doppler-gpu 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (645) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +19 -15
  3. package/package.json +77 -29
  4. package/src/cli/cli-model-resolution.js +275 -0
  5. package/src/cli/cli-output.js +453 -0
  6. package/src/cli/doppler-cli.js +817 -709
  7. package/src/cli/doppler-quickstart.d.ts +7 -23
  8. package/src/cli/doppler-quickstart.js +4 -2
  9. package/src/cli/doppler-serve.js +1 -1
  10. package/src/cli/runtime-profiles.d.ts +40 -0
  11. package/src/cli/runtime-profiles.js +225 -0
  12. package/src/client/doppler-api.js +1 -6
  13. package/src/client/doppler-registry.d.ts +12 -0
  14. package/src/client/doppler-registry.js +27 -0
  15. package/src/client/doppler-registry.json +64 -6
  16. package/src/client/failure-taxonomy.js +139 -0
  17. package/src/client/fault-injection.js +30 -0
  18. package/src/client/provider.d.ts +151 -0
  19. package/src/client/provider.js +374 -0
  20. package/src/client/receipt.js +156 -0
  21. package/src/client/runtime/index.js +56 -4
  22. package/src/client/runtime/model-manager.d.ts +0 -14
  23. package/src/client/runtime/model-manager.js +2 -42
  24. package/src/client/runtime/model-session.js +33 -0
  25. package/src/client/runtime/model-source.d.ts +13 -1
  26. package/src/client/runtime/model-source.js +128 -2
  27. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  28. package/src/client/runtime/node-quickstart-cache.js +353 -0
  29. package/src/client/runtime/source-runtime.js +1 -10
  30. package/src/client/runtime/types.d.ts +0 -60
  31. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  32. package/src/client/wrap-pipeline-handle.js +84 -0
  33. package/src/config/README.md +3 -1
  34. package/src/config/conversion/README.md +23 -157
  35. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  36. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  38. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  41. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  43. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  46. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  47. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  48. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  49. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  50. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  51. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  52. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  54. package/src/config/execution-contract-check.js +6 -5
  55. package/src/config/kernel-path-loader.d.ts +8 -2
  56. package/src/config/kernels/kernel-ref-digests.js +125 -122
  57. package/src/config/kernels/registry.d.ts +0 -35
  58. package/src/config/kernels/registry.js +5 -9
  59. package/src/config/kernels/registry.json +524 -350
  60. package/src/config/merge-helpers.d.ts +1 -1
  61. package/src/config/merge-helpers.js +1 -1
  62. package/src/config/merge.js +35 -7
  63. package/src/config/param-categories.d.ts +0 -9
  64. package/src/config/param-categories.js +1 -5
  65. package/src/config/param-validator.d.ts +2 -0
  66. package/src/config/param-validator.js +16 -0
  67. package/src/config/platforms/loader.d.ts +0 -27
  68. package/src/config/platforms/loader.js +0 -30
  69. package/src/config/required-inference-fields-contract-check.js +1 -7
  70. package/src/config/runtime/README.md +28 -0
  71. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  72. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  73. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  74. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  75. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  76. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  85. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  87. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  88. package/src/config/schema/bridge.schema.d.ts +16 -0
  89. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  90. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  91. package/src/config/schema/converter.schema.d.ts +5 -0
  92. package/src/config/schema/converter.schema.js +3 -0
  93. package/src/config/schema/debug.schema.d.ts +22 -1
  94. package/src/config/schema/debug.schema.js +1 -0
  95. package/src/config/schema/distill-training.schema.d.ts +0 -1
  96. package/src/config/schema/distill-training.schema.js +1 -1
  97. package/src/config/schema/doppler.schema.d.ts +12 -2
  98. package/src/config/schema/doppler.schema.js +108 -8
  99. package/src/config/schema/emulation.schema.d.ts +3 -0
  100. package/src/config/schema/emulation.schema.js +16 -16
  101. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  102. package/src/config/schema/execution-v1.schema.js +7 -1
  103. package/src/config/schema/index.d.ts +52 -1
  104. package/src/config/schema/index.js +14 -0
  105. package/src/config/schema/inference.schema.d.ts +3 -3
  106. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  107. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  108. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  109. package/src/config/schema/kvcache.schema.js +0 -9
  110. package/src/config/schema/loading.schema.d.ts +12 -0
  111. package/src/config/schema/loading.schema.js +4 -0
  112. package/src/config/schema/manifest.schema.d.ts +68 -1
  113. package/src/config/schema/manifest.schema.js +48 -2
  114. package/src/config/schema/moe.schema.js +0 -8
  115. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  116. package/src/config/schema/program-bundle.schema.js +393 -0
  117. package/src/config/schema/ul-training.schema.d.ts +0 -1
  118. package/src/config/schema/ul-training.schema.js +1 -1
  119. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  120. package/src/config/supported-operations.d.ts +14 -0
  121. package/src/config/supported-operations.js +34 -0
  122. package/src/config/supported-operations.json +178 -0
  123. package/src/config/training-defaults.js +1 -7
  124. package/src/config/transforms/capability-transform-resolver.js +3 -1
  125. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  126. package/src/config/transforms/execution-graph-transforms.js +693 -29
  127. package/src/converter/conversion-plan.js +13 -10
  128. package/src/converter/core.d.ts +57 -0
  129. package/src/converter/core.js +582 -13
  130. package/src/converter/parsers/diffusion.js +1 -33
  131. package/src/converter/quantization-info.d.ts +5 -4
  132. package/src/converter/quantization-info.js +39 -3
  133. package/src/converter/quantizer.d.ts +19 -0
  134. package/src/converter/quantizer.js +76 -0
  135. package/src/debug/config.d.ts +0 -4
  136. package/src/debug/config.js +0 -17
  137. package/src/debug/index.d.ts +28 -30
  138. package/src/debug/index.js +0 -18
  139. package/src/debug/stats.d.ts +0 -8
  140. package/src/debug/stats.js +3 -3
  141. package/src/debug/tensor.d.ts +9 -2
  142. package/src/debug/tensor.js +11 -4
  143. package/src/debug/trace.js +2 -15
  144. package/src/errors/doppler-error.js +20 -0
  145. package/src/experimental/browser/quantization.d.ts +5 -0
  146. package/src/experimental/distribution/shard-delivery.js +179 -23
  147. package/src/experimental/training/artifacts.js +1 -14
  148. package/src/experimental/training/checkpoint.js +1 -14
  149. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  150. package/src/experimental/training/datasets/token-batch.js +2 -2
  151. package/src/experimental/training/distillation/dataset.js +2 -15
  152. package/src/experimental/training/distillation/eval.js +3 -3
  153. package/src/experimental/training/distillation/runtime.js +3 -16
  154. package/src/experimental/training/distillation/student-fixture.js +6 -6
  155. package/src/experimental/training/lora-pipeline.js +1 -14
  156. package/src/experimental/training/objectives/base.d.ts +2 -2
  157. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  158. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  159. package/src/experimental/training/objectives/distill_kd.js +2 -2
  160. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  162. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  163. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  164. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  165. package/src/experimental/training/operator-artifacts.js +1 -14
  166. package/src/experimental/training/runner.js +1 -14
  167. package/src/experimental/training/workload-packs/registry.json +6 -6
  168. package/src/experimental/training/workloads.js +1 -1
  169. package/src/formats/gguf/types.d.ts +0 -63
  170. package/src/formats/gguf/types.js +58 -38
  171. package/src/formats/litert/types.d.ts +0 -13
  172. package/src/formats/litert/types.js +2 -2
  173. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  174. package/src/formats/rdrr/distributed/index.js +3 -0
  175. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  176. package/src/formats/rdrr/distributed/parsing.js +23 -0
  177. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  178. package/src/formats/rdrr/distributed/validation.js +131 -0
  179. package/src/formats/rdrr/groups.d.ts +1 -13
  180. package/src/formats/rdrr/groups.js +0 -34
  181. package/src/formats/rdrr/index.d.ts +9 -0
  182. package/src/formats/rdrr/index.js +9 -0
  183. package/src/formats/rdrr/integrity.d.ts +51 -0
  184. package/src/formats/rdrr/integrity.js +200 -0
  185. package/src/formats/rdrr/lowerings.d.ts +28 -0
  186. package/src/formats/rdrr/lowerings.js +112 -0
  187. package/src/formats/rdrr/manifest.d.ts +0 -14
  188. package/src/formats/rdrr/manifest.js +5 -58
  189. package/src/formats/rdrr/merkle.d.ts +24 -0
  190. package/src/formats/rdrr/merkle.js +118 -0
  191. package/src/formats/rdrr/parsing.js +4 -0
  192. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  193. package/src/formats/rdrr/slice-compiler.js +188 -0
  194. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  195. package/src/formats/rdrr/storage-descriptor.js +186 -0
  196. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  197. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  198. package/src/formats/rdrr/types.d.ts +107 -0
  199. package/src/formats/rdrr/validation.js +245 -0
  200. package/src/formats/tflite/types.d.ts +0 -31
  201. package/src/formats/tflite/types.js +5 -5
  202. package/src/gpu/command-recorder.d.ts +3 -21
  203. package/src/gpu/device.d.ts +17 -3
  204. package/src/gpu/device.js +20 -4
  205. package/src/gpu/kernel-runtime.d.ts +8 -15
  206. package/src/gpu/kernel-selection-log.d.ts +0 -8
  207. package/src/gpu/kernel-selection-log.js +0 -15
  208. package/src/gpu/kernel-tuner/cache.js +0 -3
  209. package/src/gpu/kernels/README.md +50 -18
  210. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  211. package/src/gpu/kernels/attention.d.ts +28 -19
  212. package/src/gpu/kernels/attention.js +443 -9
  213. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  214. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  215. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  216. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  217. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  218. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  219. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  220. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  223. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  224. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  225. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  226. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  227. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  228. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  229. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  230. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  231. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  232. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  233. package/src/gpu/kernels/backward/index.d.ts +6 -0
  234. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  235. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  236. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  237. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  238. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  239. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  240. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  241. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  242. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  243. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  244. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  245. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  246. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  247. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  248. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  249. package/src/gpu/kernels/constants.d.ts +9 -48
  250. package/src/gpu/kernels/constants.js +0 -23
  251. package/src/gpu/kernels/conv2d.d.ts +0 -4
  252. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  253. package/src/gpu/kernels/dequant.d.ts +0 -16
  254. package/src/gpu/kernels/dequant.js +1 -71
  255. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  256. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  257. package/src/gpu/kernels/dispatch.d.ts +12 -97
  258. package/src/gpu/kernels/dispatch.js +0 -117
  259. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  260. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  261. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  262. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  263. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  264. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  265. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  266. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  267. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  268. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  269. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  270. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  272. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  273. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  274. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  275. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  276. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  277. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  278. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  279. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  280. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  281. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  282. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  283. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  284. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  285. package/src/gpu/kernels/gather.d.ts +0 -7
  286. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  287. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  288. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  289. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  290. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  291. package/src/gpu/kernels/index.d.ts +6 -5
  292. package/src/gpu/kernels/index.js +3 -4
  293. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  294. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  295. package/src/gpu/kernels/layernorm.d.ts +0 -4
  296. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  297. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  298. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  299. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  300. package/src/gpu/kernels/logit-merge.js +1 -17
  301. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  302. package/src/gpu/kernels/matmul-selection.js +68 -1
  303. package/src/gpu/kernels/matmul.d.ts +0 -28
  304. package/src/gpu/kernels/matmul.js +101 -7
  305. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  306. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  307. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  308. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  309. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  310. package/src/gpu/kernels/moe.d.ts +0 -56
  311. package/src/gpu/kernels/moe.js +0 -15
  312. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  313. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  314. package/src/gpu/kernels/residual.d.ts +0 -12
  315. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  316. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  317. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  318. package/src/gpu/kernels/rmsnorm.js +17 -1
  319. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  320. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  321. package/src/gpu/kernels/rope.d.ts +1 -5
  322. package/src/gpu/kernels/rope.js +12 -2
  323. package/src/gpu/kernels/rope.wgsl +33 -19
  324. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  325. package/src/gpu/kernels/sample.d.ts +0 -5
  326. package/src/gpu/kernels/sample.wgsl +7 -6
  327. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  328. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  329. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  330. package/src/gpu/kernels/shader-cache.js +27 -0
  331. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  332. package/src/gpu/kernels/softmax.wgsl +4 -144
  333. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  334. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  335. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  336. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  337. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  338. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  339. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  340. package/src/gpu/kernels/utils.d.ts +28 -0
  341. package/src/gpu/kernels/utils.js +41 -0
  342. package/src/gpu/perf-guards.d.ts +0 -12
  343. package/src/gpu/perf-guards.js +0 -62
  344. package/src/gpu/profiler.d.ts +0 -21
  345. package/src/gpu/profiler.js +0 -34
  346. package/src/gpu/submit-tracker.d.ts +0 -57
  347. package/src/gpu/submit-tracker.js +3 -62
  348. package/src/gpu/tensor.d.ts +0 -23
  349. package/src/gpu/tensor.js +0 -41
  350. package/src/gpu/uniform-cache.d.ts +0 -11
  351. package/src/gpu/uniform-cache.js +0 -23
  352. package/src/index.d.ts +1 -0
  353. package/src/index.js +1 -0
  354. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  355. package/src/inference/browser-harness-model-helpers.js +22 -13
  356. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  357. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  358. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  359. package/src/inference/browser-harness-suite-helpers.js +87 -1
  360. package/src/inference/browser-harness-text-helpers.d.ts +12 -0
  361. package/src/inference/browser-harness-text-helpers.js +223 -14
  362. package/src/inference/browser-harness.d.ts +0 -29
  363. package/src/inference/browser-harness.js +264 -71
  364. package/src/inference/kv-cache/base.d.ts +6 -0
  365. package/src/inference/kv-cache/base.js +16 -1
  366. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  367. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  368. package/src/inference/kv-cache/quantized.js +2 -0
  369. package/src/inference/kv-cache/tiered.js +7 -0
  370. package/src/inference/kv-cache/types.d.ts +2 -0
  371. package/src/inference/multi-model-network.d.ts +1 -1
  372. package/src/inference/multi-model-network.js +1 -2
  373. package/src/inference/network-evolution.d.ts +0 -12
  374. package/src/inference/network-evolution.js +2 -2
  375. package/src/inference/pipelines/context.js +17 -0
  376. package/src/inference/pipelines/diffusion/init.js +2 -2
  377. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  378. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  379. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  380. package/src/inference/pipelines/diffusion/vae.js +4 -4
  381. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  382. package/src/inference/pipelines/structured/index.d.ts +2 -0
  383. package/src/inference/pipelines/structured/index.js +2 -0
  384. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  385. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  386. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  387. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  388. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  389. package/src/inference/pipelines/text/attention/projections.js +70 -23
  390. package/src/inference/pipelines/text/attention/record.js +118 -5
  391. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  392. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  393. package/src/inference/pipelines/text/attention/run.js +97 -6
  394. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  395. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  396. package/src/inference/pipelines/text/chat-format.js +1 -8
  397. package/src/inference/pipelines/text/config.d.ts +3 -25
  398. package/src/inference/pipelines/text/config.js +105 -3
  399. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  400. package/src/inference/pipelines/text/drift-policy.js +1 -1
  401. package/src/inference/pipelines/text/embed.d.ts +22 -27
  402. package/src/inference/pipelines/text/embed.js +0 -39
  403. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  404. package/src/inference/pipelines/text/execution-plan.js +15 -2
  405. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  406. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  407. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  408. package/src/inference/pipelines/text/execution-v1.js +94 -25
  409. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  410. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  411. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  412. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  413. package/src/inference/pipelines/text/generator-helpers.js +4 -5
  414. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  415. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  416. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  417. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  418. package/src/inference/pipelines/text/generator-steps.js +8 -3
  419. package/src/inference/pipelines/text/generator.d.ts +0 -6
  420. package/src/inference/pipelines/text/generator.js +223 -547
  421. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  422. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  423. package/src/inference/pipelines/text/init-rope.js +408 -0
  424. package/src/inference/pipelines/text/init.d.ts +5 -2
  425. package/src/inference/pipelines/text/init.js +183 -724
  426. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  427. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  428. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  429. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  430. package/src/inference/pipelines/text/layer-plan.js +1 -1
  431. package/src/inference/pipelines/text/layer.d.ts +70 -8
  432. package/src/inference/pipelines/text/layer.js +105 -537
  433. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  434. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  435. package/src/inference/pipelines/text/load-errors.js +59 -0
  436. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  437. package/src/inference/pipelines/text/logits/gpu.js +2 -0
  438. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  439. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  440. package/src/inference/pipelines/text/model-load.js +74 -17
  441. package/src/inference/pipelines/text/ops.d.ts +13 -2
  442. package/src/inference/pipelines/text/ops.js +1 -1
  443. package/src/inference/pipelines/text/per-layer-inputs.js +31 -6
  444. package/src/inference/pipelines/text/probes.d.ts +3 -6
  445. package/src/inference/pipelines/text/probes.js +120 -31
  446. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  447. package/src/inference/pipelines/text/sampling.js +140 -0
  448. package/src/inference/pipelines/text/stage-names.js +3 -0
  449. package/src/inference/pipelines/text/state.js +2 -0
  450. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  451. package/src/inference/pipelines/text/types.d.ts +8 -78
  452. package/src/inference/pipelines/text/weights.d.ts +0 -50
  453. package/src/inference/pipelines/text/weights.js +2 -51
  454. package/src/inference/pipelines/text.d.ts +9 -1
  455. package/src/inference/pipelines/text.js +223 -27
  456. package/src/inference/pipelines/vision/index.js +0 -107
  457. package/src/inference/runtime-model.d.ts +0 -7
  458. package/src/inference/runtime-model.js +0 -23
  459. package/src/inference/test-harness.d.ts +6 -22
  460. package/src/inference/test-harness.js +97 -23
  461. package/src/inference/tokenizer.js +1 -10
  462. package/src/inference/tokenizers/sentencepiece.js +65 -15
  463. package/src/inference/tokenizers/types.d.ts +0 -21
  464. package/src/loader/doppler-loader.d.ts +3 -0
  465. package/src/loader/doppler-loader.js +90 -12
  466. package/src/loader/embedding-loader.js +13 -3
  467. package/src/loader/loader-types.d.ts +2 -0
  468. package/src/loader/manifest-config.js +25 -1
  469. package/src/loader/memory-monitor.d.ts +17 -5
  470. package/src/loader/quantization-constants.d.ts +6 -0
  471. package/src/loader/shard-resolver.d.ts +1 -0
  472. package/src/loader/shard-resolver.js +15 -0
  473. package/src/loader/tensors/source-transform.js +6 -1
  474. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  475. package/src/loader/tensors/tensor-loader.js +84 -1
  476. package/src/memory/address-table.d.ts +1 -9
  477. package/src/memory/buffer-pool.d.ts +0 -16
  478. package/src/memory/buffer-pool.js +1 -33
  479. package/src/memory/heap-manager.d.ts +18 -43
  480. package/src/memory/unified-detect.d.ts +19 -34
  481. package/src/models/embeddinggemma.d.ts +7 -0
  482. package/src/models/embeddinggemma.js +24 -0
  483. package/src/models/family.d.ts +24 -0
  484. package/src/models/family.js +25 -0
  485. package/src/models/gemma3.d.ts +7 -0
  486. package/src/models/gemma3.js +33 -0
  487. package/src/models/gemma4.d.ts +7 -0
  488. package/src/models/gemma4.js +33 -0
  489. package/src/models/qwen3.d.ts +7 -0
  490. package/src/models/qwen3.js +36 -0
  491. package/src/rules/converter/tokenizer.rules.json +5 -0
  492. package/src/rules/execution-rules-contract-check.js +72 -3
  493. package/src/rules/inference/capability-transforms.rules.json +87 -0
  494. package/src/rules/inference/execution.rules.json +11 -1
  495. package/src/rules/inference/ffn.rules.json +6 -2
  496. package/src/rules/loader/tensor-loader.rules.json +9 -0
  497. package/src/rules/rule-registry.js +53 -75
  498. package/src/storage/artifact-storage-context.d.ts +11 -3
  499. package/src/storage/artifact-storage-context.js +231 -26
  500. package/src/storage/backends/opfs-store.js +138 -88
  501. package/src/storage/blake3.js +9 -1
  502. package/src/storage/export.js +1 -5
  503. package/src/storage/normalize-model-id.js +3 -0
  504. package/src/storage/preflight.js +11 -0
  505. package/src/storage/quickstart-downloader.js +44 -67
  506. package/src/storage/reports.js +35 -11
  507. package/src/storage/shard-manager.d.ts +8 -2
  508. package/src/storage/shard-manager.js +176 -15
  509. package/src/storage/source-artifact-store.d.ts +0 -2
  510. package/src/storage/source-artifact-store.js +3 -9
  511. package/src/sw.js +0 -3
  512. package/src/tooling/browser-command-runner.js +39 -0
  513. package/src/tooling/command-api-constants.js +1 -1
  514. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  515. package/src/tooling/command-api-family-normalizers.js +49 -7
  516. package/src/tooling/command-api-helpers.d.ts +0 -2
  517. package/src/tooling/command-api-helpers.js +6 -2
  518. package/src/tooling/command-api.d.ts +20 -1
  519. package/src/tooling/command-api.js +10 -1
  520. package/src/tooling/command-envelope.d.ts +0 -1
  521. package/src/tooling/command-envelope.js +1 -1
  522. package/src/tooling/command-runner-shared.d.ts +10 -0
  523. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  524. package/src/tooling/conversion-config-materializer.js +1 -1
  525. package/src/tooling/diagnose-runner.js +1 -4
  526. package/src/tooling/hf-registry-utils.d.ts +2 -6
  527. package/src/tooling/hf-registry-utils.js +90 -19
  528. package/src/tooling/kernel-path-builder/index.js +1 -4
  529. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  530. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  531. package/src/tooling/lean-execution-contract-runner.js +2 -2
  532. package/src/tooling/litert-package-runtime.d.ts +1 -1
  533. package/src/tooling/litert-package-runtime.js +21 -24
  534. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  535. package/src/tooling/node-browser-command-runner.js +97 -13
  536. package/src/tooling/node-command-runner.js +36 -0
  537. package/src/tooling/node-convert-worker-pool.js +12 -0
  538. package/src/tooling/node-convert-worker.js +10 -1
  539. package/src/tooling/node-converter.js +90 -1
  540. package/src/tooling/node-source-runtime.js +1 -10
  541. package/src/tooling/opfs-cache.js +1 -4
  542. package/src/tooling/precision-replay-math.js +174 -0
  543. package/src/tooling/program-bundle-host.d.ts +16 -0
  544. package/src/tooling/program-bundle-host.js +20 -0
  545. package/src/tooling/program-bundle-parity.d.ts +28 -0
  546. package/src/tooling/program-bundle-parity.js +210 -0
  547. package/src/tooling/program-bundle.d.ts +61 -0
  548. package/src/tooling/program-bundle.js +1119 -0
  549. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  550. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  551. package/src/tooling/reference-verify.js +191 -0
  552. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  553. package/src/tooling/source-artifact-adapter.js +4 -4
  554. package/src/tooling/source-package-profiles.js +1 -7
  555. package/src/tooling/source-runtime-bundle.d.ts +2 -0
  556. package/src/tooling/source-runtime-bundle.js +3 -22
  557. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  558. package/src/tooling/source-runtime-converter-config.js +13 -9
  559. package/src/tooling/source-runtime-materializer.js +1 -7
  560. package/src/tooling-exports/device.d.ts +11 -0
  561. package/src/tooling-exports/device.js +14 -0
  562. package/src/tooling-exports/manifest.d.ts +8 -0
  563. package/src/tooling-exports/manifest.js +11 -0
  564. package/src/tooling-exports/storage.d.ts +25 -0
  565. package/src/tooling-exports/storage.js +29 -0
  566. package/src/tooling-exports/structured.d.ts +1 -0
  567. package/src/tooling-exports/structured.js +5 -0
  568. package/src/tooling-exports.d.ts +23 -0
  569. package/src/tooling-exports.js +17 -0
  570. package/src/tooling-exports.shared.d.ts +27 -30
  571. package/src/tooling-exports.shared.js +28 -29
  572. package/src/types/index.d.ts +0 -3
  573. package/src/types/index.js +0 -3
  574. package/src/utils/array-buffer.d.ts +4 -0
  575. package/src/utils/array-buffer.js +9 -0
  576. package/src/utils/canonical-hash.d.ts +7 -0
  577. package/src/utils/canonical-hash.js +74 -0
  578. package/src/utils/clone-json.d.ts +1 -0
  579. package/src/utils/clone-json.js +7 -0
  580. package/src/utils/encode-utf8.js +5 -0
  581. package/src/utils/index.d.ts +4 -1
  582. package/src/utils/index.js +4 -1
  583. package/src/utils/runtime-env.js +7 -0
  584. package/src/utils/sha256.d.ts +1 -0
  585. package/src/utils/sha256.js +18 -4
  586. package/src/utils/stable-sort-object.d.ts +1 -0
  587. package/src/utils/stable-sort-object.js +13 -0
  588. package/src/version.js +1 -1
  589. package/tools/convert-safetensors-node.js +1 -2
  590. package/src/bootstrap.d.ts +0 -1
  591. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  592. package/src/config/kernels/kernel-ref.d.ts +0 -17
  593. package/src/config/kernels/kernel-ref.js +0 -75
  594. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  595. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  596. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  597. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  598. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  599. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  600. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  601. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  602. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  603. package/src/config/runtime/overlays/trace-config.json +0 -28
  604. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  605. package/src/debug/perf.d.ts +0 -33
  606. package/src/debug/perf.js +0 -51
  607. package/src/errors/index.d.ts +0 -1
  608. package/src/errors/index.js +0 -1
  609. package/src/formats/gguf/index.d.ts +0 -8
  610. package/src/formats/gguf/index.js +0 -4
  611. package/src/formats/litert/index.d.ts +0 -1
  612. package/src/formats/litert/index.js +0 -1
  613. package/src/formats/safetensors/index.d.ts +0 -8
  614. package/src/formats/safetensors/index.js +0 -4
  615. package/src/formats/tflite/index.d.ts +0 -1
  616. package/src/formats/tflite/index.js +0 -1
  617. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  618. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  619. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  620. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  621. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  622. package/src/inference/functiongemma.d.ts +0 -15
  623. package/src/inference/functiongemma.js +0 -1
  624. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  625. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  626. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  627. package/src/rules/kernels/kernel-validator.js +0 -160
  628. package/src/sw.d.ts +0 -1
  629. package/src/tooling/bench-runner.js +0 -64
  630. package/src/tooling/distill-runner.js +0 -56
  631. package/src/tooling/lora-runner.js +0 -37
  632. package/src/tooling/verify-runner.js +0 -63
  633. package/src/types/gpu.d.ts +0 -185
  634. package/src/types/gpu.js +0 -5
  635. package/src/types/inference.d.ts +0 -197
  636. package/src/types/inference.js +0 -5
  637. package/src/types/model.d.ts +0 -130
  638. package/src/types/model.js +0 -5
  639. package/src/utils/format-bytes.js +0 -20
  640. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  641. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  642. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  643. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  644. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  645. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -4,6 +4,8 @@ import { initDevice, getDevice, getKernelCapabilities } from '../gpu/device.js';
4
4
  import { parseManifest, getExpectedShardHash } from '../formats/rdrr/index.js';
5
5
  import { createPipeline } from './pipelines/text.js';
6
6
  import { createNodeFileShardStorageContext } from './pipelines/text/init.js';
7
+ import { createHttpArtifactStorageContext } from '../storage/artifact-storage-context.js';
8
+ import { resolveManifestArtifactSource } from '../client/runtime/model-source.js';
7
9
  import { log as debugLog } from '../debug/index.js';
8
10
  import { getRuntimeConfig, setRuntimeConfig } from '../config/runtime.js';
9
11
 
@@ -144,8 +146,37 @@ function toShardBytes(buffer, shardIndex) {
144
146
  throw new Error(`Shard ${shardIndex} did not return an in-memory buffer`);
145
147
  }
146
148
 
149
+ function createInitializationPhaseError(error, phase, context = {}) {
150
+ const message = error?.message || String(error);
151
+ const wrapped = new Error(
152
+ `Inference initialization phase "${phase}" failed: ${message}`,
153
+ error instanceof Error ? { cause: error } : undefined
154
+ );
155
+ wrapped.name = error?.name || 'Error';
156
+ if (error?.code !== undefined) {
157
+ wrapped.code = error.code;
158
+ }
159
+ wrapped.details = {
160
+ ...(error?.details && typeof error.details === 'object' ? error.details : {}),
161
+ initializationPhase: phase,
162
+ ...context,
163
+ };
164
+ return wrapped;
165
+ }
166
+
167
+ async function withInitializationPhase(phase, context, run) {
168
+ try {
169
+ return await run();
170
+ } catch (error) {
171
+ if (error?.details?.initializationPhase) {
172
+ throw error;
173
+ }
174
+ throw createInitializationPhaseError(error, phase, context);
175
+ }
176
+ }
147
177
 
148
- export function createHttpShardLoader(baseUrl, manifest, log) {
178
+
179
+ function createHttpShardLoader(baseUrl, manifest, log) {
149
180
  const algorithm = manifest.hashAlgorithm;
150
181
  if (!algorithm) {
151
182
  throw new Error('Manifest missing hashAlgorithm for shard delivery.');
@@ -217,11 +248,29 @@ export function createHttpShardLoader(baseUrl, manifest, log) {
217
248
  };
218
249
  }
219
250
 
220
- export function createHarnessShardStorageContext(modelUrl, manifest, log) {
251
+ export function createHarnessShardStorageContext(modelUrl, manifest, log, options = {}) {
221
252
  const nodeFileStorageContext = createNodeFileShardStorageContext(modelUrl, manifest);
222
253
  if (nodeFileStorageContext) {
223
254
  return nodeFileStorageContext;
224
255
  }
256
+ const runtimeConfig = getRuntimeConfig();
257
+ const sourceOrder = runtimeConfig?.loading?.distribution?.sourceOrder;
258
+ const httpOnly = Array.isArray(sourceOrder)
259
+ && sourceOrder.length === 1
260
+ && String(sourceOrder[0]).trim().toLowerCase() === 'http';
261
+ const explicitHttpLoadMode = String(options.loadMode ?? '').trim().toLowerCase() === 'http';
262
+ if (explicitHttpLoadMode || httpOnly) {
263
+ const httpContext = createHttpArtifactStorageContext(modelUrl, manifest, {
264
+ verifyHashes: runtimeConfig?.loading?.shardCache?.verifyHashes === true,
265
+ rangeCacheBlockBytes: runtimeConfig?.loading?.shardCache?.rangeCacheBlockBytes,
266
+ rangeCacheMaxBytes: runtimeConfig?.loading?.shardCache?.rangeCacheMaxBytes,
267
+ rangeCacheMinBytes: runtimeConfig?.loading?.shardCache?.rangeCacheMinBytes,
268
+ });
269
+ if (httpContext) {
270
+ log?.('Using HTTP range artifact storage context');
271
+ return httpContext;
272
+ }
273
+ }
225
274
  return {
226
275
  loadShard: createHttpShardLoader(modelUrl, manifest, log),
227
276
  };
@@ -232,18 +281,16 @@ export function createHarnessShardStorageContext(modelUrl, manifest, log) {
232
281
  // ============================================================================
233
282
 
234
283
 
235
- export async function fetchManifest(manifestUrl) {
284
+ async function fetchManifestPayload(manifestUrl) {
236
285
  const response = await fetch(manifestUrl);
237
286
  if (!response.ok) {
238
287
  throw new Error(`Failed to fetch manifest: ${response.status}`);
239
288
  }
240
- return parseManifest(await response.text());
241
- }
242
-
243
-
244
- export async function initializeDevice() {
245
- await initDevice();
246
- return getKernelCapabilities();
289
+ const text = await response.text();
290
+ return {
291
+ text,
292
+ manifest: parseManifest(text),
293
+ };
247
294
  }
248
295
 
249
296
 
@@ -297,7 +344,11 @@ export async function initializeInference(modelUrl, options = {}) {
297
344
  onProgress('init', 0, 'Initializing WebGPU...');
298
345
  log('Initializing WebGPU...');
299
346
 
300
- await initDevice();
347
+ await withInitializationPhase(
348
+ 'initDevice',
349
+ { modelUrl },
350
+ () => initDevice()
351
+ );
301
352
  const device = getDevice();
302
353
  const capabilities = getKernelCapabilities();
303
354
 
@@ -308,7 +359,24 @@ export async function initializeInference(modelUrl, options = {}) {
308
359
  log('Fetching manifest...');
309
360
 
310
361
  const manifestUrl = `${modelUrl}/manifest.json`;
311
- const manifest = await fetchManifest(manifestUrl);
362
+ const manifestPayload = await withInitializationPhase(
363
+ 'fetchManifest',
364
+ { modelUrl, manifestUrl },
365
+ () => fetchManifestPayload(manifestUrl)
366
+ );
367
+ const artifactSource = await withInitializationPhase(
368
+ 'resolveArtifactSource',
369
+ { modelUrl, manifestUrl },
370
+ () => resolveManifestArtifactSource({
371
+ modelId: options.modelId || manifestPayload.manifest?.modelId || modelUrl,
372
+ baseUrl: modelUrl,
373
+ manifest: null,
374
+ trace: [],
375
+ }, manifestPayload)
376
+ );
377
+ const manifest = artifactSource.manifest;
378
+ const storageManifest = artifactSource.storageManifest ?? manifest;
379
+ const storageModelUrl = artifactSource.storageBaseUrl ?? modelUrl;
312
380
 
313
381
  if (intentBundleConfig.enabled && intentBundleConfig.bundleUrl) {
314
382
  const intentBundleModule = await loadIntentBundleModule();
@@ -339,7 +407,9 @@ export async function initializeInference(modelUrl, options = {}) {
339
407
  log(`Model: ${modelLabel}`);
340
408
 
341
409
  // 3. Create shard loader
342
- const storageContext = createHarnessShardStorageContext(modelUrl, manifest, log);
410
+ const storageContext = createHarnessShardStorageContext(storageModelUrl, storageManifest, log, {
411
+ loadMode: options.loadMode ?? null,
412
+ });
343
413
 
344
414
  // 4. Build runtime options
345
415
 
@@ -351,16 +421,20 @@ export async function initializeInference(modelUrl, options = {}) {
351
421
  onProgress('pipeline', 0.2, 'Creating pipeline...');
352
422
  log('Creating pipeline...');
353
423
 
354
- const pipeline = await createPipeline( ( (manifest)), {
355
- storage: storageContext,
356
- gpu: { device },
357
- runtime,
358
- baseUrl: modelUrl,
359
- onProgress: ( progress) => {
360
- const pct = 0.2 + progress.percent * 0.8;
361
- onProgress(progress.stage || 'loading', pct, progress.message);
362
- },
363
- });
424
+ const pipeline = await withInitializationPhase(
425
+ 'createPipeline',
426
+ { modelUrl, modelId: manifest.modelId ?? null },
427
+ () => createPipeline( ( (manifest)), {
428
+ storage: storageContext,
429
+ gpu: { device },
430
+ runtime,
431
+ baseUrl: storageModelUrl,
432
+ onProgress: ( progress) => {
433
+ const pct = 0.2 + progress.percent * 0.8;
434
+ onProgress(progress.stage || 'loading', pct, progress.message);
435
+ },
436
+ })
437
+ );
364
438
 
365
439
  onProgress('complete', 1, 'Ready');
366
440
  log('Pipeline ready');
@@ -5,6 +5,7 @@ import { BaseTokenizer } from './tokenizers/base.js';
5
5
  import { TransformersTokenizer, BundledTokenizer } from './tokenizers/bundled.js';
6
6
  import { SentencePieceTokenizer } from './tokenizers/sentencepiece.js';
7
7
  import { BPETokenizer } from './tokenizers/bpe.js';
8
+ import { toArrayBuffer } from '../utils/array-buffer.js';
8
9
 
9
10
  const BUNDLED_TOKENIZER_CACHE = new Map();
10
11
 
@@ -12,16 +13,6 @@ function hasScheme(value) {
12
13
  return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
13
14
  }
14
15
 
15
- function toArrayBuffer(value, label) {
16
- if (value instanceof ArrayBuffer) {
17
- return value;
18
- }
19
- if (value instanceof Uint8Array) {
20
- return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength);
21
- }
22
- throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
23
- }
24
-
25
16
  function parseTokenizerJsonPayload(value) {
26
17
  if (value == null) {
27
18
  return null;
@@ -23,6 +23,8 @@ export class SentencePieceTokenizer extends BaseTokenizer {
23
23
 
24
24
  #unkId = 0;
25
25
 
26
+ #addDummyPrefix = true;
27
+
26
28
 
27
29
  constructor(config = {}) {
28
30
  // SentencePieceTokenizer gets vocabSize from load(), so defer validation
@@ -38,6 +40,7 @@ export class SentencePieceTokenizer extends BaseTokenizer {
38
40
  this.#algorithm = 'unigram';
39
41
  this.#byteTokens.clear();
40
42
  this.vocabSize = 0;
43
+ this.#addDummyPrefix = true;
41
44
  }
42
45
 
43
46
 
@@ -76,9 +79,10 @@ export class SentencePieceTokenizer extends BaseTokenizer {
76
79
 
77
80
 
78
81
  async #parseModelProto(buffer) {
79
- const view = new DataView(buffer);
80
82
  const bytes = new Uint8Array(buffer);
81
83
  let offset = 0;
84
+ const standardPieces = [];
85
+ const liteRTPieceCandidates = [];
82
86
 
83
87
  // SentencePiece model is a protobuf with:
84
88
  // - Field 1: trainer_spec
@@ -94,10 +98,21 @@ export class SentencePieceTokenizer extends BaseTokenizer {
94
98
  const wireType = tag & 0x7;
95
99
 
96
100
  if (fieldNumber === 1 || fieldNumber === 2) {
97
- // Skip trainer_spec and normalizer_spec (wire type 2 = length-delimited)
101
+ // Skip trainer_spec and normalizer_spec in standard ModelProto files.
102
+ // LiteRT task metadata may store a compact repeated SentencePiece list
103
+ // at field 1; collect candidates and choose it only if field 3 did not
104
+ // expose the richer standard piece list.
98
105
  if (wireType === 2) {
99
106
  const { value: length, newOffset } = this.#readVarint(bytes, offset);
100
- offset = newOffset + length;
107
+ offset = newOffset;
108
+ if (fieldNumber === 1) {
109
+ const pieceData = bytes.slice(offset, offset + length);
110
+ const candidate = this.#decodePiece(pieceData);
111
+ if (candidate && (candidate.hasScore || candidate.hasType)) {
112
+ liteRTPieceCandidates.push(candidate);
113
+ }
114
+ }
115
+ offset += length;
101
116
  }
102
117
  } else if (fieldNumber === 3 && wireType === 2) {
103
118
  // SentencePiece entry
@@ -105,7 +120,10 @@ export class SentencePieceTokenizer extends BaseTokenizer {
105
120
  offset = newOffset;
106
121
 
107
122
  const pieceData = bytes.slice(offset, offset + length);
108
- this.#parsePiece(pieceData);
123
+ const piece = this.#decodePiece(pieceData);
124
+ if (piece) {
125
+ standardPieces.push(piece);
126
+ }
109
127
  offset += length;
110
128
  } else {
111
129
  // Skip unknown field
@@ -125,6 +143,12 @@ export class SentencePieceTokenizer extends BaseTokenizer {
125
143
  }
126
144
  }
127
145
 
146
+ const isLiteRTPieceList = liteRTPieceCandidates.length > standardPieces.length;
147
+ const selectedPieces = isLiteRTPieceList ? liteRTPieceCandidates : standardPieces;
148
+ for (const [id, piece] of selectedPieces.entries()) {
149
+ this.#addPiece(piece, id);
150
+ }
151
+
128
152
  // Set up special tokens
129
153
  if (this.specialTokens.unk == null) {
130
154
  throw new Error('[Tokenizer] unk token is required for SentencePiece.');
@@ -134,15 +158,25 @@ export class SentencePieceTokenizer extends BaseTokenizer {
134
158
  // Determine algorithm from model characteristics
135
159
  // (Unigram has scores, BPE typically doesn't)
136
160
  const hasScores = [...this.#pieces.values()].some(p => p.score !== 0);
137
- this.#algorithm = selectRuleValue('converter', 'tokenizer', 'type', { hasScores });
161
+ const tokenizerRuleContext = { hasScores, isLiteRTPieceList };
162
+ this.#algorithm = selectRuleValue('converter', 'tokenizer', 'type', tokenizerRuleContext);
163
+ this.#addDummyPrefix = selectRuleValue(
164
+ 'converter',
165
+ 'tokenizer',
166
+ 'addDummyPrefix',
167
+ tokenizerRuleContext
168
+ );
138
169
  }
139
170
 
140
171
 
141
- #parsePiece(bytes) {
172
+ #decodePiece(bytes) {
142
173
  let offset = 0;
143
174
  let piece = '';
175
+ let hasPiece = false;
144
176
  let score = 0;
145
177
  let type = 1; // NORMAL by default
178
+ let hasScore = false;
179
+ let hasType = false;
146
180
 
147
181
  while (offset < bytes.length) {
148
182
  const { value: tag, newOffset: tagOffset } = this.#readVarint(bytes, offset);
@@ -156,16 +190,19 @@ export class SentencePieceTokenizer extends BaseTokenizer {
156
190
  const { value: length, newOffset } = this.#readVarint(bytes, offset);
157
191
  offset = newOffset;
158
192
  piece = new TextDecoder().decode(bytes.slice(offset, offset + length));
193
+ hasPiece = true;
159
194
  offset += length;
160
195
  } else if (fieldNumber === 2 && wireType === 5) {
161
196
  // score (float32)
162
197
  const view = new DataView(bytes.buffer, bytes.byteOffset + offset, 4);
163
198
  score = view.getFloat32(0, true);
199
+ hasScore = true;
164
200
  offset += 4;
165
201
  } else if (fieldNumber === 3 && wireType === 0) {
166
202
  // type (varint enum)
167
203
  const { value, newOffset } = this.#readVarint(bytes, offset);
168
204
  type = value;
205
+ hasType = true;
169
206
  offset = newOffset;
170
207
  } else {
171
208
  // Skip unknown
@@ -181,15 +218,26 @@ export class SentencePieceTokenizer extends BaseTokenizer {
181
218
  }
182
219
  }
183
220
 
184
- if (piece) {
185
- const id = this.#pieces.size;
186
- this.#pieces.set(piece, { id, score, type });
187
- this.#reverseVocab.set(id, piece);
188
- this.vocabSize = this.#pieces.size;
221
+ if (!hasPiece) {
222
+ return null;
223
+ }
224
+ return { piece, score, type, hasScore, hasType };
225
+ }
226
+
227
+ #addPiece(entry, id) {
228
+ if (!entry?.piece) {
229
+ return;
230
+ }
231
+ if (!this.#pieces.has(entry.piece)) {
232
+ this.#pieces.set(entry.piece, { id, score: entry.score, type: entry.type });
233
+ }
234
+ this.#reverseVocab.set(id, entry.piece);
235
+ this.vocabSize = Math.max(this.vocabSize, id + 1);
189
236
 
190
- // Track byte tokens (▁ prefix tokens and <0xXX> byte tokens)
191
- if (piece.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
192
- const byteVal = parseInt(piece.slice(3, 5), 16);
237
+ // Track byte tokens (▁ prefix tokens and <0xXX> byte tokens)
238
+ if (entry.piece.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
239
+ const byteVal = parseInt(entry.piece.slice(3, 5), 16);
240
+ if (!this.#byteTokens.has(byteVal)) {
193
241
  this.#byteTokens.set(byteVal, id);
194
242
  }
195
243
  }
@@ -255,7 +303,9 @@ export class SentencePieceTokenizer extends BaseTokenizer {
255
303
 
256
304
  // Normalize: add sentence piece prefix (▁ for word start)
257
305
  const normalized = text.replace(/ /g, '▁');
258
- const prefixed = (text.startsWith(' ') ? '' : '▁') + normalized;
306
+ const prefixed = this.#addDummyPrefix && !text.startsWith(' ')
307
+ ? `▁${normalized}`
308
+ : normalized;
259
309
 
260
310
  if (this.#algorithm === 'unigram') {
261
311
  ids.push(...this.#encodeUnigram(prefixed));
@@ -145,24 +145,3 @@ export interface BundledTokenizerJson {
145
145
  addSpacePrefix?: boolean;
146
146
  }
147
147
 
148
- /** Viterbi state for Unigram encoding */
149
- export interface ViterbiState {
150
- score: number;
151
- prev: number;
152
- tokenLen: number;
153
- isBytes?: boolean;
154
- bytes?: Uint8Array;
155
- }
156
-
157
- /** Special token pattern for matching */
158
- export interface SpecialTokenPattern {
159
- content: string;
160
- id: number;
161
- }
162
-
163
- /** Text segment (plain text or special token) */
164
- export interface TextSegment {
165
- text?: string;
166
- id?: number;
167
- isSpecial: boolean;
168
- }
@@ -97,6 +97,7 @@ export declare class DopplerLoader {
97
97
  q4kLayout: 'row' | 'col' | null;
98
98
  keepF32Weights: boolean;
99
99
  q4kMaterializationMode: 'dense' | 'fused' | 'mixed';
100
+ q4kFusedRoles: string[];
100
101
 
101
102
  constructor(loadingConfig?: LoadingConfigSchema);
102
103
 
@@ -112,6 +113,8 @@ export declare class DopplerLoader {
112
113
 
113
114
  setTensorsJsonUrl(url: string | null): void;
114
115
 
116
+ setTensorsJsonLoader(loadTensorsJson: (() => Promise<string | Record<string, unknown> | null | undefined>) | null): void;
117
+
115
118
  init(): Promise<void>;
116
119
 
117
120
  setManifest(manifest: RDRRManifest | RuntimeModelContract): void;
@@ -40,6 +40,7 @@ import {
40
40
  loadTensorToGPU,
41
41
  loadTensorToCPU,
42
42
  } from './tensors/tensor-loader.js';
43
+ import { annotateWeightLoadError } from '../inference/pipelines/text/load-errors.js';
43
44
  import { loadEmbeddings } from './embedding-loader.js';
44
45
  import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
45
46
  import { loadLayer } from './layer-loader.js';
@@ -64,6 +65,17 @@ function hasExpertGroups(manifest) {
64
65
  return Object.keys(manifest.groups).some((groupId) => groupId.includes('.expert.'));
65
66
  }
66
67
 
68
+ function isDirectSourceRuntimeManifest(manifest) {
69
+ const sourceRuntime = manifest?.metadata?.sourceRuntime;
70
+ const mode = typeof sourceRuntime?.mode === 'string'
71
+ ? sourceRuntime.mode.trim().toLowerCase()
72
+ : '';
73
+ const sourceKind = typeof sourceRuntime?.sourceKind === 'string'
74
+ ? sourceRuntime.sourceKind.trim().toLowerCase()
75
+ : '';
76
+ return mode === 'direct-source' && sourceKind !== 'rdrr';
77
+ }
78
+
67
79
  /**
68
80
  * Detect whether a manifest describes a Mixture-of-Experts model.
69
81
  * Validates that manifests with expert groups also have moeConfig.
@@ -84,6 +96,25 @@ function isGpuBufferInstance(value) {
84
96
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
85
97
  }
86
98
 
99
+ function getTensorShardIndices(location) {
100
+ if (Array.isArray(location?.spans) && location.spans.length > 0) {
101
+ return Array.from(new Set(location.spans.map((span) => span?.shardIndex).filter(Number.isInteger)));
102
+ }
103
+ return Number.isInteger(location?.shardIndex) ? [location.shardIndex] : [];
104
+ }
105
+
106
+ function annotateTensorLoadError(error, name, location, details = {}) {
107
+ return annotateWeightLoadError(error, {
108
+ tensorName: name,
109
+ tensorRole: location?.role ?? null,
110
+ tensorDtype: location?.dtype ?? null,
111
+ tensorShape: Array.isArray(location?.shape) ? [...location.shape] : null,
112
+ tensorSizeBytes: Number.isFinite(location?.size) ? location.size : null,
113
+ tensorShardIndices: getTensorShardIndices(location),
114
+ ...details,
115
+ });
116
+ }
117
+
87
118
  // Re-export types for backward compatibility
88
119
  export {
89
120
  // Types are in .d.ts file
@@ -168,6 +199,8 @@ export class DopplerLoader {
168
199
 
169
200
  q4kMaterializationMode = 'dense';
170
201
 
202
+ q4kFusedRoles = [];
203
+
171
204
  // Internal tracking
172
205
 
173
206
  #normOffsetLogged = false;
@@ -177,6 +210,7 @@ export class DopplerLoader {
177
210
  #memoryMonitor = null;
178
211
 
179
212
  #tensorsJsonUrl = null;
213
+ #loadTensorsJson = null;
180
214
 
181
215
  #loadShardOverride = null;
182
216
 
@@ -223,6 +257,9 @@ export class DopplerLoader {
223
257
  this.q4kLayout = config.q4kLayout;
224
258
  this.keepF32Weights = config.keepF32Weights;
225
259
  this.q4kMaterializationMode = config.q4kMaterializationMode ?? 'dense';
260
+ this.q4kFusedRoles = Array.isArray(config.q4kFusedRoles)
261
+ ? [...new Set(config.q4kFusedRoles.map((role) => String(role)).filter(Boolean))].sort()
262
+ : [];
226
263
  }
227
264
 
228
265
 
@@ -285,6 +322,10 @@ export class DopplerLoader {
285
322
  this.#tensorsJsonUrl = url;
286
323
  }
287
324
 
325
+ setTensorsJsonLoader(loadTensorsJson) {
326
+ this.#loadTensorsJson = typeof loadTensorsJson === 'function' ? loadTensorsJson : null;
327
+ }
328
+
288
329
 
289
330
  async #loadShard(shardIndex, options) {
290
331
  return this.shardCache.load(shardIndex, options);
@@ -371,7 +412,8 @@ export class DopplerLoader {
371
412
 
372
413
 
373
414
  #shouldStreamLargeWeight(name, location, label) {
374
- return shouldStreamLargeWeight(name, location, label, this.gpuCapabilities, this.keepF32Weights);
415
+ const manifestOverrides = this.manifest?.inference?.largeWeights?.gpuResidentOverrides ?? null;
416
+ return shouldStreamLargeWeight(name, location, label, this.gpuCapabilities, this.keepF32Weights, manifestOverrides);
375
417
  }
376
418
 
377
419
 
@@ -434,15 +476,14 @@ export class DopplerLoader {
434
476
  log.warn('Loader', 'Dense model on discrete GPU - performance limited. Consider MoE model.');
435
477
  }
436
478
 
437
- if (verifyHashes && !this.shardCache.hasCustomLoader) {
438
- // Avoid a full re-hash on every warm load. Presence check is enough to
439
- // decide "cached vs missing"; hash verification is performed at download/import time.
479
+ if (!this.shardCache.hasCustomLoader) {
440
480
  const integrity = await verifyIntegrity({ checkHashes: false });
441
481
  if (!integrity.valid) {
442
482
  throw new Error(
443
- `Model integrity check failed. ` +
483
+ `Artifact contract preflight failed for "${this.manifest?.modelId ?? modelId}". ` +
444
484
  `Missing shards: ${integrity.missingShards.length}, ` +
445
- `Corrupt shards: ${integrity.corruptShards.length}`
485
+ `corrupt shards: ${integrity.corruptShards.length}. ` +
486
+ 'Re-import, re-download, or provide a manifest with a valid weightsRef.'
446
487
  );
447
488
  }
448
489
  }
@@ -667,6 +708,7 @@ export class DopplerLoader {
667
708
 
668
709
  const locations = await buildTensorLocations(this.manifest, {
669
710
  tensorsJsonUrl: this.#tensorsJsonUrl,
711
+ loadTensorsJson: this.#loadTensorsJson,
670
712
  hasCustomLoader: this.shardCache.hasCustomLoader,
671
713
  });
672
714
 
@@ -730,7 +772,20 @@ export class DopplerLoader {
730
772
  // Range-capable custom loaders are expected to serve fine-grained tensor reads.
731
773
  // Whole-shard prefetch defeats that contract and can force invalid >4 GiB reads
732
774
  // for direct-source SafeTensors bundles.
733
- if (this.shardCache.hasCustomLoader && this.shardCache.canStreamRanges) return;
775
+ if (this.shardCache.hasCustomLoader && this.shardCache.canStreamRanges) {
776
+ if (prefetch.allowRangeLoaderPrefetch !== true) return;
777
+ if (isDirectSourceRuntimeManifest(this.manifest)) {
778
+ throw new Error(
779
+ 'runtime.loading.prefetch.allowRangeLoaderPrefetch is only supported for RDRR shard manifests; ' +
780
+ 'direct-source artifacts must use range reads.'
781
+ );
782
+ }
783
+ if (!Array.isArray(this.manifest?.shards) || this.manifest.shards.length === 0) {
784
+ throw new Error(
785
+ 'runtime.loading.prefetch.allowRangeLoaderPrefetch requires a manifest with explicit RDRR shards.'
786
+ );
787
+ }
788
+ }
734
789
 
735
790
  const layersAhead = prefetch.layersAhead;
736
791
  if (!Number.isFinite(layersAhead) || layersAhead <= 0) return;
@@ -769,9 +824,19 @@ export class DopplerLoader {
769
824
  debugTrace.loader(`Loading ${name}: shape=${JSON.stringify(location.shape)}, size=${location.size}, dtype=${location.dtype}, spans=${!!location.spans}`);
770
825
  }
771
826
 
772
- let shardData = (toGPU && this.#shouldStreamUploadToGPU(location))
773
- ? await this.#assembleShardDataToGpuBuffer(location, name)
774
- : await this.#assembleShardData(location, name);
827
+ const streamedUpload = toGPU && this.#shouldStreamUploadToGPU(location);
828
+ let shardData;
829
+ try {
830
+ shardData = streamedUpload
831
+ ? await this.#assembleShardDataToGpuBuffer(location, name)
832
+ : await this.#assembleShardData(location, name);
833
+ } catch (error) {
834
+ throw annotateTensorLoadError(error, name, location, {
835
+ tensorLoadStage: streamedUpload ? 'streamShardToGpuBuffer' : 'assembleShardData',
836
+ toGPU,
837
+ streamedUpload,
838
+ });
839
+ }
775
840
 
776
841
  if (toGPU) {
777
842
  const device = getDevice();
@@ -792,6 +857,7 @@ export class DopplerLoader {
792
857
  const config = {
793
858
  useFusedQ4K: this.useFusedQ4K,
794
859
  q4kMaterializationMode: this.q4kMaterializationMode,
860
+ q4kFusedRoles: this.q4kFusedRoles,
795
861
  keepF32Weights: this.keepF32Weights,
796
862
  q4kLayout: this.q4kLayout,
797
863
  loaderDebug: this.#loaderDebug,
@@ -799,7 +865,19 @@ export class DopplerLoader {
799
865
  allowF32UpcastNonMatmul,
800
866
  };
801
867
 
802
- const result = await loadTensorToGPU(shardData, location, name, config);
868
+ let result;
869
+ try {
870
+ result = await loadTensorToGPU(shardData, location, name, config);
871
+ } catch (error) {
872
+ if (isGpuBufferInstance(shardData)) {
873
+ releaseBuffer(shardData);
874
+ }
875
+ throw annotateTensorLoadError(error, name, location, {
876
+ tensorLoadStage: 'materializeTensorToGPU',
877
+ toGPU: true,
878
+ streamedUpload,
879
+ });
880
+ }
803
881
 
804
882
  for (const buffer of result.allocatedBuffers) {
805
883
  this.gpuBuffers.add(buffer);
@@ -859,7 +937,7 @@ export class DopplerLoader {
859
937
  let pendingBytes = null;
860
938
  const writeAlignedChunk = (bytes) => {
861
939
  if (bytes.byteLength === 0) return;
862
- device.queue.writeBuffer(raw, dstOffset, bytes, bytes.byteOffset, bytes.byteLength);
940
+ device.queue.writeBuffer(raw, dstOffset, bytes, 0, bytes.byteLength);
863
941
  dstOffset += bytes.byteLength;
864
942
  };
865
943
  const uploadChunk = (bytes) => {
@@ -67,17 +67,27 @@ function shouldUseRangeBackedEmbeddingSource(ctx, name, location) {
67
67
  return false;
68
68
  }
69
69
  if (String(location.dtype ?? '').toUpperCase() === 'F16' && ctx.hostHasShaderF16 === false) {
70
+ log.info(
71
+ 'Loader',
72
+ `Embedding "${name}" range-backed: F16 source on device without shader-f16 support.`
73
+ );
70
74
  return true;
71
75
  }
72
76
  if (hasSourceTransform(location) && typeof ctx.loadShardRange === 'function') {
73
77
  log.info(
74
78
  'Loader',
75
- `Embedding "${name}" uses sourceTransform.kind="${location.sourceTransform.kind}". ` +
76
- 'Using range-backed CPU source to avoid eager full materialization.'
79
+ `Embedding "${name}" range-backed: sourceTransform.kind="${location.sourceTransform.kind}" defers full materialization.`
77
80
  );
78
81
  return true;
79
82
  }
80
- return ctx.shouldStreamLargeWeight(name, location, 'Embedding');
83
+ const stream = ctx.shouldStreamLargeWeight(name, location, 'Embedding');
84
+ if (!stream) {
85
+ log.info(
86
+ 'Loader',
87
+ `Embedding "${name}" GPU-resident: shouldStreamLargeWeight returned false.`
88
+ );
89
+ }
90
+ return stream;
81
91
  }
82
92
 
83
93
  // ============================================================================
@@ -207,6 +207,8 @@ export interface Q4KConfig {
207
207
  keepF32Weights: boolean;
208
208
  /** Explicit dense/fused/mixed projection materialization mode */
209
209
  q4kMaterializationMode?: 'dense' | 'fused' | 'mixed';
210
+ /** Tensor roles that must keep raw Q4K materialization because the execution graph pins a fused Q4K kernel */
211
+ q4kFusedRoles?: string[];
210
212
  }
211
213
 
212
214
  /**