doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -1,4 +1,5 @@
1
1
  import { float32ToFloat16 } from '../../converter/quantizer.js';
2
+ import { normalizeTensorSourceTransform } from '../../formats/rdrr/source-transform-contract.js';
2
3
 
3
4
  function computeElementCount(shape, tensorName) {
4
5
  if (!Array.isArray(shape)) {
@@ -211,6 +212,26 @@ function validateLiteRTStorageEncoding(storageEncoding, tensorName) {
211
212
  }
212
213
  }
213
214
 
215
+ function validateLiteRTStorageLaneOrder(storageLaneOrder, storageBlockSize, tensorName) {
216
+ if (!Array.isArray(storageLaneOrder) || storageLaneOrder.length !== storageBlockSize) {
217
+ throw new Error(
218
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageLaneOrder ` +
219
+ `with ${storageBlockSize} entries.`
220
+ );
221
+ }
222
+ const seen = new Set();
223
+ for (let index = 0; index < storageLaneOrder.length; index++) {
224
+ const value = Number(storageLaneOrder[index]);
225
+ if (!Number.isInteger(value) || value < 0 || value >= storageBlockSize || seen.has(value)) {
226
+ throw new Error(
227
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT storageLaneOrder ` +
228
+ `${JSON.stringify(storageLaneOrder)}.`
229
+ );
230
+ }
231
+ seen.add(value);
232
+ }
233
+ }
234
+
214
235
  function resolveLiteRTScaleValue(storedScale, transform, tensorName, rowLabel) {
215
236
  const scaleSemantics = String(transform?.scaleSemantics || '').trim().toLowerCase();
216
237
  if (scaleSemantics === 'step') {
@@ -245,13 +266,14 @@ function getLiteRTCompanionByteLength(companionSource, tensorName, label, expect
245
266
  }
246
267
 
247
268
  export function getSourceTransformSpec(location, tensorName) {
248
- if (!hasSourceTransform(location)) {
269
+ const transform = normalizeTensorSourceTransform(location, tensorName);
270
+ if (!transform) {
249
271
  return null;
250
272
  }
251
- const transform = location.sourceTransform;
252
273
  if (transform.kind === 'affine_dequant') {
253
274
  const elementCount = computeElementCount(location.shape, tensorName);
254
275
  return {
276
+ transform,
255
277
  kind: 'affine_dequant',
256
278
  elementCount,
257
279
  outputByteLength: elementCount * 2,
@@ -314,6 +336,7 @@ export function getSourceTransformSpec(location, tensorName) {
314
336
  }
315
337
  }
316
338
  return {
339
+ transform,
317
340
  kind: 'litert_rowwise_dequant',
318
341
  rows,
319
342
  cols,
@@ -397,6 +420,33 @@ export function getSourceTransformSpec(location, tensorName) {
397
420
  );
398
421
  }
399
422
  }
423
+ const scaleCompanionDtype = String(transform?.scaleCompanionDtype || '').toUpperCase();
424
+ const scaleCompanionDequant = scaleCompanionDtype === 'UINT8'
425
+ ? transform.scaleCompanionDequant
426
+ : null;
427
+ if (scaleCompanionDtype === 'UINT8') {
428
+ if (!scaleCompanionDequant || typeof scaleCompanionDequant !== 'object') {
429
+ throw new Error(
430
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has UINT8 scale companion without scaleCompanionDequant metadata.`
431
+ );
432
+ }
433
+ const companionScale = Number(scaleCompanionDequant.scale);
434
+ const companionZeroPoint = Number(scaleCompanionDequant.zeroPoint);
435
+ if (!Number.isFinite(companionScale) || companionScale <= 0) {
436
+ throw new Error(
437
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.scale ${scaleCompanionDequant.scale}.`
438
+ );
439
+ }
440
+ if (!Number.isSafeInteger(companionZeroPoint)) {
441
+ throw new Error(
442
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.zeroPoint ${scaleCompanionDequant.zeroPoint}.`
443
+ );
444
+ }
445
+ } else if (scaleCompanionDtype !== '') {
446
+ throw new Error(
447
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has unsupported scaleCompanionDtype "${scaleCompanionDtype}".`
448
+ );
449
+ }
400
450
  const rawStorageRowBytes = computeSourceByteLength(storageCols, transform.sourceDtype, tensorName);
401
451
  const sourceByteLength = computeSourceByteLength(
402
452
  storageRows * storageCols,
@@ -404,13 +454,14 @@ export function getSourceTransformSpec(location, tensorName) {
404
454
  tensorName
405
455
  );
406
456
  return {
457
+ transform,
407
458
  kind: 'litert_axis_dequant',
408
459
  rows,
409
460
  cols,
410
461
  storageRows,
411
462
  storageCols,
412
463
  rawStorageRowBytes,
413
- scaleRowBytes: 4,
464
+ scaleRowBytes: scaleCompanionDtype === 'UINT8' ? 4 : 4,
414
465
  targetRowBytes: cols * 2,
415
466
  outputByteLength: rows * cols * 2,
416
467
  sourceByteLength,
@@ -419,6 +470,96 @@ export function getSourceTransformSpec(location, tensorName) {
419
470
  };
420
471
  }
421
472
 
473
+ if (transform.kind === 'litert_axis_blocked_dequant') {
474
+ if (transform.scheme !== 'per_axis_affine') {
475
+ throw new Error(
476
+ `[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
477
+ );
478
+ }
479
+ validateLiteRTTransformTarget(location, tensorName, transform, 'LiteRT blocked axis sourceTransform');
480
+ validateLiteRTStorageEncoding(transform.storageEncoding, tensorName);
481
+ if (!Array.isArray(transform.storageShape) || transform.storageShape.length !== 2) {
482
+ throw new Error(
483
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageShape=[rows, cols].`
484
+ );
485
+ }
486
+ const storageRows = Number(transform.storageShape[0]);
487
+ const storageCols = Number(transform.storageShape[1]);
488
+ const storageBlockSize = Number(transform.storageBlockSize);
489
+ if (
490
+ !Number.isInteger(storageRows)
491
+ || storageRows <= 0
492
+ || !Number.isInteger(storageCols)
493
+ || storageCols <= 0
494
+ || !Number.isInteger(storageBlockSize)
495
+ || storageBlockSize <= 0
496
+ ) {
497
+ throw new Error(
498
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked storage shape ` +
499
+ `${JSON.stringify({ storageShape: transform.storageShape, storageBlockSize: transform.storageBlockSize })}.`
500
+ );
501
+ }
502
+ if (transform.quantAxis !== 0) {
503
+ throw new Error(
504
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform only supports quantAxis=0.`
505
+ );
506
+ }
507
+ validateLiteRTStorageLaneOrder(transform.storageLaneOrder, storageBlockSize, tensorName);
508
+ if (storageCols !== rows || storageRows * storageBlockSize !== cols) {
509
+ throw new Error(
510
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform expects storageShape ` +
511
+ `[${cols / storageBlockSize}, ${rows}] for blockSize=${storageBlockSize}, got [${storageRows}, ${storageCols}].`
512
+ );
513
+ }
514
+ const scaleSource = transform.scaleSource;
515
+ if (!scaleSource || typeof scaleSource !== 'object') {
516
+ throw new Error(
517
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform is missing scaleSource.`
518
+ );
519
+ }
520
+ getLiteRTCompanionByteLength(scaleSource, tensorName, 'blocked axis scale', rows * 4);
521
+ if (transform.sumSource != null && typeof transform.sumSource !== 'object') {
522
+ throw new Error(
523
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has invalid sumSource.`
524
+ );
525
+ }
526
+ if (transform.sumSource != null) {
527
+ getLiteRTCompanionByteLength(transform.sumSource, tensorName, 'blocked axis sum', rows * 4);
528
+ }
529
+ if (transform.scaleSemantics !== 'step' && transform.scaleSemantics !== 'qmax_abs') {
530
+ throw new Error(
531
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has unsupported scaleSemantics "${transform.scaleSemantics}".`
532
+ );
533
+ }
534
+ if (transform.scaleSemantics === 'qmax_abs') {
535
+ const scaleDivisor = Number(transform.scaleDivisor);
536
+ if (!Number.isFinite(scaleDivisor) || scaleDivisor <= 0) {
537
+ throw new Error(
538
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires scaleDivisor > 0 for scaleSemantics="qmax_abs".`
539
+ );
540
+ }
541
+ }
542
+ const storageElementBytes = computeSourceByteLength(storageBlockSize, transform.sourceDtype, tensorName);
543
+ const rawStorageRowBytes = storageCols * storageElementBytes;
544
+ return {
545
+ transform,
546
+ kind: 'litert_axis_blocked_dequant',
547
+ rows,
548
+ cols,
549
+ storageRows,
550
+ storageCols,
551
+ rawStorageRowBytes,
552
+ storageElementBytes,
553
+ scaleRowBytes: 4,
554
+ targetRowBytes: cols * 2,
555
+ outputByteLength: rows * cols * 2,
556
+ sourceByteLength: storageRows * rawStorageRowBytes,
557
+ quantAxis: transform.quantAxis,
558
+ storageBlockSize,
559
+ storageLaneOrder: transform.storageLaneOrder.map((value) => Number(value)),
560
+ };
561
+ }
562
+
422
563
  throw new Error(
423
564
  `[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.kind "${transform.kind}".`
424
565
  );
@@ -438,9 +579,9 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
438
579
  return rawBytes;
439
580
  }
440
581
 
441
- const transform = location.sourceTransform;
582
+ const spec = getSourceTransformSpec(location, tensorName);
583
+ const transform = spec.transform;
442
584
  if (transform.kind === 'litert_rowwise_dequant') {
443
- const spec = getSourceTransformSpec(location, tensorName);
444
585
  const scaleBytes = options.scaleBytes;
445
586
  if (!(scaleBytes instanceof Uint8Array)) {
446
587
  throw new Error(
@@ -549,7 +690,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
549
690
  }
550
691
 
551
692
  if (transform.kind === 'litert_axis_dequant') {
552
- const spec = getSourceTransformSpec(location, tensorName);
553
693
  const scaleBytes = options.scaleBytes;
554
694
  if (!(scaleBytes instanceof Uint8Array)) {
555
695
  throw new Error(
@@ -598,6 +738,51 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
598
738
  ? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
599
739
  : null;
600
740
  const out = new Uint16Array(rowCount * spec.cols);
741
+ const scaleCompanionDtype = String(transform.scaleCompanionDtype || '').toUpperCase();
742
+ const scaleCompanion = scaleCompanionDtype === 'UINT8'
743
+ ? {
744
+ scale: Number(transform.scaleCompanionDequant?.scale),
745
+ zeroPoint: Number(transform.scaleCompanionDequant?.zeroPoint),
746
+ }
747
+ : null;
748
+ const scaleSubCount = scaleCompanion ? spec.scaleRowBytes : 1;
749
+ const scaleSubBlockWidth = scaleCompanion
750
+ ? Math.floor(spec.cols / scaleSubCount)
751
+ : spec.cols;
752
+ if (!Number.isInteger(scaleSubCount) || scaleSubCount <= 0) {
753
+ throw new Error(
754
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion block count ${scaleSubCount}.`
755
+ );
756
+ }
757
+ if (!Number.isInteger(scaleSubBlockWidth) || scaleSubBlockWidth <= 0) {
758
+ throw new Error(
759
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion layout for width ${spec.cols} and sub-count ${scaleSubCount}.`
760
+ );
761
+ }
762
+
763
+ // scaleBytes was loaded for the `[rowStart, rowStart + rowCount)` window,
764
+ // so its byte offsets are relative to `rowStart`. Subtract `rowStart`
765
+ // before indexing into scaleView; keep the caller contract of passing
766
+ // absolute row indices so error messages still name the real row.
767
+ const resolveScale = (logicalRow, col = 0) => {
768
+ const localRow = logicalRow - rowStart;
769
+ const rowScaleOffset = localRow * spec.scaleRowBytes;
770
+ const storedScale = scaleCompanion
771
+ ? (scaleView.getUint8(rowScaleOffset + Math.min(Math.floor(col / scaleSubBlockWidth), scaleSubCount - 1))
772
+ - scaleCompanion.zeroPoint) * scaleCompanion.scale
773
+ : scaleView.getFloat32(rowScaleOffset, true);
774
+ if (!Number.isFinite(storedScale) || storedScale <= 0) {
775
+ throw new Error(
776
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${storedScale}.`
777
+ );
778
+ }
779
+ return resolveLiteRTScaleValue(
780
+ storedScale,
781
+ transform,
782
+ tensorName,
783
+ `LiteRT axis row ${logicalRow}`
784
+ );
785
+ };
601
786
 
602
787
  if (spec.quantAxis === 1) {
603
788
  const expectedRawBytes = rowCount * spec.rawStorageRowBytes;
@@ -608,18 +793,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
608
793
  );
609
794
  }
610
795
  for (let row = 0; row < rowCount; row++) {
611
- const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
612
- if (!Number.isFinite(scale) || scale <= 0) {
613
- throw new Error(
614
- `[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${rowStart + row} has invalid scale ${scale}.`
615
- );
616
- }
617
- const resolvedScale = resolveLiteRTScaleValue(
618
- scale,
619
- transform,
620
- tensorName,
621
- `LiteRT axis row ${rowStart + row}`
622
- );
623
796
  const rawRowOffset = row * spec.rawStorageRowBytes;
624
797
  const rowBytes = rawBytes.subarray(rawRowOffset, rawRowOffset + spec.rawStorageRowBytes);
625
798
  const storedRowSum = hasSumSource
@@ -641,6 +814,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
641
814
  }
642
815
  const outRowOffset = row * spec.cols;
643
816
  for (let col = 0; col < spec.cols; col++) {
817
+ const scaleForCol = resolveScale(rowStart + row, col);
644
818
  const quantized = rowZeroPoint == null
645
819
  ? readQuantizedValue(
646
820
  rowBytes,
@@ -656,7 +830,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
656
830
  transform.storageEncoding
657
831
  ) - rowZeroPoint
658
832
  );
659
- out[outRowOffset + col] = float32ToFloat16(quantized * resolvedScale);
833
+ out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
660
834
  }
661
835
  }
662
836
  return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
@@ -697,19 +871,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
697
871
  for (let row = 0; row < rowCount; row++) {
698
872
  const logicalRow = rowStart + row;
699
873
  const localColumnIndex = localColumnBase + row;
700
- const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
701
- if (!Number.isFinite(scale) || scale <= 0) {
702
- throw new Error(
703
- `[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${scale}.`
704
- );
705
- }
706
- const resolvedScale = resolveLiteRTScaleValue(
707
- scale,
708
- transform,
709
- tensorName,
710
- `LiteRT axis row ${logicalRow}`
711
- );
712
-
713
874
  const storedValues = new Array(spec.cols);
714
875
  let storedSum = 0;
715
876
  for (let col = 0; col < spec.cols; col++) {
@@ -737,6 +898,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
737
898
 
738
899
  const outRowOffset = row * spec.cols;
739
900
  for (let col = 0; col < spec.cols; col++) {
901
+ const scaleForCol = resolveScale(logicalRow, col);
740
902
  const quantized = rowZeroPoint == null
741
903
  ? readQuantizedValue(
742
904
  rowSlices[col],
@@ -745,14 +907,135 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
745
907
  transform.storageEncoding
746
908
  )
747
909
  : (storedValues[col] - rowZeroPoint);
748
- out[outRowOffset + col] = float32ToFloat16(quantized * resolvedScale);
910
+ out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
749
911
  }
750
912
  }
751
913
  return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
752
914
  }
753
915
 
754
- const affineTransform = resolveAffineDequantTransform(location, tensorName);
916
+ if (transform.kind === 'litert_axis_blocked_dequant') {
917
+ const scaleBytes = options.scaleBytes;
918
+ if (!(scaleBytes instanceof Uint8Array)) {
919
+ throw new Error(
920
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires scaleBytes.`
921
+ );
922
+ }
923
+ const sumBytes = options.sumBytes ?? null;
924
+ const hasSumSource = transform.sumSource != null;
925
+ if (hasSumSource && !(sumBytes instanceof Uint8Array)) {
926
+ throw new Error(
927
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires sumBytes.`
928
+ );
929
+ }
930
+ const rowStart = options.rowStart == null ? 0 : Number(options.rowStart);
931
+ const rowCount = options.rowCount == null ? spec.rows : Number(options.rowCount);
932
+ if (!Number.isInteger(rowStart) || rowStart < 0) {
933
+ throw new Error(
934
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowStart ${options.rowStart}.`
935
+ );
936
+ }
937
+ if (!Number.isInteger(rowCount) || rowCount <= 0) {
938
+ throw new Error(
939
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowCount ${options.rowCount}.`
940
+ );
941
+ }
942
+ if (rowStart + rowCount > spec.rows) {
943
+ throw new Error(
944
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row range (${rowStart}..${rowStart + rowCount}) exceeds ${spec.rows} rows.`
945
+ );
946
+ }
947
+ if (scaleBytes.byteLength !== rowCount * spec.scaleRowBytes) {
948
+ throw new Error(
949
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis scale byte size mismatch. ` +
950
+ `Expected ${rowCount * spec.scaleRowBytes}, got ${scaleBytes.byteLength}.`
951
+ );
952
+ }
953
+ if (hasSumSource && sumBytes.byteLength !== rowCount * 4) {
954
+ throw new Error(
955
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sum byte size mismatch. ` +
956
+ `Expected ${rowCount * 4}, got ${sumBytes.byteLength}.`
957
+ );
958
+ }
959
+ const expectedRawBytes = spec.storageRows * rowCount * spec.storageElementBytes;
960
+ if (rawBytes.byteLength !== expectedRawBytes) {
961
+ throw new Error(
962
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis raw byte size mismatch. ` +
963
+ `Expected ${expectedRawBytes}, got ${rawBytes.byteLength}.`
964
+ );
965
+ }
966
+ const scaleView = new DataView(scaleBytes.buffer, scaleBytes.byteOffset, scaleBytes.byteLength);
967
+ const sumView = hasSumSource
968
+ ? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
969
+ : null;
970
+ const out = new Uint16Array(rowCount * spec.cols);
971
+ for (let row = 0; row < rowCount; row++) {
972
+ const logicalRow = rowStart + row;
973
+ const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
974
+ if (!Number.isFinite(scale) || scale <= 0) {
975
+ throw new Error(
976
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid scale ${scale}.`
977
+ );
978
+ }
979
+ const resolvedScale = resolveLiteRTScaleValue(
980
+ scale,
981
+ transform,
982
+ tensorName,
983
+ `LiteRT blocked axis row ${logicalRow}`
984
+ );
985
+ const storedRowSum = hasSumSource
986
+ ? sumView.getInt32(row * 4, true)
987
+ : null;
988
+ let storedSum = 0;
989
+ if (storedRowSum != null) {
990
+ for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
991
+ const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
992
+ const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
993
+ for (let lane = 0; lane < spec.storageBlockSize; lane++) {
994
+ storedSum += readStoredQuantizedValue(
995
+ blockBytes,
996
+ lane,
997
+ transform.sourceDtype,
998
+ transform.storageEncoding
999
+ );
1000
+ }
1001
+ }
1002
+ }
1003
+ const rowZeroPoint = storedRowSum == null
1004
+ ? null
1005
+ : (storedSum - storedRowSum) / spec.cols;
1006
+ if (rowZeroPoint != null && !Number.isFinite(rowZeroPoint)) {
1007
+ throw new Error(
1008
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid derived rowZeroPoint ${rowZeroPoint}.`
1009
+ );
1010
+ }
1011
+ const outRowOffset = row * spec.cols;
1012
+ for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
1013
+ const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
1014
+ const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
1015
+ for (let lane = 0; lane < spec.storageBlockSize; lane++) {
1016
+ const stored = readStoredQuantizedValue(
1017
+ blockBytes,
1018
+ lane,
1019
+ transform.sourceDtype,
1020
+ transform.storageEncoding
1021
+ );
1022
+ const quantized = rowZeroPoint == null
1023
+ ? readQuantizedValue(
1024
+ blockBytes,
1025
+ lane,
1026
+ transform.sourceDtype,
1027
+ transform.storageEncoding
1028
+ )
1029
+ : (stored - rowZeroPoint);
1030
+ const logicalCol = storageRow * spec.storageBlockSize + spec.storageLaneOrder[lane];
1031
+ out[outRowOffset + logicalCol] = float32ToFloat16(quantized * resolvedScale);
1032
+ }
1033
+ }
1034
+ }
1035
+ return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
1036
+ }
755
1037
 
1038
+ const affineTransform = transform;
756
1039
  const elementCount = computeElementCount(location.shape, tensorName);
757
1040
  const expectedBytes = computeSourceByteLength(
758
1041
  elementCount,
@@ -19,6 +19,8 @@ export interface TensorLoadConfig {
19
19
  useFusedQ4K: boolean;
20
20
  /** Q4K weight materialization mode derived from the resolved execution graph */
21
21
  q4kMaterializationMode?: 'dense' | 'fused' | 'mixed';
22
+ /** Tensor roles that require raw Q4K buffers because their graph step pins a fused Q4K kernel */
23
+ q4kFusedRoles?: string[];
22
24
  /** Debug controls for Q4K loading/dequantization */
23
25
  loaderDebug?: LoaderDebugConfigSchema | null;
24
26
  /** Keep weights as F32 (disable F16 downcasting) */
@@ -99,15 +101,6 @@ export declare function loadQ4KDequant(
99
101
  config: TensorLoadConfig
100
102
  ): Promise<TensorLoadResult>;
101
103
 
102
- /**
103
- * Load Q4K tensor to GPU with both dense and fused-compatible materializations.
104
- */
105
- export declare function loadQ4KMixed(
106
- shardData: Uint8Array,
107
- location: TensorLocation,
108
- name: string,
109
- config: TensorLoadConfig
110
- ): Promise<TensorLoadResult>;
111
104
 
112
105
  /**
113
106
  * Load Q6K tensor to GPU.
@@ -17,6 +17,7 @@ import { hasSourceTransform } from './source-transform.js';
17
17
  // ============================================================================
18
18
 
19
19
  let loggedF32UpcastNonMatmul = false;
20
+ let loggedQ4KLimitFallback = false;
20
21
 
21
22
  function isGpuBufferInstance(value) {
22
23
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
@@ -128,7 +129,10 @@ function isEmbeddingRole(location) {
128
129
 
129
130
  export function shouldUseFusedQ4K(location, config) {
130
131
  if (!config.useFusedQ4K) return false;
132
+ return canUseFusedQ4KStorage(location, config);
133
+ }
131
134
 
135
+ function canUseFusedQ4KStorage(location, config) {
132
136
  const caps = config.gpuCapabilities || getKernelCapabilities();
133
137
  if (!caps?.hasSubgroups) return false;
134
138
 
@@ -141,6 +145,75 @@ export function shouldUseFusedQ4K(location, config) {
141
145
  return true;
142
146
  }
143
147
 
148
+ function getQ4KDenseMaterializedSizeBytes(location, config) {
149
+ if (!Array.isArray(location.shape) || location.shape.length === 0) {
150
+ return null;
151
+ }
152
+ const elementCount = getShapeElementCount(location.shape);
153
+ if (!Number.isFinite(elementCount) || elementCount <= 0) {
154
+ return null;
155
+ }
156
+ const outputDtype = getQ4KOutputDtype(location, config);
157
+ const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
158
+ return elementCount * bytesPerElement;
159
+ }
160
+
161
+ function getMaxStorageBufferBindingSize() {
162
+ const device = getDevice();
163
+ const maxStorage = device?.limits?.maxStorageBufferBindingSize;
164
+ return Number.isFinite(maxStorage) && maxStorage > 0 ? maxStorage : null;
165
+ }
166
+
167
+ function resolveQ4KLimitFallback(location, config) {
168
+ if (location?.dtype !== 'Q4_K_M' && location?.dtype !== 'Q4_K') {
169
+ return {
170
+ denseExceedsBindingLimit: false,
171
+ limitFallbackEligible: false,
172
+ denseSizeBytes: null,
173
+ maxBindingSizeBytes: null,
174
+ };
175
+ }
176
+
177
+ const denseSizeBytes = getQ4KDenseMaterializedSizeBytes(location, config);
178
+ const maxBindingSizeBytes = getMaxStorageBufferBindingSize();
179
+ const denseExceedsBindingLimit = (
180
+ denseSizeBytes != null
181
+ && maxBindingSizeBytes != null
182
+ && denseSizeBytes > maxBindingSizeBytes
183
+ );
184
+ const packedSizeBytes = Number.isFinite(location.size) ? location.size : null;
185
+ const packedFitsBindingLimit = (
186
+ packedSizeBytes != null
187
+ && maxBindingSizeBytes != null
188
+ && packedSizeBytes <= maxBindingSizeBytes
189
+ );
190
+ const limitFallbackEligible = (
191
+ denseExceedsBindingLimit
192
+ && config.keepF32Weights !== true
193
+ && packedFitsBindingLimit
194
+ && canUseFusedQ4KStorage(location, config)
195
+ );
196
+
197
+ return {
198
+ denseExceedsBindingLimit,
199
+ limitFallbackEligible,
200
+ denseSizeBytes,
201
+ maxBindingSizeBytes,
202
+ };
203
+ }
204
+
205
+ function logQ4KLimitFallbackOnce(name, fallback) {
206
+ if (loggedQ4KLimitFallback) {
207
+ return;
208
+ }
209
+ loggedQ4KLimitFallback = true;
210
+ log.warn(
211
+ 'Loader',
212
+ `Q4K dense materialization for "${name}" would require ${fallback.denseSizeBytes} bytes, ` +
213
+ `exceeding maxStorageBufferBindingSize=${fallback.maxBindingSizeBytes}; retaining packed Q4K for fused matmul.`
214
+ );
215
+ }
216
+
144
217
  // ============================================================================
145
218
  // Dtype Output Selection
146
219
  // ============================================================================
@@ -389,7 +462,7 @@ export async function loadQ4KDequant(shardData, location, name, config) {
389
462
  }
390
463
  }
391
464
 
392
- export async function loadQ4KMixed(shardData, location, name, config) {
465
+ async function loadQ4KMixed(shardData, location, name, config) {
393
466
  const canMaterializeMixed = shouldUseFusedQ4K(location, config)
394
467
  && config.q4kLayout === 'row';
395
468
  if (!canMaterializeMixed) {
@@ -693,22 +766,32 @@ const GPU_LOADER_DISPATCH = {
693
766
  export async function loadTensorToGPU(shardData, location, name, config) {
694
767
  const dtype = location.dtype;
695
768
  const useFusedQ4K = shouldUseFusedQ4K(location, config);
769
+ const requiresFusedQ4KRole = Array.isArray(config?.q4kFusedRoles)
770
+ && config.q4kFusedRoles.includes(location.role);
696
771
  const caps = config?.gpuCapabilities || getKernelCapabilities();
697
772
  const platformId = getPlatformConfig()?.platform?.id ?? null;
698
773
  const q4kReferenceContext = getQ4KCpuReferenceContext(shardData, location, config);
699
774
  const q4kBasicBackendClass = platformId === 'basic'
700
775
  || (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
776
+ const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
701
777
  const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
702
778
  dtype,
779
+ role: location.role ?? null,
703
780
  useFusedQ4K,
781
+ requiresFusedQ4KRole,
704
782
  q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
705
783
  q4kCpuReferenceEligible: q4kReferenceContext.eligible,
706
784
  q4kBasicBackendClass,
785
+ q4kDenseExceedsBindingLimit: q4kLimitFallback.denseExceedsBindingLimit,
786
+ q4kLimitFallbackEligible: q4kLimitFallback.limitFallbackEligible,
707
787
  });
708
788
  const loader = GPU_LOADER_DISPATCH[loaderPath];
709
789
  if (!loader) {
710
790
  throw new Error(`Unknown GPU loader path: "${loaderPath}" for dtype "${dtype}"`);
711
791
  }
792
+ if (loaderPath === 'q4k_fused' && q4kLimitFallback.limitFallbackEligible) {
793
+ logQ4KLimitFallbackOnce(name, q4kLimitFallback);
794
+ }
712
795
  return loader(shardData, location, name, config);
713
796
  }
714
797
 
@@ -161,7 +161,7 @@ function getSourceTransformScaleLocation(location, name) {
161
161
 
162
162
  function getSourceTransformSumLocation(location) {
163
163
  const transform = location?.sourceTransform;
164
- const sumLocation = transform?.kind === 'litert_axis_dequant'
164
+ const sumLocation = transform?.kind === 'litert_axis_dequant' || transform?.kind === 'litert_axis_blocked_dequant'
165
165
  ? transform?.sumSource
166
166
  : transform?.rowSumSource;
167
167
  if (!sumLocation || typeof sumLocation !== 'object') {
@@ -177,6 +177,7 @@ async function materializeLocationBytes(rawBytes, location, name, loadShard, loa
177
177
  if (
178
178
  location.sourceTransform.kind === 'litert_rowwise_dequant'
179
179
  || location.sourceTransform.kind === 'litert_axis_dequant'
180
+ || location.sourceTransform.kind === 'litert_axis_blocked_dequant'
180
181
  ) {
181
182
  const scaleLocation = getSourceTransformScaleLocation(location, name);
182
183
  const scaleBytes = options.scaleBytes instanceof Uint8Array
@@ -284,7 +285,11 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
284
285
  if (hasSourceTransform(location)) {
285
286
  const spec = getSourceTransformSpec(location, name);
286
287
  if (
287
- (spec?.kind === 'litert_rowwise_dequant' || spec?.kind === 'litert_axis_dequant')
288
+ (
289
+ spec?.kind === 'litert_rowwise_dequant'
290
+ || spec?.kind === 'litert_axis_dequant'
291
+ || spec?.kind === 'litert_axis_blocked_dequant'
292
+ )
288
293
  && (byteOffset % spec.targetRowBytes) === 0
289
294
  && (byteLength % spec.targetRowBytes) === 0
290
295
  ) {
@@ -343,6 +348,27 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
343
348
  });
344
349
  }
345
350
 
351
+ if (spec.kind === 'litert_axis_blocked_dequant') {
352
+ const raw = new Uint8Array(spec.storageRows * rowCount * spec.storageElementBytes);
353
+ for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
354
+ const rowBytes = await loadLocationRange(
355
+ location,
356
+ name,
357
+ `LiteRT blocked axis raw storage row ${storageRow}`,
358
+ storageRow * spec.rawStorageRowBytes + rowStart * spec.storageElementBytes,
359
+ rowCount * spec.storageElementBytes,
360
+ loadShardRange
361
+ );
362
+ raw.set(rowBytes, storageRow * rowCount * spec.storageElementBytes);
363
+ }
364
+ return materializeTensorSourceTransform(raw, location, name, {
365
+ scaleBytes,
366
+ sumBytes,
367
+ rowStart,
368
+ rowCount,
369
+ });
370
+ }
371
+
346
372
  const storageColumnStart = Math.floor(rowStart / spec.storageValuesPerByte) * spec.storageValuesPerByte;
347
373
  const storageColumnEnd = Math.ceil((rowStart + rowCount) / spec.storageValuesPerByte) * spec.storageValuesPerByte;
348
374
  const rawSliceRowBytes = (storageColumnEnd - storageColumnStart) / spec.storageValuesPerByte;