doppler-gpu 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (880) hide show
  1. package/CHANGELOG.md +210 -0
  2. package/README.md +63 -38
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +111 -32
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +486 -0
  7. package/src/cli/doppler-cli.d.ts +18 -0
  8. package/src/cli/doppler-cli.js +885 -744
  9. package/src/cli/doppler-quickstart.d.ts +7 -23
  10. package/src/cli/doppler-quickstart.js +4 -2
  11. package/src/cli/doppler-serve.d.ts +210 -0
  12. package/src/cli/doppler-serve.js +469 -43
  13. package/src/cli/runtime-profiles.d.ts +40 -0
  14. package/src/cli/runtime-profiles.js +225 -0
  15. package/src/client/doppler-api.js +1 -6
  16. package/src/client/doppler-registry.d.ts +12 -0
  17. package/src/client/doppler-registry.js +27 -0
  18. package/src/client/doppler-registry.json +66 -10
  19. package/src/client/failure-taxonomy.js +139 -0
  20. package/src/client/fault-injection.js +30 -0
  21. package/src/client/provider.d.ts +151 -0
  22. package/src/client/provider.js +374 -0
  23. package/src/client/receipt.js +156 -0
  24. package/src/client/runtime/index.js +69 -4
  25. package/src/client/runtime/lora.d.ts +11 -1
  26. package/src/client/runtime/lora.js +28 -7
  27. package/src/client/runtime/model-manager.d.ts +0 -14
  28. package/src/client/runtime/model-manager.js +34 -50
  29. package/src/client/runtime/model-session.d.ts +2 -1
  30. package/src/client/runtime/model-session.js +35 -2
  31. package/src/client/runtime/model-source.d.ts +13 -1
  32. package/src/client/runtime/model-source.js +148 -3
  33. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  34. package/src/client/runtime/node-quickstart-cache.js +353 -0
  35. package/src/client/runtime/source-runtime.js +1 -10
  36. package/src/client/runtime/types.d.ts +8 -60
  37. package/src/client/wrap-pipeline-handle.d.ts +82 -0
  38. package/src/client/wrap-pipeline-handle.js +259 -0
  39. package/src/config/README.md +3 -1
  40. package/src/config/conversion/README.md +28 -157
  41. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  42. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +21 -6
  43. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +19 -12
  44. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +27 -12
  45. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  46. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  47. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +39 -16
  48. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +19 -12
  49. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +27 -12
  50. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  51. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  52. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  53. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  54. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  55. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  56. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +591 -0
  57. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  58. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +367 -61
  59. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  60. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  61. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  62. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +19 -12
  63. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +82 -36
  64. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +96 -60
  65. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +521 -0
  66. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +515 -0
  67. package/src/config/execution-contract-check.js +6 -5
  68. package/src/config/goal-completion-matrix.json +552 -0
  69. package/src/config/kernel-path-loader.d.ts +8 -2
  70. package/src/config/kernel-path-loader.js +192 -38
  71. package/src/config/kernels/kernel-ref-digests.js +163 -126
  72. package/src/config/kernels/registry.d.ts +0 -35
  73. package/src/config/kernels/registry.js +5 -9
  74. package/src/config/kernels/registry.json +3593 -1146
  75. package/src/config/merge-contract-check.js +7 -3
  76. package/src/config/merge-helpers.d.ts +3 -1
  77. package/src/config/merge-helpers.js +2 -1
  78. package/src/config/merge.d.ts +1 -0
  79. package/src/config/merge.js +65 -7
  80. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  81. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  82. package/src/config/param-categories.d.ts +0 -9
  83. package/src/config/param-categories.js +1 -5
  84. package/src/config/param-validator.d.ts +2 -0
  85. package/src/config/param-validator.js +20 -0
  86. package/src/config/platforms/loader.d.ts +0 -27
  87. package/src/config/platforms/loader.js +0 -30
  88. package/src/config/required-inference-fields-contract-check.js +1 -7
  89. package/src/config/runtime/README.md +28 -0
  90. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -1
  91. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  92. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  93. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  94. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  95. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  96. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  97. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  98. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  99. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  100. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  101. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  102. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  103. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  104. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  105. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  106. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  107. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  108. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  109. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  110. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  111. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  112. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  113. package/src/config/schema/bridge.schema.d.ts +16 -0
  114. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  115. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  116. package/src/config/schema/buffer-pool.schema.js +3 -3
  117. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  118. package/src/config/schema/command-surface-contract.schema.json +81 -0
  119. package/src/config/schema/converter.schema.d.ts +16 -0
  120. package/src/config/schema/converter.schema.js +11 -0
  121. package/src/config/schema/debug.schema.d.ts +22 -1
  122. package/src/config/schema/debug.schema.js +1 -0
  123. package/src/config/schema/distill-training.schema.d.ts +0 -1
  124. package/src/config/schema/distill-training.schema.js +1 -1
  125. package/src/config/schema/doppler.schema.d.ts +13 -2
  126. package/src/config/schema/doppler.schema.js +140 -8
  127. package/src/config/schema/emulation.schema.d.ts +3 -0
  128. package/src/config/schema/emulation.schema.js +16 -16
  129. package/src/config/schema/execution-v1.schema.d.ts +39 -1
  130. package/src/config/schema/execution-v1.schema.js +35 -20
  131. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  132. package/src/config/schema/index.d.ts +52 -1
  133. package/src/config/schema/index.js +14 -0
  134. package/src/config/schema/inference.schema.d.ts +14 -3
  135. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  136. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  137. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  138. package/src/config/schema/kvcache.schema.js +0 -9
  139. package/src/config/schema/loading.schema.d.ts +12 -0
  140. package/src/config/schema/loading.schema.js +4 -0
  141. package/src/config/schema/manifest.schema.d.ts +175 -2
  142. package/src/config/schema/manifest.schema.js +58 -3
  143. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  144. package/src/config/schema/moe.schema.d.ts +13 -0
  145. package/src/config/schema/moe.schema.js +1 -9
  146. package/src/config/schema/policy-schema-registry.json +26 -0
  147. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  148. package/src/config/schema/program-bundle.schema.js +393 -0
  149. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  150. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  151. package/src/config/schema/training-metrics.schema.js +4 -3
  152. package/src/config/schema/ul-training.schema.d.ts +0 -1
  153. package/src/config/schema/ul-training.schema.js +1 -1
  154. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  155. package/src/config/source-packages/litert/gemma-4-e2b-it.json +41 -20
  156. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  157. package/src/config/source-packages/registry.json +25 -0
  158. package/src/config/supported-operations.d.ts +14 -0
  159. package/src/config/supported-operations.js +34 -0
  160. package/src/config/supported-operations.json +183 -0
  161. package/src/config/training-defaults.js +1 -7
  162. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  163. package/src/config/transforms/capability-transform-resolver.js +3 -1
  164. package/src/config/transforms/execution-graph-transforms.d.ts +97 -0
  165. package/src/config/transforms/execution-graph-transforms.js +829 -168
  166. package/src/converter/conversion-plan.d.ts +2 -0
  167. package/src/converter/conversion-plan.js +31 -14
  168. package/src/converter/core.d.ts +65 -0
  169. package/src/converter/core.js +1153 -22
  170. package/src/converter/parsers/diffusion.js +1 -33
  171. package/src/converter/quantization-info.d.ts +5 -4
  172. package/src/converter/quantization-info.js +218 -5
  173. package/src/converter/quantizer.d.ts +24 -4
  174. package/src/converter/quantizer.js +92 -2
  175. package/src/debug/config.d.ts +0 -4
  176. package/src/debug/config.js +0 -17
  177. package/src/debug/index.d.ts +28 -30
  178. package/src/debug/index.js +0 -18
  179. package/src/debug/stats.d.ts +0 -8
  180. package/src/debug/stats.js +3 -3
  181. package/src/debug/tensor.d.ts +9 -2
  182. package/src/debug/tensor.js +11 -4
  183. package/src/debug/trace.js +2 -15
  184. package/src/errors/doppler-error.js +20 -0
  185. package/src/experimental/adapters/lora-loader.js +109 -24
  186. package/src/experimental/browser/quantization.d.ts +5 -0
  187. package/src/experimental/diffusion/index.d.ts +24 -0
  188. package/src/experimental/diffusion/index.js +10 -0
  189. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  190. package/src/experimental/distribution/descriptor-transport.js +342 -0
  191. package/src/experimental/distribution/shard-delivery.js +179 -23
  192. package/src/experimental/training/README.md +11 -0
  193. package/src/experimental/training/artifacts.js +1 -14
  194. package/src/experimental/training/autograd.js +33 -8
  195. package/src/experimental/training/checkpoint.js +1 -14
  196. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  197. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  198. package/src/experimental/training/datasets/index.d.ts +13 -1
  199. package/src/experimental/training/datasets/index.js +8 -1
  200. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  201. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  202. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  203. package/src/experimental/training/datasets/text-pairs.js +82 -5
  204. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  205. package/src/experimental/training/datasets/token-batch.js +2 -2
  206. package/src/experimental/training/distillation/dataset.js +2 -15
  207. package/src/experimental/training/distillation/eval.js +3 -3
  208. package/src/experimental/training/distillation/runtime.js +3 -16
  209. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  210. package/src/experimental/training/distillation/student-fixture.js +296 -38
  211. package/src/experimental/training/export.d.ts +9 -0
  212. package/src/experimental/training/export.js +113 -7
  213. package/src/experimental/training/index.d.ts +13 -1
  214. package/src/experimental/training/index.js +13 -1
  215. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  216. package/src/experimental/training/lora-pipeline.js +1733 -58
  217. package/src/experimental/training/lora.js +23 -1
  218. package/src/experimental/training/loss.js +1 -1
  219. package/src/experimental/training/objectives/base.d.ts +2 -2
  220. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  221. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  222. package/src/experimental/training/objectives/distill_kd.js +2 -2
  223. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  224. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  225. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  226. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  227. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  228. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  229. package/src/experimental/training/operator-agent-eval.js +326 -0
  230. package/src/experimental/training/operator-artifacts.js +1 -14
  231. package/src/experimental/training/operator-command.js +310 -51
  232. package/src/experimental/training/runner.d.ts +1 -0
  233. package/src/experimental/training/runner.js +5 -14
  234. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  235. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  236. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  237. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  238. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  239. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  240. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  241. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  242. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  243. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  244. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  245. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  246. package/src/experimental/training/workload-packs/registry.json +192 -12
  247. package/src/experimental/training/workloads.d.ts +37 -0
  248. package/src/experimental/training/workloads.js +111 -2
  249. package/src/formats/gguf/types.d.ts +0 -63
  250. package/src/formats/gguf/types.js +58 -38
  251. package/src/formats/litert/types.d.ts +0 -13
  252. package/src/formats/litert/types.js +2 -2
  253. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  254. package/src/formats/rdrr/distributed/index.js +3 -0
  255. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  256. package/src/formats/rdrr/distributed/parsing.js +23 -0
  257. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  258. package/src/formats/rdrr/distributed/validation.js +131 -0
  259. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  260. package/src/formats/rdrr/functional-descriptor.js +230 -0
  261. package/src/formats/rdrr/groups.d.ts +1 -13
  262. package/src/formats/rdrr/groups.js +0 -34
  263. package/src/formats/rdrr/index.d.ts +10 -0
  264. package/src/formats/rdrr/index.js +10 -0
  265. package/src/formats/rdrr/integrity.d.ts +51 -0
  266. package/src/formats/rdrr/integrity.js +200 -0
  267. package/src/formats/rdrr/lowerings.d.ts +28 -0
  268. package/src/formats/rdrr/lowerings.js +112 -0
  269. package/src/formats/rdrr/manifest.d.ts +0 -14
  270. package/src/formats/rdrr/manifest.js +5 -58
  271. package/src/formats/rdrr/merkle.d.ts +24 -0
  272. package/src/formats/rdrr/merkle.js +118 -0
  273. package/src/formats/rdrr/parsing.js +26 -6
  274. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  275. package/src/formats/rdrr/slice-compiler.js +188 -0
  276. package/src/formats/rdrr/source-transform-contract.js +10 -1
  277. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  278. package/src/formats/rdrr/storage-descriptor.js +193 -0
  279. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  280. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  281. package/src/formats/rdrr/types.d.ts +111 -2
  282. package/src/formats/rdrr/validation.js +260 -1
  283. package/src/formats/tflite/types.d.ts +0 -31
  284. package/src/formats/tflite/types.js +5 -5
  285. package/src/gpu/command-recorder.d.ts +45 -23
  286. package/src/gpu/command-recorder.js +122 -25
  287. package/src/gpu/device.d.ts +17 -3
  288. package/src/gpu/device.js +130 -14
  289. package/src/gpu/kernel-runtime.d.ts +8 -15
  290. package/src/gpu/kernel-selection-log.d.ts +0 -8
  291. package/src/gpu/kernel-selection-log.js +0 -15
  292. package/src/gpu/kernel-tuner/cache.js +1 -8
  293. package/src/gpu/kernels/README.md +50 -18
  294. package/src/gpu/kernels/activation_static_qdq.wgsl +3 -1
  295. package/src/gpu/kernels/attention.d.ts +28 -19
  296. package/src/gpu/kernels/attention.js +458 -16
  297. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  298. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  299. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  300. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  301. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  302. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  303. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  304. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  305. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  306. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  307. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  308. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  309. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  310. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  311. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  312. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  313. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  314. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  315. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  316. package/src/gpu/kernels/attention_streaming_f16.wgsl +34 -9
  317. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  318. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  319. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  320. package/src/gpu/kernels/backward/index.d.ts +6 -0
  321. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  322. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  323. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  324. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  325. package/src/gpu/kernels/cast.js +74 -67
  326. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  327. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  328. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  329. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  330. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +12 -50
  331. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +16 -33
  332. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  333. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +255 -170
  334. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  335. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  336. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  337. package/src/gpu/kernels/constants.d.ts +9 -48
  338. package/src/gpu/kernels/constants.js +0 -23
  339. package/src/gpu/kernels/conv2d.d.ts +0 -4
  340. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  341. package/src/gpu/kernels/dequant.d.ts +0 -16
  342. package/src/gpu/kernels/dequant.js +1 -71
  343. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  344. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  345. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  346. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  347. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  348. package/src/gpu/kernels/dispatch.d.ts +12 -97
  349. package/src/gpu/kernels/dispatch.js +13 -120
  350. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  351. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  352. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  353. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  354. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  355. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  356. package/src/gpu/kernels/fused_ffn.js +3 -4
  357. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  358. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  359. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  360. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  361. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  362. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  363. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  364. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  365. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  366. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  367. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  368. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  369. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  370. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  371. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  372. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  373. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  374. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  375. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  376. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  377. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  378. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  379. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  380. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  381. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  382. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  383. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  384. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  385. package/src/gpu/kernels/gather.d.ts +84 -8
  386. package/src/gpu/kernels/gather.js +353 -5
  387. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  388. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  389. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  390. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  391. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  392. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  393. package/src/gpu/kernels/gelu.wgsl +11 -7
  394. package/src/gpu/kernels/gelu_f16.wgsl +25 -22
  395. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  396. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  397. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  398. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  399. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  400. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  401. package/src/gpu/kernels/index.d.ts +77 -5
  402. package/src/gpu/kernels/index.js +55 -4
  403. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  404. package/src/gpu/kernels/kv-cache-write.js +126 -0
  405. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  406. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  407. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  408. package/src/gpu/kernels/layernorm.d.ts +0 -4
  409. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  410. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  411. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  412. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  413. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  414. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  415. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  416. package/src/gpu/kernels/logit-merge.js +1 -17
  417. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  418. package/src/gpu/kernels/matmul-dispatch.js +44 -2
  419. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  420. package/src/gpu/kernels/matmul-selection.js +172 -10
  421. package/src/gpu/kernels/matmul.d.ts +1 -29
  422. package/src/gpu/kernels/matmul.js +240 -38
  423. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  424. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  425. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  426. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  427. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  428. package/src/gpu/kernels/moe.d.ts +1 -56
  429. package/src/gpu/kernels/moe.js +46 -33
  430. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  431. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  432. package/src/gpu/kernels/pipeline-cache.js +24 -0
  433. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  434. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  435. package/src/gpu/kernels/rep-penalty.js +17 -9
  436. package/src/gpu/kernels/residual.d.ts +1 -12
  437. package/src/gpu/kernels/residual.js +13 -1
  438. package/src/gpu/kernels/residual.wgsl +3 -3
  439. package/src/gpu/kernels/residual_f16.wgsl +6 -4
  440. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  441. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  442. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  443. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  444. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  445. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  446. package/src/gpu/kernels/rmsnorm.d.ts +30 -7
  447. package/src/gpu/kernels/rmsnorm.js +49 -8
  448. package/src/gpu/kernels/rmsnorm.wgsl +15 -12
  449. package/src/gpu/kernels/rmsnorm_f16.wgsl +69 -39
  450. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  451. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  452. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  453. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  454. package/src/gpu/kernels/rope-qk.js +143 -0
  455. package/src/gpu/kernels/rope.d.ts +1 -5
  456. package/src/gpu/kernels/rope.js +12 -2
  457. package/src/gpu/kernels/rope.wgsl +33 -19
  458. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  459. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  460. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  461. package/src/gpu/kernels/rule-registry.js +50 -10
  462. package/src/gpu/kernels/sample.d.ts +0 -5
  463. package/src/gpu/kernels/sample.js +212 -187
  464. package/src/gpu/kernels/sample.wgsl +7 -6
  465. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  466. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  467. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  468. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  469. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  470. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  471. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  472. package/src/gpu/kernels/shader-cache.js +28 -21
  473. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  474. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  475. package/src/gpu/kernels/soft-embedding.js +447 -0
  476. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  477. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  478. package/src/gpu/kernels/softmax.wgsl +8 -223
  479. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  480. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  481. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  482. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  483. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  484. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  485. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  486. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  487. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  488. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  489. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  490. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  491. package/src/gpu/kernels/uniform-utils.js +2 -5
  492. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  493. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  494. package/src/gpu/kernels/utils.d.ts +43 -0
  495. package/src/gpu/kernels/utils.js +81 -11
  496. package/src/gpu/perf-guards.d.ts +0 -12
  497. package/src/gpu/perf-guards.js +0 -62
  498. package/src/gpu/profiler.d.ts +0 -21
  499. package/src/gpu/profiler.js +0 -34
  500. package/src/gpu/readback-utils.js +10 -2
  501. package/src/gpu/submit-tracker.d.ts +0 -57
  502. package/src/gpu/submit-tracker.js +3 -62
  503. package/src/gpu/tensor.d.ts +0 -23
  504. package/src/gpu/tensor.js +0 -41
  505. package/src/gpu/uniform-cache.d.ts +10 -12
  506. package/src/gpu/uniform-cache.js +25 -30
  507. package/src/gpu/weight-buffer.d.ts +76 -7
  508. package/src/gpu/weight-buffer.js +85 -1
  509. package/src/index.d.ts +1 -0
  510. package/src/index.js +1 -0
  511. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  512. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  513. package/src/inference/browser-harness-model-helpers.js +22 -13
  514. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  515. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  516. package/src/inference/browser-harness-suite-helpers.d.ts +13 -1
  517. package/src/inference/browser-harness-suite-helpers.js +525 -3
  518. package/src/inference/browser-harness-text-helpers.d.ts +89 -0
  519. package/src/inference/browser-harness-text-helpers.js +459 -25
  520. package/src/inference/browser-harness.d.ts +7 -28
  521. package/src/inference/browser-harness.js +556 -79
  522. package/src/inference/decode-ring.d.ts +1 -0
  523. package/src/inference/decode-ring.js +7 -5
  524. package/src/inference/kv-cache/base.d.ts +31 -0
  525. package/src/inference/kv-cache/base.js +125 -2
  526. package/src/inference/kv-cache/mixed-geometry.d.ts +11 -0
  527. package/src/inference/kv-cache/mixed-geometry.js +121 -4
  528. package/src/inference/kv-cache/quantized.js +2 -0
  529. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  530. package/src/inference/kv-cache/sliding-window.js +88 -0
  531. package/src/inference/kv-cache/tiered.js +7 -0
  532. package/src/inference/kv-cache/types.d.ts +2 -0
  533. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  534. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  535. package/src/inference/moe-router.d.ts +14 -1
  536. package/src/inference/moe-router.js +24 -1
  537. package/src/inference/multi-model-network.d.ts +1 -1
  538. package/src/inference/multi-model-network.js +1 -2
  539. package/src/inference/network-evolution.d.ts +0 -12
  540. package/src/inference/network-evolution.js +2 -2
  541. package/src/inference/pipelines/audio/gemma4.js +52 -1
  542. package/src/inference/pipelines/context.js +17 -0
  543. package/src/inference/pipelines/diffusion/init.js +2 -2
  544. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  545. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  546. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  547. package/src/inference/pipelines/diffusion/vae.js +4 -4
  548. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  549. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  550. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  551. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  552. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  553. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  554. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  555. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  556. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  557. package/src/inference/pipelines/structured/index.d.ts +2 -0
  558. package/src/inference/pipelines/structured/index.js +2 -0
  559. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  560. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  561. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  562. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  563. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  564. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  565. package/src/inference/pipelines/text/attention/projections.d.ts +73 -2
  566. package/src/inference/pipelines/text/attention/projections.js +398 -71
  567. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  568. package/src/inference/pipelines/text/attention/record.js +542 -56
  569. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  570. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  571. package/src/inference/pipelines/text/attention/run.js +343 -50
  572. package/src/inference/pipelines/text/attention/types.d.ts +7 -1
  573. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  574. package/src/inference/pipelines/text/chat-format.js +1 -8
  575. package/src/inference/pipelines/text/config.d.ts +25 -26
  576. package/src/inference/pipelines/text/config.js +304 -25
  577. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  578. package/src/inference/pipelines/text/drift-policy.js +1 -1
  579. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  580. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  581. package/src/inference/pipelines/text/embed.d.ts +25 -29
  582. package/src/inference/pipelines/text/embed.js +16 -45
  583. package/src/inference/pipelines/text/execution-plan.d.ts +13 -3
  584. package/src/inference/pipelines/text/execution-plan.js +21 -2
  585. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -4
  586. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -39
  587. package/src/inference/pipelines/text/execution-v1.d.ts +20 -0
  588. package/src/inference/pipelines/text/execution-v1.js +461 -158
  589. package/src/inference/pipelines/text/ffn/dense.js +214 -6
  590. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  591. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  592. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  593. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  594. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  595. package/src/inference/pipelines/text/ffn/standard.js +84 -17
  596. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  597. package/src/inference/pipelines/text/generator-decode-policy.js +233 -0
  598. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -2
  599. package/src/inference/pipelines/text/generator-helpers.js +9 -7
  600. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  601. package/src/inference/pipelines/text/generator-prefill-helpers.js +359 -0
  602. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -7
  603. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  604. package/src/inference/pipelines/text/generator-steps.d.ts +57 -7
  605. package/src/inference/pipelines/text/generator-steps.js +275 -79
  606. package/src/inference/pipelines/text/generator.d.ts +38 -5
  607. package/src/inference/pipelines/text/generator.js +1259 -496
  608. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  609. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  610. package/src/inference/pipelines/text/init-rope.js +408 -0
  611. package/src/inference/pipelines/text/init.d.ts +13 -3
  612. package/src/inference/pipelines/text/init.js +213 -725
  613. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  614. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  615. package/src/inference/pipelines/text/layer-plan-gpu.js +590 -0
  616. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  617. package/src/inference/pipelines/text/layer-plan.js +1 -1
  618. package/src/inference/pipelines/text/layer.d.ts +70 -8
  619. package/src/inference/pipelines/text/layer.js +373 -558
  620. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  621. package/src/inference/pipelines/text/linear-attention.js +6 -4
  622. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  623. package/src/inference/pipelines/text/load-errors.js +59 -0
  624. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  625. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  626. package/src/inference/pipelines/text/logits/gpu.js +419 -66
  627. package/src/inference/pipelines/text/logits/index.d.ts +3 -3
  628. package/src/inference/pipelines/text/logits/index.js +96 -42
  629. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  630. package/src/inference/pipelines/text/lora-apply.js +17 -0
  631. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  632. package/src/inference/pipelines/text/model-load.js +74 -17
  633. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  634. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  635. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  636. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  637. package/src/inference/pipelines/text/ops.d.ts +42 -4
  638. package/src/inference/pipelines/text/ops.js +20 -1
  639. package/src/inference/pipelines/text/per-layer-inputs.js +36 -6
  640. package/src/inference/pipelines/text/probes.d.ts +3 -6
  641. package/src/inference/pipelines/text/probes.js +119 -31
  642. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  643. package/src/inference/pipelines/text/sampling-config.js +103 -64
  644. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  645. package/src/inference/pipelines/text/sampling.js +153 -9
  646. package/src/inference/pipelines/text/stage-names.js +3 -0
  647. package/src/inference/pipelines/text/state.js +19 -0
  648. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  649. package/src/inference/pipelines/text/tsir-fixture-writer.js +408 -0
  650. package/src/inference/pipelines/text/types.d.ts +92 -81
  651. package/src/inference/pipelines/text/weights.d.ts +4 -52
  652. package/src/inference/pipelines/text/weights.js +6 -56
  653. package/src/inference/pipelines/text.d.ts +33 -1
  654. package/src/inference/pipelines/text.js +375 -55
  655. package/src/inference/pipelines/vision/gemma4.js +20 -9
  656. package/src/inference/pipelines/vision/index.js +0 -107
  657. package/src/inference/runtime-model.d.ts +0 -7
  658. package/src/inference/runtime-model.js +0 -23
  659. package/src/inference/test-harness.d.ts +6 -22
  660. package/src/inference/test-harness.js +97 -23
  661. package/src/inference/tokenizer.d.ts +30 -0
  662. package/src/inference/tokenizer.js +341 -174
  663. package/src/inference/tokenizers/bundled.js +71 -49
  664. package/src/inference/tokenizers/sentencepiece.js +65 -15
  665. package/src/inference/tokenizers/types.d.ts +0 -21
  666. package/src/loader/doppler-loader.d.ts +14 -0
  667. package/src/loader/doppler-loader.js +378 -25
  668. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  669. package/src/loader/embedding-limit-preflight.js +216 -0
  670. package/src/loader/embedding-loader.d.ts +8 -2
  671. package/src/loader/embedding-loader.js +276 -11
  672. package/src/loader/experts/expert-loader.d.ts +4 -1
  673. package/src/loader/experts/expert-loader.js +174 -5
  674. package/src/loader/final-weights-loader.d.ts +5 -0
  675. package/src/loader/final-weights-loader.js +167 -4
  676. package/src/loader/layer-loader.d.ts +2 -0
  677. package/src/loader/layer-loader.js +74 -6
  678. package/src/loader/loader-types.d.ts +61 -2
  679. package/src/loader/manifest-config.d.ts +8 -0
  680. package/src/loader/manifest-config.js +25 -1
  681. package/src/loader/memory-monitor.d.ts +17 -5
  682. package/src/loader/quantization-constants.d.ts +6 -0
  683. package/src/loader/shard-resolver.d.ts +1 -0
  684. package/src/loader/shard-resolver.js +52 -4
  685. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  686. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  687. package/src/loader/tensors/source-transform.js +6 -1
  688. package/src/loader/tensors/tensor-loader.d.ts +36 -8
  689. package/src/loader/tensors/tensor-loader.js +521 -22
  690. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  691. package/src/loader/tensors/tensor-reader.js +11 -4
  692. package/src/loader/weights.d.ts +3 -1
  693. package/src/memory/address-table.d.ts +1 -9
  694. package/src/memory/buffer-pool.d.ts +0 -16
  695. package/src/memory/buffer-pool.js +13 -42
  696. package/src/memory/heap-manager.d.ts +18 -43
  697. package/src/memory/unified-detect.d.ts +19 -34
  698. package/src/models/diffusiongemma.d.ts +7 -0
  699. package/src/models/diffusiongemma.js +25 -0
  700. package/src/models/embeddinggemma.d.ts +7 -0
  701. package/src/models/embeddinggemma.js +24 -0
  702. package/src/models/family.d.ts +24 -0
  703. package/src/models/family.js +25 -0
  704. package/src/models/gemma3.d.ts +7 -0
  705. package/src/models/gemma3.js +33 -0
  706. package/src/models/gemma4.d.ts +7 -0
  707. package/src/models/gemma4.js +57 -0
  708. package/src/models/qwen3.d.ts +7 -0
  709. package/src/models/qwen3.js +36 -0
  710. package/src/rules/converter/tokenizer.rules.json +5 -0
  711. package/src/rules/execution-rules-contract-check.js +162 -10
  712. package/src/rules/inference/capability-transforms.rules.json +111 -0
  713. package/src/rules/inference/config.rules.json +1 -0
  714. package/src/rules/inference/dtype.rules.json +7 -0
  715. package/src/rules/inference/execution.rules.json +117 -16
  716. package/src/rules/inference/ffn.rules.json +24 -2
  717. package/src/rules/kernels/gather.rules.json +81 -0
  718. package/src/rules/kernels/matmul.rules.json +19 -0
  719. package/src/rules/kernels/moe.rules.json +3 -0
  720. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  721. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  722. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  723. package/src/rules/kernels/rope-qk.rules.json +6 -0
  724. package/src/rules/loader/tensor-loader.rules.json +17 -0
  725. package/src/rules/rule-registry.js +59 -75
  726. package/src/storage/artifact-storage-context.d.ts +12 -3
  727. package/src/storage/artifact-storage-context.js +241 -26
  728. package/src/storage/backends/opfs-store.js +138 -88
  729. package/src/storage/blake3.js +9 -1
  730. package/src/storage/export.js +1 -5
  731. package/src/storage/normalize-model-id.js +3 -0
  732. package/src/storage/preflight.js +11 -0
  733. package/src/storage/quickstart-downloader.js +44 -67
  734. package/src/storage/reports.js +35 -11
  735. package/src/storage/shard-manager.d.ts +8 -2
  736. package/src/storage/shard-manager.js +176 -15
  737. package/src/storage/source-artifact-store.d.ts +0 -2
  738. package/src/storage/source-artifact-store.js +3 -9
  739. package/src/sw.js +0 -3
  740. package/src/tooling/browser-command-runner.d.ts +0 -3
  741. package/src/tooling/browser-command-runner.js +31 -4
  742. package/src/tooling/command-api-constants.js +1 -1
  743. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  744. package/src/tooling/command-api-family-normalizers.js +53 -9
  745. package/src/tooling/command-api-helpers.d.ts +0 -2
  746. package/src/tooling/command-api-helpers.js +8 -3
  747. package/src/tooling/command-api.d.ts +22 -1
  748. package/src/tooling/command-api.js +11 -1
  749. package/src/tooling/command-envelope.d.ts +0 -1
  750. package/src/tooling/command-envelope.js +1 -1
  751. package/src/tooling/command-runner-shared.d.ts +14 -0
  752. package/src/tooling/command-runner-shared.js +2 -1
  753. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  754. package/src/tooling/conversion-config-materializer.js +1 -1
  755. package/src/tooling/diagnose-runner.js +3 -4
  756. package/src/tooling/hf-registry-utils.d.ts +2 -6
  757. package/src/tooling/hf-registry-utils.js +90 -19
  758. package/src/tooling/kernel-path-builder/index.js +1 -4
  759. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  760. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  761. package/src/tooling/lean-execution-contract-runner.js +2 -2
  762. package/src/tooling/litert-package-runtime.d.ts +1 -1
  763. package/src/tooling/litert-package-runtime.js +571 -37
  764. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  765. package/src/tooling/node-browser-command-runner.js +107 -17
  766. package/src/tooling/node-command-runner.js +43 -2
  767. package/src/tooling/node-convert-worker-pool.js +15 -0
  768. package/src/tooling/node-convert-worker.js +11 -1
  769. package/src/tooling/node-converter.js +102 -1
  770. package/src/tooling/node-source-runtime.js +1 -10
  771. package/src/tooling/node-webgpu.d.ts +1 -0
  772. package/src/tooling/node-webgpu.js +23 -11
  773. package/src/tooling/opfs-cache.js +1 -4
  774. package/src/tooling/precision-replay-math.js +174 -0
  775. package/src/tooling/program-bundle-host.d.ts +16 -0
  776. package/src/tooling/program-bundle-host.js +20 -0
  777. package/src/tooling/program-bundle-parity.d.ts +28 -0
  778. package/src/tooling/program-bundle-parity.js +210 -0
  779. package/src/tooling/program-bundle.d.ts +112 -0
  780. package/src/tooling/program-bundle.js +1119 -0
  781. package/src/tooling/rdrr-integrity-refresh.d.ts +50 -0
  782. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  783. package/src/tooling/reference-verify.js +191 -0
  784. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  785. package/src/tooling/source-artifact-adapter.js +10 -5
  786. package/src/tooling/source-package-profiles.d.ts +18 -1
  787. package/src/tooling/source-package-profiles.js +110 -8
  788. package/src/tooling/source-runtime-bundle.d.ts +3 -0
  789. package/src/tooling/source-runtime-bundle.js +30 -22
  790. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  791. package/src/tooling/source-runtime-converter-config.js +25 -12
  792. package/src/tooling/source-runtime-materializer.js +1 -7
  793. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  794. package/src/tooling-experimental-exports.shared.js +10 -0
  795. package/src/tooling-exports/device.d.ts +11 -0
  796. package/src/tooling-exports/device.js +14 -0
  797. package/src/tooling-exports/manifest.d.ts +8 -0
  798. package/src/tooling-exports/manifest.js +11 -0
  799. package/src/tooling-exports/storage.d.ts +25 -0
  800. package/src/tooling-exports/storage.js +29 -0
  801. package/src/tooling-exports/structured.d.ts +1 -0
  802. package/src/tooling-exports/structured.js +5 -0
  803. package/src/tooling-exports.d.ts +23 -0
  804. package/src/tooling-exports.js +17 -0
  805. package/src/tooling-exports.shared.d.ts +27 -30
  806. package/src/tooling-exports.shared.js +28 -29
  807. package/src/types/index.d.ts +0 -3
  808. package/src/types/index.js +0 -3
  809. package/src/utils/array-buffer.d.ts +4 -0
  810. package/src/utils/array-buffer.js +9 -0
  811. package/src/utils/canonical-hash.d.ts +7 -0
  812. package/src/utils/canonical-hash.js +74 -0
  813. package/src/utils/clone-json.d.ts +1 -0
  814. package/src/utils/clone-json.js +7 -0
  815. package/src/utils/encode-utf8.js +5 -0
  816. package/src/utils/index.d.ts +4 -1
  817. package/src/utils/index.js +4 -1
  818. package/src/utils/runtime-env.js +7 -0
  819. package/src/utils/sha256.d.ts +1 -0
  820. package/src/utils/sha256.js +18 -4
  821. package/src/utils/stable-sort-object.d.ts +1 -0
  822. package/src/utils/stable-sort-object.js +13 -0
  823. package/src/version.js +1 -1
  824. package/tools/convert-safetensors-node.js +1 -2
  825. package/src/bootstrap.d.ts +0 -1
  826. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  827. package/src/config/kernels/kernel-ref.d.ts +0 -17
  828. package/src/config/kernels/kernel-ref.js +0 -75
  829. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  830. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  831. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  832. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  833. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  834. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  835. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  836. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  837. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  838. package/src/config/runtime/overlays/trace-config.json +0 -28
  839. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  840. package/src/debug/perf.d.ts +0 -33
  841. package/src/debug/perf.js +0 -51
  842. package/src/errors/index.d.ts +0 -1
  843. package/src/errors/index.js +0 -1
  844. package/src/formats/gguf/index.d.ts +0 -8
  845. package/src/formats/gguf/index.js +0 -4
  846. package/src/formats/litert/index.d.ts +0 -1
  847. package/src/formats/litert/index.js +0 -1
  848. package/src/formats/safetensors/index.d.ts +0 -8
  849. package/src/formats/safetensors/index.js +0 -4
  850. package/src/formats/tflite/index.d.ts +0 -1
  851. package/src/formats/tflite/index.js +0 -1
  852. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  853. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  854. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  855. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  856. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  857. package/src/inference/functiongemma.d.ts +0 -15
  858. package/src/inference/functiongemma.js +0 -1
  859. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  860. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  861. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  862. package/src/rules/kernels/kernel-validator.js +0 -160
  863. package/src/sw.d.ts +0 -1
  864. package/src/tooling/bench-runner.js +0 -64
  865. package/src/tooling/distill-runner.js +0 -56
  866. package/src/tooling/lora-runner.js +0 -37
  867. package/src/tooling/verify-runner.js +0 -63
  868. package/src/types/gpu.d.ts +0 -185
  869. package/src/types/gpu.js +0 -5
  870. package/src/types/inference.d.ts +0 -197
  871. package/src/types/inference.js +0 -5
  872. package/src/types/model.d.ts +0 -130
  873. package/src/types/model.js +0 -5
  874. package/src/utils/format-bytes.js +0 -20
  875. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  876. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  877. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  878. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  879. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  880. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -9,14 +9,16 @@ import { f16ToF32, convertBF16ToF32GPU, shouldDequantizeToF16, applyBufferLayout
9
9
  import { QK_K, Q4K_BLOCK_BYTES, Q6K_BLOCK_BYTES } from '../quantization-constants.js';
10
10
  import { log, trace as debugTrace } from '../../debug/index.js';
11
11
  import { selectRuleValue } from '../../rules/rule-registry.js';
12
- import { dequantizeQ4KM, dequantizeQ4KMRowWise } from '../../converter/quantizer.js';
12
+ import { dequantizeQ4KM, dequantizeQ4KMRowWise, float32ToFloat16 } from '../../converter/quantizer.js';
13
13
  import { hasSourceTransform } from './source-transform.js';
14
+ import { loadFunctionalDescriptor } from './functional-descriptor-loader.js';
14
15
 
15
16
  // ============================================================================
16
17
  // Q4K Detection
17
18
  // ============================================================================
18
19
 
19
20
  let loggedF32UpcastNonMatmul = false;
21
+ let loggedQ4KLimitFallback = false;
20
22
 
21
23
  function isGpuBufferInstance(value) {
22
24
  return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
@@ -76,6 +78,28 @@ function toUint8View(data) {
76
78
  return new Uint8Array(data);
77
79
  }
78
80
 
81
+ function toUint16View(data, label) {
82
+ const bytes = toUint8View(data);
83
+ if (bytes.byteLength % 2 !== 0) {
84
+ throw new Error(`${label}: byte length must be divisible by 2.`);
85
+ }
86
+ if (bytes.byteOffset % 2 === 0) {
87
+ return new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
88
+ }
89
+ return new Uint16Array(bytes.slice().buffer);
90
+ }
91
+
92
+ function toFloat32View(data, label) {
93
+ const bytes = toUint8View(data);
94
+ if (bytes.byteLength % 4 !== 0) {
95
+ throw new Error(`${label}: byte length must be divisible by 4.`);
96
+ }
97
+ if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
98
+ return new Float32Array(bytes.buffer);
99
+ }
100
+ return new Float32Array(bytes.slice().buffer);
101
+ }
102
+
79
103
  function resolveInputByteLength(data, fallbackSize) {
80
104
  if (data instanceof Uint8Array) return data.byteLength;
81
105
  if (ArrayBuffer.isView(data)) return data.byteLength;
@@ -107,12 +131,203 @@ function getShapeElementCount(shape) {
107
131
  return shape.reduce((product, value) => product * value, 1);
108
132
  }
109
133
 
134
+ function getStorageCompanion(shardData, location, name, role) {
135
+ const companion = shardData?.storageCompanions?.[role];
136
+ if (!companion || !(companion.bytes instanceof Uint8Array)) {
137
+ throw new Error(
138
+ `W4A16 tensor "${name}" is missing required storage companion "${role}".`
139
+ );
140
+ }
141
+ const declared = Array.isArray(location?.storage?.companions)
142
+ ? location.storage.companions.find((entry) => entry.role === role)
143
+ : null;
144
+ if (declared && companion.tensorId !== declared.tensorId) {
145
+ throw new Error(
146
+ `W4A16 tensor "${name}" companion "${role}" resolved to "${companion.tensorId}", expected "${declared.tensorId}".`
147
+ );
148
+ }
149
+ return companion;
150
+ }
151
+
152
+ function readW4A16LogicalShape(companion, fallbackShape, name) {
153
+ const location = companion.location ?? null;
154
+ const bytes = companion.bytes;
155
+ const dtype = String(location?.dtype || '').toUpperCase();
156
+ if (!Array.isArray(location?.shape) || location.shape.length !== 1 || location.shape[0] !== 2) {
157
+ throw new Error(`W4A16 tensor "${name}" shape companion must have shape [2].`);
158
+ }
159
+ if (dtype === 'I64') {
160
+ if (bytes.byteLength !== 16) {
161
+ throw new Error(`W4A16 tensor "${name}" I64 shape companion must be 16 bytes.`);
162
+ }
163
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
164
+ return [Number(view.getBigInt64(0, true)), Number(view.getBigInt64(8, true))];
165
+ }
166
+ if (dtype === 'I32' || dtype === 'U32') {
167
+ if (bytes.byteLength !== 8) {
168
+ throw new Error(`W4A16 tensor "${name}" ${dtype} shape companion must be 8 bytes.`);
169
+ }
170
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
171
+ const read = dtype === 'I32'
172
+ ? (offset) => view.getInt32(offset, true)
173
+ : (offset) => view.getUint32(offset, true);
174
+ return [read(0), read(4)];
175
+ }
176
+ if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
177
+ return fallbackShape;
178
+ }
179
+ throw new Error(`W4A16 tensor "${name}" has unsupported shape companion dtype "${location?.dtype}".`);
180
+ }
181
+
182
+ function assertW4A16Shape(shape, fallbackShape, name) {
183
+ if (!Array.isArray(shape) || shape.length !== 2) {
184
+ throw new Error(`W4A16 tensor "${name}" logical shape must be 2D.`);
185
+ }
186
+ const rows = Number(shape[0]);
187
+ const cols = Number(shape[1]);
188
+ if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
189
+ throw new Error(`W4A16 tensor "${name}" has invalid logical shape ${JSON.stringify(shape)}.`);
190
+ }
191
+ if (Array.isArray(fallbackShape) && fallbackShape.length === 2) {
192
+ if (rows !== fallbackShape[0] || cols !== fallbackShape[1]) {
193
+ throw new Error(
194
+ `W4A16 tensor "${name}" shape companion [${rows},${cols}] does not match manifest shape [${fallbackShape.join(',')}].`
195
+ );
196
+ }
197
+ }
198
+ return [rows, cols];
199
+ }
200
+
201
+ function bf16ToF32(bits) {
202
+ const floats = new Float32Array(1);
203
+ const uints = new Uint32Array(floats.buffer);
204
+ uints[0] = (bits & 0xffff) << 16;
205
+ return floats[0];
206
+ }
207
+
208
+ function readOffsetBinaryInt4(byte, highNibble) {
209
+ const value = highNibble ? ((byte >> 4) & 0x0f) : (byte & 0x0f);
210
+ return value - 8;
211
+ }
212
+
213
+ function readW4A16Scales(companion, expectedScales, name) {
214
+ const dtype = String(companion.location?.dtype || '').toUpperCase();
215
+ if (dtype === 'F16') {
216
+ const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
217
+ if (packed.length !== expectedScales) {
218
+ throw new Error(
219
+ `W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
220
+ );
221
+ }
222
+ const scales = new Float32Array(packed.length);
223
+ for (let i = 0; i < packed.length; i += 1) {
224
+ scales[i] = f16ToF32(packed[i]);
225
+ }
226
+ return scales;
227
+ }
228
+ if (dtype === 'BF16') {
229
+ const packed = toUint16View(companion.bytes, `W4A16 scales for ${name}`);
230
+ if (packed.length !== expectedScales) {
231
+ throw new Error(
232
+ `W4A16 tensor "${name}" scale count ${packed.length} does not match expected ${expectedScales}.`
233
+ );
234
+ }
235
+ const scales = new Float32Array(packed.length);
236
+ for (let i = 0; i < packed.length; i += 1) {
237
+ scales[i] = bf16ToF32(packed[i]);
238
+ }
239
+ return scales;
240
+ }
241
+ if (dtype === 'F32') {
242
+ const scales = toFloat32View(companion.bytes, `W4A16 scales for ${name}`);
243
+ if (scales.length !== expectedScales) {
244
+ throw new Error(
245
+ `W4A16 tensor "${name}" scale count ${scales.length} does not match expected ${expectedScales}.`
246
+ );
247
+ }
248
+ return scales;
249
+ }
250
+ throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${companion.location?.dtype}".`);
251
+ }
252
+
253
+ function validateW4A16PackedStorage(shardData, location, name) {
254
+ const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
255
+ const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
256
+ const [rows, cols] = assertW4A16Shape(
257
+ readW4A16LogicalShape(shapeCompanion, location.shape, name),
258
+ location.shape,
259
+ name
260
+ );
261
+ const groupsPerRow = Math.ceil(cols / 32);
262
+ const expectedPackedBytes = rows * groupsPerRow * 16;
263
+ if (shardData.byteLength !== expectedPackedBytes) {
264
+ throw new Error(
265
+ `W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
266
+ );
267
+ }
268
+ const scaleDtype = String(scaleCompanion.location?.dtype || '').toUpperCase();
269
+ const scaleBytesPerElement = scaleDtype === 'F32'
270
+ ? 4
271
+ : (scaleDtype === 'F16' || scaleDtype === 'BF16' ? 2 : null);
272
+ if (scaleBytesPerElement == null) {
273
+ throw new Error(`W4A16 tensor "${name}" has unsupported scale companion dtype "${scaleCompanion.location?.dtype}".`);
274
+ }
275
+ const expectedScaleBytes = rows * groupsPerRow * scaleBytesPerElement;
276
+ if (scaleCompanion.bytes.byteLength !== expectedScaleBytes) {
277
+ throw new Error(
278
+ `W4A16 tensor "${name}" scale byte length ${scaleCompanion.bytes.byteLength} does not match expected ${expectedScaleBytes}.`
279
+ );
280
+ }
281
+ return {
282
+ rows,
283
+ cols,
284
+ groupsPerRow,
285
+ scaleDtype: scaleDtype.toLowerCase(),
286
+ scaleBytes: scaleCompanion.bytes,
287
+ };
288
+ }
289
+
290
+ function dequantizeW4A16ToF16(shardData, location, name) {
291
+ const scaleCompanion = getStorageCompanion(shardData, location, name, 'scales');
292
+ const shapeCompanion = getStorageCompanion(shardData, location, name, 'shape');
293
+ const [rows, cols] = assertW4A16Shape(
294
+ readW4A16LogicalShape(shapeCompanion, location.shape, name),
295
+ location.shape,
296
+ name
297
+ );
298
+ const groupsPerRow = Math.ceil(cols / 32);
299
+ const expectedPackedBytes = rows * groupsPerRow * 16;
300
+ if (shardData.byteLength !== expectedPackedBytes) {
301
+ throw new Error(
302
+ `W4A16 tensor "${name}" packed byte length ${shardData.byteLength} does not match expected ${expectedPackedBytes}.`
303
+ );
304
+ }
305
+ const expectedScales = rows * groupsPerRow;
306
+ const scales = readW4A16Scales(scaleCompanion, expectedScales, name);
307
+ const out = new Uint16Array(rows * cols);
308
+ for (let row = 0; row < rows; row += 1) {
309
+ for (let group = 0; group < groupsPerRow; group += 1) {
310
+ const scale = scales[(row * groupsPerRow) + group];
311
+ const packedOffset = ((row * groupsPerRow) + group) * 16;
312
+ for (let lane = 0; lane < 32; lane += 1) {
313
+ const col = (group * 32) + lane;
314
+ if (col >= cols) break;
315
+ const byte = shardData[packedOffset + Math.floor(lane / 2)];
316
+ const quant = readOffsetBinaryInt4(byte, (lane % 2) === 1);
317
+ out[(row * cols) + col] = float32ToFloat16(quant * scale);
318
+ }
319
+ }
320
+ }
321
+ return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
322
+ }
323
+
110
324
 
111
325
  export function isPackedQ4K(location) {
112
- if (!Array.isArray(location.shape) || location.shape.length !== 2) {
326
+ if (!Array.isArray(location.shape) || location.shape.length < 2) {
113
327
  return false;
114
328
  }
115
- const [rows, cols] = location.shape;
329
+ const cols = location.shape[location.shape.length - 1];
330
+ const rows = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
116
331
  const expectedRowwise = rows * Math.ceil(cols / QK_K) * Q4K_BLOCK_BYTES;
117
332
  return location.size < expectedRowwise;
118
333
  }
@@ -128,7 +343,10 @@ function isEmbeddingRole(location) {
128
343
 
129
344
  export function shouldUseFusedQ4K(location, config) {
130
345
  if (!config.useFusedQ4K) return false;
346
+ return canUseFusedQ4KStorage(location, config);
347
+ }
131
348
 
349
+ function canUseFusedQ4KStorage(location, config) {
132
350
  const caps = config.gpuCapabilities || getKernelCapabilities();
133
351
  if (!caps?.hasSubgroups) return false;
134
352
 
@@ -141,6 +359,125 @@ export function shouldUseFusedQ4K(location, config) {
141
359
  return true;
142
360
  }
143
361
 
362
+ export function isLiteRTAffineInt4FusedEligible(location, config) {
363
+ const caps = config?.gpuCapabilities || getKernelCapabilities();
364
+ if (caps?.hasF16 !== true) return false;
365
+
366
+ if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
367
+ if (!location?.role) return false;
368
+ if (isEmbeddingRole(location)) return false;
369
+ if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
370
+ if (!shouldDequantizeToF16(location)) return false;
371
+
372
+ const transform = location.sourceTransform;
373
+ if (!transform || typeof transform !== 'object') return false;
374
+ const sourceDtype = String(transform.sourceDtype || '').toUpperCase();
375
+ const targetDtype = String(transform.targetDtype || '').toUpperCase();
376
+ const locationDtype = String(location.dtype || '').toUpperCase();
377
+ const storageEncoding = String(transform.storageEncoding || '').toLowerCase();
378
+ const scale = Number(transform.scale);
379
+ const zeroPoint = Number(transform.zeroPoint);
380
+ const storageEncodingSupported = storageEncoding === 'signed' || storageEncoding === 'offset_binary';
381
+
382
+ return transform.kind === 'affine_dequant'
383
+ && transform.scheme === 'per_tensor_affine'
384
+ && sourceDtype === 'INT4'
385
+ && targetDtype === 'F16'
386
+ && locationDtype === 'F16'
387
+ && storageEncodingSupported
388
+ && Number.isFinite(scale)
389
+ && Math.abs(scale - 0.0625) <= Number.EPSILON
390
+ && Number.isSafeInteger(zeroPoint)
391
+ && zeroPoint === 0;
392
+ }
393
+
394
+ export function isW4A16FusedEligible(location, config) {
395
+ const caps = config?.gpuCapabilities || getKernelCapabilities();
396
+ if (caps?.hasF16 !== true) return false;
397
+
398
+ if (!Array.isArray(location?.shape) || location.shape.length !== 2) return false;
399
+ if (String(location?.dtype || '').toUpperCase() !== 'W4A16') return false;
400
+ if (!location?.role) return false;
401
+ if (isEmbeddingRole(location)) return false;
402
+ if (location.role !== 'matmul' && location.role !== 'lm_head') return false;
403
+ if (!shouldDequantizeToF16(location)) return false;
404
+ if (location?.storage?.packing !== 'w4a16') return false;
405
+ if (!Array.isArray(location?.storage?.companions)) return false;
406
+
407
+ const hasScales = location.storage.companions.some((entry) => entry.role === 'scales');
408
+ const hasShape = location.storage.companions.some((entry) => entry.role === 'shape');
409
+ return hasScales && hasShape;
410
+ }
411
+
412
+ function getQ4KDenseMaterializedSizeBytes(location, config) {
413
+ if (!Array.isArray(location.shape) || location.shape.length === 0) {
414
+ return null;
415
+ }
416
+ const elementCount = getShapeElementCount(location.shape);
417
+ if (!Number.isFinite(elementCount) || elementCount <= 0) {
418
+ return null;
419
+ }
420
+ const outputDtype = getQ4KOutputDtype(location, config);
421
+ const bytesPerElement = outputDtype === 'f16' ? 2 : 4;
422
+ return elementCount * bytesPerElement;
423
+ }
424
+
425
+ function getMaxStorageBufferBindingSize() {
426
+ const device = getDevice();
427
+ const maxStorage = device?.limits?.maxStorageBufferBindingSize;
428
+ return Number.isFinite(maxStorage) && maxStorage > 0 ? maxStorage : null;
429
+ }
430
+
431
+ function resolveQ4KLimitFallback(location, config) {
432
+ if (location?.dtype !== 'Q4_K_M' && location?.dtype !== 'Q4_K') {
433
+ return {
434
+ denseExceedsBindingLimit: false,
435
+ limitFallbackEligible: false,
436
+ denseSizeBytes: null,
437
+ maxBindingSizeBytes: null,
438
+ };
439
+ }
440
+
441
+ const denseSizeBytes = getQ4KDenseMaterializedSizeBytes(location, config);
442
+ const maxBindingSizeBytes = getMaxStorageBufferBindingSize();
443
+ const denseExceedsBindingLimit = (
444
+ denseSizeBytes != null
445
+ && maxBindingSizeBytes != null
446
+ && denseSizeBytes > maxBindingSizeBytes
447
+ );
448
+ const packedSizeBytes = Number.isFinite(location.size) ? location.size : null;
449
+ const packedFitsBindingLimit = (
450
+ packedSizeBytes != null
451
+ && maxBindingSizeBytes != null
452
+ && packedSizeBytes <= maxBindingSizeBytes
453
+ );
454
+ const limitFallbackEligible = (
455
+ denseExceedsBindingLimit
456
+ && config.keepF32Weights !== true
457
+ && packedFitsBindingLimit
458
+ && canUseFusedQ4KStorage(location, config)
459
+ );
460
+
461
+ return {
462
+ denseExceedsBindingLimit,
463
+ limitFallbackEligible,
464
+ denseSizeBytes,
465
+ maxBindingSizeBytes,
466
+ };
467
+ }
468
+
469
+ function logQ4KLimitFallbackOnce(name, fallback) {
470
+ if (loggedQ4KLimitFallback) {
471
+ return;
472
+ }
473
+ loggedQ4KLimitFallback = true;
474
+ log.warn(
475
+ 'Loader',
476
+ `Q4K dense materialization for "${name}" would require ${fallback.denseSizeBytes} bytes, ` +
477
+ `exceeding maxStorageBufferBindingSize=${fallback.maxBindingSizeBytes}; retaining packed Q4K for fused matmul.`
478
+ );
479
+ }
480
+
144
481
  // ============================================================================
145
482
  // Dtype Output Selection
146
483
  // ============================================================================
@@ -220,6 +557,80 @@ export async function loadQ4KFused(shardData, location, name) {
220
557
  }
221
558
  }
222
559
 
560
+ export async function loadLiteRTInt4Fused(shardData, location, name, config = null) {
561
+ if (isGpuBufferInstance(shardData)) {
562
+ throw new Error(
563
+ `LiteRT INT4 tensor "${name}" requires raw packed source bytes before GPU upload.`
564
+ );
565
+ }
566
+ if (!isLiteRTAffineInt4FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
567
+ throw new Error(
568
+ `LiteRT INT4 tensor "${name}" does not match the fused fixed-affine contract ` +
569
+ '(INT4 -> F16, storageEncoding=signed|offset_binary, per_tensor_affine, scale=0.0625, zeroPoint=0, 2D matmul/lm_head role).'
570
+ );
571
+ }
572
+
573
+ const [rows, cols] = location.shape;
574
+ const expectedBytes = rows * Math.ceil(cols / 2);
575
+ const actualBytes = resolveInputByteLength(shardData, location.size);
576
+ if (actualBytes !== expectedBytes) {
577
+ throw new Error(
578
+ `LiteRT INT4 tensor "${name}" packed byte size mismatch. ` +
579
+ `Expected ${expectedBytes} bytes for shape [${rows},${cols}], got ${actualBytes}.`
580
+ );
581
+ }
582
+
583
+ const device = getDevice();
584
+ const buffer = acquireAlignedBuffer(actualBytes, `litert_int4_${name}`);
585
+ try {
586
+ writeBufferAligned(device, buffer, shardData);
587
+ return {
588
+ data: createWeightBuffer(buffer, 'litert_int4', 'row', location.shape, name, null, {
589
+ storageEncoding: String(location.sourceTransform.storageEncoding).toLowerCase(),
590
+ }),
591
+ allocatedBuffers: [buffer],
592
+ };
593
+ } catch (error) {
594
+ releaseBuffer(buffer);
595
+ throw error;
596
+ }
597
+ }
598
+
599
+ export async function loadW4A16Fused(shardData, location, name, config = null) {
600
+ if (isGpuBufferInstance(shardData)) {
601
+ throw new Error(
602
+ `W4A16 tensor "${name}" requires raw packed source bytes before GPU upload.`
603
+ );
604
+ }
605
+ if (!isW4A16FusedEligible(location, config ?? { gpuCapabilities: getKernelCapabilities() })) {
606
+ throw new Error(
607
+ `W4A16 tensor "${name}" does not match the fused packed contract ` +
608
+ '(dtype=W4A16, storage.packing=w4a16, scales+shape companions, 2D matmul/lm_head role).'
609
+ );
610
+ }
611
+
612
+ const storage = validateW4A16PackedStorage(shardData, location, name);
613
+ const device = getDevice();
614
+ const weightBuffer = acquireAlignedBuffer(shardData.byteLength, `w4a16_${name}`);
615
+ const scaleBuffer = acquireAlignedBuffer(storage.scaleBytes.byteLength, `w4a16_scales_${name}`);
616
+ try {
617
+ writeBufferAligned(device, weightBuffer, shardData);
618
+ writeBufferAligned(device, scaleBuffer, storage.scaleBytes);
619
+ return {
620
+ data: createWeightBuffer(weightBuffer, 'w4a16', 'row', location.shape, name, null, {
621
+ scaleBuffer,
622
+ scaleDtype: storage.scaleDtype,
623
+ groupsPerRow: storage.groupsPerRow,
624
+ }),
625
+ allocatedBuffers: [weightBuffer, scaleBuffer],
626
+ };
627
+ } catch (error) {
628
+ releaseBuffer(weightBuffer);
629
+ releaseBuffer(scaleBuffer);
630
+ throw error;
631
+ }
632
+ }
633
+
223
634
  async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name, config) {
224
635
  let dequantized = null;
225
636
  const outputDtype = getQ4KOutputDtype(location, config);
@@ -268,16 +679,23 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
268
679
  }
269
680
 
270
681
  let numBlocks = null;
682
+ let rowCount = null;
683
+ let blocksPerRow = null;
684
+ let denseElementCount = null;
271
685
  let dequantizedTensor;
272
686
  if (needsRowwise) {
273
- const rows = location.shape[0];
687
+ rowCount = location.shape.slice(0, -1).reduce((a, b) => a * b, 1);
688
+ blocksPerRow = Math.ceil(K / QK_K);
689
+ numBlocks = rowCount * blocksPerRow;
690
+ denseElementCount = rowCount * K;
274
691
  debugTrace.loader(
275
- `Dequantizing ${name} (row-wise): [${rows},${K}], K not 256-aligned, ` +
692
+ `Dequantizing ${name} (row-wise): [${rowCount},${K}], K not 256-aligned, ` +
276
693
  `outputDtype=${outputDtype}`
277
694
  );
278
- dequantizedTensor = await dequantizeRowwise(quantBuffer, rows, K, { outputDtype });
695
+ dequantizedTensor = await dequantizeRowwise(quantBuffer, rowCount, K, { outputDtype });
279
696
  } else {
280
697
  numBlocks = Math.ceil(location.size / Q4K_BLOCK_BYTES);
698
+ denseElementCount = numBlocks * QK_K;
281
699
  debugTrace.loader(
282
700
  `Dequantizing ${name}: size=${location.size}, numBlocks=${numBlocks}, ` +
283
701
  `outputDtype=${outputDtype}, expectedOutput=${numBlocks * QK_K * (outputDtype === 'f16' ? 2 : 4)}`
@@ -296,9 +714,9 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
296
714
  if (isProbeTarget) {
297
715
  try {
298
716
  const bytesPerElem = outputDtype === 'f16' ? 2 : 4;
299
- const requestedOutputBytes = numBlocks * QK_K * bytesPerElem;
717
+ const requestedOutputBytes = denseElementCount * bytesPerElem;
300
718
  const sampleCount = paritySamples;
301
- const readSize = Math.min(sampleCount * bytesPerElem, dequantized.size);
719
+ const readSize = Math.min(sampleCount * bytesPerElem, requestedOutputBytes, dequantized.size);
302
720
  const gpuRaw = await readBuffer(dequantized, readSize);
303
721
  const gpuBytes = gpuRaw instanceof ArrayBuffer
304
722
  ? new Uint8Array(gpuRaw)
@@ -334,7 +752,8 @@ async function materializeQ4KDenseBuffer(quantBuffer, shardData, location, name,
334
752
 
335
753
  log.warn('DequantProbe',
336
754
  `tensor="${name}" shape=[${location.shape}] ` +
337
- `location.size=${location.size} numBlocks=${numBlocks} outputDtype=${outputDtype} ` +
755
+ `location.size=${location.size} numBlocks=${numBlocks} blocksPerRow=${blocksPerRow} ` +
756
+ `rows=${rowCount} K=${K} denseElements=${denseElementCount} outputDtype=${outputDtype} ` +
338
757
  `bytesPerElem=${bytesPerElem} requestedOutputBytes=${requestedOutputBytes} bufSize=${dequantized.size} ` +
339
758
  `runParity=true sampleCount=${sampleCount}`
340
759
  );
@@ -389,7 +808,7 @@ export async function loadQ4KDequant(shardData, location, name, config) {
389
808
  }
390
809
  }
391
810
 
392
- export async function loadQ4KMixed(shardData, location, name, config) {
811
+ async function loadQ4KMixed(shardData, location, name, config) {
393
812
  const canMaterializeMixed = shouldUseFusedQ4K(location, config)
394
813
  && config.q4kLayout === 'row';
395
814
  if (!canMaterializeMixed) {
@@ -436,10 +855,10 @@ export async function loadQ4KMixed(shardData, location, name, config) {
436
855
 
437
856
  function getQ4KCpuReferenceContext(shardData, location, config) {
438
857
  const outputDtype = getQ4KOutputDtype(location, config);
439
- const is2DMatrix = Array.isArray(location.shape) && location.shape.length === 2;
440
- const K = is2DMatrix ? location.shape[1] : 0;
441
- const needsRowwise = is2DMatrix && K > 0 && K % QK_K !== 0;
858
+ const isMatrixLike = Array.isArray(location.shape) && location.shape.length >= 2;
859
+ const K = isMatrixLike ? location.shape[location.shape.length - 1] : 0;
442
860
  const layout = getWeightLayout(location, config);
861
+ const needsRowwise = isMatrixLike && layout === 'row' && K > 0 && K % QK_K !== 0;
443
862
  const eligible = outputDtype === 'f32'
444
863
  && !isGpuBufferInstance(shardData)
445
864
  && (!needsRowwise || layout === 'row');
@@ -655,12 +1074,58 @@ export async function loadFloat(shardData, location, name, config) {
655
1074
  }
656
1075
  }
657
1076
 
1077
+ export async function loadW4A16Dequant(shardData, location, name, config) {
1078
+ if (!config) {
1079
+ throw new Error('Tensor load config is required.');
1080
+ }
1081
+ if (isGpuBufferInstance(shardData)) {
1082
+ throw new Error(
1083
+ `W4A16 tensor "${name}" requires CPU-side storage companion materialization before GPU upload.`
1084
+ );
1085
+ }
1086
+ const f16Bytes = dequantizeW4A16ToF16(shardData, location, name);
1087
+ const device = getDevice();
1088
+ const buffer = acquireAlignedBuffer(f16Bytes.byteLength, `w4a16_dequant_${name}`);
1089
+ try {
1090
+ writeBufferAligned(device, buffer, f16Bytes);
1091
+ const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
1092
+ layout: location.layout ?? null,
1093
+ useColumnWise: false,
1094
+ });
1095
+ if (shouldDequantizeToF16(location)) {
1096
+ return {
1097
+ data: createWeightBuffer(buffer, 'f16', layout, location.shape, name),
1098
+ allocatedBuffers: [buffer],
1099
+ };
1100
+ }
1101
+ return {
1102
+ data: applyBufferLayout(buffer, location, 'f16'),
1103
+ allocatedBuffers: [buffer],
1104
+ };
1105
+ } catch (error) {
1106
+ releaseBuffer(buffer);
1107
+ throw error;
1108
+ }
1109
+ }
1110
+
658
1111
  // ============================================================================
659
1112
  // Main GPU Loading Entry Point
660
1113
  // ============================================================================
661
1114
 
662
1115
 
663
1116
  const GPU_LOADER_DISPATCH = {
1117
+ functional_descriptor: (shardData, location, name, config) => {
1118
+ debugTrace.loader(`Loading functional descriptor: ${name}`);
1119
+ return loadFunctionalDescriptor(shardData, location, name, config);
1120
+ },
1121
+ litert_int4_fused: (shardData, location, name, config) => {
1122
+ debugTrace.loader(`Loading LiteRT INT4 weight (fused): ${name} (size=${location.size})`);
1123
+ return loadLiteRTInt4Fused(shardData, location, name, config);
1124
+ },
1125
+ w4a16_fused: (shardData, location, name, config) => {
1126
+ debugTrace.loader(`Loading W4A16 weight (fused): ${name} (size=${location.size})`);
1127
+ return loadW4A16Fused(shardData, location, name, config);
1128
+ },
664
1129
  q4k_mixed: (shardData, location, name, config) => loadQ4KMixed(shardData, location, name, config),
665
1130
  q4k_fused: (shardData, location, name, _config) => {
666
1131
  debugTrace.loader(`Loading Q4K weight (fused): ${name} (size=${location.size})`);
@@ -686,52 +1151,86 @@ const GPU_LOADER_DISPATCH = {
686
1151
  }
687
1152
  ),
688
1153
  q6k: (shardData, location, name, _config) => loadQ6K(shardData, location, name),
1154
+ w4a16_dequant_reference: (shardData, location, name, config) => loadW4A16Dequant(shardData, location, name, config),
689
1155
  bf16: (shardData, location, name, config) => loadBF16(shardData, location, name, config),
690
1156
  float: (shardData, location, name, config) => loadFloat(shardData, location, name, config),
1157
+ unsupported_packed_quantization: (_shardData, location, name, _config) => {
1158
+ throw new Error(
1159
+ `Unsupported packed quantization dtype "${location.dtype}" for tensor "${name}". ` +
1160
+ 'Add a native loader and kernel path before enabling runtime execution.'
1161
+ );
1162
+ },
691
1163
  };
692
1164
 
693
1165
  export async function loadTensorToGPU(shardData, location, name, config) {
694
1166
  const dtype = location.dtype;
695
1167
  const useFusedQ4K = shouldUseFusedQ4K(location, config);
1168
+ const requiresFusedQ4KRole = Array.isArray(config?.q4kFusedRoles)
1169
+ && config.q4kFusedRoles.includes(location.role);
696
1170
  const caps = config?.gpuCapabilities || getKernelCapabilities();
697
1171
  const platformId = getPlatformConfig()?.platform?.id ?? null;
698
1172
  const q4kReferenceContext = getQ4KCpuReferenceContext(shardData, location, config);
699
1173
  const q4kBasicBackendClass = platformId === 'basic'
700
1174
  || (caps?.hasSubgroups !== true && caps?.hasF16 !== true);
1175
+ const q4kLimitFallback = resolveQ4KLimitFallback(location, config);
1176
+ const litertAffineInt4FusedEligible = isLiteRTAffineInt4FusedEligible(location, { ...config, gpuCapabilities: caps });
1177
+ const w4a16FusedEligible = isW4A16FusedEligible(location, { ...config, gpuCapabilities: caps });
701
1178
  const loaderPath = selectRuleValue('loader', 'tensorLoader', 'gpuLoaderPath', {
702
1179
  dtype,
1180
+ role: location.role ?? null,
1181
+ litertAffineInt4FusedEligible,
1182
+ w4a16FusedEligible,
703
1183
  useFusedQ4K,
1184
+ requiresFusedQ4KRole,
704
1185
  q4kMaterializationMode: config.q4kMaterializationMode ?? 'dense',
705
1186
  q4kCpuReferenceEligible: q4kReferenceContext.eligible,
706
1187
  q4kBasicBackendClass,
1188
+ q4kDenseExceedsBindingLimit: q4kLimitFallback.denseExceedsBindingLimit,
1189
+ q4kLimitFallbackEligible: q4kLimitFallback.limitFallbackEligible,
707
1190
  });
708
1191
  const loader = GPU_LOADER_DISPATCH[loaderPath];
709
1192
  if (!loader) {
710
1193
  throw new Error(`Unknown GPU loader path: "${loaderPath}" for dtype "${dtype}"`);
711
1194
  }
1195
+ if (loaderPath === 'q4k_fused' && q4kLimitFallback.limitFallbackEligible) {
1196
+ logQ4KLimitFallbackOnce(name, q4kLimitFallback);
1197
+ }
712
1198
  return loader(shardData, location, name, config);
713
1199
  }
714
1200
 
715
1201
 
716
1202
  const CPU_LOADER_DISPATCH = {
1203
+ unsupported_functional_descriptor: (_shardData, _location, name) => {
1204
+ throw new Error(
1205
+ `FUNCTIONAL_DESCRIPTOR tensor "${name ?? 'unknown'}" requires GPU materialization through loadTensorToGPU.`
1206
+ );
1207
+ },
717
1208
  raw: (shardData, _location) => shardData,
718
- bf16_to_f32: (shardData, _location) => {
719
- const bf16 = new Uint16Array(shardData.slice().buffer);
720
- return convertBF16ToF32CPU(bf16);
1209
+ w4a16_dequant_reference: (shardData, location) => {
1210
+ const f16Bytes = dequantizeW4A16ToF16(shardData, location, 'cpu');
1211
+ return convertF16ToF32CPU(toUint16View(f16Bytes, 'W4A16 CPU dequantized tensor load'));
721
1212
  },
722
- f16_to_f32: (shardData, _location) => {
723
- const f16 = new Uint16Array(shardData.slice().buffer);
724
- return convertF16ToF32CPU(f16);
1213
+ unsupported_packed_quantization: (_shardData, location) => {
1214
+ throw new Error(
1215
+ `Unsupported packed quantization dtype "${location.dtype}" for CPU tensor load. ` +
1216
+ 'Add a native loader before enabling runtime execution.'
1217
+ );
725
1218
  },
726
- f32: (shardData, _location) => new Float32Array(shardData.slice().buffer),
1219
+ bf16_to_f32: (shardData, _location) => convertBF16ToF32CPU(
1220
+ toUint16View(shardData, 'BF16 CPU tensor load')
1221
+ ),
1222
+ f16_to_f32: (shardData, _location) => convertF16ToF32CPU(
1223
+ toUint16View(shardData, 'F16 CPU tensor load')
1224
+ ),
1225
+ f32: (shardData, _location) => toFloat32View(shardData, 'F32 CPU tensor load'),
727
1226
  };
728
1227
 
729
- export function loadTensorToCPU(shardData, location) {
1228
+ export function loadTensorToCPU(shardData, location, name = null) {
730
1229
  const dtype = location.dtype;
731
1230
  const loaderPath = selectRuleValue('loader', 'tensorLoader', 'cpuLoaderPath', { dtype });
732
1231
  const loader = CPU_LOADER_DISPATCH[loaderPath];
733
1232
  if (!loader) {
734
1233
  throw new Error(`Unknown CPU loader path: "${loaderPath}" for dtype "${dtype}"`);
735
1234
  }
736
- return loader(shardData, location);
1235
+ return loader(shardData, location, name);
737
1236
  }