doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -12,6 +12,7 @@ import {
12
12
  parseLiteRTTaskFromSource,
13
13
  } from '../formats/litert/types.js';
14
14
  import { resolveDirectSourcePackageProfile } from './source-package-profiles.js';
15
+ import { cloneJsonValue } from '../utils/clone-json.js';
15
16
 
16
17
  export const LITERT_PACKAGE_SOURCE_KIND_TASK = 'litert-task';
17
18
  export const LITERT_PACKAGE_SOURCE_KIND_LITERTLM = 'litertlm';
@@ -20,13 +21,6 @@ function normalizeText(value) {
20
21
  return String(value || '').trim();
21
22
  }
22
23
 
23
- function cloneJsonValue(value) {
24
- if (typeof structuredClone === 'function') {
25
- return structuredClone(value);
26
- }
27
- return JSON.parse(JSON.stringify(value));
28
- }
29
-
30
24
  function createVirtualFile(path, offset, size, kind, options = {}) {
31
25
  return {
32
26
  path,
@@ -78,6 +72,17 @@ function resolvePackageTokenizerConfig(sourceKind, runtimeProfile) {
78
72
  return null;
79
73
  }
80
74
 
75
+ function findTFLiteMetadataEntry(parsedTFLite, name) {
76
+ const target = normalizeText(name);
77
+ if (!target) {
78
+ return null;
79
+ }
80
+ const metadataEntries = Array.isArray(parsedTFLite?.metadataEntries)
81
+ ? parsedTFLite.metadataEntries
82
+ : [];
83
+ return metadataEntries.find((entry) => normalizeText(entry?.name) === target) ?? null;
84
+ }
85
+
81
86
  function computePackedByteSize(shape, sourceDtype, tensorName) {
82
87
  if (!Array.isArray(shape) || shape.length !== 2) {
83
88
  throw new Error(
@@ -107,6 +112,47 @@ function computePackedByteSize(shape, sourceDtype, tensorName) {
107
112
  }
108
113
 
109
114
  function resolveLiteRTScaleContract(sourceDtype, tensorName, options = {}) {
115
+ const explicitScaleSemantics = normalizeText(options.scaleSemantics).toLowerCase();
116
+ if (explicitScaleSemantics) {
117
+ if (explicitScaleSemantics === 'step') {
118
+ return {
119
+ scaleSemantics: 'step',
120
+ };
121
+ }
122
+ if (explicitScaleSemantics === 'qmax_abs') {
123
+ const explicitScaleDivisor = Number(options.scaleDivisor);
124
+ if (Number.isFinite(explicitScaleDivisor) && explicitScaleDivisor > 0) {
125
+ return {
126
+ scaleSemantics: 'qmax_abs',
127
+ scaleDivisor: explicitScaleDivisor,
128
+ };
129
+ }
130
+ if (sourceDtype === 'INT8' || sourceDtype === 'UINT8') {
131
+ return {
132
+ scaleSemantics: 'qmax_abs',
133
+ scaleDivisor: 128,
134
+ };
135
+ }
136
+ if (sourceDtype === 'INT4') {
137
+ return {
138
+ scaleSemantics: 'qmax_abs',
139
+ scaleDivisor: 8,
140
+ };
141
+ }
142
+ if (sourceDtype === 'INT2') {
143
+ return {
144
+ scaleSemantics: 'qmax_abs',
145
+ scaleDivisor: 2,
146
+ };
147
+ }
148
+ throw new Error(
149
+ `direct-source runtime: unsupported LiteRT scale contract source dtype "${sourceDtype}" for "${tensorName}".`
150
+ );
151
+ }
152
+ throw new Error(
153
+ `direct-source runtime: unsupported LiteRT scaleSemantics "${options.scaleSemantics}" for "${tensorName}".`
154
+ );
155
+ }
110
156
  if (options.hasSumCompanion === true) {
111
157
  return {
112
158
  scaleSemantics: 'step',
@@ -135,6 +181,70 @@ function resolveLiteRTScaleContract(sourceDtype, tensorName, options = {}) {
135
181
  );
136
182
  }
137
183
 
184
+ function computeBlockedAxisPackedByteSize(storageShape, storageBlockSize, sourceDtype, tensorName) {
185
+ if (!Array.isArray(storageShape) || storageShape.length !== 2) {
186
+ throw new Error(
187
+ `direct-source runtime: LiteRT tensor "${tensorName}" requires an explicit blocked storageShape=[rows, cols].`
188
+ );
189
+ }
190
+ const storageRows = Number(storageShape[0]);
191
+ const storageCols = Number(storageShape[1]);
192
+ const blockSize = Number(storageBlockSize);
193
+ if (
194
+ !Number.isInteger(storageRows)
195
+ || storageRows <= 0
196
+ || !Number.isInteger(storageCols)
197
+ || storageCols <= 0
198
+ || !Number.isInteger(blockSize)
199
+ || blockSize <= 0
200
+ ) {
201
+ throw new Error(
202
+ `direct-source runtime: LiteRT tensor "${tensorName}" has invalid blocked storage shape ` +
203
+ `${JSON.stringify({ storageShape, storageBlockSize })}.`
204
+ );
205
+ }
206
+ return computePackedByteSize([storageRows, storageCols * blockSize], sourceDtype, tensorName);
207
+ }
208
+
209
+ function inferLiteRTBlockedAxisLayout(
210
+ rawTensor,
211
+ storageShape,
212
+ storageBlockSize,
213
+ tensorName = rawTensor?.name ?? 'unknown',
214
+ options = {}
215
+ ) {
216
+ const dtypeId = Number(rawTensor?.dtypeId);
217
+ const candidates = [];
218
+ if (dtypeId === 17) {
219
+ candidates.push('INT4');
220
+ } else if (dtypeId === 9) {
221
+ candidates.push('INT8', 'INT4', 'INT2');
222
+ } else if (dtypeId === 3) {
223
+ candidates.push('UINT8', 'INT4', 'INT2');
224
+ } else {
225
+ throw new Error(
226
+ `direct-source runtime: unsupported LiteRT blocked tensor dtype for "${tensorName}" (dtypeId=${dtypeId}).`
227
+ );
228
+ }
229
+
230
+ for (const sourceDtype of candidates) {
231
+ if (rawTensor.size === computeBlockedAxisPackedByteSize(storageShape, storageBlockSize, sourceDtype, tensorName)) {
232
+ const preferSignedPacked = options.preferSignedPacked === true;
233
+ return {
234
+ sourceDtype,
235
+ storageEncoding: sourceDtype === 'INT8' || sourceDtype === 'UINT8'
236
+ ? 'signed'
237
+ : (preferSignedPacked ? 'signed' : 'offset_binary'),
238
+ };
239
+ }
240
+ }
241
+
242
+ throw new Error(
243
+ `direct-source runtime: LiteRT tensor "${tensorName}" size ${rawTensor?.size} does not match any supported ` +
244
+ `blocked packed layout for storage shape ${JSON.stringify(storageShape)} and blockSize=${storageBlockSize}.`
245
+ );
246
+ }
247
+
138
248
  export function inferLiteRTRowwiseLayout(
139
249
  rawTensor,
140
250
  expectedShape,
@@ -349,7 +459,8 @@ function createLiteRTAxisTensor(
349
459
  group = null,
350
460
  logicalShape = null,
351
461
  storageShape = null,
352
- quantAxis = 1
462
+ quantAxis = 1,
463
+ scaleContractOptions = null
353
464
  ) {
354
465
  if (!rawTensor || typeof rawTensor !== 'object') {
355
466
  throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
@@ -381,6 +492,38 @@ function createLiteRTAxisTensor(
381
492
  );
382
493
  }
383
494
 
495
+ const scaleTensorSourceTransform = scaleTensor.sourceTransform;
496
+ const scaleCompanionDtype = String(scaleTensor.sourceDtype || '').toUpperCase();
497
+ const hasUint8ScaleCompanion = scaleCompanionDtype === 'UINT8';
498
+
499
+ // When the scale companion is UINT8 without affine_dequant metadata, the
500
+ // UINT8 container holds packed F32 row-scales (4 bytes per F32 value).
501
+ // This is the MediaPipe symmetric quantization convention used by LiteRT-LM
502
+ // .task weight bags (e.g. Gemma 4 E2B per_layer_embeddings): weight bytes
503
+ // are semantically INT8 (signed, zero_point=0), scale = max(|row|) / 127.
504
+ // The companion bytes are native F32 — no affine dequant needed.
505
+ const isPackedF32ScaleCompanion = hasUint8ScaleCompanion
506
+ && (!scaleTensorSourceTransform || scaleTensorSourceTransform.kind !== 'affine_dequant');
507
+
508
+ const scaleCompanionDequant = hasUint8ScaleCompanion && !isPackedF32ScaleCompanion
509
+ ? {
510
+ scale: Number(scaleTensorSourceTransform?.scale),
511
+ zeroPoint: Number(scaleTensorSourceTransform?.zeroPoint),
512
+ }
513
+ : null;
514
+ if (hasUint8ScaleCompanion && !isPackedF32ScaleCompanion) {
515
+ if (!Number.isFinite(scaleCompanionDequant.scale) || scaleCompanionDequant.scale <= 0) {
516
+ throw new Error(
517
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid scale companion affine_dequant scale ${scaleTensorSourceTransform.scale}.`
518
+ );
519
+ }
520
+ if (!Number.isSafeInteger(scaleCompanionDequant.zeroPoint)) {
521
+ throw new Error(
522
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid scale companion affine_dequant zeroPoint ${scaleTensorSourceTransform.zeroPoint}.`
523
+ );
524
+ }
525
+ }
526
+
384
527
  const logicalRows = Number(logicalShape[0]);
385
528
  const logicalCols = Number(logicalShape[1]);
386
529
  const storageRows = Number(storageShape[0]);
@@ -406,6 +549,9 @@ function createLiteRTAxisTensor(
406
549
  });
407
550
  const scaleContract = resolveLiteRTScaleContract(layout.sourceDtype, canonicalName, {
408
551
  hasSumCompanion: Boolean(sumTensor && typeof sumTensor === 'object'),
552
+ ...(scaleContractOptions && typeof scaleContractOptions === 'object'
553
+ ? scaleContractOptions
554
+ : {}),
409
555
  });
410
556
  const expectedScaleCount = quantAxis === 0 ? storageCols : storageRows;
411
557
  const scaleCount = scaleTensor.size / 4;
@@ -443,6 +589,167 @@ function createLiteRTAxisTensor(
443
589
  sourceTransform: {
444
590
  kind: 'litert_axis_dequant',
445
591
  scheme: 'per_axis_affine',
592
+ sourceDtype: isPackedF32ScaleCompanion && layout.sourceDtype === 'UINT8'
593
+ ? 'INT8'
594
+ : layout.sourceDtype,
595
+ targetDtype: 'F16',
596
+ storageEncoding: layout.storageEncoding,
597
+ scaleSemantics: scaleContract.scaleSemantics,
598
+ scaleDivisor: scaleContract.scaleDivisor,
599
+ storageShape: [storageRows, storageCols],
600
+ quantAxis,
601
+ scaleSourcePath: sourcePath,
602
+ scaleOffset: scaleTensor.offset,
603
+ scaleSize: scaleTensor.size,
604
+ ...(hasUint8ScaleCompanion && !isPackedF32ScaleCompanion
605
+ ? {
606
+ scaleCompanionDtype,
607
+ scaleCompanionDequant,
608
+ }
609
+ : {}),
610
+ ...(sumTensor && typeof sumTensor === 'object'
611
+ ? {
612
+ sumSourcePath: sourcePath,
613
+ sumOffset: sumTensor.offset,
614
+ sumSize: sumTensor.size,
615
+ }
616
+ : {}),
617
+ },
618
+ };
619
+ }
620
+
621
+ function createLiteRTBlockedAxisTensor(
622
+ rawTensor,
623
+ scaleTensor,
624
+ sumTensor,
625
+ sourcePath,
626
+ canonicalName,
627
+ role,
628
+ group = null,
629
+ logicalShape = null,
630
+ storageShape = null,
631
+ quantAxis = 0,
632
+ storageBlockSize = 4,
633
+ storageLaneOrder = null,
634
+ scaleContractOptions = null
635
+ ) {
636
+ if (!rawTensor || typeof rawTensor !== 'object') {
637
+ throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
638
+ }
639
+ if (!scaleTensor || typeof scaleTensor !== 'object') {
640
+ throw new Error(
641
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" is missing scale companion ` +
642
+ `"${rawTensor.name}_quantized_scale".`
643
+ );
644
+ }
645
+ if (!Array.isArray(logicalShape) || logicalShape.length !== 2) {
646
+ throw new Error(
647
+ `direct-source runtime: LiteRT tensor "${canonicalName}" requires an explicit 2D logical shape.`
648
+ );
649
+ }
650
+ if (!Array.isArray(storageShape) || storageShape.length !== 2) {
651
+ throw new Error(
652
+ `direct-source runtime: LiteRT tensor "${canonicalName}" requires an explicit 2D blocked storage shape.`
653
+ );
654
+ }
655
+ if (quantAxis !== 0) {
656
+ throw new Error(
657
+ `direct-source runtime: LiteRT blocked tensor "${canonicalName}" only supports quantAxis=0.`
658
+ );
659
+ }
660
+ if (scaleTensor.size % 4 !== 0) {
661
+ throw new Error(
662
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid blocked scale size ${scaleTensor.size}.`
663
+ );
664
+ }
665
+
666
+ const logicalRows = Number(logicalShape[0]);
667
+ const logicalCols = Number(logicalShape[1]);
668
+ const storageRows = Number(storageShape[0]);
669
+ const storageCols = Number(storageShape[1]);
670
+ const blockSize = Number(storageBlockSize);
671
+ if (
672
+ !Number.isInteger(logicalRows)
673
+ || logicalRows <= 0
674
+ || !Number.isInteger(logicalCols)
675
+ || logicalCols <= 0
676
+ || !Number.isInteger(storageRows)
677
+ || storageRows <= 0
678
+ || !Number.isInteger(storageCols)
679
+ || storageCols <= 0
680
+ || !Number.isInteger(blockSize)
681
+ || blockSize <= 0
682
+ ) {
683
+ throw new Error(
684
+ `direct-source runtime: LiteRT tensor "${canonicalName}" has invalid logical/blocked storage shapes ` +
685
+ `${JSON.stringify({ logicalShape, storageShape, storageBlockSize })}.`
686
+ );
687
+ }
688
+ if (storageCols !== logicalRows || storageRows * blockSize !== logicalCols) {
689
+ throw new Error(
690
+ `direct-source runtime: LiteRT blocked tensor "${canonicalName}" expects storageShape ` +
691
+ `[${logicalCols / blockSize}, ${logicalRows}] for logical shape [${logicalRows}, ${logicalCols}] and blockSize=${blockSize}. ` +
692
+ `Got [${storageRows}, ${storageCols}].`
693
+ );
694
+ }
695
+
696
+ const resolvedLaneOrder = Array.isArray(storageLaneOrder) && storageLaneOrder.length > 0
697
+ ? storageLaneOrder.map((value) => Number(value))
698
+ : Array.from({ length: blockSize }, (_value, index) => index);
699
+ if (
700
+ resolvedLaneOrder.length !== blockSize
701
+ || resolvedLaneOrder.some((value) => !Number.isInteger(value) || value < 0 || value >= blockSize)
702
+ || new Set(resolvedLaneOrder).size !== blockSize
703
+ ) {
704
+ throw new Error(
705
+ `direct-source runtime: LiteRT blocked tensor "${canonicalName}" has invalid storageLaneOrder ${JSON.stringify(storageLaneOrder)}.`
706
+ );
707
+ }
708
+
709
+ const layout = inferLiteRTBlockedAxisLayout(rawTensor, storageShape, blockSize, canonicalName, {
710
+ preferSignedPacked: !(sumTensor && typeof sumTensor === 'object'),
711
+ });
712
+ const scaleContract = resolveLiteRTScaleContract(layout.sourceDtype, canonicalName, {
713
+ hasSumCompanion: Boolean(sumTensor && typeof sumTensor === 'object'),
714
+ ...(scaleContractOptions && typeof scaleContractOptions === 'object'
715
+ ? scaleContractOptions
716
+ : {}),
717
+ });
718
+ const scaleCount = scaleTensor.size / 4;
719
+ if (scaleCount !== logicalRows) {
720
+ throw new Error(
721
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" blocked scale count ${scaleCount} ` +
722
+ `does not match logical rows ${logicalRows}.`
723
+ );
724
+ }
725
+
726
+ if (sumTensor && typeof sumTensor === 'object') {
727
+ if (sumTensor.size % 4 !== 0) {
728
+ throw new Error(
729
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid blocked sum size ${sumTensor.size}.`
730
+ );
731
+ }
732
+ const sumCount = sumTensor.size / 4;
733
+ if (sumCount !== logicalRows) {
734
+ throw new Error(
735
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" blocked sum count ${sumCount} ` +
736
+ `does not match logical rows ${logicalRows}.`
737
+ );
738
+ }
739
+ }
740
+
741
+ return {
742
+ name: canonicalName,
743
+ shape: [logicalRows, logicalCols],
744
+ dtype: 'F16',
745
+ offset: rawTensor.offset,
746
+ size: rawTensor.size,
747
+ sourcePath,
748
+ role,
749
+ ...(group ? { group } : {}),
750
+ sourceTransform: {
751
+ kind: 'litert_axis_blocked_dequant',
752
+ scheme: 'per_axis_affine',
446
753
  sourceDtype: layout.sourceDtype,
447
754
  targetDtype: 'F16',
448
755
  storageEncoding: layout.storageEncoding,
@@ -450,6 +757,8 @@ function createLiteRTAxisTensor(
450
757
  scaleDivisor: scaleContract.scaleDivisor,
451
758
  storageShape: [storageRows, storageCols],
452
759
  quantAxis,
760
+ storageBlockSize: blockSize,
761
+ storageLaneOrder: resolvedLaneOrder,
453
762
  scaleSourcePath: sourcePath,
454
763
  scaleOffset: scaleTensor.offset,
455
764
  scaleSize: scaleTensor.size,
@@ -521,21 +830,32 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
521
830
  group,
522
831
  resolvedLogicalShape,
523
832
  resolvedStorageShape,
524
- resolvedQuantAxis
833
+ resolvedQuantAxis,
834
+ {
835
+ scaleSemantics: options.scaleSemantics || 'step',
836
+ }
525
837
  )
526
838
  );
527
839
  };
528
840
 
529
- addAxisQuantized(
530
- 'transformer.embedder.input_embedding.w',
531
- 'model.language_model.embed_tokens.weight',
532
- 'embedding',
533
- 'embed',
534
- [vocabSize, hiddenSize],
535
- {
536
- transposeStorage: true,
537
- quantAxis: 0,
538
- }
841
+ normalized.push(
842
+ createLiteRTBlockedAxisTensor(
843
+ rawByName.get('transformer.embedder.input_embedding.w') ?? null,
844
+ rawByName.get('transformer.embedder.input_embedding.w_quantized_scale') ?? null,
845
+ rawByName.get('transformer.embedder.input_embedding.w.sum_i') ?? null,
846
+ sourcePath,
847
+ 'model.language_model.embed_tokens.weight',
848
+ 'embedding',
849
+ 'embed',
850
+ [vocabSize, hiddenSize],
851
+ [hiddenSize / 4, vocabSize],
852
+ 0,
853
+ 4,
854
+ [0, 1, 2, 3],
855
+ {
856
+ scaleSemantics: 'step',
857
+ }
858
+ )
539
859
  );
540
860
  addAxisQuantized(
541
861
  'transformer.embedder.per_layer_model_projection.w',
@@ -548,6 +868,16 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
548
868
  quantAxis: 0,
549
869
  }
550
870
  );
871
+ addFloat(
872
+ 'transformer.embedder.per_layer_model_projection.input_activation_static_scale',
873
+ 'model.language_model.per_layer_model_projection.input_activation_static_scale',
874
+ 'other'
875
+ );
876
+ addFloat(
877
+ 'transformer.embedder.per_layer_model_projection.output_activation_static_scale',
878
+ 'model.language_model.per_layer_model_projection.output_activation_static_scale',
879
+ 'other'
880
+ );
551
881
  addFloat(
552
882
  'transformer.embedder.per_layer_projection_norm.scale',
553
883
  'model.language_model.per_layer_projection_norm.weight',
@@ -681,8 +1011,8 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
681
1011
  'per_layer_input',
682
1012
  [vocabSizePerLayerInput, hiddenSizePerLayerInput],
683
1013
  {
684
- transposeStorage: true,
685
- quantAxis: 0,
1014
+ quantAxis: 1,
1015
+ scaleSemantics: 'step',
686
1016
  }
687
1017
  );
688
1018
  }
@@ -795,6 +1125,28 @@ async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
795
1125
  parsedTFLite = await parseTFLiteFromSource(source, {
796
1126
  allowPackedQuantization: true,
797
1127
  });
1128
+ const tokenizerMetadataEntry = findTFLiteMetadataEntry(parsedTFLite, 'spm_vocab_model');
1129
+ if (tokenizerMetadataEntry) {
1130
+ virtualFiles.push(
1131
+ createVirtualFile(
1132
+ 'TOKENIZER_MODEL',
1133
+ tokenizerMetadataEntry.offset,
1134
+ tokenizerMetadataEntry.size,
1135
+ 'tokenizer_model'
1136
+ )
1137
+ );
1138
+ }
1139
+ const llmParametersEntry = findTFLiteMetadataEntry(parsedTFLite, 'odml.infra.proto.LlmParameters');
1140
+ if (llmParametersEntry) {
1141
+ virtualFiles.push(
1142
+ createVirtualFile(
1143
+ 'METADATA',
1144
+ llmParametersEntry.offset,
1145
+ llmParametersEntry.size,
1146
+ 'litert_metadata'
1147
+ )
1148
+ );
1149
+ }
798
1150
  } else {
799
1151
  const parsedTask = await parseLiteRTTaskFromSource(source);
800
1152
  const tfliteEntryName = normalizeText(taskConfig.tfliteEntry) || LITERT_TASK_DEFAULT_TFLITE_ENTRY;
@@ -42,6 +42,21 @@ export declare function runBrowserCommandInNode(
42
42
  options?: NodeBrowserCommandRunOptions
43
43
  ): Promise<BrowserCommandRunResult>;
44
44
 
45
+ export declare function runBrowserCommandEvaluationWithTimeout<T>(
46
+ operation: () => Promise<T>,
47
+ timeoutMs: number
48
+ ): Promise<T>;
49
+
45
50
  export declare function normalizeNodeBrowserCommand(
46
51
  commandRequest: ToolingCommandRequestInput
47
52
  ): ToolingCommandRequest;
53
+
54
+ /**
55
+ * Validate a browser-relay response envelope and stamp the original
56
+ * request onto it. Throws when the envelope does not match the success
57
+ * shape.
58
+ */
59
+ export declare function finalizeBrowserRelayResponse(
60
+ response: unknown,
61
+ request: ToolingCommandRequest
62
+ ): BrowserCommandRunResult & { request: ToolingCommandRequest };