doppler-gpu 0.2.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (658) hide show
  1. package/CHANGELOG.md +182 -0
  2. package/README.md +20 -8
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +82 -33
  5. package/src/cli/cli-model-resolution.js +275 -0
  6. package/src/cli/cli-output.js +453 -0
  7. package/src/cli/doppler-cli.js +817 -709
  8. package/src/cli/doppler-quickstart.d.ts +7 -23
  9. package/src/cli/doppler-quickstart.js +20 -4
  10. package/src/cli/doppler-serve.js +1 -1
  11. package/src/cli/runtime-profiles.d.ts +40 -0
  12. package/src/cli/runtime-profiles.js +225 -0
  13. package/src/client/doppler-api.js +1 -6
  14. package/src/client/doppler-registry.d.ts +12 -0
  15. package/src/client/doppler-registry.js +27 -0
  16. package/src/client/doppler-registry.json +79 -5
  17. package/src/client/failure-taxonomy.js +139 -0
  18. package/src/client/fault-injection.js +30 -0
  19. package/src/client/provider.d.ts +151 -0
  20. package/src/client/provider.js +374 -0
  21. package/src/client/receipt.js +156 -0
  22. package/src/client/runtime/index.js +56 -4
  23. package/src/client/runtime/model-manager.d.ts +0 -14
  24. package/src/client/runtime/model-manager.js +2 -42
  25. package/src/client/runtime/model-session.js +33 -0
  26. package/src/client/runtime/model-source.d.ts +13 -1
  27. package/src/client/runtime/model-source.js +128 -2
  28. package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
  29. package/src/client/runtime/node-quickstart-cache.js +353 -0
  30. package/src/client/runtime/source-runtime.js +1 -10
  31. package/src/client/runtime/types.d.ts +0 -60
  32. package/src/client/wrap-pipeline-handle.d.ts +20 -0
  33. package/src/client/wrap-pipeline-handle.js +84 -0
  34. package/src/config/README.md +3 -1
  35. package/src/config/conversion/README.md +23 -157
  36. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
  37. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
  38. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
  39. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
  40. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
  41. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
  42. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
  43. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
  44. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
  45. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
  46. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
  47. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
  48. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
  49. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
  50. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
  53. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
  55. package/src/config/execution-contract-check.js +6 -5
  56. package/src/config/kernel-path-loader.d.ts +8 -2
  57. package/src/config/kernels/kernel-ref-digests.js +125 -122
  58. package/src/config/kernels/registry.d.ts +0 -35
  59. package/src/config/kernels/registry.js +5 -9
  60. package/src/config/kernels/registry.json +615 -368
  61. package/src/config/merge-helpers.d.ts +1 -1
  62. package/src/config/merge-helpers.js +1 -1
  63. package/src/config/merge.js +35 -7
  64. package/src/config/param-categories.d.ts +0 -9
  65. package/src/config/param-categories.js +1 -5
  66. package/src/config/param-validator.d.ts +2 -0
  67. package/src/config/param-validator.js +16 -0
  68. package/src/config/platforms/loader.d.ts +0 -27
  69. package/src/config/platforms/loader.js +0 -30
  70. package/src/config/required-inference-fields-contract-check.js +1 -7
  71. package/src/config/runtime/README.md +28 -0
  72. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  73. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
  74. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
  75. package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
  76. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
  77. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
  78. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
  79. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
  80. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
  81. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
  82. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
  83. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
  84. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
  85. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
  86. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
  87. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
  89. package/src/config/schema/benchmark.schema.d.ts +2 -0
  90. package/src/config/schema/benchmark.schema.js +1 -0
  91. package/src/config/schema/bridge.schema.d.ts +16 -0
  92. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  93. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  94. package/src/config/schema/converter.schema.d.ts +5 -0
  95. package/src/config/schema/converter.schema.js +3 -0
  96. package/src/config/schema/debug.schema.d.ts +22 -1
  97. package/src/config/schema/debug.schema.js +1 -0
  98. package/src/config/schema/distill-training.schema.d.ts +0 -1
  99. package/src/config/schema/distill-training.schema.js +1 -1
  100. package/src/config/schema/doppler.schema.d.ts +21 -3
  101. package/src/config/schema/doppler.schema.js +116 -9
  102. package/src/config/schema/emulation.schema.d.ts +3 -0
  103. package/src/config/schema/emulation.schema.js +16 -16
  104. package/src/config/schema/execution-v1.schema.d.ts +12 -1
  105. package/src/config/schema/execution-v1.schema.js +7 -1
  106. package/src/config/schema/index.d.ts +54 -1
  107. package/src/config/schema/index.js +15 -0
  108. package/src/config/schema/inference.schema.d.ts +3 -3
  109. package/src/config/schema/kernel-path.schema.d.ts +6 -27
  110. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
  111. package/src/config/schema/kernel-thresholds.schema.js +3 -7
  112. package/src/config/schema/kvcache.schema.js +0 -9
  113. package/src/config/schema/loading.schema.d.ts +12 -0
  114. package/src/config/schema/loading.schema.js +4 -0
  115. package/src/config/schema/manifest.schema.d.ts +73 -1
  116. package/src/config/schema/manifest.schema.js +48 -2
  117. package/src/config/schema/moe.schema.js +0 -8
  118. package/src/config/schema/program-bundle.schema.d.ts +233 -0
  119. package/src/config/schema/program-bundle.schema.js +393 -0
  120. package/src/config/schema/ul-training.schema.d.ts +0 -1
  121. package/src/config/schema/ul-training.schema.js +1 -1
  122. package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
  123. package/src/config/supported-operations.d.ts +14 -0
  124. package/src/config/supported-operations.js +34 -0
  125. package/src/config/supported-operations.json +178 -0
  126. package/src/config/training-defaults.js +1 -7
  127. package/src/config/transforms/capability-transform-resolver.js +3 -1
  128. package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
  129. package/src/config/transforms/execution-graph-transforms.js +693 -29
  130. package/src/converter/conversion-plan.js +13 -10
  131. package/src/converter/core.d.ts +57 -0
  132. package/src/converter/core.js +582 -13
  133. package/src/converter/parsers/diffusion.js +1 -33
  134. package/src/converter/quantization-info.d.ts +5 -4
  135. package/src/converter/quantization-info.js +39 -3
  136. package/src/converter/quantizer.d.ts +19 -0
  137. package/src/converter/quantizer.js +81 -4
  138. package/src/debug/config.d.ts +0 -4
  139. package/src/debug/config.js +0 -17
  140. package/src/debug/index.d.ts +28 -30
  141. package/src/debug/index.js +0 -18
  142. package/src/debug/stats.d.ts +0 -8
  143. package/src/debug/stats.js +3 -3
  144. package/src/debug/tensor.d.ts +9 -2
  145. package/src/debug/tensor.js +11 -4
  146. package/src/debug/trace.js +2 -15
  147. package/src/errors/doppler-error.js +20 -0
  148. package/src/experimental/browser/quantization.d.ts +5 -0
  149. package/src/experimental/distribution/shard-delivery.js +179 -23
  150. package/src/experimental/training/artifacts.js +1 -14
  151. package/src/experimental/training/checkpoint.js +1 -14
  152. package/src/experimental/training/datasets/token-batch.d.ts +1 -1
  153. package/src/experimental/training/datasets/token-batch.js +2 -2
  154. package/src/experimental/training/distillation/dataset.js +2 -15
  155. package/src/experimental/training/distillation/eval.js +3 -3
  156. package/src/experimental/training/distillation/runtime.js +3 -16
  157. package/src/experimental/training/distillation/student-fixture.js +6 -6
  158. package/src/experimental/training/lora-pipeline.js +1 -14
  159. package/src/experimental/training/objectives/base.d.ts +2 -2
  160. package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
  161. package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
  162. package/src/experimental/training/objectives/distill_kd.js +2 -2
  163. package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
  164. package/src/experimental/training/objectives/distill_triplet.js +2 -2
  165. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
  166. package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
  167. package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
  168. package/src/experimental/training/operator-artifacts.js +1 -14
  169. package/src/experimental/training/runner.js +1 -14
  170. package/src/experimental/training/workload-packs/registry.json +6 -6
  171. package/src/experimental/training/workloads.js +1 -1
  172. package/src/formats/gguf/types.d.ts +0 -63
  173. package/src/formats/gguf/types.js +58 -38
  174. package/src/formats/litert/types.d.ts +0 -13
  175. package/src/formats/litert/types.js +2 -2
  176. package/src/formats/rdrr/distributed/index.d.ts +34 -0
  177. package/src/formats/rdrr/distributed/index.js +3 -0
  178. package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
  179. package/src/formats/rdrr/distributed/parsing.js +23 -0
  180. package/src/formats/rdrr/distributed/validation.d.ts +23 -0
  181. package/src/formats/rdrr/distributed/validation.js +131 -0
  182. package/src/formats/rdrr/groups.d.ts +1 -13
  183. package/src/formats/rdrr/groups.js +0 -34
  184. package/src/formats/rdrr/index.d.ts +9 -0
  185. package/src/formats/rdrr/index.js +9 -0
  186. package/src/formats/rdrr/integrity.d.ts +51 -0
  187. package/src/formats/rdrr/integrity.js +200 -0
  188. package/src/formats/rdrr/lowerings.d.ts +28 -0
  189. package/src/formats/rdrr/lowerings.js +112 -0
  190. package/src/formats/rdrr/manifest.d.ts +0 -14
  191. package/src/formats/rdrr/manifest.js +5 -58
  192. package/src/formats/rdrr/merkle.d.ts +24 -0
  193. package/src/formats/rdrr/merkle.js +118 -0
  194. package/src/formats/rdrr/parsing.js +4 -0
  195. package/src/formats/rdrr/slice-compiler.d.ts +29 -0
  196. package/src/formats/rdrr/slice-compiler.js +188 -0
  197. package/src/formats/rdrr/source-transform-contract.js +390 -0
  198. package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
  199. package/src/formats/rdrr/storage-descriptor.js +186 -0
  200. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
  201. package/src/formats/rdrr/tensor-config-validator.js +0 -30
  202. package/src/formats/rdrr/types.d.ts +115 -1
  203. package/src/formats/rdrr/validation.js +245 -0
  204. package/src/formats/tflite/types.d.ts +8 -31
  205. package/src/formats/tflite/types.js +50 -5
  206. package/src/gpu/command-recorder.d.ts +9 -21
  207. package/src/gpu/command-recorder.js +41 -2
  208. package/src/gpu/device.d.ts +17 -3
  209. package/src/gpu/device.js +20 -4
  210. package/src/gpu/kernel-runtime.d.ts +8 -15
  211. package/src/gpu/kernel-selection-log.d.ts +0 -8
  212. package/src/gpu/kernel-selection-log.js +0 -15
  213. package/src/gpu/kernel-tuner/cache.js +0 -3
  214. package/src/gpu/kernels/README.md +50 -18
  215. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  216. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  217. package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
  218. package/src/gpu/kernels/attention.d.ts +28 -19
  219. package/src/gpu/kernels/attention.js +443 -9
  220. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
  221. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
  222. package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
  223. package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
  224. package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
  225. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
  226. package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
  228. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
  229. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
  230. package/src/gpu/kernels/attention_f16.wgsl +88 -30
  231. package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
  232. package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
  233. package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
  234. package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
  235. package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
  236. package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
  237. package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
  238. package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
  239. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
  240. package/src/gpu/kernels/backward/index.d.ts +6 -0
  241. package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
  242. package/src/gpu/kernels/backward/utils.d.ts +35 -0
  243. package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
  244. package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
  245. package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
  246. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
  247. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
  248. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
  249. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
  250. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
  251. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
  252. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
  253. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
  254. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
  255. package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
  256. package/src/gpu/kernels/constants.d.ts +9 -48
  257. package/src/gpu/kernels/constants.js +0 -23
  258. package/src/gpu/kernels/conv2d.d.ts +0 -4
  259. package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
  260. package/src/gpu/kernels/dequant.d.ts +0 -16
  261. package/src/gpu/kernels/dequant.js +1 -71
  262. package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
  263. package/src/gpu/kernels/dequant_shared.wgsl +1 -55
  264. package/src/gpu/kernels/dispatch.d.ts +12 -97
  265. package/src/gpu/kernels/dispatch.js +0 -117
  266. package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
  267. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
  269. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
  270. package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
  271. package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
  272. package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
  273. package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
  274. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
  275. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
  276. package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
  277. package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
  278. package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
  279. package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
  280. package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
  281. package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
  282. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
  283. package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
  284. package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
  285. package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
  286. package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
  287. package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
  288. package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
  289. package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
  290. package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
  291. package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
  292. package/src/gpu/kernels/gather.d.ts +0 -7
  293. package/src/gpu/kernels/gather_f16.wgsl +1 -1
  294. package/src/gpu/kernels/gelu_f16.wgsl +14 -15
  295. package/src/gpu/kernels/groupnorm.d.ts +0 -4
  296. package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
  297. package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
  298. package/src/gpu/kernels/index.d.ts +13 -5
  299. package/src/gpu/kernels/index.js +9 -4
  300. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
  301. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
  302. package/src/gpu/kernels/layernorm.d.ts +0 -4
  303. package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
  304. package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
  305. package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
  306. package/src/gpu/kernels/logit-merge.d.ts +0 -24
  307. package/src/gpu/kernels/logit-merge.js +1 -17
  308. package/src/gpu/kernels/matmul-dispatch.js +7 -1
  309. package/src/gpu/kernels/matmul-selection.js +68 -1
  310. package/src/gpu/kernels/matmul.d.ts +0 -28
  311. package/src/gpu/kernels/matmul.js +101 -7
  312. package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
  313. package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
  314. package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
  315. package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
  316. package/src/gpu/kernels/modulate_f16.wgsl +1 -1
  317. package/src/gpu/kernels/moe.d.ts +0 -56
  318. package/src/gpu/kernels/moe.js +0 -15
  319. package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
  320. package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
  321. package/src/gpu/kernels/residual.d.ts +0 -12
  322. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  323. package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
  324. package/src/gpu/kernels/rmsnorm.d.ts +0 -8
  325. package/src/gpu/kernels/rmsnorm.js +17 -1
  326. package/src/gpu/kernels/rmsnorm.wgsl +5 -6
  327. package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
  328. package/src/gpu/kernels/rope.d.ts +1 -5
  329. package/src/gpu/kernels/rope.js +12 -2
  330. package/src/gpu/kernels/rope.wgsl +33 -19
  331. package/src/gpu/kernels/rope_f16.wgsl +39 -25
  332. package/src/gpu/kernels/sample.d.ts +0 -5
  333. package/src/gpu/kernels/sample.wgsl +7 -6
  334. package/src/gpu/kernels/sample_f16.wgsl +28 -30
  335. package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
  336. package/src/gpu/kernels/shader-cache.d.ts +13 -0
  337. package/src/gpu/kernels/shader-cache.js +27 -0
  338. package/src/gpu/kernels/silu_f16.wgsl +1 -1
  339. package/src/gpu/kernels/softmax.wgsl +4 -144
  340. package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
  341. package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
  342. package/src/gpu/kernels/topk_f16.wgsl +1 -1
  343. package/src/gpu/kernels/turboquant-codebook.js +11 -40
  344. package/src/gpu/kernels/uniform-utils.d.ts +18 -0
  345. package/src/gpu/kernels/upsample2d.d.ts +0 -4
  346. package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
  347. package/src/gpu/kernels/utils.d.ts +28 -0
  348. package/src/gpu/kernels/utils.js +41 -0
  349. package/src/gpu/perf-guards.d.ts +0 -12
  350. package/src/gpu/perf-guards.js +0 -62
  351. package/src/gpu/profiler.d.ts +0 -21
  352. package/src/gpu/profiler.js +0 -34
  353. package/src/gpu/submit-tracker.d.ts +0 -57
  354. package/src/gpu/submit-tracker.js +3 -62
  355. package/src/gpu/tensor.d.ts +0 -23
  356. package/src/gpu/tensor.js +0 -41
  357. package/src/gpu/uniform-cache.d.ts +0 -11
  358. package/src/gpu/uniform-cache.js +0 -23
  359. package/src/index.d.ts +1 -0
  360. package/src/index.js +1 -0
  361. package/src/inference/browser-harness-model-helpers.d.ts +0 -9
  362. package/src/inference/browser-harness-model-helpers.js +22 -13
  363. package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
  364. package/src/inference/browser-harness-runtime-helpers.js +2 -2
  365. package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
  366. package/src/inference/browser-harness-suite-helpers.js +87 -1
  367. package/src/inference/browser-harness-text-helpers.d.ts +13 -0
  368. package/src/inference/browser-harness-text-helpers.js +267 -18
  369. package/src/inference/browser-harness.d.ts +0 -29
  370. package/src/inference/browser-harness.js +271 -71
  371. package/src/inference/kv-cache/base.d.ts +6 -0
  372. package/src/inference/kv-cache/base.js +16 -1
  373. package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
  374. package/src/inference/kv-cache/mixed-geometry.js +12 -2
  375. package/src/inference/kv-cache/quantized.js +2 -0
  376. package/src/inference/kv-cache/tiered.js +7 -0
  377. package/src/inference/kv-cache/types.d.ts +2 -0
  378. package/src/inference/multi-model-network.d.ts +1 -1
  379. package/src/inference/multi-model-network.js +1 -2
  380. package/src/inference/network-evolution.d.ts +0 -12
  381. package/src/inference/network-evolution.js +2 -2
  382. package/src/inference/pipelines/context.js +21 -2
  383. package/src/inference/pipelines/diffusion/init.js +2 -2
  384. package/src/inference/pipelines/diffusion/pipeline.js +41 -96
  385. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
  386. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
  387. package/src/inference/pipelines/diffusion/vae.js +4 -4
  388. package/src/inference/pipelines/energy/quintel.d.ts +15 -0
  389. package/src/inference/pipelines/structured/index.d.ts +2 -0
  390. package/src/inference/pipelines/structured/index.js +2 -0
  391. package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
  392. package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
  393. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
  394. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  395. package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
  396. package/src/inference/pipelines/text/attention/projections.js +70 -23
  397. package/src/inference/pipelines/text/attention/record.js +118 -5
  398. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
  399. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
  400. package/src/inference/pipelines/text/attention/run.js +97 -6
  401. package/src/inference/pipelines/text/attention/types.d.ts +2 -0
  402. package/src/inference/pipelines/text/attention/types.js +9 -4
  403. package/src/inference/pipelines/text/chat-format.d.ts +0 -10
  404. package/src/inference/pipelines/text/chat-format.js +1 -8
  405. package/src/inference/pipelines/text/config.d.ts +3 -25
  406. package/src/inference/pipelines/text/config.js +105 -3
  407. package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
  408. package/src/inference/pipelines/text/drift-policy.js +1 -1
  409. package/src/inference/pipelines/text/embed.d.ts +23 -27
  410. package/src/inference/pipelines/text/embed.js +10 -43
  411. package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
  412. package/src/inference/pipelines/text/execution-plan.js +15 -2
  413. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
  414. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
  415. package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
  416. package/src/inference/pipelines/text/execution-v1.js +94 -25
  417. package/src/inference/pipelines/text/ffn/dense.js +180 -3
  418. package/src/inference/pipelines/text/ffn/standard.js +40 -13
  419. package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
  420. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
  421. package/src/inference/pipelines/text/generator-helpers.js +20 -2
  422. package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
  423. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
  424. package/src/inference/pipelines/text/generator-runtime.js +8 -6
  425. package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
  426. package/src/inference/pipelines/text/generator-steps.js +23 -9
  427. package/src/inference/pipelines/text/generator.d.ts +0 -6
  428. package/src/inference/pipelines/text/generator.js +254 -530
  429. package/src/inference/pipelines/text/init-chat-templates.js +86 -0
  430. package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
  431. package/src/inference/pipelines/text/init-rope.js +408 -0
  432. package/src/inference/pipelines/text/init.d.ts +8 -2
  433. package/src/inference/pipelines/text/init.js +190 -710
  434. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
  435. package/src/inference/pipelines/text/kernel-trace.js +0 -33
  436. package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
  437. package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
  438. package/src/inference/pipelines/text/layer-plan.js +1 -1
  439. package/src/inference/pipelines/text/layer.d.ts +70 -8
  440. package/src/inference/pipelines/text/layer.js +105 -537
  441. package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
  442. package/src/inference/pipelines/text/load-errors.d.ts +21 -0
  443. package/src/inference/pipelines/text/load-errors.js +59 -0
  444. package/src/inference/pipelines/text/logits/cpu.js +2 -20
  445. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  446. package/src/inference/pipelines/text/logits/gpu.js +148 -11
  447. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  448. package/src/inference/pipelines/text/model-load.d.ts +5 -4
  449. package/src/inference/pipelines/text/model-load.js +74 -17
  450. package/src/inference/pipelines/text/ops.d.ts +13 -2
  451. package/src/inference/pipelines/text/ops.js +1 -1
  452. package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
  453. package/src/inference/pipelines/text/probes.d.ts +3 -6
  454. package/src/inference/pipelines/text/probes.js +198 -31
  455. package/src/inference/pipelines/text/sampling.d.ts +0 -5
  456. package/src/inference/pipelines/text/sampling.js +140 -0
  457. package/src/inference/pipelines/text/stage-names.js +12 -0
  458. package/src/inference/pipelines/text/state.js +2 -0
  459. package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
  460. package/src/inference/pipelines/text/types.d.ts +8 -78
  461. package/src/inference/pipelines/text/weights.d.ts +0 -50
  462. package/src/inference/pipelines/text/weights.js +2 -51
  463. package/src/inference/pipelines/text.d.ts +9 -1
  464. package/src/inference/pipelines/text.js +233 -29
  465. package/src/inference/pipelines/vision/index.js +0 -107
  466. package/src/inference/runtime-model.d.ts +0 -7
  467. package/src/inference/runtime-model.js +0 -23
  468. package/src/inference/test-harness.d.ts +6 -22
  469. package/src/inference/test-harness.js +97 -23
  470. package/src/inference/tokenizer.js +1 -10
  471. package/src/inference/tokenizers/sentencepiece.js +65 -15
  472. package/src/inference/tokenizers/types.d.ts +0 -21
  473. package/src/loader/doppler-loader.d.ts +3 -0
  474. package/src/loader/doppler-loader.js +90 -12
  475. package/src/loader/embedding-loader.js +30 -5
  476. package/src/loader/loader-types.d.ts +4 -0
  477. package/src/loader/manifest-config.d.ts +2 -7
  478. package/src/loader/manifest-config.js +25 -1
  479. package/src/loader/memory-monitor.d.ts +17 -5
  480. package/src/loader/per-layer-input-loader.js +114 -1
  481. package/src/loader/quantization-constants.d.ts +6 -0
  482. package/src/loader/shard-resolver.d.ts +1 -0
  483. package/src/loader/shard-resolver.js +15 -0
  484. package/src/loader/tensors/source-transform.d.ts +15 -0
  485. package/src/loader/tensors/source-transform.js +317 -34
  486. package/src/loader/tensors/tensor-loader.d.ts +2 -9
  487. package/src/loader/tensors/tensor-loader.js +84 -1
  488. package/src/loader/tensors/tensor-reader.js +28 -2
  489. package/src/memory/address-table.d.ts +1 -9
  490. package/src/memory/buffer-pool.d.ts +0 -16
  491. package/src/memory/buffer-pool.js +1 -33
  492. package/src/memory/heap-manager.d.ts +18 -43
  493. package/src/memory/unified-detect.d.ts +19 -34
  494. package/src/models/embeddinggemma.d.ts +7 -0
  495. package/src/models/embeddinggemma.js +24 -0
  496. package/src/models/family.d.ts +24 -0
  497. package/src/models/family.js +25 -0
  498. package/src/models/gemma3.d.ts +7 -0
  499. package/src/models/gemma3.js +33 -0
  500. package/src/models/gemma4.d.ts +7 -0
  501. package/src/models/gemma4.js +33 -0
  502. package/src/models/qwen3.d.ts +7 -0
  503. package/src/models/qwen3.js +36 -0
  504. package/src/rules/converter/tokenizer.rules.json +5 -0
  505. package/src/rules/execution-rules-contract-check.js +72 -3
  506. package/src/rules/inference/capability-transforms.rules.json +87 -0
  507. package/src/rules/inference/execution.rules.json +11 -1
  508. package/src/rules/inference/ffn.rules.json +6 -2
  509. package/src/rules/loader/tensor-loader.rules.json +9 -0
  510. package/src/rules/rule-registry.js +53 -75
  511. package/src/storage/artifact-storage-context.d.ts +12 -3
  512. package/src/storage/artifact-storage-context.js +389 -98
  513. package/src/storage/backends/opfs-store.js +138 -88
  514. package/src/storage/blake3.js +9 -1
  515. package/src/storage/export.js +1 -5
  516. package/src/storage/normalize-model-id.js +3 -0
  517. package/src/storage/preflight.js +22 -0
  518. package/src/storage/quickstart-downloader.js +44 -56
  519. package/src/storage/reports.js +35 -11
  520. package/src/storage/shard-manager.d.ts +8 -2
  521. package/src/storage/shard-manager.js +176 -15
  522. package/src/storage/source-artifact-store.d.ts +0 -2
  523. package/src/storage/source-artifact-store.js +3 -9
  524. package/src/sw.js +0 -3
  525. package/src/tooling/browser-command-runner.js +39 -0
  526. package/src/tooling/command-api-constants.js +1 -1
  527. package/src/tooling/command-api-family-normalizers.d.ts +1 -0
  528. package/src/tooling/command-api-family-normalizers.js +49 -7
  529. package/src/tooling/command-api-helpers.d.ts +0 -2
  530. package/src/tooling/command-api-helpers.js +6 -2
  531. package/src/tooling/command-api.d.ts +20 -1
  532. package/src/tooling/command-api.js +10 -1
  533. package/src/tooling/command-envelope.d.ts +0 -1
  534. package/src/tooling/command-envelope.js +1 -1
  535. package/src/tooling/command-runner-shared.d.ts +10 -0
  536. package/src/tooling/conversion-config-materializer.d.ts +0 -10
  537. package/src/tooling/conversion-config-materializer.js +1 -1
  538. package/src/tooling/diagnose-runner.js +1 -4
  539. package/src/tooling/hf-registry-utils.d.ts +2 -6
  540. package/src/tooling/hf-registry-utils.js +90 -19
  541. package/src/tooling/kernel-path-builder/index.js +1 -4
  542. package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
  543. package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
  544. package/src/tooling/lean-execution-contract-runner.js +2 -2
  545. package/src/tooling/litert-package-runtime.d.ts +1 -1
  546. package/src/tooling/litert-package-runtime.js +373 -21
  547. package/src/tooling/node-browser-command-runner.d.ts +15 -0
  548. package/src/tooling/node-browser-command-runner.js +97 -13
  549. package/src/tooling/node-command-runner.js +36 -0
  550. package/src/tooling/node-convert-worker-pool.js +12 -0
  551. package/src/tooling/node-convert-worker.js +10 -1
  552. package/src/tooling/node-converter.js +90 -1
  553. package/src/tooling/node-source-runtime.js +257 -150
  554. package/src/tooling/opfs-cache.js +1 -4
  555. package/src/tooling/precision-replay-math.js +174 -0
  556. package/src/tooling/program-bundle-host.d.ts +16 -0
  557. package/src/tooling/program-bundle-host.js +20 -0
  558. package/src/tooling/program-bundle-parity.d.ts +28 -0
  559. package/src/tooling/program-bundle-parity.js +210 -0
  560. package/src/tooling/program-bundle.d.ts +61 -0
  561. package/src/tooling/program-bundle.js +1119 -0
  562. package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
  563. package/src/tooling/rdrr-integrity-refresh.js +271 -0
  564. package/src/tooling/reference-verify.js +191 -0
  565. package/src/tooling/source-artifact-adapter.d.ts +0 -20
  566. package/src/tooling/source-artifact-adapter.js +4 -4
  567. package/src/tooling/source-package-profiles.js +1 -7
  568. package/src/tooling/source-runtime-bundle.d.ts +27 -0
  569. package/src/tooling/source-runtime-bundle.js +56 -23
  570. package/src/tooling/source-runtime-converter-config.d.ts +0 -5
  571. package/src/tooling/source-runtime-converter-config.js +13 -9
  572. package/src/tooling/source-runtime-materializer.js +4 -8
  573. package/src/tooling-exports/device.d.ts +11 -0
  574. package/src/tooling-exports/device.js +14 -0
  575. package/src/tooling-exports/manifest.d.ts +8 -0
  576. package/src/tooling-exports/manifest.js +11 -0
  577. package/src/tooling-exports/storage.d.ts +25 -0
  578. package/src/tooling-exports/storage.js +29 -0
  579. package/src/tooling-exports/structured.d.ts +1 -0
  580. package/src/tooling-exports/structured.js +5 -0
  581. package/src/tooling-exports.d.ts +23 -0
  582. package/src/tooling-exports.js +17 -0
  583. package/src/tooling-exports.shared.d.ts +27 -30
  584. package/src/tooling-exports.shared.js +28 -29
  585. package/src/types/index.d.ts +0 -3
  586. package/src/types/index.js +0 -3
  587. package/src/utils/array-buffer.d.ts +4 -0
  588. package/src/utils/array-buffer.js +9 -0
  589. package/src/utils/canonical-hash.d.ts +7 -0
  590. package/src/utils/canonical-hash.js +74 -0
  591. package/src/utils/clone-json.d.ts +1 -0
  592. package/src/utils/clone-json.js +7 -0
  593. package/src/utils/encode-utf8.js +5 -0
  594. package/src/utils/index.d.ts +4 -1
  595. package/src/utils/index.js +4 -1
  596. package/src/utils/runtime-env.js +7 -0
  597. package/src/utils/sha256.d.ts +1 -0
  598. package/src/utils/sha256.js +18 -4
  599. package/src/utils/stable-sort-object.d.ts +1 -0
  600. package/src/utils/stable-sort-object.js +13 -0
  601. package/src/version.js +1 -1
  602. package/tools/convert-safetensors-node.js +1 -2
  603. package/src/bootstrap.d.ts +0 -1
  604. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
  605. package/src/config/kernels/kernel-ref.d.ts +0 -17
  606. package/src/config/kernels/kernel-ref.js +0 -75
  607. package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
  608. package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
  609. package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
  610. package/src/config/runtime/kernels/fused-q4k.json +0 -13
  611. package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
  612. package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
  613. package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
  614. package/src/config/runtime/kernels/safe-q4k.json +0 -13
  615. package/src/config/runtime/overlays/debug-f32a.json +0 -3
  616. package/src/config/runtime/overlays/trace-config.json +0 -28
  617. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
  618. package/src/debug/perf.d.ts +0 -33
  619. package/src/debug/perf.js +0 -51
  620. package/src/errors/index.d.ts +0 -1
  621. package/src/errors/index.js +0 -1
  622. package/src/formats/gguf/index.d.ts +0 -8
  623. package/src/formats/gguf/index.js +0 -4
  624. package/src/formats/litert/index.d.ts +0 -1
  625. package/src/formats/litert/index.js +0 -1
  626. package/src/formats/safetensors/index.d.ts +0 -8
  627. package/src/formats/safetensors/index.js +0 -4
  628. package/src/formats/tflite/index.d.ts +0 -1
  629. package/src/formats/tflite/index.js +0 -1
  630. package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
  631. package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
  632. package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
  633. package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
  634. package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
  635. package/src/inference/functiongemma.d.ts +0 -15
  636. package/src/inference/functiongemma.js +0 -1
  637. package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
  638. package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
  639. package/src/rules/kernels/kernel-validator.d.ts +0 -24
  640. package/src/rules/kernels/kernel-validator.js +0 -160
  641. package/src/sw.d.ts +0 -1
  642. package/src/tooling/bench-runner.js +0 -64
  643. package/src/tooling/distill-runner.js +0 -56
  644. package/src/tooling/lora-runner.js +0 -37
  645. package/src/tooling/verify-runner.js +0 -63
  646. package/src/types/gpu.d.ts +0 -185
  647. package/src/types/gpu.js +0 -5
  648. package/src/types/inference.d.ts +0 -197
  649. package/src/types/inference.js +0 -5
  650. package/src/types/model.d.ts +0 -130
  651. package/src/types/model.js +0 -5
  652. package/src/utils/format-bytes.js +0 -20
  653. /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
  654. /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
  655. /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
  656. /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
  657. /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
  658. /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
@@ -6,3 +6,5 @@ export {
6
6
  isDreamStructuredModelType,
7
7
  createDreamStructuredPipeline,
8
8
  } from './json-head-pipeline.js';
9
+
10
+ export { createJsonGrammarMask } from './json-grammar-mask.js';
@@ -6,3 +6,5 @@ export {
6
6
  isDreamStructuredModelType,
7
7
  createDreamStructuredPipeline,
8
8
  } from './json-head-pipeline.js';
9
+
10
+ export { createJsonGrammarMask } from './json-grammar-mask.js';
@@ -0,0 +1,143 @@
1
+ // Minimal JSON-grammar sampling constraint.
2
+ //
3
+ // Usage:
4
+ // const mask = createJsonGrammarMask({ tokenizer });
5
+ // for await (const text of pipeline.generate(prompt, { logitMaskFn: mask })) { ... }
6
+ //
7
+ // The mask classifies each candidate token against the current parse state and
8
+ // zeroes logits for tokens that would break a valid JSON object output. It
9
+ // deliberately accepts any token that *looks like* it may extend valid JSON —
10
+ // this is a soft constraint; parsing the final output is still required to
11
+ // validate the full payload.
12
+ //
13
+ // Tokenizer assumption: `tokenizer.decode([id])` returns the literal string
14
+ // piece for that token. That's true for BPE tokenizers used by Qwen, Gemma,
15
+ // Llama. The mask is deliberately token-piece based, not character level,
16
+ // because re-encoding the stream to characters per step is prohibitively slow
17
+ // in-browser.
18
+ //
19
+ // What the mask does enforce:
20
+ // - Until the first "{" is emitted, only tokens whose piece contains "{" or
21
+ // whitespace are allowed.
22
+ // - Once inside the object, matched counts of "{" / "}" / "[" / "]" / "\""
23
+ // determine whether we are inside a string; inside a string, any token is
24
+ // allowed except one that closes more brackets than are open.
25
+ // - After a balanced closing "}" is emitted at depth 0, we forbid any
26
+ // further non-whitespace token (forces a clean stop).
27
+ //
28
+ // What the mask does NOT enforce (intentional, to keep the mask cheap):
29
+ // - Key quoting, trailing-comma avoidance, matching " pairs at token level.
30
+ // Post-decode `parseJSONObjectEnvelope` handles these via recovery
31
+ // transformations.
32
+ // - Schema/key whitelists. Callers can layer a stricter mask on top.
33
+
34
+ function pieceAt(tokenizer, tokenId) {
35
+ if (!tokenizer || typeof tokenizer.decode !== "function") return "";
36
+ try {
37
+ // decode(ids, skipSpecial, skipBos)
38
+ return String(tokenizer.decode([tokenId], true, false) || "");
39
+ } catch {
40
+ return "";
41
+ }
42
+ }
43
+
44
+ function countRunes(piece, char) {
45
+ let count = 0;
46
+ for (let i = 0; i < piece.length; i += 1) {
47
+ if (piece[i] === char) count += 1;
48
+ }
49
+ return count;
50
+ }
51
+
52
+ // Piece-level flags used by the mask.
53
+ function classifyPiece(piece) {
54
+ const openBraces = countRunes(piece, "{");
55
+ const closeBraces = countRunes(piece, "}");
56
+ const openBrackets = countRunes(piece, "[");
57
+ const closeBrackets = countRunes(piece, "]");
58
+ const quotes = countRunes(piece, "\"");
59
+ return {
60
+ openBraces,
61
+ closeBraces,
62
+ openBrackets,
63
+ closeBrackets,
64
+ quotes,
65
+ isOnlyWhitespace: /^\s*$/.test(piece),
66
+ hasOpenBrace: openBraces > 0,
67
+ };
68
+ }
69
+
70
+ /**
71
+ * Create a logit mask function that enforces a soft JSON-object grammar.
72
+ *
73
+ * @param {{ tokenizer?: { decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string } | null, cacheBudget?: number }} [opts]
74
+ * @returns {(logits: Float32Array, context: { generatedIds: number[], tokenizer?: unknown, vocabSize?: number }) => void}
75
+ */
76
+ export function createJsonGrammarMask(opts = {}) {
77
+ const pieceCache = new Map();
78
+ const cacheBudget = Math.max(1024, Math.floor(Number(opts.cacheBudget) || 32768));
79
+
80
+ function cachedPiece(tokenizer, tokenId) {
81
+ if (pieceCache.has(tokenId)) return pieceCache.get(tokenId);
82
+ if (pieceCache.size >= cacheBudget) pieceCache.clear();
83
+ const piece = pieceAt(tokenizer, tokenId);
84
+ pieceCache.set(tokenId, piece);
85
+ return piece;
86
+ }
87
+
88
+ let runningOpenBraces = 0;
89
+ let runningOpenBrackets = 0;
90
+ let runningQuotes = 0;
91
+ let sawFirstBrace = false;
92
+ let balancedClose = false;
93
+ let lastGeneratedLen = 0;
94
+
95
+ function reflectEmittedTokens(tokenizer, generatedIds) {
96
+ // Catch up to any tokens committed since we were last called.
97
+ for (let i = lastGeneratedLen; i < generatedIds.length; i += 1) {
98
+ const piece = cachedPiece(tokenizer, generatedIds[i]);
99
+ const shape = classifyPiece(piece);
100
+ runningOpenBraces += shape.openBraces - shape.closeBraces;
101
+ runningOpenBrackets += shape.openBrackets - shape.closeBrackets;
102
+ runningQuotes += shape.quotes;
103
+ if (!sawFirstBrace && shape.hasOpenBrace) sawFirstBrace = true;
104
+ if (sawFirstBrace && runningOpenBraces <= 0 && runningOpenBrackets <= 0) {
105
+ balancedClose = true;
106
+ }
107
+ }
108
+ lastGeneratedLen = generatedIds.length;
109
+ }
110
+
111
+ return function logitMask(logits, context) {
112
+ const tokenizer = (opts.tokenizer ?? context?.tokenizer) || null;
113
+ if (!tokenizer) return;
114
+ reflectEmittedTokens(tokenizer, Array.isArray(context?.generatedIds) ? context.generatedIds : []);
115
+ const insideString = runningQuotes % 2 === 1;
116
+ const vocabSize = Math.min(logits.length, Number(context?.vocabSize || logits.length));
117
+ for (let id = 0; id < vocabSize; id += 1) {
118
+ if (logits[id] <= -Infinity) continue;
119
+ const piece = cachedPiece(tokenizer, id);
120
+ if (!piece) continue;
121
+ const shape = classifyPiece(piece);
122
+ let ok = true;
123
+ if (!sawFirstBrace) {
124
+ // Before the first "{", only allow whitespace or a piece that opens a brace.
125
+ ok = shape.isOnlyWhitespace || shape.hasOpenBrace;
126
+ } else if (balancedClose) {
127
+ // After the balanced close, only whitespace is allowed (model should stop).
128
+ ok = shape.isOnlyWhitespace;
129
+ } else if (insideString) {
130
+ // Inside a string literal, accept any piece — even ones with braces or
131
+ // brackets — the parser handles them as content.
132
+ ok = true;
133
+ } else {
134
+ // Outside strings: disallow pieces that close MORE brackets than are open.
135
+ if (shape.closeBraces > runningOpenBraces + shape.openBraces) ok = false;
136
+ if (shape.closeBrackets > runningOpenBrackets + shape.openBrackets) ok = false;
137
+ }
138
+ if (!ok) {
139
+ logits[id] = -Infinity;
140
+ }
141
+ }
142
+ };
143
+ }
@@ -0,0 +1,166 @@
1
+ // Minimal TSV-grammar sampling constraint.
2
+ //
3
+ // Usage:
4
+ // const mask = createTsvGrammarMask({ tokenizer, fieldsPerLine: 4 });
5
+ // for await (const text of pipeline.generate(prompt, { logitMaskFn: mask })) { ... }
6
+ //
7
+ // Companion to json-grammar-mask.js. Use this when the model is asked to
8
+ // emit a flat tab-separated record stream (one record per line, fields
9
+ // separated by TAB, terminated by NEWLINE) rather than a JSON envelope.
10
+ // Cuts decode tokens by ~30-40% vs. JSON because the model isn't sampling
11
+ // braces/quotes/keys at every position — and Doppler's structured-
12
+ // generation primitive is logit validation only (no token forcing), so
13
+ // JSON's structural overhead is real.
14
+ //
15
+ // What the mask enforces (soft):
16
+ // - Inside a line with fewer than `fieldsPerLine - 1` tabs already
17
+ // emitted, disallow tokens whose piece contains a NEWLINE — the line
18
+ // must hit all field separators before terminating.
19
+ // - Disallow tokens that would push the per-line tab count past
20
+ // `fieldsPerLine - 1`, so the model can't emit five-field rows when
21
+ // four were specified.
22
+ // - Allow any token (including end-of-stream) after a balanced newline
23
+ // when at least one valid line has been emitted.
24
+ //
25
+ // What the mask does NOT enforce (intentional):
26
+ // - Field-content shape. The category/confidence/page-number values
27
+ // are validated post-decode by the consumer (e.g.,
28
+ // parseDelimitedExtraction + validateRedaction in Columbo). Trying
29
+ // to enforce enum prefixes at the BPE token-piece level is brittle
30
+ // because token boundaries don't align with field boundaries.
31
+ // - Empty lines. A trailing blank line is just dropped by the parser.
32
+ //
33
+ // Tokenizer assumption: same as json-grammar-mask.js — `tokenizer.decode([id])`
34
+ // returns the literal piece for that token. True for Qwen, Gemma, Llama BPE.
35
+
36
+ function pieceAt(tokenizer, tokenId) {
37
+ if (!tokenizer || typeof tokenizer.decode !== "function") return "";
38
+ try {
39
+ return String(tokenizer.decode([tokenId], true, false) || "");
40
+ } catch {
41
+ return "";
42
+ }
43
+ }
44
+
45
+ function countRunes(piece, char) {
46
+ let count = 0;
47
+ for (let i = 0; i < piece.length; i += 1) {
48
+ if (piece[i] === char) count += 1;
49
+ }
50
+ return count;
51
+ }
52
+
53
+ /**
54
+ * Per-piece flags used by the mask. Tracks tab/newline placement so the
55
+ * state machine can decide when a line is complete and whether a token
56
+ * is allowed to advance the schema.
57
+ */
58
+ function classifyPiece(piece) {
59
+ const tabs = countRunes(piece, "\t");
60
+ const newlines = countRunes(piece, "\n");
61
+ return {
62
+ tabs,
63
+ newlines,
64
+ isOnlyWhitespace: /^\s*$/.test(piece),
65
+ hasNewline: newlines > 0,
66
+ hasTab: tabs > 0,
67
+ };
68
+ }
69
+
70
+ /**
71
+ * Create a logit mask function that enforces a soft TSV grammar.
72
+ *
73
+ * @param {{
74
+ * tokenizer?: { decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string } | null,
75
+ * fieldsPerLine?: number,
76
+ * cacheBudget?: number,
77
+ * }} [opts]
78
+ * @returns {(logits: Float32Array, context: { generatedIds: number[], tokenizer?: unknown, vocabSize?: number }) => void}
79
+ */
80
+ export function createTsvGrammarMask(opts = {}) {
81
+ const fieldsPerLine = Math.max(2, Math.floor(Number(opts.fieldsPerLine) || 4));
82
+ const tabsPerLineMax = fieldsPerLine - 1;
83
+ const pieceCache = new Map();
84
+ const cacheBudget = Math.max(1024, Math.floor(Number(opts.cacheBudget) || 32768));
85
+
86
+ function cachedPiece(tokenizer, tokenId) {
87
+ if (pieceCache.has(tokenId)) return pieceCache.get(tokenId);
88
+ if (pieceCache.size >= cacheBudget) pieceCache.clear();
89
+ const piece = pieceAt(tokenizer, tokenId);
90
+ pieceCache.set(tokenId, piece);
91
+ return piece;
92
+ }
93
+
94
+ // Per-line tab count resets on each emitted newline. `linesEmitted`
95
+ // counts complete lines so the mask can permit EOS once at least one
96
+ // valid record has been produced.
97
+ let tabsOnCurrentLine = 0;
98
+ let linesEmitted = 0;
99
+ let lastGeneratedLen = 0;
100
+
101
+ function reflectEmittedTokens(tokenizer, generatedIds) {
102
+ for (let i = lastGeneratedLen; i < generatedIds.length; i += 1) {
103
+ const piece = cachedPiece(tokenizer, generatedIds[i]);
104
+ // Walk the piece char-by-char so multi-tab/newline pieces stay
105
+ // accurate (e.g., a token piece "\t\n" both completes a line
106
+ // AND resets the tab counter).
107
+ for (let c = 0; c < piece.length; c += 1) {
108
+ const ch = piece[c];
109
+ if (ch === "\n") {
110
+ if (tabsOnCurrentLine > 0) linesEmitted += 1;
111
+ tabsOnCurrentLine = 0;
112
+ } else if (ch === "\t") {
113
+ tabsOnCurrentLine += 1;
114
+ }
115
+ }
116
+ }
117
+ lastGeneratedLen = generatedIds.length;
118
+ }
119
+
120
+ /**
121
+ * Walk a candidate piece's chars and check whether any intermediate
122
+ * state would violate the per-line tab budget. Returns true iff the
123
+ * piece is structurally valid given the current `tabsOnCurrentLine`.
124
+ *
125
+ * Intra-piece state evolution matters because BPE tokens occasionally
126
+ * span line boundaries (e.g., a piece `"\t\n"` ends one line AND
127
+ * starts the next with tabsOnCurrentLine=0). A naive check that just
128
+ * sums the piece's tabs/newlines would misjudge such tokens.
129
+ */
130
+ function pieceFitsBudget(piece, startingTabs, tabsMax) {
131
+ let tabs = startingTabs;
132
+ let sawNewlineWithoutCompleteLine = false;
133
+ for (let c = 0; c < piece.length; c += 1) {
134
+ const ch = piece[c];
135
+ if (ch === "\n") {
136
+ if (tabs < tabsMax) sawNewlineWithoutCompleteLine = true;
137
+ tabs = 0;
138
+ } else if (ch === "\t") {
139
+ tabs += 1;
140
+ if (tabs > tabsMax) return false;
141
+ }
142
+ }
143
+ if (sawNewlineWithoutCompleteLine) return false;
144
+ return true;
145
+ }
146
+
147
+ return function logitMask(logits, context) {
148
+ const tokenizer = (opts.tokenizer ?? context?.tokenizer) || null;
149
+ if (!tokenizer) return;
150
+ reflectEmittedTokens(tokenizer, Array.isArray(context?.generatedIds) ? context.generatedIds : []);
151
+
152
+ const vocabSize = Math.min(logits.length, Number(context?.vocabSize || logits.length));
153
+ for (let id = 0; id < vocabSize; id += 1) {
154
+ if (logits[id] <= -Infinity) continue;
155
+ const piece = cachedPiece(tokenizer, id);
156
+ if (!piece) continue;
157
+ // Fast path: pieces with no tab/newline are always allowed —
158
+ // they're field content (snippet text, digits, decimal points).
159
+ // Avoids the per-char walk for the common case.
160
+ if (piece.indexOf("\t") < 0 && piece.indexOf("\n") < 0) continue;
161
+ if (!pieceFitsBudget(piece, tabsOnCurrentLine, tabsPerLineMax)) {
162
+ logits[id] = -Infinity;
163
+ }
164
+ }
165
+ };
166
+ }
@@ -10,11 +10,6 @@
10
10
 
11
11
  import type { AttentionConfig } from './types.js';
12
12
 
13
- /**
14
- * Required fields that must not be undefined on every AttnConfig instance.
15
- */
16
- export declare const ATTN_CONFIG_REQUIRED_FIELDS: readonly string[];
17
-
18
13
  /**
19
14
  * Validate an AttnConfig object at construction time.
20
15
  *
@@ -6,7 +6,7 @@ import { log } from '../../../../debug/index.js';
6
6
  * entry points. Optional fields (skipInputNorm, tokenIds, kernelPath,
7
7
  * disableRoPE) have defaults in the destructuring and are excluded.
8
8
  */
9
- export const ATTN_CONFIG_REQUIRED_FIELDS = Object.freeze([
9
+ const ATTN_CONFIG_REQUIRED_FIELDS = Object.freeze([
10
10
  'layerIdx',
11
11
  'numTokens',
12
12
  'isPrefill',
@@ -96,6 +96,7 @@ export interface ProjectAttentionQKVResult {
96
96
  kTensor: Tensor;
97
97
  vTensor: Tensor;
98
98
  usedFusedQKV: boolean;
99
+ valueAliasesKey: boolean;
99
100
  }
100
101
 
101
102
  export function projectAttentionQKV(options: ProjectAttentionQKVOptions): Promise<ProjectAttentionQKVResult>;
@@ -115,6 +116,7 @@ export interface ApplyAttentionQKNormOptions {
115
116
  releaseTemporary: (buffer: GPUBuffer) => void;
116
117
  onQNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
117
118
  onKNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
119
+ retainKInput?: boolean;
118
120
  }
119
121
 
120
122
  export function applyAttentionQKNorm(
@@ -157,6 +157,9 @@ async function projectSingleQkvTensor({
157
157
  matmulDebug,
158
158
  releaseTemporary,
159
159
  executionPolicies = null,
160
+ fusedNormWeight = null,
161
+ fusedNormEps = null,
162
+ fusedNormOffset = false,
160
163
  }) {
161
164
  const runMatmulForMode = getMatmulRunner(recorder);
162
165
  const layerWeight = layerWeights?.[weightKey];
@@ -178,6 +181,9 @@ async function projectSingleQkvTensor({
178
181
  outputDtype: matmulOutputDtype,
179
182
  matmulDebug,
180
183
  executionPolicies,
184
+ normWeight: fusedNormWeight,
185
+ rmsNormEps: fusedNormEps,
186
+ rmsNormOffset: fusedNormOffset,
181
187
  });
182
188
  } finally {
183
189
  releaseOwnedWeightBuffer(layerWeight, projBuffer, releaseTemporary);
@@ -269,6 +275,9 @@ async function projectQueryWithOptionalGate({
269
275
  releaseTemporary,
270
276
  attentionOutputGate,
271
277
  executionPolicies = null,
278
+ fusedNormWeight = null,
279
+ fusedNormEps = null,
280
+ fusedNormOffset = false,
272
281
  }) {
273
282
  const qSize = numHeads * headDim;
274
283
  const qWeight = layerWeights?.qProj;
@@ -296,6 +305,9 @@ async function projectQueryWithOptionalGate({
296
305
  lora,
297
306
  matmulDebug,
298
307
  releaseTemporary,
308
+ fusedNormWeight,
309
+ fusedNormEps,
310
+ fusedNormOffset,
299
311
  });
300
312
  return { qTensor, qGateTensor: null };
301
313
  }
@@ -433,6 +445,9 @@ export async function projectAttentionQKV({
433
445
  sharedKTensor = null,
434
446
  sharedVTensor = null,
435
447
  executionPolicies = null,
448
+ fusedNormWeight = null,
449
+ fusedNormEps = null,
450
+ fusedNormOffset = false,
436
451
  }) {
437
452
  const runMatmulForMode = getMatmulRunner(recorder);
438
453
  const runSplitForMode = getSplitRunner(recorder);
@@ -485,6 +500,12 @@ export async function projectAttentionQKV({
485
500
  outputDtype: projectionOutputDtype,
486
501
  matmulDebug,
487
502
  executionPolicies,
503
+ // Forward fused-rmsnorm params so the combined QKV matmul runs the
504
+ // input_norm prologue internally, eliminating the standalone rmsnorm
505
+ // dispatch upstream for layers using the useFusedQKV path.
506
+ normWeight: fusedNormWeight,
507
+ rmsNormEps: fusedNormEps,
508
+ rmsNormOffset: fusedNormOffset,
488
509
  });
489
510
  const split = await runSplitForMode(qkvTensor, {
490
511
  numTokens,
@@ -496,7 +517,14 @@ export async function projectAttentionQKV({
496
517
  if (onFusedQKV) {
497
518
  onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
498
519
  }
499
- return { qTensor: split.Q, qGateTensor: null, kTensor: split.K, vTensor: split.V, usedFusedQKV: true };
520
+ return {
521
+ qTensor: split.Q,
522
+ qGateTensor: null,
523
+ kTensor: split.K,
524
+ vTensor: split.V,
525
+ usedFusedQKV: true,
526
+ valueAliasesKey: false,
527
+ };
500
528
  } catch (error) {
501
529
  if (qkvTensor) {
502
530
  releaseTemporary(qkvTensor.buffer);
@@ -531,6 +559,9 @@ export async function projectAttentionQKV({
531
559
  releaseTemporary,
532
560
  attentionOutputGate,
533
561
  executionPolicies,
562
+ fusedNormWeight,
563
+ fusedNormEps,
564
+ fusedNormOffset,
534
565
  }));
535
566
 
536
567
  if (reuseSharedKV) {
@@ -540,6 +571,7 @@ export async function projectAttentionQKV({
540
571
  kTensor: sharedKTensor,
541
572
  vTensor: sharedVTensor,
542
573
  usedFusedQKV: false,
574
+ valueAliasesKey: false,
543
575
  };
544
576
  }
545
577
 
@@ -562,30 +594,42 @@ export async function projectAttentionQKV({
562
594
  matmulDebug,
563
595
  releaseTemporary,
564
596
  executionPolicies,
597
+ fusedNormWeight,
598
+ fusedNormEps,
599
+ fusedNormOffset,
565
600
  });
566
601
 
567
- vTensor = await projectSingleQkvTensor({
568
- recorder,
569
- normed: projectionInput,
570
- layerWeights,
571
- weightKey: 'vProj',
572
- role: 'v_proj',
573
- outputSize: numKVHeads * headDim,
574
- outputLabel: 'V',
575
- loraKey: 'v_proj',
576
- numTokens,
577
- hiddenSize,
578
- layerIdx,
579
- kernelPath,
580
- matmulOutputDtype: projectionOutputDtype,
581
- getWeightBuffer,
582
- lora,
583
- matmulDebug,
584
- releaseTemporary,
585
- executionPolicies,
586
- });
602
+ let valueAliasesKey = false;
603
+ if (layerWeights.vProj) {
604
+ vTensor = await projectSingleQkvTensor({
605
+ recorder,
606
+ normed: projectionInput,
607
+ layerWeights,
608
+ weightKey: 'vProj',
609
+ role: 'v_proj',
610
+ outputSize: numKVHeads * headDim,
611
+ outputLabel: 'V',
612
+ loraKey: 'v_proj',
613
+ numTokens,
614
+ hiddenSize,
615
+ layerIdx,
616
+ kernelPath,
617
+ matmulOutputDtype: projectionOutputDtype,
618
+ getWeightBuffer,
619
+ lora,
620
+ matmulDebug,
621
+ releaseTemporary,
622
+ executionPolicies,
623
+ fusedNormWeight,
624
+ fusedNormEps,
625
+ fusedNormOffset,
626
+ });
627
+ } else {
628
+ vTensor = kTensor;
629
+ valueAliasesKey = true;
630
+ }
587
631
 
588
- return { qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV: false };
632
+ return { qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV: false, valueAliasesKey };
589
633
  } catch (error) {
590
634
  for (const tensor of [qTensor, qGateTensor]) {
591
635
  if (tensor?.buffer) {
@@ -645,6 +689,7 @@ export async function applyAttentionQKNorm({
645
689
  onQNormApplied = null,
646
690
  onKNormApplied = null,
647
691
  skipKNorm = false,
692
+ retainKInput = false,
648
693
  }) {
649
694
  const runRmsNormForMode = getRmsNormRunner(recorder);
650
695
  let nextQ = qTensor;
@@ -683,7 +728,9 @@ export async function applyAttentionQKNorm({
683
728
  hiddenSize: headDim,
684
729
  rmsNormWeightOffset,
685
730
  });
686
- releaseTemporary(nextK.buffer);
731
+ if (!retainKInput) {
732
+ releaseTemporary(nextK.buffer);
733
+ }
687
734
  nextK = kNormedTensor;
688
735
  if (onKNormApplied) {
689
736
  await onKNormApplied(nextK);