doppler-gpu 0.4.7 → 0.4.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/CHANGELOG.md +64 -0
  2. package/README.md +93 -144
  3. package/models/catalog.json +1575 -0
  4. package/package.json +37 -12
  5. package/src/cli/doppler-cli.js +28 -7
  6. package/src/client/doppler-registry.json +2 -2
  7. package/src/client/runtime/index.js +25 -2
  8. package/src/client/runtime/model-session.d.ts +31 -1
  9. package/src/client/runtime/model-session.js +31 -0
  10. package/src/client/runtime/model-source.d.ts +19 -2
  11. package/src/client/runtime/model-source.js +45 -5
  12. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
  13. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
  14. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
  15. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
  16. package/src/config/kernel-path-loader.js +1 -0
  17. package/src/config/kernels/kernel-ref-digests.js +26 -17
  18. package/src/config/kernels/registry.json +448 -27
  19. package/src/config/merge.js +11 -0
  20. package/src/config/param-validator.js +25 -0
  21. package/src/config/runtime/default.json +3 -1
  22. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
  23. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
  24. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
  25. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
  26. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
  27. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
  28. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
  29. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
  30. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
  31. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
  32. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
  33. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
  34. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
  35. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
  36. package/src/config/runtime/profiles/rerank-stability.json +43 -0
  37. package/src/config/runtime/profiles/rerank-throughput.json +40 -0
  38. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
  39. package/src/config/schema/doppler.schema.js +40 -5
  40. package/src/config/schema/execution-v1.schema.d.ts +41 -0
  41. package/src/config/schema/execution-v1.schema.js +10 -0
  42. package/src/config/schema/index.d.ts +3 -1
  43. package/src/config/schema/speculation-self.schema.d.ts +16 -0
  44. package/src/config/transforms/capability-transform-resolver.js +7 -1
  45. package/src/config/transforms/execution-graph-transforms.js +2 -2
  46. package/src/debug/config.d.ts +4 -0
  47. package/src/experimental/orchestration/index.d.ts +10 -1
  48. package/src/experimental/training/suite.js +1 -1
  49. package/src/formats/rdrr/classification.js +3 -0
  50. package/src/formats/rdrr/integrity.d.ts +0 -2
  51. package/src/formats/rdrr/integrity.js +0 -2
  52. package/src/formats/rdrr/types.d.ts +1 -1
  53. package/src/gpu/kernels/attention.d.ts +2 -0
  54. package/src/gpu/kernels/attention.js +75 -16
  55. package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
  56. package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
  57. package/src/gpu/kernels/fused_ffn.d.ts +32 -0
  58. package/src/gpu/kernels/fused_ffn.js +540 -19
  59. package/src/gpu/kernels/fused_ffn.wgsl +2 -1
  60. package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
  61. package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
  62. package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
  63. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
  64. package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
  65. package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
  66. package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
  67. package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
  68. package/src/gpu/kernels/index.d.ts +21 -0
  69. package/src/gpu/kernels/index.js +13 -0
  70. package/src/gpu/kernels/linear-attention-core.js +413 -15
  71. package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
  72. package/src/gpu/kernels/lm-head-argmax.js +153 -57
  73. package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
  74. package/src/gpu/kernels/lm-head-select-logits.js +186 -0
  75. package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
  76. package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
  77. package/src/gpu/kernels/matmul-selection.js +5 -1
  78. package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
  79. package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
  80. package/src/gpu/kernels/rmsnorm-pair.js +113 -0
  81. package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
  82. package/src/gpu/kernels/rmsnorm-stats.js +250 -0
  83. package/src/gpu/kernels/silu.wgsl +7 -1
  84. package/src/gpu/kernels/silu_f16.wgsl +7 -1
  85. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
  86. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
  87. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
  88. package/src/gpu/tensor.d.ts +1 -0
  89. package/src/index.d.ts +1 -1
  90. package/src/index.js +1 -1
  91. package/src/inference/browser-harness-suite-helpers.js +1 -1
  92. package/src/inference/browser-harness-text-helpers.js +251 -20
  93. package/src/inference/browser-harness.js +221 -3
  94. package/src/inference/kv-cache/base.js +42 -1
  95. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
  96. package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
  97. package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
  98. package/src/inference/pipelines/text/attention/projections.js +174 -4
  99. package/src/inference/pipelines/text/attention/record.js +44 -21
  100. package/src/inference/pipelines/text/attention/run.js +18 -3
  101. package/src/inference/pipelines/text/config.d.ts +18 -0
  102. package/src/inference/pipelines/text/config.js +100 -0
  103. package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
  104. package/src/inference/pipelines/text/execution-plan.js +37 -0
  105. package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
  106. package/src/inference/pipelines/text/execution-v1.js +122 -8
  107. package/src/inference/pipelines/text/ffn/dense.js +239 -21
  108. package/src/inference/pipelines/text/ffn/standard.js +121 -38
  109. package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
  110. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  111. package/src/inference/pipelines/text/generator-runtime.js +1 -0
  112. package/src/inference/pipelines/text/generator-steps.js +2 -2
  113. package/src/inference/pipelines/text/generator.d.ts +18 -1
  114. package/src/inference/pipelines/text/generator.js +297 -19
  115. package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
  116. package/src/inference/pipelines/text/init.d.ts +22 -3
  117. package/src/inference/pipelines/text/init.js +2 -2
  118. package/src/inference/pipelines/text/layer.js +87 -8
  119. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
  120. package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
  121. package/src/inference/pipelines/text/linear-attention.js +173 -16
  122. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  123. package/src/inference/pipelines/text/logits/index.js +62 -1
  124. package/src/inference/pipelines/text/model-load.js +6 -0
  125. package/src/inference/pipelines/text/ops.js +22 -0
  126. package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
  127. package/src/inference/pipelines/text/probes.js +9 -4
  128. package/src/inference/pipelines/text/state.d.ts +4 -4
  129. package/src/inference/pipelines/text/state.js +4 -0
  130. package/src/inference/pipelines/text/types.d.ts +27 -2
  131. package/src/inference/pipelines/text.d.ts +21 -1
  132. package/src/inference/pipelines/text.js +40 -14
  133. package/src/inference/tokenizers/bundled.js +45 -23
  134. package/src/inference/tokenizers/types.d.ts +7 -4
  135. package/src/loader/doppler-loader.js +25 -2
  136. package/src/loader/index.d.ts +3 -1
  137. package/src/loader/index.js +2 -0
  138. package/src/loader/loader-types.d.ts +3 -1
  139. package/src/loader/manifest-config.d.ts +18 -0
  140. package/src/loader/manifest-config.js +11 -3
  141. package/src/loader/shard-cache.d.ts +2 -0
  142. package/src/loader/shard-cache.js +29 -1
  143. package/src/models/gemma4.js +8 -0
  144. package/src/rules/inference/capability-transforms.rules.json +145 -20
  145. package/src/rules/inference/ffn.rules.json +33 -0
  146. package/src/rules/kernels/fused-ffn.rules.json +1 -1
  147. package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
  148. package/src/rules/kernels/matmul.rules.json +1 -0
  149. package/src/rules/kernels/silu.rules.json +1 -0
  150. package/src/rules/rule-registry.js +2 -0
  151. package/src/storage/artifact-storage-context.js +36 -0
  152. package/src/tooling/node-browser-command-runner.js +42 -7
  153. package/src/tooling/opfs-cache.d.ts +21 -1
  154. package/src/tooling/opfs-cache.js +168 -32
  155. package/src/tooling-exports/storage.d.ts +1 -1
  156. package/src/tooling-exports/storage.js +1 -1
  157. package/src/types/gpu.d.ts +25 -0
  158. package/src/version.js +1 -1
  159. package/tests/kernels/browser/kernel-suite.js +1593 -0
  160. package/tests/kernels/browser/test-page.js +2629 -0
  161. package/tests/kernels/harness/benchmark.js +225 -0
  162. package/tests/kernels/harness/buffer-utils.js +88 -0
  163. package/tests/kernels/harness/tolerance.js +174 -0
  164. package/tests/kernels/reference/adam.js +15 -0
  165. package/tests/kernels/reference/attention.js +158 -0
  166. package/tests/kernels/reference/backward.js +96 -0
  167. package/tests/kernels/reference/bias-add.js +20 -0
  168. package/tests/kernels/reference/conv2d.js +58 -0
  169. package/tests/kernels/reference/cross-entropy.js +26 -0
  170. package/tests/kernels/reference/dequant.js +312 -0
  171. package/tests/kernels/reference/embed.js +15 -0
  172. package/tests/kernels/reference/gather.js +61 -0
  173. package/tests/kernels/reference/gelu.js +37 -0
  174. package/tests/kernels/reference/groupnorm.js +36 -0
  175. package/tests/kernels/reference/index.js +89 -0
  176. package/tests/kernels/reference/layernorm.js +22 -0
  177. package/tests/kernels/reference/matmul.js +57 -0
  178. package/tests/kernels/reference/modulate.js +24 -0
  179. package/tests/kernels/reference/moe-gather.js +83 -0
  180. package/tests/kernels/reference/pixel-shuffle.js +20 -0
  181. package/tests/kernels/reference/residual.js +55 -0
  182. package/tests/kernels/reference/rmsnorm.js +52 -0
  183. package/tests/kernels/reference/rope.js +79 -0
  184. package/tests/kernels/reference/sample.js +97 -0
  185. package/tests/kernels/reference/scatter-add.js +58 -0
  186. package/tests/kernels/reference/silu.js +52 -0
  187. package/tests/kernels/reference/softmax.js +86 -0
  188. package/tests/kernels/reference/split-qkv.js +58 -0
  189. package/tests/kernels/reference/topk.js +88 -0
  190. package/tests/kernels/reference/upsample2d.js +37 -0
  191. package/BRANDING.md +0 -14
  192. package/SECURITY.md +0 -19
  193. package/assets/doppler-runtime-map.svg +0 -82
  194. package/src/bootstrap.js +0 -30
  195. package/src/cli/doppler-cli.d.ts +0 -46
  196. package/src/cli/doppler-quickstart.d.ts +0 -45
  197. package/src/cli/doppler-serve.d.ts +0 -210
  198. package/src/cli/runtime-profiles.d.ts +0 -40
  199. package/src/client/doppler-api.browser.d.ts +0 -1
  200. package/src/client/doppler-api.browser.js +0 -25
  201. package/src/client/doppler-registry.d.ts +0 -54
  202. package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
  203. package/src/client/runtime/source-runtime.d.ts +0 -26
  204. package/src/client/runtime/storage.d.ts +0 -5
  205. package/src/config/README.md +0 -61
  206. package/src/config/conversion/README.md +0 -62
  207. package/src/config/goal-completion-matrix.json +0 -552
  208. package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
  209. package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
  210. package/src/config/kernels/moe/mixtral.paths.json +0 -46
  211. package/src/config/kernels/registry.d.ts +0 -51
  212. package/src/config/merge-contract-check.d.ts +0 -16
  213. package/src/config/merge-contract-check.js +0 -319
  214. package/src/config/merge-helpers.d.ts +0 -60
  215. package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
  216. package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
  217. package/src/config/param-categories.d.ts +0 -8
  218. package/src/config/param-validator.d.ts +0 -33
  219. package/src/config/platforms/loader.d.ts +0 -38
  220. package/src/config/quantization-contract-check.d.ts +0 -12
  221. package/src/config/quantization-contract-check.js +0 -91
  222. package/src/config/runtime/README.md +0 -28
  223. package/src/config/runtime-merge.d.ts +0 -5
  224. package/src/config/schema/claim-evidence-contract.schema.json +0 -93
  225. package/src/config/schema/command-surface-contract.schema.json +0 -81
  226. package/src/config/schema/goal-completion-matrix.schema.json +0 -223
  227. package/src/config/schema/model-artifact-contract.schema.json +0 -119
  228. package/src/config/schema/platform.schema.js +0 -1
  229. package/src/config/schema/policy-schema-registry.json +0 -31
  230. package/src/config/schema/subsystem-support-contract.schema.json +0 -48
  231. package/src/config/support-tiers/subsystems.json +0 -502
  232. package/src/config/supported-operations.d.ts +0 -14
  233. package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
  234. package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
  235. package/src/converter/conversion-plan.d.ts +0 -56
  236. package/src/converter/index.d.ts +0 -98
  237. package/src/converter/index.js +0 -59
  238. package/src/converter/parsers/diffusion.d.ts +0 -50
  239. package/src/converter/parsers/gguf.d.ts +0 -31
  240. package/src/converter/parsers/index.d.ts +0 -21
  241. package/src/converter/parsers/index.js +0 -12
  242. package/src/converter/parsers/transformer.d.ts +0 -31
  243. package/src/converter/quantization-info.d.ts +0 -38
  244. package/src/converter/rope-config.d.ts +0 -16
  245. package/src/converter/tokenizer-utils.d.ts +0 -12
  246. package/src/debug/reference/README.md +0 -114
  247. package/src/debug/reference/hf_attn_debug.py +0 -114
  248. package/src/debug/reference/hf_embed_check.py +0 -89
  249. package/src/debug/reference/hf_layer_out.py +0 -100
  250. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
  251. package/src/debug/reference/hf_rope_check.py +0 -116
  252. package/src/debug/reference/hf_weights.py +0 -75
  253. package/src/errors/doppler-error.d.ts +0 -20
  254. package/src/experimental/bridge/extension/background.d.ts +0 -14
  255. package/src/experimental/bridge/extension/background.js +0 -168
  256. package/src/experimental/bridge/extension/manifest.json +0 -34
  257. package/src/experimental/bridge/extension-client.d.ts +0 -114
  258. package/src/experimental/bridge/index.d.ts +0 -69
  259. package/src/experimental/bridge/node-bridge.js +0 -24
  260. package/src/experimental/bridge/protocol.d.ts +0 -96
  261. package/src/experimental/browser/gguf-importer.d.ts +0 -136
  262. package/src/experimental/browser/gguf-importer.js +0 -532
  263. package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
  264. package/src/experimental/browser/quantization.d.ts +0 -74
  265. package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
  266. package/src/experimental/browser/shard-io-browser.d.ts +0 -57
  267. package/src/experimental/browser/shard-io-browser.js +0 -89
  268. package/src/experimental/browser/tensor-source-download.d.ts +0 -27
  269. package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
  270. package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
  271. package/src/experimental/distribution/p2p-transport.js +0 -75
  272. package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
  273. package/src/experimental/hotswap/manifest.d.ts +0 -42
  274. package/src/experimental/hotswap/model-swap.js +0 -42
  275. package/src/experimental/hotswap/runtime.d.ts +0 -31
  276. package/src/experimental/training/README.md +0 -164
  277. package/src/experimental/training/artifacts.d.ts +0 -160
  278. package/src/experimental/training/attention-backward.d.ts +0 -30
  279. package/src/experimental/training/attention-forward.d.ts +0 -22
  280. package/src/experimental/training/attention-forward.js +0 -82
  281. package/src/experimental/training/autograd.d.ts +0 -51
  282. package/src/experimental/training/backward-kernels.js +0 -39
  283. package/src/experimental/training/checkpoint-watch.d.ts +0 -8
  284. package/src/experimental/training/checkpoint.d.ts +0 -36
  285. package/src/experimental/training/clip.d.ts +0 -9
  286. package/src/experimental/training/dataloader.d.ts +0 -8
  287. package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
  288. package/src/experimental/training/datasets/gepa-frontier.js +0 -220
  289. package/src/experimental/training/datasets/index.d.ts +0 -24
  290. package/src/experimental/training/datasets/index.js +0 -13
  291. package/src/experimental/training/datasets/jsonl.d.ts +0 -11
  292. package/src/experimental/training/datasets/reploid.d.ts +0 -3
  293. package/src/experimental/training/datasets/reploid.js +0 -36
  294. package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
  295. package/src/experimental/training/datasets/teacher-traces.js +0 -279
  296. package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
  297. package/src/experimental/training/datasets/token-batch.d.ts +0 -21
  298. package/src/experimental/training/datasets/token-batch.js +0 -52
  299. package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
  300. package/src/experimental/training/datasets/translation-pairs.js +0 -49
  301. package/src/experimental/training/distillation/artifacts.d.ts +0 -71
  302. package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
  303. package/src/experimental/training/distillation/dataset.d.ts +0 -59
  304. package/src/experimental/training/distillation/eval.d.ts +0 -34
  305. package/src/experimental/training/distillation/index.d.ts +0 -29
  306. package/src/experimental/training/distillation/runtime.d.ts +0 -20
  307. package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
  308. package/src/experimental/training/distillation/stage-a.d.ts +0 -45
  309. package/src/experimental/training/distillation/stage-b.d.ts +0 -24
  310. package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
  311. package/src/experimental/training/distillation/suite-data.d.ts +0 -45
  312. package/src/experimental/training/export.d.ts +0 -41
  313. package/src/experimental/training/index.d.ts +0 -74
  314. package/src/experimental/training/index.js +0 -63
  315. package/src/experimental/training/lora-pipeline.d.ts +0 -183
  316. package/src/experimental/training/lora.d.ts +0 -19
  317. package/src/experimental/training/loss-scaling.d.ts +0 -21
  318. package/src/experimental/training/loss.d.ts +0 -10
  319. package/src/experimental/training/objectives/base.d.ts +0 -58
  320. package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
  321. package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
  322. package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
  323. package/src/experimental/training/objectives/index.d.ts +0 -12
  324. package/src/experimental/training/objectives/index.js +0 -6
  325. package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
  326. package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
  327. package/src/experimental/training/operator-agent-eval.d.ts +0 -65
  328. package/src/experimental/training/operator-artifacts.d.ts +0 -62
  329. package/src/experimental/training/operator-command.d.ts +0 -5
  330. package/src/experimental/training/operator-eval.d.ts +0 -48
  331. package/src/experimental/training/operator-scoreboard.d.ts +0 -5
  332. package/src/experimental/training/optimizer.d.ts +0 -22
  333. package/src/experimental/training/runner.d.ts +0 -249
  334. package/src/experimental/training/suite.d.ts +0 -299
  335. package/src/experimental/training/tensor-factory.d.ts +0 -9
  336. package/src/experimental/training/trainer.d.ts +0 -89
  337. package/src/experimental/training/ul_dataset.d.ts +0 -47
  338. package/src/experimental/training/ul_schedule.d.ts +0 -6
  339. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
  340. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
  341. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
  342. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
  343. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
  344. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
  345. package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
  346. package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
  347. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
  348. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
  349. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
  350. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
  351. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
  352. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
  353. package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
  354. package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
  355. package/src/experimental/training/workload-packs/registry.json +0 -278
  356. package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
  357. package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
  358. package/src/experimental/training/workloads.d.ts +0 -201
  359. package/src/formats/gguf/types.d.ts +0 -74
  360. package/src/formats/index.d.ts +0 -75
  361. package/src/formats/index.js +0 -19
  362. package/src/formats/litert/types.d.ts +0 -74
  363. package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
  364. package/src/formats/rdrr/distributed/validation.d.ts +0 -23
  365. package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
  366. package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
  367. package/src/formats/safetensors/types.d.ts +0 -67
  368. package/src/formats/tflite/types.d.ts +0 -61
  369. package/src/formats/tokenizer/index.d.ts +0 -5
  370. package/src/formats/tokenizer/index.js +0 -3
  371. package/src/formats/tokenizer/types.d.ts +0 -9
  372. package/src/gpu/kernel-runtime.d.ts +0 -13
  373. package/src/gpu/kernel-selection-cache.d.ts +0 -13
  374. package/src/gpu/kernel-selection-log.d.ts +0 -4
  375. package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
  376. package/src/gpu/kernel-tuner/cache.d.ts +0 -55
  377. package/src/gpu/kernel-tuner/index.d.ts +0 -59
  378. package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
  379. package/src/gpu/kernel-tuner/types.d.ts +0 -101
  380. package/src/gpu/kernel-tuner/types.js +0 -4
  381. package/src/gpu/kernel-tuner.d.ts +0 -33
  382. package/src/gpu/kernels/README.md +0 -166
  383. package/src/gpu/kernels/check-finiteness.d.ts +0 -15
  384. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
  385. package/src/gpu/kernels/check-stop.d.ts +0 -31
  386. package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
  387. package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
  388. package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
  389. package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
  390. package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
  391. package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
  392. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
  393. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
  394. package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
  395. package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
  396. package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
  397. package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
  398. package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
  399. package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
  400. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
  401. package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
  402. package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
  403. package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
  404. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
  405. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
  406. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
  407. package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
  408. package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
  409. package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
  410. package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
  411. package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
  412. package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
  413. package/src/gpu/kernels/constants.d.ts +0 -129
  414. package/src/gpu/kernels/dispatch.d.ts +0 -72
  415. package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
  416. package/src/gpu/kernels/kernel-base.d.ts +0 -33
  417. package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
  418. package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
  419. package/src/gpu/kernels/matmul-selection.d.ts +0 -87
  420. package/src/gpu/kernels/rep-penalty.d.ts +0 -21
  421. package/src/gpu/kernels/rule-matcher.d.ts +0 -30
  422. package/src/gpu/kernels/rule-registry.d.ts +0 -7
  423. package/src/gpu/kernels/types.js +0 -4
  424. package/src/gpu/perf-guards.d.ts +0 -13
  425. package/src/gpu/profiler.d.ts +0 -93
  426. package/src/gpu/readback-utils.d.ts +0 -16
  427. package/src/gpu/readback-utils.js +0 -49
  428. package/src/gpu/submit-probe.d.ts +0 -12
  429. package/src/index-browser.d.ts +0 -2
  430. package/src/index-browser.js +0 -2
  431. package/src/inference/README.md +0 -593
  432. package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
  433. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
  434. package/src/inference/browser-harness-model-helpers.d.ts +0 -10
  435. package/src/inference/browser-harness-report-helpers.d.ts +0 -7
  436. package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
  437. package/src/inference/browser-harness-text-helpers.d.ts +0 -152
  438. package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
  439. package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
  440. package/src/inference/network-evolution.js +0 -79
  441. package/src/inference/pipelines/context.d.ts +0 -21
  442. package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
  443. package/src/inference/pipelines/diffusion/index.d.ts +0 -3
  444. package/src/inference/pipelines/diffusion/index.js +0 -3
  445. package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
  446. package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
  447. package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
  448. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
  449. package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
  450. package/src/inference/pipelines/diffusion/types.js +0 -1
  451. package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
  452. package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
  453. package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
  454. package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
  455. package/src/inference/pipelines/dream/pipeline.js +0 -8
  456. package/src/inference/pipelines/energy/index.d.ts +0 -1
  457. package/src/inference/pipelines/energy/index.js +0 -1
  458. package/src/inference/pipelines/energy/types.js +0 -1
  459. package/src/inference/pipelines/energy-head/index.d.ts +0 -6
  460. package/src/inference/pipelines/energy-head/index.js +0 -6
  461. package/src/inference/pipelines/factory.d.ts +0 -10
  462. package/src/inference/pipelines/index.d.ts +0 -22
  463. package/src/inference/pipelines/index.js +0 -19
  464. package/src/inference/pipelines/registry.d.ts +0 -15
  465. package/src/inference/pipelines/rng.d.ts +0 -2
  466. package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
  467. package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
  468. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
  469. package/src/inference/pipelines/text/attention/index.d.ts +0 -24
  470. package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
  471. package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
  472. package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
  473. package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
  474. package/src/inference/pipelines/text/attention/record.d.ts +0 -36
  475. package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
  476. package/src/inference/pipelines/text/attention/run.d.ts +0 -38
  477. package/src/inference/pipelines/text/attention/types.d.ts +0 -125
  478. package/src/inference/pipelines/text/attention.d.ts +0 -23
  479. package/src/inference/pipelines/text/attention.js +0 -12
  480. package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
  481. package/src/inference/pipelines/text/buffer-types.js +0 -4
  482. package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
  483. package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
  484. package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
  485. package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
  486. package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
  487. package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
  488. package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
  489. package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
  490. package/src/inference/pipelines/text/embed.d.ts +0 -69
  491. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
  492. package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
  493. package/src/inference/pipelines/text/failure-reduction.js +0 -180
  494. package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
  495. package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
  496. package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
  497. package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
  498. package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
  499. package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
  500. package/src/inference/pipelines/text/ffn.d.ts +0 -31
  501. package/src/inference/pipelines/text/ffn.js +0 -18
  502. package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
  503. package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
  504. package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
  505. package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
  506. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
  507. package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
  508. package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
  509. package/src/inference/pipelines/text/index.d.ts +0 -5
  510. package/src/inference/pipelines/text/index.js +0 -6
  511. package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
  512. package/src/inference/pipelines/text/layer.d.ts +0 -118
  513. package/src/inference/pipelines/text/load-errors.d.ts +0 -21
  514. package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
  515. package/src/inference/pipelines/text/logits/types.js +0 -4
  516. package/src/inference/pipelines/text/logits.d.ts +0 -27
  517. package/src/inference/pipelines/text/logits.js +0 -16
  518. package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
  519. package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
  520. package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
  521. package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
  522. package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
  523. package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
  524. package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
  525. package/src/inference/pipelines/text/operator-events.d.ts +0 -125
  526. package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
  527. package/src/inference/pipelines/text/ops.d.ts +0 -222
  528. package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
  529. package/src/inference/pipelines/text/probes.d.ts +0 -28
  530. package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
  531. package/src/inference/pipelines/text/stage-names.d.ts +0 -88
  532. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
  533. package/src/inference/pipelines/text/types.js +0 -4
  534. package/src/inference/tokenizers/base.d.ts +0 -44
  535. package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
  536. package/src/inference/tokenizers/bpe.d.ts +0 -27
  537. package/src/inference/tokenizers/bundled.d.ts +0 -67
  538. package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
  539. package/src/inference/tokenizers/types.js +0 -7
  540. package/src/loader/embedding-limit-preflight.d.ts +0 -31
  541. package/src/loader/per-layer-input-loader.d.ts +0 -30
  542. package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
  543. package/src/loader/tensors/source-transform.d.ts +0 -65
  544. package/src/memory/address-table.d.ts +0 -96
  545. package/src/rules/execution-rules-contract-check.d.ts +0 -21
  546. package/src/rules/layer-pattern-contract-check.d.ts +0 -17
  547. package/src/rules/rule-registry.d.ts +0 -77
  548. package/src/storage/artifact-storage-context.d.ts +0 -63
  549. package/src/storage/backends/idb-store.d.ts +0 -52
  550. package/src/storage/backends/memory-store.d.ts +0 -36
  551. package/src/storage/backends/opfs-store.d.ts +0 -41
  552. package/src/storage/blake3.d.ts +0 -17
  553. package/src/storage/download-types.d.ts +0 -157
  554. package/src/storage/downloader.d.ts +0 -103
  555. package/src/storage/index.d.ts +0 -274
  556. package/src/storage/index.js +0 -201
  557. package/src/storage/preflight.d.ts +0 -144
  558. package/src/storage/preflight.js +0 -360
  559. package/src/storage/quickstart-downloader.d.ts +0 -157
  560. package/src/storage/quickstart-downloader.js +0 -289
  561. package/src/storage/source-artifact-store.d.ts +0 -59
  562. package/src/sw.js +0 -184
  563. package/src/tooling/command-api-constants.d.ts +0 -9
  564. package/src/tooling/command-api-family-normalizers.d.ts +0 -10
  565. package/src/tooling/command-api-helpers.d.ts +0 -23
  566. package/src/tooling/command-runner-shared.d.ts +0 -89
  567. package/src/tooling/conversion-config-materializer.d.ts +0 -14
  568. package/src/tooling/conversion-config-materializer.js +0 -96
  569. package/src/tooling/diagnose-runner.d.ts +0 -28
  570. package/src/tooling/hf-registry-utils.d.ts +0 -55
  571. package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
  572. package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
  573. package/src/tooling/kernel-path-builder/index.d.ts +0 -65
  574. package/src/tooling/kernel-path-builder/index.js +0 -1361
  575. package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
  576. package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
  577. package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
  578. package/src/tooling/lean-execution-contract-runner.js +0 -158
  579. package/src/tooling/lean-execution-contract.d.ts +0 -16
  580. package/src/tooling/lean-execution-contract.js +0 -81
  581. package/src/tooling/litert-package-runtime.d.ts +0 -46
  582. package/src/tooling/node-convert-worker-pool.d.ts +0 -16
  583. package/src/tooling/node-convert-worker.d.ts +0 -1
  584. package/src/tooling/node-converter.d.ts +0 -1
  585. package/src/tooling/node-file-fetch.d.ts +0 -1
  586. package/src/tooling/node-source-runtime.d.ts +0 -24
  587. package/src/tooling/node-webgpu.d.ts +0 -17
  588. package/src/tooling/precision-replay-math.js +0 -174
  589. package/src/tooling/program-bundle-host.d.ts +0 -16
  590. package/src/tooling/program-bundle-host.js +0 -20
  591. package/src/tooling/runtime-input-composition.d.ts +0 -37
  592. package/src/tooling/source-package-profiles.d.ts +0 -50
  593. package/src/tooling/source-runtime-converter-config.d.ts +0 -19
  594. package/src/tooling/source-runtime-materializer.d.ts +0 -6
  595. package/src/tooling/source-runtime-materializer.js +0 -89
  596. package/src/tooling-exports.browser.d.ts +0 -7
  597. package/src/types/chrome.d.ts +0 -36
  598. package/src/types/chrome.js +0 -1
  599. package/src/types/index.d.ts +0 -1
  600. package/src/types/index.js +0 -1
  601. package/src/utils/array-buffer.d.ts +0 -4
  602. package/src/utils/canonical-hash.d.ts +0 -7
  603. package/src/utils/clone-json.d.ts +0 -1
  604. package/src/utils/hf-resolve-url.d.ts +0 -16
  605. package/src/utils/index.d.ts +0 -10
  606. package/src/utils/index.js +0 -10
  607. package/src/utils/load-json.d.ts +0 -5
  608. package/src/utils/plain-object.d.ts +0 -1
  609. package/src/utils/sha256.d.ts +0 -5
  610. package/src/utils/stable-sort-object.d.ts +0 -1
@@ -25,12 +25,26 @@ import { embed, isRangeBackedCpuEmbeddingSource, normalizeRangeBytes, decodeRang
25
25
  import { selectRuleValue } from '../../../rules/rule-registry.js';
26
26
  import { getDevice } from '../../../gpu/device.js';
27
27
  import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
28
+ import { f16ToF32 } from '../../../loader/dtype-utils.js';
28
29
 
29
30
  const pleRuntimeCache = new WeakMap();
30
31
  const pleRangeRowCache = new WeakMap();
32
+ let f16ToF32Lookup = null;
31
33
  const PLE_ACTIVATION_STATIC_QMIN = -128;
32
34
  const PLE_ACTIVATION_STATIC_QMAX = 127;
33
35
 
36
+ function getF16ToF32Lookup() {
37
+ if (f16ToF32Lookup) {
38
+ return f16ToF32Lookup;
39
+ }
40
+ const lookup = new Float32Array(1 << 16);
41
+ for (let value = 0; value < lookup.length; value += 1) {
42
+ lookup[value] = f16ToF32(value);
43
+ }
44
+ f16ToF32Lookup = lookup;
45
+ return lookup;
46
+ }
47
+
34
48
  function getPerLayerInputWeights(context) {
35
49
  const weights = context.weights.get('per_layer_inputs');
36
50
  if (!weights || typeof weights !== 'object') {
@@ -625,6 +639,28 @@ function getPleHotCachePolicy(sessionConfig) {
625
639
  return { mode: 'prepared_tokens', maxTokens, maxBytes, outputDtype };
626
640
  }
627
641
 
642
+ export function resolvePleHotVocabularyCapacity({
643
+ maxTokens,
644
+ maxBytes,
645
+ numLayers,
646
+ hiddenSize,
647
+ bytesPerElement,
648
+ vocabSize,
649
+ }) {
650
+ const bytesPerHotRow = numLayers * hiddenSize * bytesPerElement;
651
+ const tokenIndexMapBytes = vocabSize * Uint32Array.BYTES_PER_ELEMENT;
652
+ const tableBudgetBytes = Math.max(0, maxBytes - tokenIndexMapBytes);
653
+ const maxTableRows = bytesPerHotRow > 0
654
+ ? Math.floor(tableBudgetBytes / bytesPerHotRow)
655
+ : 0;
656
+ const maxHotTokens = Math.max(0, Math.min(maxTokens, maxTableRows - 1));
657
+ return {
658
+ maxHotTokens,
659
+ bytesPerHotRow,
660
+ tokenIndexMapBytes,
661
+ };
662
+ }
663
+
628
664
  export function getPleHotVocabularyRuntime(context) {
629
665
  const perLayerInputWeights = context?.weights?.get?.('per_layer_inputs');
630
666
  if (!perLayerInputWeights || typeof perLayerInputWeights !== 'object') {
@@ -1095,19 +1131,32 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1095
1131
  return null;
1096
1132
  }
1097
1133
 
1134
+ const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
1135
+ const capacity = resolvePleHotVocabularyCapacity({
1136
+ maxTokens: policy.maxTokens,
1137
+ maxBytes: policy.maxBytes,
1138
+ numLayers,
1139
+ hiddenSize: hiddenSizePerLayerInput,
1140
+ bytesPerElement,
1141
+ vocabSize: vocabSizePerLayerInput,
1142
+ });
1143
+ if (capacity.maxHotTokens <= 0) {
1144
+ return null;
1145
+ }
1146
+
1098
1147
  const tokenizer = context?.tokenizer ?? null;
1099
1148
  const tokenizerHotTokenIds = typeof tokenizer?.getHotTokenIds === 'function'
1100
- ? tokenizer.getHotTokenIds(policy.maxTokens)
1149
+ ? tokenizer.getHotTokenIds(capacity.maxHotTokens)
1101
1150
  : null;
1102
1151
  const seedTokenIds = resolvePleHotVocabularySeedTokenIds(
1103
1152
  context,
1104
- policy.maxTokens,
1153
+ capacity.maxHotTokens,
1105
1154
  vocabSizePerLayerInput
1106
1155
  );
1107
1156
  const hotTokenIds = mergePleHotVocabularyTokenIds(
1108
1157
  seedTokenIds,
1109
1158
  tokenizerHotTokenIds,
1110
- policy.maxTokens,
1159
+ capacity.maxHotTokens,
1111
1160
  vocabSizePerLayerInput
1112
1161
  );
1113
1162
  if (hotTokenIds.length === 0) {
@@ -1119,7 +1168,8 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1119
1168
  const cached = perLayerInputWeights.embedTokensPerLayerHotRuntime ?? null;
1120
1169
  if (
1121
1170
  cached
1122
- && cached.maxTokens === policy.maxTokens
1171
+ && cached.maxTokens === capacity.maxHotTokens
1172
+ && cached.maxBytes === policy.maxBytes
1123
1173
  && cached.outputDtype === policy.outputDtype
1124
1174
  && cached.vocabSize === vocabSizePerLayerInput
1125
1175
  && cached.numLayers === numLayers
@@ -1136,13 +1186,13 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1136
1186
 
1137
1187
  const sourceDtype = requirePleSourceDtype(
1138
1188
  embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
1139
- 'Gemma 4 hot vocabulary cache',
1140
- [policy.outputDtype]
1189
+ 'Gemma 4 hot vocabulary cache'
1141
1190
  );
1142
- if (sourceDtype !== policy.outputDtype) {
1191
+ const expandsF16ToF32 = sourceDtype === 'f16' && policy.outputDtype === 'f32';
1192
+ if (sourceDtype !== policy.outputDtype && !expandsF16ToF32) {
1143
1193
  throw new Error(
1144
- `Gemma 4 hot vocabulary cache requires source dtype "${policy.outputDtype}" for zero-copy row packing; ` +
1145
- `got "${sourceDtype}".`
1194
+ `Gemma 4 hot vocabulary cache cannot convert source dtype "${sourceDtype}" ` +
1195
+ `to output dtype "${policy.outputDtype}".`
1146
1196
  );
1147
1197
  }
1148
1198
 
@@ -1151,7 +1201,6 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1151
1201
  throw new Error('No GPU device available for Gemma 4 hot vocabulary cache.');
1152
1202
  }
1153
1203
 
1154
- const bytesPerElement = policy.outputDtype === 'f16' ? 2 : 4;
1155
1204
  const totalPerLayerHiddenSize = numLayers * hiddenSizePerLayerInput;
1156
1205
  const sentinelIndex = hotTokenIds.length;
1157
1206
  const hotRowCount = sentinelIndex + 1;
@@ -1193,6 +1242,10 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1193
1242
  }
1194
1243
 
1195
1244
  const { sourceRowBytes } = getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize);
1245
+ const expandedF32Row = expandsF16ToF32
1246
+ ? new Float32Array(totalPerLayerHiddenSize)
1247
+ : null;
1248
+ const f16Lookup = expandsF16ToF32 ? getF16ToF32Lookup() : null;
1196
1249
  for (let hotIndex = 0; hotIndex < hotTokenIds.length; hotIndex += 1) {
1197
1250
  const tokenId = hotTokenIds[hotIndex];
1198
1251
  const chunk = normalizeRangeBytes(
@@ -1211,6 +1264,21 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1211
1264
  hiddenSizePerLayerInput * bytesPerElement
1212
1265
  );
1213
1266
  }
1267
+ } else if (sourceDtype === 'f16') {
1268
+ const sourceWords = new Uint16Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1269
+ for (let valueIdx = 0; valueIdx < totalPerLayerHiddenSize; valueIdx += 1) {
1270
+ expandedF32Row[valueIdx] = f16Lookup[sourceWords[valueIdx]];
1271
+ }
1272
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
1273
+ const sourceStart = layerIdx * hiddenSizePerLayerInput;
1274
+ device.queue.writeBuffer(
1275
+ splitTables[layerIdx].buffer,
1276
+ hotIndex * hiddenSizePerLayerInput * bytesPerElement,
1277
+ expandedF32Row.buffer,
1278
+ sourceStart * bytesPerElement,
1279
+ hiddenSizePerLayerInput * bytesPerElement
1280
+ );
1281
+ }
1214
1282
  } else {
1215
1283
  const sourceValues = new Float32Array(chunk.buffer, chunk.byteOffset, totalPerLayerHiddenSize);
1216
1284
  for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
@@ -1235,7 +1303,8 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1235
1303
 
1236
1304
  const runtime = {
1237
1305
  mode: 'tokenizer_scores',
1238
- maxTokens: policy.maxTokens,
1306
+ maxTokens: capacity.maxHotTokens,
1307
+ maxBytes: policy.maxBytes,
1239
1308
  outputDtype: policy.outputDtype,
1240
1309
  vocabSize: vocabSizePerLayerInput,
1241
1310
  numLayers,
@@ -164,8 +164,8 @@ async function readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer) {
164
164
  return values;
165
165
  }
166
166
 
167
- function formatProbeStats(stats) {
168
- return [
167
+ function formatProbeStats(stats, firstNonFiniteIndex = -1) {
168
+ const fields = [
169
169
  `min=${formatProbeNumber(stats.min)}`,
170
170
  `max=${formatProbeNumber(stats.max)}`,
171
171
  `mean=${formatProbeNumber(stats.mean)}`,
@@ -175,7 +175,11 @@ function formatProbeStats(stats) {
175
175
  `nan=${stats.nanCount}`,
176
176
  `inf=${stats.infCount}`,
177
177
  `zero=${stats.zeroCount}`,
178
- ].join(', ');
178
+ ];
179
+ if (firstNonFiniteIndex >= 0) {
180
+ fields.push(`firstNonFinite=${firstNonFiniteIndex}`);
181
+ }
182
+ return fields.join(', ');
179
183
  }
180
184
 
181
185
 
@@ -267,7 +271,8 @@ export async function runProbes(stage, buffer, options) {
267
271
  let statsText = null;
268
272
  if (includeStats) {
269
273
  const row = await readTokenRow(buffer, tokenIdx, hiddenSize, dtype, isCpuBuffer);
270
- statsText = formatProbeStats(computeArrayStats(row));
274
+ const firstNonFiniteIndex = row.findIndex((value) => !Number.isFinite(value));
275
+ statsText = formatProbeStats(computeArrayStats(row), firstNonFiniteIndex);
271
276
  }
272
277
 
273
278
  const values = [];
@@ -28,14 +28,13 @@ import type { DopplerLoader } from '../../../loader/doppler-loader.js';
28
28
  import type { LoadedEmbeddingPostprocessor } from '../../../loader/final-weights-loader.js';
29
29
  import type { CompiledLayerPipeline } from './layer-plan.js';
30
30
  import type { ExecutionPlanState } from './execution-plan.js';
31
- import type { CompiledExecutionV1RuntimeState } from './execution-v1.js';
31
+ import type { ExecutionV1CompiledState } from './execution-v1.js';
32
32
  import type { LoRAAdapter } from './lora.js';
33
33
  import type { RuntimeConfigSchema, KernelPathSchema } from '../../../config/schema/index.js';
34
34
  import type { WeightDebugFlags } from './weights.js';
35
35
  import type { LogitsDebugFlags } from './logits/index.js';
36
36
  import type { KernelPathSource } from '../../../config/kernel-path-loader.js';
37
- import type { EmulationContext } from '/proto/simulator/index.js';
38
- import type { PipelineStorageContext } from './init.js';
37
+ import type { EmulationContext, PipelineStorageContext } from './init.js';
39
38
  import type { LinearAttentionRuntime } from './linear-attention.js';
40
39
  import type { PerLayerInputWeights } from '../../../loader/loader-types.js';
41
40
 
@@ -60,7 +59,7 @@ export class PipelineState {
60
59
  resolvedKernelPath: KernelPathSchema | null;
61
60
  kernelPathSource: KernelPathSource;
62
61
  executionPlanState: ExecutionPlanState | null;
63
- executionV1State: CompiledExecutionV1RuntimeState | null;
62
+ executionV1State: ExecutionV1CompiledState | null;
64
63
  disableRecordedLogits: boolean;
65
64
  disableFusedDecode: boolean;
66
65
 
@@ -79,6 +78,7 @@ export class PipelineState {
79
78
 
80
79
  // DopplerLoader instance
81
80
  dopplerLoader: DopplerLoader | null;
81
+ ownsDopplerLoader: boolean;
82
82
 
83
83
  // GPU context
84
84
  gpuContext: { device?: GPUDevice } | null;
@@ -75,6 +75,7 @@ export class PipelineState {
75
75
  // DopplerLoader instance
76
76
 
77
77
  this.dopplerLoader = null;
78
+ this.ownsDopplerLoader = false;
78
79
 
79
80
  // GPU context
80
81
 
@@ -111,6 +112,9 @@ export class PipelineState {
111
112
  decodeReadbackCleanupMs: 0,
112
113
  decodeReadbackCopyMs: 0,
113
114
  prefillRecordMs: 0,
115
+ prefillRecordOps: 0,
116
+ prefillRecordPasses: 0,
117
+ prefillRecordOpLabels: {},
114
118
  prefillSubmitWaitMs: 0,
115
119
  prefillProfileSteps: [],
116
120
  decodeProfileSteps: [],
@@ -46,6 +46,10 @@ export interface KVCacheSnapshot {
46
46
  linearAttention?: LinearAttentionRuntime | null;
47
47
  }
48
48
 
49
+ export interface WorkloadPhaseTiming {
50
+ [key: string]: number | string | boolean | null | Record<string, number> | WorkloadPhaseTiming | WorkloadPhaseTiming[];
51
+ }
52
+
49
53
  export interface AdvanceEmbeddingResult {
50
54
  embedding: Float32Array;
51
55
  embeddingMode: 'last' | 'mean';
@@ -175,6 +179,12 @@ export interface LayerContext {
175
179
  };
176
180
  captureConfig?: Record<string, unknown>;
177
181
  } | null;
182
+ /** Opt into fused gate/up GeGLU prefill path when runtime profile enables it. */
183
+ useFusedGateUpGelu?: boolean;
184
+ /** Opt into large-batch f16-weight/f32-activation fused gate/up prefill. */
185
+ useLargeBatchF16F32FusedGateUp?: boolean;
186
+ /** Skip KV cache writes for hidden-state-only prefill routes. */
187
+ skipKVCacheWrites?: boolean;
178
188
  }
179
189
 
180
190
  /**
@@ -408,6 +418,8 @@ export interface LogitsStepResult {
408
418
  export interface PrefillResult extends KVCacheSnapshot {
409
419
  /** Finalized logits for the next token after prefill */
410
420
  logits: Float32Array;
421
+
422
+ phase?: WorkloadPhaseTiming | null;
411
423
  }
412
424
 
413
425
  /**
@@ -425,6 +437,8 @@ export interface PrefillEmbeddingResult extends KVCacheSnapshot {
425
437
 
426
438
  /** Pooling mode used to construct embedding */
427
439
  embeddingMode: 'last' | 'mean';
440
+
441
+ phase?: WorkloadPhaseTiming | null;
428
442
  }
429
443
 
430
444
  /**
@@ -485,6 +499,7 @@ export interface LayerWeights {
485
499
  kProj: LayerWeightBuffer;
486
500
  vProj?: LayerWeightBuffer;
487
501
  oProj: LayerWeightBuffer;
502
+ qGateProj?: LayerWeightBuffer | null;
488
503
  convInProj?: LayerWeightBuffer;
489
504
  convKernel?: LayerWeightBuffer;
490
505
  convOutProj?: LayerWeightBuffer;
@@ -492,8 +507,12 @@ export interface LayerWeights {
492
507
  qkvProj?: GPUBuffer | WeightBuffer | null;
493
508
  /** Sizes for splitting fused QKV output: [qSize, kSize, vSize] in elements */
494
509
  qkvSizes?: [number, number, number];
495
- /** Data type of fused QKV weights (f16 or f32) */
496
- qkvDtype?: 'f16' | 'f32';
510
+ /** Data type of fused QKV weights (f16, f32, or q4k) */
511
+ qkvDtype?: 'f16' | 'f32' | 'q4k';
512
+ /** Fused linear-attention A/B projection weight for decode-only A+B projection fusion. */
513
+ linearABProj?: WeightBuffer | null;
514
+ /** Fused linear-attention QKV/Z projection weight for decode-only QKV+Z projection fusion. */
515
+ linearQKVZProj?: WeightBuffer | null;
497
516
 
498
517
  // FFN (dense layers)
499
518
  postAttentionNorm?: GPUBuffer | Float32Array;
@@ -610,6 +629,10 @@ export interface PipelineStats {
610
629
  decodeRecordOps?: number;
611
630
  decodeRecordPasses?: number;
612
631
  decodeRecordOpLabels?: Record<string, number>;
632
+ prefillRecordMs?: number;
633
+ prefillRecordOps?: number;
634
+ prefillRecordPasses?: number;
635
+ prefillRecordOpLabels?: Record<string, number>;
613
636
  uniformCache?: UniformCacheStats | null;
614
637
  decodeSubmitWaitMs?: number;
615
638
  decodeReadbackWaitMs?: number;
@@ -640,6 +663,7 @@ export interface PipelineStats {
640
663
  activationDtype: string;
641
664
  readbackInterval: number | null;
642
665
  readbackMode: string | null;
666
+ maxBatchDecodeTokens?: number | null;
643
667
  batchSize: number;
644
668
  stopCheckMode: string;
645
669
  disableCommandBatching?: boolean;
@@ -654,6 +678,7 @@ export interface PipelineStats {
654
678
  activationDtype: string;
655
679
  readbackInterval: number | null;
656
680
  readbackMode: string | null;
681
+ maxBatchDecodeTokens?: number | null;
657
682
  batchSize: number;
658
683
  stopCheckMode: string;
659
684
  disableCommandBatching?: boolean;
@@ -15,7 +15,7 @@ import { PipelineState } from './text/state.js';
15
15
  import { PipelineGenerator } from './text/generator.js';
16
16
  import type { Manifest } from './text/config.js';
17
17
  import type { WeightLoadResult, PipelineContexts } from './text/init.js';
18
- import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats } from './text/types.js';
18
+ import type { GenerateOptions, KVCacheSnapshot, LogitsStepResult, PrefillResult, PrefillEmbeddingResult, AdvanceEmbeddingResult, LayerWeights, ExpertWeights, RouterWeights, GenerationResult, PipelineStats, BatchingStats, WorkloadPhaseTiming } from './text/types.js';
19
19
  import type { ChatMessage } from './text/chat-format.js';
20
20
  import type { LoRAAdapter } from './text/lora.js';
21
21
  import type { DiffusionPipeline } from './diffusion/pipeline.js';
@@ -120,6 +120,7 @@ export declare class InferencePipeline extends PipelineState {
120
120
  tokens: number[];
121
121
  seqLen: number;
122
122
  embeddingMode: string;
123
+ phase?: WorkloadPhaseTiming | null;
123
124
  }>;
124
125
 
125
126
  embedBatch(prompts: string[], options?: GenerateOptions): Promise<Array<{
@@ -131,6 +132,25 @@ export declare class InferencePipeline extends PipelineState {
131
132
 
132
133
  prefillWithLogits(prompt: PromptInput, options?: GenerateOptions): Promise<PrefillResult>;
133
134
 
135
+ prefillWithTokenLogits(prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
136
+ seqLen: number;
137
+ tokens: number[];
138
+ tokenIds: number[];
139
+ logits: Float32Array;
140
+ logitsByTokenId: Record<number, number>;
141
+ phase?: WorkloadPhaseTiming | null;
142
+ }>;
143
+
144
+ prefillWithTokenLogitsFromKV(prefix: KVCacheSnapshot, prompt: PromptInput, tokenIds: readonly number[], options?: GenerateOptions): Promise<{
145
+ seqLen: number;
146
+ prefixTokens: number[];
147
+ tokens: number[];
148
+ tokenIds: number[];
149
+ logits: Float32Array;
150
+ logitsByTokenId: Record<number, number>;
151
+ phase?: WorkloadPhaseTiming | null;
152
+ }>;
153
+
134
154
  applyKVCacheSnapshot(snapshot: KVCacheSnapshot): void;
135
155
 
136
156
  generateWithPrefixKV(
@@ -299,6 +299,8 @@ export class InferencePipeline extends PipelineState {
299
299
  assignStorageContext: true,
300
300
  });
301
301
  this.runtimeConfig = runtimeConfig;
302
+ this.dopplerLoader = contexts.loader || null;
303
+ this.ownsDopplerLoader = contexts.ownsLoader === true;
302
304
  this.runtimeOverrides = contexts.runtimeConfig == null
303
305
  ? null
304
306
  : (typeof structuredClone === 'function'
@@ -400,6 +402,7 @@ export class InferencePipeline extends PipelineState {
400
402
  numLayers: Number(manifest.architecture?.numLayers ?? 0),
401
403
  capabilities,
402
404
  platform,
405
+ useGPU: this.useGPU === true,
403
406
  });
404
407
  if (executionV1Runtime.executionV1State) {
405
408
  this.runtimeConfig = executionV1Runtime.runtimeConfig;
@@ -648,6 +651,7 @@ export class InferencePipeline extends PipelineState {
648
651
  this.modelConfig.perLayerInputsSession ?? null,
649
652
  this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
650
653
  ),
654
+ loader: this.dopplerLoader ?? undefined,
651
655
  onProgress: (info) => {
652
656
  if (info.stage !== 'layers' && info.stage !== 'shards') {
653
657
  log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
@@ -675,7 +679,7 @@ export class InferencePipeline extends PipelineState {
675
679
 
676
680
  this.layerRouterWeights = result.layerRouterWeights;
677
681
 
678
- this.dopplerLoader = getDopplerLoader(this.runtimeConfig.loading);
682
+ this.dopplerLoader = result.loader ?? getDopplerLoader(this.runtimeConfig.loading);
679
683
  this.stats.loadTiming = result.loadTiming ?? this.dopplerLoader?.getLoadTiming?.() ?? null;
680
684
 
681
685
  if ((this.modelConfig).useMoE && this.moeRouter) {
@@ -687,7 +691,11 @@ export class InferencePipeline extends PipelineState {
687
691
  }
688
692
 
689
693
  if (this.useGPU && this.modelConfig) {
690
- fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath);
694
+ const session = this.runtimeConfig?.inference?.session ?? null;
695
+ fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath, {
696
+ allowQ4K: session?.useFusedQKVSplitQKNorm === true
697
+ || session?.useFusedQKVSplitQKNormRoPE === true,
698
+ });
691
699
  }
692
700
 
693
701
  if (this.useGPU && this.modelConfig) {
@@ -1577,17 +1585,23 @@ export class InferencePipeline extends PipelineState {
1577
1585
 
1578
1586
  async embed(prompt, options = {}) {
1579
1587
  assertNotAborted(options?.signal);
1580
- const result = await this.prefillWithEmbedding(prompt, {
1581
- ...options,
1582
- __skipStateSnapshot: true,
1583
- });
1584
- assertNotAborted(options?.signal);
1585
- return {
1586
- embedding: result.embedding,
1587
- tokens: result.tokens,
1588
- seqLen: result.seqLen,
1589
- embeddingMode: result.embeddingMode,
1590
- };
1588
+ this.resetForBatch();
1589
+ try {
1590
+ const result = await this.prefillWithEmbedding(prompt, {
1591
+ ...options,
1592
+ __skipStateSnapshot: true,
1593
+ });
1594
+ assertNotAborted(options?.signal);
1595
+ return {
1596
+ embedding: result.embedding,
1597
+ tokens: result.tokens,
1598
+ seqLen: result.seqLen,
1599
+ embeddingMode: result.embeddingMode,
1600
+ phase: result.phase ?? null,
1601
+ };
1602
+ } finally {
1603
+ this.resetForBatch();
1604
+ }
1591
1605
  }
1592
1606
 
1593
1607
  async embedBatch(prompts, options = {}) {
@@ -1601,7 +1615,6 @@ export class InferencePipeline extends PipelineState {
1601
1615
  // Check between every prompt so a superseded revision drops the rest.
1602
1616
  assertNotAborted(options?.signal);
1603
1617
  outputs.push(await this.embed(prompt, batchOptions));
1604
- this.resetForBatch();
1605
1618
  }
1606
1619
  return outputs;
1607
1620
  }
@@ -1746,6 +1759,14 @@ export class InferencePipeline extends PipelineState {
1746
1759
  return this.generator.prefillWithLogits(prompt, options);
1747
1760
  }
1748
1761
 
1762
+ prefillWithTokenLogits(prompt, tokenIds, options = {}) {
1763
+ return this.generator.prefillWithTokenLogits(prompt, tokenIds, options);
1764
+ }
1765
+
1766
+ prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
1767
+ return this.generator.prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options);
1768
+ }
1769
+
1749
1770
 
1750
1771
  applyKVCacheSnapshot(snapshot) {
1751
1772
  this.kvCache = snapshot.cache.clone();
@@ -1903,6 +1924,11 @@ export class InferencePipeline extends PipelineState {
1903
1924
  this.plePrefetchPending = null;
1904
1925
  this.weights.clear();
1905
1926
  this.expertWeights.clear();
1927
+ if (this.ownsDopplerLoader && this.dopplerLoader) {
1928
+ await this.dopplerLoader.unload();
1929
+ }
1930
+ this.dopplerLoader = null;
1931
+ this.ownsDopplerLoader = false;
1906
1932
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
1907
1933
  this.lora = null;
1908
1934
  destroyMoERouter(this.moeRouter);
@@ -118,6 +118,28 @@ function parseByteTokenValue(token) {
118
118
  return hi >= 0 && lo >= 0 ? (hi << 4) | lo : null;
119
119
  }
120
120
 
121
+ function decodeByteFallbackTokens(tokens) {
122
+ const parts = [];
123
+ let bytes = [];
124
+ const flushBytes = () => {
125
+ if (bytes.length === 0) return;
126
+ parts.push(new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes)));
127
+ bytes = [];
128
+ };
129
+
130
+ for (const token of tokens) {
131
+ const byteValue = parseByteTokenValue(token);
132
+ if (byteValue !== null) {
133
+ bytes.push(byteValue);
134
+ continue;
135
+ }
136
+ flushBytes();
137
+ parts.push(token);
138
+ }
139
+ flushBytes();
140
+ return parts.join('');
141
+ }
142
+
121
143
  function appendVocabEntry(token, id, vocab, reverseVocab, byteTokens) {
122
144
  const numId = typeof id === 'number' ? id : parseInt( (id), 10);
123
145
  vocab.set(token, numId);
@@ -469,9 +491,6 @@ export class BundledTokenizer extends BaseTokenizer {
469
491
  }
470
492
  log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
471
493
  this.#byteDecoder = null;
472
- if (this.#type === 'bpe') {
473
- this.#initializeByteDecoder();
474
- }
475
494
  let maxId = -1;
476
495
 
477
496
  // Handle vocab based on type
@@ -586,6 +605,9 @@ export class BundledTokenizer extends BaseTokenizer {
586
605
  const decoderPrepend = hf.decoder?.prepend_scheme === 'always' || hf.decoder?.add_prefix_space === true;
587
606
  const normalizerPrepend = hf.normalizer?.prepend_scheme === 'always' || hf.normalizer?.add_prefix_space === true;
588
607
  this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
608
+ if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
609
+ this.#initializeByteDecoder();
610
+ }
589
611
  const runtimeSpacePrefix = runtimeDefaults.addSpacePrefix;
590
612
  // Use explicit runtime config if set (non-null), otherwise auto-detect from tokenizer.json
591
613
  this.#addSpacePrefix = runtimeSpacePrefix
@@ -646,9 +668,6 @@ export class BundledTokenizer extends BaseTokenizer {
646
668
  throw new Error(`[Tokenizer] Unsupported tokenizer type: ${tokenizerJson.type}`);
647
669
  }
648
670
  this.#byteDecoder = null;
649
- if (this.#type === 'bpe') {
650
- this.#initializeByteDecoder();
651
- }
652
671
 
653
672
  // Build vocab maps
654
673
  let maxId = loadObjectVocab(tokenizerJson.vocab, this.#vocab, this.#reverseVocab, this.#byteTokens);
@@ -744,6 +763,9 @@ export class BundledTokenizer extends BaseTokenizer {
744
763
  throw new Error('[Tokenizer] addEosToken is enabled but eos token is missing.');
745
764
  }
746
765
  this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
766
+ if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
767
+ this.#initializeByteDecoder();
768
+ }
747
769
  // NOTE: Default to FALSE - first word shouldn't get space prefix
748
770
  // Space prefixes are only for words that follow a space in original text
749
771
  this.#addSpacePrefix = tokenizerJson.addSpacePrefix === true
@@ -1073,29 +1095,29 @@ export class BundledTokenizer extends BaseTokenizer {
1073
1095
 
1074
1096
  const token = this.#reverseVocab.get(id);
1075
1097
  if (token !== undefined) {
1076
- // Handle byte tokens
1077
- if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
1078
- const byteVal = parseInt(token.slice(3, 5), 16);
1079
- tokens.push(String.fromCharCode(byteVal));
1080
- } else {
1081
- tokens.push(token);
1082
- }
1098
+ tokens.push(token);
1083
1099
  }
1084
1100
  }
1085
1101
 
1086
1102
  let result;
1087
- if (this.#type === 'bpe' && this.#byteDecoder instanceof Map && this.#byteDecoder.size > 0) {
1088
- const merged = tokens.join('');
1103
+ if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
1089
1104
  const bytes = [];
1090
- for (const ch of merged) {
1091
- const mapped = this.#byteDecoder.get(ch);
1092
- if (mapped != null) {
1093
- bytes.push(mapped);
1105
+ for (const token of tokens) {
1106
+ const byteValue = parseByteTokenValue(token);
1107
+ if (byteValue !== null) {
1108
+ bytes.push(byteValue);
1094
1109
  continue;
1095
1110
  }
1096
- const fallbackBytes = new TextEncoder().encode(ch);
1097
- for (const byte of fallbackBytes) {
1098
- bytes.push(byte);
1111
+ for (const ch of token) {
1112
+ const mapped = this.#byteDecoder.get(ch);
1113
+ if (mapped != null) {
1114
+ bytes.push(mapped);
1115
+ continue;
1116
+ }
1117
+ const fallbackBytes = new TextEncoder().encode(ch);
1118
+ for (const byte of fallbackBytes) {
1119
+ bytes.push(byte);
1120
+ }
1099
1121
  }
1100
1122
  }
1101
1123
  result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
@@ -1103,7 +1125,7 @@ export class BundledTokenizer extends BaseTokenizer {
1103
1125
  result = result.replace(/▁/g, ' ');
1104
1126
  } else {
1105
1127
  // Join and convert ▁ back to spaces, handle GPT-style markers
1106
- result = tokens.join('')
1128
+ result = decodeByteFallbackTokens(tokens)
1107
1129
  .replace(/▁/g, ' ')
1108
1130
  .replace(/Ġ/g, ' ')
1109
1131
  .replace(/Ċ/g, '\n');
@@ -4,9 +4,13 @@
4
4
  * @module inference/tokenizers/types
5
5
  */
6
6
 
7
- import type { SpecialTokens } from '../../types/inference.js';
8
-
9
- export type { SpecialTokens };
7
+ export interface SpecialTokens {
8
+ bos?: number;
9
+ eos?: number;
10
+ pad?: number;
11
+ unk?: number;
12
+ mask?: number;
13
+ }
10
14
 
11
15
  /** Tokenizer Configuration */
12
16
  export interface TokenizerConfig {
@@ -144,4 +148,3 @@ export interface BundledTokenizerJson {
144
148
  addEosToken?: boolean;
145
149
  addSpacePrefix?: boolean;
146
150
  }
147
-