doppler-gpu 0.7.3 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (768) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/package.json +5 -48
  3. package/src/capsule-runtime.d.ts +2 -2
  4. package/src/client/capsule-host.browser.js +4 -4
  5. package/src/client/capsule-host.d.ts +8 -8
  6. package/src/client/capsule-host.js +4 -4
  7. package/src/client/doppler-api.d.ts +2 -2
  8. package/src/client/inspection.d.ts +3 -0
  9. package/src/client/inspection.js +29 -0
  10. package/src/client/model-host/index.d.ts +5 -4
  11. package/src/client/model-host/index.js +28 -8
  12. package/src/client/model-host/model-evidence.js +2 -0
  13. package/src/client/model-host/model-session.d.ts +7 -5
  14. package/src/client/model-host/model-session.js +10 -3
  15. package/src/client/model-host/privacy-receipt.js +22 -0
  16. package/src/client/resident-partitions.d.ts +3 -3
  17. package/src/client/resident-partitions.js +2 -2
  18. package/src/client/runtime/capsule-acquisition.d.ts +3 -3
  19. package/src/client/runtime/capsule-acquisition.js +1 -1
  20. package/src/client/runtime/capsule-adapter-execution.d.ts +2 -2
  21. package/src/client/runtime/capsule-adapter-execution.js +3 -3
  22. package/src/client/runtime/capsule-embedding.d.ts +2 -2
  23. package/src/client/runtime/capsule-embedding.js +1 -1
  24. package/src/client/runtime/capsule-forecast.d.ts +2 -2
  25. package/src/client/runtime/capsule-forecast.js +1 -1
  26. package/src/client/runtime/capsule-operation-adapters.d.ts +1 -1
  27. package/src/client/runtime/capsule-operation-adapters.js +1 -1
  28. package/src/client/runtime/capsule-operation-deltas.js +1 -1
  29. package/src/client/runtime/capsule-operation-executor.d.ts +1 -1
  30. package/src/client/runtime/capsule-operation-executor.js +2 -2
  31. package/src/client/runtime/capsule-operation-stream.d.ts +1 -1
  32. package/src/client/runtime/capsule-operation-stream.js +2 -2
  33. package/src/client/runtime/capsule-piece-reader.js +1 -1
  34. package/src/client/runtime/capsule-program-adapter.js +1 -1
  35. package/src/client/runtime/capsule-release-authorization.js +1 -1
  36. package/src/client/runtime/capsule-rerank.d.ts +2 -2
  37. package/src/client/runtime/composition-root.d.ts +6 -6
  38. package/src/client/runtime/composition-root.js +6 -6
  39. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +1 -1
  40. package/src/client/runtime/fetch-capsule-artifact-store.js +1 -1
  41. package/src/client/runtime/model-source.d.ts +4 -2
  42. package/src/client/runtime/model-source.js +5 -5
  43. package/src/client/runtime/scoped-session.d.ts +2 -2
  44. package/src/client/runtime/verified-capsule-artifact-store.d.ts +2 -2
  45. package/src/client/runtime/verified-capsule-artifact-store.js +1 -1
  46. package/src/config/backward-registry-loader.js +1 -1
  47. package/src/config/{capsule-adapter-policy.js → capsule/capsule-adapter-policy.js} +2 -2
  48. package/src/config/{capsule-adapters.d.ts → capsule/capsule-adapters.d.ts} +1 -1
  49. package/src/config/{capsule-adapters.js → capsule/capsule-adapters.js} +2 -2
  50. package/src/config/{capsule-loading.js → capsule/capsule-loading.js} +1 -1
  51. package/src/config/{capsule-operation.d.ts → capsule/capsule-operation.d.ts} +1 -1
  52. package/src/config/{capsule-operation.js → capsule/capsule-operation.js} +3 -3
  53. package/src/config/{capsule-release-contract.js → capsule/capsule-release-contract.js} +2 -2
  54. package/src/config/{capsule-release-events.d.ts → capsule/capsule-release-events.d.ts} +2 -1
  55. package/src/config/{capsule-release-events.js → capsule/capsule-release-events.js} +3 -2
  56. package/src/config/{capsule-sequence-receipt.js → capsule/capsule-sequence-receipt.js} +1 -1
  57. package/src/config/{capsule-token-selection.d.ts → capsule/capsule-token-selection.d.ts} +1 -1
  58. package/src/config/{capsule-token-selection.js → capsule/capsule-token-selection.js} +2 -2
  59. package/src/config/{capsule-v2.d.ts → capsule/capsule-v2.d.ts} +5 -4
  60. package/src/config/{capsule-v2.js → capsule/capsule-v2.js} +6 -6
  61. package/src/config/{capsule-v3.js → capsule/capsule-v3.js} +1 -1
  62. package/src/config/{capsule.js → capsule/capsule.js} +26 -7
  63. package/src/config/choice-scoring.js +1 -1
  64. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  65. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +1 -1
  66. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  67. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +2 -2
  68. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  69. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +1 -1
  70. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  71. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  72. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  73. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  74. package/src/config/conversion/gemma3/gemma-3-270m-it-w8a8-ehf16-af32.json +1 -1
  75. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  76. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  77. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +8 -8
  78. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +8 -8
  79. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +8 -8
  80. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +8 -8
  81. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +8 -8
  82. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +8 -8
  83. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +8 -8
  84. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +8 -8
  85. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +8 -8
  86. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +1 -1
  87. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  88. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +1 -1
  89. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +1 -1
  90. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +1 -1
  91. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  92. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  93. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  94. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  95. package/src/config/conversion/qwen3/qwen-3-4b-instruct-2507-q4k-ehf16-af32.json +2 -2
  96. package/src/config/conversion/qwen3/qwen-3-5-0-8b-canonical-f32.json +1 -1
  97. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  98. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  99. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  100. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  101. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +1 -1
  102. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  103. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-q4_0-passthrough.json +1 -1
  104. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-text-q4k-ehf16-af32.json +1 -1
  105. package/src/config/conversion/qwen3/qwen-3-8-flash-next-text-q4k-ehf16-af32.json +25 -22
  106. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +2 -2
  107. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  108. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  109. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +2 -2
  110. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +2 -2
  111. package/src/config/conversion/qwen3/qwen3.8-27b-q4_0-passthrough.json +1 -1
  112. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +1 -1
  113. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +1 -1
  114. package/src/config/conversion/qwen3/qwen4exp-gguf-source-policy.json +53 -0
  115. package/src/config/embedding-contract.d.ts +1 -1
  116. package/src/config/embedding-contract.js +1 -1
  117. package/src/config/execution-registry-contract.d.ts +5 -5
  118. package/src/config/execution-registry-contract.js +5 -5
  119. package/src/config/generation-contract.js +2 -1
  120. package/src/config/hardware-partition-plan.d.ts +32 -0
  121. package/src/config/hardware-partition-plan.js +78 -0
  122. package/src/config/inspection/observation-policies.json +1 -1
  123. package/src/config/kernel-path-loader.d.ts +1 -1
  124. package/src/config/kernel-path-loader.js +1 -1
  125. package/src/config/kernel-registry-contract.d.ts +2 -2
  126. package/src/config/kernel-registry-contract.js +1 -1
  127. package/src/config/kernels/kernel-ref-digests.js +508 -0
  128. package/src/config/kernels/registry.js +2 -2
  129. package/src/config/kernels/registry.json +1892 -1096
  130. package/src/config/layer-partition-plan.d.ts +4 -0
  131. package/src/config/layer-partition-plan.js +10 -4
  132. package/src/config/merge.js +1 -1
  133. package/src/config/model-session-declarations.js +85 -0
  134. package/src/config/param-validator.js +3 -3
  135. package/src/config/platforms/amd-rdna3.json +2 -2
  136. package/src/config/production-release-evidence.js +1 -1
  137. package/src/config/revocation-policy.js +6 -2
  138. package/src/config/revocation-registry.json +1 -1
  139. package/src/config/runtime/model-sessions.json +23 -0
  140. package/src/config/runtime/profiles/flash-next-gguf-control-gemv.json +72 -0
  141. package/src/config/runtime/profiles/flash-next-gguf-hc-inject-gemv.json +47 -0
  142. package/src/config/schema/capsule/index.d.ts +14 -0
  143. package/src/config/schema/capsule/index.js +10 -0
  144. package/src/config/schema/{program-bundle.schema.d.ts → capsule/program-bundle.schema.d.ts} +3 -3
  145. package/src/config/schema/{program-bundle.schema.js → capsule/program-bundle.schema.js} +6 -6
  146. package/src/config/schema/{execution-v1.schema.d.ts → execution/execution-v1.schema.d.ts} +2 -0
  147. package/src/config/schema/{execution-v1.schema.js → execution/execution-v1.schema.js} +2 -2
  148. package/src/config/schema/execution/index.d.ts +203 -0
  149. package/src/config/schema/execution/index.js +94 -0
  150. package/src/config/schema/{inference.schema.d.ts → execution/inference.schema.d.ts} +1 -1
  151. package/src/config/schema/{kernel-path.schema.d.ts → execution/kernel-path.schema.d.ts} +3 -0
  152. package/src/config/schema/{kernel-registry.schema.d.ts → execution/kernel-registry.schema.d.ts} +2 -2
  153. package/src/config/schema/{kernel-thresholds.schema.d.ts → execution/kernel-thresholds.schema.d.ts} +1 -1
  154. package/src/config/schema/{kernel-thresholds.schema.js → execution/kernel-thresholds.schema.js} +1 -1
  155. package/src/config/schema/{kvcache.schema.d.ts → execution/kvcache.schema.d.ts} +8 -2
  156. package/src/config/schema/{kvcache.schema.js → execution/kvcache.schema.js} +17 -1
  157. package/src/config/schema/{ngram-embedding.schema.d.ts → execution/ngram-embedding.schema.d.ts} +3 -1
  158. package/src/config/schema/{ngram-embedding.schema.js → execution/ngram-embedding.schema.js} +16 -1
  159. package/src/config/schema/index.d.ts +8 -897
  160. package/src/config/schema/index.js +8 -506
  161. package/src/config/schema/{converter.schema.d.ts → manifest/converter.schema.d.ts} +4 -4
  162. package/src/config/schema/{converter.schema.js → manifest/converter.schema.js} +1 -1
  163. package/src/config/schema/manifest/index.d.ts +142 -0
  164. package/src/config/schema/manifest/index.js +62 -0
  165. package/src/config/schema/manifest/manifest-inference.schema.d.ts +298 -0
  166. package/src/config/schema/{manifest.schema.d.ts → manifest/manifest.schema.d.ts} +9 -291
  167. package/src/config/schema/{manifest.schema.js → manifest/manifest.schema.js} +2 -2
  168. package/src/config/schema/{emulation.schema.d.ts → platform/emulation.schema.d.ts} +1 -1
  169. package/src/config/schema/{emulation.schema.js → platform/emulation.schema.js} +6 -6
  170. package/src/config/schema/platform/index.d.ts +72 -0
  171. package/src/config/schema/platform/index.js +27 -0
  172. package/src/config/schema/{memory-limits.schema.js → platform/memory-limits.schema.js} +1 -1
  173. package/src/config/schema/policy/index.d.ts +14 -0
  174. package/src/config/schema/policy/index.js +9 -0
  175. package/src/config/schema/{doppler.schema.d.ts → runtime/doppler.schema.d.ts} +3 -3
  176. package/src/config/schema/{doppler.schema.js → runtime/doppler.schema.js} +12 -12
  177. package/src/config/schema/runtime/index.d.ts +238 -0
  178. package/src/config/schema/runtime/index.js +135 -0
  179. package/src/config/schema/{loading.schema.d.ts → runtime/loading.schema.d.ts} +26 -1
  180. package/src/config/schema/{loading.schema.js → runtime/loading.schema.js} +88 -0
  181. package/src/config/schema/{shared-runtime.schema.d.ts → runtime/shared-runtime.schema.d.ts} +5 -5
  182. package/src/config/schema/{shared-runtime.schema.js → runtime/shared-runtime.schema.js} +4 -4
  183. package/src/config/schema/{tooling.schema.d.ts → runtime/tooling.schema.d.ts} +1 -1
  184. package/src/config/schema/training/index.d.ts +63 -0
  185. package/src/config/schema/training/index.js +43 -0
  186. package/src/config/schema/{lora.schema.js → training/lora.schema.js} +1 -1
  187. package/src/config/source-packages/litert/gemma-4-e2b-it.json +1 -1
  188. package/src/config/supported-operations.json +358 -88
  189. package/src/config/target-plan.d.ts +2 -2
  190. package/src/config/target-plan.js +2 -2
  191. package/src/config/training-defaults.d.ts +2 -2
  192. package/src/config/training-defaults.js +2 -2
  193. package/src/config/transforms/execution-graph/correctness-fallback.js +86 -0
  194. package/src/config/transforms/execution-graph/prefill-attention.js +116 -0
  195. package/src/{converter → config/transforms}/execution-graph-transforms.js +5 -194
  196. package/src/converter/core.js +18 -164
  197. package/src/converter/forge-stages.js +2 -2
  198. package/src/converter/fp8-source.js +1 -1
  199. package/src/converter/gemma4/validation.js +110 -0
  200. package/src/converter/linear-norm-gate.js +1 -1
  201. package/src/converter/manifest-builder.js +2 -9
  202. package/src/converter/parsers/gguf.js +54 -4
  203. package/src/converter/quantization-info.js +2 -2
  204. package/src/converter/qwen4-exp-source-execution.js +94 -0
  205. package/src/converter/qwen4-exp-source.js +1 -201
  206. package/src/converter/tensor-transform.js +2 -9
  207. package/src/converter/tokenizer-contract.js +2 -9
  208. package/src/converter/w8a8-quantizer.js +2 -2
  209. package/src/debug/config.js +1 -1
  210. package/src/debug/perf-guards.js +1 -1
  211. package/src/errors/doppler-error.d.ts +54 -0
  212. package/src/errors/doppler-error.js +13 -5
  213. package/src/experimental/distribution/shard-delivery/receipt.js +1 -1
  214. package/src/experimental/distribution/shard-delivery/retry.js +4 -4
  215. package/src/experimental/energy/types.d.ts +1 -1
  216. package/src/experimental/logits/cpu-output.js +1 -1
  217. package/src/experimental/training/artifacts.d.ts +2 -2
  218. package/src/experimental/training/artifacts.js +2 -2
  219. package/src/experimental/training/autograd.d.ts +1 -1
  220. package/src/experimental/training/distillation/student-fixture/execution.js +1 -1
  221. package/src/experimental/training/distillation/student-fixture/model.js +1 -1
  222. package/src/experimental/training/distillation/student-fixture.js +1 -1
  223. package/src/experimental/training/loss-scaling.d.ts +1 -1
  224. package/src/experimental/training/runner/evaluation.js +1 -1
  225. package/src/experimental/training/runner.js +1 -1
  226. package/src/experimental/training/trainer.d.ts +1 -1
  227. package/src/experimental/training/workloads.js +2 -2
  228. package/src/formats/gguf/block-storage.d.ts +9 -0
  229. package/src/formats/gguf/block-storage.js +12 -3
  230. package/src/formats/gguf/qwen-source.js +5 -3
  231. package/src/formats/gguf/qwen-source.json +8 -1
  232. package/src/formats/gguf/qwen4-exp-source.js +101 -0
  233. package/src/formats/q4k-dequant.js +1 -1
  234. package/src/formats/qwen4-exp-source-contracts.js +204 -0
  235. package/src/formats/rdrr/fp8-contract.js +1 -1
  236. package/src/formats/rdrr/fp8-storage.js +1 -1
  237. package/src/formats/rdrr/q4_0-storage.js +1 -1
  238. package/src/formats/rdrr/storage-descriptor.js +1 -1
  239. package/src/formats/rdrr/tensor-config-validator.js +3 -3
  240. package/src/formats/rdrr/validation.js +1 -1
  241. package/src/formats/rdrr/w8a8-storage.js +1 -1
  242. package/src/generation/index.d.ts +1 -1
  243. package/src/generation/index.js +1 -0
  244. package/src/gpu/device.d.ts +1 -1
  245. package/src/gpu/kernels/attention/contiguous-quantized.js +252 -0
  246. package/src/gpu/kernels/attention/tiered-quantized.js +321 -0
  247. package/src/gpu/kernels/attention/tiered.js +240 -0
  248. package/src/gpu/kernels/attention.js +5 -801
  249. package/src/gpu/kernels/backward/layernorm_backward.wgsl +3 -3
  250. package/src/gpu/kernels/channel-gain.js +100 -0
  251. package/src/gpu/kernels/channel-rotation.d.ts +16 -0
  252. package/src/gpu/kernels/channel-rotation.js +47 -0
  253. package/src/gpu/kernels/channel_energy.wgsl +17 -0
  254. package/src/gpu/kernels/channel_energy_pair.wgsl +27 -0
  255. package/src/gpu/kernels/channel_gain_control.wgsl +60 -0
  256. package/src/gpu/kernels/channel_gain_derive.wgsl +40 -0
  257. package/src/gpu/kernels/channel_gain_swiglu.wgsl +64 -0
  258. package/src/gpu/kernels/channel_rotation_apply.wgsl +38 -0
  259. package/src/gpu/kernels/channel_rotation_derive.wgsl +76 -0
  260. package/src/gpu/kernels/check-finiteness-f32.js +22 -0
  261. package/src/gpu/kernels/check_finiteness.wgsl +2 -2
  262. package/src/gpu/kernels/clamp.js +2 -2
  263. package/src/gpu/kernels/dequant.js +1 -1
  264. package/src/gpu/kernels/dispatch.js +7 -0
  265. package/src/gpu/kernels/energy_quintel_grad.wgsl +7 -7
  266. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +7 -7
  267. package/src/gpu/kernels/energy_quintel_reduce.wgsl +6 -6
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +6 -6
  269. package/src/gpu/kernels/energy_quintel_update.wgsl +7 -7
  270. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +7 -7
  271. package/src/gpu/kernels/fused_matmul_q4_widetile_f32.wgsl +174 -0
  272. package/src/gpu/kernels/fused_matmul_rmsnorm.wgsl +0 -49
  273. package/src/gpu/kernels/gated-delta-norm-gate.js +1 -1
  274. package/src/gpu/kernels/gated_delta_recurrent_gguf_register_prefill_sigmoid_gate.wgsl +730 -0
  275. package/src/gpu/kernels/gated_delta_recurrent_gguf_sigmoid_gate.wgsl +499 -0
  276. package/src/gpu/kernels/gather.js +16 -1
  277. package/src/gpu/kernels/gather_split4_packed_f16_vec4_f32_out.wgsl +104 -0
  278. package/src/gpu/kernels/generated/uniform-writers-0.js +7 -11
  279. package/src/gpu/kernels/generated/uniform-writers-1.js +43 -46
  280. package/src/gpu/kernels/generated/uniform-writers-2.js +80 -2
  281. package/src/gpu/kernels/generated/uniform-writers.js +33 -21
  282. package/src/gpu/kernels/hyper-connection-up.js +64 -0
  283. package/src/gpu/kernels/hyper-connection.js +15 -4
  284. package/src/gpu/kernels/hyper_connection_mix_branch.wgsl +105 -0
  285. package/src/gpu/kernels/hyper_connection_norm.wgsl +3 -3
  286. package/src/gpu/kernels/hyper_connection_up_mix_branch_q4_0.wgsl +164 -0
  287. package/src/gpu/kernels/kernel-execution.js +18 -7
  288. package/src/gpu/kernels/linear-attention-core.js +3 -4
  289. package/src/gpu/kernels/moe-route.js +14 -0
  290. package/src/gpu/kernels/moe_gather.wgsl +0 -63
  291. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +63 -0
  292. package/src/gpu/kernels/moe_route_q4_1_block_f32.wgsl +59 -0
  293. package/src/gpu/kernels/moe_route_remap.wgsl +22 -0
  294. package/src/gpu/kernels/paired-linear-attention.js +1 -1
  295. package/src/gpu/kernels/pipeline-cache.js +18 -2
  296. package/src/gpu/kernels/ple.js +11 -7
  297. package/src/gpu/kernels/ple_q4_0_to_f32.wgsl +32 -0
  298. package/src/gpu/kernels/qsa-indexer.d.ts +2 -0
  299. package/src/gpu/kernels/qsa-indexer.js +6 -4
  300. package/src/gpu/kernels/qsa_block_keys.wgsl +2 -1
  301. package/src/gpu/kernels/qsa_prepare.wgsl +2 -1
  302. package/src/gpu/kernels/relu.js +1 -1
  303. package/src/gpu/kernels/relu.wgsl +2 -2
  304. package/src/gpu/kernels/relu_f16.wgsl +2 -2
  305. package/src/gpu/kernels/residual.js +1 -1
  306. package/src/gpu/kernels/residual.wgsl +4 -4
  307. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  308. package/src/gpu/kernels/residual_f16_vec4.wgsl +2 -2
  309. package/src/gpu/kernels/residual_vec4.wgsl +2 -2
  310. package/src/gpu/kernels/rope-precompute.js +1 -1
  311. package/src/gpu/kernels/scatter_add.wgsl +0 -31
  312. package/src/gpu/kernels/shader-cache.js +17 -3
  313. package/src/gpu/kernels/shader-capability-filter.js +120 -0
  314. package/src/gpu/kernels/softmax.js +10 -10
  315. package/src/gpu/kernels/topk.wgsl +0 -55
  316. package/src/gpu/kernels/transpose.js +1 -1
  317. package/src/gpu/kernels/transpose.wgsl +2 -2
  318. package/src/gpu/kernels/w8a8.js +1 -1
  319. package/src/gpu/uniform-cache.d.ts +15 -1
  320. package/src/gpu/uniform-cache.js +82 -7
  321. package/src/gpu/weight-buffer.d.ts +1 -1
  322. package/src/inference/moe-router.d.ts +1 -126
  323. package/src/inference/moe-router.js +0 -149
  324. package/src/inference/pipelines/context.js +3 -15
  325. package/src/inference/pipelines/shader-scoped-pipeline.js +10 -5
  326. package/src/inference/pipelines/text/attention/interpreter/immediate.js +4 -4
  327. package/src/inference/pipelines/text/attention/interpreter/output.js +203 -0
  328. package/src/inference/pipelines/text/attention/interpreter/recorded.js +92 -67
  329. package/src/inference/pipelines/text/attention/interpreter.js +77 -350
  330. package/src/inference/pipelines/text/attention/projection-executor.js +2 -2
  331. package/src/inference/pipelines/text/attention/projection-plan.js +1 -1
  332. package/src/inference/pipelines/text/attention/projection-precision.js +132 -0
  333. package/src/inference/pipelines/text/attention/projections.js +13 -151
  334. package/src/inference/pipelines/text/attention/recorded-health.js +83 -0
  335. package/src/inference/pipelines/text/attention/rope-observation.js +1 -1
  336. package/src/inference/pipelines/text/attention/types.js +0 -13
  337. package/src/inference/pipelines/text/config/normalization.js +2 -2
  338. package/src/inference/pipelines/text/config.d.ts +4 -4
  339. package/src/inference/pipelines/text/config.js +17 -135
  340. package/src/inference/pipelines/text/{config-components.js → configuration/config-components.js} +4 -4
  341. package/src/inference/pipelines/text/configuration/model-geometry.js +114 -0
  342. package/src/inference/pipelines/text/{resolved-runtime-session.d.ts → configuration/resolved-runtime-session.d.ts} +4 -4
  343. package/src/inference/pipelines/text/{resolved-runtime-session.js → configuration/resolved-runtime-session.js} +1 -1
  344. package/src/inference/pipelines/text/{abort-contract.js → contracts/abort-contract.js} +2 -1
  345. package/src/inference/pipelines/text/contracts/metrics.d.ts +234 -0
  346. package/src/inference/pipelines/text/{embed.js → embedding/embed.js} +15 -15
  347. package/src/inference/pipelines/text/{embedding-batch.js → embedding/embedding-batch.js} +1 -1
  348. package/src/inference/pipelines/text/{embedding-extraction.js → embedding/embedding-extraction.js} +12 -12
  349. package/src/inference/pipelines/text/{embedding-normalization.js → embedding/embedding-normalization.js} +4 -4
  350. package/src/inference/pipelines/text/{execution-plan.d.ts → execution/execution-plan.d.ts} +8 -8
  351. package/src/inference/pipelines/text/{execution-plan.js → execution/execution-plan.js} +9 -9
  352. package/src/inference/pipelines/text/{execution-runtime-builders.js → execution/execution-runtime-builders.js} +3 -3
  353. package/src/inference/pipelines/text/{execution-v1.d.ts → execution/execution-v1.d.ts} +2 -2
  354. package/src/inference/pipelines/text/{execution-v1.js → execution/execution-v1.js} +13 -183
  355. package/src/inference/pipelines/text/execution/layer-contract.js +90 -0
  356. package/src/inference/pipelines/text/{layer-execution-contract.js → execution/layer-execution-contract.js} +2 -2
  357. package/src/inference/pipelines/text/{layer-input-execution.js → execution/layer-input-execution.js} +27 -36
  358. package/src/inference/pipelines/text/{layer-partition-contract.d.ts → execution/layer-partition-contract.d.ts} +3 -3
  359. package/src/inference/pipelines/text/{layer-partition-contract.js → execution/layer-partition-contract.js} +2 -2
  360. package/src/inference/pipelines/text/{layer-plan-gpu.js → execution/layer-plan-gpu.js} +12 -12
  361. package/src/inference/pipelines/text/{layer-plan.d.ts → execution/layer-plan.d.ts} +1 -1
  362. package/src/inference/pipelines/text/{layer-plan.js → execution/layer-plan.js} +1 -1
  363. package/src/inference/pipelines/text/{layer.js → execution/layer.js} +34 -142
  364. package/src/inference/pipelines/text/{operator-identity.js → execution/operator-identity.js} +1 -1
  365. package/src/inference/pipelines/text/{ops.js → execution/ops.js} +26 -26
  366. package/src/inference/pipelines/text/{paired-execution.js → execution/paired-execution.js} +9 -9
  367. package/src/inference/pipelines/text/execution-v1/hybrid-projection-contract.js +159 -0
  368. package/src/inference/pipelines/text/expert-loading-stats.d.ts +16 -0
  369. package/src/inference/pipelines/text/expert-loading-stats.js +15 -0
  370. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -4
  371. package/src/inference/pipelines/text/ffn/dense-plan.js +23 -34
  372. package/src/inference/pipelines/text/ffn/dense.js +72 -392
  373. package/src/inference/pipelines/text/ffn/fused-gate-up.js +197 -0
  374. package/src/inference/pipelines/text/ffn/moe.js +1 -1
  375. package/src/inference/pipelines/text/ffn/packed-gate-up.js +218 -0
  376. package/src/inference/pipelines/text/ffn/paired.js +3 -3
  377. package/src/inference/pipelines/text/ffn/qwen-shared-expert.js +1 -1
  378. package/src/inference/pipelines/text/ffn/sandwich.js +6 -7
  379. package/src/inference/pipelines/text/ffn/standard.js +5 -5
  380. package/src/inference/pipelines/text/ffn/types.js +6 -3
  381. package/src/inference/pipelines/text/{activity-weights.js → generator/activation/activity-weights.js} +8 -8
  382. package/src/inference/pipelines/text/generator/activation/channel-gain.d.ts +61 -0
  383. package/src/inference/pipelines/text/generator/activation/channel-gain.js +352 -0
  384. package/src/inference/pipelines/text/generator/activation/final-layer-replay.js +43 -0
  385. package/src/inference/pipelines/text/generator/{capsule-generation.js → adapters/capsule-generation.js} +5 -5
  386. package/src/inference/pipelines/text/generator/{diffusion.js → adapters/diffusion.js} +19 -19
  387. package/src/inference/pipelines/text/generator/{training.js → adapters/training.js} +21 -21
  388. package/src/inference/pipelines/text/generator/decode/advance-token.js +73 -0
  389. package/src/inference/pipelines/text/{generator-steps.js → generator/decode/batch-generation.js} +25 -288
  390. package/src/inference/pipelines/text/generator/decode/batch-readback.js +175 -0
  391. package/src/inference/pipelines/text/generator/{compiled-greedy-window.js → decode/compiled-greedy-window.js} +5 -5
  392. package/src/inference/pipelines/text/generator/{decode-components.js → decode/decode-components.js} +1 -1
  393. package/src/inference/pipelines/text/generator/{decode-runtime.js → decode/decode-loop.js} +20 -412
  394. package/src/inference/pipelines/text/generator/decode/decode-mechanics.js +288 -0
  395. package/src/inference/pipelines/text/generator/{decode-readback.js → decode/decode-readback.js} +1 -1
  396. package/src/inference/pipelines/text/generator/decode/decode-runtime.js +2 -0
  397. package/src/inference/pipelines/text/generator/{decode.js → decode/decode-step.js} +34 -313
  398. package/src/inference/pipelines/text/generator/decode/decode.js +5 -0
  399. package/src/inference/pipelines/text/generator/{greedy-head-policy.js → decode/greedy-head-policy.js} +1 -1
  400. package/src/inference/pipelines/text/generator/{recorded-step-executor.js → decode/recorded-step-executor.js} +2 -2
  401. package/src/inference/pipelines/text/generator/generator-steps.js +4 -0
  402. package/src/inference/pipelines/text/{generator.d.ts → generator/generator.d.ts} +13 -11
  403. package/src/inference/pipelines/text/{generator.js → generator/generator.js} +150 -46
  404. package/src/inference/pipelines/text/generator/{prefill-runtime.js → prefill/prefill-execution.js} +72 -278
  405. package/src/inference/pipelines/text/generator/prefill/prefill-logits.js +144 -0
  406. package/src/inference/pipelines/text/generator/{prefill-observation.js → prefill/prefill-observation.js} +2 -2
  407. package/src/inference/pipelines/text/generator/{prefill-policy.js → prefill/prefill-policy.js} +6 -1
  408. package/src/inference/pipelines/text/generator/prefill/prefill-runtime.js +2 -0
  409. package/src/inference/pipelines/text/generator/prefill/prefill-sequence.js +100 -0
  410. package/src/inference/pipelines/text/generator/{prefix-embedding.js → prefill/prefix-embedding.js} +7 -7
  411. package/src/inference/pipelines/text/{generation-request.d.ts → generator/request/generation-request.d.ts} +2 -2
  412. package/src/inference/pipelines/text/{generation-request.js → generator/request/generation-request.js} +3 -3
  413. package/src/inference/pipelines/text/generator/{prompt-input.js → request/prompt-input.js} +2 -2
  414. package/src/inference/pipelines/text/{sampling-config.d.ts → generator/sampling/sampling-config.d.ts} +1 -1
  415. package/src/inference/pipelines/text/{sampling-config.js → generator/sampling/sampling-config.js} +2 -2
  416. package/src/inference/pipelines/text/{sampling.d.ts → generator/sampling/sampling.d.ts} +15 -0
  417. package/src/inference/pipelines/text/{sampling.js → generator/sampling/sampling.js} +22 -3
  418. package/src/inference/pipelines/text/{stopping.js → generator/sampling/stopping.js} +3 -3
  419. package/src/inference/pipelines/text/generator/{token-selection.d.ts → sampling/token-selection.d.ts} +1 -1
  420. package/src/inference/pipelines/text/generator/{token-selection.js → sampling/token-selection.js} +8 -8
  421. package/src/inference/pipelines/text/generator/{activation-storage.js → session/activation-storage.js} +2 -2
  422. package/src/inference/pipelines/text/generator/{attention-lifecycle.js → session/attention-lifecycle.js} +1 -1
  423. package/src/inference/pipelines/text/generator/{diagnostic-sequence.d.ts → session/diagnostic-sequence.d.ts} +3 -3
  424. package/src/inference/pipelines/text/generator/{diagnostic-sequence.js → session/diagnostic-sequence.js} +3 -3
  425. package/src/inference/pipelines/text/generator/session/generation-runtime.js +278 -0
  426. package/src/inference/pipelines/text/{generator-runtime.d.ts → generator/session/generator-runtime.d.ts} +7 -7
  427. package/src/inference/pipelines/text/{generator-runtime.js → generator/session/generator-runtime.js} +8 -8
  428. package/src/inference/pipelines/text/generator/{recovery.js → session/recovery.js} +6 -6
  429. package/src/inference/pipelines/text/{sequence-state.d.ts → generator/session/sequence-state.d.ts} +5 -5
  430. package/src/inference/pipelines/text/{sequence-state.js → generator/session/sequence-state.js} +26 -18
  431. package/src/inference/pipelines/text/generator/{session-context.js → session/session-context.js} +12 -11
  432. package/src/inference/pipelines/text/generator/{speculative-runtime.js → session/speculative-runtime.js} +1 -1
  433. package/src/inference/pipelines/text/generator/{text.js → session/text.js} +9 -9
  434. package/src/inference/pipelines/text/{warm-generation.d.ts → generator/session/warm-generation.d.ts} +1 -1
  435. package/src/inference/pipelines/text/{warm-generation.js → generator/session/warm-generation.js} +1 -1
  436. package/src/inference/pipelines/text/generator/{whole-token-lane.js → whole-token/whole-token-lane.js} +10 -10
  437. package/src/inference/pipelines/text/generator/{whole-token-observation.d.ts → whole-token/whole-token-observation.d.ts} +2 -2
  438. package/src/inference/pipelines/text/generator/{whole-token-observation.js → whole-token/whole-token-observation.js} +5 -5
  439. package/src/inference/pipelines/text/generator/{whole-token-program.d.ts → whole-token/whole-token-program.d.ts} +3 -3
  440. package/src/inference/pipelines/text/generator/{whole-token-program.js → whole-token/whole-token-program.js} +5 -5
  441. package/src/inference/pipelines/text/hyper-connection/decoder-layer.d.ts +17 -4
  442. package/src/inference/pipelines/text/hyper-connection/decoder-layer.js +38 -14
  443. package/src/inference/pipelines/text/hyper-connection/layer-executor.js +14 -9
  444. package/src/inference/pipelines/text/hyper-connection/logits-input.js +5 -3
  445. package/src/inference/pipelines/text/hyper-connection/projection.js +52 -2
  446. package/src/inference/pipelines/text/hyper-connection/sequence-transaction.js +3 -3
  447. package/src/inference/pipelines/text/hyper-connection/session.d.ts +1 -1
  448. package/src/inference/pipelines/text/hyper-connection/session.js +9 -5
  449. package/src/inference/pipelines/text/indexed-attention/qsa-cache.d.ts +2 -1
  450. package/src/inference/pipelines/text/indexed-attention/qsa-cache.js +3 -3
  451. package/src/inference/pipelines/text/init.d.ts +4 -2
  452. package/src/inference/pipelines/text/init.js +10 -934
  453. package/src/inference/pipelines/text/initialization/init-emulation.js +76 -0
  454. package/src/inference/pipelines/text/initialization/init-kv-cache.js +547 -0
  455. package/src/inference/pipelines/text/initialization/init-moe.js +34 -0
  456. package/src/inference/pipelines/text/{init-qkv-fusion.js → initialization/init-qkv-fusion.js} +7 -7
  457. package/src/inference/pipelines/text/{init-rope.d.ts → initialization/init-rope.d.ts} +4 -4
  458. package/src/inference/pipelines/text/{init-rope.js → initialization/init-rope.js} +9 -11
  459. package/src/inference/pipelines/text/initialization/init-speculative.js +31 -0
  460. package/src/inference/pipelines/text/{init-tokenizer.js → initialization/init-tokenizer.js} +2 -2
  461. package/src/inference/pipelines/text/lifecycle.js +18 -18
  462. package/src/inference/pipelines/text/{linear-attention-ab-fusion.js → linear-attention/linear-attention-ab-fusion.js} +3 -3
  463. package/src/inference/pipelines/text/{linear-attention.d.ts → linear-attention/linear-attention.d.ts} +12 -12
  464. package/src/inference/pipelines/text/{linear-attention.js → linear-attention/linear-attention.js} +17 -17
  465. package/src/inference/pipelines/text/linear-attention/plan.js +4 -4
  466. package/src/inference/pipelines/text/linear-attention/state-checkpoint.js +1 -1
  467. package/src/inference/pipelines/text/linear-attention/state-clone.js +1 -1
  468. package/src/inference/pipelines/text/linear-attention/state-layout.js +2 -2
  469. package/src/inference/pipelines/text/loading/decode-session-config.js +295 -0
  470. package/src/inference/pipelines/text/loading/init-weights.js +254 -0
  471. package/src/inference/pipelines/text/{load-errors.js → loading/load-errors.js} +3 -3
  472. package/src/inference/pipelines/text/{model-load.js → loading/model-load.js} +14 -306
  473. package/src/inference/pipelines/text/{weight-loading-config.js → loading/weight-loading-config.js} +6 -6
  474. package/src/inference/pipelines/text/{weights.d.ts → loading/weights.d.ts} +13 -13
  475. package/src/inference/pipelines/text/{weights.js → loading/weights.js} +5 -5
  476. package/src/inference/pipelines/text/logits/gpu-executor.js +2 -2
  477. package/src/inference/pipelines/text/logits/gpu.d.ts +4 -4
  478. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  479. package/src/inference/pipelines/text/{generator → logits}/hidden-projection.js +1 -1
  480. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  481. package/src/inference/pipelines/text/logits/index.js +3 -3
  482. package/src/inference/pipelines/text/{generator → logits}/logits-config.js +1 -1
  483. package/src/inference/pipelines/text/logits/output-transform.js +1 -1
  484. package/src/inference/pipelines/text/logits/precision-policy.js +1 -1
  485. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  486. package/src/inference/pipelines/text/lora-types.d.ts +1 -1
  487. package/src/inference/pipelines/text/moe/expert-loading.js +4 -1
  488. package/src/inference/pipelines/text/moe/gpu-executor.js +44 -101
  489. package/src/inference/pipelines/text/moe/grouped-prefill.js +6 -2
  490. package/src/inference/pipelines/text/moe/qwen-route-executor.d.ts +2 -2
  491. package/src/inference/pipelines/text/moe/qwen-route-executor.js +31 -2
  492. package/src/inference/pipelines/text/moe/routed-expert-chunks.js +99 -15
  493. package/src/inference/pipelines/text/moe-cache.js +11 -77
  494. package/src/inference/pipelines/text/moe-gpu.js +33 -75
  495. package/src/inference/pipelines/text/moe-impl.d.ts +3 -38
  496. package/src/inference/pipelines/text/moe-impl.js +1 -7
  497. package/src/inference/pipelines/text/ngram/ngram-addresses.d.ts +1 -1
  498. package/src/inference/pipelines/text/ngram/ngram-addresses.js +2 -2
  499. package/src/inference/pipelines/text/ngram/ngram-table.d.ts +3 -3
  500. package/src/inference/pipelines/text/ngram/ngram-table.js +5 -6
  501. package/src/inference/pipelines/text/ngram/ple-layer.d.ts +3 -3
  502. package/src/inference/pipelines/text/ngram/ple-layer.js +3 -3
  503. package/src/inference/pipelines/text/{generator-logits-observation.js → observation/generator-logits-observation.js} +4 -4
  504. package/src/inference/pipelines/text/{kernel-trace.js → observation/kernel-trace.js} +1 -1
  505. package/src/inference/pipelines/text/{operator-events.js → observation/operator-events.js} +1 -1
  506. package/src/inference/pipelines/text/{probes.js → observation/probes.js} +13 -13
  507. package/src/inference/pipelines/text/{tsir-fixture-writer.js → observation/tsir-fixture-writer.js} +3 -3
  508. package/src/inference/pipelines/text/{execution.js → operations/execution.js} +14 -14
  509. package/src/inference/pipelines/text/{image-transcription.js → operations/image-transcription.js} +8 -8
  510. package/src/inference/pipelines/text/{partition-attempt.js → partitions/partition-attempt.js} +7 -7
  511. package/src/inference/pipelines/text/{partition-execution.js → partitions/partition-execution.js} +79 -34
  512. package/src/inference/pipelines/text/partitions/partition-wire.js +50 -0
  513. package/src/inference/pipelines/text/{resident-partition-contract.d.ts → partitions/resident-partition-contract.d.ts} +10 -8
  514. package/src/inference/pipelines/text/{resident-partition-contract.js → partitions/resident-partition-contract.js} +6 -6
  515. package/src/inference/pipelines/text/{resident-partition.js → partitions/resident-partition.js} +19 -13
  516. package/src/inference/pipelines/text/{resident-pipeline-preparation.js → partitions/resident-pipeline-preparation.js} +32 -32
  517. package/src/inference/pipelines/text/per-layer/materialize.js +4 -4
  518. package/src/inference/pipelines/text/per-layer/per-layer-inputs.js +5 -0
  519. package/src/inference/pipelines/text/per-layer/plan.js +4 -4
  520. package/src/inference/pipelines/text/{per-layer-inputs.js → per-layer/prepare.js} +14 -280
  521. package/src/inference/pipelines/text/per-layer/projection-contract.js +79 -0
  522. package/src/inference/pipelines/text/per-layer/runtime-cache.js +183 -0
  523. package/src/inference/pipelines/text/pipeline.d.ts +13 -8
  524. package/src/inference/pipelines/text/pipeline.js +78 -22
  525. package/src/inference/pipelines/text/{drift-policy.js → policy/drift-policy.js} +1 -1
  526. package/src/inference/pipelines/text/{dtype-contract.js → policy/dtype-contract.js} +2 -2
  527. package/src/inference/pipelines/text/{finiteness-guard-status.js → policy/finiteness-guard-status.js} +2 -1
  528. package/src/inference/pipelines/text/{generator-decode-policy.js → policy/generator-decode-policy.js} +5 -5
  529. package/src/inference/pipelines/text/state.d.ts +9 -8
  530. package/src/inference/pipelines/text/state.js +1 -0
  531. package/src/inference/pipelines/text/types.d.ts +14 -218
  532. package/src/inference/pipelines/text.js +2 -0
  533. package/src/inference/tokenizers/bundled/merge.js +130 -0
  534. package/src/inference/tokenizers/bundled.js +2 -36
  535. package/src/loader/doppler-loader.d.ts +17 -7
  536. package/src/loader/doppler-loader.js +27 -29
  537. package/src/loader/embedding-limit-preflight.js +29 -1
  538. package/src/loader/embedding-loader.js +18 -4
  539. package/src/loader/experts/compact-expert-cache.js +488 -0
  540. package/src/loader/experts/expert-cache.d.ts +1 -1
  541. package/src/loader/experts/expert-dequant-cache.d.ts +9 -0
  542. package/src/loader/experts/expert-dequant-cache.js +75 -0
  543. package/src/loader/experts/expert-frequency-index.js +129 -0
  544. package/src/loader/experts/expert-loader.d.ts +4 -0
  545. package/src/loader/experts/expert-loader.js +13 -1
  546. package/src/loader/experts/expert-preparation.d.ts +1 -1
  547. package/src/loader/experts/expert-preparation.js +1 -1
  548. package/src/loader/experts/loader-bindings.js +117 -13
  549. package/src/loader/experts/mapped-expert-upload-cache.d.ts +61 -0
  550. package/src/loader/experts/mapped-expert-upload-cache.js +382 -0
  551. package/src/loader/experts/qwen-paging.d.ts +1 -1
  552. package/src/loader/experts/qwen-paging.js +1 -1
  553. package/src/loader/experts/routed-expert-cache.d.ts +62 -3
  554. package/src/loader/experts/routed-expert-cache.js +451 -99
  555. package/src/loader/experts/routed-expert-plan.d.ts +2 -0
  556. package/src/loader/experts/routed-expert-plan.js +14 -8
  557. package/src/loader/experts/routed-expert-prefill.js +82 -0
  558. package/src/loader/experts/routed-expert-source.js +233 -0
  559. package/src/loader/experts/routed-expert-trace.js +77 -0
  560. package/src/loader/experts/tiered-expert-cache.d.ts +12 -0
  561. package/src/loader/experts/tiered-expert-cache.js +287 -0
  562. package/src/loader/experts/tiered-expert-plan.js +74 -0
  563. package/src/loader/final-weights-loader.d.ts +1 -1
  564. package/src/loader/hyper-connection-loader.d.ts +2 -2
  565. package/src/loader/hyper-connection-loader.js +85 -13
  566. package/src/loader/layer-loader.d.ts +3 -3
  567. package/src/loader/layer-loader.js +1 -1
  568. package/src/loader/layer-tensor-aliases.js +14 -11
  569. package/src/loader/loader-types.d.ts +6 -1
  570. package/src/loader/manifest-config.js +4 -0
  571. package/src/loader/model-executor.js +18 -1
  572. package/src/loader/model-load.d.ts +1 -1
  573. package/src/loader/model-load.js +1 -1
  574. package/src/loader/ngram-table-bindings.d.ts +3 -3
  575. package/src/loader/ngram-table-bindings.js +7 -6
  576. package/src/loader/q4k-reference-weight.js +1 -1
  577. package/src/loader/shard-cache.d.ts +8 -1
  578. package/src/loader/shard-cache.js +33 -0
  579. package/src/loader/tensors/bf16-loader.js +2 -1
  580. package/src/loader/tensors/fp8-loader.js +1 -1
  581. package/src/loader/tensors/load-executor.js +10 -2
  582. package/src/loader/tensors/tensor-loader.js +1 -1
  583. package/src/loader/tensors/tensor-reader.js +18 -5
  584. package/src/loader/tensors/w8a8-loader.js +1 -1
  585. package/src/loader/weight-representation-plan.d.ts +1 -1
  586. package/src/memory/buffer-pool/readback.js +82 -0
  587. package/src/memory/buffer-pool.d.ts +11 -2
  588. package/src/memory/buffer-pool.js +12 -73
  589. package/src/partitions.d.ts +4 -3
  590. package/src/partitions.js +3 -2
  591. package/src/{converter → rules}/capability-transform-resolver.js +3 -3
  592. package/src/rules/generated/rule-bundle.json +1 -1
  593. package/src/rules/inference/capability-transforms.rules.json +95 -28
  594. package/src/rules/rule-registry.js +2 -0
  595. package/src/storage/artifact-range-loaders.js +42 -0
  596. package/src/storage/artifact-storage-context.d.ts +4 -0
  597. package/src/storage/artifact-storage-context.js +8 -18
  598. package/src/storage/backends/idb-store.js +1 -1
  599. package/src/storage/distribution-transport.js +1 -1
  600. package/src/storage/download/http-contract.js +1 -1
  601. package/src/storage/download/http-transport.js +4 -4
  602. package/src/storage/inventory.js +1 -1
  603. package/src/storage/model-cache.d.ts +2 -1
  604. package/src/storage/model-cache.js +12 -8
  605. package/src/storage/node-file-access.js +4 -0
  606. package/src/storage/quota.d.ts +2 -1
  607. package/src/storage/quota.js +2 -1
  608. package/src/storage/shard-manager.d.ts +1 -1
  609. package/src/storage/shards/integrity.js +17 -0
  610. package/src/storage/source-artifact-store.d.ts +1 -1
  611. package/src/storage/source-artifact-store.js +1 -0
  612. package/src/storage/source-storage-context.d.ts +4 -0
  613. package/src/storage/source-storage-context.js +14 -0
  614. package/src/storage/verified-piece-gather-lane.js +192 -0
  615. package/src/storage/verified-piece-gather-worker.js +520 -0
  616. package/src/storage/verified-piece-gather.d.ts +113 -0
  617. package/src/storage/verified-piece-gather.js +199 -0
  618. package/src/storage/verified-piece-reader.d.ts +7 -0
  619. package/src/storage/verified-piece-reader.js +69 -12
  620. package/src/storage/verified-piece-storage.d.ts +3 -1
  621. package/src/storage/verified-piece-storage.js +101 -11
  622. package/src/storage/verified-piece-timings.js +25 -0
  623. package/src/tooling/browser-command-runner.js +1 -1
  624. package/src/tooling/browser-harness/benchmark-execution.js +2 -2
  625. package/src/tooling/browser-harness/benchmark-rerank.js +2 -2
  626. package/src/tooling/browser-harness/inference-execution.js +1 -1
  627. package/src/tooling/browser-harness/model-resolution.js +1 -1
  628. package/src/tooling/browser-harness/runtime-config.js +1 -1
  629. package/src/tooling/browser-harness/runtime-isolation.js +2 -5
  630. package/src/tooling/browser-harness/text-evidence.js +1 -1
  631. package/src/tooling/browser-harness-contract-helpers.js +1 -1
  632. package/src/tooling/browser-harness.js +2 -2
  633. package/src/tooling/capsule-v2.js +2 -2
  634. package/src/tooling/capsule.js +2 -2
  635. package/src/tooling/command-api-constants.js +4 -2
  636. package/src/tooling/command-api.d.ts +1 -1
  637. package/src/tooling/command-api.js +15 -21
  638. package/src/tooling/command-runtime-execution.js +1 -8
  639. package/src/tooling/diagnose-runner.js +3 -41
  640. package/src/tooling/node-browser/static-server.js +2 -2
  641. package/src/tooling/node-command-execution.js +2 -34
  642. package/src/tooling/node-converter/input.js +1 -1
  643. package/src/tooling/node-converter/output.js +2 -2
  644. package/src/tooling/node-converter.js +1 -1
  645. package/src/tooling/node-runtime-bridge.js +43 -0
  646. package/src/tooling/node-source-runtime.js +34 -35
  647. package/src/tooling/production-release.js +1 -1
  648. package/src/tooling/program-bundle/materialize.js +1 -1
  649. package/src/tooling/program-bundle-parity.d.ts +1 -1
  650. package/src/tooling/program-bundle-parity.js +1 -1
  651. package/src/tooling/program-bundle.d.ts +1 -1
  652. package/src/tooling/program-bundle.js +2 -2
  653. package/src/tooling/runtime-intent.js +1 -1
  654. package/src/tooling/source-artifact-adapter.js +3 -3
  655. package/src/tooling/source-runtime/node-loader.js +32 -5
  656. package/src/tooling/source-runtime-converter-config.js +28 -6
  657. package/src/tooling-exports.shared.d.ts +3 -3
  658. package/src/tooling-exports.shared.js +2 -2
  659. package/src/version.js +1 -1
  660. package/tests/kernels/browser/kernel-suite.js +1 -1
  661. package/tests/kernels/harness/benchmark.js +1 -1
  662. package/tools/README.md +74 -0
  663. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +0 -4407
  664. package/src/inference/pipelines/text/moe-cpu.js +0 -20
  665. /package/src/config/{capsule-loading.d.ts → capsule/capsule-loading.d.ts} +0 -0
  666. /package/src/config/{capsule-release-contract.d.ts → capsule/capsule-release-contract.d.ts} +0 -0
  667. /package/src/config/{capsule-signature.d.ts → capsule/capsule-signature.d.ts} +0 -0
  668. /package/src/config/{capsule-signature.js → capsule/capsule-signature.js} +0 -0
  669. /package/src/config/{capsule-v3.d.ts → capsule/capsule-v3.d.ts} +0 -0
  670. /package/src/config/{capsule.d.ts → capsule/capsule.d.ts} +0 -0
  671. /package/src/config/schema/{diffusion.schema.d.ts → execution/diffusion.schema.d.ts} +0 -0
  672. /package/src/config/schema/{diffusion.schema.js → execution/diffusion.schema.js} +0 -0
  673. /package/src/config/schema/{energy.schema.d.ts → execution/energy.schema.d.ts} +0 -0
  674. /package/src/config/schema/{energy.schema.js → execution/energy.schema.js} +0 -0
  675. /package/src/config/schema/{hyper-connection-decoder.schema.d.ts → execution/hyper-connection-decoder.schema.d.ts} +0 -0
  676. /package/src/config/schema/{hyper-connection-decoder.schema.js → execution/hyper-connection-decoder.schema.js} +0 -0
  677. /package/src/config/schema/{inference.schema.js → execution/inference.schema.js} +0 -0
  678. /package/src/config/schema/{kernel-path.schema.js → execution/kernel-path.schema.js} +0 -0
  679. /package/src/config/schema/{kernel-registry.schema.js → execution/kernel-registry.schema.js} +0 -0
  680. /package/src/config/schema/{kernel-warmup.schema.d.ts → execution/kernel-warmup.schema.d.ts} +0 -0
  681. /package/src/config/schema/{kernel-warmup.schema.js → execution/kernel-warmup.schema.js} +0 -0
  682. /package/src/config/schema/{speculation-self.schema.d.ts → execution/speculation-self.schema.d.ts} +0 -0
  683. /package/src/config/schema/{speculation-self.schema.js → execution/speculation-self.schema.js} +0 -0
  684. /package/src/config/schema/{speculative.schema.d.ts → execution/speculative.schema.d.ts} +0 -0
  685. /package/src/config/schema/{speculative.schema.js → execution/speculative.schema.js} +0 -0
  686. /package/src/config/schema/{conversion-report.schema.d.ts → manifest/conversion-report.schema.d.ts} +0 -0
  687. /package/src/config/schema/{conversion-report.schema.js → manifest/conversion-report.schema.js} +0 -0
  688. /package/src/config/schema/{conversion.schema.d.ts → manifest/conversion.schema.d.ts} +0 -0
  689. /package/src/config/schema/{conversion.schema.js → manifest/conversion.schema.js} +0 -0
  690. /package/src/config/schema/{fp8.schema.d.ts → manifest/fp8.schema.d.ts} +0 -0
  691. /package/src/config/schema/{fp8.schema.js → manifest/fp8.schema.js} +0 -0
  692. /package/src/config/schema/{manifest-rope.schema.d.ts → manifest/manifest-rope.schema.d.ts} +0 -0
  693. /package/src/config/schema/{quantization-defaults.schema.d.ts → manifest/quantization-defaults.schema.d.ts} +0 -0
  694. /package/src/config/schema/{quantization-defaults.schema.js → manifest/quantization-defaults.schema.js} +0 -0
  695. /package/src/config/schema/{quantization.schema.d.ts → manifest/quantization.schema.d.ts} +0 -0
  696. /package/src/config/schema/{quantization.schema.js → manifest/quantization.schema.js} +0 -0
  697. /package/src/config/schema/{w8a8.schema.d.ts → manifest/w8a8.schema.d.ts} +0 -0
  698. /package/src/config/schema/{w8a8.schema.js → manifest/w8a8.schema.js} +0 -0
  699. /package/src/config/schema/{gpu-cache.schema.d.ts → platform/gpu-cache.schema.d.ts} +0 -0
  700. /package/src/config/schema/{gpu-cache.schema.js → platform/gpu-cache.schema.js} +0 -0
  701. /package/src/config/schema/{memory-limits.schema.d.ts → platform/memory-limits.schema.d.ts} +0 -0
  702. /package/src/config/schema/{platform.schema.d.ts → platform/platform.schema.d.ts} +0 -0
  703. /package/src/config/schema/{support-tiers.schema.d.ts → policy/support-tiers.schema.d.ts} +0 -0
  704. /package/src/config/schema/{support-tiers.schema.js → policy/support-tiers.schema.js} +0 -0
  705. /package/src/config/schema/{adapter.schema.d.ts → runtime/adapter.schema.d.ts} +0 -0
  706. /package/src/config/schema/{adapter.schema.js → runtime/adapter.schema.js} +0 -0
  707. /package/src/config/schema/{benchmark.schema.d.ts → runtime/benchmark.schema.d.ts} +0 -0
  708. /package/src/config/schema/{benchmark.schema.js → runtime/benchmark.schema.js} +0 -0
  709. /package/src/config/schema/{bridge.schema.d.ts → runtime/bridge.schema.d.ts} +0 -0
  710. /package/src/config/schema/{bridge.schema.js → runtime/bridge.schema.js} +0 -0
  711. /package/src/config/schema/{browser-suite-metrics.schema.d.ts → runtime/browser-suite-metrics.schema.d.ts} +0 -0
  712. /package/src/config/schema/{browser-suite-metrics.schema.js → runtime/browser-suite-metrics.schema.js} +0 -0
  713. /package/src/config/schema/{buffer-pool.schema.d.ts → runtime/buffer-pool.schema.d.ts} +0 -0
  714. /package/src/config/schema/{buffer-pool.schema.js → runtime/buffer-pool.schema.js} +0 -0
  715. /package/src/config/schema/{debug.schema.d.ts → runtime/debug.schema.d.ts} +0 -0
  716. /package/src/config/schema/{debug.schema.js → runtime/debug.schema.js} +0 -0
  717. /package/src/config/schema/{distribution.schema.d.ts → runtime/distribution.schema.d.ts} +0 -0
  718. /package/src/config/schema/{distribution.schema.js → runtime/distribution.schema.js} +0 -0
  719. /package/src/config/schema/{ecosystem.schema.d.ts → runtime/ecosystem.schema.d.ts} +0 -0
  720. /package/src/config/schema/{ecosystem.schema.js → runtime/ecosystem.schema.js} +0 -0
  721. /package/src/config/schema/{harness.schema.d.ts → runtime/harness.schema.d.ts} +0 -0
  722. /package/src/config/schema/{harness.schema.js → runtime/harness.schema.js} +0 -0
  723. /package/src/config/schema/{hotswap.schema.d.ts → runtime/hotswap.schema.d.ts} +0 -0
  724. /package/src/config/schema/{hotswap.schema.js → runtime/hotswap.schema.js} +0 -0
  725. /package/src/config/schema/{intent-bundle.schema.d.ts → runtime/intent-bundle.schema.d.ts} +0 -0
  726. /package/src/config/schema/{intent-bundle.schema.js → runtime/intent-bundle.schema.js} +0 -0
  727. /package/src/config/schema/{runtime-profile.schema.d.ts → runtime/runtime-profile.schema.d.ts} +0 -0
  728. /package/src/config/schema/{runtime-profile.schema.js → runtime/runtime-profile.schema.js} +0 -0
  729. /package/src/config/schema/{storage.schema.d.ts → runtime/storage.schema.d.ts} +0 -0
  730. /package/src/config/schema/{storage.schema.js → runtime/storage.schema.js} +0 -0
  731. /package/src/config/schema/{tooling.schema.js → runtime/tooling.schema.js} +0 -0
  732. /package/src/config/schema/{units.schema.d.ts → runtime/units.schema.d.ts} +0 -0
  733. /package/src/config/schema/{units.schema.js → runtime/units.schema.js} +0 -0
  734. /package/src/config/schema/{backward-registry.schema.d.ts → training/backward-registry.schema.d.ts} +0 -0
  735. /package/src/config/schema/{backward-registry.schema.js → training/backward-registry.schema.js} +0 -0
  736. /package/src/config/schema/{distill-training.schema.d.ts → training/distill-training.schema.d.ts} +0 -0
  737. /package/src/config/schema/{distill-training.schema.js → training/distill-training.schema.js} +0 -0
  738. /package/src/config/schema/{lora.schema.d.ts → training/lora.schema.d.ts} +0 -0
  739. /package/src/config/schema/{training-metrics.schema.d.ts → training/training-metrics.schema.d.ts} +0 -0
  740. /package/src/config/schema/{training-metrics.schema.js → training/training-metrics.schema.js} +0 -0
  741. /package/src/config/schema/{training.schema.d.ts → training/training.schema.d.ts} +0 -0
  742. /package/src/config/schema/{training.schema.js → training/training.schema.js} +0 -0
  743. /package/src/config/schema/{ul-training.schema.d.ts → training/ul-training.schema.d.ts} +0 -0
  744. /package/src/config/schema/{ul-training.schema.js → training/ul-training.schema.js} +0 -0
  745. /package/src/{converter → config/transforms}/execution-graph/dtype.js +0 -0
  746. /package/src/{converter → config/transforms}/execution-graph/fusion.js +0 -0
  747. /package/src/{converter → config/transforms}/execution-graph/kv-storage.js +0 -0
  748. /package/src/{converter → config/transforms}/execution-graph/validation.js +0 -0
  749. /package/src/{converter → formats}/qwen4-exp-source-defaults.json +0 -0
  750. /package/src/inference/pipelines/text/{buffer-types.d.ts → contracts/buffer-types.d.ts} +0 -0
  751. /package/src/inference/pipelines/text/{embedding-contract.d.ts → embedding/embedding-contract.d.ts} +0 -0
  752. /package/src/inference/pipelines/text/{embedding-contract.js → embedding/embedding-contract.js} +0 -0
  753. /package/src/inference/pipelines/text/{precomputed-input-norm.js → execution/precomputed-input-norm.js} +0 -0
  754. /package/src/inference/pipelines/text/generator/{batch-retry-policy.js → decode/batch-retry-policy.js} +0 -0
  755. /package/src/inference/pipelines/text/{chat-format.d.ts → generator/request/chat-format.d.ts} +0 -0
  756. /package/src/inference/pipelines/text/{chat-format.js → generator/request/chat-format.js} +0 -0
  757. /package/src/inference/pipelines/text/generator/{recorder-stats.js → session/recorder-stats.js} +0 -0
  758. /package/src/inference/pipelines/text/generator/{whole-token-template.js → whole-token/whole-token-template.js} +0 -0
  759. /package/src/inference/pipelines/text/{init-chat-templates.js → initialization/init-chat-templates.js} +0 -0
  760. /package/src/inference/pipelines/text/{load-policy.js → loading/load-policy.js} +0 -0
  761. /package/src/inference/pipelines/text/{pipeline-load-timing.js → loading/pipeline-load-timing.js} +0 -0
  762. /package/src/inference/pipelines/text/{generator → logits}/hidden-projection.d.ts +0 -0
  763. /package/src/inference/pipelines/text/{operator-events.d.ts → observation/operator-events.d.ts} +0 -0
  764. /package/src/inference/pipelines/text/{stage-names.d.ts → observation/stage-names.d.ts} +0 -0
  765. /package/src/inference/pipelines/text/{stage-names.js → observation/stage-names.js} +0 -0
  766. /package/src/inference/pipelines/text/{tsir-fixture-writer.d.ts → observation/tsir-fixture-writer.d.ts} +0 -0
  767. /package/src/inference/pipelines/text/{generator → per-layer}/per-layer-buffer.js +0 -0
  768. /package/src/inference/pipelines/text/{finiteness-policy.js → policy/finiteness-policy.js} +0 -0
@@ -1,47 +1,130 @@
1
+ import { isRoutedExpertTraceActive, readRoutedExpertSelection } from '../../../../loader/experts/routed-expert-trace.js';
2
+ import { getRuntimeConfig } from '../../../../config/runtime.js';
1
3
  import { acquireBuffer, readBufferSlice, releaseBuffer } from '../../../../memory/buffer-pool.js';
2
4
  import { getDevice } from '../../../../gpu/device.js';
3
5
  import { createTensor } from '../../../../gpu/tensor.js';
4
- import { runQwenRouteExperts } from './qwen-route-executor.js';
6
+ import { runRouteRemap } from '../../../../gpu/kernels/moe-route.js';
7
+ import { runQwenRouteExperts, recordQwenRouteExperts } from './qwen-route-executor.js';
8
+
9
+ /** @param {import('../../../../loader/experts/routed-expert-cache.js').RoutedExpertWeights} weights
10
+ * @param {GPUBuffer} indices @param {number} numTokens @param {number} topK
11
+ * @param {number[]} ids @param {boolean} allowPrefill @param {string | undefined} phase
12
+ * @param {import('../../../../gpu/command-recorder.js').CommandRecorder | null} recorder */
13
+ async function remapCompactIndices(weights, indices, numTokens, topK, ids, allowPrefill, phase, recorder) {
14
+ if (!weights.sourceExpertToSlot) return null;
15
+ const sourceNumExperts = weights.sourceNumExperts;
16
+ if (!((phase === 'decode' && numTokens === 1) || (allowPrefill && phase === 'prefill'))
17
+ || !Number.isSafeInteger(sourceNumExperts) || !sourceNumExperts
18
+ || ids.some(id => id >= sourceNumExperts)) {
19
+ throw new Error('Compact expert slots require valid source IDs and an admitted decode or tiered-prefill chunk.');
20
+ }
21
+ return runRouteRemap(indices, weights.sourceExpertToSlot, numTokens * topK,
22
+ sourceNumExperts, weights.numExperts, recorder);
23
+ }
5
24
 
6
25
  // Actual GPU selections, not predictions. The next metadata read precedes the
7
26
  // current chunk's dispatch, so known-route I/O can overlap that dispatch.
8
27
  /** @type {typeof import('./routed-expert-chunks.js').executeRoutedExpertChunks} */
9
- export async function executeRoutedExpertChunks(options, loader, policy, signal) {
10
- if (options.sharedSlot || options.groupedPrefill || options.activationDtype !== 'f32') {
11
- throw new Error('Routed expert loading requires the ordinary immediate F32 route executor.');
28
+ export async function executeRoutedExpertChunks(options, loader, policy, signal, recorder = null) {
29
+ if (options.activationDtype !== 'f32') {
30
+ throw new Error('Routed expert loading requires an F32 route executor.');
31
+ }
32
+ if (options.groupedPrefill && (options.phase !== 'prefill' || options.sharedSlot || recorder)) {
33
+ throw new Error('Grouped routed experts require immediate prefill without shared-slot execution.');
34
+ }
35
+ // Shared slots keep their layer-owned shared bank. Only the routed banks and
36
+ // expert IDs are compacted. Multi-token auxiliary gate/residual slicing is
37
+ // deliberately excluded; this path hands over the complete single token.
38
+ if (options.sharedSlot && (options.phase !== 'decode' || options.numTokens !== 1 || options.sharedSlot.residual)) {
39
+ throw new Error('Paged shared slots require single-token decode without an outer residual.');
40
+ }
41
+ if (recorder && (options.phase !== 'decode' || options.numTokens !== 1)) {
42
+ throw new Error('Recorded routed experts require single-token decode.');
12
43
  }
13
44
  const { numTokens, topK, hiddenSize, indicesBuffer, weightsBuffer, inputTensor, layerIdx } = options;
14
45
  if (typeof layerIdx !== 'number' || !Number.isSafeInteger(layerIdx)) throw new Error('Routed expert loading requires an explicit layer.');
15
46
  const device = getDevice();
16
47
  if (!device) throw new Error('Routed expert loading requires the owning GPU device.');
48
+ const tiered = policy.storageMode === 'tiered-compact';
49
+ signal?.throwIfAborted();
50
+ const resident = tiered ? loader.getResidentRoutedExpertWeights?.(layerIdx) : null;
51
+ if (resident) {
52
+ if (resident.sourceExpertToSlot) throw new Error('Resident routed experts require a complete direct-index bank.');
53
+ if (isRoutedExpertTraceActive(loader)) for (let start = 0; start < numTokens; start += policy.routeChunkTokens) {
54
+ const count = Math.min(policy.routeChunkTokens, numTokens - start);
55
+ await readRoutedExpertSelection(loader, { layer: layerIdx, phase: options.phase ?? 'unspecified',
56
+ tokenOffset: start, numTokens: count, topK, source: 'resident' },
57
+ () => readBufferSlice(indicesBuffer, start * topK * 4, count * topK * 4, recorder), false);
58
+ signal?.throwIfAborted();
59
+ }
60
+
61
+ return recorder ? recordQwenRouteExperts(recorder, { ...options, weights: resident })
62
+ : runQwenRouteExperts({ ...options, weights: resident });
63
+ }
64
+ const capacity = tiered || (policy.storageMode === 'compact-decode' && options.phase === 'decode')
65
+ ? loader.getRoutedExpertCapacity?.(layerIdx, options.phase) : undefined;
66
+ if (tiered && capacity === undefined) throw new Error('Tiered routed experts require an explicit admitted capacity.');
67
+ if (capacity !== undefined && (!Number.isSafeInteger(capacity) || capacity < topK)) {
68
+ throw new Error('Routed expert capacity cannot fit one complete top-K selection.');
69
+ }
70
+ const chunkTokens = capacity === undefined ? policy.routeChunkTokens
71
+ : Math.min(policy.routeChunkTokens, Math.floor(capacity / topK));
17
72
  /** @param {number} start */
18
73
  const selections = async start => {
19
74
  signal?.throwIfAborted();
20
- const count = Math.min(policy.routeChunkTokens, numTokens - start);
21
- const data = await readBufferSlice(indicesBuffer, start * topK * 4, count * topK * 4);
75
+ const count = Math.min(chunkTokens, numTokens - start);
76
+ const profile = getRuntimeConfig().shared.debug.profiler.enabled;
77
+ const started = profile ? performance.now() : 0;
78
+ // Append the selection copy to routing's encoder and submit that prefix once.
79
+ // Mapping still settles all earlier bank consumers before cache reuse. The
80
+ // outer owner keeps temporaries until its final submission.
81
+ const data = isRoutedExpertTraceActive(loader) ? await readRoutedExpertSelection(loader, { layer: layerIdx, phase: options.phase ?? 'unspecified',
82
+ tokenOffset: start, numTokens: count, topK, source: 'paging' },
83
+ () => readBufferSlice(indicesBuffer, start * topK * 4, count * topK * 4, recorder), true)
84
+ : await readBufferSlice(indicesBuffer, start * topK * 4, count * topK * 4, recorder);
85
+ if (!data) throw new Error('Routed paging requires selection readback.');
86
+ if (profile) loader.recordRoutedReadback?.(options.phase, performance.now() - started, count * topK * 4);
22
87
  signal?.throwIfAborted();
23
88
  return [...new Set(new Uint32Array(data))];
24
89
  };
25
90
  let ids = await selections(0);
26
- if (numTokens <= policy.routeChunkTokens) {
27
- const weights = await loader.prepareRoutedExperts(layerIdx, ids, signal);
91
+ if (numTokens <= chunkTokens) {
92
+ const weights = await loader.prepareRoutedExperts(layerIdx, ids, signal, options.phase);
28
93
  signal?.throwIfAborted();
29
- return runQwenRouteExperts({ ...options, weights });
94
+ if (!weights.sourceExpertToSlot) return recorder
95
+ ? recordQwenRouteExperts(recorder, { ...options, weights })
96
+ : runQwenRouteExperts({ ...options, weights });
97
+ const remapped = await remapCompactIndices(weights, indicesBuffer, numTokens, topK, ids, tiered, options.phase, recorder);
98
+ if (!remapped) throw new Error('Compact expert remap was not produced.');
99
+ if (recorder) {
100
+ recorder.trackTemporaryBuffer(remapped);
101
+ return recordQwenRouteExperts(recorder, { ...options, weights, indicesBuffer: remapped });
102
+ }
103
+ let submitted = false;
104
+ try {
105
+ const result = await runQwenRouteExperts({ ...options, weights, indicesBuffer: remapped });
106
+ submitted = true;
107
+ return result;
108
+ } finally {
109
+ try { if (!submitted) await device.queue.onSubmittedWorkDone(); }
110
+ finally { releaseBuffer(remapped); }
111
+ }
30
112
  }
31
113
  const output = acquireBuffer(numTokens * hiddenSize * 4, undefined, 'routed_prefill_output');
32
114
  let completed = false;
33
115
  let failure;
34
116
  try {
35
- for (let start = 0; start < numTokens; start += policy.routeChunkTokens) {
117
+ for (let start = 0; start < numTokens; start += chunkTokens) {
36
118
  signal?.throwIfAborted();
37
- const count = Math.min(policy.routeChunkTokens, numTokens - start);
119
+ const count = Math.min(chunkTokens, numTokens - start);
38
120
  const next = start + count < numTokens ? selections(start + count) : null;
39
121
  // A rejection is observed immediately even if weight preparation fails.
40
122
  next?.catch(() => undefined);
41
- const preparing = loader.prepareRoutedExperts(layerIdx, ids, signal);
123
+ const selectedIds = ids;
124
+ const preparing = loader.prepareRoutedExperts(layerIdx, selectedIds, signal, options.phase);
42
125
  preparing.catch(() => undefined);
43
126
  /** @type {GPUBuffer | null} */
44
- let input = null, indices = null, routes = null;
127
+ let input = null, indices = null, routes = null, remapped = null;
45
128
  let chunkOutput = null;
46
129
  let chunkComplete = false;
47
130
  let chunkFailure;
@@ -57,9 +140,10 @@ export async function executeRoutedExpertChunks(options, loader, policy, signal)
57
140
  const weights = await preparing;
58
141
  ids = next ? await next : [];
59
142
  signal?.throwIfAborted();
143
+ remapped = await remapCompactIndices(weights, indices, count, topK, selectedIds, tiered, options.phase, null);
60
144
  chunkOutput = await runQwenRouteExperts({ ...options, numTokens: count, weights,
61
145
  inputTensor: createTensor(input, 'f32', [count, hiddenSize], 'routed_chunk'),
62
- indicesBuffer: indices, weightsBuffer: routes });
146
+ indicesBuffer: remapped ?? indices, weightsBuffer: routes });
63
147
  const combine = device.createCommandEncoder({ label: 'routed_chunk_output' });
64
148
  combine.copyBufferToBuffer(chunkOutput.buffer, 0, output, start * hiddenSize * 4, count * hiddenSize * 4);
65
149
  device.queue.submit([combine.finish()]);
@@ -73,7 +157,7 @@ export async function executeRoutedExpertChunks(options, loader, policy, signal)
73
157
  if (!chunkComplete) {
74
158
  try { await device.queue.onSubmittedWorkDone(); } catch (error) { cleanup.push(error); }
75
159
  }
76
- for (const buffer of [chunkOutput?.buffer, input, indices, routes]) {
160
+ for (const buffer of [chunkOutput?.buffer, remapped, input, indices, routes]) {
77
161
  if (buffer) { try { releaseBuffer(buffer); } catch (error) { cleanup.push(error); } }
78
162
  }
79
163
  if (cleanup.length) throw new AggregateError([chunkFailure, ...cleanup].filter(Boolean), 'Routed chunk cleanup failed.');
@@ -1,20 +1,21 @@
1
1
  import { getRuntimeConfig } from '../../../config/runtime.js';
2
2
  import { QK_K } from '../../../config/schema/index.js';
3
- import { releaseBuffer } from '../../../memory/buffer-pool.js';
4
-
5
- const dequantCache = new Map();
6
- let dequantCacheMaxEntriesOverride = null;
7
- let dequantCacheHits = 0;
8
- let dequantCacheMisses = 0;
9
3
 
10
4
  export function resolveMaxTokensPerExpert(numTokens, numExperts, topK, hiddenSize, activationDtype) {
11
5
  const routingConfig = getRuntimeConfig().inference.moe.routing;
12
6
  const {
13
- maxTokensPerExpert = 0,
14
- maxTokensPerExpertHeadroom = 2.0,
15
- maxTokensPerExpertMin = 4,
16
- maxTokensPerExpertCap = 0,
7
+ maxTokensPerExpert,
8
+ maxTokensPerExpertHeadroom,
9
+ maxTokensPerExpertMin,
10
+ maxTokensPerExpertCap,
17
11
  } = routingConfig;
12
+ for (const [name, value] of Object.entries({
13
+ maxTokensPerExpert, maxTokensPerExpertHeadroom, maxTokensPerExpertMin, maxTokensPerExpertCap,
14
+ })) {
15
+ if (!Number.isFinite(value) || value < 0) {
16
+ throw new Error(`runtime.inference.moe.routing.${name} must be a finite non-negative number, got ${String(value)}.`);
17
+ }
18
+ }
18
19
 
19
20
  let target = maxTokensPerExpert > 0
20
21
  ? maxTokensPerExpert
@@ -39,70 +40,3 @@ export function resolveMaxTokensPerExpert(numTokens, numExperts, topK, hiddenSiz
39
40
  }
40
41
  return Math.min(target, numTokens);
41
42
  }
42
-
43
- function getDequantCacheMaxEntries() {
44
- return dequantCacheMaxEntriesOverride ?? getRuntimeConfig().inference.moe.cache.dequantCacheMaxEntries;
45
- }
46
-
47
- function getDequantCacheKey(layerIdx, expertIdx, outputDtype) {
48
- return `${layerIdx}_${expertIdx}_${outputDtype}`;
49
- }
50
-
51
- export function getCachedDequant(layerIdx, expertIdx, outputDtype) {
52
- const key = getDequantCacheKey(layerIdx, expertIdx, outputDtype);
53
- const cached = dequantCache.get(key);
54
- if (cached) {
55
- cached.lastUsed = performance.now();
56
- dequantCacheHits++;
57
- }
58
- return cached;
59
- }
60
-
61
- export function setCachedDequant(layerIdx, expertIdx, outputDtype, gateUp, down) {
62
- const key = getDequantCacheKey(layerIdx, expertIdx, outputDtype);
63
- dequantCacheMisses++;
64
-
65
- if (dequantCache.size >= getDequantCacheMaxEntries()) {
66
- let oldestKey = '';
67
- let oldestTime = Infinity;
68
- for (const [k, v] of dequantCache.entries()) {
69
- if (v.lastUsed < oldestTime) {
70
- oldestTime = v.lastUsed;
71
- oldestKey = k;
72
- }
73
- }
74
- if (oldestKey) {
75
- const evicted = dequantCache.get(oldestKey);
76
- if (evicted) {
77
- releaseBuffer(evicted.gateUp);
78
- releaseBuffer(evicted.down);
79
- }
80
- dequantCache.delete(oldestKey);
81
- }
82
- }
83
-
84
- dequantCache.set(key, { gateUp, down, lastUsed: performance.now() });
85
- }
86
-
87
- export function clearDequantCache() {
88
- for (const cached of dequantCache.values()) {
89
- releaseBuffer(cached.gateUp);
90
- releaseBuffer(cached.down);
91
- }
92
- dequantCache.clear();
93
- dequantCacheHits = 0;
94
- dequantCacheMisses = 0;
95
- }
96
-
97
- export function getDequantCacheStats() {
98
- return {
99
- hits: dequantCacheHits,
100
- misses: dequantCacheMisses,
101
- size: dequantCache.size,
102
- maxEntries: getDequantCacheMaxEntries(),
103
- };
104
- }
105
-
106
- export function setDequantCacheMaxEntries(maxEntries) {
107
- dequantCacheMaxEntriesOverride = maxEntries;
108
- }
@@ -8,7 +8,7 @@ import { runSoftmaxTopK, recordSoftmaxTopK } from '../../../gpu/kernels/softmax.
8
8
  import { runMoEGather, runMoEBuildTokenOffsets, runScatterAddDynamic } from '../../../gpu/kernels/moe.js';
9
9
  import { trace, isTraceEnabled } from '../../../debug/index.js';
10
10
  import { f16ToF32Array } from '../../kv-cache/types.js';
11
- import { resolveMaxTokensPerExpert, getDequantCacheStats } from './moe-cache.js';
11
+ import { resolveMaxTokensPerExpert } from './moe-cache.js';
12
12
  import { ensureExpertLoaded } from './moe/expert-loading.js';
13
13
  import { selectRuleValue } from '../../../rules/rule-registry.js';
14
14
  import { getRuntimeConfig } from '../../../config/runtime.js';
@@ -33,7 +33,7 @@ import {
33
33
  } from './moe/plan.js';
34
34
  import { runQwenRouteExperts, recordQwenRouteExperts } from './moe/qwen-route-executor.js';
35
35
  import { executeRoutedExpertChunks } from './moe/routed-expert-chunks.js';
36
- import { releaseOrTrack } from './ops.js';
36
+ import { releaseOrTrack } from "./execution/ops.js";
37
37
 
38
38
  export { buildActiveExpertScheduleFromIndices } from './moe/plan.js';
39
39
 
@@ -75,10 +75,10 @@ const MOE_RECORDED_ROUTE_EXECUTORS = Object.freeze({
75
75
  });
76
76
 
77
77
  const MOE_EXPERT_EXECUTORS = Object.freeze({
78
- 'gpt-oss': runGptOssProfileExpert,
79
- 'gemma4-packed': runGemma4ProfileExpert,
80
- 'qwen35-packed': runGemma4ProfileExpert,
81
- mixtral: runMixtralProfileExpert,
78
+ 'gpt-oss': runGptOssExpert,
79
+ 'gemma4-packed': runGemma4Expert,
80
+ 'qwen35-packed': runGemma4Expert,
81
+ mixtral: runMixtralExpert,
82
82
  });
83
83
 
84
84
  function requireMoeExecutor(registry, id, label) {
@@ -101,64 +101,6 @@ function assertMoeExpertWeights(moeProfile, weights, expertKey) {
101
101
  }
102
102
  }
103
103
 
104
- async function runGptOssProfileExpert(args) {
105
- return runGptOssExpert(
106
- args.gathered,
107
- args.expertOutputs,
108
- args.weights,
109
- args.layerIdx,
110
- args.expertIdx,
111
- args.count,
112
- args.inputOffset,
113
- args.outputOffset,
114
- args.hiddenSize,
115
- args.intermediateSize,
116
- args.numExperts,
117
- args.activationDtype,
118
- args.swigluLimit,
119
- args.kernelPath,
120
- args.executionPolicies,
121
- args.modelType,
122
- args.vendorProfile,
123
- args.moeKernelPathProfile
124
- );
125
- }
126
-
127
- async function runGemma4ProfileExpert(args) {
128
- return runGemma4Expert(
129
- args.gathered,
130
- args.expertOutputs,
131
- args.weights,
132
- args.count,
133
- args.inputOffset,
134
- args.outputOffset,
135
- args.hiddenSize,
136
- args.intermediateSize,
137
- args.activationDtype,
138
- args.swigluLimit,
139
- args.kernelPath,
140
- args.hiddenActivation,
141
- args.moeProfile.matmulRoles
142
- );
143
- }
144
-
145
- async function runMixtralProfileExpert(args) {
146
- return runMixtralExpert(
147
- args.gathered,
148
- args.expertOutputs,
149
- args.weights,
150
- args.count,
151
- args.inputOffset,
152
- args.outputOffset,
153
- args.hiddenSize,
154
- args.intermediateSize,
155
- args.hiddenActivation,
156
- args.activationDtype,
157
- args.swigluLimit,
158
- args.kernelPath
159
- );
160
- }
161
-
162
104
  export async function moeFeedForwardGPU(
163
105
  inputBuffer,
164
106
  numTokens,
@@ -194,7 +136,18 @@ export async function moeFeedForwardGPU(
194
136
  const moeProfile = resolveMoeExecutionProfile(config, { modelType });
195
137
  const activeExpertSelection = config.activeExpertSelection === undefined
196
138
  ? resolveConfiguredMoEActiveExpertSelection() : resolveMoEActiveExpertSelection(config.activeExpertSelection);
197
- await expertLoader.prepareExpertLayer?.(layerIdx, expertWeights, requestedRecorder, config.signal, activeExpertSelection);
139
+ const segmentedRoute = requestedRecorder && expertLoader.getRoutedExpertPolicy?.()?.enabled;
140
+ if (segmentedRoute) {
141
+ if (config.phase !== 'decode' || numTokens !== 1 || config.activationDtype !== 'f32'
142
+ || config.groupedPrefill || config.sharedSlot?.residual || config.normRouter || activeExpertSelection !== 'topk-route'
143
+ || moeProfile.topkRouteExecutor !== 'qwen35-route') {
144
+ throw new Error('Recorded paged experts require single-token F32 Qwen decode without outer-residual or norm fusion.');
145
+ }
146
+ // Routed loading's preparation hook is admission-only. The supported recorded
147
+ // path flushes routing and prepares actual banks at its dependency boundary.
148
+ } else {
149
+ await expertLoader.prepareExpertLayer?.(layerIdx, expertWeights, requestedRecorder, config.signal, activeExpertSelection);
150
+ }
198
151
  const recordRouteExecutor = activeExpertSelection === 'topk-route'
199
152
  ? MOE_RECORDED_ROUTE_EXECUTORS[moeProfile.topkRouteExecutor] : null;
200
153
  const recorder = recordRouteExecutor ? requestedRecorder : null;
@@ -252,14 +205,18 @@ export async function moeFeedForwardGPU(
252
205
  let activeExpertSchedule = null;
253
206
 
254
207
  const layerRouter = layerRouterWeights?.get(layerIdx) || null;
255
- if (layerRouter) {
256
- moeRouter.loadWeights(
257
- layerRouter.weight,
258
- layerRouter.bias || null,
259
- layerRouter.scale || null,
260
- layerRouter.perExpertScale || null
208
+ if (!layerRouter) {
209
+ throw new Error(
210
+ `[MoE] Layer ${layerIdx} is an MoE layer but has no router weight; ` +
211
+ 'refusing to reuse another layer\'s router. Re-convert the model with per-layer router tensors.'
261
212
  );
262
213
  }
214
+ moeRouter.loadWeights(
215
+ layerRouter.weight,
216
+ layerRouter.bias || null,
217
+ layerRouter.scale || null,
218
+ layerRouter.perExpertScale || null
219
+ );
263
220
 
264
221
  // Immediate-only adapters must execute after their recorded producers.
265
222
  // Keep the parent recorder open for following residuals and layers.
@@ -491,7 +448,7 @@ export async function moeFeedForwardGPU(
491
448
  hiddenActivation,
492
449
  };
493
450
  outputTensor = routedPolicy?.enabled
494
- ? await executeRoutedExpertChunks(routeOptions, expertLoader, routedPolicy, config.signal)
451
+ ? await executeRoutedExpertChunks(routeOptions, expertLoader, routedPolicy, config.signal, recorder)
495
452
  : recorder
496
453
  ? await recordRouteExecutor(recorder, routeOptions)
497
454
  : await routeExecutor(routeOptions);
@@ -623,6 +580,7 @@ export async function moeFeedForwardGPU(
623
580
  vendorProfile,
624
581
  moeKernelPathProfile,
625
582
  hiddenActivation,
583
+ dequantCache: expertLoader.expertDequantCache ?? null,
626
584
  });
627
585
  perfLog(`MoE L${layerIdx} expert_exec`, stepStart, { expertIdx, count });
628
586
  }
@@ -665,7 +623,7 @@ export async function moeFeedForwardGPU(
665
623
  weightsBuffer = null;
666
624
 
667
625
  if (perfEnabled) {
668
- const cacheStats = getDequantCacheStats();
626
+ const cacheStats = expertLoader.expertDequantCache?.getStats() ?? null;
669
627
  trace.perf(`MoE L${layerIdx} done`, {
670
628
  numTokens,
671
629
  topK,
@@ -674,8 +632,8 @@ export async function moeFeedForwardGPU(
674
632
  activeExpertSelection: activeExpertSchedule?.selection ?? null,
675
633
  rowsPerExpert,
676
634
  maxTokensPerExpert,
677
- dequantCacheHits: cacheStats.hits,
678
- dequantCacheMisses: cacheStats.misses,
635
+ dequantCacheHits: cacheStats?.hits ?? null,
636
+ dequantCacheMisses: cacheStats?.misses ?? null,
679
637
  expertCache: typeof expertLoader?.getExpertCacheStats === 'function'
680
638
  ? expertLoader.getExpertCacheStats()
681
639
  : null,
@@ -19,29 +19,9 @@ import type { Tensor, TensorDtype } from '../../../gpu/tensor.js';
19
19
  import type { MoERouter } from '../../moe-router.js';
20
20
  import type { ExpertWeights } from './types.js';
21
21
  import type { KernelPathSchema } from '../../../config/schema/index.js';
22
- import type { ExecutionV1PoliciesSchema } from '../../../config/schema/execution-v1.schema.js';
22
+ import type { ExecutionV1PoliciesSchema } from "../../../config/schema/execution/execution-v1.schema.js";
23
23
  import type { WeightBuffer } from '../../../gpu/weight-buffer.js';
24
24
 
25
- /**
26
- * Clear the dequantization cache (call on model unload).
27
- */
28
- export function clearDequantCache(): void;
29
-
30
- /**
31
- * Get cache stats for debugging.
32
- */
33
- export function getDequantCacheStats(): {
34
- hits: number;
35
- misses: number;
36
- size: number;
37
- maxEntries: number;
38
- };
39
-
40
- /**
41
- * Configure dequant cache max entries at runtime.
42
- */
43
- export function setDequantCacheMaxEntries(maxEntries: number): void;
44
-
45
25
  /**
46
26
  * Configuration for MoE feed-forward.
47
27
  */
@@ -113,21 +93,10 @@ export interface ExpertLoader {
113
93
  recorder: import('../../../gpu/command-recorder.js').CommandRecorder | null,
114
94
  signal?: AbortSignal): Promise<unknown>;
115
95
  loadExpert(layerIdx: number, expertIdx: number): Promise<MoEExpertWeights | null>;
96
+ /** False once the loader's LRU cache evicted (and released) `weights`. */
97
+ isExpertResident(layerIdx: number, expertIdx: number, weights: unknown): boolean;
116
98
  }
117
99
 
118
- /**
119
- * MoE feed-forward with CPU routing.
120
- */
121
- export function moeFeedForwardCPU(
122
- hiddenStates: Float32Array,
123
- numTokens: number,
124
- config: MoEConfig,
125
- moeRouter: MoERouter,
126
- expertWeights: Map<string, MoEExpertWeights>,
127
- expertLoader: ExpertLoader,
128
- layerIdx: number
129
- ): Promise<Float32Array>;
130
-
131
100
  /**
132
101
  * MoE feed-forward fully on GPU.
133
102
  */
@@ -143,7 +112,3 @@ export function moeFeedForwardGPU(
143
112
  recorder?: import('../../../gpu/command-recorder.js').CommandRecorder | null
144
113
  ): Promise<GPUBuffer>;
145
114
 
146
- /**
147
- * Check if layer is MoE layer (some models have dense layers too).
148
- */
149
- export function isMoELayer(layerIdx: number): boolean;
@@ -1,9 +1,3 @@
1
- import { clearDequantCache, getDequantCacheStats, setDequantCacheMaxEntries } from './moe-cache.js';
2
- import { moeFeedForwardCPU } from './moe-cpu.js';
3
1
  import { moeFeedForwardGPU } from './moe-gpu.js';
4
2
 
5
- export { clearDequantCache, getDequantCacheStats, setDequantCacheMaxEntries, moeFeedForwardCPU, moeFeedForwardGPU };
6
-
7
- export function isMoELayer(_layerIdx) {
8
- return true;
9
- }
3
+ export { moeFeedForwardGPU };
@@ -1,7 +1,7 @@
1
1
  import type {
2
2
  ResolvedNgramEmbedding,
3
3
  ResolvedNgramEmbeddingLayer,
4
- } from '../../../../config/schema/ngram-embedding.schema.js';
4
+ } from "../../../../config/schema/execution/ngram-embedding.schema.js";
5
5
 
6
6
  export interface NgramHistoryCheckpoint {
7
7
  readonly epoch: number;
@@ -25,8 +25,8 @@ export class NgramTokenHistory {
25
25
  #epoch = 0;
26
26
 
27
27
  /**
28
- * @param {import('../../../../config/schema/ngram-embedding.schema.js').ResolvedNgramEmbedding} contract
29
- * @param {import('../../../../config/schema/ngram-embedding.schema.js').ResolvedNgramEmbeddingLayer} layer
28
+ * @param {import("../../../../config/schema/execution/ngram-embedding.schema.js").ResolvedNgramEmbedding} contract
29
+ * @param {import("../../../../config/schema/execution/ngram-embedding.schema.js").ResolvedNgramEmbeddingLayer} layer
30
30
  * @param {number} [initialCapacity]
31
31
  */
32
32
  constructor(contract, layer, initialCapacity = 256) {
@@ -1,4 +1,4 @@
1
- import type { ResolvedNgramEmbeddingSession } from '../../../../config/schema/ngram-embedding.schema.js';
1
+ import type { ResolvedNgramEmbeddingSession } from "../../../../config/schema/execution/ngram-embedding.schema.js";
2
2
 
3
3
  export type NgramTablePartSource = string | number;
4
4
 
@@ -22,7 +22,7 @@ export type NgramTableRangePort = (
22
22
  export interface NgramTableReaderOptions {
23
23
  rows: number;
24
24
  rowWidth: number;
25
- dtype: 'bf16';
25
+ dtype: 'bf16' | 'q4_0';
26
26
  /** Must tile [0, rows) in order. */
27
27
  parts: NgramTablePart[];
28
28
  readRange: NgramTableRangePort;
@@ -48,7 +48,7 @@ export declare class NgramTableReader {
48
48
  constructor(options: NgramTableReaderOptions);
49
49
  readonly rows: number;
50
50
  readonly rowWidth: number;
51
- readonly dtype: 'bf16';
51
+ readonly dtype: 'bf16' | 'q4_0';
52
52
  readonly rowBytes: number;
53
53
  readonly parts: readonly Readonly<Omit<NgramTablePart, 'byteLength'>>[];
54
54
  readonly rowBytesVerified: boolean;
@@ -1,3 +1,5 @@
1
+ import { ngramTableRowBytes } from "../../../../config/schema/execution/ngram-embedding.schema.js";
2
+
1
3
  // Host-resident reader of one qwen4_exp n-gram embedding table.
2
4
  //
3
5
  // The table is one logical [rows, rowWidth] matrix stored as row-contiguous
@@ -10,10 +12,9 @@
10
12
  // maxBytes; one read batch stages at most staging.maxRows rows; at most
11
13
  // reads.maxConcurrent range reads are in flight. Cancellation aborts pending
12
14
  // reads and inserts nothing from an incomplete batch. Row bytes are returned
13
- // as stored (bf16); this reader authenticates the row-to-part mapping, not the
15
+ // as stored (bf16 or block-packed q4_0); this reader authenticates the row-to-part mapping, not the
14
16
  // bytes of individual rows (see `rowBytesVerified` in stats).
15
17
 
16
- const ROW_ELEMENT_BYTES = Object.freeze({ bf16: 2 });
17
18
 
18
19
  /** @param {string} message @returns {never} */
19
20
  function fail(message) {
@@ -33,8 +34,6 @@ export class NgramTableReader {
33
34
 
34
35
  /** @param {import('./ngram-table.js').NgramTableReaderOptions} options */
35
36
  constructor({ rows, rowWidth, dtype, parts, readRange, session, rowBytesVerified = false }) {
36
- const elementBytes = ROW_ELEMENT_BYTES[/** @type {'bf16'} */ (dtype)];
37
- if (!elementBytes) fail(`unsupported table dtype ${JSON.stringify(dtype)}.`);
38
37
  if (!Number.isSafeInteger(rows) || rows < 1 || rows > 2 ** 32) fail('rows must be a positive count within u32 addressing.');
39
38
  if (!Number.isSafeInteger(rowWidth) || rowWidth < 1) fail('rowWidth must be positive.');
40
39
  if (typeof readRange !== 'function') fail('a readRange(part, offset, length, signal) port is required.');
@@ -42,7 +41,7 @@ export class NgramTableReader {
42
41
  this.rows = rows;
43
42
  this.rowWidth = rowWidth;
44
43
  this.dtype = dtype;
45
- this.rowBytes = rowWidth * elementBytes;
44
+ this.rowBytes = ngramTableRowBytes(rowWidth, dtype);
46
45
  this.readRange = readRange;
47
46
  this.session = session;
48
47
  this.rowBytesVerified = rowBytesVerified === true;
@@ -214,7 +213,7 @@ export class NgramTableReader {
214
213
  export function ngramTablePartsFromCheckpointHeaders(receipt) {
215
214
  const table = receipt?.ngramTable;
216
215
  if (!table || !Array.isArray(table.shards)) fail('checkpoint header receipt lacks ngramTable.shards.');
217
- const rowBytes = table.rowWidth * ROW_ELEMENT_BYTES[/** @type {'bf16'} */ (String(table.dtype).toLowerCase())];
216
+ const rowBytes = ngramTableRowBytes(table.rowWidth, String(table.dtype).toLowerCase());
218
217
  return table.shards.map((shard) => {
219
218
  if (shard.fileByteEnd - shard.fileByteBegin !== (shard.rowEnd - shard.rowBegin) * rowBytes) {
220
219
  fail(`shard ${shard.shard} byte span does not match its rows.`);
@@ -1,7 +1,7 @@
1
1
  import type { CommandRecorder } from '../../../../gpu/command-recorder.js';
2
2
  import type { Tensor } from '../../../../gpu/tensor.js';
3
- import type { ResolvedNgramEmbedding } from '../../../../config/schema/ngram-embedding.schema.js';
4
- import type { ResolvedHyperConnections } from '../../../../config/schema/hyper-connection-decoder.schema.js';
3
+ import type { ResolvedNgramEmbedding } from "../../../../config/schema/execution/ngram-embedding.schema.js";
4
+ import type { RuntimeHyperConnections } from '../hyper-connection/decoder-layer.js';
5
5
 
6
6
  export declare class PleConvState {
7
7
  constructor(options: { channels: number; stateLength: number });
@@ -42,7 +42,7 @@ export interface PleLayerInput {
42
42
  weights: PleLayerWeights;
43
43
  state: PleConvState;
44
44
  ngram: ResolvedNgramEmbedding;
45
- hyper: ResolvedHyperConnections;
45
+ hyper: RuntimeHyperConnections;
46
46
  numTokens: number;
47
47
  project: PleProjection;
48
48
  }
@@ -75,15 +75,15 @@ export async function runPleLayer(target, { streams, rows, weights, state, ngram
75
75
  };
76
76
  let retainedGate = null;
77
77
  let returned = false;
78
- const normOptions = { numTokens, streamCount: hyper.streamCount, streamWidth: hyper.streamWidth, eps: ngram.normEps };
78
+ const normOptions = { numTokens, streamCount: hyper.streamCount, streamWidth: hyper.streamWidth, eps: ngram.normEps, normWeightOffset: hyper.normWeightOffset };
79
79
  /** @param {import('../../../../gpu/tensor.js').Tensor} input @param {import('../../../../gpu/tensor.js').Tensor} weight */
80
80
  const norm = (input, weight) => (recorded
81
81
  ? recordHyperConnectionNorm(target, input, weight, normOptions)
82
82
  : runHyperConnectionNorm(input, weight, normOptions));
83
83
  try {
84
84
  const embedding = keep(recorded
85
- ? await recordPleRowDecode(target, rows, { numTokens, embedDim: ngram.embedDim })
86
- : await runPleRowDecode(rows, { numTokens, embedDim: ngram.embedDim }));
85
+ ? await recordPleRowDecode(target, rows, { numTokens, embedDim: ngram.embedDim, tableDtype: ngram.tableDtype })
86
+ : await runPleRowDecode(rows, { numTokens, embedDim: ngram.embedDim, tableDtype: ngram.tableDtype }));
87
87
  const key = keep(await project('ple_key_proj', embedding, weights.keyProj, channels, ngram.embedDim));
88
88
  const keyNormed = keep(await norm(key, weights.normKey));
89
89
  const value = keep(await project('ple_value_proj', embedding, weights.valueProj, hyper.streamWidth, ngram.embedDim));