doppler-gpu 0.7.2 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (770) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/package.json +5 -48
  3. package/src/capsule-runtime.d.ts +2 -2
  4. package/src/cli/doppler-cli.js +3 -1
  5. package/src/cli/doppler-serve.js +3 -1
  6. package/src/client/capsule-host.browser.js +4 -4
  7. package/src/client/capsule-host.d.ts +8 -8
  8. package/src/client/capsule-host.js +4 -4
  9. package/src/client/doppler-api.d.ts +2 -2
  10. package/src/client/inspection.d.ts +3 -0
  11. package/src/client/inspection.js +29 -0
  12. package/src/client/model-host/index.d.ts +5 -4
  13. package/src/client/model-host/index.js +28 -8
  14. package/src/client/model-host/model-evidence.js +2 -0
  15. package/src/client/model-host/model-session.d.ts +7 -5
  16. package/src/client/model-host/model-session.js +10 -3
  17. package/src/client/model-host/privacy-receipt.js +22 -0
  18. package/src/client/resident-partitions.d.ts +3 -3
  19. package/src/client/resident-partitions.js +2 -2
  20. package/src/client/runtime/capsule-acquisition.d.ts +3 -3
  21. package/src/client/runtime/capsule-acquisition.js +1 -1
  22. package/src/client/runtime/capsule-adapter-execution.d.ts +2 -2
  23. package/src/client/runtime/capsule-adapter-execution.js +3 -3
  24. package/src/client/runtime/capsule-embedding.d.ts +2 -2
  25. package/src/client/runtime/capsule-embedding.js +1 -1
  26. package/src/client/runtime/capsule-forecast.d.ts +2 -2
  27. package/src/client/runtime/capsule-forecast.js +1 -1
  28. package/src/client/runtime/capsule-operation-adapters.d.ts +1 -1
  29. package/src/client/runtime/capsule-operation-adapters.js +1 -1
  30. package/src/client/runtime/capsule-operation-deltas.js +1 -1
  31. package/src/client/runtime/capsule-operation-executor.d.ts +1 -1
  32. package/src/client/runtime/capsule-operation-executor.js +2 -2
  33. package/src/client/runtime/capsule-operation-stream.d.ts +1 -1
  34. package/src/client/runtime/capsule-operation-stream.js +2 -2
  35. package/src/client/runtime/capsule-piece-reader.js +1 -1
  36. package/src/client/runtime/capsule-program-adapter.js +1 -1
  37. package/src/client/runtime/capsule-release-authorization.js +1 -1
  38. package/src/client/runtime/capsule-rerank.d.ts +2 -2
  39. package/src/client/runtime/composition-root.d.ts +6 -6
  40. package/src/client/runtime/composition-root.js +6 -6
  41. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +1 -1
  42. package/src/client/runtime/fetch-capsule-artifact-store.js +1 -1
  43. package/src/client/runtime/model-source.d.ts +4 -2
  44. package/src/client/runtime/model-source.js +5 -5
  45. package/src/client/runtime/scoped-session.d.ts +2 -2
  46. package/src/client/runtime/verified-capsule-artifact-store.d.ts +2 -2
  47. package/src/client/runtime/verified-capsule-artifact-store.js +1 -1
  48. package/src/config/backward-registry-loader.js +1 -1
  49. package/src/config/{capsule-adapter-policy.js → capsule/capsule-adapter-policy.js} +2 -2
  50. package/src/config/{capsule-adapters.d.ts → capsule/capsule-adapters.d.ts} +1 -1
  51. package/src/config/{capsule-adapters.js → capsule/capsule-adapters.js} +2 -2
  52. package/src/config/{capsule-loading.js → capsule/capsule-loading.js} +1 -1
  53. package/src/config/{capsule-operation.d.ts → capsule/capsule-operation.d.ts} +1 -1
  54. package/src/config/{capsule-operation.js → capsule/capsule-operation.js} +3 -3
  55. package/src/config/{capsule-release-contract.js → capsule/capsule-release-contract.js} +2 -2
  56. package/src/config/{capsule-release-events.d.ts → capsule/capsule-release-events.d.ts} +2 -1
  57. package/src/config/{capsule-release-events.js → capsule/capsule-release-events.js} +3 -2
  58. package/src/config/{capsule-sequence-receipt.js → capsule/capsule-sequence-receipt.js} +1 -1
  59. package/src/config/{capsule-token-selection.d.ts → capsule/capsule-token-selection.d.ts} +1 -1
  60. package/src/config/{capsule-token-selection.js → capsule/capsule-token-selection.js} +2 -2
  61. package/src/config/{capsule-v2.d.ts → capsule/capsule-v2.d.ts} +5 -4
  62. package/src/config/{capsule-v2.js → capsule/capsule-v2.js} +6 -6
  63. package/src/config/{capsule-v3.js → capsule/capsule-v3.js} +1 -1
  64. package/src/config/{capsule.js → capsule/capsule.js} +26 -7
  65. package/src/config/choice-scoring.js +1 -1
  66. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  67. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +1 -1
  68. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  69. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +2 -2
  70. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  71. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +1 -1
  72. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  73. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  74. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  75. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  76. package/src/config/conversion/gemma3/gemma-3-270m-it-w8a8-ehf16-af32.json +1 -1
  77. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  78. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  79. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +8 -8
  80. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +8 -8
  81. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +8 -8
  82. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +8 -8
  83. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +8 -8
  84. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +8 -8
  85. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +8 -8
  86. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +8 -8
  87. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +8 -8
  88. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +1 -1
  89. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  90. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +1 -1
  91. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +1 -1
  92. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +1 -1
  93. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  94. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  95. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  96. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  97. package/src/config/conversion/qwen3/qwen-3-4b-instruct-2507-q4k-ehf16-af32.json +2 -2
  98. package/src/config/conversion/qwen3/qwen-3-5-0-8b-canonical-f32.json +1 -1
  99. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  100. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  101. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  102. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  103. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +1 -1
  104. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  105. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-q4_0-passthrough.json +1 -1
  106. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-text-q4k-ehf16-af32.json +1 -1
  107. package/src/config/conversion/qwen3/qwen-3-8-flash-next-text-q4k-ehf16-af32.json +25 -22
  108. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +2 -2
  109. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  110. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  111. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +2 -2
  112. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +2 -2
  113. package/src/config/conversion/qwen3/qwen3.8-27b-q4_0-passthrough.json +1 -1
  114. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +1 -1
  115. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +1 -1
  116. package/src/config/conversion/qwen3/qwen4exp-gguf-source-policy.json +53 -0
  117. package/src/config/embedding-contract.d.ts +1 -1
  118. package/src/config/embedding-contract.js +1 -1
  119. package/src/config/execution-registry-contract.d.ts +5 -5
  120. package/src/config/execution-registry-contract.js +5 -5
  121. package/src/config/generation-contract.js +2 -1
  122. package/src/config/hardware-partition-plan.d.ts +32 -0
  123. package/src/config/hardware-partition-plan.js +78 -0
  124. package/src/config/inspection/observation-policies.json +1 -1
  125. package/src/config/kernel-path-loader.d.ts +1 -1
  126. package/src/config/kernel-path-loader.js +1 -1
  127. package/src/config/kernel-registry-contract.d.ts +2 -2
  128. package/src/config/kernel-registry-contract.js +1 -1
  129. package/src/config/kernels/kernel-ref-digests.js +508 -0
  130. package/src/config/kernels/registry.js +2 -2
  131. package/src/config/kernels/registry.json +1892 -1096
  132. package/src/config/layer-partition-plan.d.ts +4 -0
  133. package/src/config/layer-partition-plan.js +10 -4
  134. package/src/config/merge.js +1 -1
  135. package/src/config/model-session-declarations.js +85 -0
  136. package/src/config/param-validator.js +3 -3
  137. package/src/config/platforms/amd-rdna3.json +2 -2
  138. package/src/config/production-release-evidence.js +1 -1
  139. package/src/config/revocation-policy.js +6 -2
  140. package/src/config/revocation-registry.json +1 -1
  141. package/src/config/runtime/model-sessions.json +23 -0
  142. package/src/config/runtime/profiles/flash-next-gguf-control-gemv.json +72 -0
  143. package/src/config/runtime/profiles/flash-next-gguf-hc-inject-gemv.json +47 -0
  144. package/src/config/schema/capsule/index.d.ts +14 -0
  145. package/src/config/schema/capsule/index.js +10 -0
  146. package/src/config/schema/{program-bundle.schema.d.ts → capsule/program-bundle.schema.d.ts} +3 -3
  147. package/src/config/schema/{program-bundle.schema.js → capsule/program-bundle.schema.js} +6 -6
  148. package/src/config/schema/{execution-v1.schema.d.ts → execution/execution-v1.schema.d.ts} +2 -0
  149. package/src/config/schema/{execution-v1.schema.js → execution/execution-v1.schema.js} +2 -2
  150. package/src/config/schema/execution/index.d.ts +203 -0
  151. package/src/config/schema/execution/index.js +94 -0
  152. package/src/config/schema/{inference.schema.d.ts → execution/inference.schema.d.ts} +1 -1
  153. package/src/config/schema/{kernel-path.schema.d.ts → execution/kernel-path.schema.d.ts} +3 -0
  154. package/src/config/schema/{kernel-registry.schema.d.ts → execution/kernel-registry.schema.d.ts} +2 -2
  155. package/src/config/schema/{kernel-thresholds.schema.d.ts → execution/kernel-thresholds.schema.d.ts} +1 -1
  156. package/src/config/schema/{kernel-thresholds.schema.js → execution/kernel-thresholds.schema.js} +1 -1
  157. package/src/config/schema/{kvcache.schema.d.ts → execution/kvcache.schema.d.ts} +8 -2
  158. package/src/config/schema/{kvcache.schema.js → execution/kvcache.schema.js} +17 -1
  159. package/src/config/schema/{ngram-embedding.schema.d.ts → execution/ngram-embedding.schema.d.ts} +3 -1
  160. package/src/config/schema/{ngram-embedding.schema.js → execution/ngram-embedding.schema.js} +16 -1
  161. package/src/config/schema/index.d.ts +8 -897
  162. package/src/config/schema/index.js +8 -506
  163. package/src/config/schema/{converter.schema.d.ts → manifest/converter.schema.d.ts} +4 -4
  164. package/src/config/schema/{converter.schema.js → manifest/converter.schema.js} +1 -1
  165. package/src/config/schema/manifest/index.d.ts +142 -0
  166. package/src/config/schema/manifest/index.js +62 -0
  167. package/src/config/schema/manifest/manifest-inference.schema.d.ts +298 -0
  168. package/src/config/schema/{manifest.schema.d.ts → manifest/manifest.schema.d.ts} +9 -291
  169. package/src/config/schema/{manifest.schema.js → manifest/manifest.schema.js} +2 -2
  170. package/src/config/schema/{emulation.schema.d.ts → platform/emulation.schema.d.ts} +1 -1
  171. package/src/config/schema/{emulation.schema.js → platform/emulation.schema.js} +6 -6
  172. package/src/config/schema/platform/index.d.ts +72 -0
  173. package/src/config/schema/platform/index.js +27 -0
  174. package/src/config/schema/{memory-limits.schema.js → platform/memory-limits.schema.js} +1 -1
  175. package/src/config/schema/policy/index.d.ts +14 -0
  176. package/src/config/schema/policy/index.js +9 -0
  177. package/src/config/schema/{doppler.schema.d.ts → runtime/doppler.schema.d.ts} +3 -3
  178. package/src/config/schema/{doppler.schema.js → runtime/doppler.schema.js} +12 -12
  179. package/src/config/schema/runtime/index.d.ts +238 -0
  180. package/src/config/schema/runtime/index.js +135 -0
  181. package/src/config/schema/{loading.schema.d.ts → runtime/loading.schema.d.ts} +26 -1
  182. package/src/config/schema/{loading.schema.js → runtime/loading.schema.js} +88 -0
  183. package/src/config/schema/{shared-runtime.schema.d.ts → runtime/shared-runtime.schema.d.ts} +5 -5
  184. package/src/config/schema/{shared-runtime.schema.js → runtime/shared-runtime.schema.js} +4 -4
  185. package/src/config/schema/{tooling.schema.d.ts → runtime/tooling.schema.d.ts} +1 -1
  186. package/src/config/schema/training/index.d.ts +63 -0
  187. package/src/config/schema/training/index.js +43 -0
  188. package/src/config/schema/{lora.schema.js → training/lora.schema.js} +1 -1
  189. package/src/config/source-packages/litert/gemma-4-e2b-it.json +1 -1
  190. package/src/config/supported-operations.json +358 -88
  191. package/src/config/target-plan.d.ts +2 -2
  192. package/src/config/target-plan.js +2 -2
  193. package/src/config/training-defaults.d.ts +2 -2
  194. package/src/config/training-defaults.js +2 -2
  195. package/src/config/transforms/execution-graph/correctness-fallback.js +86 -0
  196. package/src/config/transforms/execution-graph/prefill-attention.js +116 -0
  197. package/src/{converter → config/transforms}/execution-graph-transforms.js +5 -194
  198. package/src/converter/core.js +18 -164
  199. package/src/converter/forge-stages.js +2 -2
  200. package/src/converter/fp8-source.js +1 -1
  201. package/src/converter/gemma4/validation.js +110 -0
  202. package/src/converter/linear-norm-gate.js +1 -1
  203. package/src/converter/manifest-builder.js +2 -9
  204. package/src/converter/parsers/gguf.js +54 -4
  205. package/src/converter/quantization-info.js +2 -2
  206. package/src/converter/qwen4-exp-source-execution.js +94 -0
  207. package/src/converter/qwen4-exp-source.js +1 -201
  208. package/src/converter/tensor-transform.js +2 -9
  209. package/src/converter/tokenizer-contract.js +2 -9
  210. package/src/converter/w8a8-quantizer.js +2 -2
  211. package/src/debug/config.js +1 -1
  212. package/src/debug/perf-guards.js +1 -1
  213. package/src/errors/doppler-error.d.ts +54 -0
  214. package/src/errors/doppler-error.js +13 -5
  215. package/src/experimental/distribution/shard-delivery/receipt.js +1 -1
  216. package/src/experimental/distribution/shard-delivery/retry.js +4 -4
  217. package/src/experimental/energy/types.d.ts +1 -1
  218. package/src/experimental/logits/cpu-output.js +1 -1
  219. package/src/experimental/training/artifacts.d.ts +2 -2
  220. package/src/experimental/training/artifacts.js +2 -2
  221. package/src/experimental/training/autograd.d.ts +1 -1
  222. package/src/experimental/training/distillation/student-fixture/execution.js +1 -1
  223. package/src/experimental/training/distillation/student-fixture/model.js +1 -1
  224. package/src/experimental/training/distillation/student-fixture.js +1 -1
  225. package/src/experimental/training/loss-scaling.d.ts +1 -1
  226. package/src/experimental/training/runner/evaluation.js +1 -1
  227. package/src/experimental/training/runner.js +1 -1
  228. package/src/experimental/training/trainer.d.ts +1 -1
  229. package/src/experimental/training/workloads.js +2 -2
  230. package/src/formats/gguf/block-storage.d.ts +9 -0
  231. package/src/formats/gguf/block-storage.js +12 -3
  232. package/src/formats/gguf/qwen-source.js +5 -3
  233. package/src/formats/gguf/qwen-source.json +8 -1
  234. package/src/formats/gguf/qwen4-exp-source.js +101 -0
  235. package/src/formats/q4k-dequant.js +1 -1
  236. package/src/formats/qwen4-exp-source-contracts.js +204 -0
  237. package/src/formats/rdrr/fp8-contract.js +1 -1
  238. package/src/formats/rdrr/fp8-storage.js +1 -1
  239. package/src/formats/rdrr/q4_0-storage.js +1 -1
  240. package/src/formats/rdrr/storage-descriptor.js +1 -1
  241. package/src/formats/rdrr/tensor-config-validator.js +3 -3
  242. package/src/formats/rdrr/validation.js +1 -1
  243. package/src/formats/rdrr/w8a8-storage.js +1 -1
  244. package/src/generation/index.d.ts +1 -1
  245. package/src/generation/index.js +1 -0
  246. package/src/gpu/device.d.ts +1 -1
  247. package/src/gpu/kernels/attention/contiguous-quantized.js +252 -0
  248. package/src/gpu/kernels/attention/tiered-quantized.js +321 -0
  249. package/src/gpu/kernels/attention/tiered.js +240 -0
  250. package/src/gpu/kernels/attention.js +5 -801
  251. package/src/gpu/kernels/backward/layernorm_backward.wgsl +3 -3
  252. package/src/gpu/kernels/channel-gain.js +100 -0
  253. package/src/gpu/kernels/channel-rotation.d.ts +16 -0
  254. package/src/gpu/kernels/channel-rotation.js +47 -0
  255. package/src/gpu/kernels/channel_energy.wgsl +17 -0
  256. package/src/gpu/kernels/channel_energy_pair.wgsl +27 -0
  257. package/src/gpu/kernels/channel_gain_control.wgsl +60 -0
  258. package/src/gpu/kernels/channel_gain_derive.wgsl +40 -0
  259. package/src/gpu/kernels/channel_gain_swiglu.wgsl +64 -0
  260. package/src/gpu/kernels/channel_rotation_apply.wgsl +38 -0
  261. package/src/gpu/kernels/channel_rotation_derive.wgsl +76 -0
  262. package/src/gpu/kernels/check-finiteness-f32.js +22 -0
  263. package/src/gpu/kernels/check_finiteness.wgsl +2 -2
  264. package/src/gpu/kernels/clamp.js +2 -2
  265. package/src/gpu/kernels/dequant.js +1 -1
  266. package/src/gpu/kernels/dispatch.js +7 -0
  267. package/src/gpu/kernels/energy_quintel_grad.wgsl +7 -7
  268. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +7 -7
  269. package/src/gpu/kernels/energy_quintel_reduce.wgsl +6 -6
  270. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +6 -6
  271. package/src/gpu/kernels/energy_quintel_update.wgsl +7 -7
  272. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +7 -7
  273. package/src/gpu/kernels/fused_matmul_q4_widetile_f32.wgsl +174 -0
  274. package/src/gpu/kernels/fused_matmul_rmsnorm.wgsl +0 -49
  275. package/src/gpu/kernels/gated-delta-norm-gate.js +1 -1
  276. package/src/gpu/kernels/gated_delta_recurrent_gguf_register_prefill_sigmoid_gate.wgsl +730 -0
  277. package/src/gpu/kernels/gated_delta_recurrent_gguf_sigmoid_gate.wgsl +499 -0
  278. package/src/gpu/kernels/gather.js +16 -1
  279. package/src/gpu/kernels/gather_split4_packed_f16_vec4_f32_out.wgsl +104 -0
  280. package/src/gpu/kernels/generated/uniform-writers-0.js +7 -11
  281. package/src/gpu/kernels/generated/uniform-writers-1.js +43 -46
  282. package/src/gpu/kernels/generated/uniform-writers-2.js +80 -2
  283. package/src/gpu/kernels/generated/uniform-writers.js +33 -21
  284. package/src/gpu/kernels/hyper-connection-up.js +64 -0
  285. package/src/gpu/kernels/hyper-connection.js +15 -4
  286. package/src/gpu/kernels/hyper_connection_mix_branch.wgsl +105 -0
  287. package/src/gpu/kernels/hyper_connection_norm.wgsl +3 -3
  288. package/src/gpu/kernels/hyper_connection_up_mix_branch_q4_0.wgsl +164 -0
  289. package/src/gpu/kernels/kernel-execution.js +18 -7
  290. package/src/gpu/kernels/linear-attention-core.js +3 -4
  291. package/src/gpu/kernels/moe-route.js +14 -0
  292. package/src/gpu/kernels/moe_gather.wgsl +0 -63
  293. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +63 -0
  294. package/src/gpu/kernels/moe_route_q4_1_block_f32.wgsl +59 -0
  295. package/src/gpu/kernels/moe_route_remap.wgsl +22 -0
  296. package/src/gpu/kernels/paired-linear-attention.js +1 -1
  297. package/src/gpu/kernels/pipeline-cache.js +18 -2
  298. package/src/gpu/kernels/ple.js +11 -7
  299. package/src/gpu/kernels/ple_q4_0_to_f32.wgsl +32 -0
  300. package/src/gpu/kernels/qsa-indexer.d.ts +2 -0
  301. package/src/gpu/kernels/qsa-indexer.js +6 -4
  302. package/src/gpu/kernels/qsa_block_keys.wgsl +2 -1
  303. package/src/gpu/kernels/qsa_prepare.wgsl +2 -1
  304. package/src/gpu/kernels/relu.js +1 -1
  305. package/src/gpu/kernels/relu.wgsl +2 -2
  306. package/src/gpu/kernels/relu_f16.wgsl +2 -2
  307. package/src/gpu/kernels/residual.js +1 -1
  308. package/src/gpu/kernels/residual.wgsl +4 -4
  309. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  310. package/src/gpu/kernels/residual_f16_vec4.wgsl +2 -2
  311. package/src/gpu/kernels/residual_vec4.wgsl +2 -2
  312. package/src/gpu/kernels/rope-precompute.js +1 -1
  313. package/src/gpu/kernels/scatter_add.wgsl +0 -31
  314. package/src/gpu/kernels/shader-cache.js +17 -3
  315. package/src/gpu/kernels/shader-capability-filter.js +120 -0
  316. package/src/gpu/kernels/softmax.js +10 -10
  317. package/src/gpu/kernels/topk.wgsl +0 -55
  318. package/src/gpu/kernels/transpose.js +1 -1
  319. package/src/gpu/kernels/transpose.wgsl +2 -2
  320. package/src/gpu/kernels/w8a8.js +1 -1
  321. package/src/gpu/uniform-cache.d.ts +15 -1
  322. package/src/gpu/uniform-cache.js +82 -7
  323. package/src/gpu/weight-buffer.d.ts +1 -1
  324. package/src/inference/moe-router.d.ts +1 -126
  325. package/src/inference/moe-router.js +0 -149
  326. package/src/inference/pipelines/context.js +3 -15
  327. package/src/inference/pipelines/shader-scoped-pipeline.js +10 -5
  328. package/src/inference/pipelines/text/attention/interpreter/immediate.js +4 -4
  329. package/src/inference/pipelines/text/attention/interpreter/output.js +203 -0
  330. package/src/inference/pipelines/text/attention/interpreter/recorded.js +92 -67
  331. package/src/inference/pipelines/text/attention/interpreter.js +77 -350
  332. package/src/inference/pipelines/text/attention/projection-executor.js +2 -2
  333. package/src/inference/pipelines/text/attention/projection-plan.js +1 -1
  334. package/src/inference/pipelines/text/attention/projection-precision.js +132 -0
  335. package/src/inference/pipelines/text/attention/projections.js +13 -151
  336. package/src/inference/pipelines/text/attention/recorded-health.js +83 -0
  337. package/src/inference/pipelines/text/attention/rope-observation.js +1 -1
  338. package/src/inference/pipelines/text/attention/types.js +0 -13
  339. package/src/inference/pipelines/text/config/normalization.js +2 -2
  340. package/src/inference/pipelines/text/config.d.ts +4 -4
  341. package/src/inference/pipelines/text/config.js +17 -135
  342. package/src/inference/pipelines/text/{config-components.js → configuration/config-components.js} +4 -4
  343. package/src/inference/pipelines/text/configuration/model-geometry.js +114 -0
  344. package/src/inference/pipelines/text/{resolved-runtime-session.d.ts → configuration/resolved-runtime-session.d.ts} +4 -4
  345. package/src/inference/pipelines/text/{resolved-runtime-session.js → configuration/resolved-runtime-session.js} +1 -1
  346. package/src/inference/pipelines/text/{abort-contract.js → contracts/abort-contract.js} +2 -1
  347. package/src/inference/pipelines/text/contracts/metrics.d.ts +234 -0
  348. package/src/inference/pipelines/text/{embed.js → embedding/embed.js} +15 -15
  349. package/src/inference/pipelines/text/{embedding-batch.js → embedding/embedding-batch.js} +1 -1
  350. package/src/inference/pipelines/text/{embedding-extraction.js → embedding/embedding-extraction.js} +12 -12
  351. package/src/inference/pipelines/text/{embedding-normalization.js → embedding/embedding-normalization.js} +4 -4
  352. package/src/inference/pipelines/text/{execution-plan.d.ts → execution/execution-plan.d.ts} +8 -8
  353. package/src/inference/pipelines/text/{execution-plan.js → execution/execution-plan.js} +9 -9
  354. package/src/inference/pipelines/text/{execution-runtime-builders.js → execution/execution-runtime-builders.js} +3 -3
  355. package/src/inference/pipelines/text/{execution-v1.d.ts → execution/execution-v1.d.ts} +2 -2
  356. package/src/inference/pipelines/text/{execution-v1.js → execution/execution-v1.js} +13 -183
  357. package/src/inference/pipelines/text/execution/layer-contract.js +90 -0
  358. package/src/inference/pipelines/text/{layer-execution-contract.js → execution/layer-execution-contract.js} +2 -2
  359. package/src/inference/pipelines/text/{layer-input-execution.js → execution/layer-input-execution.js} +27 -36
  360. package/src/inference/pipelines/text/{layer-partition-contract.d.ts → execution/layer-partition-contract.d.ts} +3 -3
  361. package/src/inference/pipelines/text/{layer-partition-contract.js → execution/layer-partition-contract.js} +2 -2
  362. package/src/inference/pipelines/text/{layer-plan-gpu.js → execution/layer-plan-gpu.js} +12 -12
  363. package/src/inference/pipelines/text/{layer-plan.d.ts → execution/layer-plan.d.ts} +1 -1
  364. package/src/inference/pipelines/text/{layer-plan.js → execution/layer-plan.js} +1 -1
  365. package/src/inference/pipelines/text/{layer.js → execution/layer.js} +34 -142
  366. package/src/inference/pipelines/text/{operator-identity.js → execution/operator-identity.js} +1 -1
  367. package/src/inference/pipelines/text/{ops.js → execution/ops.js} +26 -26
  368. package/src/inference/pipelines/text/{paired-execution.js → execution/paired-execution.js} +9 -9
  369. package/src/inference/pipelines/text/execution-v1/hybrid-projection-contract.js +159 -0
  370. package/src/inference/pipelines/text/expert-loading-stats.d.ts +16 -0
  371. package/src/inference/pipelines/text/expert-loading-stats.js +15 -0
  372. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -4
  373. package/src/inference/pipelines/text/ffn/dense-plan.js +23 -34
  374. package/src/inference/pipelines/text/ffn/dense.js +72 -392
  375. package/src/inference/pipelines/text/ffn/fused-gate-up.js +197 -0
  376. package/src/inference/pipelines/text/ffn/moe.js +1 -1
  377. package/src/inference/pipelines/text/ffn/packed-gate-up.js +218 -0
  378. package/src/inference/pipelines/text/ffn/paired.js +3 -3
  379. package/src/inference/pipelines/text/ffn/qwen-shared-expert.js +1 -1
  380. package/src/inference/pipelines/text/ffn/sandwich.js +6 -7
  381. package/src/inference/pipelines/text/ffn/standard.js +5 -5
  382. package/src/inference/pipelines/text/ffn/types.js +6 -3
  383. package/src/inference/pipelines/text/{activity-weights.js → generator/activation/activity-weights.js} +8 -8
  384. package/src/inference/pipelines/text/generator/activation/channel-gain.d.ts +61 -0
  385. package/src/inference/pipelines/text/generator/activation/channel-gain.js +352 -0
  386. package/src/inference/pipelines/text/generator/activation/final-layer-replay.js +43 -0
  387. package/src/inference/pipelines/text/generator/{capsule-generation.js → adapters/capsule-generation.js} +5 -5
  388. package/src/inference/pipelines/text/generator/{diffusion.js → adapters/diffusion.js} +19 -19
  389. package/src/inference/pipelines/text/generator/{training.js → adapters/training.js} +21 -21
  390. package/src/inference/pipelines/text/generator/decode/advance-token.js +73 -0
  391. package/src/inference/pipelines/text/{generator-steps.js → generator/decode/batch-generation.js} +25 -288
  392. package/src/inference/pipelines/text/generator/decode/batch-readback.js +175 -0
  393. package/src/inference/pipelines/text/generator/{compiled-greedy-window.js → decode/compiled-greedy-window.js} +5 -5
  394. package/src/inference/pipelines/text/generator/{decode-components.js → decode/decode-components.js} +1 -1
  395. package/src/inference/pipelines/text/generator/{decode-runtime.js → decode/decode-loop.js} +20 -412
  396. package/src/inference/pipelines/text/generator/decode/decode-mechanics.js +288 -0
  397. package/src/inference/pipelines/text/generator/{decode-readback.js → decode/decode-readback.js} +1 -1
  398. package/src/inference/pipelines/text/generator/decode/decode-runtime.js +2 -0
  399. package/src/inference/pipelines/text/generator/{decode.js → decode/decode-step.js} +34 -313
  400. package/src/inference/pipelines/text/generator/decode/decode.js +5 -0
  401. package/src/inference/pipelines/text/generator/{greedy-head-policy.js → decode/greedy-head-policy.js} +1 -1
  402. package/src/inference/pipelines/text/generator/{recorded-step-executor.js → decode/recorded-step-executor.js} +2 -2
  403. package/src/inference/pipelines/text/generator/generator-steps.js +4 -0
  404. package/src/inference/pipelines/text/{generator.d.ts → generator/generator.d.ts} +13 -11
  405. package/src/inference/pipelines/text/{generator.js → generator/generator.js} +150 -46
  406. package/src/inference/pipelines/text/generator/{prefill-runtime.js → prefill/prefill-execution.js} +72 -278
  407. package/src/inference/pipelines/text/generator/prefill/prefill-logits.js +144 -0
  408. package/src/inference/pipelines/text/generator/{prefill-observation.js → prefill/prefill-observation.js} +2 -2
  409. package/src/inference/pipelines/text/generator/{prefill-policy.js → prefill/prefill-policy.js} +6 -1
  410. package/src/inference/pipelines/text/generator/prefill/prefill-runtime.js +2 -0
  411. package/src/inference/pipelines/text/generator/prefill/prefill-sequence.js +100 -0
  412. package/src/inference/pipelines/text/generator/{prefix-embedding.js → prefill/prefix-embedding.js} +7 -7
  413. package/src/inference/pipelines/text/{generation-request.d.ts → generator/request/generation-request.d.ts} +2 -2
  414. package/src/inference/pipelines/text/{generation-request.js → generator/request/generation-request.js} +3 -3
  415. package/src/inference/pipelines/text/generator/{prompt-input.js → request/prompt-input.js} +2 -2
  416. package/src/inference/pipelines/text/{sampling-config.d.ts → generator/sampling/sampling-config.d.ts} +1 -1
  417. package/src/inference/pipelines/text/{sampling-config.js → generator/sampling/sampling-config.js} +2 -2
  418. package/src/inference/pipelines/text/{sampling.d.ts → generator/sampling/sampling.d.ts} +15 -0
  419. package/src/inference/pipelines/text/{sampling.js → generator/sampling/sampling.js} +22 -3
  420. package/src/inference/pipelines/text/{stopping.js → generator/sampling/stopping.js} +3 -3
  421. package/src/inference/pipelines/text/generator/{token-selection.d.ts → sampling/token-selection.d.ts} +1 -1
  422. package/src/inference/pipelines/text/generator/{token-selection.js → sampling/token-selection.js} +8 -8
  423. package/src/inference/pipelines/text/generator/{activation-storage.js → session/activation-storage.js} +2 -2
  424. package/src/inference/pipelines/text/generator/{attention-lifecycle.js → session/attention-lifecycle.js} +1 -1
  425. package/src/inference/pipelines/text/generator/{diagnostic-sequence.d.ts → session/diagnostic-sequence.d.ts} +3 -3
  426. package/src/inference/pipelines/text/generator/{diagnostic-sequence.js → session/diagnostic-sequence.js} +3 -3
  427. package/src/inference/pipelines/text/generator/session/generation-runtime.js +278 -0
  428. package/src/inference/pipelines/text/{generator-runtime.d.ts → generator/session/generator-runtime.d.ts} +7 -7
  429. package/src/inference/pipelines/text/{generator-runtime.js → generator/session/generator-runtime.js} +8 -8
  430. package/src/inference/pipelines/text/generator/{recovery.js → session/recovery.js} +6 -6
  431. package/src/inference/pipelines/text/{sequence-state.d.ts → generator/session/sequence-state.d.ts} +5 -5
  432. package/src/inference/pipelines/text/{sequence-state.js → generator/session/sequence-state.js} +26 -18
  433. package/src/inference/pipelines/text/generator/{session-context.js → session/session-context.js} +12 -11
  434. package/src/inference/pipelines/text/generator/{speculative-runtime.js → session/speculative-runtime.js} +1 -1
  435. package/src/inference/pipelines/text/generator/{text.js → session/text.js} +9 -9
  436. package/src/inference/pipelines/text/{warm-generation.d.ts → generator/session/warm-generation.d.ts} +1 -1
  437. package/src/inference/pipelines/text/{warm-generation.js → generator/session/warm-generation.js} +1 -1
  438. package/src/inference/pipelines/text/generator/{whole-token-lane.js → whole-token/whole-token-lane.js} +10 -10
  439. package/src/inference/pipelines/text/generator/{whole-token-observation.d.ts → whole-token/whole-token-observation.d.ts} +2 -2
  440. package/src/inference/pipelines/text/generator/{whole-token-observation.js → whole-token/whole-token-observation.js} +5 -5
  441. package/src/inference/pipelines/text/generator/{whole-token-program.d.ts → whole-token/whole-token-program.d.ts} +3 -3
  442. package/src/inference/pipelines/text/generator/{whole-token-program.js → whole-token/whole-token-program.js} +5 -5
  443. package/src/inference/pipelines/text/hyper-connection/decoder-layer.d.ts +17 -4
  444. package/src/inference/pipelines/text/hyper-connection/decoder-layer.js +38 -14
  445. package/src/inference/pipelines/text/hyper-connection/layer-executor.js +14 -9
  446. package/src/inference/pipelines/text/hyper-connection/logits-input.js +5 -3
  447. package/src/inference/pipelines/text/hyper-connection/projection.js +52 -2
  448. package/src/inference/pipelines/text/hyper-connection/sequence-transaction.js +3 -3
  449. package/src/inference/pipelines/text/hyper-connection/session.d.ts +1 -1
  450. package/src/inference/pipelines/text/hyper-connection/session.js +9 -5
  451. package/src/inference/pipelines/text/indexed-attention/qsa-cache.d.ts +2 -1
  452. package/src/inference/pipelines/text/indexed-attention/qsa-cache.js +3 -3
  453. package/src/inference/pipelines/text/init.d.ts +4 -2
  454. package/src/inference/pipelines/text/init.js +10 -934
  455. package/src/inference/pipelines/text/initialization/init-emulation.js +76 -0
  456. package/src/inference/pipelines/text/initialization/init-kv-cache.js +547 -0
  457. package/src/inference/pipelines/text/initialization/init-moe.js +34 -0
  458. package/src/inference/pipelines/text/{init-qkv-fusion.js → initialization/init-qkv-fusion.js} +7 -7
  459. package/src/inference/pipelines/text/{init-rope.d.ts → initialization/init-rope.d.ts} +4 -4
  460. package/src/inference/pipelines/text/{init-rope.js → initialization/init-rope.js} +9 -11
  461. package/src/inference/pipelines/text/initialization/init-speculative.js +31 -0
  462. package/src/inference/pipelines/text/{init-tokenizer.js → initialization/init-tokenizer.js} +2 -2
  463. package/src/inference/pipelines/text/lifecycle.js +18 -18
  464. package/src/inference/pipelines/text/{linear-attention-ab-fusion.js → linear-attention/linear-attention-ab-fusion.js} +3 -3
  465. package/src/inference/pipelines/text/{linear-attention.d.ts → linear-attention/linear-attention.d.ts} +12 -12
  466. package/src/inference/pipelines/text/{linear-attention.js → linear-attention/linear-attention.js} +17 -17
  467. package/src/inference/pipelines/text/linear-attention/plan.js +4 -4
  468. package/src/inference/pipelines/text/linear-attention/state-checkpoint.js +1 -1
  469. package/src/inference/pipelines/text/linear-attention/state-clone.js +1 -1
  470. package/src/inference/pipelines/text/linear-attention/state-layout.js +2 -2
  471. package/src/inference/pipelines/text/loading/decode-session-config.js +295 -0
  472. package/src/inference/pipelines/text/loading/init-weights.js +254 -0
  473. package/src/inference/pipelines/text/{load-errors.js → loading/load-errors.js} +3 -3
  474. package/src/inference/pipelines/text/{model-load.js → loading/model-load.js} +14 -306
  475. package/src/inference/pipelines/text/{weight-loading-config.js → loading/weight-loading-config.js} +6 -6
  476. package/src/inference/pipelines/text/{weights.d.ts → loading/weights.d.ts} +13 -13
  477. package/src/inference/pipelines/text/{weights.js → loading/weights.js} +5 -5
  478. package/src/inference/pipelines/text/logits/gpu-executor.js +2 -2
  479. package/src/inference/pipelines/text/logits/gpu.d.ts +4 -4
  480. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  481. package/src/inference/pipelines/text/{generator → logits}/hidden-projection.js +1 -1
  482. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  483. package/src/inference/pipelines/text/logits/index.js +3 -3
  484. package/src/inference/pipelines/text/{generator → logits}/logits-config.js +1 -1
  485. package/src/inference/pipelines/text/logits/output-transform.js +1 -1
  486. package/src/inference/pipelines/text/logits/precision-policy.js +1 -1
  487. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  488. package/src/inference/pipelines/text/lora-types.d.ts +1 -1
  489. package/src/inference/pipelines/text/moe/expert-loading.js +4 -1
  490. package/src/inference/pipelines/text/moe/gpu-executor.js +44 -101
  491. package/src/inference/pipelines/text/moe/grouped-prefill.js +6 -2
  492. package/src/inference/pipelines/text/moe/qwen-route-executor.d.ts +2 -2
  493. package/src/inference/pipelines/text/moe/qwen-route-executor.js +31 -2
  494. package/src/inference/pipelines/text/moe/routed-expert-chunks.js +99 -15
  495. package/src/inference/pipelines/text/moe-cache.js +11 -77
  496. package/src/inference/pipelines/text/moe-gpu.js +33 -75
  497. package/src/inference/pipelines/text/moe-impl.d.ts +3 -38
  498. package/src/inference/pipelines/text/moe-impl.js +1 -7
  499. package/src/inference/pipelines/text/ngram/ngram-addresses.d.ts +1 -1
  500. package/src/inference/pipelines/text/ngram/ngram-addresses.js +2 -2
  501. package/src/inference/pipelines/text/ngram/ngram-table.d.ts +3 -3
  502. package/src/inference/pipelines/text/ngram/ngram-table.js +5 -6
  503. package/src/inference/pipelines/text/ngram/ple-layer.d.ts +3 -3
  504. package/src/inference/pipelines/text/ngram/ple-layer.js +3 -3
  505. package/src/inference/pipelines/text/{generator-logits-observation.js → observation/generator-logits-observation.js} +4 -4
  506. package/src/inference/pipelines/text/{kernel-trace.js → observation/kernel-trace.js} +1 -1
  507. package/src/inference/pipelines/text/{operator-events.js → observation/operator-events.js} +1 -1
  508. package/src/inference/pipelines/text/{probes.js → observation/probes.js} +13 -13
  509. package/src/inference/pipelines/text/{tsir-fixture-writer.js → observation/tsir-fixture-writer.js} +3 -3
  510. package/src/inference/pipelines/text/{execution.js → operations/execution.js} +14 -14
  511. package/src/inference/pipelines/text/{image-transcription.js → operations/image-transcription.js} +8 -8
  512. package/src/inference/pipelines/text/{partition-attempt.js → partitions/partition-attempt.js} +7 -7
  513. package/src/inference/pipelines/text/{partition-execution.js → partitions/partition-execution.js} +79 -34
  514. package/src/inference/pipelines/text/partitions/partition-wire.js +50 -0
  515. package/src/inference/pipelines/text/{resident-partition-contract.d.ts → partitions/resident-partition-contract.d.ts} +10 -8
  516. package/src/inference/pipelines/text/{resident-partition-contract.js → partitions/resident-partition-contract.js} +6 -6
  517. package/src/inference/pipelines/text/{resident-partition.js → partitions/resident-partition.js} +19 -13
  518. package/src/inference/pipelines/text/{resident-pipeline-preparation.js → partitions/resident-pipeline-preparation.js} +32 -32
  519. package/src/inference/pipelines/text/per-layer/materialize.js +4 -4
  520. package/src/inference/pipelines/text/per-layer/per-layer-inputs.js +5 -0
  521. package/src/inference/pipelines/text/per-layer/plan.js +4 -4
  522. package/src/inference/pipelines/text/{per-layer-inputs.js → per-layer/prepare.js} +14 -280
  523. package/src/inference/pipelines/text/per-layer/projection-contract.js +79 -0
  524. package/src/inference/pipelines/text/per-layer/runtime-cache.js +183 -0
  525. package/src/inference/pipelines/text/pipeline.d.ts +13 -8
  526. package/src/inference/pipelines/text/pipeline.js +78 -22
  527. package/src/inference/pipelines/text/{drift-policy.js → policy/drift-policy.js} +1 -1
  528. package/src/inference/pipelines/text/{dtype-contract.js → policy/dtype-contract.js} +2 -2
  529. package/src/inference/pipelines/text/{finiteness-guard-status.js → policy/finiteness-guard-status.js} +2 -1
  530. package/src/inference/pipelines/text/{generator-decode-policy.js → policy/generator-decode-policy.js} +5 -5
  531. package/src/inference/pipelines/text/state.d.ts +9 -8
  532. package/src/inference/pipelines/text/state.js +1 -0
  533. package/src/inference/pipelines/text/types.d.ts +14 -218
  534. package/src/inference/pipelines/text.js +2 -0
  535. package/src/inference/tokenizers/bundled/merge.js +130 -0
  536. package/src/inference/tokenizers/bundled.js +2 -36
  537. package/src/loader/doppler-loader.d.ts +17 -7
  538. package/src/loader/doppler-loader.js +27 -29
  539. package/src/loader/embedding-limit-preflight.js +29 -1
  540. package/src/loader/embedding-loader.js +18 -4
  541. package/src/loader/experts/compact-expert-cache.js +488 -0
  542. package/src/loader/experts/expert-cache.d.ts +1 -1
  543. package/src/loader/experts/expert-dequant-cache.d.ts +9 -0
  544. package/src/loader/experts/expert-dequant-cache.js +75 -0
  545. package/src/loader/experts/expert-frequency-index.js +129 -0
  546. package/src/loader/experts/expert-loader.d.ts +4 -0
  547. package/src/loader/experts/expert-loader.js +13 -1
  548. package/src/loader/experts/expert-preparation.d.ts +1 -1
  549. package/src/loader/experts/expert-preparation.js +1 -1
  550. package/src/loader/experts/loader-bindings.js +117 -13
  551. package/src/loader/experts/mapped-expert-upload-cache.d.ts +61 -0
  552. package/src/loader/experts/mapped-expert-upload-cache.js +382 -0
  553. package/src/loader/experts/qwen-paging.d.ts +1 -1
  554. package/src/loader/experts/qwen-paging.js +1 -1
  555. package/src/loader/experts/routed-expert-cache.d.ts +62 -3
  556. package/src/loader/experts/routed-expert-cache.js +451 -99
  557. package/src/loader/experts/routed-expert-plan.d.ts +2 -0
  558. package/src/loader/experts/routed-expert-plan.js +14 -8
  559. package/src/loader/experts/routed-expert-prefill.js +82 -0
  560. package/src/loader/experts/routed-expert-source.js +233 -0
  561. package/src/loader/experts/routed-expert-trace.js +77 -0
  562. package/src/loader/experts/tiered-expert-cache.d.ts +12 -0
  563. package/src/loader/experts/tiered-expert-cache.js +287 -0
  564. package/src/loader/experts/tiered-expert-plan.js +74 -0
  565. package/src/loader/final-weights-loader.d.ts +1 -1
  566. package/src/loader/hyper-connection-loader.d.ts +2 -2
  567. package/src/loader/hyper-connection-loader.js +85 -13
  568. package/src/loader/layer-loader.d.ts +3 -3
  569. package/src/loader/layer-loader.js +1 -1
  570. package/src/loader/layer-tensor-aliases.js +14 -11
  571. package/src/loader/loader-types.d.ts +6 -1
  572. package/src/loader/manifest-config.js +4 -0
  573. package/src/loader/model-executor.js +18 -1
  574. package/src/loader/model-load.d.ts +1 -1
  575. package/src/loader/model-load.js +1 -1
  576. package/src/loader/ngram-table-bindings.d.ts +3 -3
  577. package/src/loader/ngram-table-bindings.js +7 -6
  578. package/src/loader/q4k-reference-weight.js +1 -1
  579. package/src/loader/shard-cache.d.ts +8 -1
  580. package/src/loader/shard-cache.js +33 -0
  581. package/src/loader/tensors/bf16-loader.js +2 -1
  582. package/src/loader/tensors/fp8-loader.js +1 -1
  583. package/src/loader/tensors/load-executor.js +10 -2
  584. package/src/loader/tensors/tensor-loader.js +1 -1
  585. package/src/loader/tensors/tensor-reader.js +18 -5
  586. package/src/loader/tensors/w8a8-loader.js +1 -1
  587. package/src/loader/weight-representation-plan.d.ts +1 -1
  588. package/src/memory/buffer-pool/readback.js +82 -0
  589. package/src/memory/buffer-pool.d.ts +11 -2
  590. package/src/memory/buffer-pool.js +12 -73
  591. package/src/partitions.d.ts +4 -3
  592. package/src/partitions.js +3 -2
  593. package/src/{converter → rules}/capability-transform-resolver.js +3 -3
  594. package/src/rules/generated/rule-bundle.json +1 -1
  595. package/src/rules/inference/capability-transforms.rules.json +95 -28
  596. package/src/rules/rule-registry.js +2 -0
  597. package/src/storage/artifact-range-loaders.js +42 -0
  598. package/src/storage/artifact-storage-context.d.ts +4 -0
  599. package/src/storage/artifact-storage-context.js +8 -18
  600. package/src/storage/backends/idb-store.js +1 -1
  601. package/src/storage/distribution-transport.js +1 -1
  602. package/src/storage/download/http-contract.js +1 -1
  603. package/src/storage/download/http-transport.js +4 -4
  604. package/src/storage/inventory.js +1 -1
  605. package/src/storage/model-cache.d.ts +2 -1
  606. package/src/storage/model-cache.js +12 -8
  607. package/src/storage/node-file-access.js +4 -0
  608. package/src/storage/quota.d.ts +2 -1
  609. package/src/storage/quota.js +2 -1
  610. package/src/storage/shard-manager.d.ts +1 -1
  611. package/src/storage/shards/integrity.js +17 -0
  612. package/src/storage/source-artifact-store.d.ts +1 -1
  613. package/src/storage/source-artifact-store.js +1 -0
  614. package/src/storage/source-storage-context.d.ts +4 -0
  615. package/src/storage/source-storage-context.js +14 -0
  616. package/src/storage/verified-piece-gather-lane.js +192 -0
  617. package/src/storage/verified-piece-gather-worker.js +520 -0
  618. package/src/storage/verified-piece-gather.d.ts +113 -0
  619. package/src/storage/verified-piece-gather.js +199 -0
  620. package/src/storage/verified-piece-reader.d.ts +7 -0
  621. package/src/storage/verified-piece-reader.js +69 -12
  622. package/src/storage/verified-piece-storage.d.ts +3 -1
  623. package/src/storage/verified-piece-storage.js +101 -11
  624. package/src/storage/verified-piece-timings.js +25 -0
  625. package/src/tooling/browser-command-runner.js +1 -1
  626. package/src/tooling/browser-harness/benchmark-execution.js +2 -2
  627. package/src/tooling/browser-harness/benchmark-rerank.js +2 -2
  628. package/src/tooling/browser-harness/inference-execution.js +1 -1
  629. package/src/tooling/browser-harness/model-resolution.js +1 -1
  630. package/src/tooling/browser-harness/runtime-config.js +1 -1
  631. package/src/tooling/browser-harness/runtime-isolation.js +2 -5
  632. package/src/tooling/browser-harness/text-evidence.js +1 -1
  633. package/src/tooling/browser-harness-contract-helpers.js +1 -1
  634. package/src/tooling/browser-harness.js +2 -2
  635. package/src/tooling/capsule-v2.js +2 -2
  636. package/src/tooling/capsule.js +2 -2
  637. package/src/tooling/command-api-constants.js +4 -2
  638. package/src/tooling/command-api.d.ts +1 -1
  639. package/src/tooling/command-api.js +15 -21
  640. package/src/tooling/command-runtime-execution.js +1 -8
  641. package/src/tooling/diagnose-runner.js +3 -41
  642. package/src/tooling/node-browser/static-server.js +2 -2
  643. package/src/tooling/node-command-execution.js +2 -34
  644. package/src/tooling/node-converter/input.js +1 -1
  645. package/src/tooling/node-converter/output.js +2 -2
  646. package/src/tooling/node-converter.js +1 -1
  647. package/src/tooling/node-runtime-bridge.js +43 -0
  648. package/src/tooling/node-source-runtime.js +34 -35
  649. package/src/tooling/production-release.js +1 -1
  650. package/src/tooling/program-bundle/materialize.js +1 -1
  651. package/src/tooling/program-bundle-parity.d.ts +1 -1
  652. package/src/tooling/program-bundle-parity.js +1 -1
  653. package/src/tooling/program-bundle.d.ts +1 -1
  654. package/src/tooling/program-bundle.js +2 -2
  655. package/src/tooling/runtime-intent.js +1 -1
  656. package/src/tooling/source-artifact-adapter.js +3 -3
  657. package/src/tooling/source-runtime/node-loader.js +32 -5
  658. package/src/tooling/source-runtime-converter-config.js +28 -6
  659. package/src/tooling-exports.shared.d.ts +3 -3
  660. package/src/tooling-exports.shared.js +2 -2
  661. package/src/version.js +1 -1
  662. package/tests/kernels/browser/kernel-suite.js +1 -1
  663. package/tests/kernels/harness/benchmark.js +1 -1
  664. package/tools/README.md +74 -0
  665. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +0 -4407
  666. package/src/inference/pipelines/text/moe-cpu.js +0 -20
  667. /package/src/config/{capsule-loading.d.ts → capsule/capsule-loading.d.ts} +0 -0
  668. /package/src/config/{capsule-release-contract.d.ts → capsule/capsule-release-contract.d.ts} +0 -0
  669. /package/src/config/{capsule-signature.d.ts → capsule/capsule-signature.d.ts} +0 -0
  670. /package/src/config/{capsule-signature.js → capsule/capsule-signature.js} +0 -0
  671. /package/src/config/{capsule-v3.d.ts → capsule/capsule-v3.d.ts} +0 -0
  672. /package/src/config/{capsule.d.ts → capsule/capsule.d.ts} +0 -0
  673. /package/src/config/schema/{diffusion.schema.d.ts → execution/diffusion.schema.d.ts} +0 -0
  674. /package/src/config/schema/{diffusion.schema.js → execution/diffusion.schema.js} +0 -0
  675. /package/src/config/schema/{energy.schema.d.ts → execution/energy.schema.d.ts} +0 -0
  676. /package/src/config/schema/{energy.schema.js → execution/energy.schema.js} +0 -0
  677. /package/src/config/schema/{hyper-connection-decoder.schema.d.ts → execution/hyper-connection-decoder.schema.d.ts} +0 -0
  678. /package/src/config/schema/{hyper-connection-decoder.schema.js → execution/hyper-connection-decoder.schema.js} +0 -0
  679. /package/src/config/schema/{inference.schema.js → execution/inference.schema.js} +0 -0
  680. /package/src/config/schema/{kernel-path.schema.js → execution/kernel-path.schema.js} +0 -0
  681. /package/src/config/schema/{kernel-registry.schema.js → execution/kernel-registry.schema.js} +0 -0
  682. /package/src/config/schema/{kernel-warmup.schema.d.ts → execution/kernel-warmup.schema.d.ts} +0 -0
  683. /package/src/config/schema/{kernel-warmup.schema.js → execution/kernel-warmup.schema.js} +0 -0
  684. /package/src/config/schema/{speculation-self.schema.d.ts → execution/speculation-self.schema.d.ts} +0 -0
  685. /package/src/config/schema/{speculation-self.schema.js → execution/speculation-self.schema.js} +0 -0
  686. /package/src/config/schema/{speculative.schema.d.ts → execution/speculative.schema.d.ts} +0 -0
  687. /package/src/config/schema/{speculative.schema.js → execution/speculative.schema.js} +0 -0
  688. /package/src/config/schema/{conversion-report.schema.d.ts → manifest/conversion-report.schema.d.ts} +0 -0
  689. /package/src/config/schema/{conversion-report.schema.js → manifest/conversion-report.schema.js} +0 -0
  690. /package/src/config/schema/{conversion.schema.d.ts → manifest/conversion.schema.d.ts} +0 -0
  691. /package/src/config/schema/{conversion.schema.js → manifest/conversion.schema.js} +0 -0
  692. /package/src/config/schema/{fp8.schema.d.ts → manifest/fp8.schema.d.ts} +0 -0
  693. /package/src/config/schema/{fp8.schema.js → manifest/fp8.schema.js} +0 -0
  694. /package/src/config/schema/{manifest-rope.schema.d.ts → manifest/manifest-rope.schema.d.ts} +0 -0
  695. /package/src/config/schema/{quantization-defaults.schema.d.ts → manifest/quantization-defaults.schema.d.ts} +0 -0
  696. /package/src/config/schema/{quantization-defaults.schema.js → manifest/quantization-defaults.schema.js} +0 -0
  697. /package/src/config/schema/{quantization.schema.d.ts → manifest/quantization.schema.d.ts} +0 -0
  698. /package/src/config/schema/{quantization.schema.js → manifest/quantization.schema.js} +0 -0
  699. /package/src/config/schema/{w8a8.schema.d.ts → manifest/w8a8.schema.d.ts} +0 -0
  700. /package/src/config/schema/{w8a8.schema.js → manifest/w8a8.schema.js} +0 -0
  701. /package/src/config/schema/{gpu-cache.schema.d.ts → platform/gpu-cache.schema.d.ts} +0 -0
  702. /package/src/config/schema/{gpu-cache.schema.js → platform/gpu-cache.schema.js} +0 -0
  703. /package/src/config/schema/{memory-limits.schema.d.ts → platform/memory-limits.schema.d.ts} +0 -0
  704. /package/src/config/schema/{platform.schema.d.ts → platform/platform.schema.d.ts} +0 -0
  705. /package/src/config/schema/{support-tiers.schema.d.ts → policy/support-tiers.schema.d.ts} +0 -0
  706. /package/src/config/schema/{support-tiers.schema.js → policy/support-tiers.schema.js} +0 -0
  707. /package/src/config/schema/{adapter.schema.d.ts → runtime/adapter.schema.d.ts} +0 -0
  708. /package/src/config/schema/{adapter.schema.js → runtime/adapter.schema.js} +0 -0
  709. /package/src/config/schema/{benchmark.schema.d.ts → runtime/benchmark.schema.d.ts} +0 -0
  710. /package/src/config/schema/{benchmark.schema.js → runtime/benchmark.schema.js} +0 -0
  711. /package/src/config/schema/{bridge.schema.d.ts → runtime/bridge.schema.d.ts} +0 -0
  712. /package/src/config/schema/{bridge.schema.js → runtime/bridge.schema.js} +0 -0
  713. /package/src/config/schema/{browser-suite-metrics.schema.d.ts → runtime/browser-suite-metrics.schema.d.ts} +0 -0
  714. /package/src/config/schema/{browser-suite-metrics.schema.js → runtime/browser-suite-metrics.schema.js} +0 -0
  715. /package/src/config/schema/{buffer-pool.schema.d.ts → runtime/buffer-pool.schema.d.ts} +0 -0
  716. /package/src/config/schema/{buffer-pool.schema.js → runtime/buffer-pool.schema.js} +0 -0
  717. /package/src/config/schema/{debug.schema.d.ts → runtime/debug.schema.d.ts} +0 -0
  718. /package/src/config/schema/{debug.schema.js → runtime/debug.schema.js} +0 -0
  719. /package/src/config/schema/{distribution.schema.d.ts → runtime/distribution.schema.d.ts} +0 -0
  720. /package/src/config/schema/{distribution.schema.js → runtime/distribution.schema.js} +0 -0
  721. /package/src/config/schema/{ecosystem.schema.d.ts → runtime/ecosystem.schema.d.ts} +0 -0
  722. /package/src/config/schema/{ecosystem.schema.js → runtime/ecosystem.schema.js} +0 -0
  723. /package/src/config/schema/{harness.schema.d.ts → runtime/harness.schema.d.ts} +0 -0
  724. /package/src/config/schema/{harness.schema.js → runtime/harness.schema.js} +0 -0
  725. /package/src/config/schema/{hotswap.schema.d.ts → runtime/hotswap.schema.d.ts} +0 -0
  726. /package/src/config/schema/{hotswap.schema.js → runtime/hotswap.schema.js} +0 -0
  727. /package/src/config/schema/{intent-bundle.schema.d.ts → runtime/intent-bundle.schema.d.ts} +0 -0
  728. /package/src/config/schema/{intent-bundle.schema.js → runtime/intent-bundle.schema.js} +0 -0
  729. /package/src/config/schema/{runtime-profile.schema.d.ts → runtime/runtime-profile.schema.d.ts} +0 -0
  730. /package/src/config/schema/{runtime-profile.schema.js → runtime/runtime-profile.schema.js} +0 -0
  731. /package/src/config/schema/{storage.schema.d.ts → runtime/storage.schema.d.ts} +0 -0
  732. /package/src/config/schema/{storage.schema.js → runtime/storage.schema.js} +0 -0
  733. /package/src/config/schema/{tooling.schema.js → runtime/tooling.schema.js} +0 -0
  734. /package/src/config/schema/{units.schema.d.ts → runtime/units.schema.d.ts} +0 -0
  735. /package/src/config/schema/{units.schema.js → runtime/units.schema.js} +0 -0
  736. /package/src/config/schema/{backward-registry.schema.d.ts → training/backward-registry.schema.d.ts} +0 -0
  737. /package/src/config/schema/{backward-registry.schema.js → training/backward-registry.schema.js} +0 -0
  738. /package/src/config/schema/{distill-training.schema.d.ts → training/distill-training.schema.d.ts} +0 -0
  739. /package/src/config/schema/{distill-training.schema.js → training/distill-training.schema.js} +0 -0
  740. /package/src/config/schema/{lora.schema.d.ts → training/lora.schema.d.ts} +0 -0
  741. /package/src/config/schema/{training-metrics.schema.d.ts → training/training-metrics.schema.d.ts} +0 -0
  742. /package/src/config/schema/{training-metrics.schema.js → training/training-metrics.schema.js} +0 -0
  743. /package/src/config/schema/{training.schema.d.ts → training/training.schema.d.ts} +0 -0
  744. /package/src/config/schema/{training.schema.js → training/training.schema.js} +0 -0
  745. /package/src/config/schema/{ul-training.schema.d.ts → training/ul-training.schema.d.ts} +0 -0
  746. /package/src/config/schema/{ul-training.schema.js → training/ul-training.schema.js} +0 -0
  747. /package/src/{converter → config/transforms}/execution-graph/dtype.js +0 -0
  748. /package/src/{converter → config/transforms}/execution-graph/fusion.js +0 -0
  749. /package/src/{converter → config/transforms}/execution-graph/kv-storage.js +0 -0
  750. /package/src/{converter → config/transforms}/execution-graph/validation.js +0 -0
  751. /package/src/{converter → formats}/qwen4-exp-source-defaults.json +0 -0
  752. /package/src/inference/pipelines/text/{buffer-types.d.ts → contracts/buffer-types.d.ts} +0 -0
  753. /package/src/inference/pipelines/text/{embedding-contract.d.ts → embedding/embedding-contract.d.ts} +0 -0
  754. /package/src/inference/pipelines/text/{embedding-contract.js → embedding/embedding-contract.js} +0 -0
  755. /package/src/inference/pipelines/text/{precomputed-input-norm.js → execution/precomputed-input-norm.js} +0 -0
  756. /package/src/inference/pipelines/text/generator/{batch-retry-policy.js → decode/batch-retry-policy.js} +0 -0
  757. /package/src/inference/pipelines/text/{chat-format.d.ts → generator/request/chat-format.d.ts} +0 -0
  758. /package/src/inference/pipelines/text/{chat-format.js → generator/request/chat-format.js} +0 -0
  759. /package/src/inference/pipelines/text/generator/{recorder-stats.js → session/recorder-stats.js} +0 -0
  760. /package/src/inference/pipelines/text/generator/{whole-token-template.js → whole-token/whole-token-template.js} +0 -0
  761. /package/src/inference/pipelines/text/{init-chat-templates.js → initialization/init-chat-templates.js} +0 -0
  762. /package/src/inference/pipelines/text/{load-policy.js → loading/load-policy.js} +0 -0
  763. /package/src/inference/pipelines/text/{pipeline-load-timing.js → loading/pipeline-load-timing.js} +0 -0
  764. /package/src/inference/pipelines/text/{generator → logits}/hidden-projection.d.ts +0 -0
  765. /package/src/inference/pipelines/text/{operator-events.d.ts → observation/operator-events.d.ts} +0 -0
  766. /package/src/inference/pipelines/text/{stage-names.d.ts → observation/stage-names.d.ts} +0 -0
  767. /package/src/inference/pipelines/text/{stage-names.js → observation/stage-names.js} +0 -0
  768. /package/src/inference/pipelines/text/{tsir-fixture-writer.d.ts → observation/tsir-fixture-writer.d.ts} +0 -0
  769. /package/src/inference/pipelines/text/{generator → per-layer}/per-layer-buffer.js +0 -0
  770. /package/src/inference/pipelines/text/{finiteness-policy.js → policy/finiteness-policy.js} +0 -0
@@ -1,14 +1,23 @@
1
+ import { ExpertFrequencyIndex } from './expert-frequency-index.js';
2
+ import { getRuntimeConfig } from '../../config/runtime.js';
3
+ import { CompactExpertCache, compactExpertCapacity } from './compact-expert-cache.js';
1
4
  import { createWeightBuffer } from '../../gpu/weight-buffer.js';
2
- import { validateQwen35RoutedConfig } from '../../config/schema/loading.schema.js';
3
- import { loadTensorRange } from '../tensors/tensor-reader.js';
5
+ import { validateQwen35RoutedConfig } from "../../config/schema/runtime/loading.schema.js";
6
+ import { createRoutedExpertCounters, readRoutedExpertRange, assertExpertReadShape, readRoutedExpertPayload } from './routed-expert-source.js';
7
+
8
+ import { writeRoutedPrefillExpert, selectPrefillSeedExperts } from './routed-expert-prefill.js';
4
9
 
5
10
  // The cache owns individual expert payloads. One reusable logical bank preserves
6
- // the existing kernels' expert IDs, Q4K byte offsets and accumulation order.
11
+ // the existing kernels' expert IDs, source-format byte offsets and accumulation order.
7
12
  export class RoutedExpertCache {
8
13
  /** @type {import('./routed-expert-cache.js').RoutedExpertCacheOptions} */
9
14
  #options;
10
15
  /** @type {Map<string, import('./routed-expert-cache.js').RoutedExpertEntry>} */
11
16
  #entries = new Map();
17
+ /** @type {ExpertFrequencyIndex | null} */
18
+ #frequency = null;
19
+ #numLayers = 0;
20
+ #numExperts = 0;
12
21
  /** @type {Set<GPUBuffer>} */
13
22
  #owned = new Set();
14
23
  /** @type {Map<string, GPUBuffer>} */
@@ -17,30 +26,91 @@ export class RoutedExpertCache {
17
26
  #layer = -1;
18
27
  #signature = '';
19
28
  #cacheBytes = 0;
29
+ #submittedSerial = 0;
30
+ #completedSerial = 0;
20
31
  /** @type {Map<string, import('./routed-expert-cache.js').RoutedHostExpertEntry>} */
21
32
  #hostEntries = new Map();
22
33
  #hostBytes = 0;
23
34
  #hostReserved = 0;
24
35
  /** @type {Set<string>} */
25
36
  #hostSeen = new Set();
37
+ /** @type {CompactExpertCache | null} */
38
+ #compact = null;
26
39
  #workingBytes = 0;
27
40
  #staging = 0;
41
+ /** @type {Uint8Array[]} */
42
+ #readerStaging = [];
28
43
  #closed = false;
29
44
  #shutdown = new AbortController();
30
45
  /** @type {Promise<void>} */
31
46
  #mutation = Promise.resolve();
32
47
  /** @type {Promise<import('./routed-expert-cache.js').RoutedExpertWeights> | null} */
33
48
  #active = null;
34
- #stats = { sourceReadBytes: 0, sourceReads: 0, uploadBytes: 0,
35
- cacheHits: 0, cacheMisses: 0, workingHits: 0, evictions: 0,
36
- residentBytes: 0, peakResidentBytes: 0, peakAccountedGpuBytes: 0, peakStagingBytes: 0,
37
- ioWaitMs: 0, sourceReadWaitMs: 0, uploadEnqueueMs: 0, initialQueueIdleForWeightsMs: 0,
38
- hostCacheHits: 0, hostCacheMisses: 0, hostEvictions: 0, hostAdmissionSkips: 0, peakHostOwnedOrReservedBytes: 0 };
49
+ /** @type {Record<string, import('./routed-expert-cache.js').RoutedExpertPhaseProfile>} */
50
+ #phaseProfiles = {};
51
+ /** @type {import('./routed-expert-cache.js').RoutedExpertPhaseProfile | null} */
52
+ #activeProfile = null;
53
+ #stats = createRoutedExpertCounters();
39
54
 
40
55
  /** @param {import('./routed-expert-cache.js').RoutedExpertCacheOptions} options */
41
56
  constructor(options) {
42
57
  validateQwen35RoutedConfig(options.policy);
43
58
  this.#options = options;
59
+ const lease = options.rangeLease;
60
+ if (lease && (!Number.isSafeInteger(lease.concurrency ?? 1) || (lease.concurrency ?? 1) < 1
61
+ || !Number.isSafeInteger(lease.maxPartBytes) || lease.maxPartBytes < 4
62
+ || !Number.isSafeInteger(lease.memoryBytes) || lease.memoryBytes < lease.maxPartBytes * (lease.concurrency ?? 1)
63
+ || lease.memoryBytes > options.policy.stagingBytes - 8 * options.policy.readConcurrency)) {
64
+ throw new Error('Owned expert ranges must fit the aggregate staging budget with fallback scratch.');
65
+ }
66
+ this.#staging = lease?.memoryBytes ?? 0;
67
+ if (options.policy.evictionPolicy === 'lfu') {
68
+ if (typeof options.numLayers !== 'number' || typeof options.numExperts !== 'number'
69
+ || !Number.isSafeInteger(options.numLayers) || options.numLayers < 1
70
+ || !Number.isSafeInteger(options.numExperts) || options.numExperts < 1
71
+ || !Number.isSafeInteger(options.numLayers * options.numExperts)) {
72
+ throw new Error('LFU requires declared positive safe layer and expert counts.');
73
+ }
74
+ this.#numLayers = options.numLayers;
75
+ this.#numExperts = options.numExperts;
76
+ this.#frequency = new ExpertFrequencyIndex(options.numLayers * options.numExperts);
77
+ }
78
+ }
79
+
80
+ #sampleProfileMemory() {
81
+ const profile = this.#activeProfile;
82
+ if (!profile) return;
83
+ profile.peakGpuResidentBytes = Math.max(profile.peakGpuResidentBytes, this.#stats.residentBytes);
84
+ profile.peakHostOwnedOrReservedBytes = Math.max(profile.peakHostOwnedOrReservedBytes, this.#hostBytes + this.#hostReserved);
85
+ profile.peakStagingBytes = Math.max(profile.peakStagingBytes, this.#staging);
86
+ }
87
+
88
+ #clearReaderStaging() {
89
+ for (const bytes of this.#readerStaging) this.#staging -= bytes?.byteLength ?? 0;
90
+ this.#readerStaging = [];
91
+ }
92
+
93
+ /** @param {number} reader @param {number} length */
94
+ #getReaderStaging(reader, length) {
95
+ let bytes = this.#readerStaging[reader];
96
+ if (!bytes || bytes.byteLength < length) {
97
+ const previousBytes = bytes?.byteLength ?? 0;
98
+ bytes = new Uint8Array(length);
99
+ this.#staging -= previousBytes;
100
+ this.#readerStaging[reader] = bytes;
101
+ this.#staging += bytes.byteLength;
102
+ this.#stats.stagingAllocations++;
103
+ } else this.#stats.stagingReuses++;
104
+ return bytes.subarray(0, length);
105
+ }
106
+
107
+ /** @param {string} key */
108
+ #lookupHost(key) {
109
+ if (!this.#activeProfile) return this.#hostEntries.get(key);
110
+ const start = performance.now();
111
+ const entry = this.#hostEntries.get(key);
112
+ this.#activeProfile.hostLookupMs += performance.now() - start;
113
+ return entry;
44
114
  }
45
115
 
46
116
  /** @template T @param {() => Promise<T>} action @returns {Promise<T>} */
@@ -63,6 +133,7 @@ export class RoutedExpertCache {
63
133
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST, label });
64
134
  this.#owned.add(buffer);
65
135
  this.#stats.residentBytes += bytes;
136
+ this.#sampleProfileMemory();
66
137
  this.#stats.peakResidentBytes = Math.max(this.#stats.peakResidentBytes, this.#stats.residentBytes);
67
138
  this.#stats.peakAccountedGpuBytes = Math.max(this.#stats.peakAccountedGpuBytes,
68
139
  this.#options.otherGpuBytes() + this.#stats.residentBytes);
@@ -75,21 +146,67 @@ export class RoutedExpertCache {
75
146
  if (this.#owned.delete(buffer)) this.#stats.residentBytes -= buffer.size;
76
147
  }
77
148
 
78
- /** @param {number} bytes @param {number} cacheAddition @param {AbortSignal} signal */
79
- async #makeRoom(bytes, cacheAddition, signal) {
80
- const { policy, device } = this.#options;
81
- while (this.#cacheBytes + cacheAddition > policy.cacheBytes
82
- || this.#options.otherGpuBytes() + this.#stats.residentBytes + bytes > policy.gpuBudgetBytes - policy.reserveBytes) {
83
- const victim = [...this.#entries].find(([, entry]) => entry.users === 0);
84
- if (!victim) throw new Error('Routed expert budget cannot accommodate the working set and concurrent readers.');
85
- // Pins are changed under the mutation lock. A victim cannot acquire a new
86
- // copy lease while this fence settles its previously submitted work.
87
- await device.queue.onSubmittedWorkDone();
88
- signal.throwIfAborted();
89
- this.#destroyBuffer(victim[1].buffer);
90
- this.#cacheBytes -= victim[1].bytes;
91
- this.#entries.delete(victim[0]);
92
- this.#stats.evictions++;
149
+ // A fence covers only submissions issued before that call. Never mark later
150
+ // copies complete when an earlier fence resolves; entries may still be in use.
151
+ async #waitForSubmitted() {
152
+ const serial = this.#submittedSerial;
153
+ const profile = this.#activeProfile;
154
+ const start = profile ? performance.now() : 0;
155
+ try { await this.#options.device.queue.onSubmittedWorkDone(); }
156
+ finally { if (profile) profile.synchronizationWaitMs += performance.now() - start; }
157
+ this.#completedSerial = Math.max(this.#completedSerial, serial);
158
+ }
159
+
160
+ /** @param {number} bytes @param {number} cacheAddition @param {AbortSignal} signal
161
+ * @param {boolean} reusePayload @returns {Promise<GPUBuffer | null>} */
162
+ async #makeRoom(bytes, cacheAddition, signal, reusePayload = false) {
163
+ const { policy } = this.#options;
164
+ /** @type {GPUBuffer | null} */
165
+ let replacement = null;
166
+ try {
167
+ while (this.#cacheBytes + cacheAddition > policy.cacheBytes
168
+ || (policy.compactPrefillBytes > 0 && this.#stats.residentBytes
169
+ - (this.#compact?.residentBytes ?? 0) + bytes > policy.compactPrefillBytes)
170
+ || this.#options.otherGpuBytes() + this.#stats.residentBytes + bytes > policy.gpuBudgetBytes - policy.reserveBytes) {
171
+ /** @type {[string, import('./routed-expert-cache.js').RoutedExpertEntry] | undefined} */
172
+ let victim;
173
+ // LRU order already lives in Map: do not allocate the entire cache on
174
+ // every eviction just to inspect its first unpinned entry.
175
+ if (this.#frequency) {
176
+ const key = this.#frequency.oldestUnpinned(key => this.#entries.get(key)?.users !== 0);
177
+ if (key !== null) {
178
+ const entry = this.#entries.get(key);
179
+ if (!entry) throw new Error('LFU resident index disagrees with cache ownership.');
180
+ victim = [key, entry];
181
+ }
182
+ } else {
183
+ for (const candidate of this.#entries) {
184
+ if (candidate[1].users === 0) { victim = candidate; break; }
185
+ }
186
+ }
187
+ if (!victim) throw new Error('Routed expert budget cannot accommodate the working set and concurrent readers.');
188
+ // Pins are changed under the mutation lock. A victim cannot acquire a new
189
+ // copy lease while this fence settles its previously submitted work.
190
+ if (victim[1].lastUseSerial > this.#completedSerial) {
191
+ this.#stats.evictionFenceWaits++;
192
+ await this.#waitForSubmitted();
193
+ } else this.#stats.evictionFenceSkips++;
194
+ signal.throwIfAborted();
195
+ // Transfer only an exact-size, unpinned, completed payload. It remains
196
+ // owned and resident; no spare pool or extra budget is introduced.
197
+ if (reusePayload && replacement === null && victim[1].bytes === bytes) {
198
+ replacement = victim[1].buffer;
199
+ bytes = 0;
200
+ } else this.#destroyBuffer(victim[1].buffer);
201
+ this.#cacheBytes -= victim[1].bytes;
202
+ this.#entries.delete(victim[0]);
203
+ this.#frequency?.remove(victim[0]);
204
+ this.#stats.evictions++;
205
+ }
206
+ return replacement;
207
+ } catch (error) {
208
+ if (replacement) this.#destroyBuffer(replacement);
209
+ throw error;
93
210
  }
94
211
  }
95
212
 
@@ -108,6 +225,7 @@ export class RoutedExpertCache {
108
225
  #upload(entry, offset, bytes, beforeSubmit) {
109
226
  beforeSubmit();
110
227
  const started = performance.now();
228
+ entry.lastUseSerial = ++this.#submittedSerial;
111
229
  this.#options.device.queue.writeBuffer(entry.buffer, offset, bytes);
112
230
  this.#stats.uploadEnqueueMs += performance.now() - started;
113
231
  this.#stats.uploadBytes += bytes.byteLength;
@@ -115,26 +233,44 @@ export class RoutedExpertCache {
115
233
 
116
234
  /** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan @param {AbortSignal} signal */
117
235
  async #prepareWorking(plan, signal) {
118
- const signature = JSON.stringify(plan.planes.map(p => [p.role, p.location.shape, p.bytes]));
119
- if (this.#working.size && signature !== this.#signature) {
120
- await this.#options.device.queue.onSubmittedWorkDone();
236
+ const signature = JSON.stringify(plan.planes.map(p => [p.role, p.dtype, p.location.shape, p.bytes]));
237
+ const { policy } = this.#options;
238
+ const payloadBytes = policy.prefillUploadMode === 'direct-bank' ? 0 : plan.expertBytes * policy.readConcurrency;
239
+ let spareCapacity = Math.max(0, policy.gpuBudgetBytes - policy.reserveBytes
240
+ - this.#options.otherGpuBytes() - plan.bankBytes - payloadBytes);
241
+ if (this.#working.size && (signature !== this.#signature
242
+ || this.#workingBytes - plan.bankBytes > spareCapacity)) {
243
+ await this.#waitForSubmitted();
121
244
  signal.throwIfAborted();
245
+ // Capacity is owned storage, not tensor geometry. Preserve payload-reader
246
+ // headroom; trim surplus if the current budget no longer accommodates it.
122
247
  for (const [role, buffer] of this.#working) {
248
+ const next = plan.planes.find(plane => plane.role === role);
249
+ if (next && buffer.size >= next.bytes && buffer.size - next.bytes <= spareCapacity) {
250
+ spareCapacity -= buffer.size - next.bytes;
251
+ this.#stats.workingBufferReuses++;
252
+ this.#stats.workingBufferReusedBytes += buffer.size;
253
+ continue;
254
+ }
123
255
  this.#destroyBuffer(buffer);
124
256
  this.#working.delete(role);
257
+ this.#workingBytes -= buffer.size;
125
258
  }
126
- this.#workingBytes = 0;
259
+ this.#loaded.clear();
127
260
  }
128
- if (!this.#working.size) {
129
- await this.#makeRoom(plan.bankBytes, 0, signal);
261
+ const missing = plan.planes.filter(plane => !this.#working.has(plane.role));
262
+ if (missing.length) {
263
+ await this.#makeRoom(missing.reduce((bytes, plane) => bytes + plane.bytes, 0), 0, signal);
130
264
  signal.throwIfAborted();
131
- for (const plane of plan.planes) {
265
+ for (const plane of missing) {
132
266
  const buffer = this.#allocate(plane.bytes, `routed_working_${plane.role}`);
133
267
  this.#working.set(plane.role, buffer);
134
268
  this.#workingBytes += plane.bytes;
269
+ this.#stats.workingBufferAllocations++;
270
+ this.#stats.workingBufferAllocatedBytes += plane.bytes;
135
271
  }
136
- this.#signature = signature;
137
272
  }
273
+ this.#signature = signature;
138
274
  if (this.#layer !== plan.layer) {
139
275
  this.#loaded.clear();
140
276
  this.#layer = plan.layer;
@@ -145,39 +281,41 @@ export class RoutedExpertCache {
145
281
  }
146
282
 
147
283
  /** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
148
- * @param {number} expert @param {AbortSignal} signal @param {() => void} beforeSubmit */
149
- async #get(plan, expert, signal, beforeSubmit) {
284
+ * @param {number} expert @param {AbortSignal} signal @param {() => void} beforeSubmit
285
+ * @param {'prefill' | 'decode' | undefined} phase @param {number} reader */
286
+ async #get(plan, expert, signal, beforeSubmit, phase, reader) {
150
287
  const key = `${plan.layer}:${expert}`;
151
288
  const cached = await this.#locked(async () => {
152
289
  signal.throwIfAborted();
153
290
  const entry = this.#entries.get(key);
154
291
  if (entry) {
155
- this.#entries.delete(key);
156
- this.#entries.set(key, entry);
292
+ this.#touchEntry(key, entry);
157
293
  entry.users++;
158
294
  this.#stats.cacheHits++;
159
295
  }
160
296
  return entry;
161
297
  });
162
- if (cached) {
163
- const host = this.#hostEntries.get(key);
164
- if (host) { this.#hostEntries.delete(key); this.#hostEntries.set(key, host); }
165
- return cached;
166
- }
298
+ if (cached) return cached;
167
299
  this.#stats.cacheMisses++;
300
+ this.#frequency?.observe(key);
168
301
  const entry = await this.#locked(async () => {
169
- await this.#makeRoom(plan.expertBytes, plan.expertBytes, signal);
170
- signal.throwIfAborted();
171
- const buffer = this.#allocate(plan.expertBytes, `routed_expert_${key}`);
302
+ const replacement = await this.#makeRoom(plan.expertBytes, plan.expertBytes, signal, true);
303
+ if (signal.aborted) {
304
+ if (replacement) this.#destroyBuffer(replacement);
305
+ signal.throwIfAborted();
306
+ }
307
+ const buffer = replacement ?? this.#allocate(plan.expertBytes, `routed_expert_${key}`);
308
+ if (replacement) this.#stats.bufferReuses++;
172
309
  this.#cacheBytes += plan.expertBytes;
173
- return { buffer, bytes: plan.expertBytes, users: 1 };
310
+ return { buffer, bytes: plan.expertBytes, users: 1, lastUseSerial: 0 };
174
311
  });
175
- // loadTensorRange owns an assembled output as well as its source payload.
176
- const chunkBytes = Math.floor(this.#options.policy.stagingBytes / (2 * this.#options.policy.readConcurrency * 4)) * 4;
312
+ // Keep read granularity unchanged; direct fill owns only the assembled output.
313
+ const readRangeInto = this.#options.readRangeInto;
314
+ const chunkBytes = Math.floor((this.#options.policy.stagingBytes - (this.#options.rangeLease?.memoryBytes ?? 0)) / (2 * this.#options.policy.readConcurrency * 4)) * 4;
177
315
  let targetOffset = 0;
178
316
  let hostReservation = 0;
179
317
  try {
180
- const host = this.#hostEntries.get(key);
318
+ const host = this.#lookupHost(key);
181
319
  if (host) {
182
320
  this.#hostEntries.delete(key); this.#hostEntries.set(key, host);
183
321
  this.#stats.hostCacheHits++;
@@ -187,7 +325,7 @@ export class RoutedExpertCache {
187
325
  signal.throwIfAborted();
188
326
  this.#upload(entry, chunk.offset, chunk.data, beforeSubmit);
189
327
  }
190
- await this.#locked(async () => { this.#entries.set(key, entry); });
328
+ await this.#locked(async () => { this.#entries.set(key, entry); this.#frequency?.admit(key); });
191
329
  return entry;
192
330
  }
193
331
  /** @type {Array<{offset: number, data: Uint8Array}> | null} */
@@ -195,57 +333,70 @@ export class RoutedExpertCache {
195
333
  if (this.#options.policy.hostCacheBytes > 0) {
196
334
  this.#stats.hostCacheMisses++;
197
335
  const repeatOnly = this.#options.policy.hostCacheAdmission === 'repeat-read';
198
- const admit = !repeatOnly || this.#hostSeen.has(key);
199
- if (repeatOnly) this.#hostSeen.add(key);
336
+ const phaseAllowed = this.#options.policy.hostCacheAdmissionPhase === 'all' || phase === 'decode';
337
+ const admit = phaseAllowed && (!repeatOnly || this.#hostSeen.has(key));
338
+ if (phaseAllowed && repeatOnly) this.#hostSeen.add(key);
339
+ if (!phaseAllowed) this.#stats.hostPrefillAdmissionSkips++;
200
340
  if (admit) {
201
341
  this.#makeHostRoom(plan.expertBytes);
202
342
  this.#hostReserved += plan.expertBytes; hostReservation = plan.expertBytes;
343
+ this.#sampleProfileMemory();
203
344
  this.#stats.peakHostOwnedOrReservedBytes = Math.max(this.#stats.peakHostOwnedOrReservedBytes,
204
345
  this.#hostBytes + this.#hostReserved);
205
346
  chunks = [];
206
347
  } else this.#stats.hostAdmissionSkips++;
207
348
  }
208
- for (const plane of plan.planes) {
209
- for (let offset = 0; offset < plane.expertBytes;) {
349
+ if (this.#options.policy.sourceReadMode === 'expert-parallel') {
350
+ const destination = this.#getReaderStaging(reader, plan.expertBytes);
351
+ this.#sampleProfileMemory();
352
+ this.#stats.peakStagingBytes = Math.max(this.#stats.peakStagingBytes, this.#staging);
353
+ const started = performance.now();
354
+ try {
355
+ await readRoutedExpertPayload(this.#options, this.#stats, plan, expert, signal, destination, null);
210
356
  signal.throwIfAborted();
211
- const length = Math.min(chunkBytes, plane.expertBytes - offset);
212
- this.#staging += 2 * length;
213
- this.#stats.peakStagingBytes = Math.max(this.#stats.peakStagingBytes, this.#staging);
214
- const started = performance.now();
215
- try {
216
- const bytes = await loadTensorRange(plane.location, plane.name, expert * plane.expertBytes + offset, length,
217
- async (shard, start, size) => {
218
- signal.throwIfAborted();
219
- const readStarted = performance.now();
220
- const data = await this.#options.readRange(shard, start, size, signal);
221
- this.#stats.sourceReadWaitMs += performance.now() - readStarted;
222
- if (data.byteLength !== size) throw new Error('Routed expert source must return exactly the requested byte range.');
223
- this.#stats.sourceReads++;
224
- this.#stats.sourceReadBytes += data.byteLength;
225
- return data;
226
- });
357
+ this.#upload(entry, 0, destination, beforeSubmit);
358
+ } finally { this.#stats.ioWaitMs += performance.now() - started; }
359
+ } else {
360
+ for (const plane of plan.planes) {
361
+ for (let offset = 0; offset < plane.expertBytes;) {
227
362
  signal.throwIfAborted();
228
- this.#upload(entry, targetOffset + offset, bytes, beforeSubmit);
229
- chunks?.push({ offset: targetOffset + offset, data: bytes });
230
- } finally {
231
- this.#stats.ioWaitMs += performance.now() - started;
232
- this.#staging -= 2 * length;
363
+ const length = Math.min(chunkBytes, plane.expertBytes - offset);
364
+ // Each reader owns its destination until writeBuffer has copied it.
365
+ // Retained host chunks must keep independent immutable backing storage.
366
+ const destination = readRangeInto && chunks === null && plane.location.sourceTransform == null
367
+ ? this.#getReaderStaging(reader, length) : null;
368
+ const ownedStagingBytes = destination ? 0 : (readRangeInto ? 1 : 2) * length;
369
+ this.#staging += ownedStagingBytes;
370
+ this.#sampleProfileMemory();
371
+ this.#stats.peakStagingBytes = Math.max(this.#stats.peakStagingBytes, this.#staging);
372
+ const started = performance.now();
373
+ try {
374
+ const bytes = await readRoutedExpertRange(this.#options, this.#stats, plane, expert, offset, length, signal, destination);
375
+ signal.throwIfAborted();
376
+ this.#upload(entry, targetOffset + offset, bytes, beforeSubmit);
377
+ chunks?.push({ offset: targetOffset + offset, data: bytes });
378
+ } finally {
379
+ this.#stats.ioWaitMs += performance.now() - started;
380
+ this.#staging -= ownedStagingBytes;
381
+ }
382
+ offset += length;
233
383
  }
234
- offset += length;
384
+ targetOffset += plane.expertBytes;
235
385
  }
236
- targetOffset += plane.expertBytes;
237
386
  }
238
387
  if (chunks) {
239
388
  signal.throwIfAborted();
240
389
  this.#hostReserved -= hostReservation; hostReservation = 0;
241
390
  this.#hostEntries.set(key, { bytes: plan.expertBytes, chunks });
242
391
  this.#hostBytes += plan.expertBytes;
392
+ if (phase === 'prefill') this.#stats.hostPrefillAdmissions++;
393
+ if (phase === 'decode') this.#stats.hostDecodeAdmissions++;
243
394
  }
244
- await this.#locked(async () => { this.#entries.set(key, entry); });
395
+ await this.#locked(async () => { this.#entries.set(key, entry); this.#frequency?.admit(key); });
245
396
  return entry;
246
397
  } catch (error) {
247
398
  try {
248
- await this.#options.device.queue.onSubmittedWorkDone();
399
+ await this.#waitForSubmitted();
249
400
  await this.#locked(async () => { this.#destroyBuffer(entry.buffer); this.#cacheBytes -= entry.bytes; });
250
401
  } catch (cleanup) { throw new AggregateError([error, cleanup], 'Routed expert load and cleanup failed.'); }
251
402
  throw error;
@@ -254,8 +405,108 @@ export class RoutedExpertCache {
254
405
  }
255
406
  }
256
407
 
408
+ /** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
409
+ * @param {number} expert @param {AbortSignal} signal
410
+ * @param {() => void} beforeSubmit @param {number} reader */
411
+ async #writeWorking(plan, expert, signal, beforeSubmit, reader) {
412
+ const allLeased = this.#options.rangeLease && plan.planes.every(plane =>
413
+ plane.location.sourceTransform == null && (!plane.location.spans || plane.location.spans.length <= 1));
414
+ if (plan.planes.some(plane => plane.location.sourceTransform != null)
415
+ || (!allLeased && !this.#options.readRangeInto)) {
416
+ throw new Error('Direct prefill requires unchanged planes and an owned lease or direct range reader.');
417
+ }
418
+ const chunkBytes = Math.floor((this.#options.policy.stagingBytes - (this.#options.rangeLease?.memoryBytes ?? 0))
419
+ / (2 * this.#options.policy.readConcurrency * 4)) * 4;
420
+ const length = this.#options.policy.sourceReadMode === 'expert-parallel' ? plan.expertBytes : chunkBytes;
421
+ const scratch = allLeased ? null : this.#getReaderStaging(reader, length);
422
+ this.#stats.peakStagingBytes = Math.max(this.#stats.peakStagingBytes, this.#staging);
423
+ this.#sampleProfileMemory();
424
+ this.#stats.cacheMisses++;
425
+ await writeRoutedPrefillExpert(this.#options, this.#stats, plan, expert, signal, this.#working, scratch,
426
+ () => { beforeSubmit(); ++this.#submittedSerial; });
427
+ this.#loaded.add(expert);
428
+ }
429
+
430
+ /** @param {string} key @param {import('./routed-expert-cache.js').RoutedExpertEntry} entry */
431
+ #touchEntry(key, entry) {
432
+ this.#frequency?.observe(key);
433
+ this.#entries.delete(key); this.#entries.set(key, entry);
434
+ const host = this.#lookupHost(key);
435
+ if (host) { this.#hostEntries.delete(key); this.#hostEntries.set(key, host); }
436
+ }
437
+
438
+ /** @param {GPUCommandEncoder} encoder
439
+ * @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
440
+ * @param {number} expert @param {import('./routed-expert-cache.js').RoutedExpertEntry} entry */
441
+ #encodeExpertCopy(encoder, plan, expert, entry) {
442
+ let offset = 0;
443
+ for (const plane of plan.planes) {
444
+ const destination = this.#working.get(plane.role);
445
+ if (!destination) throw new Error('Routed working bank is incomplete.');
446
+ encoder.copyBufferToBuffer(entry.buffer, offset, destination, expert * plane.expertBytes, plane.expertBytes);
447
+ offset += plane.expertBytes;
448
+ }
449
+ }
450
+
451
+ /** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
452
+ * @param {number[]} ids @param {AbortSignal} signal @param {() => void} beforeSubmit */
453
+ #copyCachedHits(plan, ids, signal, beforeSubmit) {
454
+ return this.#locked(async () => {
455
+ signal.throwIfAborted();
456
+ const compact = this.#options.policy.compactPrefillReuse === 'resident-gpu' ? this.#compact : null;
457
+ const residentSlots = compact?.getResidentSlots(plan.layer,
458
+ ids.filter(expert => !this.#loaded.has(expert))) ?? new Map();
459
+ const hits = ids.flatMap(expert => {
460
+ const key = `${plan.layer}:${expert}`;
461
+ const entry = this.#entries.get(key);
462
+ return entry && !this.#loaded.has(expert) && !residentSlots.has(expert) ? [{ expert, key, entry }] : [];
463
+ });
464
+ const copied = new Set();
465
+ if (!hits.length && !residentSlots.size) return copied;
466
+ // No miss starts until this batch is submitted. Pins protect all encoded
467
+ // sources, without requiring capacity for a whole layer's missing experts.
468
+ for (const { entry } of hits) entry.users++;
469
+ try {
470
+ const encoder = this.#options.device.createCommandEncoder({ label: 'routed_expert_hit_batch' });
471
+ if (compact && residentSlots.size) compact.encodeResidentCopies(plan, residentSlots, this.#working, encoder);
472
+ for (const { expert, entry } of hits) this.#encodeExpertCopy(encoder, plan, expert, entry);
473
+ const commands = encoder.finish();
474
+ signal.throwIfAborted();
475
+ beforeSubmit();
476
+ const serial = ++this.#submittedSerial;
477
+ for (const { entry } of hits) entry.lastUseSerial = serial;
478
+ this.#options.device.queue.submit([commands]);
479
+ this.#stats.expertCopySubmissions++;
480
+ this.#stats.hitBatchSubmissions++;
481
+ this.#stats.batchedHitExperts += hits.length;
482
+ if (residentSlots.size) {
483
+ this.#stats.prefillGpuReuseBytes += residentSlots.size * plan.expertBytes;
484
+ this.#stats.prefillGpuReuseExperts += residentSlots.size;
485
+ this.#stats.prefillGpuReuseSubmissions++;
486
+ this.#stats.cacheHits += residentSlots.size;
487
+ for (const expert of residentSlots.keys()) {
488
+ this.#loaded.add(expert);
489
+ copied.add(expert);
490
+ }
491
+ }
492
+ for (const { expert, key, entry } of hits) {
493
+ this.#touchEntry(key, entry);
494
+ this.#stats.cacheHits++;
495
+ this.#loaded.add(expert); copied.add(expert);
496
+ }
497
+ return copied;
498
+ } finally {
499
+ for (const { entry } of hits) entry.users--;
500
+ }
501
+ });
502
+ }
503
+
257
504
  /** @type {import('./routed-expert-cache.js').RoutedExpertCache['prepare']} */
258
- prepare(plan, expertIds, externalSignal) {
505
+ prepare(plan, expertIds, externalSignal, phase) {
506
+ if ((phase !== undefined && phase !== 'prefill' && phase !== 'decode')
507
+ || (this.#options.policy.hostCacheAdmissionPhase === 'decode' && phase === undefined)) {
508
+ throw new Error('Routed expert phase-aware host admission requires explicit prefill or decode phase.');
509
+ }
259
510
  if (this.#closed || this.#active) throw new Error('Routed expert cache is closed or already preparing a layer.');
260
511
  const controller = new AbortController();
261
512
  const abortExternal = () => controller.abort(externalSignal?.reason);
@@ -263,9 +514,33 @@ export class RoutedExpertCache {
263
514
  externalSignal?.addEventListener('abort', abortExternal, { once: true });
264
515
  this.#shutdown.signal.addEventListener('abort', abortShutdown, { once: true });
265
516
  if (externalSignal?.aborted) abortExternal();
266
- const task = this.#prepare(plan, expertIds, controller);
517
+ const profileEnabled = getRuntimeConfig().shared.debug.profiler.enabled;
518
+ const before = profileEnabled ? { ...this.#stats } : null;
519
+ const started = profileEnabled ? performance.now() : 0;
520
+ if (profileEnabled) {
521
+ this.#activeProfile = this.#phaseProfiles[phase ?? 'unspecified'] ??= {
522
+ preparations: 0, failures: 0, prepareWallMs: 0, hostLookupMs: 0, synchronizationWaitMs: 0,
523
+ peakGpuResidentBytes: 0, peakHostOwnedOrReservedBytes: 0, peakStagingBytes: 0, counters: {},
524
+ };
525
+ this.#activeProfile.preparations++;
526
+ this.#sampleProfileMemory();
527
+ }
528
+ let succeeded = false;
529
+ const task = this.#prepare(plan, expertIds, controller, phase).then(result => { succeeded = true; return result; });
267
530
  this.#active = task;
268
531
  return task.finally(() => {
532
+ const profile = this.#activeProfile;
533
+ if (profile && before) {
534
+ profile.prepareWallMs += performance.now() - started;
535
+ if (!succeeded) profile.failures++;
536
+ const current = /** @type {Record<string, number>} */ (this.#stats);
537
+ for (const [key, value] of Object.entries(before)) {
538
+ // Peaks and gauges are not additive event counters.
539
+ if (key.startsWith('peak') || key === 'residentBytes') continue;
540
+ profile.counters[key] = (profile.counters[key] ?? 0) + current[key] - value;
541
+ }
542
+ }
543
+ this.#activeProfile = null;
269
544
  externalSignal?.removeEventListener('abort', abortExternal);
270
545
  this.#shutdown.signal.removeEventListener('abort', abortShutdown);
271
546
  this.#active = null;
@@ -274,10 +549,18 @@ export class RoutedExpertCache {
274
549
 
275
550
  /** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
276
551
  * @param {number[]} expertIds @param {AbortController} controller
552
+ * @param {'prefill' | 'decode' | undefined} phase
277
553
  * @returns {Promise<import('./routed-expert-cache.js').RoutedExpertWeights>} */
278
- async #prepare(plan, expertIds, controller) {
554
+ async #prepare(plan, expertIds, controller, phase) {
279
555
  const signal = controller.signal;
280
556
  signal.throwIfAborted();
557
+ if ((this.#options.policy.evictionPolicy === 'lfu') !== (this.#frequency !== null)) {
558
+ throw new Error('Changing routed eviction policy requires recreating the cache.');
559
+ }
560
+ if (this.#frequency && (!Number.isSafeInteger(plan.layer) || plan.layer < 0
561
+ || plan.layer >= this.#numLayers || plan.numExperts !== this.#numExperts)) {
562
+ throw new Error('LFU plan exceeds the declared model layer/expert bounds.');
563
+ }
281
564
  const ids = [...new Set(expertIds)];
282
565
  if (!ids.length || ids.some(id => !Number.isSafeInteger(id) || id < 0 || id >= plan.numExperts)) {
283
566
  throw new Error('Routed expert request contains invalid or absent selected expert IDs.');
@@ -289,47 +572,91 @@ export class RoutedExpertCache {
289
572
  && plan.expertBytes * this.#options.policy.readConcurrency > this.#options.policy.hostCacheBytes) {
290
573
  throw new Error('Routed host cache must hold one payload per concurrent reader.');
291
574
  }
575
+ assertExpertReadShape(this.#options, plan);
576
+ // Policy changes cannot leave old scratch charged alongside a different path.
577
+ const scratchLimit = Math.floor((this.#options.policy.stagingBytes - (this.#options.rangeLease?.memoryBytes ?? 0)) / (2 * this.#options.policy.readConcurrency * 4)) * 4;
578
+ if (!this.#options.readRangeInto
579
+ || this.#readerStaging.length > this.#options.policy.readConcurrency
580
+ || this.#readerStaging.some(bytes => bytes.byteLength > scratchLimit)) this.#clearReaderStaging();
581
+ if (this.#options.policy.storageMode === 'compact-decode') {
582
+ if (phase !== 'prefill' && phase !== 'decode') throw new Error('Compact experts require an explicit execution phase.');
583
+ if (phase === 'decode') {
584
+ this.#clearReaderStaging();
585
+ await this.#waitForSubmitted();
586
+ signal.throwIfAborted();
587
+ if (this.#owned.size) {
588
+ for (const buffer of this.#owned) this.#destroyBuffer(buffer);
589
+ this.#entries.clear(); this.#working.clear(); this.#loaded.clear();
590
+ this.#cacheBytes = 0; this.#workingBytes = 0; this.#signature = ''; this.#layer = -1;
591
+ }
592
+ if (!this.#compact) {
593
+ const compactLayerCount = this.#options.compactLayerCount;
594
+ if (!Number.isSafeInteger(compactLayerCount) || !compactLayerCount) throw new Error('Compact experts require declared layer count.');
595
+ this.#compact = new CompactExpertCache({ ...this.#options, compactLayerCount }, this.#stats);
596
+ }
597
+ const weights = await this.#compact.prepare(plan, ids, signal);
598
+ this.#sampleProfileMemory();
599
+ return weights;
600
+ }
601
+ this.#compact?.clearStaging();
602
+ if (this.#options.policy.compactPrefillBytes > 0) {
603
+ const payloadBytes = this.#options.policy.prefillUploadMode === 'direct-bank' ? 0 : plan.expertBytes * this.#options.policy.readConcurrency;
604
+ if (plan.bankBytes + payloadBytes > this.#options.policy.compactPrefillBytes) {
605
+ throw new Error('Compact prefill reservation cannot hold the working bank and concurrent expert payloads.');
606
+ }
607
+ } else if (this.#compact) { await this.#compact.destroy(); this.#compact = null; }
608
+ }
292
609
  await this.#locked(() => this.#prepareWorking(plan, signal));
293
610
  let cursor = 0;
294
611
  let readyAt = 0, submitted = false;
295
612
  // Only the first proven queue-idle gap is counted. Uploads and later gaps
296
613
  // are excluded; this is a lower bound, not global hardware utilization.
297
- const fence = this.#options.device.queue.onSubmittedWorkDone().then(() => { if (!submitted) readyAt = performance.now(); });
614
+ const fence = this.#waitForSubmitted().then(() => { if (!submitted) readyAt = performance.now(); });
298
615
  fence.catch(() => undefined);
299
616
  const beforeSubmit = () => {
300
617
  if (!submitted && readyAt) this.#stats.initialQueueIdleForWeightsMs += performance.now() - readyAt;
301
618
  submitted = true;
302
619
  };
303
- const work = async () => {
620
+ const hitBatch = this.#copyCachedHits(plan, ids, signal, beforeSubmit);
621
+ /** @param {number} reader */
622
+ const work = async (reader) => {
623
+ const copiedHits = await hitBatch;
304
624
  while (cursor < ids.length) {
305
625
  signal.throwIfAborted();
306
626
  const expert = ids[cursor++];
307
- if (this.#loaded.has(expert)) { this.#stats.workingHits++; continue; }
308
- const entry = await this.#get(plan, expert, signal, beforeSubmit);
627
+ if (copiedHits.has(expert)) continue;
628
+ if (this.#loaded.has(expert)) {
629
+ this.#frequency?.observe(`${plan.layer}:${expert}`);
630
+ this.#stats.workingHits++;
631
+ continue;
632
+ }
633
+ if (phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank') {
634
+ await this.#writeWorking(plan, expert, signal, beforeSubmit, reader);
635
+ continue;
636
+ }
637
+ const entry = await this.#get(plan, expert, signal, beforeSubmit, phase, reader);
309
638
  try {
310
639
  signal.throwIfAborted();
311
640
  const encoder = this.#options.device.createCommandEncoder({ label: 'routed_expert_copy' });
312
- let offset = 0;
313
- for (const plane of plan.planes) {
314
- const destination = this.#working.get(plane.role);
315
- if (!destination) throw new Error('Routed working bank is incomplete.');
316
- encoder.copyBufferToBuffer(entry.buffer, offset, destination, expert * plane.expertBytes, plane.expertBytes);
317
- offset += plane.expertBytes;
318
- }
641
+ this.#encodeExpertCopy(encoder, plan, expert, entry);
319
642
  beforeSubmit();
643
+ entry.lastUseSerial = ++this.#submittedSerial;
320
644
  this.#options.device.queue.submit([encoder.finish()]);
645
+ this.#stats.expertCopySubmissions++;
321
646
  this.#loaded.add(expert);
322
647
  } finally { entry.users--; }
323
648
  }
324
649
  };
325
- const outcomes = await Promise.allSettled(Array.from({ length: this.#options.policy.readConcurrency }, async () => {
326
- try { await work(); } catch (error) { controller.abort(error); throw error; }
650
+ const laneCount = this.#options.rangeLease && phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank'
651
+ ? Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease.concurrency ?? 1) : this.#options.policy.readConcurrency;
652
+ const outcomes = await Promise.allSettled(Array.from({ length: laneCount }, async (_, reader) => {
653
+ try { await work(reader); } catch (error) { controller.abort(error); throw error; }
327
654
  }));
328
655
  const errors = [...new Set(outcomes.filter(r => r.status === 'rejected').map(r => r.reason))];
329
656
  try { await fence; } catch (error) { errors.push(error); }
330
657
  if (errors.length || signal.aborted) {
331
658
  this.#loaded.clear();
332
- try { await this.#options.device.queue.onSubmittedWorkDone(); } catch (error) { errors.push(error); }
659
+ try { await this.#waitForSubmitted(); } catch (error) { errors.push(error); }
333
660
  if (errors.length > 1) throw new AggregateError(errors, 'Routed expert preparation failed.');
334
661
  if (errors.length) throw errors[0];
335
662
  signal.throwIfAborted();
@@ -337,9 +664,20 @@ export class RoutedExpertCache {
337
664
  const weights = Object.fromEntries(plan.planes.map(plane => {
338
665
  const buffer = this.#working.get(plane.role);
339
666
  if (!buffer) throw new Error('Routed working bank is incomplete.');
340
- return [plane.role, createWeightBuffer(buffer, 'q4k', 'row', plane.location.shape, plane.name)];
667
+ return [plane.role, createWeightBuffer(buffer, plane.dtype, 'row', plane.location.shape, plane.name)];
341
668
  }));
342
669
  if (!weights.down) throw new Error('Routed working bank has no down projection.');
670
+ if (phase === 'prefill' && this.#options.policy.prefillCompactAdmission === 'selected-frequency-first'
671
+ && !this.#compact?.hasLayer(plan.layer)) {
672
+ const compactLayerCount = this.#options.compactLayerCount;
673
+ if (!Number.isSafeInteger(compactLayerCount) || !compactLayerCount) throw new Error('Prefill seed requires assigned layer count.');
674
+ const options = { ...this.#options, compactLayerCount };
675
+ if (!this.#compact) this.#compact = new CompactExpertCache(options, this.#stats);
676
+ const selected = selectPrefillSeedExperts(expertIds, compactExpertCapacity(plan, options));
677
+ // GPU copies use already authenticated bytes and preserve original expert IDs.
678
+ this.#clearReaderStaging();
679
+ await this.#compact.prepare(plan, selected, signal, this.#working);
680
+ }
343
681
  return { expertIdx: 0, expertFormat: 'qwen3_5', numExperts: plan.numExperts,
344
682
  expertIntermediateSize: plan.intermediateSize, ...weights, down: weights.down };
345
683
  }
@@ -347,24 +685,38 @@ export class RoutedExpertCache {
347
685
  getStats() {
348
686
  this.#stats.peakAccountedGpuBytes = Math.max(this.#stats.peakAccountedGpuBytes,
349
687
  this.#options.otherGpuBytes() + this.#stats.residentBytes);
350
- return { ...this.#stats, cacheBytes: this.#cacheBytes, workingBytes: this.#workingBytes, experts: this.#entries.size,
351
- hostCacheBytes: this.#hostBytes, hostReservedBytes: this.#hostReserved, hostExperts: this.#hostEntries.size };
688
+ const compact = this.#compact?.getStats();
689
+ return { ...this.#stats,
690
+ evictionPolicy: this.#options.policy.evictionPolicy,
691
+ frequencyHistoryEntries: this.#frequency?.historySize ?? 0,
692
+ frequencyResidentEntries: this.#frequency?.residentSize ?? 0,
693
+ phaseProfiles: Object.fromEntries(Object.entries(this.#phaseProfiles).map(([phase, profile]) =>
694
+ [phase, { ...profile, counters: { ...profile.counters } }])),
695
+ experts: this.#entries.size,
696
+ hostCacheBytes: this.#hostBytes, hostReservedBytes: this.#hostReserved, hostExperts: this.#hostEntries.size,
697
+ ...compact,
698
+ cacheBytes: this.#cacheBytes + (compact?.cacheBytes ?? 0),
699
+ workingBytes: this.#workingBytes };
352
700
  }
353
701
 
354
702
  async destroy() {
355
703
  this.#closed = true;
356
704
  this.#shutdown.abort(new DOMException('Routed expert cache unloaded.', 'AbortError'));
357
705
  await this.#active?.catch(() => undefined);
706
+ this.#clearReaderStaging();
358
707
  this.#hostEntries.clear(); this.#hostSeen.clear(); this.#hostBytes = 0; this.#hostReserved = 0;
359
708
  const errors = [];
709
+ if (this.#compact) {
710
+ try { await this.#compact.destroy(); this.#compact = null; } catch (error) { errors.push(error); }
711
+ }
360
712
  if (this.#owned.size) {
361
- try { await this.#options.device.queue.onSubmittedWorkDone(); } catch (error) { errors.push(error); }
713
+ try { await this.#waitForSubmitted(); } catch (error) { errors.push(error); }
362
714
  }
363
715
  for (const buffer of this.#owned) {
364
716
  try { this.#destroyBuffer(buffer); } catch (error) { errors.push(error); }
365
717
  }
366
718
  if (!this.#owned.size) {
367
- this.#entries.clear(); this.#working.clear(); this.#loaded.clear();
719
+ this.#entries.clear(); this.#frequency?.clear(); this.#working.clear(); this.#loaded.clear();
368
720
  this.#cacheBytes = 0; this.#workingBytes = 0;
369
721
  }
370
722
  if (errors.length) throw new AggregateError(errors, 'Routed expert cache cleanup failed.');