doppler-gpu 0.7.3 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (768) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/package.json +5 -48
  3. package/src/capsule-runtime.d.ts +2 -2
  4. package/src/client/capsule-host.browser.js +4 -4
  5. package/src/client/capsule-host.d.ts +8 -8
  6. package/src/client/capsule-host.js +4 -4
  7. package/src/client/doppler-api.d.ts +2 -2
  8. package/src/client/inspection.d.ts +3 -0
  9. package/src/client/inspection.js +29 -0
  10. package/src/client/model-host/index.d.ts +5 -4
  11. package/src/client/model-host/index.js +28 -8
  12. package/src/client/model-host/model-evidence.js +2 -0
  13. package/src/client/model-host/model-session.d.ts +7 -5
  14. package/src/client/model-host/model-session.js +10 -3
  15. package/src/client/model-host/privacy-receipt.js +22 -0
  16. package/src/client/resident-partitions.d.ts +3 -3
  17. package/src/client/resident-partitions.js +2 -2
  18. package/src/client/runtime/capsule-acquisition.d.ts +3 -3
  19. package/src/client/runtime/capsule-acquisition.js +1 -1
  20. package/src/client/runtime/capsule-adapter-execution.d.ts +2 -2
  21. package/src/client/runtime/capsule-adapter-execution.js +3 -3
  22. package/src/client/runtime/capsule-embedding.d.ts +2 -2
  23. package/src/client/runtime/capsule-embedding.js +1 -1
  24. package/src/client/runtime/capsule-forecast.d.ts +2 -2
  25. package/src/client/runtime/capsule-forecast.js +1 -1
  26. package/src/client/runtime/capsule-operation-adapters.d.ts +1 -1
  27. package/src/client/runtime/capsule-operation-adapters.js +1 -1
  28. package/src/client/runtime/capsule-operation-deltas.js +1 -1
  29. package/src/client/runtime/capsule-operation-executor.d.ts +1 -1
  30. package/src/client/runtime/capsule-operation-executor.js +2 -2
  31. package/src/client/runtime/capsule-operation-stream.d.ts +1 -1
  32. package/src/client/runtime/capsule-operation-stream.js +2 -2
  33. package/src/client/runtime/capsule-piece-reader.js +1 -1
  34. package/src/client/runtime/capsule-program-adapter.js +1 -1
  35. package/src/client/runtime/capsule-release-authorization.js +1 -1
  36. package/src/client/runtime/capsule-rerank.d.ts +2 -2
  37. package/src/client/runtime/composition-root.d.ts +6 -6
  38. package/src/client/runtime/composition-root.js +6 -6
  39. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +1 -1
  40. package/src/client/runtime/fetch-capsule-artifact-store.js +1 -1
  41. package/src/client/runtime/model-source.d.ts +4 -2
  42. package/src/client/runtime/model-source.js +5 -5
  43. package/src/client/runtime/scoped-session.d.ts +2 -2
  44. package/src/client/runtime/verified-capsule-artifact-store.d.ts +2 -2
  45. package/src/client/runtime/verified-capsule-artifact-store.js +1 -1
  46. package/src/config/backward-registry-loader.js +1 -1
  47. package/src/config/{capsule-adapter-policy.js → capsule/capsule-adapter-policy.js} +2 -2
  48. package/src/config/{capsule-adapters.d.ts → capsule/capsule-adapters.d.ts} +1 -1
  49. package/src/config/{capsule-adapters.js → capsule/capsule-adapters.js} +2 -2
  50. package/src/config/{capsule-loading.js → capsule/capsule-loading.js} +1 -1
  51. package/src/config/{capsule-operation.d.ts → capsule/capsule-operation.d.ts} +1 -1
  52. package/src/config/{capsule-operation.js → capsule/capsule-operation.js} +3 -3
  53. package/src/config/{capsule-release-contract.js → capsule/capsule-release-contract.js} +2 -2
  54. package/src/config/{capsule-release-events.d.ts → capsule/capsule-release-events.d.ts} +2 -1
  55. package/src/config/{capsule-release-events.js → capsule/capsule-release-events.js} +3 -2
  56. package/src/config/{capsule-sequence-receipt.js → capsule/capsule-sequence-receipt.js} +1 -1
  57. package/src/config/{capsule-token-selection.d.ts → capsule/capsule-token-selection.d.ts} +1 -1
  58. package/src/config/{capsule-token-selection.js → capsule/capsule-token-selection.js} +2 -2
  59. package/src/config/{capsule-v2.d.ts → capsule/capsule-v2.d.ts} +5 -4
  60. package/src/config/{capsule-v2.js → capsule/capsule-v2.js} +6 -6
  61. package/src/config/{capsule-v3.js → capsule/capsule-v3.js} +1 -1
  62. package/src/config/{capsule.js → capsule/capsule.js} +26 -7
  63. package/src/config/choice-scoring.js +1 -1
  64. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  65. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +1 -1
  66. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  67. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +2 -2
  68. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  69. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +1 -1
  70. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  71. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  72. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  73. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  74. package/src/config/conversion/gemma3/gemma-3-270m-it-w8a8-ehf16-af32.json +1 -1
  75. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  76. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  77. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +8 -8
  78. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +8 -8
  79. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +8 -8
  80. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +8 -8
  81. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +8 -8
  82. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +8 -8
  83. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +8 -8
  84. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +8 -8
  85. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +8 -8
  86. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +1 -1
  87. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  88. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +1 -1
  89. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +1 -1
  90. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +1 -1
  91. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  92. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  93. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  94. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  95. package/src/config/conversion/qwen3/qwen-3-4b-instruct-2507-q4k-ehf16-af32.json +2 -2
  96. package/src/config/conversion/qwen3/qwen-3-5-0-8b-canonical-f32.json +1 -1
  97. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  98. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  99. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  100. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  101. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +1 -1
  102. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  103. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-q4_0-passthrough.json +1 -1
  104. package/src/config/conversion/qwen3/qwen-3-6-35b-a3b-text-q4k-ehf16-af32.json +1 -1
  105. package/src/config/conversion/qwen3/qwen-3-8-flash-next-text-q4k-ehf16-af32.json +25 -22
  106. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +2 -2
  107. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  108. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  109. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +2 -2
  110. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +2 -2
  111. package/src/config/conversion/qwen3/qwen3.8-27b-q4_0-passthrough.json +1 -1
  112. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +1 -1
  113. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +1 -1
  114. package/src/config/conversion/qwen3/qwen4exp-gguf-source-policy.json +53 -0
  115. package/src/config/embedding-contract.d.ts +1 -1
  116. package/src/config/embedding-contract.js +1 -1
  117. package/src/config/execution-registry-contract.d.ts +5 -5
  118. package/src/config/execution-registry-contract.js +5 -5
  119. package/src/config/generation-contract.js +2 -1
  120. package/src/config/hardware-partition-plan.d.ts +32 -0
  121. package/src/config/hardware-partition-plan.js +78 -0
  122. package/src/config/inspection/observation-policies.json +1 -1
  123. package/src/config/kernel-path-loader.d.ts +1 -1
  124. package/src/config/kernel-path-loader.js +1 -1
  125. package/src/config/kernel-registry-contract.d.ts +2 -2
  126. package/src/config/kernel-registry-contract.js +1 -1
  127. package/src/config/kernels/kernel-ref-digests.js +508 -0
  128. package/src/config/kernels/registry.js +2 -2
  129. package/src/config/kernels/registry.json +1892 -1096
  130. package/src/config/layer-partition-plan.d.ts +4 -0
  131. package/src/config/layer-partition-plan.js +10 -4
  132. package/src/config/merge.js +1 -1
  133. package/src/config/model-session-declarations.js +85 -0
  134. package/src/config/param-validator.js +3 -3
  135. package/src/config/platforms/amd-rdna3.json +2 -2
  136. package/src/config/production-release-evidence.js +1 -1
  137. package/src/config/revocation-policy.js +6 -2
  138. package/src/config/revocation-registry.json +1 -1
  139. package/src/config/runtime/model-sessions.json +23 -0
  140. package/src/config/runtime/profiles/flash-next-gguf-control-gemv.json +72 -0
  141. package/src/config/runtime/profiles/flash-next-gguf-hc-inject-gemv.json +47 -0
  142. package/src/config/schema/capsule/index.d.ts +14 -0
  143. package/src/config/schema/capsule/index.js +10 -0
  144. package/src/config/schema/{program-bundle.schema.d.ts → capsule/program-bundle.schema.d.ts} +3 -3
  145. package/src/config/schema/{program-bundle.schema.js → capsule/program-bundle.schema.js} +6 -6
  146. package/src/config/schema/{execution-v1.schema.d.ts → execution/execution-v1.schema.d.ts} +2 -0
  147. package/src/config/schema/{execution-v1.schema.js → execution/execution-v1.schema.js} +2 -2
  148. package/src/config/schema/execution/index.d.ts +203 -0
  149. package/src/config/schema/execution/index.js +94 -0
  150. package/src/config/schema/{inference.schema.d.ts → execution/inference.schema.d.ts} +1 -1
  151. package/src/config/schema/{kernel-path.schema.d.ts → execution/kernel-path.schema.d.ts} +3 -0
  152. package/src/config/schema/{kernel-registry.schema.d.ts → execution/kernel-registry.schema.d.ts} +2 -2
  153. package/src/config/schema/{kernel-thresholds.schema.d.ts → execution/kernel-thresholds.schema.d.ts} +1 -1
  154. package/src/config/schema/{kernel-thresholds.schema.js → execution/kernel-thresholds.schema.js} +1 -1
  155. package/src/config/schema/{kvcache.schema.d.ts → execution/kvcache.schema.d.ts} +8 -2
  156. package/src/config/schema/{kvcache.schema.js → execution/kvcache.schema.js} +17 -1
  157. package/src/config/schema/{ngram-embedding.schema.d.ts → execution/ngram-embedding.schema.d.ts} +3 -1
  158. package/src/config/schema/{ngram-embedding.schema.js → execution/ngram-embedding.schema.js} +16 -1
  159. package/src/config/schema/index.d.ts +8 -897
  160. package/src/config/schema/index.js +8 -506
  161. package/src/config/schema/{converter.schema.d.ts → manifest/converter.schema.d.ts} +4 -4
  162. package/src/config/schema/{converter.schema.js → manifest/converter.schema.js} +1 -1
  163. package/src/config/schema/manifest/index.d.ts +142 -0
  164. package/src/config/schema/manifest/index.js +62 -0
  165. package/src/config/schema/manifest/manifest-inference.schema.d.ts +298 -0
  166. package/src/config/schema/{manifest.schema.d.ts → manifest/manifest.schema.d.ts} +9 -291
  167. package/src/config/schema/{manifest.schema.js → manifest/manifest.schema.js} +2 -2
  168. package/src/config/schema/{emulation.schema.d.ts → platform/emulation.schema.d.ts} +1 -1
  169. package/src/config/schema/{emulation.schema.js → platform/emulation.schema.js} +6 -6
  170. package/src/config/schema/platform/index.d.ts +72 -0
  171. package/src/config/schema/platform/index.js +27 -0
  172. package/src/config/schema/{memory-limits.schema.js → platform/memory-limits.schema.js} +1 -1
  173. package/src/config/schema/policy/index.d.ts +14 -0
  174. package/src/config/schema/policy/index.js +9 -0
  175. package/src/config/schema/{doppler.schema.d.ts → runtime/doppler.schema.d.ts} +3 -3
  176. package/src/config/schema/{doppler.schema.js → runtime/doppler.schema.js} +12 -12
  177. package/src/config/schema/runtime/index.d.ts +238 -0
  178. package/src/config/schema/runtime/index.js +135 -0
  179. package/src/config/schema/{loading.schema.d.ts → runtime/loading.schema.d.ts} +26 -1
  180. package/src/config/schema/{loading.schema.js → runtime/loading.schema.js} +88 -0
  181. package/src/config/schema/{shared-runtime.schema.d.ts → runtime/shared-runtime.schema.d.ts} +5 -5
  182. package/src/config/schema/{shared-runtime.schema.js → runtime/shared-runtime.schema.js} +4 -4
  183. package/src/config/schema/{tooling.schema.d.ts → runtime/tooling.schema.d.ts} +1 -1
  184. package/src/config/schema/training/index.d.ts +63 -0
  185. package/src/config/schema/training/index.js +43 -0
  186. package/src/config/schema/{lora.schema.js → training/lora.schema.js} +1 -1
  187. package/src/config/source-packages/litert/gemma-4-e2b-it.json +1 -1
  188. package/src/config/supported-operations.json +358 -88
  189. package/src/config/target-plan.d.ts +2 -2
  190. package/src/config/target-plan.js +2 -2
  191. package/src/config/training-defaults.d.ts +2 -2
  192. package/src/config/training-defaults.js +2 -2
  193. package/src/config/transforms/execution-graph/correctness-fallback.js +86 -0
  194. package/src/config/transforms/execution-graph/prefill-attention.js +116 -0
  195. package/src/{converter → config/transforms}/execution-graph-transforms.js +5 -194
  196. package/src/converter/core.js +18 -164
  197. package/src/converter/forge-stages.js +2 -2
  198. package/src/converter/fp8-source.js +1 -1
  199. package/src/converter/gemma4/validation.js +110 -0
  200. package/src/converter/linear-norm-gate.js +1 -1
  201. package/src/converter/manifest-builder.js +2 -9
  202. package/src/converter/parsers/gguf.js +54 -4
  203. package/src/converter/quantization-info.js +2 -2
  204. package/src/converter/qwen4-exp-source-execution.js +94 -0
  205. package/src/converter/qwen4-exp-source.js +1 -201
  206. package/src/converter/tensor-transform.js +2 -9
  207. package/src/converter/tokenizer-contract.js +2 -9
  208. package/src/converter/w8a8-quantizer.js +2 -2
  209. package/src/debug/config.js +1 -1
  210. package/src/debug/perf-guards.js +1 -1
  211. package/src/errors/doppler-error.d.ts +54 -0
  212. package/src/errors/doppler-error.js +13 -5
  213. package/src/experimental/distribution/shard-delivery/receipt.js +1 -1
  214. package/src/experimental/distribution/shard-delivery/retry.js +4 -4
  215. package/src/experimental/energy/types.d.ts +1 -1
  216. package/src/experimental/logits/cpu-output.js +1 -1
  217. package/src/experimental/training/artifacts.d.ts +2 -2
  218. package/src/experimental/training/artifacts.js +2 -2
  219. package/src/experimental/training/autograd.d.ts +1 -1
  220. package/src/experimental/training/distillation/student-fixture/execution.js +1 -1
  221. package/src/experimental/training/distillation/student-fixture/model.js +1 -1
  222. package/src/experimental/training/distillation/student-fixture.js +1 -1
  223. package/src/experimental/training/loss-scaling.d.ts +1 -1
  224. package/src/experimental/training/runner/evaluation.js +1 -1
  225. package/src/experimental/training/runner.js +1 -1
  226. package/src/experimental/training/trainer.d.ts +1 -1
  227. package/src/experimental/training/workloads.js +2 -2
  228. package/src/formats/gguf/block-storage.d.ts +9 -0
  229. package/src/formats/gguf/block-storage.js +12 -3
  230. package/src/formats/gguf/qwen-source.js +5 -3
  231. package/src/formats/gguf/qwen-source.json +8 -1
  232. package/src/formats/gguf/qwen4-exp-source.js +101 -0
  233. package/src/formats/q4k-dequant.js +1 -1
  234. package/src/formats/qwen4-exp-source-contracts.js +204 -0
  235. package/src/formats/rdrr/fp8-contract.js +1 -1
  236. package/src/formats/rdrr/fp8-storage.js +1 -1
  237. package/src/formats/rdrr/q4_0-storage.js +1 -1
  238. package/src/formats/rdrr/storage-descriptor.js +1 -1
  239. package/src/formats/rdrr/tensor-config-validator.js +3 -3
  240. package/src/formats/rdrr/validation.js +1 -1
  241. package/src/formats/rdrr/w8a8-storage.js +1 -1
  242. package/src/generation/index.d.ts +1 -1
  243. package/src/generation/index.js +1 -0
  244. package/src/gpu/device.d.ts +1 -1
  245. package/src/gpu/kernels/attention/contiguous-quantized.js +252 -0
  246. package/src/gpu/kernels/attention/tiered-quantized.js +321 -0
  247. package/src/gpu/kernels/attention/tiered.js +240 -0
  248. package/src/gpu/kernels/attention.js +5 -801
  249. package/src/gpu/kernels/backward/layernorm_backward.wgsl +3 -3
  250. package/src/gpu/kernels/channel-gain.js +100 -0
  251. package/src/gpu/kernels/channel-rotation.d.ts +16 -0
  252. package/src/gpu/kernels/channel-rotation.js +47 -0
  253. package/src/gpu/kernels/channel_energy.wgsl +17 -0
  254. package/src/gpu/kernels/channel_energy_pair.wgsl +27 -0
  255. package/src/gpu/kernels/channel_gain_control.wgsl +60 -0
  256. package/src/gpu/kernels/channel_gain_derive.wgsl +40 -0
  257. package/src/gpu/kernels/channel_gain_swiglu.wgsl +64 -0
  258. package/src/gpu/kernels/channel_rotation_apply.wgsl +38 -0
  259. package/src/gpu/kernels/channel_rotation_derive.wgsl +76 -0
  260. package/src/gpu/kernels/check-finiteness-f32.js +22 -0
  261. package/src/gpu/kernels/check_finiteness.wgsl +2 -2
  262. package/src/gpu/kernels/clamp.js +2 -2
  263. package/src/gpu/kernels/dequant.js +1 -1
  264. package/src/gpu/kernels/dispatch.js +7 -0
  265. package/src/gpu/kernels/energy_quintel_grad.wgsl +7 -7
  266. package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +7 -7
  267. package/src/gpu/kernels/energy_quintel_reduce.wgsl +6 -6
  268. package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +6 -6
  269. package/src/gpu/kernels/energy_quintel_update.wgsl +7 -7
  270. package/src/gpu/kernels/energy_quintel_update_f16.wgsl +7 -7
  271. package/src/gpu/kernels/fused_matmul_q4_widetile_f32.wgsl +174 -0
  272. package/src/gpu/kernels/fused_matmul_rmsnorm.wgsl +0 -49
  273. package/src/gpu/kernels/gated-delta-norm-gate.js +1 -1
  274. package/src/gpu/kernels/gated_delta_recurrent_gguf_register_prefill_sigmoid_gate.wgsl +730 -0
  275. package/src/gpu/kernels/gated_delta_recurrent_gguf_sigmoid_gate.wgsl +499 -0
  276. package/src/gpu/kernels/gather.js +16 -1
  277. package/src/gpu/kernels/gather_split4_packed_f16_vec4_f32_out.wgsl +104 -0
  278. package/src/gpu/kernels/generated/uniform-writers-0.js +7 -11
  279. package/src/gpu/kernels/generated/uniform-writers-1.js +43 -46
  280. package/src/gpu/kernels/generated/uniform-writers-2.js +80 -2
  281. package/src/gpu/kernels/generated/uniform-writers.js +33 -21
  282. package/src/gpu/kernels/hyper-connection-up.js +64 -0
  283. package/src/gpu/kernels/hyper-connection.js +15 -4
  284. package/src/gpu/kernels/hyper_connection_mix_branch.wgsl +105 -0
  285. package/src/gpu/kernels/hyper_connection_norm.wgsl +3 -3
  286. package/src/gpu/kernels/hyper_connection_up_mix_branch_q4_0.wgsl +164 -0
  287. package/src/gpu/kernels/kernel-execution.js +18 -7
  288. package/src/gpu/kernels/linear-attention-core.js +3 -4
  289. package/src/gpu/kernels/moe-route.js +14 -0
  290. package/src/gpu/kernels/moe_gather.wgsl +0 -63
  291. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +63 -0
  292. package/src/gpu/kernels/moe_route_q4_1_block_f32.wgsl +59 -0
  293. package/src/gpu/kernels/moe_route_remap.wgsl +22 -0
  294. package/src/gpu/kernels/paired-linear-attention.js +1 -1
  295. package/src/gpu/kernels/pipeline-cache.js +18 -2
  296. package/src/gpu/kernels/ple.js +11 -7
  297. package/src/gpu/kernels/ple_q4_0_to_f32.wgsl +32 -0
  298. package/src/gpu/kernels/qsa-indexer.d.ts +2 -0
  299. package/src/gpu/kernels/qsa-indexer.js +6 -4
  300. package/src/gpu/kernels/qsa_block_keys.wgsl +2 -1
  301. package/src/gpu/kernels/qsa_prepare.wgsl +2 -1
  302. package/src/gpu/kernels/relu.js +1 -1
  303. package/src/gpu/kernels/relu.wgsl +2 -2
  304. package/src/gpu/kernels/relu_f16.wgsl +2 -2
  305. package/src/gpu/kernels/residual.js +1 -1
  306. package/src/gpu/kernels/residual.wgsl +4 -4
  307. package/src/gpu/kernels/residual_f16.wgsl +2 -2
  308. package/src/gpu/kernels/residual_f16_vec4.wgsl +2 -2
  309. package/src/gpu/kernels/residual_vec4.wgsl +2 -2
  310. package/src/gpu/kernels/rope-precompute.js +1 -1
  311. package/src/gpu/kernels/scatter_add.wgsl +0 -31
  312. package/src/gpu/kernels/shader-cache.js +17 -3
  313. package/src/gpu/kernels/shader-capability-filter.js +120 -0
  314. package/src/gpu/kernels/softmax.js +10 -10
  315. package/src/gpu/kernels/topk.wgsl +0 -55
  316. package/src/gpu/kernels/transpose.js +1 -1
  317. package/src/gpu/kernels/transpose.wgsl +2 -2
  318. package/src/gpu/kernels/w8a8.js +1 -1
  319. package/src/gpu/uniform-cache.d.ts +15 -1
  320. package/src/gpu/uniform-cache.js +82 -7
  321. package/src/gpu/weight-buffer.d.ts +1 -1
  322. package/src/inference/moe-router.d.ts +1 -126
  323. package/src/inference/moe-router.js +0 -149
  324. package/src/inference/pipelines/context.js +3 -15
  325. package/src/inference/pipelines/shader-scoped-pipeline.js +10 -5
  326. package/src/inference/pipelines/text/attention/interpreter/immediate.js +4 -4
  327. package/src/inference/pipelines/text/attention/interpreter/output.js +203 -0
  328. package/src/inference/pipelines/text/attention/interpreter/recorded.js +92 -67
  329. package/src/inference/pipelines/text/attention/interpreter.js +77 -350
  330. package/src/inference/pipelines/text/attention/projection-executor.js +2 -2
  331. package/src/inference/pipelines/text/attention/projection-plan.js +1 -1
  332. package/src/inference/pipelines/text/attention/projection-precision.js +132 -0
  333. package/src/inference/pipelines/text/attention/projections.js +13 -151
  334. package/src/inference/pipelines/text/attention/recorded-health.js +83 -0
  335. package/src/inference/pipelines/text/attention/rope-observation.js +1 -1
  336. package/src/inference/pipelines/text/attention/types.js +0 -13
  337. package/src/inference/pipelines/text/config/normalization.js +2 -2
  338. package/src/inference/pipelines/text/config.d.ts +4 -4
  339. package/src/inference/pipelines/text/config.js +17 -135
  340. package/src/inference/pipelines/text/{config-components.js → configuration/config-components.js} +4 -4
  341. package/src/inference/pipelines/text/configuration/model-geometry.js +114 -0
  342. package/src/inference/pipelines/text/{resolved-runtime-session.d.ts → configuration/resolved-runtime-session.d.ts} +4 -4
  343. package/src/inference/pipelines/text/{resolved-runtime-session.js → configuration/resolved-runtime-session.js} +1 -1
  344. package/src/inference/pipelines/text/{abort-contract.js → contracts/abort-contract.js} +2 -1
  345. package/src/inference/pipelines/text/contracts/metrics.d.ts +234 -0
  346. package/src/inference/pipelines/text/{embed.js → embedding/embed.js} +15 -15
  347. package/src/inference/pipelines/text/{embedding-batch.js → embedding/embedding-batch.js} +1 -1
  348. package/src/inference/pipelines/text/{embedding-extraction.js → embedding/embedding-extraction.js} +12 -12
  349. package/src/inference/pipelines/text/{embedding-normalization.js → embedding/embedding-normalization.js} +4 -4
  350. package/src/inference/pipelines/text/{execution-plan.d.ts → execution/execution-plan.d.ts} +8 -8
  351. package/src/inference/pipelines/text/{execution-plan.js → execution/execution-plan.js} +9 -9
  352. package/src/inference/pipelines/text/{execution-runtime-builders.js → execution/execution-runtime-builders.js} +3 -3
  353. package/src/inference/pipelines/text/{execution-v1.d.ts → execution/execution-v1.d.ts} +2 -2
  354. package/src/inference/pipelines/text/{execution-v1.js → execution/execution-v1.js} +13 -183
  355. package/src/inference/pipelines/text/execution/layer-contract.js +90 -0
  356. package/src/inference/pipelines/text/{layer-execution-contract.js → execution/layer-execution-contract.js} +2 -2
  357. package/src/inference/pipelines/text/{layer-input-execution.js → execution/layer-input-execution.js} +27 -36
  358. package/src/inference/pipelines/text/{layer-partition-contract.d.ts → execution/layer-partition-contract.d.ts} +3 -3
  359. package/src/inference/pipelines/text/{layer-partition-contract.js → execution/layer-partition-contract.js} +2 -2
  360. package/src/inference/pipelines/text/{layer-plan-gpu.js → execution/layer-plan-gpu.js} +12 -12
  361. package/src/inference/pipelines/text/{layer-plan.d.ts → execution/layer-plan.d.ts} +1 -1
  362. package/src/inference/pipelines/text/{layer-plan.js → execution/layer-plan.js} +1 -1
  363. package/src/inference/pipelines/text/{layer.js → execution/layer.js} +34 -142
  364. package/src/inference/pipelines/text/{operator-identity.js → execution/operator-identity.js} +1 -1
  365. package/src/inference/pipelines/text/{ops.js → execution/ops.js} +26 -26
  366. package/src/inference/pipelines/text/{paired-execution.js → execution/paired-execution.js} +9 -9
  367. package/src/inference/pipelines/text/execution-v1/hybrid-projection-contract.js +159 -0
  368. package/src/inference/pipelines/text/expert-loading-stats.d.ts +16 -0
  369. package/src/inference/pipelines/text/expert-loading-stats.js +15 -0
  370. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -4
  371. package/src/inference/pipelines/text/ffn/dense-plan.js +23 -34
  372. package/src/inference/pipelines/text/ffn/dense.js +72 -392
  373. package/src/inference/pipelines/text/ffn/fused-gate-up.js +197 -0
  374. package/src/inference/pipelines/text/ffn/moe.js +1 -1
  375. package/src/inference/pipelines/text/ffn/packed-gate-up.js +218 -0
  376. package/src/inference/pipelines/text/ffn/paired.js +3 -3
  377. package/src/inference/pipelines/text/ffn/qwen-shared-expert.js +1 -1
  378. package/src/inference/pipelines/text/ffn/sandwich.js +6 -7
  379. package/src/inference/pipelines/text/ffn/standard.js +5 -5
  380. package/src/inference/pipelines/text/ffn/types.js +6 -3
  381. package/src/inference/pipelines/text/{activity-weights.js → generator/activation/activity-weights.js} +8 -8
  382. package/src/inference/pipelines/text/generator/activation/channel-gain.d.ts +61 -0
  383. package/src/inference/pipelines/text/generator/activation/channel-gain.js +352 -0
  384. package/src/inference/pipelines/text/generator/activation/final-layer-replay.js +43 -0
  385. package/src/inference/pipelines/text/generator/{capsule-generation.js → adapters/capsule-generation.js} +5 -5
  386. package/src/inference/pipelines/text/generator/{diffusion.js → adapters/diffusion.js} +19 -19
  387. package/src/inference/pipelines/text/generator/{training.js → adapters/training.js} +21 -21
  388. package/src/inference/pipelines/text/generator/decode/advance-token.js +73 -0
  389. package/src/inference/pipelines/text/{generator-steps.js → generator/decode/batch-generation.js} +25 -288
  390. package/src/inference/pipelines/text/generator/decode/batch-readback.js +175 -0
  391. package/src/inference/pipelines/text/generator/{compiled-greedy-window.js → decode/compiled-greedy-window.js} +5 -5
  392. package/src/inference/pipelines/text/generator/{decode-components.js → decode/decode-components.js} +1 -1
  393. package/src/inference/pipelines/text/generator/{decode-runtime.js → decode/decode-loop.js} +20 -412
  394. package/src/inference/pipelines/text/generator/decode/decode-mechanics.js +288 -0
  395. package/src/inference/pipelines/text/generator/{decode-readback.js → decode/decode-readback.js} +1 -1
  396. package/src/inference/pipelines/text/generator/decode/decode-runtime.js +2 -0
  397. package/src/inference/pipelines/text/generator/{decode.js → decode/decode-step.js} +34 -313
  398. package/src/inference/pipelines/text/generator/decode/decode.js +5 -0
  399. package/src/inference/pipelines/text/generator/{greedy-head-policy.js → decode/greedy-head-policy.js} +1 -1
  400. package/src/inference/pipelines/text/generator/{recorded-step-executor.js → decode/recorded-step-executor.js} +2 -2
  401. package/src/inference/pipelines/text/generator/generator-steps.js +4 -0
  402. package/src/inference/pipelines/text/{generator.d.ts → generator/generator.d.ts} +13 -11
  403. package/src/inference/pipelines/text/{generator.js → generator/generator.js} +150 -46
  404. package/src/inference/pipelines/text/generator/{prefill-runtime.js → prefill/prefill-execution.js} +72 -278
  405. package/src/inference/pipelines/text/generator/prefill/prefill-logits.js +144 -0
  406. package/src/inference/pipelines/text/generator/{prefill-observation.js → prefill/prefill-observation.js} +2 -2
  407. package/src/inference/pipelines/text/generator/{prefill-policy.js → prefill/prefill-policy.js} +6 -1
  408. package/src/inference/pipelines/text/generator/prefill/prefill-runtime.js +2 -0
  409. package/src/inference/pipelines/text/generator/prefill/prefill-sequence.js +100 -0
  410. package/src/inference/pipelines/text/generator/{prefix-embedding.js → prefill/prefix-embedding.js} +7 -7
  411. package/src/inference/pipelines/text/{generation-request.d.ts → generator/request/generation-request.d.ts} +2 -2
  412. package/src/inference/pipelines/text/{generation-request.js → generator/request/generation-request.js} +3 -3
  413. package/src/inference/pipelines/text/generator/{prompt-input.js → request/prompt-input.js} +2 -2
  414. package/src/inference/pipelines/text/{sampling-config.d.ts → generator/sampling/sampling-config.d.ts} +1 -1
  415. package/src/inference/pipelines/text/{sampling-config.js → generator/sampling/sampling-config.js} +2 -2
  416. package/src/inference/pipelines/text/{sampling.d.ts → generator/sampling/sampling.d.ts} +15 -0
  417. package/src/inference/pipelines/text/{sampling.js → generator/sampling/sampling.js} +22 -3
  418. package/src/inference/pipelines/text/{stopping.js → generator/sampling/stopping.js} +3 -3
  419. package/src/inference/pipelines/text/generator/{token-selection.d.ts → sampling/token-selection.d.ts} +1 -1
  420. package/src/inference/pipelines/text/generator/{token-selection.js → sampling/token-selection.js} +8 -8
  421. package/src/inference/pipelines/text/generator/{activation-storage.js → session/activation-storage.js} +2 -2
  422. package/src/inference/pipelines/text/generator/{attention-lifecycle.js → session/attention-lifecycle.js} +1 -1
  423. package/src/inference/pipelines/text/generator/{diagnostic-sequence.d.ts → session/diagnostic-sequence.d.ts} +3 -3
  424. package/src/inference/pipelines/text/generator/{diagnostic-sequence.js → session/diagnostic-sequence.js} +3 -3
  425. package/src/inference/pipelines/text/generator/session/generation-runtime.js +278 -0
  426. package/src/inference/pipelines/text/{generator-runtime.d.ts → generator/session/generator-runtime.d.ts} +7 -7
  427. package/src/inference/pipelines/text/{generator-runtime.js → generator/session/generator-runtime.js} +8 -8
  428. package/src/inference/pipelines/text/generator/{recovery.js → session/recovery.js} +6 -6
  429. package/src/inference/pipelines/text/{sequence-state.d.ts → generator/session/sequence-state.d.ts} +5 -5
  430. package/src/inference/pipelines/text/{sequence-state.js → generator/session/sequence-state.js} +26 -18
  431. package/src/inference/pipelines/text/generator/{session-context.js → session/session-context.js} +12 -11
  432. package/src/inference/pipelines/text/generator/{speculative-runtime.js → session/speculative-runtime.js} +1 -1
  433. package/src/inference/pipelines/text/generator/{text.js → session/text.js} +9 -9
  434. package/src/inference/pipelines/text/{warm-generation.d.ts → generator/session/warm-generation.d.ts} +1 -1
  435. package/src/inference/pipelines/text/{warm-generation.js → generator/session/warm-generation.js} +1 -1
  436. package/src/inference/pipelines/text/generator/{whole-token-lane.js → whole-token/whole-token-lane.js} +10 -10
  437. package/src/inference/pipelines/text/generator/{whole-token-observation.d.ts → whole-token/whole-token-observation.d.ts} +2 -2
  438. package/src/inference/pipelines/text/generator/{whole-token-observation.js → whole-token/whole-token-observation.js} +5 -5
  439. package/src/inference/pipelines/text/generator/{whole-token-program.d.ts → whole-token/whole-token-program.d.ts} +3 -3
  440. package/src/inference/pipelines/text/generator/{whole-token-program.js → whole-token/whole-token-program.js} +5 -5
  441. package/src/inference/pipelines/text/hyper-connection/decoder-layer.d.ts +17 -4
  442. package/src/inference/pipelines/text/hyper-connection/decoder-layer.js +38 -14
  443. package/src/inference/pipelines/text/hyper-connection/layer-executor.js +14 -9
  444. package/src/inference/pipelines/text/hyper-connection/logits-input.js +5 -3
  445. package/src/inference/pipelines/text/hyper-connection/projection.js +52 -2
  446. package/src/inference/pipelines/text/hyper-connection/sequence-transaction.js +3 -3
  447. package/src/inference/pipelines/text/hyper-connection/session.d.ts +1 -1
  448. package/src/inference/pipelines/text/hyper-connection/session.js +9 -5
  449. package/src/inference/pipelines/text/indexed-attention/qsa-cache.d.ts +2 -1
  450. package/src/inference/pipelines/text/indexed-attention/qsa-cache.js +3 -3
  451. package/src/inference/pipelines/text/init.d.ts +4 -2
  452. package/src/inference/pipelines/text/init.js +10 -934
  453. package/src/inference/pipelines/text/initialization/init-emulation.js +76 -0
  454. package/src/inference/pipelines/text/initialization/init-kv-cache.js +547 -0
  455. package/src/inference/pipelines/text/initialization/init-moe.js +34 -0
  456. package/src/inference/pipelines/text/{init-qkv-fusion.js → initialization/init-qkv-fusion.js} +7 -7
  457. package/src/inference/pipelines/text/{init-rope.d.ts → initialization/init-rope.d.ts} +4 -4
  458. package/src/inference/pipelines/text/{init-rope.js → initialization/init-rope.js} +9 -11
  459. package/src/inference/pipelines/text/initialization/init-speculative.js +31 -0
  460. package/src/inference/pipelines/text/{init-tokenizer.js → initialization/init-tokenizer.js} +2 -2
  461. package/src/inference/pipelines/text/lifecycle.js +18 -18
  462. package/src/inference/pipelines/text/{linear-attention-ab-fusion.js → linear-attention/linear-attention-ab-fusion.js} +3 -3
  463. package/src/inference/pipelines/text/{linear-attention.d.ts → linear-attention/linear-attention.d.ts} +12 -12
  464. package/src/inference/pipelines/text/{linear-attention.js → linear-attention/linear-attention.js} +17 -17
  465. package/src/inference/pipelines/text/linear-attention/plan.js +4 -4
  466. package/src/inference/pipelines/text/linear-attention/state-checkpoint.js +1 -1
  467. package/src/inference/pipelines/text/linear-attention/state-clone.js +1 -1
  468. package/src/inference/pipelines/text/linear-attention/state-layout.js +2 -2
  469. package/src/inference/pipelines/text/loading/decode-session-config.js +295 -0
  470. package/src/inference/pipelines/text/loading/init-weights.js +254 -0
  471. package/src/inference/pipelines/text/{load-errors.js → loading/load-errors.js} +3 -3
  472. package/src/inference/pipelines/text/{model-load.js → loading/model-load.js} +14 -306
  473. package/src/inference/pipelines/text/{weight-loading-config.js → loading/weight-loading-config.js} +6 -6
  474. package/src/inference/pipelines/text/{weights.d.ts → loading/weights.d.ts} +13 -13
  475. package/src/inference/pipelines/text/{weights.js → loading/weights.js} +5 -5
  476. package/src/inference/pipelines/text/logits/gpu-executor.js +2 -2
  477. package/src/inference/pipelines/text/logits/gpu.d.ts +4 -4
  478. package/src/inference/pipelines/text/logits/gpu.js +2 -2
  479. package/src/inference/pipelines/text/{generator → logits}/hidden-projection.js +1 -1
  480. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  481. package/src/inference/pipelines/text/logits/index.js +3 -3
  482. package/src/inference/pipelines/text/{generator → logits}/logits-config.js +1 -1
  483. package/src/inference/pipelines/text/logits/output-transform.js +1 -1
  484. package/src/inference/pipelines/text/logits/precision-policy.js +1 -1
  485. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  486. package/src/inference/pipelines/text/lora-types.d.ts +1 -1
  487. package/src/inference/pipelines/text/moe/expert-loading.js +4 -1
  488. package/src/inference/pipelines/text/moe/gpu-executor.js +44 -101
  489. package/src/inference/pipelines/text/moe/grouped-prefill.js +6 -2
  490. package/src/inference/pipelines/text/moe/qwen-route-executor.d.ts +2 -2
  491. package/src/inference/pipelines/text/moe/qwen-route-executor.js +31 -2
  492. package/src/inference/pipelines/text/moe/routed-expert-chunks.js +99 -15
  493. package/src/inference/pipelines/text/moe-cache.js +11 -77
  494. package/src/inference/pipelines/text/moe-gpu.js +33 -75
  495. package/src/inference/pipelines/text/moe-impl.d.ts +3 -38
  496. package/src/inference/pipelines/text/moe-impl.js +1 -7
  497. package/src/inference/pipelines/text/ngram/ngram-addresses.d.ts +1 -1
  498. package/src/inference/pipelines/text/ngram/ngram-addresses.js +2 -2
  499. package/src/inference/pipelines/text/ngram/ngram-table.d.ts +3 -3
  500. package/src/inference/pipelines/text/ngram/ngram-table.js +5 -6
  501. package/src/inference/pipelines/text/ngram/ple-layer.d.ts +3 -3
  502. package/src/inference/pipelines/text/ngram/ple-layer.js +3 -3
  503. package/src/inference/pipelines/text/{generator-logits-observation.js → observation/generator-logits-observation.js} +4 -4
  504. package/src/inference/pipelines/text/{kernel-trace.js → observation/kernel-trace.js} +1 -1
  505. package/src/inference/pipelines/text/{operator-events.js → observation/operator-events.js} +1 -1
  506. package/src/inference/pipelines/text/{probes.js → observation/probes.js} +13 -13
  507. package/src/inference/pipelines/text/{tsir-fixture-writer.js → observation/tsir-fixture-writer.js} +3 -3
  508. package/src/inference/pipelines/text/{execution.js → operations/execution.js} +14 -14
  509. package/src/inference/pipelines/text/{image-transcription.js → operations/image-transcription.js} +8 -8
  510. package/src/inference/pipelines/text/{partition-attempt.js → partitions/partition-attempt.js} +7 -7
  511. package/src/inference/pipelines/text/{partition-execution.js → partitions/partition-execution.js} +79 -34
  512. package/src/inference/pipelines/text/partitions/partition-wire.js +50 -0
  513. package/src/inference/pipelines/text/{resident-partition-contract.d.ts → partitions/resident-partition-contract.d.ts} +10 -8
  514. package/src/inference/pipelines/text/{resident-partition-contract.js → partitions/resident-partition-contract.js} +6 -6
  515. package/src/inference/pipelines/text/{resident-partition.js → partitions/resident-partition.js} +19 -13
  516. package/src/inference/pipelines/text/{resident-pipeline-preparation.js → partitions/resident-pipeline-preparation.js} +32 -32
  517. package/src/inference/pipelines/text/per-layer/materialize.js +4 -4
  518. package/src/inference/pipelines/text/per-layer/per-layer-inputs.js +5 -0
  519. package/src/inference/pipelines/text/per-layer/plan.js +4 -4
  520. package/src/inference/pipelines/text/{per-layer-inputs.js → per-layer/prepare.js} +14 -280
  521. package/src/inference/pipelines/text/per-layer/projection-contract.js +79 -0
  522. package/src/inference/pipelines/text/per-layer/runtime-cache.js +183 -0
  523. package/src/inference/pipelines/text/pipeline.d.ts +13 -8
  524. package/src/inference/pipelines/text/pipeline.js +78 -22
  525. package/src/inference/pipelines/text/{drift-policy.js → policy/drift-policy.js} +1 -1
  526. package/src/inference/pipelines/text/{dtype-contract.js → policy/dtype-contract.js} +2 -2
  527. package/src/inference/pipelines/text/{finiteness-guard-status.js → policy/finiteness-guard-status.js} +2 -1
  528. package/src/inference/pipelines/text/{generator-decode-policy.js → policy/generator-decode-policy.js} +5 -5
  529. package/src/inference/pipelines/text/state.d.ts +9 -8
  530. package/src/inference/pipelines/text/state.js +1 -0
  531. package/src/inference/pipelines/text/types.d.ts +14 -218
  532. package/src/inference/pipelines/text.js +2 -0
  533. package/src/inference/tokenizers/bundled/merge.js +130 -0
  534. package/src/inference/tokenizers/bundled.js +2 -36
  535. package/src/loader/doppler-loader.d.ts +17 -7
  536. package/src/loader/doppler-loader.js +27 -29
  537. package/src/loader/embedding-limit-preflight.js +29 -1
  538. package/src/loader/embedding-loader.js +18 -4
  539. package/src/loader/experts/compact-expert-cache.js +488 -0
  540. package/src/loader/experts/expert-cache.d.ts +1 -1
  541. package/src/loader/experts/expert-dequant-cache.d.ts +9 -0
  542. package/src/loader/experts/expert-dequant-cache.js +75 -0
  543. package/src/loader/experts/expert-frequency-index.js +129 -0
  544. package/src/loader/experts/expert-loader.d.ts +4 -0
  545. package/src/loader/experts/expert-loader.js +13 -1
  546. package/src/loader/experts/expert-preparation.d.ts +1 -1
  547. package/src/loader/experts/expert-preparation.js +1 -1
  548. package/src/loader/experts/loader-bindings.js +117 -13
  549. package/src/loader/experts/mapped-expert-upload-cache.d.ts +61 -0
  550. package/src/loader/experts/mapped-expert-upload-cache.js +382 -0
  551. package/src/loader/experts/qwen-paging.d.ts +1 -1
  552. package/src/loader/experts/qwen-paging.js +1 -1
  553. package/src/loader/experts/routed-expert-cache.d.ts +62 -3
  554. package/src/loader/experts/routed-expert-cache.js +451 -99
  555. package/src/loader/experts/routed-expert-plan.d.ts +2 -0
  556. package/src/loader/experts/routed-expert-plan.js +14 -8
  557. package/src/loader/experts/routed-expert-prefill.js +82 -0
  558. package/src/loader/experts/routed-expert-source.js +233 -0
  559. package/src/loader/experts/routed-expert-trace.js +77 -0
  560. package/src/loader/experts/tiered-expert-cache.d.ts +12 -0
  561. package/src/loader/experts/tiered-expert-cache.js +287 -0
  562. package/src/loader/experts/tiered-expert-plan.js +74 -0
  563. package/src/loader/final-weights-loader.d.ts +1 -1
  564. package/src/loader/hyper-connection-loader.d.ts +2 -2
  565. package/src/loader/hyper-connection-loader.js +85 -13
  566. package/src/loader/layer-loader.d.ts +3 -3
  567. package/src/loader/layer-loader.js +1 -1
  568. package/src/loader/layer-tensor-aliases.js +14 -11
  569. package/src/loader/loader-types.d.ts +6 -1
  570. package/src/loader/manifest-config.js +4 -0
  571. package/src/loader/model-executor.js +18 -1
  572. package/src/loader/model-load.d.ts +1 -1
  573. package/src/loader/model-load.js +1 -1
  574. package/src/loader/ngram-table-bindings.d.ts +3 -3
  575. package/src/loader/ngram-table-bindings.js +7 -6
  576. package/src/loader/q4k-reference-weight.js +1 -1
  577. package/src/loader/shard-cache.d.ts +8 -1
  578. package/src/loader/shard-cache.js +33 -0
  579. package/src/loader/tensors/bf16-loader.js +2 -1
  580. package/src/loader/tensors/fp8-loader.js +1 -1
  581. package/src/loader/tensors/load-executor.js +10 -2
  582. package/src/loader/tensors/tensor-loader.js +1 -1
  583. package/src/loader/tensors/tensor-reader.js +18 -5
  584. package/src/loader/tensors/w8a8-loader.js +1 -1
  585. package/src/loader/weight-representation-plan.d.ts +1 -1
  586. package/src/memory/buffer-pool/readback.js +82 -0
  587. package/src/memory/buffer-pool.d.ts +11 -2
  588. package/src/memory/buffer-pool.js +12 -73
  589. package/src/partitions.d.ts +4 -3
  590. package/src/partitions.js +3 -2
  591. package/src/{converter → rules}/capability-transform-resolver.js +3 -3
  592. package/src/rules/generated/rule-bundle.json +1 -1
  593. package/src/rules/inference/capability-transforms.rules.json +95 -28
  594. package/src/rules/rule-registry.js +2 -0
  595. package/src/storage/artifact-range-loaders.js +42 -0
  596. package/src/storage/artifact-storage-context.d.ts +4 -0
  597. package/src/storage/artifact-storage-context.js +8 -18
  598. package/src/storage/backends/idb-store.js +1 -1
  599. package/src/storage/distribution-transport.js +1 -1
  600. package/src/storage/download/http-contract.js +1 -1
  601. package/src/storage/download/http-transport.js +4 -4
  602. package/src/storage/inventory.js +1 -1
  603. package/src/storage/model-cache.d.ts +2 -1
  604. package/src/storage/model-cache.js +12 -8
  605. package/src/storage/node-file-access.js +4 -0
  606. package/src/storage/quota.d.ts +2 -1
  607. package/src/storage/quota.js +2 -1
  608. package/src/storage/shard-manager.d.ts +1 -1
  609. package/src/storage/shards/integrity.js +17 -0
  610. package/src/storage/source-artifact-store.d.ts +1 -1
  611. package/src/storage/source-artifact-store.js +1 -0
  612. package/src/storage/source-storage-context.d.ts +4 -0
  613. package/src/storage/source-storage-context.js +14 -0
  614. package/src/storage/verified-piece-gather-lane.js +192 -0
  615. package/src/storage/verified-piece-gather-worker.js +520 -0
  616. package/src/storage/verified-piece-gather.d.ts +113 -0
  617. package/src/storage/verified-piece-gather.js +199 -0
  618. package/src/storage/verified-piece-reader.d.ts +7 -0
  619. package/src/storage/verified-piece-reader.js +69 -12
  620. package/src/storage/verified-piece-storage.d.ts +3 -1
  621. package/src/storage/verified-piece-storage.js +101 -11
  622. package/src/storage/verified-piece-timings.js +25 -0
  623. package/src/tooling/browser-command-runner.js +1 -1
  624. package/src/tooling/browser-harness/benchmark-execution.js +2 -2
  625. package/src/tooling/browser-harness/benchmark-rerank.js +2 -2
  626. package/src/tooling/browser-harness/inference-execution.js +1 -1
  627. package/src/tooling/browser-harness/model-resolution.js +1 -1
  628. package/src/tooling/browser-harness/runtime-config.js +1 -1
  629. package/src/tooling/browser-harness/runtime-isolation.js +2 -5
  630. package/src/tooling/browser-harness/text-evidence.js +1 -1
  631. package/src/tooling/browser-harness-contract-helpers.js +1 -1
  632. package/src/tooling/browser-harness.js +2 -2
  633. package/src/tooling/capsule-v2.js +2 -2
  634. package/src/tooling/capsule.js +2 -2
  635. package/src/tooling/command-api-constants.js +4 -2
  636. package/src/tooling/command-api.d.ts +1 -1
  637. package/src/tooling/command-api.js +15 -21
  638. package/src/tooling/command-runtime-execution.js +1 -8
  639. package/src/tooling/diagnose-runner.js +3 -41
  640. package/src/tooling/node-browser/static-server.js +2 -2
  641. package/src/tooling/node-command-execution.js +2 -34
  642. package/src/tooling/node-converter/input.js +1 -1
  643. package/src/tooling/node-converter/output.js +2 -2
  644. package/src/tooling/node-converter.js +1 -1
  645. package/src/tooling/node-runtime-bridge.js +43 -0
  646. package/src/tooling/node-source-runtime.js +34 -35
  647. package/src/tooling/production-release.js +1 -1
  648. package/src/tooling/program-bundle/materialize.js +1 -1
  649. package/src/tooling/program-bundle-parity.d.ts +1 -1
  650. package/src/tooling/program-bundle-parity.js +1 -1
  651. package/src/tooling/program-bundle.d.ts +1 -1
  652. package/src/tooling/program-bundle.js +2 -2
  653. package/src/tooling/runtime-intent.js +1 -1
  654. package/src/tooling/source-artifact-adapter.js +3 -3
  655. package/src/tooling/source-runtime/node-loader.js +32 -5
  656. package/src/tooling/source-runtime-converter-config.js +28 -6
  657. package/src/tooling-exports.shared.d.ts +3 -3
  658. package/src/tooling-exports.shared.js +2 -2
  659. package/src/version.js +1 -1
  660. package/tests/kernels/browser/kernel-suite.js +1 -1
  661. package/tests/kernels/harness/benchmark.js +1 -1
  662. package/tools/README.md +74 -0
  663. package/src/config/runtime/profiles/qwen3-6-35b-a3b-q4_0-pooled-q8-common-diagnostic.json +0 -4407
  664. package/src/inference/pipelines/text/moe-cpu.js +0 -20
  665. /package/src/config/{capsule-loading.d.ts → capsule/capsule-loading.d.ts} +0 -0
  666. /package/src/config/{capsule-release-contract.d.ts → capsule/capsule-release-contract.d.ts} +0 -0
  667. /package/src/config/{capsule-signature.d.ts → capsule/capsule-signature.d.ts} +0 -0
  668. /package/src/config/{capsule-signature.js → capsule/capsule-signature.js} +0 -0
  669. /package/src/config/{capsule-v3.d.ts → capsule/capsule-v3.d.ts} +0 -0
  670. /package/src/config/{capsule.d.ts → capsule/capsule.d.ts} +0 -0
  671. /package/src/config/schema/{diffusion.schema.d.ts → execution/diffusion.schema.d.ts} +0 -0
  672. /package/src/config/schema/{diffusion.schema.js → execution/diffusion.schema.js} +0 -0
  673. /package/src/config/schema/{energy.schema.d.ts → execution/energy.schema.d.ts} +0 -0
  674. /package/src/config/schema/{energy.schema.js → execution/energy.schema.js} +0 -0
  675. /package/src/config/schema/{hyper-connection-decoder.schema.d.ts → execution/hyper-connection-decoder.schema.d.ts} +0 -0
  676. /package/src/config/schema/{hyper-connection-decoder.schema.js → execution/hyper-connection-decoder.schema.js} +0 -0
  677. /package/src/config/schema/{inference.schema.js → execution/inference.schema.js} +0 -0
  678. /package/src/config/schema/{kernel-path.schema.js → execution/kernel-path.schema.js} +0 -0
  679. /package/src/config/schema/{kernel-registry.schema.js → execution/kernel-registry.schema.js} +0 -0
  680. /package/src/config/schema/{kernel-warmup.schema.d.ts → execution/kernel-warmup.schema.d.ts} +0 -0
  681. /package/src/config/schema/{kernel-warmup.schema.js → execution/kernel-warmup.schema.js} +0 -0
  682. /package/src/config/schema/{speculation-self.schema.d.ts → execution/speculation-self.schema.d.ts} +0 -0
  683. /package/src/config/schema/{speculation-self.schema.js → execution/speculation-self.schema.js} +0 -0
  684. /package/src/config/schema/{speculative.schema.d.ts → execution/speculative.schema.d.ts} +0 -0
  685. /package/src/config/schema/{speculative.schema.js → execution/speculative.schema.js} +0 -0
  686. /package/src/config/schema/{conversion-report.schema.d.ts → manifest/conversion-report.schema.d.ts} +0 -0
  687. /package/src/config/schema/{conversion-report.schema.js → manifest/conversion-report.schema.js} +0 -0
  688. /package/src/config/schema/{conversion.schema.d.ts → manifest/conversion.schema.d.ts} +0 -0
  689. /package/src/config/schema/{conversion.schema.js → manifest/conversion.schema.js} +0 -0
  690. /package/src/config/schema/{fp8.schema.d.ts → manifest/fp8.schema.d.ts} +0 -0
  691. /package/src/config/schema/{fp8.schema.js → manifest/fp8.schema.js} +0 -0
  692. /package/src/config/schema/{manifest-rope.schema.d.ts → manifest/manifest-rope.schema.d.ts} +0 -0
  693. /package/src/config/schema/{quantization-defaults.schema.d.ts → manifest/quantization-defaults.schema.d.ts} +0 -0
  694. /package/src/config/schema/{quantization-defaults.schema.js → manifest/quantization-defaults.schema.js} +0 -0
  695. /package/src/config/schema/{quantization.schema.d.ts → manifest/quantization.schema.d.ts} +0 -0
  696. /package/src/config/schema/{quantization.schema.js → manifest/quantization.schema.js} +0 -0
  697. /package/src/config/schema/{w8a8.schema.d.ts → manifest/w8a8.schema.d.ts} +0 -0
  698. /package/src/config/schema/{w8a8.schema.js → manifest/w8a8.schema.js} +0 -0
  699. /package/src/config/schema/{gpu-cache.schema.d.ts → platform/gpu-cache.schema.d.ts} +0 -0
  700. /package/src/config/schema/{gpu-cache.schema.js → platform/gpu-cache.schema.js} +0 -0
  701. /package/src/config/schema/{memory-limits.schema.d.ts → platform/memory-limits.schema.d.ts} +0 -0
  702. /package/src/config/schema/{platform.schema.d.ts → platform/platform.schema.d.ts} +0 -0
  703. /package/src/config/schema/{support-tiers.schema.d.ts → policy/support-tiers.schema.d.ts} +0 -0
  704. /package/src/config/schema/{support-tiers.schema.js → policy/support-tiers.schema.js} +0 -0
  705. /package/src/config/schema/{adapter.schema.d.ts → runtime/adapter.schema.d.ts} +0 -0
  706. /package/src/config/schema/{adapter.schema.js → runtime/adapter.schema.js} +0 -0
  707. /package/src/config/schema/{benchmark.schema.d.ts → runtime/benchmark.schema.d.ts} +0 -0
  708. /package/src/config/schema/{benchmark.schema.js → runtime/benchmark.schema.js} +0 -0
  709. /package/src/config/schema/{bridge.schema.d.ts → runtime/bridge.schema.d.ts} +0 -0
  710. /package/src/config/schema/{bridge.schema.js → runtime/bridge.schema.js} +0 -0
  711. /package/src/config/schema/{browser-suite-metrics.schema.d.ts → runtime/browser-suite-metrics.schema.d.ts} +0 -0
  712. /package/src/config/schema/{browser-suite-metrics.schema.js → runtime/browser-suite-metrics.schema.js} +0 -0
  713. /package/src/config/schema/{buffer-pool.schema.d.ts → runtime/buffer-pool.schema.d.ts} +0 -0
  714. /package/src/config/schema/{buffer-pool.schema.js → runtime/buffer-pool.schema.js} +0 -0
  715. /package/src/config/schema/{debug.schema.d.ts → runtime/debug.schema.d.ts} +0 -0
  716. /package/src/config/schema/{debug.schema.js → runtime/debug.schema.js} +0 -0
  717. /package/src/config/schema/{distribution.schema.d.ts → runtime/distribution.schema.d.ts} +0 -0
  718. /package/src/config/schema/{distribution.schema.js → runtime/distribution.schema.js} +0 -0
  719. /package/src/config/schema/{ecosystem.schema.d.ts → runtime/ecosystem.schema.d.ts} +0 -0
  720. /package/src/config/schema/{ecosystem.schema.js → runtime/ecosystem.schema.js} +0 -0
  721. /package/src/config/schema/{harness.schema.d.ts → runtime/harness.schema.d.ts} +0 -0
  722. /package/src/config/schema/{harness.schema.js → runtime/harness.schema.js} +0 -0
  723. /package/src/config/schema/{hotswap.schema.d.ts → runtime/hotswap.schema.d.ts} +0 -0
  724. /package/src/config/schema/{hotswap.schema.js → runtime/hotswap.schema.js} +0 -0
  725. /package/src/config/schema/{intent-bundle.schema.d.ts → runtime/intent-bundle.schema.d.ts} +0 -0
  726. /package/src/config/schema/{intent-bundle.schema.js → runtime/intent-bundle.schema.js} +0 -0
  727. /package/src/config/schema/{runtime-profile.schema.d.ts → runtime/runtime-profile.schema.d.ts} +0 -0
  728. /package/src/config/schema/{runtime-profile.schema.js → runtime/runtime-profile.schema.js} +0 -0
  729. /package/src/config/schema/{storage.schema.d.ts → runtime/storage.schema.d.ts} +0 -0
  730. /package/src/config/schema/{storage.schema.js → runtime/storage.schema.js} +0 -0
  731. /package/src/config/schema/{tooling.schema.js → runtime/tooling.schema.js} +0 -0
  732. /package/src/config/schema/{units.schema.d.ts → runtime/units.schema.d.ts} +0 -0
  733. /package/src/config/schema/{units.schema.js → runtime/units.schema.js} +0 -0
  734. /package/src/config/schema/{backward-registry.schema.d.ts → training/backward-registry.schema.d.ts} +0 -0
  735. /package/src/config/schema/{backward-registry.schema.js → training/backward-registry.schema.js} +0 -0
  736. /package/src/config/schema/{distill-training.schema.d.ts → training/distill-training.schema.d.ts} +0 -0
  737. /package/src/config/schema/{distill-training.schema.js → training/distill-training.schema.js} +0 -0
  738. /package/src/config/schema/{lora.schema.d.ts → training/lora.schema.d.ts} +0 -0
  739. /package/src/config/schema/{training-metrics.schema.d.ts → training/training-metrics.schema.d.ts} +0 -0
  740. /package/src/config/schema/{training-metrics.schema.js → training/training-metrics.schema.js} +0 -0
  741. /package/src/config/schema/{training.schema.d.ts → training/training.schema.d.ts} +0 -0
  742. /package/src/config/schema/{training.schema.js → training/training.schema.js} +0 -0
  743. /package/src/config/schema/{ul-training.schema.d.ts → training/ul-training.schema.d.ts} +0 -0
  744. /package/src/config/schema/{ul-training.schema.js → training/ul-training.schema.js} +0 -0
  745. /package/src/{converter → config/transforms}/execution-graph/dtype.js +0 -0
  746. /package/src/{converter → config/transforms}/execution-graph/fusion.js +0 -0
  747. /package/src/{converter → config/transforms}/execution-graph/kv-storage.js +0 -0
  748. /package/src/{converter → config/transforms}/execution-graph/validation.js +0 -0
  749. /package/src/{converter → formats}/qwen4-exp-source-defaults.json +0 -0
  750. /package/src/inference/pipelines/text/{buffer-types.d.ts → contracts/buffer-types.d.ts} +0 -0
  751. /package/src/inference/pipelines/text/{embedding-contract.d.ts → embedding/embedding-contract.d.ts} +0 -0
  752. /package/src/inference/pipelines/text/{embedding-contract.js → embedding/embedding-contract.js} +0 -0
  753. /package/src/inference/pipelines/text/{precomputed-input-norm.js → execution/precomputed-input-norm.js} +0 -0
  754. /package/src/inference/pipelines/text/generator/{batch-retry-policy.js → decode/batch-retry-policy.js} +0 -0
  755. /package/src/inference/pipelines/text/{chat-format.d.ts → generator/request/chat-format.d.ts} +0 -0
  756. /package/src/inference/pipelines/text/{chat-format.js → generator/request/chat-format.js} +0 -0
  757. /package/src/inference/pipelines/text/generator/{recorder-stats.js → session/recorder-stats.js} +0 -0
  758. /package/src/inference/pipelines/text/generator/{whole-token-template.js → whole-token/whole-token-template.js} +0 -0
  759. /package/src/inference/pipelines/text/{init-chat-templates.js → initialization/init-chat-templates.js} +0 -0
  760. /package/src/inference/pipelines/text/{load-policy.js → loading/load-policy.js} +0 -0
  761. /package/src/inference/pipelines/text/{pipeline-load-timing.js → loading/pipeline-load-timing.js} +0 -0
  762. /package/src/inference/pipelines/text/{generator → logits}/hidden-projection.d.ts +0 -0
  763. /package/src/inference/pipelines/text/{operator-events.d.ts → observation/operator-events.d.ts} +0 -0
  764. /package/src/inference/pipelines/text/{stage-names.d.ts → observation/stage-names.d.ts} +0 -0
  765. /package/src/inference/pipelines/text/{stage-names.js → observation/stage-names.js} +0 -0
  766. /package/src/inference/pipelines/text/{tsir-fixture-writer.d.ts → observation/tsir-fixture-writer.d.ts} +0 -0
  767. /package/src/inference/pipelines/text/{generator → per-layer}/per-layer-buffer.js +0 -0
  768. /package/src/inference/pipelines/text/{finiteness-policy.js → policy/finiteness-policy.js} +0 -0
@@ -1,38 +1,4 @@
1
- import { isPlainObject } from '../../../formats/plain-object.js';
2
- import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
3
- import { scopeKVCacheLayerRange, scopeKVCacheLayerIndices, resolveKVCacheLayerCount } from '../../kv-cache/layer-range.js';
4
- import { KVCache } from '../../kv-cache/base.js';
5
- import { SlidingWindowKVCache } from '../../kv-cache/sliding-window.js';
6
- import { TieredKVCache } from '../../kv-cache/tiered.js';
7
- import { BasisDecomposedPagedCache } from '../../kv-cache/basis-decomposed-paged.js';
8
- import { QuantizedKVCache } from '../../kv-cache/quantized.js';
9
- import { MixedGeometryKVCache } from '../../kv-cache/mixed-geometry.js';
10
- import {
11
- retainTurboQuantSharedBuffers,
12
- } from '../../../gpu/kernels/turboquant-codebook.js';
13
- import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
14
- import { selectRuleValue as selectKernelRuleValue } from '../../../gpu/kernels/rule-registry.js';
15
- import { MoERouter } from '../../moe-router.js';
16
- import { SpeculativeDecoder } from '../../speculative.js';
17
- import { createDopplerLoader } from '../../../loader/doppler-loader.js';
18
- import { log } from '../../../debug/index.js';
19
- import { getRuntimeConfig } from '../../../config/runtime.js';
20
- import { PAGED_LAYOUT_SEQ_LEN_THRESHOLD } from '../../../config/schema/index.js';
21
- import { selectRuleValue } from '../../../rules/rule-registry.js';
22
- import { resolvePerLayerInputsSession } from './generator/session-context.js';
23
- import {
24
- createHttpArtifactStorageContext,
25
- createNodeFileArtifactStorageContext,
26
- } from '../../../storage/artifact-storage-context.js';
27
- import { hasLinearAttentionLayers } from './linear-attention.js';
28
- import { isConvLayerType } from './layer-input-execution.js';
29
- import { resolveShardHashVerification } from './load-policy.js';
30
- import { rewriteWeightLoadError } from './load-errors.js';
31
- import { resolveWeightLoadingConfig } from './weight-loading-config.js';
32
- import { toArrayBuffer } from '../../../formats/array-buffer.js';
33
-
34
- // Re-exports for backwards compatibility
35
- export { initRoPEFrequencies, isGPURoPEBuffers } from './init-rope.js';
1
+ export { initRoPEFrequencies, isGPURoPEBuffers } from "./initialization/init-rope.js";
36
2
  export {
37
3
  applyChatTemplate,
38
4
  applyGemmaChatTemplate,
@@ -41,902 +7,12 @@ export {
41
7
  applyGptOssChatTemplate,
42
8
  applyQwenChatTemplate,
43
9
  isStopToken,
44
- } from './init-chat-templates.js';
45
- export { fuseQKVWeights } from './init-qkv-fusion.js';
46
- export { resolveWeightLoadingConfig, resolveQ4KConfig } from './weight-loading-config.js';
47
-
48
- function resolveErrorMessage(error) {
49
- if (error && typeof error === 'object' && typeof error.message === 'string') {
50
- return error.message;
51
- }
52
- if (typeof error === 'string') {
53
- return error;
54
- }
55
- return String(error);
56
- }
57
-
58
- function isRuntimeInferenceConfigInput(value) {
59
- if (!isPlainObject(value)) {
60
- return false;
61
- }
62
- return Object.prototype.hasOwnProperty.call(value, 'session')
63
- || Object.prototype.hasOwnProperty.call(value, 'kvcache')
64
- || Object.prototype.hasOwnProperty.call(value, 'compute')
65
- || Object.prototype.hasOwnProperty.call(value, 'batching')
66
- || Object.prototype.hasOwnProperty.call(value, 'generation')
67
- || Object.prototype.hasOwnProperty.call(value, 'kernelPath')
68
- || Object.prototype.hasOwnProperty.call(value, 'pipeline');
69
- }
70
-
71
- function resolveRuntimeKVConfig(runtimeInput) {
72
- if (runtimeInput != null && !isRuntimeInferenceConfigInput(runtimeInput)) {
73
- return runtimeInput;
74
- }
75
- const runtimeInference = runtimeInput ?? getRuntimeConfig().inference;
76
- if (runtimeInference?.session?.kvcache) {
77
- return runtimeInference.session.kvcache;
78
- }
79
- throw new Error(
80
- 'runtime.inference.session.kvcache is required for live KV cache creation. ' +
81
- 'Top-level runtime.inference.kvcache is not supported on the live path.'
82
- );
83
- }
84
-
85
- function assertSupportedTurboQuantMode(mode, label) {
86
- if (String(mode ?? '').trim().toLowerCase() !== 'turboquant_outlier') {
87
- return;
88
- }
89
- throw new Error(
90
- `${label}="turboquant_outlier" is not supported yet. ` +
91
- 'TurboQuant outlier high-precision buffers and decode kernels are not wired end to end; ' +
92
- 'use "turboquant" or "turboquant_prod".'
93
- );
94
- }
95
-
96
- function toUint8Array(value, label) {
97
- if (value instanceof Uint8Array) {
98
- return value;
99
- }
100
- if (value instanceof ArrayBuffer) {
101
- return new Uint8Array(value);
102
- }
103
- throw new Error(`${label} must return ArrayBuffer or Uint8Array.`);
104
- }
105
-
106
- function createStorageContextLoadError(error, operation, index, context = {}) {
107
- const message = error?.message || String(error);
108
- const wrapped = new Error(
109
- `storageContext.${operation} failed for shard ${index}: ${message}`,
110
- error instanceof Error ? { cause: error } : undefined
111
- );
112
- wrapped.name = error?.name || 'Error';
113
- if (error?.code !== undefined) {
114
- wrapped.code = error.code;
115
- }
116
- wrapped.details = {
117
- ...(error?.details && typeof error.details === 'object' ? error.details : {}),
118
- storageOperation: operation,
119
- shardIndex: index,
120
- ...context,
121
- };
122
- return wrapped;
123
- }
124
-
125
- function isRDRRManifest(manifest) {
126
- return manifest !== null && typeof manifest === 'object' && Array.isArray( (manifest).shards);
127
- }
128
-
129
- export function createNodeFileShardStorageContext(baseUrl, manifest, options = {}) {
130
- return createNodeFileArtifactStorageContext(baseUrl, manifest, options);
131
- }
132
-
133
- function createRemoteStorageContext(baseUrl, manifest, options = {}) {
134
- const nodeFileStorageContext = createNodeFileArtifactStorageContext(baseUrl, manifest, options);
135
- if (nodeFileStorageContext) {
136
- return nodeFileStorageContext;
137
- }
138
- return createHttpArtifactStorageContext(baseUrl, manifest, options);
139
- }
140
-
141
-
142
- function normalizeLayerType(layerType) {
143
- return typeof layerType === 'string' ? layerType.trim().toLowerCase() : '';
144
- }
145
-
146
- function isSlidingLayerType(layerType) {
147
- const normalized = normalizeLayerType(layerType);
148
- return normalized === 'sliding_attention'
149
- || normalized === 'local_attention'
150
- || normalized === 'local'
151
- || normalized === 'sliding';
152
- }
153
-
154
- function hasFullAttentionLayers(layerTypes) {
155
- if (!Array.isArray(layerTypes) || layerTypes.length === 0) {
156
- return false;
157
- }
158
- return layerTypes.some((layerType) => !isSlidingLayerType(layerType));
159
- }
160
-
161
- function resolveContiguousKVPolicy(modelConfig) {
162
- if (hasFullAttentionLayers(modelConfig.layerTypes)) {
163
- return {
164
- forceContiguousKVCache: true,
165
- reason: 'layerPattern',
166
- };
167
- }
168
- if (modelConfig.layerTypes == null && modelConfig.slidingWindow == null) {
169
- return {
170
- forceContiguousKVCache: true,
171
- reason: 'slidingWindow',
172
- };
173
- }
174
- return {
175
- forceContiguousKVCache: false,
176
- reason: 'none',
177
- };
178
- }
179
-
180
- function layerOwnsNoAttentionKV(layerType) {
181
- return hasLinearAttentionLayers([layerType]) || isConvLayerType(layerType);
182
- }
183
-
184
- function usesMixedGeometryKVCache(modelConfig) {
185
- if (!modelConfig || typeof modelConfig !== 'object') {
186
- return false;
187
- }
188
- const hasMixedAttentionGeometry = Number.isFinite(modelConfig.globalHeadDim)
189
- && modelConfig.globalHeadDim > 0
190
- && modelConfig.globalHeadDim !== modelConfig.headDim;
191
- const hasSharedKvLayers = Number.isFinite(modelConfig.numKvSharedLayers)
192
- && modelConfig.numKvSharedLayers > 0;
193
- const hasExplicitLayerTypes = Array.isArray(modelConfig.layerTypes)
194
- && modelConfig.layerTypes.length === modelConfig.numLayers;
195
- return hasExplicitLayerTypes && (hasMixedAttentionGeometry || hasSharedKvLayers);
196
- }
197
-
198
- function resolveMixedGeometrySlidingLayerLayout(modelConfig) {
199
- const diffusionGemmaContract = modelConfig?.diffusionGemma ?? null;
200
- if (diffusionGemmaContract == null) {
201
- return 'ring';
202
- }
203
- if (diffusionGemmaContract.decoderCacheMode === 'encoder_kv_readonly_canvas_concat') {
204
- return 'contiguous';
205
- }
206
- throw new Error(
207
- `Unsupported DiffusionGemma decoderCacheMode="${String(diffusionGemmaContract.decoderCacheMode)}" ` +
208
- 'for mixed-geometry KV cache.'
209
- );
210
- }
211
-
212
- function getRequiredKernelMaxKVLen(operation, variant, label) {
213
- const config = getKernelConfig(operation, variant);
214
- const maxKVLen = config.variantMetadata?.maxKVLen;
215
- if (!Number.isFinite(maxKVLen) || maxKVLen <= 0) {
216
- throw new Error(`${label} kernel "${variant}" is missing variantMetadata.maxKVLen.`);
217
- }
218
- return maxKVLen;
219
- }
220
-
221
- function resolveTieredRequestedQuantMode(runtimeKV) {
222
- const tiering = requirePlainObject(
223
- runtimeKV?.tiering,
224
- 'runtime.inference.session.kvcache.tiering'
225
- );
226
- const tieringMode = requireNonEmptyString(
227
- tiering.mode,
228
- 'runtime.inference.session.kvcache.tiering.mode'
229
- ).toLowerCase();
230
- assertSupportedTurboQuantMode(tieringMode, 'runtime.inference.session.kvcache.tiering.mode');
231
- const compression = requirePlainObject(
232
- tiering.compression,
233
- 'runtime.inference.session.kvcache.tiering.compression'
234
- );
235
- const compressionMode = requireNonEmptyString(
236
- compression.mode,
237
- 'runtime.inference.session.kvcache.tiering.compression.mode'
238
- ).toLowerCase();
239
- assertSupportedTurboQuantMode(
240
- compressionMode,
241
- 'runtime.inference.session.kvcache.tiering.compression.mode'
242
- );
243
- const gating = requirePlainObject(
244
- tiering.gating,
245
- 'runtime.inference.session.kvcache.tiering.gating'
246
- );
247
- const gatingMode = requireNonEmptyString(
248
- gating.mode,
249
- 'runtime.inference.session.kvcache.tiering.gating.mode'
250
- ).toLowerCase();
251
- if (gatingMode === 'force_off') {
252
- return 'none';
253
- }
254
- return compressionMode;
255
- }
256
-
257
- function resolveContiguousRequestedQuantMode(runtimeKV) {
258
- const quantization = requirePlainObject(
259
- runtimeKV?.quantization,
260
- 'runtime.inference.session.kvcache.quantization'
261
- );
262
- const quantMode = requireNonEmptyString(
263
- quantization.mode,
264
- 'runtime.inference.session.kvcache.quantization.mode'
265
- ).toLowerCase();
266
- assertSupportedTurboQuantMode(
267
- quantMode,
268
- 'runtime.inference.session.kvcache.quantization.mode'
269
- );
270
- return quantMode;
271
- }
272
-
273
- function assertQuantizedKVKernelSupport(modelConfig, cacheLayout, cacheMaxSeqLen, runtimeKV) {
274
- if (cacheLayout === 'contiguous_quantized') {
275
- if (modelConfig.headDim > 256) {
276
- throw new Error(
277
- `Contiguous quantized KV cache requires headDim <= 256; got ${modelConfig.headDim}.`
278
- );
279
- }
280
- const quantMode = resolveContiguousRequestedQuantMode(runtimeKV);
281
- const variant = selectKernelRuleValue('attention', 'contiguousQuantVariant', { mode: quantMode });
282
- const maxKVLen = getRequiredKernelMaxKVLen(
283
- 'attention_contiguous_quant',
284
- variant,
285
- 'Contiguous quant attention'
286
- );
287
- if (cacheMaxSeqLen > maxKVLen) {
288
- throw new Error(
289
- `Contiguous quantized KV cache requires maxSeqLen <= ${maxKVLen}; got ${cacheMaxSeqLen}.`
290
- );
291
- }
292
- return;
293
- }
294
-
295
- if (cacheLayout !== 'tiered') {
296
- return;
297
- }
298
-
299
- const coldQuantMode = resolveTieredRequestedQuantMode(runtimeKV);
300
- if (coldQuantMode === 'none') {
301
- return;
302
- }
303
- if (modelConfig.headDim > 256) {
304
- throw new Error(
305
- `Tiered quantized KV cache requires headDim <= 256; got ${modelConfig.headDim}.`
306
- );
307
- }
308
- const variant = selectKernelRuleValue('attention', 'tieredQuantVariant', { mode: coldQuantMode });
309
- const maxKVLen = getRequiredKernelMaxKVLen(
310
- 'attention_tiered_quant',
311
- variant,
312
- 'Tiered quant attention'
313
- );
314
- if (cacheMaxSeqLen > maxKVLen) {
315
- throw new Error(
316
- `Tiered quantized KV cache requires maxSeqLen <= ${maxKVLen}; got ${cacheMaxSeqLen}.`
317
- );
318
- }
319
- }
320
-
321
- // ============================================================================
322
- // KV Cache Setup
323
- // ============================================================================
324
-
325
- export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig, layerRange = null) {
326
- let allocatedLayerCount = resolveKVCacheLayerCount(modelConfig.numLayers, layerRange);
327
- if (modelConfig?.decodeStrategy === 'replay_prefill') {
328
- throw new Error(
329
- 'Live KV cache creation is not supported for models that require replay-prefill decode. ' +
330
- 'Skip createKVCache() when the model config does not resolve explicit layerTypes for mixed-geometry/shared-KV decode.'
331
- );
332
- }
333
- const runtimeKV = resolveRuntimeKVConfig(runtimeConfig);
334
- const requiresMixedGeometryKVCache = usesMixedGeometryKVCache(modelConfig);
335
- if (layerRange !== null && requiresMixedGeometryKVCache) {
336
- throw new Error('Resident partition KV cache does not support mixed geometry or shared KV layers.');
337
- }
338
- // Linear attention and short-convolution layers own recurrent or convolution
339
- // state, not per-token attention keys/values. Keep mixed geometry/shared-KV
340
- // ownership unchanged; its source-layer mapping is a separate cache contract.
341
- let allocatedLayerIndices = null;
342
- if (!requiresMixedGeometryKVCache && Array.isArray(modelConfig.layerTypes)
343
- && modelConfig.layerTypes.some(layerOwnsNoAttentionKV)) {
344
- if (modelConfig.layerTypes.length !== modelConfig.numLayers) {
345
- throw new Error('Hybrid KV allocation requires explicit types for every model layer.');
346
- }
347
- const firstLayer = layerRange === null ? 0 : layerRange[0];
348
- allocatedLayerIndices = Array.from({ length: allocatedLayerCount }, (_, index) => firstLayer + index)
349
- .filter(layer => !layerOwnsNoAttentionKV(modelConfig.layerTypes[layer]));
350
- allocatedLayerCount = allocatedLayerIndices.length;
351
- }
352
- const contiguousKVPolicy = resolveContiguousKVPolicy(modelConfig);
353
- const forceContiguousKVCache = contiguousKVPolicy.forceContiguousKVCache;
354
- const modelMaxSeqLen = modelConfig.maxSeqLen;
355
- if (!Number.isFinite(modelMaxSeqLen) || modelMaxSeqLen <= 0) {
356
- throw new Error('Model config is missing maxSeqLen.');
357
- }
358
- let slidingWindow = modelConfig.slidingWindow;
359
-
360
- let cacheMaxSeqLen = modelMaxSeqLen;
361
- if (Number.isFinite(runtimeKV.maxSeqLen) && runtimeKV.maxSeqLen > 0) {
362
- cacheMaxSeqLen = Math.min(cacheMaxSeqLen, runtimeKV.maxSeqLen);
363
- }
364
-
365
-
366
- let cacheLayout = runtimeKV.layout;
367
- if (!cacheLayout) {
368
- throw new Error('runtime.inference.session.kvcache.layout is required.');
369
- }
370
- if (cacheLayout === 'tiered' && !runtimeKV.tiering) {
371
- throw new Error('runtime.inference.session.kvcache.tiering is required for tiered layout.');
372
- }
373
- const tieringMode = runtimeKV.tiering?.mode;
374
- if (tieringMode == null) {
375
- throw new Error('runtime.inference.session.kvcache.tiering.mode is required.');
376
- }
377
- let layoutSource = 'runtime';
378
- if (tieringMode !== 'off' && cacheLayout !== 'tiered') {
379
- if (cacheLayout !== 'contiguous') {
380
- throw new Error('runtime.inference.session.kvcache.layout must be "tiered" when tiering.mode is enabled.');
381
- }
382
- cacheLayout = 'tiered';
383
- layoutSource = 'tiering';
384
- }
385
- if (!forceContiguousKVCache && cacheLayout === 'contiguous' && cacheMaxSeqLen >= PAGED_LAYOUT_SEQ_LEN_THRESHOLD) {
386
- cacheLayout = 'paged';
387
- layoutSource = 'threshold';
388
- }
389
- const quantMode = resolveContiguousRequestedQuantMode(runtimeKV);
390
- if (forceContiguousKVCache && cacheLayout === 'contiguous' && quantMode !== 'none') {
391
- cacheLayout = 'contiguous_quantized';
392
- layoutSource = 'quantization';
393
- }
394
- if (forceContiguousKVCache && cacheLayout === 'paged') {
395
- throw new Error(
396
- 'Paged KV cache layout is not supported for models with full-attention layers. ' +
397
- 'Set runtime.inference.session.kvcache.layout to "contiguous" instead.'
398
- );
399
- }
400
- if (layerRange !== null && cacheLayout !== 'contiguous') {
401
- throw new Error(`Resident partition KV cache requires resolved contiguous layout; got ${cacheLayout}.`);
402
- }
403
- if (requiresMixedGeometryKVCache) {
404
- if (cacheLayout !== 'contiguous') {
405
- throw new Error(
406
- `Mixed-geometry incremental KV cache requires layout="contiguous"; got "${cacheLayout}". ` +
407
- 'Disable tiering and quantized KV cache for this model.'
408
- );
409
- }
410
- if (runtimeKV.tiering?.mode != null && runtimeKV.tiering.mode !== 'off') {
411
- throw new Error(
412
- `Mixed-geometry incremental KV cache requires tiering.mode="off"; got "${runtimeKV.tiering.mode}".`
413
- );
414
- }
415
- if (resolveContiguousRequestedQuantMode(runtimeKV) !== 'none') {
416
- throw new Error(
417
- 'Mixed-geometry incremental KV cache does not support contiguous quantization yet. ' +
418
- 'Set runtime.inference.session.kvcache.quantization.mode="none".'
419
- );
420
- }
421
- if (!useGPU) {
422
- throw new Error(
423
- 'Mixed-geometry incremental KV cache requires GPU execution. ' +
424
- 'Use a WebGPU-capable surface.'
425
- );
426
- }
427
- }
428
- if (debug && cacheLayout !== runtimeKV.layout) {
429
- log.debug('Pipeline', `KV cache layout override: ${runtimeKV.layout} -> ${cacheLayout} (${layoutSource})`);
430
- }
431
-
432
- // Sliding-window attention only needs a bounded KV cache on contiguous layouts.
433
- if (slidingWindow && Number.isFinite(slidingWindow) && slidingWindow > 0) {
434
- if (runtimeKV.windowSize > 0) {
435
- slidingWindow = Math.min(slidingWindow, runtimeKV.windowSize);
436
- }
437
- if (!forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered') {
438
- cacheMaxSeqLen = Math.min(cacheMaxSeqLen, slidingWindow);
439
- }
440
- }
441
-
442
- // Use f16 KV cache when supported to reduce VRAM.
443
- // For models with attention logit softcapping, allow forcing F32 via runtime config
444
- // to avoid precision issues in attention. See: https://github.com/ggerganov/llama.cpp/issues/8853
445
- const gpuCaps = getKernelCapabilities();
446
- // Use config value directly instead of model detection flag (manifest-first architecture)
447
- // Check > 0 to allow explicit "disabled" encoding as 0 or null
448
- const attnSoftcap = modelConfig.attnLogitSoftcapping;
449
- const hasAttnSoftcapping = attnSoftcap != null && attnSoftcap > 0;
450
- const forceF32Softcap = runtimeKV.forceF32Softcap === true;
451
- const forceF32KV = hasAttnSoftcapping && forceF32Softcap;
452
-
453
- const kvDtype = selectRuleValue('inference', 'dtype', 'kvCacheDtype', {
454
- requested: runtimeKV.kvDtype,
455
- useGPU,
456
- hasF16: gpuCaps.hasF16,
457
- forceF32: forceF32KV,
458
- });
459
- if (forceF32KV && debug) {
460
- log.debug('Pipeline', `Forcing F32 KV cache (attnLogitSoftcapping=${modelConfig.attnLogitSoftcapping}, forceF32Softcap=true)`);
461
- }
462
- if (cacheLayout === 'tiered' && kvDtype !== 'f16') {
463
- throw new Error('Tiered KV cache requires kvDtype="f16" (no f32 tiered kernels yet).');
464
- }
465
- if (cacheLayout === 'contiguous_quantized' && kvDtype !== 'f16') {
466
- throw new Error('Contiguous quantized KV cache requires kvDtype="f16".');
467
- }
468
- if (cacheLayout === 'contiguous_quantized' && !useGPU) {
469
- throw new Error('Contiguous quantized KV cache requires GPU.');
470
- }
471
-
472
- if (useGPU && (cacheLayout === 'paged' || cacheLayout === 'tiered' || cacheLayout === 'bdpa' || cacheLayout === 'contiguous_quantized')) {
473
- const limits = getDeviceLimits();
474
- if (limits) {
475
- const maxLayerHeadDim = Math.max(
476
- modelConfig.headDim,
477
- Number.isFinite(modelConfig.globalHeadDim) && modelConfig.globalHeadDim > 0
478
- ? modelConfig.globalHeadDim
479
- : modelConfig.headDim
480
- );
481
- const bytesPerToken = modelConfig.numKVHeads * maxLayerHeadDim * (kvDtype === 'f16' ? 2 : 4);
482
- const maxByBinding = Math.floor(limits.maxStorageBufferBindingSize / bytesPerToken);
483
- const maxByBuffer = Math.floor(limits.maxBufferSize / bytesPerToken);
484
- const fallbackMax = Number.isFinite(runtimeKV.gpuPagedFallbackMaxSeqLen) && runtimeKV.gpuPagedFallbackMaxSeqLen > 0
485
- ? runtimeKV.gpuPagedFallbackMaxSeqLen
486
- : Infinity;
487
- const limitMax = Math.min(maxByBinding, maxByBuffer, fallbackMax);
488
- if (!Number.isFinite(limitMax) || limitMax <= 0) {
489
- throw new Error('KV cache maxSeqLen exceeds device buffer limits.');
490
- }
491
- if (Number.isFinite(limitMax) && limitMax > 0 && limitMax < cacheMaxSeqLen) {
492
- log.warn(
493
- 'Pipeline',
494
- `KV cache maxSeqLen capped ${cacheMaxSeqLen} -> ${limitMax} (layout=${cacheLayout}, limit=${limits.maxStorageBufferBindingSize}).`
495
- );
496
- cacheMaxSeqLen = limitMax;
497
- }
498
- }
499
- }
500
-
501
- assertQuantizedKVKernelSupport(modelConfig, cacheLayout, cacheMaxSeqLen, runtimeKV);
502
-
503
-
504
- const cacheConfig = {
505
- numLayers: allocatedLayerCount,
506
- numHeads: modelConfig.numKVHeads,
507
- headDim: modelConfig.headDim,
508
- maxSeqLen: cacheMaxSeqLen,
509
- useGPU,
510
- layout: cacheLayout,
511
- kvDtype,
512
- bdpaVocabSize: runtimeKV.bdpaVocabSize,
513
- pageSize: runtimeKV.pageSize,
514
- };
515
-
516
-
517
- let kvCache;
518
-
519
- if (requiresMixedGeometryKVCache) {
520
- kvCache = new MixedGeometryKVCache({
521
- ...cacheConfig,
522
- numHeads: modelConfig.numKVHeads,
523
- globalNumHeads: modelConfig.numGlobalKVHeads ?? modelConfig.numKVHeads,
524
- globalHeadDim: modelConfig.globalHeadDim ?? null,
525
- slidingWindow,
526
- slidingLayerLayout: resolveMixedGeometrySlidingLayerLayout(modelConfig),
527
- layerTypes: modelConfig.layerTypes,
528
- });
529
- } else if (modelConfig.slidingWindow && !forceContiguousKVCache && cacheLayout !== 'paged' && cacheLayout !== 'tiered' && cacheLayout !== 'bdpa') {
530
- kvCache = new SlidingWindowKVCache({
531
- ...cacheConfig,
532
- windowSize: slidingWindow ?? modelConfig.slidingWindow,
533
- });
534
- } else if (cacheLayout === 'bdpa') {
535
- kvCache = new BasisDecomposedPagedCache({
536
- ...cacheConfig,
537
- });
538
- } else if (cacheLayout === 'tiered') {
539
- kvCache = new TieredKVCache({
540
- ...cacheConfig,
541
- tiering: runtimeKV.tiering,
542
- });
543
- } else if (cacheLayout === 'contiguous_quantized') {
544
- const quantCfg = requirePlainObject(
545
- runtimeKV.quantization,
546
- 'runtime.inference.kvcache.quantization'
547
- );
548
- const bitWidth = requirePositiveInteger(
549
- quantCfg.bitWidth,
550
- 'runtime.inference.kvcache.quantization.bitWidth'
551
- );
552
- const qCache = new QuantizedKVCache({
553
- ...cacheConfig,
554
- quantMode,
555
- bitWidth,
556
- prodMode: quantCfg.prodMode === true,
557
- });
558
- const device = getDevice();
559
- qCache.setSharedBuffers(retainTurboQuantSharedBuffers(device, {
560
- headDim: modelConfig.headDim,
561
- bitWidth,
562
- prodMode: quantCfg.prodMode === true,
563
- }));
564
- kvCache = qCache;
565
- } else {
566
- kvCache = new KVCache(cacheConfig);
567
- }
568
-
569
- // Diagnostic logging: actual vs requested KV cache parameters
570
- const requestedKvDtype = runtimeKV.kvDtype ?? 'unset';
571
- const requestedMaxSeqLen = runtimeKV.maxSeqLen ?? 'unset';
572
- const actualKvDtype = kvDtype;
573
- const actualMaxSeqLen = cacheMaxSeqLen;
574
-
575
- if (requestedKvDtype !== 'unset' && requestedKvDtype !== actualKvDtype) {
576
- log.warn(
577
- 'Pipeline',
578
- `KV cache kvDtype mismatch: requested=${requestedKvDtype}, actual=${actualKvDtype} ` +
579
- `(rule-based resolution may have changed the dtype due to GPU capabilities or softcap policy)`
580
- );
581
- }
582
- if (requestedMaxSeqLen !== 'unset' && requestedMaxSeqLen !== actualMaxSeqLen) {
583
- log.debug(
584
- 'Pipeline',
585
- `KV cache maxSeqLen adjusted: requested=${requestedMaxSeqLen}, actual=${actualMaxSeqLen}`
586
- );
587
- }
588
-
589
- log.info(
590
- 'Pipeline',
591
- `KV cache allocated: kvDtype=${actualKvDtype}, maxSeqLen=${actualMaxSeqLen}, ` +
592
- `layout=${cacheLayout}, modelMaxSeqLen=${modelMaxSeqLen}, ` +
593
- `requestedKvDtype=${requestedKvDtype}, requestedMaxSeqLen=${requestedMaxSeqLen}`
594
- );
595
-
596
- if (debug) {
597
- if (contiguousKVPolicy.reason === 'layerPattern') {
598
- log.debug('Pipeline', 'Layer pattern includes full-attention layers; paged layout blocked, contiguous enforced.');
599
- } else if (contiguousKVPolicy.reason === 'slidingWindow') {
600
- log.debug('Pipeline', 'Model declares attention.slidingWindow=null without explicit layerTypes; treating KV layout as contiguous-compatible.');
601
- }
602
- const isSliding = kvCache instanceof SlidingWindowKVCache;
603
- log.debug('Pipeline', `KV cache: type=${kvCache?.constructor?.name || 'unknown'}, kvDtype=${kvCache.kvDtype}, layout=${kvCache.layout}, maxSeqLen=${kvCache.maxSeqLen}, windowSize=${isSliding ? kvCache.windowSize : null}`);
604
- }
605
-
606
- if (allocatedLayerIndices !== null) return scopeKVCacheLayerIndices(kvCache, allocatedLayerIndices);
607
- return layerRange === null ? kvCache : scopeKVCacheLayerRange(kvCache, layerRange[0]);
608
- }
609
-
610
- function requirePlainObject(value, label) {
611
- if (!isPlainObject(value)) {
612
- throw new Error(`${label} is required.`);
613
- }
614
- return value;
615
- }
616
-
617
- function requireNonEmptyString(value, label) {
618
- if (typeof value !== 'string' || value.trim().length === 0) {
619
- throw new Error(`${label} is required.`);
620
- }
621
- return value.trim();
622
- }
623
-
624
- function requirePositiveInteger(value, label) {
625
- if (!Number.isInteger(value) || value <= 0) {
626
- throw new Error(`${label} must be a positive integer.`);
627
- }
628
- return value;
629
- }
630
-
631
- // ============================================================================
632
- // Tokenizer Setup
633
- // ============================================================================
634
-
635
- export { initTokenizer } from './init-tokenizer.js';
636
-
637
- // ============================================================================
638
- // Weight Loading
639
- // ============================================================================
640
-
641
- export async function loadWeights(manifest, modelConfig, options = {}) {
642
- const {
643
- onProgress,
644
- loadingConfig,
645
- baseUrl,
646
- loaderDebug,
647
- perLayerInputSession,
648
- } = options;
649
- // Resolve the policy before creating a context so the context it creates
650
- // carries the same decision; a context must never shadow the configured policy.
651
- const configuredVerifyHashes = resolveShardHashVerification({ options, loadingConfig }).verifyHashes;
652
- const runtimeStorageContext = options.storageContext
653
- ?? createRemoteStorageContext(baseUrl, manifest, { verifyHashes: configuredVerifyHashes });
654
- if (typeof runtimeStorageContext?.preflight === 'function') {
655
- await runtimeStorageContext.preflight();
656
- }
657
- const { verifyHashes } = resolveShardHashVerification({
658
- options,
659
- loadingConfig,
660
- storageContext: runtimeStorageContext,
661
- });
662
-
663
- const dopplerLoader = options.loader ?? createDopplerLoader(loadingConfig);
664
- const keepF32Weights = options.keepF32Weights === true;
665
- dopplerLoader.setQ4KConfig({
666
- representationPaths: options.reachableKernelPaths ?? [options.resolvedKernelPath].filter(Boolean),
667
- ...resolveWeightLoadingConfig(
668
- manifest,
669
- options.resolvedKernelPath ?? null,
670
- options.kernelPathSource ?? 'none',
671
- keepF32Weights
672
- ),
673
- });
674
- dopplerLoader.setLoaderDebugConfig(loaderDebug ?? null);
675
- dopplerLoader.setPerLayerInputSession(
676
- resolvePerLayerInputsSession(
677
- modelConfig.perLayerInputsSession ?? null,
678
- perLayerInputSession ?? null
679
- )
680
- );
681
-
682
- const tensorsFile = isRDRRManifest(manifest) ? manifest.tensorsFile : null;
683
- if (baseUrl && tensorsFile) {
684
- const base = baseUrl.replace(/\/$/, '');
685
- const filename = tensorsFile.replace(/^\/+/, '');
686
- dopplerLoader.setTensorsJsonUrl(`${base}/${filename}`);
687
- } else {
688
- dopplerLoader.setTensorsJsonUrl(null);
689
- }
690
- dopplerLoader.setTensorsJsonLoader(
691
- typeof runtimeStorageContext?.loadTensorsJson === 'function'
692
- ? () => runtimeStorageContext.loadTensorsJson()
693
- : null
694
- );
695
-
696
- // Configure custom shard loader if provided (Native Bridge or direct-source bundle)
697
- const hasLoadShard = typeof runtimeStorageContext?.loadShard === 'function';
698
- const hasLoadShardRange = typeof runtimeStorageContext?.loadShardRange === 'function';
699
- const hasStreamShardRange = typeof runtimeStorageContext?.streamShardRange === 'function';
700
- if (hasLoadShard || hasLoadShardRange) {
701
- log.debug('Pipeline', 'Using custom shard loader (Native Bridge or external)');
702
-
703
- const loadShard = async (index) => {
704
- try {
705
- if (hasLoadShard) {
706
- const data = await runtimeStorageContext.loadShard(index);
707
- return toUint8Array(data, 'storageContext.loadShard');
708
- }
709
- const rangeData = await runtimeStorageContext.loadShardRange(index, 0, null);
710
- return toUint8Array(rangeData, 'storageContext.loadShardRange');
711
- } catch (error) {
712
- throw createStorageContextLoadError(error, hasLoadShard ? 'loadShard' : 'loadShardRange', index);
713
- }
714
- };
715
-
716
- const loadShardRange = hasLoadShardRange
717
- ? async (index, offset, length = null) => {
718
- try {
719
- const data = await runtimeStorageContext.loadShardRange(index, offset, length);
720
- return toArrayBuffer(data, 'storageContext.loadShardRange');
721
- } catch (error) {
722
- throw createStorageContextLoadError(error, 'loadShardRange', index, { offset, length });
723
- }
724
- }
725
- : null;
726
-
727
- const streamShardRange = hasStreamShardRange
728
- ? async function* (index, offset = 0, length = null, streamOptions = {}) {
729
- try {
730
- for await (const chunk of runtimeStorageContext.streamShardRange(index, offset, length, streamOptions)) {
731
- yield toUint8Array(chunk, 'storageContext.streamShardRange');
732
- }
733
- } catch (error) {
734
- throw createStorageContextLoadError(error, 'streamShardRange', index, { offset, length });
735
- }
736
- }
737
- : null;
738
-
739
- dopplerLoader.setCustomShardLoader(loadShard, {
740
- verify: verifyHashes,
741
- loadShardRange,
742
- streamShardRange,
743
- loadAuxiliaryFile: typeof runtimeStorageContext?.loadAuxiliaryFile === 'function'
744
- ? (path) => runtimeStorageContext.loadAuxiliaryFile(path)
745
- : null,
746
- });
747
- if (isRDRRManifest(manifest)) {
748
- dopplerLoader.setManifest(manifest);
749
- }
750
- }
751
-
752
- await dopplerLoader.init();
753
-
754
- // Load model via DopplerLoader
755
- const modelId = manifest.modelId;
756
- if (!modelId) {
757
- throw new Error('Manifest is missing modelId. Re-convert the model with modelId set.');
758
- }
759
- try {
760
- await dopplerLoader.load(modelId, {
761
- verifyHashes, partition: options.partition, signal: options.signal,
762
- onProgress: onProgress || ((info) => {
763
- // Shard and layer progress are logged by loader with source info
764
- if (info.stage !== 'layers' && info.stage !== 'shards') {
765
- log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%`);
766
- }
767
- }),
768
- });
769
- } catch (error) {
770
- throw rewriteWeightLoadError(error, { modelId });
771
- }
772
-
773
- // Map layer weights
774
-
775
- const layerWeights = new Map();
776
- for (let l = 0; l < modelConfig.numLayers; l++) {
777
- const weights = dopplerLoader.getLayerWeights(l);
778
- if (weights) {
779
- layerWeights.set(`layer_${l}`, weights);
780
- }
781
- }
782
- // Collect per-layer router weights for MoE
783
-
784
- const layerRouterWeights = new Map();
785
- if (modelConfig.useMoE) {
786
- for (let l = 0; l < modelConfig.numLayers; l++) {
787
- const weights = layerWeights.get(`layer_${l}`);
788
- if (weights?.routerWeight) {
789
- layerRouterWeights.set(l, {
790
- weight: weights.routerWeight,
791
- bias: weights.routerBias || null,
792
- scale: weights.routerScale || null,
793
- perExpertScale: weights.routerPerExpertScale || null,
794
- });
795
- }
796
- }
797
- log.debug('Pipeline', `MoE expert preparation: ${dopplerLoader.getLoadTiming()?.expertPreparation?.status}`);
798
- }
799
-
800
- return {
801
- loader: dopplerLoader,
802
- layerWeights,
803
- embeddings: dopplerLoader.embeddings,
804
- lmHead: dopplerLoader.lmHead,
805
- lmHeadBias: dopplerLoader.lmHeadBias,
806
- finalNorm: dopplerLoader.finalNorm,
807
- hyperConnectionMixer: dopplerLoader.hyperConnectionMixer,
808
- finalNormBias: dopplerLoader.finalNormBias,
809
- embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
810
- diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
811
- perLayerInputWeights: dopplerLoader.perLayerInputWeights,
812
- layerRouterWeights,
813
- loadTiming: typeof dopplerLoader.getLoadTiming === 'function'
814
- ? dopplerLoader.getLoadTiming()
815
- : null,
816
- };
817
- }
818
-
819
- // ============================================================================
820
- // MoE Router Setup
821
- // ============================================================================
822
-
823
- export function initMoERouter(modelConfig, moeRoutingConfig, layerWeights) {
824
- if (!modelConfig.useMoE) return null;
825
-
826
- const router = new MoERouter({
827
- numExperts: modelConfig.numExperts,
828
- topK: modelConfig.moeTopK,
829
- hiddenSize: modelConfig.hiddenSize,
830
- normalizeWeights: moeRoutingConfig.normalizeWeights,
831
- });
832
-
833
- // Find first layer with router weights
834
- for (let l = 0; l < modelConfig.numLayers; l++) {
835
- const weights = layerWeights.get(`layer_${l}`);
836
- if (weights?.routerWeight) {
837
- router.loadWeights(
838
- weights.routerWeight,
839
- weights.routerBias || null,
840
- weights.routerScale || null,
841
- weights.routerPerExpertScale || null
842
- );
843
- log.debug('Pipeline', `Loaded MoE router from layer ${l}${weights.routerBias ? ' (with bias)' : ''}`);
844
- break;
845
- }
846
- }
847
-
848
- return router;
849
- }
850
-
851
- // ============================================================================
852
- // Speculative Decoder Setup
853
- // ============================================================================
854
-
855
- // EXPERIMENTAL: Speculative decoding is parsed and initialized but the full
856
- // verify-and-accept loop is not yet wired into the generation pipeline.
857
- // Enabling this will create the decoder state but decoded tokens are not
858
- // verified against the draft model. Do not rely on this for production use.
859
- export function initSpeculativeDecoder(manifest, speculativeConfig) {
860
- if (!manifest.draftModel) return null;
861
- if (manifest.draftModel.numTokens == null) {
862
- throw new Error(`Manifest "${manifest.modelId}" is missing draftModel.numTokens.`);
863
- }
864
-
865
- log.warn(
866
- 'Pipeline',
867
- `Speculative decoding enabled for "${manifest.modelId}" but this feature is experimental and not fully wired. ` +
868
- 'The draft-verify-accept loop is incomplete. Generated output is from the base model only.'
869
- );
870
-
871
- return new SpeculativeDecoder({
872
- numDraftTokens: manifest.draftModel.numTokens,
873
- maxRejectionRetries: speculativeConfig.maxRejectionRetries,
874
- enableTreeDraft: speculativeConfig.enableTreeDraft,
875
- temperature: speculativeConfig.temperature,
876
- randomSeed: speculativeConfig.randomSeed,
877
- });
878
- }
879
- // ============================================================================
880
- // Emulation Setup
881
- // ============================================================================
882
-
883
- export async function initEmulation(runtimeConfig) {
884
- const emulationConfig = runtimeConfig?.emulation;
885
-
886
- // Skip if emulation is not enabled
887
- if (!emulationConfig?.enabled) {
888
- return null;
889
- }
890
-
891
- try {
892
- const simulatorModuleRoot = '/proto/simulator';
893
- const simulatorEnvSpecifier = `${simulatorModuleRoot}/env.js`;
894
- const simulatorIndexSpecifier = `${simulatorModuleRoot}/index.js`;
895
-
896
- // Dynamically import to avoid loading emulation code when disabled
897
- const { setSimulatorEnv } = await import(simulatorEnvSpecifier);
898
- const { createEmulationConfig, formatBytes, formatBandwidth } = await import('../../../config/schema/emulation.schema.js');
899
- const { EmulatedVramStore, detectLocalResources } = await import('../../../storage/emulated-vram.js');
900
- const { getBufferPool } = await import('../../../memory/buffer-pool.js');
901
- const { createEmulationContext, isEmulationSupported } = await import(simulatorIndexSpecifier);
902
-
903
- setSimulatorEnv({
904
- log,
905
- bufferPool: getBufferPool,
906
- createEmulationConfig,
907
- formatBytes,
908
- formatBandwidth,
909
- detectLocalResources,
910
- createVramStore: (config, budgets) =>
911
- new EmulatedVramStore(config.opfsRootPath, budgets.vramBudgetBytes, budgets.ramBudgetBytes),
912
- });
913
-
914
- const supported = await isEmulationSupported();
915
- if (!supported) {
916
- throw new Error('Emulation requested but not supported in this environment.');
917
- }
918
-
919
- // Create emulation context
920
- log.info('Pipeline', `Initializing emulation for ${emulationConfig.targetChip}`);
921
- const ctx = await createEmulationContext(emulationConfig);
922
-
923
- log.info('Pipeline', `Emulation ready: ${ctx.config.topology.gpuCount} virtual GPUs, timing mode: ${ctx.config.timingMode}`);
924
-
925
- return ctx;
926
- } catch (err) {
927
- const message = resolveErrorMessage(err);
928
- log.error('Pipeline', `Failed to initialize emulation: ${message}`);
929
- throw new Error(`Failed to initialize emulation: ${message}`);
930
- }
931
- }
932
-
933
- export async function destroyEmulation(emulation) {
934
- if (emulation) {
935
- try {
936
- await emulation.destroy();
937
- log.info('Pipeline', 'Emulation context destroyed');
938
- } catch (err) {
939
- log.warn('Pipeline', `Error destroying emulation: ${err.message}`);
940
- }
941
- }
942
- }
10
+ } from "./initialization/init-chat-templates.js";
11
+ export { fuseQKVWeights } from "./initialization/init-qkv-fusion.js";
12
+ export { resolveWeightLoadingConfig, resolveQ4KConfig } from "./loading/weight-loading-config.js";
13
+ export { initTokenizer } from "./initialization/init-tokenizer.js";
14
+ export { createKVCache } from './initialization/init-kv-cache.js';
15
+ export { createNodeFileShardStorageContext, loadWeights } from "./loading/init-weights.js";
16
+ export { initMoERouter } from './initialization/init-moe.js';
17
+ export { initSpeculativeDecoder } from './initialization/init-speculative.js';
18
+ export { initEmulation, destroyEmulation } from './initialization/init-emulation.js';