doppler-gpu 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (927) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +99 -86
  6. package/src/capsule-runtime.d.ts +23 -0
  7. package/src/capsule-runtime.js +23 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +1 -0
  24. package/src/client/capsule-host.d.ts +3 -0
  25. package/src/client/capsule-host.js +1 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +36 -1
  28. package/src/client/doppler-api.js +40 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +322 -0
  45. package/src/client/{runtime → model-host}/model-session.js +190 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +68 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +100 -0
  64. package/src/client/runtime/capsule-program-adapter.js +151 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +85 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +74 -0
  72. package/src/client/runtime/composition-root.js +344 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +30 -0
  92. package/src/client/runtime/session-controller.js +170 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +11 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +58 -0
  109. package/src/config/capsule-operations.json +18 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  128. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
  129. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  130. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  132. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  134. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  135. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  136. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
  137. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
  140. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
  141. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
  142. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  143. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  145. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  146. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  147. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  148. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  149. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  150. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  151. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  152. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  153. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  154. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
  155. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  157. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  158. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  160. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
  161. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  162. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  163. package/src/config/dtype-transition-contract.js +40 -0
  164. package/src/config/embedding-contract.js +26 -0
  165. package/src/config/embedding-reference.js +85 -0
  166. package/src/config/evidence/source-intake-policy.json +1 -1
  167. package/src/config/execution-contract-check.d.ts +16 -73
  168. package/src/config/execution-contract-check.js +7 -429
  169. package/src/config/forecast-manifest.js +56 -0
  170. package/src/config/heterogeneous-attention-contract.js +46 -0
  171. package/src/config/initial-execution-identity.d.ts +58 -0
  172. package/src/config/initial-execution-identity.js +331 -0
  173. package/src/config/kernel-path-loader.d.ts +10 -0
  174. package/src/config/kernel-path-loader.js +23 -8
  175. package/src/config/kernel-registry-contract.d.ts +25 -0
  176. package/src/config/kernel-registry-contract.js +46 -0
  177. package/src/config/kernels/kernel-ref-digests.js +50 -2
  178. package/src/config/kernels/registry.js +1 -1
  179. package/src/config/kernels/registry.json +13592 -11377
  180. package/src/config/lora-layouts.d.ts +10 -0
  181. package/src/config/lora-layouts.js +17 -0
  182. package/src/config/lora-layouts.json +9 -0
  183. package/src/config/manifest-inference-validation.js +575 -0
  184. package/src/config/merge/execution-patches.js +12 -0
  185. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  186. package/src/config/merge/value-selection.js +38 -0
  187. package/src/config/merge.js +43 -11
  188. package/src/config/model-ir-v2.d.ts +57 -0
  189. package/src/config/model-ir-v2.js +258 -0
  190. package/src/config/model-ir.d.ts +49 -0
  191. package/src/config/model-ir.js +180 -0
  192. package/src/config/param-validator.js +1 -1
  193. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  194. package/src/config/platforms/loader.js +2 -1
  195. package/src/config/production-release-evidence.d.ts +23 -0
  196. package/src/config/production-release-evidence.js +423 -0
  197. package/src/config/production-release.js +321 -0
  198. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  199. package/src/config/required-inference-fields-contract-check.js +2 -2
  200. package/src/config/rerank-reference.d.ts +28 -0
  201. package/src/config/rerank-reference.js +89 -0
  202. package/src/config/revocation-policy.d.ts +65 -0
  203. package/src/config/revocation-policy.js +290 -0
  204. package/src/config/revocation-registry.json +11 -0
  205. package/src/config/revocation-updates.d.ts +68 -0
  206. package/src/config/revocation-updates.js +440 -0
  207. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  208. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  209. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  210. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  211. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  212. package/src/config/runtime-merge.js +1 -1
  213. package/src/config/runtime.js +1 -1
  214. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  215. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  216. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  217. package/src/config/schema/conversion-report.schema.js +29 -0
  218. package/src/config/schema/converter.schema.d.ts +1 -0
  219. package/src/config/schema/debug.schema.d.ts +1 -0
  220. package/src/config/schema/doppler.schema.js +5 -5
  221. package/src/config/schema/emulation.schema.js +1 -1
  222. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  223. package/src/config/schema/execution-v1.schema.js +19 -0
  224. package/src/config/schema/harness.schema.d.ts +13 -0
  225. package/src/config/schema/harness.schema.js +6 -0
  226. package/src/config/schema/inference.schema.d.ts +21 -2
  227. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  228. package/src/config/schema/manifest.schema.d.ts +26 -7
  229. package/src/config/schema/manifest.schema.js +10 -1
  230. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  231. package/src/config/schema/program-bundle.schema.js +145 -10
  232. package/src/config/sequence-reference.d.ts +26 -0
  233. package/src/config/sequence-reference.js +100 -0
  234. package/src/config/target-plan.d.ts +108 -0
  235. package/src/config/target-plan.js +411 -0
  236. package/src/config/training-defaults.js +1 -1
  237. package/src/config/transforms/capability-transform-resolver.js +4 -105
  238. package/src/config/transforms/execution-graph/dtype.js +591 -0
  239. package/src/config/transforms/execution-graph/fusion.js +436 -0
  240. package/src/config/transforms/execution-graph/validation.js +414 -0
  241. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  242. package/src/converter/artifact-identity.js +475 -0
  243. package/src/converter/conversion-plan.js +1 -1
  244. package/src/converter/core.js +94 -2096
  245. package/src/converter/forge-candidate-evaluation.js +271 -0
  246. package/src/converter/forge-qualification.js +162 -0
  247. package/src/converter/forge-stages.js +798 -0
  248. package/src/converter/manifest-builder.js +683 -0
  249. package/src/converter/quantization-info.js +4 -1
  250. package/src/converter/quantizer.js +58 -9
  251. package/src/converter/shard-packer.d.ts +1 -1
  252. package/src/converter/shard-packer.js +2 -2
  253. package/src/converter/tensor-transform.js +640 -0
  254. package/src/converter/tokenizer-contract.js +243 -0
  255. package/src/converter/tokenizer-manifest.js +243 -0
  256. package/src/debug/history.js +0 -8
  257. package/src/debug/index.js +0 -10
  258. package/src/debug/tensor.js +1 -1
  259. package/src/debug/trace.js +0 -7
  260. package/src/experimental/adapters/adapter-manager.js +2 -0
  261. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  262. package/src/experimental/adapters/adapter-manifest.js +11 -0
  263. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  264. package/src/experimental/adapters/artifact-contract.js +5 -2
  265. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  266. package/src/experimental/adapters/lora-loader.js +110 -48
  267. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  268. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  269. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  270. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  271. package/src/experimental/browser/browser-converter.js +11 -11
  272. package/src/experimental/diffusion/index.d.ts +1 -1
  273. package/src/experimental/diffusion/index.js +1 -1
  274. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  275. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  276. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  277. package/src/experimental/diffusion/vae.js +596 -0
  278. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  279. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  280. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  281. package/src/experimental/distribution/shard-delivery.js +4 -1709
  282. package/src/experimental/energy/index.d.ts +2 -2
  283. package/src/experimental/energy/index.js +2 -2
  284. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  285. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  286. package/src/experimental/logits/cpu-output.d.ts +27 -0
  287. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  288. package/src/experimental/logits/cpu.d.ts +87 -0
  289. package/src/experimental/logits/cpu.js +95 -0
  290. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  291. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  292. package/src/experimental/orchestration/index.d.ts +1 -1
  293. package/src/experimental/orchestration/index.js +1 -1
  294. package/src/experimental/training/artifacts.js +2 -2
  295. package/src/experimental/training/checkpoint.js +2 -2
  296. package/src/experimental/training/distillation/artifacts.js +1 -1
  297. package/src/experimental/training/distillation/dataset.js +2 -2
  298. package/src/experimental/training/distillation/eval.js +1 -1
  299. package/src/experimental/training/distillation/runtime.js +2 -2
  300. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  301. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  302. package/src/experimental/training/distillation/student-fixture.js +2 -469
  303. package/src/experimental/training/lora/checkpoint.js +746 -0
  304. package/src/experimental/training/lora/execution.js +550 -0
  305. package/src/experimental/training/lora/plan.js +387 -0
  306. package/src/experimental/training/lora/recovery.js +418 -0
  307. package/src/experimental/training/lora-pipeline.js +7 -1949
  308. package/src/experimental/training/operator-artifacts.js +2 -2
  309. package/src/experimental/training/operator-command.js +4 -4
  310. package/src/experimental/training/runner/evaluation.js +62 -0
  311. package/src/experimental/training/runner/lifecycle.js +257 -0
  312. package/src/experimental/training/runner.js +5 -268
  313. package/src/experimental/training/suite/execution.js +879 -0
  314. package/src/experimental/training/suite/plan.js +855 -0
  315. package/src/experimental/training/suite.js +5 -1659
  316. package/src/experimental/training/workloads.d.ts +5 -5
  317. package/src/experimental/training/workloads.js +6 -6
  318. package/src/experimental/turboquant/codebook.js +470 -0
  319. package/src/formats/quantization-constants.d.ts +11 -0
  320. package/src/formats/quantization-constants.js +11 -0
  321. package/src/formats/rdrr/integrity.js +1 -1
  322. package/src/formats/rdrr/lowerings.js +0 -29
  323. package/src/formats/rdrr/merkle.js +1 -1
  324. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  325. package/src/formats/rdrr/validation.js +3 -3
  326. package/src/gpu/command-recorder.d.ts +10 -1
  327. package/src/gpu/command-recorder.js +58 -14
  328. package/src/gpu/constant-buffer.js +29 -0
  329. package/src/gpu/device-state.js +36 -0
  330. package/src/gpu/device.js +6 -41
  331. package/src/gpu/kernel-runtime.js +2 -1
  332. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  333. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  334. package/src/gpu/kernels/attention/executors.js +960 -0
  335. package/src/gpu/kernels/attention/plan.js +472 -0
  336. package/src/gpu/kernels/attention.d.ts +4 -0
  337. package/src/gpu/kernels/attention.js +145 -1545
  338. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  339. package/src/gpu/kernels/backward/adam.js +2 -1
  340. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  341. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  342. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  343. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  344. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  345. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  346. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  347. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  348. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  349. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  350. package/src/gpu/kernels/backward/executor.js +158 -0
  351. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  352. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  353. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  354. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  355. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  356. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  357. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  358. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  359. package/src/gpu/kernels/backward/matmul.js +133 -0
  360. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  361. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  362. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  363. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  364. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  365. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  366. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  367. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  368. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  369. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  370. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  371. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  372. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  374. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  376. package/src/gpu/kernels/backward/utils.js +2 -392
  377. package/src/gpu/kernels/bind-group-validation.js +16 -0
  378. package/src/gpu/kernels/cast.js +2 -1
  379. package/src/gpu/kernels/check-finiteness.js +13 -49
  380. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  381. package/src/gpu/kernels/check-stop.js +14 -33
  382. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  383. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  384. package/src/gpu/kernels/check_stop.wgsl +20 -0
  385. package/src/gpu/kernels/clamp.js +1 -1
  386. package/src/gpu/kernels/conv2d.js +1 -1
  387. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  388. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  389. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  390. package/src/gpu/kernels/dequant.js +7 -2
  391. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  392. package/src/gpu/kernels/dispatch.js +12 -1
  393. package/src/gpu/kernels/embedding-pool.js +72 -0
  394. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  395. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  396. package/src/gpu/kernels/energy.js +2 -1
  397. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  398. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  399. package/src/gpu/kernels/fused_ffn.js +10 -220
  400. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  401. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  402. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  403. package/src/gpu/kernels/gated-short-conv.js +11 -69
  404. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  405. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  406. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  407. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  408. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  409. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  410. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  411. package/src/gpu/kernels/gather.d.ts +1 -1
  412. package/src/gpu/kernels/gather.js +30 -6
  413. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  414. package/src/gpu/kernels/gelu.js +1 -1
  415. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  416. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  417. package/src/gpu/kernels/groupnorm.js +1 -1
  418. package/src/gpu/kernels/index.d.ts +38 -5
  419. package/src/gpu/kernels/index.js +18 -4
  420. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  421. package/src/gpu/kernels/kernel-base.js +1 -1
  422. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  423. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  424. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  425. package/src/gpu/kernels/kernel-configs.js +26 -54
  426. package/src/gpu/kernels/kernel-execution.js +141 -0
  427. package/src/gpu/kernels/kv-cache-write.js +4 -4
  428. package/src/gpu/kernels/kv-quantize.js +4 -1
  429. package/src/gpu/kernels/l2-normalize.js +36 -0
  430. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  431. package/src/gpu/kernels/layernorm.d.ts +4 -4
  432. package/src/gpu/kernels/layernorm.js +44 -7
  433. package/src/gpu/kernels/linear-attention-core.js +29 -449
  434. package/src/gpu/kernels/linear_attention.js +1 -1
  435. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  436. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  437. package/src/gpu/kernels/logit-finalize.js +103 -0
  438. package/src/gpu/kernels/logit-merge.js +13 -125
  439. package/src/gpu/kernels/logit-scatter.js +40 -0
  440. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  441. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  442. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  443. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  444. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  445. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  446. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  447. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  448. package/src/gpu/kernels/matmul-selection.js +3 -2
  449. package/src/gpu/kernels/matmul.js +4 -15
  450. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  451. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  452. package/src/gpu/kernels/modulate.js +1 -1
  453. package/src/gpu/kernels/moe.js +3 -1
  454. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  455. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  456. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  457. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  458. package/src/gpu/kernels/patch_context.wgsl +29 -0
  459. package/src/gpu/kernels/pipeline-cache.js +3 -1
  460. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  461. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  462. package/src/gpu/kernels/relu.js +1 -1
  463. package/src/gpu/kernels/repeat_channels.js +1 -1
  464. package/src/gpu/kernels/residual.js +2 -2
  465. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  466. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  467. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  468. package/src/gpu/kernels/rmsnorm.js +1 -1
  469. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  470. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  471. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  472. package/src/gpu/kernels/rope-precompute.js +160 -0
  473. package/src/gpu/kernels/rope-qk.js +1 -1
  474. package/src/gpu/kernels/rope.js +2 -2
  475. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  476. package/src/gpu/kernels/sample.js +5 -1
  477. package/src/gpu/kernels/scale.js +1 -1
  478. package/src/gpu/kernels/shader-cache.js +8 -4
  479. package/src/gpu/kernels/shader-source-scope.js +68 -0
  480. package/src/gpu/kernels/silu.js +2 -1
  481. package/src/gpu/kernels/soft-embedding.js +3 -1
  482. package/src/gpu/kernels/softmax.js +4 -2
  483. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  484. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  485. package/src/gpu/kernels/split_qg.js +1 -1
  486. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  487. package/src/gpu/kernels/split_qkv.js +18 -3
  488. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  489. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  490. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  491. package/src/gpu/kernels/transpose.js +1 -1
  492. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  493. package/src/gpu/kernels/uniform-utils.js +7 -2
  494. package/src/gpu/kernels/upsample2d.js +1 -1
  495. package/src/gpu/kernels/utils.js +9 -330
  496. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  497. package/src/gpu/kernels/vision-average-pool.js +68 -0
  498. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  499. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  500. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  501. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  502. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  503. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  504. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  505. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  506. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  507. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  508. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  509. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  510. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  511. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  512. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  513. package/src/gpu/readback-buffer.js +20 -0
  514. package/src/gpu/submit-probe.js +2 -26
  515. package/src/gpu/weight-buffer.d.ts +1 -0
  516. package/src/gpu/weight-buffer.js +7 -0
  517. package/src/index.d.ts +10 -0
  518. package/src/index.js +1 -0
  519. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  520. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  521. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  522. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  523. package/src/inference/browser-harness/inference-execution.js +543 -0
  524. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  525. package/src/inference/browser-harness/model-resolution.js +137 -0
  526. package/src/inference/browser-harness/report.js +338 -0
  527. package/src/inference/browser-harness/request.js +263 -0
  528. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  529. package/src/inference/browser-harness/runtime-config.js +255 -0
  530. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  531. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  532. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  533. package/src/inference/browser-harness/runtime-report.js +117 -0
  534. package/src/inference/browser-harness/suite-summary.js +39 -0
  535. package/src/inference/browser-harness/text-evidence.js +694 -0
  536. package/src/inference/browser-harness/text-execution.js +757 -0
  537. package/src/inference/browser-harness/text-input.js +725 -0
  538. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  539. package/src/inference/browser-harness-contract-helpers.js +1 -1
  540. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  541. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  542. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  543. package/src/inference/browser-harness-text-helpers.js +31 -2199
  544. package/src/inference/browser-harness.d.ts +15 -2
  545. package/src/inference/browser-harness.js +38 -1952
  546. package/src/inference/kv-cache/base.js +67 -381
  547. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  548. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  549. package/src/inference/kv-cache/quantized.d.ts +2 -2
  550. package/src/inference/kv-cache/quantized.js +4 -8
  551. package/src/inference/kv-cache/update.js +309 -0
  552. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  553. package/src/inference/model-checkpoint-evidence.js +264 -0
  554. package/src/inference/moe-router.js +36 -216
  555. package/src/inference/multi-model-network.js +5 -341
  556. package/src/inference/pipelines/audio/index.js +1 -20
  557. package/src/inference/pipelines/audio/mel.js +0 -38
  558. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  559. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  560. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  561. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  562. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  563. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  564. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  565. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  566. package/src/inference/pipelines/factory.js +14 -3
  567. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  568. package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
  569. package/src/inference/pipelines/text/abort-contract.js +22 -0
  570. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  571. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  572. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  573. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  574. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  575. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  576. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  577. package/src/inference/pipelines/text/attention/plan.js +17 -0
  578. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  579. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  580. package/src/inference/pipelines/text/attention/projections.js +20 -537
  581. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  582. package/src/inference/pipelines/text/attention/record.js +0 -6
  583. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  584. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  585. package/src/inference/pipelines/text/attention/run.js +0 -6
  586. package/src/inference/pipelines/text/attention/types.js +3 -31
  587. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  588. package/src/inference/pipelines/text/chat-format.js +42 -0
  589. package/src/inference/pipelines/text/config/normalization.js +729 -0
  590. package/src/inference/pipelines/text/config/validation.js +9 -0
  591. package/src/inference/pipelines/text/config.d.ts +12 -1
  592. package/src/inference/pipelines/text/config.js +14 -1228
  593. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  594. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  595. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  596. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  597. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  598. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  599. package/src/inference/pipelines/text/embed.js +178 -241
  600. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  601. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  602. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  603. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  604. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  605. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  606. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  607. package/src/inference/pipelines/text/execution-v1.js +7 -283
  608. package/src/inference/pipelines/text/execution.js +517 -0
  609. package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
  610. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  611. package/src/inference/pipelines/text/ffn/dense.js +4 -968
  612. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  613. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  614. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  615. package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
  616. package/src/inference/pipelines/text/generator/decode.js +952 -0
  617. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  618. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  619. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  620. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  621. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  622. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  623. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  624. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  625. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  626. package/src/inference/pipelines/text/generator/text.js +448 -0
  627. package/src/inference/pipelines/text/generator/training.js +752 -0
  628. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  629. package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
  630. package/src/inference/pipelines/text/generator-runtime.js +5 -135
  631. package/src/inference/pipelines/text/generator-steps.js +30 -972
  632. package/src/inference/pipelines/text/generator.js +165 -3544
  633. package/src/inference/pipelines/text/image-transcription.js +304 -0
  634. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  635. package/src/inference/pipelines/text/init-rope.js +37 -189
  636. package/src/inference/pipelines/text/init.d.ts +8 -3
  637. package/src/inference/pipelines/text/init.js +6 -216
  638. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  639. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  640. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  641. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  642. package/src/inference/pipelines/text/layer-plan.js +45 -3
  643. package/src/inference/pipelines/text/layer.js +52 -522
  644. package/src/inference/pipelines/text/lifecycle.js +839 -0
  645. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  646. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  647. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  648. package/src/inference/pipelines/text/linear-attention.js +7 -537
  649. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  650. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  651. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  652. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  653. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  654. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  655. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  656. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  657. package/src/inference/pipelines/text/logits/index.js +62 -102
  658. package/src/inference/pipelines/text/logits/output-transform.js +47 -0
  659. package/src/inference/pipelines/text/logits/plan.js +184 -0
  660. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  661. package/src/inference/pipelines/text/logits/readback.js +9 -0
  662. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  663. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  664. package/src/inference/pipelines/text/lora-apply.js +12 -4
  665. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  666. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  667. package/src/inference/pipelines/text/model-load.js +0 -20
  668. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  669. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  670. package/src/inference/pipelines/text/moe/plan.js +47 -0
  671. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  672. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  673. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  674. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  675. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  676. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  677. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  678. package/src/inference/pipelines/text/probes.js +20 -0
  679. package/src/inference/pipelines/text/state.d.ts +8 -0
  680. package/src/inference/pipelines/text/state.js +1 -0
  681. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  682. package/src/inference/pipelines/text/types.d.ts +4 -0
  683. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  684. package/src/inference/pipelines/text.d.ts +2 -2
  685. package/src/inference/pipelines/text.js +46 -1633
  686. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  687. package/src/inference/pipelines/video/index.js +0 -21
  688. package/src/inference/pipelines/vision/attention.js +59 -0
  689. package/src/inference/pipelines/vision/encoder.js +254 -369
  690. package/src/inference/pipelines/vision/gemma4.js +204 -318
  691. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  692. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  693. package/src/inference/pipelines/vision/glmocr.js +557 -0
  694. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  695. package/src/inference/pipelines/vision/index.js +8 -27
  696. package/src/inference/pipelines/vision/ops.js +11 -33
  697. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  698. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  699. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  700. package/src/inference/refactor-receipt.js +2 -2
  701. package/src/inference/rerank.js +182 -0
  702. package/src/inference/tokenizer.js +1 -1
  703. package/src/inference/tokenizers/behavior-flags.js +6 -3
  704. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  705. package/src/inference/tokenizers/bundled/registry.js +216 -0
  706. package/src/inference/tokenizers/bundled.js +23 -353
  707. package/src/loader/doppler-loader.d.ts +4 -0
  708. package/src/loader/doppler-loader.js +119 -930
  709. package/src/loader/embedding-limit-preflight.js +11 -4
  710. package/src/loader/embedding-loader.js +1 -3
  711. package/src/loader/gpu-tensor-materialization.js +20 -0
  712. package/src/loader/layer-loader.js +15 -119
  713. package/src/loader/layer-tensor-aliases.js +99 -0
  714. package/src/loader/layer-weight-shape.js +29 -0
  715. package/src/loader/load-timing.js +64 -0
  716. package/src/loader/loader-types.d.ts +3 -0
  717. package/src/loader/model-executor.js +819 -0
  718. package/src/loader/model-load-validation.js +15 -0
  719. package/src/loader/quantization-constants.d.ts +11 -29
  720. package/src/loader/quantization-constants.js +1 -2
  721. package/src/loader/shard-resolver.js +0 -14
  722. package/src/loader/tensor-load-error.js +14 -0
  723. package/src/loader/tensor-shard-indices.js +8 -0
  724. package/src/loader/tensors/bf16-loader.js +125 -0
  725. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  726. package/src/loader/tensors/load-executor.js +577 -0
  727. package/src/loader/tensors/source-transform/executor.js +123 -0
  728. package/src/loader/tensors/source-transform/plan.js +107 -0
  729. package/src/loader/tensors/source-transform.js +15 -264
  730. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  731. package/src/loader/tensors/tensor-loader.js +7 -653
  732. package/src/memory/buffer-pool.js +7 -29
  733. package/src/rules/capability-transform-resolver.js +80 -0
  734. package/src/rules/execution-contract.d.ts +73 -0
  735. package/src/rules/execution-contract.js +429 -0
  736. package/src/rules/execution-rules-contract-check.js +1 -1
  737. package/src/rules/generated/rule-bundle.json +4305 -0
  738. package/src/rules/inference/capability-transforms.rules.json +3 -3
  739. package/src/rules/layer-pattern-contract-check.js +1 -1
  740. package/src/rules/rule-registry.js +70 -57
  741. package/src/storage/artifact-storage-context.js +2 -2
  742. package/src/storage/backend-selection.js +43 -0
  743. package/src/storage/distribution-transport.js +5 -0
  744. package/src/storage/download/integrity.d.ts +33 -0
  745. package/src/storage/download/integrity.js +130 -0
  746. package/src/storage/download/resume-inspection.d.ts +22 -0
  747. package/src/storage/download/resume-inspection.js +134 -0
  748. package/src/storage/download/retry.js +57 -0
  749. package/src/storage/download/state.js +131 -0
  750. package/src/storage/download/transport.js +99 -0
  751. package/src/storage/downloader.js +74 -453
  752. package/src/storage/model-cache.d.ts +1 -0
  753. package/src/storage/model-cache.js +51 -11
  754. package/src/storage/shard-manager.d.ts +5 -0
  755. package/src/storage/shard-manager.js +50 -369
  756. package/src/storage/shards/index.js +60 -0
  757. package/src/storage/shards/integrity.js +225 -0
  758. package/src/storage/shards/lifecycle.js +22 -0
  759. package/src/storage/source-artifact-store.js +14 -4
  760. package/src/tooling/boundary-evidence.d.ts +4 -4
  761. package/src/tooling/boundary-evidence.js +20 -20
  762. package/src/tooling/browser-command-runner.js +13 -6
  763. package/src/tooling/capsule-v2.d.ts +11 -0
  764. package/src/tooling/capsule-v2.js +58 -0
  765. package/src/tooling/capsule.js +20 -0
  766. package/src/tooling/command-api-constants.js +1 -1
  767. package/src/tooling/command-api-family-normalizers.js +163 -5
  768. package/src/tooling/command-api.d.ts +42 -3
  769. package/src/tooling/command-api.js +7 -2
  770. package/src/tooling/command-envelope.js +1 -1
  771. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  772. package/src/tooling/command-runner.html +49 -1
  773. package/src/tooling/command-validation.js +1 -24
  774. package/src/tooling/conversion-run-timing.js +24 -0
  775. package/src/tooling/diagnose-runner.js +11 -9
  776. package/src/tooling/distribution-shard-transport.js +11 -0
  777. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  778. package/src/tooling/execution-cost-ledger.js +197 -15
  779. package/src/tooling/hf-registry/catalog-io.js +63 -0
  780. package/src/tooling/litert/package.js +863 -0
  781. package/src/tooling/litert-package-runtime.js +3 -849
  782. package/src/tooling/model-capsule-forge.js +478 -0
  783. package/src/tooling/node-browser/execution.d.ts +40 -0
  784. package/src/tooling/node-browser/execution.js +308 -0
  785. package/src/tooling/node-browser/static-server.js +188 -0
  786. package/src/tooling/node-browser/transport.d.ts +80 -0
  787. package/src/tooling/node-browser/transport.js +622 -0
  788. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  789. package/src/tooling/node-browser-command-runner.js +10 -1112
  790. package/src/tooling/node-capsule-artifact-store.js +44 -0
  791. package/src/tooling/node-command-execution.js +237 -0
  792. package/src/tooling/node-command-runner.js +26 -205
  793. package/src/tooling/node-converter/input.js +521 -0
  794. package/src/tooling/node-converter/output.js +244 -0
  795. package/src/tooling/node-converter-chunk-batches.js +21 -0
  796. package/src/tooling/node-converter.js +7 -720
  797. package/src/tooling/node-source-runtime.js +16 -276
  798. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  799. package/src/tooling/node-webgpu.js +178 -453
  800. package/src/tooling/opfs-cache.d.ts +5 -0
  801. package/src/tooling/opfs-cache.js +4 -0
  802. package/src/tooling/production-release.d.ts +19 -0
  803. package/src/tooling/production-release.js +647 -0
  804. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  805. package/src/tooling/program-bundle/materialize.js +324 -0
  806. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  807. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  808. package/src/tooling/program-bundle/validation.js +31 -0
  809. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  810. package/src/tooling/program-bundle-parity.d.ts +11 -4
  811. package/src/tooling/program-bundle-parity.js +262 -109
  812. package/src/tooling/program-bundle.d.ts +7 -0
  813. package/src/tooling/program-bundle.js +186 -591
  814. package/src/tooling/provider-v1-contract.d.ts +61 -0
  815. package/src/tooling/provider-v1-contract.js +192 -0
  816. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  817. package/src/tooling/registered-variant-calibration-job.js +210 -0
  818. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  819. package/src/tooling/registered-variant-calibration.js +57 -7
  820. package/src/tooling/runtime-optimization/candidates.js +664 -0
  821. package/src/tooling/runtime-optimization-campaign.js +166 -0
  822. package/src/tooling/runtime-optimization-index.js +16 -1
  823. package/src/tooling/runtime-optimization.d.ts +44 -0
  824. package/src/tooling/runtime-optimization.js +13 -656
  825. package/src/tooling/source-intake.d.ts +1 -1
  826. package/src/tooling/source-intake.js +34 -17
  827. package/src/tooling/source-package-profiles.js +1 -1
  828. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  829. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  830. package/src/tooling/source-runtime/node-loader.js +258 -0
  831. package/src/tooling/source-runtime-bundle.js +26 -411
  832. package/src/tooling/source-runtime-converter-config.js +1 -1
  833. package/src/tooling-experimental-exports.js +0 -5
  834. package/src/tooling-exports/evidence.d.ts +14 -3
  835. package/src/tooling-exports/evidence.js +14 -3
  836. package/src/tooling-exports/storage.d.ts +5 -0
  837. package/src/tooling-exports/storage.js +4 -0
  838. package/src/tooling-exports.d.ts +3 -0
  839. package/src/tooling-exports.js +2 -0
  840. package/src/tooling-exports.shared.d.ts +17 -3
  841. package/src/tooling-exports.shared.js +9 -3
  842. package/src/training.d.ts +6 -6
  843. package/src/training.js +4 -4
  844. package/src/version.js +1 -1
  845. package/tests/kernels/browser/kernel-suite.js +512 -0
  846. package/tests/kernels/browser/test-page.js +169 -14
  847. package/tests/kernels/harness/tolerance.js +1 -0
  848. package/tests/kernels/reference/index.js +13 -1
  849. package/tests/kernels/reference/vision.js +160 -0
  850. package/src/converter/quantizer.d.ts +0 -121
  851. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  852. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  853. package/src/gpu/kernels/utils.d.ts +0 -147
  854. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  855. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  856. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  857. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  858. package/src/rules/converter/execution.rules.json +0 -20
  859. package/src/rules/converter/tensor-roles.rules.json +0 -13
  860. package/src/rules/converter/tokenizer.rules.json +0 -12
  861. package/src/rules/inference/attention.rules.json +0 -69
  862. package/src/rules/inference/config.rules.json +0 -82
  863. package/src/rules/inference/dtype.rules.json +0 -113
  864. package/src/rules/inference/execution.rules.json +0 -195
  865. package/src/rules/inference/ffn.rules.json +0 -123
  866. package/src/rules/inference/layer-pattern.rules.json +0 -16
  867. package/src/rules/inference/layer.rules.json +0 -7
  868. package/src/rules/inference/moe.rules.json +0 -145
  869. package/src/rules/kernels/attention.rules.json +0 -68
  870. package/src/rules/kernels/conv2d.rules.json +0 -6
  871. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  872. package/src/rules/kernels/dequant.rules.json +0 -58
  873. package/src/rules/kernels/energy.rules.json +0 -22
  874. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  875. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  876. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  877. package/src/rules/kernels/gather.rules.json +0 -93
  878. package/src/rules/kernels/gelu.rules.json +0 -11
  879. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  880. package/src/rules/kernels/groupnorm.rules.json +0 -10
  881. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  882. package/src/rules/kernels/layernorm.rules.json +0 -6
  883. package/src/rules/kernels/linear-attention.rules.json +0 -6
  884. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  885. package/src/rules/kernels/matmul.rules.json +0 -95
  886. package/src/rules/kernels/modulate.rules.json +0 -6
  887. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  888. package/src/rules/kernels/moe.rules.json +0 -14
  889. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  890. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  891. package/src/rules/kernels/relu.rules.json +0 -6
  892. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  893. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  894. package/src/rules/kernels/residual.rules.json +0 -12
  895. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  896. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  897. package/src/rules/kernels/rope-qk.rules.json +0 -6
  898. package/src/rules/kernels/rope.rules.json +0 -6
  899. package/src/rules/kernels/sample.rules.json +0 -6
  900. package/src/rules/kernels/scale.rules.json +0 -8
  901. package/src/rules/kernels/silu.rules.json +0 -22
  902. package/src/rules/kernels/softmax.rules.json +0 -25
  903. package/src/rules/kernels/split-qg.rules.json +0 -6
  904. package/src/rules/kernels/split-qkv.rules.json +0 -6
  905. package/src/rules/kernels/upsample2d.rules.json +0 -6
  906. package/src/rules/loader/tensor-loader.rules.json +0 -42
  907. package/src/rules/loader/weights.rules.json +0 -42
  908. package/src/rules/tooling/command-runtime.rules.json +0 -65
  909. package/src/tooling/hf-registry-utils.js +0 -395
  910. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  911. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  912. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  913. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  914. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  915. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  916. /package/src/{utils → formats}/array-buffer.js +0 -0
  917. /package/src/{utils → formats}/canonical-hash.js +0 -0
  918. /package/src/{utils → formats}/clone-json.js +0 -0
  919. /package/src/{utils → formats}/encode-utf8.js +0 -0
  920. /package/src/{utils → formats}/load-json.js +0 -0
  921. /package/src/{utils → formats}/plain-object.js +0 -0
  922. /package/src/{utils → formats}/sha256.js +0 -0
  923. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  924. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  925. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  926. /package/src/{utils → storage}/runtime-env.js +0 -0
  927. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -0,0 +1,304 @@
1
+ import { releaseBuffer } from '../../../memory/buffer-pool.js';
2
+ import { formatChatMessages } from './chat-format.js';
3
+ import { assertNotAborted } from './abort-contract.js';
4
+ import {
5
+ assertMultimodalSequenceCapacity,
6
+ buildConservativeMultimodalGenerationOptions,
7
+ expandImagePlaceholderTokenIds,
8
+ resolveMultimodalMaxTokens,
9
+ resolveSingleSpecialTokenId,
10
+ } from './modality-token-contract.js';
11
+ import { runProbes } from './probes.js';
12
+
13
+ export function createImageTranscriptionResourceScope(
14
+ pipeline,
15
+ encodeResult,
16
+ releaseFeature = releaseBuffer
17
+ ) {
18
+ if (!pipeline || typeof pipeline !== 'object') {
19
+ throw new Error('[Pipeline] Image transcription resource scope requires a pipeline.');
20
+ }
21
+ if (!encodeResult || typeof encodeResult !== 'object') {
22
+ throw new Error('[Pipeline] Image transcription resource scope requires an encode result.');
23
+ }
24
+ if (typeof releaseFeature !== 'function') {
25
+ throw new Error('[Pipeline] Image transcription resource scope requires a release function.');
26
+ }
27
+
28
+ const originalRopeCos = pipeline.ropeFreqsCos;
29
+ const originalRopeSin = pipeline.ropeFreqsSin;
30
+ let glmOcrRopeOverride = null;
31
+ let hasRun = false;
32
+ let runActive = false;
33
+ let hasFinished = false;
34
+ let generationStarted = false;
35
+
36
+ return {
37
+ setGlmOcrRopeOverride(override) {
38
+ if (generationStarted || hasFinished) {
39
+ throw new Error('[Pipeline] Cannot register an MRoPE override after transcription started.');
40
+ }
41
+ if (
42
+ !override
43
+ || typeof override !== 'object'
44
+ || !override.cos
45
+ || !override.sin
46
+ || typeof override.release !== 'function'
47
+ ) {
48
+ throw new Error('[Pipeline] GLM-OCR MRoPE override is missing cos, sin, or release.');
49
+ }
50
+ if (glmOcrRopeOverride) {
51
+ throw new Error('[Pipeline] GLM-OCR MRoPE override is already registered.');
52
+ }
53
+ glmOcrRopeOverride = override;
54
+ pipeline.ropeFreqsCos = override.cos;
55
+ pipeline.ropeFreqsSin = override.sin;
56
+ },
57
+
58
+ async runGeneration(task) {
59
+ if (!runActive) {
60
+ throw new Error('[Pipeline] Image transcription generation requires an active resource scope.');
61
+ }
62
+ if (generationStarted) {
63
+ throw new Error('[Pipeline] Image transcription generation may run only once.');
64
+ }
65
+ if (typeof task !== 'function') {
66
+ throw new Error('[Pipeline] Image transcription generation requires a task.');
67
+ }
68
+ generationStarted = true;
69
+ return task();
70
+ },
71
+
72
+ async run(task) {
73
+ if (hasRun) {
74
+ throw new Error('[Pipeline] Image transcription resource scope may run only once.');
75
+ }
76
+ if (typeof task !== 'function') {
77
+ throw new Error('[Pipeline] Image transcription resource scope requires a task.');
78
+ }
79
+ hasRun = true;
80
+ runActive = true;
81
+ try {
82
+ return await task();
83
+ } finally {
84
+ runActive = false;
85
+ hasFinished = true;
86
+ try {
87
+ if (glmOcrRopeOverride) {
88
+ pipeline.ropeFreqsCos = originalRopeCos;
89
+ pipeline.ropeFreqsSin = originalRopeSin;
90
+ glmOcrRopeOverride.release();
91
+ }
92
+ } finally {
93
+ if (encodeResult.features) {
94
+ releaseFeature(encodeResult.features);
95
+ }
96
+ }
97
+ }
98
+ },
99
+ };
100
+ }
101
+
102
+ export async function transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget, signal }) {
103
+ assertNotAborted(signal);
104
+ if (!this.visionCapable) {
105
+ throw new Error(
106
+ 'Pipeline does not support image transcription (no image_token_id in manifest).'
107
+ );
108
+ }
109
+ await this._ensureVisionWeightsLoaded();
110
+ assertNotAborted(signal);
111
+
112
+ this.reset();
113
+
114
+ // Lazy-load vision module (avoids GPU kernel dependency for text-only pipelines)
115
+ const { encodeImage } = await import('../vision/index.js');
116
+
117
+ // Step 1: Encode image through vision pipeline
118
+ const probeVisionTensor = async (stage, buffer, options = {}) => runProbes(stage, buffer, {
119
+ ...options,
120
+ probes: this.runtimeConfig.shared.debug.probes,
121
+ recorder: null,
122
+ operatorDiagnostics: this.operatorDiagnostics,
123
+ dtype: options.dtype ?? 'f32',
124
+ });
125
+ const encodeResult = await encodeImage({
126
+ pixels: imageBytes,
127
+ width,
128
+ height,
129
+ visionConfig: this.visionConfig,
130
+ weights: this.visionWeights,
131
+ softTokenBudget,
132
+ probeTensor: probeVisionTensor,
133
+ });
134
+ await probeVisionTensor('image_features_out', encodeResult.features, {
135
+ numTokens: encodeResult.numTokens,
136
+ hiddenSize: this.visionConfig.outHiddenSize,
137
+ });
138
+ const transcriptionResources = createImageTranscriptionResourceScope(this, encodeResult);
139
+
140
+ return transcriptionResources.run(async () => {
141
+ // Step 2: Build the multimodal prompt from the model's chat template and
142
+ // expand the single <|image|> placeholder into the exact visual-token span.
143
+ const visionArchitecture = this.visionConfig?.visionArchitecture;
144
+ const isGemma4 = visionArchitecture === 'gemma4';
145
+ const isGlmOcr = visionArchitecture === 'glmocr';
146
+ if (!isGemma4 && !isGlmOcr) {
147
+ throw new Error(
148
+ `[Pipeline] transcribeImage: unsupported vision architecture "${visionArchitecture ?? 'unknown'}". ` +
149
+ 'Supported transcription paths: Gemma 4 and GLM-OCR.'
150
+ );
151
+ }
152
+ const requestedPrompt = prompt ?? (isGlmOcr
153
+ ? 'Text Recognition:'
154
+ : 'Describe the image in one short sentence.');
155
+ const imageTokenId = this.visionConfig?.imageTokenId ?? this.modelConfig?.imageTokenId;
156
+ if (imageTokenId == null) {
157
+ throw new Error(
158
+ 'Pipeline missing image_token_id. Re-convert the model with image token metadata.'
159
+ );
160
+ }
161
+ const expectedTemplateType = isGlmOcr ? 'glmocr' : 'gemma4';
162
+ const templateType = this.modelConfig?.chatTemplateType ?? expectedTemplateType;
163
+ if (templateType !== expectedTemplateType) {
164
+ throw new Error(
165
+ `[Pipeline] transcribeImage: ${visionArchitecture} requires chatTemplate.type="${expectedTemplateType}", ` +
166
+ `got ${JSON.stringify(templateType)}.`
167
+ );
168
+ }
169
+ const chatTemplateThinking = this.modelConfig?.chatTemplateThinking;
170
+ const chatOptions = chatTemplateThinking == null
171
+ ? undefined
172
+ : { thinking: chatTemplateThinking };
173
+ const multimodalPrompt = formatChatMessages([
174
+ {
175
+ role: 'user',
176
+ content: [
177
+ { type: 'image' },
178
+ { type: 'text', text: requestedPrompt },
179
+ ],
180
+ },
181
+ ], templateType, chatOptions);
182
+ const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
183
+ const imageTokenSpanLength = encodeResult.numTokens;
184
+ const effectiveBudget = softTokenBudget ?? this.visionConfig?.defaultOutputLength;
185
+ const maxImageTokenSpanLength = Number(effectiveBudget);
186
+ if (!Number.isFinite(maxImageTokenSpanLength) || maxImageTokenSpanLength < 1 || Math.floor(maxImageTokenSpanLength) !== maxImageTokenSpanLength) {
187
+ throw new Error(
188
+ `[Pipeline] transcribeImage: invalid soft token budget ${effectiveBudget}. ` +
189
+ 'Expected a positive integer from the resolved vision config or softTokenBudget parameter.'
190
+ );
191
+ }
192
+ if (imageTokenSpanLength > maxImageTokenSpanLength) {
193
+ throw new Error(
194
+ `[Pipeline] transcribeImage: encoded ${visionArchitecture} image produced ${imageTokenSpanLength} soft tokens, ` +
195
+ `which exceeds the effective soft token budget=${maxImageTokenSpanLength}.`
196
+ );
197
+ }
198
+ const expansionOptions = isGemma4
199
+ ? {
200
+ boiTokenId: resolveSingleSpecialTokenId(this.tokenizer, '<|image>', 'Gemma 4 BOI token'),
201
+ eoiTokenId: resolveSingleSpecialTokenId(this.tokenizer, '<image|>', 'Gemma 4 EOI token'),
202
+ }
203
+ : {};
204
+ const { inputIds: fullTokenIds, imageStartOffset } = expandImagePlaceholderTokenIds(
205
+ promptTokenIds,
206
+ imageTokenId,
207
+ imageTokenSpanLength,
208
+ expansionOptions
209
+ );
210
+ const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
211
+ if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
212
+ throw new Error(
213
+ `[Pipeline] transcribeImage: ${visionArchitecture} multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
214
+ );
215
+ }
216
+
217
+ // Step 3: Generate with embedding override at the image token offset.
218
+ const tokens = [];
219
+ const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
220
+ assertMultimodalSequenceCapacity({
221
+ inputTokenCount: fullTokenIds.length,
222
+ maxTokens: maxGen,
223
+ maxSeqLen: this.kvCache?.maxSeqLen,
224
+ });
225
+ const stopTokenIds = this.modelConfig.stopTokenIds;
226
+
227
+ if (isGlmOcr) {
228
+ if (
229
+ this.modelConfig.ropeInterleaved !== true
230
+ || this.modelConfig.mropeInterleaved !== true
231
+ || !Array.isArray(this.modelConfig.mropeSection)
232
+ ) {
233
+ throw new Error(
234
+ '[Pipeline] transcribeImage: GLM-OCR requires interleaved 3-axis MRoPE metadata.'
235
+ );
236
+ }
237
+ const { buildGlmOcrRopePositionPlan, uploadGlmOcrRopeFrequencies } = await import(
238
+ '../vision/glmocr-rope.js'
239
+ );
240
+ const [gridTemporal, gridHeight, gridWidth] = encodeResult.gridThw ?? [];
241
+ if (gridTemporal !== 1) {
242
+ throw new Error(
243
+ `[Pipeline] transcribeImage: GLM-OCR image path requires grid temporal extent 1, got ${gridTemporal}.`
244
+ );
245
+ }
246
+ const positionPlan = buildGlmOcrRopePositionPlan({
247
+ promptLength: fullTokenIds.length,
248
+ capacity: fullTokenIds.length + maxGen,
249
+ imageStartOffset,
250
+ imageTokenLength: imageTokenSpanLength,
251
+ gridHeight,
252
+ gridWidth,
253
+ mergeSize: this.visionConfig.spatialMergeSize,
254
+ });
255
+ const glmOcrRopeOverride = await uploadGlmOcrRopeFrequencies(positionPlan, {
256
+ rotaryDim: this.modelConfig.ropeRotaryDim,
257
+ frequencyBaseDim: this.modelConfig.ropeFrequencyBaseDim,
258
+ ropeTheta: this.modelConfig.ropeTheta,
259
+ mropeSection: this.modelConfig.mropeSection,
260
+ });
261
+ transcriptionResources.setGlmOcrRopeOverride(glmOcrRopeOverride);
262
+ }
263
+
264
+ const generationOptions = {
265
+ inputIds: fullTokenIds,
266
+ embeddingOverrides: {
267
+ prefixLength: encodeResult.numTokens,
268
+ offset: imageStartOffset,
269
+ embeddings: encodeResult.features,
270
+ },
271
+ __internalEmbeddingInputSpan: {
272
+ offset: imageStartOffset,
273
+ length: encodeResult.numTokens,
274
+ tokenId: padTokenId,
275
+ },
276
+ ...(isGemma4
277
+ ? {
278
+ __internalMultimodalBidirectionalSpan: {
279
+ offset: imageStartOffset,
280
+ length: encodeResult.numTokens,
281
+ },
282
+ }
283
+ : {}),
284
+ maxTokens: maxGen,
285
+ temperature: 0,
286
+ topK: 1,
287
+ topP: 1,
288
+ repetitionPenalty: 1,
289
+ };
290
+ const generation = await transcriptionResources.runGeneration(
291
+ () => this.generator.generateTokenIds(
292
+ '',
293
+ buildConservativeMultimodalGenerationOptions(generationOptions)
294
+ )
295
+ );
296
+ for (const token of generation.tokenIds ?? []) {
297
+ if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
298
+ tokens.push(token);
299
+ }
300
+ const text = this.tokenizer.decode(tokens);
301
+ return { text, tokens };
302
+ });
303
+ }
304
+
@@ -47,8 +47,7 @@ function normalizeProjectionDtype(weight) {
47
47
  const dtype = typeof weight?.dtype === 'string'
48
48
  ? weight.dtype.toLowerCase()
49
49
  : null;
50
- if (dtype === 'bf16') return 'f16';
51
- if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') return dtype;
50
+ if (dtype === 'bf16' || dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') return dtype;
52
51
  return null;
53
52
  }
54
53
 
@@ -73,8 +72,8 @@ function resolveProjectionStorageFormat(weight, expectedRows, hiddenSize) {
73
72
  };
74
73
  }
75
74
 
76
- if (dtype === 'f16' || dtype === 'f32') {
77
- const bytesPerElement = dtype === 'f16' ? 2 : 4;
75
+ if (dtype === 'bf16' || dtype === 'f16' || dtype === 'f32') {
76
+ const bytesPerElement = dtype === 'f32' ? 4 : 2;
78
77
  const byteLength = expectedRows * hiddenSize * bytesPerElement;
79
78
  if (weight?.buffer?.size < byteLength) {
80
79
  return null;
@@ -157,7 +156,6 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
157
156
  const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
158
157
  const qSize = numHeads * headDim;
159
158
  const hasAttentionOutputGate = modelConfig?.attentionOutputGate === true;
160
- const qProjSize = hasAttentionOutputGate ? qSize * 2 : qSize;
161
159
  const kSize = numKVHeads * headDim;
162
160
  const vSize = numKVHeads * headDim;
163
161
  const qkvSize = qSize + kSize + vSize;
@@ -211,6 +209,8 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
211
209
  continue;
212
210
  }
213
211
 
212
+ const hasSeparateGateProjection = hasAttentionOutputGate && Boolean(weights.qGateProj);
213
+ const qProjSize = hasAttentionOutputGate && !hasSeparateGateProjection ? qSize * 2 : qSize;
214
214
  const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
215
215
  const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
216
216
  const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
@@ -225,8 +225,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
225
225
  continue;
226
226
  }
227
227
 
228
+ // BF16 is a distinct numeric encoding, not an F16 storage alias. The
229
+ // current fused-QKV kernels do not declare BF16 weight decoding, so keep
230
+ // these source projections separate until a signed plan selects a BF16
231
+ // fused implementation explicitly.
232
+ if (qFormat.dtype === 'bf16') {
233
+ log.debug('QKV Fusion', `Layer ${l}: BF16 projections require split execution`);
234
+ continue;
235
+ }
236
+
228
237
  if (requiresQ4K && qFormat.dtype !== 'q4k') {
229
- log.debug('QKV Fusion', `Layer ${l}: qkv_proj requires Q4K weights, skipping ${qFormat.dtype} pack`);
238
+ log.debug('QKV Fusion', `Layer ${l}: qkv_proj requires Q4K weights, skipping ${qFormat.dtype} capsule`);
230
239
  continue;
231
240
  }
232
241
  if (qFormat.dtype === 'q4k' && !allowQ4K) {
@@ -248,7 +257,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
248
257
  }
249
258
 
250
259
  const qBytes = qSize * qFormat.bytesPerRow;
251
- const qGateBytes = hasAttentionOutputGate ? qBytes : 0;
260
+ const qGateBytes = hasAttentionOutputGate && !hasSeparateGateProjection ? qBytes : 0;
252
261
 
253
262
  // Create fused QKV buffer: [qkvSize, hiddenSize] row-major.
254
263
  // attentionOutputGate models keep Q rows in qkvProj and gate rows in qGateProj.
@@ -257,7 +266,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
257
266
  size: qBytes + kFormat.byteLength + vFormat.byteLength,
258
267
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
259
268
  });
260
- const qGateBuffer = hasAttentionOutputGate
269
+ const qGateBuffer = hasAttentionOutputGate && !hasSeparateGateProjection
261
270
  ? device.createBuffer({
262
271
  label: `layer_${l}_q_gate_proj`,
263
272
  size: qGateBytes,
@@ -292,7 +301,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
292
301
  }
293
302
 
294
303
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
295
- if (hasAttentionOutputGate) {
304
+ if (hasAttentionOutputGate && !hasSeparateGateProjection) {
296
305
  for (let head = 0; head < numHeads; head++) {
297
306
  const srcHeadOffset = head * headDim * 2 * qFormat.bytesPerRow;
298
307
  const dstHeadOffset = head * headDim * qFormat.bytesPerRow;
@@ -1,14 +1,16 @@
1
1
  import { getDevice } from '../../../gpu/device.js';
2
- import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
2
+ import { isBufferActive } from '../../../memory/buffer-pool.js';
3
3
  import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
4
4
  import { log } from '../../../debug/index.js';
5
+ import { runRoPEPrecompute } from '../../../gpu/kernels/rope-precompute.js';
6
+ import { getShaderScopeCacheKey } from '../../../gpu/kernels/shader-source-scope.js';
5
7
 
6
8
  // ============================================================================
7
9
  // RoPE Initialization
8
10
  // ============================================================================
9
11
 
10
12
 
11
- function computeRoPEFreqsForTheta(
13
+ async function computeRoPEFreqsForTheta(
12
14
  theta,
13
15
  rotaryDim,
14
16
  frequencyBaseDim,
@@ -17,120 +19,15 @@ function computeRoPEFreqsForTheta(
17
19
  ropeScalingType,
18
20
  ropeScaling
19
21
  ) {
20
- const halfDim = rotaryDim / 2;
21
-
22
- // Compute base frequencies: theta_i = 1 / (base^(2i/d))
23
- const freqs = new Float32Array(halfDim);
24
- for (let i = 0; i < halfDim; i++) {
25
- freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
26
- }
27
-
28
- // Compute per-dimension scaling factors
29
- const scales = new Float32Array(halfDim);
30
- const isYarn = ropeScalingType === 'yarn';
31
- const isLongRoPE = ropeScalingType === 'longrope';
32
- let magnitudeScale = 1;
33
- if (isYarn) {
34
- // YARN scaling - validate ALL required params (fail fast on incomplete manifest)
35
- if (ropeScaling?.factor == null ||
36
- ropeScaling?.beta_fast == null || ropeScaling?.beta_slow == null ||
37
- ropeScaling?.original_max_position_embeddings == null) {
38
- throw new Error(
39
- `RoPE scaling type is 'yarn' but YARN params missing. ` +
40
- `Manifest must provide factor, beta_fast, beta_slow, and original_max_position_embeddings. ` +
41
- `Got: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}, ` +
42
- `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
43
- );
44
- }
45
- // Extract validated YARN params (no hidden defaults - all guaranteed non-null)
46
- const yarnFactor = ropeScaling.factor;
47
- const yarnBetaFast = ropeScaling.beta_fast;
48
- const yarnBetaSlow = ropeScaling.beta_slow;
49
- const originalMaxPos = ropeScaling.original_max_position_embeddings;
50
-
51
- // YARN: wavelength-based interpolation
52
- for (let i = 0; i < halfDim; i++) {
53
- const wavelength = (2 * Math.PI) / freqs[i];
54
- const lowThresh = originalMaxPos / yarnBetaSlow;
55
- const highThresh = originalMaxPos / yarnBetaFast;
56
-
57
- if (wavelength < highThresh) {
58
- scales[i] = 1.0;
59
- } else if (wavelength > lowThresh) {
60
- scales[i] = yarnFactor;
61
- } else {
62
- const t = (wavelength - highThresh) / (lowThresh - highThresh);
63
- scales[i] = 1.0 + (yarnFactor - 1.0) * t;
64
- }
65
- }
66
- } else if (isLongRoPE) {
67
- if (
68
- !Array.isArray(ropeScaling?.short_factor)
69
- || !Array.isArray(ropeScaling?.long_factor)
70
- || ropeScaling?.original_max_position_embeddings == null
71
- ) {
72
- throw new Error(
73
- `RoPE scaling type is 'longrope' but LongRoPE params are missing. ` +
74
- `Manifest must provide short_factor, long_factor, and original_max_position_embeddings. ` +
75
- `Got: short_factor=${Array.isArray(ropeScaling?.short_factor) ? ropeScaling.short_factor.length : ropeScaling?.short_factor}, ` +
76
- `long_factor=${Array.isArray(ropeScaling?.long_factor) ? ropeScaling.long_factor.length : ropeScaling?.long_factor}, ` +
77
- `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
78
- );
79
- }
80
- if (ropeScaling.short_factor.length !== halfDim || ropeScaling.long_factor.length !== halfDim) {
81
- throw new Error(
82
- `LongRoPE factor length mismatch: expected ${halfDim}, ` +
83
- `got short_factor=${ropeScaling.short_factor.length}, long_factor=${ropeScaling.long_factor.length}.`
84
- );
85
- }
86
- const originalMaxPos = Number(ropeScaling.original_max_position_embeddings);
87
- if (!Number.isFinite(originalMaxPos) || originalMaxPos <= 0) {
88
- throw new Error(`LongRoPE original_max_position_embeddings must be positive; got "${String(originalMaxPos)}".`);
89
- }
90
- const selectedFactors = maxSeqLen > originalMaxPos
91
- ? ropeScaling.long_factor
92
- : ropeScaling.short_factor;
93
- for (let i = 0; i < halfDim; i++) {
94
- const factor = Number(selectedFactors[i]);
95
- if (!Number.isFinite(factor) || factor <= 0) {
96
- throw new Error(`LongRoPE factor[${i}] must be a positive finite number; got "${String(selectedFactors[i])}".`);
97
- }
98
- scales[i] = factor;
99
- }
100
- magnitudeScale = Math.sqrt(1 + Math.log(maxSeqLen / originalMaxPos) / Math.log(originalMaxPos));
101
- if (!Number.isFinite(magnitudeScale) || magnitudeScale <= 0) {
102
- throw new Error(
103
- `LongRoPE magnitude scale is invalid for maxSeqLen=${maxSeqLen}, ` +
104
- `original_max_position_embeddings=${originalMaxPos}.`
105
- );
106
- }
107
- } else {
108
- // Linear scaling: uniform across all dimensions
109
- if (ropeScalingType != null && ropeScalingType !== 'linear') {
110
- throw new Error(
111
- `Unsupported RoPE scaling type "${ropeScalingType}". ` +
112
- 'Supported types: null, "linear", "yarn", "longrope".'
113
- );
114
- }
115
- for (let i = 0; i < halfDim; i++) {
116
- scales[i] = ropeScale;
117
- }
118
- }
119
-
120
- // Compute cos/sin for each position
121
- const cosValues = new Float32Array(maxSeqLen * halfDim);
122
- const sinValues = new Float32Array(maxSeqLen * halfDim);
123
-
124
- for (let pos = 0; pos < maxSeqLen; pos++) {
125
- for (let i = 0; i < halfDim; i++) {
126
- const scaledPos = pos / scales[i];
127
- const angle = scaledPos * freqs[i];
128
- cosValues[pos * halfDim + i] = Math.cos(angle) * magnitudeScale;
129
- sinValues[pos * halfDim + i] = Math.sin(angle) * magnitudeScale;
130
- }
131
- }
132
-
133
- return { cos: cosValues, sin: sinValues };
22
+ return runRoPEPrecompute({
23
+ theta,
24
+ rotaryDim,
25
+ frequencyBaseDim,
26
+ maxSeqLen,
27
+ ropeScale,
28
+ scalingType: ropeScalingType,
29
+ scaling: ropeScaling,
30
+ });
134
31
  }
135
32
 
136
33
  function isSameRoPEScalingConfig(
@@ -157,8 +54,6 @@ function isSameRoPEScalingConfig(
157
54
  }
158
55
 
159
56
  const GPU_ROPE_BUFFER_CACHE = new WeakMap();
160
- const CPU_ROPE_BUFFER_CACHE = new Map();
161
-
162
57
  function isLiveCachedRopeBuffer(buffer) {
163
58
  return buffer == null || isBufferActive(buffer);
164
59
  }
@@ -239,7 +134,7 @@ function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
239
134
 
240
135
 
241
136
  export async function initRoPEFrequencies(config, useGPU) {
242
- const cacheKey = buildRoPECacheKey(config);
137
+ const cacheKey = `${getShaderScopeCacheKey()}:${buildRoPECacheKey(config)}`;
243
138
  const {
244
139
  headDim,
245
140
  localHeadDim,
@@ -309,8 +204,25 @@ export async function initRoPEFrequencies(config, useGPU) {
309
204
  const isYarn = ropeScalingType === 'yarn';
310
205
  const isLocalYarn = resolvedLocalScalingType === 'yarn';
311
206
 
207
+ const device = getDevice();
208
+ if (!useGPU || !device) {
209
+ throw new Error('RoPE frequency initialization requires the declared WebGPU execution path.');
210
+ }
211
+ let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
212
+ if (!perDeviceCache) {
213
+ perDeviceCache = new Map();
214
+ GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
215
+ }
216
+ const cachedBuffers = perDeviceCache.get(cacheKey);
217
+ if (cachedBuffers) {
218
+ if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
219
+ return cachedBuffers;
220
+ }
221
+ perDeviceCache.delete(cacheKey);
222
+ }
223
+
312
224
  // Compute global (full_attention) frequencies
313
- const globalFreqs = computeRoPEFreqsForTheta(
225
+ const globalFreqs = await computeRoPEFreqsForTheta(
314
226
  ropeTheta,
315
227
  resolvedRotaryDim,
316
228
  resolvedFrequencyBaseDim,
@@ -335,7 +247,7 @@ export async function initRoPEFrequencies(config, useGPU) {
335
247
  resolvedLocalScaling
336
248
  );
337
249
  if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
338
- localFreqs = computeRoPEFreqsForTheta(
250
+ localFreqs = await computeRoPEFreqsForTheta(
339
251
  resolvedLocalTheta,
340
252
  resolvedLocalRotaryDim,
341
253
  resolvedLocalFrequencyBaseDim,
@@ -366,73 +278,9 @@ export async function initRoPEFrequencies(config, useGPU) {
366
278
  );
367
279
  }
368
280
 
369
- // Upload to GPU if available
370
- const device = getDevice();
371
- if (device && useGPU) {
372
- let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
373
- if (!perDeviceCache) {
374
- perDeviceCache = new Map();
375
- GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
376
- }
377
- const cachedBuffers = perDeviceCache.get(cacheKey);
378
- if (cachedBuffers) {
379
- if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
380
- return cachedBuffers;
381
- }
382
- perDeviceCache.delete(cacheKey);
383
- }
384
- let cosBuffer = null;
385
- let sinBuffer = null;
386
- let localCosBuffer = null;
387
- let localSinBuffer = null;
388
- try {
389
- cosBuffer = acquireBuffer(globalFreqs.cos.byteLength, undefined, 'rope_cos');
390
- sinBuffer = acquireBuffer(globalFreqs.sin.byteLength, undefined, 'rope_sin');
391
- device.queue.writeBuffer(cosBuffer, 0, globalFreqs.cos.buffer, globalFreqs.cos.byteOffset, globalFreqs.cos.byteLength);
392
- device.queue.writeBuffer(sinBuffer, 0, globalFreqs.sin.buffer, globalFreqs.sin.byteOffset, globalFreqs.sin.byteLength);
393
-
394
- if (localFreqs) {
395
- localCosBuffer = acquireBuffer(localFreqs.cos.byteLength, undefined, 'rope_local_cos');
396
- localSinBuffer = acquireBuffer(localFreqs.sin.byteLength, undefined, 'rope_local_sin');
397
- device.queue.writeBuffer(localCosBuffer, 0, localFreqs.cos.buffer, localFreqs.cos.byteOffset, localFreqs.cos.byteLength);
398
- device.queue.writeBuffer(localSinBuffer, 0, localFreqs.sin.buffer, localFreqs.sin.byteOffset, localFreqs.sin.byteLength);
399
- }
400
- } catch (error) {
401
- for (const buffer of [cosBuffer, sinBuffer, localCosBuffer, localSinBuffer]) {
402
- if (buffer) {
403
- releaseBuffer(buffer);
404
- }
405
- }
406
- throw error;
407
- }
408
-
409
- log.debug(
410
- 'Pipeline',
411
- `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
412
- `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
413
- `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
414
- `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
415
- `interleaved=${mropeInterleaved === true}`
416
- );
417
-
418
- const buffers = {
419
- cos: cosBuffer,
420
- sin: sinBuffer,
421
- localCos: localCosBuffer,
422
- localSin: localSinBuffer,
423
- };
424
- perDeviceCache.set(cacheKey, buffers);
425
- return buffers;
426
- }
427
-
428
- const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
429
- if (cachedCpuBuffers) {
430
- return cachedCpuBuffers;
431
- }
432
-
433
281
  log.debug(
434
282
  'Pipeline',
435
- `RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
283
+ `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
436
284
  `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
437
285
  `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
438
286
  `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
@@ -442,10 +290,10 @@ export async function initRoPEFrequencies(config, useGPU) {
442
290
  const buffers = {
443
291
  cos: globalFreqs.cos,
444
292
  sin: globalFreqs.sin,
445
- localCos: localFreqs?.cos,
446
- localSin: localFreqs?.sin,
293
+ localCos: localFreqs?.cos ?? null,
294
+ localSin: localFreqs?.sin ?? null,
447
295
  };
448
- CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
296
+ perDeviceCache.set(cacheKey, buffers);
449
297
  return buffers;
450
298
  }
451
299