doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -0,0 +1,304 @@
1
+ import { releaseBuffer } from '../../../memory/buffer-pool.js';
2
+ import { formatChatMessages } from './chat-format.js';
3
+ import { assertNotAborted } from './abort-contract.js';
4
+ import {
5
+ assertMultimodalSequenceCapacity,
6
+ buildConservativeMultimodalGenerationOptions,
7
+ expandImagePlaceholderTokenIds,
8
+ resolveMultimodalMaxTokens,
9
+ resolveSingleSpecialTokenId,
10
+ } from './modality-token-contract.js';
11
+ import { runProbes } from './probes.js';
12
+
13
+ export function createImageTranscriptionResourceScope(
14
+ pipeline,
15
+ encodeResult,
16
+ releaseFeature = releaseBuffer
17
+ ) {
18
+ if (!pipeline || typeof pipeline !== 'object') {
19
+ throw new Error('[Pipeline] Image transcription resource scope requires a pipeline.');
20
+ }
21
+ if (!encodeResult || typeof encodeResult !== 'object') {
22
+ throw new Error('[Pipeline] Image transcription resource scope requires an encode result.');
23
+ }
24
+ if (typeof releaseFeature !== 'function') {
25
+ throw new Error('[Pipeline] Image transcription resource scope requires a release function.');
26
+ }
27
+
28
+ const originalRopeCos = pipeline.ropeFreqsCos;
29
+ const originalRopeSin = pipeline.ropeFreqsSin;
30
+ let glmOcrRopeOverride = null;
31
+ let hasRun = false;
32
+ let runActive = false;
33
+ let hasFinished = false;
34
+ let generationStarted = false;
35
+
36
+ return {
37
+ setGlmOcrRopeOverride(override) {
38
+ if (generationStarted || hasFinished) {
39
+ throw new Error('[Pipeline] Cannot register an MRoPE override after transcription started.');
40
+ }
41
+ if (
42
+ !override
43
+ || typeof override !== 'object'
44
+ || !override.cos
45
+ || !override.sin
46
+ || typeof override.release !== 'function'
47
+ ) {
48
+ throw new Error('[Pipeline] GLM-OCR MRoPE override is missing cos, sin, or release.');
49
+ }
50
+ if (glmOcrRopeOverride) {
51
+ throw new Error('[Pipeline] GLM-OCR MRoPE override is already registered.');
52
+ }
53
+ glmOcrRopeOverride = override;
54
+ pipeline.ropeFreqsCos = override.cos;
55
+ pipeline.ropeFreqsSin = override.sin;
56
+ },
57
+
58
+ async runGeneration(task) {
59
+ if (!runActive) {
60
+ throw new Error('[Pipeline] Image transcription generation requires an active resource scope.');
61
+ }
62
+ if (generationStarted) {
63
+ throw new Error('[Pipeline] Image transcription generation may run only once.');
64
+ }
65
+ if (typeof task !== 'function') {
66
+ throw new Error('[Pipeline] Image transcription generation requires a task.');
67
+ }
68
+ generationStarted = true;
69
+ return task();
70
+ },
71
+
72
+ async run(task) {
73
+ if (hasRun) {
74
+ throw new Error('[Pipeline] Image transcription resource scope may run only once.');
75
+ }
76
+ if (typeof task !== 'function') {
77
+ throw new Error('[Pipeline] Image transcription resource scope requires a task.');
78
+ }
79
+ hasRun = true;
80
+ runActive = true;
81
+ try {
82
+ return await task();
83
+ } finally {
84
+ runActive = false;
85
+ hasFinished = true;
86
+ try {
87
+ if (glmOcrRopeOverride) {
88
+ pipeline.ropeFreqsCos = originalRopeCos;
89
+ pipeline.ropeFreqsSin = originalRopeSin;
90
+ glmOcrRopeOverride.release();
91
+ }
92
+ } finally {
93
+ if (encodeResult.features) {
94
+ releaseFeature(encodeResult.features);
95
+ }
96
+ }
97
+ }
98
+ },
99
+ };
100
+ }
101
+
102
+ export async function transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget, signal }) {
103
+ assertNotAborted(signal);
104
+ if (!this.visionCapable) {
105
+ throw new Error(
106
+ 'Pipeline does not support image transcription (no image_token_id in manifest).'
107
+ );
108
+ }
109
+ await this._ensureVisionWeightsLoaded();
110
+ assertNotAborted(signal);
111
+
112
+ this.reset();
113
+
114
+ // Lazy-load vision module (avoids GPU kernel dependency for text-only pipelines)
115
+ const { encodeImage } = await import('../vision/index.js');
116
+
117
+ // Step 1: Encode image through vision pipeline
118
+ const probeVisionTensor = async (stage, buffer, options = {}) => runProbes(stage, buffer, {
119
+ ...options,
120
+ probes: this.runtimeConfig.shared.debug.probes,
121
+ recorder: null,
122
+ operatorDiagnostics: this.operatorDiagnostics,
123
+ dtype: options.dtype ?? 'f32',
124
+ });
125
+ const encodeResult = await encodeImage({
126
+ pixels: imageBytes,
127
+ width,
128
+ height,
129
+ visionConfig: this.visionConfig,
130
+ weights: this.visionWeights,
131
+ softTokenBudget,
132
+ probeTensor: probeVisionTensor,
133
+ });
134
+ await probeVisionTensor('image_features_out', encodeResult.features, {
135
+ numTokens: encodeResult.numTokens,
136
+ hiddenSize: this.visionConfig.outHiddenSize,
137
+ });
138
+ const transcriptionResources = createImageTranscriptionResourceScope(this, encodeResult);
139
+
140
+ return transcriptionResources.run(async () => {
141
+ // Step 2: Build the multimodal prompt from the model's chat template and
142
+ // expand the single <|image|> placeholder into the exact visual-token span.
143
+ const visionArchitecture = this.visionConfig?.visionArchitecture;
144
+ const isGemma4 = visionArchitecture === 'gemma4';
145
+ const isGlmOcr = visionArchitecture === 'glmocr';
146
+ if (!isGemma4 && !isGlmOcr) {
147
+ throw new Error(
148
+ `[Pipeline] transcribeImage: unsupported vision architecture "${visionArchitecture ?? 'unknown'}". ` +
149
+ 'Supported transcription paths: Gemma 4 and GLM-OCR.'
150
+ );
151
+ }
152
+ const requestedPrompt = prompt ?? (isGlmOcr
153
+ ? 'Text Recognition:'
154
+ : 'Describe the image in one short sentence.');
155
+ const imageTokenId = this.visionConfig?.imageTokenId ?? this.modelConfig?.imageTokenId;
156
+ if (imageTokenId == null) {
157
+ throw new Error(
158
+ 'Pipeline missing image_token_id. Re-convert the model with image token metadata.'
159
+ );
160
+ }
161
+ const expectedTemplateType = isGlmOcr ? 'glmocr' : 'gemma4';
162
+ const templateType = this.modelConfig?.chatTemplateType ?? expectedTemplateType;
163
+ if (templateType !== expectedTemplateType) {
164
+ throw new Error(
165
+ `[Pipeline] transcribeImage: ${visionArchitecture} requires chatTemplate.type="${expectedTemplateType}", ` +
166
+ `got ${JSON.stringify(templateType)}.`
167
+ );
168
+ }
169
+ const chatTemplateThinking = this.modelConfig?.chatTemplateThinking;
170
+ const chatOptions = chatTemplateThinking == null
171
+ ? undefined
172
+ : { thinking: chatTemplateThinking };
173
+ const multimodalPrompt = formatChatMessages([
174
+ {
175
+ role: 'user',
176
+ content: [
177
+ { type: 'image' },
178
+ { type: 'text', text: requestedPrompt },
179
+ ],
180
+ },
181
+ ], templateType, chatOptions);
182
+ const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
183
+ const imageTokenSpanLength = encodeResult.numTokens;
184
+ const effectiveBudget = softTokenBudget ?? this.visionConfig?.defaultOutputLength;
185
+ const maxImageTokenSpanLength = Number(effectiveBudget);
186
+ if (!Number.isFinite(maxImageTokenSpanLength) || maxImageTokenSpanLength < 1 || Math.floor(maxImageTokenSpanLength) !== maxImageTokenSpanLength) {
187
+ throw new Error(
188
+ `[Pipeline] transcribeImage: invalid soft token budget ${effectiveBudget}. ` +
189
+ 'Expected a positive integer from the resolved vision config or softTokenBudget parameter.'
190
+ );
191
+ }
192
+ if (imageTokenSpanLength > maxImageTokenSpanLength) {
193
+ throw new Error(
194
+ `[Pipeline] transcribeImage: encoded ${visionArchitecture} image produced ${imageTokenSpanLength} soft tokens, ` +
195
+ `which exceeds the effective soft token budget=${maxImageTokenSpanLength}.`
196
+ );
197
+ }
198
+ const expansionOptions = isGemma4
199
+ ? {
200
+ boiTokenId: resolveSingleSpecialTokenId(this.tokenizer, '<|image>', 'Gemma 4 BOI token'),
201
+ eoiTokenId: resolveSingleSpecialTokenId(this.tokenizer, '<image|>', 'Gemma 4 EOI token'),
202
+ }
203
+ : {};
204
+ const { inputIds: fullTokenIds, imageStartOffset } = expandImagePlaceholderTokenIds(
205
+ promptTokenIds,
206
+ imageTokenId,
207
+ imageTokenSpanLength,
208
+ expansionOptions
209
+ );
210
+ const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
211
+ if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
212
+ throw new Error(
213
+ `[Pipeline] transcribeImage: ${visionArchitecture} multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
214
+ );
215
+ }
216
+
217
+ // Step 3: Generate with embedding override at the image token offset.
218
+ const tokens = [];
219
+ const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
220
+ assertMultimodalSequenceCapacity({
221
+ inputTokenCount: fullTokenIds.length,
222
+ maxTokens: maxGen,
223
+ maxSeqLen: this.kvCache?.maxSeqLen,
224
+ });
225
+ const stopTokenIds = this.modelConfig.stopTokenIds;
226
+
227
+ if (isGlmOcr) {
228
+ if (
229
+ this.modelConfig.ropeInterleaved !== true
230
+ || this.modelConfig.mropeInterleaved !== true
231
+ || !Array.isArray(this.modelConfig.mropeSection)
232
+ ) {
233
+ throw new Error(
234
+ '[Pipeline] transcribeImage: GLM-OCR requires interleaved 3-axis MRoPE metadata.'
235
+ );
236
+ }
237
+ const { buildGlmOcrRopePositionPlan, uploadGlmOcrRopeFrequencies } = await import(
238
+ '../vision/glmocr-rope.js'
239
+ );
240
+ const [gridTemporal, gridHeight, gridWidth] = encodeResult.gridThw ?? [];
241
+ if (gridTemporal !== 1) {
242
+ throw new Error(
243
+ `[Pipeline] transcribeImage: GLM-OCR image path requires grid temporal extent 1, got ${gridTemporal}.`
244
+ );
245
+ }
246
+ const positionPlan = buildGlmOcrRopePositionPlan({
247
+ promptLength: fullTokenIds.length,
248
+ capacity: fullTokenIds.length + maxGen,
249
+ imageStartOffset,
250
+ imageTokenLength: imageTokenSpanLength,
251
+ gridHeight,
252
+ gridWidth,
253
+ mergeSize: this.visionConfig.spatialMergeSize,
254
+ });
255
+ const glmOcrRopeOverride = await uploadGlmOcrRopeFrequencies(positionPlan, {
256
+ rotaryDim: this.modelConfig.ropeRotaryDim,
257
+ frequencyBaseDim: this.modelConfig.ropeFrequencyBaseDim,
258
+ ropeTheta: this.modelConfig.ropeTheta,
259
+ mropeSection: this.modelConfig.mropeSection,
260
+ });
261
+ transcriptionResources.setGlmOcrRopeOverride(glmOcrRopeOverride);
262
+ }
263
+
264
+ const generationOptions = {
265
+ inputIds: fullTokenIds,
266
+ embeddingOverrides: {
267
+ prefixLength: encodeResult.numTokens,
268
+ offset: imageStartOffset,
269
+ embeddings: encodeResult.features,
270
+ },
271
+ __internalEmbeddingInputSpan: {
272
+ offset: imageStartOffset,
273
+ length: encodeResult.numTokens,
274
+ tokenId: padTokenId,
275
+ },
276
+ ...(isGemma4
277
+ ? {
278
+ __internalMultimodalBidirectionalSpan: {
279
+ offset: imageStartOffset,
280
+ length: encodeResult.numTokens,
281
+ },
282
+ }
283
+ : {}),
284
+ maxTokens: maxGen,
285
+ temperature: 0,
286
+ topK: 1,
287
+ topP: 1,
288
+ repetitionPenalty: 1,
289
+ };
290
+ const generation = await transcriptionResources.runGeneration(
291
+ () => this.generator.generateTokenIds(
292
+ '',
293
+ buildConservativeMultimodalGenerationOptions(generationOptions)
294
+ )
295
+ );
296
+ for (const token of generation.tokenIds ?? []) {
297
+ if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
298
+ tokens.push(token);
299
+ }
300
+ const text = this.tokenizer.decode(tokens);
301
+ return { text, tokens };
302
+ });
303
+ }
304
+
@@ -47,8 +47,7 @@ function normalizeProjectionDtype(weight) {
47
47
  const dtype = typeof weight?.dtype === 'string'
48
48
  ? weight.dtype.toLowerCase()
49
49
  : null;
50
- if (dtype === 'bf16') return 'f16';
51
- if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') return dtype;
50
+ if (dtype === 'bf16' || dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') return dtype;
52
51
  return null;
53
52
  }
54
53
 
@@ -73,8 +72,8 @@ function resolveProjectionStorageFormat(weight, expectedRows, hiddenSize) {
73
72
  };
74
73
  }
75
74
 
76
- if (dtype === 'f16' || dtype === 'f32') {
77
- const bytesPerElement = dtype === 'f16' ? 2 : 4;
75
+ if (dtype === 'bf16' || dtype === 'f16' || dtype === 'f32') {
76
+ const bytesPerElement = dtype === 'f32' ? 4 : 2;
78
77
  const byteLength = expectedRows * hiddenSize * bytesPerElement;
79
78
  if (weight?.buffer?.size < byteLength) {
80
79
  return null;
@@ -157,7 +156,6 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
157
156
  const { numLayers, numHeads, numKVHeads, headDim, hiddenSize } = modelConfig;
158
157
  const qSize = numHeads * headDim;
159
158
  const hasAttentionOutputGate = modelConfig?.attentionOutputGate === true;
160
- const qProjSize = hasAttentionOutputGate ? qSize * 2 : qSize;
161
159
  const kSize = numKVHeads * headDim;
162
160
  const vSize = numKVHeads * headDim;
163
161
  const qkvSize = qSize + kSize + vSize;
@@ -211,6 +209,8 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
211
209
  continue;
212
210
  }
213
211
 
212
+ const hasSeparateGateProjection = hasAttentionOutputGate && Boolean(weights.qGateProj);
213
+ const qProjSize = hasAttentionOutputGate && !hasSeparateGateProjection ? qSize * 2 : qSize;
214
214
  const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
215
215
  const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
216
216
  const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
@@ -225,8 +225,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
225
225
  continue;
226
226
  }
227
227
 
228
+ // BF16 is a distinct numeric encoding, not an F16 storage alias. The
229
+ // current fused-QKV kernels do not declare BF16 weight decoding, so keep
230
+ // these source projections separate until a signed plan selects a BF16
231
+ // fused implementation explicitly.
232
+ if (qFormat.dtype === 'bf16') {
233
+ log.debug('QKV Fusion', `Layer ${l}: BF16 projections require split execution`);
234
+ continue;
235
+ }
236
+
228
237
  if (requiresQ4K && qFormat.dtype !== 'q4k') {
229
- log.debug('QKV Fusion', `Layer ${l}: qkv_proj requires Q4K weights, skipping ${qFormat.dtype} pack`);
238
+ log.debug('QKV Fusion', `Layer ${l}: qkv_proj requires Q4K weights, skipping ${qFormat.dtype} capsule`);
230
239
  continue;
231
240
  }
232
241
  if (qFormat.dtype === 'q4k' && !allowQ4K) {
@@ -248,7 +257,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
248
257
  }
249
258
 
250
259
  const qBytes = qSize * qFormat.bytesPerRow;
251
- const qGateBytes = hasAttentionOutputGate ? qBytes : 0;
260
+ const qGateBytes = hasAttentionOutputGate && !hasSeparateGateProjection ? qBytes : 0;
252
261
 
253
262
  // Create fused QKV buffer: [qkvSize, hiddenSize] row-major.
254
263
  // attentionOutputGate models keep Q rows in qkvProj and gate rows in qGateProj.
@@ -257,7 +266,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
257
266
  size: qBytes + kFormat.byteLength + vFormat.byteLength,
258
267
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
259
268
  });
260
- const qGateBuffer = hasAttentionOutputGate
269
+ const qGateBuffer = hasAttentionOutputGate && !hasSeparateGateProjection
261
270
  ? device.createBuffer({
262
271
  label: `layer_${l}_q_gate_proj`,
263
272
  size: qGateBytes,
@@ -292,7 +301,7 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
292
301
  }
293
302
 
294
303
  const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
295
- if (hasAttentionOutputGate) {
304
+ if (hasAttentionOutputGate && !hasSeparateGateProjection) {
296
305
  for (let head = 0; head < numHeads; head++) {
297
306
  const srcHeadOffset = head * headDim * 2 * qFormat.bytesPerRow;
298
307
  const dstHeadOffset = head * headDim * qFormat.bytesPerRow;
@@ -1,14 +1,16 @@
1
1
  import { getDevice } from '../../../gpu/device.js';
2
- import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
2
+ import { isBufferActive } from '../../../memory/buffer-pool.js';
3
3
  import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
4
4
  import { log } from '../../../debug/index.js';
5
+ import { runRoPEPrecompute } from '../../../gpu/kernels/rope-precompute.js';
6
+ import { getShaderScopeCacheKey } from '../../../gpu/kernels/shader-source-scope.js';
5
7
 
6
8
  // ============================================================================
7
9
  // RoPE Initialization
8
10
  // ============================================================================
9
11
 
10
12
 
11
- function computeRoPEFreqsForTheta(
13
+ async function computeRoPEFreqsForTheta(
12
14
  theta,
13
15
  rotaryDim,
14
16
  frequencyBaseDim,
@@ -17,120 +19,15 @@ function computeRoPEFreqsForTheta(
17
19
  ropeScalingType,
18
20
  ropeScaling
19
21
  ) {
20
- const halfDim = rotaryDim / 2;
21
-
22
- // Compute base frequencies: theta_i = 1 / (base^(2i/d))
23
- const freqs = new Float32Array(halfDim);
24
- for (let i = 0; i < halfDim; i++) {
25
- freqs[i] = 1.0 / Math.pow(theta, (2 * i) / frequencyBaseDim);
26
- }
27
-
28
- // Compute per-dimension scaling factors
29
- const scales = new Float32Array(halfDim);
30
- const isYarn = ropeScalingType === 'yarn';
31
- const isLongRoPE = ropeScalingType === 'longrope';
32
- let magnitudeScale = 1;
33
- if (isYarn) {
34
- // YARN scaling - validate ALL required params (fail fast on incomplete manifest)
35
- if (ropeScaling?.factor == null ||
36
- ropeScaling?.beta_fast == null || ropeScaling?.beta_slow == null ||
37
- ropeScaling?.original_max_position_embeddings == null) {
38
- throw new Error(
39
- `RoPE scaling type is 'yarn' but YARN params missing. ` +
40
- `Manifest must provide factor, beta_fast, beta_slow, and original_max_position_embeddings. ` +
41
- `Got: factor=${ropeScaling?.factor}, beta_fast=${ropeScaling?.beta_fast}, beta_slow=${ropeScaling?.beta_slow}, ` +
42
- `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
43
- );
44
- }
45
- // Extract validated YARN params (no hidden defaults - all guaranteed non-null)
46
- const yarnFactor = ropeScaling.factor;
47
- const yarnBetaFast = ropeScaling.beta_fast;
48
- const yarnBetaSlow = ropeScaling.beta_slow;
49
- const originalMaxPos = ropeScaling.original_max_position_embeddings;
50
-
51
- // YARN: wavelength-based interpolation
52
- for (let i = 0; i < halfDim; i++) {
53
- const wavelength = (2 * Math.PI) / freqs[i];
54
- const lowThresh = originalMaxPos / yarnBetaSlow;
55
- const highThresh = originalMaxPos / yarnBetaFast;
56
-
57
- if (wavelength < highThresh) {
58
- scales[i] = 1.0;
59
- } else if (wavelength > lowThresh) {
60
- scales[i] = yarnFactor;
61
- } else {
62
- const t = (wavelength - highThresh) / (lowThresh - highThresh);
63
- scales[i] = 1.0 + (yarnFactor - 1.0) * t;
64
- }
65
- }
66
- } else if (isLongRoPE) {
67
- if (
68
- !Array.isArray(ropeScaling?.short_factor)
69
- || !Array.isArray(ropeScaling?.long_factor)
70
- || ropeScaling?.original_max_position_embeddings == null
71
- ) {
72
- throw new Error(
73
- `RoPE scaling type is 'longrope' but LongRoPE params are missing. ` +
74
- `Manifest must provide short_factor, long_factor, and original_max_position_embeddings. ` +
75
- `Got: short_factor=${Array.isArray(ropeScaling?.short_factor) ? ropeScaling.short_factor.length : ropeScaling?.short_factor}, ` +
76
- `long_factor=${Array.isArray(ropeScaling?.long_factor) ? ropeScaling.long_factor.length : ropeScaling?.long_factor}, ` +
77
- `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
78
- );
79
- }
80
- if (ropeScaling.short_factor.length !== halfDim || ropeScaling.long_factor.length !== halfDim) {
81
- throw new Error(
82
- `LongRoPE factor length mismatch: expected ${halfDim}, ` +
83
- `got short_factor=${ropeScaling.short_factor.length}, long_factor=${ropeScaling.long_factor.length}.`
84
- );
85
- }
86
- const originalMaxPos = Number(ropeScaling.original_max_position_embeddings);
87
- if (!Number.isFinite(originalMaxPos) || originalMaxPos <= 0) {
88
- throw new Error(`LongRoPE original_max_position_embeddings must be positive; got "${String(originalMaxPos)}".`);
89
- }
90
- const selectedFactors = maxSeqLen > originalMaxPos
91
- ? ropeScaling.long_factor
92
- : ropeScaling.short_factor;
93
- for (let i = 0; i < halfDim; i++) {
94
- const factor = Number(selectedFactors[i]);
95
- if (!Number.isFinite(factor) || factor <= 0) {
96
- throw new Error(`LongRoPE factor[${i}] must be a positive finite number; got "${String(selectedFactors[i])}".`);
97
- }
98
- scales[i] = factor;
99
- }
100
- magnitudeScale = Math.sqrt(1 + Math.log(maxSeqLen / originalMaxPos) / Math.log(originalMaxPos));
101
- if (!Number.isFinite(magnitudeScale) || magnitudeScale <= 0) {
102
- throw new Error(
103
- `LongRoPE magnitude scale is invalid for maxSeqLen=${maxSeqLen}, ` +
104
- `original_max_position_embeddings=${originalMaxPos}.`
105
- );
106
- }
107
- } else {
108
- // Linear scaling: uniform across all dimensions
109
- if (ropeScalingType != null && ropeScalingType !== 'linear') {
110
- throw new Error(
111
- `Unsupported RoPE scaling type "${ropeScalingType}". ` +
112
- 'Supported types: null, "linear", "yarn", "longrope".'
113
- );
114
- }
115
- for (let i = 0; i < halfDim; i++) {
116
- scales[i] = ropeScale;
117
- }
118
- }
119
-
120
- // Compute cos/sin for each position
121
- const cosValues = new Float32Array(maxSeqLen * halfDim);
122
- const sinValues = new Float32Array(maxSeqLen * halfDim);
123
-
124
- for (let pos = 0; pos < maxSeqLen; pos++) {
125
- for (let i = 0; i < halfDim; i++) {
126
- const scaledPos = pos / scales[i];
127
- const angle = scaledPos * freqs[i];
128
- cosValues[pos * halfDim + i] = Math.cos(angle) * magnitudeScale;
129
- sinValues[pos * halfDim + i] = Math.sin(angle) * magnitudeScale;
130
- }
131
- }
132
-
133
- return { cos: cosValues, sin: sinValues };
22
+ return runRoPEPrecompute({
23
+ theta,
24
+ rotaryDim,
25
+ frequencyBaseDim,
26
+ maxSeqLen,
27
+ ropeScale,
28
+ scalingType: ropeScalingType,
29
+ scaling: ropeScaling,
30
+ });
134
31
  }
135
32
 
136
33
  function isSameRoPEScalingConfig(
@@ -157,8 +54,6 @@ function isSameRoPEScalingConfig(
157
54
  }
158
55
 
159
56
  const GPU_ROPE_BUFFER_CACHE = new WeakMap();
160
- const CPU_ROPE_BUFFER_CACHE = new Map();
161
-
162
57
  function isLiveCachedRopeBuffer(buffer) {
163
58
  return buffer == null || isBufferActive(buffer);
164
59
  }
@@ -239,7 +134,7 @@ function resolveFrequencyBaseDim(rotaryDim, frequencyBaseDim, label) {
239
134
 
240
135
 
241
136
  export async function initRoPEFrequencies(config, useGPU) {
242
- const cacheKey = buildRoPECacheKey(config);
137
+ const cacheKey = `${getShaderScopeCacheKey()}:${buildRoPECacheKey(config)}`;
243
138
  const {
244
139
  headDim,
245
140
  localHeadDim,
@@ -309,8 +204,25 @@ export async function initRoPEFrequencies(config, useGPU) {
309
204
  const isYarn = ropeScalingType === 'yarn';
310
205
  const isLocalYarn = resolvedLocalScalingType === 'yarn';
311
206
 
207
+ const device = getDevice();
208
+ if (!useGPU || !device) {
209
+ throw new Error('RoPE frequency initialization requires the declared WebGPU execution path.');
210
+ }
211
+ let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
212
+ if (!perDeviceCache) {
213
+ perDeviceCache = new Map();
214
+ GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
215
+ }
216
+ const cachedBuffers = perDeviceCache.get(cacheKey);
217
+ if (cachedBuffers) {
218
+ if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
219
+ return cachedBuffers;
220
+ }
221
+ perDeviceCache.delete(cacheKey);
222
+ }
223
+
312
224
  // Compute global (full_attention) frequencies
313
- const globalFreqs = computeRoPEFreqsForTheta(
225
+ const globalFreqs = await computeRoPEFreqsForTheta(
314
226
  ropeTheta,
315
227
  resolvedRotaryDim,
316
228
  resolvedFrequencyBaseDim,
@@ -335,7 +247,7 @@ export async function initRoPEFrequencies(config, useGPU) {
335
247
  resolvedLocalScaling
336
248
  );
337
249
  if (hasDistinctLocalTheta || hasDistinctLocalScaling || hasDistinctLocalDim) {
338
- localFreqs = computeRoPEFreqsForTheta(
250
+ localFreqs = await computeRoPEFreqsForTheta(
339
251
  resolvedLocalTheta,
340
252
  resolvedLocalRotaryDim,
341
253
  resolvedLocalFrequencyBaseDim,
@@ -366,73 +278,9 @@ export async function initRoPEFrequencies(config, useGPU) {
366
278
  );
367
279
  }
368
280
 
369
- // Upload to GPU if available
370
- const device = getDevice();
371
- if (device && useGPU) {
372
- let perDeviceCache = GPU_ROPE_BUFFER_CACHE.get(device);
373
- if (!perDeviceCache) {
374
- perDeviceCache = new Map();
375
- GPU_ROPE_BUFFER_CACHE.set(device, perDeviceCache);
376
- }
377
- const cachedBuffers = perDeviceCache.get(cacheKey);
378
- if (cachedBuffers) {
379
- if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
380
- return cachedBuffers;
381
- }
382
- perDeviceCache.delete(cacheKey);
383
- }
384
- let cosBuffer = null;
385
- let sinBuffer = null;
386
- let localCosBuffer = null;
387
- let localSinBuffer = null;
388
- try {
389
- cosBuffer = acquireBuffer(globalFreqs.cos.byteLength, undefined, 'rope_cos');
390
- sinBuffer = acquireBuffer(globalFreqs.sin.byteLength, undefined, 'rope_sin');
391
- device.queue.writeBuffer(cosBuffer, 0, globalFreqs.cos.buffer, globalFreqs.cos.byteOffset, globalFreqs.cos.byteLength);
392
- device.queue.writeBuffer(sinBuffer, 0, globalFreqs.sin.buffer, globalFreqs.sin.byteOffset, globalFreqs.sin.byteLength);
393
-
394
- if (localFreqs) {
395
- localCosBuffer = acquireBuffer(localFreqs.cos.byteLength, undefined, 'rope_local_cos');
396
- localSinBuffer = acquireBuffer(localFreqs.sin.byteLength, undefined, 'rope_local_sin');
397
- device.queue.writeBuffer(localCosBuffer, 0, localFreqs.cos.buffer, localFreqs.cos.byteOffset, localFreqs.cos.byteLength);
398
- device.queue.writeBuffer(localSinBuffer, 0, localFreqs.sin.buffer, localFreqs.sin.byteOffset, localFreqs.sin.byteLength);
399
- }
400
- } catch (error) {
401
- for (const buffer of [cosBuffer, sinBuffer, localCosBuffer, localSinBuffer]) {
402
- if (buffer) {
403
- releaseBuffer(buffer);
404
- }
405
- }
406
- throw error;
407
- }
408
-
409
- log.debug(
410
- 'Pipeline',
411
- `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
412
- `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
413
- `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
414
- `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
415
- `interleaved=${mropeInterleaved === true}`
416
- );
417
-
418
- const buffers = {
419
- cos: cosBuffer,
420
- sin: sinBuffer,
421
- localCos: localCosBuffer,
422
- localSin: localSinBuffer,
423
- };
424
- perDeviceCache.set(cacheKey, buffers);
425
- return buffers;
426
- }
427
-
428
- const cachedCpuBuffers = CPU_ROPE_BUFFER_CACHE.get(cacheKey);
429
- if (cachedCpuBuffers) {
430
- return cachedCpuBuffers;
431
- }
432
-
433
281
  log.debug(
434
282
  'Pipeline',
435
- `RoPE frequencies initialized (CPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
283
+ `RoPE frequencies initialized (GPU): ${maxSeqLen} positions, dim=${halfDim}, headDim=${headDim}, rotaryDim=${resolvedRotaryDim}, ` +
436
284
  `theta=${ropeTheta}${hasDistinctLocalTheta ? `, localTheta=${resolvedLocalTheta}` : ''}, ` +
437
285
  `${hasDistinctLocalDim ? `localRotaryDim=${resolvedLocalRotaryDim}, ` : ''}` +
438
286
  `scaling=${ropeScalingType ?? 'none'}:${ropeScale}${hasDistinctLocalScaling ? `, localScaling=${resolvedLocalScalingType ?? 'none'}:${resolvedLocalScale}` : ''}, ` +
@@ -442,10 +290,10 @@ export async function initRoPEFrequencies(config, useGPU) {
442
290
  const buffers = {
443
291
  cos: globalFreqs.cos,
444
292
  sin: globalFreqs.sin,
445
- localCos: localFreqs?.cos,
446
- localSin: localFreqs?.sin,
293
+ localCos: localFreqs?.cos ?? null,
294
+ localSin: localFreqs?.sin ?? null,
447
295
  };
448
- CPU_ROPE_BUFFER_CACHE.set(cacheKey, buffers);
296
+ perDeviceCache.set(cacheKey, buffers);
449
297
  return buffers;
450
298
  }
451
299