doppler-gpu 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (927) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +99 -86
  6. package/src/capsule-runtime.d.ts +23 -0
  7. package/src/capsule-runtime.js +23 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +1 -0
  24. package/src/client/capsule-host.d.ts +3 -0
  25. package/src/client/capsule-host.js +1 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +36 -1
  28. package/src/client/doppler-api.js +40 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +322 -0
  45. package/src/client/{runtime → model-host}/model-session.js +190 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +68 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +100 -0
  64. package/src/client/runtime/capsule-program-adapter.js +151 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +85 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +74 -0
  72. package/src/client/runtime/composition-root.js +344 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +30 -0
  92. package/src/client/runtime/session-controller.js +170 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +72 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +11 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +58 -0
  109. package/src/config/capsule-operations.json +18 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  128. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +41 -1
  129. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  130. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +1 -1
  132. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +1 -1
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +1 -1
  134. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +1 -1
  135. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +1 -1
  136. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +2 -2
  137. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +1 -1
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +1 -1
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +1 -1
  140. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +2 -2
  141. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +1 -1
  142. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  143. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +1 -1
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  145. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  146. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +1 -1
  147. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  148. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  149. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  150. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +1 -1
  151. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +1 -1
  152. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  153. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +1 -1
  154. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +2 -2
  155. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  157. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +1 -1
  158. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  160. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +59 -7
  161. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  162. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  163. package/src/config/dtype-transition-contract.js +40 -0
  164. package/src/config/embedding-contract.js +26 -0
  165. package/src/config/embedding-reference.js +85 -0
  166. package/src/config/evidence/source-intake-policy.json +1 -1
  167. package/src/config/execution-contract-check.d.ts +16 -73
  168. package/src/config/execution-contract-check.js +7 -429
  169. package/src/config/forecast-manifest.js +56 -0
  170. package/src/config/heterogeneous-attention-contract.js +46 -0
  171. package/src/config/initial-execution-identity.d.ts +58 -0
  172. package/src/config/initial-execution-identity.js +331 -0
  173. package/src/config/kernel-path-loader.d.ts +10 -0
  174. package/src/config/kernel-path-loader.js +23 -8
  175. package/src/config/kernel-registry-contract.d.ts +25 -0
  176. package/src/config/kernel-registry-contract.js +46 -0
  177. package/src/config/kernels/kernel-ref-digests.js +50 -2
  178. package/src/config/kernels/registry.js +1 -1
  179. package/src/config/kernels/registry.json +13592 -11377
  180. package/src/config/lora-layouts.d.ts +10 -0
  181. package/src/config/lora-layouts.js +17 -0
  182. package/src/config/lora-layouts.json +9 -0
  183. package/src/config/manifest-inference-validation.js +575 -0
  184. package/src/config/merge/execution-patches.js +12 -0
  185. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  186. package/src/config/merge/value-selection.js +38 -0
  187. package/src/config/merge.js +43 -11
  188. package/src/config/model-ir-v2.d.ts +57 -0
  189. package/src/config/model-ir-v2.js +258 -0
  190. package/src/config/model-ir.d.ts +49 -0
  191. package/src/config/model-ir.js +180 -0
  192. package/src/config/param-validator.js +1 -1
  193. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  194. package/src/config/platforms/loader.js +2 -1
  195. package/src/config/production-release-evidence.d.ts +23 -0
  196. package/src/config/production-release-evidence.js +423 -0
  197. package/src/config/production-release.js +321 -0
  198. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  199. package/src/config/required-inference-fields-contract-check.js +2 -2
  200. package/src/config/rerank-reference.d.ts +28 -0
  201. package/src/config/rerank-reference.js +89 -0
  202. package/src/config/revocation-policy.d.ts +65 -0
  203. package/src/config/revocation-policy.js +290 -0
  204. package/src/config/revocation-registry.json +11 -0
  205. package/src/config/revocation-updates.d.ts +68 -0
  206. package/src/config/revocation-updates.js +440 -0
  207. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  208. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  209. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  210. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  211. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  212. package/src/config/runtime-merge.js +1 -1
  213. package/src/config/runtime.js +1 -1
  214. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  215. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  216. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  217. package/src/config/schema/conversion-report.schema.js +29 -0
  218. package/src/config/schema/converter.schema.d.ts +1 -0
  219. package/src/config/schema/debug.schema.d.ts +1 -0
  220. package/src/config/schema/doppler.schema.js +5 -5
  221. package/src/config/schema/emulation.schema.js +1 -1
  222. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  223. package/src/config/schema/execution-v1.schema.js +19 -0
  224. package/src/config/schema/harness.schema.d.ts +13 -0
  225. package/src/config/schema/harness.schema.js +6 -0
  226. package/src/config/schema/inference.schema.d.ts +21 -2
  227. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  228. package/src/config/schema/manifest.schema.d.ts +26 -7
  229. package/src/config/schema/manifest.schema.js +10 -1
  230. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  231. package/src/config/schema/program-bundle.schema.js +145 -10
  232. package/src/config/sequence-reference.d.ts +26 -0
  233. package/src/config/sequence-reference.js +100 -0
  234. package/src/config/target-plan.d.ts +108 -0
  235. package/src/config/target-plan.js +411 -0
  236. package/src/config/training-defaults.js +1 -1
  237. package/src/config/transforms/capability-transform-resolver.js +4 -105
  238. package/src/config/transforms/execution-graph/dtype.js +591 -0
  239. package/src/config/transforms/execution-graph/fusion.js +436 -0
  240. package/src/config/transforms/execution-graph/validation.js +414 -0
  241. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  242. package/src/converter/artifact-identity.js +475 -0
  243. package/src/converter/conversion-plan.js +1 -1
  244. package/src/converter/core.js +94 -2096
  245. package/src/converter/forge-candidate-evaluation.js +271 -0
  246. package/src/converter/forge-qualification.js +162 -0
  247. package/src/converter/forge-stages.js +798 -0
  248. package/src/converter/manifest-builder.js +683 -0
  249. package/src/converter/quantization-info.js +4 -1
  250. package/src/converter/quantizer.js +58 -9
  251. package/src/converter/shard-packer.d.ts +1 -1
  252. package/src/converter/shard-packer.js +2 -2
  253. package/src/converter/tensor-transform.js +640 -0
  254. package/src/converter/tokenizer-contract.js +243 -0
  255. package/src/converter/tokenizer-manifest.js +243 -0
  256. package/src/debug/history.js +0 -8
  257. package/src/debug/index.js +0 -10
  258. package/src/debug/tensor.js +1 -1
  259. package/src/debug/trace.js +0 -7
  260. package/src/experimental/adapters/adapter-manager.js +2 -0
  261. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  262. package/src/experimental/adapters/adapter-manifest.js +11 -0
  263. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  264. package/src/experimental/adapters/artifact-contract.js +5 -2
  265. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  266. package/src/experimental/adapters/lora-loader.js +110 -48
  267. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  268. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  269. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  270. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  271. package/src/experimental/browser/browser-converter.js +11 -11
  272. package/src/experimental/diffusion/index.d.ts +1 -1
  273. package/src/experimental/diffusion/index.js +1 -1
  274. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  275. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  276. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  277. package/src/experimental/diffusion/vae.js +596 -0
  278. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  279. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  280. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  281. package/src/experimental/distribution/shard-delivery.js +4 -1709
  282. package/src/experimental/energy/index.d.ts +2 -2
  283. package/src/experimental/energy/index.js +2 -2
  284. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  285. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  286. package/src/experimental/logits/cpu-output.d.ts +27 -0
  287. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  288. package/src/experimental/logits/cpu.d.ts +87 -0
  289. package/src/experimental/logits/cpu.js +95 -0
  290. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  291. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  292. package/src/experimental/orchestration/index.d.ts +1 -1
  293. package/src/experimental/orchestration/index.js +1 -1
  294. package/src/experimental/training/artifacts.js +2 -2
  295. package/src/experimental/training/checkpoint.js +2 -2
  296. package/src/experimental/training/distillation/artifacts.js +1 -1
  297. package/src/experimental/training/distillation/dataset.js +2 -2
  298. package/src/experimental/training/distillation/eval.js +1 -1
  299. package/src/experimental/training/distillation/runtime.js +2 -2
  300. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  301. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  302. package/src/experimental/training/distillation/student-fixture.js +2 -469
  303. package/src/experimental/training/lora/checkpoint.js +746 -0
  304. package/src/experimental/training/lora/execution.js +550 -0
  305. package/src/experimental/training/lora/plan.js +387 -0
  306. package/src/experimental/training/lora/recovery.js +418 -0
  307. package/src/experimental/training/lora-pipeline.js +7 -1949
  308. package/src/experimental/training/operator-artifacts.js +2 -2
  309. package/src/experimental/training/operator-command.js +4 -4
  310. package/src/experimental/training/runner/evaluation.js +62 -0
  311. package/src/experimental/training/runner/lifecycle.js +257 -0
  312. package/src/experimental/training/runner.js +5 -268
  313. package/src/experimental/training/suite/execution.js +879 -0
  314. package/src/experimental/training/suite/plan.js +855 -0
  315. package/src/experimental/training/suite.js +5 -1659
  316. package/src/experimental/training/workloads.d.ts +5 -5
  317. package/src/experimental/training/workloads.js +6 -6
  318. package/src/experimental/turboquant/codebook.js +470 -0
  319. package/src/formats/quantization-constants.d.ts +11 -0
  320. package/src/formats/quantization-constants.js +11 -0
  321. package/src/formats/rdrr/integrity.js +1 -1
  322. package/src/formats/rdrr/lowerings.js +0 -29
  323. package/src/formats/rdrr/merkle.js +1 -1
  324. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  325. package/src/formats/rdrr/validation.js +3 -3
  326. package/src/gpu/command-recorder.d.ts +10 -1
  327. package/src/gpu/command-recorder.js +58 -14
  328. package/src/gpu/constant-buffer.js +29 -0
  329. package/src/gpu/device-state.js +36 -0
  330. package/src/gpu/device.js +6 -41
  331. package/src/gpu/kernel-runtime.js +2 -1
  332. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  333. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  334. package/src/gpu/kernels/attention/executors.js +960 -0
  335. package/src/gpu/kernels/attention/plan.js +472 -0
  336. package/src/gpu/kernels/attention.d.ts +4 -0
  337. package/src/gpu/kernels/attention.js +145 -1545
  338. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  339. package/src/gpu/kernels/backward/adam.js +2 -1
  340. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  341. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  342. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  343. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  344. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  345. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  346. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  347. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  348. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  349. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  350. package/src/gpu/kernels/backward/executor.js +158 -0
  351. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  352. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  353. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  354. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  355. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  356. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  357. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  358. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  359. package/src/gpu/kernels/backward/matmul.js +133 -0
  360. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  361. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  362. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  363. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  364. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  365. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  366. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  367. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  368. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  369. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  370. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  371. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  372. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  374. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  376. package/src/gpu/kernels/backward/utils.js +2 -392
  377. package/src/gpu/kernels/bind-group-validation.js +16 -0
  378. package/src/gpu/kernels/cast.js +2 -1
  379. package/src/gpu/kernels/check-finiteness.js +13 -49
  380. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  381. package/src/gpu/kernels/check-stop.js +14 -33
  382. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  383. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  384. package/src/gpu/kernels/check_stop.wgsl +20 -0
  385. package/src/gpu/kernels/clamp.js +1 -1
  386. package/src/gpu/kernels/conv2d.js +1 -1
  387. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  388. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  389. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  390. package/src/gpu/kernels/dequant.js +7 -2
  391. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  392. package/src/gpu/kernels/dispatch.js +12 -1
  393. package/src/gpu/kernels/embedding-pool.js +72 -0
  394. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  395. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  396. package/src/gpu/kernels/energy.js +2 -1
  397. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  398. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  399. package/src/gpu/kernels/fused_ffn.js +10 -220
  400. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  401. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  402. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  403. package/src/gpu/kernels/gated-short-conv.js +11 -69
  404. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  405. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  406. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  407. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  408. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  409. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  410. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  411. package/src/gpu/kernels/gather.d.ts +1 -1
  412. package/src/gpu/kernels/gather.js +30 -6
  413. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  414. package/src/gpu/kernels/gelu.js +1 -1
  415. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  416. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  417. package/src/gpu/kernels/groupnorm.js +1 -1
  418. package/src/gpu/kernels/index.d.ts +38 -5
  419. package/src/gpu/kernels/index.js +18 -4
  420. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  421. package/src/gpu/kernels/kernel-base.js +1 -1
  422. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  423. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  424. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  425. package/src/gpu/kernels/kernel-configs.js +26 -54
  426. package/src/gpu/kernels/kernel-execution.js +141 -0
  427. package/src/gpu/kernels/kv-cache-write.js +4 -4
  428. package/src/gpu/kernels/kv-quantize.js +4 -1
  429. package/src/gpu/kernels/l2-normalize.js +36 -0
  430. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  431. package/src/gpu/kernels/layernorm.d.ts +4 -4
  432. package/src/gpu/kernels/layernorm.js +44 -7
  433. package/src/gpu/kernels/linear-attention-core.js +29 -449
  434. package/src/gpu/kernels/linear_attention.js +1 -1
  435. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  436. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  437. package/src/gpu/kernels/logit-finalize.js +103 -0
  438. package/src/gpu/kernels/logit-merge.js +13 -125
  439. package/src/gpu/kernels/logit-scatter.js +40 -0
  440. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  441. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  442. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  443. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  444. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  445. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  446. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  447. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  448. package/src/gpu/kernels/matmul-selection.js +3 -2
  449. package/src/gpu/kernels/matmul.js +4 -15
  450. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  451. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  452. package/src/gpu/kernels/modulate.js +1 -1
  453. package/src/gpu/kernels/moe.js +3 -1
  454. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  455. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  456. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  457. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  458. package/src/gpu/kernels/patch_context.wgsl +29 -0
  459. package/src/gpu/kernels/pipeline-cache.js +3 -1
  460. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  461. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  462. package/src/gpu/kernels/relu.js +1 -1
  463. package/src/gpu/kernels/repeat_channels.js +1 -1
  464. package/src/gpu/kernels/residual.js +2 -2
  465. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  466. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  467. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  468. package/src/gpu/kernels/rmsnorm.js +1 -1
  469. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  470. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  471. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  472. package/src/gpu/kernels/rope-precompute.js +160 -0
  473. package/src/gpu/kernels/rope-qk.js +1 -1
  474. package/src/gpu/kernels/rope.js +2 -2
  475. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  476. package/src/gpu/kernels/sample.js +5 -1
  477. package/src/gpu/kernels/scale.js +1 -1
  478. package/src/gpu/kernels/shader-cache.js +8 -4
  479. package/src/gpu/kernels/shader-source-scope.js +68 -0
  480. package/src/gpu/kernels/silu.js +2 -1
  481. package/src/gpu/kernels/soft-embedding.js +3 -1
  482. package/src/gpu/kernels/softmax.js +4 -2
  483. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  484. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  485. package/src/gpu/kernels/split_qg.js +1 -1
  486. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  487. package/src/gpu/kernels/split_qkv.js +18 -3
  488. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  489. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  490. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  491. package/src/gpu/kernels/transpose.js +1 -1
  492. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  493. package/src/gpu/kernels/uniform-utils.js +7 -2
  494. package/src/gpu/kernels/upsample2d.js +1 -1
  495. package/src/gpu/kernels/utils.js +9 -330
  496. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  497. package/src/gpu/kernels/vision-average-pool.js +68 -0
  498. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  499. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  500. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  501. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  502. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  503. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  504. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  505. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  506. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  507. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  508. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  509. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  510. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  511. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  512. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  513. package/src/gpu/readback-buffer.js +20 -0
  514. package/src/gpu/submit-probe.js +2 -26
  515. package/src/gpu/weight-buffer.d.ts +1 -0
  516. package/src/gpu/weight-buffer.js +7 -0
  517. package/src/index.d.ts +10 -0
  518. package/src/index.js +1 -0
  519. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  520. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  521. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  522. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  523. package/src/inference/browser-harness/inference-execution.js +543 -0
  524. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  525. package/src/inference/browser-harness/model-resolution.js +137 -0
  526. package/src/inference/browser-harness/report.js +338 -0
  527. package/src/inference/browser-harness/request.js +263 -0
  528. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  529. package/src/inference/browser-harness/runtime-config.js +255 -0
  530. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  531. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  532. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  533. package/src/inference/browser-harness/runtime-report.js +117 -0
  534. package/src/inference/browser-harness/suite-summary.js +39 -0
  535. package/src/inference/browser-harness/text-evidence.js +694 -0
  536. package/src/inference/browser-harness/text-execution.js +757 -0
  537. package/src/inference/browser-harness/text-input.js +725 -0
  538. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  539. package/src/inference/browser-harness-contract-helpers.js +1 -1
  540. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  541. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  542. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  543. package/src/inference/browser-harness-text-helpers.js +31 -2199
  544. package/src/inference/browser-harness.d.ts +15 -2
  545. package/src/inference/browser-harness.js +38 -1952
  546. package/src/inference/kv-cache/base.js +67 -381
  547. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  548. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  549. package/src/inference/kv-cache/quantized.d.ts +2 -2
  550. package/src/inference/kv-cache/quantized.js +4 -8
  551. package/src/inference/kv-cache/update.js +309 -0
  552. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  553. package/src/inference/model-checkpoint-evidence.js +264 -0
  554. package/src/inference/moe-router.js +36 -216
  555. package/src/inference/multi-model-network.js +5 -341
  556. package/src/inference/pipelines/audio/index.js +1 -20
  557. package/src/inference/pipelines/audio/mel.js +0 -38
  558. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  559. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  560. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  561. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  562. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  563. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  564. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  565. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  566. package/src/inference/pipelines/factory.js +14 -3
  567. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  568. package/src/inference/pipelines/structured/json-grammar-mask.js +0 -6
  569. package/src/inference/pipelines/text/abort-contract.js +22 -0
  570. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  571. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  572. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  573. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  574. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  575. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  576. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  577. package/src/inference/pipelines/text/attention/plan.js +17 -0
  578. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  579. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  580. package/src/inference/pipelines/text/attention/projections.js +20 -537
  581. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  582. package/src/inference/pipelines/text/attention/record.js +0 -6
  583. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  584. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  585. package/src/inference/pipelines/text/attention/run.js +0 -6
  586. package/src/inference/pipelines/text/attention/types.js +3 -31
  587. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  588. package/src/inference/pipelines/text/chat-format.js +42 -0
  589. package/src/inference/pipelines/text/config/normalization.js +729 -0
  590. package/src/inference/pipelines/text/config/validation.js +9 -0
  591. package/src/inference/pipelines/text/config.d.ts +12 -1
  592. package/src/inference/pipelines/text/config.js +14 -1228
  593. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  594. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  595. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  596. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  597. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  598. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  599. package/src/inference/pipelines/text/embed.js +178 -241
  600. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  601. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  602. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  603. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  604. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  605. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  606. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  607. package/src/inference/pipelines/text/execution-v1.js +7 -283
  608. package/src/inference/pipelines/text/execution.js +517 -0
  609. package/src/inference/pipelines/text/ffn/dense-executor.js +718 -0
  610. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  611. package/src/inference/pipelines/text/ffn/dense.js +4 -968
  612. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  613. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  614. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  615. package/src/inference/pipelines/text/generator/decode-runtime.js +991 -0
  616. package/src/inference/pipelines/text/generator/decode.js +952 -0
  617. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  618. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  619. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  620. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  621. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  622. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  623. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  624. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  625. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  626. package/src/inference/pipelines/text/generator/text.js +448 -0
  627. package/src/inference/pipelines/text/generator/training.js +752 -0
  628. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  629. package/src/inference/pipelines/text/generator-logits-observation.js +45 -0
  630. package/src/inference/pipelines/text/generator-runtime.js +5 -135
  631. package/src/inference/pipelines/text/generator-steps.js +30 -972
  632. package/src/inference/pipelines/text/generator.js +165 -3544
  633. package/src/inference/pipelines/text/image-transcription.js +304 -0
  634. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  635. package/src/inference/pipelines/text/init-rope.js +37 -189
  636. package/src/inference/pipelines/text/init.d.ts +8 -3
  637. package/src/inference/pipelines/text/init.js +6 -216
  638. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  639. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  640. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  641. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  642. package/src/inference/pipelines/text/layer-plan.js +45 -3
  643. package/src/inference/pipelines/text/layer.js +52 -522
  644. package/src/inference/pipelines/text/lifecycle.js +839 -0
  645. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  646. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  647. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  648. package/src/inference/pipelines/text/linear-attention.js +7 -537
  649. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  650. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  651. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  652. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  653. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  654. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  655. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  656. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  657. package/src/inference/pipelines/text/logits/index.js +62 -102
  658. package/src/inference/pipelines/text/logits/output-transform.js +47 -0
  659. package/src/inference/pipelines/text/logits/plan.js +184 -0
  660. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  661. package/src/inference/pipelines/text/logits/readback.js +9 -0
  662. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  663. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  664. package/src/inference/pipelines/text/lora-apply.js +12 -4
  665. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  666. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  667. package/src/inference/pipelines/text/model-load.js +0 -20
  668. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  669. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  670. package/src/inference/pipelines/text/moe/plan.js +47 -0
  671. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  672. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  673. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  674. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  675. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  676. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  677. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  678. package/src/inference/pipelines/text/probes.js +20 -0
  679. package/src/inference/pipelines/text/state.d.ts +8 -0
  680. package/src/inference/pipelines/text/state.js +1 -0
  681. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  682. package/src/inference/pipelines/text/types.d.ts +4 -0
  683. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  684. package/src/inference/pipelines/text.d.ts +2 -2
  685. package/src/inference/pipelines/text.js +46 -1633
  686. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  687. package/src/inference/pipelines/video/index.js +0 -21
  688. package/src/inference/pipelines/vision/attention.js +59 -0
  689. package/src/inference/pipelines/vision/encoder.js +254 -369
  690. package/src/inference/pipelines/vision/gemma4.js +204 -318
  691. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  692. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  693. package/src/inference/pipelines/vision/glmocr.js +557 -0
  694. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  695. package/src/inference/pipelines/vision/index.js +8 -27
  696. package/src/inference/pipelines/vision/ops.js +11 -33
  697. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  698. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  699. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  700. package/src/inference/refactor-receipt.js +2 -2
  701. package/src/inference/rerank.js +182 -0
  702. package/src/inference/tokenizer.js +1 -1
  703. package/src/inference/tokenizers/behavior-flags.js +6 -3
  704. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  705. package/src/inference/tokenizers/bundled/registry.js +216 -0
  706. package/src/inference/tokenizers/bundled.js +23 -353
  707. package/src/loader/doppler-loader.d.ts +4 -0
  708. package/src/loader/doppler-loader.js +119 -930
  709. package/src/loader/embedding-limit-preflight.js +11 -4
  710. package/src/loader/embedding-loader.js +1 -3
  711. package/src/loader/gpu-tensor-materialization.js +20 -0
  712. package/src/loader/layer-loader.js +15 -119
  713. package/src/loader/layer-tensor-aliases.js +99 -0
  714. package/src/loader/layer-weight-shape.js +29 -0
  715. package/src/loader/load-timing.js +64 -0
  716. package/src/loader/loader-types.d.ts +3 -0
  717. package/src/loader/model-executor.js +819 -0
  718. package/src/loader/model-load-validation.js +15 -0
  719. package/src/loader/quantization-constants.d.ts +11 -29
  720. package/src/loader/quantization-constants.js +1 -2
  721. package/src/loader/shard-resolver.js +0 -14
  722. package/src/loader/tensor-load-error.js +14 -0
  723. package/src/loader/tensor-shard-indices.js +8 -0
  724. package/src/loader/tensors/bf16-loader.js +125 -0
  725. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  726. package/src/loader/tensors/load-executor.js +577 -0
  727. package/src/loader/tensors/source-transform/executor.js +123 -0
  728. package/src/loader/tensors/source-transform/plan.js +107 -0
  729. package/src/loader/tensors/source-transform.js +15 -264
  730. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  731. package/src/loader/tensors/tensor-loader.js +7 -653
  732. package/src/memory/buffer-pool.js +7 -29
  733. package/src/rules/capability-transform-resolver.js +80 -0
  734. package/src/rules/execution-contract.d.ts +73 -0
  735. package/src/rules/execution-contract.js +429 -0
  736. package/src/rules/execution-rules-contract-check.js +1 -1
  737. package/src/rules/generated/rule-bundle.json +4305 -0
  738. package/src/rules/inference/capability-transforms.rules.json +3 -3
  739. package/src/rules/layer-pattern-contract-check.js +1 -1
  740. package/src/rules/rule-registry.js +70 -57
  741. package/src/storage/artifact-storage-context.js +2 -2
  742. package/src/storage/backend-selection.js +43 -0
  743. package/src/storage/distribution-transport.js +5 -0
  744. package/src/storage/download/integrity.d.ts +33 -0
  745. package/src/storage/download/integrity.js +130 -0
  746. package/src/storage/download/resume-inspection.d.ts +22 -0
  747. package/src/storage/download/resume-inspection.js +134 -0
  748. package/src/storage/download/retry.js +57 -0
  749. package/src/storage/download/state.js +131 -0
  750. package/src/storage/download/transport.js +99 -0
  751. package/src/storage/downloader.js +74 -453
  752. package/src/storage/model-cache.d.ts +1 -0
  753. package/src/storage/model-cache.js +51 -11
  754. package/src/storage/shard-manager.d.ts +5 -0
  755. package/src/storage/shard-manager.js +50 -369
  756. package/src/storage/shards/index.js +60 -0
  757. package/src/storage/shards/integrity.js +225 -0
  758. package/src/storage/shards/lifecycle.js +22 -0
  759. package/src/storage/source-artifact-store.js +14 -4
  760. package/src/tooling/boundary-evidence.d.ts +4 -4
  761. package/src/tooling/boundary-evidence.js +20 -20
  762. package/src/tooling/browser-command-runner.js +13 -6
  763. package/src/tooling/capsule-v2.d.ts +11 -0
  764. package/src/tooling/capsule-v2.js +58 -0
  765. package/src/tooling/capsule.js +20 -0
  766. package/src/tooling/command-api-constants.js +1 -1
  767. package/src/tooling/command-api-family-normalizers.js +163 -5
  768. package/src/tooling/command-api.d.ts +42 -3
  769. package/src/tooling/command-api.js +7 -2
  770. package/src/tooling/command-envelope.js +1 -1
  771. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  772. package/src/tooling/command-runner.html +49 -1
  773. package/src/tooling/command-validation.js +1 -24
  774. package/src/tooling/conversion-run-timing.js +24 -0
  775. package/src/tooling/diagnose-runner.js +11 -9
  776. package/src/tooling/distribution-shard-transport.js +11 -0
  777. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  778. package/src/tooling/execution-cost-ledger.js +197 -15
  779. package/src/tooling/hf-registry/catalog-io.js +63 -0
  780. package/src/tooling/litert/package.js +863 -0
  781. package/src/tooling/litert-package-runtime.js +3 -849
  782. package/src/tooling/model-capsule-forge.js +478 -0
  783. package/src/tooling/node-browser/execution.d.ts +40 -0
  784. package/src/tooling/node-browser/execution.js +308 -0
  785. package/src/tooling/node-browser/static-server.js +188 -0
  786. package/src/tooling/node-browser/transport.d.ts +80 -0
  787. package/src/tooling/node-browser/transport.js +622 -0
  788. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  789. package/src/tooling/node-browser-command-runner.js +10 -1112
  790. package/src/tooling/node-capsule-artifact-store.js +44 -0
  791. package/src/tooling/node-command-execution.js +237 -0
  792. package/src/tooling/node-command-runner.js +26 -205
  793. package/src/tooling/node-converter/input.js +521 -0
  794. package/src/tooling/node-converter/output.js +244 -0
  795. package/src/tooling/node-converter-chunk-batches.js +21 -0
  796. package/src/tooling/node-converter.js +7 -720
  797. package/src/tooling/node-source-runtime.js +16 -276
  798. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  799. package/src/tooling/node-webgpu.js +178 -453
  800. package/src/tooling/opfs-cache.d.ts +5 -0
  801. package/src/tooling/opfs-cache.js +4 -0
  802. package/src/tooling/production-release.d.ts +19 -0
  803. package/src/tooling/production-release.js +647 -0
  804. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  805. package/src/tooling/program-bundle/materialize.js +324 -0
  806. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  807. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  808. package/src/tooling/program-bundle/validation.js +31 -0
  809. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  810. package/src/tooling/program-bundle-parity.d.ts +11 -4
  811. package/src/tooling/program-bundle-parity.js +262 -109
  812. package/src/tooling/program-bundle.d.ts +7 -0
  813. package/src/tooling/program-bundle.js +186 -591
  814. package/src/tooling/provider-v1-contract.d.ts +61 -0
  815. package/src/tooling/provider-v1-contract.js +192 -0
  816. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  817. package/src/tooling/registered-variant-calibration-job.js +210 -0
  818. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  819. package/src/tooling/registered-variant-calibration.js +57 -7
  820. package/src/tooling/runtime-optimization/candidates.js +664 -0
  821. package/src/tooling/runtime-optimization-campaign.js +166 -0
  822. package/src/tooling/runtime-optimization-index.js +16 -1
  823. package/src/tooling/runtime-optimization.d.ts +44 -0
  824. package/src/tooling/runtime-optimization.js +13 -656
  825. package/src/tooling/source-intake.d.ts +1 -1
  826. package/src/tooling/source-intake.js +34 -17
  827. package/src/tooling/source-package-profiles.js +1 -1
  828. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  829. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  830. package/src/tooling/source-runtime/node-loader.js +258 -0
  831. package/src/tooling/source-runtime-bundle.js +26 -411
  832. package/src/tooling/source-runtime-converter-config.js +1 -1
  833. package/src/tooling-experimental-exports.js +0 -5
  834. package/src/tooling-exports/evidence.d.ts +14 -3
  835. package/src/tooling-exports/evidence.js +14 -3
  836. package/src/tooling-exports/storage.d.ts +5 -0
  837. package/src/tooling-exports/storage.js +4 -0
  838. package/src/tooling-exports.d.ts +3 -0
  839. package/src/tooling-exports.js +2 -0
  840. package/src/tooling-exports.shared.d.ts +17 -3
  841. package/src/tooling-exports.shared.js +9 -3
  842. package/src/training.d.ts +6 -6
  843. package/src/training.js +4 -4
  844. package/src/version.js +1 -1
  845. package/tests/kernels/browser/kernel-suite.js +512 -0
  846. package/tests/kernels/browser/test-page.js +169 -14
  847. package/tests/kernels/harness/tolerance.js +1 -0
  848. package/tests/kernels/reference/index.js +13 -1
  849. package/tests/kernels/reference/vision.js +160 -0
  850. package/src/converter/quantizer.d.ts +0 -121
  851. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  852. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  853. package/src/gpu/kernels/utils.d.ts +0 -147
  854. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  855. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  856. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  857. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  858. package/src/rules/converter/execution.rules.json +0 -20
  859. package/src/rules/converter/tensor-roles.rules.json +0 -13
  860. package/src/rules/converter/tokenizer.rules.json +0 -12
  861. package/src/rules/inference/attention.rules.json +0 -69
  862. package/src/rules/inference/config.rules.json +0 -82
  863. package/src/rules/inference/dtype.rules.json +0 -113
  864. package/src/rules/inference/execution.rules.json +0 -195
  865. package/src/rules/inference/ffn.rules.json +0 -123
  866. package/src/rules/inference/layer-pattern.rules.json +0 -16
  867. package/src/rules/inference/layer.rules.json +0 -7
  868. package/src/rules/inference/moe.rules.json +0 -145
  869. package/src/rules/kernels/attention.rules.json +0 -68
  870. package/src/rules/kernels/conv2d.rules.json +0 -6
  871. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  872. package/src/rules/kernels/dequant.rules.json +0 -58
  873. package/src/rules/kernels/energy.rules.json +0 -22
  874. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  875. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  876. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  877. package/src/rules/kernels/gather.rules.json +0 -93
  878. package/src/rules/kernels/gelu.rules.json +0 -11
  879. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  880. package/src/rules/kernels/groupnorm.rules.json +0 -10
  881. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  882. package/src/rules/kernels/layernorm.rules.json +0 -6
  883. package/src/rules/kernels/linear-attention.rules.json +0 -6
  884. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  885. package/src/rules/kernels/matmul.rules.json +0 -95
  886. package/src/rules/kernels/modulate.rules.json +0 -6
  887. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  888. package/src/rules/kernels/moe.rules.json +0 -14
  889. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  890. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  891. package/src/rules/kernels/relu.rules.json +0 -6
  892. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  893. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  894. package/src/rules/kernels/residual.rules.json +0 -12
  895. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  896. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  897. package/src/rules/kernels/rope-qk.rules.json +0 -6
  898. package/src/rules/kernels/rope.rules.json +0 -6
  899. package/src/rules/kernels/sample.rules.json +0 -6
  900. package/src/rules/kernels/scale.rules.json +0 -8
  901. package/src/rules/kernels/silu.rules.json +0 -22
  902. package/src/rules/kernels/softmax.rules.json +0 -25
  903. package/src/rules/kernels/split-qg.rules.json +0 -6
  904. package/src/rules/kernels/split-qkv.rules.json +0 -6
  905. package/src/rules/kernels/upsample2d.rules.json +0 -6
  906. package/src/rules/loader/tensor-loader.rules.json +0 -42
  907. package/src/rules/loader/weights.rules.json +0 -42
  908. package/src/rules/tooling/command-runtime.rules.json +0 -65
  909. package/src/tooling/hf-registry-utils.js +0 -395
  910. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  911. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  912. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  913. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  914. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  915. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  916. /package/src/{utils → formats}/array-buffer.js +0 -0
  917. /package/src/{utils → formats}/canonical-hash.js +0 -0
  918. /package/src/{utils → formats}/clone-json.js +0 -0
  919. /package/src/{utils → formats}/encode-utf8.js +0 -0
  920. /package/src/{utils → formats}/load-json.js +0 -0
  921. /package/src/{utils → formats}/plain-object.js +0 -0
  922. /package/src/{utils → formats}/sha256.js +0 -0
  923. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  924. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  925. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  926. /package/src/{utils → storage}/runtime-env.js +0 -0
  927. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -32,7 +32,7 @@ import {
32
32
  initTokenizerFromManifest,
33
33
  assertManifestComputeLaneBinding,
34
34
  } from './text/model-load.js';
35
- import { resolvePerLayerInputsSession } from './text/generator-helpers.js';
35
+ import { resolvePerLayerInputsSession } from './text/generator/session-context.js';
36
36
  import { getKernelPathActivationDtype } from '../../config/kernel-path-loader.js';
37
37
  import { applyPipelineDebugConfig } from './text/debug-utils.js';
38
38
  import { resolveLayerPipeline } from './text/layer-plan.js';
@@ -51,259 +51,34 @@ import { registerPipeline, getPipelineFactory } from './registry.js';
51
51
  import { selectRuleValue } from '../../rules/rule-registry.js';
52
52
  import { createObservationContext } from '../observation-context.js';
53
53
  import { createResolvedRuntimeSession } from './text/resolved-runtime-session.js';
54
-
55
- // AbortSignal contract: every public inference primitive on this pipeline
56
- // accepts `options.signal` (or `args.signal`). When the signal aborts the
57
- // call rejects with `AbortError` so callers (the dream typing-clock
58
- // controller, batch eval scripts, etc.) can cancel superseded work without
59
- // waiting for the model to finish. Internal checkpoints fire between
60
- // dispatches; mid-dispatch cancellation is not provided because WebGPU
61
- // command-buffer cancellation is not exposed by the spec.
62
- export class AbortError extends Error {
63
- constructor(message = 'Doppler: aborted') {
64
- super(message);
65
- this.name = 'AbortError';
66
- this.code = 'ABORT_ERR';
67
- }
68
- }
69
-
70
- export function isAbortError(err) {
71
- return !!err && (err.name === 'AbortError' || err.code === 'ABORT_ERR' || err.code === 20);
72
- }
73
-
74
- function assertNotAborted(signal) {
75
- if (signal && signal.aborted) {
76
- throw new AbortError(typeof signal.reason === 'string' ? signal.reason : 'Doppler: aborted');
77
- }
78
- }
79
-
80
- function poolSequenceTokenEmbeddings(tokenEmbeddings, tokenIds, hiddenSize, pooling) {
81
- if (!(tokenEmbeddings instanceof Float32Array)) {
82
- throw new Error('Sequence token embeddings are unavailable.');
83
- }
84
- if (tokenEmbeddings.length !== tokenIds.length * hiddenSize) {
85
- throw new Error(
86
- `Sequence embedding shape mismatch: expected ${tokenIds.length * hiddenSize}, got ${tokenEmbeddings.length}.`
87
- );
88
- }
89
- const excluded = new Set(pooling.excludeTokenIds);
90
- const includedIndices = [];
91
- for (let index = 0; index < tokenIds.length; index += 1) {
92
- if (!excluded.has(tokenIds[index])) includedIndices.push(index);
93
- }
94
- if (includedIndices.length === 0) {
95
- throw new Error('Sequence pooling excluded every input token.');
96
- }
97
- const pooled = new Float32Array(hiddenSize);
98
- if (pooling.mode === 'last') {
99
- const tokenIndex = includedIndices[includedIndices.length - 1];
100
- const offset = tokenIndex * hiddenSize;
101
- pooled.set(tokenEmbeddings.subarray(offset, offset + hiddenSize));
102
- } else {
103
- for (const tokenIndex of includedIndices) {
104
- const offset = tokenIndex * hiddenSize;
105
- for (let column = 0; column < hiddenSize; column += 1) {
106
- pooled[column] += tokenEmbeddings[offset + column];
107
- }
108
- }
109
- const scale = 1 / includedIndices.length;
110
- for (let column = 0; column < hiddenSize; column += 1) {
111
- pooled[column] *= scale;
112
- }
113
- }
114
- return {
115
- pooled,
116
- tokenMask: Uint8Array.from(tokenIds, (tokenId) => excluded.has(tokenId) ? 0 : 1),
117
- includedTokenCount: includedIndices.length,
118
- };
119
- }
54
+ import { assertBundledAdapterAuthorized } from '../../config/revocation-policy.js';
55
+ import { assertNotAborted } from './text/abort-contract.js';
56
+ export { AbortError, isAbortError } from './text/abort-contract.js';
57
+ import { destroyMoERouter } from './text/pipeline-load-timing.js';
58
+ export { buildConservativeMultimodalGenerationOptions } from './text/modality-token-contract.js';
120
59
  import { initConvLayerState } from './text/ops.js';
121
60
  import { destroyPleBufferCache, destroyPleRuntimeCache } from './text/per-layer-inputs.js';
122
-
123
- function destroyMoERouter(router) {
124
- if (router && typeof router.destroy === 'function') {
125
- router.destroy();
126
- }
127
- }
128
-
129
- function createPipelineLoadPhaseError(error, phase, context = {}) {
130
- const message = error?.message || String(error);
131
- const wrapped = new Error(
132
- `Pipeline load phase "${phase}" failed: ${message}`,
133
- error instanceof Error ? { cause: error } : undefined
134
- );
135
- wrapped.name = error?.name || 'Error';
136
- if (error?.code !== undefined) {
137
- wrapped.code = error.code;
138
- }
139
- wrapped.details = {
140
- ...(error?.details && typeof error.details === 'object' ? error.details : {}),
141
- pipelineLoadPhase: phase,
142
- ...context,
143
- };
144
- return wrapped;
145
- }
146
-
147
- async function withPipelineLoadPhase(phase, context, run) {
148
- try {
149
- return await run();
150
- } catch (error) {
151
- if (error?.details?.pipelineLoadPhase) {
152
- throw error;
153
- }
154
- throw createPipelineLoadPhaseError(error, phase, context);
155
- }
156
- }
157
-
158
- function roundPipelineTimingMs(value) {
159
- return Number.isFinite(value) ? Number(value.toFixed(3)) : null;
160
- }
161
-
162
- function createPipelineLoadTiming(modelId) {
163
- return {
164
- schemaVersion: 1,
165
- source: 'doppler-pipeline',
166
- modelId: typeof modelId === 'string' ? modelId : null,
167
- status: 'running',
168
- phasesMs: {
169
- reset: null,
170
- configResolution: null,
171
- kernelWarmup: null,
172
- tokenizer: null,
173
- executionSetup: null,
174
- loadWeights: null,
175
- rope: null,
176
- convStates: null,
177
- },
178
- details: {
179
- tokenizer: null,
180
- },
181
- totalMs: null,
182
- };
183
- }
184
-
185
- function finishPipelineLoadTimingPhase(loadTiming, phase, startMs) {
186
- if (!loadTiming?.phasesMs || !phase) return;
187
- const elapsedMs = roundPipelineTimingMs(performance.now() - startMs);
188
- const currentMs = loadTiming.phasesMs[phase];
189
- loadTiming.phasesMs[phase] = Number.isFinite(currentMs)
190
- ? roundPipelineTimingMs(currentMs + elapsedMs)
191
- : elapsedMs;
192
- }
193
-
194
- async function timedPipelineLoadPhase(loadTiming, phase, context, run) {
195
- const startMs = performance.now();
196
- try {
197
- return await withPipelineLoadPhase(phase, context, run);
198
- } finally {
199
- finishPipelineLoadTimingPhase(loadTiming, phase, startMs);
200
- }
201
- }
202
-
203
- function resolveSingleSpecialTokenId(tokenizer, tokenText, label) {
204
- const rawTokenIds = tokenizer?.encode?.(tokenText);
205
- const tokenIds = Array.isArray(rawTokenIds)
206
- ? rawTokenIds
207
- : (ArrayBuffer.isView(rawTokenIds) ? Array.from(rawTokenIds) : null);
208
- if (!Array.isArray(tokenIds) || tokenIds.length !== 1) {
209
- throw new Error(
210
- `[Pipeline] transcribeImage: tokenizer must encode ${label} "${tokenText}" as exactly one token.`
211
- );
212
- }
213
- const tokenId = Number(tokenIds[0]);
214
- if (!Number.isFinite(tokenId) || Math.floor(tokenId) !== tokenId || tokenId < 0) {
215
- throw new Error(
216
- `[Pipeline] transcribeImage: tokenizer returned invalid ${label} token id "${tokenIds[0]}".`
217
- );
218
- }
219
- return tokenId;
220
- }
221
-
222
- function expandImagePlaceholderTokenIds(tokenIds, imageTokenId, numImageTokens, options = {}) {
223
- const normalizedTokenIds = Array.isArray(tokenIds)
224
- ? Int32Array.from(tokenIds)
225
- : (ArrayBuffer.isView(tokenIds) ? Int32Array.from(tokenIds) : null);
226
- if (!(normalizedTokenIds instanceof Int32Array)) {
227
- throw new Error(
228
- '[Pipeline] transcribeImage: tokenizer.encode() must return an array or typed array of token IDs.'
229
- );
230
- }
231
- if (!Number.isFinite(numImageTokens) || Math.floor(numImageTokens) !== numImageTokens || numImageTokens < 1) {
232
- throw new Error(
233
- `[Pipeline] transcribeImage: image token span must be a positive integer, got ${numImageTokens}.`
234
- );
235
- }
236
-
237
- let placeholderIndex = -1;
238
- let placeholderCount = 0;
239
- for (let i = 0; i < normalizedTokenIds.length; i++) {
240
- if (normalizedTokenIds[i] !== imageTokenId) continue;
241
- if (placeholderIndex < 0) {
242
- placeholderIndex = i;
243
- }
244
- placeholderCount++;
245
- }
246
-
247
- if (placeholderCount !== 1) {
248
- throw new Error(
249
- `[Pipeline] transcribeImage: expected exactly one image_token_id (${imageTokenId}) placeholder ` +
250
- `from the chat template, got ${placeholderCount}.`
251
- );
252
- }
253
-
254
- const boiTokenId = Number.isInteger(options.boiTokenId) ? options.boiTokenId : null;
255
- const eoiTokenId = Number.isInteger(options.eoiTokenId) ? options.eoiTokenId : null;
256
- const prefixExtra = boiTokenId == null ? 0 : 1;
257
- const suffixExtra = eoiTokenId == null ? 0 : 1;
258
- const expandedLength = normalizedTokenIds.length - 1 + prefixExtra + numImageTokens + suffixExtra;
259
- const expanded = new Int32Array(expandedLength);
260
- expanded.set(normalizedTokenIds.subarray(0, placeholderIndex), 0);
261
- let writeOffset = placeholderIndex;
262
- if (boiTokenId != null) {
263
- expanded[writeOffset++] = boiTokenId;
264
- }
265
- expanded.fill(imageTokenId, writeOffset, writeOffset + numImageTokens);
266
- const imageStartOffset = writeOffset;
267
- writeOffset += numImageTokens;
268
- if (eoiTokenId != null) {
269
- expanded[writeOffset++] = eoiTokenId;
270
- }
271
- expanded.set(
272
- normalizedTokenIds.subarray(placeholderIndex + 1),
273
- writeOffset
274
- );
275
-
276
- return {
277
- inputIds: expanded,
278
- imageStartOffset,
279
- };
280
- }
281
-
282
- export function buildConservativeMultimodalGenerationOptions(options = {}) {
283
- return {
284
- ...options,
285
- disableCommandBatching: true,
286
- disableMultiTokenDecode: true,
287
- stopCheckMode: 'per-token',
288
- };
289
- }
290
-
291
- function resolveMultimodalMaxTokens(runtimeConfig, requestedMaxTokens) {
292
- if (requestedMaxTokens !== undefined) {
293
- return requirePositiveInteger(requestedMaxTokens, 'maxTokens');
294
- }
295
- return requirePositiveInteger(
296
- runtimeConfig?.inference?.generation?.multimodalMaxTokens,
297
- 'runtime.inference.generation.multimodalMaxTokens'
298
- );
299
- }
300
-
301
- function requirePositiveInteger(value, label) {
302
- if (!Number.isInteger(value) || value <= 0) {
303
- throw new Error(`${label} must be a positive integer.`);
304
- }
305
- return value;
306
- }
61
+ import {
62
+ initialize as initializeImpl,
63
+ loadModel as loadModelImpl,
64
+ _loadWeights as _loadWeightsImpl,
65
+ _initRoPE as _initRoPEImpl,
66
+ _initConvLayerStates as _initConvLayerStatesImpl,
67
+ _loadVisionWeights as _loadVisionWeightsImpl,
68
+ _ensureVisionWeightsLoaded as _ensureVisionWeightsLoadedImpl,
69
+ _loadAudioWeights as _loadAudioWeightsImpl,
70
+ _ensureAudioWeightsLoaded as _ensureAudioWeightsLoadedImpl,
71
+ } from './text/lifecycle.js';
72
+ import {
73
+ transcribeVideo as transcribeVideoImpl,
74
+ transcribeAudio as transcribeAudioImpl,
75
+ embed as embedImpl,
76
+ embedBatch as embedBatchImpl,
77
+ encodeSequence as encodeSequenceImpl,
78
+ embedImage as embedImageImpl,
79
+ embedAudio as embedAudioImpl,
80
+ } from './text/execution.js';
81
+ import { transcribeImage as transcribeImageImpl } from './text/image-transcription.js';
307
82
 
308
83
  // ============================================================================
309
84
  // Main Inference Pipeline Class
@@ -317,7 +92,6 @@ export class InferencePipeline extends PipelineState {
317
92
 
318
93
  _onProgress = null;
319
94
 
320
-
321
95
  _preloadedWeights = null;
322
96
  runtimeOverrides = null;
323
97
 
@@ -333,1213 +107,70 @@ export class InferencePipeline extends PipelineState {
333
107
  // Initialization
334
108
  // ==========================================================================
335
109
 
336
-
337
110
  async initialize(contexts = {}) {
338
- const { runtimeConfig, sharedDebug } = applyPipelineContexts(this, contexts, {
339
- assignGpuContext: true,
340
- assignUseGPU: true,
341
- assignMemoryContext: true,
342
- assignStorageContext: true,
343
- });
344
- this.runtimeConfig = runtimeConfig;
345
- this.commandContext = contexts.commandContext ?? null;
346
- this.observationContext = createObservationContext({
347
- runtimeConfig,
348
- commandContext: this.commandContext,
349
- });
350
- this.dopplerLoader = contexts.loader || null;
351
- this.ownsDopplerLoader = contexts.ownsLoader === true;
352
- this.runtimeOverrides = contexts.runtimeConfig == null
353
- ? null
354
- : (typeof structuredClone === 'function'
355
- ? structuredClone(contexts.runtimeConfig)
356
- : JSON.parse(JSON.stringify(contexts.runtimeConfig)));
357
- applyPipelineDebugConfig(sharedDebug?.pipeline);
358
- configurePerfGuards(sharedDebug?.perfGuards);
359
-
360
- if (!this.gpuContext?.device && typeof globalThis.navigator !== 'undefined' && globalThis.navigator?.gpu) {
361
- const device = await initDevice();
362
- if (!device || typeof device !== 'object' || typeof device.createBuffer !== 'function' || !device.queue) {
363
- throw new Error(
364
- 'GPU device initialization returned an invalid device object. ' +
365
- 'Expected an object with queue and createBuffer. Check WebGPU adapter availability.'
366
- );
367
- }
368
- this.gpuContext = { device };
369
- this.useGPU = true;
370
- }
371
-
372
- this.emulation = await initEmulation(this.runtimeConfig);
373
-
374
- this.debug = sharedDebug?.pipeline?.enabled === true;
375
- log.debug('Pipeline', 'Initialized', { useGPU: this.useGPU, debug: this.debug });
111
+ return initializeImpl.call(this, contexts);
376
112
  }
377
113
 
378
-
379
114
  async loadModel(manifest) {
380
- const loadStart = performance.now();
381
- const pipelineLoadTiming = createPipelineLoadTiming(manifest?.modelId ?? null);
382
- const resetStart = performance.now();
383
- this.stats.pipelineLoadTiming = pipelineLoadTiming;
384
- this.manifest = manifest;
385
- this.decodeRing?.release();
386
- if (this.sampleReadbackBuffer) {
387
- this.sampleReadbackBuffer.destroy();
388
- this.sampleReadbackBuffer = null;
389
- }
390
- this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
391
- destroyMoERouter(this.moeRouter);
392
- this.moeRouter = null;
393
- finishPipelineLoadTimingPhase(pipelineLoadTiming, 'reset', resetStart);
394
-
395
- // ========================================================================
396
- // Config Resolution Passes
397
- //
398
- // The following passes mutate this.runtimeConfig in a fixed order.
399
- // Each pass is allowed to read the full runtimeConfig but must only
400
- // mutate its own documented subset. Reordering passes may change
401
- // resolved values.
402
- //
403
- // Phase 1 — applyExecutionV1RuntimeConfig
404
- // Reads: manifest.inference.execution, kernelCapabilities, platform
405
- // Mutates: runtimeConfig.inference (kernelPath, pipeline, compute,
406
- // session via runtimeInferencePatch)
407
- //
408
- // Phase 2 — parseModelConfig + applyModelBatchingRuntimeDefaults
409
- // Reads: manifest.architecture, runtimeConfig.inference.modelOverrides
410
- // Mutates: runtimeConfig.inference.batching,
411
- // runtimeConfig.inference.generation
412
- //
413
- // Phase 3 — resolveKernelPathState
414
- // Reads: manifest, modelConfig.kernelPath, runtimeConfig.inference.kernelPath
415
- // Mutates: runtimeConfig.inference.compute.activationDtype,
416
- // runtimeConfig.inference.session.kvcache.kvDtype,
417
- // runtimeConfig.inference.session.compute.defaults.outputDtype
418
- //
419
- // Phase 4 — _resolveLayerPipeline
420
- // Reads: runtimeConfig.inference.pipeline, modelConfig.layerPipeline,
421
- // executionV1State.runtimeInferencePatch.pipeline
422
- // Mutates: this.layerPipelinePlan (does not mutate runtimeConfig)
423
- // ========================================================================
424
-
425
- let configResolutionPhase = 0;
426
- const configResolutionStart = performance.now();
427
-
428
- // Phase 1: execution-v1 runtime config
429
- configResolutionPhase = 1;
430
- log.debug('Pipeline', `Config resolution phase ${configResolutionPhase}: applyExecutionV1RuntimeConfig`);
431
- if (hasExecutionV1(manifest.inference)) {
432
- let capabilities = null;
433
- let platform = null;
434
- try {
435
- capabilities = getKernelCapabilities();
436
- } catch {
437
- // Device not yet initialized — transforms will be skipped
438
- }
439
- try {
440
- platform = getPlatform();
441
- } catch {
442
- // Platform not yet initialized — use null fallback
443
- }
444
-
445
- const executionV1Runtime = applyExecutionV1RuntimeConfig({
446
- runtimeConfig: this.runtimeConfig,
447
- runtimeOverrides: this.runtimeOverrides,
448
- manifest,
449
- modelId: manifest.modelId ?? 'model',
450
- numLayers: Number(manifest.architecture?.numLayers ?? 0),
451
- capabilities,
452
- platform,
453
- useGPU: this.useGPU === true,
454
- });
455
- if (executionV1Runtime.executionV1State) {
456
- this.runtimeConfig = executionV1Runtime.runtimeConfig;
457
- this.executionV1State = executionV1Runtime.executionV1State;
458
- const transformInfo = this.executionV1State.appliedTransforms?.length > 0
459
- ? `, transforms=[${this.executionV1State.appliedTransforms.join(', ')}]`
460
- : '';
461
- const fallbackInfo = this.executionV1State.fallbackKernelPath
462
- ? ', fallbackKernelPath=yes'
463
- : '';
464
- const laneIntegrity = this.executionV1State.laneIntegrity;
465
- const laneInfo = laneIntegrity?.status === 'transformed'
466
- ? `, laneIntegrity=transformed(declared=${laneIntegrity.declared.activationDtype}/${laneIntegrity.declared.kvDtype},` +
467
- `executed=${laneIntegrity.executed.activationDtype}/${laneIntegrity.executed.kvDtype})`
468
- : '';
469
- log.info(
470
- 'Pipeline',
471
- `Execution v1 enabled (steps=${this.executionV1State.resolvedSteps.all.length}, ` +
472
- `kernelPathInline=${this.executionV1State.runtimeInferencePatch.kernelPath ? 'yes' : 'no'}, ` +
473
- `pipelineInline=${this.executionV1State.runtimeInferencePatch.pipeline ? 'yes' : 'no'}` +
474
- `${transformInfo}${laneInfo}${fallbackInfo})`
475
- );
476
- }
477
- }
478
-
479
- // Phase 2: model config + batching defaults
480
- configResolutionPhase = 2;
481
- log.debug('Pipeline', `Config resolution phase ${configResolutionPhase}: parseModelConfig + applyModelBatchingRuntimeDefaults`);
482
- const modelOverrides = (this.runtimeConfig.inference.modelOverrides);
483
- this.modelConfig = parseModelConfig(manifest, modelOverrides);
484
- this.runtimeConfig = applyModelBatchingRuntimeDefaults(
485
- this.runtimeConfig,
486
- manifest,
487
- this.modelConfig,
488
- this.runtimeOverrides
489
- );
490
- this.useTiedEmbeddings = this.modelConfig.useTiedEmbeddings;
491
- this.embeddingVocabSize = this.modelConfig.embeddingVocabSize;
492
- this.embeddingTranspose = this.modelConfig.embeddingTranspose;
493
-
494
- // Vision capability detection — gated by manifest fields
495
- const imageTokenId = manifest.image_token_id;
496
- const hasVisionQuant = manifest.quantizationInfo?.vision != null;
497
- if (Number.isInteger(imageTokenId) && imageTokenId > 0 && hasVisionQuant) {
498
- this.visionCapable = true;
499
- this.imageTokenId = imageTokenId;
500
- this.visionConfig = this.modelConfig.visionConfig;
501
- if (!this.visionConfig) {
502
- throw new Error(
503
- `Manifest declares image_token_id=${imageTokenId} and quantizationInfo.vision ` +
504
- 'but no vision_config was resolved. Check conversion config.'
505
- );
506
- }
507
- log.info('Pipeline', `Vision capable: imageTokenId=${imageTokenId}`);
508
- } else {
509
- this.visionCapable = false;
510
- }
511
-
512
- // Audio capability detection — gated by manifest fields
513
- const audioTokenId = manifest.audio_token_id;
514
- const hasAudioQuant = manifest.quantizationInfo?.audio != null;
515
- if (Number.isInteger(audioTokenId) && audioTokenId > 0 && hasAudioQuant) {
516
- this.audioCapable = true;
517
- this.audioTokenId = audioTokenId;
518
- this.audioConfig = this.modelConfig.audioConfig;
519
- if (!this.audioConfig) {
520
- throw new Error(
521
- `Manifest declares audio_token_id=${audioTokenId} and quantizationInfo.audio ` +
522
- 'but no audio_config was resolved. Check conversion config.'
523
- );
524
- }
525
- log.info('Pipeline', `Audio capable: audioTokenId=${audioTokenId}`);
526
- } else {
527
- this.audioCapable = false;
528
- }
529
- finishPipelineLoadTimingPhase(pipelineLoadTiming, 'configResolution', configResolutionStart);
530
-
531
- await timedPipelineLoadPhase(
532
- pipelineLoadTiming,
533
- 'kernelWarmup',
534
- { modelId: manifest.modelId ?? null },
535
- () => runKernelWarmup({
536
- useGPU: this.useGPU,
537
- kernelWarmup: this.runtimeConfig.shared?.kernelWarmup,
538
- })
539
- );
540
-
541
- const executionSetupStart = performance.now();
542
- // Phase 3: kernel path resolution + dtype contract
543
- configResolutionPhase = 3;
544
- log.debug('Pipeline', `Config resolution phase ${configResolutionPhase}: resolveKernelPathState`);
545
- const kernelPathState = resolveKernelPathState({
546
- manifest,
547
- runtimeConfig: this.runtimeConfig,
548
- runtimeOverrides: this.runtimeOverrides,
549
- modelConfig: this.modelConfig,
550
- });
551
- this.resolvedKernelPath = kernelPathState.resolvedKernelPath;
552
- this.kernelPathSource = kernelPathState.kernelPathSource;
553
- this.runtimeConfig = kernelPathState.runtimeConfig;
554
-
555
- // Phase 4: layer pipeline resolution
556
- configResolutionPhase = 4;
557
- log.debug('Pipeline', `Config resolution phase ${configResolutionPhase}: _resolveLayerPipeline`);
558
- this._resolveLayerPipeline();
559
- log.debug('Pipeline', `Config resolution complete (${configResolutionPhase} phases)`);
560
-
561
- const cfg = this.modelConfig;
562
- const moeStr = cfg.useMoE ? `, MoE(${cfg.numExperts}x${cfg.moeTopK})` : '';
563
- const kernelInfo = this.resolvedKernelPath ? `kernelPath=${this.resolvedKernelPath.id}` : 'kernelPath=none';
564
- log.info('Pipeline', `${cfg.numLayers}L/${cfg.hiddenSize}H/${cfg.numHeads}heads (${cfg.headDim}dim)${moeStr}, ${kernelInfo}`);
565
- finishPipelineLoadTimingPhase(pipelineLoadTiming, 'executionSetup', executionSetupStart);
566
-
567
- this.tokenizer = await timedPipelineLoadPhase(
568
- pipelineLoadTiming,
569
- 'tokenizer',
570
- { modelId: manifest.modelId ?? null },
571
- () => initTokenizerFromManifest(
572
- manifest,
573
- this.baseUrl,
574
- this.storageContext
575
- )
576
- );
577
- pipelineLoadTiming.details.tokenizer = typeof this.tokenizer?.getLoadTiming === 'function'
578
- ? this.tokenizer.getLoadTiming()
579
- : null;
580
- const tokenizerVocabSize = this.tokenizer.getVocabSize();
581
- if (Number.isFinite(tokenizerVocabSize) && tokenizerVocabSize > 0) {
582
- if (tokenizerVocabSize !== this.modelConfig.vocabSize) {
583
- log.info('Pipeline', `Tokenizer vocabSize=${tokenizerVocabSize} differs from model=${this.modelConfig.vocabSize}, using model size`);
584
- }
585
- }
586
-
587
- const postTokenizerExecutionSetupStart = performance.now();
588
- // Manifest quantizationInfo.compute is the binding lane identity.
589
- assertManifestComputeLaneBinding({ manifest, runtimeConfig: this.runtimeConfig });
590
-
591
- // Initialize KV cache
592
- if (this.modelConfig.decodeStrategy === 'replay_prefill') {
593
- this.kvCache = null;
594
- log.warn(
595
- 'Pipeline',
596
- 'Replay-prefill decode enabled for this model. Incremental KV-cache decode is disabled ' +
597
- 'because the model config did not resolve explicit layerTypes for mixed-geometry/shared-KV decode.'
598
- );
599
- } else {
600
- this.kvCache = createKVCache(this.modelConfig, this.useGPU, this.debug, this.runtimeConfig.inference);
601
- }
602
- this.executionPlanState = compileExecutionPlanState({
603
- runtimeConfig: this.runtimeConfig,
604
- resolvedKernelPath: this.resolvedKernelPath,
605
- kernelPathSource: this.kernelPathSource,
606
- fallbackKernelPath: this.executionV1State?.fallbackKernelPath ?? null,
607
- });
608
- const activeExecutionPlan = resolveActiveExecutionPlan(this);
609
- log.info(
610
- 'Pipeline',
611
- `Execution plan: active=${activeExecutionPlan.id}, dtype=${activeExecutionPlan.activationDtype}, ` +
612
- `kernelPath=${activeExecutionPlan.kernelPathId ?? 'none'}`
613
- );
614
-
615
- // Issue 1: Validate dtype consistency across all three resolution paths
616
- // (execution plan, runtimeConfig.inference.compute, and layer context).
617
- // The layer context is not yet built at this point, so pass null for it.
618
- // This logs a warning if the execution plan and runtimeConfig disagree.
619
- assertDtypeConsistency(this.executionPlanState, this.runtimeConfig, null);
620
-
621
- const kpActivation = getKernelPathActivationDtype(this.resolvedKernelPath);
622
- if (kpActivation && kpActivation !== activeExecutionPlan.activationDtype) {
623
- throw new Error(
624
- `Dtype contract violation: execution plan activationDtype="${activeExecutionPlan.activationDtype}" ` +
625
- `but kernel path "${this.resolvedKernelPath.id}" declares activationDtype="${kpActivation}".`
626
- );
627
- }
628
- this.resolvedRuntimeSession = createResolvedRuntimeSession({
629
- manifest,
630
- modelConfig: this.modelConfig,
631
- runtimeConfig: this.runtimeConfig,
632
- resolvedKernelPath: this.resolvedKernelPath,
633
- kernelPathSource: this.kernelPathSource,
634
- executionV1State: this.executionV1State,
635
- executionPlanState: this.executionPlanState,
636
- });
637
-
638
- // Initialize MoE router if needed
639
- if (this.modelConfig.useMoE) {
640
- this.moeRouter = new MoERouter({
641
- numExperts: this.modelConfig.numExperts,
642
- topK: this.modelConfig.moeTopK,
643
- hiddenSize: this.modelConfig.hiddenSize,
644
- normalizeWeights: this.runtimeConfig.inference.moe.routing.normalizeWeights,
645
- });
646
- }
647
-
648
- // Initialize speculative decoder
649
- if (manifest.draftModel) {
650
- this.speculativeDecoder = initSpeculativeDecoder(
651
- manifest,
652
- this.runtimeConfig.inference.speculative
653
- );
654
- }
655
- finishPipelineLoadTimingPhase(
656
- pipelineLoadTiming,
657
- 'executionSetup',
658
- postTokenizerExecutionSetupStart
659
- );
660
-
661
- // Load weights
662
- await timedPipelineLoadPhase(
663
- pipelineLoadTiming,
664
- 'loadWeights',
665
- { modelId: manifest.modelId ?? null },
666
- () => this._loadWeights()
667
- );
668
-
669
- // Initialize RoPE frequencies
670
- await timedPipelineLoadPhase(
671
- pipelineLoadTiming,
672
- 'rope',
673
- { modelId: manifest.modelId ?? null },
674
- () => this._initRoPE()
675
- );
676
-
677
- // Initialize conv layer states for gated short conv layers (LFM2)
678
- await timedPipelineLoadPhase(
679
- pipelineLoadTiming,
680
- 'convStates',
681
- { modelId: manifest.modelId ?? null },
682
- () => this._initConvLayerStates()
683
- );
684
-
685
- this.isLoaded = true;
686
- const loadMs = performance.now() - loadStart;
687
- this.stats.modelLoadMs = loadMs;
688
- pipelineLoadTiming.totalMs = roundPipelineTimingMs(loadMs);
689
- pipelineLoadTiming.status = 'complete';
690
- log.info('Pipeline', `Model loaded successfully (${loadMs.toFixed(0)}ms)`);
115
+ return loadModelImpl.call(this, manifest);
691
116
  }
692
117
 
693
-
694
118
  async _loadWeights() {
695
- const result = this._preloadedWeights || await loadWeights(
696
- (this.manifest),
697
- (this.modelConfig),
698
- {
699
- storageContext: this.storageContext ?? undefined,
700
- loadingConfig: this.runtimeConfig.loading,
701
- baseUrl: this.baseUrl ?? undefined,
702
- resolvedKernelPath: this.resolvedKernelPath,
703
- kernelPathSource: this.kernelPathSource,
704
- keepF32Weights: this.runtimeConfig.inference.compute.keepF32Weights === true,
705
- loaderDebug: this.runtimeConfig?.shared?.debug?.loader ?? null,
706
- perLayerInputSession: resolvePerLayerInputsSession(
707
- this.modelConfig.perLayerInputsSession ?? null,
708
- this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
709
- ),
710
- loader: this.dopplerLoader ?? undefined,
711
- onProgress: (info) => {
712
- if (info.stage !== 'layers' && info.stage !== 'shards') {
713
- log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
714
- }
715
- if (this._onProgress) {
716
- this._onProgress({
717
- percent: info.progress * 100,
718
- message: info.message,
719
- stage: info.stage,
720
- layer: info.layer,
721
- total: info.total,
722
- });
723
- }
724
- },
725
- }
726
- );
727
-
728
- result.layerWeights.forEach((w, k) => this.weights.set(k, w));
729
- this.weights.set('embed', result.embeddings);
730
- this.weights.set('lm_head', result.lmHead);
731
- this.weights.set('lm_head_bias', result.lmHeadBias);
732
- if (result.lmHeadBias) {
733
- // Recorded/fused logits currently keep their result on GPU, while the
734
- // manifest-owned decoder bias is loaded as an exact CPU tensor. Route
735
- // biased heads through computeLogits until the execution graph declares
736
- // and records an explicit GPU bias-add step.
737
- this.disableRecordedLogits = true;
738
- this.disableFusedDecode = true;
739
- }
740
- this.weights.set('final_norm', result.finalNorm);
741
- this.weights.set('final_norm_bias', result.finalNormBias);
742
- if (result.finalNormBias) {
743
- this.disableRecordedLogits = true;
744
- this.disableFusedDecode = true;
745
- }
746
- this.weights.set('diffusion_gemma_self_conditioning', result.diffusionGemmaSelfConditioning);
747
- this.weights.set('per_layer_inputs', result.perLayerInputWeights);
748
- this.embeddingPostprocessor = result.embeddingPostprocessor;
749
-
750
- this.layerRouterWeights = result.layerRouterWeights;
751
-
752
- this.dopplerLoader = result.loader ?? getDopplerLoader(this.runtimeConfig.loading);
753
- this.stats.loadTiming = result.loadTiming ?? this.dopplerLoader?.getLoadTiming?.() ?? null;
754
-
755
- if ((this.modelConfig).useMoE && this.moeRouter) {
756
- this.moeRouter = initMoERouter(
757
- (this.modelConfig),
758
- this.runtimeConfig.inference.moe.routing,
759
- result.layerWeights
760
- );
761
- }
762
-
763
- if (this.useGPU && this.modelConfig) {
764
- const session = this.runtimeConfig?.inference?.session ?? null;
765
- fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath, {
766
- allowQ4K: session?.useFusedQKVSplitQKNorm === true
767
- || session?.useFusedQKVSplitQKNormRoPE === true,
768
- });
769
- }
770
-
771
- if (this.useGPU && this.modelConfig) {
772
- const activeExecutionPlan = resolveActiveExecutionPlan(this);
773
- try {
774
- this.decodeBuffers?.ensureBuffers({
775
- hiddenSize: this.modelConfig.hiddenSize,
776
- intermediateSize: this.modelConfig.maxIntermediateSize,
777
- activationDtype: activeExecutionPlan.activationDtype,
778
- enablePingPong: true,
779
- });
780
-
781
- const device = getDevice();
782
- if (device) {
783
- this.finitenessBuffer = device.createBuffer({
784
- label: 'finiteness_status',
785
- size: 16,
786
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST,
787
- });
788
- }
789
- } catch (bufferError) {
790
- this.decodeBuffers?.release();
791
- if (this.finitenessBuffer) {
792
- this.finitenessBuffer.destroy();
793
- this.finitenessBuffer = null;
794
- }
795
- throw bufferError;
796
- }
797
- }
119
+ return _loadWeightsImpl.call(this);
798
120
  }
799
121
 
800
-
801
122
  setPreloadedWeights(weights) {
802
123
  this._preloadedWeights = weights;
803
124
  }
804
125
 
805
-
806
126
  async _initRoPE() {
807
- const config = (this.modelConfig);
808
- const maxSeqLen = config.maxSeqLen;
809
- const ropeBuffers = await initRoPEFrequencies({
810
- headDim: config.globalHeadDim ?? config.headDim,
811
- localHeadDim: config.headDim,
812
- rotaryDim: config.ropeRotaryDim,
813
- ropeLocalRotaryDim: config.ropeLocalRotaryDim,
814
- ropeFrequencyBaseDim: config.ropeFrequencyBaseDim,
815
- ropeLocalFrequencyBaseDim: config.ropeLocalFrequencyBaseDim,
816
- maxSeqLen,
817
- ropeTheta: config.ropeTheta,
818
- ropeLocalTheta: config.ropeLocalTheta,
819
- mropeInterleaved: config.mropeInterleaved,
820
- mropeSection: config.mropeSection,
821
- partialRotaryFactor: config.partialRotaryFactor,
822
- ropeLocalPartialRotaryFactor: config.ropeLocalPartialRotaryFactor,
823
- ropeScale: config.ropeScale,
824
- ropeLocalScale: config.ropeLocalScale,
825
- ropeScalingType: config.ropeScalingType,
826
- ropeLocalScalingType: config.ropeLocalScalingType,
827
- ropeScaling: config.ropeScaling,
828
- ropeLocalScaling: config.ropeLocalScaling,
829
- }, this.useGPU);
830
- this.ropeFreqsCos = ropeBuffers.cos;
831
- this.ropeFreqsSin = ropeBuffers.sin;
832
- this.ropeLocalCos = ropeBuffers.localCos ?? null;
833
- this.ropeLocalSin = ropeBuffers.localSin ?? null;
127
+ return _initRoPEImpl.call(this);
834
128
  }
835
129
 
836
-
837
130
  async _initConvLayerStates() {
838
- const config = this.modelConfig;
839
- if (!config?.layerTypes) return;
840
- const { getDevice } = await import('../../gpu/device.js');
841
- const device = getDevice();
842
- if (!device) return;
843
-
844
- const hiddenSize = config.hiddenSize;
845
- const convStates = new Map();
846
-
847
- for (let i = 0; i < config.layerTypes.length; i++) {
848
- const lt = String(config.layerTypes[i] ?? '').toLowerCase();
849
- if (lt !== 'conv' && lt !== 'convolution') continue;
850
-
851
- const layerWeights = this.weights.get(`layer_${i}`);
852
- if (!layerWeights) continue;
853
- const convKernel = layerWeights?.convKernel;
854
- if (!convKernel) continue;
855
-
856
- const convState = {};
857
- try {
858
- await initConvLayerState(
859
- convState,
860
- convKernel,
861
- layerWeights.convInProj ?? null,
862
- hiddenSize,
863
- `L${i}.conv`,
864
- i
865
- );
866
- if (!convState.convWeightGPU || !convState.convStateGPU) {
867
- continue;
868
- }
869
- convStates.set(i, convState);
870
- } catch (e) {
871
- log.warn('Pipeline', `Conv layer ${i} state init failed: ${e.message}`);
872
- }
873
- }
874
-
875
- if (convStates.size > 0) {
876
- this.convLayerStates = convStates;
877
- log.info('Pipeline', `Initialized ${convStates.size} conv layer states (kernelSize=${convStates.values().next().value?.kernelSize})`);
878
- }
131
+ return _initConvLayerStatesImpl.call(this);
879
132
  }
880
133
 
881
-
882
134
  async _loadVisionWeights() {
883
- const loader = this.dopplerLoader ?? getDopplerLoader(this.runtimeConfig.loading);
884
- const vc = this.visionConfig;
885
- const depth = vc.depth;
886
-
887
- const loadRequiredTensor = async (name, toGPU = true) => {
888
- const tensor = await loader.loadTensor(name, toGPU, true);
889
- if (!tensor) {
890
- throw new Error(`Vision tensor "${name}" is missing from the converted artifact.`);
891
- }
892
- return tensor;
893
- };
894
- const loadScalar = async (name) => {
895
- const tensor = await loadRequiredTensor(name, false);
896
- if (tensor instanceof Float32Array) {
897
- if (tensor.length !== 1) {
898
- throw new Error(`Vision scalar "${name}" must be a single-element tensor, got length=${tensor.length}.`);
899
- }
900
- return tensor[0];
901
- }
902
- if (ArrayBuffer.isView(tensor) && tensor.length === 1) {
903
- return Number(tensor[0]);
904
- }
905
- if (typeof tensor === 'number') {
906
- return tensor;
907
- }
908
- throw new Error(
909
- `Vision scalar "${name}" must decode to a single numeric value, ` +
910
- `got ${tensor?.constructor?.name ?? typeof tensor} length=${tensor?.length ?? 'N/A'}.`
911
- );
912
- };
913
- const loadClipRange = async (prefix) => ({
914
- inputMin: await loadScalar(`${prefix}.input_min`),
915
- inputMax: await loadScalar(`${prefix}.input_max`),
916
- outputMin: await loadScalar(`${prefix}.output_min`),
917
- outputMax: await loadScalar(`${prefix}.output_max`),
918
- });
919
-
920
- if (vc.visionArchitecture === 'gemma4') {
921
- const isEncoderFree = (depth === 0);
922
- const visionWeights = {
923
- textHiddenSize: this.modelConfig.hiddenSize,
924
- patchInputProj: await loadRequiredTensor('model.vision_tower.patch_embedder.input_proj.weight'),
925
- patchPositionEmbeddingTable: await loadRequiredTensor('model.vision_tower.patch_embedder.position_embedding_table', false),
926
- projector: isEncoderFree
927
- ? await loader.loadTensor('model.embed_vision.embedding_projection.weight', true, true)
928
- : await loadRequiredTensor('model.embed_vision.embedding_projection.weight'),
929
- layers: [],
930
- };
931
-
932
- if (!(visionWeights.patchPositionEmbeddingTable instanceof Float32Array)) {
933
- throw new Error(
934
- 'Gemma 4 vision position_embedding_table must decode to Float32Array on CPU. ' +
935
- 'Re-convert the artifact if this tensor was quantized incorrectly.'
936
- );
937
- }
938
-
939
- for (let i = 0; i < depth; i++) {
940
- const prefix = `model.vision_tower.encoder.layers.${i}`;
941
- const attnPrefix = `${prefix}.self_attn`;
942
- const mlpPrefix = `${prefix}.mlp`;
943
- visionWeights.layers.push({
944
- inputLayerNorm: await loadRequiredTensor(`${prefix}.input_layernorm.weight`),
945
- postAttentionLayerNorm: await loadRequiredTensor(`${prefix}.post_attention_layernorm.weight`),
946
- preFeedforwardLayerNorm: await loadRequiredTensor(`${prefix}.pre_feedforward_layernorm.weight`),
947
- postFeedforwardLayerNorm: await loadRequiredTensor(`${prefix}.post_feedforward_layernorm.weight`),
948
- qNorm: await loadRequiredTensor(`${attnPrefix}.q_norm.weight`),
949
- kNorm: await loadRequiredTensor(`${attnPrefix}.k_norm.weight`),
950
- qProj: await loadRequiredTensor(`${attnPrefix}.q_proj.linear.weight`),
951
- kProj: await loadRequiredTensor(`${attnPrefix}.k_proj.linear.weight`),
952
- vProj: await loadRequiredTensor(`${attnPrefix}.v_proj.linear.weight`),
953
- oProj: await loadRequiredTensor(`${attnPrefix}.o_proj.linear.weight`),
954
- qProjClip: await loadClipRange(`${attnPrefix}.q_proj`),
955
- kProjClip: await loadClipRange(`${attnPrefix}.k_proj`),
956
- vProjClip: await loadClipRange(`${attnPrefix}.v_proj`),
957
- oProjClip: await loadClipRange(`${attnPrefix}.o_proj`),
958
- gateProj: await loadRequiredTensor(`${mlpPrefix}.gate_proj.linear.weight`),
959
- upProj: await loadRequiredTensor(`${mlpPrefix}.up_proj.linear.weight`),
960
- downProj: await loadRequiredTensor(`${mlpPrefix}.down_proj.linear.weight`),
961
- gateProjClip: await loadClipRange(`${mlpPrefix}.gate_proj`),
962
- upProjClip: await loadClipRange(`${mlpPrefix}.up_proj`),
963
- downProjClip: await loadClipRange(`${mlpPrefix}.down_proj`),
964
- });
965
- }
966
-
967
- this.visionWeights = visionWeights;
968
- log.info('Pipeline', `Vision weights loaded (${depth} Gemma 4 encoder layers)`);
969
- return;
970
- }
971
-
972
- const visionWeights = {};
973
-
974
- // Patch embedding weights
975
- const patchProjName = 'visual.patch_embed.proj.weight';
976
- const patchProjBiasName = 'visual.patch_embed.proj.bias';
977
- visionWeights.patchProjWeight = await loader.loadTensor(patchProjName, true, true);
978
- visionWeights.patchProjBias = await loader.loadTensor(patchProjBiasName, true, true);
979
-
980
- // Vision encoder layer weights
981
- visionWeights.layers = [];
982
- for (let i = 0; i < depth; i++) {
983
- const prefix = `visual.blocks.${i}`;
984
- const layerW = {
985
- norm1Weight: await loader.loadTensor(`${prefix}.norm1.weight`, true, true),
986
- norm2Weight: await loader.loadTensor(`${prefix}.norm2.weight`, true, true),
987
- qkvWeight: await loader.loadTensor(`${prefix}.attn.qkv.weight`, true, true),
988
- qkvBias: await loader.loadTensor(`${prefix}.attn.qkv.bias`, true, true),
989
- projWeight: await loader.loadTensor(`${prefix}.attn.proj.weight`, true, true),
990
- projBias: await loader.loadTensor(`${prefix}.attn.proj.bias`, true, true),
991
- fc1Weight: await loader.loadTensor(`${prefix}.mlp.fc1.weight`, true, true),
992
- fc1Bias: await loader.loadTensor(`${prefix}.mlp.fc1.bias`, true, true),
993
- fc2Weight: await loader.loadTensor(`${prefix}.mlp.fc2.weight`, true, true),
994
- fc2Bias: await loader.loadTensor(`${prefix}.mlp.fc2.bias`, true, true),
995
- };
996
- visionWeights.layers.push(layerW);
997
- }
998
-
999
- // Spatial merge projection
1000
- visionWeights.mergerLnWeight = await loader.loadTensor('visual.merger.ln_q.weight', true, true);
1001
- visionWeights.mergerMlp0Weight = await loader.loadTensor('visual.merger.mlp.0.weight', true, true);
1002
- visionWeights.mergerMlp0Bias = await loader.loadTensor('visual.merger.mlp.0.bias', true, true);
1003
- visionWeights.mergerMlp2Weight = await loader.loadTensor('visual.merger.mlp.2.weight', true, true);
1004
- visionWeights.mergerMlp2Bias = await loader.loadTensor('visual.merger.mlp.2.bias', true, true);
1005
-
1006
- this.visionWeights = visionWeights;
1007
- log.info('Pipeline', `Vision weights loaded (${depth} encoder layers)`);
135
+ return _loadVisionWeightsImpl.call(this);
1008
136
  }
1009
137
 
1010
138
  async _ensureVisionWeightsLoaded() {
1011
- if (!this.visionCapable) {
1012
- throw new Error(
1013
- 'Pipeline does not support vision weights (no image_token_id in manifest).'
1014
- );
1015
- }
1016
- if (this.visionWeights) {
1017
- return;
1018
- }
1019
- log.info('Pipeline', 'Loading vision weights on demand');
1020
- await this._loadVisionWeights();
139
+ return _ensureVisionWeightsLoadedImpl.call(this);
1021
140
  }
1022
141
 
1023
-
1024
142
  async _loadAudioWeights() {
1025
- const loader = this.dopplerLoader ?? getDopplerLoader(this.runtimeConfig.loading);
1026
- const ac = this.audioConfig;
1027
- const depth = ac.depth;
1028
-
1029
- const loadRequiredTensor = async (name, toGPU = true) => {
1030
- const tensor = await loader.loadTensor(name, toGPU, true);
1031
- if (!tensor) {
1032
- throw new Error(`Audio tensor "${name}" is missing from the converted artifact.`);
1033
- }
1034
- return tensor;
1035
- };
1036
- const loadScalar = async (name) => {
1037
- const tensor = await loadRequiredTensor(name, false);
1038
- if (tensor instanceof Float32Array) {
1039
- if (tensor.length !== 1) {
1040
- throw new Error(`Audio scalar "${name}" must be a single-element tensor, got length=${tensor.length}.`);
1041
- }
1042
- return tensor[0];
1043
- }
1044
- if (ArrayBuffer.isView(tensor) && tensor.length === 1) {
1045
- return Number(tensor[0]);
1046
- }
1047
- if (typeof tensor === 'number') {
1048
- return tensor;
1049
- }
1050
- throw new Error(
1051
- `Audio scalar "${name}" must decode to a single numeric value, ` +
1052
- `got ${tensor?.constructor?.name ?? typeof tensor} length=${tensor?.length ?? 'N/A'}.`
1053
- );
1054
- };
1055
- const loadClipRange = async (prefix) => ({
1056
- inputMin: await loadScalar(`${prefix}.input_min`),
1057
- inputMax: await loadScalar(`${prefix}.input_max`),
1058
- outputMin: await loadScalar(`${prefix}.output_min`),
1059
- outputMax: await loadScalar(`${prefix}.output_max`),
1060
- });
1061
-
1062
- const isEncoderFree = (depth === 0);
1063
- const audioWeights = {
1064
- // Subsampling
1065
- subsampleConv0Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.conv.weight'),
1066
- subsampleNorm0Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer0.norm.weight'),
1067
- subsampleConv1Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.conv.weight'),
1068
- subsampleNorm1Weight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.layer1.norm.weight'),
1069
- subsampleInputProjWeight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.subsample_conv_projection.input_proj_linear.weight'),
1070
- // Output
1071
- outputProjWeight: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.output_proj.weight'),
1072
- outputProjBias: isEncoderFree ? null : await loadRequiredTensor('model.audio_tower.output_proj.bias'),
1073
- audioEmbeddingProjWeight: await loadRequiredTensor('model.embed_audio.embedding_projection.weight'),
1074
- layers: [],
1075
- };
1076
-
1077
- for (let i = 0; i < depth; i++) {
1078
- const prefix = `model.audio_tower.layers.${i}`;
1079
- const layer = {
1080
- // Feed-forward 1 (Macaron half-step)
1081
- feedForward1: {
1082
- preLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward1.pre_layer_norm.weight`),
1083
- ffwLayer1Weight: await loadRequiredTensor(`${prefix}.feed_forward1.ffw_layer_1.linear.weight`),
1084
- ffwLayer1Clip: await loadClipRange(`${prefix}.feed_forward1.ffw_layer_1`),
1085
- ffwLayer2Weight: await loadRequiredTensor(`${prefix}.feed_forward1.ffw_layer_2.linear.weight`),
1086
- ffwLayer2Clip: await loadClipRange(`${prefix}.feed_forward1.ffw_layer_2`),
1087
- postLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward1.post_layer_norm.weight`),
1088
- },
1089
- // Self-attention
1090
- normPreAttn: await loadRequiredTensor(`${prefix}.norm_pre_attn.weight`),
1091
- qProj: await loadRequiredTensor(`${prefix}.self_attn.q_proj.linear.weight`),
1092
- qProjClip: await loadClipRange(`${prefix}.self_attn.q_proj`),
1093
- kProj: await loadRequiredTensor(`${prefix}.self_attn.k_proj.linear.weight`),
1094
- kProjClip: await loadClipRange(`${prefix}.self_attn.k_proj`),
1095
- vProj: await loadRequiredTensor(`${prefix}.self_attn.v_proj.linear.weight`),
1096
- vProjClip: await loadClipRange(`${prefix}.self_attn.v_proj`),
1097
- perDimScale: await loadRequiredTensor(`${prefix}.self_attn.per_dim_scale`),
1098
- relativeKProj: await loadRequiredTensor(`${prefix}.self_attn.relative_k_proj.weight`),
1099
- postProj: await loadRequiredTensor(`${prefix}.self_attn.post.linear.weight`),
1100
- postProjClip: await loadClipRange(`${prefix}.self_attn.post`),
1101
- normPostAttn: await loadRequiredTensor(`${prefix}.norm_post_attn.weight`),
1102
- // Convolution module (LConv1D)
1103
- lconvPreLayerNorm: await loadRequiredTensor(`${prefix}.lconv1d.pre_layer_norm.weight`),
1104
- lconvLinearStartWeight: await loadRequiredTensor(`${prefix}.lconv1d.linear_start.linear.weight`),
1105
- lconvLinearStartClip: await loadClipRange(`${prefix}.lconv1d.linear_start`),
1106
- lconvDepthwiseWeight: await loadRequiredTensor(`${prefix}.lconv1d.depthwise_conv1d.weight`),
1107
- lconvConvNorm: await loadRequiredTensor(`${prefix}.lconv1d.conv_norm.weight`),
1108
- lconvLinearEndWeight: await loadRequiredTensor(`${prefix}.lconv1d.linear_end.linear.weight`),
1109
- lconvLinearEndClip: await loadClipRange(`${prefix}.lconv1d.linear_end`),
1110
- // Feed-forward 2 (Macaron half-step)
1111
- feedForward2: {
1112
- preLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward2.pre_layer_norm.weight`),
1113
- ffwLayer1Weight: await loadRequiredTensor(`${prefix}.feed_forward2.ffw_layer_1.linear.weight`),
1114
- ffwLayer1Clip: await loadClipRange(`${prefix}.feed_forward2.ffw_layer_1`),
1115
- ffwLayer2Weight: await loadRequiredTensor(`${prefix}.feed_forward2.ffw_layer_2.linear.weight`),
1116
- ffwLayer2Clip: await loadClipRange(`${prefix}.feed_forward2.ffw_layer_2`),
1117
- postLayerNorm: await loadRequiredTensor(`${prefix}.feed_forward2.post_layer_norm.weight`),
1118
- },
1119
- // Final layer norm
1120
- normOut: await loadRequiredTensor(`${prefix}.norm_out.weight`),
1121
- };
1122
- audioWeights.layers.push(layer);
1123
- }
1124
-
1125
- this.audioWeights = audioWeights;
1126
- log.info('Pipeline', `Audio weights loaded (${depth} conformer layers)`);
143
+ return _loadAudioWeightsImpl.call(this);
1127
144
  }
1128
145
 
1129
146
  async _ensureAudioWeightsLoaded() {
1130
- if (!this.audioCapable) {
1131
- throw new Error(
1132
- 'Pipeline does not support audio weights (no audio_token_id in manifest).'
1133
- );
1134
- }
1135
- if (this.audioWeights) {
1136
- return;
1137
- }
1138
- log.info('Pipeline', 'Loading audio weights on demand');
1139
- await this._loadAudioWeights();
147
+ return _ensureAudioWeightsLoadedImpl.call(this);
1140
148
  }
1141
149
 
1142
-
1143
150
  // ==========================================================================
1144
151
  // Vision: transcribeImage
1145
152
  // ==========================================================================
1146
153
 
1147
- /**
1148
- * Transcribe text from an image using the vision encoder and text decoder.
1149
- *
1150
- * @param {object} params
1151
- * @param {Uint8Array|Float32Array} params.imageBytes Raw image pixel data
1152
- * @param {number} params.width Image width
1153
- * @param {number} params.height Image height
1154
- * @param {string} [params.prompt] Custom transcription prompt
1155
- * @param {number} [params.maxTokens] Max tokens to generate
1156
- * @param {number} [params.softTokenBudget] Per-request soft token budget (Gemma 4 tiers: 70/140/280/560/1120)
1157
- * @returns {Promise<{ text: string, tokens: number[] }>}
1158
- */
1159
154
  async transcribeImage({ imageBytes, width, height, prompt, maxTokens, softTokenBudget, signal }) {
1160
- assertNotAborted(signal);
1161
- if (!this.visionCapable) {
1162
- throw new Error(
1163
- 'Pipeline does not support image transcription (no image_token_id in manifest).'
1164
- );
1165
- }
1166
- await this._ensureVisionWeightsLoaded();
1167
- assertNotAborted(signal);
1168
-
1169
- this.reset();
1170
-
1171
- // Lazy-load vision module (avoids GPU kernel dependency for text-only pipelines)
1172
- const { encodeImage } = await import('./vision/index.js');
1173
-
1174
- // Step 1: Encode image through vision pipeline
1175
- const encodeResult = await encodeImage({
1176
- pixels: imageBytes,
1177
- width,
1178
- height,
1179
- visionConfig: this.visionConfig,
1180
- weights: this.visionWeights,
1181
- softTokenBudget,
1182
- });
1183
-
1184
- // Step 2: Build the multimodal prompt from the model's chat template and
1185
- // expand the single <|image|> placeholder into the exact visual-token span.
1186
- const requestedPrompt = prompt ?? 'Describe the image in one short sentence.';
1187
- const imageTokenId = this.visionConfig?.imageTokenId ?? this.modelConfig?.imageTokenId;
1188
- if (imageTokenId == null) {
1189
- throw new Error(
1190
- 'Pipeline missing image_token_id. Re-convert the model with image token metadata.'
1191
- );
1192
- }
1193
- if (this.visionConfig?.visionArchitecture !== 'gemma4') {
1194
- throw new Error(
1195
- `[Pipeline] transcribeImage: unsupported vision architecture "${this.visionConfig?.visionArchitecture ?? 'unknown'}". ` +
1196
- 'This runtime path currently requires Gemma 4 multimodal prompt expansion.'
1197
- );
1198
- }
1199
- const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
1200
- const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
1201
- const multimodalPrompt = formatChatMessages([
1202
- {
1203
- role: 'user',
1204
- content: [
1205
- { type: 'image' },
1206
- { type: 'text', text: requestedPrompt },
1207
- ],
1208
- },
1209
- ], templateType, chatOptions);
1210
- const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
1211
- const imageTokenSpanLength = encodeResult.numTokens;
1212
- const effectiveBudget = softTokenBudget ?? this.visionConfig?.defaultOutputLength;
1213
- const maxImageTokenSpanLength = Number(effectiveBudget);
1214
- if (!Number.isFinite(maxImageTokenSpanLength) || maxImageTokenSpanLength < 1 || Math.floor(maxImageTokenSpanLength) !== maxImageTokenSpanLength) {
1215
- throw new Error(
1216
- `[Pipeline] transcribeImage: invalid soft token budget ${effectiveBudget}. ` +
1217
- 'Expected a positive integer from the resolved vision config or softTokenBudget parameter.'
1218
- );
1219
- }
1220
- if (imageTokenSpanLength > maxImageTokenSpanLength) {
1221
- throw new Error(
1222
- `[Pipeline] transcribeImage: encoded Gemma 4 image produced ${imageTokenSpanLength} soft tokens, ` +
1223
- `which exceeds the effective soft token budget=${maxImageTokenSpanLength}.`
1224
- );
1225
- }
1226
- const boiTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|image>', 'Gemma 4 BOI token');
1227
- const eoiTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<image|>', 'Gemma 4 EOI token');
1228
- const { inputIds: fullTokenIds, imageStartOffset } = expandImagePlaceholderTokenIds(
1229
- promptTokenIds,
1230
- imageTokenId,
1231
- imageTokenSpanLength,
1232
- { boiTokenId, eoiTokenId }
1233
- );
1234
- const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
1235
- if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
1236
- throw new Error(
1237
- `[Pipeline] transcribeImage: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
1238
- );
1239
- }
1240
-
1241
- // Step 3: Generate with embedding override at the image token offset.
1242
- const tokens = [];
1243
- const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
1244
- const stopTokenIds = this.modelConfig.stopTokenIds;
1245
-
1246
- try {
1247
- const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
1248
- inputIds: fullTokenIds,
1249
- embeddingOverrides: {
1250
- prefixLength: encodeResult.numTokens,
1251
- offset: imageStartOffset,
1252
- embeddings: encodeResult.features,
1253
- },
1254
- __internalEmbeddingInputSpan: {
1255
- offset: imageStartOffset,
1256
- length: encodeResult.numTokens,
1257
- tokenId: padTokenId,
1258
- },
1259
- __internalMultimodalBidirectionalSpan: {
1260
- offset: imageStartOffset,
1261
- length: encodeResult.numTokens,
1262
- },
1263
- maxTokens: maxGen,
1264
- temperature: 0,
1265
- topK: 1,
1266
- topP: 1,
1267
- repetitionPenalty: 1,
1268
- }));
1269
- for (const token of generation.tokenIds ?? []) {
1270
- if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
1271
- tokens.push(token);
1272
- }
1273
- } finally {
1274
- if (encodeResult.features) {
1275
- releaseBuffer(encodeResult.features);
1276
- }
1277
- }
1278
-
1279
- const text = this.tokenizer.decode(tokens);
1280
- return { text, tokens };
155
+ return transcribeImageImpl.apply(this, arguments);
1281
156
  }
1282
157
 
1283
-
1284
158
  // ==========================================================================
1285
159
  // Video: transcribeVideo
1286
160
  // ==========================================================================
1287
161
 
1288
- /**
1289
- * Transcribe text from video using the vision encoder (per-frame) and text decoder.
1290
- *
1291
- * Video is processed as sampled frames through the existing vision encoder.
1292
- * Each frame is encoded independently and the visual tokens are concatenated.
1293
- *
1294
- * @param {object} params
1295
- * @param {Array<{ pixels: Uint8Array|Float32Array, width: number, height: number }>} params.frames Decoded video frames
1296
- * @param {string} [params.prompt] Custom transcription prompt
1297
- * @param {number} [params.maxTokens] Max tokens to generate
1298
- * @param {number} [params.maxFrames=8] Maximum frames to sample
1299
- * @param {number} [params.perFrameSoftTokenBudget] Soft token budget per frame
1300
- * @returns {Promise<{ text: string, tokens: number[] }>}
1301
- */
1302
162
  async transcribeVideo({ frames, prompt, maxTokens, maxFrames, perFrameSoftTokenBudget, signal }) {
1303
- assertNotAborted(signal);
1304
- if (!this.visionCapable) {
1305
- throw new Error(
1306
- 'Pipeline does not support video transcription (no image_token_id in manifest for vision encoder).'
1307
- );
1308
- }
1309
- await this._ensureVisionWeightsLoaded();
1310
-
1311
- this.reset();
1312
-
1313
- // Lazy-load video module
1314
- const { encodeVideo } = await import('./video/index.js');
1315
-
1316
- // Step 1: Encode video frames through vision pipeline
1317
- const encodeResult = await encodeVideo({
1318
- frames,
1319
- visionConfig: this.visionConfig,
1320
- weights: this.visionWeights,
1321
- maxFrames: maxFrames ?? 8,
1322
- perFrameSoftTokenBudget,
1323
- });
1324
-
1325
- // Step 2: Build the multimodal prompt with <|video|> placeholder
1326
- const requestedPrompt = prompt ?? 'Describe the video in one short sentence.';
1327
- const videoTokenId = this.tokenizer?.model?.tokenToId?.('<|video_token|>')
1328
- ?? this.tokenizer?.encode?.('<|video|>')?.find?.((id) => id !== undefined)
1329
- ?? null;
1330
- // Fall back to image token ID for video placeholder expansion
1331
- const placeholderTokenId = videoTokenId ?? this.visionConfig?.imageTokenId ?? this.imageTokenId;
1332
- if (placeholderTokenId == null) {
1333
- throw new Error(
1334
- 'Pipeline missing video/image token ID for video placeholder expansion.'
1335
- );
1336
- }
1337
-
1338
- const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
1339
- const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
1340
- const multimodalPrompt = formatChatMessages([
1341
- {
1342
- role: 'user',
1343
- content: [
1344
- { type: 'video' },
1345
- { type: 'text', text: requestedPrompt },
1346
- ],
1347
- },
1348
- ], templateType, chatOptions);
1349
- const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
1350
- const videoTokenSpanLength = encodeResult.numTokens;
1351
-
1352
- // Resolve BOV/EOV tokens (reuse image BOI/EOI if video-specific ones don't exist)
1353
- const bovTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|video|>', 'Gemma 4 BOV token');
1354
- const eovTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<video|>', 'Gemma 4 EOV token');
1355
-
1356
- const { inputIds: fullTokenIds, imageStartOffset: videoStartOffset } = expandImagePlaceholderTokenIds(
1357
- promptTokenIds,
1358
- placeholderTokenId,
1359
- videoTokenSpanLength,
1360
- { boiTokenId: bovTokenId, eoiTokenId: eovTokenId }
1361
- );
1362
-
1363
- const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
1364
- if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
1365
- throw new Error(
1366
- `[Pipeline] transcribeVideo: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
1367
- );
1368
- }
1369
-
1370
- // Step 3: Generate with embedding override at the video token offset
1371
- const tokens = [];
1372
- const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
1373
- const stopTokenIds = this.modelConfig.stopTokenIds;
1374
-
1375
- try {
1376
- const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
1377
- inputIds: fullTokenIds,
1378
- embeddingOverrides: {
1379
- prefixLength: encodeResult.numTokens,
1380
- offset: videoStartOffset,
1381
- embeddings: encodeResult.features,
1382
- },
1383
- __internalEmbeddingInputSpan: {
1384
- offset: videoStartOffset,
1385
- length: encodeResult.numTokens,
1386
- tokenId: padTokenId,
1387
- },
1388
- __internalMultimodalBidirectionalSpan: {
1389
- offset: videoStartOffset,
1390
- length: encodeResult.numTokens,
1391
- },
1392
- maxTokens: maxGen,
1393
- temperature: 0,
1394
- topK: 1,
1395
- topP: 1,
1396
- repetitionPenalty: 1,
1397
- }));
1398
- for (const token of generation.tokenIds ?? []) {
1399
- if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
1400
- tokens.push(token);
1401
- }
1402
- } finally {
1403
- if (encodeResult.features) {
1404
- releaseBuffer(encodeResult.features);
1405
- }
1406
- }
1407
-
1408
- const text = this.tokenizer.decode(tokens);
1409
- return { text, tokens };
163
+ return transcribeVideoImpl.apply(this, arguments);
1410
164
  }
1411
165
 
1412
-
1413
166
  // ==========================================================================
1414
167
  // Audio: transcribeAudio
1415
168
  // ==========================================================================
1416
169
 
1417
- /**
1418
- * Transcribe text from audio using the audio encoder and text decoder.
1419
- *
1420
- * @param {object} params
1421
- * @param {Float32Array} params.audio Raw audio PCM (mono, 16kHz)
1422
- * @param {string} [params.prompt] Custom transcription prompt
1423
- * @param {number} [params.maxTokens] Max tokens to generate
1424
- * @returns {Promise<{ text: string, tokens: number[] }>}
1425
- */
1426
170
  async transcribeAudio({ audio, prompt, maxTokens, signal }) {
1427
- assertNotAborted(signal);
1428
- if (!this.audioCapable) {
1429
- throw new Error(
1430
- 'Pipeline does not support audio transcription (no audio_token_id in manifest).'
1431
- );
1432
- }
1433
- await this._ensureAudioWeightsLoaded();
1434
-
1435
- this.reset();
1436
-
1437
- // Lazy-load audio modules
1438
- const { encodeAudio } = await import('./audio/index.js');
1439
-
1440
- let encodeResult;
1441
- if (this.audioConfig.depth === 0) {
1442
- encodeResult = await encodeAudio({
1443
- rawAudio: audio,
1444
- audioConfig: this.audioConfig,
1445
- weights: this.audioWeights,
1446
- });
1447
- } else {
1448
- const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1449
- const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1450
- encodeResult = await encodeAudio({
1451
- melFeatures,
1452
- numFrames,
1453
- nMels,
1454
- audioConfig: this.audioConfig,
1455
- weights: this.audioWeights,
1456
- });
1457
- }
1458
-
1459
- // Step 3: Build the multimodal prompt with <|audio|> placeholder
1460
- const requestedPrompt = prompt ?? 'Transcribe the audio.';
1461
- const audioTokenId = this.audioConfig?.audioTokenId ?? this.audioTokenId;
1462
- if (audioTokenId == null) {
1463
- throw new Error(
1464
- 'Pipeline missing audio_token_id. Re-convert the model with audio token metadata.'
1465
- );
1466
- }
1467
- const templateType = this.modelConfig?.chatTemplateType ?? 'gemma4';
1468
- const chatOptions = this.modelConfig?.chatTemplateThinking === true ? { thinking: true } : undefined;
1469
- const multimodalPrompt = formatChatMessages([
1470
- {
1471
- role: 'user',
1472
- content: [
1473
- { type: 'audio' },
1474
- { type: 'text', text: requestedPrompt },
1475
- ],
1476
- },
1477
- ], templateType, chatOptions);
1478
- const promptTokenIds = this.tokenizer.encode(multimodalPrompt);
1479
- const audioTokenSpanLength = encodeResult.numTokens;
1480
-
1481
- // Resolve BOA/EOA tokens
1482
- const boaTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<|audio|>', 'Gemma 4 BOA token');
1483
- const eoaTokenId = resolveSingleSpecialTokenId(this.tokenizer, '<audio|>', 'Gemma 4 EOA token');
1484
-
1485
- // Expand single audio placeholder token into the full audio token span
1486
- const { inputIds: fullTokenIds, imageStartOffset: audioStartOffset } = expandImagePlaceholderTokenIds(
1487
- promptTokenIds,
1488
- audioTokenId,
1489
- audioTokenSpanLength,
1490
- { boiTokenId: boaTokenId, eoiTokenId: eoaTokenId }
1491
- );
1492
-
1493
- const padTokenId = this.tokenizer?.getSpecialTokens?.()?.pad;
1494
- if (!Number.isFinite(padTokenId) || Math.floor(padTokenId) !== padTokenId || padTokenId < 0) {
1495
- throw new Error(
1496
- `[Pipeline] transcribeAudio: Gemma 4 multimodal prefill requires a tokenizer pad token ID, got ${padTokenId}.`
1497
- );
1498
- }
1499
-
1500
- // Step 4: Generate with embedding override at the audio token offset
1501
- const tokens = [];
1502
- const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
1503
- const stopTokenIds = this.modelConfig.stopTokenIds;
1504
-
1505
- try {
1506
- const generation = await this.generator.generateTokenIds('', buildConservativeMultimodalGenerationOptions({
1507
- inputIds: fullTokenIds,
1508
- embeddingOverrides: {
1509
- prefixLength: encodeResult.numTokens,
1510
- offset: audioStartOffset,
1511
- embeddings: encodeResult.features,
1512
- },
1513
- __internalEmbeddingInputSpan: {
1514
- offset: audioStartOffset,
1515
- length: encodeResult.numTokens,
1516
- tokenId: padTokenId,
1517
- },
1518
- __internalMultimodalBidirectionalSpan: {
1519
- offset: audioStartOffset,
1520
- length: encodeResult.numTokens,
1521
- },
1522
- maxTokens: maxGen,
1523
- temperature: 0,
1524
- topK: 1,
1525
- topP: 1,
1526
- repetitionPenalty: 1,
1527
- }));
1528
- for (const token of generation.tokenIds ?? []) {
1529
- if (Array.isArray(stopTokenIds) && stopTokenIds.includes(token)) break;
1530
- tokens.push(token);
1531
- }
1532
- } finally {
1533
- if (encodeResult.features) {
1534
- releaseBuffer(encodeResult.features);
1535
- }
1536
- }
1537
-
1538
- const text = this.tokenizer.decode(tokens);
1539
- return { text, tokens };
171
+ return transcribeAudioImpl.apply(this, arguments);
1540
172
  }
1541
173
 
1542
-
1543
174
  // ==========================================================================
1544
175
  // Capability Detection
1545
176
  // ==========================================================================
@@ -1554,7 +185,6 @@ export class InferencePipeline extends PipelineState {
1554
185
  return Object.freeze(caps);
1555
186
  }
1556
187
 
1557
-
1558
188
  // Layer pipeline precedence (lowest to highest):
1559
189
  // 1. execution-v1-produced pipeline (via runtimeInferencePatch.pipeline)
1560
190
  // 2. model config pipeline (manifest inference.pipeline)
@@ -1598,7 +228,6 @@ export class InferencePipeline extends PipelineState {
1598
228
  // Generation Delegates
1599
229
  // ==========================================================================
1600
230
 
1601
-
1602
231
  generate(prompt, options = {}) {
1603
232
  assertNotAborted(options?.signal);
1604
233
  return this.generator.generate(prompt, options);
@@ -1633,7 +262,6 @@ export class InferencePipeline extends PipelineState {
1633
262
  return this.generator.advanceWithTokenAndEmbedding(tokenId, options);
1634
263
  }
1635
264
 
1636
-
1637
265
  prefillKVOnly(prompt, options = {}) {
1638
266
  assertNotAborted(options?.signal);
1639
267
  return this.generator.prefillKVOnly(prompt, options);
@@ -1659,95 +287,15 @@ export class InferencePipeline extends PipelineState {
1659
287
  }
1660
288
 
1661
289
  async embed(prompt, options = {}) {
1662
- assertNotAborted(options?.signal);
1663
- this.resetForBatch();
1664
- try {
1665
- const result = await this.prefillWithEmbedding(prompt, {
1666
- ...options,
1667
- __skipStateSnapshot: true,
1668
- });
1669
- assertNotAborted(options?.signal);
1670
- return {
1671
- embedding: result.embedding,
1672
- tokens: result.tokens,
1673
- seqLen: result.seqLen,
1674
- embeddingMode: result.embeddingMode,
1675
- phase: result.phase ?? null,
1676
- };
1677
- } finally {
1678
- this.resetForBatch();
1679
- }
290
+ return embedImpl.apply(this, arguments);
1680
291
  }
1681
292
 
1682
293
  async embedBatch(prompts, options = {}) {
1683
- if (!Array.isArray(prompts)) {
1684
- throw new Error('embedBatch expects an array of prompts');
1685
- }
1686
- assertNotAborted(options?.signal);
1687
- const batchOptions = { ...options, __skipStateSnapshot: true };
1688
- const outputs = [];
1689
- for (const prompt of prompts) {
1690
- // Check between every prompt so a superseded revision drops the rest.
1691
- assertNotAborted(options?.signal);
1692
- outputs.push(await this.embed(prompt, batchOptions));
1693
- }
1694
- return outputs;
294
+ return embedBatchImpl.apply(this, arguments);
1695
295
  }
1696
296
 
1697
297
  async encodeSequence(sequence, options = {}) {
1698
- assertNotAborted(options?.signal);
1699
- const contract = this.manifest?.inference?.sequence ?? null;
1700
- if (this.manifest?.inference?.supportsSequence !== true || !contract) {
1701
- throw new Error('Model manifest does not declare sequence encoding support.');
1702
- }
1703
- if (typeof sequence !== 'string' || sequence.length === 0) {
1704
- throw new Error('encodeSequence expects a non-empty sequence string.');
1705
- }
1706
- const includeTokenEmbeddings = options.includeTokenEmbeddings
1707
- ?? contract.tokenEmbeddings;
1708
- const includeLogits = options.includeLogits === true;
1709
- if (includeTokenEmbeddings && contract.tokenEmbeddings !== true) {
1710
- throw new Error('Model manifest does not permit token-level sequence embeddings.');
1711
- }
1712
- if (includeLogits && contract.logits !== true) {
1713
- throw new Error('Model manifest does not permit sequence logits.');
1714
- }
1715
- const needsTokenEmbeddings = includeTokenEmbeddings || contract.pooledEmbedding !== null;
1716
-
1717
- this.resetForBatch();
1718
- try {
1719
- const result = await this.prefillWithEmbedding(sequence, {
1720
- ...options,
1721
- embeddingMode: contract.pooledEmbedding?.mode ?? 'last',
1722
- __skipStateSnapshot: true,
1723
- __returnTokenEmbeddings: needsTokenEmbeddings,
1724
- __returnSequenceLogits: includeLogits,
1725
- });
1726
- assertNotAborted(options?.signal);
1727
- const hiddenSize = this.modelConfig.hiddenSize;
1728
- const pooledResult = contract.pooledEmbedding
1729
- ? poolSequenceTokenEmbeddings(
1730
- result.tokenEmbeddings,
1731
- result.tokens,
1732
- hiddenSize,
1733
- contract.pooledEmbedding
1734
- )
1735
- : null;
1736
- return {
1737
- alphabet: contract.alphabet,
1738
- tokens: result.tokens,
1739
- tokenMask: pooledResult?.tokenMask ?? new Uint8Array(result.tokens.length),
1740
- includedTokenCount: pooledResult?.includedTokenCount ?? 0,
1741
- tokenEmbeddings: includeTokenEmbeddings ? result.tokenEmbeddings : null,
1742
- pooledEmbedding: pooledResult?.pooled ?? null,
1743
- logits: includeLogits ? result.logits : null,
1744
- embeddingDim: hiddenSize,
1745
- vocabSize: this.modelConfig.vocabSize,
1746
- phase: result.phase ?? null,
1747
- };
1748
- } finally {
1749
- this.resetForBatch();
1750
- }
298
+ return encodeSequenceImpl.apply(this, arguments);
1751
299
  }
1752
300
 
1753
301
  // Run the vision encoder over a single image and return a mean-pooled
@@ -1755,61 +303,7 @@ export class InferencePipeline extends PipelineState {
1755
303
  // (jpeg/png -> RGBA pixels) belongs to the caller; this method takes
1756
304
  // already-decoded pixel data.
1757
305
  async embedImage({ pixels, width, height, softTokenBudget, signal } = {}) {
1758
- assertNotAborted(signal);
1759
- if (!this.visionCapable) {
1760
- throw new Error(
1761
- 'Pipeline does not support image embedding (no image_token_id in manifest).'
1762
- );
1763
- }
1764
- if (pixels == null) {
1765
- throw new Error('[Pipeline] embedImage: pixels are required.');
1766
- }
1767
- if (!Number.isFinite(width) || width <= 0 || !Number.isFinite(height) || height <= 0) {
1768
- throw new Error('[Pipeline] embedImage: width and height must be positive integers.');
1769
- }
1770
- await this._ensureVisionWeightsLoaded();
1771
- this.reset();
1772
-
1773
- const { encodeImage } = await import('./vision/index.js');
1774
- const encodeResult = await encodeImage({
1775
- pixels,
1776
- width,
1777
- height,
1778
- visionConfig: this.visionConfig,
1779
- weights: this.visionWeights,
1780
- softTokenBudget,
1781
- });
1782
-
1783
- const hiddenSize = this.modelConfig.hiddenSize;
1784
- const numTokens = encodeResult.numTokens;
1785
- if (!Number.isFinite(numTokens) || numTokens < 1) {
1786
- releaseBuffer(encodeResult.features);
1787
- throw new Error(`[Pipeline] embedImage: encoder produced ${numTokens} soft tokens; expected >= 1.`);
1788
- }
1789
- try {
1790
- const bytes = await readBuffer(
1791
- encodeResult.features,
1792
- numTokens * hiddenSize * Float32Array.BYTES_PER_ELEMENT
1793
- );
1794
- const features = new Float32Array(bytes);
1795
- const pooled = new Float32Array(hiddenSize);
1796
- for (let t = 0; t < numTokens; t++) {
1797
- const base = t * hiddenSize;
1798
- for (let d = 0; d < hiddenSize; d++) {
1799
- pooled[d] += features[base + d];
1800
- }
1801
- }
1802
- const inv = 1 / numTokens;
1803
- for (let d = 0; d < hiddenSize; d++) pooled[d] *= inv;
1804
- return {
1805
- embedding: pooled,
1806
- embeddingDim: hiddenSize,
1807
- numTokens,
1808
- embeddingMode: 'mean',
1809
- };
1810
- } finally {
1811
- releaseBuffer(encodeResult.features);
1812
- }
306
+ return embedImageImpl.apply(this, arguments);
1813
307
  }
1814
308
 
1815
309
  // Run the audio encoder over a single PCM segment and return a mean-pooled
@@ -1817,73 +311,7 @@ export class InferencePipeline extends PipelineState {
1817
311
  // text hidden size in Gemma 4). Decoder responsibility (webm/opus/wav ->
1818
312
  // Float32 PCM at the model's expected sample rate) belongs to the caller.
1819
313
  async embedAudio({ audio, signal } = {}) {
1820
- assertNotAborted(signal);
1821
- if (!this.audioCapable) {
1822
- throw new Error(
1823
- 'Pipeline does not support audio embedding (no audio_token_id in manifest).'
1824
- );
1825
- }
1826
- if (audio == null) {
1827
- throw new Error('[Pipeline] embedAudio: audio is required.');
1828
- }
1829
- await this._ensureAudioWeightsLoaded();
1830
- this.reset();
1831
-
1832
- const { encodeAudio } = await import('./audio/index.js');
1833
-
1834
- let encodeResult;
1835
- if (this.audioConfig.depth === 0) {
1836
- encodeResult = await encodeAudio({
1837
- rawAudio: audio,
1838
- audioConfig: this.audioConfig,
1839
- weights: this.audioWeights,
1840
- });
1841
- } else {
1842
- const { extractLogMelSpectrogram } = await import('./audio/mel.js');
1843
- const { features: melFeatures, numFrames, nMels } = extractLogMelSpectrogram(audio);
1844
- encodeResult = await encodeAudio({
1845
- melFeatures,
1846
- numFrames,
1847
- nMels,
1848
- audioConfig: this.audioConfig,
1849
- weights: this.audioWeights,
1850
- });
1851
- }
1852
-
1853
- const hiddenSize = Number(this.audioConfig?.outputProjDims ?? this.modelConfig?.hiddenSize);
1854
- const numTokens = encodeResult.numTokens;
1855
- if (!Number.isFinite(hiddenSize) || hiddenSize < 1) {
1856
- releaseBuffer(encodeResult.features);
1857
- throw new Error('[Pipeline] embedAudio: audioConfig.outputProjDims is missing or invalid.');
1858
- }
1859
- if (!Number.isFinite(numTokens) || numTokens < 1) {
1860
- releaseBuffer(encodeResult.features);
1861
- throw new Error(`[Pipeline] embedAudio: encoder produced ${numTokens} tokens; expected >= 1.`);
1862
- }
1863
- try {
1864
- const bytes = await readBuffer(
1865
- encodeResult.features,
1866
- numTokens * hiddenSize * Float32Array.BYTES_PER_ELEMENT
1867
- );
1868
- const features = new Float32Array(bytes);
1869
- const pooled = new Float32Array(hiddenSize);
1870
- for (let t = 0; t < numTokens; t++) {
1871
- const base = t * hiddenSize;
1872
- for (let d = 0; d < hiddenSize; d++) {
1873
- pooled[d] += features[base + d];
1874
- }
1875
- }
1876
- const inv = 1 / numTokens;
1877
- for (let d = 0; d < hiddenSize; d++) pooled[d] *= inv;
1878
- return {
1879
- embedding: pooled,
1880
- embeddingDim: hiddenSize,
1881
- numTokens,
1882
- embeddingMode: 'mean',
1883
- };
1884
- } finally {
1885
- releaseBuffer(encodeResult.features);
1886
- }
314
+ return embedAudioImpl.apply(this, arguments);
1887
315
  }
1888
316
 
1889
317
  prefillWithLogits(prompt, options = {}) {
@@ -1898,7 +326,6 @@ export class InferencePipeline extends PipelineState {
1898
326
  return this.generator.prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options);
1899
327
  }
1900
328
 
1901
-
1902
329
  applyKVCacheSnapshot(snapshot) {
1903
330
  this.kvCache = snapshot.cache.clone();
1904
331
  if (this.useGPU && this.kvCache) {
@@ -1923,7 +350,6 @@ export class InferencePipeline extends PipelineState {
1923
350
  this.currentSeqLen = snapshot.seqLen;
1924
351
  }
1925
352
 
1926
-
1927
353
  generateWithPrefixKV(prefix, prompt, options = {}) {
1928
354
  return this.generator.generateWithPrefixKV(prefix, prompt, options);
1929
355
  }
@@ -1932,7 +358,6 @@ export class InferencePipeline extends PipelineState {
1932
358
  // Utility Methods
1933
359
  // ==========================================================================
1934
360
 
1935
-
1936
361
  getStats() {
1937
362
  const stats = { ...this.stats };
1938
363
  stats.batching ??= { ...this.batchingStats };
@@ -1994,12 +419,10 @@ export class InferencePipeline extends PipelineState {
1994
419
  return stats;
1995
420
  }
1996
421
 
1997
-
1998
422
  getBatchingStats() {
1999
423
  return { ...this.batchingStats };
2000
424
  }
2001
425
 
2002
-
2003
426
  getMemoryStats() {
2004
427
 
2005
428
  const stats = { used: 0 };
@@ -2025,14 +448,12 @@ export class InferencePipeline extends PipelineState {
2025
448
  return stats;
2026
449
  }
2027
450
 
2028
-
2029
451
  getKVCacheStats() {
2030
452
  if (!this.kvCache) return null;
2031
453
  const { seqLen, maxSeqLen } = this.kvCache.getMemoryStats();
2032
454
  return { seqLen, maxSeqLen };
2033
455
  }
2034
456
 
2035
-
2036
457
  getBufferPool() {
2037
458
  try {
2038
459
  return getGlobalBufferPool();
@@ -2041,7 +462,6 @@ export class InferencePipeline extends PipelineState {
2041
462
  }
2042
463
  }
2043
464
 
2044
-
2045
465
  async unload() {
2046
466
  const storageContext = this.storageContext;
2047
467
  this.storageContext = null;
@@ -2062,6 +482,7 @@ export class InferencePipeline extends PipelineState {
2062
482
  this.ownsDopplerLoader = false;
2063
483
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
2064
484
  this.lora = null;
485
+ this.revocationIdentity = null;
2065
486
  destroyMoERouter(this.moeRouter);
2066
487
  this.moeRouter = null;
2067
488
  if (this.finitenessBuffer) {
@@ -2081,17 +502,15 @@ export class InferencePipeline extends PipelineState {
2081
502
  log.info('Pipeline', 'Unloaded');
2082
503
  }
2083
504
 
2084
-
2085
505
  setLoRAAdapter(adapter) {
506
+ assertBundledAdapterAuthorized(adapter);
2086
507
  this.lora = adapter;
2087
508
  }
2088
509
 
2089
-
2090
510
  getActiveLoRA() {
2091
511
  return this.lora;
2092
512
  }
2093
513
 
2094
-
2095
514
  reset() {
2096
515
  this.kvCache?.clear();
2097
516
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
@@ -2115,10 +534,6 @@ export class InferencePipeline extends PipelineState {
2115
534
  this.stats.attentionInputs = [];
2116
535
  }
2117
536
 
2118
- /**
2119
- * Lightweight reset for batch embedding: clears KV cache and sequence state
2120
- * without resetting stats or debug flags (they are overwritten on each prefill).
2121
- */
2122
537
  resetForBatch() {
2123
538
  this.kvCache?.clear();
2124
539
  this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
@@ -2128,7 +543,6 @@ export class InferencePipeline extends PipelineState {
2128
543
  this.decodeRing?.reset();
2129
544
  }
2130
545
 
2131
-
2132
546
  releaseGPUResources() {
2133
547
  this.decodeBuffers?.release();
2134
548
  this.decodeRing?.release();
@@ -2153,7 +567,6 @@ export class InferencePipeline extends PipelineState {
2153
567
  // Factory Function
2154
568
  // ============================================================================
2155
569
 
2156
-
2157
570
  async function createTransformerPipeline(manifest, contexts = {}) {
2158
571
  return createInitializedPipeline(InferencePipeline, manifest, contexts);
2159
572
  }