doppler-gpu 0.5.1 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (953) hide show
  1. package/CHANGELOG.md +33 -577
  2. package/LICENSE +21 -201
  3. package/README.md +165 -222
  4. package/models/catalog.json +241 -33
  5. package/package.json +106 -87
  6. package/src/capsule-runtime.d.ts +25 -0
  7. package/src/capsule-runtime.js +24 -0
  8. package/src/capsule.d.ts +8 -0
  9. package/src/capsule.js +4 -0
  10. package/src/cli/argument-contract.js +200 -0
  11. package/src/cli/browser-report-output.js +24 -0
  12. package/src/cli/cli-model-resolution.js +2 -2
  13. package/src/cli/cli-output.js +10 -0
  14. package/src/cli/commands/bundle.js +828 -0
  15. package/src/cli/doppler-cli.js +50 -1457
  16. package/src/cli/doppler-serve.js +18 -244
  17. package/src/cli/output/manifest.js +368 -0
  18. package/src/cli/runtime-profiles.js +1 -1
  19. package/src/cli/serve/capsule-handler.d.ts +33 -0
  20. package/src/cli/serve/capsule-handler.js +105 -0
  21. package/src/cli/serve/capsule-http.js +102 -0
  22. package/src/cli/serve/receipt.js +269 -0
  23. package/src/client/capsule-host.browser.js +2 -0
  24. package/src/client/capsule-host.d.ts +5 -0
  25. package/src/client/capsule-host.js +2 -0
  26. package/src/client/doppler-api.browser.js +42 -0
  27. package/src/client/doppler-api.d.ts +38 -1
  28. package/src/client/doppler-api.js +41 -1
  29. package/src/client/doppler-registry.js +15 -7
  30. package/src/client/electron/index.d.ts +4 -0
  31. package/src/client/electron/index.js +13 -0
  32. package/src/client/electron/ipc-contract.d.ts +9 -0
  33. package/src/client/electron/ipc-contract.js +56 -0
  34. package/src/client/electron/release-state.d.ts +53 -0
  35. package/src/client/electron/release-state.js +425 -0
  36. package/src/client/electron/renderer-runtime.d.ts +19 -0
  37. package/src/client/electron/renderer-runtime.js +104 -0
  38. package/src/client/failure-taxonomy.js +1 -28
  39. package/src/client/fault-injection.js +0 -8
  40. package/src/client/inspection.js +1 -1
  41. package/src/client/model-host/index.d.ts +140 -0
  42. package/src/client/model-host/index.js +548 -0
  43. package/src/client/model-host/model-evidence.js +255 -0
  44. package/src/client/model-host/model-session.d.ts +323 -0
  45. package/src/client/{runtime → model-host}/model-session.js +202 -148
  46. package/src/client/provider.d.ts +18 -2
  47. package/src/client/provider.js +26 -3
  48. package/src/client/receipt.js +30 -72
  49. package/src/client/runtime/capsule-acquisition.d.ts +18 -0
  50. package/src/client/runtime/capsule-acquisition.js +103 -0
  51. package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
  52. package/src/client/runtime/capsule-adapter-execution.js +63 -0
  53. package/src/client/runtime/capsule-artifact-source.js +62 -0
  54. package/src/client/runtime/capsule-embedding.d.ts +29 -0
  55. package/src/client/runtime/capsule-embedding.js +76 -0
  56. package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
  57. package/src/client/runtime/capsule-forecast-program.js +85 -0
  58. package/src/client/runtime/capsule-forecast.d.ts +17 -0
  59. package/src/client/runtime/capsule-forecast.js +31 -0
  60. package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
  61. package/src/client/runtime/capsule-operation-adapters.js +69 -0
  62. package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
  63. package/src/client/runtime/capsule-operation-executor.js +101 -0
  64. package/src/client/runtime/capsule-program-adapter.js +153 -0
  65. package/src/client/runtime/capsule-release-authorization.js +25 -0
  66. package/src/client/runtime/capsule-rerank.d.ts +50 -0
  67. package/src/client/runtime/capsule-rerank.js +105 -0
  68. package/src/client/runtime/capsule-session-execution.js +89 -0
  69. package/src/client/runtime/command-executor.d.ts +15 -0
  70. package/src/client/runtime/command-executor.js +128 -0
  71. package/src/client/runtime/composition-root.d.ts +77 -0
  72. package/src/client/runtime/composition-root.js +352 -0
  73. package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
  74. package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
  75. package/src/client/runtime/index.d.ts +1 -117
  76. package/src/client/runtime/index.js +1 -433
  77. package/src/client/runtime/lora.d.ts +14 -0
  78. package/src/client/runtime/lora.js +19 -1
  79. package/src/client/runtime/model-manager.js +12 -0
  80. package/src/client/runtime/model-rerank.js +30 -0
  81. package/src/client/runtime/model-session.d.ts +1 -221
  82. package/src/client/runtime/model-source.d.ts +9 -0
  83. package/src/client/runtime/model-source.js +46 -17
  84. package/src/client/runtime/node-quickstart-cache.js +1 -1
  85. package/src/client/runtime/resolution-policy.d.ts +20 -0
  86. package/src/client/runtime/resolution-policy.js +84 -0
  87. package/src/client/runtime/resource-binder.d.ts +24 -0
  88. package/src/client/runtime/resource-binder.js +135 -0
  89. package/src/client/runtime/scoped-session.d.ts +10 -1
  90. package/src/client/runtime/scoped-session.js +20 -1
  91. package/src/client/runtime/session-controller.d.ts +19 -0
  92. package/src/client/runtime/session-controller.js +92 -0
  93. package/src/client/runtime/source-runtime.js +1 -1
  94. package/src/client/runtime/target-selector.d.ts +9 -0
  95. package/src/client/runtime/target-selector.js +2 -0
  96. package/src/client/runtime/types.d.ts +1 -0
  97. package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
  98. package/src/client/wrap-pipeline-handle.js +3 -9
  99. package/src/config/backward-registry-loader.js +1 -1
  100. package/src/config/capsule-adapter-policy.js +30 -0
  101. package/src/config/capsule-adapters.d.ts +21 -0
  102. package/src/config/capsule-adapters.js +40 -0
  103. package/src/config/capsule-adapters.json +9 -0
  104. package/src/config/capsule-loading.d.ts +2 -0
  105. package/src/config/capsule-loading.js +14 -0
  106. package/src/config/capsule-loading.schema.json +12 -0
  107. package/src/config/capsule-operation.d.ts +19 -0
  108. package/src/config/capsule-operation.js +63 -0
  109. package/src/config/capsule-operations.json +17 -0
  110. package/src/config/capsule-release-contract.d.ts +66 -0
  111. package/src/config/capsule-release-contract.js +197 -0
  112. package/src/config/capsule-release-events.d.ts +53 -0
  113. package/src/config/capsule-release-events.js +172 -0
  114. package/src/config/capsule-sequence-receipt.js +19 -0
  115. package/src/config/capsule-serve.d.ts +9 -0
  116. package/src/config/capsule-serve.js +28 -0
  117. package/src/config/capsule-serve.schema.json +14 -0
  118. package/src/config/capsule-signature.d.ts +6 -0
  119. package/src/config/capsule-signature.js +50 -0
  120. package/src/config/capsule-v2.d.ts +70 -0
  121. package/src/config/capsule-v2.js +431 -0
  122. package/src/config/capsule-v3.d.ts +19 -0
  123. package/src/config/capsule-v3.js +72 -0
  124. package/src/config/capsule.d.ts +22 -0
  125. package/src/config/capsule.js +52 -0
  126. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
  127. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  128. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
  129. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
  130. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
  131. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  132. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
  133. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
  134. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
  135. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
  136. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
  137. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
  138. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
  139. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
  140. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
  141. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
  142. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  143. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  144. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
  145. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
  146. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
  147. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  148. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
  149. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  150. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  151. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  152. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
  153. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
  154. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
  155. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
  156. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
  157. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
  158. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
  159. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
  160. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
  161. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
  162. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
  163. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
  164. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
  165. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
  166. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
  167. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
  168. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
  169. package/src/config/dtype-transition-contract.js +40 -0
  170. package/src/config/embedding-contract.js +26 -0
  171. package/src/config/embedding-reference.js +85 -0
  172. package/src/config/evidence/source-intake-policy.json +1 -1
  173. package/src/config/execution-contract-check.d.ts +16 -73
  174. package/src/config/execution-contract-check.js +7 -429
  175. package/src/config/forecast-manifest.js +56 -0
  176. package/src/config/generation-contract.d.ts +39 -0
  177. package/src/config/generation-contract.js +66 -0
  178. package/src/config/generation-contract.json +32 -0
  179. package/src/config/heterogeneous-attention-contract.js +46 -0
  180. package/src/config/initial-execution-identity.d.ts +58 -0
  181. package/src/config/initial-execution-identity.js +331 -0
  182. package/src/config/kernel-path-loader.d.ts +17 -0
  183. package/src/config/kernel-path-loader.js +42 -8
  184. package/src/config/kernel-registry-contract.d.ts +25 -0
  185. package/src/config/kernel-registry-contract.js +46 -0
  186. package/src/config/kernels/kernel-ref-digests.js +66 -18
  187. package/src/config/kernels/registry.js +1 -1
  188. package/src/config/kernels/registry.json +13617 -11377
  189. package/src/config/lora-layouts.d.ts +10 -0
  190. package/src/config/lora-layouts.js +17 -0
  191. package/src/config/lora-layouts.json +9 -0
  192. package/src/config/manifest-inference-validation.js +575 -0
  193. package/src/config/merge/execution-patches.js +12 -0
  194. package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
  195. package/src/config/merge/value-selection.js +38 -0
  196. package/src/config/merge.js +43 -11
  197. package/src/config/model-ir-v2.d.ts +57 -0
  198. package/src/config/model-ir-v2.js +258 -0
  199. package/src/config/model-ir.d.ts +49 -0
  200. package/src/config/model-ir.js +180 -0
  201. package/src/config/param-validator.js +1 -1
  202. package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
  203. package/src/config/platforms/loader.js +2 -1
  204. package/src/config/production-release-evidence.d.ts +23 -0
  205. package/src/config/production-release-evidence.js +423 -0
  206. package/src/config/production-release.js +321 -0
  207. package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
  208. package/src/config/required-inference-fields-contract-check.js +2 -2
  209. package/src/config/rerank-reference.d.ts +28 -0
  210. package/src/config/rerank-reference.js +89 -0
  211. package/src/config/revocation-policy.d.ts +65 -0
  212. package/src/config/revocation-policy.js +290 -0
  213. package/src/config/revocation-registry.json +11 -0
  214. package/src/config/revocation-updates.d.ts +68 -0
  215. package/src/config/revocation-updates.js +440 -0
  216. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
  217. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
  218. package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
  219. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
  220. package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
  221. package/src/config/runtime-merge.js +1 -1
  222. package/src/config/runtime.js +1 -1
  223. package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
  224. package/src/config/schema/browser-suite-metrics.schema.js +2 -0
  225. package/src/config/schema/conversion-report.schema.d.ts +3 -0
  226. package/src/config/schema/conversion-report.schema.js +29 -0
  227. package/src/config/schema/converter.schema.d.ts +1 -0
  228. package/src/config/schema/debug.schema.d.ts +1 -0
  229. package/src/config/schema/doppler.schema.js +6 -5
  230. package/src/config/schema/emulation.schema.js +1 -1
  231. package/src/config/schema/execution-v1.schema.d.ts +3 -0
  232. package/src/config/schema/execution-v1.schema.js +19 -0
  233. package/src/config/schema/harness.schema.d.ts +13 -0
  234. package/src/config/schema/harness.schema.js +6 -0
  235. package/src/config/schema/inference.schema.d.ts +21 -2
  236. package/src/config/schema/kernel-thresholds.schema.js +1 -1
  237. package/src/config/schema/manifest.schema.d.ts +26 -7
  238. package/src/config/schema/manifest.schema.js +10 -1
  239. package/src/config/schema/program-bundle.schema.d.ts +52 -6
  240. package/src/config/schema/program-bundle.schema.js +145 -10
  241. package/src/config/sequence-reference.d.ts +26 -0
  242. package/src/config/sequence-reference.js +100 -0
  243. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  244. package/src/config/target-plan.d.ts +108 -0
  245. package/src/config/target-plan.js +411 -0
  246. package/src/config/training-defaults.js +1 -1
  247. package/src/config/transforms/capability-transform-resolver.js +4 -105
  248. package/src/config/transforms/execution-graph/dtype.js +591 -0
  249. package/src/config/transforms/execution-graph/fusion.js +436 -0
  250. package/src/config/transforms/execution-graph/validation.js +414 -0
  251. package/src/config/transforms/execution-graph-transforms.js +13 -1419
  252. package/src/converter/artifact-identity.js +475 -0
  253. package/src/converter/conversion-plan.js +1 -1
  254. package/src/converter/core.js +94 -2096
  255. package/src/converter/forge-candidate-evaluation.js +271 -0
  256. package/src/converter/forge-qualification.js +163 -0
  257. package/src/converter/forge-stages.js +800 -0
  258. package/src/converter/manifest-builder.js +683 -0
  259. package/src/converter/quantization-info.js +4 -1
  260. package/src/converter/quantizer.js +76 -13
  261. package/src/converter/shard-packer.d.ts +1 -1
  262. package/src/converter/shard-packer.js +2 -2
  263. package/src/converter/tensor-transform.js +640 -0
  264. package/src/converter/tokenizer-contract.js +243 -0
  265. package/src/converter/tokenizer-manifest.js +243 -0
  266. package/src/debug/history.js +0 -8
  267. package/src/debug/index.js +0 -10
  268. package/src/debug/tensor.js +1 -1
  269. package/src/debug/trace.js +0 -7
  270. package/src/experimental/adapters/adapter-manager.js +2 -0
  271. package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
  272. package/src/experimental/adapters/adapter-manifest.js +11 -0
  273. package/src/experimental/adapters/artifact-contract.d.ts +2 -2
  274. package/src/experimental/adapters/artifact-contract.js +5 -2
  275. package/src/experimental/adapters/lora-loader.d.ts +10 -1
  276. package/src/experimental/adapters/lora-loader.js +110 -48
  277. package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
  278. package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
  279. package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
  280. package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
  281. package/src/experimental/browser/browser-converter.js +11 -11
  282. package/src/experimental/diffusion/index.d.ts +1 -1
  283. package/src/experimental/diffusion/index.js +1 -1
  284. package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
  285. package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
  286. package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
  287. package/src/experimental/diffusion/vae.js +596 -0
  288. package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
  289. package/src/experimental/distribution/shard-delivery/retry.js +392 -0
  290. package/src/experimental/distribution/shard-delivery/transport.js +831 -0
  291. package/src/experimental/distribution/shard-delivery.js +4 -1709
  292. package/src/experimental/energy/index.d.ts +2 -2
  293. package/src/experimental/energy/index.js +2 -2
  294. package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
  295. package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
  296. package/src/experimental/logits/cpu-output.d.ts +27 -0
  297. package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
  298. package/src/experimental/logits/cpu.d.ts +87 -0
  299. package/src/experimental/logits/cpu.js +95 -0
  300. package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
  301. package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
  302. package/src/experimental/orchestration/index.d.ts +1 -1
  303. package/src/experimental/orchestration/index.js +1 -1
  304. package/src/experimental/training/artifacts.js +2 -2
  305. package/src/experimental/training/checkpoint.js +2 -2
  306. package/src/experimental/training/distillation/artifacts.js +1 -1
  307. package/src/experimental/training/distillation/dataset.js +2 -2
  308. package/src/experimental/training/distillation/eval.js +1 -1
  309. package/src/experimental/training/distillation/runtime.js +2 -2
  310. package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
  311. package/src/experimental/training/distillation/student-fixture/model.js +353 -0
  312. package/src/experimental/training/distillation/student-fixture.js +2 -469
  313. package/src/experimental/training/lora/checkpoint.js +746 -0
  314. package/src/experimental/training/lora/execution.js +550 -0
  315. package/src/experimental/training/lora/plan.js +387 -0
  316. package/src/experimental/training/lora/recovery.js +418 -0
  317. package/src/experimental/training/lora-pipeline.js +7 -1949
  318. package/src/experimental/training/operator-artifacts.js +2 -2
  319. package/src/experimental/training/operator-command.js +4 -4
  320. package/src/experimental/training/runner/evaluation.js +62 -0
  321. package/src/experimental/training/runner/lifecycle.js +257 -0
  322. package/src/experimental/training/runner.js +5 -268
  323. package/src/experimental/training/suite/execution.js +879 -0
  324. package/src/experimental/training/suite/plan.js +855 -0
  325. package/src/experimental/training/suite.js +5 -1659
  326. package/src/experimental/training/workloads.d.ts +5 -5
  327. package/src/experimental/training/workloads.js +6 -6
  328. package/src/experimental/turboquant/codebook.js +470 -0
  329. package/src/formats/quantization-constants.d.ts +11 -0
  330. package/src/formats/quantization-constants.js +11 -0
  331. package/src/formats/rdrr/integrity.js +1 -1
  332. package/src/formats/rdrr/lowerings.js +0 -29
  333. package/src/formats/rdrr/merkle.js +1 -1
  334. package/src/formats/rdrr/tensor-config-validator.js +25 -5
  335. package/src/formats/rdrr/validation.js +3 -3
  336. package/src/gpu/command-recorder.d.ts +10 -1
  337. package/src/gpu/command-recorder.js +58 -14
  338. package/src/gpu/constant-buffer.js +29 -0
  339. package/src/gpu/device-state.js +44 -0
  340. package/src/gpu/device.js +11 -41
  341. package/src/gpu/kernel-runtime.js +2 -1
  342. package/src/gpu/kernels/activation-static-qdq.js +1 -1
  343. package/src/gpu/kernels/append_embedding.wgsl +14 -0
  344. package/src/gpu/kernels/attention/executors.js +960 -0
  345. package/src/gpu/kernels/attention/plan.js +472 -0
  346. package/src/gpu/kernels/attention.d.ts +4 -0
  347. package/src/gpu/kernels/attention.js +145 -1545
  348. package/src/gpu/kernels/attention_relative.wgsl +59 -0
  349. package/src/gpu/kernels/backward/adam.js +2 -1
  350. package/src/gpu/kernels/backward/attention_backward.js +1 -1
  351. package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
  352. package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
  353. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
  354. package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
  355. package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
  356. package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
  357. package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
  358. package/src/gpu/kernels/backward/embed_backward.js +2 -1
  359. package/src/gpu/kernels/backward/executor.d.ts +34 -0
  360. package/src/gpu/kernels/backward/executor.js +158 -0
  361. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
  362. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
  363. package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
  364. package/src/gpu/kernels/backward/gelu_backward.js +1 -1
  365. package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
  366. package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
  367. package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
  368. package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
  369. package/src/gpu/kernels/backward/matmul.js +133 -0
  370. package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
  371. package/src/gpu/kernels/backward/matmul_backward.js +1 -1
  372. package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
  373. package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
  374. package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
  375. package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
  376. package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
  377. package/src/gpu/kernels/backward/rope_backward.js +2 -1
  378. package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
  379. package/src/gpu/kernels/backward/scale_backward.js +2 -1
  380. package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
  381. package/src/gpu/kernels/backward/silu_backward.js +1 -1
  382. package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
  383. package/src/gpu/kernels/backward/softmax_backward.js +1 -1
  384. package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
  385. package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
  386. package/src/gpu/kernels/backward/utils.js +2 -392
  387. package/src/gpu/kernels/bind-group-validation.js +16 -0
  388. package/src/gpu/kernels/cast.js +2 -1
  389. package/src/gpu/kernels/check-finiteness.js +13 -49
  390. package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
  391. package/src/gpu/kernels/check-stop.js +14 -33
  392. package/src/gpu/kernels/check_finiteness.wgsl +33 -0
  393. package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
  394. package/src/gpu/kernels/check_stop.wgsl +20 -0
  395. package/src/gpu/kernels/clamp.js +1 -1
  396. package/src/gpu/kernels/conv2d.js +1 -1
  397. package/src/gpu/kernels/cross_entropy_loss.js +1 -1
  398. package/src/gpu/kernels/depthwise_conv1d.js +1 -1
  399. package/src/gpu/kernels/depthwise_conv2d.js +1 -1
  400. package/src/gpu/kernels/dequant.js +7 -2
  401. package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
  402. package/src/gpu/kernels/dispatch.js +12 -1
  403. package/src/gpu/kernels/embedding-pool.js +72 -0
  404. package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
  405. package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
  406. package/src/gpu/kernels/energy.js +2 -1
  407. package/src/gpu/kernels/forecast_output.wgsl +17 -0
  408. package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
  409. package/src/gpu/kernels/fused_ffn.js +10 -220
  410. package/src/gpu/kernels/fused_matmul_residual.js +2 -1
  411. package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
  412. package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
  413. package/src/gpu/kernels/gated-short-conv.js +11 -69
  414. package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
  415. package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
  416. package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
  417. package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
  418. package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
  419. package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
  420. package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
  421. package/src/gpu/kernels/gather.d.ts +1 -1
  422. package/src/gpu/kernels/gather.js +30 -6
  423. package/src/gpu/kernels/gather_bf16.wgsl +69 -0
  424. package/src/gpu/kernels/gelu.d.ts +4 -0
  425. package/src/gpu/kernels/gelu.js +12 -2
  426. package/src/gpu/kernels/gelu.wgsl +16 -0
  427. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  428. package/src/gpu/kernels/gemma4-route-expert.js +1 -1
  429. package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
  430. package/src/gpu/kernels/groupnorm.js +1 -1
  431. package/src/gpu/kernels/index.d.ts +38 -5
  432. package/src/gpu/kernels/index.js +18 -4
  433. package/src/gpu/kernels/instance_stats.wgsl +26 -0
  434. package/src/gpu/kernels/kernel-base.js +1 -1
  435. package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
  436. package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
  437. package/src/gpu/kernels/kernel-configs.d.ts +10 -61
  438. package/src/gpu/kernels/kernel-configs.js +26 -54
  439. package/src/gpu/kernels/kernel-execution.js +141 -0
  440. package/src/gpu/kernels/kv-cache-write.js +4 -4
  441. package/src/gpu/kernels/kv-quantize.js +4 -1
  442. package/src/gpu/kernels/l2-normalize.js +36 -0
  443. package/src/gpu/kernels/l2_normalize.wgsl +53 -0
  444. package/src/gpu/kernels/layernorm.d.ts +4 -4
  445. package/src/gpu/kernels/layernorm.js +44 -7
  446. package/src/gpu/kernels/linear-attention-core.js +29 -449
  447. package/src/gpu/kernels/linear_attention.js +1 -1
  448. package/src/gpu/kernels/lm-head-argmax.js +2 -2
  449. package/src/gpu/kernels/lm-head-select-logits.js +2 -2
  450. package/src/gpu/kernels/logit-finalize.js +103 -0
  451. package/src/gpu/kernels/logit-merge.js +13 -125
  452. package/src/gpu/kernels/logit-scatter.js +40 -0
  453. package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
  454. package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
  455. package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
  456. package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
  457. package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
  458. package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
  459. package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
  460. package/src/gpu/kernels/matmul-dispatch.js +6 -1
  461. package/src/gpu/kernels/matmul-selection.js +3 -2
  462. package/src/gpu/kernels/matmul.js +4 -15
  463. package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
  464. package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
  465. package/src/gpu/kernels/modulate.js +1 -1
  466. package/src/gpu/kernels/moe.js +3 -1
  467. package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
  468. package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
  469. package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
  470. package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
  471. package/src/gpu/kernels/patch_context.wgsl +29 -0
  472. package/src/gpu/kernels/pipeline-cache.js +3 -1
  473. package/src/gpu/kernels/pixel_shuffle.js +1 -1
  474. package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
  475. package/src/gpu/kernels/relu.js +1 -1
  476. package/src/gpu/kernels/rep-penalty.js +30 -5
  477. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  478. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  479. package/src/gpu/kernels/repeat_channels.js +1 -1
  480. package/src/gpu/kernels/residual.js +2 -2
  481. package/src/gpu/kernels/rmsnorm-pair.js +1 -1
  482. package/src/gpu/kernels/rmsnorm-qk.js +1 -1
  483. package/src/gpu/kernels/rmsnorm-stats.js +14 -113
  484. package/src/gpu/kernels/rmsnorm.js +1 -1
  485. package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
  486. package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
  487. package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
  488. package/src/gpu/kernels/rope-precompute.js +160 -0
  489. package/src/gpu/kernels/rope-qk.js +1 -1
  490. package/src/gpu/kernels/rope.js +2 -2
  491. package/src/gpu/kernels/rope_precompute.wgsl +74 -0
  492. package/src/gpu/kernels/sample.d.ts +1 -0
  493. package/src/gpu/kernels/sample.js +36 -17
  494. package/src/gpu/kernels/sample.wgsl +112 -157
  495. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  496. package/src/gpu/kernels/scale.js +1 -1
  497. package/src/gpu/kernels/shader-cache.js +8 -4
  498. package/src/gpu/kernels/shader-source-scope.js +68 -0
  499. package/src/gpu/kernels/silu.js +2 -1
  500. package/src/gpu/kernels/soft-embedding.js +3 -1
  501. package/src/gpu/kernels/softmax.js +4 -2
  502. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
  503. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
  504. package/src/gpu/kernels/split_qg.js +1 -1
  505. package/src/gpu/kernels/split_qkv.d.ts +2 -0
  506. package/src/gpu/kernels/split_qkv.js +18 -3
  507. package/src/gpu/kernels/split_qkv.wgsl +5 -2
  508. package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
  509. package/src/gpu/kernels/submit_probe.wgsl +6 -0
  510. package/src/gpu/kernels/transpose.js +1 -1
  511. package/src/gpu/kernels/turboquant-codebook.js +4 -535
  512. package/src/gpu/kernels/uniform-utils.js +7 -2
  513. package/src/gpu/kernels/upsample2d.js +1 -1
  514. package/src/gpu/kernels/utils.js +9 -330
  515. package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
  516. package/src/gpu/kernels/vision-average-pool.js +68 -0
  517. package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
  518. package/src/gpu/kernels/vision-patch-embed.js +117 -0
  519. package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
  520. package/src/gpu/kernels/vision-position-embedding.js +79 -0
  521. package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
  522. package/src/gpu/kernels/vision-rope-2d.js +73 -0
  523. package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
  524. package/src/gpu/kernels/vision-spatial-merge.js +100 -0
  525. package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
  526. package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
  527. package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
  528. package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
  529. package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
  530. package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
  531. package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
  532. package/src/gpu/readback-buffer.js +20 -0
  533. package/src/gpu/submit-probe.js +2 -26
  534. package/src/gpu/weight-buffer.d.ts +1 -0
  535. package/src/gpu/weight-buffer.js +7 -0
  536. package/src/index.d.ts +12 -0
  537. package/src/index.js +2 -0
  538. package/src/inference/browser-harness/benchmark-execution.js +872 -0
  539. package/src/inference/browser-harness/benchmark-rerank.js +284 -0
  540. package/src/inference/browser-harness/decode-diagnostics.js +241 -0
  541. package/src/inference/browser-harness/diffusion-performance.js +102 -0
  542. package/src/inference/browser-harness/inference-execution.js +543 -0
  543. package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
  544. package/src/inference/browser-harness/model-resolution.js +137 -0
  545. package/src/inference/browser-harness/report.js +338 -0
  546. package/src/inference/browser-harness/request.js +263 -0
  547. package/src/inference/browser-harness/runtime-config.d.ts +25 -0
  548. package/src/inference/browser-harness/runtime-config.js +255 -0
  549. package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
  550. package/src/inference/browser-harness/runtime-isolation.js +45 -0
  551. package/src/inference/browser-harness/runtime-report.d.ts +26 -0
  552. package/src/inference/browser-harness/runtime-report.js +117 -0
  553. package/src/inference/browser-harness/suite-summary.js +39 -0
  554. package/src/inference/browser-harness/text-evidence.js +694 -0
  555. package/src/inference/browser-harness/text-execution.js +757 -0
  556. package/src/inference/browser-harness/text-input.js +725 -0
  557. package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
  558. package/src/inference/browser-harness-contract-helpers.js +1 -1
  559. package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
  560. package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
  561. package/src/inference/browser-harness-runtime-helpers.js +3 -423
  562. package/src/inference/browser-harness-text-helpers.js +31 -2199
  563. package/src/inference/browser-harness.d.ts +15 -2
  564. package/src/inference/browser-harness.js +38 -1952
  565. package/src/inference/kv-cache/base.js +67 -381
  566. package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
  567. package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
  568. package/src/inference/kv-cache/quantized.d.ts +2 -2
  569. package/src/inference/kv-cache/quantized.js +4 -8
  570. package/src/inference/kv-cache/update.js +309 -0
  571. package/src/inference/model-checkpoint-evidence.d.ts +75 -0
  572. package/src/inference/model-checkpoint-evidence.js +264 -0
  573. package/src/inference/moe-router.js +36 -216
  574. package/src/inference/multi-model-network.js +5 -341
  575. package/src/inference/pipelines/audio/index.js +1 -20
  576. package/src/inference/pipelines/audio/mel.js +0 -38
  577. package/src/inference/pipelines/context.js +6 -2
  578. package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
  579. package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
  580. package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
  581. package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
  582. package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
  583. package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
  584. package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
  585. package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
  586. package/src/inference/pipelines/factory.js +14 -3
  587. package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
  588. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  589. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
  590. package/src/inference/pipelines/text/abort-contract.js +22 -0
  591. package/src/inference/pipelines/text/attention/attn-config.js +1 -16
  592. package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
  593. package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
  594. package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
  595. package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
  596. package/src/inference/pipelines/text/attention/interpreter.js +89 -439
  597. package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
  598. package/src/inference/pipelines/text/attention/plan.js +17 -0
  599. package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
  600. package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
  601. package/src/inference/pipelines/text/attention/projections.js +20 -537
  602. package/src/inference/pipelines/text/attention/query-transform.js +11 -0
  603. package/src/inference/pipelines/text/attention/record.js +0 -6
  604. package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
  605. package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
  606. package/src/inference/pipelines/text/attention/run.js +0 -6
  607. package/src/inference/pipelines/text/attention/types.js +3 -31
  608. package/src/inference/pipelines/text/chat-format.d.ts +3 -0
  609. package/src/inference/pipelines/text/chat-format.js +42 -0
  610. package/src/inference/pipelines/text/config/normalization.js +729 -0
  611. package/src/inference/pipelines/text/config/validation.js +9 -0
  612. package/src/inference/pipelines/text/config.d.ts +12 -1
  613. package/src/inference/pipelines/text/config.js +14 -1228
  614. package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
  615. package/src/inference/pipelines/text/debug-utils/index.js +3 -8
  616. package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
  617. package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
  618. package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
  619. package/src/inference/pipelines/text/dtype-contract.js +1 -68
  620. package/src/inference/pipelines/text/embed.js +178 -241
  621. package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
  622. package/src/inference/pipelines/text/embedding-contract.js +21 -0
  623. package/src/inference/pipelines/text/embedding-extraction.js +225 -0
  624. package/src/inference/pipelines/text/embedding-normalization.js +39 -0
  625. package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
  626. package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
  627. package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
  628. package/src/inference/pipelines/text/execution-v1.js +7 -283
  629. package/src/inference/pipelines/text/execution.js +517 -0
  630. package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
  631. package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
  632. package/src/inference/pipelines/text/ffn/dense.js +6 -970
  633. package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
  634. package/src/inference/pipelines/text/ffn/standard.js +2 -1
  635. package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
  636. package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
  637. package/src/inference/pipelines/text/generator/decode.js +964 -0
  638. package/src/inference/pipelines/text/generator/diffusion.js +607 -0
  639. package/src/inference/pipelines/text/generator/logits-config.js +63 -0
  640. package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
  641. package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
  642. package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
  643. package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
  644. package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
  645. package/src/inference/pipelines/text/generator/recovery.js +310 -0
  646. package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
  647. package/src/inference/pipelines/text/generator/text.js +448 -0
  648. package/src/inference/pipelines/text/generator/training.js +753 -0
  649. package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
  650. package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
  651. package/src/inference/pipelines/text/generator-runtime.js +23 -154
  652. package/src/inference/pipelines/text/generator-steps.js +45 -980
  653. package/src/inference/pipelines/text/generator.js +179 -3552
  654. package/src/inference/pipelines/text/image-transcription.js +304 -0
  655. package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
  656. package/src/inference/pipelines/text/init-rope.js +37 -189
  657. package/src/inference/pipelines/text/init.d.ts +8 -3
  658. package/src/inference/pipelines/text/init.js +6 -216
  659. package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
  660. package/src/inference/pipelines/text/layer-input-execution.js +435 -0
  661. package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
  662. package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
  663. package/src/inference/pipelines/text/layer-plan.js +45 -3
  664. package/src/inference/pipelines/text/layer.js +52 -522
  665. package/src/inference/pipelines/text/lifecycle.js +839 -0
  666. package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
  667. package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
  668. package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
  669. package/src/inference/pipelines/text/linear-attention.js +28 -534
  670. package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
  671. package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
  672. package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
  673. package/src/inference/pipelines/text/logits/cpu.js +8 -95
  674. package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
  675. package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
  676. package/src/inference/pipelines/text/logits/gpu.js +116 -579
  677. package/src/inference/pipelines/text/logits/index.d.ts +2 -1
  678. package/src/inference/pipelines/text/logits/index.js +62 -102
  679. package/src/inference/pipelines/text/logits/output-transform.js +54 -0
  680. package/src/inference/pipelines/text/logits/plan.js +184 -0
  681. package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
  682. package/src/inference/pipelines/text/logits/readback.js +9 -0
  683. package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
  684. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  685. package/src/inference/pipelines/text/lora-apply.js +12 -4
  686. package/src/inference/pipelines/text/lora-types.d.ts +12 -0
  687. package/src/inference/pipelines/text/modality-token-contract.js +115 -0
  688. package/src/inference/pipelines/text/model-load.js +0 -20
  689. package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
  690. package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
  691. package/src/inference/pipelines/text/moe/plan.js +47 -0
  692. package/src/inference/pipelines/text/moe-cpu.js +9 -135
  693. package/src/inference/pipelines/text/moe-gpu.js +21 -545
  694. package/src/inference/pipelines/text/normalization-contract.js +17 -0
  695. package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
  696. package/src/inference/pipelines/text/per-layer/plan.js +362 -0
  697. package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
  698. package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
  699. package/src/inference/pipelines/text/probes.js +20 -0
  700. package/src/inference/pipelines/text/sampling-config.js +22 -113
  701. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  702. package/src/inference/pipelines/text/sampling.js +18 -375
  703. package/src/inference/pipelines/text/state.d.ts +8 -0
  704. package/src/inference/pipelines/text/state.js +1 -0
  705. package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
  706. package/src/inference/pipelines/text/types.d.ts +7 -3
  707. package/src/inference/pipelines/text/weight-loading-config.js +194 -0
  708. package/src/inference/pipelines/text.d.ts +2 -2
  709. package/src/inference/pipelines/text.js +46 -1633
  710. package/src/inference/pipelines/video/frame-sampler.js +0 -13
  711. package/src/inference/pipelines/video/index.js +0 -21
  712. package/src/inference/pipelines/vision/attention.js +59 -0
  713. package/src/inference/pipelines/vision/encoder.js +254 -369
  714. package/src/inference/pipelines/vision/gemma4.js +204 -318
  715. package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
  716. package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
  717. package/src/inference/pipelines/vision/glmocr.js +557 -0
  718. package/src/inference/pipelines/vision/image-preprocess.js +0 -27
  719. package/src/inference/pipelines/vision/index.js +8 -27
  720. package/src/inference/pipelines/vision/ops.js +11 -33
  721. package/src/inference/pipelines/vision/patch-embed.js +51 -145
  722. package/src/inference/pipelines/vision/spatial-merge.js +78 -0
  723. package/src/inference/pipelines/vision/weight-loading.js +79 -0
  724. package/src/inference/refactor-receipt.js +2 -2
  725. package/src/inference/rerank.js +182 -0
  726. package/src/inference/token-sampling.js +319 -0
  727. package/src/inference/tokenizer.js +1 -1
  728. package/src/inference/tokenizers/behavior-flags.js +6 -3
  729. package/src/inference/tokenizers/bundled/decoders.js +133 -0
  730. package/src/inference/tokenizers/bundled/registry.js +216 -0
  731. package/src/inference/tokenizers/bundled.js +23 -353
  732. package/src/loader/doppler-loader.d.ts +4 -0
  733. package/src/loader/doppler-loader.js +119 -930
  734. package/src/loader/embedding-limit-preflight.js +11 -4
  735. package/src/loader/embedding-loader.js +1 -3
  736. package/src/loader/gpu-tensor-materialization.js +20 -0
  737. package/src/loader/layer-loader.js +15 -119
  738. package/src/loader/layer-tensor-aliases.js +99 -0
  739. package/src/loader/layer-weight-shape.js +29 -0
  740. package/src/loader/load-timing.js +64 -0
  741. package/src/loader/loader-types.d.ts +3 -0
  742. package/src/loader/model-executor.js +819 -0
  743. package/src/loader/model-load-validation.js +15 -0
  744. package/src/loader/quantization-constants.d.ts +11 -29
  745. package/src/loader/quantization-constants.js +1 -2
  746. package/src/loader/shard-resolver.js +0 -14
  747. package/src/loader/tensor-load-error.js +14 -0
  748. package/src/loader/tensor-shard-indices.js +8 -0
  749. package/src/loader/tensors/bf16-loader.js +125 -0
  750. package/src/loader/tensors/functional-descriptor-loader.js +3 -375
  751. package/src/loader/tensors/load-executor.js +577 -0
  752. package/src/loader/tensors/source-transform/executor.js +123 -0
  753. package/src/loader/tensors/source-transform/plan.js +107 -0
  754. package/src/loader/tensors/source-transform.js +15 -264
  755. package/src/loader/tensors/tensor-loader.d.ts +4 -4
  756. package/src/loader/tensors/tensor-loader.js +7 -653
  757. package/src/memory/buffer-pool.js +7 -29
  758. package/src/rules/capability-transform-resolver.js +80 -0
  759. package/src/rules/execution-contract.d.ts +73 -0
  760. package/src/rules/execution-contract.js +429 -0
  761. package/src/rules/execution-rules-contract-check.js +1 -1
  762. package/src/rules/generated/rule-bundle.json +4305 -0
  763. package/src/rules/inference/capability-transforms.rules.json +3 -3
  764. package/src/rules/layer-pattern-contract-check.js +1 -1
  765. package/src/rules/rule-registry.js +70 -57
  766. package/src/storage/artifact-storage-context.js +2 -2
  767. package/src/storage/backend-selection.js +43 -0
  768. package/src/storage/backends/opfs-store.d.ts +41 -0
  769. package/src/storage/distribution-transport.js +5 -0
  770. package/src/storage/download/integrity.d.ts +33 -0
  771. package/src/storage/download/integrity.js +130 -0
  772. package/src/storage/download/resume-inspection.d.ts +22 -0
  773. package/src/storage/download/resume-inspection.js +134 -0
  774. package/src/storage/download/retry.js +57 -0
  775. package/src/storage/download/state.js +131 -0
  776. package/src/storage/download/transport.js +99 -0
  777. package/src/storage/downloader.js +74 -453
  778. package/src/storage/model-cache.d.ts +1 -0
  779. package/src/storage/model-cache.js +51 -11
  780. package/src/storage/shard-manager.d.ts +5 -0
  781. package/src/storage/shard-manager.js +50 -369
  782. package/src/storage/shards/index.js +60 -0
  783. package/src/storage/shards/integrity.js +225 -0
  784. package/src/storage/shards/lifecycle.js +22 -0
  785. package/src/storage/source-artifact-store.js +14 -4
  786. package/src/tooling/boundary-evidence.d.ts +4 -4
  787. package/src/tooling/boundary-evidence.js +20 -20
  788. package/src/tooling/browser-command-runner.js +13 -6
  789. package/src/tooling/capsule-v2.d.ts +11 -0
  790. package/src/tooling/capsule-v2.js +58 -0
  791. package/src/tooling/capsule.js +20 -0
  792. package/src/tooling/command-api-constants.js +1 -1
  793. package/src/tooling/command-api-family-normalizers.js +163 -5
  794. package/src/tooling/command-api.d.ts +42 -3
  795. package/src/tooling/command-api.js +7 -2
  796. package/src/tooling/command-envelope.js +1 -1
  797. package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
  798. package/src/tooling/command-runner.html +49 -1
  799. package/src/tooling/command-validation.js +1 -24
  800. package/src/tooling/conversion-run-timing.js +24 -0
  801. package/src/tooling/diagnose-runner.js +11 -9
  802. package/src/tooling/distribution-shard-transport.js +11 -0
  803. package/src/tooling/execution-cost-ledger.d.ts +76 -2
  804. package/src/tooling/execution-cost-ledger.js +197 -15
  805. package/src/tooling/hf-registry/catalog-io.js +63 -0
  806. package/src/tooling/litert/package.js +863 -0
  807. package/src/tooling/litert-package-runtime.js +3 -849
  808. package/src/tooling/model-capsule-forge.js +478 -0
  809. package/src/tooling/node-browser/execution.d.ts +40 -0
  810. package/src/tooling/node-browser/execution.js +308 -0
  811. package/src/tooling/node-browser/static-server.js +189 -0
  812. package/src/tooling/node-browser/transport.d.ts +80 -0
  813. package/src/tooling/node-browser/transport.js +622 -0
  814. package/src/tooling/node-browser-command-runner.d.ts +18 -78
  815. package/src/tooling/node-browser-command-runner.js +10 -1112
  816. package/src/tooling/node-capsule-artifact-store.js +44 -0
  817. package/src/tooling/node-command-execution.js +237 -0
  818. package/src/tooling/node-command-runner.js +26 -205
  819. package/src/tooling/node-converter/input.js +521 -0
  820. package/src/tooling/node-converter/output.js +244 -0
  821. package/src/tooling/node-converter-chunk-batches.js +21 -0
  822. package/src/tooling/node-converter.js +7 -720
  823. package/src/tooling/node-source-runtime.js +16 -276
  824. package/src/tooling/node-webgpu-provider.v1.json +33 -0
  825. package/src/tooling/node-webgpu.js +178 -453
  826. package/src/tooling/opfs-cache.d.ts +5 -0
  827. package/src/tooling/opfs-cache.js +4 -0
  828. package/src/tooling/production-release.d.ts +19 -0
  829. package/src/tooling/production-release.js +647 -0
  830. package/src/tooling/program-bundle/embedding-reference.js +19 -0
  831. package/src/tooling/program-bundle/materialize.js +324 -0
  832. package/src/tooling/program-bundle/rerank-reference.js +22 -0
  833. package/src/tooling/program-bundle/sequence-reference.js +42 -0
  834. package/src/tooling/program-bundle/validation.js +31 -0
  835. package/src/tooling/program-bundle/wgsl-closure.js +254 -0
  836. package/src/tooling/program-bundle-parity.d.ts +11 -4
  837. package/src/tooling/program-bundle-parity.js +262 -109
  838. package/src/tooling/program-bundle.d.ts +7 -0
  839. package/src/tooling/program-bundle.js +186 -591
  840. package/src/tooling/provider-v1-contract.d.ts +61 -0
  841. package/src/tooling/provider-v1-contract.js +192 -0
  842. package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
  843. package/src/tooling/registered-variant-calibration-job.js +210 -0
  844. package/src/tooling/registered-variant-calibration.d.ts +167 -7
  845. package/src/tooling/registered-variant-calibration.js +57 -7
  846. package/src/tooling/runtime-optimization/candidates.js +664 -0
  847. package/src/tooling/runtime-optimization-campaign.js +166 -0
  848. package/src/tooling/runtime-optimization-index.js +16 -1
  849. package/src/tooling/runtime-optimization.d.ts +44 -0
  850. package/src/tooling/runtime-optimization.js +13 -656
  851. package/src/tooling/source-intake.d.ts +1 -1
  852. package/src/tooling/source-intake.js +34 -17
  853. package/src/tooling/source-package-profiles.js +1 -1
  854. package/src/tooling/source-runtime/bundle-materialize.js +298 -0
  855. package/src/tooling/source-runtime/bundle-plan.js +121 -0
  856. package/src/tooling/source-runtime/node-loader.js +258 -0
  857. package/src/tooling/source-runtime-bundle.js +26 -411
  858. package/src/tooling/source-runtime-converter-config.js +1 -1
  859. package/src/tooling-experimental-exports.js +0 -5
  860. package/src/tooling-exports/evidence.d.ts +14 -3
  861. package/src/tooling-exports/evidence.js +14 -3
  862. package/src/tooling-exports/storage.d.ts +7 -0
  863. package/src/tooling-exports/storage.js +5 -0
  864. package/src/tooling-exports.d.ts +3 -0
  865. package/src/tooling-exports.js +2 -0
  866. package/src/tooling-exports.shared.d.ts +17 -3
  867. package/src/tooling-exports.shared.js +9 -3
  868. package/src/training.d.ts +6 -6
  869. package/src/training.js +4 -4
  870. package/src/version.js +1 -1
  871. package/tests/kernels/browser/kernel-suite.js +512 -0
  872. package/tests/kernels/browser/test-page.js +170 -14
  873. package/tests/kernels/harness/tolerance.js +1 -0
  874. package/tests/kernels/reference/index.js +13 -1
  875. package/tests/kernels/reference/vision.js +160 -0
  876. package/src/converter/quantizer.d.ts +0 -121
  877. package/src/gpu/kernels/backward/utils.d.ts +0 -90
  878. package/src/gpu/kernels/uniform-utils.d.ts +0 -66
  879. package/src/gpu/kernels/utils.d.ts +0 -147
  880. package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
  881. package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
  882. package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
  883. package/src/inference/pipelines/text/moe-helpers.js +0 -21
  884. package/src/rules/converter/execution.rules.json +0 -20
  885. package/src/rules/converter/tensor-roles.rules.json +0 -13
  886. package/src/rules/converter/tokenizer.rules.json +0 -12
  887. package/src/rules/inference/attention.rules.json +0 -69
  888. package/src/rules/inference/config.rules.json +0 -82
  889. package/src/rules/inference/dtype.rules.json +0 -113
  890. package/src/rules/inference/execution.rules.json +0 -195
  891. package/src/rules/inference/ffn.rules.json +0 -123
  892. package/src/rules/inference/layer-pattern.rules.json +0 -16
  893. package/src/rules/inference/layer.rules.json +0 -7
  894. package/src/rules/inference/moe.rules.json +0 -145
  895. package/src/rules/kernels/attention.rules.json +0 -68
  896. package/src/rules/kernels/conv2d.rules.json +0 -6
  897. package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
  898. package/src/rules/kernels/dequant.rules.json +0 -58
  899. package/src/rules/kernels/energy.rules.json +0 -22
  900. package/src/rules/kernels/fused-ffn.rules.json +0 -15
  901. package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
  902. package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
  903. package/src/rules/kernels/gather.rules.json +0 -93
  904. package/src/rules/kernels/gelu.rules.json +0 -11
  905. package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
  906. package/src/rules/kernels/groupnorm.rules.json +0 -10
  907. package/src/rules/kernels/kv_quantize.rules.json +0 -9
  908. package/src/rules/kernels/layernorm.rules.json +0 -6
  909. package/src/rules/kernels/linear-attention.rules.json +0 -6
  910. package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
  911. package/src/rules/kernels/matmul.rules.json +0 -95
  912. package/src/rules/kernels/modulate.rules.json +0 -6
  913. package/src/rules/kernels/moe.rules.gptoss.json +0 -105
  914. package/src/rules/kernels/moe.rules.json +0 -14
  915. package/src/rules/kernels/moe.rules.mixtral.json +0 -75
  916. package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
  917. package/src/rules/kernels/relu.rules.json +0 -6
  918. package/src/rules/kernels/rep-penalty.rules.json +0 -6
  919. package/src/rules/kernels/repeat-channels.rules.json +0 -6
  920. package/src/rules/kernels/residual.rules.json +0 -12
  921. package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
  922. package/src/rules/kernels/rmsnorm.rules.json +0 -46
  923. package/src/rules/kernels/rope-qk.rules.json +0 -6
  924. package/src/rules/kernels/rope.rules.json +0 -6
  925. package/src/rules/kernels/sample.rules.json +0 -6
  926. package/src/rules/kernels/scale.rules.json +0 -8
  927. package/src/rules/kernels/silu.rules.json +0 -22
  928. package/src/rules/kernels/softmax.rules.json +0 -25
  929. package/src/rules/kernels/split-qg.rules.json +0 -6
  930. package/src/rules/kernels/split-qkv.rules.json +0 -6
  931. package/src/rules/kernels/upsample2d.rules.json +0 -6
  932. package/src/rules/loader/tensor-loader.rules.json +0 -42
  933. package/src/rules/loader/weights.rules.json +0 -42
  934. package/src/rules/tooling/command-runtime.rules.json +0 -65
  935. package/src/tooling/hf-registry-utils.js +0 -395
  936. /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
  937. /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
  938. /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
  939. /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
  940. /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
  941. /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
  942. /package/src/{utils → formats}/array-buffer.js +0 -0
  943. /package/src/{utils → formats}/canonical-hash.js +0 -0
  944. /package/src/{utils → formats}/clone-json.js +0 -0
  945. /package/src/{utils → formats}/encode-utf8.js +0 -0
  946. /package/src/{utils → formats}/load-json.js +0 -0
  947. /package/src/{utils → formats}/plain-object.js +0 -0
  948. /package/src/{utils → formats}/sha256.js +0 -0
  949. /package/src/{utils → formats}/stable-sort-object.js +0 -0
  950. /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
  951. /package/src/{utils → storage}/hf-resolve-url.js +0 -0
  952. /package/src/{utils → storage}/runtime-env.js +0 -0
  953. /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
@@ -1,2199 +1,31 @@
1
- import { readBuffer } from '../memory/buffer-pool.js';
2
- import { CAPTURE_LEVELS, createDefaultCaptureConfig } from '../debug/capture-policy.js';
3
- import { selectRuleValue } from '../rules/rule-registry.js';
4
- import { loadJson } from '../utils/load-json.js';
5
- import { isPlainObject } from '../utils/plain-object.js';
6
- import { cloneJsonValue } from '../utils/clone-json.js';
7
- import { sha256BytesHex } from '../utils/sha256.js';
8
- import { resolvePromptInput } from './pipelines/text/generator-prefill-helpers.js';
9
- import { isExecutionObservationRequested } from '../tooling/execution-cost-ledger.js';
10
-
11
- const DEFAULT_IMAGE_TRANSCRIPTION_PROMPT = 'Describe the image in one short sentence.';
12
- const DEFAULT_IMAGE_TRANSCRIPTION_SOFT_TOKEN_BUDGET = 70;
13
- const EMBEDDING_PREVIEW_LENGTH = 16;
14
- const GENERATION_TOKEN_DIAGNOSTIC_LIMIT = 32;
15
- const DECODE_RECORD_TOP_OP_LIMIT = 20;
16
- const RERANK_SCORE_POLICIES = new Set(['logit_difference', 'true_logit']);
17
-
18
- export function normalizeDecodeRecordOpLabels(value) {
19
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
20
- return null;
21
- }
22
- const entries = [];
23
- for (const [label, rawCount] of Object.entries(value)) {
24
- const count = Number(rawCount);
25
- if (typeof label !== 'string' || label.length === 0 || !Number.isFinite(count) || count <= 0) {
26
- continue;
27
- }
28
- entries.push([label, count]);
29
- }
30
- if (entries.length === 0) {
31
- return null;
32
- }
33
- entries.sort((a, b) => {
34
- const countDelta = b[1] - a[1];
35
- return countDelta !== 0 ? countDelta : a[0].localeCompare(b[0]);
36
- });
37
- return Object.fromEntries(entries);
38
- }
39
-
40
- export function buildDecodeRecordTopOps(labelCounts, totalOps = null, limit = DECODE_RECORD_TOP_OP_LIMIT) {
41
- const normalized = normalizeDecodeRecordOpLabels(labelCounts);
42
- if (!normalized) {
43
- return [];
44
- }
45
- const entries = Object.entries(normalized);
46
- const denominator = Number.isFinite(totalOps) && totalOps > 0
47
- ? totalOps
48
- : entries.reduce((sum, [, count]) => sum + count, 0);
49
- const maxEntries = Number.isFinite(limit) && limit > 0
50
- ? Math.floor(limit)
51
- : DECODE_RECORD_TOP_OP_LIMIT;
52
- return entries.slice(0, maxEntries).map(([label, count]) => ({
53
- label,
54
- count,
55
- shareOfOps: denominator > 0 ? count / denominator : null,
56
- }));
57
- }
58
-
59
- function normalizeDecodeRecordOpGroupLabel(label) {
60
- const grouped = label
61
- .replace(/^L\d+[.:]/, '')
62
- .replace(/:L\d+(?=:|$)/g, '');
63
- return grouped.length > 0 ? grouped : label;
64
- }
65
-
66
- export function groupDecodeRecordOpLabels(labelCounts) {
67
- const normalized = normalizeDecodeRecordOpLabels(labelCounts);
68
- if (!normalized) {
69
- return null;
70
- }
71
- const groups = {};
72
- for (const [label, count] of Object.entries(normalized)) {
73
- const groupLabel = normalizeDecodeRecordOpGroupLabel(label);
74
- groups[groupLabel] = (groups[groupLabel] ?? 0) + count;
75
- }
76
- return normalizeDecodeRecordOpLabels(groups);
77
- }
78
-
79
- export function buildDecodeRecordTopOpGroups(labelCounts, totalOps = null, limit = DECODE_RECORD_TOP_OP_LIMIT) {
80
- return buildDecodeRecordTopOps(
81
- groupDecodeRecordOpLabels(labelCounts),
82
- totalOps,
83
- limit
84
- );
85
- }
86
-
87
- function nonNegativeFiniteNumber(value) {
88
- const number = Number(value);
89
- return Number.isFinite(number) && number >= 0 ? number : null;
90
- }
91
-
92
- export function normalizeUniformCacheStats(value) {
93
- if (!value || typeof value !== 'object' || Array.isArray(value)) {
94
- return null;
95
- }
96
- const hits = nonNegativeFiniteNumber(value.hits);
97
- const misses = nonNegativeFiniteNumber(value.misses);
98
- const evictions = nonNegativeFiniteNumber(value.evictions);
99
- const currentSize = nonNegativeFiniteNumber(value.currentSize);
100
- const pendingDestruction = nonNegativeFiniteNumber(value.pendingDestruction);
101
- const totalLookups = Number.isFinite(hits) && Number.isFinite(misses)
102
- ? hits + misses
103
- : null;
104
- const stats = {};
105
- if (Number.isFinite(hits)) stats.hits = hits;
106
- if (Number.isFinite(misses)) stats.misses = misses;
107
- if (Number.isFinite(totalLookups)) stats.totalLookups = totalLookups;
108
- if (Number.isFinite(totalLookups) && totalLookups > 0) stats.hitRateRatio = hits / totalLookups;
109
- if (typeof value.hitRate === 'string' && value.hitRate.length > 0) stats.hitRate = value.hitRate;
110
- if (Number.isFinite(evictions)) stats.evictions = evictions;
111
- if (Number.isFinite(currentSize)) stats.currentSize = currentSize;
112
- if (Number.isFinite(pendingDestruction)) stats.pendingDestruction = pendingDestruction;
113
- return Object.keys(stats).length > 0 ? stats : null;
114
- }
115
-
116
- function normalizeTokenIdArray(value, label) {
117
- const raw = ArrayBuffer.isView(value) ? Array.from(value) : value;
118
- if (!Array.isArray(raw)) {
119
- throw new Error(`${label} must be an array or typed array of token IDs.`);
120
- }
121
- return raw.map((entry) => {
122
- const tokenId = Number(entry);
123
- if (!Number.isInteger(tokenId) || tokenId < 0) {
124
- throw new Error(`${label} contains invalid token ID ${entry}.`);
125
- }
126
- return tokenId;
127
- });
128
- }
129
-
130
- function resolveGenerationUseChatTemplate(pipeline, runtimeConfig, runOverrides, promptInput) {
131
- if (typeof runOverrides?.useChatTemplate === 'boolean') {
132
- return runOverrides.useChatTemplate;
133
- }
134
- if (typeof runtimeConfig?.inference?.chatTemplate?.enabled === 'boolean') {
135
- return runtimeConfig.inference.chatTemplate.enabled;
136
- }
137
- if (isStructuredPromptInput(promptInput)) {
138
- return true;
139
- }
140
- if (typeof pipeline?.modelConfig?.chatTemplateEnabled === 'boolean') {
141
- return pipeline.modelConfig.chatTemplateEnabled;
142
- }
143
- return false;
144
- }
145
-
146
- function resolvePromptTokenIdsForTranscript(pipeline, promptInput, useChatTemplate) {
147
- if (!pipeline?.tokenizer || typeof pipeline.tokenizer.encode !== 'function') {
148
- return null;
149
- }
150
- const processedPrompt = resolvePromptInput(
151
- { modelConfig: pipeline.modelConfig ?? {} },
152
- promptInput,
153
- useChatTemplate,
154
- 'browserHarness.referenceTranscript'
155
- );
156
- return normalizeTokenIdArray(
157
- pipeline.tokenizer.encode(processedPrompt),
158
- 'browserHarness.referenceTranscript.promptTokenIds'
159
- );
160
- }
161
-
162
- function bytesFromArrayBufferView(view) {
163
- return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
164
- }
165
-
166
- function digestBytes(bytes) {
167
- return `sha256:${sha256BytesHex(bytes)}`;
168
- }
169
-
170
- function selectTopLogits(logits, limit, decodeToken) {
171
- const top = [];
172
- const count = Math.max(1, Math.floor(limit));
173
- for (let tokenId = 0; tokenId < logits.length; tokenId++) {
174
- const logit = logits[tokenId];
175
- if (!Number.isFinite(logit)) continue;
176
- const candidate = { tokenId, logit };
177
- let insertAt = top.length;
178
- while (
179
- insertAt > 0
180
- && (
181
- candidate.logit > top[insertAt - 1].logit
182
- || (candidate.logit === top[insertAt - 1].logit && candidate.tokenId < top[insertAt - 1].tokenId)
183
- )
184
- ) {
185
- insertAt -= 1;
186
- }
187
- top.splice(insertAt, 0, candidate);
188
- if (top.length > count) {
189
- top.pop();
190
- }
191
- }
192
- return top.map((entry) => ({
193
- tokenId: entry.tokenId,
194
- logit: entry.logit,
195
- text: typeof decodeToken === 'function' ? decodeToken(entry.tokenId) : null,
196
- }));
197
- }
198
-
199
- function getReferenceTranscriptRuntimeConfig(runtimeConfig) {
200
- const config = runtimeConfig?.shared?.harness?.referenceTranscript;
201
- return isPlainObject(config) ? config : null;
202
- }
203
-
204
- function shouldCaptureReferenceLogits(runOverrides, runtimeConfig) {
205
- const referenceConfig = getReferenceTranscriptRuntimeConfig(runtimeConfig);
206
- return runOverrides?.diagnostics?.referenceTranscript?.captureLogits === true
207
- || runOverrides?.diagnostics?.captureLogits === true
208
- || referenceConfig?.captureLogits === true;
209
- }
210
-
211
- function shouldCaptureReferenceKvBytes(runOverrides, runtimeConfig) {
212
- const referenceConfig = getReferenceTranscriptRuntimeConfig(runtimeConfig);
213
- return runOverrides?.diagnostics?.referenceTranscript?.captureKvBytes === true
214
- || runOverrides?.diagnostics?.captureKvBytes === true
215
- || referenceConfig?.captureKvBytes === true;
216
- }
217
-
218
- function shouldEnableReferenceTranscriptDiagnostics(runOverrides, runtimeConfig) {
219
- const referenceConfig = getReferenceTranscriptRuntimeConfig(runtimeConfig);
220
- return runOverrides?.diagnostics?.enabled === true
221
- || referenceConfig?.enabled === true
222
- || referenceConfig?.captureLogits === true
223
- || referenceConfig?.captureKvBytes === true;
224
- }
225
-
226
- export function digestLogitsForTranscript(logits, context) {
227
- if (!(logits instanceof Float32Array)) {
228
- throw new Error('reference transcript logits capture requires Float32Array logits.');
229
- }
230
- const digest = digestBytes(bytesFromArrayBufferView(logits));
231
- const topK = Number.isInteger(context?.topK) ? Math.max(1, context.topK) : 8;
232
- const decodeToken = typeof context?.decodeToken === 'function' ? context.decodeToken : null;
233
- return {
234
- index: Number.isInteger(context?.index) ? context.index : null,
235
- tokenId: Number.isInteger(context?.tokenId) ? context.tokenId : null,
236
- inputTokenCount: Number.isInteger(context?.inputTokenCount) ? context.inputTokenCount : null,
237
- dtype: 'f32',
238
- elementCount: logits.length,
239
- digest,
240
- top: selectTopLogits(logits, topK, decodeToken),
241
- };
242
- }
243
-
244
- async function digestKvLayerBytes(layer, layerIdx, kvCache) {
245
- const seqLen = Number.isFinite(layer?.seqLen) ? Math.max(0, Math.floor(layer.seqLen)) : 0;
246
- const byteLength = seqLen * kvCache.kvSize * kvCache.bytesPerElem;
247
- if (byteLength < 1) {
248
- return {
249
- layer: layerIdx,
250
- seqLen,
251
- keyBytes: 0,
252
- valueBytes: 0,
253
- keyDigest: digestBytes(new Uint8Array()),
254
- valueDigest: digestBytes(new Uint8Array()),
255
- };
256
- }
257
-
258
- if (layer?.keysGPU && layer?.valuesGPU) {
259
- const [keyBuffer, valueBuffer] = await Promise.all([
260
- readBuffer(layer.keysGPU, byteLength),
261
- readBuffer(layer.valuesGPU, byteLength),
262
- ]);
263
- return {
264
- layer: layerIdx,
265
- seqLen,
266
- keyBytes: byteLength,
267
- valueBytes: byteLength,
268
- keyDigest: digestBytes(new Uint8Array(keyBuffer)),
269
- valueDigest: digestBytes(new Uint8Array(valueBuffer)),
270
- };
271
- }
272
-
273
- const elementCount = seqLen * kvCache.kvSize;
274
- if (layer?.keys instanceof Float32Array && layer?.values instanceof Float32Array) {
275
- const keys = layer.keys.subarray(0, elementCount);
276
- const values = layer.values.subarray(0, elementCount);
277
- return {
278
- layer: layerIdx,
279
- seqLen,
280
- keyBytes: keys.byteLength,
281
- valueBytes: values.byteLength,
282
- keyDigest: digestBytes(bytesFromArrayBufferView(keys)),
283
- valueDigest: digestBytes(bytesFromArrayBufferView(values)),
284
- };
285
- }
286
-
287
- throw new Error(`reference transcript KV byte capture unsupported for layer ${layerIdx}.`);
288
- }
289
-
290
- export async function captureKvCacheByteProof(pipeline, enabled) {
291
- if (!enabled) return null;
292
- const kvCache = pipeline?.kvCache ?? null;
293
- if (!kvCache || !Array.isArray(kvCache.layers)) {
294
- return null;
295
- }
296
- if (kvCache.layout !== 'contiguous') {
297
- throw new Error(
298
- `reference transcript KV byte capture only supports contiguous KV cache layout; got ${kvCache.layout}.`
299
- );
300
- }
301
- const layers = [];
302
- for (let layerIdx = 0; layerIdx < kvCache.layers.length; layerIdx += 1) {
303
- layers.push(await digestKvLayerBytes(kvCache.layers[layerIdx], layerIdx, kvCache));
304
- }
305
- const canonicalBytes = new TextEncoder().encode(JSON.stringify({
306
- mode: 'sha256-layer-kv-bytes',
307
- layout: kvCache.layout,
308
- kvDtype: kvCache.kvDtype ?? null,
309
- kvSize: kvCache.kvSize,
310
- bytesPerElem: kvCache.bytesPerElem,
311
- layers,
312
- }));
313
- return {
314
- mode: 'sha256-layer-kv-bytes',
315
- layout: kvCache.layout,
316
- kvDtype: kvCache.kvDtype ?? null,
317
- layerCount: layers.length,
318
- digest: digestBytes(canonicalBytes),
319
- layers,
320
- };
321
- }
322
-
323
- const embeddingSemanticFixtureAsset = await loadJson(
324
- './fixtures/embedding-semantic-fixtures.json',
325
- import.meta.url,
326
- 'Failed to load embedding semantic fixtures'
327
- );
328
-
329
- const rerankSemanticFixtureAsset = await loadJson(
330
- './fixtures/rerank-semantic-fixtures.json',
331
- import.meta.url,
332
- 'Failed to load rerank semantic fixtures'
333
- );
334
-
335
- function asText(value) {
336
- if (typeof value !== 'string') return null;
337
- const trimmed = value.trim();
338
- return trimmed || null;
339
- }
340
-
341
- function normalizeRetrievalFixtures(cases) {
342
- if (!Array.isArray(cases)) return null;
343
- const normalized = [];
344
- for (let i = 0; i < cases.length; i++) {
345
- const entry = cases[i];
346
- if (!entry || typeof entry !== 'object') continue;
347
-
348
- const query = asText(entry.query);
349
- const docs = Array.isArray(entry.docs) ? entry.docs.map(asText).filter(Boolean) : [];
350
- if (!query || docs.length === 0 || !Number.isFinite(entry.expectedDoc)) {
351
- continue;
352
- }
353
- const expectedDoc = Math.floor(entry.expectedDoc);
354
- normalized.push({
355
- id: asText(entry.id) ?? `case-${i + 1}`,
356
- query,
357
- docs,
358
- expectedDoc: Math.max(0, Math.min(expectedDoc, docs.length - 1)),
359
- });
360
- }
361
- return normalized.length > 0 ? normalized : null;
362
- }
363
-
364
- function normalizePairFixtures(cases) {
365
- if (!Array.isArray(cases)) return null;
366
- const normalized = [];
367
- for (let i = 0; i < cases.length; i++) {
368
- const entry = cases[i];
369
- if (!entry || typeof entry !== 'object') continue;
370
-
371
- const anchor = asText(entry.anchor);
372
- const positive = asText(entry.positive);
373
- const negative = asText(entry.negative);
374
- if (!anchor || !positive || !negative) {
375
- continue;
376
- }
377
- normalized.push({
378
- id: asText(entry.id) ?? `pair-${i + 1}`,
379
- anchor,
380
- positive,
381
- negative,
382
- });
383
- }
384
- return normalized.length > 0 ? normalized : null;
385
- }
386
-
387
- function normalizeLengthStabilityCases(cases) {
388
- if (!Array.isArray(cases)) return null;
389
- const normalized = [];
390
- for (let i = 0; i < cases.length; i++) {
391
- const entry = cases[i];
392
- if (!entry || typeof entry !== 'object') continue;
393
- const short_ = asText(entry.short);
394
- const medium = asText(entry.medium);
395
- const long_ = asText(entry.long);
396
- if (!short_ || !medium || !long_) continue;
397
- normalized.push({
398
- id: asText(entry.id) ?? `length-${i + 1}`,
399
- short: short_,
400
- medium,
401
- long: long_,
402
- maxCosineDrift: Number.isFinite(entry.maxCosineDrift) ? entry.maxCosineDrift : 0.25,
403
- });
404
- }
405
- return normalized.length > 0 ? normalized : null;
406
- }
407
-
408
- function normalizeThroughputCorpus(corpus) {
409
- if (!Array.isArray(corpus)) return null;
410
- const normalized = corpus.map(asText).filter(Boolean);
411
- return normalized.length > 0 ? normalized : null;
412
- }
413
-
414
- function normalizeRerankCases(cases) {
415
- if (!Array.isArray(cases)) return null;
416
- const normalized = [];
417
- for (let i = 0; i < cases.length; i++) {
418
- const entry = cases[i];
419
- if (!entry || typeof entry !== 'object') continue;
420
-
421
- const query = asText(entry.query);
422
- const positive = asText(entry.positive);
423
- const negative = asText(entry.negative);
424
- if (!query || !positive || !negative) {
425
- continue;
426
- }
427
- normalized.push({
428
- id: asText(entry.id) ?? `rerank-${i + 1}`,
429
- query,
430
- positive,
431
- negative,
432
- });
433
- }
434
- return normalized.length > 0 ? normalized : null;
435
- }
436
-
437
- function resolveDefaultRerankSemanticFixtures() {
438
- const defaults = rerankSemanticFixtureAsset?.defaults;
439
- if (!isPlainObject(defaults)) {
440
- throw new Error('Rerank semantic fixture asset must define defaults.');
441
- }
442
-
443
- const cases = normalizeRerankCases(defaults.cases);
444
- if (!cases) {
445
- throw new Error('Rerank semantic fixture asset must define cases.');
446
- }
447
- if (!Number.isFinite(defaults.minPairAcc)) {
448
- throw new Error('Rerank semantic fixture asset must define minPairAcc.');
449
- }
450
- if (!Number.isFinite(defaults.minScoreMargin)) {
451
- throw new Error('Rerank semantic fixture asset must define minScoreMargin.');
452
- }
453
-
454
- return {
455
- cases,
456
- minPairAcc: Math.max(0, Math.min(1, Number(defaults.minPairAcc))),
457
- minScoreMargin: Number(defaults.minScoreMargin),
458
- };
459
- }
460
-
461
- const DEFAULT_RERANK_SEMANTIC_FIXTURES = resolveDefaultRerankSemanticFixtures();
462
-
463
- export function getDefaultRerankSemanticFixtures() {
464
- return cloneJsonValue(DEFAULT_RERANK_SEMANTIC_FIXTURES);
465
- }
466
-
467
- function resolveDefaultEmbeddingSemanticFixtures() {
468
- const defaults = embeddingSemanticFixtureAsset?.defaults;
469
- if (!isPlainObject(defaults)) {
470
- throw new Error('Embedding semantic fixture asset must define defaults.');
471
- }
472
-
473
- const retrievalCases = normalizeRetrievalFixtures(defaults.retrievalCases);
474
- if (!retrievalCases) {
475
- throw new Error('Embedding semantic fixture asset must define retrievalCases.');
476
- }
477
-
478
- const pairCases = normalizePairFixtures(defaults.pairCases);
479
- if (!pairCases) {
480
- throw new Error('Embedding semantic fixture asset must define pairCases.');
481
- }
482
-
483
- if (!Number.isFinite(defaults.minRetrievalTop1Acc)) {
484
- throw new Error('Embedding semantic fixture asset must define minRetrievalTop1Acc.');
485
- }
486
- if (!Number.isFinite(defaults.minPairAcc)) {
487
- throw new Error('Embedding semantic fixture asset must define minPairAcc.');
488
- }
489
- if (!Number.isFinite(defaults.pairMargin)) {
490
- throw new Error('Embedding semantic fixture asset must define pairMargin.');
491
- }
492
-
493
- return {
494
- retrievalCases,
495
- pairCases,
496
- lengthStabilityCases: normalizeLengthStabilityCases(defaults.lengthStabilityCases) ?? [],
497
- throughputCorpus: normalizeThroughputCorpus(defaults.throughputCorpus) ?? [],
498
- minRetrievalTop1Acc: Math.max(0, Math.min(1, Number(defaults.minRetrievalTop1Acc))),
499
- minPairAcc: Math.max(0, Math.min(1, Number(defaults.minPairAcc))),
500
- pairMargin: Number(defaults.pairMargin),
501
- };
502
- }
503
-
504
- const DEFAULT_EMBEDDING_SEMANTIC_FIXTURES = resolveDefaultEmbeddingSemanticFixtures();
505
-
506
- export function getDefaultEmbeddingSemanticFixtures() {
507
- return cloneJsonValue(DEFAULT_EMBEDDING_SEMANTIC_FIXTURES);
508
- }
509
-
510
- function resolveEmbeddingSemanticFixtures(runtimeConfig, options = null) {
511
- const overrides = isPlainObject(options?.embeddingSemantic)
512
- ? options.embeddingSemantic
513
- : null;
514
- const runtimeOverrides = runtimeConfig?.shared?.benchmark?.run?.embeddingSemantic;
515
- const source = overrides ?? (isPlainObject(runtimeOverrides) ? runtimeOverrides : null);
516
-
517
- const retrievalCases = normalizeRetrievalFixtures(source?.retrievalCases)
518
- ?? DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.retrievalCases;
519
- const pairCases = normalizePairFixtures(source?.pairCases)
520
- ?? DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.pairCases;
521
- const minRetrievalTop1Acc = Number.isFinite(source?.minRetrievalTop1Acc)
522
- ? Math.max(0, Math.min(1, Number(source.minRetrievalTop1Acc)))
523
- : DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.minRetrievalTop1Acc;
524
- const minPairAcc = Number.isFinite(source?.minPairAcc)
525
- ? Math.max(0, Math.min(1, Number(source.minPairAcc)))
526
- : DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.minPairAcc;
527
- const pairMargin = Number.isFinite(source?.pairMargin)
528
- ? Number(source.pairMargin)
529
- : DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.pairMargin;
530
-
531
- const lengthStabilityCases = normalizeLengthStabilityCases(source?.lengthStabilityCases)
532
- ?? DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.lengthStabilityCases;
533
- const throughputCorpus = normalizeThroughputCorpus(source?.throughputCorpus)
534
- ?? DEFAULT_EMBEDDING_SEMANTIC_FIXTURES.throughputCorpus;
535
-
536
- return {
537
- retrievalCases,
538
- pairCases,
539
- lengthStabilityCases,
540
- throughputCorpus,
541
- minRetrievalTop1Acc,
542
- minPairAcc,
543
- pairMargin,
544
- };
545
- }
546
-
547
- function resolveEmbeddingSemanticStyle(pipeline) {
548
- const manifest = pipeline?.manifest ?? null;
549
- const style = selectRuleValue('inference', 'config', 'embeddingSemanticStyle', {
550
- modelType: String(manifest?.modelType ?? '').toLowerCase(),
551
- manifestModelType: String(
552
- manifest?.config?.model_type
553
- ?? manifest?.config?.text_config?.model_type
554
- ?? ''
555
- ).toLowerCase(),
556
- sourceCheckpointId: String(manifest?.artifactIdentity?.sourceCheckpointId ?? ''),
557
- });
558
- if (typeof style === 'string' && style.length > 0) {
559
- return style;
560
- }
561
- return 'default';
562
- }
563
-
564
- function formatEmbeddingSemanticText(text, kind, style) {
565
- if (style === 'embeddinggemma') {
566
- if (kind === 'query') {
567
- return `task: search result | query: ${text}`;
568
- }
569
- if (kind === 'document') {
570
- return `title: None | text: ${text}`;
571
- }
572
- }
573
- if (style === 'qwen3_embedding') {
574
- if (kind === 'query') {
575
- return `Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: ${text}`;
576
- }
577
- return text;
578
- }
579
- return text;
580
- }
581
-
582
- export function resolvePrompt(runtimeConfig) {
583
- const runtimePrompt = runtimeConfig?.inference?.prompt;
584
- if (typeof runtimePrompt === 'string' && runtimePrompt.trim()) {
585
- return runtimePrompt.trim();
586
- }
587
- throw new Error('Harness embedding requires explicit runtime.inference.prompt.');
588
- }
589
-
590
- function assertRerankTokenId(value, label) {
591
- const tokenId = Number(value);
592
- if (!Number.isInteger(tokenId) || tokenId < 0) {
593
- throw new Error(`Manifest rerank config requires non-negative integer ${label}.`);
594
- }
595
- return tokenId;
596
- }
597
-
598
- function assertRerankText(value, label, preserve = false) {
599
- if (typeof value !== 'string') {
600
- throw new Error(`Manifest rerank config requires non-empty ${label}.`);
601
- }
602
- const text = value.trim();
603
- if (!text) {
604
- throw new Error(`Manifest rerank config requires non-empty ${label}.`);
605
- }
606
- return preserve ? value : text;
607
- }
608
-
609
- export function resolveRerankScoringConfig(pipeline) {
610
- const config = pipeline?.manifest?.inference?.rerank;
611
- if (!isPlainObject(config)) {
612
- throw new Error('Rerank workload requires manifest.inference.rerank scoring config.');
613
- }
614
- const format = assertRerankText(config.format, 'format');
615
- if (format !== 'qwen3_yes_no_logit') {
616
- throw new Error(`Unsupported rerank scoring format "${format}".`);
617
- }
618
- const trueTokenId = assertRerankTokenId(config.trueTokenId, 'trueTokenId');
619
- const falseTokenId = assertRerankTokenId(config.falseTokenId, 'falseTokenId');
620
- if (trueTokenId === falseTokenId) {
621
- throw new Error('Manifest rerank config trueTokenId and falseTokenId must be distinct.');
622
- }
623
- const score = assertRerankText(config.score, 'score');
624
- if (!RERANK_SCORE_POLICIES.has(score)) {
625
- throw new Error(`Unsupported rerank score policy "${score}".`);
626
- }
627
- const probability = assertRerankText(config.probability, 'probability');
628
- if (probability !== 'sigmoid') {
629
- throw new Error(`Unsupported rerank probability policy "${probability}".`);
630
- }
631
- return {
632
- format,
633
- instruction: assertRerankText(config.instruction, 'instruction'),
634
- inputTemplate: assertRerankText(config.inputTemplate, 'inputTemplate', true),
635
- prefix: assertRerankText(config.prefix, 'prefix', true),
636
- suffix: assertRerankText(config.suffix, 'suffix', true),
637
- trueToken: assertRerankText(config.trueToken, 'trueToken'),
638
- trueTokenId,
639
- falseToken: assertRerankText(config.falseToken, 'falseToken'),
640
- falseTokenId,
641
- score,
642
- probability,
643
- };
644
- }
645
-
646
- function replaceRerankTemplate(template, values) {
647
- let output = template;
648
- for (const [key, value] of Object.entries(values)) {
649
- const placeholder = `{${key}}`;
650
- if (!output.includes(placeholder)) {
651
- throw new Error(`Manifest rerank inputTemplate is missing ${placeholder}.`);
652
- }
653
- output = output.split(placeholder).join(value);
654
- }
655
- return output;
656
- }
657
-
658
- export function formatRerankPrompt(query, document, scoringConfig) {
659
- const instruction = assertRerankText(scoringConfig?.instruction, 'instruction');
660
- const normalizedQuery = assertRerankText(query, 'query');
661
- const normalizedDocument = assertRerankText(document, 'document');
662
- const input = replaceRerankTemplate(
663
- assertRerankText(scoringConfig?.inputTemplate, 'inputTemplate', true),
664
- {
665
- instruction,
666
- query: normalizedQuery,
667
- document: normalizedDocument,
668
- }
669
- );
670
- return `${assertRerankText(scoringConfig?.prefix, 'prefix', true)}${input}${assertRerankText(scoringConfig?.suffix, 'suffix', true)}`;
671
- }
672
-
673
- function sigmoid(value) {
674
- return 1 / (1 + Math.exp(-value));
675
- }
676
-
677
- function computeRerankScore(scoringConfig, trueLogit, falseLogit) {
678
- if (scoringConfig.score === 'logit_difference') {
679
- return trueLogit - falseLogit;
680
- }
681
- if (scoringConfig.score === 'true_logit') {
682
- return trueLogit;
683
- }
684
- throw new Error(`Unsupported rerank score policy "${scoringConfig.score}".`);
685
- }
686
-
687
- function assertLogitsVector(value) {
688
- if (!ArrayBuffer.isView(value) && !Array.isArray(value)) {
689
- throw new Error('Rerank prefillWithLogits result must include a logits vector.');
690
- }
691
- return value;
692
- }
693
-
694
- function buildRerankScoreRecord(query, document, prompt, tokenCount, trueLogit, falseLogit, config, scoringPath, phase = null) {
695
- if (!Number.isFinite(trueLogit) || !Number.isFinite(falseLogit)) {
696
- throw new Error(
697
- `Rerank logits missing finite yes/no scores at token IDs ${config.trueTokenId}/${config.falseTokenId}.`
698
- );
699
- }
700
- const score = computeRerankScore(config, trueLogit, falseLogit);
701
- const probability = sigmoid(score);
702
- return {
703
- query,
704
- document,
705
- prompt,
706
- tokenCount,
707
- score,
708
- probability,
709
- trueLogit,
710
- falseLogit,
711
- trueTokenId: config.trueTokenId,
712
- falseTokenId: config.falseTokenId,
713
- scoringPath,
714
- phase,
715
- };
716
- }
717
-
718
- export async function scoreRerankDocument(pipeline, query, document, scoringConfig = null, options = {}) {
719
- if (!pipeline || (typeof pipeline.prefillWithTokenLogits !== 'function' && typeof pipeline.prefillWithLogits !== 'function')) {
720
- throw new Error('Rerank workload requires pipeline.prefillWithTokenLogits() or pipeline.prefillWithLogits().');
721
- }
722
- const config = scoringConfig ?? resolveRerankScoringConfig(pipeline);
723
- const prompt = formatRerankPrompt(query, document, config);
724
- pipeline.reset?.();
725
- let trueLogit;
726
- let falseLogit;
727
- let tokenCount = 0;
728
- let scoringPath = 'full-logits';
729
- let resultPhase = null;
730
- const totalStart = performance.now();
731
- if (typeof pipeline.prefillWithTokenLogits === 'function') {
732
- const prefillCallStart = performance.now();
733
- const result = await pipeline.prefillWithTokenLogits(
734
- prompt,
735
- [config.trueTokenId, config.falseTokenId],
736
- {
737
- useChatTemplate: false,
738
- benchmark: options.benchmark === true,
739
- }
740
- );
741
- const prefillCallMs = performance.now() - prefillCallStart;
742
- const logits = assertLogitsVector(result?.logits);
743
- trueLogit = Number(logits[0]);
744
- falseLogit = Number(logits[1]);
745
- tokenCount = Number.isFinite(result?.tokens?.length) ? result.tokens.length : 0;
746
- scoringPath = 'selected-token-logits';
747
- resultPhase = {
748
- ...(isPlainObject(result?.phase) ? result.phase : {}),
749
- prefillCallMs,
750
- };
751
- } else {
752
- const prefillCallStart = performance.now();
753
- const result = await pipeline.prefillWithLogits(prompt, {
754
- useChatTemplate: false,
755
- benchmark: options.benchmark === true,
756
- });
757
- const prefillCallMs = performance.now() - prefillCallStart;
758
- const logits = assertLogitsVector(result?.logits);
759
- trueLogit = Number(logits[config.trueTokenId]);
760
- falseLogit = Number(logits[config.falseTokenId]);
761
- tokenCount = Number.isFinite(result?.tokens?.length) ? result.tokens.length : 0;
762
- resultPhase = {
763
- ...(isPlainObject(result?.phase) ? result.phase : {}),
764
- prefillCallMs,
765
- };
766
- }
767
- return buildRerankScoreRecord(
768
- query,
769
- document,
770
- prompt,
771
- tokenCount,
772
- trueLogit,
773
- falseLogit,
774
- config,
775
- scoringPath,
776
- {
777
- ...(resultPhase ?? {}),
778
- totalMs: performance.now() - totalStart,
779
- promptChars: prompt.length,
780
- }
781
- );
782
- }
783
-
784
- function resolveRerankInput(runtimeConfig, runOverrides = null) {
785
- const source = isPlainObject(runOverrides?.rerank)
786
- ? runOverrides.rerank
787
- : runtimeConfig?.inference?.rerank;
788
- if (!isPlainObject(source)) {
789
- throw new Error('Harness rerank requires explicit runtime.inference.rerank.');
790
- }
791
- const query = asText(source.query);
792
- if (!query) {
793
- throw new Error('Harness rerank requires non-empty runtime.inference.rerank.query.');
794
- }
795
- const documents = Array.isArray(source.documents)
796
- ? source.documents.map(asText).filter(Boolean)
797
- : [];
798
- if (documents.length === 0) {
799
- throw new Error('Harness rerank requires non-empty runtime.inference.rerank.documents.');
800
- }
801
- return { query, documents };
802
- }
803
-
804
- function summarizeRerankScores(scores) {
805
- const sorted = [...scores].sort((a, b) => {
806
- const scoreDelta = b.score - a.score;
807
- return scoreDelta !== 0 ? scoreDelta : a.index - b.index;
808
- });
809
- return {
810
- ranking: sorted.map((entry, rank) => ({
811
- rank: rank + 1,
812
- index: entry.index,
813
- document: entry.document,
814
- score: Number(entry.score.toFixed(6)),
815
- probability: Number(entry.probability.toFixed(6)),
816
- trueLogit: Number(entry.trueLogit.toFixed(6)),
817
- falseLogit: Number(entry.falseLogit.toFixed(6)),
818
- tokenCount: entry.tokenCount,
819
- scoringPath: entry.scoringPath,
820
- })),
821
- top: sorted[0] ?? null,
822
- };
823
- }
824
-
825
- function getPipelineTokenizer(pipeline) {
826
- const tokenizer = pipeline?.tokenizer;
827
- return tokenizer && typeof tokenizer.encode === 'function' ? tokenizer : null;
828
- }
829
-
830
- function longestCommonTokenPrefixLength(rows) {
831
- if (!Array.isArray(rows) || rows.length === 0) {
832
- return 0;
833
- }
834
- const first = rows[0];
835
- let length = Array.isArray(first) ? first.length : 0;
836
- for (let rowIndex = 1; rowIndex < rows.length; rowIndex += 1) {
837
- const row = rows[rowIndex];
838
- if (!Array.isArray(row)) {
839
- return 0;
840
- }
841
- length = Math.min(length, row.length);
842
- for (let index = 0; index < length; index += 1) {
843
- if (row[index] !== first[index]) {
844
- length = index;
845
- break;
846
- }
847
- }
848
- }
849
- return length;
850
- }
851
-
852
- async function createRerankPrefixContext(pipeline, query, documents, config, options = {}) {
853
- const totalStart = performance.now();
854
- if (
855
- typeof pipeline?.prefillKVOnly !== 'function'
856
- || typeof pipeline?.prefillWithTokenLogits !== 'function'
857
- || typeof pipeline?.resetToSeqLen !== 'function'
858
- ) {
859
- return null;
860
- }
861
- const tokenizer = getPipelineTokenizer(pipeline);
862
- if (!tokenizer) {
863
- return null;
864
- }
865
- const prompts = documents.map((document) => formatRerankPrompt(query, document, config));
866
- const tokenRows = prompts.map((prompt, index) => normalizeTokenIdArray(
867
- tokenizer.encode(prompt),
868
- `rerank.prompt[${index}].tokenIds`
869
- ));
870
- const prefixLength = longestCommonTokenPrefixLength(tokenRows);
871
- if (prefixLength <= 0 || tokenRows.some((row) => row.length <= prefixLength)) {
872
- return null;
873
- }
874
- const prefixTokens = tokenRows[0].slice(0, prefixLength);
875
- pipeline.reset?.();
876
- const prefillStart = performance.now();
877
- const prefix = await pipeline.prefillKVOnly('', {
878
- useChatTemplate: false,
879
- inputIds: prefixTokens,
880
- benchmark: options.benchmark === true,
881
- });
882
- const prefillMs = performance.now() - prefillStart;
883
- return {
884
- prefix,
885
- seqLen: prefix.seqLen,
886
- prefixTokens,
887
- prompts,
888
- tokenRows,
889
- suffixRows: tokenRows.map((row) => row.slice(prefixLength)),
890
- phase: {
891
- totalMs: performance.now() - totalStart,
892
- prefillMs,
893
- prefixTokens: prefixTokens.length,
894
- documentCount: documents.length,
895
- },
896
- };
897
- }
898
-
899
- async function scoreRerankDocumentFromPrefix(pipeline, query, document, config, prefixContext, index, options = {}) {
900
- const suffixTokens = prefixContext.suffixRows[index];
901
- const totalStart = performance.now();
902
- try {
903
- const prefillCallStart = performance.now();
904
- const result = await pipeline.prefillWithTokenLogits(
905
- '',
906
- [config.trueTokenId, config.falseTokenId],
907
- {
908
- useChatTemplate: false,
909
- inputIds: suffixTokens,
910
- benchmark: options.benchmark === true,
911
- }
912
- );
913
- const prefillCallMs = performance.now() - prefillCallStart;
914
- const logits = assertLogitsVector(result?.logits);
915
- const trueLogit = Number(logits[0]);
916
- const falseLogit = Number(logits[1]);
917
- return buildRerankScoreRecord(
918
- query,
919
- document,
920
- prefixContext.prompts[index],
921
- prefixContext.prefixTokens.length + suffixTokens.length,
922
- trueLogit,
923
- falseLogit,
924
- config,
925
- 'prefix-selected-token-logits',
926
- {
927
- ...(isPlainObject(result?.phase) ? result.phase : {}),
928
- totalMs: performance.now() - totalStart,
929
- prefillCallMs,
930
- suffixTokens: suffixTokens.length,
931
- }
932
- );
933
- } finally {
934
- pipeline.resetToSeqLen(prefixContext.seqLen);
935
- }
936
- }
937
-
938
- export async function runRerank(pipeline, runtimeConfig, runOverrides = null) {
939
- const input = resolveRerankInput(runtimeConfig, runOverrides);
940
- const config = resolveRerankScoringConfig(pipeline);
941
- const start = performance.now();
942
- const scores = [];
943
- const prefixStart = performance.now();
944
- const prefixContext = await createRerankPrefixContext(pipeline, input.query, input.documents, config, {
945
- benchmark: runOverrides?.benchmark === true,
946
- });
947
- const prefixMs = performance.now() - prefixStart;
948
- for (let i = 0; i < input.documents.length; i++) {
949
- const scored = prefixContext
950
- ? await scoreRerankDocumentFromPrefix(
951
- pipeline,
952
- input.query,
953
- input.documents[i],
954
- config,
955
- prefixContext,
956
- i,
957
- {
958
- benchmark: runOverrides?.benchmark === true,
959
- }
960
- )
961
- : await scoreRerankDocument(
962
- pipeline,
963
- input.query,
964
- input.documents[i],
965
- config,
966
- {
967
- benchmark: runOverrides?.benchmark === true,
968
- }
969
- );
970
- scores.push({
971
- index: i,
972
- ...scored,
973
- });
974
- }
975
- const summary = summarizeRerankScores(scores);
976
- const durationMs = Math.max(1, performance.now() - start);
977
- const documentDurations = scores
978
- .map((entry) => Number(entry.phase?.totalMs))
979
- .filter((value) => Number.isFinite(value));
980
- const documentTotalMs = documentDurations.reduce((sum, value) => sum + value, 0);
981
- return {
982
- query: input.query,
983
- documents: input.documents,
984
- documentCount: input.documents.length,
985
- scores,
986
- ranking: summary.ranking,
987
- topDocument: summary.top
988
- ? {
989
- index: summary.top.index,
990
- document: summary.top.document,
991
- score: summary.top.score,
992
- probability: summary.top.probability,
993
- }
994
- : null,
995
- phase: {
996
- totalMs: durationMs,
997
- prefixMs,
998
- prefixApplied: prefixContext != null,
999
- prefixTokens: Number.isFinite(prefixContext?.prefixTokens?.length)
1000
- ? prefixContext.prefixTokens.length
1001
- : 0,
1002
- prefix: prefixContext?.phase ?? null,
1003
- documentCount: input.documents.length,
1004
- documentTotalMs,
1005
- maxDocumentMs: documentDurations.length > 0 ? Math.max(...documentDurations) : 0,
1006
- avgDocumentMs: documentDurations.length > 0 ? documentTotalMs / documentDurations.length : 0,
1007
- documents: scores.map((entry) => ({
1008
- index: entry.index,
1009
- scoringPath: entry.scoringPath,
1010
- tokenCount: entry.tokenCount,
1011
- phase: entry.phase ?? null,
1012
- })),
1013
- },
1014
- durationMs,
1015
- };
1016
- }
1017
-
1018
- function resolveRerankSemanticFixtures(runtimeConfig, options = null) {
1019
- const overrides = isPlainObject(options?.rerankSemantic)
1020
- ? options.rerankSemantic
1021
- : null;
1022
- const runtimeOverrides = runtimeConfig?.shared?.benchmark?.run?.rerankSemantic;
1023
- const source = overrides ?? (isPlainObject(runtimeOverrides) ? runtimeOverrides : null);
1024
- const cases = normalizeRerankCases(source?.cases)
1025
- ?? DEFAULT_RERANK_SEMANTIC_FIXTURES.cases;
1026
- const minPairAcc = Number.isFinite(source?.minPairAcc)
1027
- ? Math.max(0, Math.min(1, Number(source.minPairAcc)))
1028
- : DEFAULT_RERANK_SEMANTIC_FIXTURES.minPairAcc;
1029
- const minScoreMargin = Number.isFinite(source?.minScoreMargin)
1030
- ? Number(source.minScoreMargin)
1031
- : DEFAULT_RERANK_SEMANTIC_FIXTURES.minScoreMargin;
1032
- return {
1033
- cases,
1034
- minPairAcc,
1035
- minScoreMargin,
1036
- };
1037
- }
1038
-
1039
- export async function runRerankSemanticChecks(pipeline, options = null) {
1040
- const fixture = resolveRerankSemanticFixtures(pipeline?.runtimeConfig ?? {}, options);
1041
- const config = resolveRerankScoringConfig(pipeline);
1042
- const start = performance.now();
1043
- const pairs = [];
1044
- let pairPassed = 0;
1045
- for (const testCase of fixture.cases) {
1046
- const positive = await scoreRerankDocument(
1047
- pipeline,
1048
- testCase.query,
1049
- testCase.positive,
1050
- config
1051
- );
1052
- const negative = await scoreRerankDocument(
1053
- pipeline,
1054
- testCase.query,
1055
- testCase.negative,
1056
- config
1057
- );
1058
- const margin = positive.score - negative.score;
1059
- const passed = Number.isFinite(margin) && margin > fixture.minScoreMargin;
1060
- if (passed) pairPassed++;
1061
- pairs.push({
1062
- id: testCase.id,
1063
- query: testCase.query,
1064
- positive: testCase.positive,
1065
- negative: testCase.negative,
1066
- passed,
1067
- positiveScore: Number(positive.score.toFixed(6)),
1068
- negativeScore: Number(negative.score.toFixed(6)),
1069
- positiveProbability: Number(positive.probability.toFixed(6)),
1070
- negativeProbability: Number(negative.probability.toFixed(6)),
1071
- margin: Number.isFinite(margin) ? Number(margin.toFixed(6)) : null,
1072
- });
1073
- }
1074
- const pairAcc = pairs.length > 0 ? pairPassed / pairs.length : 0;
1075
- const passed = pairAcc >= fixture.minPairAcc;
1076
- return {
1077
- passed,
1078
- pairAcc,
1079
- pairPassed,
1080
- pairTotal: pairs.length,
1081
- minPairAcc: Number(fixture.minPairAcc.toFixed(4)),
1082
- minScoreMargin: Number(fixture.minScoreMargin.toFixed(4)),
1083
- failedCaseIds: pairs.filter((item) => !item.passed).map((item) => item.id),
1084
- pairs,
1085
- durationMs: Math.max(1, performance.now() - start),
1086
- };
1087
- }
1088
-
1089
- function isStructuredPromptInput(value) {
1090
- return Array.isArray(value) || (value != null && typeof value === 'object');
1091
- }
1092
-
1093
- function clonePromptInput(promptInput) {
1094
- if (!isStructuredPromptInput(promptInput)) {
1095
- return promptInput;
1096
- }
1097
- if (typeof structuredClone === 'function') {
1098
- return structuredClone(promptInput);
1099
- }
1100
- return JSON.parse(JSON.stringify(promptInput));
1101
- }
1102
-
1103
- function resolvePromptTemplateType(source) {
1104
- const sourceTemplateType = asText(source?.chatTemplateType);
1105
- if (sourceTemplateType) {
1106
- return sourceTemplateType;
1107
- }
1108
- const modelConfigTemplateType = asText(source?.modelConfig?.chatTemplateType);
1109
- if (modelConfigTemplateType) {
1110
- return modelConfigTemplateType;
1111
- }
1112
- return asText(source?.manifest?.inference?.chatTemplate?.type);
1113
- }
1114
-
1115
- function assertPromptContract(runtimePrompt, templateType, source = 'runtime.inference.prompt') {
1116
- if (templateType !== 'translategemma') {
1117
- return;
1118
- }
1119
- if (runtimePrompt === undefined || runtimePrompt === null) {
1120
- return;
1121
- }
1122
- if (typeof runtimePrompt === 'string') {
1123
- throw new Error(
1124
- `TranslateGemma harness prompt contract violation: ${source} must be ` +
1125
- '{ messages: [...] } with source_lang_code/target_lang_code blocks, not a plain string.'
1126
- );
1127
- }
1128
- if (!isStructuredPromptInput(runtimePrompt)) {
1129
- throw new Error(
1130
- `TranslateGemma harness prompt contract violation: ${source} must be ` +
1131
- '{ messages: [...] } with source_lang_code/target_lang_code blocks.'
1132
- );
1133
- }
1134
- }
1135
-
1136
- function describePromptInput(promptInput) {
1137
- if (typeof promptInput === 'string') {
1138
- return promptInput.trim() || '[empty prompt]';
1139
- }
1140
- if (isPlainObject(promptInput?.image) && typeof promptInput?.prompt === 'string') {
1141
- const width = Number.isFinite(promptInput.image.width) ? promptInput.image.width : '?';
1142
- const height = Number.isFinite(promptInput.image.height) ? promptInput.image.height : '?';
1143
- const source = asText(promptInput.image.source) ?? 'image';
1144
- return `${source} ${width}x${height}: ${promptInput.prompt}`;
1145
- }
1146
- const firstMessage = Array.isArray(promptInput?.messages)
1147
- ? promptInput.messages[0]
1148
- : null;
1149
- const firstContent = Array.isArray(firstMessage?.content)
1150
- ? firstMessage.content[0]
1151
- : null;
1152
- const sourceLang = asText(firstContent?.source_lang_code);
1153
- const targetLang = asText(firstContent?.target_lang_code);
1154
- const text = asText(firstContent?.text);
1155
- if (sourceLang && targetLang) {
1156
- return `${sourceLang} -> ${targetLang}: ${text || '[non-text request]'}`;
1157
- }
1158
- const stringContent = asText(firstMessage?.content);
1159
- if (stringContent) {
1160
- const role = asText(firstMessage?.role) || 'user';
1161
- return `${role}: ${stringContent}`;
1162
- }
1163
- try {
1164
- return JSON.stringify(promptInput);
1165
- } catch {
1166
- return '[structured prompt]';
1167
- }
1168
- }
1169
-
1170
- function decodeBase64ToBytes(base64, label) {
1171
- const normalized = asText(base64);
1172
- if (!normalized) {
1173
- throw new Error(`${label} must be a non-empty base64 string.`);
1174
- }
1175
- if (typeof Buffer !== 'undefined') {
1176
- return new Uint8Array(Buffer.from(normalized, 'base64'));
1177
- }
1178
- if (typeof atob === 'function') {
1179
- const binary = atob(normalized);
1180
- const bytes = new Uint8Array(binary.length);
1181
- for (let i = 0; i < binary.length; i++) {
1182
- bytes[i] = binary.charCodeAt(i);
1183
- }
1184
- return bytes;
1185
- }
1186
- throw new Error(`${label} requires a base64 decoder in this runtime.`);
1187
- }
1188
-
1189
- function assertRawImageByteLength(bytes, width, height, label) {
1190
- const expectedRgb = width * height * 3;
1191
- const expectedRgba = width * height * 4;
1192
- if (bytes.length !== expectedRgb && bytes.length !== expectedRgba) {
1193
- throw new Error(
1194
- `${label} must contain width*height*3 or width*height*4 bytes. ` +
1195
- `Got ${bytes.length} for ${width}x${height}.`
1196
- );
1197
- }
1198
- }
1199
-
1200
- function normalizeRawImageBytes(value, width, height, label) {
1201
- let bytes = null;
1202
- if (ArrayBuffer.isView(value)) {
1203
- bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
1204
- } else if (Array.isArray(value)) {
1205
- const normalized = new Uint8Array(value.length);
1206
- for (let i = 0; i < value.length; i++) {
1207
- const parsed = Number(value[i]);
1208
- if (!Number.isInteger(parsed) || parsed < 0 || parsed > 255) {
1209
- throw new Error(`${label}[${i}] must be an integer in [0, 255].`);
1210
- }
1211
- normalized[i] = parsed;
1212
- }
1213
- bytes = normalized;
1214
- }
1215
- if (!(bytes instanceof Uint8Array)) {
1216
- throw new Error(`${label} must be an array or typed array.`);
1217
- }
1218
- assertRawImageByteLength(bytes, width, height, label);
1219
- return new Uint8Array(bytes);
1220
- }
1221
-
1222
- function createCanvasForImageDecode(width, height) {
1223
- if (typeof OffscreenCanvas === 'function') {
1224
- return new OffscreenCanvas(width, height);
1225
- }
1226
- if (typeof document !== 'undefined' && typeof document.createElement === 'function') {
1227
- const canvas = document.createElement('canvas');
1228
- canvas.width = width;
1229
- canvas.height = height;
1230
- return canvas;
1231
- }
1232
- return null;
1233
- }
1234
-
1235
- function isNodeRuntime() {
1236
- return typeof process !== 'undefined' && !!process.versions?.node;
1237
- }
1238
-
1239
- async function decodeImageUrlToPixelsOnNode(url) {
1240
- let sharpModule = null;
1241
- try {
1242
- sharpModule = await import('sharp');
1243
- } catch (error) {
1244
- throw new Error(
1245
- `URL-backed inferenceInput.image.url on the node surface requires the optional "sharp" decoder. ${error?.message || error}`
1246
- );
1247
- }
1248
- const sharp = typeof sharpModule?.default === 'function'
1249
- ? sharpModule.default
1250
- : sharpModule;
1251
- if (typeof sharp !== 'function') {
1252
- throw new Error('Node image decode requires sharp to export a callable default.');
1253
- }
1254
-
1255
- const response = await fetch(url);
1256
- if (!response.ok) {
1257
- throw new Error(`image fetch failed: HTTP ${response.status}`);
1258
- }
1259
- const sourceBytes = new Uint8Array(await response.arrayBuffer());
1260
- const decoded = await sharp(sourceBytes)
1261
- .ensureAlpha()
1262
- .raw()
1263
- .toBuffer({ resolveWithObject: true });
1264
- return {
1265
- imageBytes: new Uint8Array(decoded.data),
1266
- width: decoded.info.width,
1267
- height: decoded.info.height,
1268
- };
1269
- }
1270
-
1271
- async function decodeImageUrlToPixels(url) {
1272
- if (isNodeRuntime()) {
1273
- return decodeImageUrlToPixelsOnNode(url);
1274
- }
1275
- const response = await fetch(url);
1276
- if (!response.ok) {
1277
- throw new Error(`image fetch failed: HTTP ${response.status}`);
1278
- }
1279
- if (typeof createImageBitmap !== 'function') {
1280
- throw new Error(
1281
- 'URL-backed inferenceInput.image.url requires createImageBitmap support. ' +
1282
- 'Use raw pixels or run on a browser-capable surface.'
1283
- );
1284
- }
1285
-
1286
- const imageBlob = await response.blob();
1287
- const imageBitmap = await createImageBitmap(imageBlob);
1288
- try {
1289
- const canvas = createCanvasForImageDecode(imageBitmap.width, imageBitmap.height);
1290
- if (!canvas) {
1291
- throw new Error(
1292
- 'URL-backed inferenceInput.image.url requires OffscreenCanvas or a DOM canvas in this runtime. ' +
1293
- 'Use raw pixels or run on a browser-capable surface.'
1294
- );
1295
- }
1296
- const context = canvas.getContext('2d', { willReadFrequently: true });
1297
- if (!context || typeof context.getImageData !== 'function') {
1298
- throw new Error('Image decode canvas did not provide a readable 2D context.');
1299
- }
1300
- context.drawImage(imageBitmap, 0, 0);
1301
- const imageData = context.getImageData(0, 0, imageBitmap.width, imageBitmap.height);
1302
- return {
1303
- imageBytes: new Uint8Array(imageData.data),
1304
- width: imageBitmap.width,
1305
- height: imageBitmap.height,
1306
- };
1307
- } finally {
1308
- imageBitmap.close?.();
1309
- }
1310
- }
1311
-
1312
- async function resolveInferenceImagePayload(imageInput) {
1313
- if (!isPlainObject(imageInput)) {
1314
- throw new Error('inference image input must be an object.');
1315
- }
1316
-
1317
- if (typeof imageInput.url === 'string' && imageInput.url.trim()) {
1318
- const decoded = await decodeImageUrlToPixels(imageInput.url.trim());
1319
- return {
1320
- imageBytes: decoded.imageBytes,
1321
- width: decoded.width,
1322
- height: decoded.height,
1323
- descriptor: {
1324
- source: 'url',
1325
- width: decoded.width,
1326
- height: decoded.height,
1327
- url: imageInput.url.trim(),
1328
- },
1329
- };
1330
- }
1331
-
1332
- const width = Math.max(1, Math.floor(Number(imageInput.width)));
1333
- const height = Math.max(1, Math.floor(Number(imageInput.height)));
1334
- if (!Number.isFinite(width) || !Number.isFinite(height)) {
1335
- throw new Error('Raw inference image input requires positive integer width and height.');
1336
- }
1337
-
1338
- if (typeof imageInput.pixelDataBase64 === 'string' && imageInput.pixelDataBase64.trim()) {
1339
- const decodedBytes = decodeBase64ToBytes(
1340
- imageInput.pixelDataBase64,
1341
- 'inferenceInput.image.pixelDataBase64'
1342
- );
1343
- assertRawImageByteLength(
1344
- decodedBytes,
1345
- width,
1346
- height,
1347
- 'inferenceInput.image.pixelDataBase64'
1348
- );
1349
- return {
1350
- imageBytes: decodedBytes,
1351
- width,
1352
- height,
1353
- descriptor: {
1354
- source: 'pixelDataBase64',
1355
- width,
1356
- height,
1357
- },
1358
- };
1359
- }
1360
-
1361
- return {
1362
- imageBytes: normalizeRawImageBytes(
1363
- imageInput.pixels,
1364
- width,
1365
- height,
1366
- 'inferenceInput.image.pixels'
1367
- ),
1368
- width,
1369
- height,
1370
- descriptor: {
1371
- source: 'pixels',
1372
- width,
1373
- height,
1374
- },
1375
- };
1376
- }
1377
-
1378
- function resolveGenerationPromptInput(runtimeConfig, runOverrides = null, source = null) {
1379
- const templateType = resolvePromptTemplateType(source);
1380
- const overridePrompt = runOverrides?.prompt;
1381
- assertPromptContract(overridePrompt, templateType, 'runOverrides.prompt');
1382
- if (typeof overridePrompt === 'string' && overridePrompt.trim()) {
1383
- return overridePrompt;
1384
- }
1385
- if (isStructuredPromptInput(overridePrompt)) {
1386
- return clonePromptInput(overridePrompt);
1387
- }
1388
-
1389
- const runtimePrompt = runtimeConfig?.inference?.prompt;
1390
- assertPromptContract(runtimePrompt, templateType, 'runtimeConfig.inference.prompt');
1391
- if (typeof runtimePrompt === 'string' && runtimePrompt.trim()) {
1392
- return runtimePrompt;
1393
- }
1394
- if (isStructuredPromptInput(runtimePrompt)) {
1395
- return clonePromptInput(runtimePrompt);
1396
- }
1397
-
1398
- throw new Error('Harness generation requires explicit runOverrides.prompt or runtime.inference.prompt.');
1399
- }
1400
-
1401
- function resolveMaxTokens(runtimeConfig) {
1402
- const runtimeMax = runtimeConfig?.inference?.generation?.maxTokens;
1403
- if (Number.isFinite(runtimeMax) && runtimeMax > 0) {
1404
- return Math.floor(runtimeMax);
1405
- }
1406
- throw new Error('Harness generation requires explicit runtime.inference.generation.maxTokens.');
1407
- }
1408
-
1409
- function resolveAutomaticGenerationDiagnostics(runtimeConfig, runOverrides = null) {
1410
- const overrideDiagnostics = runOverrides?.diagnostics ?? null;
1411
- if (overrideDiagnostics?.enabled === true) {
1412
- return overrideDiagnostics;
1413
- }
1414
-
1415
- const diagnosticsPolicy = runtimeConfig?.shared?.tooling?.diagnostics ?? 'off';
1416
- if (diagnosticsPolicy !== 'always') {
1417
- return overrideDiagnostics;
1418
- }
1419
-
1420
- return {
1421
- enabled: true,
1422
- captureConfig: {
1423
- ...createDefaultCaptureConfig(),
1424
- enabled: true,
1425
- defaultLevel: CAPTURE_LEVELS.NONE,
1426
- },
1427
- };
1428
- }
1429
-
1430
- export function resolveBenchmarkRunSettings(runtimeConfig, source = null) {
1431
- const benchConfig = runtimeConfig?.shared?.benchmark?.run || {};
1432
- const runtimeSampling = isPlainObject(runtimeConfig?.inference?.sampling)
1433
- ? runtimeConfig.inference.sampling
1434
- : {};
1435
- const benchSampling = isPlainObject(benchConfig?.sampling)
1436
- ? benchConfig.sampling
1437
- : {};
1438
- const runSeed = Number.isFinite(benchConfig.seed)
1439
- ? Math.max(0, Math.floor(benchConfig.seed))
1440
- : null;
1441
- const runtimeSeed = Number.isFinite(runtimeSampling.seed)
1442
- ? Math.max(0, Math.floor(runtimeSampling.seed))
1443
- : null;
1444
- const benchSeed = Number.isFinite(benchSampling.seed)
1445
- ? Math.max(0, Math.floor(benchSampling.seed))
1446
- : null;
1447
- const mergedSeed = runSeed != null
1448
- ? runSeed
1449
- : benchSeed != null
1450
- ? benchSeed
1451
- : runtimeSeed;
1452
- const promptInput = typeof benchConfig.customPrompt === 'string' && benchConfig.customPrompt.trim()
1453
- ? benchConfig.customPrompt
1454
- : resolveGenerationPromptInput(runtimeConfig, null, source);
1455
- const maxTokens = Number.isFinite(benchConfig.maxNewTokens)
1456
- ? Math.max(1, Math.floor(benchConfig.maxNewTokens))
1457
- : resolveMaxTokens(runtimeConfig);
1458
- const sampling = {
1459
- ...runtimeSampling,
1460
- ...benchSampling,
1461
- };
1462
- if (Number.isFinite(mergedSeed)) {
1463
- sampling.seed = mergedSeed;
1464
- }
1465
-
1466
- return {
1467
- warmupRuns: Math.max(0, Math.floor(benchConfig.warmupRuns ?? 0)),
1468
- timedRuns: Math.max(1, Math.floor(benchConfig.timedRuns ?? 1)),
1469
- ...(Number.isFinite(mergedSeed) ? { seed: mergedSeed } : {}),
1470
- prompt: promptInput,
1471
- promptLabel: describePromptInput(promptInput),
1472
- maxTokens,
1473
- sampling,
1474
- };
1475
- }
1476
-
1477
- function summarizeEmbeddingValues(embedding) {
1478
- const values = ArrayBuffer.isView(embedding) || Array.isArray(embedding) ? embedding : null;
1479
- const embeddingDim = Number.isFinite(values?.length) ? values.length : 0;
1480
- const preview = [];
1481
-
1482
- let nonFiniteCount = 0;
1483
- let finiteCount = 0;
1484
- let min = Infinity;
1485
- let max = -Infinity;
1486
- let maxAbs = 0;
1487
- let sum = 0;
1488
- let sumSq = 0;
1489
-
1490
- for (let i = 0; i < embeddingDim; i++) {
1491
- const value = Number(values[i]);
1492
- if (preview.length < EMBEDDING_PREVIEW_LENGTH) {
1493
- preview.push(Number.isFinite(value) ? Number(value.toFixed(6)) : null);
1494
- }
1495
- if (!Number.isFinite(value)) {
1496
- nonFiniteCount++;
1497
- continue;
1498
- }
1499
- finiteCount++;
1500
- if (value < min) min = value;
1501
- if (value > max) max = value;
1502
- const abs = Math.abs(value);
1503
- if (abs > maxAbs) maxAbs = abs;
1504
- sum += value;
1505
- sumSq += value * value;
1506
- }
1507
-
1508
- const mean = finiteCount > 0 ? (sum / finiteCount) : null;
1509
- const variance = finiteCount > 0 ? Math.max(0, (sumSq / finiteCount) - ((mean || 0) * (mean || 0))) : null;
1510
- const stdDev = variance == null ? null : Math.sqrt(variance);
1511
- const l2Norm = finiteCount > 0 ? Math.sqrt(sumSq) : null;
1512
- const finiteRatio = embeddingDim > 0 ? finiteCount / embeddingDim : 0;
1513
-
1514
- return {
1515
- embeddingDim,
1516
- nonFiniteCount,
1517
- finiteCount,
1518
- finiteRatio,
1519
- min: finiteCount > 0 ? min : null,
1520
- max: finiteCount > 0 ? max : null,
1521
- maxAbs: finiteCount > 0 ? maxAbs : null,
1522
- mean,
1523
- stdDev,
1524
- l2Norm,
1525
- preview,
1526
- };
1527
- }
1528
-
1529
- function cosineSimilarity(a, b) {
1530
- if (!a || !b || !Number.isFinite(a.length) || !Number.isFinite(b.length)) return NaN;
1531
- if (a.length !== b.length || a.length === 0) return NaN;
1532
- let dot = 0;
1533
- let normA = 0;
1534
- let normB = 0;
1535
- for (let i = 0; i < a.length; i++) {
1536
- const av = Number(a[i]);
1537
- const bv = Number(b[i]);
1538
- if (!Number.isFinite(av) || !Number.isFinite(bv)) return NaN;
1539
- dot += av * bv;
1540
- normA += av * av;
1541
- normB += bv * bv;
1542
- }
1543
- if (normA <= 0 || normB <= 0) return NaN;
1544
- return dot / Math.sqrt(normA * normB);
1545
- }
1546
-
1547
- function top1Index(values) {
1548
- let best = -1;
1549
- let bestValue = -Infinity;
1550
- for (let i = 0; i < values.length; i++) {
1551
- const value = Number(values[i]);
1552
- if (!Number.isFinite(value)) continue;
1553
- if (value > bestValue) {
1554
- bestValue = value;
1555
- best = i;
1556
- }
1557
- }
1558
- return best;
1559
- }
1560
-
1561
- async function embedStandaloneText(pipeline, text) {
1562
- pipeline.reset?.();
1563
- const result = await pipeline.embed(text);
1564
- const embedding = result?.embedding;
1565
- if (!embedding || !Number.isFinite(embedding.length) || embedding.length <= 0) {
1566
- throw new Error('Semantic check embedding is missing.');
1567
- }
1568
- return embedding;
1569
- }
1570
-
1571
- export async function runEmbeddingSemanticChecks(pipeline, options = null) {
1572
- const config = resolveEmbeddingSemanticFixtures(
1573
- pipeline?.runtimeConfig ?? {},
1574
- options
1575
- );
1576
- const start = performance.now();
1577
- const semanticStyle = resolveEmbeddingSemanticStyle(pipeline);
1578
- const retrieval = [];
1579
- let retrievalPassed = 0;
1580
-
1581
- for (const testCase of config.retrievalCases) {
1582
- const formattedQuery = formatEmbeddingSemanticText(testCase.query, 'query', semanticStyle);
1583
- const queryEmbedding = await embedStandaloneText(
1584
- pipeline,
1585
- formattedQuery
1586
- );
1587
- const docEmbeddings = [];
1588
- const docs = [];
1589
- for (const doc of testCase.docs) {
1590
- const formattedDoc = formatEmbeddingSemanticText(doc, 'document', semanticStyle);
1591
- docEmbeddings.push(await embedStandaloneText(
1592
- pipeline,
1593
- formattedDoc
1594
- ));
1595
- docs.push({
1596
- text: doc,
1597
- formattedText: formattedDoc,
1598
- });
1599
- }
1600
- const sims = docEmbeddings.map((docEmbedding) => cosineSimilarity(queryEmbedding, docEmbedding));
1601
- const topDoc = top1Index(sims);
1602
- const passed = topDoc === testCase.expectedDoc;
1603
- if (passed) retrievalPassed++;
1604
- retrieval.push({
1605
- id: testCase.id,
1606
- query: testCase.query,
1607
- formattedQuery,
1608
- docs,
1609
- passed,
1610
- expectedDoc: testCase.expectedDoc,
1611
- topDoc,
1612
- sims: sims.map((v) => (Number.isFinite(v) ? Number(v.toFixed(6)) : null)),
1613
- });
1614
- }
1615
-
1616
- const pairs = [];
1617
- let pairPassed = 0;
1618
- for (const testCase of config.pairCases) {
1619
- const formattedAnchor = formatEmbeddingSemanticText(testCase.anchor, 'query', semanticStyle);
1620
- const anchor = await embedStandaloneText(
1621
- pipeline,
1622
- formattedAnchor
1623
- );
1624
- const formattedPositive = formatEmbeddingSemanticText(testCase.positive, 'query', semanticStyle);
1625
- const positive = await embedStandaloneText(
1626
- pipeline,
1627
- formattedPositive
1628
- );
1629
- const formattedNegative = formatEmbeddingSemanticText(testCase.negative, 'query', semanticStyle);
1630
- const negative = await embedStandaloneText(
1631
- pipeline,
1632
- formattedNegative
1633
- );
1634
- const simPos = cosineSimilarity(anchor, positive);
1635
- const simNeg = cosineSimilarity(anchor, negative);
1636
- const margin = simPos - simNeg;
1637
- const passed = Number.isFinite(margin) && margin > config.pairMargin;
1638
- if (passed) pairPassed++;
1639
- pairs.push({
1640
- id: testCase.id,
1641
- anchor: testCase.anchor,
1642
- formattedAnchor,
1643
- positive: testCase.positive,
1644
- formattedPositive,
1645
- negative: testCase.negative,
1646
- formattedNegative,
1647
- passed,
1648
- simPos: Number.isFinite(simPos) ? Number(simPos.toFixed(6)) : null,
1649
- simNeg: Number.isFinite(simNeg) ? Number(simNeg.toFixed(6)) : null,
1650
- margin: Number.isFinite(margin) ? Number(margin.toFixed(6)) : null,
1651
- });
1652
- }
1653
-
1654
- const lengthStability = [];
1655
- let lengthStabilityPassed = 0;
1656
- for (const testCase of config.lengthStabilityCases) {
1657
- const shortEmb = await embedStandaloneText(
1658
- pipeline,
1659
- formatEmbeddingSemanticText(testCase.short, 'document', semanticStyle)
1660
- );
1661
- const mediumEmb = await embedStandaloneText(
1662
- pipeline,
1663
- formatEmbeddingSemanticText(testCase.medium, 'document', semanticStyle)
1664
- );
1665
- const longEmb = await embedStandaloneText(
1666
- pipeline,
1667
- formatEmbeddingSemanticText(testCase.long, 'document', semanticStyle)
1668
- );
1669
- const simShortMedium = cosineSimilarity(shortEmb, mediumEmb);
1670
- const simShortLong = cosineSimilarity(shortEmb, longEmb);
1671
- const simMediumLong = cosineSimilarity(mediumEmb, longEmb);
1672
- const minSim = Math.min(
1673
- Number.isFinite(simShortMedium) ? simShortMedium : -1,
1674
- Number.isFinite(simShortLong) ? simShortLong : -1,
1675
- Number.isFinite(simMediumLong) ? simMediumLong : -1
1676
- );
1677
- const maxDrift = 1 - minSim;
1678
- const passed = Number.isFinite(maxDrift) && maxDrift <= testCase.maxCosineDrift;
1679
- if (passed) lengthStabilityPassed++;
1680
- lengthStability.push({
1681
- id: testCase.id,
1682
- passed,
1683
- simShortMedium: Number.isFinite(simShortMedium) ? Number(simShortMedium.toFixed(6)) : null,
1684
- simShortLong: Number.isFinite(simShortLong) ? Number(simShortLong.toFixed(6)) : null,
1685
- simMediumLong: Number.isFinite(simMediumLong) ? Number(simMediumLong.toFixed(6)) : null,
1686
- maxDrift: Number.isFinite(maxDrift) ? Number(maxDrift.toFixed(6)) : null,
1687
- maxCosineDrift: testCase.maxCosineDrift,
1688
- });
1689
- }
1690
-
1691
- let throughput = null;
1692
- if (config.throughputCorpus.length > 0) {
1693
- const corpusStart = performance.now();
1694
- for (const text of config.throughputCorpus) {
1695
- await embedStandaloneText(
1696
- pipeline,
1697
- formatEmbeddingSemanticText(text, 'document', semanticStyle)
1698
- );
1699
- }
1700
- const corpusDurationMs = Math.max(1, performance.now() - corpusStart);
1701
- throughput = {
1702
- corpusSize: config.throughputCorpus.length,
1703
- durationMs: Number(corpusDurationMs.toFixed(1)),
1704
- docsPerSecond: Number((config.throughputCorpus.length / (corpusDurationMs / 1000)).toFixed(2)),
1705
- };
1706
- }
1707
-
1708
- const retrievalTop1Acc = retrieval.length > 0 ? retrievalPassed / retrieval.length : 0;
1709
- const pairAcc = pairs.length > 0 ? pairPassed / pairs.length : 0;
1710
- const lengthStabilityAcc = lengthStability.length > 0
1711
- ? lengthStabilityPassed / lengthStability.length : 1;
1712
- const passed = retrievalTop1Acc >= config.minRetrievalTop1Acc
1713
- && pairAcc >= config.minPairAcc;
1714
- const failedCaseIds = [
1715
- ...retrieval.filter((item) => !item.passed).map((item) => `retrieval:${item.id}`),
1716
- ...pairs.filter((item) => !item.passed).map((item) => `pair:${item.id}`),
1717
- ...lengthStability.filter((item) => !item.passed).map((item) => `length:${item.id}`),
1718
- ];
1719
-
1720
- return {
1721
- passed,
1722
- style: semanticStyle,
1723
- retrievalTop1Acc,
1724
- pairAcc,
1725
- lengthStabilityAcc,
1726
- retrievalPassed,
1727
- retrievalTotal: retrieval.length,
1728
- pairPassed,
1729
- pairTotal: pairs.length,
1730
- lengthStabilityPassed,
1731
- lengthStabilityTotal: lengthStability.length,
1732
- minRetrievalTop1Acc: Number(config.minRetrievalTop1Acc.toFixed(4)),
1733
- minPairAcc: Number(config.minPairAcc.toFixed(4)),
1734
- pairMarginThreshold: Number(config.pairMargin.toFixed(4)),
1735
- failedCaseIds,
1736
- retrieval,
1737
- pairs,
1738
- lengthStability,
1739
- throughput,
1740
- durationMs: Math.max(1, performance.now() - start),
1741
- };
1742
- }
1743
-
1744
- const SPECIAL_TOKEN_RE = /^(<pad>|<unused\d*>|<eos>|<bos>|<s>|<\/s>|\[PAD\]|\[UNK\]|\[SEP\]|\[CLS\]|<[^>]{1,32}>)$/i;
1745
- const PAD_DOMINANCE_THRESHOLD = 0.5;
1746
-
1747
- function isSpecialLikeTokenText(value) {
1748
- if (typeof value !== 'string') return false;
1749
- return SPECIAL_TOKEN_RE.test(value.trim());
1750
- }
1751
-
1752
- function summarizeGenerationTokens(tokenRecords) {
1753
- const records = Array.isArray(tokenRecords) ? tokenRecords : [];
1754
- const preview = records.slice(0, GENERATION_TOKEN_DIAGNOSTIC_LIMIT).map((record) => ({
1755
- id: record.id,
1756
- text: record.text,
1757
- fallbackText: record.fallbackText,
1758
- }));
1759
- let emptyTextCount = 0;
1760
- let specialLikeTextCount = 0;
1761
- let specialLikeFallbackCount = 0;
1762
- for (const record of records) {
1763
- if (!record || typeof record !== 'object') continue;
1764
- if (typeof record.text === 'string' && record.text.length === 0) {
1765
- emptyTextCount += 1;
1766
- }
1767
- if (isSpecialLikeTokenText(record.text)) {
1768
- specialLikeTextCount += 1;
1769
- }
1770
- if (isSpecialLikeTokenText(record.fallbackText)) {
1771
- specialLikeFallbackCount += 1;
1772
- }
1773
- }
1774
- return {
1775
- preview,
1776
- total: records.length,
1777
- omitted: Math.max(0, records.length - preview.length),
1778
- emptyTextCount,
1779
- specialLikeTextCount,
1780
- specialLikeFallbackCount,
1781
- };
1782
- }
1783
-
1784
- function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1785
- const stats = typeof pipeline?.getStats === 'function'
1786
- ? (pipeline.getStats() || {})
1787
- : {};
1788
- const memoryStats = typeof pipeline?.getMemoryStats === 'function'
1789
- ? (pipeline.getMemoryStats() || {})
1790
- : {};
1791
- const kvMemory = memoryStats.kvCache && typeof memoryStats.kvCache === 'object'
1792
- ? memoryStats.kvCache
1793
- : null;
1794
- const prefillMs = Number.isFinite(stats.prefillTimeMs) ? stats.prefillTimeMs : 0;
1795
- const ttftMs = Number.isFinite(stats.ttftMs) ? stats.ttftMs : prefillMs;
1796
- const decodeMs = Number.isFinite(stats.decodeTimeMs) ? stats.decodeTimeMs : 0;
1797
- const prefillTokens = Number.isFinite(stats.prefillTokens) ? stats.prefillTokens : 0;
1798
- const decodeTokens = Number.isFinite(stats.decodeTokens)
1799
- ? stats.decodeTokens
1800
- : Math.max(0, tokenCount - 1);
1801
- const decodeTokensPerSec = decodeMs > 0
1802
- ? (decodeTokens / decodeMs) * 1000
1803
- : 0;
1804
- const prefillTokensPerSec = prefillMs > 0
1805
- ? (prefillTokens / prefillMs) * 1000
1806
- : 0;
1807
- const prefillTokensPerSecTtft = ttftMs > 0
1808
- ? (prefillTokens / ttftMs) * 1000
1809
- : 0;
1810
- const gpu = {};
1811
- if (Number.isFinite(stats.gpuTimePrefillMs)) gpu.prefillMs = stats.gpuTimePrefillMs;
1812
- if (Number.isFinite(stats.gpuTimeDecodeMs)) gpu.decodeMs = stats.gpuTimeDecodeMs;
1813
- if (Number.isFinite(stats.decodeRecordMs)) gpu.decodeRecordMs = stats.decodeRecordMs;
1814
- if (Number.isFinite(stats.decodeRecordOps)) gpu.decodeRecordOps = stats.decodeRecordOps;
1815
- if (Number.isFinite(stats.decodeRecordPasses)) gpu.decodeRecordPasses = stats.decodeRecordPasses;
1816
- const decodeRecordOpLabels = normalizeDecodeRecordOpLabels(stats.decodeRecordOpLabels);
1817
- if (decodeRecordOpLabels) {
1818
- const decodeRecordTopOps = buildDecodeRecordTopOps(decodeRecordOpLabels, stats.decodeRecordOps);
1819
- const decodeRecordTopOpGroups = buildDecodeRecordTopOpGroups(decodeRecordOpLabels, stats.decodeRecordOps);
1820
- gpu.decodeRecordOpLabels = decodeRecordOpLabels;
1821
- gpu.decodeRecordUniqueOpLabels = Object.keys(decodeRecordOpLabels).length;
1822
- gpu.decodeRecordTopOps = decodeRecordTopOps;
1823
- gpu.decodeRecordTopOpGroups = decodeRecordTopOpGroups;
1824
- }
1825
- if (
1826
- Number.isFinite(stats.decodeRecordMs) &&
1827
- Number.isFinite(stats.decodeRecordOps) &&
1828
- stats.decodeRecordOps > 0
1829
- ) {
1830
- gpu.decodeRecordMsPerOp = stats.decodeRecordMs / stats.decodeRecordOps;
1831
- }
1832
- if (
1833
- Number.isFinite(stats.decodeRecordMs) &&
1834
- Number.isFinite(stats.decodeRecordPasses) &&
1835
- stats.decodeRecordPasses > 0
1836
- ) {
1837
- gpu.decodeRecordMsPerPass = stats.decodeRecordMs / stats.decodeRecordPasses;
1838
- }
1839
- if (
1840
- Number.isFinite(stats.decodeRecordOps) &&
1841
- Number.isFinite(stats.decodeRecordPasses) &&
1842
- stats.decodeRecordOps > 0
1843
- ) {
1844
- gpu.decodeRecordPassesPerOp = stats.decodeRecordPasses / stats.decodeRecordOps;
1845
- }
1846
- if (
1847
- Number.isFinite(stats.decodeRecordPasses) &&
1848
- Number.isFinite(stats.batching?.executedBatchTokens) &&
1849
- stats.batching.executedBatchTokens > 0
1850
- ) {
1851
- gpu.decodeRecordPassesPerExecutedBatchToken = stats.decodeRecordPasses / stats.batching.executedBatchTokens;
1852
- }
1853
- if (
1854
- Number.isFinite(stats.decodeRecordMs) &&
1855
- Number.isFinite(stats.batching?.executedBatchTokens) &&
1856
- stats.batching.executedBatchTokens > 0
1857
- ) {
1858
- gpu.decodeRecordMsPerExecutedBatchToken = stats.decodeRecordMs / stats.batching.executedBatchTokens;
1859
- }
1860
- if (
1861
- Number.isFinite(stats.decodeRecordOps) &&
1862
- Number.isFinite(stats.batching?.executedBatchTokens) &&
1863
- stats.batching.executedBatchTokens > 0
1864
- ) {
1865
- gpu.decodeRecordOpsPerExecutedBatchToken = stats.decodeRecordOps / stats.batching.executedBatchTokens;
1866
- }
1867
- const uniformCacheStats = normalizeUniformCacheStats(stats.uniformCache);
1868
- if (uniformCacheStats) {
1869
- gpu.uniformCache = uniformCacheStats;
1870
- }
1871
- if (Number.isFinite(stats.decodeSubmitWaitMs)) gpu.decodeSubmitWaitMs = stats.decodeSubmitWaitMs;
1872
- if (Number.isFinite(stats.decodeReadbackWaitMs)) gpu.decodeReadbackWaitMs = stats.decodeReadbackWaitMs;
1873
- if (Number.isFinite(stats.decodeReadbackMapWaitMs)) gpu.decodeReadbackMapWaitMs = stats.decodeReadbackMapWaitMs;
1874
- if (Number.isFinite(stats.decodeReadbackCleanupMs)) gpu.decodeReadbackCleanupMs = stats.decodeReadbackCleanupMs;
1875
- if (Number.isFinite(stats.decodeReadbackCopyMs)) gpu.decodeReadbackCopyMs = stats.decodeReadbackCopyMs;
1876
- if (Number.isFinite(stats.prefillRecordMs)) gpu.prefillRecordMs = stats.prefillRecordMs;
1877
- if (Number.isFinite(stats.prefillRecordOps)) gpu.prefillRecordOps = stats.prefillRecordOps;
1878
- if (Number.isFinite(stats.prefillRecordPasses)) gpu.prefillRecordPasses = stats.prefillRecordPasses;
1879
- const prefillRecordOpLabels = normalizeDecodeRecordOpLabels(stats.prefillRecordOpLabels);
1880
- if (prefillRecordOpLabels) {
1881
- const prefillRecordTopOps = buildDecodeRecordTopOps(prefillRecordOpLabels, stats.prefillRecordOps);
1882
- const prefillRecordTopOpGroups = buildDecodeRecordTopOpGroups(prefillRecordOpLabels, stats.prefillRecordOps);
1883
- gpu.prefillRecordOpLabels = prefillRecordOpLabels;
1884
- gpu.prefillRecordUniqueOpLabels = Object.keys(prefillRecordOpLabels).length;
1885
- gpu.prefillRecordTopOps = prefillRecordTopOps;
1886
- gpu.prefillRecordTopOpGroups = prefillRecordTopOpGroups;
1887
- }
1888
- if (Number.isFinite(stats.prefillSubmitWaitMs)) gpu.prefillSubmitWaitMs = stats.prefillSubmitWaitMs;
1889
- if (
1890
- Number.isFinite(decodeMs) &&
1891
- Number.isFinite(stats.decodeRecordMs) &&
1892
- Number.isFinite(stats.decodeSubmitWaitMs) &&
1893
- Number.isFinite(stats.decodeReadbackWaitMs)
1894
- ) {
1895
- const decodeGpuWaitMs = Math.max(stats.decodeSubmitWaitMs, stats.decodeReadbackWaitMs);
1896
- gpu.decodeOrchestrationMs = decodeMs - stats.decodeRecordMs - decodeGpuWaitMs;
1897
- }
1898
- if (Number.isFinite(stats.singleTokenSubmitWaitMs)) gpu.singleTokenSubmitWaitMs = stats.singleTokenSubmitWaitMs;
1899
- if (Number.isFinite(stats.singleTokenReadbackWaitMs)) gpu.singleTokenReadbackWaitMs = stats.singleTokenReadbackWaitMs;
1900
- if (Number.isFinite(stats.singleTokenReadbackMapWaitMs)) gpu.singleTokenReadbackMapWaitMs = stats.singleTokenReadbackMapWaitMs;
1901
- if (Number.isFinite(stats.singleTokenReadbackCleanupMs)) gpu.singleTokenReadbackCleanupMs = stats.singleTokenReadbackCleanupMs;
1902
- if (Number.isFinite(stats.singleTokenReadbackCopyMs)) gpu.singleTokenReadbackCopyMs = stats.singleTokenReadbackCopyMs;
1903
- if (Number.isFinite(stats.singleTokenOrchestrationMs)) gpu.singleTokenOrchestrationMs = stats.singleTokenOrchestrationMs;
1904
- const gpuPhase = Object.keys(gpu).length > 0 ? gpu : null;
1905
- const batching = {};
1906
- if (Number.isFinite(stats.batching?.batchedForwardCalls)) {
1907
- batching.batchedForwardCalls = stats.batching.batchedForwardCalls;
1908
- }
1909
- if (Number.isFinite(stats.batching?.unbatchedForwardCalls)) {
1910
- batching.unbatchedForwardCalls = stats.batching.unbatchedForwardCalls;
1911
- }
1912
- if (Number.isFinite(stats.batching?.totalBatchedTimeMs)) {
1913
- batching.totalBatchedTimeMs = stats.batching.totalBatchedTimeMs;
1914
- }
1915
- if (Number.isFinite(stats.batching?.totalUnbatchedTimeMs)) {
1916
- batching.totalUnbatchedTimeMs = stats.batching.totalUnbatchedTimeMs;
1917
- }
1918
- if (Number.isFinite(stats.batching?.gpuSubmissions)) {
1919
- batching.gpuSubmissions = stats.batching.gpuSubmissions;
1920
- }
1921
- if (Number.isFinite(stats.batching?.requestedBatchTokens)) {
1922
- batching.requestedBatchTokens = stats.batching.requestedBatchTokens;
1923
- }
1924
- if (Number.isFinite(stats.batching?.effectiveBatchTokens)) {
1925
- batching.effectiveBatchTokens = stats.batching.effectiveBatchTokens;
1926
- }
1927
- if (Number.isFinite(stats.batching?.executedBatchTokens)) {
1928
- batching.executedBatchTokens = stats.batching.executedBatchTokens;
1929
- }
1930
- if (Number.isFinite(stats.batching?.resolvedBatchTokens)) {
1931
- batching.resolvedBatchTokens = stats.batching.resolvedBatchTokens;
1932
- }
1933
- if (Number.isFinite(stats.batching?.maxBatchTokenCap)) {
1934
- batching.maxBatchTokenCap = stats.batching.maxBatchTokenCap;
1935
- }
1936
- if (Number.isFinite(stats.batching?.batchClampCount)) {
1937
- batching.batchClampCount = stats.batching.batchClampCount;
1938
- }
1939
- const batchingPhase = Object.keys(batching).length > 0 ? batching : null;
1940
- const plePreparedTokenCache = {};
1941
- if (Number.isFinite(stats.plePreparedTokenCacheHits)) {
1942
- plePreparedTokenCache.hits = stats.plePreparedTokenCacheHits;
1943
- }
1944
- if (Number.isFinite(stats.plePreparedTokenCacheMisses)) {
1945
- plePreparedTokenCache.misses = stats.plePreparedTokenCacheMisses;
1946
- }
1947
- if (Number.isFinite(stats.plePreparedTokenCacheEntries)) {
1948
- plePreparedTokenCache.entries = stats.plePreparedTokenCacheEntries;
1949
- }
1950
- if (Number.isFinite(stats.plePreparedTokenCacheBytes)) {
1951
- plePreparedTokenCache.bytes = stats.plePreparedTokenCacheBytes;
1952
- }
1953
- if (Number.isFinite(stats.pleWriteBufferCount)) {
1954
- plePreparedTokenCache.writeBufferCount = stats.pleWriteBufferCount;
1955
- }
1956
- if (Number.isFinite(stats.pleWriteBufferBytes)) {
1957
- plePreparedTokenCache.writeBufferBytes = stats.pleWriteBufferBytes;
1958
- }
1959
- const plePreparedTokenCachePhase = Object.keys(plePreparedTokenCache).length > 0
1960
- ? plePreparedTokenCache
1961
- : null;
1962
- const wallMs = Number.isFinite(stats.totalTimeMs) ? stats.totalTimeMs : durationMs;
1963
-
1964
- return {
1965
- phase: {
1966
- totalMs: prefillMs + decodeMs,
1967
- wallMs,
1968
- ttftMs,
1969
- prefillMs,
1970
- decodeMs,
1971
- prefillTokens,
1972
- decodeTokens,
1973
- prefillTokensPerSec,
1974
- prefillTokensPerSecTtft,
1975
- decodeTokensPerSec,
1976
- gpu: gpuPhase,
1977
- prefillProfileSteps: Array.isArray(stats.prefillProfileSteps)
1978
- ? stats.prefillProfileSteps
1979
- : null,
1980
- decodeProfileSteps: Array.isArray(stats.decodeProfileSteps)
1981
- ? stats.decodeProfileSteps
1982
- : null,
1983
- decodeMode: stats.decodeMode ?? null,
1984
- batchGuardReason: stats.batchGuardReason ?? null,
1985
- stopReason: stats.stopReason ?? null,
1986
- stopTokenId: Number.isInteger(stats.stopTokenId) ? stats.stopTokenId : null,
1987
- batching: batchingPhase,
1988
- plePreparedTokenCache: plePreparedTokenCachePhase,
1989
- kvCache: kvMemory
1990
- ? {
1991
- layout: kvMemory.layout ?? null,
1992
- kvDtype: kvMemory.kvDtype ?? null,
1993
- seqLen: Number.isFinite(kvMemory.seqLen) ? kvMemory.seqLen : null,
1994
- maxSeqLen: Number.isFinite(kvMemory.maxSeqLen) ? kvMemory.maxSeqLen : null,
1995
- usedBytes: Number.isFinite(kvMemory.used) ? kvMemory.used : null,
1996
- allocatedBytes: Number.isFinite(kvMemory.allocated) ? kvMemory.allocated : null,
1997
- counters: kvMemory.counters ?? null,
1998
- }
1999
- : null,
2000
- executionPlan: stats.executionPlan ?? null,
2001
- kernelPathId: stats.kernelPathId ?? null,
2002
- operatorDiagnostics: stats.operatorDiagnostics ?? null,
2003
- kernelPathSource: stats.kernelPathSource ?? null,
2004
- },
2005
- };
2006
- }
2007
-
2008
- export function isCoherentOutput(tokens, output) {
2009
- if (tokens.length === 0) return false;
2010
- const specialTokenCount = tokens.filter((t) => SPECIAL_TOKEN_RE.test(String(t).trim())).length;
2011
- if (specialTokenCount / tokens.length >= PAD_DOMINANCE_THRESHOLD) return false;
2012
- const cleanedOutput = String(output || '')
2013
- .replace(/<[^>\n]{1,80}>/g, ' ')
2014
- .replace(/\s+/g, ' ')
2015
- .trim();
2016
- return cleanedOutput.length > 0;
2017
- }
2018
-
2019
- export async function runGeneration(pipeline, runtimeConfig, runOverrides = null) {
2020
- const tokens = [];
2021
- const tokenIds = [];
2022
- const tokenRecords = [];
2023
- const logitsDigests = [];
2024
- const promptInput = resolveGenerationPromptInput(runtimeConfig, runOverrides, pipeline);
2025
- const promptLabel = describePromptInput(promptInput);
2026
- const useChatTemplate = resolveGenerationUseChatTemplate(pipeline, runtimeConfig, runOverrides, promptInput);
2027
- const promptTokenIds = resolvePromptTokenIdsForTranscript(pipeline, promptInput, useChatTemplate);
2028
- const maxTokens = Number.isFinite(runOverrides?.maxTokens)
2029
- ? Math.max(1, Math.floor(runOverrides.maxTokens))
2030
- : resolveMaxTokens(runtimeConfig);
2031
- const sampling = {
2032
- ...(runtimeConfig.inference?.sampling || {}),
2033
- ...(isPlainObject(runOverrides?.sampling) ? runOverrides.sampling : {}),
2034
- };
2035
- const seed = Number.isFinite(runOverrides?.seed)
2036
- ? Math.max(0, Math.floor(runOverrides.seed))
2037
- : Number.isFinite(sampling.seed)
2038
- ? Math.max(0, Math.floor(sampling.seed))
2039
- : null;
2040
- const debugProbes = runtimeConfig.shared?.debug?.probes || [];
2041
- const executionObserverEnabled = isExecutionObservationRequested(runtimeConfig);
2042
- const profile = executionObserverEnabled;
2043
- const explicitDiagnosticsEnabled = runtimeConfig.shared?.harness?.mode === 'diagnose'
2044
- || shouldEnableReferenceTranscriptDiagnostics(runOverrides, runtimeConfig);
2045
- const disableCommandBatchingForDiagnostics = explicitDiagnosticsEnabled
2046
- || (Array.isArray(debugProbes) && debugProbes.length > 0);
2047
- const start = performance.now();
2048
- const diagnostics = resolveAutomaticGenerationDiagnostics(runtimeConfig, runOverrides);
2049
- const captureLogits = shouldCaptureReferenceLogits(runOverrides, runtimeConfig);
2050
-
2051
- for await (const tokenText of pipeline.generate(promptInput, {
2052
- maxTokens,
2053
- ...(Number.isFinite(seed) ? { seed } : {}),
2054
- temperature: sampling.temperature,
2055
- topP: sampling.topP,
2056
- topK: sampling.topK,
2057
- repetitionPenalty: sampling.repetitionPenalty,
2058
- greedyThreshold: sampling.greedyThreshold,
2059
- useChatTemplate,
2060
- benchmark: runOverrides?.benchmark === true,
2061
- profile,
2062
- executionObserver: executionObserverEnabled,
2063
- ...(disableCommandBatchingForDiagnostics ? { disableCommandBatching: true } : {}),
2064
- diagnostics,
2065
- ...(captureLogits ? {
2066
- onLogits: (logits, context) => {
2067
- logitsDigests.push(digestLogitsForTranscript(logits, {
2068
- ...context,
2069
- index: logitsDigests.length,
2070
- decodeToken: (tokenId) => pipeline?.tokenizer?.decode?.([tokenId], false, false) ?? null,
2071
- }));
2072
- },
2073
- } : {}),
2074
- onToken: (tokenId, tokenText) => {
2075
- tokenIds.push(tokenId);
2076
- tokenRecords.push({
2077
- id: tokenId,
2078
- text: typeof tokenText === 'string' ? tokenText : '',
2079
- fallbackText: pipeline?.tokenizer?.decode?.([tokenId], false, false) ?? '',
2080
- });
2081
- },
2082
- })) {
2083
- if (typeof tokenText === 'string') {
2084
- tokens.push(tokenText);
2085
- }
2086
- }
2087
-
2088
- const durationMs = Math.max(1, performance.now() - start);
2089
- const tokensPerSec = (tokens.length / durationMs) * 1000;
2090
- const { phase } = buildGenerationPhaseFromStats(pipeline, durationMs, tokenIds.length);
2091
- const kvCacheByteProof = await captureKvCacheByteProof(
2092
- pipeline,
2093
- shouldCaptureReferenceKvBytes(runOverrides, runtimeConfig)
2094
- );
2095
-
2096
- return {
2097
- ...(Number.isFinite(seed) ? { seed } : {}),
2098
- prompt: promptLabel,
2099
- promptInput,
2100
- promptTokenIds,
2101
- maxTokens,
2102
- tokens,
2103
- tokenIds,
2104
- tokenDiagnostics: summarizeGenerationTokens(tokenRecords),
2105
- logitsDigests,
2106
- kvCacheByteProof,
2107
- output: tokens.join(''),
2108
- durationMs,
2109
- tokensPerSec,
2110
- phase,
2111
- };
2112
- }
2113
-
2114
- export async function runImageTranscription(pipeline, runtimeConfig, runOverrides = null) {
2115
- const imageInput = runOverrides?.image;
2116
- if (!isPlainObject(imageInput)) {
2117
- throw new Error('Image transcription requires inferenceInput.image.');
2118
- }
2119
- const prompt = typeof runOverrides?.prompt === 'string' && runOverrides.prompt.trim()
2120
- ? runOverrides.prompt.trim()
2121
- : DEFAULT_IMAGE_TRANSCRIPTION_PROMPT;
2122
- const maxTokens = Number.isFinite(runOverrides?.maxTokens)
2123
- ? Math.max(1, Math.floor(runOverrides.maxTokens))
2124
- : resolveMaxTokens(runtimeConfig);
2125
- const softTokenBudget = Number.isFinite(runOverrides?.softTokenBudget)
2126
- ? Math.max(1, Math.floor(runOverrides.softTokenBudget))
2127
- : DEFAULT_IMAGE_TRANSCRIPTION_SOFT_TOKEN_BUDGET;
2128
- const {
2129
- imageBytes,
2130
- width,
2131
- height,
2132
- descriptor,
2133
- } = await resolveInferenceImagePayload(imageInput);
2134
- const start = performance.now();
2135
- const result = await pipeline.transcribeImage({
2136
- imageBytes,
2137
- width,
2138
- height,
2139
- prompt,
2140
- maxTokens,
2141
- softTokenBudget,
2142
- });
2143
- const durationMs = Math.max(1, performance.now() - start);
2144
- const tokenIds = Array.isArray(result?.tokens)
2145
- ? result.tokens.map((value) => Number(value)).filter((value) => Number.isInteger(value))
2146
- : [];
2147
- const tokenRecords = tokenIds.map((tokenId) => {
2148
- const decoded = pipeline?.tokenizer?.decode?.([tokenId], false, false) ?? '';
2149
- return {
2150
- id: tokenId,
2151
- text: decoded,
2152
- fallbackText: decoded,
2153
- };
2154
- });
2155
- const { phase } = buildGenerationPhaseFromStats(pipeline, durationMs, tokenIds.length);
2156
- return {
2157
- inputMode: 'image_to_text',
2158
- prompt: `image ${width}x${height}: ${prompt}`,
2159
- promptInput: {
2160
- prompt,
2161
- image: descriptor,
2162
- },
2163
- maxTokens,
2164
- tokens: tokenRecords.map((record) => record.text),
2165
- tokenIds,
2166
- tokenDiagnostics: summarizeGenerationTokens(tokenRecords),
2167
- output: typeof result?.text === 'string' ? result.text : tokenRecords.map((record) => record.text).join(''),
2168
- durationMs,
2169
- tokensPerSec: tokenIds.length > 0 ? (tokenIds.length / durationMs) * 1000 : 0,
2170
- phase,
2171
- };
2172
- }
2173
-
2174
- export async function runTextInference(pipeline, runtimeConfig, runOverrides = null) {
2175
- if (isPlainObject(runOverrides?.image)) {
2176
- return runImageTranscription(pipeline, runtimeConfig, runOverrides);
2177
- }
2178
- return runGeneration(pipeline, runtimeConfig, runOverrides);
2179
- }
2180
-
2181
- export async function runEmbedding(pipeline, runtimeConfig, runOverrides = null) {
2182
- const prompt = typeof runOverrides?.prompt === 'string' && runOverrides.prompt.trim()
2183
- ? runOverrides.prompt.trim()
2184
- : resolvePrompt(runtimeConfig);
2185
- const start = performance.now();
2186
- const result = await pipeline.embed(prompt, {
2187
- benchmark: runOverrides?.benchmark === true,
2188
- });
2189
- const durationMs = Math.max(1, performance.now() - start);
2190
- const tokenCount = Number.isFinite(result?.tokens?.length) ? result.tokens.length : 0;
2191
- const stats = summarizeEmbeddingValues(result?.embedding);
2192
- return {
2193
- prompt,
2194
- tokenCount,
2195
- durationMs,
2196
- phase: result?.phase ?? null,
2197
- ...stats,
2198
- };
2199
- }
1
+ export {
2
+ formatRerankPrompt,
3
+ resolveRerankScoringConfig,
4
+ scoreRerankDocument,
5
+ } from './rerank.js';
6
+ export {
7
+ getDefaultEmbeddingSemanticFixtures,
8
+ getDefaultRerankSemanticFixtures,
9
+ resolveBenchmarkRunSettings,
10
+ resolvePrompt,
11
+ } from './browser-harness/text-input.js';
12
+ export {
13
+ buildDecodeRecordTopOpGroups,
14
+ buildDecodeRecordTopOps,
15
+ captureKvCacheByteProof,
16
+ digestLogitsForTranscript,
17
+ groupDecodeRecordOpLabels,
18
+ isCoherentOutput,
19
+ normalizeDecodeRecordOpLabels,
20
+ normalizeUniformCacheStats,
21
+ } from './browser-harness/text-evidence.js';
22
+ export {
23
+ runEmbedding,
24
+ runEmbeddingSemanticChecks,
25
+ runGeneration,
26
+ runImageTranscription,
27
+ runRerank,
28
+ runRerankSemanticChecks,
29
+ runSequenceEncoding,
30
+ runTextInference,
31
+ } from './browser-harness/text-execution.js';