doppler-gpu 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (607) hide show
  1. package/CHANGELOG.md +29 -1
  2. package/README.md +97 -29
  3. package/package.json +40 -27
  4. package/src/cli/config/doppler-quickstart.json +10 -0
  5. package/src/cli/doppler-cli.js +66 -22
  6. package/src/cli/doppler-quickstart.d.ts +61 -0
  7. package/src/cli/doppler-quickstart.js +299 -0
  8. package/src/cli/doppler-serve.js +395 -0
  9. package/src/client/doppler-api.browser.js +13 -377
  10. package/src/client/doppler-api.d.ts +35 -56
  11. package/src/client/doppler-api.js +12 -363
  12. package/src/client/doppler-registry.json +55 -2
  13. package/src/client/runtime/index.d.ts +63 -0
  14. package/src/client/runtime/index.js +173 -0
  15. package/src/client/runtime/lora.d.ts +36 -0
  16. package/src/client/runtime/lora.js +165 -0
  17. package/src/client/{doppler-provider → runtime}/model-manager.d.ts +1 -1
  18. package/src/client/{doppler-provider → runtime}/model-manager.js +56 -30
  19. package/src/client/runtime/model-session.d.ts +72 -0
  20. package/src/client/runtime/model-session.js +122 -0
  21. package/src/client/runtime/model-source.d.ts +43 -0
  22. package/src/client/runtime/model-source.js +101 -0
  23. package/src/client/{doppler-provider → runtime}/source-runtime.d.ts +8 -5
  24. package/src/client/runtime/source-runtime.js +919 -0
  25. package/src/client/runtime/storage.d.ts +5 -0
  26. package/src/client/runtime/storage.js +51 -0
  27. package/src/client/{doppler-provider → runtime}/types.d.ts +14 -2
  28. package/src/config/README.md +2 -0
  29. package/src/config/conversion/README.md +47 -7
  30. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +67 -5
  31. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +62 -11
  32. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +79 -18
  33. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +61 -13
  34. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +53 -10
  35. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +66 -18
  36. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +256 -0
  37. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +49 -8
  38. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +45 -6
  39. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +45 -6
  40. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +57 -9
  41. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +105 -26
  42. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +83 -33
  43. package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +33 -5
  44. package/src/config/execution-contract-check.d.ts +2 -1
  45. package/src/config/execution-contract-check.js +187 -53
  46. package/src/config/kernel-path-loader.d.ts +36 -0
  47. package/src/config/kernel-path-loader.js +97 -10
  48. package/src/config/kernels/kernel-ref-digests.js +40 -24
  49. package/src/config/kernels/registry.js +0 -1
  50. package/src/config/kernels/registry.json +772 -66
  51. package/src/config/merge-contract-check.js +3 -0
  52. package/src/config/merge.d.ts +18 -1
  53. package/src/config/merge.js +56 -0
  54. package/src/config/param-categories.js +5 -0
  55. package/src/config/param-validator.js +42 -26
  56. package/src/config/platforms/apple-m3.json +1 -1
  57. package/src/config/platforms/loader.js +21 -16
  58. package/src/config/required-inference-fields-contract-check.js +16 -0
  59. package/src/config/runtime/compute/f16-batched.json +2 -0
  60. package/src/config/runtime/default.json +1 -0
  61. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +57 -0
  62. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +57 -0
  63. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +2 -0
  64. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +57 -0
  65. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +57 -0
  66. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +81 -0
  67. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +5 -0
  68. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +81 -0
  69. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +38 -0
  70. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +63 -0
  71. package/src/config/runtime/profiles/gemma4-e2b-throughput.json +39 -0
  72. package/src/config/runtime/profiles/low-memory.json +1 -0
  73. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +29 -0
  74. package/src/config/runtime/profiles/turboquant-contiguous.json +29 -0
  75. package/src/config/runtime/profiles/turboquant.json +36 -0
  76. package/src/config/runtime/profiles/vector-stability.json +1 -0
  77. package/src/config/runtime/tiers/gemma4-128gb.json +79 -0
  78. package/src/config/runtime/tiers/gemma4-16gb.json +15 -2
  79. package/src/config/runtime/tiers/gemma4-24gb.json +15 -2
  80. package/src/config/runtime/tiers/gemma4-32gb.json +15 -2
  81. package/src/config/runtime.js +11 -4
  82. package/src/config/schema/converter.schema.d.ts +4 -2
  83. package/src/config/schema/converter.schema.js +7 -5
  84. package/src/config/schema/doppler.schema.d.ts +26 -2
  85. package/src/config/schema/doppler.schema.js +77 -28
  86. package/src/config/schema/execution-v1.schema.d.ts +78 -5
  87. package/src/config/schema/execution-v1.schema.js +62 -1
  88. package/src/config/schema/index.d.ts +22 -29
  89. package/src/config/schema/index.js +15 -17
  90. package/src/config/schema/inference.schema.d.ts +26 -4
  91. package/src/config/schema/kernel-path.schema.d.ts +30 -3
  92. package/src/config/schema/kernel-registry.schema.d.ts +1 -1
  93. package/src/config/schema/kernel-registry.schema.js +1 -0
  94. package/src/config/schema/kvcache.schema.js +7 -0
  95. package/src/config/schema/loading.schema.d.ts +28 -0
  96. package/src/config/schema/loading.schema.js +9 -0
  97. package/src/config/schema/manifest.schema.d.ts +79 -5
  98. package/src/config/schema/manifest.schema.js +7 -1
  99. package/src/config/schema/platform.schema.d.ts +10 -8
  100. package/src/config/schema/speculation-self.schema.js +0 -5
  101. package/src/config/schema/support-tiers.schema.d.ts +53 -0
  102. package/src/config/schema/support-tiers.schema.js +128 -0
  103. package/src/config/source-packages/litert/gemma-4-e2b-it.json +622 -0
  104. package/src/config/source-packages/registry.json +15 -0
  105. package/src/config/support-tiers/subsystems.json +502 -0
  106. package/src/config/transforms/capability-transform-resolver.d.ts +16 -5
  107. package/src/config/transforms/capability-transform-resolver.js +44 -9
  108. package/src/config/transforms/execution-graph-transforms.d.ts +113 -2
  109. package/src/config/transforms/execution-graph-transforms.js +1220 -8
  110. package/src/converter/conversion-plan.js +20 -8
  111. package/src/converter/core.d.ts +48 -1
  112. package/src/converter/core.js +249 -62
  113. package/src/converter/parsers/gguf.d.ts +10 -1
  114. package/src/converter/rope-config.d.ts +1 -0
  115. package/src/converter/rope-config.js +75 -5
  116. package/src/converter/shard-packer.js +2 -2
  117. package/src/debug/tensor.js +13 -1
  118. package/src/{adapters → experimental/adapters}/adapter-manager.d.ts +1 -1
  119. package/src/{adapters → experimental/adapters}/adapter-manager.js +2 -2
  120. package/src/{adapters → experimental/adapters}/adapter-manifest.d.ts +1 -1
  121. package/src/{adapters → experimental/adapters}/adapter-manifest.js +1 -1
  122. package/src/{adapters → experimental/adapters}/adapter-registry.d.ts +1 -1
  123. package/src/{adapters → experimental/adapters}/adapter-registry.js +1 -1
  124. package/src/{adapters → experimental/adapters}/lora-loader.d.ts +1 -1
  125. package/src/{adapters → experimental/adapters}/lora-loader.js +2 -2
  126. package/src/{bridge → experimental/bridge}/extension-client.js +2 -2
  127. package/src/{bridge → experimental/bridge}/node-bridge.js +2 -4
  128. package/src/{bridge → experimental/bridge}/protocol.js +1 -1
  129. package/src/{browser → experimental/browser}/browser-converter.d.ts +4 -4
  130. package/src/{browser → experimental/browser}/browser-converter.js +12 -12
  131. package/src/{browser → experimental/browser}/gguf-importer.d.ts +1 -1
  132. package/src/{browser → experimental/browser}/gguf-importer.js +8 -8
  133. package/src/{browser → experimental/browser}/gguf-parser-browser.d.ts +2 -2
  134. package/src/{browser → experimental/browser}/gguf-parser-browser.js +3 -3
  135. package/src/{browser → experimental/browser}/quantization.d.ts +3 -3
  136. package/src/{browser → experimental/browser}/quantization.js +4 -4
  137. package/src/{browser → experimental/browser}/safetensors-parser-browser.d.ts +3 -3
  138. package/src/{browser → experimental/browser}/safetensors-parser-browser.js +3 -3
  139. package/src/{browser → experimental/browser}/shard-io-browser.d.ts +1 -1
  140. package/src/{browser → experimental/browser}/shard-io-browser.js +2 -2
  141. package/src/{browser → experimental/browser}/tensor-source-download.js +3 -3
  142. package/src/experimental/diffusion/index.d.ts +8 -0
  143. package/src/experimental/diffusion/index.js +8 -0
  144. package/src/{distribution → experimental/distribution}/p2p-transport-contract.js +1 -1
  145. package/src/{distribution → experimental/distribution}/p2p-transport.js +1 -1
  146. package/src/{distribution → experimental/distribution}/shard-delivery.js +5 -5
  147. package/src/experimental/energy/index.d.ts +2 -0
  148. package/src/experimental/energy/index.js +2 -0
  149. package/src/{hotswap → experimental/hotswap}/intent-bundle.d.ts +1 -1
  150. package/src/{hotswap → experimental/hotswap}/intent-bundle.js +2 -2
  151. package/src/{hotswap → experimental/hotswap}/manifest.d.ts +2 -2
  152. package/src/{hotswap → experimental/hotswap}/manifest.js +1 -1
  153. package/src/{hotswap → experimental/hotswap}/model-swap.js +1 -1
  154. package/src/{hotswap → experimental/hotswap}/runtime.d.ts +1 -1
  155. package/src/experimental/orchestration/index.d.ts +12 -0
  156. package/src/experimental/orchestration/index.js +12 -0
  157. package/src/{training → experimental/training}/README.md +1 -1
  158. package/src/{training → experimental/training}/artifacts.d.ts +3 -3
  159. package/src/{training → experimental/training}/artifacts.js +3 -3
  160. package/src/{training → experimental/training}/attention-backward.d.ts +1 -1
  161. package/src/{training → experimental/training}/attention-backward.js +3 -3
  162. package/src/{training → experimental/training}/attention-forward.d.ts +2 -2
  163. package/src/{training → experimental/training}/attention-forward.js +1 -1
  164. package/src/{training → experimental/training}/autograd.d.ts +2 -2
  165. package/src/{training → experimental/training}/autograd.js +7 -7
  166. package/src/{training → experimental/training}/backward-kernels.js +1 -1
  167. package/src/{training → experimental/training}/checkpoint.js +1 -1
  168. package/src/{training → experimental/training}/clip.d.ts +2 -2
  169. package/src/{training → experimental/training}/clip.js +3 -3
  170. package/src/{training → experimental/training}/export.d.ts +2 -2
  171. package/src/{training → experimental/training}/export.js +2 -2
  172. package/src/{training → experimental/training}/lora-pipeline.js +4 -4
  173. package/src/{training → experimental/training}/lora.d.ts +1 -1
  174. package/src/{training → experimental/training}/lora.js +4 -4
  175. package/src/{training → experimental/training}/loss-scaling.d.ts +2 -2
  176. package/src/{training → experimental/training}/loss-scaling.js +3 -3
  177. package/src/{training → experimental/training}/loss.d.ts +2 -2
  178. package/src/{training → experimental/training}/loss.js +1 -1
  179. package/src/{training → experimental/training}/operator-artifacts.js +1 -1
  180. package/src/{training → experimental/training}/optimizer.d.ts +2 -2
  181. package/src/{training → experimental/training}/optimizer.js +3 -3
  182. package/src/{training → experimental/training}/runner.d.ts +2 -2
  183. package/src/{training → experimental/training}/runner.js +5 -5
  184. package/src/{training → experimental/training}/suite.js +16 -16
  185. package/src/{training → experimental/training}/tensor-factory.d.ts +1 -1
  186. package/src/{training → experimental/training}/tensor-factory.js +2 -2
  187. package/src/{training → experimental/training}/trainer.d.ts +3 -3
  188. package/src/{training → experimental/training}/trainer.js +3 -3
  189. package/src/{training → experimental/training}/ul_dataset.d.ts +2 -2
  190. package/src/{training → experimental/training}/ul_dataset.js +1 -1
  191. package/src/{training → experimental/training}/ul_schedule.d.ts +1 -1
  192. package/src/{training → experimental/training}/workloads.js +4 -4
  193. package/src/formats/index.d.ts +24 -0
  194. package/src/formats/index.js +6 -0
  195. package/src/formats/litert/index.d.ts +1 -0
  196. package/src/formats/litert/index.js +1 -0
  197. package/src/formats/litert/types.d.ts +87 -0
  198. package/src/formats/litert/types.js +568 -0
  199. package/src/formats/rdrr/classification.js +16 -8
  200. package/src/formats/rdrr/parsing.d.ts +3 -2
  201. package/src/formats/rdrr/types.d.ts +33 -0
  202. package/src/formats/rdrr/validation.js +16 -5
  203. package/src/formats/tflite/index.d.ts +1 -0
  204. package/src/formats/tflite/index.js +1 -0
  205. package/src/formats/tflite/types.d.ts +84 -0
  206. package/src/formats/tflite/types.js +633 -0
  207. package/src/generation/index.d.ts +16 -14
  208. package/src/generation/index.js +1 -32
  209. package/src/gpu/command-recorder.d.ts +17 -1
  210. package/src/gpu/command-recorder.js +47 -10
  211. package/src/gpu/device.d.ts +3 -1
  212. package/src/gpu/device.js +17 -11
  213. package/src/gpu/kernel-selector.js +3 -1
  214. package/src/gpu/kernels/attention.d.ts +2 -0
  215. package/src/gpu/kernels/attention.js +359 -14
  216. package/src/gpu/kernels/attention.wgsl +20 -1
  217. package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +180 -0
  218. package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +192 -0
  219. package/src/gpu/kernels/attention_decode_online_f16.wgsl +36 -18
  220. package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +36 -18
  221. package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +269 -0
  222. package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +281 -0
  223. package/src/gpu/kernels/attention_head256_f16kv.wgsl +269 -0
  224. package/src/gpu/kernels/attention_streaming.wgsl +22 -3
  225. package/src/gpu/kernels/attention_streaming_f16.wgsl +1 -1
  226. package/src/gpu/kernels/attention_streaming_f16kv.wgsl +1 -1
  227. package/src/gpu/kernels/backward/attention_backward.d.ts +14 -1
  228. package/src/gpu/kernels/check-hot-vocab-stop.d.ts +18 -0
  229. package/src/gpu/kernels/check-hot-vocab-stop.js +108 -0
  230. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +1 -1
  231. package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +1 -1
  232. package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +126 -194
  233. package/src/gpu/kernels/codegen/wgsl-variants.js +24 -6
  234. package/src/gpu/kernels/depthwise_conv1d.d.ts +23 -0
  235. package/src/gpu/kernels/depthwise_conv1d.js +51 -0
  236. package/src/gpu/kernels/depthwise_conv1d.wgsl +46 -0
  237. package/src/gpu/kernels/feature-check.js +0 -1
  238. package/src/gpu/kernels/fused_ffn.js +67 -22
  239. package/src/gpu/kernels/fused_ffn_q4k.wgsl +48 -43
  240. package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +329 -0
  241. package/src/gpu/kernels/fused_matmul_q4.wgsl +417 -0
  242. package/src/gpu/kernels/fused_matmul_q4_batched_multicol_shared.wgsl +174 -0
  243. package/src/gpu/kernels/gather.d.ts +4 -0
  244. package/src/gpu/kernels/gather.js +13 -2
  245. package/src/gpu/kernels/gather.wgsl +5 -3
  246. package/src/gpu/kernels/gather_f16.wgsl +5 -3
  247. package/src/gpu/kernels/index.d.ts +7 -0
  248. package/src/gpu/kernels/index.js +6 -0
  249. package/src/gpu/kernels/kernel-configs.js +1 -0
  250. package/src/gpu/kernels/kv-quantize.js +87 -55
  251. package/src/gpu/kernels/kv_quantize_turboquant.wgsl +153 -0
  252. package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +230 -0
  253. package/src/gpu/kernels/linear-attention-core.d.ts +3 -0
  254. package/src/gpu/kernels/linear-attention-core.js +145 -56
  255. package/src/gpu/kernels/matmul-dispatch.js +3 -0
  256. package/src/gpu/kernels/matmul-selection.js +31 -42
  257. package/src/gpu/kernels/matmul.d.ts +3 -0
  258. package/src/gpu/kernels/matmul.js +72 -34
  259. package/src/gpu/kernels/residual.d.ts +2 -0
  260. package/src/gpu/kernels/residual.js +43 -4
  261. package/src/gpu/kernels/rope.d.ts +2 -0
  262. package/src/gpu/kernels/rope.js +19 -6
  263. package/src/gpu/kernels/rope.wgsl +24 -12
  264. package/src/gpu/kernels/rope_f16.wgsl +97 -38
  265. package/src/gpu/kernels/scale.d.ts +5 -0
  266. package/src/gpu/kernels/scale.js +6 -1
  267. package/src/gpu/kernels/scale.wgsl +3 -1
  268. package/src/gpu/kernels/scale_f16.wgsl +32 -0
  269. package/src/gpu/kernels/turboquant-codebook.js +564 -0
  270. package/src/gpu/kernels/uniform-utils.js +26 -0
  271. package/src/gpu/kernels/utils.js +6 -1
  272. package/src/gpu/submit-probe.d.ts +12 -0
  273. package/src/gpu/submit-probe.js +77 -0
  274. package/src/gpu/weight-buffer.d.ts +26 -3
  275. package/src/gpu/weight-buffer.js +62 -2
  276. package/src/index-browser.d.ts +1 -91
  277. package/src/index-browser.js +0 -66
  278. package/src/index.d.ts +1 -102
  279. package/src/index.js +0 -74
  280. package/src/inference/browser-harness-model-helpers.d.ts +3 -0
  281. package/src/inference/browser-harness-model-helpers.js +134 -11
  282. package/src/inference/browser-harness-runtime-helpers.js +1 -9
  283. package/src/inference/browser-harness-text-helpers.js +441 -70
  284. package/src/inference/browser-harness.js +61 -4
  285. package/src/inference/decode-ring.d.ts +2 -0
  286. package/src/inference/decode-ring.js +20 -0
  287. package/src/inference/kv-cache/index.d.ts +3 -0
  288. package/src/inference/kv-cache/index.js +2 -0
  289. package/src/inference/kv-cache/mixed-geometry.d.ts +84 -0
  290. package/src/inference/kv-cache/mixed-geometry.js +422 -0
  291. package/src/inference/kv-cache/quantized.d.ts +56 -0
  292. package/src/inference/kv-cache/quantized.js +383 -0
  293. package/src/inference/kv-cache/tiered.d.ts +20 -3
  294. package/src/inference/kv-cache/tiered.js +203 -65
  295. package/src/inference/kv-cache/types.d.ts +49 -6
  296. package/src/inference/kv-cache.d.ts +3 -0
  297. package/src/inference/kv-cache.js +2 -0
  298. package/src/inference/multi-model-network.js +2 -1
  299. package/src/inference/pipelines/audio/gemma4.js +655 -0
  300. package/src/inference/pipelines/audio/index.js +39 -0
  301. package/src/inference/pipelines/audio/mel.js +171 -0
  302. package/src/inference/pipelines/context.js +9 -1
  303. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +2 -0
  304. package/src/inference/pipelines/shared/clipped-linear.js +46 -0
  305. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +2 -1
  306. package/src/inference/pipelines/text/attention/dispatch-params.js +52 -8
  307. package/src/inference/pipelines/text/attention/output-projection.js +3 -3
  308. package/src/inference/pipelines/text/attention/precision-contract.d.ts +36 -0
  309. package/src/inference/pipelines/text/attention/precision-contract.js +115 -0
  310. package/src/inference/pipelines/text/attention/projections.d.ts +23 -0
  311. package/src/inference/pipelines/text/attention/projections.js +182 -15
  312. package/src/inference/pipelines/text/attention/quant-options.js +54 -0
  313. package/src/inference/pipelines/text/attention/record.js +257 -69
  314. package/src/inference/pipelines/text/attention/run.js +306 -94
  315. package/src/inference/pipelines/text/attention/types.d.ts +11 -0
  316. package/src/inference/pipelines/text/chat-format.d.ts +8 -1
  317. package/src/inference/pipelines/text/chat-format.js +89 -5
  318. package/src/inference/pipelines/text/config.d.ts +78 -26
  319. package/src/inference/pipelines/text/config.js +722 -28
  320. package/src/inference/pipelines/text/dtype-contract.d.ts +15 -0
  321. package/src/inference/pipelines/text/dtype-contract.js +38 -0
  322. package/src/inference/pipelines/text/embed.d.ts +4 -0
  323. package/src/inference/pipelines/text/embed.js +165 -172
  324. package/src/inference/pipelines/text/execution-plan.d.ts +19 -0
  325. package/src/inference/pipelines/text/execution-plan.js +112 -20
  326. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +8 -4
  327. package/src/inference/pipelines/text/execution-runtime-builders.js +90 -49
  328. package/src/inference/pipelines/text/execution-v1.d.ts +23 -4
  329. package/src/inference/pipelines/text/execution-v1.js +411 -33
  330. package/src/inference/pipelines/text/ffn/dense.d.ts +59 -0
  331. package/src/inference/pipelines/text/ffn/dense.js +575 -63
  332. package/src/inference/pipelines/text/ffn/moe.js +1 -0
  333. package/src/inference/pipelines/text/ffn/sandwich.js +20 -4
  334. package/src/inference/pipelines/text/ffn/standard.js +28 -4
  335. package/src/inference/pipelines/text/finiteness-policy.d.ts +4 -0
  336. package/src/inference/pipelines/text/finiteness-policy.js +14 -11
  337. package/src/inference/pipelines/text/generator-helpers.d.ts +7 -0
  338. package/src/inference/pipelines/text/generator-helpers.js +39 -4
  339. package/src/inference/pipelines/text/generator-runtime.js +45 -8
  340. package/src/inference/pipelines/text/generator-steps.js +407 -75
  341. package/src/inference/pipelines/text/generator.d.ts +6 -0
  342. package/src/inference/pipelines/text/generator.js +1179 -305
  343. package/src/inference/pipelines/text/init.d.ts +39 -7
  344. package/src/inference/pipelines/text/init.js +525 -110
  345. package/src/inference/pipelines/text/layer-plan.d.ts +1 -0
  346. package/src/inference/pipelines/text/layer-plan.js +2 -0
  347. package/src/inference/pipelines/text/layer.js +402 -19
  348. package/src/inference/pipelines/text/linear-attention.d.ts +4 -0
  349. package/src/inference/pipelines/text/linear-attention.js +112 -41
  350. package/src/inference/pipelines/text/logits/gpu.d.ts +2 -1
  351. package/src/inference/pipelines/text/logits/gpu.js +334 -15
  352. package/src/inference/pipelines/text/logits/index.d.ts +11 -0
  353. package/src/inference/pipelines/text/logits/index.js +227 -29
  354. package/src/inference/pipelines/text/logits/types.d.ts +3 -0
  355. package/src/inference/pipelines/text/logits/utils.js +3 -1
  356. package/src/inference/pipelines/text/lora-apply.js +3 -3
  357. package/src/inference/pipelines/text/model-load.d.ts +2 -1
  358. package/src/inference/pipelines/text/model-load.js +205 -178
  359. package/src/inference/pipelines/text/moe-gpu.js +8 -0
  360. package/src/inference/pipelines/text/moe-impl.d.ts +2 -0
  361. package/src/inference/pipelines/text/ops.d.ts +16 -2
  362. package/src/inference/pipelines/text/ops.js +11 -2
  363. package/src/inference/pipelines/text/per-layer-inputs.d.ts +107 -0
  364. package/src/inference/pipelines/text/per-layer-inputs.js +1581 -0
  365. package/src/inference/pipelines/text/probes.js +6 -0
  366. package/src/inference/pipelines/text/sampling-config.js +9 -1
  367. package/src/inference/pipelines/text/stage-names.js +3 -0
  368. package/src/inference/pipelines/text/state.d.ts +12 -3
  369. package/src/inference/pipelines/text/state.js +19 -0
  370. package/src/inference/pipelines/text/types.d.ts +79 -7
  371. package/src/inference/pipelines/text/weights.d.ts +2 -2
  372. package/src/inference/pipelines/text/weights.js +4 -0
  373. package/src/inference/pipelines/text.d.ts +6 -0
  374. package/src/inference/pipelines/text.js +860 -11
  375. package/src/inference/pipelines/video/frame-sampler.js +36 -0
  376. package/src/inference/pipelines/video/index.js +87 -0
  377. package/src/inference/pipelines/vision/encoder.js +27 -6
  378. package/src/inference/pipelines/vision/gemma4.js +731 -0
  379. package/src/inference/pipelines/vision/image-preprocess.js +22 -7
  380. package/src/inference/pipelines/vision/index.js +35 -10
  381. package/src/inference/pipelines/vision/ops.js +31 -26
  382. package/src/inference/pipelines/vision/patch-embed.js +22 -10
  383. package/src/inference/runtime-model.d.ts +89 -0
  384. package/src/inference/runtime-model.js +152 -0
  385. package/src/inference/test-harness.d.ts +8 -1
  386. package/src/inference/test-harness.js +54 -20
  387. package/src/inference/tokenizer.d.ts +5 -0
  388. package/src/inference/tokenizer.js +43 -0
  389. package/src/inference/tokenizers/base.d.ts +5 -0
  390. package/src/inference/tokenizers/base.js +5 -0
  391. package/src/inference/tokenizers/behavior-flags.d.ts +9 -0
  392. package/src/inference/tokenizers/behavior-flags.js +100 -0
  393. package/src/inference/tokenizers/bundled.d.ts +4 -0
  394. package/src/inference/tokenizers/bundled.js +110 -0
  395. package/src/inference/tokenizers/sentencepiece.d.ts +2 -0
  396. package/src/inference/tokenizers/sentencepiece.js +16 -0
  397. package/src/inference/tokenizers/types.d.ts +2 -0
  398. package/src/loader/doppler-loader.d.ts +10 -3
  399. package/src/loader/doppler-loader.js +103 -5
  400. package/src/loader/embedding-loader.d.ts +5 -1
  401. package/src/loader/embedding-loader.js +57 -6
  402. package/src/loader/final-weights-loader.js +44 -5
  403. package/src/loader/layer-loader.js +210 -8
  404. package/src/loader/loader-state.js +2 -2
  405. package/src/loader/loader-types.d.ts +17 -1
  406. package/src/loader/memory-monitor.d.ts +8 -0
  407. package/src/loader/memory-monitor.js +21 -0
  408. package/src/loader/multi-model-loader.d.ts +9 -1
  409. package/src/loader/multi-model-loader.js +9 -1
  410. package/src/loader/per-layer-input-loader.d.ts +30 -0
  411. package/src/loader/per-layer-input-loader.js +478 -0
  412. package/src/loader/shard-cache.d.ts +4 -3
  413. package/src/loader/shard-resolver.js +2 -0
  414. package/src/loader/tensors/source-transform.d.ts +50 -0
  415. package/src/loader/tensors/source-transform.js +776 -0
  416. package/src/loader/tensors/tensor-loader.d.ts +12 -0
  417. package/src/loader/tensors/tensor-loader.js +208 -123
  418. package/src/loader/tensors/tensor-reader.d.ts +8 -0
  419. package/src/loader/tensors/tensor-reader.js +283 -15
  420. package/src/loaders/index.d.ts +4 -0
  421. package/src/loaders/index.js +4 -0
  422. package/src/memory/buffer-pool.d.ts +4 -0
  423. package/src/memory/buffer-pool.js +36 -0
  424. package/src/rules/execution-rules-contract-check.d.ts +4 -0
  425. package/src/rules/execution-rules-contract-check.js +150 -24
  426. package/src/rules/inference/attention.rules.json +15 -0
  427. package/src/rules/inference/capability-transforms.rules.json +30 -0
  428. package/src/rules/inference/execution.rules.json +54 -2
  429. package/src/rules/inference/ffn.rules.json +33 -0
  430. package/src/rules/kernels/attention.rules.json +7 -0
  431. package/src/rules/kernels/fused-ffn.rules.json +4 -2
  432. package/src/rules/kernels/kv_quantize.rules.json +2 -0
  433. package/src/rules/kernels/matmul.rules.json +14 -0
  434. package/src/rules/kernels/scale.rules.json +2 -0
  435. package/src/rules/loader/tensor-loader.rules.json +1 -0
  436. package/src/storage/artifact-storage-context.d.ts +53 -0
  437. package/src/storage/artifact-storage-context.js +560 -0
  438. package/src/storage/downloader.js +7 -1
  439. package/src/storage/index.d.ts +18 -0
  440. package/src/storage/index.js +13 -0
  441. package/src/storage/preflight.js +22 -0
  442. package/src/storage/quickstart-downloader.js +35 -2
  443. package/src/storage/registry.d.ts +1 -0
  444. package/src/storage/registry.js +26 -4
  445. package/src/storage/shard-manager.d.ts +4 -0
  446. package/src/storage/shard-manager.js +25 -6
  447. package/src/storage/source-artifact-store.d.ts +9 -0
  448. package/src/storage/source-artifact-store.js +134 -0
  449. package/src/tooling/browser-command-runner.js +12 -5
  450. package/src/tooling/command-api-family-normalizers.js +144 -0
  451. package/src/tooling/command-api-helpers.js +1 -0
  452. package/src/tooling/command-api.d.ts +32 -9
  453. package/src/tooling/command-api.js +0 -25
  454. package/src/tooling/command-runner-shared.js +1 -2
  455. package/src/tooling/command-runner.html +5 -2
  456. package/src/tooling/diagnose-runner.js +1 -1
  457. package/src/tooling/kernel-path-builder/index.js +48 -32
  458. package/src/tooling/lean-execution-contract.d.ts +1 -1
  459. package/src/tooling/lean-execution-contract.js +2 -149
  460. package/src/tooling/litert-package-runtime.d.ts +46 -0
  461. package/src/tooling/litert-package-runtime.js +950 -0
  462. package/src/tooling/node-browser-command-runner.d.ts +13 -0
  463. package/src/tooling/node-browser-command-runner.js +163 -3
  464. package/src/tooling/node-command-runner.js +14 -5
  465. package/src/tooling/node-converter.js +226 -63
  466. package/src/tooling/node-source-runtime.d.ts +8 -3
  467. package/src/tooling/node-source-runtime.js +561 -236
  468. package/src/tooling/node-webgpu.js +7 -30
  469. package/src/tooling/opfs-cache.d.ts +18 -1
  470. package/src/tooling/opfs-cache.js +126 -9
  471. package/src/tooling/runtime-input-composition.d.ts +0 -1
  472. package/src/tooling/runtime-input-composition.js +0 -5
  473. package/src/tooling/source-artifact-adapter.d.ts +134 -0
  474. package/src/tooling/source-artifact-adapter.js +366 -0
  475. package/src/tooling/source-package-profiles.d.ts +33 -0
  476. package/src/tooling/source-package-profiles.js +57 -0
  477. package/src/tooling/source-runtime-bundle.d.ts +46 -6
  478. package/src/tooling/source-runtime-bundle.js +371 -60
  479. package/src/tooling/source-runtime-converter-config.d.ts +24 -0
  480. package/src/tooling/source-runtime-converter-config.js +297 -0
  481. package/src/tooling-experimental-exports.browser.js +2 -0
  482. package/src/tooling-experimental-exports.d.ts +7 -0
  483. package/src/tooling-experimental-exports.js +7 -0
  484. package/src/tooling-experimental-exports.shared.d.ts +26 -0
  485. package/src/tooling-experimental-exports.shared.js +29 -0
  486. package/src/tooling-exports.shared.d.ts +0 -23
  487. package/src/tooling-exports.shared.js +6 -28
  488. package/src/version.js +1 -1
  489. package/src/client/doppler-provider/generation.d.ts +0 -25
  490. package/src/client/doppler-provider/generation.js +0 -126
  491. package/src/client/doppler-provider/index.d.ts +0 -2
  492. package/src/client/doppler-provider/index.js +0 -3
  493. package/src/client/doppler-provider/provider.d.ts +0 -5
  494. package/src/client/doppler-provider/provider.js +0 -102
  495. package/src/client/doppler-provider/source-runtime.js +0 -677
  496. package/src/client/doppler-provider.d.ts +0 -46
  497. package/src/client/doppler-provider.js +0 -36
  498. package/src/config/schema/inference-defaults.schema.d.ts +0 -276
  499. package/src/config/schema/inference-defaults.schema.js +0 -189
  500. package/src/diffusion/index.d.ts +0 -8
  501. package/src/diffusion/index.js +0 -8
  502. package/src/energy/index.d.ts +0 -2
  503. package/src/energy/index.js +0 -2
  504. package/src/index-internal.d.ts +0 -2
  505. package/src/index-internal.js +0 -2
  506. /package/src/client/{doppler-provider → runtime}/types.js +0 -0
  507. /package/src/{adapters → experimental/adapters}/index.d.ts +0 -0
  508. /package/src/{adapters → experimental/adapters}/index.js +0 -0
  509. /package/src/{bridge → experimental/bridge}/extension/background.d.ts +0 -0
  510. /package/src/{bridge → experimental/bridge}/extension/background.js +0 -0
  511. /package/src/{bridge → experimental/bridge}/extension/manifest.json +0 -0
  512. /package/src/{bridge → experimental/bridge}/extension-client.d.ts +0 -0
  513. /package/src/{bridge → experimental/bridge}/index.d.ts +0 -0
  514. /package/src/{bridge → experimental/bridge}/index.js +0 -0
  515. /package/src/{bridge → experimental/bridge}/protocol.d.ts +0 -0
  516. /package/src/{browser → experimental/browser}/file-picker.d.ts +0 -0
  517. /package/src/{browser → experimental/browser}/file-picker.js +0 -0
  518. /package/src/{browser → experimental/browser}/tensor-source-download.d.ts +0 -0
  519. /package/src/{browser → experimental/browser}/tensor-source-file.d.ts +0 -0
  520. /package/src/{browser → experimental/browser}/tensor-source-file.js +0 -0
  521. /package/src/{browser → experimental/browser}/tensor-source-http.d.ts +0 -0
  522. /package/src/{browser → experimental/browser}/tensor-source-http.js +0 -0
  523. /package/src/{diffusion → experimental/diffusion}/config/expansion-lanes.json +0 -0
  524. /package/src/{diffusion → experimental/diffusion}/image-regression.d.ts +0 -0
  525. /package/src/{diffusion → experimental/diffusion}/image-regression.js +0 -0
  526. /package/src/{distribution → experimental/distribution}/p2p-control-plane.d.ts +0 -0
  527. /package/src/{distribution → experimental/distribution}/p2p-control-plane.js +0 -0
  528. /package/src/{distribution → experimental/distribution}/p2p-observability.d.ts +0 -0
  529. /package/src/{distribution → experimental/distribution}/p2p-observability.js +0 -0
  530. /package/src/{distribution → experimental/distribution}/p2p-transport-contract.d.ts +0 -0
  531. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.d.ts +0 -0
  532. /package/src/{distribution → experimental/distribution}/p2p-webrtc-browser.js +0 -0
  533. /package/src/{distribution → experimental/distribution}/shard-delivery.d.ts +0 -0
  534. /package/src/{hotswap → experimental/hotswap}/runtime.js +0 -0
  535. /package/src/{training → experimental/training}/checkpoint-watch.d.ts +0 -0
  536. /package/src/{training → experimental/training}/checkpoint-watch.js +0 -0
  537. /package/src/{training → experimental/training}/checkpoint.d.ts +0 -0
  538. /package/src/{training → experimental/training}/dataloader.d.ts +0 -0
  539. /package/src/{training → experimental/training}/dataloader.js +0 -0
  540. /package/src/{training → experimental/training}/datasets/index.d.ts +0 -0
  541. /package/src/{training → experimental/training}/datasets/index.js +0 -0
  542. /package/src/{training → experimental/training}/datasets/jsonl.d.ts +0 -0
  543. /package/src/{training → experimental/training}/datasets/jsonl.js +0 -0
  544. /package/src/{training → experimental/training}/datasets/reploid.d.ts +0 -0
  545. /package/src/{training → experimental/training}/datasets/reploid.js +0 -0
  546. /package/src/{training → experimental/training}/datasets/text-pairs.d.ts +0 -0
  547. /package/src/{training → experimental/training}/datasets/text-pairs.js +0 -0
  548. /package/src/{training → experimental/training}/datasets/token-batch.d.ts +0 -0
  549. /package/src/{training → experimental/training}/datasets/token-batch.js +0 -0
  550. /package/src/{training → experimental/training}/datasets/translation-pairs.d.ts +0 -0
  551. /package/src/{training → experimental/training}/datasets/translation-pairs.js +0 -0
  552. /package/src/{training → experimental/training}/distillation/artifacts.d.ts +0 -0
  553. /package/src/{training → experimental/training}/distillation/artifacts.js +0 -0
  554. /package/src/{training → experimental/training}/distillation/checkpoint-watch.d.ts +0 -0
  555. /package/src/{training → experimental/training}/distillation/checkpoint-watch.js +0 -0
  556. /package/src/{training → experimental/training}/distillation/dataset.d.ts +0 -0
  557. /package/src/{training → experimental/training}/distillation/dataset.js +0 -0
  558. /package/src/{training → experimental/training}/distillation/eval.d.ts +0 -0
  559. /package/src/{training → experimental/training}/distillation/eval.js +0 -0
  560. /package/src/{training → experimental/training}/distillation/index.d.ts +0 -0
  561. /package/src/{training → experimental/training}/distillation/index.js +0 -0
  562. /package/src/{training → experimental/training}/distillation/runtime.d.ts +0 -0
  563. /package/src/{training → experimental/training}/distillation/runtime.js +0 -0
  564. /package/src/{training → experimental/training}/distillation/scoreboard.d.ts +0 -0
  565. /package/src/{training → experimental/training}/distillation/scoreboard.js +0 -0
  566. /package/src/{training → experimental/training}/distillation/stage-a.d.ts +0 -0
  567. /package/src/{training → experimental/training}/distillation/stage-a.js +0 -0
  568. /package/src/{training → experimental/training}/distillation/stage-b.d.ts +0 -0
  569. /package/src/{training → experimental/training}/distillation/stage-b.js +0 -0
  570. /package/src/{training → experimental/training}/distillation/student-fixture.d.ts +0 -0
  571. /package/src/{training → experimental/training}/distillation/student-fixture.js +0 -0
  572. /package/src/{training → experimental/training}/distillation/suite-data.d.ts +0 -0
  573. /package/src/{training → experimental/training}/distillation/suite-data.js +0 -0
  574. /package/src/{training → experimental/training}/index.d.ts +0 -0
  575. /package/src/{training → experimental/training}/index.js +0 -0
  576. /package/src/{training → experimental/training}/lora-pipeline.d.ts +0 -0
  577. /package/src/{training → experimental/training}/objectives/base.d.ts +0 -0
  578. /package/src/{training → experimental/training}/objectives/base.js +0 -0
  579. /package/src/{training → experimental/training}/objectives/cross_entropy.d.ts +0 -0
  580. /package/src/{training → experimental/training}/objectives/cross_entropy.js +0 -0
  581. /package/src/{training → experimental/training}/objectives/distill_kd.d.ts +0 -0
  582. /package/src/{training → experimental/training}/objectives/distill_kd.js +0 -0
  583. /package/src/{training → experimental/training}/objectives/distill_triplet.d.ts +0 -0
  584. /package/src/{training → experimental/training}/objectives/distill_triplet.js +0 -0
  585. /package/src/{training → experimental/training}/objectives/index.d.ts +0 -0
  586. /package/src/{training → experimental/training}/objectives/index.js +0 -0
  587. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.d.ts +0 -0
  588. /package/src/{training → experimental/training}/objectives/ul_stage1_joint.js +0 -0
  589. /package/src/{training → experimental/training}/objectives/ul_stage2_base.d.ts +0 -0
  590. /package/src/{training → experimental/training}/objectives/ul_stage2_base.js +0 -0
  591. /package/src/{training → experimental/training}/operator-artifacts.d.ts +0 -0
  592. /package/src/{training → experimental/training}/operator-command.d.ts +0 -0
  593. /package/src/{training → experimental/training}/operator-command.js +0 -0
  594. /package/src/{training → experimental/training}/operator-eval.d.ts +0 -0
  595. /package/src/{training → experimental/training}/operator-eval.js +0 -0
  596. /package/src/{training → experimental/training}/operator-scoreboard.d.ts +0 -0
  597. /package/src/{training → experimental/training}/operator-scoreboard.js +0 -0
  598. /package/src/{training → experimental/training}/suite.d.ts +0 -0
  599. /package/src/{training → experimental/training}/ul_schedule.js +0 -0
  600. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-medium.json +0 -0
  601. /package/src/{training → experimental/training}/workload-packs/distill-translategemma-tiny.json +0 -0
  602. /package/src/{training → experimental/training}/workload-packs/lora-toy-medium.json +0 -0
  603. /package/src/{training → experimental/training}/workload-packs/lora-toy-tiny.json +0 -0
  604. /package/src/{training → experimental/training}/workload-packs/registry.json +0 -0
  605. /package/src/{training → experimental/training}/workload-packs/ul-training-medium.json +0 -0
  606. /package/src/{training → experimental/training}/workload-packs/ul-training-tiny.json +0 -0
  607. /package/src/{training → experimental/training}/workloads.d.ts +0 -0
@@ -1,8 +1,17 @@
1
1
  import { log as debugLog } from '../debug/index.js';
2
+ import { CAPTURE_LEVELS, createDefaultCaptureConfig } from '../debug/capture-policy.js';
2
3
  import { selectRuleValue } from '../rules/rule-registry.js';
3
4
  import { loadJson } from '../utils/load-json.js';
4
5
  import { isPlainObject } from '../utils/plain-object.js';
5
- import { DEFAULT_SAMPLING_DEFAULTS } from '../config/schema/inference-defaults.schema.js';
6
+
7
+ const DEFAULT_SAMPLING_DEFAULTS = Object.freeze({
8
+ temperature: 1.0,
9
+ topP: 0.95,
10
+ topK: 50,
11
+ repetitionPenalty: 1.1,
12
+ greedyThreshold: 0.01,
13
+ repetitionPenaltyWindow: 100,
14
+ });
6
15
 
7
16
  const DEFAULT_HARNESS_PROMPT = 'The color of the sky is';
8
17
  const DEFAULT_RUNTIME_PLACEHOLDER_PROMPT = 'Hello from Doppler.';
@@ -29,6 +38,8 @@ const DEFAULT_TRANSLATEGEMMA_PROMPT = Object.freeze({
29
38
  }),
30
39
  ]),
31
40
  });
41
+ const DEFAULT_IMAGE_TRANSCRIPTION_PROMPT = 'Describe the image in one short sentence.';
42
+ const DEFAULT_IMAGE_TRANSCRIPTION_SOFT_TOKEN_BUDGET = 70;
32
43
  const DEFAULT_HARNESS_MAX_TOKENS = 32;
33
44
  const EMBEDDING_PREVIEW_LENGTH = 16;
34
45
  const GENERATION_TOKEN_DIAGNOSTIC_LIMIT = 32;
@@ -36,24 +47,35 @@ let defaultsWarningEmitted = false;
36
47
 
37
48
  function warnIfUsingDefaults(runtimeConfig) {
38
49
  if (defaultsWarningEmitted) return;
39
- const hasPrompt = typeof runtimeConfig?.inference?.prompt === 'string'
40
- && runtimeConfig.inference.prompt.trim().length > 0;
50
+ const promptOverride = runtimeConfig?.inference?.prompt;
51
+ const hasPrompt = (typeof promptOverride === 'string' && promptOverride.trim().length > 0)
52
+ || (Array.isArray(promptOverride) && promptOverride.length > 0)
53
+ || isStructuredPromptInput(promptOverride);
41
54
  const hasSampling = isPlainObject(runtimeConfig?.inference?.sampling)
42
55
  && Object.keys(runtimeConfig.inference.sampling).length > 0;
43
- const hasMaxTokens = Number.isFinite(runtimeConfig?.inference?.batching?.maxTokens);
56
+ const hasMaxTokens = Number.isFinite(runtimeConfig?.inference?.generation?.maxTokens);
44
57
  if (hasPrompt && hasSampling && hasMaxTokens) return;
45
58
  defaultsWarningEmitted = true;
46
- const defaults = [
47
- ` prompt: "${DEFAULT_HARNESS_PROMPT}"`,
48
- ` maxTokens: ${DEFAULT_HARNESS_MAX_TOKENS}`,
49
- ` temperature: ${DEFAULT_SAMPLING_DEFAULTS.temperature}`,
50
- ` topK: ${DEFAULT_SAMPLING_DEFAULTS.topK}`,
51
- ` topP: ${DEFAULT_SAMPLING_DEFAULTS.topP}`,
52
- ];
59
+ const missingFields = [];
60
+ const defaults = [];
61
+ if (!hasPrompt) {
62
+ missingFields.push('prompt');
63
+ defaults.push(` prompt: "${DEFAULT_HARNESS_PROMPT}"`);
64
+ }
65
+ if (!hasMaxTokens) {
66
+ missingFields.push('generation.maxTokens');
67
+ defaults.push(` maxTokens: ${DEFAULT_HARNESS_MAX_TOKENS}`);
68
+ }
69
+ if (!hasSampling) {
70
+ missingFields.push('sampling');
71
+ defaults.push(` temperature: ${DEFAULT_SAMPLING_DEFAULTS.temperature}`);
72
+ defaults.push(` topK: ${DEFAULT_SAMPLING_DEFAULTS.topK}`);
73
+ defaults.push(` topP: ${DEFAULT_SAMPLING_DEFAULTS.topP}`);
74
+ }
53
75
  debugLog.warn('Harness',
54
- 'Running with default inference parameters (no runtime config override):\n'
76
+ `Running with default inference parameters for missing fields: ${missingFields.join(', ')}.\n`
55
77
  + defaults.join('\n')
56
- + '\n See: src/config/schema/inference-defaults.schema.js'
78
+ + '\n Provide explicit runtime.inference.sampling and generation.maxTokens if you want harness-stable settings.'
57
79
  );
58
80
  }
59
81
  const embeddingSemanticFixtureAsset = await loadJson(
@@ -339,6 +361,12 @@ function describePromptInput(promptInput) {
339
361
  if (typeof promptInput === 'string') {
340
362
  return promptInput.trim() || DEFAULT_HARNESS_PROMPT;
341
363
  }
364
+ if (isPlainObject(promptInput?.image) && typeof promptInput?.prompt === 'string') {
365
+ const width = Number.isFinite(promptInput.image.width) ? promptInput.image.width : '?';
366
+ const height = Number.isFinite(promptInput.image.height) ? promptInput.image.height : '?';
367
+ const source = asText(promptInput.image.source) ?? 'image';
368
+ return `${source} ${width}x${height}: ${promptInput.prompt}`;
369
+ }
342
370
  const firstMessage = Array.isArray(promptInput?.messages)
343
371
  ? promptInput.messages[0]
344
372
  : null;
@@ -363,6 +391,214 @@ function describePromptInput(promptInput) {
363
391
  }
364
392
  }
365
393
 
394
+ function decodeBase64ToBytes(base64, label) {
395
+ const normalized = asText(base64);
396
+ if (!normalized) {
397
+ throw new Error(`${label} must be a non-empty base64 string.`);
398
+ }
399
+ if (typeof Buffer !== 'undefined') {
400
+ return new Uint8Array(Buffer.from(normalized, 'base64'));
401
+ }
402
+ if (typeof atob === 'function') {
403
+ const binary = atob(normalized);
404
+ const bytes = new Uint8Array(binary.length);
405
+ for (let i = 0; i < binary.length; i++) {
406
+ bytes[i] = binary.charCodeAt(i);
407
+ }
408
+ return bytes;
409
+ }
410
+ throw new Error(`${label} requires a base64 decoder in this runtime.`);
411
+ }
412
+
413
+ function assertRawImageByteLength(bytes, width, height, label) {
414
+ const expectedRgb = width * height * 3;
415
+ const expectedRgba = width * height * 4;
416
+ if (bytes.length !== expectedRgb && bytes.length !== expectedRgba) {
417
+ throw new Error(
418
+ `${label} must contain width*height*3 or width*height*4 bytes. ` +
419
+ `Got ${bytes.length} for ${width}x${height}.`
420
+ );
421
+ }
422
+ }
423
+
424
+ function normalizeRawImageBytes(value, width, height, label) {
425
+ let bytes = null;
426
+ if (ArrayBuffer.isView(value)) {
427
+ bytes = new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
428
+ } else if (Array.isArray(value)) {
429
+ const normalized = new Uint8Array(value.length);
430
+ for (let i = 0; i < value.length; i++) {
431
+ const parsed = Number(value[i]);
432
+ if (!Number.isInteger(parsed) || parsed < 0 || parsed > 255) {
433
+ throw new Error(`${label}[${i}] must be an integer in [0, 255].`);
434
+ }
435
+ normalized[i] = parsed;
436
+ }
437
+ bytes = normalized;
438
+ }
439
+ if (!(bytes instanceof Uint8Array)) {
440
+ throw new Error(`${label} must be an array or typed array.`);
441
+ }
442
+ assertRawImageByteLength(bytes, width, height, label);
443
+ return new Uint8Array(bytes);
444
+ }
445
+
446
+ function createCanvasForImageDecode(width, height) {
447
+ if (typeof OffscreenCanvas === 'function') {
448
+ return new OffscreenCanvas(width, height);
449
+ }
450
+ if (typeof document !== 'undefined' && typeof document.createElement === 'function') {
451
+ const canvas = document.createElement('canvas');
452
+ canvas.width = width;
453
+ canvas.height = height;
454
+ return canvas;
455
+ }
456
+ return null;
457
+ }
458
+
459
+ function isNodeRuntime() {
460
+ return typeof process !== 'undefined' && !!process.versions?.node;
461
+ }
462
+
463
+ async function decodeImageUrlToPixelsOnNode(url) {
464
+ let sharpModule = null;
465
+ try {
466
+ sharpModule = await import('sharp');
467
+ } catch (error) {
468
+ throw new Error(
469
+ `URL-backed inferenceInput.image.url on the node surface requires the optional "sharp" decoder. ${error?.message || error}`
470
+ );
471
+ }
472
+ const sharp = typeof sharpModule?.default === 'function'
473
+ ? sharpModule.default
474
+ : sharpModule;
475
+ if (typeof sharp !== 'function') {
476
+ throw new Error('Node image decode requires sharp to export a callable default.');
477
+ }
478
+
479
+ const response = await fetch(url);
480
+ if (!response.ok) {
481
+ throw new Error(`image fetch failed: HTTP ${response.status}`);
482
+ }
483
+ const sourceBytes = new Uint8Array(await response.arrayBuffer());
484
+ const decoded = await sharp(sourceBytes)
485
+ .ensureAlpha()
486
+ .raw()
487
+ .toBuffer({ resolveWithObject: true });
488
+ return {
489
+ imageBytes: new Uint8Array(decoded.data),
490
+ width: decoded.info.width,
491
+ height: decoded.info.height,
492
+ };
493
+ }
494
+
495
+ async function decodeImageUrlToPixels(url) {
496
+ if (isNodeRuntime()) {
497
+ return decodeImageUrlToPixelsOnNode(url);
498
+ }
499
+ const response = await fetch(url);
500
+ if (!response.ok) {
501
+ throw new Error(`image fetch failed: HTTP ${response.status}`);
502
+ }
503
+ if (typeof createImageBitmap !== 'function') {
504
+ throw new Error(
505
+ 'URL-backed inferenceInput.image.url requires createImageBitmap support. ' +
506
+ 'Use raw pixels or run on a browser-capable surface.'
507
+ );
508
+ }
509
+
510
+ const imageBlob = await response.blob();
511
+ const imageBitmap = await createImageBitmap(imageBlob);
512
+ try {
513
+ const canvas = createCanvasForImageDecode(imageBitmap.width, imageBitmap.height);
514
+ if (!canvas) {
515
+ throw new Error(
516
+ 'URL-backed inferenceInput.image.url requires OffscreenCanvas or a DOM canvas in this runtime. ' +
517
+ 'Use raw pixels or run on a browser-capable surface.'
518
+ );
519
+ }
520
+ const context = canvas.getContext('2d', { willReadFrequently: true });
521
+ if (!context || typeof context.getImageData !== 'function') {
522
+ throw new Error('Image decode canvas did not provide a readable 2D context.');
523
+ }
524
+ context.drawImage(imageBitmap, 0, 0);
525
+ const imageData = context.getImageData(0, 0, imageBitmap.width, imageBitmap.height);
526
+ return {
527
+ imageBytes: new Uint8Array(imageData.data),
528
+ width: imageBitmap.width,
529
+ height: imageBitmap.height,
530
+ };
531
+ } finally {
532
+ imageBitmap.close?.();
533
+ }
534
+ }
535
+
536
+ async function resolveInferenceImagePayload(imageInput) {
537
+ if (!isPlainObject(imageInput)) {
538
+ throw new Error('inference image input must be an object.');
539
+ }
540
+
541
+ if (typeof imageInput.url === 'string' && imageInput.url.trim()) {
542
+ const decoded = await decodeImageUrlToPixels(imageInput.url.trim());
543
+ return {
544
+ imageBytes: decoded.imageBytes,
545
+ width: decoded.width,
546
+ height: decoded.height,
547
+ descriptor: {
548
+ source: 'url',
549
+ width: decoded.width,
550
+ height: decoded.height,
551
+ url: imageInput.url.trim(),
552
+ },
553
+ };
554
+ }
555
+
556
+ const width = Math.max(1, Math.floor(Number(imageInput.width)));
557
+ const height = Math.max(1, Math.floor(Number(imageInput.height)));
558
+ if (!Number.isFinite(width) || !Number.isFinite(height)) {
559
+ throw new Error('Raw inference image input requires positive integer width and height.');
560
+ }
561
+
562
+ if (typeof imageInput.pixelDataBase64 === 'string' && imageInput.pixelDataBase64.trim()) {
563
+ const decodedBytes = decodeBase64ToBytes(
564
+ imageInput.pixelDataBase64,
565
+ 'inferenceInput.image.pixelDataBase64'
566
+ );
567
+ assertRawImageByteLength(
568
+ decodedBytes,
569
+ width,
570
+ height,
571
+ 'inferenceInput.image.pixelDataBase64'
572
+ );
573
+ return {
574
+ imageBytes: decodedBytes,
575
+ width,
576
+ height,
577
+ descriptor: {
578
+ source: 'pixelDataBase64',
579
+ width,
580
+ height,
581
+ },
582
+ };
583
+ }
584
+
585
+ return {
586
+ imageBytes: normalizeRawImageBytes(
587
+ imageInput.pixels,
588
+ width,
589
+ height,
590
+ 'inferenceInput.image.pixels'
591
+ ),
592
+ width,
593
+ height,
594
+ descriptor: {
595
+ source: 'pixels',
596
+ width,
597
+ height,
598
+ },
599
+ };
600
+ }
601
+
366
602
  function resolveGenerationPromptInput(runtimeConfig, runOverrides = null, source = null) {
367
603
  const templateType = resolvePromptTemplateType(source);
368
604
  const overridePrompt = runOverrides?.prompt;
@@ -390,13 +626,34 @@ function resolveGenerationPromptInput(runtimeConfig, runOverrides = null, source
390
626
  }
391
627
 
392
628
  function resolveMaxTokens(runtimeConfig) {
393
- const runtimeMax = runtimeConfig?.inference?.batching?.maxTokens;
629
+ const runtimeMax = runtimeConfig?.inference?.generation?.maxTokens;
394
630
  if (Number.isFinite(runtimeMax)) {
395
631
  return Math.max(1, Math.floor(runtimeMax));
396
632
  }
397
633
  return DEFAULT_HARNESS_MAX_TOKENS;
398
634
  }
399
635
 
636
+ function resolveAutomaticGenerationDiagnostics(runtimeConfig, runOverrides = null) {
637
+ const overrideDiagnostics = runOverrides?.diagnostics ?? null;
638
+ if (overrideDiagnostics?.enabled === true) {
639
+ return overrideDiagnostics;
640
+ }
641
+
642
+ const diagnosticsPolicy = runtimeConfig?.shared?.tooling?.diagnostics ?? 'off';
643
+ if (diagnosticsPolicy !== 'always') {
644
+ return overrideDiagnostics;
645
+ }
646
+
647
+ return {
648
+ enabled: true,
649
+ captureConfig: {
650
+ ...createDefaultCaptureConfig(),
651
+ enabled: true,
652
+ defaultLevel: CAPTURE_LEVELS.NONE,
653
+ },
654
+ };
655
+ }
656
+
400
657
  export function resolveBenchmarkRunSettings(runtimeConfig, source = null) {
401
658
  const benchConfig = runtimeConfig?.shared?.benchmark?.run || {};
402
659
  const runtimeSampling = isPlainObject(runtimeConfig?.inference?.sampling)
@@ -732,6 +989,100 @@ function summarizeGenerationTokens(tokenRecords) {
732
989
  };
733
990
  }
734
991
 
992
+ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
993
+ const stats = typeof pipeline?.getStats === 'function'
994
+ ? (pipeline.getStats() || {})
995
+ : {};
996
+ const prefillMs = Number.isFinite(stats.prefillTimeMs) ? stats.prefillTimeMs : 0;
997
+ const ttftMs = Number.isFinite(stats.ttftMs) ? stats.ttftMs : prefillMs;
998
+ const decodeMs = Number.isFinite(stats.decodeTimeMs) ? stats.decodeTimeMs : 0;
999
+ const prefillTokens = Number.isFinite(stats.prefillTokens) ? stats.prefillTokens : 0;
1000
+ const decodeTokens = Number.isFinite(stats.decodeTokens)
1001
+ ? stats.decodeTokens
1002
+ : Math.max(0, tokenCount - 1);
1003
+ const decodeTokensPerSec = decodeMs > 0
1004
+ ? (decodeTokens / decodeMs) * 1000
1005
+ : 0;
1006
+ const prefillTokensPerSec = prefillMs > 0
1007
+ ? (prefillTokens / prefillMs) * 1000
1008
+ : 0;
1009
+ const prefillTokensPerSecTtft = ttftMs > 0
1010
+ ? (prefillTokens / ttftMs) * 1000
1011
+ : 0;
1012
+ const gpu = {};
1013
+ if (Number.isFinite(stats.gpuTimePrefillMs)) gpu.prefillMs = stats.gpuTimePrefillMs;
1014
+ if (Number.isFinite(stats.gpuTimeDecodeMs)) gpu.decodeMs = stats.gpuTimeDecodeMs;
1015
+ if (Number.isFinite(stats.decodeRecordMs)) gpu.decodeRecordMs = stats.decodeRecordMs;
1016
+ if (Number.isFinite(stats.decodeSubmitWaitMs)) gpu.decodeSubmitWaitMs = stats.decodeSubmitWaitMs;
1017
+ if (Number.isFinite(stats.decodeReadbackWaitMs)) gpu.decodeReadbackWaitMs = stats.decodeReadbackWaitMs;
1018
+ if (Number.isFinite(stats.singleTokenSubmitWaitMs)) gpu.singleTokenSubmitWaitMs = stats.singleTokenSubmitWaitMs;
1019
+ if (Number.isFinite(stats.singleTokenReadbackWaitMs)) gpu.singleTokenReadbackWaitMs = stats.singleTokenReadbackWaitMs;
1020
+ if (Number.isFinite(stats.singleTokenOrchestrationMs)) gpu.singleTokenOrchestrationMs = stats.singleTokenOrchestrationMs;
1021
+ const gpuPhase = Object.keys(gpu).length > 0 ? gpu : null;
1022
+ const batching = {};
1023
+ if (Number.isFinite(stats.batching?.batchedForwardCalls)) {
1024
+ batching.batchedForwardCalls = stats.batching.batchedForwardCalls;
1025
+ }
1026
+ if (Number.isFinite(stats.batching?.unbatchedForwardCalls)) {
1027
+ batching.unbatchedForwardCalls = stats.batching.unbatchedForwardCalls;
1028
+ }
1029
+ if (Number.isFinite(stats.batching?.totalBatchedTimeMs)) {
1030
+ batching.totalBatchedTimeMs = stats.batching.totalBatchedTimeMs;
1031
+ }
1032
+ if (Number.isFinite(stats.batching?.totalUnbatchedTimeMs)) {
1033
+ batching.totalUnbatchedTimeMs = stats.batching.totalUnbatchedTimeMs;
1034
+ }
1035
+ if (Number.isFinite(stats.batching?.gpuSubmissions)) {
1036
+ batching.gpuSubmissions = stats.batching.gpuSubmissions;
1037
+ }
1038
+ const batchingPhase = Object.keys(batching).length > 0 ? batching : null;
1039
+ const plePreparedTokenCache = {};
1040
+ if (Number.isFinite(stats.plePreparedTokenCacheHits)) {
1041
+ plePreparedTokenCache.hits = stats.plePreparedTokenCacheHits;
1042
+ }
1043
+ if (Number.isFinite(stats.plePreparedTokenCacheMisses)) {
1044
+ plePreparedTokenCache.misses = stats.plePreparedTokenCacheMisses;
1045
+ }
1046
+ if (Number.isFinite(stats.plePreparedTokenCacheEntries)) {
1047
+ plePreparedTokenCache.entries = stats.plePreparedTokenCacheEntries;
1048
+ }
1049
+ if (Number.isFinite(stats.plePreparedTokenCacheBytes)) {
1050
+ plePreparedTokenCache.bytes = stats.plePreparedTokenCacheBytes;
1051
+ }
1052
+ const plePreparedTokenCachePhase = Object.keys(plePreparedTokenCache).length > 0
1053
+ ? plePreparedTokenCache
1054
+ : null;
1055
+
1056
+ return {
1057
+ phase: {
1058
+ totalMs: Number.isFinite(stats.totalTimeMs) ? stats.totalTimeMs : durationMs,
1059
+ ttftMs,
1060
+ prefillMs,
1061
+ decodeMs,
1062
+ prefillTokens,
1063
+ decodeTokens,
1064
+ prefillTokensPerSec,
1065
+ prefillTokensPerSecTtft,
1066
+ decodeTokensPerSec,
1067
+ gpu: gpuPhase,
1068
+ prefillProfileSteps: Array.isArray(stats.prefillProfileSteps)
1069
+ ? stats.prefillProfileSteps
1070
+ : null,
1071
+ decodeProfileSteps: Array.isArray(stats.decodeProfileSteps)
1072
+ ? stats.decodeProfileSteps
1073
+ : null,
1074
+ decodeMode: stats.decodeMode ?? null,
1075
+ batchGuardReason: stats.batchGuardReason ?? null,
1076
+ batching: batchingPhase,
1077
+ plePreparedTokenCache: plePreparedTokenCachePhase,
1078
+ executionPlan: stats.executionPlan ?? null,
1079
+ kernelPathId: stats.kernelPathId ?? null,
1080
+ operatorDiagnostics: stats.operatorDiagnostics ?? null,
1081
+ kernelPathSource: stats.kernelPathSource ?? null,
1082
+ },
1083
+ };
1084
+ }
1085
+
735
1086
  export function isCoherentOutput(tokens, output) {
736
1087
  if (tokens.length === 0) return false;
737
1088
  const specialTokenCount = tokens.filter((t) => SPECIAL_TOKEN_RE.test(String(t).trim())).length;
@@ -756,17 +1107,19 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
756
1107
  const maxTokens = Number.isFinite(runOverrides?.maxTokens)
757
1108
  ? Math.max(1, Math.floor(runOverrides.maxTokens))
758
1109
  : resolveMaxTokens(runtimeConfig);
759
- const sampling = isPlainObject(runOverrides?.sampling)
760
- ? runOverrides.sampling
761
- : (runtimeConfig.inference?.sampling || {});
1110
+ const sampling = {
1111
+ ...DEFAULT_SAMPLING_DEFAULTS,
1112
+ ...(runtimeConfig.inference?.sampling || {}),
1113
+ ...(isPlainObject(runOverrides?.sampling) ? runOverrides.sampling : {}),
1114
+ };
762
1115
  const debugProbes = runtimeConfig.shared?.debug?.probes || [];
763
1116
  const profile = runtimeConfig.shared?.debug?.profiler?.enabled === true;
764
- const diagnosticsEnabled = runtimeConfig.shared?.harness?.mode === 'diagnose'
1117
+ const explicitDiagnosticsEnabled = runtimeConfig.shared?.harness?.mode === 'diagnose'
765
1118
  || runOverrides?.diagnostics?.enabled === true;
766
- const disableCommandBatching = diagnosticsEnabled
1119
+ const disableCommandBatching = explicitDiagnosticsEnabled
767
1120
  || (Array.isArray(debugProbes) && debugProbes.length > 0);
768
1121
  const start = performance.now();
769
- const diagnostics = runOverrides?.diagnostics ?? null;
1122
+ const diagnostics = resolveAutomaticGenerationDiagnostics(runtimeConfig, runOverrides);
770
1123
 
771
1124
  for await (const tokenText of pipeline.generate(promptInput, {
772
1125
  maxTokens,
@@ -795,38 +1148,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
795
1148
 
796
1149
  const durationMs = Math.max(1, performance.now() - start);
797
1150
  const tokensPerSec = (tokens.length / durationMs) * 1000;
798
- const stats = typeof pipeline?.getStats === 'function'
799
- ? (pipeline.getStats() || {})
800
- : {};
801
- const prefillMs = Number.isFinite(stats.prefillTimeMs) ? stats.prefillTimeMs : 0;
802
- const ttftMs = Number.isFinite(stats.ttftMs) ? stats.ttftMs : prefillMs;
803
- const decodeMs = Number.isFinite(stats.decodeTimeMs) ? stats.decodeTimeMs : 0;
804
- const prefillTokens = Number.isFinite(stats.prefillTokens) ? stats.prefillTokens : 0;
805
- const decodeTokens = Number.isFinite(stats.decodeTokens)
806
- ? stats.decodeTokens
807
- : Math.max(0, tokens.length - 1);
808
- const decodeTokensPerSec = decodeMs > 0
809
- ? (decodeTokens / decodeMs) * 1000
810
- : 0;
811
- const prefillTokensPerSec = prefillMs > 0
812
- ? (prefillTokens / prefillMs) * 1000
813
- : 0;
814
- const prefillTokensPerSecTtft = ttftMs > 0
815
- ? (prefillTokens / ttftMs) * 1000
816
- : 0;
817
- const gpu = {};
818
- if (Number.isFinite(stats.gpuTimePrefillMs)) gpu.prefillMs = stats.gpuTimePrefillMs;
819
- if (Number.isFinite(stats.gpuTimeDecodeMs)) gpu.decodeMs = stats.gpuTimeDecodeMs;
820
- if (Number.isFinite(stats.decodeRecordMs)) gpu.decodeRecordMs = stats.decodeRecordMs;
821
- if (Number.isFinite(stats.decodeSubmitWaitMs)) gpu.decodeSubmitWaitMs = stats.decodeSubmitWaitMs;
822
- if (Number.isFinite(stats.decodeReadbackWaitMs)) gpu.decodeReadbackWaitMs = stats.decodeReadbackWaitMs;
823
- if (Number.isFinite(stats.singleTokenSubmitWaitMs)) gpu.singleTokenSubmitWaitMs = stats.singleTokenSubmitWaitMs;
824
- if (Number.isFinite(stats.singleTokenReadbackWaitMs)) gpu.singleTokenReadbackWaitMs = stats.singleTokenReadbackWaitMs;
825
- if (Number.isFinite(stats.singleTokenOrchestrationMs)) gpu.singleTokenOrchestrationMs = stats.singleTokenOrchestrationMs;
826
- const gpuPhase = Object.keys(gpu).length > 0 ? gpu : null;
827
- const decodeProfileSteps = Array.isArray(stats.decodeProfileSteps)
828
- ? stats.decodeProfileSteps
829
- : null;
1151
+ const { phase } = buildGenerationPhaseFromStats(pipeline, durationMs, tokenIds.length);
830
1152
 
831
1153
  return {
832
1154
  prompt: promptLabel,
@@ -838,28 +1160,77 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
838
1160
  output: tokens.join(''),
839
1161
  durationMs,
840
1162
  tokensPerSec,
841
- phase: {
842
- totalMs: Number.isFinite(stats.totalTimeMs) ? stats.totalTimeMs : durationMs,
843
- ttftMs,
844
- prefillMs,
845
- decodeMs,
846
- prefillTokens,
847
- decodeTokens,
848
- prefillTokensPerSec,
849
- prefillTokensPerSecTtft,
850
- decodeTokensPerSec,
851
- gpu: gpuPhase,
852
- decodeProfileSteps,
853
- decodeMode: stats.decodeMode ?? null,
854
- batchGuardReason: stats.batchGuardReason ?? null,
855
- executionPlan: stats.executionPlan ?? null,
856
- kernelPathId: stats.kernelPathId ?? null,
857
- operatorDiagnostics: stats.operatorDiagnostics ?? null,
858
- kernelPathSource: stats.kernelPathSource ?? null,
1163
+ phase,
1164
+ };
1165
+ }
1166
+
1167
+ export async function runImageTranscription(pipeline, runtimeConfig, runOverrides = null) {
1168
+ const imageInput = runOverrides?.image;
1169
+ if (!isPlainObject(imageInput)) {
1170
+ throw new Error('Image transcription requires inferenceInput.image.');
1171
+ }
1172
+ const prompt = typeof runOverrides?.prompt === 'string' && runOverrides.prompt.trim()
1173
+ ? runOverrides.prompt.trim()
1174
+ : DEFAULT_IMAGE_TRANSCRIPTION_PROMPT;
1175
+ const maxTokens = Number.isFinite(runOverrides?.maxTokens)
1176
+ ? Math.max(1, Math.floor(runOverrides.maxTokens))
1177
+ : resolveMaxTokens(runtimeConfig);
1178
+ const softTokenBudget = Number.isFinite(runOverrides?.softTokenBudget)
1179
+ ? Math.max(1, Math.floor(runOverrides.softTokenBudget))
1180
+ : DEFAULT_IMAGE_TRANSCRIPTION_SOFT_TOKEN_BUDGET;
1181
+ const {
1182
+ imageBytes,
1183
+ width,
1184
+ height,
1185
+ descriptor,
1186
+ } = await resolveInferenceImagePayload(imageInput);
1187
+ const start = performance.now();
1188
+ const result = await pipeline.transcribeImage({
1189
+ imageBytes,
1190
+ width,
1191
+ height,
1192
+ prompt,
1193
+ maxTokens,
1194
+ softTokenBudget,
1195
+ });
1196
+ const durationMs = Math.max(1, performance.now() - start);
1197
+ const tokenIds = Array.isArray(result?.tokens)
1198
+ ? result.tokens.map((value) => Number(value)).filter((value) => Number.isInteger(value))
1199
+ : [];
1200
+ const tokenRecords = tokenIds.map((tokenId) => {
1201
+ const decoded = pipeline?.tokenizer?.decode?.([tokenId], false, false) ?? '';
1202
+ return {
1203
+ id: tokenId,
1204
+ text: decoded,
1205
+ fallbackText: decoded,
1206
+ };
1207
+ });
1208
+ const { phase } = buildGenerationPhaseFromStats(pipeline, durationMs, tokenIds.length);
1209
+ return {
1210
+ inputMode: 'image_to_text',
1211
+ prompt: `image ${width}x${height}: ${prompt}`,
1212
+ promptInput: {
1213
+ prompt,
1214
+ image: descriptor,
859
1215
  },
1216
+ maxTokens,
1217
+ tokens: tokenRecords.map((record) => record.text),
1218
+ tokenIds,
1219
+ tokenDiagnostics: summarizeGenerationTokens(tokenRecords),
1220
+ output: typeof result?.text === 'string' ? result.text : tokenRecords.map((record) => record.text).join(''),
1221
+ durationMs,
1222
+ tokensPerSec: tokenIds.length > 0 ? (tokenIds.length / durationMs) * 1000 : 0,
1223
+ phase,
860
1224
  };
861
1225
  }
862
1226
 
1227
+ export async function runTextInference(pipeline, runtimeConfig, runOverrides = null) {
1228
+ if (isPlainObject(runOverrides?.image)) {
1229
+ return runImageTranscription(pipeline, runtimeConfig, runOverrides);
1230
+ }
1231
+ return runGeneration(pipeline, runtimeConfig, runOverrides);
1232
+ }
1233
+
863
1234
  export async function runEmbedding(pipeline, runtimeConfig, runOverrides = null) {
864
1235
  warnIfUsingDefaults(runtimeConfig);
865
1236
  const prompt = typeof runOverrides?.prompt === 'string' && runOverrides.prompt.trim()