@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (701) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +83 -0
  3. package/package.json +82 -15
  4. package/src/actions/generate-media.d.ts +59 -0
  5. package/src/actions/generate-media.d.ts.map +1 -0
  6. package/src/actions/generate-media.ts +647 -0
  7. package/src/actions/identify-speaker.d.ts +23 -0
  8. package/src/actions/identify-speaker.d.ts.map +1 -0
  9. package/src/actions/identify-speaker.ts +171 -0
  10. package/src/actions/transcription-control.d.ts +29 -0
  11. package/src/actions/transcription-control.d.ts.map +1 -0
  12. package/src/actions/transcription-control.test.ts +100 -0
  13. package/src/actions/transcription-control.ts +127 -0
  14. package/src/adapters/capacitor-llama/__tests__/compat-behavior.test.ts +218 -0
  15. package/src/adapters/capacitor-llama/__tests__/index.test.ts +68 -0
  16. package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts +215 -0
  17. package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts +174 -0
  18. package/src/adapters/capacitor-llama/environment.ts +71 -0
  19. package/src/adapters/capacitor-llama/index.browser.ts +83 -0
  20. package/src/adapters/capacitor-llama/index.ts +807 -0
  21. package/src/adapters/capacitor-llama/loader.ts +109 -0
  22. package/src/adapters/capacitor-llama/structured-output.ts +165 -0
  23. package/src/adapters/capacitor-llama/text-streaming.ts +227 -0
  24. package/src/adapters/capacitor-llama/types.ts +374 -0
  25. package/src/backends/apple-foundation.ts +127 -0
  26. package/src/index.d.ts +8 -0
  27. package/src/index.d.ts.map +1 -0
  28. package/src/index.ts +62 -0
  29. package/src/local-inference-routes.d.ts +38 -0
  30. package/src/local-inference-routes.d.ts.map +1 -0
  31. package/src/local-inference-routes.test.ts +344 -0
  32. package/src/local-inference-routes.ts +1543 -0
  33. package/src/provider.d.ts +21 -0
  34. package/src/provider.d.ts.map +1 -0
  35. package/src/provider.ts +1082 -0
  36. package/src/routes/compat-helpers.d.ts +18 -0
  37. package/src/routes/compat-helpers.d.ts.map +1 -0
  38. package/src/routes/compat-helpers.ts +274 -0
  39. package/src/routes/family-member-route.d.ts +62 -0
  40. package/src/routes/family-member-route.d.ts.map +1 -0
  41. package/src/routes/family-member-route.ts +353 -0
  42. package/src/routes/index.d.ts +19 -0
  43. package/src/routes/index.d.ts.map +1 -0
  44. package/src/routes/index.ts +60 -0
  45. package/src/routes/live-diarization-route.d.ts +26 -0
  46. package/src/routes/live-diarization-route.d.ts.map +1 -0
  47. package/src/routes/live-diarization-route.test.ts +213 -0
  48. package/src/routes/live-diarization-route.ts +122 -0
  49. package/src/routes/local-inference-asr-route.d.ts +4 -0
  50. package/src/routes/local-inference-asr-route.d.ts.map +1 -0
  51. package/src/routes/local-inference-asr-route.test.ts +205 -0
  52. package/src/routes/local-inference-asr-route.ts +163 -0
  53. package/src/routes/local-inference-asr-transcribe.d.ts +20 -0
  54. package/src/routes/local-inference-asr-transcribe.d.ts.map +1 -0
  55. package/src/routes/local-inference-asr-transcribe.test.ts +118 -0
  56. package/src/routes/local-inference-asr-transcribe.ts +97 -0
  57. package/src/routes/local-inference-compat-routes.d.ts +16 -0
  58. package/src/routes/local-inference-compat-routes.d.ts.map +1 -0
  59. package/src/routes/local-inference-compat-routes.test.ts +485 -0
  60. package/src/routes/local-inference-compat-routes.ts +808 -0
  61. package/src/routes/local-inference-tts-route.d.ts +7 -0
  62. package/src/routes/local-inference-tts-route.d.ts.map +1 -0
  63. package/src/routes/local-inference-tts-route.test.ts +179 -0
  64. package/src/routes/local-inference-tts-route.ts +230 -0
  65. package/src/routes/transcript-audio-store.d.ts +15 -0
  66. package/src/routes/transcript-audio-store.d.ts.map +1 -0
  67. package/src/routes/transcript-audio-store.ts +27 -0
  68. package/src/routes/transcripts-routes.d.ts +36 -0
  69. package/src/routes/transcripts-routes.d.ts.map +1 -0
  70. package/src/routes/transcripts-routes.test.ts +144 -0
  71. package/src/routes/transcripts-routes.ts +159 -0
  72. package/src/routes/voice-first-run-routes.d.ts +62 -0
  73. package/src/routes/voice-first-run-routes.d.ts.map +1 -0
  74. package/src/routes/voice-first-run-routes.ts +524 -0
  75. package/src/routes/voice-models-routes.d.ts +62 -0
  76. package/src/routes/voice-models-routes.d.ts.map +1 -0
  77. package/src/routes/voice-models-routes.ts +554 -0
  78. package/src/routes/voice-profile-plugin-routes.d.ts +19 -0
  79. package/src/routes/voice-profile-plugin-routes.d.ts.map +1 -0
  80. package/src/routes/voice-profile-plugin-routes.ts +138 -0
  81. package/src/routes/voice-profiles-management-routes.d.ts +52 -0
  82. package/src/routes/voice-profiles-management-routes.d.ts.map +1 -0
  83. package/src/routes/voice-profiles-management-routes.ts +476 -0
  84. package/src/routes/voice-speaker-profile-routes.d.ts +57 -0
  85. package/src/routes/voice-speaker-profile-routes.d.ts.map +1 -0
  86. package/src/routes/voice-speaker-profile-routes.ts +199 -0
  87. package/src/runtime/aosp-llama-loader-selection.test.ts +80 -0
  88. package/src/runtime/capacitor-llama.d.ts +25 -0
  89. package/src/runtime/embedding-manager-support.d.ts +77 -0
  90. package/src/runtime/embedding-manager-support.d.ts.map +1 -0
  91. package/src/runtime/embedding-manager-support.ts +497 -0
  92. package/src/runtime/embedding-presets.d.ts +16 -0
  93. package/src/runtime/embedding-presets.d.ts.map +1 -0
  94. package/src/runtime/embedding-presets.ts +81 -0
  95. package/src/runtime/embedding-warmup-policy.d.ts +14 -0
  96. package/src/runtime/embedding-warmup-policy.d.ts.map +1 -0
  97. package/src/runtime/embedding-warmup-policy.test.ts +53 -0
  98. package/src/runtime/embedding-warmup-policy.ts +48 -0
  99. package/src/runtime/ensure-local-inference-handler.d.ts +62 -0
  100. package/src/runtime/ensure-local-inference-handler.d.ts.map +1 -0
  101. package/src/runtime/ensure-local-inference-handler.test.ts +528 -0
  102. package/src/runtime/ensure-local-inference-handler.ts +1448 -0
  103. package/src/runtime/index.d.ts +15 -0
  104. package/src/runtime/index.d.ts.map +1 -0
  105. package/src/runtime/index.ts +33 -0
  106. package/src/runtime/mobile-local-inference-gate.d.ts +31 -0
  107. package/src/runtime/mobile-local-inference-gate.d.ts.map +1 -0
  108. package/src/runtime/mobile-local-inference-gate.test.ts +69 -0
  109. package/src/runtime/mobile-local-inference-gate.ts +44 -0
  110. package/src/runtime/voice-entity-binding.d.ts +103 -0
  111. package/src/runtime/voice-entity-binding.d.ts.map +1 -0
  112. package/src/runtime/voice-entity-binding.transcript.test.ts +69 -0
  113. package/src/runtime/voice-entity-binding.ts +328 -0
  114. package/src/services/README.md +71 -0
  115. package/src/services/__tests__/backend-selector.test.ts +101 -0
  116. package/src/services/__tests__/checkpoint-manager.test.ts +376 -0
  117. package/src/services/__tests__/gpu-autotune.test.ts +400 -0
  118. package/src/services/__tests__/llm-streaming-binding.test.ts +85 -0
  119. package/src/services/__tests__/planner-grammar.test.ts +372 -0
  120. package/src/services/__tests__/runtime-target.test.ts +176 -0
  121. package/src/services/active-model-switch-rollback.test.ts +183 -0
  122. package/src/services/active-model.d.ts +282 -0
  123. package/src/services/active-model.d.ts.map +1 -0
  124. package/src/services/active-model.ts +1213 -0
  125. package/src/services/assignments.d.ts +71 -0
  126. package/src/services/assignments.d.ts.map +1 -0
  127. package/src/services/assignments.test.ts +80 -0
  128. package/src/services/assignments.ts +230 -0
  129. package/src/services/backend-selector.ts +95 -0
  130. package/src/services/backend.d.ts +346 -0
  131. package/src/services/backend.d.ts.map +1 -0
  132. package/src/services/backend.ts +612 -0
  133. package/src/services/bionic-host-loader.d.ts +46 -0
  134. package/src/services/bionic-host-loader.d.ts.map +1 -0
  135. package/src/services/bionic-host-loader.test.ts +133 -0
  136. package/src/services/bionic-host-loader.ts +180 -0
  137. package/src/services/bundled-models.d.ts +34 -0
  138. package/src/services/bundled-models.d.ts.map +1 -0
  139. package/src/services/bundled-models.ts +129 -0
  140. package/src/services/cache-bridge.d.ts +206 -0
  141. package/src/services/cache-bridge.d.ts.map +1 -0
  142. package/src/services/cache-bridge.test.ts +516 -0
  143. package/src/services/cache-bridge.ts +423 -0
  144. package/src/services/catalog.d.ts +10 -0
  145. package/src/services/catalog.d.ts.map +1 -0
  146. package/src/services/catalog.test.ts +238 -0
  147. package/src/services/catalog.ts +27 -0
  148. package/src/services/checkpoint-client.d.ts +109 -0
  149. package/src/services/checkpoint-client.d.ts.map +1 -0
  150. package/src/services/checkpoint-client.ts +258 -0
  151. package/src/services/checkpoint-manager.ts +474 -0
  152. package/src/services/cloud-fallback.d.ts +102 -0
  153. package/src/services/cloud-fallback.d.ts.map +1 -0
  154. package/src/services/cloud-fallback.ts +230 -0
  155. package/src/services/conversation-registry.d.ts +142 -0
  156. package/src/services/conversation-registry.d.ts.map +1 -0
  157. package/src/services/conversation-registry.test.ts +235 -0
  158. package/src/services/conversation-registry.ts +264 -0
  159. package/src/services/desktop-fused-ffi-backend-runtime.d.ts +95 -0
  160. package/src/services/desktop-fused-ffi-backend-runtime.d.ts.map +1 -0
  161. package/src/services/desktop-fused-ffi-backend-runtime.ts +339 -0
  162. package/src/services/device-bridge.d.ts +188 -0
  163. package/src/services/device-bridge.d.ts.map +1 -0
  164. package/src/services/device-bridge.ts +1237 -0
  165. package/src/services/device-resource-metrics.d.ts +149 -0
  166. package/src/services/device-resource-metrics.d.ts.map +1 -0
  167. package/src/services/device-resource-metrics.test.ts +98 -0
  168. package/src/services/device-resource-metrics.ts +346 -0
  169. package/src/services/device-tier.d.ts +115 -0
  170. package/src/services/device-tier.d.ts.map +1 -0
  171. package/src/services/device-tier.test.ts +371 -0
  172. package/src/services/device-tier.ts +410 -0
  173. package/src/services/downloader.d.ts +82 -0
  174. package/src/services/downloader.d.ts.map +1 -0
  175. package/src/services/downloader.test.ts +747 -0
  176. package/src/services/downloader.ts +925 -0
  177. package/src/services/engine-direct-bundle.test.ts +58 -0
  178. package/src/services/engine-streaming.test.ts +80 -0
  179. package/src/services/engine.d.ts +540 -0
  180. package/src/services/engine.d.ts.map +1 -0
  181. package/src/services/engine.ts +1909 -0
  182. package/src/services/ensure-local-artifacts.integration.test.ts +273 -0
  183. package/src/services/ensure-local-artifacts.test.ts +368 -0
  184. package/src/services/ensure-local-artifacts.ts +351 -0
  185. package/src/services/external-scanner.d.ts +17 -0
  186. package/src/services/external-scanner.d.ts.map +1 -0
  187. package/src/services/external-scanner.ts +312 -0
  188. package/src/services/ffi-llm-mock.ts +354 -0
  189. package/src/services/ffi-llm-streaming-abi.ts +442 -0
  190. package/src/services/ffi-streaming-backend.d.ts +180 -0
  191. package/src/services/ffi-streaming-backend.d.ts.map +1 -0
  192. package/src/services/ffi-streaming-backend.ts +382 -0
  193. package/src/services/ffi-streaming-runner.d.ts +122 -0
  194. package/src/services/ffi-streaming-runner.d.ts.map +1 -0
  195. package/src/services/ffi-streaming-runner.test.ts +60 -0
  196. package/src/services/ffi-streaming-runner.ts +354 -0
  197. package/src/services/ffi-unload-ordering.test.ts +162 -0
  198. package/src/services/gpu-autotune.ts +534 -0
  199. package/src/services/gpu-detect.d.ts +56 -0
  200. package/src/services/gpu-detect.d.ts.map +1 -0
  201. package/src/services/gpu-detect.ts +139 -0
  202. package/src/services/handler-registry.d.ts +72 -0
  203. package/src/services/handler-registry.d.ts.map +1 -0
  204. package/src/services/handler-registry.ts +240 -0
  205. package/src/services/hardware.d.ts +63 -0
  206. package/src/services/hardware.d.ts.map +1 -0
  207. package/src/services/hardware.test.ts +231 -0
  208. package/src/services/hardware.ts +410 -0
  209. package/src/services/hf-search.d.ts +26 -0
  210. package/src/services/hf-search.d.ts.map +1 -0
  211. package/src/services/hf-search.test.ts +69 -0
  212. package/src/services/hf-search.ts +420 -0
  213. package/src/services/image-description-runtime.d.ts +14 -0
  214. package/src/services/image-description-runtime.d.ts.map +1 -0
  215. package/src/services/image-description-runtime.test.ts +61 -0
  216. package/src/services/image-description-runtime.ts +118 -0
  217. package/src/services/imagegen/aosp-unavailable.d.ts +134 -0
  218. package/src/services/imagegen/aosp-unavailable.d.ts.map +1 -0
  219. package/src/services/imagegen/aosp-unavailable.ts +229 -0
  220. package/src/services/imagegen/backend-selector.d.ts +118 -0
  221. package/src/services/imagegen/backend-selector.d.ts.map +1 -0
  222. package/src/services/imagegen/backend-selector.ts +277 -0
  223. package/src/services/imagegen/coreml-unavailable.d.ts +105 -0
  224. package/src/services/imagegen/coreml-unavailable.d.ts.map +1 -0
  225. package/src/services/imagegen/coreml-unavailable.ts +237 -0
  226. package/src/services/imagegen/errors.d.ts +16 -0
  227. package/src/services/imagegen/errors.d.ts.map +1 -0
  228. package/src/services/imagegen/errors.ts +40 -0
  229. package/src/services/imagegen/index.d.ts +58 -0
  230. package/src/services/imagegen/index.d.ts.map +1 -0
  231. package/src/services/imagegen/index.ts +144 -0
  232. package/src/services/imagegen/mflux.d.ts +74 -0
  233. package/src/services/imagegen/mflux.d.ts.map +1 -0
  234. package/src/services/imagegen/mflux.ts +313 -0
  235. package/src/services/imagegen/sd-cpp.d.ts +180 -0
  236. package/src/services/imagegen/sd-cpp.d.ts.map +1 -0
  237. package/src/services/imagegen/sd-cpp.ts +718 -0
  238. package/src/services/imagegen/tensorrt-unavailable.d.ts +83 -0
  239. package/src/services/imagegen/tensorrt-unavailable.d.ts.map +1 -0
  240. package/src/services/imagegen/tensorrt-unavailable.ts +295 -0
  241. package/src/services/imagegen/types.d.ts +181 -0
  242. package/src/services/imagegen/types.d.ts.map +1 -0
  243. package/src/services/imagegen/types.ts +193 -0
  244. package/src/services/index.d.ts +29 -0
  245. package/src/services/index.d.ts.map +1 -0
  246. package/src/services/index.ts +211 -0
  247. package/src/services/inference-capabilities.d.ts +132 -0
  248. package/src/services/inference-capabilities.d.ts.map +1 -0
  249. package/src/services/inference-capabilities.test.ts +75 -0
  250. package/src/services/inference-capabilities.ts +204 -0
  251. package/src/services/inference-telemetry.d.ts +59 -0
  252. package/src/services/inference-telemetry.d.ts.map +1 -0
  253. package/src/services/inference-telemetry.ts +143 -0
  254. package/src/services/ios-llama-streaming.ts +248 -0
  255. package/src/services/kv-spill.d.ts +189 -0
  256. package/src/services/kv-spill.d.ts.map +1 -0
  257. package/src/services/kv-spill.test.ts +222 -0
  258. package/src/services/kv-spill.ts +356 -0
  259. package/src/services/latency-trace.d.ts +346 -0
  260. package/src/services/latency-trace.d.ts.map +1 -0
  261. package/src/services/latency-trace.test.ts +266 -0
  262. package/src/services/latency-trace.ts +844 -0
  263. package/src/services/llama-server-metrics.ts +304 -0
  264. package/src/services/llm-streaming-binding.d.ts +96 -0
  265. package/src/services/llm-streaming-binding.d.ts.map +1 -0
  266. package/src/services/llm-streaming-binding.ts +136 -0
  267. package/src/services/load-args.d.ts +82 -0
  268. package/src/services/load-args.d.ts.map +1 -0
  269. package/src/services/load-args.ts +81 -0
  270. package/src/services/manifest/eliza-1.manifest.v1.json +708 -0
  271. package/src/services/manifest/index.d.ts +4 -0
  272. package/src/services/manifest/index.d.ts.map +1 -0
  273. package/src/services/manifest/index.ts +66 -0
  274. package/src/services/manifest/manifest.test.ts +689 -0
  275. package/src/services/manifest/schema.d.ts +713 -0
  276. package/src/services/manifest/schema.d.ts.map +1 -0
  277. package/src/services/manifest/schema.ts +653 -0
  278. package/src/services/manifest/types.d.ts +30 -0
  279. package/src/services/manifest/types.d.ts.map +1 -0
  280. package/src/services/manifest/types.ts +55 -0
  281. package/src/services/manifest/validator.d.ts +66 -0
  282. package/src/services/manifest/validator.d.ts.map +1 -0
  283. package/src/services/manifest/validator.ts +567 -0
  284. package/src/services/memory-arbiter.d.ts +318 -0
  285. package/src/services/memory-arbiter.d.ts.map +1 -0
  286. package/src/services/memory-arbiter.test.ts +419 -0
  287. package/src/services/memory-arbiter.ts +925 -0
  288. package/src/services/memory-monitor.d.ts +122 -0
  289. package/src/services/memory-monitor.d.ts.map +1 -0
  290. package/src/services/memory-monitor.test.ts +208 -0
  291. package/src/services/memory-monitor.ts +297 -0
  292. package/src/services/memory-pressure.d.ts +130 -0
  293. package/src/services/memory-pressure.d.ts.map +1 -0
  294. package/src/services/memory-pressure.ts +414 -0
  295. package/src/services/mtp-doctor.d.ts +13 -0
  296. package/src/services/mtp-doctor.d.ts.map +1 -0
  297. package/src/services/mtp-doctor.ts +78 -0
  298. package/src/services/network-policy.d.ts +127 -0
  299. package/src/services/network-policy.d.ts.map +1 -0
  300. package/src/services/network-policy.ts +346 -0
  301. package/src/services/paths.d.ts +6 -0
  302. package/src/services/paths.d.ts.map +1 -0
  303. package/src/services/paths.ts +25 -0
  304. package/src/services/planner-skeleton.d.ts +124 -0
  305. package/src/services/planner-skeleton.d.ts.map +1 -0
  306. package/src/services/planner-skeleton.ts +175 -0
  307. package/src/services/providers.d.ts +38 -0
  308. package/src/services/providers.d.ts.map +1 -0
  309. package/src/services/providers.ts +507 -0
  310. package/src/services/ram-budget-cache.test.ts +163 -0
  311. package/src/services/ram-budget.d.ts +110 -0
  312. package/src/services/ram-budget.d.ts.map +1 -0
  313. package/src/services/ram-budget.ts +0 -0
  314. package/src/services/readiness.d.ts +9 -0
  315. package/src/services/readiness.d.ts.map +1 -0
  316. package/src/services/readiness.test.ts +87 -0
  317. package/src/services/readiness.ts +238 -0
  318. package/src/services/recommendation.d.ts +111 -0
  319. package/src/services/recommendation.d.ts.map +1 -0
  320. package/src/services/recommendation.ts +671 -0
  321. package/src/services/registry.d.ts +35 -0
  322. package/src/services/registry.d.ts.map +1 -0
  323. package/src/services/registry.ts +151 -0
  324. package/src/services/router-handler.d.ts +92 -0
  325. package/src/services/router-handler.d.ts.map +1 -0
  326. package/src/services/router-handler.test.ts +45 -0
  327. package/src/services/router-handler.ts +407 -0
  328. package/src/services/routing-policy.d.ts +69 -0
  329. package/src/services/routing-policy.d.ts.map +1 -0
  330. package/src/services/routing-policy.test.ts +164 -0
  331. package/src/services/routing-policy.ts +297 -0
  332. package/src/services/routing-preferences.d.ts +8 -0
  333. package/src/services/routing-preferences.d.ts.map +1 -0
  334. package/src/services/routing-preferences.ts +17 -0
  335. package/src/services/runtime-target.d.ts +98 -0
  336. package/src/services/runtime-target.d.ts.map +1 -0
  337. package/src/services/runtime-target.ts +154 -0
  338. package/src/services/service.d.ts +128 -0
  339. package/src/services/service.d.ts.map +1 -0
  340. package/src/services/service.test.ts +223 -0
  341. package/src/services/service.ts +735 -0
  342. package/src/services/session-pool.d.ts +72 -0
  343. package/src/services/session-pool.d.ts.map +1 -0
  344. package/src/services/session-pool.ts +153 -0
  345. package/src/services/structured-output/deterministic-repair.d.ts +23 -0
  346. package/src/services/structured-output/deterministic-repair.d.ts.map +1 -0
  347. package/src/services/structured-output/deterministic-repair.test.ts +169 -0
  348. package/src/services/structured-output/deterministic-repair.ts +443 -0
  349. package/src/services/structured-output/index.ts +4 -0
  350. package/src/services/structured-output.d.ts +311 -0
  351. package/src/services/structured-output.d.ts.map +1 -0
  352. package/src/services/structured-output.test.ts +483 -0
  353. package/src/services/structured-output.ts +712 -0
  354. package/src/services/system-memory.d.ts +33 -0
  355. package/src/services/system-memory.d.ts.map +1 -0
  356. package/src/services/system-memory.test.ts +47 -0
  357. package/src/services/system-memory.ts +67 -0
  358. package/src/services/transcription-priority.test.ts +211 -0
  359. package/src/services/types.d.ts +19 -0
  360. package/src/services/types.d.ts.map +1 -0
  361. package/src/services/types.ts +55 -0
  362. package/src/services/verify-on-device.d.ts +34 -0
  363. package/src/services/verify-on-device.d.ts.map +1 -0
  364. package/src/services/verify-on-device.test.ts +87 -0
  365. package/src/services/verify-on-device.ts +127 -0
  366. package/src/services/verify.d.ts +8 -0
  367. package/src/services/verify.d.ts.map +1 -0
  368. package/src/services/verify.ts +13 -0
  369. package/src/services/vision/aosp-unavailable.d.ts +115 -0
  370. package/src/services/vision/aosp-unavailable.d.ts.map +1 -0
  371. package/src/services/vision/aosp-unavailable.ts +163 -0
  372. package/src/services/vision/capacitor-llama.d.ts +99 -0
  373. package/src/services/vision/capacitor-llama.d.ts.map +1 -0
  374. package/src/services/vision/capacitor-llama.ts +255 -0
  375. package/src/services/vision/cloud-fallback.d.ts +47 -0
  376. package/src/services/vision/cloud-fallback.d.ts.map +1 -0
  377. package/src/services/vision/cloud-fallback.test.ts +243 -0
  378. package/src/services/vision/cloud-fallback.ts +268 -0
  379. package/src/services/vision/fallback-chain.test.ts +86 -0
  380. package/src/services/vision/hash.d.ts +71 -0
  381. package/src/services/vision/hash.d.ts.map +1 -0
  382. package/src/services/vision/hash.ts +157 -0
  383. package/src/services/vision/index.d.ts +95 -0
  384. package/src/services/vision/index.d.ts.map +1 -0
  385. package/src/services/vision/index.ts +251 -0
  386. package/src/services/vision/llama-server.d.ts +73 -0
  387. package/src/services/vision/llama-server.d.ts.map +1 -0
  388. package/src/services/vision/llama-server.ts +177 -0
  389. package/src/services/vision/types.d.ts +153 -0
  390. package/src/services/vision/types.d.ts.map +1 -0
  391. package/src/services/vision/types.ts +154 -0
  392. package/src/services/vision/vast-fallback.d.ts +18 -0
  393. package/src/services/vision/vast-fallback.d.ts.map +1 -0
  394. package/src/services/vision/vast-fallback.ts +127 -0
  395. package/src/services/vision-embedding-cache.d.ts +98 -0
  396. package/src/services/vision-embedding-cache.d.ts.map +1 -0
  397. package/src/services/vision-embedding-cache.ts +189 -0
  398. package/src/services/voice/VOICE_WORKBENCH.md +88 -0
  399. package/src/services/voice/__test-helpers__/fake-ffi.ts +94 -0
  400. package/src/services/voice/__test-helpers__/synthetic-speech.ts +124 -0
  401. package/src/services/voice/__tests__/checkpoint-manager.test.ts +241 -0
  402. package/src/services/voice/__tests__/checkpoint-policy.test.ts +270 -0
  403. package/src/services/voice/__tests__/eager-context-builder.test.ts +257 -0
  404. package/src/services/voice/__tests__/eliza1-eot-scorer.test.ts +288 -0
  405. package/src/services/voice/__tests__/eot-classifier.test.ts +431 -0
  406. package/src/services/voice/__tests__/optimistic-rollback.test.ts +312 -0
  407. package/src/services/voice/__tests__/prefill-client.test.ts +266 -0
  408. package/src/services/voice/__tests__/prefix-preserving-queue.test.ts +208 -0
  409. package/src/services/voice/__tests__/streaming-asr.test.ts +450 -0
  410. package/src/services/voice/__tests__/streaming-transcriber.test.ts +339 -0
  411. package/src/services/voice/__tests__/turn-detector-resolver.test.ts +195 -0
  412. package/src/services/voice/__tests__/voice-state-machine-prefill.test.ts +275 -0
  413. package/src/services/voice/__tests__/voice-state-machine.test.ts +354 -0
  414. package/src/services/voice/asr-timed.real.test.ts +141 -0
  415. package/src/services/voice/audio-frame-consumer.d.ts +212 -0
  416. package/src/services/voice/audio-frame-consumer.d.ts.map +1 -0
  417. package/src/services/voice/audio-frame-consumer.test.ts +343 -0
  418. package/src/services/voice/audio-frame-consumer.ts +491 -0
  419. package/src/services/voice/barge-in.d.ts +112 -0
  420. package/src/services/voice/barge-in.d.ts.map +1 -0
  421. package/src/services/voice/barge-in.test.ts +244 -0
  422. package/src/services/voice/barge-in.ts +336 -0
  423. package/src/services/voice/cancellation-coordinator.d.ts +127 -0
  424. package/src/services/voice/cancellation-coordinator.d.ts.map +1 -0
  425. package/src/services/voice/cancellation-coordinator.test.ts +196 -0
  426. package/src/services/voice/cancellation-coordinator.ts +269 -0
  427. package/src/services/voice/checkpoint-manager.d.ts +199 -0
  428. package/src/services/voice/checkpoint-manager.d.ts.map +1 -0
  429. package/src/services/voice/checkpoint-manager.ts +401 -0
  430. package/src/services/voice/checkpoint-policy.ts +336 -0
  431. package/src/services/voice/composite-eot-classifier.test.ts +59 -0
  432. package/src/services/voice/e2e-harness.test.ts +182 -0
  433. package/src/services/voice/e2e-harness.ts +743 -0
  434. package/src/services/voice/eager-context-builder.d.ts +170 -0
  435. package/src/services/voice/eager-context-builder.d.ts.map +1 -0
  436. package/src/services/voice/eager-context-builder.ts +262 -0
  437. package/src/services/voice/eliza1-eot-scorer.d.ts +124 -0
  438. package/src/services/voice/eliza1-eot-scorer.d.ts.map +1 -0
  439. package/src/services/voice/eliza1-eot-scorer.ts +242 -0
  440. package/src/services/voice/embedding-server.ts +200 -0
  441. package/src/services/voice/embedding.d.ts +133 -0
  442. package/src/services/voice/embedding.d.ts.map +1 -0
  443. package/src/services/voice/embedding.test.ts +131 -0
  444. package/src/services/voice/embedding.ts +243 -0
  445. package/src/services/voice/emotion-attribution.d.ts +68 -0
  446. package/src/services/voice/emotion-attribution.d.ts.map +1 -0
  447. package/src/services/voice/emotion-attribution.test.ts +129 -0
  448. package/src/services/voice/emotion-attribution.ts +361 -0
  449. package/src/services/voice/engine-bridge-cancellation.test.ts +422 -0
  450. package/src/services/voice/engine-bridge.d.ts +759 -0
  451. package/src/services/voice/engine-bridge.d.ts.map +1 -0
  452. package/src/services/voice/engine-bridge.test.ts +384 -0
  453. package/src/services/voice/engine-bridge.ts +2302 -0
  454. package/src/services/voice/eot-classifier-ggml.d.ts +179 -0
  455. package/src/services/voice/eot-classifier-ggml.d.ts.map +1 -0
  456. package/src/services/voice/eot-classifier-ggml.ts +566 -0
  457. package/src/services/voice/eot-classifier.d.ts +214 -0
  458. package/src/services/voice/eot-classifier.d.ts.map +1 -0
  459. package/src/services/voice/eot-classifier.ts +533 -0
  460. package/src/services/voice/errors.d.ts +20 -0
  461. package/src/services/voice/errors.d.ts.map +1 -0
  462. package/src/services/voice/errors.ts +32 -0
  463. package/src/services/voice/expressive-tags.d.ts +158 -0
  464. package/src/services/voice/expressive-tags.d.ts.map +1 -0
  465. package/src/services/voice/expressive-tags.ts +405 -0
  466. package/src/services/voice/ffi-bindings.d.ts +674 -0
  467. package/src/services/voice/ffi-bindings.d.ts.map +1 -0
  468. package/src/services/voice/ffi-bindings.test.ts +728 -0
  469. package/src/services/voice/ffi-bindings.ts +3225 -0
  470. package/src/services/voice/first-line-cache.d.ts +181 -0
  471. package/src/services/voice/first-line-cache.d.ts.map +1 -0
  472. package/src/services/voice/first-line-cache.ts +725 -0
  473. package/src/services/voice/fused-eot-scorer.d.ts +51 -0
  474. package/src/services/voice/fused-eot-scorer.d.ts.map +1 -0
  475. package/src/services/voice/fused-eot-scorer.ts +135 -0
  476. package/src/services/voice/index.d.ts +91 -0
  477. package/src/services/voice/index.d.ts.map +1 -0
  478. package/src/services/voice/index.ts +481 -0
  479. package/src/services/voice/kokoro/__tests__/kokoro-backend.test.ts +151 -0
  480. package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.real.test.ts +151 -0
  481. package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.test.ts +60 -0
  482. package/src/services/voice/kokoro/__tests__/kokoro-engine-discovery.test.ts +277 -0
  483. package/src/services/voice/kokoro/__tests__/kokoro-ffi-runtime.test.ts +235 -0
  484. package/src/services/voice/kokoro/__tests__/kokoro-runtime.test.ts +95 -0
  485. package/src/services/voice/kokoro/__tests__/phonemizer.test.ts +53 -0
  486. package/src/services/voice/kokoro/__tests__/runtime-selection.test.ts +231 -0
  487. package/src/services/voice/kokoro/__tests__/voices.test.ts +57 -0
  488. package/src/services/voice/kokoro/index.ts +79 -0
  489. package/src/services/voice/kokoro/kokoro-backend.d.ts +72 -0
  490. package/src/services/voice/kokoro/kokoro-backend.d.ts.map +1 -0
  491. package/src/services/voice/kokoro/kokoro-backend.ts +207 -0
  492. package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts +58 -0
  493. package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts.map +1 -0
  494. package/src/services/voice/kokoro/kokoro-engine-discovery.ts +177 -0
  495. package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts +75 -0
  496. package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts.map +1 -0
  497. package/src/services/voice/kokoro/kokoro-ffi-runtime.ts +233 -0
  498. package/src/services/voice/kokoro/kokoro-runtime.d.ts +100 -0
  499. package/src/services/voice/kokoro/kokoro-runtime.d.ts.map +1 -0
  500. package/src/services/voice/kokoro/kokoro-runtime.ts +170 -0
  501. package/src/services/voice/kokoro/phoneme-stream.ts +123 -0
  502. package/src/services/voice/kokoro/phonemizer.d.ts +50 -0
  503. package/src/services/voice/kokoro/phonemizer.d.ts.map +1 -0
  504. package/src/services/voice/kokoro/phonemizer.ts +344 -0
  505. package/src/services/voice/kokoro/pick-runtime.d.ts +61 -0
  506. package/src/services/voice/kokoro/pick-runtime.d.ts.map +1 -0
  507. package/src/services/voice/kokoro/pick-runtime.test.ts +91 -0
  508. package/src/services/voice/kokoro/pick-runtime.ts +130 -0
  509. package/src/services/voice/kokoro/runtime-selection.d.ts +92 -0
  510. package/src/services/voice/kokoro/runtime-selection.d.ts.map +1 -0
  511. package/src/services/voice/kokoro/runtime-selection.ts +237 -0
  512. package/src/services/voice/kokoro/types.d.ts +82 -0
  513. package/src/services/voice/kokoro/types.d.ts.map +1 -0
  514. package/src/services/voice/kokoro/types.ts +95 -0
  515. package/src/services/voice/kokoro/voice-presets.d.ts +23 -0
  516. package/src/services/voice/kokoro/voice-presets.d.ts.map +1 -0
  517. package/src/services/voice/kokoro/voice-presets.ts +129 -0
  518. package/src/services/voice/kokoro/voices.d.ts +30 -0
  519. package/src/services/voice/kokoro/voices.d.ts.map +1 -0
  520. package/src/services/voice/kokoro/voices.ts +64 -0
  521. package/src/services/voice/lifecycle.d.ts +135 -0
  522. package/src/services/voice/lifecycle.d.ts.map +1 -0
  523. package/src/services/voice/lifecycle.test.ts +315 -0
  524. package/src/services/voice/lifecycle.ts +301 -0
  525. package/src/services/voice/live-diarization-session.d.ts +96 -0
  526. package/src/services/voice/live-diarization-session.d.ts.map +1 -0
  527. package/src/services/voice/live-diarization-session.ts +289 -0
  528. package/src/services/voice/mic-source.d.ts +136 -0
  529. package/src/services/voice/mic-source.d.ts.map +1 -0
  530. package/src/services/voice/mic-source.test.ts +210 -0
  531. package/src/services/voice/mic-source.ts +503 -0
  532. package/src/services/voice/optimistic-policy.d.ts +109 -0
  533. package/src/services/voice/optimistic-policy.d.ts.map +1 -0
  534. package/src/services/voice/optimistic-policy.test.ts +101 -0
  535. package/src/services/voice/optimistic-policy.ts +192 -0
  536. package/src/services/voice/optimistic-rollback.ts +343 -0
  537. package/src/services/voice/partial-stabilizer.d.ts +73 -0
  538. package/src/services/voice/partial-stabilizer.d.ts.map +1 -0
  539. package/src/services/voice/partial-stabilizer.test.ts +68 -0
  540. package/src/services/voice/partial-stabilizer.ts +140 -0
  541. package/src/services/voice/phoneme-tokenizer.d.ts +49 -0
  542. package/src/services/voice/phoneme-tokenizer.d.ts.map +1 -0
  543. package/src/services/voice/phoneme-tokenizer.ts +158 -0
  544. package/src/services/voice/phrase-cache.d.ts +76 -0
  545. package/src/services/voice/phrase-cache.d.ts.map +1 -0
  546. package/src/services/voice/phrase-cache.test.ts +242 -0
  547. package/src/services/voice/phrase-cache.ts +186 -0
  548. package/src/services/voice/phrase-chunker.d.ts +62 -0
  549. package/src/services/voice/phrase-chunker.d.ts.map +1 -0
  550. package/src/services/voice/phrase-chunker.test.ts +239 -0
  551. package/src/services/voice/phrase-chunker.ts +281 -0
  552. package/src/services/voice/pipeline-impls.d.ts +151 -0
  553. package/src/services/voice/pipeline-impls.d.ts.map +1 -0
  554. package/src/services/voice/pipeline-impls.l6.test.ts +110 -0
  555. package/src/services/voice/pipeline-impls.test.ts +292 -0
  556. package/src/services/voice/pipeline-impls.ts +315 -0
  557. package/src/services/voice/pipeline.d.ts +216 -0
  558. package/src/services/voice/pipeline.d.ts.map +1 -0
  559. package/src/services/voice/pipeline.ts +505 -0
  560. package/src/services/voice/prefill-client.d.ts +123 -0
  561. package/src/services/voice/prefill-client.d.ts.map +1 -0
  562. package/src/services/voice/prefill-client.ts +316 -0
  563. package/src/services/voice/prefix-preserving-queue.d.ts +113 -0
  564. package/src/services/voice/prefix-preserving-queue.d.ts.map +1 -0
  565. package/src/services/voice/prefix-preserving-queue.ts +162 -0
  566. package/src/services/voice/profile-store.d.ts +248 -0
  567. package/src/services/voice/profile-store.d.ts.map +1 -0
  568. package/src/services/voice/profile-store.ts +887 -0
  569. package/src/services/voice/real-audio-decode.test.ts +148 -0
  570. package/src/services/voice/ring-buffer.d.ts +40 -0
  571. package/src/services/voice/ring-buffer.d.ts.map +1 -0
  572. package/src/services/voice/ring-buffer.test.ts +129 -0
  573. package/src/services/voice/ring-buffer.ts +123 -0
  574. package/src/services/voice/rollback-queue.d.ts +24 -0
  575. package/src/services/voice/rollback-queue.d.ts.map +1 -0
  576. package/src/services/voice/rollback-queue.ts +74 -0
  577. package/src/services/voice/samantha-preset-placeholder.d.ts +67 -0
  578. package/src/services/voice/samantha-preset-placeholder.d.ts.map +1 -0
  579. package/src/services/voice/samantha-preset-placeholder.test.ts +97 -0
  580. package/src/services/voice/samantha-preset-placeholder.ts +148 -0
  581. package/src/services/voice/samantha-preset-regenerator.d.ts +87 -0
  582. package/src/services/voice/samantha-preset-regenerator.d.ts.map +1 -0
  583. package/src/services/voice/samantha-preset-regenerator.ts +393 -0
  584. package/src/services/voice/scheduler.d.ts +146 -0
  585. package/src/services/voice/scheduler.d.ts.map +1 -0
  586. package/src/services/voice/scheduler.t2.test.ts +141 -0
  587. package/src/services/voice/scheduler.ts +927 -0
  588. package/src/services/voice/shared-resources.d.ts +190 -0
  589. package/src/services/voice/shared-resources.d.ts.map +1 -0
  590. package/src/services/voice/shared-resources.ts +320 -0
  591. package/src/services/voice/speaker/attribution-pipeline.d.ts +74 -0
  592. package/src/services/voice/speaker/attribution-pipeline.d.ts.map +1 -0
  593. package/src/services/voice/speaker/attribution-pipeline.ts +386 -0
  594. package/src/services/voice/speaker/diarizer-fused.d.ts +59 -0
  595. package/src/services/voice/speaker/diarizer-fused.d.ts.map +1 -0
  596. package/src/services/voice/speaker/diarizer-fused.real.test.ts +100 -0
  597. package/src/services/voice/speaker/diarizer-fused.ts +154 -0
  598. package/src/services/voice/speaker/diarizer.d.ts +75 -0
  599. package/src/services/voice/speaker/diarizer.d.ts.map +1 -0
  600. package/src/services/voice/speaker/diarizer.ts +218 -0
  601. package/src/services/voice/speaker/encoder-fused.d.ts +60 -0
  602. package/src/services/voice/speaker/encoder-fused.d.ts.map +1 -0
  603. package/src/services/voice/speaker/encoder-fused.real.test.ts +113 -0
  604. package/src/services/voice/speaker/encoder-fused.ts +138 -0
  605. package/src/services/voice/speaker/encoder-ggml.d.ts +33 -0
  606. package/src/services/voice/speaker/encoder-ggml.d.ts.map +1 -0
  607. package/src/services/voice/speaker/encoder-ggml.ts +79 -0
  608. package/src/services/voice/speaker/encoder.d.ts +37 -0
  609. package/src/services/voice/speaker/encoder.d.ts.map +1 -0
  610. package/src/services/voice/speaker/encoder.ts +105 -0
  611. package/src/services/voice/speaker-imprint.d.ts +83 -0
  612. package/src/services/voice/speaker-imprint.d.ts.map +1 -0
  613. package/src/services/voice/speaker-imprint.test.ts +185 -0
  614. package/src/services/voice/speaker-imprint.ts +312 -0
  615. package/src/services/voice/speaker-preset-cache.d.ts +77 -0
  616. package/src/services/voice/speaker-preset-cache.d.ts.map +1 -0
  617. package/src/services/voice/speaker-preset-cache.test.ts +154 -0
  618. package/src/services/voice/speaker-preset-cache.ts +195 -0
  619. package/src/services/voice/streaming-asr/streaming-pipeline-adapter.ts +292 -0
  620. package/src/services/voice/system-audio-sink.d.ts +73 -0
  621. package/src/services/voice/system-audio-sink.d.ts.map +1 -0
  622. package/src/services/voice/system-audio-sink.test.ts +29 -0
  623. package/src/services/voice/system-audio-sink.ts +366 -0
  624. package/src/services/voice/transcriber.d.ts +244 -0
  625. package/src/services/voice/transcriber.d.ts.map +1 -0
  626. package/src/services/voice/transcriber.test.ts +392 -0
  627. package/src/services/voice/transcriber.ts +704 -0
  628. package/src/services/voice/transcript-knowledge.d.ts +37 -0
  629. package/src/services/voice/transcript-knowledge.d.ts.map +1 -0
  630. package/src/services/voice/transcript-knowledge.test.ts +68 -0
  631. package/src/services/voice/transcript-knowledge.ts +75 -0
  632. package/src/services/voice/transcript-service.d.ts +41 -0
  633. package/src/services/voice/transcript-service.d.ts.map +1 -0
  634. package/src/services/voice/transcript-service.test.ts +137 -0
  635. package/src/services/voice/transcript-service.ts +141 -0
  636. package/src/services/voice/transcript-store.d.ts +53 -0
  637. package/src/services/voice/transcript-store.d.ts.map +1 -0
  638. package/src/services/voice/transcript-store.test.ts +153 -0
  639. package/src/services/voice/transcript-store.ts +132 -0
  640. package/src/services/voice/turn-controller.d.ts +183 -0
  641. package/src/services/voice/turn-controller.d.ts.map +1 -0
  642. package/src/services/voice/turn-controller.test.ts +575 -0
  643. package/src/services/voice/turn-controller.ts +596 -0
  644. package/src/services/voice/types.d.ts +643 -0
  645. package/src/services/voice/types.d.ts.map +1 -0
  646. package/src/services/voice/types.ts +699 -0
  647. package/src/services/voice/vad.d.ts +282 -0
  648. package/src/services/voice/vad.d.ts.map +1 -0
  649. package/src/services/voice/vad.test.ts +480 -0
  650. package/src/services/voice/vad.ts +827 -0
  651. package/src/services/voice/vad.v1-v4.test.ts +222 -0
  652. package/src/services/voice/voice-budget.d.ts +241 -0
  653. package/src/services/voice/voice-budget.d.ts.map +1 -0
  654. package/src/services/voice/voice-budget.test.ts +418 -0
  655. package/src/services/voice/voice-budget.ts +635 -0
  656. package/src/services/voice/voice-duet.test.ts +375 -0
  657. package/src/services/voice/voice-emotion-classifier.d.ts +95 -0
  658. package/src/services/voice/voice-emotion-classifier.d.ts.map +1 -0
  659. package/src/services/voice/voice-emotion-classifier.test.ts +210 -0
  660. package/src/services/voice/voice-emotion-classifier.ts +273 -0
  661. package/src/services/voice/voice-preset-format.d.ts +158 -0
  662. package/src/services/voice/voice-preset-format.d.ts.map +1 -0
  663. package/src/services/voice/voice-preset-format.ts +700 -0
  664. package/src/services/voice/voice-preset-generator.test.ts +89 -0
  665. package/src/services/voice/voice-profile-artifact.d.ts +116 -0
  666. package/src/services/voice/voice-profile-artifact.d.ts.map +1 -0
  667. package/src/services/voice/voice-profile-artifact.test.ts +138 -0
  668. package/src/services/voice/voice-profile-artifact.ts +518 -0
  669. package/src/services/voice/voice-profile-routes.d.ts +83 -0
  670. package/src/services/voice/voice-profile-routes.d.ts.map +1 -0
  671. package/src/services/voice/voice-profile-routes.test.ts +429 -0
  672. package/src/services/voice/voice-profile-routes.ts +425 -0
  673. package/src/services/voice/voice-scenario.ts +154 -0
  674. package/src/services/voice/voice-settings.d.ts +82 -0
  675. package/src/services/voice/voice-settings.d.ts.map +1 -0
  676. package/src/services/voice/voice-settings.ts +172 -0
  677. package/src/services/voice/voice-state-machine.d.ts +364 -0
  678. package/src/services/voice/voice-state-machine.d.ts.map +1 -0
  679. package/src/services/voice/voice-state-machine.ts +727 -0
  680. package/src/services/voice/voice-workbench-report.test.ts +168 -0
  681. package/src/services/voice/voice-workbench-report.ts +326 -0
  682. package/src/services/voice/voice-workbench.test.ts +158 -0
  683. package/src/services/voice/voice.test.ts +1070 -0
  684. package/src/services/voice/wake-word-ggml.d.ts +101 -0
  685. package/src/services/voice/wake-word-ggml.d.ts.map +1 -0
  686. package/src/services/voice/wake-word-ggml.ts +320 -0
  687. package/src/services/voice/wake-word.d.ts +255 -0
  688. package/src/services/voice/wake-word.d.ts.map +1 -0
  689. package/src/services/voice/wake-word.test.ts +298 -0
  690. package/src/services/voice/wake-word.ts +554 -0
  691. package/src/services/voice/wrap-with-first-line-cache.d.ts +70 -0
  692. package/src/services/voice/wrap-with-first-line-cache.d.ts.map +1 -0
  693. package/src/services/voice/wrap-with-first-line-cache.ts +267 -0
  694. package/src/services/voice-model-updater.d.ts +240 -0
  695. package/src/services/voice-model-updater.d.ts.map +1 -0
  696. package/src/services/voice-model-updater.ts +724 -0
  697. package/src/services/voice-prewarm.d.ts +3 -0
  698. package/src/services/voice-prewarm.d.ts.map +1 -0
  699. package/src/services/voice-prewarm.ts +51 -0
  700. package/dist/index.d.ts +0 -37
  701. package/dist/index.js +0 -1098
@@ -0,0 +1,255 @@
1
+ /**
2
+ * Capacitor-llama vision-describe backend (WS2).
3
+ *
4
+ * Wraps the in-process capacitor-llama binding's multimodal projector
5
+ * (mtmd) surface and exposes the WS2 `VisionDescribeBackend` contract.
6
+ *
7
+ * State of the binding (2026-05-19):
8
+ * `llama-cpp-capacitor`'s `LlamaContext` exposes `initMultimodal` and
9
+ * `getMultimodalSupport`, which load an mmproj alongside the chat
10
+ * target. The desktop bun:ffi shim does not yet bind those symbols —
11
+ * the desktop FFI path returns `binding_missing_mtmd` until the shim
12
+ * adds `mtmd_init_from_file` + the encode/decode helpers.
13
+ *
14
+ * What this module does today:
15
+ * - Provides the WS2-shaped backend so plugin-vision / plugin-image-gen /
16
+ * computer-use can compile against a stable contract.
17
+ * - When the binding exposes the mtmd API, the backend dispatches
18
+ * through it.
19
+ * - Otherwise, the backend falls back to an injected
20
+ * `VisionManagerLike` implementation when one is supplied (kept as a
21
+ * pluggable seam for tests and out-of-tree integrations).
22
+ * - When neither path is wired, `describe()` throws a structured
23
+ * `VisionBackendUnavailableError` the arbiter surfaces upward.
24
+ *
25
+ * GPU validation status (this host has neither GPU):
26
+ * The mtmd encode path is GPU-accelerated when the underlying llama.cpp
27
+ * build dispatches `llama_image_t` through the model's batch path. We
28
+ * document the on-device validation that's required for each GPU
29
+ * family at the bottom of this file's tests (see
30
+ * `__tests__/vision-describe.test.ts`). Until those run on real
31
+ * hardware, GPU-backed vision is "implementation present, not
32
+ * validated".
33
+ */
34
+
35
+ import { existsSync, promises as fs } from "node:fs";
36
+ import { resolveImageBytes } from "./hash";
37
+ import type {
38
+ VisionDescribeBackend,
39
+ VisionDescribeBackendOptions,
40
+ VisionDescribeLoadArgs,
41
+ VisionDescribeRequest,
42
+ VisionDescribeResult,
43
+ } from "./types";
44
+
45
+ export class VisionBackendUnavailableError extends Error {
46
+ readonly code = "VISION_BACKEND_UNAVAILABLE";
47
+ constructor(
48
+ readonly backendId: string,
49
+ readonly reason:
50
+ | "binding_missing_mtmd"
51
+ | "no_fallback_present"
52
+ | "mmproj_missing",
53
+ message: string,
54
+ ) {
55
+ super(message);
56
+ this.name = "VisionBackendUnavailableError";
57
+ }
58
+ }
59
+
60
+ /**
61
+ * Optional shape the Capacitor-llama binding exposes once the mtmd typed
62
+ * wrappers land in the shared adapter. The backend only consumes
63
+ * `describeWithMmproj`, which wraps `LlamaContext.initMultimodal` +
64
+ * `completion(...)` with `media_paths`. Backends that don't satisfy this
65
+ * shape are treated as "binding without mtmd support" and the fallback
66
+ * path is used.
67
+ */
68
+ export interface CapacitorLlamaMtmdBinding {
69
+ loadVisionModel(args: {
70
+ modelPath: string;
71
+ mmprojPath: string;
72
+ gpuLayers?: number | "auto" | "max";
73
+ contextSize?: number;
74
+ }): Promise<CapacitorLlamaMtmdHandle>;
75
+ }
76
+
77
+ export interface CapacitorLlamaMtmdHandle {
78
+ describeWithMmproj(args: {
79
+ imageBytes: Uint8Array;
80
+ mimeType?: string;
81
+ prompt: string;
82
+ maxTokens?: number;
83
+ temperature?: number;
84
+ signal?: AbortSignal;
85
+ projectedTokens?: VisionDescribeBackendOptions["projectedTokens"];
86
+ }): Promise<{ text: string; projectorMs?: number; decodeMs?: number }>;
87
+ dispose(): Promise<void>;
88
+ }
89
+
90
+ /**
91
+ * Optional VisionManager-shape fallback. Kept available as a pluggable
92
+ * injection point for tests and out-of-tree integrations that want to
93
+ * supply their own image captioning implementation.
94
+ */
95
+ export interface VisionManagerLike {
96
+ processImage(
97
+ dataUrl: string,
98
+ ): Promise<{ title: string; description: string }>;
99
+ }
100
+
101
+ export interface CapacitorLlamaVisionBackendOptions {
102
+ loadArgs: VisionDescribeLoadArgs;
103
+ /**
104
+ * Injected by tests and by the shared mtmd typed wrappers. When
105
+ * provided the backend uses the mtmd path.
106
+ */
107
+ mtmd?: CapacitorLlamaMtmdBinding;
108
+ /**
109
+ * Caption-only fallback. Optional — when present the backend uses it
110
+ * as last resort, after mtmd. Backends that have neither throw a
111
+ * structured `VisionBackendUnavailableError`.
112
+ */
113
+ visionManager?: VisionManagerLike;
114
+ }
115
+
116
+ const DEFAULT_PROMPT = "Describe what is in this image.";
117
+
118
+ export async function loadCapacitorLlamaVisionBackend(
119
+ opts: CapacitorLlamaVisionBackendOptions,
120
+ ): Promise<VisionDescribeBackend> {
121
+ const { loadArgs, mtmd, visionManager } = opts;
122
+
123
+ if (mtmd) {
124
+ // Validate mmproj presence here so we surface a clean error before
125
+ // burning a load (the binding's own error would be cryptic).
126
+ if (!existsSync(loadArgs.mmprojPath)) {
127
+ throw new VisionBackendUnavailableError(
128
+ "capacitor-llama",
129
+ "mmproj_missing",
130
+ `[vision/capacitor-llama] mmproj GGUF not found: ${loadArgs.mmprojPath}`,
131
+ );
132
+ }
133
+ const handle = await mtmd.loadVisionModel({
134
+ modelPath: loadArgs.modelPath,
135
+ mmprojPath: loadArgs.mmprojPath,
136
+ gpuLayers: loadArgs.gpuLayers,
137
+ contextSize: loadArgs.contextSize,
138
+ });
139
+ return {
140
+ id: "capacitor-llama",
141
+ async describe(
142
+ request: VisionDescribeRequest,
143
+ args?: VisionDescribeBackendOptions,
144
+ ): Promise<VisionDescribeResult> {
145
+ const { bytes, mimeType } = resolveImageBytes(request.image);
146
+ const result = await handle.describeWithMmproj({
147
+ imageBytes: bytes,
148
+ mimeType,
149
+ prompt: request.prompt ?? DEFAULT_PROMPT,
150
+ maxTokens: request.maxTokens,
151
+ temperature: request.temperature,
152
+ signal: request.signal,
153
+ projectedTokens: args?.projectedTokens,
154
+ });
155
+ return shapeResult(result.text, {
156
+ projectorMs: result.projectorMs,
157
+ decodeMs: result.decodeMs,
158
+ cacheHit: Boolean(args?.projectedTokens),
159
+ });
160
+ },
161
+ async dispose() {
162
+ await handle.dispose();
163
+ },
164
+ };
165
+ }
166
+
167
+ if (visionManager) {
168
+ return {
169
+ id: "capacitor-llama",
170
+ async describe(
171
+ request: VisionDescribeRequest,
172
+ ): Promise<VisionDescribeResult> {
173
+ const dataUrl = await imageInputToDataUrl(request.image);
174
+ const result = await visionManager.processImage(dataUrl);
175
+ return {
176
+ title: result.title,
177
+ description: result.description,
178
+ cacheHit: false,
179
+ };
180
+ },
181
+ async dispose() {
182
+ // VisionManager is a process-singleton owned by LocalAIManager;
183
+ // its lifetime is decoupled from the WS2 backend. Disposing the
184
+ // backend here is a no-op — the manager stays warm for legacy
185
+ // callers that haven't moved off LocalAIManager.describeImage yet.
186
+ },
187
+ };
188
+ }
189
+
190
+ throw new VisionBackendUnavailableError(
191
+ "capacitor-llama",
192
+ "binding_missing_mtmd",
193
+ "[vision/capacitor-llama] no mtmd binding and no VisionManager fallback was provided. Wire up the Capacitor-llama mtmd adapter (initMultimodal + media_paths completion) or pass a VisionManager fallback in options.",
194
+ );
195
+ }
196
+
197
+ function shapeResult(
198
+ text: string,
199
+ telemetry: { projectorMs?: number; decodeMs?: number; cacheHit?: boolean },
200
+ ): VisionDescribeResult {
201
+ const trimmed = text.trim();
202
+ if (!trimmed) {
203
+ throw new Error("[vision/capacitor-llama] backend returned empty text");
204
+ }
205
+ const title = trimmed.split(/[.!?]/, 1)[0]?.trim() || "Image";
206
+ return {
207
+ title,
208
+ description: trimmed,
209
+ ...telemetry,
210
+ };
211
+ }
212
+
213
+ async function imageInputToDataUrl(
214
+ input: VisionDescribeRequest["image"],
215
+ ): Promise<string> {
216
+ switch (input.kind) {
217
+ case "dataUrl":
218
+ return input.dataUrl;
219
+ case "base64":
220
+ return `data:${input.mimeType ?? "image/png"};base64,${input.base64}`;
221
+ case "bytes": {
222
+ const mimeType = input.mimeType ?? "image/png";
223
+ const base64 = Buffer.from(input.bytes).toString("base64");
224
+ return `data:${mimeType};base64,${base64}`;
225
+ }
226
+ case "url": {
227
+ const url = input.url;
228
+ if (url.startsWith("data:")) return url;
229
+ if (url.startsWith("file://") || url.startsWith("/")) {
230
+ const filePath = url.startsWith("file://") ? url.slice(7) : url;
231
+ const bytes = await fs.readFile(filePath);
232
+ const mimeType = input.mimeType ?? guessMimeFromPath(filePath);
233
+ return `data:${mimeType};base64,${bytes.toString("base64")}`;
234
+ }
235
+ const res = await fetch(url);
236
+ if (!res.ok) {
237
+ throw new Error(
238
+ `[vision/capacitor-llama] failed to fetch image: ${res.status} ${res.statusText}`,
239
+ );
240
+ }
241
+ const buf = new Uint8Array(await res.arrayBuffer());
242
+ const mimeType =
243
+ input.mimeType ?? res.headers.get("content-type") ?? "image/png";
244
+ return `data:${mimeType};base64,${Buffer.from(buf).toString("base64")}`;
245
+ }
246
+ }
247
+ }
248
+
249
+ function guessMimeFromPath(p: string): string {
250
+ const lower = p.toLowerCase();
251
+ if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg";
252
+ if (lower.endsWith(".webp")) return "image/webp";
253
+ if (lower.endsWith(".gif")) return "image/gif";
254
+ return "image/png";
255
+ }
@@ -0,0 +1,47 @@
1
+ /**
2
+ * Soft cloud fallback wrapper for local IMAGE_DESCRIPTION handlers.
3
+ *
4
+ * The local vision path can report recoverable unavailability without forcing
5
+ * callers to know which remote provider is paired. The wrapper keeps that
6
+ * state explicit: handlers return a normal image description or a typed
7
+ * fallback outcome that the next layer can handle.
8
+ */
9
+ import type { ImageDescriptionParams, ImageDescriptionResult } from "@elizaos/core";
10
+ export type VisionFallbackReason = "local-unavailable" | "local-overloaded" | "local-error" | "local-aborted-pre-completion" | "local-not-registered" | "cloud-unavailable" | "cloud-error" | "vast-unavailable" | "vast-error";
11
+ export type LocalVisionOutcome = ImageDescriptionResult | string | {
12
+ kind: "ok";
13
+ result: ImageDescriptionResult | string;
14
+ } | {
15
+ kind: "fallback";
16
+ reason: VisionFallbackReason;
17
+ cause?: Error;
18
+ };
19
+ export type LocalVisionResult = Exclude<LocalVisionOutcome, {
20
+ kind: "fallback";
21
+ reason: VisionFallbackReason;
22
+ cause?: Error;
23
+ }>;
24
+ export type LocalImageDescriptionHandler = (params: ImageDescriptionParams | string) => Promise<LocalVisionOutcome>;
25
+ export type WrappedImageDescriptionHandler = LocalImageDescriptionHandler;
26
+ export interface VisionCloudFallbackOptions {
27
+ enabled?: boolean;
28
+ token?: string;
29
+ apiKey?: string;
30
+ baseUrl?: string;
31
+ fetch?: typeof fetch;
32
+ handler?: (params: ImageDescriptionParams | string, reason: VisionFallbackReason) => Promise<LocalVisionOutcome>;
33
+ log?: (message: string, detail?: Record<string, unknown>) => void;
34
+ }
35
+ export declare function classifyLocalVisionError(error: unknown): {
36
+ fallback: boolean;
37
+ reason: VisionFallbackReason;
38
+ cause?: Error;
39
+ };
40
+ export declare function isVisionFallbackOutcome(outcome: LocalVisionOutcome): outcome is {
41
+ kind: "fallback";
42
+ reason: VisionFallbackReason;
43
+ cause?: Error;
44
+ };
45
+ export declare function normalizeVisionDescription(result: LocalVisionResult): ImageDescriptionResult;
46
+ export declare function wrapImageDescriptionHandlerWithCloudFallback(local: LocalImageDescriptionHandler, options?: VisionCloudFallbackOptions): WrappedImageDescriptionHandler;
47
+ //# sourceMappingURL=cloud-fallback.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cloud-fallback.d.ts","sourceRoot":"","sources":["cloud-fallback.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACX,sBAAsB,EACtB,sBAAsB,EACtB,MAAM,eAAe,CAAC;AAEvB,MAAM,MAAM,oBAAoB,GAC7B,mBAAmB,GACnB,kBAAkB,GAClB,aAAa,GACb,8BAA8B,GAC9B,sBAAsB,GACtB,mBAAmB,GACnB,aAAa,GACb,kBAAkB,GAClB,YAAY,CAAC;AAEhB,MAAM,MAAM,kBAAkB,GAC3B,sBAAsB,GACtB,MAAM,GACN;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC;AAErE,MAAM,MAAM,iBAAiB,GAAG,OAAO,CACtC,kBAAkB,EAClB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,oBAAoB,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CACjE,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,CAC1C,MAAM,EAAE,sBAAsB,GAAG,MAAM,KACnC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAEjC,MAAM,MAAM,8BAA8B,GAAG,4BAA4B,CAAC;AAE1E,MAAM,WAAW,0BAA0B;IAC1C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,OAAO,CAAC,EAAE,CACT,MAAM,EAAE,sBAAsB,GAAG,MAAM,EACvC,MAAM,EAAE,oBAAoB,KACxB,OAAO,CAAC,kBAAkB,CAAC,CAAC;IACjC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CAClE;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG;IACzD,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,CAAC,EAAE,KAAK,CAAC;CACd,CA2CA;AAED,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,kBAAkB,GACzB,OAAO,IAAI;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,KAAK,CAAC,EAAE,KAAK,CAAC;CACd,CAOA;AAED,wBAAgB,0BAA0B,CACzC,MAAM,EAAE,iBAAiB,GACvB,sBAAsB,CAiCxB;AAqED,wBAAgB,4CAA4C,CAC3D,KAAK,EAAE,4BAA4B,EACnC,OAAO,GAAE,0BAA+B,GACtC,8BAA8B,CAyChC"}
@@ -0,0 +1,243 @@
1
+ /**
2
+ * IMAGE_DESCRIPTION fallback chain tests.
3
+ *
4
+ * Exercises the local→cloud→vast wrappers in isolation. The wrappers are
5
+ * pure functions over an injected `fetch`, so we never touch global env
6
+ * here — every knob is passed in via options.
7
+ *
8
+ * What we assert:
9
+ * - Local success: cloud `fetch` is never called.
10
+ * - Local fallback: cloud `fetch` is invoked with the right URL/body/auth.
11
+ * - Cloud 5xx: the cloud-wrapped handler re-emits a `{kind:"fallback"}` so
12
+ * the vast layer can take over; vast `fetch` is then invoked.
13
+ * - Both unavailable (no token / no key): the original local fallback
14
+ * propagates unchanged so an outer layer (or the runtime) can act.
15
+ */
16
+
17
+ import type {
18
+ ImageDescriptionParams,
19
+ ImageDescriptionResult,
20
+ } from "@elizaos/core";
21
+ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
22
+ import {
23
+ type LocalImageDescriptionHandler,
24
+ type LocalVisionOutcome,
25
+ wrapImageDescriptionHandlerWithCloudFallback,
26
+ } from "./cloud-fallback";
27
+ import { wrapImageDescriptionHandlerWithVastFallback } from "./vast-fallback";
28
+
29
+ // Quarantine the env vars the wrappers read so a developer machine that
30
+ // happens to have ELIZA_CLOUD_TOKEN / ELIZA_VAST_* exported doesn't make the
31
+ // "unconfigured" assertions flake into a real outbound HTTP call.
32
+ const ENV_KEYS = [
33
+ "ELIZA_CLOUD_TOKEN",
34
+ "ELIZA_CLOUD_API_KEY",
35
+ "ELIZA_CLOUD_BASE_URL",
36
+ "ELIZA_VAST_BASE_URL",
37
+ "ELIZA_VAST_API_KEY",
38
+ ] as const;
39
+ const savedEnv: Record<string, string | undefined> = {};
40
+
41
+ beforeEach(() => {
42
+ for (const key of ENV_KEYS) {
43
+ savedEnv[key] = process.env[key];
44
+ delete process.env[key];
45
+ }
46
+ });
47
+
48
+ afterEach(() => {
49
+ for (const key of ENV_KEYS) {
50
+ if (savedEnv[key] === undefined) delete process.env[key];
51
+ else process.env[key] = savedEnv[key];
52
+ }
53
+ });
54
+
55
+ const PNG_URL = "https://example.invalid/cat.png";
56
+
57
+ function jsonResponse(body: unknown, status = 200): Response {
58
+ return new Response(JSON.stringify(body), {
59
+ status,
60
+ headers: { "content-type": "application/json" },
61
+ });
62
+ }
63
+
64
+ function textResponse(body: string, status = 500): Response {
65
+ return new Response(body, {
66
+ status,
67
+ headers: { "content-type": "text/plain" },
68
+ });
69
+ }
70
+
71
+ function okLocal(result: ImageDescriptionResult): LocalImageDescriptionHandler {
72
+ return async () => result;
73
+ }
74
+
75
+ function fallbackLocal(
76
+ reason: LocalVisionOutcome extends { kind: "fallback"; reason: infer R }
77
+ ? R
78
+ : never,
79
+ ): LocalImageDescriptionHandler {
80
+ return async () =>
81
+ ({
82
+ kind: "fallback",
83
+ reason,
84
+ }) satisfies LocalVisionOutcome;
85
+ }
86
+
87
+ describe("wrapImageDescriptionHandlerWithCloudFallback", () => {
88
+ it("returns local result without invoking cloud when local succeeds", async () => {
89
+ const fetchMock = vi.fn();
90
+ const wrapped = wrapImageDescriptionHandlerWithCloudFallback(
91
+ okLocal({ title: "Cat", description: "A small ginger cat." }),
92
+ { token: "tk_test", baseUrl: "https://cloud.test", fetch: fetchMock },
93
+ );
94
+ const result = await wrapped({
95
+ imageUrl: PNG_URL,
96
+ } as ImageDescriptionParams);
97
+ expect(result).toEqual({
98
+ title: "Cat",
99
+ description: "A small ginger cat.",
100
+ });
101
+ expect(fetchMock).not.toHaveBeenCalled();
102
+ });
103
+
104
+ it("invokes cloud /v1/vision/describe with bearer + body when local falls back", async () => {
105
+ const fetchMock = vi.fn(async () =>
106
+ jsonResponse({
107
+ title: "Cloud title",
108
+ description: "Cloud description.",
109
+ }),
110
+ );
111
+ const wrapped = wrapImageDescriptionHandlerWithCloudFallback(
112
+ fallbackLocal("local-unavailable"),
113
+ { token: "tk_test", baseUrl: "https://cloud.test", fetch: fetchMock },
114
+ );
115
+ const result = await wrapped({
116
+ imageUrl: PNG_URL,
117
+ prompt: "describe",
118
+ } as ImageDescriptionParams);
119
+ expect(result).toEqual({
120
+ title: "Cloud title",
121
+ description: "Cloud description.",
122
+ });
123
+ expect(fetchMock).toHaveBeenCalledTimes(1);
124
+ const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
125
+ expect(url).toBe("https://cloud.test/v1/vision/describe");
126
+ expect(init.method).toBe("POST");
127
+ expect((init.headers as Record<string, string>).authorization).toBe(
128
+ "Bearer tk_test",
129
+ );
130
+ const body = JSON.parse(init.body as string) as {
131
+ image: { kind: string; url: string };
132
+ prompt: string;
133
+ };
134
+ expect(body.image).toEqual({ kind: "url", url: PNG_URL });
135
+ expect(body.prompt).toBe("describe");
136
+ });
137
+
138
+ it("forwards URL inputs as-is (no local fetch first)", async () => {
139
+ const fetchMock = vi.fn(async () =>
140
+ jsonResponse({ title: "T", description: "D" }),
141
+ );
142
+ const wrapped = wrapImageDescriptionHandlerWithCloudFallback(
143
+ fallbackLocal("local-unavailable"),
144
+ { token: "tk_test", baseUrl: "https://cloud.test", fetch: fetchMock },
145
+ );
146
+ await wrapped({ imageUrl: PNG_URL } as ImageDescriptionParams);
147
+ // Only one outbound call — to the cloud describe endpoint, not to the URL.
148
+ expect(fetchMock).toHaveBeenCalledTimes(1);
149
+ const [url] = fetchMock.mock.calls[0] as [string];
150
+ expect(url).toBe("https://cloud.test/v1/vision/describe");
151
+ });
152
+
153
+ it("propagates fallback when no cloud token is configured", async () => {
154
+ const fetchMock = vi.fn();
155
+ const wrapped = wrapImageDescriptionHandlerWithCloudFallback(
156
+ fallbackLocal("local-unavailable"),
157
+ { baseUrl: "https://cloud.test", fetch: fetchMock },
158
+ );
159
+ const out = await wrapped({ imageUrl: PNG_URL } as ImageDescriptionParams);
160
+ expect(fetchMock).not.toHaveBeenCalled();
161
+ expect(out).toMatchObject({
162
+ kind: "fallback",
163
+ reason: "local-unavailable",
164
+ });
165
+ });
166
+
167
+ it("propagates fallback when cloud responds with 5xx", async () => {
168
+ const fetchMock = vi.fn(async () => textResponse("upstream down", 503));
169
+ const wrapped = wrapImageDescriptionHandlerWithCloudFallback(
170
+ fallbackLocal("local-error"),
171
+ { token: "tk_test", baseUrl: "https://cloud.test", fetch: fetchMock },
172
+ );
173
+ const out = await wrapped({ imageUrl: PNG_URL } as ImageDescriptionParams);
174
+ expect(fetchMock).toHaveBeenCalledTimes(1);
175
+ expect(out).toMatchObject({ kind: "fallback", reason: "local-error" });
176
+ expect((out as { cause?: Error }).cause?.message ?? "").toContain("503");
177
+ });
178
+ });
179
+
180
+ describe("wrapImageDescriptionHandlerWithVastFallback", () => {
181
+ it("invokes vast when inner handler returns fallback and vast is configured", async () => {
182
+ const innerFetch = vi.fn(async () => textResponse("cloud sad", 503));
183
+ const inner = wrapImageDescriptionHandlerWithCloudFallback(
184
+ fallbackLocal("local-unavailable"),
185
+ { token: "tk_test", baseUrl: "https://cloud.test", fetch: innerFetch },
186
+ );
187
+ const vastFetch = vi.fn(async () =>
188
+ jsonResponse({ title: "VTitle", description: "VDescription" }),
189
+ );
190
+ const wrapped = wrapImageDescriptionHandlerWithVastFallback(inner, {
191
+ baseUrl: "https://vast.test",
192
+ apiKey: "vast_key",
193
+ fetch: vastFetch,
194
+ });
195
+ const result = await wrapped({
196
+ imageUrl: PNG_URL,
197
+ } as ImageDescriptionParams);
198
+ expect(innerFetch).toHaveBeenCalledTimes(1);
199
+ expect(vastFetch).toHaveBeenCalledTimes(1);
200
+ const [url, init] = vastFetch.mock.calls[0] as [string, RequestInit];
201
+ expect(url).toBe("https://vast.test/v1/vision/describe");
202
+ expect(init.method).toBe("POST");
203
+ expect((init.headers as Record<string, string>).authorization).toBe(
204
+ "Bearer vast_key",
205
+ );
206
+ expect(result).toEqual({ title: "VTitle", description: "VDescription" });
207
+ });
208
+
209
+ it("does not call vast when inner handler succeeds", async () => {
210
+ const inner = wrapImageDescriptionHandlerWithCloudFallback(
211
+ okLocal({ title: "Local", description: "Local desc." }),
212
+ { token: "tk_test", baseUrl: "https://cloud.test", fetch: vi.fn() },
213
+ );
214
+ const vastFetch = vi.fn();
215
+ const wrapped = wrapImageDescriptionHandlerWithVastFallback(inner, {
216
+ baseUrl: "https://vast.test",
217
+ apiKey: "vast_key",
218
+ fetch: vastFetch,
219
+ });
220
+ const result = await wrapped({
221
+ imageUrl: PNG_URL,
222
+ } as ImageDescriptionParams);
223
+ expect(vastFetch).not.toHaveBeenCalled();
224
+ expect(result).toEqual({ title: "Local", description: "Local desc." });
225
+ });
226
+
227
+ it("propagates fallback when both cloud and vast are unconfigured", async () => {
228
+ const inner = wrapImageDescriptionHandlerWithCloudFallback(
229
+ fallbackLocal("local-unavailable"),
230
+ { fetch: vi.fn() },
231
+ );
232
+ const vastFetch = vi.fn();
233
+ const wrapped = wrapImageDescriptionHandlerWithVastFallback(inner, {
234
+ fetch: vastFetch,
235
+ });
236
+ const out = await wrapped({ imageUrl: PNG_URL } as ImageDescriptionParams);
237
+ expect(vastFetch).not.toHaveBeenCalled();
238
+ expect(out).toMatchObject({
239
+ kind: "fallback",
240
+ reason: "local-unavailable",
241
+ });
242
+ });
243
+ });