@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.11-beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (676) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +83 -0
  3. package/package.json +81 -15
  4. package/src/actions/generate-media.d.ts +59 -0
  5. package/src/actions/generate-media.d.ts.map +1 -0
  6. package/src/actions/generate-media.ts +647 -0
  7. package/src/actions/identify-speaker.d.ts +23 -0
  8. package/src/actions/identify-speaker.d.ts.map +1 -0
  9. package/src/actions/identify-speaker.ts +171 -0
  10. package/src/adapters/capacitor-llama/__tests__/compat-behavior.test.ts +218 -0
  11. package/src/adapters/capacitor-llama/__tests__/index.test.ts +68 -0
  12. package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts +215 -0
  13. package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts +174 -0
  14. package/src/adapters/capacitor-llama/environment.ts +71 -0
  15. package/src/adapters/capacitor-llama/index.browser.ts +83 -0
  16. package/src/adapters/capacitor-llama/index.ts +807 -0
  17. package/src/adapters/capacitor-llama/loader.ts +109 -0
  18. package/src/adapters/capacitor-llama/structured-output.ts +165 -0
  19. package/src/adapters/capacitor-llama/text-streaming.ts +227 -0
  20. package/src/adapters/capacitor-llama/types.ts +374 -0
  21. package/src/backends/apple-foundation.ts +127 -0
  22. package/src/index.d.ts +7 -0
  23. package/src/index.d.ts.map +1 -0
  24. package/src/index.ts +54 -0
  25. package/src/local-inference-routes.d.ts +38 -0
  26. package/src/local-inference-routes.d.ts.map +1 -0
  27. package/src/local-inference-routes.test.ts +344 -0
  28. package/src/local-inference-routes.ts +1543 -0
  29. package/src/provider.d.ts +21 -0
  30. package/src/provider.d.ts.map +1 -0
  31. package/src/provider.ts +1171 -0
  32. package/src/routes/compat-helpers.d.ts +18 -0
  33. package/src/routes/compat-helpers.d.ts.map +1 -0
  34. package/src/routes/compat-helpers.ts +274 -0
  35. package/src/routes/family-member-route.d.ts +62 -0
  36. package/src/routes/family-member-route.d.ts.map +1 -0
  37. package/src/routes/family-member-route.ts +353 -0
  38. package/src/routes/index.d.ts +19 -0
  39. package/src/routes/index.d.ts.map +1 -0
  40. package/src/routes/index.ts +60 -0
  41. package/src/routes/live-diarization-route.d.ts +26 -0
  42. package/src/routes/live-diarization-route.d.ts.map +1 -0
  43. package/src/routes/live-diarization-route.test.ts +213 -0
  44. package/src/routes/live-diarization-route.ts +122 -0
  45. package/src/routes/local-inference-asr-route.d.ts +4 -0
  46. package/src/routes/local-inference-asr-route.d.ts.map +1 -0
  47. package/src/routes/local-inference-asr-route.test.ts +190 -0
  48. package/src/routes/local-inference-asr-route.ts +213 -0
  49. package/src/routes/local-inference-compat-routes.d.ts +16 -0
  50. package/src/routes/local-inference-compat-routes.d.ts.map +1 -0
  51. package/src/routes/local-inference-compat-routes.test.ts +423 -0
  52. package/src/routes/local-inference-compat-routes.ts +782 -0
  53. package/src/routes/local-inference-tts-route.d.ts +7 -0
  54. package/src/routes/local-inference-tts-route.d.ts.map +1 -0
  55. package/src/routes/local-inference-tts-route.test.ts +179 -0
  56. package/src/routes/local-inference-tts-route.ts +230 -0
  57. package/src/routes/voice-first-run-routes.d.ts +62 -0
  58. package/src/routes/voice-first-run-routes.d.ts.map +1 -0
  59. package/src/routes/voice-first-run-routes.ts +524 -0
  60. package/src/routes/voice-models-routes.d.ts +62 -0
  61. package/src/routes/voice-models-routes.d.ts.map +1 -0
  62. package/src/routes/voice-models-routes.ts +554 -0
  63. package/src/routes/voice-profile-plugin-routes.d.ts +19 -0
  64. package/src/routes/voice-profile-plugin-routes.d.ts.map +1 -0
  65. package/src/routes/voice-profile-plugin-routes.ts +138 -0
  66. package/src/routes/voice-profiles-management-routes.d.ts +52 -0
  67. package/src/routes/voice-profiles-management-routes.d.ts.map +1 -0
  68. package/src/routes/voice-profiles-management-routes.ts +476 -0
  69. package/src/routes/voice-speaker-profile-routes.d.ts +57 -0
  70. package/src/routes/voice-speaker-profile-routes.d.ts.map +1 -0
  71. package/src/routes/voice-speaker-profile-routes.ts +199 -0
  72. package/src/runtime/aosp-llama-loader-selection.test.ts +80 -0
  73. package/src/runtime/capacitor-llama.d.ts +25 -0
  74. package/src/runtime/embedding-manager-support.d.ts +77 -0
  75. package/src/runtime/embedding-manager-support.d.ts.map +1 -0
  76. package/src/runtime/embedding-manager-support.ts +497 -0
  77. package/src/runtime/embedding-presets.d.ts +16 -0
  78. package/src/runtime/embedding-presets.d.ts.map +1 -0
  79. package/src/runtime/embedding-presets.ts +81 -0
  80. package/src/runtime/embedding-warmup-policy.d.ts +14 -0
  81. package/src/runtime/embedding-warmup-policy.d.ts.map +1 -0
  82. package/src/runtime/embedding-warmup-policy.test.ts +53 -0
  83. package/src/runtime/embedding-warmup-policy.ts +48 -0
  84. package/src/runtime/ensure-local-inference-handler.d.ts +53 -0
  85. package/src/runtime/ensure-local-inference-handler.d.ts.map +1 -0
  86. package/src/runtime/ensure-local-inference-handler.test.ts +528 -0
  87. package/src/runtime/ensure-local-inference-handler.ts +1398 -0
  88. package/src/runtime/index.d.ts +14 -0
  89. package/src/runtime/index.d.ts.map +1 -0
  90. package/src/runtime/index.ts +27 -0
  91. package/src/runtime/mobile-local-inference-gate.d.ts +31 -0
  92. package/src/runtime/mobile-local-inference-gate.d.ts.map +1 -0
  93. package/src/runtime/mobile-local-inference-gate.test.ts +69 -0
  94. package/src/runtime/mobile-local-inference-gate.ts +44 -0
  95. package/src/runtime/voice-entity-binding.d.ts +103 -0
  96. package/src/runtime/voice-entity-binding.d.ts.map +1 -0
  97. package/src/runtime/voice-entity-binding.transcript.test.ts +69 -0
  98. package/src/runtime/voice-entity-binding.ts +328 -0
  99. package/src/services/README.md +71 -0
  100. package/src/services/__tests__/backend-selector.test.ts +101 -0
  101. package/src/services/__tests__/checkpoint-manager.test.ts +376 -0
  102. package/src/services/__tests__/gpu-autotune.test.ts +400 -0
  103. package/src/services/__tests__/llm-streaming-binding.test.ts +85 -0
  104. package/src/services/__tests__/planner-grammar.test.ts +372 -0
  105. package/src/services/__tests__/runtime-target.test.ts +176 -0
  106. package/src/services/active-model-switch-rollback.test.ts +183 -0
  107. package/src/services/active-model.d.ts +282 -0
  108. package/src/services/active-model.d.ts.map +1 -0
  109. package/src/services/active-model.ts +1213 -0
  110. package/src/services/asr/errors.d.ts +21 -0
  111. package/src/services/asr/errors.d.ts.map +1 -0
  112. package/src/services/asr/errors.ts +50 -0
  113. package/src/services/asr/hash.d.ts +28 -0
  114. package/src/services/asr/hash.d.ts.map +1 -0
  115. package/src/services/asr/hash.ts +49 -0
  116. package/src/services/asr/index.d.ts +76 -0
  117. package/src/services/asr/index.d.ts.map +1 -0
  118. package/src/services/asr/index.ts +178 -0
  119. package/src/services/asr/types.d.ts +91 -0
  120. package/src/services/asr/types.d.ts.map +1 -0
  121. package/src/services/asr/types.ts +95 -0
  122. package/src/services/assignments.d.ts +71 -0
  123. package/src/services/assignments.d.ts.map +1 -0
  124. package/src/services/assignments.test.ts +80 -0
  125. package/src/services/assignments.ts +230 -0
  126. package/src/services/backend-selector.ts +95 -0
  127. package/src/services/backend.d.ts +346 -0
  128. package/src/services/backend.d.ts.map +1 -0
  129. package/src/services/backend.ts +612 -0
  130. package/src/services/bundled-models.d.ts +34 -0
  131. package/src/services/bundled-models.d.ts.map +1 -0
  132. package/src/services/bundled-models.ts +129 -0
  133. package/src/services/cache-bridge.d.ts +206 -0
  134. package/src/services/cache-bridge.d.ts.map +1 -0
  135. package/src/services/cache-bridge.test.ts +516 -0
  136. package/src/services/cache-bridge.ts +423 -0
  137. package/src/services/catalog.d.ts +10 -0
  138. package/src/services/catalog.d.ts.map +1 -0
  139. package/src/services/catalog.test.ts +240 -0
  140. package/src/services/catalog.ts +27 -0
  141. package/src/services/checkpoint-client.d.ts +109 -0
  142. package/src/services/checkpoint-client.d.ts.map +1 -0
  143. package/src/services/checkpoint-client.ts +258 -0
  144. package/src/services/checkpoint-manager.ts +474 -0
  145. package/src/services/cloud-fallback.d.ts +102 -0
  146. package/src/services/cloud-fallback.d.ts.map +1 -0
  147. package/src/services/cloud-fallback.ts +230 -0
  148. package/src/services/conversation-registry.d.ts +142 -0
  149. package/src/services/conversation-registry.d.ts.map +1 -0
  150. package/src/services/conversation-registry.test.ts +235 -0
  151. package/src/services/conversation-registry.ts +264 -0
  152. package/src/services/desktop-fused-ffi-backend-runtime.d.ts +92 -0
  153. package/src/services/desktop-fused-ffi-backend-runtime.d.ts.map +1 -0
  154. package/src/services/desktop-fused-ffi-backend-runtime.ts +333 -0
  155. package/src/services/device-bridge.d.ts +188 -0
  156. package/src/services/device-bridge.d.ts.map +1 -0
  157. package/src/services/device-bridge.ts +1237 -0
  158. package/src/services/device-resource-metrics.d.ts +149 -0
  159. package/src/services/device-resource-metrics.d.ts.map +1 -0
  160. package/src/services/device-resource-metrics.test.ts +98 -0
  161. package/src/services/device-resource-metrics.ts +346 -0
  162. package/src/services/device-tier.d.ts +115 -0
  163. package/src/services/device-tier.d.ts.map +1 -0
  164. package/src/services/device-tier.test.ts +371 -0
  165. package/src/services/device-tier.ts +410 -0
  166. package/src/services/downloader.d.ts +82 -0
  167. package/src/services/downloader.d.ts.map +1 -0
  168. package/src/services/downloader.test.ts +724 -0
  169. package/src/services/downloader.ts +899 -0
  170. package/src/services/engine-direct-bundle.test.ts +58 -0
  171. package/src/services/engine-streaming.test.ts +80 -0
  172. package/src/services/engine.d.ts +534 -0
  173. package/src/services/engine.d.ts.map +1 -0
  174. package/src/services/engine.ts +1891 -0
  175. package/src/services/ensure-local-artifacts.integration.test.ts +273 -0
  176. package/src/services/ensure-local-artifacts.test.ts +368 -0
  177. package/src/services/ensure-local-artifacts.ts +351 -0
  178. package/src/services/external-scanner.d.ts +17 -0
  179. package/src/services/external-scanner.d.ts.map +1 -0
  180. package/src/services/external-scanner.ts +312 -0
  181. package/src/services/ffi-llm-mock.ts +354 -0
  182. package/src/services/ffi-llm-streaming-abi.ts +442 -0
  183. package/src/services/ffi-streaming-backend.d.ts +180 -0
  184. package/src/services/ffi-streaming-backend.d.ts.map +1 -0
  185. package/src/services/ffi-streaming-backend.ts +382 -0
  186. package/src/services/ffi-streaming-runner.d.ts +122 -0
  187. package/src/services/ffi-streaming-runner.d.ts.map +1 -0
  188. package/src/services/ffi-streaming-runner.test.ts +60 -0
  189. package/src/services/ffi-streaming-runner.ts +354 -0
  190. package/src/services/ffi-unload-ordering.test.ts +162 -0
  191. package/src/services/gpu-autotune.ts +534 -0
  192. package/src/services/gpu-detect.ts +139 -0
  193. package/src/services/handler-registry.d.ts +72 -0
  194. package/src/services/handler-registry.d.ts.map +1 -0
  195. package/src/services/handler-registry.ts +240 -0
  196. package/src/services/hardware.d.ts +63 -0
  197. package/src/services/hardware.d.ts.map +1 -0
  198. package/src/services/hardware.test.ts +183 -0
  199. package/src/services/hardware.ts +404 -0
  200. package/src/services/hf-search.d.ts +26 -0
  201. package/src/services/hf-search.d.ts.map +1 -0
  202. package/src/services/hf-search.test.ts +69 -0
  203. package/src/services/hf-search.ts +420 -0
  204. package/src/services/image-description-runtime.d.ts +14 -0
  205. package/src/services/image-description-runtime.d.ts.map +1 -0
  206. package/src/services/image-description-runtime.test.ts +61 -0
  207. package/src/services/image-description-runtime.ts +118 -0
  208. package/src/services/imagegen/aosp-unavailable.d.ts +134 -0
  209. package/src/services/imagegen/aosp-unavailable.d.ts.map +1 -0
  210. package/src/services/imagegen/aosp-unavailable.ts +229 -0
  211. package/src/services/imagegen/backend-selector.d.ts +118 -0
  212. package/src/services/imagegen/backend-selector.d.ts.map +1 -0
  213. package/src/services/imagegen/backend-selector.ts +281 -0
  214. package/src/services/imagegen/coreml-unavailable.d.ts +105 -0
  215. package/src/services/imagegen/coreml-unavailable.d.ts.map +1 -0
  216. package/src/services/imagegen/coreml-unavailable.ts +237 -0
  217. package/src/services/imagegen/errors.d.ts +16 -0
  218. package/src/services/imagegen/errors.d.ts.map +1 -0
  219. package/src/services/imagegen/errors.ts +40 -0
  220. package/src/services/imagegen/index.d.ts +58 -0
  221. package/src/services/imagegen/index.d.ts.map +1 -0
  222. package/src/services/imagegen/index.ts +144 -0
  223. package/src/services/imagegen/mflux.d.ts +74 -0
  224. package/src/services/imagegen/mflux.d.ts.map +1 -0
  225. package/src/services/imagegen/mflux.ts +313 -0
  226. package/src/services/imagegen/sd-cpp.d.ts +180 -0
  227. package/src/services/imagegen/sd-cpp.d.ts.map +1 -0
  228. package/src/services/imagegen/sd-cpp.ts +718 -0
  229. package/src/services/imagegen/tensorrt-unavailable.d.ts +83 -0
  230. package/src/services/imagegen/tensorrt-unavailable.d.ts.map +1 -0
  231. package/src/services/imagegen/tensorrt-unavailable.ts +295 -0
  232. package/src/services/imagegen/types.d.ts +181 -0
  233. package/src/services/imagegen/types.d.ts.map +1 -0
  234. package/src/services/imagegen/types.ts +193 -0
  235. package/src/services/index.d.ts +30 -0
  236. package/src/services/index.d.ts.map +1 -0
  237. package/src/services/index.ts +225 -0
  238. package/src/services/inference-capabilities.d.ts +132 -0
  239. package/src/services/inference-capabilities.d.ts.map +1 -0
  240. package/src/services/inference-capabilities.test.ts +75 -0
  241. package/src/services/inference-capabilities.ts +204 -0
  242. package/src/services/inference-telemetry.d.ts +59 -0
  243. package/src/services/inference-telemetry.d.ts.map +1 -0
  244. package/src/services/inference-telemetry.ts +143 -0
  245. package/src/services/ios-llama-streaming.ts +248 -0
  246. package/src/services/kv-spill.d.ts +189 -0
  247. package/src/services/kv-spill.d.ts.map +1 -0
  248. package/src/services/kv-spill.test.ts +222 -0
  249. package/src/services/kv-spill.ts +356 -0
  250. package/src/services/latency-trace.d.ts +346 -0
  251. package/src/services/latency-trace.d.ts.map +1 -0
  252. package/src/services/latency-trace.test.ts +266 -0
  253. package/src/services/latency-trace.ts +844 -0
  254. package/src/services/llama-server-metrics.ts +304 -0
  255. package/src/services/llm-streaming-binding.d.ts +96 -0
  256. package/src/services/llm-streaming-binding.d.ts.map +1 -0
  257. package/src/services/llm-streaming-binding.ts +136 -0
  258. package/src/services/load-args.d.ts +82 -0
  259. package/src/services/load-args.d.ts.map +1 -0
  260. package/src/services/load-args.ts +81 -0
  261. package/src/services/manifest/eliza-1.manifest.v1.json +708 -0
  262. package/src/services/manifest/index.d.ts +4 -0
  263. package/src/services/manifest/index.d.ts.map +1 -0
  264. package/src/services/manifest/index.ts +66 -0
  265. package/src/services/manifest/manifest.test.ts +693 -0
  266. package/src/services/manifest/schema.d.ts +715 -0
  267. package/src/services/manifest/schema.d.ts.map +1 -0
  268. package/src/services/manifest/schema.ts +655 -0
  269. package/src/services/manifest/types.d.ts +30 -0
  270. package/src/services/manifest/types.d.ts.map +1 -0
  271. package/src/services/manifest/types.ts +55 -0
  272. package/src/services/manifest/validator.d.ts +66 -0
  273. package/src/services/manifest/validator.d.ts.map +1 -0
  274. package/src/services/manifest/validator.ts +569 -0
  275. package/src/services/memory-arbiter.d.ts +343 -0
  276. package/src/services/memory-arbiter.d.ts.map +1 -0
  277. package/src/services/memory-arbiter.test.ts +419 -0
  278. package/src/services/memory-arbiter.ts +1000 -0
  279. package/src/services/memory-monitor.d.ts +119 -0
  280. package/src/services/memory-monitor.d.ts.map +1 -0
  281. package/src/services/memory-monitor.test.ts +208 -0
  282. package/src/services/memory-monitor.ts +296 -0
  283. package/src/services/memory-pressure.d.ts +127 -0
  284. package/src/services/memory-pressure.d.ts.map +1 -0
  285. package/src/services/memory-pressure.ts +413 -0
  286. package/src/services/mtp-doctor.d.ts +13 -0
  287. package/src/services/mtp-doctor.d.ts.map +1 -0
  288. package/src/services/mtp-doctor.ts +78 -0
  289. package/src/services/network-policy.d.ts +127 -0
  290. package/src/services/network-policy.d.ts.map +1 -0
  291. package/src/services/network-policy.ts +346 -0
  292. package/src/services/paths.d.ts +6 -0
  293. package/src/services/paths.d.ts.map +1 -0
  294. package/src/services/paths.ts +25 -0
  295. package/src/services/planner-skeleton.d.ts +124 -0
  296. package/src/services/planner-skeleton.d.ts.map +1 -0
  297. package/src/services/planner-skeleton.ts +175 -0
  298. package/src/services/providers.d.ts +38 -0
  299. package/src/services/providers.d.ts.map +1 -0
  300. package/src/services/providers.ts +507 -0
  301. package/src/services/ram-budget-cache.test.ts +163 -0
  302. package/src/services/ram-budget.d.ts +110 -0
  303. package/src/services/ram-budget.d.ts.map +1 -0
  304. package/src/services/ram-budget.ts +0 -0
  305. package/src/services/readiness.d.ts +9 -0
  306. package/src/services/readiness.d.ts.map +1 -0
  307. package/src/services/readiness.test.ts +87 -0
  308. package/src/services/readiness.ts +238 -0
  309. package/src/services/recommendation.d.ts +111 -0
  310. package/src/services/recommendation.d.ts.map +1 -0
  311. package/src/services/recommendation.ts +672 -0
  312. package/src/services/registry.d.ts +35 -0
  313. package/src/services/registry.d.ts.map +1 -0
  314. package/src/services/registry.ts +151 -0
  315. package/src/services/router-handler.d.ts +92 -0
  316. package/src/services/router-handler.d.ts.map +1 -0
  317. package/src/services/router-handler.test.ts +45 -0
  318. package/src/services/router-handler.ts +376 -0
  319. package/src/services/routing-policy.d.ts +55 -0
  320. package/src/services/routing-policy.d.ts.map +1 -0
  321. package/src/services/routing-policy.ts +228 -0
  322. package/src/services/routing-preferences.d.ts +8 -0
  323. package/src/services/routing-preferences.d.ts.map +1 -0
  324. package/src/services/routing-preferences.ts +15 -0
  325. package/src/services/runtime-target.d.ts +98 -0
  326. package/src/services/runtime-target.d.ts.map +1 -0
  327. package/src/services/runtime-target.ts +154 -0
  328. package/src/services/service.d.ts +128 -0
  329. package/src/services/service.d.ts.map +1 -0
  330. package/src/services/service.test.ts +223 -0
  331. package/src/services/service.ts +735 -0
  332. package/src/services/session-pool.d.ts +72 -0
  333. package/src/services/session-pool.d.ts.map +1 -0
  334. package/src/services/session-pool.ts +153 -0
  335. package/src/services/structured-output/deterministic-repair.d.ts +23 -0
  336. package/src/services/structured-output/deterministic-repair.d.ts.map +1 -0
  337. package/src/services/structured-output/deterministic-repair.test.ts +169 -0
  338. package/src/services/structured-output/deterministic-repair.ts +443 -0
  339. package/src/services/structured-output/index.ts +4 -0
  340. package/src/services/structured-output.d.ts +311 -0
  341. package/src/services/structured-output.d.ts.map +1 -0
  342. package/src/services/structured-output.test.ts +483 -0
  343. package/src/services/structured-output.ts +712 -0
  344. package/src/services/transcription-priority.test.ts +211 -0
  345. package/src/services/tts/errors.ts +46 -0
  346. package/src/services/tts/index.ts +214 -0
  347. package/src/services/tts/tts-audio-cache.ts +235 -0
  348. package/src/services/tts/types.ts +157 -0
  349. package/src/services/types.d.ts +19 -0
  350. package/src/services/types.d.ts.map +1 -0
  351. package/src/services/types.ts +55 -0
  352. package/src/services/verify-on-device.d.ts +34 -0
  353. package/src/services/verify-on-device.d.ts.map +1 -0
  354. package/src/services/verify-on-device.test.ts +87 -0
  355. package/src/services/verify-on-device.ts +127 -0
  356. package/src/services/verify.d.ts +8 -0
  357. package/src/services/verify.d.ts.map +1 -0
  358. package/src/services/verify.ts +13 -0
  359. package/src/services/vision/aosp-unavailable.d.ts +115 -0
  360. package/src/services/vision/aosp-unavailable.d.ts.map +1 -0
  361. package/src/services/vision/aosp-unavailable.ts +163 -0
  362. package/src/services/vision/capacitor-llama.d.ts +99 -0
  363. package/src/services/vision/capacitor-llama.d.ts.map +1 -0
  364. package/src/services/vision/capacitor-llama.ts +255 -0
  365. package/src/services/vision/cloud-fallback.d.ts +47 -0
  366. package/src/services/vision/cloud-fallback.d.ts.map +1 -0
  367. package/src/services/vision/cloud-fallback.test.ts +243 -0
  368. package/src/services/vision/cloud-fallback.ts +268 -0
  369. package/src/services/vision/fallback-chain.test.ts +86 -0
  370. package/src/services/vision/hash.d.ts +71 -0
  371. package/src/services/vision/hash.d.ts.map +1 -0
  372. package/src/services/vision/hash.ts +157 -0
  373. package/src/services/vision/index.d.ts +95 -0
  374. package/src/services/vision/index.d.ts.map +1 -0
  375. package/src/services/vision/index.ts +251 -0
  376. package/src/services/vision/llama-server.d.ts +73 -0
  377. package/src/services/vision/llama-server.d.ts.map +1 -0
  378. package/src/services/vision/llama-server.ts +177 -0
  379. package/src/services/vision/types.d.ts +153 -0
  380. package/src/services/vision/types.d.ts.map +1 -0
  381. package/src/services/vision/types.ts +154 -0
  382. package/src/services/vision/vast-fallback.d.ts +18 -0
  383. package/src/services/vision/vast-fallback.d.ts.map +1 -0
  384. package/src/services/vision/vast-fallback.ts +127 -0
  385. package/src/services/vision-embedding-cache.d.ts +98 -0
  386. package/src/services/vision-embedding-cache.d.ts.map +1 -0
  387. package/src/services/vision-embedding-cache.ts +189 -0
  388. package/src/services/voice/VOICE_WORKBENCH.md +88 -0
  389. package/src/services/voice/__test-helpers__/fake-ffi.ts +92 -0
  390. package/src/services/voice/__test-helpers__/synthetic-speech.ts +124 -0
  391. package/src/services/voice/__tests__/checkpoint-manager.test.ts +241 -0
  392. package/src/services/voice/__tests__/checkpoint-policy.test.ts +270 -0
  393. package/src/services/voice/__tests__/eager-context-builder.test.ts +257 -0
  394. package/src/services/voice/__tests__/eliza1-eot-scorer.test.ts +288 -0
  395. package/src/services/voice/__tests__/eot-classifier.test.ts +431 -0
  396. package/src/services/voice/__tests__/optimistic-rollback.test.ts +312 -0
  397. package/src/services/voice/__tests__/prefill-client.test.ts +266 -0
  398. package/src/services/voice/__tests__/prefix-preserving-queue.test.ts +208 -0
  399. package/src/services/voice/__tests__/streaming-asr.test.ts +450 -0
  400. package/src/services/voice/__tests__/streaming-transcriber.test.ts +339 -0
  401. package/src/services/voice/__tests__/turn-detector-resolver.test.ts +197 -0
  402. package/src/services/voice/__tests__/voice-state-machine-prefill.test.ts +275 -0
  403. package/src/services/voice/__tests__/voice-state-machine.test.ts +354 -0
  404. package/src/services/voice/audio-frame-consumer.d.ts +212 -0
  405. package/src/services/voice/audio-frame-consumer.d.ts.map +1 -0
  406. package/src/services/voice/audio-frame-consumer.test.ts +343 -0
  407. package/src/services/voice/audio-frame-consumer.ts +491 -0
  408. package/src/services/voice/barge-in.d.ts +112 -0
  409. package/src/services/voice/barge-in.d.ts.map +1 -0
  410. package/src/services/voice/barge-in.test.ts +244 -0
  411. package/src/services/voice/barge-in.ts +336 -0
  412. package/src/services/voice/cancellation-coordinator.d.ts +127 -0
  413. package/src/services/voice/cancellation-coordinator.d.ts.map +1 -0
  414. package/src/services/voice/cancellation-coordinator.test.ts +196 -0
  415. package/src/services/voice/cancellation-coordinator.ts +269 -0
  416. package/src/services/voice/checkpoint-manager.d.ts +199 -0
  417. package/src/services/voice/checkpoint-manager.d.ts.map +1 -0
  418. package/src/services/voice/checkpoint-manager.ts +401 -0
  419. package/src/services/voice/checkpoint-policy.ts +336 -0
  420. package/src/services/voice/composite-eot-classifier.test.ts +59 -0
  421. package/src/services/voice/e2e-harness.test.ts +182 -0
  422. package/src/services/voice/e2e-harness.ts +743 -0
  423. package/src/services/voice/eager-context-builder.d.ts +170 -0
  424. package/src/services/voice/eager-context-builder.d.ts.map +1 -0
  425. package/src/services/voice/eager-context-builder.ts +262 -0
  426. package/src/services/voice/eliza1-eot-scorer.d.ts +124 -0
  427. package/src/services/voice/eliza1-eot-scorer.d.ts.map +1 -0
  428. package/src/services/voice/eliza1-eot-scorer.ts +242 -0
  429. package/src/services/voice/embedding-server.ts +200 -0
  430. package/src/services/voice/embedding.d.ts +133 -0
  431. package/src/services/voice/embedding.d.ts.map +1 -0
  432. package/src/services/voice/embedding.test.ts +148 -0
  433. package/src/services/voice/embedding.ts +244 -0
  434. package/src/services/voice/emotion-attribution.d.ts +68 -0
  435. package/src/services/voice/emotion-attribution.d.ts.map +1 -0
  436. package/src/services/voice/emotion-attribution.test.ts +129 -0
  437. package/src/services/voice/emotion-attribution.ts +361 -0
  438. package/src/services/voice/engine-bridge-cancellation.test.ts +422 -0
  439. package/src/services/voice/engine-bridge.d.ts +746 -0
  440. package/src/services/voice/engine-bridge.d.ts.map +1 -0
  441. package/src/services/voice/engine-bridge.test.ts +384 -0
  442. package/src/services/voice/engine-bridge.ts +2226 -0
  443. package/src/services/voice/eot-classifier-ggml.d.ts +179 -0
  444. package/src/services/voice/eot-classifier-ggml.d.ts.map +1 -0
  445. package/src/services/voice/eot-classifier-ggml.ts +566 -0
  446. package/src/services/voice/eot-classifier.d.ts +214 -0
  447. package/src/services/voice/eot-classifier.d.ts.map +1 -0
  448. package/src/services/voice/eot-classifier.ts +533 -0
  449. package/src/services/voice/errors.d.ts +20 -0
  450. package/src/services/voice/errors.d.ts.map +1 -0
  451. package/src/services/voice/errors.ts +32 -0
  452. package/src/services/voice/expressive-tags.d.ts +158 -0
  453. package/src/services/voice/expressive-tags.d.ts.map +1 -0
  454. package/src/services/voice/expressive-tags.ts +405 -0
  455. package/src/services/voice/ffi-bindings.d.ts +636 -0
  456. package/src/services/voice/ffi-bindings.d.ts.map +1 -0
  457. package/src/services/voice/ffi-bindings.test.ts +671 -0
  458. package/src/services/voice/ffi-bindings.ts +3050 -0
  459. package/src/services/voice/first-line-cache.d.ts +181 -0
  460. package/src/services/voice/first-line-cache.d.ts.map +1 -0
  461. package/src/services/voice/first-line-cache.ts +725 -0
  462. package/src/services/voice/fused-eot-scorer.d.ts +51 -0
  463. package/src/services/voice/fused-eot-scorer.d.ts.map +1 -0
  464. package/src/services/voice/fused-eot-scorer.ts +135 -0
  465. package/src/services/voice/index.d.ts +91 -0
  466. package/src/services/voice/index.d.ts.map +1 -0
  467. package/src/services/voice/index.ts +481 -0
  468. package/src/services/voice/kokoro/__tests__/kokoro-backend.test.ts +151 -0
  469. package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.real.test.ts +151 -0
  470. package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.test.ts +60 -0
  471. package/src/services/voice/kokoro/__tests__/kokoro-engine-discovery.test.ts +277 -0
  472. package/src/services/voice/kokoro/__tests__/kokoro-ffi-runtime.test.ts +235 -0
  473. package/src/services/voice/kokoro/__tests__/kokoro-runtime.test.ts +95 -0
  474. package/src/services/voice/kokoro/__tests__/phonemizer.test.ts +53 -0
  475. package/src/services/voice/kokoro/__tests__/runtime-selection.test.ts +231 -0
  476. package/src/services/voice/kokoro/__tests__/voices.test.ts +57 -0
  477. package/src/services/voice/kokoro/index.ts +79 -0
  478. package/src/services/voice/kokoro/kokoro-backend.d.ts +72 -0
  479. package/src/services/voice/kokoro/kokoro-backend.d.ts.map +1 -0
  480. package/src/services/voice/kokoro/kokoro-backend.ts +207 -0
  481. package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts +58 -0
  482. package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts.map +1 -0
  483. package/src/services/voice/kokoro/kokoro-engine-discovery.ts +177 -0
  484. package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts +75 -0
  485. package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts.map +1 -0
  486. package/src/services/voice/kokoro/kokoro-ffi-runtime.ts +233 -0
  487. package/src/services/voice/kokoro/kokoro-runtime.d.ts +100 -0
  488. package/src/services/voice/kokoro/kokoro-runtime.d.ts.map +1 -0
  489. package/src/services/voice/kokoro/kokoro-runtime.ts +170 -0
  490. package/src/services/voice/kokoro/phoneme-stream.ts +123 -0
  491. package/src/services/voice/kokoro/phonemizer.d.ts +50 -0
  492. package/src/services/voice/kokoro/phonemizer.d.ts.map +1 -0
  493. package/src/services/voice/kokoro/phonemizer.ts +344 -0
  494. package/src/services/voice/kokoro/pick-runtime.d.ts +61 -0
  495. package/src/services/voice/kokoro/pick-runtime.d.ts.map +1 -0
  496. package/src/services/voice/kokoro/pick-runtime.test.ts +91 -0
  497. package/src/services/voice/kokoro/pick-runtime.ts +130 -0
  498. package/src/services/voice/kokoro/runtime-selection.d.ts +92 -0
  499. package/src/services/voice/kokoro/runtime-selection.d.ts.map +1 -0
  500. package/src/services/voice/kokoro/runtime-selection.ts +237 -0
  501. package/src/services/voice/kokoro/types.d.ts +82 -0
  502. package/src/services/voice/kokoro/types.d.ts.map +1 -0
  503. package/src/services/voice/kokoro/types.ts +95 -0
  504. package/src/services/voice/kokoro/voice-presets.d.ts +23 -0
  505. package/src/services/voice/kokoro/voice-presets.d.ts.map +1 -0
  506. package/src/services/voice/kokoro/voice-presets.ts +129 -0
  507. package/src/services/voice/kokoro/voices.d.ts +30 -0
  508. package/src/services/voice/kokoro/voices.d.ts.map +1 -0
  509. package/src/services/voice/kokoro/voices.ts +64 -0
  510. package/src/services/voice/lifecycle.d.ts +135 -0
  511. package/src/services/voice/lifecycle.d.ts.map +1 -0
  512. package/src/services/voice/lifecycle.test.ts +315 -0
  513. package/src/services/voice/lifecycle.ts +301 -0
  514. package/src/services/voice/live-diarization-session.d.ts +96 -0
  515. package/src/services/voice/live-diarization-session.d.ts.map +1 -0
  516. package/src/services/voice/live-diarization-session.ts +289 -0
  517. package/src/services/voice/mic-source.d.ts +136 -0
  518. package/src/services/voice/mic-source.d.ts.map +1 -0
  519. package/src/services/voice/mic-source.test.ts +210 -0
  520. package/src/services/voice/mic-source.ts +503 -0
  521. package/src/services/voice/optimistic-policy.d.ts +109 -0
  522. package/src/services/voice/optimistic-policy.d.ts.map +1 -0
  523. package/src/services/voice/optimistic-policy.test.ts +101 -0
  524. package/src/services/voice/optimistic-policy.ts +192 -0
  525. package/src/services/voice/optimistic-rollback.ts +343 -0
  526. package/src/services/voice/partial-stabilizer.d.ts +73 -0
  527. package/src/services/voice/partial-stabilizer.d.ts.map +1 -0
  528. package/src/services/voice/partial-stabilizer.test.ts +68 -0
  529. package/src/services/voice/partial-stabilizer.ts +140 -0
  530. package/src/services/voice/phoneme-tokenizer.d.ts +49 -0
  531. package/src/services/voice/phoneme-tokenizer.d.ts.map +1 -0
  532. package/src/services/voice/phoneme-tokenizer.ts +158 -0
  533. package/src/services/voice/phrase-cache.d.ts +76 -0
  534. package/src/services/voice/phrase-cache.d.ts.map +1 -0
  535. package/src/services/voice/phrase-cache.test.ts +242 -0
  536. package/src/services/voice/phrase-cache.ts +186 -0
  537. package/src/services/voice/phrase-chunker.d.ts +62 -0
  538. package/src/services/voice/phrase-chunker.d.ts.map +1 -0
  539. package/src/services/voice/phrase-chunker.test.ts +239 -0
  540. package/src/services/voice/phrase-chunker.ts +281 -0
  541. package/src/services/voice/pipeline-impls.d.ts +151 -0
  542. package/src/services/voice/pipeline-impls.d.ts.map +1 -0
  543. package/src/services/voice/pipeline-impls.l6.test.ts +110 -0
  544. package/src/services/voice/pipeline-impls.test.ts +292 -0
  545. package/src/services/voice/pipeline-impls.ts +315 -0
  546. package/src/services/voice/pipeline.d.ts +216 -0
  547. package/src/services/voice/pipeline.d.ts.map +1 -0
  548. package/src/services/voice/pipeline.ts +505 -0
  549. package/src/services/voice/prefill-client.d.ts +123 -0
  550. package/src/services/voice/prefill-client.d.ts.map +1 -0
  551. package/src/services/voice/prefill-client.ts +316 -0
  552. package/src/services/voice/prefix-preserving-queue.d.ts +113 -0
  553. package/src/services/voice/prefix-preserving-queue.d.ts.map +1 -0
  554. package/src/services/voice/prefix-preserving-queue.ts +162 -0
  555. package/src/services/voice/profile-store.d.ts +248 -0
  556. package/src/services/voice/profile-store.d.ts.map +1 -0
  557. package/src/services/voice/profile-store.ts +887 -0
  558. package/src/services/voice/ring-buffer.d.ts +40 -0
  559. package/src/services/voice/ring-buffer.d.ts.map +1 -0
  560. package/src/services/voice/ring-buffer.ts +105 -0
  561. package/src/services/voice/rollback-queue.d.ts +24 -0
  562. package/src/services/voice/rollback-queue.d.ts.map +1 -0
  563. package/src/services/voice/rollback-queue.ts +74 -0
  564. package/src/services/voice/samantha-preset-placeholder.d.ts +67 -0
  565. package/src/services/voice/samantha-preset-placeholder.d.ts.map +1 -0
  566. package/src/services/voice/samantha-preset-placeholder.test.ts +97 -0
  567. package/src/services/voice/samantha-preset-placeholder.ts +148 -0
  568. package/src/services/voice/samantha-preset-regenerator.d.ts +87 -0
  569. package/src/services/voice/samantha-preset-regenerator.d.ts.map +1 -0
  570. package/src/services/voice/samantha-preset-regenerator.ts +393 -0
  571. package/src/services/voice/scheduler.d.ts +146 -0
  572. package/src/services/voice/scheduler.d.ts.map +1 -0
  573. package/src/services/voice/scheduler.t2.test.ts +141 -0
  574. package/src/services/voice/scheduler.ts +927 -0
  575. package/src/services/voice/shared-resources.d.ts +190 -0
  576. package/src/services/voice/shared-resources.d.ts.map +1 -0
  577. package/src/services/voice/shared-resources.ts +320 -0
  578. package/src/services/voice/speaker/attribution-pipeline.d.ts +74 -0
  579. package/src/services/voice/speaker/attribution-pipeline.d.ts.map +1 -0
  580. package/src/services/voice/speaker/attribution-pipeline.ts +386 -0
  581. package/src/services/voice/speaker/diarizer-fused.d.ts +59 -0
  582. package/src/services/voice/speaker/diarizer-fused.d.ts.map +1 -0
  583. package/src/services/voice/speaker/diarizer-fused.real.test.ts +100 -0
  584. package/src/services/voice/speaker/diarizer-fused.ts +154 -0
  585. package/src/services/voice/speaker/diarizer.d.ts +75 -0
  586. package/src/services/voice/speaker/diarizer.d.ts.map +1 -0
  587. package/src/services/voice/speaker/diarizer.ts +218 -0
  588. package/src/services/voice/speaker/encoder-fused.d.ts +60 -0
  589. package/src/services/voice/speaker/encoder-fused.d.ts.map +1 -0
  590. package/src/services/voice/speaker/encoder-fused.real.test.ts +113 -0
  591. package/src/services/voice/speaker/encoder-fused.ts +138 -0
  592. package/src/services/voice/speaker/encoder-ggml.d.ts +33 -0
  593. package/src/services/voice/speaker/encoder-ggml.d.ts.map +1 -0
  594. package/src/services/voice/speaker/encoder-ggml.ts +79 -0
  595. package/src/services/voice/speaker/encoder.d.ts +37 -0
  596. package/src/services/voice/speaker/encoder.d.ts.map +1 -0
  597. package/src/services/voice/speaker/encoder.ts +105 -0
  598. package/src/services/voice/speaker-imprint.d.ts +83 -0
  599. package/src/services/voice/speaker-imprint.d.ts.map +1 -0
  600. package/src/services/voice/speaker-imprint.test.ts +185 -0
  601. package/src/services/voice/speaker-imprint.ts +312 -0
  602. package/src/services/voice/speaker-preset-cache.d.ts +77 -0
  603. package/src/services/voice/speaker-preset-cache.d.ts.map +1 -0
  604. package/src/services/voice/speaker-preset-cache.test.ts +154 -0
  605. package/src/services/voice/speaker-preset-cache.ts +195 -0
  606. package/src/services/voice/streaming-asr/streaming-pipeline-adapter.ts +292 -0
  607. package/src/services/voice/system-audio-sink.d.ts +73 -0
  608. package/src/services/voice/system-audio-sink.d.ts.map +1 -0
  609. package/src/services/voice/system-audio-sink.test.ts +29 -0
  610. package/src/services/voice/system-audio-sink.ts +366 -0
  611. package/src/services/voice/transcriber.d.ts +244 -0
  612. package/src/services/voice/transcriber.d.ts.map +1 -0
  613. package/src/services/voice/transcriber.test.ts +392 -0
  614. package/src/services/voice/transcriber.ts +704 -0
  615. package/src/services/voice/turn-controller.d.ts +183 -0
  616. package/src/services/voice/turn-controller.d.ts.map +1 -0
  617. package/src/services/voice/turn-controller.test.ts +575 -0
  618. package/src/services/voice/turn-controller.ts +596 -0
  619. package/src/services/voice/types.d.ts +643 -0
  620. package/src/services/voice/types.d.ts.map +1 -0
  621. package/src/services/voice/types.ts +699 -0
  622. package/src/services/voice/vad.d.ts +282 -0
  623. package/src/services/voice/vad.d.ts.map +1 -0
  624. package/src/services/voice/vad.test.ts +480 -0
  625. package/src/services/voice/vad.ts +827 -0
  626. package/src/services/voice/vad.v1-v4.test.ts +222 -0
  627. package/src/services/voice/voice-budget.d.ts +241 -0
  628. package/src/services/voice/voice-budget.d.ts.map +1 -0
  629. package/src/services/voice/voice-budget.test.ts +420 -0
  630. package/src/services/voice/voice-budget.ts +656 -0
  631. package/src/services/voice/voice-duet.test.ts +375 -0
  632. package/src/services/voice/voice-emotion-classifier.d.ts +95 -0
  633. package/src/services/voice/voice-emotion-classifier.d.ts.map +1 -0
  634. package/src/services/voice/voice-emotion-classifier.test.ts +210 -0
  635. package/src/services/voice/voice-emotion-classifier.ts +273 -0
  636. package/src/services/voice/voice-preset-format.d.ts +158 -0
  637. package/src/services/voice/voice-preset-format.d.ts.map +1 -0
  638. package/src/services/voice/voice-preset-format.ts +700 -0
  639. package/src/services/voice/voice-preset-generator.test.ts +89 -0
  640. package/src/services/voice/voice-profile-artifact.d.ts +116 -0
  641. package/src/services/voice/voice-profile-artifact.d.ts.map +1 -0
  642. package/src/services/voice/voice-profile-artifact.test.ts +138 -0
  643. package/src/services/voice/voice-profile-artifact.ts +518 -0
  644. package/src/services/voice/voice-profile-routes.d.ts +83 -0
  645. package/src/services/voice/voice-profile-routes.d.ts.map +1 -0
  646. package/src/services/voice/voice-profile-routes.test.ts +429 -0
  647. package/src/services/voice/voice-profile-routes.ts +425 -0
  648. package/src/services/voice/voice-scenario.ts +154 -0
  649. package/src/services/voice/voice-settings.d.ts +82 -0
  650. package/src/services/voice/voice-settings.d.ts.map +1 -0
  651. package/src/services/voice/voice-settings.ts +172 -0
  652. package/src/services/voice/voice-state-machine.d.ts +364 -0
  653. package/src/services/voice/voice-state-machine.d.ts.map +1 -0
  654. package/src/services/voice/voice-state-machine.ts +727 -0
  655. package/src/services/voice/voice-workbench-report.test.ts +168 -0
  656. package/src/services/voice/voice-workbench-report.ts +326 -0
  657. package/src/services/voice/voice-workbench.test.ts +158 -0
  658. package/src/services/voice/voice.test.ts +1070 -0
  659. package/src/services/voice/wake-word-ggml.d.ts +101 -0
  660. package/src/services/voice/wake-word-ggml.d.ts.map +1 -0
  661. package/src/services/voice/wake-word-ggml.ts +320 -0
  662. package/src/services/voice/wake-word.d.ts +255 -0
  663. package/src/services/voice/wake-word.d.ts.map +1 -0
  664. package/src/services/voice/wake-word.test.ts +298 -0
  665. package/src/services/voice/wake-word.ts +554 -0
  666. package/src/services/voice/wrap-with-first-line-cache.d.ts +70 -0
  667. package/src/services/voice/wrap-with-first-line-cache.d.ts.map +1 -0
  668. package/src/services/voice/wrap-with-first-line-cache.ts +267 -0
  669. package/src/services/voice-model-updater.d.ts +240 -0
  670. package/src/services/voice-model-updater.d.ts.map +1 -0
  671. package/src/services/voice-model-updater.ts +724 -0
  672. package/src/services/voice-prewarm.d.ts +3 -0
  673. package/src/services/voice-prewarm.d.ts.map +1 -0
  674. package/src/services/voice-prewarm.ts +51 -0
  675. package/dist/index.d.ts +0 -37
  676. package/dist/index.js +0 -1098
@@ -0,0 +1,2226 @@
1
+ /**
2
+ * Engine ↔ voice scheduler bridge.
3
+ *
4
+ * Adapts the live `LocalInferenceEngine` (`engine.ts`) plus the MTP
5
+ * llama-server (`ffi-streaming-backend.ts`) onto the voice scaffold's
6
+ * `VoiceScheduler`. See `packages/inference/AGENTS.md` §4 for the
7
+ * streaming graph this implements:
8
+ *
9
+ * ASR → text tokens → MTP drafter ↔ target verifier (text model)
10
+ * → phrase chunker → speaker preset cache + phrase cache
11
+ * → OmniVoice TTS → PCM ring buffer → audio out
12
+ *
13
+ * Plus rollback queue (MTP rejection → cancel pending TTS chunks)
14
+ * and barge-in cancellation (mic VAD → drain ring buffer + cancel TTS).
15
+ *
16
+ * Two TTS backends are exposed:
17
+ * - `StubOmniVoiceBackend`: deterministic synthetic PCM. Used by tests
18
+ * and any path that wants the streaming graph without real audio.
19
+ * - `FfiOmniVoiceBackend`: forwards through the fused
20
+ * `libelizainference.{dylib,so,dll}` ABI. The bridge creates the
21
+ * context lazily when voice is armed or first used, so voice-off
22
+ * does not keep OmniVoice weights resident.
23
+ *
24
+ * Per AGENTS.md §3 + §9 (no defensive code, no log-and-continue), every
25
+ * startup precondition surfaces as a thrown `VoiceStartupError`. There
26
+ * is no silent fallback to text-only.
27
+ */
28
+
29
+ import { existsSync, readdirSync, statSync } from "node:fs";
30
+ import os from "node:os";
31
+ import path from "node:path";
32
+ import type { IAgentRuntime } from "@elizaos/core";
33
+ import { logger } from "@elizaos/core";
34
+ import type { VoiceCancellationReason } from "@elizaos/shared";
35
+ import { localInferenceRoot } from "../paths";
36
+ import {
37
+ type CoordinatorRuntime,
38
+ VoiceCancellationCoordinator,
39
+ } from "./cancellation-coordinator";
40
+ import { VoiceStartupError } from "./errors";
41
+ import type {
42
+ ElizaInferenceContextHandle,
43
+ ElizaInferenceFfi,
44
+ NativeVerifierEvent,
45
+ } from "./ffi-bindings";
46
+ import { loadElizaInferenceFfi } from "./ffi-bindings";
47
+ import { KokoroTtsBackend } from "./kokoro/kokoro-backend";
48
+ import type { KokoroEngineDiscoveryResult } from "./kokoro/kokoro-engine-discovery";
49
+ import { pickKokoroRuntimeBackend } from "./kokoro/pick-runtime";
50
+ import {
51
+ VoiceLifecycle,
52
+ VoiceLifecycleError,
53
+ type VoiceLifecycleLoaders,
54
+ } from "./lifecycle";
55
+ import {
56
+ OptimisticGenerationPolicy,
57
+ type OptimisticPolicyOptions,
58
+ resolvePowerSourceState,
59
+ } from "./optimistic-policy";
60
+ import {
61
+ type CachedPhraseAudio,
62
+ DEFAULT_PHRASE_CACHE_SEED,
63
+ FIRST_AUDIO_FILLERS,
64
+ PhraseCache,
65
+ } from "./phrase-cache";
66
+ import {
67
+ VoicePipeline,
68
+ type VoicePipelineConfig,
69
+ type VoicePipelineDeps,
70
+ type VoicePipelineEvents,
71
+ } from "./pipeline";
72
+ import {
73
+ MissingAsrTranscriber,
74
+ MtpDraftProposer,
75
+ MtpTargetVerifier,
76
+ type MtpTextRunner,
77
+ } from "./pipeline-impls";
78
+ import type { VoiceProfileStore } from "./profile-store";
79
+ import { type SchedulerEvents, VoiceScheduler } from "./scheduler";
80
+ import {
81
+ type MmapRegionHandle,
82
+ SharedResourceRegistry,
83
+ } from "./shared-resources";
84
+ import {
85
+ type VoiceAttributionOutput,
86
+ VoiceAttributionPipeline,
87
+ } from "./speaker/attribution-pipeline";
88
+ import {
89
+ type Diarizer,
90
+ PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID,
91
+ } from "./speaker/diarizer";
92
+ import { FusedDiarizer } from "./speaker/diarizer-fused";
93
+ import type { SpeakerEncoder } from "./speaker/encoder";
94
+ import { FusedSpeakerEncoder } from "./speaker/encoder-fused";
95
+ import {
96
+ SPEAKER_GGML_EMBEDDING_DIM,
97
+ SPEAKER_GGML_SAMPLE_RATE,
98
+ } from "./speaker/encoder-ggml";
99
+ import {
100
+ DEFAULT_VOICE_PRESET_REL_PATH,
101
+ SpeakerPresetCache,
102
+ } from "./speaker-preset-cache";
103
+ import {
104
+ ASR_SAMPLE_RATE,
105
+ AsrUnavailableError,
106
+ createStreamingTranscriber,
107
+ resampleLinear,
108
+ } from "./transcriber";
109
+ import type {
110
+ AudioChunk,
111
+ AudioSink,
112
+ OmniVoiceBackend,
113
+ Phrase,
114
+ RejectedTokenRange,
115
+ SchedulerConfig,
116
+ SpeakerPreset,
117
+ StreamingTranscriber,
118
+ TextToken,
119
+ TranscriptionAudio,
120
+ VadEventSource,
121
+ } from "./types";
122
+
123
+ const SAMPLE_RATE_DEFAULT = 24_000;
124
+ const RING_BUFFER_CAPACITY_DEFAULT = SAMPLE_RATE_DEFAULT * 4; // 4s
125
+ /**
126
+ * Runtime default for the no-punctuation phrase cap (`PhraseChunker.maxTokensPerPhrase`).
127
+ * Punctuation (`, . ! ?`) is still the primary boundary; this only bounds
128
+ * a run-on token stream. Kept small — equal to the MTP draft window
129
+ * (`DEFAULT_VOICE_MAX_DRAFT_TOKENS` in `engine.ts`) — so first-audio latency
130
+ * is bounded (a phrase ≈ one draft round of audio, not 30 words) and a
131
+ * MTP-reject rollback drops at most one un-spoken chunk (AGENTS.md §4 —
132
+ * "small chunk = low latency cost on rollback"). Override per bridge via
133
+ * `maxTokensPerPhrase` or `ELIZA_VOICE_MAX_TOKENS_PER_PHRASE`. The
134
+ * `PhraseChunker` primitive keeps the AGENTS-spec 30-word default for
135
+ * non-runtime callers.
136
+ */
137
+ const PHRASE_MAX_TOKENS_DEFAULT = 8;
138
+ const STUB_PCM_MS_PER_PHRASE = 100;
139
+ const STUB_PCM_STREAM_CHUNKS = 4;
140
+
141
+ /**
142
+ * Resolve the `speaker_preset_id` value to send across the FFI boundary.
143
+ *
144
+ * Historically this returned `null` for the default voice — the C side then
145
+ * treated `null` as "auto-voice mode" and ignored any preset file under
146
+ * `cache/voice-preset-default.bin`. That was the right behaviour when the
147
+ * default preset was a 256-fp32-zero placeholder; it's wrong now that the
148
+ * default preset can be a real (v2) OmniVoice sam freeze. With ABI v4
149
+ * the FFI bridge looks up `<bundle>/cache/voice-preset-<id>.bin` when the
150
+ * id is supplied and applies the `(instruct, ref_audio_tokens, ref_text)`
151
+ * triple to `ov_tts_params` — so we must always pass the id.
152
+ *
153
+ * The only case we return `null` is when the preset shape is degenerate
154
+ * (no embedding, no ref-audio-tokens, no instruct) — i.e. an explicit
155
+ * "no preset" signal from a caller that doesn't want a voice bound. The
156
+ * FFI side honours `null` by running OmniVoice's intrinsic auto-voice
157
+ * path.
158
+ */
159
+ function ffiSpeakerPresetId(preset: SpeakerPreset): string | null {
160
+ const hasV2Payload =
161
+ (preset.instruct !== undefined && preset.instruct.length > 0) ||
162
+ (preset.refText !== undefined && preset.refText.length > 0) ||
163
+ (preset.refAudioTokens !== undefined &&
164
+ preset.refAudioTokens.tokens.length > 0);
165
+ const hasEmbedding = preset.embedding.length > 0;
166
+ if (!hasV2Payload && !hasEmbedding) {
167
+ // Degenerate preset (e.g. the 1052-byte all-zero placeholder). The C
168
+ // side cannot do anything useful with it; let OmniVoice pick its own
169
+ // voice via the auto-voice path.
170
+ return null;
171
+ }
172
+ return preset.voiceId;
173
+ }
174
+
175
+ /** Re-exported from `./errors` so existing `engine-bridge` importers don't churn. */
176
+ export { VoiceStartupError };
177
+
178
+ /**
179
+ * Native verifier callbacks report rejected token ranges as half-open
180
+ * `[from, to)` intervals. The scheduler rollback queue uses inclusive
181
+ * token indexes, so convert in exactly one place.
182
+ */
183
+ export function nativeRejectedRangeToRollbackRange(
184
+ event: Pick<NativeVerifierEvent, "rejectedFrom" | "rejectedTo">,
185
+ ): RejectedTokenRange | null {
186
+ if (event.rejectedFrom < 0 || event.rejectedTo <= event.rejectedFrom) {
187
+ return null;
188
+ }
189
+ return {
190
+ fromIndex: event.rejectedFrom,
191
+ toIndex: event.rejectedTo - 1,
192
+ };
193
+ }
194
+
195
+ /**
196
+ * One PCM segment delivered to a `StreamingTtsBackend.synthesizeStream`
197
+ * consumer (W9's scheduler) as TTS decodes it. `isFinal` marks the
198
+ * zero-length tail chunk that closes the phrase.
199
+ */
200
+ export interface TtsPcmChunk {
201
+ pcm: Float32Array;
202
+ sampleRate: number;
203
+ isFinal: boolean;
204
+ }
205
+
206
+ /**
207
+ * Streaming-TTS seam between the fused `libelizainference` runtime and
208
+ * W9's voice scheduler. The scheduler calls `synthesizeStream(...)` for
209
+ * a phrase and writes each delivered `pcm` segment into the
210
+ * `PcmRingBuffer` on the same scheduler tick (AGENTS.md §4 —
211
+ * phrase-chunk → TTS within one scheduler tick); returning `true` from
212
+ * `onChunk` (or flipping `cancelSignal.cancelled`) hard-cancels the
213
+ * in-flight forward pass at the next kernel boundary (barge-in /
214
+ * MTP-rejected tail).
215
+ *
216
+ * Both `OmniVoiceBackend` implementations in this module satisfy it:
217
+ * - `FfiOmniVoiceBackend` forwards to
218
+ * `eliza_inference_tts_synthesize_stream` when the loaded build
219
+ * advertises streaming TTS (`tts_stream_supported() == 1`), else it
220
+ * synthesizes whole and emits the result as one body chunk + a final
221
+ * tail (no silent "streaming" lie — the chunk count just collapses
222
+ * to one when the build is non-streaming);
223
+ * - `StubOmniVoiceBackend` emits deterministic synthetic PCM split
224
+ * into a fixed number of chunks so scheduler tests can observe the
225
+ * incremental handoff without a real model.
226
+ */
227
+ export interface StreamingTtsBackend {
228
+ /**
229
+ * Synthesize `phrase` with `preset` and deliver PCM in chunks. The
230
+ * scheduler owns the ring-buffer write inside `onChunk`. Resolves with
231
+ * `cancelled: true` if `onChunk` requested a stop (or `cancelSignal`
232
+ * was set), `false` on a clean finish. The final `onChunk` call always
233
+ * has `isFinal: true` (possibly a zero-length `pcm`) so the consumer
234
+ * can settle per-phrase state.
235
+ */
236
+ synthesizeStream(args: {
237
+ phrase: Phrase;
238
+ preset: SpeakerPreset;
239
+ cancelSignal: { cancelled: boolean };
240
+ onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
241
+ onKernelTick?: () => void;
242
+ }): Promise<{ cancelled: boolean }>;
243
+ }
244
+
245
+ /** True when `backend` implements the `StreamingTtsBackend` seam. */
246
+ export function isStreamingTtsBackend(
247
+ backend: OmniVoiceBackend,
248
+ ): backend is OmniVoiceBackend & StreamingTtsBackend {
249
+ return (
250
+ typeof (backend as Partial<StreamingTtsBackend>).synthesizeStream ===
251
+ "function"
252
+ );
253
+ }
254
+
255
+ /**
256
+ * Deterministic test TTS backend. Each phrase yields
257
+ * `STUB_PCM_MS_PER_PHRASE` ms of silence (zeros), with the
258
+ * cancel signal honoured at the kernel-tick boundary so barge-in tests
259
+ * observe cancellation without waiting on a real model.
260
+ */
261
+ export class StubOmniVoiceBackend
262
+ implements OmniVoiceBackend, StreamingTtsBackend
263
+ {
264
+ readonly id = "stub" as const;
265
+ private readonly sampleRate: number;
266
+ calls = 0;
267
+ streamCalls = 0;
268
+
269
+ constructor(sampleRate = SAMPLE_RATE_DEFAULT) {
270
+ this.sampleRate = sampleRate;
271
+ }
272
+
273
+ async synthesize(args: {
274
+ phrase: Phrase;
275
+ preset: SpeakerPreset;
276
+ cancelSignal: { cancelled: boolean };
277
+ onKernelTick?: () => void;
278
+ }): Promise<AudioChunk> {
279
+ this.calls++;
280
+ args.onKernelTick?.();
281
+ const samples = Math.floor(
282
+ (this.sampleRate * STUB_PCM_MS_PER_PHRASE) / 1000,
283
+ );
284
+ const pcm = new Float32Array(samples);
285
+ return {
286
+ phraseId: args.phrase.id,
287
+ fromIndex: args.phrase.fromIndex,
288
+ toIndex: args.phrase.toIndex,
289
+ pcm,
290
+ sampleRate: this.sampleRate,
291
+ };
292
+ }
293
+
294
+ async synthesizeStream(args: {
295
+ phrase: Phrase;
296
+ preset: SpeakerPreset;
297
+ cancelSignal: { cancelled: boolean };
298
+ onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
299
+ onKernelTick?: () => void;
300
+ }): Promise<{ cancelled: boolean }> {
301
+ this.streamCalls++;
302
+ const totalSamples = Math.floor(
303
+ (this.sampleRate * STUB_PCM_MS_PER_PHRASE) / 1000,
304
+ );
305
+ const perChunk = Math.max(
306
+ 1,
307
+ Math.ceil(totalSamples / STUB_PCM_STREAM_CHUNKS),
308
+ );
309
+ let cancelled = false;
310
+ for (let off = 0; off < totalSamples; off += perChunk) {
311
+ args.onKernelTick?.();
312
+ if (args.cancelSignal.cancelled) {
313
+ cancelled = true;
314
+ break;
315
+ }
316
+ const n = Math.min(perChunk, totalSamples - off);
317
+ const want = args.onChunk({
318
+ pcm: new Float32Array(n),
319
+ sampleRate: this.sampleRate,
320
+ isFinal: false,
321
+ });
322
+ if (want === true || args.cancelSignal.cancelled) {
323
+ cancelled = true;
324
+ break;
325
+ }
326
+ }
327
+ args.onChunk({
328
+ pcm: new Float32Array(0),
329
+ sampleRate: this.sampleRate,
330
+ isFinal: true,
331
+ });
332
+ return { cancelled };
333
+ }
334
+ }
335
+
336
+ /**
337
+ * FFI-backed TTS backend. Forwards each `synthesize()` call through the
338
+ * fused `libelizainference` ABI declared in
339
+ * `packages/app-core/scripts/omnivoice-fuse/ffi.h`. The library handle
340
+ * + a per-engine context pointer are held by the bridge and passed in
341
+ * at construction so this backend stays a thin adapter.
342
+ *
343
+ * Until the real fused build ships, the binding is exercised against
344
+ * the compatibility C library at `scripts/omnivoice-fuse/ffi-stub.c`, which returns
345
+ * `ELIZA_ERR_NOT_IMPLEMENTED` for `tts_synthesize` — the binding then
346
+ * raises `VoiceLifecycleError({code:"kernel-missing"})`. The adapter
347
+ * re-wraps that as `VoiceStartupError("missing-fused-build", ...)` so
348
+ * the engine layer's startup-error taxonomy stays unified. No silent
349
+ * fallback (AGENTS.md §3 + §9).
350
+ */
351
+ export class FfiOmniVoiceBackend
352
+ implements OmniVoiceBackend, StreamingTtsBackend
353
+ {
354
+ readonly id = "ffi" as const;
355
+ private readonly ffi: ElizaInferenceFfi;
356
+ private readonly getContext: () => ElizaInferenceContextHandle;
357
+ private readonly sampleRate: number;
358
+ private readonly maxSecondsPerPhrase: number;
359
+
360
+ constructor(args: {
361
+ ffi: ElizaInferenceFfi;
362
+ ctx?: ElizaInferenceContextHandle;
363
+ getContext?: () => ElizaInferenceContextHandle;
364
+ sampleRate?: number;
365
+ maxSecondsPerPhrase?: number;
366
+ }) {
367
+ this.ffi = args.ffi;
368
+ this.getContext =
369
+ args.getContext ??
370
+ (() => {
371
+ if (args.ctx === undefined) {
372
+ throw new VoiceStartupError(
373
+ "missing-fused-build",
374
+ "[voice] FFI backend has no context provider",
375
+ );
376
+ }
377
+ return args.ctx;
378
+ });
379
+ this.sampleRate = args.sampleRate ?? SAMPLE_RATE_DEFAULT;
380
+ this.maxSecondsPerPhrase = args.maxSecondsPerPhrase ?? 6;
381
+ }
382
+
383
+ /** True when the loaded `libelizainference` advertises streaming TTS. */
384
+ supportsStreamingTts(): boolean {
385
+ return this.ffi.ttsStreamSupported();
386
+ }
387
+
388
+ /**
389
+ * One-shot synthesis returning the whole phrase as an `AudioChunk`.
390
+ * When the loaded build advertises streaming TTS this routes through
391
+ * `eliza_inference_tts_synthesize_stream` and concatenates the
392
+ * delivered chunks (so the chunk-aware native path is exercised even
393
+ * for whole-phrase callers); otherwise it uses the batch
394
+ * `eliza_inference_tts_synthesize` symbol. `cancelSignal` is honoured
395
+ * at chunk boundaries — a cancelled stream returns whatever was
396
+ * synthesized so far.
397
+ */
398
+ async synthesize(args: {
399
+ phrase: Phrase;
400
+ preset: SpeakerPreset;
401
+ cancelSignal: { cancelled: boolean };
402
+ onKernelTick?: () => void;
403
+ }): Promise<AudioChunk> {
404
+ args.onKernelTick?.();
405
+ const ctx = this.getContext();
406
+ if (this.ffi.ttsStreamSupported()) {
407
+ const parts: Float32Array[] = [];
408
+ let total = 0;
409
+ this.ffi.ttsSynthesizeStream({
410
+ ctx,
411
+ text: args.phrase.text,
412
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
413
+ onChunk: ({ pcm, isFinal }) => {
414
+ args.onKernelTick?.();
415
+ if (!isFinal && pcm.length > 0) {
416
+ parts.push(pcm);
417
+ total += pcm.length;
418
+ }
419
+ return args.cancelSignal.cancelled === true;
420
+ },
421
+ });
422
+ const merged = new Float32Array(total);
423
+ let off = 0;
424
+ for (const part of parts) {
425
+ merged.set(part, off);
426
+ off += part.length;
427
+ }
428
+ return {
429
+ phraseId: args.phrase.id,
430
+ fromIndex: args.phrase.fromIndex,
431
+ toIndex: args.phrase.toIndex,
432
+ pcm: merged,
433
+ sampleRate: this.sampleRate,
434
+ };
435
+ }
436
+ const out = new Float32Array(this.sampleRate * this.maxSecondsPerPhrase);
437
+ const samples = this.ffi.ttsSynthesize({
438
+ ctx,
439
+ text: args.phrase.text,
440
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
441
+ out,
442
+ });
443
+ return {
444
+ phraseId: args.phrase.id,
445
+ fromIndex: args.phrase.fromIndex,
446
+ toIndex: args.phrase.toIndex,
447
+ pcm: out.subarray(0, samples),
448
+ sampleRate: this.sampleRate,
449
+ };
450
+ }
451
+
452
+ /**
453
+ * Streaming synthesis: forwards to `eliza_inference_tts_synthesize_stream`
454
+ * when the build advertises a streaming decoder. When it does NOT
455
+ * (`tts_stream_supported() == 0`), this still satisfies the seam — but
456
+ * with exactly one body chunk + one final tail (the batch synthesis
457
+ * result), so the caller never mistakes a non-streaming build for a
458
+ * streaming one (no fallback sludge — the chunk count is the honest
459
+ * signal). The native side checks `ctx->tts_cancel` (set via
460
+ * `eliza_inference_cancel_tts`) on top of the `onChunk` return value.
461
+ * A non-streaming build cannot be interrupted while the native batch
462
+ * forward pass is inside `ttsSynthesize`; it only observes cancellation
463
+ * before emitting the body chunk. Barge-in-critical product paths should
464
+ * require `supportsStreamingTts()`.
465
+ */
466
+ async synthesizeStream(args: {
467
+ phrase: Phrase;
468
+ preset: SpeakerPreset;
469
+ cancelSignal: { cancelled: boolean };
470
+ onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
471
+ onKernelTick?: () => void;
472
+ }): Promise<{ cancelled: boolean }> {
473
+ const ctx = this.getContext();
474
+ if (this.ffi.ttsStreamSupported()) {
475
+ const { cancelled } = this.ffi.ttsSynthesizeStream({
476
+ ctx,
477
+ text: args.phrase.text,
478
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
479
+ onChunk: ({ pcm, isFinal }) => {
480
+ args.onKernelTick?.();
481
+ if (args.cancelSignal.cancelled) return true;
482
+ const want = args.onChunk({
483
+ pcm,
484
+ sampleRate: this.sampleRate,
485
+ isFinal,
486
+ });
487
+ // Re-read the (mutable) cancel flag — the chunk callback or a
488
+ // concurrent barge-in may have flipped it.
489
+ return want === true || args.cancelSignal.cancelled;
490
+ },
491
+ });
492
+ return { cancelled };
493
+ }
494
+ // Non-streaming build: one batch forward pass, surfaced as a single
495
+ // body chunk + final tail.
496
+ args.onKernelTick?.();
497
+ const out = new Float32Array(this.sampleRate * this.maxSecondsPerPhrase);
498
+ const samples = this.ffi.ttsSynthesize({
499
+ ctx,
500
+ text: args.phrase.text,
501
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
502
+ out,
503
+ });
504
+ let cancelled = args.cancelSignal.cancelled === true;
505
+ if (!cancelled && samples > 0) {
506
+ const want = args.onChunk({
507
+ pcm: out.subarray(0, samples),
508
+ sampleRate: this.sampleRate,
509
+ isFinal: false,
510
+ });
511
+ cancelled = want === true || args.cancelSignal.cancelled === true;
512
+ }
513
+ args.onChunk({
514
+ pcm: new Float32Array(0),
515
+ sampleRate: this.sampleRate,
516
+ isFinal: true,
517
+ });
518
+ return { cancelled };
519
+ }
520
+
521
+ /** Hard-cancel any in-flight TTS forward pass on this backend's context. */
522
+ cancelTts(): void {
523
+ this.ffi.cancelTts(this.getContext());
524
+ }
525
+
526
+ /**
527
+ * Batch transcription. One-shot callers should use the fused batch ABI
528
+ * directly so the native side receives the original sample-rate metadata
529
+ * and can apply its own audio preprocessing. Live mic streaming remains
530
+ * available through `EngineVoiceBridge.createStreamingTranscriber()`.
531
+ */
532
+ async transcribe(args: TranscriptionAudio): Promise<string> {
533
+ return this.ffi.asrTranscribe({
534
+ ctx: this.getContext(),
535
+ pcm: args.pcm,
536
+ sampleRateHz: args.sampleRate,
537
+ });
538
+ }
539
+ }
540
+
541
+ export interface EngineVoiceBridgeOptions {
542
+ /**
543
+ * Bundle root on disk. Must contain `cache/voice-preset-default.bin`
544
+ * and the FFI library (`lib/libelizainference.{dylib,so}`) when
545
+ * `useFfiBackend === true`.
546
+ */
547
+ bundleRoot: string;
548
+ /**
549
+ * When true, use `FfiOmniVoiceBackend`. When false, use the deterministic test backend
550
+ * only for lifecycle/unit tests; live sessions and direct synthesis reject
551
+ * the deterministic test backend before user-visible audio can be emitted.
552
+ */
553
+ useFfiBackend: boolean;
554
+ /** Override sample rate. Defaults to 24 kHz. */
555
+ sampleRate?: number;
556
+ /** Override ring buffer capacity (samples). Defaults to 4 s @ 24 kHz. */
557
+ ringBufferCapacity?: number;
558
+ /** Phrase chunker `maxTokensPerPhrase` (no-punctuation run-on cap). Defaults to
559
+ * `ELIZA_VOICE_MAX_TOKENS_PER_PHRASE` or 8 (one MTP draft round). */
560
+ maxTokensPerPhrase?: number;
561
+ /** Max concurrent TTS phrase dispatches. Defaults to env or scheduler default. */
562
+ maxInFlightPhrases?: number;
563
+ /**
564
+ * Pre-warmed phrase cache entries. Per AGENTS.md §4, a precomputed
565
+ * phrase cache for common assistant utterances is mandatory for the
566
+ * first-byte-latency win. Empty by default — callers wire actual
567
+ * entries from the bundle when available.
568
+ */
569
+ prewarmedPhrases?: ReadonlyArray<CachedPhraseAudio>;
570
+ /**
571
+ * Optional sink override (e.g. for tests or for routing PCM to a
572
+ * platform-specific audio device). Defaults to the in-memory sink the
573
+ * scheduler creates.
574
+ */
575
+ sink?: AudioSink;
576
+ /** Optional scheduler event listeners (rollback, audio, cancel). */
577
+ events?: SchedulerEvents;
578
+ /**
579
+ * Optional override for the TTS backend. When set, supersedes
580
+ * `useFfiBackend`. Tests use this to inject a controllable backend
581
+ * (e.g. one that holds synthesis open until a deferred resolves) so
582
+ * rollback timing can be observed deterministically.
583
+ */
584
+ backendOverride?: OmniVoiceBackend;
585
+ /**
586
+ * Override only the TTS backend while keeping the fused bundle lifecycle
587
+ * and ASR FFI loaded. Used when a bundle falls back from OmniVoice speech
588
+ * to Kokoro speech but still needs bundled Qwen3-ASR for mic input.
589
+ */
590
+ ttsBackendOverride?: OmniVoiceBackend;
591
+ /** Optional speaker preset paired with `ttsBackendOverride`. */
592
+ speakerPresetOverride?: SpeakerPreset;
593
+ /**
594
+ * Optional shared resource registry. When the bridge is created
595
+ * inside an engine that already owns one (text + voice on the same
596
+ * tokenizer / mmap regions), the engine passes its registry in so
597
+ * voice ref-counts against the same canonical resources. Tests can
598
+ * leave this unset to get a private registry.
599
+ */
600
+ sharedResources?: SharedResourceRegistry;
601
+ /**
602
+ * Optional lifecycle loaders override. Production wires real
603
+ * `madvise`-backed mmap handles via the FFI; tests inject mocks so
604
+ * the disarm path can assert eviction without a real file mapping.
605
+ * When unset, default loaders are derived from the bundle root.
606
+ */
607
+ lifecycleLoaders?: VoiceLifecycleLoaders;
608
+ /**
609
+ * Construct a `KokoroTtsBackend` directly and skip the bundle-root +
610
+ * speaker-preset + FFI checks the fused omnivoice path requires.
611
+ * Kokoro voices are picked by id (`KOKORO_VOICE_PACKS`), so the bundle's
612
+ * per-user speaker preset is not used. Mutually exclusive with
613
+ * `useFfiBackend: true` and `backendOverride`. Lifecycle loaders
614
+ * default to empty lifecycle handles (ORT owns the model memory; nothing to
615
+ * mmap-evict).
616
+ */
617
+ kokoroOnly?: KokoroEngineDiscoveryResult;
618
+ /**
619
+ * Optional pre-loaded fused inference handle for the `kokoroOnly` path. When
620
+ * set, the Kokoro FFI runtime reuses it instead of dlopen-ing a second copy
621
+ * of `libelizainference` (tests inject a stub; production may share the
622
+ * engine's handle).
623
+ */
624
+ kokoroFfi?: ElizaInferenceFfi;
625
+ /**
626
+ * Optional voice-profile store for speaker-attribution. When set, the
627
+ * bridge constructs a `VoiceAttributionPipeline` and runs attribution
628
+ * in parallel with ASR on every turn via `runVoiceTurn`. Callers receive
629
+ * the resolved `VoiceAttributionOutput` via `onAttribution` in the turn
630
+ * events passed to `runVoiceTurn`.
631
+ *
632
+ * When absent, attribution is skipped and the pipeline operates exactly
633
+ * as before (no diarizer / encoder overhead).
634
+ */
635
+ profileStore?: VoiceProfileStore;
636
+ /**
637
+ * W3-9 / F1 — the agent runtime. When supplied, the bridge constructs a
638
+ * `VoiceCancellationCoordinator` and an `OptimisticGenerationPolicy`
639
+ * scoped to this voice session. The coordinator owns one cancellation
640
+ * token per `roomId` and fans abort out to:
641
+ * 1. `runtime.turnControllers.abortTurn(roomId, reason)` — the
642
+ * planner-loop / action handlers / streaming `useModel` see the
643
+ * abort within one tick.
644
+ * 2. The slot-abort callback (`slotAbort`) when the LM slot id is
645
+ * registered with the turn.
646
+ * 3. The TTS hard-stop callback (`ttsStop`), which the bridge wires
647
+ * to its existing `triggerBargeIn()` (audio sink drain + FFI/HTTP
648
+ * synthesis cancel).
649
+ * 4. The standard `AbortSignal` every fetch / `useModel` / FFI call
650
+ * that took `token.signal` honours.
651
+ *
652
+ * The reverse direction (runtime → voice) is wired symmetrically via
653
+ * the coordinator's `runtime.turnControllers.onEvent` subscription.
654
+ *
655
+ * Omit to keep the prior behaviour — the bridge then exposes no
656
+ * coordinator / policy and callers fall back to the legacy
657
+ * `BargeInController` + `triggerBargeIn()` surface.
658
+ *
659
+ * Structural type — `CoordinatorRuntime` is the minimum surface the
660
+ * coordinator needs (`turnControllers.{abortTurn, onEvent}`). Production
661
+ * passes a full `IAgentRuntime`; tests can pass a fake matching the
662
+ * structural shape.
663
+ */
664
+ runtime?: IAgentRuntime | CoordinatorRuntime;
665
+ /**
666
+ * W3-9 / F1 — optional `OptimisticGenerationPolicy` overrides. When
667
+ * `runtime` is set and `optimisticPolicyOptions` is omitted, the bridge
668
+ * constructs a default policy gated on the resolved power source
669
+ * (plugged-in / battery / unknown) and the canonical EOT threshold.
670
+ */
671
+ optimisticPolicyOptions?: OptimisticPolicyOptions;
672
+ /**
673
+ * W3-9 / F1 — optional LM slot-abort callback for the cancellation
674
+ * coordinator. Production wires this to `MtpLlamaServer.abortSlot`
675
+ * once a slot id is known per turn. The bridge passes this directly
676
+ * into the coordinator; the bridge itself does not own slot ids.
677
+ *
678
+ * Has no effect when `runtime` is unset (no coordinator is constructed).
679
+ */
680
+ slotAbort?: (slotId: number, reason: VoiceCancellationReason) => void;
681
+ /**
682
+ * Live speaker-attribution gating. When set alongside a `profileStore` AND
683
+ * a full `runtime` (with `emitEvent`), `runVoiceTurn` automatically:
684
+ * 1. emits `VOICE_TURN_OBSERVED` for every attributed turn, and
685
+ * 2. folds the diarization decision into the turn's `voiceTurnSignal`
686
+ * (stamped onto `output.turn.metadata`) so the server gate
687
+ * `core.voice_turn_signal` can suppress confident-bystander cross-talk.
688
+ *
689
+ * `knownSpeakerEntityIds` / `ownerEntityId` may be functions so the caller
690
+ * can resolve the enrolled-speaker set lazily per turn (the household roster
691
+ * changes as people are named). When omitted, attribution still emits
692
+ * `VOICE_TURN_OBSERVED` and produces a fail-open signal (no bystander
693
+ * suppression — every attribution is treated as potentially addressed to us).
694
+ */
695
+ liveAttribution?: LiveAttributionConfig;
696
+ }
697
+
698
+ /** Gating inputs for the automatic live-attribution → voiceTurnSignal seam. */
699
+ export interface LiveAttributionConfig {
700
+ /** Owner / primary-enrolled entity id (always allowed to speak). */
701
+ ownerEntityId?: string | (() => string | null | undefined);
702
+ /** Entity ids the agent answers without a wake word (owner + enrolled). */
703
+ knownSpeakerEntityIds?:
704
+ | readonly string[]
705
+ | (() => readonly string[] | undefined);
706
+ /** True when a wake word fired within the recent listen window. */
707
+ wakeWordActive?: boolean | (() => boolean);
708
+ }
709
+
710
+ export function createKokoroTtsBackend(
711
+ kokoro: KokoroEngineDiscoveryResult,
712
+ opts: { bundleRoot?: string; ffi?: ElizaInferenceFfi } = {},
713
+ ): KokoroTtsBackend {
714
+ // In-process FFI is the sole Kokoro synthesis path on every platform — it
715
+ // runs inside the fused libelizainference handle, the only path that ships
716
+ // on iOS / Google Play (no local TCP socket). The legacy HTTP `fork`
717
+ // (llama-server /v1/audio/speech) runtime was removed. An already-loaded
718
+ // fused handle may be injected (`opts.ffi`) so Kokoro reuses it instead of
719
+ // dlopen-ing a second copy of the lib.
720
+ const decision = pickKokoroRuntimeBackend({
721
+ defaultBackend: "ffi",
722
+ ffi: {
723
+ layout: kokoro.layout,
724
+ bundleRoot: opts.bundleRoot,
725
+ ...(opts.ffi ? { ffi: opts.ffi } : {}),
726
+ },
727
+ });
728
+ logger.info(
729
+ `[voice/kokoro] runtime backend=${decision.backend} reason="${decision.reason}"`,
730
+ );
731
+ return new KokoroTtsBackend({
732
+ layout: kokoro.layout,
733
+ runtime: decision.runtime,
734
+ defaultVoiceId: kokoro.defaultVoiceId,
735
+ });
736
+ }
737
+
738
+ export function createKokoroSpeakerPreset(
739
+ kokoro: KokoroEngineDiscoveryResult,
740
+ ): SpeakerPreset {
741
+ return {
742
+ voiceId: kokoro.defaultVoiceId,
743
+ embedding: new Float32Array(0),
744
+ bytes: new Uint8Array(0),
745
+ };
746
+ }
747
+
748
+ /**
749
+ * Per-turn events that include the optional attribution result alongside
750
+ * the existing `VoicePipelineEvents`. The attribution runs in parallel
751
+ * with ASR; it resolves some time after `onAsrComplete` and before
752
+ * `onComplete`.
753
+ */
754
+ export interface VoiceTurnEvents extends VoicePipelineEvents {
755
+ /**
756
+ * Called once per turn when the `VoiceAttributionPipeline` resolves
757
+ * (diarizer + encoder + profile-store match). Only fired when the
758
+ * bridge was constructed with a `profileStore`. May arrive after
759
+ * `onAsrComplete` but before `onComplete`. Fire-and-forget from the
760
+ * bridge's perspective — callers attach the metadata to the turn's
761
+ * transcript asynchronously.
762
+ */
763
+ onAttribution?(output: VoiceAttributionOutput): void;
764
+ }
765
+
766
+ /**
767
+ * Internal helper: construct the W3-9 cancellation coordinator + the
768
+ * optimistic-generation policy for a session, given the bridge options.
769
+ * Returns null/null when no runtime was supplied (the bridge then operates
770
+ * without the W3-9 surface — back-compat for callers that haven't adopted
771
+ * the canonical cancellation token yet).
772
+ *
773
+ * Lives outside the class so both `start()` and `startKokoroOnly()` can
774
+ * share it without duplicating the construction order (the coordinator's
775
+ * `ttsStop` callback closes over the to-be-constructed bridge — we plumb
776
+ * that through `setTtsStop` after the bridge is built).
777
+ */
778
+ interface PendingCancellationWiring {
779
+ coordinator: VoiceCancellationCoordinator;
780
+ policy: OptimisticGenerationPolicy;
781
+ /** Wire the bridge's `triggerBargeIn` as the ttsStop callback. */
782
+ bindTtsStop(stop: () => void): void;
783
+ }
784
+
785
+ /**
786
+ * True when `runtime` is a full `IAgentRuntime` (exposes `emitEvent`) rather
787
+ * than the structural `CoordinatorRuntime` a test may pass. Only an
788
+ * event-capable runtime can drive the automatic `VOICE_TURN_OBSERVED` emit.
789
+ */
790
+ function isEventRuntime(
791
+ runtime: IAgentRuntime | CoordinatorRuntime | undefined,
792
+ ): runtime is IAgentRuntime {
793
+ return (
794
+ runtime !== undefined &&
795
+ typeof (runtime as { emitEvent?: unknown }).emitEvent === "function"
796
+ );
797
+ }
798
+
799
+ /**
800
+ * Flatten the (possibly lazy) `LiveAttributionConfig` into the plain options
801
+ * the runtime helper consumes. Resolved per turn so a changing household roster
802
+ * is picked up without re-arming voice.
803
+ */
804
+ function resolveLiveAttributionOptions(cfg: LiveAttributionConfig | null): {
805
+ ownerEntityId?: string | null;
806
+ knownSpeakerEntityIds?: readonly string[];
807
+ wakeWordActive?: boolean;
808
+ } {
809
+ if (!cfg) return {};
810
+ const ownerEntityId =
811
+ typeof cfg.ownerEntityId === "function"
812
+ ? cfg.ownerEntityId()
813
+ : cfg.ownerEntityId;
814
+ const knownSpeakerEntityIds =
815
+ typeof cfg.knownSpeakerEntityIds === "function"
816
+ ? cfg.knownSpeakerEntityIds()
817
+ : cfg.knownSpeakerEntityIds;
818
+ const wakeWordActive =
819
+ typeof cfg.wakeWordActive === "function"
820
+ ? cfg.wakeWordActive()
821
+ : cfg.wakeWordActive;
822
+ return {
823
+ ...(ownerEntityId !== undefined ? { ownerEntityId } : {}),
824
+ ...(knownSpeakerEntityIds !== undefined ? { knownSpeakerEntityIds } : {}),
825
+ ...(wakeWordActive !== undefined ? { wakeWordActive } : {}),
826
+ };
827
+ }
828
+
829
+ function buildCancellationWiring(
830
+ opts: EngineVoiceBridgeOptions,
831
+ ): PendingCancellationWiring | null {
832
+ if (!opts.runtime) return null;
833
+ let ttsStopHandler: (() => void) | null = null;
834
+ const coordinator = new VoiceCancellationCoordinator({
835
+ runtime: opts.runtime,
836
+ ...(opts.slotAbort ? { slotAbort: opts.slotAbort } : {}),
837
+ ttsStop: () => {
838
+ if (ttsStopHandler) {
839
+ ttsStopHandler();
840
+ }
841
+ },
842
+ });
843
+ const policy = new OptimisticGenerationPolicy(
844
+ opts.optimisticPolicyOptions ?? {},
845
+ );
846
+ policy.setPowerSource(resolvePowerSourceState());
847
+ return {
848
+ coordinator,
849
+ policy,
850
+ bindTtsStop(stop) {
851
+ ttsStopHandler = stop;
852
+ },
853
+ };
854
+ }
855
+
856
+ /**
857
+ * Wires the voice scaffold (`VoiceScheduler` + helpers) onto the engine.
858
+ * One bridge per active voice session — created in
859
+ * `LocalInferenceEngine.startVoice()` and disposed when the engine
860
+ * unloads or `stopVoice()` is called.
861
+ */
862
+ export class EngineVoiceBridge {
863
+ readonly scheduler: VoiceScheduler;
864
+ readonly backend: OmniVoiceBackend;
865
+ readonly lifecycle: VoiceLifecycle;
866
+ /** Loaded FFI handle when running against the fused build (else null). */
867
+ readonly ffi: ElizaInferenceFfi | null;
868
+ /** Lazily-created FFI context this bridge owns; destroyed in `dispose()`. */
869
+ private readonly ffiContextRef: FfiContextRef | null;
870
+ readonly asrAvailable: boolean;
871
+ private readonly bundleRoot: string;
872
+ /** The phrase cache the scheduler dispatches against — held so the bridge
873
+ * can answer "is phrase X cached" for the first-audio filler and seed the
874
+ * idle-time auto-prewarm. */
875
+ private readonly phraseCache: PhraseCache;
876
+ /** In-flight fused turn (`runVoiceTurn`), if any — cancelled on barge-in. */
877
+ private activePipeline: VoicePipeline | null = null;
878
+ /**
879
+ * Optional attribution pipeline. Populated when the bridge was created
880
+ * with a `profileStore` option. When present, `runVoiceTurn` fires
881
+ * attribution in parallel with ASR and delivers the result via
882
+ * `VoiceTurnEvents.onAttribution`.
883
+ */
884
+ private readonly attributionPipeline: VoiceAttributionPipeline | null;
885
+ /**
886
+ * Full agent runtime, retained only when `opts.runtime` supports
887
+ * `emitEvent` (i.e. it is a real `IAgentRuntime`, not the structural
888
+ * `CoordinatorRuntime` a test may pass). Used by the automatic
889
+ * live-attribution seam in `runVoiceTurn` to emit `VOICE_TURN_OBSERVED`.
890
+ * Null when no event-capable runtime was supplied.
891
+ */
892
+ private readonly eventRuntime: IAgentRuntime | null;
893
+ /** Gating inputs for the live-attribution → voiceTurnSignal seam. */
894
+ private readonly liveAttribution: LiveAttributionConfig | null;
895
+ /**
896
+ * W3-9 / F1 — voice cancellation coordinator. Populated when the bridge
897
+ * was created with a `runtime` option. Owns one
898
+ * `VoiceCancellationToken` per active `roomId` and fans abort out to
899
+ * the runtime turn controller, the LM slot, the TTS pipeline, and the
900
+ * standard `AbortSignal`. See `cancellation-coordinator.ts` for the
901
+ * full contract.
902
+ */
903
+ private readonly cancellationCoordinator: VoiceCancellationCoordinator | null;
904
+ /**
905
+ * W3-9 / F1 — optimistic-generation policy. Constructed once per
906
+ * session when `runtime` is supplied. Gates the speculative LM prefill
907
+ * at the `firePrefill` site (see `voice-state-machine.ts`). Hot-swappable
908
+ * via `setPowerSource()` / `setOverride()` from Settings or a device-
909
+ * event listener.
910
+ */
911
+ private readonly optimisticGenerationPolicy: OptimisticGenerationPolicy | null;
912
+ /**
913
+ * W3-9 / F1 — per-room `BargeInController` bindings the bridge owns.
914
+ * Holds the unsubscribe handle returned by
915
+ * `coordinator.bindBargeInController` so `dispose()` can tear them down.
916
+ */
917
+ private readonly bargeInBindings = new Map<string, () => void>();
918
+
919
+ private constructor(
920
+ scheduler: VoiceScheduler,
921
+ backend: OmniVoiceBackend,
922
+ bundleRoot: string,
923
+ lifecycle: VoiceLifecycle,
924
+ ffi: ElizaInferenceFfi | null,
925
+ ffiContextRef: FfiContextRef | null,
926
+ asrAvailable: boolean,
927
+ phraseCache: PhraseCache,
928
+ attributionPipeline: VoiceAttributionPipeline | null = null,
929
+ cancellationCoordinator: VoiceCancellationCoordinator | null = null,
930
+ optimisticGenerationPolicy: OptimisticGenerationPolicy | null = null,
931
+ eventRuntime: IAgentRuntime | null = null,
932
+ liveAttribution: LiveAttributionConfig | null = null,
933
+ ) {
934
+ this.scheduler = scheduler;
935
+ this.backend = backend;
936
+ this.bundleRoot = bundleRoot;
937
+ this.lifecycle = lifecycle;
938
+ this.ffi = ffi;
939
+ this.ffiContextRef = ffiContextRef;
940
+ this.asrAvailable = asrAvailable;
941
+ this.phraseCache = phraseCache;
942
+ this.attributionPipeline = attributionPipeline;
943
+ this.cancellationCoordinator = cancellationCoordinator;
944
+ this.optimisticGenerationPolicy = optimisticGenerationPolicy;
945
+ this.eventRuntime = eventRuntime;
946
+ this.liveAttribution = liveAttribution;
947
+ }
948
+
949
+ get ffiCtx(): ElizaInferenceContextHandle | null {
950
+ return this.ffiContextRef?.current ?? null;
951
+ }
952
+
953
+ /**
954
+ * Tear down the FFI context the bridge owns. Idempotent; safe to call
955
+ * multiple times. Callers should `disarm()` first to drop voice
956
+ * resources, then `dispose()` to close the FFI handle.
957
+ */
958
+ dispose(): void {
959
+ // W3-9 / F1 — tear down barge-in bindings + the cancellation
960
+ // coordinator first so any armed turn aborts with reason=external
961
+ // before the FFI context goes away.
962
+ for (const unsub of Array.from(this.bargeInBindings.values())) {
963
+ try {
964
+ unsub();
965
+ } catch {
966
+ // Best-effort teardown.
967
+ }
968
+ }
969
+ this.bargeInBindings.clear();
970
+ if (this.cancellationCoordinator) {
971
+ try {
972
+ this.cancellationCoordinator.dispose();
973
+ } catch {
974
+ // Coordinator dispose must not block FFI teardown.
975
+ }
976
+ }
977
+ if (this.ffi) {
978
+ const ctx = this.ffiContextRef?.current ?? null;
979
+ if (ctx !== null) {
980
+ this.ffi.destroy(ctx);
981
+ if (this.ffiContextRef) this.ffiContextRef.current = null;
982
+ }
983
+ this.ffi.close();
984
+ }
985
+ }
986
+
987
+ /**
988
+ * Start the voice session for a bundle. Validates the bundle layout
989
+ * up-front (per AGENTS.md §3 + §7 — required artifacts checked before
990
+ * activation) and throws `VoiceStartupError` for any missing piece.
991
+ * No partial activation: either the scheduler exists and is wired or
992
+ * the call throws.
993
+ */
994
+ static start(opts: EngineVoiceBridgeOptions): EngineVoiceBridge {
995
+ if (opts.kokoroOnly) {
996
+ if (opts.useFfiBackend || opts.backendOverride) {
997
+ throw new VoiceStartupError(
998
+ "invalid-options",
999
+ "[voice] kokoroOnly cannot be combined with useFfiBackend or backendOverride. Caller must pick exactly one backend path.",
1000
+ );
1001
+ }
1002
+ return EngineVoiceBridge.startKokoroOnly(opts);
1003
+ }
1004
+ if (!opts.bundleRoot || !existsSync(opts.bundleRoot)) {
1005
+ throw new VoiceStartupError(
1006
+ "missing-bundle-root",
1007
+ `[voice] Bundle root does not exist: ${opts.bundleRoot}`,
1008
+ );
1009
+ }
1010
+
1011
+ const presetPath = path.join(
1012
+ opts.bundleRoot,
1013
+ DEFAULT_VOICE_PRESET_REL_PATH,
1014
+ );
1015
+ if (!existsSync(presetPath)) {
1016
+ throw new VoiceStartupError(
1017
+ "missing-speaker-preset",
1018
+ `[voice] Bundle is missing required speaker preset at ${presetPath}. The default voice MUST ship as a precomputed embedding (AGENTS.md §4).`,
1019
+ );
1020
+ }
1021
+
1022
+ const sampleRate = opts.sampleRate ?? SAMPLE_RATE_DEFAULT;
1023
+ const presetCache = new SpeakerPresetCache();
1024
+ const { preset, phrases: seedPhrases } = presetCache.loadFromBundle({
1025
+ bundleRoot: opts.bundleRoot,
1026
+ });
1027
+ const schedulerPreset = opts.speakerPresetOverride ?? preset;
1028
+
1029
+ const phraseCache = new PhraseCache();
1030
+ phraseCache.seed(seedPhrases);
1031
+ for (const entry of opts.prewarmedPhrases ?? []) {
1032
+ phraseCache.put(entry);
1033
+ }
1034
+
1035
+ // FFI binding + per-bridge context. When the bridge runs against
1036
+ // the real fused build, the same `ffi`/`ctx` pair is shared by:
1037
+ // - the TTS backend (`FfiOmniVoiceBackend.synthesize`),
1038
+ // - the lifecycle loaders (`MmapRegionHandle.evictPages` calls
1039
+ // `ffi.mmapEvict(ctx, "tts" | "asr")`).
1040
+ // Tests can opt out by either passing `lifecycleLoaders` (mocks
1041
+ // `evictPages`) or `backendOverride` (mocks the backend) or
1042
+ // setting `useFfiBackend: false` (test TTS + empty evict transition).
1043
+ let ffiHandle: ElizaInferenceFfi | null = null;
1044
+ let ffiContextRef: FfiContextRef | null = null;
1045
+ let backend: OmniVoiceBackend;
1046
+ const asrAvailable = bundleHasRegularFile(
1047
+ path.join(opts.bundleRoot, "asr"),
1048
+ );
1049
+ if (opts.backendOverride && opts.ttsBackendOverride) {
1050
+ throw new VoiceStartupError(
1051
+ "invalid-options",
1052
+ "[voice] backendOverride and ttsBackendOverride are mutually exclusive.",
1053
+ );
1054
+ }
1055
+ if (opts.backendOverride && opts.useFfiBackend) {
1056
+ throw new VoiceStartupError(
1057
+ "missing-fused-build",
1058
+ "[voice] backendOverride cannot be combined with useFfiBackend=true. Voice-on production paths must load libelizainference and verify its ABI instead of bypassing the fused runtime.",
1059
+ );
1060
+ }
1061
+ if (opts.backendOverride) {
1062
+ backend = opts.backendOverride;
1063
+ } else if (opts.useFfiBackend) {
1064
+ const libPath = locateBundleLibrary(opts.bundleRoot);
1065
+ if (!existsSync(libPath)) {
1066
+ throw new VoiceStartupError(
1067
+ "missing-ffi",
1068
+ `[voice] Fused omnivoice library not found under ${path.join(opts.bundleRoot, "lib")} (tried ${libraryFilenames().join(", ")}). Build via packages/app-core/scripts/build-llama-cpp-mtp.mjs (omnivoice-fuse target).`,
1069
+ );
1070
+ }
1071
+ ffiHandle = loadElizaInferenceFfi(libPath);
1072
+ const contextRef: FfiContextRef = {
1073
+ current: null,
1074
+ ensure: () => {
1075
+ if (!ffiHandle) {
1076
+ throw new VoiceStartupError(
1077
+ "missing-ffi",
1078
+ "[voice] FFI context requested without a loaded libelizainference handle",
1079
+ );
1080
+ }
1081
+ if (contextRef.current === null) {
1082
+ contextRef.current = ffiHandle.create(opts.bundleRoot);
1083
+ }
1084
+ return contextRef.current;
1085
+ },
1086
+ };
1087
+ ffiContextRef = contextRef;
1088
+ backend =
1089
+ opts.ttsBackendOverride ??
1090
+ new FfiOmniVoiceBackend({
1091
+ ffi: ffiHandle,
1092
+ getContext: contextRef.ensure,
1093
+ sampleRate,
1094
+ });
1095
+ } else {
1096
+ backend = opts.ttsBackendOverride ?? new StubOmniVoiceBackend(sampleRate);
1097
+ }
1098
+
1099
+ const config: SchedulerConfig = {
1100
+ chunkerConfig: {
1101
+ maxTokensPerPhrase:
1102
+ opts.maxTokensPerPhrase ??
1103
+ readPositiveIntEnv("ELIZA_VOICE_MAX_TOKENS_PER_PHRASE") ??
1104
+ PHRASE_MAX_TOKENS_DEFAULT,
1105
+ },
1106
+ preset: schedulerPreset,
1107
+ ringBufferCapacity:
1108
+ opts.ringBufferCapacity ?? RING_BUFFER_CAPACITY_DEFAULT,
1109
+ sampleRate,
1110
+ maxInFlightPhrases:
1111
+ opts.maxInFlightPhrases ??
1112
+ readPositiveIntEnv("ELIZA_VOICE_MAX_IN_FLIGHT_PHRASES"),
1113
+ };
1114
+
1115
+ const sinkOverride = opts.sink;
1116
+ const scheduler = new VoiceScheduler(
1117
+ config,
1118
+ sinkOverride
1119
+ ? { backend, sink: sinkOverride, phraseCache }
1120
+ : { backend, phraseCache },
1121
+ opts.events ?? {},
1122
+ );
1123
+
1124
+ // Wire the voice lifecycle. The lifecycle starts in `voice-off` —
1125
+ // heavy resources (TTS + ASR mmap regions) are loaded only when
1126
+ // `arm()` is called. The default loaders derive an mmap-style
1127
+ // handle from the bundle's `tts/` and `asr/` directories so that
1128
+ // production paths get real eviction calls; tests inject
1129
+ // `lifecycleLoaders` to assert the disarm path.
1130
+ const registry = opts.sharedResources ?? new SharedResourceRegistry();
1131
+ const loaders =
1132
+ opts.lifecycleLoaders ??
1133
+ defaultLifecycleLoaders(opts.bundleRoot, ffiHandle, ffiContextRef);
1134
+ const lifecycle = new VoiceLifecycle({ registry, loaders });
1135
+
1136
+ // Wire speaker-attribution when a profile store is provided. The
1137
+ // attribution pipeline wraps the fused encoder + diarizer + profile-store.
1138
+ // Both run through the ONE fused `libelizainference` handle via its
1139
+ // `eliza_inference_speaker_*` / `_diariz_*` ABI — there is no standalone
1140
+ // `libvoice_classifier` runtime.
1141
+ //
1142
+ // Fail-fast at ARM time: the fused speaker ABI is probed synchronously
1143
+ // here (`FusedSpeakerEncoder.isSupported`). When the build does not
1144
+ // advertise it, this throws `VoiceStartupError` rather than silently
1145
+ // degrading attribution to "unknown speaker" on the first turn. The
1146
+ // native session `load()` runs lazily on first encode/diarize, but the
1147
+ // capability is decided up front.
1148
+ let attributionPipeline: VoiceAttributionPipeline | null = null;
1149
+ if (opts.profileStore) {
1150
+ const fusedFfi = ffiHandle;
1151
+ const fusedCtx = ffiContextRef;
1152
+ if (!fusedFfi || !fusedCtx) {
1153
+ throw new VoiceStartupError(
1154
+ "missing-fused-build",
1155
+ "[voice] Speaker-attribution requires the fused libelizainference handle (useFfiBackend). No standalone speaker runtime exists.",
1156
+ );
1157
+ }
1158
+ if (!FusedSpeakerEncoder.isSupported(fusedFfi)) {
1159
+ throw new VoiceStartupError(
1160
+ "missing-fused-build",
1161
+ "[voice] The loaded libelizainference build lacks the speaker ABI (eliza_inference_speaker_supported() == 0). Rebuild with the WeSpeaker forward graph linked in (eliza_inference_speaker_* symbols).",
1162
+ );
1163
+ }
1164
+ // Fused encoder: probe passed above; the native session opens lazily
1165
+ // on first encode() so voice-off does not keep the model resident.
1166
+ let resolvedEncoder: SpeakerEncoder | null = null;
1167
+ let encoderLoadError: Error | null = null;
1168
+ const lazyEncoder: SpeakerEncoder = {
1169
+ embeddingDim: SPEAKER_GGML_EMBEDDING_DIM,
1170
+ sampleRate: SPEAKER_GGML_SAMPLE_RATE,
1171
+ async encode(pcm: Float32Array): Promise<Float32Array> {
1172
+ if (encoderLoadError) throw encoderLoadError;
1173
+ if (!resolvedEncoder) {
1174
+ try {
1175
+ resolvedEncoder = await FusedSpeakerEncoder.load({
1176
+ ffi: fusedFfi,
1177
+ ctx: () => fusedCtx.ensure(),
1178
+ });
1179
+ } catch (err) {
1180
+ encoderLoadError =
1181
+ err instanceof Error ? err : new Error(String(err));
1182
+ throw encoderLoadError;
1183
+ }
1184
+ }
1185
+ return resolvedEncoder.encode(pcm);
1186
+ },
1187
+ async dispose(): Promise<void> {
1188
+ await resolvedEncoder?.dispose();
1189
+ },
1190
+ };
1191
+ // Fused diarizer (optional). When the build does not advertise the
1192
+ // diarizer ABI, attribution runs without it — a single-speaker turn
1193
+ // collapses to one segment (the attribution-pipeline localSpeakerId=0
1194
+ // path). The diarizer is NOT a fail-fast gate (unlike the encoder):
1195
+ // it refines multi-speaker windows, it is not required to attribute a
1196
+ // single speaker.
1197
+ let lazyDiarizer: Diarizer | undefined;
1198
+ if (FusedDiarizer.isSupported(fusedFfi)) {
1199
+ let resolvedDiarizer: Diarizer | null = null;
1200
+ let diarizerLoadError: Error | null = null;
1201
+ lazyDiarizer = {
1202
+ modelId: PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID,
1203
+ sampleRate: SPEAKER_GGML_SAMPLE_RATE,
1204
+ async diarizeWindow(pcm: Float32Array) {
1205
+ if (diarizerLoadError) throw diarizerLoadError;
1206
+ if (!resolvedDiarizer) {
1207
+ try {
1208
+ resolvedDiarizer = await FusedDiarizer.load({
1209
+ ffi: fusedFfi,
1210
+ ctx: () => fusedCtx.ensure(),
1211
+ });
1212
+ } catch (err) {
1213
+ diarizerLoadError =
1214
+ err instanceof Error ? err : new Error(String(err));
1215
+ throw diarizerLoadError;
1216
+ }
1217
+ }
1218
+ return resolvedDiarizer.diarizeWindow(pcm);
1219
+ },
1220
+ async dispose(): Promise<void> {
1221
+ await resolvedDiarizer?.dispose();
1222
+ },
1223
+ };
1224
+ }
1225
+ attributionPipeline = new VoiceAttributionPipeline({
1226
+ encoder: lazyEncoder,
1227
+ ...(lazyDiarizer ? { diarizer: lazyDiarizer } : {}),
1228
+ profileStore: opts.profileStore,
1229
+ });
1230
+ }
1231
+
1232
+ // W3-9 / F1 — construct the cancellation coordinator + optimistic policy
1233
+ // when a runtime is supplied. The coordinator's ttsStop callback closes
1234
+ // over `bridge.triggerBargeIn()`, which is wired below once the bridge
1235
+ // is constructed.
1236
+ const wiring = buildCancellationWiring(opts);
1237
+
1238
+ const bridge = new EngineVoiceBridge(
1239
+ scheduler,
1240
+ backend,
1241
+ opts.bundleRoot,
1242
+ lifecycle,
1243
+ ffiHandle,
1244
+ ffiContextRef,
1245
+ asrAvailable,
1246
+ phraseCache,
1247
+ attributionPipeline,
1248
+ wiring?.coordinator ?? null,
1249
+ wiring?.policy ?? null,
1250
+ isEventRuntime(opts.runtime) ? opts.runtime : null,
1251
+ opts.liveAttribution ?? null,
1252
+ );
1253
+ if (wiring) wiring.bindTtsStop(() => bridge.triggerBargeIn());
1254
+ return bridge;
1255
+ }
1256
+
1257
+ /**
1258
+ * Kokoro-only path. Skips bundle-root / speaker-preset / FFI checks
1259
+ * (Kokoro picks voices by id against `KOKORO_VOICE_PACKS`) and
1260
+ * synthesizes a minimal `SpeakerPreset` keyed to the discovered voice
1261
+ * id. Defaults lifecycle loaders to empty handles since ORT owns the
1262
+ * model memory. `asrAvailable` is `false`: callers needing ASR
1263
+ * construct `createStreamingTranscriber` directly.
1264
+ */
1265
+ private static startKokoroOnly(
1266
+ opts: EngineVoiceBridgeOptions,
1267
+ ): EngineVoiceBridge {
1268
+ if (!opts.kokoroOnly) {
1269
+ throw new VoiceStartupError(
1270
+ "invalid-options",
1271
+ "[voice] startKokoroOnly called without `kokoroOnly` config — this is an internal error.",
1272
+ );
1273
+ }
1274
+ const kokoro = opts.kokoroOnly;
1275
+ const sampleRate = opts.sampleRate ?? kokoro.layout.sampleRate;
1276
+ const workDir =
1277
+ opts.bundleRoot && existsSync(opts.bundleRoot)
1278
+ ? opts.bundleRoot
1279
+ : localInferenceRoot();
1280
+
1281
+ // Synthesize a minimal preset. Kokoro's `resolveVoice(preset)` looks
1282
+ // up `preset.voiceId` against `KOKORO_VOICE_PACKS`; the embedding +
1283
+ // bytes fields are ignored on this path (voice cloning is OmniVoice-only).
1284
+ const preset = createKokoroSpeakerPreset(kokoro);
1285
+
1286
+ // Anchor the in-process Kokoro FFI ctx at the Eliza-1 bundle root when
1287
+ // one is present; otherwise the runtime anchors at the Kokoro model root.
1288
+ const backend = createKokoroTtsBackend(kokoro, {
1289
+ bundleRoot:
1290
+ opts.bundleRoot && existsSync(opts.bundleRoot)
1291
+ ? opts.bundleRoot
1292
+ : undefined,
1293
+ ...(opts.kokoroFfi ? { ffi: opts.kokoroFfi } : {}),
1294
+ });
1295
+
1296
+ const phraseCache = new PhraseCache();
1297
+ for (const entry of opts.prewarmedPhrases ?? []) {
1298
+ phraseCache.put(entry);
1299
+ }
1300
+
1301
+ const config: SchedulerConfig = {
1302
+ chunkerConfig: {
1303
+ maxTokensPerPhrase:
1304
+ opts.maxTokensPerPhrase ??
1305
+ readPositiveIntEnv("ELIZA_VOICE_MAX_TOKENS_PER_PHRASE") ??
1306
+ PHRASE_MAX_TOKENS_DEFAULT,
1307
+ },
1308
+ preset,
1309
+ ringBufferCapacity:
1310
+ opts.ringBufferCapacity ?? RING_BUFFER_CAPACITY_DEFAULT,
1311
+ sampleRate,
1312
+ maxInFlightPhrases:
1313
+ opts.maxInFlightPhrases ??
1314
+ readPositiveIntEnv("ELIZA_VOICE_MAX_IN_FLIGHT_PHRASES"),
1315
+ };
1316
+
1317
+ const sinkOverride = opts.sink;
1318
+ const scheduler = new VoiceScheduler(
1319
+ config,
1320
+ sinkOverride
1321
+ ? { backend, sink: sinkOverride, phraseCache }
1322
+ : { backend, phraseCache },
1323
+ opts.events ?? {},
1324
+ );
1325
+
1326
+ const registry = opts.sharedResources ?? new SharedResourceRegistry();
1327
+ const loaders = opts.lifecycleLoaders ?? kokoroOnlyLifecycleLoaders();
1328
+ const lifecycle = new VoiceLifecycle({ registry, loaders });
1329
+
1330
+ const wiring = buildCancellationWiring(opts);
1331
+
1332
+ const bridge = new EngineVoiceBridge(
1333
+ scheduler,
1334
+ backend,
1335
+ workDir,
1336
+ lifecycle,
1337
+ null, // no FFI handle on Kokoro-only
1338
+ null, // no FFI context on Kokoro-only
1339
+ false, // ASR is not served from this path
1340
+ phraseCache,
1341
+ null, // no profile store on Kokoro-only
1342
+ wiring?.coordinator ?? null,
1343
+ wiring?.policy ?? null,
1344
+ );
1345
+ if (wiring) wiring.bindTtsStop(() => bridge.triggerBargeIn());
1346
+ return bridge;
1347
+ }
1348
+
1349
+ /**
1350
+ * True when this bridge runs against a TTS backend that produces real
1351
+ * audio — i.e. anything but the `StubOmniVoiceBackend` (which yields
1352
+ * zeros and is tests-only). The prewarm + first-audio-filler paths gate
1353
+ * on this so the cache never holds silence (AGENTS.md §3 — no fake data).
1354
+ */
1355
+ hasRealTtsBackend(): boolean {
1356
+ return !(this.backend instanceof StubOmniVoiceBackend);
1357
+ }
1358
+
1359
+ /**
1360
+ * Lazy-load the TTS mmap region, optional ASR region, and the voice
1361
+ * scheduler nodes via the lifecycle state machine. Idempotent for
1362
+ * repeated calls in `voice-on` (returns the existing armed resources).
1363
+ * Surfaces RAM pressure / mmap-fail / kernel-missing as `VoiceLifecycleError` —
1364
+ * see `lifecycle.ts` for the full error taxonomy.
1365
+ */
1366
+ async arm(): Promise<void> {
1367
+ if (this.lifecycle.current().kind === "voice-on") return;
1368
+ await this.lifecycle.arm();
1369
+ }
1370
+
1371
+ /**
1372
+ * Drain in-flight TTS, settle the scheduler, then disarm the
1373
+ * lifecycle. Disarm calls `evictPages()` (madvise / VirtualUnlock
1374
+ * equivalent) on the TTS + optional ASR mmap regions and releases every
1375
+ * voice-only ref. Speaker preset + phrase cache survive in the
1376
+ * registry as small LRU entries (KB-scale; not worth evicting).
1377
+ */
1378
+ async disarm(): Promise<void> {
1379
+ if (this.lifecycle.current().kind !== "voice-on") return;
1380
+ await this.settle();
1381
+ await this.lifecycle.disarm();
1382
+ }
1383
+
1384
+ /**
1385
+ * Forward an accepted text token from the verifier into the scheduler.
1386
+ * Tokens that fill a phrase trigger TTS dispatch on the same scheduler
1387
+ * tick (AGENTS.md §4 — no buffering past phrase boundaries).
1388
+ */
1389
+ async pushAcceptedToken(
1390
+ token: TextToken,
1391
+ acceptedAt = Date.now(),
1392
+ ): Promise<void> {
1393
+ await this.scheduler.accept(token, acceptedAt);
1394
+ }
1395
+
1396
+ /**
1397
+ * MTP rejection → rollback queue. The scheduler cancels any
1398
+ * in-flight TTS forward pass for phrases that overlap the rejected
1399
+ * token range and emits an `onRollback` event for observability.
1400
+ * Already-played audio cannot be unplayed; the chunker is sized so
1401
+ * rollback is rare and cheap.
1402
+ */
1403
+ async pushRejectedRange(range: RejectedTokenRange): Promise<void> {
1404
+ await this.scheduler.reject(range);
1405
+ }
1406
+
1407
+ /**
1408
+ * Voice activity detected on the mic input → cancel everything.
1409
+ * Drains the ring buffer immediately, flushes the chunker queue, and
1410
+ * marks every in-flight cancel signal so synthesise loops exit at the
1411
+ * next kernel boundary (AGENTS.md §4 — barge-in cancellation MUST be
1412
+ * within one kernel tick).
1413
+ */
1414
+ triggerBargeIn(): void {
1415
+ // Cancel the text side first (stop ASR / drafter / verifier at the next
1416
+ // kernel boundary), then the audio side (ring-buffer drain + chunker
1417
+ // flush + in-flight TTS cancel). The pipeline also wires its own
1418
+ // barge-in listener onto the scheduler, so `onMicActive()` alone would
1419
+ // suffice — calling `cancel()` first just stops the next HTTP body
1420
+ // sooner.
1421
+ this.activePipeline?.cancel();
1422
+ this.scheduler.bargeIn.onMicActive();
1423
+ }
1424
+
1425
+ /**
1426
+ * W3-9 / F1 — the canonical voice cancellation coordinator for this
1427
+ * session, or `null` when the bridge was constructed without a
1428
+ * `runtime` option. Callers (turn controller, mic VAD source, UI cancel
1429
+ * route) use this to arm per-turn tokens, fire `bargeIn(roomId)` on
1430
+ * VAD speech-start, fire `revokeEot(roomId)` when the turn detector
1431
+ * revokes a tentative EOT, etc. See
1432
+ * `plugins/plugin-local-inference/docs/voice-cancellation-contract.md`.
1433
+ */
1434
+ cancellationCoordinatorOrNull(): VoiceCancellationCoordinator | null {
1435
+ return this.cancellationCoordinator;
1436
+ }
1437
+
1438
+ /**
1439
+ * W3-9 / F1 — the optimistic-generation policy for this session, or
1440
+ * `null` when the bridge was constructed without a `runtime` option.
1441
+ * The bridge primes it with the resolved power source at construction
1442
+ * time; callers can mutate it via `setPowerSource()` / `setOverride()`
1443
+ * to respond to Settings toggles or battery-state events.
1444
+ */
1445
+ optimisticPolicyOrNull(): OptimisticGenerationPolicy | null {
1446
+ return this.optimisticGenerationPolicy;
1447
+ }
1448
+
1449
+ /**
1450
+ * W3-9 / F1 — bind the scheduler's `BargeInController` into the
1451
+ * cancellation coordinator for `roomId`. Subsequent
1452
+ * `BargeInController.hardStop()` calls (typically fired by the
1453
+ * ASR-confirmed barge-in words ladder) translate into
1454
+ * `coordinator.bargeIn(roomId)` so the canonical token (and every
1455
+ * downstream consumer: runtime turn abort, LM slot abort, TTS stop,
1456
+ * AbortSignal) sees the abort.
1457
+ *
1458
+ * Idempotent per `roomId` — repeated calls for the same room return
1459
+ * the same unsubscribe handle (the prior binding is torn down first).
1460
+ *
1461
+ * When the bridge was constructed without a `runtime` option, this returns
1462
+ * an empty unsubscribe. Callers should still call it
1463
+ * unconditionally — back-compat for the legacy path is automatic.
1464
+ */
1465
+ bindBargeInControllerForRoom(roomId: string): () => void {
1466
+ if (!this.cancellationCoordinator) {
1467
+ return () => undefined;
1468
+ }
1469
+ const existing = this.bargeInBindings.get(roomId);
1470
+ if (existing) existing();
1471
+ const unsub = this.cancellationCoordinator.bindBargeInController(
1472
+ roomId,
1473
+ this.scheduler.bargeIn,
1474
+ );
1475
+ this.bargeInBindings.set(roomId, unsub);
1476
+ return () => {
1477
+ unsub();
1478
+ if (this.bargeInBindings.get(roomId) === unsub) {
1479
+ this.bargeInBindings.delete(roomId);
1480
+ }
1481
+ };
1482
+ }
1483
+
1484
+ /**
1485
+ * Drain pending phrase data and wait for in-flight TTS to settle.
1486
+ * Used at the end of a turn so callers can synchronise on a quiescent
1487
+ * scheduler before they tear it down.
1488
+ */
1489
+ async settle(): Promise<void> {
1490
+ await this.scheduler.flushPending();
1491
+ await this.scheduler.waitIdle();
1492
+ }
1493
+
1494
+ async synthesizeTextToWav(
1495
+ text: string,
1496
+ signal?: AbortSignal,
1497
+ ): Promise<Uint8Array> {
1498
+ this.assertVoiceOn("synthesize speech");
1499
+ if (!this.hasRealTtsBackend()) {
1500
+ throw new VoiceStartupError(
1501
+ "missing-fused-build",
1502
+ "[voice] Direct speech synthesis requires a fused OmniVoice backend. The deterministic test backend is only allowed in scheduler/unit tests.",
1503
+ );
1504
+ }
1505
+ const chunk = await this.scheduler.synthesizeText(text, signal);
1506
+ return encodeMonoPcm16Wav(chunk.pcm, chunk.sampleRate);
1507
+ }
1508
+
1509
+ /**
1510
+ * The streaming-TTS seam W9's scheduler drives: returns the active
1511
+ * backend as a `StreamingTtsBackend` (`FfiOmniVoiceBackend` against the
1512
+ * fused build, `StubOmniVoiceBackend` for tests). The scheduler calls
1513
+ * `synthesizeStream(...)` for each phrase and writes the delivered PCM
1514
+ * segments into its `PcmRingBuffer` on the same scheduler tick. Returns
1515
+ * null when an injected `backendOverride` does not implement the seam.
1516
+ */
1517
+ streamingTtsBackend(): StreamingTtsBackend | null {
1518
+ return isStreamingTtsBackend(this.backend) ? this.backend : null;
1519
+ }
1520
+
1521
+ /**
1522
+ * True when the loaded fused `libelizainference` runs the MTP
1523
+ * speculative loop in-process and can emit native accept/reject
1524
+ * verifier events. When true, callers (W9's turn controller /
1525
+ * `ffi-streaming-backend.ts` wiring) should subscribe via
1526
+ * `subscribeNativeVerifier()` and SKIP the `llama-server` SSE
1527
+ * `{"verifier":{"rejected":[a,b]}}` side-channel — the SSE path stays
1528
+ * only as the non-fused desktop text fallback. False whenever there is
1529
+ * no FFI handle or the build pre-dates the verifier callback.
1530
+ */
1531
+ hasNativeVerifier(): boolean {
1532
+ // ABI v3 exports `eliza_inference_set_verifier_callback`, but the
1533
+ // current generated adapter returns ELIZA_ERR_NOT_IMPLEMENTED until the
1534
+ // native MTP speculative loop is ported into libelizainference. Do
1535
+ // not let callers skip the SSE verifier fallback merely because the
1536
+ // symbol exists.
1537
+ return false;
1538
+ }
1539
+
1540
+ /**
1541
+ * Register the native MTP verifier callback on the fused runtime
1542
+ * and adapt each `NativeVerifierEvent` into the rollback-queue domain:
1543
+ * accepted/corrected token-id ranges become `VerifierStreamEvent`s and
1544
+ * rejected ranges become `RejectedTokenRange`s fed to `pushRejectedRange`.
1545
+ * The returned handle MUST be `close()`d (clears the native callback +
1546
+ * frees the bun:ffi `JSCallback`). Throws if no fused runtime is loaded.
1547
+ *
1548
+ * `onEvent` (optional) also receives the raw `NativeVerifierEvent` for
1549
+ * callers that want the accepted-token stream (W9's phrase-chunker can
1550
+ * commit accepted draft tokens directly off this instead of round-trip
1551
+ * SSE deltas).
1552
+ */
1553
+ subscribeNativeVerifier(onEvent?: (event: NativeVerifierEvent) => void): {
1554
+ close(): void;
1555
+ } {
1556
+ if (!this.ffi) {
1557
+ throw new VoiceStartupError(
1558
+ "missing-ffi",
1559
+ "[voice] subscribeNativeVerifier requires a loaded fused libelizainference handle",
1560
+ );
1561
+ }
1562
+ const ctx = this.ffiContextRef
1563
+ ? this.ffiContextRef.ensure()
1564
+ : (() => {
1565
+ throw new VoiceStartupError(
1566
+ "missing-ffi",
1567
+ "[voice] subscribeNativeVerifier: no FFI context provider",
1568
+ );
1569
+ })();
1570
+ return this.ffi.setVerifierCallback(ctx, (event) => {
1571
+ onEvent?.(event);
1572
+ const rollback = nativeRejectedRangeToRollbackRange(event);
1573
+ if (rollback) {
1574
+ void this.pushRejectedRange(rollback);
1575
+ }
1576
+ });
1577
+ }
1578
+
1579
+ async prewarmPhrases(
1580
+ texts: ReadonlyArray<string>,
1581
+ opts: { concurrency?: number } = {},
1582
+ ): Promise<{ warmed: number; cached: number }> {
1583
+ this.assertVoiceOn("prewarm voice phrases");
1584
+ return this.scheduler.prewarmPhrases(texts, opts);
1585
+ }
1586
+
1587
+ /**
1588
+ * Idle-time auto-prewarm hook: synthesize the canonical phrase-cache seed
1589
+ * (`DEFAULT_PHRASE_CACHE_SEED`) so common openers/acks are cached before
1590
+ * the next turn. The voice bridge / connector calls this when the loop is
1591
+ * idle. No-op (returns `{ warmed: 0, cached: 0 }`) unless a real TTS
1592
+ * backend is present and voice is armed — we never cache the test backend's zeros
1593
+ * (AGENTS.md §3).
1594
+ */
1595
+ async prewarmIdlePhrases(
1596
+ opts: { concurrency?: number } = {},
1597
+ ): Promise<{ warmed: number; cached: number }> {
1598
+ if (!this.hasRealTtsBackend()) return { warmed: 0, cached: 0 };
1599
+ if (this.lifecycle.current().kind !== "voice-on") {
1600
+ return { warmed: 0, cached: 0 };
1601
+ }
1602
+ return this.scheduler.prewarmPhrases(DEFAULT_PHRASE_CACHE_SEED, opts);
1603
+ }
1604
+
1605
+ /**
1606
+ * First-audio filler (AGENTS.md §4 / H4): the instant W1's VAD fires
1607
+ * `speech-start`, play a short cached acknowledgement ("one sec", "okay",
1608
+ * …) into the audio sink to mask first-token latency. W9's turn controller
1609
+ * owns the call site (it gets the `speech-start` event and the cutover to
1610
+ * real `replyText` audio); this method is the seam.
1611
+ *
1612
+ * It only ever plays audio that is *already in the phrase cache* — it does
1613
+ * not synthesize. Returns the filler text that was played, or `null` if no
1614
+ * filler was played (no real TTS backend, voice not armed, or none of the
1615
+ * filler phrases are cached). When real reply audio is ready, W9 cuts over
1616
+ * by writing it through the scheduler as usual (a `triggerBargeIn()` or a
1617
+ * direct `ringBuffer.drain()` truncates any still-playing filler first).
1618
+ */
1619
+ playFirstAudioFiller(): string | null {
1620
+ if (!this.hasRealTtsBackend()) return null;
1621
+ if (this.lifecycle.current().kind !== "voice-on") return null;
1622
+ for (const text of FIRST_AUDIO_FILLERS) {
1623
+ const cached = this.phraseCache.get(text);
1624
+ if (!cached || cached.pcm.length === 0) continue;
1625
+ this.scheduler.ringBuffer.write(cached.pcm);
1626
+ const flushed = this.scheduler.ringBuffer.flushToSink();
1627
+ this.scheduler.markAgentSpeakingForAudio(flushed, cached.sampleRate);
1628
+ return cached.text;
1629
+ }
1630
+ return null;
1631
+ }
1632
+
1633
+ /**
1634
+ * Construct a `StreamingTranscriber` for live ASR — the contract the
1635
+ * voice turn controller (W9) feeds mic frames into and the barge-in
1636
+ * word-confirm gate (W1) listens to. Resolves the adapter chain:
1637
+ * fused `libelizainference` streaming ASR (final path, gated on a
1638
+ * working decoder AND a bundled ASR model) → fused batch ASR over the
1639
+ * same bundled model → `AsrUnavailableError`. The Eliza-1 bridge runs
1640
+ * only the fused path; the whisper.cpp interim fallback has been removed.
1641
+ *
1642
+ * Pass W1's `vad` event stream to gate decoding to active speech
1643
+ * windows. Caller owns the returned transcriber's lifecycle (`dispose()`).
1644
+ */
1645
+ createStreamingTranscriber(opts?: {
1646
+ vad?: VadEventSource;
1647
+ }): StreamingTranscriber {
1648
+ this.assertVoiceOn("create streaming transcriber");
1649
+ const contextRef = this.ffiContextRef;
1650
+ return createStreamingTranscriber({
1651
+ ffi: this.ffi,
1652
+ getContext: contextRef ? () => contextRef.ensure() : undefined,
1653
+ asrBundlePresent: this.asrAvailable,
1654
+ vad: opts?.vad,
1655
+ });
1656
+ }
1657
+
1658
+ /**
1659
+ * Batch transcription: one-shot over a whole PCM buffer. When the active
1660
+ * backend exposes the fused batch ASR ABI, use it directly so the native
1661
+ * side receives the original sample rate and can apply its own resampling.
1662
+ * Otherwise drive a `StreamingTranscriber` (fused streaming ASR →
1663
+ * fused-batch interim) by feeding the buffer as a single frame and
1664
+ * `flush()`ing. Throws `AsrUnavailableError` when no ASR backend is
1665
+ * available — never a silent empty string.
1666
+ */
1667
+ async transcribePcm(
1668
+ args: TranscriptionAudio,
1669
+ signal?: AbortSignal,
1670
+ ): Promise<string> {
1671
+ this.assertVoiceOn("transcribe audio");
1672
+ if (signal?.aborted) {
1673
+ throw signal.reason instanceof Error
1674
+ ? signal.reason
1675
+ : new DOMException("Aborted", "AbortError");
1676
+ }
1677
+ const backendBatch = this.backend as OmniVoiceBackend & {
1678
+ transcribe?: (args: TranscriptionAudio) => Promise<string>;
1679
+ };
1680
+ if (typeof backendBatch.transcribe === "function") {
1681
+ const transcript = await backendBatch.transcribe(args);
1682
+ if (signal?.aborted) {
1683
+ throw signal.reason instanceof Error
1684
+ ? signal.reason
1685
+ : new DOMException("Aborted", "AbortError");
1686
+ }
1687
+ return transcript;
1688
+ }
1689
+ if (
1690
+ this.ffi &&
1691
+ this.ffiContextRef &&
1692
+ this.asrAvailable &&
1693
+ typeof this.ffi.asrTranscribe === "function"
1694
+ ) {
1695
+ const pcm =
1696
+ args.sampleRate === ASR_SAMPLE_RATE
1697
+ ? args.pcm
1698
+ : resampleLinear(args.pcm, args.sampleRate, ASR_SAMPLE_RATE);
1699
+ const transcript = this.ffi
1700
+ .asrTranscribe({
1701
+ ctx: this.ffiContextRef.ensure(),
1702
+ pcm,
1703
+ sampleRateHz: ASR_SAMPLE_RATE,
1704
+ })
1705
+ .trim();
1706
+ if (signal?.aborted) {
1707
+ throw signal.reason instanceof Error
1708
+ ? signal.reason
1709
+ : new DOMException("Aborted", "AbortError");
1710
+ }
1711
+ return transcript;
1712
+ }
1713
+ const transcriber = this.createStreamingTranscriber();
1714
+ const abort = () => transcriber.dispose();
1715
+ try {
1716
+ signal?.addEventListener("abort", abort, { once: true });
1717
+ transcriber.feed({
1718
+ pcm: args.pcm,
1719
+ sampleRate: args.sampleRate,
1720
+ timestampMs: 0,
1721
+ });
1722
+ const final = await transcriber.flush();
1723
+ if (signal?.aborted) {
1724
+ throw signal.reason instanceof Error
1725
+ ? signal.reason
1726
+ : new DOMException("Aborted", "AbortError");
1727
+ }
1728
+ return final.partial;
1729
+ } finally {
1730
+ signal?.removeEventListener("abort", abort);
1731
+ transcriber.dispose();
1732
+ }
1733
+ }
1734
+
1735
+ /**
1736
+ * Run one fused mic→speech turn through the overlapped `VoicePipeline`
1737
+ * (AGENTS.md §4): ASR streams; the instant its last token lands the
1738
+ * MTP drafter and the target verifier kick off concurrently, accepted
1739
+ * tokens flow into this bridge's phrase chunker → TTS → ring buffer on
1740
+ * the same tick, rejected draft tails roll back not-yet-spoken audio, and
1741
+ * a mic-VAD barge-in cancels everything at the next kernel boundary.
1742
+ *
1743
+ * The drafter + verifier are wired against the running MTP llama-server
1744
+ * (`textRunner`); the transcriber is the fused ABI's ASR when this bridge
1745
+ * was started with the FFI backend and the bundle ships an `asr/` region.
1746
+ * In voice mode a missing ASR region is a hard `VoiceStartupError` — no
1747
+ * silent cloud fallback (AGENTS.md §3 + §7).
1748
+ *
1749
+ * Resolves with the turn's exit reason. Throws if no turn is wired or one
1750
+ * is already in flight. The created pipeline is held until the turn ends
1751
+ * so `bargeIn()` can cancel it.
1752
+ */
1753
+ async runVoiceTurn(
1754
+ audio: TranscriptionAudio,
1755
+ textRunner: MtpTextRunner,
1756
+ config: VoicePipelineConfig,
1757
+ events?: VoiceTurnEvents,
1758
+ ): Promise<"done" | "token-cap" | "cancelled"> {
1759
+ this.assertVoiceOn("run a voice turn");
1760
+ // If a profileStore was wired, kick off speaker-attribution in parallel
1761
+ // with ASR. The attribution uses the same PCM buffer as the transcriber
1762
+ // but runs through the diarizer + encoder + profile-store independently.
1763
+ // It is fire-and-forget from the pipeline's perspective: the result
1764
+ // arrives via `onAttribution` asynchronously (possibly after onComplete).
1765
+ if (
1766
+ this.attributionPipeline &&
1767
+ (events?.onAttribution || this.eventRuntime)
1768
+ ) {
1769
+ const onAttribution = events?.onAttribution;
1770
+ const attribution = this.attributionPipeline;
1771
+ const eventRuntime = this.eventRuntime;
1772
+ const liveAttribution = this.liveAttribution;
1773
+ const turnId = `turn-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
1774
+ void attribution
1775
+ .attribute({
1776
+ turnId,
1777
+ pcm: audio.pcm,
1778
+ })
1779
+ .then(async (output) => {
1780
+ // Automatic seam: when a full runtime is wired, emit
1781
+ // VOICE_TURN_OBSERVED and fold the speaker decision into the
1782
+ // turn's voiceTurnSignal BEFORE handing the (now-stamped)
1783
+ // output to the caller. Any caller with a profileStore +
1784
+ // runtime gets diarization-driven gating for free.
1785
+ if (eventRuntime) {
1786
+ const { handleLiveVoiceAttribution } = await import(
1787
+ "../../runtime/voice-entity-binding.js"
1788
+ );
1789
+ await handleLiveVoiceAttribution(
1790
+ eventRuntime,
1791
+ output,
1792
+ resolveLiveAttributionOptions(liveAttribution),
1793
+ );
1794
+ }
1795
+ onAttribution?.(output);
1796
+ })
1797
+ .catch((err: unknown) => {
1798
+ // Attribution failures must not crash the turn. Log and continue.
1799
+ logger.warn(
1800
+ {
1801
+ turnId,
1802
+ error: err instanceof Error ? err.message : String(err),
1803
+ },
1804
+ "[voice-bridge] speaker attribution failed",
1805
+ );
1806
+ });
1807
+ }
1808
+ const pipeline = this.buildPipeline(textRunner, config, events);
1809
+ this.activePipeline = pipeline;
1810
+ try {
1811
+ return await pipeline.run(audio);
1812
+ } finally {
1813
+ if (this.activePipeline === pipeline) this.activePipeline = null;
1814
+ }
1815
+ }
1816
+
1817
+ /** Construct the `VoicePipeline` for this bridge (no-run). Exposed for tests. */
1818
+ buildPipeline(
1819
+ textRunner: MtpTextRunner,
1820
+ config: VoicePipelineConfig,
1821
+ events?: VoicePipelineEvents,
1822
+ ): VoicePipeline {
1823
+ const transcriber = this.resolveTranscriber();
1824
+ const deps: VoicePipelineDeps = {
1825
+ scheduler: this.scheduler,
1826
+ transcriber,
1827
+ drafter: new MtpDraftProposer(textRunner),
1828
+ verifier: new MtpTargetVerifier(textRunner),
1829
+ };
1830
+ return new VoicePipeline(deps, config, events);
1831
+ }
1832
+
1833
+ /**
1834
+ * Resolve the pipeline's ASR backend: a live `StreamingTranscriber` —
1835
+ * the fused `eliza_inference_asr_stream_*` decoder when the loaded build
1836
+ * advertises one and the bundle ships an `asr/` region, else the fused
1837
+ * batch ASR adapter. The `VoicePipeline` drives it as a batch
1838
+ * (feed the whole utterance, `flush()`, split the transcript into
1839
+ * tokens). When no ASR backend is available the failure is surfaced as a
1840
+ * `MissingAsrTranscriber` that throws on first use — AGENTS.md §3, no
1841
+ * silent cloud fallback.
1842
+ */
1843
+ private resolveTranscriber(): StreamingTranscriber {
1844
+ const ctxRef = this.ffiContextRef;
1845
+ try {
1846
+ return createStreamingTranscriber({
1847
+ ffi: this.ffi,
1848
+ getContext: ctxRef ? () => ctxRef.ensure() : undefined,
1849
+ asrBundlePresent: this.asrAvailable,
1850
+ });
1851
+ } catch (err) {
1852
+ if (err instanceof AsrUnavailableError) {
1853
+ return new MissingAsrTranscriber(err.message);
1854
+ }
1855
+ throw err;
1856
+ }
1857
+ }
1858
+
1859
+ /** Diagnostic accessor — bundle root the bridge is wired against. */
1860
+ bundlePath(): string {
1861
+ return this.bundleRoot;
1862
+ }
1863
+
1864
+ private assertVoiceOn(action: string): void {
1865
+ const state = this.lifecycle.current();
1866
+ if (state.kind === "voice-on") return;
1867
+ if (state.kind === "voice-error") {
1868
+ throw state.error;
1869
+ }
1870
+ throw new VoiceLifecycleError(
1871
+ "illegal-transition",
1872
+ `[voice] Cannot ${action} while lifecycle is ${state.kind}. Call armVoice() and wait for voice-on first.`,
1873
+ );
1874
+ }
1875
+ }
1876
+
1877
+ export function encodeMonoPcm16Wav(
1878
+ pcm: Float32Array,
1879
+ sampleRate: number,
1880
+ ): Uint8Array {
1881
+ const channels = 1;
1882
+ const bytesPerSample = 2;
1883
+ const dataBytes = pcm.length * bytesPerSample;
1884
+ const out = new Uint8Array(44 + dataBytes);
1885
+ const view = new DataView(out.buffer);
1886
+ writeAscii(out, 0, "RIFF");
1887
+ view.setUint32(4, 36 + dataBytes, true);
1888
+ writeAscii(out, 8, "WAVE");
1889
+ writeAscii(out, 12, "fmt ");
1890
+ view.setUint32(16, 16, true);
1891
+ view.setUint16(20, 1, true);
1892
+ view.setUint16(22, channels, true);
1893
+ view.setUint32(24, sampleRate, true);
1894
+ view.setUint32(28, sampleRate * channels * bytesPerSample, true);
1895
+ view.setUint16(32, channels * bytesPerSample, true);
1896
+ view.setUint16(34, bytesPerSample * 8, true);
1897
+ writeAscii(out, 36, "data");
1898
+ view.setUint32(40, dataBytes, true);
1899
+
1900
+ let offset = 44;
1901
+ for (const sample of pcm) {
1902
+ const clamped = Math.max(-1, Math.min(1, sample));
1903
+ const value = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
1904
+ view.setInt16(offset, Math.round(value), true);
1905
+ offset += bytesPerSample;
1906
+ }
1907
+ return out;
1908
+ }
1909
+
1910
+ export function decodeMonoPcm16Wav(bytes: Uint8Array): TranscriptionAudio {
1911
+ if (bytes.byteLength < 44) {
1912
+ throw new Error("[voice] WAV input is too short to contain a header");
1913
+ }
1914
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
1915
+ if (
1916
+ readAscii(bytes, 0, 4) !== "RIFF" ||
1917
+ readAscii(bytes, 8, 4) !== "WAVE" ||
1918
+ readAscii(bytes, 12, 4) !== "fmt "
1919
+ ) {
1920
+ throw new Error("[voice] Local transcription expects mono PCM16 WAV bytes");
1921
+ }
1922
+ const audioFormat = view.getUint16(20, true);
1923
+ const channels = view.getUint16(22, true);
1924
+ const sampleRate = view.getUint32(24, true);
1925
+ const bitsPerSample = view.getUint16(34, true);
1926
+ if (audioFormat !== 1 || channels !== 1 || bitsPerSample !== 16) {
1927
+ throw new Error(
1928
+ `[voice] Local transcription expects mono PCM16 WAV (format=1 channels=1 bits=16); got format=${audioFormat} channels=${channels} bits=${bitsPerSample}`,
1929
+ );
1930
+ }
1931
+
1932
+ let pos = 36;
1933
+ while (pos + 8 <= bytes.byteLength) {
1934
+ const chunkId = readAscii(bytes, pos, 4);
1935
+ const chunkBytes = view.getUint32(pos + 4, true);
1936
+ const dataStart = pos + 8;
1937
+ if (chunkId === "data") {
1938
+ if (dataStart + chunkBytes > bytes.byteLength) {
1939
+ throw new Error("[voice] WAV data chunk exceeds input length");
1940
+ }
1941
+ if (chunkBytes % 2 !== 0) {
1942
+ throw new Error("[voice] WAV PCM16 data chunk has odd byte length");
1943
+ }
1944
+ const pcm = new Float32Array(chunkBytes / 2);
1945
+ for (let i = 0; i < pcm.length; i++) {
1946
+ pcm[i] = view.getInt16(dataStart + i * 2, true) / 0x8000;
1947
+ }
1948
+ return { pcm, sampleRate };
1949
+ }
1950
+ pos = dataStart + chunkBytes + (chunkBytes % 2);
1951
+ }
1952
+ throw new Error("[voice] WAV input is missing a data chunk");
1953
+ }
1954
+
1955
+ function writeAscii(out: Uint8Array, offset: number, text: string): void {
1956
+ for (let i = 0; i < text.length; i++) {
1957
+ out[offset + i] = text.charCodeAt(i);
1958
+ }
1959
+ }
1960
+
1961
+ function readAscii(bytes: Uint8Array, offset: number, length: number): string {
1962
+ let out = "";
1963
+ for (let i = 0; i < length; i++) {
1964
+ out += String.fromCharCode(bytes[offset + i]);
1965
+ }
1966
+ return out;
1967
+ }
1968
+
1969
+ function readPositiveIntEnv(name: string): number | undefined {
1970
+ const raw = process.env[name]?.trim();
1971
+ if (!raw) return undefined;
1972
+ const value = Number.parseInt(raw, 10);
1973
+ return Number.isFinite(value) && value > 0 ? value : undefined;
1974
+ }
1975
+
1976
+ /**
1977
+ * Default lifecycle loaders derived from the bundle layout (per
1978
+ * AGENTS.md §2: `tts/omnivoice-<size>.gguf` + `asr/...`).
1979
+ *
1980
+ * When a live `ffi`/`ctx` pair is passed in, arming calls
1981
+ * `ffi.mmapAcquire(ctx, "tts" | "asr")` before the lifecycle can enter
1982
+ * `voice-on`, and the returned handles' `evictPages()` calls forward
1983
+ * to `ffi.mmapEvict(ctx, "tts" | "asr")`. The C ABI is declared in
1984
+ * `scripts/omnivoice-fuse/ffi.h`. Production builds may implement this
1985
+ * as page eviction or as a full voice-runtime unload for mobile RAM
1986
+ * pressure; callers must reacquire before using the region again. The
1987
+ * compatibility library returns `ELIZA_ERR_NOT_IMPLEMENTED`, which the binding raises as
1988
+ * `VoiceLifecycleError({code:"kernel-missing"})`.
1989
+ *
1990
+ * When `ffi` is null, acquire/evict are documented empty transitions — used by the
1991
+ * development TTS path in tests + dev (no real mmap exists). Directory and
1992
+ * "contains at least one file" checks still run for both TTS and ASR.
1993
+ * ASR never gets a virtual fallback: voice-on requires a real bundled ASR
1994
+ * model file so the FFI path can acquire the `"asr"` region and surface
1995
+ * the fused ABI's diagnostic if the runtime lacks the required region support.
1996
+ */
1997
+ interface FfiContextRef {
1998
+ current: ElizaInferenceContextHandle | null;
1999
+ ensure(): ElizaInferenceContextHandle;
2000
+ }
2001
+
2002
+ function ensureContext(
2003
+ ref: ElizaInferenceContextHandle | FfiContextRef | null,
2004
+ ): ElizaInferenceContextHandle | null {
2005
+ if (ref === null) return null;
2006
+ if (typeof ref === "object" && "ensure" in ref) return ref.ensure();
2007
+ return ref;
2008
+ }
2009
+
2010
+ /**
2011
+ * No-op lifecycle loaders for the Kokoro-only bridge. ORT owns the
2012
+ * model memory; nothing to mmap-acquire or evict. ASR is not served
2013
+ * from this path — callers that need ASR construct
2014
+ * `createStreamingTranscriber` directly (the fused-only chain in
2015
+ * `transcriber.ts`: fused streaming → fused batch → AsrUnavailableError).
2016
+ */
2017
+ function kokoroOnlyLifecycleLoaders(): VoiceLifecycleLoaders {
2018
+ const noopMmap = (id: string): MmapRegionHandle => ({
2019
+ id,
2020
+ path: "",
2021
+ sizeBytes: 0,
2022
+ async evictPages() {
2023
+ // Nothing to evict — ORT owns the model bytes.
2024
+ },
2025
+ async release() {
2026
+ // No mmap region to release.
2027
+ },
2028
+ });
2029
+ return {
2030
+ loadTtsRegion: async () => noopMmap("kokoro:tts"),
2031
+ loadAsrRegion: async () => noopMmap("kokoro:asr"),
2032
+ loadVoiceCaches: async () => ({
2033
+ id: "kokoro:voice-caches",
2034
+ async release() {},
2035
+ }),
2036
+ loadVoiceSchedulerNodes: async () => ({
2037
+ id: "kokoro:voice-scheduler-nodes",
2038
+ async release() {},
2039
+ }),
2040
+ };
2041
+ }
2042
+
2043
+ function defaultLifecycleLoaders(
2044
+ bundleRoot: string,
2045
+ ffi: ElizaInferenceFfi | null,
2046
+ ctx: ElizaInferenceContextHandle | FfiContextRef | null,
2047
+ ): VoiceLifecycleLoaders {
2048
+ return {
2049
+ loadTtsRegion: async () =>
2050
+ bundleMmapRegion(path.join(bundleRoot, "tts"), "tts", ffi, ctx),
2051
+ loadAsrRegion: async () =>
2052
+ bundleMmapRegion(path.join(bundleRoot, "asr"), "asr", ffi, ctx),
2053
+ loadVoiceCaches: async () => ({
2054
+ id: `voice-caches:${bundleRoot}`,
2055
+ async release() {
2056
+ // Caches stay live in the SpeakerPresetCache + PhraseCache
2057
+ // singletons; the registry refcount is the only thing that
2058
+ // drops on disarm.
2059
+ },
2060
+ }),
2061
+ loadVoiceSchedulerNodes: async () => ({
2062
+ id: `voice-scheduler-nodes:${bundleRoot}`,
2063
+ async release() {
2064
+ // Scheduler nodes (chunker, rollback, ring buffer, barge-in)
2065
+ // are owned by the bridge's `scheduler` field — no extra
2066
+ // teardown beyond the refcount drop.
2067
+ },
2068
+ }),
2069
+ };
2070
+ }
2071
+
2072
+ /**
2073
+ * Build an `MmapRegionHandle` for a bundle subdirectory. Refuses to
2074
+ * fabricate a region when the directory is missing — that surfaces as
2075
+ * `VoiceLifecycleError` via the lifecycle's `arm-failed`/`mmap-fail`
2076
+ * mapping (no silent fallback to a smaller voice model — AGENTS.md §3).
2077
+ *
2078
+ * `mmapAcquire()` / `evictPages()` forward to the FFI binding when one
2079
+ * is supplied. With no FFI handle (test mode), those calls return without
2080
+ * touching native memory because no real mmap was made. The lifecycle test
2081
+ * still asserts the call shape via injected mocks.
2082
+ */
2083
+ function bundleMmapRegion(
2084
+ dir: string,
2085
+ kind: "tts" | "asr",
2086
+ ffi: ElizaInferenceFfi | null,
2087
+ ctx: ElizaInferenceContextHandle | FfiContextRef | null,
2088
+ ): MmapRegionHandle {
2089
+ if (!existsSync(dir)) {
2090
+ throw new Error(
2091
+ `[voice] mmap MAP_FAILED: ${kind} directory missing at ${dir}`,
2092
+ );
2093
+ }
2094
+ if (!directoryHasRegularFile(dir)) {
2095
+ throw new Error(
2096
+ `[voice] mmap MAP_FAILED: ${kind} directory has no model files at ${dir}`,
2097
+ );
2098
+ }
2099
+ // Stat the directory to get a stable inode for id derivation. Real
2100
+ // FFI will mmap each weight file independently; this default loader
2101
+ // collapses them into one region per kind for refcount purposes.
2102
+ const st = statSync(dir);
2103
+ const handle = ffi ? ensureContext(ctx) : null;
2104
+ if (ffi && handle !== null) {
2105
+ // Real fused build: load or re-page the heavy voice region now.
2106
+ // A compatibility runtime without region support returns ELIZA_ERR_NOT_IMPLEMENTED,
2107
+ // which surfaces as VoiceLifecycleError({code:"kernel-missing"})
2108
+ // before the lifecycle can enter voice-on.
2109
+ ffi.mmapAcquire(handle, kind);
2110
+ }
2111
+ return {
2112
+ id: `mmap:${kind}:${st.ino}`,
2113
+ path: dir,
2114
+ sizeBytes: st.size,
2115
+ async evictPages() {
2116
+ const evictHandle = ffi ? ensureContext(ctx) : null;
2117
+ if (ffi && evictHandle !== null) {
2118
+ // Real fused build: madvise / VirtualUnlock through the C ABI.
2119
+ // Throws VoiceLifecycleError on a negative return — the
2120
+ // lifecycle catches and re-classifies via `disarm-failed`.
2121
+ ffi.mmapEvict(evictHandle, kind);
2122
+ }
2123
+ // Else: no FFI handle (test TTS / no fused build) — nothing to
2124
+ // evict.
2125
+ },
2126
+ async release() {
2127
+ // The FFI owns the actual mmap; release is a refcount drop on
2128
+ // the JS side. The fused build's destroy path flushes any
2129
+ // remaining pages when the context is destroyed.
2130
+ },
2131
+ };
2132
+ }
2133
+
2134
+ /** Re-export for the engine and tests that want the default loader. */
2135
+ export { defaultLifecycleLoaders };
2136
+
2137
+ /**
2138
+ * Platform-specific shared-library suffix for the fused omnivoice build.
2139
+ * macOS dylib, Linux/Android so, Windows dll. Windows artifacts have
2140
+ * used both `elizainference.dll` and `libelizainference.dll` names in
2141
+ * cross-build toolchains, so the runtime probes both.
2142
+ */
2143
+ function libraryFilenames(): string[] {
2144
+ if (process.platform === "darwin") return ["libelizainference.dylib"];
2145
+ if (process.platform === "win32") {
2146
+ return ["elizainference.dll", "libelizainference.dll"];
2147
+ }
2148
+ return ["libelizainference.so"];
2149
+ }
2150
+
2151
+ function locateBundleLibrary(bundleRoot: string): string {
2152
+ const exact = process.env.ELIZA_INFERENCE_LIBRARY?.trim();
2153
+ if (exact && existsSync(exact)) return exact;
2154
+
2155
+ const dirs = [
2156
+ path.join(bundleRoot, "lib"),
2157
+ exact ? path.dirname(exact) : null,
2158
+ process.env.ELIZA_INFERENCE_LIB_DIR?.trim() || null,
2159
+ ...managedFusedRuntimeDirs(),
2160
+ ].filter((dir): dir is string => Boolean(dir));
2161
+
2162
+ for (const dir of dirs) {
2163
+ for (const name of libraryFilenames()) {
2164
+ const candidate = path.join(dir, name);
2165
+ if (existsSync(candidate)) return candidate;
2166
+ }
2167
+ }
2168
+ return path.join(
2169
+ dirs[0] ?? path.join(bundleRoot, "lib"),
2170
+ libraryFilenames()[0] ?? "libelizainference.so",
2171
+ );
2172
+ }
2173
+
2174
+ function bundleHasOmniVoiceWeights(bundleRoot: string): boolean {
2175
+ const ttsDir = path.join(bundleRoot, "tts");
2176
+ if (!existsSync(ttsDir)) return false;
2177
+ try {
2178
+ return readdirSync(ttsDir, { withFileTypes: true }).some(
2179
+ (entry) => entry.isFile() && /^omnivoice-.+\.gguf$/i.test(entry.name),
2180
+ );
2181
+ } catch {
2182
+ return false;
2183
+ }
2184
+ }
2185
+
2186
+ export function isOmniVoiceBundleAvailable(bundleRoot: string): boolean {
2187
+ if (!bundleRoot || !existsSync(bundleRoot)) return false;
2188
+ const presetPath = path.join(bundleRoot, DEFAULT_VOICE_PRESET_REL_PATH);
2189
+ return (
2190
+ existsSync(presetPath) &&
2191
+ bundleHasOmniVoiceWeights(bundleRoot) &&
2192
+ existsSync(locateBundleLibrary(bundleRoot))
2193
+ );
2194
+ }
2195
+
2196
+ function directoryHasRegularFile(dir: string): boolean {
2197
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
2198
+ if (entry.isFile()) return true;
2199
+ }
2200
+ return false;
2201
+ }
2202
+
2203
+ function bundleHasRegularFile(dir: string): boolean {
2204
+ if (!existsSync(dir)) return false;
2205
+ try {
2206
+ return directoryHasRegularFile(dir);
2207
+ } catch {
2208
+ return false;
2209
+ }
2210
+ }
2211
+
2212
+ function managedFusedRuntimeDirs(): string[] {
2213
+ if (process.env.ELIZA_INFERENCE_MANAGED_LOOKUP?.trim() === "0") {
2214
+ return [];
2215
+ }
2216
+ const root = localInferenceRoot();
2217
+ const platform = process.platform;
2218
+ const arch = os.arch();
2219
+ const candidates = [
2220
+ `${platform}-${arch}-metal-fused`,
2221
+ `${platform}-${arch}-vulkan-fused`,
2222
+ `${platform}-${arch}-cuda-fused`,
2223
+ `${platform}-${arch}-cpu-fused`,
2224
+ ];
2225
+ return candidates.map((target) => path.join(root, "bin", "mtp", target));
2226
+ }