@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (701) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +83 -0
  3. package/package.json +82 -15
  4. package/src/actions/generate-media.d.ts +59 -0
  5. package/src/actions/generate-media.d.ts.map +1 -0
  6. package/src/actions/generate-media.ts +647 -0
  7. package/src/actions/identify-speaker.d.ts +23 -0
  8. package/src/actions/identify-speaker.d.ts.map +1 -0
  9. package/src/actions/identify-speaker.ts +171 -0
  10. package/src/actions/transcription-control.d.ts +29 -0
  11. package/src/actions/transcription-control.d.ts.map +1 -0
  12. package/src/actions/transcription-control.test.ts +100 -0
  13. package/src/actions/transcription-control.ts +127 -0
  14. package/src/adapters/capacitor-llama/__tests__/compat-behavior.test.ts +218 -0
  15. package/src/adapters/capacitor-llama/__tests__/index.test.ts +68 -0
  16. package/src/adapters/capacitor-llama/__tests__/structured-output.test.ts +215 -0
  17. package/src/adapters/capacitor-llama/__tests__/text-streaming.test.ts +174 -0
  18. package/src/adapters/capacitor-llama/environment.ts +71 -0
  19. package/src/adapters/capacitor-llama/index.browser.ts +83 -0
  20. package/src/adapters/capacitor-llama/index.ts +807 -0
  21. package/src/adapters/capacitor-llama/loader.ts +109 -0
  22. package/src/adapters/capacitor-llama/structured-output.ts +165 -0
  23. package/src/adapters/capacitor-llama/text-streaming.ts +227 -0
  24. package/src/adapters/capacitor-llama/types.ts +374 -0
  25. package/src/backends/apple-foundation.ts +127 -0
  26. package/src/index.d.ts +8 -0
  27. package/src/index.d.ts.map +1 -0
  28. package/src/index.ts +62 -0
  29. package/src/local-inference-routes.d.ts +38 -0
  30. package/src/local-inference-routes.d.ts.map +1 -0
  31. package/src/local-inference-routes.test.ts +344 -0
  32. package/src/local-inference-routes.ts +1543 -0
  33. package/src/provider.d.ts +21 -0
  34. package/src/provider.d.ts.map +1 -0
  35. package/src/provider.ts +1082 -0
  36. package/src/routes/compat-helpers.d.ts +18 -0
  37. package/src/routes/compat-helpers.d.ts.map +1 -0
  38. package/src/routes/compat-helpers.ts +274 -0
  39. package/src/routes/family-member-route.d.ts +62 -0
  40. package/src/routes/family-member-route.d.ts.map +1 -0
  41. package/src/routes/family-member-route.ts +353 -0
  42. package/src/routes/index.d.ts +19 -0
  43. package/src/routes/index.d.ts.map +1 -0
  44. package/src/routes/index.ts +60 -0
  45. package/src/routes/live-diarization-route.d.ts +26 -0
  46. package/src/routes/live-diarization-route.d.ts.map +1 -0
  47. package/src/routes/live-diarization-route.test.ts +213 -0
  48. package/src/routes/live-diarization-route.ts +122 -0
  49. package/src/routes/local-inference-asr-route.d.ts +4 -0
  50. package/src/routes/local-inference-asr-route.d.ts.map +1 -0
  51. package/src/routes/local-inference-asr-route.test.ts +205 -0
  52. package/src/routes/local-inference-asr-route.ts +163 -0
  53. package/src/routes/local-inference-asr-transcribe.d.ts +20 -0
  54. package/src/routes/local-inference-asr-transcribe.d.ts.map +1 -0
  55. package/src/routes/local-inference-asr-transcribe.test.ts +118 -0
  56. package/src/routes/local-inference-asr-transcribe.ts +97 -0
  57. package/src/routes/local-inference-compat-routes.d.ts +16 -0
  58. package/src/routes/local-inference-compat-routes.d.ts.map +1 -0
  59. package/src/routes/local-inference-compat-routes.test.ts +485 -0
  60. package/src/routes/local-inference-compat-routes.ts +808 -0
  61. package/src/routes/local-inference-tts-route.d.ts +7 -0
  62. package/src/routes/local-inference-tts-route.d.ts.map +1 -0
  63. package/src/routes/local-inference-tts-route.test.ts +179 -0
  64. package/src/routes/local-inference-tts-route.ts +230 -0
  65. package/src/routes/transcript-audio-store.d.ts +15 -0
  66. package/src/routes/transcript-audio-store.d.ts.map +1 -0
  67. package/src/routes/transcript-audio-store.ts +27 -0
  68. package/src/routes/transcripts-routes.d.ts +36 -0
  69. package/src/routes/transcripts-routes.d.ts.map +1 -0
  70. package/src/routes/transcripts-routes.test.ts +144 -0
  71. package/src/routes/transcripts-routes.ts +159 -0
  72. package/src/routes/voice-first-run-routes.d.ts +62 -0
  73. package/src/routes/voice-first-run-routes.d.ts.map +1 -0
  74. package/src/routes/voice-first-run-routes.ts +524 -0
  75. package/src/routes/voice-models-routes.d.ts +62 -0
  76. package/src/routes/voice-models-routes.d.ts.map +1 -0
  77. package/src/routes/voice-models-routes.ts +554 -0
  78. package/src/routes/voice-profile-plugin-routes.d.ts +19 -0
  79. package/src/routes/voice-profile-plugin-routes.d.ts.map +1 -0
  80. package/src/routes/voice-profile-plugin-routes.ts +138 -0
  81. package/src/routes/voice-profiles-management-routes.d.ts +52 -0
  82. package/src/routes/voice-profiles-management-routes.d.ts.map +1 -0
  83. package/src/routes/voice-profiles-management-routes.ts +476 -0
  84. package/src/routes/voice-speaker-profile-routes.d.ts +57 -0
  85. package/src/routes/voice-speaker-profile-routes.d.ts.map +1 -0
  86. package/src/routes/voice-speaker-profile-routes.ts +199 -0
  87. package/src/runtime/aosp-llama-loader-selection.test.ts +80 -0
  88. package/src/runtime/capacitor-llama.d.ts +25 -0
  89. package/src/runtime/embedding-manager-support.d.ts +77 -0
  90. package/src/runtime/embedding-manager-support.d.ts.map +1 -0
  91. package/src/runtime/embedding-manager-support.ts +497 -0
  92. package/src/runtime/embedding-presets.d.ts +16 -0
  93. package/src/runtime/embedding-presets.d.ts.map +1 -0
  94. package/src/runtime/embedding-presets.ts +81 -0
  95. package/src/runtime/embedding-warmup-policy.d.ts +14 -0
  96. package/src/runtime/embedding-warmup-policy.d.ts.map +1 -0
  97. package/src/runtime/embedding-warmup-policy.test.ts +53 -0
  98. package/src/runtime/embedding-warmup-policy.ts +48 -0
  99. package/src/runtime/ensure-local-inference-handler.d.ts +62 -0
  100. package/src/runtime/ensure-local-inference-handler.d.ts.map +1 -0
  101. package/src/runtime/ensure-local-inference-handler.test.ts +528 -0
  102. package/src/runtime/ensure-local-inference-handler.ts +1448 -0
  103. package/src/runtime/index.d.ts +15 -0
  104. package/src/runtime/index.d.ts.map +1 -0
  105. package/src/runtime/index.ts +33 -0
  106. package/src/runtime/mobile-local-inference-gate.d.ts +31 -0
  107. package/src/runtime/mobile-local-inference-gate.d.ts.map +1 -0
  108. package/src/runtime/mobile-local-inference-gate.test.ts +69 -0
  109. package/src/runtime/mobile-local-inference-gate.ts +44 -0
  110. package/src/runtime/voice-entity-binding.d.ts +103 -0
  111. package/src/runtime/voice-entity-binding.d.ts.map +1 -0
  112. package/src/runtime/voice-entity-binding.transcript.test.ts +69 -0
  113. package/src/runtime/voice-entity-binding.ts +328 -0
  114. package/src/services/README.md +71 -0
  115. package/src/services/__tests__/backend-selector.test.ts +101 -0
  116. package/src/services/__tests__/checkpoint-manager.test.ts +376 -0
  117. package/src/services/__tests__/gpu-autotune.test.ts +400 -0
  118. package/src/services/__tests__/llm-streaming-binding.test.ts +85 -0
  119. package/src/services/__tests__/planner-grammar.test.ts +372 -0
  120. package/src/services/__tests__/runtime-target.test.ts +176 -0
  121. package/src/services/active-model-switch-rollback.test.ts +183 -0
  122. package/src/services/active-model.d.ts +282 -0
  123. package/src/services/active-model.d.ts.map +1 -0
  124. package/src/services/active-model.ts +1213 -0
  125. package/src/services/assignments.d.ts +71 -0
  126. package/src/services/assignments.d.ts.map +1 -0
  127. package/src/services/assignments.test.ts +80 -0
  128. package/src/services/assignments.ts +230 -0
  129. package/src/services/backend-selector.ts +95 -0
  130. package/src/services/backend.d.ts +346 -0
  131. package/src/services/backend.d.ts.map +1 -0
  132. package/src/services/backend.ts +612 -0
  133. package/src/services/bionic-host-loader.d.ts +46 -0
  134. package/src/services/bionic-host-loader.d.ts.map +1 -0
  135. package/src/services/bionic-host-loader.test.ts +133 -0
  136. package/src/services/bionic-host-loader.ts +180 -0
  137. package/src/services/bundled-models.d.ts +34 -0
  138. package/src/services/bundled-models.d.ts.map +1 -0
  139. package/src/services/bundled-models.ts +129 -0
  140. package/src/services/cache-bridge.d.ts +206 -0
  141. package/src/services/cache-bridge.d.ts.map +1 -0
  142. package/src/services/cache-bridge.test.ts +516 -0
  143. package/src/services/cache-bridge.ts +423 -0
  144. package/src/services/catalog.d.ts +10 -0
  145. package/src/services/catalog.d.ts.map +1 -0
  146. package/src/services/catalog.test.ts +238 -0
  147. package/src/services/catalog.ts +27 -0
  148. package/src/services/checkpoint-client.d.ts +109 -0
  149. package/src/services/checkpoint-client.d.ts.map +1 -0
  150. package/src/services/checkpoint-client.ts +258 -0
  151. package/src/services/checkpoint-manager.ts +474 -0
  152. package/src/services/cloud-fallback.d.ts +102 -0
  153. package/src/services/cloud-fallback.d.ts.map +1 -0
  154. package/src/services/cloud-fallback.ts +230 -0
  155. package/src/services/conversation-registry.d.ts +142 -0
  156. package/src/services/conversation-registry.d.ts.map +1 -0
  157. package/src/services/conversation-registry.test.ts +235 -0
  158. package/src/services/conversation-registry.ts +264 -0
  159. package/src/services/desktop-fused-ffi-backend-runtime.d.ts +95 -0
  160. package/src/services/desktop-fused-ffi-backend-runtime.d.ts.map +1 -0
  161. package/src/services/desktop-fused-ffi-backend-runtime.ts +339 -0
  162. package/src/services/device-bridge.d.ts +188 -0
  163. package/src/services/device-bridge.d.ts.map +1 -0
  164. package/src/services/device-bridge.ts +1237 -0
  165. package/src/services/device-resource-metrics.d.ts +149 -0
  166. package/src/services/device-resource-metrics.d.ts.map +1 -0
  167. package/src/services/device-resource-metrics.test.ts +98 -0
  168. package/src/services/device-resource-metrics.ts +346 -0
  169. package/src/services/device-tier.d.ts +115 -0
  170. package/src/services/device-tier.d.ts.map +1 -0
  171. package/src/services/device-tier.test.ts +371 -0
  172. package/src/services/device-tier.ts +410 -0
  173. package/src/services/downloader.d.ts +82 -0
  174. package/src/services/downloader.d.ts.map +1 -0
  175. package/src/services/downloader.test.ts +747 -0
  176. package/src/services/downloader.ts +925 -0
  177. package/src/services/engine-direct-bundle.test.ts +58 -0
  178. package/src/services/engine-streaming.test.ts +80 -0
  179. package/src/services/engine.d.ts +540 -0
  180. package/src/services/engine.d.ts.map +1 -0
  181. package/src/services/engine.ts +1909 -0
  182. package/src/services/ensure-local-artifacts.integration.test.ts +273 -0
  183. package/src/services/ensure-local-artifacts.test.ts +368 -0
  184. package/src/services/ensure-local-artifacts.ts +351 -0
  185. package/src/services/external-scanner.d.ts +17 -0
  186. package/src/services/external-scanner.d.ts.map +1 -0
  187. package/src/services/external-scanner.ts +312 -0
  188. package/src/services/ffi-llm-mock.ts +354 -0
  189. package/src/services/ffi-llm-streaming-abi.ts +442 -0
  190. package/src/services/ffi-streaming-backend.d.ts +180 -0
  191. package/src/services/ffi-streaming-backend.d.ts.map +1 -0
  192. package/src/services/ffi-streaming-backend.ts +382 -0
  193. package/src/services/ffi-streaming-runner.d.ts +122 -0
  194. package/src/services/ffi-streaming-runner.d.ts.map +1 -0
  195. package/src/services/ffi-streaming-runner.test.ts +60 -0
  196. package/src/services/ffi-streaming-runner.ts +354 -0
  197. package/src/services/ffi-unload-ordering.test.ts +162 -0
  198. package/src/services/gpu-autotune.ts +534 -0
  199. package/src/services/gpu-detect.d.ts +56 -0
  200. package/src/services/gpu-detect.d.ts.map +1 -0
  201. package/src/services/gpu-detect.ts +139 -0
  202. package/src/services/handler-registry.d.ts +72 -0
  203. package/src/services/handler-registry.d.ts.map +1 -0
  204. package/src/services/handler-registry.ts +240 -0
  205. package/src/services/hardware.d.ts +63 -0
  206. package/src/services/hardware.d.ts.map +1 -0
  207. package/src/services/hardware.test.ts +231 -0
  208. package/src/services/hardware.ts +410 -0
  209. package/src/services/hf-search.d.ts +26 -0
  210. package/src/services/hf-search.d.ts.map +1 -0
  211. package/src/services/hf-search.test.ts +69 -0
  212. package/src/services/hf-search.ts +420 -0
  213. package/src/services/image-description-runtime.d.ts +14 -0
  214. package/src/services/image-description-runtime.d.ts.map +1 -0
  215. package/src/services/image-description-runtime.test.ts +61 -0
  216. package/src/services/image-description-runtime.ts +118 -0
  217. package/src/services/imagegen/aosp-unavailable.d.ts +134 -0
  218. package/src/services/imagegen/aosp-unavailable.d.ts.map +1 -0
  219. package/src/services/imagegen/aosp-unavailable.ts +229 -0
  220. package/src/services/imagegen/backend-selector.d.ts +118 -0
  221. package/src/services/imagegen/backend-selector.d.ts.map +1 -0
  222. package/src/services/imagegen/backend-selector.ts +277 -0
  223. package/src/services/imagegen/coreml-unavailable.d.ts +105 -0
  224. package/src/services/imagegen/coreml-unavailable.d.ts.map +1 -0
  225. package/src/services/imagegen/coreml-unavailable.ts +237 -0
  226. package/src/services/imagegen/errors.d.ts +16 -0
  227. package/src/services/imagegen/errors.d.ts.map +1 -0
  228. package/src/services/imagegen/errors.ts +40 -0
  229. package/src/services/imagegen/index.d.ts +58 -0
  230. package/src/services/imagegen/index.d.ts.map +1 -0
  231. package/src/services/imagegen/index.ts +144 -0
  232. package/src/services/imagegen/mflux.d.ts +74 -0
  233. package/src/services/imagegen/mflux.d.ts.map +1 -0
  234. package/src/services/imagegen/mflux.ts +313 -0
  235. package/src/services/imagegen/sd-cpp.d.ts +180 -0
  236. package/src/services/imagegen/sd-cpp.d.ts.map +1 -0
  237. package/src/services/imagegen/sd-cpp.ts +718 -0
  238. package/src/services/imagegen/tensorrt-unavailable.d.ts +83 -0
  239. package/src/services/imagegen/tensorrt-unavailable.d.ts.map +1 -0
  240. package/src/services/imagegen/tensorrt-unavailable.ts +295 -0
  241. package/src/services/imagegen/types.d.ts +181 -0
  242. package/src/services/imagegen/types.d.ts.map +1 -0
  243. package/src/services/imagegen/types.ts +193 -0
  244. package/src/services/index.d.ts +29 -0
  245. package/src/services/index.d.ts.map +1 -0
  246. package/src/services/index.ts +211 -0
  247. package/src/services/inference-capabilities.d.ts +132 -0
  248. package/src/services/inference-capabilities.d.ts.map +1 -0
  249. package/src/services/inference-capabilities.test.ts +75 -0
  250. package/src/services/inference-capabilities.ts +204 -0
  251. package/src/services/inference-telemetry.d.ts +59 -0
  252. package/src/services/inference-telemetry.d.ts.map +1 -0
  253. package/src/services/inference-telemetry.ts +143 -0
  254. package/src/services/ios-llama-streaming.ts +248 -0
  255. package/src/services/kv-spill.d.ts +189 -0
  256. package/src/services/kv-spill.d.ts.map +1 -0
  257. package/src/services/kv-spill.test.ts +222 -0
  258. package/src/services/kv-spill.ts +356 -0
  259. package/src/services/latency-trace.d.ts +346 -0
  260. package/src/services/latency-trace.d.ts.map +1 -0
  261. package/src/services/latency-trace.test.ts +266 -0
  262. package/src/services/latency-trace.ts +844 -0
  263. package/src/services/llama-server-metrics.ts +304 -0
  264. package/src/services/llm-streaming-binding.d.ts +96 -0
  265. package/src/services/llm-streaming-binding.d.ts.map +1 -0
  266. package/src/services/llm-streaming-binding.ts +136 -0
  267. package/src/services/load-args.d.ts +82 -0
  268. package/src/services/load-args.d.ts.map +1 -0
  269. package/src/services/load-args.ts +81 -0
  270. package/src/services/manifest/eliza-1.manifest.v1.json +708 -0
  271. package/src/services/manifest/index.d.ts +4 -0
  272. package/src/services/manifest/index.d.ts.map +1 -0
  273. package/src/services/manifest/index.ts +66 -0
  274. package/src/services/manifest/manifest.test.ts +689 -0
  275. package/src/services/manifest/schema.d.ts +713 -0
  276. package/src/services/manifest/schema.d.ts.map +1 -0
  277. package/src/services/manifest/schema.ts +653 -0
  278. package/src/services/manifest/types.d.ts +30 -0
  279. package/src/services/manifest/types.d.ts.map +1 -0
  280. package/src/services/manifest/types.ts +55 -0
  281. package/src/services/manifest/validator.d.ts +66 -0
  282. package/src/services/manifest/validator.d.ts.map +1 -0
  283. package/src/services/manifest/validator.ts +567 -0
  284. package/src/services/memory-arbiter.d.ts +318 -0
  285. package/src/services/memory-arbiter.d.ts.map +1 -0
  286. package/src/services/memory-arbiter.test.ts +419 -0
  287. package/src/services/memory-arbiter.ts +925 -0
  288. package/src/services/memory-monitor.d.ts +122 -0
  289. package/src/services/memory-monitor.d.ts.map +1 -0
  290. package/src/services/memory-monitor.test.ts +208 -0
  291. package/src/services/memory-monitor.ts +297 -0
  292. package/src/services/memory-pressure.d.ts +130 -0
  293. package/src/services/memory-pressure.d.ts.map +1 -0
  294. package/src/services/memory-pressure.ts +414 -0
  295. package/src/services/mtp-doctor.d.ts +13 -0
  296. package/src/services/mtp-doctor.d.ts.map +1 -0
  297. package/src/services/mtp-doctor.ts +78 -0
  298. package/src/services/network-policy.d.ts +127 -0
  299. package/src/services/network-policy.d.ts.map +1 -0
  300. package/src/services/network-policy.ts +346 -0
  301. package/src/services/paths.d.ts +6 -0
  302. package/src/services/paths.d.ts.map +1 -0
  303. package/src/services/paths.ts +25 -0
  304. package/src/services/planner-skeleton.d.ts +124 -0
  305. package/src/services/planner-skeleton.d.ts.map +1 -0
  306. package/src/services/planner-skeleton.ts +175 -0
  307. package/src/services/providers.d.ts +38 -0
  308. package/src/services/providers.d.ts.map +1 -0
  309. package/src/services/providers.ts +507 -0
  310. package/src/services/ram-budget-cache.test.ts +163 -0
  311. package/src/services/ram-budget.d.ts +110 -0
  312. package/src/services/ram-budget.d.ts.map +1 -0
  313. package/src/services/ram-budget.ts +0 -0
  314. package/src/services/readiness.d.ts +9 -0
  315. package/src/services/readiness.d.ts.map +1 -0
  316. package/src/services/readiness.test.ts +87 -0
  317. package/src/services/readiness.ts +238 -0
  318. package/src/services/recommendation.d.ts +111 -0
  319. package/src/services/recommendation.d.ts.map +1 -0
  320. package/src/services/recommendation.ts +671 -0
  321. package/src/services/registry.d.ts +35 -0
  322. package/src/services/registry.d.ts.map +1 -0
  323. package/src/services/registry.ts +151 -0
  324. package/src/services/router-handler.d.ts +92 -0
  325. package/src/services/router-handler.d.ts.map +1 -0
  326. package/src/services/router-handler.test.ts +45 -0
  327. package/src/services/router-handler.ts +407 -0
  328. package/src/services/routing-policy.d.ts +69 -0
  329. package/src/services/routing-policy.d.ts.map +1 -0
  330. package/src/services/routing-policy.test.ts +164 -0
  331. package/src/services/routing-policy.ts +297 -0
  332. package/src/services/routing-preferences.d.ts +8 -0
  333. package/src/services/routing-preferences.d.ts.map +1 -0
  334. package/src/services/routing-preferences.ts +17 -0
  335. package/src/services/runtime-target.d.ts +98 -0
  336. package/src/services/runtime-target.d.ts.map +1 -0
  337. package/src/services/runtime-target.ts +154 -0
  338. package/src/services/service.d.ts +128 -0
  339. package/src/services/service.d.ts.map +1 -0
  340. package/src/services/service.test.ts +223 -0
  341. package/src/services/service.ts +735 -0
  342. package/src/services/session-pool.d.ts +72 -0
  343. package/src/services/session-pool.d.ts.map +1 -0
  344. package/src/services/session-pool.ts +153 -0
  345. package/src/services/structured-output/deterministic-repair.d.ts +23 -0
  346. package/src/services/structured-output/deterministic-repair.d.ts.map +1 -0
  347. package/src/services/structured-output/deterministic-repair.test.ts +169 -0
  348. package/src/services/structured-output/deterministic-repair.ts +443 -0
  349. package/src/services/structured-output/index.ts +4 -0
  350. package/src/services/structured-output.d.ts +311 -0
  351. package/src/services/structured-output.d.ts.map +1 -0
  352. package/src/services/structured-output.test.ts +483 -0
  353. package/src/services/structured-output.ts +712 -0
  354. package/src/services/system-memory.d.ts +33 -0
  355. package/src/services/system-memory.d.ts.map +1 -0
  356. package/src/services/system-memory.test.ts +47 -0
  357. package/src/services/system-memory.ts +67 -0
  358. package/src/services/transcription-priority.test.ts +211 -0
  359. package/src/services/types.d.ts +19 -0
  360. package/src/services/types.d.ts.map +1 -0
  361. package/src/services/types.ts +55 -0
  362. package/src/services/verify-on-device.d.ts +34 -0
  363. package/src/services/verify-on-device.d.ts.map +1 -0
  364. package/src/services/verify-on-device.test.ts +87 -0
  365. package/src/services/verify-on-device.ts +127 -0
  366. package/src/services/verify.d.ts +8 -0
  367. package/src/services/verify.d.ts.map +1 -0
  368. package/src/services/verify.ts +13 -0
  369. package/src/services/vision/aosp-unavailable.d.ts +115 -0
  370. package/src/services/vision/aosp-unavailable.d.ts.map +1 -0
  371. package/src/services/vision/aosp-unavailable.ts +163 -0
  372. package/src/services/vision/capacitor-llama.d.ts +99 -0
  373. package/src/services/vision/capacitor-llama.d.ts.map +1 -0
  374. package/src/services/vision/capacitor-llama.ts +255 -0
  375. package/src/services/vision/cloud-fallback.d.ts +47 -0
  376. package/src/services/vision/cloud-fallback.d.ts.map +1 -0
  377. package/src/services/vision/cloud-fallback.test.ts +243 -0
  378. package/src/services/vision/cloud-fallback.ts +268 -0
  379. package/src/services/vision/fallback-chain.test.ts +86 -0
  380. package/src/services/vision/hash.d.ts +71 -0
  381. package/src/services/vision/hash.d.ts.map +1 -0
  382. package/src/services/vision/hash.ts +157 -0
  383. package/src/services/vision/index.d.ts +95 -0
  384. package/src/services/vision/index.d.ts.map +1 -0
  385. package/src/services/vision/index.ts +251 -0
  386. package/src/services/vision/llama-server.d.ts +73 -0
  387. package/src/services/vision/llama-server.d.ts.map +1 -0
  388. package/src/services/vision/llama-server.ts +177 -0
  389. package/src/services/vision/types.d.ts +153 -0
  390. package/src/services/vision/types.d.ts.map +1 -0
  391. package/src/services/vision/types.ts +154 -0
  392. package/src/services/vision/vast-fallback.d.ts +18 -0
  393. package/src/services/vision/vast-fallback.d.ts.map +1 -0
  394. package/src/services/vision/vast-fallback.ts +127 -0
  395. package/src/services/vision-embedding-cache.d.ts +98 -0
  396. package/src/services/vision-embedding-cache.d.ts.map +1 -0
  397. package/src/services/vision-embedding-cache.ts +189 -0
  398. package/src/services/voice/VOICE_WORKBENCH.md +88 -0
  399. package/src/services/voice/__test-helpers__/fake-ffi.ts +94 -0
  400. package/src/services/voice/__test-helpers__/synthetic-speech.ts +124 -0
  401. package/src/services/voice/__tests__/checkpoint-manager.test.ts +241 -0
  402. package/src/services/voice/__tests__/checkpoint-policy.test.ts +270 -0
  403. package/src/services/voice/__tests__/eager-context-builder.test.ts +257 -0
  404. package/src/services/voice/__tests__/eliza1-eot-scorer.test.ts +288 -0
  405. package/src/services/voice/__tests__/eot-classifier.test.ts +431 -0
  406. package/src/services/voice/__tests__/optimistic-rollback.test.ts +312 -0
  407. package/src/services/voice/__tests__/prefill-client.test.ts +266 -0
  408. package/src/services/voice/__tests__/prefix-preserving-queue.test.ts +208 -0
  409. package/src/services/voice/__tests__/streaming-asr.test.ts +450 -0
  410. package/src/services/voice/__tests__/streaming-transcriber.test.ts +339 -0
  411. package/src/services/voice/__tests__/turn-detector-resolver.test.ts +195 -0
  412. package/src/services/voice/__tests__/voice-state-machine-prefill.test.ts +275 -0
  413. package/src/services/voice/__tests__/voice-state-machine.test.ts +354 -0
  414. package/src/services/voice/asr-timed.real.test.ts +141 -0
  415. package/src/services/voice/audio-frame-consumer.d.ts +212 -0
  416. package/src/services/voice/audio-frame-consumer.d.ts.map +1 -0
  417. package/src/services/voice/audio-frame-consumer.test.ts +343 -0
  418. package/src/services/voice/audio-frame-consumer.ts +491 -0
  419. package/src/services/voice/barge-in.d.ts +112 -0
  420. package/src/services/voice/barge-in.d.ts.map +1 -0
  421. package/src/services/voice/barge-in.test.ts +244 -0
  422. package/src/services/voice/barge-in.ts +336 -0
  423. package/src/services/voice/cancellation-coordinator.d.ts +127 -0
  424. package/src/services/voice/cancellation-coordinator.d.ts.map +1 -0
  425. package/src/services/voice/cancellation-coordinator.test.ts +196 -0
  426. package/src/services/voice/cancellation-coordinator.ts +269 -0
  427. package/src/services/voice/checkpoint-manager.d.ts +199 -0
  428. package/src/services/voice/checkpoint-manager.d.ts.map +1 -0
  429. package/src/services/voice/checkpoint-manager.ts +401 -0
  430. package/src/services/voice/checkpoint-policy.ts +336 -0
  431. package/src/services/voice/composite-eot-classifier.test.ts +59 -0
  432. package/src/services/voice/e2e-harness.test.ts +182 -0
  433. package/src/services/voice/e2e-harness.ts +743 -0
  434. package/src/services/voice/eager-context-builder.d.ts +170 -0
  435. package/src/services/voice/eager-context-builder.d.ts.map +1 -0
  436. package/src/services/voice/eager-context-builder.ts +262 -0
  437. package/src/services/voice/eliza1-eot-scorer.d.ts +124 -0
  438. package/src/services/voice/eliza1-eot-scorer.d.ts.map +1 -0
  439. package/src/services/voice/eliza1-eot-scorer.ts +242 -0
  440. package/src/services/voice/embedding-server.ts +200 -0
  441. package/src/services/voice/embedding.d.ts +133 -0
  442. package/src/services/voice/embedding.d.ts.map +1 -0
  443. package/src/services/voice/embedding.test.ts +131 -0
  444. package/src/services/voice/embedding.ts +243 -0
  445. package/src/services/voice/emotion-attribution.d.ts +68 -0
  446. package/src/services/voice/emotion-attribution.d.ts.map +1 -0
  447. package/src/services/voice/emotion-attribution.test.ts +129 -0
  448. package/src/services/voice/emotion-attribution.ts +361 -0
  449. package/src/services/voice/engine-bridge-cancellation.test.ts +422 -0
  450. package/src/services/voice/engine-bridge.d.ts +759 -0
  451. package/src/services/voice/engine-bridge.d.ts.map +1 -0
  452. package/src/services/voice/engine-bridge.test.ts +384 -0
  453. package/src/services/voice/engine-bridge.ts +2302 -0
  454. package/src/services/voice/eot-classifier-ggml.d.ts +179 -0
  455. package/src/services/voice/eot-classifier-ggml.d.ts.map +1 -0
  456. package/src/services/voice/eot-classifier-ggml.ts +566 -0
  457. package/src/services/voice/eot-classifier.d.ts +214 -0
  458. package/src/services/voice/eot-classifier.d.ts.map +1 -0
  459. package/src/services/voice/eot-classifier.ts +533 -0
  460. package/src/services/voice/errors.d.ts +20 -0
  461. package/src/services/voice/errors.d.ts.map +1 -0
  462. package/src/services/voice/errors.ts +32 -0
  463. package/src/services/voice/expressive-tags.d.ts +158 -0
  464. package/src/services/voice/expressive-tags.d.ts.map +1 -0
  465. package/src/services/voice/expressive-tags.ts +405 -0
  466. package/src/services/voice/ffi-bindings.d.ts +674 -0
  467. package/src/services/voice/ffi-bindings.d.ts.map +1 -0
  468. package/src/services/voice/ffi-bindings.test.ts +728 -0
  469. package/src/services/voice/ffi-bindings.ts +3225 -0
  470. package/src/services/voice/first-line-cache.d.ts +181 -0
  471. package/src/services/voice/first-line-cache.d.ts.map +1 -0
  472. package/src/services/voice/first-line-cache.ts +725 -0
  473. package/src/services/voice/fused-eot-scorer.d.ts +51 -0
  474. package/src/services/voice/fused-eot-scorer.d.ts.map +1 -0
  475. package/src/services/voice/fused-eot-scorer.ts +135 -0
  476. package/src/services/voice/index.d.ts +91 -0
  477. package/src/services/voice/index.d.ts.map +1 -0
  478. package/src/services/voice/index.ts +481 -0
  479. package/src/services/voice/kokoro/__tests__/kokoro-backend.test.ts +151 -0
  480. package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.real.test.ts +151 -0
  481. package/src/services/voice/kokoro/__tests__/kokoro-engine-bridge.test.ts +60 -0
  482. package/src/services/voice/kokoro/__tests__/kokoro-engine-discovery.test.ts +277 -0
  483. package/src/services/voice/kokoro/__tests__/kokoro-ffi-runtime.test.ts +235 -0
  484. package/src/services/voice/kokoro/__tests__/kokoro-runtime.test.ts +95 -0
  485. package/src/services/voice/kokoro/__tests__/phonemizer.test.ts +53 -0
  486. package/src/services/voice/kokoro/__tests__/runtime-selection.test.ts +231 -0
  487. package/src/services/voice/kokoro/__tests__/voices.test.ts +57 -0
  488. package/src/services/voice/kokoro/index.ts +79 -0
  489. package/src/services/voice/kokoro/kokoro-backend.d.ts +72 -0
  490. package/src/services/voice/kokoro/kokoro-backend.d.ts.map +1 -0
  491. package/src/services/voice/kokoro/kokoro-backend.ts +207 -0
  492. package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts +58 -0
  493. package/src/services/voice/kokoro/kokoro-engine-discovery.d.ts.map +1 -0
  494. package/src/services/voice/kokoro/kokoro-engine-discovery.ts +177 -0
  495. package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts +75 -0
  496. package/src/services/voice/kokoro/kokoro-ffi-runtime.d.ts.map +1 -0
  497. package/src/services/voice/kokoro/kokoro-ffi-runtime.ts +233 -0
  498. package/src/services/voice/kokoro/kokoro-runtime.d.ts +100 -0
  499. package/src/services/voice/kokoro/kokoro-runtime.d.ts.map +1 -0
  500. package/src/services/voice/kokoro/kokoro-runtime.ts +170 -0
  501. package/src/services/voice/kokoro/phoneme-stream.ts +123 -0
  502. package/src/services/voice/kokoro/phonemizer.d.ts +50 -0
  503. package/src/services/voice/kokoro/phonemizer.d.ts.map +1 -0
  504. package/src/services/voice/kokoro/phonemizer.ts +344 -0
  505. package/src/services/voice/kokoro/pick-runtime.d.ts +61 -0
  506. package/src/services/voice/kokoro/pick-runtime.d.ts.map +1 -0
  507. package/src/services/voice/kokoro/pick-runtime.test.ts +91 -0
  508. package/src/services/voice/kokoro/pick-runtime.ts +130 -0
  509. package/src/services/voice/kokoro/runtime-selection.d.ts +92 -0
  510. package/src/services/voice/kokoro/runtime-selection.d.ts.map +1 -0
  511. package/src/services/voice/kokoro/runtime-selection.ts +237 -0
  512. package/src/services/voice/kokoro/types.d.ts +82 -0
  513. package/src/services/voice/kokoro/types.d.ts.map +1 -0
  514. package/src/services/voice/kokoro/types.ts +95 -0
  515. package/src/services/voice/kokoro/voice-presets.d.ts +23 -0
  516. package/src/services/voice/kokoro/voice-presets.d.ts.map +1 -0
  517. package/src/services/voice/kokoro/voice-presets.ts +129 -0
  518. package/src/services/voice/kokoro/voices.d.ts +30 -0
  519. package/src/services/voice/kokoro/voices.d.ts.map +1 -0
  520. package/src/services/voice/kokoro/voices.ts +64 -0
  521. package/src/services/voice/lifecycle.d.ts +135 -0
  522. package/src/services/voice/lifecycle.d.ts.map +1 -0
  523. package/src/services/voice/lifecycle.test.ts +315 -0
  524. package/src/services/voice/lifecycle.ts +301 -0
  525. package/src/services/voice/live-diarization-session.d.ts +96 -0
  526. package/src/services/voice/live-diarization-session.d.ts.map +1 -0
  527. package/src/services/voice/live-diarization-session.ts +289 -0
  528. package/src/services/voice/mic-source.d.ts +136 -0
  529. package/src/services/voice/mic-source.d.ts.map +1 -0
  530. package/src/services/voice/mic-source.test.ts +210 -0
  531. package/src/services/voice/mic-source.ts +503 -0
  532. package/src/services/voice/optimistic-policy.d.ts +109 -0
  533. package/src/services/voice/optimistic-policy.d.ts.map +1 -0
  534. package/src/services/voice/optimistic-policy.test.ts +101 -0
  535. package/src/services/voice/optimistic-policy.ts +192 -0
  536. package/src/services/voice/optimistic-rollback.ts +343 -0
  537. package/src/services/voice/partial-stabilizer.d.ts +73 -0
  538. package/src/services/voice/partial-stabilizer.d.ts.map +1 -0
  539. package/src/services/voice/partial-stabilizer.test.ts +68 -0
  540. package/src/services/voice/partial-stabilizer.ts +140 -0
  541. package/src/services/voice/phoneme-tokenizer.d.ts +49 -0
  542. package/src/services/voice/phoneme-tokenizer.d.ts.map +1 -0
  543. package/src/services/voice/phoneme-tokenizer.ts +158 -0
  544. package/src/services/voice/phrase-cache.d.ts +76 -0
  545. package/src/services/voice/phrase-cache.d.ts.map +1 -0
  546. package/src/services/voice/phrase-cache.test.ts +242 -0
  547. package/src/services/voice/phrase-cache.ts +186 -0
  548. package/src/services/voice/phrase-chunker.d.ts +62 -0
  549. package/src/services/voice/phrase-chunker.d.ts.map +1 -0
  550. package/src/services/voice/phrase-chunker.test.ts +239 -0
  551. package/src/services/voice/phrase-chunker.ts +281 -0
  552. package/src/services/voice/pipeline-impls.d.ts +151 -0
  553. package/src/services/voice/pipeline-impls.d.ts.map +1 -0
  554. package/src/services/voice/pipeline-impls.l6.test.ts +110 -0
  555. package/src/services/voice/pipeline-impls.test.ts +292 -0
  556. package/src/services/voice/pipeline-impls.ts +315 -0
  557. package/src/services/voice/pipeline.d.ts +216 -0
  558. package/src/services/voice/pipeline.d.ts.map +1 -0
  559. package/src/services/voice/pipeline.ts +505 -0
  560. package/src/services/voice/prefill-client.d.ts +123 -0
  561. package/src/services/voice/prefill-client.d.ts.map +1 -0
  562. package/src/services/voice/prefill-client.ts +316 -0
  563. package/src/services/voice/prefix-preserving-queue.d.ts +113 -0
  564. package/src/services/voice/prefix-preserving-queue.d.ts.map +1 -0
  565. package/src/services/voice/prefix-preserving-queue.ts +162 -0
  566. package/src/services/voice/profile-store.d.ts +248 -0
  567. package/src/services/voice/profile-store.d.ts.map +1 -0
  568. package/src/services/voice/profile-store.ts +887 -0
  569. package/src/services/voice/real-audio-decode.test.ts +148 -0
  570. package/src/services/voice/ring-buffer.d.ts +40 -0
  571. package/src/services/voice/ring-buffer.d.ts.map +1 -0
  572. package/src/services/voice/ring-buffer.test.ts +129 -0
  573. package/src/services/voice/ring-buffer.ts +123 -0
  574. package/src/services/voice/rollback-queue.d.ts +24 -0
  575. package/src/services/voice/rollback-queue.d.ts.map +1 -0
  576. package/src/services/voice/rollback-queue.ts +74 -0
  577. package/src/services/voice/samantha-preset-placeholder.d.ts +67 -0
  578. package/src/services/voice/samantha-preset-placeholder.d.ts.map +1 -0
  579. package/src/services/voice/samantha-preset-placeholder.test.ts +97 -0
  580. package/src/services/voice/samantha-preset-placeholder.ts +148 -0
  581. package/src/services/voice/samantha-preset-regenerator.d.ts +87 -0
  582. package/src/services/voice/samantha-preset-regenerator.d.ts.map +1 -0
  583. package/src/services/voice/samantha-preset-regenerator.ts +393 -0
  584. package/src/services/voice/scheduler.d.ts +146 -0
  585. package/src/services/voice/scheduler.d.ts.map +1 -0
  586. package/src/services/voice/scheduler.t2.test.ts +141 -0
  587. package/src/services/voice/scheduler.ts +927 -0
  588. package/src/services/voice/shared-resources.d.ts +190 -0
  589. package/src/services/voice/shared-resources.d.ts.map +1 -0
  590. package/src/services/voice/shared-resources.ts +320 -0
  591. package/src/services/voice/speaker/attribution-pipeline.d.ts +74 -0
  592. package/src/services/voice/speaker/attribution-pipeline.d.ts.map +1 -0
  593. package/src/services/voice/speaker/attribution-pipeline.ts +386 -0
  594. package/src/services/voice/speaker/diarizer-fused.d.ts +59 -0
  595. package/src/services/voice/speaker/diarizer-fused.d.ts.map +1 -0
  596. package/src/services/voice/speaker/diarizer-fused.real.test.ts +100 -0
  597. package/src/services/voice/speaker/diarizer-fused.ts +154 -0
  598. package/src/services/voice/speaker/diarizer.d.ts +75 -0
  599. package/src/services/voice/speaker/diarizer.d.ts.map +1 -0
  600. package/src/services/voice/speaker/diarizer.ts +218 -0
  601. package/src/services/voice/speaker/encoder-fused.d.ts +60 -0
  602. package/src/services/voice/speaker/encoder-fused.d.ts.map +1 -0
  603. package/src/services/voice/speaker/encoder-fused.real.test.ts +113 -0
  604. package/src/services/voice/speaker/encoder-fused.ts +138 -0
  605. package/src/services/voice/speaker/encoder-ggml.d.ts +33 -0
  606. package/src/services/voice/speaker/encoder-ggml.d.ts.map +1 -0
  607. package/src/services/voice/speaker/encoder-ggml.ts +79 -0
  608. package/src/services/voice/speaker/encoder.d.ts +37 -0
  609. package/src/services/voice/speaker/encoder.d.ts.map +1 -0
  610. package/src/services/voice/speaker/encoder.ts +105 -0
  611. package/src/services/voice/speaker-imprint.d.ts +83 -0
  612. package/src/services/voice/speaker-imprint.d.ts.map +1 -0
  613. package/src/services/voice/speaker-imprint.test.ts +185 -0
  614. package/src/services/voice/speaker-imprint.ts +312 -0
  615. package/src/services/voice/speaker-preset-cache.d.ts +77 -0
  616. package/src/services/voice/speaker-preset-cache.d.ts.map +1 -0
  617. package/src/services/voice/speaker-preset-cache.test.ts +154 -0
  618. package/src/services/voice/speaker-preset-cache.ts +195 -0
  619. package/src/services/voice/streaming-asr/streaming-pipeline-adapter.ts +292 -0
  620. package/src/services/voice/system-audio-sink.d.ts +73 -0
  621. package/src/services/voice/system-audio-sink.d.ts.map +1 -0
  622. package/src/services/voice/system-audio-sink.test.ts +29 -0
  623. package/src/services/voice/system-audio-sink.ts +366 -0
  624. package/src/services/voice/transcriber.d.ts +244 -0
  625. package/src/services/voice/transcriber.d.ts.map +1 -0
  626. package/src/services/voice/transcriber.test.ts +392 -0
  627. package/src/services/voice/transcriber.ts +704 -0
  628. package/src/services/voice/transcript-knowledge.d.ts +37 -0
  629. package/src/services/voice/transcript-knowledge.d.ts.map +1 -0
  630. package/src/services/voice/transcript-knowledge.test.ts +68 -0
  631. package/src/services/voice/transcript-knowledge.ts +75 -0
  632. package/src/services/voice/transcript-service.d.ts +41 -0
  633. package/src/services/voice/transcript-service.d.ts.map +1 -0
  634. package/src/services/voice/transcript-service.test.ts +137 -0
  635. package/src/services/voice/transcript-service.ts +141 -0
  636. package/src/services/voice/transcript-store.d.ts +53 -0
  637. package/src/services/voice/transcript-store.d.ts.map +1 -0
  638. package/src/services/voice/transcript-store.test.ts +153 -0
  639. package/src/services/voice/transcript-store.ts +132 -0
  640. package/src/services/voice/turn-controller.d.ts +183 -0
  641. package/src/services/voice/turn-controller.d.ts.map +1 -0
  642. package/src/services/voice/turn-controller.test.ts +575 -0
  643. package/src/services/voice/turn-controller.ts +596 -0
  644. package/src/services/voice/types.d.ts +643 -0
  645. package/src/services/voice/types.d.ts.map +1 -0
  646. package/src/services/voice/types.ts +699 -0
  647. package/src/services/voice/vad.d.ts +282 -0
  648. package/src/services/voice/vad.d.ts.map +1 -0
  649. package/src/services/voice/vad.test.ts +480 -0
  650. package/src/services/voice/vad.ts +827 -0
  651. package/src/services/voice/vad.v1-v4.test.ts +222 -0
  652. package/src/services/voice/voice-budget.d.ts +241 -0
  653. package/src/services/voice/voice-budget.d.ts.map +1 -0
  654. package/src/services/voice/voice-budget.test.ts +418 -0
  655. package/src/services/voice/voice-budget.ts +635 -0
  656. package/src/services/voice/voice-duet.test.ts +375 -0
  657. package/src/services/voice/voice-emotion-classifier.d.ts +95 -0
  658. package/src/services/voice/voice-emotion-classifier.d.ts.map +1 -0
  659. package/src/services/voice/voice-emotion-classifier.test.ts +210 -0
  660. package/src/services/voice/voice-emotion-classifier.ts +273 -0
  661. package/src/services/voice/voice-preset-format.d.ts +158 -0
  662. package/src/services/voice/voice-preset-format.d.ts.map +1 -0
  663. package/src/services/voice/voice-preset-format.ts +700 -0
  664. package/src/services/voice/voice-preset-generator.test.ts +89 -0
  665. package/src/services/voice/voice-profile-artifact.d.ts +116 -0
  666. package/src/services/voice/voice-profile-artifact.d.ts.map +1 -0
  667. package/src/services/voice/voice-profile-artifact.test.ts +138 -0
  668. package/src/services/voice/voice-profile-artifact.ts +518 -0
  669. package/src/services/voice/voice-profile-routes.d.ts +83 -0
  670. package/src/services/voice/voice-profile-routes.d.ts.map +1 -0
  671. package/src/services/voice/voice-profile-routes.test.ts +429 -0
  672. package/src/services/voice/voice-profile-routes.ts +425 -0
  673. package/src/services/voice/voice-scenario.ts +154 -0
  674. package/src/services/voice/voice-settings.d.ts +82 -0
  675. package/src/services/voice/voice-settings.d.ts.map +1 -0
  676. package/src/services/voice/voice-settings.ts +172 -0
  677. package/src/services/voice/voice-state-machine.d.ts +364 -0
  678. package/src/services/voice/voice-state-machine.d.ts.map +1 -0
  679. package/src/services/voice/voice-state-machine.ts +727 -0
  680. package/src/services/voice/voice-workbench-report.test.ts +168 -0
  681. package/src/services/voice/voice-workbench-report.ts +326 -0
  682. package/src/services/voice/voice-workbench.test.ts +158 -0
  683. package/src/services/voice/voice.test.ts +1070 -0
  684. package/src/services/voice/wake-word-ggml.d.ts +101 -0
  685. package/src/services/voice/wake-word-ggml.d.ts.map +1 -0
  686. package/src/services/voice/wake-word-ggml.ts +320 -0
  687. package/src/services/voice/wake-word.d.ts +255 -0
  688. package/src/services/voice/wake-word.d.ts.map +1 -0
  689. package/src/services/voice/wake-word.test.ts +298 -0
  690. package/src/services/voice/wake-word.ts +554 -0
  691. package/src/services/voice/wrap-with-first-line-cache.d.ts +70 -0
  692. package/src/services/voice/wrap-with-first-line-cache.d.ts.map +1 -0
  693. package/src/services/voice/wrap-with-first-line-cache.ts +267 -0
  694. package/src/services/voice-model-updater.d.ts +240 -0
  695. package/src/services/voice-model-updater.d.ts.map +1 -0
  696. package/src/services/voice-model-updater.ts +724 -0
  697. package/src/services/voice-prewarm.d.ts +3 -0
  698. package/src/services/voice-prewarm.d.ts.map +1 -0
  699. package/src/services/voice-prewarm.ts +51 -0
  700. package/dist/index.d.ts +0 -37
  701. package/dist/index.js +0 -1098
@@ -0,0 +1,2302 @@
1
+ /**
2
+ * Engine ↔ voice scheduler bridge.
3
+ *
4
+ * Adapts the live `LocalInferenceEngine` (`engine.ts`) plus the MTP
5
+ * llama-server (`ffi-streaming-backend.ts`) onto the voice scaffold's
6
+ * `VoiceScheduler`. See `packages/inference/AGENTS.md` §4 for the
7
+ * streaming graph this implements:
8
+ *
9
+ * ASR → text tokens → MTP drafter ↔ target verifier (text model)
10
+ * → phrase chunker → speaker preset cache + phrase cache
11
+ * → OmniVoice TTS → PCM ring buffer → audio out
12
+ *
13
+ * Plus rollback queue (MTP rejection → cancel pending TTS chunks)
14
+ * and barge-in cancellation (mic VAD → drain ring buffer + cancel TTS).
15
+ *
16
+ * Two TTS backends are exposed:
17
+ * - `StubOmniVoiceBackend`: deterministic synthetic PCM. Used by tests
18
+ * and any path that wants the streaming graph without real audio.
19
+ * - `FfiOmniVoiceBackend`: forwards through the fused
20
+ * `libelizainference.{dylib,so,dll}` ABI. The bridge creates the
21
+ * context lazily when voice is armed or first used, so voice-off
22
+ * does not keep OmniVoice weights resident.
23
+ *
24
+ * Per AGENTS.md §3 + §9 (no defensive code, no log-and-continue), every
25
+ * startup precondition surfaces as a thrown `VoiceStartupError`. There
26
+ * is no silent fallback to text-only.
27
+ */
28
+
29
+ import { existsSync, readdirSync, statSync } from "node:fs";
30
+ import os from "node:os";
31
+ import path from "node:path";
32
+ import type { IAgentRuntime } from "@elizaos/core";
33
+ import { logger } from "@elizaos/core";
34
+ import type { VoiceCancellationReason } from "@elizaos/shared";
35
+ import { localInferenceRoot } from "../paths";
36
+ import {
37
+ type CoordinatorRuntime,
38
+ VoiceCancellationCoordinator,
39
+ } from "./cancellation-coordinator";
40
+ import { VoiceStartupError } from "./errors";
41
+ import type {
42
+ AsrWordTiming,
43
+ ElizaInferenceContextHandle,
44
+ ElizaInferenceFfi,
45
+ NativeVerifierEvent,
46
+ } from "./ffi-bindings";
47
+ import { loadElizaInferenceFfi } from "./ffi-bindings";
48
+ import { KokoroTtsBackend } from "./kokoro/kokoro-backend";
49
+ import type { KokoroEngineDiscoveryResult } from "./kokoro/kokoro-engine-discovery";
50
+ import { pickKokoroRuntimeBackend } from "./kokoro/pick-runtime";
51
+ import {
52
+ VoiceLifecycle,
53
+ VoiceLifecycleError,
54
+ type VoiceLifecycleLoaders,
55
+ } from "./lifecycle";
56
+ import {
57
+ OptimisticGenerationPolicy,
58
+ type OptimisticPolicyOptions,
59
+ resolvePowerSourceState,
60
+ } from "./optimistic-policy";
61
+ import {
62
+ type CachedPhraseAudio,
63
+ DEFAULT_PHRASE_CACHE_SEED,
64
+ FIRST_AUDIO_FILLERS,
65
+ PhraseCache,
66
+ } from "./phrase-cache";
67
+ import {
68
+ VoicePipeline,
69
+ type VoicePipelineConfig,
70
+ type VoicePipelineDeps,
71
+ type VoicePipelineEvents,
72
+ } from "./pipeline";
73
+ import {
74
+ MissingAsrTranscriber,
75
+ MtpDraftProposer,
76
+ MtpTargetVerifier,
77
+ type MtpTextRunner,
78
+ } from "./pipeline-impls";
79
+ import type { VoiceProfileStore } from "./profile-store";
80
+ import { type SchedulerEvents, VoiceScheduler } from "./scheduler";
81
+ import {
82
+ type MmapRegionHandle,
83
+ SharedResourceRegistry,
84
+ } from "./shared-resources";
85
+ import {
86
+ type VoiceAttributionOutput,
87
+ VoiceAttributionPipeline,
88
+ } from "./speaker/attribution-pipeline";
89
+ import {
90
+ type Diarizer,
91
+ PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID,
92
+ } from "./speaker/diarizer";
93
+ import { FusedDiarizer } from "./speaker/diarizer-fused";
94
+ import type { SpeakerEncoder } from "./speaker/encoder";
95
+ import { FusedSpeakerEncoder } from "./speaker/encoder-fused";
96
+ import {
97
+ SPEAKER_GGML_EMBEDDING_DIM,
98
+ SPEAKER_GGML_SAMPLE_RATE,
99
+ } from "./speaker/encoder-ggml";
100
+ import {
101
+ DEFAULT_VOICE_PRESET_REL_PATH,
102
+ SpeakerPresetCache,
103
+ } from "./speaker-preset-cache";
104
+ import {
105
+ ASR_SAMPLE_RATE,
106
+ AsrUnavailableError,
107
+ createStreamingTranscriber,
108
+ resampleLinear,
109
+ } from "./transcriber";
110
+ import type {
111
+ AudioChunk,
112
+ AudioSink,
113
+ OmniVoiceBackend,
114
+ Phrase,
115
+ RejectedTokenRange,
116
+ SchedulerConfig,
117
+ SpeakerPreset,
118
+ StreamingTranscriber,
119
+ TextToken,
120
+ TranscriptionAudio,
121
+ VadEventSource,
122
+ } from "./types";
123
+
124
+ const SAMPLE_RATE_DEFAULT = 24_000;
125
+ const RING_BUFFER_CAPACITY_DEFAULT = SAMPLE_RATE_DEFAULT * 4; // 4s
126
+ /**
127
+ * Runtime default for the no-punctuation phrase cap (`PhraseChunker.maxTokensPerPhrase`).
128
+ * Punctuation (`, . ! ?`) is still the primary boundary; this only bounds
129
+ * a run-on token stream. Kept small — equal to the MTP draft window
130
+ * (`DEFAULT_VOICE_MAX_DRAFT_TOKENS` in `engine.ts`) — so first-audio latency
131
+ * is bounded (a phrase ≈ one draft round of audio, not 30 words) and a
132
+ * MTP-reject rollback drops at most one un-spoken chunk (AGENTS.md §4 —
133
+ * "small chunk = low latency cost on rollback"). Override per bridge via
134
+ * `maxTokensPerPhrase` or `ELIZA_VOICE_MAX_TOKENS_PER_PHRASE`. The
135
+ * `PhraseChunker` primitive keeps the AGENTS-spec 30-word default for
136
+ * non-runtime callers.
137
+ */
138
+ const PHRASE_MAX_TOKENS_DEFAULT = 8;
139
+ const STUB_PCM_MS_PER_PHRASE = 100;
140
+ const STUB_PCM_STREAM_CHUNKS = 4;
141
+
142
+ /**
143
+ * Resolve the `speaker_preset_id` value to send across the FFI boundary.
144
+ *
145
+ * Historically this returned `null` for the default voice — the C side then
146
+ * treated `null` as "auto-voice mode" and ignored any preset file under
147
+ * `cache/voice-preset-default.bin`. That was the right behaviour when the
148
+ * default preset was a 256-fp32-zero placeholder; it's wrong now that the
149
+ * default preset can be a real (v2) OmniVoice sam freeze. With ABI v4
150
+ * the FFI bridge looks up `<bundle>/cache/voice-preset-<id>.bin` when the
151
+ * id is supplied and applies the `(instruct, ref_audio_tokens, ref_text)`
152
+ * triple to `ov_tts_params` — so we must always pass the id.
153
+ *
154
+ * The only case we return `null` is when the preset shape is degenerate
155
+ * (no embedding, no ref-audio-tokens, no instruct) — i.e. an explicit
156
+ * "no preset" signal from a caller that doesn't want a voice bound. The
157
+ * FFI side honours `null` by running OmniVoice's intrinsic auto-voice
158
+ * path.
159
+ */
160
+ function ffiSpeakerPresetId(preset: SpeakerPreset): string | null {
161
+ const hasV2Payload =
162
+ (preset.instruct !== undefined && preset.instruct.length > 0) ||
163
+ (preset.refText !== undefined && preset.refText.length > 0) ||
164
+ (preset.refAudioTokens !== undefined &&
165
+ preset.refAudioTokens.tokens.length > 0);
166
+ const hasEmbedding = preset.embedding.length > 0;
167
+ if (!hasV2Payload && !hasEmbedding) {
168
+ // Degenerate preset (e.g. the 1052-byte all-zero placeholder). The C
169
+ // side cannot do anything useful with it; let OmniVoice pick its own
170
+ // voice via the auto-voice path.
171
+ return null;
172
+ }
173
+ return preset.voiceId;
174
+ }
175
+
176
+ /** Re-exported from `./errors` so existing `engine-bridge` importers don't churn. */
177
+ export { VoiceStartupError };
178
+
179
+ /**
180
+ * Native verifier callbacks report rejected token ranges as half-open
181
+ * `[from, to)` intervals. The scheduler rollback queue uses inclusive
182
+ * token indexes, so convert in exactly one place.
183
+ */
184
+ export function nativeRejectedRangeToRollbackRange(
185
+ event: Pick<NativeVerifierEvent, "rejectedFrom" | "rejectedTo">,
186
+ ): RejectedTokenRange | null {
187
+ if (event.rejectedFrom < 0 || event.rejectedTo <= event.rejectedFrom) {
188
+ return null;
189
+ }
190
+ return {
191
+ fromIndex: event.rejectedFrom,
192
+ toIndex: event.rejectedTo - 1,
193
+ };
194
+ }
195
+
196
+ /**
197
+ * One PCM segment delivered to a `StreamingTtsBackend.synthesizeStream`
198
+ * consumer (W9's scheduler) as TTS decodes it. `isFinal` marks the
199
+ * zero-length tail chunk that closes the phrase.
200
+ */
201
+ export interface TtsPcmChunk {
202
+ pcm: Float32Array;
203
+ sampleRate: number;
204
+ isFinal: boolean;
205
+ }
206
+
207
+ /**
208
+ * Streaming-TTS seam between the fused `libelizainference` runtime and
209
+ * W9's voice scheduler. The scheduler calls `synthesizeStream(...)` for
210
+ * a phrase and writes each delivered `pcm` segment into the
211
+ * `PcmRingBuffer` on the same scheduler tick (AGENTS.md §4 —
212
+ * phrase-chunk → TTS within one scheduler tick); returning `true` from
213
+ * `onChunk` (or flipping `cancelSignal.cancelled`) hard-cancels the
214
+ * in-flight forward pass at the next kernel boundary (barge-in /
215
+ * MTP-rejected tail).
216
+ *
217
+ * Both `OmniVoiceBackend` implementations in this module satisfy it:
218
+ * - `FfiOmniVoiceBackend` forwards to
219
+ * `eliza_inference_tts_synthesize_stream` when the loaded build
220
+ * advertises streaming TTS (`tts_stream_supported() == 1`), else it
221
+ * synthesizes whole and emits the result as one body chunk + a final
222
+ * tail (no silent "streaming" lie — the chunk count just collapses
223
+ * to one when the build is non-streaming);
224
+ * - `StubOmniVoiceBackend` emits deterministic synthetic PCM split
225
+ * into a fixed number of chunks so scheduler tests can observe the
226
+ * incremental handoff without a real model.
227
+ */
228
+ export interface StreamingTtsBackend {
229
+ /**
230
+ * Synthesize `phrase` with `preset` and deliver PCM in chunks. The
231
+ * scheduler owns the ring-buffer write inside `onChunk`. Resolves with
232
+ * `cancelled: true` if `onChunk` requested a stop (or `cancelSignal`
233
+ * was set), `false` on a clean finish. The final `onChunk` call always
234
+ * has `isFinal: true` (possibly a zero-length `pcm`) so the consumer
235
+ * can settle per-phrase state.
236
+ */
237
+ synthesizeStream(args: {
238
+ phrase: Phrase;
239
+ preset: SpeakerPreset;
240
+ cancelSignal: { cancelled: boolean };
241
+ onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
242
+ onKernelTick?: () => void;
243
+ }): Promise<{ cancelled: boolean }>;
244
+ }
245
+
246
+ /** True when `backend` implements the `StreamingTtsBackend` seam. */
247
+ export function isStreamingTtsBackend(
248
+ backend: OmniVoiceBackend,
249
+ ): backend is OmniVoiceBackend & StreamingTtsBackend {
250
+ return (
251
+ typeof (backend as Partial<StreamingTtsBackend>).synthesizeStream ===
252
+ "function"
253
+ );
254
+ }
255
+
256
+ /**
257
+ * Deterministic test TTS backend. Each phrase yields
258
+ * `STUB_PCM_MS_PER_PHRASE` ms of silence (zeros), with the
259
+ * cancel signal honoured at the kernel-tick boundary so barge-in tests
260
+ * observe cancellation without waiting on a real model.
261
+ */
262
+ export class StubOmniVoiceBackend
263
+ implements OmniVoiceBackend, StreamingTtsBackend
264
+ {
265
+ readonly id = "stub" as const;
266
+ private readonly sampleRate: number;
267
+ calls = 0;
268
+ streamCalls = 0;
269
+
270
+ constructor(sampleRate = SAMPLE_RATE_DEFAULT) {
271
+ this.sampleRate = sampleRate;
272
+ }
273
+
274
+ async synthesize(args: {
275
+ phrase: Phrase;
276
+ preset: SpeakerPreset;
277
+ cancelSignal: { cancelled: boolean };
278
+ onKernelTick?: () => void;
279
+ }): Promise<AudioChunk> {
280
+ this.calls++;
281
+ args.onKernelTick?.();
282
+ const samples = Math.floor(
283
+ (this.sampleRate * STUB_PCM_MS_PER_PHRASE) / 1000,
284
+ );
285
+ const pcm = new Float32Array(samples);
286
+ return {
287
+ phraseId: args.phrase.id,
288
+ fromIndex: args.phrase.fromIndex,
289
+ toIndex: args.phrase.toIndex,
290
+ pcm,
291
+ sampleRate: this.sampleRate,
292
+ };
293
+ }
294
+
295
+ async synthesizeStream(args: {
296
+ phrase: Phrase;
297
+ preset: SpeakerPreset;
298
+ cancelSignal: { cancelled: boolean };
299
+ onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
300
+ onKernelTick?: () => void;
301
+ }): Promise<{ cancelled: boolean }> {
302
+ this.streamCalls++;
303
+ const totalSamples = Math.floor(
304
+ (this.sampleRate * STUB_PCM_MS_PER_PHRASE) / 1000,
305
+ );
306
+ const perChunk = Math.max(
307
+ 1,
308
+ Math.ceil(totalSamples / STUB_PCM_STREAM_CHUNKS),
309
+ );
310
+ let cancelled = false;
311
+ for (let off = 0; off < totalSamples; off += perChunk) {
312
+ args.onKernelTick?.();
313
+ if (args.cancelSignal.cancelled) {
314
+ cancelled = true;
315
+ break;
316
+ }
317
+ const n = Math.min(perChunk, totalSamples - off);
318
+ const want = args.onChunk({
319
+ pcm: new Float32Array(n),
320
+ sampleRate: this.sampleRate,
321
+ isFinal: false,
322
+ });
323
+ if (want === true || args.cancelSignal.cancelled) {
324
+ cancelled = true;
325
+ break;
326
+ }
327
+ }
328
+ args.onChunk({
329
+ pcm: new Float32Array(0),
330
+ sampleRate: this.sampleRate,
331
+ isFinal: true,
332
+ });
333
+ return { cancelled };
334
+ }
335
+ }
336
+
337
+ /**
338
+ * FFI-backed TTS backend. Forwards each `synthesize()` call through the
339
+ * fused `libelizainference` ABI declared in
340
+ * `packages/app-core/scripts/omnivoice-fuse/ffi.h`. The library handle
341
+ * + a per-engine context pointer are held by the bridge and passed in
342
+ * at construction so this backend stays a thin adapter.
343
+ *
344
+ * Until the real fused build ships, the binding is exercised against
345
+ * the compatibility C library at `scripts/omnivoice-fuse/ffi-stub.c`, which returns
346
+ * `ELIZA_ERR_NOT_IMPLEMENTED` for `tts_synthesize` — the binding then
347
+ * raises `VoiceLifecycleError({code:"kernel-missing"})`. The adapter
348
+ * re-wraps that as `VoiceStartupError("missing-fused-build", ...)` so
349
+ * the engine layer's startup-error taxonomy stays unified. No silent
350
+ * fallback (AGENTS.md §3 + §9).
351
+ */
352
+ export class FfiOmniVoiceBackend
353
+ implements OmniVoiceBackend, StreamingTtsBackend
354
+ {
355
+ readonly id = "ffi" as const;
356
+ private readonly ffi: ElizaInferenceFfi;
357
+ private readonly getContext: () => ElizaInferenceContextHandle;
358
+ private readonly sampleRate: number;
359
+ private readonly maxSecondsPerPhrase: number;
360
+
361
+ constructor(args: {
362
+ ffi: ElizaInferenceFfi;
363
+ ctx?: ElizaInferenceContextHandle;
364
+ getContext?: () => ElizaInferenceContextHandle;
365
+ sampleRate?: number;
366
+ maxSecondsPerPhrase?: number;
367
+ }) {
368
+ this.ffi = args.ffi;
369
+ this.getContext =
370
+ args.getContext ??
371
+ (() => {
372
+ if (args.ctx === undefined) {
373
+ throw new VoiceStartupError(
374
+ "missing-fused-build",
375
+ "[voice] FFI backend has no context provider",
376
+ );
377
+ }
378
+ return args.ctx;
379
+ });
380
+ this.sampleRate = args.sampleRate ?? SAMPLE_RATE_DEFAULT;
381
+ this.maxSecondsPerPhrase = args.maxSecondsPerPhrase ?? 6;
382
+ }
383
+
384
+ /** True when the loaded `libelizainference` advertises streaming TTS. */
385
+ supportsStreamingTts(): boolean {
386
+ return this.ffi.ttsStreamSupported();
387
+ }
388
+
389
+ /**
390
+ * One-shot synthesis returning the whole phrase as an `AudioChunk`.
391
+ * When the loaded build advertises streaming TTS this routes through
392
+ * `eliza_inference_tts_synthesize_stream` and concatenates the
393
+ * delivered chunks (so the chunk-aware native path is exercised even
394
+ * for whole-phrase callers); otherwise it uses the batch
395
+ * `eliza_inference_tts_synthesize` symbol. `cancelSignal` is honoured
396
+ * at chunk boundaries — a cancelled stream returns whatever was
397
+ * synthesized so far.
398
+ */
399
+ async synthesize(args: {
400
+ phrase: Phrase;
401
+ preset: SpeakerPreset;
402
+ cancelSignal: { cancelled: boolean };
403
+ onKernelTick?: () => void;
404
+ }): Promise<AudioChunk> {
405
+ args.onKernelTick?.();
406
+ const ctx = this.getContext();
407
+ if (this.ffi.ttsStreamSupported()) {
408
+ const parts: Float32Array[] = [];
409
+ let total = 0;
410
+ this.ffi.ttsSynthesizeStream({
411
+ ctx,
412
+ text: args.phrase.text,
413
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
414
+ onChunk: ({ pcm, isFinal }) => {
415
+ args.onKernelTick?.();
416
+ if (!isFinal && pcm.length > 0) {
417
+ parts.push(pcm);
418
+ total += pcm.length;
419
+ }
420
+ return args.cancelSignal.cancelled === true;
421
+ },
422
+ });
423
+ const merged = new Float32Array(total);
424
+ let off = 0;
425
+ for (const part of parts) {
426
+ merged.set(part, off);
427
+ off += part.length;
428
+ }
429
+ return {
430
+ phraseId: args.phrase.id,
431
+ fromIndex: args.phrase.fromIndex,
432
+ toIndex: args.phrase.toIndex,
433
+ pcm: merged,
434
+ sampleRate: this.sampleRate,
435
+ };
436
+ }
437
+ const out = new Float32Array(this.sampleRate * this.maxSecondsPerPhrase);
438
+ const samples = this.ffi.ttsSynthesize({
439
+ ctx,
440
+ text: args.phrase.text,
441
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
442
+ out,
443
+ });
444
+ return {
445
+ phraseId: args.phrase.id,
446
+ fromIndex: args.phrase.fromIndex,
447
+ toIndex: args.phrase.toIndex,
448
+ pcm: out.subarray(0, samples),
449
+ sampleRate: this.sampleRate,
450
+ };
451
+ }
452
+
453
+ /**
454
+ * Streaming synthesis: forwards to `eliza_inference_tts_synthesize_stream`
455
+ * when the build advertises a streaming decoder. When it does NOT
456
+ * (`tts_stream_supported() == 0`), this still satisfies the seam — but
457
+ * with exactly one body chunk + one final tail (the batch synthesis
458
+ * result), so the caller never mistakes a non-streaming build for a
459
+ * streaming one (no fallback sludge — the chunk count is the honest
460
+ * signal). The native side checks `ctx->tts_cancel` (set via
461
+ * `eliza_inference_cancel_tts`) on top of the `onChunk` return value.
462
+ * A non-streaming build cannot be interrupted while the native batch
463
+ * forward pass is inside `ttsSynthesize`; it only observes cancellation
464
+ * before emitting the body chunk. Barge-in-critical product paths should
465
+ * require `supportsStreamingTts()`.
466
+ */
467
+ async synthesizeStream(args: {
468
+ phrase: Phrase;
469
+ preset: SpeakerPreset;
470
+ cancelSignal: { cancelled: boolean };
471
+ onChunk: (chunk: TtsPcmChunk) => boolean | undefined;
472
+ onKernelTick?: () => void;
473
+ }): Promise<{ cancelled: boolean }> {
474
+ const ctx = this.getContext();
475
+ if (this.ffi.ttsStreamSupported()) {
476
+ const { cancelled } = this.ffi.ttsSynthesizeStream({
477
+ ctx,
478
+ text: args.phrase.text,
479
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
480
+ onChunk: ({ pcm, isFinal }) => {
481
+ args.onKernelTick?.();
482
+ if (args.cancelSignal.cancelled) return true;
483
+ const want = args.onChunk({
484
+ pcm,
485
+ sampleRate: this.sampleRate,
486
+ isFinal,
487
+ });
488
+ // Re-read the (mutable) cancel flag — the chunk callback or a
489
+ // concurrent barge-in may have flipped it.
490
+ return want === true || args.cancelSignal.cancelled;
491
+ },
492
+ });
493
+ return { cancelled };
494
+ }
495
+ // Non-streaming build: one batch forward pass, surfaced as a single
496
+ // body chunk + final tail.
497
+ args.onKernelTick?.();
498
+ const out = new Float32Array(this.sampleRate * this.maxSecondsPerPhrase);
499
+ const samples = this.ffi.ttsSynthesize({
500
+ ctx,
501
+ text: args.phrase.text,
502
+ speakerPresetId: ffiSpeakerPresetId(args.preset),
503
+ out,
504
+ });
505
+ let cancelled = args.cancelSignal.cancelled === true;
506
+ if (!cancelled && samples > 0) {
507
+ const want = args.onChunk({
508
+ pcm: out.subarray(0, samples),
509
+ sampleRate: this.sampleRate,
510
+ isFinal: false,
511
+ });
512
+ cancelled = want === true || args.cancelSignal.cancelled === true;
513
+ }
514
+ args.onChunk({
515
+ pcm: new Float32Array(0),
516
+ sampleRate: this.sampleRate,
517
+ isFinal: true,
518
+ });
519
+ return { cancelled };
520
+ }
521
+
522
+ /** Hard-cancel any in-flight TTS forward pass on this backend's context. */
523
+ cancelTts(): void {
524
+ this.ffi.cancelTts(this.getContext());
525
+ }
526
+
527
+ /**
528
+ * Batch transcription. One-shot callers should use the fused batch ABI
529
+ * directly so the native side receives the original sample-rate metadata
530
+ * and can apply its own audio preprocessing. Live mic streaming remains
531
+ * available through `EngineVoiceBridge.createStreamingTranscriber()`.
532
+ */
533
+ async transcribe(args: TranscriptionAudio): Promise<string> {
534
+ return this.ffi.asrTranscribe({
535
+ ctx: this.getContext(),
536
+ pcm: args.pcm,
537
+ sampleRateHz: args.sampleRate,
538
+ });
539
+ }
540
+
541
+ /** Transcribe + per-word timings when the fused build is ABI v12+; otherwise
542
+ * the text with empty `words` (the caller degrades to segment highlight). */
543
+ async transcribeTimed(
544
+ args: TranscriptionAudio,
545
+ ): Promise<{ text: string; words: AsrWordTiming[] }> {
546
+ if (this.ffi.timedAsrSupported()) {
547
+ const res = this.ffi.asrTranscribeTimed({
548
+ ctx: this.getContext(),
549
+ pcm: args.pcm,
550
+ sampleRateHz: args.sampleRate,
551
+ });
552
+ return { text: res.text.trim(), words: res.words };
553
+ }
554
+ logger.debug(
555
+ "[FfiOmniVoiceBackend] timedAsrSupported()===false on the active fused build — per-word timings dropped, transcript player degrades to segment-level highlight",
556
+ );
557
+ return { text: (await this.transcribe(args)).trim(), words: [] };
558
+ }
559
+ }
560
+
561
+ export interface EngineVoiceBridgeOptions {
562
+ /**
563
+ * Bundle root on disk. Must contain `cache/voice-preset-default.bin`
564
+ * and the FFI library (`lib/libelizainference.{dylib,so}`) when
565
+ * `useFfiBackend === true`.
566
+ */
567
+ bundleRoot: string;
568
+ /**
569
+ * When true, use `FfiOmniVoiceBackend`. When false, use the deterministic test backend
570
+ * only for lifecycle/unit tests; live sessions and direct synthesis reject
571
+ * the deterministic test backend before user-visible audio can be emitted.
572
+ */
573
+ useFfiBackend: boolean;
574
+ /** Override sample rate. Defaults to 24 kHz. */
575
+ sampleRate?: number;
576
+ /** Override ring buffer capacity (samples). Defaults to 4 s @ 24 kHz. */
577
+ ringBufferCapacity?: number;
578
+ /** Phrase chunker `maxTokensPerPhrase` (no-punctuation run-on cap). Defaults to
579
+ * `ELIZA_VOICE_MAX_TOKENS_PER_PHRASE` or 8 (one MTP draft round). */
580
+ maxTokensPerPhrase?: number;
581
+ /** Max concurrent TTS phrase dispatches. Defaults to env or scheduler default. */
582
+ maxInFlightPhrases?: number;
583
+ /**
584
+ * Pre-warmed phrase cache entries. Per AGENTS.md §4, a precomputed
585
+ * phrase cache for common assistant utterances is mandatory for the
586
+ * first-byte-latency win. Empty by default — callers wire actual
587
+ * entries from the bundle when available.
588
+ */
589
+ prewarmedPhrases?: ReadonlyArray<CachedPhraseAudio>;
590
+ /**
591
+ * Optional sink override (e.g. for tests or for routing PCM to a
592
+ * platform-specific audio device). Defaults to the in-memory sink the
593
+ * scheduler creates.
594
+ */
595
+ sink?: AudioSink;
596
+ /** Optional scheduler event listeners (rollback, audio, cancel). */
597
+ events?: SchedulerEvents;
598
+ /**
599
+ * Optional override for the TTS backend. When set, supersedes
600
+ * `useFfiBackend`. Tests use this to inject a controllable backend
601
+ * (e.g. one that holds synthesis open until a deferred resolves) so
602
+ * rollback timing can be observed deterministically.
603
+ */
604
+ backendOverride?: OmniVoiceBackend;
605
+ /**
606
+ * Override only the TTS backend while keeping the fused bundle lifecycle
607
+ * and ASR FFI loaded. Used when a bundle falls back from OmniVoice speech
608
+ * to Kokoro speech but still needs bundled Qwen3-ASR for mic input.
609
+ */
610
+ ttsBackendOverride?: OmniVoiceBackend;
611
+ /** Optional speaker preset paired with `ttsBackendOverride`. */
612
+ speakerPresetOverride?: SpeakerPreset;
613
+ /**
614
+ * Optional shared resource registry. When the bridge is created
615
+ * inside an engine that already owns one (text + voice on the same
616
+ * tokenizer / mmap regions), the engine passes its registry in so
617
+ * voice ref-counts against the same canonical resources. Tests can
618
+ * leave this unset to get a private registry.
619
+ */
620
+ sharedResources?: SharedResourceRegistry;
621
+ /**
622
+ * Optional lifecycle loaders override. Production wires real
623
+ * `madvise`-backed mmap handles via the FFI; tests inject mocks so
624
+ * the disarm path can assert eviction without a real file mapping.
625
+ * When unset, default loaders are derived from the bundle root.
626
+ */
627
+ lifecycleLoaders?: VoiceLifecycleLoaders;
628
+ /**
629
+ * Construct a `KokoroTtsBackend` directly and skip the bundle-root +
630
+ * speaker-preset + FFI checks the fused omnivoice path requires.
631
+ * Kokoro voices are picked by id (`KOKORO_VOICE_PACKS`), so the bundle's
632
+ * per-user speaker preset is not used. Mutually exclusive with
633
+ * `useFfiBackend: true` and `backendOverride`. Lifecycle loaders
634
+ * default to empty lifecycle handles (ORT owns the model memory; nothing to
635
+ * mmap-evict).
636
+ */
637
+ kokoroOnly?: KokoroEngineDiscoveryResult;
638
+ /**
639
+ * Optional pre-loaded fused inference handle for the `kokoroOnly` path. When
640
+ * set, the Kokoro FFI runtime reuses it instead of dlopen-ing a second copy
641
+ * of `libelizainference` (tests inject a stub; production may share the
642
+ * engine's handle).
643
+ */
644
+ kokoroFfi?: ElizaInferenceFfi;
645
+ /**
646
+ * Optional voice-profile store for speaker-attribution. When set, the
647
+ * bridge constructs a `VoiceAttributionPipeline` and runs attribution
648
+ * in parallel with ASR on every turn via `runVoiceTurn`. Callers receive
649
+ * the resolved `VoiceAttributionOutput` via `onAttribution` in the turn
650
+ * events passed to `runVoiceTurn`.
651
+ *
652
+ * When absent, attribution is skipped and the pipeline operates exactly
653
+ * as before (no diarizer / encoder overhead).
654
+ */
655
+ profileStore?: VoiceProfileStore;
656
+ /**
657
+ * W3-9 / F1 — the agent runtime. When supplied, the bridge constructs a
658
+ * `VoiceCancellationCoordinator` and an `OptimisticGenerationPolicy`
659
+ * scoped to this voice session. The coordinator owns one cancellation
660
+ * token per `roomId` and fans abort out to:
661
+ * 1. `runtime.turnControllers.abortTurn(roomId, reason)` — the
662
+ * planner-loop / action handlers / streaming `useModel` see the
663
+ * abort within one tick.
664
+ * 2. The slot-abort callback (`slotAbort`) when the LM slot id is
665
+ * registered with the turn.
666
+ * 3. The TTS hard-stop callback (`ttsStop`), which the bridge wires
667
+ * to its existing `triggerBargeIn()` (audio sink drain + FFI/HTTP
668
+ * synthesis cancel).
669
+ * 4. The standard `AbortSignal` every fetch / `useModel` / FFI call
670
+ * that took `token.signal` honours.
671
+ *
672
+ * The reverse direction (runtime → voice) is wired symmetrically via
673
+ * the coordinator's `runtime.turnControllers.onEvent` subscription.
674
+ *
675
+ * Omit to keep the prior behaviour — the bridge then exposes no
676
+ * coordinator / policy and callers fall back to the legacy
677
+ * `BargeInController` + `triggerBargeIn()` surface.
678
+ *
679
+ * Structural type — `CoordinatorRuntime` is the minimum surface the
680
+ * coordinator needs (`turnControllers.{abortTurn, onEvent}`). Production
681
+ * passes a full `IAgentRuntime`; tests can pass a fake matching the
682
+ * structural shape.
683
+ */
684
+ runtime?: IAgentRuntime | CoordinatorRuntime;
685
+ /**
686
+ * W3-9 / F1 — optional `OptimisticGenerationPolicy` overrides. When
687
+ * `runtime` is set and `optimisticPolicyOptions` is omitted, the bridge
688
+ * constructs a default policy gated on the resolved power source
689
+ * (plugged-in / battery / unknown) and the canonical EOT threshold.
690
+ */
691
+ optimisticPolicyOptions?: OptimisticPolicyOptions;
692
+ /**
693
+ * W3-9 / F1 — optional LM slot-abort callback for the cancellation
694
+ * coordinator. Production wires this to `MtpLlamaServer.abortSlot`
695
+ * once a slot id is known per turn. The bridge passes this directly
696
+ * into the coordinator; the bridge itself does not own slot ids.
697
+ *
698
+ * Has no effect when `runtime` is unset (no coordinator is constructed).
699
+ */
700
+ slotAbort?: (slotId: number, reason: VoiceCancellationReason) => void;
701
+ /**
702
+ * Live speaker-attribution gating. When set alongside a `profileStore` AND
703
+ * a full `runtime` (with `emitEvent`), `runVoiceTurn` automatically:
704
+ * 1. emits `VOICE_TURN_OBSERVED` for every attributed turn, and
705
+ * 2. folds the diarization decision into the turn's `voiceTurnSignal`
706
+ * (stamped onto `output.turn.metadata`) so the server gate
707
+ * `core.voice_turn_signal` can suppress confident-bystander cross-talk.
708
+ *
709
+ * `knownSpeakerEntityIds` / `ownerEntityId` may be functions so the caller
710
+ * can resolve the enrolled-speaker set lazily per turn (the household roster
711
+ * changes as people are named). When omitted, attribution still emits
712
+ * `VOICE_TURN_OBSERVED` and produces a fail-open signal (no bystander
713
+ * suppression — every attribution is treated as potentially addressed to us).
714
+ */
715
+ liveAttribution?: LiveAttributionConfig;
716
+ }
717
+
718
+ /** Gating inputs for the automatic live-attribution → voiceTurnSignal seam. */
719
+ export interface LiveAttributionConfig {
720
+ /** Owner / primary-enrolled entity id (always allowed to speak). */
721
+ ownerEntityId?: string | (() => string | null | undefined);
722
+ /** Entity ids the agent answers without a wake word (owner + enrolled). */
723
+ knownSpeakerEntityIds?:
724
+ | readonly string[]
725
+ | (() => readonly string[] | undefined);
726
+ /** True when a wake word fired within the recent listen window. */
727
+ wakeWordActive?: boolean | (() => boolean);
728
+ }
729
+
730
+ export function createKokoroTtsBackend(
731
+ kokoro: KokoroEngineDiscoveryResult,
732
+ opts: { bundleRoot?: string; ffi?: ElizaInferenceFfi } = {},
733
+ ): KokoroTtsBackend {
734
+ // In-process FFI is the sole Kokoro synthesis path on every platform — it
735
+ // runs inside the fused libelizainference handle, the only path that ships
736
+ // on iOS / Google Play (no local TCP socket). The legacy HTTP `fork`
737
+ // (llama-server /v1/audio/speech) runtime was removed. An already-loaded
738
+ // fused handle may be injected (`opts.ffi`) so Kokoro reuses it instead of
739
+ // dlopen-ing a second copy of the lib.
740
+ const decision = pickKokoroRuntimeBackend({
741
+ defaultBackend: "ffi",
742
+ ffi: {
743
+ layout: kokoro.layout,
744
+ bundleRoot: opts.bundleRoot,
745
+ ...(opts.ffi ? { ffi: opts.ffi } : {}),
746
+ },
747
+ });
748
+ logger.info(
749
+ `[voice/kokoro] runtime backend=${decision.backend} reason="${decision.reason}"`,
750
+ );
751
+ return new KokoroTtsBackend({
752
+ layout: kokoro.layout,
753
+ runtime: decision.runtime,
754
+ defaultVoiceId: kokoro.defaultVoiceId,
755
+ });
756
+ }
757
+
758
+ export function createKokoroSpeakerPreset(
759
+ kokoro: KokoroEngineDiscoveryResult,
760
+ ): SpeakerPreset {
761
+ return {
762
+ voiceId: kokoro.defaultVoiceId,
763
+ embedding: new Float32Array(0),
764
+ bytes: new Uint8Array(0),
765
+ };
766
+ }
767
+
768
+ /**
769
+ * Per-turn events that include the optional attribution result alongside
770
+ * the existing `VoicePipelineEvents`. The attribution runs in parallel
771
+ * with ASR; it resolves some time after `onAsrComplete` and before
772
+ * `onComplete`.
773
+ */
774
+ export interface VoiceTurnEvents extends VoicePipelineEvents {
775
+ /**
776
+ * Called once per turn when the `VoiceAttributionPipeline` resolves
777
+ * (diarizer + encoder + profile-store match). Only fired when the
778
+ * bridge was constructed with a `profileStore`. May arrive after
779
+ * `onAsrComplete` but before `onComplete`. Fire-and-forget from the
780
+ * bridge's perspective — callers attach the metadata to the turn's
781
+ * transcript asynchronously.
782
+ */
783
+ onAttribution?(output: VoiceAttributionOutput): void;
784
+ }
785
+
786
+ /**
787
+ * Internal helper: construct the W3-9 cancellation coordinator + the
788
+ * optimistic-generation policy for a session, given the bridge options.
789
+ * Returns null/null when no runtime was supplied (the bridge then operates
790
+ * without the W3-9 surface — back-compat for callers that haven't adopted
791
+ * the canonical cancellation token yet).
792
+ *
793
+ * Lives outside the class so both `start()` and `startKokoroOnly()` can
794
+ * share it without duplicating the construction order (the coordinator's
795
+ * `ttsStop` callback closes over the to-be-constructed bridge — we plumb
796
+ * that through `setTtsStop` after the bridge is built).
797
+ */
798
+ interface PendingCancellationWiring {
799
+ coordinator: VoiceCancellationCoordinator;
800
+ policy: OptimisticGenerationPolicy;
801
+ /** Wire the bridge's `triggerBargeIn` as the ttsStop callback. */
802
+ bindTtsStop(stop: () => void): void;
803
+ }
804
+
805
+ /**
806
+ * True when `runtime` is a full `IAgentRuntime` (exposes `emitEvent`) rather
807
+ * than the structural `CoordinatorRuntime` a test may pass. Only an
808
+ * event-capable runtime can drive the automatic `VOICE_TURN_OBSERVED` emit.
809
+ */
810
+ function isEventRuntime(
811
+ runtime: IAgentRuntime | CoordinatorRuntime | undefined,
812
+ ): runtime is IAgentRuntime {
813
+ return (
814
+ runtime !== undefined &&
815
+ typeof (runtime as { emitEvent?: unknown }).emitEvent === "function"
816
+ );
817
+ }
818
+
819
+ /**
820
+ * Flatten the (possibly lazy) `LiveAttributionConfig` into the plain options
821
+ * the runtime helper consumes. Resolved per turn so a changing household roster
822
+ * is picked up without re-arming voice.
823
+ */
824
+ function resolveLiveAttributionOptions(cfg: LiveAttributionConfig | null): {
825
+ ownerEntityId?: string | null;
826
+ knownSpeakerEntityIds?: readonly string[];
827
+ wakeWordActive?: boolean;
828
+ } {
829
+ if (!cfg) return {};
830
+ const ownerEntityId =
831
+ typeof cfg.ownerEntityId === "function"
832
+ ? cfg.ownerEntityId()
833
+ : cfg.ownerEntityId;
834
+ const knownSpeakerEntityIds =
835
+ typeof cfg.knownSpeakerEntityIds === "function"
836
+ ? cfg.knownSpeakerEntityIds()
837
+ : cfg.knownSpeakerEntityIds;
838
+ const wakeWordActive =
839
+ typeof cfg.wakeWordActive === "function"
840
+ ? cfg.wakeWordActive()
841
+ : cfg.wakeWordActive;
842
+ return {
843
+ ...(ownerEntityId !== undefined ? { ownerEntityId } : {}),
844
+ ...(knownSpeakerEntityIds !== undefined ? { knownSpeakerEntityIds } : {}),
845
+ ...(wakeWordActive !== undefined ? { wakeWordActive } : {}),
846
+ };
847
+ }
848
+
849
+ function buildCancellationWiring(
850
+ opts: EngineVoiceBridgeOptions,
851
+ ): PendingCancellationWiring | null {
852
+ if (!opts.runtime) return null;
853
+ let ttsStopHandler: (() => void) | null = null;
854
+ const coordinator = new VoiceCancellationCoordinator({
855
+ runtime: opts.runtime,
856
+ ...(opts.slotAbort ? { slotAbort: opts.slotAbort } : {}),
857
+ ttsStop: () => {
858
+ if (ttsStopHandler) {
859
+ ttsStopHandler();
860
+ }
861
+ },
862
+ });
863
+ const policy = new OptimisticGenerationPolicy(
864
+ opts.optimisticPolicyOptions ?? {},
865
+ );
866
+ policy.setPowerSource(resolvePowerSourceState());
867
+ return {
868
+ coordinator,
869
+ policy,
870
+ bindTtsStop(stop) {
871
+ ttsStopHandler = stop;
872
+ },
873
+ };
874
+ }
875
+
876
+ /**
877
+ * Wires the voice scaffold (`VoiceScheduler` + helpers) onto the engine.
878
+ * One bridge per active voice session — created in
879
+ * `LocalInferenceEngine.startVoice()` and disposed when the engine
880
+ * unloads or `stopVoice()` is called.
881
+ */
882
+ export class EngineVoiceBridge {
883
+ readonly scheduler: VoiceScheduler;
884
+ readonly backend: OmniVoiceBackend;
885
+ readonly lifecycle: VoiceLifecycle;
886
+ /** Loaded FFI handle when running against the fused build (else null). */
887
+ readonly ffi: ElizaInferenceFfi | null;
888
+ /** Lazily-created FFI context this bridge owns; destroyed in `dispose()`. */
889
+ private readonly ffiContextRef: FfiContextRef | null;
890
+ readonly asrAvailable: boolean;
891
+ private readonly bundleRoot: string;
892
+ /** The phrase cache the scheduler dispatches against — held so the bridge
893
+ * can answer "is phrase X cached" for the first-audio filler and seed the
894
+ * idle-time auto-prewarm. */
895
+ private readonly phraseCache: PhraseCache;
896
+ /** In-flight fused turn (`runVoiceTurn`), if any — cancelled on barge-in. */
897
+ private activePipeline: VoicePipeline | null = null;
898
+ /**
899
+ * Optional attribution pipeline. Populated when the bridge was created
900
+ * with a `profileStore` option. When present, `runVoiceTurn` fires
901
+ * attribution in parallel with ASR and delivers the result via
902
+ * `VoiceTurnEvents.onAttribution`.
903
+ */
904
+ private readonly attributionPipeline: VoiceAttributionPipeline | null;
905
+ /**
906
+ * Full agent runtime, retained only when `opts.runtime` supports
907
+ * `emitEvent` (i.e. it is a real `IAgentRuntime`, not the structural
908
+ * `CoordinatorRuntime` a test may pass). Used by the automatic
909
+ * live-attribution seam in `runVoiceTurn` to emit `VOICE_TURN_OBSERVED`.
910
+ * Null when no event-capable runtime was supplied.
911
+ */
912
+ private readonly eventRuntime: IAgentRuntime | null;
913
+ /** Gating inputs for the live-attribution → voiceTurnSignal seam. */
914
+ private readonly liveAttribution: LiveAttributionConfig | null;
915
+ /**
916
+ * W3-9 / F1 — voice cancellation coordinator. Populated when the bridge
917
+ * was created with a `runtime` option. Owns one
918
+ * `VoiceCancellationToken` per active `roomId` and fans abort out to
919
+ * the runtime turn controller, the LM slot, the TTS pipeline, and the
920
+ * standard `AbortSignal`. See `cancellation-coordinator.ts` for the
921
+ * full contract.
922
+ */
923
+ private readonly cancellationCoordinator: VoiceCancellationCoordinator | null;
924
+ /**
925
+ * W3-9 / F1 — optimistic-generation policy. Constructed once per
926
+ * session when `runtime` is supplied. Gates the speculative LM prefill
927
+ * at the `firePrefill` site (see `voice-state-machine.ts`). Hot-swappable
928
+ * via `setPowerSource()` / `setOverride()` from Settings or a device-
929
+ * event listener.
930
+ */
931
+ private readonly optimisticGenerationPolicy: OptimisticGenerationPolicy | null;
932
+ /**
933
+ * W3-9 / F1 — per-room `BargeInController` bindings the bridge owns.
934
+ * Holds the unsubscribe handle returned by
935
+ * `coordinator.bindBargeInController` so `dispose()` can tear them down.
936
+ */
937
+ private readonly bargeInBindings = new Map<string, () => void>();
938
+
939
+ private constructor(
940
+ scheduler: VoiceScheduler,
941
+ backend: OmniVoiceBackend,
942
+ bundleRoot: string,
943
+ lifecycle: VoiceLifecycle,
944
+ ffi: ElizaInferenceFfi | null,
945
+ ffiContextRef: FfiContextRef | null,
946
+ asrAvailable: boolean,
947
+ phraseCache: PhraseCache,
948
+ attributionPipeline: VoiceAttributionPipeline | null = null,
949
+ cancellationCoordinator: VoiceCancellationCoordinator | null = null,
950
+ optimisticGenerationPolicy: OptimisticGenerationPolicy | null = null,
951
+ eventRuntime: IAgentRuntime | null = null,
952
+ liveAttribution: LiveAttributionConfig | null = null,
953
+ ) {
954
+ this.scheduler = scheduler;
955
+ this.backend = backend;
956
+ this.bundleRoot = bundleRoot;
957
+ this.lifecycle = lifecycle;
958
+ this.ffi = ffi;
959
+ this.ffiContextRef = ffiContextRef;
960
+ this.asrAvailable = asrAvailable;
961
+ this.phraseCache = phraseCache;
962
+ this.attributionPipeline = attributionPipeline;
963
+ this.cancellationCoordinator = cancellationCoordinator;
964
+ this.optimisticGenerationPolicy = optimisticGenerationPolicy;
965
+ this.eventRuntime = eventRuntime;
966
+ this.liveAttribution = liveAttribution;
967
+ }
968
+
969
+ get ffiCtx(): ElizaInferenceContextHandle | null {
970
+ return this.ffiContextRef?.current ?? null;
971
+ }
972
+
973
+ /**
974
+ * Tear down the FFI context the bridge owns. Idempotent; safe to call
975
+ * multiple times. Callers should `disarm()` first to drop voice
976
+ * resources, then `dispose()` to close the FFI handle.
977
+ */
978
+ dispose(): void {
979
+ // W3-9 / F1 — tear down barge-in bindings + the cancellation
980
+ // coordinator first so any armed turn aborts with reason=external
981
+ // before the FFI context goes away.
982
+ for (const unsub of Array.from(this.bargeInBindings.values())) {
983
+ try {
984
+ unsub();
985
+ } catch {
986
+ // Best-effort teardown.
987
+ }
988
+ }
989
+ this.bargeInBindings.clear();
990
+ if (this.cancellationCoordinator) {
991
+ try {
992
+ this.cancellationCoordinator.dispose();
993
+ } catch {
994
+ // Coordinator dispose must not block FFI teardown.
995
+ }
996
+ }
997
+ if (this.ffi) {
998
+ const ctx = this.ffiContextRef?.current ?? null;
999
+ if (ctx !== null) {
1000
+ this.ffi.destroy(ctx);
1001
+ if (this.ffiContextRef) this.ffiContextRef.current = null;
1002
+ }
1003
+ this.ffi.close();
1004
+ }
1005
+ }
1006
+
1007
+ /**
1008
+ * Start the voice session for a bundle. Validates the bundle layout
1009
+ * up-front (per AGENTS.md §3 + §7 — required artifacts checked before
1010
+ * activation) and throws `VoiceStartupError` for any missing piece.
1011
+ * No partial activation: either the scheduler exists and is wired or
1012
+ * the call throws.
1013
+ */
1014
+ static start(opts: EngineVoiceBridgeOptions): EngineVoiceBridge {
1015
+ if (opts.kokoroOnly) {
1016
+ if (opts.useFfiBackend || opts.backendOverride) {
1017
+ throw new VoiceStartupError(
1018
+ "invalid-options",
1019
+ "[voice] kokoroOnly cannot be combined with useFfiBackend or backendOverride. Caller must pick exactly one backend path.",
1020
+ );
1021
+ }
1022
+ return EngineVoiceBridge.startKokoroOnly(opts);
1023
+ }
1024
+ if (!opts.bundleRoot || !existsSync(opts.bundleRoot)) {
1025
+ throw new VoiceStartupError(
1026
+ "missing-bundle-root",
1027
+ `[voice] Bundle root does not exist: ${opts.bundleRoot}`,
1028
+ );
1029
+ }
1030
+
1031
+ const presetPath = path.join(
1032
+ opts.bundleRoot,
1033
+ DEFAULT_VOICE_PRESET_REL_PATH,
1034
+ );
1035
+ if (!existsSync(presetPath)) {
1036
+ throw new VoiceStartupError(
1037
+ "missing-speaker-preset",
1038
+ `[voice] Bundle is missing required speaker preset at ${presetPath}. The default voice MUST ship as a precomputed embedding (AGENTS.md §4).`,
1039
+ );
1040
+ }
1041
+
1042
+ const sampleRate = opts.sampleRate ?? SAMPLE_RATE_DEFAULT;
1043
+ const presetCache = new SpeakerPresetCache();
1044
+ const { preset, phrases: seedPhrases } = presetCache.loadFromBundle({
1045
+ bundleRoot: opts.bundleRoot,
1046
+ });
1047
+ const schedulerPreset = opts.speakerPresetOverride ?? preset;
1048
+
1049
+ const phraseCache = new PhraseCache();
1050
+ phraseCache.seed(seedPhrases);
1051
+ for (const entry of opts.prewarmedPhrases ?? []) {
1052
+ phraseCache.put(entry);
1053
+ }
1054
+
1055
+ // FFI binding + per-bridge context. When the bridge runs against
1056
+ // the real fused build, the same `ffi`/`ctx` pair is shared by:
1057
+ // - the TTS backend (`FfiOmniVoiceBackend.synthesize`),
1058
+ // - the lifecycle loaders (`MmapRegionHandle.evictPages` calls
1059
+ // `ffi.mmapEvict(ctx, "tts" | "asr")`).
1060
+ // Tests can opt out by either passing `lifecycleLoaders` (mocks
1061
+ // `evictPages`) or `backendOverride` (mocks the backend) or
1062
+ // setting `useFfiBackend: false` (test TTS + empty evict transition).
1063
+ let ffiHandle: ElizaInferenceFfi | null = null;
1064
+ let ffiContextRef: FfiContextRef | null = null;
1065
+ let backend: OmniVoiceBackend;
1066
+ const asrAvailable = bundleHasRegularFile(
1067
+ path.join(opts.bundleRoot, "asr"),
1068
+ );
1069
+ if (opts.backendOverride && opts.ttsBackendOverride) {
1070
+ throw new VoiceStartupError(
1071
+ "invalid-options",
1072
+ "[voice] backendOverride and ttsBackendOverride are mutually exclusive.",
1073
+ );
1074
+ }
1075
+ if (opts.backendOverride && opts.useFfiBackend) {
1076
+ throw new VoiceStartupError(
1077
+ "missing-fused-build",
1078
+ "[voice] backendOverride cannot be combined with useFfiBackend=true. Voice-on production paths must load libelizainference and verify its ABI instead of bypassing the fused runtime.",
1079
+ );
1080
+ }
1081
+ if (opts.backendOverride) {
1082
+ backend = opts.backendOverride;
1083
+ } else if (opts.useFfiBackend) {
1084
+ const libPath = locateBundleLibrary(opts.bundleRoot);
1085
+ if (!existsSync(libPath)) {
1086
+ throw new VoiceStartupError(
1087
+ "missing-ffi",
1088
+ `[voice] Fused omnivoice library not found under ${path.join(opts.bundleRoot, "lib")} (tried ${libraryFilenames().join(", ")}). Build via packages/app-core/scripts/build-llama-cpp-mtp.mjs (omnivoice-fuse target).`,
1089
+ );
1090
+ }
1091
+ ffiHandle = loadElizaInferenceFfi(libPath);
1092
+ const contextRef: FfiContextRef = {
1093
+ current: null,
1094
+ ensure: () => {
1095
+ if (!ffiHandle) {
1096
+ throw new VoiceStartupError(
1097
+ "missing-ffi",
1098
+ "[voice] FFI context requested without a loaded libelizainference handle",
1099
+ );
1100
+ }
1101
+ if (contextRef.current === null) {
1102
+ contextRef.current = ffiHandle.create(opts.bundleRoot);
1103
+ }
1104
+ return contextRef.current;
1105
+ },
1106
+ };
1107
+ ffiContextRef = contextRef;
1108
+ backend =
1109
+ opts.ttsBackendOverride ??
1110
+ new FfiOmniVoiceBackend({
1111
+ ffi: ffiHandle,
1112
+ getContext: contextRef.ensure,
1113
+ sampleRate,
1114
+ });
1115
+ } else {
1116
+ backend = opts.ttsBackendOverride ?? new StubOmniVoiceBackend(sampleRate);
1117
+ }
1118
+
1119
+ const config: SchedulerConfig = {
1120
+ chunkerConfig: {
1121
+ maxTokensPerPhrase:
1122
+ opts.maxTokensPerPhrase ??
1123
+ readPositiveIntEnv("ELIZA_VOICE_MAX_TOKENS_PER_PHRASE") ??
1124
+ PHRASE_MAX_TOKENS_DEFAULT,
1125
+ },
1126
+ preset: schedulerPreset,
1127
+ ringBufferCapacity:
1128
+ opts.ringBufferCapacity ?? RING_BUFFER_CAPACITY_DEFAULT,
1129
+ sampleRate,
1130
+ maxInFlightPhrases:
1131
+ opts.maxInFlightPhrases ??
1132
+ readPositiveIntEnv("ELIZA_VOICE_MAX_IN_FLIGHT_PHRASES"),
1133
+ };
1134
+
1135
+ const sinkOverride = opts.sink;
1136
+ const scheduler = new VoiceScheduler(
1137
+ config,
1138
+ sinkOverride
1139
+ ? { backend, sink: sinkOverride, phraseCache }
1140
+ : { backend, phraseCache },
1141
+ opts.events ?? {},
1142
+ );
1143
+
1144
+ // Wire the voice lifecycle. The lifecycle starts in `voice-off` —
1145
+ // heavy resources (TTS + ASR mmap regions) are loaded only when
1146
+ // `arm()` is called. The default loaders derive an mmap-style
1147
+ // handle from the bundle's `tts/` and `asr/` directories so that
1148
+ // production paths get real eviction calls; tests inject
1149
+ // `lifecycleLoaders` to assert the disarm path.
1150
+ const registry = opts.sharedResources ?? new SharedResourceRegistry();
1151
+ const loaders =
1152
+ opts.lifecycleLoaders ??
1153
+ defaultLifecycleLoaders(opts.bundleRoot, ffiHandle, ffiContextRef);
1154
+ const lifecycle = new VoiceLifecycle({ registry, loaders });
1155
+
1156
+ // Wire speaker-attribution when a profile store is provided. The
1157
+ // attribution pipeline wraps the fused encoder + diarizer + profile-store.
1158
+ // Both run through the ONE fused `libelizainference` handle via its
1159
+ // `eliza_inference_speaker_*` / `_diariz_*` ABI — there is no standalone
1160
+ // `libvoice_classifier` runtime.
1161
+ //
1162
+ // Fail-fast at ARM time: the fused speaker ABI is probed synchronously
1163
+ // here (`FusedSpeakerEncoder.isSupported`). When the build does not
1164
+ // advertise it, this throws `VoiceStartupError` rather than silently
1165
+ // degrading attribution to "unknown speaker" on the first turn. The
1166
+ // native session `load()` runs lazily on first encode/diarize, but the
1167
+ // capability is decided up front.
1168
+ let attributionPipeline: VoiceAttributionPipeline | null = null;
1169
+ if (opts.profileStore) {
1170
+ const fusedFfi = ffiHandle;
1171
+ const fusedCtx = ffiContextRef;
1172
+ if (!fusedFfi || !fusedCtx) {
1173
+ throw new VoiceStartupError(
1174
+ "missing-fused-build",
1175
+ "[voice] Speaker-attribution requires the fused libelizainference handle (useFfiBackend). No standalone speaker runtime exists.",
1176
+ );
1177
+ }
1178
+ if (!FusedSpeakerEncoder.isSupported(fusedFfi)) {
1179
+ throw new VoiceStartupError(
1180
+ "missing-fused-build",
1181
+ "[voice] The loaded libelizainference build lacks the speaker ABI (eliza_inference_speaker_supported() == 0). Rebuild with the WeSpeaker forward graph linked in (eliza_inference_speaker_* symbols).",
1182
+ );
1183
+ }
1184
+ // Fused encoder: probe passed above; the native session opens lazily
1185
+ // on first encode() so voice-off does not keep the model resident.
1186
+ let resolvedEncoder: SpeakerEncoder | null = null;
1187
+ let encoderLoadError: Error | null = null;
1188
+ const lazyEncoder: SpeakerEncoder = {
1189
+ embeddingDim: SPEAKER_GGML_EMBEDDING_DIM,
1190
+ sampleRate: SPEAKER_GGML_SAMPLE_RATE,
1191
+ async encode(pcm: Float32Array): Promise<Float32Array> {
1192
+ if (encoderLoadError) throw encoderLoadError;
1193
+ if (!resolvedEncoder) {
1194
+ try {
1195
+ resolvedEncoder = await FusedSpeakerEncoder.load({
1196
+ ffi: fusedFfi,
1197
+ ctx: () => fusedCtx.ensure(),
1198
+ });
1199
+ } catch (err) {
1200
+ encoderLoadError =
1201
+ err instanceof Error ? err : new Error(String(err));
1202
+ throw encoderLoadError;
1203
+ }
1204
+ }
1205
+ return resolvedEncoder.encode(pcm);
1206
+ },
1207
+ async dispose(): Promise<void> {
1208
+ await resolvedEncoder?.dispose();
1209
+ },
1210
+ };
1211
+ // Fused diarizer (optional). When the build does not advertise the
1212
+ // diarizer ABI, attribution runs without it — a single-speaker turn
1213
+ // collapses to one segment (the attribution-pipeline localSpeakerId=0
1214
+ // path). The diarizer is NOT a fail-fast gate (unlike the encoder):
1215
+ // it refines multi-speaker windows, it is not required to attribute a
1216
+ // single speaker.
1217
+ let lazyDiarizer: Diarizer | undefined;
1218
+ if (FusedDiarizer.isSupported(fusedFfi)) {
1219
+ let resolvedDiarizer: Diarizer | null = null;
1220
+ let diarizerLoadError: Error | null = null;
1221
+ lazyDiarizer = {
1222
+ modelId: PYANNOTE_SEGMENTATION_3_INT8_MODEL_ID,
1223
+ sampleRate: SPEAKER_GGML_SAMPLE_RATE,
1224
+ async diarizeWindow(pcm: Float32Array) {
1225
+ if (diarizerLoadError) throw diarizerLoadError;
1226
+ if (!resolvedDiarizer) {
1227
+ try {
1228
+ resolvedDiarizer = await FusedDiarizer.load({
1229
+ ffi: fusedFfi,
1230
+ ctx: () => fusedCtx.ensure(),
1231
+ });
1232
+ } catch (err) {
1233
+ diarizerLoadError =
1234
+ err instanceof Error ? err : new Error(String(err));
1235
+ throw diarizerLoadError;
1236
+ }
1237
+ }
1238
+ return resolvedDiarizer.diarizeWindow(pcm);
1239
+ },
1240
+ async dispose(): Promise<void> {
1241
+ await resolvedDiarizer?.dispose();
1242
+ },
1243
+ };
1244
+ }
1245
+ attributionPipeline = new VoiceAttributionPipeline({
1246
+ encoder: lazyEncoder,
1247
+ ...(lazyDiarizer ? { diarizer: lazyDiarizer } : {}),
1248
+ profileStore: opts.profileStore,
1249
+ });
1250
+ }
1251
+
1252
+ // W3-9 / F1 — construct the cancellation coordinator + optimistic policy
1253
+ // when a runtime is supplied. The coordinator's ttsStop callback closes
1254
+ // over `bridge.triggerBargeIn()`, which is wired below once the bridge
1255
+ // is constructed.
1256
+ const wiring = buildCancellationWiring(opts);
1257
+
1258
+ const bridge = new EngineVoiceBridge(
1259
+ scheduler,
1260
+ backend,
1261
+ opts.bundleRoot,
1262
+ lifecycle,
1263
+ ffiHandle,
1264
+ ffiContextRef,
1265
+ asrAvailable,
1266
+ phraseCache,
1267
+ attributionPipeline,
1268
+ wiring?.coordinator ?? null,
1269
+ wiring?.policy ?? null,
1270
+ isEventRuntime(opts.runtime) ? opts.runtime : null,
1271
+ opts.liveAttribution ?? null,
1272
+ );
1273
+ if (wiring) wiring.bindTtsStop(() => bridge.triggerBargeIn());
1274
+ return bridge;
1275
+ }
1276
+
1277
+ /**
1278
+ * Kokoro-only path. Skips bundle-root / speaker-preset / FFI checks
1279
+ * (Kokoro picks voices by id against `KOKORO_VOICE_PACKS`) and
1280
+ * synthesizes a minimal `SpeakerPreset` keyed to the discovered voice
1281
+ * id. Defaults lifecycle loaders to empty handles since ORT owns the
1282
+ * model memory. `asrAvailable` is `false`: callers needing ASR
1283
+ * construct `createStreamingTranscriber` directly.
1284
+ */
1285
+ private static startKokoroOnly(
1286
+ opts: EngineVoiceBridgeOptions,
1287
+ ): EngineVoiceBridge {
1288
+ if (!opts.kokoroOnly) {
1289
+ throw new VoiceStartupError(
1290
+ "invalid-options",
1291
+ "[voice] startKokoroOnly called without `kokoroOnly` config — this is an internal error.",
1292
+ );
1293
+ }
1294
+ const kokoro = opts.kokoroOnly;
1295
+ const sampleRate = opts.sampleRate ?? kokoro.layout.sampleRate;
1296
+ const workDir =
1297
+ opts.bundleRoot && existsSync(opts.bundleRoot)
1298
+ ? opts.bundleRoot
1299
+ : localInferenceRoot();
1300
+
1301
+ // Synthesize a minimal preset. Kokoro's `resolveVoice(preset)` looks
1302
+ // up `preset.voiceId` against `KOKORO_VOICE_PACKS`; the embedding +
1303
+ // bytes fields are ignored on this path (voice cloning is OmniVoice-only).
1304
+ const preset = createKokoroSpeakerPreset(kokoro);
1305
+
1306
+ // Anchor the in-process Kokoro FFI ctx at the Eliza-1 bundle root when
1307
+ // one is present; otherwise the runtime anchors at the Kokoro model root.
1308
+ const backend = createKokoroTtsBackend(kokoro, {
1309
+ bundleRoot:
1310
+ opts.bundleRoot && existsSync(opts.bundleRoot)
1311
+ ? opts.bundleRoot
1312
+ : undefined,
1313
+ ...(opts.kokoroFfi ? { ffi: opts.kokoroFfi } : {}),
1314
+ });
1315
+
1316
+ const phraseCache = new PhraseCache();
1317
+ for (const entry of opts.prewarmedPhrases ?? []) {
1318
+ phraseCache.put(entry);
1319
+ }
1320
+
1321
+ const config: SchedulerConfig = {
1322
+ chunkerConfig: {
1323
+ maxTokensPerPhrase:
1324
+ opts.maxTokensPerPhrase ??
1325
+ readPositiveIntEnv("ELIZA_VOICE_MAX_TOKENS_PER_PHRASE") ??
1326
+ PHRASE_MAX_TOKENS_DEFAULT,
1327
+ },
1328
+ preset,
1329
+ ringBufferCapacity:
1330
+ opts.ringBufferCapacity ?? RING_BUFFER_CAPACITY_DEFAULT,
1331
+ sampleRate,
1332
+ maxInFlightPhrases:
1333
+ opts.maxInFlightPhrases ??
1334
+ readPositiveIntEnv("ELIZA_VOICE_MAX_IN_FLIGHT_PHRASES"),
1335
+ };
1336
+
1337
+ const sinkOverride = opts.sink;
1338
+ const scheduler = new VoiceScheduler(
1339
+ config,
1340
+ sinkOverride
1341
+ ? { backend, sink: sinkOverride, phraseCache }
1342
+ : { backend, phraseCache },
1343
+ opts.events ?? {},
1344
+ );
1345
+
1346
+ const registry = opts.sharedResources ?? new SharedResourceRegistry();
1347
+ const loaders = opts.lifecycleLoaders ?? kokoroOnlyLifecycleLoaders();
1348
+ const lifecycle = new VoiceLifecycle({ registry, loaders });
1349
+
1350
+ const wiring = buildCancellationWiring(opts);
1351
+
1352
+ const bridge = new EngineVoiceBridge(
1353
+ scheduler,
1354
+ backend,
1355
+ workDir,
1356
+ lifecycle,
1357
+ null, // no FFI handle on Kokoro-only
1358
+ null, // no FFI context on Kokoro-only
1359
+ false, // ASR is not served from this path
1360
+ phraseCache,
1361
+ null, // no profile store on Kokoro-only
1362
+ wiring?.coordinator ?? null,
1363
+ wiring?.policy ?? null,
1364
+ );
1365
+ if (wiring) wiring.bindTtsStop(() => bridge.triggerBargeIn());
1366
+ return bridge;
1367
+ }
1368
+
1369
+ /**
1370
+ * True when this bridge runs against a TTS backend that produces real
1371
+ * audio — i.e. anything but the `StubOmniVoiceBackend` (which yields
1372
+ * zeros and is tests-only). The prewarm + first-audio-filler paths gate
1373
+ * on this so the cache never holds silence (AGENTS.md §3 — no fake data).
1374
+ */
1375
+ hasRealTtsBackend(): boolean {
1376
+ return !(this.backend instanceof StubOmniVoiceBackend);
1377
+ }
1378
+
1379
+ /**
1380
+ * Lazy-load the TTS mmap region, optional ASR region, and the voice
1381
+ * scheduler nodes via the lifecycle state machine. Idempotent for
1382
+ * repeated calls in `voice-on` (returns the existing armed resources).
1383
+ * Surfaces RAM pressure / mmap-fail / kernel-missing as `VoiceLifecycleError` —
1384
+ * see `lifecycle.ts` for the full error taxonomy.
1385
+ */
1386
+ async arm(): Promise<void> {
1387
+ if (this.lifecycle.current().kind === "voice-on") return;
1388
+ await this.lifecycle.arm();
1389
+ }
1390
+
1391
+ /**
1392
+ * Drain in-flight TTS, settle the scheduler, then disarm the
1393
+ * lifecycle. Disarm calls `evictPages()` (madvise / VirtualUnlock
1394
+ * equivalent) on the TTS + optional ASR mmap regions and releases every
1395
+ * voice-only ref. Speaker preset + phrase cache survive in the
1396
+ * registry as small LRU entries (KB-scale; not worth evicting).
1397
+ */
1398
+ async disarm(): Promise<void> {
1399
+ if (this.lifecycle.current().kind !== "voice-on") return;
1400
+ await this.settle();
1401
+ await this.lifecycle.disarm();
1402
+ }
1403
+
1404
+ /**
1405
+ * Forward an accepted text token from the verifier into the scheduler.
1406
+ * Tokens that fill a phrase trigger TTS dispatch on the same scheduler
1407
+ * tick (AGENTS.md §4 — no buffering past phrase boundaries).
1408
+ */
1409
+ async pushAcceptedToken(
1410
+ token: TextToken,
1411
+ acceptedAt = Date.now(),
1412
+ ): Promise<void> {
1413
+ await this.scheduler.accept(token, acceptedAt);
1414
+ }
1415
+
1416
+ /**
1417
+ * MTP rejection → rollback queue. The scheduler cancels any
1418
+ * in-flight TTS forward pass for phrases that overlap the rejected
1419
+ * token range and emits an `onRollback` event for observability.
1420
+ * Already-played audio cannot be unplayed; the chunker is sized so
1421
+ * rollback is rare and cheap.
1422
+ */
1423
+ async pushRejectedRange(range: RejectedTokenRange): Promise<void> {
1424
+ await this.scheduler.reject(range);
1425
+ }
1426
+
1427
+ /**
1428
+ * Voice activity detected on the mic input → cancel everything.
1429
+ * Drains the ring buffer immediately, flushes the chunker queue, and
1430
+ * marks every in-flight cancel signal so synthesise loops exit at the
1431
+ * next kernel boundary (AGENTS.md §4 — barge-in cancellation MUST be
1432
+ * within one kernel tick).
1433
+ */
1434
+ triggerBargeIn(): void {
1435
+ // Cancel the text side first (stop ASR / drafter / verifier at the next
1436
+ // kernel boundary), then the audio side (ring-buffer drain + chunker
1437
+ // flush + in-flight TTS cancel). The pipeline also wires its own
1438
+ // barge-in listener onto the scheduler, so `onMicActive()` alone would
1439
+ // suffice — calling `cancel()` first just stops the next HTTP body
1440
+ // sooner.
1441
+ this.activePipeline?.cancel();
1442
+ this.scheduler.bargeIn.onMicActive();
1443
+ }
1444
+
1445
+ /**
1446
+ * W3-9 / F1 — the canonical voice cancellation coordinator for this
1447
+ * session, or `null` when the bridge was constructed without a
1448
+ * `runtime` option. Callers (turn controller, mic VAD source, UI cancel
1449
+ * route) use this to arm per-turn tokens, fire `bargeIn(roomId)` on
1450
+ * VAD speech-start, fire `revokeEot(roomId)` when the turn detector
1451
+ * revokes a tentative EOT, etc. See
1452
+ * `plugins/plugin-local-inference/docs/voice-cancellation-contract.md`.
1453
+ */
1454
+ cancellationCoordinatorOrNull(): VoiceCancellationCoordinator | null {
1455
+ return this.cancellationCoordinator;
1456
+ }
1457
+
1458
+ /**
1459
+ * W3-9 / F1 — the optimistic-generation policy for this session, or
1460
+ * `null` when the bridge was constructed without a `runtime` option.
1461
+ * The bridge primes it with the resolved power source at construction
1462
+ * time; callers can mutate it via `setPowerSource()` / `setOverride()`
1463
+ * to respond to Settings toggles or battery-state events.
1464
+ */
1465
+ optimisticPolicyOrNull(): OptimisticGenerationPolicy | null {
1466
+ return this.optimisticGenerationPolicy;
1467
+ }
1468
+
1469
+ /**
1470
+ * W3-9 / F1 — bind the scheduler's `BargeInController` into the
1471
+ * cancellation coordinator for `roomId`. Subsequent
1472
+ * `BargeInController.hardStop()` calls (typically fired by the
1473
+ * ASR-confirmed barge-in words ladder) translate into
1474
+ * `coordinator.bargeIn(roomId)` so the canonical token (and every
1475
+ * downstream consumer: runtime turn abort, LM slot abort, TTS stop,
1476
+ * AbortSignal) sees the abort.
1477
+ *
1478
+ * Idempotent per `roomId` — repeated calls for the same room return
1479
+ * the same unsubscribe handle (the prior binding is torn down first).
1480
+ *
1481
+ * When the bridge was constructed without a `runtime` option, this returns
1482
+ * an empty unsubscribe. Callers should still call it
1483
+ * unconditionally — back-compat for the legacy path is automatic.
1484
+ */
1485
+ bindBargeInControllerForRoom(roomId: string): () => void {
1486
+ if (!this.cancellationCoordinator) {
1487
+ return () => undefined;
1488
+ }
1489
+ const existing = this.bargeInBindings.get(roomId);
1490
+ if (existing) existing();
1491
+ const unsub = this.cancellationCoordinator.bindBargeInController(
1492
+ roomId,
1493
+ this.scheduler.bargeIn,
1494
+ );
1495
+ this.bargeInBindings.set(roomId, unsub);
1496
+ return () => {
1497
+ unsub();
1498
+ if (this.bargeInBindings.get(roomId) === unsub) {
1499
+ this.bargeInBindings.delete(roomId);
1500
+ }
1501
+ };
1502
+ }
1503
+
1504
+ /**
1505
+ * Drain pending phrase data and wait for in-flight TTS to settle.
1506
+ * Used at the end of a turn so callers can synchronise on a quiescent
1507
+ * scheduler before they tear it down.
1508
+ */
1509
+ async settle(): Promise<void> {
1510
+ await this.scheduler.flushPending();
1511
+ await this.scheduler.waitIdle();
1512
+ }
1513
+
1514
+ async synthesizeTextToWav(
1515
+ text: string,
1516
+ signal?: AbortSignal,
1517
+ ): Promise<Uint8Array> {
1518
+ this.assertVoiceOn("synthesize speech");
1519
+ if (!this.hasRealTtsBackend()) {
1520
+ throw new VoiceStartupError(
1521
+ "missing-fused-build",
1522
+ "[voice] Direct speech synthesis requires a fused OmniVoice backend. The deterministic test backend is only allowed in scheduler/unit tests.",
1523
+ );
1524
+ }
1525
+ const chunk = await this.scheduler.synthesizeText(text, signal);
1526
+ return encodeMonoPcm16Wav(chunk.pcm, chunk.sampleRate);
1527
+ }
1528
+
1529
+ /**
1530
+ * The streaming-TTS seam W9's scheduler drives: returns the active
1531
+ * backend as a `StreamingTtsBackend` (`FfiOmniVoiceBackend` against the
1532
+ * fused build, `StubOmniVoiceBackend` for tests). The scheduler calls
1533
+ * `synthesizeStream(...)` for each phrase and writes the delivered PCM
1534
+ * segments into its `PcmRingBuffer` on the same scheduler tick. Returns
1535
+ * null when an injected `backendOverride` does not implement the seam.
1536
+ */
1537
+ streamingTtsBackend(): StreamingTtsBackend | null {
1538
+ return isStreamingTtsBackend(this.backend) ? this.backend : null;
1539
+ }
1540
+
1541
+ /**
1542
+ * True when the loaded fused `libelizainference` runs the MTP
1543
+ * speculative loop in-process and can emit native accept/reject
1544
+ * verifier events. When true, callers (W9's turn controller /
1545
+ * `ffi-streaming-backend.ts` wiring) should subscribe via
1546
+ * `subscribeNativeVerifier()` and SKIP the `llama-server` SSE
1547
+ * `{"verifier":{"rejected":[a,b]}}` side-channel — the SSE path stays
1548
+ * only as the non-fused desktop text fallback. False whenever there is
1549
+ * no FFI handle or the build pre-dates the verifier callback.
1550
+ */
1551
+ hasNativeVerifier(): boolean {
1552
+ // ABI v3 exports `eliza_inference_set_verifier_callback`, but the
1553
+ // current generated adapter returns ELIZA_ERR_NOT_IMPLEMENTED until the
1554
+ // native MTP speculative loop is ported into libelizainference. Do
1555
+ // not let callers skip the SSE verifier fallback merely because the
1556
+ // symbol exists.
1557
+ return false;
1558
+ }
1559
+
1560
+ /**
1561
+ * Register the native MTP verifier callback on the fused runtime
1562
+ * and adapt each `NativeVerifierEvent` into the rollback-queue domain:
1563
+ * accepted/corrected token-id ranges become `VerifierStreamEvent`s and
1564
+ * rejected ranges become `RejectedTokenRange`s fed to `pushRejectedRange`.
1565
+ * The returned handle MUST be `close()`d (clears the native callback +
1566
+ * frees the bun:ffi `JSCallback`). Throws if no fused runtime is loaded.
1567
+ *
1568
+ * `onEvent` (optional) also receives the raw `NativeVerifierEvent` for
1569
+ * callers that want the accepted-token stream (W9's phrase-chunker can
1570
+ * commit accepted draft tokens directly off this instead of round-trip
1571
+ * SSE deltas).
1572
+ */
1573
+ subscribeNativeVerifier(onEvent?: (event: NativeVerifierEvent) => void): {
1574
+ close(): void;
1575
+ } {
1576
+ if (!this.ffi) {
1577
+ throw new VoiceStartupError(
1578
+ "missing-ffi",
1579
+ "[voice] subscribeNativeVerifier requires a loaded fused libelizainference handle",
1580
+ );
1581
+ }
1582
+ const ctx = this.ffiContextRef
1583
+ ? this.ffiContextRef.ensure()
1584
+ : (() => {
1585
+ throw new VoiceStartupError(
1586
+ "missing-ffi",
1587
+ "[voice] subscribeNativeVerifier: no FFI context provider",
1588
+ );
1589
+ })();
1590
+ return this.ffi.setVerifierCallback(ctx, (event) => {
1591
+ onEvent?.(event);
1592
+ const rollback = nativeRejectedRangeToRollbackRange(event);
1593
+ if (rollback) {
1594
+ void this.pushRejectedRange(rollback);
1595
+ }
1596
+ });
1597
+ }
1598
+
1599
+ async prewarmPhrases(
1600
+ texts: ReadonlyArray<string>,
1601
+ opts: { concurrency?: number } = {},
1602
+ ): Promise<{ warmed: number; cached: number }> {
1603
+ this.assertVoiceOn("prewarm voice phrases");
1604
+ return this.scheduler.prewarmPhrases(texts, opts);
1605
+ }
1606
+
1607
+ /**
1608
+ * Idle-time auto-prewarm hook: synthesize the canonical phrase-cache seed
1609
+ * (`DEFAULT_PHRASE_CACHE_SEED`) so common openers/acks are cached before
1610
+ * the next turn. The voice bridge / connector calls this when the loop is
1611
+ * idle. No-op (returns `{ warmed: 0, cached: 0 }`) unless a real TTS
1612
+ * backend is present and voice is armed — we never cache the test backend's zeros
1613
+ * (AGENTS.md §3).
1614
+ */
1615
+ async prewarmIdlePhrases(
1616
+ opts: { concurrency?: number } = {},
1617
+ ): Promise<{ warmed: number; cached: number }> {
1618
+ if (!this.hasRealTtsBackend()) return { warmed: 0, cached: 0 };
1619
+ if (this.lifecycle.current().kind !== "voice-on") {
1620
+ return { warmed: 0, cached: 0 };
1621
+ }
1622
+ return this.scheduler.prewarmPhrases(DEFAULT_PHRASE_CACHE_SEED, opts);
1623
+ }
1624
+
1625
+ /**
1626
+ * First-audio filler (AGENTS.md §4 / H4): the instant W1's VAD fires
1627
+ * `speech-start`, play a short cached acknowledgement ("one sec", "okay",
1628
+ * …) into the audio sink to mask first-token latency. W9's turn controller
1629
+ * owns the call site (it gets the `speech-start` event and the cutover to
1630
+ * real `replyText` audio); this method is the seam.
1631
+ *
1632
+ * It only ever plays audio that is *already in the phrase cache* — it does
1633
+ * not synthesize. Returns the filler text that was played, or `null` if no
1634
+ * filler was played (no real TTS backend, voice not armed, or none of the
1635
+ * filler phrases are cached). When real reply audio is ready, W9 cuts over
1636
+ * by writing it through the scheduler as usual (a `triggerBargeIn()` or a
1637
+ * direct `ringBuffer.drain()` truncates any still-playing filler first).
1638
+ */
1639
+ playFirstAudioFiller(): string | null {
1640
+ if (!this.hasRealTtsBackend()) return null;
1641
+ if (this.lifecycle.current().kind !== "voice-on") return null;
1642
+ for (const text of FIRST_AUDIO_FILLERS) {
1643
+ const cached = this.phraseCache.get(text);
1644
+ if (!cached || cached.pcm.length === 0) continue;
1645
+ this.scheduler.ringBuffer.write(cached.pcm);
1646
+ const flushed = this.scheduler.ringBuffer.flushToSink();
1647
+ this.scheduler.markAgentSpeakingForAudio(flushed, cached.sampleRate);
1648
+ return cached.text;
1649
+ }
1650
+ return null;
1651
+ }
1652
+
1653
+ /**
1654
+ * Construct a `StreamingTranscriber` for live ASR — the contract the
1655
+ * voice turn controller (W9) feeds mic frames into and the barge-in
1656
+ * word-confirm gate (W1) listens to. Resolves the adapter chain:
1657
+ * fused `libelizainference` streaming ASR (final path, gated on a
1658
+ * working decoder AND a bundled ASR model) → fused batch ASR over the
1659
+ * same bundled model → `AsrUnavailableError`. The Eliza-1 bridge runs
1660
+ * only the fused path; the whisper.cpp interim fallback has been removed.
1661
+ *
1662
+ * Pass W1's `vad` event stream to gate decoding to active speech
1663
+ * windows. Caller owns the returned transcriber's lifecycle (`dispose()`).
1664
+ */
1665
+ createStreamingTranscriber(opts?: {
1666
+ vad?: VadEventSource;
1667
+ }): StreamingTranscriber {
1668
+ this.assertVoiceOn("create streaming transcriber");
1669
+ const contextRef = this.ffiContextRef;
1670
+ return createStreamingTranscriber({
1671
+ ffi: this.ffi,
1672
+ getContext: contextRef ? () => contextRef.ensure() : undefined,
1673
+ asrBundlePresent: this.asrAvailable,
1674
+ vad: opts?.vad,
1675
+ });
1676
+ }
1677
+
1678
+ /**
1679
+ * Batch transcription: one-shot over a whole PCM buffer. When the active
1680
+ * backend exposes the fused batch ASR ABI, use it directly so the native
1681
+ * side receives the original sample rate and can apply its own resampling.
1682
+ * Otherwise drive a `StreamingTranscriber` (fused streaming ASR →
1683
+ * fused-batch interim) by feeding the buffer as a single frame and
1684
+ * `flush()`ing. Throws `AsrUnavailableError` when no ASR backend is
1685
+ * available — never a silent empty string.
1686
+ */
1687
+ /** Transcribe + per-word timings through the fused ASR (v12). Prefers the
1688
+ * backend's timed path; falls back to the plain transcript with empty
1689
+ * `words` when timing isn't available. */
1690
+ async transcribePcmTimed(
1691
+ args: TranscriptionAudio,
1692
+ signal?: AbortSignal,
1693
+ ): Promise<{ text: string; words: AsrWordTiming[] }> {
1694
+ this.assertVoiceOn("transcribe audio");
1695
+ if (signal?.aborted) {
1696
+ throw signal.reason instanceof Error
1697
+ ? signal.reason
1698
+ : new DOMException("Aborted", "AbortError");
1699
+ }
1700
+ const backendTimed = this.backend as OmniVoiceBackend & {
1701
+ transcribeTimed?: (
1702
+ args: TranscriptionAudio,
1703
+ ) => Promise<{ text: string; words: AsrWordTiming[] }>;
1704
+ };
1705
+ if (typeof backendTimed.transcribeTimed === "function") {
1706
+ const result = await backendTimed.transcribeTimed(args);
1707
+ if (signal?.aborted) {
1708
+ throw signal.reason instanceof Error
1709
+ ? signal.reason
1710
+ : new DOMException("Aborted", "AbortError");
1711
+ }
1712
+ return result;
1713
+ }
1714
+ if (
1715
+ this.ffi &&
1716
+ this.ffiContextRef &&
1717
+ this.asrAvailable &&
1718
+ this.ffi.timedAsrSupported()
1719
+ ) {
1720
+ const pcm =
1721
+ args.sampleRate === ASR_SAMPLE_RATE
1722
+ ? args.pcm
1723
+ : resampleLinear(args.pcm, args.sampleRate, ASR_SAMPLE_RATE);
1724
+ const res = this.ffi.asrTranscribeTimed({
1725
+ ctx: this.ffiContextRef.ensure(),
1726
+ pcm,
1727
+ sampleRateHz: ASR_SAMPLE_RATE,
1728
+ });
1729
+ if (signal?.aborted) {
1730
+ throw signal.reason instanceof Error
1731
+ ? signal.reason
1732
+ : new DOMException("Aborted", "AbortError");
1733
+ }
1734
+ return { text: res.text.trim(), words: res.words };
1735
+ }
1736
+ // No timed path available — degrade to the text-only transcript.
1737
+ logger.debug(
1738
+ "[EngineVoiceBridge] timedAsrSupported()===false on the active fused build — per-word timings dropped, transcript player degrades to segment-level highlight",
1739
+ );
1740
+ return { text: await this.transcribePcm(args, signal), words: [] };
1741
+ }
1742
+
1743
+ async transcribePcm(
1744
+ args: TranscriptionAudio,
1745
+ signal?: AbortSignal,
1746
+ ): Promise<string> {
1747
+ this.assertVoiceOn("transcribe audio");
1748
+ if (signal?.aborted) {
1749
+ throw signal.reason instanceof Error
1750
+ ? signal.reason
1751
+ : new DOMException("Aborted", "AbortError");
1752
+ }
1753
+ const backendBatch = this.backend as OmniVoiceBackend & {
1754
+ transcribe?: (args: TranscriptionAudio) => Promise<string>;
1755
+ };
1756
+ if (typeof backendBatch.transcribe === "function") {
1757
+ const transcript = await backendBatch.transcribe(args);
1758
+ if (signal?.aborted) {
1759
+ throw signal.reason instanceof Error
1760
+ ? signal.reason
1761
+ : new DOMException("Aborted", "AbortError");
1762
+ }
1763
+ return transcript;
1764
+ }
1765
+ if (
1766
+ this.ffi &&
1767
+ this.ffiContextRef &&
1768
+ this.asrAvailable &&
1769
+ typeof this.ffi.asrTranscribe === "function"
1770
+ ) {
1771
+ const pcm =
1772
+ args.sampleRate === ASR_SAMPLE_RATE
1773
+ ? args.pcm
1774
+ : resampleLinear(args.pcm, args.sampleRate, ASR_SAMPLE_RATE);
1775
+ const transcript = this.ffi
1776
+ .asrTranscribe({
1777
+ ctx: this.ffiContextRef.ensure(),
1778
+ pcm,
1779
+ sampleRateHz: ASR_SAMPLE_RATE,
1780
+ })
1781
+ .trim();
1782
+ if (signal?.aborted) {
1783
+ throw signal.reason instanceof Error
1784
+ ? signal.reason
1785
+ : new DOMException("Aborted", "AbortError");
1786
+ }
1787
+ return transcript;
1788
+ }
1789
+ const transcriber = this.createStreamingTranscriber();
1790
+ const abort = () => transcriber.dispose();
1791
+ try {
1792
+ signal?.addEventListener("abort", abort, { once: true });
1793
+ transcriber.feed({
1794
+ pcm: args.pcm,
1795
+ sampleRate: args.sampleRate,
1796
+ timestampMs: 0,
1797
+ });
1798
+ const final = await transcriber.flush();
1799
+ if (signal?.aborted) {
1800
+ throw signal.reason instanceof Error
1801
+ ? signal.reason
1802
+ : new DOMException("Aborted", "AbortError");
1803
+ }
1804
+ return final.partial;
1805
+ } finally {
1806
+ signal?.removeEventListener("abort", abort);
1807
+ transcriber.dispose();
1808
+ }
1809
+ }
1810
+
1811
+ /**
1812
+ * Run one fused mic→speech turn through the overlapped `VoicePipeline`
1813
+ * (AGENTS.md §4): ASR streams; the instant its last token lands the
1814
+ * MTP drafter and the target verifier kick off concurrently, accepted
1815
+ * tokens flow into this bridge's phrase chunker → TTS → ring buffer on
1816
+ * the same tick, rejected draft tails roll back not-yet-spoken audio, and
1817
+ * a mic-VAD barge-in cancels everything at the next kernel boundary.
1818
+ *
1819
+ * The drafter + verifier are wired against the running MTP llama-server
1820
+ * (`textRunner`); the transcriber is the fused ABI's ASR when this bridge
1821
+ * was started with the FFI backend and the bundle ships an `asr/` region.
1822
+ * In voice mode a missing ASR region is a hard `VoiceStartupError` — no
1823
+ * silent cloud fallback (AGENTS.md §3 + §7).
1824
+ *
1825
+ * Resolves with the turn's exit reason. Throws if no turn is wired or one
1826
+ * is already in flight. The created pipeline is held until the turn ends
1827
+ * so `bargeIn()` can cancel it.
1828
+ */
1829
+ async runVoiceTurn(
1830
+ audio: TranscriptionAudio,
1831
+ textRunner: MtpTextRunner,
1832
+ config: VoicePipelineConfig,
1833
+ events?: VoiceTurnEvents,
1834
+ ): Promise<"done" | "token-cap" | "cancelled"> {
1835
+ this.assertVoiceOn("run a voice turn");
1836
+ // If a profileStore was wired, kick off speaker-attribution in parallel
1837
+ // with ASR. The attribution uses the same PCM buffer as the transcriber
1838
+ // but runs through the diarizer + encoder + profile-store independently.
1839
+ // It is fire-and-forget from the pipeline's perspective: the result
1840
+ // arrives via `onAttribution` asynchronously (possibly after onComplete).
1841
+ if (
1842
+ this.attributionPipeline &&
1843
+ (events?.onAttribution || this.eventRuntime)
1844
+ ) {
1845
+ const onAttribution = events?.onAttribution;
1846
+ const attribution = this.attributionPipeline;
1847
+ const eventRuntime = this.eventRuntime;
1848
+ const liveAttribution = this.liveAttribution;
1849
+ const turnId = `turn-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
1850
+ void attribution
1851
+ .attribute({
1852
+ turnId,
1853
+ pcm: audio.pcm,
1854
+ })
1855
+ .then(async (output) => {
1856
+ // Automatic seam: when a full runtime is wired, emit
1857
+ // VOICE_TURN_OBSERVED and fold the speaker decision into the
1858
+ // turn's voiceTurnSignal BEFORE handing the (now-stamped)
1859
+ // output to the caller. Any caller with a profileStore +
1860
+ // runtime gets diarization-driven gating for free.
1861
+ if (eventRuntime) {
1862
+ const { handleLiveVoiceAttribution } = await import(
1863
+ "../../runtime/voice-entity-binding.js"
1864
+ );
1865
+ await handleLiveVoiceAttribution(
1866
+ eventRuntime,
1867
+ output,
1868
+ resolveLiveAttributionOptions(liveAttribution),
1869
+ );
1870
+ }
1871
+ onAttribution?.(output);
1872
+ })
1873
+ .catch((err: unknown) => {
1874
+ // Attribution failures must not crash the turn. Log and continue.
1875
+ logger.warn(
1876
+ {
1877
+ turnId,
1878
+ error: err instanceof Error ? err.message : String(err),
1879
+ },
1880
+ "[voice-bridge] speaker attribution failed",
1881
+ );
1882
+ });
1883
+ }
1884
+ const pipeline = this.buildPipeline(textRunner, config, events);
1885
+ this.activePipeline = pipeline;
1886
+ try {
1887
+ return await pipeline.run(audio);
1888
+ } finally {
1889
+ if (this.activePipeline === pipeline) this.activePipeline = null;
1890
+ }
1891
+ }
1892
+
1893
+ /** Construct the `VoicePipeline` for this bridge (no-run). Exposed for tests. */
1894
+ buildPipeline(
1895
+ textRunner: MtpTextRunner,
1896
+ config: VoicePipelineConfig,
1897
+ events?: VoicePipelineEvents,
1898
+ ): VoicePipeline {
1899
+ const transcriber = this.resolveTranscriber();
1900
+ const deps: VoicePipelineDeps = {
1901
+ scheduler: this.scheduler,
1902
+ transcriber,
1903
+ drafter: new MtpDraftProposer(textRunner),
1904
+ verifier: new MtpTargetVerifier(textRunner),
1905
+ };
1906
+ return new VoicePipeline(deps, config, events);
1907
+ }
1908
+
1909
+ /**
1910
+ * Resolve the pipeline's ASR backend: a live `StreamingTranscriber` —
1911
+ * the fused `eliza_inference_asr_stream_*` decoder when the loaded build
1912
+ * advertises one and the bundle ships an `asr/` region, else the fused
1913
+ * batch ASR adapter. The `VoicePipeline` drives it as a batch
1914
+ * (feed the whole utterance, `flush()`, split the transcript into
1915
+ * tokens). When no ASR backend is available the failure is surfaced as a
1916
+ * `MissingAsrTranscriber` that throws on first use — AGENTS.md §3, no
1917
+ * silent cloud fallback.
1918
+ */
1919
+ private resolveTranscriber(): StreamingTranscriber {
1920
+ const ctxRef = this.ffiContextRef;
1921
+ try {
1922
+ return createStreamingTranscriber({
1923
+ ffi: this.ffi,
1924
+ getContext: ctxRef ? () => ctxRef.ensure() : undefined,
1925
+ asrBundlePresent: this.asrAvailable,
1926
+ });
1927
+ } catch (err) {
1928
+ if (err instanceof AsrUnavailableError) {
1929
+ return new MissingAsrTranscriber(err.message);
1930
+ }
1931
+ throw err;
1932
+ }
1933
+ }
1934
+
1935
+ /** Diagnostic accessor — bundle root the bridge is wired against. */
1936
+ bundlePath(): string {
1937
+ return this.bundleRoot;
1938
+ }
1939
+
1940
+ private assertVoiceOn(action: string): void {
1941
+ const state = this.lifecycle.current();
1942
+ if (state.kind === "voice-on") return;
1943
+ if (state.kind === "voice-error") {
1944
+ throw state.error;
1945
+ }
1946
+ throw new VoiceLifecycleError(
1947
+ "illegal-transition",
1948
+ `[voice] Cannot ${action} while lifecycle is ${state.kind}. Call armVoice() and wait for voice-on first.`,
1949
+ );
1950
+ }
1951
+ }
1952
+
1953
+ export function encodeMonoPcm16Wav(
1954
+ pcm: Float32Array,
1955
+ sampleRate: number,
1956
+ ): Uint8Array {
1957
+ const channels = 1;
1958
+ const bytesPerSample = 2;
1959
+ const dataBytes = pcm.length * bytesPerSample;
1960
+ const out = new Uint8Array(44 + dataBytes);
1961
+ const view = new DataView(out.buffer);
1962
+ writeAscii(out, 0, "RIFF");
1963
+ view.setUint32(4, 36 + dataBytes, true);
1964
+ writeAscii(out, 8, "WAVE");
1965
+ writeAscii(out, 12, "fmt ");
1966
+ view.setUint32(16, 16, true);
1967
+ view.setUint16(20, 1, true);
1968
+ view.setUint16(22, channels, true);
1969
+ view.setUint32(24, sampleRate, true);
1970
+ view.setUint32(28, sampleRate * channels * bytesPerSample, true);
1971
+ view.setUint16(32, channels * bytesPerSample, true);
1972
+ view.setUint16(34, bytesPerSample * 8, true);
1973
+ writeAscii(out, 36, "data");
1974
+ view.setUint32(40, dataBytes, true);
1975
+
1976
+ let offset = 44;
1977
+ for (const sample of pcm) {
1978
+ const clamped = Math.max(-1, Math.min(1, sample));
1979
+ const value = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
1980
+ view.setInt16(offset, Math.round(value), true);
1981
+ offset += bytesPerSample;
1982
+ }
1983
+ return out;
1984
+ }
1985
+
1986
+ export function decodeMonoPcm16Wav(bytes: Uint8Array): TranscriptionAudio {
1987
+ if (bytes.byteLength < 44) {
1988
+ throw new Error("[voice] WAV input is too short to contain a header");
1989
+ }
1990
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
1991
+ if (
1992
+ readAscii(bytes, 0, 4) !== "RIFF" ||
1993
+ readAscii(bytes, 8, 4) !== "WAVE" ||
1994
+ readAscii(bytes, 12, 4) !== "fmt "
1995
+ ) {
1996
+ throw new Error("[voice] Local transcription expects mono PCM16 WAV bytes");
1997
+ }
1998
+ const audioFormat = view.getUint16(20, true);
1999
+ const channels = view.getUint16(22, true);
2000
+ const sampleRate = view.getUint32(24, true);
2001
+ const bitsPerSample = view.getUint16(34, true);
2002
+ if (audioFormat !== 1 || channels !== 1 || bitsPerSample !== 16) {
2003
+ throw new Error(
2004
+ `[voice] Local transcription expects mono PCM16 WAV (format=1 channels=1 bits=16); got format=${audioFormat} channels=${channels} bits=${bitsPerSample}`,
2005
+ );
2006
+ }
2007
+
2008
+ let pos = 36;
2009
+ while (pos + 8 <= bytes.byteLength) {
2010
+ const chunkId = readAscii(bytes, pos, 4);
2011
+ const chunkBytes = view.getUint32(pos + 4, true);
2012
+ const dataStart = pos + 8;
2013
+ if (chunkId === "data") {
2014
+ if (dataStart + chunkBytes > bytes.byteLength) {
2015
+ throw new Error("[voice] WAV data chunk exceeds input length");
2016
+ }
2017
+ if (chunkBytes % 2 !== 0) {
2018
+ throw new Error("[voice] WAV PCM16 data chunk has odd byte length");
2019
+ }
2020
+ const pcm = new Float32Array(chunkBytes / 2);
2021
+ for (let i = 0; i < pcm.length; i++) {
2022
+ pcm[i] = view.getInt16(dataStart + i * 2, true) / 0x8000;
2023
+ }
2024
+ return { pcm, sampleRate };
2025
+ }
2026
+ pos = dataStart + chunkBytes + (chunkBytes % 2);
2027
+ }
2028
+ throw new Error("[voice] WAV input is missing a data chunk");
2029
+ }
2030
+
2031
+ function writeAscii(out: Uint8Array, offset: number, text: string): void {
2032
+ for (let i = 0; i < text.length; i++) {
2033
+ out[offset + i] = text.charCodeAt(i);
2034
+ }
2035
+ }
2036
+
2037
+ function readAscii(bytes: Uint8Array, offset: number, length: number): string {
2038
+ let out = "";
2039
+ for (let i = 0; i < length; i++) {
2040
+ out += String.fromCharCode(bytes[offset + i]);
2041
+ }
2042
+ return out;
2043
+ }
2044
+
2045
+ function readPositiveIntEnv(name: string): number | undefined {
2046
+ const raw = process.env[name]?.trim();
2047
+ if (!raw) return undefined;
2048
+ const value = Number.parseInt(raw, 10);
2049
+ return Number.isFinite(value) && value > 0 ? value : undefined;
2050
+ }
2051
+
2052
+ /**
2053
+ * Default lifecycle loaders derived from the bundle layout (per
2054
+ * AGENTS.md §2: `tts/omnivoice-<size>.gguf` + `asr/...`).
2055
+ *
2056
+ * When a live `ffi`/`ctx` pair is passed in, arming calls
2057
+ * `ffi.mmapAcquire(ctx, "tts" | "asr")` before the lifecycle can enter
2058
+ * `voice-on`, and the returned handles' `evictPages()` calls forward
2059
+ * to `ffi.mmapEvict(ctx, "tts" | "asr")`. The C ABI is declared in
2060
+ * `scripts/omnivoice-fuse/ffi.h`. Production builds may implement this
2061
+ * as page eviction or as a full voice-runtime unload for mobile RAM
2062
+ * pressure; callers must reacquire before using the region again. The
2063
+ * compatibility library returns `ELIZA_ERR_NOT_IMPLEMENTED`, which the binding raises as
2064
+ * `VoiceLifecycleError({code:"kernel-missing"})`.
2065
+ *
2066
+ * When `ffi` is null, acquire/evict are documented empty transitions — used by the
2067
+ * development TTS path in tests + dev (no real mmap exists). Directory and
2068
+ * "contains at least one file" checks still run for both TTS and ASR.
2069
+ * ASR never gets a virtual fallback: voice-on requires a real bundled ASR
2070
+ * model file so the FFI path can acquire the `"asr"` region and surface
2071
+ * the fused ABI's diagnostic if the runtime lacks the required region support.
2072
+ */
2073
+ interface FfiContextRef {
2074
+ current: ElizaInferenceContextHandle | null;
2075
+ ensure(): ElizaInferenceContextHandle;
2076
+ }
2077
+
2078
+ function ensureContext(
2079
+ ref: ElizaInferenceContextHandle | FfiContextRef | null,
2080
+ ): ElizaInferenceContextHandle | null {
2081
+ if (ref === null) return null;
2082
+ if (typeof ref === "object" && "ensure" in ref) return ref.ensure();
2083
+ return ref;
2084
+ }
2085
+
2086
+ /**
2087
+ * No-op lifecycle loaders for the Kokoro-only bridge. ORT owns the
2088
+ * model memory; nothing to mmap-acquire or evict. ASR is not served
2089
+ * from this path — callers that need ASR construct
2090
+ * `createStreamingTranscriber` directly (the fused-only chain in
2091
+ * `transcriber.ts`: fused streaming → fused batch → AsrUnavailableError).
2092
+ */
2093
+ function kokoroOnlyLifecycleLoaders(): VoiceLifecycleLoaders {
2094
+ const noopMmap = (id: string): MmapRegionHandle => ({
2095
+ id,
2096
+ path: "",
2097
+ sizeBytes: 0,
2098
+ async evictPages() {
2099
+ // Nothing to evict — ORT owns the model bytes.
2100
+ },
2101
+ async release() {
2102
+ // No mmap region to release.
2103
+ },
2104
+ });
2105
+ return {
2106
+ loadTtsRegion: async () => noopMmap("kokoro:tts"),
2107
+ loadAsrRegion: async () => noopMmap("kokoro:asr"),
2108
+ loadVoiceCaches: async () => ({
2109
+ id: "kokoro:voice-caches",
2110
+ async release() {},
2111
+ }),
2112
+ loadVoiceSchedulerNodes: async () => ({
2113
+ id: "kokoro:voice-scheduler-nodes",
2114
+ async release() {},
2115
+ }),
2116
+ };
2117
+ }
2118
+
2119
+ function defaultLifecycleLoaders(
2120
+ bundleRoot: string,
2121
+ ffi: ElizaInferenceFfi | null,
2122
+ ctx: ElizaInferenceContextHandle | FfiContextRef | null,
2123
+ ): VoiceLifecycleLoaders {
2124
+ return {
2125
+ loadTtsRegion: async () =>
2126
+ bundleMmapRegion(path.join(bundleRoot, "tts"), "tts", ffi, ctx),
2127
+ loadAsrRegion: async () =>
2128
+ bundleMmapRegion(path.join(bundleRoot, "asr"), "asr", ffi, ctx),
2129
+ loadVoiceCaches: async () => ({
2130
+ id: `voice-caches:${bundleRoot}`,
2131
+ async release() {
2132
+ // Caches stay live in the SpeakerPresetCache + PhraseCache
2133
+ // singletons; the registry refcount is the only thing that
2134
+ // drops on disarm.
2135
+ },
2136
+ }),
2137
+ loadVoiceSchedulerNodes: async () => ({
2138
+ id: `voice-scheduler-nodes:${bundleRoot}`,
2139
+ async release() {
2140
+ // Scheduler nodes (chunker, rollback, ring buffer, barge-in)
2141
+ // are owned by the bridge's `scheduler` field — no extra
2142
+ // teardown beyond the refcount drop.
2143
+ },
2144
+ }),
2145
+ };
2146
+ }
2147
+
2148
+ /**
2149
+ * Build an `MmapRegionHandle` for a bundle subdirectory. Refuses to
2150
+ * fabricate a region when the directory is missing — that surfaces as
2151
+ * `VoiceLifecycleError` via the lifecycle's `arm-failed`/`mmap-fail`
2152
+ * mapping (no silent fallback to a smaller voice model — AGENTS.md §3).
2153
+ *
2154
+ * `mmapAcquire()` / `evictPages()` forward to the FFI binding when one
2155
+ * is supplied. With no FFI handle (test mode), those calls return without
2156
+ * touching native memory because no real mmap was made. The lifecycle test
2157
+ * still asserts the call shape via injected mocks.
2158
+ */
2159
+ function bundleMmapRegion(
2160
+ dir: string,
2161
+ kind: "tts" | "asr",
2162
+ ffi: ElizaInferenceFfi | null,
2163
+ ctx: ElizaInferenceContextHandle | FfiContextRef | null,
2164
+ ): MmapRegionHandle {
2165
+ if (!existsSync(dir)) {
2166
+ throw new Error(
2167
+ `[voice] mmap MAP_FAILED: ${kind} directory missing at ${dir}`,
2168
+ );
2169
+ }
2170
+ if (!directoryHasRegularFile(dir)) {
2171
+ throw new Error(
2172
+ `[voice] mmap MAP_FAILED: ${kind} directory has no model files at ${dir}`,
2173
+ );
2174
+ }
2175
+ // Stat the directory to get a stable inode for id derivation. Real
2176
+ // FFI will mmap each weight file independently; this default loader
2177
+ // collapses them into one region per kind for refcount purposes.
2178
+ const st = statSync(dir);
2179
+ const handle = ffi ? ensureContext(ctx) : null;
2180
+ if (ffi && handle !== null) {
2181
+ // Real fused build: load or re-page the heavy voice region now.
2182
+ // A compatibility runtime without region support returns ELIZA_ERR_NOT_IMPLEMENTED,
2183
+ // which surfaces as VoiceLifecycleError({code:"kernel-missing"})
2184
+ // before the lifecycle can enter voice-on.
2185
+ ffi.mmapAcquire(handle, kind);
2186
+ }
2187
+ return {
2188
+ id: `mmap:${kind}:${st.ino}`,
2189
+ path: dir,
2190
+ sizeBytes: st.size,
2191
+ async evictPages() {
2192
+ const evictHandle = ffi ? ensureContext(ctx) : null;
2193
+ if (ffi && evictHandle !== null) {
2194
+ // Real fused build: madvise / VirtualUnlock through the C ABI.
2195
+ // Throws VoiceLifecycleError on a negative return — the
2196
+ // lifecycle catches and re-classifies via `disarm-failed`.
2197
+ ffi.mmapEvict(evictHandle, kind);
2198
+ }
2199
+ // Else: no FFI handle (test TTS / no fused build) — nothing to
2200
+ // evict.
2201
+ },
2202
+ async release() {
2203
+ // The FFI owns the actual mmap; release is a refcount drop on
2204
+ // the JS side. The fused build's destroy path flushes any
2205
+ // remaining pages when the context is destroyed.
2206
+ },
2207
+ };
2208
+ }
2209
+
2210
+ /** Re-export for the engine and tests that want the default loader. */
2211
+ export { defaultLifecycleLoaders };
2212
+
2213
+ /**
2214
+ * Platform-specific shared-library suffix for the fused omnivoice build.
2215
+ * macOS dylib, Linux/Android so, Windows dll. Windows artifacts have
2216
+ * used both `elizainference.dll` and `libelizainference.dll` names in
2217
+ * cross-build toolchains, so the runtime probes both.
2218
+ */
2219
+ function libraryFilenames(): string[] {
2220
+ if (process.platform === "darwin") return ["libelizainference.dylib"];
2221
+ if (process.platform === "win32") {
2222
+ return ["elizainference.dll", "libelizainference.dll"];
2223
+ }
2224
+ return ["libelizainference.so"];
2225
+ }
2226
+
2227
+ function locateBundleLibrary(bundleRoot: string): string {
2228
+ const exact = process.env.ELIZA_INFERENCE_LIBRARY?.trim();
2229
+ if (exact && existsSync(exact)) return exact;
2230
+
2231
+ const dirs = [
2232
+ path.join(bundleRoot, "lib"),
2233
+ exact ? path.dirname(exact) : null,
2234
+ process.env.ELIZA_INFERENCE_LIB_DIR?.trim() || null,
2235
+ ...managedFusedRuntimeDirs(),
2236
+ ].filter((dir): dir is string => Boolean(dir));
2237
+
2238
+ for (const dir of dirs) {
2239
+ for (const name of libraryFilenames()) {
2240
+ const candidate = path.join(dir, name);
2241
+ if (existsSync(candidate)) return candidate;
2242
+ }
2243
+ }
2244
+ return path.join(
2245
+ dirs[0] ?? path.join(bundleRoot, "lib"),
2246
+ libraryFilenames()[0] ?? "libelizainference.so",
2247
+ );
2248
+ }
2249
+
2250
+ function bundleHasOmniVoiceWeights(bundleRoot: string): boolean {
2251
+ const ttsDir = path.join(bundleRoot, "tts");
2252
+ if (!existsSync(ttsDir)) return false;
2253
+ try {
2254
+ return readdirSync(ttsDir, { withFileTypes: true }).some(
2255
+ (entry) => entry.isFile() && /^omnivoice-.+\.gguf$/i.test(entry.name),
2256
+ );
2257
+ } catch {
2258
+ return false;
2259
+ }
2260
+ }
2261
+
2262
+ export function isOmniVoiceBundleAvailable(bundleRoot: string): boolean {
2263
+ if (!bundleRoot || !existsSync(bundleRoot)) return false;
2264
+ const presetPath = path.join(bundleRoot, DEFAULT_VOICE_PRESET_REL_PATH);
2265
+ return (
2266
+ existsSync(presetPath) &&
2267
+ bundleHasOmniVoiceWeights(bundleRoot) &&
2268
+ existsSync(locateBundleLibrary(bundleRoot))
2269
+ );
2270
+ }
2271
+
2272
+ function directoryHasRegularFile(dir: string): boolean {
2273
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
2274
+ if (entry.isFile()) return true;
2275
+ }
2276
+ return false;
2277
+ }
2278
+
2279
+ function bundleHasRegularFile(dir: string): boolean {
2280
+ if (!existsSync(dir)) return false;
2281
+ try {
2282
+ return directoryHasRegularFile(dir);
2283
+ } catch {
2284
+ return false;
2285
+ }
2286
+ }
2287
+
2288
+ function managedFusedRuntimeDirs(): string[] {
2289
+ if (process.env.ELIZA_INFERENCE_MANAGED_LOOKUP?.trim() === "0") {
2290
+ return [];
2291
+ }
2292
+ const root = localInferenceRoot();
2293
+ const platform = process.platform;
2294
+ const arch = os.arch();
2295
+ const candidates = [
2296
+ `${platform}-${arch}-metal-fused`,
2297
+ `${platform}-${arch}-vulkan-fused`,
2298
+ `${platform}-${arch}-cuda-fused`,
2299
+ `${platform}-${arch}-cpu-fused`,
2300
+ ];
2301
+ return candidates.map((target) => path.join(root, "bin", "mtp", target));
2302
+ }