@elizaos/app-core 2.0.0-beta.3 → 2.0.3-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1466) hide show
  1. package/agent-bridge.d.ts +27 -0
  2. package/agent-bridge.d.ts.map +1 -0
  3. package/agent-bridge.js +26 -0
  4. package/api/__tests__/sandbox-test-helpers.js +1 -1
  5. package/api/auth/audit.d.ts +1 -1
  6. package/api/auth/audit.js +1 -1
  7. package/api/auth/auth-context.d.ts +1 -1
  8. package/api/auth/auth-context.js +1 -1
  9. package/api/auth/bootstrap-token.d.ts +1 -1
  10. package/api/auth/bootstrap-token.js +2 -2
  11. package/api/auth/index.d.ts +9 -10
  12. package/api/auth/index.d.ts.map +1 -1
  13. package/api/auth/index.js +9 -10
  14. package/api/auth/passwords.js +2 -2
  15. package/api/auth/sensitive-rate-limit.d.ts +1 -4
  16. package/api/auth/sensitive-rate-limit.d.ts.map +1 -1
  17. package/api/auth/sensitive-rate-limit.js +6 -6
  18. package/api/auth/sessions.d.ts +1 -1
  19. package/api/auth/sessions.js +2 -2
  20. package/api/auth-bootstrap-routes.d.ts +7 -14
  21. package/api/auth-bootstrap-routes.d.ts.map +1 -1
  22. package/api/auth-bootstrap-routes.js +14 -27
  23. package/api/auth-pairing-routes.d.ts +17 -0
  24. package/api/auth-pairing-routes.d.ts.map +1 -0
  25. package/api/auth-pairing-routes.js +299 -0
  26. package/api/auth-session-routes.d.ts +2 -2
  27. package/api/auth-session-routes.d.ts.map +1 -1
  28. package/api/auth-session-routes.js +36 -15
  29. package/api/auth.d.ts +13 -20
  30. package/api/auth.d.ts.map +1 -1
  31. package/api/auth.js +32 -27
  32. package/api/automations-compat-routes.d.ts +1 -1
  33. package/api/automations-compat-routes.d.ts.map +1 -1
  34. package/api/automations-compat-routes.js +5 -5
  35. package/api/background-tasks-routes.d.ts +4 -0
  36. package/api/background-tasks-routes.d.ts.map +1 -0
  37. package/api/background-tasks-routes.js +63 -0
  38. package/api/catalog-routes.d.ts +1 -1
  39. package/api/catalog-routes.js +3 -3
  40. package/api/cloud-pair-route.d.ts +26 -0
  41. package/api/cloud-pair-route.d.ts.map +1 -0
  42. package/api/cloud-pair-route.js +222 -0
  43. package/api/cloud-voice-routes.d.ts +52 -0
  44. package/api/cloud-voice-routes.d.ts.map +1 -0
  45. package/api/cloud-voice-routes.js +50 -0
  46. package/api/compat-route-shared.d.ts +14 -7
  47. package/api/compat-route-shared.d.ts.map +1 -1
  48. package/api/compat-route-shared.js +27 -188
  49. package/api/credential-resolver.d.ts +2 -2
  50. package/api/credential-resolver.d.ts.map +1 -1
  51. package/api/credential-resolver.js +8 -2
  52. package/api/database-rows-compat-routes.d.ts +1 -1
  53. package/api/database-rows-compat-routes.d.ts.map +1 -1
  54. package/api/database-rows-compat-routes.js +69 -31
  55. package/api/dev-boot-history.d.ts +26 -0
  56. package/api/dev-boot-history.d.ts.map +1 -0
  57. package/api/dev-boot-history.js +69 -0
  58. package/api/dev-compat-routes.d.ts +6 -1
  59. package/api/dev-compat-routes.d.ts.map +1 -1
  60. package/api/dev-compat-routes.js +127 -4
  61. package/api/dev-console-log.d.ts +2 -2
  62. package/api/dev-console-log.d.ts.map +1 -1
  63. package/api/dev-console-log.js +8 -5
  64. package/api/dev-route-catalog.d.ts +58 -0
  65. package/api/dev-route-catalog.d.ts.map +1 -0
  66. package/api/dev-route-catalog.js +477 -0
  67. package/api/dev-stack.d.ts.map +1 -1
  68. package/api/dev-stack.js +6 -9
  69. package/api/drop-status-compat-route.d.ts +3 -0
  70. package/api/drop-status-compat-route.d.ts.map +1 -0
  71. package/api/drop-status-compat-route.js +12 -0
  72. package/api/first-run-routes.d.ts +4 -0
  73. package/api/first-run-routes.d.ts.map +1 -0
  74. package/api/first-run-routes.js +208 -0
  75. package/api/first-run-tts-route.d.ts +19 -0
  76. package/api/first-run-tts-route.d.ts.map +1 -0
  77. package/api/first-run-tts-route.js +59 -0
  78. package/api/internal-routes.d.ts +23 -0
  79. package/api/internal-routes.d.ts.map +1 -0
  80. package/api/internal-routes.js +203 -0
  81. package/api/ios-local-agent-transport.d.ts +36 -0
  82. package/api/ios-local-agent-transport.d.ts.map +1 -0
  83. package/api/ios-local-agent-transport.js +566 -0
  84. package/api/onboarding-voice-lines.d.ts +23 -0
  85. package/api/onboarding-voice-lines.d.ts.map +1 -0
  86. package/api/onboarding-voice-lines.js +8 -0
  87. package/api/perf-instrument.d.ts +43 -0
  88. package/api/perf-instrument.d.ts.map +1 -0
  89. package/api/perf-instrument.js +113 -0
  90. package/api/response.d.ts.map +1 -1
  91. package/api/response.js +14 -14
  92. package/api/runtime-mode-routes.d.ts +1 -1
  93. package/api/runtime-mode-routes.d.ts.map +1 -1
  94. package/api/runtime-mode-routes.js +2 -2
  95. package/api/secrets-inventory-routes.js +2 -2
  96. package/api/secrets-manager-routes.d.ts +1 -1
  97. package/api/secrets-manager-routes.d.ts.map +1 -1
  98. package/api/secrets-manager-routes.js +9 -10
  99. package/api/sensitive-request-routes.d.ts +2 -2
  100. package/api/sensitive-request-routes.js +5 -5
  101. package/api/server-config-filter.d.ts.map +1 -1
  102. package/api/server-config-filter.js +0 -1
  103. package/api/server-cors.d.ts.map +1 -1
  104. package/api/server-cors.js +13 -2
  105. package/api/server-first-run-helpers.d.ts +26 -0
  106. package/api/server-first-run-helpers.d.ts.map +1 -0
  107. package/api/server-first-run-helpers.js +268 -0
  108. package/api/server-security.js +1 -1
  109. package/api/server-startup.d.ts.map +1 -1
  110. package/api/server-startup.js +3 -4
  111. package/api/server-wallet-trade.js +1 -1
  112. package/api/server.d.ts +11 -11
  113. package/api/server.d.ts.map +1 -1
  114. package/api/server.js +227 -139
  115. package/api/setup-contract.d.ts +63 -0
  116. package/api/setup-contract.d.ts.map +1 -0
  117. package/api/setup-contract.js +39 -0
  118. package/api/training-benchmarks.d.ts +97 -0
  119. package/api/training-benchmarks.d.ts.map +1 -0
  120. package/api/training-benchmarks.js +329 -0
  121. package/api/workbench-compat-routes.d.ts +1 -1
  122. package/api/workbench-compat-routes.js +2 -2
  123. package/benchmark/cerebras-autowire.d.ts +28 -0
  124. package/benchmark/cerebras-autowire.d.ts.map +1 -0
  125. package/benchmark/cerebras-autowire.js +62 -0
  126. package/benchmark/lifeops-bench-handler.d.ts +36 -0
  127. package/benchmark/lifeops-bench-handler.d.ts.map +1 -1
  128. package/benchmark/lifeops-bench-handler.js +63 -1
  129. package/benchmark/lifeops-fake-backend.d.ts +39 -0
  130. package/benchmark/lifeops-fake-backend.d.ts.map +1 -1
  131. package/benchmark/lifeops-fake-backend.js +993 -21
  132. package/benchmark/mock-plugin.d.ts.map +1 -1
  133. package/benchmark/mock-plugin.js +0 -24
  134. package/benchmark/plugin.d.ts +2 -1
  135. package/benchmark/plugin.d.ts.map +1 -1
  136. package/benchmark/plugin.js +989 -68
  137. package/benchmark/replay-capture.d.ts +2 -2
  138. package/benchmark/replay-capture.d.ts.map +1 -1
  139. package/benchmark/replay-capture.js +3 -3
  140. package/benchmark/server-utils.d.ts +164 -11
  141. package/benchmark/server-utils.d.ts.map +1 -1
  142. package/benchmark/server-utils.js +625 -62
  143. package/benchmark/server.d.ts.map +1 -1
  144. package/benchmark/server.js +1962 -118
  145. package/boot-profile.d.ts +3 -0
  146. package/boot-profile.d.ts.map +1 -0
  147. package/boot-profile.js +30 -0
  148. package/browser.d.ts +23 -1
  149. package/browser.d.ts.map +1 -1
  150. package/browser.js +20 -1
  151. package/cli/argv.js +1 -1
  152. package/cli/banner.js +1 -1
  153. package/cli/command-format.js +2 -2
  154. package/cli/doctor/checks.d.ts.map +1 -1
  155. package/cli/doctor/checks.js +6 -6
  156. package/cli/plugins-cli.d.ts.map +1 -1
  157. package/cli/plugins-cli.js +82 -32
  158. package/cli/profile.d.ts.map +1 -1
  159. package/cli/profile.js +5 -4
  160. package/cli/program/build-program.js +4 -4
  161. package/cli/program/command-registry.d.ts.map +1 -1
  162. package/cli/program/command-registry.js +13 -11
  163. package/cli/program/help.js +5 -5
  164. package/cli/program/preaction.js +5 -5
  165. package/cli/program/register.auth.d.ts +27 -1
  166. package/cli/program/register.auth.d.ts.map +1 -1
  167. package/cli/program/register.auth.js +193 -12
  168. package/cli/program/register.capability-router.d.ts +29 -0
  169. package/cli/program/register.capability-router.d.ts.map +1 -0
  170. package/cli/program/register.capability-router.js +568 -0
  171. package/cli/program/register.config.js +1 -1
  172. package/cli/program/register.configure.d.ts.map +1 -1
  173. package/cli/program/register.configure.js +1 -1
  174. package/cli/program/register.dashboard.d.ts.map +1 -1
  175. package/cli/program/register.dashboard.js +17 -8
  176. package/cli/program/register.db.d.ts.map +1 -1
  177. package/cli/program/register.db.js +3 -4
  178. package/cli/program/register.doctor.js +7 -7
  179. package/cli/program/register.setup.d.ts.map +1 -1
  180. package/cli/program/register.setup.js +14 -10
  181. package/cli/program/register.start.d.ts.map +1 -1
  182. package/cli/program/register.start.js +5 -3
  183. package/cli/program/register.subclis.js +3 -3
  184. package/cli/program/register.update.d.ts +6 -0
  185. package/cli/program/register.update.d.ts.map +1 -1
  186. package/cli/program/register.update.js +58 -6
  187. package/cli/program.d.ts +1 -1
  188. package/cli/program.js +1 -1
  189. package/cli/run-main.js +4 -4
  190. package/config/app-config.d.ts +2 -0
  191. package/config/app-config.d.ts.map +1 -0
  192. package/config/app-config.js +1 -0
  193. package/connectors/capacitor-jsc.d.ts.map +1 -1
  194. package/connectors/capacitor-jsc.js +16 -10
  195. package/connectors/capacitor-quickjs.d.ts.map +1 -1
  196. package/connectors/capacitor-quickjs.js +18 -13
  197. package/connectors/capacitor-sqlite.d.ts.map +1 -1
  198. package/connectors/capacitor-sqlite.js +27 -12
  199. package/dispatch/approval-queue.d.ts +37 -0
  200. package/dispatch/approval-queue.d.ts.map +1 -0
  201. package/dispatch/approval-queue.js +25 -0
  202. package/dispatch/channel-registry.d.ts +30 -0
  203. package/dispatch/channel-registry.d.ts.map +1 -0
  204. package/dispatch/channel-registry.js +22 -0
  205. package/dispatch/connector-registry.d.ts +39 -0
  206. package/dispatch/connector-registry.d.ts.map +1 -0
  207. package/dispatch/connector-registry.js +24 -0
  208. package/dispatch/index.d.ts +14 -0
  209. package/dispatch/index.d.ts.map +1 -0
  210. package/dispatch/index.js +13 -0
  211. package/dispatch/send-policy.d.ts +36 -0
  212. package/dispatch/send-policy.d.ts.map +1 -0
  213. package/dispatch/send-policy.js +16 -0
  214. package/entry.js +28 -11
  215. package/first-run/first-run-config.d.ts +55 -0
  216. package/first-run/first-run-config.d.ts.map +1 -0
  217. package/first-run/first-run-config.js +178 -0
  218. package/first-run/runtime-target.d.ts +4 -0
  219. package/first-run/runtime-target.d.ts.map +1 -0
  220. package/first-run/runtime-target.js +13 -0
  221. package/index.d.ts +46 -34
  222. package/index.d.ts.map +1 -1
  223. package/index.js +53 -33
  224. package/package.json +166 -51
  225. package/packaging/PUBLISHING_GUIDE.md +20 -20
  226. package/packaging/debian/apt-repo-config/README.md +18 -0
  227. package/packaging/debian/apt-repo-config/conf/distributions +11 -0
  228. package/packaging/flatpak/README.md +26 -16
  229. package/packaging/flatpak/ai.elizaos.App.metainfo.xml +17 -12
  230. package/packaging/flatpak/ai.elizaos.App.store.yml +5 -5
  231. package/packaging/flatpak/ai.elizaos.App.yml +10 -24
  232. package/packaging/flatpak/elizaos-app-wrapper.store.sh +2 -2
  233. package/packaging/flatpak/generate-sources.sh +74 -0
  234. package/packaging/flatpak/node-sources.json +7930 -0
  235. package/packaging/inno/build-inno.ps1 +34 -9
  236. package/packaging/msix/AppxManifest.store.xml +1 -1
  237. package/packaging/msix/README.md +39 -19
  238. package/packaging/msix/build-msix.ps1 +44 -14
  239. package/packaging/snap/snapcraft.yaml +22 -21
  240. package/packaging/test-packaging.sh +2 -2
  241. package/permissions/types.d.ts +1 -1
  242. package/permissions/types.js +1 -1
  243. package/platform/elizaos-agent-browser-stub.d.ts +143 -0
  244. package/platform/elizaos-agent-browser-stub.d.ts.map +1 -0
  245. package/platform/elizaos-agent-browser-stub.js +157 -0
  246. package/platform/elizaos-plugin-elizacloud-browser-stub.d.ts +34 -0
  247. package/platform/elizaos-plugin-elizacloud-browser-stub.d.ts.map +1 -0
  248. package/platform/elizaos-plugin-elizacloud-browser-stub.js +51 -0
  249. package/platform/empty-node-module.d.ts +148 -0
  250. package/platform/empty-node-module.d.ts.map +1 -1
  251. package/platform/empty-node-module.js +140 -3
  252. package/platform/ios-runtime-backends.d.ts +83 -0
  253. package/platform/ios-runtime-backends.d.ts.map +1 -0
  254. package/platform/ios-runtime-backends.js +133 -0
  255. package/platform/ios-runtime-bridge.d.ts +15 -0
  256. package/platform/ios-runtime-bridge.d.ts.map +1 -0
  257. package/platform/ios-runtime-bridge.js +526 -0
  258. package/platform/native-library-policy.d.ts +23 -0
  259. package/platform/native-library-policy.d.ts.map +1 -0
  260. package/platform/native-library-policy.js +112 -0
  261. package/platform/native-plugin-entrypoints.d.ts +19 -0
  262. package/platform/native-plugin-entrypoints.d.ts.map +1 -0
  263. package/platform/native-plugin-entrypoints.js +29 -0
  264. package/platforms/android/README.md +68 -10
  265. package/platforms/android/app/build.gradle +291 -3
  266. package/platforms/android/app/capacitor.build.gradle +18 -1
  267. package/platforms/android/app/proguard-rules.pro +17 -2
  268. package/platforms/android/app/src/androidTest/java/ai/elizaos/app/ElizaOsInstrumentedTest.java +1 -1
  269. package/platforms/android/app/src/main/AndroidManifest.xml +335 -18
  270. package/platforms/android/app/src/main/assets/runners/eliza-tasks.js +177 -0
  271. package/platforms/android/app/src/main/elizavoice-jni/CMakeLists.txt +100 -0
  272. package/platforms/android/app/src/main/elizavoice-jni/elizavoice-jni.cpp +1629 -0
  273. package/platforms/android/app/src/main/java/ai/elizaos/app/AgentPlugin.java +111 -171
  274. package/platforms/android/app/src/main/java/ai/elizaos/app/AndroidVirtualizationBridge.java +284 -0
  275. package/platforms/android/app/src/main/java/ai/elizaos/app/BatteryOptimizationPlugin.java +95 -0
  276. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaAccessibilityService.java +55 -0
  277. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaAgentService.java +1283 -141
  278. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaAndroidSystemBridge.java +83 -0
  279. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaAssistActivity.java +50 -1
  280. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaBionicInferenceServer.java +740 -0
  281. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaBootReceiver.java +90 -8
  282. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaBrowserActivity.java +2 -2
  283. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaCalendarActivity.java +1 -1
  284. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaCameraActivity.java +1 -1
  285. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaClockActivity.java +2 -2
  286. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaContactsActivity.java +1 -1
  287. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaDialActivity.java +1 -1
  288. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaInCallService.java +1 -1
  289. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaMmsReceiver.java +1 -1
  290. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaNativeBridge.java +22 -0
  291. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaNotificationListenerService.java +45 -0
  292. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaQuickActionsWidgetProvider.java +68 -0
  293. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaShareActivity.java +132 -0
  294. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaSmsComposeActivity.java +1 -1
  295. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaSmsGatewayService.java +268 -0
  296. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaSmsReceiver.java +12 -1
  297. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaTasksWorker.java +194 -0
  298. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaVoiceCaptureService.java +198 -0
  299. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaVoiceNative.java +255 -0
  300. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaVoicePlugin.java +505 -0
  301. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaVoiceTileService.java +39 -0
  302. package/platforms/android/app/src/main/java/ai/elizaos/app/ElizaWorkScheduler.java +60 -0
  303. package/platforms/android/app/src/main/java/ai/elizaos/app/GatewayConnectionService.java +53 -19
  304. package/platforms/android/app/src/main/java/ai/elizaos/app/MainActivity.java +160 -33
  305. package/platforms/android/app/src/main/java/ai/elizaos/app/ResourceProbePlugin.java +169 -0
  306. package/platforms/android/app/src/main/java/ai/elizaos/app/VoiceCapturePlugin.java +119 -0
  307. package/platforms/android/app/src/main/res/drawable/eliza_widget_background.xml +10 -0
  308. package/platforms/android/app/src/main/res/drawable/eliza_widget_button_background.xml +13 -0
  309. package/platforms/android/app/src/main/res/drawable/splash.png +0 -0
  310. package/platforms/android/app/src/main/res/drawable-land-hdpi/splash.png +0 -0
  311. package/platforms/android/app/src/main/res/drawable-land-mdpi/splash.png +0 -0
  312. package/platforms/android/app/src/main/res/drawable-land-xhdpi/splash.png +0 -0
  313. package/platforms/android/app/src/main/res/drawable-land-xxhdpi/splash.png +0 -0
  314. package/platforms/android/app/src/main/res/drawable-land-xxxhdpi/splash.png +0 -0
  315. package/platforms/android/app/src/main/res/drawable-port-hdpi/splash.png +0 -0
  316. package/platforms/android/app/src/main/res/drawable-port-mdpi/splash.png +0 -0
  317. package/platforms/android/app/src/main/res/drawable-port-xhdpi/splash.png +0 -0
  318. package/platforms/android/app/src/main/res/drawable-port-xxhdpi/splash.png +0 -0
  319. package/platforms/android/app/src/main/res/drawable-port-xxxhdpi/splash.png +0 -0
  320. package/platforms/android/app/src/main/res/layout/eliza_quick_actions_widget.xml +86 -0
  321. package/platforms/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml +2 -1
  322. package/platforms/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml +2 -1
  323. package/platforms/android/app/src/main/res/mipmap-hdpi/ic_launcher.png +0 -0
  324. package/platforms/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png +0 -0
  325. package/platforms/android/app/src/main/res/mipmap-hdpi/ic_launcher_monochrome.png +0 -0
  326. package/platforms/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png +0 -0
  327. package/platforms/android/app/src/main/res/mipmap-mdpi/ic_launcher.png +0 -0
  328. package/platforms/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png +0 -0
  329. package/platforms/android/app/src/main/res/mipmap-mdpi/ic_launcher_monochrome.png +0 -0
  330. package/platforms/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png +0 -0
  331. package/platforms/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png +0 -0
  332. package/platforms/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png +0 -0
  333. package/platforms/android/app/src/main/res/mipmap-xhdpi/ic_launcher_monochrome.png +0 -0
  334. package/platforms/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png +0 -0
  335. package/platforms/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png +0 -0
  336. package/platforms/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png +0 -0
  337. package/platforms/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_monochrome.png +0 -0
  338. package/platforms/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png +0 -0
  339. package/platforms/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png +0 -0
  340. package/platforms/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png +0 -0
  341. package/platforms/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_monochrome.png +0 -0
  342. package/platforms/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png +0 -0
  343. package/platforms/android/app/src/main/res/values/android_app_actions.xml +48 -0
  344. package/platforms/android/app/src/main/res/values/colors.xml +12 -0
  345. package/platforms/android/app/src/main/res/values/ic_launcher_background.xml +2 -2
  346. package/platforms/android/app/src/main/res/values/strings.xml +2 -2
  347. package/platforms/android/app/src/main/res/values/styles.xml +28 -1
  348. package/platforms/android/app/src/main/res/xml/eliza_accessibility_service.xml +9 -0
  349. package/platforms/android/app/src/main/res/xml/eliza_quick_actions_widget.xml +13 -0
  350. package/platforms/android/app/src/main/res/xml/shortcuts.xml +121 -0
  351. package/platforms/android/build.gradle +2 -2
  352. package/platforms/android/capacitor-cordova-android-plugins/build.gradle +9 -3
  353. package/platforms/android/capacitor-cordova-android-plugins/cordova.variables.gradle +6 -2
  354. package/platforms/android/capacitor-cordova-android-plugins/src/main/AndroidManifest.xml +7 -2
  355. package/platforms/android/capacitor-cordova-android-plugins/src/main/java/.gitkeep +0 -1
  356. package/platforms/android/capacitor.settings.gradle +66 -16
  357. package/platforms/android/gradle.properties +1 -0
  358. package/platforms/android/update-manifest/generate-manifest.mjs +97 -0
  359. package/platforms/android/update-manifest/schema.json +26 -0
  360. package/platforms/apple-store-entitlements.reviewed.json +155 -0
  361. package/platforms/electrobun/.generated/brand-config.json +3 -2
  362. package/platforms/electrobun/LICENSE +21 -0
  363. package/platforms/electrobun/README.md +15 -1
  364. package/platforms/electrobun/assets/appIcon.icns +0 -0
  365. package/platforms/electrobun/assets/appIcon.ico +0 -0
  366. package/platforms/electrobun/assets/appIcon.iconset/icon_128x128.png +0 -0
  367. package/platforms/electrobun/assets/appIcon.iconset/icon_128x128@2x.png +0 -0
  368. package/platforms/electrobun/assets/appIcon.iconset/icon_16x16.png +0 -0
  369. package/platforms/electrobun/assets/appIcon.iconset/icon_16x16@2x.png +0 -0
  370. package/platforms/electrobun/assets/appIcon.iconset/icon_256x256.png +0 -0
  371. package/platforms/electrobun/assets/appIcon.iconset/icon_256x256@2x.png +0 -0
  372. package/platforms/electrobun/assets/appIcon.iconset/icon_32x32.png +0 -0
  373. package/platforms/electrobun/assets/appIcon.iconset/icon_32x32@2x.png +0 -0
  374. package/platforms/electrobun/assets/appIcon.iconset/icon_512x512.png +0 -0
  375. package/platforms/electrobun/assets/brand-config.json +6 -6
  376. package/platforms/electrobun/biome.json +9 -9
  377. package/platforms/electrobun/docs/capability-collapse-matrix.json +318 -0
  378. package/platforms/electrobun/docs/capability-collapse-matrix.md +129 -0
  379. package/platforms/electrobun/docs/capability-routing.md +86 -0
  380. package/platforms/electrobun/docs/database-boot-policy.md +90 -0
  381. package/platforms/electrobun/docs/riscv64-port.md +175 -0
  382. package/platforms/electrobun/docs/startup-first-run-cleanup.md +18 -0
  383. package/platforms/electrobun/docs/trace-first-annotations.md +52 -0
  384. package/platforms/electrobun/docs/ui-boundary-audit.json +580 -0
  385. package/platforms/electrobun/docs/ui-boundary-audit.md +257 -0
  386. package/platforms/electrobun/electrobun.config.ts +591 -364
  387. package/platforms/electrobun/entitlements/JUSTIFICATIONS.md +141 -0
  388. package/platforms/electrobun/entitlements/README.md +34 -6
  389. package/platforms/electrobun/entitlements/mas-bun.entitlements +15 -0
  390. package/platforms/electrobun/entitlements/mas.entitlements +6 -4
  391. package/platforms/electrobun/native/macos/window-effects.mm +1526 -0
  392. package/platforms/electrobun/package.json +21 -13
  393. package/platforms/electrobun/remotes/fs/README.md +70 -0
  394. package/platforms/electrobun/remotes/fs/electrobun.config.ts +38 -0
  395. package/platforms/electrobun/remotes/fs/package.json +12 -0
  396. package/platforms/electrobun/remotes/fs/plugin.json +25 -0
  397. package/platforms/electrobun/remotes/fs/src/bun/errors.ts +57 -0
  398. package/platforms/electrobun/remotes/fs/src/bun/file-limits.ts +50 -0
  399. package/platforms/electrobun/remotes/fs/src/bun/fs-service.ts +389 -0
  400. package/platforms/electrobun/remotes/fs/src/bun/path-guard.ts +270 -0
  401. package/platforms/electrobun/remotes/fs/src/bun/protocol.ts +149 -0
  402. package/platforms/electrobun/remotes/fs/src/bun/worker.ts +174 -0
  403. package/platforms/electrobun/remotes/fs/src/dev/phase5-smoke.ts +171 -0
  404. package/platforms/electrobun/remotes/fs/src/web/index.html +8 -0
  405. package/platforms/electrobun/remotes/git/README.md +75 -0
  406. package/platforms/electrobun/remotes/git/electrobun.config.ts +44 -0
  407. package/platforms/electrobun/remotes/git/package.json +12 -0
  408. package/platforms/electrobun/remotes/git/plugin.json +31 -0
  409. package/platforms/electrobun/remotes/git/src/bun/errors.ts +69 -0
  410. package/platforms/electrobun/remotes/git/src/bun/git-command.ts +156 -0
  411. package/platforms/electrobun/remotes/git/src/bun/git-service.ts +446 -0
  412. package/platforms/electrobun/remotes/git/src/bun/operation-history.ts +124 -0
  413. package/platforms/electrobun/remotes/git/src/bun/protocol.ts +252 -0
  414. package/platforms/electrobun/remotes/git/src/bun/worker.ts +316 -0
  415. package/platforms/electrobun/remotes/git/src/dev/phase7-smoke.ts +141 -0
  416. package/platforms/electrobun/remotes/git/src/web/index.html +8 -0
  417. package/platforms/electrobun/remotes/local-model/README.md +138 -0
  418. package/platforms/electrobun/remotes/local-model/electrobun.config.ts +46 -0
  419. package/platforms/electrobun/remotes/local-model/package.json +12 -0
  420. package/platforms/electrobun/remotes/local-model/plugin.json +33 -0
  421. package/platforms/electrobun/remotes/local-model/src/bun/download-state.ts +115 -0
  422. package/platforms/electrobun/remotes/local-model/src/bun/eliza1-catalog.ts +405 -0
  423. package/platforms/electrobun/remotes/local-model/src/bun/errors.ts +74 -0
  424. package/platforms/electrobun/remotes/local-model/src/bun/hf-eliza1-client.ts +169 -0
  425. package/platforms/electrobun/remotes/local-model/src/bun/local-inference-api-client.ts +245 -0
  426. package/platforms/electrobun/remotes/local-model/src/bun/model-service.ts +490 -0
  427. package/platforms/electrobun/remotes/local-model/src/bun/protocol.ts +301 -0
  428. package/platforms/electrobun/remotes/local-model/src/bun/worker.ts +248 -0
  429. package/platforms/electrobun/remotes/local-model/src/dev/phase8-smoke.ts +116 -0
  430. package/platforms/electrobun/remotes/local-model/src/web/index.html +13 -0
  431. package/platforms/electrobun/remotes/pty/README.md +65 -0
  432. package/platforms/electrobun/remotes/pty/electrobun.config.ts +47 -0
  433. package/platforms/electrobun/remotes/pty/package.json +12 -0
  434. package/platforms/electrobun/remotes/pty/plugin.json +34 -0
  435. package/platforms/electrobun/remotes/pty/src/bun/errors.ts +57 -0
  436. package/platforms/electrobun/remotes/pty/src/bun/output-buffer.ts +127 -0
  437. package/platforms/electrobun/remotes/pty/src/bun/protocol.ts +192 -0
  438. package/platforms/electrobun/remotes/pty/src/bun/pty-service.ts +562 -0
  439. package/platforms/electrobun/remotes/pty/src/bun/worker.ts +218 -0
  440. package/platforms/electrobun/remotes/pty/src/dev/phase6-smoke.ts +127 -0
  441. package/platforms/electrobun/remotes/pty/src/web/index.html +8 -0
  442. package/platforms/electrobun/remotes/runtime/README.md +370 -0
  443. package/platforms/electrobun/remotes/runtime/electrobun.config.ts +48 -0
  444. package/platforms/electrobun/remotes/runtime/package.json +14 -0
  445. package/platforms/electrobun/remotes/runtime/plugin.json +30 -0
  446. package/platforms/electrobun/remotes/runtime/src/bun/api-client.ts +620 -0
  447. package/platforms/electrobun/remotes/runtime/src/bun/errors.ts +45 -0
  448. package/platforms/electrobun/remotes/runtime/src/bun/log-buffer.ts +33 -0
  449. package/platforms/electrobun/remotes/runtime/src/bun/protocol.ts +366 -0
  450. package/platforms/electrobun/remotes/runtime/src/bun/route-discovery.ts +419 -0
  451. package/platforms/electrobun/remotes/runtime/src/bun/runtime-manager.ts +423 -0
  452. package/platforms/electrobun/remotes/runtime/src/bun/sse-parser.ts +99 -0
  453. package/platforms/electrobun/remotes/runtime/src/bun/stream-manager.ts +887 -0
  454. package/platforms/electrobun/remotes/runtime/src/bun/worker.ts +1231 -0
  455. package/platforms/electrobun/remotes/runtime/src/dev/phase1-smoke.ts +34 -0
  456. package/platforms/electrobun/remotes/runtime/src/dev/phase2-smoke.ts +91 -0
  457. package/platforms/electrobun/remotes/runtime/src/dev/phase3-smoke.ts +137 -0
  458. package/platforms/electrobun/remotes/runtime/src/web/index.css +187 -0
  459. package/platforms/electrobun/remotes/runtime/src/web/index.html +76 -0
  460. package/platforms/electrobun/remotes/runtime/src/web/index.ts +192 -0
  461. package/platforms/electrobun/remotes/surface/README.md +201 -0
  462. package/platforms/electrobun/remotes/surface/electrobun.config.ts +38 -0
  463. package/platforms/electrobun/remotes/surface/package.json +12 -0
  464. package/platforms/electrobun/remotes/surface/plugin.json +28 -0
  465. package/platforms/electrobun/remotes/surface/src/bun/worker.ts +132 -0
  466. package/platforms/electrobun/remotes/surface/src/dev/phase4-smoke.ts +566 -0
  467. package/platforms/electrobun/remotes/surface/src/protocol/event-types.ts +84 -0
  468. package/platforms/electrobun/remotes/surface/src/protocol/runtime-client.ts +673 -0
  469. package/platforms/electrobun/remotes/surface/src/web/app.ts +595 -0
  470. package/platforms/electrobun/remotes/surface/src/web/index.css +460 -0
  471. package/platforms/electrobun/remotes/surface/src/web/index.html +466 -0
  472. package/platforms/electrobun/remotes/surface/src/web/index.ts +5 -0
  473. package/platforms/electrobun/remotes/surface/src/web/render.ts +455 -0
  474. package/platforms/electrobun/remotes/surface/src/web/state.ts +427 -0
  475. package/platforms/electrobun/scripts/build-macos-effects.sh +4 -0
  476. package/platforms/electrobun/scripts/ensure-build-folder.ts +28 -0
  477. package/platforms/electrobun/scripts/local-adhoc-sign-macos.ts +159 -159
  478. package/platforms/electrobun/scripts/postwrap-diagnostics.ts +424 -339
  479. package/platforms/electrobun/scripts/postwrap-sign-runtime-macos.ts +302 -271
  480. package/platforms/electrobun/scripts/smoke-test-windows.ps1 +17 -16
  481. package/platforms/electrobun/scripts/smoke-test.sh +5 -7
  482. package/platforms/electrobun/scripts/sync-web-assets.mjs +13 -13
  483. package/platforms/electrobun/scripts/verify-rpc-handlers.ts +109 -110
  484. package/platforms/electrobun/scripts/verify-windows-installer-proof.ps1 +3 -8
  485. package/platforms/electrobun/src/__stubs__/bun-ffi.ts +31 -31
  486. package/platforms/electrobun/src/__stubs__/electrobun-bun.ts +1 -1
  487. package/platforms/electrobun/src/agent-ready-state.ts +8 -8
  488. package/platforms/electrobun/src/agent-reset-from-main.test.ts +162 -0
  489. package/platforms/electrobun/src/agent-reset-from-main.ts +62 -62
  490. package/platforms/electrobun/src/agent-status-rpc.test.ts +95 -0
  491. package/platforms/electrobun/src/agent-status-rpc.ts +156 -0
  492. package/platforms/electrobun/src/api-base.test.ts +247 -0
  493. package/platforms/electrobun/src/api-base.ts +202 -93
  494. package/platforms/electrobun/src/application-menu-action-registry.ts +9 -9
  495. package/platforms/electrobun/src/application-menu.ts +341 -348
  496. package/platforms/electrobun/src/background-notice.ts +36 -36
  497. package/platforms/electrobun/src/boot-progress.test.ts +188 -0
  498. package/platforms/electrobun/src/boot-progress.ts +111 -0
  499. package/platforms/electrobun/src/brand-config.test.ts +39 -0
  500. package/platforms/electrobun/src/brand-config.ts +141 -129
  501. package/platforms/electrobun/src/bridge/browser-tabs-renderer-registry.ts +28 -28
  502. package/platforms/electrobun/src/bridge/electrobun-boot-config.ts +42 -0
  503. package/platforms/electrobun/src/bridge/electrobun-crypto-ready.ts +120 -0
  504. package/platforms/electrobun/src/bridge/electrobun-direct-rpc.ts +342 -357
  505. package/platforms/electrobun/src/bridge/electrobun-stub.ts +13 -13
  506. package/platforms/electrobun/src/browser-workspace-bridge-server.ts +285 -243
  507. package/platforms/electrobun/src/cloud-auth-window.ts +136 -136
  508. package/platforms/electrobun/src/cloud-disconnect-from-main.ts +90 -90
  509. package/platforms/electrobun/src/config-and-auth-rpc.test.ts +256 -0
  510. package/platforms/electrobun/src/config-and-auth-rpc.ts +302 -0
  511. package/platforms/electrobun/src/conversations-and-character-rpc.test.ts +185 -0
  512. package/platforms/electrobun/src/conversations-and-character-rpc.ts +131 -0
  513. package/platforms/electrobun/src/dashboard-rpc.test.ts +200 -0
  514. package/platforms/electrobun/src/dashboard-rpc.ts +344 -0
  515. package/platforms/electrobun/src/database/database-lock.ts +141 -0
  516. package/platforms/electrobun/src/database/database-mode.ts +149 -0
  517. package/platforms/electrobun/src/database/database-recovery.ts +72 -0
  518. package/platforms/electrobun/src/database/database-snapshot.ts +190 -0
  519. package/platforms/electrobun/src/database/database.test.ts +196 -0
  520. package/platforms/electrobun/src/database/index.ts +5 -0
  521. package/platforms/electrobun/src/database/pglite-paths.ts +100 -0
  522. package/platforms/electrobun/src/desktop-deep-link-events.test.ts +30 -0
  523. package/platforms/electrobun/src/desktop-deep-link-events.ts +17 -0
  524. package/platforms/electrobun/src/desktop-http-request.test.ts +102 -73
  525. package/platforms/electrobun/src/desktop-http-request.ts +97 -85
  526. package/platforms/electrobun/src/desktop-pill-config.test.ts +27 -0
  527. package/platforms/electrobun/src/desktop-pill-config.ts +40 -0
  528. package/platforms/electrobun/src/desktop-test-bridge-server.ts +232 -204
  529. package/platforms/electrobun/src/desktop-tray-config.test.ts +87 -0
  530. package/platforms/electrobun/src/desktop-tray-config.ts +84 -0
  531. package/platforms/electrobun/src/devtools-layout.ts +41 -41
  532. package/platforms/electrobun/src/diagnostic-format.test.ts +71 -0
  533. package/platforms/electrobun/src/diagnostic-format.ts +75 -36
  534. package/platforms/electrobun/src/dynamic-view-rpc-schema.test.ts +37 -0
  535. package/platforms/electrobun/src/dynamic-views/README.md +44 -0
  536. package/platforms/electrobun/src/dynamic-views/demo/agent-run-trace.html +135 -0
  537. package/platforms/electrobun/src/dynamic-views/errors.ts +29 -0
  538. package/platforms/electrobun/src/dynamic-views/host.test.ts +353 -0
  539. package/platforms/electrobun/src/dynamic-views/host.ts +332 -0
  540. package/platforms/electrobun/src/dynamic-views/index.ts +57 -0
  541. package/platforms/electrobun/src/dynamic-views/kiosk-canvas.ts +89 -0
  542. package/platforms/electrobun/src/dynamic-views/registry.test.ts +139 -0
  543. package/platforms/electrobun/src/dynamic-views/registry.ts +196 -0
  544. package/platforms/electrobun/src/dynamic-views/session-manager.test.ts +355 -0
  545. package/platforms/electrobun/src/dynamic-views/session-manager.ts +348 -0
  546. package/platforms/electrobun/src/dynamic-views/types.ts +105 -0
  547. package/platforms/electrobun/src/electrobun-boot-config.test.ts +50 -0
  548. package/platforms/electrobun/src/electrobun-config.test.ts +62 -0
  549. package/platforms/electrobun/src/electrobun-crypto-ready.test.ts +65 -0
  550. package/platforms/electrobun/src/electrobun-window-options.ts +25 -0
  551. package/platforms/electrobun/src/extension-rpc.test.ts +88 -0
  552. package/platforms/electrobun/src/extension-rpc.ts +102 -0
  553. package/platforms/electrobun/src/fatal-shutdown.test.ts +12 -16
  554. package/platforms/electrobun/src/fatal-shutdown.ts +1 -1
  555. package/platforms/electrobun/src/first-party-remotes.test.ts +169 -0
  556. package/platforms/electrobun/src/first-party-remotes.ts +297 -0
  557. package/platforms/electrobun/src/first-run-rpc.test.ts +192 -0
  558. package/platforms/electrobun/src/first-run-rpc.ts +146 -0
  559. package/platforms/electrobun/src/floating-chat-window.ts +181 -181
  560. package/platforms/electrobun/src/inbox-rpc.test.ts +123 -0
  561. package/platforms/electrobun/src/inbox-rpc.ts +158 -0
  562. package/platforms/electrobun/src/index.ts +2608 -2098
  563. package/platforms/electrobun/src/kiosk-mode.ts +50 -0
  564. package/platforms/electrobun/src/launch/index.ts +4 -0
  565. package/platforms/electrobun/src/launch/launch-dynamic-view.ts +37 -0
  566. package/platforms/electrobun/src/launch/launch-orchestrator.test.ts +224 -0
  567. package/platforms/electrobun/src/launch/launch-orchestrator.ts +456 -0
  568. package/platforms/electrobun/src/launch/launch-store.test.ts +97 -0
  569. package/platforms/electrobun/src/launch/launch-store.ts +134 -0
  570. package/platforms/electrobun/src/launch/types.ts +103 -0
  571. package/platforms/electrobun/src/launch/views/launch-diagnostics.html +205 -0
  572. package/platforms/electrobun/src/lifecycle/agent-ready-publish.test.ts +50 -0
  573. package/platforms/electrobun/src/lifecycle/agent-ready-publish.ts +27 -0
  574. package/platforms/electrobun/src/lifecycle/api-base-owner.test.ts +83 -0
  575. package/platforms/electrobun/src/lifecycle/api-base-owner.ts +63 -31
  576. package/platforms/electrobun/src/lifecycle/desktop-session-prime.ts +44 -44
  577. package/platforms/electrobun/src/logger.ts +14 -14
  578. package/platforms/electrobun/src/main-window-runtime.ts +83 -83
  579. package/platforms/electrobun/src/main-window-session.test.ts +109 -0
  580. package/platforms/electrobun/src/main-window-session.ts +87 -51
  581. package/platforms/electrobun/src/menu-reset-from-main.ts +158 -158
  582. package/platforms/electrobun/src/native/agent-env.test.ts +52 -0
  583. package/platforms/electrobun/src/native/agent-runtime-layout.test.ts +42 -0
  584. package/platforms/electrobun/src/native/agent-state-dir.test.ts +91 -0
  585. package/platforms/electrobun/src/native/agent.ts +2133 -1682
  586. package/platforms/electrobun/src/native/auth-bridge.test.ts +67 -0
  587. package/platforms/electrobun/src/native/auth-bridge.ts +464 -360
  588. package/platforms/electrobun/src/native/browser-workspace.ts +723 -471
  589. package/platforms/electrobun/src/native/camera.ts +50 -50
  590. package/platforms/electrobun/src/native/canvas.ts +444 -445
  591. package/platforms/electrobun/src/native/credentials.ts +678 -616
  592. package/platforms/electrobun/src/native/desktop-window.test.ts +416 -0
  593. package/platforms/electrobun/src/native/desktop.ts +2200 -2156
  594. package/platforms/electrobun/src/native/editor-bridge.ts +201 -201
  595. package/platforms/electrobun/src/native/file-watcher.ts +154 -154
  596. package/platforms/electrobun/src/native/gateway.ts +179 -180
  597. package/platforms/electrobun/src/native/gpu-window.ts +256 -256
  598. package/platforms/electrobun/src/native/index.ts +82 -74
  599. package/platforms/electrobun/src/native/location.test.ts +44 -0
  600. package/platforms/electrobun/src/native/location.ts +90 -80
  601. package/platforms/electrobun/src/native/loopback-port.ts +60 -60
  602. package/platforms/electrobun/src/native/mac-window-effects.ts +166 -104
  603. package/platforms/electrobun/src/native/music-player.ts +38 -38
  604. package/platforms/electrobun/src/native/permissions-shared.ts +249 -150
  605. package/platforms/electrobun/src/native/permissions.ts +301 -208
  606. package/platforms/electrobun/src/native/power-state.ts +129 -129
  607. package/platforms/electrobun/src/native/remote-plugin-host.test.ts +1394 -0
  608. package/platforms/electrobun/src/native/remote-plugin-host.ts +1541 -0
  609. package/platforms/electrobun/src/native/screencapture.ts +667 -573
  610. package/platforms/electrobun/src/native/steward.ts +207 -204
  611. package/platforms/electrobun/src/native/swabble.ts +68 -324
  612. package/platforms/electrobun/src/native/talkmode.ts +253 -422
  613. package/platforms/electrobun/src/native/webgpu-browser-support.test.ts +18 -0
  614. package/platforms/electrobun/src/native/webgpu-browser-support.ts +165 -147
  615. package/platforms/electrobun/src/native-onboarding.ts +270 -0
  616. package/platforms/electrobun/src/onboarding-overlay-window.ts +141 -0
  617. package/platforms/electrobun/src/persisted-deployment.ts +91 -0
  618. package/platforms/electrobun/src/pill-window.test.ts +91 -0
  619. package/platforms/electrobun/src/pill-window.ts +99 -0
  620. package/platforms/electrobun/src/preload-validation.ts +44 -44
  621. package/platforms/electrobun/src/preload.js +1 -1
  622. package/platforms/electrobun/src/print-electrobun-dev-settings-banner.ts +120 -120
  623. package/platforms/electrobun/src/renderer-api-proxy.test.ts +73 -0
  624. package/platforms/electrobun/src/renderer-api-proxy.ts +86 -0
  625. package/platforms/electrobun/src/renderer-static.test.ts +53 -0
  626. package/platforms/electrobun/src/renderer-static.ts +144 -57
  627. package/platforms/electrobun/src/rpc-handler-slices.ts +121 -0
  628. package/platforms/electrobun/src/rpc-handlers.test.ts +267 -0
  629. package/platforms/electrobun/src/rpc-handlers.ts +1306 -913
  630. package/platforms/electrobun/src/rpc-parse-utils.ts +57 -0
  631. package/platforms/electrobun/src/rpc-port-resolver.test.ts +45 -0
  632. package/platforms/electrobun/src/rpc-port-resolver.ts +31 -0
  633. package/platforms/electrobun/src/rpc-schema.ts +2556 -1619
  634. package/platforms/electrobun/src/runtime-layout.ts +105 -105
  635. package/platforms/electrobun/src/runtime-permissions.ts +95 -95
  636. package/platforms/electrobun/src/runtime-rpc.test.ts +126 -0
  637. package/platforms/electrobun/src/runtime-rpc.ts +237 -0
  638. package/platforms/electrobun/src/screenshot-dev-server.ts +87 -87
  639. package/platforms/electrobun/src/settings-mutations-rpc.test.ts +193 -0
  640. package/platforms/electrobun/src/settings-mutations-rpc.ts +220 -0
  641. package/platforms/electrobun/src/startup-trace.ts +274 -270
  642. package/platforms/electrobun/src/subscription-rpc.test.ts +89 -0
  643. package/platforms/electrobun/src/subscription-rpc.ts +192 -0
  644. package/platforms/electrobun/src/surface-windows.test.ts +355 -0
  645. package/platforms/electrobun/src/surface-windows.ts +410 -410
  646. package/platforms/electrobun/src/trace/README.md +73 -0
  647. package/platforms/electrobun/src/trace/errors.ts +21 -0
  648. package/platforms/electrobun/src/trace/index.ts +40 -0
  649. package/platforms/electrobun/src/trace/trace-dynamic-view.ts +40 -0
  650. package/platforms/electrobun/src/trace/trace-host-requests.ts +473 -0
  651. package/platforms/electrobun/src/trace/trace-service.test.ts +186 -0
  652. package/platforms/electrobun/src/trace/trace-service.ts +324 -0
  653. package/platforms/electrobun/src/trace/trace-store.test.ts +141 -0
  654. package/platforms/electrobun/src/trace/trace-store.ts +551 -0
  655. package/platforms/electrobun/src/trace/types.ts +250 -0
  656. package/platforms/electrobun/src/trace/views/agent-run-trace.html +311 -0
  657. package/platforms/electrobun/src/types/web-speech.d.ts +28 -28
  658. package/platforms/electrobun/src/types.ts +5 -5
  659. package/platforms/electrobun/src/update-availability.test.ts +72 -0
  660. package/platforms/electrobun/src/update-availability.ts +90 -0
  661. package/platforms/electrobun/src/update-rpc.test.ts +83 -0
  662. package/platforms/electrobun/src/update-rpc.ts +123 -0
  663. package/platforms/electrobun/src/voice/README.md +184 -0
  664. package/platforms/electrobun/src/voice/errors.ts +42 -0
  665. package/platforms/electrobun/src/voice/index.ts +78 -0
  666. package/platforms/electrobun/src/voice/types.ts +316 -0
  667. package/platforms/electrobun/src/voice/voice-host-requests.ts +259 -0
  668. package/platforms/electrobun/src/voice/voice-latency-budget.test.ts +66 -0
  669. package/platforms/electrobun/src/voice/voice-latency-budget.ts +243 -0
  670. package/platforms/electrobun/src/voice/voice-live-validation.test.ts +357 -0
  671. package/platforms/electrobun/src/voice/voice-live-validation.ts +838 -0
  672. package/platforms/electrobun/src/voice/voice-pipeline.ts +250 -0
  673. package/platforms/electrobun/src/voice/voice-playback-adapter.ts +31 -0
  674. package/platforms/electrobun/src/voice/voice-runtime-adapter.test.ts +213 -0
  675. package/platforms/electrobun/src/voice/voice-runtime-adapter.ts +687 -0
  676. package/platforms/electrobun/src/voice/voice-service.test.ts +587 -0
  677. package/platforms/electrobun/src/voice/voice-service.ts +1067 -0
  678. package/platforms/electrobun/src/voice/voice-stream-coordinator.test.ts +115 -0
  679. package/platforms/electrobun/src/voice/voice-stream-coordinator.ts +270 -0
  680. package/platforms/electrobun/src/voice/voice-trace.ts +97 -0
  681. package/platforms/electrobun/src/voice/voice-tts-chunker.test.ts +91 -0
  682. package/platforms/electrobun/src/voice/voice-tts-chunker.ts +194 -0
  683. package/platforms/electrobun/src/windows-cef-profile.ts +88 -88
  684. package/platforms/electrobun/tsconfig.json +88 -12
  685. package/platforms/electrobun/update-channels.json +22 -0
  686. package/platforms/electrobun/vitest.electrobun.config.ts +72 -42
  687. package/platforms/ios/App/App/App.entitlements +4 -0
  688. package/platforms/ios/App/App/AppDelegate.swift +80 -18
  689. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ios-marketing-1024.png +0 -0
  690. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-20x20@1x.png +0 -0
  691. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-20x20@2x.png +0 -0
  692. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-29x29@1x.png +0 -0
  693. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-29x29@2x.png +0 -0
  694. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-40x40@1x.png +0 -0
  695. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-40x40@2x.png +0 -0
  696. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-76x76@1x.png +0 -0
  697. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-76x76@2x.png +0 -0
  698. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-ipad-83_5x83_5@2x.png +0 -0
  699. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-20x20@2x.png +0 -0
  700. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-20x20@3x.png +0 -0
  701. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-29x29@2x.png +0 -0
  702. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-29x29@3x.png +0 -0
  703. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-40x40@2x.png +0 -0
  704. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-40x40@3x.png +0 -0
  705. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-60x60@2x.png +0 -0
  706. package/platforms/ios/App/App/Assets.xcassets/AppIcon.appiconset/AppIcon-iphone-60x60@3x.png +0 -0
  707. package/platforms/ios/App/App/Base.lproj/LaunchScreen.storyboard +4 -9
  708. package/platforms/ios/App/App/ComputerUseBridge.swift +589 -0
  709. package/platforms/ios/App/App/DeviceActivityMonitorExtension/DeviceActivityMonitorExtension.entitlements +12 -0
  710. package/platforms/ios/App/App/DeviceActivityMonitorExtension/DeviceActivityMonitorExtension.swift +34 -0
  711. package/platforms/ios/App/App/DeviceActivityMonitorExtension/Info.plist +29 -0
  712. package/platforms/ios/App/App/DeviceActivityReportExtension/DeviceActivityReportExtension.entitlements +12 -0
  713. package/platforms/ios/App/App/DeviceActivityReportExtension/DeviceActivityReportExtension.swift +53 -0
  714. package/platforms/ios/App/App/DeviceActivityReportExtension/Info.plist +27 -0
  715. package/platforms/ios/App/App/ElizaAppIntents.swift +183 -0
  716. package/platforms/ios/App/App/ElizaIntentPlugin.swift +342 -5
  717. package/platforms/ios/App/App/Info.plist +17 -1
  718. package/platforms/ios/App/App/runners/eliza-tasks.js +177 -0
  719. package/platforms/ios/App/App.xcodeproj/project.pbxproj +262 -6
  720. package/platforms/ios/App/BroadcastExtension/SampleHandler.swift +100 -0
  721. package/platforms/ios/App/Podfile +5 -0
  722. package/platforms/ios/App/Podfile.lock +83 -59
  723. package/register-runtime-hooks.js +11 -5
  724. package/registry/index.d.ts +1 -5
  725. package/registry/index.d.ts.map +1 -1
  726. package/registry/index.js +10 -68
  727. package/runtime/android-avf-microdroid-bridge.d.ts +29 -0
  728. package/runtime/android-avf-microdroid-bridge.d.ts.map +1 -0
  729. package/runtime/android-avf-microdroid-bridge.js +149 -0
  730. package/runtime/api-dev-settings-banner.d.ts.map +1 -1
  731. package/runtime/api-dev-settings-banner.js +5 -13
  732. package/runtime/app-core-runtime-hooks.d.ts +21 -0
  733. package/runtime/app-core-runtime-hooks.d.ts.map +1 -0
  734. package/runtime/app-core-runtime-hooks.js +10 -0
  735. package/runtime/autonomy-policy.d.ts +2 -0
  736. package/runtime/autonomy-policy.d.ts.map +1 -0
  737. package/runtime/autonomy-policy.js +4 -0
  738. package/runtime/bundled-fused-lib.d.ts +36 -0
  739. package/runtime/bundled-fused-lib.d.ts.map +1 -0
  740. package/runtime/bundled-fused-lib.js +77 -0
  741. package/runtime/channel-plugin-map.d.ts +1 -5
  742. package/runtime/channel-plugin-map.d.ts.map +1 -1
  743. package/runtime/channel-plugin-map.js +11 -31
  744. package/runtime/desktop/AppWindowRenderer.d.ts +17 -0
  745. package/runtime/desktop/AppWindowRenderer.d.ts.map +1 -0
  746. package/runtime/desktop/AppWindowRenderer.js +360 -0
  747. package/runtime/desktop/DesktopSurfaceNavigationRuntime.d.ts +2 -0
  748. package/runtime/desktop/DesktopSurfaceNavigationRuntime.d.ts.map +1 -0
  749. package/runtime/desktop/DesktopSurfaceNavigationRuntime.js +41 -0
  750. package/runtime/desktop/DesktopTrayRuntime.d.ts +2 -0
  751. package/runtime/desktop/DesktopTrayRuntime.d.ts.map +1 -0
  752. package/runtime/desktop/DesktopTrayRuntime.js +174 -0
  753. package/runtime/desktop/DetachedShellRoot.d.ts +10 -0
  754. package/runtime/desktop/DetachedShellRoot.d.ts.map +1 -0
  755. package/runtime/desktop/DetachedShellRoot.js +111 -0
  756. package/runtime/desktop/index.d.ts +6 -0
  757. package/runtime/desktop/index.d.ts.map +1 -0
  758. package/runtime/desktop/index.js +5 -0
  759. package/runtime/desktop/tray-menu.d.ts +20 -0
  760. package/runtime/desktop/tray-menu.d.ts.map +1 -0
  761. package/runtime/desktop/tray-menu.js +143 -0
  762. package/runtime/dev-server.d.ts +1 -1
  763. package/runtime/dev-server.d.ts.map +1 -1
  764. package/runtime/dev-server.js +104 -19
  765. package/runtime/eliza.d.ts +75 -1
  766. package/runtime/eliza.d.ts.map +1 -1
  767. package/runtime/eliza.js +606 -122
  768. package/runtime/ensure-text-to-speech-handler.d.ts.map +1 -1
  769. package/runtime/ensure-text-to-speech-handler.js +10 -3
  770. package/runtime/mobile-safe-runtime.d.ts +181 -2
  771. package/runtime/mobile-safe-runtime.d.ts.map +1 -1
  772. package/runtime/mobile-safe-runtime.js +1019 -12
  773. package/runtime/mode/remote-forwarder.d.ts.map +1 -1
  774. package/runtime/mode/remote-forwarder.js +2 -2
  775. package/runtime/mode/route-mode-guard.d.ts +2 -3
  776. package/runtime/mode/route-mode-guard.d.ts.map +1 -1
  777. package/runtime/mode/route-mode-guard.js +4 -5
  778. package/runtime/mode/route-mode-matrix.d.ts +1 -1
  779. package/runtime/mode/route-mode-matrix.d.ts.map +1 -1
  780. package/runtime/mode/route-mode-matrix.js +14 -1
  781. package/runtime/mode/runtime-mode.d.ts +1 -1
  782. package/runtime/mode/runtime-mode.js +1 -1
  783. package/runtime/runtime-bootstrap-policy.d.ts.map +1 -1
  784. package/runtime/runtime-bootstrap-policy.js +14 -2
  785. package/runtime/telegram-standalone-handler.d.ts.map +1 -1
  786. package/runtime/telegram-standalone-handler.js +10 -9
  787. package/runtime/tts-cache-wiring.d.ts +29 -0
  788. package/runtime/tts-cache-wiring.d.ts.map +1 -0
  789. package/runtime/tts-cache-wiring.js +114 -0
  790. package/runtime/voice-warmup.d.ts +66 -0
  791. package/runtime/voice-warmup.d.ts.map +1 -0
  792. package/runtime/voice-warmup.js +101 -0
  793. package/scripts/android-sms-gateway-template.test.mjs +1014 -0
  794. package/scripts/aosp/README.md +29 -16
  795. package/scripts/aosp/compile-libllama-zig-pin.test.mjs +147 -0
  796. package/scripts/aosp/compile-libllama.mjs +1750 -267
  797. package/scripts/aosp/compile-libllama.test.mjs +133 -0
  798. package/scripts/aosp/compile-shim.mjs +68 -19
  799. package/scripts/aosp/deploy-pixel.mjs +405 -0
  800. package/scripts/aosp/lib/load-variant-config.mjs +3 -3
  801. package/scripts/aosp/llama-cpp-patches/README.md +8 -8
  802. package/scripts/aosp/llama-cpp-patches/apply-patches.mjs +23 -6
  803. package/scripts/aosp/llama-cpp-patches/polarquant/README.md +37 -0
  804. package/scripts/aosp/llama-cpp-patches/qjl/README.md +37 -0
  805. package/scripts/aosp/seccomp-shim/sigsys-handler-arm64.c +169 -0
  806. package/scripts/aosp/seccomp-shim/sigsys-handler-riscv64.c +217 -0
  807. package/scripts/aosp/smoke-cuttlefish.mjs +34 -4
  808. package/scripts/aosp/stage-default-models.mjs +59 -35
  809. package/scripts/aosp/stage-default-models.test.mjs +23 -0
  810. package/scripts/aosp/variant-config-schema.ts +2 -2
  811. package/scripts/assert-required-bundled-packages.test.ts +555 -0
  812. package/scripts/audit-apple-store-sandbox.mjs +146 -0
  813. package/scripts/audit-live-test-surface.mjs +5 -2
  814. package/scripts/build-capacitor-app.mjs +21 -0
  815. package/scripts/build-flatpak.mjs +5 -5
  816. package/scripts/build-helpers/arm64-simd.mjs +72 -0
  817. package/scripts/build-helpers/omnivoice-merged.mjs +87 -0
  818. package/scripts/build-helpers/verify-fused-symbols.mjs +600 -0
  819. package/scripts/build-helpers/verify-fused-symbols.test.mjs +0 -0
  820. package/scripts/build-image.sh +1 -1
  821. package/scripts/build-llama-cpp-mtp.mjs +488 -0
  822. package/scripts/build-native-plugins.mjs +230 -18
  823. package/scripts/build-patched-electrobun-cli.mjs +68 -10
  824. package/scripts/build-win.mjs +1 -1
  825. package/scripts/bun-riscv64/Dockerfile +418 -0
  826. package/scripts/bun-riscv64/README.md +316 -0
  827. package/scripts/bun-riscv64/build.sh +469 -0
  828. package/scripts/bun-riscv64/bun-patches/0001-config-add-riscv64-arch.patch +74 -0
  829. package/scripts/bun-riscv64/bun-patches/0002-flags-add-riscv64-march-mabi.patch +16 -0
  830. package/scripts/bun-riscv64/bun-patches/0003-zig-add-riscv64-target-triple-and-cpu.patch +26 -0
  831. package/scripts/bun-riscv64/bun-patches/0004-webkit-force-local-mode-on-riscv64.patch +33 -0
  832. package/scripts/bun-riscv64/bun-patches/0005-tinycc-disable-on-riscv64.patch +16 -0
  833. package/scripts/bun-riscv64/bun-patches/0006-build-add-riscv64-cli-validation.patch +15 -0
  834. package/scripts/bun-riscv64/bun-patches/0007-deps-per-dep-riscv64-checks.patch +24 -0
  835. package/scripts/bun-riscv64/bun-patches/0008-source-stabilize-riscv64-musl-build.patch +226 -0
  836. package/scripts/bun-riscv64/bun-patches/0009-disable-wasm-streaming-hooks-for-c-loop.patch +162 -0
  837. package/scripts/bun-riscv64/bun-patches/0010-disable-inspector-profiler-for-riscv64-c-loop.patch +80 -0
  838. package/scripts/bun-riscv64/bun-patches/0011-process-arch-add-riscv64.patch +23 -0
  839. package/scripts/bun-riscv64/bun-patches/0012-cpu-features-add-riscv64-fallback.patch +13 -0
  840. package/scripts/bun-riscv64/bun-patches/0013-disable-console-inspector-hooks-for-riscv64-c-loop.patch +43 -0
  841. package/scripts/bun-riscv64/bun-patches/0014-disable-custom-inspector-dispatchers-on-riscv64.patch +127 -0
  842. package/scripts/bun-riscv64/bun-patches/0015-disable-jsc-profiler-builtins-on-riscv64.patch +75 -0
  843. package/scripts/bun-riscv64/bun-patches/0016-node-vm-disable-jit-cached-data-on-riscv64-c-loop.patch +96 -0
  844. package/scripts/bun-riscv64/bun-patches/0017-disable-performance-domjit-signature-on-riscv64-c-loop.patch +34 -0
  845. package/scripts/bun-riscv64/bun-patches/0018-fix-serialized-script-identifier-big-endian-path.patch +19 -0
  846. package/scripts/bun-riscv64/bun-patches/0019-add-wtf-timer-fire-bridge-for-c-loop.patch +24 -0
  847. package/scripts/bun-riscv64/bun-patches/0020-run-riscv64-smoke-test-under-qemu.patch +13 -0
  848. package/scripts/bun-riscv64/bun-patches/0021-fix-riscv64-linux-open-flags.patch +25 -0
  849. package/scripts/bun-riscv64/bun-patches/0022-zlib-riscv64-generic-kernels.patch +25 -0
  850. package/scripts/bun-riscv64/bun-patches/README.md +127 -0
  851. package/scripts/bun-riscv64/bun-version.json +202 -0
  852. package/scripts/bun-riscv64/run-build.sh +162 -0
  853. package/scripts/bun-riscv64/rust-core/0001-riscv64-rust-core-port.patch +868 -0
  854. package/scripts/bun-riscv64/rust-core/0002-second-wave-riscv64-source-gaps.patch +130 -0
  855. package/scripts/bun-riscv64/rust-core/0003-third-wave-riscv64-crash-handler-gaps.patch +78 -0
  856. package/scripts/bun-riscv64/rust-core/0004-rust-target-cpu-riscv64.patch +39 -0
  857. package/scripts/bun-riscv64/rust-core/0005-fifth-wave-riscv64-source-gaps.patch +96 -0
  858. package/scripts/bun-riscv64/rust-core/0006-cpp-wasm-and-inspector-guards-riscv64.patch +91 -0
  859. package/scripts/bun-riscv64/rust-core/0007-bun-alloc-max-align-t-riscv64.patch +36 -0
  860. package/scripts/bun-riscv64/rust-core/0008-workspace-lints-warn-not-deny-riscv64.patch +75 -0
  861. package/scripts/bun-riscv64/rust-core/0009-zigglobalobject-wasm-streaming-guards-riscv64.patch +109 -0
  862. package/scripts/bun-riscv64/rust-core/0010-tcc-externs-stub-on-riscv64.patch +62 -0
  863. package/scripts/bun-riscv64/rust-core/0011-clippy-ptr-cast-lints-warn-riscv64.patch +61 -0
  864. package/scripts/bun-riscv64/rust-core/README.md +80 -0
  865. package/scripts/bun-riscv64/rust-core/webkit-patches/0003-disable-dfg-ftl-on-riscv64.patch +60 -0
  866. package/scripts/bun-riscv64/rust-core/webkit-patches/0004-riscv64-do-not-force-wasm-in-c-loop.patch +31 -0
  867. package/scripts/bun-riscv64/rust-core/webkit-patches/0005-domjit-effect-allow-no-dfg-c-loop.patch +40 -0
  868. package/scripts/bun-riscv64/rust-core/webkit-patches/0006-disable-usewasm-when-webassembly-compiled-out.patch +33 -0
  869. package/scripts/bun-riscv64/rust-core/webkit-patches/0007-restore-dropped-includes-and-llint-fwd-decl.patch +31 -0
  870. package/scripts/bun-riscv64/validate.sh +264 -0
  871. package/scripts/bun-riscv64/webkit-patches/0001-cherry-pick-llint-riscv64.recipe +155 -0
  872. package/scripts/bun-riscv64/webkit-patches/0002-cherry-pick-baseline-jit-riscv64.recipe +40 -0
  873. package/scripts/bun-riscv64/webkit-patches/0003-disable-dfg-ftl-on-riscv64.patch +60 -0
  874. package/scripts/bun-riscv64/webkit-patches/0004-riscv64-do-not-force-wasm-in-c-loop.patch +31 -0
  875. package/scripts/bun-riscv64/webkit-patches/0005-domjit-effect-allow-no-dfg-c-loop.patch +40 -0
  876. package/scripts/bun-riscv64/webkit-patches/0006-disable-usewasm-when-webassembly-compiled-out.patch +33 -0
  877. package/scripts/bun-riscv64/webkit-patches/0007-restore-dropped-includes-and-llint-fwd-decl.patch +72 -0
  878. package/scripts/bun-riscv64/webkit-patches/README.md +146 -0
  879. package/scripts/check-homepage-public-readiness.mjs +353 -0
  880. package/scripts/check-homepage-release-data.mjs +110 -0
  881. package/scripts/check-i18n.mjs +1 -1
  882. package/scripts/check-real-local-chat.ts +147 -0
  883. package/scripts/check-real-local-provisioning.ts +104 -0
  884. package/scripts/check-real-local-reset.ts +249 -0
  885. package/scripts/check-sms-gateway-completion-audit.mjs +428 -0
  886. package/scripts/check-sms-gateway-readiness.mjs +266 -0
  887. package/scripts/clean-repo.mjs +5 -5
  888. package/scripts/codesign-mas.mjs +222 -16
  889. package/scripts/collect-docker-runtime-deps.mjs +229 -0
  890. package/scripts/continue-sms-gateway-work.mjs +121 -0
  891. package/scripts/copy-runtime-node-modules.ts +1050 -195
  892. package/scripts/deploy-cloud-api-production-gateway.mjs +52 -0
  893. package/scripts/desktop-build.mjs +965 -98
  894. package/scripts/dev-platform.mjs +346 -102
  895. package/scripts/dev-startup-smoke.mjs +248 -0
  896. package/scripts/dev-ui.mjs +418 -176
  897. package/scripts/disable-local-eliza-workspace.mjs +35 -0
  898. package/scripts/docker-ci-smoke.sh +465 -96
  899. package/scripts/docker-entrypoint.sh +62 -1
  900. package/scripts/docker-entrypoint.test.ts +283 -0
  901. package/scripts/ensure-avatars.mjs +2 -2
  902. package/scripts/ensure-electrobun-core.mjs +1 -1
  903. package/scripts/ensure-generated-core-proto-js.mjs +1 -1
  904. package/scripts/ensure-type-package-aliases.mjs +62 -5
  905. package/scripts/ensure-vision-deps.mjs +20 -1
  906. package/scripts/entry.ts +1 -1
  907. package/scripts/ffi-stub/Makefile +64 -0
  908. package/scripts/ffi-stub/README.md +391 -0
  909. package/scripts/ffi-stub/asr-ffi-smoke.ts +139 -0
  910. package/scripts/ffi-stub/ffi-stub.c +539 -0
  911. package/scripts/ffi-stub/ffi.h +538 -0
  912. package/scripts/ffi-stub/libelizainference_stub.so +0 -0
  913. package/scripts/ffi-stub/tts-stream-ffi-smoke.ts +349 -0
  914. package/scripts/generate-first-run-voicelines.mjs +194 -0
  915. package/scripts/generate-plugin-index.js +4 -3
  916. package/scripts/generate-static-asset-manifest.mjs +1 -1
  917. package/scripts/i18n-dynamic-keys.json +5 -5
  918. package/scripts/init-submodules.mjs +2 -2
  919. package/scripts/install-android-sms-gateway.md +177 -0
  920. package/scripts/install-android-sms-gateway.mjs +1088 -0
  921. package/scripts/ios-xcframework/README.md +74 -72
  922. package/scripts/ios-xcframework/build-xcframework.mjs +204 -43
  923. package/scripts/ios-xcframework/run-physical-device-smoke.mjs +1943 -0
  924. package/scripts/ios-xcframework/runtime-symbol-shim.c +450 -0
  925. package/scripts/kernel-patches/cpu-polar-kernels.mjs +441 -0
  926. package/scripts/kernel-patches/cpu-simd-kernels.mjs +253 -0
  927. package/scripts/kernel-patches/cpu-thread-parallelism.mjs +368 -0
  928. package/scripts/kernel-patches/cuda-kernels.mjs +117 -0
  929. package/scripts/kernel-patches/metal-kernels.mjs +1698 -109
  930. package/scripts/kernel-patches/server-omnivoice-route.mjs +718 -0
  931. package/scripts/kernel-patches/server-structured-output.mjs +279 -0
  932. package/scripts/kernel-patches/vulkan-dispatch-log.mjs +166 -0
  933. package/scripts/kernel-patches/vulkan-dispatch-log.test.mjs +50 -0
  934. package/scripts/kernel-patches/vulkan-dispatch-patches/01-vulkan-shaders-gen.patch +30 -16
  935. package/scripts/kernel-patches/vulkan-dispatch-patches/02-ggml-vulkan-pipelines.patch +75 -30
  936. package/scripts/kernel-patches/vulkan-kernels.mjs +800 -49
  937. package/scripts/lib/agent-source-watcher.mjs +173 -0
  938. package/scripts/lib/agent-source-watcher.test.mjs +184 -0
  939. package/scripts/lib/api-supervisor.mjs +78 -9
  940. package/scripts/lib/api-supervisor.test.mjs +121 -0
  941. package/scripts/lib/app-dir.mjs +2 -16
  942. package/scripts/lib/apple-entitlement-audit.mjs +655 -0
  943. package/scripts/lib/apple-entitlement-audit.test.mjs +144 -0
  944. package/scripts/lib/artifact-staleness.d.mts +33 -0
  945. package/scripts/lib/artifact-staleness.mjs +122 -0
  946. package/scripts/lib/artifact-staleness.test.mts +98 -0
  947. package/scripts/lib/bun-version-guard.mjs +13 -13
  948. package/scripts/lib/capacitor-plugin-build-needed.mjs +4 -3
  949. package/scripts/lib/capacitor-plugin-names.mjs +30 -14
  950. package/scripts/lib/desktop-preflight.mjs +9 -5
  951. package/scripts/lib/desktop-startup-embedding-warmup-policy.mjs +51 -0
  952. package/scripts/lib/desktop-startup-embedding-warmup-policy.test.mjs +55 -0
  953. package/scripts/lib/duet-bridge.d.mts +63 -0
  954. package/scripts/lib/duet-bridge.mjs +193 -0
  955. package/scripts/lib/ios-plist-url-scheme.mjs +80 -0
  956. package/scripts/lib/mobile-build-decisions.mjs +97 -0
  957. package/scripts/lib/node-path-env.mjs +4 -2
  958. package/scripts/lib/orchestrator-desktop-dev-banner.mjs +12 -3
  959. package/scripts/lib/patch-bun-exports.mjs +90 -27
  960. package/scripts/lib/patch-bun-exports.test.mjs +79 -0
  961. package/scripts/lib/renderer-build-action.mjs +35 -0
  962. package/scripts/lib/renderer-build-action.test.mjs +70 -0
  963. package/scripts/lib/renderer-build-manifest.d.mts +65 -0
  964. package/scripts/lib/renderer-build-manifest.mjs +257 -0
  965. package/scripts/lib/renderer-build-manifest.test.mts +277 -0
  966. package/scripts/lib/stage-android-agent.mjs +757 -99
  967. package/scripts/lib/sync-eliza-env-aliases.mjs +3 -25
  968. package/scripts/lib/ui-smoke-stub-decision.mjs +33 -0
  969. package/scripts/lib/ui-smoke-stub-decision.test.mjs +46 -0
  970. package/scripts/lib/verify-ondevice-artifact.d.mts +16 -0
  971. package/scripts/lib/verify-ondevice-artifact.mjs +61 -0
  972. package/scripts/lib/verify-ondevice-artifact.test.mts +98 -0
  973. package/scripts/lib/vite-renderer-dist-stale.mjs +5 -0
  974. package/scripts/lib/voice-latency-report.mjs +154 -0
  975. package/scripts/lib/websocket-pending-queue.ts +92 -0
  976. package/scripts/lifeops-prompt-benchmark.ts +21 -12
  977. package/scripts/link-docker-local-app-packages.mjs +109 -36
  978. package/scripts/local-stt-bench.ts +192 -0
  979. package/scripts/maintain-cloud-api-production-gateway.mjs +54 -0
  980. package/scripts/mas-smoke.mjs +459 -0
  981. package/scripts/mas-smoke.test.mjs +220 -0
  982. package/scripts/mobile-auth-simulator-smoke.mjs +0 -1
  983. package/scripts/normalize-eliza-capture.ts +97 -0
  984. package/scripts/omnivoice-fuse/prepare.mjs +2543 -23
  985. package/scripts/pack-upstreams.mjs +65 -5
  986. package/scripts/package-electrobun-linux.mjs +303 -0
  987. package/scripts/patch-deps.mjs +5 -3
  988. package/scripts/patches/llama-asr-word-timestamps.patch +218 -0
  989. package/scripts/playwright-ui-live-stack.ts +588 -157
  990. package/scripts/playwright-ui-smoke-api-stub.mjs +3495 -111
  991. package/scripts/pre-review-local.mjs +2 -2
  992. package/scripts/prepare-ios-cocoapods.sh +41 -3
  993. package/scripts/release-check.ts +180 -84
  994. package/scripts/relink-workspace-packages-to-dist.mjs +21 -4
  995. package/scripts/rt.mjs +16 -1
  996. package/scripts/run-biome-check.mjs +1 -1
  997. package/scripts/run-coding-agent-e2e.mjs +3 -3
  998. package/scripts/run-eliza-app-core-script.mjs +34 -0
  999. package/scripts/run-local-plugin-live-smoke.mjs +66 -2
  1000. package/scripts/run-mobile-build-android-app-actions.test.mjs +426 -0
  1001. package/scripts/run-mobile-build-brand-separation.test.mts +33 -0
  1002. package/scripts/run-mobile-build-ios-engine-gate.test.mjs +85 -0
  1003. package/scripts/run-mobile-build-mtp-staleness.test.mts +198 -0
  1004. package/scripts/run-mobile-build.mjs +5082 -610
  1005. package/scripts/run-node-runtime.mjs +184 -7
  1006. package/scripts/run-node-runtime.test.mjs +167 -0
  1007. package/scripts/run-node-tsx.mjs +80 -33
  1008. package/scripts/run-node.mjs +41 -1
  1009. package/scripts/run-production-build.mjs +34 -27
  1010. package/scripts/run-release-check.mjs +19 -0
  1011. package/scripts/run-release-contract-suite.mjs +107 -14
  1012. package/scripts/run-ui-smoke-playwright-suite.mjs +0 -2
  1013. package/scripts/runtime-package-manifest.ts +21 -3
  1014. package/scripts/serve-real-local-agent.ts +72 -0
  1015. package/scripts/setup-upstreams.mjs +42 -1
  1016. package/scripts/sms-gateway-status.mjs +194 -0
  1017. package/scripts/stage-android-agent.test.mjs +97 -0
  1018. package/scripts/stage-desktop-fused-lib.mjs +453 -0
  1019. package/scripts/stage-elizavoice-lib.mjs +363 -0
  1020. package/scripts/startup-integration-script-drift.test.ts +82 -4
  1021. package/scripts/streaming-pipeline-bench.ts +543 -0
  1022. package/scripts/sync-homepage-porkbun-dns.mjs +262 -0
  1023. package/scripts/test-sms-gateway-software.mjs +100 -0
  1024. package/scripts/type-audit.mjs +1 -1
  1025. package/scripts/validate-bluebubbles-outbound.mjs +293 -0
  1026. package/scripts/validate-cdn-assets.mjs +15 -7
  1027. package/scripts/validate-regression-matrix.mjs +109 -8
  1028. package/scripts/verify-android-sms-gateway-e2e.mjs +362 -0
  1029. package/scripts/verify-bluebubbles-gateway-e2e.mjs +191 -0
  1030. package/scripts/verify-bluebubbles-inbound-readiness.mjs +88 -0
  1031. package/scripts/verify-cloud-api-production-deploy.mjs +87 -0
  1032. package/scripts/verify-cloud-sms-onboarding-flow.mjs +336 -0
  1033. package/scripts/verify-ondevice-artifact.mjs +111 -0
  1034. package/scripts/voice/freeze-voice.mjs +521 -0
  1035. package/scripts/voice-attribution-smoke.ts +644 -0
  1036. package/scripts/voice-create-profile.mjs +379 -0
  1037. package/scripts/voice-duet.mjs +1355 -0
  1038. package/scripts/voice-e2e-hardware.ts +877 -0
  1039. package/scripts/voice-interactive.mjs +1721 -0
  1040. package/scripts/voice-latency-report.mjs +96 -0
  1041. package/scripts/voice-latency-report.test.ts +176 -0
  1042. package/scripts/voice-preset/build-default-voice-preset.mjs +249 -0
  1043. package/scripts/voice-preset/build-onboarding-voice.mjs +281 -0
  1044. package/scripts/watch-sms-gateway-readiness.mjs +303 -0
  1045. package/scripts/write-homepage-release-data.mjs +458 -26
  1046. package/security/agent-vault-id.d.ts +2 -2
  1047. package/security/agent-vault-id.js +1 -1
  1048. package/security/hydrate-wallet-keys-from-platform-store.d.ts.map +1 -1
  1049. package/security/hydrate-wallet-keys-from-platform-store.js +23 -14
  1050. package/security/platform-secure-store-node.d.ts +3 -3
  1051. package/security/platform-secure-store-node.js +3 -3
  1052. package/security/wallet-os-store-actions.d.ts +0 -9
  1053. package/security/wallet-os-store-actions.d.ts.map +1 -1
  1054. package/security/wallet-os-store-actions.js +3 -10
  1055. package/services/account-pool.d.ts +23 -14
  1056. package/services/account-pool.d.ts.map +1 -1
  1057. package/services/account-pool.js +86 -24
  1058. package/services/account-usage.d.ts.map +1 -1
  1059. package/services/account-usage.js +2 -5
  1060. package/services/ambient-audio/consent.d.ts +9 -0
  1061. package/services/ambient-audio/consent.d.ts.map +1 -0
  1062. package/services/ambient-audio/consent.js +28 -0
  1063. package/services/ambient-audio/index.d.ts +7 -0
  1064. package/services/ambient-audio/index.d.ts.map +1 -0
  1065. package/services/ambient-audio/index.js +4 -0
  1066. package/services/ambient-audio/replay-buffer.d.ts +14 -0
  1067. package/services/ambient-audio/replay-buffer.d.ts.map +1 -0
  1068. package/services/ambient-audio/replay-buffer.js +66 -0
  1069. package/services/ambient-audio/response-gate.d.ts +3 -0
  1070. package/services/ambient-audio/response-gate.d.ts.map +1 -0
  1071. package/services/ambient-audio/response-gate.js +33 -0
  1072. package/services/ambient-audio/service.d.ts +22 -0
  1073. package/services/ambient-audio/service.d.ts.map +1 -0
  1074. package/services/ambient-audio/service.js +47 -0
  1075. package/services/ambient-audio/types.d.ts +42 -0
  1076. package/services/ambient-audio/types.d.ts.map +1 -0
  1077. package/services/app-updates/update-policy.d.ts +64 -0
  1078. package/services/app-updates/update-policy.d.ts.map +1 -0
  1079. package/services/app-updates/update-policy.js +228 -0
  1080. package/services/auth-store.d.ts +37 -1
  1081. package/services/auth-store.d.ts.map +1 -1
  1082. package/services/auth-store.js +59 -26
  1083. package/services/cloud-jwks-store.d.ts +3 -3
  1084. package/services/cloud-jwks-store.d.ts.map +1 -1
  1085. package/services/cloud-jwks-store.js +5 -8
  1086. package/services/coding-account-bridge.d.ts +71 -0
  1087. package/services/coding-account-bridge.d.ts.map +1 -0
  1088. package/services/coding-account-bridge.js +267 -0
  1089. package/services/connector-target-catalog.d.ts +11 -4
  1090. package/services/connector-target-catalog.d.ts.map +1 -1
  1091. package/services/connector-target-catalog.js +7 -4
  1092. package/services/credential-tunnel-service.d.ts +66 -0
  1093. package/services/credential-tunnel-service.d.ts.map +1 -0
  1094. package/services/credential-tunnel-service.js +227 -0
  1095. package/services/github-credentials.d.ts +1 -1
  1096. package/services/github-credentials.js +1 -1
  1097. package/services/inference-abort.d.ts +47 -0
  1098. package/services/inference-abort.d.ts.map +1 -0
  1099. package/services/inference-abort.js +76 -0
  1100. package/services/persistence.d.ts +2 -3
  1101. package/services/persistence.d.ts.map +1 -1
  1102. package/services/persistence.js +2 -3
  1103. package/services/phrase-chunked-tts.d.ts +136 -0
  1104. package/services/phrase-chunked-tts.d.ts.map +1 -0
  1105. package/services/phrase-chunked-tts.js +208 -0
  1106. package/services/sandbox-registry.d.ts +78 -0
  1107. package/services/sandbox-registry.d.ts.map +1 -0
  1108. package/services/sandbox-registry.js +323 -0
  1109. package/services/secrets-manager-installer.d.ts +8 -1
  1110. package/services/secrets-manager-installer.d.ts.map +1 -1
  1111. package/services/secrets-manager-installer.js +27 -2
  1112. package/services/sensitive-requests/cloud-link-adapter.d.ts +15 -0
  1113. package/services/sensitive-requests/cloud-link-adapter.d.ts.map +1 -0
  1114. package/services/sensitive-requests/cloud-link-adapter.js +73 -0
  1115. package/services/sensitive-requests/index.d.ts +27 -0
  1116. package/services/sensitive-requests/index.d.ts.map +1 -0
  1117. package/services/sensitive-requests/index.js +51 -0
  1118. package/services/sensitive-requests/instruct-dm-only-adapter.d.ts +14 -0
  1119. package/services/sensitive-requests/instruct-dm-only-adapter.d.ts.map +1 -0
  1120. package/services/sensitive-requests/instruct-dm-only-adapter.js +22 -0
  1121. package/services/sensitive-requests/owner-app-inline-adapter.d.ts +3 -0
  1122. package/services/sensitive-requests/owner-app-inline-adapter.d.ts.map +1 -0
  1123. package/services/sensitive-requests/owner-app-inline-adapter.js +146 -0
  1124. package/services/sensitive-requests/owner-app-oauth-adapter.d.ts +3 -0
  1125. package/services/sensitive-requests/owner-app-oauth-adapter.d.ts.map +1 -0
  1126. package/services/sensitive-requests/owner-app-oauth-adapter.js +156 -0
  1127. package/services/sensitive-requests/public-link-adapter.d.ts +14 -0
  1128. package/services/sensitive-requests/public-link-adapter.d.ts.map +1 -0
  1129. package/services/sensitive-requests/public-link-adapter.js +86 -0
  1130. package/services/sensitive-requests/tunnel-link-adapter.d.ts +17 -0
  1131. package/services/sensitive-requests/tunnel-link-adapter.d.ts.map +1 -0
  1132. package/services/sensitive-requests/tunnel-link-adapter.js +38 -0
  1133. package/services/steward-credentials.d.ts +1 -1
  1134. package/services/steward-credentials.d.ts.map +1 -1
  1135. package/services/steward-credentials.js +10 -6
  1136. package/services/steward-sidecar/health-check.d.ts.map +1 -1
  1137. package/services/steward-sidecar/health-check.js +4 -3
  1138. package/services/steward-sidecar/process-management.d.ts +1 -1
  1139. package/services/steward-sidecar/process-management.d.ts.map +1 -1
  1140. package/services/steward-sidecar/process-management.js +9 -3
  1141. package/services/steward-sidecar/types.d.ts +1 -1
  1142. package/services/steward-sidecar/types.d.ts.map +1 -1
  1143. package/services/steward-sidecar/wallet-setup.d.ts +1 -1
  1144. package/services/steward-sidecar/wallet-setup.d.ts.map +1 -1
  1145. package/services/steward-sidecar/wallet-setup.js +8 -7
  1146. package/services/steward-sidecar.d.ts +5 -5
  1147. package/services/steward-sidecar.d.ts.map +1 -1
  1148. package/services/steward-sidecar.js +27 -19
  1149. package/services/task-host-capabilities.d.ts +60 -0
  1150. package/services/task-host-capabilities.d.ts.map +1 -0
  1151. package/services/task-host-capabilities.js +122 -0
  1152. package/services/tool-call-cache/index.d.ts +2 -2
  1153. package/services/tool-call-cache/index.d.ts.map +1 -1
  1154. package/services/tool-call-cache/index.js +1 -1
  1155. package/services/trigger-event-bridge.js +1 -1
  1156. package/services/tunnel-to-mobile/index.d.ts +2 -0
  1157. package/services/tunnel-to-mobile/index.d.ts.map +1 -0
  1158. package/services/tunnel-to-mobile/index.js +1 -0
  1159. package/services/tunnel-to-mobile/tunnel-to-mobile-client.d.ts +105 -0
  1160. package/services/tunnel-to-mobile/tunnel-to-mobile-client.d.ts.map +1 -0
  1161. package/services/tunnel-to-mobile/tunnel-to-mobile-client.js +190 -0
  1162. package/services/vault-bootstrap.d.ts.map +1 -1
  1163. package/services/vault-bootstrap.js +52 -21
  1164. package/services/vault-mirror.d.ts +1 -1
  1165. package/services/vault-mirror.d.ts.map +1 -1
  1166. package/services/vault-mirror.js +29 -6
  1167. package/services/voice-profiles/diarization-pipeline.d.ts +6 -0
  1168. package/services/voice-profiles/diarization-pipeline.d.ts.map +1 -0
  1169. package/services/voice-profiles/diarization-pipeline.js +20 -0
  1170. package/services/voice-profiles/index.d.ts +12 -0
  1171. package/services/voice-profiles/index.d.ts.map +1 -0
  1172. package/services/voice-profiles/index.js +5 -0
  1173. package/services/voice-profiles/nickname-evaluator.d.ts +14 -0
  1174. package/services/voice-profiles/nickname-evaluator.d.ts.map +1 -0
  1175. package/services/voice-profiles/nickname-evaluator.js +46 -0
  1176. package/services/voice-profiles/owner-confidence.d.ts +10 -0
  1177. package/services/voice-profiles/owner-confidence.d.ts.map +1 -0
  1178. package/services/voice-profiles/owner-confidence.js +38 -0
  1179. package/services/voice-profiles/private-challenge.d.ts +20 -0
  1180. package/services/voice-profiles/private-challenge.d.ts.map +1 -0
  1181. package/services/voice-profiles/private-challenge.js +44 -0
  1182. package/services/voice-profiles/store.d.ts +21 -0
  1183. package/services/voice-profiles/store.d.ts.map +1 -0
  1184. package/services/voice-profiles/store.js +50 -0
  1185. package/services/voice-profiles/types.d.ts +38 -0
  1186. package/services/voice-profiles/types.d.ts.map +1 -0
  1187. package/services/voice-profiles/types.js +1 -0
  1188. package/styles/electrobun-mac-window-drag.css +4 -4
  1189. package/test/helpers/__tests__/live-agent-test.smoke.test.ts +43 -70
  1190. package/test/helpers/browser-mocks.ts +2 -2
  1191. package/test/helpers/conditional-tests.ts +2 -2
  1192. package/test/helpers/live-agent-test.ts +537 -551
  1193. package/test/helpers/live-provider.test.ts +95 -5
  1194. package/test/helpers/live-provider.ts +234 -56
  1195. package/test/helpers/live-runtime-server.ts +4 -4
  1196. package/test/helpers/pglite-runtime.ts +1 -1
  1197. package/test/helpers/real-runtime.ts +72 -20
  1198. package/test/helpers/trajectory-harness.ts +11 -7
  1199. package/test/scripts/start-eliza-live.ts +9 -0
  1200. package/test/scripts/test-parallel.mjs +1 -1
  1201. package/test/scripts/test-root-unit.mjs +6 -7
  1202. package/test/scripts/websocket-pending-queue.test.ts +119 -0
  1203. package/test-support/test-helpers.d.ts +0 -33
  1204. package/test-support/test-helpers.d.ts.map +1 -1
  1205. package/test-support/test-helpers.js +81 -189
  1206. package/ui-compat.d.ts +14 -2
  1207. package/ui-compat.d.ts.map +1 -1
  1208. package/ui-compat.js +23 -3
  1209. package/api/auth-pairing-compat-routes.d.ts +0 -17
  1210. package/api/auth-pairing-compat-routes.d.ts.map +0 -1
  1211. package/api/auth-pairing-compat-routes.js +0 -301
  1212. package/api/local-inference-compat-routes.d.ts +0 -16
  1213. package/api/local-inference-compat-routes.d.ts.map +0 -1
  1214. package/api/local-inference-compat-routes.js +0 -617
  1215. package/api/onboarding-compat-routes.d.ts +0 -4
  1216. package/api/onboarding-compat-routes.d.ts.map +0 -1
  1217. package/api/onboarding-compat-routes.js +0 -207
  1218. package/api/plugins-compat-routes.d.ts +0 -103
  1219. package/api/plugins-compat-routes.d.ts.map +0 -1
  1220. package/api/plugins-compat-routes.js +0 -1181
  1221. package/api/server-onboarding-compat.d.ts +0 -31
  1222. package/api/server-onboarding-compat.d.ts.map +0 -1
  1223. package/api/server-onboarding-compat.js +0 -283
  1224. package/benchmark/cua-routes.d.ts +0 -10
  1225. package/benchmark/cua-routes.d.ts.map +0 -1
  1226. package/benchmark/cua-routes.js +0 -179
  1227. package/benchmark/mock-plugin-base.d.ts +0 -9
  1228. package/benchmark/mock-plugin-base.d.ts.map +0 -1
  1229. package/benchmark/mock-plugin-base.js +0 -325
  1230. package/cli/parse-duration.d.ts +0 -5
  1231. package/cli/parse-duration.d.ts.map +0 -1
  1232. package/cli/parse-duration.js +0 -27
  1233. package/patches/llama-cpp-capacitor@0.1.5.patch +0 -2387
  1234. package/platform/agent-browser-stub.d.ts +0 -27
  1235. package/platform/agent-browser-stub.d.ts.map +0 -1
  1236. package/platform/agent-browser-stub.js +0 -16
  1237. package/platforms/android/app/src/androidTest/java/com/getcapacitor/myapp/ExampleInstrumentedTest.java +0 -26
  1238. package/platforms/android/app/src/main/res/drawable/ic_launcher_background.xml +0 -170
  1239. package/platforms/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml +0 -34
  1240. package/platforms/android/app/src/test/java/com/getcapacitor/myapp/ExampleUnitTest.java +0 -18
  1241. package/platforms/electrobun/assets/appIcon.iconset/icon_512x512@2x.png +0 -0
  1242. package/platforms/electrobun/assets/appIcon.png +0 -0
  1243. package/platforms/electrobun/scripts/build-whisper-universal.sh +0 -137
  1244. package/platforms/electrobun/scripts/build-whisper.sh +0 -95
  1245. package/platforms/electrobun/scripts/ensure-whisper-model.sh +0 -93
  1246. package/platforms/electrobun/src/libMacWindowEffects.dylib +0 -0
  1247. package/platforms/electrobun/src/native/whisper.ts +0 -280
  1248. package/platforms/ios/App/App/Assets.xcassets/Splash.imageset/splash-2732x2732-1.png +0 -0
  1249. package/platforms/ios/App/App/Assets.xcassets/Splash.imageset/splash-2732x2732-2.png +0 -0
  1250. package/platforms/ios/App/App/Assets.xcassets/Splash.imageset/splash-2732x2732.png +0 -0
  1251. package/registry/generate-apps.d.ts +0 -2
  1252. package/registry/generate-apps.d.ts.map +0 -1
  1253. package/registry/generate-apps.js +0 -338
  1254. package/registry/generate.d.ts +0 -2
  1255. package/registry/generate.d.ts.map +0 -1
  1256. package/registry/generate.js +0 -506
  1257. package/registry/loader.d.ts +0 -27
  1258. package/registry/loader.d.ts.map +0 -1
  1259. package/registry/loader.js +0 -111
  1260. package/registry/schema.d.ts +0 -989
  1261. package/registry/schema.d.ts.map +0 -1
  1262. package/registry/schema.js +0 -264
  1263. package/runtime/embedding-manager-support.d.ts +0 -77
  1264. package/runtime/embedding-manager-support.d.ts.map +0 -1
  1265. package/runtime/embedding-manager-support.js +0 -309
  1266. package/runtime/embedding-presets.d.ts +0 -5
  1267. package/runtime/embedding-presets.d.ts.map +0 -1
  1268. package/runtime/embedding-presets.js +0 -47
  1269. package/runtime/embedding-warmup-policy.d.ts +0 -13
  1270. package/runtime/embedding-warmup-policy.d.ts.map +0 -1
  1271. package/runtime/embedding-warmup-policy.js +0 -33
  1272. package/runtime/ensure-local-inference-handler.d.ts +0 -25
  1273. package/runtime/ensure-local-inference-handler.d.ts.map +0 -1
  1274. package/runtime/ensure-local-inference-handler.js +0 -389
  1275. package/runtime/mobile-local-inference-gate.d.ts +0 -21
  1276. package/runtime/mobile-local-inference-gate.d.ts.map +0 -1
  1277. package/runtime/mobile-local-inference-gate.js +0 -24
  1278. package/scripts/aosp/avd-test.mjs +0 -403
  1279. package/scripts/aosp/boot-validate.mjs +0 -536
  1280. package/scripts/aosp/build-aosp.mjs +0 -448
  1281. package/scripts/aosp/build-bootanimation.mjs +0 -178
  1282. package/scripts/aosp/capture-screens.mjs +0 -325
  1283. package/scripts/aosp/e2e-validate.mjs +0 -225
  1284. package/scripts/aosp/lint-init-rc.mjs +0 -258
  1285. package/scripts/aosp/llama-shim/eliza_llama_shim.c +0 -276
  1286. package/scripts/aosp/sim.mjs +0 -277
  1287. package/scripts/aosp/sync-to-aosp.mjs +0 -134
  1288. package/scripts/aosp/validate.mjs +0 -1273
  1289. package/scripts/build-llama-cpp-dflash.mjs +0 -1866
  1290. package/scripts/generate-onboarding-voicelines.mjs +0 -194
  1291. package/scripts/generated/static-asset-manifest.json +0 -4
  1292. package/scripts/normalize-parallax-capture.ts +0 -97
  1293. package/scripts/omnivoice-fuse/Makefile +0 -44
  1294. package/scripts/omnivoice-fuse/README.md +0 -266
  1295. package/scripts/omnivoice-fuse/cmake-graft.mjs +0 -180
  1296. package/scripts/omnivoice-fuse/ffi-stub.c +0 -222
  1297. package/scripts/omnivoice-fuse/ffi.h +0 -158
  1298. package/scripts/omnivoice-fuse/libelizainference_stub.dylib +0 -0
  1299. package/scripts/omnivoice-fuse/verify-symbols.mjs +0 -138
  1300. package/security/cloud-secret-store.d.ts +0 -34
  1301. package/security/cloud-secret-store.d.ts.map +0 -1
  1302. package/security/cloud-secret-store.js +0 -65
  1303. package/security/export-guard.d.ts +0 -34
  1304. package/security/export-guard.d.ts.map +0 -1
  1305. package/security/export-guard.js +0 -127
  1306. package/services/local-inference/__stress__/cache-stress-helpers.d.ts +0 -76
  1307. package/services/local-inference/__stress__/cache-stress-helpers.d.ts.map +0 -1
  1308. package/services/local-inference/__stress__/cache-stress-helpers.js +0 -238
  1309. package/services/local-inference/active-model.d.ts +0 -180
  1310. package/services/local-inference/active-model.d.ts.map +0 -1
  1311. package/services/local-inference/active-model.js +0 -362
  1312. package/services/local-inference/assignments.d.ts +0 -58
  1313. package/services/local-inference/assignments.d.ts.map +0 -1
  1314. package/services/local-inference/assignments.js +0 -179
  1315. package/services/local-inference/backend.d.ts +0 -200
  1316. package/services/local-inference/backend.d.ts.map +0 -1
  1317. package/services/local-inference/backend.js +0 -242
  1318. package/services/local-inference/bundled-models.d.ts +0 -34
  1319. package/services/local-inference/bundled-models.d.ts.map +0 -1
  1320. package/services/local-inference/bundled-models.js +0 -104
  1321. package/services/local-inference/cache-bridge.d.ts +0 -184
  1322. package/services/local-inference/cache-bridge.d.ts.map +0 -1
  1323. package/services/local-inference/cache-bridge.js +0 -333
  1324. package/services/local-inference/catalog.d.ts +0 -57
  1325. package/services/local-inference/catalog.d.ts.map +0 -1
  1326. package/services/local-inference/catalog.js +0 -262
  1327. package/services/local-inference/conversation-registry.d.ts +0 -122
  1328. package/services/local-inference/conversation-registry.d.ts.map +0 -1
  1329. package/services/local-inference/conversation-registry.js +0 -182
  1330. package/services/local-inference/device-bridge.d.ts +0 -139
  1331. package/services/local-inference/device-bridge.d.ts.map +0 -1
  1332. package/services/local-inference/device-bridge.js +0 -774
  1333. package/services/local-inference/dflash-doctor.d.ts +0 -27
  1334. package/services/local-inference/dflash-doctor.d.ts.map +0 -1
  1335. package/services/local-inference/dflash-doctor.js +0 -149
  1336. package/services/local-inference/dflash-server.d.ts +0 -248
  1337. package/services/local-inference/dflash-server.d.ts.map +0 -1
  1338. package/services/local-inference/dflash-server.js +0 -1076
  1339. package/services/local-inference/downloader.d.ts +0 -48
  1340. package/services/local-inference/downloader.d.ts.map +0 -1
  1341. package/services/local-inference/downloader.js +0 -688
  1342. package/services/local-inference/engine.d.ts +0 -282
  1343. package/services/local-inference/engine.d.ts.map +0 -1
  1344. package/services/local-inference/engine.js +0 -743
  1345. package/services/local-inference/external-scanner.d.ts +0 -17
  1346. package/services/local-inference/external-scanner.d.ts.map +0 -1
  1347. package/services/local-inference/external-scanner.js +0 -261
  1348. package/services/local-inference/handler-registry.d.ts +0 -72
  1349. package/services/local-inference/handler-registry.d.ts.map +0 -1
  1350. package/services/local-inference/handler-registry.js +0 -159
  1351. package/services/local-inference/hardware.d.ts +0 -26
  1352. package/services/local-inference/hardware.d.ts.map +0 -1
  1353. package/services/local-inference/hardware.js +0 -139
  1354. package/services/local-inference/hf-search.d.ts +0 -19
  1355. package/services/local-inference/hf-search.d.ts.map +0 -1
  1356. package/services/local-inference/hf-search.js +0 -169
  1357. package/services/local-inference/index.d.ts +0 -10
  1358. package/services/local-inference/index.d.ts.map +0 -1
  1359. package/services/local-inference/index.js +0 -7
  1360. package/services/local-inference/llama-server-metrics.d.ts +0 -108
  1361. package/services/local-inference/llama-server-metrics.d.ts.map +0 -1
  1362. package/services/local-inference/llama-server-metrics.js +0 -175
  1363. package/services/local-inference/manifest/index.d.ts +0 -4
  1364. package/services/local-inference/manifest/index.d.ts.map +0 -1
  1365. package/services/local-inference/manifest/index.js +0 -5
  1366. package/services/local-inference/manifest/schema.d.ts +0 -419
  1367. package/services/local-inference/manifest/schema.d.ts.map +0 -1
  1368. package/services/local-inference/manifest/schema.js +0 -227
  1369. package/services/local-inference/manifest/types.d.ts +0 -23
  1370. package/services/local-inference/manifest/types.d.ts.map +0 -1
  1371. package/services/local-inference/manifest/types.js +0 -5
  1372. package/services/local-inference/manifest/validator.d.ts +0 -43
  1373. package/services/local-inference/manifest/validator.d.ts.map +0 -1
  1374. package/services/local-inference/manifest/validator.js +0 -187
  1375. package/services/local-inference/paths.d.ts +0 -8
  1376. package/services/local-inference/paths.d.ts.map +0 -1
  1377. package/services/local-inference/paths.js +0 -7
  1378. package/services/local-inference/providers.d.ts +0 -61
  1379. package/services/local-inference/providers.d.ts.map +0 -1
  1380. package/services/local-inference/providers.js +0 -334
  1381. package/services/local-inference/ram-budget.d.ts +0 -57
  1382. package/services/local-inference/ram-budget.d.ts.map +0 -1
  1383. package/services/local-inference/ram-budget.js +0 -107
  1384. package/services/local-inference/readiness.d.ts +0 -9
  1385. package/services/local-inference/readiness.d.ts.map +0 -1
  1386. package/services/local-inference/readiness.js +0 -153
  1387. package/services/local-inference/recommendation.d.ts +0 -62
  1388. package/services/local-inference/recommendation.d.ts.map +0 -1
  1389. package/services/local-inference/recommendation.js +0 -309
  1390. package/services/local-inference/registry.d.ts +0 -35
  1391. package/services/local-inference/registry.d.ts.map +0 -1
  1392. package/services/local-inference/registry.js +0 -117
  1393. package/services/local-inference/router-handler.d.ts +0 -51
  1394. package/services/local-inference/router-handler.d.ts.map +0 -1
  1395. package/services/local-inference/router-handler.js +0 -165
  1396. package/services/local-inference/routing-policy.d.ts +0 -55
  1397. package/services/local-inference/routing-policy.d.ts.map +0 -1
  1398. package/services/local-inference/routing-policy.js +0 -195
  1399. package/services/local-inference/routing-preferences.d.ts +0 -8
  1400. package/services/local-inference/routing-preferences.d.ts.map +0 -1
  1401. package/services/local-inference/routing-preferences.js +0 -7
  1402. package/services/local-inference/service.d.ts +0 -88
  1403. package/services/local-inference/service.d.ts.map +0 -1
  1404. package/services/local-inference/service.js +0 -210
  1405. package/services/local-inference/session-pool.d.ts +0 -72
  1406. package/services/local-inference/session-pool.d.ts.map +0 -1
  1407. package/services/local-inference/session-pool.js +0 -125
  1408. package/services/local-inference/types.d.ts +0 -309
  1409. package/services/local-inference/types.d.ts.map +0 -1
  1410. package/services/local-inference/types.js +0 -23
  1411. package/services/local-inference/verify.d.ts +0 -8
  1412. package/services/local-inference/verify.d.ts.map +0 -1
  1413. package/services/local-inference/verify.js +0 -7
  1414. package/services/local-inference/voice/barge-in.d.ts +0 -15
  1415. package/services/local-inference/voice/barge-in.d.ts.map +0 -1
  1416. package/services/local-inference/voice/barge-in.js +0 -20
  1417. package/services/local-inference/voice/engine-bridge.d.ts +0 -256
  1418. package/services/local-inference/voice/engine-bridge.d.ts.map +0 -1
  1419. package/services/local-inference/voice/engine-bridge.js +0 -398
  1420. package/services/local-inference/voice/ffi-bindings.d.ts +0 -114
  1421. package/services/local-inference/voice/ffi-bindings.d.ts.map +0 -1
  1422. package/services/local-inference/voice/ffi-bindings.js +0 -281
  1423. package/services/local-inference/voice/index.d.ts +0 -51
  1424. package/services/local-inference/voice/index.d.ts.map +0 -1
  1425. package/services/local-inference/voice/index.js +0 -50
  1426. package/services/local-inference/voice/lifecycle.d.ts +0 -135
  1427. package/services/local-inference/voice/lifecycle.d.ts.map +0 -1
  1428. package/services/local-inference/voice/lifecycle.js +0 -189
  1429. package/services/local-inference/voice/phoneme-tokenizer.d.ts +0 -58
  1430. package/services/local-inference/voice/phoneme-tokenizer.d.ts.map +0 -1
  1431. package/services/local-inference/voice/phoneme-tokenizer.js +0 -53
  1432. package/services/local-inference/voice/phrase-cache.d.ts +0 -24
  1433. package/services/local-inference/voice/phrase-cache.d.ts.map +0 -1
  1434. package/services/local-inference/voice/phrase-cache.js +0 -32
  1435. package/services/local-inference/voice/phrase-chunker.d.ts +0 -20
  1436. package/services/local-inference/voice/phrase-chunker.d.ts.map +0 -1
  1437. package/services/local-inference/voice/phrase-chunker.js +0 -85
  1438. package/services/local-inference/voice/ring-buffer.d.ts +0 -40
  1439. package/services/local-inference/voice/ring-buffer.d.ts.map +0 -1
  1440. package/services/local-inference/voice/ring-buffer.js +0 -85
  1441. package/services/local-inference/voice/rollback-queue.d.ts +0 -24
  1442. package/services/local-inference/voice/rollback-queue.d.ts.map +0 -1
  1443. package/services/local-inference/voice/rollback-queue.js +0 -49
  1444. package/services/local-inference/voice/scheduler.d.ts +0 -47
  1445. package/services/local-inference/voice/scheduler.d.ts.map +0 -1
  1446. package/services/local-inference/voice/scheduler.js +0 -123
  1447. package/services/local-inference/voice/shared-resources.d.ts +0 -119
  1448. package/services/local-inference/voice/shared-resources.d.ts.map +0 -1
  1449. package/services/local-inference/voice/shared-resources.js +0 -83
  1450. package/services/local-inference/voice/speaker-preset-cache.d.ts +0 -28
  1451. package/services/local-inference/voice/speaker-preset-cache.d.ts.map +0 -1
  1452. package/services/local-inference/voice/speaker-preset-cache.js +0 -44
  1453. package/services/local-inference/voice/types.d.ts +0 -80
  1454. package/services/local-inference/voice/types.d.ts.map +0 -1
  1455. package/services/local-inference/voice/voice-preset-format.d.ts +0 -56
  1456. package/services/local-inference/voice/voice-preset-format.d.ts.map +0 -1
  1457. package/services/local-inference/voice/voice-preset-format.js +0 -184
  1458. package/services/plugin-installer.d.ts +0 -22
  1459. package/services/plugin-installer.d.ts.map +0 -1
  1460. package/services/plugin-installer.js +0 -41
  1461. package/test/helpers/i18n.ts +0 -5
  1462. package/test/helpers/loopback.ts +0 -33
  1463. package/test/helpers/react-test.ts +0 -40
  1464. package/test/helpers/skip-without.ts +0 -53
  1465. package/test/scripts/task-agent-live-smoke.ts +0 -1335
  1466. /package/services/{local-inference/voice → ambient-audio}/types.js +0 -0
@@ -0,0 +1,1629 @@
1
+ // elizavoice-jni.cpp
2
+ //
3
+ // JNI bridge for the fused, NDK/bionic-built `libelizainference.so` (the
4
+ // omnivoice `elizainference` target — all four voice runtimes fused at ABI
5
+ // v7) running INSIDE the normal Android APK process (ai.elizaos.app,
6
+ // Capacitor/bionic), NOT the separate musl bun agent.
7
+ //
8
+ // Phase 3a proved the load + a single VAD op. Phase 3b wires the WHOLE
9
+ // mic→attributed-turn pipeline through this host:
10
+ //
11
+ // - Context lifecycle: contextCreate / contextDestroy.
12
+ // - The four fused classifier ops, each wrapping the matching
13
+ // `eliza_inference_*` symbols:
14
+ // VAD open / processBatch / reset / close
15
+ // wakeword open / scoreBatch / reset / close
16
+ // speaker open / embed / close
17
+ // diariz open / segment / close
18
+ // - A native streaming PIPELINE (pipelineOpen / pipelineProcess /
19
+ // pipelineFlush / pipelineReset / pipelineClose) that runs the VAD
20
+ // hot-loop + turn segmentation NATIVELY (no per-512-window JS↔native
21
+ // bridge call): JS hands it an audio-frame batch, it streams the PCM
22
+ // through VAD, applies the onset/offset/pause/end-hangover state machine
23
+ // (ported from VadDetector in vad.ts), buffers the turn PCM, and on
24
+ // speech-end runs speaker-embed + diariz natively and returns a
25
+ // TURN-LEVEL JSON result (speaker 256-float embedding + diariz int8
26
+ // labels) to JS. JS then applies the ambient gate + builds the
27
+ // voiceTurnSignal.
28
+ //
29
+ // PCM marshalling: Java `float[]` (16 kHz mono fp32 in [-1, 1]) in via
30
+ // GetFloatArrayElements (JNI_ABORT release — read-only, no copy-back).
31
+ // Outputs: the 256-float speaker embedding and the per-frame diariz int8
32
+ // labels are returned as Java arrays; turn metadata is returned as a JSON
33
+ // string the JS side parses.
34
+ //
35
+ // All sessions are reference-counted by an opaque jlong handle (the native
36
+ // pointer). Idempotent close. Errors surface as a thrown RuntimeException
37
+ // (so the Capacitor plugin rejects the call) — no swallow, no defaulted
38
+ // result (AGENTS.md §3, §9).
39
+
40
+ #include <jni.h>
41
+ #include <android/log.h>
42
+ #include <algorithm>
43
+ #include <cmath>
44
+ #include <cstdint>
45
+ #include <cstdlib>
46
+ #include <cstring>
47
+ #include <memory>
48
+ #include <string>
49
+ #include <vector>
50
+
51
+ #include "eliza-inference-ffi.h"
52
+
53
+ #define LOG_TAG "ElizaVoiceJni"
54
+ #define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
55
+ #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
56
+
57
+ namespace {
58
+
59
+ // Sample-rate / window constants (must mirror the JS contract).
60
+ constexpr int kSampleRate = 16000;
61
+ constexpr size_t kVadWindow = 512; // Silero v5 window @ 16 kHz
62
+ constexpr size_t kWakeFrame = 1280; // openWakeWord 80 ms frame @ 16 kHz
63
+ constexpr size_t kSpeakerEmbeddingDim = 256;
64
+ constexpr size_t kDiarizWindow = 80000; // pyannote 5 s window @ 16 kHz
65
+ constexpr size_t kDiarizLabelCap = 2048;
66
+
67
+ jstring to_jstring(JNIEnv* env, const std::string& s) {
68
+ return env->NewStringUTF(s.c_str());
69
+ }
70
+
71
+ std::string from_jstring(JNIEnv* env, jstring js) {
72
+ if (js == nullptr) return std::string();
73
+ const char* chars = env->GetStringUTFChars(js, nullptr);
74
+ std::string out(chars ? chars : "");
75
+ if (chars) env->ReleaseStringUTFChars(js, chars);
76
+ return out;
77
+ }
78
+
79
+ // Throw a java.lang.RuntimeException; the Capacitor plugin maps that to a
80
+ // rejected call. Frees the heap-allocated C diagnostic when present.
81
+ void throw_runtime(JNIEnv* env, const std::string& stage, char* outError) {
82
+ std::string msg = "[elizavoice-jni] " + stage;
83
+ if (outError) {
84
+ msg += ": ";
85
+ msg += outError;
86
+ std::free(outError);
87
+ }
88
+ LOGE("%s", msg.c_str());
89
+ jclass cls = env->FindClass("java/lang/RuntimeException");
90
+ if (cls) env->ThrowNew(cls, msg.c_str());
91
+ }
92
+
93
+ // Copy a Java float[] into a std::vector<float> (read-only view, no copy-back).
94
+ std::vector<float> read_float_array(JNIEnv* env, jfloatArray arr) {
95
+ if (arr == nullptr) return {};
96
+ const jsize n = env->GetArrayLength(arr);
97
+ std::vector<float> out(static_cast<size_t>(n));
98
+ if (n > 0) {
99
+ env->GetFloatArrayRegion(arr, 0, n, out.data());
100
+ }
101
+ return out;
102
+ }
103
+
104
+ float rms_of(const float* pcm, size_t n) {
105
+ if (n == 0) return 0.0f;
106
+ double acc = 0.0;
107
+ for (size_t i = 0; i < n; ++i) acc += static_cast<double>(pcm[i]) * pcm[i];
108
+ return static_cast<float>(std::sqrt(acc / static_cast<double>(n)));
109
+ }
110
+
111
+ // --- Native turn-segmentation state machine (ports VadDetector core) ------
112
+ //
113
+ // Phases mirror vad.ts: idle → speaking → paused → (end) → idle. Thresholds
114
+ // are the LiveDiarizationSession defaults (onset 0.5, pause hangover 120 ms,
115
+ // end hangover 500 ms, min speech 250 ms). The adaptive-hangover (V4) and
116
+ // fast-endpoint (V1) refinements are intentionally NOT ported — this native
117
+ // loop only needs robust turn boundaries; the JS layer no longer runs the
118
+ // per-window detector when the native pipeline owns segmentation.
119
+ struct VadSegmenter {
120
+ float onsetThreshold = 0.5f;
121
+ float offsetThreshold = 0.35f; // onset - 0.15
122
+ double pauseHangoverMs = 120.0;
123
+ double endHangoverMs = 500.0;
124
+ double minSpeechMs = 250.0;
125
+ double windowDurationMs = 1000.0 * kVadWindow / kSampleRate; // 32 ms
126
+
127
+ enum Phase { Idle, Speaking, Paused } phase = Idle;
128
+ double clockMs = 0.0;
129
+ double speechStartMs = 0.0;
130
+ double lastSpeechMs = 0.0;
131
+ double pauseStartedMs = 0.0;
132
+
133
+ // Result flags set by step(): the caller acts on speechStarted /
134
+ // speechEnded between feeding windows.
135
+ bool speechStarted = false;
136
+ bool speechEnded = false;
137
+ double endedSpeechDurationMs = 0.0;
138
+
139
+ void reset() {
140
+ phase = Idle;
141
+ clockMs = 0.0;
142
+ speechStartMs = lastSpeechMs = pauseStartedMs = 0.0;
143
+ speechStarted = speechEnded = false;
144
+ endedSpeechDurationMs = 0.0;
145
+ }
146
+
147
+ // Advance one window's worth of clock with the VAD probability. Returns via
148
+ // speechStarted / speechEnded the boundary events for this window.
149
+ void step(float prob) {
150
+ speechStarted = false;
151
+ speechEnded = false;
152
+ clockMs += windowDurationMs;
153
+ const double now = clockMs;
154
+ const bool isSpeechFrame = prob >= onsetThreshold;
155
+ const bool aboveOffset = prob >= offsetThreshold;
156
+
157
+ switch (phase) {
158
+ case Idle:
159
+ if (isSpeechFrame) {
160
+ phase = Speaking;
161
+ speechStartMs = now - windowDurationMs;
162
+ lastSpeechMs = now;
163
+ speechStarted = true;
164
+ }
165
+ break;
166
+ case Speaking:
167
+ if (aboveOffset) lastSpeechMs = now;
168
+ if (now - lastSpeechMs >= pauseHangoverMs) {
169
+ phase = Paused;
170
+ pauseStartedMs = lastSpeechMs;
171
+ }
172
+ break;
173
+ case Paused:
174
+ if (isSpeechFrame) {
175
+ phase = Speaking;
176
+ lastSpeechMs = now;
177
+ } else if (now - pauseStartedMs >= endHangoverMs) {
178
+ endedSpeechDurationMs = lastSpeechMs - speechStartMs;
179
+ phase = Idle;
180
+ speechEnded = true;
181
+ }
182
+ break;
183
+ }
184
+ }
185
+
186
+ // Force-close an open turn (on flush). Returns true when a turn was open.
187
+ bool forceEnd() {
188
+ speechStarted = false;
189
+ speechEnded = false;
190
+ if (phase == Idle) return false;
191
+ endedSpeechDurationMs = lastSpeechMs - speechStartMs;
192
+ phase = Idle;
193
+ speechEnded = true;
194
+ return true;
195
+ }
196
+ };
197
+
198
+ // --- Pipeline session -----------------------------------------------------
199
+ //
200
+ // Owns one fused VAD + speaker + diariz session set anchored to a context,
201
+ // plus the native turn-segmentation state machine and the in-flight turn PCM
202
+ // buffer. Runs entirely in the bionic app process.
203
+ struct PipelineSession {
204
+ EliInferenceContext* ctx = nullptr; // borrowed (owned by the context handle)
205
+ EliVad* vad = nullptr;
206
+ EliSpeaker* speaker = nullptr;
207
+ EliDiariz* diariz = nullptr;
208
+
209
+ VadSegmenter seg;
210
+
211
+ // Carry-over PCM not yet aligned to a 512-window boundary.
212
+ std::vector<float> pending;
213
+ // The in-flight turn's PCM (between speech-start and speech-end), with a
214
+ // small pre-roll so the leading word is not clipped.
215
+ std::vector<float> turnPcm;
216
+ std::vector<float> preRoll; // ring of pre-speech-start PCM
217
+ size_t preRollSamples = static_cast<size_t>(0.3 * kSampleRate);
218
+ size_t maxTurnSamples = static_cast<size_t>(30 * kSampleRate);
219
+ bool capturing = false;
220
+ int turnSeq = 0;
221
+
222
+ // Completed turns produced by the last process()/flush() call, encoded as
223
+ // JSON the JS side parses. Drained by the caller.
224
+ std::vector<std::string> turns;
225
+ // Parallel arrays for the heavy payloads (embedding + labels) so JS can
226
+ // read them without re-parsing big JSON.
227
+ std::vector<std::vector<float>> turnEmbeddings;
228
+ std::vector<std::vector<int8_t>> turnLabels;
229
+ std::vector<std::vector<float>> turnPcms;
230
+ };
231
+
232
+ // Append `n` samples to a bounded pre-roll ring, dropping oldest beyond cap.
233
+ void push_preroll(PipelineSession* s, const float* pcm, size_t n) {
234
+ if (s->preRollSamples == 0) return;
235
+ s->preRoll.insert(s->preRoll.end(), pcm, pcm + n);
236
+ if (s->preRoll.size() > s->preRollSamples) {
237
+ const size_t drop = s->preRoll.size() - s->preRollSamples;
238
+ s->preRoll.erase(s->preRoll.begin(),
239
+ s->preRoll.begin() + static_cast<long>(drop));
240
+ }
241
+ }
242
+
243
+ // Run speaker-embed + diariz on a finalized turn and stage the JSON + payloads.
244
+ // Returns false (with *outError set) on a native op failure.
245
+ bool finalize_turn(PipelineSession* s, char** outError) {
246
+ if (s->turnPcm.empty()) return true;
247
+ const size_t samples = s->turnPcm.size();
248
+ const int turnId = s->turnSeq++;
249
+
250
+ // Speaker embedding over the whole turn (WeSpeaker min ~ 0.5 s; the JS
251
+ // attribution pipeline re-checks WESPEAKER_MIN_SAMPLES, but the native op
252
+ // needs enough audio — run it and let the value flow; too-short turns just
253
+ // yield a weak embedding the matcher discounts).
254
+ std::vector<float> embedding(kSpeakerEmbeddingDim, 0.0f);
255
+ bool haveEmbedding = false;
256
+ if (s->speaker && samples >= kSampleRate / 2) {
257
+ const int rc = eliza_inference_speaker_embed(
258
+ s->speaker, s->turnPcm.data(), samples, embedding.data(), outError);
259
+ if (rc != ELIZA_OK) return false;
260
+ haveEmbedding = true;
261
+ }
262
+
263
+ // Diarizer over a single 5 s window: pyannote takes exactly 80000 samples.
264
+ // Center-crop or zero-pad the turn to the fixed window (the JS reducer maps
265
+ // frame labels back to ms via the fixed stride, so the window is canonical).
266
+ std::vector<int8_t> labels;
267
+ bool haveLabels = false;
268
+ if (s->diariz) {
269
+ std::vector<float> window(kDiarizWindow, 0.0f);
270
+ const size_t copy = samples < kDiarizWindow ? samples : kDiarizWindow;
271
+ std::memcpy(window.data(), s->turnPcm.data(), copy * sizeof(float));
272
+ std::vector<int8_t> out(kDiarizLabelCap, 0);
273
+ size_t nLabels = out.size();
274
+ const int rc = eliza_inference_diariz_segment(
275
+ s->diariz, window.data(), window.size(), out.data(), &nLabels,
276
+ outError);
277
+ if (rc != ELIZA_OK) return false;
278
+ labels.assign(out.begin(), out.begin() + static_cast<long>(nLabels));
279
+ haveLabels = true;
280
+ }
281
+
282
+ const double durationMs = 1000.0 * static_cast<double>(samples) / kSampleRate;
283
+
284
+ // Log per-op turn evidence to logcat (the in-process verification channel:
285
+ // these lines are emitted by the ai.elizaos.app pid, proving all four ops —
286
+ // VAD segmentation + speaker embed + diariz — ran in the bionic app
287
+ // process, independent of the JS↔Capacitor result delivery).
288
+ double embNorm = 0.0, embAbsMax = 0.0;
289
+ for (size_t i = 0; i < embedding.size(); ++i) {
290
+ embNorm += static_cast<double>(embedding[i]) * embedding[i];
291
+ const double a = std::fabs(static_cast<double>(embedding[i]));
292
+ if (a > embAbsMax) embAbsMax = a;
293
+ }
294
+ embNorm = std::sqrt(embNorm);
295
+ // Diariz label histogram (how many distinct powerset classes fired).
296
+ int distinctLabels = 0;
297
+ bool seen[8] = {false, false, false, false, false, false, false, false};
298
+ for (int8_t l : labels) {
299
+ if (l >= 0 && l < 8 && !seen[l]) { seen[l] = true; distinctLabels++; }
300
+ }
301
+ LOGI("TURN jni_%d: samples=%zu (%.2fs) | speaker: embDim=%zu norm=%.4f "
302
+ "absMax=%.4f emb[0..3]=[%.4f,%.4f,%.4f,%.4f] | diariz: frames=%zu "
303
+ "distinctClasses=%d",
304
+ turnId, samples, durationMs / 1000.0,
305
+ haveEmbedding ? embedding.size() : 0, embNorm, embAbsMax,
306
+ embedding.size() > 0 ? embedding[0] : 0.0f,
307
+ embedding.size() > 1 ? embedding[1] : 0.0f,
308
+ embedding.size() > 2 ? embedding[2] : 0.0f,
309
+ embedding.size() > 3 ? embedding[3] : 0.0f,
310
+ labels.size(), distinctLabels);
311
+
312
+ std::string json = "{\"turnId\":\"jni_" + std::to_string(turnId) +
313
+ "\",\"samples\":" + std::to_string(samples) +
314
+ ",\"durationMs\":" + std::to_string(durationMs) +
315
+ ",\"hasEmbedding\":" + (haveEmbedding ? "true" : "false") +
316
+ ",\"embNorm\":" + std::to_string(embNorm) +
317
+ ",\"diarizFrames\":" + std::to_string(labels.size()) +
318
+ ",\"diarizDistinctClasses\":" +
319
+ std::to_string(distinctLabels) + "}";
320
+ s->turns.push_back(std::move(json));
321
+ s->turnEmbeddings.push_back(haveEmbedding ? std::move(embedding)
322
+ : std::vector<float>{});
323
+ s->turnLabels.push_back(haveLabels ? std::move(labels)
324
+ : std::vector<int8_t>{});
325
+ s->turnPcms.push_back(std::move(s->turnPcm));
326
+ s->turnPcm.clear();
327
+ return true;
328
+ }
329
+
330
+ // Drain the pending buffer in 512-window steps, driving the VAD + segmenter.
331
+ bool drain_windows(PipelineSession* s, char** outError) {
332
+ while (s->pending.size() >= kVadWindow) {
333
+ float prob = -1.0f;
334
+ const int rc = eliza_inference_vad_process(
335
+ s->vad, s->pending.data(), kVadWindow, &prob, outError);
336
+ if (rc != ELIZA_OK) return false;
337
+
338
+ // Buffer this window into turn or pre-roll BEFORE the state transition
339
+ // so a speech-start seeds the turn with the pre-roll + this window.
340
+ const float* win = s->pending.data();
341
+ if (s->capturing) {
342
+ s->turnPcm.insert(s->turnPcm.end(), win, win + kVadWindow);
343
+ } else {
344
+ push_preroll(s, win, kVadWindow);
345
+ }
346
+
347
+ s->seg.step(prob);
348
+ if (s->seg.speechStarted) {
349
+ s->capturing = true;
350
+ // Seed the turn with the pre-roll (leading word) + this window.
351
+ s->turnPcm = s->preRoll;
352
+ s->turnPcm.insert(s->turnPcm.end(), win, win + kVadWindow);
353
+ s->preRoll.clear();
354
+ }
355
+ if (s->seg.speechEnded) {
356
+ s->capturing = false;
357
+ eliza_inference_vad_reset(s->vad, outError); // best-effort
358
+ if (!finalize_turn(s, outError)) return false;
359
+ }
360
+ // Hard cap on a runaway turn.
361
+ if (s->capturing && s->turnPcm.size() >= s->maxTurnSamples) {
362
+ s->capturing = false;
363
+ s->seg.forceEnd();
364
+ eliza_inference_vad_reset(s->vad, outError);
365
+ if (!finalize_turn(s, outError)) return false;
366
+ }
367
+
368
+ // Advance the pending window.
369
+ s->pending.erase(s->pending.begin(),
370
+ s->pending.begin() + static_cast<long>(kVadWindow));
371
+ }
372
+ return true;
373
+ }
374
+
375
+ // Build the JSON returned to JS for a process()/flush() call: the turns
376
+ // completed in this call. Embeddings + labels are exposed via the parallel
377
+ // getters so JS reads them per-index.
378
+ std::string drain_turns_json(PipelineSession* s) {
379
+ std::string out = "[";
380
+ for (size_t i = 0; i < s->turns.size(); ++i) {
381
+ if (i > 0) out += ",";
382
+ out += s->turns[i];
383
+ }
384
+ out += "]";
385
+ return out;
386
+ }
387
+
388
+ // Close a self-test session's classifier handles + its owning context.
389
+ void cleanup_session_for_selftest(PipelineSession* s, EliInferenceContext* ctx) {
390
+ if (s->diariz) eliza_inference_diariz_close(s->diariz);
391
+ if (s->speaker) eliza_inference_speaker_close(s->speaker);
392
+ if (s->vad) eliza_inference_vad_close(s->vad);
393
+ delete s;
394
+ if (ctx) eliza_inference_destroy(ctx);
395
+ }
396
+
397
+ // True unless the env var is explicitly "0"/"false"/"no"/"off" (case-insensitive).
398
+ // Absent / unrecognized → the `fallback`.
399
+ bool bionic_bool_env_or_default(const char* name, bool fallback) {
400
+ const char* v = std::getenv(name);
401
+ if (!v || v[0] == '\0') return fallback;
402
+ std::string s(v);
403
+ std::transform(s.begin(), s.end(), s.begin(),
404
+ [](unsigned char c) { return std::tolower(c); });
405
+ if (s == "0" || s == "false" || s == "no" || s == "off") return false;
406
+ if (s == "1" || s == "true" || s == "yes" || s == "on") return true;
407
+ return fallback;
408
+ }
409
+
410
+ int bionic_int_env_or_default(const char* name, int fallback) {
411
+ const char* v = std::getenv(name);
412
+ if (!v || v[0] == '\0') return fallback;
413
+ int parsed = std::atoi(v);
414
+ return parsed > 0 ? parsed : fallback;
415
+ }
416
+
417
+ // Arm the eliza-1 text-model bionic config for runtime optimizations that are
418
+ // correct for the shipped model.
419
+ //
420
+ // KV-quant stays OFF by default. The shipped Eliza-1 tiers are Gemma 4, whose
421
+ // KV is already minimal by construction (MQA + windowed-SWA + shared-KV, dual
422
+ // head dims 512 global / 256 SWA) and runs on stock f16/q8_0 KV. The legacy
423
+ // QJL1_256 / fused QJL-TBQ kernels are head_dim=128 and dimensionally
424
+ // inapplicable to Gemma, so cache_type_k=qjl1_256/cache_type_v=tbq3_0 is not a
425
+ // shipping path; F16 KV is correct. ELIZA_BIONIC_KV_QUANT=1 is left as an
426
+ // explicit lab override for head_dim=128 test bundles only.
427
+ //
428
+ // MTP is enabled only when the caller supplies a Gemma separate-drafter GGUF.
429
+ // Omitting cfg.mtp_drafter_path would select the retired same-file NextN path,
430
+ // so shipped Gemma bundles without a staged drafter run plain decode even if
431
+ // ELIZA_BIONIC_MTP is set in the app process.
432
+ //
433
+ // The two static names below outlive the cfg they're attached to (cfg is a
434
+ // stack struct consumed synchronously by eliza_inference_llm_stream_open).
435
+ void arm_bionic_text_cfg(eliza_llm_stream_config_t& cfg,
436
+ const char* bundle_dir = nullptr) {
437
+ static const char* kKvTypeK = "qjl1_256";
438
+ static const char* kKvTypeV = "tbq3_0";
439
+ if (bionic_bool_env_or_default("ELIZA_BIONIC_KV_QUANT", false)) {
440
+ cfg.cache_type_k = kKvTypeK;
441
+ cfg.cache_type_v = kKvTypeV;
442
+ LOGI("bionic text cfg: KV-quant ON by explicit override (k=%s v=%s)",
443
+ kKvTypeK, kKvTypeV);
444
+ } else {
445
+ LOGI("bionic text cfg: KV-quant OFF (stock f16/q8_0 KV; Gemma 4 KV is "
446
+ "already minimal)");
447
+ }
448
+
449
+ (void)bundle_dir;
450
+ const bool has_drafter =
451
+ cfg.mtp_drafter_path && cfg.mtp_drafter_path[0] != '\0';
452
+ if (has_drafter && bionic_bool_env_or_default("ELIZA_BIONIC_MTP", true)) {
453
+ int draft_min = bionic_int_env_or_default("ELIZA_BIONIC_MTP_DRAFT_MIN", 1);
454
+ int draft_max = bionic_int_env_or_default("ELIZA_BIONIC_MTP_DRAFT_MAX", 1);
455
+ if (draft_min < 1) draft_min = 1;
456
+ if (draft_max < draft_min) draft_max = draft_min;
457
+ cfg.draft_min = draft_min;
458
+ cfg.draft_max = draft_max;
459
+ LOGI("bionic text cfg: MTP ON (draft_min=%d draft_max=%d, drafter=%s)",
460
+ draft_min, draft_max, cfg.mtp_drafter_path);
461
+ } else {
462
+ LOGI("bionic text cfg: MTP OFF (%s)",
463
+ has_drafter ? "disabled by ELIZA_BIONIC_MTP" : "no drafter");
464
+ }
465
+ }
466
+
467
+ } // namespace
468
+
469
+ extern "C" {
470
+
471
+ // ── ABI / capability probes (Phase 3a, retained) ─────────────────────────
472
+
473
+ JNIEXPORT jstring JNICALL
474
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVoiceAbiVersion(JNIEnv* env, jclass) {
475
+ const char* abi = eliza_inference_abi_version();
476
+ LOGI("eliza_inference_abi_version() = %s", abi ? abi : "(null)");
477
+ return to_jstring(env, abi ? std::string(abi) : std::string());
478
+ }
479
+
480
+ JNIEXPORT jint JNICALL
481
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVadSupported(JNIEnv*, jclass) {
482
+ return static_cast<jint>(eliza_inference_vad_supported());
483
+ }
484
+
485
+ JNIEXPORT jint JNICALL
486
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeWakewordSupported(JNIEnv*, jclass) {
487
+ return static_cast<jint>(eliza_inference_wakeword_supported());
488
+ }
489
+
490
+ JNIEXPORT jint JNICALL
491
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeSpeakerSupported(JNIEnv*, jclass) {
492
+ return static_cast<jint>(eliza_inference_speaker_supported());
493
+ }
494
+
495
+ JNIEXPORT jint JNICALL
496
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeDiarizSupported(JNIEnv*, jclass) {
497
+ return static_cast<jint>(eliza_inference_diariz_supported());
498
+ }
499
+
500
+ // ── Context lifecycle ────────────────────────────────────────────────────
501
+
502
+ JNIEXPORT jlong JNICALL
503
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeContextCreate(JNIEnv* env, jclass,
504
+ jstring jBundleDir) {
505
+ const std::string bundleDir = from_jstring(env, jBundleDir);
506
+ if (bundleDir.empty()) {
507
+ throw_runtime(env, "contextCreate: empty bundle dir", nullptr);
508
+ return 0;
509
+ }
510
+ char* outError = nullptr;
511
+ EliInferenceContext* ctx =
512
+ eliza_inference_create(bundleDir.c_str(), &outError);
513
+ if (ctx == nullptr) {
514
+ throw_runtime(env, "eliza_inference_create returned null", outError);
515
+ return 0;
516
+ }
517
+ LOGI("contextCreate(%s) -> %p", bundleDir.c_str(), (void*)ctx);
518
+ return reinterpret_cast<jlong>(ctx);
519
+ }
520
+
521
+ JNIEXPORT void JNICALL
522
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeContextDestroy(JNIEnv*, jclass,
523
+ jlong handle) {
524
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(handle);
525
+ if (ctx) eliza_inference_destroy(ctx);
526
+ }
527
+
528
+ // ── VAD direct ops (evidence / self-test) ────────────────────────────────
529
+
530
+ JNIEXPORT jlong JNICALL
531
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVadOpen(JNIEnv* env, jclass,
532
+ jlong ctxHandle) {
533
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
534
+ char* outError = nullptr;
535
+ EliVad* vad = eliza_inference_vad_open(ctx, kSampleRate, &outError);
536
+ if (!vad) {
537
+ throw_runtime(env, "vad_open returned null", outError);
538
+ return 0;
539
+ }
540
+ return reinterpret_cast<jlong>(vad);
541
+ }
542
+
543
+ // Process N 512-sample windows in one call; returns the per-window
544
+ // probabilities as a Java float[] (length floor(samples/512)). Zero per-window
545
+ // bridge calls — the whole batch runs natively.
546
+ JNIEXPORT jfloatArray JNICALL
547
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVadProcessBatch(JNIEnv* env, jclass,
548
+ jlong vadHandle,
549
+ jfloatArray jPcm) {
550
+ auto* vad = reinterpret_cast<EliVad*>(vadHandle);
551
+ const std::vector<float> pcm = read_float_array(env, jPcm);
552
+ const size_t windows = pcm.size() / kVadWindow;
553
+ std::vector<float> probs(windows, 0.0f);
554
+ char* outError = nullptr;
555
+ for (size_t w = 0; w < windows; ++w) {
556
+ float p = -1.0f;
557
+ const int rc = eliza_inference_vad_process(
558
+ vad, pcm.data() + w * kVadWindow, kVadWindow, &p, &outError);
559
+ if (rc != ELIZA_OK) {
560
+ throw_runtime(env, "vad_process(batch)", outError);
561
+ return nullptr;
562
+ }
563
+ probs[w] = p;
564
+ }
565
+ jfloatArray out = env->NewFloatArray(static_cast<jsize>(windows));
566
+ if (out && windows > 0) {
567
+ env->SetFloatArrayRegion(out, 0, static_cast<jsize>(windows),
568
+ probs.data());
569
+ }
570
+ return out;
571
+ }
572
+
573
+ JNIEXPORT void JNICALL
574
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVadReset(JNIEnv*, jclass,
575
+ jlong vadHandle) {
576
+ auto* vad = reinterpret_cast<EliVad*>(vadHandle);
577
+ char* outError = nullptr;
578
+ eliza_inference_vad_reset(vad, &outError);
579
+ if (outError) std::free(outError);
580
+ }
581
+
582
+ JNIEXPORT void JNICALL
583
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVadClose(JNIEnv*, jclass,
584
+ jlong vadHandle) {
585
+ eliza_inference_vad_close(reinterpret_cast<EliVad*>(vadHandle));
586
+ }
587
+
588
+ // ── Wake-word ops ────────────────────────────────────────────────────────
589
+
590
+ JNIEXPORT jlong JNICALL
591
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeWakewordOpen(JNIEnv* env, jclass,
592
+ jlong ctxHandle,
593
+ jstring jHead) {
594
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
595
+ const std::string head = from_jstring(env, jHead);
596
+ char* outError = nullptr;
597
+ EliWakeWord* wake = eliza_inference_wakeword_open(
598
+ ctx, kSampleRate, head.empty() ? "hey-eliza" : head.c_str(), &outError);
599
+ if (!wake) {
600
+ throw_runtime(env, "wakeword_open returned null", outError);
601
+ return 0;
602
+ }
603
+ return reinterpret_cast<jlong>(wake);
604
+ }
605
+
606
+ // Score N 1280-sample frames in one call; returns the per-frame P(wake) as a
607
+ // Java float[] (length floor(samples/1280)).
608
+ JNIEXPORT jfloatArray JNICALL
609
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeWakewordScoreBatch(JNIEnv* env,
610
+ jclass,
611
+ jlong wakeHandle,
612
+ jfloatArray jPcm) {
613
+ auto* wake = reinterpret_cast<EliWakeWord*>(wakeHandle);
614
+ const std::vector<float> pcm = read_float_array(env, jPcm);
615
+ const size_t frames = pcm.size() / kWakeFrame;
616
+ std::vector<float> scores(frames, 0.0f);
617
+ char* outError = nullptr;
618
+ for (size_t f = 0; f < frames; ++f) {
619
+ float p = -1.0f;
620
+ const int rc = eliza_inference_wakeword_score(
621
+ wake, pcm.data() + f * kWakeFrame, kWakeFrame, &p, &outError);
622
+ if (rc != ELIZA_OK) {
623
+ throw_runtime(env, "wakeword_score(batch)", outError);
624
+ return nullptr;
625
+ }
626
+ scores[f] = p;
627
+ }
628
+ float maxScore = 0.0f;
629
+ for (float v : scores) maxScore = std::max(maxScore, v);
630
+ LOGI("wakeword scoreBatch: frames=%zu maxP=%.4f", frames, maxScore);
631
+ jfloatArray out = env->NewFloatArray(static_cast<jsize>(frames));
632
+ if (out && frames > 0) {
633
+ env->SetFloatArrayRegion(out, 0, static_cast<jsize>(frames),
634
+ scores.data());
635
+ }
636
+ return out;
637
+ }
638
+
639
+ JNIEXPORT void JNICALL
640
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeWakewordReset(JNIEnv*, jclass,
641
+ jlong wakeHandle) {
642
+ auto* wake = reinterpret_cast<EliWakeWord*>(wakeHandle);
643
+ char* outError = nullptr;
644
+ eliza_inference_wakeword_reset(wake, &outError);
645
+ if (outError) std::free(outError);
646
+ }
647
+
648
+ JNIEXPORT void JNICALL
649
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeWakewordClose(JNIEnv*, jclass,
650
+ jlong wakeHandle) {
651
+ eliza_inference_wakeword_close(reinterpret_cast<EliWakeWord*>(wakeHandle));
652
+ }
653
+
654
+ // ── Speaker encoder ops ──────────────────────────────────────────────────
655
+
656
+ JNIEXPORT jlong JNICALL
657
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeSpeakerOpen(JNIEnv* env, jclass,
658
+ jlong ctxHandle,
659
+ jstring jGgufPath) {
660
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
661
+ const std::string gguf = from_jstring(env, jGgufPath);
662
+ char* outError = nullptr;
663
+ EliSpeaker* sp = eliza_inference_speaker_open(
664
+ ctx, gguf.empty() ? nullptr : gguf.c_str(), &outError);
665
+ if (!sp) {
666
+ throw_runtime(env, "speaker_open returned null", outError);
667
+ return 0;
668
+ }
669
+ return reinterpret_cast<jlong>(sp);
670
+ }
671
+
672
+ // Embed a turn's PCM into a 256-float L2-normalized speaker embedding.
673
+ JNIEXPORT jfloatArray JNICALL
674
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeSpeakerEmbed(JNIEnv* env, jclass,
675
+ jlong spHandle,
676
+ jfloatArray jPcm) {
677
+ auto* sp = reinterpret_cast<EliSpeaker*>(spHandle);
678
+ const std::vector<float> pcm = read_float_array(env, jPcm);
679
+ std::vector<float> emb(kSpeakerEmbeddingDim, 0.0f);
680
+ char* outError = nullptr;
681
+ const int rc = eliza_inference_speaker_embed(sp, pcm.data(), pcm.size(),
682
+ emb.data(), &outError);
683
+ if (rc != ELIZA_OK) {
684
+ throw_runtime(env, "speaker_embed", outError);
685
+ return nullptr;
686
+ }
687
+ jfloatArray out = env->NewFloatArray(static_cast<jsize>(kSpeakerEmbeddingDim));
688
+ if (out) {
689
+ env->SetFloatArrayRegion(out, 0, static_cast<jsize>(kSpeakerEmbeddingDim),
690
+ emb.data());
691
+ }
692
+ return out;
693
+ }
694
+
695
+ JNIEXPORT void JNICALL
696
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeSpeakerClose(JNIEnv*, jclass,
697
+ jlong spHandle) {
698
+ eliza_inference_speaker_close(reinterpret_cast<EliSpeaker*>(spHandle));
699
+ }
700
+
701
+ // ── Diarizer ops ─────────────────────────────────────────────────────────
702
+
703
+ JNIEXPORT jlong JNICALL
704
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeDiarizOpen(JNIEnv* env, jclass,
705
+ jlong ctxHandle,
706
+ jstring jGgufPath) {
707
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
708
+ const std::string gguf = from_jstring(env, jGgufPath);
709
+ char* outError = nullptr;
710
+ EliDiariz* di = eliza_inference_diariz_open(
711
+ ctx, gguf.empty() ? nullptr : gguf.c_str(), &outError);
712
+ if (!di) {
713
+ throw_runtime(env, "diariz_open returned null", outError);
714
+ return 0;
715
+ }
716
+ return reinterpret_cast<jlong>(di);
717
+ }
718
+
719
+ // Segment a fixed 80000-sample (5 s) window into per-frame int8 powerset
720
+ // labels; returns the labels as a Java byte[].
721
+ JNIEXPORT jbyteArray JNICALL
722
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeDiarizSegment(JNIEnv* env, jclass,
723
+ jlong diHandle,
724
+ jfloatArray jPcm) {
725
+ auto* di = reinterpret_cast<EliDiariz*>(diHandle);
726
+ std::vector<float> pcm = read_float_array(env, jPcm);
727
+ // pyannote takes exactly 80000 samples — pad or crop.
728
+ std::vector<float> window(kDiarizWindow, 0.0f);
729
+ const size_t copy = pcm.size() < kDiarizWindow ? pcm.size() : kDiarizWindow;
730
+ std::memcpy(window.data(), pcm.data(), copy * sizeof(float));
731
+ std::vector<int8_t> out(kDiarizLabelCap, 0);
732
+ size_t nLabels = out.size();
733
+ char* outError = nullptr;
734
+ const int rc = eliza_inference_diariz_segment(
735
+ di, window.data(), window.size(), out.data(), &nLabels, &outError);
736
+ if (rc != ELIZA_OK) {
737
+ throw_runtime(env, "diariz_segment", outError);
738
+ return nullptr;
739
+ }
740
+ jbyteArray ja = env->NewByteArray(static_cast<jsize>(nLabels));
741
+ if (ja && nLabels > 0) {
742
+ env->SetByteArrayRegion(ja, 0, static_cast<jsize>(nLabels),
743
+ reinterpret_cast<const jbyte*>(out.data()));
744
+ }
745
+ return ja;
746
+ }
747
+
748
+ JNIEXPORT void JNICALL
749
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeDiarizClose(JNIEnv*, jclass,
750
+ jlong diHandle) {
751
+ eliza_inference_diariz_close(reinterpret_cast<EliDiariz*>(diHandle));
752
+ }
753
+
754
+ // ── Native streaming pipeline (the hot-loop owner) ───────────────────────
755
+
756
+ // Open a pipeline session on a context: opens VAD + speaker + diariz (each
757
+ // best-effort; a missing classifier is reported as null and that turn payload
758
+ // is empty). Returns an opaque handle.
759
+ JNIEXPORT jlong JNICALL
760
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineOpen(JNIEnv* env, jclass,
761
+ jlong ctxHandle) {
762
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
763
+ if (!ctx) {
764
+ throw_runtime(env, "pipelineOpen: null context", nullptr);
765
+ return 0;
766
+ }
767
+ auto* s = new PipelineSession();
768
+ s->ctx = ctx;
769
+ char* outError = nullptr;
770
+
771
+ s->vad = eliza_inference_vad_open(ctx, kSampleRate, &outError);
772
+ if (!s->vad) {
773
+ delete s;
774
+ throw_runtime(env, "pipelineOpen: vad_open returned null", outError);
775
+ return 0;
776
+ }
777
+ // Speaker + diariz are required for a real attributed turn; open both.
778
+ s->speaker = eliza_inference_speaker_open(ctx, nullptr, &outError);
779
+ if (!s->speaker) {
780
+ eliza_inference_vad_close(s->vad);
781
+ delete s;
782
+ throw_runtime(env, "pipelineOpen: speaker_open returned null", outError);
783
+ return 0;
784
+ }
785
+ s->diariz = eliza_inference_diariz_open(ctx, nullptr, &outError);
786
+ if (!s->diariz) {
787
+ eliza_inference_speaker_close(s->speaker);
788
+ eliza_inference_vad_close(s->vad);
789
+ delete s;
790
+ throw_runtime(env, "pipelineOpen: diariz_open returned null", outError);
791
+ return 0;
792
+ }
793
+ LOGI("pipelineOpen -> %p (vad+speaker+diariz)", (void*)s);
794
+ return reinterpret_cast<jlong>(s);
795
+ }
796
+
797
+ // Feed one audio-frame batch (16 kHz mono fp32) into the pipeline. Runs the VAD
798
+ // hot-loop + segmentation natively; on each speech-end runs speaker+diariz.
799
+ // Returns a JSON array of turns completed in THIS call (may be empty). The
800
+ // heavy embedding/labels payloads are read per-index via the getters below.
801
+ JNIEXPORT jstring JNICALL
802
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineProcess(JNIEnv* env, jclass,
803
+ jlong handle,
804
+ jfloatArray jPcm) {
805
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
806
+ if (!s) {
807
+ throw_runtime(env, "pipelineProcess: null session", nullptr);
808
+ return nullptr;
809
+ }
810
+ s->turns.clear();
811
+ s->turnEmbeddings.clear();
812
+ s->turnLabels.clear();
813
+ s->turnPcms.clear();
814
+
815
+ const std::vector<float> pcm = read_float_array(env, jPcm);
816
+ s->pending.insert(s->pending.end(), pcm.begin(), pcm.end());
817
+ char* outError = nullptr;
818
+ if (!drain_windows(s, &outError)) {
819
+ throw_runtime(env, "pipelineProcess: drain", outError);
820
+ return nullptr;
821
+ }
822
+ if (!s->turns.empty()) {
823
+ LOGI("pipelineProcess: fed %zu samples, %zu turn(s) completed, "
824
+ "capturing=%d", pcm.size(), s->turns.size(), s->capturing ? 1 : 0);
825
+ }
826
+ return to_jstring(env, drain_turns_json(s));
827
+ }
828
+
829
+ // Force-finalize any open turn (call at end-of-capture). Returns the JSON array
830
+ // of any turn completed by the flush.
831
+ JNIEXPORT jstring JNICALL
832
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineFlush(JNIEnv* env, jclass,
833
+ jlong handle) {
834
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
835
+ if (!s) {
836
+ throw_runtime(env, "pipelineFlush: null session", nullptr);
837
+ return nullptr;
838
+ }
839
+ s->turns.clear();
840
+ s->turnEmbeddings.clear();
841
+ s->turnLabels.clear();
842
+ s->turnPcms.clear();
843
+ if (s->seg.forceEnd()) {
844
+ s->capturing = false;
845
+ char* outError = nullptr;
846
+ eliza_inference_vad_reset(s->vad, &outError);
847
+ if (outError) std::free(outError);
848
+ outError = nullptr;
849
+ if (!finalize_turn(s, &outError)) {
850
+ throw_runtime(env, "pipelineFlush: finalize", outError);
851
+ return nullptr;
852
+ }
853
+ }
854
+ return to_jstring(env, drain_turns_json(s));
855
+ }
856
+
857
+ // Read the speaker embedding (256 floats) for the i-th turn produced by the
858
+ // last process()/flush() call. Empty array when that turn had no embedding.
859
+ JNIEXPORT jfloatArray JNICALL
860
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineTurnEmbedding(JNIEnv* env,
861
+ jclass,
862
+ jlong handle,
863
+ jint index) {
864
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
865
+ if (!s || index < 0 ||
866
+ static_cast<size_t>(index) >= s->turnEmbeddings.size()) {
867
+ return env->NewFloatArray(0);
868
+ }
869
+ const auto& emb = s->turnEmbeddings[static_cast<size_t>(index)];
870
+ jfloatArray out = env->NewFloatArray(static_cast<jsize>(emb.size()));
871
+ if (out && !emb.empty()) {
872
+ env->SetFloatArrayRegion(out, 0, static_cast<jsize>(emb.size()),
873
+ emb.data());
874
+ }
875
+ return out;
876
+ }
877
+
878
+ // Read the diariz int8 frame labels for the i-th turn.
879
+ JNIEXPORT jbyteArray JNICALL
880
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineTurnLabels(JNIEnv* env,
881
+ jclass,
882
+ jlong handle,
883
+ jint index) {
884
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
885
+ if (!s || index < 0 ||
886
+ static_cast<size_t>(index) >= s->turnLabels.size()) {
887
+ return env->NewByteArray(0);
888
+ }
889
+ const auto& labels = s->turnLabels[static_cast<size_t>(index)];
890
+ jbyteArray out = env->NewByteArray(static_cast<jsize>(labels.size()));
891
+ if (out && !labels.empty()) {
892
+ env->SetByteArrayRegion(out, 0, static_cast<jsize>(labels.size()),
893
+ reinterpret_cast<const jbyte*>(labels.data()));
894
+ }
895
+ return out;
896
+ }
897
+
898
+ // Read the segmented turn PCM for the i-th turn. Empty array when unavailable.
899
+ JNIEXPORT jfloatArray JNICALL
900
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineTurnPcm(JNIEnv* env,
901
+ jclass,
902
+ jlong handle,
903
+ jint index) {
904
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
905
+ if (!s || index < 0 ||
906
+ static_cast<size_t>(index) >= s->turnPcms.size()) {
907
+ return env->NewFloatArray(0);
908
+ }
909
+ const auto& pcm = s->turnPcms[static_cast<size_t>(index)];
910
+ jfloatArray out = env->NewFloatArray(static_cast<jsize>(pcm.size()));
911
+ if (out && !pcm.empty()) {
912
+ env->SetFloatArrayRegion(out, 0, static_cast<jsize>(pcm.size()),
913
+ pcm.data());
914
+ }
915
+ return out;
916
+ }
917
+
918
+ JNIEXPORT void JNICALL
919
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineReset(JNIEnv*, jclass,
920
+ jlong handle) {
921
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
922
+ if (!s) return;
923
+ s->seg.reset();
924
+ s->pending.clear();
925
+ s->turnPcm.clear();
926
+ s->turnPcms.clear();
927
+ s->preRoll.clear();
928
+ s->capturing = false;
929
+ char* outError = nullptr;
930
+ if (s->vad) eliza_inference_vad_reset(s->vad, &outError);
931
+ if (outError) std::free(outError);
932
+ }
933
+
934
+ JNIEXPORT void JNICALL
935
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineClose(JNIEnv*, jclass,
936
+ jlong handle) {
937
+ auto* s = reinterpret_cast<PipelineSession*>(handle);
938
+ if (!s) return;
939
+ if (s->diariz) eliza_inference_diariz_close(s->diariz);
940
+ if (s->speaker) eliza_inference_speaker_close(s->speaker);
941
+ if (s->vad) eliza_inference_vad_close(s->vad);
942
+ delete s;
943
+ }
944
+
945
+ // ── Pipeline self-test (one native call: ctx→open→feed→flush) ────────────
946
+ //
947
+ // Runs the WHOLE native pipeline on a complete PCM buffer in ONE call: creates
948
+ // a context, opens the VAD+speaker+diariz pipeline, streams the PCM through the
949
+ // native VAD hot-loop + turn segmentation, flushes, and returns the turn JSON
950
+ // (the per-op TURN logcat lines are the in-process evidence). Single call so it
951
+ // does not depend on chained Capacitor awaits. `feedSamples` chunks the feed to
952
+ // mimic the live audioFrame batching (0 = feed the whole buffer at once).
953
+ JNIEXPORT jstring JNICALL
954
+ Java_ai_elizaos_app_ElizaVoiceNative_nativePipelineSelfTest(JNIEnv* env, jclass,
955
+ jstring jBundleDir,
956
+ jfloatArray jPcm,
957
+ jint feedSamples) {
958
+ const std::string bundleDir = from_jstring(env, jBundleDir);
959
+ char* outError = nullptr;
960
+ EliInferenceContext* ctx =
961
+ eliza_inference_create(bundleDir.c_str(), &outError);
962
+ if (!ctx) {
963
+ throw_runtime(env, "pipelineSelfTest: create", outError);
964
+ return nullptr;
965
+ }
966
+ auto* s = new PipelineSession();
967
+ s->ctx = ctx;
968
+ s->vad = eliza_inference_vad_open(ctx, kSampleRate, &outError);
969
+ if (!s->vad) { delete s; eliza_inference_destroy(ctx);
970
+ throw_runtime(env, "pipelineSelfTest: vad_open", outError); return nullptr; }
971
+ s->speaker = eliza_inference_speaker_open(ctx, nullptr, &outError);
972
+ if (!s->speaker) { eliza_inference_vad_close(s->vad); delete s;
973
+ eliza_inference_destroy(ctx);
974
+ throw_runtime(env, "pipelineSelfTest: speaker_open", outError); return nullptr; }
975
+ s->diariz = eliza_inference_diariz_open(ctx, nullptr, &outError);
976
+ if (!s->diariz) { eliza_inference_speaker_close(s->speaker);
977
+ eliza_inference_vad_close(s->vad); delete s; eliza_inference_destroy(ctx);
978
+ throw_runtime(env, "pipelineSelfTest: diariz_open", outError); return nullptr; }
979
+
980
+ const std::vector<float> pcm = read_float_array(env, jPcm);
981
+ LOGI("pipelineSelfTest: feeding %zu samples (%.2fs), chunk=%d",
982
+ pcm.size(), pcm.size() / (double)kSampleRate, feedSamples);
983
+ const size_t chunk = feedSamples > 0 ? static_cast<size_t>(feedSamples)
984
+ : pcm.size();
985
+ std::vector<std::string> allTurns;
986
+ for (size_t off = 0; off < pcm.size(); off += chunk) {
987
+ const size_t end = std::min(off + chunk, pcm.size());
988
+ s->pending.insert(s->pending.end(), pcm.begin() + off, pcm.begin() + end);
989
+ s->turns.clear(); s->turnEmbeddings.clear(); s->turnLabels.clear();
990
+ if (!drain_windows(s, &outError)) {
991
+ cleanup_session_for_selftest(s, ctx);
992
+ throw_runtime(env, "pipelineSelfTest: drain", outError);
993
+ return nullptr;
994
+ }
995
+ for (auto& t : s->turns) allTurns.push_back(t);
996
+ }
997
+ // Flush any open turn (the speech-end via end-hangover may already have
998
+ // fired mid-stream; a force-flush catches a turn still open at EOF).
999
+ s->turns.clear(); s->turnEmbeddings.clear(); s->turnLabels.clear();
1000
+ if (s->seg.forceEnd()) {
1001
+ s->capturing = false;
1002
+ eliza_inference_vad_reset(s->vad, &outError);
1003
+ if (outError) { std::free(outError); outError = nullptr; }
1004
+ if (finalize_turn(s, &outError)) {
1005
+ for (auto& t : s->turns) allTurns.push_back(t);
1006
+ } else if (outError) { std::free(outError); outError = nullptr; }
1007
+ }
1008
+ std::string json = "[";
1009
+ for (size_t i = 0; i < allTurns.size(); ++i) {
1010
+ if (i) json += ",";
1011
+ json += allTurns[i];
1012
+ }
1013
+ json += "]";
1014
+ LOGI("pipelineSelfTest: %zu total turn(s)", allTurns.size());
1015
+ cleanup_session_for_selftest(s, ctx);
1016
+ return to_jstring(env, json);
1017
+ }
1018
+
1019
+ // ── Wake-word self-test (one native call: open + score pos + score neg) ──
1020
+ //
1021
+ // Scores two pre-decoded float[] clips through one wake-word session
1022
+ // (reset between them) and logs both max P(wake) to logcat. Single call so the
1023
+ // JS side does not chain awaits across the broken-under-CDP Capacitor result
1024
+ // delivery — the in-process evidence is the logcat line.
1025
+ JNIEXPORT jstring JNICALL
1026
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeWakewordSelfTest(JNIEnv* env, jclass,
1027
+ jstring jBundleDir,
1028
+ jfloatArray jPos,
1029
+ jfloatArray jNeg) {
1030
+ const std::string bundleDir = from_jstring(env, jBundleDir);
1031
+ char* outError = nullptr;
1032
+ EliInferenceContext* ctx =
1033
+ eliza_inference_create(bundleDir.c_str(), &outError);
1034
+ if (!ctx) {
1035
+ throw_runtime(env, "wakewordSelfTest: create", outError);
1036
+ return nullptr;
1037
+ }
1038
+ EliWakeWord* wake =
1039
+ eliza_inference_wakeword_open(ctx, kSampleRate, "hey-eliza", &outError);
1040
+ if (!wake) {
1041
+ eliza_inference_destroy(ctx);
1042
+ throw_runtime(env, "wakewordSelfTest: wakeword_open", outError);
1043
+ return nullptr;
1044
+ }
1045
+ auto scoreMax = [&](jfloatArray jPcm) -> float {
1046
+ const std::vector<float> pcm = read_float_array(env, jPcm);
1047
+ const size_t frames = pcm.size() / kWakeFrame;
1048
+ float maxP = 0.0f;
1049
+ for (size_t f = 0; f < frames; ++f) {
1050
+ float p = -1.0f;
1051
+ char* e = nullptr;
1052
+ if (eliza_inference_wakeword_score(wake, pcm.data() + f * kWakeFrame,
1053
+ kWakeFrame, &p, &e) == ELIZA_OK) {
1054
+ maxP = std::max(maxP, p);
1055
+ }
1056
+ if (e) std::free(e);
1057
+ }
1058
+ return maxP;
1059
+ };
1060
+ const float posMax = scoreMax(jPos);
1061
+ eliza_inference_wakeword_reset(wake, &outError);
1062
+ if (outError) { std::free(outError); outError = nullptr; }
1063
+ const float negMax = scoreMax(jNeg);
1064
+ eliza_inference_wakeword_close(wake);
1065
+ eliza_inference_destroy(ctx);
1066
+ LOGI("WAKEWORD SELFTEST: posMax=%.4f negMax=%.4f (pos should >> neg)",
1067
+ posMax, negMax);
1068
+ std::string j = "{\"posMax\":" + std::to_string(posMax) +
1069
+ ",\"negMax\":" + std::to_string(negMax) + "}";
1070
+ return to_jstring(env, j);
1071
+ }
1072
+
1073
+ // ── VAD self-test (Phase 3a, retained for the existing harness) ──────────
1074
+
1075
+ JNIEXPORT jstring JNICALL
1076
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVadSelfTest(JNIEnv* env, jclass,
1077
+ jstring jBundleDir) {
1078
+ const char* abi = eliza_inference_abi_version();
1079
+ const std::string abiStr = abi ? std::string(abi) : std::string();
1080
+ const int supported = eliza_inference_vad_supported();
1081
+ const std::string bundleDir = from_jstring(env, jBundleDir);
1082
+
1083
+ auto fail = [&](const char* stage, const std::string& err) -> jstring {
1084
+ std::string j = std::string("{\"ok\":false,\"stage\":\"") + stage +
1085
+ "\",\"error\":\"" + err + "\",\"abi\":\"" + abiStr +
1086
+ "\",\"supported\":" + std::to_string(supported) + "}";
1087
+ LOGE("nativeVadSelfTest failed at %s: %s", stage, err.c_str());
1088
+ return to_jstring(env, j);
1089
+ };
1090
+
1091
+ if (bundleDir.empty()) return fail("bundle_dir", "empty bundle dir");
1092
+
1093
+ char* outError = nullptr;
1094
+ EliInferenceContext* ctx =
1095
+ eliza_inference_create(bundleDir.c_str(), &outError);
1096
+ if (ctx == nullptr) {
1097
+ std::string err =
1098
+ outError ? std::string(outError) : std::string("create returned null");
1099
+ if (outError) std::free(outError);
1100
+ return fail("create", err);
1101
+ }
1102
+ EliVad* vad = eliza_inference_vad_open(ctx, kSampleRate, &outError);
1103
+ if (vad == nullptr) {
1104
+ std::string err = outError ? std::string(outError)
1105
+ : std::string("vad_open returned null");
1106
+ if (outError) std::free(outError);
1107
+ eliza_inference_destroy(ctx);
1108
+ return fail("vad_open", err);
1109
+ }
1110
+ std::vector<float> pcm(kVadWindow);
1111
+ for (size_t i = 0; i < pcm.size(); ++i) {
1112
+ pcm[i] = 0.25f *
1113
+ std::sin(2.0 * M_PI * 200.0 * (double)i / kSampleRate);
1114
+ }
1115
+ float probability = -1.0f;
1116
+ int rc = eliza_inference_vad_process(vad, pcm.data(), pcm.size(),
1117
+ &probability, &outError);
1118
+ if (rc != ELIZA_OK) {
1119
+ std::string err =
1120
+ outError ? std::string(outError) : ("rc=" + std::to_string(rc));
1121
+ if (outError) std::free(outError);
1122
+ eliza_inference_vad_close(vad);
1123
+ eliza_inference_destroy(ctx);
1124
+ return fail("vad_process", err);
1125
+ }
1126
+ if (outError) std::free(outError);
1127
+ eliza_inference_vad_close(vad);
1128
+ eliza_inference_destroy(ctx);
1129
+
1130
+ const bool finite = std::isfinite(probability);
1131
+ std::string j = std::string("{\"ok\":") + (finite ? "true" : "false") +
1132
+ ",\"probability\":" + std::to_string(probability) +
1133
+ ",\"abi\":\"" + abiStr + "\",\"supported\":" +
1134
+ std::to_string(supported) + "}";
1135
+ LOGI("nativeVadSelfTest ok: probability=%f abi=%s supported=%d", probability,
1136
+ abiStr.c_str(), supported);
1137
+ return to_jstring(env, j);
1138
+ }
1139
+
1140
+ // ── Text generation (LLM) ops — the GPU-accelerated text path ────────────
1141
+ //
1142
+ // Wrap the fused streaming-LLM ABI (eliza_inference_llm_stream_*), pooled
1143
+ // embeddings (eliza_inference_embed), end-of-turn scoring
1144
+ // (eliza_inference_llm_eot_score, ABI v11), and the tokenizer. When this JNI
1145
+ // host is built against the DYNAMIC-Vulkan libelizainference (libggml-vulkan.so
1146
+ // staged alongside it), llm_stream_open offloads the model to the GPU in the
1147
+ // bionic app process automatically — the path the musl bun agent cannot take.
1148
+
1149
+ JNIEXPORT jint JNICALL
1150
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamSupported(JNIEnv*, jclass) {
1151
+ return static_cast<jint>(eliza_inference_llm_stream_supported());
1152
+ }
1153
+
1154
+ JNIEXPORT jint JNICALL
1155
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeEmbedSupported(JNIEnv*, jclass) {
1156
+ return static_cast<jint>(eliza_inference_embed_supported());
1157
+ }
1158
+
1159
+ JNIEXPORT jint JNICALL
1160
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeEotSupported(JNIEnv*, jclass) {
1161
+ return static_cast<jint>(eliza_inference_llm_eot_supported());
1162
+ }
1163
+
1164
+ // Tokenize text -> int[] token ids. addSpecial adds BOS; parseSpecial renders
1165
+ // special tokens (<|im_start|> etc.) from the input.
1166
+ JNIEXPORT jintArray JNICALL
1167
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeTokenize(JNIEnv* env, jclass,
1168
+ jlong ctxHandle,
1169
+ jstring jText,
1170
+ jboolean addSpecial,
1171
+ jboolean parseSpecial) {
1172
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1173
+ const std::string text = from_jstring(env, jText);
1174
+ int* toks = nullptr;
1175
+ size_t n = 0;
1176
+ char* outError = nullptr;
1177
+ const int rc = eliza_inference_tokenize(
1178
+ ctx, text.c_str(), text.size(), addSpecial ? 1 : 0,
1179
+ parseSpecial ? 1 : 0, &toks, &n, &outError);
1180
+ if (rc != ELIZA_OK) {
1181
+ throw_runtime(env, "tokenize", outError);
1182
+ return nullptr;
1183
+ }
1184
+ jintArray out = env->NewIntArray(static_cast<jsize>(n));
1185
+ if (out && n > 0) {
1186
+ env->SetIntArrayRegion(out, 0, static_cast<jsize>(n),
1187
+ reinterpret_cast<const jint*>(toks));
1188
+ }
1189
+ if (toks) eliza_inference_free_tokens(toks);
1190
+ return out;
1191
+ }
1192
+
1193
+ // Pooled, L2-normalized sentence embedding (pooling: 1=MEAN default) ->
1194
+ // float[n_embd].
1195
+ JNIEXPORT jfloatArray JNICALL
1196
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeEmbed(JNIEnv* env, jclass,
1197
+ jlong ctxHandle, jstring jText,
1198
+ jint pooling) {
1199
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1200
+ const std::string text = from_jstring(env, jText);
1201
+ std::vector<float> out(4096, 0.0f);
1202
+ int dim = 0;
1203
+ char* outError = nullptr;
1204
+ const int rc = eliza_inference_embed(ctx, text.c_str(), text.size(),
1205
+ pooling > 0 ? pooling : 1, out.data(),
1206
+ out.size(), &dim, &outError);
1207
+ if (rc != ELIZA_OK) {
1208
+ throw_runtime(env, "embed", outError);
1209
+ return nullptr;
1210
+ }
1211
+ jfloatArray ja = env->NewFloatArray(dim);
1212
+ if (ja && dim > 0) env->SetFloatArrayRegion(ja, 0, dim, out.data());
1213
+ return ja;
1214
+ }
1215
+
1216
+ // End-of-turn score: next-token P(targetToken | tokens) -> float.
1217
+ JNIEXPORT jfloat JNICALL
1218
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeEotScore(JNIEnv* env, jclass,
1219
+ jlong ctxHandle,
1220
+ jintArray jTokens,
1221
+ jint targetToken) {
1222
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1223
+ const jsize n = env->GetArrayLength(jTokens);
1224
+ std::vector<int32_t> toks(static_cast<size_t>(n));
1225
+ if (n > 0) {
1226
+ env->GetIntArrayRegion(jTokens, 0, n,
1227
+ reinterpret_cast<jint*>(toks.data()));
1228
+ }
1229
+ float prob = 0.0f, topProb = 0.0f;
1230
+ int32_t topTok = -1;
1231
+ char* outError = nullptr;
1232
+ const int rc = eliza_inference_llm_eot_score(ctx, toks.data(), toks.size(),
1233
+ targetToken, &prob, &topTok,
1234
+ &topProb, &outError);
1235
+ if (rc != ELIZA_OK) {
1236
+ throw_runtime(env, "eot_score", outError);
1237
+ return 0.0f;
1238
+ }
1239
+ return prob;
1240
+ }
1241
+
1242
+ // Open a streaming-LLM session. nGpuLayers: -1 = all-GPU (default), 0 = CPU
1243
+ // (the lib ignores 0 when libggml-vulkan is linked; the CPU/GPU choice is the
1244
+ // staged LIB VARIANT, see the per-device selection). drafterPath ("" = none)
1245
+ // enables MTP speculative decoding.
1246
+ JNIEXPORT jlong JNICALL
1247
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamOpen(
1248
+ JNIEnv* env, jclass, jlong ctxHandle, jint maxTokens, jfloat temperature,
1249
+ jfloat topP, jint topK, jint nGpuLayers, jstring jDrafterPath) {
1250
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1251
+ const std::string drafter = from_jstring(env, jDrafterPath);
1252
+ eliza_llm_stream_config_t cfg;
1253
+ std::memset(&cfg, 0, sizeof(cfg));
1254
+ cfg.max_tokens = maxTokens;
1255
+ cfg.temperature = temperature;
1256
+ cfg.top_p = topP > 0 ? topP : 1.0f;
1257
+ cfg.top_k = topK;
1258
+ cfg.repeat_penalty = 1.0f;
1259
+ cfg.n_gpu_layers = nGpuLayers;
1260
+ cfg.mtp_drafter_path = drafter.empty() ? nullptr : drafter.c_str();
1261
+ // Arm safe runtime optimizations: stock f16/q8_0 KV for the shipped Gemma 4
1262
+ // tiers, and separate-drafter MTP only when the caller passes a drafter.
1263
+ arm_bionic_text_cfg(cfg);
1264
+ if (!drafter.empty() && cfg.draft_min <= 0) {
1265
+ // Separate drafter supplied but env left the window 0; give it the
1266
+ // single-head default so the drafter actually drives speculation.
1267
+ cfg.draft_min = 1;
1268
+ cfg.draft_max = 1;
1269
+ }
1270
+ char* outError = nullptr;
1271
+ EliLlmStream* s = eliza_inference_llm_stream_open(ctx, &cfg, &outError);
1272
+ if (!s) {
1273
+ throw_runtime(env, "llm_stream_open returned null", outError);
1274
+ return 0;
1275
+ }
1276
+ return reinterpret_cast<jlong>(s);
1277
+ }
1278
+
1279
+ JNIEXPORT void JNICALL
1280
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamPrefill(JNIEnv* env, jclass,
1281
+ jlong streamHandle,
1282
+ jintArray jTokens) {
1283
+ auto* s = reinterpret_cast<EliLlmStream*>(streamHandle);
1284
+ const jsize n = env->GetArrayLength(jTokens);
1285
+ std::vector<int32_t> toks(static_cast<size_t>(n));
1286
+ if (n > 0) {
1287
+ env->GetIntArrayRegion(jTokens, 0, n,
1288
+ reinterpret_cast<jint*>(toks.data()));
1289
+ }
1290
+ char* outError = nullptr;
1291
+ const int rc = eliza_inference_llm_stream_prefill(s, toks.data(),
1292
+ toks.size(), &outError);
1293
+ if (rc != ELIZA_OK) throw_runtime(env, "llm_stream_prefill", outError);
1294
+ }
1295
+
1296
+ // Pull the next decode step. Returns JSON {text, done, drafted, accepted}:
1297
+ // `text` is the detokenized chunk (may span multiple committed tokens via MTP),
1298
+ // `done` true at the final step. `text` is JSON-escaped.
1299
+ JNIEXPORT jstring JNICALL
1300
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamNext(JNIEnv* env, jclass,
1301
+ jlong streamHandle) {
1302
+ auto* s = reinterpret_cast<EliLlmStream*>(streamHandle);
1303
+ int32_t toks[256];
1304
+ char text[4096];
1305
+ size_t nout = 0;
1306
+ int32_t drafted = 0, accepted = 0;
1307
+ char* outError = nullptr;
1308
+ const int rc = eliza_inference_llm_stream_next(
1309
+ s, toks, 256, &nout, text, sizeof(text), &drafted, &accepted, &outError);
1310
+ if (rc < 0) {
1311
+ throw_runtime(env, "llm_stream_next", outError);
1312
+ return nullptr;
1313
+ }
1314
+ std::string esc;
1315
+ for (const char* p = text; *p; ++p) {
1316
+ switch (*p) {
1317
+ case '"': esc += "\\\""; break;
1318
+ case '\\': esc += "\\\\"; break;
1319
+ case '\n': esc += "\\n"; break;
1320
+ case '\r': esc += "\\r"; break;
1321
+ case '\t': esc += "\\t"; break;
1322
+ default:
1323
+ if (static_cast<unsigned char>(*p) < 0x20) {
1324
+ char buf[8];
1325
+ std::snprintf(buf, sizeof(buf), "\\u%04x",
1326
+ static_cast<unsigned char>(*p));
1327
+ esc += buf;
1328
+ } else {
1329
+ esc += *p;
1330
+ }
1331
+ }
1332
+ }
1333
+ std::string json = "{\"text\":\"" + esc +
1334
+ "\",\"done\":" + (rc == 1 ? "true" : "false") +
1335
+ ",\"nout\":" + std::to_string(nout) +
1336
+ ",\"drafted\":" + std::to_string(drafted) +
1337
+ ",\"accepted\":" + std::to_string(accepted) + "}";
1338
+ return to_jstring(env, json);
1339
+ }
1340
+
1341
+ JNIEXPORT void JNICALL
1342
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamClose(JNIEnv*, jclass,
1343
+ jlong streamHandle) {
1344
+ eliza_inference_llm_stream_close(
1345
+ reinterpret_cast<EliLlmStream*>(streamHandle));
1346
+ }
1347
+
1348
+ // Reset a persistent stream (clear KV + sampler + counters) for warm reuse.
1349
+ // Returns 1 on success, 0 if the stream can't be reset (MTP / null).
1350
+ JNIEXPORT jint JNICALL
1351
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamReset(JNIEnv*, jclass,
1352
+ jlong streamHandle) {
1353
+ const int rc = eliza_inference_llm_stream_reset(
1354
+ reinterpret_cast<EliLlmStream*>(streamHandle));
1355
+ return static_cast<jint>(rc == ELIZA_OK ? 1 : 0);
1356
+ }
1357
+
1358
+ // Prefix-preserving reset: keep the first nKeep tokens of KV resident, drop the
1359
+ // rest. Returns the n_keep actually applied (>= 0), or a negative ELIZA_* on a
1360
+ // NULL / MTP / unopened stream (caller falls back to a full reset + prefill).
1361
+ JNIEXPORT jint JNICALL
1362
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmStreamResetKeep(JNIEnv*, jclass,
1363
+ jlong streamHandle,
1364
+ jint nKeep) {
1365
+ const int rc = eliza_inference_llm_stream_reset_keep(
1366
+ reinterpret_cast<EliLlmStream*>(streamHandle), static_cast<int32_t>(nKeep));
1367
+ return static_cast<jint>(rc);
1368
+ }
1369
+
1370
+ // ── LLM self-test (one native call: ctx→tokenize→stream→generate) ─────────
1371
+ //
1372
+ // THE KEYSTONE PROOF: runs a whole greedy text generation in ONE native call,
1373
+ // in the bionic app process, against whatever libelizainference.so is staged
1374
+ // into jniLibs. When that lib is the dynamic-Vulkan variant, ggml-vulkan logs
1375
+ // "Found 1 Vulkan devices: Mali-G715" + "offloaded N/N layers to GPU" to
1376
+ // logcat (the in-process GPU evidence). Returns JSON {ok,text,tokens,ms,tokS}.
1377
+ JNIEXPORT jstring JNICALL
1378
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeLlmSelfTest(JNIEnv* env, jclass,
1379
+ jstring jBundleDir,
1380
+ jstring jPrompt,
1381
+ jint maxTokens) {
1382
+ const std::string bundleDir = from_jstring(env, jBundleDir);
1383
+ const std::string prompt = from_jstring(env, jPrompt);
1384
+ const int genCap = maxTokens > 0 ? maxTokens : 32;
1385
+ char* outError = nullptr;
1386
+
1387
+ EliInferenceContext* ctx =
1388
+ eliza_inference_create(bundleDir.c_str(), &outError);
1389
+ if (!ctx) { throw_runtime(env, "llmSelfTest: create", outError); return nullptr; }
1390
+
1391
+ int* tok = nullptr; size_t tn = 0;
1392
+ if (eliza_inference_tokenize(ctx, prompt.c_str(), prompt.size(), 1, 1, &tok,
1393
+ &tn, &outError) != ELIZA_OK) {
1394
+ eliza_inference_destroy(ctx);
1395
+ throw_runtime(env, "llmSelfTest: tokenize", outError);
1396
+ return nullptr;
1397
+ }
1398
+
1399
+ eliza_llm_stream_config_t cfg;
1400
+ std::memset(&cfg, 0, sizeof(cfg));
1401
+ cfg.max_tokens = genCap;
1402
+ cfg.temperature = 0.0f; // greedy, deterministic
1403
+ cfg.top_k = 1;
1404
+ cfg.top_p = 1.0f;
1405
+ cfg.repeat_penalty = 1.0f;
1406
+ cfg.n_gpu_layers = -1; // all-GPU when the vulkan lib is staged
1407
+ // Arm safe runtime optimizations (stock f16/q8_0 KV for shipped Gemma 4
1408
+ // tiers; no MTP here because nativeLlmSelfTest has no drafter argument).
1409
+ arm_bionic_text_cfg(cfg, bundleDir.c_str());
1410
+ EliLlmStream* s = eliza_inference_llm_stream_open(ctx, &cfg, &outError);
1411
+ if (!s) {
1412
+ if (tok) eliza_inference_free_tokens(tok);
1413
+ eliza_inference_destroy(ctx);
1414
+ throw_runtime(env, "llmSelfTest: stream_open", outError);
1415
+ return nullptr;
1416
+ }
1417
+
1418
+ const double t0 = []() {
1419
+ timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
1420
+ return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
1421
+ }();
1422
+ if (eliza_inference_llm_stream_prefill(s, reinterpret_cast<int32_t*>(tok),
1423
+ tn, &outError) != ELIZA_OK) {
1424
+ eliza_inference_llm_stream_close(s);
1425
+ if (tok) eliza_inference_free_tokens(tok);
1426
+ eliza_inference_destroy(ctx);
1427
+ throw_runtime(env, "llmSelfTest: prefill", outError);
1428
+ return nullptr;
1429
+ }
1430
+
1431
+ std::string text;
1432
+ int produced = 0;
1433
+ while (produced < genCap) {
1434
+ int32_t toks[256]; char chunk[4096]; size_t nout = 0;
1435
+ int32_t dd = 0, da = 0;
1436
+ const int rc = eliza_inference_llm_stream_next(
1437
+ s, toks, 256, &nout, chunk, sizeof(chunk), &dd, &da, &outError);
1438
+ if (rc < 0) break;
1439
+ text += chunk;
1440
+ produced += static_cast<int>(nout);
1441
+ if (rc == 1) break;
1442
+ }
1443
+ const double t1 = []() {
1444
+ timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
1445
+ return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
1446
+ }();
1447
+ eliza_inference_llm_stream_close(s);
1448
+ if (tok) eliza_inference_free_tokens(tok);
1449
+ eliza_inference_destroy(ctx);
1450
+
1451
+ const double ms = t1 - t0;
1452
+ const double tokS = ms > 0 ? produced * 1000.0 / ms : 0.0;
1453
+ LOGI("LLM SELFTEST: generated %d tokens in %.0fms (%.2f tok/s) — \"%.80s\"",
1454
+ produced, ms, tokS, text.c_str());
1455
+
1456
+ // JSON-escape the generated text.
1457
+ std::string esc;
1458
+ for (char c : text) {
1459
+ switch (c) {
1460
+ case '"': esc += "\\\""; break;
1461
+ case '\\': esc += "\\\\"; break;
1462
+ case '\n': esc += "\\n"; break;
1463
+ case '\r': esc += "\\r"; break;
1464
+ case '\t': esc += "\\t"; break;
1465
+ default:
1466
+ if (static_cast<unsigned char>(c) < 0x20) {
1467
+ char b[8]; std::snprintf(b, sizeof(b), "\\u%04x",
1468
+ static_cast<unsigned char>(c));
1469
+ esc += b;
1470
+ } else esc += c;
1471
+ }
1472
+ }
1473
+ std::string json = "{\"ok\":true,\"tokens\":" + std::to_string(produced) +
1474
+ ",\"ms\":" + std::to_string(ms) + ",\"tokS\":" +
1475
+ std::to_string(tokS) + ",\"text\":\"" + esc + "\"}";
1476
+ return to_jstring(env, json);
1477
+ }
1478
+
1479
+ // ── Kokoro TTS (ABI v10) ─────────────────────────────────────────────────
1480
+ // Synthesize speech in-process via the fused Kokoro-82M head. This is what lets
1481
+ // the Android app speak with the real on-device voice instead of falling back to
1482
+ // the platform TextToSpeech: TalkMode (this bionic process) → bionic host "tts"
1483
+ // op → here. Returns a float[] of 24 kHz PCM (the model's native rate).
1484
+
1485
+ JNIEXPORT jint JNICALL
1486
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeKokoroSampleRate(JNIEnv*, jclass,
1487
+ jlong ctxHandle) {
1488
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1489
+ return ctx ? eliza_inference_kokoro_sample_rate(ctx) : -1;
1490
+ }
1491
+
1492
+ JNIEXPORT jfloatArray JNICALL
1493
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeKokoroSynthesize(JNIEnv* env, jclass,
1494
+ jlong ctxHandle,
1495
+ jstring jGguf,
1496
+ jstring jVoiceBin,
1497
+ jstring jText,
1498
+ jfloat speed) {
1499
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1500
+ if (!ctx) {
1501
+ throw_runtime(env, "kokoroSynthesize: null context", nullptr);
1502
+ return nullptr;
1503
+ }
1504
+ const std::string gguf = from_jstring(env, jGguf);
1505
+ const std::string voiceBin = from_jstring(env, jVoiceBin);
1506
+ const std::string text = from_jstring(env, jText);
1507
+ char* err = nullptr;
1508
+ // style_dim 256 for Kokoro v1.0. Reloads only when the model/voice changed
1509
+ // (the FFI caches the resident model + voice preset).
1510
+ if (eliza_inference_kokoro_load(ctx, gguf.c_str(), voiceBin.c_str(), 256, &err) != 0) {
1511
+ throw_runtime(env, "kokoro_load failed", err);
1512
+ return nullptr;
1513
+ }
1514
+ // Cap at 30 s @ 24 kHz — far longer than any single reply phrase.
1515
+ const size_t cap = 24000u * 30u;
1516
+ std::vector<float> pcm(cap);
1517
+ err = nullptr;
1518
+ int n = eliza_inference_kokoro_synthesize(
1519
+ ctx, text.c_str(), text.size(), speed, pcm.data(), cap, &err);
1520
+ if (n < 0) {
1521
+ throw_runtime(env, "kokoro_synthesize failed", err);
1522
+ return nullptr;
1523
+ }
1524
+ jfloatArray out = env->NewFloatArray(n);
1525
+ if (!out) return nullptr;
1526
+ env->SetFloatArrayRegion(out, 0, n, pcm.data());
1527
+ LOGI("nativeKokoroSynthesize: %zu chars -> %d samples", text.size(), n);
1528
+ return out;
1529
+ }
1530
+
1531
+ // ── Batch ASR (synchronous, VAD-free) ────────────────────────────────────
1532
+ //
1533
+ // The streaming pipeline (nativePipelineProcess) is VAD-gated and needs the
1534
+ // VAD/diariz/speaker GGUFs staged; this is the DIRECT audio-in/text-out
1535
+ // transcribe the fused lib exposes (eliza_inference_asr_transcribe), which
1536
+ // only mmap-acquires the `asr/` weights on the resident context. The bionic
1537
+ // host's op="asr" calls this so the agent's TRANSCRIPTION delegate gets a
1538
+ // real on-device transcript without the full attribution pipeline.
1539
+ JNIEXPORT jstring JNICALL
1540
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeAsrTranscribe(JNIEnv* env, jclass,
1541
+ jlong ctxHandle,
1542
+ jfloatArray jPcm,
1543
+ jint sampleRate) {
1544
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1545
+ if (!ctx) {
1546
+ throw_runtime(env, "asrTranscribe: null context", nullptr);
1547
+ return nullptr;
1548
+ }
1549
+ const jsize n = env->GetArrayLength(jPcm);
1550
+ jfloat* pcm = env->GetFloatArrayElements(jPcm, nullptr);
1551
+ if (!pcm) {
1552
+ throw_runtime(env, "asrTranscribe: null PCM", nullptr);
1553
+ return nullptr;
1554
+ }
1555
+ // The ASR weights are a voice-only mmap region that must be armed before
1556
+ // transcribe (VoiceLifecycle normally does this on voice-on). The agent's
1557
+ // one-shot transcribe path has no lifecycle, so arm it here (idempotent if
1558
+ // already acquired).
1559
+ char* acqErr = nullptr;
1560
+ if (eliza_inference_mmap_acquire(ctx, "asr", &acqErr) != 0) {
1561
+ env->ReleaseFloatArrayElements(jPcm, pcm, JNI_ABORT);
1562
+ throw_runtime(env, "asr mmap_acquire", acqErr);
1563
+ return nullptr;
1564
+ }
1565
+ // 64 KiB transcript cap — far longer than any single utterance.
1566
+ std::vector<char> out(65536, 0);
1567
+ char* err = nullptr;
1568
+ const int rc = eliza_inference_asr_transcribe(
1569
+ ctx, reinterpret_cast<const float*>(pcm), static_cast<size_t>(n),
1570
+ sampleRate > 0 ? sampleRate : kSampleRate, out.data(), out.size(),
1571
+ &err);
1572
+ env->ReleaseFloatArrayElements(jPcm, pcm, JNI_ABORT);
1573
+ if (rc < 0) {
1574
+ throw_runtime(env, "asr_transcribe", err);
1575
+ return nullptr;
1576
+ }
1577
+ LOGI("nativeAsrTranscribe: %d samples @ %d Hz -> %d transcript bytes",
1578
+ (int)n, (int)sampleRate, rc);
1579
+ return to_jstring(env, std::string(out.data()));
1580
+ }
1581
+
1582
+ // ── mmproj vision (ABI v9) ───────────────────────────────────────────────
1583
+ JNIEXPORT jint JNICALL
1584
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeVisionSupported(JNIEnv*, jclass) {
1585
+ return static_cast<jint>(eliza_inference_vision_supported());
1586
+ }
1587
+
1588
+ // Describe a raw PNG/JPEG/WebP image with the resident TEXT model + an mmproj
1589
+ // projector (eliza_inference_describe_image). The bionic host's op="image"
1590
+ // calls this so the agent's IMAGE_DESCRIPTION delegate runs screen/vision
1591
+ // recognition fully on-device.
1592
+ JNIEXPORT jstring JNICALL
1593
+ Java_ai_elizaos_app_ElizaVoiceNative_nativeDescribeImage(JNIEnv* env, jclass,
1594
+ jlong ctxHandle,
1595
+ jbyteArray jImage,
1596
+ jstring jMmproj,
1597
+ jstring jPrompt) {
1598
+ auto* ctx = reinterpret_cast<EliInferenceContext*>(ctxHandle);
1599
+ if (!ctx) {
1600
+ throw_runtime(env, "describeImage: null context", nullptr);
1601
+ return nullptr;
1602
+ }
1603
+ const std::string mmproj = from_jstring(env, jMmproj);
1604
+ const std::string prompt = from_jstring(env, jPrompt);
1605
+ const jsize n = env->GetArrayLength(jImage);
1606
+ jbyte* img = env->GetByteArrayElements(jImage, nullptr);
1607
+ if (!img) {
1608
+ throw_runtime(env, "describeImage: null image bytes", nullptr);
1609
+ return nullptr;
1610
+ }
1611
+ std::vector<char> out(16384, 0);
1612
+ char* err = nullptr;
1613
+ const int rc = eliza_inference_describe_image(
1614
+ ctx, reinterpret_cast<const unsigned char*>(img),
1615
+ static_cast<size_t>(n),
1616
+ mmproj.empty() ? nullptr : mmproj.c_str(),
1617
+ prompt.empty() ? nullptr : prompt.c_str(), out.data(), out.size(),
1618
+ &err);
1619
+ env->ReleaseByteArrayElements(jImage, img, JNI_ABORT);
1620
+ if (rc < 0) {
1621
+ throw_runtime(env, "describe_image", err);
1622
+ return nullptr;
1623
+ }
1624
+ LOGI("nativeDescribeImage: %d image bytes -> %d description bytes", (int)n,
1625
+ rc);
1626
+ return to_jstring(env, std::string(out.data()));
1627
+ }
1628
+
1629
+ } // extern "C"