@vellumai/assistant 0.6.2 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (895) hide show
  1. package/ARCHITECTURE.md +273 -10
  2. package/Dockerfile +2 -3
  3. package/bun.lock +41 -49
  4. package/bunfig.toml +3 -0
  5. package/docs/architecture/memory.md +1 -1
  6. package/docs/backup-troubleshooting.md +52 -0
  7. package/docs/browser-use-architecture-phase2.md +174 -0
  8. package/docs/stt-provider-onboarding.md +120 -0
  9. package/knip.json +12 -2
  10. package/node_modules/@vellumai/ces-contracts/bun.lock +8 -6
  11. package/node_modules/@vellumai/ces-contracts/package.json +3 -3
  12. package/node_modules/@vellumai/ces-contracts/src/rpc.ts +42 -0
  13. package/openapi.yaml +1111 -86
  14. package/package.json +40 -42
  15. package/scripts/generate-openapi.ts +0 -2
  16. package/scripts/test.sh +73 -18
  17. package/src/__tests__/acp-session.test.ts +43 -0
  18. package/src/__tests__/agent-image-optimize.test.ts +28 -0
  19. package/src/__tests__/agent-loop.test.ts +123 -0
  20. package/src/__tests__/anthropic-provider.test.ts +263 -10
  21. package/src/__tests__/app-builder-tool-scripts.test.ts +1 -0
  22. package/src/__tests__/app-executors.test.ts +1 -0
  23. package/src/__tests__/app-source-watcher.test.ts +37 -11
  24. package/src/__tests__/approval-routes-http.test.ts +178 -1
  25. package/src/__tests__/auto-analysis-end-to-end.test.ts +550 -0
  26. package/src/__tests__/auto-analysis-prompt.test.ts +50 -0
  27. package/src/__tests__/browser-fill-credential.test.ts +240 -94
  28. package/src/__tests__/browser-manager.test.ts +40 -27
  29. package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
  30. package/src/__tests__/browser-skill-endstate.test.ts +31 -7
  31. package/src/__tests__/btw-routes.test.ts +7 -0
  32. package/src/__tests__/call-controller.test.ts +581 -20
  33. package/src/__tests__/catalog-files.test.ts +1000 -0
  34. package/src/__tests__/channel-approvals.test.ts +53 -0
  35. package/src/__tests__/channel-invite-transport.test.ts +2 -2
  36. package/src/__tests__/channel-readiness-routes.test.ts +16 -20
  37. package/src/__tests__/channel-readiness-service.test.ts +12 -7
  38. package/src/__tests__/checker.test.ts +157 -10
  39. package/src/__tests__/clawhub-files.test.ts +347 -0
  40. package/src/__tests__/commit-message-enrichment-service.test.ts +36 -19
  41. package/src/__tests__/config-analysis.test.ts +100 -0
  42. package/src/__tests__/config-managed-gemini-defaults.test.ts +326 -0
  43. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  44. package/src/__tests__/config-schema.test.ts +1248 -224
  45. package/src/__tests__/config-watcher-cleanup-throttle.test.ts +339 -0
  46. package/src/__tests__/config-watcher.test.ts +43 -8
  47. package/src/__tests__/confirmation-request-guardian-bridge.test.ts +23 -0
  48. package/src/__tests__/contact-store-user-file.test.ts +512 -0
  49. package/src/__tests__/contacts-write.test.ts +197 -0
  50. package/src/__tests__/context-overflow-approval.test.ts +16 -1
  51. package/src/__tests__/context-window-manager.test.ts +88 -0
  52. package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
  53. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -1
  54. package/src/__tests__/conversation-agent-loop.test.ts +99 -3
  55. package/src/__tests__/conversation-analysis-routes.test.ts +2 -2
  56. package/src/__tests__/conversation-attachments.test.ts +80 -4
  57. package/src/__tests__/conversation-confirmation-signals.test.ts +290 -0
  58. package/src/__tests__/conversation-error.test.ts +70 -0
  59. package/src/__tests__/conversation-fork-crud.test.ts +17 -0
  60. package/src/__tests__/conversation-history-web-search.test.ts +12 -4
  61. package/src/__tests__/conversation-host-access-routes.test.ts +229 -0
  62. package/src/__tests__/conversation-init.benchmark.test.ts +6 -1
  63. package/src/__tests__/conversation-inject-context.test.ts +103 -0
  64. package/src/__tests__/conversation-launcher-skill-regression.test.ts +51 -0
  65. package/src/__tests__/conversation-list-source.test.ts +145 -0
  66. package/src/__tests__/conversation-pre-run-repair.test.ts +2 -0
  67. package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
  68. package/src/__tests__/conversation-queue.test.ts +946 -62
  69. package/src/__tests__/conversation-routes-disk-view.test.ts +275 -0
  70. package/src/__tests__/conversation-routes-guardian-reply.test.ts +16 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +324 -46
  73. package/src/__tests__/conversation-skill-tools.test.ts +7 -4
  74. package/src/__tests__/conversation-slash-commands.test.ts +33 -0
  75. package/src/__tests__/conversation-slash-queue.test.ts +89 -18
  76. package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
  77. package/src/__tests__/conversation-starter-routes.test.ts +126 -0
  78. package/src/__tests__/conversation-starters-cadence.test.ts +161 -0
  79. package/src/__tests__/conversation-store.test.ts +195 -0
  80. package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +226 -0
  81. package/src/__tests__/conversation-workspace-cache-state.test.ts +193 -0
  82. package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
  83. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
  84. package/src/__tests__/credential-execution-approval-bridge.test.ts +32 -1
  85. package/src/__tests__/credential-health-service.test.ts +352 -0
  86. package/src/__tests__/credential-security-invariants.test.ts +6 -3
  87. package/src/__tests__/credential-vault-unit.test.ts +383 -7
  88. package/src/__tests__/credential-vault.test.ts +152 -13
  89. package/src/__tests__/credentials-cli.test.ts +42 -18
  90. package/src/__tests__/cross-provider-web-search.test.ts +146 -35
  91. package/src/__tests__/date-context.test.ts +4 -4
  92. package/src/__tests__/deterministic-verification-control-plane.test.ts +10 -1
  93. package/src/__tests__/device-id.test.ts +112 -0
  94. package/src/__tests__/docker-signing-key-bootstrap.test.ts +167 -4
  95. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +1 -3
  96. package/src/__tests__/email-html-renderer.test.ts +71 -0
  97. package/src/__tests__/email-invite-adapter.test.ts +36 -32
  98. package/src/__tests__/embedding-managed-proxy-selection.test.ts +256 -0
  99. package/src/__tests__/emit-event-signal.test.ts +71 -0
  100. package/src/__tests__/extension-id-sync-guard.test.ts +222 -0
  101. package/src/__tests__/fixtures/mock-chrome-extension.ts +386 -0
  102. package/src/__tests__/gateway-only-enforcement.test.ts +206 -1
  103. package/src/__tests__/gateway-only-guard.test.ts +2 -0
  104. package/src/__tests__/gemini-provider.test.ts +66 -2
  105. package/src/__tests__/get-skill-detail-audit.test.ts +325 -0
  106. package/src/__tests__/gmail-archive-fallback.test.ts +193 -0
  107. package/src/__tests__/gmail-archive-gate.test.ts +246 -0
  108. package/src/__tests__/gmail-preferences.test.ts +117 -0
  109. package/src/__tests__/guardian-routing-invariants.test.ts +70 -2
  110. package/src/__tests__/headless-browser-interactions.test.ts +738 -359
  111. package/src/__tests__/headless-browser-mode.test.ts +614 -0
  112. package/src/__tests__/headless-browser-navigate.test.ts +528 -49
  113. package/src/__tests__/headless-browser-read-tools.test.ts +274 -100
  114. package/src/__tests__/headless-browser-snapshot.test.ts +250 -77
  115. package/src/__tests__/heartbeat-service.test.ts +70 -17
  116. package/src/__tests__/home-state-routes.test.ts +162 -0
  117. package/src/__tests__/host-bash-proxy.test.ts +145 -1
  118. package/src/__tests__/host-browser-e2e-cloud.test.ts +596 -0
  119. package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +286 -0
  120. package/src/__tests__/host-browser-e2e-self-hosted.test.ts +374 -0
  121. package/src/__tests__/host-browser-event-routes.test.ts +350 -0
  122. package/src/__tests__/host-browser-proxy.test.ts +444 -0
  123. package/src/__tests__/host-browser-routes.test.ts +198 -0
  124. package/src/__tests__/host-browser-ws-events-e2e.test.ts +423 -0
  125. package/src/__tests__/host-cu-proxy.test.ts +166 -1
  126. package/src/__tests__/host-file-proxy.test.ts +185 -1
  127. package/src/__tests__/host-file-read-tool.test.ts +52 -0
  128. package/src/__tests__/host-proxy-interface.test.ts +165 -0
  129. package/src/__tests__/host-shell-tool.test.ts +1 -11
  130. package/src/__tests__/http-user-message-parity.test.ts +1 -0
  131. package/src/__tests__/identity-intro-cache.test.ts +40 -10
  132. package/src/__tests__/init-feature-flag-overrides.test.ts +38 -112
  133. package/src/__tests__/integration-status.test.ts +6 -7
  134. package/src/__tests__/jobs-store-upsert-debounced.test.ts +141 -0
  135. package/src/__tests__/list-messages-tool-merge.test.ts +37 -12
  136. package/src/__tests__/llm-context-normalization.test.ts +488 -0
  137. package/src/__tests__/llm-context-route-provider.test.ts +86 -5
  138. package/src/__tests__/llm-usage-store.test.ts +363 -0
  139. package/src/__tests__/mcp-client-auth.test.ts +40 -4
  140. package/src/__tests__/mcp-health-check.test.ts +10 -3
  141. package/src/__tests__/media-stream-output.test.ts +555 -0
  142. package/src/__tests__/media-stream-parser.test.ts +374 -0
  143. package/src/__tests__/media-stream-server-integration.test.ts +1234 -0
  144. package/src/__tests__/media-stream-stt-session.test.ts +588 -0
  145. package/src/__tests__/media-turn-detector.test.ts +440 -0
  146. package/src/__tests__/message-queue.test.ts +125 -0
  147. package/src/__tests__/migration-cross-version-compatibility.test.ts +3 -1
  148. package/src/__tests__/migration-export-http.test.ts +67 -8
  149. package/src/__tests__/migration-export-streaming.test.ts +66 -0
  150. package/src/__tests__/migration-import-commit-http.test.ts +109 -7
  151. package/src/__tests__/migration-import-preflight-http.test.ts +6 -5
  152. package/src/__tests__/migration-validate-http.test.ts +3 -3
  153. package/src/__tests__/mock-gateway-ipc.ts +151 -0
  154. package/src/__tests__/model-intents.test.ts +2 -2
  155. package/src/__tests__/native-host-marker-sync-guard.test.ts +157 -0
  156. package/src/__tests__/oauth-apps-routes.test.ts +18 -12
  157. package/src/__tests__/oauth-cli.test.ts +709 -60
  158. package/src/__tests__/oauth-connect-orchestrator.test.ts +118 -24
  159. package/src/__tests__/oauth-provider-seed-logos.test.ts +23 -0
  160. package/src/__tests__/oauth-provider-serializer.test.ts +147 -10
  161. package/src/__tests__/oauth-provider-visibility.test.ts +19 -21
  162. package/src/__tests__/oauth-providers-routes.test.ts +52 -14
  163. package/src/__tests__/oauth-store.test.ts +1465 -176
  164. package/src/__tests__/oauth2-gateway-transport.test.ts +460 -26
  165. package/src/__tests__/onboarding-template-contract.test.ts +81 -70
  166. package/src/__tests__/openai-provider.test.ts +178 -2
  167. package/src/__tests__/openai-responses-cutover-guard.test.ts +184 -0
  168. package/src/__tests__/openai-responses-provider.test.ts +1105 -0
  169. package/src/__tests__/openrouter-token-estimation.test.ts +100 -0
  170. package/src/__tests__/outlook-categories.test.ts +1 -1
  171. package/src/__tests__/outlook-client-automation.test.ts +1 -1
  172. package/src/__tests__/outlook-compose-tools.test.ts +1 -1
  173. package/src/__tests__/outlook-email-watcher.test.ts +1 -1
  174. package/src/__tests__/outlook-follow-up.test.ts +1 -1
  175. package/src/__tests__/outlook-messaging-provider.test.ts +2 -2
  176. package/src/__tests__/outlook-trash.test.ts +1 -1
  177. package/src/__tests__/outlook-unsubscribe.test.ts +32 -3
  178. package/src/__tests__/permission-checker-host-gate.test.ts +74 -14
  179. package/src/__tests__/permission-mode.test.ts +28 -56
  180. package/src/__tests__/persona-resolver.test.ts +251 -0
  181. package/src/__tests__/platform-bash-auto-approve.test.ts +4 -0
  182. package/src/__tests__/platform-callback-registration.test.ts +19 -0
  183. package/src/__tests__/platform.test.ts +92 -1
  184. package/src/__tests__/post-turn-tool-result-truncation.test.ts +343 -0
  185. package/src/__tests__/prechat-onboarding-contract.test.ts +267 -0
  186. package/src/__tests__/pricing.test.ts +174 -0
  187. package/src/__tests__/proxy-approval-callback.test.ts +18 -0
  188. package/src/__tests__/qdrant-manager.test.ts +29 -8
  189. package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +194 -0
  190. package/src/__tests__/relationship-state-contract.test.ts +175 -0
  191. package/src/__tests__/relay-server.test.ts +423 -5
  192. package/src/__tests__/require-fresh-approval.test.ts +40 -1
  193. package/src/__tests__/sanitize-config-for-transfer.test.ts +132 -0
  194. package/src/__tests__/schedule-routes.test.ts +162 -0
  195. package/src/__tests__/search-skills-unified.test.ts +118 -0
  196. package/src/__tests__/secret-detection-handler.test.ts +84 -0
  197. package/src/__tests__/secret-ingress-http.test.ts +1 -0
  198. package/src/__tests__/secret-scanner-executor.test.ts +4 -0
  199. package/src/__tests__/secure-keys.test.ts +107 -0
  200. package/src/__tests__/send-endpoint-busy.test.ts +8 -1
  201. package/src/__tests__/sequence-store.test.ts +1 -1
  202. package/src/__tests__/server-history-render.test.ts +49 -0
  203. package/src/__tests__/set-permission-mode.test.ts +13 -250
  204. package/src/__tests__/settings-routes.test.ts +201 -0
  205. package/src/__tests__/skill-load-feature-flag.test.ts +1 -0
  206. package/src/__tests__/skills-file-content-endpoint.test.ts +801 -0
  207. package/src/__tests__/skills-files-catalog-fallback.test.ts +738 -0
  208. package/src/__tests__/skills.test.ts +5 -2
  209. package/src/__tests__/skillssh-files.test.ts +446 -0
  210. package/src/__tests__/slack-block-formatting.test.ts +110 -0
  211. package/src/__tests__/slack-channel-config.test.ts +576 -16
  212. package/src/__tests__/stt-catalog-parity.test.ts +282 -0
  213. package/src/__tests__/stt-stream-session.test.ts +535 -0
  214. package/src/__tests__/subagent-detail.test.ts +44 -2
  215. package/src/__tests__/subagent-disposal.test.ts +1 -0
  216. package/src/__tests__/subagent-fork-notifications.test.ts +291 -0
  217. package/src/__tests__/subagent-fork-spawn.test.ts +384 -0
  218. package/src/__tests__/subagent-manager-notify.test.ts +1 -0
  219. package/src/__tests__/subagent-notify-parent.test.ts +1 -0
  220. package/src/__tests__/subagent-spawn-tool-fork.test.ts +411 -0
  221. package/src/__tests__/subagent-tools.test.ts +1 -0
  222. package/src/__tests__/subagent-types.test.ts +1 -0
  223. package/src/__tests__/system-prompt-ask-mode.test.ts +27 -71
  224. package/src/__tests__/system-prompt.test.ts +184 -27
  225. package/src/__tests__/task-scheduler.test.ts +32 -6
  226. package/src/__tests__/telegram-config.test.ts +10 -13
  227. package/src/__tests__/telephony-stt-routing.test.ts +329 -0
  228. package/src/__tests__/terminal-tools.test.ts +25 -5
  229. package/src/__tests__/test-preload.ts +18 -0
  230. package/src/__tests__/test-support/browser-skill-harness.ts +4 -1
  231. package/src/__tests__/tool-approval-handler.test.ts +73 -0
  232. package/src/__tests__/tool-executor-lifecycle-events.test.ts +9 -5
  233. package/src/__tests__/tool-executor-shell-integration.test.ts +4 -0
  234. package/src/__tests__/tool-executor.test.ts +33 -24
  235. package/src/__tests__/tool-result-truncation.test.ts +36 -0
  236. package/src/__tests__/tool-side-effects-slack-dm.test.ts +22 -0
  237. package/src/__tests__/top-level-renderer.test.ts +73 -1
  238. package/src/__tests__/transport-hints-queue.test.ts +14 -29
  239. package/src/__tests__/trust-store.test.ts +7 -1
  240. package/src/__tests__/trusted-contact-approval-notifier.test.ts +1 -1
  241. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +109 -0
  242. package/src/__tests__/tts-catalog-parity.test.ts +345 -0
  243. package/src/__tests__/twilio-routes-twiml.test.ts +512 -114
  244. package/src/__tests__/twilio-routes.test.ts +376 -0
  245. package/src/__tests__/unicode.test.ts +293 -0
  246. package/src/__tests__/update-bulletin-format.test.ts +59 -0
  247. package/src/__tests__/update-bulletin.test.ts +206 -5
  248. package/src/__tests__/usage-routes.test.ts +25 -4
  249. package/src/__tests__/user-reference.test.ts +46 -61
  250. package/src/__tests__/v2-consent-policy.test.ts +103 -0
  251. package/src/__tests__/verification-control-plane-policy.test.ts +4 -0
  252. package/src/__tests__/voice-config-update.test.ts +403 -0
  253. package/src/__tests__/voice-quality.test.ts +434 -19
  254. package/src/__tests__/workspace-heartbeat-service.test.ts +7 -0
  255. package/src/__tests__/workspace-migration-033-stt-service-explicit-config.test.ts +547 -0
  256. package/src/__tests__/workspace-migration-034-remove-calls-voice-transcription-provider.test.ts +596 -0
  257. package/src/__tests__/workspace-migration-drop-user-md.test.ts +368 -0
  258. package/src/__tests__/workspace-migration-meets.test.ts +244 -0
  259. package/src/__tests__/workspace-migration-seed-device-id.test.ts +14 -20
  260. package/src/__tests__/workspace-policy.test.ts +2 -0
  261. package/src/acp/client-handler.ts +30 -4
  262. package/src/agent/image-optimize.ts +24 -12
  263. package/src/agent/loop.ts +55 -9
  264. package/src/approvals/guardian-request-resolvers.ts +21 -15
  265. package/src/backup/__tests__/backup-key.test.ts +152 -0
  266. package/src/backup/__tests__/backup-worker.test.ts +767 -0
  267. package/src/backup/__tests__/list-snapshots.test.ts +87 -0
  268. package/src/backup/__tests__/local-writer.test.ts +218 -0
  269. package/src/backup/__tests__/offsite-writer.test.ts +641 -0
  270. package/src/backup/__tests__/paths.test.ts +300 -0
  271. package/src/backup/__tests__/restore.test.ts +498 -0
  272. package/src/backup/__tests__/snapshot-lock.test.ts +352 -0
  273. package/src/backup/__tests__/stream-crypt.test.ts +228 -0
  274. package/src/backup/backup-key.ts +137 -0
  275. package/src/backup/backup-worker.ts +459 -0
  276. package/src/backup/list-snapshots.ts +147 -0
  277. package/src/backup/local-writer.ts +133 -0
  278. package/src/backup/offsite-writer.ts +222 -0
  279. package/src/backup/paths.ts +226 -0
  280. package/src/backup/restore.ts +322 -0
  281. package/src/backup/snapshot-lock.ts +431 -0
  282. package/src/backup/stream-crypt.ts +263 -0
  283. package/src/browser-session/__tests__/manager.test.ts +297 -0
  284. package/src/browser-session/backends/cdp-inspect.ts +30 -0
  285. package/src/browser-session/backends/extension.ts +26 -0
  286. package/src/browser-session/backends/local.ts +24 -0
  287. package/src/browser-session/events.ts +164 -0
  288. package/src/browser-session/index.ts +27 -0
  289. package/src/browser-session/manager.ts +159 -0
  290. package/src/browser-session/types.ts +28 -0
  291. package/src/bundler/package-resolver.ts +4 -0
  292. package/src/calls/audio-store.ts +11 -5
  293. package/src/calls/call-controller.ts +226 -71
  294. package/src/calls/call-domain.ts +9 -0
  295. package/src/calls/call-speech-output.ts +190 -0
  296. package/src/calls/call-transport.ts +77 -0
  297. package/src/calls/media-stream-audio-transcode.ts +173 -0
  298. package/src/calls/media-stream-output.ts +660 -0
  299. package/src/calls/media-stream-parser.ts +300 -0
  300. package/src/calls/media-stream-protocol.ts +166 -0
  301. package/src/calls/media-stream-server.ts +592 -0
  302. package/src/calls/media-stream-stt-session.ts +460 -0
  303. package/src/calls/media-turn-detector.ts +230 -0
  304. package/src/calls/relay-server.ts +90 -75
  305. package/src/calls/resolve-call-tts-provider.ts +136 -0
  306. package/src/calls/telephony-stt-routing.ts +145 -0
  307. package/src/calls/tts-call-strategy.ts +161 -0
  308. package/src/calls/tts-text-sanitizer.ts +32 -16
  309. package/src/calls/twilio-routes.ts +281 -17
  310. package/src/calls/voice-quality.ts +78 -35
  311. package/src/calls/voice-session-bridge.ts +8 -1
  312. package/src/channels/__tests__/types.test.ts +134 -0
  313. package/src/channels/types.ts +69 -3
  314. package/src/cli/__tests__/run-assistant-command.ts +11 -1
  315. package/src/cli/commands/__tests__/backup.test.ts +1165 -0
  316. package/src/cli/commands/__tests__/domain-register.test.ts +234 -0
  317. package/src/cli/commands/__tests__/domain-status.test.ts +132 -0
  318. package/src/cli/commands/__tests__/email-attachment.test.ts +422 -0
  319. package/src/cli/commands/__tests__/email-download.test.ts +16 -1
  320. package/src/cli/commands/__tests__/email-list.test.ts +22 -4
  321. package/src/cli/commands/__tests__/email-register.test.ts +4 -4
  322. package/src/cli/commands/__tests__/email-send.test.ts +37 -4
  323. package/src/cli/commands/__tests__/email-status.test.ts +5 -1
  324. package/src/cli/commands/__tests__/email-unregister.test.ts +34 -5
  325. package/src/cli/commands/backup.ts +993 -0
  326. package/src/cli/commands/conversations.ts +77 -0
  327. package/src/cli/commands/credentials.ts +3 -4
  328. package/src/cli/commands/domain.ts +210 -0
  329. package/src/cli/commands/email.ts +273 -16
  330. package/src/cli/commands/mcp.ts +16 -4
  331. package/src/cli/commands/oauth/__tests__/connect.test.ts +56 -44
  332. package/src/cli/commands/oauth/__tests__/disconnect.test.ts +21 -21
  333. package/src/cli/commands/oauth/__tests__/mode.test.ts +17 -17
  334. package/src/cli/commands/oauth/__tests__/ping.test.ts +16 -16
  335. package/src/cli/commands/oauth/__tests__/providers-delete.test.ts +32 -33
  336. package/src/cli/commands/oauth/__tests__/providers-register.test.ts +330 -0
  337. package/src/cli/commands/oauth/__tests__/providers-update.test.ts +117 -12
  338. package/src/cli/commands/oauth/__tests__/status.test.ts +10 -10
  339. package/src/cli/commands/oauth/__tests__/token.test.ts +7 -7
  340. package/src/cli/commands/oauth/apps.ts +7 -4
  341. package/src/cli/commands/oauth/connect.ts +6 -3
  342. package/src/cli/commands/oauth/disconnect.ts +1 -1
  343. package/src/cli/commands/oauth/mode.ts +12 -3
  344. package/src/cli/commands/oauth/providers.ts +215 -36
  345. package/src/cli/commands/oauth/shared.ts +7 -6
  346. package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +254 -0
  347. package/src/cli/commands/platform/__tests__/connect.test.ts +6 -0
  348. package/src/cli/commands/platform/__tests__/disconnect.test.ts +7 -1
  349. package/src/cli/commands/platform/__tests__/status.test.ts +6 -0
  350. package/src/cli/commands/platform/index.ts +107 -10
  351. package/src/cli/commands/usage.ts +10 -9
  352. package/src/cli/lib/daemon-credential-client.ts +4 -0
  353. package/src/cli/program.ts +30 -4
  354. package/src/config/__tests__/backup-schema.test.ts +134 -0
  355. package/src/config/assistant-feature-flags.ts +61 -62
  356. package/src/config/bundled-skills/app-builder/SKILL.md +26 -249
  357. package/src/config/bundled-skills/app-builder/references/CUSTOM_ROUTES.md +141 -0
  358. package/src/config/bundled-skills/app-builder/references/INTERACTION_HOOKS.md +56 -0
  359. package/src/config/bundled-skills/app-builder/references/WIDGETS.md +125 -0
  360. package/src/config/bundled-skills/browser/SKILL.md +30 -5
  361. package/src/config/bundled-skills/browser/TOOLS.json +123 -0
  362. package/src/config/bundled-skills/browser/tools/browser-attach.ts +12 -0
  363. package/src/config/bundled-skills/browser/tools/browser-detach.ts +12 -0
  364. package/src/config/bundled-skills/browser/tools/browser-status.ts +12 -0
  365. package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +17 -0
  366. package/src/config/bundled-skills/contacts/SKILL.md +5 -2
  367. package/src/config/bundled-skills/document/SKILL.md +4 -0
  368. package/src/config/bundled-skills/gmail/SKILL.md +54 -8
  369. package/src/config/bundled-skills/gmail/TOOLS.json +33 -3
  370. package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +116 -9
  371. package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +138 -11
  372. package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +59 -0
  373. package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +82 -0
  374. package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +113 -17
  375. package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -2
  376. package/src/config/bundled-skills/media-processing/SKILL.md +3 -9
  377. package/src/config/bundled-skills/media-processing/TOOLS.json +1 -6
  378. package/src/config/bundled-skills/media-processing/__tests__/audio-transcribe.test.ts +125 -0
  379. package/src/config/bundled-skills/media-processing/__tests__/extract-keyframes.test.ts +181 -0
  380. package/src/config/bundled-skills/media-processing/__tests__/preprocess-audio.test.ts +141 -0
  381. package/src/config/bundled-skills/media-processing/services/audio-transcribe.ts +32 -87
  382. package/src/config/bundled-skills/media-processing/services/preprocess.ts +8 -4
  383. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +0 -10
  384. package/src/config/bundled-skills/messaging/SKILL.md +3 -3
  385. package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +2 -2
  386. package/src/config/bundled-skills/outlook/SKILL.md +9 -2
  387. package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +2 -2
  388. package/src/config/bundled-skills/phone-calls/SKILL.md +2 -2
  389. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +27 -18
  390. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +3 -3
  391. package/src/config/bundled-skills/settings/TOOLS.json +3 -3
  392. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +26 -22
  393. package/src/config/bundled-skills/slack/SKILL.md +1 -0
  394. package/src/config/bundled-skills/subagent/SKILL.md +21 -0
  395. package/src/config/bundled-skills/subagent/TOOLS.json +8 -4
  396. package/src/config/bundled-skills/tasks/SKILL.md +5 -0
  397. package/src/config/bundled-skills/transcribe/SKILL.md +9 -14
  398. package/src/config/bundled-skills/transcribe/TOOLS.json +2 -7
  399. package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +256 -0
  400. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +38 -188
  401. package/src/config/bundled-tool-registry.ts +8 -0
  402. package/src/config/env-registry.ts +38 -0
  403. package/src/config/env.ts +49 -4
  404. package/src/config/feature-flag-registry.json +85 -14
  405. package/src/config/loader.ts +82 -13
  406. package/src/config/sanitize-for-transfer.ts +47 -0
  407. package/src/config/schema.ts +81 -15
  408. package/src/config/schemas/__tests__/stt.test.ts +43 -0
  409. package/src/config/schemas/analysis.ts +51 -0
  410. package/src/config/schemas/backup.ts +72 -0
  411. package/src/config/schemas/calls.ts +1 -26
  412. package/src/config/schemas/elevenlabs.ts +0 -59
  413. package/src/config/schemas/filing.ts +47 -7
  414. package/src/config/schemas/heartbeat.ts +27 -5
  415. package/src/config/schemas/host-browser.ts +112 -0
  416. package/src/config/schemas/inference.ts +1 -1
  417. package/src/config/schemas/memory-lifecycle.ts +14 -2
  418. package/src/config/schemas/memory-retrieval.ts +103 -0
  419. package/src/config/schemas/security.ts +0 -6
  420. package/src/config/schemas/services.ts +52 -0
  421. package/src/config/schemas/stt.ts +59 -0
  422. package/src/config/schemas/tts.ts +230 -0
  423. package/src/config/schemas/updates.ts +14 -0
  424. package/src/config/skills.ts +4 -0
  425. package/src/config/types.ts +4 -1
  426. package/src/contacts/contact-store.ts +56 -11
  427. package/src/contacts/contacts-write.ts +38 -1
  428. package/src/context/post-turn-tool-result-truncation.ts +177 -0
  429. package/src/context/tool-result-truncation.ts +2 -1
  430. package/src/context/window-manager.ts +61 -10
  431. package/src/credential-execution/approval-bridge.ts +49 -15
  432. package/src/credential-execution/executable-discovery.ts +12 -2
  433. package/src/credential-execution/process-manager.ts +33 -2
  434. package/src/credential-health/credential-health-service.ts +366 -0
  435. package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +324 -0
  436. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +497 -0
  437. package/src/daemon/__tests__/conversation-tool-setup.test.ts +195 -0
  438. package/src/daemon/__tests__/lifecycle-startup-ordering.test.ts +127 -0
  439. package/src/daemon/app-source-watcher.ts +35 -0
  440. package/src/daemon/config-watcher.ts +99 -5
  441. package/src/daemon/context-overflow-approval.ts +5 -0
  442. package/src/daemon/conversation-agent-loop-handlers.ts +23 -2
  443. package/src/daemon/conversation-agent-loop.ts +153 -42
  444. package/src/daemon/conversation-attachments.ts +40 -0
  445. package/src/daemon/conversation-error.ts +11 -0
  446. package/src/daemon/conversation-history.ts +40 -6
  447. package/src/daemon/conversation-launch.ts +220 -0
  448. package/src/daemon/conversation-lifecycle.ts +59 -9
  449. package/src/daemon/conversation-messaging.ts +37 -3
  450. package/src/daemon/conversation-notifiers.ts +5 -0
  451. package/src/daemon/conversation-process.ts +622 -13
  452. package/src/daemon/conversation-queue-manager.ts +24 -0
  453. package/src/daemon/conversation-runtime-assembly.ts +128 -36
  454. package/src/daemon/conversation-slash.ts +36 -0
  455. package/src/daemon/conversation-surfaces.ts +131 -40
  456. package/src/daemon/conversation-tool-setup.ts +99 -8
  457. package/src/daemon/conversation-usage.ts +7 -4
  458. package/src/daemon/conversation-workspace.ts +12 -0
  459. package/src/daemon/conversation.ts +292 -16
  460. package/src/daemon/date-context.ts +10 -10
  461. package/src/daemon/first-greeting.ts +3 -2
  462. package/src/daemon/handlers/config-slack-channel.ts +269 -94
  463. package/src/daemon/handlers/conversations.ts +13 -141
  464. package/src/daemon/handlers/shared.ts +80 -0
  465. package/src/daemon/handlers/skills.ts +483 -44
  466. package/src/daemon/host-bash-proxy.ts +48 -13
  467. package/src/daemon/host-browser-proxy.ts +192 -0
  468. package/src/daemon/host-cu-proxy.ts +36 -11
  469. package/src/daemon/host-file-proxy.ts +57 -9
  470. package/src/daemon/lifecycle.ts +179 -28
  471. package/src/daemon/message-protocol.ts +13 -0
  472. package/src/daemon/message-types/conversations.ts +89 -14
  473. package/src/daemon/message-types/home.ts +40 -0
  474. package/src/daemon/message-types/host-browser.ts +100 -0
  475. package/src/daemon/message-types/meet.ts +143 -0
  476. package/src/daemon/message-types/messages.ts +19 -5
  477. package/src/daemon/message-types/schedules.ts +34 -2
  478. package/src/daemon/message-types/skills.ts +26 -0
  479. package/src/daemon/message-types/subagents.ts +2 -0
  480. package/src/daemon/message-types/surfaces.ts +2 -0
  481. package/src/daemon/server.ts +439 -14
  482. package/src/daemon/shutdown-handlers.ts +32 -4
  483. package/src/daemon/shutdown-registry.ts +40 -0
  484. package/src/daemon/tool-side-effects.ts +15 -0
  485. package/src/daemon/transport-hints.ts +5 -24
  486. package/src/email/html-renderer.ts +76 -0
  487. package/src/heartbeat/heartbeat-service.ts +93 -7
  488. package/src/home/__tests__/assistant-feed-authoring.test.ts +156 -0
  489. package/src/home/__tests__/emit-feed-event.test.ts +169 -0
  490. package/src/home/__tests__/feed-scheduler.test.ts +194 -0
  491. package/src/home/__tests__/feed-types.test.ts +275 -0
  492. package/src/home/__tests__/feed-writer.test.ts +688 -0
  493. package/src/home/__tests__/phase5-exit-criteria.test.ts +212 -0
  494. package/src/home/__tests__/platform-gmail-digest.test.ts +222 -0
  495. package/src/home/__tests__/progress-formula.test.ts +213 -0
  496. package/src/home/__tests__/relationship-state-writer.test.ts +740 -0
  497. package/src/home/__tests__/rollup-producer.test.ts +398 -0
  498. package/src/home/assistant-feed-authoring.ts +124 -0
  499. package/src/home/emit-feed-event.ts +158 -0
  500. package/src/home/feed-scheduler.ts +247 -0
  501. package/src/home/feed-types.ts +181 -0
  502. package/src/home/feed-writer.ts +469 -0
  503. package/src/home/platform-gmail-digest.ts +163 -0
  504. package/src/home/progress-formula.ts +86 -0
  505. package/src/home/relationship-state-writer.ts +824 -0
  506. package/src/home/relationship-state.ts +143 -0
  507. package/src/home/rollup-producer.ts +384 -0
  508. package/src/hooks/runner.ts +7 -0
  509. package/src/inbound/platform-callback-registration.ts +30 -20
  510. package/src/inbound/public-ingress-urls.ts +12 -0
  511. package/src/instrument.ts +1 -1
  512. package/src/ipc/__tests__/cli-ipc.test.ts +200 -0
  513. package/src/ipc/cli-client.ts +151 -0
  514. package/src/ipc/cli-server.ts +234 -0
  515. package/src/ipc/gateway-client.ts +180 -0
  516. package/src/ipc/routes/index.ts +5 -0
  517. package/src/ipc/routes/wake-conversation.ts +19 -0
  518. package/src/mcp/client.ts +59 -24
  519. package/src/memory/__tests__/auto-analysis-enqueue.test.ts +356 -0
  520. package/src/memory/__tests__/auto-analysis-guard.test.ts +57 -0
  521. package/src/memory/__tests__/conversation-analyze-job.test.ts +232 -0
  522. package/src/memory/__tests__/find-analysis-conversation.test.ts +196 -0
  523. package/src/memory/app-store.ts +31 -1
  524. package/src/memory/attachments-store.ts +70 -0
  525. package/src/memory/auto-analysis-enqueue.ts +127 -0
  526. package/src/memory/auto-analysis-guard.ts +27 -0
  527. package/src/memory/cleanup-schedule-state.ts +37 -0
  528. package/src/memory/conversation-analyze-job.ts +73 -0
  529. package/src/memory/conversation-crud.ts +122 -0
  530. package/src/memory/conversation-disk-view.ts +7 -0
  531. package/src/memory/conversation-group-migration.ts +34 -2
  532. package/src/memory/conversation-queries.ts +6 -5
  533. package/src/memory/conversation-starters-cadence.ts +76 -0
  534. package/src/memory/conversation-title-service.ts +5 -2
  535. package/src/memory/db-init.ts +18 -0
  536. package/src/memory/db-maintenance.ts +108 -0
  537. package/src/memory/db.ts +1 -0
  538. package/src/memory/embedding-backend.test.ts +75 -0
  539. package/src/memory/embedding-backend.ts +131 -5
  540. package/src/memory/embedding-gemini.test.ts +54 -0
  541. package/src/memory/embedding-gemini.ts +20 -9
  542. package/src/memory/embedding-local.ts +176 -17
  543. package/src/memory/graph/consolidation.ts +10 -23
  544. package/src/memory/graph/conversation-graph-memory.ts +15 -0
  545. package/src/memory/graph/extraction-job.ts +15 -0
  546. package/src/memory/graph/extraction.test.ts +23 -0
  547. package/src/memory/graph/extraction.ts +8 -0
  548. package/src/memory/graph/retriever.ts +67 -40
  549. package/src/memory/graph/scoring.test.ts +186 -0
  550. package/src/memory/graph/scoring.ts +31 -1
  551. package/src/memory/graph/store.test.ts +7 -3
  552. package/src/memory/graph/store.ts +47 -12
  553. package/src/memory/graph/tools.ts +1 -1
  554. package/src/memory/group-crud.ts +6 -1
  555. package/src/memory/indexer.ts +95 -16
  556. package/src/memory/job-handlers/cleanup.ts +11 -8
  557. package/src/memory/job-handlers/conversation-starters.ts +16 -10
  558. package/src/memory/jobs-store.ts +64 -4
  559. package/src/memory/jobs-worker.ts +22 -9
  560. package/src/memory/llm-usage-store.ts +137 -60
  561. package/src/memory/migrations/213-oauth-providers-scope-separator.ts +13 -0
  562. package/src/memory/migrations/214-oauth-providers-refresh-url.ts +11 -0
  563. package/src/memory/migrations/215-oauth-providers-revoke.ts +14 -0
  564. package/src/memory/migrations/216-oauth-providers-token-auth-method.ts +30 -0
  565. package/src/memory/migrations/217-conversation-host-access.ts +40 -0
  566. package/src/memory/migrations/218-oauth-providers-logo-url.ts +11 -0
  567. package/src/memory/migrations/219-oauth-providers-token-exchange-body-format.ts +15 -0
  568. package/src/memory/migrations/220-normalize-user-file-by-principal.ts +190 -0
  569. package/src/memory/migrations/221-conversations-archived-at.ts +16 -0
  570. package/src/memory/migrations/index.ts +12 -0
  571. package/src/memory/migrations/registry.ts +16 -0
  572. package/src/memory/qdrant-manager.ts +43 -16
  573. package/src/memory/schema/conversations.ts +3 -0
  574. package/src/memory/schema/oauth.ts +21 -13
  575. package/src/memory/usage-buckets.ts +396 -0
  576. package/src/messaging/providers/gmail/client.ts +57 -6
  577. package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +282 -0
  578. package/src/messaging/providers/slack/adapter.ts +143 -38
  579. package/src/messaging/providers/slack/client.ts +16 -0
  580. package/src/messaging/providers/slack/types.ts +4 -0
  581. package/src/notifications/decision-engine.ts +3 -3
  582. package/src/notifications/signal.ts +5 -0
  583. package/src/oauth/AGENTS.md +76 -0
  584. package/src/oauth/__tests__/identity-verifier.test.ts +25 -19
  585. package/src/oauth/__tests__/seed-providers-managed.test.ts +32 -0
  586. package/src/oauth/byo-connection.test.ts +26 -9
  587. package/src/oauth/byo-connection.ts +10 -8
  588. package/src/oauth/connect-orchestrator.ts +25 -21
  589. package/src/oauth/connect-types.ts +3 -3
  590. package/src/oauth/connection-resolver.test.ts +17 -4
  591. package/src/oauth/connection-resolver.ts +22 -18
  592. package/src/oauth/connection.ts +3 -1
  593. package/src/oauth/manual-token-connection.ts +13 -13
  594. package/src/oauth/oauth-store.ts +223 -100
  595. package/src/oauth/platform-connection.test.ts +101 -3
  596. package/src/oauth/platform-connection.ts +56 -35
  597. package/src/oauth/provider-serializer.ts +31 -5
  598. package/src/oauth/revoke.ts +76 -0
  599. package/src/oauth/seed-providers.ts +133 -87
  600. package/src/oauth/token-persistence.ts +1 -1
  601. package/src/permissions/checker.ts +16 -6
  602. package/src/permissions/defaults.ts +49 -1
  603. package/src/permissions/permission-mode.ts +4 -11
  604. package/src/permissions/prompter.ts +13 -1
  605. package/src/permissions/trust-store.ts +3 -3
  606. package/src/permissions/v2-consent-policy.ts +87 -0
  607. package/src/permissions/workspace-policy.ts +3 -0
  608. package/src/platform/client.test.ts +10 -0
  609. package/src/platform/sync-identity.ts +129 -0
  610. package/src/prompts/persona-resolver.ts +126 -2
  611. package/src/prompts/system-prompt.ts +76 -38
  612. package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +3 -65
  613. package/src/prompts/templates/BOOTSTRAP.md +59 -105
  614. package/src/prompts/templates/SOUL.md +3 -1
  615. package/src/prompts/templates/UPDATES.md +12 -0
  616. package/src/prompts/templates/channels/slack.md +20 -0
  617. package/src/prompts/update-bulletin-format.ts +26 -9
  618. package/src/prompts/update-bulletin.ts +34 -23
  619. package/src/prompts/user-reference.ts +20 -17
  620. package/src/providers/__tests__/provider-secret-catalog.test.ts +42 -0
  621. package/src/providers/anthropic/client.ts +157 -60
  622. package/src/providers/fireworks/client.ts +2 -2
  623. package/src/providers/gemini/client.ts +9 -1
  624. package/src/providers/model-catalog.ts +6 -0
  625. package/src/providers/model-intents.ts +4 -4
  626. package/src/providers/ollama/client.ts +2 -2
  627. package/src/providers/openai/chat-completions-provider.ts +474 -0
  628. package/src/providers/openai/client.ts +25 -440
  629. package/src/providers/openai/responses-provider.ts +502 -0
  630. package/src/providers/openrouter/client.ts +101 -4
  631. package/src/providers/provider-secret-catalog.ts +139 -0
  632. package/src/providers/registry.ts +2 -2
  633. package/src/providers/retry.ts +14 -3
  634. package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +251 -0
  635. package/src/providers/speech-to-text/__tests__/resolve.test.ts +828 -0
  636. package/src/providers/speech-to-text/deepgram-realtime.test.ts +980 -0
  637. package/src/providers/speech-to-text/deepgram-realtime.ts +767 -0
  638. package/src/providers/speech-to-text/deepgram.test.ts +332 -0
  639. package/src/providers/speech-to-text/deepgram.ts +115 -0
  640. package/src/providers/speech-to-text/google-gemini-live-stream.test.ts +743 -0
  641. package/src/providers/speech-to-text/google-gemini-live-stream.ts +625 -0
  642. package/src/providers/speech-to-text/google-gemini.test.ts +226 -0
  643. package/src/providers/speech-to-text/google-gemini.ts +101 -0
  644. package/src/providers/speech-to-text/openai-whisper-stream.test.ts +564 -0
  645. package/src/providers/speech-to-text/openai-whisper-stream.ts +381 -0
  646. package/src/providers/speech-to-text/openai-whisper.test.ts +1 -37
  647. package/src/providers/speech-to-text/openai-whisper.ts +63 -33
  648. package/src/providers/speech-to-text/provider-catalog.ts +306 -0
  649. package/src/providers/speech-to-text/resolve.ts +386 -6
  650. package/src/providers/types.ts +10 -1
  651. package/src/runtime/AGENTS.md +65 -0
  652. package/src/runtime/__tests__/agent-wake.test.ts +831 -0
  653. package/src/runtime/__tests__/browser-extension-pair-routes.test.ts +715 -0
  654. package/src/runtime/__tests__/capability-tokens.test.ts +258 -0
  655. package/src/runtime/__tests__/chrome-extension-registry.test.ts +518 -0
  656. package/src/runtime/__tests__/runtime-mode.test.ts +62 -0
  657. package/src/runtime/__tests__/slack-block-formatting.test.ts +481 -0
  658. package/src/runtime/agent-wake.ts +512 -0
  659. package/src/runtime/assistant-event-hub.ts +2 -2
  660. package/src/runtime/auth/__tests__/guard-tests.test.ts +1 -0
  661. package/src/runtime/auth/__tests__/middleware.test.ts +116 -1
  662. package/src/runtime/auth/__tests__/route-policy.test.ts +48 -0
  663. package/src/runtime/auth/middleware.ts +98 -0
  664. package/src/runtime/auth/route-policy.ts +33 -9
  665. package/src/runtime/auth/token-service.ts +56 -1
  666. package/src/runtime/btw-sidechain.ts +2 -0
  667. package/src/runtime/capability-tokens.ts +414 -0
  668. package/src/runtime/channel-approvals.ts +18 -5
  669. package/src/runtime/channel-invite-transport.ts +1 -1
  670. package/src/runtime/channel-invite-transports/email.ts +14 -6
  671. package/src/runtime/channel-readiness-service.ts +12 -22
  672. package/src/runtime/chrome-extension-registry.ts +368 -0
  673. package/src/runtime/confirmation-request-guardian-bridge.ts +6 -0
  674. package/src/runtime/guardian-decision-types.ts +7 -0
  675. package/src/runtime/http-server.ts +815 -75
  676. package/src/runtime/http-types.ts +6 -2
  677. package/src/runtime/migrations/__tests__/rebind-secrets-credentials.test.ts +172 -0
  678. package/src/runtime/migrations/__tests__/vbundle-builder-credentials.test.ts +276 -0
  679. package/src/runtime/migrations/__tests__/vbundle-import-credentials.test.ts +198 -0
  680. package/src/runtime/migrations/__tests__/vbundle-legacy-user-md.test.ts +360 -0
  681. package/src/runtime/migrations/migration-transport.ts +7 -0
  682. package/src/runtime/migrations/migration-wizard.ts +23 -2
  683. package/src/runtime/migrations/rebind-secrets-screen.ts +76 -15
  684. package/src/runtime/migrations/vbundle-builder.ts +145 -38
  685. package/src/runtime/migrations/vbundle-import-analyzer.ts +96 -1
  686. package/src/runtime/migrations/vbundle-importer.ts +89 -5
  687. package/src/runtime/pending-interactions.ts +18 -13
  688. package/src/runtime/routes/__tests__/backup-routes.test.ts +967 -0
  689. package/src/runtime/routes/__tests__/home-feed-routes.test.ts +507 -0
  690. package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +208 -0
  691. package/src/runtime/routes/__tests__/stt-routes.test.ts +406 -0
  692. package/src/runtime/routes/__tests__/tts-routes.test.ts +474 -0
  693. package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +148 -17
  694. package/src/runtime/routes/app-management-routes.ts +12 -18
  695. package/src/runtime/routes/approval-routes.ts +90 -16
  696. package/src/runtime/routes/attachment-routes.test.ts +9 -3
  697. package/src/runtime/routes/attachment-routes.ts +216 -17
  698. package/src/runtime/routes/backup-routes.ts +519 -0
  699. package/src/runtime/routes/browser-extension-pair-routes.ts +556 -0
  700. package/src/runtime/routes/btw-routes.ts +8 -6
  701. package/src/runtime/routes/contact-routes.test.ts +298 -0
  702. package/src/runtime/routes/contact-routes.ts +132 -5
  703. package/src/runtime/routes/conversation-analysis-routes.ts +22 -141
  704. package/src/runtime/routes/conversation-management-routes.ts +223 -0
  705. package/src/runtime/routes/conversation-routes.ts +598 -103
  706. package/src/runtime/routes/conversation-starter-routes.ts +78 -16
  707. package/src/runtime/routes/filing-routes.ts +93 -0
  708. package/src/runtime/routes/guardian-action-routes.ts +24 -13
  709. package/src/runtime/routes/home-feed-routes.ts +334 -0
  710. package/src/runtime/routes/home-state-routes.ts +138 -0
  711. package/src/runtime/routes/host-browser-routes.ts +268 -0
  712. package/src/runtime/routes/host-file-routes.ts +9 -1
  713. package/src/runtime/routes/identity-intro-cache.ts +7 -3
  714. package/src/runtime/routes/identity-routes.ts +262 -33
  715. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +46 -39
  716. package/src/runtime/routes/inbound-stages/transcribe-audio.ts +15 -15
  717. package/src/runtime/routes/integrations/slack/__tests__/channel.test.ts +137 -0
  718. package/src/runtime/routes/integrations/slack/__tests__/share.test.ts +179 -0
  719. package/src/runtime/routes/integrations/slack/channel.ts +11 -3
  720. package/src/runtime/routes/integrations/slack/share.ts +45 -7
  721. package/src/runtime/routes/llm-context-normalization.ts +303 -0
  722. package/src/runtime/routes/log-export-routes.ts +42 -22
  723. package/src/runtime/routes/memory-item-routes.test.ts +3 -2
  724. package/src/runtime/routes/memory-item-routes.ts +1 -7
  725. package/src/runtime/routes/migration-routes.ts +122 -2
  726. package/src/runtime/routes/oauth-apps.ts +15 -17
  727. package/src/runtime/routes/oauth-providers.ts +4 -0
  728. package/src/runtime/routes/schedule-routes.ts +24 -11
  729. package/src/runtime/routes/settings-routes.ts +31 -102
  730. package/src/runtime/routes/skills-routes.ts +128 -9
  731. package/src/runtime/routes/stt-routes.ts +233 -0
  732. package/src/runtime/routes/subagents-routes.ts +14 -10
  733. package/src/runtime/routes/surface-action-routes.ts +41 -2
  734. package/src/runtime/routes/tts-routes.ts +108 -24
  735. package/src/runtime/routes/usage-routes.ts +38 -9
  736. package/src/runtime/routes/user-route-dispatcher.ts +50 -5
  737. package/src/runtime/routes/user-routes.ts +13 -1
  738. package/src/runtime/routes/work-items-routes.ts +8 -1
  739. package/src/runtime/routes/workspace-routes.test.ts +22 -0
  740. package/src/runtime/routes/workspace-routes.ts +8 -1
  741. package/src/runtime/routes/workspace-utils.ts +2 -0
  742. package/src/runtime/runtime-mode.ts +33 -0
  743. package/src/runtime/services/__tests__/analyze-conversation.test.ts +444 -0
  744. package/src/runtime/services/__tests__/analyze-deps-singleton.test.ts +67 -0
  745. package/src/runtime/services/__tests__/auto-analysis-prompt.test.ts +53 -0
  746. package/src/runtime/services/__tests__/manual-analysis-prompt.test.ts +41 -0
  747. package/src/runtime/services/analyze-conversation.ts +344 -0
  748. package/src/runtime/services/analyze-deps-singleton.ts +32 -0
  749. package/src/runtime/services/auto-analysis-prompt.ts +55 -0
  750. package/src/runtime/skill-route-registry.ts +49 -0
  751. package/src/runtime/slack-block-formatting.ts +437 -10
  752. package/src/schedule/scheduler.ts +57 -5
  753. package/src/security/ces-credential-client.ts +20 -0
  754. package/src/security/ces-rpc-credential-backend.ts +17 -0
  755. package/src/security/credential-backend.ts +5 -0
  756. package/src/security/oauth2.ts +68 -29
  757. package/src/security/secure-keys.ts +143 -27
  758. package/src/security/token-manager.ts +31 -10
  759. package/src/sequence/engine.ts +23 -0
  760. package/src/sequence/types.ts +1 -1
  761. package/src/skills/catalog-files.ts +554 -0
  762. package/src/skills/category-inference.ts +122 -0
  763. package/src/skills/clawhub-files.ts +213 -0
  764. package/src/skills/clawhub.ts +84 -23
  765. package/src/skills/skill-file-provider.ts +40 -0
  766. package/src/skills/skillssh-files.ts +395 -0
  767. package/src/skills/skillssh-registry.ts +4 -4
  768. package/src/stt/__tests__/daemon-batch-transcriber.test.ts +392 -0
  769. package/src/stt/__tests__/types.test.ts +89 -0
  770. package/src/stt/daemon-batch-transcriber.ts +195 -0
  771. package/src/stt/stt-stream-session.ts +499 -0
  772. package/src/stt/types.ts +330 -0
  773. package/src/stt/wav-encoder.test.ts +373 -0
  774. package/src/stt/wav-encoder.ts +175 -0
  775. package/src/subagent/manager.ts +169 -40
  776. package/src/subagent/types.ts +19 -0
  777. package/src/tools/apps/executors.ts +11 -2
  778. package/src/tools/browser/__tests__/auth-detector.test.ts +202 -108
  779. package/src/tools/browser/__tests__/browser-mode.test.ts +119 -0
  780. package/src/tools/browser/__tests__/browser-status.test.ts +123 -0
  781. package/src/tools/browser/auth-detector.ts +43 -12
  782. package/src/tools/browser/browser-execution.ts +1787 -342
  783. package/src/tools/browser/browser-manager.ts +81 -12
  784. package/src/tools/browser/browser-mode-constants.ts +12 -0
  785. package/src/tools/browser/browser-mode.ts +92 -0
  786. package/src/tools/browser/browser-status-constants.ts +33 -0
  787. package/src/tools/browser/cdp-client/__tests__/accessibility-snapshot.test.ts +318 -0
  788. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +1175 -0
  789. package/src/tools/browser/cdp-client/__tests__/cdp-inspect-client.test.ts +1263 -0
  790. package/src/tools/browser/cdp-client/__tests__/extension-cdp-client.test.ts +359 -0
  791. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +1993 -0
  792. package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-nested-frames.json +64 -0
  793. package/src/tools/browser/cdp-client/__tests__/fixtures/ax-tree-simple.json +69 -0
  794. package/src/tools/browser/cdp-client/__tests__/local-cdp-client.test.ts +310 -0
  795. package/src/tools/browser/cdp-client/__tests__/types.test.ts +96 -0
  796. package/src/tools/browser/cdp-client/accessibility-snapshot.ts +387 -0
  797. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +695 -0
  798. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/discovery.test.ts +1007 -0
  799. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +580 -0
  800. package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +744 -0
  801. package/src/tools/browser/cdp-client/cdp-inspect/ws-transport.ts +579 -0
  802. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +868 -0
  803. package/src/tools/browser/cdp-client/errors.ts +49 -0
  804. package/src/tools/browser/cdp-client/extension-cdp-client.ts +148 -0
  805. package/src/tools/browser/cdp-client/factory.ts +914 -0
  806. package/src/tools/browser/cdp-client/index.ts +28 -0
  807. package/src/tools/browser/cdp-client/local-cdp-client.ts +187 -0
  808. package/src/tools/browser/cdp-client/types.ts +120 -0
  809. package/src/tools/credentials/vault.ts +35 -6
  810. package/src/tools/filesystem/edit.ts +1 -1
  811. package/src/tools/filesystem/list.ts +1 -1
  812. package/src/tools/filesystem/read.ts +1 -1
  813. package/src/tools/filesystem/write.ts +2 -1
  814. package/src/tools/host-filesystem/edit.ts +1 -1
  815. package/src/tools/host-filesystem/read.ts +12 -15
  816. package/src/tools/host-filesystem/write.ts +1 -1
  817. package/src/tools/host-terminal/host-shell.ts +21 -16
  818. package/src/tools/network/web-fetch.ts +5 -2
  819. package/src/tools/network/web-search.ts +5 -2
  820. package/src/tools/permission-checker.ts +77 -82
  821. package/src/tools/registry.ts +0 -2
  822. package/src/tools/secret-detection-handler.ts +34 -0
  823. package/src/tools/shared/filesystem/image-read.ts +61 -40
  824. package/src/tools/shared/shell-output.ts +3 -1
  825. package/src/tools/side-effects.ts +2 -0
  826. package/src/tools/skills/sandbox-runner.ts +3 -2
  827. package/src/tools/subagent/spawn.ts +47 -3
  828. package/src/tools/subagent/status.ts +2 -0
  829. package/src/tools/system/register.ts +2 -16
  830. package/src/tools/terminal/safe-env.ts +15 -0
  831. package/src/tools/terminal/shell.ts +36 -20
  832. package/src/tools/tool-approval-handler.ts +48 -2
  833. package/src/tools/tool-manifest.ts +21 -0
  834. package/src/tools/types.ts +19 -0
  835. package/src/tools/ui-surface/definitions.ts +6 -1
  836. package/src/tts/__tests__/provider-adapters.test.ts +834 -0
  837. package/src/tts/__tests__/provider-catalog-consistency.test.ts +196 -0
  838. package/src/tts/__tests__/provider-catalog.test.ts +183 -0
  839. package/src/tts/__tests__/provider-registry.test.ts +90 -0
  840. package/src/tts/provider-catalog.ts +201 -0
  841. package/src/tts/provider-registry.ts +73 -0
  842. package/src/tts/providers/deepgram-provider.ts +219 -0
  843. package/src/tts/providers/elevenlabs-provider.ts +211 -0
  844. package/src/tts/providers/fish-audio-provider.ts +183 -0
  845. package/src/tts/providers/index.ts +42 -0
  846. package/src/tts/providers/register-builtins.ts +130 -0
  847. package/src/tts/synthesize-text.ts +110 -0
  848. package/src/tts/tts-config-resolver.ts +78 -0
  849. package/src/tts/types.ts +153 -0
  850. package/src/types/onboarding-context.ts +7 -0
  851. package/src/util/abort-reasons.ts +58 -0
  852. package/src/util/device-id.ts +32 -16
  853. package/src/util/errors.ts +9 -1
  854. package/src/util/platform.ts +63 -24
  855. package/src/util/pricing.ts +66 -3
  856. package/src/util/spawn.ts +1 -1
  857. package/src/util/truncate.ts +4 -2
  858. package/src/util/unicode.ts +201 -0
  859. package/src/version.ts +19 -24
  860. package/src/watcher/engine.ts +23 -0
  861. package/src/watcher/watcher-store.ts +31 -0
  862. package/src/workspace/migrations/003-seed-device-id.ts +9 -3
  863. package/src/workspace/migrations/017-seed-persona-dirs.ts +68 -4
  864. package/src/workspace/migrations/029-seed-pkb.ts +1 -1
  865. package/src/workspace/migrations/031-drop-user-md.ts +317 -0
  866. package/src/workspace/migrations/031-llm-log-retention-zero-to-null.ts +73 -0
  867. package/src/workspace/migrations/032-tts-provider-unification.ts +227 -0
  868. package/src/workspace/migrations/033-stt-service-explicit-config.ts +122 -0
  869. package/src/workspace/migrations/034-remove-calls-voice-transcription-provider.ts +215 -0
  870. package/src/workspace/migrations/035-seed-slack-channel-persona.ts +50 -0
  871. package/src/workspace/migrations/036-update-pkb-index-bar.ts +37 -0
  872. package/src/workspace/migrations/037-create-meets-dir.ts +61 -0
  873. package/src/workspace/migrations/registry.ts +16 -0
  874. package/src/workspace/top-level-renderer.ts +31 -1
  875. package/src/workspace/turn-commit.ts +31 -0
  876. package/src/__tests__/chrome-cdp.test.ts +0 -419
  877. package/src/__tests__/email-cli.test.ts +0 -297
  878. package/src/__tests__/email-service-config-fallback.test.ts +0 -102
  879. package/src/__tests__/permission-mode-sse.test.ts +0 -418
  880. package/src/__tests__/permission-mode-store.test.ts +0 -277
  881. package/src/browser-extension-relay/protocol.ts +0 -63
  882. package/src/browser-extension-relay/server.ts +0 -203
  883. package/src/cli/commands/browser-relay.ts +0 -536
  884. package/src/config/schemas/sandbox.ts +0 -14
  885. package/src/email/guardrails.ts +0 -221
  886. package/src/email/provider.ts +0 -117
  887. package/src/email/providers/agentmail.ts +0 -361
  888. package/src/email/providers/index.ts +0 -65
  889. package/src/email/service.ts +0 -384
  890. package/src/email/types.ts +0 -126
  891. package/src/permissions/permission-mode-store.ts +0 -180
  892. package/src/prompts/templates/USER.md +0 -13
  893. package/src/providers/speech-to-text/types.ts +0 -17
  894. package/src/tools/browser/chrome-cdp.ts +0 -239
  895. package/src/tools/system/set-permission-mode.ts +0 -103
@@ -0,0 +1,743 @@
1
+ import { afterEach, beforeEach, describe, expect, test } from "bun:test";
2
+
3
+ import type { SttStreamServerEvent } from "../../stt/types.js";
4
+ import { GoogleGeminiLiveStreamingTranscriber } from "./google-gemini-live-stream.js";
5
+
6
+ const TEST_API_KEY = "google-live-test-key";
7
+
8
+ // ---------------------------------------------------------------------------
9
+ // Mock Live session
10
+ // ---------------------------------------------------------------------------
11
+
12
+ interface LiveCallbacksForTests {
13
+ onopen?: () => void;
14
+ onmessage: (msg: unknown) => void;
15
+ onerror?: (ev: unknown) => void;
16
+ onclose?: (ev: { code: number; reason: string }) => void;
17
+ }
18
+
19
+ interface CapturedConnect {
20
+ params: {
21
+ model: string;
22
+ config?: {
23
+ responseModalities?: unknown;
24
+ inputAudioTranscription?: unknown;
25
+ systemInstruction?: unknown;
26
+ };
27
+ callbacks: LiveCallbacksForTests;
28
+ };
29
+ session: MockLiveSession;
30
+ }
31
+
32
+ /**
33
+ * Minimal mock of the Live API Session returned by `ai.live.connect`.
34
+ * Tests drive behavior via the captured `callbacks` on the connect
35
+ * invocation plus helper methods (`simulateMessage`, `simulateClose`,
36
+ * `simulateError`).
37
+ */
38
+ class MockLiveSession {
39
+ sentInputs: unknown[] = [];
40
+ closeCalled = false;
41
+ private readonly callbacks: LiveCallbacksForTests;
42
+
43
+ constructor(callbacks: LiveCallbacksForTests) {
44
+ this.callbacks = callbacks;
45
+ }
46
+
47
+ sendRealtimeInput(params: unknown): void {
48
+ this.sentInputs.push(params);
49
+ }
50
+
51
+ sendClientContent(_params: unknown): void {
52
+ // Not used by the adapter but present to satisfy structural typing.
53
+ }
54
+
55
+ close(): void {
56
+ this.closeCalled = true;
57
+ }
58
+
59
+ // ── Test helpers ──────────────────────────────────────────────────
60
+
61
+ simulateMessage(msg: Record<string, unknown>): void {
62
+ this.callbacks.onmessage(msg);
63
+ }
64
+
65
+ simulateClose(code = 1000, reason = ""): void {
66
+ this.callbacks.onclose?.({ code, reason });
67
+ }
68
+
69
+ simulateError(ev: unknown): void {
70
+ this.callbacks.onerror?.(ev);
71
+ }
72
+ }
73
+
74
+ // ---------------------------------------------------------------------------
75
+ // Helpers
76
+ // ---------------------------------------------------------------------------
77
+
78
+ /** Collect all events emitted during a test. */
79
+ function createEventCollector(): {
80
+ events: SttStreamServerEvent[];
81
+ onEvent: (event: SttStreamServerEvent) => void;
82
+ } {
83
+ const events: SttStreamServerEvent[] = [];
84
+ return {
85
+ events,
86
+ onEvent: (event: SttStreamServerEvent) => events.push(event),
87
+ };
88
+ }
89
+
90
+ /**
91
+ * Install a mock GoogleGenAI constructor on the module's `client` field
92
+ * before `start()` runs. We patch the adapter's private client after
93
+ * construction to a minimal object that exposes `live.connect()`.
94
+ */
95
+ function installMockClient(
96
+ transcriber: GoogleGeminiLiveStreamingTranscriber,
97
+ opts: {
98
+ /** Whether to invoke `onopen` on the next microtask. Default: true. */
99
+ autoOpen?: boolean;
100
+ /** If provided, delay the mock's `connect` resolution by N ms. */
101
+ resolveDelayMs?: number;
102
+ } = {},
103
+ ): {
104
+ capturedCalls: CapturedConnect[];
105
+ } {
106
+ const capturedCalls: CapturedConnect[] = [];
107
+ const autoOpen = opts.autoOpen ?? true;
108
+
109
+ const mockLive = {
110
+ connect: (params: CapturedConnect["params"]): Promise<MockLiveSession> => {
111
+ const session = new MockLiveSession(params.callbacks);
112
+ capturedCalls.push({ params, session });
113
+
114
+ // Fire onopen on the next microtask by default.
115
+ if (autoOpen) {
116
+ queueMicrotask(() => {
117
+ params.callbacks.onopen?.();
118
+ });
119
+ }
120
+
121
+ if (opts.resolveDelayMs !== undefined) {
122
+ return new Promise((resolve) =>
123
+ setTimeout(() => resolve(session), opts.resolveDelayMs),
124
+ );
125
+ }
126
+ return Promise.resolve(session);
127
+ },
128
+ };
129
+
130
+ (transcriber as unknown as { client: { live: typeof mockLive } }).client = {
131
+ live: mockLive,
132
+ };
133
+
134
+ return { capturedCalls };
135
+ }
136
+
137
+ // ---------------------------------------------------------------------------
138
+ // Test suite
139
+ // ---------------------------------------------------------------------------
140
+
141
+ describe("GoogleGeminiLiveStreamingTranscriber", () => {
142
+ let transcriber: GoogleGeminiLiveStreamingTranscriber;
143
+
144
+ beforeEach(() => {
145
+ transcriber = new GoogleGeminiLiveStreamingTranscriber(TEST_API_KEY, {
146
+ // Long inactivity timeout to avoid test flakes.
147
+ inactivityTimeoutMs: 60_000,
148
+ });
149
+ });
150
+
151
+ afterEach(() => {
152
+ // Best-effort stop in case a test forgot.
153
+ try {
154
+ transcriber.stop();
155
+ } catch {
156
+ // ignore
157
+ }
158
+ });
159
+
160
+ // ── Helper: start a session ────────────────────────────────────────
161
+
162
+ async function startSession(
163
+ options?: {
164
+ transcriberOptions?: ConstructorParameters<
165
+ typeof GoogleGeminiLiveStreamingTranscriber
166
+ >[1];
167
+ installOptions?: Parameters<typeof installMockClient>[1];
168
+ },
169
+ ): Promise<{
170
+ transcriber: GoogleGeminiLiveStreamingTranscriber;
171
+ session: MockLiveSession;
172
+ events: SttStreamServerEvent[];
173
+ capturedCalls: CapturedConnect[];
174
+ }> {
175
+ const t = options?.transcriberOptions
176
+ ? new GoogleGeminiLiveStreamingTranscriber(
177
+ TEST_API_KEY,
178
+ options.transcriberOptions,
179
+ )
180
+ : transcriber;
181
+ const { capturedCalls } = installMockClient(t, options?.installOptions);
182
+ const { events, onEvent } = createEventCollector();
183
+ await t.start(onEvent);
184
+ const session = capturedCalls[0]?.session;
185
+ if (!session) throw new Error("No connect call captured");
186
+ return { transcriber: t, session, events, capturedCalls };
187
+ }
188
+
189
+ // ─────────────────────────────────────────────────────────────────
190
+ // Lifecycle
191
+ // ─────────────────────────────────────────────────────────────────
192
+
193
+ test("start() resolves after onopen fires", async () => {
194
+ installMockClient(transcriber);
195
+ const { onEvent } = createEventCollector();
196
+ await transcriber.start(onEvent);
197
+ // No error = success
198
+ });
199
+
200
+ test("start() rejects after connectTimeoutMs if open never fires", async () => {
201
+ const t = new GoogleGeminiLiveStreamingTranscriber(TEST_API_KEY, {
202
+ connectTimeoutMs: 30,
203
+ });
204
+ installMockClient(t, { autoOpen: false });
205
+
206
+ const { onEvent } = createEventCollector();
207
+ await expect(t.start(onEvent)).rejects.toThrow(
208
+ "Gemini Live connect timeout",
209
+ );
210
+ });
211
+
212
+ test("start() throws if called twice", async () => {
213
+ const { transcriber: t } = await startSession();
214
+ await expect(t.start(() => {})).rejects.toThrow("start() called twice");
215
+ });
216
+
217
+ test("sendAudio() is a no-op before start() resolves (session not yet set)", () => {
218
+ installMockClient(transcriber);
219
+ // Do not await start() — session is null.
220
+ transcriber.sendAudio(Buffer.from([1, 2, 3]), "audio/pcm;rate=16000");
221
+ // No throw; nothing else observable since no session was created yet.
222
+ });
223
+
224
+ test("stop() is idempotent", async () => {
225
+ const { transcriber: t, session, events } = await startSession();
226
+ t.stop();
227
+ session.simulateClose(1000, "normal");
228
+ t.stop(); // Second call should be a no-op.
229
+
230
+ const closedEvents = events.filter((e) => e.type === "closed");
231
+ expect(closedEvents).toHaveLength(1);
232
+ });
233
+
234
+ // ─────────────────────────────────────────────────────────────────
235
+ // Setup config
236
+ // ─────────────────────────────────────────────────────────────────
237
+
238
+ test("connect() is called with TEXT-only modality and inputAudioTranscription enabled", async () => {
239
+ const { capturedCalls } = await startSession();
240
+
241
+ expect(capturedCalls).toHaveLength(1);
242
+ const params = capturedCalls[0].params;
243
+ expect(params.model).toBe("gemini-live-2.5-flash-preview");
244
+ expect(params.config?.responseModalities).toEqual(["TEXT"]);
245
+ expect(params.config?.inputAudioTranscription).toEqual({});
246
+ expect(typeof params.config?.systemInstruction).toBe("string");
247
+ });
248
+
249
+ // ─────────────────────────────────────────────────────────────────
250
+ // Audio forwarding
251
+ // ─────────────────────────────────────────────────────────────────
252
+
253
+ test("sendAudio() forwards base64-encoded PCM with the provided mimeType", async () => {
254
+ const { transcriber: t, session } = await startSession();
255
+
256
+ const audio = Buffer.from([1, 2, 3]);
257
+ t.sendAudio(audio, "audio/pcm;rate=16000");
258
+
259
+ expect(session.sentInputs).toHaveLength(1);
260
+ const input = session.sentInputs[0] as {
261
+ audio?: { data: string; mimeType: string };
262
+ };
263
+ expect(input.audio?.data).toBe(audio.toString("base64"));
264
+ expect(input.audio?.mimeType).toBe("audio/pcm;rate=16000");
265
+ });
266
+
267
+ test("sendAudio() normalizes bare audio/pcm to include the configured sample rate", async () => {
268
+ const { transcriber: t, session } = await startSession({
269
+ transcriberOptions: {
270
+ pcmSampleRate: 24_000,
271
+ inactivityTimeoutMs: 60_000,
272
+ },
273
+ });
274
+
275
+ t.sendAudio(Buffer.from([9, 9, 9]), "audio/pcm");
276
+
277
+ const input = session.sentInputs[0] as {
278
+ audio?: { data: string; mimeType: string };
279
+ };
280
+ expect(input.audio?.mimeType).toBe("audio/pcm;rate=24000");
281
+ });
282
+
283
+ test("sendAudio() passes non-PCM mime types through unchanged", async () => {
284
+ const { transcriber: t, session } = await startSession();
285
+
286
+ t.sendAudio(Buffer.from([1, 2]), "audio/webm");
287
+
288
+ const input = session.sentInputs[0] as {
289
+ audio?: { data: string; mimeType: string };
290
+ };
291
+ expect(input.audio?.mimeType).toBe("audio/webm");
292
+ });
293
+
294
+ // ─────────────────────────────────────────────────────────────────
295
+ // Transcription events: partial → final
296
+ // ─────────────────────────────────────────────────────────────────
297
+
298
+ test("emits partial events for accumulated inputTranscription text", async () => {
299
+ const { session, events } = await startSession();
300
+
301
+ session.simulateMessage({
302
+ serverContent: {
303
+ inputTranscription: { text: "hello" },
304
+ },
305
+ });
306
+ session.simulateMessage({
307
+ serverContent: {
308
+ inputTranscription: { text: " world" },
309
+ },
310
+ });
311
+
312
+ expect(events).toHaveLength(2);
313
+ expect(events[0]).toEqual({ type: "partial", text: "hello" });
314
+ expect(events[1]).toEqual({ type: "partial", text: "hello world" });
315
+ });
316
+
317
+ test("emits final event on generationComplete and resets accumulator", async () => {
318
+ const { session, events } = await startSession();
319
+
320
+ session.simulateMessage({
321
+ serverContent: { inputTranscription: { text: "hello" } },
322
+ });
323
+ session.simulateMessage({
324
+ serverContent: { inputTranscription: { text: " world" } },
325
+ });
326
+ session.simulateMessage({
327
+ serverContent: { generationComplete: true },
328
+ });
329
+
330
+ expect(events).toEqual([
331
+ { type: "partial", text: "hello" },
332
+ { type: "partial", text: "hello world" },
333
+ { type: "final", text: "hello world" },
334
+ ]);
335
+
336
+ // New turn starts fresh.
337
+ session.simulateMessage({
338
+ serverContent: { inputTranscription: { text: "again" } },
339
+ });
340
+ expect(events[events.length - 1]).toEqual({
341
+ type: "partial",
342
+ text: "again",
343
+ });
344
+ });
345
+
346
+ test("emits final event on turnComplete as well as generationComplete", async () => {
347
+ const { session, events } = await startSession();
348
+
349
+ session.simulateMessage({
350
+ serverContent: { inputTranscription: { text: "done" } },
351
+ });
352
+ session.simulateMessage({
353
+ serverContent: { turnComplete: true },
354
+ });
355
+
356
+ expect(events.filter((e) => e.type === "final")).toEqual([
357
+ { type: "final", text: "done" },
358
+ ]);
359
+ });
360
+
361
+ test("dedupes repeated partials when the transcription text does not change", async () => {
362
+ const { session, events } = await startSession();
363
+
364
+ session.simulateMessage({
365
+ serverContent: { inputTranscription: { text: "hello" } },
366
+ });
367
+ // Same text repeated — no new partial.
368
+ session.simulateMessage({
369
+ serverContent: { inputTranscription: { text: "" } },
370
+ });
371
+ session.simulateMessage({
372
+ serverContent: { inputTranscription: {} },
373
+ });
374
+
375
+ const partials = events.filter((e) => e.type === "partial");
376
+ expect(partials).toHaveLength(1);
377
+ expect(partials[0]).toEqual({ type: "partial", text: "hello" });
378
+ });
379
+
380
+ test("ignores serverContent.modelTurn payloads that don't carry inputTranscription", async () => {
381
+ const { session, events } = await startSession();
382
+
383
+ session.simulateMessage({
384
+ serverContent: {
385
+ modelTurn: { parts: [{ text: "I'm staying silent" }] },
386
+ },
387
+ });
388
+
389
+ expect(events).toHaveLength(0);
390
+ });
391
+
392
+ // ─────────────────────────────────────────────────────────────────
393
+ // Stop lifecycle
394
+ // ─────────────────────────────────────────────────────────────────
395
+
396
+ test("stop() signals audioStreamEnd and emits final + closed after provider closes", async () => {
397
+ const { transcriber: t, session, events } = await startSession();
398
+
399
+ session.simulateMessage({
400
+ serverContent: { inputTranscription: { text: "goodbye" } },
401
+ });
402
+
403
+ t.stop();
404
+
405
+ const endSignal = session.sentInputs.find(
406
+ (input) => (input as { audioStreamEnd?: boolean }).audioStreamEnd,
407
+ );
408
+ expect(endSignal).toEqual({ audioStreamEnd: true });
409
+
410
+ // Provider completes the turn then closes normally.
411
+ session.simulateClose(1000, "normal");
412
+
413
+ const finals = events.filter((e) => e.type === "final");
414
+ expect(finals).toEqual([{ type: "final", text: "goodbye" }]);
415
+
416
+ const closedEvents = events.filter((e) => e.type === "closed");
417
+ expect(closedEvents).toHaveLength(1);
418
+ });
419
+
420
+ test("stop() emits final from accumulated turn text even if provider closes without turnComplete", async () => {
421
+ const { transcriber: t, session, events } = await startSession();
422
+
423
+ session.simulateMessage({
424
+ serverContent: { inputTranscription: { text: "partial only" } },
425
+ });
426
+ t.stop();
427
+ session.simulateClose(1000, "normal");
428
+
429
+ const finals = events.filter((e) => e.type === "final");
430
+ expect(finals).toEqual([{ type: "final", text: "partial only" }]);
431
+ });
432
+
433
+ test("stop() falls back to emitting empty final and closed if no audio was ever transcribed", async () => {
434
+ const { transcriber: t, session, events } = await startSession();
435
+
436
+ t.stop();
437
+ session.simulateClose(1000, "normal");
438
+
439
+ expect(events).toEqual([
440
+ { type: "final", text: "" },
441
+ { type: "closed" },
442
+ ]);
443
+ });
444
+
445
+ // ─────────────────────────────────────────────────────────────────
446
+ // Close-code categorization
447
+ // ─────────────────────────────────────────────────────────────────
448
+
449
+ test("unexpected close with 1008 maps to auth error category", async () => {
450
+ const { session, events } = await startSession();
451
+
452
+ session.simulateClose(1008, "auth failed");
453
+
454
+ const errs = events.filter((e) => e.type === "error");
455
+ expect(errs).toHaveLength(1);
456
+ expect((errs[0] as { category: string }).category).toBe("auth");
457
+ expect(events.filter((e) => e.type === "closed")).toHaveLength(1);
458
+ });
459
+
460
+ test("unexpected close with 4001 maps to auth error category", async () => {
461
+ const { session, events } = await startSession();
462
+
463
+ session.simulateClose(4001, "invalid api key");
464
+
465
+ const errs = events.filter((e) => e.type === "error");
466
+ expect((errs[0] as { category: string }).category).toBe("auth");
467
+ });
468
+
469
+ test("unexpected close with 1013 maps to rate-limit error category", async () => {
470
+ const { session, events } = await startSession();
471
+
472
+ session.simulateClose(1013, "try again later");
473
+
474
+ const errs = events.filter((e) => e.type === "error");
475
+ expect((errs[0] as { category: string }).category).toBe("rate-limit");
476
+ });
477
+
478
+ test("unexpected close with arbitrary code maps to provider-error", async () => {
479
+ const { session, events } = await startSession();
480
+
481
+ session.simulateClose(1006, "abnormal closure");
482
+
483
+ const errs = events.filter((e) => e.type === "error");
484
+ expect((errs[0] as { category: string }).category).toBe("provider-error");
485
+ expect((errs[0] as { message: string }).message).toContain("1006");
486
+ });
487
+
488
+ test("session-level error event emits provider-error + closed", async () => {
489
+ const { session, events } = await startSession();
490
+
491
+ session.simulateError({ message: "boom" });
492
+
493
+ const errs = events.filter((e) => e.type === "error");
494
+ const closedEvents = events.filter((e) => e.type === "closed");
495
+ expect(errs).toHaveLength(1);
496
+ expect((errs[0] as { category: string }).category).toBe("provider-error");
497
+ expect((errs[0] as { message: string }).message).toContain("boom");
498
+ expect(closedEvents).toHaveLength(1);
499
+ });
500
+
501
+ // ─────────────────────────────────────────────────────────────────
502
+ // Inactivity timeout
503
+ // ─────────────────────────────────────────────────────────────────
504
+
505
+ test("inactivity timeout emits timeout error and closed", async () => {
506
+ const { events } = await startSession({
507
+ transcriberOptions: { inactivityTimeoutMs: 50 },
508
+ });
509
+
510
+ await new Promise((resolve) => setTimeout(resolve, 120));
511
+
512
+ const errs = events.filter((e) => e.type === "error");
513
+ const closedEvents = events.filter((e) => e.type === "closed");
514
+ expect(errs).toHaveLength(1);
515
+ expect((errs[0] as { category: string }).category).toBe("timeout");
516
+ expect((errs[0] as { message: string }).message).toContain("inactivity");
517
+ expect(closedEvents).toHaveLength(1);
518
+ });
519
+
520
+ test("inactivity timer resets on incoming server messages", async () => {
521
+ const { session, events } = await startSession({
522
+ transcriberOptions: { inactivityTimeoutMs: 100 },
523
+ });
524
+
525
+ // Send a message before the timer fires.
526
+ await new Promise((resolve) => setTimeout(resolve, 60));
527
+ session.simulateMessage({
528
+ serverContent: { inputTranscription: { text: "hi" } },
529
+ });
530
+
531
+ await new Promise((resolve) => setTimeout(resolve, 60));
532
+
533
+ // Should not have timed out because the timer reset on the message.
534
+ expect(events.filter((e) => e.type === "error")).toHaveLength(0);
535
+ });
536
+
537
+ // ─────────────────────────────────────────────────────────────────
538
+ // No events after closed
539
+ // ─────────────────────────────────────────────────────────────────
540
+
541
+ test("no events are emitted between final and closed on stop()", async () => {
542
+ const { transcriber: t, session, events } = await startSession();
543
+
544
+ session.simulateMessage({
545
+ serverContent: { inputTranscription: { text: "abc" } },
546
+ });
547
+ t.stop();
548
+
549
+ // Partial-style messages after stop() should not produce additional
550
+ // partial events (they are dropped because `stopping` is true) and
551
+ // never interleave between final and closed. The text is still
552
+ // accumulated so the flushed final reflects all audio that arrived
553
+ // during the grace period.
554
+ session.simulateMessage({
555
+ serverContent: { inputTranscription: { text: " late" } },
556
+ });
557
+
558
+ session.simulateClose(1000, "");
559
+
560
+ const types = events.map((e) => e.type);
561
+ const lastTwo = types.slice(-2);
562
+ expect(lastTwo).toEqual(["final", "closed"]);
563
+
564
+ // No partials emitted after stop() — only the initial pre-stop
565
+ // partial, then the final and closed.
566
+ const partialsAfterStop = events
567
+ .slice(1)
568
+ .filter((e) => e.type === "partial");
569
+ expect(partialsAfterStop).toHaveLength(0);
570
+
571
+ // But the final should include text that arrived during the grace
572
+ // period, since we kept accumulating.
573
+ const finals = events.filter((e) => e.type === "final");
574
+ expect(finals).toEqual([{ type: "final", text: "abc late" }]);
575
+ });
576
+
577
+ test("no events emitted after closed on unexpected close", async () => {
578
+ const { session, events } = await startSession();
579
+
580
+ session.simulateError(new Error("boom"));
581
+
582
+ const count = events.length;
583
+
584
+ session.simulateMessage({
585
+ serverContent: { inputTranscription: { text: "late" } },
586
+ });
587
+ session.simulateClose(1000, "");
588
+
589
+ expect(events.length).toBe(count);
590
+ });
591
+
592
+ // ─────────────────────────────────────────────────────────────────
593
+ // Regression: double-final on stop → turnComplete → close race
594
+ // ─────────────────────────────────────────────────────────────────
595
+
596
+ test(
597
+ "does not emit a second empty final when provider closes normally after a completion signal",
598
+ async () => {
599
+ const { transcriber: t, session, events } = await startSession();
600
+
601
+ session.simulateMessage({
602
+ serverContent: { inputTranscription: { text: "hello" } },
603
+ });
604
+ t.stop();
605
+
606
+ // Server flushes turnComplete in response to audioStreamEnd, then
607
+ // closes normally. Both events used to produce a final event, with
608
+ // the second being an empty string — storage-writer in Meet would
609
+ // write an empty transcript line.
610
+ session.simulateMessage({
611
+ serverContent: { turnComplete: true },
612
+ });
613
+ session.simulateClose(1000, "normal");
614
+
615
+ const finals = events.filter((e) => e.type === "final");
616
+ expect(finals).toEqual([{ type: "final", text: "hello" }]);
617
+ expect(events.filter((e) => e.type === "closed")).toHaveLength(1);
618
+ },
619
+ );
620
+
621
+ test(
622
+ "still emits a final on normal close when no completion signal arrived during the grace period",
623
+ async () => {
624
+ const { transcriber: t, session, events } = await startSession();
625
+
626
+ session.simulateMessage({
627
+ serverContent: { inputTranscription: { text: "midstream" } },
628
+ });
629
+ t.stop();
630
+ // No turnComplete/generationComplete before close — the flush is
631
+ // the only source of the final.
632
+ session.simulateClose(1000, "normal");
633
+
634
+ const finals = events.filter((e) => e.type === "final");
635
+ expect(finals).toEqual([{ type: "final", text: "midstream" }]);
636
+ },
637
+ );
638
+
639
+ // ─────────────────────────────────────────────────────────────────
640
+ // Regression: session leak on connect timeout
641
+ // ─────────────────────────────────────────────────────────────────
642
+
643
+ test(
644
+ "closes the underlying session when start() times out after the SDK resolved a session handle",
645
+ async () => {
646
+ const t = new GoogleGeminiLiveStreamingTranscriber(TEST_API_KEY, {
647
+ connectTimeoutMs: 20,
648
+ });
649
+ // autoOpen=false so `onopen` never fires; connectPromise still
650
+ // resolves synchronously with the session — this is the leak path.
651
+ const { capturedCalls } = installMockClient(t, { autoOpen: false });
652
+
653
+ const { onEvent } = createEventCollector();
654
+ await expect(t.start(onEvent)).rejects.toThrow(
655
+ "Gemini Live connect timeout",
656
+ );
657
+
658
+ const session = capturedCalls[0]?.session;
659
+ expect(session).toBeDefined();
660
+ // The session handle must have been closed by start()'s catch
661
+ // block via forceCloseSession() — otherwise the WebSocket leaks.
662
+ expect(session!.closeCalled).toBe(true);
663
+ },
664
+ );
665
+
666
+ // ─────────────────────────────────────────────────────────────────
667
+ // Regression: honor inputTranscription.finished
668
+ // ─────────────────────────────────────────────────────────────────
669
+
670
+ test(
671
+ "emits final on inputTranscription.finished=true (SDK says it's independent of model turns)",
672
+ async () => {
673
+ const { session, events } = await startSession();
674
+
675
+ session.simulateMessage({
676
+ serverContent: { inputTranscription: { text: "quick brown fox" } },
677
+ });
678
+ session.simulateMessage({
679
+ serverContent: { inputTranscription: { finished: true } },
680
+ });
681
+
682
+ const finals = events.filter((e) => e.type === "final");
683
+ expect(finals).toEqual([{ type: "final", text: "quick brown fox" }]);
684
+ },
685
+ );
686
+
687
+ test(
688
+ "finished=true also suppresses the trailing empty final on subsequent normal close",
689
+ async () => {
690
+ const { transcriber: t, session, events } = await startSession();
691
+
692
+ session.simulateMessage({
693
+ serverContent: { inputTranscription: { text: "done" } },
694
+ });
695
+ session.simulateMessage({
696
+ serverContent: { inputTranscription: { finished: true } },
697
+ });
698
+
699
+ t.stop();
700
+ session.simulateClose(1000, "normal");
701
+
702
+ const finals = events.filter((e) => e.type === "final");
703
+ expect(finals).toEqual([{ type: "final", text: "done" }]);
704
+ },
705
+ );
706
+
707
+ // ─────────────────────────────────────────────────────────────────
708
+ // Regression: MIME normalization preserves non-rate parameters
709
+ // ─────────────────────────────────────────────────────────────────
710
+
711
+ test(
712
+ "normalizePcmMimeType appends rate= without dropping other PCM parameters",
713
+ async () => {
714
+ const { transcriber: t, session } = await startSession({
715
+ transcriberOptions: {
716
+ pcmSampleRate: 16_000,
717
+ inactivityTimeoutMs: 60_000,
718
+ },
719
+ });
720
+
721
+ t.sendAudio(Buffer.from([1, 2, 3]), "audio/pcm;encoding=linear16");
722
+
723
+ const input = session.sentInputs[0] as {
724
+ audio?: { data: string; mimeType: string };
725
+ };
726
+ // The original `encoding=linear16` parameter must be preserved;
727
+ // only the missing `rate=` is appended.
728
+ expect(input.audio?.mimeType).toBe(
729
+ "audio/pcm;encoding=linear16;rate=16000",
730
+ );
731
+ },
732
+ );
733
+
734
+ // ─────────────────────────────────────────────────────────────────
735
+ // Provider identity
736
+ // ─────────────────────────────────────────────────────────────────
737
+
738
+ test("reports correct providerId and boundaryId", () => {
739
+ const t = new GoogleGeminiLiveStreamingTranscriber(TEST_API_KEY);
740
+ expect(t.providerId).toBe("google-gemini");
741
+ expect(t.boundaryId).toBe("daemon-streaming");
742
+ });
743
+ });