@agent-native/core 0.98.0 → 0.98.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (345) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +32 -0
  3. package/corpus/core/docs/content/locales/ar-SA/pr-visual-recap.mdx +25 -1
  4. package/corpus/core/docs/content/locales/ar-SA/template-plan.mdx +1 -1
  5. package/corpus/core/docs/content/locales/de-DE/pr-visual-recap.mdx +25 -1
  6. package/corpus/core/docs/content/locales/de-DE/template-plan.mdx +1 -1
  7. package/corpus/core/docs/content/locales/es-ES/pr-visual-recap.mdx +25 -1
  8. package/corpus/core/docs/content/locales/es-ES/template-plan.mdx +1 -1
  9. package/corpus/core/docs/content/locales/fr-FR/pr-visual-recap.mdx +25 -1
  10. package/corpus/core/docs/content/locales/fr-FR/template-plan.mdx +1 -1
  11. package/corpus/core/docs/content/locales/hi-IN/pr-visual-recap.mdx +25 -1
  12. package/corpus/core/docs/content/locales/hi-IN/template-plan.mdx +1 -1
  13. package/corpus/core/docs/content/locales/ja-JP/pr-visual-recap.mdx +25 -1
  14. package/corpus/core/docs/content/locales/ja-JP/template-plan.mdx +1 -1
  15. package/corpus/core/docs/content/locales/ko-KR/pr-visual-recap.mdx +25 -1
  16. package/corpus/core/docs/content/locales/ko-KR/template-plan.mdx +1 -1
  17. package/corpus/core/docs/content/locales/pt-BR/pr-visual-recap.mdx +25 -1
  18. package/corpus/core/docs/content/locales/pt-BR/template-plan.mdx +1 -1
  19. package/corpus/core/docs/content/locales/zh-CN/pr-visual-recap.mdx +25 -1
  20. package/corpus/core/docs/content/locales/zh-CN/template-plan.mdx +1 -1
  21. package/corpus/core/docs/content/locales/zh-TW/pr-visual-recap.mdx +25 -1
  22. package/corpus/core/docs/content/locales/zh-TW/template-plan.mdx +1 -1
  23. package/corpus/core/docs/content/pr-visual-recap.mdx +50 -1
  24. package/corpus/core/docs/content/template-plan.mdx +1 -1
  25. package/corpus/core/package.json +2 -2
  26. package/corpus/core/src/action-change-marker.ts +8 -1
  27. package/corpus/core/src/changelog/parse.ts +8 -3
  28. package/corpus/core/src/cli/plan-local.ts +43 -11
  29. package/corpus/core/src/cli/pr-visual-recap-workflow.ts +2 -9
  30. package/corpus/core/src/cli/recap.ts +356 -4
  31. package/corpus/core/src/cli/skills-content/local-files.ts +17 -12
  32. package/corpus/core/src/client/AgentPanel.tsx +2 -3
  33. package/corpus/core/src/client/AssistantChat.tsx +83 -39
  34. package/corpus/core/src/client/analytics.ts +125 -14
  35. package/corpus/core/src/client/blocks/library/wireframe-kit.tsx +16 -2
  36. package/corpus/core/src/client/chat-model-groups.ts +28 -0
  37. package/corpus/core/src/client/composer/RealtimeVoiceMode.tsx +442 -85
  38. package/corpus/core/src/client/composer/TiptapComposer.tsx +46 -29
  39. package/corpus/core/src/client/composer/VoiceButton.tsx +2 -1
  40. package/corpus/core/src/client/composer/index.ts +5 -0
  41. package/corpus/core/src/client/composer/useRealtimeVoiceMode.tsx +1206 -67
  42. package/corpus/core/src/client/index.ts +10 -0
  43. package/corpus/core/src/client/progress/RunsTray.tsx +13 -1
  44. package/corpus/core/src/client/session-replay.ts +10 -0
  45. package/corpus/core/src/client/settings/DemoModeSection.tsx +4 -28
  46. package/corpus/core/src/client/settings/SecretsSection.tsx +387 -266
  47. package/corpus/core/src/client/settings/SettingsPanel.tsx +117 -38
  48. package/corpus/core/src/client/url-scrub.ts +13 -0
  49. package/corpus/core/src/client/use-action.ts +12 -0
  50. package/corpus/core/src/client/use-agent-chat-home-handoff.ts +8 -1
  51. package/corpus/core/src/client/use-agent-engine-configured.ts +17 -3
  52. package/corpus/core/src/client/use-db-sync.ts +84 -34
  53. package/corpus/core/src/client/use-demo-mode-status.ts +44 -0
  54. package/corpus/core/src/client/use-pausing-interval.ts +14 -4
  55. package/corpus/core/src/integrations/scope-store.ts +1 -1
  56. package/corpus/core/src/integrations/usage-budget-store.ts +1 -1
  57. package/corpus/core/src/localization/default-messages.ts +44 -1
  58. package/corpus/core/src/mcp-client/workspace-servers.ts +4 -4
  59. package/corpus/core/src/secrets/crypto.ts +28 -8
  60. package/corpus/core/src/server/action-change.ts +5 -1
  61. package/corpus/core/src/server/action-routes.ts +8 -0
  62. package/corpus/core/src/server/poll.ts +1 -0
  63. package/corpus/core/src/server/realtime-voice.ts +299 -13
  64. package/corpus/core/src/styles/agent-native.css +2 -2
  65. package/corpus/core/src/templates/default/.agents/skills/real-time-sync/SKILL.md +2 -2
  66. package/corpus/core/src/templates/workspace-core/.agents/skills/real-time-sync/SKILL.md +2 -2
  67. package/corpus/core/src/templates/workspace-core/.agents/skills/secrets/SKILL.md +7 -3
  68. package/corpus/core/src/templates/workspace-core/.agents/skills/voice-transcription/SKILL.md +39 -9
  69. package/corpus/core/src/vite/client.ts +9 -1
  70. package/corpus/templates/analytics/.agents/skills/real-time-sync/SKILL.md +2 -2
  71. package/corpus/templates/analytics/.agents/skills/session-replay/SKILL.md +10 -5
  72. package/corpus/templates/analytics/AGENTS.md +17 -13
  73. package/corpus/templates/analytics/actions/navigate.ts +59 -5
  74. package/corpus/templates/analytics/actions/view-screen.ts +3 -6
  75. package/corpus/templates/analytics/app/components/ProviderCorpusJobNotifier.tsx +20 -3
  76. package/corpus/templates/analytics/app/components/dashboard/SqlChart.tsx +20 -5
  77. package/corpus/templates/analytics/app/components/layout/Header.tsx +1 -1
  78. package/corpus/templates/analytics/app/components/layout/Sidebar.tsx +21 -6
  79. package/corpus/templates/analytics/app/global.css +19 -20
  80. package/corpus/templates/analytics/app/lib/demo-chart-trend.ts +156 -0
  81. package/corpus/templates/analytics/app/lib/provider-corpus-job-sync.ts +26 -0
  82. package/corpus/templates/analytics/app/pages/Settings.tsx +35 -341
  83. package/corpus/templates/analytics/app/pages/monitoring/ErrorsPanel.tsx +38 -7
  84. package/corpus/templates/analytics/app/pages/monitoring/MonitoringPage.tsx +1 -1
  85. package/corpus/templates/analytics/app/pages/monitoring/errors/i18n.ts +3 -0
  86. package/corpus/templates/analytics/app/pages/sessions/SessionDetailPage.tsx +333 -272
  87. package/corpus/templates/analytics/app/pages/sessions/SessionDevToolsPanel.tsx +72 -17
  88. package/corpus/templates/analytics/app/pages/sessions/SessionsPage.tsx +297 -108
  89. package/corpus/templates/analytics/app/root.tsx +28 -8
  90. package/corpus/templates/analytics/changelog/2026-06-24-first-party-signup-and-pageview-panels-now-honor-dashboard-filters.md +2 -1
  91. package/corpus/templates/analytics/changelog/2026-06-25-referrer-dashboard-panels-show-external-domains.md +2 -1
  92. package/corpus/templates/analytics/changelog/2026-07-06-mobile-chat-header-is-more-compact-and-the-menu-button-opens-navigation.md +1 -0
  93. package/corpus/templates/analytics/changelog/2026-07-08-dashboard-history-and-chat-context.md +1 -0
  94. package/corpus/templates/analytics/changelog/2026-07-08-settings-are-cleaner-searchable-and-alerts-have-their-own-tab.md +1 -0
  95. package/corpus/templates/analytics/changelog/2026-07-08-sidebar-sections-stay-collapsed-until-opened.md +1 -0
  96. package/corpus/templates/analytics/changelog/2026-07-11-analytics-no-longer-repeatedly-restarts-background-requests-.md +6 -0
  97. package/corpus/templates/analytics/changelog/2026-07-11-analytics-only-asks-for-replay-storage-when-you-enable-sessi.md +6 -0
  98. package/corpus/templates/analytics/changelog/2026-07-11-charts-now-show-their-loading-skeleton-while-refreshing.md +6 -0
  99. package/corpus/templates/analytics/changelog/2026-07-11-realtime-voice-now-shows-a-clear-connection-indicator-comple.md +6 -0
  100. package/corpus/templates/analytics/changelog/2026-07-11-replay-event-counts-stay-hidden-until-recordings-finish-load.md +6 -0
  101. package/corpus/templates/analytics/changelog/2026-07-11-replay-storage-key-mismatches-show-actionable-setup-guidance.md +6 -0
  102. package/corpus/templates/analytics/changelog/2026-07-11-replay-timelines-group-continuous-scrolling-and-dev-tools-co.md +6 -0
  103. package/corpus/templates/analytics/changelog/2026-07-11-session-filters-now-stay-compact-in-a-settings-menu-at-mediu.md +6 -0
  104. package/corpus/templates/analytics/changelog/2026-07-11-session-replay-console-errors-now-link-to-matching-issues-or.md +6 -0
  105. package/corpus/templates/analytics/changelog/2026-07-11-session-replays-load-large-recordings-faster-and-surface-fai.md +6 -0
  106. package/corpus/templates/analytics/changelog/2026-07-11-session-replays-preserve-recorded-styles-and-viewport-layout.md +6 -0
  107. package/corpus/templates/analytics/changelog/2026-07-11-session-replays-stay-centered-instead-of-clipping-wide-recor.md +6 -0
  108. package/corpus/templates/analytics/changelog/2026-07-12-daily-dashboard-emails-reliably-include-the-complete-dashboa.md +6 -0
  109. package/corpus/templates/analytics/changelog/2026-07-12-demo-mode-charts-trend-upward.md +6 -0
  110. package/corpus/templates/analytics/package.json +1 -1
  111. package/corpus/templates/analytics/seeds/dashboards/agent-native-templates-first-party.json +1 -303
  112. package/corpus/templates/analytics/server/handlers/session-replay.ts +57 -0
  113. package/corpus/templates/analytics/server/jobs/dashboard-report.ts +2 -0
  114. package/corpus/templates/analytics/server/lib/dashboard-catalog.ts +1 -1
  115. package/corpus/templates/analytics/server/lib/dashboard-report.ts +17 -33
  116. package/corpus/templates/analytics/server/lib/demo-dashboards.ts +16 -3
  117. package/corpus/templates/analytics/server/lib/session-replay-agent-context.ts +12 -0
  118. package/corpus/templates/analytics/server/lib/session-replay.ts +208 -2
  119. package/corpus/templates/analytics/server/plugins/onboarding.ts +7 -79
  120. package/corpus/templates/analytics/server/routes/api/session-replay/recordings/[recordingId]/chunks.get.ts +1 -0
  121. package/corpus/templates/assets/.agents/skills/real-time-sync/SKILL.md +2 -2
  122. package/corpus/templates/assets/app/global.css +4 -4
  123. package/corpus/templates/assets/app/routes/brand-kits.$id.tsx +0 -1
  124. package/corpus/templates/assets/app/routes/library.tsx +0 -1
  125. package/corpus/templates/brain/.agents/skills/real-time-sync/SKILL.md +2 -2
  126. package/corpus/templates/brain/app/global.css +5 -5
  127. package/corpus/templates/brain/app/root.tsx +0 -3
  128. package/corpus/templates/calendar/.agents/skills/real-time-sync/SKILL.md +2 -2
  129. package/corpus/templates/calendar/app/root.tsx +1 -0
  130. package/corpus/templates/chat/.agents/skills/real-time-sync/SKILL.md +2 -2
  131. package/corpus/templates/clips/app/components/player/video-player.tsx +5 -1
  132. package/corpus/templates/clips/app/hooks/use-auto-title.ts +74 -8
  133. package/corpus/templates/clips/app/hooks/use-library.ts +12 -25
  134. package/corpus/templates/clips/app/routes/_app.settings._index.tsx +20 -22
  135. package/corpus/templates/clips/changelog/2026-07-11-clips-no-longer-stay-stuck-on-preparing-clip-when-they-are-r.md +6 -0
  136. package/corpus/templates/clips/changelog/2026-07-11-generated-titles-no-longer-cause-continuous-background-refreshes.md +6 -0
  137. package/corpus/templates/clips/changelog/2026-07-11-s3-storage-setup-stays-collapsed-until-you-choose-it-as-the-.md +6 -0
  138. package/corpus/templates/content/.agents/skills/real-time-sync/SKILL.md +2 -2
  139. package/corpus/templates/content/app/components/sidebar/DocumentSidebar.tsx +2 -2
  140. package/corpus/templates/content/app/hooks/use-comments.ts +0 -1
  141. package/corpus/templates/content/changelog/2026-07-11-comments-now-update-from-live-collaboration-events-without-c.md +6 -0
  142. package/corpus/templates/content/changelog/2026-07-12-the-left-sidebar-stays-fixed-horizontally-while-scrolling-on.md +6 -0
  143. package/corpus/templates/design/app/components/layout/Header.tsx +1 -1
  144. package/corpus/templates/forms/.agents/skills/real-time-sync/SKILL.md +2 -2
  145. package/corpus/templates/macros/app/components/layout/AppLayout.tsx +1 -2
  146. package/corpus/templates/macros/app/root.tsx +0 -3
  147. package/corpus/templates/mail/.agents/skills/real-time-sync/SKILL.md +2 -2
  148. package/corpus/templates/mail/app/hooks/use-compose-state.ts +1 -0
  149. package/corpus/templates/mail/app/root.tsx +1 -12
  150. package/corpus/templates/plan/.agents/skills/real-time-sync/SKILL.md +2 -2
  151. package/corpus/templates/plan/.agents/skills/visual-plan/references/local-files.md +17 -12
  152. package/corpus/templates/plan/.agents/skills/visual-recap/references/local-files.md +17 -12
  153. package/corpus/templates/plan/actions/validate-local-plan-source.ts +4 -4
  154. package/corpus/templates/plan/app/components/layout/Sidebar.tsx +2 -1
  155. package/corpus/templates/plan/app/hooks/use-navigation-state.ts +5 -2
  156. package/corpus/templates/plan/app/hooks/use-plans.ts +6 -13
  157. package/corpus/templates/plan/app/lib/plan-local-bridge.ts +41 -0
  158. package/corpus/templates/plan/app/lib/plan-tracking.ts +3 -0
  159. package/corpus/templates/plan/app/pages/PlansPage.tsx +14 -36
  160. package/corpus/templates/plan/app/root.tsx +13 -3
  161. package/corpus/templates/plan/changelog/2026-07-11-plan-refreshes-only-when-content-changes.md +6 -0
  162. package/corpus/templates/plan/changelog/2026-07-12-local-plan-source-now-stays-on-device.md +6 -0
  163. package/corpus/templates/slides/app/components/comments/SlideCommentsPanel.tsx +4 -1
  164. package/corpus/templates/slides/app/components/editor/EditorToolbar.tsx +1 -1
  165. package/corpus/templates/slides/app/components/layout/Header.tsx +1 -1
  166. package/corpus/templates/slides/app/context/DeckContext.tsx +1 -0
  167. package/corpus/templates/slides/app/hooks/use-slide-comments.ts +7 -31
  168. package/corpus/templates/slides/changelog/2026-07-11-comments-now-update-from-live-collaboration-events-without-c.md +6 -0
  169. package/dist/action-change-marker.d.ts +1 -0
  170. package/dist/action-change-marker.d.ts.map +1 -1
  171. package/dist/action-change-marker.js +7 -1
  172. package/dist/action-change-marker.js.map +1 -1
  173. package/dist/changelog/parse.d.ts +4 -2
  174. package/dist/changelog/parse.d.ts.map +1 -1
  175. package/dist/changelog/parse.js +5 -3
  176. package/dist/changelog/parse.js.map +1 -1
  177. package/dist/cli/plan-local.d.ts +6 -6
  178. package/dist/cli/plan-local.d.ts.map +1 -1
  179. package/dist/cli/plan-local.js +36 -7
  180. package/dist/cli/plan-local.js.map +1 -1
  181. package/dist/cli/pr-visual-recap-workflow.d.ts +2 -8
  182. package/dist/cli/pr-visual-recap-workflow.d.ts.map +1 -1
  183. package/dist/cli/pr-visual-recap-workflow.js +2 -8
  184. package/dist/cli/pr-visual-recap-workflow.js.map +1 -1
  185. package/dist/cli/recap.d.ts +22 -1
  186. package/dist/cli/recap.d.ts.map +1 -1
  187. package/dist/cli/recap.js +286 -2
  188. package/dist/cli/recap.js.map +1 -1
  189. package/dist/cli/skills-content/local-files.d.ts +1 -1
  190. package/dist/cli/skills-content/local-files.d.ts.map +1 -1
  191. package/dist/cli/skills-content/local-files.js +17 -12
  192. package/dist/cli/skills-content/local-files.js.map +1 -1
  193. package/dist/client/AgentPanel.d.ts.map +1 -1
  194. package/dist/client/AgentPanel.js +2 -3
  195. package/dist/client/AgentPanel.js.map +1 -1
  196. package/dist/client/AssistantChat.d.ts +7 -0
  197. package/dist/client/AssistantChat.d.ts.map +1 -1
  198. package/dist/client/AssistantChat.js +62 -40
  199. package/dist/client/AssistantChat.js.map +1 -1
  200. package/dist/client/analytics.d.ts +8 -0
  201. package/dist/client/analytics.d.ts.map +1 -1
  202. package/dist/client/analytics.js +103 -13
  203. package/dist/client/analytics.js.map +1 -1
  204. package/dist/client/blocks/library/wireframe-kit.d.ts +2 -0
  205. package/dist/client/blocks/library/wireframe-kit.d.ts.map +1 -1
  206. package/dist/client/blocks/library/wireframe-kit.js +9 -2
  207. package/dist/client/blocks/library/wireframe-kit.js.map +1 -1
  208. package/dist/client/chat-model-groups.d.ts.map +1 -1
  209. package/dist/client/chat-model-groups.js +24 -0
  210. package/dist/client/chat-model-groups.js.map +1 -1
  211. package/dist/client/composer/RealtimeVoiceMode.d.ts +40 -4
  212. package/dist/client/composer/RealtimeVoiceMode.d.ts.map +1 -1
  213. package/dist/client/composer/RealtimeVoiceMode.js +172 -42
  214. package/dist/client/composer/RealtimeVoiceMode.js.map +1 -1
  215. package/dist/client/composer/TiptapComposer.d.ts +4 -0
  216. package/dist/client/composer/TiptapComposer.d.ts.map +1 -1
  217. package/dist/client/composer/TiptapComposer.js +47 -29
  218. package/dist/client/composer/TiptapComposer.js.map +1 -1
  219. package/dist/client/composer/VoiceButton.d.ts.map +1 -1
  220. package/dist/client/composer/VoiceButton.js +2 -2
  221. package/dist/client/composer/VoiceButton.js.map +1 -1
  222. package/dist/client/composer/index.d.ts +2 -2
  223. package/dist/client/composer/index.d.ts.map +1 -1
  224. package/dist/client/composer/index.js +1 -1
  225. package/dist/client/composer/index.js.map +1 -1
  226. package/dist/client/composer/useRealtimeVoiceMode.d.ts +100 -3
  227. package/dist/client/composer/useRealtimeVoiceMode.d.ts.map +1 -1
  228. package/dist/client/composer/useRealtimeVoiceMode.js +952 -51
  229. package/dist/client/composer/useRealtimeVoiceMode.js.map +1 -1
  230. package/dist/client/index.d.ts +3 -2
  231. package/dist/client/index.d.ts.map +1 -1
  232. package/dist/client/index.js +3 -2
  233. package/dist/client/index.js.map +1 -1
  234. package/dist/client/progress/RunsTray.d.ts.map +1 -1
  235. package/dist/client/progress/RunsTray.js +10 -1
  236. package/dist/client/progress/RunsTray.js.map +1 -1
  237. package/dist/client/session-replay.d.ts +2 -0
  238. package/dist/client/session-replay.d.ts.map +1 -1
  239. package/dist/client/session-replay.js +7 -0
  240. package/dist/client/session-replay.js.map +1 -1
  241. package/dist/client/settings/DemoModeSection.d.ts.map +1 -1
  242. package/dist/client/settings/DemoModeSection.js +4 -23
  243. package/dist/client/settings/DemoModeSection.js.map +1 -1
  244. package/dist/client/settings/SecretsSection.d.ts +2 -3
  245. package/dist/client/settings/SecretsSection.d.ts.map +1 -1
  246. package/dist/client/settings/SecretsSection.js +46 -22
  247. package/dist/client/settings/SecretsSection.js.map +1 -1
  248. package/dist/client/settings/SettingsPanel.d.ts +9 -0
  249. package/dist/client/settings/SettingsPanel.d.ts.map +1 -1
  250. package/dist/client/settings/SettingsPanel.js +46 -12
  251. package/dist/client/settings/SettingsPanel.js.map +1 -1
  252. package/dist/client/url-scrub.d.ts.map +1 -1
  253. package/dist/client/url-scrub.js +14 -0
  254. package/dist/client/url-scrub.js.map +1 -1
  255. package/dist/client/use-action.d.ts +2 -0
  256. package/dist/client/use-action.d.ts.map +1 -1
  257. package/dist/client/use-action.js +10 -0
  258. package/dist/client/use-action.js.map +1 -1
  259. package/dist/client/use-agent-chat-home-handoff.d.ts +2 -0
  260. package/dist/client/use-agent-chat-home-handoff.d.ts.map +1 -1
  261. package/dist/client/use-agent-chat-home-handoff.js +7 -1
  262. package/dist/client/use-agent-chat-home-handoff.js.map +1 -1
  263. package/dist/client/use-agent-engine-configured.d.ts.map +1 -1
  264. package/dist/client/use-agent-engine-configured.js +17 -3
  265. package/dist/client/use-agent-engine-configured.js.map +1 -1
  266. package/dist/client/use-db-sync.d.ts +8 -2
  267. package/dist/client/use-db-sync.d.ts.map +1 -1
  268. package/dist/client/use-db-sync.js +57 -32
  269. package/dist/client/use-db-sync.js.map +1 -1
  270. package/dist/client/use-demo-mode-status.d.ts +12 -0
  271. package/dist/client/use-demo-mode-status.d.ts.map +1 -0
  272. package/dist/client/use-demo-mode-status.js +30 -0
  273. package/dist/client/use-demo-mode-status.js.map +1 -0
  274. package/dist/client/use-pausing-interval.d.ts.map +1 -1
  275. package/dist/client/use-pausing-interval.js +16 -4
  276. package/dist/client/use-pausing-interval.js.map +1 -1
  277. package/dist/collab/struct-routes.d.ts +1 -1
  278. package/dist/integrations/scope-store.js +1 -1
  279. package/dist/integrations/scope-store.js.map +1 -1
  280. package/dist/integrations/usage-budget-store.js +1 -1
  281. package/dist/integrations/usage-budget-store.js.map +1 -1
  282. package/dist/localization/default-messages.d.ts +41 -1
  283. package/dist/localization/default-messages.d.ts.map +1 -1
  284. package/dist/localization/default-messages.js +41 -1
  285. package/dist/localization/default-messages.js.map +1 -1
  286. package/dist/mcp-client/workspace-servers.js +4 -4
  287. package/dist/mcp-client/workspace-servers.js.map +1 -1
  288. package/dist/notifications/routes.d.ts +3 -3
  289. package/dist/progress/routes.d.ts +1 -1
  290. package/dist/provider-api/corpus-jobs.d.ts +2 -2
  291. package/dist/resources/handlers.d.ts +3 -3
  292. package/dist/secrets/crypto.d.ts +7 -6
  293. package/dist/secrets/crypto.d.ts.map +1 -1
  294. package/dist/secrets/crypto.js +30 -11
  295. package/dist/secrets/crypto.js.map +1 -1
  296. package/dist/server/action-change.d.ts +1 -0
  297. package/dist/server/action-change.d.ts.map +1 -1
  298. package/dist/server/action-change.js +4 -1
  299. package/dist/server/action-change.js.map +1 -1
  300. package/dist/server/action-routes.d.ts.map +1 -1
  301. package/dist/server/action-routes.js +5 -0
  302. package/dist/server/action-routes.js.map +1 -1
  303. package/dist/server/agent-engine-api-key-route.d.ts +1 -1
  304. package/dist/server/poll.d.ts.map +1 -1
  305. package/dist/server/poll.js +1 -0
  306. package/dist/server/poll.js.map +1 -1
  307. package/dist/server/realtime-voice.d.ts +7 -0
  308. package/dist/server/realtime-voice.d.ts.map +1 -1
  309. package/dist/server/realtime-voice.js +222 -12
  310. package/dist/server/realtime-voice.js.map +1 -1
  311. package/dist/styles/agent-native.css +2 -2
  312. package/dist/templates/default/.agents/skills/real-time-sync/SKILL.md +2 -2
  313. package/dist/templates/workspace-core/.agents/skills/real-time-sync/SKILL.md +2 -2
  314. package/dist/templates/workspace-core/.agents/skills/secrets/SKILL.md +7 -3
  315. package/dist/templates/workspace-core/.agents/skills/voice-transcription/SKILL.md +39 -9
  316. package/dist/vite/client.d.ts.map +1 -1
  317. package/dist/vite/client.js +6 -1
  318. package/dist/vite/client.js.map +1 -1
  319. package/docs/content/locales/ar-SA/pr-visual-recap.mdx +25 -1
  320. package/docs/content/locales/ar-SA/template-plan.mdx +1 -1
  321. package/docs/content/locales/de-DE/pr-visual-recap.mdx +25 -1
  322. package/docs/content/locales/de-DE/template-plan.mdx +1 -1
  323. package/docs/content/locales/es-ES/pr-visual-recap.mdx +25 -1
  324. package/docs/content/locales/es-ES/template-plan.mdx +1 -1
  325. package/docs/content/locales/fr-FR/pr-visual-recap.mdx +25 -1
  326. package/docs/content/locales/fr-FR/template-plan.mdx +1 -1
  327. package/docs/content/locales/hi-IN/pr-visual-recap.mdx +25 -1
  328. package/docs/content/locales/hi-IN/template-plan.mdx +1 -1
  329. package/docs/content/locales/ja-JP/pr-visual-recap.mdx +25 -1
  330. package/docs/content/locales/ja-JP/template-plan.mdx +1 -1
  331. package/docs/content/locales/ko-KR/pr-visual-recap.mdx +25 -1
  332. package/docs/content/locales/ko-KR/template-plan.mdx +1 -1
  333. package/docs/content/locales/pt-BR/pr-visual-recap.mdx +25 -1
  334. package/docs/content/locales/pt-BR/template-plan.mdx +1 -1
  335. package/docs/content/locales/zh-CN/pr-visual-recap.mdx +25 -1
  336. package/docs/content/locales/zh-CN/template-plan.mdx +1 -1
  337. package/docs/content/locales/zh-TW/pr-visual-recap.mdx +25 -1
  338. package/docs/content/locales/zh-TW/template-plan.mdx +1 -1
  339. package/docs/content/pr-visual-recap.mdx +50 -1
  340. package/docs/content/template-plan.mdx +1 -1
  341. package/package.json +2 -2
  342. package/src/templates/default/.agents/skills/real-time-sync/SKILL.md +2 -2
  343. package/src/templates/workspace-core/.agents/skills/real-time-sync/SKILL.md +2 -2
  344. package/src/templates/workspace-core/.agents/skills/secrets/SKILL.md +7 -3
  345. package/src/templates/workspace-core/.agents/skills/voice-transcription/SKILL.md +39 -9
@@ -211,6 +211,7 @@ const messages = {
211
211
  cliRequiresDevModeDescription:
212
212
  "Run this app locally with pnpm dev or use Builder.io to access the CLI terminal.",
213
213
  toggleAgent: "Toggle agent",
214
+ checkingAiConnection: "Checking AI connection...",
214
215
  voiceMode: {
215
216
  entryButtonLabel: "Use microphone",
216
217
  promptTitle: "Talk to your app",
@@ -227,7 +228,47 @@ const messages = {
227
228
  showChat: "Show chat",
228
229
  hideChat: "Hide chat",
229
230
  end: "End voice mode",
230
- microphoneSettings: "Microphone settings",
231
+ voiceSettings: "Voice settings",
232
+ settings: {
233
+ microphone: "Microphone",
234
+ defaultMicrophone: "System default",
235
+ microphoneSwitchFailed:
236
+ "Could not switch microphones. Your current microphone is still active.",
237
+ language: "Language",
238
+ autoLanguage: "Auto",
239
+ languages: {
240
+ en: "English",
241
+ es: "Spanish",
242
+ fr: "French",
243
+ de: "German",
244
+ it: "Italian",
245
+ pt: "Portuguese",
246
+ ja: "Japanese",
247
+ ko: "Korean",
248
+ zh: "Chinese",
249
+ },
250
+ intelligence: "Intelligence",
251
+ intelligenceLevels: {
252
+ instant: "Instant",
253
+ balanced: "Balanced",
254
+ deep: "Deep",
255
+ },
256
+ voiceStyle: "Voice style",
257
+ voiceChangePending:
258
+ "Your new voice will apply next time you start voice mode.",
259
+ voiceDescriptions: {
260
+ marin: "Warm and natural",
261
+ cedar: "Clear and grounded",
262
+ coral: "Friendly and bright",
263
+ sage: "Calm and thoughtful",
264
+ verse: "Expressive and versatile",
265
+ alloy: "Balanced and neutral",
266
+ ash: "Smooth and confident",
267
+ ballad: "Warm and expressive",
268
+ echo: "Clear and direct",
269
+ shimmer: "Light and upbeat",
270
+ },
271
+ },
231
272
  status: {
232
273
  connecting: "Connecting",
233
274
  listening: "Listening",
@@ -242,6 +283,8 @@ const messages = {
242
283
  responseFailed: "OpenAI could not complete the voice response.",
243
284
  sessionFailed: "The realtime voice session encountered an error.",
244
285
  channelDisconnected: "The realtime voice control channel disconnected.",
286
+ connectionTimedOut:
287
+ "The realtime voice connection timed out. Try again after the app finishes loading.",
245
288
  connectionFailed: "The realtime voice connection failed.",
246
289
  offerFailed: "The browser did not create an audio offer.",
247
290
  },
@@ -265,8 +265,8 @@ async function selectWorkspaceMcpResourceRows(options?: {
265
265
  AND wr.scope = ?
266
266
  AND wr.path LIKE ?
267
267
  AND (
268
- (? IS NOT NULL AND wr.org_id = ?)
269
- OR (? IS NOT NULL AND wr.org_id IS NULL AND lower(wr.owner_email) = lower(?))
268
+ (CAST(? AS TEXT) IS NOT NULL AND wr.org_id = ?)
269
+ OR (CAST(? AS TEXT) IS NOT NULL AND wr.org_id IS NULL AND lower(wr.owner_email) = lower(?))
270
270
  )
271
271
  `,
272
272
  args: [
@@ -304,8 +304,8 @@ async function selectWorkspaceMcpResourceRows(options?: {
304
304
  AND wg.status = ?
305
305
  AND wg.app_id = ?
306
306
  AND (
307
- (? IS NOT NULL AND wr.org_id = ?)
308
- OR (? IS NOT NULL AND wr.org_id IS NULL AND lower(wr.owner_email) = lower(?))
307
+ (CAST(? AS TEXT) IS NOT NULL AND wr.org_id = ?)
308
+ OR (CAST(? AS TEXT) IS NOT NULL AND wr.org_id IS NULL AND lower(wr.owner_email) = lower(?))
309
309
  )
310
310
  AND (
311
311
  (wr.org_id IS NOT NULL AND wg.org_id = wr.org_id)
@@ -5,12 +5,13 @@
5
5
  * credentials (`resolveCredential` / `saveCredential`, stored in `settings`) so
6
6
  * there is a single crypto implementation and a single key story.
7
7
  *
8
- * The encryption key is derived from `SECRETS_ENCRYPTION_KEY` (preferred) or the
9
- * existing `BETTER_AUTH_SECRET` env var (fallback so templates don't need a
10
- * second secret during development). In production we refuse to start without
11
- * one of them a CWD-derived fallback would be effectively static (e.g.
12
- * `/var/task` on Lambda), so anyone with read access to the DB could decrypt
13
- * every secret.
8
+ * The encryption key is derived from `<APP_NAME>_SECRETS_ENCRYPTION_KEY` when
9
+ * set, then `SECRETS_ENCRYPTION_KEY`, then `BETTER_AUTH_SECRET`. The app-scoped
10
+ * key lets a local multi-app workspace read one app's encrypted production data
11
+ * without replacing the shared local auth secret. In production we refuse to
12
+ * start without configured key material a CWD-derived fallback would be
13
+ * effectively static (e.g. `/var/task` on Lambda), so anyone with read access
14
+ * to the DB could decrypt every secret.
14
15
  *
15
16
  * Encrypted values are tagged `v1:<iv-hex>:<ct-hex>:<tag-hex>`. The `v1:` prefix
16
17
  * lets readers distinguish ciphertext from legacy plaintext during migration.
@@ -55,6 +56,17 @@ function processCwd(): string {
55
56
  return process.cwd();
56
57
  }
57
58
 
59
+ function appScopedEncryptionKey(): string | undefined {
60
+ if (typeof process === "undefined") return undefined;
61
+ const appName = process.env.APP_NAME?.trim() // guard:allow-env-credential — deploy-level app configuration selects the scoped encryption key.
62
+ .toUpperCase()
63
+ .replace(/[^A-Z0-9]+/g, "_")
64
+ .replace(/^_+|_+$/g, "");
65
+ return appName
66
+ ? process.env[`${appName}_SECRETS_ENCRYPTION_KEY`] // guard:allow-env-credential — deploy-level app encryption material, never a user credential.
67
+ : undefined;
68
+ }
69
+
58
70
  /**
59
71
  * Derive a 32-byte AES key from the configured secret material via SHA-256.
60
72
  * Re-derived per call (cheap, stateless, and makes rotation easy).
@@ -62,6 +74,7 @@ function processCwd(): string {
62
74
  export function getSecretEncryptionKey(): Buffer {
63
75
  const { createHash } = requireNodeCrypto();
64
76
  const explicit =
77
+ appScopedEncryptionKey() ||
65
78
  (typeof process === "undefined"
66
79
  ? undefined
67
80
  : process.env.SECRETS_ENCRYPTION_KEY) ||
@@ -71,9 +84,16 @@ export function getSecretEncryptionKey(): Buffer {
71
84
 
72
85
  if (!explicit) {
73
86
  if (processNodeEnv() === "production") {
87
+ const appName =
88
+ typeof process === "undefined"
89
+ ? undefined
90
+ : process.env.APP_NAME?.trim() // guard:allow-env-credential — deploy-level app configuration selects the scoped encryption key.
91
+ .toUpperCase()
92
+ .replace(/[^A-Z0-9]+/g, "_")
93
+ .replace(/^_+|_+$/g, "");
74
94
  throw new Error(
75
95
  "[agent-native/secrets] Refusing to start in production without an encryption key. " +
76
- "Set SECRETS_ENCRYPTION_KEY (preferred) or BETTER_AUTH_SECRET in the deploy environment. " +
96
+ `Set ${appName ? `${appName}_SECRETS_ENCRYPTION_KEY, ` : ""}SECRETS_ENCRYPTION_KEY, or BETTER_AUTH_SECRET in the deploy environment. ` +
77
97
  "The previous CWD-derived fallback was effectively static (e.g. `/var/task` on Lambda), " +
78
98
  "which means anyone with read access to the secrets table could decrypt every user's secrets.",
79
99
  );
@@ -83,7 +103,7 @@ export function getSecretEncryptionKey(): Buffer {
83
103
  // eslint-disable-next-line no-console
84
104
  console.warn(
85
105
  "[agent-native/secrets] SECRETS_ENCRYPTION_KEY not set — using a machine-local fallback. " +
86
- "Set SECRETS_ENCRYPTION_KEY (or BETTER_AUTH_SECRET) for production. " +
106
+ "Set an app-scoped *_SECRETS_ENCRYPTION_KEY, SECRETS_ENCRYPTION_KEY, or BETTER_AUTH_SECRET for production. " +
87
107
  "Production deploys without one of these env vars now hard-fail.",
88
108
  );
89
109
  }
@@ -12,6 +12,7 @@ export interface NotifyActionChangeOptions {
12
12
  actionName: string;
13
13
  owner?: string;
14
14
  orgId?: string;
15
+ requestSource?: string;
15
16
  }
16
17
 
17
18
  function actionChangeTarget(
@@ -22,6 +23,7 @@ function actionChangeTarget(
22
23
  actionName: options.actionName,
23
24
  owner,
24
25
  orgId: owner ? undefined : (options.orgId ?? getRequestOrgId()),
26
+ requestSource: options.requestSource,
25
27
  };
26
28
  }
27
29
 
@@ -38,7 +40,7 @@ export async function writeActionChangeMarker(
38
40
  ...actionChangeMarkerValue(target),
39
41
  nonce: `${Date.now()}-${Math.random().toString(36).slice(2)}`,
40
42
  },
41
- { requestSource: "agent" },
43
+ { requestSource: options.requestSource ?? "agent" },
42
44
  );
43
45
  }
44
46
 
@@ -52,11 +54,13 @@ export async function notifyActionChange(
52
54
  key: options.actionName,
53
55
  ...(target.owner ? { owner: target.owner } : {}),
54
56
  ...(target.orgId ? { orgId: target.orgId } : {}),
57
+ ...(options.requestSource ? { requestSource: options.requestSource } : {}),
55
58
  });
56
59
 
57
60
  await writeActionChangeMarker({
58
61
  actionName: options.actionName,
59
62
  ...(target.owner ? { owner: target.owner } : {}),
60
63
  ...(target.orgId ? { orgId: target.orgId } : {}),
64
+ ...(options.requestSource ? { requestSource: options.requestSource } : {}),
61
65
  });
62
66
  }
@@ -371,6 +371,14 @@ export function mountActionRoutes(
371
371
  await notifyActionChange({
372
372
  actionName: name,
373
373
  ...(userEmail ? { owner: userEmail } : {}),
374
+ ...(getHeader(event, "x-request-source")
375
+ ? {
376
+ requestSource: getHeader(
377
+ event,
378
+ "x-request-source",
379
+ ) as string,
380
+ }
381
+ : {}),
374
382
  });
375
383
  } catch {
376
384
  // ignore
@@ -604,6 +604,7 @@ function recordActionChanges(targets: ActionChangeTarget[]): void {
604
604
  key: target.actionName ?? "*",
605
605
  ...(target.owner ? { owner: target.owner } : {}),
606
606
  ...(target.orgId ? { orgId: target.orgId } : {}),
607
+ ...(target.requestSource ? { requestSource: target.requestSource } : {}),
607
608
  });
608
609
  }
609
610
  }
@@ -18,6 +18,8 @@ import {
18
18
  sanitizeToolErrorText,
19
19
  sanitizeToolErrorValue,
20
20
  } from "../agent/tool-error-redaction.js";
21
+ import { TOOL_SEARCH_ACTION_NAME } from "../agent/tool-search.js";
22
+ import { parseAcceptLanguage } from "../localization/server.js";
21
23
  import { getSession } from "./auth.js";
22
24
  import {
23
25
  getBuilderGatewayBaseUrl,
@@ -37,6 +39,10 @@ export const REALTIME_VOICE_MAX_TOOL_OUTPUT_CHARS = 16_000;
37
39
  export const REALTIME_VOICE_MAX_TOOLS = 32;
38
40
  export const REALTIME_VOICE_MAX_TOOL_SCHEMA_BYTES = 32_000;
39
41
  export const REALTIME_VOICE_MAX_SESSION_BYTES = 64_000;
42
+ export const REALTIME_VOICE_TOOL_GRANT_TTL_MS = 10 * 60 * 1_000;
43
+ export const REALTIME_VOICE_MAX_TOOL_GRANT_SESSIONS = 256;
44
+ export const REALTIME_VOICE_CAPABILITY_HEADER =
45
+ "X-Agent-Native-Realtime-Capability";
40
46
 
41
47
  const OPENAI_REALTIME_CALLS_URL = "https://api.openai.com/v1/realtime/calls";
42
48
  const DEFAULT_MODEL = "gpt-realtime-2.1";
@@ -50,6 +56,37 @@ const REALTIME_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/;
50
56
  const CALL_ID = /^[A-Za-z0-9_-]{1,256}$/;
51
57
  const SESSION_ID = /^[A-Za-z0-9_.:-]{1,256}$/;
52
58
  const BROWSER_TAB_ID = /^[A-Za-z0-9_-]{1,96}$/;
59
+ const ISO_639_1_LANGUAGE = /^[A-Za-z]{2}$/;
60
+ const REALTIME_VOICE_BUILT_INS = new Set([
61
+ "alloy",
62
+ "ash",
63
+ "ballad",
64
+ "coral",
65
+ "echo",
66
+ "sage",
67
+ "shimmer",
68
+ "verse",
69
+ "marin",
70
+ "cedar",
71
+ ]);
72
+ const REALTIME_VOICE_REASONING_EFFORT = {
73
+ instant: "minimal",
74
+ balanced: "low",
75
+ deep: "medium",
76
+ } as const;
77
+
78
+ /**
79
+ * Realtime sessions have a deliberately bounded tool manifest. Keep the
80
+ * context/navigation tools ahead of large template registries so voice can
81
+ * always see and operate the same UI navigation surface as text chat.
82
+ */
83
+ const REALTIME_VOICE_PRIORITY_TOOLS = [
84
+ "navigate",
85
+ "set-url-path",
86
+ "set-search-params",
87
+ "view-screen",
88
+ TOOL_SEARCH_ACTION_NAME,
89
+ ] as const;
53
90
 
54
91
  export interface RealtimeVoiceRequestContext {
55
92
  event: H3Event;
@@ -105,6 +142,17 @@ interface RealtimeFunctionTool {
105
142
  parameters: Record<string, unknown>;
106
143
  }
107
144
 
145
+ interface RealtimeToolCapability {
146
+ userEmail: string;
147
+ orgId?: string;
148
+ browserTabId?: string;
149
+ expiresAt: number;
150
+ initialNames: Set<string>;
151
+ names: Set<string>;
152
+ }
153
+
154
+ type RealtimeToolCapabilityStore = Map<string, RealtimeToolCapability>;
155
+
108
156
  interface AuthenticatedVoiceContext extends RealtimeVoiceRequestContext {
109
157
  timezone?: string;
110
158
  }
@@ -126,6 +174,50 @@ function configuredIdentifier(
126
174
  : fallback;
127
175
  }
128
176
 
177
+ export function resolveRealtimeVoiceTranscriptionLanguage(
178
+ acceptLanguage: string | null | undefined,
179
+ ): string {
180
+ for (const locale of parseAcceptLanguage(acceptLanguage)) {
181
+ const primaryLanguage = locale.split("-")[0];
182
+ if (ISO_639_1_LANGUAGE.test(primaryLanguage)) {
183
+ return primaryLanguage.toLowerCase();
184
+ }
185
+ }
186
+ return "en";
187
+ }
188
+
189
+ export function resolveRealtimeVoiceLanguagePreference(
190
+ value: string | null | undefined,
191
+ acceptLanguage: string | null | undefined,
192
+ ): string {
193
+ const normalized = value?.trim().toLowerCase();
194
+ return normalized && ISO_639_1_LANGUAGE.test(normalized)
195
+ ? normalized
196
+ : resolveRealtimeVoiceTranscriptionLanguage(acceptLanguage);
197
+ }
198
+
199
+ export function resolveRealtimeVoiceReasoningEffort(
200
+ value: string | null | undefined,
201
+ ): "minimal" | "low" | "medium" {
202
+ const normalized = value?.trim().toLowerCase();
203
+ return normalized &&
204
+ Object.hasOwn(REALTIME_VOICE_REASONING_EFFORT, normalized)
205
+ ? REALTIME_VOICE_REASONING_EFFORT[
206
+ normalized as keyof typeof REALTIME_VOICE_REASONING_EFFORT
207
+ ]
208
+ : "low";
209
+ }
210
+
211
+ export function resolveRealtimeVoicePreference(
212
+ value: string | null | undefined,
213
+ fallback: string,
214
+ ): string {
215
+ const normalized = value?.trim().toLowerCase();
216
+ return normalized && REALTIME_VOICE_BUILT_INS.has(normalized)
217
+ ? normalized
218
+ : fallback;
219
+ }
220
+
129
221
  function truncate(value: string, maxChars: number): string {
130
222
  if (value.length <= maxChars) return value;
131
223
  return `${value.slice(0, maxChars)}\n\n...[truncated]`;
@@ -218,7 +310,22 @@ function buildRealtimeTools(
218
310
  parameters,
219
311
  });
220
312
  }
221
- return tools;
313
+ const priority = new Map<string, number>(
314
+ REALTIME_VOICE_PRIORITY_TOOLS.map((name, index) => [name, index]),
315
+ );
316
+ return tools
317
+ .map((tool, index) => ({ tool, index }))
318
+ .sort((left, right) => {
319
+ const leftPriority = priority.get(left.tool.name);
320
+ const rightPriority = priority.get(right.tool.name);
321
+ if (leftPriority !== undefined || rightPriority !== undefined) {
322
+ if (leftPriority === undefined) return 1;
323
+ if (rightPriority === undefined) return -1;
324
+ return leftPriority - rightPriority;
325
+ }
326
+ return left.index - right.index;
327
+ })
328
+ .map(({ tool }) => tool);
222
329
  }
223
330
 
224
331
  function packRealtimeTools(
@@ -243,6 +350,124 @@ function packRealtimeTools(
243
350
  return packed;
244
351
  }
245
352
 
353
+ function mintRealtimeToolCapability(): string {
354
+ return globalThis.crypto.randomUUID().replaceAll("-", "");
355
+ }
356
+
357
+ function cleanRealtimeToolCapabilities(
358
+ capabilities: RealtimeToolCapabilityStore,
359
+ now = Date.now(),
360
+ ): void {
361
+ for (const [key, capability] of capabilities) {
362
+ if (capability.expiresAt <= now) capabilities.delete(key);
363
+ }
364
+ while (capabilities.size > REALTIME_VOICE_MAX_TOOL_GRANT_SESSIONS) {
365
+ let oldestKey: string | undefined;
366
+ let oldestExpiry = Number.POSITIVE_INFINITY;
367
+ for (const [key, capability] of capabilities) {
368
+ if (capability.expiresAt < oldestExpiry) {
369
+ oldestKey = key;
370
+ oldestExpiry = capability.expiresAt;
371
+ }
372
+ }
373
+ if (!oldestKey) break;
374
+ capabilities.delete(oldestKey);
375
+ }
376
+ }
377
+
378
+ function registerRealtimeToolCapability(
379
+ capabilities: RealtimeToolCapabilityStore,
380
+ auth: AuthenticatedVoiceContext,
381
+ initialNames: Iterable<string>,
382
+ ): string {
383
+ cleanRealtimeToolCapabilities(capabilities);
384
+ const token = mintRealtimeToolCapability();
385
+ capabilities.set(token, {
386
+ userEmail: auth.userEmail.trim().toLowerCase(),
387
+ ...(auth.orgId ? { orgId: auth.orgId } : {}),
388
+ ...(auth.browserTabId ? { browserTabId: auth.browserTabId } : {}),
389
+ expiresAt: Date.now() + REALTIME_VOICE_TOOL_GRANT_TTL_MS,
390
+ initialNames: new Set(initialNames),
391
+ names: new Set(),
392
+ });
393
+ cleanRealtimeToolCapabilities(capabilities);
394
+ return token;
395
+ }
396
+
397
+ function resolveRealtimeToolCapability(
398
+ capabilities: RealtimeToolCapabilityStore,
399
+ token: string | undefined,
400
+ auth: AuthenticatedVoiceContext,
401
+ ): RealtimeToolCapability | null {
402
+ cleanRealtimeToolCapabilities(capabilities);
403
+ if (!token) return null;
404
+ const capability = capabilities.get(token);
405
+ if (!capability) return null;
406
+ if (
407
+ capability.userEmail !== auth.userEmail.trim().toLowerCase() ||
408
+ capability.orgId !== auth.orgId ||
409
+ capability.browserTabId !== auth.browserTabId
410
+ ) {
411
+ return null;
412
+ }
413
+ capability.expiresAt = Date.now() + REALTIME_VOICE_TOOL_GRANT_TTL_MS;
414
+ return capability;
415
+ }
416
+
417
+ function parseSuccessfulToolSearchNames(output: string): string[] {
418
+ try {
419
+ const parsed = JSON.parse(output) as { results?: unknown };
420
+ if (!Array.isArray(parsed.results)) return [];
421
+ const names: string[] = [];
422
+ for (const result of parsed.results) {
423
+ if (!isRecord(result) || typeof result.name !== "string") continue;
424
+ if (!REALTIME_TOOL_NAME.test(result.name)) continue;
425
+ if (!names.includes(result.name)) names.push(result.name);
426
+ if (names.length >= REALTIME_VOICE_MAX_TOOLS) break;
427
+ }
428
+ return names;
429
+ } catch {
430
+ return [];
431
+ }
432
+ }
433
+
434
+ function grantDiscoveredRealtimeTools(input: {
435
+ request: NonNullable<ReturnType<typeof parseToolRequest>>;
436
+ result: RealtimeVoiceToolExecutionResult;
437
+ toolsByName: ReadonlyMap<string, RealtimeFunctionTool>;
438
+ initialAllowedNames: ReadonlySet<string>;
439
+ capability: RealtimeToolCapability;
440
+ }): RealtimeFunctionTool[] {
441
+ const query = input.request.args.query;
442
+ if (
443
+ input.request.name !== TOOL_SEARCH_ACTION_NAME ||
444
+ input.result.status !== "completed" ||
445
+ typeof query !== "string" ||
446
+ !query.trim()
447
+ ) {
448
+ return [];
449
+ }
450
+
451
+ const candidates = parseSuccessfulToolSearchNames(input.result.output)
452
+ .filter((name) => !input.initialAllowedNames.has(name))
453
+ .map((name) => input.toolsByName.get(name))
454
+ .filter((tool): tool is RealtimeFunctionTool => Boolean(tool));
455
+ const boundedTools = packRealtimeTools({}, candidates);
456
+ const expandedTools: RealtimeFunctionTool[] = [];
457
+ for (const tool of boundedTools) {
458
+ if (
459
+ !input.capability.names.has(tool.name) &&
460
+ input.capability.names.size >= REALTIME_VOICE_MAX_TOOLS
461
+ ) {
462
+ continue;
463
+ }
464
+ input.capability.names.add(tool.name);
465
+ expandedTools.push(tool);
466
+ }
467
+ input.capability.expiresAt = Date.now() + REALTIME_VOICE_TOOL_GRANT_TTL_MS;
468
+ return expandedTools;
469
+ }
470
+
246
471
  function declaredBodyBytes(event: H3Event): number | undefined {
247
472
  const raw = readSafeHeader(event, "content-length");
248
473
  if (!raw) return undefined;
@@ -328,6 +553,7 @@ function invalidMethod(event: H3Event): { error: string } {
328
553
 
329
554
  function createSessionHandler(
330
555
  tools: RealtimeFunctionTool[],
556
+ capabilities: RealtimeToolCapabilityStore,
331
557
  options: MountRealtimeVoiceRoutesOptions,
332
558
  ) {
333
559
  return defineEventHandler(async (event: H3Event) => {
@@ -394,14 +620,30 @@ function createSessionHandler(
394
620
  }
395
621
 
396
622
  const instructions = await buildInstructions(auth, options);
623
+ const transcriptionLanguage = resolveRealtimeVoiceLanguagePreference(
624
+ readSafeHeader(event, "x-agent-native-realtime-language"),
625
+ readSafeHeader(event, "accept-language"),
626
+ );
627
+ const reasoningEffort = resolveRealtimeVoiceReasoningEffort(
628
+ readSafeHeader(event, "x-agent-native-realtime-intelligence"),
629
+ );
630
+ const voice = resolveRealtimeVoicePreference(
631
+ readSafeHeader(event, "x-agent-native-realtime-voice"),
632
+ configuredIdentifier(options.voice, DEFAULT_VOICE),
633
+ );
397
634
  const sessionBase = {
398
635
  type: "realtime",
399
636
  model: configuredIdentifier(options.model, DEFAULT_MODEL),
400
637
  instructions,
638
+ parallel_tool_calls: false,
639
+ reasoning: { effort: reasoningEffort },
401
640
  output_modalities: ["audio"],
402
641
  audio: {
403
642
  input: {
404
- transcription: { model: "gpt-4o-mini-transcribe" },
643
+ transcription: {
644
+ model: "gpt-4o-mini-transcribe",
645
+ language: transcriptionLanguage,
646
+ },
405
647
  turn_detection: {
406
648
  type: "semantic_vad",
407
649
  create_response: true,
@@ -410,13 +652,14 @@ function createSessionHandler(
410
652
  },
411
653
  },
412
654
  output: {
413
- voice: configuredIdentifier(options.voice, DEFAULT_VOICE),
655
+ voice,
414
656
  },
415
657
  },
416
658
  };
659
+ const packedTools = packRealtimeTools(sessionBase, tools);
417
660
  const session = {
418
661
  ...sessionBase,
419
- tools: packRealtimeTools(sessionBase, tools),
662
+ tools: packedTools,
420
663
  tool_choice: "auto",
421
664
  };
422
665
 
@@ -491,6 +734,15 @@ function createSessionHandler(
491
734
 
492
735
  setResponseStatus(event, upstream.status);
493
736
  setResponseHeader(event, "Content-Type", "application/sdp");
737
+ setResponseHeader(
738
+ event,
739
+ REALTIME_VOICE_CAPABILITY_HEADER,
740
+ registerRealtimeToolCapability(
741
+ capabilities,
742
+ auth,
743
+ packedTools.map((tool) => tool.name),
744
+ ),
745
+ );
494
746
  return answerSdp;
495
747
  },
496
748
  );
@@ -562,7 +814,8 @@ function normalizeExecutionResult(
562
814
  }
563
815
 
564
816
  function createToolHandler(
565
- allowedToolNames: ReadonlySet<string>,
817
+ toolsByName: ReadonlyMap<string, RealtimeFunctionTool>,
818
+ capabilities: RealtimeToolCapabilityStore,
566
819
  options: MountRealtimeVoiceRoutesOptions,
567
820
  ) {
568
821
  return defineEventHandler(async (event: H3Event) => {
@@ -578,6 +831,15 @@ function createToolHandler(
578
831
  setResponseStatus(event, 401);
579
832
  return { error: "Authentication required" };
580
833
  }
834
+ const capability = resolveRealtimeToolCapability(
835
+ capabilities,
836
+ readSafeHeader(event, REALTIME_VOICE_CAPABILITY_HEADER),
837
+ auth,
838
+ );
839
+ if (!capability) {
840
+ setResponseStatus(event, 403);
841
+ return { error: "Invalid or expired realtime voice capability" };
842
+ }
581
843
 
582
844
  const contentType = readSafeHeader(event, "content-type")?.toLowerCase();
583
845
  if (!contentType?.includes("application/json")) {
@@ -608,12 +870,22 @@ function createToolHandler(
608
870
  setResponseStatus(event, 400);
609
871
  return { error: "Invalid realtime tool request" };
610
872
  }
611
- if (!allowedToolNames.has(request.name)) {
873
+ if (
874
+ request.browserTabId !== undefined &&
875
+ request.browserTabId !== capability.browserTabId
876
+ ) {
877
+ setResponseStatus(event, 403);
878
+ return { error: "Realtime voice browser tab mismatch" };
879
+ }
880
+ if (
881
+ !capability.initialNames.has(request.name) &&
882
+ !capability.names.has(request.name)
883
+ ) {
612
884
  setResponseStatus(event, 404);
613
885
  return { error: "Unknown realtime voice tool" };
614
886
  }
615
887
 
616
- const browserTabId = request.browserTabId ?? auth.browserTabId;
888
+ const browserTabId = capability.browserTabId;
617
889
  const threadId = request.sessionId
618
890
  ? `realtime:${request.sessionId}`
619
891
  : `realtime:${request.callId}`;
@@ -635,9 +907,21 @@ function createToolHandler(
635
907
  userEmail: auth.userEmail,
636
908
  orgId: auth.orgId,
637
909
  ...request,
910
+ ...(browserTabId ? { browserTabId } : {}),
638
911
  }),
639
912
  );
640
- return { callId: request.callId, ...result };
913
+ const expandedTools = grantDiscoveredRealtimeTools({
914
+ request,
915
+ result,
916
+ toolsByName,
917
+ initialAllowedNames: capability.initialNames,
918
+ capability,
919
+ });
920
+ return {
921
+ callId: request.callId,
922
+ ...result,
923
+ ...(expandedTools.length > 0 ? { expandedTools } : {}),
924
+ };
641
925
  } catch (error) {
642
926
  setResponseStatus(event, 500);
643
927
  return {
@@ -665,14 +949,16 @@ export function mountRealtimeVoiceRoutes(
665
949
  }
666
950
 
667
951
  const tools = buildRealtimeTools(actions);
668
- const allowedToolNames = new Set(
669
- tools.slice(0, REALTIME_VOICE_MAX_TOOLS).map((tool) => tool.name),
670
- );
952
+ const toolsByName = new Map(tools.map((tool) => [tool.name, tool]));
953
+ const capabilities: RealtimeToolCapabilityStore = new Map();
671
954
  const app = getH3App(nitroApp);
672
- app.use(REALTIME_VOICE_SESSION_PATH, createSessionHandler(tools, options));
955
+ app.use(
956
+ REALTIME_VOICE_SESSION_PATH,
957
+ createSessionHandler(tools, capabilities, options),
958
+ );
673
959
  app.use(
674
960
  REALTIME_VOICE_TOOL_PATH,
675
- createToolHandler(allowedToolNames, options),
961
+ createToolHandler(toolsByName, capabilities, options),
676
962
  );
677
963
  return {
678
964
  sessionPath: REALTIME_VOICE_SESSION_PATH,
@@ -733,7 +733,7 @@
733
733
  }
734
734
 
735
735
  .agent-composer-root--hero {
736
- min-height: 7.5rem;
736
+ min-height: 5.625rem;
737
737
  border-radius: 1rem;
738
738
  border-color: hsl(var(--border) / 0.75);
739
739
  background: hsl(var(--muted) / 0.62);
@@ -751,7 +751,7 @@
751
751
  }
752
752
 
753
753
  .agent-composer-root--hero [data-agent-composer-slot="editor-input"] {
754
- min-height: 4rem;
754
+ min-height: 3rem;
755
755
  font-size: 1rem;
756
756
  line-height: 1.75rem;
757
757
  }