@memberjunction/ng-conversations 5.40.2 → 5.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (378) hide show
  1. package/README.md +57 -0
  2. package/dist/__tests__/channel-optional-surface.test.d.ts +2 -0
  3. package/dist/__tests__/channel-optional-surface.test.d.ts.map +1 -0
  4. package/dist/__tests__/channel-optional-surface.test.js +53 -0
  5. package/dist/__tests__/channel-optional-surface.test.js.map +1 -0
  6. package/dist/__tests__/chat-events.test.d.ts +14 -0
  7. package/dist/__tests__/chat-events.test.d.ts.map +1 -0
  8. package/dist/__tests__/chat-events.test.js +109 -0
  9. package/dist/__tests__/chat-events.test.js.map +1 -0
  10. package/dist/__tests__/conversation-naming.test.d.ts +2 -0
  11. package/dist/__tests__/conversation-naming.test.d.ts.map +1 -0
  12. package/dist/__tests__/conversation-naming.test.js +110 -0
  13. package/dist/__tests__/conversation-naming.test.js.map +1 -0
  14. package/dist/__tests__/delegation-result-parser.test.d.ts +2 -0
  15. package/dist/__tests__/delegation-result-parser.test.d.ts.map +1 -0
  16. package/dist/__tests__/delegation-result-parser.test.js +107 -0
  17. package/dist/__tests__/delegation-result-parser.test.js.map +1 -0
  18. package/dist/__tests__/event-wiring.test.d.ts +15 -0
  19. package/dist/__tests__/event-wiring.test.d.ts.map +1 -0
  20. package/dist/__tests__/event-wiring.test.js +100 -0
  21. package/dist/__tests__/event-wiring.test.js.map +1 -0
  22. package/dist/__tests__/narration-template.test.d.ts +2 -0
  23. package/dist/__tests__/narration-template.test.d.ts.map +1 -0
  24. package/dist/__tests__/narration-template.test.js +76 -0
  25. package/dist/__tests__/narration-template.test.js.map +1 -0
  26. package/dist/__tests__/realtime-agent-picker-models.test.d.ts +2 -0
  27. package/dist/__tests__/realtime-agent-picker-models.test.d.ts.map +1 -0
  28. package/dist/__tests__/realtime-agent-picker-models.test.js +49 -0
  29. package/dist/__tests__/realtime-agent-picker-models.test.js.map +1 -0
  30. package/dist/__tests__/realtime-audio-visuals.test.d.ts +2 -0
  31. package/dist/__tests__/realtime-audio-visuals.test.d.ts.map +1 -0
  32. package/dist/__tests__/realtime-audio-visuals.test.js +123 -0
  33. package/dist/__tests__/realtime-audio-visuals.test.js.map +1 -0
  34. package/dist/__tests__/realtime-delegation-card-cancel.test.d.ts +2 -0
  35. package/dist/__tests__/realtime-delegation-card-cancel.test.d.ts.map +1 -0
  36. package/dist/__tests__/realtime-delegation-card-cancel.test.js +48 -0
  37. package/dist/__tests__/realtime-delegation-card-cancel.test.js.map +1 -0
  38. package/dist/__tests__/realtime-disclosure.test.d.ts +2 -0
  39. package/dist/__tests__/realtime-disclosure.test.d.ts.map +1 -0
  40. package/dist/__tests__/realtime-disclosure.test.js +164 -0
  41. package/dist/__tests__/realtime-disclosure.test.js.map +1 -0
  42. package/dist/__tests__/realtime-pairing.test.d.ts +2 -0
  43. package/dist/__tests__/realtime-pairing.test.d.ts.map +1 -0
  44. package/dist/__tests__/realtime-pairing.test.js +207 -0
  45. package/dist/__tests__/realtime-pairing.test.js.map +1 -0
  46. package/dist/__tests__/realtime-review-lifecycle.test.d.ts +2 -0
  47. package/dist/__tests__/realtime-review-lifecycle.test.d.ts.map +1 -0
  48. package/dist/__tests__/realtime-review-lifecycle.test.js +154 -0
  49. package/dist/__tests__/realtime-review-lifecycle.test.js.map +1 -0
  50. package/dist/__tests__/realtime-session-cancel-usage.test.d.ts +2 -0
  51. package/dist/__tests__/realtime-session-cancel-usage.test.d.ts.map +1 -0
  52. package/dist/__tests__/realtime-session-cancel-usage.test.js +230 -0
  53. package/dist/__tests__/realtime-session-cancel-usage.test.js.map +1 -0
  54. package/dist/__tests__/realtime-session-channels.test.d.ts +2 -0
  55. package/dist/__tests__/realtime-session-channels.test.d.ts.map +1 -0
  56. package/dist/__tests__/realtime-session-channels.test.js +252 -0
  57. package/dist/__tests__/realtime-session-channels.test.js.map +1 -0
  58. package/dist/__tests__/realtime-session-client-tools.test.d.ts +2 -0
  59. package/dist/__tests__/realtime-session-client-tools.test.d.ts.map +1 -0
  60. package/dist/__tests__/realtime-session-client-tools.test.js +103 -0
  61. package/dist/__tests__/realtime-session-client-tools.test.js.map +1 -0
  62. package/dist/__tests__/realtime-session-minimized.test.d.ts +2 -0
  63. package/dist/__tests__/realtime-session-minimized.test.d.ts.map +1 -0
  64. package/dist/__tests__/realtime-session-minimized.test.js +32 -0
  65. package/dist/__tests__/realtime-session-minimized.test.js.map +1 -0
  66. package/dist/__tests__/realtime-session-mint.test.d.ts +2 -0
  67. package/dist/__tests__/realtime-session-mint.test.d.ts.map +1 -0
  68. package/dist/__tests__/realtime-session-mint.test.js +69 -0
  69. package/dist/__tests__/realtime-session-mint.test.js.map +1 -0
  70. package/dist/__tests__/realtime-session-policy.test.d.ts +2 -0
  71. package/dist/__tests__/realtime-session-policy.test.d.ts.map +1 -0
  72. package/dist/__tests__/realtime-session-policy.test.js +303 -0
  73. package/dist/__tests__/realtime-session-policy.test.js.map +1 -0
  74. package/dist/__tests__/realtime-session-review.service.test.d.ts +2 -0
  75. package/dist/__tests__/realtime-session-review.service.test.d.ts.map +1 -0
  76. package/dist/__tests__/realtime-session-review.service.test.js +743 -0
  77. package/dist/__tests__/realtime-session-review.service.test.js.map +1 -0
  78. package/dist/__tests__/realtime-session-state.test.d.ts +2 -0
  79. package/dist/__tests__/realtime-session-state.test.d.ts.map +1 -0
  80. package/dist/__tests__/realtime-session-state.test.js +83 -0
  81. package/dist/__tests__/realtime-session-state.test.js.map +1 -0
  82. package/dist/__tests__/realtime-session-timeline-card.test.d.ts +2 -0
  83. package/dist/__tests__/realtime-session-timeline-card.test.d.ts.map +1 -0
  84. package/dist/__tests__/realtime-session-timeline-card.test.js +106 -0
  85. package/dist/__tests__/realtime-session-timeline-card.test.js.map +1 -0
  86. package/dist/__tests__/realtime-session-timeline.test.d.ts +2 -0
  87. package/dist/__tests__/realtime-session-timeline.test.d.ts.map +1 -0
  88. package/dist/__tests__/realtime-session-timeline.test.js +142 -0
  89. package/dist/__tests__/realtime-session-timeline.test.js.map +1 -0
  90. package/dist/__tests__/realtime-sessions-adapter.test.d.ts +19 -0
  91. package/dist/__tests__/realtime-sessions-adapter.test.d.ts.map +1 -0
  92. package/dist/__tests__/realtime-sessions-adapter.test.js +188 -0
  93. package/dist/__tests__/realtime-sessions-adapter.test.js.map +1 -0
  94. package/dist/__tests__/realtime-surface-panel-prefs.test.d.ts +2 -0
  95. package/dist/__tests__/realtime-surface-panel-prefs.test.d.ts.map +1 -0
  96. package/dist/__tests__/realtime-surface-panel-prefs.test.js +100 -0
  97. package/dist/__tests__/realtime-surface-panel-prefs.test.js.map +1 -0
  98. package/dist/__tests__/realtime-surface-tabs-model.test.d.ts +2 -0
  99. package/dist/__tests__/realtime-surface-tabs-model.test.d.ts.map +1 -0
  100. package/dist/__tests__/realtime-surface-tabs-model.test.js +193 -0
  101. package/dist/__tests__/realtime-surface-tabs-model.test.js.map +1 -0
  102. package/dist/__tests__/remote-browser-audio-player.test.d.ts +2 -0
  103. package/dist/__tests__/remote-browser-audio-player.test.d.ts.map +1 -0
  104. package/dist/__tests__/remote-browser-audio-player.test.js +137 -0
  105. package/dist/__tests__/remote-browser-audio-player.test.js.map +1 -0
  106. package/dist/__tests__/remote-browser-channel.test.d.ts +2 -0
  107. package/dist/__tests__/remote-browser-channel.test.d.ts.map +1 -0
  108. package/dist/__tests__/remote-browser-channel.test.js +423 -0
  109. package/dist/__tests__/remote-browser-channel.test.js.map +1 -0
  110. package/dist/__tests__/slot-defaults.test.d.ts +24 -0
  111. package/dist/__tests__/slot-defaults.test.d.ts.map +1 -0
  112. package/dist/__tests__/slot-defaults.test.js +63 -0
  113. package/dist/__tests__/slot-defaults.test.js.map +1 -0
  114. package/dist/__tests__/user-authorization.test.d.ts +2 -0
  115. package/dist/__tests__/user-authorization.test.d.ts.map +1 -0
  116. package/dist/__tests__/user-authorization.test.js +97 -0
  117. package/dist/__tests__/user-authorization.test.js.map +1 -0
  118. package/dist/__tests__/voice-session-narration.test.d.ts +2 -0
  119. package/dist/__tests__/voice-session-narration.test.d.ts.map +1 -0
  120. package/dist/__tests__/voice-session-narration.test.js +609 -0
  121. package/dist/__tests__/voice-session-narration.test.js.map +1 -0
  122. package/dist/__tests__/whiteboard-artifact-viewer.test.d.ts +2 -0
  123. package/dist/__tests__/whiteboard-artifact-viewer.test.d.ts.map +1 -0
  124. package/dist/__tests__/whiteboard-artifact-viewer.test.js +101 -0
  125. package/dist/__tests__/whiteboard-artifact-viewer.test.js.map +1 -0
  126. package/dist/__tests__/whiteboard-channel.test.d.ts +2 -0
  127. package/dist/__tests__/whiteboard-channel.test.d.ts.map +1 -0
  128. package/dist/__tests__/whiteboard-channel.test.js +260 -0
  129. package/dist/__tests__/whiteboard-channel.test.js.map +1 -0
  130. package/dist/__tests__/whiteboard-restore-state.test.d.ts +2 -0
  131. package/dist/__tests__/whiteboard-restore-state.test.d.ts.map +1 -0
  132. package/dist/__tests__/whiteboard-restore-state.test.js +108 -0
  133. package/dist/__tests__/whiteboard-restore-state.test.js.map +1 -0
  134. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts +205 -3
  135. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts.map +1 -1
  136. package/dist/lib/components/conversation/conversation-chat-area.component.js +911 -342
  137. package/dist/lib/components/conversation/conversation-chat-area.component.js.map +1 -1
  138. package/dist/lib/components/mention/mention-dropdown.component.js +35 -17
  139. package/dist/lib/components/mention/mention-dropdown.component.js.map +1 -1
  140. package/dist/lib/components/mention/mention-editor.component.d.ts +4 -0
  141. package/dist/lib/components/mention/mention-editor.component.d.ts.map +1 -1
  142. package/dist/lib/components/mention/mention-editor.component.js +43 -19
  143. package/dist/lib/components/mention/mention-editor.component.js.map +1 -1
  144. package/dist/lib/components/message/message-input-box.component.d.ts +17 -1
  145. package/dist/lib/components/message/message-input-box.component.d.ts.map +1 -1
  146. package/dist/lib/components/message/message-input-box.component.js +73 -15
  147. package/dist/lib/components/message/message-input-box.component.js.map +1 -1
  148. package/dist/lib/components/message/message-input.component.d.ts +142 -6
  149. package/dist/lib/components/message/message-input.component.d.ts.map +1 -1
  150. package/dist/lib/components/message/message-input.component.js +328 -82
  151. package/dist/lib/components/message/message-input.component.js.map +1 -1
  152. package/dist/lib/components/message/message-item.component.d.ts +28 -3
  153. package/dist/lib/components/message/message-item.component.d.ts.map +1 -1
  154. package/dist/lib/components/message/message-item.component.js +180 -108
  155. package/dist/lib/components/message/message-item.component.js.map +1 -1
  156. package/dist/lib/components/message/message-list.component.d.ts +81 -2
  157. package/dist/lib/components/message/message-list.component.d.ts.map +1 -1
  158. package/dist/lib/components/message/message-list.component.js +252 -87
  159. package/dist/lib/components/message/message-list.component.js.map +1 -1
  160. package/dist/lib/components/realtime/channels/base-realtime-channel-client.d.ts +282 -0
  161. package/dist/lib/components/realtime/channels/base-realtime-channel-client.d.ts.map +1 -0
  162. package/dist/lib/components/realtime/channels/base-realtime-channel-client.js +158 -0
  163. package/dist/lib/components/realtime/channels/base-realtime-channel-client.js.map +1 -0
  164. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.d.ts +25 -0
  165. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.d.ts.map +1 -0
  166. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.js +140 -0
  167. package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.js.map +1 -0
  168. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.d.ts +35 -0
  169. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.d.ts.map +1 -0
  170. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.js +58 -0
  171. package/dist/lib/components/realtime/channels/realtime-channel-pane.component.js.map +1 -0
  172. package/dist/lib/components/realtime/realtime-activity-rail.component.d.ts +63 -0
  173. package/dist/lib/components/realtime/realtime-activity-rail.component.d.ts.map +1 -0
  174. package/dist/lib/components/realtime/realtime-activity-rail.component.js +260 -0
  175. package/dist/lib/components/realtime/realtime-activity-rail.component.js.map +1 -0
  176. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts +117 -0
  177. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts.map +1 -0
  178. package/dist/lib/components/realtime/realtime-agent-banner.component.js +504 -0
  179. package/dist/lib/components/realtime/realtime-agent-banner.component.js.map +1 -0
  180. package/dist/lib/components/realtime/realtime-agent-picker.component.d.ts +168 -0
  181. package/dist/lib/components/realtime/realtime-agent-picker.component.d.ts.map +1 -0
  182. package/dist/lib/components/realtime/realtime-agent-picker.component.js +556 -0
  183. package/dist/lib/components/realtime/realtime-agent-picker.component.js.map +1 -0
  184. package/dist/lib/components/realtime/realtime-audio-visuals.d.ts +97 -0
  185. package/dist/lib/components/realtime/realtime-audio-visuals.d.ts.map +1 -0
  186. package/dist/lib/components/realtime/realtime-audio-visuals.js +139 -0
  187. package/dist/lib/components/realtime/realtime-audio-visuals.js.map +1 -0
  188. package/dist/lib/components/realtime/realtime-channel-strip.component.d.ts +29 -0
  189. package/dist/lib/components/realtime/realtime-channel-strip.component.d.ts.map +1 -0
  190. package/dist/lib/components/realtime/realtime-channel-strip.component.js +69 -0
  191. package/dist/lib/components/realtime/realtime-channel-strip.component.js.map +1 -0
  192. package/dist/lib/components/realtime/realtime-composer.component.d.ts +65 -0
  193. package/dist/lib/components/realtime/realtime-composer.component.d.ts.map +1 -0
  194. package/dist/lib/components/realtime/realtime-composer.component.js +256 -0
  195. package/dist/lib/components/realtime/realtime-composer.component.js.map +1 -0
  196. package/dist/lib/components/realtime/realtime-delegation-card.component.d.ts +71 -0
  197. package/dist/lib/components/realtime/realtime-delegation-card.component.d.ts.map +1 -0
  198. package/dist/lib/components/realtime/realtime-delegation-card.component.js +324 -0
  199. package/dist/lib/components/realtime/realtime-delegation-card.component.js.map +1 -0
  200. package/dist/lib/components/realtime/realtime-disclosure.d.ts +135 -0
  201. package/dist/lib/components/realtime/realtime-disclosure.d.ts.map +1 -0
  202. package/dist/lib/components/realtime/realtime-disclosure.js +188 -0
  203. package/dist/lib/components/realtime/realtime-disclosure.js.map +1 -0
  204. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts +491 -0
  205. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts.map +1 -0
  206. package/dist/lib/components/realtime/realtime-session-overlay.component.js +1274 -0
  207. package/dist/lib/components/realtime/realtime-session-overlay.component.js.map +1 -0
  208. package/dist/lib/components/realtime/realtime-session-state.d.ts +191 -0
  209. package/dist/lib/components/realtime/realtime-session-state.d.ts.map +1 -0
  210. package/dist/lib/components/realtime/realtime-session-state.js +244 -0
  211. package/dist/lib/components/realtime/realtime-session-state.js.map +1 -0
  212. package/dist/lib/components/realtime/realtime-session-thread.component.d.ts +56 -0
  213. package/dist/lib/components/realtime/realtime-session-thread.component.d.ts.map +1 -0
  214. package/dist/lib/components/realtime/realtime-session-thread.component.js +246 -0
  215. package/dist/lib/components/realtime/realtime-session-thread.component.js.map +1 -0
  216. package/dist/lib/components/realtime/realtime-session-timeline-card.component.d.ts +51 -0
  217. package/dist/lib/components/realtime/realtime-session-timeline-card.component.d.ts.map +1 -0
  218. package/dist/lib/components/realtime/realtime-session-timeline-card.component.js +193 -0
  219. package/dist/lib/components/realtime/realtime-session-timeline-card.component.js.map +1 -0
  220. package/dist/lib/components/realtime/realtime-surface-panel-prefs.d.ts +77 -0
  221. package/dist/lib/components/realtime/realtime-surface-panel-prefs.d.ts.map +1 -0
  222. package/dist/lib/components/realtime/realtime-surface-panel-prefs.js +114 -0
  223. package/dist/lib/components/realtime/realtime-surface-panel-prefs.js.map +1 -0
  224. package/dist/lib/components/realtime/realtime-surface-tabs.component.d.ts +173 -0
  225. package/dist/lib/components/realtime/realtime-surface-tabs.component.d.ts.map +1 -0
  226. package/dist/lib/components/realtime/realtime-surface-tabs.component.js +496 -0
  227. package/dist/lib/components/realtime/realtime-surface-tabs.component.js.map +1 -0
  228. package/dist/lib/components/realtime/realtime-surface-tabs.model.d.ts +181 -0
  229. package/dist/lib/components/realtime/realtime-surface-tabs.model.d.ts.map +1 -0
  230. package/dist/lib/components/realtime/realtime-surface-tabs.model.js +223 -0
  231. package/dist/lib/components/realtime/realtime-surface-tabs.model.js.map +1 -0
  232. package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.d.ts +163 -0
  233. package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.d.ts.map +1 -0
  234. package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.js +309 -0
  235. package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.js.map +1 -0
  236. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts +168 -0
  237. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map +1 -0
  238. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js +524 -0
  239. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js.map +1 -0
  240. package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.d.ts +346 -0
  241. package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.d.ts.map +1 -0
  242. package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.js +851 -0
  243. package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.js.map +1 -0
  244. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts +86 -0
  245. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts.map +1 -0
  246. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js +210 -0
  247. package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js.map +1 -0
  248. package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.d.ts +48 -0
  249. package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.d.ts.map +1 -0
  250. package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.js +180 -0
  251. package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.js.map +1 -0
  252. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts +119 -0
  253. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts.map +1 -0
  254. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js +274 -0
  255. package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js.map +1 -0
  256. package/dist/lib/components/slots/mj-chat-agent-presence-default.component.d.ts +11 -0
  257. package/dist/lib/components/slots/mj-chat-agent-presence-default.component.d.ts.map +1 -0
  258. package/dist/lib/components/slots/mj-chat-agent-presence-default.component.js +98 -0
  259. package/dist/lib/components/slots/mj-chat-agent-presence-default.component.js.map +1 -0
  260. package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.d.ts +9 -0
  261. package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.d.ts.map +1 -0
  262. package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.js +35 -0
  263. package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.js.map +1 -0
  264. package/dist/lib/components/slots/mj-chat-empty-state-default.component.d.ts +28 -0
  265. package/dist/lib/components/slots/mj-chat-empty-state-default.component.d.ts.map +1 -0
  266. package/dist/lib/components/slots/mj-chat-empty-state-default.component.js +104 -0
  267. package/dist/lib/components/slots/mj-chat-empty-state-default.component.js.map +1 -0
  268. package/dist/lib/components/slots/mj-chat-header-default.component.d.ts +11 -0
  269. package/dist/lib/components/slots/mj-chat-header-default.component.d.ts.map +1 -0
  270. package/dist/lib/components/slots/mj-chat-header-default.component.js +103 -0
  271. package/dist/lib/components/slots/mj-chat-header-default.component.js.map +1 -0
  272. package/dist/lib/components/slots/mj-chat-message-bubble-default.component.d.ts +15 -0
  273. package/dist/lib/components/slots/mj-chat-message-bubble-default.component.d.ts.map +1 -0
  274. package/dist/lib/components/slots/mj-chat-message-bubble-default.component.js +73 -0
  275. package/dist/lib/components/slots/mj-chat-message-bubble-default.component.js.map +1 -0
  276. package/dist/lib/components/slots/mj-chat-message-extra-default.component.d.ts +9 -0
  277. package/dist/lib/components/slots/mj-chat-message-extra-default.component.d.ts.map +1 -0
  278. package/dist/lib/components/slots/mj-chat-message-extra-default.component.js +34 -0
  279. package/dist/lib/components/slots/mj-chat-message-extra-default.component.js.map +1 -0
  280. package/dist/lib/components/slots/slot-interfaces.d.ts +95 -0
  281. package/dist/lib/components/slots/slot-interfaces.d.ts.map +1 -0
  282. package/dist/lib/components/slots/slot-interfaces.js +18 -0
  283. package/dist/lib/components/slots/slot-interfaces.js.map +1 -0
  284. package/dist/lib/components/workspace/conversation-workspace.component.d.ts +11 -0
  285. package/dist/lib/components/workspace/conversation-workspace.component.d.ts.map +1 -1
  286. package/dist/lib/components/workspace/conversation-workspace.component.js +28 -4
  287. package/dist/lib/components/workspace/conversation-workspace.component.js.map +1 -1
  288. package/dist/lib/conversations.module.d.ts +12 -1
  289. package/dist/lib/conversations.module.d.ts.map +1 -1
  290. package/dist/lib/conversations.module.js +93 -5
  291. package/dist/lib/conversations.module.js.map +1 -1
  292. package/dist/lib/directives/chat-slot.directive.d.ts +44 -0
  293. package/dist/lib/directives/chat-slot.directive.d.ts.map +1 -0
  294. package/dist/lib/directives/chat-slot.directive.js +54 -0
  295. package/dist/lib/directives/chat-slot.directive.js.map +1 -0
  296. package/dist/lib/events/chat-events.d.ts +137 -0
  297. package/dist/lib/events/chat-events.d.ts.map +1 -0
  298. package/dist/lib/events/chat-events.js +189 -0
  299. package/dist/lib/events/chat-events.js.map +1 -0
  300. package/dist/lib/models/conversation-state.model.d.ts +2 -1
  301. package/dist/lib/models/conversation-state.model.d.ts.map +1 -1
  302. package/dist/lib/models/conversation-state.model.js.map +1 -1
  303. package/dist/lib/services/artifact-state.service.d.ts.map +1 -1
  304. package/dist/lib/services/artifact-state.service.js +23 -6
  305. package/dist/lib/services/artifact-state.service.js.map +1 -1
  306. package/dist/lib/services/conversation-agent.service.d.ts +60 -74
  307. package/dist/lib/services/conversation-agent.service.d.ts.map +1 -1
  308. package/dist/lib/services/conversation-agent.service.js +100 -313
  309. package/dist/lib/services/conversation-agent.service.js.map +1 -1
  310. package/dist/lib/services/conversation-bridge.service.d.ts +11 -70
  311. package/dist/lib/services/conversation-bridge.service.d.ts.map +1 -1
  312. package/dist/lib/services/conversation-bridge.service.js +51 -85
  313. package/dist/lib/services/conversation-bridge.service.js.map +1 -1
  314. package/dist/lib/services/conversation-naming.d.ts +63 -0
  315. package/dist/lib/services/conversation-naming.d.ts.map +1 -0
  316. package/dist/lib/services/conversation-naming.js +58 -0
  317. package/dist/lib/services/conversation-naming.js.map +1 -0
  318. package/dist/lib/services/conversation-streaming.service.d.ts +24 -154
  319. package/dist/lib/services/conversation-streaming.service.d.ts.map +1 -1
  320. package/dist/lib/services/conversation-streaming.service.js +39 -361
  321. package/dist/lib/services/conversation-streaming.service.js.map +1 -1
  322. package/dist/lib/services/conversations-runtime-bootstrap.service.d.ts +10 -0
  323. package/dist/lib/services/conversations-runtime-bootstrap.service.d.ts.map +1 -0
  324. package/dist/lib/services/conversations-runtime-bootstrap.service.js +104 -0
  325. package/dist/lib/services/conversations-runtime-bootstrap.service.js.map +1 -0
  326. package/dist/lib/services/delegation-result-parser.d.ts +45 -0
  327. package/dist/lib/services/delegation-result-parser.d.ts.map +1 -0
  328. package/dist/lib/services/delegation-result-parser.js +48 -0
  329. package/dist/lib/services/delegation-result-parser.js.map +1 -0
  330. package/dist/lib/services/mention-autocomplete.service.d.ts +19 -4
  331. package/dist/lib/services/mention-autocomplete.service.d.ts.map +1 -1
  332. package/dist/lib/services/mention-autocomplete.service.js +65 -4
  333. package/dist/lib/services/mention-autocomplete.service.js.map +1 -1
  334. package/dist/lib/services/mention-parser.service.d.ts +8 -53
  335. package/dist/lib/services/mention-parser.service.d.ts.map +1 -1
  336. package/dist/lib/services/mention-parser.service.js +32 -243
  337. package/dist/lib/services/mention-parser.service.js.map +1 -1
  338. package/dist/lib/services/narration-template.d.ts +42 -0
  339. package/dist/lib/services/narration-template.d.ts.map +1 -0
  340. package/dist/lib/services/narration-template.js +73 -0
  341. package/dist/lib/services/narration-template.js.map +1 -0
  342. package/dist/lib/services/realtime-pairing.d.ts +120 -0
  343. package/dist/lib/services/realtime-pairing.d.ts.map +1 -0
  344. package/dist/lib/services/realtime-pairing.js +150 -0
  345. package/dist/lib/services/realtime-pairing.js.map +1 -0
  346. package/dist/lib/services/realtime-session-review.service.d.ts +233 -0
  347. package/dist/lib/services/realtime-session-review.service.d.ts.map +1 -0
  348. package/dist/lib/services/realtime-session-review.service.js +417 -0
  349. package/dist/lib/services/realtime-session-review.service.js.map +1 -0
  350. package/dist/lib/services/realtime-session.service.d.ts +739 -0
  351. package/dist/lib/services/realtime-session.service.d.ts.map +1 -0
  352. package/dist/lib/services/realtime-session.service.js +1647 -0
  353. package/dist/lib/services/realtime-session.service.js.map +1 -0
  354. package/dist/lib/services/realtime-sessions-adapter.d.ts +54 -0
  355. package/dist/lib/services/realtime-sessions-adapter.d.ts.map +1 -0
  356. package/dist/lib/services/realtime-sessions-adapter.js +154 -0
  357. package/dist/lib/services/realtime-sessions-adapter.js.map +1 -0
  358. package/dist/lib/services/user-authorization.d.ts +67 -0
  359. package/dist/lib/services/user-authorization.d.ts.map +1 -0
  360. package/dist/lib/services/user-authorization.js +66 -0
  361. package/dist/lib/services/user-authorization.js.map +1 -0
  362. package/dist/lib/utils/realtime-session-timeline.d.ts +84 -0
  363. package/dist/lib/utils/realtime-session-timeline.d.ts.map +1 -0
  364. package/dist/lib/utils/realtime-session-timeline.js +94 -0
  365. package/dist/lib/utils/realtime-session-timeline.js.map +1 -0
  366. package/dist/public-api.d.ts +41 -0
  367. package/dist/public-api.d.ts.map +1 -1
  368. package/dist/public-api.js +50 -0
  369. package/dist/public-api.js.map +1 -1
  370. package/package.json +27 -24
  371. package/dist/__tests__/conversation-bridge.service.test.d.ts +0 -2
  372. package/dist/__tests__/conversation-bridge.service.test.d.ts.map +0 -1
  373. package/dist/__tests__/conversation-bridge.service.test.js +0 -98
  374. package/dist/__tests__/conversation-bridge.service.test.js.map +0 -1
  375. package/dist/__tests__/mention-parser.test.d.ts +0 -2
  376. package/dist/__tests__/mention-parser.test.d.ts.map +0 -1
  377. package/dist/__tests__/mention-parser.test.js +0 -154
  378. package/dist/__tests__/mention-parser.test.js.map +0 -1
@@ -0,0 +1,1647 @@
1
+ import { Injectable } from '@angular/core';
2
+ import { BehaviorSubject, Subject } from 'rxjs';
3
+ import { Metadata } from '@memberjunction/core';
4
+ import { AIEngineBase } from '@memberjunction/ai-engine-base';
5
+ import { MJGlobal } from '@memberjunction/global';
6
+ import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient } from '@memberjunction/ai-realtime-client';
7
+ import { BuildNarrationInstructions } from './narration-template';
8
+ import { ParseDelegationResultJson } from './delegation-result-parser';
9
+ import { BaseRealtimeChannelClient } from '../components/realtime/channels/base-realtime-channel-client';
10
+ import * as i0 from "@angular/core";
11
+ // Tree-shaking prevention: the OpenAI client is resolved dynamically through the
12
+ // ClassFactory (by the server-reported Provider key), so this static call is what keeps
13
+ // its @RegisterClass side effect from being eliminated by the bundler.
14
+ // NOTE: the interactive-channel plugins (resolved dynamically from the `MJ: AI Agent
15
+ // Channels` registry by ClientPluginClass key) get the same treatment, but their Load
16
+ // calls live in `conversations.module.ts` — plugins carry Angular surface COMPONENTS,
17
+ // and this service stays component-free (it must stay importable in plain-node tests).
18
+ LoadOpenAIRealtimeClient();
19
+ LoadGeminiRealtimeClient();
20
+ LoadElevenLabsRealtimeClient();
21
+ LoadAssemblyAIRealtimeClient();
22
+ LoadxAIRealtimeClient();
23
+ /**
24
+ * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
25
+ * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
26
+ * frames never transit the MJ server (low latency); only tool calls and final
27
+ * transcripts are relayed back to MJ over GraphQL.
28
+ *
29
+ * This service is PROVIDER-AGNOSTIC policy/orchestration. All provider wire concerns
30
+ * (transport, event translation, the response state machine, narration-kind tagging,
31
+ * playback tracking) live in a {@link BaseRealtimeClient} driver resolved through the
32
+ * MJ ClassFactory by the server-reported `Provider` key (e.g. `'openai'` →
33
+ * `OpenAIRealtimeClient`). Future providers (Gemini Live, …) snap in by registering a
34
+ * new driver — this service does not change.
35
+ *
36
+ * The Realtime Co-Agent (server-side) fronts the conversation's current agent — the server
37
+ * bakes the companion instructions + tool set into `SessionConfigJson`, which the client
38
+ * driver applies verbatim.
39
+ *
40
+ * Lifecycle: {@link StartVoiceSession} → live duplex → {@link EndVoiceSession}.
41
+ */
42
+ export class RealtimeSessionService {
43
+ // ── Reactive UI state ──────────────────────────────────────────────────────
44
+ _connectionState$ = new BehaviorSubject('closed');
45
+ _captions$ = new BehaviorSubject([]);
46
+ _active$ = new BehaviorSubject(false);
47
+ _delegationProgress$ = new Subject();
48
+ _delegationResult$ = new Subject();
49
+ _delegationNarration$ = new Subject();
50
+ _agentName$ = new BehaviorSubject('Sage');
51
+ _modelName$ = new BehaviorSubject(null);
52
+ _minimized$ = new BehaviorSubject(false);
53
+ _activeChannels$ = new BehaviorSubject([]);
54
+ _channelFocus$ = new Subject();
55
+ // ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
56
+ // bridge into @memberjunction/conversations-runtime's framework-agnostic
57
+ // SessionsObserver). Why not derive from Active$ + agentSessionId? Because
58
+ // Active$ flips true before mintSession resolves and sets agentSessionId —
59
+ // a naive Active$ subscription would emit session-started with sessionId === null.
60
+ // Emitting explicitly avoids the race entirely. ───
61
+ _sessionStarted$ = new Subject();
62
+ _sessionEnded$ = new Subject();
63
+ _channelActivity$ = new Subject();
64
+ /** Current connection / turn state. */
65
+ ConnectionState$ = this._connectionState$.asObservable();
66
+ /** Live captions for both sides of the conversation. */
67
+ Captions$ = this._captions$.asObservable();
68
+ /** True while a session is open (mic button active, overlay shown). */
69
+ Active$ = this._active$.asObservable();
70
+ /**
71
+ * Progress updates from a delegated agent run (e.g. Sage) while the realtime model waits on it.
72
+ * The future overlay subscribes to render a "working" card; the model also narrates these aloud.
73
+ */
74
+ DelegationProgress$ = this._delegationProgress$.asObservable();
75
+ /** Terminal result of a delegation, so the overlay can complete the working card with real content. */
76
+ DelegationResult$ = this._delegationResult$.asObservable();
77
+ /**
78
+ * EPHEMERAL spoken progress narrations (see {@link VoiceDelegationNarration}). These are
79
+ * deliberately kept OUT of {@link Captions$} and never relayed/persisted — the overlay
80
+ * renders them as a transient "live note" near the active working card.
81
+ */
82
+ DelegationNarration$ = this._delegationNarration$.asObservable();
83
+ /** Display name of the agent the active session fronts (set at session start). */
84
+ AgentName$ = this._agentName$.asObservable();
85
+ /**
86
+ * Display name of the realtime MODEL the active session runs on (server-reported at session
87
+ * start, e.g. "GPT Realtime 2"). `null` before a session starts / when the server didn't report
88
+ * one. The overlay banner shows it subtly next to the agent identity.
89
+ */
90
+ ModelName$ = this._modelName$.asObservable();
91
+ /**
92
+ * True while the active call overlay is MINIMIZED to the host's floating "on call" pill
93
+ * (e.g. after a dev link navigated away). The mic and session stay fully live — this is
94
+ * pure presentation state, reset to `false` at session start and teardown.
95
+ */
96
+ Minimized$ = this._minimized$.asObservable();
97
+ /**
98
+ * The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
99
+ * Channels` registry at session start (one instance per session, per channel). Emits
100
+ * `[]` before a session starts and after teardown. The overlay subscribes to register
101
+ * one surface tab per plugin — it never knows any concrete channel type.
102
+ */
103
+ ActiveChannels$ = this._activeChannels$.asObservable();
104
+ /**
105
+ * Channel requests to enter / leave the FOCUS layout (see
106
+ * {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
107
+ * `SetFocusMode` — e.g. the whiteboard's "Focus board" toggle.
108
+ */
109
+ ChannelFocus$ = this._channelFocus$.asObservable();
110
+ /**
111
+ * Fired EXACTLY ONCE per session after both `agentSessionId` is set AND the
112
+ * realtime client is connected. Carries the server-issued `sessionId` and the
113
+ * `ChannelName` of each plugin resolved at session mint. Consumed by
114
+ * `RealtimeSessionsAdapter` (in this package) to feed
115
+ * `@memberjunction/conversations-runtime`'s `SessionsObserver`.
116
+ *
117
+ * **Why this exists separately from `Active$`:** `Active$` flips `true` BEFORE
118
+ * `mintSession` resolves, so `agentSessionId` is still `null` at that moment.
119
+ * Subscribers correlating `(Active$, agentSessionId)` would race; this event
120
+ * removes the race.
121
+ */
122
+ SessionStarted$ = this._sessionStarted$.asObservable();
123
+ /**
124
+ * Fired EXACTLY ONCE per session as teardown begins, with the prior
125
+ * `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
126
+ * sessionId) and the client-distinguishable reason — `'explicit'` when the
127
+ * user called `EndVoiceSession`, `'error'` when teardown ran from a catch
128
+ * block. Server-side close paths (janitor, shutdown) do NOT propagate here —
129
+ * they happen out-of-process and have no client push channel today.
130
+ */
131
+ SessionEnded$ = this._sessionEnded$.asObservable();
132
+ /**
133
+ * Fires with the channel PLUGIN every time the agent ACTS on that channel (a tool call
134
+ * was routed to its local executor — e.g. the agent drew on the whiteboard). The overlay
135
+ * uses the FIRST emission per channel to auto-reveal + focus the channel's surface tab,
136
+ * so the user discovers the surface the moment the agent starts using it. Finer-grained
137
+ * than {@link SessionStarted$}/{@link SessionEnded$} (per tool call, not per session).
138
+ */
139
+ ChannelActivity$ = this._channelActivity$.asObservable();
140
+ /** Synchronous access to the session's active interactive-channel plugins. */
141
+ get ActiveChannels() {
142
+ return this._activeChannels$.value;
143
+ }
144
+ /** Synchronous access to the display name of the agent the active session fronts. */
145
+ get CurrentAgentName() {
146
+ return this._agentName$.value;
147
+ }
148
+ /**
149
+ * ID of the active server-side agent session (`MJ: AI Agent Sessions`), or `null` when no
150
+ * session is open / the session hasn't been minted yet. Powers the overlay's gear-gated
151
+ * "Open session" dev link.
152
+ */
153
+ /** Conversation id the SERVER created for this session (null when the host supplied one). */
154
+ createdConversationId = null;
155
+ /** The session's conversation id (supplied or server-created). */
156
+ sessionConversationId = null;
157
+ /** First final user utterance of the live session (the naming seed). */
158
+ firstUserTranscript = null;
159
+ /**
160
+ * When the active/last session CREATED its conversation (started without one), the new
161
+ * conversation's id — the host uses it to refresh the cached list, conditionally select
162
+ * it on close, and auto-name it. Null when the session joined an existing conversation.
163
+ */
164
+ get SessionCreatedConversationId() {
165
+ return this.createdConversationId;
166
+ }
167
+ /** The first final user utterance of the session (naming seed); null before the user speaks. */
168
+ get FirstUserTranscript() {
169
+ return this.firstUserTranscript;
170
+ }
171
+ get CurrentAgentSessionId() {
172
+ return this.agentSessionId;
173
+ }
174
+ /** Synchronous access to the minimized presentation state. */
175
+ get IsMinimized() {
176
+ return this._minimized$.value;
177
+ }
178
+ /**
179
+ * Minimizes / restores the active call overlay (host renders the floating pill while
180
+ * minimized). Presentation-only — the live audio session is untouched.
181
+ */
182
+ SetMinimized(minimized) {
183
+ if (this._minimized$.value !== minimized) {
184
+ this._minimized$.next(minimized);
185
+ }
186
+ }
187
+ // ── Session internals ──────────────────────────────────────────────────────
188
+ /** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
189
+ client = null;
190
+ /** The mic capture stream — acquired here (permission UX) and handed to the client. */
191
+ localStream = null;
192
+ agentSessionId = null;
193
+ /**
194
+ * The DB-driven narration instruction template (server-resolved at session start, containing a
195
+ * `{{ progressMessage }}` placeholder). `null` when the deployment hasn't synced the narration
196
+ * prompt — {@link buildNarrationInstructions} then falls back to the built-in wording.
197
+ */
198
+ narrationTemplate = null;
199
+ // ── Delegated-run progress streaming ───────────────────────────────────────
200
+ /** First spoken update fires no earlier than this long after delegated work starts. */
201
+ static FirstNarrationDelayMs = 5000;
202
+ /** Minimum gap between SUBSEQUENT spoken updates (the 7–10s band; floods aggregate). */
203
+ static NarrationIntervalMs = 8000;
204
+ /** Retry delay when the fire moment finds the model busy / audio still playing. */
205
+ static NarrationBusyRetryMs = 1500;
206
+ /** Max progress messages aggregated into one spoken digest. */
207
+ static MaxDigestMessages = 4;
208
+ /** Max prior spoken narrations chained into the instructions (anti-repetition). */
209
+ static MaxPriorNarrations = 3;
210
+ /**
211
+ * Aggregation buffer: distinct progress messages since the last spoken update (oldest
212
+ * first, capped at {@link RealtimeSessionService.MaxDigestMessages}). A flood of small
213
+ * updates becomes ONE digest; the buffer is discarded when the result lands first.
214
+ */
215
+ pendingNarrationMessages = [];
216
+ /**
217
+ * Tool calls currently executing on the server. Progress events ride PubSub and can
218
+ * lag the (fast) mutation result — any progress for a call NOT in this set is stale
219
+ * (already completed) and is dropped, so we never narrate "starting up" after the
220
+ * answer was already spoken.
221
+ */
222
+ inFlightCallIds = new Set();
223
+ /** Timer for the deferred narration; cancelled when the delegation result lands first. */
224
+ narrationTimer = null;
225
+ /**
226
+ * Call ids the USER explicitly cancelled via {@link CancelDelegation} /
227
+ * {@link CancelInFlightDelegations}. Their cards were already flipped to the
228
+ * "Cancelled by user" failed result, so when the original tool mutation later resolves
229
+ * with the aborted run's outcome, {@link emitDelegationResult} skips the duplicate card
230
+ * emission (the model still receives the tool result). Cleared at teardown.
231
+ */
232
+ cancelledCallIds = new Set();
233
+ // ── Usage telemetry relay (B7) ─────────────────────────────────────────────
234
+ /** Debounce window for relaying accumulated usage deltas to the server. */
235
+ static UsageFlushDebounceMs = 10000;
236
+ /** Accumulated input-token delta since the last flush. */
237
+ pendingUsageInput = 0;
238
+ /** Accumulated output-token delta since the last flush. */
239
+ pendingUsageOutput = 0;
240
+ /** Pending debounced usage flush; also force-flushed at teardown. */
241
+ usageFlushTimer = null;
242
+ /** Active push-status subscription that feeds delegation progress; cleared on teardown. */
243
+ delegationProgressSub = null;
244
+ /** Timestamp (ms) of the last narration we triggered; 0 = never. */
245
+ lastDelegationNarrationAt = 0;
246
+ /** When the current delegation burst began (first in-flight call); anchors the 5s first update. */
247
+ delegationBurstStartedAt = 0;
248
+ /** Spoken updates so far in this burst (1-based numbering for the instructions). */
249
+ narrationCount = 0;
250
+ /** What the model actually SAID for prior updates this burst — chained in so it never repeats itself. */
251
+ spokenNarrations = [];
252
+ /** Tail message of the last digest, so an identical trailing progress event isn't re-buffered. */
253
+ lastNarratedTail = '';
254
+ /**
255
+ * Registry of CLIENT-EXECUTED UI tool handlers, keyed by tool-name prefix (e.g.
256
+ * `'Whiteboard_'`). Tool calls whose name matches a registered prefix run LOCALLY through the
257
+ * handler (never relayed to the server); everything else takes the standard server-relay path.
258
+ * Cleared at teardown.
259
+ */
260
+ clientToolHandlers = new Map();
261
+ // ── Interactive channels (registry-resolved plugins) ───────────────────────
262
+ /** Debounce window for persisting a channel's state of record after a change burst. */
263
+ static ChannelSaveDebounceMs = 3000;
264
+ /**
265
+ * Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
266
+ * LATEST serialized state plus the session id captured while the session was live —
267
+ * the teardown flush runs as the live id is being torn down, so the capture guarantees
268
+ * the final save still lands on the just-closed session.
269
+ */
270
+ pendingChannelSaves = new Map();
271
+ _provider = null;
272
+ /**
273
+ * Metadata provider used for the GraphQL relay mutations. Falls back to the
274
+ * global default when unset (single-provider apps see no change).
275
+ */
276
+ get Provider() {
277
+ return this._provider ?? Metadata.Provider;
278
+ }
279
+ set Provider(value) {
280
+ this._provider = value;
281
+ }
282
+ /** True when a session is currently open. */
283
+ get IsActive() {
284
+ return this._active$.value;
285
+ }
286
+ /**
287
+ * Start a client-direct voice session fronting `targetAgentId`.
288
+ *
289
+ * @param targetAgentId The agent the Realtime Co-Agent voices on behalf of.
290
+ * @param conversationId Optional existing conversation to bind + seed context from.
291
+ * @param lastSessionId Optional prior session to chain to (resume / continuation).
292
+ * @param agentName Optional display name of the target agent — resolved by the caller
293
+ * (which knows the conversation's routing context) and surfaced on {@link AgentName$}
294
+ * so ANY host (composer trigger, chat-area overlay) can render it without re-resolving.
295
+ * @param preferredModelId Optional EXPLICIT realtime model choice (`MJ: AI Models.ID`). When
296
+ * set, the server uses exactly that model and FAILS with a clear reason if it can't (no
297
+ * silent fallback). Omit for the server's automatic (highest-PowerRank) selection.
298
+ * @param clientTools Optional EXTRA client-executed UI tool declarations to expose to the
299
+ * realtime model alongside the server's stable tool set and the interactive-channel
300
+ * tools (which are aggregated automatically from the registry-resolved plugins — see
301
+ * {@link ActiveChannels$}). The server only DECLARES these — execution stays in the
302
+ * browser via handlers registered with {@link RegisterClientToolHandler}. This is an
303
+ * extension point for hosts with bespoke (non-channel) UI tools; most callers omit it.
304
+ * @param coAgentId Optional EXPLICIT co-agent choice (`MJ: AI Agents.ID` of an Active,
305
+ * Realtime-type agent) — the highest-precedence step of the server's co-agent resolution
306
+ * chain. When set, the server uses exactly that co-agent and FAILS with a clear reason if
307
+ * it can't (no silent fallback). Omit to let server metadata drive the choice: the target
308
+ * agent's `DefaultCoAgentID`, then the type-level `AIAgentCoAgent` default row, then the global Realtime Co-Agent.
309
+ * @param configOverridesJson Optional JSON payload of SESSION CONFIG overrides (e.g.
310
+ * `{"realtime":{"modelPreference":"<modelId>"}}`), forwarded verbatim on the mint
311
+ * mutation. The server enforces the `Realtime: Advanced Session Controls`
312
+ * authorization on any overrides — hosts only populate this from authorization-gated
313
+ * pickers, and never synthesize overrides beyond what the user explicitly chose.
314
+ * Omit/`null` for the server's defaults (today's behavior).
315
+ */
316
+ async StartVoiceSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson) {
317
+ if (this.IsActive) {
318
+ return; // a session is already running — ignore duplicate starts
319
+ }
320
+ if (agentName) {
321
+ this._agentName$.next(agentName);
322
+ }
323
+ this.resetState();
324
+ this._active$.next(true);
325
+ this._connectionState$.next('connecting');
326
+ try {
327
+ // Resolve + initialize the interactive-channel plugins FIRST: their client-executed
328
+ // tool sets must be declared to the realtime model at session mint.
329
+ const allClientTools = [...(clientTools ?? []), ...(await this.startChannels())];
330
+ const session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson);
331
+ this.agentSessionId = session.AgentSessionId;
332
+ // A null input conversationId means the SERVER created a fresh conversation for
333
+ // this session — track it so the host can fold it into the cached list, select
334
+ // it on close, and auto-name it (via the shared naming helper).
335
+ this.createdConversationId = !conversationId && session.ConversationId ? session.ConversationId : null;
336
+ this.sessionConversationId = session.ConversationId ?? conversationId ?? null;
337
+ this.firstUserTranscript = null;
338
+ this.narrationTemplate = session.NarrationInstructionsTemplate ?? null;
339
+ this._modelName$.next(session.ModelName ?? null);
340
+ // Resume continuity: rehydrate channel plugins from the PRIOR session's saved states
341
+ // (e.g. the whiteboard) BEFORE any surface binds — tolerant, never blocks the start.
342
+ this.applyPriorChannelStates(session.PriorChannelStatesJson);
343
+ const client = this.createRealtimeClient(session.Provider);
344
+ this.client = client;
345
+ this.wireClientHandlers(client);
346
+ this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
347
+ await client.Connect(this.buildClientConfig(session), this.localStream);
348
+ this.subscribeDelegationProgress();
349
+ // State advances to 'listening' once the provider control channel opens
350
+ // (driven by the client's OnStateChange events).
351
+ // Surface a generic session-started event for the conversations runtime
352
+ // SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
353
+ // agentSessionId is set (line ~468) AND the realtime client is connected,
354
+ // so consumers can act on it without re-checking either condition.
355
+ this._sessionStarted$.next({
356
+ sessionId: this.agentSessionId,
357
+ channelNames: this._activeChannels$.value.map(c => c.ChannelName),
358
+ });
359
+ }
360
+ catch (error) {
361
+ console.error('[RealtimeSession] Failed to start session:', error);
362
+ this._connectionState$.next('error');
363
+ await this.teardown(false);
364
+ }
365
+ }
366
+ /**
367
+ * End the active session: stop the mic, tear down the provider connection, and close
368
+ * the server-side agent session. Safe to call when no session is active.
369
+ */
370
+ async EndVoiceSession() {
371
+ if (!this.IsActive && !this.agentSessionId) {
372
+ return;
373
+ }
374
+ await this.teardown(true);
375
+ }
376
+ /**
377
+ * Inject a typed message into the live session as a user turn.
378
+ *
379
+ * Decomposed into two steps, each mirroring an existing voice path so the typed
380
+ * turn behaves identically to a spoken one:
381
+ * 1. {@link BaseRealtimeClient.SendText} injects the text as user input and triggers a
382
+ * reply through the SAME collision-safe path tool results use — so it queues behind
383
+ * any in-flight response (progress narration / prior turn) instead of colliding.
384
+ * 2. Relay the turn through the same caption + transcript paths user speech uses
385
+ * ({@link onUserTranscript}) so it shows in the live thread AND persists to MJ.
386
+ *
387
+ * No-op when no session is open / the control channel isn't ready, or when the text is empty.
388
+ */
389
+ SendText(text) {
390
+ const trimmed = text?.trim() ?? '';
391
+ if (trimmed.length === 0) {
392
+ return;
393
+ }
394
+ const client = this.client;
395
+ if (!client || !this.isSessionLive()) {
396
+ return;
397
+ }
398
+ client.SendText(trimmed);
399
+ // Relay as a user turn — same path spoken input uses (caption + persisted transcript).
400
+ void this.onUserTranscript(trimmed);
401
+ }
402
+ /** Mute / unmute the local microphone track. Returns the new muted state. */
403
+ ToggleMute() {
404
+ const tracks = this.localStream?.getAudioTracks() ?? [];
405
+ if (tracks.length === 0) {
406
+ return false;
407
+ }
408
+ const muted = tracks[0].enabled; // currently enabled → becomes muted
409
+ this.client?.SetMuted(muted);
410
+ return muted;
411
+ }
412
+ // ── Client-executed UI tools ───────────────────────────────────────────────
413
+ /**
414
+ * Registers a handler for CLIENT-EXECUTED UI tools whose names start with `toolNamePrefix`
415
+ * (e.g. `'Whiteboard_'` → all `Whiteboard_*` calls). Matching tool calls execute LOCALLY via
416
+ * the handler — they are never relayed to the server — and the handler's result JSON is sent
417
+ * back to the model as the `tool_response`. Re-registering the same prefix replaces the
418
+ * handler. The registry is cleared at session teardown.
419
+ */
420
+ RegisterClientToolHandler(toolNamePrefix, handler) {
421
+ this.clientToolHandlers.set(toolNamePrefix, handler);
422
+ }
423
+ /** Removes the handler registered for `toolNamePrefix` (no-op when none is registered). */
424
+ UnregisterClientToolHandler(toolNamePrefix) {
425
+ this.clientToolHandlers.delete(toolNamePrefix);
426
+ }
427
+ /**
428
+ * Feeds a background context note into the live model (no spoken reply is requested) — the
429
+ * perception channel interactive surfaces use (e.g. the whiteboard's coalesced scene deltas).
430
+ * No-op when no session is live.
431
+ */
432
+ SendContextNote(text) {
433
+ const trimmed = text?.trim() ?? '';
434
+ if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
435
+ return;
436
+ }
437
+ this.client.SendContextNote(trimmed);
438
+ }
439
+ /**
440
+ * The active client's current audio activity (per-direction RMS levels + spectrum
441
+ * bins), or `null` when no session is live or the driver attached no audio meters.
442
+ * Sampled by the overlay's animation-frame loop to drive the audio-reactive orb/EQ —
443
+ * a cheap analyser read, never provider traffic.
444
+ */
445
+ GetAudioActivity() {
446
+ return this.client?.GetAudioActivity() ?? null;
447
+ }
448
+ // ── Interactive channels (registry-driven plugins) ─────────────────────────
449
+ /**
450
+ * Resolves, instantiates and initializes the session's interactive-channel plugins from
451
+ * the `MJ: AI Agent Channels` registry, publishes them on {@link ActiveChannels$}, and
452
+ * returns their aggregated client-executed tool declarations for the session mint.
453
+ * Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
454
+ * session itself always proceeds.
455
+ */
456
+ async startChannels() {
457
+ const channels = await this.loadActiveChannels();
458
+ for (const plugin of channels) {
459
+ this.initializeChannel(plugin);
460
+ }
461
+ this._activeChannels$.next(channels);
462
+ return channels.flatMap(plugin => plugin.GetToolDefinitions());
463
+ }
464
+ /**
465
+ * Loads the ACTIVE channel definitions from the registry and resolves each row's
466
+ * `ClientPluginClass` through the MJ ClassFactory into a per-session plugin instance —
467
+ * the client-side mirror of how realtime-model drivers resolve from `BaseRealtimeModel`
468
+ * / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
469
+ * never fatal.
470
+ */
471
+ async loadActiveChannels() {
472
+ const rows = await this.fetchChannelDefinitions();
473
+ const channels = [];
474
+ for (const row of rows) {
475
+ const plugin = this.resolveChannelPlugin(row);
476
+ if (plugin) {
477
+ channels.push(plugin);
478
+ }
479
+ }
480
+ return channels;
481
+ }
482
+ /**
483
+ * Reads the ACTIVE `MJ: AI Agent Channels` rows from {@link AIEngineBase}'s cached
484
+ * `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView
485
+ * round-trip; the engine's BaseEntity-event reactivity keeps the registry fresh).
486
+ * Failures are logged and degrade to an empty list — channel availability must
487
+ * never block the voice session.
488
+ */
489
+ async fetchChannelDefinitions() {
490
+ try {
491
+ const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
492
+ await engine.Config(false, undefined, this.Provider);
493
+ return (engine.AgentChannels ?? [])
494
+ .filter(c => c.IsActive)
495
+ .map(c => ({ ID: c.ID, Name: c.Name, ClientPluginClass: c.ClientPluginClass }));
496
+ }
497
+ catch (error) {
498
+ console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
499
+ return [];
500
+ }
501
+ }
502
+ /**
503
+ * Resolves one registry row's `ClientPluginClass` via the ClassFactory (registration
504
+ * checked first, exactly like the realtime-client drivers) and instantiates a fresh
505
+ * per-session plugin. Returns `null` (logged) when no plugin is registered for the key
506
+ * — e.g. its Load function was never called or the package isn't included client-side.
507
+ */
508
+ resolveChannelPlugin(row) {
509
+ const key = row.ClientPluginClass?.trim();
510
+ if (!key) {
511
+ console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
512
+ return null;
513
+ }
514
+ const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
515
+ if (!registration) {
516
+ console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
517
+ return null;
518
+ }
519
+ const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
520
+ if (!plugin) {
521
+ console.warn(`[RealtimeSession] Failed to instantiate client plugin for channel '${row.Name}' (key '${key}').`);
522
+ return null;
523
+ }
524
+ return plugin;
525
+ }
526
+ /**
527
+ * Wires one plugin into the session: hands it its host context and registers its
528
+ * prefix-routed local tool executor (so `<ToolNamePrefix>*` calls run in the browser
529
+ * through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the server relay).
530
+ */
531
+ initializeChannel(plugin) {
532
+ plugin.Initialize(this.buildChannelContext(plugin));
533
+ this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
534
+ // The agent is ACTING on this channel — surface-discovery signal for the overlay
535
+ // (first activity auto-reveals + focuses the channel tab) before the tool applies.
536
+ this._channelActivity$.next(plugin);
537
+ return plugin.ApplyAgentTool(toolName, argsJson);
538
+ });
539
+ }
540
+ /** Builds the host-services context one channel plugin sees (its only line to the session). */
541
+ buildChannelContext(plugin) {
542
+ // Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
543
+ // field (not the object literal's `this`) every time it's accessed.
544
+ const service = this;
545
+ return {
546
+ AgentName: this.CurrentAgentName,
547
+ SendContextNote: (text) => this.SendContextNote(text),
548
+ RequestSpokenResponse: (instructions) => this.requestChannelSpokenResponse(instructions),
549
+ RequestSave: (stateJson) => this.scheduleChannelSave(plugin.ChannelName, stateJson),
550
+ SaveAsArtifact: (name, contentJson) => this.saveChannelArtifact(plugin.ChannelName, name, contentJson),
551
+ SetFocusMode: (on) => this._channelFocus$.next({ Channel: plugin, Focused: on }),
552
+ // Live session id + GraphQL escape hatch for SERVER-BACKED channels (e.g. Remote
553
+ // Browser). `get` so a channel always reads the CURRENT id — it's null at Initialize
554
+ // (the plugin is built before mintSession resolves) and set once the session is live.
555
+ get AgentSessionID() {
556
+ return service.agentSessionId;
557
+ },
558
+ ExecuteServerAction: (query, variables) => this.executeChannelServerAction(query, variables)
559
+ };
560
+ }
561
+ /**
562
+ * Runs a channel-specific GraphQL operation through the live session's provider (the
563
+ * {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
564
+ * transport/server error is logged and resolves to `null` so the calling channel can map
565
+ * the failure to a model-readable result string without `try/catch`.
566
+ */
567
+ async executeChannelServerAction(query, variables) {
568
+ try {
569
+ const result = await this.gql().ExecuteGQL(query, variables);
570
+ return result ?? null;
571
+ }
572
+ catch (error) {
573
+ console.error('[RealtimeSession] Channel server action failed:', error);
574
+ return null;
575
+ }
576
+ }
577
+ /**
578
+ * A channel asked the live model to SPEAK in reaction to channel input (e.g. a widget
579
+ * submission) — routed through the client's spoken-update channel. No-op when the
580
+ * session isn't live; empty instructions are dropped.
581
+ */
582
+ requestChannelSpokenResponse(instructions) {
583
+ const trimmed = instructions?.trim() ?? '';
584
+ if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
585
+ return;
586
+ }
587
+ this.client.RequestSpokenUpdate(trimmed);
588
+ }
589
+ /**
590
+ * Applies the PRIOR session's saved channel states (resume continuity): parses the
591
+ * server-supplied map and offers each entry to the matching active plugin via
592
+ * {@link BaseRealtimeChannelClient.RestoreState}. Fully tolerant — malformed payloads,
593
+ * unknown channels, and plugin rejections are logged and skipped; the session start is
594
+ * never affected.
595
+ */
596
+ applyPriorChannelStates(statesJson) {
597
+ if (!statesJson) {
598
+ return;
599
+ }
600
+ let states;
601
+ try {
602
+ const parsed = JSON.parse(statesJson);
603
+ if (parsed === null || typeof parsed !== 'object') {
604
+ return;
605
+ }
606
+ states = parsed;
607
+ }
608
+ catch {
609
+ console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
610
+ return;
611
+ }
612
+ for (const plugin of this._activeChannels$.value) {
613
+ const state = states[plugin.ChannelName];
614
+ if (typeof state === 'string' && state.length > 0) {
615
+ try {
616
+ const restored = plugin.RestoreState(state);
617
+ if (!restored) {
618
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
619
+ }
620
+ }
621
+ catch (error) {
622
+ console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
623
+ }
624
+ }
625
+ }
626
+ }
627
+ /**
628
+ * Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
629
+ * `SaveSessionChannelArtifact` mutation — the channel-context capability behind e.g. the
630
+ * whiteboard's "Save to artifacts". Best-effort: returns the created Artifact ID, or null
631
+ * on any failure (logged, never thrown). Uses the live session id, falling back to the
632
+ * teardown-captured one so "save my board" works right after the call ends.
633
+ */
634
+ async saveChannelArtifact(channelName, name, contentJson) {
635
+ const sessionId = this.agentSessionId ?? this.lastKnownSessionIdForSaves();
636
+ if (!sessionId || !name.trim() || !contentJson) {
637
+ return null;
638
+ }
639
+ try {
640
+ const result = await this.gql().ExecuteGQL(`mutation SaveSessionChannelArtifact($agentSessionId: String!, $channelName: String!, $name: String!, $contentJson: String!) {
641
+ SaveSessionChannelArtifact(agentSessionId: $agentSessionId, channelName: $channelName, name: $name, contentJson: $contentJson) {
642
+ Success
643
+ ErrorMessage
644
+ ArtifactID
645
+ ArtifactVersionID
646
+ }
647
+ }`, { agentSessionId: sessionId, channelName, name: name.trim(), contentJson });
648
+ const payload = result?.SaveSessionChannelArtifact;
649
+ if (!payload?.Success) {
650
+ console.warn(`[RealtimeSession] Save-as-artifact failed for '${channelName}': ${payload?.ErrorMessage ?? 'unknown error'}`);
651
+ return null;
652
+ }
653
+ return payload.ArtifactID ?? null;
654
+ }
655
+ catch (error) {
656
+ console.warn(`[RealtimeSession] Save-as-artifact errored for '${channelName}':`, error);
657
+ return null;
658
+ }
659
+ }
660
+ /** Most recent session id captured by the save pipeline (post-teardown saves). */
661
+ lastKnownSessionIdForSaves() {
662
+ for (const pending of this.pendingChannelSaves.values()) {
663
+ if (pending.SessionID) {
664
+ return pending.SessionID;
665
+ }
666
+ }
667
+ return null;
668
+ }
669
+ /**
670
+ * Schedules the DEBOUNCED state-of-record save for a channel: each request replaces the
671
+ * pending payload (latest state wins) and re-arms the timer; the session id is captured
672
+ * while live so the teardown flush can persist onto the just-closed session.
673
+ */
674
+ scheduleChannelSave(channelName, stateJson) {
675
+ const pending = this.pendingChannelSaves.get(channelName);
676
+ if (pending) {
677
+ clearTimeout(pending.Timer);
678
+ }
679
+ this.pendingChannelSaves.set(channelName, {
680
+ Timer: setTimeout(() => this.flushChannelSave(channelName), RealtimeSessionService.ChannelSaveDebounceMs),
681
+ StateJson: stateJson,
682
+ SessionID: this.agentSessionId ?? pending?.SessionID ?? null
683
+ });
684
+ }
685
+ /** Fires one pending channel save (best-effort; {@link SaveChannelState} logs failures). */
686
+ flushChannelSave(channelName) {
687
+ const pending = this.pendingChannelSaves.get(channelName);
688
+ if (!pending) {
689
+ return;
690
+ }
691
+ this.pendingChannelSaves.delete(channelName);
692
+ clearTimeout(pending.Timer);
693
+ void this.SaveChannelState(channelName, pending.StateJson, pending.SessionID);
694
+ }
695
+ /** Final teardown flush: persist every channel's unsaved state immediately. */
696
+ flushAllChannelSaves() {
697
+ for (const channelName of [...this.pendingChannelSaves.keys()]) {
698
+ this.flushChannelSave(channelName);
699
+ }
700
+ }
701
+ /** Disposes all channel plugins (errors contained per plugin) and clears the live set. */
702
+ disposeChannels() {
703
+ for (const plugin of this._activeChannels$.value) {
704
+ try {
705
+ plugin.Dispose();
706
+ }
707
+ catch (error) {
708
+ console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' Dispose failed:`, error);
709
+ }
710
+ }
711
+ if (this._activeChannels$.value.length > 0) {
712
+ this._activeChannels$.next([]);
713
+ }
714
+ }
715
+ // ── Realtime client resolution + wiring ────────────────────────────────────
716
+ /**
717
+ * Resolves the provider-direct realtime client for `provider` through the MJ
718
+ * ClassFactory — the client-side mirror of how server drivers are resolved from
719
+ * `BaseRealtimeModel`. Throws a clear error when no driver is registered for the
720
+ * provider (e.g. its Load function was never called).
721
+ */
722
+ createRealtimeClient(provider) {
723
+ const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeClient, provider);
724
+ if (!registration) {
725
+ throw new Error(`No realtime client registered for provider '${provider}'. ` +
726
+ `Ensure the provider's client driver package is imported and its Load function called.`);
727
+ }
728
+ const client = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeClient, provider);
729
+ if (!client) {
730
+ throw new Error(`Failed to instantiate the realtime client for provider '${provider}'`);
731
+ }
732
+ return client;
733
+ }
734
+ /** Builds the client-direct session config the realtime client connects with. */
735
+ buildClientConfig(session) {
736
+ return {
737
+ Provider: session.Provider,
738
+ Model: session.Model,
739
+ EphemeralToken: session.EphemeralToken,
740
+ ExpiresAt: session.ExpiresAt,
741
+ SessionConfig: this.parseSessionConfig(session.SessionConfigJson)
742
+ };
743
+ }
744
+ /**
745
+ * Parses the server-built session config JSON. On failure, logs and returns an empty
746
+ * object — the client treats an empty config as "nothing to apply", so the session
747
+ * still opens (mirroring the prior behavior of skipping the config update).
748
+ */
749
+ parseSessionConfig(sessionConfigJson) {
750
+ if (!sessionConfigJson) {
751
+ return {};
752
+ }
753
+ try {
754
+ return JSON.parse(sessionConfigJson);
755
+ }
756
+ catch (error) {
757
+ console.error('[RealtimeSession] Failed to parse/apply SessionConfigJson:', error);
758
+ return {};
759
+ }
760
+ }
761
+ /** Subscribes this service's policy handlers to the realtime client's events. */
762
+ wireClientHandlers(client) {
763
+ client.OnStateChange((state) => this.onClientStateChange(state));
764
+ client.OnTranscript((transcript) => {
765
+ void this.onClientTranscript(transcript);
766
+ });
767
+ client.OnToolCall((call) => {
768
+ void this.handleToolCall(call);
769
+ });
770
+ client.OnError((error) => {
771
+ console.error('[RealtimeSession] Provider error event:', error);
772
+ });
773
+ // Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
774
+ // the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
775
+ // without usage events simply never emit — registering is always safe.
776
+ client.OnUsage((usage) => this.onUsageDelta(usage));
777
+ // TRUE BARGE-IN (user input cut off active model output — the driver already stopped
778
+ // the speech): the user took the floor, so any pending/queued progress narration is
779
+ // stale — cancel it; the next progress event re-schedules at the session-global pace.
780
+ // HOST POLICY (deliberate): barge-in does NOT abort in-flight delegated runs — the
781
+ // narration design EXPECTS the user to keep talking while delegated work runs, so
782
+ // killing the work on speech would cancel exactly the jobs the user asked for.
783
+ // Explicit cancellation is a separate, intentional act: the overlay's per-card ✕
784
+ // calls {@link CancelDelegation} (server cancel channel) instead.
785
+ client.OnInterruption(() => {
786
+ this.cancelPendingNarration();
787
+ });
788
+ }
789
+ /** Maps a client state event onto the UI connection state. */
790
+ onClientStateChange(state) {
791
+ const mapped = this.mapClientState(state);
792
+ if (mapped) {
793
+ this._connectionState$.next(mapped);
794
+ }
795
+ }
796
+ /**
797
+ * Translates {@link RealtimeClientState} into {@link VoiceConnectionState}. `'connected'`
798
+ * is suppressed (the UI stays 'connecting' until the control channel opens → 'listening'),
799
+ * and `'closed'` never overwrites a terminal 'error' the service itself recorded.
800
+ */
801
+ mapClientState(state) {
802
+ switch (state) {
803
+ case 'connecting':
804
+ return 'connecting';
805
+ case 'connected':
806
+ return null;
807
+ case 'listening':
808
+ return 'listening';
809
+ case 'speaking':
810
+ return 'speaking';
811
+ case 'error':
812
+ return 'error';
813
+ case 'closed':
814
+ return this._connectionState$.value === 'error' ? null : 'closed';
815
+ }
816
+ }
817
+ /** True when the live control channel is usable (open and not torn down / failed). */
818
+ isSessionLive() {
819
+ const state = this._connectionState$.value;
820
+ return state === 'listening' || state === 'speaking' || state === 'thinking';
821
+ }
822
+ // ── Transcript policy ──────────────────────────────────────────────────────
823
+ /**
824
+ * Applies transcript policy to client transcript events. Interim deltas are ignored
825
+ * (the client already drives the speaking state). Final NORMAL assistant turns become
826
+ * captions + persisted transcripts; final NARRATION turns are EPHEMERAL by product
827
+ * decision — emitted on {@link DelegationNarration$} only, never a caption, never
828
+ * relayed/persisted. User turns ride the caption + relay path.
829
+ */
830
+ async onClientTranscript(transcript) {
831
+ if (!transcript.IsFinal) {
832
+ return;
833
+ }
834
+ if (transcript.Role === 'Assistant') {
835
+ if (transcript.Kind === 'narration') {
836
+ this._delegationNarration$.next({ Text: transcript.Text });
837
+ // Remember what was actually SAID so later updates build on it instead of repeating.
838
+ this.spokenNarrations.push(transcript.Text);
839
+ if (this.spokenNarrations.length > RealtimeSessionService.MaxPriorNarrations) {
840
+ this.spokenNarrations.shift();
841
+ }
842
+ }
843
+ else if (transcript.ReplacesPrevious) {
844
+ // CORRECTION (e.g. ElevenLabs post-barge-in re-finalization): this final
845
+ // SUPERSEDES the previous final assistant turn — replace the caption in place
846
+ // and tell the server to update the persisted turn instead of appending.
847
+ this.replaceLastCaption('Assistant', transcript.Text);
848
+ await this.relayTranscript('assistant', transcript.Text, true);
849
+ }
850
+ else {
851
+ this.appendCaption({ Role: 'Assistant', Text: transcript.Text });
852
+ await this.relayTranscript('assistant', transcript.Text);
853
+ }
854
+ }
855
+ else if (transcript.ReplacesPrevious) {
856
+ // STREAMING user transcription: providers like Grok emit the growing utterance as repeated
857
+ // events (each the full text so far), flagging all but the first ReplacesPrevious. Update the
858
+ // in-place User caption + persisted turn instead of stacking a new bubble per increment — the
859
+ // same correction semantics the assistant branch uses. (OpenAI sends one final → the else path.)
860
+ this.replaceLastCaption('User', transcript.Text);
861
+ await this.relayTranscript('user', transcript.Text, true);
862
+ }
863
+ else {
864
+ await this.onUserTranscript(transcript.Text);
865
+ }
866
+ }
867
+ /**
868
+ * Replaces the LAST caption of `role` in place (correction semantics); falls back to a
869
+ * plain append when no such caption exists yet (e.g. the superseded turn predates this
870
+ * client's caption window).
871
+ */
872
+ replaceLastCaption(role, text) {
873
+ const captions = this._captions$.value;
874
+ for (let i = captions.length - 1; i >= 0; i--) {
875
+ if (captions[i].Role === role) {
876
+ const next = [...captions];
877
+ next[i] = { Role: role, Text: text };
878
+ this._captions$.next(next);
879
+ return;
880
+ }
881
+ }
882
+ this.appendCaption({ Role: role, Text: text });
883
+ }
884
+ /** Finalizes the user turn: push a caption + relay the final transcript. */
885
+ async onUserTranscript(transcript) {
886
+ if (transcript.trim().length === 0) {
887
+ return;
888
+ }
889
+ if (this.firstUserTranscript === null) {
890
+ // First spoken user utterance — the naming seed for a session-created conversation.
891
+ this.firstUserTranscript = transcript;
892
+ }
893
+ this.appendCaption({ Role: 'User', Text: transcript });
894
+ await this.relayTranscript('user', transcript);
895
+ }
896
+ // ── Tool calling ───────────────────────────────────────────────────────────
897
+ /**
898
+ * Routes a provider tool call: names matching a registered client-tool prefix execute
899
+ * LOCALLY (UI tools — see {@link RegisterClientToolHandler}); everything else executes on
900
+ * the MJ server. Either way the result feeds back to the model via
901
+ * {@link BaseRealtimeClient.SendToolResult} so it speaks the outcome.
902
+ */
903
+ async handleToolCall(call) {
904
+ const clientHandler = this.findClientToolHandler(call.ToolName);
905
+ if (clientHandler) {
906
+ // Local UI tool: no server relay, no 'thinking' turn-state / narration burst — these
907
+ // are fast, in-browser surface mutations (e.g. drawing on the whiteboard).
908
+ const resultJson = await this.executeClientTool(clientHandler, call);
909
+ this.client?.SendToolResult(call.CallID, resultJson);
910
+ // Observability: record the channel tool call on the co-agent's run (run-only — NOT a chat
911
+ // turn). Without this the run shows speech but never the browser_/Whiteboard_ actions the
912
+ // co-agent took. Fire-and-forget; never disturbs the live surface mutation.
913
+ void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
914
+ return;
915
+ }
916
+ this._connectionState$.next('thinking');
917
+ if (this.inFlightCallIds.size === 0) {
918
+ // A fresh delegation burst: anchor the first-update delay and clear the digest
919
+ // buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
920
+ // is SESSION-global — sequential tool calls seconds apart must not re-arm the
921
+ // faster first-update path, which read as "no debounce") and spokenNarrations
922
+ // (so the story never repeats across closely-spaced calls).
923
+ this.delegationBurstStartedAt = Date.now();
924
+ this.narrationCount = 0;
925
+ this.pendingNarrationMessages = [];
926
+ this.lastNarratedTail = '';
927
+ }
928
+ this.inFlightCallIds.add(call.CallID);
929
+ try {
930
+ const resultJson = await this.executeSessionTool(call.CallID, call.ToolName, call.ArgumentsJson);
931
+ this.emitDelegationResult(call.CallID, resultJson);
932
+ this.client?.SendToolResult(call.CallID, resultJson);
933
+ }
934
+ catch (error) {
935
+ console.error('[RealtimeSession] Tool execution failed:', error);
936
+ // Feed the error back so the model can narrate it rather than going silent.
937
+ // success:false matters: ParseDelegationResultJson treats anything else as
938
+ // success, which would flip the overlay's working card to a SUCCESS card
939
+ // carrying the error text (matches the server broker's failure shape).
940
+ const errorJson = JSON.stringify({
941
+ success: false,
942
+ error: error instanceof Error ? error.message : String(error)
943
+ });
944
+ this.emitDelegationResult(call.CallID, errorJson);
945
+ this.client?.SendToolResult(call.CallID, errorJson);
946
+ }
947
+ }
948
+ /** Finds the registered client-tool handler whose prefix matches `toolName`, or `null`. */
949
+ findClientToolHandler(toolName) {
950
+ for (const [prefix, handler] of this.clientToolHandlers) {
951
+ if (toolName.startsWith(prefix)) {
952
+ return handler;
953
+ }
954
+ }
955
+ return null;
956
+ }
957
+ /**
958
+ * Executes one client-tool call through its handler, wrapping any thrown error into a
959
+ * `{ success: false, error }` JSON payload so the model can narrate the failure instead of
960
+ * the call going silent.
961
+ */
962
+ async executeClientTool(handler, call) {
963
+ try {
964
+ return await handler(call.ToolName, call.ArgumentsJson);
965
+ }
966
+ catch (error) {
967
+ console.error('[RealtimeSession] Client tool execution failed:', error);
968
+ return JSON.stringify({
969
+ success: false,
970
+ error: error instanceof Error ? error.message : String(error)
971
+ });
972
+ }
973
+ }
974
+ /**
975
+ * Emits a delegation result so the overlay's "working" card flips to a result card with real
976
+ * content. Parses the broker's `{success, output, runId}` | `{success:false, error}` shape via
977
+ * {@link ParseDelegationResultJson}; if it isn't JSON, surfaces the raw string. Only delegation
978
+ * cards (created from progress events) react — non-delegation tool results have no card and are
979
+ * harmlessly ignored downstream. The `runId` (the delegated `MJ: AI Agent Runs` record) rides
980
+ * along as {@link VoiceDelegationResult.RunID} for the overlay's dev links, and any `artifacts`
981
+ * ride along as {@link VoiceDelegationResult.Artifacts} for the surface panel's artifact tabs.
982
+ */
983
+ emitDelegationResult(callId, resultJson) {
984
+ // The result will be spoken next — a deferred interim update is now pointless
985
+ // (this is what keeps fast agents like Sage from narrating over their own answer),
986
+ // and any progress still in the PubSub pipe for this call is stale.
987
+ this.inFlightCallIds.delete(callId);
988
+ this.cancelPendingNarration();
989
+ if (this.cancelledCallIds.delete(callId)) {
990
+ // The user explicitly cancelled this call: its card already flipped to the
991
+ // "Cancelled by user" failed result, so the aborted run's late outcome must not
992
+ // overwrite it. (The tool result still flows back to the model via the caller.)
993
+ return;
994
+ }
995
+ const parsed = ParseDelegationResultJson(resultJson);
996
+ this._delegationResult$.next({
997
+ CallID: callId,
998
+ Success: parsed.Success,
999
+ Output: parsed.Output,
1000
+ RunID: parsed.RunID,
1001
+ Artifacts: parsed.Artifacts
1002
+ });
1003
+ }
1004
+ // ── Explicit delegation cancellation (server cancel channel) ───────────────
1005
+ /**
1006
+ * Cancels ONE in-flight delegated tool call — the overlay's per-card ✕ affordance.
1007
+ *
1008
+ * EXPLICIT USER INTENT ONLY (deliberate host policy): true barge-in never aborts
1009
+ * delegations — the narration design expects the user to talk while delegated work runs.
1010
+ * Calls the `CancelRealtimeSessionTool` mutation (ownership-gated server-side); when the
1011
+ * server reports it aborted the run, the card is flipped immediately to a FAILED
1012
+ * "Cancelled by user" result and the eventual late result from the aborted run is
1013
+ * suppressed (see {@link emitDelegationResult}).
1014
+ *
1015
+ * @returns `true` when the server aborted the in-flight run; `false` when there was
1016
+ * nothing to cancel (the work finished first — its real result is already racing in)
1017
+ * or the mutation failed (logged, never thrown).
1018
+ */
1019
+ async CancelDelegation(callId) {
1020
+ if (!this.agentSessionId || !this.inFlightCallIds.has(callId)) {
1021
+ return false;
1022
+ }
1023
+ const aborted = await this.cancelSessionTool(callId);
1024
+ if (aborted <= 0) {
1025
+ return false; // finished first / nothing in flight server-side — let the real result land
1026
+ }
1027
+ this.surfaceUserCancellation(callId);
1028
+ return true;
1029
+ }
1030
+ /**
1031
+ * Cancels EVERY in-flight delegated tool call for the active session (callId-less form of
1032
+ * the `CancelRealtimeSessionTool` mutation). Exposed for host policies that need a
1033
+ * sweep-cancel (e.g. an explicit "stop everything" affordance) — NOT wired to barge-in,
1034
+ * by the same deliberate policy as {@link CancelDelegation}.
1035
+ *
1036
+ * @returns The number of in-flight runs the server aborted (0 when nothing was tracked
1037
+ * in flight client-side, nothing was in flight server-side, or the mutation failed).
1038
+ */
1039
+ async CancelInFlightDelegations() {
1040
+ if (!this.agentSessionId || this.inFlightCallIds.size === 0) {
1041
+ return 0;
1042
+ }
1043
+ const aborted = await this.cancelSessionTool(null);
1044
+ if (aborted <= 0) {
1045
+ return 0;
1046
+ }
1047
+ for (const callId of [...this.inFlightCallIds]) {
1048
+ this.surfaceUserCancellation(callId);
1049
+ }
1050
+ return aborted;
1051
+ }
1052
+ /** Flips a cancelled call's card to the failed "Cancelled by user" result and suppresses the late real result. */
1053
+ surfaceUserCancellation(callId) {
1054
+ this.inFlightCallIds.delete(callId);
1055
+ this.cancelledCallIds.add(callId);
1056
+ this.cancelPendingNarration();
1057
+ this._delegationResult$.next({
1058
+ CallID: callId,
1059
+ Success: false,
1060
+ Output: 'Cancelled by user'
1061
+ });
1062
+ }
1063
+ /**
1064
+ * Calls the `CancelRealtimeSessionTool` mutation and unwraps its structured
1065
+ * `{ AbortedCount, Success, ErrorMessage }` result. Returns the aborted count —
1066
+ * 0 on a structured failure or a thrown transport error (both logged, never thrown).
1067
+ */
1068
+ async cancelSessionTool(callId) {
1069
+ try {
1070
+ const mutation = `
1071
+ mutation CancelRealtimeSessionTool($agentSessionId: String!, $callId: String) {
1072
+ CancelRealtimeSessionTool(agentSessionId: $agentSessionId, callId: $callId) {
1073
+ AbortedCount
1074
+ Success
1075
+ ErrorMessage
1076
+ }
1077
+ }
1078
+ `;
1079
+ const result = await this.gql().ExecuteGQL(mutation, { agentSessionId: this.agentSessionId, callId });
1080
+ const payload = result?.CancelRealtimeSessionTool;
1081
+ if (!payload?.Success) {
1082
+ console.warn(`[RealtimeSession] Cancel reported failure: ${payload?.ErrorMessage ?? 'unknown error'}`);
1083
+ return 0;
1084
+ }
1085
+ return typeof payload.AbortedCount === 'number' ? payload.AbortedCount : 0;
1086
+ }
1087
+ catch (error) {
1088
+ console.error('[RealtimeSession] Failed to cancel in-flight delegation(s):', error);
1089
+ return 0;
1090
+ }
1091
+ }
1092
+ // ── Session minting (GraphQL) ──────────────────────────────────────────────
1093
+ /** Calls the `StartRealtimeClientSession` mutation to obtain an ephemeral token + config. */
1094
+ async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson) {
1095
+ const mutation = `
1096
+ mutation StartRealtimeClientSession($targetAgentId: String!, $conversationId: String, $lastSessionId: String, $preferredModelId: String, $clientToolsJson: String, $coAgentId: String, $configOverridesJson: String) {
1097
+ StartRealtimeClientSession(targetAgentId: $targetAgentId, conversationId: $conversationId, lastSessionId: $lastSessionId, preferredModelId: $preferredModelId, clientToolsJson: $clientToolsJson, coAgentId: $coAgentId, configOverridesJson: $configOverridesJson) {
1098
+ AgentSessionId
1099
+ ConversationId
1100
+ Provider
1101
+ Model
1102
+ EphemeralToken
1103
+ ExpiresAt
1104
+ SessionConfigJson
1105
+ ModelName
1106
+ NarrationInstructionsTemplate
1107
+ PriorChannelStatesJson
1108
+ }
1109
+ }
1110
+ `;
1111
+ const variables = {
1112
+ targetAgentId,
1113
+ conversationId: conversationId ?? null,
1114
+ lastSessionId: lastSessionId ?? null,
1115
+ preferredModelId: preferredModelId ?? null,
1116
+ clientToolsJson: clientTools && clientTools.length > 0 ? JSON.stringify(clientTools) : null,
1117
+ coAgentId: coAgentId ?? null,
1118
+ configOverridesJson: configOverridesJson ?? null
1119
+ };
1120
+ const result = await this.gql().ExecuteGQL(mutation, variables);
1121
+ const payload = result?.StartRealtimeClientSession;
1122
+ if (!payload?.EphemeralToken) {
1123
+ throw new Error('StartRealtimeClientSession returned no ephemeral token');
1124
+ }
1125
+ return payload;
1126
+ }
1127
+ /** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
1128
+ async executeSessionTool(callId, toolName, argsJson) {
1129
+ if (!this.agentSessionId) {
1130
+ throw new Error('No active agent session for tool execution');
1131
+ }
1132
+ const mutation = `
1133
+ mutation ExecuteRealtimeSessionTool($agentSessionId: String!, $callId: String!, $toolName: String!, $argsJson: String!) {
1134
+ ExecuteRealtimeSessionTool(agentSessionId: $agentSessionId, callId: $callId, toolName: $toolName, argsJson: $argsJson)
1135
+ }
1136
+ `;
1137
+ const result = await this.gql().ExecuteGQL(mutation, {
1138
+ agentSessionId: this.agentSessionId,
1139
+ callId,
1140
+ toolName,
1141
+ argsJson
1142
+ });
1143
+ return result?.ExecuteRealtimeSessionTool ?? '{}';
1144
+ }
1145
+ /**
1146
+ * Persists an interactive channel's state of record (e.g. the whiteboard's serialized scene)
1147
+ * onto the session's `MJ: AI Agent Session Channels` row via `SaveSessionChannelState`.
1148
+ *
1149
+ * @param channelName The channel definition name (e.g. `'Whiteboard'`).
1150
+ * @param stateJson The serialized channel state.
1151
+ * @param agentSessionId Optional EXPLICIT session id. The debounced channel-save pipeline
1152
+ * captures the id while the session is live and passes it here, so the final teardown
1153
+ * flush still lands on the just-closed session. Falls back to the active session's id;
1154
+ * returns `false` when neither is available.
1155
+ * @returns Whether the server persisted the state. Failures are logged, never thrown — channel
1156
+ * persistence is best-effort and must not disturb the live call.
1157
+ */
1158
+ async SaveChannelState(channelName, stateJson, agentSessionId) {
1159
+ const sessionId = agentSessionId ?? this.agentSessionId;
1160
+ if (!sessionId) {
1161
+ return false;
1162
+ }
1163
+ try {
1164
+ const mutation = `
1165
+ mutation SaveSessionChannelState($agentSessionId: String!, $channelName: String!, $stateJson: String!) {
1166
+ SaveSessionChannelState(agentSessionId: $agentSessionId, channelName: $channelName, stateJson: $stateJson)
1167
+ }
1168
+ `;
1169
+ const result = await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, channelName, stateJson });
1170
+ return result?.SaveSessionChannelState ?? false;
1171
+ }
1172
+ catch (error) {
1173
+ console.error('[RealtimeSession] Failed to save channel state:', error);
1174
+ return false;
1175
+ }
1176
+ }
1177
+ // ── Transcript relay (GraphQL) ─────────────────────────────────────────────
1178
+ /**
1179
+ * Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
1180
+ * @param replacesPrevious CORRECTION semantics: the server updates the session's most
1181
+ * recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
1182
+ * post-barge-in `agent_response_correction`).
1183
+ */
1184
+ async relayTranscript(role, text, replacesPrevious = false) {
1185
+ if (!this.agentSessionId) {
1186
+ return;
1187
+ }
1188
+ try {
1189
+ const mutation = `
1190
+ mutation RelayRealtimeTranscript($agentSessionId: String!, $role: String!, $text: String!, $replacesPrevious: Boolean) {
1191
+ RelayRealtimeTranscript(agentSessionId: $agentSessionId, role: $role, text: $text, replacesPrevious: $replacesPrevious)
1192
+ }
1193
+ `;
1194
+ await this.gql().ExecuteGQL(mutation, {
1195
+ agentSessionId: this.agentSessionId,
1196
+ role,
1197
+ text,
1198
+ replacesPrevious
1199
+ });
1200
+ }
1201
+ catch (error) {
1202
+ console.error('[RealtimeSession] Failed to relay transcript:', error);
1203
+ }
1204
+ }
1205
+ /**
1206
+ * Relays a co-agent CHANNEL tool-call turn (browser_ / Whiteboard_ etc.) to the session's run for
1207
+ * observability via `RelayRealtimeToolTurn` — so the co-agent's AIPromptRun shows what it DID, not
1208
+ * just what it said. Run-only by design: deliberately NOT a `ConversationDetail` turn, so the chat
1209
+ * thread stays speech-only. Best-effort — a failed relay never disturbs the live call.
1210
+ */
1211
+ async relayToolTurn(toolName, argsJson, resultJson) {
1212
+ if (!this.agentSessionId) {
1213
+ return;
1214
+ }
1215
+ try {
1216
+ const mutation = `
1217
+ mutation RelayRealtimeToolTurn($agentSessionId: String!, $toolName: String!, $argsJson: String, $resultJson: String) {
1218
+ RelayRealtimeToolTurn(agentSessionId: $agentSessionId, toolName: $toolName, argsJson: $argsJson, resultJson: $resultJson)
1219
+ }
1220
+ `;
1221
+ await this.gql().ExecuteGQL(mutation, {
1222
+ agentSessionId: this.agentSessionId,
1223
+ toolName,
1224
+ argsJson,
1225
+ resultJson
1226
+ });
1227
+ }
1228
+ catch (error) {
1229
+ console.error('[RealtimeSession] Failed to relay tool turn:', error);
1230
+ }
1231
+ }
1232
+ // ── Usage telemetry relay (B7) ─────────────────────────────────────────────
1233
+ /**
1234
+ * Accumulates one usage DELTA from the realtime client (per-response token counts —
1235
+ * the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
1236
+ * values are clamped to 0; an all-zero delta is dropped without arming the timer.
1237
+ */
1238
+ onUsageDelta(usage) {
1239
+ const input = this.clampUsageDelta(usage.InputTokens);
1240
+ const output = this.clampUsageDelta(usage.OutputTokens);
1241
+ if (input === 0 && output === 0) {
1242
+ return;
1243
+ }
1244
+ this.pendingUsageInput += input;
1245
+ this.pendingUsageOutput += output;
1246
+ if (!this.usageFlushTimer) {
1247
+ this.usageFlushTimer = setTimeout(() => {
1248
+ this.usageFlushTimer = null;
1249
+ void this.flushPendingUsage();
1250
+ }, RealtimeSessionService.UsageFlushDebounceMs);
1251
+ }
1252
+ }
1253
+ /** Clamps a driver-reported token delta: undefined / negative / non-finite become 0. */
1254
+ clampUsageDelta(value) {
1255
+ return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
1256
+ }
1257
+ /**
1258
+ * Relays the accumulated usage deltas to the server via `RelayRealtimeUsage` (which
1259
+ * accumulates them onto the co-agent `AIPromptRun`). Best-effort: a failed relay
1260
+ * re-accumulates the captured deltas so the next debounce / teardown flush retries —
1261
+ * usage telemetry must never disturb the live call.
1262
+ *
1263
+ * @param agentSessionId Optional EXPLICIT session id (the teardown flush runs while the
1264
+ * live id is still set, but accepts it as a parameter for symmetry with channel saves).
1265
+ */
1266
+ async flushPendingUsage(agentSessionId) {
1267
+ const sessionId = agentSessionId ?? this.agentSessionId;
1268
+ const input = this.pendingUsageInput;
1269
+ const output = this.pendingUsageOutput;
1270
+ if (!sessionId || (input === 0 && output === 0)) {
1271
+ return;
1272
+ }
1273
+ this.pendingUsageInput = 0;
1274
+ this.pendingUsageOutput = 0;
1275
+ try {
1276
+ const mutation = `
1277
+ mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!) {
1278
+ RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens)
1279
+ }
1280
+ `;
1281
+ await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
1282
+ }
1283
+ catch (error) {
1284
+ console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
1285
+ // Re-accumulate so a later debounce / the teardown flush retries the same deltas.
1286
+ this.pendingUsageInput += input;
1287
+ this.pendingUsageOutput += output;
1288
+ }
1289
+ }
1290
+ /** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
1291
+ resetUsageRelay() {
1292
+ if (this.usageFlushTimer) {
1293
+ clearTimeout(this.usageFlushTimer);
1294
+ this.usageFlushTimer = null;
1295
+ }
1296
+ this.pendingUsageInput = 0;
1297
+ this.pendingUsageOutput = 0;
1298
+ }
1299
+ // ── Delegated-run progress streaming ───────────────────────────────────────
1300
+ /**
1301
+ * Subscribes to the server's push-status topic (scoped by the GraphQL transport
1302
+ * sessionId) to receive delegated-run progress for the active voice session.
1303
+ * Each matching event is surfaced on {@link DelegationProgress$} and narrated.
1304
+ */
1305
+ subscribeDelegationProgress() {
1306
+ if (this.delegationProgressSub) {
1307
+ return; // already subscribed for this session
1308
+ }
1309
+ const transportSessionId = this.gql().sessionId;
1310
+ this.lastDelegationNarrationAt = 0;
1311
+ this.delegationProgressSub = this.gql()
1312
+ .PushStatusUpdates(transportSessionId)
1313
+ .subscribe({
1314
+ next: (raw) => this.onDelegationStatusMessage(raw),
1315
+ error: (err) => console.error('[RealtimeSession] Delegation progress stream error:', err)
1316
+ });
1317
+ }
1318
+ /**
1319
+ * Parses one push-status message and routes it: a Remote Browser screencast frame goes to the active
1320
+ * Remote Browser channel's canvas; a delegation-progress event is dispatched + narrated. Other shapes
1321
+ * (normal agent-run streams) are ignored. Screencast frames are checked FIRST and short-circuit, so the
1322
+ * delegation path is untouched.
1323
+ */
1324
+ onDelegationStatusMessage(raw) {
1325
+ const frame = this.parseScreencastFrame(raw);
1326
+ if (frame) {
1327
+ this.routeScreencastFrame(frame);
1328
+ return;
1329
+ }
1330
+ const audio = this.parseAudioChunk(raw);
1331
+ if (audio) {
1332
+ this.routeAudioChunk(audio);
1333
+ return;
1334
+ }
1335
+ const progress = this.parseProgress(raw);
1336
+ if (progress) {
1337
+ this.dispatchProgress(progress);
1338
+ }
1339
+ }
1340
+ /**
1341
+ * Parses a push-status message and returns it only when it's a Remote Browser screencast frame for the
1342
+ * active session — otherwise `null` (ignored, so delegation progress falls through). Matched by
1343
+ * `resolver` + `type`, then scoped to THIS session by `agentSessionID`.
1344
+ */
1345
+ parseScreencastFrame(raw) {
1346
+ let payload;
1347
+ try {
1348
+ payload = JSON.parse(raw);
1349
+ }
1350
+ catch {
1351
+ return null;
1352
+ }
1353
+ const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
1354
+ payload?.type === 'RemoteBrowserScreencastFrame' &&
1355
+ payload?.agentSessionID === this.agentSessionId &&
1356
+ typeof payload?.dataBase64 === 'string';
1357
+ return matches ? payload : null;
1358
+ }
1359
+ /**
1360
+ * Forwards a screencast frame to the active Remote Browser channel plugin so it paints the frame on its
1361
+ * surface canvas. The plugin is found among the session's active channels by its `ChannelName`; located
1362
+ * via a structural guard so the service stays decoupled from the concrete channel class.
1363
+ */
1364
+ routeScreencastFrame(frame) {
1365
+ for (const channel of this._activeChannels$.value) {
1366
+ if (channel.ChannelName === 'Remote Browser' && this.hasOnScreencastFrame(channel)) {
1367
+ channel.OnScreencastFrame(frame.dataBase64);
1368
+ return;
1369
+ }
1370
+ }
1371
+ }
1372
+ /** Structural guard: true when the channel exposes an `OnScreencastFrame(dataBase64)` method. */
1373
+ hasOnScreencastFrame(channel) {
1374
+ return typeof channel.OnScreencastFrame === 'function';
1375
+ }
1376
+ /**
1377
+ * Parses a push-status message and returns it only when it's a Remote Browser audio chunk for the active
1378
+ * session — otherwise `null` (ignored). Matched by `resolver` + `type`, then scoped to THIS session by
1379
+ * `agentSessionID`.
1380
+ */
1381
+ parseAudioChunk(raw) {
1382
+ let payload;
1383
+ try {
1384
+ payload = JSON.parse(raw);
1385
+ }
1386
+ catch {
1387
+ return null;
1388
+ }
1389
+ const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
1390
+ payload?.type === 'RemoteBrowserAudioChunk' &&
1391
+ payload?.agentSessionID === this.agentSessionId &&
1392
+ typeof payload?.dataBase64 === 'string';
1393
+ return matches ? payload : null;
1394
+ }
1395
+ /**
1396
+ * Forwards an audio chunk to the active Remote Browser channel plugin so it plays the chunk through its
1397
+ * client-side audio player. The plugin is found among the session's active channels by its `ChannelName`;
1398
+ * located via a structural guard so the service stays decoupled from the concrete channel class.
1399
+ */
1400
+ routeAudioChunk(chunk) {
1401
+ for (const channel of this._activeChannels$.value) {
1402
+ if (channel.ChannelName === 'Remote Browser' && this.hasOnAudioChunk(channel)) {
1403
+ channel.OnAudioChunk({
1404
+ dataBase64: chunk.dataBase64,
1405
+ codec: chunk.codec,
1406
+ sampleRate: chunk.sampleRate,
1407
+ channels: chunk.channels,
1408
+ seq: chunk.seq,
1409
+ });
1410
+ return;
1411
+ }
1412
+ }
1413
+ }
1414
+ /** Structural guard: true when the channel exposes an `OnAudioChunk(chunk)` method. */
1415
+ hasOnAudioChunk(channel) {
1416
+ return typeof channel.OnAudioChunk === 'function';
1417
+ }
1418
+ /**
1419
+ * Parses a push-status message and returns it only when it's a delegation
1420
+ * progress event for the active voice session — otherwise `null` (ignored).
1421
+ */
1422
+ parseProgress(raw) {
1423
+ let payload;
1424
+ try {
1425
+ payload = JSON.parse(raw);
1426
+ }
1427
+ catch {
1428
+ return null; // non-JSON or unrelated frame
1429
+ }
1430
+ const matches = payload?.resolver === 'RealtimeClientSessionResolver' &&
1431
+ payload?.type === 'RealtimeDelegationProgress' &&
1432
+ payload?.agentSessionID === this.agentSessionId;
1433
+ if (!matches) {
1434
+ return null;
1435
+ }
1436
+ return {
1437
+ CallID: payload.callID,
1438
+ Step: payload.step,
1439
+ Message: payload.message,
1440
+ Percentage: payload.percentage
1441
+ };
1442
+ }
1443
+ /** Emits the progress to the UI observable and feeds it to the realtime model. */
1444
+ dispatchProgress(progress) {
1445
+ // Drop stale progress: PubSub delivery can lag the mutation result, so events for a
1446
+ // call that already completed (or was never seen) must not update cards or narrate.
1447
+ if (!this.inFlightCallIds.has(progress.CallID)) {
1448
+ return;
1449
+ }
1450
+ this._delegationProgress$.next(progress);
1451
+ this.narrateProgress(progress);
1452
+ }
1453
+ /**
1454
+ * Injects the progress into the model's context as a background note every time,
1455
+ * then (throttled) asks the model to briefly voice a reassuring update so the
1456
+ * background work doesn't sit in silence — without chattering or interrupting.
1457
+ */
1458
+ narrateProgress(progress) {
1459
+ const client = this.client;
1460
+ if (!client) {
1461
+ return;
1462
+ }
1463
+ client.SendContextNote(`[delegated-agent progress] ${progress.Message}`);
1464
+ // Floods of small updates AGGREGATE: each distinct message joins the digest buffer,
1465
+ // and ONE spoken update fires per window (first at ~5s into the burst, then every
1466
+ // ~8s). The buffer is discarded if the final result lands first.
1467
+ this.bufferNarrationMessage(progress.Message);
1468
+ if (this.pendingNarrationMessages.length > 0 && !this.narrationTimer) {
1469
+ this.narrationTimer = setTimeout(() => this.fireDeferredNarration(), this.nextNarrationDelayMs());
1470
+ }
1471
+ }
1472
+ /** Adds a progress message to the digest buffer (deduped, capped, oldest-first). */
1473
+ bufferNarrationMessage(message) {
1474
+ if (message === this.lastNarratedTail || this.pendingNarrationMessages.includes(message)) {
1475
+ return;
1476
+ }
1477
+ this.pendingNarrationMessages.push(message);
1478
+ if (this.pendingNarrationMessages.length > RealtimeSessionService.MaxDigestMessages) {
1479
+ this.pendingNarrationMessages.shift();
1480
+ }
1481
+ }
1482
+ /**
1483
+ * ms until the next spoken update is allowed. Two constraints, BOTH enforced:
1484
+ * - first update of a burst: no earlier than ~5s after the burst started;
1485
+ * - ~8s since the last spoken update, SESSION-global — so sequential tool calls
1486
+ * that reset the burst can never narrate faster than the interval.
1487
+ */
1488
+ nextNarrationDelayMs() {
1489
+ const now = Date.now();
1490
+ const firstAnchor = this.narrationCount === 0
1491
+ ? this.delegationBurstStartedAt + RealtimeSessionService.FirstNarrationDelayMs
1492
+ : 0;
1493
+ const spacingFloor = this.lastDelegationNarrationAt > 0
1494
+ ? this.lastDelegationNarrationAt + RealtimeSessionService.NarrationIntervalMs
1495
+ : 0;
1496
+ return Math.max(250, Math.max(firstAnchor, spacingFloor) - now);
1497
+ }
1498
+ /**
1499
+ * Speaks the aggregated progress digest — unless the work already finished (buffer
1500
+ * cancelled) or the model is busy / audio is still playing, in which case it retries
1501
+ * shortly with the buffer intact (work is still running, so the update stays relevant).
1502
+ */
1503
+ fireDeferredNarration() {
1504
+ this.narrationTimer = null;
1505
+ const client = this.client;
1506
+ if (this.pendingNarrationMessages.length === 0 || !client || this.inFlightCallIds.size === 0) {
1507
+ this.pendingNarrationMessages = [];
1508
+ return;
1509
+ }
1510
+ if (client.IsBusy || client.IsAudioPlaying) {
1511
+ this.narrationTimer = setTimeout(() => this.fireDeferredNarration(), RealtimeSessionService.NarrationBusyRetryMs);
1512
+ return;
1513
+ }
1514
+ const digest = this.pendingNarrationMessages.join(' → ');
1515
+ this.lastNarratedTail = this.pendingNarrationMessages[this.pendingNarrationMessages.length - 1];
1516
+ this.pendingNarrationMessages = [];
1517
+ this.narrationCount++;
1518
+ this.lastDelegationNarrationAt = Date.now();
1519
+ client.RequestSpokenUpdate(this.buildNarrationInstructions(digest));
1520
+ }
1521
+ /** Cancels any deferred narration — the result is about to be spoken, so it's moot. */
1522
+ cancelPendingNarration() {
1523
+ if (this.narrationTimer) {
1524
+ clearTimeout(this.narrationTimer);
1525
+ this.narrationTimer = null;
1526
+ }
1527
+ this.pendingNarrationMessages = [];
1528
+ }
1529
+ /**
1530
+ * Builds the one-off instructions for a short spoken update that conveys THIS specific
1531
+ * progress message naturally — strictly first person, since the co-agent owns the work.
1532
+ * The wording is DB-driven: the server-resolved `Realtime Co-Agent - Progress Narration`
1533
+ * template (substituting `{{ progressMessage }}`) when present, otherwise the built-in
1534
+ * fallback so deployments that haven't synced the prompt behave exactly as before.
1535
+ * The client tags the resulting turn as narration, keeping it EPHEMERAL — surfaced on
1536
+ * {@link DelegationNarration$} instead of becoming a caption / persisted ConversationDetail.
1537
+ */
1538
+ buildNarrationInstructions(digest) {
1539
+ return BuildNarrationInstructions(this.narrationTemplate, digest, {
1540
+ PriorNarrations: this.spokenNarrations.slice(-RealtimeSessionService.MaxPriorNarrations),
1541
+ UpdateNumber: this.narrationCount
1542
+ });
1543
+ }
1544
+ /** Tears down the delegation progress subscription and resets the narration throttle. */
1545
+ teardownDelegationProgress() {
1546
+ if (this.delegationProgressSub) {
1547
+ this.delegationProgressSub.unsubscribe();
1548
+ this.delegationProgressSub = null;
1549
+ }
1550
+ this.cancelPendingNarration();
1551
+ this.inFlightCallIds.clear();
1552
+ this.cancelledCallIds.clear();
1553
+ this.lastDelegationNarrationAt = 0;
1554
+ this.delegationBurstStartedAt = 0;
1555
+ this.narrationCount = 0;
1556
+ this.spokenNarrations = [];
1557
+ this.lastNarratedTail = '';
1558
+ }
1559
+ // ── Teardown ───────────────────────────────────────────────────────────────
1560
+ /**
1561
+ * Tears down all client resources and (optionally) closes the server session.
1562
+ * @param closeServerSession when true, calls `CloseAgentSession` on the server.
1563
+ */
1564
+ async teardown(closeServerSession) {
1565
+ this.teardownDelegationProgress();
1566
+ // Channels first: flush any unsaved channel state WHILE the live session id is still
1567
+ // set (the captured per-save id covers the race anyway), then dispose the plugins.
1568
+ this.flushAllChannelSaves();
1569
+ this.disposeChannels();
1570
+ // Defensive: stop the mic even when Connect never ran (the client also stops the
1571
+ // tracks it was handed — track.stop() is idempotent).
1572
+ this.localStream?.getTracks().forEach(t => t.stop());
1573
+ this.localStream = null;
1574
+ if (this.client) {
1575
+ await this.client.Disconnect();
1576
+ this.client = null;
1577
+ }
1578
+ // Final usage flush WHILE the live session id is still set (the relay mutation also
1579
+ // accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
1580
+ if (this.usageFlushTimer) {
1581
+ clearTimeout(this.usageFlushTimer);
1582
+ this.usageFlushTimer = null;
1583
+ }
1584
+ await this.flushPendingUsage(this.agentSessionId);
1585
+ this.resetUsageRelay();
1586
+ if (closeServerSession && this.agentSessionId) {
1587
+ await this.closeServerSession(this.agentSessionId);
1588
+ }
1589
+ // Capture the session id BEFORE we null it so the lifecycle emit carries it.
1590
+ // Skip emitting when there was no live session (defensive — teardown is safe
1591
+ // to call without an active session).
1592
+ const closedSessionId = this.agentSessionId;
1593
+ this.agentSessionId = null;
1594
+ this.narrationTemplate = null;
1595
+ this.clientToolHandlers.clear();
1596
+ this._modelName$.next(null);
1597
+ this.SetMinimized(false);
1598
+ this._active$.next(false);
1599
+ if (this._connectionState$.value !== 'error') {
1600
+ this._connectionState$.next('closed');
1601
+ }
1602
+ // Surface generic session-ended for the conversations runtime bridge.
1603
+ // `closeServerSession=true` means the user explicitly called EndVoiceSession;
1604
+ // `false` means teardown ran from a catch block (start path error path).
1605
+ if (closedSessionId) {
1606
+ this._sessionEnded$.next({
1607
+ sessionId: closedSessionId,
1608
+ reason: closeServerSession ? 'explicit' : 'error',
1609
+ });
1610
+ }
1611
+ }
1612
+ /** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
1613
+ async closeServerSession(agentSessionId) {
1614
+ try {
1615
+ const mutation = `
1616
+ mutation CloseAgentSession($agentSessionId: String!) {
1617
+ CloseAgentSession(agentSessionId: $agentSessionId)
1618
+ }
1619
+ `;
1620
+ await this.gql().ExecuteGQL(mutation, { agentSessionId });
1621
+ }
1622
+ catch (error) {
1623
+ console.error('[RealtimeSession] Failed to close server session:', error);
1624
+ }
1625
+ }
1626
+ // ── Helpers ────────────────────────────────────────────────────────────────
1627
+ /** Pushes a caption onto the live list (immutable update for change detection). */
1628
+ appendCaption(caption) {
1629
+ this._captions$.next([...this._captions$.value, caption]);
1630
+ }
1631
+ /** Resets reactive + internal state at the start of a session. */
1632
+ resetState() {
1633
+ this._captions$.next([]);
1634
+ this.SetMinimized(false);
1635
+ }
1636
+ /** The GraphQL provider used for relay mutations. */
1637
+ gql() {
1638
+ return this.Provider;
1639
+ }
1640
+ static ɵfac = function RealtimeSessionService_Factory(__ngFactoryType__) { return new (__ngFactoryType__ || RealtimeSessionService)(); };
1641
+ static ɵprov = /*@__PURE__*/ i0.ɵɵdefineInjectable({ token: RealtimeSessionService, factory: RealtimeSessionService.ɵfac, providedIn: 'root' });
1642
+ }
1643
+ (() => { (typeof ngDevMode === "undefined" || ngDevMode) && i0.ɵsetClassMetadata(RealtimeSessionService, [{
1644
+ type: Injectable,
1645
+ args: [{ providedIn: 'root' }]
1646
+ }], null, null); })();
1647
+ //# sourceMappingURL=realtime-session.service.js.map