@memberjunction/ng-conversations 5.40.1 → 5.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -0
- package/dist/__tests__/channel-optional-surface.test.d.ts +2 -0
- package/dist/__tests__/channel-optional-surface.test.d.ts.map +1 -0
- package/dist/__tests__/channel-optional-surface.test.js +53 -0
- package/dist/__tests__/channel-optional-surface.test.js.map +1 -0
- package/dist/__tests__/chat-events.test.d.ts +14 -0
- package/dist/__tests__/chat-events.test.d.ts.map +1 -0
- package/dist/__tests__/chat-events.test.js +109 -0
- package/dist/__tests__/chat-events.test.js.map +1 -0
- package/dist/__tests__/conversation-naming.test.d.ts +2 -0
- package/dist/__tests__/conversation-naming.test.d.ts.map +1 -0
- package/dist/__tests__/conversation-naming.test.js +110 -0
- package/dist/__tests__/conversation-naming.test.js.map +1 -0
- package/dist/__tests__/delegation-result-parser.test.d.ts +2 -0
- package/dist/__tests__/delegation-result-parser.test.d.ts.map +1 -0
- package/dist/__tests__/delegation-result-parser.test.js +107 -0
- package/dist/__tests__/delegation-result-parser.test.js.map +1 -0
- package/dist/__tests__/event-wiring.test.d.ts +15 -0
- package/dist/__tests__/event-wiring.test.d.ts.map +1 -0
- package/dist/__tests__/event-wiring.test.js +100 -0
- package/dist/__tests__/event-wiring.test.js.map +1 -0
- package/dist/__tests__/narration-template.test.d.ts +2 -0
- package/dist/__tests__/narration-template.test.d.ts.map +1 -0
- package/dist/__tests__/narration-template.test.js +76 -0
- package/dist/__tests__/narration-template.test.js.map +1 -0
- package/dist/__tests__/realtime-agent-picker-models.test.d.ts +2 -0
- package/dist/__tests__/realtime-agent-picker-models.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-agent-picker-models.test.js +49 -0
- package/dist/__tests__/realtime-agent-picker-models.test.js.map +1 -0
- package/dist/__tests__/realtime-audio-visuals.test.d.ts +2 -0
- package/dist/__tests__/realtime-audio-visuals.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-audio-visuals.test.js +123 -0
- package/dist/__tests__/realtime-audio-visuals.test.js.map +1 -0
- package/dist/__tests__/realtime-delegation-card-cancel.test.d.ts +2 -0
- package/dist/__tests__/realtime-delegation-card-cancel.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-delegation-card-cancel.test.js +48 -0
- package/dist/__tests__/realtime-delegation-card-cancel.test.js.map +1 -0
- package/dist/__tests__/realtime-disclosure.test.d.ts +2 -0
- package/dist/__tests__/realtime-disclosure.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-disclosure.test.js +164 -0
- package/dist/__tests__/realtime-disclosure.test.js.map +1 -0
- package/dist/__tests__/realtime-pairing.test.d.ts +2 -0
- package/dist/__tests__/realtime-pairing.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-pairing.test.js +207 -0
- package/dist/__tests__/realtime-pairing.test.js.map +1 -0
- package/dist/__tests__/realtime-review-lifecycle.test.d.ts +2 -0
- package/dist/__tests__/realtime-review-lifecycle.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-review-lifecycle.test.js +154 -0
- package/dist/__tests__/realtime-review-lifecycle.test.js.map +1 -0
- package/dist/__tests__/realtime-session-cancel-usage.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-cancel-usage.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-cancel-usage.test.js +230 -0
- package/dist/__tests__/realtime-session-cancel-usage.test.js.map +1 -0
- package/dist/__tests__/realtime-session-channels.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-channels.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-channels.test.js +252 -0
- package/dist/__tests__/realtime-session-channels.test.js.map +1 -0
- package/dist/__tests__/realtime-session-client-tools.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-client-tools.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-client-tools.test.js +103 -0
- package/dist/__tests__/realtime-session-client-tools.test.js.map +1 -0
- package/dist/__tests__/realtime-session-minimized.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-minimized.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-minimized.test.js +32 -0
- package/dist/__tests__/realtime-session-minimized.test.js.map +1 -0
- package/dist/__tests__/realtime-session-mint.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-mint.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-mint.test.js +69 -0
- package/dist/__tests__/realtime-session-mint.test.js.map +1 -0
- package/dist/__tests__/realtime-session-policy.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-policy.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-policy.test.js +303 -0
- package/dist/__tests__/realtime-session-policy.test.js.map +1 -0
- package/dist/__tests__/realtime-session-review.service.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-review.service.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-review.service.test.js +743 -0
- package/dist/__tests__/realtime-session-review.service.test.js.map +1 -0
- package/dist/__tests__/realtime-session-state.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-state.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-state.test.js +83 -0
- package/dist/__tests__/realtime-session-state.test.js.map +1 -0
- package/dist/__tests__/realtime-session-timeline-card.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-timeline-card.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-timeline-card.test.js +106 -0
- package/dist/__tests__/realtime-session-timeline-card.test.js.map +1 -0
- package/dist/__tests__/realtime-session-timeline.test.d.ts +2 -0
- package/dist/__tests__/realtime-session-timeline.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-session-timeline.test.js +142 -0
- package/dist/__tests__/realtime-session-timeline.test.js.map +1 -0
- package/dist/__tests__/realtime-sessions-adapter.test.d.ts +19 -0
- package/dist/__tests__/realtime-sessions-adapter.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-sessions-adapter.test.js +188 -0
- package/dist/__tests__/realtime-sessions-adapter.test.js.map +1 -0
- package/dist/__tests__/realtime-surface-panel-prefs.test.d.ts +2 -0
- package/dist/__tests__/realtime-surface-panel-prefs.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-surface-panel-prefs.test.js +100 -0
- package/dist/__tests__/realtime-surface-panel-prefs.test.js.map +1 -0
- package/dist/__tests__/realtime-surface-tabs-model.test.d.ts +2 -0
- package/dist/__tests__/realtime-surface-tabs-model.test.d.ts.map +1 -0
- package/dist/__tests__/realtime-surface-tabs-model.test.js +193 -0
- package/dist/__tests__/realtime-surface-tabs-model.test.js.map +1 -0
- package/dist/__tests__/remote-browser-audio-player.test.d.ts +2 -0
- package/dist/__tests__/remote-browser-audio-player.test.d.ts.map +1 -0
- package/dist/__tests__/remote-browser-audio-player.test.js +137 -0
- package/dist/__tests__/remote-browser-audio-player.test.js.map +1 -0
- package/dist/__tests__/remote-browser-channel.test.d.ts +2 -0
- package/dist/__tests__/remote-browser-channel.test.d.ts.map +1 -0
- package/dist/__tests__/remote-browser-channel.test.js +423 -0
- package/dist/__tests__/remote-browser-channel.test.js.map +1 -0
- package/dist/__tests__/slot-defaults.test.d.ts +24 -0
- package/dist/__tests__/slot-defaults.test.d.ts.map +1 -0
- package/dist/__tests__/slot-defaults.test.js +63 -0
- package/dist/__tests__/slot-defaults.test.js.map +1 -0
- package/dist/__tests__/user-authorization.test.d.ts +2 -0
- package/dist/__tests__/user-authorization.test.d.ts.map +1 -0
- package/dist/__tests__/user-authorization.test.js +97 -0
- package/dist/__tests__/user-authorization.test.js.map +1 -0
- package/dist/__tests__/voice-session-narration.test.d.ts +2 -0
- package/dist/__tests__/voice-session-narration.test.d.ts.map +1 -0
- package/dist/__tests__/voice-session-narration.test.js +609 -0
- package/dist/__tests__/voice-session-narration.test.js.map +1 -0
- package/dist/__tests__/whiteboard-artifact-viewer.test.d.ts +2 -0
- package/dist/__tests__/whiteboard-artifact-viewer.test.d.ts.map +1 -0
- package/dist/__tests__/whiteboard-artifact-viewer.test.js +101 -0
- package/dist/__tests__/whiteboard-artifact-viewer.test.js.map +1 -0
- package/dist/__tests__/whiteboard-channel.test.d.ts +2 -0
- package/dist/__tests__/whiteboard-channel.test.d.ts.map +1 -0
- package/dist/__tests__/whiteboard-channel.test.js +260 -0
- package/dist/__tests__/whiteboard-channel.test.js.map +1 -0
- package/dist/__tests__/whiteboard-restore-state.test.d.ts +2 -0
- package/dist/__tests__/whiteboard-restore-state.test.d.ts.map +1 -0
- package/dist/__tests__/whiteboard-restore-state.test.js +108 -0
- package/dist/__tests__/whiteboard-restore-state.test.js.map +1 -0
- package/dist/lib/components/conversation/conversation-chat-area.component.d.ts +205 -3
- package/dist/lib/components/conversation/conversation-chat-area.component.d.ts.map +1 -1
- package/dist/lib/components/conversation/conversation-chat-area.component.js +911 -342
- package/dist/lib/components/conversation/conversation-chat-area.component.js.map +1 -1
- package/dist/lib/components/mention/mention-dropdown.component.js +35 -17
- package/dist/lib/components/mention/mention-dropdown.component.js.map +1 -1
- package/dist/lib/components/mention/mention-editor.component.d.ts +4 -0
- package/dist/lib/components/mention/mention-editor.component.d.ts.map +1 -1
- package/dist/lib/components/mention/mention-editor.component.js +43 -19
- package/dist/lib/components/mention/mention-editor.component.js.map +1 -1
- package/dist/lib/components/message/message-input-box.component.d.ts +17 -1
- package/dist/lib/components/message/message-input-box.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-input-box.component.js +73 -15
- package/dist/lib/components/message/message-input-box.component.js.map +1 -1
- package/dist/lib/components/message/message-input.component.d.ts +142 -6
- package/dist/lib/components/message/message-input.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-input.component.js +328 -82
- package/dist/lib/components/message/message-input.component.js.map +1 -1
- package/dist/lib/components/message/message-item.component.d.ts +28 -3
- package/dist/lib/components/message/message-item.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-item.component.js +180 -108
- package/dist/lib/components/message/message-item.component.js.map +1 -1
- package/dist/lib/components/message/message-list.component.d.ts +81 -2
- package/dist/lib/components/message/message-list.component.d.ts.map +1 -1
- package/dist/lib/components/message/message-list.component.js +252 -87
- package/dist/lib/components/message/message-list.component.js.map +1 -1
- package/dist/lib/components/realtime/channels/base-realtime-channel-client.d.ts +282 -0
- package/dist/lib/components/realtime/channels/base-realtime-channel-client.d.ts.map +1 -0
- package/dist/lib/components/realtime/channels/base-realtime-channel-client.js +158 -0
- package/dist/lib/components/realtime/channels/base-realtime-channel-client.js.map +1 -0
- package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.d.ts +25 -0
- package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.js +140 -0
- package/dist/lib/components/realtime/channels/channel-onboarding-panel.component.js.map +1 -0
- package/dist/lib/components/realtime/channels/realtime-channel-pane.component.d.ts +35 -0
- package/dist/lib/components/realtime/channels/realtime-channel-pane.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/channels/realtime-channel-pane.component.js +58 -0
- package/dist/lib/components/realtime/channels/realtime-channel-pane.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-activity-rail.component.d.ts +63 -0
- package/dist/lib/components/realtime/realtime-activity-rail.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-activity-rail.component.js +260 -0
- package/dist/lib/components/realtime/realtime-activity-rail.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts +117 -0
- package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-agent-banner.component.js +504 -0
- package/dist/lib/components/realtime/realtime-agent-banner.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-agent-picker.component.d.ts +168 -0
- package/dist/lib/components/realtime/realtime-agent-picker.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-agent-picker.component.js +556 -0
- package/dist/lib/components/realtime/realtime-agent-picker.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-audio-visuals.d.ts +97 -0
- package/dist/lib/components/realtime/realtime-audio-visuals.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-audio-visuals.js +139 -0
- package/dist/lib/components/realtime/realtime-audio-visuals.js.map +1 -0
- package/dist/lib/components/realtime/realtime-channel-strip.component.d.ts +29 -0
- package/dist/lib/components/realtime/realtime-channel-strip.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-channel-strip.component.js +69 -0
- package/dist/lib/components/realtime/realtime-channel-strip.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-composer.component.d.ts +65 -0
- package/dist/lib/components/realtime/realtime-composer.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-composer.component.js +256 -0
- package/dist/lib/components/realtime/realtime-composer.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-delegation-card.component.d.ts +71 -0
- package/dist/lib/components/realtime/realtime-delegation-card.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-delegation-card.component.js +324 -0
- package/dist/lib/components/realtime/realtime-delegation-card.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-disclosure.d.ts +135 -0
- package/dist/lib/components/realtime/realtime-disclosure.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-disclosure.js +188 -0
- package/dist/lib/components/realtime/realtime-disclosure.js.map +1 -0
- package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts +491 -0
- package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-session-overlay.component.js +1274 -0
- package/dist/lib/components/realtime/realtime-session-overlay.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-session-state.d.ts +191 -0
- package/dist/lib/components/realtime/realtime-session-state.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-session-state.js +244 -0
- package/dist/lib/components/realtime/realtime-session-state.js.map +1 -0
- package/dist/lib/components/realtime/realtime-session-thread.component.d.ts +56 -0
- package/dist/lib/components/realtime/realtime-session-thread.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-session-thread.component.js +246 -0
- package/dist/lib/components/realtime/realtime-session-thread.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-session-timeline-card.component.d.ts +51 -0
- package/dist/lib/components/realtime/realtime-session-timeline-card.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-session-timeline-card.component.js +193 -0
- package/dist/lib/components/realtime/realtime-session-timeline-card.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-surface-panel-prefs.d.ts +77 -0
- package/dist/lib/components/realtime/realtime-surface-panel-prefs.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-surface-panel-prefs.js +114 -0
- package/dist/lib/components/realtime/realtime-surface-panel-prefs.js.map +1 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.component.d.ts +173 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.component.js +496 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.component.js.map +1 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.model.d.ts +181 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.model.d.ts.map +1 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.model.js +223 -0
- package/dist/lib/components/realtime/realtime-surface-tabs.model.js.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.d.ts +163 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.d.ts.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.js +309 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.js.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts +168 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js +524 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.d.ts +346 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.js +851 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-surface.component.js.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts +86 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.d.ts.map +1 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js +210 -0
- package/dist/lib/components/realtime/remote-browser/remote-browser-tools.js.map +1 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.d.ts +48 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.d.ts.map +1 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.js +180 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-artifact-viewer.component.js.map +1 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts +119 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.d.ts.map +1 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js +274 -0
- package/dist/lib/components/realtime/whiteboard/whiteboard-channel.js.map +1 -0
- package/dist/lib/components/slots/mj-chat-agent-presence-default.component.d.ts +11 -0
- package/dist/lib/components/slots/mj-chat-agent-presence-default.component.d.ts.map +1 -0
- package/dist/lib/components/slots/mj-chat-agent-presence-default.component.js +98 -0
- package/dist/lib/components/slots/mj-chat-agent-presence-default.component.js.map +1 -0
- package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.d.ts +9 -0
- package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.d.ts.map +1 -0
- package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.js +35 -0
- package/dist/lib/components/slots/mj-chat-demonstration-surface-default.component.js.map +1 -0
- package/dist/lib/components/slots/mj-chat-empty-state-default.component.d.ts +28 -0
- package/dist/lib/components/slots/mj-chat-empty-state-default.component.d.ts.map +1 -0
- package/dist/lib/components/slots/mj-chat-empty-state-default.component.js +104 -0
- package/dist/lib/components/slots/mj-chat-empty-state-default.component.js.map +1 -0
- package/dist/lib/components/slots/mj-chat-header-default.component.d.ts +11 -0
- package/dist/lib/components/slots/mj-chat-header-default.component.d.ts.map +1 -0
- package/dist/lib/components/slots/mj-chat-header-default.component.js +103 -0
- package/dist/lib/components/slots/mj-chat-header-default.component.js.map +1 -0
- package/dist/lib/components/slots/mj-chat-message-bubble-default.component.d.ts +15 -0
- package/dist/lib/components/slots/mj-chat-message-bubble-default.component.d.ts.map +1 -0
- package/dist/lib/components/slots/mj-chat-message-bubble-default.component.js +73 -0
- package/dist/lib/components/slots/mj-chat-message-bubble-default.component.js.map +1 -0
- package/dist/lib/components/slots/mj-chat-message-extra-default.component.d.ts +9 -0
- package/dist/lib/components/slots/mj-chat-message-extra-default.component.d.ts.map +1 -0
- package/dist/lib/components/slots/mj-chat-message-extra-default.component.js +34 -0
- package/dist/lib/components/slots/mj-chat-message-extra-default.component.js.map +1 -0
- package/dist/lib/components/slots/slot-interfaces.d.ts +95 -0
- package/dist/lib/components/slots/slot-interfaces.d.ts.map +1 -0
- package/dist/lib/components/slots/slot-interfaces.js +18 -0
- package/dist/lib/components/slots/slot-interfaces.js.map +1 -0
- package/dist/lib/components/workspace/conversation-workspace.component.d.ts +11 -0
- package/dist/lib/components/workspace/conversation-workspace.component.d.ts.map +1 -1
- package/dist/lib/components/workspace/conversation-workspace.component.js +28 -4
- package/dist/lib/components/workspace/conversation-workspace.component.js.map +1 -1
- package/dist/lib/conversations.module.d.ts +12 -1
- package/dist/lib/conversations.module.d.ts.map +1 -1
- package/dist/lib/conversations.module.js +93 -5
- package/dist/lib/conversations.module.js.map +1 -1
- package/dist/lib/directives/chat-slot.directive.d.ts +44 -0
- package/dist/lib/directives/chat-slot.directive.d.ts.map +1 -0
- package/dist/lib/directives/chat-slot.directive.js +54 -0
- package/dist/lib/directives/chat-slot.directive.js.map +1 -0
- package/dist/lib/events/chat-events.d.ts +137 -0
- package/dist/lib/events/chat-events.d.ts.map +1 -0
- package/dist/lib/events/chat-events.js +189 -0
- package/dist/lib/events/chat-events.js.map +1 -0
- package/dist/lib/models/conversation-state.model.d.ts +2 -1
- package/dist/lib/models/conversation-state.model.d.ts.map +1 -1
- package/dist/lib/models/conversation-state.model.js.map +1 -1
- package/dist/lib/services/artifact-state.service.d.ts.map +1 -1
- package/dist/lib/services/artifact-state.service.js +23 -6
- package/dist/lib/services/artifact-state.service.js.map +1 -1
- package/dist/lib/services/conversation-agent.service.d.ts +60 -74
- package/dist/lib/services/conversation-agent.service.d.ts.map +1 -1
- package/dist/lib/services/conversation-agent.service.js +100 -313
- package/dist/lib/services/conversation-agent.service.js.map +1 -1
- package/dist/lib/services/conversation-bridge.service.d.ts +11 -70
- package/dist/lib/services/conversation-bridge.service.d.ts.map +1 -1
- package/dist/lib/services/conversation-bridge.service.js +51 -85
- package/dist/lib/services/conversation-bridge.service.js.map +1 -1
- package/dist/lib/services/conversation-naming.d.ts +63 -0
- package/dist/lib/services/conversation-naming.d.ts.map +1 -0
- package/dist/lib/services/conversation-naming.js +58 -0
- package/dist/lib/services/conversation-naming.js.map +1 -0
- package/dist/lib/services/conversation-streaming.service.d.ts +24 -154
- package/dist/lib/services/conversation-streaming.service.d.ts.map +1 -1
- package/dist/lib/services/conversation-streaming.service.js +39 -361
- package/dist/lib/services/conversation-streaming.service.js.map +1 -1
- package/dist/lib/services/conversations-runtime-bootstrap.service.d.ts +10 -0
- package/dist/lib/services/conversations-runtime-bootstrap.service.d.ts.map +1 -0
- package/dist/lib/services/conversations-runtime-bootstrap.service.js +104 -0
- package/dist/lib/services/conversations-runtime-bootstrap.service.js.map +1 -0
- package/dist/lib/services/delegation-result-parser.d.ts +45 -0
- package/dist/lib/services/delegation-result-parser.d.ts.map +1 -0
- package/dist/lib/services/delegation-result-parser.js +48 -0
- package/dist/lib/services/delegation-result-parser.js.map +1 -0
- package/dist/lib/services/mention-autocomplete.service.d.ts +19 -4
- package/dist/lib/services/mention-autocomplete.service.d.ts.map +1 -1
- package/dist/lib/services/mention-autocomplete.service.js +65 -4
- package/dist/lib/services/mention-autocomplete.service.js.map +1 -1
- package/dist/lib/services/mention-parser.service.d.ts +8 -53
- package/dist/lib/services/mention-parser.service.d.ts.map +1 -1
- package/dist/lib/services/mention-parser.service.js +32 -243
- package/dist/lib/services/mention-parser.service.js.map +1 -1
- package/dist/lib/services/narration-template.d.ts +42 -0
- package/dist/lib/services/narration-template.d.ts.map +1 -0
- package/dist/lib/services/narration-template.js +73 -0
- package/dist/lib/services/narration-template.js.map +1 -0
- package/dist/lib/services/realtime-pairing.d.ts +120 -0
- package/dist/lib/services/realtime-pairing.d.ts.map +1 -0
- package/dist/lib/services/realtime-pairing.js +150 -0
- package/dist/lib/services/realtime-pairing.js.map +1 -0
- package/dist/lib/services/realtime-session-review.service.d.ts +233 -0
- package/dist/lib/services/realtime-session-review.service.d.ts.map +1 -0
- package/dist/lib/services/realtime-session-review.service.js +417 -0
- package/dist/lib/services/realtime-session-review.service.js.map +1 -0
- package/dist/lib/services/realtime-session.service.d.ts +739 -0
- package/dist/lib/services/realtime-session.service.d.ts.map +1 -0
- package/dist/lib/services/realtime-session.service.js +1647 -0
- package/dist/lib/services/realtime-session.service.js.map +1 -0
- package/dist/lib/services/realtime-sessions-adapter.d.ts +54 -0
- package/dist/lib/services/realtime-sessions-adapter.d.ts.map +1 -0
- package/dist/lib/services/realtime-sessions-adapter.js +154 -0
- package/dist/lib/services/realtime-sessions-adapter.js.map +1 -0
- package/dist/lib/services/user-authorization.d.ts +67 -0
- package/dist/lib/services/user-authorization.d.ts.map +1 -0
- package/dist/lib/services/user-authorization.js +66 -0
- package/dist/lib/services/user-authorization.js.map +1 -0
- package/dist/lib/utils/realtime-session-timeline.d.ts +84 -0
- package/dist/lib/utils/realtime-session-timeline.d.ts.map +1 -0
- package/dist/lib/utils/realtime-session-timeline.js +94 -0
- package/dist/lib/utils/realtime-session-timeline.js.map +1 -0
- package/dist/public-api.d.ts +41 -0
- package/dist/public-api.d.ts.map +1 -1
- package/dist/public-api.js +50 -0
- package/dist/public-api.js.map +1 -1
- package/package.json +27 -24
- package/dist/__tests__/conversation-bridge.service.test.d.ts +0 -2
- package/dist/__tests__/conversation-bridge.service.test.d.ts.map +0 -1
- package/dist/__tests__/conversation-bridge.service.test.js +0 -98
- package/dist/__tests__/conversation-bridge.service.test.js.map +0 -1
- package/dist/__tests__/mention-parser.test.d.ts +0 -2
- package/dist/__tests__/mention-parser.test.d.ts.map +0 -1
- package/dist/__tests__/mention-parser.test.js +0 -154
- package/dist/__tests__/mention-parser.test.js.map +0 -1
|
@@ -0,0 +1,1647 @@
|
|
|
1
|
+
import { Injectable } from '@angular/core';
|
|
2
|
+
import { BehaviorSubject, Subject } from 'rxjs';
|
|
3
|
+
import { Metadata } from '@memberjunction/core';
|
|
4
|
+
import { AIEngineBase } from '@memberjunction/ai-engine-base';
|
|
5
|
+
import { MJGlobal } from '@memberjunction/global';
|
|
6
|
+
import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient } from '@memberjunction/ai-realtime-client';
|
|
7
|
+
import { BuildNarrationInstructions } from './narration-template';
|
|
8
|
+
import { ParseDelegationResultJson } from './delegation-result-parser';
|
|
9
|
+
import { BaseRealtimeChannelClient } from '../components/realtime/channels/base-realtime-channel-client';
|
|
10
|
+
import * as i0 from "@angular/core";
|
|
11
|
+
// Tree-shaking prevention: the OpenAI client is resolved dynamically through the
|
|
12
|
+
// ClassFactory (by the server-reported Provider key), so this static call is what keeps
|
|
13
|
+
// its @RegisterClass side effect from being eliminated by the bundler.
|
|
14
|
+
// NOTE: the interactive-channel plugins (resolved dynamically from the `MJ: AI Agent
|
|
15
|
+
// Channels` registry by ClientPluginClass key) get the same treatment, but their Load
|
|
16
|
+
// calls live in `conversations.module.ts` — plugins carry Angular surface COMPONENTS,
|
|
17
|
+
// and this service stays component-free (it must stay importable in plain-node tests).
|
|
18
|
+
LoadOpenAIRealtimeClient();
|
|
19
|
+
LoadGeminiRealtimeClient();
|
|
20
|
+
LoadElevenLabsRealtimeClient();
|
|
21
|
+
LoadAssemblyAIRealtimeClient();
|
|
22
|
+
LoadxAIRealtimeClient();
|
|
23
|
+
/**
|
|
24
|
+
* Drives a **client-direct** real-time voice session: the browser mints an ephemeral
|
|
25
|
+
* token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
|
|
26
|
+
* frames never transit the MJ server (low latency); only tool calls and final
|
|
27
|
+
* transcripts are relayed back to MJ over GraphQL.
|
|
28
|
+
*
|
|
29
|
+
* This service is PROVIDER-AGNOSTIC policy/orchestration. All provider wire concerns
|
|
30
|
+
* (transport, event translation, the response state machine, narration-kind tagging,
|
|
31
|
+
* playback tracking) live in a {@link BaseRealtimeClient} driver resolved through the
|
|
32
|
+
* MJ ClassFactory by the server-reported `Provider` key (e.g. `'openai'` →
|
|
33
|
+
* `OpenAIRealtimeClient`). Future providers (Gemini Live, …) snap in by registering a
|
|
34
|
+
* new driver — this service does not change.
|
|
35
|
+
*
|
|
36
|
+
* The Realtime Co-Agent (server-side) fronts the conversation's current agent — the server
|
|
37
|
+
* bakes the companion instructions + tool set into `SessionConfigJson`, which the client
|
|
38
|
+
* driver applies verbatim.
|
|
39
|
+
*
|
|
40
|
+
* Lifecycle: {@link StartVoiceSession} → live duplex → {@link EndVoiceSession}.
|
|
41
|
+
*/
|
|
42
|
+
export class RealtimeSessionService {
|
|
43
|
+
// ── Reactive UI state ──────────────────────────────────────────────────────
|
|
44
|
+
_connectionState$ = new BehaviorSubject('closed');
|
|
45
|
+
_captions$ = new BehaviorSubject([]);
|
|
46
|
+
_active$ = new BehaviorSubject(false);
|
|
47
|
+
_delegationProgress$ = new Subject();
|
|
48
|
+
_delegationResult$ = new Subject();
|
|
49
|
+
_delegationNarration$ = new Subject();
|
|
50
|
+
_agentName$ = new BehaviorSubject('Sage');
|
|
51
|
+
_modelName$ = new BehaviorSubject(null);
|
|
52
|
+
_minimized$ = new BehaviorSubject(false);
|
|
53
|
+
_activeChannels$ = new BehaviorSubject([]);
|
|
54
|
+
_channelFocus$ = new Subject();
|
|
55
|
+
// ─── Generic session-lifecycle events (consumed by RealtimeSessionsAdapter to
|
|
56
|
+
// bridge into @memberjunction/conversations-runtime's framework-agnostic
|
|
57
|
+
// SessionsObserver). Why not derive from Active$ + agentSessionId? Because
|
|
58
|
+
// Active$ flips true before mintSession resolves and sets agentSessionId —
|
|
59
|
+
// a naive Active$ subscription would emit session-started with sessionId === null.
|
|
60
|
+
// Emitting explicitly avoids the race entirely. ───
|
|
61
|
+
_sessionStarted$ = new Subject();
|
|
62
|
+
_sessionEnded$ = new Subject();
|
|
63
|
+
_channelActivity$ = new Subject();
|
|
64
|
+
/** Current connection / turn state. */
|
|
65
|
+
ConnectionState$ = this._connectionState$.asObservable();
|
|
66
|
+
/** Live captions for both sides of the conversation. */
|
|
67
|
+
Captions$ = this._captions$.asObservable();
|
|
68
|
+
/** True while a session is open (mic button active, overlay shown). */
|
|
69
|
+
Active$ = this._active$.asObservable();
|
|
70
|
+
/**
|
|
71
|
+
* Progress updates from a delegated agent run (e.g. Sage) while the realtime model waits on it.
|
|
72
|
+
* The future overlay subscribes to render a "working" card; the model also narrates these aloud.
|
|
73
|
+
*/
|
|
74
|
+
DelegationProgress$ = this._delegationProgress$.asObservable();
|
|
75
|
+
/** Terminal result of a delegation, so the overlay can complete the working card with real content. */
|
|
76
|
+
DelegationResult$ = this._delegationResult$.asObservable();
|
|
77
|
+
/**
|
|
78
|
+
* EPHEMERAL spoken progress narrations (see {@link VoiceDelegationNarration}). These are
|
|
79
|
+
* deliberately kept OUT of {@link Captions$} and never relayed/persisted — the overlay
|
|
80
|
+
* renders them as a transient "live note" near the active working card.
|
|
81
|
+
*/
|
|
82
|
+
DelegationNarration$ = this._delegationNarration$.asObservable();
|
|
83
|
+
/** Display name of the agent the active session fronts (set at session start). */
|
|
84
|
+
AgentName$ = this._agentName$.asObservable();
|
|
85
|
+
/**
|
|
86
|
+
* Display name of the realtime MODEL the active session runs on (server-reported at session
|
|
87
|
+
* start, e.g. "GPT Realtime 2"). `null` before a session starts / when the server didn't report
|
|
88
|
+
* one. The overlay banner shows it subtly next to the agent identity.
|
|
89
|
+
*/
|
|
90
|
+
ModelName$ = this._modelName$.asObservable();
|
|
91
|
+
/**
|
|
92
|
+
* True while the active call overlay is MINIMIZED to the host's floating "on call" pill
|
|
93
|
+
* (e.g. after a dev link navigated away). The mic and session stay fully live — this is
|
|
94
|
+
* pure presentation state, reset to `false` at session start and teardown.
|
|
95
|
+
*/
|
|
96
|
+
Minimized$ = this._minimized$.asObservable();
|
|
97
|
+
/**
|
|
98
|
+
* The session's ACTIVE interactive-channel plugins, resolved from the `MJ: AI Agent
|
|
99
|
+
* Channels` registry at session start (one instance per session, per channel). Emits
|
|
100
|
+
* `[]` before a session starts and after teardown. The overlay subscribes to register
|
|
101
|
+
* one surface tab per plugin — it never knows any concrete channel type.
|
|
102
|
+
*/
|
|
103
|
+
ActiveChannels$ = this._activeChannels$.asObservable();
|
|
104
|
+
/**
|
|
105
|
+
* Channel requests to enter / leave the FOCUS layout (see
|
|
106
|
+
* {@link RealtimeChannelFocusEvent}). Fired when a plugin calls its host context's
|
|
107
|
+
* `SetFocusMode` — e.g. the whiteboard's "Focus board" toggle.
|
|
108
|
+
*/
|
|
109
|
+
ChannelFocus$ = this._channelFocus$.asObservable();
|
|
110
|
+
/**
|
|
111
|
+
* Fired EXACTLY ONCE per session after both `agentSessionId` is set AND the
|
|
112
|
+
* realtime client is connected. Carries the server-issued `sessionId` and the
|
|
113
|
+
* `ChannelName` of each plugin resolved at session mint. Consumed by
|
|
114
|
+
* `RealtimeSessionsAdapter` (in this package) to feed
|
|
115
|
+
* `@memberjunction/conversations-runtime`'s `SessionsObserver`.
|
|
116
|
+
*
|
|
117
|
+
* **Why this exists separately from `Active$`:** `Active$` flips `true` BEFORE
|
|
118
|
+
* `mintSession` resolves, so `agentSessionId` is still `null` at that moment.
|
|
119
|
+
* Subscribers correlating `(Active$, agentSessionId)` would race; this event
|
|
120
|
+
* removes the race.
|
|
121
|
+
*/
|
|
122
|
+
SessionStarted$ = this._sessionStarted$.asObservable();
|
|
123
|
+
/**
|
|
124
|
+
* Fired EXACTLY ONCE per session as teardown begins, with the prior
|
|
125
|
+
* `agentSessionId` (so subscribers can correlate against `SessionStarted$`'s
|
|
126
|
+
* sessionId) and the client-distinguishable reason — `'explicit'` when the
|
|
127
|
+
* user called `EndVoiceSession`, `'error'` when teardown ran from a catch
|
|
128
|
+
* block. Server-side close paths (janitor, shutdown) do NOT propagate here —
|
|
129
|
+
* they happen out-of-process and have no client push channel today.
|
|
130
|
+
*/
|
|
131
|
+
SessionEnded$ = this._sessionEnded$.asObservable();
|
|
132
|
+
/**
|
|
133
|
+
* Fires with the channel PLUGIN every time the agent ACTS on that channel (a tool call
|
|
134
|
+
* was routed to its local executor — e.g. the agent drew on the whiteboard). The overlay
|
|
135
|
+
* uses the FIRST emission per channel to auto-reveal + focus the channel's surface tab,
|
|
136
|
+
* so the user discovers the surface the moment the agent starts using it. Finer-grained
|
|
137
|
+
* than {@link SessionStarted$}/{@link SessionEnded$} (per tool call, not per session).
|
|
138
|
+
*/
|
|
139
|
+
ChannelActivity$ = this._channelActivity$.asObservable();
|
|
140
|
+
/** Synchronous access to the session's active interactive-channel plugins. */
|
|
141
|
+
get ActiveChannels() {
|
|
142
|
+
return this._activeChannels$.value;
|
|
143
|
+
}
|
|
144
|
+
/** Synchronous access to the display name of the agent the active session fronts. */
|
|
145
|
+
get CurrentAgentName() {
|
|
146
|
+
return this._agentName$.value;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* ID of the active server-side agent session (`MJ: AI Agent Sessions`), or `null` when no
|
|
150
|
+
* session is open / the session hasn't been minted yet. Powers the overlay's gear-gated
|
|
151
|
+
* "Open session" dev link.
|
|
152
|
+
*/
|
|
153
|
+
/** Conversation id the SERVER created for this session (null when the host supplied one). */
|
|
154
|
+
createdConversationId = null;
|
|
155
|
+
/** The session's conversation id (supplied or server-created). */
|
|
156
|
+
sessionConversationId = null;
|
|
157
|
+
/** First final user utterance of the live session (the naming seed). */
|
|
158
|
+
firstUserTranscript = null;
|
|
159
|
+
/**
|
|
160
|
+
* When the active/last session CREATED its conversation (started without one), the new
|
|
161
|
+
* conversation's id — the host uses it to refresh the cached list, conditionally select
|
|
162
|
+
* it on close, and auto-name it. Null when the session joined an existing conversation.
|
|
163
|
+
*/
|
|
164
|
+
get SessionCreatedConversationId() {
|
|
165
|
+
return this.createdConversationId;
|
|
166
|
+
}
|
|
167
|
+
/** The first final user utterance of the session (naming seed); null before the user speaks. */
|
|
168
|
+
get FirstUserTranscript() {
|
|
169
|
+
return this.firstUserTranscript;
|
|
170
|
+
}
|
|
171
|
+
get CurrentAgentSessionId() {
|
|
172
|
+
return this.agentSessionId;
|
|
173
|
+
}
|
|
174
|
+
/** Synchronous access to the minimized presentation state. */
|
|
175
|
+
get IsMinimized() {
|
|
176
|
+
return this._minimized$.value;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Minimizes / restores the active call overlay (host renders the floating pill while
|
|
180
|
+
* minimized). Presentation-only — the live audio session is untouched.
|
|
181
|
+
*/
|
|
182
|
+
SetMinimized(minimized) {
|
|
183
|
+
if (this._minimized$.value !== minimized) {
|
|
184
|
+
this._minimized$.next(minimized);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
// ── Session internals ──────────────────────────────────────────────────────
|
|
188
|
+
/** The provider-direct realtime client driving the live session (ClassFactory-resolved). */
|
|
189
|
+
client = null;
|
|
190
|
+
/** The mic capture stream — acquired here (permission UX) and handed to the client. */
|
|
191
|
+
localStream = null;
|
|
192
|
+
agentSessionId = null;
|
|
193
|
+
/**
|
|
194
|
+
* The DB-driven narration instruction template (server-resolved at session start, containing a
|
|
195
|
+
* `{{ progressMessage }}` placeholder). `null` when the deployment hasn't synced the narration
|
|
196
|
+
* prompt — {@link buildNarrationInstructions} then falls back to the built-in wording.
|
|
197
|
+
*/
|
|
198
|
+
narrationTemplate = null;
|
|
199
|
+
// ── Delegated-run progress streaming ───────────────────────────────────────
|
|
200
|
+
/** First spoken update fires no earlier than this long after delegated work starts. */
|
|
201
|
+
static FirstNarrationDelayMs = 5000;
|
|
202
|
+
/** Minimum gap between SUBSEQUENT spoken updates (the 7–10s band; floods aggregate). */
|
|
203
|
+
static NarrationIntervalMs = 8000;
|
|
204
|
+
/** Retry delay when the fire moment finds the model busy / audio still playing. */
|
|
205
|
+
static NarrationBusyRetryMs = 1500;
|
|
206
|
+
/** Max progress messages aggregated into one spoken digest. */
|
|
207
|
+
static MaxDigestMessages = 4;
|
|
208
|
+
/** Max prior spoken narrations chained into the instructions (anti-repetition). */
|
|
209
|
+
static MaxPriorNarrations = 3;
|
|
210
|
+
/**
|
|
211
|
+
* Aggregation buffer: distinct progress messages since the last spoken update (oldest
|
|
212
|
+
* first, capped at {@link RealtimeSessionService.MaxDigestMessages}). A flood of small
|
|
213
|
+
* updates becomes ONE digest; the buffer is discarded when the result lands first.
|
|
214
|
+
*/
|
|
215
|
+
pendingNarrationMessages = [];
|
|
216
|
+
/**
|
|
217
|
+
* Tool calls currently executing on the server. Progress events ride PubSub and can
|
|
218
|
+
* lag the (fast) mutation result — any progress for a call NOT in this set is stale
|
|
219
|
+
* (already completed) and is dropped, so we never narrate "starting up" after the
|
|
220
|
+
* answer was already spoken.
|
|
221
|
+
*/
|
|
222
|
+
inFlightCallIds = new Set();
|
|
223
|
+
/** Timer for the deferred narration; cancelled when the delegation result lands first. */
|
|
224
|
+
narrationTimer = null;
|
|
225
|
+
/**
|
|
226
|
+
* Call ids the USER explicitly cancelled via {@link CancelDelegation} /
|
|
227
|
+
* {@link CancelInFlightDelegations}. Their cards were already flipped to the
|
|
228
|
+
* "Cancelled by user" failed result, so when the original tool mutation later resolves
|
|
229
|
+
* with the aborted run's outcome, {@link emitDelegationResult} skips the duplicate card
|
|
230
|
+
* emission (the model still receives the tool result). Cleared at teardown.
|
|
231
|
+
*/
|
|
232
|
+
cancelledCallIds = new Set();
|
|
233
|
+
// ── Usage telemetry relay (B7) ─────────────────────────────────────────────
|
|
234
|
+
/** Debounce window for relaying accumulated usage deltas to the server. */
|
|
235
|
+
static UsageFlushDebounceMs = 10000;
|
|
236
|
+
/** Accumulated input-token delta since the last flush. */
|
|
237
|
+
pendingUsageInput = 0;
|
|
238
|
+
/** Accumulated output-token delta since the last flush. */
|
|
239
|
+
pendingUsageOutput = 0;
|
|
240
|
+
/** Pending debounced usage flush; also force-flushed at teardown. */
|
|
241
|
+
usageFlushTimer = null;
|
|
242
|
+
/** Active push-status subscription that feeds delegation progress; cleared on teardown. */
|
|
243
|
+
delegationProgressSub = null;
|
|
244
|
+
/** Timestamp (ms) of the last narration we triggered; 0 = never. */
|
|
245
|
+
lastDelegationNarrationAt = 0;
|
|
246
|
+
/** When the current delegation burst began (first in-flight call); anchors the 5s first update. */
|
|
247
|
+
delegationBurstStartedAt = 0;
|
|
248
|
+
/** Spoken updates so far in this burst (1-based numbering for the instructions). */
|
|
249
|
+
narrationCount = 0;
|
|
250
|
+
/** What the model actually SAID for prior updates this burst — chained in so it never repeats itself. */
|
|
251
|
+
spokenNarrations = [];
|
|
252
|
+
/** Tail message of the last digest, so an identical trailing progress event isn't re-buffered. */
|
|
253
|
+
lastNarratedTail = '';
|
|
254
|
+
/**
|
|
255
|
+
* Registry of CLIENT-EXECUTED UI tool handlers, keyed by tool-name prefix (e.g.
|
|
256
|
+
* `'Whiteboard_'`). Tool calls whose name matches a registered prefix run LOCALLY through the
|
|
257
|
+
* handler (never relayed to the server); everything else takes the standard server-relay path.
|
|
258
|
+
* Cleared at teardown.
|
|
259
|
+
*/
|
|
260
|
+
clientToolHandlers = new Map();
|
|
261
|
+
// ── Interactive channels (registry-resolved plugins) ───────────────────────
|
|
262
|
+
/** Debounce window for persisting a channel's state of record after a change burst. */
|
|
263
|
+
static ChannelSaveDebounceMs = 3000;
|
|
264
|
+
/**
|
|
265
|
+
* Pending DEBOUNCED channel-state saves, keyed by channel name. Each entry keeps the
|
|
266
|
+
* LATEST serialized state plus the session id captured while the session was live —
|
|
267
|
+
* the teardown flush runs as the live id is being torn down, so the capture guarantees
|
|
268
|
+
* the final save still lands on the just-closed session.
|
|
269
|
+
*/
|
|
270
|
+
pendingChannelSaves = new Map();
|
|
271
|
+
_provider = null;
|
|
272
|
+
/**
|
|
273
|
+
* Metadata provider used for the GraphQL relay mutations. Falls back to the
|
|
274
|
+
* global default when unset (single-provider apps see no change).
|
|
275
|
+
*/
|
|
276
|
+
get Provider() {
|
|
277
|
+
return this._provider ?? Metadata.Provider;
|
|
278
|
+
}
|
|
279
|
+
set Provider(value) {
|
|
280
|
+
this._provider = value;
|
|
281
|
+
}
|
|
282
|
+
/** True when a session is currently open. */
|
|
283
|
+
get IsActive() {
|
|
284
|
+
return this._active$.value;
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Start a client-direct voice session fronting `targetAgentId`.
|
|
288
|
+
*
|
|
289
|
+
* @param targetAgentId The agent the Realtime Co-Agent voices on behalf of.
|
|
290
|
+
* @param conversationId Optional existing conversation to bind + seed context from.
|
|
291
|
+
* @param lastSessionId Optional prior session to chain to (resume / continuation).
|
|
292
|
+
* @param agentName Optional display name of the target agent — resolved by the caller
|
|
293
|
+
* (which knows the conversation's routing context) and surfaced on {@link AgentName$}
|
|
294
|
+
* so ANY host (composer trigger, chat-area overlay) can render it without re-resolving.
|
|
295
|
+
* @param preferredModelId Optional EXPLICIT realtime model choice (`MJ: AI Models.ID`). When
|
|
296
|
+
* set, the server uses exactly that model and FAILS with a clear reason if it can't (no
|
|
297
|
+
* silent fallback). Omit for the server's automatic (highest-PowerRank) selection.
|
|
298
|
+
* @param clientTools Optional EXTRA client-executed UI tool declarations to expose to the
|
|
299
|
+
* realtime model alongside the server's stable tool set and the interactive-channel
|
|
300
|
+
* tools (which are aggregated automatically from the registry-resolved plugins — see
|
|
301
|
+
* {@link ActiveChannels$}). The server only DECLARES these — execution stays in the
|
|
302
|
+
* browser via handlers registered with {@link RegisterClientToolHandler}. This is an
|
|
303
|
+
* extension point for hosts with bespoke (non-channel) UI tools; most callers omit it.
|
|
304
|
+
* @param coAgentId Optional EXPLICIT co-agent choice (`MJ: AI Agents.ID` of an Active,
|
|
305
|
+
* Realtime-type agent) — the highest-precedence step of the server's co-agent resolution
|
|
306
|
+
* chain. When set, the server uses exactly that co-agent and FAILS with a clear reason if
|
|
307
|
+
* it can't (no silent fallback). Omit to let server metadata drive the choice: the target
|
|
308
|
+
* agent's `DefaultCoAgentID`, then the type-level `AIAgentCoAgent` default row, then the global Realtime Co-Agent.
|
|
309
|
+
* @param configOverridesJson Optional JSON payload of SESSION CONFIG overrides (e.g.
|
|
310
|
+
* `{"realtime":{"modelPreference":"<modelId>"}}`), forwarded verbatim on the mint
|
|
311
|
+
* mutation. The server enforces the `Realtime: Advanced Session Controls`
|
|
312
|
+
* authorization on any overrides — hosts only populate this from authorization-gated
|
|
313
|
+
* pickers, and never synthesize overrides beyond what the user explicitly chose.
|
|
314
|
+
* Omit/`null` for the server's defaults (today's behavior).
|
|
315
|
+
*/
|
|
316
|
+
async StartVoiceSession(targetAgentId, conversationId, lastSessionId, agentName, preferredModelId, clientTools, coAgentId, configOverridesJson) {
|
|
317
|
+
if (this.IsActive) {
|
|
318
|
+
return; // a session is already running — ignore duplicate starts
|
|
319
|
+
}
|
|
320
|
+
if (agentName) {
|
|
321
|
+
this._agentName$.next(agentName);
|
|
322
|
+
}
|
|
323
|
+
this.resetState();
|
|
324
|
+
this._active$.next(true);
|
|
325
|
+
this._connectionState$.next('connecting');
|
|
326
|
+
try {
|
|
327
|
+
// Resolve + initialize the interactive-channel plugins FIRST: their client-executed
|
|
328
|
+
// tool sets must be declared to the realtime model at session mint.
|
|
329
|
+
const allClientTools = [...(clientTools ?? []), ...(await this.startChannels())];
|
|
330
|
+
const session = await this.mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, allClientTools, coAgentId, configOverridesJson);
|
|
331
|
+
this.agentSessionId = session.AgentSessionId;
|
|
332
|
+
// A null input conversationId means the SERVER created a fresh conversation for
|
|
333
|
+
// this session — track it so the host can fold it into the cached list, select
|
|
334
|
+
// it on close, and auto-name it (via the shared naming helper).
|
|
335
|
+
this.createdConversationId = !conversationId && session.ConversationId ? session.ConversationId : null;
|
|
336
|
+
this.sessionConversationId = session.ConversationId ?? conversationId ?? null;
|
|
337
|
+
this.firstUserTranscript = null;
|
|
338
|
+
this.narrationTemplate = session.NarrationInstructionsTemplate ?? null;
|
|
339
|
+
this._modelName$.next(session.ModelName ?? null);
|
|
340
|
+
// Resume continuity: rehydrate channel plugins from the PRIOR session's saved states
|
|
341
|
+
// (e.g. the whiteboard) BEFORE any surface binds — tolerant, never blocks the start.
|
|
342
|
+
this.applyPriorChannelStates(session.PriorChannelStatesJson);
|
|
343
|
+
const client = this.createRealtimeClient(session.Provider);
|
|
344
|
+
this.client = client;
|
|
345
|
+
this.wireClientHandlers(client);
|
|
346
|
+
this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
347
|
+
await client.Connect(this.buildClientConfig(session), this.localStream);
|
|
348
|
+
this.subscribeDelegationProgress();
|
|
349
|
+
// State advances to 'listening' once the provider control channel opens
|
|
350
|
+
// (driven by the client's OnStateChange events).
|
|
351
|
+
// Surface a generic session-started event for the conversations runtime
|
|
352
|
+
// SessionsObserver bridge. Emitting AFTER Connect() guarantees both that
|
|
353
|
+
// agentSessionId is set (line ~468) AND the realtime client is connected,
|
|
354
|
+
// so consumers can act on it without re-checking either condition.
|
|
355
|
+
this._sessionStarted$.next({
|
|
356
|
+
sessionId: this.agentSessionId,
|
|
357
|
+
channelNames: this._activeChannels$.value.map(c => c.ChannelName),
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
catch (error) {
|
|
361
|
+
console.error('[RealtimeSession] Failed to start session:', error);
|
|
362
|
+
this._connectionState$.next('error');
|
|
363
|
+
await this.teardown(false);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* End the active session: stop the mic, tear down the provider connection, and close
|
|
368
|
+
* the server-side agent session. Safe to call when no session is active.
|
|
369
|
+
*/
|
|
370
|
+
async EndVoiceSession() {
|
|
371
|
+
if (!this.IsActive && !this.agentSessionId) {
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
await this.teardown(true);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Inject a typed message into the live session as a user turn.
|
|
378
|
+
*
|
|
379
|
+
* Decomposed into two steps, each mirroring an existing voice path so the typed
|
|
380
|
+
* turn behaves identically to a spoken one:
|
|
381
|
+
* 1. {@link BaseRealtimeClient.SendText} injects the text as user input and triggers a
|
|
382
|
+
* reply through the SAME collision-safe path tool results use — so it queues behind
|
|
383
|
+
* any in-flight response (progress narration / prior turn) instead of colliding.
|
|
384
|
+
* 2. Relay the turn through the same caption + transcript paths user speech uses
|
|
385
|
+
* ({@link onUserTranscript}) so it shows in the live thread AND persists to MJ.
|
|
386
|
+
*
|
|
387
|
+
* No-op when no session is open / the control channel isn't ready, or when the text is empty.
|
|
388
|
+
*/
|
|
389
|
+
SendText(text) {
|
|
390
|
+
const trimmed = text?.trim() ?? '';
|
|
391
|
+
if (trimmed.length === 0) {
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
const client = this.client;
|
|
395
|
+
if (!client || !this.isSessionLive()) {
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
client.SendText(trimmed);
|
|
399
|
+
// Relay as a user turn — same path spoken input uses (caption + persisted transcript).
|
|
400
|
+
void this.onUserTranscript(trimmed);
|
|
401
|
+
}
|
|
402
|
+
/** Mute / unmute the local microphone track. Returns the new muted state. */
|
|
403
|
+
ToggleMute() {
|
|
404
|
+
const tracks = this.localStream?.getAudioTracks() ?? [];
|
|
405
|
+
if (tracks.length === 0) {
|
|
406
|
+
return false;
|
|
407
|
+
}
|
|
408
|
+
const muted = tracks[0].enabled; // currently enabled → becomes muted
|
|
409
|
+
this.client?.SetMuted(muted);
|
|
410
|
+
return muted;
|
|
411
|
+
}
|
|
412
|
+
// ── Client-executed UI tools ───────────────────────────────────────────────
|
|
413
|
+
/**
|
|
414
|
+
* Registers a handler for CLIENT-EXECUTED UI tools whose names start with `toolNamePrefix`
|
|
415
|
+
* (e.g. `'Whiteboard_'` → all `Whiteboard_*` calls). Matching tool calls execute LOCALLY via
|
|
416
|
+
* the handler — they are never relayed to the server — and the handler's result JSON is sent
|
|
417
|
+
* back to the model as the `tool_response`. Re-registering the same prefix replaces the
|
|
418
|
+
* handler. The registry is cleared at session teardown.
|
|
419
|
+
*/
|
|
420
|
+
RegisterClientToolHandler(toolNamePrefix, handler) {
|
|
421
|
+
this.clientToolHandlers.set(toolNamePrefix, handler);
|
|
422
|
+
}
|
|
423
|
+
/** Removes the handler registered for `toolNamePrefix` (no-op when none is registered). */
|
|
424
|
+
UnregisterClientToolHandler(toolNamePrefix) {
|
|
425
|
+
this.clientToolHandlers.delete(toolNamePrefix);
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Feeds a background context note into the live model (no spoken reply is requested) — the
|
|
429
|
+
* perception channel interactive surfaces use (e.g. the whiteboard's coalesced scene deltas).
|
|
430
|
+
* No-op when no session is live.
|
|
431
|
+
*/
|
|
432
|
+
SendContextNote(text) {
|
|
433
|
+
const trimmed = text?.trim() ?? '';
|
|
434
|
+
if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
this.client.SendContextNote(trimmed);
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* The active client's current audio activity (per-direction RMS levels + spectrum
|
|
441
|
+
* bins), or `null` when no session is live or the driver attached no audio meters.
|
|
442
|
+
* Sampled by the overlay's animation-frame loop to drive the audio-reactive orb/EQ —
|
|
443
|
+
* a cheap analyser read, never provider traffic.
|
|
444
|
+
*/
|
|
445
|
+
GetAudioActivity() {
|
|
446
|
+
return this.client?.GetAudioActivity() ?? null;
|
|
447
|
+
}
|
|
448
|
+
// ── Interactive channels (registry-driven plugins) ─────────────────────────
|
|
449
|
+
/**
|
|
450
|
+
* Resolves, instantiates and initializes the session's interactive-channel plugins from
|
|
451
|
+
* the `MJ: AI Agent Channels` registry, publishes them on {@link ActiveChannels$}, and
|
|
452
|
+
* returns their aggregated client-executed tool declarations for the session mint.
|
|
453
|
+
* Tolerant by design: registry/resolution failures degrade to "no channels" — the voice
|
|
454
|
+
* session itself always proceeds.
|
|
455
|
+
*/
|
|
456
|
+
async startChannels() {
|
|
457
|
+
const channels = await this.loadActiveChannels();
|
|
458
|
+
for (const plugin of channels) {
|
|
459
|
+
this.initializeChannel(plugin);
|
|
460
|
+
}
|
|
461
|
+
this._activeChannels$.next(channels);
|
|
462
|
+
return channels.flatMap(plugin => plugin.GetToolDefinitions());
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Loads the ACTIVE channel definitions from the registry and resolves each row's
|
|
466
|
+
* `ClientPluginClass` through the MJ ClassFactory into a per-session plugin instance —
|
|
467
|
+
* the client-side mirror of how realtime-model drivers resolve from `BaseRealtimeModel`
|
|
468
|
+
* / `BaseRealtimeClient`. Rows whose plugin class isn't registered are skipped (logged),
|
|
469
|
+
* never fatal.
|
|
470
|
+
*/
|
|
471
|
+
async loadActiveChannels() {
|
|
472
|
+
const rows = await this.fetchChannelDefinitions();
|
|
473
|
+
const channels = [];
|
|
474
|
+
for (const row of rows) {
|
|
475
|
+
const plugin = this.resolveChannelPlugin(row);
|
|
476
|
+
if (plugin) {
|
|
477
|
+
channels.push(plugin);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
return channels;
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* Reads the ACTIVE `MJ: AI Agent Channels` rows from {@link AIEngineBase}'s cached
|
|
484
|
+
* `AgentChannels` (provider-scoped engine instance, lazy `Config` — no RunView
|
|
485
|
+
* round-trip; the engine's BaseEntity-event reactivity keeps the registry fresh).
|
|
486
|
+
* Failures are logged and degrade to an empty list — channel availability must
|
|
487
|
+
* never block the voice session.
|
|
488
|
+
*/
|
|
489
|
+
async fetchChannelDefinitions() {
|
|
490
|
+
try {
|
|
491
|
+
const engine = AIEngineBase.GetProviderInstance(this.Provider, AIEngineBase);
|
|
492
|
+
await engine.Config(false, undefined, this.Provider);
|
|
493
|
+
return (engine.AgentChannels ?? [])
|
|
494
|
+
.filter(c => c.IsActive)
|
|
495
|
+
.map(c => ({ ID: c.ID, Name: c.Name, ClientPluginClass: c.ClientPluginClass }));
|
|
496
|
+
}
|
|
497
|
+
catch (error) {
|
|
498
|
+
console.warn('[RealtimeSession] Channel registry unavailable — starting with no channels:', error);
|
|
499
|
+
return [];
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Resolves one registry row's `ClientPluginClass` via the ClassFactory (registration
|
|
504
|
+
* checked first, exactly like the realtime-client drivers) and instantiates a fresh
|
|
505
|
+
* per-session plugin. Returns `null` (logged) when no plugin is registered for the key
|
|
506
|
+
* — e.g. its Load function was never called or the package isn't included client-side.
|
|
507
|
+
*/
|
|
508
|
+
resolveChannelPlugin(row) {
|
|
509
|
+
const key = row.ClientPluginClass?.trim();
|
|
510
|
+
if (!key) {
|
|
511
|
+
console.warn(`[RealtimeSession] Channel '${row.Name}' has no ClientPluginClass — skipping.`);
|
|
512
|
+
return null;
|
|
513
|
+
}
|
|
514
|
+
const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeChannelClient, key);
|
|
515
|
+
if (!registration) {
|
|
516
|
+
console.warn(`[RealtimeSession] No client plugin registered for channel '${row.Name}' (key '${key}') — skipping.`);
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
const plugin = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeChannelClient, key);
|
|
520
|
+
if (!plugin) {
|
|
521
|
+
console.warn(`[RealtimeSession] Failed to instantiate client plugin for channel '${row.Name}' (key '${key}').`);
|
|
522
|
+
return null;
|
|
523
|
+
}
|
|
524
|
+
return plugin;
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Wires one plugin into the session: hands it its host context and registers its
|
|
528
|
+
* prefix-routed local tool executor (so `<ToolNamePrefix>*` calls run in the browser
|
|
529
|
+
* through {@link BaseRealtimeChannelClient.ApplyAgentTool}, never the server relay).
|
|
530
|
+
*/
|
|
531
|
+
initializeChannel(plugin) {
|
|
532
|
+
plugin.Initialize(this.buildChannelContext(plugin));
|
|
533
|
+
this.RegisterClientToolHandler(plugin.ToolNamePrefix, (toolName, argsJson) => {
|
|
534
|
+
// The agent is ACTING on this channel — surface-discovery signal for the overlay
|
|
535
|
+
// (first activity auto-reveals + focuses the channel tab) before the tool applies.
|
|
536
|
+
this._channelActivity$.next(plugin);
|
|
537
|
+
return plugin.ApplyAgentTool(toolName, argsJson);
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
/** Builds the host-services context one channel plugin sees (its only line to the session). */
|
|
541
|
+
buildChannelContext(plugin) {
|
|
542
|
+
// Capture the service in a local so the AgentSessionID getter reads the SERVICE's live
|
|
543
|
+
// field (not the object literal's `this`) every time it's accessed.
|
|
544
|
+
const service = this;
|
|
545
|
+
return {
|
|
546
|
+
AgentName: this.CurrentAgentName,
|
|
547
|
+
SendContextNote: (text) => this.SendContextNote(text),
|
|
548
|
+
RequestSpokenResponse: (instructions) => this.requestChannelSpokenResponse(instructions),
|
|
549
|
+
RequestSave: (stateJson) => this.scheduleChannelSave(plugin.ChannelName, stateJson),
|
|
550
|
+
SaveAsArtifact: (name, contentJson) => this.saveChannelArtifact(plugin.ChannelName, name, contentJson),
|
|
551
|
+
SetFocusMode: (on) => this._channelFocus$.next({ Channel: plugin, Focused: on }),
|
|
552
|
+
// Live session id + GraphQL escape hatch for SERVER-BACKED channels (e.g. Remote
|
|
553
|
+
// Browser). `get` so a channel always reads the CURRENT id — it's null at Initialize
|
|
554
|
+
// (the plugin is built before mintSession resolves) and set once the session is live.
|
|
555
|
+
get AgentSessionID() {
|
|
556
|
+
return service.agentSessionId;
|
|
557
|
+
},
|
|
558
|
+
ExecuteServerAction: (query, variables) => this.executeChannelServerAction(query, variables)
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Runs a channel-specific GraphQL operation through the live session's provider (the
|
|
563
|
+
* {@link RealtimeChannelContext.ExecuteServerAction} implementation). Best-effort: any
|
|
564
|
+
* transport/server error is logged and resolves to `null` so the calling channel can map
|
|
565
|
+
* the failure to a model-readable result string without `try/catch`.
|
|
566
|
+
*/
|
|
567
|
+
async executeChannelServerAction(query, variables) {
|
|
568
|
+
try {
|
|
569
|
+
const result = await this.gql().ExecuteGQL(query, variables);
|
|
570
|
+
return result ?? null;
|
|
571
|
+
}
|
|
572
|
+
catch (error) {
|
|
573
|
+
console.error('[RealtimeSession] Channel server action failed:', error);
|
|
574
|
+
return null;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* A channel asked the live model to SPEAK in reaction to channel input (e.g. a widget
|
|
579
|
+
* submission) — routed through the client's spoken-update channel. No-op when the
|
|
580
|
+
* session isn't live; empty instructions are dropped.
|
|
581
|
+
*/
|
|
582
|
+
requestChannelSpokenResponse(instructions) {
|
|
583
|
+
const trimmed = instructions?.trim() ?? '';
|
|
584
|
+
if (trimmed.length === 0 || !this.client || !this.isSessionLive()) {
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
this.client.RequestSpokenUpdate(trimmed);
|
|
588
|
+
}
|
|
589
|
+
/**
|
|
590
|
+
* Applies the PRIOR session's saved channel states (resume continuity): parses the
|
|
591
|
+
* server-supplied map and offers each entry to the matching active plugin via
|
|
592
|
+
* {@link BaseRealtimeChannelClient.RestoreState}. Fully tolerant — malformed payloads,
|
|
593
|
+
* unknown channels, and plugin rejections are logged and skipped; the session start is
|
|
594
|
+
* never affected.
|
|
595
|
+
*/
|
|
596
|
+
applyPriorChannelStates(statesJson) {
|
|
597
|
+
if (!statesJson) {
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
let states;
|
|
601
|
+
try {
|
|
602
|
+
const parsed = JSON.parse(statesJson);
|
|
603
|
+
if (parsed === null || typeof parsed !== 'object') {
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
states = parsed;
|
|
607
|
+
}
|
|
608
|
+
catch {
|
|
609
|
+
console.warn('[RealtimeSession] PriorChannelStatesJson was malformed — starting channels fresh');
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
612
|
+
for (const plugin of this._activeChannels$.value) {
|
|
613
|
+
const state = states[plugin.ChannelName];
|
|
614
|
+
if (typeof state === 'string' && state.length > 0) {
|
|
615
|
+
try {
|
|
616
|
+
const restored = plugin.RestoreState(state);
|
|
617
|
+
if (!restored) {
|
|
618
|
+
console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' declined its prior-session state — starting fresh`);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
catch (error) {
|
|
622
|
+
console.warn(`[RealtimeSession] Channel '${plugin.ChannelName}' restore threw — starting fresh`, error);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Persists a channel's state as a first-class versioned artifact (`MJ: Artifacts`) via the
|
|
629
|
+
* `SaveSessionChannelArtifact` mutation — the channel-context capability behind e.g. the
|
|
630
|
+
* whiteboard's "Save to artifacts". Best-effort: returns the created Artifact ID, or null
|
|
631
|
+
* on any failure (logged, never thrown). Uses the live session id, falling back to the
|
|
632
|
+
* teardown-captured one so "save my board" works right after the call ends.
|
|
633
|
+
*/
|
|
634
|
+
async saveChannelArtifact(channelName, name, contentJson) {
|
|
635
|
+
const sessionId = this.agentSessionId ?? this.lastKnownSessionIdForSaves();
|
|
636
|
+
if (!sessionId || !name.trim() || !contentJson) {
|
|
637
|
+
return null;
|
|
638
|
+
}
|
|
639
|
+
try {
|
|
640
|
+
const result = await this.gql().ExecuteGQL(`mutation SaveSessionChannelArtifact($agentSessionId: String!, $channelName: String!, $name: String!, $contentJson: String!) {
|
|
641
|
+
SaveSessionChannelArtifact(agentSessionId: $agentSessionId, channelName: $channelName, name: $name, contentJson: $contentJson) {
|
|
642
|
+
Success
|
|
643
|
+
ErrorMessage
|
|
644
|
+
ArtifactID
|
|
645
|
+
ArtifactVersionID
|
|
646
|
+
}
|
|
647
|
+
}`, { agentSessionId: sessionId, channelName, name: name.trim(), contentJson });
|
|
648
|
+
const payload = result?.SaveSessionChannelArtifact;
|
|
649
|
+
if (!payload?.Success) {
|
|
650
|
+
console.warn(`[RealtimeSession] Save-as-artifact failed for '${channelName}': ${payload?.ErrorMessage ?? 'unknown error'}`);
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
return payload.ArtifactID ?? null;
|
|
654
|
+
}
|
|
655
|
+
catch (error) {
|
|
656
|
+
console.warn(`[RealtimeSession] Save-as-artifact errored for '${channelName}':`, error);
|
|
657
|
+
return null;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
/** Most recent session id captured by the save pipeline (post-teardown saves). */
|
|
661
|
+
lastKnownSessionIdForSaves() {
|
|
662
|
+
for (const pending of this.pendingChannelSaves.values()) {
|
|
663
|
+
if (pending.SessionID) {
|
|
664
|
+
return pending.SessionID;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
return null;
|
|
668
|
+
}
|
|
669
|
+
/**
|
|
670
|
+
* Schedules the DEBOUNCED state-of-record save for a channel: each request replaces the
|
|
671
|
+
* pending payload (latest state wins) and re-arms the timer; the session id is captured
|
|
672
|
+
* while live so the teardown flush can persist onto the just-closed session.
|
|
673
|
+
*/
|
|
674
|
+
scheduleChannelSave(channelName, stateJson) {
|
|
675
|
+
const pending = this.pendingChannelSaves.get(channelName);
|
|
676
|
+
if (pending) {
|
|
677
|
+
clearTimeout(pending.Timer);
|
|
678
|
+
}
|
|
679
|
+
this.pendingChannelSaves.set(channelName, {
|
|
680
|
+
Timer: setTimeout(() => this.flushChannelSave(channelName), RealtimeSessionService.ChannelSaveDebounceMs),
|
|
681
|
+
StateJson: stateJson,
|
|
682
|
+
SessionID: this.agentSessionId ?? pending?.SessionID ?? null
|
|
683
|
+
});
|
|
684
|
+
}
|
|
685
|
+
/** Fires one pending channel save (best-effort; {@link SaveChannelState} logs failures). */
|
|
686
|
+
flushChannelSave(channelName) {
|
|
687
|
+
const pending = this.pendingChannelSaves.get(channelName);
|
|
688
|
+
if (!pending) {
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
this.pendingChannelSaves.delete(channelName);
|
|
692
|
+
clearTimeout(pending.Timer);
|
|
693
|
+
void this.SaveChannelState(channelName, pending.StateJson, pending.SessionID);
|
|
694
|
+
}
|
|
695
|
+
/** Final teardown flush: persist every channel's unsaved state immediately. */
|
|
696
|
+
flushAllChannelSaves() {
|
|
697
|
+
for (const channelName of [...this.pendingChannelSaves.keys()]) {
|
|
698
|
+
this.flushChannelSave(channelName);
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
/** Disposes all channel plugins (errors contained per plugin) and clears the live set. */
|
|
702
|
+
disposeChannels() {
|
|
703
|
+
for (const plugin of this._activeChannels$.value) {
|
|
704
|
+
try {
|
|
705
|
+
plugin.Dispose();
|
|
706
|
+
}
|
|
707
|
+
catch (error) {
|
|
708
|
+
console.error(`[RealtimeSession] Channel '${plugin.ChannelName}' Dispose failed:`, error);
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
if (this._activeChannels$.value.length > 0) {
|
|
712
|
+
this._activeChannels$.next([]);
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
// ── Realtime client resolution + wiring ────────────────────────────────────
|
|
716
|
+
/**
|
|
717
|
+
* Resolves the provider-direct realtime client for `provider` through the MJ
|
|
718
|
+
* ClassFactory — the client-side mirror of how server drivers are resolved from
|
|
719
|
+
* `BaseRealtimeModel`. Throws a clear error when no driver is registered for the
|
|
720
|
+
* provider (e.g. its Load function was never called).
|
|
721
|
+
*/
|
|
722
|
+
createRealtimeClient(provider) {
|
|
723
|
+
const registration = MJGlobal.Instance.ClassFactory.GetRegistration(BaseRealtimeClient, provider);
|
|
724
|
+
if (!registration) {
|
|
725
|
+
throw new Error(`No realtime client registered for provider '${provider}'. ` +
|
|
726
|
+
`Ensure the provider's client driver package is imported and its Load function called.`);
|
|
727
|
+
}
|
|
728
|
+
const client = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeClient, provider);
|
|
729
|
+
if (!client) {
|
|
730
|
+
throw new Error(`Failed to instantiate the realtime client for provider '${provider}'`);
|
|
731
|
+
}
|
|
732
|
+
return client;
|
|
733
|
+
}
|
|
734
|
+
/** Builds the client-direct session config the realtime client connects with. */
|
|
735
|
+
buildClientConfig(session) {
|
|
736
|
+
return {
|
|
737
|
+
Provider: session.Provider,
|
|
738
|
+
Model: session.Model,
|
|
739
|
+
EphemeralToken: session.EphemeralToken,
|
|
740
|
+
ExpiresAt: session.ExpiresAt,
|
|
741
|
+
SessionConfig: this.parseSessionConfig(session.SessionConfigJson)
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* Parses the server-built session config JSON. On failure, logs and returns an empty
|
|
746
|
+
* object — the client treats an empty config as "nothing to apply", so the session
|
|
747
|
+
* still opens (mirroring the prior behavior of skipping the config update).
|
|
748
|
+
*/
|
|
749
|
+
parseSessionConfig(sessionConfigJson) {
|
|
750
|
+
if (!sessionConfigJson) {
|
|
751
|
+
return {};
|
|
752
|
+
}
|
|
753
|
+
try {
|
|
754
|
+
return JSON.parse(sessionConfigJson);
|
|
755
|
+
}
|
|
756
|
+
catch (error) {
|
|
757
|
+
console.error('[RealtimeSession] Failed to parse/apply SessionConfigJson:', error);
|
|
758
|
+
return {};
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
/** Subscribes this service's policy handlers to the realtime client's events. */
|
|
762
|
+
wireClientHandlers(client) {
|
|
763
|
+
client.OnStateChange((state) => this.onClientStateChange(state));
|
|
764
|
+
client.OnTranscript((transcript) => {
|
|
765
|
+
void this.onClientTranscript(transcript);
|
|
766
|
+
});
|
|
767
|
+
client.OnToolCall((call) => {
|
|
768
|
+
void this.handleToolCall(call);
|
|
769
|
+
});
|
|
770
|
+
client.OnError((error) => {
|
|
771
|
+
console.error('[RealtimeSession] Provider error event:', error);
|
|
772
|
+
});
|
|
773
|
+
// Usage telemetry: accumulate the driver's per-response token DELTAS and relay them to
|
|
774
|
+
// the server (onto the co-agent AIPromptRun) debounced + once at teardown. Providers
|
|
775
|
+
// without usage events simply never emit — registering is always safe.
|
|
776
|
+
client.OnUsage((usage) => this.onUsageDelta(usage));
|
|
777
|
+
// TRUE BARGE-IN (user input cut off active model output — the driver already stopped
|
|
778
|
+
// the speech): the user took the floor, so any pending/queued progress narration is
|
|
779
|
+
// stale — cancel it; the next progress event re-schedules at the session-global pace.
|
|
780
|
+
// HOST POLICY (deliberate): barge-in does NOT abort in-flight delegated runs — the
|
|
781
|
+
// narration design EXPECTS the user to keep talking while delegated work runs, so
|
|
782
|
+
// killing the work on speech would cancel exactly the jobs the user asked for.
|
|
783
|
+
// Explicit cancellation is a separate, intentional act: the overlay's per-card ✕
|
|
784
|
+
// calls {@link CancelDelegation} (server cancel channel) instead.
|
|
785
|
+
client.OnInterruption(() => {
|
|
786
|
+
this.cancelPendingNarration();
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
/** Maps a client state event onto the UI connection state. */
|
|
790
|
+
onClientStateChange(state) {
|
|
791
|
+
const mapped = this.mapClientState(state);
|
|
792
|
+
if (mapped) {
|
|
793
|
+
this._connectionState$.next(mapped);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Translates {@link RealtimeClientState} into {@link VoiceConnectionState}. `'connected'`
|
|
798
|
+
* is suppressed (the UI stays 'connecting' until the control channel opens → 'listening'),
|
|
799
|
+
* and `'closed'` never overwrites a terminal 'error' the service itself recorded.
|
|
800
|
+
*/
|
|
801
|
+
mapClientState(state) {
|
|
802
|
+
switch (state) {
|
|
803
|
+
case 'connecting':
|
|
804
|
+
return 'connecting';
|
|
805
|
+
case 'connected':
|
|
806
|
+
return null;
|
|
807
|
+
case 'listening':
|
|
808
|
+
return 'listening';
|
|
809
|
+
case 'speaking':
|
|
810
|
+
return 'speaking';
|
|
811
|
+
case 'error':
|
|
812
|
+
return 'error';
|
|
813
|
+
case 'closed':
|
|
814
|
+
return this._connectionState$.value === 'error' ? null : 'closed';
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
/** True when the live control channel is usable (open and not torn down / failed). */
|
|
818
|
+
isSessionLive() {
|
|
819
|
+
const state = this._connectionState$.value;
|
|
820
|
+
return state === 'listening' || state === 'speaking' || state === 'thinking';
|
|
821
|
+
}
|
|
822
|
+
// ── Transcript policy ──────────────────────────────────────────────────────
|
|
823
|
+
/**
|
|
824
|
+
* Applies transcript policy to client transcript events. Interim deltas are ignored
|
|
825
|
+
* (the client already drives the speaking state). Final NORMAL assistant turns become
|
|
826
|
+
* captions + persisted transcripts; final NARRATION turns are EPHEMERAL by product
|
|
827
|
+
* decision — emitted on {@link DelegationNarration$} only, never a caption, never
|
|
828
|
+
* relayed/persisted. User turns ride the caption + relay path.
|
|
829
|
+
*/
|
|
830
|
+
async onClientTranscript(transcript) {
|
|
831
|
+
if (!transcript.IsFinal) {
|
|
832
|
+
return;
|
|
833
|
+
}
|
|
834
|
+
if (transcript.Role === 'Assistant') {
|
|
835
|
+
if (transcript.Kind === 'narration') {
|
|
836
|
+
this._delegationNarration$.next({ Text: transcript.Text });
|
|
837
|
+
// Remember what was actually SAID so later updates build on it instead of repeating.
|
|
838
|
+
this.spokenNarrations.push(transcript.Text);
|
|
839
|
+
if (this.spokenNarrations.length > RealtimeSessionService.MaxPriorNarrations) {
|
|
840
|
+
this.spokenNarrations.shift();
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
else if (transcript.ReplacesPrevious) {
|
|
844
|
+
// CORRECTION (e.g. ElevenLabs post-barge-in re-finalization): this final
|
|
845
|
+
// SUPERSEDES the previous final assistant turn — replace the caption in place
|
|
846
|
+
// and tell the server to update the persisted turn instead of appending.
|
|
847
|
+
this.replaceLastCaption('Assistant', transcript.Text);
|
|
848
|
+
await this.relayTranscript('assistant', transcript.Text, true);
|
|
849
|
+
}
|
|
850
|
+
else {
|
|
851
|
+
this.appendCaption({ Role: 'Assistant', Text: transcript.Text });
|
|
852
|
+
await this.relayTranscript('assistant', transcript.Text);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
else if (transcript.ReplacesPrevious) {
|
|
856
|
+
// STREAMING user transcription: providers like Grok emit the growing utterance as repeated
|
|
857
|
+
// events (each the full text so far), flagging all but the first ReplacesPrevious. Update the
|
|
858
|
+
// in-place User caption + persisted turn instead of stacking a new bubble per increment — the
|
|
859
|
+
// same correction semantics the assistant branch uses. (OpenAI sends one final → the else path.)
|
|
860
|
+
this.replaceLastCaption('User', transcript.Text);
|
|
861
|
+
await this.relayTranscript('user', transcript.Text, true);
|
|
862
|
+
}
|
|
863
|
+
else {
|
|
864
|
+
await this.onUserTranscript(transcript.Text);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
/**
|
|
868
|
+
* Replaces the LAST caption of `role` in place (correction semantics); falls back to a
|
|
869
|
+
* plain append when no such caption exists yet (e.g. the superseded turn predates this
|
|
870
|
+
* client's caption window).
|
|
871
|
+
*/
|
|
872
|
+
replaceLastCaption(role, text) {
|
|
873
|
+
const captions = this._captions$.value;
|
|
874
|
+
for (let i = captions.length - 1; i >= 0; i--) {
|
|
875
|
+
if (captions[i].Role === role) {
|
|
876
|
+
const next = [...captions];
|
|
877
|
+
next[i] = { Role: role, Text: text };
|
|
878
|
+
this._captions$.next(next);
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
this.appendCaption({ Role: role, Text: text });
|
|
883
|
+
}
|
|
884
|
+
/** Finalizes the user turn: push a caption + relay the final transcript. */
|
|
885
|
+
async onUserTranscript(transcript) {
|
|
886
|
+
if (transcript.trim().length === 0) {
|
|
887
|
+
return;
|
|
888
|
+
}
|
|
889
|
+
if (this.firstUserTranscript === null) {
|
|
890
|
+
// First spoken user utterance — the naming seed for a session-created conversation.
|
|
891
|
+
this.firstUserTranscript = transcript;
|
|
892
|
+
}
|
|
893
|
+
this.appendCaption({ Role: 'User', Text: transcript });
|
|
894
|
+
await this.relayTranscript('user', transcript);
|
|
895
|
+
}
|
|
896
|
+
// ── Tool calling ───────────────────────────────────────────────────────────
|
|
897
|
+
/**
|
|
898
|
+
* Routes a provider tool call: names matching a registered client-tool prefix execute
|
|
899
|
+
* LOCALLY (UI tools — see {@link RegisterClientToolHandler}); everything else executes on
|
|
900
|
+
* the MJ server. Either way the result feeds back to the model via
|
|
901
|
+
* {@link BaseRealtimeClient.SendToolResult} so it speaks the outcome.
|
|
902
|
+
*/
|
|
903
|
+
async handleToolCall(call) {
|
|
904
|
+
const clientHandler = this.findClientToolHandler(call.ToolName);
|
|
905
|
+
if (clientHandler) {
|
|
906
|
+
// Local UI tool: no server relay, no 'thinking' turn-state / narration burst — these
|
|
907
|
+
// are fast, in-browser surface mutations (e.g. drawing on the whiteboard).
|
|
908
|
+
const resultJson = await this.executeClientTool(clientHandler, call);
|
|
909
|
+
this.client?.SendToolResult(call.CallID, resultJson);
|
|
910
|
+
// Observability: record the channel tool call on the co-agent's run (run-only — NOT a chat
|
|
911
|
+
// turn). Without this the run shows speech but never the browser_/Whiteboard_ actions the
|
|
912
|
+
// co-agent took. Fire-and-forget; never disturbs the live surface mutation.
|
|
913
|
+
void this.relayToolTurn(call.ToolName, call.ArgumentsJson, resultJson);
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
this._connectionState$.next('thinking');
|
|
917
|
+
if (this.inFlightCallIds.size === 0) {
|
|
918
|
+
// A fresh delegation burst: anchor the first-update delay and clear the digest
|
|
919
|
+
// buffer. Deliberately NOT reset: lastDelegationNarrationAt (the 8s spacing floor
|
|
920
|
+
// is SESSION-global — sequential tool calls seconds apart must not re-arm the
|
|
921
|
+
// faster first-update path, which read as "no debounce") and spokenNarrations
|
|
922
|
+
// (so the story never repeats across closely-spaced calls).
|
|
923
|
+
this.delegationBurstStartedAt = Date.now();
|
|
924
|
+
this.narrationCount = 0;
|
|
925
|
+
this.pendingNarrationMessages = [];
|
|
926
|
+
this.lastNarratedTail = '';
|
|
927
|
+
}
|
|
928
|
+
this.inFlightCallIds.add(call.CallID);
|
|
929
|
+
try {
|
|
930
|
+
const resultJson = await this.executeSessionTool(call.CallID, call.ToolName, call.ArgumentsJson);
|
|
931
|
+
this.emitDelegationResult(call.CallID, resultJson);
|
|
932
|
+
this.client?.SendToolResult(call.CallID, resultJson);
|
|
933
|
+
}
|
|
934
|
+
catch (error) {
|
|
935
|
+
console.error('[RealtimeSession] Tool execution failed:', error);
|
|
936
|
+
// Feed the error back so the model can narrate it rather than going silent.
|
|
937
|
+
// success:false matters: ParseDelegationResultJson treats anything else as
|
|
938
|
+
// success, which would flip the overlay's working card to a SUCCESS card
|
|
939
|
+
// carrying the error text (matches the server broker's failure shape).
|
|
940
|
+
const errorJson = JSON.stringify({
|
|
941
|
+
success: false,
|
|
942
|
+
error: error instanceof Error ? error.message : String(error)
|
|
943
|
+
});
|
|
944
|
+
this.emitDelegationResult(call.CallID, errorJson);
|
|
945
|
+
this.client?.SendToolResult(call.CallID, errorJson);
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
/** Finds the registered client-tool handler whose prefix matches `toolName`, or `null`. */
|
|
949
|
+
findClientToolHandler(toolName) {
|
|
950
|
+
for (const [prefix, handler] of this.clientToolHandlers) {
|
|
951
|
+
if (toolName.startsWith(prefix)) {
|
|
952
|
+
return handler;
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
return null;
|
|
956
|
+
}
|
|
957
|
+
/**
|
|
958
|
+
* Executes one client-tool call through its handler, wrapping any thrown error into a
|
|
959
|
+
* `{ success: false, error }` JSON payload so the model can narrate the failure instead of
|
|
960
|
+
* the call going silent.
|
|
961
|
+
*/
|
|
962
|
+
async executeClientTool(handler, call) {
|
|
963
|
+
try {
|
|
964
|
+
return await handler(call.ToolName, call.ArgumentsJson);
|
|
965
|
+
}
|
|
966
|
+
catch (error) {
|
|
967
|
+
console.error('[RealtimeSession] Client tool execution failed:', error);
|
|
968
|
+
return JSON.stringify({
|
|
969
|
+
success: false,
|
|
970
|
+
error: error instanceof Error ? error.message : String(error)
|
|
971
|
+
});
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Emits a delegation result so the overlay's "working" card flips to a result card with real
|
|
976
|
+
* content. Parses the broker's `{success, output, runId}` | `{success:false, error}` shape via
|
|
977
|
+
* {@link ParseDelegationResultJson}; if it isn't JSON, surfaces the raw string. Only delegation
|
|
978
|
+
* cards (created from progress events) react — non-delegation tool results have no card and are
|
|
979
|
+
* harmlessly ignored downstream. The `runId` (the delegated `MJ: AI Agent Runs` record) rides
|
|
980
|
+
* along as {@link VoiceDelegationResult.RunID} for the overlay's dev links, and any `artifacts`
|
|
981
|
+
* ride along as {@link VoiceDelegationResult.Artifacts} for the surface panel's artifact tabs.
|
|
982
|
+
*/
|
|
983
|
+
emitDelegationResult(callId, resultJson) {
|
|
984
|
+
// The result will be spoken next — a deferred interim update is now pointless
|
|
985
|
+
// (this is what keeps fast agents like Sage from narrating over their own answer),
|
|
986
|
+
// and any progress still in the PubSub pipe for this call is stale.
|
|
987
|
+
this.inFlightCallIds.delete(callId);
|
|
988
|
+
this.cancelPendingNarration();
|
|
989
|
+
if (this.cancelledCallIds.delete(callId)) {
|
|
990
|
+
// The user explicitly cancelled this call: its card already flipped to the
|
|
991
|
+
// "Cancelled by user" failed result, so the aborted run's late outcome must not
|
|
992
|
+
// overwrite it. (The tool result still flows back to the model via the caller.)
|
|
993
|
+
return;
|
|
994
|
+
}
|
|
995
|
+
const parsed = ParseDelegationResultJson(resultJson);
|
|
996
|
+
this._delegationResult$.next({
|
|
997
|
+
CallID: callId,
|
|
998
|
+
Success: parsed.Success,
|
|
999
|
+
Output: parsed.Output,
|
|
1000
|
+
RunID: parsed.RunID,
|
|
1001
|
+
Artifacts: parsed.Artifacts
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
// ── Explicit delegation cancellation (server cancel channel) ───────────────
|
|
1005
|
+
/**
|
|
1006
|
+
* Cancels ONE in-flight delegated tool call — the overlay's per-card ✕ affordance.
|
|
1007
|
+
*
|
|
1008
|
+
* EXPLICIT USER INTENT ONLY (deliberate host policy): true barge-in never aborts
|
|
1009
|
+
* delegations — the narration design expects the user to talk while delegated work runs.
|
|
1010
|
+
* Calls the `CancelRealtimeSessionTool` mutation (ownership-gated server-side); when the
|
|
1011
|
+
* server reports it aborted the run, the card is flipped immediately to a FAILED
|
|
1012
|
+
* "Cancelled by user" result and the eventual late result from the aborted run is
|
|
1013
|
+
* suppressed (see {@link emitDelegationResult}).
|
|
1014
|
+
*
|
|
1015
|
+
* @returns `true` when the server aborted the in-flight run; `false` when there was
|
|
1016
|
+
* nothing to cancel (the work finished first — its real result is already racing in)
|
|
1017
|
+
* or the mutation failed (logged, never thrown).
|
|
1018
|
+
*/
|
|
1019
|
+
async CancelDelegation(callId) {
|
|
1020
|
+
if (!this.agentSessionId || !this.inFlightCallIds.has(callId)) {
|
|
1021
|
+
return false;
|
|
1022
|
+
}
|
|
1023
|
+
const aborted = await this.cancelSessionTool(callId);
|
|
1024
|
+
if (aborted <= 0) {
|
|
1025
|
+
return false; // finished first / nothing in flight server-side — let the real result land
|
|
1026
|
+
}
|
|
1027
|
+
this.surfaceUserCancellation(callId);
|
|
1028
|
+
return true;
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* Cancels EVERY in-flight delegated tool call for the active session (callId-less form of
|
|
1032
|
+
* the `CancelRealtimeSessionTool` mutation). Exposed for host policies that need a
|
|
1033
|
+
* sweep-cancel (e.g. an explicit "stop everything" affordance) — NOT wired to barge-in,
|
|
1034
|
+
* by the same deliberate policy as {@link CancelDelegation}.
|
|
1035
|
+
*
|
|
1036
|
+
* @returns The number of in-flight runs the server aborted (0 when nothing was tracked
|
|
1037
|
+
* in flight client-side, nothing was in flight server-side, or the mutation failed).
|
|
1038
|
+
*/
|
|
1039
|
+
async CancelInFlightDelegations() {
|
|
1040
|
+
if (!this.agentSessionId || this.inFlightCallIds.size === 0) {
|
|
1041
|
+
return 0;
|
|
1042
|
+
}
|
|
1043
|
+
const aborted = await this.cancelSessionTool(null);
|
|
1044
|
+
if (aborted <= 0) {
|
|
1045
|
+
return 0;
|
|
1046
|
+
}
|
|
1047
|
+
for (const callId of [...this.inFlightCallIds]) {
|
|
1048
|
+
this.surfaceUserCancellation(callId);
|
|
1049
|
+
}
|
|
1050
|
+
return aborted;
|
|
1051
|
+
}
|
|
1052
|
+
/** Flips a cancelled call's card to the failed "Cancelled by user" result and suppresses the late real result. */
|
|
1053
|
+
surfaceUserCancellation(callId) {
|
|
1054
|
+
this.inFlightCallIds.delete(callId);
|
|
1055
|
+
this.cancelledCallIds.add(callId);
|
|
1056
|
+
this.cancelPendingNarration();
|
|
1057
|
+
this._delegationResult$.next({
|
|
1058
|
+
CallID: callId,
|
|
1059
|
+
Success: false,
|
|
1060
|
+
Output: 'Cancelled by user'
|
|
1061
|
+
});
|
|
1062
|
+
}
|
|
1063
|
+
/**
|
|
1064
|
+
* Calls the `CancelRealtimeSessionTool` mutation and unwraps its structured
|
|
1065
|
+
* `{ AbortedCount, Success, ErrorMessage }` result. Returns the aborted count —
|
|
1066
|
+
* 0 on a structured failure or a thrown transport error (both logged, never thrown).
|
|
1067
|
+
*/
|
|
1068
|
+
async cancelSessionTool(callId) {
|
|
1069
|
+
try {
|
|
1070
|
+
const mutation = `
|
|
1071
|
+
mutation CancelRealtimeSessionTool($agentSessionId: String!, $callId: String) {
|
|
1072
|
+
CancelRealtimeSessionTool(agentSessionId: $agentSessionId, callId: $callId) {
|
|
1073
|
+
AbortedCount
|
|
1074
|
+
Success
|
|
1075
|
+
ErrorMessage
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
`;
|
|
1079
|
+
const result = await this.gql().ExecuteGQL(mutation, { agentSessionId: this.agentSessionId, callId });
|
|
1080
|
+
const payload = result?.CancelRealtimeSessionTool;
|
|
1081
|
+
if (!payload?.Success) {
|
|
1082
|
+
console.warn(`[RealtimeSession] Cancel reported failure: ${payload?.ErrorMessage ?? 'unknown error'}`);
|
|
1083
|
+
return 0;
|
|
1084
|
+
}
|
|
1085
|
+
return typeof payload.AbortedCount === 'number' ? payload.AbortedCount : 0;
|
|
1086
|
+
}
|
|
1087
|
+
catch (error) {
|
|
1088
|
+
console.error('[RealtimeSession] Failed to cancel in-flight delegation(s):', error);
|
|
1089
|
+
return 0;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
// ── Session minting (GraphQL) ──────────────────────────────────────────────
|
|
1093
|
+
/** Calls the `StartRealtimeClientSession` mutation to obtain an ephemeral token + config. */
|
|
1094
|
+
async mintSession(targetAgentId, conversationId, lastSessionId, preferredModelId, clientTools, coAgentId, configOverridesJson) {
|
|
1095
|
+
const mutation = `
|
|
1096
|
+
mutation StartRealtimeClientSession($targetAgentId: String!, $conversationId: String, $lastSessionId: String, $preferredModelId: String, $clientToolsJson: String, $coAgentId: String, $configOverridesJson: String) {
|
|
1097
|
+
StartRealtimeClientSession(targetAgentId: $targetAgentId, conversationId: $conversationId, lastSessionId: $lastSessionId, preferredModelId: $preferredModelId, clientToolsJson: $clientToolsJson, coAgentId: $coAgentId, configOverridesJson: $configOverridesJson) {
|
|
1098
|
+
AgentSessionId
|
|
1099
|
+
ConversationId
|
|
1100
|
+
Provider
|
|
1101
|
+
Model
|
|
1102
|
+
EphemeralToken
|
|
1103
|
+
ExpiresAt
|
|
1104
|
+
SessionConfigJson
|
|
1105
|
+
ModelName
|
|
1106
|
+
NarrationInstructionsTemplate
|
|
1107
|
+
PriorChannelStatesJson
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
`;
|
|
1111
|
+
const variables = {
|
|
1112
|
+
targetAgentId,
|
|
1113
|
+
conversationId: conversationId ?? null,
|
|
1114
|
+
lastSessionId: lastSessionId ?? null,
|
|
1115
|
+
preferredModelId: preferredModelId ?? null,
|
|
1116
|
+
clientToolsJson: clientTools && clientTools.length > 0 ? JSON.stringify(clientTools) : null,
|
|
1117
|
+
coAgentId: coAgentId ?? null,
|
|
1118
|
+
configOverridesJson: configOverridesJson ?? null
|
|
1119
|
+
};
|
|
1120
|
+
const result = await this.gql().ExecuteGQL(mutation, variables);
|
|
1121
|
+
const payload = result?.StartRealtimeClientSession;
|
|
1122
|
+
if (!payload?.EphemeralToken) {
|
|
1123
|
+
throw new Error('StartRealtimeClientSession returned no ephemeral token');
|
|
1124
|
+
}
|
|
1125
|
+
return payload;
|
|
1126
|
+
}
|
|
1127
|
+
/** Calls the `ExecuteRealtimeSessionTool` mutation; returns the ResultJson string. */
|
|
1128
|
+
async executeSessionTool(callId, toolName, argsJson) {
|
|
1129
|
+
if (!this.agentSessionId) {
|
|
1130
|
+
throw new Error('No active agent session for tool execution');
|
|
1131
|
+
}
|
|
1132
|
+
const mutation = `
|
|
1133
|
+
mutation ExecuteRealtimeSessionTool($agentSessionId: String!, $callId: String!, $toolName: String!, $argsJson: String!) {
|
|
1134
|
+
ExecuteRealtimeSessionTool(agentSessionId: $agentSessionId, callId: $callId, toolName: $toolName, argsJson: $argsJson)
|
|
1135
|
+
}
|
|
1136
|
+
`;
|
|
1137
|
+
const result = await this.gql().ExecuteGQL(mutation, {
|
|
1138
|
+
agentSessionId: this.agentSessionId,
|
|
1139
|
+
callId,
|
|
1140
|
+
toolName,
|
|
1141
|
+
argsJson
|
|
1142
|
+
});
|
|
1143
|
+
return result?.ExecuteRealtimeSessionTool ?? '{}';
|
|
1144
|
+
}
|
|
1145
|
+
/**
|
|
1146
|
+
* Persists an interactive channel's state of record (e.g. the whiteboard's serialized scene)
|
|
1147
|
+
* onto the session's `MJ: AI Agent Session Channels` row via `SaveSessionChannelState`.
|
|
1148
|
+
*
|
|
1149
|
+
* @param channelName The channel definition name (e.g. `'Whiteboard'`).
|
|
1150
|
+
* @param stateJson The serialized channel state.
|
|
1151
|
+
* @param agentSessionId Optional EXPLICIT session id. The debounced channel-save pipeline
|
|
1152
|
+
* captures the id while the session is live and passes it here, so the final teardown
|
|
1153
|
+
* flush still lands on the just-closed session. Falls back to the active session's id;
|
|
1154
|
+
* returns `false` when neither is available.
|
|
1155
|
+
* @returns Whether the server persisted the state. Failures are logged, never thrown — channel
|
|
1156
|
+
* persistence is best-effort and must not disturb the live call.
|
|
1157
|
+
*/
|
|
1158
|
+
async SaveChannelState(channelName, stateJson, agentSessionId) {
|
|
1159
|
+
const sessionId = agentSessionId ?? this.agentSessionId;
|
|
1160
|
+
if (!sessionId) {
|
|
1161
|
+
return false;
|
|
1162
|
+
}
|
|
1163
|
+
try {
|
|
1164
|
+
const mutation = `
|
|
1165
|
+
mutation SaveSessionChannelState($agentSessionId: String!, $channelName: String!, $stateJson: String!) {
|
|
1166
|
+
SaveSessionChannelState(agentSessionId: $agentSessionId, channelName: $channelName, stateJson: $stateJson)
|
|
1167
|
+
}
|
|
1168
|
+
`;
|
|
1169
|
+
const result = await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, channelName, stateJson });
|
|
1170
|
+
return result?.SaveSessionChannelState ?? false;
|
|
1171
|
+
}
|
|
1172
|
+
catch (error) {
|
|
1173
|
+
console.error('[RealtimeSession] Failed to save channel state:', error);
|
|
1174
|
+
return false;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
// ── Transcript relay (GraphQL) ─────────────────────────────────────────────
|
|
1178
|
+
/**
|
|
1179
|
+
* Relays a final transcript turn to MJ via `RelayRealtimeTranscript`.
|
|
1180
|
+
* @param replacesPrevious CORRECTION semantics: the server updates the session's most
|
|
1181
|
+
* recent persisted turn of this role IN PLACE instead of appending (e.g. ElevenLabs'
|
|
1182
|
+
* post-barge-in `agent_response_correction`).
|
|
1183
|
+
*/
|
|
1184
|
+
async relayTranscript(role, text, replacesPrevious = false) {
|
|
1185
|
+
if (!this.agentSessionId) {
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
try {
|
|
1189
|
+
const mutation = `
|
|
1190
|
+
mutation RelayRealtimeTranscript($agentSessionId: String!, $role: String!, $text: String!, $replacesPrevious: Boolean) {
|
|
1191
|
+
RelayRealtimeTranscript(agentSessionId: $agentSessionId, role: $role, text: $text, replacesPrevious: $replacesPrevious)
|
|
1192
|
+
}
|
|
1193
|
+
`;
|
|
1194
|
+
await this.gql().ExecuteGQL(mutation, {
|
|
1195
|
+
agentSessionId: this.agentSessionId,
|
|
1196
|
+
role,
|
|
1197
|
+
text,
|
|
1198
|
+
replacesPrevious
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
catch (error) {
|
|
1202
|
+
console.error('[RealtimeSession] Failed to relay transcript:', error);
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
/**
|
|
1206
|
+
* Relays a co-agent CHANNEL tool-call turn (browser_ / Whiteboard_ etc.) to the session's run for
|
|
1207
|
+
* observability via `RelayRealtimeToolTurn` — so the co-agent's AIPromptRun shows what it DID, not
|
|
1208
|
+
* just what it said. Run-only by design: deliberately NOT a `ConversationDetail` turn, so the chat
|
|
1209
|
+
* thread stays speech-only. Best-effort — a failed relay never disturbs the live call.
|
|
1210
|
+
*/
|
|
1211
|
+
async relayToolTurn(toolName, argsJson, resultJson) {
|
|
1212
|
+
if (!this.agentSessionId) {
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
try {
|
|
1216
|
+
const mutation = `
|
|
1217
|
+
mutation RelayRealtimeToolTurn($agentSessionId: String!, $toolName: String!, $argsJson: String, $resultJson: String) {
|
|
1218
|
+
RelayRealtimeToolTurn(agentSessionId: $agentSessionId, toolName: $toolName, argsJson: $argsJson, resultJson: $resultJson)
|
|
1219
|
+
}
|
|
1220
|
+
`;
|
|
1221
|
+
await this.gql().ExecuteGQL(mutation, {
|
|
1222
|
+
agentSessionId: this.agentSessionId,
|
|
1223
|
+
toolName,
|
|
1224
|
+
argsJson,
|
|
1225
|
+
resultJson
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
catch (error) {
|
|
1229
|
+
console.error('[RealtimeSession] Failed to relay tool turn:', error);
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
// ── Usage telemetry relay (B7) ─────────────────────────────────────────────
|
|
1233
|
+
/**
|
|
1234
|
+
* Accumulates one usage DELTA from the realtime client (per-response token counts —
|
|
1235
|
+
* the `OnUsage` contract shape) and schedules the debounced relay. Negative / non-finite
|
|
1236
|
+
* values are clamped to 0; an all-zero delta is dropped without arming the timer.
|
|
1237
|
+
*/
|
|
1238
|
+
onUsageDelta(usage) {
|
|
1239
|
+
const input = this.clampUsageDelta(usage.InputTokens);
|
|
1240
|
+
const output = this.clampUsageDelta(usage.OutputTokens);
|
|
1241
|
+
if (input === 0 && output === 0) {
|
|
1242
|
+
return;
|
|
1243
|
+
}
|
|
1244
|
+
this.pendingUsageInput += input;
|
|
1245
|
+
this.pendingUsageOutput += output;
|
|
1246
|
+
if (!this.usageFlushTimer) {
|
|
1247
|
+
this.usageFlushTimer = setTimeout(() => {
|
|
1248
|
+
this.usageFlushTimer = null;
|
|
1249
|
+
void this.flushPendingUsage();
|
|
1250
|
+
}, RealtimeSessionService.UsageFlushDebounceMs);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
/** Clamps a driver-reported token delta: undefined / negative / non-finite become 0. */
|
|
1254
|
+
clampUsageDelta(value) {
|
|
1255
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
1256
|
+
}
|
|
1257
|
+
/**
|
|
1258
|
+
* Relays the accumulated usage deltas to the server via `RelayRealtimeUsage` (which
|
|
1259
|
+
* accumulates them onto the co-agent `AIPromptRun`). Best-effort: a failed relay
|
|
1260
|
+
* re-accumulates the captured deltas so the next debounce / teardown flush retries —
|
|
1261
|
+
* usage telemetry must never disturb the live call.
|
|
1262
|
+
*
|
|
1263
|
+
* @param agentSessionId Optional EXPLICIT session id (the teardown flush runs while the
|
|
1264
|
+
* live id is still set, but accepts it as a parameter for symmetry with channel saves).
|
|
1265
|
+
*/
|
|
1266
|
+
async flushPendingUsage(agentSessionId) {
|
|
1267
|
+
const sessionId = agentSessionId ?? this.agentSessionId;
|
|
1268
|
+
const input = this.pendingUsageInput;
|
|
1269
|
+
const output = this.pendingUsageOutput;
|
|
1270
|
+
if (!sessionId || (input === 0 && output === 0)) {
|
|
1271
|
+
return;
|
|
1272
|
+
}
|
|
1273
|
+
this.pendingUsageInput = 0;
|
|
1274
|
+
this.pendingUsageOutput = 0;
|
|
1275
|
+
try {
|
|
1276
|
+
const mutation = `
|
|
1277
|
+
mutation RelayRealtimeUsage($agentSessionId: String!, $inputTokens: Int!, $outputTokens: Int!) {
|
|
1278
|
+
RelayRealtimeUsage(agentSessionId: $agentSessionId, inputTokens: $inputTokens, outputTokens: $outputTokens)
|
|
1279
|
+
}
|
|
1280
|
+
`;
|
|
1281
|
+
await this.gql().ExecuteGQL(mutation, { agentSessionId: sessionId, inputTokens: input, outputTokens: output });
|
|
1282
|
+
}
|
|
1283
|
+
catch (error) {
|
|
1284
|
+
console.error('[RealtimeSession] Failed to relay usage telemetry:', error);
|
|
1285
|
+
// Re-accumulate so a later debounce / the teardown flush retries the same deltas.
|
|
1286
|
+
this.pendingUsageInput += input;
|
|
1287
|
+
this.pendingUsageOutput += output;
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
/** Cancels the pending debounced usage flush and zeroes the accumulators (teardown tail). */
|
|
1291
|
+
resetUsageRelay() {
|
|
1292
|
+
if (this.usageFlushTimer) {
|
|
1293
|
+
clearTimeout(this.usageFlushTimer);
|
|
1294
|
+
this.usageFlushTimer = null;
|
|
1295
|
+
}
|
|
1296
|
+
this.pendingUsageInput = 0;
|
|
1297
|
+
this.pendingUsageOutput = 0;
|
|
1298
|
+
}
|
|
1299
|
+
// ── Delegated-run progress streaming ───────────────────────────────────────
|
|
1300
|
+
/**
|
|
1301
|
+
* Subscribes to the server's push-status topic (scoped by the GraphQL transport
|
|
1302
|
+
* sessionId) to receive delegated-run progress for the active voice session.
|
|
1303
|
+
* Each matching event is surfaced on {@link DelegationProgress$} and narrated.
|
|
1304
|
+
*/
|
|
1305
|
+
subscribeDelegationProgress() {
|
|
1306
|
+
if (this.delegationProgressSub) {
|
|
1307
|
+
return; // already subscribed for this session
|
|
1308
|
+
}
|
|
1309
|
+
const transportSessionId = this.gql().sessionId;
|
|
1310
|
+
this.lastDelegationNarrationAt = 0;
|
|
1311
|
+
this.delegationProgressSub = this.gql()
|
|
1312
|
+
.PushStatusUpdates(transportSessionId)
|
|
1313
|
+
.subscribe({
|
|
1314
|
+
next: (raw) => this.onDelegationStatusMessage(raw),
|
|
1315
|
+
error: (err) => console.error('[RealtimeSession] Delegation progress stream error:', err)
|
|
1316
|
+
});
|
|
1317
|
+
}
|
|
1318
|
+
/**
|
|
1319
|
+
* Parses one push-status message and routes it: a Remote Browser screencast frame goes to the active
|
|
1320
|
+
* Remote Browser channel's canvas; a delegation-progress event is dispatched + narrated. Other shapes
|
|
1321
|
+
* (normal agent-run streams) are ignored. Screencast frames are checked FIRST and short-circuit, so the
|
|
1322
|
+
* delegation path is untouched.
|
|
1323
|
+
*/
|
|
1324
|
+
onDelegationStatusMessage(raw) {
|
|
1325
|
+
const frame = this.parseScreencastFrame(raw);
|
|
1326
|
+
if (frame) {
|
|
1327
|
+
this.routeScreencastFrame(frame);
|
|
1328
|
+
return;
|
|
1329
|
+
}
|
|
1330
|
+
const audio = this.parseAudioChunk(raw);
|
|
1331
|
+
if (audio) {
|
|
1332
|
+
this.routeAudioChunk(audio);
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
const progress = this.parseProgress(raw);
|
|
1336
|
+
if (progress) {
|
|
1337
|
+
this.dispatchProgress(progress);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Parses a push-status message and returns it only when it's a Remote Browser screencast frame for the
|
|
1342
|
+
* active session — otherwise `null` (ignored, so delegation progress falls through). Matched by
|
|
1343
|
+
* `resolver` + `type`, then scoped to THIS session by `agentSessionID`.
|
|
1344
|
+
*/
|
|
1345
|
+
parseScreencastFrame(raw) {
|
|
1346
|
+
let payload;
|
|
1347
|
+
try {
|
|
1348
|
+
payload = JSON.parse(raw);
|
|
1349
|
+
}
|
|
1350
|
+
catch {
|
|
1351
|
+
return null;
|
|
1352
|
+
}
|
|
1353
|
+
const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
|
|
1354
|
+
payload?.type === 'RemoteBrowserScreencastFrame' &&
|
|
1355
|
+
payload?.agentSessionID === this.agentSessionId &&
|
|
1356
|
+
typeof payload?.dataBase64 === 'string';
|
|
1357
|
+
return matches ? payload : null;
|
|
1358
|
+
}
|
|
1359
|
+
/**
|
|
1360
|
+
* Forwards a screencast frame to the active Remote Browser channel plugin so it paints the frame on its
|
|
1361
|
+
* surface canvas. The plugin is found among the session's active channels by its `ChannelName`; located
|
|
1362
|
+
* via a structural guard so the service stays decoupled from the concrete channel class.
|
|
1363
|
+
*/
|
|
1364
|
+
routeScreencastFrame(frame) {
|
|
1365
|
+
for (const channel of this._activeChannels$.value) {
|
|
1366
|
+
if (channel.ChannelName === 'Remote Browser' && this.hasOnScreencastFrame(channel)) {
|
|
1367
|
+
channel.OnScreencastFrame(frame.dataBase64);
|
|
1368
|
+
return;
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
}
|
|
1372
|
+
/** Structural guard: true when the channel exposes an `OnScreencastFrame(dataBase64)` method. */
|
|
1373
|
+
hasOnScreencastFrame(channel) {
|
|
1374
|
+
return typeof channel.OnScreencastFrame === 'function';
|
|
1375
|
+
}
|
|
1376
|
+
/**
|
|
1377
|
+
* Parses a push-status message and returns it only when it's a Remote Browser audio chunk for the active
|
|
1378
|
+
* session — otherwise `null` (ignored). Matched by `resolver` + `type`, then scoped to THIS session by
|
|
1379
|
+
* `agentSessionID`.
|
|
1380
|
+
*/
|
|
1381
|
+
parseAudioChunk(raw) {
|
|
1382
|
+
let payload;
|
|
1383
|
+
try {
|
|
1384
|
+
payload = JSON.parse(raw);
|
|
1385
|
+
}
|
|
1386
|
+
catch {
|
|
1387
|
+
return null;
|
|
1388
|
+
}
|
|
1389
|
+
const matches = payload?.resolver === 'RemoteBrowserActionResolver' &&
|
|
1390
|
+
payload?.type === 'RemoteBrowserAudioChunk' &&
|
|
1391
|
+
payload?.agentSessionID === this.agentSessionId &&
|
|
1392
|
+
typeof payload?.dataBase64 === 'string';
|
|
1393
|
+
return matches ? payload : null;
|
|
1394
|
+
}
|
|
1395
|
+
/**
|
|
1396
|
+
* Forwards an audio chunk to the active Remote Browser channel plugin so it plays the chunk through its
|
|
1397
|
+
* client-side audio player. The plugin is found among the session's active channels by its `ChannelName`;
|
|
1398
|
+
* located via a structural guard so the service stays decoupled from the concrete channel class.
|
|
1399
|
+
*/
|
|
1400
|
+
routeAudioChunk(chunk) {
|
|
1401
|
+
for (const channel of this._activeChannels$.value) {
|
|
1402
|
+
if (channel.ChannelName === 'Remote Browser' && this.hasOnAudioChunk(channel)) {
|
|
1403
|
+
channel.OnAudioChunk({
|
|
1404
|
+
dataBase64: chunk.dataBase64,
|
|
1405
|
+
codec: chunk.codec,
|
|
1406
|
+
sampleRate: chunk.sampleRate,
|
|
1407
|
+
channels: chunk.channels,
|
|
1408
|
+
seq: chunk.seq,
|
|
1409
|
+
});
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
/** Structural guard: true when the channel exposes an `OnAudioChunk(chunk)` method. */
|
|
1415
|
+
hasOnAudioChunk(channel) {
|
|
1416
|
+
return typeof channel.OnAudioChunk === 'function';
|
|
1417
|
+
}
|
|
1418
|
+
/**
|
|
1419
|
+
* Parses a push-status message and returns it only when it's a delegation
|
|
1420
|
+
* progress event for the active voice session — otherwise `null` (ignored).
|
|
1421
|
+
*/
|
|
1422
|
+
parseProgress(raw) {
|
|
1423
|
+
let payload;
|
|
1424
|
+
try {
|
|
1425
|
+
payload = JSON.parse(raw);
|
|
1426
|
+
}
|
|
1427
|
+
catch {
|
|
1428
|
+
return null; // non-JSON or unrelated frame
|
|
1429
|
+
}
|
|
1430
|
+
const matches = payload?.resolver === 'RealtimeClientSessionResolver' &&
|
|
1431
|
+
payload?.type === 'RealtimeDelegationProgress' &&
|
|
1432
|
+
payload?.agentSessionID === this.agentSessionId;
|
|
1433
|
+
if (!matches) {
|
|
1434
|
+
return null;
|
|
1435
|
+
}
|
|
1436
|
+
return {
|
|
1437
|
+
CallID: payload.callID,
|
|
1438
|
+
Step: payload.step,
|
|
1439
|
+
Message: payload.message,
|
|
1440
|
+
Percentage: payload.percentage
|
|
1441
|
+
};
|
|
1442
|
+
}
|
|
1443
|
+
/** Emits the progress to the UI observable and feeds it to the realtime model. */
|
|
1444
|
+
dispatchProgress(progress) {
|
|
1445
|
+
// Drop stale progress: PubSub delivery can lag the mutation result, so events for a
|
|
1446
|
+
// call that already completed (or was never seen) must not update cards or narrate.
|
|
1447
|
+
if (!this.inFlightCallIds.has(progress.CallID)) {
|
|
1448
|
+
return;
|
|
1449
|
+
}
|
|
1450
|
+
this._delegationProgress$.next(progress);
|
|
1451
|
+
this.narrateProgress(progress);
|
|
1452
|
+
}
|
|
1453
|
+
/**
|
|
1454
|
+
* Injects the progress into the model's context as a background note every time,
|
|
1455
|
+
* then (throttled) asks the model to briefly voice a reassuring update so the
|
|
1456
|
+
* background work doesn't sit in silence — without chattering or interrupting.
|
|
1457
|
+
*/
|
|
1458
|
+
narrateProgress(progress) {
|
|
1459
|
+
const client = this.client;
|
|
1460
|
+
if (!client) {
|
|
1461
|
+
return;
|
|
1462
|
+
}
|
|
1463
|
+
client.SendContextNote(`[delegated-agent progress] ${progress.Message}`);
|
|
1464
|
+
// Floods of small updates AGGREGATE: each distinct message joins the digest buffer,
|
|
1465
|
+
// and ONE spoken update fires per window (first at ~5s into the burst, then every
|
|
1466
|
+
// ~8s). The buffer is discarded if the final result lands first.
|
|
1467
|
+
this.bufferNarrationMessage(progress.Message);
|
|
1468
|
+
if (this.pendingNarrationMessages.length > 0 && !this.narrationTimer) {
|
|
1469
|
+
this.narrationTimer = setTimeout(() => this.fireDeferredNarration(), this.nextNarrationDelayMs());
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
/** Adds a progress message to the digest buffer (deduped, capped, oldest-first). */
|
|
1473
|
+
bufferNarrationMessage(message) {
|
|
1474
|
+
if (message === this.lastNarratedTail || this.pendingNarrationMessages.includes(message)) {
|
|
1475
|
+
return;
|
|
1476
|
+
}
|
|
1477
|
+
this.pendingNarrationMessages.push(message);
|
|
1478
|
+
if (this.pendingNarrationMessages.length > RealtimeSessionService.MaxDigestMessages) {
|
|
1479
|
+
this.pendingNarrationMessages.shift();
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
/**
|
|
1483
|
+
* ms until the next spoken update is allowed. Two constraints, BOTH enforced:
|
|
1484
|
+
* - first update of a burst: no earlier than ~5s after the burst started;
|
|
1485
|
+
* - ~8s since the last spoken update, SESSION-global — so sequential tool calls
|
|
1486
|
+
* that reset the burst can never narrate faster than the interval.
|
|
1487
|
+
*/
|
|
1488
|
+
nextNarrationDelayMs() {
|
|
1489
|
+
const now = Date.now();
|
|
1490
|
+
const firstAnchor = this.narrationCount === 0
|
|
1491
|
+
? this.delegationBurstStartedAt + RealtimeSessionService.FirstNarrationDelayMs
|
|
1492
|
+
: 0;
|
|
1493
|
+
const spacingFloor = this.lastDelegationNarrationAt > 0
|
|
1494
|
+
? this.lastDelegationNarrationAt + RealtimeSessionService.NarrationIntervalMs
|
|
1495
|
+
: 0;
|
|
1496
|
+
return Math.max(250, Math.max(firstAnchor, spacingFloor) - now);
|
|
1497
|
+
}
|
|
1498
|
+
/**
|
|
1499
|
+
* Speaks the aggregated progress digest — unless the work already finished (buffer
|
|
1500
|
+
* cancelled) or the model is busy / audio is still playing, in which case it retries
|
|
1501
|
+
* shortly with the buffer intact (work is still running, so the update stays relevant).
|
|
1502
|
+
*/
|
|
1503
|
+
fireDeferredNarration() {
|
|
1504
|
+
this.narrationTimer = null;
|
|
1505
|
+
const client = this.client;
|
|
1506
|
+
if (this.pendingNarrationMessages.length === 0 || !client || this.inFlightCallIds.size === 0) {
|
|
1507
|
+
this.pendingNarrationMessages = [];
|
|
1508
|
+
return;
|
|
1509
|
+
}
|
|
1510
|
+
if (client.IsBusy || client.IsAudioPlaying) {
|
|
1511
|
+
this.narrationTimer = setTimeout(() => this.fireDeferredNarration(), RealtimeSessionService.NarrationBusyRetryMs);
|
|
1512
|
+
return;
|
|
1513
|
+
}
|
|
1514
|
+
const digest = this.pendingNarrationMessages.join(' → ');
|
|
1515
|
+
this.lastNarratedTail = this.pendingNarrationMessages[this.pendingNarrationMessages.length - 1];
|
|
1516
|
+
this.pendingNarrationMessages = [];
|
|
1517
|
+
this.narrationCount++;
|
|
1518
|
+
this.lastDelegationNarrationAt = Date.now();
|
|
1519
|
+
client.RequestSpokenUpdate(this.buildNarrationInstructions(digest));
|
|
1520
|
+
}
|
|
1521
|
+
/** Cancels any deferred narration — the result is about to be spoken, so it's moot. */
|
|
1522
|
+
cancelPendingNarration() {
|
|
1523
|
+
if (this.narrationTimer) {
|
|
1524
|
+
clearTimeout(this.narrationTimer);
|
|
1525
|
+
this.narrationTimer = null;
|
|
1526
|
+
}
|
|
1527
|
+
this.pendingNarrationMessages = [];
|
|
1528
|
+
}
|
|
1529
|
+
/**
|
|
1530
|
+
* Builds the one-off instructions for a short spoken update that conveys THIS specific
|
|
1531
|
+
* progress message naturally — strictly first person, since the co-agent owns the work.
|
|
1532
|
+
* The wording is DB-driven: the server-resolved `Realtime Co-Agent - Progress Narration`
|
|
1533
|
+
* template (substituting `{{ progressMessage }}`) when present, otherwise the built-in
|
|
1534
|
+
* fallback so deployments that haven't synced the prompt behave exactly as before.
|
|
1535
|
+
* The client tags the resulting turn as narration, keeping it EPHEMERAL — surfaced on
|
|
1536
|
+
* {@link DelegationNarration$} instead of becoming a caption / persisted ConversationDetail.
|
|
1537
|
+
*/
|
|
1538
|
+
buildNarrationInstructions(digest) {
|
|
1539
|
+
return BuildNarrationInstructions(this.narrationTemplate, digest, {
|
|
1540
|
+
PriorNarrations: this.spokenNarrations.slice(-RealtimeSessionService.MaxPriorNarrations),
|
|
1541
|
+
UpdateNumber: this.narrationCount
|
|
1542
|
+
});
|
|
1543
|
+
}
|
|
1544
|
+
/** Tears down the delegation progress subscription and resets the narration throttle. */
|
|
1545
|
+
teardownDelegationProgress() {
|
|
1546
|
+
if (this.delegationProgressSub) {
|
|
1547
|
+
this.delegationProgressSub.unsubscribe();
|
|
1548
|
+
this.delegationProgressSub = null;
|
|
1549
|
+
}
|
|
1550
|
+
this.cancelPendingNarration();
|
|
1551
|
+
this.inFlightCallIds.clear();
|
|
1552
|
+
this.cancelledCallIds.clear();
|
|
1553
|
+
this.lastDelegationNarrationAt = 0;
|
|
1554
|
+
this.delegationBurstStartedAt = 0;
|
|
1555
|
+
this.narrationCount = 0;
|
|
1556
|
+
this.spokenNarrations = [];
|
|
1557
|
+
this.lastNarratedTail = '';
|
|
1558
|
+
}
|
|
1559
|
+
// ── Teardown ───────────────────────────────────────────────────────────────
|
|
1560
|
+
/**
|
|
1561
|
+
* Tears down all client resources and (optionally) closes the server session.
|
|
1562
|
+
* @param closeServerSession when true, calls `CloseAgentSession` on the server.
|
|
1563
|
+
*/
|
|
1564
|
+
async teardown(closeServerSession) {
|
|
1565
|
+
this.teardownDelegationProgress();
|
|
1566
|
+
// Channels first: flush any unsaved channel state WHILE the live session id is still
|
|
1567
|
+
// set (the captured per-save id covers the race anyway), then dispose the plugins.
|
|
1568
|
+
this.flushAllChannelSaves();
|
|
1569
|
+
this.disposeChannels();
|
|
1570
|
+
// Defensive: stop the mic even when Connect never ran (the client also stops the
|
|
1571
|
+
// tracks it was handed — track.stop() is idempotent).
|
|
1572
|
+
this.localStream?.getTracks().forEach(t => t.stop());
|
|
1573
|
+
this.localStream = null;
|
|
1574
|
+
if (this.client) {
|
|
1575
|
+
await this.client.Disconnect();
|
|
1576
|
+
this.client = null;
|
|
1577
|
+
}
|
|
1578
|
+
// Final usage flush WHILE the live session id is still set (the relay mutation also
|
|
1579
|
+
// accepts a Closed session, so ordering vs. CloseAgentSession is belt-and-braces).
|
|
1580
|
+
if (this.usageFlushTimer) {
|
|
1581
|
+
clearTimeout(this.usageFlushTimer);
|
|
1582
|
+
this.usageFlushTimer = null;
|
|
1583
|
+
}
|
|
1584
|
+
await this.flushPendingUsage(this.agentSessionId);
|
|
1585
|
+
this.resetUsageRelay();
|
|
1586
|
+
if (closeServerSession && this.agentSessionId) {
|
|
1587
|
+
await this.closeServerSession(this.agentSessionId);
|
|
1588
|
+
}
|
|
1589
|
+
// Capture the session id BEFORE we null it so the lifecycle emit carries it.
|
|
1590
|
+
// Skip emitting when there was no live session (defensive — teardown is safe
|
|
1591
|
+
// to call without an active session).
|
|
1592
|
+
const closedSessionId = this.agentSessionId;
|
|
1593
|
+
this.agentSessionId = null;
|
|
1594
|
+
this.narrationTemplate = null;
|
|
1595
|
+
this.clientToolHandlers.clear();
|
|
1596
|
+
this._modelName$.next(null);
|
|
1597
|
+
this.SetMinimized(false);
|
|
1598
|
+
this._active$.next(false);
|
|
1599
|
+
if (this._connectionState$.value !== 'error') {
|
|
1600
|
+
this._connectionState$.next('closed');
|
|
1601
|
+
}
|
|
1602
|
+
// Surface generic session-ended for the conversations runtime bridge.
|
|
1603
|
+
// `closeServerSession=true` means the user explicitly called EndVoiceSession;
|
|
1604
|
+
// `false` means teardown ran from a catch block (start path error path).
|
|
1605
|
+
if (closedSessionId) {
|
|
1606
|
+
this._sessionEnded$.next({
|
|
1607
|
+
sessionId: closedSessionId,
|
|
1608
|
+
reason: closeServerSession ? 'explicit' : 'error',
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
/** Calls the `CloseAgentSession` mutation (provisioned in P4b). */
|
|
1613
|
+
async closeServerSession(agentSessionId) {
|
|
1614
|
+
try {
|
|
1615
|
+
const mutation = `
|
|
1616
|
+
mutation CloseAgentSession($agentSessionId: String!) {
|
|
1617
|
+
CloseAgentSession(agentSessionId: $agentSessionId)
|
|
1618
|
+
}
|
|
1619
|
+
`;
|
|
1620
|
+
await this.gql().ExecuteGQL(mutation, { agentSessionId });
|
|
1621
|
+
}
|
|
1622
|
+
catch (error) {
|
|
1623
|
+
console.error('[RealtimeSession] Failed to close server session:', error);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
// ── Helpers ────────────────────────────────────────────────────────────────
|
|
1627
|
+
/** Pushes a caption onto the live list (immutable update for change detection). */
|
|
1628
|
+
appendCaption(caption) {
|
|
1629
|
+
this._captions$.next([...this._captions$.value, caption]);
|
|
1630
|
+
}
|
|
1631
|
+
/** Resets reactive + internal state at the start of a session. */
|
|
1632
|
+
resetState() {
|
|
1633
|
+
this._captions$.next([]);
|
|
1634
|
+
this.SetMinimized(false);
|
|
1635
|
+
}
|
|
1636
|
+
/** The GraphQL provider used for relay mutations. */
|
|
1637
|
+
gql() {
|
|
1638
|
+
return this.Provider;
|
|
1639
|
+
}
|
|
1640
|
+
static ɵfac = function RealtimeSessionService_Factory(__ngFactoryType__) { return new (__ngFactoryType__ || RealtimeSessionService)(); };
|
|
1641
|
+
static ɵprov = /*@__PURE__*/ i0.ɵɵdefineInjectable({ token: RealtimeSessionService, factory: RealtimeSessionService.ɵfac, providedIn: 'root' });
|
|
1642
|
+
}
|
|
1643
|
+
(() => { (typeof ngDevMode === "undefined" || ngDevMode) && i0.ɵsetClassMetadata(RealtimeSessionService, [{
|
|
1644
|
+
type: Injectable,
|
|
1645
|
+
args: [{ providedIn: 'root' }]
|
|
1646
|
+
}], null, null); })();
|
|
1647
|
+
//# sourceMappingURL=realtime-session.service.js.map
|