@memberjunction/ng-conversations 6.1.1 → 6.1.3

package/dist/lib/services/realtime-session.service.js

@@ -4,6 +4,7 @@ import { Metadata } from '@memberjunction/core';
 import { UserInfoEngine } from '@memberjunction/core-entities';
 import { AIEngineBase } from '@memberjunction/ai-engine-base';
 import { MJGlobal } from '@memberjunction/global';
+import { DEFAULT_REALTIME_AUDIO_TRACKS } from '@memberjunction/ai';
 import { BaseRealtimeClient, LoadAssemblyAIRealtimeClient, LoadElevenLabsRealtimeClient, LoadGeminiRealtimeClient, LoadHuggingFaceRealtimeClient, LoadOpenAIRealtimeClient, LoadxAIRealtimeClient } from '@memberjunction/ai-realtime-client';
 import { BuildNarrationInstructions } from './narration-template';
 import { ParseDelegationResultJson, FormatToolName } from './delegation-result-parser';
@@ -30,6 +31,45 @@ LoadElevenLabsRealtimeClient();
 LoadAssemblyAIRealtimeClient();
 LoadxAIRealtimeClient();
 LoadHuggingFaceRealtimeClient();
+/**
+ * Converts a {@link RealtimeTrackDescriptor} to its JSON form for the session-config bag.
+ *
+ * Every field's VALUE is already JSON-safe; the interface simply is not assignable to `JSONValue`
+ * because it declares no index signature and `UsageBasis` is `readonly`. Written out field by field
+ * rather than asserted, so adding a descriptor field is a compile error here instead of a field that
+ * silently stops reaching the driver.
+ */
+function trackDescriptorToJSON(track) {
+    const json = { Modality: track.Modality, Direction: track.Direction };
+    if (track.Encoding !== undefined) {
+        json['Encoding'] = track.Encoding;
+    }
+    if (track.Rate !== undefined) {
+        json['Rate'] = track.Rate;
+    }
+    if (track.UsageBasis !== undefined) {
+        json['UsageBasis'] = [...track.UsageBasis];
+    }
+    if (track.RequiresConsent !== undefined) {
+        json['RequiresConsent'] = track.RequiresConsent;
+    }
+    return json;
+}
+/**
+ * Reads the `Direction:Modality` dedupe key off an already-JSON track entry, or `null` when the
+ * entry is not a track-shaped object. Used for tracks the mint supplied, which arrive as raw JSON.
+ */
+function trackKeyFromJSON(raw) {
+    if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) {
+        return null;
+    }
+    const direction = raw['Direction'];
+    const modality = raw['Modality'];
+    if (typeof direction !== 'string' || typeof modality !== 'string') {
+        return null;
+    }
+    return `${direction}:${modality}`;
+}
 /**
  * Drives a **client-direct** real-time voice session: the browser mints an ephemeral
  * token from the MJ server, then connects DIRECTLY to the realtime provider. Audio
@@ -60,6 +100,7 @@ export class RealtimeSessionService {
     _delegationProgress$ = new Subject();
     _delegationResult$ = new Subject();
     _delegationNarration$ = new Subject();
+    _thoughtNarration$ = new Subject();
     _agentName$ = new BehaviorSubject('Sage');
     _modelName$ = new BehaviorSubject(null);
     _minimized$ = new BehaviorSubject(false);
@@ -93,6 +134,11 @@ export class RealtimeSessionService {
      * renders them as a transient "live note" near the active working card.
      */
     DelegationNarration$ = this._delegationNarration$.asObservable();
+    /**
+     * Model-authored thought / reasoning narrations (see {@link RealtimeThoughtNarration}). These are
+     * reasoning summaries author-emitted during extended thinking, separate from spoken progress updates.
+     */
+    ThoughtNarration$ = this._thoughtNarration$.asObservable();
     /** Display name of the agent the active session fronts (set at session start). */
     AgentName$ = this._agentName$.asObservable();
     /**
@@ -538,6 +584,15 @@ export class RealtimeSessionService {
             this.wireClientHandlers(client);
             this.localStream = await navigator.mediaDevices.getUserMedia({ audio: true });
             await client.Connect(this.buildClientConfig(session), this.localStream);
+            // Notify active channels that the session client is connected and tracks are established
+            for (const channel of this._activeChannels$.value) {
+                try {
+                    channel.OnSessionStarted?.();
+                }
+                catch (err) {
+                    console.error(`[RealtimeSession] Error in channel '${channel.ChannelName}' OnSessionStarted:`, err);
+                }
+            }
             // Start browser-side recording (mic + agent mix) when consented. Best-effort: an
             // unsupported browser / missing remote stream degrades gracefully (mic-only or off)
             // and never blocks the call. The remote stream may still be null here (the WebRTC
@@ -689,6 +744,27 @@ export class RealtimeSessionService {
     GetAudioActivity() {
         return this.client?.GetAudioActivity() ?? null;
     }
+    /**
+     * The active {@link BaseRealtimeClient} driving the media plane, or null when not connected.
+     */
+    get Client() {
+        return this.client;
+    }
+    /**
+     * Relays a video frame to the underlying realtime client if active.
+     */
+    SendVideoFrame(base64Image, mimeType) {
+        if (!this.client || !this.isSessionLive()) {
+            return;
+        }
+        this.client.SendVideoFrame?.(base64Image, mimeType);
+    }
+    /**
+     * Checks whether a media track is established on the active realtime client.
+     */
+    IsTrackEstablished(modality, direction) {
+        return this.client?.IsTrackEstablished(modality, direction) ?? false;
+    }
     // ── Browser-side call recording ────────────────────────────────────────────
     /**
      * Reads the per-user recording-consent preference from `MJ: User Settings` (via
@@ -1040,7 +1116,12 @@ export class RealtimeSessionService {
             // App-context stream + client-tool execution for the headless ClientContextChannel. The host
             // (Explorer) feeds both; absent on hosts that supply no app context / register no client tools.
             AppContext$: this.AppContext$,
-            ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params)
+            ExecuteClientTool: (name, params) => this.executeAppClientTool(name, params),
+            get Client() {
+                return service.client;
+            },
+            SendVideoFrame: (base64Image, mimeType) => this.SendVideoFrame(base64Image, mimeType),
+            IsTrackEstablished: (modality, direction) => this.IsTrackEstablished(modality, direction),
         };
     }
     /**
@@ -1266,14 +1347,44 @@ export class RealtimeSessionService {
         }
         return client;
     }
-    /** Builds the client-direct session config the realtime client connects with. */
+    /**
+     * Builds the client-direct session config the realtime client connects with.
+     * Aggregates tracks sourced by active channels into `requestedTracks` so the driver
+     * can negotiate them (e.g., establishing inbound video streaming for Whiteboard / RemoteBrowser).
+     */
     buildClientConfig(session) {
+        const sessionConfig = this.parseSessionConfig(session.SessionConfigJson);
+        const channelTracks = this._activeChannels$.value.flatMap((c) => c.GetSourcedTracks());
+        if (channelTracks.length > 0) {
+            // `requestedTracks` crosses a JSON boundary — the driver reads it back out of the session
+            // config bag (`GeminiRealtimeClient.parseSessionConfig`). A `RealtimeTrackDescriptor` is NOT
+            // structurally a `JSONValue`: it has no index signature and `UsageBasis` is readonly, so the
+            // conversion is written out rather than asserted. Dedupe key and precedence are unchanged —
+            // audio floor first, then anything the mint supplied, then the channels' own tracks.
+            const existing = Array.isArray(sessionConfig['requestedTracks'])
+                ? sessionConfig['requestedTracks']
+                : [];
+            const trackMap = new Map();
+            for (const t of DEFAULT_REALTIME_AUDIO_TRACKS) {
+                trackMap.set(`${t.Direction}:${t.Modality}`, trackDescriptorToJSON(t));
+            }
+            for (const raw of existing) {
+                const key = trackKeyFromJSON(raw);
+                if (key) {
+                    trackMap.set(key, raw);
+                }
+            }
+            for (const t of channelTracks) {
+                trackMap.set(`${t.Direction}:${t.Modality}`, trackDescriptorToJSON(t));
+            }
+            sessionConfig['requestedTracks'] = Array.from(trackMap.values());
+        }
         return {
             Provider: session.Provider,
             Model: session.Model,
             EphemeralToken: session.EphemeralToken,
             ExpiresAt: session.ExpiresAt,
-            SessionConfig: this.parseSessionConfig(session.SessionConfigJson)
+            SessionConfig: sessionConfig
         };
     }
     /**
@@ -1391,11 +1502,20 @@ export class RealtimeSessionService {
             this.hasActiveInterimUserCaption = false;
             this.pendingUserCaption = '';
             if (transcript.Kind === 'narration') {
-                this._delegationNarration$.next({ Text: transcript.Text });
-                // Remember what was actually SAID so later updates build on it instead of repeating.
-                this.spokenNarrations.push(transcript.Text);
-                if (this.spokenNarrations.length > RealtimeSessionService.MaxPriorNarrations) {
-                    this.spokenNarrations.shift();
+                if (transcript.IsThought) {
+                    this._thoughtNarration$.next({
+                        CallID: 'thought-session',
+                        Text: transcript.Text,
+                        IsFinal: transcript.IsFinal ?? true,
+                    });
+                }
+                else {
+                    this._delegationNarration$.next({ Text: transcript.Text });
+                    // Remember what was actually SAID so later updates build on it instead of repeating.
+                    this.spokenNarrations.push(transcript.Text);
+                    if (this.spokenNarrations.length > RealtimeSessionService.MaxPriorNarrations) {
+                        this.spokenNarrations.shift();
+                    }
                 }
             }
             else if (transcript.ReplacesPrevious) {