@chat21/chat21-web-widget 5.1.32-rc8 → 5.1.33-rc8

package/src/app/providers/voice/voice-streaming.types.ts

@@ -0,0 +1,113 @@
+/**
+ * Configurazione sessione WSS /ws/voice.
+ *
+ * L'URL di connessione porta solo: token, mimeType, sttProvider, ttsProvider (ADR-002).
+ * I campi di identità di sessione viaggiano nel config frame JSON inviato subito dopo onopen.
+ *
+ * Il config frame fonde i campi di routing Chat21 (`sender`, `recipient`, `lang`) con la struttura
+ * prodotta da `chat21client.js#sendMessage` (`text`, `type`, `recipient_fullname`, `sender_fullname`,
+ * `attributes`, `metadata`, `channel_type`), così il proxy riceve lo stesso payload di un normale
+ * messaggio Chat21.
+ */
+export interface VoiceStreamingSessionConfig {
+  /** JWT auth token — finisce in `?token=` nell'URL. */
+  token: string;
+  /** Chat21 userId — campo `sender` del config frame. */
+  sender: string;
+  /** Chat21 conversationId, es. `support-group-<projectId>-<requestId>` — campo `recipient` del config frame. */
+  recipient: string;
+  /** Codice lingua BCP-47, default `'en'` — campo `lang` del config frame. */
+  lang?: string;
+  sttProvider?: string;
+  ttsProvider?: string;
+  /**
+   * Base URL del WebSocket *senza* query, incluso path.
+   * Esempio: `wss://proxy.example.com/ws/voice` o `ws://127.0.0.1:4587/ws/voice` (mock locale)
+   * Se assente, si usa `voiceProxyWsBaseUrl` dal widget config caricato con `AppConfigService.getConfig()`.
+   */
+  wsBaseUrl?: string;
+  /** Default 1000 — intervallo `MediaRecorder.start(timeslice)` in ms */
+  timesliceMs?: number;
+  /** Se valorizzato, ha precedenza sulle euristiche (es. `audio/webm;codecs=opus`) */
+  mimeType?: string;
+
+  // ── Campi sendMessage (chat21client.js#sendMessage outgoing_message) ──────────────────────────
+  /** Testo del messaggio — default `""` per il config frame. Corrisponde a `text` in `sendMessage`. */
+  text?: string;
+  /** Tipo del messaggio — default `"text"`. Corrisponde a `type` in `sendMessage`. */
+  type?: string;
+  /** Nome completo del destinatario. Corrisponde a `recipient_fullname` in `sendMessage`. */
+  recipient_fullname?: string;
+  /** Nome completo del mittente. Corrisponde a `sender_fullname` in `sendMessage`. */
+  sender_fullname?: string;
+  /** Attributi del messaggio (es. lingua, info utente). Corrisponde a `attributes` in `sendMessage`. */
+  attributes?: Record<string, unknown>;
+  /** Metadata del messaggio — default `""`. Corrisponde a `metadata` in `sendMessage`. */
+  metadata?: unknown;
+  /** Tipo di canale (es. `"direct"`). Corrisponde a `channel_type` in `sendMessage`. */
+  channel_type?: string;
+}
+
+export type VoiceStreamingConnectionState =
+  | 'idle'
+  | 'connecting'
+  | 'open'
+  | 'streaming'
+  | 'stopping'
+  | 'closed'
+  | 'error';
+
+export interface VoiceStreamingServerMessage {
+  /** Originale: text JSON o testo, binary come ArrayBuffer */
+  data: string | ArrayBuffer;
+  isBinary: boolean;
+}
+
+export interface VoiceStreamingStopOptions {
+  discard?: boolean;
+  /** Dopo `MediaRecorder.stop`, attende un messaggio testuale dal server con l’URL (JSON `url` / `audioUrl` / …) prima di chiudere il socket */
+  awaitServerResultUrl?: boolean;
+  serverResultTimeoutMs?: number;
+}
+
+export interface VoiceStreamingStopResult {
+  blob: Blob | null;
+  mimeType: string;
+  /** Estratto dal messaggio testuale del server al termine dello stream, se `awaitServerResultUrl` e protocollo coerente */
+  resultUrl: string | null;
+}
+
+/** Messaggio di controllo JSON dal proxy voce (`msg.event`). */
+export type VoiceWsServerEventName =
+  | 'ready'
+  | 'session_started'
+  | 'listening'
+  | 'transcript'
+  | 'thinking'
+  | 'speaking'
+  | 'done'
+  | 'barge_in'
+  | 'error';
+
+/** Messaggio di controllo JSON dal proxy (`msg.event`); altri campi sono ignorati se non gestiti. */
+export type VoiceWsControlMessage = {
+  event: VoiceWsServerEventName;
+  requestId?: string;
+  text?: string;
+  isFinal?: boolean;
+  message?: string;
+} & Record<string, unknown>;
+
+/** Single word with its karaoke highlight state. */
+export interface VoiceTtsKaraokeWord {
+  text: string;
+  state: 'future' | 'active' | 'past';
+}
+
+/** Emitted on each word transition while TTS plays over WebSocket. */
+export interface VoiceTtsKaraokeFrame {
+  /** Full text of the utterance being spoken. */
+  text: string;
+  words: ReadonlyArray<VoiceTtsKaraokeWord>;
+  activeIndex: number;
+}

package/src/app/providers/voice/voice.service.spec.ts

@@ -1,11 +1,31 @@
 import { TestBed } from '@angular/core/testing';
+import { Subject } from 'rxjs';
 
 import { VoiceService } from './voice.service';
 import { VadService } from './vad.service';
+import { VoiceStreamingService } from './voice-streaming.service';
+import { TtsAudioPlaybackCoordinator } from '../tts-audio-playback-coordinator.service';
+import { VoiceWsControlMessage } from './voice-streaming.types';
+
+/** Stream con traccia audio reale (Web Audio), richiesto da `createMediaStreamSource` nei test WSS/legacy. */
+function createFakeMicStreamWithAudioTrack(): MediaStream {
+  const ctx = new AudioContext();
+  const dest = ctx.createMediaStreamDestination();
+  const osc = ctx.createOscillator();
+  const gain = ctx.createGain();
+  gain.gain.value = 0.00001;
+  osc.connect(gain);
+  gain.connect(dest);
+  osc.start(0);
+  return dest.stream;
+}
 
 describe('VoiceService', () => {
   let service: VoiceService;
   let vadService: jasmine.SpyObj<VadService>;
+  let wsControl$: Subject<VoiceWsControlMessage>;
+  let ttsBinaryChunk$: Subject<ArrayBuffer>;
+  let voiceStreamingMock: jasmine.SpyObj<VoiceStreamingService>;
 
   let mockVad: { start: jasmine.Spy; pause: jasmine.Spy; destroy: jasmine.Spy };
 
@@ -19,12 +39,36 @@ describe('VoiceService', () => {
     vadService.ensureOnnxRuntimeEnv.and.returnValue(Promise.resolve());
     vadService.createMicVad.and.returnValue(Promise.resolve(mockVad as any));
 
+    wsControl$ = new Subject<VoiceWsControlMessage>();
+    ttsBinaryChunk$ = new Subject<ArrayBuffer>();
+
+    voiceStreamingMock = jasmine.createSpyObj<VoiceStreamingService>(
+      'VoiceStreamingService',
+      ['start', 'stop', 'setAudioMuted', 'sendPlaybackComplete', 'sendBargeIn'],
+    );
+    voiceStreamingMock.start.and.returnValue(Promise.resolve());
+    voiceStreamingMock.stop.and.returnValue(
+      Promise.resolve({ blob: null, mimeType: '', resultUrl: null }),
+    );
+    // Expose the subjects as readonly observables via Object.defineProperty
+    Object.defineProperty(voiceStreamingMock, 'wsControl$', { get: () => wsControl$.asObservable() });
+    Object.defineProperty(voiceStreamingMock, 'ttsBinaryChunk$', { get: () => ttsBinaryChunk$.asObservable() });
+
+    const ttsMock = { isTTSPlaying$: { subscribe: () => ({ unsubscribe: () => undefined }) } };
+
     TestBed.configureTestingModule({
-      providers: [VoiceService, { provide: VadService, useValue: vadService }],
+      providers: [
+        VoiceService,
+        { provide: VadService, useValue: vadService },
+        { provide: VoiceStreamingService, useValue: voiceStreamingMock },
+        { provide: TtsAudioPlaybackCoordinator, useValue: ttsMock },
+      ],
     });
     service = TestBed.inject(VoiceService);
   });
 
+  // ── Existing session lifecycle tests ──────────────────────────────────────
+
   it('startSession should call ensureOnnxRuntimeEnv', async () => {
     const stream = new MediaStream();
     spyOn(navigator.mediaDevices, 'getUserMedia').and.returnValue(Promise.resolve(stream));
@@ -47,9 +91,22 @@ describe('VoiceService', () => {
     expect(mockVad.start).toHaveBeenCalled();
   });
 
+  it('startSession with voiceIngressStream should not use MicVAD', async () => {
+    const stream = createFakeMicStreamWithAudioTrack();
+    spyOn(navigator.mediaDevices, 'getUserMedia').and.returnValue(Promise.resolve(stream));
+
+    await service.startSession({
+      voiceIngressStream: { token: 'JWT x', sender: 'user1', recipient: 'support-group-p1-req1' },
+    });
+
+    expect(vadService.ensureOnnxRuntimeEnv).not.toHaveBeenCalled();
+    expect(vadService.createMicVad).not.toHaveBeenCalled();
+  });
+
   it('stopSession should destroy VAD and stop tracks', async () => {
-    const track = jasmine.createSpyObj<MediaStreamTrack>('MediaStreamTrack', ['stop']);
-    const stream = new MediaStream([track]);
+    const stream = createFakeMicStreamWithAudioTrack();
+    const track = stream.getAudioTracks()[0];
+    spyOn(track, 'stop').and.callThrough();
     spyOn(navigator.mediaDevices, 'getUserMedia').and.returnValue(Promise.resolve(stream));
 
     await service.startSession({ onRecordingComplete: () => {} });
@@ -57,4 +114,147 @@ describe('VoiceService', () => {
 
     expect(track.stop).toHaveBeenCalled();
   });
+
+  // ── Playback-gated listening re-enablement tests ──────────────────────────
+
+  /**
+   * Start a WSS session and return a helper that tracks _isAcquisitionBlocked$ emissions.
+   */
+  async function startWssSession(): Promise<boolean[]> {
+    const stream = createFakeMicStreamWithAudioTrack();
+    spyOn(navigator.mediaDevices, 'getUserMedia').and.returnValue(Promise.resolve(stream));
+    const blocked: boolean[] = [];
+    service.isAcquisitionBlocked$.subscribe((v) => blocked.push(v));
+    await service.startSession({
+      voiceIngressStream: { token: 'JWT x', sender: 'user1', recipient: 'support-group-p1-req1' },
+    });
+    return blocked;
+  }
+
+  it('acquisition stays blocked after _flushTtsUnblock; unblocks only on "listening"', async () => {
+    const blocked = await startWssSession();
+    const initialLen = blocked.length;
+
+    // Simulate proxy sequence: speaking → binary audio → done
+    wsControl$.next({ event: 'speaking', text: 'hello' } as VoiceWsControlMessage);
+    // Emit a tiny audio buffer so _activeTtsSources increments
+    ttsBinaryChunk$.next(new ArrayBuffer(4));
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage);
+
+    // sendPlaybackComplete must NOT have been called yet (audio hasn't ended)
+    expect(voiceStreamingMock.sendPlaybackComplete).not.toHaveBeenCalled();
+
+    // _isAcquisitionBlocked$ must still be true — no premature unblock
+    const afterDone = blocked.slice(initialLen);
+    expect(afterDone.every((v) => v === true)).toBeTrue();
+
+    // Now simulate "listening" arriving from proxy
+    wsControl$.next({ event: 'listening' } as VoiceWsControlMessage);
+
+    const afterListening = blocked[blocked.length - 1];
+    expect(afterListening).toBeFalse();
+    expect(voiceStreamingMock.setAudioMuted).not.toHaveBeenCalled();
+  });
+
+  it('empty-audio path: sendPlaybackComplete immediately but acquisition stays blocked until "listening"', async () => {
+    const blocked = await startWssSession();
+    const initialLen = blocked.length;
+
+    // Simulate done arriving with NO binary audio (_activeTtsSources === 0)
+    wsControl$.next({ event: 'speaking', text: 'hello' } as VoiceWsControlMessage);
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage);
+
+    // Proxy signalled immediately
+    expect(voiceStreamingMock.sendPlaybackComplete).toHaveBeenCalledTimes(1);
+
+    // Acquisition must still be blocked — proxy hasn't confirmed LISTENING yet
+    const afterDone = blocked.slice(initialLen);
+    expect(afterDone.every((v) => v === true)).toBeTrue();
+
+    // Unblock only after proxy confirms
+    wsControl$.next({ event: 'listening' } as VoiceWsControlMessage);
+    expect(blocked[blocked.length - 1]).toBeFalse();
+  });
+
+  it('"listening" event unblocks acquisition without mic mute toggles (AEC keeps capture open)', async () => {
+    await startWssSession();
+
+    wsControl$.next({ event: 'speaking', text: 'hi' } as VoiceWsControlMessage);
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage);
+    wsControl$.next({ event: 'listening' } as VoiceWsControlMessage);
+
+    expect(voiceStreamingMock.setAudioMuted).not.toHaveBeenCalled();
+    expect((service as any)._isAcquisitionBlocked$.getValue()).toBe(false);
+  });
+
+  // ── Audio preemption tests (SPEC-002) ────────────────────────────────────
+
+  it('second "speaking" cancels first audio: sendPlaybackComplete called exactly once for the new turn', async () => {
+    await startWssSession();
+    voiceStreamingMock.sendPlaybackComplete.calls.reset();
+
+    // First turn: audio chunk arrives → _activeTtsSources = 1 (sync) → done sets _unblockAfterTts
+    wsControl$.next({ event: 'speaking', text: 'first' } as VoiceWsControlMessage);
+    ttsBinaryChunk$.next(new ArrayBuffer(4));           // _activeTtsSources++ synchronously
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage); // _unblockAfterTts = true
+
+    // Second turn preempts while first audio is still "playing"
+    wsControl$.next({ event: 'speaking', text: 'second' } as VoiceWsControlMessage);
+    // _cancelAllTtsAudio() resets _activeTtsSources=0, _unblockAfterTts=false
+
+    // done with no audio → sendPlaybackComplete immediately (new turn, _activeTtsSources = 0)
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage);
+
+    expect(voiceStreamingMock.sendPlaybackComplete).toHaveBeenCalledTimes(1);
+  });
+
+  it('second "speaking" resets counters so first audio ending does not trigger spurious sendPlaybackComplete', async () => {
+    await startWssSession();
+    voiceStreamingMock.sendPlaybackComplete.calls.reset();
+
+    wsControl$.next({ event: 'speaking', text: 'first' } as VoiceWsControlMessage);
+    ttsBinaryChunk$.next(new ArrayBuffer(4));
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage);
+
+    // Preempt
+    wsControl$.next({ event: 'speaking', text: 'second' } as VoiceWsControlMessage);
+
+    // Simulate first audio's onended firing AFTER the cancel (delayed Web Audio callback).
+    (service as any)._onTtsSourceEnded();
+
+    // _unblockAfterTts was cleared by cancel; no sendPlaybackComplete should fire
+    expect(voiceStreamingMock.sendPlaybackComplete).not.toHaveBeenCalled();
+  });
+
+  // ── Barge-in ──────────────────────────────────────────────────────────────
+
+  it('barge_in event cancels TTS audio and unblocks acquisition without sending tts_playback_complete', async () => {
+    await startWssSession();
+    voiceStreamingMock.sendPlaybackComplete.calls.reset();
+
+    // Simulate bot speaking with audio in flight
+    wsControl$.next({ event: 'speaking', text: 'hello' } as VoiceWsControlMessage);
+    ttsBinaryChunk$.next(new ArrayBuffer(4));   // _activeTtsSources++ synchronously
+    wsControl$.next({ event: 'done' } as VoiceWsControlMessage); // _unblockAfterTts = true
+
+    // Proxy detects user speech and sends barge_in
+    wsControl$.next({ event: 'barge_in' } as VoiceWsControlMessage);
+
+    // tts_playback_complete must NOT be sent — it was an interruption, not a completion
+    expect(voiceStreamingMock.sendPlaybackComplete).not.toHaveBeenCalled();
+    expect(voiceStreamingMock.setAudioMuted).not.toHaveBeenCalled();
+    expect((service as any)._isAcquisitionBlocked$.getValue()).toBe(false);
+  });
+
+  it('barge_in while no TTS is active does not throw and still unblocks acquisition', async () => {
+    await startWssSession();
+    voiceStreamingMock.sendPlaybackComplete.calls.reset();
+
+    // No speaking event — mic was never muted
+    expect(() => {
+      wsControl$.next({ event: 'barge_in' } as VoiceWsControlMessage);
+    }).not.toThrow();
+
+    expect(voiceStreamingMock.sendPlaybackComplete).not.toHaveBeenCalled();
+  });
 });