@bytexbyte/nxtlinq-ai-agent-web-development 0.1.1

package/src/voice/useVoiceTranscriptMessages.ts

@@ -0,0 +1,184 @@
+import type {
+  Message,
+  VoiceDoneEvent,
+  VoiceTranscriptEvent,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { mergeStreamingTranscript } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useCallback, useRef } from 'react';
+import { flushSync } from 'react-dom';
+
+type InteractionMode = 'text' | 'voice';
+
+type VoiceTranscriptAgentApi = {
+  getMessages: () => Message[];
+  setMessages: (messages: Message[]) => void;
+  syncVoiceTurnHistory: (options?: { last?: number }) => Promise<void>;
+};
+
+const STREAM_PREFIX = 'voice-stream-';
+
+function voiceMeta(sessionId: string | null) {
+  return {
+    voiceRealtime: true as const,
+    voiceSessionId: sessionId ?? undefined,
+  };
+}
+
+export function useVoiceTranscriptMessages(
+  api: VoiceTranscriptAgentApi,
+  interactionMode: InteractionMode,
+  voiceSessionId: string | null,
+  getVoiceSessionId?: () => string | null,
+) {
+  const streamIdRef = useRef<string | null>(null);
+  const sessionIdRef = useRef(voiceSessionId);
+  sessionIdRef.current = voiceSessionId;
+
+  const resolveSessionId = useCallback(
+    () => getVoiceSessionId?.() ?? sessionIdRef.current,
+    [getVoiceSessionId],
+  );
+
+  const isVoiceUiActive = useCallback(
+    () => interactionMode === 'voice',
+    [interactionMode],
+  );
+
+  const upsertStreaming = useCallback(
+    (text: string) => {
+      const messages = api.getMessages();
+      let streamId = streamIdRef.current;
+      if (!streamId) {
+        streamId = `${STREAM_PREFIX}${Date.now()}`;
+        streamIdRef.current = streamId;
+      }
+      const idx = messages.findIndex((m) => m.id === streamId);
+      const partialContent =
+        idx >= 0
+          ? mergeStreamingTranscript(messages[idx]?.partialContent ?? '', text)
+          : text;
+      const meta = voiceMeta(resolveSessionId());
+      const apply = () => {
+        if (idx >= 0) {
+          api.setMessages(
+            messages.map((m, i) =>
+              i === idx
+                ? { ...m, partialContent, isStreaming: true, metadata: { ...m.metadata, ...meta } }
+                : m,
+            ),
+          );
+          return;
+        }
+        api.setMessages([
+          ...messages,
+          {
+            id: streamId,
+            role: 'assistant',
+            content: '',
+            partialContent,
+            isStreaming: true,
+            timestamp: new Date().toISOString(),
+            metadata: meta,
+          },
+        ]);
+      };
+      flushSync(apply);
+    },
+    [api, resolveSessionId],
+  );
+
+  const finalizeAssistant = useCallback(
+    (text: string, messageId?: string | null) => {
+      const trimmed = text.trim();
+      streamIdRef.current = null;
+      if (!trimmed) return;
+
+      const messages = api.getMessages();
+      const streamIdx = messages.findIndex((m) => m.isStreaming && m.role === 'assistant');
+      if (streamIdx >= 0) {
+        api.setMessages(
+          messages.map((m, i) =>
+            i === streamIdx
+              ? {
+                  ...m,
+                  id: messageId ?? m.id,
+                  content: trimmed,
+                  partialContent: undefined,
+                  isStreaming: false,
+                  metadata: { ...m.metadata, ...voiceMeta(resolveSessionId()) },
+                }
+              : m,
+          ),
+        );
+        return;
+      }
+      const last = messages[messages.length - 1];
+      if (last?.role === 'assistant' && last.content === trimmed) return;
+      api.setMessages([
+        ...messages,
+        {
+          id: messageId ?? `voice-asst-${Date.now()}`,
+          role: 'assistant',
+          content: trimmed,
+          timestamp: new Date().toISOString(),
+          metadata: voiceMeta(resolveSessionId()),
+        },
+      ]);
+    },
+    [api, resolveSessionId],
+  );
+
+  const handleTranscript = useCallback(
+    (event: VoiceTranscriptEvent) => {
+      if (!isVoiceUiActive()) return;
+      const text = event.text?.trim() ?? '';
+      if (event.role === 'assistant') {
+        // Keep one streaming bubble for the whole turn; finalize only in handleDone.
+        if (text) upsertStreaming(text);
+        return;
+      }
+      if (event.role === 'user' && !event.interim && text) {
+        const messages = api.getMessages();
+        const last = messages[messages.length - 1];
+        if (last?.role === 'user' && last.content === text) return;
+        api.setMessages([
+          ...messages,
+          {
+            id: `voice-user-${Date.now()}`,
+            role: 'user',
+            content: text,
+            timestamp: new Date().toISOString(),
+            metadata: voiceMeta(resolveSessionId()),
+          },
+        ]);
+      }
+    },
+    [api, finalizeAssistant, isVoiceUiActive, resolveSessionId, upsertStreaming],
+  );
+
+  const handleDone = useCallback(
+    (event: VoiceDoneEvent) => {
+      if (!isVoiceUiActive()) return;
+      if (event.guardrailsBlocked || event.billingBlocked || event.error) {
+        streamIdRef.current = null;
+        return;
+      }
+      const reply = event.replyText?.trim() ?? '';
+      if (reply) {
+        finalizeAssistant(reply, event.assistantMessageId ?? undefined);
+      } else {
+        streamIdRef.current = null;
+      }
+      void api.syncVoiceTurnHistory({ last: 20 }).catch((err) => {
+        console.warn('[nxtlinq] syncVoiceTurnHistory after voice turn failed', err);
+      });
+    },
+    [api, finalizeAssistant, isVoiceUiActive],
+  );
+
+  const clearVoiceStream = useCallback(() => {
+    streamIdRef.current = null;
+  }, []);
+
+  return { handleTranscript, handleDone, clearVoiceStream };
+}

package/src/voice/useWsRealtimeAudio.ts

@@ -0,0 +1,141 @@
+import type { VoiceSession, VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useCallback, useEffect, useRef } from 'react';
+import { BrowserWsPcmPlayer } from './ws/BrowserWsPcmPlayer';
+import { BrowserWsPcmRecorder } from './ws/BrowserWsPcmRecorder';
+import { useVoiceSilenceCommit } from './useVoiceSilenceCommit';
+
+type WsVoiceCallbacks = {
+  onOpen?: () => void;
+  onAudioDelta?: (pcm16: Int16Array | ArrayBuffer) => void;
+  onClose?: (reason: string) => void;
+  onError?: (err: Error) => void;
+};
+
+export type UseWsRealtimeAudioOptions = {
+  voiceStatus: VoiceStatus;
+  muteAfterSilenceCommit: () => void;
+};
+
+export function useWsRealtimeAudio(
+  isCaptureActive: boolean,
+  isVoiceActive: boolean,
+  options: UseWsRealtimeAudioOptions,
+) {
+  const playerRef = useRef<BrowserWsPcmPlayer | null>(null);
+  const recorderRef = useRef<BrowserWsPcmRecorder | null>(null);
+  const sessionRef = useRef<VoiceSession | null>(null);
+  const captureGenRef = useRef(0);
+  const isCaptureActiveRef = useRef(isCaptureActive);
+  isCaptureActiveRef.current = isCaptureActive;
+  const prevCaptureActiveRef = useRef(isCaptureActive);
+
+  const getSession = useCallback(() => sessionRef.current, []);
+  const muteAfterSilenceCommitRef = useRef(options.muteAfterSilenceCommit);
+  muteAfterSilenceCommitRef.current = options.muteAfterSilenceCommit;
+
+  const silence = useVoiceSilenceCommit(
+    getSession,
+    () => muteAfterSilenceCommitRef.current(),
+    options.voiceStatus,
+  );
+  const silenceRef = useRef(silence);
+  silenceRef.current = silence;
+
+  const ensurePlayer = useCallback(async () => {
+    if (!playerRef.current) {
+      playerRef.current = new BrowserWsPcmPlayer();
+      playerRef.current.prewarm();
+    }
+    await playerRef.current.ensureRunning();
+  }, []);
+
+  const stopCapture = useCallback((commit: boolean) => {
+    captureGenRef.current += 1;
+    const s = silenceRef.current;
+    s.clearPoll();
+    // Commit before recorder.stop() — stop() must not clear the OpenAI input buffer first.
+    if (commit && !s.consumeSkipCommitOnMute()) {
+      s.tryCommit('manual');
+    }
+    recorderRef.current?.stop();
+  }, []);
+
+  const startCapture = useCallback(async () => {
+    const session = sessionRef.current;
+    if (!session) return;
+    const gen = ++captureGenRef.current;
+    const s = silenceRef.current;
+    if (!recorderRef.current) recorderRef.current = new BrowserWsPcmRecorder();
+    playerRef.current?.clearQueue();
+    recorderRef.current.bindSession(session);
+    recorderRef.current.setOnRms(s.onSpeechRms);
+    recorderRef.current.setShouldAppend(() => s.hasHadSpeech());
+    // Let mic-unmute state settle before getUserMedia (Berify beginMicCapture).
+    await new Promise<void>((resolve) => {
+      requestAnimationFrame(() => requestAnimationFrame(() => resolve()));
+    });
+    if (gen !== captureGenRef.current || !isCaptureActiveRef.current) return;
+    s.resetTurn();
+    try {
+      await recorderRef.current.start();
+      if (gen !== captureGenRef.current || !isCaptureActiveRef.current) {
+        recorderRef.current.stop();
+        return;
+      }
+      s.startPoll();
+    } catch (err) {
+      s.clearPoll();
+      console.error('[nxtlinq] mic capture start failed', err);
+    }
+  }, []);
+
+  const cleanup = useCallback(() => {
+    silenceRef.current.clearPoll();
+    recorderRef.current?.cleanup();
+    recorderRef.current = null;
+    playerRef.current?.cleanup();
+    playerRef.current = null;
+    sessionRef.current = null;
+  }, []);
+
+  const bindSession = useCallback((session: VoiceSession | null, captureWhenUnmuted = false) => {
+    sessionRef.current = session;
+    recorderRef.current?.bindSession(session);
+    if (session && captureWhenUnmuted && isCaptureActiveRef.current) {
+      void startCapture();
+    }
+  }, [startCapture]);
+
+  const buildCallbacks = useCallback((overrides?: Partial<WsVoiceCallbacks>): WsVoiceCallbacks => ({
+    onOpen: () => {
+      void ensurePlayer();
+      overrides?.onOpen?.();
+    },
+    onAudioDelta: (pcm16) => {
+      void ensurePlayer().then(() => playerRef.current?.addAudio(pcm16));
+      overrides?.onAudioDelta?.(pcm16);
+    },
+    onClose: (reason) => {
+      cleanup();
+      overrides?.onClose?.(reason);
+    },
+    onError: (err) => {
+      cleanup();
+      overrides?.onError?.(err);
+    },
+  }), [cleanup, ensurePlayer]);
+
+  useEffect(() => {
+    if (!isVoiceActive) {
+      prevCaptureActiveRef.current = false;
+      cleanup();
+      return;
+    }
+    const prev = prevCaptureActiveRef.current;
+    prevCaptureActiveRef.current = isCaptureActive;
+    if (isCaptureActive && !prev) void startCapture();
+    else if (!isCaptureActive && prev) stopCapture(true);
+  }, [isCaptureActive, isVoiceActive, startCapture, stopCapture, cleanup]);
+
+  return { bindSession, buildCallbacks, getOutputAudioLevel: () => playerRef.current?.getAudioLevel() ?? 0 };
+}

package/src/voice/voiceMicConstants.ts

@@ -0,0 +1,13 @@
+import type { VoiceStatus } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export const ASSISTANT_MIC_HOLD_STATUSES: ReadonlySet<VoiceStatus> = new Set([
+  'transcribing',
+  'thinking',
+  'generating',
+  'speaking',
+]);
+
+export const SPEAKER_ACTIVE_STATUSES: ReadonlySet<VoiceStatus> = new Set([
+  'generating',
+  'speaking',
+]);

package/src/voice/ws/BrowserWsPcmPlayer.ts

@@ -0,0 +1,139 @@
+import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
+
+const LEVEL_SCALE = 3.2;
+const LOOKAHEAD_SECONDS = 0.02;
+const CROSSFADE_SECONDS = 0.04;
+
+function applyFade(buffer: Float32Array, fadeSamples: number): void {
+  const fade = Math.min(fadeSamples, Math.floor(buffer.length / 2));
+  if (fade <= 0) return;
+  for (let i = 0; i < fade; i += 1) {
+    buffer[i]! *= i / fade;
+    buffer[buffer.length - 1 - i]! *= (fade - i) / fade;
+  }
+}
+
+export class BrowserWsPcmPlayer {
+  private readonly sampleRate = WS_REALTIME_SAMPLE_RATE;
+  private readonly fadeSamples = Math.max(1, Math.floor(this.sampleRate * 0.01));
+  private audioContext: AudioContext | null = null;
+  private gainNode: GainNode | null = null;
+  private analyserNode: AnalyserNode | null = null;
+  private analyserBuffer: Float32Array | null = null;
+  private readonly activeSources = new Set<AudioBufferSourceNode>();
+  private queue: AudioBuffer[] = [];
+  private isPlaying = true;
+  private playHead = 0;
+  private lastChunkRms = 0;
+  private lastChunkAt = 0;
+
+  async ensureRunning(): Promise<void> {
+    const ctx = this.ensureContext();
+    if (ctx.state === 'suspended') await ctx.resume();
+    this.isPlaying = true;
+  }
+
+  prewarm(): void {
+    this.ensureContext();
+  }
+
+  addAudio(pcm16: Int16Array | ArrayBuffer): void {
+    const ctx = this.ensureContext();
+    const int16 = pcm16 instanceof ArrayBuffer ? new Int16Array(pcm16) : pcm16;
+    if (int16.length === 0) return;
+    const floats = new Float32Array(int16.length);
+    let sumSq = 0;
+    for (let i = 0; i < int16.length; i += 1) {
+      const sample = int16[i]! / 32768;
+      floats[i] = Math.max(-1, Math.min(1, sample));
+      sumSq += sample * sample;
+    }
+    applyFade(floats, this.fadeSamples);
+    this.lastChunkRms = Math.min(1, Math.sqrt(sumSq / int16.length) * LEVEL_SCALE);
+    this.lastChunkAt = Date.now();
+    const buffer = ctx.createBuffer(1, floats.length, this.sampleRate);
+    buffer.getChannelData(0).set(floats);
+    this.queue.push(buffer);
+    this.isPlaying = true;
+    this.pipelineQueue();
+  }
+
+  getAudioLevel(): number {
+    const analyser = this.analyserNode;
+    const buf = this.analyserBuffer;
+    if (analyser && buf && this.activeSources.size > 0) {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      analyser.getFloatTimeDomainData(buf as any);
+      let sumSq = 0;
+      for (let i = 0; i < buf.length; i += 1) {
+        const v = buf[i]!;
+        sumSq += v * v;
+      }
+      return Math.min(1, Math.sqrt(sumSq / buf.length) * LEVEL_SCALE);
+    }
+    return Date.now() - this.lastChunkAt < 500 ? this.lastChunkRms : 0;
+  }
+
+  clearQueue(): void {
+    for (const source of this.activeSources) {
+      try { source.stop(); } catch { /* noop */ }
+    }
+    this.activeSources.clear();
+    this.queue = [];
+    this.playHead = this.audioContext?.currentTime ?? 0;
+    this.lastChunkRms = 0;
+    this.lastChunkAt = 0;
+  }
+
+  cleanup(): void {
+    this.clearQueue();
+    this.isPlaying = false;
+    void this.audioContext?.close();
+    this.analyserNode = null;
+    this.analyserBuffer = null;
+    this.gainNode = null;
+    this.audioContext = null;
+  }
+
+  private ensureContext(): AudioContext {
+    if (!this.audioContext) {
+      this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
+      this.gainNode = this.audioContext.createGain();
+      this.analyserNode = this.audioContext.createAnalyser();
+      this.analyserNode.fftSize = 512;
+      this.analyserBuffer = new Float32Array(this.analyserNode.fftSize);
+      this.gainNode.connect(this.analyserNode);
+      this.analyserNode.connect(this.audioContext.destination);
+      this.playHead = this.audioContext.currentTime;
+    }
+    return this.audioContext;
+  }
+
+  private pipelineQueue(): void {
+    const ctx = this.audioContext;
+    const gain = this.gainNode;
+    if (!ctx || !gain || !this.isPlaying) return;
+    while (this.queue.length > 0) {
+      const audioBuffer = this.queue.shift();
+      if (!audioBuffer) return;
+      const source = ctx.createBufferSource();
+      source.buffer = audioBuffer;
+      const perSourceGain = ctx.createGain();
+      source.connect(perSourceGain);
+      perSourceGain.connect(gain);
+      const now = ctx.currentTime;
+      const startAt = Math.max((this.playHead || now) - CROSSFADE_SECONDS * 0.75, now + LOOKAHEAD_SECONDS);
+      const duration = audioBuffer.duration;
+      const endAt = startAt + duration;
+      const fade = Math.min(CROSSFADE_SECONDS, duration / 2);
+      perSourceGain.gain.setValueAtTime(0, startAt);
+      perSourceGain.gain.linearRampToValueAtTime(1, startAt + fade);
+      perSourceGain.gain.setValueAtTime(1, endAt - fade);
+      perSourceGain.gain.linearRampToValueAtTime(0, endAt);
+      this.playHead = endAt;
+      this.activeSources.add(source);
+      source.start(startAt);
+      source.onended = () => this.activeSources.delete(source);
+    }
+  }
+}

package/src/voice/ws/BrowserWsPcmRecorder.ts

@@ -0,0 +1,83 @@
+import type { VoiceSession } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { float32ToPcm16 } from './float32ToPcm16';
+import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
+
+function computeRms(channel: Float32Array): number {
+  let sum = 0;
+  for (let i = 0; i < channel.length; i += 1) {
+    const v = channel[i]!;
+    sum += v * v;
+  }
+  return channel.length > 0 ? Math.sqrt(sum / channel.length) : 0;
+}
+
+/** getUserMedia only on start() — Berify hold-to-talk aligned. */
+export class BrowserWsPcmRecorder {
+  private session: VoiceSession | null = null;
+  private onRms?: (rms: number) => void;
+  private stream: MediaStream | null = null;
+  private audioContext: AudioContext | null = null;
+  private processor: ScriptProcessorNode | null = null;
+  private source: MediaStreamAudioSourceNode | null = null;
+  private silentGain: GainNode | null = null;
+
+  private shouldAppend?: () => boolean;
+
+  bindSession(session: VoiceSession | null): void {
+    this.session = session;
+  }
+
+  setShouldAppend(handler: () => boolean): void {
+    this.shouldAppend = handler;
+  }
+
+  setOnRms(handler: (rms: number) => void): void {
+    this.onRms = handler;
+  }
+
+  async start(): Promise<void> {
+    if (this.processor) return;
+    this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const ctx = new AudioContext({ sampleRate: WS_REALTIME_SAMPLE_RATE });
+    this.audioContext = ctx;
+    this.source = ctx.createMediaStreamSource(this.stream);
+    this.processor = ctx.createScriptProcessor(4096, 1, 1);
+    this.silentGain = ctx.createGain();
+    this.silentGain.gain.value = 0;
+    this.source.connect(this.processor);
+    this.processor.connect(this.silentGain);
+    this.silentGain.connect(ctx.destination);
+    this.processor.onaudioprocess = (event) => {
+      if (!this.session?.appendInputAudio) return;
+      const channel = event.inputBuffer.getChannelData(0);
+      const rms = computeRms(channel);
+      this.onRms?.(rms);
+      // Berify: only stream PCM after RMS crosses speech threshold.
+      if (this.shouldAppend && !this.shouldAppend()) return;
+      this.session.appendInputAudio(float32ToPcm16(channel));
+    };
+    if (ctx.state === 'suspended') {
+      await ctx.resume();
+    }
+  }
+
+  stop(): void {
+    this.processor?.disconnect();
+    this.source?.disconnect();
+    this.silentGain?.disconnect();
+    this.processor = null;
+    this.source = null;
+    this.silentGain = null;
+    for (const track of this.stream?.getTracks() ?? []) {
+      track.stop();
+    }
+    this.stream = null;
+    void this.audioContext?.close();
+    this.audioContext = null;
+  }
+
+  cleanup(): void {
+    this.stop();
+    this.session = null;
+  }
+}

package/src/voice/ws/float32ToPcm16.ts

@@ -0,0 +1,8 @@
+export function float32ToPcm16(input: Float32Array): Int16Array {
+  const out = new Int16Array(input.length);
+  for (let i = 0; i < input.length; i += 1) {
+    const sample = Math.max(-1, Math.min(1, input[i]!));
+    out[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+  }
+  return out;
+}

package/src/voice/ws/voiceSilenceConstants.ts

@@ -0,0 +1,4 @@
+export const MIC_SPEECH_RMS_THRESHOLD = 0.008;
+export const MIC_BARGE_IN_RMS_THRESHOLD = 0.045;
+export const MIC_SILENCE_COMMIT_MS = 400;
+export const MIC_SILENCE_POLL_MS = 100;

package/src/voice/ws/wsRealtimeConstants.ts

@@ -0,0 +1 @@
+export const WS_REALTIME_SAMPLE_RATE = 24000;

package/src/webAgentDefaults.ts

@@ -0,0 +1,12 @@
+import type { AgentConfig } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+/** Browser SDK defaults — integrators do not pass these props. */
+export const WEB_AGENT_DEFAULTS = {
+  voiceTransport: 'webrtc',
+  voiceMode: 'realtime',
+  defaultModel: 'open-ai-stream',
+} as const satisfies Partial<AgentConfig>;
+
+export function applyWebAgentDefaults<C extends AgentConfig>(config: C): C {
+  return { ...WEB_AGENT_DEFAULTS, ...config };
+}