@bytexbyte/nxtlinq-ai-agent-ui-react-native-development 0.2.0 → 0.3.0

package/src/voice/wsPcmPlayer.ts

@@ -0,0 +1,166 @@
+import {
+  AnalyserNode,
+  AudioBuffer,
+  AudioBufferSourceNode,
+  AudioContext,
+  GainNode,
+} from 'react-native-audio-api';
+import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
+
+const LEVEL_SCALE = 3.2;
+const LOOKAHEAD_SECONDS = 0.02;
+const CROSSFADE_SECONDS = 0.04;
+
+function applyFade(buffer: Float32Array, fadeSamples: number): void {
+  const fade = Math.min(fadeSamples, Math.floor(buffer.length / 2));
+  if (fade <= 0) return;
+  for (let i = 0; i < fade; i += 1) {
+    buffer[i]! *= i / fade;
+    buffer[buffer.length - 1 - i]! *= (fade - i) / fade;
+  }
+}
+
+export class WsPcmPlayer {
+  private readonly sampleRate = WS_REALTIME_SAMPLE_RATE;
+  private readonly fadeSamples = Math.max(1, Math.floor(this.sampleRate * 0.01));
+  private audioContext: AudioContext | null = null;
+  private gainNode: GainNode | null = null;
+  private analyserNode: AnalyserNode | null = null;
+  private analyserBuffer: Float32Array | null = null;
+  private readonly activeSources = new Set<AudioBufferSourceNode>();
+  private queue: AudioBuffer[] = [];
+  private isPlaying = true;
+  private playHead = 0;
+  private lastChunkRms = 0;
+  private lastChunkAt = 0;
+
+  async ensureRunning(): Promise<void> {
+    const ctx = this.ensureContext();
+    if (ctx.state === 'suspended') {
+      await ctx.resume();
+    }
+    this.isPlaying = true;
+  }
+
+  prewarm(): void {
+    this.ensureContext();
+  }
+
+  addAudio(pcm16: Int16Array | ArrayBuffer): void {
+    const ctx = this.ensureContext();
+    const int16 = pcm16 instanceof ArrayBuffer ? new Int16Array(pcm16) : pcm16;
+    if (int16.length === 0) return;
+
+    const floats = new Float32Array(int16.length);
+    let sumSq = 0;
+    for (let i = 0; i < int16.length; i += 1) {
+      const sample = int16[i]! / 32768;
+      floats[i] = Math.max(-1, Math.min(1, sample));
+      sumSq += sample * sample;
+    }
+    applyFade(floats, this.fadeSamples);
+    this.lastChunkRms = Math.min(1, Math.sqrt(sumSq / int16.length) * LEVEL_SCALE);
+    this.lastChunkAt = Date.now();
+
+    const buffer = ctx.createBuffer(1, floats.length, this.sampleRate);
+    buffer.getChannelData(0).set(floats);
+    this.queue.push(buffer);
+    this.isPlaying = true;
+    this.pipelineQueue();
+  }
+
+  getAudioLevel(): number {
+    const analyser = this.analyserNode;
+    const buf = this.analyserBuffer;
+    if (analyser && buf && this.activeSources.size > 0) {
+      analyser.getFloatTimeDomainData(buf);
+      let sumSq = 0;
+      for (let i = 0; i < buf.length; i += 1) {
+        const v = buf[i]!;
+        sumSq += v * v;
+      }
+      return Math.min(1, Math.sqrt(sumSq / buf.length) * LEVEL_SCALE);
+    }
+    if (Date.now() - this.lastChunkAt < 500) {
+      return this.lastChunkRms;
+    }
+    return 0;
+  }
+
+  clearQueue(): void {
+    for (const source of this.activeSources) {
+      try {
+        source.stop();
+      } catch {
+        /* already stopped */
+      }
+    }
+    this.activeSources.clear();
+    this.queue = [];
+    this.playHead = this.audioContext?.currentTime ?? 0;
+    this.lastChunkRms = 0;
+    this.lastChunkAt = 0;
+  }
+
+  cleanup(): void {
+    this.clearQueue();
+    this.isPlaying = false;
+    this.analyserNode = null;
+    this.analyserBuffer = null;
+    this.gainNode = null;
+    this.audioContext = null;
+  }
+
+  private ensureContext(): AudioContext {
+    if (!this.audioContext) {
+      this.audioContext = new AudioContext({ sampleRate: this.sampleRate });
+      this.gainNode = this.audioContext.createGain();
+      this.analyserNode = this.audioContext.createAnalyser();
+      this.analyserNode.fftSize = 512;
+      this.analyserBuffer = new Float32Array(this.analyserNode.fftSize);
+      this.gainNode.connect(this.analyserNode);
+      this.analyserNode.connect(this.audioContext.destination);
+      this.playHead = this.audioContext.currentTime;
+    }
+    return this.audioContext;
+  }
+
+  /** Berify WavStreamPlayer: schedule chunks with lookahead + crossfade, no gaps. */
+  private pipelineQueue(): void {
+    const ctx = this.audioContext;
+    const gain = this.gainNode;
+    if (!ctx || !gain || !this.isPlaying) return;
+
+    while (this.queue.length > 0) {
+      const audioBuffer = this.queue.shift();
+      if (!audioBuffer) return;
+
+      const source = ctx.createBufferSource();
+      source.buffer = audioBuffer;
+      const perSourceGain = ctx.createGain();
+      source.connect(perSourceGain);
+      perSourceGain.connect(gain);
+
+      const now = ctx.currentTime;
+      const startAt = Math.max(
+        (this.playHead || now) - CROSSFADE_SECONDS * 0.75,
+        now + LOOKAHEAD_SECONDS,
+      );
+      const duration = audioBuffer.duration;
+      const endAt = startAt + duration;
+      const fade = Math.min(CROSSFADE_SECONDS, duration / 2);
+
+      perSourceGain.gain.setValueAtTime(0, startAt);
+      perSourceGain.gain.linearRampToValueAtTime(1, startAt + fade);
+      perSourceGain.gain.setValueAtTime(1, endAt - fade);
+      perSourceGain.gain.linearRampToValueAtTime(0, endAt);
+
+      this.playHead = endAt;
+      this.activeSources.add(source);
+      source.start(startAt);
+      source.onEnded = () => {
+        this.activeSources.delete(source);
+      };
+    }
+  }
+}

package/src/voice/wsPcmRecorder.ts

@@ -0,0 +1,152 @@
+import type { VoiceSession } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import {
+  AudioManager,
+  AudioRecorder,
+  type AudioBuffer,
+} from 'react-native-audio-api';
+import { float32ToPcm16 } from './float32ToPcm16';
+import { WS_REALTIME_SAMPLE_RATE } from './wsRealtimeConstants';
+
+/**
+ * Berify {@link WavRecorder}-aligned lifecycle: reuse one native AudioRecorder,
+ * configure playAndRecord on start, reset to playback on stop.
+ */
+export class WsPcmRecorder {
+  private recorder: AudioRecorder | null = null;
+  private recording = false;
+  private audioReadyAttached = false;
+  private session: VoiceSession | null = null;
+  private onRms?: (rms: number) => void;
+
+  async initialize(options?: { force?: boolean }): Promise<void> {
+    if (options?.force && this.recorder) {
+      this.disposeRecorderInternal();
+    }
+    if (this.recorder) return;
+    await AudioManager.requestRecordingPermissions();
+    this.recorder = new AudioRecorder({
+      sampleRate: WS_REALTIME_SAMPLE_RATE,
+      bufferLengthInSamples: WS_REALTIME_SAMPLE_RATE,
+    });
+  }
+
+  bindSession(session: VoiceSession | null): void {
+    this.session = session;
+  }
+
+  setOnRms(handler: (rms: number) => void): void {
+    this.onRms = handler;
+  }
+
+  private disposeRecorderInternal(): void {
+    if (!this.recorder) return;
+    try {
+      this.recorder.stop();
+    } catch {
+      /* noop */
+    }
+    this.recorder = null;
+    this.audioReadyAttached = false;
+    this.recording = false;
+    this.resetSession();
+  }
+
+  async start(): Promise<void> {
+    await this.initialize();
+    const recorder = this.recorder;
+    if (!recorder) return;
+
+    if (this.recording) {
+      this.recording = false;
+      try {
+        recorder.stop();
+      } catch {
+        /* noop */
+      }
+    }
+
+    this.configureSession();
+    if (!this.audioReadyAttached) {
+      recorder.onAudioReady(({ buffer }) => {
+        this.handleAudioReady(buffer);
+      });
+      this.audioReadyAttached = true;
+    }
+
+    try {
+      recorder.start();
+      this.recording = true;
+    } catch (err) {
+      this.recording = false;
+      console.error('[WsPcmRecorder] start failed', err);
+      throw err;
+    }
+  }
+
+  stop(): void {
+    if (!this.recorder || !this.recording) return;
+    try {
+      this.recorder.stop();
+    } catch (err) {
+      console.warn('[WsPcmRecorder] stop', err);
+    }
+    this.recording = false;
+    this.resetSession();
+  }
+
+  cleanup(): void {
+    if (this.recorder) {
+      try {
+        if (this.recording) {
+          this.stop();
+        } else {
+          this.recorder.stop();
+        }
+      } catch {
+        /* noop */
+      }
+    }
+    this.recorder = null;
+    this.audioReadyAttached = false;
+    this.recording = false;
+    this.session = null;
+    this.onRms = undefined;
+  }
+
+  get isRecording(): boolean {
+    return this.recording;
+  }
+
+  private configureSession(): void {
+    AudioManager.setAudioSessionOptions({
+      iosCategory: 'playAndRecord',
+      iosMode: 'voiceChat',
+      iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'],
+      iosAllowHaptics: true,
+    });
+  }
+
+  private resetSession(): void {
+    try {
+      AudioManager.setAudioSessionOptions({
+        iosCategory: 'playback',
+        iosMode: 'default',
+      });
+    } catch {
+      /* noop */
+    }
+  }
+
+  private handleAudioReady(buffer: AudioBuffer): void {
+    const channel = buffer.getChannelData(0);
+    let sum = 0;
+    for (let i = 0; i < channel.length; i += 1) {
+      const sample = channel[i]!;
+      sum += sample * sample;
+    }
+    const rms = channel.length > 0 ? Math.sqrt(sum / channel.length) : 0;
+    this.onRms?.(rms);
+    if (!this.recording || !this.session?.appendInputAudio) return;
+    this.session.appendInputAudio(float32ToPcm16(channel));
+  }
+}

package/src/voice/wsRealtimeConstants.ts

@@ -0,0 +1 @@
+export const WS_REALTIME_SAMPLE_RATE = 24000;

package/src/components/AgentRemoteAudio.tsx

@@ -1,105 +0,0 @@
-import React, { useEffect, useState } from 'react';
-import { StyleSheet } from 'react-native';
-import { useAgentAssistant } from '../context/AgentAssistantContext';
-import {
-  applyRemoteAudioPlaybackGain,
-  DEFAULT_REMOTE_AUDIO_GAIN,
-} from '@bytexbyte/nxtlinq-ai-agent-core-development';
-
-type RTCViewComponent = React.ComponentType<{
-  streamURL: string;
-  objectFit?: string;
-  style?: object;
-  zOrder?: number;
-}>;
-
-let RTCView: RTCViewComponent | null = null;
-try {
-  // eslint-disable-next-line @typescript-eslint/no-require-imports
-  RTCView = require('react-native-webrtc').RTCView as RTCViewComponent;
-} catch {
-  RTCView = null;
-}
-
-type StreamWithUrl = {
-  toURL?: () => string;
-  getAudioTracks?: () => Array<{ enabled: boolean; kind?: string }>;
-};
-
-export type AgentRemoteAudioProps = {
-  /**
-   * Playback gain for assistant WebRTC audio (`MediaStreamTrack._setVolume`, 0–10).
-   * Text TTS uses react-native-video at volume 1 and sounds louder by default.
-   */
-  remoteAudioGain?: number;
-};
-
-/**
- * Plays remote WebRTC assistant audio. On iOS/Android, {@link RTCView} is required
- * (enabling tracks alone is not enough).
- */
-export function AgentRemoteAudio({
-  remoteAudioGain = DEFAULT_REMOTE_AUDIO_GAIN,
-}: AgentRemoteAudioProps): React.ReactElement | null {
-  const { getRemoteAudioStream, interactionMode, voiceSessionId, voiceStatus } =
-    useAgentAssistant();
-  const [streamUrl, setStreamUrl] = useState<string | null>(null);
-
-  useEffect(() => {
-    if (interactionMode !== 'voice' || !voiceSessionId) {
-      setStreamUrl(null);
-      return;
-    }
-
-    const sync = () => {
-      const stream = getRemoteAudioStream() as StreamWithUrl | null;
-      applyRemoteAudioPlaybackGain(
-        stream as Parameters<typeof applyRemoteAudioPlaybackGain>[0],
-        remoteAudioGain,
-      );
-      const tracks = stream?.getAudioTracks?.() ?? [];
-      for (const track of tracks) {
-        if (track.kind === 'audio' || track.kind == null) {
-          track.enabled = true;
-        }
-      }
-      const url = stream?.toURL?.() ?? null;
-      setStreamUrl((prev) => (prev === url ? prev : url));
-    };
-
-    sync();
-    const id = setInterval(sync, 150);
-    return () => clearInterval(id);
-  }, [
-    getRemoteAudioStream,
-    interactionMode,
-    voiceSessionId,
-    voiceStatus,
-    remoteAudioGain,
-  ]);
-
-  if (!RTCView || !streamUrl || interactionMode !== 'voice') {
-    return null;
-  }
-
-  return (
-    <RTCView
-      key={streamUrl}
-      streamURL={streamUrl}
-      objectFit="cover"
-      zOrder={-1}
-      style={styles.hidden}
-    />
-  );
-}
-
-const styles = StyleSheet.create({
-  hidden: {
-    position: 'absolute',
-    width: 1,
-    height: 1,
-    opacity: 0.01,
-    left: -1000,
-    top: -1000,
-  },
-});