ibi-ai-talk 1.0.0

package/src/utils/recorder.js

@@ -0,0 +1,420 @@
+// 音频录制模块
+import { initOpusEncoder } from "./opus-codec.js";
+import { getAudioPlayer } from "./player.js";
+
+// 音频录制器类
+export class AudioRecorder {
+  constructor() {
+    this.isRecording = false;
+    this.audioContext = null;
+    this.analyser = null;
+    this.audioProcessor = null;
+    this.audioProcessorType = null;
+    this.audioSource = null;
+    this.opusEncoder = null;
+    this.pcmDataBuffer = new Int16Array();
+    this.audioBuffers = [];
+    this.totalAudioSize = 0;
+    this.visualizationRequest = null;
+    this.recordingTimer = null;
+    this.websocket = null;
+
+    // 回调函数
+    this.onRecordingStart = null;
+    this.onRecordingStop = null;
+    this.onVisualizerUpdate = null;
+  }
+
+  // 设置WebSocket实例
+  setWebSocket(ws) {
+    this.websocket = ws;
+  }
+
+  // 获取AudioContext实例
+  getAudioContext() {
+    const audioPlayer = getAudioPlayer();
+    return audioPlayer.getAudioContext();
+  }
+
+  // 初始化编码器
+  initEncoder() {
+    if (!this.opusEncoder) {
+      this.opusEncoder = initOpusEncoder();
+    }
+    return this.opusEncoder;
+  }
+
+  // PCM处理器代码
+  getAudioProcessorCode() {
+    return `
+            class AudioRecorderProcessor extends AudioWorkletProcessor {
+                constructor() {
+                    super();
+                    this.buffers = [];
+                    this.frameSize = 960;
+                    this.buffer = new Int16Array(this.frameSize);
+                    this.bufferIndex = 0;
+                    this.isRecording = false;
+
+                    this.port.onmessage = (event) => {
+                        if (event.data.command === 'start') {
+                            this.isRecording = true;
+                            this.port.postMessage({ type: 'status', status: 'started' });
+                        } else if (event.data.command === 'stop') {
+                            this.isRecording = false;
+
+                            if (this.bufferIndex > 0) {
+                                const finalBuffer = this.buffer.slice(0, this.bufferIndex);
+                                this.port.postMessage({
+                                    type: 'buffer',
+                                    buffer: finalBuffer
+                                });
+                                this.bufferIndex = 0;
+                            }
+
+                            this.port.postMessage({ type: 'status', status: 'stopped' });
+                        }
+                    };
+                }
+
+                process(inputs, outputs, parameters) {
+                    if (!this.isRecording) return true;
+
+                    const input = inputs[0][0];
+                    if (!input) return true;
+
+                    for (let i = 0; i < input.length; i++) {
+                        if (this.bufferIndex >= this.frameSize) {
+                            this.port.postMessage({
+                                type: 'buffer',
+                                buffer: this.buffer.slice(0)
+                            });
+                            this.bufferIndex = 0;
+                        }
+
+                        this.buffer[this.bufferIndex++] = Math.max(-32768, Math.min(32767, Math.floor(input[i] * 32767)));
+                    }
+
+                    return true;
+                }
+            }
+
+            registerProcessor('audio-recorder-processor', AudioRecorderProcessor);
+        `;
+  }
+
+  // 创建音频处理器
+  async createAudioProcessor() {
+    this.audioContext = this.getAudioContext();
+
+    try {
+      if (this.audioContext.audioWorklet) {
+        const blob = new Blob([this.getAudioProcessorCode()], {
+          type: "application/javascript",
+        });
+        const url = URL.createObjectURL(blob);
+        await this.audioContext.audioWorklet.addModule(url);
+        URL.revokeObjectURL(url);
+
+        const audioProcessor = new AudioWorkletNode(
+          this.audioContext,
+          "audio-recorder-processor"
+        );
+
+        audioProcessor.port.onmessage = (event) => {
+          if (event.data.type === "buffer") {
+            this.processPCMBuffer(event.data.buffer);
+          }
+        };
+
+        console.log("使用AudioWorklet处理音频", "success");
+
+        const silent = this.audioContext.createGain();
+        silent.gain.value = 0;
+        audioProcessor.connect(silent);
+        silent.connect(this.audioContext.destination);
+        return { node: audioProcessor, type: "worklet" };
+      } else {
+        console.log(
+          "AudioWorklet不可用，使用ScriptProcessorNode作为回退方案",
+          "warning"
+        );
+        return this.createScriptProcessor();
+      }
+    } catch (error) {
+      console.log(
+        `创建音频处理器失败: ${error.message}，尝试回退方案`,
+        "error"
+      );
+      return this.createScriptProcessor();
+    }
+  }
+
+  // 创建ScriptProcessor作为回退
+  createScriptProcessor() {
+    try {
+      const frameSize = 4096;
+      const scriptProcessor = this.audioContext.createScriptProcessor(
+        frameSize,
+        1,
+        1
+      );
+
+      scriptProcessor.onaudioprocess = (event) => {
+        if (!this.isRecording) return;
+
+        const input = event.inputBuffer.getChannelData(0);
+        const buffer = new Int16Array(input.length);
+
+        for (let i = 0; i < input.length; i++) {
+          buffer[i] = Math.max(
+            -32768,
+            Math.min(32767, Math.floor(input[i] * 32767))
+          );
+        }
+
+        this.processPCMBuffer(buffer);
+      };
+
+      const silent = this.audioContext.createGain();
+      silent.gain.value = 0;
+      scriptProcessor.connect(silent);
+      silent.connect(this.audioContext.destination);
+
+      console.log("使用ScriptProcessorNode作为回退方案成功", "warning");
+      return { node: scriptProcessor, type: "processor" };
+    } catch (fallbackError) {
+      console.log(`回退方案也失败: ${fallbackError.message}`, "error");
+      return null;
+    }
+  }
+
+  // 处理PCM缓冲数据
+  processPCMBuffer(buffer) {
+    if (!this.isRecording) return;
+
+    const newBuffer = new Int16Array(this.pcmDataBuffer.length + buffer.length);
+    newBuffer.set(this.pcmDataBuffer);
+    newBuffer.set(buffer, this.pcmDataBuffer.length);
+    this.pcmDataBuffer = newBuffer;
+
+    const samplesPerFrame = 960;
+
+    while (this.pcmDataBuffer.length >= samplesPerFrame) {
+      const frameData = this.pcmDataBuffer.slice(0, samplesPerFrame);
+      this.pcmDataBuffer = this.pcmDataBuffer.slice(samplesPerFrame);
+
+      this.encodeAndSendOpus(frameData);
+    }
+  }
+
+  // 编码并发送Opus数据
+  encodeAndSendOpus(pcmData = null) {
+    if (!this.opusEncoder) {
+      console.log("Opus编码器未初始化", "error");
+      return;
+    }
+
+    try {
+      if (pcmData) {
+        const opusData = this.opusEncoder.encode(pcmData);
+        if (opusData && opusData.length > 0) {
+          this.audioBuffers.push(opusData.buffer);
+          this.totalAudioSize += opusData.length;
+
+          if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+            try {
+              this.websocket.send(opusData.buffer);
+            } catch (error) {
+              console.log(`WebSocket发送错误: ${error.message}`, "error");
+            }
+          }
+        } else {
+          log("Opus编码失败，无有效数据返回", "error");
+        }
+      } else {
+        if (this.pcmDataBuffer.length > 0) {
+          const samplesPerFrame = 960;
+          if (this.pcmDataBuffer.length < samplesPerFrame) {
+            const paddedBuffer = new Int16Array(samplesPerFrame);
+            paddedBuffer.set(this.pcmDataBuffer);
+            this.encodeAndSendOpus(paddedBuffer);
+          } else {
+            this.encodeAndSendOpus(
+              this.pcmDataBuffer.slice(0, samplesPerFrame)
+            );
+          }
+          this.pcmDataBuffer = new Int16Array(0);
+        }
+      }
+    } catch (error) {
+      console.log(`Opus编码错误: ${error.message}`, "error");
+    }
+  }
+
+  // 开始录音
+  async start() {
+    try {
+      if (!this.initEncoder()) {
+        console.log("无法启动录音: Opus编码器初始化失败", "error");
+        return false;
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          sampleRate: 16000,
+          channelCount: 1,
+          latency: { ideal: 0.02, max: 0.05 },
+          // Chrome 扩展参数（非标准，可能变动）
+          googNoiseSuppression: true, // 启用 Chrome 噪声抑制
+          googNoiseSuppression2: 3, // 级别设置（1-3，数值越高抑制越强，不同版本可能有差异）
+          googAutoGainControl: true, // 自动增益控制
+          googHighpassFilter: true, // 高通滤波器（过滤低频噪声）
+        },
+      });
+
+      this.audioContext = this.getAudioContext();
+
+      if (this.audioContext.state === "suspended") {
+        await this.audioContext.resume();
+      }
+
+      const processorResult = await this.createAudioProcessor();
+      if (!processorResult) {
+        console.log("无法创建音频处理器", "error");
+        return false;
+      }
+
+      this.audioProcessor = processorResult.node;
+      this.audioProcessorType = processorResult.type;
+
+      this.audioSource = this.audioContext.createMediaStreamSource(stream);
+      this.analyser = this.audioContext.createAnalyser();
+      this.analyser.fftSize = 2048;
+
+      this.audioSource.connect(this.analyser);
+      this.audioSource.connect(this.audioProcessor);
+
+      this.pcmDataBuffer = new Int16Array();
+      this.audioBuffers = [];
+      this.totalAudioSize = 0;
+      this.isRecording = true;
+
+      if (this.audioProcessorType === "worklet" && this.audioProcessor.port) {
+        this.audioProcessor.port.postMessage({ command: "start" });
+      }
+
+      // 发送监听开始消息
+      if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+        const listenMessage = {
+          type: "listen",
+          mode: localStorage.getItem("listenMode") || "wakeup",
+          state: "start",
+        };
+
+        console.log(
+          `发送录音开始消息: ${JSON.stringify(listenMessage)}`,
+          "info"
+        );
+        this.websocket.send(JSON.stringify(listenMessage));
+      } else {
+        console.log("WebSocket未连接，无法发送开始消息", "error");
+        return false;
+      }
+
+      // 启动录音计时器
+      let recordingSeconds = 0;
+      this.recordingTimer = setInterval(() => {
+        recordingSeconds += 0.1;
+        if (this.onRecordingStart) {
+          this.onRecordingStart(recordingSeconds);
+        }
+      }, 100);
+
+      console.log("开始PCM直接录音", "success");
+      return true;
+    } catch (error) {
+      console.log(`直接录音启动错误: ${error.message}`, "error");
+      this.isRecording = false;
+      return false;
+    }
+  }
+
+  // 停止录音
+  stop() {
+    if (!this.isRecording) return false;
+
+    try {
+      this.isRecording = false;
+
+      if (this.audioProcessor) {
+        if (this.audioProcessorType === "worklet" && this.audioProcessor.port) {
+          this.audioProcessor.port.postMessage({ command: "stop" });
+        }
+
+        this.audioProcessor.disconnect();
+        this.audioProcessor = null;
+      }
+
+      if (this.audioSource) {
+        this.audioSource.disconnect();
+        this.audioSource = null;
+      }
+
+      if (this.visualizationRequest) {
+        cancelAnimationFrame(this.visualizationRequest);
+        this.visualizationRequest = null;
+      }
+
+      if (this.recordingTimer) {
+        clearInterval(this.recordingTimer);
+        this.recordingTimer = null;
+      }
+
+      // 编码并发送剩余的数据
+      this.encodeAndSendOpus();
+
+      // 发送结束信号
+      if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+        const emptyOpusFrame = new Uint8Array(0);
+        this.websocket.send(emptyOpusFrame);
+
+        const stopMessage = {
+          type: "listen",
+          mode: localStorage.getItem("listenMode") || "wakeup",
+          state: "stop",
+        };
+
+        this.websocket.send(JSON.stringify(stopMessage));
+        console.log("已发送录音停止信号", "info");
+      }
+
+      if (this.onRecordingStop) {
+        this.onRecordingStop();
+      }
+
+      console.log("停止PCM直接录音", "success");
+      return true;
+    } catch (error) {
+      console.log(`直接录音停止错误: ${error.message}`, "error");
+      return false;
+    }
+  }
+
+  // 获取分析器
+  getAnalyser() {
+    return this.analyser;
+  }
+}
+
+// 创建单例
+let audioRecorderInstance = null;
+
+export function getAudioRecorder() {
+  if (!audioRecorderInstance) {
+    audioRecorderInstance = new AudioRecorder();
+  }
+  return audioRecorderInstance;
+}

package/src/utils/stream-context.js

@@ -0,0 +1,184 @@
+import BlockingQueue from "./blocking-queue.js";
+
+// 音频流播放上下文类
+export class StreamingContext {
+  constructor(
+    opusDecoder,
+    audioContext,
+    sampleRate,
+    channels,
+    minAudioDuration
+  ) {
+    this.opusDecoder = opusDecoder;
+    this.audioContext = audioContext;
+
+    // 音频参数
+    this.sampleRate = sampleRate;
+    this.channels = channels;
+    this.minAudioDuration = minAudioDuration;
+
+    // 初始化队列和状态
+    this.queue = []; // 已解码的PCM队列。正在播放
+    this.activeQueue = new BlockingQueue(); // 已解码的PCM队列。准备播放
+    this.pendingAudioBufferQueue = []; // 待处理的缓存队列
+    this.audioBufferQueue = new BlockingQueue(); // 缓存队列
+    this.playing = false; // 是否正在播放
+    this.endOfStream = false; // 是否收到结束信号
+    this.source = null; // 当前音频源
+    this.totalSamples = 0; // 累积的总样本数
+    this.lastPlayTime = 0; // 上次播放的时间戳
+  }
+
+  // 缓存音频数组
+  pushAudioBuffer(item) {
+    this.audioBufferQueue.enqueue(...item);
+  }
+
+  // 获取需要处理缓存队列，单线程：在audioBufferQueue一直更新的状态下不会出现安全问题
+  async getPendingAudioBufferQueue() {
+    // 原子交换 + 清空
+    [this.pendingAudioBufferQueue, this.audioBufferQueue] = [
+      await this.audioBufferQueue.dequeue(),
+      new BlockingQueue(),
+    ];
+  }
+
+  // 获取正在播放已解码的PCM队列，单线程：在activeQueue一直更新的状态下不会出现安全问题
+  async getQueue(minSamples) {
+    let TepArray = [];
+    const num =
+      minSamples - this.queue.length > 0 ? minSamples - this.queue.length : 1;
+    // 原子交换 + 清空
+    [TepArray, this.activeQueue] = [
+      await this.activeQueue.dequeue(num),
+      new BlockingQueue(),
+    ];
+    this.queue.push(...TepArray);
+  }
+
+  // 将Int16音频数据转换为Float32音频数据
+  convertInt16ToFloat32(int16Data) {
+    const float32Data = new Float32Array(int16Data.length);
+    for (let i = 0; i < int16Data.length; i++) {
+      // 将[-32768,32767]范围转换为[-1,1]，统一使用32768.0避免不对称失真
+      float32Data[i] = int16Data[i] / 32768.0;
+    }
+    return float32Data;
+  }
+
+  // 将Opus数据解码为PCM
+  async decodeOpusFrames() {
+    if (!this.opusDecoder) {
+      console.log("Opus解码器未初始化，无法解码", "error");
+      return;
+    } else {
+      console.log("Opus解码器启动", "info");
+    }
+
+    while (true) {
+      let decodedSamples = [];
+      for (const frame of this.pendingAudioBufferQueue) {
+        try {
+          // 使用Opus解码器解码
+          const frameData = this.opusDecoder.decode(frame);
+          if (frameData && frameData.length > 0) {
+            // 转换为Float32
+            const floatData = this.convertInt16ToFloat32(frameData);
+            // 使用循环替代展开运算符
+            for (let i = 0; i < floatData.length; i++) {
+              decodedSamples.push(floatData[i]);
+            }
+          }
+        } catch (error) {
+          console.log("Opus解码失败: " + error.message, "error");
+        }
+      }
+
+      if (decodedSamples.length > 0) {
+        // 使用循环替代展开运算符
+        for (let i = 0; i < decodedSamples.length; i++) {
+          this.activeQueue.enqueue(decodedSamples[i]);
+        }
+        this.totalSamples += decodedSamples.length;
+      } else {
+        console.log("没有成功解码的样本", "warning");
+      }
+      await this.getPendingAudioBufferQueue();
+    }
+  }
+
+  // 开始播放音频
+  async startPlaying() {
+    let scheduledEndTime = this.audioContext.currentTime; // 跟踪已调度音频的结束时间
+
+    while (true) {
+      // 初始缓冲：等待足够的样本再开始播放
+      const minSamples = this.sampleRate * this.minAudioDuration * 2;
+      if (!this.playing && this.queue.length < minSamples) {
+        await this.getQueue(minSamples);
+      }
+      this.playing = true;
+
+      // 持续播放队列中的音频，每次播放一个小块
+      while (this.playing && this.queue.length > 0) {
+        // 每次播放120ms的音频（2个Opus包）
+        const playDuration = 0.12;
+        const targetSamples = Math.floor(this.sampleRate * playDuration);
+        const actualSamples = Math.min(this.queue.length, targetSamples);
+
+        if (actualSamples === 0) break;
+
+        const currentSamples = this.queue.splice(0, actualSamples);
+        const audioBuffer = this.audioContext.createBuffer(
+          this.channels,
+          currentSamples.length,
+          this.sampleRate
+        );
+        audioBuffer.copyToChannel(new Float32Array(currentSamples), 0);
+
+        // 创建音频源
+        this.source = this.audioContext.createBufferSource();
+        this.source.buffer = audioBuffer;
+
+        // 精确调度播放时间
+        const currentTime = this.audioContext.currentTime;
+        const startTime = Math.max(scheduledEndTime, currentTime);
+
+        // 直接连接到输出
+        this.source.connect(this.audioContext.destination);
+
+        this.source.start(startTime);
+
+        // 更新下一个音频块的调度时间
+        const duration = audioBuffer.duration;
+        scheduledEndTime = startTime + duration;
+        this.lastPlayTime = startTime;
+
+        // 如果队列中数据不足，等待新数据
+        if (this.queue.length < targetSamples) {
+          break;
+        }
+      }
+
+      // 等待新数据
+      await this.getQueue(minSamples);
+    }
+  }
+}
+
+// 创建streamingContext实例的工厂函数
+export function createStreamingContext(
+  opusDecoder,
+  audioContext,
+  sampleRate,
+  channels,
+  minAudioDuration
+) {
+  return new StreamingContext(
+    opusDecoder,
+    audioContext,
+    sampleRate,
+    channels,
+    minAudioDuration
+  );
+}