byt-lingxiao-ai 0.3.28 → 0.3.29

package/components/mixins/audioMixin.js

@@ -1,20 +1,94 @@
 export default {
   data() {
     return {
+      audioContext: null,
+      workletNode: null,
+      mediaStreamSource: null,
       isRecording: false,
       isMicAvailable: false,
-      audioContext: null,
-      microphone: null,
-      processor: null,
-      audioBuffer: new Float32Array(0)
-    }
+      audioData: new Float32Array(128)
+    };
   },
   methods: {
+    async initAudioWorklet() {
+      if (this.audioContext) return;
+
+      const AudioContext = window.AudioContext || window.webkitAudioContext;
+      this.audioContext = new AudioContext({ sampleRate: this.SAMPLE_RATE });
+
+      const processorCode = `
+        class RecorderProcessor extends AudioWorkletProcessor {
+          constructor() {
+            super();
+            this.buffer = [];
+            this.frameSize = ${this.FRAME_SIZE};
+          }
+
+          process(inputs) {
+            const input = inputs[0];
+            if (!input || !input[0]) return true;
+
+            const channel = input[0];
+
+            for (let i = 0; i < channel.length; i++) {
+              this.buffer.push(channel[i]);
+            }
+
+            while (this.buffer.length >= this.frameSize) {
+              const frame = this.buffer.splice(0, this.frameSize);
+
+              // PCM
+              const pcm = this.floatTo16BitPCM(frame);
+              this.port.postMessage({
+                type: 'audio',
+                payload: pcm
+              });
+
+              // Visual（下采样）
+              this.port.postMessage({
+                type: 'visual',
+                payload: new Float32Array(frame.slice(0, 128))
+              });
+            }
+            return true;
+          }
+
+          floatTo16BitPCM(float32Array) {
+            const pcm = new Int16Array(float32Array.length);
+            for (let i = 0; i < float32Array.length; i++) {
+              let s = Math.max(-1, Math.min(1, float32Array[i]));
+              pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+            }
+            return pcm.buffer;
+          }
+        }
+
+        registerProcessor('recorder-processor', RecorderProcessor);
+      `;
+
+      const blob = new Blob([processorCode], {
+        type: 'application/javascript'
+      });
+      const moduleUrl = URL.createObjectURL(blob);
+
+      await this.audioContext.audioWorklet.addModule(moduleUrl);
+
+      this.workletNode = new AudioWorkletNode(
+        this.audioContext,
+        'recorder-processor'
+      );
+
+      this.workletNode.port.onmessage = this.handleWorkletMessage;
+    },
     async initAudio() {
       if (this.isRecording) return;
+
       try {
         this.isMicAvailable = true;
-        const stream = await navigator.mediaDevices.getUserMedia({
+
+        await this.initAudioWorklet();
+
+        this.stream = await navigator.mediaDevices.getUserMedia({
           audio: {
             sampleRate: this.SAMPLE_RATE,
             channelCount: 1,
@@ -24,102 +98,85 @@ export default {
           }
         });
 
-        this.audioContext = new AudioContext({ sampleRate: this.SAMPLE_RATE });
-        this.microphone = this.audioContext.createMediaStreamSource(stream);
-        this.processor = this.audioContext.createScriptProcessor(this.FRAME_SIZE, 1, 1);
-        this.processor.onaudioprocess = this.processAudio;
-        
-        this.microphone.connect(this.processor);
-        this.processor.connect(this.audioContext.destination);
+        this.mediaStreamSource =
+          this.audioContext.createMediaStreamSource(this.stream);
+
+        this.mediaStreamSource.connect(this.workletNode);
+
+        if (this.audioContext.state === 'suspended') {
+          await this.audioContext.resume();
+        }
 
         this.isRecording = true;
-        console.log(`录音中 (采样率: ${this.audioContext.sampleRate}Hz)`);
-      } catch (error) {
-        console.error("音频初始化失败:", error);
+        console.log(`AudioWorklet 录音中 (${this.SAMPLE_RATE}Hz)`);
+
+      } catch (e) {
+        console.error('音频初始化失败:', e);
         this.isRecording = false;
         this.isMicAvailable = false;
       }
     },
+    handleWorkletMessage(e) {
+      const { type, payload } = e.data;
 
-    processAudio(event) {
-      if (!this.isRecording) return;
-      const inputData = event.inputBuffer.getChannelData(0);
-
-      const tempBuffer = new Float32Array(this.audioBuffer.length + inputData.length);
-      tempBuffer.set(this.audioBuffer, 0);
-      tempBuffer.set(inputData, this.audioBuffer.length);
-      this.audioBuffer = tempBuffer;
-
-      while (this.audioBuffer.length >= this.FRAME_SIZE) {
-        const frame = this.audioBuffer.slice(0, this.FRAME_SIZE);
-        this.audioBuffer = this.audioBuffer.slice(this.FRAME_SIZE);
-        const pcmData = this.floatTo16BitPCM(frame);
-
-        if (this.ws && this.ws.readyState === this.ws.OPEN) {
-          this.ws.send(pcmData);
+      if (type === 'audio') {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+          this.ws.send(payload);
         }
       }
     },
+    stopRecording() {
+      if (!this.isRecording) return;
 
-    floatTo16BitPCM(input) {
-      const output = new Int16Array(input.length);
-      for (let i = 0; i < input.length; i++) {
-        const s = Math.max(-1, Math.min(1, input[i]));
-        output[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
+      if (this.mediaStreamSource) this.mediaStreamSource.disconnect();
+      if (this.workletNode) this.workletNode.disconnect();
+
+      if (this.stream) {
+        this.stream.getTracks().forEach(t => t.stop());
       }
-      return output.buffer;
-    },
 
-    stopRecording() {
-      if (!this.isRecording) return;
-      if (this.microphone) this.microphone.disconnect();
-      if (this.processor) this.processor.disconnect();
-      if (this.analyser) this.analyser.disconnect();
       if (this.audioContext) {
-        this.audioContext.close().catch(e => {
-          console.error("关闭音频上下文失败:", e);
-        });
+        this.audioContext.close();
+        this.audioContext = null;
       }
+
       this.isRecording = false;
+      console.log('录音已停止');
     },
+    handleWebSocketMessage(data) {
+      if (data.type === 'detection') {
+        if (this.robotStatus === 'speaking') {
+          this.robotStatus = 'waiting';
+        }
+        this.avaterStatus = 'normal';
+        this.startTime = Date.now();
+      }
+
+      if (data.type === 'Collecting') {
+        this.avaterStatus = 'thinking';
+      }
 
+      if (data.type === 'command') {
+        if (this.startTime) {
+          console.log(
+            `[Latency] ${Date.now() - this.startTime}ms`
+          );
+          this.startTime = null;
+        }
+        this.analyzeAudioCommand(data.category);
+      }
+    },
     play() {
-      this.robotStatus = 'speaking';
-      this.$refs.audioPlayer.play();
+      this.$refs.audioPlayer?.play();
     },
-
     pause() {
-      console.log('暂停播放');
-      this.robotStatus = 'waiting';
-      this.$refs.audioPlayer.pause();
+      this.$refs.audioPlayer?.pause();
     },
-
     stop() {
-      this.robotStatus = 'leaving';
-      this.$refs.audioPlayer.pause();
-      this.$refs.audioPlayer.currentTime = 0;
-      this.jumpedTimePoints.clear();
-    },
-
-    analyzeAudioCommand(command) {
-      console.log('分析音频命令:', command);
-      if (command === 'C5') {
-        this.robotStatus = 'entering';
-        setTimeout(() => {
-          this.robotStatus = 'speaking';
-          this.play();
-        }, 3000);
-      } else if (command === 'C8') {
-        this.robotStatus = 'speaking';
-        this.play();
-      } else if (command === 'C7') {
-        this.robotStatus = 'waiting';
-        this.pause();
-      } else if (command === 'C6') {
-        this.robotStatus = 'leaving';
-        this.avaterStatus = 'normal';
-        this.stop();
-      }
+      const p = this.$refs.audioPlayer;
+      if (!p) return;
+      p.pause();
+      p.currentTime = 0;
     }
   }
-}
+};

package/components/mixins/webSocketMixin.js

@@ -12,11 +12,7 @@ export default {
   methods: {
     initWebSocket() {
       try {
-        // this.ws = new WebSocket('ws://10.2.233.41:9999');
-        // 测试
-        // console.log('WS_URL:', WS_URL)
         this.ws = new WebSocket(WS_URL);
-        // this.ws = new WebSocket('ws://192.168.8.87/audio/ws/');
         this.ws.binaryType = 'arraybuffer';
 
         this.ws.onopen = async () => {
@@ -74,7 +70,6 @@ export default {
         console.error('WebSocket连接失败:', error);
       }
     },
-
     handleWebSocketMessage(data) {
       if (data.type === 'detection') {
         console.log('检测到唤醒词');
@@ -95,13 +90,15 @@ export default {
         console.log('状态: 处理中');
 
         // 性能检测终点
-        if (this.startTime) {
-          const latency = Date.now() - this.startTime;
-          console.log(`[Latency] 完整命令处理耗时: ${latency}ms`); // 记录端到端延迟
-          this.startTime = null;
-        }
+        // if (this.startTime) {
+        //   const latency = Date.now() - this.startTime;
+        //   console.log(`[Latency] 完整命令处理耗时: ${latency}ms`); // 记录端到端延迟
+        //   this.startTime = null;
+        // }
 
         this.analyzeAudioCommand(data.category);
+      } else if (data.type === 'voice') {
+        this.analyzeVoiceCommand(data.category);
       } else {
         console.log('状态: 其他');
         this.avaterStatus = 'normal';