byt-lingxiao-ai 0.1.5 → 0.1.7

package/README.md

@@ -1,24 +0,0 @@
-# byt-lingxiao-test
-
-## Project setup
-```
-yarn install
-```
-
-### Compiles and hot-reloads for development
-```
-yarn serve
-```
-
-### Compiles and minifies for production
-```
-yarn build
-```
-
-### Lints and fixes files
-```
-yarn lint
-```
-
-### Customize configuration
-See [Configuration Reference](https://cli.vuejs.org/config/).

package/components/ChatWindow.vue

@@ -1,6 +1,17 @@
 <template>
   <div class="chat">
-    <div class="chat-ai" @click="toggleWindow">
+    <!-- 隐藏的音频播放器 -->
+    <audio 
+      ref="audioPlayer"
+      class="hidden-audio"
+      @timeupdate="onTimeUpdate"
+      @ended="onAudioEnded"
+    >
+      <source :src="audioSrc" type="audio/mp3">
+      您的浏览器不支持音频元素。
+    </audio>
+    <div v-if="robotStatus !== 'leaving'" :class="['chat-robot', robotStatus]" ></div>
+    <div v-else class="chat-ai" @click="toggleWindow">
       <div class="chat-ai-avater"></div>
       <div class="chat-ai-text">凌霄AI</div>
     </div>
@@ -63,7 +74,7 @@
               @keydown="handleKeyDown"
             ></el-input>
             <div class="chat-window-bar">
-              <div class="chat-window-send">
+              <div class="chat-window-send" @click="handleSend">
                 <svg
                   xmlns="http://www.w3.org/2000/svg"
                   width="20"
@@ -100,22 +111,21 @@
   </div>
 </template>
 <script>
-// ... existing code ...
-
+const SAMPLE_RATE = 16000;
+const FRAME_SIZE = 512;
 export default {
   name: 'ChatWindow',
   props: {
     appendToBody: {
       type: Boolean,
       default: true,
-    },
+    }
   },
   data() {
     return {
-      wsUrl: '',
-      inputMessage: '',
-      visible: false,
-      // ... existing code ...
+      audioSrc: '/audio/byt.mp3', // 音频URL
+      inputMessage: '', // 输入消息
+      visible: false, // 窗口是否可见
       messages: [
         {
           id: 1,
@@ -145,27 +155,297 @@ export default {
           time: '',
           content: '你好，欢迎来到凌霄大模型AI对话。',
         },
-      ],
+      ], // 消息列表
+      isRecording: false, // 正在录制
+      isMicAvailable: false, // 麦克风是否可用
+      mediaRecorder: null, // 媒体录制器
+      ws: null, // WebSocket连接
+      // wsUrl: 'wss://mes.shnfonline.top:8316/ai_model/ws/voice-stream', // WebSocket URL
+      wsUrl: 'ws://192.168.8.9:9999/ai_model/ws/voice-stream', // WebSocket URL
+      isConnected: false, // WebSocket是否已连接
+      audioContext: null, // 音频上下文
+      microphone: null, // 麦克风输入节点
+      processor: null, // 音频处理节点
+      robotStatus: 'leaving', // 机器人状态 waiting, speaking, leaving, entering
+      audioBuffer: new Float32Array(0) // 音频缓冲区
     }
   },
   mounted() {
-    this.scrollToBottom()
+    this.initWebSocket()
 
     // 处理append-to-body逻辑
     if (this.appendToBody) {
       this.appendToBodyHandler()
     }
   },
+  unmounted() {
+    this.closeWebSocket()
+    this.stopRecording()
+  },
   beforeDestroy() {
     // 组件销毁前，如果元素被移动到body中，需要移除
     if (this.appendToBody && this.$el.parentElement === document.body) {
       document.body.removeChild(this.$el)
     }
+    this.closeWebSocket()
+    this.stopRecording()
   },
   methods: {
-    initWebSocket() {},
-    initAudio() {},
-    handleSend() {},
+    // 处理音频播放结束事件
+    onAudioEnded() {
+      this.robotStatus = 'leaving'
+    },
+    // 处理音频播放时间更新事件
+    onTimeUpdate() {
+      const audio = this.$refs.audioPlayer
+      const currentTime = audio.currentTime
+
+      if (!this.jumpedTimePoints) {
+        this.jumpedTimePoints = new Set()
+      }
+
+      const timeJumpPoints = [
+        { time: 30, url: '', title: '生产管理' }, 
+        { time: 50, url: '', title: '生产信息总览' }, 
+        { time: 60, url: '', title: '能源管理' },
+        { time: 70, url: '', title: '设备管理' }
+      ]
+      // 检查当前时间是否达到跳转点
+      timeJumpPoints.forEach(point => {
+        // 使用一定的误差范围，确保不会因为播放进度的微小差异而错过跳转点
+        if (currentTime >= point.time && currentTime < point.time + 0.5 && !this.jumpedTimePoints.has(point.time)) {
+          this.jumpedTimePoints.add(point.time)
+          console.log(`到达时间点 ${point.time} 秒，跳转到 ${point.title}`)
+          this.$appOptions.store.dispatch('tags/addTagview', {
+            path: point.url,
+            name: point.title,
+            meta: {
+              title: point.title
+            }
+          })
+          this.$appOptions.router.push({
+            path: point.url
+          })
+        }
+      })
+
+      console.log('当前播放时间:', currentTime)
+    },
+    play() {
+      this.robotStatus = 'speaking'
+      this.$refs.audioPlayer.play()
+    },
+    pause() {
+      this.robotStatus = 'waiting'
+      this.$refs.audioPlayer.pause()
+    },
+    stop() {
+      this.robotStatus = 'leaving'
+      this.$refs.audioPlayer.pause()
+      this.$refs.audioPlayer.currentTime = 0
+    },
+    initWebSocket() {
+      try {
+        this.ws = new WebSocket(this.wsUrl)
+        this.ws.binaryType = 'arraybuffer'
+
+        this.ws.onopen = async () => {
+          console.log('连接成功')
+          this.isConnected = true
+          this.initAudio()
+        }
+        this.ws.onmessage = (event) => {
+          try {
+            console.log("收到原始消息:", event.data);
+            // 二进制数据直接返回
+            if (event.data instanceof ArrayBuffer) {
+              console.log("收到二进制音频数据");
+              return;
+            }
+            // 解析JSON数据
+            const data = JSON.parse(event.data);
+            console.log("解析后的数据:", data);
+
+            if (data.type === 'config'){
+              console.log('配置信息:', data);
+            } else if (data.code === 0) {
+              if (data.data.type === 'detection') {
+                console.log('检测到唤醒词...');
+              } else if (data.data.type === 'Collecting') {
+                console.log('状态: 采集中...');
+              } else if (data.data.type === 'command') {
+                // 根据指令改变机器人的状态
+                console.log('状态: 处理中...')
+                this.analyzeAudioCommand(data.data.category)
+              } else {
+                console.log('状态: 其他...')
+              }
+            } else {
+              console.error("服务器返回错误:", data.msg);
+            }
+          } catch (error) {
+            console.error("消息解析错误:", error);
+          }
+        }
+        this.ws.onclose = () => {
+          console.log('连接关闭')
+          this.isConnected = false
+          if (this.isRecording) {
+            this.stopRecording()
+          }
+          setTimeout(() => {
+            console.log('尝试重新建立连接')
+            if (!this.isConnected) {
+              // 尝试重连
+              this.initWebSocket()
+            }
+          }, 3000)
+        }
+      } catch (error) {
+        console.error('WebSocket连接失败:', error);
+      }
+    },
+    closeWebSocket() {
+      if (this.ws) {
+        this.ws.close()
+      }
+    },
+    async initAudio() {
+      if (this.isRecording) return;
+      try {
+        this.isMicAvailable = true;
+        // 2. 获取麦克风权限
+        const stream = await navigator.mediaDevices.getUserMedia({
+          audio: {
+            sampleRate: SAMPLE_RATE, // 请求指定采样率
+            channelCount: 1,         // 单声道
+            noiseSuppression: true,
+            echoCancellation: true
+          }
+        });
+
+        // 3. 创建音频处理环境
+        this.audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
+        const actualSampleRate = this.audioContext.sampleRate;
+        this.microphone = this.audioContext.createMediaStreamSource(stream);
+
+        // 4. 创建音频处理器
+        this.processor = this.audioContext.createScriptProcessor(FRAME_SIZE, 1, 1);
+        this.processor.onaudioprocess = this.processAudio;
+        // 连接处理链
+        this.microphone.connect(this.processor);
+        this.processor.connect(this.audioContext.destination);
+
+        this.isRecording = true;
+
+        console.log(`状态: 录音中 (采样率: ${actualSampleRate}Hz)`)
+      } catch (error) {
+        console.error("音频初始化失败:", error);
+        this.isRecording = false
+        this.isMicAvailable = false
+      }
+    },
+    async handleSend() {
+      if (!this.inputMessage.trim()) {
+        return
+      }
+      const message = this.inputMessage.trim();
+      // 发送消息
+      this.messages.push({
+        id: this.messages.length + 1,
+        type: 'user',
+        sender: '用户',
+        time: new Date().toLocaleTimeString(),
+        content: this.inputMessage,
+      })
+      this.inputMessage = ''
+      
+      try {
+        const token = `Bearer 24ab99b4-4b59-42a0-84df-1d73a96e70cd`
+        const response = await fetch('/bytserver/api-model/chat/stream', {
+          timeout: 30000,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json' ,
+            'Authorization': token,
+          },
+          body: JSON.stringify({ content: message })
+        });
+        if (!response.ok) {
+          throw new Error(`${response.status}`);
+        }
+
+        console.log('响应状态:', response.status);
+        // 解析响应流
+        this.parseResponseStream(response.body)
+      } catch (error) {
+        console.error('发送消息失败:', error)
+      }
+    },
+    // 分析音频命令
+    analyzeAudioCommand(command) {
+      console.log('分析音频命令:', command)
+      // 解析到开始导览，执行机器人进入动画
+      if (command === 'C5') {
+        this.robotStatus = 'entering'
+        setTimeout(() => {
+          this.robotStatus = 'speaking'
+          this.play()
+        }, 3000)
+      }
+      // 继续导览
+      else if (command === 'C8') {
+        this.robotStatus = 'speaking'
+        this.play()
+      }
+      // 解析到暂停导览，执行机器人暂停动画
+      else if (command === 'C7') {
+        this.robotStatus = 'waiting'
+        this.pause()
+      }
+      // 解析到结束导览，执行机器人离开动画
+      else if (command === 'C6') {
+        this.robotStatus = 'leaving'
+        this.stop()
+      }
+    },
+    // 解析响应流
+    parseResponseStream(body) {
+      console.log(body)
+    },
+    processAudio(event) {
+      if (!this.isRecording) return;
+      // 5. 获取音频数据并处理
+      const inputData = event.inputBuffer.getChannelData(0);
+
+      // 累积音频数据
+      const tempBuffer = new Float32Array(this.audioBuffer.length + inputData.length);
+      tempBuffer.set(this.audioBuffer, 0);
+      tempBuffer.set(inputData, this.audioBuffer.length);
+      this.audioBuffer = tempBuffer;
+
+      // 当累积足够一帧时发送
+      while (this.audioBuffer.length >= FRAME_SIZE) {
+        const frame = this.audioBuffer.slice(0, FRAME_SIZE);
+        this.audioBuffer = this.audioBuffer.slice(FRAME_SIZE);
+
+        // 转换为16位PCM
+        const pcmData = this.floatTo16BitPCM(frame);
+
+        // 通过WebSocket发送
+        if (this.ws && this.ws.readyState === this.ws.OPEN) {
+          this.ws.send(pcmData);
+        }
+      }
+    },
+    floatTo16BitPCM(input) {
+      const output = new Int16Array(input.length);
+      for (let i = 0; i < input.length; i++) {
+        const s = Math.max(-1, Math.min(1, input[i]));
+        output[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
+      }
+      return output.buffer;
+    },
     toggleWindow() {
       this.visible = !this.visible
     },
@@ -189,6 +469,18 @@ export default {
         this.handleSend()
       }
     },
+    stopRecording() {
+      if (!this.isRecording) return;
+      if (this.microphone) this.microphone.disconnect();
+      if (this.processor) this.processor.disconnect();
+      if (this.analyser) this.analyser.disconnect();
+      if (this.audioContext) {
+        this.audioContext.close().catch(e => {
+          console.error("关闭音频上下文失败:", e);
+        });
+      }
+      this.isRecording = false;
+    },
     // 添加到body的处理函数
     appendToBodyHandler() {
       // 确保DOM已经渲染完成
@@ -216,6 +508,12 @@ export default {
 }
 </script>
 <style scoped>
+.hidden-audio {
+  position: absolute;
+  left: -9999px;
+  opacity: 0;
+  pointer-events: none;
+}
 .chat-overlay {
   position: fixed;
   top: 0;
@@ -268,6 +566,22 @@ export default {
   cursor: pointer;
   user-select: none;
 }
+.chat-robot {
+  width: 150px;
+  height: 200px;
+}
+.chat-robot.entering {
+    background-image: url('./assets/entering.png');
+    background-size: cover;
+  }
+.chat-robot.waiting {
+    background-image: url('./assets/waiting.png');
+    background-size: cover;
+  }
+.chat-robot.speaking {
+    background-image: url('./assets/speaking.png');
+    background-size: cover;
+  }
 .chat-ai-avater {
   border-radius: 67px;
   border: 1px solid #0f66e4;
@@ -333,8 +647,8 @@ export default {
   background: #fff;
   box-shadow: 0 2px 20px 0 rgba(0, 0, 0, 0.16);
   position: absolute;
-  bottom: 0;
-  right: 50px;
+  bottom: 20px;
+  right: 60px;
   overflow: hidden;
   display: flex;
   flex-direction: column;

package/components/index.js

@@ -4,10 +4,12 @@ const components = {
   ChatWindow,
 };
 
-const install = function(Vue) {
+const install = function(Vue, options = {}) {
   if (install.installed) return;
   install.installed = true;
 
+  Vue.prototype.$appOptions = options;
+
   Object.keys(components).forEach(name => {
     Vue.component(name, components[name]);
   });