@fe-free/ai 4.1.26 → 4.1.27

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @fe-free/ai
 
+## 4.1.27
+
+### Patch Changes
+
+- feat: ai voice
+  - @fe-free/core@4.1.27
+  - @fe-free/icons@4.1.27
+  - @fe-free/tool@4.1.27
+
 ## 4.1.26
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fe-free/ai",
-  "version": "4.1.26",
+  "version": "4.1.27",
   "description": "",
   "main": "./src/index.ts",
   "author": "",
@@ -19,7 +19,7 @@
     "lodash-es": "^4.17.21",
     "uuid": "^13.0.0",
     "zustand": "^4.5.7",
-    "@fe-free/core": "4.1.26"
+    "@fe-free/core": "4.1.27"
   },
   "peerDependencies": {
     "antd": "^5.27.1",
@@ -29,8 +29,8 @@
     "i18next-icu": "^2.4.1",
     "react": "^19.2.0",
     "react-i18next": "^16.4.0",
-    "@fe-free/icons": "4.1.26",
-    "@fe-free/tool": "4.1.26"
+    "@fe-free/icons": "4.1.27",
+    "@fe-free/tool": "4.1.27"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",

package/src/ai.stories.tsx

@@ -5,6 +5,7 @@ import {
   EnumChatMessageStatus,
   EnumChatMessageType,
   generateUUID,
+  getRecordAudioOfPCM,
   Markdown,
   MessageActions,
   Messages,
@@ -12,7 +13,7 @@ import {
 } from '@fe-free/ai';
 import { sleep } from '@fe-free/tool';
 import type { Meta } from '@storybook/react-vite';
-import { Button, Divider } from 'antd';
+import { App, Button, Divider } from 'antd';
 import { set } from 'lodash-es';
 import { useCallback, useEffect, useMemo } from 'react';
 
@@ -54,6 +55,8 @@ function Component() {
   const updateMessage = useChatStore((state) => state.updateMessage);
   const { chatStatus } = useChatStoreComputed();
 
+  const { message } = App.useApp();
+
   // init from cache
   useEffect(() => {
     const cacheMessages = localStorage.getItem('chatMessages');
@@ -103,6 +106,10 @@ function Component() {
     [addMessage, updateMessage],
   );
 
+  const { start: startRecord, stop: stopRecord } = useMemo(() => {
+    return getRecordAudioOfPCM();
+  }, []);
+
   return (
     <div>
       <div>
@@ -139,6 +146,28 @@ function Component() {
                 onChange={(v) => setSenderValue(v)}
                 loading={loading}
                 onSubmit={handleSubmit}
+                allowSpeech={{
+                  onRecordStart: async () => {
+                    console.log('onRecordStart');
+                    try {
+                      await startRecord({
+                        onAudio: (data) => {
+                          console.log('onAudio', data);
+                        },
+                        onError: (err) => {
+                          message.error(err.message);
+                        },
+                      });
+                    } catch (err) {
+                      console.error(err);
+                    }
+                  },
+                  onRecordEnd: async (isSend) => {
+                    console.log('onRecordEnd', isSend);
+                    const voiceData = await stopRecord();
+                    console.log('voiceData', voiceData);
+                  },
+                }}
               />
             </div>
           }

package/src/index.ts

@@ -15,4 +15,4 @@ export { EnumChatMessageStatus, EnumChatMessageType } from './store/types';
 export type { ChatMessage, ChatMessageOfAI, ChatMessageOfUser } from './store/types';
 export { fetchStream } from './stream';
 export { Tip } from './tip';
-export { recordAudioOfPCM } from './voice';
+export { getRecordAudioOfBlob, getRecordAudioOfPCM } from './voice';

package/src/m_sender/m_sender.stories.tsx

@@ -1,7 +1,7 @@
-import { MSender } from '@fe-free/ai';
-import { sleep } from '@fe-free/tool';
+import { getRecordAudioOfBlob, getRecordAudioOfPCM, MSender } from '@fe-free/ai';
 import type { Meta, StoryObj } from '@storybook/react-vite';
-import { useCallback, useState } from 'react';
+import { App } from 'antd';
+import { useCallback, useMemo, useState } from 'react';
 import type { MSenderProps, MSenderValue } from './types';
 
 const meta: Meta<typeof MSender> = {
@@ -61,8 +61,11 @@ export const Loading: Story = {
 
 export const AllowSpeech: Story = {
   render: (props) => {
-    // 假设是字符串，实则是 buffer
-    const [recordVoice, setRecordVoice] = useState<string | undefined>(undefined);
+    const { message } = App.useApp();
+
+    const { start: startRecord, stop: stopRecord } = useMemo(() => {
+      return getRecordAudioOfPCM();
+    }, []);
 
     const handleSubmit = (value: MSenderValue) => {
       console.log('handleSubmit', value);
@@ -70,22 +73,34 @@ export const AllowSpeech: Story = {
 
     const handleRecordStart = useCallback(async () => {
       // 假设这是录音的文本
-      setRecordVoice('这是录音的文本');
+      try {
+        await startRecord({
+          onAudio: (data) => {
+            console.log('onAudio', data);
+          },
+          onError: (err) => {
+            message.error(err.message);
+          },
+        });
+      } catch (err) {
+        console.error(err);
+      }
 
       return;
     }, []);
 
     const handleRecordEnd = useCallback(
-      async (isSend: boolean) => {
+      async (isSend) => {
         console.log('handleRecordEnd isSend', isSend);
-        if (isSend) {
-          await sleep(1000);
-          const recordResult = recordVoice;
 
-          handleSubmit({ ...(props.value || {}), text: recordResult });
+        const voiceData = await stopRecord();
+        console.log('voiceData', voiceData);
+
+        if (isSend) {
+          handleSubmit({ ...(props.value || {}), text: '假设这是识别的文字' });
         }
       },
-      [props.value, recordVoice],
+      [props.value, stopRecord],
     );
 
     return (
@@ -122,4 +137,63 @@ export const AllowSpeech: Story = {
   },
 };
 
+export const AllowSpeech2: Story = {
+  render: (props) => {
+    const { message } = App.useApp();
+
+    const { start: startRecord, stop: stopRecord } = useMemo(() => {
+      return getRecordAudioOfBlob();
+    }, []);
+
+    const handleSubmit = (value: MSenderValue) => {
+      console.log('handleSubmit', value);
+    };
+
+    const handleRecordStart = useCallback(async () => {
+      // 假设这是录音的文本
+      try {
+        await startRecord({
+          onAudio: (data) => {
+            console.log('onAudio', data);
+          },
+          onError: (err) => {
+            message.error(err.message);
+          },
+        });
+      } catch (err) {
+        console.error(err);
+      }
+
+      return;
+    }, []);
+
+    const handleRecordEnd = useCallback(
+      async (isSend) => {
+        console.log('handleRecordEnd isSend', isSend);
+
+        const voiceData = await stopRecord();
+        console.log('voiceData', voiceData);
+
+        if (isSend) {
+          handleSubmit({ ...(props.value || {}), text: '假设这是识别的文字' });
+        }
+      },
+      [props.value, stopRecord],
+    );
+
+    return (
+      <div className="flex flex-col gap-10">
+        <Component
+          {...props}
+          defaultType="record"
+          allowSpeech={{
+            onRecordStart: handleRecordStart,
+            onRecordEnd: handleRecordEnd,
+          }}
+        />
+      </div>
+    );
+  },
+};
+
 export default meta;

package/src/voice/index.ts

@@ -1,33 +1,188 @@
-async function recordAudioOfPCM({ onAudio }: { onAudio: (data: ArrayBufferLike) => void }) {
-  // --- 初始化音频 ---
-  const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
-  const audioContext = new AudioContext({ sampleRate: 16000 });
-  const sourceNode = audioContext.createMediaStreamSource(micStream);
-  // ScriptProcessorNode（4096 是稳定 buffer）
-  const processorNode = audioContext.createScriptProcessor(4096, 1, 1);
-
-  processorNode.onaudioprocess = function (event) {
-    const float32Data = event.inputBuffer.getChannelData(0); // float32
-
-    // === 转成 Int16 PCM ===
-    const pcm16 = new Int16Array(float32Data.length);
-    for (let i = 0; i < float32Data.length; i++) {
-      const s = Math.max(-1, Math.min(1, float32Data[i]));
-      pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
-    }
+function getRecordAudioOfPCM() {
+  let processorNode: ScriptProcessorNode | null = null;
+  let sourceNode: MediaStreamAudioSourceNode | null = null;
+  let audioContext: AudioContext | null = null;
+  let micStream: MediaStream | null = null;
 
-    onAudio(pcm16.buffer);
-  };
+  let data: ArrayBufferLike[] = [];
+
+  async function start({
+    onAudio,
+    onError,
+  }: {
+    onAudio: (data: ArrayBufferLike) => void;
+    onError?: (error: Error) => void;
+  }): Promise<void> {
+    try {
+      // --- 初始化音频 ---
+      micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      audioContext = new AudioContext({ sampleRate: 16000 });
+      sourceNode = audioContext.createMediaStreamSource(micStream);
+      // ScriptProcessorNode（4096 是稳定 buffer）
+      processorNode = audioContext.createScriptProcessor(4096, 1, 1);
+
+      data = [];
+
+      processorNode.onaudioprocess = function (event) {
+        const float32Data = event.inputBuffer.getChannelData(0); // float32
+
+        // === 转成 Int16 PCM ===
+        const pcm16 = new Int16Array(float32Data.length);
+        for (let i = 0; i < float32Data.length; i++) {
+          const s = Math.max(-1, Math.min(1, float32Data[i]));
+          pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+        }
+
+        data.push(pcm16.buffer);
+
+        onAudio(pcm16.buffer);
+      };
 
-  sourceNode.connect(processorNode);
-  processorNode.connect(audioContext.destination);
+      sourceNode.connect(processorNode);
+      processorNode.connect(audioContext.destination);
+    } catch (err) {
+      if (err instanceof DOMException && err.name === 'NotAllowedError') {
+        onError?.(new Error('请允许麦克风权限'));
+      } else if (err instanceof DOMException && err.name === 'NotFoundError') {
+        onError?.(new Error('未找到麦克风设备'));
+      } else if (err instanceof DOMException && err.name === 'NotReadableError') {
+        onError?.(new Error('麦克风被其他应用占用'));
+      } else {
+        onError?.(new Error('启动录音失败'));
+      }
 
-  return () => {
+      throw err;
+    }
+  }
+
+  async function stop(): Promise<{ data: ArrayBufferLike[] }> {
     if (processorNode) processorNode.disconnect();
     if (sourceNode) sourceNode.disconnect();
     if (audioContext) audioContext.close();
     if (micStream) micStream.getTracks().forEach((track) => track.stop());
+
+    const result = data;
+    data = [];
+
+    return { data: result };
+  }
+
+  return {
+    start,
+    stop,
+  };
+}
+
+function getRecordAudioOfBlob() {
+  let mediaRecorder: MediaRecorder | null = null;
+  let micStream: MediaStream | null = null;
+  let chunks: Blob[] = [];
+
+  async function start({
+    onAudio,
+    onError,
+    mimeType = 'audio/webm',
+  }: {
+    onAudio?: (blob: Blob) => void;
+    onError?: (error: Error) => void;
+    mimeType?: string;
+  }): Promise<void> {
+    try {
+      // 获取麦克风权限
+      micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+      // 检查浏览器是否支持指定的 MIME 类型
+      let finalMimeType = mimeType;
+      if (!MediaRecorder.isTypeSupported(mimeType)) {
+        // 如果不支持，尝试使用默认类型
+        finalMimeType = '';
+        console.warn(`不支持的 MIME 类型: ${mimeType}，使用默认类型`);
+      }
+
+      // 创建 MediaRecorder 实例
+      mediaRecorder = new MediaRecorder(micStream, {
+        mimeType: finalMimeType || undefined,
+      });
+
+      chunks = [];
+
+      // 监听数据可用事件
+      mediaRecorder.ondataavailable = (event) => {
+        if (event.data && event.data.size > 0) {
+          chunks.push(event.data);
+          onAudio?.(event.data);
+        }
+      };
+
+      // 监听错误事件
+      mediaRecorder.onerror = () => {
+        const error = new Error('MediaRecorder 录音错误');
+        onError?.(error);
+      };
+
+      // 开始录音
+      mediaRecorder.start(100); // 每 100ms 触发一次 dataavailable 事件
+    } catch (err) {
+      if (err instanceof DOMException && err.name === 'NotAllowedError') {
+        onError?.(new Error('请允许麦克风权限'));
+      } else if (err instanceof DOMException && err.name === 'NotFoundError') {
+        onError?.(new Error('未找到麦克风设备'));
+      } else if (err instanceof DOMException && err.name === 'NotReadableError') {
+        onError?.(new Error('麦克风被其他应用占用'));
+      } else {
+        onError?.(new Error('启动录音失败'));
+      }
+
+      throw err;
+    }
+  }
+
+  async function stop(): Promise<{ data: Blob; base64: string }> {
+    return new Promise((resolve, reject) => {
+      if (!mediaRecorder) {
+        reject(new Error('MediaRecorder 未初始化'));
+        return;
+      }
+
+      function doStop() {
+        const blob = new Blob(chunks, { type: mediaRecorder?.mimeType || 'audio/webm' });
+        chunks = [];
+
+        // 停止所有轨道
+        if (micStream) {
+          micStream.getTracks().forEach((track) => track.stop());
+        }
+
+        // 将 Blob 转换为 base64
+        const reader = new FileReader();
+        reader.onloadend = () => {
+          const base64String = reader.result as string;
+          resolve({ data: blob, base64: base64String });
+        };
+        reader.onerror = () => {
+          reject(new Error('转换为 base64 失败'));
+        };
+        reader.readAsDataURL(blob);
+      }
+
+      // 监听停止事件
+      mediaRecorder.onstop = () => {
+        doStop();
+      };
+
+      // 如果正在录音，则停止
+      if (mediaRecorder.state === 'recording') {
+        mediaRecorder.stop();
+      } else {
+        doStop();
+      }
+    });
+  }
+
+  return {
+    start,
+    stop,
   };
 }
 
-export { recordAudioOfPCM };
+export { getRecordAudioOfBlob, getRecordAudioOfPCM };