@fe-free/ai 4.1.26 → 4.1.28

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @fe-free/ai
 
+## 4.1.28
+
+### Patch Changes
+
+- feat: ai
+  - @fe-free/core@4.1.28
+  - @fe-free/icons@4.1.28
+  - @fe-free/tool@4.1.28
+
+## 4.1.27
+
+### Patch Changes
+
+- feat: ai voice
+  - @fe-free/core@4.1.27
+  - @fe-free/icons@4.1.27
+  - @fe-free/tool@4.1.27
+
 ## 4.1.26
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fe-free/ai",
-  "version": "4.1.26",
+  "version": "4.1.28",
   "description": "",
   "main": "./src/index.ts",
   "author": "",
@@ -19,7 +19,7 @@
     "lodash-es": "^4.17.21",
     "uuid": "^13.0.0",
     "zustand": "^4.5.7",
-    "@fe-free/core": "4.1.26"
+    "@fe-free/core": "4.1.28"
   },
   "peerDependencies": {
     "antd": "^5.27.1",
@@ -29,8 +29,8 @@
     "i18next-icu": "^2.4.1",
     "react": "^19.2.0",
     "react-i18next": "^16.4.0",
-    "@fe-free/icons": "4.1.26",
-    "@fe-free/tool": "4.1.26"
+    "@fe-free/icons": "4.1.28",
+    "@fe-free/tool": "4.1.28"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",

package/src/ai.stories.tsx

@@ -5,6 +5,7 @@ import {
   EnumChatMessageStatus,
   EnumChatMessageType,
   generateUUID,
+  getRecordAudioOfPCM,
   Markdown,
   MessageActions,
   Messages,
@@ -12,7 +13,7 @@ import {
 } from '@fe-free/ai';
 import { sleep } from '@fe-free/tool';
 import type { Meta } from '@storybook/react-vite';
-import { Button, Divider } from 'antd';
+import { App, Button, Divider } from 'antd';
 import { set } from 'lodash-es';
 import { useCallback, useEffect, useMemo } from 'react';
 
@@ -54,6 +55,8 @@ function Component() {
   const updateMessage = useChatStore((state) => state.updateMessage);
   const { chatStatus } = useChatStoreComputed();
 
+  const { message } = App.useApp();
+
   // init from cache
   useEffect(() => {
     const cacheMessages = localStorage.getItem('chatMessages');
@@ -103,6 +106,10 @@ function Component() {
     [addMessage, updateMessage],
   );
 
+  const { start: startRecord, stop: stopRecord } = useMemo(() => {
+    return getRecordAudioOfPCM();
+  }, []);
+
   return (
     <div>
       <div>
@@ -139,6 +146,28 @@ function Component() {
                 onChange={(v) => setSenderValue(v)}
                 loading={loading}
                 onSubmit={handleSubmit}
+                allowSpeech={{
+                  onRecordStart: async () => {
+                    console.log('onRecordStart');
+                    try {
+                      await startRecord({
+                        onAudio: (data) => {
+                          console.log('onAudio', data);
+                        },
+                        onError: (err) => {
+                          message.error(err.message);
+                        },
+                      });
+                    } catch (err) {
+                      console.error(err);
+                    }
+                  },
+                  onRecordEnd: async (isSend) => {
+                    console.log('onRecordEnd', isSend);
+                    const voiceData = await stopRecord();
+                    console.log('voiceData', voiceData);
+                  },
+                }}
               />
             </div>
           }

package/src/index.ts

@@ -15,4 +15,4 @@ export { EnumChatMessageStatus, EnumChatMessageType } from './store/types';
 export type { ChatMessage, ChatMessageOfAI, ChatMessageOfUser } from './store/types';
 export { fetchStream } from './stream';
 export { Tip } from './tip';
-export { recordAudioOfPCM } from './voice';
+export { getRecordAudioOfBlob, getRecordAudioOfPCM } from './voice';

package/src/m_sender/actions.tsx

@@ -12,7 +12,7 @@ function Actions(
     setType: (type: 'input' | 'record') => void;
   },
 ) {
-  const { refText, loading, onSubmit, value, onChange, setType, allowSpeech } = props;
+  const { loading, onSubmit, value, onChange, setType, allowSpeech } = props;
 
   const isLoading = loading;
 
@@ -33,10 +33,10 @@ function Actions(
       // reset
       onChange?.({});
 
-      // focus
-      refText.current?.focus();
+      // 移动端 不 focus
+      // refText.current?.focus();
     }
-  }, [isLoading, value, onSubmit, onChange, refText]);
+  }, [isLoading, value, onSubmit, onChange]);
 
   return (
     <div className="mr-1 flex items-center gap-2">
@@ -44,7 +44,7 @@ function Actions(
         <Button
           type="primary"
           shape="circle"
-          icon={<Icons component={IconRecord} />}
+          icon={<Icons component={IconRecord} className="!text-lg" />}
           onClick={() => {
             setType('record');
           }}
@@ -53,7 +53,7 @@ function Actions(
         <Button
           type="primary"
           shape="circle"
-          icon={<Icons component={SendIcon} />}
+          icon={<Icons component={SendIcon} className="!text-lg" />}
           loading={isLoading}
           onClick={handleSubmit}
         />

package/src/m_sender/m_sender.stories.tsx

@@ -1,7 +1,7 @@
-import { MSender } from '@fe-free/ai';
-import { sleep } from '@fe-free/tool';
+import { getRecordAudioOfBlob, getRecordAudioOfPCM, MSender } from '@fe-free/ai';
 import type { Meta, StoryObj } from '@storybook/react-vite';
-import { useCallback, useState } from 'react';
+import { App } from 'antd';
+import { useCallback, useMemo, useState } from 'react';
 import type { MSenderProps, MSenderValue } from './types';
 
 const meta: Meta<typeof MSender> = {
@@ -61,31 +61,23 @@ export const Loading: Story = {
 
 export const AllowSpeech: Story = {
   render: (props) => {
-    // 假设是字符串，实则是 buffer
-    const [recordVoice, setRecordVoice] = useState<string | undefined>(undefined);
-
     const handleSubmit = (value: MSenderValue) => {
       console.log('handleSubmit', value);
     };
 
     const handleRecordStart = useCallback(async () => {
-      // 假设这是录音的文本
-      setRecordVoice('这是录音的文本');
-
-      return;
+      // fake
     }, []);
 
     const handleRecordEnd = useCallback(
-      async (isSend: boolean) => {
+      async (isSend) => {
         console.log('handleRecordEnd isSend', isSend);
-        if (isSend) {
-          await sleep(1000);
-          const recordResult = recordVoice;
 
-          handleSubmit({ ...(props.value || {}), text: recordResult });
+        if (isSend) {
+          handleSubmit({ ...(props.value || {}), text: '假设这是识别的文字' });
         }
       },
-      [props.value, recordVoice],
+      [props.value],
     );
 
     return (
@@ -122,4 +114,118 @@ export const AllowSpeech: Story = {
   },
 };
 
+export const AllowSpeechPCM: Story = {
+  render: (props) => {
+    const { message } = App.useApp();
+
+    const { start: startRecord, stop: stopRecord } = useMemo(() => {
+      return getRecordAudioOfPCM();
+    }, []);
+
+    const handleSubmit = (value: MSenderValue) => {
+      console.log('handleSubmit', value);
+    };
+
+    const handleRecordStart = useCallback(async () => {
+      // 假设这是录音的文本
+      try {
+        await startRecord({
+          onAudio: (data) => {
+            console.log('onAudio', data);
+          },
+          onError: (err) => {
+            message.error(err.message);
+          },
+        });
+      } catch (err) {
+        console.error(err);
+      }
+
+      return;
+    }, []);
+
+    const handleRecordEnd = useCallback(
+      async (isSend) => {
+        console.log('handleRecordEnd isSend', isSend);
+
+        const voiceData = await stopRecord();
+        console.log('voiceData', voiceData);
+
+        if (isSend) {
+          handleSubmit({ ...(props.value || {}), text: '假设这是识别的文字' });
+        }
+      },
+      [props.value, stopRecord],
+    );
+
+    return (
+      <Component
+        {...props}
+        defaultType="record"
+        allowSpeech={{
+          onRecordStart: handleRecordStart,
+          onRecordEnd: handleRecordEnd,
+        }}
+      />
+    );
+  },
+};
+
+export const AllowSpeechBlob: Story = {
+  render: (props) => {
+    const { message } = App.useApp();
+
+    const { start: startRecord, stop: stopRecord } = useMemo(() => {
+      return getRecordAudioOfBlob();
+    }, []);
+
+    const handleSubmit = (value: MSenderValue) => {
+      console.log('handleSubmit', value);
+    };
+
+    const handleRecordStart = useCallback(async () => {
+      // 假设这是录音的文本
+      try {
+        await startRecord({
+          onAudio: (data) => {
+            console.log('onAudio', data);
+          },
+          onError: (err) => {
+            message.error(err.message);
+          },
+        });
+      } catch (err) {
+        console.error(err);
+      }
+
+      return;
+    }, []);
+
+    const handleRecordEnd = useCallback(
+      async (isSend) => {
+        console.log('handleRecordEnd isSend', isSend);
+
+        const voiceData = await stopRecord();
+        console.log('voiceData', voiceData);
+
+        if (isSend) {
+          handleSubmit({ ...(props.value || {}), text: '假设这是识别的文字' });
+        }
+      },
+      [props.value, stopRecord],
+    );
+
+    return (
+      <Component
+        {...props}
+        defaultType="record"
+        allowSpeech={{
+          onRecordStart: handleRecordStart,
+          onRecordEnd: handleRecordEnd,
+        }}
+      />
+    );
+  },
+};
+
 export default meta;

package/src/messages/message_actions.tsx

@@ -8,9 +8,18 @@ import {
   LikeOutlined,
 } from '@fe-free/icons';
 import { App, Button, Tooltip } from 'antd';
+import classNames from 'classnames';
 import { useCallback, useEffect, useState } from 'react';
 
-function MessageActionOfCopy({ value, onCopied }: { value: string; onCopied?: () => void }) {
+function MessageActionOfCopy({
+  value,
+  onCopied,
+  className,
+}: {
+  value: string;
+  onCopied?: () => void;
+  className?: string;
+}) {
   const [active, setActive] = useState(false);
   const { message } = App.useApp();
 
@@ -22,7 +31,11 @@ function MessageActionOfCopy({ value, onCopied }: { value: string; onCopied?: ()
 
   return (
     <Tooltip title="复制">
-      <Copy value={value} className="cursor-pointer text-03" onCopied={handleCopied}>
+      <Copy
+        value={value}
+        className={classNames('cursor-pointer text-03', className)}
+        onCopied={handleCopied}
+      >
         <Button
           type="text"
           size="small"
@@ -37,9 +50,11 @@ function MessageActionOfCopy({ value, onCopied }: { value: string; onCopied?: ()
 function MessageActionOfLike({
   active: propsActive,
   onClick,
+  className,
 }: {
   active?: boolean;
   onClick?: (active: boolean) => Promise<void>;
+  className?: string;
 }) {
   const { message } = App.useApp();
   const [active, setActive] = useState(propsActive || false);
@@ -60,7 +75,7 @@ function MessageActionOfLike({
         type="text"
         onClick={handleClick}
         size="small"
-        className="text-03"
+        className={classNames('text-03', className)}
         icon={active ? <LikeFilled /> : <LikeOutlined />}
       />
     </Tooltip>
@@ -70,9 +85,11 @@ function MessageActionOfLike({
 function MessageActionOfDislike({
   active: propsActive,
   onClick,
+  className,
 }: {
   active?: boolean;
   onClick?: (active: boolean) => Promise<void>;
+  className?: string;
 }) {
   const [active, setActive] = useState(propsActive || false);
   const { message } = App.useApp();
@@ -93,7 +110,7 @@ function MessageActionOfDislike({
         type="text"
         onClick={handleClick}
         size="small"
-        className="text-03"
+        className={classNames('text-03', className)}
         icon={active ? <DislikeFilled /> : <DislikeOutlined />}
       />
     </Tooltip>
@@ -103,9 +120,11 @@ function MessageActionOfDislike({
 function MessageActionOfLinkAndDislike({
   value: propsValue,
   onChange,
+  className,
 }: {
   value?: -1 | 0 | 1;
   onChange?: (value: -1 | 0 | 1) => void;
+  className?: string;
 }) {
   const [value, setValue] = useState<(-1 | 0 | 1) | undefined>(propsValue);
 
@@ -122,6 +141,7 @@ function MessageActionOfLinkAndDislike({
           await Promise.resolve(onChange?.(newValue));
           setValue(newValue);
         }}
+        className={className}
       />
       <MessageActionOfDislike
         active={value === -1}
@@ -130,6 +150,7 @@ function MessageActionOfLinkAndDislike({
           await Promise.resolve(onChange?.(newValue));
           setValue(newValue);
         }}
+        className={className}
       />
     </>
   );

package/src/voice/index.ts

@@ -1,33 +1,188 @@
-async function recordAudioOfPCM({ onAudio }: { onAudio: (data: ArrayBufferLike) => void }) {
-  // --- 初始化音频 ---
-  const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
-  const audioContext = new AudioContext({ sampleRate: 16000 });
-  const sourceNode = audioContext.createMediaStreamSource(micStream);
-  // ScriptProcessorNode（4096 是稳定 buffer）
-  const processorNode = audioContext.createScriptProcessor(4096, 1, 1);
-
-  processorNode.onaudioprocess = function (event) {
-    const float32Data = event.inputBuffer.getChannelData(0); // float32
-
-    // === 转成 Int16 PCM ===
-    const pcm16 = new Int16Array(float32Data.length);
-    for (let i = 0; i < float32Data.length; i++) {
-      const s = Math.max(-1, Math.min(1, float32Data[i]));
-      pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
-    }
+function getRecordAudioOfPCM() {
+  let processorNode: ScriptProcessorNode | null = null;
+  let sourceNode: MediaStreamAudioSourceNode | null = null;
+  let audioContext: AudioContext | null = null;
+  let micStream: MediaStream | null = null;
 
-    onAudio(pcm16.buffer);
-  };
+  let data: ArrayBufferLike[] = [];
+
+  async function start({
+    onAudio,
+    onError,
+  }: {
+    onAudio: (data: ArrayBufferLike) => void;
+    onError?: (error: Error) => void;
+  }): Promise<void> {
+    try {
+      // --- 初始化音频 ---
+      micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      audioContext = new AudioContext({ sampleRate: 16000 });
+      sourceNode = audioContext.createMediaStreamSource(micStream);
+      // ScriptProcessorNode（4096 是稳定 buffer）
+      processorNode = audioContext.createScriptProcessor(4096, 1, 1);
+
+      data = [];
+
+      processorNode.onaudioprocess = function (event) {
+        const float32Data = event.inputBuffer.getChannelData(0); // float32
+
+        // === 转成 Int16 PCM ===
+        const pcm16 = new Int16Array(float32Data.length);
+        for (let i = 0; i < float32Data.length; i++) {
+          const s = Math.max(-1, Math.min(1, float32Data[i]));
+          pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+        }
+
+        data.push(pcm16.buffer);
+
+        onAudio(pcm16.buffer);
+      };
 
-  sourceNode.connect(processorNode);
-  processorNode.connect(audioContext.destination);
+      sourceNode.connect(processorNode);
+      processorNode.connect(audioContext.destination);
+    } catch (err) {
+      if (err instanceof DOMException && err.name === 'NotAllowedError') {
+        onError?.(new Error('请允许麦克风权限'));
+      } else if (err instanceof DOMException && err.name === 'NotFoundError') {
+        onError?.(new Error('未找到麦克风设备'));
+      } else if (err instanceof DOMException && err.name === 'NotReadableError') {
+        onError?.(new Error('麦克风被其他应用占用'));
+      } else {
+        onError?.(new Error('启动录音失败'));
+      }
 
-  return () => {
+      throw err;
+    }
+  }
+
+  async function stop(): Promise<{ data: ArrayBufferLike[] }> {
     if (processorNode) processorNode.disconnect();
     if (sourceNode) sourceNode.disconnect();
     if (audioContext) audioContext.close();
     if (micStream) micStream.getTracks().forEach((track) => track.stop());
+
+    const result = data;
+    data = [];
+
+    return { data: result };
+  }
+
+  return {
+    start,
+    stop,
+  };
+}
+
+function getRecordAudioOfBlob() {
+  let mediaRecorder: MediaRecorder | null = null;
+  let micStream: MediaStream | null = null;
+  let chunks: Blob[] = [];
+
+  async function start({
+    onAudio,
+    onError,
+    mimeType = 'audio/webm',
+  }: {
+    onAudio?: (blob: Blob) => void;
+    onError?: (error: Error) => void;
+    mimeType?: string;
+  }): Promise<void> {
+    try {
+      // 获取麦克风权限
+      micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+      // 检查浏览器是否支持指定的 MIME 类型
+      let finalMimeType = mimeType;
+      if (!MediaRecorder.isTypeSupported(mimeType)) {
+        // 如果不支持，尝试使用默认类型
+        finalMimeType = '';
+        console.warn(`不支持的 MIME 类型: ${mimeType}，使用默认类型`);
+      }
+
+      // 创建 MediaRecorder 实例
+      mediaRecorder = new MediaRecorder(micStream, {
+        mimeType: finalMimeType || undefined,
+      });
+
+      chunks = [];
+
+      // 监听数据可用事件
+      mediaRecorder.ondataavailable = (event) => {
+        if (event.data && event.data.size > 0) {
+          chunks.push(event.data);
+          onAudio?.(event.data);
+        }
+      };
+
+      // 监听错误事件
+      mediaRecorder.onerror = () => {
+        const error = new Error('MediaRecorder 录音错误');
+        onError?.(error);
+      };
+
+      // 开始录音
+      mediaRecorder.start(100); // 每 100ms 触发一次 dataavailable 事件
+    } catch (err) {
+      if (err instanceof DOMException && err.name === 'NotAllowedError') {
+        onError?.(new Error('请允许麦克风权限'));
+      } else if (err instanceof DOMException && err.name === 'NotFoundError') {
+        onError?.(new Error('未找到麦克风设备'));
+      } else if (err instanceof DOMException && err.name === 'NotReadableError') {
+        onError?.(new Error('麦克风被其他应用占用'));
+      } else {
+        onError?.(new Error('启动录音失败'));
+      }
+
+      throw err;
+    }
+  }
+
+  async function stop(): Promise<{ data: Blob; base64: string }> {
+    return new Promise((resolve, reject) => {
+      if (!mediaRecorder) {
+        reject(new Error('MediaRecorder 未初始化'));
+        return;
+      }
+
+      function doStop() {
+        const blob = new Blob(chunks, { type: mediaRecorder?.mimeType || 'audio/webm' });
+        chunks = [];
+
+        // 停止所有轨道
+        if (micStream) {
+          micStream.getTracks().forEach((track) => track.stop());
+        }
+
+        // 将 Blob 转换为 base64
+        const reader = new FileReader();
+        reader.onloadend = () => {
+          const base64String = reader.result as string;
+          resolve({ data: blob, base64: base64String });
+        };
+        reader.onerror = () => {
+          reject(new Error('转换为 base64 失败'));
+        };
+        reader.readAsDataURL(blob);
+      }
+
+      // 监听停止事件
+      mediaRecorder.onstop = () => {
+        doStop();
+      };
+
+      // 如果正在录音，则停止
+      if (mediaRecorder.state === 'recording') {
+        mediaRecorder.stop();
+      } else {
+        doStop();
+      }
+    });
+  }
+
+  return {
+    start,
+    stop,
   };
 }
 
-export { recordAudioOfPCM };
+export { getRecordAudioOfBlob, getRecordAudioOfPCM };