@baishuyun/chat-sdk 0.1.6 → 0.1.8

package/src/hooks/use-voice-input.ts

@@ -0,0 +1,254 @@
+import { useCallback, useRef, useState } from 'react';
+
+export interface UseVoiceInputOptions {
+  /** ASR WebSocket 端点，如 ws://localhost:3001/web/api/asr */
+  endpoint?: string;
+  /** 收到识别文本 */
+  onText: (text: string) => void;
+  /** 识别完成 */
+  onDone: () => void;
+  /** 发生错误 */
+  onError: (msg: string) => void;
+}
+
+export interface UseVoiceInputReturn {
+  /** 是否正在录音（麦克风采集音频中） */
+  isRecording: boolean;
+  /** 是否正在连接（获取权限 / WebSocket 握手） */
+  isConnecting: boolean;
+  /** 是否正在识别（已停止录音，等待后端返回结果） */
+  isRecognizing: boolean;
+  /** 开始录音 */
+  start: () => Promise<void>;
+  /** 停止录音 */
+  stop: () => void;
+}
+
+/**
+ * 语音输入 Hook
+ *
+ * 使用浏览器原生 Web Audio API（AudioContext + ScriptProcessorNode）
+ * 直接录制 pcm_s16le 格式音频，通过 WebSocket 发送到后端
+ * 无第三方语音识别 SDK，只与 Node 后端通信
+ */
+export function useVoiceInput(options: UseVoiceInputOptions): UseVoiceInputReturn {
+  const { endpoint, onText, onDone, onError } = options;
+
+  const [isRecording, setIsRecording] = useState(false);
+  const [isConnecting, setIsConnecting] = useState(false);
+  const [isRecognizing, setIsRecognizing] = useState(false);
+
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const processorRef = useRef<ScriptProcessorNode | null>(null);
+  const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null);
+  const wsRef = useRef<WebSocket | null>(null);
+  const streamRef = useRef<MediaStream | null>(null);
+
+  const cleanup = useCallback(() => {
+    if (processorRef.current) {
+      try {
+        processorRef.current.disconnect();
+      } catch {
+        // ignore
+      }
+      processorRef.current = null;
+    }
+
+    if (sourceRef.current) {
+      try {
+        sourceRef.current.disconnect();
+      } catch {
+        // ignore
+      }
+      sourceRef.current = null;
+    }
+
+    if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
+      try {
+        audioContextRef.current.close();
+      } catch {
+        // ignore
+      }
+      audioContextRef.current = null;
+    }
+
+    if (wsRef.current) {
+      try {
+        wsRef.current.close();
+      } catch {
+        // ignore
+      }
+      wsRef.current = null;
+    }
+
+    if (streamRef.current) {
+      streamRef.current.getTracks().forEach((track) => track.stop());
+      streamRef.current = null;
+    }
+
+    setIsRecording(false);
+    setIsConnecting(false);
+    setIsRecognizing(false);
+  }, []);
+
+  const start = useCallback(async () => {
+    if (!endpoint) {
+      onError('ASR 服务未配置');
+      return;
+    }
+
+    // 清理之前的状态
+    cleanup();
+
+    setIsConnecting(true);
+
+    try {
+      // 1. 获取麦克风权限
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          sampleRate: 16000,
+          channelCount: 1,
+          echoCancellation: true,
+          noiseSuppression: true,
+        },
+      });
+      streamRef.current = stream;
+
+      // 2. 创建 AudioContext（16kHz 单声道，匹配火山引擎要求）
+      const audioContext = new AudioContext({ sampleRate: 16000 });
+      audioContextRef.current = audioContext;
+
+      const source = audioContext.createMediaStreamSource(stream);
+      sourceRef.current = source;
+
+      // 3. 创建 ScriptProcessorNode（bufferSize=2048，约 128ms @ 16kHz，符合火山引擎建议的 100-200ms）
+      const processor = audioContext.createScriptProcessor(2048, 1, 1);
+      processorRef.current = processor;
+
+      // 4. 创建 WebSocket 连接
+      const ws = new WebSocket(endpoint);
+      ws.binaryType = 'arraybuffer';
+      wsRef.current = ws;
+
+      let hasReceivedResult = false;
+
+      ws.onopen = () => {
+        // WebSocket 连接成功后，连接音频处理节点开始录音
+        source.connect(processor);
+        processor.connect(audioContext.destination);
+
+        processor.onaudioprocess = (e) => {
+          if (ws.readyState !== WebSocket.OPEN) return;
+
+          const inputData = e.inputBuffer.getChannelData(0); // Float32Array
+          const pcmData = float32ToInt16(inputData);
+          ws.send(pcmData.buffer);
+        };
+
+        setIsConnecting(false);
+        setIsRecording(true);
+      };
+
+      ws.onmessage = (event) => {
+        try {
+          const data = JSON.parse(event.data as string);
+          if (data.type === 'result' || data.type === 'final') {
+            hasReceivedResult = true;
+            onText(data.text || '');
+          }
+          if (data.type === 'done') {
+            setIsRecognizing(false);
+            onDone();
+          }
+          if (data.type === 'error') {
+            setIsRecognizing(false);
+            onError(data.message || '语音识别出错');
+          }
+        } catch {
+          // 忽略非 JSON 消息
+        }
+      };
+
+      ws.onerror = () => {
+        onError('语音识别连接失败');
+        cleanup();
+      };
+
+      ws.onclose = () => {
+        // 如果已经收到过结果但还没收到 done，可能是连接异常关闭
+        if (hasReceivedResult) {
+          setIsRecognizing(false);
+        }
+        setIsRecording(false);
+        setIsConnecting(false);
+      };
+    } catch (error) {
+      setIsConnecting(false);
+      const msg = error instanceof Error ? error.message : '录音启动失败';
+      if (msg.includes('Permission denied') || msg.includes('NotAllowedError')) {
+        onError('麦克风权限被拒绝');
+      } else if (msg.includes('NotFoundError')) {
+        onError('未找到麦克风设备');
+      } else {
+        onError(msg);
+      }
+    }
+  }, [endpoint, onText, onDone, onError, cleanup]);
+
+  const stop = useCallback(() => {
+    // 断开音频处理节点，停止录音
+    if (processorRef.current) {
+      processorRef.current.onaudioprocess = null;
+      try {
+        processorRef.current.disconnect();
+      } catch {
+        // ignore
+      }
+    }
+
+    if (sourceRef.current) {
+      try {
+        sourceRef.current.disconnect();
+      } catch {
+        // ignore
+      }
+    }
+
+    if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
+      try {
+        audioContextRef.current.close();
+      } catch {
+        // ignore
+      }
+    }
+
+    // 停止麦克风轨道
+    if (streamRef.current) {
+      streamRef.current.getTracks().forEach((track) => track.stop());
+      streamRef.current = null;
+    }
+
+    // 发送结束信号
+    if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+      wsRef.current.send('end');
+    }
+
+    // 停止录音，进入识别中状态
+    setIsRecording(false);
+    setIsRecognizing(true);
+  }, []);
+
+  return { isRecording, isConnecting, isRecognizing, start, stop };
+}
+
+/**
+ * Float32Array (-1.0 ~ 1.0) 转换为 Int16Array (pcm_s16le)
+ */
+function float32ToInt16(float32Array: Float32Array): Int16Array {
+  const int16Array = new Int16Array(float32Array.length);
+  for (let i = 0; i < float32Array.length; i++) {
+    const s = Math.max(-1, Math.min(1, float32Array[i]));
+    int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+  }
+  return int16Array;
+}

package/src/lib/utils.ts

@@ -419,17 +419,87 @@ export const formatJSONIfValid = (str: string): string | null => {
   }
 };
 
+const MIME_MAP: Record<string, string> = {
+  // 文档
+  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  xls: 'application/vnd.ms-excel',
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  pdf: 'application/pdf',
+  txt: 'text/plain',
+  // 图片
+  png: 'image/png',
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  gif: 'image/gif',
+  webp: 'image/webp',
+  svg: 'image/svg+xml',
+  // 音视频
+  mp4: 'video/mp4',
+  mp3: 'audio/mpeg',
+  // 压缩包
+  zip: 'application/zip',
+  rar: 'application/x-rar-compressed',
+  '7z': 'application/x-7z-compressed',
+  // 代码/数据
+  json: 'application/json',
+  js: 'application/javascript',
+  ts: 'application/typescript',
+  csv: 'text/csv',
+  xml: 'application/xml',
+  // 默认回退
+  default: 'application/octet-stream',
+};
+
+const getMimeType = (filename: string, serverType: string | null): string => {
+  // 如果服务器给了具体类型（不是 octet-stream），就信任它
+  if (serverType && serverType !== 'application/octet-stream') {
+    return serverType;
+  }
+
+  // 否则从扩展名推断
+  const ext = filename.split('.').pop()?.toLowerCase() || '';
+  return MIME_MAP[ext] || MIME_MAP.default;
+};
+
 /**
  * 从 URL 下载文件，返回 File 对象
  */
 export const downloadFile = async (url: string): Promise<File> => {
-  const response = await fetch(url);
+  const response = await fetch(url, {
+    credentials: 'include', // 必须携带，因为 Allow-Credentials: true
+    mode: 'cors',
+  });
+
   if (!response.ok) {
-    throw new Error(`Failed to download file from ${url}: ${response.status}`);
+    throw new Error(`Failed to download: ${response.status}`);
   }
+
   const blob = await response.blob();
-  const contentType = response.headers.get('content-type') || 'application/octet-stream';
-  const filename = new URL(url).pathname.split('/').pop() || 'attachment';
+
+  // 1. 从 Content-Disposition 解析文件名（优先）
+  const disposition = response.headers.get('content-disposition');
+  let filename = 'attachment';
+
+  if (disposition) {
+    // 尝试 filename*=UTF-8'' 格式（RFC 5987）
+    const utf8Match = disposition.match(/filename\*=UTF-8''([^;]+)/i);
+    // 尝试 filename="..." 或 filename=... 格式（需要解码 MinIO 的 URL 编码）
+    const asciiMatch = disposition.match(/filename=["']?([^"';]+)["']?/i);
+
+    if (utf8Match) {
+      filename = decodeURIComponent(utf8Match[1]);
+    } else if (asciiMatch) {
+      // MinIO 特殊处理：filename 可能是 URL 编码的
+      filename = decodeURIComponent(asciiMatch[1]);
+    }
+  }
+
+  // 2. 修正 MIME 类型（如果服务器返回的是 octet-stream）
+  let contentType = response.headers.get('content-type') || 'application/octet-stream';
+  if (contentType === 'application/octet-stream') {
+    contentType = getMimeType(filename, contentType);
+  }
+
   return new File([blob], filename, { type: contentType });
 };
 
@@ -468,12 +538,18 @@ export const urls2fileParts = async (attachmentsUrl: string[], uploadEndpoint: s
     return [];
   }
 
-  const results = await Promise.all(
-    attachmentsUrl.map(async (url) => {
-      const file = await downloadFile(url);
-      return uploadFile(file, uploadEndpoint);
-    })
-  );
+  let results: (Attachment | undefined)[];
+  try {
+    results = await Promise.all(
+      attachmentsUrl.map(async (url) => {
+        const file = await downloadFile(url);
+        return uploadFile(file, uploadEndpoint);
+      })
+    );
+  } catch (error) {
+    console.error('Error processing attachments:', error);
+    return [];
+  }
 
   return results
     .filter((attachment): attachment is Attachment => attachment !== undefined)

package/src/sdk.impl.tsx

@@ -133,16 +133,19 @@ export class ChatSDK implements IChatSDK {
     let fileParts: ChatMessage['parts'] = await urls2fileParts(attachmentsUrl || [], uploadEndpoint);
     this.Store.getState().clearGlobalFakeLoadingMessage();
 
+    const textParts = text.trim() ? [{ type: 'text' as const, text }] : [];
+
     const msg = fileParts.length > 0
       ? {
           role: 'user',
           parts: [
             ...fileParts,
-            { type: 'text' as const, text },
+            ...textParts,
           ],
         } as ChatMessage
       : { text };
 
+
     this._sendMessage?.(
       msg,
       {