@baishuyun/chat-backend 0.0.19 → 0.0.20

package/src/services/asr/asr-websocket.ts

@@ -0,0 +1,231 @@
+import WebSocket from 'ws';
+import type { IncomingMessage } from 'node:http';
+import config from 'config';
+import crypto from 'crypto';
+import {
+  createFullClientRequest,
+  createAudioOnlyRequest,
+  parseHeader,
+  readPayloadSize,
+  parseJsonPayload,
+  parseErrorPayload,
+  MSG_TYPE_FULL_SERVER_RESPONSE,
+  MSG_TYPE_ERROR_RESPONSE,
+} from './volc-protocol.js';
+import { logger } from '../../logger/index.js';
+import type { ASRConfig, ASRResponse } from './volc-protocol.js';
+
+/**
+ * 处理浏览器 ASR WebSocket 连接
+ * 作为浏览器与火山引擎 ASR 之间的双向代理
+ */
+export function handleASRConnection(clientWs: WebSocket, req: IncomingMessage): void {
+  const asrConfig = config.get<ASRConfig>('agent.asr');
+
+  if (!asrConfig?.appid || !asrConfig?.token || !asrConfig?.cluster) {
+    logger.error('ASR configuration missing: appid, token, cluster are required');
+    clientWs.close(1011, 'ASR config missing');
+    return;
+  }
+
+  logger.info('ASR client connected');
+
+  // 音频数据缓冲区（在火山连接建立前暂存）
+  const audioBuffer: Buffer[] = [];
+  let isVolcReady = false;
+  let isClosing = false;
+
+  // v3 接口地址：双向流式优化版
+  const volcUrl = `wss://${asrConfig.host}/api/v3/sauc/bigmodel_async`;
+
+  // 鉴权 Header（v3 大模型语音识别通过 HTTP Header 鉴权）
+  const connectId = crypto.randomUUID();
+  const requestHeaders: Record<string, string> = {
+    'X-Api-App-Key': asrConfig.appid,
+    'X-Api-Access-Key': asrConfig.token,
+    'X-Api-Resource-Id': asrConfig.cluster,
+    'X-Api-Connect-Id': connectId,
+  };
+
+  logger.info(
+    { url: volcUrl, headers: { ...requestHeaders, 'X-Api-Access-Key': '***' } },
+    'Connecting to Volc ASR v3'
+  );
+
+  const volcWs = new WebSocket(volcUrl, {
+    headers: requestHeaders,
+    perMessageDeflate: false,
+  });
+
+  volcWs.on('unexpected-response', (req, res) => {
+    let body = '';
+    res.on('data', (chunk) => { body += chunk; });
+    res.on('end', () => {
+      logger.error(
+        { statusCode: res.statusCode, statusMessage: res.statusMessage, body },
+        'Volc ASR unexpected response'
+      );
+    });
+  });
+
+  // 火山引擎连接建立后立即发送初始化请求并开始转发音频
+  volcWs.on('open', () => {
+    logger.info('Connected to Volc ASR v3, sending init request');
+    volcWs.send(createFullClientRequest());
+
+    // 火山引擎不返回初始化确认，直接开始等待音频
+    // 立即开始转发缓冲的音频数据
+    isVolcReady = true;
+    while (audioBuffer.length > 0) {
+      const buffered = audioBuffer.shift()!;
+      if (volcWs.readyState === WebSocket.OPEN) {
+        volcWs.send(createAudioOnlyRequest(buffered));
+      }
+    }
+  });
+
+  // 处理火山引擎返回的消息（二进制帧）
+  volcWs.on('message', (data: Buffer) => {
+    if (isClosing) return;
+
+    if (!Buffer.isBuffer(data) || data.length < 4) {
+      logger.warn({ len: data.length, type: typeof data }, 'Invalid ASR message');
+      return;
+    }
+
+    const { msgType, flags, bodyOffset } = parseHeader(data);
+
+    if (msgType === MSG_TYPE_FULL_SERVER_RESPONSE) {
+      // 根据 flags 判断是否有 sequence（flags bit0=1 表示有正 sequence）
+      const hasSequence = (flags & 0b0001) !== 0;
+      const payloadSizeOffset = hasSequence ? bodyOffset + 4 : bodyOffset;
+      const payloadOffset = payloadSizeOffset + 4;
+
+      if (data.length < payloadOffset) {
+        logger.warn({ len: data.length, payloadOffset, hex: data.toString('hex') }, 'ASR response too short');
+        return;
+      }
+
+      const payloadSize = readPayloadSize(data, payloadSizeOffset);
+
+      if (data.length < payloadOffset + payloadSize) {
+        logger.warn({ len: data.length, payloadSize, payloadOffset }, 'ASR response incomplete');
+        return;
+      }
+
+      const payload = data.slice(payloadOffset, payloadOffset + payloadSize);
+      const response = parseJsonPayload(payload);
+
+      if (!response) {
+        logger.warn({ payloadStr: payload.toString('utf-8').slice(0, 200) }, 'Failed to parse ASR JSON');
+        return;
+      }
+
+      const text = response.result?.text;
+      const utterances = response.result?.utterances;
+      const isDefinite = utterances?.some((u) => u.definite);
+
+      logger.info({ text, definite: isDefinite }, 'ASR result');
+
+      // 转发识别文本给浏览器
+      if (text !== undefined && clientWs.readyState === WebSocket.OPEN) {
+        clientWs.send(
+          JSON.stringify({
+            type: isDefinite ? 'final' : 'result',
+            text,
+          })
+        );
+      }
+
+      if (isDefinite && clientWs.readyState === WebSocket.OPEN) {
+        clientWs.send(JSON.stringify({ type: 'done' }));
+      }
+    } else if (msgType === MSG_TYPE_ERROR_RESPONSE) {
+      const error = parseErrorPayload(data, bodyOffset);
+      logger.error({ error }, 'ASR error response');
+      if (clientWs.readyState === WebSocket.OPEN) {
+        clientWs.send(
+          JSON.stringify({
+            type: 'error',
+            message: error?.message || `ASR error code: ${error?.code}`,
+          })
+        );
+      }
+    }
+  });
+
+  // 处理浏览器发来的消息
+  clientWs.on('message', (data: Buffer | ArrayBuffer | string) => {
+    if (isClosing) return;
+
+    // 结束信号
+    if (typeof data === 'string' && data === 'end') {
+      if (volcWs.readyState === WebSocket.OPEN) {
+        volcWs.send(createAudioOnlyRequest(Buffer.alloc(0), true));
+      }
+      return;
+    }
+
+    // 将数据转为 Buffer
+    let audioData: Buffer;
+    if (Buffer.isBuffer(data)) {
+      audioData = data;
+    } else if (data instanceof ArrayBuffer) {
+      audioData = Buffer.from(data);
+    } else {
+      return;
+    }
+
+    if (audioData.length === 0) return;
+
+    // 火山连接就绪则直接转发，否则缓冲
+    if (isVolcReady && volcWs.readyState === WebSocket.OPEN) {
+      volcWs.send(createAudioOnlyRequest(audioData));
+    } else {
+      audioBuffer.push(audioData);
+    }
+  });
+
+  // 浏览器关闭
+  clientWs.on('close', () => {
+    logger.info('ASR client disconnected');
+    isClosing = true;
+    if (volcWs.readyState === WebSocket.OPEN) {
+      volcWs.send(createAudioOnlyRequest(Buffer.alloc(0), true));
+      setTimeout(() => volcWs.close(), 500);
+    } else {
+      volcWs.close();
+    }
+  });
+
+  // 火山引擎关闭
+  volcWs.on('close', () => {
+    logger.info('Volc ASR connection closed');
+    isClosing = true;
+    if (clientWs.readyState === WebSocket.OPEN) {
+      clientWs.close();
+    }
+  });
+
+  // 浏览器端错误
+  clientWs.on('error', (err) => {
+    logger.error({ err }, 'Client WebSocket error');
+    isClosing = true;
+    volcWs.close();
+  });
+
+  // 火山引擎错误
+  volcWs.on('error', (err) => {
+    logger.error({ err }, 'Volc ASR WebSocket error');
+    isClosing = true;
+    if (clientWs.readyState === WebSocket.OPEN) {
+      clientWs.send(
+        JSON.stringify({
+          type: 'error',
+          message: 'ASR service connection error',
+        })
+      );
+      clientWs.close(1011, 'ASR error');
+    }
+  });
+}

package/src/services/asr/volc-protocol.ts

@@ -0,0 +1,220 @@
+/**
+ * 火山引擎 ASR v3 大模型语音识别 WebSocket 二进制协议
+ * 协议文档: https://www.volcengine.com/docs/6561/1354869
+ *
+ * v3 协议要点：
+ * - 4 字节 header
+ * - payloadSize 为 4 字节 uint32 大端
+ * - FullServerResponse 包含 4 字节 sequence 字段
+ * - ErrorResponse 包含 errorCode(4B) + errorMsgSize(4B) + errorMsg
+ * - 鉴权在 HTTP Header 中，不在 payload
+ */
+
+const PROTOCOL_VERSION = 0x01;
+const HEADER_SIZE_VALUE = 0x01; // actual header size = 1 * 4 = 4 bytes
+
+/** 消息类型 */
+export const MSG_TYPE_FULL_CLIENT_REQUEST = 0x01;
+export const MSG_TYPE_AUDIO_ONLY_REQUEST = 0x02;
+export const MSG_TYPE_FULL_SERVER_RESPONSE = 0x09;
+export const MSG_TYPE_ERROR_RESPONSE = 0x0f;
+
+/** 序列化方式 */
+const SERIALIZATION_JSON = 0x01;
+const SERIALIZATION_NONE = 0x00;
+
+/** 压缩方式 */
+const COMPRESSION_NONE = 0x00;
+
+/**
+ * 构建 4 字节协议头
+ *
+ * byte0: version(4bit) | header_size(4bit)
+ * byte1: msg_type(4bit) | flags(4bit)
+ * byte2: serialization(4bit) | compression(4bit)
+ * byte3: reserved
+ */
+function build4ByteHeader(
+  msgType: number,
+  flags: number,
+  serialization: number
+): Buffer {
+  const header = Buffer.alloc(4);
+  header[0] = (PROTOCOL_VERSION << 4) | HEADER_SIZE_VALUE;
+  header[1] = (msgType << 4) | (flags & 0x0f);
+  header[2] = (serialization << 4) | COMPRESSION_NONE;
+  header[3] = 0x00;
+  return header;
+}
+
+/** 构建完整消息：header + payloadSize(4B) + payload */
+function buildMessage(
+  msgType: number,
+  flags: number,
+  serialization: number,
+  payload: Buffer
+): Buffer {
+  const header = build4ByteHeader(msgType, flags, serialization);
+  const payloadSize = Buffer.alloc(4);
+  payloadSize.writeUInt32BE(payload.length, 0);
+  return Buffer.concat([header, payloadSize, payload]);
+}
+
+/** 解析后的协议头信息 */
+export interface ParsedMessage {
+  msgType: number;
+  flags: number;
+  /** header 之后的偏移量（header 4B + 扩展字段） */
+  bodyOffset: number;
+}
+
+/**
+ * 解析 4 字节协议头
+ */
+export function parseHeader(buffer: Buffer): ParsedMessage {
+  const version = buffer[0] >> 4;
+  const headerSizeValue = buffer[0] & 0x0f;
+  const headerSize = headerSizeValue * 4;
+  const msgType = buffer[1] >> 4;
+  const flags = buffer[1] & 0x0f;
+  const serialization = buffer[2] >> 4;
+  const compression = buffer[2] & 0x0f;
+
+  return {
+    msgType,
+    flags,
+    bodyOffset: headerSize,
+  };
+}
+
+/**
+ * 读取 payload size（4 字节 uint32 大端）
+ */
+export function readPayloadSize(buffer: Buffer, offset: number): number {
+  return buffer.readUInt32BE(offset);
+}
+
+/** ASR 配置（鉴权信息在 HTTP Header 中，不在 payload） */
+export interface ASRConfig {
+  appid: string;
+  token: string;
+  secretKey: string;
+  cluster: string;
+  host: string;
+}
+
+/**
+ * 创建 FullClientRequest（初始化请求）
+ *
+ * Payload JSON 格式：
+ * {
+ *   user: { uid: string },
+ *   audio: { format, rate, bits, channel, codec, language },
+ *   request: { model_name, enable_itn, enable_punc, ... }
+ * }
+ */
+export function createFullClientRequest(): Buffer {
+  const payload = Buffer.from(
+    JSON.stringify({
+      user: {
+        uid: '0',
+      },
+      audio: {
+        format: 'pcm',
+        rate: 16000,
+        bits: 16,
+        channel: 1,
+        codec: 'raw',
+        language: 'zh-CN',
+      },
+      request: {
+        model_name: 'bigmodel',
+        enable_itn: true,
+        enable_punc: true,
+        enable_ddc: false,
+      },
+    }),
+    'utf-8'
+  );
+
+  return buildMessage(
+    MSG_TYPE_FULL_CLIENT_REQUEST,
+    0b0000,
+    SERIALIZATION_JSON,
+    payload
+  );
+}
+
+/**
+ * 创建 AudioOnlyRequest（音频数据请求）
+ *
+ * flags:
+ * - 0b0000: 普通音频包
+ * - 0b0010: 最后一包音频（负包）
+ */
+export function createAudioOnlyRequest(
+  audioData: Buffer,
+  isLast: boolean = false
+): Buffer {
+  const flags = isLast ? 0b0010 : 0b0000;
+  return buildMessage(
+    MSG_TYPE_AUDIO_ONLY_REQUEST,
+    flags,
+    SERIALIZATION_NONE,
+    audioData
+  );
+}
+
+/** 火山引擎响应结果 */
+export interface ASRResponse {
+  result?: {
+    text?: string;
+    utterances?: Array<{
+      text: string;
+      definite: boolean;
+      start_time: number;
+      end_time: number;
+    }>;
+    [key: string]: unknown;
+  };
+  audio_info?: {
+    duration: number;
+  };
+  [key: string]: unknown;
+}
+
+/**
+ * 解析 JSON payload
+ */
+export function parseJsonPayload(payloadBuffer: Buffer): ASRResponse | null {
+  try {
+    return JSON.parse(payloadBuffer.toString('utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * 解析 ErrorResponse
+ *
+ * 格式：header(4B) + errorCode(4B) + errorMsgSize(4B) + errorMsg
+ */
+export function parseErrorPayload(buffer: Buffer, bodyOffset: number): { code: number; message: string } | null {
+  const errorCodeOffset = bodyOffset;
+  const errorMsgSizeOffset = errorCodeOffset + 4;
+  const errorMsgOffset = errorMsgSizeOffset + 4;
+
+  if (buffer.length < errorMsgOffset) {
+    return null;
+  }
+
+  const code = buffer.readUInt32BE(errorCodeOffset);
+  const msgSize = buffer.readUInt32BE(errorMsgSizeOffset);
+
+  if (buffer.length < errorMsgOffset + msgSize) {
+    return null;
+  }
+
+  const message = buffer.slice(errorMsgOffset, errorMsgOffset + msgSize).toString('utf-8');
+  return { code, message };
+}

package/src/utils/createFakeUIMessageStreamResponse.ts

@@ -0,0 +1,27 @@
+import { createUIMessageStream, createUIMessageStreamResponse, generateId } from 'ai';
+import { createSpecialPartMeta } from './createSpecialPartMeta.js';
+
+/**
+ * 创建一个与 `streamText().toUIMessageStreamResponse()` 格式一致的假流式响应，
+ * 用于在无法调用真实模型时向客户端推送一条指定的文本消息。
+ */
+export const createFakeUIMessageStreamResponse = (text: string): Response => {
+  const stream = createUIMessageStream({
+    execute: ({ writer }) => {
+      const messageId = generateId();
+      const textId = generateId();
+      writer.write({ type: 'start', messageId });
+      writer.write({ type: 'text-start', id: textId });
+      writer.write({
+        type: 'text-delta',
+        id: textId,
+        delta: text,
+        providerMetadata: createSpecialPartMeta('error-bot-id-required'),
+      });
+      writer.write({ type: 'text-end', id: textId });
+      writer.write({ type: 'finish', finishReason: 'stop' });
+    },
+  });
+
+  return createUIMessageStreamResponse({ stream });
+};

package/src/utils/createJsonStreamTransformer.ts

@@ -31,6 +31,8 @@ export interface JsonStreamTransformerOptions<T> {
   onErrorChunk?: (chunk: LanguageModelV2StreamPart) => void;
   /** JSON 解析出错时的回调，返回要展示的错误消息（可选） */
   onParseError?: (error: any) => string;
+
+  bypassParseError?: boolean; // 是否完全绕过解析错误处理，直接透传原始 chunk（不推荐，除非特殊场景）
 }
 
 /**
@@ -75,6 +77,7 @@ class JsonStreamProcessor<T> {
   private parseCompleted: Promise<void> | null = null;
   private resolveParseCompleted: (() => void) | null = null;
   private terminated = false;
+  private passthroughMode = false; // 当启用且遇到非 JSON chunk 时，切换到透传模式
 
   constructor(options: JsonStreamTransformerOptions<T>) {
     this.options = options;
@@ -96,6 +99,11 @@ class JsonStreamProcessor<T> {
       return;
     }
 
+    if (this.passthroughMode) {
+      controller.enqueue(chunk);
+      return;
+    }
+
     try {
       // 1) error chunk → 通知下游后终止
       if (chunk.type === 'error') {
@@ -117,6 +125,11 @@ class JsonStreamProcessor<T> {
         this.parser.write(stripCodeFence(chunk.delta));
       }
     } catch (error) {
+      if (this.options.bypassParseError) {
+        this.passthroughMode = true;
+        controller.enqueue(chunk);
+        return;
+      }
       this.enqueueError(controller, error);
     }
   }
@@ -169,6 +182,11 @@ class JsonStreamProcessor<T> {
     };
 
     this.parser.onError = (err: any) => {
+      if (this.options.bypassParseError) {
+        this.passthroughMode = true;
+        return;
+      }
+
       if (!this.terminated) {
         const msg = this.options.onParseError
           ? this.options.onParseError(err)
@@ -180,8 +198,9 @@ class JsonStreamProcessor<T> {
     };
 
     this.parser.onEnd = () => {
-      logger.debug('Parsing completed');
-      enqueue(' ', JSON.stringify({ appendConfirm: 'save-fields' }));
+      if (!this.options.bypassParseError) {
+        enqueue(' ', JSON.stringify({ appendConfirm: 'save-fields' }));
+      }
       this.completeParsing();
     };
   }

package/src/utils/createSpecialPartMeta.ts

@@ -0,0 +1,7 @@
+import type { SpecialPart } from '@baishuyun/types';
+
+export const createSpecialPartMeta = (type: SpecialPart, content?: string) => {
+  return {
+    raw: { data: { specialPart: true, type: 'error-bot-id-required' } },
+  };
+};