@baishuyun/chat-backend 0.0.19 → 0.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/config/default.ts +8 -0
  3. package/dist/config/default.js +12 -5
  4. package/dist/src/app/main.js +9 -0
  5. package/dist/src/config/hono.config.js +2 -1
  6. package/dist/src/const/error_code.js +3 -0
  7. package/dist/src/controllers/agent/bots.controller.js +1 -1
  8. package/dist/src/controllers/common/connect.controll.js +4 -2
  9. package/dist/src/controllers/common/model.js +2 -0
  10. package/dist/src/controllers/common/transformer-factory/agent-calling-listener.js +44 -0
  11. package/dist/src/controllers/form/attachment-upload.controller.js +15 -15
  12. package/dist/src/controllers/form/build/build.controller.js +3 -2
  13. package/dist/src/controllers/form/conversation/clear.controller.js +6 -6
  14. package/dist/src/controllers/form/fill/batch-fill.controller.js +3 -4
  15. package/dist/src/controllers/form/fill/createBatchFillingTransformStream.js +1 -17
  16. package/dist/src/controllers/form/fill/fill.controller.js +2 -1
  17. package/dist/src/controllers/form/fill/utils.js +16 -0
  18. package/dist/src/controllers/report/query/query.controller.js +1 -1
  19. package/dist/src/middleware/botRouter.js +34 -0
  20. package/dist/src/routes/common/common.route.js +2 -1
  21. package/dist/src/services/asr/asr-websocket.js +183 -0
  22. package/dist/src/services/asr/volc-protocol.js +143 -0
  23. package/dist/src/services/fetchCozeInfo.js +1 -1
  24. package/dist/src/utils/createFakeUIMessageStreamResponse.js +25 -0
  25. package/dist/src/utils/createJsonStreamTransformer.js +17 -2
  26. package/dist/src/utils/createSpecialPartMeta.js +5 -0
  27. package/package.json +7 -5
  28. package/src/app/main.ts +12 -0
  29. package/src/config/hono.config.ts +2 -1
  30. package/src/const/error_code.ts +3 -0
  31. package/src/controllers/common/connect.controll.ts +5 -2
  32. package/src/controllers/common/model.ts +2 -0
  33. package/src/controllers/common/transformer-factory/agent-calling-listener.ts +69 -0
  34. package/src/controllers/form/build/build.controller.ts +5 -2
  35. package/src/controllers/form/fill/batch-fill.controller.ts +3 -5
  36. package/src/controllers/form/fill/createBatchFillingTransformStream.ts +1 -20
  37. package/src/controllers/form/fill/fill.controller.ts +2 -0
  38. package/src/controllers/form/fill/utils.ts +18 -0
  39. package/src/middleware/botRouter.ts +41 -0
  40. package/src/routes/common/common.route.ts +2 -1
  41. package/src/services/asr/asr-websocket.ts +231 -0
  42. package/src/services/asr/volc-protocol.ts +220 -0
  43. package/src/utils/createFakeUIMessageStreamResponse.ts +27 -0
  44. package/src/utils/createJsonStreamTransformer.ts +21 -2
  45. package/src/utils/createSpecialPartMeta.ts +7 -0
package/CHANGELOG.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # @baishuyun/chat-backend
2
2
 
3
+ ## 0.0.20
4
+
5
+ ### Patch Changes
6
+
7
+ - Updated dependencies
8
+ - @baishuyun/coze-provider@1.0.0
9
+ - @baishuyun/agents@1.0.0
10
+ - @baishuyun/types@2.0.0
11
+
3
12
  ## 0.0.19
4
13
 
5
14
  ### Patch Changes
package/config/default.ts CHANGED
@@ -49,5 +49,13 @@ export default {
49
49
  apiKey: process.env.BOT_API_KEY, // load from env
50
50
  },
51
51
  },
52
+
53
+ asr: {
54
+ appid: process.env.VOLC_APPID || '',
55
+ token: process.env.VOLC_TOKEN || '',
56
+ secretKey: process.env.VOLC_SECRET_KEY || '',
57
+ cluster: process.env.VOLC_CLUSTER || '',
58
+ host: process.env.VOLC_ASR_HOST || 'openspeech.bytedance.com',
59
+ },
52
60
  },
53
61
  };
@@ -14,22 +14,22 @@ export default {
14
14
  },
15
15
  agent: {
16
16
  host: process.env.AGENT_HOST || '47.99.202.157',
17
- apiAuthKey: process.env.COZE_API_KEY,
17
+ apiAuthKey: process.env.BOT_API_KEY,
18
18
  userId: process.env.BUILTIN_COZE_USER_ID,
19
19
  common: {
20
- baseUrl: `http://${process.env.AGENT_HOST}/v3/`,
20
+ baseUrl: `https://${process.env.AGENT_HOST}/v3/`,
21
21
  apiKey: process.env.BOT_API_KEY, // load from env
22
22
  },
23
23
  form: {
24
24
  build: {
25
25
  botId: process.env.BUILD_BOT_ID || '7579927677256073216',
26
- baseUrl: `http://${process.env.AGENT_HOST}/v3/`,
26
+ baseUrl: `https://${process.env.AGENT_HOST}/v3/`,
27
27
  apiKey: process.env.BOT_API_KEY, // load from env
28
28
  intentBotId: process.env.FORM_BUILD_INTENT_BOT_ID || '7615799745042186240',
29
29
  },
30
30
  fill: {
31
31
  botId: process.env.FILL_BOT_ID || '7586483957357608960',
32
- baseUrl: `http://${process.env.AGENT_HOST}/v3/`,
32
+ baseUrl: `https://${process.env.AGENT_HOST}/v3/`,
33
33
  apiKey: process.env.BOT_API_KEY, // load from env
34
34
  ocrApiKey: process.env.OCR_API_KEY || '', // load from env
35
35
  },
@@ -37,9 +37,16 @@ export default {
37
37
  report: {
38
38
  query: {
39
39
  botId: process.env.QUERY_BOT_ID || '7595888372090929152',
40
- baseUrl: `http://${process.env.AGENT_HOST}/v3/`,
40
+ baseUrl: `https://${process.env.AGENT_HOST}/v3/`,
41
41
  apiKey: process.env.BOT_API_KEY, // load from env
42
42
  },
43
43
  },
44
+ asr: {
45
+ appid: process.env.VOLC_APPID || '',
46
+ token: process.env.VOLC_TOKEN || '',
47
+ secretKey: process.env.VOLC_SECRET_KEY || '',
48
+ cluster: process.env.VOLC_CLUSTER || '',
49
+ host: process.env.VOLC_ASR_HOST || 'openspeech.bytedance.com',
50
+ },
44
51
  },
45
52
  };
@@ -22,3 +22,12 @@ app.get('/web/api/health', (c) => {
22
22
  service: 'hono-app',
23
23
  });
24
24
  });
25
+ // 挂载 ASR WebSocket Server
26
+ import { WebSocketServer } from 'ws';
27
+ import { server } from '../config/hono.config.js';
28
+ import { handleASRConnection } from '../services/asr/asr-websocket.js';
29
+ const wss = new WebSocketServer({
30
+ server: server,
31
+ path: '/web/api/asr',
32
+ });
33
+ wss.on('connection', handleASRConnection);
@@ -36,11 +36,12 @@ app.onError((err, c) => {
36
36
  // 返回统一的 JSON 错误响应
37
37
  return c.json({ message: 'Internal Server Error' }, 500);
38
38
  });
39
- serve({
39
+ const server = serve({
40
40
  fetch: app.fetch,
41
41
  hostname: config.get('app.host') || '',
42
42
  port: config.get('app.port') || 3001,
43
43
  }, (info) => {
44
44
  logger.info(`Server is running on http://localhost:${info.port}`);
45
45
  });
46
+ export { server };
46
47
  export default app;
@@ -0,0 +1,3 @@
1
+ export const ERROR_CODE = {
2
+ BOT_ID_MISSING: 10001,
3
+ };
@@ -3,7 +3,7 @@ import { logger } from '../../logger/index.js';
3
3
  import { safeJsonParser } from '../../utils/safeJsonParser.js';
4
4
  export const listBots = async (c) => {
5
5
  const agentHost = config.get('agent.host');
6
- const apiBots = `http://${agentHost}/v1/bots`;
6
+ const apiBots = `https://${agentHost}/v1/bots`;
7
7
  const cozeInfo = c.get('X-Coze-Info');
8
8
  logger.debug(`Fetching bots from ${apiBots} with cozeToken: ${cozeInfo.cozeToken}`);
9
9
  const result = await fetch(apiBots, {
@@ -1,5 +1,6 @@
1
1
  import { convertToModelMessages, streamText } from 'ai';
2
2
  import { createBaseModel } from './model.js';
3
+ import { logger } from '../../logger/index.js';
3
4
  export const connectToAgent = async (c) => {
4
5
  let requestBody;
5
6
  try {
@@ -12,13 +13,14 @@ export const connectToAgent = async (c) => {
12
13
  const cozeInfo = c.get('X-Coze-Info');
13
14
  const messages = requestBody.messages;
14
15
  const botId = c.req.header('X-Bot-Id') || '';
16
+ logger.debug('enter common controller');
15
17
  const stream = streamText({
16
- model: createBaseModel(botId, cozeInfo.cozeToken),
18
+ model: createBaseModel(botId, cozeInfo?.cozeToken),
17
19
  messages: convertToModelMessages(messages),
18
20
  includeRawChunks: true,
19
21
  headers: {
20
22
  'x-user-var': requestBody.userVar,
21
- 'x-user-id': cozeInfo.userId,
23
+ 'x-user-id': cozeInfo?.userId,
22
24
  },
23
25
  });
24
26
  return stream.toUIMessageStreamResponse();
@@ -1,10 +1,12 @@
1
1
  import { createCoze } from '@baishuyun/coze-provider';
2
2
  import config from 'config';
3
+ import { createAgentCallingListener } from './transformer-factory/agent-calling-listener.js';
3
4
  export const createBaseModel = (botId, token) => {
4
5
  const coze = createCoze({
5
6
  apiKey: token || config.get('agent.common.apiKey'),
6
7
  baseURL: config.get('agent.common.baseUrl'),
7
8
  botId: botId,
9
+ extraStreamTransformers: [createAgentCallingListener],
8
10
  });
9
11
  return coze.chat('chat');
10
12
  };
@@ -0,0 +1,44 @@
1
+ import { createJsonStreamTransformer, } from '../../../utils/createJsonStreamTransformer.js';
2
+ import { JSONParser } from '@streamparser/json';
3
+ import { isTargetElement } from '../../report/query/utils.js';
4
+ function createJSONParser() {
5
+ return new JSONParser({
6
+ stringBufferSize: undefined,
7
+ numberBufferSize: undefined,
8
+ separator: '',
9
+ paths: ['$.type', '$.prompt', '$.agent'],
10
+ keepStack: true,
11
+ });
12
+ }
13
+ export const fieldHandlers = [
14
+ {
15
+ path: '$.type',
16
+ handler: (ctx, value) => ctx.setPartialResult({ type: value }),
17
+ },
18
+ {
19
+ path: '$.prompt',
20
+ handler: (ctx, value) => ctx.setPartialResult({ prompt: value }),
21
+ },
22
+ {
23
+ path: '$.agent',
24
+ handler: (ctx, value) => ctx.setPartialResult({ agent: value }),
25
+ },
26
+ ];
27
+ function handleParsedValue(ctx) {
28
+ const { parsedInfo, getResult, currentChunkId, deltaChunkEnqueuer: enqueueTextDelta, ctrl } = ctx;
29
+ const { value } = parsedInfo;
30
+ for (const { path, handler } of fieldHandlers) {
31
+ if (isTargetElement(path, parsedInfo)) {
32
+ handler(ctx, value);
33
+ break;
34
+ }
35
+ }
36
+ enqueueTextDelta(`${JSON.stringify(value)},`, { type: 'agent-calling', result: JSON.stringify(getResult()) }, currentChunkId, true);
37
+ }
38
+ export const createAgentCallingListener = () => {
39
+ return createJsonStreamTransformer({
40
+ bypassParseError: true,
41
+ createJSONParser,
42
+ onParseValue: handleParsedValue,
43
+ });
44
+ };
@@ -1,22 +1,22 @@
1
- import {} from "hono";
2
- import config from "config";
3
- import { logger } from "../../logger/index.js";
4
- import { parseImg } from "./fill/utils.js";
1
+ import {} from 'hono';
2
+ import config from 'config';
3
+ import { logger } from '../../logger/index.js';
4
+ import { parseImg } from './fill/utils.js';
5
5
  export const uploadAttachment = async (c) => {
6
6
  const formData = await c.req.formData();
7
- const file = formData.get("file"); // as FormData | null;
7
+ const file = formData.get('file'); // as FormData | null;
8
8
  // 校验文件是否存在
9
9
  if (!file || !(file instanceof Blob)) {
10
- return c.json({ error: "请上传有效的文件" }, 400);
10
+ return c.json({ error: '请上传有效的文件' }, 400);
11
11
  }
12
12
  const forwardFormData = new FormData();
13
- forwardFormData.append("file", file);
14
- const apiKey = config.get("agent.apiAuthKey");
15
- const host = config.get("agent.host");
16
- const api = `http://${host}/v1/files/upload`;
13
+ forwardFormData.append('file', file);
14
+ const apiKey = config.get('agent.apiAuthKey');
15
+ const host = config.get('agent.host');
16
+ const api = `https://${host}/v1/files/upload`;
17
17
  logger.debug(c.body);
18
18
  const response = await fetch(api, {
19
- method: "POST",
19
+ method: 'POST',
20
20
  body: formData,
21
21
  headers: {
22
22
  // Add any auth headers
@@ -24,9 +24,7 @@ export const uploadAttachment = async (c) => {
24
24
  },
25
25
  });
26
26
  if (!response.ok) {
27
- const errorData = await response
28
- .json()
29
- .catch(() => ({ message: "上传失败" }));
27
+ const errorData = await response.json().catch(() => ({ message: '上传失败' }));
30
28
  return c.json({
31
29
  error: `目标接口返回错误: ${errorData.message}`,
32
30
  status: response.status,
@@ -35,8 +33,10 @@ export const uploadAttachment = async (c) => {
35
33
  const data = await response.json();
36
34
  const { url, uri, ...rest } = data?.data || {};
37
35
  const orcResult = await parseImg(file);
36
+ // replace file url protol to https
37
+ const secureUrl = url.replace('http://', 'https://');
38
38
  return c.json({
39
- url,
39
+ url: secureUrl,
40
40
  name: uri,
41
41
  contentType: file.type,
42
42
  parsedData: orcResult,
@@ -37,13 +37,14 @@ export const buildForm = async (c) => {
37
37
  }
38
38
  return {
39
39
  type: "text",
40
- text: `【确认搭建:${formName}】 ${p.text}`
40
+ text: `【确认搭建:${formName}】 ${p.text}`
41
41
  };
42
42
  });
43
43
  }
44
+ const lastSecondMsg = allMsg.length > 1 ? allMsg[allMsg.length - 2] : null;
44
45
  const stream = streamText({
45
46
  model,
46
- messages: convertToModelMessages([lastUserMsg]),
47
+ messages: convertToModelMessages([lastSecondMsg, lastUserMsg].filter(Boolean)),
47
48
  includeRawChunks: true,
48
49
  headers: {
49
50
  "x-user-stage": intent,
@@ -1,18 +1,18 @@
1
- import config from "config";
1
+ import config from 'config';
2
2
  export const clearConversation = async (c) => {
3
3
  let requestBody;
4
- const agentHost = config.get("agent.host");
5
- const baseUrl = `http://${agentHost}/v1`;
6
- const apiKey = config.get("agent.apiAuthKey");
4
+ const agentHost = config.get('agent.host');
5
+ const baseUrl = `https://${agentHost}/v1`;
6
+ const apiKey = config.get('agent.apiAuthKey');
7
7
  try {
8
8
  const json = await c.req.json();
9
9
  requestBody = json;
10
10
  }
11
11
  catch (_) {
12
- return c.json({ error: "Invalid JSON" }, 400);
12
+ return c.json({ error: 'Invalid JSON' }, 400);
13
13
  }
14
14
  const result = await fetch(`${baseUrl}/conversations/${requestBody.conversationId}/clear`, {
15
- method: "POST",
15
+ method: 'POST',
16
16
  headers: {
17
17
  // Add any auth headers
18
18
  Authorization: `Bearer ${apiKey}`,
@@ -2,7 +2,7 @@ import {} from 'hono';
2
2
  import { convertToModelMessages, streamText, createUIMessageStream, createUIMessageStreamResponse, generateId, } from 'ai';
3
3
  import { logger } from '../../../logger/index.js';
4
4
  import { createBatchFillingModel } from './model.js';
5
- import { mode2part } from './utils.js';
5
+ import { extraGeneratePromptFromRandomContent, mode2part } from './utils.js';
6
6
  const getModelMessagesFromUserMessages = ({ continueMessageId, currentMsg, formStructure, mode, messages, }) => {
7
7
  // init fill
8
8
  if (continueMessageId && currentMsg) {
@@ -12,7 +12,7 @@ const getModelMessagesFromUserMessages = ({ continueMessageId, currentMsg, formS
12
12
  parts: [
13
13
  {
14
14
  type: 'text',
15
- text: `formStructure: ${JSON.stringify(formStructure)}`,
15
+ text: `\n\n${extraGeneratePromptFromRandomContent()}\n\nformStructure: ${JSON.stringify(formStructure)}\n\n`,
16
16
  },
17
17
  mode2part(mode),
18
18
  {
@@ -30,7 +30,7 @@ const getModelMessagesFromUserMessages = ({ continueMessageId, currentMsg, formS
30
30
  };
31
31
  lastUserMsg.parts.push({
32
32
  type: 'text',
33
- text: `formStructure: ${JSON.stringify(formStructure)}`,
33
+ text: `\n\n${extraGeneratePromptFromRandomContent()}\n\nformStructure: ${JSON.stringify(formStructure)}\n\n`,
34
34
  });
35
35
  lastUserMsg.parts.push(mode2part(mode));
36
36
  return convertToModelMessages([lastUserMsg]);
@@ -118,7 +118,6 @@ async function handleContinueGeneration(c, { messages, continueMessageId, mode,
118
118
  mode,
119
119
  formStructure,
120
120
  });
121
- logger.debug(`continue modelMessages ${JSON.stringify(modelMessages)}`);
122
121
  const result = streamText({
123
122
  model: createBatchFillingModel(),
124
123
  messages: modelMessages,
@@ -85,23 +85,7 @@ export const createBatchFillingResultTransformer = (enableJsonParser) => {
85
85
  controller.terminate(); // 信号通知下游可读流已关闭
86
86
  return;
87
87
  }
88
- const subFormField = getLastSubFormField(parsedInfo);
89
- if (subFormField) {
90
- // enqueueTextDelta(
91
- // JSON.stringify(value),
92
- // {
93
- // type: 'mcp-fields-json',
94
- // field: subFormField.value,
95
- // },
96
- // id,
97
- // true
98
- // );
99
- return;
100
- }
101
- if (value.fieldType === 'subform') {
102
- return;
103
- }
104
- // logger.debug(`id in onValue: ${id}`);
88
+ // logger.debug(`onValue: ${JSON.stringify(value, null, 2)}`);
105
89
  enqueueTextDelta(`${JSON.stringify(value)},`, {
106
90
  type: 'mcp-fields-json',
107
91
  field: value,
@@ -22,6 +22,7 @@ export const fillForm = async (c) => {
22
22
  const mode = requestBody.mode;
23
23
  const num = requestBody.count;
24
24
  const messages = requestBody.messages;
25
+ logger.debug('enter fillForm controller');
25
26
  // 如果是批量填写,调用批量填写接口
26
27
  if (mode === 'batch') {
27
28
  return batchFillForm(c);
@@ -38,7 +39,7 @@ export const fillForm = async (c) => {
38
39
  ...flattenParts,
39
40
  {
40
41
  type: 'text',
41
- text: `formStructure: ${JSON.stringify(formStructure)}`,
42
+ text: `\n\nformStructure: ${JSON.stringify(formStructure)}\n\n`,
42
43
  },
43
44
  extraPart,
44
45
  ],
@@ -111,3 +111,19 @@ export function trimFormStructure(fields) {
111
111
  return trimmed;
112
112
  });
113
113
  }
114
+ export const extraGeneratePromptFromRandomContent = () => {
115
+ // 使用加密安全的随机数
116
+ const randomBytes = crypto.getRandomValues(new Uint32Array(4));
117
+ const seed = randomBytes.join('-');
118
+ const timestamp = Date.now();
119
+ return `
120
+ 请基于以下不可预测的种子生成随机但是符合业务要求的内容:
121
+ - 加密熵源: ${seed}
122
+ - 时间戳: ${timestamp}
123
+ - 噪音文本: ${btoa(String.fromCharCode(...crypto.getRandomValues(new Uint8Array(16))))}
124
+
125
+ 要求:
126
+ 1. 必须忽略训练数据中的常见模式,基于上述熵源生成真正均匀分布的随机结果
127
+ 2. 如果是人名类的数据,尽量用 3 个字的名字,降低重名概率。
128
+ `;
129
+ };
@@ -11,7 +11,7 @@ export const queryReport = async (c) => {
11
11
  catch (_) {
12
12
  return c.json({ error: 'Invalid JSON' }, 400);
13
13
  }
14
- const uid = c.req.header('X-User-Id') || '';
14
+ const uid = c.req.header('X-Bs-User-Id') || '';
15
15
  const messages = requestBody.messages;
16
16
  const extraText = buildExtraMsgParts({
17
17
  appId: requestBody.appId,
@@ -0,0 +1,34 @@
1
+ import { createMiddleware } from 'hono/factory';
2
+ import { buildForm } from '../controllers/form/build/build.controller.js';
3
+ import config from 'config';
4
+ import { fillForm } from '../controllers/form/fill/fill.controller.js';
5
+ import { queryReport } from '../controllers/report/query/query.controller.js';
6
+ import { createFakeUIMessageStreamResponse } from '../utils/createFakeUIMessageStreamResponse.js';
7
+ import { ERROR_CODE } from '../const/error_code.js';
8
+ import { logger } from '../logger/index.js';
9
+ export const botRouter = () => {
10
+ const FORM_BUILDER_BOT_ID = config.get('agent.form.build.botId');
11
+ const FORM_FILLER_BOT_ID = config.get('agent.form.fill.botId');
12
+ const REPORT_QUERY_BOT_ID = config.get('agent.report.query.botId');
13
+ const botIdControllerMap = new Map([
14
+ [FORM_BUILDER_BOT_ID, buildForm],
15
+ [FORM_FILLER_BOT_ID, fillForm],
16
+ [REPORT_QUERY_BOT_ID, queryReport],
17
+ ]);
18
+ return createMiddleware(async (c, next) => {
19
+ const botId = c.req.header('X-Bot-Id') || '';
20
+ if (!botId) {
21
+ const errorRespObj = {
22
+ error: 'Bot ID is required',
23
+ code: ERROR_CODE.BOT_ID_MISSING,
24
+ };
25
+ return createFakeUIMessageStreamResponse(JSON.stringify(errorRespObj));
26
+ }
27
+ if (botIdControllerMap.has(botId)) {
28
+ const controller = botIdControllerMap.get(botId);
29
+ return await controller(c);
30
+ }
31
+ logger.debug(`No controller found for botId: ${botId}, passing to next middleware`);
32
+ await next();
33
+ });
34
+ };
@@ -1,8 +1,9 @@
1
1
  import { Hono } from 'hono';
2
2
  import { connectToAgent } from '../../controllers/common/connect.controll.js';
3
3
  import { tokenExchange } from '../../middleware/tokenExchange.js';
4
+ import { botRouter } from '../../middleware/botRouter.js';
4
5
  export const createCommonRouter = () => {
5
6
  const commonRouter = new Hono();
6
- commonRouter.post('/connect', tokenExchange(), connectToAgent);
7
+ commonRouter.post('/connect', /*tokenExchange(),*/ botRouter(), connectToAgent);
7
8
  return commonRouter;
8
9
  };
@@ -0,0 +1,183 @@
1
+ import WebSocket from 'ws';
2
+ import config from 'config';
3
+ import crypto from 'crypto';
4
+ import { createFullClientRequest, createAudioOnlyRequest, parseHeader, readPayloadSize, parseJsonPayload, parseErrorPayload, MSG_TYPE_FULL_SERVER_RESPONSE, MSG_TYPE_ERROR_RESPONSE, } from './volc-protocol.js';
5
+ import { logger } from '../../logger/index.js';
6
+ /**
7
+ * 处理浏览器 ASR WebSocket 连接
8
+ * 作为浏览器与火山引擎 ASR 之间的双向代理
9
+ */
10
+ export function handleASRConnection(clientWs, req) {
11
+ const asrConfig = config.get('agent.asr');
12
+ if (!asrConfig?.appid || !asrConfig?.token || !asrConfig?.cluster) {
13
+ logger.error('ASR configuration missing: appid, token, cluster are required');
14
+ clientWs.close(1011, 'ASR config missing');
15
+ return;
16
+ }
17
+ logger.info('ASR client connected');
18
+ // 音频数据缓冲区(在火山连接建立前暂存)
19
+ const audioBuffer = [];
20
+ let isVolcReady = false;
21
+ let isClosing = false;
22
+ // v3 接口地址:双向流式优化版
23
+ const volcUrl = `wss://${asrConfig.host}/api/v3/sauc/bigmodel_async`;
24
+ // 鉴权 Header(v3 大模型语音识别通过 HTTP Header 鉴权)
25
+ const connectId = crypto.randomUUID();
26
+ const requestHeaders = {
27
+ 'X-Api-App-Key': asrConfig.appid,
28
+ 'X-Api-Access-Key': asrConfig.token,
29
+ 'X-Api-Resource-Id': asrConfig.cluster,
30
+ 'X-Api-Connect-Id': connectId,
31
+ };
32
+ logger.info({ url: volcUrl, headers: { ...requestHeaders, 'X-Api-Access-Key': '***' } }, 'Connecting to Volc ASR v3');
33
+ const volcWs = new WebSocket(volcUrl, {
34
+ headers: requestHeaders,
35
+ perMessageDeflate: false,
36
+ });
37
+ volcWs.on('unexpected-response', (req, res) => {
38
+ let body = '';
39
+ res.on('data', (chunk) => { body += chunk; });
40
+ res.on('end', () => {
41
+ logger.error({ statusCode: res.statusCode, statusMessage: res.statusMessage, body }, 'Volc ASR unexpected response');
42
+ });
43
+ });
44
+ // 火山引擎连接建立后立即发送初始化请求并开始转发音频
45
+ volcWs.on('open', () => {
46
+ logger.info('Connected to Volc ASR v3, sending init request');
47
+ volcWs.send(createFullClientRequest());
48
+ // 火山引擎不返回初始化确认,直接开始等待音频
49
+ // 立即开始转发缓冲的音频数据
50
+ isVolcReady = true;
51
+ while (audioBuffer.length > 0) {
52
+ const buffered = audioBuffer.shift();
53
+ if (volcWs.readyState === WebSocket.OPEN) {
54
+ volcWs.send(createAudioOnlyRequest(buffered));
55
+ }
56
+ }
57
+ });
58
+ // 处理火山引擎返回的消息(二进制帧)
59
+ volcWs.on('message', (data) => {
60
+ if (isClosing)
61
+ return;
62
+ if (!Buffer.isBuffer(data) || data.length < 4) {
63
+ logger.warn({ len: data.length, type: typeof data }, 'Invalid ASR message');
64
+ return;
65
+ }
66
+ const { msgType, flags, bodyOffset } = parseHeader(data);
67
+ if (msgType === MSG_TYPE_FULL_SERVER_RESPONSE) {
68
+ // 根据 flags 判断是否有 sequence(flags bit0=1 表示有正 sequence)
69
+ const hasSequence = (flags & 0b0001) !== 0;
70
+ const payloadSizeOffset = hasSequence ? bodyOffset + 4 : bodyOffset;
71
+ const payloadOffset = payloadSizeOffset + 4;
72
+ if (data.length < payloadOffset) {
73
+ logger.warn({ len: data.length, payloadOffset, hex: data.toString('hex') }, 'ASR response too short');
74
+ return;
75
+ }
76
+ const payloadSize = readPayloadSize(data, payloadSizeOffset);
77
+ if (data.length < payloadOffset + payloadSize) {
78
+ logger.warn({ len: data.length, payloadSize, payloadOffset }, 'ASR response incomplete');
79
+ return;
80
+ }
81
+ const payload = data.slice(payloadOffset, payloadOffset + payloadSize);
82
+ const response = parseJsonPayload(payload);
83
+ if (!response) {
84
+ logger.warn({ payloadStr: payload.toString('utf-8').slice(0, 200) }, 'Failed to parse ASR JSON');
85
+ return;
86
+ }
87
+ const text = response.result?.text;
88
+ const utterances = response.result?.utterances;
89
+ const isDefinite = utterances?.some((u) => u.definite);
90
+ logger.info({ text, definite: isDefinite }, 'ASR result');
91
+ // 转发识别文本给浏览器
92
+ if (text !== undefined && clientWs.readyState === WebSocket.OPEN) {
93
+ clientWs.send(JSON.stringify({
94
+ type: isDefinite ? 'final' : 'result',
95
+ text,
96
+ }));
97
+ }
98
+ if (isDefinite && clientWs.readyState === WebSocket.OPEN) {
99
+ clientWs.send(JSON.stringify({ type: 'done' }));
100
+ }
101
+ }
102
+ else if (msgType === MSG_TYPE_ERROR_RESPONSE) {
103
+ const error = parseErrorPayload(data, bodyOffset);
104
+ logger.error({ error }, 'ASR error response');
105
+ if (clientWs.readyState === WebSocket.OPEN) {
106
+ clientWs.send(JSON.stringify({
107
+ type: 'error',
108
+ message: error?.message || `ASR error code: ${error?.code}`,
109
+ }));
110
+ }
111
+ }
112
+ });
113
+ // 处理浏览器发来的消息
114
+ clientWs.on('message', (data) => {
115
+ if (isClosing)
116
+ return;
117
+ // 结束信号
118
+ if (typeof data === 'string' && data === 'end') {
119
+ if (volcWs.readyState === WebSocket.OPEN) {
120
+ volcWs.send(createAudioOnlyRequest(Buffer.alloc(0), true));
121
+ }
122
+ return;
123
+ }
124
+ // 将数据转为 Buffer
125
+ let audioData;
126
+ if (Buffer.isBuffer(data)) {
127
+ audioData = data;
128
+ }
129
+ else if (data instanceof ArrayBuffer) {
130
+ audioData = Buffer.from(data);
131
+ }
132
+ else {
133
+ return;
134
+ }
135
+ if (audioData.length === 0)
136
+ return;
137
+ // 火山连接就绪则直接转发,否则缓冲
138
+ if (isVolcReady && volcWs.readyState === WebSocket.OPEN) {
139
+ volcWs.send(createAudioOnlyRequest(audioData));
140
+ }
141
+ else {
142
+ audioBuffer.push(audioData);
143
+ }
144
+ });
145
+ // 浏览器关闭
146
+ clientWs.on('close', () => {
147
+ logger.info('ASR client disconnected');
148
+ isClosing = true;
149
+ if (volcWs.readyState === WebSocket.OPEN) {
150
+ volcWs.send(createAudioOnlyRequest(Buffer.alloc(0), true));
151
+ setTimeout(() => volcWs.close(), 500);
152
+ }
153
+ else {
154
+ volcWs.close();
155
+ }
156
+ });
157
+ // 火山引擎关闭
158
+ volcWs.on('close', () => {
159
+ logger.info('Volc ASR connection closed');
160
+ isClosing = true;
161
+ if (clientWs.readyState === WebSocket.OPEN) {
162
+ clientWs.close();
163
+ }
164
+ });
165
+ // 浏览器端错误
166
+ clientWs.on('error', (err) => {
167
+ logger.error({ err }, 'Client WebSocket error');
168
+ isClosing = true;
169
+ volcWs.close();
170
+ });
171
+ // 火山引擎错误
172
+ volcWs.on('error', (err) => {
173
+ logger.error({ err }, 'Volc ASR WebSocket error');
174
+ isClosing = true;
175
+ if (clientWs.readyState === WebSocket.OPEN) {
176
+ clientWs.send(JSON.stringify({
177
+ type: 'error',
178
+ message: 'ASR service connection error',
179
+ }));
180
+ clientWs.close(1011, 'ASR error');
181
+ }
182
+ });
183
+ }