ai_co_discuss 0.3.26 → 0.3.27

package/backend/server.py

@@ -1721,7 +1721,7 @@ class DiscussionHandler(BaseHTTPRequestHandler):
         site = self.local_request_site(method)
         route = parts[2:]
         if method == "OPTIONS" and route in (["settings"], ["launch"], ["room-models"], ["room-member"], ["room-member-access"], ["room-member-allow"], ["room-member-adapter"], ["room-member-model"], ["room-profiles"], ["room-restore"], ["room-fork"], ["models"], ["api-key"], ["api-check"],
-                                             ["scheduler-check"], ["tts-settings"], ["tts"], ["pick-directory"], ["projects"], ["presets"], ["machine-link"], ["admin-account"]):
+                                             ["scheduler-check"], ["tts-settings"], ["tts"], ["stt"], ["pick-directory"], ["projects"], ["presets"], ["machine-link"], ["admin-account"]):
             return self.respond({"ok": True})
         if route == ["settings"] and method in ("GET", "POST"):
             command = "settings"
@@ -1739,7 +1739,7 @@ class DiscussionHandler(BaseHTTPRequestHandler):
             command = "api-check"
         elif route == ["scheduler-check"] and method == "POST":
             command = "scheduler-check"
-        elif route in (["tts-settings"], ["tts"]) and method == "POST":
+        elif route in (["tts-settings"], ["tts"], ["stt"]) and method == "POST":
             command = route[0]
         elif route in (["room-member"], ["room-member-access"], ["room-member-allow"], ["room-member-adapter"], ["room-member-model"]) and method == "POST":
             command = route[0]

package/docs/runner.md

@@ -88,9 +88,14 @@ API 成员可以担任专家或主持人。房间指定了项目目录时，它
 
 翻开成员卡片，“提示”一行可以给这位专家单独写几句要求，例如“回答先给结论”。它只发给这位专家，不改变权限；部分软件带有预设，可随时恢复。卡片背面的书签图标把这张卡片的描述、提示词、项目权限、模型与强度存为专家预设，之后在“添加”行一键加入，见 [使用指南](workflow.md#成员管理)。
 
-## 朗读（可选）
+## 朗读与语音输入（可选）
 
-在网页“设置”的“朗读”中填写火山引擎 API Key，选择音色、语速，可写一句语气提示（如“像朋友聊天一样自然地读”）。使用豆包语音合成 2.0，需先在火山引擎控制台开通该服务和所选音色，费用按你的账号计费；密钥只保存在本机。每条消息的工具栏有“朗读”，在消息里选中文字后点“听当前”从选中处开始；读完这条会接着读后面的消息直到最新，正在读的句子会高亮，代码块跳过。朗读由本机服务合成，手机等其他设备经已配对电脑使用，电脑关机时不可用。
+在火山引擎控制台开通「语音合成2.0」（朗读）和「录音文件识别2.0」（语音输入），复制 API Key，填进网页“设置”的“语音”。朗读还可选音色、语速，改语气提示（默认是“像给朋友讲故事一样”的自然语气）。密钥只保存在本机，费用按你的账号计费。
+
+- 朗读：每条消息的工具栏有“朗读”（手机上在“…”菜单里），在消息里选中文字后点“听当前”从选中处开始；读完这条会接着读后面的消息直到最新，正在读的句子会高亮并自动滚到视野内，点一下高亮的句子暂停，再点继续；代码块跳过。
+- 语音输入：点输入框右下角的麦克风开始说话，再点一次转成文字插入输入框，单次最长 5 分钟。浏览器会先请求麦克风权限。
+
+两者都由本机服务调用火山引擎，手机等其他设备经已配对电脑使用，电脑关机时不可用。
 
 ## 权限
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai_co_discuss",
-  "version": "0.3.26",
+  "version": "0.3.27",
   "bin": {
     "codiscuss": "runner/codiscuss.mjs"
   },

package/runner/discussion_launch.mjs

@@ -16,6 +16,7 @@ import { ADAPTER_IDS, ADAPTER_PROFILES, adapter_profile, platform_adapters, same
 import { API_PROVIDERS, api_check, api_models, save_api_key, saved_api_keys, validate_api_profile } from './discussion_api.mjs';
 import { DEFAULT_SCHEDULER_MODE, SCHEDULER_MODES, SCHEDULER_PROVIDERS, scheduler_check } from './scheduler.mjs';
 import { save_tts_settings, tts_settings, tts_speak } from './tts.mjs';
+import { stt_part } from './stt.mjs';
 import { access_levels, BUILTIN_EXPERT_PRESETS, MAX_EXPERT_PROMPT, PROJECT_ACCESS } from './expert_prompts.mjs';
 import { acquire_lock, api, DiscussionStore, service_drafts, validate_config } from './discussion_runner.mjs';
 import { FreeRoomRunner, free_task_files } from './free_runner.mjs';
@@ -59,6 +60,7 @@ const HELP = `用法：codiscuss launch settings [--file FILE]
       codiscuss launch scheduler-check --file FILE
       codiscuss launch tts-settings [--file FILE]
       codiscuss launch tts --file FILE
+      codiscuss launch stt --file FILE
       codiscuss launch room-member --file FILE
       codiscuss launch room-member-access --file FILE
       codiscuss launch room-member-allow --file FILE
@@ -83,6 +85,7 @@ api-check 用 API profile 向模型发送一次极短请求，核对地址、密
 scheduler-check 用智能调度配置（provider、model、api_key_ref）向决策模型发送一道是非题，核对供应商、模型与密钥是否可用。
 tts-settings 读取朗读设置；内容非空时保存（音色 speaker、语速 speech_rate、语气提示 context、api_key_ref）。
 tts 用朗读设置把 text（不超过 300 字）合成为 MP3，返回 base64。
+stt 按序接收录音片段（upload_id、index、base64 PCM 16 kHz 单声道、final）；最后一段时识别整段录音并返回 text。
 pick-directory 在本机弹出系统目录选择窗口，返回所选路径；取消时不返回路径。
 projects 读取本机已配置的项目（名称与绝对路径）；提供 JSON（projects 列表）时整体替换保存。
 presets 读取本机的专家预设（含内置预设）与各平台默认描述；提供 JSON（presets 列表和/或 description_templates）时替换对应部分。
@@ -1303,6 +1306,7 @@ export async function launch_main(args = process.argv.slice(2), input_text) {
   else if (positionals[0] === 'scheduler-check' && input) result = await scheduler_check(home, input);
   else if (positionals[0] === 'tts-settings') result = input && Object.keys(input).length ? save_tts_settings(home, input) : tts_settings(home);
   else if (positionals[0] === 'tts' && input) result = await tts_speak(home, input);
+  else if (positionals[0] === 'stt' && input) result = await stt_part(home, input);
   else if (positionals[0] === 'room-models' && input) result = await room_models(home, input);
   else if (positionals[0] === 'models' && adapter_profile(input?.adapter)?.kind === 'api') result = await api_models(home, input);
   else if (positionals[0] === 'models' && input) {

package/runner/remote_launch.mjs

@@ -27,7 +27,7 @@ const RELAY_CIPHER_INFO = 'codiscuss-relay-v1';
 const LAUNCH_SCRIPT = fileURLToPath(new URL('./discussion_launch.mjs', import.meta.url));
 // What a connected browser may ask the local bridge for, minus the folder dialog nobody would see and pairing itself.
 const RELAYED_ACTIONS = new Set(['settings', 'launch', 'room-models', 'room-member', 'room-member-access', 'room-member-allow', 'room-member-adapter',
-  'room-member-model', 'room-profiles', 'room-restore', 'room-fork', 'models', 'api-key', 'api-check', 'scheduler-check', 'tts-settings', 'tts', 'projects', 'presets', 'admin-account']);
+  'room-member-model', 'room-profiles', 'room-restore', 'room-fork', 'models', 'api-key', 'api-check', 'scheduler-check', 'tts-settings', 'tts', 'stt', 'projects', 'presets', 'admin-account']);
 export const ROOM_ACTIONS = new Set(['launch', 'room-models', 'room-member', 'room-member-access', 'room-member-allow', 'room-member-adapter',
   'room-member-model', 'room-profiles', 'room-restore', 'room-fork']);
 

package/runner/stt.mjs

@@ -0,0 +1,101 @@
+/**
+ * Voice input: the page records 16 kHz mono PCM and sends it in small parts (each fits one relayed request);
+ * the last part assembles a WAV file and Volcengine's flash recognizer turns it into text. The key stays local.
+ */
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { read_saved_api_key } from './discussion_api.mjs';
+import { TTS_BASE_URL, tts_settings } from './tts.mjs';
+
+const STT_ENDPOINT = `${TTS_BASE_URL}/api/v3/auc/bigmodel/recognize/flash`;
+const STT_RESOURCE_ID = 'volc.bigasr.auc_turbo';
+const STT_DONE_CODE = '20000000';
+const STT_SILENT_CODE = '20000003';
+const STT_TIMEOUT_MS = 50_000;
+const SAMPLE_RATE = 16000;
+// About 5 minutes of 16-bit mono audio at 16 kHz.
+export const STT_MAX_BYTES = 10 * 1024 * 1024;
+const PART_MAX_BYTES = 256 * 1024;
+const STALE_MS = 60 * 60 * 1000;
+const STATUS_MESSAGES = {
+  45000001: '请求参数有误',
+  45000002: '没有录到声音',
+  45000151: '音频格式不受支持',
+  55000031: '火山引擎语音识别繁忙，请稍后再试',
+};
+
+/** Prefix raw 16-bit mono PCM with a WAV header. */
+export function pcm_to_wav(pcm) {
+  const header = Buffer.alloc(44);
+  header.write('RIFF', 0); header.writeUInt32LE(36 + pcm.length, 4); header.write('WAVE', 8);
+  header.write('fmt ', 12); header.writeUInt32LE(16, 16); header.writeUInt16LE(1, 20); header.writeUInt16LE(1, 22);
+  header.writeUInt32LE(SAMPLE_RATE, 24); header.writeUInt32LE(SAMPLE_RATE * 2, 28); header.writeUInt16LE(2, 32); header.writeUInt16LE(16, 34);
+  header.write('data', 36); header.writeUInt32LE(pcm.length, 40);
+  return Buffer.concat([header, pcm]);
+}
+
+/** Drop recordings abandoned mid-way (closed page, lost connection). */
+function clean_stale(directory) {
+  if (!existsSync(directory)) return;
+  for (const name of readdirSync(directory)) {
+    const path = join(directory, name);
+    try { if (Date.now() - statSync(path).mtimeMs > STALE_MS) rmSync(path, { recursive: true, force: true }); } catch { /* Gone already. */ }
+  }
+}
+
+/** Send a finished recording to the recognizer and return its text. */
+async function recognize(key, wav, request) {
+  let response;
+  try {
+    response = await request(STT_ENDPOINT, { method: 'POST', redirect: 'error', signal: AbortSignal.timeout(STT_TIMEOUT_MS),
+      headers: { 'Content-Type': 'application/json', 'X-Api-Key': key, 'X-Api-Resource-Id': STT_RESOURCE_ID,
+        'X-Api-Request-Id': randomUUID(), 'X-Api-Sequence': '-1' },
+      body: JSON.stringify({ user: { uid: 'codiscuss' }, audio: { data: wav.toString('base64') }, request: { model_name: 'bigmodel', enable_punc: true, enable_itn: true } }) });
+  } catch { throw new Error('语音识别请求超时或连接失败，请检查网络后重试'); }
+  if (response.status === 401 || response.status === 403) throw new Error('火山引擎拒绝了请求：API Key 无效或未开通录音文件识别 2.0');
+  const code = response.headers.get('X-Api-Status-Code');
+  if (code === STT_SILENT_CODE) return '';
+  if (code !== STT_DONE_CODE) {
+    const message = STATUS_MESSAGES[code] || response.headers.get('X-Api-Message') || `HTTP ${response.status}`;
+    throw new Error(`语音识别失败（${code || response.status}）：${message}`);
+  }
+  const data = await response.json();
+  return String(data?.result?.text || '').trim();
+}
+
+/**
+ * Store one part of a recording; the final part recognizes the whole recording and returns its text.
+ * @param {string} home Local service directory.
+ * @param {{ upload_id: string, index: number, data: string, final?: boolean }} input Base64 PCM part in order.
+ * @returns {Promise<{ received: number } | { text: string }>}
+ */
+export async function stt_part(home, input, request = fetch) {
+  const { upload_id, index, data, final } = input || {};
+  if (!/^[a-f0-9-]{36}$/i.test(upload_id || '') || !Number.isInteger(index) || index < 0 || typeof data !== 'string') throw new Error('录音片段无效');
+  const settings = tts_settings(home);
+  if (!settings.api_key_ref) throw new Error('尚未配置语音：请在设置的“语音”中填写火山引擎 API Key');
+  const part = Buffer.from(data, 'base64');
+  if (part.length > PART_MAX_BYTES || part.length % 2) throw new Error('录音片段无效');
+  const root = join(home, 'stt');
+  mkdirSync(root, { recursive: true, mode: 0o700 });
+  clean_stale(root);
+  const directory = join(root, upload_id.toLowerCase());
+  mkdirSync(directory, { recursive: true, mode: 0o700 });
+  writeFileSync(join(directory, `${index}.pcm`), part, { mode: 0o600 });
+  if (!final) return { received: index };
+  try {
+    const parts = [];
+    for (let position = 0; position <= index; position += 1) {
+      const path = join(directory, `${position}.pcm`);
+      if (!existsSync(path)) throw new Error('录音片段缺失，请重新录制');
+      parts.push(readFileSync(path));
+    }
+    const pcm = Buffer.concat(parts);
+    if (pcm.length > STT_MAX_BYTES) throw new Error('录音超过 5 分钟，请分段录制');
+    if (!pcm.length) return { text: '' };
+    const key = read_saved_api_key(home, TTS_BASE_URL, settings.api_key_ref);
+    return { text: await recognize(key, pcm_to_wav(pcm), request) };
+  } finally { rmSync(directory, { recursive: true, force: true }); }
+}

package/runner/tts.mjs

@@ -11,13 +11,40 @@ const TTS_ENDPOINT = `${TTS_BASE_URL}/api/v3/tts/unidirectional`;
 const TTS_RESOURCE_ID = 'seed-tts-2.0';
 const TTS_DONE_CODE = 20000000;
 const TTS_TIMEOUT_MS = 45_000;
-// The page sends one sentence at a time; this bounds a single relayed answer well under the relay frame size.
+// The page sends a paragraph of up to 200 characters; at about 8 KB of MP3 per second this keeps one relayed answer under 1 MiB.
 export const TTS_MAX_TEXT = 300;
 const MAX_CONTEXT = 200;
 const SETTINGS_FILE = 'tts-settings.json';
-export const TTS_DEFAULTS = { speaker: 'zh_female_vv_uranus_bigtts', speech_rate: 0, context: '' };
+// A default tone instruction (context_texts) so reading sounds told rather than recited; users may change or clear it.
+export const TTS_DEFAULT_CONTEXT = '请用自然放松的语气朗读，像给朋友讲故事一样，有轻重缓急，句子之间自然停顿。';
+export const TTS_DEFAULTS = { speaker: 'zh_female_vv_uranus_bigtts', speech_rate: 0, context: TTS_DEFAULT_CONTEXT };
 // Voices a user can pick without looking up IDs; any other 2.0 voice ID enabled in the console may be typed.
-export const TTS_VOICES = [{ id: 'zh_female_vv_uranus_bigtts', label: 'Vivi（通用女声）' }];
+export const TTS_VOICES = [
+  { id: 'zh_female_vv_uranus_bigtts', label: 'Vivi · 通用女声' },
+  { id: 'zh_female_xiaohe_uranus_bigtts', label: '小何 · 通用女声' },
+  { id: 'zh_male_m191_uranus_bigtts', label: '云舟 · 通用男声' },
+  { id: 'zh_male_taocheng_uranus_bigtts', label: '小天 · 通用男声' },
+  { id: 'zh_male_liufei_uranus_bigtts', label: '刘飞 · 通用男声' },
+  { id: 'zh_female_sophie_uranus_bigtts', label: '魅力苏菲 · 女声' },
+  { id: 'zh_female_qingxinnvsheng_uranus_bigtts', label: '清新女声' },
+  { id: 'zh_female_cancan_uranus_bigtts', label: '知性灿灿 · 女声' },
+  { id: 'zh_female_zhixingnv_uranus_bigtts', label: '知性女声' },
+  { id: 'zh_female_wenroushunv_uranus_bigtts', label: '温柔淑女 · 女声' },
+  { id: 'zh_female_linjianvhai_uranus_bigtts', label: '邻家女孩' },
+  { id: 'zh_female_shuangkuaisisi_uranus_bigtts', label: '爽快思思 · 女声' },
+  { id: 'zh_female_gaolengyujie_uranus_bigtts', label: '高冷御姐 · 女声' },
+  { id: 'zh_female_xiaoxue_uranus_bigtts', label: '儿童绘本 · 女声' },
+  { id: 'zh_male_ruyaqingnian_uranus_bigtts', label: '儒雅青年 · 男声' },
+  { id: 'zh_male_yangguangqingnian_uranus_bigtts', label: '阳光青年 · 男声' },
+  { id: 'zh_male_yuanboxiaoshu_uranus_bigtts', label: '渊博小叔 · 男声' },
+  { id: 'zh_male_cixingjieshuonan_uranus_bigtts', label: '磁性解说 · 男声' },
+  { id: 'zh_male_shenyeboke_uranus_bigtts', label: '深夜播客 · 男声' },
+  { id: 'zh_male_xuanyijieshuo_uranus_bigtts', label: '悬疑解说 · 男声' },
+  { id: 'zh_male_baqiqingshu_uranus_bigtts', label: '霸气青叔 · 有声书男声' },
+  { id: 'zh_male_yizhipiannan_uranus_bigtts', label: '译制片男声' },
+  { id: 'zh_male_jieshuoxiaoming_uranus_bigtts', label: '解说小明 · 男声' },
+  { id: 'zh_male_wennuanahu_uranus_bigtts', label: '温暖阿虎 · 男声' },
+];
 const STATUS_MESSAGES = {
   40402003: '文本超出单次合成长度上限',
   45000000: '音色不可用：请确认已在火山引擎控制台开通该豆包语音合成 2.0 音色，并核对音色 ID',
@@ -69,18 +96,27 @@ function tts_error(code, message) {
   return new Error(`语音合成失败（${code}）：${STATUS_MESSAGES[code] || message || '请核对 API Key 与音色'}`);
 }
 
-/** Synthesize one piece of text and return it as base64 MP3. */
+/** Flatten the per-word timings the service returns with subtitles, as { w: word, t: start in ms }. */
+function word_marks(objects) {
+  return objects.flatMap(item => item.sentence?.words || []).map(word => ({ w: String(word.word || ''), t: Math.round(Number(word.startTime || 0) * 1000) }));
+}
+
+/**
+ * Synthesize one paragraph and return it as base64 MP3 with word timings for highlighting.
+ * Requests of one reading pass share `section_id`, so the service keeps the earlier paragraphs as context.
+ */
 export async function tts_speak(home, input, request = fetch) {
   const text = typeof input?.text === 'string' ? input.text.trim() : '';
   if (!text) throw new Error('朗读文本为空');
   if (text.length > TTS_MAX_TEXT) throw new Error(`单次朗读不超过 ${TTS_MAX_TEXT} 字`);
   const settings = tts_settings(home);
-  if (!settings.api_key_ref) throw new Error('尚未配置朗读：请在设置的“朗读”中填写火山引擎 API Key');
+  if (!settings.api_key_ref) throw new Error('尚未配置语音：请在设置的“语音”中填写火山引擎 API Key');
   const key = read_saved_api_key(home, TTS_BASE_URL, settings.api_key_ref);
-  const additions = settings.context ? { context_texts: [settings.context] } : null;
+  const section_id = /^[a-f0-9-]{36}$/i.test(input.section_id || '') ? input.section_id : undefined;
+  const additions = { ...(settings.context ? { context_texts: [settings.context] } : {}), ...(section_id ? { section_id } : {}) };
   const body = { user: { uid: 'codiscuss' }, req_params: { text, speaker: settings.speaker,
-    audio_params: { format: 'mp3', sample_rate: 24000, speech_rate: settings.speech_rate },
-    ...(additions ? { additions: JSON.stringify(additions) } : {}) } };
+    audio_params: { format: 'mp3', sample_rate: 24000, speech_rate: settings.speech_rate, enable_subtitle: true },
+    ...(Object.keys(additions).length ? { additions: JSON.stringify(additions) } : {}) } };
   let response;
   try {
     response = await request(TTS_ENDPOINT, { method: 'POST', redirect: 'error', signal: AbortSignal.timeout(TTS_TIMEOUT_MS),
@@ -91,7 +127,7 @@ export async function tts_speak(home, input, request = fetch) {
   const objects = split_json_objects(await response.text());
   const chunks = [];
   for (const item of objects) {
-    if (item.code === TTS_DONE_CODE) return { audio: Buffer.concat(chunks).toString('base64'), format: 'mp3' };
+    if (item.code === TTS_DONE_CODE) return { audio: Buffer.concat(chunks).toString('base64'), format: 'mp3', marks: word_marks(objects) };
     if (item.code !== 0) throw tts_error(item.code, item.message);
     if (item.data) chunks.push(Buffer.from(item.data, 'base64'));
   }