agent-loft 0.2.113 → 0.2.114

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
54
54
  - **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
55
55
  - 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
56
56
  - 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
57
- - 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框里点声波按钮,和 Codex 桌面端的语音模式一样边说边聊,可以随时打断。输入框换成通话条(音量动画、状态、实时字幕、静音、结束);要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3、WebRTC),声音在浏览器和 OpenAI 之间直连,不经过服务器;用的是 Codex 登录的 ChatGPT 账号。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
57
+ - 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框里点声波按钮,和 Codex 桌面端的语音模式一样边说边聊,可以随时打断。输入框换成通话条(音量动画、状态、实时字幕、静音、结束);要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话条上的齿轮里可以选音色和回答用的语言(记在这台设备上)。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
58
58
  - 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
59
59
  - 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
60
60
  - 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-loft",
3
- "version": "0.2.113",
3
+ "version": "0.2.114",
4
4
  "type": "module",
5
5
  "description": "多个 agent 的工作台:把本机 Claude Code / Codex 放到一处统一管理,用飞书和 Web 页面随时驱动——会话、审批、模型切换都共用",
6
6
  "keywords": [
@@ -58,9 +58,11 @@
58
58
  "echarts": "^6.1.0",
59
59
  "marked": "^16.0.0",
60
60
  "mermaid": "^12.2.0",
61
+ "opusscript": "^0.1.1",
61
62
  "qrcode": "^1.5.4",
62
63
  "sharp": "^0.35.5",
63
64
  "tsx": "^4.20.6",
65
+ "werift": "^0.27.0",
64
66
  "ws": "^8.18.0"
65
67
  },
66
68
  "devDependencies": {
@@ -20,6 +20,7 @@ import type {
20
20
  InteractionSpec,
21
21
  InterruptOptions,
22
22
  QuestionSpec,
23
+ RealtimeOptions,
23
24
  RuntimeInfo,
24
25
  SavedAttachment,
25
26
  SessionOptions,
@@ -816,12 +817,12 @@ export class CodexSession implements AgentSession {
816
817
  // ---------- 实时语音对话(thread/realtime/*,实验接口) ----------
817
818
 
818
819
  /**
819
- * 浏览器建好 RTCPeerConnection(麦克风 + oai-events 数据通道)把 offer 给过来,app-server 去 OpenAI 建通话,
820
- * 对端 answer 从 thread/realtime/sdp 通知回来。声音在浏览器和 OpenAI 之间直连,不经过这里。
820
+ * offer 是服务器这边的 WebRTC 端点(core/realtime-relay.ts)建的,app-server 去 OpenAI 建通话,
821
+ * 对端 answer 从 thread/realtime/sdp 通知回来。声音经服务器中转,浏览器那头走 WebSocket。
821
822
  * 实测:ChatGPT 账号登录只能走 WebRTC(WebSocket 要 API key),协议版本要 v3。
822
823
  * 语音模型把要干的活转交给这条 thread,app-server 自己开一轮(见 turn/started 里的 externalPending)。
823
824
  */
824
- async startRealtime(offer: string): Promise<string> {
825
+ async startRealtime(offer: string, opts: RealtimeOptions = {}): Promise<string> {
825
826
  if (this.disposed) throw new Error("会话已关闭");
826
827
  if (this.rt) await this.stopRealtime();
827
828
  const threadId = await this.open();
@@ -853,6 +854,7 @@ export class CodexSession implements AgentSession {
853
854
  // 启动背景里有这台机器上最近的会话和目录结构:只给内置专家的管理员
854
855
  includeStartupContext: !expert && this.opts.skills?.admin === true,
855
856
  ...(context ? { realtimeStartInstructions: context } : {}),
857
+ ...realtimeVoiceParams(opts),
856
858
  });
857
859
  return await answer;
858
860
  } catch (err) {
@@ -1336,6 +1338,32 @@ function kindMark(kind: unknown): string {
1336
1338
  return k === "add" ? "+" : k === "delete" ? "-" : "~";
1337
1339
  }
1338
1340
 
1341
+ /** v3 实时语音能用的音色(实测是 thread/realtime/listVoices 里 v1 那组,给 v2 的会报不支持;默认 cove) */
1342
+ export const REALTIME_VOICES = ["cove", "juniper", "maple", "spruce", "ember", "vale", "breeze", "arbor", "sol"] as const;
1343
+
1344
+ /**
1345
+ * 语音模型说哪种话:协议里没有语言参数,用 v3 的 initialItems 放一条 developer 指令。
1346
+ * 不在表里(或「自动」)就不加,跟着用户说的语言走
1347
+ */
1348
+ export const REALTIME_LANGUAGES: Record<string, string> = {
1349
+ "zh-CN": "Always speak Mandarin Chinese (普通话) and use Simplified Chinese in any text, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
1350
+ "zh-TW": "Always speak Mandarin Chinese (國語) and use Traditional Chinese in any text, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
1351
+ yue: "Always speak Cantonese (粵語/廣東話), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
1352
+ en: "Always speak English, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
1353
+ ja: "Always speak Japanese (日本語), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
1354
+ ko: "Always speak Korean (한국어), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
1355
+ };
1356
+
1357
+ /** 页面选的音色、语言 → thread/realtime/start 的参数;认不出的忽略(用 Codex 默认) */
1358
+ export function realtimeVoiceParams(opts: RealtimeOptions): Record<string, unknown> {
1359
+ const voice = (REALTIME_VOICES as readonly string[]).includes(opts.voice ?? "") ? opts.voice : undefined;
1360
+ const language = opts.language ? REALTIME_LANGUAGES[opts.language] : undefined;
1361
+ return {
1362
+ ...(voice ? { voice } : {}),
1363
+ ...(language ? { initialItems: [{ role: "developer", text: language }] } : {}),
1364
+ };
1365
+ }
1366
+
1339
1367
  /** 语音转交的一轮没拿到用户说的话时显示的占位 */
1340
1368
  const VOICE_FALLBACK = `${VOICE_MARK}(语音)`;
1341
1369
 
@@ -19,6 +19,7 @@ import { skillEvents } from "../../core/skills.js";
19
19
  import { expertEvents } from "../../core/experts.js";
20
20
  import { enforceTerminalAccess, killAllTerminals, terminalEvents, watchTerminalAccess } from "../../core/terminals.js";
21
21
  import { RUNNING_VERSION } from "../../core/upgrade.js";
22
+ import { RealtimeRelay, RELAY_RATE } from "../../core/realtime-relay.js";
22
23
  import type { RealtimeEvent, Settlement } from "../../core/types.js";
23
24
  import { getUser, isAdmin, legacyOwnerId, userEvents, type User } from "../../core/users.js";
24
25
  import { currentUser, handleApi, takeUploads } from "./api.js";
@@ -268,8 +269,24 @@ type Client = {
268
269
  pushEndpoint: string | null;
269
270
  /** 最后一次收到这个页面的消息(含 ping) */
270
271
  seen: number;
272
+ /** 这个页面正在打的语音对话:声音经这里中转(relay 建好之前是 undefined) */
273
+ rt?: { key: string; relay?: RealtimeRelay };
271
274
  };
272
275
 
276
+ /** 一块麦克风 PCM 的上限(正常是 20ms ≈ 1KB) */
277
+ const MAX_AUDIO_CHUNK = 64 * 1024;
278
+ /** 发给页面的声音积压超过这么多就丢(手机网络卡住时别越攒越多、越放越晚) */
279
+ const MAX_AUDIO_BACKLOG = 256 * 1024;
280
+
281
+ /** 收掉这个页面的语音中转;stopCall:顺带让 Codex 挂断 */
282
+ function endRelay(client: Client, stopCall: boolean): void {
283
+ const rt = client.rt;
284
+ if (!rt) return;
285
+ client.rt = undefined;
286
+ rt.relay?.close();
287
+ if (stopCall) void hub.realtimeStop(rt.key).catch(() => {});
288
+ }
289
+
273
290
  /** 每次进程启动换一个:客户端发现变了,就知道 seq 已经从头计数,得整页重拉 */
274
291
  const EPOCH = Math.random().toString(36).slice(2, 10);
275
292
 
@@ -305,7 +322,10 @@ function broadcastEvent(env: EventEnvelope): void {
305
322
 
306
323
  /** 语音对话的字幕和状态:只推给订阅了这条对话的页面(不进缓冲区,刷新后不重放) */
307
324
  function broadcastRealtime({ key, ev }: { key: string; ev: RealtimeEvent }): void {
308
- for (const client of clients) if (client.subs.has(key) && canSeeKey(client.user, key)) send(client, { t: "rt", key, ev });
325
+ for (const client of clients) {
326
+ if (ev.type === "closed" && client.rt?.key === key && client.rt.relay) endRelay(client, false);
327
+ if (client.subs.has(key) && canSeeKey(client.user, key)) send(client, { t: "rt", key, ev });
328
+ }
309
329
  }
310
330
 
311
331
  function broadcastInteraction(i: Interaction): void {
@@ -533,19 +553,45 @@ function handleClientMessage(client: Client, raw: string): void {
533
553
  }
534
554
 
535
555
  case "rt": {
536
- // 实时语音对话:start 带浏览器的 SDP offer,ack 回对端 answer;stop 挂断
556
+ // 实时语音对话:服务器自己和 OpenAI 建 WebRTC,声音在这条 WebSocket 上中转(二进制帧,24kHz 16 位 PCM)。
557
+ // start 等连上了才 ack;stop 挂断
537
558
  if (!getConversation(key) || !canSeeKey(user, key)) return ack(false, { error: "会话不存在" });
538
559
  if (msg.op === "start") {
539
- const sdp = typeof msg.sdp === "string" ? msg.sdp : "";
540
- if (!sdp || sdp.length > 50_000) return ack(false, { error: "SDP 不对" });
560
+ endRelay(client, false);
561
+ const rt: NonNullable<Client["rt"]> = { key };
562
+ client.rt = rt;
541
563
  client.subs.add(key);
542
- void hub
543
- .realtimeStart(key, sdp)
544
- .then((answer) => ack(true, { sdp: answer }))
545
- .catch((err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
564
+ const opts = {
565
+ ...(typeof msg.voice === "string" && msg.voice ? { voice: msg.voice } : {}),
566
+ ...(typeof msg.language === "string" && msg.language ? { language: msg.language } : {}),
567
+ };
568
+ void (async () => {
569
+ const { relay, offer } = await RealtimeRelay.create();
570
+ if (client.rt !== rt) {
571
+ relay.close();
572
+ throw new Error("已挂断");
573
+ }
574
+ rt.relay = relay;
575
+ relay.onAudio = (pcm) => {
576
+ if (client.ws.readyState === client.ws.OPEN && client.ws.bufferedAmount < MAX_AUDIO_BACKLOG) client.ws.send(pcm, { binary: true });
577
+ };
578
+ relay.onFailed = (reason) => {
579
+ if (client.rt !== rt) return;
580
+ endRelay(client, true);
581
+ send(client, { t: "rt", key, ev: { type: "closed", reason } });
582
+ };
583
+ const answer = await hub.realtimeStart(key, offer, opts);
584
+ if (client.rt !== rt) throw new Error("已挂断");
585
+ await relay.accept(answer);
586
+ ack(true, { rate: RELAY_RATE });
587
+ })().catch((err) => {
588
+ if (client.rt === rt) endRelay(client, true);
589
+ ack(false, { error: err instanceof Error ? err.message : String(err) });
590
+ });
546
591
  return;
547
592
  }
548
593
  if (msg.op === "stop") {
594
+ if (client.rt?.key === key) endRelay(client, false);
549
595
  void hub.realtimeStop(key).then(() => ack(true), (err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
550
596
  return;
551
597
  }
@@ -653,15 +699,27 @@ export async function startWeb(): Promise<void> {
653
699
  const client: Client = { ws, subs: new Set(), alive: true, token, user, visible: false, viewing: null, foreground: false, pushEndpoint: null, seen: Date.now() };
654
700
  clients.add(client);
655
701
  ws.on("pong", () => (client.alive = true));
656
- ws.on("message", (data) => {
702
+ ws.on("message", (data, isBinary) => {
657
703
  try {
704
+ // 二进制帧只有一种:语音对话里麦克风的 PCM
705
+ if (isBinary) {
706
+ const chunk = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as Buffer);
707
+ if (chunk.length <= MAX_AUDIO_CHUNK) client.rt?.relay?.pushAudio(chunk);
708
+ return;
709
+ }
658
710
  handleClientMessage(client, data.toString());
659
711
  } catch (err) {
660
712
  console.error("[web] 处理 WebSocket 消息出错:", err);
661
713
  }
662
714
  });
663
- ws.on("close", () => clients.delete(client));
664
- ws.on("error", () => clients.delete(client));
715
+ ws.on("close", () => {
716
+ clients.delete(client);
717
+ endRelay(client, true);
718
+ });
719
+ ws.on("error", () => {
720
+ clients.delete(client);
721
+ endRelay(client, true);
722
+ });
665
723
  send(client, { t: "hello", seq: hub.currentSeq(), epoch: EPOCH, version: RUNNING_VERSION });
666
724
  });
667
725
  } catch (err) {
package/src/core/hub.ts CHANGED
@@ -50,6 +50,7 @@ import type {
50
50
  InteractionSpec,
51
51
  ModelInfo,
52
52
  RealtimeEvent,
53
+ RealtimeOptions,
53
54
  RuntimeInfo,
54
55
  SessionSummary,
55
56
  StepKind,
@@ -2057,7 +2058,7 @@ class Hub extends EventEmitter<{
2057
2058
  // ---------- 实时语音对话(只有 Codex) ----------
2058
2059
 
2059
2060
  /** 开始通话:offer 是浏览器的 WebRTC SDP,返回对端 answer。会话没拉起来就按绑定拉起 */
2060
- async realtimeStart(key: string, offer: string): Promise<string> {
2061
+ async realtimeStart(key: string, offer: string, opts?: RealtimeOptions): Promise<string> {
2061
2062
  const conv = getConversation(key);
2062
2063
  if (!conv) throw new Error("会话不存在");
2063
2064
  if (conv.agent !== "codex") throw new Error("只有 Codex 的对话能语音对话");
@@ -2065,7 +2066,7 @@ class Hub extends EventEmitter<{
2065
2066
  const session = this.session(key);
2066
2067
  if (!session.startRealtime) throw new Error("这个 agent 不支持语音对话");
2067
2068
  touchConversation(key);
2068
- return session.startRealtime(offer);
2069
+ return session.startRealtime(offer, opts);
2069
2070
  }
2070
2071
 
2071
2072
  async realtimeStop(key: string): Promise<void> {
@@ -0,0 +1,142 @@
1
+ /**
2
+ * 实时语音的中转:服务器自己当 WebRTC 端点连 OpenAI,浏览器那头走现有的 WebSocket 收发 PCM。
3
+ * 这样声音和网页走同一条路(反向代理、Cloudflare Tunnel 都能过),浏览器不用直连 OpenAI。
4
+ *
5
+ * 浏览器 ⇄ 这里:16 位小端、单声道、24kHz 的 PCM(每块多长都行)。
6
+ * 这里 ⇄ OpenAI:Opus(RTP 时钟 48kHz,每包 20ms),加上 OpenAI 要求的 oai-events 数据通道。
7
+ */
8
+ import OpusScript from "opusscript";
9
+ import { MediaStreamTrack, RTCPeerConnection, RTCRtpCodecParameters, RtpHeader, RtpPacket } from "werift";
10
+
11
+ export const RELAY_RATE = 24000;
12
+ /** 20ms 一帧:24kHz 下 480 个采样;RTP 时间戳按 48kHz 走,一帧加 960 */
13
+ const FRAME = 480;
14
+ const FRAME_BYTES = FRAME * 2;
15
+ const RTP_STEP = 960;
16
+ /** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
17
+ const MAX_PENDING = RELAY_RATE * 2;
18
+
19
+ export class RealtimeRelay {
20
+ /** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
21
+ onAudio: (pcm: Buffer) => void = () => {};
22
+ /** 和 OpenAI 的连接断了(不是 close() 主动关的) */
23
+ onFailed: (reason: string) => void = () => {};
24
+
25
+ private readonly pc: RTCPeerConnection;
26
+ private readonly track = new MediaStreamTrack({ kind: "audio" });
27
+ private readonly encoder = new OpusScript(RELAY_RATE, 1, OpusScript.Application.VOIP);
28
+ private readonly decoder = new OpusScript(RELAY_RATE, 1, OpusScript.Application.VOIP);
29
+ private pending = Buffer.alloc(0);
30
+ private seq = Math.floor(Math.random() * 0xffff);
31
+ private ts = Math.floor(Math.random() * 0xffffffff);
32
+ private closed = false;
33
+ private dropTimer: NodeJS.Timeout | undefined;
34
+
35
+ private constructor() {
36
+ this.pc = new RTCPeerConnection({
37
+ codecs: { audio: [new RTCRtpCodecParameters({ mimeType: "audio/opus", clockRate: 48000, channels: 2 })] },
38
+ });
39
+ const transceiver = this.pc.addTransceiver(this.track, { direction: "sendrecv" });
40
+ transceiver.onTrack.subscribe((remote) => {
41
+ remote.onReceiveRtp.subscribe((rtp) => {
42
+ if (this.closed || !rtp.payload.length) return;
43
+ try {
44
+ this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
45
+ } catch {
46
+ // 坏包丢掉
47
+ }
48
+ });
49
+ });
50
+ // OpenAI 要求 offer 里有这个数据通道(事件我们走 Codex 的通知,用不上它)
51
+ this.pc.createDataChannel("oai-events");
52
+ this.pc.connectionStateChange.subscribe((state) => {
53
+ if (this.closed) return;
54
+ clearTimeout(this.dropTimer);
55
+ if (state === "failed" || state === "closed") this.fail("和语音服务的连接断开了");
56
+ // 网络抖一下会短暂 disconnected,几秒内没恢复才算断
57
+ else if (state === "disconnected") this.dropTimer = setTimeout(() => this.fail("和语音服务的连接断开了"), 5000);
58
+ });
59
+ }
60
+
61
+ /** 建好连接对象和 offer(ICE 候选收齐了写在里面),offer 交给 Codex 去 OpenAI 建通话 */
62
+ static async create(): Promise<{ relay: RealtimeRelay; offer: string }> {
63
+ const relay = new RealtimeRelay();
64
+ await relay.pc.setLocalDescription(await relay.pc.createOffer());
65
+ await relay.gathered(1500);
66
+ const offer = relay.pc.localDescription?.sdp;
67
+ if (!offer) {
68
+ relay.close();
69
+ throw new Error("没能建立语音连接");
70
+ }
71
+ return { relay, offer };
72
+ }
73
+
74
+ /** OpenAI 的 answer 回来了:连上 */
75
+ async accept(answer: string): Promise<void> {
76
+ if (this.closed) throw new Error("已挂断");
77
+ await this.pc.setRemoteDescription({ type: "answer", sdp: answer });
78
+ }
79
+
80
+ /** 浏览器发来的麦克风 PCM:凑够 20ms 就编码发出去 */
81
+ pushAudio(pcm: Buffer): void {
82
+ if (this.closed || !pcm.length) return;
83
+ this.pending = this.pending.length ? Buffer.concat([this.pending, pcm]) : Buffer.from(pcm);
84
+ if (this.pending.length > MAX_PENDING) this.pending = this.pending.subarray(this.pending.length - MAX_PENDING);
85
+ while (this.pending.length >= FRAME_BYTES) {
86
+ const frame = this.pending.subarray(0, FRAME_BYTES);
87
+ this.pending = this.pending.subarray(FRAME_BYTES);
88
+ this.sendFrame(frame);
89
+ }
90
+ }
91
+
92
+ close(): void {
93
+ if (this.closed) return;
94
+ this.closed = true;
95
+ clearTimeout(this.dropTimer);
96
+ this.pending = Buffer.alloc(0);
97
+ void this.pc.close().catch(() => {});
98
+ for (const codec of [this.encoder, this.decoder]) {
99
+ try {
100
+ codec.delete();
101
+ } catch {
102
+ // 已经释放
103
+ }
104
+ }
105
+ }
106
+
107
+ private sendFrame(frame: Buffer): void {
108
+ let payload: Buffer;
109
+ try {
110
+ payload = this.encoder.encode(frame, FRAME);
111
+ } catch {
112
+ return;
113
+ }
114
+ const header = new RtpHeader({ sequenceNumber: this.seq, timestamp: this.ts, marker: false, payloadType: 111 });
115
+ this.seq = (this.seq + 1) & 0xffff;
116
+ this.ts = (this.ts + RTP_STEP) >>> 0;
117
+ try {
118
+ this.track.writeRtp(new RtpPacket(header, Buffer.from(payload)));
119
+ } catch {
120
+ // 还没连上 / 已经关了
121
+ }
122
+ }
123
+
124
+ private fail(reason: string): void {
125
+ if (this.closed) return;
126
+ this.close();
127
+ this.onFailed(reason);
128
+ }
129
+
130
+ private gathered(ms: number): Promise<void> {
131
+ if (this.pc.iceGatheringState === "complete") return Promise.resolve();
132
+ return new Promise((resolve) => {
133
+ const timer = setTimeout(done, ms);
134
+ const sub = this.pc.iceGatheringStateChange.subscribe((state) => state === "complete" && done());
135
+ function done(): void {
136
+ clearTimeout(timer);
137
+ sub.unSubscribe();
138
+ resolve();
139
+ }
140
+ });
141
+ }
142
+ }
package/src/core/types.ts CHANGED
@@ -144,6 +144,9 @@ export type AgentEvents = {
144
144
  onRealtime?: (ev: RealtimeEvent) => void;
145
145
  };
146
146
 
147
+ /** 实时语音对话的音色和语言(页面上选的,没选就用 Codex 默认) */
148
+ export type RealtimeOptions = { voice?: string; language?: string };
149
+
147
150
  /** 实时语音对话的事件(Codex 的 thread/realtime/*) */
148
151
  export type RealtimeEvent =
149
152
  | { type: "started" }
@@ -315,7 +318,7 @@ export interface AgentSession {
315
318
  /** 开关计划模式。Claude 立即生效,Codex 下一轮生效 */
316
319
  setPlan?(on: boolean): Promise<void>;
317
320
  /** 开始实时语音对话(只有 Codex):offer 是浏览器的 WebRTC SDP,返回对端的 answer */
318
- startRealtime?(offer: string): Promise<string>;
321
+ startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
319
322
  /** 挂断实时语音对话;没在通话就什么都不做 */
320
323
  stopRealtime?(): Promise<void>;
321
324
  }
@@ -679,6 +679,44 @@ textarea.sending {
679
679
  background: color-mix(in srgb, var(--danger) 12%, transparent);
680
680
  }
681
681
 
682
+ .call-ctl.active {
683
+ color: var(--accent);
684
+ background: color-mix(in srgb, var(--accent) 12%, transparent);
685
+ }
686
+
687
+ .call-opts {
688
+ display: flex;
689
+ flex-wrap: wrap;
690
+ align-items: center;
691
+ gap: 8px 16px;
692
+ margin-top: 10px;
693
+ padding-top: 10px;
694
+ border-top: 1px solid color-mix(in srgb, var(--border) 70%, transparent);
695
+ }
696
+
697
+ .call-opts[hidden] {
698
+ display: none;
699
+ }
700
+
701
+ .call-opt {
702
+ display: inline-flex;
703
+ align-items: center;
704
+ gap: 6px;
705
+ font-size: 13px;
706
+ color: var(--fg-soft);
707
+ }
708
+
709
+ .call-opt select {
710
+ font-size: 13px;
711
+ max-width: 180px;
712
+ }
713
+
714
+ .call-opts-hint {
715
+ flex-basis: 100%;
716
+ font-size: 12px;
717
+ color: var(--fg-faint);
718
+ }
719
+
682
720
  .call-ctl.call-end {
683
721
  padding: 0 16px;
684
722
  font-size: 14px;
@@ -0,0 +1,110 @@
1
+ // 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
2
+ // rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
3
+ // rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
4
+
5
+ const RATE = 24000;
6
+ const CHUNK = 480; // 20ms
7
+
8
+ class Capture extends AudioWorkletProcessor {
9
+ constructor() {
10
+ super();
11
+ this.step = sampleRate / RATE; // 每出一个 24kHz 采样,输入前进多少
12
+ this.pos = 0; // 下一个输出采样在输入里的位置(相对 prev 之后)
13
+ this.prev = 0; // 上一块的最后一个采样,插值跨块用
14
+ this.out = new Int16Array(CHUNK);
15
+ this.n = 0;
16
+ }
17
+
18
+ process(inputs) {
19
+ const input = inputs[0]?.[0];
20
+ if (!input) return true;
21
+ // 多声道输入只取第一路就够(getUserMedia 一般是单声道)
22
+ const len = input.length;
23
+ // 位置 -1 对应 prev,0..len-1 对应 input
24
+ while (this.pos < len) {
25
+ const i = Math.floor(this.pos);
26
+ const f = this.pos - i;
27
+ const a = i < 0 ? this.prev : input[i];
28
+ const b = i + 1 < len ? input[i + 1] : input[len - 1];
29
+ let v = a + (b - a) * f;
30
+ v = v < -1 ? -1 : v > 1 ? 1 : v;
31
+ this.out[this.n++] = v < 0 ? v * 0x8000 : v * 0x7fff;
32
+ if (this.n === CHUNK) {
33
+ this.port.postMessage(this.out.buffer, [this.out.buffer]);
34
+ this.out = new Int16Array(CHUNK);
35
+ this.n = 0;
36
+ }
37
+ this.pos += this.step;
38
+ }
39
+ this.pos -= len;
40
+ this.prev = input[len - 1];
41
+ return true;
42
+ }
43
+ }
44
+
45
+ class Play extends AudioWorkletProcessor {
46
+ constructor() {
47
+ super();
48
+ this.step = RATE / sampleRate; // 每出一个输出采样,24kHz 的缓冲前进多少
49
+ this.size = RATE * 4; // 环形缓冲最多 4 秒
50
+ this.buf = new Float32Array(this.size);
51
+ this.read = 0; // 读位置(小数)
52
+ this.avail = 0; // 还没放的采样数
53
+ this.write = 0;
54
+ this.playing = false;
55
+ this.port.onmessage = (e) => {
56
+ if (e.data === "flush") {
57
+ this.avail = 0;
58
+ this.playing = false;
59
+ return;
60
+ }
61
+ const pcm = new Int16Array(e.data);
62
+ // 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
63
+ const over = this.avail + pcm.length - this.size;
64
+ if (over > 0) {
65
+ this.read = (this.read + over) % this.size;
66
+ this.avail -= over;
67
+ }
68
+ for (let i = 0; i < pcm.length; i++) {
69
+ this.buf[this.write] = pcm[i] / 0x8000;
70
+ this.write = (this.write + 1) % this.size;
71
+ }
72
+ this.avail += pcm.length;
73
+ // 太多了(超过 1 秒)说明延迟越积越大,追到只剩 0.2 秒
74
+ if (this.avail > RATE) {
75
+ const skip = this.avail - RATE / 5;
76
+ this.read = (this.read + skip) % this.size;
77
+ this.avail -= skip;
78
+ }
79
+ };
80
+ }
81
+
82
+ process(_inputs, outputs) {
83
+ const out = outputs[0];
84
+ const ch = out[0];
85
+ if (!ch) return true;
86
+ // 先攒 60ms 再开始放,扛一点网络抖动;放空了就停下来重新攒
87
+ if (!this.playing && this.avail >= RATE * 0.06) this.playing = true;
88
+ for (let i = 0; i < ch.length; i++) {
89
+ if (!this.playing || this.avail < 2) {
90
+ ch[i] = 0;
91
+ this.playing = false;
92
+ continue;
93
+ }
94
+ const i0 = Math.floor(this.read);
95
+ const f = this.read - i0;
96
+ const a = this.buf[i0 % this.size];
97
+ const b = this.buf[(i0 + 1) % this.size];
98
+ ch[i] = a + (b - a) * f;
99
+ const next = this.read + this.step;
100
+ const used = Math.floor(next) - i0;
101
+ this.avail -= used;
102
+ this.read = next % this.size;
103
+ }
104
+ for (let c = 1; c < out.length; c++) out[c].set(ch);
105
+ return true;
106
+ }
107
+ }
108
+
109
+ registerProcessor("rt-capture", Capture);
110
+ registerProcessor("rt-play", Play);
package/web/lib/ws.js CHANGED
@@ -11,7 +11,7 @@ import { setBusy } from "../views/composer.js";
11
11
  import { SH, shownHere } from "./shell.js";
12
12
  import { currentPushSub } from "./push.js";
13
13
  import { renderScheduled } from "../views/schedule.js";
14
- import { onRealtimeEvent } from "../views/voice-call.js";
14
+ import { onRealtimeAudio, onRealtimeEvent, onSocketLost } from "../views/voice-call.js";
15
15
  import { reloadExperts } from "../views/experts.js";
16
16
 
17
17
  // ---------- WebSocket ----------
@@ -34,6 +34,8 @@ export const WS = {
34
34
  if (this.sock && (this.sock.readyState === 0 || this.sock.readyState === 1)) return;
35
35
  const proto = location.protocol === "https:" ? "wss" : "ws";
36
36
  const sock = new WebSocket(`${proto}://${location.host}/ws`);
37
+ // 二进制帧只有语音对话的声音
38
+ sock.binaryType = "arraybuffer";
37
39
  this.sock = sock;
38
40
  sock.onopen = () => {
39
41
  this.connected = true;
@@ -42,10 +44,11 @@ export const WS = {
42
44
  clearInterval(this.pingTimer);
43
45
  this.pingTimer = setInterval(() => this.raw({ t: "ping" }), 25000);
44
46
  };
45
- sock.onmessage = (e) => onWsMessage(JSON.parse(e.data));
47
+ sock.onmessage = (e) => (typeof e.data === "string" ? onWsMessage(JSON.parse(e.data)) : onRealtimeAudio(e.data));
46
48
  sock.onclose = async (e) => {
47
49
  this.connected = false;
48
50
  clearInterval(this.pingTimer);
51
+ if (this.sock === sock) onSocketLost();
49
52
  for (const w of this.waiters.values()) w.reject(new Error("连接已断开"));
50
53
  this.waiters.clear();
51
54
  if (this.sock !== sock) return;
@@ -6,35 +6,89 @@ import { currentAgentId } from "./slash.js";
6
6
  import { cancelVoice } from "./voice.js";
7
7
 
8
8
  // ---------- 实时语音对话(只有 Codex) ----------
9
- // 浏览器建 RTCPeerConnection(麦克风 + oai-events 数据通道),offer 交给服务端,服务端让 Codex app-server 去 OpenAI 建通话,
10
- // answer 拿回来连上。声音在浏览器和 OpenAI 之间直连,不经过 agent-loft。
9
+ // 声音经服务器中转:服务器自己和 OpenAI 建 WebRTC(core/realtime-relay.ts),这里和服务器之间走现有的 WebSocket,
10
+ // 两头都是 24kHz、16 位、单声道的 PCM(二进制帧)。采集、播放在 AudioWorklet 里做(lib/rt-worklet.js)。
11
+ // 这样能打开网页的地方就能语音(反向代理、Cloudflare Tunnel 都能过),浏览器不直连 OpenAI。
11
12
  // 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
12
13
  // 通话时输入框换成通话条:音量动画、状态、字幕、静音、挂断(和 Codex 桌面端的语音模式一样)。
13
14
 
14
15
  /** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
15
- const supported = !desktop && !!window.RTCPeerConnection && !!navigator.mediaDevices?.getUserMedia;
16
+ const supported = !desktop && !!window.AudioWorkletNode && !!navigator.mediaDevices?.getUserMedia;
17
+
18
+ /**
19
+ * 回复的声音直接从 AudioContext 放出来,iOS / Safari 的回声消除照样管得到;Chrome 等的回声消除只认 WebRTC 放出来的声音,
20
+ * 不绕一下模型会听到自己说话、自己打断自己:在本页里用一对 RTCPeerConnection 环回一下,再交给 <audio> 放
21
+ */
22
+ const ua = navigator.userAgent;
23
+ const appleAudio = /iPad|iPhone|iPod/.test(ua) || (/Macintosh/.test(ua) && navigator.maxTouchPoints > 1) || (/Safari\//.test(ua) && !/Chrome|Chromium|Edg|OPR|Firefox/.test(ua));
16
24
 
17
25
  const call = {
18
26
  /** idle / connecting / live */
19
27
  state: "idle",
20
28
  key: null,
21
- pc: null,
22
29
  mic: null,
30
+ /** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
31
+ play: null,
32
+ /** Chrome 等的回声消除环回(见 appleAudio) */
33
+ loop: null,
23
34
  audio: null,
24
35
  muted: false,
25
36
  startedAt: 0,
26
37
  raf: 0,
27
- ctx: null,
38
+ /** 音量表接上的节点,挂断时断开(AudioContext 本身留着,见 audioCtx) */
39
+ nodes: [],
28
40
  micLevel: null,
29
41
  outLevel: null,
30
42
  /** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
31
43
  caption: { role: null, text: "", done: false },
32
44
  panel: null,
33
45
  els: null,
34
- /** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
35
- dropTimer: 0,
36
46
  };
37
47
 
48
+ // ---------- 音色和语言(按设备记在浏览器里) ----------
49
+
50
+ const PREFS_KEY = "agent-loft.voiceCall";
51
+ /** 和服务端 REALTIME_VOICES 一致(就是 ChatGPT 语音模式那几个);空 = Codex 默认(cove) */
52
+ const VOICES = [
53
+ ["", "默认(Cove)"],
54
+ ["cove", "Cove · 男声,沉着直接"],
55
+ ["juniper", "Juniper · 女声,开朗"],
56
+ ["maple", "Maple · 女声,爽朗坦率"],
57
+ ["vale", "Vale · 女声,明亮好奇"],
58
+ ["sol", "Sol · 女声,从容干练"],
59
+ ["spruce", "Spruce · 男声,平和"],
60
+ ["ember", "Ember · 男声,自信乐观"],
61
+ ["arbor", "Arbor · 男声,随和"],
62
+ ["breeze", "Breeze · 活泼真诚"],
63
+ ];
64
+ /** 和服务端 REALTIME_LANGUAGES 一致;空 = 跟着你说的语言 */
65
+ const LANGUAGES = [
66
+ ["", "跟着我说的语言"],
67
+ ["zh-CN", "普通话(简体)"],
68
+ ["zh-TW", "国语(繁体)"],
69
+ ["yue", "粤语"],
70
+ ["en", "English"],
71
+ ["ja", "日本語"],
72
+ ["ko", "한국어"],
73
+ ];
74
+
75
+ function loadPrefs() {
76
+ try {
77
+ const p = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "{}");
78
+ return { voice: typeof p.voice === "string" ? p.voice : "", language: typeof p.language === "string" ? p.language : "" };
79
+ } catch {
80
+ return { voice: "", language: "" };
81
+ }
82
+ }
83
+
84
+ function savePrefs(p) {
85
+ try {
86
+ localStorage.setItem(PREFS_KEY, JSON.stringify(p));
87
+ } catch {
88
+ // 隐私模式等存不了:这次通话照样用
89
+ }
90
+ }
91
+
38
92
  const ICON_WAVE =
39
93
  '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
40
94
  const ICON_MIC_OFF =
@@ -76,6 +130,8 @@ async function startCall() {
76
130
  // 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
77
131
  call.audio = h("audio", { autoplay: true, playsinline: true });
78
132
  call.audio.play().catch(() => {});
133
+ // AudioContext 也要在点击里建 / 恢复:iOS 上点击之外起的一直是 suspended,第二通起就没声音、录不到
134
+ audioCtx();
79
135
  mountPanel();
80
136
  try {
81
137
  const key = await ensureConversation();
@@ -83,31 +139,53 @@ async function startCall() {
83
139
  call.key = key;
84
140
  call.mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
85
141
  if (call.state === "idle") return stopTracks();
86
- const pc = new RTCPeerConnection();
87
- call.pc = pc;
88
- for (const track of call.mic.getTracks()) pc.addTrack(track, call.mic);
89
- // OpenAI 要求 offer 里有这个数据通道(事件走这里;字幕我们用服务端转来的)
90
- pc.createDataChannel("oai-events");
91
- pc.ontrack = (e) => {
92
- const stream = e.streams[0] ?? new MediaStream([e.track]);
93
- call.audio.srcObject = stream;
94
- call.audio.play().catch(() => {});
95
- call.outLevel = meter(stream);
96
- };
97
- pc.onconnectionstatechange = () => {
98
- const st = pc.connectionState;
99
- clearTimeout(call.dropTimer);
100
- if (st === "failed" || st === "closed") endCall("语音连接断开了");
101
- // 网络抖一下会短暂 disconnected,几秒内没恢复才算断
102
- else if (st === "disconnected") call.dropTimer = setTimeout(() => endCall("语音连接断开了"), 5000);
142
+ // iOS 上偶尔拿到的是已经结束 / 被静音的轨道(音频会话被别的占着):别假装在听
143
+ const track = call.mic.getAudioTracks()[0];
144
+ if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
145
+ if (track.muted) await waitUnmute(track, 1500);
146
+ const ctx = audioCtx();
147
+ if (!ctx) throw new Error("浏览器不支持音频处理");
148
+ await loadWorklet(ctx);
149
+ if (call.state === "idle") return stopTracks();
150
+ // 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
151
+ const src = ctx.createMediaStreamSource(call.mic);
152
+ const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
153
+ // 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
154
+ const sink = ctx.createGain();
155
+ sink.gain.value = 0;
156
+ src.connect(capture);
157
+ capture.connect(sink);
158
+ sink.connect(ctx.destination);
159
+ capture.port.onmessage = (e) => {
160
+ const sock = WS.sock;
161
+ // 网络卡住时别越攒越多(攒下的说出去也晚了)
162
+ if (call.state === "live" && sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
103
163
  };
104
- call.micLevel = meter(call.mic);
105
- await pc.setLocalDescription(await pc.createOffer());
106
- await iceGathered(pc, 1500);
164
+ // 服务器发来的 → 播放 → 扬声器
165
+ const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
166
+ call.play = play;
167
+ call.nodes.push(src, capture, sink, play);
168
+ call.micLevel = meter(src);
169
+ call.outLevel = meter(play);
170
+ if (appleAudio) {
171
+ play.connect(ctx.destination);
172
+ } else {
173
+ const dest = ctx.createMediaStreamDestination();
174
+ play.connect(dest);
175
+ call.nodes.push(dest);
176
+ try {
177
+ call.loop = await loopback(dest.stream);
178
+ call.audio.srcObject = call.loop.stream;
179
+ call.audio.play().catch(() => {});
180
+ } catch {
181
+ // 环回建不起来:直接放(可能有回声)
182
+ play.connect(ctx.destination);
183
+ }
184
+ }
107
185
  if (call.state === "idle") return;
108
- const res = await WS.request({ t: "rt", op: "start", key, sdp: pc.localDescription.sdp }, 30_000);
186
+ const prefs = loadPrefs();
187
+ await WS.request({ t: "rt", op: "start", key, voice: prefs.voice, language: prefs.language }, 40_000);
109
188
  if (call.state === "idle") return;
110
- await pc.setRemoteDescription({ type: "answer", sdp: res.sdp });
111
189
  call.state = "live";
112
190
  call.startedAt = Date.now();
113
191
  render();
@@ -123,15 +201,26 @@ export function endCall(reason, fromServer = false) {
123
201
  const key = call.key;
124
202
  call.state = "idle";
125
203
  if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
126
- clearTimeout(call.dropTimer);
127
204
  cancelAnimationFrame(call.raf);
128
- call.pc?.close();
129
- call.pc = null;
130
205
  stopTracks();
131
- if (call.audio) call.audio.srcObject = null;
206
+ call.loop?.close();
207
+ call.loop = null;
208
+ call.play = null;
209
+ if (call.audio) {
210
+ call.audio.pause();
211
+ call.audio.srcObject = null;
212
+ }
132
213
  call.audio = null;
133
- call.ctx?.close().catch(() => {});
134
- call.ctx = null;
214
+ // 不 close:iOS 上关掉接过麦克风的 AudioContext,之后再开麦克风常常是静音的。断开节点、先暂停,下次点击时恢复
215
+ for (const n of call.nodes) {
216
+ try {
217
+ n.disconnect();
218
+ } catch {
219
+ // 已经断开
220
+ }
221
+ }
222
+ call.nodes = [];
223
+ if (sharedCtx?.state === "running") sharedCtx.suspend().catch(() => {});
135
224
  call.micLevel = call.outLevel = null;
136
225
  call.key = null;
137
226
  call.panel?.remove();
@@ -179,12 +268,14 @@ function mountPanel() {
179
268
  if (!call.panel) {
180
269
  const bars = [0, 1, 2, 3, 4].map(() => h("span", { class: "call-bar" }));
181
270
  const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
271
+ const gear = h("button", { class: "call-ctl", title: "音色和语言", "aria-label": "音色和语言", "aria-expanded": "false", onclick: toggleOpts }, icon("gear"));
182
272
  const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "结束");
183
273
  const status = h("span", { class: "call-status" });
184
274
  const time = h("span", { class: "call-time" });
185
275
  const who = h("span", { class: "call-who" });
186
276
  const text = h("span", { class: "call-text" });
187
- call.els = { bars, mute, status, time, who, text };
277
+ const opts = optsRow();
278
+ call.els = { bars, mute, gear, opts, status, time, who, text };
188
279
  call.panel = h(
189
280
  "div",
190
281
  { class: "call-panel", role: "region", "aria-label": "语音对话" },
@@ -193,8 +284,9 @@ function mountPanel() {
193
284
  { class: "call-main" },
194
285
  h("div", { class: "call-wave", "aria-hidden": "true" }, bars),
195
286
  h("div", { class: "call-info" }, h("div", { class: "call-line" }, status, time), h("div", { class: "call-caption", "aria-live": "polite" }, who, text)),
196
- h("div", { class: "call-ctls" }, mute, end),
287
+ h("div", { class: "call-ctls" }, mute, gear, end),
197
288
  ),
289
+ opts,
198
290
  );
199
291
  }
200
292
  ui.composer.classList.add("in-call");
@@ -204,6 +296,43 @@ function mountPanel() {
204
296
  call.raf = requestAnimationFrame(tick);
205
297
  }
206
298
 
299
+ /** 通话条下面展开的一行:音色、语言。通话中改了就用新设置重新连一次(Codex 那边对话上下文还在) */
300
+ function optsRow() {
301
+ const prefs = loadPrefs();
302
+ const pick = (label, list, value, name) => {
303
+ const sel = h("select", { "aria-label": label }, list.map(([v, t]) => h("option", { value: v }, t)));
304
+ sel.value = list.some(([v]) => v === value) ? value : "";
305
+ sel.addEventListener("change", () => {
306
+ savePrefs({ ...loadPrefs(), [name]: sel.value });
307
+ restartCall();
308
+ });
309
+ return h("label", { class: "call-opt" }, h("span", {}, label), sel);
310
+ };
311
+ return h(
312
+ "div",
313
+ { class: "call-opts", hidden: true },
314
+ pick("音色", VOICES, prefs.voice, "voice"),
315
+ pick("语言", LANGUAGES, prefs.language, "language"),
316
+ h("div", { class: "call-opts-hint" }, "记在这台设备上。通话中改了会重新连接,对话内容不受影响"),
317
+ );
318
+ }
319
+
320
+ function toggleOpts() {
321
+ const { opts, gear } = call.els;
322
+ opts.hidden = !opts.hidden;
323
+ gear.classList.toggle("active", !opts.hidden);
324
+ gear.setAttribute("aria-expanded", String(!opts.hidden));
325
+ }
326
+
327
+ /** 用新的音色 / 语言重连:挂掉当前这通,马上再开(在选择框的操作里,iOS 也允许播放) */
328
+ function restartCall() {
329
+ if (call.state === "idle") return;
330
+ endCall("");
331
+ void startCall();
332
+ // startCall 同步就把新的通话条挂上了:设置行保持展开,方便接着试别的音色
333
+ if (call.els) toggleOpts();
334
+ }
335
+
207
336
  function toggleMute() {
208
337
  call.muted = !call.muted;
209
338
  for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
@@ -255,15 +384,42 @@ function clock(ms) {
255
384
  return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
256
385
  }
257
386
 
258
- /** 一路声音的音量(0~1 的均方根),给动画用 */
259
- function meter(stream) {
387
+ /** 整个页面共用一个 AudioContext(iOS 上反复新建、关闭会把麦克风搞坏),在点击里调用来建 / 恢复 */
388
+ let sharedCtx = null;
389
+ function audioCtx() {
260
390
  try {
261
- call.ctx ??= new AudioContext();
262
- void call.ctx.resume();
263
- const src = call.ctx.createMediaStreamSource(stream);
264
- const an = call.ctx.createAnalyser();
391
+ if (!sharedCtx || sharedCtx.state === "closed") sharedCtx = new (window.AudioContext ?? window.webkitAudioContext)();
392
+ if (sharedCtx.state !== "running") sharedCtx.resume().catch(() => {});
393
+ } catch {
394
+ sharedCtx = null;
395
+ }
396
+ return sharedCtx;
397
+ }
398
+
399
+ /** 轨道 muted 时等它恢复一会儿;还是不行就报错 */
400
+ function waitUnmute(track, ms) {
401
+ return new Promise((resolve, reject) => {
402
+ const timer = setTimeout(() => {
403
+ track.removeEventListener("unmute", ok);
404
+ reject(new Error("麦克风被系统静音了,请检查是否有其他应用在用麦克风,或刷新页面后重试"));
405
+ }, ms);
406
+ const ok = () => {
407
+ clearTimeout(timer);
408
+ resolve();
409
+ };
410
+ track.addEventListener("unmute", ok, { once: true });
411
+ });
412
+ }
413
+
414
+ /** 一个节点的音量(0~1 的均方根),给动画用 */
415
+ function meter(node) {
416
+ try {
417
+ const ctx = audioCtx();
418
+ if (!ctx) return null;
419
+ const an = ctx.createAnalyser();
265
420
  an.fftSize = 512;
266
- src.connect(an);
421
+ node.connect(an);
422
+ call.nodes.push(an);
267
423
  const buf = new Float32Array(an.fftSize);
268
424
  return () => {
269
425
  an.getFloatTimeDomainData(buf);
@@ -276,18 +432,54 @@ function meter(stream) {
276
432
  }
277
433
  }
278
434
 
279
- /** 等 ICE 候选收集完(最多 ms 毫秒):offer 里带上候选,不用另外传 */
280
- function iceGathered(pc, ms) {
281
- if (pc.iceGatheringState === "complete") return Promise.resolve();
282
- return new Promise((resolve) => {
283
- const done = () => {
284
- pc.removeEventListener("icegatheringstatechange", check);
285
- resolve();
286
- };
287
- const check = () => pc.iceGatheringState === "complete" && done();
288
- pc.addEventListener("icegatheringstatechange", check);
289
- setTimeout(done, ms);
290
- });
435
+ /** 每个 AudioContext 加载一次 worklet */
436
+ const workletLoaded = new WeakMap();
437
+ function loadWorklet(ctx) {
438
+ let p = workletLoaded.get(ctx);
439
+ if (!p) {
440
+ p = ctx.audioWorklet.addModule(new URL("../lib/rt-worklet.js", import.meta.url));
441
+ p.catch(() => workletLoaded.delete(ctx));
442
+ workletLoaded.set(ctx, p);
443
+ }
444
+ return p;
445
+ }
446
+
447
+ /** 本页里一对 RTCPeerConnection 把声音环回一下,让浏览器的回声消除认得它(见 appleAudio) */
448
+ async function loopback(stream) {
449
+ const a = new RTCPeerConnection();
450
+ const b = new RTCPeerConnection();
451
+ const close = () => {
452
+ a.close();
453
+ b.close();
454
+ };
455
+ try {
456
+ a.onicecandidate = (e) => e.candidate && b.addIceCandidate(e.candidate).catch(() => {});
457
+ b.onicecandidate = (e) => e.candidate && a.addIceCandidate(e.candidate).catch(() => {});
458
+ const got = new Promise((resolve, reject) => {
459
+ b.ontrack = (e) => resolve(e.streams[0] ?? new MediaStream([e.track]));
460
+ setTimeout(() => reject(new Error("环回超时")), 3000);
461
+ });
462
+ for (const t of stream.getTracks()) a.addTrack(t, stream);
463
+ await a.setLocalDescription(await a.createOffer());
464
+ await b.setRemoteDescription(a.localDescription);
465
+ await b.setLocalDescription(await b.createAnswer());
466
+ await a.setRemoteDescription(b.localDescription);
467
+ return { stream: await got, close };
468
+ } catch (err) {
469
+ close();
470
+ throw err;
471
+ }
472
+ }
473
+
474
+ /** 服务器转来的回复声音(WebSocket 二进制帧:24kHz 16 位 PCM) */
475
+ export function onRealtimeAudio(buf) {
476
+ if (call.state === "idle" || !call.play) return;
477
+ call.play.port.postMessage(buf, [buf]);
478
+ }
479
+
480
+ /** WebSocket 断了:声音走不了了,挂断 */
481
+ export function onSocketLost() {
482
+ endCall("和服务器的连接断开了,语音对话已结束", true);
291
483
  }
292
484
 
293
485
  window.addEventListener("pagehide", () => endCall());