agent-loft 0.2.113 → 0.2.115

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,48 +1,109 @@
1
1
  import { h, icon, toast } from "../lib/dom.js";
2
2
  import { desktop, S, ui } from "../state.js";
3
3
  import { WS } from "../lib/ws.js";
4
- import { ensureConversation } from "./composer.js";
4
+ import { ensureConversation, updateSendBtn } from "./composer.js";
5
5
  import { currentAgentId } from "./slash.js";
6
6
  import { cancelVoice } from "./voice.js";
7
7
 
8
8
  // ---------- 实时语音对话(只有 Codex) ----------
9
- // 浏览器建 RTCPeerConnection(麦克风 + oai-events 数据通道),offer 交给服务端,服务端让 Codex app-server 去 OpenAI 建通话,
10
- // answer 拿回来连上。声音在浏览器和 OpenAI 之间直连,不经过 agent-loft。
9
+ // 声音经服务器中转:服务器自己和 OpenAI 建 WebRTC(core/realtime-relay.ts),这里和服务器之间走现有的 WebSocket,
10
+ // 两头都是 24kHz、16 位、单声道的 PCM(二进制帧)。采集、播放在 AudioWorklet 里做(lib/rt-worklet.js)。
11
+ // 这样能打开网页的地方就能语音(反向代理、Cloudflare Tunnel 都能过),浏览器不直连 OpenAI。
11
12
  // 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
12
- // 通话时输入框换成通话条:音量动画、状态、字幕、静音、挂断(和 Codex 桌面端的语音模式一样)。
13
+ // 通话默认打开沉浸式光球界面,也可以收起成输入框位置的通话条。
13
14
 
14
15
  /** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
15
- const supported = !desktop && !!window.RTCPeerConnection && !!navigator.mediaDevices?.getUserMedia;
16
+ const supported = !desktop && !!window.AudioWorkletNode && !!navigator.mediaDevices?.getUserMedia;
17
+
18
+ /**
19
+ * 回复的声音直接从 AudioContext 放出来,iOS / Safari 的回声消除照样管得到;Chrome 等的回声消除只认 WebRTC 放出来的声音,
20
+ * 不绕一下模型会听到自己说话、自己打断自己:在本页里用一对 RTCPeerConnection 环回一下,再交给 <audio> 放
21
+ */
22
+ const ua = navigator.userAgent;
23
+ const appleAudio = /iPad|iPhone|iPod/.test(ua) || (/Macintosh/.test(ua) && navigator.maxTouchPoints > 1) || (/Safari\//.test(ua) && !/Chrome|Chromium|Edg|OPR|Firefox/.test(ua));
16
24
 
17
25
  const call = {
18
26
  /** idle / connecting / live */
19
27
  state: "idle",
20
28
  key: null,
21
- pc: null,
22
29
  mic: null,
30
+ /** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
31
+ play: null,
32
+ /** Chrome 等的回声消除环回(见 appleAudio) */
33
+ loop: null,
23
34
  audio: null,
24
35
  muted: false,
25
36
  startedAt: 0,
26
37
  raf: 0,
27
- ctx: null,
38
+ /** 音量表接上的节点,挂断时断开(AudioContext 本身留着,见 audioCtx) */
39
+ nodes: [],
28
40
  micLevel: null,
29
41
  outLevel: null,
30
42
  /** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
31
43
  caption: { role: null, text: "", done: false },
32
44
  panel: null,
45
+ dialog: null,
46
+ compact: false,
47
+ session: 0,
33
48
  els: null,
34
- /** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
35
- dropTimer: 0,
36
49
  };
37
50
 
51
+ // ---------- 音色和语言(按设备记在浏览器里) ----------
52
+
53
+ const PREFS_KEY = "agent-loft.voiceCall";
54
+ /** 和服务端 REALTIME_VOICES 一致(就是 ChatGPT 语音模式那几个);空 = Codex 默认(cove) */
55
+ const VOICES = [
56
+ ["", "默认(Cove)"],
57
+ ["cove", "Cove · 男声,沉着直接"],
58
+ ["juniper", "Juniper · 女声,开朗"],
59
+ ["maple", "Maple · 女声,爽朗坦率"],
60
+ ["vale", "Vale · 女声,明亮好奇"],
61
+ ["sol", "Sol · 女声,从容干练"],
62
+ ["spruce", "Spruce · 男声,平和"],
63
+ ["ember", "Ember · 男声,自信乐观"],
64
+ ["arbor", "Arbor · 男声,随和"],
65
+ ["breeze", "Breeze · 活泼真诚"],
66
+ ];
67
+ /** 和服务端 REALTIME_LANGUAGES 一致;空 = 跟着你说的语言 */
68
+ const LANGUAGES = [
69
+ ["", "跟着我说的语言"],
70
+ ["zh-CN", "普通话(简体)"],
71
+ ["zh-TW", "国语(繁体)"],
72
+ ["yue", "粤语"],
73
+ ["en", "English"],
74
+ ["ja", "日本語"],
75
+ ["ko", "한국어"],
76
+ ];
77
+
78
+ function loadPrefs() {
79
+ try {
80
+ const p = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "{}");
81
+ return { voice: typeof p.voice === "string" ? p.voice : "", language: typeof p.language === "string" ? p.language : "" };
82
+ } catch {
83
+ return { voice: "", language: "" };
84
+ }
85
+ }
86
+
87
+ function savePrefs(p) {
88
+ try {
89
+ localStorage.setItem(PREFS_KEY, JSON.stringify(p));
90
+ } catch {
91
+ // 隐私模式等存不了:这次通话照样用
92
+ }
93
+ }
94
+
38
95
  const ICON_WAVE =
39
96
  '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
40
97
  const ICON_MIC_OFF =
41
98
  '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 3l18 18M9 9v3a3 3 0 0 0 5.1 2.1M15 9.3V5a3 3 0 0 0-5.9-.6"/><path d="M5 10a7 7 0 0 0 11.7 5.2M19 10a7 7 0 0 1-.6 2.8M12 17v5"/></svg>';
42
99
 
43
100
  /** 输入框里的「语音对话」按钮 */
101
+ export function canCall() {
102
+ return supported && currentAgentId() === "codex";
103
+ }
104
+
44
105
  export function callButton() {
45
- ui.callBtn = h("button", { class: "icon-btn call-btn", title: "语音对话", "aria-label": "语音对话", html: ICON_WAVE, onclick: () => void startCall() });
106
+ ui.callBtn = h("button", { class: "send-btn call-btn", title: "语音对话", "aria-label": "语音对话", html: ICON_WAVE, onclick: () => void startCall() });
46
107
  syncCall();
47
108
  return ui.callBtn;
48
109
  }
@@ -56,11 +117,14 @@ export function syncCall() {
56
117
  if (call.state !== "idle") {
57
118
  if (call.key && S.current !== call.key) {
58
119
  endCall("切换了对话,语音对话已结束");
59
- } else if (call.panel && !call.panel.isConnected) {
60
- mountPanel();
120
+ } else if (call.panel) {
121
+ ui.composer?.classList.add("in-call");
122
+ if (!call.panel.isConnected || (call.compact && call.panel.previousElementSibling !== ui.composer)) {
123
+ mountPanel();
124
+ }
61
125
  }
62
126
  }
63
- if (ui.callBtn) ui.callBtn.hidden = !supported || currentAgentId() !== "codex";
127
+ updateSendBtn();
64
128
  }
65
129
 
66
130
  async function startCall() {
@@ -70,48 +134,84 @@ async function startCall() {
70
134
  return;
71
135
  }
72
136
  cancelVoice();
137
+ const session = ++call.session;
73
138
  call.state = "connecting";
139
+ call.compact = false;
74
140
  call.muted = false;
75
141
  call.caption = { role: null, text: "", done: false };
76
142
  // 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
77
143
  call.audio = h("audio", { autoplay: true, playsinline: true });
78
144
  call.audio.play().catch(() => {});
145
+ // AudioContext 也要在点击里建 / 恢复:iOS 上点击之外起的一直是 suspended,第二通起就没声音、录不到
146
+ audioCtx();
79
147
  mountPanel();
80
148
  try {
81
149
  const key = await ensureConversation();
82
- if (call.state === "idle") return;
150
+ if (session !== call.session) return;
83
151
  call.key = key;
84
- call.mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
85
- if (call.state === "idle") return stopTracks();
86
- const pc = new RTCPeerConnection();
87
- call.pc = pc;
88
- for (const track of call.mic.getTracks()) pc.addTrack(track, call.mic);
89
- // OpenAI 要求 offer 里有这个数据通道(事件走这里;字幕我们用服务端转来的)
90
- pc.createDataChannel("oai-events");
91
- pc.ontrack = (e) => {
92
- const stream = e.streams[0] ?? new MediaStream([e.track]);
93
- call.audio.srcObject = stream;
94
- call.audio.play().catch(() => {});
95
- call.outLevel = meter(stream);
96
- };
97
- pc.onconnectionstatechange = () => {
98
- const st = pc.connectionState;
99
- clearTimeout(call.dropTimer);
100
- if (st === "failed" || st === "closed") endCall("语音连接断开了");
101
- // 网络抖一下会短暂 disconnected,几秒内没恢复才算断
102
- else if (st === "disconnected") call.dropTimer = setTimeout(() => endCall("语音连接断开了"), 5000);
152
+ const mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
153
+ if (session !== call.session) {
154
+ for (const track of mic.getTracks()) track.stop();
155
+ return;
156
+ }
157
+ call.mic = mic;
158
+ // iOS 上偶尔拿到的是已经结束 / 被静音的轨道(音频会话被别的占着):别假装在听
159
+ const track = call.mic.getAudioTracks()[0];
160
+ if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
161
+ track.enabled = !call.muted;
162
+ if (track.muted) await waitUnmute(track, 1500);
163
+ if (session !== call.session) return;
164
+ const ctx = audioCtx();
165
+ if (!ctx) throw new Error("浏览器不支持音频处理");
166
+ await loadWorklet(ctx);
167
+ if (session !== call.session) return;
168
+ // 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
169
+ const src = ctx.createMediaStreamSource(call.mic);
170
+ const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
171
+ // 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
172
+ const sink = ctx.createGain();
173
+ sink.gain.value = 0;
174
+ src.connect(capture);
175
+ capture.connect(sink);
176
+ sink.connect(ctx.destination);
177
+ capture.port.onmessage = (e) => {
178
+ const sock = WS.sock;
179
+ // 网络卡住时别越攒越多(攒下的说出去也晚了)
180
+ if (call.state === "live" && sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
103
181
  };
104
- call.micLevel = meter(call.mic);
105
- await pc.setLocalDescription(await pc.createOffer());
106
- await iceGathered(pc, 1500);
107
- if (call.state === "idle") return;
108
- const res = await WS.request({ t: "rt", op: "start", key, sdp: pc.localDescription.sdp }, 30_000);
109
- if (call.state === "idle") return;
110
- await pc.setRemoteDescription({ type: "answer", sdp: res.sdp });
182
+ // 服务器发来的 → 播放 → 扬声器
183
+ const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
184
+ call.play = play;
185
+ call.nodes.push(src, capture, sink, play);
186
+ call.micLevel = meter(src);
187
+ call.outLevel = meter(play);
188
+ if (appleAudio) {
189
+ play.connect(ctx.destination);
190
+ } else {
191
+ const dest = ctx.createMediaStreamDestination();
192
+ play.connect(dest);
193
+ call.nodes.push(dest);
194
+ try {
195
+ const loop = await loopback(dest.stream);
196
+ if (session !== call.session) return loop.close();
197
+ call.loop = loop;
198
+ call.audio.srcObject = call.loop.stream;
199
+ call.audio.play().catch(() => {});
200
+ } catch {
201
+ // 环回建不起来:直接放(可能有回声)
202
+ if (session !== call.session) return;
203
+ play.connect(ctx.destination);
204
+ }
205
+ }
206
+ if (session !== call.session) return;
207
+ const prefs = loadPrefs();
208
+ await WS.request({ t: "rt", op: "start", key, voice: prefs.voice, language: prefs.language }, 40_000);
209
+ if (session !== call.session) return;
111
210
  call.state = "live";
112
211
  call.startedAt = Date.now();
113
212
  render();
114
213
  } catch (err) {
214
+ if (session !== call.session) return;
115
215
  const message = err?.name === "NotAllowedError" ? "没有麦克风权限,请在浏览器里允许使用麦克风" : err?.name === "NotFoundError" ? "没找到可用的麦克风" : err?.message ?? String(err);
116
216
  endCall(`语音对话没能开始:${message}`);
117
217
  }
@@ -121,23 +221,39 @@ async function startCall() {
121
221
  export function endCall(reason, fromServer = false) {
122
222
  if (call.state === "idle") return;
123
223
  const key = call.key;
224
+ ++call.session;
124
225
  call.state = "idle";
125
226
  if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
126
- clearTimeout(call.dropTimer);
127
227
  cancelAnimationFrame(call.raf);
128
- call.pc?.close();
129
- call.pc = null;
130
228
  stopTracks();
131
- if (call.audio) call.audio.srcObject = null;
229
+ call.loop?.close();
230
+ call.loop = null;
231
+ call.play = null;
232
+ if (call.audio) {
233
+ call.audio.pause();
234
+ call.audio.srcObject = null;
235
+ }
132
236
  call.audio = null;
133
- call.ctx?.close().catch(() => {});
134
- call.ctx = null;
237
+ // 不 close:iOS 上关掉接过麦克风的 AudioContext,之后再开麦克风常常是静音的。断开节点、先暂停,下次点击时恢复
238
+ for (const n of call.nodes) {
239
+ try {
240
+ n.disconnect();
241
+ } catch {
242
+ // 已经断开
243
+ }
244
+ }
245
+ call.nodes = [];
246
+ if (sharedCtx?.state === "running") sharedCtx.suspend().catch(() => {});
135
247
  call.micLevel = call.outLevel = null;
136
248
  call.key = null;
137
249
  call.panel?.remove();
250
+ call.dialog?.close();
251
+ call.dialog?.remove();
252
+ call.dialog = null;
138
253
  call.panel = call.els = null;
139
254
  ui.composer?.classList.remove("in-call");
140
255
  syncCall();
256
+ (ui.callBtn && !ui.callBtn.hidden ? ui.callBtn : ui.sendBtn)?.focus({ preventScroll: true });
141
257
  if (reason) toast(reason);
142
258
  }
143
259
 
@@ -172,38 +288,111 @@ export function onRealtimeEvent(msg) {
172
288
  renderCaption();
173
289
  }
174
290
 
175
- // ---------- 通话条 ----------
291
+ // ---------- 通话界面 ----------
176
292
 
177
293
  function mountPanel() {
178
294
  if (!ui.composer) return;
179
295
  if (!call.panel) {
180
- const bars = [0, 1, 2, 3, 4].map(() => h("span", { class: "call-bar" }));
296
+ const orb = h("div", { class: "call-orb", "aria-hidden": "true" },
297
+ h("div", { class: "call-orb-surface" }, h("span", { class: "call-orb-cloud cloud-a" }), h("span", { class: "call-orb-cloud cloud-b" }), h("span", { class: "call-orb-cloud cloud-c" })),
298
+ );
181
299
  const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
182
- const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "结束");
183
- const status = h("span", { class: "call-status" });
300
+ const gear = h("button", { class: "call-ctl", title: "音色和语言", "aria-label": "音色和语言", "aria-expanded": "false", onclick: toggleOpts }, icon("gear"));
301
+ const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, icon("x"));
302
+ const minimize = h("button", { class: "call-minimize icon-btn", title: "收起通话", "aria-label": "收起通话", onclick: () => setCompact(true) }, icon("down"));
303
+ const expand = h("button", { class: "call-expand call-ctl", title: "展开通话", "aria-label": "展开通话", onclick: () => setCompact(false) }, icon("expand"));
304
+ const status = h("span", { class: "call-status", role: "status" });
184
305
  const time = h("span", { class: "call-time" });
185
306
  const who = h("span", { class: "call-who" });
186
307
  const text = h("span", { class: "call-text" });
187
- call.els = { bars, mute, status, time, who, text };
308
+ const opts = optsRow();
309
+ const hint = h("p", { class: "call-hint" }, "直接开口,随时打断");
310
+ call.els = { orb, mute, gear, opts, status, time, who, text, hint, expand, minimize };
188
311
  call.panel = h(
189
312
  "div",
190
313
  { class: "call-panel", role: "region", "aria-label": "语音对话" },
314
+ h("header", { class: "call-header" },
315
+ h("div", { class: "call-brand" }, h("span", { class: "call-brand-icon", html: ICON_WAVE }), h("span", {}, "Codex", h("small", {}, "语音对话"))),
316
+ h("div", { class: "call-session" }, h("span", { class: "call-live-dot" }), time), minimize,
317
+ ),
191
318
  h(
192
319
  "div",
193
320
  { class: "call-main" },
194
- h("div", { class: "call-wave", "aria-hidden": "true" }, bars),
195
- h("div", { class: "call-info" }, h("div", { class: "call-line" }, status, time), h("div", { class: "call-caption", "aria-live": "polite" }, who, text)),
196
- h("div", { class: "call-ctls" }, mute, end),
321
+ h("div", { class: "call-stage" }, orb),
322
+ h("div", { class: "call-info" }, status, hint),
197
323
  ),
324
+ h("div", { class: "call-caption", "aria-live": "polite", "aria-atomic": "true" }, who, text),
325
+ opts,
326
+ h("footer", { class: "call-footer" }, h("div", { class: "call-ctls" }, expand,
327
+ h("div", { class: "call-control" }, mute, h("span", {}, "麦克风")),
328
+ h("div", { class: "call-control" }, gear, h("span", {}, "音色与语言")),
329
+ h("div", { class: "call-control" }, end, h("span", {}, "结束")),
330
+ ), h("span", { class: "call-footer-note" }, "可以收起通话,查看聊天")),
198
331
  );
332
+ call.dialog = h("dialog", { class: "call-dialog", "aria-label": "Codex 语音对话", oncancel: (e) => { e.preventDefault(); setCompact(true); } });
199
333
  }
200
334
  ui.composer.classList.add("in-call");
201
- ui.composer.after(call.panel);
335
+ if (call.compact) ui.composer.after(call.panel);
336
+ else {
337
+ if (!call.dialog.isConnected) document.body.append(call.dialog);
338
+ call.dialog.append(call.panel);
339
+ if (!call.dialog.open) call.dialog.showModal();
340
+ }
202
341
  render();
203
342
  cancelAnimationFrame(call.raf);
204
343
  call.raf = requestAnimationFrame(tick);
205
344
  }
206
345
 
346
+ /** 收起只切换视图,麦克风和播放保持连接。Esc 也收起,避免误挂断。 */
347
+ function setCompact(compact) {
348
+ if (!call.panel || call.compact === compact) return;
349
+ call.compact = compact;
350
+ call.panel.classList.toggle("compact", compact);
351
+ if (compact) call.dialog.close();
352
+ mountPanel();
353
+ (compact ? call.els.expand : call.els.minimize).focus({ preventScroll: true });
354
+ }
355
+
356
+ /** 通话条下面展开的一行:音色、语言。通话中改了就用新设置重新连一次(Codex 那边对话上下文还在) */
357
+ function optsRow() {
358
+ const prefs = loadPrefs();
359
+ const pick = (label, list, value, name) => {
360
+ const sel = h("select", { "aria-label": label }, list.map(([v, t]) => h("option", { value: v }, t)));
361
+ sel.value = list.some(([v]) => v === value) ? value : "";
362
+ sel.addEventListener("change", () => {
363
+ savePrefs({ ...loadPrefs(), [name]: sel.value });
364
+ restartCall();
365
+ });
366
+ return h("label", { class: "call-opt" }, h("span", {}, label), sel);
367
+ };
368
+ return h(
369
+ "div",
370
+ { class: "call-opts", hidden: true },
371
+ pick("音色", VOICES, prefs.voice, "voice"),
372
+ pick("语言", LANGUAGES, prefs.language, "language"),
373
+ h("div", { class: "call-opts-hint" }, "记在这台设备上。通话中改了会重新连接,对话内容不受影响"),
374
+ );
375
+ }
376
+
377
+ function toggleOpts() {
378
+ const { opts, gear } = call.els;
379
+ opts.hidden = !opts.hidden;
380
+ gear.classList.toggle("active", !opts.hidden);
381
+ gear.setAttribute("aria-expanded", String(!opts.hidden));
382
+ }
383
+
384
+ /** 用新的音色 / 语言重连:挂掉当前这通,马上再开(在选择框的操作里,iOS 也允许播放) */
385
+ function restartCall() {
386
+ if (call.state === "idle") return;
387
+ const { compact, muted } = call;
388
+ endCall("");
389
+ void startCall();
390
+ if (muted) toggleMute();
391
+ if (compact) setCompact(true);
392
+ // startCall 同步就把新的通话条挂上了:设置行保持展开,方便接着试别的音色
393
+ if (call.els) toggleOpts();
394
+ }
395
+
207
396
  function toggleMute() {
208
397
  call.muted = !call.muted;
209
398
  for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
@@ -216,8 +405,10 @@ function render() {
216
405
  mute.innerHTML = call.muted ? ICON_MIC_OFF : icon("mic").innerHTML;
217
406
  mute.title = call.muted ? "取消静音" : "静音";
218
407
  mute.setAttribute("aria-label", mute.title);
408
+ mute.setAttribute("aria-pressed", String(call.muted));
219
409
  mute.classList.toggle("on", call.muted);
220
410
  call.panel.classList.toggle("connecting", call.state === "connecting");
411
+ call.panel.classList.toggle("muted", call.muted);
221
412
  renderCaption();
222
413
  }
223
414
 
@@ -225,28 +416,34 @@ function renderCaption() {
225
416
  if (!call.els) return;
226
417
  const { who, text } = call.els;
227
418
  const c = call.caption;
419
+ const caption = text.parentElement;
420
+ const follow = caption.scrollHeight - caption.scrollTop - caption.clientHeight < 32;
228
421
  who.textContent = c.text ? (c.role === "user" ? "你" : "Codex") : "";
229
- text.textContent = c.text || (call.state === "connecting" ? "" : "说点什么吧,可以随时打断它");
422
+ if (text.textContent !== c.text) {
423
+ text.textContent = c.text || "";
424
+ if (follow) requestAnimationFrame(() => { if (caption.isConnected) caption.scrollTop = caption.scrollHeight; });
425
+ }
230
426
  call.panel.classList.toggle("has-caption", !!c.text);
231
427
  }
232
428
 
233
- /** 每帧:音量条、状态、时长 */
429
+ /** 每帧读取实际音量;光球平滑响应声音,状态和时长仅在变化时写入 DOM。 */
234
430
  function tick() {
235
431
  if (!call.els) return;
236
432
  const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
237
433
  const out = call.outLevel?.() ?? 0;
238
434
  const speaking = out > 0.02;
239
435
  const level = Math.min(1, (speaking ? out : mic) * 6);
240
- call.els.bars.forEach((bar, i) => {
241
- const shape = [0.45, 0.75, 1, 0.75, 0.45][i];
242
- const wobble = call.state === "live" ? 0.15 + 0.85 * level * shape * (0.8 + 0.2 * Math.sin(Date.now() / 120 + i)) : 0.2;
243
- bar.style.transform = `scaleY(${Math.max(0.15, wobble).toFixed(3)})`;
244
- });
436
+ call.els.orb.style.setProperty("--voice-scale", (1 + level * 0.14).toFixed(3));
245
437
  call.panel.classList.toggle("speaking", speaking);
246
438
  const busy = S.busy && S.current === call.key;
247
- call.els.status.textContent =
439
+ call.panel.classList.toggle("processing", !!busy && !speaking);
440
+ const status =
248
441
  call.state === "connecting" ? "正在连接…" : speaking ? "Codex 正在说" : busy ? "Codex 正在处理…" : call.muted ? "已静音" : "正在听";
249
- call.els.time.textContent = call.state === "live" ? clock(Date.now() - call.startedAt) : "";
442
+ const hint = call.state === "connecting" ? "正在为你接通 Codex" : call.muted ? "麦克风已关闭,仍可听到回复" : busy && !speaking ? "完成后会接着告诉你" : "直接开口,随时打断";
443
+ const time = call.state === "live" ? clock(Date.now() - call.startedAt) : "连接中";
444
+ for (const [el, value] of [[call.els.status, status], [call.els.time, time], [call.els.hint, hint]]) {
445
+ if (el.textContent !== value) el.textContent = value;
446
+ }
250
447
  call.raf = requestAnimationFrame(tick);
251
448
  }
252
449
 
@@ -255,15 +452,42 @@ function clock(ms) {
255
452
  return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
256
453
  }
257
454
 
258
- /** 一路声音的音量(0~1 的均方根),给动画用 */
259
- function meter(stream) {
455
+ /** 整个页面共用一个 AudioContext(iOS 上反复新建、关闭会把麦克风搞坏),在点击里调用来建 / 恢复 */
456
+ let sharedCtx = null;
457
+ function audioCtx() {
260
458
  try {
261
- call.ctx ??= new AudioContext();
262
- void call.ctx.resume();
263
- const src = call.ctx.createMediaStreamSource(stream);
264
- const an = call.ctx.createAnalyser();
459
+ if (!sharedCtx || sharedCtx.state === "closed") sharedCtx = new (window.AudioContext ?? window.webkitAudioContext)();
460
+ if (sharedCtx.state !== "running") sharedCtx.resume().catch(() => {});
461
+ } catch {
462
+ sharedCtx = null;
463
+ }
464
+ return sharedCtx;
465
+ }
466
+
467
+ /** 轨道 muted 时等它恢复一会儿;还是不行就报错 */
468
+ function waitUnmute(track, ms) {
469
+ return new Promise((resolve, reject) => {
470
+ const timer = setTimeout(() => {
471
+ track.removeEventListener("unmute", ok);
472
+ reject(new Error("麦克风被系统静音了,请检查是否有其他应用在用麦克风,或刷新页面后重试"));
473
+ }, ms);
474
+ const ok = () => {
475
+ clearTimeout(timer);
476
+ resolve();
477
+ };
478
+ track.addEventListener("unmute", ok, { once: true });
479
+ });
480
+ }
481
+
482
+ /** 一个节点的音量(0~1 的均方根),给动画用 */
483
+ function meter(node) {
484
+ try {
485
+ const ctx = audioCtx();
486
+ if (!ctx) return null;
487
+ const an = ctx.createAnalyser();
265
488
  an.fftSize = 512;
266
- src.connect(an);
489
+ node.connect(an);
490
+ call.nodes.push(an);
267
491
  const buf = new Float32Array(an.fftSize);
268
492
  return () => {
269
493
  an.getFloatTimeDomainData(buf);
@@ -276,18 +500,54 @@ function meter(stream) {
276
500
  }
277
501
  }
278
502
 
279
- /** 等 ICE 候选收集完(最多 ms 毫秒):offer 里带上候选,不用另外传 */
280
- function iceGathered(pc, ms) {
281
- if (pc.iceGatheringState === "complete") return Promise.resolve();
282
- return new Promise((resolve) => {
283
- const done = () => {
284
- pc.removeEventListener("icegatheringstatechange", check);
285
- resolve();
286
- };
287
- const check = () => pc.iceGatheringState === "complete" && done();
288
- pc.addEventListener("icegatheringstatechange", check);
289
- setTimeout(done, ms);
290
- });
503
+ /** 每个 AudioContext 加载一次 worklet */
504
+ const workletLoaded = new WeakMap();
505
+ function loadWorklet(ctx) {
506
+ let p = workletLoaded.get(ctx);
507
+ if (!p) {
508
+ p = ctx.audioWorklet.addModule(new URL("../lib/rt-worklet.js", import.meta.url));
509
+ p.catch(() => workletLoaded.delete(ctx));
510
+ workletLoaded.set(ctx, p);
511
+ }
512
+ return p;
513
+ }
514
+
515
+ /** 本页里一对 RTCPeerConnection 把声音环回一下,让浏览器的回声消除认得它(见 appleAudio) */
516
+ async function loopback(stream) {
517
+ const a = new RTCPeerConnection();
518
+ const b = new RTCPeerConnection();
519
+ const close = () => {
520
+ a.close();
521
+ b.close();
522
+ };
523
+ try {
524
+ a.onicecandidate = (e) => e.candidate && b.addIceCandidate(e.candidate).catch(() => {});
525
+ b.onicecandidate = (e) => e.candidate && a.addIceCandidate(e.candidate).catch(() => {});
526
+ const got = new Promise((resolve, reject) => {
527
+ b.ontrack = (e) => resolve(e.streams[0] ?? new MediaStream([e.track]));
528
+ setTimeout(() => reject(new Error("环回超时")), 3000);
529
+ });
530
+ for (const t of stream.getTracks()) a.addTrack(t, stream);
531
+ await a.setLocalDescription(await a.createOffer());
532
+ await b.setRemoteDescription(a.localDescription);
533
+ await b.setLocalDescription(await b.createAnswer());
534
+ await a.setRemoteDescription(b.localDescription);
535
+ return { stream: await got, close };
536
+ } catch (err) {
537
+ close();
538
+ throw err;
539
+ }
540
+ }
541
+
542
+ /** 服务器转来的回复声音(WebSocket 二进制帧:24kHz 16 位 PCM) */
543
+ export function onRealtimeAudio(buf) {
544
+ if (call.state === "idle" || !call.play) return;
545
+ call.play.port.postMessage(buf, [buf]);
546
+ }
547
+
548
+ /** WebSocket 断了:声音走不了了,挂断 */
549
+ export function onSocketLost() {
550
+ endCall("和服务器的连接断开了,语音对话已结束", true);
291
551
  }
292
552
 
293
553
  window.addEventListener("pagehide", () => endCall());