agent-loft 0.2.113 → 0.2.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -1
- package/src/agents/codex/session.ts +31 -3
- package/src/channels/web/server.ts +69 -11
- package/src/core/hub.ts +3 -2
- package/src/core/realtime-relay.ts +142 -0
- package/src/core/types.ts +4 -1
- package/web/css/composer.css +4 -159
- package/web/css/voice-call.css +196 -0
- package/web/index.html +1 -0
- package/web/lib/rt-worklet.js +110 -0
- package/web/lib/ws.js +5 -2
- package/web/views/composer.js +10 -4
- package/web/views/voice-call.js +343 -83
package/web/views/voice-call.js
CHANGED
|
@@ -1,48 +1,109 @@
|
|
|
1
1
|
import { h, icon, toast } from "../lib/dom.js";
|
|
2
2
|
import { desktop, S, ui } from "../state.js";
|
|
3
3
|
import { WS } from "../lib/ws.js";
|
|
4
|
-
import { ensureConversation } from "./composer.js";
|
|
4
|
+
import { ensureConversation, updateSendBtn } from "./composer.js";
|
|
5
5
|
import { currentAgentId } from "./slash.js";
|
|
6
6
|
import { cancelVoice } from "./voice.js";
|
|
7
7
|
|
|
8
8
|
// ---------- 实时语音对话(只有 Codex) ----------
|
|
9
|
-
//
|
|
10
|
-
//
|
|
9
|
+
// 声音经服务器中转:服务器自己和 OpenAI 建 WebRTC(core/realtime-relay.ts),这里和服务器之间走现有的 WebSocket,
|
|
10
|
+
// 两头都是 24kHz、16 位、单声道的 PCM(二进制帧)。采集、播放在 AudioWorklet 里做(lib/rt-worklet.js)。
|
|
11
|
+
// 这样能打开网页的地方就能语音(反向代理、Cloudflare Tunnel 都能过),浏览器不直连 OpenAI。
|
|
11
12
|
// 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
|
|
12
|
-
//
|
|
13
|
+
// 通话默认打开沉浸式光球界面,也可以收起成输入框位置的通话条。
|
|
13
14
|
|
|
14
15
|
/** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
|
|
15
|
-
const supported = !desktop && !!window.
|
|
16
|
+
const supported = !desktop && !!window.AudioWorkletNode && !!navigator.mediaDevices?.getUserMedia;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 回复的声音直接从 AudioContext 放出来,iOS / Safari 的回声消除照样管得到;Chrome 等的回声消除只认 WebRTC 放出来的声音,
|
|
20
|
+
* 不绕一下模型会听到自己说话、自己打断自己:在本页里用一对 RTCPeerConnection 环回一下,再交给 <audio> 放
|
|
21
|
+
*/
|
|
22
|
+
const ua = navigator.userAgent;
|
|
23
|
+
const appleAudio = /iPad|iPhone|iPod/.test(ua) || (/Macintosh/.test(ua) && navigator.maxTouchPoints > 1) || (/Safari\//.test(ua) && !/Chrome|Chromium|Edg|OPR|Firefox/.test(ua));
|
|
16
24
|
|
|
17
25
|
const call = {
|
|
18
26
|
/** idle / connecting / live */
|
|
19
27
|
state: "idle",
|
|
20
28
|
key: null,
|
|
21
|
-
pc: null,
|
|
22
29
|
mic: null,
|
|
30
|
+
/** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
|
|
31
|
+
play: null,
|
|
32
|
+
/** Chrome 等的回声消除环回(见 appleAudio) */
|
|
33
|
+
loop: null,
|
|
23
34
|
audio: null,
|
|
24
35
|
muted: false,
|
|
25
36
|
startedAt: 0,
|
|
26
37
|
raf: 0,
|
|
27
|
-
|
|
38
|
+
/** 音量表接上的节点,挂断时断开(AudioContext 本身留着,见 audioCtx) */
|
|
39
|
+
nodes: [],
|
|
28
40
|
micLevel: null,
|
|
29
41
|
outLevel: null,
|
|
30
42
|
/** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
|
|
31
43
|
caption: { role: null, text: "", done: false },
|
|
32
44
|
panel: null,
|
|
45
|
+
dialog: null,
|
|
46
|
+
compact: false,
|
|
47
|
+
session: 0,
|
|
33
48
|
els: null,
|
|
34
|
-
/** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
|
|
35
|
-
dropTimer: 0,
|
|
36
49
|
};
|
|
37
50
|
|
|
51
|
+
// ---------- 音色和语言(按设备记在浏览器里) ----------
|
|
52
|
+
|
|
53
|
+
const PREFS_KEY = "agent-loft.voiceCall";
|
|
54
|
+
/** 和服务端 REALTIME_VOICES 一致(就是 ChatGPT 语音模式那几个);空 = Codex 默认(cove) */
|
|
55
|
+
const VOICES = [
|
|
56
|
+
["", "默认(Cove)"],
|
|
57
|
+
["cove", "Cove · 男声,沉着直接"],
|
|
58
|
+
["juniper", "Juniper · 女声,开朗"],
|
|
59
|
+
["maple", "Maple · 女声,爽朗坦率"],
|
|
60
|
+
["vale", "Vale · 女声,明亮好奇"],
|
|
61
|
+
["sol", "Sol · 女声,从容干练"],
|
|
62
|
+
["spruce", "Spruce · 男声,平和"],
|
|
63
|
+
["ember", "Ember · 男声,自信乐观"],
|
|
64
|
+
["arbor", "Arbor · 男声,随和"],
|
|
65
|
+
["breeze", "Breeze · 活泼真诚"],
|
|
66
|
+
];
|
|
67
|
+
/** 和服务端 REALTIME_LANGUAGES 一致;空 = 跟着你说的语言 */
|
|
68
|
+
const LANGUAGES = [
|
|
69
|
+
["", "跟着我说的语言"],
|
|
70
|
+
["zh-CN", "普通话(简体)"],
|
|
71
|
+
["zh-TW", "国语(繁体)"],
|
|
72
|
+
["yue", "粤语"],
|
|
73
|
+
["en", "English"],
|
|
74
|
+
["ja", "日本語"],
|
|
75
|
+
["ko", "한국어"],
|
|
76
|
+
];
|
|
77
|
+
|
|
78
|
+
function loadPrefs() {
|
|
79
|
+
try {
|
|
80
|
+
const p = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "{}");
|
|
81
|
+
return { voice: typeof p.voice === "string" ? p.voice : "", language: typeof p.language === "string" ? p.language : "" };
|
|
82
|
+
} catch {
|
|
83
|
+
return { voice: "", language: "" };
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function savePrefs(p) {
|
|
88
|
+
try {
|
|
89
|
+
localStorage.setItem(PREFS_KEY, JSON.stringify(p));
|
|
90
|
+
} catch {
|
|
91
|
+
// 隐私模式等存不了:这次通话照样用
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
38
95
|
const ICON_WAVE =
|
|
39
96
|
'<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
|
|
40
97
|
const ICON_MIC_OFF =
|
|
41
98
|
'<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 3l18 18M9 9v3a3 3 0 0 0 5.1 2.1M15 9.3V5a3 3 0 0 0-5.9-.6"/><path d="M5 10a7 7 0 0 0 11.7 5.2M19 10a7 7 0 0 1-.6 2.8M12 17v5"/></svg>';
|
|
42
99
|
|
|
43
100
|
/** 输入框里的「语音对话」按钮 */
|
|
101
|
+
export function canCall() {
|
|
102
|
+
return supported && currentAgentId() === "codex";
|
|
103
|
+
}
|
|
104
|
+
|
|
44
105
|
export function callButton() {
|
|
45
|
-
ui.callBtn = h("button", { class: "
|
|
106
|
+
ui.callBtn = h("button", { class: "send-btn call-btn", title: "语音对话", "aria-label": "语音对话", html: ICON_WAVE, onclick: () => void startCall() });
|
|
46
107
|
syncCall();
|
|
47
108
|
return ui.callBtn;
|
|
48
109
|
}
|
|
@@ -56,11 +117,14 @@ export function syncCall() {
|
|
|
56
117
|
if (call.state !== "idle") {
|
|
57
118
|
if (call.key && S.current !== call.key) {
|
|
58
119
|
endCall("切换了对话,语音对话已结束");
|
|
59
|
-
} else if (call.panel
|
|
60
|
-
|
|
120
|
+
} else if (call.panel) {
|
|
121
|
+
ui.composer?.classList.add("in-call");
|
|
122
|
+
if (!call.panel.isConnected || (call.compact && call.panel.previousElementSibling !== ui.composer)) {
|
|
123
|
+
mountPanel();
|
|
124
|
+
}
|
|
61
125
|
}
|
|
62
126
|
}
|
|
63
|
-
|
|
127
|
+
updateSendBtn();
|
|
64
128
|
}
|
|
65
129
|
|
|
66
130
|
async function startCall() {
|
|
@@ -70,48 +134,84 @@ async function startCall() {
|
|
|
70
134
|
return;
|
|
71
135
|
}
|
|
72
136
|
cancelVoice();
|
|
137
|
+
const session = ++call.session;
|
|
73
138
|
call.state = "connecting";
|
|
139
|
+
call.compact = false;
|
|
74
140
|
call.muted = false;
|
|
75
141
|
call.caption = { role: null, text: "", done: false };
|
|
76
142
|
// 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
|
|
77
143
|
call.audio = h("audio", { autoplay: true, playsinline: true });
|
|
78
144
|
call.audio.play().catch(() => {});
|
|
145
|
+
// AudioContext 也要在点击里建 / 恢复:iOS 上点击之外起的一直是 suspended,第二通起就没声音、录不到
|
|
146
|
+
audioCtx();
|
|
79
147
|
mountPanel();
|
|
80
148
|
try {
|
|
81
149
|
const key = await ensureConversation();
|
|
82
|
-
if (call.
|
|
150
|
+
if (session !== call.session) return;
|
|
83
151
|
call.key = key;
|
|
84
|
-
|
|
85
|
-
if (call.
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
152
|
+
const mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
|
|
153
|
+
if (session !== call.session) {
|
|
154
|
+
for (const track of mic.getTracks()) track.stop();
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
call.mic = mic;
|
|
158
|
+
// iOS 上偶尔拿到的是已经结束 / 被静音的轨道(音频会话被别的占着):别假装在听
|
|
159
|
+
const track = call.mic.getAudioTracks()[0];
|
|
160
|
+
if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
|
|
161
|
+
track.enabled = !call.muted;
|
|
162
|
+
if (track.muted) await waitUnmute(track, 1500);
|
|
163
|
+
if (session !== call.session) return;
|
|
164
|
+
const ctx = audioCtx();
|
|
165
|
+
if (!ctx) throw new Error("浏览器不支持音频处理");
|
|
166
|
+
await loadWorklet(ctx);
|
|
167
|
+
if (session !== call.session) return;
|
|
168
|
+
// 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
|
|
169
|
+
const src = ctx.createMediaStreamSource(call.mic);
|
|
170
|
+
const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
171
|
+
// 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
|
|
172
|
+
const sink = ctx.createGain();
|
|
173
|
+
sink.gain.value = 0;
|
|
174
|
+
src.connect(capture);
|
|
175
|
+
capture.connect(sink);
|
|
176
|
+
sink.connect(ctx.destination);
|
|
177
|
+
capture.port.onmessage = (e) => {
|
|
178
|
+
const sock = WS.sock;
|
|
179
|
+
// 网络卡住时别越攒越多(攒下的说出去也晚了)
|
|
180
|
+
if (call.state === "live" && sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
|
|
103
181
|
};
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
182
|
+
// 服务器发来的 → 播放 → 扬声器
|
|
183
|
+
const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
184
|
+
call.play = play;
|
|
185
|
+
call.nodes.push(src, capture, sink, play);
|
|
186
|
+
call.micLevel = meter(src);
|
|
187
|
+
call.outLevel = meter(play);
|
|
188
|
+
if (appleAudio) {
|
|
189
|
+
play.connect(ctx.destination);
|
|
190
|
+
} else {
|
|
191
|
+
const dest = ctx.createMediaStreamDestination();
|
|
192
|
+
play.connect(dest);
|
|
193
|
+
call.nodes.push(dest);
|
|
194
|
+
try {
|
|
195
|
+
const loop = await loopback(dest.stream);
|
|
196
|
+
if (session !== call.session) return loop.close();
|
|
197
|
+
call.loop = loop;
|
|
198
|
+
call.audio.srcObject = call.loop.stream;
|
|
199
|
+
call.audio.play().catch(() => {});
|
|
200
|
+
} catch {
|
|
201
|
+
// 环回建不起来:直接放(可能有回声)
|
|
202
|
+
if (session !== call.session) return;
|
|
203
|
+
play.connect(ctx.destination);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (session !== call.session) return;
|
|
207
|
+
const prefs = loadPrefs();
|
|
208
|
+
await WS.request({ t: "rt", op: "start", key, voice: prefs.voice, language: prefs.language }, 40_000);
|
|
209
|
+
if (session !== call.session) return;
|
|
111
210
|
call.state = "live";
|
|
112
211
|
call.startedAt = Date.now();
|
|
113
212
|
render();
|
|
114
213
|
} catch (err) {
|
|
214
|
+
if (session !== call.session) return;
|
|
115
215
|
const message = err?.name === "NotAllowedError" ? "没有麦克风权限,请在浏览器里允许使用麦克风" : err?.name === "NotFoundError" ? "没找到可用的麦克风" : err?.message ?? String(err);
|
|
116
216
|
endCall(`语音对话没能开始:${message}`);
|
|
117
217
|
}
|
|
@@ -121,23 +221,39 @@ async function startCall() {
|
|
|
121
221
|
export function endCall(reason, fromServer = false) {
|
|
122
222
|
if (call.state === "idle") return;
|
|
123
223
|
const key = call.key;
|
|
224
|
+
++call.session;
|
|
124
225
|
call.state = "idle";
|
|
125
226
|
if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
|
|
126
|
-
clearTimeout(call.dropTimer);
|
|
127
227
|
cancelAnimationFrame(call.raf);
|
|
128
|
-
call.pc?.close();
|
|
129
|
-
call.pc = null;
|
|
130
228
|
stopTracks();
|
|
131
|
-
|
|
229
|
+
call.loop?.close();
|
|
230
|
+
call.loop = null;
|
|
231
|
+
call.play = null;
|
|
232
|
+
if (call.audio) {
|
|
233
|
+
call.audio.pause();
|
|
234
|
+
call.audio.srcObject = null;
|
|
235
|
+
}
|
|
132
236
|
call.audio = null;
|
|
133
|
-
|
|
134
|
-
call.
|
|
237
|
+
// 不 close:iOS 上关掉接过麦克风的 AudioContext,之后再开麦克风常常是静音的。断开节点、先暂停,下次点击时恢复
|
|
238
|
+
for (const n of call.nodes) {
|
|
239
|
+
try {
|
|
240
|
+
n.disconnect();
|
|
241
|
+
} catch {
|
|
242
|
+
// 已经断开
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
call.nodes = [];
|
|
246
|
+
if (sharedCtx?.state === "running") sharedCtx.suspend().catch(() => {});
|
|
135
247
|
call.micLevel = call.outLevel = null;
|
|
136
248
|
call.key = null;
|
|
137
249
|
call.panel?.remove();
|
|
250
|
+
call.dialog?.close();
|
|
251
|
+
call.dialog?.remove();
|
|
252
|
+
call.dialog = null;
|
|
138
253
|
call.panel = call.els = null;
|
|
139
254
|
ui.composer?.classList.remove("in-call");
|
|
140
255
|
syncCall();
|
|
256
|
+
(ui.callBtn && !ui.callBtn.hidden ? ui.callBtn : ui.sendBtn)?.focus({ preventScroll: true });
|
|
141
257
|
if (reason) toast(reason);
|
|
142
258
|
}
|
|
143
259
|
|
|
@@ -172,38 +288,111 @@ export function onRealtimeEvent(msg) {
|
|
|
172
288
|
renderCaption();
|
|
173
289
|
}
|
|
174
290
|
|
|
175
|
-
// ----------
|
|
291
|
+
// ---------- 通话界面 ----------
|
|
176
292
|
|
|
177
293
|
function mountPanel() {
|
|
178
294
|
if (!ui.composer) return;
|
|
179
295
|
if (!call.panel) {
|
|
180
|
-
const
|
|
296
|
+
const orb = h("div", { class: "call-orb", "aria-hidden": "true" },
|
|
297
|
+
h("div", { class: "call-orb-surface" }, h("span", { class: "call-orb-cloud cloud-a" }), h("span", { class: "call-orb-cloud cloud-b" }), h("span", { class: "call-orb-cloud cloud-c" })),
|
|
298
|
+
);
|
|
181
299
|
const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
|
|
182
|
-
const
|
|
183
|
-
const
|
|
300
|
+
const gear = h("button", { class: "call-ctl", title: "音色和语言", "aria-label": "音色和语言", "aria-expanded": "false", onclick: toggleOpts }, icon("gear"));
|
|
301
|
+
const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, icon("x"));
|
|
302
|
+
const minimize = h("button", { class: "call-minimize icon-btn", title: "收起通话", "aria-label": "收起通话", onclick: () => setCompact(true) }, icon("down"));
|
|
303
|
+
const expand = h("button", { class: "call-expand call-ctl", title: "展开通话", "aria-label": "展开通话", onclick: () => setCompact(false) }, icon("expand"));
|
|
304
|
+
const status = h("span", { class: "call-status", role: "status" });
|
|
184
305
|
const time = h("span", { class: "call-time" });
|
|
185
306
|
const who = h("span", { class: "call-who" });
|
|
186
307
|
const text = h("span", { class: "call-text" });
|
|
187
|
-
|
|
308
|
+
const opts = optsRow();
|
|
309
|
+
const hint = h("p", { class: "call-hint" }, "直接开口,随时打断");
|
|
310
|
+
call.els = { orb, mute, gear, opts, status, time, who, text, hint, expand, minimize };
|
|
188
311
|
call.panel = h(
|
|
189
312
|
"div",
|
|
190
313
|
{ class: "call-panel", role: "region", "aria-label": "语音对话" },
|
|
314
|
+
h("header", { class: "call-header" },
|
|
315
|
+
h("div", { class: "call-brand" }, h("span", { class: "call-brand-icon", html: ICON_WAVE }), h("span", {}, "Codex", h("small", {}, "语音对话"))),
|
|
316
|
+
h("div", { class: "call-session" }, h("span", { class: "call-live-dot" }), time), minimize,
|
|
317
|
+
),
|
|
191
318
|
h(
|
|
192
319
|
"div",
|
|
193
320
|
{ class: "call-main" },
|
|
194
|
-
h("div", { class: "call-
|
|
195
|
-
h("div", { class: "call-info" },
|
|
196
|
-
h("div", { class: "call-ctls" }, mute, end),
|
|
321
|
+
h("div", { class: "call-stage" }, orb),
|
|
322
|
+
h("div", { class: "call-info" }, status, hint),
|
|
197
323
|
),
|
|
324
|
+
h("div", { class: "call-caption", "aria-live": "polite", "aria-atomic": "true" }, who, text),
|
|
325
|
+
opts,
|
|
326
|
+
h("footer", { class: "call-footer" }, h("div", { class: "call-ctls" }, expand,
|
|
327
|
+
h("div", { class: "call-control" }, mute, h("span", {}, "麦克风")),
|
|
328
|
+
h("div", { class: "call-control" }, gear, h("span", {}, "音色与语言")),
|
|
329
|
+
h("div", { class: "call-control" }, end, h("span", {}, "结束")),
|
|
330
|
+
), h("span", { class: "call-footer-note" }, "可以收起通话,查看聊天")),
|
|
198
331
|
);
|
|
332
|
+
call.dialog = h("dialog", { class: "call-dialog", "aria-label": "Codex 语音对话", oncancel: (e) => { e.preventDefault(); setCompact(true); } });
|
|
199
333
|
}
|
|
200
334
|
ui.composer.classList.add("in-call");
|
|
201
|
-
ui.composer.after(call.panel);
|
|
335
|
+
if (call.compact) ui.composer.after(call.panel);
|
|
336
|
+
else {
|
|
337
|
+
if (!call.dialog.isConnected) document.body.append(call.dialog);
|
|
338
|
+
call.dialog.append(call.panel);
|
|
339
|
+
if (!call.dialog.open) call.dialog.showModal();
|
|
340
|
+
}
|
|
202
341
|
render();
|
|
203
342
|
cancelAnimationFrame(call.raf);
|
|
204
343
|
call.raf = requestAnimationFrame(tick);
|
|
205
344
|
}
|
|
206
345
|
|
|
346
|
+
/** 收起只切换视图,麦克风和播放保持连接。Esc 也收起,避免误挂断。 */
|
|
347
|
+
function setCompact(compact) {
|
|
348
|
+
if (!call.panel || call.compact === compact) return;
|
|
349
|
+
call.compact = compact;
|
|
350
|
+
call.panel.classList.toggle("compact", compact);
|
|
351
|
+
if (compact) call.dialog.close();
|
|
352
|
+
mountPanel();
|
|
353
|
+
(compact ? call.els.expand : call.els.minimize).focus({ preventScroll: true });
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** 通话条下面展开的一行:音色、语言。通话中改了就用新设置重新连一次(Codex 那边对话上下文还在) */
|
|
357
|
+
function optsRow() {
|
|
358
|
+
const prefs = loadPrefs();
|
|
359
|
+
const pick = (label, list, value, name) => {
|
|
360
|
+
const sel = h("select", { "aria-label": label }, list.map(([v, t]) => h("option", { value: v }, t)));
|
|
361
|
+
sel.value = list.some(([v]) => v === value) ? value : "";
|
|
362
|
+
sel.addEventListener("change", () => {
|
|
363
|
+
savePrefs({ ...loadPrefs(), [name]: sel.value });
|
|
364
|
+
restartCall();
|
|
365
|
+
});
|
|
366
|
+
return h("label", { class: "call-opt" }, h("span", {}, label), sel);
|
|
367
|
+
};
|
|
368
|
+
return h(
|
|
369
|
+
"div",
|
|
370
|
+
{ class: "call-opts", hidden: true },
|
|
371
|
+
pick("音色", VOICES, prefs.voice, "voice"),
|
|
372
|
+
pick("语言", LANGUAGES, prefs.language, "language"),
|
|
373
|
+
h("div", { class: "call-opts-hint" }, "记在这台设备上。通话中改了会重新连接,对话内容不受影响"),
|
|
374
|
+
);
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
function toggleOpts() {
|
|
378
|
+
const { opts, gear } = call.els;
|
|
379
|
+
opts.hidden = !opts.hidden;
|
|
380
|
+
gear.classList.toggle("active", !opts.hidden);
|
|
381
|
+
gear.setAttribute("aria-expanded", String(!opts.hidden));
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** 用新的音色 / 语言重连:挂掉当前这通,马上再开(在选择框的操作里,iOS 也允许播放) */
|
|
385
|
+
function restartCall() {
|
|
386
|
+
if (call.state === "idle") return;
|
|
387
|
+
const { compact, muted } = call;
|
|
388
|
+
endCall("");
|
|
389
|
+
void startCall();
|
|
390
|
+
if (muted) toggleMute();
|
|
391
|
+
if (compact) setCompact(true);
|
|
392
|
+
// startCall 同步就把新的通话条挂上了:设置行保持展开,方便接着试别的音色
|
|
393
|
+
if (call.els) toggleOpts();
|
|
394
|
+
}
|
|
395
|
+
|
|
207
396
|
function toggleMute() {
|
|
208
397
|
call.muted = !call.muted;
|
|
209
398
|
for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
|
|
@@ -216,8 +405,10 @@ function render() {
|
|
|
216
405
|
mute.innerHTML = call.muted ? ICON_MIC_OFF : icon("mic").innerHTML;
|
|
217
406
|
mute.title = call.muted ? "取消静音" : "静音";
|
|
218
407
|
mute.setAttribute("aria-label", mute.title);
|
|
408
|
+
mute.setAttribute("aria-pressed", String(call.muted));
|
|
219
409
|
mute.classList.toggle("on", call.muted);
|
|
220
410
|
call.panel.classList.toggle("connecting", call.state === "connecting");
|
|
411
|
+
call.panel.classList.toggle("muted", call.muted);
|
|
221
412
|
renderCaption();
|
|
222
413
|
}
|
|
223
414
|
|
|
@@ -225,28 +416,34 @@ function renderCaption() {
|
|
|
225
416
|
if (!call.els) return;
|
|
226
417
|
const { who, text } = call.els;
|
|
227
418
|
const c = call.caption;
|
|
419
|
+
const caption = text.parentElement;
|
|
420
|
+
const follow = caption.scrollHeight - caption.scrollTop - caption.clientHeight < 32;
|
|
228
421
|
who.textContent = c.text ? (c.role === "user" ? "你" : "Codex") : "";
|
|
229
|
-
text.textContent
|
|
422
|
+
if (text.textContent !== c.text) {
|
|
423
|
+
text.textContent = c.text || "";
|
|
424
|
+
if (follow) requestAnimationFrame(() => { if (caption.isConnected) caption.scrollTop = caption.scrollHeight; });
|
|
425
|
+
}
|
|
230
426
|
call.panel.classList.toggle("has-caption", !!c.text);
|
|
231
427
|
}
|
|
232
428
|
|
|
233
|
-
/**
|
|
429
|
+
/** 每帧读取实际音量;光球平滑响应声音,状态和时长仅在变化时写入 DOM。 */
|
|
234
430
|
function tick() {
|
|
235
431
|
if (!call.els) return;
|
|
236
432
|
const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
|
|
237
433
|
const out = call.outLevel?.() ?? 0;
|
|
238
434
|
const speaking = out > 0.02;
|
|
239
435
|
const level = Math.min(1, (speaking ? out : mic) * 6);
|
|
240
|
-
call.els.
|
|
241
|
-
const shape = [0.45, 0.75, 1, 0.75, 0.45][i];
|
|
242
|
-
const wobble = call.state === "live" ? 0.15 + 0.85 * level * shape * (0.8 + 0.2 * Math.sin(Date.now() / 120 + i)) : 0.2;
|
|
243
|
-
bar.style.transform = `scaleY(${Math.max(0.15, wobble).toFixed(3)})`;
|
|
244
|
-
});
|
|
436
|
+
call.els.orb.style.setProperty("--voice-scale", (1 + level * 0.14).toFixed(3));
|
|
245
437
|
call.panel.classList.toggle("speaking", speaking);
|
|
246
438
|
const busy = S.busy && S.current === call.key;
|
|
247
|
-
call.
|
|
439
|
+
call.panel.classList.toggle("processing", !!busy && !speaking);
|
|
440
|
+
const status =
|
|
248
441
|
call.state === "connecting" ? "正在连接…" : speaking ? "Codex 正在说" : busy ? "Codex 正在处理…" : call.muted ? "已静音" : "正在听";
|
|
249
|
-
|
|
442
|
+
const hint = call.state === "connecting" ? "正在为你接通 Codex" : call.muted ? "麦克风已关闭,仍可听到回复" : busy && !speaking ? "完成后会接着告诉你" : "直接开口,随时打断";
|
|
443
|
+
const time = call.state === "live" ? clock(Date.now() - call.startedAt) : "连接中";
|
|
444
|
+
for (const [el, value] of [[call.els.status, status], [call.els.time, time], [call.els.hint, hint]]) {
|
|
445
|
+
if (el.textContent !== value) el.textContent = value;
|
|
446
|
+
}
|
|
250
447
|
call.raf = requestAnimationFrame(tick);
|
|
251
448
|
}
|
|
252
449
|
|
|
@@ -255,15 +452,42 @@ function clock(ms) {
|
|
|
255
452
|
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
|
|
256
453
|
}
|
|
257
454
|
|
|
258
|
-
/**
|
|
259
|
-
|
|
455
|
+
/** 整个页面共用一个 AudioContext(iOS 上反复新建、关闭会把麦克风搞坏),在点击里调用来建 / 恢复 */
|
|
456
|
+
let sharedCtx = null;
|
|
457
|
+
function audioCtx() {
|
|
260
458
|
try {
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
459
|
+
if (!sharedCtx || sharedCtx.state === "closed") sharedCtx = new (window.AudioContext ?? window.webkitAudioContext)();
|
|
460
|
+
if (sharedCtx.state !== "running") sharedCtx.resume().catch(() => {});
|
|
461
|
+
} catch {
|
|
462
|
+
sharedCtx = null;
|
|
463
|
+
}
|
|
464
|
+
return sharedCtx;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** 轨道 muted 时等它恢复一会儿;还是不行就报错 */
|
|
468
|
+
function waitUnmute(track, ms) {
|
|
469
|
+
return new Promise((resolve, reject) => {
|
|
470
|
+
const timer = setTimeout(() => {
|
|
471
|
+
track.removeEventListener("unmute", ok);
|
|
472
|
+
reject(new Error("麦克风被系统静音了,请检查是否有其他应用在用麦克风,或刷新页面后重试"));
|
|
473
|
+
}, ms);
|
|
474
|
+
const ok = () => {
|
|
475
|
+
clearTimeout(timer);
|
|
476
|
+
resolve();
|
|
477
|
+
};
|
|
478
|
+
track.addEventListener("unmute", ok, { once: true });
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** 一个节点的音量(0~1 的均方根),给动画用 */
|
|
483
|
+
function meter(node) {
|
|
484
|
+
try {
|
|
485
|
+
const ctx = audioCtx();
|
|
486
|
+
if (!ctx) return null;
|
|
487
|
+
const an = ctx.createAnalyser();
|
|
265
488
|
an.fftSize = 512;
|
|
266
|
-
|
|
489
|
+
node.connect(an);
|
|
490
|
+
call.nodes.push(an);
|
|
267
491
|
const buf = new Float32Array(an.fftSize);
|
|
268
492
|
return () => {
|
|
269
493
|
an.getFloatTimeDomainData(buf);
|
|
@@ -276,18 +500,54 @@ function meter(stream) {
|
|
|
276
500
|
}
|
|
277
501
|
}
|
|
278
502
|
|
|
279
|
-
/**
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
503
|
+
/** 每个 AudioContext 加载一次 worklet */
|
|
504
|
+
const workletLoaded = new WeakMap();
|
|
505
|
+
function loadWorklet(ctx) {
|
|
506
|
+
let p = workletLoaded.get(ctx);
|
|
507
|
+
if (!p) {
|
|
508
|
+
p = ctx.audioWorklet.addModule(new URL("../lib/rt-worklet.js", import.meta.url));
|
|
509
|
+
p.catch(() => workletLoaded.delete(ctx));
|
|
510
|
+
workletLoaded.set(ctx, p);
|
|
511
|
+
}
|
|
512
|
+
return p;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/** 本页里一对 RTCPeerConnection 把声音环回一下,让浏览器的回声消除认得它(见 appleAudio) */
|
|
516
|
+
async function loopback(stream) {
|
|
517
|
+
const a = new RTCPeerConnection();
|
|
518
|
+
const b = new RTCPeerConnection();
|
|
519
|
+
const close = () => {
|
|
520
|
+
a.close();
|
|
521
|
+
b.close();
|
|
522
|
+
};
|
|
523
|
+
try {
|
|
524
|
+
a.onicecandidate = (e) => e.candidate && b.addIceCandidate(e.candidate).catch(() => {});
|
|
525
|
+
b.onicecandidate = (e) => e.candidate && a.addIceCandidate(e.candidate).catch(() => {});
|
|
526
|
+
const got = new Promise((resolve, reject) => {
|
|
527
|
+
b.ontrack = (e) => resolve(e.streams[0] ?? new MediaStream([e.track]));
|
|
528
|
+
setTimeout(() => reject(new Error("环回超时")), 3000);
|
|
529
|
+
});
|
|
530
|
+
for (const t of stream.getTracks()) a.addTrack(t, stream);
|
|
531
|
+
await a.setLocalDescription(await a.createOffer());
|
|
532
|
+
await b.setRemoteDescription(a.localDescription);
|
|
533
|
+
await b.setLocalDescription(await b.createAnswer());
|
|
534
|
+
await a.setRemoteDescription(b.localDescription);
|
|
535
|
+
return { stream: await got, close };
|
|
536
|
+
} catch (err) {
|
|
537
|
+
close();
|
|
538
|
+
throw err;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** 服务器转来的回复声音(WebSocket 二进制帧:24kHz 16 位 PCM) */
|
|
543
|
+
export function onRealtimeAudio(buf) {
|
|
544
|
+
if (call.state === "idle" || !call.play) return;
|
|
545
|
+
call.play.port.postMessage(buf, [buf]);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** WebSocket 断了:声音走不了了,挂断 */
|
|
549
|
+
export function onSocketLost() {
|
|
550
|
+
endCall("和服务器的连接断开了,语音对话已结束", true);
|
|
291
551
|
}
|
|
292
552
|
|
|
293
553
|
window.addEventListener("pagehide", () => endCall());
|