agent-loft 0.2.113 → 0.2.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -1
- package/src/agents/codex/session.ts +31 -3
- package/src/channels/web/server.ts +69 -11
- package/src/core/hub.ts +3 -2
- package/src/core/realtime-relay.ts +142 -0
- package/src/core/types.ts +4 -1
- package/web/css/composer.css +38 -0
- package/web/lib/rt-worklet.js +110 -0
- package/web/lib/ws.js +5 -2
- package/web/views/voice-call.js +248 -56
package/README.md
CHANGED
|
@@ -54,7 +54,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
|
|
|
54
54
|
- **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
|
|
55
55
|
- 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
|
|
56
56
|
- 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
|
|
57
|
-
- 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框里点声波按钮,和 Codex 桌面端的语音模式一样边说边聊,可以随时打断。输入框换成通话条(音量动画、状态、实时字幕、静音、结束);要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3
|
|
57
|
+
- 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框里点声波按钮,和 Codex 桌面端的语音模式一样边说边聊,可以随时打断。输入框换成通话条(音量动画、状态、实时字幕、静音、结束);要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话条上的齿轮里可以选音色和回答用的语言(记在这台设备上)。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
|
|
58
58
|
- 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
|
|
59
59
|
- 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
|
|
60
60
|
- 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-loft",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.114",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "多个 agent 的工作台:把本机 Claude Code / Codex 放到一处统一管理,用飞书和 Web 页面随时驱动——会话、审批、模型切换都共用",
|
|
6
6
|
"keywords": [
|
|
@@ -58,9 +58,11 @@
|
|
|
58
58
|
"echarts": "^6.1.0",
|
|
59
59
|
"marked": "^16.0.0",
|
|
60
60
|
"mermaid": "^12.2.0",
|
|
61
|
+
"opusscript": "^0.1.1",
|
|
61
62
|
"qrcode": "^1.5.4",
|
|
62
63
|
"sharp": "^0.35.5",
|
|
63
64
|
"tsx": "^4.20.6",
|
|
65
|
+
"werift": "^0.27.0",
|
|
64
66
|
"ws": "^8.18.0"
|
|
65
67
|
},
|
|
66
68
|
"devDependencies": {
|
|
@@ -20,6 +20,7 @@ import type {
|
|
|
20
20
|
InteractionSpec,
|
|
21
21
|
InterruptOptions,
|
|
22
22
|
QuestionSpec,
|
|
23
|
+
RealtimeOptions,
|
|
23
24
|
RuntimeInfo,
|
|
24
25
|
SavedAttachment,
|
|
25
26
|
SessionOptions,
|
|
@@ -816,12 +817,12 @@ export class CodexSession implements AgentSession {
|
|
|
816
817
|
// ---------- 实时语音对话(thread/realtime/*,实验接口) ----------
|
|
817
818
|
|
|
818
819
|
/**
|
|
819
|
-
*
|
|
820
|
-
* 对端 answer 从 thread/realtime/sdp
|
|
820
|
+
* offer 是服务器这边的 WebRTC 端点(core/realtime-relay.ts)建的,app-server 去 OpenAI 建通话,
|
|
821
|
+
* 对端 answer 从 thread/realtime/sdp 通知回来。声音经服务器中转,浏览器那头走 WebSocket。
|
|
821
822
|
* 实测:ChatGPT 账号登录只能走 WebRTC(WebSocket 要 API key),协议版本要 v3。
|
|
822
823
|
* 语音模型把要干的活转交给这条 thread,app-server 自己开一轮(见 turn/started 里的 externalPending)。
|
|
823
824
|
*/
|
|
824
|
-
async startRealtime(offer: string): Promise<string> {
|
|
825
|
+
async startRealtime(offer: string, opts: RealtimeOptions = {}): Promise<string> {
|
|
825
826
|
if (this.disposed) throw new Error("会话已关闭");
|
|
826
827
|
if (this.rt) await this.stopRealtime();
|
|
827
828
|
const threadId = await this.open();
|
|
@@ -853,6 +854,7 @@ export class CodexSession implements AgentSession {
|
|
|
853
854
|
// 启动背景里有这台机器上最近的会话和目录结构:只给内置专家的管理员
|
|
854
855
|
includeStartupContext: !expert && this.opts.skills?.admin === true,
|
|
855
856
|
...(context ? { realtimeStartInstructions: context } : {}),
|
|
857
|
+
...realtimeVoiceParams(opts),
|
|
856
858
|
});
|
|
857
859
|
return await answer;
|
|
858
860
|
} catch (err) {
|
|
@@ -1336,6 +1338,32 @@ function kindMark(kind: unknown): string {
|
|
|
1336
1338
|
return k === "add" ? "+" : k === "delete" ? "-" : "~";
|
|
1337
1339
|
}
|
|
1338
1340
|
|
|
1341
|
+
/** v3 实时语音能用的音色(实测是 thread/realtime/listVoices 里 v1 那组,给 v2 的会报不支持;默认 cove) */
|
|
1342
|
+
export const REALTIME_VOICES = ["cove", "juniper", "maple", "spruce", "ember", "vale", "breeze", "arbor", "sol"] as const;
|
|
1343
|
+
|
|
1344
|
+
/**
|
|
1345
|
+
* 语音模型说哪种话:协议里没有语言参数,用 v3 的 initialItems 放一条 developer 指令。
|
|
1346
|
+
* 不在表里(或「自动」)就不加,跟着用户说的语言走
|
|
1347
|
+
*/
|
|
1348
|
+
export const REALTIME_LANGUAGES: Record<string, string> = {
|
|
1349
|
+
"zh-CN": "Always speak Mandarin Chinese (普通话) and use Simplified Chinese in any text, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1350
|
+
"zh-TW": "Always speak Mandarin Chinese (國語) and use Traditional Chinese in any text, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1351
|
+
yue: "Always speak Cantonese (粵語/廣東話), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1352
|
+
en: "Always speak English, no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1353
|
+
ja: "Always speak Japanese (日本語), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1354
|
+
ko: "Always speak Korean (한국어), no matter what language the user speaks, unless the user explicitly asks you to switch languages.",
|
|
1355
|
+
};
|
|
1356
|
+
|
|
1357
|
+
/** 页面选的音色、语言 → thread/realtime/start 的参数;认不出的忽略(用 Codex 默认) */
|
|
1358
|
+
export function realtimeVoiceParams(opts: RealtimeOptions): Record<string, unknown> {
|
|
1359
|
+
const voice = (REALTIME_VOICES as readonly string[]).includes(opts.voice ?? "") ? opts.voice : undefined;
|
|
1360
|
+
const language = opts.language ? REALTIME_LANGUAGES[opts.language] : undefined;
|
|
1361
|
+
return {
|
|
1362
|
+
...(voice ? { voice } : {}),
|
|
1363
|
+
...(language ? { initialItems: [{ role: "developer", text: language }] } : {}),
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1339
1367
|
/** 语音转交的一轮没拿到用户说的话时显示的占位 */
|
|
1340
1368
|
const VOICE_FALLBACK = `${VOICE_MARK}(语音)`;
|
|
1341
1369
|
|
|
@@ -19,6 +19,7 @@ import { skillEvents } from "../../core/skills.js";
|
|
|
19
19
|
import { expertEvents } from "../../core/experts.js";
|
|
20
20
|
import { enforceTerminalAccess, killAllTerminals, terminalEvents, watchTerminalAccess } from "../../core/terminals.js";
|
|
21
21
|
import { RUNNING_VERSION } from "../../core/upgrade.js";
|
|
22
|
+
import { RealtimeRelay, RELAY_RATE } from "../../core/realtime-relay.js";
|
|
22
23
|
import type { RealtimeEvent, Settlement } from "../../core/types.js";
|
|
23
24
|
import { getUser, isAdmin, legacyOwnerId, userEvents, type User } from "../../core/users.js";
|
|
24
25
|
import { currentUser, handleApi, takeUploads } from "./api.js";
|
|
@@ -268,8 +269,24 @@ type Client = {
|
|
|
268
269
|
pushEndpoint: string | null;
|
|
269
270
|
/** 最后一次收到这个页面的消息(含 ping) */
|
|
270
271
|
seen: number;
|
|
272
|
+
/** 这个页面正在打的语音对话:声音经这里中转(relay 建好之前是 undefined) */
|
|
273
|
+
rt?: { key: string; relay?: RealtimeRelay };
|
|
271
274
|
};
|
|
272
275
|
|
|
276
|
+
/** 一块麦克风 PCM 的上限(正常是 20ms ≈ 1KB) */
|
|
277
|
+
const MAX_AUDIO_CHUNK = 64 * 1024;
|
|
278
|
+
/** 发给页面的声音积压超过这么多就丢(手机网络卡住时别越攒越多、越放越晚) */
|
|
279
|
+
const MAX_AUDIO_BACKLOG = 256 * 1024;
|
|
280
|
+
|
|
281
|
+
/** 收掉这个页面的语音中转;stopCall:顺带让 Codex 挂断 */
|
|
282
|
+
function endRelay(client: Client, stopCall: boolean): void {
|
|
283
|
+
const rt = client.rt;
|
|
284
|
+
if (!rt) return;
|
|
285
|
+
client.rt = undefined;
|
|
286
|
+
rt.relay?.close();
|
|
287
|
+
if (stopCall) void hub.realtimeStop(rt.key).catch(() => {});
|
|
288
|
+
}
|
|
289
|
+
|
|
273
290
|
/** 每次进程启动换一个:客户端发现变了,就知道 seq 已经从头计数,得整页重拉 */
|
|
274
291
|
const EPOCH = Math.random().toString(36).slice(2, 10);
|
|
275
292
|
|
|
@@ -305,7 +322,10 @@ function broadcastEvent(env: EventEnvelope): void {
|
|
|
305
322
|
|
|
306
323
|
/** 语音对话的字幕和状态:只推给订阅了这条对话的页面(不进缓冲区,刷新后不重放) */
|
|
307
324
|
function broadcastRealtime({ key, ev }: { key: string; ev: RealtimeEvent }): void {
|
|
308
|
-
for (const client of clients)
|
|
325
|
+
for (const client of clients) {
|
|
326
|
+
if (ev.type === "closed" && client.rt?.key === key && client.rt.relay) endRelay(client, false);
|
|
327
|
+
if (client.subs.has(key) && canSeeKey(client.user, key)) send(client, { t: "rt", key, ev });
|
|
328
|
+
}
|
|
309
329
|
}
|
|
310
330
|
|
|
311
331
|
function broadcastInteraction(i: Interaction): void {
|
|
@@ -533,19 +553,45 @@ function handleClientMessage(client: Client, raw: string): void {
|
|
|
533
553
|
}
|
|
534
554
|
|
|
535
555
|
case "rt": {
|
|
536
|
-
//
|
|
556
|
+
// 实时语音对话:服务器自己和 OpenAI 建 WebRTC,声音在这条 WebSocket 上中转(二进制帧,24kHz 16 位 PCM)。
|
|
557
|
+
// start 等连上了才 ack;stop 挂断
|
|
537
558
|
if (!getConversation(key) || !canSeeKey(user, key)) return ack(false, { error: "会话不存在" });
|
|
538
559
|
if (msg.op === "start") {
|
|
539
|
-
|
|
540
|
-
|
|
560
|
+
endRelay(client, false);
|
|
561
|
+
const rt: NonNullable<Client["rt"]> = { key };
|
|
562
|
+
client.rt = rt;
|
|
541
563
|
client.subs.add(key);
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
564
|
+
const opts = {
|
|
565
|
+
...(typeof msg.voice === "string" && msg.voice ? { voice: msg.voice } : {}),
|
|
566
|
+
...(typeof msg.language === "string" && msg.language ? { language: msg.language } : {}),
|
|
567
|
+
};
|
|
568
|
+
void (async () => {
|
|
569
|
+
const { relay, offer } = await RealtimeRelay.create();
|
|
570
|
+
if (client.rt !== rt) {
|
|
571
|
+
relay.close();
|
|
572
|
+
throw new Error("已挂断");
|
|
573
|
+
}
|
|
574
|
+
rt.relay = relay;
|
|
575
|
+
relay.onAudio = (pcm) => {
|
|
576
|
+
if (client.ws.readyState === client.ws.OPEN && client.ws.bufferedAmount < MAX_AUDIO_BACKLOG) client.ws.send(pcm, { binary: true });
|
|
577
|
+
};
|
|
578
|
+
relay.onFailed = (reason) => {
|
|
579
|
+
if (client.rt !== rt) return;
|
|
580
|
+
endRelay(client, true);
|
|
581
|
+
send(client, { t: "rt", key, ev: { type: "closed", reason } });
|
|
582
|
+
};
|
|
583
|
+
const answer = await hub.realtimeStart(key, offer, opts);
|
|
584
|
+
if (client.rt !== rt) throw new Error("已挂断");
|
|
585
|
+
await relay.accept(answer);
|
|
586
|
+
ack(true, { rate: RELAY_RATE });
|
|
587
|
+
})().catch((err) => {
|
|
588
|
+
if (client.rt === rt) endRelay(client, true);
|
|
589
|
+
ack(false, { error: err instanceof Error ? err.message : String(err) });
|
|
590
|
+
});
|
|
546
591
|
return;
|
|
547
592
|
}
|
|
548
593
|
if (msg.op === "stop") {
|
|
594
|
+
if (client.rt?.key === key) endRelay(client, false);
|
|
549
595
|
void hub.realtimeStop(key).then(() => ack(true), (err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
|
|
550
596
|
return;
|
|
551
597
|
}
|
|
@@ -653,15 +699,27 @@ export async function startWeb(): Promise<void> {
|
|
|
653
699
|
const client: Client = { ws, subs: new Set(), alive: true, token, user, visible: false, viewing: null, foreground: false, pushEndpoint: null, seen: Date.now() };
|
|
654
700
|
clients.add(client);
|
|
655
701
|
ws.on("pong", () => (client.alive = true));
|
|
656
|
-
ws.on("message", (data) => {
|
|
702
|
+
ws.on("message", (data, isBinary) => {
|
|
657
703
|
try {
|
|
704
|
+
// 二进制帧只有一种:语音对话里麦克风的 PCM
|
|
705
|
+
if (isBinary) {
|
|
706
|
+
const chunk = Array.isArray(data) ? Buffer.concat(data) : Buffer.from(data as Buffer);
|
|
707
|
+
if (chunk.length <= MAX_AUDIO_CHUNK) client.rt?.relay?.pushAudio(chunk);
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
658
710
|
handleClientMessage(client, data.toString());
|
|
659
711
|
} catch (err) {
|
|
660
712
|
console.error("[web] 处理 WebSocket 消息出错:", err);
|
|
661
713
|
}
|
|
662
714
|
});
|
|
663
|
-
ws.on("close", () =>
|
|
664
|
-
|
|
715
|
+
ws.on("close", () => {
|
|
716
|
+
clients.delete(client);
|
|
717
|
+
endRelay(client, true);
|
|
718
|
+
});
|
|
719
|
+
ws.on("error", () => {
|
|
720
|
+
clients.delete(client);
|
|
721
|
+
endRelay(client, true);
|
|
722
|
+
});
|
|
665
723
|
send(client, { t: "hello", seq: hub.currentSeq(), epoch: EPOCH, version: RUNNING_VERSION });
|
|
666
724
|
});
|
|
667
725
|
} catch (err) {
|
package/src/core/hub.ts
CHANGED
|
@@ -50,6 +50,7 @@ import type {
|
|
|
50
50
|
InteractionSpec,
|
|
51
51
|
ModelInfo,
|
|
52
52
|
RealtimeEvent,
|
|
53
|
+
RealtimeOptions,
|
|
53
54
|
RuntimeInfo,
|
|
54
55
|
SessionSummary,
|
|
55
56
|
StepKind,
|
|
@@ -2057,7 +2058,7 @@ class Hub extends EventEmitter<{
|
|
|
2057
2058
|
// ---------- 实时语音对话(只有 Codex) ----------
|
|
2058
2059
|
|
|
2059
2060
|
/** 开始通话:offer 是浏览器的 WebRTC SDP,返回对端 answer。会话没拉起来就按绑定拉起 */
|
|
2060
|
-
async realtimeStart(key: string, offer: string): Promise<string> {
|
|
2061
|
+
async realtimeStart(key: string, offer: string, opts?: RealtimeOptions): Promise<string> {
|
|
2061
2062
|
const conv = getConversation(key);
|
|
2062
2063
|
if (!conv) throw new Error("会话不存在");
|
|
2063
2064
|
if (conv.agent !== "codex") throw new Error("只有 Codex 的对话能语音对话");
|
|
@@ -2065,7 +2066,7 @@ class Hub extends EventEmitter<{
|
|
|
2065
2066
|
const session = this.session(key);
|
|
2066
2067
|
if (!session.startRealtime) throw new Error("这个 agent 不支持语音对话");
|
|
2067
2068
|
touchConversation(key);
|
|
2068
|
-
return session.startRealtime(offer);
|
|
2069
|
+
return session.startRealtime(offer, opts);
|
|
2069
2070
|
}
|
|
2070
2071
|
|
|
2071
2072
|
async realtimeStop(key: string): Promise<void> {
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 实时语音的中转:服务器自己当 WebRTC 端点连 OpenAI,浏览器那头走现有的 WebSocket 收发 PCM。
|
|
3
|
+
* 这样声音和网页走同一条路(反向代理、Cloudflare Tunnel 都能过),浏览器不用直连 OpenAI。
|
|
4
|
+
*
|
|
5
|
+
* 浏览器 ⇄ 这里:16 位小端、单声道、24kHz 的 PCM(每块多长都行)。
|
|
6
|
+
* 这里 ⇄ OpenAI:Opus(RTP 时钟 48kHz,每包 20ms),加上 OpenAI 要求的 oai-events 数据通道。
|
|
7
|
+
*/
|
|
8
|
+
import OpusScript from "opusscript";
|
|
9
|
+
import { MediaStreamTrack, RTCPeerConnection, RTCRtpCodecParameters, RtpHeader, RtpPacket } from "werift";
|
|
10
|
+
|
|
11
|
+
export const RELAY_RATE = 24000;
|
|
12
|
+
/** 20ms 一帧:24kHz 下 480 个采样;RTP 时间戳按 48kHz 走,一帧加 960 */
|
|
13
|
+
const FRAME = 480;
|
|
14
|
+
const FRAME_BYTES = FRAME * 2;
|
|
15
|
+
const RTP_STEP = 960;
|
|
16
|
+
/** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
|
|
17
|
+
const MAX_PENDING = RELAY_RATE * 2;
|
|
18
|
+
|
|
19
|
+
export class RealtimeRelay {
|
|
20
|
+
/** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
|
|
21
|
+
onAudio: (pcm: Buffer) => void = () => {};
|
|
22
|
+
/** 和 OpenAI 的连接断了(不是 close() 主动关的) */
|
|
23
|
+
onFailed: (reason: string) => void = () => {};
|
|
24
|
+
|
|
25
|
+
private readonly pc: RTCPeerConnection;
|
|
26
|
+
private readonly track = new MediaStreamTrack({ kind: "audio" });
|
|
27
|
+
private readonly encoder = new OpusScript(RELAY_RATE, 1, OpusScript.Application.VOIP);
|
|
28
|
+
private readonly decoder = new OpusScript(RELAY_RATE, 1, OpusScript.Application.VOIP);
|
|
29
|
+
private pending = Buffer.alloc(0);
|
|
30
|
+
private seq = Math.floor(Math.random() * 0xffff);
|
|
31
|
+
private ts = Math.floor(Math.random() * 0xffffffff);
|
|
32
|
+
private closed = false;
|
|
33
|
+
private dropTimer: NodeJS.Timeout | undefined;
|
|
34
|
+
|
|
35
|
+
private constructor() {
|
|
36
|
+
this.pc = new RTCPeerConnection({
|
|
37
|
+
codecs: { audio: [new RTCRtpCodecParameters({ mimeType: "audio/opus", clockRate: 48000, channels: 2 })] },
|
|
38
|
+
});
|
|
39
|
+
const transceiver = this.pc.addTransceiver(this.track, { direction: "sendrecv" });
|
|
40
|
+
transceiver.onTrack.subscribe((remote) => {
|
|
41
|
+
remote.onReceiveRtp.subscribe((rtp) => {
|
|
42
|
+
if (this.closed || !rtp.payload.length) return;
|
|
43
|
+
try {
|
|
44
|
+
this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
|
|
45
|
+
} catch {
|
|
46
|
+
// 坏包丢掉
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
// OpenAI 要求 offer 里有这个数据通道(事件我们走 Codex 的通知,用不上它)
|
|
51
|
+
this.pc.createDataChannel("oai-events");
|
|
52
|
+
this.pc.connectionStateChange.subscribe((state) => {
|
|
53
|
+
if (this.closed) return;
|
|
54
|
+
clearTimeout(this.dropTimer);
|
|
55
|
+
if (state === "failed" || state === "closed") this.fail("和语音服务的连接断开了");
|
|
56
|
+
// 网络抖一下会短暂 disconnected,几秒内没恢复才算断
|
|
57
|
+
else if (state === "disconnected") this.dropTimer = setTimeout(() => this.fail("和语音服务的连接断开了"), 5000);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** 建好连接对象和 offer(ICE 候选收齐了写在里面),offer 交给 Codex 去 OpenAI 建通话 */
|
|
62
|
+
static async create(): Promise<{ relay: RealtimeRelay; offer: string }> {
|
|
63
|
+
const relay = new RealtimeRelay();
|
|
64
|
+
await relay.pc.setLocalDescription(await relay.pc.createOffer());
|
|
65
|
+
await relay.gathered(1500);
|
|
66
|
+
const offer = relay.pc.localDescription?.sdp;
|
|
67
|
+
if (!offer) {
|
|
68
|
+
relay.close();
|
|
69
|
+
throw new Error("没能建立语音连接");
|
|
70
|
+
}
|
|
71
|
+
return { relay, offer };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** OpenAI 的 answer 回来了:连上 */
|
|
75
|
+
async accept(answer: string): Promise<void> {
|
|
76
|
+
if (this.closed) throw new Error("已挂断");
|
|
77
|
+
await this.pc.setRemoteDescription({ type: "answer", sdp: answer });
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** 浏览器发来的麦克风 PCM:凑够 20ms 就编码发出去 */
|
|
81
|
+
pushAudio(pcm: Buffer): void {
|
|
82
|
+
if (this.closed || !pcm.length) return;
|
|
83
|
+
this.pending = this.pending.length ? Buffer.concat([this.pending, pcm]) : Buffer.from(pcm);
|
|
84
|
+
if (this.pending.length > MAX_PENDING) this.pending = this.pending.subarray(this.pending.length - MAX_PENDING);
|
|
85
|
+
while (this.pending.length >= FRAME_BYTES) {
|
|
86
|
+
const frame = this.pending.subarray(0, FRAME_BYTES);
|
|
87
|
+
this.pending = this.pending.subarray(FRAME_BYTES);
|
|
88
|
+
this.sendFrame(frame);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
close(): void {
|
|
93
|
+
if (this.closed) return;
|
|
94
|
+
this.closed = true;
|
|
95
|
+
clearTimeout(this.dropTimer);
|
|
96
|
+
this.pending = Buffer.alloc(0);
|
|
97
|
+
void this.pc.close().catch(() => {});
|
|
98
|
+
for (const codec of [this.encoder, this.decoder]) {
|
|
99
|
+
try {
|
|
100
|
+
codec.delete();
|
|
101
|
+
} catch {
|
|
102
|
+
// 已经释放
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private sendFrame(frame: Buffer): void {
|
|
108
|
+
let payload: Buffer;
|
|
109
|
+
try {
|
|
110
|
+
payload = this.encoder.encode(frame, FRAME);
|
|
111
|
+
} catch {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
const header = new RtpHeader({ sequenceNumber: this.seq, timestamp: this.ts, marker: false, payloadType: 111 });
|
|
115
|
+
this.seq = (this.seq + 1) & 0xffff;
|
|
116
|
+
this.ts = (this.ts + RTP_STEP) >>> 0;
|
|
117
|
+
try {
|
|
118
|
+
this.track.writeRtp(new RtpPacket(header, Buffer.from(payload)));
|
|
119
|
+
} catch {
|
|
120
|
+
// 还没连上 / 已经关了
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private fail(reason: string): void {
|
|
125
|
+
if (this.closed) return;
|
|
126
|
+
this.close();
|
|
127
|
+
this.onFailed(reason);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
private gathered(ms: number): Promise<void> {
|
|
131
|
+
if (this.pc.iceGatheringState === "complete") return Promise.resolve();
|
|
132
|
+
return new Promise((resolve) => {
|
|
133
|
+
const timer = setTimeout(done, ms);
|
|
134
|
+
const sub = this.pc.iceGatheringStateChange.subscribe((state) => state === "complete" && done());
|
|
135
|
+
function done(): void {
|
|
136
|
+
clearTimeout(timer);
|
|
137
|
+
sub.unSubscribe();
|
|
138
|
+
resolve();
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
package/src/core/types.ts
CHANGED
|
@@ -144,6 +144,9 @@ export type AgentEvents = {
|
|
|
144
144
|
onRealtime?: (ev: RealtimeEvent) => void;
|
|
145
145
|
};
|
|
146
146
|
|
|
147
|
+
/** 实时语音对话的音色和语言(页面上选的,没选就用 Codex 默认) */
|
|
148
|
+
export type RealtimeOptions = { voice?: string; language?: string };
|
|
149
|
+
|
|
147
150
|
/** 实时语音对话的事件(Codex 的 thread/realtime/*) */
|
|
148
151
|
export type RealtimeEvent =
|
|
149
152
|
| { type: "started" }
|
|
@@ -315,7 +318,7 @@ export interface AgentSession {
|
|
|
315
318
|
/** 开关计划模式。Claude 立即生效,Codex 下一轮生效 */
|
|
316
319
|
setPlan?(on: boolean): Promise<void>;
|
|
317
320
|
/** 开始实时语音对话(只有 Codex):offer 是浏览器的 WebRTC SDP,返回对端的 answer */
|
|
318
|
-
startRealtime?(offer: string): Promise<string>;
|
|
321
|
+
startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
|
|
319
322
|
/** 挂断实时语音对话;没在通话就什么都不做 */
|
|
320
323
|
stopRealtime?(): Promise<void>;
|
|
321
324
|
}
|
package/web/css/composer.css
CHANGED
|
@@ -679,6 +679,44 @@ textarea.sending {
|
|
|
679
679
|
background: color-mix(in srgb, var(--danger) 12%, transparent);
|
|
680
680
|
}
|
|
681
681
|
|
|
682
|
+
.call-ctl.active {
|
|
683
|
+
color: var(--accent);
|
|
684
|
+
background: color-mix(in srgb, var(--accent) 12%, transparent);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
.call-opts {
|
|
688
|
+
display: flex;
|
|
689
|
+
flex-wrap: wrap;
|
|
690
|
+
align-items: center;
|
|
691
|
+
gap: 8px 16px;
|
|
692
|
+
margin-top: 10px;
|
|
693
|
+
padding-top: 10px;
|
|
694
|
+
border-top: 1px solid color-mix(in srgb, var(--border) 70%, transparent);
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
.call-opts[hidden] {
|
|
698
|
+
display: none;
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
.call-opt {
|
|
702
|
+
display: inline-flex;
|
|
703
|
+
align-items: center;
|
|
704
|
+
gap: 6px;
|
|
705
|
+
font-size: 13px;
|
|
706
|
+
color: var(--fg-soft);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
.call-opt select {
|
|
710
|
+
font-size: 13px;
|
|
711
|
+
max-width: 180px;
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
.call-opts-hint {
|
|
715
|
+
flex-basis: 100%;
|
|
716
|
+
font-size: 12px;
|
|
717
|
+
color: var(--fg-faint);
|
|
718
|
+
}
|
|
719
|
+
|
|
682
720
|
.call-ctl.call-end {
|
|
683
721
|
padding: 0 16px;
|
|
684
722
|
font-size: 14px;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
|
|
2
|
+
// rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
|
|
3
|
+
// rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
|
|
4
|
+
|
|
5
|
+
const RATE = 24000;
|
|
6
|
+
const CHUNK = 480; // 20ms
|
|
7
|
+
|
|
8
|
+
class Capture extends AudioWorkletProcessor {
|
|
9
|
+
constructor() {
|
|
10
|
+
super();
|
|
11
|
+
this.step = sampleRate / RATE; // 每出一个 24kHz 采样,输入前进多少
|
|
12
|
+
this.pos = 0; // 下一个输出采样在输入里的位置(相对 prev 之后)
|
|
13
|
+
this.prev = 0; // 上一块的最后一个采样,插值跨块用
|
|
14
|
+
this.out = new Int16Array(CHUNK);
|
|
15
|
+
this.n = 0;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
process(inputs) {
|
|
19
|
+
const input = inputs[0]?.[0];
|
|
20
|
+
if (!input) return true;
|
|
21
|
+
// 多声道输入只取第一路就够(getUserMedia 一般是单声道)
|
|
22
|
+
const len = input.length;
|
|
23
|
+
// 位置 -1 对应 prev,0..len-1 对应 input
|
|
24
|
+
while (this.pos < len) {
|
|
25
|
+
const i = Math.floor(this.pos);
|
|
26
|
+
const f = this.pos - i;
|
|
27
|
+
const a = i < 0 ? this.prev : input[i];
|
|
28
|
+
const b = i + 1 < len ? input[i + 1] : input[len - 1];
|
|
29
|
+
let v = a + (b - a) * f;
|
|
30
|
+
v = v < -1 ? -1 : v > 1 ? 1 : v;
|
|
31
|
+
this.out[this.n++] = v < 0 ? v * 0x8000 : v * 0x7fff;
|
|
32
|
+
if (this.n === CHUNK) {
|
|
33
|
+
this.port.postMessage(this.out.buffer, [this.out.buffer]);
|
|
34
|
+
this.out = new Int16Array(CHUNK);
|
|
35
|
+
this.n = 0;
|
|
36
|
+
}
|
|
37
|
+
this.pos += this.step;
|
|
38
|
+
}
|
|
39
|
+
this.pos -= len;
|
|
40
|
+
this.prev = input[len - 1];
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
class Play extends AudioWorkletProcessor {
|
|
46
|
+
constructor() {
|
|
47
|
+
super();
|
|
48
|
+
this.step = RATE / sampleRate; // 每出一个输出采样,24kHz 的缓冲前进多少
|
|
49
|
+
this.size = RATE * 4; // 环形缓冲最多 4 秒
|
|
50
|
+
this.buf = new Float32Array(this.size);
|
|
51
|
+
this.read = 0; // 读位置(小数)
|
|
52
|
+
this.avail = 0; // 还没放的采样数
|
|
53
|
+
this.write = 0;
|
|
54
|
+
this.playing = false;
|
|
55
|
+
this.port.onmessage = (e) => {
|
|
56
|
+
if (e.data === "flush") {
|
|
57
|
+
this.avail = 0;
|
|
58
|
+
this.playing = false;
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const pcm = new Int16Array(e.data);
|
|
62
|
+
// 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
|
|
63
|
+
const over = this.avail + pcm.length - this.size;
|
|
64
|
+
if (over > 0) {
|
|
65
|
+
this.read = (this.read + over) % this.size;
|
|
66
|
+
this.avail -= over;
|
|
67
|
+
}
|
|
68
|
+
for (let i = 0; i < pcm.length; i++) {
|
|
69
|
+
this.buf[this.write] = pcm[i] / 0x8000;
|
|
70
|
+
this.write = (this.write + 1) % this.size;
|
|
71
|
+
}
|
|
72
|
+
this.avail += pcm.length;
|
|
73
|
+
// 太多了(超过 1 秒)说明延迟越积越大,追到只剩 0.2 秒
|
|
74
|
+
if (this.avail > RATE) {
|
|
75
|
+
const skip = this.avail - RATE / 5;
|
|
76
|
+
this.read = (this.read + skip) % this.size;
|
|
77
|
+
this.avail -= skip;
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
process(_inputs, outputs) {
|
|
83
|
+
const out = outputs[0];
|
|
84
|
+
const ch = out[0];
|
|
85
|
+
if (!ch) return true;
|
|
86
|
+
// 先攒 60ms 再开始放,扛一点网络抖动;放空了就停下来重新攒
|
|
87
|
+
if (!this.playing && this.avail >= RATE * 0.06) this.playing = true;
|
|
88
|
+
for (let i = 0; i < ch.length; i++) {
|
|
89
|
+
if (!this.playing || this.avail < 2) {
|
|
90
|
+
ch[i] = 0;
|
|
91
|
+
this.playing = false;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
const i0 = Math.floor(this.read);
|
|
95
|
+
const f = this.read - i0;
|
|
96
|
+
const a = this.buf[i0 % this.size];
|
|
97
|
+
const b = this.buf[(i0 + 1) % this.size];
|
|
98
|
+
ch[i] = a + (b - a) * f;
|
|
99
|
+
const next = this.read + this.step;
|
|
100
|
+
const used = Math.floor(next) - i0;
|
|
101
|
+
this.avail -= used;
|
|
102
|
+
this.read = next % this.size;
|
|
103
|
+
}
|
|
104
|
+
for (let c = 1; c < out.length; c++) out[c].set(ch);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
registerProcessor("rt-capture", Capture);
|
|
110
|
+
registerProcessor("rt-play", Play);
|
package/web/lib/ws.js
CHANGED
|
@@ -11,7 +11,7 @@ import { setBusy } from "../views/composer.js";
|
|
|
11
11
|
import { SH, shownHere } from "./shell.js";
|
|
12
12
|
import { currentPushSub } from "./push.js";
|
|
13
13
|
import { renderScheduled } from "../views/schedule.js";
|
|
14
|
-
import { onRealtimeEvent } from "../views/voice-call.js";
|
|
14
|
+
import { onRealtimeAudio, onRealtimeEvent, onSocketLost } from "../views/voice-call.js";
|
|
15
15
|
import { reloadExperts } from "../views/experts.js";
|
|
16
16
|
|
|
17
17
|
// ---------- WebSocket ----------
|
|
@@ -34,6 +34,8 @@ export const WS = {
|
|
|
34
34
|
if (this.sock && (this.sock.readyState === 0 || this.sock.readyState === 1)) return;
|
|
35
35
|
const proto = location.protocol === "https:" ? "wss" : "ws";
|
|
36
36
|
const sock = new WebSocket(`${proto}://${location.host}/ws`);
|
|
37
|
+
// 二进制帧只有语音对话的声音
|
|
38
|
+
sock.binaryType = "arraybuffer";
|
|
37
39
|
this.sock = sock;
|
|
38
40
|
sock.onopen = () => {
|
|
39
41
|
this.connected = true;
|
|
@@ -42,10 +44,11 @@ export const WS = {
|
|
|
42
44
|
clearInterval(this.pingTimer);
|
|
43
45
|
this.pingTimer = setInterval(() => this.raw({ t: "ping" }), 25000);
|
|
44
46
|
};
|
|
45
|
-
sock.onmessage = (e) => onWsMessage(JSON.parse(e.data));
|
|
47
|
+
sock.onmessage = (e) => (typeof e.data === "string" ? onWsMessage(JSON.parse(e.data)) : onRealtimeAudio(e.data));
|
|
46
48
|
sock.onclose = async (e) => {
|
|
47
49
|
this.connected = false;
|
|
48
50
|
clearInterval(this.pingTimer);
|
|
51
|
+
if (this.sock === sock) onSocketLost();
|
|
49
52
|
for (const w of this.waiters.values()) w.reject(new Error("连接已断开"));
|
|
50
53
|
this.waiters.clear();
|
|
51
54
|
if (this.sock !== sock) return;
|
package/web/views/voice-call.js
CHANGED
|
@@ -6,35 +6,89 @@ import { currentAgentId } from "./slash.js";
|
|
|
6
6
|
import { cancelVoice } from "./voice.js";
|
|
7
7
|
|
|
8
8
|
// ---------- 实时语音对话(只有 Codex) ----------
|
|
9
|
-
//
|
|
10
|
-
//
|
|
9
|
+
// 声音经服务器中转:服务器自己和 OpenAI 建 WebRTC(core/realtime-relay.ts),这里和服务器之间走现有的 WebSocket,
|
|
10
|
+
// 两头都是 24kHz、16 位、单声道的 PCM(二进制帧)。采集、播放在 AudioWorklet 里做(lib/rt-worklet.js)。
|
|
11
|
+
// 这样能打开网页的地方就能语音(反向代理、Cloudflare Tunnel 都能过),浏览器不直连 OpenAI。
|
|
11
12
|
// 语音模型把要干的活转交给这条对话里的 Codex:那一轮像打字发的消息一样出现在对话里(用户消息前面带 🎙️)。
|
|
12
13
|
// 通话时输入框换成通话条:音量动画、状态、字幕、静音、挂断(和 Codex 桌面端的语音模式一样)。
|
|
13
14
|
|
|
14
15
|
/** 桌面端(Electron)没申请麦克风权限:macOS 上直接用会被系统拦掉,先不放 */
|
|
15
|
-
const supported = !desktop && !!window.
|
|
16
|
+
const supported = !desktop && !!window.AudioWorkletNode && !!navigator.mediaDevices?.getUserMedia;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* 回复的声音直接从 AudioContext 放出来,iOS / Safari 的回声消除照样管得到;Chrome 等的回声消除只认 WebRTC 放出来的声音,
|
|
20
|
+
* 不绕一下模型会听到自己说话、自己打断自己:在本页里用一对 RTCPeerConnection 环回一下,再交给 <audio> 放
|
|
21
|
+
*/
|
|
22
|
+
const ua = navigator.userAgent;
|
|
23
|
+
const appleAudio = /iPad|iPhone|iPod/.test(ua) || (/Macintosh/.test(ua) && navigator.maxTouchPoints > 1) || (/Safari\//.test(ua) && !/Chrome|Chromium|Edg|OPR|Firefox/.test(ua));
|
|
16
24
|
|
|
17
25
|
const call = {
|
|
18
26
|
/** idle / connecting / live */
|
|
19
27
|
state: "idle",
|
|
20
28
|
key: null,
|
|
21
|
-
pc: null,
|
|
22
29
|
mic: null,
|
|
30
|
+
/** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
|
|
31
|
+
play: null,
|
|
32
|
+
/** Chrome 等的回声消除环回(见 appleAudio) */
|
|
33
|
+
loop: null,
|
|
23
34
|
audio: null,
|
|
24
35
|
muted: false,
|
|
25
36
|
startedAt: 0,
|
|
26
37
|
raf: 0,
|
|
27
|
-
|
|
38
|
+
/** 音量表接上的节点,挂断时断开(AudioContext 本身留着,见 audioCtx) */
|
|
39
|
+
nodes: [],
|
|
28
40
|
micLevel: null,
|
|
29
41
|
outLevel: null,
|
|
30
42
|
/** 字幕:谁在说、说了什么(一句说完 done 之后,下一句同一个人开口就换新的) */
|
|
31
43
|
caption: { role: null, text: "", done: false },
|
|
32
44
|
panel: null,
|
|
33
45
|
els: null,
|
|
34
|
-
/** 连接短暂断开(disconnected)时等一会儿再判定挂了 */
|
|
35
|
-
dropTimer: 0,
|
|
36
46
|
};
|
|
37
47
|
|
|
48
|
+
// ---------- 音色和语言(按设备记在浏览器里) ----------
|
|
49
|
+
|
|
50
|
+
const PREFS_KEY = "agent-loft.voiceCall";
|
|
51
|
+
/** 和服务端 REALTIME_VOICES 一致(就是 ChatGPT 语音模式那几个);空 = Codex 默认(cove) */
|
|
52
|
+
const VOICES = [
|
|
53
|
+
["", "默认(Cove)"],
|
|
54
|
+
["cove", "Cove · 男声,沉着直接"],
|
|
55
|
+
["juniper", "Juniper · 女声,开朗"],
|
|
56
|
+
["maple", "Maple · 女声,爽朗坦率"],
|
|
57
|
+
["vale", "Vale · 女声,明亮好奇"],
|
|
58
|
+
["sol", "Sol · 女声,从容干练"],
|
|
59
|
+
["spruce", "Spruce · 男声,平和"],
|
|
60
|
+
["ember", "Ember · 男声,自信乐观"],
|
|
61
|
+
["arbor", "Arbor · 男声,随和"],
|
|
62
|
+
["breeze", "Breeze · 活泼真诚"],
|
|
63
|
+
];
|
|
64
|
+
/** 和服务端 REALTIME_LANGUAGES 一致;空 = 跟着你说的语言 */
|
|
65
|
+
const LANGUAGES = [
|
|
66
|
+
["", "跟着我说的语言"],
|
|
67
|
+
["zh-CN", "普通话(简体)"],
|
|
68
|
+
["zh-TW", "国语(繁体)"],
|
|
69
|
+
["yue", "粤语"],
|
|
70
|
+
["en", "English"],
|
|
71
|
+
["ja", "日本語"],
|
|
72
|
+
["ko", "한국어"],
|
|
73
|
+
];
|
|
74
|
+
|
|
75
|
+
function loadPrefs() {
|
|
76
|
+
try {
|
|
77
|
+
const p = JSON.parse(localStorage.getItem(PREFS_KEY) ?? "{}");
|
|
78
|
+
return { voice: typeof p.voice === "string" ? p.voice : "", language: typeof p.language === "string" ? p.language : "" };
|
|
79
|
+
} catch {
|
|
80
|
+
return { voice: "", language: "" };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function savePrefs(p) {
|
|
85
|
+
try {
|
|
86
|
+
localStorage.setItem(PREFS_KEY, JSON.stringify(p));
|
|
87
|
+
} catch {
|
|
88
|
+
// 隐私模式等存不了:这次通话照样用
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
38
92
|
const ICON_WAVE =
|
|
39
93
|
'<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M4 10v4M8 6v12M12 3v18M16 7v10M20 10v4"/></svg>';
|
|
40
94
|
const ICON_MIC_OFF =
|
|
@@ -76,6 +130,8 @@ async function startCall() {
|
|
|
76
130
|
// 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
|
|
77
131
|
call.audio = h("audio", { autoplay: true, playsinline: true });
|
|
78
132
|
call.audio.play().catch(() => {});
|
|
133
|
+
// AudioContext 也要在点击里建 / 恢复:iOS 上点击之外起的一直是 suspended,第二通起就没声音、录不到
|
|
134
|
+
audioCtx();
|
|
79
135
|
mountPanel();
|
|
80
136
|
try {
|
|
81
137
|
const key = await ensureConversation();
|
|
@@ -83,31 +139,53 @@ async function startCall() {
|
|
|
83
139
|
call.key = key;
|
|
84
140
|
call.mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
|
|
85
141
|
if (call.state === "idle") return stopTracks();
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
};
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
142
|
+
// iOS 上偶尔拿到的是已经结束 / 被静音的轨道(音频会话被别的占着):别假装在听
|
|
143
|
+
const track = call.mic.getAudioTracks()[0];
|
|
144
|
+
if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
|
|
145
|
+
if (track.muted) await waitUnmute(track, 1500);
|
|
146
|
+
const ctx = audioCtx();
|
|
147
|
+
if (!ctx) throw new Error("浏览器不支持音频处理");
|
|
148
|
+
await loadWorklet(ctx);
|
|
149
|
+
if (call.state === "idle") return stopTracks();
|
|
150
|
+
// 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
|
|
151
|
+
const src = ctx.createMediaStreamSource(call.mic);
|
|
152
|
+
const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
153
|
+
// 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
|
|
154
|
+
const sink = ctx.createGain();
|
|
155
|
+
sink.gain.value = 0;
|
|
156
|
+
src.connect(capture);
|
|
157
|
+
capture.connect(sink);
|
|
158
|
+
sink.connect(ctx.destination);
|
|
159
|
+
capture.port.onmessage = (e) => {
|
|
160
|
+
const sock = WS.sock;
|
|
161
|
+
// 网络卡住时别越攒越多(攒下的说出去也晚了)
|
|
162
|
+
if (call.state === "live" && sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
|
|
103
163
|
};
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
164
|
+
// 服务器发来的 → 播放 → 扬声器
|
|
165
|
+
const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
166
|
+
call.play = play;
|
|
167
|
+
call.nodes.push(src, capture, sink, play);
|
|
168
|
+
call.micLevel = meter(src);
|
|
169
|
+
call.outLevel = meter(play);
|
|
170
|
+
if (appleAudio) {
|
|
171
|
+
play.connect(ctx.destination);
|
|
172
|
+
} else {
|
|
173
|
+
const dest = ctx.createMediaStreamDestination();
|
|
174
|
+
play.connect(dest);
|
|
175
|
+
call.nodes.push(dest);
|
|
176
|
+
try {
|
|
177
|
+
call.loop = await loopback(dest.stream);
|
|
178
|
+
call.audio.srcObject = call.loop.stream;
|
|
179
|
+
call.audio.play().catch(() => {});
|
|
180
|
+
} catch {
|
|
181
|
+
// 环回建不起来:直接放(可能有回声)
|
|
182
|
+
play.connect(ctx.destination);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
107
185
|
if (call.state === "idle") return;
|
|
108
|
-
const
|
|
186
|
+
const prefs = loadPrefs();
|
|
187
|
+
await WS.request({ t: "rt", op: "start", key, voice: prefs.voice, language: prefs.language }, 40_000);
|
|
109
188
|
if (call.state === "idle") return;
|
|
110
|
-
await pc.setRemoteDescription({ type: "answer", sdp: res.sdp });
|
|
111
189
|
call.state = "live";
|
|
112
190
|
call.startedAt = Date.now();
|
|
113
191
|
render();
|
|
@@ -123,15 +201,26 @@ export function endCall(reason, fromServer = false) {
|
|
|
123
201
|
const key = call.key;
|
|
124
202
|
call.state = "idle";
|
|
125
203
|
if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
|
|
126
|
-
clearTimeout(call.dropTimer);
|
|
127
204
|
cancelAnimationFrame(call.raf);
|
|
128
|
-
call.pc?.close();
|
|
129
|
-
call.pc = null;
|
|
130
205
|
stopTracks();
|
|
131
|
-
|
|
206
|
+
call.loop?.close();
|
|
207
|
+
call.loop = null;
|
|
208
|
+
call.play = null;
|
|
209
|
+
if (call.audio) {
|
|
210
|
+
call.audio.pause();
|
|
211
|
+
call.audio.srcObject = null;
|
|
212
|
+
}
|
|
132
213
|
call.audio = null;
|
|
133
|
-
|
|
134
|
-
call.
|
|
214
|
+
// 不 close:iOS 上关掉接过麦克风的 AudioContext,之后再开麦克风常常是静音的。断开节点、先暂停,下次点击时恢复
|
|
215
|
+
for (const n of call.nodes) {
|
|
216
|
+
try {
|
|
217
|
+
n.disconnect();
|
|
218
|
+
} catch {
|
|
219
|
+
// 已经断开
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
call.nodes = [];
|
|
223
|
+
if (sharedCtx?.state === "running") sharedCtx.suspend().catch(() => {});
|
|
135
224
|
call.micLevel = call.outLevel = null;
|
|
136
225
|
call.key = null;
|
|
137
226
|
call.panel?.remove();
|
|
@@ -179,12 +268,14 @@ function mountPanel() {
|
|
|
179
268
|
if (!call.panel) {
|
|
180
269
|
const bars = [0, 1, 2, 3, 4].map(() => h("span", { class: "call-bar" }));
|
|
181
270
|
const mute = h("button", { class: "call-ctl", title: "静音", "aria-label": "静音", onclick: toggleMute }, icon("mic"));
|
|
271
|
+
const gear = h("button", { class: "call-ctl", title: "音色和语言", "aria-label": "音色和语言", "aria-expanded": "false", onclick: toggleOpts }, icon("gear"));
|
|
182
272
|
const end = h("button", { class: "call-ctl call-end", title: "结束语音对话", "aria-label": "结束语音对话", onclick: () => endCall() }, "结束");
|
|
183
273
|
const status = h("span", { class: "call-status" });
|
|
184
274
|
const time = h("span", { class: "call-time" });
|
|
185
275
|
const who = h("span", { class: "call-who" });
|
|
186
276
|
const text = h("span", { class: "call-text" });
|
|
187
|
-
|
|
277
|
+
const opts = optsRow();
|
|
278
|
+
call.els = { bars, mute, gear, opts, status, time, who, text };
|
|
188
279
|
call.panel = h(
|
|
189
280
|
"div",
|
|
190
281
|
{ class: "call-panel", role: "region", "aria-label": "语音对话" },
|
|
@@ -193,8 +284,9 @@ function mountPanel() {
|
|
|
193
284
|
{ class: "call-main" },
|
|
194
285
|
h("div", { class: "call-wave", "aria-hidden": "true" }, bars),
|
|
195
286
|
h("div", { class: "call-info" }, h("div", { class: "call-line" }, status, time), h("div", { class: "call-caption", "aria-live": "polite" }, who, text)),
|
|
196
|
-
h("div", { class: "call-ctls" }, mute, end),
|
|
287
|
+
h("div", { class: "call-ctls" }, mute, gear, end),
|
|
197
288
|
),
|
|
289
|
+
opts,
|
|
198
290
|
);
|
|
199
291
|
}
|
|
200
292
|
ui.composer.classList.add("in-call");
|
|
@@ -204,6 +296,43 @@ function mountPanel() {
|
|
|
204
296
|
call.raf = requestAnimationFrame(tick);
|
|
205
297
|
}
|
|
206
298
|
|
|
299
|
+
/** 通话条下面展开的一行:音色、语言。通话中改了就用新设置重新连一次(Codex 那边对话上下文还在) */
|
|
300
|
+
function optsRow() {
|
|
301
|
+
const prefs = loadPrefs();
|
|
302
|
+
const pick = (label, list, value, name) => {
|
|
303
|
+
const sel = h("select", { "aria-label": label }, list.map(([v, t]) => h("option", { value: v }, t)));
|
|
304
|
+
sel.value = list.some(([v]) => v === value) ? value : "";
|
|
305
|
+
sel.addEventListener("change", () => {
|
|
306
|
+
savePrefs({ ...loadPrefs(), [name]: sel.value });
|
|
307
|
+
restartCall();
|
|
308
|
+
});
|
|
309
|
+
return h("label", { class: "call-opt" }, h("span", {}, label), sel);
|
|
310
|
+
};
|
|
311
|
+
return h(
|
|
312
|
+
"div",
|
|
313
|
+
{ class: "call-opts", hidden: true },
|
|
314
|
+
pick("音色", VOICES, prefs.voice, "voice"),
|
|
315
|
+
pick("语言", LANGUAGES, prefs.language, "language"),
|
|
316
|
+
h("div", { class: "call-opts-hint" }, "记在这台设备上。通话中改了会重新连接,对话内容不受影响"),
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function toggleOpts() {
|
|
321
|
+
const { opts, gear } = call.els;
|
|
322
|
+
opts.hidden = !opts.hidden;
|
|
323
|
+
gear.classList.toggle("active", !opts.hidden);
|
|
324
|
+
gear.setAttribute("aria-expanded", String(!opts.hidden));
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** 用新的音色 / 语言重连:挂掉当前这通,马上再开(在选择框的操作里,iOS 也允许播放) */
|
|
328
|
+
function restartCall() {
|
|
329
|
+
if (call.state === "idle") return;
|
|
330
|
+
endCall("");
|
|
331
|
+
void startCall();
|
|
332
|
+
// startCall 同步就把新的通话条挂上了:设置行保持展开,方便接着试别的音色
|
|
333
|
+
if (call.els) toggleOpts();
|
|
334
|
+
}
|
|
335
|
+
|
|
207
336
|
function toggleMute() {
|
|
208
337
|
call.muted = !call.muted;
|
|
209
338
|
for (const t of call.mic?.getAudioTracks() ?? []) t.enabled = !call.muted;
|
|
@@ -255,15 +384,42 @@ function clock(ms) {
|
|
|
255
384
|
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
|
|
256
385
|
}
|
|
257
386
|
|
|
258
|
-
/**
|
|
259
|
-
|
|
387
|
+
/** 整个页面共用一个 AudioContext(iOS 上反复新建、关闭会把麦克风搞坏),在点击里调用来建 / 恢复 */
|
|
388
|
+
let sharedCtx = null;
|
|
389
|
+
function audioCtx() {
|
|
260
390
|
try {
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
391
|
+
if (!sharedCtx || sharedCtx.state === "closed") sharedCtx = new (window.AudioContext ?? window.webkitAudioContext)();
|
|
392
|
+
if (sharedCtx.state !== "running") sharedCtx.resume().catch(() => {});
|
|
393
|
+
} catch {
|
|
394
|
+
sharedCtx = null;
|
|
395
|
+
}
|
|
396
|
+
return sharedCtx;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/** 轨道 muted 时等它恢复一会儿;还是不行就报错 */
|
|
400
|
+
function waitUnmute(track, ms) {
|
|
401
|
+
return new Promise((resolve, reject) => {
|
|
402
|
+
const timer = setTimeout(() => {
|
|
403
|
+
track.removeEventListener("unmute", ok);
|
|
404
|
+
reject(new Error("麦克风被系统静音了,请检查是否有其他应用在用麦克风,或刷新页面后重试"));
|
|
405
|
+
}, ms);
|
|
406
|
+
const ok = () => {
|
|
407
|
+
clearTimeout(timer);
|
|
408
|
+
resolve();
|
|
409
|
+
};
|
|
410
|
+
track.addEventListener("unmute", ok, { once: true });
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/** 一个节点的音量(0~1 的均方根),给动画用 */
|
|
415
|
+
function meter(node) {
|
|
416
|
+
try {
|
|
417
|
+
const ctx = audioCtx();
|
|
418
|
+
if (!ctx) return null;
|
|
419
|
+
const an = ctx.createAnalyser();
|
|
265
420
|
an.fftSize = 512;
|
|
266
|
-
|
|
421
|
+
node.connect(an);
|
|
422
|
+
call.nodes.push(an);
|
|
267
423
|
const buf = new Float32Array(an.fftSize);
|
|
268
424
|
return () => {
|
|
269
425
|
an.getFloatTimeDomainData(buf);
|
|
@@ -276,18 +432,54 @@ function meter(stream) {
|
|
|
276
432
|
}
|
|
277
433
|
}
|
|
278
434
|
|
|
279
|
-
/**
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
435
|
+
/** 每个 AudioContext 加载一次 worklet */
|
|
436
|
+
const workletLoaded = new WeakMap();
|
|
437
|
+
function loadWorklet(ctx) {
|
|
438
|
+
let p = workletLoaded.get(ctx);
|
|
439
|
+
if (!p) {
|
|
440
|
+
p = ctx.audioWorklet.addModule(new URL("../lib/rt-worklet.js", import.meta.url));
|
|
441
|
+
p.catch(() => workletLoaded.delete(ctx));
|
|
442
|
+
workletLoaded.set(ctx, p);
|
|
443
|
+
}
|
|
444
|
+
return p;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** 本页里一对 RTCPeerConnection 把声音环回一下,让浏览器的回声消除认得它(见 appleAudio) */
|
|
448
|
+
async function loopback(stream) {
|
|
449
|
+
const a = new RTCPeerConnection();
|
|
450
|
+
const b = new RTCPeerConnection();
|
|
451
|
+
const close = () => {
|
|
452
|
+
a.close();
|
|
453
|
+
b.close();
|
|
454
|
+
};
|
|
455
|
+
try {
|
|
456
|
+
a.onicecandidate = (e) => e.candidate && b.addIceCandidate(e.candidate).catch(() => {});
|
|
457
|
+
b.onicecandidate = (e) => e.candidate && a.addIceCandidate(e.candidate).catch(() => {});
|
|
458
|
+
const got = new Promise((resolve, reject) => {
|
|
459
|
+
b.ontrack = (e) => resolve(e.streams[0] ?? new MediaStream([e.track]));
|
|
460
|
+
setTimeout(() => reject(new Error("环回超时")), 3000);
|
|
461
|
+
});
|
|
462
|
+
for (const t of stream.getTracks()) a.addTrack(t, stream);
|
|
463
|
+
await a.setLocalDescription(await a.createOffer());
|
|
464
|
+
await b.setRemoteDescription(a.localDescription);
|
|
465
|
+
await b.setLocalDescription(await b.createAnswer());
|
|
466
|
+
await a.setRemoteDescription(b.localDescription);
|
|
467
|
+
return { stream: await got, close };
|
|
468
|
+
} catch (err) {
|
|
469
|
+
close();
|
|
470
|
+
throw err;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** 服务器转来的回复声音(WebSocket 二进制帧:24kHz 16 位 PCM) */
|
|
475
|
+
export function onRealtimeAudio(buf) {
|
|
476
|
+
if (call.state === "idle" || !call.play) return;
|
|
477
|
+
call.play.port.postMessage(buf, [buf]);
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** WebSocket 断了:声音走不了了,挂断 */
|
|
481
|
+
export function onSocketLost() {
|
|
482
|
+
endCall("和服务器的连接断开了,语音对话已结束", true);
|
|
291
483
|
}
|
|
292
484
|
|
|
293
485
|
window.addEventListener("pagehide", () => endCall());
|