agent-loft 0.2.116 → 0.2.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/channels/web/server.ts +32 -2
- package/src/core/realtime-relay.ts +89 -9
- package/web/lib/rt-worklet.js +74 -14
- package/web/views/voice-call.js +131 -9
package/README.md
CHANGED
|
@@ -54,7 +54,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
|
|
|
54
54
|
- **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
|
|
55
55
|
- 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
|
|
56
56
|
- 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
|
|
57
|
-
- 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框为空且空闲时,最右侧显示声波按钮;输入文字或添加附件后切换为发送,回复生成中仍可停止。点声波按钮打开沉浸式通话界面:蓝白光球随声音流动、缩放,配有状态、实时字幕、静音和结束按钮,可以随时打断,也可以收起成通话条继续查看聊天;要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI
|
|
57
|
+
- 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框为空且空闲时,最右侧显示声波按钮;输入文字或添加附件后切换为发送,回复生成中仍可停止。点声波按钮打开沉浸式通话界面:蓝白光球随声音流动、缩放,配有状态、实时字幕、静音和结束按钮,可以随时打断,也可以收起成通话条继续查看聊天;要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话界面的齿轮里可以选音色和回答用的语言(记在这台设备上),切换设置后重连会保留静音状态和收起状态。通话中会保持屏幕常亮;iOS 锁屏时系统不让网页收音(PWA 也一样),解锁回来会自动恢复麦克风,不用重打。每通电话结束会在服务日志里记一行 `[rt]` 传输统计(丢包、断流、播放放空),方便排查卡顿。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
|
|
58
58
|
- 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
|
|
59
59
|
- 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
|
|
60
60
|
- 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
|
package/package.json
CHANGED
|
@@ -270,7 +270,7 @@ type Client = {
|
|
|
270
270
|
/** 最后一次收到这个页面的消息(含 ping) */
|
|
271
271
|
seen: number;
|
|
272
272
|
/** 这个页面正在打的语音对话:声音经这里中转(relay 建好之前是 undefined) */
|
|
273
|
-
rt?: { key: string; relay?: RealtimeRelay };
|
|
273
|
+
rt?: { key: string; relay?: RealtimeRelay; startedAt?: number };
|
|
274
274
|
};
|
|
275
275
|
|
|
276
276
|
/** 一块麦克风 PCM 的上限(正常是 20ms ≈ 1KB) */
|
|
@@ -283,10 +283,36 @@ function endRelay(client: Client, stopCall: boolean): void {
|
|
|
283
283
|
const rt = client.rt;
|
|
284
284
|
if (!rt) return;
|
|
285
285
|
client.rt = undefined;
|
|
286
|
-
rt.relay
|
|
286
|
+
if (rt.relay) {
|
|
287
|
+
rt.relay.close();
|
|
288
|
+
if (rt.startedAt) console.log(`[rt] ${rt.key} 通话结束 ${relaySummary(rt.relay.stats, Date.now() - rt.startedAt)}`);
|
|
289
|
+
}
|
|
287
290
|
if (stopCall) void hub.realtimeStop(rt.key).catch(() => {});
|
|
288
291
|
}
|
|
289
292
|
|
|
293
|
+
function relaySummary(s: RealtimeRelay["stats"], ms: number): string {
|
|
294
|
+
const pct = (n: number, total: number) => (total ? `${((n / total) * 100).toFixed(1)}%` : "0%");
|
|
295
|
+
return [
|
|
296
|
+
`时长 ${Math.round(ms / 1000)}s`,
|
|
297
|
+
`下行 收 ${s.rxPackets} 包 丢 ${s.rxLost}(${pct(s.rxLost, s.rxPackets + s.rxLost)})补 ${s.rxConcealed}`,
|
|
298
|
+
`上行 发 ${s.txFrames} 帧 断流 ${s.txUnderruns} 次 丢 ${s.txDropped} 帧`,
|
|
299
|
+
].join(" · ");
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** 浏览器那头报上来的播放 / 采集统计(数字字段才要,防止乱写日志) */
|
|
303
|
+
function clientRtSummary(raw: unknown): string | undefined {
|
|
304
|
+
if (!raw || typeof raw !== "object") return undefined;
|
|
305
|
+
const parts: string[] = [];
|
|
306
|
+
const labels: Record<string, string> = { starves: "播放放空", jitter: "其中抖动", jbMs: "缓冲", skippedMs: "跳过", sendDropped: "采集丢块", micLost: "麦克风中断", ua: "" };
|
|
307
|
+
for (const [k, label] of Object.entries(labels)) {
|
|
308
|
+
const v = (raw as Record<string, unknown>)[k];
|
|
309
|
+
if (k === "ua") {
|
|
310
|
+
if (typeof v === "string") parts.push(v.slice(0, 40).replace(/[^\w ./-]/g, ""));
|
|
311
|
+
} else if (typeof v === "number" && Number.isFinite(v)) parts.push(`${label} ${Math.round(v)}${k.endsWith("Ms") ? "ms" : ""}`);
|
|
312
|
+
}
|
|
313
|
+
return parts.length ? parts.join(" · ") : undefined;
|
|
314
|
+
}
|
|
315
|
+
|
|
290
316
|
/** 每次进程启动换一个:客户端发现变了,就知道 seq 已经从头计数,得整页重拉 */
|
|
291
317
|
const EPOCH = Math.random().toString(36).slice(2, 10);
|
|
292
318
|
|
|
@@ -572,6 +598,7 @@ function handleClientMessage(client: Client, raw: string): void {
|
|
|
572
598
|
throw new Error("已挂断");
|
|
573
599
|
}
|
|
574
600
|
rt.relay = relay;
|
|
601
|
+
rt.startedAt = Date.now();
|
|
575
602
|
relay.onAudio = (pcm) => {
|
|
576
603
|
if (client.ws.readyState === client.ws.OPEN && client.ws.bufferedAmount < MAX_AUDIO_BACKLOG) client.ws.send(pcm, { binary: true });
|
|
577
604
|
};
|
|
@@ -590,6 +617,9 @@ function handleClientMessage(client: Client, raw: string): void {
|
|
|
590
617
|
});
|
|
591
618
|
return;
|
|
592
619
|
}
|
|
620
|
+
const clientStats = clientRtSummary(msg.stats);
|
|
621
|
+
if (clientStats) console.log(`[rt] ${key} 浏览器 ${clientStats}`);
|
|
622
|
+
if (msg.op === "stats") return ack(true);
|
|
593
623
|
if (msg.op === "stop") {
|
|
594
624
|
if (client.rt?.key === key) endRelay(client, false);
|
|
595
625
|
void hub.realtimeStop(key).then(() => ack(true), (err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
|
|
@@ -13,13 +13,36 @@ export const RELAY_RATE = 24000;
|
|
|
13
13
|
const FRAME = 480;
|
|
14
14
|
const FRAME_BYTES = FRAME * 2;
|
|
15
15
|
const RTP_STEP = 960;
|
|
16
|
-
/**
|
|
17
|
-
|
|
16
|
+
/**
|
|
17
|
+
* 上行(麦克风)按 20ms 的节拍匀速发给 OpenAI:浏览器经 WebSocket(TCP、过代理 / 隧道)送来的声音是一阵一阵的,
|
|
18
|
+
* 来了就发,OpenAI 那头的抖动缓冲会把晚到的丢掉、把空档当成你说完了,识别断断续续。
|
|
19
|
+
* 断流后攒够 PREBUFFER 再开始发;积压超过 CATCHUP 就每拍多发一帧追上;超过 MAX_PENDING 丢掉最旧的
|
|
20
|
+
*/
|
|
21
|
+
const PREBUFFER = FRAME_BYTES * 3;
|
|
22
|
+
const CATCHUP = FRAME_BYTES * 10;
|
|
23
|
+
const MAX_PENDING = FRAME_BYTES * 50;
|
|
24
|
+
const FRAME_MS = 20;
|
|
25
|
+
const PACE_MS = 10;
|
|
18
26
|
/**
|
|
19
27
|
* 收的方向丢了几个包以内用 Opus 的丢包补偿填上(解码空包 = PLC,一次给 120ms,按丢的帧数截)。
|
|
20
28
|
* 服务器到 OpenAI 这段常过代理,实测丢 2~3%:不填的话浏览器那头的缓冲每丢一包就少 20ms,过一阵就放空、一卡一卡
|
|
21
29
|
*/
|
|
22
30
|
const MAX_CONCEAL = 6;
|
|
31
|
+
/** libopus 的 CTL 编号 */
|
|
32
|
+
const OPUS_SET_INBAND_FEC = 4012;
|
|
33
|
+
const OPUS_SET_PACKET_LOSS_PERC = 4014;
|
|
34
|
+
|
|
35
|
+
/** 一通电话的传输统计(挂断时打到日志里,用来判断卡在哪一段) */
|
|
36
|
+
export interface RelayStats {
|
|
37
|
+
/** 下行(OpenAI → 这里)收到的包、序号上看丢了的包、用丢包补偿填上的帧 */
|
|
38
|
+
rxPackets: number;
|
|
39
|
+
rxLost: number;
|
|
40
|
+
rxConcealed: number;
|
|
41
|
+
/** 上行(这里 → OpenAI)发出的帧、断流(浏览器没及时送来)次数、积压太多丢掉的帧 */
|
|
42
|
+
txFrames: number;
|
|
43
|
+
txUnderruns: number;
|
|
44
|
+
txDropped: number;
|
|
45
|
+
}
|
|
23
46
|
|
|
24
47
|
export class RealtimeRelay {
|
|
25
48
|
/** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
|
|
@@ -38,8 +61,23 @@ export class RealtimeRelay {
|
|
|
38
61
|
/** 收到的上一个 RTP 包的序号 */
|
|
39
62
|
private lastSeq: number | undefined;
|
|
40
63
|
private dropTimer: NodeJS.Timeout | undefined;
|
|
64
|
+
private paceTimer: NodeJS.Timeout | undefined;
|
|
65
|
+
/** 上行正在匀速发(断流后要重新攒够 PREBUFFER) */
|
|
66
|
+
private sending = false;
|
|
67
|
+
/** 下一帧该发的时刻、上一帧实际发出的时刻(performance.now) */
|
|
68
|
+
private nextAt = 0;
|
|
69
|
+
private lastSentAt = 0;
|
|
70
|
+
readonly stats: RelayStats = { rxPackets: 0, rxLost: 0, rxConcealed: 0, txFrames: 0, txUnderruns: 0, txDropped: 0 };
|
|
41
71
|
|
|
42
72
|
private constructor() {
|
|
73
|
+
// 带内 FEC:下一个包里捎上前一个包的低码率副本,丢一个包 OpenAI 那头能补回来(服务器到 OpenAI 常过代理,会丢包)
|
|
74
|
+
try {
|
|
75
|
+
this.encoder.setBitrate(32000);
|
|
76
|
+
this.encoder.encoderCTL(OPUS_SET_INBAND_FEC, 1);
|
|
77
|
+
this.encoder.encoderCTL(OPUS_SET_PACKET_LOSS_PERC, 10);
|
|
78
|
+
} catch {
|
|
79
|
+
// 设不上就用默认的
|
|
80
|
+
}
|
|
43
81
|
this.pc = new RTCPeerConnection({
|
|
44
82
|
codecs: { audio: [new RTCRtpCodecParameters({ mimeType: "audio/opus", clockRate: 48000, channels: 2 })] },
|
|
45
83
|
});
|
|
@@ -52,9 +90,13 @@ export class RealtimeRelay {
|
|
|
52
90
|
const gap = (seq - this.lastSeq) & 0xffff;
|
|
53
91
|
// 重复的、迟到的(已经补过或跳过了):丢掉,别打乱顺序
|
|
54
92
|
if (gap === 0 || gap > 0x8000) return;
|
|
55
|
-
if (gap > 1
|
|
93
|
+
if (gap > 1) {
|
|
94
|
+
this.stats.rxLost += gap - 1;
|
|
95
|
+
if (gap - 1 <= MAX_CONCEAL) this.conceal(gap - 1);
|
|
96
|
+
}
|
|
56
97
|
}
|
|
57
98
|
this.lastSeq = seq;
|
|
99
|
+
this.stats.rxPackets++;
|
|
58
100
|
try {
|
|
59
101
|
this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
|
|
60
102
|
} catch {
|
|
@@ -92,22 +134,58 @@ export class RealtimeRelay {
|
|
|
92
134
|
await this.pc.setRemoteDescription({ type: "answer", sdp: answer });
|
|
93
135
|
}
|
|
94
136
|
|
|
95
|
-
/** 浏览器发来的麦克风 PCM
|
|
137
|
+
/** 浏览器发来的麦克风 PCM:先攒着,由节拍器每 20ms 编码发一帧(见 PREBUFFER) */
|
|
96
138
|
pushAudio(pcm: Buffer): void {
|
|
97
139
|
if (this.closed || !pcm.length) return;
|
|
98
140
|
this.pending = this.pending.length ? Buffer.concat([this.pending, pcm]) : Buffer.from(pcm);
|
|
99
|
-
if (this.pending.length > MAX_PENDING)
|
|
100
|
-
|
|
101
|
-
const
|
|
102
|
-
this.
|
|
103
|
-
this.
|
|
141
|
+
if (this.pending.length > MAX_PENDING) {
|
|
142
|
+
const over = this.pending.length - MAX_PENDING;
|
|
143
|
+
const frames = Math.ceil(over / FRAME_BYTES);
|
|
144
|
+
this.stats.txDropped += frames;
|
|
145
|
+
this.pending = this.pending.subarray(Math.min(this.pending.length, frames * FRAME_BYTES));
|
|
146
|
+
}
|
|
147
|
+
this.paceTimer ??= setInterval(() => this.pace(), PACE_MS);
|
|
148
|
+
this.pace();
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** 节拍器:到点就发一帧;没货了算断流,等重新攒够再发 */
|
|
152
|
+
private pace(now = performance.now()): void {
|
|
153
|
+
if (this.closed) return;
|
|
154
|
+
if (!this.sending) {
|
|
155
|
+
if (this.pending.length < PREBUFFER) return;
|
|
156
|
+
this.sending = true;
|
|
157
|
+
// 断了一阵:时间戳跟着真实时间往前跳,OpenAI 那头当成中间没声音,而不是把后面的当迟到的扔掉
|
|
158
|
+
if (this.lastSentAt) {
|
|
159
|
+
const skipped = Math.round((now - this.lastSentAt) / FRAME_MS) - 1;
|
|
160
|
+
if (skipped > 0) this.ts = (this.ts + skipped * RTP_STEP) >>> 0;
|
|
161
|
+
}
|
|
162
|
+
this.nextAt = now;
|
|
104
163
|
}
|
|
164
|
+
while (now >= this.nextAt) {
|
|
165
|
+
if (this.pending.length < FRAME_BYTES) {
|
|
166
|
+
this.sending = false;
|
|
167
|
+
this.stats.txUnderruns++;
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
this.sendFrame(this.takeFrame());
|
|
171
|
+
// 积压多了(浏览器卡了一阵又一下子涌来):这一拍多发一帧,慢慢追回来
|
|
172
|
+
if (this.pending.length > CATCHUP) this.sendFrame(this.takeFrame());
|
|
173
|
+
this.lastSentAt = now;
|
|
174
|
+
this.nextAt += FRAME_MS;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
private takeFrame(): Buffer {
|
|
179
|
+
const frame = this.pending.subarray(0, FRAME_BYTES);
|
|
180
|
+
this.pending = this.pending.subarray(FRAME_BYTES);
|
|
181
|
+
return frame;
|
|
105
182
|
}
|
|
106
183
|
|
|
107
184
|
close(): void {
|
|
108
185
|
if (this.closed) return;
|
|
109
186
|
this.closed = true;
|
|
110
187
|
clearTimeout(this.dropTimer);
|
|
188
|
+
clearInterval(this.paceTimer);
|
|
111
189
|
this.pending = Buffer.alloc(0);
|
|
112
190
|
void this.pc.close().catch(() => {});
|
|
113
191
|
for (const codec of [this.encoder, this.decoder]) {
|
|
@@ -129,6 +207,7 @@ export class RealtimeRelay {
|
|
|
129
207
|
const header = new RtpHeader({ sequenceNumber: this.seq, timestamp: this.ts, marker: false, payloadType: 111 });
|
|
130
208
|
this.seq = (this.seq + 1) & 0xffff;
|
|
131
209
|
this.ts = (this.ts + RTP_STEP) >>> 0;
|
|
210
|
+
this.stats.txFrames++;
|
|
132
211
|
try {
|
|
133
212
|
this.track.writeRtp(new RtpPacket(header, Buffer.from(payload)));
|
|
134
213
|
} catch {
|
|
@@ -141,6 +220,7 @@ export class RealtimeRelay {
|
|
|
141
220
|
try {
|
|
142
221
|
const pcm = this.decoder.decode(Buffer.alloc(0));
|
|
143
222
|
this.onAudio(Buffer.from(pcm.subarray(0, Math.min(pcm.length, frames * FRAME_BYTES))));
|
|
223
|
+
this.stats.rxConcealed += frames;
|
|
144
224
|
} catch {
|
|
145
225
|
// 补不了就算了
|
|
146
226
|
}
|
package/web/lib/rt-worklet.js
CHANGED
|
@@ -1,22 +1,27 @@
|
|
|
1
1
|
// 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
|
|
2
|
-
// rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
|
|
2
|
+
// rt-capture:麦克风 → 低通(防混叠)→ 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
|
|
3
3
|
// rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
|
|
4
4
|
// 放空之后很快(JB_JITTER_SEC 内)又来了声音 = 说到一半网络抖了:把缓冲加长一点;隔得久的是正常停顿,不算。
|
|
5
|
-
//
|
|
5
|
+
// 一阵子没再放空就慢慢缩回去;攒得比目标多(卡了一阵又一下子涌来)就稍微放快一点追回来,
|
|
6
|
+
// 多得离谱才直接跳(跳会丢字,听起来就是断断续续)
|
|
7
|
+
// 每 STATS_SEC 秒把统计(放空几次、缓冲多长、跳过多少)报给主线程,挂断时带给服务器记日志
|
|
6
8
|
|
|
7
9
|
const RATE = 24000;
|
|
8
10
|
const CHUNK = 480; // 20ms
|
|
9
|
-
/** 播放缓冲(24kHz
|
|
10
|
-
const JB_START = RATE * 0.
|
|
11
|
-
const JB_MIN = RATE * 0.
|
|
12
|
-
const JB_MAX = RATE * 0.
|
|
13
|
-
const JB_STEP = RATE * 0.
|
|
11
|
+
/** 播放缓冲(24kHz 采样数):起步、最短、最长、每次放空加多少。手机走流量 / 隧道,起步给足一点 */
|
|
12
|
+
const JB_START = RATE * 0.18;
|
|
13
|
+
const JB_MIN = RATE * 0.1;
|
|
14
|
+
const JB_MAX = RATE * 0.6;
|
|
15
|
+
const JB_STEP = RATE * 0.08;
|
|
14
16
|
/** 这么久(秒)没放空,缓冲缩短 20ms */
|
|
15
17
|
const JB_DECAY_SEC = 8;
|
|
16
18
|
/** 放空后这么久(秒)内又来了声音,算网络抖动 */
|
|
17
19
|
const JB_JITTER_SEC = 0.3;
|
|
18
|
-
/**
|
|
19
|
-
const
|
|
20
|
+
/** 比目标多出这么多就放快 JB_FAST 倍追(音调高一点点,听不太出来);多出 JB_SLACK 才直接跳回目标 */
|
|
21
|
+
const JB_AHEAD = RATE * 0.15;
|
|
22
|
+
const JB_FAST = 1.06;
|
|
23
|
+
const JB_SLACK = RATE * 1.5;
|
|
24
|
+
const STATS_SEC = 2;
|
|
20
25
|
/** 没攒够目标、但这么久(秒)没再来声音:先放掉(很短的一句别卡在缓冲里) */
|
|
21
26
|
const JB_FLUSH_SEC = 0.1;
|
|
22
27
|
|
|
@@ -28,12 +33,22 @@ class Capture extends AudioWorkletProcessor {
|
|
|
28
33
|
this.prev = 0; // 上一块的最后一个采样,插值跨块用
|
|
29
34
|
this.out = new Int16Array(CHUNK);
|
|
30
35
|
this.n = 0;
|
|
36
|
+
// 降到 24kHz 之前先把 10.5kHz 以上滤掉(两节巴特沃斯二阶 = 四阶),不然高频折回来成了杂音,识别更差
|
|
37
|
+
this.lp = sampleRate > RATE * 1.05 ? [biquadLowpass(10500, sampleRate, 0.5412), biquadLowpass(10500, sampleRate, 1.3066)] : null;
|
|
38
|
+
this.tmp = new Float32Array(128);
|
|
31
39
|
}
|
|
32
40
|
|
|
33
41
|
process(inputs) {
|
|
34
|
-
|
|
42
|
+
let input = inputs[0]?.[0];
|
|
35
43
|
if (!input) return true;
|
|
36
44
|
// 多声道输入只取第一路就够(getUserMedia 一般是单声道)
|
|
45
|
+
if (this.lp) {
|
|
46
|
+
if (this.tmp.length < input.length) this.tmp = new Float32Array(input.length);
|
|
47
|
+
const x = this.tmp.subarray(0, input.length);
|
|
48
|
+
x.set(input);
|
|
49
|
+
for (const f of this.lp) f(x);
|
|
50
|
+
input = x;
|
|
51
|
+
}
|
|
37
52
|
const len = input.length;
|
|
38
53
|
// 位置 -1 对应 prev,0..len-1 对应 input
|
|
39
54
|
while (this.pos < len) {
|
|
@@ -74,6 +89,9 @@ class Play extends AudioWorkletProcessor {
|
|
|
74
89
|
this.starvedAt = -1;
|
|
75
90
|
/** 最近一次收到声音的时刻(currentFrame) */
|
|
76
91
|
this.lastData = 0;
|
|
92
|
+
/** 统计:说着说着放空几次、其中算网络抖动的几次、直接跳过了多少采样 */
|
|
93
|
+
this.stats = { starves: 0, jitter: 0, skipped: 0 };
|
|
94
|
+
this.statsAt = 0;
|
|
77
95
|
this.port.onmessage = (e) => {
|
|
78
96
|
if (e.data === "flush") {
|
|
79
97
|
this.avail = 0;
|
|
@@ -85,7 +103,10 @@ class Play extends AudioWorkletProcessor {
|
|
|
85
103
|
const pcm = new Int16Array(e.data);
|
|
86
104
|
this.lastData = currentFrame;
|
|
87
105
|
if (this.starvedAt >= 0) {
|
|
88
|
-
if (currentFrame - this.starvedAt < sampleRate * JB_JITTER_SEC)
|
|
106
|
+
if (currentFrame - this.starvedAt < sampleRate * JB_JITTER_SEC) {
|
|
107
|
+
this.target = Math.min(JB_MAX, this.target + JB_STEP);
|
|
108
|
+
this.stats.jitter++;
|
|
109
|
+
}
|
|
89
110
|
this.starvedAt = -1;
|
|
90
111
|
}
|
|
91
112
|
// 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
|
|
@@ -93,17 +114,19 @@ class Play extends AudioWorkletProcessor {
|
|
|
93
114
|
if (over > 0) {
|
|
94
115
|
this.read = (this.read + over) % this.size;
|
|
95
116
|
this.avail -= over;
|
|
117
|
+
this.stats.skipped += over;
|
|
96
118
|
}
|
|
97
119
|
for (let i = 0; i < pcm.length; i++) {
|
|
98
120
|
this.buf[this.write] = pcm[i] / 0x8000;
|
|
99
121
|
this.write = (this.write + 1) % this.size;
|
|
100
122
|
}
|
|
101
123
|
this.avail += pcm.length;
|
|
102
|
-
//
|
|
124
|
+
// 多得离谱(卡了很久又一下子涌来):直接跳回目标;多一点的在 process 里放快追
|
|
103
125
|
if (this.avail > this.target + JB_SLACK) {
|
|
104
126
|
const skip = this.avail - this.target;
|
|
105
127
|
this.read = (this.read + skip) % this.size;
|
|
106
128
|
this.avail -= skip;
|
|
129
|
+
this.stats.skipped += skip;
|
|
107
130
|
}
|
|
108
131
|
};
|
|
109
132
|
}
|
|
@@ -117,9 +140,13 @@ class Play extends AudioWorkletProcessor {
|
|
|
117
140
|
this.playing = true;
|
|
118
141
|
this.smooth = 0;
|
|
119
142
|
}
|
|
143
|
+
const step = this.avail > this.target + JB_AHEAD ? this.step * JB_FAST : this.step;
|
|
120
144
|
for (let i = 0; i < ch.length; i++) {
|
|
121
145
|
if (!this.playing || this.avail < 2) {
|
|
122
|
-
if (this.playing)
|
|
146
|
+
if (this.playing) {
|
|
147
|
+
this.starvedAt = currentFrame + i;
|
|
148
|
+
this.stats.starves++;
|
|
149
|
+
}
|
|
123
150
|
ch[i] = 0;
|
|
124
151
|
this.playing = false;
|
|
125
152
|
continue;
|
|
@@ -129,7 +156,7 @@ class Play extends AudioWorkletProcessor {
|
|
|
129
156
|
const a = this.buf[i0 % this.size];
|
|
130
157
|
const b = this.buf[(i0 + 1) % this.size];
|
|
131
158
|
ch[i] = a + (b - a) * f;
|
|
132
|
-
const next = this.read +
|
|
159
|
+
const next = this.read + step;
|
|
133
160
|
const used = Math.floor(next) - i0;
|
|
134
161
|
this.avail -= used;
|
|
135
162
|
this.read = next % this.size;
|
|
@@ -139,9 +166,42 @@ class Play extends AudioWorkletProcessor {
|
|
|
139
166
|
this.target = Math.max(JB_MIN, this.target - RATE * 0.02);
|
|
140
167
|
}
|
|
141
168
|
for (let c = 1; c < out.length; c++) out[c].set(ch);
|
|
169
|
+
if (currentFrame - this.statsAt > sampleRate * STATS_SEC) {
|
|
170
|
+
this.statsAt = currentFrame;
|
|
171
|
+
const { starves, jitter, skipped } = this.stats;
|
|
172
|
+
this.port.postMessage({ starves, jitter, jbMs: (this.target / RATE) * 1000, skippedMs: (skipped / RATE) * 1000 });
|
|
173
|
+
}
|
|
142
174
|
return true;
|
|
143
175
|
}
|
|
144
176
|
}
|
|
145
177
|
|
|
178
|
+
/** 二阶低通(RBJ biquad),返回一个就地处理 Float32Array 的函数,状态跨块保留 */
|
|
179
|
+
function biquadLowpass(freq, rate, q) {
|
|
180
|
+
const w = (2 * Math.PI * freq) / rate;
|
|
181
|
+
const alpha = Math.sin(w) / (2 * q);
|
|
182
|
+
const cos = Math.cos(w);
|
|
183
|
+
const a0 = 1 + alpha;
|
|
184
|
+
const b0 = (1 - cos) / 2 / a0;
|
|
185
|
+
const b1 = (1 - cos) / a0;
|
|
186
|
+
const b2 = b0;
|
|
187
|
+
const a1 = (-2 * cos) / a0;
|
|
188
|
+
const a2 = (1 - alpha) / a0;
|
|
189
|
+
let x1 = 0;
|
|
190
|
+
let x2 = 0;
|
|
191
|
+
let y1 = 0;
|
|
192
|
+
let y2 = 0;
|
|
193
|
+
return (buf) => {
|
|
194
|
+
for (let i = 0; i < buf.length; i++) {
|
|
195
|
+
const x = buf[i];
|
|
196
|
+
const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
|
|
197
|
+
x2 = x1;
|
|
198
|
+
x1 = x;
|
|
199
|
+
y2 = y1;
|
|
200
|
+
y1 = y < 1e-25 && y > -1e-25 ? 0 : y;
|
|
201
|
+
buf[i] = y;
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
|
|
146
206
|
registerProcessor("rt-capture", Capture);
|
|
147
207
|
registerProcessor("rt-play", Play);
|
package/web/views/voice-call.js
CHANGED
|
@@ -29,6 +29,17 @@ const call = {
|
|
|
29
29
|
mic: null,
|
|
30
30
|
/** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
|
|
31
31
|
play: null,
|
|
32
|
+
/** 采集用的 worklet 节点、接着麦克风的源节点(锁屏回来麦克风换了要重接) */
|
|
33
|
+
capture: null,
|
|
34
|
+
src: null,
|
|
35
|
+
/** 传输统计:play 那头报上来的 + 这边数的,挂断时带给服务器记日志 */
|
|
36
|
+
stats: null,
|
|
37
|
+
/** 屏幕常亮锁(通话中别自动锁屏:iOS 锁屏后网页收不到音) */
|
|
38
|
+
wakeLock: null,
|
|
39
|
+
/** 最近一次听到声音的时刻:状态文字防闪(字和字之间的停顿别跳回「正在听」) */
|
|
40
|
+
loudAt: 0,
|
|
41
|
+
/** 正在重新拿麦克风 */
|
|
42
|
+
recovering: false,
|
|
32
43
|
/** Chrome 等的回声消除环回(见 appleAudio) */
|
|
33
44
|
loop: null,
|
|
34
45
|
audio: null,
|
|
@@ -139,6 +150,9 @@ async function startCall() {
|
|
|
139
150
|
call.compact = false;
|
|
140
151
|
call.muted = false;
|
|
141
152
|
call.caption = { role: null, text: "", done: false };
|
|
153
|
+
call.stats = { sendDropped: 0, micLost: 0 };
|
|
154
|
+
call.loudAt = 0;
|
|
155
|
+
call.recovering = false;
|
|
142
156
|
// 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
|
|
143
157
|
call.audio = h("audio", { autoplay: true, playsinline: true });
|
|
144
158
|
call.audio.play().catch(() => {});
|
|
@@ -160,30 +174,35 @@ async function startCall() {
|
|
|
160
174
|
if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
|
|
161
175
|
track.enabled = !call.muted;
|
|
162
176
|
if (track.muted) await waitUnmute(track, 1500);
|
|
177
|
+
watchTrack(track);
|
|
163
178
|
if (session !== call.session) return;
|
|
164
179
|
const ctx = audioCtx();
|
|
165
180
|
if (!ctx) throw new Error("浏览器不支持音频处理");
|
|
166
181
|
await loadWorklet(ctx);
|
|
167
182
|
if (session !== call.session) return;
|
|
168
|
-
// 麦克风 →
|
|
169
|
-
const src = ctx.createMediaStreamSource(call.mic);
|
|
183
|
+
// 麦克风 → 采集(低通、重采样到 24kHz,20ms 一块)→ WebSocket
|
|
170
184
|
const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
185
|
+
call.capture = capture;
|
|
171
186
|
// 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
|
|
172
187
|
const sink = ctx.createGain();
|
|
173
188
|
sink.gain.value = 0;
|
|
174
|
-
src.connect(capture);
|
|
175
189
|
capture.connect(sink);
|
|
176
190
|
sink.connect(ctx.destination);
|
|
177
191
|
capture.port.onmessage = (e) => {
|
|
192
|
+
if (call.state !== "live") return;
|
|
178
193
|
const sock = WS.sock;
|
|
179
194
|
// 网络卡住时别越攒越多(攒下的说出去也晚了)
|
|
180
|
-
if (
|
|
195
|
+
if (sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
|
|
196
|
+
else call.stats.sendDropped++;
|
|
181
197
|
};
|
|
198
|
+
attachMic(ctx);
|
|
182
199
|
// 服务器发来的 → 播放 → 扬声器
|
|
183
200
|
const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
|
|
201
|
+
play.port.onmessage = (e) => {
|
|
202
|
+
if (e.data && typeof e.data === "object") Object.assign(call.stats, e.data);
|
|
203
|
+
};
|
|
184
204
|
call.play = play;
|
|
185
|
-
call.nodes.push(
|
|
186
|
-
call.micLevel = meter(src);
|
|
205
|
+
call.nodes.push(capture, sink, play);
|
|
187
206
|
call.outLevel = meter(play);
|
|
188
207
|
if (appleAudio) {
|
|
189
208
|
play.connect(ctx.destination);
|
|
@@ -209,6 +228,7 @@ async function startCall() {
|
|
|
209
228
|
if (session !== call.session) return;
|
|
210
229
|
call.state = "live";
|
|
211
230
|
call.startedAt = Date.now();
|
|
231
|
+
void keepAwake();
|
|
212
232
|
render();
|
|
213
233
|
} catch (err) {
|
|
214
234
|
if (session !== call.session) return;
|
|
@@ -222,13 +242,17 @@ export function endCall(reason, fromServer = false) {
|
|
|
222
242
|
if (call.state === "idle") return;
|
|
223
243
|
const key = call.key;
|
|
224
244
|
++call.session;
|
|
245
|
+
const wasLive = call.state === "live";
|
|
225
246
|
call.state = "idle";
|
|
226
|
-
|
|
247
|
+
const stats = wasLive && call.stats ? { ...call.stats, ua: deviceLabel() } : undefined;
|
|
248
|
+
if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key, stats });
|
|
249
|
+
else if (key && stats) WS.raw({ t: "rt", op: "stats", key, stats });
|
|
227
250
|
cancelAnimationFrame(call.raf);
|
|
251
|
+
releaseWakeLock();
|
|
228
252
|
stopTracks();
|
|
229
253
|
call.loop?.close();
|
|
230
254
|
call.loop = null;
|
|
231
|
-
call.play = null;
|
|
255
|
+
call.play = call.capture = call.src = null;
|
|
232
256
|
if (call.audio) {
|
|
233
257
|
call.audio.pause();
|
|
234
258
|
call.audio.srcObject = null;
|
|
@@ -262,6 +286,102 @@ function stopTracks() {
|
|
|
262
286
|
call.mic = null;
|
|
263
287
|
}
|
|
264
288
|
|
|
289
|
+
/** 把 call.mic 接到采集节点上(换了麦克风先断开旧的) */
|
|
290
|
+
function attachMic(ctx) {
|
|
291
|
+
try {
|
|
292
|
+
call.src?.disconnect();
|
|
293
|
+
} catch {
|
|
294
|
+
// 已经断开
|
|
295
|
+
}
|
|
296
|
+
const src = ctx.createMediaStreamSource(call.mic);
|
|
297
|
+
src.connect(call.capture);
|
|
298
|
+
call.src = src;
|
|
299
|
+
call.nodes.push(src);
|
|
300
|
+
call.micLevel = meter(src);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** 盯着麦克风轨道:被系统结束了(锁屏、来电、被别的应用抢走)就试着重新拿 */
|
|
304
|
+
function watchTrack(track) {
|
|
305
|
+
const session = call.session;
|
|
306
|
+
track.addEventListener?.("ended", () => {
|
|
307
|
+
if (session !== call.session || call.state === "idle" || call.mic?.getAudioTracks()[0] !== track) return;
|
|
308
|
+
call.stats.micLost++;
|
|
309
|
+
if (document.visibilityState === "visible") void recoverMic();
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* 从后台 / 锁屏回来:iOS 上锁屏时网页拿不到麦克风(PWA 也一样,系统不让),AudioContext 会被打断成 interrupted,
|
|
315
|
+
* 麦克风轨道可能已经结束或一直静音。这里把 AudioContext 恢复、麦克风重新拿一次接回去
|
|
316
|
+
*/
|
|
317
|
+
async function recoverMic() {
|
|
318
|
+
if (call.state !== "live" || call.recovering) return;
|
|
319
|
+
const session = call.session;
|
|
320
|
+
call.recovering = true;
|
|
321
|
+
try {
|
|
322
|
+
const ctx = audioCtx();
|
|
323
|
+
if (!ctx) return;
|
|
324
|
+
const old = call.mic?.getAudioTracks()[0];
|
|
325
|
+
if (old && old.readyState === "live" && !old.muted) return;
|
|
326
|
+
if (old?.muted) {
|
|
327
|
+
// 刚回到前台,系统一般马上就还回来
|
|
328
|
+
const back = await waitUnmute(old, 800).then(() => true, () => false);
|
|
329
|
+
if (back || session !== call.session) return;
|
|
330
|
+
}
|
|
331
|
+
const mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
|
|
332
|
+
if (session !== call.session) {
|
|
333
|
+
for (const t of mic.getTracks()) t.stop();
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
stopTracks();
|
|
337
|
+
call.mic = mic;
|
|
338
|
+
const track = mic.getAudioTracks()[0];
|
|
339
|
+
if (track) {
|
|
340
|
+
track.enabled = !call.muted;
|
|
341
|
+
watchTrack(track);
|
|
342
|
+
}
|
|
343
|
+
attachMic(ctx);
|
|
344
|
+
toast("麦克风已恢复(锁屏时 iOS 不让网页收音)");
|
|
345
|
+
} catch {
|
|
346
|
+
if (session === call.session) toast("麦克风被系统中断了,挂断后重新打一通试试", 4000);
|
|
347
|
+
} finally {
|
|
348
|
+
if (session === call.session) call.recovering = false;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/** 通话中保持屏幕常亮:iOS 自动锁屏后网页收不到音(手动按锁屏键挡不住) */
|
|
353
|
+
async function keepAwake() {
|
|
354
|
+
if (!navigator.wakeLock || document.visibilityState !== "visible" || call.state !== "live") return;
|
|
355
|
+
try {
|
|
356
|
+
const lock = await navigator.wakeLock.request("screen");
|
|
357
|
+
if (call.state !== "live") return void lock.release().catch(() => {});
|
|
358
|
+
call.wakeLock = lock;
|
|
359
|
+
// 切到后台系统会自动释放,回来时再要
|
|
360
|
+
lock.addEventListener?.("release", () => {
|
|
361
|
+
if (call.wakeLock === lock) call.wakeLock = null;
|
|
362
|
+
});
|
|
363
|
+
} catch {
|
|
364
|
+
// 不支持 / 低电量模式下拒绝:算了
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function releaseWakeLock() {
|
|
369
|
+
const lock = call.wakeLock;
|
|
370
|
+
call.wakeLock = null;
|
|
371
|
+
lock?.release().catch(() => {});
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function deviceLabel() {
|
|
375
|
+
const standalone = window.matchMedia?.("(display-mode: standalone)").matches || navigator.standalone === true;
|
|
376
|
+
return `${appleAudio ? "Apple" : "Other"}${standalone ? " PWA" : ""}`;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
document.addEventListener("visibilitychange", () => {
|
|
380
|
+
if (document.visibilityState !== "visible" || call.state !== "live") return;
|
|
381
|
+
void keepAwake();
|
|
382
|
+
void recoverMic();
|
|
383
|
+
});
|
|
384
|
+
|
|
265
385
|
/** 服务端转来的通话事件(ws 的 t: "rt") */
|
|
266
386
|
export function onRealtimeEvent(msg) {
|
|
267
387
|
if (call.state === "idle" || msg.key !== call.key) return;
|
|
@@ -431,7 +551,9 @@ function tick() {
|
|
|
431
551
|
if (!call.els) return;
|
|
432
552
|
const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
|
|
433
553
|
const out = call.outLevel?.() ?? 0;
|
|
434
|
-
|
|
554
|
+
// 字和字之间的停顿也会掉到阈值以下:响过之后 0.4 秒内都算在说,状态别来回闪
|
|
555
|
+
if (out > 0.02) call.loudAt = performance.now();
|
|
556
|
+
const speaking = out > 0.02 || performance.now() - call.loudAt < 400;
|
|
435
557
|
const level = Math.min(1, (speaking ? out : mic) * 6);
|
|
436
558
|
call.els.orb.style.setProperty("--voice-scale", (1 + level * 0.14).toFixed(3));
|
|
437
559
|
call.panel.classList.toggle("speaking", speaking);
|