agent-loft 0.2.115 → 0.2.117

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,7 @@ Logo 就是这个意思:一大两小几颗星(多个 agent)站在同一张
54
54
  - **回复里的相册**:agent 用 ```` ```album ```` JSON 描述多张本地照片,网页以大图轮播展示,可左右滑动、点击箭头或底部缩略图切换,按需加载照片;点开预览后支持左右切换、缩放、拖动、方向键和 Esc。可附名称、拍摄日期、说明和查询总数,每个相册最多 200 张;飞书显示简短摘要。写法自动附在所有专家的提示词里,图片沿用会话文件权限。
55
55
  - 聊天右上角「更多」和侧栏会话右键菜单共用分叉、重命名、在飞书继续、删除操作,按会话类型和可用机器人显示。
56
56
  - 语音输入:用浏览器自带的语音识别(Chrome / Edge / Safari,需要 HTTPS)。
57
- - 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框为空且空闲时,最右侧显示声波按钮;输入文字或添加附件后切换为发送,回复生成中仍可停止。点声波按钮打开沉浸式通话界面:蓝白光球随声音流动、缩放,配有状态、实时字幕、静音和结束按钮,可以随时打断,也可以收起成通话条继续查看聊天;要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话界面的齿轮里可以选音色和回答用的语言(记在这台设备上),切换设置后重连会保留静音状态和收起状态。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
57
+ - 语音对话(只有 Codex,包括 Codex 引擎的自定义专家):输入框为空且空闲时,最右侧显示声波按钮;输入文字或添加附件后切换为发送,回复生成中仍可停止。点声波按钮打开沉浸式通话界面:蓝白光球随声音流动、缩放,配有状态、实时字幕、静音和结束按钮,可以随时打断,也可以收起成通话条继续查看聊天;要动手的事语音模型会转交给这条对话里的 Codex,那一轮像发了一条消息一样出现在对话里(前面带 🎙️),授权卡照常弹。走 Codex 的实时语音(v3),用的是 Codex 登录的 ChatGPT 账号。声音经服务器中转:服务器自己和 OpenAI 建 WebRTC,浏览器和服务器之间走网页原有的 WebSocket,所以能打开网页的地方(反向代理、Cloudflare Tunnel)就能语音,手机不用直连 OpenAI。通话界面的齿轮里可以选音色和回答用的语言(记在这台设备上),切换设置后重连会保留静音状态和收起状态。通话中会保持屏幕常亮;iOS 锁屏时系统不让网页收音(PWA 也一样),解锁回来会自动恢复麦克风,不用重打。每通电话结束会在服务日志里记一行 `[rt]` 传输统计(丢包、断流、播放放空),方便排查卡顿。开通话前会把这条对话当前的权限和沙箱同步给 Codex,转交出来的轮次一样受限;机器上最近会话的启动背景只给内置 Codex 的管理员。需要 HTTPS 和麦克风权限;桌面端(Electron)里暂不提供。闲聊的字幕不存进聊天记录,切走对话或关掉页面会挂断。
58
58
  - 新对话页:点大标题里的目录名,在固定高度的目录弹窗里逐层选工作目录,也能新建文件夹;输入框上方选专家,鼠标停在专家上看简介,右下角选模型和思考程度。对话从头到尾一个专家,要换专家就新建对话;模型和思考程度随时可换。
59
59
  - 输入框里的小圆环显示上下文窗口占用;模型按钮显示实际的模型和思考程度(如「Opus 5.5 high」)。
60
60
  - 滚动和 ChatGPT 一样:发出的问题顶到最上面、下面留白,回复长到屏幕底部才跟着往下滚;回复过程中自己滚动了就不再自动滚,下次发消息再恢复。
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-loft",
3
- "version": "0.2.115",
3
+ "version": "0.2.117",
4
4
  "type": "module",
5
5
  "description": "多个 agent 的工作台:把本机 Claude Code / Codex 放到一处统一管理,用飞书和 Web 页面随时驱动——会话、审批、模型切换都共用",
6
6
  "keywords": [
@@ -867,6 +867,10 @@ export class CodexSession implements AgentSession {
867
867
  }
868
868
  }
869
869
 
870
+ inRealtime(): boolean {
871
+ return !!this.rt;
872
+ }
873
+
870
874
  async stopRealtime(): Promise<void> {
871
875
  if (!this.rt) return;
872
876
  this.rt.fail?.(new Error("已挂断"));
@@ -270,7 +270,7 @@ type Client = {
270
270
  /** 最后一次收到这个页面的消息(含 ping) */
271
271
  seen: number;
272
272
  /** 这个页面正在打的语音对话:声音经这里中转(relay 建好之前是 undefined) */
273
- rt?: { key: string; relay?: RealtimeRelay };
273
+ rt?: { key: string; relay?: RealtimeRelay; startedAt?: number };
274
274
  };
275
275
 
276
276
  /** 一块麦克风 PCM 的上限(正常是 20ms ≈ 1KB) */
@@ -283,10 +283,36 @@ function endRelay(client: Client, stopCall: boolean): void {
283
283
  const rt = client.rt;
284
284
  if (!rt) return;
285
285
  client.rt = undefined;
286
- rt.relay?.close();
286
+ if (rt.relay) {
287
+ rt.relay.close();
288
+ if (rt.startedAt) console.log(`[rt] ${rt.key} 通话结束 ${relaySummary(rt.relay.stats, Date.now() - rt.startedAt)}`);
289
+ }
287
290
  if (stopCall) void hub.realtimeStop(rt.key).catch(() => {});
288
291
  }
289
292
 
293
+ function relaySummary(s: RealtimeRelay["stats"], ms: number): string {
294
+ const pct = (n: number, total: number) => (total ? `${((n / total) * 100).toFixed(1)}%` : "0%");
295
+ return [
296
+ `时长 ${Math.round(ms / 1000)}s`,
297
+ `下行 收 ${s.rxPackets} 包 丢 ${s.rxLost}(${pct(s.rxLost, s.rxPackets + s.rxLost)})补 ${s.rxConcealed}`,
298
+ `上行 发 ${s.txFrames} 帧 断流 ${s.txUnderruns} 次 丢 ${s.txDropped} 帧`,
299
+ ].join(" · ");
300
+ }
301
+
302
+ /** 浏览器那头报上来的播放 / 采集统计(数字字段才要,防止乱写日志) */
303
+ function clientRtSummary(raw: unknown): string | undefined {
304
+ if (!raw || typeof raw !== "object") return undefined;
305
+ const parts: string[] = [];
306
+ const labels: Record<string, string> = { starves: "播放放空", jitter: "其中抖动", jbMs: "缓冲", skippedMs: "跳过", sendDropped: "采集丢块", micLost: "麦克风中断", ua: "" };
307
+ for (const [k, label] of Object.entries(labels)) {
308
+ const v = (raw as Record<string, unknown>)[k];
309
+ if (k === "ua") {
310
+ if (typeof v === "string") parts.push(v.slice(0, 40).replace(/[^\w ./-]/g, ""));
311
+ } else if (typeof v === "number" && Number.isFinite(v)) parts.push(`${label} ${Math.round(v)}${k.endsWith("Ms") ? "ms" : ""}`);
312
+ }
313
+ return parts.length ? parts.join(" · ") : undefined;
314
+ }
315
+
290
316
  /** 每次进程启动换一个:客户端发现变了,就知道 seq 已经从头计数,得整页重拉 */
291
317
  const EPOCH = Math.random().toString(36).slice(2, 10);
292
318
 
@@ -572,6 +598,7 @@ function handleClientMessage(client: Client, raw: string): void {
572
598
  throw new Error("已挂断");
573
599
  }
574
600
  rt.relay = relay;
601
+ rt.startedAt = Date.now();
575
602
  relay.onAudio = (pcm) => {
576
603
  if (client.ws.readyState === client.ws.OPEN && client.ws.bufferedAmount < MAX_AUDIO_BACKLOG) client.ws.send(pcm, { binary: true });
577
604
  };
@@ -590,6 +617,9 @@ function handleClientMessage(client: Client, raw: string): void {
590
617
  });
591
618
  return;
592
619
  }
620
+ const clientStats = clientRtSummary(msg.stats);
621
+ if (clientStats) console.log(`[rt] ${key} 浏览器 ${clientStats}`);
622
+ if (msg.op === "stats") return ack(true);
593
623
  if (msg.op === "stop") {
594
624
  if (client.rt?.key === key) endRelay(client, false);
595
625
  void hub.realtimeStop(key).then(() => ack(true), (err) => ack(false, { error: err instanceof Error ? err.message : String(err) }));
package/src/core/hub.ts CHANGED
@@ -846,7 +846,10 @@ class Hub extends EventEmitter<{
846
846
  this.externalTurn(key, text, anchor);
847
847
  },
848
848
  // 不看 alive:会话被换下去时发的「通话结束」也要让页面知道
849
- onRealtime: (ev) => this.emit("realtime", { key, ev }),
849
+ onRealtime: (ev) => {
850
+ this.emit("realtime", { key, ev });
851
+ if (ev.type === "closed") this.realtimeEnded(key);
852
+ },
850
853
  onStep: (kind, text) => alive() && this.emitEvent(key, { type: "step", kind, text }),
851
854
  onTodos: (todos) => alive() && this.emitEvent(key, { type: "todos", todos }),
852
855
  onTextDelta: (text) => alive() && this.emitEvent(key, { type: "delta", text }),
@@ -1847,7 +1850,21 @@ class Hub extends EventEmitter<{
1847
1850
  || (this.pendingOps.get(key) ?? 0) > 0
1848
1851
  || this.isolatedRuns.has(key)
1849
1852
  || this.rewinding.has(key)
1850
- || interactions.listPending(key).length > 0;
1853
+ || interactions.listPending(key).length > 0
1854
+ || this.inCall(key);
1855
+ }
1856
+
1857
+ /** 正在实时语音通话:通话本身不走轮次,不算进 isBusy,但进程不能闲置回收、也不能因为记忆更新被换掉 */
1858
+ private inCall(key: string): boolean {
1859
+ return this.live.get(key)?.session.inRealtime?.() === true;
1860
+ }
1861
+
1862
+ /** 通话结束:从这时起重新算闲置;通话期间攒下的「记忆已更新」这时再换进程 */
1863
+ private realtimeEnded(key: string): void {
1864
+ const live = this.live.get(key);
1865
+ if (!live) return;
1866
+ this.liveUsedAt.set(live.id, Date.now());
1867
+ this.dropIfStale(key);
1851
1868
  }
1852
1869
 
1853
1870
  /** 关掉空闲太久的会话进程(只关进程,不动绑定),见 LIVE_IDLE_MS */
@@ -1877,13 +1894,13 @@ class Hub extends EventEmitter<{
1877
1894
 
1878
1895
  private dropIfStale(key: string): void {
1879
1896
  const live = this.live.get(key);
1880
- if (!live || !this.staleLive.has(live.id) || this.isBusy(key)) return;
1897
+ if (!live || !this.staleLive.has(live.id) || this.isBusy(key) || this.inCall(key)) return;
1881
1898
  this.staleLive.delete(live.id);
1882
1899
  void this.serial(key, async () => {
1883
1900
  // 排队期间可能已经换了进程、或者又开始说话了:那就留到下一轮结束
1884
1901
  const now = this.live.get(key);
1885
1902
  if (now?.id !== live.id) return;
1886
- if (this.isBusy(key)) {
1903
+ if (this.isBusy(key) || this.inCall(key)) {
1887
1904
  this.staleLive.add(live.id);
1888
1905
  return;
1889
1906
  }
@@ -2071,6 +2088,7 @@ class Hub extends EventEmitter<{
2071
2088
 
2072
2089
  async realtimeStop(key: string): Promise<void> {
2073
2090
  await this.live.get(key)?.session.stopRealtime?.();
2091
+ this.realtimeEnded(key);
2074
2092
  }
2075
2093
 
2076
2094
  /**
@@ -13,8 +13,36 @@ export const RELAY_RATE = 24000;
13
13
  const FRAME = 480;
14
14
  const FRAME_BYTES = FRAME * 2;
15
15
  const RTP_STEP = 960;
16
- /** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
17
- const MAX_PENDING = RELAY_RATE * 2;
16
+ /**
17
+ * 上行(麦克风)按 20ms 的节拍匀速发给 OpenAI:浏览器经 WebSocket(TCP、过代理 / 隧道)送来的声音是一阵一阵的,
18
+ * 来了就发,OpenAI 那头的抖动缓冲会把晚到的丢掉、把空档当成你说完了,识别断断续续。
19
+ * 断流后攒够 PREBUFFER 再开始发;积压超过 CATCHUP 就每拍多发一帧追上;超过 MAX_PENDING 丢掉最旧的
20
+ */
21
+ const PREBUFFER = FRAME_BYTES * 3;
22
+ const CATCHUP = FRAME_BYTES * 10;
23
+ const MAX_PENDING = FRAME_BYTES * 50;
24
+ const FRAME_MS = 20;
25
+ const PACE_MS = 10;
26
+ /**
27
+ * 收的方向丢了几个包以内用 Opus 的丢包补偿填上(解码空包 = PLC,一次给 120ms,按丢的帧数截)。
28
+ * 服务器到 OpenAI 这段常过代理,实测丢 2~3%:不填的话浏览器那头的缓冲每丢一包就少 20ms,过一阵就放空、一卡一卡
29
+ */
30
+ const MAX_CONCEAL = 6;
31
+ /** libopus 的 CTL 编号 */
32
+ const OPUS_SET_INBAND_FEC = 4012;
33
+ const OPUS_SET_PACKET_LOSS_PERC = 4014;
34
+
35
+ /** 一通电话的传输统计(挂断时打到日志里,用来判断卡在哪一段) */
36
+ export interface RelayStats {
37
+ /** 下行(OpenAI → 这里)收到的包、序号上看丢了的包、用丢包补偿填上的帧 */
38
+ rxPackets: number;
39
+ rxLost: number;
40
+ rxConcealed: number;
41
+ /** 上行(这里 → OpenAI)发出的帧、断流(浏览器没及时送来)次数、积压太多丢掉的帧 */
42
+ txFrames: number;
43
+ txUnderruns: number;
44
+ txDropped: number;
45
+ }
18
46
 
19
47
  export class RealtimeRelay {
20
48
  /** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
@@ -30,9 +58,26 @@ export class RealtimeRelay {
30
58
  private seq = Math.floor(Math.random() * 0xffff);
31
59
  private ts = Math.floor(Math.random() * 0xffffffff);
32
60
  private closed = false;
61
+ /** 收到的上一个 RTP 包的序号 */
62
+ private lastSeq: number | undefined;
33
63
  private dropTimer: NodeJS.Timeout | undefined;
64
+ private paceTimer: NodeJS.Timeout | undefined;
65
+ /** 上行正在匀速发(断流后要重新攒够 PREBUFFER) */
66
+ private sending = false;
67
+ /** 下一帧该发的时刻、上一帧实际发出的时刻(performance.now) */
68
+ private nextAt = 0;
69
+ private lastSentAt = 0;
70
+ readonly stats: RelayStats = { rxPackets: 0, rxLost: 0, rxConcealed: 0, txFrames: 0, txUnderruns: 0, txDropped: 0 };
34
71
 
35
72
  private constructor() {
73
+ // 带内 FEC:下一个包里捎上前一个包的低码率副本,丢一个包 OpenAI 那头能补回来(服务器到 OpenAI 常过代理,会丢包)
74
+ try {
75
+ this.encoder.setBitrate(32000);
76
+ this.encoder.encoderCTL(OPUS_SET_INBAND_FEC, 1);
77
+ this.encoder.encoderCTL(OPUS_SET_PACKET_LOSS_PERC, 10);
78
+ } catch {
79
+ // 设不上就用默认的
80
+ }
36
81
  this.pc = new RTCPeerConnection({
37
82
  codecs: { audio: [new RTCRtpCodecParameters({ mimeType: "audio/opus", clockRate: 48000, channels: 2 })] },
38
83
  });
@@ -40,6 +85,18 @@ export class RealtimeRelay {
40
85
  transceiver.onTrack.subscribe((remote) => {
41
86
  remote.onReceiveRtp.subscribe((rtp) => {
42
87
  if (this.closed || !rtp.payload.length) return;
88
+ const seq = rtp.header.sequenceNumber;
89
+ if (this.lastSeq !== undefined) {
90
+ const gap = (seq - this.lastSeq) & 0xffff;
91
+ // 重复的、迟到的(已经补过或跳过了):丢掉,别打乱顺序
92
+ if (gap === 0 || gap > 0x8000) return;
93
+ if (gap > 1) {
94
+ this.stats.rxLost += gap - 1;
95
+ if (gap - 1 <= MAX_CONCEAL) this.conceal(gap - 1);
96
+ }
97
+ }
98
+ this.lastSeq = seq;
99
+ this.stats.rxPackets++;
43
100
  try {
44
101
  this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
45
102
  } catch {
@@ -77,22 +134,58 @@ export class RealtimeRelay {
77
134
  await this.pc.setRemoteDescription({ type: "answer", sdp: answer });
78
135
  }
79
136
 
80
- /** 浏览器发来的麦克风 PCM:凑够 20ms 就编码发出去 */
137
+ /** 浏览器发来的麦克风 PCM:先攒着,由节拍器每 20ms 编码发一帧(见 PREBUFFER) */
81
138
  pushAudio(pcm: Buffer): void {
82
139
  if (this.closed || !pcm.length) return;
83
140
  this.pending = this.pending.length ? Buffer.concat([this.pending, pcm]) : Buffer.from(pcm);
84
- if (this.pending.length > MAX_PENDING) this.pending = this.pending.subarray(this.pending.length - MAX_PENDING);
85
- while (this.pending.length >= FRAME_BYTES) {
86
- const frame = this.pending.subarray(0, FRAME_BYTES);
87
- this.pending = this.pending.subarray(FRAME_BYTES);
88
- this.sendFrame(frame);
141
+ if (this.pending.length > MAX_PENDING) {
142
+ const over = this.pending.length - MAX_PENDING;
143
+ const frames = Math.ceil(over / FRAME_BYTES);
144
+ this.stats.txDropped += frames;
145
+ this.pending = this.pending.subarray(Math.min(this.pending.length, frames * FRAME_BYTES));
146
+ }
147
+ this.paceTimer ??= setInterval(() => this.pace(), PACE_MS);
148
+ this.pace();
149
+ }
150
+
151
+ /** 节拍器:到点就发一帧;没货了算断流,等重新攒够再发 */
152
+ private pace(now = performance.now()): void {
153
+ if (this.closed) return;
154
+ if (!this.sending) {
155
+ if (this.pending.length < PREBUFFER) return;
156
+ this.sending = true;
157
+ // 断了一阵:时间戳跟着真实时间往前跳,OpenAI 那头当成中间没声音,而不是把后面的当迟到的扔掉
158
+ if (this.lastSentAt) {
159
+ const skipped = Math.round((now - this.lastSentAt) / FRAME_MS) - 1;
160
+ if (skipped > 0) this.ts = (this.ts + skipped * RTP_STEP) >>> 0;
161
+ }
162
+ this.nextAt = now;
89
163
  }
164
+ while (now >= this.nextAt) {
165
+ if (this.pending.length < FRAME_BYTES) {
166
+ this.sending = false;
167
+ this.stats.txUnderruns++;
168
+ return;
169
+ }
170
+ this.sendFrame(this.takeFrame());
171
+ // 积压多了(浏览器卡了一阵又一下子涌来):这一拍多发一帧,慢慢追回来
172
+ if (this.pending.length > CATCHUP) this.sendFrame(this.takeFrame());
173
+ this.lastSentAt = now;
174
+ this.nextAt += FRAME_MS;
175
+ }
176
+ }
177
+
178
+ private takeFrame(): Buffer {
179
+ const frame = this.pending.subarray(0, FRAME_BYTES);
180
+ this.pending = this.pending.subarray(FRAME_BYTES);
181
+ return frame;
90
182
  }
91
183
 
92
184
  close(): void {
93
185
  if (this.closed) return;
94
186
  this.closed = true;
95
187
  clearTimeout(this.dropTimer);
188
+ clearInterval(this.paceTimer);
96
189
  this.pending = Buffer.alloc(0);
97
190
  void this.pc.close().catch(() => {});
98
191
  for (const codec of [this.encoder, this.decoder]) {
@@ -114,6 +207,7 @@ export class RealtimeRelay {
114
207
  const header = new RtpHeader({ sequenceNumber: this.seq, timestamp: this.ts, marker: false, payloadType: 111 });
115
208
  this.seq = (this.seq + 1) & 0xffff;
116
209
  this.ts = (this.ts + RTP_STEP) >>> 0;
210
+ this.stats.txFrames++;
117
211
  try {
118
212
  this.track.writeRtp(new RtpPacket(header, Buffer.from(payload)));
119
213
  } catch {
@@ -121,6 +215,17 @@ export class RealtimeRelay {
121
215
  }
122
216
  }
123
217
 
218
+ /** 丢了 frames 个 20ms 的包:用解码器的丢包补偿顶上同样长的声音 */
219
+ private conceal(frames: number): void {
220
+ try {
221
+ const pcm = this.decoder.decode(Buffer.alloc(0));
222
+ this.onAudio(Buffer.from(pcm.subarray(0, Math.min(pcm.length, frames * FRAME_BYTES))));
223
+ this.stats.rxConcealed += frames;
224
+ } catch {
225
+ // 补不了就算了
226
+ }
227
+ }
228
+
124
229
  private fail(reason: string): void {
125
230
  if (this.closed) return;
126
231
  this.close();
package/src/core/types.ts CHANGED
@@ -321,6 +321,8 @@ export interface AgentSession {
321
321
  startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
322
322
  /** 挂断实时语音对话;没在通话就什么都不做 */
323
323
  stopRealtime?(): Promise<void>;
324
+ /** 正在实时语音通话(含建连中):这期间进程不能当闲置回收 */
325
+ inRealtime?(): boolean;
324
326
  }
325
327
 
326
328
  export type Diagnosis = {
@@ -1,9 +1,29 @@
1
1
  // 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
2
- // rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
2
+ // rt-capture:麦克风 → 低通(防混叠)→ 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
3
3
  // rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
4
+ // 放空之后很快(JB_JITTER_SEC 内)又来了声音 = 说到一半网络抖了:把缓冲加长一点;隔得久的是正常停顿,不算。
5
+ // 一阵子没再放空就慢慢缩回去;攒得比目标多(卡了一阵又一下子涌来)就稍微放快一点追回来,
6
+ // 多得离谱才直接跳(跳会丢字,听起来就是断断续续)
7
+ // 每 STATS_SEC 秒把统计(放空几次、缓冲多长、跳过多少)报给主线程,挂断时带给服务器记日志
4
8
 
5
9
  const RATE = 24000;
6
10
  const CHUNK = 480; // 20ms
11
+ /** 播放缓冲(24kHz 采样数):起步、最短、最长、每次放空加多少。手机走流量 / 隧道,起步给足一点 */
12
+ const JB_START = RATE * 0.18;
13
+ const JB_MIN = RATE * 0.1;
14
+ const JB_MAX = RATE * 0.6;
15
+ const JB_STEP = RATE * 0.08;
16
+ /** 这么久(秒)没放空,缓冲缩短 20ms */
17
+ const JB_DECAY_SEC = 8;
18
+ /** 放空后这么久(秒)内又来了声音,算网络抖动 */
19
+ const JB_JITTER_SEC = 0.3;
20
+ /** 比目标多出这么多就放快 JB_FAST 倍追(音调高一点点,听不太出来);多出 JB_SLACK 才直接跳回目标 */
21
+ const JB_AHEAD = RATE * 0.15;
22
+ const JB_FAST = 1.06;
23
+ const JB_SLACK = RATE * 1.5;
24
+ const STATS_SEC = 2;
25
+ /** 没攒够目标、但这么久(秒)没再来声音:先放掉(很短的一句别卡在缓冲里) */
26
+ const JB_FLUSH_SEC = 0.1;
7
27
 
8
28
  class Capture extends AudioWorkletProcessor {
9
29
  constructor() {
@@ -13,12 +33,22 @@ class Capture extends AudioWorkletProcessor {
13
33
  this.prev = 0; // 上一块的最后一个采样,插值跨块用
14
34
  this.out = new Int16Array(CHUNK);
15
35
  this.n = 0;
36
+ // 降到 24kHz 之前先把 10.5kHz 以上滤掉(两节巴特沃斯二阶 = 四阶),不然高频折回来成了杂音,识别更差
37
+ this.lp = sampleRate > RATE * 1.05 ? [biquadLowpass(10500, sampleRate, 0.5412), biquadLowpass(10500, sampleRate, 1.3066)] : null;
38
+ this.tmp = new Float32Array(128);
16
39
  }
17
40
 
18
41
  process(inputs) {
19
- const input = inputs[0]?.[0];
42
+ let input = inputs[0]?.[0];
20
43
  if (!input) return true;
21
44
  // 多声道输入只取第一路就够(getUserMedia 一般是单声道)
45
+ if (this.lp) {
46
+ if (this.tmp.length < input.length) this.tmp = new Float32Array(input.length);
47
+ const x = this.tmp.subarray(0, input.length);
48
+ x.set(input);
49
+ for (const f of this.lp) f(x);
50
+ input = x;
51
+ }
22
52
  const len = input.length;
23
53
  // 位置 -1 对应 prev,0..len-1 对应 input
24
54
  while (this.pos < len) {
@@ -52,29 +82,51 @@ class Play extends AudioWorkletProcessor {
52
82
  this.avail = 0; // 还没放的采样数
53
83
  this.write = 0;
54
84
  this.playing = false;
85
+ this.target = JB_START;
86
+ /** 上次放空 / 缩短缓冲以来放了多少个输出采样 */
87
+ this.smooth = 0;
88
+ /** 说着说着放空的时刻(currentFrame),等下一块声音来了判断是抖动还是停顿 */
89
+ this.starvedAt = -1;
90
+ /** 最近一次收到声音的时刻(currentFrame) */
91
+ this.lastData = 0;
92
+ /** 统计:说着说着放空几次、其中算网络抖动的几次、直接跳过了多少采样 */
93
+ this.stats = { starves: 0, jitter: 0, skipped: 0 };
94
+ this.statsAt = 0;
55
95
  this.port.onmessage = (e) => {
56
96
  if (e.data === "flush") {
57
97
  this.avail = 0;
58
98
  this.playing = false;
99
+ this.target = JB_START;
100
+ this.starvedAt = -1;
59
101
  return;
60
102
  }
61
103
  const pcm = new Int16Array(e.data);
104
+ this.lastData = currentFrame;
105
+ if (this.starvedAt >= 0) {
106
+ if (currentFrame - this.starvedAt < sampleRate * JB_JITTER_SEC) {
107
+ this.target = Math.min(JB_MAX, this.target + JB_STEP);
108
+ this.stats.jitter++;
109
+ }
110
+ this.starvedAt = -1;
111
+ }
62
112
  // 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
63
113
  const over = this.avail + pcm.length - this.size;
64
114
  if (over > 0) {
65
115
  this.read = (this.read + over) % this.size;
66
116
  this.avail -= over;
117
+ this.stats.skipped += over;
67
118
  }
68
119
  for (let i = 0; i < pcm.length; i++) {
69
120
  this.buf[this.write] = pcm[i] / 0x8000;
70
121
  this.write = (this.write + 1) % this.size;
71
122
  }
72
123
  this.avail += pcm.length;
73
- // 太多了(超过 1 秒)说明延迟越积越大,追到只剩 0.2 秒
74
- if (this.avail > RATE) {
75
- const skip = this.avail - RATE / 5;
124
+ // 多得离谱(卡了很久又一下子涌来):直接跳回目标;多一点的在 process 里放快追
125
+ if (this.avail > this.target + JB_SLACK) {
126
+ const skip = this.avail - this.target;
76
127
  this.read = (this.read + skip) % this.size;
77
128
  this.avail -= skip;
129
+ this.stats.skipped += skip;
78
130
  }
79
131
  };
80
132
  }
@@ -83,10 +135,18 @@ class Play extends AudioWorkletProcessor {
83
135
  const out = outputs[0];
84
136
  const ch = out[0];
85
137
  if (!ch) return true;
86
- // 先攒 60ms 再开始放,扛一点网络抖动;放空了就停下来重新攒
87
- if (!this.playing && this.avail >= RATE * 0.06) this.playing = true;
138
+ // 先攒够目标再开始放,扛网络抖动;放空了就停下来重新攒,下次多攒一点
139
+ if (!this.playing && this.avail >= 2 && (this.avail >= this.target || currentFrame - this.lastData > sampleRate * JB_FLUSH_SEC)) {
140
+ this.playing = true;
141
+ this.smooth = 0;
142
+ }
143
+ const step = this.avail > this.target + JB_AHEAD ? this.step * JB_FAST : this.step;
88
144
  for (let i = 0; i < ch.length; i++) {
89
145
  if (!this.playing || this.avail < 2) {
146
+ if (this.playing) {
147
+ this.starvedAt = currentFrame + i;
148
+ this.stats.starves++;
149
+ }
90
150
  ch[i] = 0;
91
151
  this.playing = false;
92
152
  continue;
@@ -96,15 +156,52 @@ class Play extends AudioWorkletProcessor {
96
156
  const a = this.buf[i0 % this.size];
97
157
  const b = this.buf[(i0 + 1) % this.size];
98
158
  ch[i] = a + (b - a) * f;
99
- const next = this.read + this.step;
159
+ const next = this.read + step;
100
160
  const used = Math.floor(next) - i0;
101
161
  this.avail -= used;
102
162
  this.read = next % this.size;
103
163
  }
164
+ if (this.playing && (this.smooth += ch.length) > sampleRate * JB_DECAY_SEC) {
165
+ this.smooth = 0;
166
+ this.target = Math.max(JB_MIN, this.target - RATE * 0.02);
167
+ }
104
168
  for (let c = 1; c < out.length; c++) out[c].set(ch);
169
+ if (currentFrame - this.statsAt > sampleRate * STATS_SEC) {
170
+ this.statsAt = currentFrame;
171
+ const { starves, jitter, skipped } = this.stats;
172
+ this.port.postMessage({ starves, jitter, jbMs: (this.target / RATE) * 1000, skippedMs: (skipped / RATE) * 1000 });
173
+ }
105
174
  return true;
106
175
  }
107
176
  }
108
177
 
178
+ /** 二阶低通(RBJ biquad),返回一个就地处理 Float32Array 的函数,状态跨块保留 */
179
+ function biquadLowpass(freq, rate, q) {
180
+ const w = (2 * Math.PI * freq) / rate;
181
+ const alpha = Math.sin(w) / (2 * q);
182
+ const cos = Math.cos(w);
183
+ const a0 = 1 + alpha;
184
+ const b0 = (1 - cos) / 2 / a0;
185
+ const b1 = (1 - cos) / a0;
186
+ const b2 = b0;
187
+ const a1 = (-2 * cos) / a0;
188
+ const a2 = (1 - alpha) / a0;
189
+ let x1 = 0;
190
+ let x2 = 0;
191
+ let y1 = 0;
192
+ let y2 = 0;
193
+ return (buf) => {
194
+ for (let i = 0; i < buf.length; i++) {
195
+ const x = buf[i];
196
+ const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
197
+ x2 = x1;
198
+ x1 = x;
199
+ y2 = y1;
200
+ y1 = y < 1e-25 && y > -1e-25 ? 0 : y;
201
+ buf[i] = y;
202
+ }
203
+ };
204
+ }
205
+
109
206
  registerProcessor("rt-capture", Capture);
110
207
  registerProcessor("rt-play", Play);
@@ -29,6 +29,17 @@ const call = {
29
29
  mic: null,
30
30
  /** 播放用的 worklet 节点:服务器发来的 PCM 交给它 */
31
31
  play: null,
32
+ /** 采集用的 worklet 节点、接着麦克风的源节点(锁屏回来麦克风换了要重接) */
33
+ capture: null,
34
+ src: null,
35
+ /** 传输统计:play 那头报上来的 + 这边数的,挂断时带给服务器记日志 */
36
+ stats: null,
37
+ /** 屏幕常亮锁(通话中别自动锁屏:iOS 锁屏后网页收不到音) */
38
+ wakeLock: null,
39
+ /** 最近一次听到声音的时刻:状态文字防闪(字和字之间的停顿别跳回「正在听」) */
40
+ loudAt: 0,
41
+ /** 正在重新拿麦克风 */
42
+ recovering: false,
32
43
  /** Chrome 等的回声消除环回(见 appleAudio) */
33
44
  loop: null,
34
45
  audio: null,
@@ -139,6 +150,9 @@ async function startCall() {
139
150
  call.compact = false;
140
151
  call.muted = false;
141
152
  call.caption = { role: null, text: "", done: false };
153
+ call.stats = { sendDropped: 0, micLost: 0 };
154
+ call.loudAt = 0;
155
+ call.recovering = false;
142
156
  // 在点击里建好、先 play 一次:iOS 上不是用户手势里起的播放会被拦
143
157
  call.audio = h("audio", { autoplay: true, playsinline: true });
144
158
  call.audio.play().catch(() => {});
@@ -160,30 +174,35 @@ async function startCall() {
160
174
  if (!track || track.readyState !== "live") throw new Error("麦克风没能打开,请关掉其他在用麦克风的页面或应用后重试");
161
175
  track.enabled = !call.muted;
162
176
  if (track.muted) await waitUnmute(track, 1500);
177
+ watchTrack(track);
163
178
  if (session !== call.session) return;
164
179
  const ctx = audioCtx();
165
180
  if (!ctx) throw new Error("浏览器不支持音频处理");
166
181
  await loadWorklet(ctx);
167
182
  if (session !== call.session) return;
168
- // 麦克风 → 采集(重采样到 24kHz,20ms 一块)→ WebSocket
169
- const src = ctx.createMediaStreamSource(call.mic);
183
+ // 麦克风 → 采集(低通、重采样到 24kHz,20ms 一块)→ WebSocket
170
184
  const capture = new AudioWorkletNode(ctx, "rt-capture", { numberOfInputs: 1, numberOfOutputs: 1, outputChannelCount: [1] });
185
+ call.capture = capture;
171
186
  // 没有输出接到 destination 的节点有的浏览器不跑:接一个静音的
172
187
  const sink = ctx.createGain();
173
188
  sink.gain.value = 0;
174
- src.connect(capture);
175
189
  capture.connect(sink);
176
190
  sink.connect(ctx.destination);
177
191
  capture.port.onmessage = (e) => {
192
+ if (call.state !== "live") return;
178
193
  const sock = WS.sock;
179
194
  // 网络卡住时别越攒越多(攒下的说出去也晚了)
180
- if (call.state === "live" && sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
195
+ if (sock?.readyState === 1 && sock.bufferedAmount < 64 * 1024) sock.send(e.data);
196
+ else call.stats.sendDropped++;
181
197
  };
198
+ attachMic(ctx);
182
199
  // 服务器发来的 → 播放 → 扬声器
183
200
  const play = new AudioWorkletNode(ctx, "rt-play", { numberOfInputs: 0, numberOfOutputs: 1, outputChannelCount: [1] });
201
+ play.port.onmessage = (e) => {
202
+ if (e.data && typeof e.data === "object") Object.assign(call.stats, e.data);
203
+ };
184
204
  call.play = play;
185
- call.nodes.push(src, capture, sink, play);
186
- call.micLevel = meter(src);
205
+ call.nodes.push(capture, sink, play);
187
206
  call.outLevel = meter(play);
188
207
  if (appleAudio) {
189
208
  play.connect(ctx.destination);
@@ -209,6 +228,7 @@ async function startCall() {
209
228
  if (session !== call.session) return;
210
229
  call.state = "live";
211
230
  call.startedAt = Date.now();
231
+ void keepAwake();
212
232
  render();
213
233
  } catch (err) {
214
234
  if (session !== call.session) return;
@@ -222,13 +242,17 @@ export function endCall(reason, fromServer = false) {
222
242
  if (call.state === "idle") return;
223
243
  const key = call.key;
224
244
  ++call.session;
245
+ const wasLive = call.state === "live";
225
246
  call.state = "idle";
226
- if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key });
247
+ const stats = wasLive && call.stats ? { ...call.stats, ua: deviceLabel() } : undefined;
248
+ if (key && !fromServer) WS.raw({ t: "rt", op: "stop", key, stats });
249
+ else if (key && stats) WS.raw({ t: "rt", op: "stats", key, stats });
227
250
  cancelAnimationFrame(call.raf);
251
+ releaseWakeLock();
228
252
  stopTracks();
229
253
  call.loop?.close();
230
254
  call.loop = null;
231
- call.play = null;
255
+ call.play = call.capture = call.src = null;
232
256
  if (call.audio) {
233
257
  call.audio.pause();
234
258
  call.audio.srcObject = null;
@@ -262,6 +286,102 @@ function stopTracks() {
262
286
  call.mic = null;
263
287
  }
264
288
 
289
+ /** 把 call.mic 接到采集节点上(换了麦克风先断开旧的) */
290
+ function attachMic(ctx) {
291
+ try {
292
+ call.src?.disconnect();
293
+ } catch {
294
+ // 已经断开
295
+ }
296
+ const src = ctx.createMediaStreamSource(call.mic);
297
+ src.connect(call.capture);
298
+ call.src = src;
299
+ call.nodes.push(src);
300
+ call.micLevel = meter(src);
301
+ }
302
+
303
+ /** 盯着麦克风轨道:被系统结束了(锁屏、来电、被别的应用抢走)就试着重新拿 */
304
+ function watchTrack(track) {
305
+ const session = call.session;
306
+ track.addEventListener?.("ended", () => {
307
+ if (session !== call.session || call.state === "idle" || call.mic?.getAudioTracks()[0] !== track) return;
308
+ call.stats.micLost++;
309
+ if (document.visibilityState === "visible") void recoverMic();
310
+ });
311
+ }
312
+
313
+ /**
314
+ * 从后台 / 锁屏回来:iOS 上锁屏时网页拿不到麦克风(PWA 也一样,系统不让),AudioContext 会被打断成 interrupted,
315
+ * 麦克风轨道可能已经结束或一直静音。这里把 AudioContext 恢复、麦克风重新拿一次接回去
316
+ */
317
+ async function recoverMic() {
318
+ if (call.state !== "live" || call.recovering) return;
319
+ const session = call.session;
320
+ call.recovering = true;
321
+ try {
322
+ const ctx = audioCtx();
323
+ if (!ctx) return;
324
+ const old = call.mic?.getAudioTracks()[0];
325
+ if (old && old.readyState === "live" && !old.muted) return;
326
+ if (old?.muted) {
327
+ // 刚回到前台,系统一般马上就还回来
328
+ const back = await waitUnmute(old, 800).then(() => true, () => false);
329
+ if (back || session !== call.session) return;
330
+ }
331
+ const mic = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } });
332
+ if (session !== call.session) {
333
+ for (const t of mic.getTracks()) t.stop();
334
+ return;
335
+ }
336
+ stopTracks();
337
+ call.mic = mic;
338
+ const track = mic.getAudioTracks()[0];
339
+ if (track) {
340
+ track.enabled = !call.muted;
341
+ watchTrack(track);
342
+ }
343
+ attachMic(ctx);
344
+ toast("麦克风已恢复(锁屏时 iOS 不让网页收音)");
345
+ } catch {
346
+ if (session === call.session) toast("麦克风被系统中断了,挂断后重新打一通试试", 4000);
347
+ } finally {
348
+ if (session === call.session) call.recovering = false;
349
+ }
350
+ }
351
+
352
+ /** 通话中保持屏幕常亮:iOS 自动锁屏后网页收不到音(手动按锁屏键挡不住) */
353
+ async function keepAwake() {
354
+ if (!navigator.wakeLock || document.visibilityState !== "visible" || call.state !== "live") return;
355
+ try {
356
+ const lock = await navigator.wakeLock.request("screen");
357
+ if (call.state !== "live") return void lock.release().catch(() => {});
358
+ call.wakeLock = lock;
359
+ // 切到后台系统会自动释放,回来时再要
360
+ lock.addEventListener?.("release", () => {
361
+ if (call.wakeLock === lock) call.wakeLock = null;
362
+ });
363
+ } catch {
364
+ // 不支持 / 低电量模式下拒绝:算了
365
+ }
366
+ }
367
+
368
+ function releaseWakeLock() {
369
+ const lock = call.wakeLock;
370
+ call.wakeLock = null;
371
+ lock?.release().catch(() => {});
372
+ }
373
+
374
+ function deviceLabel() {
375
+ const standalone = window.matchMedia?.("(display-mode: standalone)").matches || navigator.standalone === true;
376
+ return `${appleAudio ? "Apple" : "Other"}${standalone ? " PWA" : ""}`;
377
+ }
378
+
379
+ document.addEventListener("visibilitychange", () => {
380
+ if (document.visibilityState !== "visible" || call.state !== "live") return;
381
+ void keepAwake();
382
+ void recoverMic();
383
+ });
384
+
265
385
  /** 服务端转来的通话事件(ws 的 t: "rt") */
266
386
  export function onRealtimeEvent(msg) {
267
387
  if (call.state === "idle" || msg.key !== call.key) return;
@@ -431,7 +551,9 @@ function tick() {
431
551
  if (!call.els) return;
432
552
  const mic = call.muted ? 0 : (call.micLevel?.() ?? 0);
433
553
  const out = call.outLevel?.() ?? 0;
434
- const speaking = out > 0.02;
554
+ // 字和字之间的停顿也会掉到阈值以下:响过之后 0.4 秒内都算在说,状态别来回闪
555
+ if (out > 0.02) call.loudAt = performance.now();
556
+ const speaking = out > 0.02 || performance.now() - call.loudAt < 400;
435
557
  const level = Math.min(1, (speaking ? out : mic) * 6);
436
558
  call.els.orb.style.setProperty("--voice-scale", (1 + level * 0.14).toFixed(3));
437
559
  call.panel.classList.toggle("speaking", speaking);