agent-loft 0.2.115 → 0.2.116

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-loft",
3
- "version": "0.2.115",
3
+ "version": "0.2.116",
4
4
  "type": "module",
5
5
  "description": "多个 agent 的工作台:把本机 Claude Code / Codex 放到一处统一管理,用飞书和 Web 页面随时驱动——会话、审批、模型切换都共用",
6
6
  "keywords": [
@@ -867,6 +867,10 @@ export class CodexSession implements AgentSession {
867
867
  }
868
868
  }
869
869
 
870
+ inRealtime(): boolean {
871
+ return !!this.rt;
872
+ }
873
+
870
874
  async stopRealtime(): Promise<void> {
871
875
  if (!this.rt) return;
872
876
  this.rt.fail?.(new Error("已挂断"));
package/src/core/hub.ts CHANGED
@@ -846,7 +846,10 @@ class Hub extends EventEmitter<{
846
846
  this.externalTurn(key, text, anchor);
847
847
  },
848
848
  // 不看 alive:会话被换下去时发的「通话结束」也要让页面知道
849
- onRealtime: (ev) => this.emit("realtime", { key, ev }),
849
+ onRealtime: (ev) => {
850
+ this.emit("realtime", { key, ev });
851
+ if (ev.type === "closed") this.realtimeEnded(key);
852
+ },
850
853
  onStep: (kind, text) => alive() && this.emitEvent(key, { type: "step", kind, text }),
851
854
  onTodos: (todos) => alive() && this.emitEvent(key, { type: "todos", todos }),
852
855
  onTextDelta: (text) => alive() && this.emitEvent(key, { type: "delta", text }),
@@ -1847,7 +1850,21 @@ class Hub extends EventEmitter<{
1847
1850
  || (this.pendingOps.get(key) ?? 0) > 0
1848
1851
  || this.isolatedRuns.has(key)
1849
1852
  || this.rewinding.has(key)
1850
- || interactions.listPending(key).length > 0;
1853
+ || interactions.listPending(key).length > 0
1854
+ || this.inCall(key);
1855
+ }
1856
+
1857
+ /** 正在实时语音通话:通话本身不走轮次,不算进 isBusy,但进程不能闲置回收、也不能因为记忆更新被换掉 */
1858
+ private inCall(key: string): boolean {
1859
+ return this.live.get(key)?.session.inRealtime?.() === true;
1860
+ }
1861
+
1862
+ /** 通话结束:从这时起重新算闲置;通话期间攒下的「记忆已更新」这时再换进程 */
1863
+ private realtimeEnded(key: string): void {
1864
+ const live = this.live.get(key);
1865
+ if (!live) return;
1866
+ this.liveUsedAt.set(live.id, Date.now());
1867
+ this.dropIfStale(key);
1851
1868
  }
1852
1869
 
1853
1870
  /** 关掉空闲太久的会话进程(只关进程,不动绑定),见 LIVE_IDLE_MS */
@@ -1877,13 +1894,13 @@ class Hub extends EventEmitter<{
1877
1894
 
1878
1895
  private dropIfStale(key: string): void {
1879
1896
  const live = this.live.get(key);
1880
- if (!live || !this.staleLive.has(live.id) || this.isBusy(key)) return;
1897
+ if (!live || !this.staleLive.has(live.id) || this.isBusy(key) || this.inCall(key)) return;
1881
1898
  this.staleLive.delete(live.id);
1882
1899
  void this.serial(key, async () => {
1883
1900
  // 排队期间可能已经换了进程、或者又开始说话了:那就留到下一轮结束
1884
1901
  const now = this.live.get(key);
1885
1902
  if (now?.id !== live.id) return;
1886
- if (this.isBusy(key)) {
1903
+ if (this.isBusy(key) || this.inCall(key)) {
1887
1904
  this.staleLive.add(live.id);
1888
1905
  return;
1889
1906
  }
@@ -2071,6 +2088,7 @@ class Hub extends EventEmitter<{
2071
2088
 
2072
2089
  async realtimeStop(key: string): Promise<void> {
2073
2090
  await this.live.get(key)?.session.stopRealtime?.();
2091
+ this.realtimeEnded(key);
2074
2092
  }
2075
2093
 
2076
2094
  /**
@@ -15,6 +15,11 @@ const FRAME_BYTES = FRAME * 2;
15
15
  const RTP_STEP = 960;
16
16
  /** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
17
17
  const MAX_PENDING = RELAY_RATE * 2;
18
+ /**
19
+ * 收的方向丢了几个包以内用 Opus 的丢包补偿填上(解码空包 = PLC,一次给 120ms,按丢的帧数截)。
20
+ * 服务器到 OpenAI 这段常过代理,实测丢 2~3%:不填的话浏览器那头的缓冲每丢一包就少 20ms,过一阵就放空、一卡一卡
21
+ */
22
+ const MAX_CONCEAL = 6;
18
23
 
19
24
  export class RealtimeRelay {
20
25
  /** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
@@ -30,6 +35,8 @@ export class RealtimeRelay {
30
35
  private seq = Math.floor(Math.random() * 0xffff);
31
36
  private ts = Math.floor(Math.random() * 0xffffffff);
32
37
  private closed = false;
38
+ /** 收到的上一个 RTP 包的序号 */
39
+ private lastSeq: number | undefined;
33
40
  private dropTimer: NodeJS.Timeout | undefined;
34
41
 
35
42
  private constructor() {
@@ -40,6 +47,14 @@ export class RealtimeRelay {
40
47
  transceiver.onTrack.subscribe((remote) => {
41
48
  remote.onReceiveRtp.subscribe((rtp) => {
42
49
  if (this.closed || !rtp.payload.length) return;
50
+ const seq = rtp.header.sequenceNumber;
51
+ if (this.lastSeq !== undefined) {
52
+ const gap = (seq - this.lastSeq) & 0xffff;
53
+ // 重复的、迟到的(已经补过或跳过了):丢掉,别打乱顺序
54
+ if (gap === 0 || gap > 0x8000) return;
55
+ if (gap > 1 && gap - 1 <= MAX_CONCEAL) this.conceal(gap - 1);
56
+ }
57
+ this.lastSeq = seq;
43
58
  try {
44
59
  this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
45
60
  } catch {
@@ -121,6 +136,16 @@ export class RealtimeRelay {
121
136
  }
122
137
  }
123
138
 
139
+ /** 丢了 frames 个 20ms 的包:用解码器的丢包补偿顶上同样长的声音 */
140
+ private conceal(frames: number): void {
141
+ try {
142
+ const pcm = this.decoder.decode(Buffer.alloc(0));
143
+ this.onAudio(Buffer.from(pcm.subarray(0, Math.min(pcm.length, frames * FRAME_BYTES))));
144
+ } catch {
145
+ // 补不了就算了
146
+ }
147
+ }
148
+
124
149
  private fail(reason: string): void {
125
150
  if (this.closed) return;
126
151
  this.close();
package/src/core/types.ts CHANGED
@@ -321,6 +321,8 @@ export interface AgentSession {
321
321
  startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
322
322
  /** 挂断实时语音对话;没在通话就什么都不做 */
323
323
  stopRealtime?(): Promise<void>;
324
+ /** 正在实时语音通话(含建连中):这期间进程不能当闲置回收 */
325
+ inRealtime?(): boolean;
324
326
  }
325
327
 
326
328
  export type Diagnosis = {
@@ -1,9 +1,24 @@
1
1
  // 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
2
2
  // rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
3
3
  // rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
4
+ // 放空之后很快(JB_JITTER_SEC 内)又来了声音 = 说到一半网络抖了:把缓冲加长一点;隔得久的是正常停顿,不算。
5
+ // 一阵子没再放空就慢慢缩回去;攒得比目标多太多(卡了一阵又一下子涌来)就跳回目标,免得越放越晚
4
6
 
5
7
  const RATE = 24000;
6
8
  const CHUNK = 480; // 20ms
9
+ /** 播放缓冲(24kHz 采样数):起步、最短、最长、每次放空加多少 */
10
+ const JB_START = RATE * 0.1;
11
+ const JB_MIN = RATE * 0.06;
12
+ const JB_MAX = RATE * 0.4;
13
+ const JB_STEP = RATE * 0.06;
14
+ /** 这么久(秒)没放空,缓冲缩短 20ms */
15
+ const JB_DECAY_SEC = 8;
16
+ /** 放空后这么久(秒)内又来了声音,算网络抖动 */
17
+ const JB_JITTER_SEC = 0.3;
18
+ /** 比目标多出这么多就跳回目标 */
19
+ const JB_SLACK = RATE * 0.3;
20
+ /** 没攒够目标、但这么久(秒)没再来声音:先放掉(很短的一句别卡在缓冲里) */
21
+ const JB_FLUSH_SEC = 0.1;
7
22
 
8
23
  class Capture extends AudioWorkletProcessor {
9
24
  constructor() {
@@ -52,13 +67,27 @@ class Play extends AudioWorkletProcessor {
52
67
  this.avail = 0; // 还没放的采样数
53
68
  this.write = 0;
54
69
  this.playing = false;
70
+ this.target = JB_START;
71
+ /** 上次放空 / 缩短缓冲以来放了多少个输出采样 */
72
+ this.smooth = 0;
73
+ /** 说着说着放空的时刻(currentFrame),等下一块声音来了判断是抖动还是停顿 */
74
+ this.starvedAt = -1;
75
+ /** 最近一次收到声音的时刻(currentFrame) */
76
+ this.lastData = 0;
55
77
  this.port.onmessage = (e) => {
56
78
  if (e.data === "flush") {
57
79
  this.avail = 0;
58
80
  this.playing = false;
81
+ this.target = JB_START;
82
+ this.starvedAt = -1;
59
83
  return;
60
84
  }
61
85
  const pcm = new Int16Array(e.data);
86
+ this.lastData = currentFrame;
87
+ if (this.starvedAt >= 0) {
88
+ if (currentFrame - this.starvedAt < sampleRate * JB_JITTER_SEC) this.target = Math.min(JB_MAX, this.target + JB_STEP);
89
+ this.starvedAt = -1;
90
+ }
62
91
  // 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
63
92
  const over = this.avail + pcm.length - this.size;
64
93
  if (over > 0) {
@@ -70,9 +99,9 @@ class Play extends AudioWorkletProcessor {
70
99
  this.write = (this.write + 1) % this.size;
71
100
  }
72
101
  this.avail += pcm.length;
73
- // 太多了(超过 1 秒)说明延迟越积越大,追到只剩 0.2 秒
74
- if (this.avail > RATE) {
75
- const skip = this.avail - RATE / 5;
102
+ // 太多了说明延迟越积越大,追回到目标
103
+ if (this.avail > this.target + JB_SLACK) {
104
+ const skip = this.avail - this.target;
76
105
  this.read = (this.read + skip) % this.size;
77
106
  this.avail -= skip;
78
107
  }
@@ -83,10 +112,14 @@ class Play extends AudioWorkletProcessor {
83
112
  const out = outputs[0];
84
113
  const ch = out[0];
85
114
  if (!ch) return true;
86
- // 先攒 60ms 再开始放,扛一点网络抖动;放空了就停下来重新攒
87
- if (!this.playing && this.avail >= RATE * 0.06) this.playing = true;
115
+ // 先攒够目标再开始放,扛网络抖动;放空了就停下来重新攒,下次多攒一点
116
+ if (!this.playing && this.avail >= 2 && (this.avail >= this.target || currentFrame - this.lastData > sampleRate * JB_FLUSH_SEC)) {
117
+ this.playing = true;
118
+ this.smooth = 0;
119
+ }
88
120
  for (let i = 0; i < ch.length; i++) {
89
121
  if (!this.playing || this.avail < 2) {
122
+ if (this.playing) this.starvedAt = currentFrame + i;
90
123
  ch[i] = 0;
91
124
  this.playing = false;
92
125
  continue;
@@ -101,6 +134,10 @@ class Play extends AudioWorkletProcessor {
101
134
  this.avail -= used;
102
135
  this.read = next % this.size;
103
136
  }
137
+ if (this.playing && (this.smooth += ch.length) > sampleRate * JB_DECAY_SEC) {
138
+ this.smooth = 0;
139
+ this.target = Math.max(JB_MIN, this.target - RATE * 0.02);
140
+ }
104
141
  for (let c = 1; c < out.length; c++) out[c].set(ch);
105
142
  return true;
106
143
  }