agent-loft 0.2.115 → 0.2.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agents/codex/session.ts +4 -0
- package/src/core/hub.ts +22 -4
- package/src/core/realtime-relay.ts +25 -0
- package/src/core/types.ts +2 -0
- package/web/lib/rt-worklet.js +42 -5
package/package.json
CHANGED
package/src/core/hub.ts
CHANGED
|
@@ -846,7 +846,10 @@ class Hub extends EventEmitter<{
|
|
|
846
846
|
this.externalTurn(key, text, anchor);
|
|
847
847
|
},
|
|
848
848
|
// 不看 alive:会话被换下去时发的「通话结束」也要让页面知道
|
|
849
|
-
onRealtime: (ev) =>
|
|
849
|
+
onRealtime: (ev) => {
|
|
850
|
+
this.emit("realtime", { key, ev });
|
|
851
|
+
if (ev.type === "closed") this.realtimeEnded(key);
|
|
852
|
+
},
|
|
850
853
|
onStep: (kind, text) => alive() && this.emitEvent(key, { type: "step", kind, text }),
|
|
851
854
|
onTodos: (todos) => alive() && this.emitEvent(key, { type: "todos", todos }),
|
|
852
855
|
onTextDelta: (text) => alive() && this.emitEvent(key, { type: "delta", text }),
|
|
@@ -1847,7 +1850,21 @@ class Hub extends EventEmitter<{
|
|
|
1847
1850
|
|| (this.pendingOps.get(key) ?? 0) > 0
|
|
1848
1851
|
|| this.isolatedRuns.has(key)
|
|
1849
1852
|
|| this.rewinding.has(key)
|
|
1850
|
-
|| interactions.listPending(key).length > 0
|
|
1853
|
+
|| interactions.listPending(key).length > 0
|
|
1854
|
+
|| this.inCall(key);
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
/** 正在实时语音通话:通话本身不走轮次,不算进 isBusy,但进程不能闲置回收、也不能因为记忆更新被换掉 */
|
|
1858
|
+
private inCall(key: string): boolean {
|
|
1859
|
+
return this.live.get(key)?.session.inRealtime?.() === true;
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
/** 通话结束:从这时起重新算闲置;通话期间攒下的「记忆已更新」这时再换进程 */
|
|
1863
|
+
private realtimeEnded(key: string): void {
|
|
1864
|
+
const live = this.live.get(key);
|
|
1865
|
+
if (!live) return;
|
|
1866
|
+
this.liveUsedAt.set(live.id, Date.now());
|
|
1867
|
+
this.dropIfStale(key);
|
|
1851
1868
|
}
|
|
1852
1869
|
|
|
1853
1870
|
/** 关掉空闲太久的会话进程(只关进程,不动绑定),见 LIVE_IDLE_MS */
|
|
@@ -1877,13 +1894,13 @@ class Hub extends EventEmitter<{
|
|
|
1877
1894
|
|
|
1878
1895
|
private dropIfStale(key: string): void {
|
|
1879
1896
|
const live = this.live.get(key);
|
|
1880
|
-
if (!live || !this.staleLive.has(live.id) || this.isBusy(key)) return;
|
|
1897
|
+
if (!live || !this.staleLive.has(live.id) || this.isBusy(key) || this.inCall(key)) return;
|
|
1881
1898
|
this.staleLive.delete(live.id);
|
|
1882
1899
|
void this.serial(key, async () => {
|
|
1883
1900
|
// 排队期间可能已经换了进程、或者又开始说话了:那就留到下一轮结束
|
|
1884
1901
|
const now = this.live.get(key);
|
|
1885
1902
|
if (now?.id !== live.id) return;
|
|
1886
|
-
if (this.isBusy(key)) {
|
|
1903
|
+
if (this.isBusy(key) || this.inCall(key)) {
|
|
1887
1904
|
this.staleLive.add(live.id);
|
|
1888
1905
|
return;
|
|
1889
1906
|
}
|
|
@@ -2071,6 +2088,7 @@ class Hub extends EventEmitter<{
|
|
|
2071
2088
|
|
|
2072
2089
|
async realtimeStop(key: string): Promise<void> {
|
|
2073
2090
|
await this.live.get(key)?.session.stopRealtime?.();
|
|
2091
|
+
this.realtimeEnded(key);
|
|
2074
2092
|
}
|
|
2075
2093
|
|
|
2076
2094
|
/**
|
|
@@ -15,6 +15,11 @@ const FRAME_BYTES = FRAME * 2;
|
|
|
15
15
|
const RTP_STEP = 960;
|
|
16
16
|
/** 浏览器发过来的还没凑够一帧的、积压的上限(约 1 秒),再多说明来不及发,丢掉旧的 */
|
|
17
17
|
const MAX_PENDING = RELAY_RATE * 2;
|
|
18
|
+
/**
|
|
19
|
+
* 收的方向丢了几个包以内用 Opus 的丢包补偿填上(解码空包 = PLC,一次给 120ms,按丢的帧数截)。
|
|
20
|
+
* 服务器到 OpenAI 这段常过代理,实测丢 2~3%:不填的话浏览器那头的缓冲每丢一包就少 20ms,过一阵就放空、一卡一卡
|
|
21
|
+
*/
|
|
22
|
+
const MAX_CONCEAL = 6;
|
|
18
23
|
|
|
19
24
|
export class RealtimeRelay {
|
|
20
25
|
/** 收到 OpenAI 的声音(解码后的 PCM),交给浏览器 */
|
|
@@ -30,6 +35,8 @@ export class RealtimeRelay {
|
|
|
30
35
|
private seq = Math.floor(Math.random() * 0xffff);
|
|
31
36
|
private ts = Math.floor(Math.random() * 0xffffffff);
|
|
32
37
|
private closed = false;
|
|
38
|
+
/** 收到的上一个 RTP 包的序号 */
|
|
39
|
+
private lastSeq: number | undefined;
|
|
33
40
|
private dropTimer: NodeJS.Timeout | undefined;
|
|
34
41
|
|
|
35
42
|
private constructor() {
|
|
@@ -40,6 +47,14 @@ export class RealtimeRelay {
|
|
|
40
47
|
transceiver.onTrack.subscribe((remote) => {
|
|
41
48
|
remote.onReceiveRtp.subscribe((rtp) => {
|
|
42
49
|
if (this.closed || !rtp.payload.length) return;
|
|
50
|
+
const seq = rtp.header.sequenceNumber;
|
|
51
|
+
if (this.lastSeq !== undefined) {
|
|
52
|
+
const gap = (seq - this.lastSeq) & 0xffff;
|
|
53
|
+
// 重复的、迟到的(已经补过或跳过了):丢掉,别打乱顺序
|
|
54
|
+
if (gap === 0 || gap > 0x8000) return;
|
|
55
|
+
if (gap > 1 && gap - 1 <= MAX_CONCEAL) this.conceal(gap - 1);
|
|
56
|
+
}
|
|
57
|
+
this.lastSeq = seq;
|
|
43
58
|
try {
|
|
44
59
|
this.onAudio(Buffer.from(this.decoder.decode(rtp.payload)));
|
|
45
60
|
} catch {
|
|
@@ -121,6 +136,16 @@ export class RealtimeRelay {
|
|
|
121
136
|
}
|
|
122
137
|
}
|
|
123
138
|
|
|
139
|
+
/** 丢了 frames 个 20ms 的包:用解码器的丢包补偿顶上同样长的声音 */
|
|
140
|
+
private conceal(frames: number): void {
|
|
141
|
+
try {
|
|
142
|
+
const pcm = this.decoder.decode(Buffer.alloc(0));
|
|
143
|
+
this.onAudio(Buffer.from(pcm.subarray(0, Math.min(pcm.length, frames * FRAME_BYTES))));
|
|
144
|
+
} catch {
|
|
145
|
+
// 补不了就算了
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
124
149
|
private fail(reason: string): void {
|
|
125
150
|
if (this.closed) return;
|
|
126
151
|
this.close();
|
package/src/core/types.ts
CHANGED
|
@@ -321,6 +321,8 @@ export interface AgentSession {
|
|
|
321
321
|
startRealtime?(offer: string, opts?: RealtimeOptions): Promise<string>;
|
|
322
322
|
/** 挂断实时语音对话;没在通话就什么都不做 */
|
|
323
323
|
stopRealtime?(): Promise<void>;
|
|
324
|
+
/** 正在实时语音通话(含建连中):这期间进程不能当闲置回收 */
|
|
325
|
+
inRealtime?(): boolean;
|
|
324
326
|
}
|
|
325
327
|
|
|
326
328
|
export type Diagnosis = {
|
package/web/lib/rt-worklet.js
CHANGED
|
@@ -1,9 +1,24 @@
|
|
|
1
1
|
// 语音对话的音频处理(AudioWorklet,跑在音频线程里)。声音经服务器中转:两头都是 24kHz、16 位、单声道的 PCM。
|
|
2
2
|
// rt-capture:麦克风 → 重采样到 24kHz → 每 20ms 一块 Int16 发给主线程(主线程走 WebSocket 发给服务器)
|
|
3
3
|
// rt-play:服务器发来的 Int16 24kHz → 攒一点缓冲 → 重采样到 AudioContext 的采样率放出来
|
|
4
|
+
// 放空之后很快(JB_JITTER_SEC 内)又来了声音 = 说到一半网络抖了:把缓冲加长一点;隔得久的是正常停顿,不算。
|
|
5
|
+
// 一阵子没再放空就慢慢缩回去;攒得比目标多太多(卡了一阵又一下子涌来)就跳回目标,免得越放越晚
|
|
4
6
|
|
|
5
7
|
const RATE = 24000;
|
|
6
8
|
const CHUNK = 480; // 20ms
|
|
9
|
+
/** 播放缓冲(24kHz 采样数):起步、最短、最长、每次放空加多少 */
|
|
10
|
+
const JB_START = RATE * 0.1;
|
|
11
|
+
const JB_MIN = RATE * 0.06;
|
|
12
|
+
const JB_MAX = RATE * 0.4;
|
|
13
|
+
const JB_STEP = RATE * 0.06;
|
|
14
|
+
/** 这么久(秒)没放空,缓冲缩短 20ms */
|
|
15
|
+
const JB_DECAY_SEC = 8;
|
|
16
|
+
/** 放空后这么久(秒)内又来了声音,算网络抖动 */
|
|
17
|
+
const JB_JITTER_SEC = 0.3;
|
|
18
|
+
/** 比目标多出这么多就跳回目标 */
|
|
19
|
+
const JB_SLACK = RATE * 0.3;
|
|
20
|
+
/** 没攒够目标、但这么久(秒)没再来声音:先放掉(很短的一句别卡在缓冲里) */
|
|
21
|
+
const JB_FLUSH_SEC = 0.1;
|
|
7
22
|
|
|
8
23
|
class Capture extends AudioWorkletProcessor {
|
|
9
24
|
constructor() {
|
|
@@ -52,13 +67,27 @@ class Play extends AudioWorkletProcessor {
|
|
|
52
67
|
this.avail = 0; // 还没放的采样数
|
|
53
68
|
this.write = 0;
|
|
54
69
|
this.playing = false;
|
|
70
|
+
this.target = JB_START;
|
|
71
|
+
/** 上次放空 / 缩短缓冲以来放了多少个输出采样 */
|
|
72
|
+
this.smooth = 0;
|
|
73
|
+
/** 说着说着放空的时刻(currentFrame),等下一块声音来了判断是抖动还是停顿 */
|
|
74
|
+
this.starvedAt = -1;
|
|
75
|
+
/** 最近一次收到声音的时刻(currentFrame) */
|
|
76
|
+
this.lastData = 0;
|
|
55
77
|
this.port.onmessage = (e) => {
|
|
56
78
|
if (e.data === "flush") {
|
|
57
79
|
this.avail = 0;
|
|
58
80
|
this.playing = false;
|
|
81
|
+
this.target = JB_START;
|
|
82
|
+
this.starvedAt = -1;
|
|
59
83
|
return;
|
|
60
84
|
}
|
|
61
85
|
const pcm = new Int16Array(e.data);
|
|
86
|
+
this.lastData = currentFrame;
|
|
87
|
+
if (this.starvedAt >= 0) {
|
|
88
|
+
if (currentFrame - this.starvedAt < sampleRate * JB_JITTER_SEC) this.target = Math.min(JB_MAX, this.target + JB_STEP);
|
|
89
|
+
this.starvedAt = -1;
|
|
90
|
+
}
|
|
62
91
|
// 放不下(网络卡了一阵又一下子涌来):丢掉最旧的,宁可跳一下也不越放越晚
|
|
63
92
|
const over = this.avail + pcm.length - this.size;
|
|
64
93
|
if (over > 0) {
|
|
@@ -70,9 +99,9 @@ class Play extends AudioWorkletProcessor {
|
|
|
70
99
|
this.write = (this.write + 1) % this.size;
|
|
71
100
|
}
|
|
72
101
|
this.avail += pcm.length;
|
|
73
|
-
//
|
|
74
|
-
if (this.avail >
|
|
75
|
-
const skip = this.avail -
|
|
102
|
+
// 太多了说明延迟越积越大,追回到目标
|
|
103
|
+
if (this.avail > this.target + JB_SLACK) {
|
|
104
|
+
const skip = this.avail - this.target;
|
|
76
105
|
this.read = (this.read + skip) % this.size;
|
|
77
106
|
this.avail -= skip;
|
|
78
107
|
}
|
|
@@ -83,10 +112,14 @@ class Play extends AudioWorkletProcessor {
|
|
|
83
112
|
const out = outputs[0];
|
|
84
113
|
const ch = out[0];
|
|
85
114
|
if (!ch) return true;
|
|
86
|
-
//
|
|
87
|
-
if (!this.playing && this.avail >=
|
|
115
|
+
// 先攒够目标再开始放,扛网络抖动;放空了就停下来重新攒,下次多攒一点
|
|
116
|
+
if (!this.playing && this.avail >= 2 && (this.avail >= this.target || currentFrame - this.lastData > sampleRate * JB_FLUSH_SEC)) {
|
|
117
|
+
this.playing = true;
|
|
118
|
+
this.smooth = 0;
|
|
119
|
+
}
|
|
88
120
|
for (let i = 0; i < ch.length; i++) {
|
|
89
121
|
if (!this.playing || this.avail < 2) {
|
|
122
|
+
if (this.playing) this.starvedAt = currentFrame + i;
|
|
90
123
|
ch[i] = 0;
|
|
91
124
|
this.playing = false;
|
|
92
125
|
continue;
|
|
@@ -101,6 +134,10 @@ class Play extends AudioWorkletProcessor {
|
|
|
101
134
|
this.avail -= used;
|
|
102
135
|
this.read = next % this.size;
|
|
103
136
|
}
|
|
137
|
+
if (this.playing && (this.smooth += ch.length) > sampleRate * JB_DECAY_SEC) {
|
|
138
|
+
this.smooth = 0;
|
|
139
|
+
this.target = Math.max(JB_MIN, this.target - RATE * 0.02);
|
|
140
|
+
}
|
|
104
141
|
for (let c = 1; c < out.length; c++) out[c].set(ch);
|
|
105
142
|
return true;
|
|
106
143
|
}
|