@kuralle-syrinx/browser-client 4.4.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-state.d.ts +35 -0
- package/dist/agent-state.d.ts.map +1 -0
- package/dist/agent-state.js +76 -0
- package/dist/agent-state.js.map +1 -0
- package/dist/audio.d.ts +61 -0
- package/dist/audio.d.ts.map +1 -0
- package/dist/audio.js +276 -0
- package/dist/audio.js.map +1 -0
- package/dist/browser-opus.d.ts +41 -0
- package/dist/browser-opus.d.ts.map +1 -0
- package/dist/browser-opus.js +81 -0
- package/dist/browser-opus.js.map +1 -0
- package/dist/index.d.ts +270 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +744 -0
- package/dist/index.js.map +1 -0
- package/dist/session-metrics.d.ts +28 -0
- package/dist/session-metrics.d.ts.map +1 -0
- package/dist/session-metrics.js +60 -0
- package/dist/session-metrics.js.map +1 -0
- package/dist/session-record.d.ts +112 -0
- package/dist/session-record.d.ts.map +1 -0
- package/dist/session-record.js +181 -0
- package/dist/session-record.js.map +1 -0
- package/dist/transport.d.ts +16 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/transport.js +3 -0
- package/dist/transport.js.map +1 -0
- package/dist/turn-recorder.d.ts +56 -0
- package/dist/turn-recorder.d.ts.map +1 -0
- package/dist/turn-recorder.js +168 -0
- package/dist/turn-recorder.js.map +1 -0
- package/dist/turn-timeline.d.ts +51 -0
- package/dist/turn-timeline.d.ts.map +1 -0
- package/dist/turn-timeline.js +92 -0
- package/dist/turn-timeline.js.map +1 -0
- package/dist/websocket-transport.d.ts +20 -0
- package/dist/websocket-transport.d.ts.map +1 -0
- package/dist/websocket-transport.js +92 -0
- package/dist/websocket-transport.js.map +1 -0
- package/package.json +35 -11
- package/src/agent-state.test.ts +143 -0
- package/src/audio.test.ts +130 -0
- package/src/index.test.ts +842 -0
- package/src/jitter-buffer.test.ts +275 -0
- package/src/session-metrics.test.ts +102 -0
- package/src/session-record.test.ts +219 -0
- package/src/studio-page.test.ts +69 -0
- package/src/transport.test.ts +233 -0
- package/src/turn-recorder.test.ts +138 -0
- package/src/turn-timeline.test.ts +151 -0
- package/index.html +0 -1073
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
// WT-03: Client jitter buffer tests
|
|
3
|
+
|
|
4
|
+
import { beforeEach, describe, expect, it, vi } from "vitest";
|
|
5
|
+
import { AudioJitterBuffer } from "./audio.js";
|
|
6
|
+
|
|
7
|
+
// Mock AudioContext for testing
|
|
8
|
+
class MockAudioContext {
|
|
9
|
+
public currentTime = 0;
|
|
10
|
+
public destination = {};
|
|
11
|
+
private nextBufferId = 1;
|
|
12
|
+
|
|
13
|
+
createBuffer(channels: number, length: number, sampleRate: number): MockAudioBuffer {
|
|
14
|
+
return new MockAudioBuffer(channels, length, sampleRate, this.nextBufferId++);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
createBufferSource(): MockAudioBufferSource {
|
|
18
|
+
return new MockAudioBufferSource();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
class MockAudioBuffer {
|
|
23
|
+
constructor(
|
|
24
|
+
public numberOfChannels: number,
|
|
25
|
+
public length: number,
|
|
26
|
+
public sampleRate: number,
|
|
27
|
+
public _id: number
|
|
28
|
+
) {}
|
|
29
|
+
|
|
30
|
+
get duration(): number {
|
|
31
|
+
return this.length / this.sampleRate;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
copyToChannel(_data: Float32Array, _channel: number): void {
|
|
35
|
+
// Mock implementation
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
class MockAudioBufferSource {
|
|
40
|
+
public buffer: MockAudioBuffer | null = null;
|
|
41
|
+
public onended: (() => void) | null = null;
|
|
42
|
+
private connected = false;
|
|
43
|
+
private started = false;
|
|
44
|
+
private startTime = 0;
|
|
45
|
+
|
|
46
|
+
connect(_destination: unknown): void {
|
|
47
|
+
this.connected = true;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
start(when = 0): void {
|
|
51
|
+
this.started = true;
|
|
52
|
+
this.startTime = when;
|
|
53
|
+
|
|
54
|
+
// Simulate completion after buffer duration
|
|
55
|
+
if (this.buffer && this.onended) {
|
|
56
|
+
setTimeout(() => {
|
|
57
|
+
this.onended?.();
|
|
58
|
+
}, this.buffer.duration * 1000);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
stop(): void {
|
|
63
|
+
this.started = false;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe("AudioJitterBuffer", () => {
|
|
68
|
+
let mockContext: MockAudioContext;
|
|
69
|
+
let jitterBuffer: AudioJitterBuffer;
|
|
70
|
+
|
|
71
|
+
beforeEach(() => {
|
|
72
|
+
vi.useFakeTimers();
|
|
73
|
+
mockContext = new MockAudioContext();
|
|
74
|
+
jitterBuffer = new AudioJitterBuffer(mockContext as any, {
|
|
75
|
+
sampleRateHz: 16000,
|
|
76
|
+
targetBufferMs: 100
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it("enqueues and schedules audio frames with target buffer delay", () => {
|
|
81
|
+
const pcm16Data = new Int16Array(320); // 20ms at 16kHz
|
|
82
|
+
pcm16Data.fill(1000);
|
|
83
|
+
|
|
84
|
+
mockContext.currentTime = 1.0;
|
|
85
|
+
jitterBuffer.enqueue(pcm16Data.buffer, "test-context");
|
|
86
|
+
|
|
87
|
+
// Should schedule audio ~100ms + frame duration in the future
|
|
88
|
+
expect(jitterBuffer.bufferedDurationMs).toBeCloseTo(120, -1);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it("schedules multiple frames contiguously", () => {
|
|
92
|
+
const frameData = new Int16Array(320); // 20ms at 16kHz
|
|
93
|
+
frameData.fill(1000);
|
|
94
|
+
|
|
95
|
+
mockContext.currentTime = 1.0;
|
|
96
|
+
|
|
97
|
+
// Enqueue 3 frames
|
|
98
|
+
jitterBuffer.enqueue(frameData.buffer, "test-context");
|
|
99
|
+
jitterBuffer.enqueue(frameData.buffer, "test-context");
|
|
100
|
+
jitterBuffer.enqueue(frameData.buffer, "test-context");
|
|
101
|
+
|
|
102
|
+
// Should buffer ~160ms total (100ms initial + 3*20ms)
|
|
103
|
+
expect(jitterBuffer.bufferedDurationMs).toBeCloseTo(160, -1);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("tracks active context IDs", () => {
|
|
107
|
+
const frameData = new Int16Array(320);
|
|
108
|
+
frameData.fill(1000);
|
|
109
|
+
|
|
110
|
+
jitterBuffer.enqueue(frameData.buffer, "context-1");
|
|
111
|
+
jitterBuffer.enqueue(frameData.buffer, "context-2");
|
|
112
|
+
|
|
113
|
+
expect(jitterBuffer.activeContextIds).toContain("context-1");
|
|
114
|
+
expect(jitterBuffer.activeContextIds).toContain("context-2");
|
|
115
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(2);
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it("reports played-out ms clamped to the scheduled window (heard clock)", () => {
|
|
119
|
+
const frameData = new Int16Array(320); // 20ms at 16kHz
|
|
120
|
+
frameData.fill(1000);
|
|
121
|
+
mockContext.currentTime = 1.0;
|
|
122
|
+
jitterBuffer.enqueue(frameData.buffer, "ctx"); // scheduled at 1.1, ends 1.12
|
|
123
|
+
|
|
124
|
+
// Before the frame starts playing → nothing heard yet.
|
|
125
|
+
expect(jitterBuffer.playedOutMs("ctx")).toBe(0);
|
|
126
|
+
expect(jitterBuffer.isPlayoutComplete("ctx")).toBe(false);
|
|
127
|
+
|
|
128
|
+
// Halfway through the 20ms frame → ~10ms heard.
|
|
129
|
+
mockContext.currentTime = 1.11;
|
|
130
|
+
expect(jitterBuffer.playedOutMs("ctx")).toBeCloseTo(10, 0);
|
|
131
|
+
|
|
132
|
+
// Past the end → clamped to the full 20ms, and complete.
|
|
133
|
+
mockContext.currentTime = 1.2;
|
|
134
|
+
expect(jitterBuffer.playedOutMs("ctx")).toBeCloseTo(20, 0);
|
|
135
|
+
expect(jitterBuffer.isPlayoutComplete("ctx")).toBe(true);
|
|
136
|
+
|
|
137
|
+
// Unknown context reports 0.
|
|
138
|
+
expect(jitterBuffer.playedOutMs("nope")).toBe(0);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("clears specific context frames on context-specific clear", () => {
|
|
142
|
+
const frameData = new Int16Array(320);
|
|
143
|
+
frameData.fill(1000);
|
|
144
|
+
|
|
145
|
+
jitterBuffer.enqueue(frameData.buffer, "context-1");
|
|
146
|
+
jitterBuffer.enqueue(frameData.buffer, "context-2");
|
|
147
|
+
jitterBuffer.enqueue(frameData.buffer, "context-1");
|
|
148
|
+
|
|
149
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(2);
|
|
150
|
+
|
|
151
|
+
jitterBuffer.clear("context-1");
|
|
152
|
+
|
|
153
|
+
expect(jitterBuffer.activeContextIds).toContain("context-2");
|
|
154
|
+
expect(jitterBuffer.activeContextIds).not.toContain("context-1");
|
|
155
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(1);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it("resets schedule baseline after context-specific clear", () => {
|
|
159
|
+
const frameData = new Int16Array(320);
|
|
160
|
+
frameData.fill(1000);
|
|
161
|
+
|
|
162
|
+
mockContext.currentTime = 1.0;
|
|
163
|
+
for (let i = 0; i < 50; i += 1) {
|
|
164
|
+
jitterBuffer.enqueue(frameData.buffer, "assistant");
|
|
165
|
+
}
|
|
166
|
+
expect(jitterBuffer.bufferedDurationMs).toBeGreaterThan(1000);
|
|
167
|
+
|
|
168
|
+
jitterBuffer.clear("assistant");
|
|
169
|
+
expect(jitterBuffer.bufferedDurationMs).toBe(0);
|
|
170
|
+
|
|
171
|
+
mockContext.currentTime = 1.05;
|
|
172
|
+
jitterBuffer.enqueue(frameData.buffer, "user-barge");
|
|
173
|
+
expect(jitterBuffer.bufferedDurationMs).toBeCloseTo(120, -1);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("zeros buffered duration when the cleared context was the only one scheduled", () => {
|
|
177
|
+
const frameData = new Int16Array(320);
|
|
178
|
+
frameData.fill(1000);
|
|
179
|
+
|
|
180
|
+
mockContext.currentTime = 2.0;
|
|
181
|
+
jitterBuffer.enqueue(frameData.buffer, "solo-context");
|
|
182
|
+
jitterBuffer.enqueue(frameData.buffer, "solo-context");
|
|
183
|
+
expect(jitterBuffer.bufferedDurationMs).toBeGreaterThan(0);
|
|
184
|
+
|
|
185
|
+
jitterBuffer.clear("solo-context");
|
|
186
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(0);
|
|
187
|
+
expect(jitterBuffer.bufferedDurationMs).toBe(0);
|
|
188
|
+
|
|
189
|
+
mockContext.currentTime = 8.0;
|
|
190
|
+
jitterBuffer.enqueue(frameData.buffer, "next-context");
|
|
191
|
+
expect(jitterBuffer.bufferedDurationMs).toBeCloseTo(120, -1);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it("clears all frames on global clear", () => {
|
|
195
|
+
const frameData = new Int16Array(320);
|
|
196
|
+
frameData.fill(1000);
|
|
197
|
+
|
|
198
|
+
jitterBuffer.enqueue(frameData.buffer, "context-1");
|
|
199
|
+
jitterBuffer.enqueue(frameData.buffer, "context-2");
|
|
200
|
+
|
|
201
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(2);
|
|
202
|
+
|
|
203
|
+
jitterBuffer.clear();
|
|
204
|
+
|
|
205
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(0);
|
|
206
|
+
expect(jitterBuffer.bufferedDurationMs).toBe(0);
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it("establishes new baseline when buffer is empty", () => {
|
|
210
|
+
const frameData = new Int16Array(320);
|
|
211
|
+
frameData.fill(1000);
|
|
212
|
+
|
|
213
|
+
mockContext.currentTime = 1.0;
|
|
214
|
+
jitterBuffer.enqueue(frameData.buffer, "context-1");
|
|
215
|
+
|
|
216
|
+
const initialBuffer = jitterBuffer.bufferedDurationMs;
|
|
217
|
+
|
|
218
|
+
// Clear and advance time
|
|
219
|
+
jitterBuffer.clear();
|
|
220
|
+
mockContext.currentTime = 2.0;
|
|
221
|
+
|
|
222
|
+
jitterBuffer.enqueue(frameData.buffer, "context-2");
|
|
223
|
+
|
|
224
|
+
// Should establish new baseline at current time + target buffer
|
|
225
|
+
expect(jitterBuffer.bufferedDurationMs).toBeCloseTo(initialBuffer, -1);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it("handles PCM16 to Float32 conversion correctly", () => {
|
|
229
|
+
// Test with known values
|
|
230
|
+
const pcm16Data = new Int16Array([0, 16383, -16384, 32767, -32768]);
|
|
231
|
+
const expectedFloat32 = [0, 0.5, -0.5, 1.0, -1.0];
|
|
232
|
+
|
|
233
|
+
let capturedFloat32: Float32Array | null = null;
|
|
234
|
+
|
|
235
|
+
// Mock copyToChannel to capture the converted data
|
|
236
|
+
MockAudioBuffer.prototype.copyToChannel = function(data: Float32Array, _channel: number) {
|
|
237
|
+
capturedFloat32 = new Float32Array(data);
|
|
238
|
+
};
|
|
239
|
+
|
|
240
|
+
jitterBuffer.enqueue(pcm16Data.buffer);
|
|
241
|
+
|
|
242
|
+
expect(capturedFloat32).not.toBeNull();
|
|
243
|
+
expect(capturedFloat32![0]).toBeCloseTo(expectedFloat32[0]!, 5);
|
|
244
|
+
expect(capturedFloat32![1]).toBeCloseTo(expectedFloat32[1]!, 3);
|
|
245
|
+
expect(capturedFloat32![2]).toBeCloseTo(expectedFloat32[2]!, 3);
|
|
246
|
+
expect(capturedFloat32![3]).toBeCloseTo(expectedFloat32[3]!, 3);
|
|
247
|
+
expect(capturedFloat32![4]).toBeCloseTo(expectedFloat32[4]!, 3);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
it("handles empty or invalid audio data gracefully", () => {
|
|
251
|
+
const emptyData = new ArrayBuffer(0);
|
|
252
|
+
|
|
253
|
+
expect(() => {
|
|
254
|
+
jitterBuffer.enqueue(emptyData, "test-context");
|
|
255
|
+
}).not.toThrow();
|
|
256
|
+
|
|
257
|
+
expect(jitterBuffer.activeContextIds).toHaveLength(0);
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
it("advances scheduled time correctly for consecutive frames", () => {
|
|
261
|
+
const frame1 = new Int16Array(160).fill(1000); // 10ms at 16kHz
|
|
262
|
+
const frame2 = new Int16Array(320).fill(1000); // 20ms at 16kHz
|
|
263
|
+
|
|
264
|
+
mockContext.currentTime = 0;
|
|
265
|
+
|
|
266
|
+
jitterBuffer.enqueue(frame1.buffer, "context-1");
|
|
267
|
+
const buffer1 = jitterBuffer.bufferedDurationMs;
|
|
268
|
+
|
|
269
|
+
jitterBuffer.enqueue(frame2.buffer, "context-1");
|
|
270
|
+
const buffer2 = jitterBuffer.bufferedDurationMs;
|
|
271
|
+
|
|
272
|
+
// Second frame should add its duration to the buffer
|
|
273
|
+
expect(buffer2 - buffer1).toBeCloseTo(20, -1);
|
|
274
|
+
});
|
|
275
|
+
});
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
|
|
5
|
+
import type { TurnRecord } from "./session-record.js";
|
|
6
|
+
import { buildSessionMetrics, percentile } from "./session-metrics.js";
|
|
7
|
+
|
|
8
|
+
const turn = (turnId: string, timings?: TurnRecord["timings"]): TurnRecord => ({
|
|
9
|
+
turnId,
|
|
10
|
+
startedAtMs: 0,
|
|
11
|
+
events: [],
|
|
12
|
+
agentText: "",
|
|
13
|
+
toolCalls: [],
|
|
14
|
+
ttsAudioBytes: 0,
|
|
15
|
+
errors: [],
|
|
16
|
+
complete: true,
|
|
17
|
+
droppedEvents: 0,
|
|
18
|
+
...(timings ? { timings } : {}),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
describe("percentile — nearest-rank", () => {
|
|
22
|
+
it("returns a value that actually occurred", () => {
|
|
23
|
+
const xs = [10, 20, 30, 40, 50];
|
|
24
|
+
// Every result is a real measurement, traceable to a turn — not an interpolation.
|
|
25
|
+
expect(xs).toContain(percentile(xs, 50));
|
|
26
|
+
expect(xs).toContain(percentile(xs, 95));
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("handles the single-sample case", () => {
|
|
30
|
+
expect(percentile([42], 50)).toBe(42);
|
|
31
|
+
expect(percentile([42], 95)).toBe(42);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("returns 0 for an empty set rather than NaN", () => {
|
|
35
|
+
expect(percentile([], 50)).toBe(0);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it("p95 lands on the top of the distribution", () => {
|
|
39
|
+
const xs = Array.from({ length: 100 }, (_, i) => i + 1);
|
|
40
|
+
expect(percentile(xs, 95)).toBe(95);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe("session metrics — aggregates", () => {
|
|
45
|
+
it("computes per-stage stats only for stages that have data", () => {
|
|
46
|
+
const m = buildSessionMetrics([
|
|
47
|
+
turn("t1", { sttMs: 100, llmTTFTMs: 900, e2eMs: 1500 }),
|
|
48
|
+
turn("t2", { sttMs: 200, llmTTFTMs: 1100, e2eMs: 1900 }),
|
|
49
|
+
]);
|
|
50
|
+
expect(m.stages.map((s) => s.stage)).toEqual(["stt", "llm", "e2e"]); // no tts data → no tts row
|
|
51
|
+
// Nearest-rank, not interpolated: p50 of [100, 200] is 100, a turn that
|
|
52
|
+
// actually happened — not 150, which no turn ever measured. Deliberate; do
|
|
53
|
+
// not "correct" this to an average.
|
|
54
|
+
expect(m.stages.find((s) => s.stage === "stt")?.medianMs).toBe(100);
|
|
55
|
+
expect(m.stages.find((s) => s.stage === "e2e")?.maxMs).toBe(1900);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("counts measured turns separately from total turns", () => {
|
|
59
|
+
// The Workers path emits no metrics, so a session can have turns and no measurements.
|
|
60
|
+
const m = buildSessionMetrics([turn("t1", { e2eMs: 1200 }), turn("t2"), turn("t3")]);
|
|
61
|
+
expect(m.turnCount).toBe(3);
|
|
62
|
+
expect(m.measuredTurnCount).toBe(1);
|
|
63
|
+
expect(m.unavailable).toBe(false);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("reports unavailable when nothing carried timings", () => {
|
|
67
|
+
const m = buildSessionMetrics([turn("t1"), turn("t2")]);
|
|
68
|
+
expect(m.unavailable).toBe(true);
|
|
69
|
+
expect(m.stages).toHaveLength(0);
|
|
70
|
+
expect(m.turnCount).toBe(2); // still says how many turns happened
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("handles an empty session", () => {
|
|
74
|
+
const m = buildSessionMetrics([]);
|
|
75
|
+
expect(m.turnCount).toBe(0);
|
|
76
|
+
expect(m.unavailable).toBe(true);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
describe("session metrics — the floor, not just the ceiling", () => {
|
|
81
|
+
it("flags turns that replied implausibly fast", () => {
|
|
82
|
+
const m = buildSessionMetrics([
|
|
83
|
+
turn("slow", { e2eMs: 2000 }),
|
|
84
|
+
turn("premature", { e2eMs: 420 }),
|
|
85
|
+
turn("normal", { e2eMs: 1100 }),
|
|
86
|
+
]);
|
|
87
|
+
// 420ms is not a win — the endpointer almost certainly cut the caller off.
|
|
88
|
+
expect(m.suspiciouslyFastTurnIds).toEqual(["premature"]);
|
|
89
|
+
expect(m.floorMs).toBe(700);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("does not flag a zero or missing e2e as fast", () => {
|
|
93
|
+
const m = buildSessionMetrics([turn("a", { e2eMs: 0 }), turn("b", { sttMs: 50 })]);
|
|
94
|
+
expect(m.suspiciouslyFastTurnIds).toEqual([]);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("respects a caller-supplied floor", () => {
|
|
98
|
+
// A triage agent talking to distressed callers wants a much higher floor.
|
|
99
|
+
const m = buildSessionMetrics([turn("t1", { e2eMs: 1500 })], 2000);
|
|
100
|
+
expect(m.suspiciouslyFastTurnIds).toEqual(["t1"]);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
applyMessage,
|
|
7
|
+
buildSessionRecord,
|
|
8
|
+
DEFAULT_LIMITS,
|
|
9
|
+
emptySessionRecord,
|
|
10
|
+
type SessionRecord,
|
|
11
|
+
type ToolCall,
|
|
12
|
+
type TurnRecord,
|
|
13
|
+
} from "./session-record.js";
|
|
14
|
+
|
|
15
|
+
const at = (message: unknown, atMs: number) => ({ message: message as never, atMs });
|
|
16
|
+
|
|
17
|
+
/** Index into turns/toolCalls with a loud failure instead of a non-null assertion. */
|
|
18
|
+
function turnAt(r: SessionRecord, i: number): TurnRecord {
|
|
19
|
+
const t = r.turns[i];
|
|
20
|
+
if (!t) throw new Error(`expected a turn at index ${i}, got ${r.turns.length} turn(s)`);
|
|
21
|
+
return t;
|
|
22
|
+
}
|
|
23
|
+
function toolAt(t: TurnRecord, i: number): ToolCall {
|
|
24
|
+
const c = t.toolCalls[i];
|
|
25
|
+
if (!c) throw new Error(`expected a tool call at index ${i}, got ${t.toolCalls.length}`);
|
|
26
|
+
return c;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe("session record — config", () => {
|
|
30
|
+
it("captures negotiated params from ready", () => {
|
|
31
|
+
const r = buildSessionRecord([
|
|
32
|
+
at({ type: "ready", sessionId: "s1", resumeWindowMs: 30_000, audio: { inputSampleRateHz: 16000, outputSampleRateHz: 24000, encoding: "pcm_s16le", channels: 1, binaryEnvelope: "syrinx.audio.v1" } }, 0),
|
|
33
|
+
]);
|
|
34
|
+
expect(r.config.sessionId).toBe("s1");
|
|
35
|
+
expect(r.config.inputSampleRateHz).toBe(16000);
|
|
36
|
+
expect(r.config.outputSampleRateHz).toBe(24000);
|
|
37
|
+
expect(r.config.encoding).toBe("pcm_s16le");
|
|
38
|
+
expect(r.config.binaryEnvelope).toBe("syrinx.audio.v1");
|
|
39
|
+
expect(r.config.resumeWindowMs).toBe(30_000);
|
|
40
|
+
expect(r.turns).toHaveLength(0);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("keeps a caller-supplied wsUrl", () => {
|
|
44
|
+
const r = applyMessage(emptySessionRecord({ wsUrl: "ws://x/ws" }), { type: "ready" } as never, 0);
|
|
45
|
+
expect(r.config.wsUrl).toBe("ws://x/ws");
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
describe("session record — unknown message types", () => {
|
|
50
|
+
// Proven on the Cloudflare path: the agents SDK sends these before `ready`.
|
|
51
|
+
it("tolerates cf_agent_* arriving before ready without throwing or creating turns", () => {
|
|
52
|
+
const r = buildSessionRecord([
|
|
53
|
+
at({ type: "cf_agent_identity", id: "abc" }, 0),
|
|
54
|
+
at({ type: "cf_agent_mcp_servers", servers: [] }, 1),
|
|
55
|
+
at({ type: "ready", audio: { inputSampleRateHz: 16000 } }, 2),
|
|
56
|
+
]);
|
|
57
|
+
expect(r.turns).toHaveLength(0);
|
|
58
|
+
expect(r.sessionEvents).toHaveLength(3);
|
|
59
|
+
expect(r.config.inputSampleRateHz).toBe(16000);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("retains an unknown type that carries a turnId without corrupting the turn", () => {
|
|
63
|
+
const r = buildSessionRecord([
|
|
64
|
+
at({ type: "agent_chunk", turnId: "t1", text: "hi" }, 0),
|
|
65
|
+
at({ type: "some_future_message", turnId: "t1", whatever: true }, 1),
|
|
66
|
+
]);
|
|
67
|
+
expect(turnAt(r, 0).agentText).toBe("hi");
|
|
68
|
+
expect(turnAt(r, 0).events).toHaveLength(2);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
describe("session record — turn assembly", () => {
|
|
73
|
+
it("folds a complete turn across every mutating message type", () => {
|
|
74
|
+
const r = buildSessionRecord([
|
|
75
|
+
at({ type: "speech_started", turnId: "t1" }, 0),
|
|
76
|
+
at({ type: "speech_ended", turnId: "t1" }, 100),
|
|
77
|
+
at({ type: "stt_chunk", turnId: "t1", transcript: "what is the" }, 120),
|
|
78
|
+
at({ type: "stt_output", turnId: "t1", transcript: "what is the deadline", confidence: 0.94 }, 200),
|
|
79
|
+
at({ type: "agent_tool_call", turnId: "t1", id: "c1", name: "lookup", args: { q: 1 } }, 210),
|
|
80
|
+
at({ type: "tool_call_started", turnId: "t1", toolId: "c1", toolName: "lookup" }, 211),
|
|
81
|
+
at({ type: "tool_call_delayed", turnId: "t1", toolId: "c1", toolName: "lookup", afterMs: 800 }, 1000),
|
|
82
|
+
at({ type: "agent_tool_result", turnId: "t1", id: "c1", result: "March 1" }, 1100),
|
|
83
|
+
at({ type: "tool_call_complete", turnId: "t1", toolId: "c1", toolName: "lookup" }, 1101),
|
|
84
|
+
at({ type: "agent_chunk", turnId: "t1", text: "The deadline " }, 1200),
|
|
85
|
+
at({ type: "agent_chunk", turnId: "t1", text: "is March 1." }, 1250),
|
|
86
|
+
at({ type: "tts_chunk", turnId: "t1", byteLength: 4000 }, 1300),
|
|
87
|
+
at({ type: "tts_chunk", turnId: "t1", byteLength: 6000 }, 1350),
|
|
88
|
+
at({ type: "agent_end", turnId: "t1" }, 1400),
|
|
89
|
+
at({ type: "tts_end", turnId: "t1" }, 1500),
|
|
90
|
+
at({ type: "turn_complete", turnId: "t1", transcript: "what is the deadline" }, 1550),
|
|
91
|
+
at({ type: "metrics", turnId: "t1", speechEndMs: 100, llmTTFTMs: 900, ttsTTFBMs: 100, e2eMs: 1450 }, 1600),
|
|
92
|
+
]);
|
|
93
|
+
|
|
94
|
+
const t = turnAt(r, 0);
|
|
95
|
+
expect(r.turns).toHaveLength(1);
|
|
96
|
+
expect(t.userTranscript).toBe("what is the deadline");
|
|
97
|
+
expect(t.userInterim).toBeUndefined(); // interim replaced in place by the final
|
|
98
|
+
expect(t.userConfidence).toBe(0.94);
|
|
99
|
+
expect(t.agentText).toBe("The deadline is March 1."); // chunks concatenated in order
|
|
100
|
+
expect(t.ttsAudioBytes).toBe(10_000);
|
|
101
|
+
expect(t.complete).toBe(true);
|
|
102
|
+
expect(t.timings?.llmTTFTMs).toBe(900);
|
|
103
|
+
expect(t.timings).not.toHaveProperty("turnId"); // envelope fields stripped
|
|
104
|
+
expect(t.toolCalls).toHaveLength(1); // one call, not four — merged by id
|
|
105
|
+
expect(toolAt(t, 0)).toMatchObject({ name: "lookup", result: "March 1", phase: "complete", afterMs: 800 });
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("records an interruption with its reason", () => {
|
|
109
|
+
const r = buildSessionRecord([
|
|
110
|
+
at({ type: "agent_chunk", turnId: "t1", text: "The dead" }, 0),
|
|
111
|
+
at({ type: "agent_interrupted", turnId: "t1", reason: "user_speech" }, 1400),
|
|
112
|
+
]);
|
|
113
|
+
expect(turnAt(r, 0).interrupted).toEqual({ atMs: 1400, reason: "user_speech" });
|
|
114
|
+
expect(turnAt(r, 0).complete).toBe(false);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it("keeps a turn that never completes", () => {
|
|
118
|
+
const r = buildSessionRecord([at({ type: "speech_started", turnId: "t1" }, 0)]);
|
|
119
|
+
expect(turnAt(r, 0).complete).toBe(false);
|
|
120
|
+
expect(turnAt(r, 0).timings).toBeUndefined(); // Workers path emits no metrics
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("collects errors without ending the turn", () => {
|
|
124
|
+
const r = buildSessionRecord([
|
|
125
|
+
at({ type: "agent_chunk", turnId: "t1", text: "a" }, 0),
|
|
126
|
+
at({ type: "error", turnId: "t1", component: "llm", category: "recoverable", message: "rate limited" }, 10),
|
|
127
|
+
at({ type: "agent_chunk", turnId: "t1", text: "b" }, 20),
|
|
128
|
+
]);
|
|
129
|
+
expect(turnAt(r, 0).errors).toEqual([{ atMs: 10, component: "llm", category: "recoverable", message: "rate limited" }]);
|
|
130
|
+
expect(turnAt(r, 0).agentText).toBe("ab");
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it("separates turns by turnId and tolerates interleaving", () => {
|
|
134
|
+
const r = buildSessionRecord([
|
|
135
|
+
at({ type: "agent_chunk", turnId: "t1", text: "one" }, 0),
|
|
136
|
+
at({ type: "agent_chunk", turnId: "t2", text: "two" }, 1),
|
|
137
|
+
at({ type: "agent_chunk", turnId: "t1", text: "!" }, 2),
|
|
138
|
+
]);
|
|
139
|
+
expect(r.turns.map((t) => t.agentText)).toEqual(["one!", "two"]);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it("creates a turn from whichever message arrives first, out of order", () => {
|
|
143
|
+
const r = buildSessionRecord([
|
|
144
|
+
at({ type: "turn_complete", turnId: "t1", transcript: "late" }, 0),
|
|
145
|
+
at({ type: "speech_started", turnId: "t1" }, 1),
|
|
146
|
+
]);
|
|
147
|
+
expect(r.turns).toHaveLength(1);
|
|
148
|
+
expect(turnAt(r, 0).userTranscript).toBe("late");
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
describe("session record — bounded by construction", () => {
|
|
153
|
+
it("evicts oldest turns past the cap and counts the loss", () => {
|
|
154
|
+
const limits = { maxTurns: 3, maxEventsPerTurn: 100 };
|
|
155
|
+
const msgs = Array.from({ length: 6 }, (_, i) => at({ type: "agent_chunk", turnId: `t${i}`, text: "x" }, i));
|
|
156
|
+
const r = buildSessionRecord(msgs, {}, limits);
|
|
157
|
+
expect(r.turns).toHaveLength(3);
|
|
158
|
+
expect(r.turns.map((t) => t.turnId)).toEqual(["t3", "t4", "t5"]); // newest kept
|
|
159
|
+
expect(r.droppedTurns).toBe(3); // surfaced, not silent
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it("caps events per turn and counts the loss", () => {
|
|
163
|
+
const limits = { maxEventsPerTurn: 5, maxTurns: 10 };
|
|
164
|
+
const msgs = Array.from({ length: 12 }, (_, i) => at({ type: "tts_chunk", turnId: "t1", byteLength: 1 }, i));
|
|
165
|
+
const r = buildSessionRecord(msgs, {}, limits);
|
|
166
|
+
expect(turnAt(r, 0).events).toHaveLength(5);
|
|
167
|
+
expect(turnAt(r, 0).droppedEvents).toBe(7);
|
|
168
|
+
expect(turnAt(r, 0).ttsAudioBytes).toBe(12); // derived state survives event eviction
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it("has sane defaults", () => {
|
|
172
|
+
expect(DEFAULT_LIMITS.maxTurns).toBe(50);
|
|
173
|
+
expect(DEFAULT_LIMITS.maxEventsPerTurn).toBe(500);
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
describe("session record — purity", () => {
|
|
178
|
+
it("does not mutate the input record", () => {
|
|
179
|
+
const before: SessionRecord = emptySessionRecord();
|
|
180
|
+
const after = applyMessage(before, { type: "agent_chunk", turnId: "t1", text: "x" } as never, 0);
|
|
181
|
+
expect(before.turns).toHaveLength(0);
|
|
182
|
+
expect(after.turns).toHaveLength(1);
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("is deterministic — same input, same output", () => {
|
|
186
|
+
const msgs = [at({ type: "agent_chunk", turnId: "t1", text: "a" }, 0), at({ type: "turn_complete", turnId: "t1" }, 1)];
|
|
187
|
+
expect(buildSessionRecord(msgs)).toEqual(buildSessionRecord(msgs));
|
|
188
|
+
});
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
describe("session record — endpointing decision", () => {
|
|
192
|
+
it("folds the owner/reason from metrics onto the turn, not into timings", () => {
|
|
193
|
+
const r = buildSessionRecord([
|
|
194
|
+
at({ type: "metrics", turnId: "t1", speechEndMs: 100, e2eMs: 500, endpointingOwner: "smart_turn", endpointingReason: "end_of_speech" }, 0),
|
|
195
|
+
]);
|
|
196
|
+
const t = turnAt(r, 0);
|
|
197
|
+
expect(t.endpointingOwner).toBe("smart_turn");
|
|
198
|
+
expect(t.endpointingReason).toBe("end_of_speech");
|
|
199
|
+
// The decision is a fact about the turn, not a timing — it must not leak into timings.
|
|
200
|
+
expect(t.timings).not.toHaveProperty("endpointingOwner");
|
|
201
|
+
expect(t.timings).not.toHaveProperty("endpointingReason");
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it("omits the decision when metrics does not carry it (absent means absent)", () => {
|
|
205
|
+
const r = buildSessionRecord([
|
|
206
|
+
at({ type: "metrics", turnId: "t1", speechEndMs: 100, e2eMs: 500 }, 0),
|
|
207
|
+
]);
|
|
208
|
+
const t = turnAt(r, 0);
|
|
209
|
+
expect(t.endpointingOwner).toBeUndefined();
|
|
210
|
+
expect(t.endpointingReason).toBeUndefined();
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
it("carries a force-finalized reason onto the turn", () => {
|
|
214
|
+
const r = buildSessionRecord([
|
|
215
|
+
at({ type: "metrics", turnId: "t1", speechEndMs: 100, e2eMs: 500, endpointingOwner: "provider_stt", endpointingReason: "force_finalized" }, 0),
|
|
216
|
+
]);
|
|
217
|
+
expect(turnAt(r, 0).endpointingReason).toBe("force_finalized");
|
|
218
|
+
});
|
|
219
|
+
});
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
// Guards the standalone studio page (packages/voice-client-browser/index.html), which is
|
|
4
|
+
// served raw by the dev server as a fallback when apps/studio has not been built. Its assistant-audio decoder is PCM16-only (no Opus
|
|
5
|
+
// decoder), so it must declare a pcm_s16le downlink capability on connect — otherwise the
|
|
6
|
+
// server streams Opus envelopes it rejects with "PCM16 payload must contain an even number
|
|
7
|
+
// of bytes" / "durationMs mismatch". Regression guard for that contract.
|
|
8
|
+
|
|
9
|
+
import { readFileSync } from "node:fs";
|
|
10
|
+
import { fileURLToPath } from "node:url";
|
|
11
|
+
import { describe, expect, it } from "vitest";
|
|
12
|
+
|
|
13
|
+
const html = readFileSync(fileURLToPath(new URL("../index.html", import.meta.url)), "utf8");
|
|
14
|
+
|
|
15
|
+
describe("studio index.html downlink codec negotiation", () => {
|
|
16
|
+
it("declares pcm_s16le downlink capability on connect", () => {
|
|
17
|
+
// The page has no Opus decoder, so it must negotiate PCM downlink.
|
|
18
|
+
expect(html).not.toMatch(/OpusDecoder|decodeOpus/);
|
|
19
|
+
const open = html.slice(html.indexOf('addEventListener("open"'), html.indexOf('addEventListener("close"'));
|
|
20
|
+
expect(open).toMatch(/"codec_capability"/);
|
|
21
|
+
expect(open).toMatch(/downlinkEncoding:\s*"pcm_s16le"/);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it("rejects a non-pcm_s16le downlink encoding loudly instead of failing as cryptic PCM", () => {
|
|
25
|
+
// If the server ever sends a non-PCM envelope (codec negotiation failure), the decoder
|
|
26
|
+
// must branch on metadata.encoding and surface a clear error — not the misleading
|
|
27
|
+
// "PCM16 payload must contain an even number of bytes" the byte invariants would throw.
|
|
28
|
+
const decode = html.slice(html.indexOf("function decodeAssistantAudio"), html.indexOf("function hasPrefix"));
|
|
29
|
+
expect(decode).toMatch(/metadata\.encoding\s*&&\s*metadata\.encoding\s*!==\s*"pcm_s16le"/);
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe("studio index.html capture turn lifecycle", () => {
|
|
34
|
+
it("keeps the capture context open across VAD speech_ended until the server commits the turn", () => {
|
|
35
|
+
const speechEnded = html.slice(
|
|
36
|
+
html.indexOf('message.type === "speech_ended"'),
|
|
37
|
+
html.indexOf('message.type === "audio_clear"'),
|
|
38
|
+
);
|
|
39
|
+
expect(speechEnded).toMatch(/drainPcmQueue\(turn\.id\)/);
|
|
40
|
+
expect(speechEnded).not.toMatch(/activeTurn\s*=\s*null/);
|
|
41
|
+
|
|
42
|
+
const turnComplete = html.slice(
|
|
43
|
+
html.indexOf('message.type === "turn_complete"'),
|
|
44
|
+
html.indexOf('message.type === "agent_tool_call"'),
|
|
45
|
+
);
|
|
46
|
+
expect(turnComplete).toMatch(/activeTurn\?\.id\s*===\s*turn\.id/);
|
|
47
|
+
expect(turnComplete).toMatch(/activeTurn\s*=\s*null/);
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe("studio index.html local speech-start barge-in", () => {
|
|
52
|
+
it("flushes local assistant playout before sending a server client_interrupt", () => {
|
|
53
|
+
const handler = html.slice(
|
|
54
|
+
html.indexOf("function handleLocalSpeechStart"),
|
|
55
|
+
html.indexOf("function createCaptureTurn"),
|
|
56
|
+
);
|
|
57
|
+
expect(handler).toMatch(/flushOutputAudio\(\)[\s\S]*sendClientInterrupt/);
|
|
58
|
+
expect(handler).toMatch(/local_vad_speech_start/);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("does not use browser speech_ended or silence as an EOS signal", () => {
|
|
62
|
+
const capture = html.slice(
|
|
63
|
+
html.indexOf("processorNode.onaudioprocess"),
|
|
64
|
+
html.indexOf("sourceNode.connect"),
|
|
65
|
+
);
|
|
66
|
+
expect(capture).toMatch(/handleLocalSpeechStart/);
|
|
67
|
+
expect(capture).not.toMatch(/eos\.turn_complete|turn_complete|client_eos|speech_ended/);
|
|
68
|
+
});
|
|
69
|
+
});
|