@kuralle-syrinx/browser-client 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -0
- package/package.json +12 -4
- package/src/agent-state.ts +107 -0
- package/src/index.ts +37 -1
- package/src/session-metrics.ts +90 -0
- package/src/session-record.ts +307 -0
- package/src/turn-recorder.ts +207 -0
- package/src/turn-timeline.ts +148 -0
- package/src/audio.test.ts +0 -130
- package/src/index.test.ts +0 -818
- package/src/jitter-buffer.test.ts +0 -275
- package/src/studio-page.test.ts +0 -69
- package/src/transport.test.ts +0 -233
- package/tsconfig.json +0 -21
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// SessionRecord — a structured, bounded view of what happened in a voice session,
|
|
4
|
+
// assembled from the messages the server already sends. Every debugging surface
|
|
5
|
+
// (turn timeline, event log, metrics, fixture capture) reads this one shape, and a
|
|
6
|
+
// fixture is a SessionRecord trimmed to one turn.
|
|
7
|
+
//
|
|
8
|
+
// Deliberately dependency-free and DOM-free: this module is exported from the
|
|
9
|
+
// `/record` subpath so a Node consumer can import it without pulling AudioContext
|
|
10
|
+
// or the opus codec. Assembly is a pure reduction over messages — no sockets, no
|
|
11
|
+
// timers, no React — which is what makes every consumer testable against recorded
|
|
12
|
+
// fixtures instead of a live provider.
|
|
13
|
+
|
|
14
|
+
import type { SyrinxStudioMessage } from "./index.js";
|
|
15
|
+
|
|
16
|
+
/** Per-turn latency decomposition, from the `metrics` message. */
|
|
17
|
+
export interface TurnTimings {
|
|
18
|
+
readonly speechEndMs?: number;
|
|
19
|
+
readonly textReadyMs?: number;
|
|
20
|
+
readonly firstAudioByteMs?: number;
|
|
21
|
+
readonly firstAudioPlayedMs?: number;
|
|
22
|
+
readonly lastAudioPlayedMs?: number;
|
|
23
|
+
readonly sttMs?: number;
|
|
24
|
+
readonly llmTTFTMs?: number;
|
|
25
|
+
readonly ttsTTFBMs?: number;
|
|
26
|
+
readonly e2eMs?: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Negotiated session parameters. Recorded so a replayed fixture cannot silently mislead. */
|
|
30
|
+
export interface SessionConfig {
|
|
31
|
+
readonly wsUrl?: string;
|
|
32
|
+
readonly sessionId?: string;
|
|
33
|
+
readonly inputSampleRateHz?: number;
|
|
34
|
+
readonly outputSampleRateHz?: number;
|
|
35
|
+
readonly encoding?: "pcm_s16le" | "opus";
|
|
36
|
+
readonly binaryEnvelope?: string;
|
|
37
|
+
readonly rawBinaryInput?: boolean;
|
|
38
|
+
/**
|
|
39
|
+
* Frame duration the host paces to. Node states it; the Workers/DO host does not
|
|
40
|
+
* (measured live, 2026-07-25) — which is exactly the kind of divergence the session
|
|
41
|
+
* info panel exists to make visible, so it is captured rather than dropped.
|
|
42
|
+
*/
|
|
43
|
+
readonly targetFrameDurationMs?: number;
|
|
44
|
+
readonly resumeWindowMs?: number;
|
|
45
|
+
/** Only present once the server exposes it; the timeline omits the marker rather than guessing. */
|
|
46
|
+
readonly endpointingOwner?: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface ToolCall {
|
|
50
|
+
readonly id?: string;
|
|
51
|
+
readonly name?: string;
|
|
52
|
+
readonly args?: unknown;
|
|
53
|
+
readonly result?: unknown;
|
|
54
|
+
/** `delayed` means it outlived the cue threshold — a slow tool, not a failed one. */
|
|
55
|
+
readonly phase?: "started" | "delayed" | "complete" | "failed";
|
|
56
|
+
readonly afterMs?: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface RecordedEvent {
|
|
60
|
+
/** ms since the record started — comparable across turns. */
|
|
61
|
+
readonly atMs: number;
|
|
62
|
+
readonly message: SyrinxStudioMessage | { readonly type: string; readonly [k: string]: unknown };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface TurnRecord {
|
|
66
|
+
readonly turnId: string;
|
|
67
|
+
readonly startedAtMs: number;
|
|
68
|
+
readonly events: readonly RecordedEvent[];
|
|
69
|
+
/** Absent on runtimes that do not emit `metrics` — the Workers path does not. */
|
|
70
|
+
readonly timings?: TurnTimings;
|
|
71
|
+
readonly userTranscript?: string;
|
|
72
|
+
/** Latest interim, cleared once the final arrives. */
|
|
73
|
+
readonly userInterim?: string;
|
|
74
|
+
readonly userConfidence?: number;
|
|
75
|
+
readonly agentText: string;
|
|
76
|
+
readonly toolCalls: readonly ToolCall[];
|
|
77
|
+
readonly ttsAudioBytes: number;
|
|
78
|
+
readonly interrupted?: { readonly atMs: number; readonly reason?: string };
|
|
79
|
+
readonly errors: readonly { readonly atMs: number; readonly component?: string; readonly category?: string; readonly message: string }[];
|
|
80
|
+
readonly complete: boolean;
|
|
81
|
+
/** Non-zero when this turn's event list hit the cap. Surfaced, never silent. */
|
|
82
|
+
readonly droppedEvents: number;
|
|
83
|
+
/** Which owner decided the turn ended. Omitted when the backend did not say. */
|
|
84
|
+
readonly endpointingOwner?: string;
|
|
85
|
+
/** Why the turn ended. Omitted when the backend did not say. */
|
|
86
|
+
readonly endpointingReason?: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export interface SessionRecord {
|
|
90
|
+
readonly config: SessionConfig;
|
|
91
|
+
readonly turns: readonly TurnRecord[];
|
|
92
|
+
/** Non-zero when oldest turns were evicted. Surfaced, never silent. */
|
|
93
|
+
readonly droppedTurns: number;
|
|
94
|
+
/** Messages that arrived before any turn existed, or with no turnId. */
|
|
95
|
+
readonly sessionEvents: readonly RecordedEvent[];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export interface SessionRecordLimits {
|
|
99
|
+
readonly maxTurns: number;
|
|
100
|
+
readonly maxEventsPerTurn: number;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export const DEFAULT_LIMITS: SessionRecordLimits = { maxTurns: 50, maxEventsPerTurn: 500 };
|
|
104
|
+
|
|
105
|
+
export function emptySessionRecord(config: SessionConfig = {}): SessionRecord {
|
|
106
|
+
return { config, turns: [], droppedTurns: 0, sessionEvents: [] };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// A message we do not recognise — the Cloudflare agents SDK sends `cf_agent_identity`
|
|
110
|
+
// and `cf_agent_mcp_servers` before `ready`, and future SDK versions will add more.
|
|
111
|
+
// These are retained in the event stream but must never affect turn assembly.
|
|
112
|
+
type AnyMessage = SyrinxStudioMessage | { readonly type: string; readonly [k: string]: unknown };
|
|
113
|
+
|
|
114
|
+
const turnIdOf = (m: AnyMessage): string | undefined =>
|
|
115
|
+
typeof (m as { turnId?: unknown }).turnId === "string" ? (m as { turnId: string }).turnId : undefined;
|
|
116
|
+
|
|
117
|
+
function newTurn(turnId: string, atMs: number): TurnRecord {
|
|
118
|
+
return {
|
|
119
|
+
turnId,
|
|
120
|
+
startedAtMs: atMs,
|
|
121
|
+
events: [],
|
|
122
|
+
agentText: "",
|
|
123
|
+
toolCalls: [],
|
|
124
|
+
ttsAudioBytes: 0,
|
|
125
|
+
errors: [],
|
|
126
|
+
complete: false,
|
|
127
|
+
droppedEvents: 0,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function pushEvent(turn: TurnRecord, ev: RecordedEvent, limits: SessionRecordLimits): TurnRecord {
|
|
132
|
+
if (turn.events.length < limits.maxEventsPerTurn) {
|
|
133
|
+
return { ...turn, events: [...turn.events, ev] };
|
|
134
|
+
}
|
|
135
|
+
// Drop the oldest, keep the newest, and count what was lost.
|
|
136
|
+
return { ...turn, events: [...turn.events.slice(1), ev], droppedEvents: turn.droppedEvents + 1 };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function upsertToolCall(calls: readonly ToolCall[], key: string | undefined, patch: Partial<ToolCall>): readonly ToolCall[] {
|
|
140
|
+
// A later phase message carries fewer fields than an earlier one — `tool_call_complete`
|
|
141
|
+
// has no `afterMs` and may have no `toolName`. Spreading those as explicit `undefined`
|
|
142
|
+
// would erase what `tool_call_delayed`/`agent_tool_call` already established, so drop
|
|
143
|
+
// undefined keys before merging.
|
|
144
|
+
const defined = Object.fromEntries(Object.entries(patch).filter(([, v]) => v !== undefined)) as Partial<ToolCall>;
|
|
145
|
+
const i = key === undefined ? -1 : calls.findIndex((c) => c.id === key);
|
|
146
|
+
if (i === -1) return [...calls, { ...defined, id: key }];
|
|
147
|
+
const next = [...calls];
|
|
148
|
+
next[i] = { ...next[i], ...defined };
|
|
149
|
+
return next;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Fold one message into the record. Pure: same inputs, same output, no side effects.
|
|
154
|
+
*
|
|
155
|
+
* `atMs` is the caller's clock (ms since the record started) rather than anything
|
|
156
|
+
* read from the message, so the record stays deterministic under test.
|
|
157
|
+
*/
|
|
158
|
+
export function applyMessage(
|
|
159
|
+
record: SessionRecord,
|
|
160
|
+
message: AnyMessage,
|
|
161
|
+
atMs: number,
|
|
162
|
+
limits: SessionRecordLimits = DEFAULT_LIMITS,
|
|
163
|
+
): SessionRecord {
|
|
164
|
+
const ev: RecordedEvent = { atMs, message };
|
|
165
|
+
const type = typeof message?.type === "string" ? message.type : "";
|
|
166
|
+
|
|
167
|
+
// `ready` carries the negotiated config and belongs to the session, not a turn.
|
|
168
|
+
if (type === "ready") {
|
|
169
|
+
const m = message as Extract<SyrinxStudioMessage, { type: "ready" }>;
|
|
170
|
+
return {
|
|
171
|
+
...record,
|
|
172
|
+
config: {
|
|
173
|
+
...record.config,
|
|
174
|
+
sessionId: m.sessionId ?? record.config.sessionId,
|
|
175
|
+
resumeWindowMs: m.resumeWindowMs ?? record.config.resumeWindowMs,
|
|
176
|
+
inputSampleRateHz: m.audio?.inputSampleRateHz ?? record.config.inputSampleRateHz,
|
|
177
|
+
outputSampleRateHz: m.audio?.outputSampleRateHz ?? record.config.outputSampleRateHz,
|
|
178
|
+
encoding: m.audio?.encoding ?? record.config.encoding,
|
|
179
|
+
binaryEnvelope: m.audio?.binaryEnvelope ?? record.config.binaryEnvelope,
|
|
180
|
+
rawBinaryInput: m.audio?.rawBinaryInput ?? record.config.rawBinaryInput,
|
|
181
|
+
targetFrameDurationMs: m.audio?.targetFrameDurationMs ?? record.config.targetFrameDurationMs,
|
|
182
|
+
},
|
|
183
|
+
sessionEvents: [...record.sessionEvents, ev],
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const turnId = turnIdOf(message);
|
|
188
|
+
// No turn to attribute this to — including every unknown type that carries no
|
|
189
|
+
// turnId, e.g. the agents-SDK `cf_agent_*` messages. Retained, but inert.
|
|
190
|
+
if (turnId === undefined) {
|
|
191
|
+
return { ...record, sessionEvents: [...record.sessionEvents, ev] };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let turns = record.turns;
|
|
195
|
+
let droppedTurns = record.droppedTurns;
|
|
196
|
+
let current = turns.find((t) => t.turnId === turnId);
|
|
197
|
+
if (!current) {
|
|
198
|
+
// Out-of-order arrival is fine: a turn is created by whichever of its
|
|
199
|
+
// messages lands first, not by a designated "start" message.
|
|
200
|
+
current = newTurn(turnId, atMs);
|
|
201
|
+
let appended = [...turns, current];
|
|
202
|
+
if (appended.length > limits.maxTurns) {
|
|
203
|
+
appended = appended.slice(appended.length - limits.maxTurns);
|
|
204
|
+
droppedTurns += 1;
|
|
205
|
+
}
|
|
206
|
+
turns = appended;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
let turn = pushEvent(current, ev, limits);
|
|
210
|
+
|
|
211
|
+
switch (type) {
|
|
212
|
+
case "stt_chunk":
|
|
213
|
+
turn = { ...turn, userInterim: (message as { transcript: string }).transcript };
|
|
214
|
+
break;
|
|
215
|
+
case "stt_output": {
|
|
216
|
+
const m = message as { transcript: string; confidence?: number };
|
|
217
|
+
turn = { ...turn, userTranscript: m.transcript, userInterim: undefined, userConfidence: m.confidence };
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
case "agent_chunk":
|
|
221
|
+
turn = { ...turn, agentText: turn.agentText + (message as { text: string }).text };
|
|
222
|
+
break;
|
|
223
|
+
case "agent_tool_call": {
|
|
224
|
+
const m = message as { id?: string; name: string; args?: unknown };
|
|
225
|
+
turn = { ...turn, toolCalls: upsertToolCall(turn.toolCalls, m.id, { name: m.name, args: m.args }) };
|
|
226
|
+
break;
|
|
227
|
+
}
|
|
228
|
+
case "agent_tool_result": {
|
|
229
|
+
const m = message as { id?: string; result?: unknown };
|
|
230
|
+
turn = { ...turn, toolCalls: upsertToolCall(turn.toolCalls, m.id, { result: m.result }) };
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
case "tool_call_started":
|
|
234
|
+
case "tool_call_delayed":
|
|
235
|
+
case "tool_call_complete":
|
|
236
|
+
case "tool_call_failed": {
|
|
237
|
+
const m = message as { toolId?: string; toolName?: string; afterMs?: number };
|
|
238
|
+
const phase = type.slice("tool_call_".length) as ToolCall["phase"];
|
|
239
|
+
turn = {
|
|
240
|
+
...turn,
|
|
241
|
+
toolCalls: upsertToolCall(turn.toolCalls, m.toolId, { name: m.toolName, phase, afterMs: m.afterMs }),
|
|
242
|
+
};
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
case "tts_chunk":
|
|
246
|
+
turn = { ...turn, ttsAudioBytes: turn.ttsAudioBytes + ((message as { byteLength?: number }).byteLength ?? 0) };
|
|
247
|
+
break;
|
|
248
|
+
case "agent_interrupted":
|
|
249
|
+
turn = { ...turn, interrupted: { atMs, reason: (message as { reason?: string }).reason } };
|
|
250
|
+
break;
|
|
251
|
+
case "turn_complete":
|
|
252
|
+
turn = {
|
|
253
|
+
...turn,
|
|
254
|
+
complete: true,
|
|
255
|
+
userTranscript: (message as { transcript?: string }).transcript ?? turn.userTranscript,
|
|
256
|
+
};
|
|
257
|
+
break;
|
|
258
|
+
case "metrics": {
|
|
259
|
+
const {
|
|
260
|
+
type: _t,
|
|
261
|
+
turnId: _i,
|
|
262
|
+
correlationId: _c,
|
|
263
|
+
endpointingOwner: owner,
|
|
264
|
+
endpointingReason: reason,
|
|
265
|
+
...timings
|
|
266
|
+
} = message as Record<string, unknown>;
|
|
267
|
+
// The endpointing decision is a fact about the turn, not a timing, so it lives
|
|
268
|
+
// on the turn itself — not inside `timings`. Preserve an earlier value if this
|
|
269
|
+
// metrics message omits it (re-emission should not erase a known decision).
|
|
270
|
+
const nextOwner = typeof owner === "string" ? owner : turn.endpointingOwner;
|
|
271
|
+
const nextReason = typeof reason === "string" ? reason : turn.endpointingReason;
|
|
272
|
+
turn = {
|
|
273
|
+
...turn,
|
|
274
|
+
timings: timings as TurnTimings,
|
|
275
|
+
...(nextOwner !== undefined ? { endpointingOwner: nextOwner } : {}),
|
|
276
|
+
...(nextReason !== undefined ? { endpointingReason: nextReason } : {}),
|
|
277
|
+
};
|
|
278
|
+
break;
|
|
279
|
+
}
|
|
280
|
+
case "error": {
|
|
281
|
+
const m = message as { component?: string; category?: string; message: string };
|
|
282
|
+
turn = {
|
|
283
|
+
...turn,
|
|
284
|
+
errors: [...turn.errors, { atMs, component: m.component, category: m.category, message: m.message }],
|
|
285
|
+
};
|
|
286
|
+
break;
|
|
287
|
+
}
|
|
288
|
+
default:
|
|
289
|
+
// Unknown or non-mutating type (speech_started, tts_end, audio_clear, …):
|
|
290
|
+
// already recorded in `events`; nothing further to derive.
|
|
291
|
+
break;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
return { ...record, turns: turns.map((t) => (t.turnId === turnId ? turn : t)), droppedTurns };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Fold a whole message stream. Equivalent to repeated `applyMessage`. */
|
|
298
|
+
export function buildSessionRecord(
|
|
299
|
+
messages: readonly { readonly message: AnyMessage; readonly atMs: number }[],
|
|
300
|
+
config: SessionConfig = {},
|
|
301
|
+
limits: SessionRecordLimits = DEFAULT_LIMITS,
|
|
302
|
+
): SessionRecord {
|
|
303
|
+
return messages.reduce(
|
|
304
|
+
(rec, { message, atMs }) => applyMessage(rec, message, atMs, limits),
|
|
305
|
+
emptySessionRecord(config),
|
|
306
|
+
);
|
|
307
|
+
}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// TurnAudioRecorder — keeps the microphone PCM you already streamed, sliced per turn,
|
|
4
|
+
// so a turn can be saved as a fixture. Without this the studio sends audio to the
|
|
5
|
+
// server and keeps none of it, and the "capture a fixture" loop cannot close.
|
|
6
|
+
//
|
|
7
|
+
// Dependency-free and DOM-free, exported from the `/turn-recorder` subpath: it takes
|
|
8
|
+
// Int16 frames and returns WAV bytes, so the same code runs in a browser, in Node, and
|
|
9
|
+
// in CI. It holds no socket and no AudioContext — you push it the frames you were
|
|
10
|
+
// sending anyway, and the server's own messages drive the turn boundaries.
|
|
11
|
+
|
|
12
|
+
import type { SyrinxStudioMessage } from "./index.js";
|
|
13
|
+
|
|
14
|
+
export interface TurnAudioRecorderOptions {
|
|
15
|
+
/** Turns to retain audio for. Older blobs are dropped; the TurnRecord itself is not ours to evict. */
|
|
16
|
+
readonly maxTurns?: number;
|
|
17
|
+
/**
|
|
18
|
+
* Audio kept from *before* `speech_started`, in ms.
|
|
19
|
+
*
|
|
20
|
+
* Load-bearing, not a nicety: VAD announces speech after onset, so a recorder that
|
|
21
|
+
* starts buffering on `speech_started` clips the first phoneme. A fixture missing its
|
|
22
|
+
* onset transcribes differently from the live turn it was captured from, which makes
|
|
23
|
+
* it worse than no fixture — it looks like a real regression.
|
|
24
|
+
*/
|
|
25
|
+
readonly preRollMs?: number;
|
|
26
|
+
/** Hard ceiling per turn. A stuck endpointer must not grow a buffer without bound. */
|
|
27
|
+
readonly maxTurnMs?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const DEFAULT_RECORDER_OPTIONS: Required<TurnAudioRecorderOptions> = {
|
|
31
|
+
maxTurns: 10,
|
|
32
|
+
preRollMs: 300,
|
|
33
|
+
maxTurnMs: 60_000,
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export interface RecordedTurnAudio {
|
|
37
|
+
readonly turnId: string;
|
|
38
|
+
readonly sampleRateHz: number;
|
|
39
|
+
readonly durationMs: number;
|
|
40
|
+
readonly byteLength: number;
|
|
41
|
+
/** True when the turn hit `maxTurnMs` and the tail was dropped. Surfaced, never silent. */
|
|
42
|
+
readonly truncated: boolean;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const DEFAULT_SAMPLE_RATE_HZ = 16_000;
|
|
46
|
+
|
|
47
|
+
/** Minimal RIFF/WAVE header + samples. Mono PCM16 — the format every STT accepts. */
|
|
48
|
+
export function encodeWav(samples: Int16Array, sampleRateHz: number): Uint8Array {
|
|
49
|
+
if (!Number.isFinite(sampleRateHz) || sampleRateHz <= 0) {
|
|
50
|
+
throw new Error(`encodeWav: invalid sampleRateHz ${String(sampleRateHz)}`);
|
|
51
|
+
}
|
|
52
|
+
const dataBytes = samples.length * 2;
|
|
53
|
+
const out = new Uint8Array(44 + dataBytes);
|
|
54
|
+
const view = new DataView(out.buffer);
|
|
55
|
+
const ascii = (offset: number, text: string): void => {
|
|
56
|
+
for (let i = 0; i < text.length; i += 1) out[offset + i] = text.charCodeAt(i);
|
|
57
|
+
};
|
|
58
|
+
ascii(0, "RIFF");
|
|
59
|
+
view.setUint32(4, 36 + dataBytes, true);
|
|
60
|
+
ascii(8, "WAVE");
|
|
61
|
+
ascii(12, "fmt ");
|
|
62
|
+
view.setUint32(16, 16, true); // PCM chunk size
|
|
63
|
+
view.setUint16(20, 1, true); // PCM
|
|
64
|
+
view.setUint16(22, 1, true); // mono
|
|
65
|
+
view.setUint32(24, sampleRateHz, true);
|
|
66
|
+
view.setUint32(28, sampleRateHz * 2, true); // byte rate
|
|
67
|
+
view.setUint16(32, 2, true); // block align
|
|
68
|
+
view.setUint16(34, 16, true); // bits per sample
|
|
69
|
+
ascii(36, "data");
|
|
70
|
+
view.setUint32(40, dataBytes, true);
|
|
71
|
+
// Copy through a DataView so the result is little-endian on every host, rather than
|
|
72
|
+
// inheriting the platform's byte order via Int16Array.set.
|
|
73
|
+
for (let i = 0; i < samples.length; i += 1) {
|
|
74
|
+
view.setInt16(44 + i * 2, samples[i] ?? 0, true);
|
|
75
|
+
}
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface OpenSegment {
|
|
80
|
+
samples: Int16Array[];
|
|
81
|
+
length: number;
|
|
82
|
+
truncated: boolean;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export class TurnAudioRecorder {
|
|
86
|
+
private readonly options: Required<TurnAudioRecorderOptions>;
|
|
87
|
+
private sampleRateHz = DEFAULT_SAMPLE_RATE_HZ;
|
|
88
|
+
/** Rolling pre-roll, trimmed to preRollMs on every push. */
|
|
89
|
+
private preRoll: Int16Array[] = [];
|
|
90
|
+
private preRollLength = 0;
|
|
91
|
+
private open: OpenSegment | undefined;
|
|
92
|
+
/** Sealed but not yet attributed to a turnId — turnId arrives with a later message. */
|
|
93
|
+
private pending: { samples: Int16Array; truncated: boolean } | undefined;
|
|
94
|
+
private readonly turns = new Map<string, { samples: Int16Array; truncated: boolean }>();
|
|
95
|
+
|
|
96
|
+
constructor(options: TurnAudioRecorderOptions = {}) {
|
|
97
|
+
this.options = { ...DEFAULT_RECORDER_OPTIONS, ...options };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Push the uplink frame you are already sending. Cheap: retains the reference, copies on seal. */
|
|
101
|
+
pushFrame(frame: Int16Array): void {
|
|
102
|
+
if (frame.length === 0) return;
|
|
103
|
+
if (this.open) {
|
|
104
|
+
const cap = Math.floor((this.options.maxTurnMs / 1000) * this.sampleRateHz);
|
|
105
|
+
if (this.open.length >= cap) {
|
|
106
|
+
this.open.truncated = true;
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
this.open.samples.push(frame);
|
|
110
|
+
this.open.length += frame.length;
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.preRoll.push(frame);
|
|
114
|
+
this.preRollLength += frame.length;
|
|
115
|
+
const keep = Math.floor((this.options.preRollMs / 1000) * this.sampleRateHz);
|
|
116
|
+
while (this.preRollLength - (this.preRoll[0]?.length ?? 0) >= keep && this.preRoll.length > 1) {
|
|
117
|
+
this.preRollLength -= this.preRoll.shift()?.length ?? 0;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Feed the same messages you feed the SessionRecord. Turn boundaries come from the server. */
|
|
122
|
+
onMessage(message: SyrinxStudioMessage | { readonly type: string; readonly [k: string]: unknown }): void {
|
|
123
|
+
const type = message.type;
|
|
124
|
+
if (type === "ready") {
|
|
125
|
+
const rate = (message as { audio?: { inputSampleRateHz?: unknown } }).audio?.inputSampleRateHz;
|
|
126
|
+
if (typeof rate === "number" && Number.isFinite(rate) && rate > 0) this.sampleRateHz = rate;
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const turnId = typeof (message as { turnId?: unknown }).turnId === "string"
|
|
130
|
+
? (message as { turnId: string }).turnId
|
|
131
|
+
: undefined;
|
|
132
|
+
|
|
133
|
+
if (type === "speech_started") {
|
|
134
|
+
// Start from the pre-roll so the onset is present.
|
|
135
|
+
this.open = { samples: [...this.preRoll], length: this.preRollLength, truncated: false };
|
|
136
|
+
this.preRoll = [];
|
|
137
|
+
this.preRollLength = 0;
|
|
138
|
+
if (turnId !== undefined) this.attributeOnSeal = turnId;
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
if (type === "speech_ended" || type === "stt_output" || type === "turn_complete") {
|
|
143
|
+
this.seal(turnId);
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private attributeOnSeal: string | undefined;
|
|
149
|
+
|
|
150
|
+
private seal(turnId: string | undefined): void {
|
|
151
|
+
if (this.open) {
|
|
152
|
+
const merged = new Int16Array(this.open.length);
|
|
153
|
+
let offset = 0;
|
|
154
|
+
for (const chunk of this.open.samples) {
|
|
155
|
+
merged.set(chunk, offset);
|
|
156
|
+
offset += chunk.length;
|
|
157
|
+
}
|
|
158
|
+
this.pending = { samples: merged, truncated: this.open.truncated };
|
|
159
|
+
this.open = undefined;
|
|
160
|
+
}
|
|
161
|
+
if (!this.pending) return;
|
|
162
|
+
const id = turnId ?? this.attributeOnSeal;
|
|
163
|
+
if (id === undefined) return; // wait for a message that names the turn
|
|
164
|
+
this.attributeOnSeal = undefined;
|
|
165
|
+
this.turns.set(id, this.pending);
|
|
166
|
+
this.pending = undefined;
|
|
167
|
+
// Bounded by construction: audio blobs, not the records, are the memory risk.
|
|
168
|
+
while (this.turns.size > this.options.maxTurns) {
|
|
169
|
+
const oldest = this.turns.keys().next().value;
|
|
170
|
+
if (oldest === undefined) break;
|
|
171
|
+
this.turns.delete(oldest);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** WAV bytes for a turn, or undefined if it was never captured or has been evicted. */
|
|
176
|
+
getWav(turnId: string): Uint8Array | undefined {
|
|
177
|
+
const entry = this.turns.get(turnId);
|
|
178
|
+
return entry ? encodeWav(entry.samples, this.sampleRateHz) : undefined;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** What audio is retained right now, oldest first. */
|
|
182
|
+
list(): readonly RecordedTurnAudio[] {
|
|
183
|
+
return [...this.turns.entries()].map(([turnId, entry]) => ({
|
|
184
|
+
turnId,
|
|
185
|
+
sampleRateHz: this.sampleRateHz,
|
|
186
|
+
durationMs: Math.round((entry.samples.length / this.sampleRateHz) * 1000),
|
|
187
|
+
byteLength: entry.samples.length * 2,
|
|
188
|
+
truncated: entry.truncated,
|
|
189
|
+
}));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Total retained audio in bytes — what a memory readout should show. */
|
|
193
|
+
retainedBytes(): number {
|
|
194
|
+
let total = 0;
|
|
195
|
+
for (const entry of this.turns.values()) total += entry.samples.length * 2;
|
|
196
|
+
return total;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
reset(): void {
|
|
200
|
+
this.preRoll = [];
|
|
201
|
+
this.preRollLength = 0;
|
|
202
|
+
this.open = undefined;
|
|
203
|
+
this.pending = undefined;
|
|
204
|
+
this.attributeOnSeal = undefined;
|
|
205
|
+
this.turns.clear();
|
|
206
|
+
}
|
|
207
|
+
}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Turn timeline — the per-turn latency waterfall, computed from the `metrics`
|
|
4
|
+
// message the server already sends.
|
|
5
|
+
//
|
|
6
|
+
// "It cut me off" is not actionable; a lane showing speech ending at 2.1s and the
|
|
7
|
+
// endpoint firing at 2.16s is. This is the view no competing playground ships,
|
|
8
|
+
// and the data has been arriving in the browser all along.
|
|
9
|
+
//
|
|
10
|
+
// Pure and DOM-free, so a timeline can be rendered from a recorded fixture in a
|
|
11
|
+
// test rather than only from a live provider.
|
|
12
|
+
|
|
13
|
+
import type { TurnRecord, TurnTimings } from "./session-record.js";
|
|
14
|
+
|
|
15
|
+
export interface TimelineSegment {
|
|
16
|
+
readonly key: string;
|
|
17
|
+
/** Plain language. The reader does not know what `eos.turn_complete` is. */
|
|
18
|
+
readonly label: string;
|
|
19
|
+
readonly startMs: number;
|
|
20
|
+
readonly durationMs: number;
|
|
21
|
+
/** True for the longest segment — where the time actually went. */
|
|
22
|
+
readonly slowest: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface TurnTimeline {
|
|
26
|
+
readonly turnId: string;
|
|
27
|
+
readonly segments: readonly TimelineSegment[];
|
|
28
|
+
readonly totalMs: number;
|
|
29
|
+
/** Why there is nothing to draw, when there is nothing to draw. */
|
|
30
|
+
readonly unavailable?: "no-metrics" | "insufficient-marks";
|
|
31
|
+
/**
|
|
32
|
+
* Set when the agent replied implausibly fast, which almost always means the
|
|
33
|
+
* endpointer fired while the caller was still speaking. Most tools celebrate
|
|
34
|
+
* this number; it is usually a defect.
|
|
35
|
+
*/
|
|
36
|
+
readonly suspiciouslyFast?: { readonly totalMs: number; readonly floorMs: number };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Below this, a reply is not "fast" — the endpointer almost certainly cut the
|
|
41
|
+
* caller off. Asserting a floor as well as a ceiling is the inversion no
|
|
42
|
+
* surveyed framework supports.
|
|
43
|
+
*/
|
|
44
|
+
export const FAST_TURN_FLOOR_MS = 700;
|
|
45
|
+
|
|
46
|
+
// Ordered marks. Each segment spans one mark to the next, so a missing mark
|
|
47
|
+
// collapses its segment rather than corrupting the ones around it.
|
|
48
|
+
const MARKS: readonly { readonly field: keyof TurnTimings; readonly label: string }[] = [
|
|
49
|
+
{ field: "speechEndMs", label: "You stopped speaking" },
|
|
50
|
+
{ field: "textReadyMs", label: "Deciding you're done, transcribing, and thinking" },
|
|
51
|
+
// Segments are labelled by the mark that ENDS them, so this one covers
|
|
52
|
+
// textReady -> firstAudioByte: the reply text already exists and we are waiting on
|
|
53
|
+
// speech. That is time-to-first-audio, which `session-metrics` reports as
|
|
54
|
+
// "Voice (to first audio)" from the same field. Calling it "Thinking" made the two
|
|
55
|
+
// panels name one quantity two different things.
|
|
56
|
+
{ field: "firstAudioByteMs", label: "Voice (to first audio)" },
|
|
57
|
+
{ field: "firstAudioPlayedMs", label: "First audio out" },
|
|
58
|
+
{ field: "lastAudioPlayedMs", label: "Agent speaking" },
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
export function buildTurnTimeline(turn: TurnRecord): TurnTimeline {
|
|
62
|
+
const t = turn.timings;
|
|
63
|
+
if (!t) {
|
|
64
|
+
// The Workers path emits no `metrics` at all (proven by LDT-18). Say so
|
|
65
|
+
// rather than rendering an empty lane that looks like a zero-latency turn.
|
|
66
|
+
return { turnId: turn.turnId, segments: [], totalMs: 0, unavailable: "no-metrics" };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const points = MARKS.map(({ field, label }) => ({ label, at: t[field] })).filter(
|
|
70
|
+
(p): p is { label: string; at: number } => typeof p.at === "number",
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
if (points.length < 2) {
|
|
74
|
+
return { turnId: turn.turnId, segments: [], totalMs: t.e2eMs ?? 0, unavailable: "insufficient-marks" };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const origin = points[0]?.at ?? 0;
|
|
78
|
+
const raw = points.slice(1).map((p, i) => {
|
|
79
|
+
const prev = points[i];
|
|
80
|
+
const startMs = (prev?.at ?? origin) - origin;
|
|
81
|
+
return { key: p.label, label: p.label, startMs, durationMs: Math.max(0, p.at - (prev?.at ?? origin)) };
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
const longest = raw.reduce((max, s) => Math.max(max, s.durationMs), 0);
|
|
85
|
+
const segments: TimelineSegment[] = raw.map((s) => ({ ...s, slowest: longest > 0 && s.durationMs === longest }));
|
|
86
|
+
|
|
87
|
+
const totalMs = t.e2eMs ?? (points[points.length - 1]?.at ?? origin) - origin;
|
|
88
|
+
const suspiciouslyFast =
|
|
89
|
+
totalMs > 0 && totalMs < FAST_TURN_FLOOR_MS ? { totalMs, floorMs: FAST_TURN_FLOOR_MS } : undefined;
|
|
90
|
+
|
|
91
|
+
return { turnId: turn.turnId, segments, totalMs, ...(suspiciouslyFast ? { suspiciouslyFast } : {}) };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function buildTimelines(turns: readonly TurnRecord[]): readonly TurnTimeline[] {
|
|
95
|
+
return turns.map(buildTurnTimeline);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// -----------------------------------------------------------------------------
|
|
99
|
+
// Endpointing decision rendering
|
|
100
|
+
// -----------------------------------------------------------------------------
|
|
101
|
+
// The timeline can already show WHEN a turn ended; this names WHO decided and WHY.
|
|
102
|
+
// That turns "it cut me off" from a feeling into a named cause. Pure and DOM-free
|
|
103
|
+
// so it is testable from a fixture; the component layer just renders `text`.
|
|
104
|
+
|
|
105
|
+
export type EndpointingMarkerKind = "typed" | "endpoint" | "unknown";
|
|
106
|
+
|
|
107
|
+
export interface EndpointingMarker {
|
|
108
|
+
readonly kind: EndpointingMarkerKind;
|
|
109
|
+
/** Plain language — never a packet or message name (`eos.turn_complete`, raw `provider_stt`, …). */
|
|
110
|
+
readonly text: string;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Turn an endpointing decision into the words a person reads. Internal type
|
|
115
|
+
* values (`provider_stt`, `smart_turn`, `timer`, `text`, `force_finalized`, …)
|
|
116
|
+
* stay internal; this is the only place they cross into user-facing text.
|
|
117
|
+
*
|
|
118
|
+
* A genuinely unknown decision is returned as `kind: "unknown"` rather than
|
|
119
|
+
* guessed — a debugging surface that fabricates a cause is worse than one that
|
|
120
|
+
* says it does not know.
|
|
121
|
+
*/
|
|
122
|
+
export function endpointingMarker(owner?: string, reason?: string): EndpointingMarker {
|
|
123
|
+
if (owner === "text") {
|
|
124
|
+
return {
|
|
125
|
+
kind: "typed",
|
|
126
|
+
text: "You typed this turn — nothing transcribed you and nothing judged when you finished, so no endpointer ran.",
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
const who =
|
|
130
|
+
owner === "provider_stt"
|
|
131
|
+
? "The speech-to-text provider"
|
|
132
|
+
: owner === "smart_turn"
|
|
133
|
+
? "The Smart Turn model"
|
|
134
|
+
: owner === "timer"
|
|
135
|
+
? "A turn timer"
|
|
136
|
+
: undefined;
|
|
137
|
+
if (who === undefined) {
|
|
138
|
+
return {
|
|
139
|
+
kind: "unknown",
|
|
140
|
+
text: "This backend did not report what ended the turn, so the cause is unknown.",
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
const text =
|
|
144
|
+
reason === "force_finalized"
|
|
145
|
+
? `${who} force-finalized the transcript after a timeout, not a natural end of speech.`
|
|
146
|
+
: `${who} decided you had finished speaking.`;
|
|
147
|
+
return { kind: "endpoint", text };
|
|
148
|
+
}
|