@crewhaus/barge-in-controller 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +49 -0
- package/dist/index.js +91 -0
- package/package.json +10 -7
- package/src/index.test.ts +0 -302
- package/src/index.ts +0 -124
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R16 `barge-in-controller` — Section 24 VOICE.
|
|
3
|
+
*
|
|
4
|
+
* Coordinator between `vad-engine` and a `RealtimeAdapter`. Watches
|
|
5
|
+
* inbound user audio frames; when the user starts speaking AND the
|
|
6
|
+
* agent is currently playing audio back, fires `adapter.interrupt()`
|
|
7
|
+
* so the agent's response is cancelled and the user's turn takes over.
|
|
8
|
+
*
|
|
9
|
+
* Hysteresis: a single 30ms speech frame is not enough to barge in
|
|
10
|
+
* (rejects a 50ms cough). The controller waits for `triggerFrames`
|
|
11
|
+
* consecutive speech-classified frames within a `windowMs` window
|
|
12
|
+
* before firing. Defaults: 4 frames in 200ms.
|
|
13
|
+
*
|
|
14
|
+
* Playback tracking: the controller listens for `audio_chunk` events on
|
|
15
|
+
* the adapter to mark the agent as "speaking", and for
|
|
16
|
+
* `transcript_final` / `interrupt` / `disconnect` to mark the agent as
|
|
17
|
+
* "idle". Barge-in only fires while the agent is speaking — when it's
|
|
18
|
+
* idle, `feedAudioFrame` is a no-op (the user's normal turn ends with
|
|
19
|
+
* server VAD, no need to interrupt anything).
|
|
20
|
+
*/
|
|
21
|
+
import { type VadDetector } from "@crewhaus/vad-engine";
|
|
22
|
+
import type { RealtimeAdapter } from "@crewhaus/voice-runtime";
|
|
23
|
+
export type BargeInOptions = {
|
|
24
|
+
readonly adapter: RealtimeAdapter;
|
|
25
|
+
readonly detector?: VadDetector;
|
|
26
|
+
/**
|
|
27
|
+
* Consecutive speech frames inside `windowMs` to trigger an
|
|
28
|
+
* interrupt. Defaults to 4 (≈ 120ms of continuous speech at 30ms
|
|
29
|
+
* framing).
|
|
30
|
+
*/
|
|
31
|
+
readonly triggerFrames?: number;
|
|
32
|
+
/** Sliding-window bound for the trigger count. Defaults to 200ms. */
|
|
33
|
+
readonly windowMs?: number;
|
|
34
|
+
/** Per-frame elapsed time advances. Defaults to 30ms. */
|
|
35
|
+
readonly frameMs?: number;
|
|
36
|
+
readonly onBargeIn?: (info: {
|
|
37
|
+
reason: string;
|
|
38
|
+
speechFrames: number;
|
|
39
|
+
}) => void;
|
|
40
|
+
};
|
|
41
|
+
export interface BargeInController {
|
|
42
|
+
/** Push one inbound user-mic PCM frame. Returns the VAD verdict for diagnostics. */
|
|
43
|
+
feedAudioFrame(frame: Int16Array): "speech" | "silence" | "transitioning";
|
|
44
|
+
/** Whether the agent is currently speaking (set by adapter `audio_chunk` events). */
|
|
45
|
+
isAgentSpeaking(): boolean;
|
|
46
|
+
/** Detach the adapter listener. */
|
|
47
|
+
stop(): void;
|
|
48
|
+
}
|
|
49
|
+
export declare function createBargeInController(opts: BargeInOptions): BargeInController;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R16 `barge-in-controller` — Section 24 VOICE.
|
|
3
|
+
*
|
|
4
|
+
* Coordinator between `vad-engine` and a `RealtimeAdapter`. Watches
|
|
5
|
+
* inbound user audio frames; when the user starts speaking AND the
|
|
6
|
+
* agent is currently playing audio back, fires `adapter.interrupt()`
|
|
7
|
+
* so the agent's response is cancelled and the user's turn takes over.
|
|
8
|
+
*
|
|
9
|
+
* Hysteresis: a single 30ms speech frame is not enough to barge in
|
|
10
|
+
* (rejects a 50ms cough). The controller waits for `triggerFrames`
|
|
11
|
+
* consecutive speech-classified frames within a `windowMs` window
|
|
12
|
+
* before firing. Defaults: 4 frames in 200ms.
|
|
13
|
+
*
|
|
14
|
+
* Playback tracking: the controller listens for `audio_chunk` events on
|
|
15
|
+
* the adapter to mark the agent as "speaking", and for
|
|
16
|
+
* `transcript_final` / `interrupt` / `disconnect` to mark the agent as
|
|
17
|
+
* "idle". Barge-in only fires while the agent is speaking — when it's
|
|
18
|
+
* idle, `feedAudioFrame` is a no-op (the user's normal turn ends with
|
|
19
|
+
* server VAD, no need to interrupt anything).
|
|
20
|
+
*/
|
|
21
|
+
import { createVadDetector } from "@crewhaus/vad-engine";
|
|
22
|
+
const DEFAULT_TRIGGER_FRAMES = 4;
|
|
23
|
+
const DEFAULT_WINDOW_MS = 200;
|
|
24
|
+
const DEFAULT_FRAME_MS = 30;
|
|
25
|
+
export function createBargeInController(opts) {
|
|
26
|
+
const detector = opts.detector ?? createVadDetector({ aggressiveness: 1 });
|
|
27
|
+
const triggerFrames = opts.triggerFrames ?? DEFAULT_TRIGGER_FRAMES;
|
|
28
|
+
const windowMs = opts.windowMs ?? DEFAULT_WINDOW_MS;
|
|
29
|
+
const frameMs = opts.frameMs ?? DEFAULT_FRAME_MS;
|
|
30
|
+
let agentSpeaking = false;
|
|
31
|
+
let virtualNowMs = 0;
|
|
32
|
+
// Sliding window of speech frame timestamps.
|
|
33
|
+
const speechFrameStamps = [];
|
|
34
|
+
const off = opts.adapter.on((event) => {
|
|
35
|
+
if (event.kind === "audio_chunk") {
|
|
36
|
+
agentSpeaking = true;
|
|
37
|
+
}
|
|
38
|
+
else if (event.kind === "transcript_final" ||
|
|
39
|
+
event.kind === "interrupt" ||
|
|
40
|
+
event.kind === "disconnect") {
|
|
41
|
+
agentSpeaking = false;
|
|
42
|
+
// Reset window so a stray pre-event frame doesn't carry over.
|
|
43
|
+
speechFrameStamps.length = 0;
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
return {
|
|
47
|
+
feedAudioFrame(frame) {
|
|
48
|
+
virtualNowMs += frameMs;
|
|
49
|
+
const verdict = detector.detect(frame);
|
|
50
|
+
if (!agentSpeaking) {
|
|
51
|
+
// No audio playing → no barge-in needed. Still advance the
|
|
52
|
+
// window so it doesn't accumulate stale frames.
|
|
53
|
+
if (verdict === "speech") {
|
|
54
|
+
speechFrameStamps.push(virtualNowMs);
|
|
55
|
+
}
|
|
56
|
+
// Trim window
|
|
57
|
+
const cutoff = virtualNowMs - windowMs;
|
|
58
|
+
while (speechFrameStamps.length > 0 && (speechFrameStamps[0] ?? cutoff) < cutoff) {
|
|
59
|
+
speechFrameStamps.shift();
|
|
60
|
+
}
|
|
61
|
+
return verdict;
|
|
62
|
+
}
|
|
63
|
+
if (verdict === "speech") {
|
|
64
|
+
speechFrameStamps.push(virtualNowMs);
|
|
65
|
+
}
|
|
66
|
+
const cutoff = virtualNowMs - windowMs;
|
|
67
|
+
while (speechFrameStamps.length > 0 && (speechFrameStamps[0] ?? cutoff) < cutoff) {
|
|
68
|
+
speechFrameStamps.shift();
|
|
69
|
+
}
|
|
70
|
+
if (speechFrameStamps.length >= triggerFrames) {
|
|
71
|
+
const count = speechFrameStamps.length;
|
|
72
|
+
speechFrameStamps.length = 0;
|
|
73
|
+
agentSpeaking = false;
|
|
74
|
+
try {
|
|
75
|
+
opts.adapter.interrupt("user barged in");
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
/* tolerate adapter disconnect mid-barge-in */
|
|
79
|
+
}
|
|
80
|
+
opts.onBargeIn?.({ reason: "user barged in", speechFrames: count });
|
|
81
|
+
}
|
|
82
|
+
return verdict;
|
|
83
|
+
},
|
|
84
|
+
isAgentSpeaking() {
|
|
85
|
+
return agentSpeaking;
|
|
86
|
+
},
|
|
87
|
+
stop() {
|
|
88
|
+
off();
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
package/package.json
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/barge-in-controller",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Barge-in coordinator: watches inbound audio via vad-engine, fires interrupt() on adapter when user speaks during agent playback (Section 24 VOICE)",
|
|
6
|
-
"main": "
|
|
7
|
-
"types": "
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
8
|
"exports": {
|
|
9
|
-
".":
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
10
13
|
},
|
|
11
14
|
"scripts": {
|
|
12
15
|
"test": "bun test src"
|
|
13
16
|
},
|
|
14
17
|
"dependencies": {
|
|
15
|
-
"@crewhaus/vad-engine": "0.1.
|
|
16
|
-
"@crewhaus/voice-runtime": "0.1.
|
|
18
|
+
"@crewhaus/vad-engine": "0.1.6",
|
|
19
|
+
"@crewhaus/voice-runtime": "0.1.6"
|
|
17
20
|
},
|
|
18
21
|
"license": "Apache-2.0",
|
|
19
22
|
"author": {
|
|
@@ -33,5 +36,5 @@
|
|
|
33
36
|
"publishConfig": {
|
|
34
37
|
"access": "public"
|
|
35
38
|
},
|
|
36
|
-
"files": ["
|
|
39
|
+
"files": ["dist", "README.md", "LICENSE", "NOTICE"]
|
|
37
40
|
}
|
package/src/index.test.ts
DELETED
|
@@ -1,302 +0,0 @@
|
|
|
1
|
-
import { describe, expect, test } from "bun:test";
|
|
2
|
-
import type { RealtimeAdapter, RealtimeEvent, RealtimeEventHandler } from "@crewhaus/voice-runtime";
|
|
3
|
-
import { createBargeInController } from "./index.js";
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Stub adapter that exposes the event-bus + records interrupt() calls.
|
|
7
|
-
*/
|
|
8
|
-
function makeStubAdapter(): {
|
|
9
|
-
adapter: RealtimeAdapter;
|
|
10
|
-
fire(event: RealtimeEvent): void;
|
|
11
|
-
interrupts: string[];
|
|
12
|
-
} {
|
|
13
|
-
const handlers = new Set<RealtimeEventHandler>();
|
|
14
|
-
const interrupts: string[] = [];
|
|
15
|
-
const adapter: RealtimeAdapter = {
|
|
16
|
-
providerId: "openai",
|
|
17
|
-
sampleRate: 24_000,
|
|
18
|
-
get connected() {
|
|
19
|
-
return true;
|
|
20
|
-
},
|
|
21
|
-
async connect() {},
|
|
22
|
-
sendAudio() {},
|
|
23
|
-
sendText() {},
|
|
24
|
-
interrupt(reason) {
|
|
25
|
-
interrupts.push(reason);
|
|
26
|
-
// Mimic the real adapter: emit a synthetic interrupt event.
|
|
27
|
-
for (const h of handlers) h({ kind: "interrupt", reason });
|
|
28
|
-
},
|
|
29
|
-
on(h) {
|
|
30
|
-
handlers.add(h);
|
|
31
|
-
return () => {
|
|
32
|
-
handlers.delete(h);
|
|
33
|
-
};
|
|
34
|
-
},
|
|
35
|
-
async disconnect() {},
|
|
36
|
-
};
|
|
37
|
-
return {
|
|
38
|
-
adapter,
|
|
39
|
-
fire: (event) => {
|
|
40
|
-
for (const h of handlers) h(event);
|
|
41
|
-
},
|
|
42
|
-
interrupts,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
function speechFrame(): Int16Array {
|
|
47
|
-
// Energy + ZCR sufficient to trigger speech verdict at aggressiveness 1.
|
|
48
|
-
const out = new Int16Array(720);
|
|
49
|
-
for (let i = 0; i < 720; i++) {
|
|
50
|
-
const sig =
|
|
51
|
-
0.7 * Math.sin((2 * Math.PI * 500 * i) / 24_000) +
|
|
52
|
-
0.3 * Math.sin((2 * Math.PI * 1500 * i) / 24_000);
|
|
53
|
-
out[i] = Math.round(sig * 8000);
|
|
54
|
-
}
|
|
55
|
-
return out;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
function silentFrame(): Int16Array {
|
|
59
|
-
return new Int16Array(720);
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
describe("createBargeInController", () => {
|
|
63
|
-
test("does NOT interrupt when the agent is idle (T1)", () => {
|
|
64
|
-
const { adapter, interrupts } = makeStubAdapter();
|
|
65
|
-
const ctrl = createBargeInController({ adapter });
|
|
66
|
-
for (let i = 0; i < 10; i++) ctrl.feedAudioFrame(speechFrame());
|
|
67
|
-
expect(interrupts).toHaveLength(0);
|
|
68
|
-
ctrl.stop();
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
test("does NOT interrupt on a single speech frame (hysteresis)", () => {
|
|
72
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
73
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 4 });
|
|
74
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
75
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
76
|
-
ctrl.feedAudioFrame(silentFrame());
|
|
77
|
-
expect(interrupts).toHaveLength(0);
|
|
78
|
-
ctrl.stop();
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
test("interrupts after triggerFrames consecutive speech frames during agent playback (T3)", () => {
|
|
82
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
83
|
-
const onBargeIn: { hits: number } = { hits: 0 };
|
|
84
|
-
const ctrl = createBargeInController({
|
|
85
|
-
adapter,
|
|
86
|
-
triggerFrames: 4,
|
|
87
|
-
windowMs: 200,
|
|
88
|
-
frameMs: 30,
|
|
89
|
-
onBargeIn: () => {
|
|
90
|
-
onBargeIn.hits += 1;
|
|
91
|
-
},
|
|
92
|
-
});
|
|
93
|
-
// Mark agent speaking.
|
|
94
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
95
|
-
expect(ctrl.isAgentSpeaking()).toBe(true);
|
|
96
|
-
|
|
97
|
-
for (let i = 0; i < 4; i++) ctrl.feedAudioFrame(speechFrame());
|
|
98
|
-
expect(interrupts).toHaveLength(1);
|
|
99
|
-
expect(interrupts[0]).toBe("user barged in");
|
|
100
|
-
expect(onBargeIn.hits).toBe(1);
|
|
101
|
-
expect(ctrl.isAgentSpeaking()).toBe(false);
|
|
102
|
-
ctrl.stop();
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
test("idle agent → speech doesn't trigger interrupt", () => {
|
|
106
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
107
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 2, windowMs: 200, frameMs: 30 });
|
|
108
|
-
// Without an audio_chunk event, agent stays idle.
|
|
109
|
-
for (let i = 0; i < 10; i++) ctrl.feedAudioFrame(speechFrame());
|
|
110
|
-
expect(interrupts).toHaveLength(0);
|
|
111
|
-
// Now mark agent idle->speaking and feed speech: SHOULD trigger.
|
|
112
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
113
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
114
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
115
|
-
expect(interrupts).toHaveLength(1);
|
|
116
|
-
ctrl.stop();
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
test("transcript_final marks agent idle and resets the speech-frame window", () => {
|
|
120
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
121
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 4, windowMs: 200, frameMs: 30 });
|
|
122
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
123
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
124
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
125
|
-
fire({ kind: "transcript_final", text: "..." });
|
|
126
|
-
expect(ctrl.isAgentSpeaking()).toBe(false);
|
|
127
|
-
// Now agent idle: 2 more speech frames shouldn't trigger.
|
|
128
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
129
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
130
|
-
expect(interrupts).toHaveLength(0);
|
|
131
|
-
ctrl.stop();
|
|
132
|
-
});
|
|
133
|
-
|
|
134
|
-
test("disconnect marks agent idle and resets the speech-frame window", () => {
|
|
135
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
136
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 4, windowMs: 200, frameMs: 30 });
|
|
137
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
138
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
139
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
140
|
-
fire({ kind: "disconnect", code: 1000, reason: "bye" });
|
|
141
|
-
expect(ctrl.isAgentSpeaking()).toBe(false);
|
|
142
|
-
// Window was reset on disconnect: 2 more frames (had agent stayed speaking)
|
|
143
|
-
// would have hit triggerFrames=4, but the agent is idle so nothing fires.
|
|
144
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
145
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
146
|
-
expect(interrupts).toHaveLength(0);
|
|
147
|
-
ctrl.stop();
|
|
148
|
-
});
|
|
149
|
-
|
|
150
|
-
test("returns the raw VAD verdict for diagnostics (speech + silence)", () => {
|
|
151
|
-
const { adapter, fire } = makeStubAdapter();
|
|
152
|
-
const ctrl = createBargeInController({
|
|
153
|
-
adapter,
|
|
154
|
-
triggerFrames: 99,
|
|
155
|
-
windowMs: 200,
|
|
156
|
-
frameMs: 30,
|
|
157
|
-
});
|
|
158
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
159
|
-
// Agent is speaking; a silent frame must not be counted as speech, and the
|
|
160
|
-
// verdict is surfaced verbatim.
|
|
161
|
-
expect(ctrl.feedAudioFrame(silentFrame())).toBe("silence");
|
|
162
|
-
expect(ctrl.feedAudioFrame(speechFrame())).toBe("speech");
|
|
163
|
-
ctrl.stop();
|
|
164
|
-
});
|
|
165
|
-
|
|
166
|
-
test("does NOT interrupt when speech frames age out of the sliding window", () => {
|
|
167
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
168
|
-
// windowMs (60) only holds ~2 frames at frameMs 30, so a steady stream of
|
|
169
|
-
// speech never accumulates the 4 required *within the window*.
|
|
170
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 4, windowMs: 60, frameMs: 30 });
|
|
171
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
172
|
-
for (let i = 0; i < 20; i++) {
|
|
173
|
-
expect(ctrl.feedAudioFrame(speechFrame())).toBe("speech");
|
|
174
|
-
}
|
|
175
|
-
expect(interrupts).toHaveLength(0);
|
|
176
|
-
expect(ctrl.isAgentSpeaking()).toBe(true);
|
|
177
|
-
ctrl.stop();
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
test("idle agent: speech frames also age out of the window (no stale carry-over)", () => {
|
|
181
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
182
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 4, windowMs: 60, frameMs: 30 });
|
|
183
|
-
// Agent idle: feed a long stream of speech. The window trims as it grows so
|
|
184
|
-
// it never accumulates, then a later audio_chunk + brief speech is still
|
|
185
|
-
// governed solely by the in-window count.
|
|
186
|
-
for (let i = 0; i < 20; i++) ctrl.feedAudioFrame(speechFrame());
|
|
187
|
-
expect(interrupts).toHaveLength(0);
|
|
188
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
189
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
190
|
-
expect(interrupts).toHaveLength(0);
|
|
191
|
-
ctrl.stop();
|
|
192
|
-
});
|
|
193
|
-
|
|
194
|
-
test("fires onBargeIn with the in-window speech-frame count", () => {
|
|
195
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
196
|
-
const seen: Array<{ reason: string; speechFrames: number }> = [];
|
|
197
|
-
const ctrl = createBargeInController({
|
|
198
|
-
adapter,
|
|
199
|
-
triggerFrames: 4,
|
|
200
|
-
windowMs: 200,
|
|
201
|
-
frameMs: 30,
|
|
202
|
-
onBargeIn: (info) => seen.push(info),
|
|
203
|
-
});
|
|
204
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
205
|
-
for (let i = 0; i < 4; i++) ctrl.feedAudioFrame(speechFrame());
|
|
206
|
-
expect(interrupts).toEqual(["user barged in"]);
|
|
207
|
-
expect(seen).toEqual([{ reason: "user barged in", speechFrames: 4 }]);
|
|
208
|
-
ctrl.stop();
|
|
209
|
-
});
|
|
210
|
-
|
|
211
|
-
test("interrupts even when no onBargeIn callback is supplied", () => {
|
|
212
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
213
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 2, windowMs: 200, frameMs: 30 });
|
|
214
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
215
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
216
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
217
|
-
expect(interrupts).toEqual(["user barged in"]);
|
|
218
|
-
ctrl.stop();
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
test("tolerates an adapter whose interrupt() throws mid-barge-in", () => {
|
|
222
|
-
// Custom adapter: interrupt() throws (e.g. socket already closed). The
|
|
223
|
-
// controller must swallow it, still mark the agent idle, and still fire
|
|
224
|
-
// the onBargeIn observer.
|
|
225
|
-
const handlers = new Set<RealtimeEventHandler>();
|
|
226
|
-
let interruptCalls = 0;
|
|
227
|
-
const adapter: RealtimeAdapter = {
|
|
228
|
-
providerId: "openai",
|
|
229
|
-
sampleRate: 24_000,
|
|
230
|
-
get connected() {
|
|
231
|
-
return true;
|
|
232
|
-
},
|
|
233
|
-
async connect() {},
|
|
234
|
-
sendAudio() {},
|
|
235
|
-
sendText() {},
|
|
236
|
-
interrupt() {
|
|
237
|
-
interruptCalls += 1;
|
|
238
|
-
throw new Error("socket closed");
|
|
239
|
-
},
|
|
240
|
-
on(h) {
|
|
241
|
-
handlers.add(h);
|
|
242
|
-
return () => {
|
|
243
|
-
handlers.delete(h);
|
|
244
|
-
};
|
|
245
|
-
},
|
|
246
|
-
async disconnect() {},
|
|
247
|
-
};
|
|
248
|
-
const fire = (event: RealtimeEvent): void => {
|
|
249
|
-
for (const h of handlers) h(event);
|
|
250
|
-
};
|
|
251
|
-
let info: { reason: string; speechFrames: number } | undefined;
|
|
252
|
-
const ctrl = createBargeInController({
|
|
253
|
-
adapter,
|
|
254
|
-
triggerFrames: 4,
|
|
255
|
-
onBargeIn: (i) => {
|
|
256
|
-
info = i;
|
|
257
|
-
},
|
|
258
|
-
});
|
|
259
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
260
|
-
expect(() => {
|
|
261
|
-
for (let i = 0; i < 4; i++) ctrl.feedAudioFrame(speechFrame());
|
|
262
|
-
}).not.toThrow();
|
|
263
|
-
expect(interruptCalls).toBe(1);
|
|
264
|
-
expect(info).toEqual({ reason: "user barged in", speechFrames: 4 });
|
|
265
|
-
expect(ctrl.isAgentSpeaking()).toBe(false);
|
|
266
|
-
ctrl.stop();
|
|
267
|
-
});
|
|
268
|
-
|
|
269
|
-
test("uses an injected detector instead of the default", () => {
|
|
270
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
271
|
-
let detectCalls = 0;
|
|
272
|
-
// Deterministic detector that always classifies frames as speech.
|
|
273
|
-
const detector = {
|
|
274
|
-
aggressiveness: 2 as const,
|
|
275
|
-
detect(): "speech" | "silence" | "transitioning" {
|
|
276
|
-
detectCalls += 1;
|
|
277
|
-
return "speech";
|
|
278
|
-
},
|
|
279
|
-
};
|
|
280
|
-
const ctrl = createBargeInController({ adapter, detector, triggerFrames: 2 });
|
|
281
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
282
|
-
// Even silent frames count as speech because the injected detector says so.
|
|
283
|
-
expect(ctrl.feedAudioFrame(silentFrame())).toBe("speech");
|
|
284
|
-
expect(ctrl.feedAudioFrame(silentFrame())).toBe("speech");
|
|
285
|
-
expect(detectCalls).toBe(2);
|
|
286
|
-
expect(interrupts).toEqual(["user barged in"]);
|
|
287
|
-
ctrl.stop();
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
test("stop() detaches the adapter listener (later events are ignored)", () => {
|
|
291
|
-
const { adapter, fire, interrupts } = makeStubAdapter();
|
|
292
|
-
const ctrl = createBargeInController({ adapter, triggerFrames: 2 });
|
|
293
|
-
ctrl.stop();
|
|
294
|
-
// After stop(), an audio_chunk must not flip the agent to speaking, so a
|
|
295
|
-
// subsequent speech burst cannot barge in.
|
|
296
|
-
fire({ kind: "audio_chunk", pcm: new Int16Array(0), sampleRate: 24_000 });
|
|
297
|
-
expect(ctrl.isAgentSpeaking()).toBe(false);
|
|
298
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
299
|
-
ctrl.feedAudioFrame(speechFrame());
|
|
300
|
-
expect(interrupts).toHaveLength(0);
|
|
301
|
-
});
|
|
302
|
-
});
|
package/src/index.ts
DELETED
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Catalog R16 `barge-in-controller` — Section 24 VOICE.
|
|
3
|
-
*
|
|
4
|
-
* Coordinator between `vad-engine` and a `RealtimeAdapter`. Watches
|
|
5
|
-
* inbound user audio frames; when the user starts speaking AND the
|
|
6
|
-
* agent is currently playing audio back, fires `adapter.interrupt()`
|
|
7
|
-
* so the agent's response is cancelled and the user's turn takes over.
|
|
8
|
-
*
|
|
9
|
-
* Hysteresis: a single 30ms speech frame is not enough to barge in
|
|
10
|
-
* (rejects a 50ms cough). The controller waits for `triggerFrames`
|
|
11
|
-
* consecutive speech-classified frames within a `windowMs` window
|
|
12
|
-
* before firing. Defaults: 4 frames in 200ms.
|
|
13
|
-
*
|
|
14
|
-
* Playback tracking: the controller listens for `audio_chunk` events on
|
|
15
|
-
* the adapter to mark the agent as "speaking", and for
|
|
16
|
-
* `transcript_final` / `interrupt` / `disconnect` to mark the agent as
|
|
17
|
-
* "idle". Barge-in only fires while the agent is speaking — when it's
|
|
18
|
-
* idle, `feedAudioFrame` is a no-op (the user's normal turn ends with
|
|
19
|
-
* server VAD, no need to interrupt anything).
|
|
20
|
-
*/
|
|
21
|
-
import { type VadDetector, createVadDetector } from "@crewhaus/vad-engine";
|
|
22
|
-
import type { RealtimeAdapter } from "@crewhaus/voice-runtime";
|
|
23
|
-
|
|
24
|
-
export type BargeInOptions = {
|
|
25
|
-
readonly adapter: RealtimeAdapter;
|
|
26
|
-
readonly detector?: VadDetector;
|
|
27
|
-
/**
|
|
28
|
-
* Consecutive speech frames inside `windowMs` to trigger an
|
|
29
|
-
* interrupt. Defaults to 4 (≈ 120ms of continuous speech at 30ms
|
|
30
|
-
* framing).
|
|
31
|
-
*/
|
|
32
|
-
readonly triggerFrames?: number;
|
|
33
|
-
/** Sliding-window bound for the trigger count. Defaults to 200ms. */
|
|
34
|
-
readonly windowMs?: number;
|
|
35
|
-
/** Per-frame elapsed time advances. Defaults to 30ms. */
|
|
36
|
-
readonly frameMs?: number;
|
|
37
|
-
readonly onBargeIn?: (info: { reason: string; speechFrames: number }) => void;
|
|
38
|
-
};
|
|
39
|
-
|
|
40
|
-
export interface BargeInController {
|
|
41
|
-
/** Push one inbound user-mic PCM frame. Returns the VAD verdict for diagnostics. */
|
|
42
|
-
feedAudioFrame(frame: Int16Array): "speech" | "silence" | "transitioning";
|
|
43
|
-
/** Whether the agent is currently speaking (set by adapter `audio_chunk` events). */
|
|
44
|
-
isAgentSpeaking(): boolean;
|
|
45
|
-
/** Detach the adapter listener. */
|
|
46
|
-
stop(): void;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
const DEFAULT_TRIGGER_FRAMES = 4;
|
|
50
|
-
const DEFAULT_WINDOW_MS = 200;
|
|
51
|
-
const DEFAULT_FRAME_MS = 30;
|
|
52
|
-
|
|
53
|
-
export function createBargeInController(opts: BargeInOptions): BargeInController {
|
|
54
|
-
const detector = opts.detector ?? createVadDetector({ aggressiveness: 1 });
|
|
55
|
-
const triggerFrames = opts.triggerFrames ?? DEFAULT_TRIGGER_FRAMES;
|
|
56
|
-
const windowMs = opts.windowMs ?? DEFAULT_WINDOW_MS;
|
|
57
|
-
const frameMs = opts.frameMs ?? DEFAULT_FRAME_MS;
|
|
58
|
-
|
|
59
|
-
let agentSpeaking = false;
|
|
60
|
-
let virtualNowMs = 0;
|
|
61
|
-
// Sliding window of speech frame timestamps.
|
|
62
|
-
const speechFrameStamps: number[] = [];
|
|
63
|
-
|
|
64
|
-
const off = opts.adapter.on((event) => {
|
|
65
|
-
if (event.kind === "audio_chunk") {
|
|
66
|
-
agentSpeaking = true;
|
|
67
|
-
} else if (
|
|
68
|
-
event.kind === "transcript_final" ||
|
|
69
|
-
event.kind === "interrupt" ||
|
|
70
|
-
event.kind === "disconnect"
|
|
71
|
-
) {
|
|
72
|
-
agentSpeaking = false;
|
|
73
|
-
// Reset window so a stray pre-event frame doesn't carry over.
|
|
74
|
-
speechFrameStamps.length = 0;
|
|
75
|
-
}
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
return {
|
|
79
|
-
feedAudioFrame(frame) {
|
|
80
|
-
virtualNowMs += frameMs;
|
|
81
|
-
const verdict = detector.detect(frame);
|
|
82
|
-
if (!agentSpeaking) {
|
|
83
|
-
// No audio playing → no barge-in needed. Still advance the
|
|
84
|
-
// window so it doesn't accumulate stale frames.
|
|
85
|
-
if (verdict === "speech") {
|
|
86
|
-
speechFrameStamps.push(virtualNowMs);
|
|
87
|
-
}
|
|
88
|
-
// Trim window
|
|
89
|
-
const cutoff = virtualNowMs - windowMs;
|
|
90
|
-
while (speechFrameStamps.length > 0 && (speechFrameStamps[0] ?? cutoff) < cutoff) {
|
|
91
|
-
speechFrameStamps.shift();
|
|
92
|
-
}
|
|
93
|
-
return verdict;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
if (verdict === "speech") {
|
|
97
|
-
speechFrameStamps.push(virtualNowMs);
|
|
98
|
-
}
|
|
99
|
-
const cutoff = virtualNowMs - windowMs;
|
|
100
|
-
while (speechFrameStamps.length > 0 && (speechFrameStamps[0] ?? cutoff) < cutoff) {
|
|
101
|
-
speechFrameStamps.shift();
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
if (speechFrameStamps.length >= triggerFrames) {
|
|
105
|
-
const count = speechFrameStamps.length;
|
|
106
|
-
speechFrameStamps.length = 0;
|
|
107
|
-
agentSpeaking = false;
|
|
108
|
-
try {
|
|
109
|
-
opts.adapter.interrupt("user barged in");
|
|
110
|
-
} catch {
|
|
111
|
-
/* tolerate adapter disconnect mid-barge-in */
|
|
112
|
-
}
|
|
113
|
-
opts.onBargeIn?.({ reason: "user barged in", speechFrames: count });
|
|
114
|
-
}
|
|
115
|
-
return verdict;
|
|
116
|
-
},
|
|
117
|
-
isAgentSpeaking() {
|
|
118
|
-
return agentSpeaking;
|
|
119
|
-
},
|
|
120
|
-
stop() {
|
|
121
|
-
off();
|
|
122
|
-
},
|
|
123
|
-
};
|
|
124
|
-
}
|