@kuralle-syrinx/server-websocket 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +23 -4
- package/src/backchannel-cue-fixtures.ts +29 -0
- package/src/background-audio.test.ts +52 -0
- package/src/background-audio.ts +97 -38
- package/src/edge-twilio.ts +3 -2
- package/src/edge.ts +4 -2
- package/src/index.ts +5 -0
- package/src/outbound-playout-pipeline.ts +2 -2
- package/src/smartpbx.ts +1 -0
- package/src/telnyx.ts +1 -0
- package/src/twilio.ts +1 -0
package/package.json
CHANGED
|
@@ -1,9 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/server-websocket",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"
|
|
5
|
+
"description": "Node WebSocket voice host for Syrinx — browser transport plus Twilio/Telnyx/SmartPBX telephony adapters, background audio, admission control",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"voice",
|
|
8
|
+
"voice-agent",
|
|
9
|
+
"speech",
|
|
10
|
+
"syrinx",
|
|
11
|
+
"telephony",
|
|
12
|
+
"twilio",
|
|
13
|
+
"telnyx"
|
|
14
|
+
],
|
|
6
15
|
"license": "MIT",
|
|
16
|
+
"homepage": "https://github.com/kuralle/syrinx#readme",
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "git+https://github.com/kuralle/syrinx.git",
|
|
20
|
+
"directory": "packages/server-websocket"
|
|
21
|
+
},
|
|
22
|
+
"bugs": {
|
|
23
|
+
"url": "https://github.com/kuralle/syrinx/issues"
|
|
24
|
+
},
|
|
25
|
+
"type": "module",
|
|
7
26
|
"main": "./src/index.ts",
|
|
8
27
|
"types": "./src/index.ts",
|
|
9
28
|
"exports": {
|
|
@@ -15,8 +34,8 @@
|
|
|
15
34
|
"dependencies": {
|
|
16
35
|
"@evan/opus": "1.0.3",
|
|
17
36
|
"ws": "^8.18.0",
|
|
18
|
-
"@kuralle-syrinx/core": "4.
|
|
19
|
-
"@kuralle-syrinx/ws": "4.
|
|
37
|
+
"@kuralle-syrinx/core": "4.2.0",
|
|
38
|
+
"@kuralle-syrinx/ws": "4.2.0"
|
|
20
39
|
},
|
|
21
40
|
"devDependencies": {
|
|
22
41
|
"@types/node": "^22.0.0",
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Deterministic placeholder backchannel cue PCM (16 kHz mono). Real voice-matched
|
|
4
|
+
// assets + locale table are a follow-up — these shaped tones prove the byte-config path.
|
|
5
|
+
|
|
6
|
+
import { synthesizeTonePcm16 } from "@kuralle-syrinx/core";
|
|
7
|
+
import { pcm16BytesToSamples } from "@kuralle-syrinx/core/audio";
|
|
8
|
+
import type { BackgroundAudioSource } from "./background-audio.js";
|
|
9
|
+
|
|
10
|
+
const CUE_SAMPLE_RATE_HZ = 16_000;
|
|
11
|
+
|
|
12
|
+
function placeholderCue(frequencyHz: number, durationMs: number, gain: number): BackgroundAudioSource {
|
|
13
|
+
const bytes = synthesizeTonePcm16({
|
|
14
|
+
frequencyHz,
|
|
15
|
+
durationMs,
|
|
16
|
+
sampleRateHz: CUE_SAMPLE_RATE_HZ,
|
|
17
|
+
amplitude: 0.35,
|
|
18
|
+
});
|
|
19
|
+
return { pcm: pcm16BytesToSamples(bytes), sampleRateHz: CUE_SAMPLE_RATE_HZ, gain };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Placeholder cue map — pass as `backgroundAudio.cues` at session init. */
|
|
23
|
+
export function buildPlaceholderBackchannelCues(): Readonly<Record<string, BackgroundAudioSource>> {
|
|
24
|
+
return {
|
|
25
|
+
mm_hmm: placeholderCue(440, 400, 0.65),
|
|
26
|
+
yeah: placeholderCue(520, 350, 0.6),
|
|
27
|
+
got_it: placeholderCue(380, 450, 0.6),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -160,6 +160,58 @@ describe("BackgroundAudioMixer", () => {
|
|
|
160
160
|
expect(mixer.idleFrame(1, 16000, t0 + 300)).toBeNull(); // episode fully over
|
|
161
161
|
});
|
|
162
162
|
|
|
163
|
+
it("IP-C3: plays a one-shot cue in an idle frame with thinking ducked under it", () => {
|
|
164
|
+
const ramp = Int16Array.from({ length: 2000 }, (_, i) => (i % 2 === 0 ? 800 : -800));
|
|
165
|
+
const cue = Int16Array.from({ length: 160 }, (_, i) => Math.round(6000 * Math.sin((2 * Math.PI * i) / 160)));
|
|
166
|
+
const mixer = new BackgroundAudioMixer({
|
|
167
|
+
thinking: { pcm: ramp, sampleRateHz: 16000, gain: 1 },
|
|
168
|
+
cues: { mm_hmm: { pcm: cue, sampleRateHz: 16000, gain: 1 } },
|
|
169
|
+
fadeMs: 0,
|
|
170
|
+
});
|
|
171
|
+
const t0 = 12_000_000;
|
|
172
|
+
mixer.setThinking(true);
|
|
173
|
+
expect(mixer.queueCue("mm_hmm")).toBe(true);
|
|
174
|
+
|
|
175
|
+
const frame = mixer.idleFrame(10, 16000, t0)!;
|
|
176
|
+
const samples = samplesOf(frame);
|
|
177
|
+
expect(samples.some((s) => Math.abs(s) > 1000)).toBe(true);
|
|
178
|
+
expect(samples.every((s) => s <= 32767 && s >= -32768)).toBe(true);
|
|
179
|
+
|
|
180
|
+
mixer.setThinking(true);
|
|
181
|
+
expect(mixer.queueCue("missing")).toBe(false);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it("IP-C3: thinking loop continues before and after a one-shot cue", () => {
|
|
185
|
+
const thinking = Int16Array.from({ length: 1000 }, (_, i) => i);
|
|
186
|
+
const cue = new Int16Array(32).fill(5000);
|
|
187
|
+
const mixer = new BackgroundAudioMixer({
|
|
188
|
+
thinking: { pcm: thinking, sampleRateHz: 16000, gain: 1 },
|
|
189
|
+
cues: { mm_hmm: { pcm: cue, sampleRateHz: 16000, gain: 1 } },
|
|
190
|
+
fadeMs: 0,
|
|
191
|
+
});
|
|
192
|
+
const t0 = 13_000_000;
|
|
193
|
+
mixer.setThinking(true);
|
|
194
|
+
const before = samplesOf(mixer.idleFrame(1, 16000, t0)!);
|
|
195
|
+
expect(before[0]).toBe(0);
|
|
196
|
+
|
|
197
|
+
mixer.queueCue("mm_hmm");
|
|
198
|
+
mixer.idleFrame(1, 16000, t0 + 100);
|
|
199
|
+
const after = samplesOf(mixer.idleFrame(1, 16000, t0 + 200)!);
|
|
200
|
+
expect(after[0]).toBeGreaterThan(0);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it("IP-C3: resolves cues from config bytes without filesystem access", () => {
|
|
204
|
+
const cuePcm = new Int16Array(80).fill(1200);
|
|
205
|
+
const mixer = new BackgroundAudioMixer({
|
|
206
|
+
cues: { mm_hmm: { pcm: cuePcm, sampleRateHz: 16000, gain: 0.8 } },
|
|
207
|
+
fadeMs: 0,
|
|
208
|
+
});
|
|
209
|
+
expect(mixer.hasCue("mm_hmm")).toBe(true);
|
|
210
|
+
expect(mixer.queueCue("mm_hmm")).toBe(true);
|
|
211
|
+
const frame = mixer.idleFrame(5, 16000, 14_000_000)!;
|
|
212
|
+
expect(samplesOf(frame).some((s) => s !== 0)).toBe(true);
|
|
213
|
+
});
|
|
214
|
+
|
|
163
215
|
it("fades each new thinking episode in from silence", () => {
|
|
164
216
|
const mixer = new BackgroundAudioMixer({
|
|
165
217
|
thinking: { pcm: new Int16Array(1000).fill(1000), sampleRateHz: 16000, gain: 1 },
|
package/src/background-audio.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Background audio for the outbound path: a looped ambient bed (which doubles
|
|
4
4
|
// as telephony comfort noise between turns), an optional "thinking" loop keyed
|
|
5
|
-
// off the G3 tool-call cues, and ducking under
|
|
5
|
+
// off the G3 tool-call cues, one-shot backchannel cues, and ducking under speech.
|
|
6
6
|
//
|
|
7
7
|
// Syrinx wires (Twilio media stream, browser jitter buffer) are single ORDERED
|
|
8
8
|
// streams — unlike LiveKit's WebRTC rooms there is no client-side track mixing
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
// Prior art: Pipecat SoundfileMixer (transport-level mix + volume + loop),
|
|
16
16
|
// LiveKit BackgroundAudioPlayer (ambient/thinking split, state-keyed thinking).
|
|
17
17
|
|
|
18
|
-
import type { VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
18
|
+
import type { InteractionBackchannelPacket, VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
19
|
+
import { Route } from "@kuralle-syrinx/core";
|
|
19
20
|
import { pcm16BytesToSamples, pcm16SamplesToBytes, resamplePcm16 } from "@kuralle-syrinx/core/audio";
|
|
20
21
|
|
|
21
22
|
export interface BackgroundAudioSource {
|
|
@@ -31,6 +32,8 @@ export interface BackgroundAudioConfig {
|
|
|
31
32
|
readonly ambient?: BackgroundAudioSource;
|
|
32
33
|
/** Looped while `setThinking(true)` — the audible face of a pending tool call. */
|
|
33
34
|
readonly thinking?: BackgroundAudioSource;
|
|
35
|
+
/** Pre-cached one-shot backchannel cues keyed by cue id (IP-C3). */
|
|
36
|
+
readonly cues?: Readonly<Record<string, BackgroundAudioSource>>;
|
|
34
37
|
/**
|
|
35
38
|
* Extra multiplier on background sources while assistant speech is being
|
|
36
39
|
* mixed, so the bed never muddies the voice. 1 disables ducking. @default 0.5
|
|
@@ -48,8 +51,10 @@ export interface BackgroundAudioConfig {
|
|
|
48
51
|
|
|
49
52
|
const DEFAULT_AMBIENT_GAIN = 0.25;
|
|
50
53
|
const DEFAULT_THINKING_GAIN = 0.4;
|
|
54
|
+
const DEFAULT_CUE_GAIN = 0.65;
|
|
51
55
|
const DEFAULT_DUCK = 0.5;
|
|
52
56
|
const DEFAULT_FADE_MS = 250;
|
|
57
|
+
const CUE_THINKING_DUCK = 0.25;
|
|
53
58
|
|
|
54
59
|
interface LoopedSource {
|
|
55
60
|
readonly original: BackgroundAudioSource;
|
|
@@ -61,6 +66,14 @@ interface LoopedSource {
|
|
|
61
66
|
positionRateHz: number;
|
|
62
67
|
}
|
|
63
68
|
|
|
69
|
+
interface OneShotSource {
|
|
70
|
+
readonly original: BackgroundAudioSource;
|
|
71
|
+
readonly gain: number;
|
|
72
|
+
readonly byRate: Map<number, Int16Array>;
|
|
73
|
+
position: number;
|
|
74
|
+
positionRateHz: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
64
77
|
function loopedSource(source: BackgroundAudioSource, defaultGain: number): LoopedSource {
|
|
65
78
|
return {
|
|
66
79
|
original: source,
|
|
@@ -71,14 +84,26 @@ function loopedSource(source: BackgroundAudioSource, defaultGain: number): Loope
|
|
|
71
84
|
};
|
|
72
85
|
}
|
|
73
86
|
|
|
74
|
-
function
|
|
87
|
+
function oneShotSource(source: BackgroundAudioSource, defaultGain: number): OneShotSource {
|
|
88
|
+
return {
|
|
89
|
+
original: source,
|
|
90
|
+
gain: source.gain ?? defaultGain,
|
|
91
|
+
byRate: new Map(),
|
|
92
|
+
position: 0,
|
|
93
|
+
positionRateHz: source.sampleRateHz,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function samplesAtRate(
|
|
98
|
+
source: LoopedSource | OneShotSource,
|
|
99
|
+
rateHz: number,
|
|
100
|
+
): Int16Array {
|
|
75
101
|
let samples = source.byRate.get(rateHz);
|
|
76
102
|
if (!samples) {
|
|
77
103
|
samples = resamplePcm16(source.original.pcm, source.original.sampleRateHz, rateHz);
|
|
78
104
|
source.byRate.set(rateHz, samples);
|
|
79
105
|
}
|
|
80
106
|
if (source.positionRateHz !== rateHz) {
|
|
81
|
-
// Wire rate changed mid-connection (rare): carry the position over proportionally.
|
|
82
107
|
source.position = Math.round((source.position * rateHz) / source.positionRateHz);
|
|
83
108
|
source.positionRateHz = rateHz;
|
|
84
109
|
}
|
|
@@ -108,6 +133,22 @@ function addLooped(
|
|
|
108
133
|
source.position = pos;
|
|
109
134
|
}
|
|
110
135
|
|
|
136
|
+
function addOneShot(target: Float64Array, source: OneShotSource, rateHz: number): boolean {
|
|
137
|
+
const samples = samplesAtRate(source, rateHz);
|
|
138
|
+
if (samples.length === 0 || source.gain === 0) return true;
|
|
139
|
+
let pos = source.position;
|
|
140
|
+
for (let i = 0; i < target.length; i += 1) {
|
|
141
|
+
if (pos >= samples.length) {
|
|
142
|
+
source.position = samples.length;
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
target[i]! += samples[pos]! * source.gain;
|
|
146
|
+
pos += 1;
|
|
147
|
+
}
|
|
148
|
+
source.position = pos;
|
|
149
|
+
return pos >= samples.length;
|
|
150
|
+
}
|
|
151
|
+
|
|
111
152
|
/** Equal-power fade-in gain for sample `n` of a `total`-sample ramp (1 past the ramp). */
|
|
112
153
|
function fadeInGain(n: number, total: number): number {
|
|
113
154
|
if (total <= 0 || n >= total) return 1;
|
|
@@ -132,59 +173,65 @@ function clipToPcm16Bytes(mix: Float64Array): Uint8Array {
|
|
|
132
173
|
export class BackgroundAudioMixer {
|
|
133
174
|
private readonly ambient: LoopedSource | null;
|
|
134
175
|
private readonly thinking: LoopedSource | null;
|
|
176
|
+
private readonly cueCatalog: ReadonlyMap<string, BackgroundAudioSource>;
|
|
135
177
|
private readonly duck: number;
|
|
136
178
|
private readonly fadeMs: number;
|
|
137
|
-
/**
|
|
138
|
-
* Thinking episode lifecycle: "stopping" keeps the loop audible for one
|
|
139
|
-
* fade-out ramp after `setThinking(false)` instead of hard-cutting it.
|
|
140
|
-
*/
|
|
141
179
|
private thinkingState: "off" | "on" | "stopping" = "off";
|
|
142
|
-
/** Samples rendered since the current thinking episode started (fade-in). */
|
|
143
180
|
private thinkingFadeInPos = 0;
|
|
144
|
-
/** Samples rendered since the stop was requested (fade-out). */
|
|
145
181
|
private thinkingFadeOutPos = 0;
|
|
146
|
-
/** Samples of ambient rendered so far (one-time fade-in at bed start). */
|
|
147
182
|
private ambientFadeInPos = 0;
|
|
148
|
-
/** Realtime estimate of when already-mixed speech finishes playing out. */
|
|
149
183
|
private speakingUntilMs = 0;
|
|
184
|
+
private pendingCue: OneShotSource | null = null;
|
|
150
185
|
|
|
151
186
|
constructor(config: BackgroundAudioConfig) {
|
|
152
187
|
this.ambient = config.ambient ? loopedSource(config.ambient, DEFAULT_AMBIENT_GAIN) : null;
|
|
153
188
|
this.thinking = config.thinking ? loopedSource(config.thinking, DEFAULT_THINKING_GAIN) : null;
|
|
189
|
+
this.cueCatalog = new Map(Object.entries(config.cues ?? {}));
|
|
154
190
|
this.duck = config.duckWhileSpeaking ?? DEFAULT_DUCK;
|
|
155
191
|
this.fadeMs = config.fadeMs ?? DEFAULT_FADE_MS;
|
|
156
192
|
}
|
|
157
193
|
|
|
158
194
|
get hasSources(): boolean {
|
|
159
|
-
return this.ambient !== null || this.thinking !== null;
|
|
195
|
+
return this.ambient !== null || this.thinking !== null || this.cueCatalog.size > 0;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
hasCue(cueId: string): boolean {
|
|
199
|
+
return this.cueCatalog.has(cueId);
|
|
160
200
|
}
|
|
161
201
|
|
|
162
202
|
isSpeaking(nowMs = Date.now()): boolean {
|
|
163
203
|
return nowMs < this.speakingUntilMs;
|
|
164
204
|
}
|
|
165
205
|
|
|
206
|
+
/** Queue a one-shot backchannel cue by id. Returns false when the cue is unknown. */
|
|
207
|
+
queueCue(cueId: string): boolean {
|
|
208
|
+
const source = this.cueCatalog.get(cueId);
|
|
209
|
+
if (!source) return false;
|
|
210
|
+
this.pendingCue = oneShotSource(source, DEFAULT_CUE_GAIN);
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
|
|
166
214
|
/** G3 cue wiring: started/delayed → true, complete/failed → false. */
|
|
167
215
|
setThinking(on: boolean): void {
|
|
168
216
|
if (on) {
|
|
169
217
|
if (this.thinkingState === "on") return;
|
|
170
218
|
this.thinkingState = "on";
|
|
171
219
|
this.thinkingFadeInPos = 0;
|
|
172
|
-
if (this.thinking) this.thinking.position = 0;
|
|
220
|
+
if (this.thinking) this.thinking.position = 0;
|
|
173
221
|
return;
|
|
174
222
|
}
|
|
175
223
|
if (this.thinkingState !== "on") return;
|
|
176
|
-
// Audible for one fade-out ramp, then off (immediately off when fades are disabled).
|
|
177
224
|
this.thinkingState = this.fadeMs > 0 ? "stopping" : "off";
|
|
178
225
|
this.thinkingFadeOutPos = 0;
|
|
179
226
|
}
|
|
180
227
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
*/
|
|
228
|
+
private thinkingScale(bedScale: number): number {
|
|
229
|
+
return this.pendingCue ? bedScale * CUE_THINKING_DUCK : bedScale;
|
|
230
|
+
}
|
|
231
|
+
|
|
186
232
|
private addBed(mixBuf: Float64Array, sampleRateHz: number, bedScale: number): void {
|
|
187
233
|
const fadeSamples = Math.round((this.fadeMs / 1000) * sampleRateHz);
|
|
234
|
+
const thinkingScale = this.thinkingScale(bedScale);
|
|
188
235
|
|
|
189
236
|
if (this.ambient) {
|
|
190
237
|
const startPos = this.ambientFadeInPos;
|
|
@@ -201,7 +248,7 @@ export class BackgroundAudioMixer {
|
|
|
201
248
|
const envelope = startPos >= fadeSamples
|
|
202
249
|
? undefined
|
|
203
250
|
: (i: number) => fadeInGain(startPos + i, fadeSamples);
|
|
204
|
-
addLooped(mixBuf, this.thinking, sampleRateHz, this.thinking.gain *
|
|
251
|
+
addLooped(mixBuf, this.thinking, sampleRateHz, this.thinking.gain * thinkingScale, envelope);
|
|
205
252
|
this.thinkingFadeInPos = Math.min(fadeSamples, startPos + mixBuf.length);
|
|
206
253
|
} else {
|
|
207
254
|
const startPos = this.thinkingFadeOutPos;
|
|
@@ -209,7 +256,7 @@ export class BackgroundAudioMixer {
|
|
|
209
256
|
mixBuf,
|
|
210
257
|
this.thinking,
|
|
211
258
|
sampleRateHz,
|
|
212
|
-
this.thinking.gain *
|
|
259
|
+
this.thinking.gain * thinkingScale,
|
|
213
260
|
(i: number) => fadeOutGain(startPos + i, fadeSamples),
|
|
214
261
|
);
|
|
215
262
|
this.thinkingFadeOutPos = startPos + mixBuf.length;
|
|
@@ -218,46 +265,58 @@ export class BackgroundAudioMixer {
|
|
|
218
265
|
}
|
|
219
266
|
}
|
|
220
267
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
268
|
+
private addPendingCue(mixBuf: Float64Array, sampleRateHz: number): void {
|
|
269
|
+
if (!this.pendingCue) return;
|
|
270
|
+
const finished = addOneShot(mixBuf, this.pendingCue, sampleRateHz);
|
|
271
|
+
if (finished) this.pendingCue = null;
|
|
272
|
+
}
|
|
273
|
+
|
|
226
274
|
mix(chunk: Uint8Array, sampleRateHz: number, nowMs = Date.now()): Uint8Array {
|
|
227
275
|
const speech = pcm16BytesToSamples(chunk);
|
|
228
276
|
const durationMs = (speech.length / sampleRateHz) * 1000;
|
|
229
277
|
this.speakingUntilMs = Math.max(this.speakingUntilMs, nowMs) + durationMs;
|
|
230
278
|
|
|
231
|
-
if (!this.hasSources) return chunk;
|
|
279
|
+
if (!this.hasSources && !this.pendingCue) return chunk;
|
|
232
280
|
const mixBuf = new Float64Array(speech.length);
|
|
233
281
|
for (let i = 0; i < speech.length; i += 1) mixBuf[i] = speech[i]!;
|
|
234
282
|
this.addBed(mixBuf, sampleRateHz, this.duck);
|
|
235
283
|
return clipToPcm16Bytes(mixBuf);
|
|
236
284
|
}
|
|
237
285
|
|
|
238
|
-
/**
|
|
239
|
-
* Bed-only frame for the gaps between turns (telephony comfort noise), or
|
|
240
|
-
* null while speech is still playing out / there is nothing to play.
|
|
241
|
-
*/
|
|
242
286
|
idleFrame(frameMs: number, sampleRateHz: number, nowMs = Date.now()): Uint8Array | null {
|
|
243
287
|
if (this.isSpeaking(nowMs)) return null;
|
|
244
288
|
const playThinking = this.thinking !== null && this.thinkingState !== "off";
|
|
245
|
-
if (!this.ambient && !playThinking) return null;
|
|
289
|
+
if (!this.ambient && !playThinking && !this.pendingCue) return null;
|
|
246
290
|
|
|
247
291
|
const sampleCount = Math.max(1, Math.round((frameMs / 1000) * sampleRateHz));
|
|
248
292
|
const mixBuf = new Float64Array(sampleCount);
|
|
249
293
|
this.addBed(mixBuf, sampleRateHz, 1);
|
|
294
|
+
this.addPendingCue(mixBuf, sampleRateHz);
|
|
250
295
|
return clipToPcm16Bytes(mixBuf);
|
|
251
296
|
}
|
|
252
297
|
}
|
|
253
298
|
|
|
254
|
-
/**
|
|
255
|
-
* Drive the mixer's thinking loop from the session's G3 tool-call cues:
|
|
256
|
-
* started/delayed → thinking on; complete/failed → off. Listener lifetime is
|
|
257
|
-
* the session's — both are per-connection.
|
|
258
|
-
*/
|
|
259
299
|
export function wireBackgroundThinking(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
|
|
260
300
|
session.on("tool_call_cue", (event) => {
|
|
261
301
|
mixer.setThinking(event.phase === "started" || event.phase === "delayed");
|
|
262
302
|
});
|
|
263
303
|
}
|
|
304
|
+
|
|
305
|
+
export function wireBackgroundBackchannel(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
|
|
306
|
+
session.bus.on("interaction.backchannel", (pkt) => {
|
|
307
|
+
const backchannel = pkt as InteractionBackchannelPacket;
|
|
308
|
+
if (mixer.queueCue(backchannel.cue)) return;
|
|
309
|
+
session.bus.push(Route.Background, {
|
|
310
|
+
kind: "metric.conversation",
|
|
311
|
+
contextId: backchannel.contextId,
|
|
312
|
+
timestampMs: backchannel.timestampMs,
|
|
313
|
+
name: "backchannel.suppressed_missing_asset",
|
|
314
|
+
value: backchannel.cue,
|
|
315
|
+
});
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
export function wireBackgroundAudio(session: VoiceAgentSession, mixer: BackgroundAudioMixer): void {
|
|
320
|
+
wireBackgroundThinking(session, mixer);
|
|
321
|
+
wireBackgroundBackchannel(session, mixer);
|
|
322
|
+
}
|
package/src/edge-twilio.ts
CHANGED
|
@@ -28,7 +28,7 @@ import {
|
|
|
28
28
|
import type { ManagedSocket, SocketData } from "@kuralle-syrinx/ws";
|
|
29
29
|
import {
|
|
30
30
|
BackgroundAudioMixer,
|
|
31
|
-
|
|
31
|
+
wireBackgroundAudio,
|
|
32
32
|
type BackgroundAudioConfig,
|
|
33
33
|
} from "./background-audio.js";
|
|
34
34
|
import {
|
|
@@ -208,7 +208,7 @@ export async function runTwilioEdgeWebSocketConnection(
|
|
|
208
208
|
? new BackgroundAudioMixer(options.backgroundAudio)
|
|
209
209
|
: null;
|
|
210
210
|
if (backgroundAudio) {
|
|
211
|
-
|
|
211
|
+
wireBackgroundAudio(session, backgroundAudio);
|
|
212
212
|
const idleFrameMs = options.backgroundIdleFrameMs ?? 200;
|
|
213
213
|
const idleTimer = setInterval(() => {
|
|
214
214
|
if (!streamSid || stopped || closed || !socket.isOpen) return;
|
|
@@ -296,6 +296,7 @@ export async function runTwilioEdgeWebSocketConnection(
|
|
|
296
296
|
contextId,
|
|
297
297
|
timestampMs: Date.now(),
|
|
298
298
|
audio: pcm16SamplesToBytes(pcm16k),
|
|
299
|
+
sampleRateHz: engineRate,
|
|
299
300
|
});
|
|
300
301
|
return;
|
|
301
302
|
}
|
package/src/edge.ts
CHANGED
|
@@ -16,7 +16,7 @@ import { TimerScheduler, type Scheduler } from "@kuralle-syrinx/core";
|
|
|
16
16
|
import { type StreamingPcm16Resampler } from "@kuralle-syrinx/core/audio";
|
|
17
17
|
import {
|
|
18
18
|
BackgroundAudioMixer,
|
|
19
|
-
|
|
19
|
+
wireBackgroundAudio,
|
|
20
20
|
type BackgroundAudioConfig,
|
|
21
21
|
} from "./background-audio.js";
|
|
22
22
|
export type { BackgroundAudioConfig, BackgroundAudioSource } from "./background-audio.js";
|
|
@@ -370,7 +370,7 @@ function wireEdgeSessionEvents(
|
|
|
370
370
|
recorder?: EdgeRecorder,
|
|
371
371
|
backgroundAudio?: BackgroundAudioMixer,
|
|
372
372
|
): void {
|
|
373
|
-
if (backgroundAudio)
|
|
373
|
+
if (backgroundAudio) wireBackgroundAudio(session, backgroundAudio);
|
|
374
374
|
const onSession = <K extends keyof VoiceAgentSessionEvents>(
|
|
375
375
|
event: K,
|
|
376
376
|
handler: VoiceAgentSessionEvents[K],
|
|
@@ -494,6 +494,7 @@ function handleClientMessage(
|
|
|
494
494
|
contextId: nextContextId,
|
|
495
495
|
timestampMs: Date.now(),
|
|
496
496
|
audio,
|
|
497
|
+
sampleRateHz: inputSampleRateHz,
|
|
497
498
|
} satisfies UserAudioReceivedPacket);
|
|
498
499
|
return nextContextId;
|
|
499
500
|
}
|
|
@@ -557,6 +558,7 @@ function handleClientMessage(
|
|
|
557
558
|
contextId: nextContextId,
|
|
558
559
|
timestampMs: Date.now(),
|
|
559
560
|
audio,
|
|
561
|
+
sampleRateHz: inputSampleRateHz,
|
|
560
562
|
} satisfies UserAudioReceivedPacket);
|
|
561
563
|
return nextContextId;
|
|
562
564
|
}
|
package/src/index.ts
CHANGED
|
@@ -63,9 +63,12 @@ export { validateTwilioSignature } from "./twilio-auth.js";
|
|
|
63
63
|
export {
|
|
64
64
|
BackgroundAudioMixer,
|
|
65
65
|
wireBackgroundThinking,
|
|
66
|
+
wireBackgroundBackchannel,
|
|
67
|
+
wireBackgroundAudio,
|
|
66
68
|
type BackgroundAudioConfig,
|
|
67
69
|
type BackgroundAudioSource,
|
|
68
70
|
} from "./background-audio.js";
|
|
71
|
+
export { buildPlaceholderBackchannelCues } from "./backchannel-cue-fixtures.js";
|
|
69
72
|
export { installGracefulShutdown, type GracefulClosable } from "./websocket-lifecycle.js";
|
|
70
73
|
|
|
71
74
|
export interface VoiceWebSocketServerOptions {
|
|
@@ -711,6 +714,7 @@ function handleClientMessage(
|
|
|
711
714
|
contextId: nextContextId,
|
|
712
715
|
timestampMs: Date.now(),
|
|
713
716
|
audio,
|
|
717
|
+
sampleRateHz: inputSampleRateHz,
|
|
714
718
|
} satisfies UserAudioReceivedPacket);
|
|
715
719
|
return nextContextId;
|
|
716
720
|
}
|
|
@@ -770,6 +774,7 @@ function handleClientMessage(
|
|
|
770
774
|
contextId: nextContextId,
|
|
771
775
|
timestampMs: Date.now(),
|
|
772
776
|
audio: resampleAudioBytes(decodeStrictBase64(message.audio, "audio"), sourceSampleRateHz, inputSampleRateHz, streamingResamplers),
|
|
777
|
+
sampleRateHz: inputSampleRateHz,
|
|
773
778
|
} satisfies UserAudioReceivedPacket);
|
|
774
779
|
return nextContextId;
|
|
775
780
|
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import { Route, type InterruptTtsPacket, type TextToSpeechAudioPacket, type TextToSpeechEndPacket, type VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
4
4
|
import { WebSocket } from "ws";
|
|
5
5
|
import type { BackgroundAudioMixer } from "./background-audio.js";
|
|
6
|
-
import {
|
|
6
|
+
import { wireBackgroundAudio } from "./background-audio.js";
|
|
7
7
|
import type { PacedPlayoutFrame } from "./paced-playout.js";
|
|
8
8
|
import { PacedPlayoutQueue } from "./paced-playout.js";
|
|
9
9
|
import { PlayoutProgressEmitter } from "./playout-progress.js";
|
|
@@ -81,7 +81,7 @@ export function wireTelephonyOutboundPipeline(args: {
|
|
|
81
81
|
readonly backgroundAudio?: BackgroundAudioMixer;
|
|
82
82
|
}): TelephonyOutboundHandle {
|
|
83
83
|
const { session, socket, disposers, outboundFrameDurationMs, maxQueuedOutputAudioMs, callbacks, backgroundAudio } = args;
|
|
84
|
-
if (backgroundAudio)
|
|
84
|
+
if (backgroundAudio) wireBackgroundAudio(session, backgroundAudio);
|
|
85
85
|
|
|
86
86
|
const recordDiscardedPlayout = (discardedMs: number, reason: string): void => {
|
|
87
87
|
if (discardedMs <= 0) return;
|
package/src/smartpbx.ts
CHANGED
package/src/telnyx.ts
CHANGED