@kuralle-syrinx/server-websocket 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -5
- package/src/carrier-commands.ts +332 -0
- package/src/edge-telnyx.ts +389 -0
- package/src/edge.ts +23 -2
- package/src/index.ts +25 -2
- package/src/telnyx-codec.ts +163 -0
- package/src/telnyx.ts +54 -59
- package/src/turn-metrics.ts +107 -5
- package/src/twilio.ts +23 -0
- package/src/wire-carrier-control.ts +179 -0
- package/src/admission-control.test.ts +0 -270
- package/src/background-audio.test.ts +0 -224
- package/src/browser-opus.test.ts +0 -41
- package/src/browser-pacing.test.ts +0 -440
- package/src/edge-twilio.test.ts +0 -293
- package/src/edge.test.ts +0 -591
- package/src/graceful-drain.test.ts +0 -306
- package/src/inbound-audio.test.ts +0 -58
- package/src/index.test.ts +0 -2074
- package/src/outbound-playout-pipeline.test.ts +0 -314
- package/src/paced-playout.test.ts +0 -247
- package/src/playout-progress.test.ts +0 -56
- package/src/session-store.test.ts +0 -274
- package/src/smartpbx.test.ts +0 -967
- package/src/telnyx.test.ts +0 -1457
- package/src/transport-host.test.ts +0 -51
- package/src/turn-metrics.test.ts +0 -327
- package/src/twilio-auth.test.ts +0 -36
- package/src/twilio.test.ts +0 -1275
- package/src/websocket-close.test.ts +0 -63
- package/src/websocket-lifecycle.test.ts +0 -49
- package/tsconfig.json +0 -22
- package/vitest.config.ts +0 -7
package/src/telnyx.ts
CHANGED
|
@@ -5,12 +5,9 @@ import { createServer, type Server as HttpServer } from "node:http";
|
|
|
5
5
|
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
|
6
6
|
import { Route, type VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
7
7
|
import {
|
|
8
|
-
|
|
9
|
-
decodeMuLawToPcm16,
|
|
10
|
-
encodePcm16ToMuLaw,
|
|
8
|
+
createG722DecoderState,
|
|
11
9
|
pcm16BytesToSamples,
|
|
12
10
|
pcm16SamplesToBytes,
|
|
13
|
-
pcm16SamplesToBigEndianBytes,
|
|
14
11
|
resamplePcm16Streaming,
|
|
15
12
|
type StreamingPcm16Resampler,
|
|
16
13
|
} from "@kuralle-syrinx/core/audio";
|
|
@@ -30,12 +27,28 @@ import { wireTelephonyOutboundPipeline, installTelephonyTurnRotation } from "./o
|
|
|
30
27
|
import {
|
|
31
28
|
decodeStrictBase64,
|
|
32
29
|
nonNegativeInteger,
|
|
33
|
-
numberFromString,
|
|
34
30
|
optionalNonNegativeIntegerString,
|
|
35
31
|
optionalPositiveIntegerString,
|
|
36
32
|
positiveInteger,
|
|
37
33
|
rawDataToText,
|
|
38
34
|
} from "./transport-helpers.js";
|
|
35
|
+
import type { FetchLike, TelnyxRestCredentials, WarmTransferSummarizer } from "./carrier-commands.js";
|
|
36
|
+
import { wireCarrierControl } from "./wire-carrier-control.js";
|
|
37
|
+
import {
|
|
38
|
+
createTelnyxG722State,
|
|
39
|
+
decodeTelnyxInboundPayload,
|
|
40
|
+
defaultTelnyxContextId,
|
|
41
|
+
encodeTelnyxOutboundPayload,
|
|
42
|
+
splitTelnyxEncodedFrames,
|
|
43
|
+
validateTelnyxStart,
|
|
44
|
+
wireSampleRateForCodec,
|
|
45
|
+
type TelnyxCodec,
|
|
46
|
+
type TelnyxG722State,
|
|
47
|
+
type TelnyxStartPayload,
|
|
48
|
+
} from "./telnyx-codec.js";
|
|
49
|
+
|
|
50
|
+
export type { TelnyxCodec, TelnyxStartPayload } from "./telnyx-codec.js";
|
|
51
|
+
export { validateTelnyxStart, defaultTelnyxContextId } from "./telnyx-codec.js";
|
|
39
52
|
|
|
40
53
|
export interface TelnyxMediaStreamServerOptions {
|
|
41
54
|
readonly server?: HttpServer;
|
|
@@ -47,7 +60,7 @@ export interface TelnyxMediaStreamServerOptions {
|
|
|
47
60
|
readonly inputSampleRateHz?: number;
|
|
48
61
|
readonly outputSampleRateHz?: number;
|
|
49
62
|
/** Must match the `stream_bidirectional_codec` selected when starting the Telnyx call stream. */
|
|
50
|
-
readonly bidirectionalCodec?:
|
|
63
|
+
readonly bidirectionalCodec?: TelnyxCodec;
|
|
51
64
|
readonly outboundFrameDurationMs?: number;
|
|
52
65
|
readonly maxQueuedOutputAudioMs?: number;
|
|
53
66
|
/** Ambient/thinking bed mixed (ducked) under assistant speech (see BackgroundAudioConfig). */
|
|
@@ -61,6 +74,15 @@ export interface TelnyxMediaStreamServerOptions {
|
|
|
61
74
|
readonly maxConcurrentSessions?: number;
|
|
62
75
|
readonly maxConcurrentSessionsScope?: "path" | "server";
|
|
63
76
|
readonly onTransportMetric?: (name: string) => void;
|
|
77
|
+
/**
|
|
78
|
+
* Injected Call Control credentials for dtmf.send / call.transfer dispatch.
|
|
79
|
+
* Workers: pass secrets from `env`. Never read from process.env here.
|
|
80
|
+
* Live dispatch unverified without credentials.
|
|
81
|
+
*/
|
|
82
|
+
readonly callControlCredentials?: TelnyxRestCredentials;
|
|
83
|
+
readonly callControlFetch?: FetchLike;
|
|
84
|
+
readonly callControlWebhookUrl?: string;
|
|
85
|
+
readonly warmTransferSummarizer?: WarmTransferSummarizer;
|
|
64
86
|
}
|
|
65
87
|
|
|
66
88
|
export interface TelnyxMediaStreamServer {
|
|
@@ -70,17 +92,6 @@ export interface TelnyxMediaStreamServer {
|
|
|
70
92
|
close(opts?: GracefulCloseOptions): Promise<void>;
|
|
71
93
|
}
|
|
72
94
|
|
|
73
|
-
export interface TelnyxStartPayload {
|
|
74
|
-
readonly stream_id?: string;
|
|
75
|
-
readonly call_control_id?: string;
|
|
76
|
-
readonly call_session_id?: string;
|
|
77
|
-
readonly media_format?: {
|
|
78
|
-
readonly encoding?: string;
|
|
79
|
-
readonly sample_rate?: number | string;
|
|
80
|
-
readonly channels?: number | string;
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
|
|
84
95
|
interface TelnyxMediaMessage {
|
|
85
96
|
readonly event?: string;
|
|
86
97
|
readonly stream_id?: string;
|
|
@@ -102,10 +113,9 @@ interface PendingTelnyxMediaFrame {
|
|
|
102
113
|
readonly pcm: Int16Array;
|
|
103
114
|
}
|
|
104
115
|
|
|
105
|
-
type TelnyxCodec = "PCMU" | "L16";
|
|
106
|
-
|
|
107
116
|
interface TelnyxConnectionState {
|
|
108
117
|
streamId: string;
|
|
118
|
+
callControlId: string;
|
|
109
119
|
contextId: string;
|
|
110
120
|
contextBase: string;
|
|
111
121
|
turnCounter: number;
|
|
@@ -125,6 +135,8 @@ interface TelnyxConnectionState {
|
|
|
125
135
|
onPlaybackMarkReceived?: () => void;
|
|
126
136
|
clearPlayout: (reason: string) => void;
|
|
127
137
|
readonly streamingResamplers: Map<string, StreamingPcm16Resampler>;
|
|
138
|
+
/** Stateful G.722 encoder/decoder (only used when codec is G722). */
|
|
139
|
+
g722: TelnyxG722State;
|
|
128
140
|
}
|
|
129
141
|
|
|
130
142
|
const DEFAULT_ENGINE_SAMPLE_RATE_HZ = 16000;
|
|
@@ -155,7 +167,7 @@ export async function createTelnyxMediaStreamServer(
|
|
|
155
167
|
const sessions = new Set<VoiceAgentSession>();
|
|
156
168
|
const inputSampleRateHz = positiveInteger(options.inputSampleRateHz) ?? DEFAULT_ENGINE_SAMPLE_RATE_HZ;
|
|
157
169
|
const bidirectionalCodec: TelnyxCodec = options.bidirectionalCodec ?? "PCMU";
|
|
158
|
-
const outboundSampleRateHz = bidirectionalCodec
|
|
170
|
+
const outboundSampleRateHz = wireSampleRateForCodec(bidirectionalCodec);
|
|
159
171
|
const outboundFrameDurationMs = positiveInteger(options.outboundFrameDurationMs) ?? DEFAULT_OUTBOUND_FRAME_DURATION_MS;
|
|
160
172
|
const maxQueuedOutputAudioMs = positiveInteger(options.maxQueuedOutputAudioMs) ?? DEFAULT_MAX_QUEUED_OUTPUT_AUDIO_MS;
|
|
161
173
|
const maxInboundReorderFrames = positiveInteger(options.maxInboundReorderFrames) ?? DEFAULT_MAX_INBOUND_REORDER_FRAMES;
|
|
@@ -172,6 +184,7 @@ export async function createTelnyxMediaStreamServer(
|
|
|
172
184
|
const adapter: TransportAdapter<TelnyxConnectionState> = {
|
|
173
185
|
createState: () => ({
|
|
174
186
|
streamId: "",
|
|
187
|
+
callControlId: "",
|
|
175
188
|
contextId: "",
|
|
176
189
|
contextBase: "",
|
|
177
190
|
turnCounter: 0,
|
|
@@ -190,6 +203,7 @@ export async function createTelnyxMediaStreamServer(
|
|
|
190
203
|
pendingEndMarkName: "",
|
|
191
204
|
clearPlayout: () => undefined,
|
|
192
205
|
streamingResamplers: new Map(),
|
|
206
|
+
g722: createTelnyxG722State(bidirectionalCodec),
|
|
193
207
|
}),
|
|
194
208
|
|
|
195
209
|
async acquireSession({ request, shouldAbort, onSessionCreated }) {
|
|
@@ -221,6 +235,15 @@ export async function createTelnyxMediaStreamServer(
|
|
|
221
235
|
};
|
|
222
236
|
state.onPlaybackMarkReceived = sendPendingEndMark;
|
|
223
237
|
|
|
238
|
+
wireCarrierControl(session, disposers, {
|
|
239
|
+
carrier: "telnyx",
|
|
240
|
+
getCallControlId: () => state.callControlId || undefined,
|
|
241
|
+
credentials: options.callControlCredentials,
|
|
242
|
+
fetchImpl: options.callControlFetch,
|
|
243
|
+
webhookUrl: options.callControlWebhookUrl,
|
|
244
|
+
warmSummarizer: options.warmTransferSummarizer,
|
|
245
|
+
});
|
|
246
|
+
|
|
224
247
|
const outbound = wireTelephonyOutboundPipeline({
|
|
225
248
|
session,
|
|
226
249
|
socket,
|
|
@@ -323,10 +346,16 @@ export async function createTelnyxMediaStreamServer(
|
|
|
323
346
|
const format = validateTelnyxStart(start);
|
|
324
347
|
state.streamId = message.stream_id ?? start.stream_id ?? "";
|
|
325
348
|
if (!state.streamId) throw new Error("Telnyx start event is missing stream_id");
|
|
349
|
+
state.callControlId = start.call_control_id?.trim() ?? "";
|
|
326
350
|
state.contextId = contextIdFn(start);
|
|
327
351
|
state.contextBase = state.contextId;
|
|
328
352
|
state.inboundCodec = format.codec;
|
|
329
353
|
state.inboundSampleRateHz = format.sampleRateHz;
|
|
354
|
+
// G.722 is stateful — reset decoder on each stream start. Keep 16 kHz
|
|
355
|
+
// for inbound (do NOT downsample before STT; that is the quality pitfall).
|
|
356
|
+
if (format.codec === "G722") {
|
|
357
|
+
state.g722.decoder = createG722DecoderState();
|
|
358
|
+
}
|
|
330
359
|
state.started = true;
|
|
331
360
|
state.nextInboundMediaChunk = 1;
|
|
332
361
|
state.inboundMediaReorderBuffer.clear();
|
|
@@ -338,7 +367,7 @@ export async function createTelnyxMediaStreamServer(
|
|
|
338
367
|
const payload = message.media?.payload;
|
|
339
368
|
if (!payload) throw new Error("Telnyx media event is missing media.payload");
|
|
340
369
|
const encoded = decodeStrictBase64(payload, "media.payload");
|
|
341
|
-
const pcm = decodeInboundPayload(encoded, state
|
|
370
|
+
const pcm = decodeInboundPayload(encoded, state);
|
|
342
371
|
const chunk = optionalPositiveIntegerString(message.media?.chunk, "Telnyx media.chunk");
|
|
343
372
|
if (chunk === undefined) {
|
|
344
373
|
emitTelnyxMediaFrame(session, state, { chunk: 0, timestamp: message.media?.timestamp, pcm }, inputSampleRateHz);
|
|
@@ -657,25 +686,8 @@ function rememberTelnyxMediaTimestamp(
|
|
|
657
686
|
state.lastInboundMediaTimestampMs = timestampMs;
|
|
658
687
|
}
|
|
659
688
|
|
|
660
|
-
function
|
|
661
|
-
|
|
662
|
-
if (!format) throw new Error("Telnyx start event is missing media_format");
|
|
663
|
-
const encoding = format.encoding?.trim().toUpperCase();
|
|
664
|
-
if (encoding !== "PCMU" && encoding !== "L16") {
|
|
665
|
-
throw new Error(`Unsupported Telnyx media encoding: ${format.encoding ?? "unknown"}`);
|
|
666
|
-
}
|
|
667
|
-
const sampleRateHz = numberFromString(format.sample_rate);
|
|
668
|
-
if (encoding === "PCMU" && sampleRateHz !== 8000) throw new Error(`Unsupported Telnyx PCMU sample rate: ${String(format.sample_rate)}`);
|
|
669
|
-
if (encoding === "L16" && sampleRateHz !== 16000) throw new Error(`Unsupported Telnyx L16 sample rate: ${String(format.sample_rate)}`);
|
|
670
|
-
if (sampleRateHz === null) throw new Error(`Unsupported Telnyx sample rate: ${String(format.sample_rate)}`);
|
|
671
|
-
const channels = numberFromString(format.channels);
|
|
672
|
-
if (channels !== 1) throw new Error(`Unsupported Telnyx channel count: ${String(format.channels)}`);
|
|
673
|
-
return { codec: encoding, sampleRateHz };
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
function decodeInboundPayload(input: Uint8Array, codec: TelnyxCodec): Int16Array {
|
|
677
|
-
if (codec === "PCMU") return decodeMuLawToPcm16(input);
|
|
678
|
-
return bigEndianPcm16BytesToSamples(input);
|
|
689
|
+
function decodeInboundPayload(input: Uint8Array, state: TelnyxConnectionState): Int16Array {
|
|
690
|
+
return decodeTelnyxInboundPayload(input, state.inboundCodec, state.g722);
|
|
679
691
|
}
|
|
680
692
|
|
|
681
693
|
function encodeOutboundPayload(
|
|
@@ -686,25 +698,8 @@ function encodeOutboundPayload(
|
|
|
686
698
|
): Uint8Array[] {
|
|
687
699
|
const samples = pcm16BytesToSamples(audio);
|
|
688
700
|
const resampled = resamplePcm16Streaming(state.streamingResamplers, samples, sourceSampleRateHz, state.outboundSampleRateHz);
|
|
689
|
-
const encoded = state.outboundCodec
|
|
690
|
-
|
|
691
|
-
: pcm16SamplesToBigEndianBytes(resampled);
|
|
692
|
-
const frameBytes = Math.max(1, Math.round((state.outboundSampleRateHz * frameDurationMs) / 1000) * (state.outboundCodec === "L16" ? 2 : 1));
|
|
693
|
-
const frames: Uint8Array[] = [];
|
|
694
|
-
for (let offset = 0; offset < encoded.byteLength; offset += frameBytes) {
|
|
695
|
-
frames.push(encoded.subarray(offset, Math.min(encoded.byteLength, offset + frameBytes)));
|
|
696
|
-
}
|
|
697
|
-
return frames;
|
|
698
|
-
}
|
|
699
|
-
|
|
700
|
-
function defaultTelnyxContextId(start: TelnyxStartPayload): string {
|
|
701
|
-
const callControlId = start.call_control_id?.trim();
|
|
702
|
-
if (callControlId) return `telnyx-${callControlId}`;
|
|
703
|
-
const callSessionId = start.call_session_id?.trim();
|
|
704
|
-
if (callSessionId) return `telnyx-${callSessionId}`;
|
|
705
|
-
const streamId = start.stream_id?.trim();
|
|
706
|
-
if (streamId) return `telnyx-${streamId}`;
|
|
707
|
-
return `telnyx-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
|
701
|
+
const encoded = encodeTelnyxOutboundPayload(resampled, state.outboundCodec, state.g722);
|
|
702
|
+
return splitTelnyxEncodedFrames(encoded, state.outboundCodec, state.outboundSampleRateHz, frameDurationMs);
|
|
708
703
|
}
|
|
709
704
|
|
|
710
705
|
function sendTelnyxError(socket: WebSocket, streamId: string, message: string, maxBufferedAmountBytes: number): void {
|
package/src/turn-metrics.ts
CHANGED
|
@@ -8,8 +8,11 @@ import {
|
|
|
8
8
|
type PipelineBus,
|
|
9
9
|
type SttResultPacket,
|
|
10
10
|
type TextToSpeechAudioPacket,
|
|
11
|
+
type TextToSpeechEndPacket,
|
|
11
12
|
type TextToSpeechPlayoutProgressPacket,
|
|
12
13
|
type TextToSpeechPlayoutStartedPacket,
|
|
14
|
+
type TurnEndOwner,
|
|
15
|
+
type TurnEndReason,
|
|
13
16
|
type VadSpeechEndedPacket,
|
|
14
17
|
} from "@kuralle-syrinx/core";
|
|
15
18
|
|
|
@@ -22,6 +25,12 @@ export interface TurnTimestampState {
|
|
|
22
25
|
firstAudioByteMs: number;
|
|
23
26
|
firstAudioPlayedMs: number;
|
|
24
27
|
lastAudioPlayedMs: number;
|
|
28
|
+
/** Who decided the turn ended — captured from the completing eos packet. */
|
|
29
|
+
endpointingOwner?: TurnEndOwner;
|
|
30
|
+
/** Why the turn ended — captured from the completing eos packet. */
|
|
31
|
+
endpointingReason?: TurnEndReason;
|
|
32
|
+
/** Set when the STT force-finalize watchdog fired for this turn. */
|
|
33
|
+
forceFinalized?: boolean;
|
|
25
34
|
}
|
|
26
35
|
|
|
27
36
|
export interface BrowserMetricsMessage {
|
|
@@ -43,8 +52,36 @@ export interface BrowserMetricsMessage {
|
|
|
43
52
|
readonly endpointDelayMs?: number;
|
|
44
53
|
readonly totalMs?: number;
|
|
45
54
|
};
|
|
55
|
+
/** Which owner decided the turn ended. Omitted when the backend did not say. */
|
|
56
|
+
readonly endpointingOwner?: TurnEndOwner;
|
|
57
|
+
/** Why the turn ended. Omitted when the backend did not say. */
|
|
58
|
+
readonly endpointingReason?: TurnEndReason;
|
|
46
59
|
}
|
|
47
60
|
|
|
61
|
+
/**
|
|
62
|
+
* The field set `buildBrowserMetricsMessage` populates for an audio-originated turn
|
|
63
|
+
* (`vad.speech_ended` + `stt.result` + `llm.delta` + `tts.audio` all measured) —
|
|
64
|
+
* everything EXCEPT `firstAudioPlayedMs`/`lastAudioPlayedMs`, which depend on a
|
|
65
|
+
* playout-completion signal whose *source* legitimately differs per transport
|
|
66
|
+
* (Node's browser websocket path paces server-side; the Workers/DO edge path is
|
|
67
|
+
* client-paced). Both runtimes call this same builder and must emit this same core
|
|
68
|
+
* set; the parity tests in both packages assert against this constant so a runtime
|
|
69
|
+
* that silently drops the `metrics` message, or diverges in shape, fails loudly.
|
|
70
|
+
*/
|
|
71
|
+
export const CORE_METRICS_FIELDS = [
|
|
72
|
+
"type",
|
|
73
|
+
"turnId",
|
|
74
|
+
"correlationId",
|
|
75
|
+
"speechEndMs",
|
|
76
|
+
"textReadyMs",
|
|
77
|
+
"firstAudioByteMs",
|
|
78
|
+
"sttMs",
|
|
79
|
+
"llmTTFTMs",
|
|
80
|
+
"ttsTTFBMs",
|
|
81
|
+
"e2eMs",
|
|
82
|
+
"eouBudgetMs",
|
|
83
|
+
] as const;
|
|
84
|
+
|
|
48
85
|
function positiveDelta(endMs: number, startMs: number): number | undefined {
|
|
49
86
|
if (endMs <= 0 || startMs <= 0 || endMs < startMs) return undefined;
|
|
50
87
|
return endMs - startMs;
|
|
@@ -81,6 +118,15 @@ export function buildBrowserMetricsMessage(
|
|
|
81
118
|
}
|
|
82
119
|
: undefined;
|
|
83
120
|
|
|
121
|
+
// The endpointing decision is a fact about the turn, not a timing measurement, so
|
|
122
|
+
// it is emitted whenever it is known — never coerced to a zero. A force-finalize
|
|
123
|
+
// watchdog firing overrides the emitter's reason: the STT plugin that produced the
|
|
124
|
+
// completing eos cannot know the watchdog fired, but this mark (stt.force_finalized)
|
|
125
|
+
// arrives independently on the bus, so the truth wins here regardless of emitter.
|
|
126
|
+
const endpointingOwner = timestamps.endpointingOwner;
|
|
127
|
+
const endpointingReason =
|
|
128
|
+
timestamps.forceFinalized === true ? "force_finalized" : timestamps.endpointingReason;
|
|
129
|
+
|
|
84
130
|
return {
|
|
85
131
|
type: "metrics",
|
|
86
132
|
turnId,
|
|
@@ -95,6 +141,8 @@ export function buildBrowserMetricsMessage(
|
|
|
95
141
|
...(ttsTTFBMs !== undefined ? { ttsTTFBMs } : {}),
|
|
96
142
|
...(e2eFromPlayout !== undefined ? { e2eMs: e2eFromPlayout } : e2eFromByte !== undefined ? { e2eMs: e2eFromByte } : {}),
|
|
97
143
|
...(eouBudgetMs !== undefined ? { eouBudgetMs } : {}),
|
|
144
|
+
...(endpointingOwner !== undefined ? { endpointingOwner } : {}),
|
|
145
|
+
...(endpointingReason !== undefined ? { endpointingReason } : {}),
|
|
98
146
|
};
|
|
99
147
|
}
|
|
100
148
|
|
|
@@ -111,6 +159,23 @@ function emptyTurnState(): TurnTimestampState {
|
|
|
111
159
|
};
|
|
112
160
|
}
|
|
113
161
|
|
|
162
|
+
export interface TurnMetricsTrackerOptions {
|
|
163
|
+
/**
|
|
164
|
+
* Additionally finalize (emit + clear) a turn's metrics on `tts.end` when no
|
|
165
|
+
* `tts.playout_progress` completion has arrived yet. The Node browser websocket
|
|
166
|
+
* path always gets a `tts.playout_progress` completion shortly after `tts.end`
|
|
167
|
+
* from its own server-side pacer (playout-progress.ts) — turning this on there
|
|
168
|
+
* would race a premature, playout-less emission ahead of the real one, so it
|
|
169
|
+
* defaults off and Node leaves it unset. The Workers/DO edge path is
|
|
170
|
+
* client-paced (the browser reports playout over the wire); a client that never
|
|
171
|
+
* reports it — a non-browser client, or a text-only smoke probe — would
|
|
172
|
+
* otherwise leave a turn's metrics pending forever, so edge opts in. If a
|
|
173
|
+
* richer client-reported completion arrives after this floor already fired,
|
|
174
|
+
* it is a no-op: the turn was already finalized and cleared.
|
|
175
|
+
*/
|
|
176
|
+
readonly finalizeOnTtsEnd?: boolean;
|
|
177
|
+
}
|
|
178
|
+
|
|
114
179
|
export class TurnMetricsTracker {
|
|
115
180
|
private readonly turns: Map<string, TurnTimestampState>;
|
|
116
181
|
|
|
@@ -118,6 +183,7 @@ export class TurnMetricsTracker {
|
|
|
118
183
|
private readonly bus: PipelineBus,
|
|
119
184
|
private readonly onEmit: (message: BrowserMetricsMessage) => void,
|
|
120
185
|
persistedTurns?: Map<string, TurnTimestampState>,
|
|
186
|
+
private readonly options: TurnMetricsTrackerOptions = {},
|
|
121
187
|
) {
|
|
122
188
|
this.turns = persistedTurns ?? new Map();
|
|
123
189
|
}
|
|
@@ -136,16 +202,29 @@ export class TurnMetricsTracker {
|
|
|
136
202
|
}),
|
|
137
203
|
this.bus.on("metric.conversation", (pkt) => {
|
|
138
204
|
const metric = pkt as ConversationMetricPacket;
|
|
139
|
-
if (metric.name
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
205
|
+
if (metric.name === "vad.stop_hangover_ms") {
|
|
206
|
+
const hangoverMs = Number(metric.value);
|
|
207
|
+
if (Number.isNaN(hangoverMs)) return;
|
|
208
|
+
const state = this.turnState(metric.contextId);
|
|
209
|
+
if (state.vadStopHangoverMs === 0) state.vadStopHangoverMs = hangoverMs;
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (metric.name === "stt.force_finalized") {
|
|
213
|
+
this.turnState(metric.contextId).forceFinalized = true;
|
|
214
|
+
}
|
|
144
215
|
}),
|
|
145
216
|
this.bus.on("eos.turn_complete", (pkt) => {
|
|
146
217
|
const eos = pkt as EndOfSpeechPacket;
|
|
147
218
|
const state = this.turnState(eos.contextId);
|
|
148
219
|
if (state.eosMs === 0) state.eosMs = eos.timestampMs;
|
|
220
|
+
// First eos wins (mirrors eosMs): the completing eos is the first, and any
|
|
221
|
+
// duplicate is dropped downstream. Owner/reason are facts, not measurements.
|
|
222
|
+
if (state.endpointingOwner === undefined && eos.endpointingOwner !== undefined) {
|
|
223
|
+
state.endpointingOwner = eos.endpointingOwner;
|
|
224
|
+
}
|
|
225
|
+
if (state.endpointingReason === undefined && eos.endpointingReason !== undefined) {
|
|
226
|
+
state.endpointingReason = eos.endpointingReason;
|
|
227
|
+
}
|
|
149
228
|
}),
|
|
150
229
|
this.bus.on("llm.delta", (pkt) => {
|
|
151
230
|
const delta = pkt as LlmDeltaPacket;
|
|
@@ -170,12 +249,35 @@ export class TurnMetricsTracker {
|
|
|
170
249
|
const progress = pkt as TextToSpeechPlayoutProgressPacket;
|
|
171
250
|
const state = this.turns.get(progress.contextId);
|
|
172
251
|
if (!state) return;
|
|
252
|
+
// The edge transport never emits `tts.playout_started` — the browser reports
|
|
253
|
+
// its own playout clock and that is forwarded here (edge.ts). So take the
|
|
254
|
+
// first progress report as the moment audio started playing. On the Node path
|
|
255
|
+
// `playout_started` has already set this, and the `=== 0` guard leaves it be.
|
|
256
|
+
if (state.firstAudioPlayedMs === 0) state.firstAudioPlayedMs = progress.timestampMs;
|
|
173
257
|
if (progress.complete) {
|
|
174
258
|
state.lastAudioPlayedMs = progress.timestampMs;
|
|
175
259
|
this.onEmit(buildBrowserMetricsMessage(progress.contextId, state));
|
|
176
260
|
this.turns.delete(progress.contextId);
|
|
177
261
|
}
|
|
178
262
|
}),
|
|
263
|
+
this.bus.on("tts.end", (pkt) => {
|
|
264
|
+
if (!this.options.finalizeOnTtsEnd) return;
|
|
265
|
+
const end = pkt as TextToSpeechEndPacket;
|
|
266
|
+
const state = this.turns.get(end.contextId);
|
|
267
|
+
if (!state) return;
|
|
268
|
+
// `tts.end` means synthesis finished, which is EARLIER than playback finishing.
|
|
269
|
+
// So if this client is pacing playout, firing here would emit a poorer message
|
|
270
|
+
// and delete the turn before the real completion arrived — costing
|
|
271
|
+
// firstAudioPlayedMs, lastAudioPlayedMs, and downgrading e2eMs from
|
|
272
|
+
// "to first audio played" to "to first byte" on this runtime only. That is the
|
|
273
|
+
// exact runtime drift this tracker exists to prevent.
|
|
274
|
+
//
|
|
275
|
+
// A non-zero firstAudioPlayedMs means playout has been reported, so wait.
|
|
276
|
+
// The floor is only for clients that never report at all.
|
|
277
|
+
if (state.firstAudioPlayedMs !== 0) return;
|
|
278
|
+
this.onEmit(buildBrowserMetricsMessage(end.contextId, state));
|
|
279
|
+
this.turns.delete(end.contextId);
|
|
280
|
+
}),
|
|
179
281
|
this.bus.on("interrupt.tts", (pkt) => {
|
|
180
282
|
this.turns.delete((pkt as InterruptTtsPacket).contextId);
|
|
181
283
|
}),
|
package/src/twilio.ts
CHANGED
|
@@ -33,6 +33,8 @@ import {
|
|
|
33
33
|
positiveInteger,
|
|
34
34
|
rawDataToText,
|
|
35
35
|
} from "./transport-helpers.js";
|
|
36
|
+
import type { FetchLike, TwilioRestCredentials, WarmTransferSummarizer } from "./carrier-commands.js";
|
|
37
|
+
import { wireCarrierControl } from "./wire-carrier-control.js";
|
|
36
38
|
|
|
37
39
|
export interface TwilioMediaStreamServerOptions {
|
|
38
40
|
readonly server?: HttpServer;
|
|
@@ -56,6 +58,15 @@ export interface TwilioMediaStreamServerOptions {
|
|
|
56
58
|
readonly maxConcurrentSessions?: number;
|
|
57
59
|
readonly maxConcurrentSessionsScope?: "path" | "server";
|
|
58
60
|
readonly onTransportMetric?: (name: string) => void;
|
|
61
|
+
/**
|
|
62
|
+
* Injected REST credentials for dtmf.send / call.transfer dispatch.
|
|
63
|
+
* Workers: pass secrets from `env`. Never read from process.env here.
|
|
64
|
+
* Live dispatch unverified without credentials.
|
|
65
|
+
*/
|
|
66
|
+
readonly restCredentials?: TwilioRestCredentials;
|
|
67
|
+
readonly restFetch?: FetchLike;
|
|
68
|
+
readonly transferRedirectUrl?: string;
|
|
69
|
+
readonly warmTransferSummarizer?: WarmTransferSummarizer;
|
|
59
70
|
}
|
|
60
71
|
|
|
61
72
|
export interface TwilioMediaStreamServer {
|
|
@@ -92,6 +103,7 @@ interface TwilioMediaMessage {
|
|
|
92
103
|
|
|
93
104
|
interface TwilioConnectionState {
|
|
94
105
|
streamSid: string;
|
|
106
|
+
callSid: string;
|
|
95
107
|
contextId: string;
|
|
96
108
|
contextBase: string;
|
|
97
109
|
turnCounter: number;
|
|
@@ -152,6 +164,7 @@ export async function createTwilioMediaStreamServer(
|
|
|
152
164
|
const adapter: TransportAdapter<TwilioConnectionState> = {
|
|
153
165
|
createState: () => ({
|
|
154
166
|
streamSid: "",
|
|
167
|
+
callSid: "",
|
|
155
168
|
contextId: "",
|
|
156
169
|
contextBase: "",
|
|
157
170
|
turnCounter: 0,
|
|
@@ -197,6 +210,15 @@ export async function createTwilioMediaStreamServer(
|
|
|
197
210
|
};
|
|
198
211
|
state.onPlaybackMarkReceived = sendPendingEndMark;
|
|
199
212
|
|
|
213
|
+
wireCarrierControl(session, disposers, {
|
|
214
|
+
carrier: "twilio",
|
|
215
|
+
getCallSid: () => state.callSid || undefined,
|
|
216
|
+
credentials: options.restCredentials,
|
|
217
|
+
fetchImpl: options.restFetch,
|
|
218
|
+
redirectUrl: options.transferRedirectUrl,
|
|
219
|
+
warmSummarizer: options.warmTransferSummarizer,
|
|
220
|
+
});
|
|
221
|
+
|
|
200
222
|
const outbound = wireTelephonyOutboundPipeline({
|
|
201
223
|
session,
|
|
202
224
|
socket,
|
|
@@ -304,6 +326,7 @@ export async function createTwilioMediaStreamServer(
|
|
|
304
326
|
validateTwilioStart(start, twilioSampleRateHz);
|
|
305
327
|
state.streamSid = start.streamSid ?? message.streamSid ?? "";
|
|
306
328
|
if (!state.streamSid) throw new Error("Twilio start event is missing streamSid");
|
|
329
|
+
state.callSid = start.callSid?.trim() ?? "";
|
|
307
330
|
state.contextId = contextIdFn(start);
|
|
308
331
|
state.contextBase = state.contextId;
|
|
309
332
|
state.started = true;
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Bus wiring for dtmf.send + call.transfer → pure constructor + injectable dispatch.
|
|
4
|
+
// Mechanism unit-tested; live carrier HTTP unverified.
|
|
5
|
+
|
|
6
|
+
import { Route, type CallTransferPacket, type DtmfSendPacket, type VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
7
|
+
import {
|
|
8
|
+
buildTelnyxSendDtmf,
|
|
9
|
+
buildTelnyxTransfer,
|
|
10
|
+
buildTwilioSendDigits,
|
|
11
|
+
buildTwilioTransfer,
|
|
12
|
+
dispatchTelnyxCommand,
|
|
13
|
+
dispatchTwilioCommand,
|
|
14
|
+
resolveTransferSummary,
|
|
15
|
+
type FetchLike,
|
|
16
|
+
type TelnyxRestCredentials,
|
|
17
|
+
type TwilioRestCredentials,
|
|
18
|
+
type WarmTransferSummarizer,
|
|
19
|
+
} from "./carrier-commands.js";
|
|
20
|
+
|
|
21
|
+
export interface TwilioCarrierControlOptions {
|
|
22
|
+
readonly carrier: "twilio";
|
|
23
|
+
/** Call SID for the active media stream (injected; may resolve lazily). */
|
|
24
|
+
getCallSid: () => string | undefined;
|
|
25
|
+
readonly credentials?: TwilioRestCredentials;
|
|
26
|
+
readonly fetchImpl?: FetchLike;
|
|
27
|
+
readonly redirectUrl?: string;
|
|
28
|
+
readonly warmSummarizer?: WarmTransferSummarizer;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface TelnyxCarrierControlOptions {
|
|
32
|
+
readonly carrier: "telnyx";
|
|
33
|
+
getCallControlId: () => string | undefined;
|
|
34
|
+
readonly credentials?: TelnyxRestCredentials;
|
|
35
|
+
readonly fetchImpl?: FetchLike;
|
|
36
|
+
readonly webhookUrl?: string;
|
|
37
|
+
readonly warmSummarizer?: WarmTransferSummarizer;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export type CarrierControlOptions = TwilioCarrierControlOptions | TelnyxCarrierControlOptions;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Subscribe to `dtmf.send` and `call.transfer` on the session bus.
|
|
44
|
+
* When credentials are absent, still builds the command and emits a metric so
|
|
45
|
+
* unit tests can assert the constructor path without live HTTP.
|
|
46
|
+
*/
|
|
47
|
+
export function wireCarrierControl(
|
|
48
|
+
session: VoiceAgentSession,
|
|
49
|
+
disposers: Array<() => void>,
|
|
50
|
+
options: CarrierControlOptions,
|
|
51
|
+
): void {
|
|
52
|
+
disposers.push(
|
|
53
|
+
session.bus.on("dtmf.send", (pkt) => {
|
|
54
|
+
void handleDtmfSend(session, pkt as DtmfSendPacket, options);
|
|
55
|
+
}),
|
|
56
|
+
session.bus.on("call.transfer", (pkt) => {
|
|
57
|
+
void handleCallTransfer(session, pkt as CallTransferPacket, options);
|
|
58
|
+
}),
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function handleDtmfSend(
|
|
63
|
+
session: VoiceAgentSession,
|
|
64
|
+
pkt: DtmfSendPacket,
|
|
65
|
+
options: CarrierControlOptions,
|
|
66
|
+
): Promise<void> {
|
|
67
|
+
try {
|
|
68
|
+
if (options.carrier === "twilio") {
|
|
69
|
+
const callSid = options.getCallSid();
|
|
70
|
+
if (!callSid) {
|
|
71
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.missing_call_handle", "twilio");
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const command = buildTwilioSendDigits(callSid, pkt);
|
|
75
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.built", JSON.stringify({ carrier: "twilio", digits: command.form.SendDigits }));
|
|
76
|
+
if (!options.credentials) {
|
|
77
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.no_credentials", "twilio");
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const res = await dispatchTwilioCommand(options.credentials, command, options.fetchImpl);
|
|
81
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.dispatched", String(res.status));
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const callControlId = options.getCallControlId();
|
|
86
|
+
if (!callControlId) {
|
|
87
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.missing_call_handle", "telnyx");
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
const command = buildTelnyxSendDtmf(callControlId, pkt);
|
|
91
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.built", JSON.stringify({ carrier: "telnyx", digits: command.json.digits }));
|
|
92
|
+
if (!options.credentials) {
|
|
93
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.no_credentials", "telnyx");
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
const res = await dispatchTelnyxCommand(options.credentials, command, options.fetchImpl);
|
|
97
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.dispatched", String(res.status));
|
|
98
|
+
} catch (err) {
|
|
99
|
+
emitMetric(session, pkt.contextId, "carrier.dtmf_send.error", err instanceof Error ? err.message : String(err));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function handleCallTransfer(
|
|
104
|
+
session: VoiceAgentSession,
|
|
105
|
+
pkt: CallTransferPacket,
|
|
106
|
+
options: CarrierControlOptions,
|
|
107
|
+
): Promise<void> {
|
|
108
|
+
try {
|
|
109
|
+
const summary = await resolveTransferSummary(pkt, options.warmSummarizer);
|
|
110
|
+
const resolved = summary !== undefined ? { ...pkt, summary } : pkt;
|
|
111
|
+
|
|
112
|
+
if (options.carrier === "twilio") {
|
|
113
|
+
const callSid = options.getCallSid();
|
|
114
|
+
if (!callSid) {
|
|
115
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.missing_call_handle", "twilio");
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const command = buildTwilioTransfer(callSid, resolved, {
|
|
119
|
+
redirectUrl: options.redirectUrl,
|
|
120
|
+
});
|
|
121
|
+
emitMetric(
|
|
122
|
+
session,
|
|
123
|
+
pkt.contextId,
|
|
124
|
+
"carrier.transfer.built",
|
|
125
|
+
JSON.stringify({
|
|
126
|
+
carrier: "twilio",
|
|
127
|
+
mode: command.mode,
|
|
128
|
+
target: resolved.target,
|
|
129
|
+
summary: command.summary ?? null,
|
|
130
|
+
}),
|
|
131
|
+
);
|
|
132
|
+
if (!options.credentials) {
|
|
133
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.no_credentials", "twilio");
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
const res = await dispatchTwilioCommand(options.credentials, command, options.fetchImpl);
|
|
137
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.dispatched", String(res.status));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const callControlId = options.getCallControlId();
|
|
142
|
+
if (!callControlId) {
|
|
143
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.missing_call_handle", "telnyx");
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const command = buildTelnyxTransfer(callControlId, resolved, {
|
|
147
|
+
webhookUrl: options.webhookUrl,
|
|
148
|
+
});
|
|
149
|
+
emitMetric(
|
|
150
|
+
session,
|
|
151
|
+
pkt.contextId,
|
|
152
|
+
"carrier.transfer.built",
|
|
153
|
+
JSON.stringify({
|
|
154
|
+
carrier: "telnyx",
|
|
155
|
+
mode: command.mode,
|
|
156
|
+
target: command.json.to,
|
|
157
|
+
summary: command.summary ?? null,
|
|
158
|
+
}),
|
|
159
|
+
);
|
|
160
|
+
if (!options.credentials) {
|
|
161
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.no_credentials", "telnyx");
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const res = await dispatchTelnyxCommand(options.credentials, command, options.fetchImpl);
|
|
165
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.dispatched", String(res.status));
|
|
166
|
+
} catch (err) {
|
|
167
|
+
emitMetric(session, pkt.contextId, "carrier.transfer.error", err instanceof Error ? err.message : String(err));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function emitMetric(session: VoiceAgentSession, contextId: string, name: string, value: string): void {
|
|
172
|
+
session.bus.push(Route.Background, {
|
|
173
|
+
kind: "metric.conversation",
|
|
174
|
+
contextId,
|
|
175
|
+
timestampMs: Date.now(),
|
|
176
|
+
name,
|
|
177
|
+
value,
|
|
178
|
+
});
|
|
179
|
+
}
|