botium-connector-voip 0.0.29 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/botium-connector-voip-cjs.js +998 -55
- package/dist/botium-connector-voip-cjs.js.map +1 -1
- package/dist/botium-connector-voip-es.js +998 -55
- package/dist/botium-connector-voip-es.js.map +1 -1
- package/index.js +21 -0
- package/package.json +1 -1
- package/src/connector.js +977 -51
|
@@ -26,6 +26,12 @@ const debug$3 = debug__default["default"]('botium-connector-voip');
|
|
|
26
26
|
// debug = high-frequency diagnostics (DEBUG=botium-connector-voip). warn = degraded but continuing.
|
|
27
27
|
// error = abort/failure. No secrets in info; STT text only as length or truncated in info.
|
|
28
28
|
|
|
29
|
+
/** WS frame types logged only at start/end handlers — not per-chunk (hundreds per call). */
|
|
30
|
+
const WS_DEBUG_SILENT_TYPES = new Set(['audioStreamChunk', 'fullRecordChunk']);
|
|
31
|
+
const AGENT_SPEECH_RMS_WINDOW_MS = 100;
|
|
32
|
+
const DEFAULT_AGENT_SPEECH_RMS_THRESHOLD = 500;
|
|
33
|
+
const DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS = 2;
|
|
34
|
+
const WS_DEBUG_BASE64_FIELD_NAMES = new Set(['chunk', 'buffer', 'base64', 'fullRecord', 'full_record', 'audio', 'audioData', 'b64_buffer']);
|
|
29
35
|
const _info = (event, data) => {
|
|
30
36
|
const parts = Object.entries({
|
|
31
37
|
event,
|
|
@@ -39,12 +45,34 @@ const sanitizeDtmfDigits = raw => {
|
|
|
39
45
|
if (raw == null) return '';
|
|
40
46
|
return String(raw).replace(/[^0-9*#ABCDabcd]/g, '').toUpperCase();
|
|
41
47
|
};
|
|
48
|
+
|
|
49
|
+
// Matches voipcall.generate_dtmf_sequence defaults (100 ms tone, 50 ms pause between digits).
|
|
50
|
+
const DTMF_TONE_MS = 100;
|
|
51
|
+
const DTMF_PAUSE_MS = 50;
|
|
52
|
+
const DTMF_MS_PER_DIGIT = 200;
|
|
53
|
+
const DEFAULT_AUDIO_STREAM_INTERVAL_MS = 250;
|
|
54
|
+
const audioStreamIntervalMs = () => {
|
|
55
|
+
const n = Number(process.env.VOIP_AUDIO_STREAM_INTERVAL_MS || process.env.AUDIO_STREAM_INTERVAL_MS);
|
|
56
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AUDIO_STREAM_INTERVAL_MS;
|
|
57
|
+
};
|
|
58
|
+
const dtmfPlaybackMs = digitCount => {
|
|
59
|
+
if (!digitCount || digitCount <= 0) return 0;
|
|
60
|
+
return digitCount * DTMF_TONE_MS + Math.max(0, digitCount - 1) * DTMF_PAUSE_MS;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/** Wait before turn-audio slice so DTMF PCM is flushed through audioStream (250 ms default). */
|
|
64
|
+
const dtmfTurnAudioWaitMs = digitCount => {
|
|
65
|
+
const playbackMs = dtmfPlaybackMs(digitCount);
|
|
66
|
+
const streamMs = audioStreamIntervalMs();
|
|
67
|
+
return Math.max(digitCount * DTMF_MS_PER_DIGIT, playbackMs + streamMs + 50);
|
|
68
|
+
};
|
|
42
69
|
const Capabilities = {
|
|
43
70
|
VOIP_STT_URL_STREAM: 'VOIP_STT_URL_STREAM',
|
|
44
71
|
VOIP_STT_PARAMS_STREAM: 'VOIP_STT_PARAMS_STREAM',
|
|
45
72
|
VOIP_STT_METHOD_STREAM: 'VOIP_STT_METHOD_STREAM',
|
|
46
73
|
VOIP_STT_BODY_STREAM: 'VOIP_STT_BODY_STREAM',
|
|
47
74
|
VOIP_STT_BODY: 'VOIP_STT_BODY',
|
|
75
|
+
VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 'VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS',
|
|
48
76
|
VOIP_STT_HEADERS: 'VOIP_STT_HEADERS',
|
|
49
77
|
VOIP_STT_TIMEOUT: 'VOIP_STT_TIMEOUT',
|
|
50
78
|
VOIP_STT_MESSAGE_HANDLING: 'VOIP_STT_MESSAGE_HANDLING',
|
|
@@ -84,6 +112,8 @@ const Capabilities = {
|
|
|
84
112
|
VOIP_ICE_TURN_PROTOCOL: 'VOIP_ICE_TURN_PROTOCOL',
|
|
85
113
|
VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 'VOIP_WEBSOCKET_CONNECT_MAXRETRIES',
|
|
86
114
|
VOIP_WEBSOCKET_CONNECT_TIMEOUT: 'VOIP_WEBSOCKET_CONNECT_TIMEOUT',
|
|
115
|
+
VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 'VOIP_CALL_SETUP_RETRY_487_MAXRETRIES',
|
|
116
|
+
VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 'VOIP_CALL_SETUP_RETRY_487_TIMEOUT',
|
|
87
117
|
VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_ENABLE',
|
|
88
118
|
VOIP_SILENCE_DURATION_TIMEOUT: 'VOIP_SILENCE_DURATION_TIMEOUT',
|
|
89
119
|
VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE',
|
|
@@ -92,7 +122,10 @@ const Capabilities = {
|
|
|
92
122
|
VOIP_USE_GLOBAL_VOIP_WORKER: 'VOIP_USE_GLOBAL_VOIP_WORKER',
|
|
93
123
|
VOIP_USER_INPUT_PREFER_VOICE: 'VOIP_USER_INPUT_PREFER_VOICE',
|
|
94
124
|
VOIP_EMIT_SPECULATIVE_TEXT: 'VOIP_EMIT_SPECULATIVE_TEXT',
|
|
95
|
-
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE'
|
|
125
|
+
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE',
|
|
126
|
+
VOIP_TURN_AUDIO_ENABLE: 'VOIP_TURN_AUDIO_ENABLE',
|
|
127
|
+
VOIP_TURN_AUDIO_PADDING_MS: 'VOIP_TURN_AUDIO_PADDING_MS',
|
|
128
|
+
VOIP_TURN_AUDIO_OFFSET_MS: 'VOIP_TURN_AUDIO_OFFSET_MS'
|
|
96
129
|
};
|
|
97
130
|
const Defaults = {
|
|
98
131
|
VOIP_STT_METHOD: 'POST',
|
|
@@ -108,15 +141,54 @@ const Defaults = {
|
|
|
108
141
|
VOIP_STT_MESSAGE_HANDLING_PUNCTUATION: '.!?',
|
|
109
142
|
VOIP_WEBSOCKET_CONNECT_TIMEOUT: 4000,
|
|
110
143
|
VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 5,
|
|
144
|
+
// Retry the whole call setup when the SIP peer terminates the INVITE with a
|
|
145
|
+
// transient '487 Request Terminated' before the call connects.
|
|
146
|
+
VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 2,
|
|
147
|
+
VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 2000,
|
|
111
148
|
VOIP_SILENCE_DURATION_TIMEOUT: 2500,
|
|
112
149
|
VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: false,
|
|
113
150
|
VOIP_SILENCE_DURATION_TIMEOUT_START: 1000,
|
|
114
151
|
VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: false,
|
|
115
152
|
VOIP_STT_CONFIDENCE_THRESHOLD: 0.5,
|
|
153
|
+
VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 500,
|
|
116
154
|
VOIP_USE_GLOBAL_VOIP_WORKER: false,
|
|
117
155
|
VOIP_SIP_PROTOCOL: 'TCP',
|
|
118
156
|
VOIP_USER_INPUT_PREFER_VOICE: true,
|
|
119
|
-
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false
|
|
157
|
+
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false,
|
|
158
|
+
VOIP_TURN_AUDIO_ENABLE: true,
|
|
159
|
+
VOIP_TURN_AUDIO_PADDING_MS: 150,
|
|
160
|
+
VOIP_TURN_AUDIO_OFFSET_MS: 0
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// Inject the Azure end-of-speech segmentation timeout into the STT body. botium-speech-processing
|
|
164
|
+
// applies azure.config.properties via SpeechConfig.setProperty(), so this controls how long Azure
|
|
165
|
+
// waits after the last word before emitting final=true. Only applied for the Azure engine, never
|
|
166
|
+
// overrides a value already present in the profile config, and clones so the capability object stays
|
|
167
|
+
// untouched. Set the capability to 0/empty to disable injection.
|
|
168
|
+
const injectAzureSegmentationTimeout = (body, sttParams, timeoutMs) => {
|
|
169
|
+
const ms = Number(timeoutMs);
|
|
170
|
+
if (!lodash__default["default"].isFinite(ms) || ms <= 0) return body;
|
|
171
|
+
const isAzure = sttParams && sttParams.stt === 'azure' || body && typeof body === 'object' && body.azure || typeof body === 'string' && body.indexOf('azure') !== -1;
|
|
172
|
+
if (!isAzure) return body;
|
|
173
|
+
let next;
|
|
174
|
+
if (body == null) {
|
|
175
|
+
next = {};
|
|
176
|
+
} else if (typeof body === 'string') {
|
|
177
|
+
try {
|
|
178
|
+
next = JSON.parse(body);
|
|
179
|
+
} catch (err) {
|
|
180
|
+
return body;
|
|
181
|
+
}
|
|
182
|
+
} else {
|
|
183
|
+
next = lodash__default["default"].cloneDeep(body);
|
|
184
|
+
}
|
|
185
|
+
if (!lodash__default["default"].isObject(next.azure)) next.azure = {};
|
|
186
|
+
if (!lodash__default["default"].isObject(next.azure.config)) next.azure.config = {};
|
|
187
|
+
if (!lodash__default["default"].isObject(next.azure.config.properties)) next.azure.config.properties = {};
|
|
188
|
+
if (next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs == null) {
|
|
189
|
+
next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs = String(Math.round(ms));
|
|
190
|
+
}
|
|
191
|
+
return next;
|
|
120
192
|
};
|
|
121
193
|
const TTS_HTTP_AGENT = new http__default["default"].Agent({
|
|
122
194
|
keepAlive: true
|
|
@@ -145,6 +217,8 @@ class BotiumConnectorVoip {
|
|
|
145
217
|
// For debugging latency between incoming STT (bot says) and outgoing audio (sendAudio)
|
|
146
218
|
this._lastBotSaysQueuedAt = null;
|
|
147
219
|
this._lastBotSaysText = null;
|
|
220
|
+
this._replyTrace = null;
|
|
221
|
+
this._activeUserSaysVoipAgent = null;
|
|
148
222
|
this._speculativeTurnToken = 0;
|
|
149
223
|
}
|
|
150
224
|
async Validate() {
|
|
@@ -193,6 +267,27 @@ class BotiumConnectorVoip {
|
|
|
193
267
|
this.convoStep = null;
|
|
194
268
|
this._lastBotSaysQueuedAt = null;
|
|
195
269
|
this._lastBotSaysText = null;
|
|
270
|
+
this._replyTrace = null;
|
|
271
|
+
this._activeUserSaysVoipAgent = null;
|
|
272
|
+
this.audioStream = {
|
|
273
|
+
format: null,
|
|
274
|
+
pcmParts: [],
|
|
275
|
+
totalBytes: 0,
|
|
276
|
+
complete: false
|
|
277
|
+
};
|
|
278
|
+
this._turnAudioCounter = 0;
|
|
279
|
+
this._meTurnAudioOrdinal = -1; // 0-based ordinal of REAL user turns this session
|
|
280
|
+
this._lastBotTurnStartSec = null;
|
|
281
|
+
// Per-turn audio is NOT sliced inline (that would force UserSays to wait for the
|
|
282
|
+
// recording to catch up with playback). Instead each turn records a lightweight
|
|
283
|
+
// descriptor here; slices are cut and emitted as MESSAGE_ATTACHMENT progressively,
|
|
284
|
+
// as soon as the recording buffer has reached each turn's playback end (so the live
|
|
285
|
+
// UI can show them mid-run), with a forced flush at session end for any remainder.
|
|
286
|
+
// Zero added latency for TTS and DTMF turns.
|
|
287
|
+
this._pendingTurnAudio = [];
|
|
288
|
+
this._turnAudioPrevEndSec = null;
|
|
289
|
+
this._turnAudioForceDone = false;
|
|
290
|
+
this._trailingBotAudioEmitted = false;
|
|
196
291
|
if (this.ttsCache) {
|
|
197
292
|
this.ttsCache.clear();
|
|
198
293
|
}
|
|
@@ -200,6 +295,7 @@ class BotiumConnectorVoip {
|
|
|
200
295
|
const queuedAt = Date.now();
|
|
201
296
|
this._lastBotSaysQueuedAt = queuedAt;
|
|
202
297
|
this._lastBotSaysText = botMsg && botMsg.messageText ? String(botMsg.messageText) : null;
|
|
298
|
+
this._captureBotQueuedForReplyTrace(queuedAt);
|
|
203
299
|
// Stamp the wall-clock instant at which the connector released the bot
|
|
204
300
|
// utterance to botium-core's queue. Paired with `_receivedAtMs` (last
|
|
205
301
|
// STT final frame) this gives the true "join silence" the connector
|
|
@@ -207,8 +303,33 @@ class BotiumConnectorVoip {
|
|
|
207
303
|
// message up with WaitBotSays().
|
|
208
304
|
if (botMsg && botMsg.sourceData) {
|
|
209
305
|
const head = Array.isArray(botMsg.sourceData) ? botMsg.sourceData[0] : botMsg.sourceData;
|
|
210
|
-
if (head && typeof head === 'object'
|
|
211
|
-
head.flushedAtMs = queuedAt;
|
|
306
|
+
if (head && typeof head === 'object') {
|
|
307
|
+
if (!('flushedAtMs' in head)) head.flushedAtMs = queuedAt;
|
|
308
|
+
if (this._replyTrace) {
|
|
309
|
+
if (lodash__default["default"].isFinite(this._replyTrace.psstFireDelayMs)) head.psstFireDelayMs = this._replyTrace.psstFireDelayMs;
|
|
310
|
+
if (lodash__default["default"].isFinite(this._replyTrace.psstScheduledMs)) head.psstScheduledMs = this._replyTrace.psstScheduledMs;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
// A turn = bot speaks first, then me responds.
|
|
315
|
+
// Save the bot's audio start so UserSays can slice the full exchange
|
|
316
|
+
// (bot + me) once me has finished speaking.
|
|
317
|
+
if (this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE] && botMsg && !(botMsg instanceof Error)) {
|
|
318
|
+
try {
|
|
319
|
+
const sd = botMsg.sourceData;
|
|
320
|
+
let startSec = null;
|
|
321
|
+
if (Array.isArray(sd) && sd.length > 0) {
|
|
322
|
+
startSec = lodash__default["default"].get(sd, '[0].data.start', null);
|
|
323
|
+
} else if (sd && typeof sd === 'object') {
|
|
324
|
+
startSec = lodash__default["default"].get(sd, 'data.start', null);
|
|
325
|
+
}
|
|
326
|
+
// Only record the FIRST bot message's start; if several bot messages
|
|
327
|
+
// arrive before the next UserSays they all belong to the same turn.
|
|
328
|
+
if (lodash__default["default"].isFinite(startSec) && this._lastBotTurnStartSec === null) {
|
|
329
|
+
this._lastBotTurnStartSec = startSec;
|
|
330
|
+
}
|
|
331
|
+
} catch (err) {
|
|
332
|
+
debug$3(`${this.sessionId} - sendBotMsg: saving turn start error: ${err && err.message}`);
|
|
212
333
|
}
|
|
213
334
|
}
|
|
214
335
|
setTimeout(() => this.queueBotSays(botMsg), 0);
|
|
@@ -229,6 +350,10 @@ class BotiumConnectorVoip {
|
|
|
229
350
|
botMsg.sourceData[0].silenceDuration = lodash__default["default"].isFinite(firstStart) ? firstStart : null;
|
|
230
351
|
botMsg.sourceData[0].voiceDuration = lodash__default["default"].isFinite(firstStart) && lodash__default["default"].isFinite(lastEnd) ? lastEnd - firstStart : null;
|
|
231
352
|
}
|
|
353
|
+
const lastSpeechEnd = lodash__default["default"].get(botMsgs, `[${botMsgs.length - 1}].sourceData.data.speechEndSec`, null);
|
|
354
|
+
if (lodash__default["default"].isFinite(lastSpeechEnd) && botMsg.sourceData[0] && botMsg.sourceData[0].data) {
|
|
355
|
+
botMsg.sourceData[0].data.speechEndSec = lastSpeechEnd;
|
|
356
|
+
}
|
|
232
357
|
return botMsg;
|
|
233
358
|
};
|
|
234
359
|
|
|
@@ -264,6 +389,10 @@ class BotiumConnectorVoip {
|
|
|
264
389
|
}
|
|
265
390
|
const bufferedAtArm = this.botMsgs.length;
|
|
266
391
|
const armedAt = Date.now();
|
|
392
|
+
this._markReplyTrace({
|
|
393
|
+
psstTimerArmedAtMs: armedAt,
|
|
394
|
+
psstScheduledMs: joinTimeoutMs || 0
|
|
395
|
+
});
|
|
267
396
|
_info('psst_timer_armed', {
|
|
268
397
|
sessionId: this.sessionId,
|
|
269
398
|
joinTimeoutMs: joinTimeoutMs || 0,
|
|
@@ -287,6 +416,10 @@ class BotiumConnectorVoip {
|
|
|
287
416
|
}
|
|
288
417
|
this.silenceTimeout = setTimeout(() => {
|
|
289
418
|
const fireDelay = Date.now() - armedAt;
|
|
419
|
+
this._markReplyTrace({
|
|
420
|
+
psstTimerFiredAtMs: Date.now(),
|
|
421
|
+
psstFireDelayMs: fireDelay
|
|
422
|
+
});
|
|
290
423
|
if (this.botMsgs.length > 0) {
|
|
291
424
|
_info('psst_timer_fired', {
|
|
292
425
|
sessionId: this.sessionId,
|
|
@@ -438,15 +571,17 @@ class BotiumConnectorVoip {
|
|
|
438
571
|
await connectHttp(retryIndex + 1);
|
|
439
572
|
}
|
|
440
573
|
};
|
|
441
|
-
await connectHttp();
|
|
442
|
-
if (httpInitRetries > 0) {
|
|
443
|
-
_info('connected_after_retries', {
|
|
444
|
-
phase: 'initCall',
|
|
445
|
-
retries: httpInitRetries
|
|
446
|
-
});
|
|
447
|
-
}
|
|
448
574
|
return new Promise((resolve, reject) => {
|
|
449
|
-
|
|
575
|
+
// Worker session details (port) come from connectHttp() and are recomputed
|
|
576
|
+
// on every (re)try, since each setup attempt gets a fresh worker session.
|
|
577
|
+
let wsEndpoint = null;
|
|
578
|
+
const computeWsEndpoint = () => `${this.caps[Capabilities.VOIP_USE_GLOBAL_VOIP_WORKER] ? process.env.BOTIUM_VOIP_WORKER_URL : this.caps[Capabilities.VOIP_WORKER_URL]}/ws/${data.port}`;
|
|
579
|
+
// 487-retry bookkeeping: a transient '487 Request Terminated' that arrives
|
|
580
|
+
// before the call connects re-runs the whole setup (fresh initCall -> ws
|
|
581
|
+
// -> SIP INVITE) up to max487Retries times.
|
|
582
|
+
let setup487Retries = 0;
|
|
583
|
+
let convoStepListenerAttached = false;
|
|
584
|
+
const max487Retries = this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_MAXRETRIES];
|
|
450
585
|
const connectWs = retryIndex => {
|
|
451
586
|
retryIndex = retryIndex || 0;
|
|
452
587
|
return new Promise((resolve, reject) => {
|
|
@@ -480,7 +615,7 @@ class BotiumConnectorVoip {
|
|
|
480
615
|
});
|
|
481
616
|
});
|
|
482
617
|
};
|
|
483
|
-
|
|
618
|
+
const onWsConnected = wsRetries => {
|
|
484
619
|
if (wsRetries > 0) {
|
|
485
620
|
_info('connected_after_retries', {
|
|
486
621
|
phase: 'websocket',
|
|
@@ -494,26 +629,32 @@ class BotiumConnectorVoip {
|
|
|
494
629
|
this.caps[Capabilities.VOIP_ICE_STUN_SERVERS] = this.caps[Capabilities.VOIP_ICE_STUN_SERVERS].split(',');
|
|
495
630
|
}
|
|
496
631
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
632
|
+
|
|
633
|
+
// Attach this once: the listener reads this.ws dynamically, so it keeps
|
|
634
|
+
// working across 487 setup retries that swap out the websocket.
|
|
635
|
+
if (!convoStepListenerAttached) {
|
|
636
|
+
convoStepListenerAttached = true;
|
|
637
|
+
this.eventEmitter.on('CONVO_STEP_NEXT', (container, convoStep) => {
|
|
638
|
+
this.convoStep = convoStep;
|
|
639
|
+
this._maybePrefetchTts(convoStep);
|
|
640
|
+
// For PSST: send join silence duration per step to VOIP worker (controls PSST silence trigger)
|
|
641
|
+
try {
|
|
642
|
+
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'PSST' && this.ws && this.ws.readyState === ws__default["default"].OPEN) {
|
|
643
|
+
const silenceMs = this._getEffectiveJoinTimeoutMs(convoStep, this.botMsgs);
|
|
644
|
+
if (lodash__default["default"].isFinite(silenceMs) && silenceMs > 0 && this.sessionId) {
|
|
645
|
+
debug$3(`PSST: sending silenceDurationMs=${silenceMs} for sessionId=${this.sessionId}`);
|
|
646
|
+
this.ws.send(JSON.stringify({
|
|
647
|
+
METHOD: 'setSttSilenceDuration',
|
|
648
|
+
sessionId: this.sessionId,
|
|
649
|
+
silenceDurationMs: silenceMs
|
|
650
|
+
}));
|
|
651
|
+
}
|
|
511
652
|
}
|
|
653
|
+
} catch (err) {
|
|
654
|
+
debug$3(`Failed sending PSST silence duration to VOIP worker: ${err.message || err}`);
|
|
512
655
|
}
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
}
|
|
516
|
-
});
|
|
656
|
+
});
|
|
657
|
+
}
|
|
517
658
|
this.silence = null;
|
|
518
659
|
this.msgCount = 0;
|
|
519
660
|
this.sttPartialCount = 0;
|
|
@@ -561,11 +702,12 @@ class BotiumConnectorVoip {
|
|
|
561
702
|
ICE_TURN_PROTOCOL: this.caps[Capabilities.VOIP_ICE_TURN_PROTOCOL] || 'TCP',
|
|
562
703
|
MIN_SILENCE_DURATION: this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT_ENABLE] ? this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT] : null,
|
|
563
704
|
SDP_MEDIA_TYPE_TEXT_ENABLE: !!this.caps[Capabilities.VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE],
|
|
705
|
+
AUDIO_STREAM: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE],
|
|
564
706
|
STT_LEGACY: sttLegacy,
|
|
565
707
|
STT_CONFIG: {
|
|
566
708
|
stt_url: sttUrl,
|
|
567
709
|
stt_params: this.caps[Capabilities.VOIP_STT_PARAMS_STREAM],
|
|
568
|
-
stt_body: this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null
|
|
710
|
+
stt_body: injectAzureSegmentationTimeout(this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null, this.caps[Capabilities.VOIP_STT_PARAMS_STREAM], this.caps[Capabilities.VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS])
|
|
569
711
|
},
|
|
570
712
|
TTS_CONFIG: {
|
|
571
713
|
tts_url: this.caps[Capabilities.VOIP_TTS_URL],
|
|
@@ -630,7 +772,7 @@ class BotiumConnectorVoip {
|
|
|
630
772
|
// (fullRecord*) and hard errors so `full_record.wav` is delivered
|
|
631
773
|
// on early-completion hangups. Post-Stop STT frames remain blocked.
|
|
632
774
|
if (this.stopCalled) {
|
|
633
|
-
const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error'];
|
|
775
|
+
const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error', 'audioStreamStart', 'audioStreamChunk', 'audioStreamEnd'];
|
|
634
776
|
if (!parsedData || !allowedPostStopTypes.includes(parsedData.type)) {
|
|
635
777
|
debug$3(`${this.sessionId} - Stop already called, ignoring incoming message`);
|
|
636
778
|
return;
|
|
@@ -642,7 +784,7 @@ class BotiumConnectorVoip {
|
|
|
642
784
|
if (!obj || typeof obj !== 'object') return;
|
|
643
785
|
for (const key of Object.keys(obj)) {
|
|
644
786
|
const val = obj[key];
|
|
645
|
-
if (typeof val === 'string' && val.length > 500) {
|
|
787
|
+
if (typeof val === 'string' && val.length > 0 && (WS_DEBUG_BASE64_FIELD_NAMES.has(key) || val.length > 500)) {
|
|
646
788
|
obj[key] = `<base64:${val.length}chars>`;
|
|
647
789
|
} else if (val && typeof val === 'object' && !Array.isArray(val)) {
|
|
648
790
|
sanitizeBase64Fields(val, `${prefix}${key}.`);
|
|
@@ -650,7 +792,9 @@ class BotiumConnectorVoip {
|
|
|
650
792
|
}
|
|
651
793
|
};
|
|
652
794
|
sanitizeBase64Fields(parsedDataLog);
|
|
653
|
-
|
|
795
|
+
if (!WS_DEBUG_SILENT_TYPES.has(parsedData?.type)) {
|
|
796
|
+
debug$3(JSON.stringify(parsedDataLog, null, 2));
|
|
797
|
+
}
|
|
654
798
|
const _extractFullRecordBase64 = pd => {
|
|
655
799
|
if (!pd) return null;
|
|
656
800
|
// Different VOIP workers may put the payload in various fields - search all string fields
|
|
@@ -754,6 +898,9 @@ class BotiumConnectorVoip {
|
|
|
754
898
|
reject(new Error('Error: Sip Registration failed'));
|
|
755
899
|
}
|
|
756
900
|
if (parsedData && parsedData.type === 'callinfo' && parsedData.status === 'connected') {
|
|
901
|
+
// Mark connected so a later terminal error is delivered to the bot
|
|
902
|
+
// conversation instead of triggering a (now pointless) setup retry.
|
|
903
|
+
this.connected = true;
|
|
757
904
|
_info('callinfo_connected', {
|
|
758
905
|
sessionId: this.sessionId
|
|
759
906
|
});
|
|
@@ -788,6 +935,23 @@ class BotiumConnectorVoip {
|
|
|
788
935
|
});
|
|
789
936
|
}
|
|
790
937
|
if (parsedData && parsedData.type === 'error') {
|
|
938
|
+
const errMsg = parsedData.message || '';
|
|
939
|
+
// The worker reports the SIP code inside the message string
|
|
940
|
+
// ("Disconnected because of error - Reason: 487 Request Terminated")
|
|
941
|
+
// and does not set a dedicated `code` field, so detect 487 from both.
|
|
942
|
+
const is487 = parsedData.code === 487 || parsedData.code === '487' || /\b487\b/.test(errMsg) || /request terminated/i.test(errMsg);
|
|
943
|
+
// A transient '487 Request Terminated' that arrives before the call
|
|
944
|
+
// connects: retry the whole call setup instead of failing the test.
|
|
945
|
+
if (is487 && !this.connected && setup487Retries < max487Retries) {
|
|
946
|
+
_info('ws_error_msg', {
|
|
947
|
+
sessionId: this.sessionId,
|
|
948
|
+
message: parsedData.message || null,
|
|
949
|
+
code: parsedData.code || null,
|
|
950
|
+
retrying487: true
|
|
951
|
+
});
|
|
952
|
+
retryCallSetup487(errMsg);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
791
955
|
flushPendingBotMsgs('error');
|
|
792
956
|
// Ensure buffered recording is not lost on terminal worker errors.
|
|
793
957
|
this._emitBufferedFullRecordIfAny('error_buffered');
|
|
@@ -801,6 +965,46 @@ class BotiumConnectorVoip {
|
|
|
801
965
|
sendBotMsg(new Error(`Error: ${parsedData.message}`));
|
|
802
966
|
}
|
|
803
967
|
|
|
968
|
+
// Per-turn audio stream: continuous PCM chunks received during the call.
|
|
969
|
+
// The connector buffers them so _sliceTurnAudio() can extract per-turn segments.
|
|
970
|
+
if (parsedData && parsedData.type === 'audioStreamStart') {
|
|
971
|
+
this.audioStream = {
|
|
972
|
+
format: {
|
|
973
|
+
sampleRate: parsedData.sampleRate,
|
|
974
|
+
channels: parsedData.channels,
|
|
975
|
+
bitsPerSample: parsedData.bitsPerSample,
|
|
976
|
+
dataOffset: parsedData.dataOffset
|
|
977
|
+
},
|
|
978
|
+
pcmParts: [],
|
|
979
|
+
totalBytes: 0,
|
|
980
|
+
complete: false
|
|
981
|
+
};
|
|
982
|
+
debug$3(`${this.sessionId} - audioStreamStart sampleRate=${parsedData.sampleRate} channels=${parsedData.channels} bitsPerSample=${parsedData.bitsPerSample}`);
|
|
983
|
+
}
|
|
984
|
+
if (parsedData && parsedData.type === 'audioStreamChunk') {
|
|
985
|
+
if (this.audioStream && parsedData.chunk) {
|
|
986
|
+
try {
|
|
987
|
+
const buf = Buffer.from(parsedData.chunk, 'base64');
|
|
988
|
+
this.audioStream.pcmParts.push(buf);
|
|
989
|
+
this.audioStream.totalBytes += buf.length;
|
|
990
|
+
this._maybeDetectAgentAudibleOnRecording(this._activeUserSaysVoipAgent);
|
|
991
|
+
// Emit any per-turn audio whose playback the recording has now caught up to,
|
|
992
|
+
// so the live transcript can show it mid-run (no UserSays latency).
|
|
993
|
+
this._emitReadyTurnAudio('audioStreamChunk', false);
|
|
994
|
+
} catch (e) {
|
|
995
|
+
debug$3(`${this.sessionId} - audioStreamChunk decode error: ${e && e.message}`);
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
if (parsedData && parsedData.type === 'audioStreamEnd') {
|
|
1000
|
+
if (this.audioStream) {
|
|
1001
|
+
this.audioStream.complete = true;
|
|
1002
|
+
}
|
|
1003
|
+
debug$3(`${this.sessionId} - audioStreamEnd totalBytes=${parsedData.totalBytes}`);
|
|
1004
|
+
// Buffer is complete — cut and emit the per-turn audio now.
|
|
1005
|
+
this._flushPendingTurnAudio('audioStreamEnd');
|
|
1006
|
+
}
|
|
1007
|
+
|
|
804
1008
|
// Full record streaming support:
|
|
805
1009
|
// - some VOIP workers send the recording in chunks and an end marker
|
|
806
1010
|
if (parsedData && parsedData.type === 'fullRecordStart') {
|
|
@@ -820,11 +1024,49 @@ class BotiumConnectorVoip {
|
|
|
820
1024
|
source: 'fullRecordEnd',
|
|
821
1025
|
base64Len
|
|
822
1026
|
});
|
|
1027
|
+
// Emit per-turn audio before `this.end = true` so it is captured by the
|
|
1028
|
+
// worker before Stop() resolves (no-op if audioStreamEnd already flushed).
|
|
1029
|
+
this._flushPendingTurnAudio('fullRecordEnd');
|
|
823
1030
|
// Flush before `this.end = true` so the buffered final STT is not
|
|
824
1031
|
// dropped when Stop() clears the PSST silence timer on teardown.
|
|
825
1032
|
flushPendingBotMsgs('fullRecordEnd');
|
|
826
1033
|
this.end = true;
|
|
827
1034
|
}
|
|
1035
|
+
if (parsedData && parsedData.type === 'agentPlaybackStarted') {
|
|
1036
|
+
const playbackData = parsedData.data || {};
|
|
1037
|
+
const playedSec = playbackData.playedRecordingStartSec;
|
|
1038
|
+
const active = this._activeUserSaysVoipAgent;
|
|
1039
|
+
if (active && lodash__default["default"].isFinite(playedSec)) {
|
|
1040
|
+
active.playedRecordingStartSec = playedSec;
|
|
1041
|
+
active.playbackAtMs = playbackData.playbackAtMs;
|
|
1042
|
+
if (lodash__default["default"].isFinite(playbackData.requestedDurationMs)) {
|
|
1043
|
+
active.playbackRequestedDurationMs = playbackData.requestedDurationMs;
|
|
1044
|
+
}
|
|
1045
|
+
if (lodash__default["default"].isFinite(playbackData.digitCount)) {
|
|
1046
|
+
active.digitCount = playbackData.digitCount;
|
|
1047
|
+
}
|
|
1048
|
+
this._markReplyTrace({
|
|
1049
|
+
playedRecordingStartSec: playedSec,
|
|
1050
|
+
playbackAtMs: playbackData.playbackAtMs
|
|
1051
|
+
});
|
|
1052
|
+
const heardSec = playbackData.wireKind === 'dtmf' ? playedSec : this._applyAgentHeardRecordingStartSec(active);
|
|
1053
|
+
if (lodash__default["default"].isFinite(heardSec)) {
|
|
1054
|
+
if (playbackData.wireKind === 'dtmf') {
|
|
1055
|
+
active.heardRecordingStartSec = heardSec;
|
|
1056
|
+
this._markReplyTrace({
|
|
1057
|
+
heardRecordingStartSec: heardSec
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
1060
|
+
debug$3(`${this.sessionId} - agent audible on recording at ${heardSec}s (played=${playedSec}s)`);
|
|
1061
|
+
}
|
|
1062
|
+
// Log the heard reply-trace now that playback is audible — earlier than the
|
|
1063
|
+
// old turn-audio callback and free of the next-turn _replyTrace race. The
|
|
1064
|
+
// agent's end on the recording = played + playback length.
|
|
1065
|
+
const playbackSec = lodash__default["default"].isFinite(active.playbackRequestedDurationMs) && active.playbackRequestedDurationMs > 0 ? active.playbackRequestedDurationMs / 1000 : playbackData.wireKind === 'dtmf' ? this._dtmfPlaybackSec(active, active.digitCount || 1) : null;
|
|
1066
|
+
const agentEndSec = lodash__default["default"].isFinite(playbackSec) ? playedSec + playbackSec : undefined;
|
|
1067
|
+
this._logReplyTraceHeard(agentEndSec);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
828
1070
|
if (parsedData && parsedData.type === 'silence') {
|
|
829
1071
|
if (lodash__default["default"].isNil(this._getIgnoreSilenceDurationAsserterLogicHook(this.convoStep))) {
|
|
830
1072
|
if (!this._hasJoinLogicHookOrRule(this.convoStep) && parsedData.data.silence.length > 0) {
|
|
@@ -953,8 +1195,13 @@ class BotiumConnectorVoip {
|
|
|
953
1195
|
messageLength: msgLen,
|
|
954
1196
|
confidence: this._getConfidenceScore(parsedData),
|
|
955
1197
|
threshold: confidenceThreshold,
|
|
956
|
-
accepted: successfulConfidenceScore
|
|
1198
|
+
accepted: successfulConfidenceScore,
|
|
1199
|
+
segmentEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.end')) ? parsedData.data.end : null,
|
|
1200
|
+
speechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.speechEndSec')) ? parsedData.data.speechEndSec : null
|
|
957
1201
|
});
|
|
1202
|
+
if (successfulConfidenceScore) {
|
|
1203
|
+
this._captureSttFinalForReplyTrace(parsedData, msgPreview);
|
|
1204
|
+
}
|
|
958
1205
|
// ORIGINAL: always emit final message immediately (ignore JOIN hooks/rules).
|
|
959
1206
|
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'ORIGINAL') {
|
|
960
1207
|
let botMsg = {
|
|
@@ -1051,9 +1298,49 @@ class BotiumConnectorVoip {
|
|
|
1051
1298
|
}
|
|
1052
1299
|
}
|
|
1053
1300
|
});
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1301
|
+
};
|
|
1302
|
+
const retryCallSetup487 = reasonMessage => {
|
|
1303
|
+
setup487Retries++;
|
|
1304
|
+
_info('call_setup_retry_487', {
|
|
1305
|
+
sessionId: this.sessionId,
|
|
1306
|
+
attempt: setup487Retries,
|
|
1307
|
+
max: max487Retries,
|
|
1308
|
+
reason: reasonMessage || null
|
|
1309
|
+
});
|
|
1310
|
+
debug$3(`Call setup 487 retry ${setup487Retries}/${max487Retries}: ${reasonMessage}`);
|
|
1311
|
+
// Tear down the dead websocket so its stale handlers cannot fire while
|
|
1312
|
+
// the next attempt establishes a fresh worker session.
|
|
1313
|
+
try {
|
|
1314
|
+
if (this.ws) {
|
|
1315
|
+
this.ws.removeAllListeners();
|
|
1316
|
+
this.ws.terminate();
|
|
1317
|
+
}
|
|
1318
|
+
} catch (err) {
|
|
1319
|
+
debug$3(`Call setup 487 retry: websocket teardown failed: ${err && err.message}`);
|
|
1320
|
+
}
|
|
1321
|
+
this.ws = null;
|
|
1322
|
+
this.wsOpened = false;
|
|
1323
|
+
this.end = false;
|
|
1324
|
+
setTimeout(() => {
|
|
1325
|
+
establishCall().catch(err => reject(new Error('Error: ' + err)));
|
|
1326
|
+
}, this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_TIMEOUT]);
|
|
1327
|
+
};
|
|
1328
|
+
const establishCall = async () => {
|
|
1329
|
+
// Each (re)try gets a fresh worker session: new initCall -> new port ->
|
|
1330
|
+
// new websocket -> new SIP INVITE.
|
|
1331
|
+
httpInitRetries = 0;
|
|
1332
|
+
await connectHttp();
|
|
1333
|
+
if (httpInitRetries > 0) {
|
|
1334
|
+
_info('connected_after_retries', {
|
|
1335
|
+
phase: 'initCall',
|
|
1336
|
+
retries: httpInitRetries
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
wsEndpoint = computeWsEndpoint();
|
|
1340
|
+
const wsRetries = await connectWs();
|
|
1341
|
+
onWsConnected(wsRetries);
|
|
1342
|
+
};
|
|
1343
|
+
establishCall().catch(err => reject(new Error('Error: ' + err)));
|
|
1057
1344
|
});
|
|
1058
1345
|
}
|
|
1059
1346
|
async UserSays(msg) {
|
|
@@ -1063,6 +1350,10 @@ class BotiumConnectorVoip {
|
|
|
1063
1350
|
const hasDtmf = !!(msg && msg.buttons && msg.buttons.length > 0);
|
|
1064
1351
|
const dtmfMatch = msg && msg.messageText && msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
|
|
1065
1352
|
const inputType = hasDtmf || dtmfMatch ? 'dtmf' : hasText && hasVoiceMedia ? 'mixed' : hasText ? 'text' : hasVoiceMedia ? 'media' : 'unknown';
|
|
1353
|
+
// A real user turn is one that sends content (text/media/dtmf). 'unknown' turns send nothing and
|
|
1354
|
+
// surface server-side as skippable me-steps, so they must NOT consume an ordinal slot - this keeps
|
|
1355
|
+
// meTurnIndex aligned with the server's non-skippable me-step indices when placing turn audio.
|
|
1356
|
+
const meTurnIndex = inputType !== 'unknown' ? this._meTurnAudioOrdinal = (this._meTurnAudioOrdinal ?? -1) + 1 : null;
|
|
1066
1357
|
const msgPreview = hasText && msg.messageText ? String(msg.messageText).trim().substring(0, 80) : '';
|
|
1067
1358
|
_info('user_says', {
|
|
1068
1359
|
sessionId: this.sessionId,
|
|
@@ -1071,6 +1362,7 @@ class BotiumConnectorVoip {
|
|
|
1071
1362
|
messageLength: hasText && msg.messageText ? msg.messageText.length : undefined,
|
|
1072
1363
|
mediaSize: hasVoiceMedia && msg.media[0] && Buffer.isBuffer(msg.media[0].buffer) ? msg.media[0].buffer.length : undefined
|
|
1073
1364
|
});
|
|
1365
|
+
this._captureUserSaysStart(msgPreview);
|
|
1074
1366
|
// Avoid logging large buffers/base64 (can break job logs and overwhelm stdout)
|
|
1075
1367
|
try {
|
|
1076
1368
|
const safeLog = {
|
|
@@ -1105,17 +1397,45 @@ class BotiumConnectorVoip {
|
|
|
1105
1397
|
// the coach can place the agent turn on the recording timeline.
|
|
1106
1398
|
// `requestedDurationMs` is the best estimate of on-wire playback
|
|
1107
1399
|
// length (DTMF tones × digits, TTS synth output, parsed media duration).
|
|
1400
|
+
const recordingSecNow = () => this._recordingSecNow();
|
|
1108
1401
|
const stampAgentWire = (wireKind, requestedDurationMs, extras = {}) => {
|
|
1402
|
+
const wireRecordingStartSec = recordingSecNow();
|
|
1109
1403
|
msg.voipAgent = {
|
|
1110
1404
|
wireSentAtMs: Date.now(),
|
|
1111
1405
|
inputType,
|
|
1112
1406
|
wireKind,
|
|
1113
1407
|
requestedDurationMs: Math.max(0, Math.round(requestedDurationMs || 0)),
|
|
1408
|
+
...(wireRecordingStartSec != null ? {
|
|
1409
|
+
wireRecordingStartSec
|
|
1410
|
+
} : {}),
|
|
1411
|
+
// Carry the coach's reply-decision trace (set on the outgoing message before UserSays)
|
|
1412
|
+
// into voipAgent here, while we own the object and before MESSAGE_SENTTOBOT fires — so
|
|
1413
|
+
// the LIVE transcript snapshot (a deep copy taken at that event) already has it, not
|
|
1414
|
+
// just the final result. Lets the UI split "Tester reply decision" live.
|
|
1415
|
+
...(msg && msg.coachTrace && typeof msg.coachTrace === 'object' ? {
|
|
1416
|
+
coachTrace: msg.coachTrace
|
|
1417
|
+
} : {}),
|
|
1114
1418
|
...extras
|
|
1115
1419
|
};
|
|
1420
|
+
this._activeUserSaysVoipAgent = msg.voipAgent;
|
|
1421
|
+
this._captureAgentWire(msg.voipAgent, inputType);
|
|
1422
|
+
};
|
|
1423
|
+
const sendAgentWire = request => {
|
|
1424
|
+
this._sendUserSaysWs(request);
|
|
1425
|
+
this._markReplyTrace({
|
|
1426
|
+
sendAudioAtMs: Date.now()
|
|
1427
|
+
});
|
|
1428
|
+
this._logReplyTrace('wire_sent');
|
|
1429
|
+
// Finalize the wall pipeline NOW, while _replyTrace still holds this turn's data
|
|
1430
|
+
// (userSaysAtMs / ttsStartAtMs / ttsEndAtMs / wireAtMs / sendAudioAtMs are all set by
|
|
1431
|
+
// this point). The deferred turn-audio callback runs only after the agent audio is
|
|
1432
|
+
// "heard" on the recording, which nulls _replyTrace (_logReplyTraceHeard); finalizing
|
|
1433
|
+
// there would drop wallPipeline entirely and collapse the reply-delay breakdown into a
|
|
1434
|
+
// single "Audio send" bucket on the UI.
|
|
1435
|
+
this._finalizeWallPipeline(msg.voipAgent);
|
|
1116
1436
|
};
|
|
1117
1437
|
// Twilio default: 100 ms tone + 100 ms gap per digit. Drives agent-bar width only.
|
|
1118
|
-
const
|
|
1438
|
+
const DTMF_AGENT_BAR_MS_PER_DIGIT = DTMF_MS_PER_DIGIT;
|
|
1119
1439
|
if (msg && msg.buttons && msg.buttons.length > 0) {
|
|
1120
1440
|
const digits = sanitizeDtmfDigits(msg.buttons[0].payload);
|
|
1121
1441
|
if (!digits) {
|
|
@@ -1128,10 +1448,13 @@ class BotiumConnectorVoip {
|
|
|
1128
1448
|
digits,
|
|
1129
1449
|
sessionId: this.sessionId
|
|
1130
1450
|
});
|
|
1131
|
-
stampAgentWire('dtmf', digits.length *
|
|
1132
|
-
digitCount: digits.length
|
|
1451
|
+
stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
|
|
1452
|
+
digitCount: digits.length,
|
|
1453
|
+
dtmfDigits: digits
|
|
1133
1454
|
});
|
|
1134
|
-
|
|
1455
|
+
sendAgentWire(request);
|
|
1456
|
+
// Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
|
|
1457
|
+
duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
|
|
1135
1458
|
} else if (msg && msg.messageText) {
|
|
1136
1459
|
// Check for DTMF tag in messageText: <DTMF>1234</DTMF>
|
|
1137
1460
|
const dtmfMatch = msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
|
|
@@ -1152,13 +1475,14 @@ class BotiumConnectorVoip {
|
|
|
1152
1475
|
digits,
|
|
1153
1476
|
sessionId: this.sessionId
|
|
1154
1477
|
});
|
|
1155
|
-
stampAgentWire('dtmf', digits.length *
|
|
1156
|
-
digitCount: digits.length
|
|
1478
|
+
stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
|
|
1479
|
+
digitCount: digits.length,
|
|
1480
|
+
dtmfDigits: digits
|
|
1157
1481
|
});
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
if (!skipTtsForMixedInput) {
|
|
1482
|
+
sendAgentWire(request);
|
|
1483
|
+
// Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
|
|
1484
|
+
duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
|
|
1485
|
+
} else if (!skipTtsForMixedInput) {
|
|
1162
1486
|
if (!this.axiosTtsParams) {
|
|
1163
1487
|
if (!(msg.media && msg.media.length > 0 && msg.media[0].buffer)) {
|
|
1164
1488
|
return reject(new Error('TTS not configured, only audio input supported'));
|
|
@@ -1170,10 +1494,17 @@ class BotiumConnectorVoip {
|
|
|
1170
1494
|
msg.sourceData = ttsRequest;
|
|
1171
1495
|
let ttsResult = null;
|
|
1172
1496
|
const ttsStartedAt = Date.now();
|
|
1497
|
+
this._markReplyTrace({
|
|
1498
|
+
ttsStartAtMs: ttsStartedAt
|
|
1499
|
+
});
|
|
1173
1500
|
let ttsSynthMs = 0;
|
|
1174
1501
|
try {
|
|
1175
1502
|
ttsResult = await this._getTtsAudio(ttsRequest, msg.messageText);
|
|
1176
1503
|
ttsSynthMs = Date.now() - ttsStartedAt;
|
|
1504
|
+
this._markReplyTrace({
|
|
1505
|
+
ttsEndAtMs: Date.now(),
|
|
1506
|
+
ttsSynthMs
|
|
1507
|
+
});
|
|
1177
1508
|
} catch (err) {
|
|
1178
1509
|
return reject(new Error(`TTS "${msg.messageText}" failed - ${this._getAxiosErrOutput(err)}`));
|
|
1179
1510
|
}
|
|
@@ -1209,7 +1540,7 @@ class BotiumConnectorVoip {
|
|
|
1209
1540
|
ttsSynthMs,
|
|
1210
1541
|
textLength: msg.messageText ? msg.messageText.length : 0
|
|
1211
1542
|
});
|
|
1212
|
-
|
|
1543
|
+
sendAgentWire(request);
|
|
1213
1544
|
} else {
|
|
1214
1545
|
return reject(new Error('TTS failed, response is empty'));
|
|
1215
1546
|
}
|
|
@@ -1234,7 +1565,7 @@ class BotiumConnectorVoip {
|
|
|
1234
1565
|
stampAgentWire('media', 0, {
|
|
1235
1566
|
mediaUri: msg.media[0].mediaUri || null
|
|
1236
1567
|
});
|
|
1237
|
-
|
|
1568
|
+
sendAgentWire(request);
|
|
1238
1569
|
msg.attachments.push({
|
|
1239
1570
|
name: msg.media[0].mediaUri,
|
|
1240
1571
|
mimeType: msg.media[0].mimeType,
|
|
@@ -1256,16 +1587,514 @@ class BotiumConnectorVoip {
|
|
|
1256
1587
|
}
|
|
1257
1588
|
}
|
|
1258
1589
|
const requestedDurationMs = Math.max(0, Math.round((duration || 0) * 1000));
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1590
|
+
|
|
1591
|
+
// Record this turn's slice bounds and resolve immediately — the actual
|
|
1592
|
+
// slice is cut from the complete buffer at session end (see
|
|
1593
|
+
// _flushPendingTurnAudio). This keeps UserSays from waiting for the
|
|
1594
|
+
// recording to catch up with playback (no latency for TTS or DTMF). The
|
|
1595
|
+
// heard-trace telemetry (voip_reply_trace_heard) is logged from the
|
|
1596
|
+
// agentPlaybackStarted handler once playback is audible on the recording.
|
|
1597
|
+
this._recordPendingTurnAudio(msg.voipAgent, requestedDurationMs, meTurnIndex);
|
|
1598
|
+
resolve();
|
|
1263
1599
|
} catch (err) {
|
|
1264
1600
|
reject(err);
|
|
1265
1601
|
}
|
|
1266
1602
|
}, 0);
|
|
1267
1603
|
});
|
|
1268
1604
|
}
|
|
1605
|
+
_recordingSecNow() {
|
|
1606
|
+
const fmt = this.audioStream && this.audioStream.format;
|
|
1607
|
+
const bytesPerSec = fmt ? fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8) : null;
|
|
1608
|
+
if (!bytesPerSec || !this.audioStream || !(this.audioStream.totalBytes > 0)) return null;
|
|
1609
|
+
return this.audioStream.totalBytes / bytesPerSec;
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
/** Expected on-recording playback length of a DTMF turn (worker-reported, else estimated). */
|
|
1613
|
+
_dtmfPlaybackSec(voipAgent, digitCount) {
|
|
1614
|
+
const playbackMs = voipAgent && voipAgent.playbackRequestedDurationMs;
|
|
1615
|
+
if (lodash__default["default"].isFinite(playbackMs) && playbackMs > 0) return playbackMs / 1000;
|
|
1616
|
+
return dtmfPlaybackMs(digitCount) / 1000;
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
/**
|
|
1620
|
+
* Snapshot the bounds needed to slice this turn's audio later, then return.
|
|
1621
|
+
* `_lastBotTurnStartSec` is the bot-speech anchor; it is consumed here so the
|
|
1622
|
+
* next bot message sets a fresh one. The end of the agent's playback is read
|
|
1623
|
+
* from voipAgent.playedRecordingStartSec (filled asynchronously by the
|
|
1624
|
+
* agentPlaybackStarted handler) at flush time. Consecutive agent turns with no
|
|
1625
|
+
* bot message in between (anchor null) chain off the previous turn's end.
|
|
1626
|
+
*/
|
|
1627
|
+
_recordPendingTurnAudio(voipAgent, requestedDurationMs, meTurnIndex) {
|
|
1628
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1629
|
+
const botAnchorSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
|
|
1630
|
+
this._lastBotTurnStartSec = null;
|
|
1631
|
+
this._pendingTurnAudio.push({
|
|
1632
|
+
botAnchorSec,
|
|
1633
|
+
voipAgent: voipAgent || null,
|
|
1634
|
+
requestedDurationMs: Math.max(0, requestedDurationMs || 0),
|
|
1635
|
+
meTurnIndex: Number.isInteger(meTurnIndex) && meTurnIndex >= 0 ? meTurnIndex : null
|
|
1636
|
+
});
|
|
1637
|
+
_info('turn_audio_recorded', {
|
|
1638
|
+
sessionId: this.sessionId,
|
|
1639
|
+
meTurnIndex,
|
|
1640
|
+
botAnchorSec,
|
|
1641
|
+
wireKind: voipAgent && voipAgent.wireKind,
|
|
1642
|
+
pending: this._pendingTurnAudio.length
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
/**
|
|
1647
|
+
* Emit per-turn audio (turn_N.wav) for every pending turn whose playback end the
|
|
1648
|
+
* recording buffer has already reached, slicing from the live buffer and emitting a
|
|
1649
|
+
* MESSAGE_ATTACHMENT carrying meTurnIndex. Turns are processed in order (the chain
|
|
1650
|
+
* start of a follow-up turn with no bot anchor depends on the previous turn's end), so
|
|
1651
|
+
* a not-yet-ready turn stops the pass until more audio streams in. With force=true
|
|
1652
|
+
* (session end) the remainder is emitted even if the buffer is incomplete.
|
|
1653
|
+
*/
|
|
1654
|
+
_emitReadyTurnAudio(reason, force) {
|
|
1655
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1656
|
+
const pending = this._pendingTurnAudio || [];
|
|
1657
|
+
if (!pending.length) return;
|
|
1658
|
+
if (!this.audioStream || !this.audioStream.format) return; // no PCM yet
|
|
1659
|
+
const slackSec = (audioStreamIntervalMs() + 50) / 1000;
|
|
1660
|
+
const recNow = this._recordingSecNow();
|
|
1661
|
+
for (const t of pending) {
|
|
1662
|
+
if (t.emitted) continue;
|
|
1663
|
+
try {
|
|
1664
|
+
const va = t.voipAgent || {};
|
|
1665
|
+
const playbackSec = va.wireKind === 'dtmf' ? this._dtmfPlaybackSec(va, va.digitCount || 1) : lodash__default["default"].isFinite(va.playbackRequestedDurationMs) && va.playbackRequestedDurationMs > 0 ? va.playbackRequestedDurationMs / 1000 : t.requestedDurationMs / 1000;
|
|
1666
|
+
const startSec = lodash__default["default"].isFinite(t.botAnchorSec) ? t.botAnchorSec : this._turnAudioPrevEndSec;
|
|
1667
|
+
// End = where the agent's playback finished on the recording. _sliceTurnAudio
|
|
1668
|
+
// adds the configured padding on top.
|
|
1669
|
+
const anchorSec = lodash__default["default"].isFinite(va.playedRecordingStartSec) ? va.playedRecordingStartSec : lodash__default["default"].isFinite(va.heardRecordingStartSec) ? va.heardRecordingStartSec : lodash__default["default"].isFinite(va.wireRecordingStartSec) ? va.wireRecordingStartSec : null;
|
|
1670
|
+
let endSec = lodash__default["default"].isFinite(anchorSec) ? anchorSec + playbackSec : null;
|
|
1671
|
+
if (!lodash__default["default"].isFinite(endSec) && lodash__default["default"].isFinite(startSec)) endSec = startSec + playbackSec;
|
|
1672
|
+
|
|
1673
|
+
// Progressive: only emit once the recording has reached this turn's end (plus a
|
|
1674
|
+
// stream-flush slack). Stop the pass otherwise — later turns must wait their turn.
|
|
1675
|
+
if (!force) {
|
|
1676
|
+
if (!lodash__default["default"].isFinite(anchorSec) || !lodash__default["default"].isFinite(endSec)) break;
|
|
1677
|
+
if (!lodash__default["default"].isFinite(recNow) || recNow < endSec + slackSec) break;
|
|
1678
|
+
}
|
|
1679
|
+
if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) {
|
|
1680
|
+
debug$3(`${this.sessionId} - turnAudio: skip turn (start=${startSec} end=${endSec} reason=${reason})`);
|
|
1681
|
+
t.emitted = true;
|
|
1682
|
+
if (lodash__default["default"].isFinite(endSec)) this._turnAudioPrevEndSec = endSec;
|
|
1683
|
+
continue;
|
|
1684
|
+
}
|
|
1685
|
+
const audioBase64 = this._sliceTurnAudio(startSec, endSec);
|
|
1686
|
+
t.emitted = true;
|
|
1687
|
+
this._turnAudioPrevEndSec = endSec;
|
|
1688
|
+
if (!audioBase64) continue;
|
|
1689
|
+
this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
|
|
1690
|
+
this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
|
|
1691
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1692
|
+
mimeType: 'audio/wav',
|
|
1693
|
+
base64: audioBase64,
|
|
1694
|
+
meTurnIndex: t.meTurnIndex,
|
|
1695
|
+
// 0-based real-user-turn ordinal (authoritative for placement)
|
|
1696
|
+
sessionContext: {
|
|
1697
|
+
testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
|
|
1698
|
+
testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
|
|
1699
|
+
}
|
|
1700
|
+
});
|
|
1701
|
+
_info('turn_audio_emitted', {
|
|
1702
|
+
sessionId: this.sessionId,
|
|
1703
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1704
|
+
meTurnIndex: t.meTurnIndex,
|
|
1705
|
+
startSec: Number(startSec.toFixed(2)),
|
|
1706
|
+
endSec: Number(endSec.toFixed(2)),
|
|
1707
|
+
reason
|
|
1708
|
+
});
|
|
1709
|
+
} catch (err) {
|
|
1710
|
+
debug$3(`${this.sessionId} - emitReadyTurnAudio: turn slice error: ${err && err.message}`);
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
/**
|
|
1716
|
+
* Force-emit any per-turn audio not yet sent (session end). Idempotent.
|
|
1717
|
+
*/
|
|
1718
|
+
_flushPendingTurnAudio(reason) {
|
|
1719
|
+
if (this._turnAudioForceDone) return;
|
|
1720
|
+
this._turnAudioForceDone = true;
|
|
1721
|
+
const pending = this._pendingTurnAudio || [];
|
|
1722
|
+
_info('turn_audio_flush_enter', {
|
|
1723
|
+
sessionId: this.sessionId,
|
|
1724
|
+
reason,
|
|
1725
|
+
pending: pending.length,
|
|
1726
|
+
emittedAlready: pending.filter(t => t.emitted).length,
|
|
1727
|
+
hasFormat: !!(this.audioStream && this.audioStream.format),
|
|
1728
|
+
totalBytes: this.audioStream && this.audioStream.totalBytes,
|
|
1729
|
+
turnAudioEnable: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]
|
|
1730
|
+
});
|
|
1731
|
+
this._emitReadyTurnAudio(reason, true);
|
|
1732
|
+
this._emitTrailingBotAudio(reason);
|
|
1733
|
+
_info('turn_audio_flush_done', {
|
|
1734
|
+
sessionId: this.sessionId,
|
|
1735
|
+
reason,
|
|
1736
|
+
emitted: this._turnAudioCounter || 0,
|
|
1737
|
+
pending: pending.length
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
/**
|
|
1742
|
+
* If the call ended on a bot turn (a bot message arrived with no me-response after it),
|
|
1743
|
+
* `_lastBotTurnStartSec` was never consumed by a turn. Slice that trailing bot audio
|
|
1744
|
+
* (bot start → end of recording) and emit it as a turn clip flagged `trailingBot` so the
|
|
1745
|
+
* UI/server place it on the final bot step.
|
|
1746
|
+
*/
|
|
1747
|
+
_emitTrailingBotAudio(reason) {
|
|
1748
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1749
|
+
if (this._trailingBotAudioEmitted) return;
|
|
1750
|
+
const startSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
|
|
1751
|
+
if (!lodash__default["default"].isFinite(startSec)) return;
|
|
1752
|
+
if (!this.audioStream || !this.audioStream.format) return;
|
|
1753
|
+
const endSec = this._recordingSecNow();
|
|
1754
|
+
if (!lodash__default["default"].isFinite(endSec) || endSec <= startSec) return;
|
|
1755
|
+
let audioBase64 = null;
|
|
1756
|
+
try {
|
|
1757
|
+
audioBase64 = this._sliceTurnAudio(startSec, endSec);
|
|
1758
|
+
} catch (err) {
|
|
1759
|
+
debug$3(`${this.sessionId} - emitTrailingBotAudio: slice error: ${err && err.message}`);
|
|
1760
|
+
return;
|
|
1761
|
+
}
|
|
1762
|
+
if (!audioBase64) return;
|
|
1763
|
+
this._trailingBotAudioEmitted = true;
|
|
1764
|
+
this._lastBotTurnStartSec = null;
|
|
1765
|
+
this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
|
|
1766
|
+
this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
|
|
1767
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1768
|
+
mimeType: 'audio/wav',
|
|
1769
|
+
base64: audioBase64,
|
|
1770
|
+
trailingBot: true,
|
|
1771
|
+
// place on the final bot step (no me-response followed)
|
|
1772
|
+
sessionContext: {
|
|
1773
|
+
testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
|
|
1774
|
+
testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
|
|
1775
|
+
}
|
|
1776
|
+
});
|
|
1777
|
+
_info('turn_audio_trailing_emitted', {
|
|
1778
|
+
sessionId: this.sessionId,
|
|
1779
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1780
|
+
startSec: Number(startSec.toFixed(2)),
|
|
1781
|
+
endSec: Number(endSec.toFixed(2)),
|
|
1782
|
+
reason
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
_agentSpeechRmsThreshold() {
|
|
1786
|
+
const raw = process.env.VOIP_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1787
|
+
const n = raw != null ? Number(raw) : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1788
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1789
|
+
}
|
|
1790
|
+
_agentSpeechSustainedWindows() {
|
|
1791
|
+
const raw = process.env.VOIP_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1792
|
+
const n = raw != null ? parseInt(raw, 10) : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1793
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1794
|
+
}
|
|
1795
|
+
_audioStreamBytesPerSec() {
|
|
1796
|
+
const fmt = this.audioStream && this.audioStream.format;
|
|
1797
|
+
if (!fmt) return null;
|
|
1798
|
+
return fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8);
|
|
1799
|
+
}
|
|
1800
|
+
_pcmBufferRms(pcm, bitsPerSample) {
|
|
1801
|
+
if (!pcm || pcm.length < 2 || bitsPerSample !== 16) return 0;
|
|
1802
|
+
let sum = 0;
|
|
1803
|
+
let count = 0;
|
|
1804
|
+
for (let i = 0; i + 1 < pcm.length; i += 2) {
|
|
1805
|
+
const sample = pcm.readInt16LE(i);
|
|
1806
|
+
sum += sample * sample;
|
|
1807
|
+
count += 1;
|
|
1808
|
+
}
|
|
1809
|
+
return count > 0 ? Math.sqrt(sum / count) : 0;
|
|
1810
|
+
}
|
|
1811
|
+
_readWavPcmInfo(wavBuffer) {
|
|
1812
|
+
if (!wavBuffer || wavBuffer.length < 44) return null;
|
|
1813
|
+
if (wavBuffer.toString('ascii', 0, 4) !== 'RIFF' || wavBuffer.toString('ascii', 8, 12) !== 'WAVE') {
|
|
1814
|
+
return null;
|
|
1815
|
+
}
|
|
1816
|
+
let offset = 12;
|
|
1817
|
+
let sampleRate = null;
|
|
1818
|
+
let channels = null;
|
|
1819
|
+
let bitsPerSample = null;
|
|
1820
|
+
let dataOffset = null;
|
|
1821
|
+
let dataLength = null;
|
|
1822
|
+
while (offset + 8 <= wavBuffer.length) {
|
|
1823
|
+
const chunkId = wavBuffer.toString('ascii', offset, offset + 4);
|
|
1824
|
+
const chunkSize = wavBuffer.readUInt32LE(offset + 4);
|
|
1825
|
+
const chunkStart = offset + 8;
|
|
1826
|
+
if (chunkId === 'fmt ' && chunkSize >= 16) {
|
|
1827
|
+
channels = wavBuffer.readUInt16LE(chunkStart + 2);
|
|
1828
|
+
sampleRate = wavBuffer.readUInt32LE(chunkStart + 4);
|
|
1829
|
+
bitsPerSample = wavBuffer.readUInt16LE(chunkStart + 14);
|
|
1830
|
+
} else if (chunkId === 'data') {
|
|
1831
|
+
dataOffset = chunkStart;
|
|
1832
|
+
dataLength = chunkSize;
|
|
1833
|
+
break;
|
|
1834
|
+
}
|
|
1835
|
+
offset = chunkStart + chunkSize + chunkSize % 2;
|
|
1836
|
+
}
|
|
1837
|
+
if (!sampleRate || !channels || !bitsPerSample || dataOffset == null) return null;
|
|
1838
|
+
const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
|
|
1839
|
+
if (!bytesPerSec) return null;
|
|
1840
|
+
const pcmLength = dataLength != null ? Math.min(dataLength, wavBuffer.length - dataOffset) : wavBuffer.length - dataOffset;
|
|
1841
|
+
return {
|
|
1842
|
+
pcmOffset: dataOffset,
|
|
1843
|
+
pcmLength,
|
|
1844
|
+
bytesPerSec,
|
|
1845
|
+
bitsPerSample,
|
|
1846
|
+
sampleRate,
|
|
1847
|
+
channels
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
_findAudibleLeadInSecFromPcm(pcm, bytesPerSec, bitsPerSample) {
|
|
1851
|
+
if (!pcm || !bytesPerSec) return null;
|
|
1852
|
+
const threshold = this._agentSpeechRmsThreshold();
|
|
1853
|
+
const sustainedWindows = this._agentSpeechSustainedWindows();
|
|
1854
|
+
const windowBytes = Math.max(2, Math.floor(bytesPerSec * (AGENT_SPEECH_RMS_WINDOW_MS / 1000)));
|
|
1855
|
+
const hopBytes = Math.max(2, Math.floor(windowBytes / 2));
|
|
1856
|
+
let streak = 0;
|
|
1857
|
+
let onsetPos = null;
|
|
1858
|
+
for (let pos = 0; pos + windowBytes <= pcm.length; pos += hopBytes) {
|
|
1859
|
+
const rms = this._pcmBufferRms(pcm.subarray(pos, pos + windowBytes), bitsPerSample);
|
|
1860
|
+
if (rms >= threshold) {
|
|
1861
|
+
if (streak === 0) onsetPos = pos;
|
|
1862
|
+
streak += 1;
|
|
1863
|
+
if (streak >= sustainedWindows) {
|
|
1864
|
+
return onsetPos / bytesPerSec;
|
|
1865
|
+
}
|
|
1866
|
+
} else {
|
|
1867
|
+
streak = 0;
|
|
1868
|
+
onsetPos = null;
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
return null;
|
|
1872
|
+
}
|
|
1873
|
+
_findAudibleLeadInSecFromWavBuffer(wavBuffer) {
|
|
1874
|
+
const info = this._readWavPcmInfo(wavBuffer);
|
|
1875
|
+
if (!info) return null;
|
|
1876
|
+
const pcm = wavBuffer.subarray(info.pcmOffset, info.pcmOffset + info.pcmLength);
|
|
1877
|
+
return this._findAudibleLeadInSecFromPcm(pcm, info.bytesPerSec, info.bitsPerSample);
|
|
1878
|
+
}
|
|
1879
|
+
_findAudibleRecordingStartSecOnStream(playedSec, wireSec) {
|
|
1880
|
+
if (!lodash__default["default"].isFinite(playedSec)) return null;
|
|
1881
|
+
const bytesPerSec = this._audioStreamBytesPerSec();
|
|
1882
|
+
const stream = this.audioStream;
|
|
1883
|
+
if (!bytesPerSec || !stream || !stream.pcmParts.length) return null;
|
|
1884
|
+
const startByte = Math.max(0, Math.floor(playedSec * bytesPerSec));
|
|
1885
|
+
const pcm = Buffer.concat(stream.pcmParts);
|
|
1886
|
+
if (startByte >= pcm.length) return null;
|
|
1887
|
+
const bitsPerSample = stream.format.bitsPerSample;
|
|
1888
|
+
const leadInSec = this._findAudibleLeadInSecFromPcm(pcm.subarray(startByte), bytesPerSec, bitsPerSample);
|
|
1889
|
+
if (!lodash__default["default"].isFinite(leadInSec)) return null;
|
|
1890
|
+
let heardSec = playedSec + leadInSec;
|
|
1891
|
+
if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
|
|
1892
|
+
return heardSec;
|
|
1893
|
+
}
|
|
1894
|
+
_findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments) {
|
|
1895
|
+
if (!lodash__default["default"].isFinite(playedSec) || !lodash__default["default"].isArray(attachments)) return null;
|
|
1896
|
+
const tts = attachments.find(a => a && a.name === 'tts.wav' && a.base64);
|
|
1897
|
+
if (!tts) return null;
|
|
1898
|
+
try {
|
|
1899
|
+
const wavBuffer = Buffer.from(tts.base64, 'base64');
|
|
1900
|
+
const leadInSec = this._findAudibleLeadInSecFromWavBuffer(wavBuffer);
|
|
1901
|
+
if (!lodash__default["default"].isFinite(leadInSec)) return null;
|
|
1902
|
+
let heardSec = playedSec + leadInSec;
|
|
1903
|
+
if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
|
|
1904
|
+
return heardSec;
|
|
1905
|
+
} catch (err) {
|
|
1906
|
+
debug$3(`${this.sessionId} - TTS lead-in scan failed: ${err && err.message}`);
|
|
1907
|
+
return null;
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
_resolveAgentHeardRecordingStartSec(voipAgent, attachments) {
|
|
1911
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
|
|
1912
|
+
const playedSec = voipAgent.playedRecordingStartSec;
|
|
1913
|
+
const wireSec = voipAgent.wireRecordingStartSec;
|
|
1914
|
+
const fromTts = this._findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments);
|
|
1915
|
+
const fromStream = this._findAudibleRecordingStartSecOnStream(playedSec, wireSec);
|
|
1916
|
+
const candidates = [fromTts, fromStream].filter(s => lodash__default["default"].isFinite(s));
|
|
1917
|
+
if (!candidates.length) return null;
|
|
1918
|
+
// Prefer the later onset — mixed recording can spike before clear TTS speech.
|
|
1919
|
+
return Math.max(...candidates);
|
|
1920
|
+
}
|
|
1921
|
+
_applyAgentHeardRecordingStartSec(voipAgent, attachments) {
|
|
1922
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
|
|
1923
|
+
const heardSec = this._resolveAgentHeardRecordingStartSec(voipAgent, attachments);
|
|
1924
|
+
if (!lodash__default["default"].isFinite(heardSec) || heardSec <= voipAgent.playedRecordingStartSec) {
|
|
1925
|
+
return lodash__default["default"].isFinite(voipAgent.heardRecordingStartSec) ? voipAgent.heardRecordingStartSec : null;
|
|
1926
|
+
}
|
|
1927
|
+
const prev = voipAgent.heardRecordingStartSec;
|
|
1928
|
+
if (lodash__default["default"].isFinite(prev) && prev >= heardSec) return prev;
|
|
1929
|
+
voipAgent.heardRecordingStartSec = heardSec;
|
|
1930
|
+
this._markReplyTrace({
|
|
1931
|
+
heardRecordingStartSec: heardSec
|
|
1932
|
+
});
|
|
1933
|
+
return heardSec;
|
|
1934
|
+
}
|
|
1935
|
+
_maybeDetectAgentAudibleOnRecording(voipAgent) {
|
|
1936
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return;
|
|
1937
|
+
this._applyAgentHeardRecordingStartSec(voipAgent);
|
|
1938
|
+
}
|
|
1939
|
+
_markReplyTrace(patch) {
|
|
1940
|
+
if (!this._replyTrace || !patch) return;
|
|
1941
|
+
Object.assign(this._replyTrace, patch);
|
|
1942
|
+
}
|
|
1943
|
+
_captureSttFinalForReplyTrace(parsedData, msgPreview) {
|
|
1944
|
+
const data = parsedData && parsedData.data;
|
|
1945
|
+
const atMs = parsedData._receivedAtMs || Date.now();
|
|
1946
|
+
const recordingAtSttFinalSec = this._recordingSecNow();
|
|
1947
|
+
if (parsedData && lodash__default["default"].isFinite(recordingAtSttFinalSec)) {
|
|
1948
|
+
parsedData.recordingAtSttFinalSec = recordingAtSttFinalSec;
|
|
1949
|
+
}
|
|
1950
|
+
// Dedicated, self-documenting anchor for the downstream "STT transport" sub-phase
|
|
1951
|
+
// (receivedAtMs - finalEmittedWallMs). Unlike the generic _receivedAtMs (stamped on
|
|
1952
|
+
// every WS frame), this is set only on the accepted STT-final.
|
|
1953
|
+
if (parsedData && lodash__default["default"].isFinite(atMs)) {
|
|
1954
|
+
parsedData.sttFinalReceivedAtMs = atMs;
|
|
1955
|
+
}
|
|
1956
|
+
this._replyTrace = {
|
|
1957
|
+
sessionId: this.sessionId,
|
|
1958
|
+
botMessagePreview: msgPreview || undefined,
|
|
1959
|
+
sttFinalAtMs: atMs,
|
|
1960
|
+
sttRecordingStartSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'start')) ? data.start : null,
|
|
1961
|
+
sttRecordingEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'end')) ? data.end : null,
|
|
1962
|
+
sttSpeechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'speechEndSec')) ? data.speechEndSec : null,
|
|
1963
|
+
recordingAtSttFinalSec: lodash__default["default"].isFinite(recordingAtSttFinalSec) ? recordingAtSttFinalSec : null,
|
|
1964
|
+
queueAtMs: null,
|
|
1965
|
+
recordingAtQueueSec: null,
|
|
1966
|
+
psstTimerArmedAtMs: null,
|
|
1967
|
+
psstScheduledMs: null,
|
|
1968
|
+
psstTimerFiredAtMs: null,
|
|
1969
|
+
psstFireDelayMs: null,
|
|
1970
|
+
userSaysAtMs: null,
|
|
1971
|
+
coachWaitMs: null,
|
|
1972
|
+
ttsStartAtMs: null,
|
|
1973
|
+
ttsEndAtMs: null,
|
|
1974
|
+
ttsSynthMs: null,
|
|
1975
|
+
wireAtMs: null,
|
|
1976
|
+
wireRecordingStartSec: null,
|
|
1977
|
+
sendAudioAtMs: null,
|
|
1978
|
+
playedRecordingStartSec: null,
|
|
1979
|
+
playbackAtMs: null,
|
|
1980
|
+
heardRecordingStartSec: null,
|
|
1981
|
+
agentEndRecordingSec: null,
|
|
1982
|
+
wireKind: null,
|
|
1983
|
+
inputType: null,
|
|
1984
|
+
requestedDurationMs: null,
|
|
1985
|
+
meMessagePreview: null
|
|
1986
|
+
};
|
|
1987
|
+
}
|
|
1988
|
+
_captureBotQueuedForReplyTrace(queuedAt) {
|
|
1989
|
+
if (!this._replyTrace) return;
|
|
1990
|
+
this._replyTrace.queueAtMs = queuedAt;
|
|
1991
|
+
this._replyTrace.recordingAtQueueSec = this._recordingSecNow();
|
|
1992
|
+
}
|
|
1993
|
+
_captureUserSaysStart(msgPreview) {
|
|
1994
|
+
if (!this._replyTrace) return;
|
|
1995
|
+
const now = Date.now();
|
|
1996
|
+
this._replyTrace.userSaysAtMs = now;
|
|
1997
|
+
this._replyTrace.meMessagePreview = msgPreview || undefined;
|
|
1998
|
+
const queueAt = this._replyTrace.queueAtMs || this._lastBotSaysQueuedAt;
|
|
1999
|
+
if (lodash__default["default"].isFinite(queueAt)) {
|
|
2000
|
+
if (!this._replyTrace.queueAtMs) this._replyTrace.queueAtMs = queueAt;
|
|
2001
|
+
this._replyTrace.coachWaitMs = now - queueAt;
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
_captureAgentWire(voipAgent, inputType) {
|
|
2005
|
+
if (!this._replyTrace || !voipAgent) return;
|
|
2006
|
+
this._replyTrace.wireAtMs = voipAgent.wireSentAtMs;
|
|
2007
|
+
this._replyTrace.wireRecordingStartSec = voipAgent.wireRecordingStartSec;
|
|
2008
|
+
this._replyTrace.wireKind = voipAgent.wireKind;
|
|
2009
|
+
this._replyTrace.inputType = inputType;
|
|
2010
|
+
this._replyTrace.requestedDurationMs = voipAgent.requestedDurationMs;
|
|
2011
|
+
if (lodash__default["default"].isFinite(voipAgent.ttsSynthMs)) this._replyTrace.ttsSynthMs = voipAgent.ttsSynthMs;
|
|
2012
|
+
}
|
|
2013
|
+
_finalizeWallPipeline(voipAgent) {
|
|
2014
|
+
const t = this._replyTrace;
|
|
2015
|
+
if (!voipAgent || !t) return;
|
|
2016
|
+
voipAgent.wallPipeline = {
|
|
2017
|
+
psstScheduledMs: lodash__default["default"].isFinite(t.psstScheduledMs) ? t.psstScheduledMs : null,
|
|
2018
|
+
psstFireDelayMs: lodash__default["default"].isFinite(t.psstFireDelayMs) ? t.psstFireDelayMs : null,
|
|
2019
|
+
coachWaitMs: lodash__default["default"].isFinite(t.coachWaitMs) ? t.coachWaitMs : null,
|
|
2020
|
+
userSaysAtMs: lodash__default["default"].isFinite(t.userSaysAtMs) ? t.userSaysAtMs : null,
|
|
2021
|
+
ttsStartAtMs: lodash__default["default"].isFinite(t.ttsStartAtMs) ? t.ttsStartAtMs : null,
|
|
2022
|
+
ttsEndAtMs: lodash__default["default"].isFinite(t.ttsEndAtMs) ? t.ttsEndAtMs : null,
|
|
2023
|
+
ttsSynthMs: lodash__default["default"].isFinite(t.ttsSynthMs) ? t.ttsSynthMs : null,
|
|
2024
|
+
wireAtMs: lodash__default["default"].isFinite(t.wireAtMs) ? t.wireAtMs : null,
|
|
2025
|
+
sendAudioAtMs: lodash__default["default"].isFinite(t.sendAudioAtMs) ? t.sendAudioAtMs : null
|
|
2026
|
+
};
|
|
2027
|
+
}
|
|
2028
|
+
_replyTraceMsFromSttFinal(atMs) {
|
|
2029
|
+
const anchor = this._replyTrace && this._replyTrace.sttFinalAtMs;
|
|
2030
|
+
if (!lodash__default["default"].isFinite(anchor) || !lodash__default["default"].isFinite(atMs)) return null;
|
|
2031
|
+
return Math.round(atMs - anchor);
|
|
2032
|
+
}
|
|
2033
|
+
_replyTraceRecMs(fromSec, toSec) {
|
|
2034
|
+
if (!lodash__default["default"].isFinite(fromSec) || !lodash__default["default"].isFinite(toSec)) return null;
|
|
2035
|
+
return Math.round((toSec - fromSec) * 1000);
|
|
2036
|
+
}
|
|
2037
|
+
_logReplyTrace(trigger) {
|
|
2038
|
+
const t = this._replyTrace;
|
|
2039
|
+
if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
|
|
2040
|
+
_info('voip_reply_trace', {
|
|
2041
|
+
sessionId: t.sessionId,
|
|
2042
|
+
trigger,
|
|
2043
|
+
botPreview: t.botMessagePreview,
|
|
2044
|
+
mePreview: t.meMessagePreview,
|
|
2045
|
+
sttRecordingStartSec: t.sttRecordingStartSec,
|
|
2046
|
+
sttRecordingEndSec: t.sttRecordingEndSec,
|
|
2047
|
+
sttSpeechEndSec: t.sttSpeechEndSec,
|
|
2048
|
+
recordingAtSttFinalSec: t.recordingAtSttFinalSec,
|
|
2049
|
+
recordingAtQueueSec: t.recordingAtQueueSec,
|
|
2050
|
+
wireRecordingStartSec: t.wireRecordingStartSec,
|
|
2051
|
+
wireKind: t.wireKind,
|
|
2052
|
+
inputType: t.inputType,
|
|
2053
|
+
requestedDurationMs: t.requestedDurationMs,
|
|
2054
|
+
ttsSynthMs: t.ttsSynthMs,
|
|
2055
|
+
coachWaitMs: t.coachWaitMs,
|
|
2056
|
+
psstScheduledMs: t.psstScheduledMs,
|
|
2057
|
+
psstFireDelayMs: t.psstFireDelayMs,
|
|
2058
|
+
ms_sttFinal_to_queue: this._replyTraceMsFromSttFinal(t.queueAtMs),
|
|
2059
|
+
ms_sttFinal_to_psstFire: this._replyTraceMsFromSttFinal(t.psstTimerFiredAtMs),
|
|
2060
|
+
ms_sttFinal_to_userSays: this._replyTraceMsFromSttFinal(t.userSaysAtMs),
|
|
2061
|
+
ms_sttFinal_to_ttsStart: this._replyTraceMsFromSttFinal(t.ttsStartAtMs),
|
|
2062
|
+
ms_sttFinal_to_ttsEnd: this._replyTraceMsFromSttFinal(t.ttsEndAtMs),
|
|
2063
|
+
ms_sttFinal_to_wire: this._replyTraceMsFromSttFinal(t.wireAtMs),
|
|
2064
|
+
ms_sttFinal_to_sendAudio: this._replyTraceMsFromSttFinal(t.sendAudioAtMs),
|
|
2065
|
+
ms_userSays_to_ttsStart: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.ttsStartAtMs) ? Math.round(t.ttsStartAtMs - t.userSaysAtMs) : null,
|
|
2066
|
+
ms_userSays_to_wire: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.wireAtMs) ? Math.round(t.wireAtMs - t.userSaysAtMs) : null,
|
|
2067
|
+
ms_queue_to_userSays: t.coachWaitMs,
|
|
2068
|
+
recMs_sttEnd_to_queue: this._replyTraceRecMs(t.sttRecordingEndSec, t.recordingAtQueueSec),
|
|
2069
|
+
recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
|
|
2070
|
+
recMs_speechEnd_to_wire: this._replyTraceRecMs(t.sttSpeechEndSec, t.wireRecordingStartSec)
|
|
2071
|
+
});
|
|
2072
|
+
}
|
|
2073
|
+
_logReplyTraceHeard(agentEndRecordingSec) {
|
|
2074
|
+
const t = this._replyTrace;
|
|
2075
|
+
if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
|
|
2076
|
+
const heardSec = t.heardRecordingStartSec;
|
|
2077
|
+
const playedSec = t.playedRecordingStartSec;
|
|
2078
|
+
if (lodash__default["default"].isFinite(agentEndRecordingSec)) {
|
|
2079
|
+
t.agentEndRecordingSec = agentEndRecordingSec;
|
|
2080
|
+
}
|
|
2081
|
+
_info('voip_reply_trace_heard', {
|
|
2082
|
+
sessionId: t.sessionId,
|
|
2083
|
+
playedRecordingStartSec: playedSec,
|
|
2084
|
+
heardRecordingStartSec: heardSec,
|
|
2085
|
+
agentEndRecordingSec: t.agentEndRecordingSec,
|
|
2086
|
+
wireRecordingStartSec: t.wireRecordingStartSec,
|
|
2087
|
+
sttRecordingEndSec: t.sttRecordingEndSec,
|
|
2088
|
+
sttSpeechEndSec: t.sttSpeechEndSec,
|
|
2089
|
+
recMs_sttEnd_to_played: this._replyTraceRecMs(t.sttRecordingEndSec, playedSec),
|
|
2090
|
+
recMs_speechEnd_to_played: this._replyTraceRecMs(t.sttSpeechEndSec, playedSec),
|
|
2091
|
+
recMs_sttEnd_to_heard: this._replyTraceRecMs(t.sttRecordingEndSec, heardSec),
|
|
2092
|
+
recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
|
|
2093
|
+
recMs_wire_to_played: this._replyTraceRecMs(t.wireRecordingStartSec, playedSec),
|
|
2094
|
+
recMs_wire_to_heard: this._replyTraceRecMs(t.wireRecordingStartSec, heardSec)
|
|
2095
|
+
});
|
|
2096
|
+
this._replyTrace = null;
|
|
2097
|
+
}
|
|
1269
2098
|
_voipWsCanSend() {
|
|
1270
2099
|
return !this.stopCalled && this.ws && this.ws.readyState === ws__default["default"].OPEN;
|
|
1271
2100
|
}
|
|
@@ -1318,6 +2147,8 @@ class BotiumConnectorVoip {
|
|
|
1318
2147
|
if (typeof this._emitBufferedFullRecordIfAny === 'function') {
|
|
1319
2148
|
this._emitBufferedFullRecordIfAny('stop_final_guard');
|
|
1320
2149
|
}
|
|
2150
|
+
// Last-resort flush in case neither audioStreamEnd nor fullRecordEnd fired.
|
|
2151
|
+
this._flushPendingTurnAudio('stop_final_guard');
|
|
1321
2152
|
}
|
|
1322
2153
|
this._emitBufferedFullRecordIfAny = null;
|
|
1323
2154
|
}
|
|
@@ -1662,6 +2493,100 @@ class BotiumConnectorVoip {
|
|
|
1662
2493
|
}
|
|
1663
2494
|
return null;
|
|
1664
2495
|
}
|
|
2496
|
+
|
|
2497
|
+
/**
|
|
2498
|
+
* Build a well-formed WAV Buffer from raw PCM bytes and a format descriptor.
|
|
2499
|
+
* @param {Buffer} pcm raw PCM bytes (no header)
|
|
2500
|
+
* @param {{ sampleRate: number, channels: number, bitsPerSample: number }} fmt
|
|
2501
|
+
* @returns {Buffer}
|
|
2502
|
+
*/
|
|
2503
|
+
_buildWavBuffer(pcm, fmt) {
|
|
2504
|
+
const {
|
|
2505
|
+
sampleRate,
|
|
2506
|
+
channels,
|
|
2507
|
+
bitsPerSample
|
|
2508
|
+
} = fmt;
|
|
2509
|
+
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
|
2510
|
+
const blockAlign = channels * (bitsPerSample / 8);
|
|
2511
|
+
const dataSize = pcm.length;
|
|
2512
|
+
const header = Buffer.alloc(44);
|
|
2513
|
+
header.write('RIFF', 0);
|
|
2514
|
+
header.writeUInt32LE(36 + dataSize, 4);
|
|
2515
|
+
header.write('WAVE', 8);
|
|
2516
|
+
header.write('fmt ', 12);
|
|
2517
|
+
header.writeUInt32LE(16, 16); // fmt chunk size
|
|
2518
|
+
header.writeUInt16LE(1, 20); // PCM format
|
|
2519
|
+
header.writeUInt16LE(channels, 22);
|
|
2520
|
+
header.writeUInt32LE(sampleRate, 24);
|
|
2521
|
+
header.writeUInt32LE(byteRate, 28);
|
|
2522
|
+
header.writeUInt16LE(blockAlign, 32);
|
|
2523
|
+
header.writeUInt16LE(bitsPerSample, 34);
|
|
2524
|
+
header.write('data', 36);
|
|
2525
|
+
header.writeUInt32LE(dataSize, 40);
|
|
2526
|
+
return Buffer.concat([header, pcm]);
|
|
2527
|
+
}
|
|
2528
|
+
|
|
2529
|
+
/**
|
|
2530
|
+
* Slice a segment of the continuously buffered PCM audio stream and return
|
|
2531
|
+
* it as a base64-encoded WAV string.
|
|
2532
|
+
*
|
|
2533
|
+
* @param {number} startSec start of the segment (seconds from call connect)
|
|
2534
|
+
* @param {number} endSec end of the segment (seconds from call connect)
|
|
2535
|
+
* @returns {string|null} base64 WAV or null if the stream is not ready
|
|
2536
|
+
*/
|
|
2537
|
+
_sliceTurnAudio(startSec, endSec) {
|
|
2538
|
+
const stream = this.audioStream;
|
|
2539
|
+
if (!stream || !stream.format || !stream.pcmParts || !stream.pcmParts.length) return null;
|
|
2540
|
+
if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) return null;
|
|
2541
|
+
const {
|
|
2542
|
+
sampleRate,
|
|
2543
|
+
channels,
|
|
2544
|
+
bitsPerSample
|
|
2545
|
+
} = stream.format;
|
|
2546
|
+
const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
|
|
2547
|
+
const frameBytes = channels * (bitsPerSample / 8);
|
|
2548
|
+
const offsetSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_OFFSET_MS] || 0) / 1000;
|
|
2549
|
+
const paddingSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_PADDING_MS] || 0) / 1000;
|
|
2550
|
+
const adjStart = Math.max(0, startSec + offsetSec);
|
|
2551
|
+
const adjEnd = endSec + paddingSec;
|
|
2552
|
+
|
|
2553
|
+
// Frame-align the byte boundaries.
|
|
2554
|
+
const startByte = Math.floor(adjStart * bytesPerSec / frameBytes) * frameBytes;
|
|
2555
|
+
const endByte = Math.ceil(adjEnd * bytesPerSec / frameBytes) * frameBytes;
|
|
2556
|
+
if (startByte >= stream.totalBytes) {
|
|
2557
|
+
debug$3(`${this.sessionId} - _sliceTurnAudio: startByte ${startByte} >= totalBytes ${stream.totalBytes}, skipping`);
|
|
2558
|
+
return null;
|
|
2559
|
+
}
|
|
2560
|
+
const clampedEnd = Math.min(endByte, stream.totalBytes);
|
|
2561
|
+
const sliceLen = clampedEnd - startByte;
|
|
2562
|
+
if (sliceLen <= 0) return null;
|
|
2563
|
+
|
|
2564
|
+
// Materialise only the bytes we need from the part list.
|
|
2565
|
+
const pcm = Buffer.allocUnsafe(sliceLen);
|
|
2566
|
+
let written = 0;
|
|
2567
|
+
let offset = 0;
|
|
2568
|
+
for (const part of stream.pcmParts) {
|
|
2569
|
+
const partEnd = offset + part.length;
|
|
2570
|
+
if (partEnd <= startByte) {
|
|
2571
|
+
offset += part.length;
|
|
2572
|
+
continue;
|
|
2573
|
+
}
|
|
2574
|
+
if (offset >= clampedEnd) break;
|
|
2575
|
+
const copyFrom = Math.max(0, startByte - offset);
|
|
2576
|
+
const copyTo = Math.min(part.length, clampedEnd - offset);
|
|
2577
|
+
part.copy(pcm, written, copyFrom, copyTo);
|
|
2578
|
+
written += copyTo - copyFrom;
|
|
2579
|
+
offset += part.length;
|
|
2580
|
+
}
|
|
2581
|
+
if (written === 0) return null;
|
|
2582
|
+
const slicedPcm = written < sliceLen ? pcm.slice(0, written) : pcm;
|
|
2583
|
+
const wavBuf = this._buildWavBuffer(slicedPcm, {
|
|
2584
|
+
sampleRate,
|
|
2585
|
+
channels,
|
|
2586
|
+
bitsPerSample
|
|
2587
|
+
});
|
|
2588
|
+
return wavBuf.toString('base64');
|
|
2589
|
+
}
|
|
1665
2590
|
}
|
|
1666
2591
|
var connector = BotiumConnectorVoip;
|
|
1667
2592
|
|
|
@@ -1745,6 +2670,24 @@ var botiumConnectorVoip = {
|
|
|
1745
2670
|
type: 'boolean',
|
|
1746
2671
|
required: false,
|
|
1747
2672
|
advanced: true
|
|
2673
|
+
}, {
|
|
2674
|
+
name: 'VOIP_TURN_AUDIO_ENABLE',
|
|
2675
|
+
label: 'Attach per-turn audio to each transcript message',
|
|
2676
|
+
type: 'boolean',
|
|
2677
|
+
required: false,
|
|
2678
|
+
advanced: true
|
|
2679
|
+
}, {
|
|
2680
|
+
name: 'VOIP_TURN_AUDIO_PADDING_MS',
|
|
2681
|
+
label: 'Extra milliseconds appended after each turn audio slice (absorbs STT boundary jitter)',
|
|
2682
|
+
type: 'int',
|
|
2683
|
+
required: false,
|
|
2684
|
+
advanced: true
|
|
2685
|
+
}, {
|
|
2686
|
+
name: 'VOIP_TURN_AUDIO_OFFSET_MS',
|
|
2687
|
+
label: 'Millisecond offset applied to every turn audio start time (positive = shift right)',
|
|
2688
|
+
type: 'int',
|
|
2689
|
+
required: false,
|
|
2690
|
+
advanced: true
|
|
1748
2691
|
}]
|
|
1749
2692
|
},
|
|
1750
2693
|
PluginLogicHooks: {
|