botium-connector-voip 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/botium-connector-voip-cjs.js +1121 -87
- package/dist/botium-connector-voip-cjs.js.map +1 -1
- package/dist/botium-connector-voip-es.js +1121 -87
- package/dist/botium-connector-voip-es.js.map +1 -1
- package/index.js +35 -0
- package/package.json +1 -1
- package/src/connector.js +1085 -85
|
@@ -26,11 +26,17 @@ const debug$3 = debug__default["default"]('botium-connector-voip');
|
|
|
26
26
|
// debug = high-frequency diagnostics (DEBUG=botium-connector-voip). warn = degraded but continuing.
|
|
27
27
|
// error = abort/failure. No secrets in info; STT text only as length or truncated in info.
|
|
28
28
|
|
|
29
|
+
/** WS frame types logged only at start/end handlers — not per-chunk (hundreds per call). */
|
|
30
|
+
const WS_DEBUG_SILENT_TYPES = new Set(['audioStreamChunk', 'fullRecordChunk']);
|
|
31
|
+
const AGENT_SPEECH_RMS_WINDOW_MS = 100;
|
|
32
|
+
const DEFAULT_AGENT_SPEECH_RMS_THRESHOLD = 500;
|
|
33
|
+
const DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS = 2;
|
|
34
|
+
const WS_DEBUG_BASE64_FIELD_NAMES = new Set(['chunk', 'buffer', 'base64', 'fullRecord', 'full_record', 'audio', 'audioData', 'b64_buffer']);
|
|
29
35
|
const _info = (event, data) => {
|
|
30
36
|
const parts = Object.entries({
|
|
31
37
|
event,
|
|
32
38
|
...data
|
|
33
|
-
}).filter(([, v]) => v != null && v !== '').map(([k, v]) => `${k}=${
|
|
39
|
+
}).filter(([, v]) => v != null && v !== '').map(([k, v]) => `${k}=${JSON.stringify(v)}`);
|
|
34
40
|
console.info(`[botium-connector-voip] ${parts.join(' ')}`);
|
|
35
41
|
};
|
|
36
42
|
|
|
@@ -39,18 +45,41 @@ const sanitizeDtmfDigits = raw => {
|
|
|
39
45
|
if (raw == null) return '';
|
|
40
46
|
return String(raw).replace(/[^0-9*#ABCDabcd]/g, '').toUpperCase();
|
|
41
47
|
};
|
|
48
|
+
|
|
49
|
+
// Matches voipcall.generate_dtmf_sequence defaults (100 ms tone, 50 ms pause between digits).
|
|
50
|
+
const DTMF_TONE_MS = 100;
|
|
51
|
+
const DTMF_PAUSE_MS = 50;
|
|
52
|
+
const DTMF_MS_PER_DIGIT = 200;
|
|
53
|
+
const DEFAULT_AUDIO_STREAM_INTERVAL_MS = 250;
|
|
54
|
+
const audioStreamIntervalMs = () => {
|
|
55
|
+
const n = Number(process.env.VOIP_AUDIO_STREAM_INTERVAL_MS || process.env.AUDIO_STREAM_INTERVAL_MS);
|
|
56
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AUDIO_STREAM_INTERVAL_MS;
|
|
57
|
+
};
|
|
58
|
+
const dtmfPlaybackMs = digitCount => {
|
|
59
|
+
if (!digitCount || digitCount <= 0) return 0;
|
|
60
|
+
return digitCount * DTMF_TONE_MS + Math.max(0, digitCount - 1) * DTMF_PAUSE_MS;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/** Wait before turn-audio slice so DTMF PCM is flushed through audioStream (250 ms default). */
|
|
64
|
+
const dtmfTurnAudioWaitMs = digitCount => {
|
|
65
|
+
const playbackMs = dtmfPlaybackMs(digitCount);
|
|
66
|
+
const streamMs = audioStreamIntervalMs();
|
|
67
|
+
return Math.max(digitCount * DTMF_MS_PER_DIGIT, playbackMs + streamMs + 50);
|
|
68
|
+
};
|
|
42
69
|
const Capabilities = {
|
|
43
70
|
VOIP_STT_URL_STREAM: 'VOIP_STT_URL_STREAM',
|
|
44
71
|
VOIP_STT_PARAMS_STREAM: 'VOIP_STT_PARAMS_STREAM',
|
|
45
72
|
VOIP_STT_METHOD_STREAM: 'VOIP_STT_METHOD_STREAM',
|
|
46
73
|
VOIP_STT_BODY_STREAM: 'VOIP_STT_BODY_STREAM',
|
|
47
74
|
VOIP_STT_BODY: 'VOIP_STT_BODY',
|
|
75
|
+
VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 'VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS',
|
|
48
76
|
VOIP_STT_HEADERS: 'VOIP_STT_HEADERS',
|
|
49
77
|
VOIP_STT_TIMEOUT: 'VOIP_STT_TIMEOUT',
|
|
50
78
|
VOIP_STT_MESSAGE_HANDLING: 'VOIP_STT_MESSAGE_HANDLING',
|
|
51
79
|
VOIP_STT_MESSAGE_HANDLING_TIMEOUT: 'VOIP_STT_MESSAGE_HANDLING_TIMEOUT',
|
|
52
80
|
VOIP_STT_MESSAGE_HANDLING_TIMEOUT_SUBSEQUENT: 'VOIP_STT_MESSAGE_HANDLING_TIMEOUT_SUBSEQUENT',
|
|
53
81
|
VOIP_JOIN_SILENCE_DURATION_BY_SUBSTRING: 'VOIP_JOIN_SILENCE_DURATION_BY_SUBSTRING',
|
|
82
|
+
VOIP_STT_DICTIONARY_REPLACEMENTS: 'VOIP_STT_DICTIONARY_REPLACEMENTS',
|
|
54
83
|
VOIP_STT_MESSAGE_HANDLING_DELIMITER: 'VOIP_STT_MESSAGE_HANDLING_DELIMITER',
|
|
55
84
|
VOIP_STT_MESSAGE_HANDLING_PUNCTUATION: 'VOIP_STT_MESSAGE_HANDLING_PUNCTUATION',
|
|
56
85
|
VOIP_TTS_URL: 'VOIP_TTS_URL',
|
|
@@ -83,6 +112,8 @@ const Capabilities = {
|
|
|
83
112
|
VOIP_ICE_TURN_PROTOCOL: 'VOIP_ICE_TURN_PROTOCOL',
|
|
84
113
|
VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 'VOIP_WEBSOCKET_CONNECT_MAXRETRIES',
|
|
85
114
|
VOIP_WEBSOCKET_CONNECT_TIMEOUT: 'VOIP_WEBSOCKET_CONNECT_TIMEOUT',
|
|
115
|
+
VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 'VOIP_CALL_SETUP_RETRY_487_MAXRETRIES',
|
|
116
|
+
VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 'VOIP_CALL_SETUP_RETRY_487_TIMEOUT',
|
|
86
117
|
VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_ENABLE',
|
|
87
118
|
VOIP_SILENCE_DURATION_TIMEOUT: 'VOIP_SILENCE_DURATION_TIMEOUT',
|
|
88
119
|
VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE',
|
|
@@ -90,7 +121,11 @@ const Capabilities = {
|
|
|
90
121
|
VOIP_STT_CONFIDENCE_THRESHOLD: 'VOIP_STT_CONFIDENCE_THRESHOLD',
|
|
91
122
|
VOIP_USE_GLOBAL_VOIP_WORKER: 'VOIP_USE_GLOBAL_VOIP_WORKER',
|
|
92
123
|
VOIP_USER_INPUT_PREFER_VOICE: 'VOIP_USER_INPUT_PREFER_VOICE',
|
|
93
|
-
VOIP_EMIT_SPECULATIVE_TEXT: 'VOIP_EMIT_SPECULATIVE_TEXT'
|
|
124
|
+
VOIP_EMIT_SPECULATIVE_TEXT: 'VOIP_EMIT_SPECULATIVE_TEXT',
|
|
125
|
+
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE',
|
|
126
|
+
VOIP_TURN_AUDIO_ENABLE: 'VOIP_TURN_AUDIO_ENABLE',
|
|
127
|
+
VOIP_TURN_AUDIO_PADDING_MS: 'VOIP_TURN_AUDIO_PADDING_MS',
|
|
128
|
+
VOIP_TURN_AUDIO_OFFSET_MS: 'VOIP_TURN_AUDIO_OFFSET_MS'
|
|
94
129
|
};
|
|
95
130
|
const Defaults = {
|
|
96
131
|
VOIP_STT_METHOD: 'POST',
|
|
@@ -106,14 +141,54 @@ const Defaults = {
|
|
|
106
141
|
VOIP_STT_MESSAGE_HANDLING_PUNCTUATION: '.!?',
|
|
107
142
|
VOIP_WEBSOCKET_CONNECT_TIMEOUT: 4000,
|
|
108
143
|
VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 5,
|
|
144
|
+
// Retry the whole call setup when the SIP peer terminates the INVITE with a
|
|
145
|
+
// transient '487 Request Terminated' before the call connects.
|
|
146
|
+
VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 2,
|
|
147
|
+
VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 2000,
|
|
109
148
|
VOIP_SILENCE_DURATION_TIMEOUT: 2500,
|
|
110
149
|
VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: false,
|
|
111
150
|
VOIP_SILENCE_DURATION_TIMEOUT_START: 1000,
|
|
112
151
|
VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: false,
|
|
113
152
|
VOIP_STT_CONFIDENCE_THRESHOLD: 0.5,
|
|
153
|
+
VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 500,
|
|
114
154
|
VOIP_USE_GLOBAL_VOIP_WORKER: false,
|
|
115
155
|
VOIP_SIP_PROTOCOL: 'TCP',
|
|
116
|
-
VOIP_USER_INPUT_PREFER_VOICE: true
|
|
156
|
+
VOIP_USER_INPUT_PREFER_VOICE: true,
|
|
157
|
+
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false,
|
|
158
|
+
VOIP_TURN_AUDIO_ENABLE: true,
|
|
159
|
+
VOIP_TURN_AUDIO_PADDING_MS: 150,
|
|
160
|
+
VOIP_TURN_AUDIO_OFFSET_MS: 0
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// Inject the Azure end-of-speech segmentation timeout into the STT body. botium-speech-processing
|
|
164
|
+
// applies azure.config.properties via SpeechConfig.setProperty(), so this controls how long Azure
|
|
165
|
+
// waits after the last word before emitting final=true. Only applied for the Azure engine, never
|
|
166
|
+
// overrides a value already present in the profile config, and clones so the capability object stays
|
|
167
|
+
// untouched. Set the capability to 0/empty to disable injection.
|
|
168
|
+
const injectAzureSegmentationTimeout = (body, sttParams, timeoutMs) => {
|
|
169
|
+
const ms = Number(timeoutMs);
|
|
170
|
+
if (!lodash__default["default"].isFinite(ms) || ms <= 0) return body;
|
|
171
|
+
const isAzure = sttParams && sttParams.stt === 'azure' || body && typeof body === 'object' && body.azure || typeof body === 'string' && body.indexOf('azure') !== -1;
|
|
172
|
+
if (!isAzure) return body;
|
|
173
|
+
let next;
|
|
174
|
+
if (body == null) {
|
|
175
|
+
next = {};
|
|
176
|
+
} else if (typeof body === 'string') {
|
|
177
|
+
try {
|
|
178
|
+
next = JSON.parse(body);
|
|
179
|
+
} catch (err) {
|
|
180
|
+
return body;
|
|
181
|
+
}
|
|
182
|
+
} else {
|
|
183
|
+
next = lodash__default["default"].cloneDeep(body);
|
|
184
|
+
}
|
|
185
|
+
if (!lodash__default["default"].isObject(next.azure)) next.azure = {};
|
|
186
|
+
if (!lodash__default["default"].isObject(next.azure.config)) next.azure.config = {};
|
|
187
|
+
if (!lodash__default["default"].isObject(next.azure.config.properties)) next.azure.config.properties = {};
|
|
188
|
+
if (next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs == null) {
|
|
189
|
+
next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs = String(Math.round(ms));
|
|
190
|
+
}
|
|
191
|
+
return next;
|
|
117
192
|
};
|
|
118
193
|
const TTS_HTTP_AGENT = new http__default["default"].Agent({
|
|
119
194
|
keepAlive: true
|
|
@@ -142,6 +217,8 @@ class BotiumConnectorVoip {
|
|
|
142
217
|
// For debugging latency between incoming STT (bot says) and outgoing audio (sendAudio)
|
|
143
218
|
this._lastBotSaysQueuedAt = null;
|
|
144
219
|
this._lastBotSaysText = null;
|
|
220
|
+
this._replyTrace = null;
|
|
221
|
+
this._activeUserSaysVoipAgent = null;
|
|
145
222
|
this._speculativeTurnToken = 0;
|
|
146
223
|
}
|
|
147
224
|
async Validate() {
|
|
@@ -190,6 +267,27 @@ class BotiumConnectorVoip {
|
|
|
190
267
|
this.convoStep = null;
|
|
191
268
|
this._lastBotSaysQueuedAt = null;
|
|
192
269
|
this._lastBotSaysText = null;
|
|
270
|
+
this._replyTrace = null;
|
|
271
|
+
this._activeUserSaysVoipAgent = null;
|
|
272
|
+
this.audioStream = {
|
|
273
|
+
format: null,
|
|
274
|
+
pcmParts: [],
|
|
275
|
+
totalBytes: 0,
|
|
276
|
+
complete: false
|
|
277
|
+
};
|
|
278
|
+
this._turnAudioCounter = 0;
|
|
279
|
+
this._meTurnAudioOrdinal = -1; // 0-based ordinal of REAL user turns this session
|
|
280
|
+
this._lastBotTurnStartSec = null;
|
|
281
|
+
// Per-turn audio is NOT sliced inline (that would force UserSays to wait for the
|
|
282
|
+
// recording to catch up with playback). Instead each turn records a lightweight
|
|
283
|
+
// descriptor here; slices are cut and emitted as MESSAGE_ATTACHMENT progressively,
|
|
284
|
+
// as soon as the recording buffer has reached each turn's playback end (so the live
|
|
285
|
+
// UI can show them mid-run), with a forced flush at session end for any remainder.
|
|
286
|
+
// Zero added latency for TTS and DTMF turns.
|
|
287
|
+
this._pendingTurnAudio = [];
|
|
288
|
+
this._turnAudioPrevEndSec = null;
|
|
289
|
+
this._turnAudioForceDone = false;
|
|
290
|
+
this._trailingBotAudioEmitted = false;
|
|
193
291
|
if (this.ttsCache) {
|
|
194
292
|
this.ttsCache.clear();
|
|
195
293
|
}
|
|
@@ -197,6 +295,7 @@ class BotiumConnectorVoip {
|
|
|
197
295
|
const queuedAt = Date.now();
|
|
198
296
|
this._lastBotSaysQueuedAt = queuedAt;
|
|
199
297
|
this._lastBotSaysText = botMsg && botMsg.messageText ? String(botMsg.messageText) : null;
|
|
298
|
+
this._captureBotQueuedForReplyTrace(queuedAt);
|
|
200
299
|
// Stamp the wall-clock instant at which the connector released the bot
|
|
201
300
|
// utterance to botium-core's queue. Paired with `_receivedAtMs` (last
|
|
202
301
|
// STT final frame) this gives the true "join silence" the connector
|
|
@@ -204,8 +303,33 @@ class BotiumConnectorVoip {
|
|
|
204
303
|
// message up with WaitBotSays().
|
|
205
304
|
if (botMsg && botMsg.sourceData) {
|
|
206
305
|
const head = Array.isArray(botMsg.sourceData) ? botMsg.sourceData[0] : botMsg.sourceData;
|
|
207
|
-
if (head && typeof head === 'object'
|
|
208
|
-
head.flushedAtMs = queuedAt;
|
|
306
|
+
if (head && typeof head === 'object') {
|
|
307
|
+
if (!('flushedAtMs' in head)) head.flushedAtMs = queuedAt;
|
|
308
|
+
if (this._replyTrace) {
|
|
309
|
+
if (lodash__default["default"].isFinite(this._replyTrace.psstFireDelayMs)) head.psstFireDelayMs = this._replyTrace.psstFireDelayMs;
|
|
310
|
+
if (lodash__default["default"].isFinite(this._replyTrace.psstScheduledMs)) head.psstScheduledMs = this._replyTrace.psstScheduledMs;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
// A turn = bot speaks first, then me responds.
|
|
315
|
+
// Save the bot's audio start so UserSays can slice the full exchange
|
|
316
|
+
// (bot + me) once me has finished speaking.
|
|
317
|
+
if (this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE] && botMsg && !(botMsg instanceof Error)) {
|
|
318
|
+
try {
|
|
319
|
+
const sd = botMsg.sourceData;
|
|
320
|
+
let startSec = null;
|
|
321
|
+
if (Array.isArray(sd) && sd.length > 0) {
|
|
322
|
+
startSec = lodash__default["default"].get(sd, '[0].data.start', null);
|
|
323
|
+
} else if (sd && typeof sd === 'object') {
|
|
324
|
+
startSec = lodash__default["default"].get(sd, 'data.start', null);
|
|
325
|
+
}
|
|
326
|
+
// Only record the FIRST bot message's start; if several bot messages
|
|
327
|
+
// arrive before the next UserSays they all belong to the same turn.
|
|
328
|
+
if (lodash__default["default"].isFinite(startSec) && this._lastBotTurnStartSec === null) {
|
|
329
|
+
this._lastBotTurnStartSec = startSec;
|
|
330
|
+
}
|
|
331
|
+
} catch (err) {
|
|
332
|
+
debug$3(`${this.sessionId} - sendBotMsg: saving turn start error: ${err && err.message}`);
|
|
209
333
|
}
|
|
210
334
|
}
|
|
211
335
|
setTimeout(() => this.queueBotSays(botMsg), 0);
|
|
@@ -226,6 +350,10 @@ class BotiumConnectorVoip {
|
|
|
226
350
|
botMsg.sourceData[0].silenceDuration = lodash__default["default"].isFinite(firstStart) ? firstStart : null;
|
|
227
351
|
botMsg.sourceData[0].voiceDuration = lodash__default["default"].isFinite(firstStart) && lodash__default["default"].isFinite(lastEnd) ? lastEnd - firstStart : null;
|
|
228
352
|
}
|
|
353
|
+
const lastSpeechEnd = lodash__default["default"].get(botMsgs, `[${botMsgs.length - 1}].sourceData.data.speechEndSec`, null);
|
|
354
|
+
if (lodash__default["default"].isFinite(lastSpeechEnd) && botMsg.sourceData[0] && botMsg.sourceData[0].data) {
|
|
355
|
+
botMsg.sourceData[0].data.speechEndSec = lastSpeechEnd;
|
|
356
|
+
}
|
|
229
357
|
return botMsg;
|
|
230
358
|
};
|
|
231
359
|
|
|
@@ -261,6 +389,10 @@ class BotiumConnectorVoip {
|
|
|
261
389
|
}
|
|
262
390
|
const bufferedAtArm = this.botMsgs.length;
|
|
263
391
|
const armedAt = Date.now();
|
|
392
|
+
this._markReplyTrace({
|
|
393
|
+
psstTimerArmedAtMs: armedAt,
|
|
394
|
+
psstScheduledMs: joinTimeoutMs || 0
|
|
395
|
+
});
|
|
264
396
|
_info('psst_timer_armed', {
|
|
265
397
|
sessionId: this.sessionId,
|
|
266
398
|
joinTimeoutMs: joinTimeoutMs || 0,
|
|
@@ -284,6 +416,10 @@ class BotiumConnectorVoip {
|
|
|
284
416
|
}
|
|
285
417
|
this.silenceTimeout = setTimeout(() => {
|
|
286
418
|
const fireDelay = Date.now() - armedAt;
|
|
419
|
+
this._markReplyTrace({
|
|
420
|
+
psstTimerFiredAtMs: Date.now(),
|
|
421
|
+
psstFireDelayMs: fireDelay
|
|
422
|
+
});
|
|
287
423
|
if (this.botMsgs.length > 0) {
|
|
288
424
|
_info('psst_timer_fired', {
|
|
289
425
|
sessionId: this.sessionId,
|
|
@@ -435,15 +571,17 @@ class BotiumConnectorVoip {
|
|
|
435
571
|
await connectHttp(retryIndex + 1);
|
|
436
572
|
}
|
|
437
573
|
};
|
|
438
|
-
await connectHttp();
|
|
439
|
-
if (httpInitRetries > 0) {
|
|
440
|
-
_info('connected_after_retries', {
|
|
441
|
-
phase: 'initCall',
|
|
442
|
-
retries: httpInitRetries
|
|
443
|
-
});
|
|
444
|
-
}
|
|
445
574
|
return new Promise((resolve, reject) => {
|
|
446
|
-
|
|
575
|
+
// Worker session details (port) come from connectHttp() and are recomputed
|
|
576
|
+
// on every (re)try, since each setup attempt gets a fresh worker session.
|
|
577
|
+
let wsEndpoint = null;
|
|
578
|
+
const computeWsEndpoint = () => `${this.caps[Capabilities.VOIP_USE_GLOBAL_VOIP_WORKER] ? process.env.BOTIUM_VOIP_WORKER_URL : this.caps[Capabilities.VOIP_WORKER_URL]}/ws/${data.port}`;
|
|
579
|
+
// 487-retry bookkeeping: a transient '487 Request Terminated' that arrives
|
|
580
|
+
// before the call connects re-runs the whole setup (fresh initCall -> ws
|
|
581
|
+
// -> SIP INVITE) up to max487Retries times.
|
|
582
|
+
let setup487Retries = 0;
|
|
583
|
+
let convoStepListenerAttached = false;
|
|
584
|
+
const max487Retries = this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_MAXRETRIES];
|
|
447
585
|
const connectWs = retryIndex => {
|
|
448
586
|
retryIndex = retryIndex || 0;
|
|
449
587
|
return new Promise((resolve, reject) => {
|
|
@@ -477,7 +615,7 @@ class BotiumConnectorVoip {
|
|
|
477
615
|
});
|
|
478
616
|
});
|
|
479
617
|
};
|
|
480
|
-
|
|
618
|
+
const onWsConnected = wsRetries => {
|
|
481
619
|
if (wsRetries > 0) {
|
|
482
620
|
_info('connected_after_retries', {
|
|
483
621
|
phase: 'websocket',
|
|
@@ -491,26 +629,32 @@ class BotiumConnectorVoip {
|
|
|
491
629
|
this.caps[Capabilities.VOIP_ICE_STUN_SERVERS] = this.caps[Capabilities.VOIP_ICE_STUN_SERVERS].split(',');
|
|
492
630
|
}
|
|
493
631
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
632
|
+
|
|
633
|
+
// Attach this once: the listener reads this.ws dynamically, so it keeps
|
|
634
|
+
// working across 487 setup retries that swap out the websocket.
|
|
635
|
+
if (!convoStepListenerAttached) {
|
|
636
|
+
convoStepListenerAttached = true;
|
|
637
|
+
this.eventEmitter.on('CONVO_STEP_NEXT', (container, convoStep) => {
|
|
638
|
+
this.convoStep = convoStep;
|
|
639
|
+
this._maybePrefetchTts(convoStep);
|
|
640
|
+
// For PSST: send join silence duration per step to VOIP worker (controls PSST silence trigger)
|
|
641
|
+
try {
|
|
642
|
+
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'PSST' && this.ws && this.ws.readyState === ws__default["default"].OPEN) {
|
|
643
|
+
const silenceMs = this._getEffectiveJoinTimeoutMs(convoStep, this.botMsgs);
|
|
644
|
+
if (lodash__default["default"].isFinite(silenceMs) && silenceMs > 0 && this.sessionId) {
|
|
645
|
+
debug$3(`PSST: sending silenceDurationMs=${silenceMs} for sessionId=${this.sessionId}`);
|
|
646
|
+
this.ws.send(JSON.stringify({
|
|
647
|
+
METHOD: 'setSttSilenceDuration',
|
|
648
|
+
sessionId: this.sessionId,
|
|
649
|
+
silenceDurationMs: silenceMs
|
|
650
|
+
}));
|
|
651
|
+
}
|
|
508
652
|
}
|
|
653
|
+
} catch (err) {
|
|
654
|
+
debug$3(`Failed sending PSST silence duration to VOIP worker: ${err.message || err}`);
|
|
509
655
|
}
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
}
|
|
513
|
-
});
|
|
656
|
+
});
|
|
657
|
+
}
|
|
514
658
|
this.silence = null;
|
|
515
659
|
this.msgCount = 0;
|
|
516
660
|
this.sttPartialCount = 0;
|
|
@@ -557,11 +701,13 @@ class BotiumConnectorVoip {
|
|
|
557
701
|
ICE_TURN_PASSWORD: this.caps[Capabilities.VOIP_ICE_TURN_PASSWORD],
|
|
558
702
|
ICE_TURN_PROTOCOL: this.caps[Capabilities.VOIP_ICE_TURN_PROTOCOL] || 'TCP',
|
|
559
703
|
MIN_SILENCE_DURATION: this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT_ENABLE] ? this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT] : null,
|
|
704
|
+
SDP_MEDIA_TYPE_TEXT_ENABLE: !!this.caps[Capabilities.VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE],
|
|
705
|
+
AUDIO_STREAM: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE],
|
|
560
706
|
STT_LEGACY: sttLegacy,
|
|
561
707
|
STT_CONFIG: {
|
|
562
708
|
stt_url: sttUrl,
|
|
563
709
|
stt_params: this.caps[Capabilities.VOIP_STT_PARAMS_STREAM],
|
|
564
|
-
stt_body: this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null
|
|
710
|
+
stt_body: injectAzureSegmentationTimeout(this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null, this.caps[Capabilities.VOIP_STT_PARAMS_STREAM], this.caps[Capabilities.VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS])
|
|
565
711
|
},
|
|
566
712
|
TTS_CONFIG: {
|
|
567
713
|
tts_url: this.caps[Capabilities.VOIP_TTS_URL],
|
|
@@ -626,7 +772,7 @@ class BotiumConnectorVoip {
|
|
|
626
772
|
// (fullRecord*) and hard errors so `full_record.wav` is delivered
|
|
627
773
|
// on early-completion hangups. Post-Stop STT frames remain blocked.
|
|
628
774
|
if (this.stopCalled) {
|
|
629
|
-
const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error'];
|
|
775
|
+
const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error', 'audioStreamStart', 'audioStreamChunk', 'audioStreamEnd'];
|
|
630
776
|
if (!parsedData || !allowedPostStopTypes.includes(parsedData.type)) {
|
|
631
777
|
debug$3(`${this.sessionId} - Stop already called, ignoring incoming message`);
|
|
632
778
|
return;
|
|
@@ -638,7 +784,7 @@ class BotiumConnectorVoip {
|
|
|
638
784
|
if (!obj || typeof obj !== 'object') return;
|
|
639
785
|
for (const key of Object.keys(obj)) {
|
|
640
786
|
const val = obj[key];
|
|
641
|
-
if (typeof val === 'string' && val.length > 500) {
|
|
787
|
+
if (typeof val === 'string' && val.length > 0 && (WS_DEBUG_BASE64_FIELD_NAMES.has(key) || val.length > 500)) {
|
|
642
788
|
obj[key] = `<base64:${val.length}chars>`;
|
|
643
789
|
} else if (val && typeof val === 'object' && !Array.isArray(val)) {
|
|
644
790
|
sanitizeBase64Fields(val, `${prefix}${key}.`);
|
|
@@ -646,7 +792,9 @@ class BotiumConnectorVoip {
|
|
|
646
792
|
}
|
|
647
793
|
};
|
|
648
794
|
sanitizeBase64Fields(parsedDataLog);
|
|
649
|
-
|
|
795
|
+
if (!WS_DEBUG_SILENT_TYPES.has(parsedData?.type)) {
|
|
796
|
+
debug$3(JSON.stringify(parsedDataLog, null, 2));
|
|
797
|
+
}
|
|
650
798
|
const _extractFullRecordBase64 = pd => {
|
|
651
799
|
if (!pd) return null;
|
|
652
800
|
// Different VOIP workers may put the payload in various fields - search all string fields
|
|
@@ -750,6 +898,9 @@ class BotiumConnectorVoip {
|
|
|
750
898
|
reject(new Error('Error: Sip Registration failed'));
|
|
751
899
|
}
|
|
752
900
|
if (parsedData && parsedData.type === 'callinfo' && parsedData.status === 'connected') {
|
|
901
|
+
// Mark connected so a later terminal error is delivered to the bot
|
|
902
|
+
// conversation instead of triggering a (now pointless) setup retry.
|
|
903
|
+
this.connected = true;
|
|
753
904
|
_info('callinfo_connected', {
|
|
754
905
|
sessionId: this.sessionId
|
|
755
906
|
});
|
|
@@ -784,6 +935,23 @@ class BotiumConnectorVoip {
|
|
|
784
935
|
});
|
|
785
936
|
}
|
|
786
937
|
if (parsedData && parsedData.type === 'error') {
|
|
938
|
+
const errMsg = parsedData.message || '';
|
|
939
|
+
// The worker reports the SIP code inside the message string
|
|
940
|
+
// ("Disconnected because of error - Reason: 487 Request Terminated")
|
|
941
|
+
// and does not set a dedicated `code` field, so detect 487 from both.
|
|
942
|
+
const is487 = parsedData.code === 487 || parsedData.code === '487' || /\b487\b/.test(errMsg) || /request terminated/i.test(errMsg);
|
|
943
|
+
// A transient '487 Request Terminated' that arrives before the call
|
|
944
|
+
// connects: retry the whole call setup instead of failing the test.
|
|
945
|
+
if (is487 && !this.connected && setup487Retries < max487Retries) {
|
|
946
|
+
_info('ws_error_msg', {
|
|
947
|
+
sessionId: this.sessionId,
|
|
948
|
+
message: parsedData.message || null,
|
|
949
|
+
code: parsedData.code || null,
|
|
950
|
+
retrying487: true
|
|
951
|
+
});
|
|
952
|
+
retryCallSetup487(errMsg);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
787
955
|
flushPendingBotMsgs('error');
|
|
788
956
|
// Ensure buffered recording is not lost on terminal worker errors.
|
|
789
957
|
this._emitBufferedFullRecordIfAny('error_buffered');
|
|
@@ -797,6 +965,46 @@ class BotiumConnectorVoip {
|
|
|
797
965
|
sendBotMsg(new Error(`Error: ${parsedData.message}`));
|
|
798
966
|
}
|
|
799
967
|
|
|
968
|
+
// Per-turn audio stream: continuous PCM chunks received during the call.
|
|
969
|
+
// The connector buffers them so _sliceTurnAudio() can extract per-turn segments.
|
|
970
|
+
if (parsedData && parsedData.type === 'audioStreamStart') {
|
|
971
|
+
this.audioStream = {
|
|
972
|
+
format: {
|
|
973
|
+
sampleRate: parsedData.sampleRate,
|
|
974
|
+
channels: parsedData.channels,
|
|
975
|
+
bitsPerSample: parsedData.bitsPerSample,
|
|
976
|
+
dataOffset: parsedData.dataOffset
|
|
977
|
+
},
|
|
978
|
+
pcmParts: [],
|
|
979
|
+
totalBytes: 0,
|
|
980
|
+
complete: false
|
|
981
|
+
};
|
|
982
|
+
debug$3(`${this.sessionId} - audioStreamStart sampleRate=${parsedData.sampleRate} channels=${parsedData.channels} bitsPerSample=${parsedData.bitsPerSample}`);
|
|
983
|
+
}
|
|
984
|
+
if (parsedData && parsedData.type === 'audioStreamChunk') {
|
|
985
|
+
if (this.audioStream && parsedData.chunk) {
|
|
986
|
+
try {
|
|
987
|
+
const buf = Buffer.from(parsedData.chunk, 'base64');
|
|
988
|
+
this.audioStream.pcmParts.push(buf);
|
|
989
|
+
this.audioStream.totalBytes += buf.length;
|
|
990
|
+
this._maybeDetectAgentAudibleOnRecording(this._activeUserSaysVoipAgent);
|
|
991
|
+
// Emit any per-turn audio whose playback the recording has now caught up to,
|
|
992
|
+
// so the live transcript can show it mid-run (no UserSays latency).
|
|
993
|
+
this._emitReadyTurnAudio('audioStreamChunk', false);
|
|
994
|
+
} catch (e) {
|
|
995
|
+
debug$3(`${this.sessionId} - audioStreamChunk decode error: ${e && e.message}`);
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
if (parsedData && parsedData.type === 'audioStreamEnd') {
|
|
1000
|
+
if (this.audioStream) {
|
|
1001
|
+
this.audioStream.complete = true;
|
|
1002
|
+
}
|
|
1003
|
+
debug$3(`${this.sessionId} - audioStreamEnd totalBytes=${parsedData.totalBytes}`);
|
|
1004
|
+
// Buffer is complete — cut and emit the per-turn audio now.
|
|
1005
|
+
this._flushPendingTurnAudio('audioStreamEnd');
|
|
1006
|
+
}
|
|
1007
|
+
|
|
800
1008
|
// Full record streaming support:
|
|
801
1009
|
// - some VOIP workers send the recording in chunks and an end marker
|
|
802
1010
|
if (parsedData && parsedData.type === 'fullRecordStart') {
|
|
@@ -816,11 +1024,49 @@ class BotiumConnectorVoip {
|
|
|
816
1024
|
source: 'fullRecordEnd',
|
|
817
1025
|
base64Len
|
|
818
1026
|
});
|
|
1027
|
+
// Emit per-turn audio before `this.end = true` so it is captured by the
|
|
1028
|
+
// worker before Stop() resolves (no-op if audioStreamEnd already flushed).
|
|
1029
|
+
this._flushPendingTurnAudio('fullRecordEnd');
|
|
819
1030
|
// Flush before `this.end = true` so the buffered final STT is not
|
|
820
1031
|
// dropped when Stop() clears the PSST silence timer on teardown.
|
|
821
1032
|
flushPendingBotMsgs('fullRecordEnd');
|
|
822
1033
|
this.end = true;
|
|
823
1034
|
}
|
|
1035
|
+
if (parsedData && parsedData.type === 'agentPlaybackStarted') {
|
|
1036
|
+
const playbackData = parsedData.data || {};
|
|
1037
|
+
const playedSec = playbackData.playedRecordingStartSec;
|
|
1038
|
+
const active = this._activeUserSaysVoipAgent;
|
|
1039
|
+
if (active && lodash__default["default"].isFinite(playedSec)) {
|
|
1040
|
+
active.playedRecordingStartSec = playedSec;
|
|
1041
|
+
active.playbackAtMs = playbackData.playbackAtMs;
|
|
1042
|
+
if (lodash__default["default"].isFinite(playbackData.requestedDurationMs)) {
|
|
1043
|
+
active.playbackRequestedDurationMs = playbackData.requestedDurationMs;
|
|
1044
|
+
}
|
|
1045
|
+
if (lodash__default["default"].isFinite(playbackData.digitCount)) {
|
|
1046
|
+
active.digitCount = playbackData.digitCount;
|
|
1047
|
+
}
|
|
1048
|
+
this._markReplyTrace({
|
|
1049
|
+
playedRecordingStartSec: playedSec,
|
|
1050
|
+
playbackAtMs: playbackData.playbackAtMs
|
|
1051
|
+
});
|
|
1052
|
+
const heardSec = playbackData.wireKind === 'dtmf' ? playedSec : this._applyAgentHeardRecordingStartSec(active);
|
|
1053
|
+
if (lodash__default["default"].isFinite(heardSec)) {
|
|
1054
|
+
if (playbackData.wireKind === 'dtmf') {
|
|
1055
|
+
active.heardRecordingStartSec = heardSec;
|
|
1056
|
+
this._markReplyTrace({
|
|
1057
|
+
heardRecordingStartSec: heardSec
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
1060
|
+
debug$3(`${this.sessionId} - agent audible on recording at ${heardSec}s (played=${playedSec}s)`);
|
|
1061
|
+
}
|
|
1062
|
+
// Log the heard reply-trace now that playback is audible — earlier than the
|
|
1063
|
+
// old turn-audio callback and free of the next-turn _replyTrace race. The
|
|
1064
|
+
// agent's end on the recording = played + playback length.
|
|
1065
|
+
const playbackSec = lodash__default["default"].isFinite(active.playbackRequestedDurationMs) && active.playbackRequestedDurationMs > 0 ? active.playbackRequestedDurationMs / 1000 : playbackData.wireKind === 'dtmf' ? this._dtmfPlaybackSec(active, active.digitCount || 1) : null;
|
|
1066
|
+
const agentEndSec = lodash__default["default"].isFinite(playbackSec) ? playedSec + playbackSec : undefined;
|
|
1067
|
+
this._logReplyTraceHeard(agentEndSec);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
824
1070
|
if (parsedData && parsedData.type === 'silence') {
|
|
825
1071
|
if (lodash__default["default"].isNil(this._getIgnoreSilenceDurationAsserterLogicHook(this.convoStep))) {
|
|
826
1072
|
if (!this._hasJoinLogicHookOrRule(this.convoStep) && parsedData.data.silence.length > 0) {
|
|
@@ -846,14 +1092,15 @@ class BotiumConnectorVoip {
|
|
|
846
1092
|
this.sttPartialCount++;
|
|
847
1093
|
const partialText = parsedData.data.message;
|
|
848
1094
|
if (typeof partialText === 'string' && partialText.trim().length > 0) {
|
|
1095
|
+
const replacementResult = this._applySttDictionaryReplacements(partialText);
|
|
849
1096
|
this.lastPartialBotMsg = {
|
|
850
|
-
messageText:
|
|
851
|
-
sourceData: Object.assign({}, parsedData, {
|
|
1097
|
+
messageText: replacementResult.text,
|
|
1098
|
+
sourceData: Object.assign({}, this._decorateSourceDataWithSttDictionaryReplacements(parsedData, replacementResult), {
|
|
852
1099
|
partialRecovery: true
|
|
853
1100
|
})
|
|
854
1101
|
};
|
|
855
1102
|
}
|
|
856
|
-
const partialPreview = typeof partialText === 'string' ? partialText.trim()
|
|
1103
|
+
const partialPreview = typeof partialText === 'string' ? partialText.trim() : '';
|
|
857
1104
|
_info('stt_partial_received', {
|
|
858
1105
|
sessionId: this.sessionId,
|
|
859
1106
|
partialIndex: this.sttPartialCount,
|
|
@@ -935,26 +1182,33 @@ class BotiumConnectorVoip {
|
|
|
935
1182
|
const confidenceThreshold = this._getConfidenceScoreLogicHook(this.convoStep) && this._getConfidenceScoreLogicHook(this.convoStep).args[0] || this.caps[Capabilities.VOIP_STT_CONFIDENCE_THRESHOLD];
|
|
936
1183
|
const successfulConfidenceScore = this._getConfidenceScore(parsedData) >= confidenceThreshold;
|
|
937
1184
|
const msgText = parsedData.data.message || '';
|
|
1185
|
+
const replacementResult = this._applySttDictionaryReplacements(msgText);
|
|
1186
|
+
const normalizedMsgText = replacementResult.text;
|
|
938
1187
|
const msgLen = typeof msgText === 'string' ? msgText.length : 0;
|
|
939
|
-
const msgPreview = typeof
|
|
1188
|
+
const msgPreview = typeof normalizedMsgText === 'string' ? normalizedMsgText.trim() : '';
|
|
940
1189
|
// A final supersedes the cached interim; clear to avoid duplicate tail emission.
|
|
941
1190
|
this.lastPartialBotMsg = null;
|
|
942
|
-
debug$3(`Message: ${
|
|
1191
|
+
debug$3(`Message: ${normalizedMsgText} / Confidence Score: ${this._getConfidenceScore(parsedData)} (Threshold: ${confidenceThreshold})`);
|
|
943
1192
|
_info('stt_final', {
|
|
944
1193
|
sessionId: this.sessionId,
|
|
945
1194
|
message: msgPreview,
|
|
946
1195
|
messageLength: msgLen,
|
|
947
1196
|
confidence: this._getConfidenceScore(parsedData),
|
|
948
1197
|
threshold: confidenceThreshold,
|
|
949
|
-
accepted: successfulConfidenceScore
|
|
1198
|
+
accepted: successfulConfidenceScore,
|
|
1199
|
+
segmentEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.end')) ? parsedData.data.end : null,
|
|
1200
|
+
speechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.speechEndSec')) ? parsedData.data.speechEndSec : null
|
|
950
1201
|
});
|
|
1202
|
+
if (successfulConfidenceScore) {
|
|
1203
|
+
this._captureSttFinalForReplyTrace(parsedData, msgPreview);
|
|
1204
|
+
}
|
|
951
1205
|
// ORIGINAL: always emit final message immediately (ignore JOIN hooks/rules).
|
|
952
1206
|
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'ORIGINAL') {
|
|
953
1207
|
let botMsg = {
|
|
954
|
-
messageText:
|
|
1208
|
+
messageText: normalizedMsgText
|
|
955
1209
|
};
|
|
956
1210
|
if (this.firstMsg) {
|
|
957
|
-
const sourceData = parsedData;
|
|
1211
|
+
const sourceData = this._decorateSourceDataWithSttDictionaryReplacements(parsedData, replacementResult);
|
|
958
1212
|
sourceData.silenceDuration = parsedData.data.start;
|
|
959
1213
|
sourceData.voiceDuration = parsedData.data.end - parsedData.data.start;
|
|
960
1214
|
botMsg = Object.assign({}, botMsg, {
|
|
@@ -962,7 +1216,7 @@ class BotiumConnectorVoip {
|
|
|
962
1216
|
});
|
|
963
1217
|
this.firstMsg = false;
|
|
964
1218
|
} else {
|
|
965
|
-
const sourceData = parsedData;
|
|
1219
|
+
const sourceData = this._decorateSourceDataWithSttDictionaryReplacements(parsedData, replacementResult);
|
|
966
1220
|
const start = lodash__default["default"].get(parsedData, 'data.start', null);
|
|
967
1221
|
const prevEnd = lodash__default["default"].get(this.prevData, 'data.end', null);
|
|
968
1222
|
sourceData.silenceDuration = lodash__default["default"].isFinite(start) && lodash__default["default"].isFinite(prevEnd) ? start - prevEnd : lodash__default["default"].isFinite(start) ? start : null;
|
|
@@ -980,10 +1234,10 @@ class BotiumConnectorVoip {
|
|
|
980
1234
|
}
|
|
981
1235
|
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'SPLIT' || this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'EXPAND') {
|
|
982
1236
|
let botMsg = {
|
|
983
|
-
messageText:
|
|
1237
|
+
messageText: normalizedMsgText
|
|
984
1238
|
};
|
|
985
1239
|
if (this.firstMsg) {
|
|
986
|
-
const sourceData = parsedData;
|
|
1240
|
+
const sourceData = this._decorateSourceDataWithSttDictionaryReplacements(parsedData, replacementResult);
|
|
987
1241
|
sourceData.silenceDuration = parsedData.data.start;
|
|
988
1242
|
sourceData.voiceDuration = parsedData.data.end - parsedData.data.start;
|
|
989
1243
|
botMsg = Object.assign({}, botMsg, {
|
|
@@ -991,7 +1245,7 @@ class BotiumConnectorVoip {
|
|
|
991
1245
|
});
|
|
992
1246
|
this.firstMsg = false;
|
|
993
1247
|
} else {
|
|
994
|
-
const sourceData = parsedData;
|
|
1248
|
+
const sourceData = this._decorateSourceDataWithSttDictionaryReplacements(parsedData, replacementResult);
|
|
995
1249
|
const start = lodash__default["default"].get(parsedData, 'data.start', null);
|
|
996
1250
|
const prevEnd = lodash__default["default"].get(this.prevData, 'data.end', null);
|
|
997
1251
|
sourceData.silenceDuration = lodash__default["default"].isFinite(start) && lodash__default["default"].isFinite(prevEnd) ? start - prevEnd : lodash__default["default"].isFinite(start) ? start : null;
|
|
@@ -1010,8 +1264,8 @@ class BotiumConnectorVoip {
|
|
|
1010
1264
|
}
|
|
1011
1265
|
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'JOIN' || this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'PSST' || this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'CONCAT') {
|
|
1012
1266
|
const botMsg = {
|
|
1013
|
-
messageText:
|
|
1014
|
-
sourceData: parsedData
|
|
1267
|
+
messageText: normalizedMsgText,
|
|
1268
|
+
sourceData: this._decorateSourceDataWithSttDictionaryReplacements(parsedData, replacementResult)
|
|
1015
1269
|
};
|
|
1016
1270
|
this.prevData = parsedData;
|
|
1017
1271
|
if (successfulConfidenceScore) {
|
|
@@ -1044,9 +1298,49 @@ class BotiumConnectorVoip {
|
|
|
1044
1298
|
}
|
|
1045
1299
|
}
|
|
1046
1300
|
});
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
|
|
1301
|
+
};
|
|
1302
|
+
const retryCallSetup487 = reasonMessage => {
|
|
1303
|
+
setup487Retries++;
|
|
1304
|
+
_info('call_setup_retry_487', {
|
|
1305
|
+
sessionId: this.sessionId,
|
|
1306
|
+
attempt: setup487Retries,
|
|
1307
|
+
max: max487Retries,
|
|
1308
|
+
reason: reasonMessage || null
|
|
1309
|
+
});
|
|
1310
|
+
debug$3(`Call setup 487 retry ${setup487Retries}/${max487Retries}: ${reasonMessage}`);
|
|
1311
|
+
// Tear down the dead websocket so its stale handlers cannot fire while
|
|
1312
|
+
// the next attempt establishes a fresh worker session.
|
|
1313
|
+
try {
|
|
1314
|
+
if (this.ws) {
|
|
1315
|
+
this.ws.removeAllListeners();
|
|
1316
|
+
this.ws.terminate();
|
|
1317
|
+
}
|
|
1318
|
+
} catch (err) {
|
|
1319
|
+
debug$3(`Call setup 487 retry: websocket teardown failed: ${err && err.message}`);
|
|
1320
|
+
}
|
|
1321
|
+
this.ws = null;
|
|
1322
|
+
this.wsOpened = false;
|
|
1323
|
+
this.end = false;
|
|
1324
|
+
setTimeout(() => {
|
|
1325
|
+
establishCall().catch(err => reject(new Error('Error: ' + err)));
|
|
1326
|
+
}, this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_TIMEOUT]);
|
|
1327
|
+
};
|
|
1328
|
+
const establishCall = async () => {
|
|
1329
|
+
// Each (re)try gets a fresh worker session: new initCall -> new port ->
|
|
1330
|
+
// new websocket -> new SIP INVITE.
|
|
1331
|
+
httpInitRetries = 0;
|
|
1332
|
+
await connectHttp();
|
|
1333
|
+
if (httpInitRetries > 0) {
|
|
1334
|
+
_info('connected_after_retries', {
|
|
1335
|
+
phase: 'initCall',
|
|
1336
|
+
retries: httpInitRetries
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
wsEndpoint = computeWsEndpoint();
|
|
1340
|
+
const wsRetries = await connectWs();
|
|
1341
|
+
onWsConnected(wsRetries);
|
|
1342
|
+
};
|
|
1343
|
+
establishCall().catch(err => reject(new Error('Error: ' + err)));
|
|
1050
1344
|
});
|
|
1051
1345
|
}
|
|
1052
1346
|
async UserSays(msg) {
|
|
@@ -1056,6 +1350,10 @@ class BotiumConnectorVoip {
|
|
|
1056
1350
|
const hasDtmf = !!(msg && msg.buttons && msg.buttons.length > 0);
|
|
1057
1351
|
const dtmfMatch = msg && msg.messageText && msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
|
|
1058
1352
|
const inputType = hasDtmf || dtmfMatch ? 'dtmf' : hasText && hasVoiceMedia ? 'mixed' : hasText ? 'text' : hasVoiceMedia ? 'media' : 'unknown';
|
|
1353
|
+
// A real user turn is one that sends content (text/media/dtmf). 'unknown' turns send nothing and
|
|
1354
|
+
// surface server-side as skippable me-steps, so they must NOT consume an ordinal slot - this keeps
|
|
1355
|
+
// meTurnIndex aligned with the server's non-skippable me-step indices when placing turn audio.
|
|
1356
|
+
const meTurnIndex = inputType !== 'unknown' ? this._meTurnAudioOrdinal = (this._meTurnAudioOrdinal ?? -1) + 1 : null;
|
|
1059
1357
|
const msgPreview = hasText && msg.messageText ? String(msg.messageText).trim().substring(0, 80) : '';
|
|
1060
1358
|
_info('user_says', {
|
|
1061
1359
|
sessionId: this.sessionId,
|
|
@@ -1064,6 +1362,7 @@ class BotiumConnectorVoip {
|
|
|
1064
1362
|
messageLength: hasText && msg.messageText ? msg.messageText.length : undefined,
|
|
1065
1363
|
mediaSize: hasVoiceMedia && msg.media[0] && Buffer.isBuffer(msg.media[0].buffer) ? msg.media[0].buffer.length : undefined
|
|
1066
1364
|
});
|
|
1365
|
+
this._captureUserSaysStart(msgPreview);
|
|
1067
1366
|
// Avoid logging large buffers/base64 (can break job logs and overwhelm stdout)
|
|
1068
1367
|
try {
|
|
1069
1368
|
const safeLog = {
|
|
@@ -1098,17 +1397,45 @@ class BotiumConnectorVoip {
|
|
|
1098
1397
|
// the coach can place the agent turn on the recording timeline.
|
|
1099
1398
|
// `requestedDurationMs` is the best estimate of on-wire playback
|
|
1100
1399
|
// length (DTMF tones × digits, TTS synth output, parsed media duration).
|
|
1400
|
+
const recordingSecNow = () => this._recordingSecNow();
|
|
1101
1401
|
const stampAgentWire = (wireKind, requestedDurationMs, extras = {}) => {
|
|
1402
|
+
const wireRecordingStartSec = recordingSecNow();
|
|
1102
1403
|
msg.voipAgent = {
|
|
1103
1404
|
wireSentAtMs: Date.now(),
|
|
1104
1405
|
inputType,
|
|
1105
1406
|
wireKind,
|
|
1106
1407
|
requestedDurationMs: Math.max(0, Math.round(requestedDurationMs || 0)),
|
|
1408
|
+
...(wireRecordingStartSec != null ? {
|
|
1409
|
+
wireRecordingStartSec
|
|
1410
|
+
} : {}),
|
|
1411
|
+
// Carry the coach's reply-decision trace (set on the outgoing message before UserSays)
|
|
1412
|
+
// into voipAgent here, while we own the object and before MESSAGE_SENTTOBOT fires — so
|
|
1413
|
+
// the LIVE transcript snapshot (a deep copy taken at that event) already has it, not
|
|
1414
|
+
// just the final result. Lets the UI split "Tester reply decision" live.
|
|
1415
|
+
...(msg && msg.coachTrace && typeof msg.coachTrace === 'object' ? {
|
|
1416
|
+
coachTrace: msg.coachTrace
|
|
1417
|
+
} : {}),
|
|
1107
1418
|
...extras
|
|
1108
1419
|
};
|
|
1420
|
+
this._activeUserSaysVoipAgent = msg.voipAgent;
|
|
1421
|
+
this._captureAgentWire(msg.voipAgent, inputType);
|
|
1422
|
+
};
|
|
1423
|
+
const sendAgentWire = request => {
|
|
1424
|
+
this._sendUserSaysWs(request);
|
|
1425
|
+
this._markReplyTrace({
|
|
1426
|
+
sendAudioAtMs: Date.now()
|
|
1427
|
+
});
|
|
1428
|
+
this._logReplyTrace('wire_sent');
|
|
1429
|
+
// Finalize the wall pipeline NOW, while _replyTrace still holds this turn's data
|
|
1430
|
+
// (userSaysAtMs / ttsStartAtMs / ttsEndAtMs / wireAtMs / sendAudioAtMs are all set by
|
|
1431
|
+
// this point). The deferred turn-audio callback runs only after the agent audio is
|
|
1432
|
+
// "heard" on the recording, which nulls _replyTrace (_logReplyTraceHeard); finalizing
|
|
1433
|
+
// there would drop wallPipeline entirely and collapse the reply-delay breakdown into a
|
|
1434
|
+
// single "Audio send" bucket on the UI.
|
|
1435
|
+
this._finalizeWallPipeline(msg.voipAgent);
|
|
1109
1436
|
};
|
|
1110
1437
|
// Twilio default: 100 ms tone + 100 ms gap per digit. Drives agent-bar width only.
|
|
1111
|
-
const
|
|
1438
|
+
const DTMF_AGENT_BAR_MS_PER_DIGIT = DTMF_MS_PER_DIGIT;
|
|
1112
1439
|
if (msg && msg.buttons && msg.buttons.length > 0) {
|
|
1113
1440
|
const digits = sanitizeDtmfDigits(msg.buttons[0].payload);
|
|
1114
1441
|
if (!digits) {
|
|
@@ -1121,10 +1448,13 @@ class BotiumConnectorVoip {
|
|
|
1121
1448
|
digits,
|
|
1122
1449
|
sessionId: this.sessionId
|
|
1123
1450
|
});
|
|
1124
|
-
stampAgentWire('dtmf', digits.length *
|
|
1125
|
-
digitCount: digits.length
|
|
1451
|
+
stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
|
|
1452
|
+
digitCount: digits.length,
|
|
1453
|
+
dtmfDigits: digits
|
|
1126
1454
|
});
|
|
1127
|
-
|
|
1455
|
+
sendAgentWire(request);
|
|
1456
|
+
// Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
|
|
1457
|
+
duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
|
|
1128
1458
|
} else if (msg && msg.messageText) {
|
|
1129
1459
|
// Check for DTMF tag in messageText: <DTMF>1234</DTMF>
|
|
1130
1460
|
const dtmfMatch = msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
|
|
@@ -1145,13 +1475,14 @@ class BotiumConnectorVoip {
|
|
|
1145
1475
|
digits,
|
|
1146
1476
|
sessionId: this.sessionId
|
|
1147
1477
|
});
|
|
1148
|
-
stampAgentWire('dtmf', digits.length *
|
|
1149
|
-
digitCount: digits.length
|
|
1478
|
+
stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
|
|
1479
|
+
digitCount: digits.length,
|
|
1480
|
+
dtmfDigits: digits
|
|
1150
1481
|
});
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
if (!skipTtsForMixedInput) {
|
|
1482
|
+
sendAgentWire(request);
|
|
1483
|
+
// Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
|
|
1484
|
+
duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
|
|
1485
|
+
} else if (!skipTtsForMixedInput) {
|
|
1155
1486
|
if (!this.axiosTtsParams) {
|
|
1156
1487
|
if (!(msg.media && msg.media.length > 0 && msg.media[0].buffer)) {
|
|
1157
1488
|
return reject(new Error('TTS not configured, only audio input supported'));
|
|
@@ -1163,10 +1494,17 @@ class BotiumConnectorVoip {
|
|
|
1163
1494
|
msg.sourceData = ttsRequest;
|
|
1164
1495
|
let ttsResult = null;
|
|
1165
1496
|
const ttsStartedAt = Date.now();
|
|
1497
|
+
this._markReplyTrace({
|
|
1498
|
+
ttsStartAtMs: ttsStartedAt
|
|
1499
|
+
});
|
|
1166
1500
|
let ttsSynthMs = 0;
|
|
1167
1501
|
try {
|
|
1168
1502
|
ttsResult = await this._getTtsAudio(ttsRequest, msg.messageText);
|
|
1169
1503
|
ttsSynthMs = Date.now() - ttsStartedAt;
|
|
1504
|
+
this._markReplyTrace({
|
|
1505
|
+
ttsEndAtMs: Date.now(),
|
|
1506
|
+
ttsSynthMs
|
|
1507
|
+
});
|
|
1170
1508
|
} catch (err) {
|
|
1171
1509
|
return reject(new Error(`TTS "${msg.messageText}" failed - ${this._getAxiosErrOutput(err)}`));
|
|
1172
1510
|
}
|
|
@@ -1202,7 +1540,7 @@ class BotiumConnectorVoip {
|
|
|
1202
1540
|
ttsSynthMs,
|
|
1203
1541
|
textLength: msg.messageText ? msg.messageText.length : 0
|
|
1204
1542
|
});
|
|
1205
|
-
|
|
1543
|
+
sendAgentWire(request);
|
|
1206
1544
|
} else {
|
|
1207
1545
|
return reject(new Error('TTS failed, response is empty'));
|
|
1208
1546
|
}
|
|
@@ -1227,7 +1565,7 @@ class BotiumConnectorVoip {
|
|
|
1227
1565
|
stampAgentWire('media', 0, {
|
|
1228
1566
|
mediaUri: msg.media[0].mediaUri || null
|
|
1229
1567
|
});
|
|
1230
|
-
|
|
1568
|
+
sendAgentWire(request);
|
|
1231
1569
|
msg.attachments.push({
|
|
1232
1570
|
name: msg.media[0].mediaUri,
|
|
1233
1571
|
mimeType: msg.media[0].mimeType,
|
|
@@ -1249,16 +1587,514 @@ class BotiumConnectorVoip {
|
|
|
1249
1587
|
}
|
|
1250
1588
|
}
|
|
1251
1589
|
const requestedDurationMs = Math.max(0, Math.round((duration || 0) * 1000));
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1590
|
+
|
|
1591
|
+
// Record this turn's slice bounds and resolve immediately — the actual
|
|
1592
|
+
// slice is cut from the complete buffer at session end (see
|
|
1593
|
+
// _flushPendingTurnAudio). This keeps UserSays from waiting for the
|
|
1594
|
+
// recording to catch up with playback (no latency for TTS or DTMF). The
|
|
1595
|
+
// heard-trace telemetry (voip_reply_trace_heard) is logged from the
|
|
1596
|
+
// agentPlaybackStarted handler once playback is audible on the recording.
|
|
1597
|
+
this._recordPendingTurnAudio(msg.voipAgent, requestedDurationMs, meTurnIndex);
|
|
1598
|
+
resolve();
|
|
1256
1599
|
} catch (err) {
|
|
1257
1600
|
reject(err);
|
|
1258
1601
|
}
|
|
1259
1602
|
}, 0);
|
|
1260
1603
|
});
|
|
1261
1604
|
}
|
|
1605
|
+
_recordingSecNow() {
|
|
1606
|
+
const fmt = this.audioStream && this.audioStream.format;
|
|
1607
|
+
const bytesPerSec = fmt ? fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8) : null;
|
|
1608
|
+
if (!bytesPerSec || !this.audioStream || !(this.audioStream.totalBytes > 0)) return null;
|
|
1609
|
+
return this.audioStream.totalBytes / bytesPerSec;
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
/** Expected on-recording playback length of a DTMF turn (worker-reported, else estimated). */
|
|
1613
|
+
_dtmfPlaybackSec(voipAgent, digitCount) {
|
|
1614
|
+
const playbackMs = voipAgent && voipAgent.playbackRequestedDurationMs;
|
|
1615
|
+
if (lodash__default["default"].isFinite(playbackMs) && playbackMs > 0) return playbackMs / 1000;
|
|
1616
|
+
return dtmfPlaybackMs(digitCount) / 1000;
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
/**
|
|
1620
|
+
* Snapshot the bounds needed to slice this turn's audio later, then return.
|
|
1621
|
+
* `_lastBotTurnStartSec` is the bot-speech anchor; it is consumed here so the
|
|
1622
|
+
* next bot message sets a fresh one. The end of the agent's playback is read
|
|
1623
|
+
* from voipAgent.playedRecordingStartSec (filled asynchronously by the
|
|
1624
|
+
* agentPlaybackStarted handler) at flush time. Consecutive agent turns with no
|
|
1625
|
+
* bot message in between (anchor null) chain off the previous turn's end.
|
|
1626
|
+
*/
|
|
1627
|
+
_recordPendingTurnAudio(voipAgent, requestedDurationMs, meTurnIndex) {
|
|
1628
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1629
|
+
const botAnchorSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
|
|
1630
|
+
this._lastBotTurnStartSec = null;
|
|
1631
|
+
this._pendingTurnAudio.push({
|
|
1632
|
+
botAnchorSec,
|
|
1633
|
+
voipAgent: voipAgent || null,
|
|
1634
|
+
requestedDurationMs: Math.max(0, requestedDurationMs || 0),
|
|
1635
|
+
meTurnIndex: Number.isInteger(meTurnIndex) && meTurnIndex >= 0 ? meTurnIndex : null
|
|
1636
|
+
});
|
|
1637
|
+
_info('turn_audio_recorded', {
|
|
1638
|
+
sessionId: this.sessionId,
|
|
1639
|
+
meTurnIndex,
|
|
1640
|
+
botAnchorSec,
|
|
1641
|
+
wireKind: voipAgent && voipAgent.wireKind,
|
|
1642
|
+
pending: this._pendingTurnAudio.length
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
/**
|
|
1647
|
+
* Emit per-turn audio (turn_N.wav) for every pending turn whose playback end the
|
|
1648
|
+
* recording buffer has already reached, slicing from the live buffer and emitting a
|
|
1649
|
+
* MESSAGE_ATTACHMENT carrying meTurnIndex. Turns are processed in order (the chain
|
|
1650
|
+
* start of a follow-up turn with no bot anchor depends on the previous turn's end), so
|
|
1651
|
+
* a not-yet-ready turn stops the pass until more audio streams in. With force=true
|
|
1652
|
+
* (session end) the remainder is emitted even if the buffer is incomplete.
|
|
1653
|
+
*/
|
|
1654
|
+
_emitReadyTurnAudio(reason, force) {
|
|
1655
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1656
|
+
const pending = this._pendingTurnAudio || [];
|
|
1657
|
+
if (!pending.length) return;
|
|
1658
|
+
if (!this.audioStream || !this.audioStream.format) return; // no PCM yet
|
|
1659
|
+
const slackSec = (audioStreamIntervalMs() + 50) / 1000;
|
|
1660
|
+
const recNow = this._recordingSecNow();
|
|
1661
|
+
for (const t of pending) {
|
|
1662
|
+
if (t.emitted) continue;
|
|
1663
|
+
try {
|
|
1664
|
+
const va = t.voipAgent || {};
|
|
1665
|
+
const playbackSec = va.wireKind === 'dtmf' ? this._dtmfPlaybackSec(va, va.digitCount || 1) : lodash__default["default"].isFinite(va.playbackRequestedDurationMs) && va.playbackRequestedDurationMs > 0 ? va.playbackRequestedDurationMs / 1000 : t.requestedDurationMs / 1000;
|
|
1666
|
+
const startSec = lodash__default["default"].isFinite(t.botAnchorSec) ? t.botAnchorSec : this._turnAudioPrevEndSec;
|
|
1667
|
+
// End = where the agent's playback finished on the recording. _sliceTurnAudio
|
|
1668
|
+
// adds the configured padding on top.
|
|
1669
|
+
const anchorSec = lodash__default["default"].isFinite(va.playedRecordingStartSec) ? va.playedRecordingStartSec : lodash__default["default"].isFinite(va.heardRecordingStartSec) ? va.heardRecordingStartSec : lodash__default["default"].isFinite(va.wireRecordingStartSec) ? va.wireRecordingStartSec : null;
|
|
1670
|
+
let endSec = lodash__default["default"].isFinite(anchorSec) ? anchorSec + playbackSec : null;
|
|
1671
|
+
if (!lodash__default["default"].isFinite(endSec) && lodash__default["default"].isFinite(startSec)) endSec = startSec + playbackSec;
|
|
1672
|
+
|
|
1673
|
+
// Progressive: only emit once the recording has reached this turn's end (plus a
|
|
1674
|
+
// stream-flush slack). Stop the pass otherwise — later turns must wait their turn.
|
|
1675
|
+
if (!force) {
|
|
1676
|
+
if (!lodash__default["default"].isFinite(anchorSec) || !lodash__default["default"].isFinite(endSec)) break;
|
|
1677
|
+
if (!lodash__default["default"].isFinite(recNow) || recNow < endSec + slackSec) break;
|
|
1678
|
+
}
|
|
1679
|
+
if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) {
|
|
1680
|
+
debug$3(`${this.sessionId} - turnAudio: skip turn (start=${startSec} end=${endSec} reason=${reason})`);
|
|
1681
|
+
t.emitted = true;
|
|
1682
|
+
if (lodash__default["default"].isFinite(endSec)) this._turnAudioPrevEndSec = endSec;
|
|
1683
|
+
continue;
|
|
1684
|
+
}
|
|
1685
|
+
const audioBase64 = this._sliceTurnAudio(startSec, endSec);
|
|
1686
|
+
t.emitted = true;
|
|
1687
|
+
this._turnAudioPrevEndSec = endSec;
|
|
1688
|
+
if (!audioBase64) continue;
|
|
1689
|
+
this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
|
|
1690
|
+
this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
|
|
1691
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1692
|
+
mimeType: 'audio/wav',
|
|
1693
|
+
base64: audioBase64,
|
|
1694
|
+
meTurnIndex: t.meTurnIndex,
|
|
1695
|
+
// 0-based real-user-turn ordinal (authoritative for placement)
|
|
1696
|
+
sessionContext: {
|
|
1697
|
+
testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
|
|
1698
|
+
testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
|
|
1699
|
+
}
|
|
1700
|
+
});
|
|
1701
|
+
_info('turn_audio_emitted', {
|
|
1702
|
+
sessionId: this.sessionId,
|
|
1703
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1704
|
+
meTurnIndex: t.meTurnIndex,
|
|
1705
|
+
startSec: Number(startSec.toFixed(2)),
|
|
1706
|
+
endSec: Number(endSec.toFixed(2)),
|
|
1707
|
+
reason
|
|
1708
|
+
});
|
|
1709
|
+
} catch (err) {
|
|
1710
|
+
debug$3(`${this.sessionId} - emitReadyTurnAudio: turn slice error: ${err && err.message}`);
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
/**
|
|
1716
|
+
* Force-emit any per-turn audio not yet sent (session end). Idempotent.
|
|
1717
|
+
*/
|
|
1718
|
+
_flushPendingTurnAudio(reason) {
|
|
1719
|
+
if (this._turnAudioForceDone) return;
|
|
1720
|
+
this._turnAudioForceDone = true;
|
|
1721
|
+
const pending = this._pendingTurnAudio || [];
|
|
1722
|
+
_info('turn_audio_flush_enter', {
|
|
1723
|
+
sessionId: this.sessionId,
|
|
1724
|
+
reason,
|
|
1725
|
+
pending: pending.length,
|
|
1726
|
+
emittedAlready: pending.filter(t => t.emitted).length,
|
|
1727
|
+
hasFormat: !!(this.audioStream && this.audioStream.format),
|
|
1728
|
+
totalBytes: this.audioStream && this.audioStream.totalBytes,
|
|
1729
|
+
turnAudioEnable: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]
|
|
1730
|
+
});
|
|
1731
|
+
this._emitReadyTurnAudio(reason, true);
|
|
1732
|
+
this._emitTrailingBotAudio(reason);
|
|
1733
|
+
_info('turn_audio_flush_done', {
|
|
1734
|
+
sessionId: this.sessionId,
|
|
1735
|
+
reason,
|
|
1736
|
+
emitted: this._turnAudioCounter || 0,
|
|
1737
|
+
pending: pending.length
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
/**
|
|
1742
|
+
* If the call ended on a bot turn (a bot message arrived with no me-response after it),
|
|
1743
|
+
* `_lastBotTurnStartSec` was never consumed by a turn. Slice that trailing bot audio
|
|
1744
|
+
* (bot start → end of recording) and emit it as a turn clip flagged `trailingBot` so the
|
|
1745
|
+
* UI/server place it on the final bot step.
|
|
1746
|
+
*/
|
|
1747
|
+
_emitTrailingBotAudio(reason) {
|
|
1748
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1749
|
+
if (this._trailingBotAudioEmitted) return;
|
|
1750
|
+
const startSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
|
|
1751
|
+
if (!lodash__default["default"].isFinite(startSec)) return;
|
|
1752
|
+
if (!this.audioStream || !this.audioStream.format) return;
|
|
1753
|
+
const endSec = this._recordingSecNow();
|
|
1754
|
+
if (!lodash__default["default"].isFinite(endSec) || endSec <= startSec) return;
|
|
1755
|
+
let audioBase64 = null;
|
|
1756
|
+
try {
|
|
1757
|
+
audioBase64 = this._sliceTurnAudio(startSec, endSec);
|
|
1758
|
+
} catch (err) {
|
|
1759
|
+
debug$3(`${this.sessionId} - emitTrailingBotAudio: slice error: ${err && err.message}`);
|
|
1760
|
+
return;
|
|
1761
|
+
}
|
|
1762
|
+
if (!audioBase64) return;
|
|
1763
|
+
this._trailingBotAudioEmitted = true;
|
|
1764
|
+
this._lastBotTurnStartSec = null;
|
|
1765
|
+
this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
|
|
1766
|
+
this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
|
|
1767
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1768
|
+
mimeType: 'audio/wav',
|
|
1769
|
+
base64: audioBase64,
|
|
1770
|
+
trailingBot: true,
|
|
1771
|
+
// place on the final bot step (no me-response followed)
|
|
1772
|
+
sessionContext: {
|
|
1773
|
+
testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
|
|
1774
|
+
testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
|
|
1775
|
+
}
|
|
1776
|
+
});
|
|
1777
|
+
_info('turn_audio_trailing_emitted', {
|
|
1778
|
+
sessionId: this.sessionId,
|
|
1779
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1780
|
+
startSec: Number(startSec.toFixed(2)),
|
|
1781
|
+
endSec: Number(endSec.toFixed(2)),
|
|
1782
|
+
reason
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
_agentSpeechRmsThreshold() {
|
|
1786
|
+
const raw = process.env.VOIP_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1787
|
+
const n = raw != null ? Number(raw) : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1788
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1789
|
+
}
|
|
1790
|
+
_agentSpeechSustainedWindows() {
|
|
1791
|
+
const raw = process.env.VOIP_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1792
|
+
const n = raw != null ? parseInt(raw, 10) : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1793
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1794
|
+
}
|
|
1795
|
+
_audioStreamBytesPerSec() {
|
|
1796
|
+
const fmt = this.audioStream && this.audioStream.format;
|
|
1797
|
+
if (!fmt) return null;
|
|
1798
|
+
return fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8);
|
|
1799
|
+
}
|
|
1800
|
+
_pcmBufferRms(pcm, bitsPerSample) {
|
|
1801
|
+
if (!pcm || pcm.length < 2 || bitsPerSample !== 16) return 0;
|
|
1802
|
+
let sum = 0;
|
|
1803
|
+
let count = 0;
|
|
1804
|
+
for (let i = 0; i + 1 < pcm.length; i += 2) {
|
|
1805
|
+
const sample = pcm.readInt16LE(i);
|
|
1806
|
+
sum += sample * sample;
|
|
1807
|
+
count += 1;
|
|
1808
|
+
}
|
|
1809
|
+
return count > 0 ? Math.sqrt(sum / count) : 0;
|
|
1810
|
+
}
|
|
1811
|
+
_readWavPcmInfo(wavBuffer) {
|
|
1812
|
+
if (!wavBuffer || wavBuffer.length < 44) return null;
|
|
1813
|
+
if (wavBuffer.toString('ascii', 0, 4) !== 'RIFF' || wavBuffer.toString('ascii', 8, 12) !== 'WAVE') {
|
|
1814
|
+
return null;
|
|
1815
|
+
}
|
|
1816
|
+
let offset = 12;
|
|
1817
|
+
let sampleRate = null;
|
|
1818
|
+
let channels = null;
|
|
1819
|
+
let bitsPerSample = null;
|
|
1820
|
+
let dataOffset = null;
|
|
1821
|
+
let dataLength = null;
|
|
1822
|
+
while (offset + 8 <= wavBuffer.length) {
|
|
1823
|
+
const chunkId = wavBuffer.toString('ascii', offset, offset + 4);
|
|
1824
|
+
const chunkSize = wavBuffer.readUInt32LE(offset + 4);
|
|
1825
|
+
const chunkStart = offset + 8;
|
|
1826
|
+
if (chunkId === 'fmt ' && chunkSize >= 16) {
|
|
1827
|
+
channels = wavBuffer.readUInt16LE(chunkStart + 2);
|
|
1828
|
+
sampleRate = wavBuffer.readUInt32LE(chunkStart + 4);
|
|
1829
|
+
bitsPerSample = wavBuffer.readUInt16LE(chunkStart + 14);
|
|
1830
|
+
} else if (chunkId === 'data') {
|
|
1831
|
+
dataOffset = chunkStart;
|
|
1832
|
+
dataLength = chunkSize;
|
|
1833
|
+
break;
|
|
1834
|
+
}
|
|
1835
|
+
offset = chunkStart + chunkSize + chunkSize % 2;
|
|
1836
|
+
}
|
|
1837
|
+
if (!sampleRate || !channels || !bitsPerSample || dataOffset == null) return null;
|
|
1838
|
+
const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
|
|
1839
|
+
if (!bytesPerSec) return null;
|
|
1840
|
+
const pcmLength = dataLength != null ? Math.min(dataLength, wavBuffer.length - dataOffset) : wavBuffer.length - dataOffset;
|
|
1841
|
+
return {
|
|
1842
|
+
pcmOffset: dataOffset,
|
|
1843
|
+
pcmLength,
|
|
1844
|
+
bytesPerSec,
|
|
1845
|
+
bitsPerSample,
|
|
1846
|
+
sampleRate,
|
|
1847
|
+
channels
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
_findAudibleLeadInSecFromPcm(pcm, bytesPerSec, bitsPerSample) {
|
|
1851
|
+
if (!pcm || !bytesPerSec) return null;
|
|
1852
|
+
const threshold = this._agentSpeechRmsThreshold();
|
|
1853
|
+
const sustainedWindows = this._agentSpeechSustainedWindows();
|
|
1854
|
+
const windowBytes = Math.max(2, Math.floor(bytesPerSec * (AGENT_SPEECH_RMS_WINDOW_MS / 1000)));
|
|
1855
|
+
const hopBytes = Math.max(2, Math.floor(windowBytes / 2));
|
|
1856
|
+
let streak = 0;
|
|
1857
|
+
let onsetPos = null;
|
|
1858
|
+
for (let pos = 0; pos + windowBytes <= pcm.length; pos += hopBytes) {
|
|
1859
|
+
const rms = this._pcmBufferRms(pcm.subarray(pos, pos + windowBytes), bitsPerSample);
|
|
1860
|
+
if (rms >= threshold) {
|
|
1861
|
+
if (streak === 0) onsetPos = pos;
|
|
1862
|
+
streak += 1;
|
|
1863
|
+
if (streak >= sustainedWindows) {
|
|
1864
|
+
return onsetPos / bytesPerSec;
|
|
1865
|
+
}
|
|
1866
|
+
} else {
|
|
1867
|
+
streak = 0;
|
|
1868
|
+
onsetPos = null;
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
return null;
|
|
1872
|
+
}
|
|
1873
|
+
_findAudibleLeadInSecFromWavBuffer(wavBuffer) {
|
|
1874
|
+
const info = this._readWavPcmInfo(wavBuffer);
|
|
1875
|
+
if (!info) return null;
|
|
1876
|
+
const pcm = wavBuffer.subarray(info.pcmOffset, info.pcmOffset + info.pcmLength);
|
|
1877
|
+
return this._findAudibleLeadInSecFromPcm(pcm, info.bytesPerSec, info.bitsPerSample);
|
|
1878
|
+
}
|
|
1879
|
+
_findAudibleRecordingStartSecOnStream(playedSec, wireSec) {
|
|
1880
|
+
if (!lodash__default["default"].isFinite(playedSec)) return null;
|
|
1881
|
+
const bytesPerSec = this._audioStreamBytesPerSec();
|
|
1882
|
+
const stream = this.audioStream;
|
|
1883
|
+
if (!bytesPerSec || !stream || !stream.pcmParts.length) return null;
|
|
1884
|
+
const startByte = Math.max(0, Math.floor(playedSec * bytesPerSec));
|
|
1885
|
+
const pcm = Buffer.concat(stream.pcmParts);
|
|
1886
|
+
if (startByte >= pcm.length) return null;
|
|
1887
|
+
const bitsPerSample = stream.format.bitsPerSample;
|
|
1888
|
+
const leadInSec = this._findAudibleLeadInSecFromPcm(pcm.subarray(startByte), bytesPerSec, bitsPerSample);
|
|
1889
|
+
if (!lodash__default["default"].isFinite(leadInSec)) return null;
|
|
1890
|
+
let heardSec = playedSec + leadInSec;
|
|
1891
|
+
if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
|
|
1892
|
+
return heardSec;
|
|
1893
|
+
}
|
|
1894
|
+
_findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments) {
|
|
1895
|
+
if (!lodash__default["default"].isFinite(playedSec) || !lodash__default["default"].isArray(attachments)) return null;
|
|
1896
|
+
const tts = attachments.find(a => a && a.name === 'tts.wav' && a.base64);
|
|
1897
|
+
if (!tts) return null;
|
|
1898
|
+
try {
|
|
1899
|
+
const wavBuffer = Buffer.from(tts.base64, 'base64');
|
|
1900
|
+
const leadInSec = this._findAudibleLeadInSecFromWavBuffer(wavBuffer);
|
|
1901
|
+
if (!lodash__default["default"].isFinite(leadInSec)) return null;
|
|
1902
|
+
let heardSec = playedSec + leadInSec;
|
|
1903
|
+
if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
|
|
1904
|
+
return heardSec;
|
|
1905
|
+
} catch (err) {
|
|
1906
|
+
debug$3(`${this.sessionId} - TTS lead-in scan failed: ${err && err.message}`);
|
|
1907
|
+
return null;
|
|
1908
|
+
}
|
|
1909
|
+
}
|
|
1910
|
+
_resolveAgentHeardRecordingStartSec(voipAgent, attachments) {
|
|
1911
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
|
|
1912
|
+
const playedSec = voipAgent.playedRecordingStartSec;
|
|
1913
|
+
const wireSec = voipAgent.wireRecordingStartSec;
|
|
1914
|
+
const fromTts = this._findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments);
|
|
1915
|
+
const fromStream = this._findAudibleRecordingStartSecOnStream(playedSec, wireSec);
|
|
1916
|
+
const candidates = [fromTts, fromStream].filter(s => lodash__default["default"].isFinite(s));
|
|
1917
|
+
if (!candidates.length) return null;
|
|
1918
|
+
// Prefer the later onset — mixed recording can spike before clear TTS speech.
|
|
1919
|
+
return Math.max(...candidates);
|
|
1920
|
+
}
|
|
1921
|
+
_applyAgentHeardRecordingStartSec(voipAgent, attachments) {
|
|
1922
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
|
|
1923
|
+
const heardSec = this._resolveAgentHeardRecordingStartSec(voipAgent, attachments);
|
|
1924
|
+
if (!lodash__default["default"].isFinite(heardSec) || heardSec <= voipAgent.playedRecordingStartSec) {
|
|
1925
|
+
return lodash__default["default"].isFinite(voipAgent.heardRecordingStartSec) ? voipAgent.heardRecordingStartSec : null;
|
|
1926
|
+
}
|
|
1927
|
+
const prev = voipAgent.heardRecordingStartSec;
|
|
1928
|
+
if (lodash__default["default"].isFinite(prev) && prev >= heardSec) return prev;
|
|
1929
|
+
voipAgent.heardRecordingStartSec = heardSec;
|
|
1930
|
+
this._markReplyTrace({
|
|
1931
|
+
heardRecordingStartSec: heardSec
|
|
1932
|
+
});
|
|
1933
|
+
return heardSec;
|
|
1934
|
+
}
|
|
1935
|
+
_maybeDetectAgentAudibleOnRecording(voipAgent) {
|
|
1936
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return;
|
|
1937
|
+
this._applyAgentHeardRecordingStartSec(voipAgent);
|
|
1938
|
+
}
|
|
1939
|
+
_markReplyTrace(patch) {
|
|
1940
|
+
if (!this._replyTrace || !patch) return;
|
|
1941
|
+
Object.assign(this._replyTrace, patch);
|
|
1942
|
+
}
|
|
1943
|
+
_captureSttFinalForReplyTrace(parsedData, msgPreview) {
|
|
1944
|
+
const data = parsedData && parsedData.data;
|
|
1945
|
+
const atMs = parsedData._receivedAtMs || Date.now();
|
|
1946
|
+
const recordingAtSttFinalSec = this._recordingSecNow();
|
|
1947
|
+
if (parsedData && lodash__default["default"].isFinite(recordingAtSttFinalSec)) {
|
|
1948
|
+
parsedData.recordingAtSttFinalSec = recordingAtSttFinalSec;
|
|
1949
|
+
}
|
|
1950
|
+
// Dedicated, self-documenting anchor for the downstream "STT transport" sub-phase
|
|
1951
|
+
// (receivedAtMs - finalEmittedWallMs). Unlike the generic _receivedAtMs (stamped on
|
|
1952
|
+
// every WS frame), this is set only on the accepted STT-final.
|
|
1953
|
+
if (parsedData && lodash__default["default"].isFinite(atMs)) {
|
|
1954
|
+
parsedData.sttFinalReceivedAtMs = atMs;
|
|
1955
|
+
}
|
|
1956
|
+
this._replyTrace = {
|
|
1957
|
+
sessionId: this.sessionId,
|
|
1958
|
+
botMessagePreview: msgPreview || undefined,
|
|
1959
|
+
sttFinalAtMs: atMs,
|
|
1960
|
+
sttRecordingStartSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'start')) ? data.start : null,
|
|
1961
|
+
sttRecordingEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'end')) ? data.end : null,
|
|
1962
|
+
sttSpeechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'speechEndSec')) ? data.speechEndSec : null,
|
|
1963
|
+
recordingAtSttFinalSec: lodash__default["default"].isFinite(recordingAtSttFinalSec) ? recordingAtSttFinalSec : null,
|
|
1964
|
+
queueAtMs: null,
|
|
1965
|
+
recordingAtQueueSec: null,
|
|
1966
|
+
psstTimerArmedAtMs: null,
|
|
1967
|
+
psstScheduledMs: null,
|
|
1968
|
+
psstTimerFiredAtMs: null,
|
|
1969
|
+
psstFireDelayMs: null,
|
|
1970
|
+
userSaysAtMs: null,
|
|
1971
|
+
coachWaitMs: null,
|
|
1972
|
+
ttsStartAtMs: null,
|
|
1973
|
+
ttsEndAtMs: null,
|
|
1974
|
+
ttsSynthMs: null,
|
|
1975
|
+
wireAtMs: null,
|
|
1976
|
+
wireRecordingStartSec: null,
|
|
1977
|
+
sendAudioAtMs: null,
|
|
1978
|
+
playedRecordingStartSec: null,
|
|
1979
|
+
playbackAtMs: null,
|
|
1980
|
+
heardRecordingStartSec: null,
|
|
1981
|
+
agentEndRecordingSec: null,
|
|
1982
|
+
wireKind: null,
|
|
1983
|
+
inputType: null,
|
|
1984
|
+
requestedDurationMs: null,
|
|
1985
|
+
meMessagePreview: null
|
|
1986
|
+
};
|
|
1987
|
+
}
|
|
1988
|
+
_captureBotQueuedForReplyTrace(queuedAt) {
|
|
1989
|
+
if (!this._replyTrace) return;
|
|
1990
|
+
this._replyTrace.queueAtMs = queuedAt;
|
|
1991
|
+
this._replyTrace.recordingAtQueueSec = this._recordingSecNow();
|
|
1992
|
+
}
|
|
1993
|
+
_captureUserSaysStart(msgPreview) {
|
|
1994
|
+
if (!this._replyTrace) return;
|
|
1995
|
+
const now = Date.now();
|
|
1996
|
+
this._replyTrace.userSaysAtMs = now;
|
|
1997
|
+
this._replyTrace.meMessagePreview = msgPreview || undefined;
|
|
1998
|
+
const queueAt = this._replyTrace.queueAtMs || this._lastBotSaysQueuedAt;
|
|
1999
|
+
if (lodash__default["default"].isFinite(queueAt)) {
|
|
2000
|
+
if (!this._replyTrace.queueAtMs) this._replyTrace.queueAtMs = queueAt;
|
|
2001
|
+
this._replyTrace.coachWaitMs = now - queueAt;
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
_captureAgentWire(voipAgent, inputType) {
|
|
2005
|
+
if (!this._replyTrace || !voipAgent) return;
|
|
2006
|
+
this._replyTrace.wireAtMs = voipAgent.wireSentAtMs;
|
|
2007
|
+
this._replyTrace.wireRecordingStartSec = voipAgent.wireRecordingStartSec;
|
|
2008
|
+
this._replyTrace.wireKind = voipAgent.wireKind;
|
|
2009
|
+
this._replyTrace.inputType = inputType;
|
|
2010
|
+
this._replyTrace.requestedDurationMs = voipAgent.requestedDurationMs;
|
|
2011
|
+
if (lodash__default["default"].isFinite(voipAgent.ttsSynthMs)) this._replyTrace.ttsSynthMs = voipAgent.ttsSynthMs;
|
|
2012
|
+
}
|
|
2013
|
+
_finalizeWallPipeline(voipAgent) {
|
|
2014
|
+
const t = this._replyTrace;
|
|
2015
|
+
if (!voipAgent || !t) return;
|
|
2016
|
+
voipAgent.wallPipeline = {
|
|
2017
|
+
psstScheduledMs: lodash__default["default"].isFinite(t.psstScheduledMs) ? t.psstScheduledMs : null,
|
|
2018
|
+
psstFireDelayMs: lodash__default["default"].isFinite(t.psstFireDelayMs) ? t.psstFireDelayMs : null,
|
|
2019
|
+
coachWaitMs: lodash__default["default"].isFinite(t.coachWaitMs) ? t.coachWaitMs : null,
|
|
2020
|
+
userSaysAtMs: lodash__default["default"].isFinite(t.userSaysAtMs) ? t.userSaysAtMs : null,
|
|
2021
|
+
ttsStartAtMs: lodash__default["default"].isFinite(t.ttsStartAtMs) ? t.ttsStartAtMs : null,
|
|
2022
|
+
ttsEndAtMs: lodash__default["default"].isFinite(t.ttsEndAtMs) ? t.ttsEndAtMs : null,
|
|
2023
|
+
ttsSynthMs: lodash__default["default"].isFinite(t.ttsSynthMs) ? t.ttsSynthMs : null,
|
|
2024
|
+
wireAtMs: lodash__default["default"].isFinite(t.wireAtMs) ? t.wireAtMs : null,
|
|
2025
|
+
sendAudioAtMs: lodash__default["default"].isFinite(t.sendAudioAtMs) ? t.sendAudioAtMs : null
|
|
2026
|
+
};
|
|
2027
|
+
}
|
|
2028
|
+
_replyTraceMsFromSttFinal(atMs) {
|
|
2029
|
+
const anchor = this._replyTrace && this._replyTrace.sttFinalAtMs;
|
|
2030
|
+
if (!lodash__default["default"].isFinite(anchor) || !lodash__default["default"].isFinite(atMs)) return null;
|
|
2031
|
+
return Math.round(atMs - anchor);
|
|
2032
|
+
}
|
|
2033
|
+
_replyTraceRecMs(fromSec, toSec) {
|
|
2034
|
+
if (!lodash__default["default"].isFinite(fromSec) || !lodash__default["default"].isFinite(toSec)) return null;
|
|
2035
|
+
return Math.round((toSec - fromSec) * 1000);
|
|
2036
|
+
}
|
|
2037
|
+
_logReplyTrace(trigger) {
|
|
2038
|
+
const t = this._replyTrace;
|
|
2039
|
+
if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
|
|
2040
|
+
_info('voip_reply_trace', {
|
|
2041
|
+
sessionId: t.sessionId,
|
|
2042
|
+
trigger,
|
|
2043
|
+
botPreview: t.botMessagePreview,
|
|
2044
|
+
mePreview: t.meMessagePreview,
|
|
2045
|
+
sttRecordingStartSec: t.sttRecordingStartSec,
|
|
2046
|
+
sttRecordingEndSec: t.sttRecordingEndSec,
|
|
2047
|
+
sttSpeechEndSec: t.sttSpeechEndSec,
|
|
2048
|
+
recordingAtSttFinalSec: t.recordingAtSttFinalSec,
|
|
2049
|
+
recordingAtQueueSec: t.recordingAtQueueSec,
|
|
2050
|
+
wireRecordingStartSec: t.wireRecordingStartSec,
|
|
2051
|
+
wireKind: t.wireKind,
|
|
2052
|
+
inputType: t.inputType,
|
|
2053
|
+
requestedDurationMs: t.requestedDurationMs,
|
|
2054
|
+
ttsSynthMs: t.ttsSynthMs,
|
|
2055
|
+
coachWaitMs: t.coachWaitMs,
|
|
2056
|
+
psstScheduledMs: t.psstScheduledMs,
|
|
2057
|
+
psstFireDelayMs: t.psstFireDelayMs,
|
|
2058
|
+
ms_sttFinal_to_queue: this._replyTraceMsFromSttFinal(t.queueAtMs),
|
|
2059
|
+
ms_sttFinal_to_psstFire: this._replyTraceMsFromSttFinal(t.psstTimerFiredAtMs),
|
|
2060
|
+
ms_sttFinal_to_userSays: this._replyTraceMsFromSttFinal(t.userSaysAtMs),
|
|
2061
|
+
ms_sttFinal_to_ttsStart: this._replyTraceMsFromSttFinal(t.ttsStartAtMs),
|
|
2062
|
+
ms_sttFinal_to_ttsEnd: this._replyTraceMsFromSttFinal(t.ttsEndAtMs),
|
|
2063
|
+
ms_sttFinal_to_wire: this._replyTraceMsFromSttFinal(t.wireAtMs),
|
|
2064
|
+
ms_sttFinal_to_sendAudio: this._replyTraceMsFromSttFinal(t.sendAudioAtMs),
|
|
2065
|
+
ms_userSays_to_ttsStart: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.ttsStartAtMs) ? Math.round(t.ttsStartAtMs - t.userSaysAtMs) : null,
|
|
2066
|
+
ms_userSays_to_wire: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.wireAtMs) ? Math.round(t.wireAtMs - t.userSaysAtMs) : null,
|
|
2067
|
+
ms_queue_to_userSays: t.coachWaitMs,
|
|
2068
|
+
recMs_sttEnd_to_queue: this._replyTraceRecMs(t.sttRecordingEndSec, t.recordingAtQueueSec),
|
|
2069
|
+
recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
|
|
2070
|
+
recMs_speechEnd_to_wire: this._replyTraceRecMs(t.sttSpeechEndSec, t.wireRecordingStartSec)
|
|
2071
|
+
});
|
|
2072
|
+
}
|
|
2073
|
+
_logReplyTraceHeard(agentEndRecordingSec) {
|
|
2074
|
+
const t = this._replyTrace;
|
|
2075
|
+
if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
|
|
2076
|
+
const heardSec = t.heardRecordingStartSec;
|
|
2077
|
+
const playedSec = t.playedRecordingStartSec;
|
|
2078
|
+
if (lodash__default["default"].isFinite(agentEndRecordingSec)) {
|
|
2079
|
+
t.agentEndRecordingSec = agentEndRecordingSec;
|
|
2080
|
+
}
|
|
2081
|
+
_info('voip_reply_trace_heard', {
|
|
2082
|
+
sessionId: t.sessionId,
|
|
2083
|
+
playedRecordingStartSec: playedSec,
|
|
2084
|
+
heardRecordingStartSec: heardSec,
|
|
2085
|
+
agentEndRecordingSec: t.agentEndRecordingSec,
|
|
2086
|
+
wireRecordingStartSec: t.wireRecordingStartSec,
|
|
2087
|
+
sttRecordingEndSec: t.sttRecordingEndSec,
|
|
2088
|
+
sttSpeechEndSec: t.sttSpeechEndSec,
|
|
2089
|
+
recMs_sttEnd_to_played: this._replyTraceRecMs(t.sttRecordingEndSec, playedSec),
|
|
2090
|
+
recMs_speechEnd_to_played: this._replyTraceRecMs(t.sttSpeechEndSec, playedSec),
|
|
2091
|
+
recMs_sttEnd_to_heard: this._replyTraceRecMs(t.sttRecordingEndSec, heardSec),
|
|
2092
|
+
recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
|
|
2093
|
+
recMs_wire_to_played: this._replyTraceRecMs(t.wireRecordingStartSec, playedSec),
|
|
2094
|
+
recMs_wire_to_heard: this._replyTraceRecMs(t.wireRecordingStartSec, heardSec)
|
|
2095
|
+
});
|
|
2096
|
+
this._replyTrace = null;
|
|
2097
|
+
}
|
|
1262
2098
|
_voipWsCanSend() {
|
|
1263
2099
|
return !this.stopCalled && this.ws && this.ws.readyState === ws__default["default"].OPEN;
|
|
1264
2100
|
}
|
|
@@ -1311,6 +2147,8 @@ class BotiumConnectorVoip {
|
|
|
1311
2147
|
if (typeof this._emitBufferedFullRecordIfAny === 'function') {
|
|
1312
2148
|
this._emitBufferedFullRecordIfAny('stop_final_guard');
|
|
1313
2149
|
}
|
|
2150
|
+
// Last-resort flush in case neither audioStreamEnd nor fullRecordEnd fired.
|
|
2151
|
+
this._flushPendingTurnAudio('stop_final_guard');
|
|
1314
2152
|
}
|
|
1315
2153
|
this._emitBufferedFullRecordIfAny = null;
|
|
1316
2154
|
}
|
|
@@ -1521,27 +2359,99 @@ class BotiumConnectorVoip {
|
|
|
1521
2359
|
};
|
|
1522
2360
|
}).filter(Boolean);
|
|
1523
2361
|
}
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
if (
|
|
2362
|
+
|
|
2363
|
+
// Matches a substring rule against ONLY the latest buffered STT final chunk.
|
|
2364
|
+
// This keeps the rule's custom timeout in effect for just the one following
|
|
2365
|
+
// final: once a final no longer matches, callers fall back to the default
|
|
2366
|
+
// timeout (the convoStep expected text and the cumulative buffer are
|
|
2367
|
+
// deliberately not considered, so a stale match cannot stick).
|
|
2368
|
+
_getJoinRuleBySubstring(botMsgs) {
|
|
2369
|
+
if (!lodash__default["default"].isArray(botMsgs) || botMsgs.length === 0) return null;
|
|
2370
|
+
const last = botMsgs[botMsgs.length - 1];
|
|
2371
|
+
const text = last && typeof last.messageText === 'string' ? last.messageText : '';
|
|
2372
|
+
if (!text) return null;
|
|
2373
|
+
const loweredText = text.toLowerCase();
|
|
1535
2374
|
const rules = this._normalizeJoinRulesBySubstring();
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
2375
|
+
return rules.find(rule => loweredText.includes(rule.substring.toLowerCase())) || null;
|
|
2376
|
+
}
|
|
2377
|
+
_normalizeSttDictionaryReplacements() {
|
|
2378
|
+
const rawRules = this.caps[Capabilities.VOIP_STT_DICTIONARY_REPLACEMENTS];
|
|
2379
|
+
if (lodash__default["default"].isNil(rawRules) || rawRules === '') return [];
|
|
2380
|
+
let parsedRules = rawRules;
|
|
2381
|
+
if (lodash__default["default"].isString(rawRules)) {
|
|
2382
|
+
try {
|
|
2383
|
+
parsedRules = JSON.parse(rawRules);
|
|
2384
|
+
} catch (err) {
|
|
2385
|
+
debug$3(`Invalid ${Capabilities.VOIP_STT_DICTIONARY_REPLACEMENTS} JSON: ${err.message || err}`);
|
|
2386
|
+
return [];
|
|
2387
|
+
}
|
|
1540
2388
|
}
|
|
1541
|
-
|
|
2389
|
+
if (!lodash__default["default"].isArray(parsedRules)) {
|
|
2390
|
+
debug$3(`Invalid ${Capabilities.VOIP_STT_DICTIONARY_REPLACEMENTS}: expected array`);
|
|
2391
|
+
return [];
|
|
2392
|
+
}
|
|
2393
|
+
return parsedRules.map(rule => {
|
|
2394
|
+
if (!rule || typeof rule !== 'object') return null;
|
|
2395
|
+
const fromValues = lodash__default["default"].isArray(rule.from) ? rule.from : [rule.from];
|
|
2396
|
+
const from = lodash__default["default"].uniq(fromValues.map(value => value != null ? String(value).trim() : '').filter(Boolean));
|
|
2397
|
+
const to = rule.to != null ? String(rule.to).trim() : '';
|
|
2398
|
+
if (from.length === 0 || !to) return null;
|
|
2399
|
+
return {
|
|
2400
|
+
from,
|
|
2401
|
+
to
|
|
2402
|
+
};
|
|
2403
|
+
}).filter(Boolean);
|
|
2404
|
+
}
|
|
2405
|
+
_applySttDictionaryReplacements(text) {
|
|
2406
|
+
if (!lodash__default["default"].isString(text) || text.length === 0) return {
|
|
2407
|
+
text,
|
|
2408
|
+
applied: []
|
|
2409
|
+
};
|
|
2410
|
+
const replacements = this._normalizeSttDictionaryReplacements();
|
|
2411
|
+
if (replacements.length === 0) return {
|
|
2412
|
+
text,
|
|
2413
|
+
applied: []
|
|
2414
|
+
};
|
|
2415
|
+
const replacementsByFrom = new Map();
|
|
2416
|
+
const fromAlternatives = [];
|
|
2417
|
+
replacements.forEach(rule => {
|
|
2418
|
+
rule.from.forEach(from => {
|
|
2419
|
+
const fromKey = from.toLowerCase();
|
|
2420
|
+
if (!replacementsByFrom.has(fromKey)) {
|
|
2421
|
+
replacementsByFrom.set(fromKey, rule.to);
|
|
2422
|
+
fromAlternatives.push(from);
|
|
2423
|
+
}
|
|
2424
|
+
});
|
|
2425
|
+
});
|
|
2426
|
+
if (fromAlternatives.length === 0) return {
|
|
2427
|
+
text,
|
|
2428
|
+
applied: []
|
|
2429
|
+
};
|
|
2430
|
+
fromAlternatives.sort((a, b) => b.length - a.length);
|
|
2431
|
+
const matcher = new RegExp(fromAlternatives.map(value => lodash__default["default"].escapeRegExp(value)).join('|'), 'gi');
|
|
2432
|
+
const applied = [];
|
|
2433
|
+
const replacedText = text.replace(matcher, match => {
|
|
2434
|
+
const to = replacementsByFrom.get(match.toLowerCase());
|
|
2435
|
+
applied.push({
|
|
2436
|
+
from: match,
|
|
2437
|
+
to
|
|
2438
|
+
});
|
|
2439
|
+
return to;
|
|
2440
|
+
});
|
|
2441
|
+
return {
|
|
2442
|
+
text: replacedText,
|
|
2443
|
+
applied
|
|
2444
|
+
};
|
|
2445
|
+
}
|
|
2446
|
+
_decorateSourceDataWithSttDictionaryReplacements(sourceData, replacementResult) {
|
|
2447
|
+
if (!replacementResult || !lodash__default["default"].isArray(replacementResult.applied) || replacementResult.applied.length === 0) return sourceData;
|
|
2448
|
+
return Object.assign({}, sourceData, {
|
|
2449
|
+
sttDictionaryOriginalMessage: replacementResult.text === sourceData?.data?.message ? undefined : sourceData?.data?.message,
|
|
2450
|
+
sttDictionaryReplacements: replacementResult.applied
|
|
2451
|
+
});
|
|
1542
2452
|
}
|
|
1543
2453
|
_hasJoinLogicHookOrRule(convoStep) {
|
|
1544
|
-
return !lodash__default["default"].isNil(this._getJoinLogicHook(convoStep)) || !lodash__default["default"].isNil(this._getJoinRuleBySubstring(
|
|
2454
|
+
return !lodash__default["default"].isNil(this._getJoinLogicHook(convoStep)) || !lodash__default["default"].isNil(this._getJoinRuleBySubstring(this.botMsgs));
|
|
1545
2455
|
}
|
|
1546
2456
|
_toJoinTimeoutMs(ms, isPsst) {
|
|
1547
2457
|
const parsed = parseInt(ms, 10);
|
|
@@ -1556,7 +2466,7 @@ class BotiumConnectorVoip {
|
|
|
1556
2466
|
const joinHookTimeoutMs = this._toJoinTimeoutMs(joinLogicHook.args[0], isPsst);
|
|
1557
2467
|
if (lodash__default["default"].isFinite(joinHookTimeoutMs) && joinHookTimeoutMs > 0) return joinHookTimeoutMs;
|
|
1558
2468
|
}
|
|
1559
|
-
const joinRule = this._getJoinRuleBySubstring(
|
|
2469
|
+
const joinRule = this._getJoinRuleBySubstring(botMsgs);
|
|
1560
2470
|
if (joinRule) {
|
|
1561
2471
|
const joinRuleTimeoutMs = this._toJoinTimeoutMs(joinRule.timeoutMs, isPsst);
|
|
1562
2472
|
if (lodash__default["default"].isFinite(joinRuleTimeoutMs) && joinRuleTimeoutMs > 0) return joinRuleTimeoutMs;
|
|
@@ -1583,6 +2493,100 @@ class BotiumConnectorVoip {
|
|
|
1583
2493
|
}
|
|
1584
2494
|
return null;
|
|
1585
2495
|
}
|
|
2496
|
+
|
|
2497
|
+
/**
|
|
2498
|
+
* Build a well-formed WAV Buffer from raw PCM bytes and a format descriptor.
|
|
2499
|
+
* @param {Buffer} pcm raw PCM bytes (no header)
|
|
2500
|
+
* @param {{ sampleRate: number, channels: number, bitsPerSample: number }} fmt
|
|
2501
|
+
* @returns {Buffer}
|
|
2502
|
+
*/
|
|
2503
|
+
_buildWavBuffer(pcm, fmt) {
|
|
2504
|
+
const {
|
|
2505
|
+
sampleRate,
|
|
2506
|
+
channels,
|
|
2507
|
+
bitsPerSample
|
|
2508
|
+
} = fmt;
|
|
2509
|
+
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
|
2510
|
+
const blockAlign = channels * (bitsPerSample / 8);
|
|
2511
|
+
const dataSize = pcm.length;
|
|
2512
|
+
const header = Buffer.alloc(44);
|
|
2513
|
+
header.write('RIFF', 0);
|
|
2514
|
+
header.writeUInt32LE(36 + dataSize, 4);
|
|
2515
|
+
header.write('WAVE', 8);
|
|
2516
|
+
header.write('fmt ', 12);
|
|
2517
|
+
header.writeUInt32LE(16, 16); // fmt chunk size
|
|
2518
|
+
header.writeUInt16LE(1, 20); // PCM format
|
|
2519
|
+
header.writeUInt16LE(channels, 22);
|
|
2520
|
+
header.writeUInt32LE(sampleRate, 24);
|
|
2521
|
+
header.writeUInt32LE(byteRate, 28);
|
|
2522
|
+
header.writeUInt16LE(blockAlign, 32);
|
|
2523
|
+
header.writeUInt16LE(bitsPerSample, 34);
|
|
2524
|
+
header.write('data', 36);
|
|
2525
|
+
header.writeUInt32LE(dataSize, 40);
|
|
2526
|
+
return Buffer.concat([header, pcm]);
|
|
2527
|
+
}
|
|
2528
|
+
|
|
2529
|
+
/**
|
|
2530
|
+
* Slice a segment of the continuously buffered PCM audio stream and return
|
|
2531
|
+
* it as a base64-encoded WAV string.
|
|
2532
|
+
*
|
|
2533
|
+
* @param {number} startSec start of the segment (seconds from call connect)
|
|
2534
|
+
* @param {number} endSec end of the segment (seconds from call connect)
|
|
2535
|
+
* @returns {string|null} base64 WAV or null if the stream is not ready
|
|
2536
|
+
*/
|
|
2537
|
+
_sliceTurnAudio(startSec, endSec) {
|
|
2538
|
+
const stream = this.audioStream;
|
|
2539
|
+
if (!stream || !stream.format || !stream.pcmParts || !stream.pcmParts.length) return null;
|
|
2540
|
+
if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) return null;
|
|
2541
|
+
const {
|
|
2542
|
+
sampleRate,
|
|
2543
|
+
channels,
|
|
2544
|
+
bitsPerSample
|
|
2545
|
+
} = stream.format;
|
|
2546
|
+
const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
|
|
2547
|
+
const frameBytes = channels * (bitsPerSample / 8);
|
|
2548
|
+
const offsetSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_OFFSET_MS] || 0) / 1000;
|
|
2549
|
+
const paddingSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_PADDING_MS] || 0) / 1000;
|
|
2550
|
+
const adjStart = Math.max(0, startSec + offsetSec);
|
|
2551
|
+
const adjEnd = endSec + paddingSec;
|
|
2552
|
+
|
|
2553
|
+
// Frame-align the byte boundaries.
|
|
2554
|
+
const startByte = Math.floor(adjStart * bytesPerSec / frameBytes) * frameBytes;
|
|
2555
|
+
const endByte = Math.ceil(adjEnd * bytesPerSec / frameBytes) * frameBytes;
|
|
2556
|
+
if (startByte >= stream.totalBytes) {
|
|
2557
|
+
debug$3(`${this.sessionId} - _sliceTurnAudio: startByte ${startByte} >= totalBytes ${stream.totalBytes}, skipping`);
|
|
2558
|
+
return null;
|
|
2559
|
+
}
|
|
2560
|
+
const clampedEnd = Math.min(endByte, stream.totalBytes);
|
|
2561
|
+
const sliceLen = clampedEnd - startByte;
|
|
2562
|
+
if (sliceLen <= 0) return null;
|
|
2563
|
+
|
|
2564
|
+
// Materialise only the bytes we need from the part list.
|
|
2565
|
+
const pcm = Buffer.allocUnsafe(sliceLen);
|
|
2566
|
+
let written = 0;
|
|
2567
|
+
let offset = 0;
|
|
2568
|
+
for (const part of stream.pcmParts) {
|
|
2569
|
+
const partEnd = offset + part.length;
|
|
2570
|
+
if (partEnd <= startByte) {
|
|
2571
|
+
offset += part.length;
|
|
2572
|
+
continue;
|
|
2573
|
+
}
|
|
2574
|
+
if (offset >= clampedEnd) break;
|
|
2575
|
+
const copyFrom = Math.max(0, startByte - offset);
|
|
2576
|
+
const copyTo = Math.min(part.length, clampedEnd - offset);
|
|
2577
|
+
part.copy(pcm, written, copyFrom, copyTo);
|
|
2578
|
+
written += copyTo - copyFrom;
|
|
2579
|
+
offset += part.length;
|
|
2580
|
+
}
|
|
2581
|
+
if (written === 0) return null;
|
|
2582
|
+
const slicedPcm = written < sliceLen ? pcm.slice(0, written) : pcm;
|
|
2583
|
+
const wavBuf = this._buildWavBuffer(slicedPcm, {
|
|
2584
|
+
sampleRate,
|
|
2585
|
+
channels,
|
|
2586
|
+
bitsPerSample
|
|
2587
|
+
});
|
|
2588
|
+
return wavBuf.toString('base64');
|
|
2589
|
+
}
|
|
1586
2590
|
}
|
|
1587
2591
|
var connector = BotiumConnectorVoip;
|
|
1588
2592
|
|
|
@@ -1654,6 +2658,36 @@ var botiumConnectorVoip = {
|
|
|
1654
2658
|
type: 'json',
|
|
1655
2659
|
required: false,
|
|
1656
2660
|
advanced: true
|
|
2661
|
+
}, {
|
|
2662
|
+
name: 'VOIP_STT_DICTIONARY_REPLACEMENTS',
|
|
2663
|
+
label: 'STT dictionary replacements',
|
|
2664
|
+
type: 'json',
|
|
2665
|
+
required: false,
|
|
2666
|
+
advanced: true
|
|
2667
|
+
}, {
|
|
2668
|
+
name: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE',
|
|
2669
|
+
label: 'Enable SDP media type text',
|
|
2670
|
+
type: 'boolean',
|
|
2671
|
+
required: false,
|
|
2672
|
+
advanced: true
|
|
2673
|
+
}, {
|
|
2674
|
+
name: 'VOIP_TURN_AUDIO_ENABLE',
|
|
2675
|
+
label: 'Attach per-turn audio to each transcript message',
|
|
2676
|
+
type: 'boolean',
|
|
2677
|
+
required: false,
|
|
2678
|
+
advanced: true
|
|
2679
|
+
}, {
|
|
2680
|
+
name: 'VOIP_TURN_AUDIO_PADDING_MS',
|
|
2681
|
+
label: 'Extra milliseconds appended after each turn audio slice (absorbs STT boundary jitter)',
|
|
2682
|
+
type: 'int',
|
|
2683
|
+
required: false,
|
|
2684
|
+
advanced: true
|
|
2685
|
+
}, {
|
|
2686
|
+
name: 'VOIP_TURN_AUDIO_OFFSET_MS',
|
|
2687
|
+
label: 'Millisecond offset applied to every turn audio start time (positive = shift right)',
|
|
2688
|
+
type: 'int',
|
|
2689
|
+
required: false,
|
|
2690
|
+
advanced: true
|
|
1657
2691
|
}]
|
|
1658
2692
|
},
|
|
1659
2693
|
PluginLogicHooks: {
|