botium-connector-voip 0.0.29 → 0.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/botium-connector-voip-cjs.js +1069 -63
- package/dist/botium-connector-voip-cjs.js.map +1 -1
- package/dist/botium-connector-voip-es.js +1069 -63
- package/dist/botium-connector-voip-es.js.map +1 -1
- package/index.js +21 -0
- package/package.json +1 -1
- package/src/connector.js +1047 -59
|
@@ -26,6 +26,12 @@ const debug$3 = debug__default["default"]('botium-connector-voip');
|
|
|
26
26
|
// debug = high-frequency diagnostics (DEBUG=botium-connector-voip). warn = degraded but continuing.
|
|
27
27
|
// error = abort/failure. No secrets in info; STT text only as length or truncated in info.
|
|
28
28
|
|
|
29
|
+
/** WS frame types logged only at start/end handlers — not per-chunk (hundreds per call). */
|
|
30
|
+
const WS_DEBUG_SILENT_TYPES = new Set(['audioStreamChunk', 'fullRecordChunk']);
|
|
31
|
+
const AGENT_SPEECH_RMS_WINDOW_MS = 100;
|
|
32
|
+
const DEFAULT_AGENT_SPEECH_RMS_THRESHOLD = 500;
|
|
33
|
+
const DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS = 2;
|
|
34
|
+
const WS_DEBUG_BASE64_FIELD_NAMES = new Set(['chunk', 'buffer', 'base64', 'fullRecord', 'full_record', 'audio', 'audioData', 'b64_buffer']);
|
|
29
35
|
const _info = (event, data) => {
|
|
30
36
|
const parts = Object.entries({
|
|
31
37
|
event,
|
|
@@ -39,17 +45,40 @@ const sanitizeDtmfDigits = raw => {
|
|
|
39
45
|
if (raw == null) return '';
|
|
40
46
|
return String(raw).replace(/[^0-9*#ABCDabcd]/g, '').toUpperCase();
|
|
41
47
|
};
|
|
48
|
+
|
|
49
|
+
// Matches voipcall.generate_dtmf_sequence defaults (100 ms tone, 50 ms pause between digits).
|
|
50
|
+
const DTMF_TONE_MS = 100;
|
|
51
|
+
const DTMF_PAUSE_MS = 50;
|
|
52
|
+
const DTMF_MS_PER_DIGIT = 200;
|
|
53
|
+
const DEFAULT_AUDIO_STREAM_INTERVAL_MS = 250;
|
|
54
|
+
const audioStreamIntervalMs = () => {
|
|
55
|
+
const n = Number(process.env.VOIP_AUDIO_STREAM_INTERVAL_MS || process.env.AUDIO_STREAM_INTERVAL_MS);
|
|
56
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AUDIO_STREAM_INTERVAL_MS;
|
|
57
|
+
};
|
|
58
|
+
const dtmfPlaybackMs = digitCount => {
|
|
59
|
+
if (!digitCount || digitCount <= 0) return 0;
|
|
60
|
+
return digitCount * DTMF_TONE_MS + Math.max(0, digitCount - 1) * DTMF_PAUSE_MS;
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/** Wait before turn-audio slice so DTMF PCM is flushed through audioStream (250 ms default). */
|
|
64
|
+
const dtmfTurnAudioWaitMs = digitCount => {
|
|
65
|
+
const playbackMs = dtmfPlaybackMs(digitCount);
|
|
66
|
+
const streamMs = audioStreamIntervalMs();
|
|
67
|
+
return Math.max(digitCount * DTMF_MS_PER_DIGIT, playbackMs + streamMs + 50);
|
|
68
|
+
};
|
|
42
69
|
const Capabilities = {
|
|
43
70
|
VOIP_STT_URL_STREAM: 'VOIP_STT_URL_STREAM',
|
|
44
71
|
VOIP_STT_PARAMS_STREAM: 'VOIP_STT_PARAMS_STREAM',
|
|
45
72
|
VOIP_STT_METHOD_STREAM: 'VOIP_STT_METHOD_STREAM',
|
|
46
73
|
VOIP_STT_BODY_STREAM: 'VOIP_STT_BODY_STREAM',
|
|
47
74
|
VOIP_STT_BODY: 'VOIP_STT_BODY',
|
|
75
|
+
VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 'VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS',
|
|
48
76
|
VOIP_STT_HEADERS: 'VOIP_STT_HEADERS',
|
|
49
77
|
VOIP_STT_TIMEOUT: 'VOIP_STT_TIMEOUT',
|
|
50
78
|
VOIP_STT_MESSAGE_HANDLING: 'VOIP_STT_MESSAGE_HANDLING',
|
|
51
79
|
VOIP_STT_MESSAGE_HANDLING_TIMEOUT: 'VOIP_STT_MESSAGE_HANDLING_TIMEOUT',
|
|
52
80
|
VOIP_STT_MESSAGE_HANDLING_TIMEOUT_SUBSEQUENT: 'VOIP_STT_MESSAGE_HANDLING_TIMEOUT_SUBSEQUENT',
|
|
81
|
+
VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS: 'VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS',
|
|
53
82
|
VOIP_JOIN_SILENCE_DURATION_BY_SUBSTRING: 'VOIP_JOIN_SILENCE_DURATION_BY_SUBSTRING',
|
|
54
83
|
VOIP_STT_DICTIONARY_REPLACEMENTS: 'VOIP_STT_DICTIONARY_REPLACEMENTS',
|
|
55
84
|
VOIP_STT_MESSAGE_HANDLING_DELIMITER: 'VOIP_STT_MESSAGE_HANDLING_DELIMITER',
|
|
@@ -84,6 +113,8 @@ const Capabilities = {
|
|
|
84
113
|
VOIP_ICE_TURN_PROTOCOL: 'VOIP_ICE_TURN_PROTOCOL',
|
|
85
114
|
VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 'VOIP_WEBSOCKET_CONNECT_MAXRETRIES',
|
|
86
115
|
VOIP_WEBSOCKET_CONNECT_TIMEOUT: 'VOIP_WEBSOCKET_CONNECT_TIMEOUT',
|
|
116
|
+
VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 'VOIP_CALL_SETUP_RETRY_487_MAXRETRIES',
|
|
117
|
+
VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 'VOIP_CALL_SETUP_RETRY_487_TIMEOUT',
|
|
87
118
|
VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_ENABLE',
|
|
88
119
|
VOIP_SILENCE_DURATION_TIMEOUT: 'VOIP_SILENCE_DURATION_TIMEOUT',
|
|
89
120
|
VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE',
|
|
@@ -92,7 +123,10 @@ const Capabilities = {
|
|
|
92
123
|
VOIP_USE_GLOBAL_VOIP_WORKER: 'VOIP_USE_GLOBAL_VOIP_WORKER',
|
|
93
124
|
VOIP_USER_INPUT_PREFER_VOICE: 'VOIP_USER_INPUT_PREFER_VOICE',
|
|
94
125
|
VOIP_EMIT_SPECULATIVE_TEXT: 'VOIP_EMIT_SPECULATIVE_TEXT',
|
|
95
|
-
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE'
|
|
126
|
+
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE',
|
|
127
|
+
VOIP_TURN_AUDIO_ENABLE: 'VOIP_TURN_AUDIO_ENABLE',
|
|
128
|
+
VOIP_TURN_AUDIO_PADDING_MS: 'VOIP_TURN_AUDIO_PADDING_MS',
|
|
129
|
+
VOIP_TURN_AUDIO_OFFSET_MS: 'VOIP_TURN_AUDIO_OFFSET_MS'
|
|
96
130
|
};
|
|
97
131
|
const Defaults = {
|
|
98
132
|
VOIP_STT_METHOD: 'POST',
|
|
@@ -104,19 +138,62 @@ const Defaults = {
|
|
|
104
138
|
VOIP_TTS_PREFETCH_ENABLE: true,
|
|
105
139
|
VOIP_STT_MESSAGE_HANDLING: 'ORIGINAL',
|
|
106
140
|
VOIP_STT_MESSAGE_HANDLING_TIMEOUT: 2500,
|
|
141
|
+
// Extra wall-clock grace added to the JOIN/PSST flush window when the last buffered final was
|
|
142
|
+
// cut mid-utterance (no Azure-detected end-of-speech silence). Absorbs STT delivery latency so a
|
|
143
|
+
// paused-then-resumed prompt is joined instead of split. Set to 0 to disable. See _getPsstLatencyGraceMs.
|
|
144
|
+
VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS: 1500,
|
|
107
145
|
VOIP_STT_MESSAGE_HANDLING_DELIMITER: '. ',
|
|
108
146
|
VOIP_STT_MESSAGE_HANDLING_PUNCTUATION: '.!?',
|
|
109
147
|
VOIP_WEBSOCKET_CONNECT_TIMEOUT: 4000,
|
|
110
148
|
VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 5,
|
|
149
|
+
// Retry the whole call setup when the SIP peer terminates the INVITE with a
|
|
150
|
+
// transient '487 Request Terminated' before the call connects.
|
|
151
|
+
VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 2,
|
|
152
|
+
VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 2000,
|
|
111
153
|
VOIP_SILENCE_DURATION_TIMEOUT: 2500,
|
|
112
154
|
VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: false,
|
|
113
155
|
VOIP_SILENCE_DURATION_TIMEOUT_START: 1000,
|
|
114
156
|
VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: false,
|
|
115
157
|
VOIP_STT_CONFIDENCE_THRESHOLD: 0.5,
|
|
158
|
+
VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 500,
|
|
116
159
|
VOIP_USE_GLOBAL_VOIP_WORKER: false,
|
|
117
160
|
VOIP_SIP_PROTOCOL: 'TCP',
|
|
118
161
|
VOIP_USER_INPUT_PREFER_VOICE: true,
|
|
119
|
-
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false
|
|
162
|
+
VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false,
|
|
163
|
+
VOIP_TURN_AUDIO_ENABLE: true,
|
|
164
|
+
VOIP_TURN_AUDIO_PADDING_MS: 150,
|
|
165
|
+
VOIP_TURN_AUDIO_OFFSET_MS: 0
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
// Inject the Azure end-of-speech segmentation timeout into the STT body. botium-speech-processing
|
|
169
|
+
// applies azure.config.properties via SpeechConfig.setProperty(), so this controls how long Azure
|
|
170
|
+
// waits after the last word before emitting final=true. Only applied for the Azure engine, never
|
|
171
|
+
// overrides a value already present in the profile config, and clones so the capability object stays
|
|
172
|
+
// untouched. Set the capability to 0/empty to disable injection.
|
|
173
|
+
const injectAzureSegmentationTimeout = (body, sttParams, timeoutMs) => {
|
|
174
|
+
const ms = Number(timeoutMs);
|
|
175
|
+
if (!lodash__default["default"].isFinite(ms) || ms <= 0) return body;
|
|
176
|
+
const isAzure = sttParams && sttParams.stt === 'azure' || body && typeof body === 'object' && body.azure || typeof body === 'string' && body.indexOf('azure') !== -1;
|
|
177
|
+
if (!isAzure) return body;
|
|
178
|
+
let next;
|
|
179
|
+
if (body == null) {
|
|
180
|
+
next = {};
|
|
181
|
+
} else if (typeof body === 'string') {
|
|
182
|
+
try {
|
|
183
|
+
next = JSON.parse(body);
|
|
184
|
+
} catch (err) {
|
|
185
|
+
return body;
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
next = lodash__default["default"].cloneDeep(body);
|
|
189
|
+
}
|
|
190
|
+
if (!lodash__default["default"].isObject(next.azure)) next.azure = {};
|
|
191
|
+
if (!lodash__default["default"].isObject(next.azure.config)) next.azure.config = {};
|
|
192
|
+
if (!lodash__default["default"].isObject(next.azure.config.properties)) next.azure.config.properties = {};
|
|
193
|
+
if (next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs == null) {
|
|
194
|
+
next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs = String(Math.round(ms));
|
|
195
|
+
}
|
|
196
|
+
return next;
|
|
120
197
|
};
|
|
121
198
|
const TTS_HTTP_AGENT = new http__default["default"].Agent({
|
|
122
199
|
keepAlive: true
|
|
@@ -145,6 +222,8 @@ class BotiumConnectorVoip {
|
|
|
145
222
|
// For debugging latency between incoming STT (bot says) and outgoing audio (sendAudio)
|
|
146
223
|
this._lastBotSaysQueuedAt = null;
|
|
147
224
|
this._lastBotSaysText = null;
|
|
225
|
+
this._replyTrace = null;
|
|
226
|
+
this._activeUserSaysVoipAgent = null;
|
|
148
227
|
this._speculativeTurnToken = 0;
|
|
149
228
|
}
|
|
150
229
|
async Validate() {
|
|
@@ -193,6 +272,27 @@ class BotiumConnectorVoip {
|
|
|
193
272
|
this.convoStep = null;
|
|
194
273
|
this._lastBotSaysQueuedAt = null;
|
|
195
274
|
this._lastBotSaysText = null;
|
|
275
|
+
this._replyTrace = null;
|
|
276
|
+
this._activeUserSaysVoipAgent = null;
|
|
277
|
+
this.audioStream = {
|
|
278
|
+
format: null,
|
|
279
|
+
pcmParts: [],
|
|
280
|
+
totalBytes: 0,
|
|
281
|
+
complete: false
|
|
282
|
+
};
|
|
283
|
+
this._turnAudioCounter = 0;
|
|
284
|
+
this._meTurnAudioOrdinal = -1; // 0-based ordinal of REAL user turns this session
|
|
285
|
+
this._lastBotTurnStartSec = null;
|
|
286
|
+
// Per-turn audio is NOT sliced inline (that would force UserSays to wait for the
|
|
287
|
+
// recording to catch up with playback). Instead each turn records a lightweight
|
|
288
|
+
// descriptor here; slices are cut and emitted as MESSAGE_ATTACHMENT progressively,
|
|
289
|
+
// as soon as the recording buffer has reached each turn's playback end (so the live
|
|
290
|
+
// UI can show them mid-run), with a forced flush at session end for any remainder.
|
|
291
|
+
// Zero added latency for TTS and DTMF turns.
|
|
292
|
+
this._pendingTurnAudio = [];
|
|
293
|
+
this._turnAudioPrevEndSec = null;
|
|
294
|
+
this._turnAudioForceDone = false;
|
|
295
|
+
this._trailingBotAudioEmitted = false;
|
|
196
296
|
if (this.ttsCache) {
|
|
197
297
|
this.ttsCache.clear();
|
|
198
298
|
}
|
|
@@ -200,6 +300,7 @@ class BotiumConnectorVoip {
|
|
|
200
300
|
const queuedAt = Date.now();
|
|
201
301
|
this._lastBotSaysQueuedAt = queuedAt;
|
|
202
302
|
this._lastBotSaysText = botMsg && botMsg.messageText ? String(botMsg.messageText) : null;
|
|
303
|
+
this._captureBotQueuedForReplyTrace(queuedAt);
|
|
203
304
|
// Stamp the wall-clock instant at which the connector released the bot
|
|
204
305
|
// utterance to botium-core's queue. Paired with `_receivedAtMs` (last
|
|
205
306
|
// STT final frame) this gives the true "join silence" the connector
|
|
@@ -207,8 +308,33 @@ class BotiumConnectorVoip {
|
|
|
207
308
|
// message up with WaitBotSays().
|
|
208
309
|
if (botMsg && botMsg.sourceData) {
|
|
209
310
|
const head = Array.isArray(botMsg.sourceData) ? botMsg.sourceData[0] : botMsg.sourceData;
|
|
210
|
-
if (head && typeof head === 'object'
|
|
211
|
-
head.flushedAtMs = queuedAt;
|
|
311
|
+
if (head && typeof head === 'object') {
|
|
312
|
+
if (!('flushedAtMs' in head)) head.flushedAtMs = queuedAt;
|
|
313
|
+
if (this._replyTrace) {
|
|
314
|
+
if (lodash__default["default"].isFinite(this._replyTrace.psstFireDelayMs)) head.psstFireDelayMs = this._replyTrace.psstFireDelayMs;
|
|
315
|
+
if (lodash__default["default"].isFinite(this._replyTrace.psstScheduledMs)) head.psstScheduledMs = this._replyTrace.psstScheduledMs;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
// A turn = bot speaks first, then me responds.
|
|
320
|
+
// Save the bot's audio start so UserSays can slice the full exchange
|
|
321
|
+
// (bot + me) once me has finished speaking.
|
|
322
|
+
if (this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE] && botMsg && !(botMsg instanceof Error)) {
|
|
323
|
+
try {
|
|
324
|
+
const sd = botMsg.sourceData;
|
|
325
|
+
let startSec = null;
|
|
326
|
+
if (Array.isArray(sd) && sd.length > 0) {
|
|
327
|
+
startSec = lodash__default["default"].get(sd, '[0].data.start', null);
|
|
328
|
+
} else if (sd && typeof sd === 'object') {
|
|
329
|
+
startSec = lodash__default["default"].get(sd, 'data.start', null);
|
|
330
|
+
}
|
|
331
|
+
// Only record the FIRST bot message's start; if several bot messages
|
|
332
|
+
// arrive before the next UserSays they all belong to the same turn.
|
|
333
|
+
if (lodash__default["default"].isFinite(startSec) && this._lastBotTurnStartSec === null) {
|
|
334
|
+
this._lastBotTurnStartSec = startSec;
|
|
335
|
+
}
|
|
336
|
+
} catch (err) {
|
|
337
|
+
debug$3(`${this.sessionId} - sendBotMsg: saving turn start error: ${err && err.message}`);
|
|
212
338
|
}
|
|
213
339
|
}
|
|
214
340
|
setTimeout(() => this.queueBotSays(botMsg), 0);
|
|
@@ -229,6 +355,10 @@ class BotiumConnectorVoip {
|
|
|
229
355
|
botMsg.sourceData[0].silenceDuration = lodash__default["default"].isFinite(firstStart) ? firstStart : null;
|
|
230
356
|
botMsg.sourceData[0].voiceDuration = lodash__default["default"].isFinite(firstStart) && lodash__default["default"].isFinite(lastEnd) ? lastEnd - firstStart : null;
|
|
231
357
|
}
|
|
358
|
+
const lastSpeechEnd = lodash__default["default"].get(botMsgs, `[${botMsgs.length - 1}].sourceData.data.speechEndSec`, null);
|
|
359
|
+
if (lodash__default["default"].isFinite(lastSpeechEnd) && botMsg.sourceData[0] && botMsg.sourceData[0].data) {
|
|
360
|
+
botMsg.sourceData[0].data.speechEndSec = lastSpeechEnd;
|
|
361
|
+
}
|
|
232
362
|
return botMsg;
|
|
233
363
|
};
|
|
234
364
|
|
|
@@ -258,25 +388,41 @@ class BotiumConnectorVoip {
|
|
|
258
388
|
const isJoinMethod = sttHandling === 'JOIN' || sttHandling === 'PSST' || sttHandling === 'CONCAT' || this._hasJoinLogicHookOrRule(this.convoStep);
|
|
259
389
|
if (!isJoinMethod) return;
|
|
260
390
|
const joinTimeoutMs = this._getEffectiveJoinTimeoutMs(this.convoStep, this.botMsgs);
|
|
391
|
+
// Variant-3 latency grace: when the last buffered final was cut mid-utterance (no
|
|
392
|
+
// Azure-detected end-of-speech silence) a continuation is plausible but its STT delivery
|
|
393
|
+
// lags behind the audio by ~1s. Extend the wall-clock flush window by the grace so the
|
|
394
|
+
// resumed segment's partial/final (or a worker speechResumed event) can re-arm this timer
|
|
395
|
+
// and be joined, instead of flushing the first fragment on its own. A natural-ended final
|
|
396
|
+
// keeps the base window and flushes fast.
|
|
397
|
+
const graceMs = this._getPsstLatencyGraceMs();
|
|
398
|
+
const graceApplied = graceMs > 0 && lodash__default["default"].isFinite(joinTimeoutMs) && joinTimeoutMs > 0 && !this._isLastFinalNaturalEnd();
|
|
399
|
+
const effectiveWindowMs = graceApplied ? joinTimeoutMs + graceMs : joinTimeoutMs || 0;
|
|
261
400
|
if (this.silenceTimeout) {
|
|
262
401
|
clearTimeout(this.silenceTimeout);
|
|
263
402
|
this.silenceTimeout = null;
|
|
264
403
|
}
|
|
265
404
|
const bufferedAtArm = this.botMsgs.length;
|
|
266
405
|
const armedAt = Date.now();
|
|
406
|
+
this._markReplyTrace({
|
|
407
|
+
psstTimerArmedAtMs: armedAt,
|
|
408
|
+
psstScheduledMs: effectiveWindowMs || 0
|
|
409
|
+
});
|
|
267
410
|
_info('psst_timer_armed', {
|
|
268
411
|
sessionId: this.sessionId,
|
|
269
|
-
joinTimeoutMs:
|
|
412
|
+
joinTimeoutMs: effectiveWindowMs || 0,
|
|
413
|
+
baseTimeoutMs: joinTimeoutMs || 0,
|
|
414
|
+
graceMs: graceApplied ? graceMs : 0,
|
|
415
|
+
graceApplied,
|
|
270
416
|
bufferedChunks: bufferedAtArm,
|
|
271
417
|
stopCalled: !!this.stopCalled
|
|
272
418
|
});
|
|
273
|
-
// Emit the authoritative
|
|
419
|
+
// Emit the authoritative flush window so downstream consumers (e.g.
|
|
274
420
|
// SpeculationBuffer) can size their quiet threshold relative to it.
|
|
275
|
-
if (this.eventEmitter && lodash__default["default"].isFinite(
|
|
421
|
+
if (this.eventEmitter && lodash__default["default"].isFinite(effectiveWindowMs) && effectiveWindowMs > 0) {
|
|
276
422
|
try {
|
|
277
423
|
this.eventEmitter.emit('voip.psstTimerArmed', {
|
|
278
424
|
sessionId: this.sessionId,
|
|
279
|
-
joinTimeoutMs,
|
|
425
|
+
joinTimeoutMs: effectiveWindowMs,
|
|
280
426
|
bufferedChunks: bufferedAtArm,
|
|
281
427
|
armedAt
|
|
282
428
|
});
|
|
@@ -287,15 +433,20 @@ class BotiumConnectorVoip {
|
|
|
287
433
|
}
|
|
288
434
|
this.silenceTimeout = setTimeout(() => {
|
|
289
435
|
const fireDelay = Date.now() - armedAt;
|
|
436
|
+
this._markReplyTrace({
|
|
437
|
+
psstTimerFiredAtMs: Date.now(),
|
|
438
|
+
psstFireDelayMs: fireDelay
|
|
439
|
+
});
|
|
290
440
|
if (this.botMsgs.length > 0) {
|
|
291
441
|
_info('psst_timer_fired', {
|
|
292
442
|
sessionId: this.sessionId,
|
|
293
443
|
bufferedChunks: this.botMsgs.length,
|
|
294
444
|
actualDelayMs: fireDelay,
|
|
295
|
-
scheduledDelayMs:
|
|
445
|
+
scheduledDelayMs: effectiveWindowMs || 0,
|
|
446
|
+
graceApplied,
|
|
296
447
|
outcome: 'emit'
|
|
297
448
|
});
|
|
298
|
-
debug$3('Silence Duration Timeout (JOIN/PSST):',
|
|
449
|
+
debug$3('Silence Duration Timeout (JOIN/PSST):', effectiveWindowMs, 'ms');
|
|
299
450
|
sendBotMsg(joinBotMsg(this.botMsgs, this.joinLastPrevMsg));
|
|
300
451
|
this.firstMsg = false;
|
|
301
452
|
this.joinLastPrevMsg = this.botMsgs[this.botMsgs.length - 1];
|
|
@@ -308,11 +459,11 @@ class BotiumConnectorVoip {
|
|
|
308
459
|
sessionId: this.sessionId,
|
|
309
460
|
bufferedChunks: 0,
|
|
310
461
|
actualDelayMs: fireDelay,
|
|
311
|
-
scheduledDelayMs:
|
|
462
|
+
scheduledDelayMs: effectiveWindowMs || 0,
|
|
312
463
|
outcome: 'noop_empty_buffer'
|
|
313
464
|
});
|
|
314
465
|
}
|
|
315
|
-
},
|
|
466
|
+
}, effectiveWindowMs || 0);
|
|
316
467
|
};
|
|
317
468
|
|
|
318
469
|
// Flush buffered STT chunks on teardown so a late final is not lost when
|
|
@@ -438,15 +589,17 @@ class BotiumConnectorVoip {
|
|
|
438
589
|
await connectHttp(retryIndex + 1);
|
|
439
590
|
}
|
|
440
591
|
};
|
|
441
|
-
await connectHttp();
|
|
442
|
-
if (httpInitRetries > 0) {
|
|
443
|
-
_info('connected_after_retries', {
|
|
444
|
-
phase: 'initCall',
|
|
445
|
-
retries: httpInitRetries
|
|
446
|
-
});
|
|
447
|
-
}
|
|
448
592
|
return new Promise((resolve, reject) => {
|
|
449
|
-
|
|
593
|
+
// Worker session details (port) come from connectHttp() and are recomputed
|
|
594
|
+
// on every (re)try, since each setup attempt gets a fresh worker session.
|
|
595
|
+
let wsEndpoint = null;
|
|
596
|
+
const computeWsEndpoint = () => `${this.caps[Capabilities.VOIP_USE_GLOBAL_VOIP_WORKER] ? process.env.BOTIUM_VOIP_WORKER_URL : this.caps[Capabilities.VOIP_WORKER_URL]}/ws/${data.port}`;
|
|
597
|
+
// 487-retry bookkeeping: a transient '487 Request Terminated' that arrives
|
|
598
|
+
// before the call connects re-runs the whole setup (fresh initCall -> ws
|
|
599
|
+
// -> SIP INVITE) up to max487Retries times.
|
|
600
|
+
let setup487Retries = 0;
|
|
601
|
+
let convoStepListenerAttached = false;
|
|
602
|
+
const max487Retries = this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_MAXRETRIES];
|
|
450
603
|
const connectWs = retryIndex => {
|
|
451
604
|
retryIndex = retryIndex || 0;
|
|
452
605
|
return new Promise((resolve, reject) => {
|
|
@@ -480,7 +633,7 @@ class BotiumConnectorVoip {
|
|
|
480
633
|
});
|
|
481
634
|
});
|
|
482
635
|
};
|
|
483
|
-
|
|
636
|
+
const onWsConnected = wsRetries => {
|
|
484
637
|
if (wsRetries > 0) {
|
|
485
638
|
_info('connected_after_retries', {
|
|
486
639
|
phase: 'websocket',
|
|
@@ -494,26 +647,32 @@ class BotiumConnectorVoip {
|
|
|
494
647
|
this.caps[Capabilities.VOIP_ICE_STUN_SERVERS] = this.caps[Capabilities.VOIP_ICE_STUN_SERVERS].split(',');
|
|
495
648
|
}
|
|
496
649
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
650
|
+
|
|
651
|
+
// Attach this once: the listener reads this.ws dynamically, so it keeps
|
|
652
|
+
// working across 487 setup retries that swap out the websocket.
|
|
653
|
+
if (!convoStepListenerAttached) {
|
|
654
|
+
convoStepListenerAttached = true;
|
|
655
|
+
this.eventEmitter.on('CONVO_STEP_NEXT', (container, convoStep) => {
|
|
656
|
+
this.convoStep = convoStep;
|
|
657
|
+
this._maybePrefetchTts(convoStep);
|
|
658
|
+
// For PSST: send join silence duration per step to VOIP worker (controls PSST silence trigger)
|
|
659
|
+
try {
|
|
660
|
+
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'PSST' && this.ws && this.ws.readyState === ws__default["default"].OPEN) {
|
|
661
|
+
const silenceMs = this._getEffectiveJoinTimeoutMs(convoStep, this.botMsgs);
|
|
662
|
+
if (lodash__default["default"].isFinite(silenceMs) && silenceMs > 0 && this.sessionId) {
|
|
663
|
+
debug$3(`PSST: sending silenceDurationMs=${silenceMs} for sessionId=${this.sessionId}`);
|
|
664
|
+
this.ws.send(JSON.stringify({
|
|
665
|
+
METHOD: 'setSttSilenceDuration',
|
|
666
|
+
sessionId: this.sessionId,
|
|
667
|
+
silenceDurationMs: silenceMs
|
|
668
|
+
}));
|
|
669
|
+
}
|
|
511
670
|
}
|
|
671
|
+
} catch (err) {
|
|
672
|
+
debug$3(`Failed sending PSST silence duration to VOIP worker: ${err.message || err}`);
|
|
512
673
|
}
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
}
|
|
516
|
-
});
|
|
674
|
+
});
|
|
675
|
+
}
|
|
517
676
|
this.silence = null;
|
|
518
677
|
this.msgCount = 0;
|
|
519
678
|
this.sttPartialCount = 0;
|
|
@@ -561,11 +720,12 @@ class BotiumConnectorVoip {
|
|
|
561
720
|
ICE_TURN_PROTOCOL: this.caps[Capabilities.VOIP_ICE_TURN_PROTOCOL] || 'TCP',
|
|
562
721
|
MIN_SILENCE_DURATION: this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT_ENABLE] ? this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT] : null,
|
|
563
722
|
SDP_MEDIA_TYPE_TEXT_ENABLE: !!this.caps[Capabilities.VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE],
|
|
723
|
+
AUDIO_STREAM: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE],
|
|
564
724
|
STT_LEGACY: sttLegacy,
|
|
565
725
|
STT_CONFIG: {
|
|
566
726
|
stt_url: sttUrl,
|
|
567
727
|
stt_params: this.caps[Capabilities.VOIP_STT_PARAMS_STREAM],
|
|
568
|
-
stt_body: this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null
|
|
728
|
+
stt_body: injectAzureSegmentationTimeout(this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null, this.caps[Capabilities.VOIP_STT_PARAMS_STREAM], this.caps[Capabilities.VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS])
|
|
569
729
|
},
|
|
570
730
|
TTS_CONFIG: {
|
|
571
731
|
tts_url: this.caps[Capabilities.VOIP_TTS_URL],
|
|
@@ -630,7 +790,7 @@ class BotiumConnectorVoip {
|
|
|
630
790
|
// (fullRecord*) and hard errors so `full_record.wav` is delivered
|
|
631
791
|
// on early-completion hangups. Post-Stop STT frames remain blocked.
|
|
632
792
|
if (this.stopCalled) {
|
|
633
|
-
const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error'];
|
|
793
|
+
const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error', 'audioStreamStart', 'audioStreamChunk', 'audioStreamEnd'];
|
|
634
794
|
if (!parsedData || !allowedPostStopTypes.includes(parsedData.type)) {
|
|
635
795
|
debug$3(`${this.sessionId} - Stop already called, ignoring incoming message`);
|
|
636
796
|
return;
|
|
@@ -642,7 +802,7 @@ class BotiumConnectorVoip {
|
|
|
642
802
|
if (!obj || typeof obj !== 'object') return;
|
|
643
803
|
for (const key of Object.keys(obj)) {
|
|
644
804
|
const val = obj[key];
|
|
645
|
-
if (typeof val === 'string' && val.length > 500) {
|
|
805
|
+
if (typeof val === 'string' && val.length > 0 && (WS_DEBUG_BASE64_FIELD_NAMES.has(key) || val.length > 500)) {
|
|
646
806
|
obj[key] = `<base64:${val.length}chars>`;
|
|
647
807
|
} else if (val && typeof val === 'object' && !Array.isArray(val)) {
|
|
648
808
|
sanitizeBase64Fields(val, `${prefix}${key}.`);
|
|
@@ -650,7 +810,9 @@ class BotiumConnectorVoip {
|
|
|
650
810
|
}
|
|
651
811
|
};
|
|
652
812
|
sanitizeBase64Fields(parsedDataLog);
|
|
653
|
-
|
|
813
|
+
if (!WS_DEBUG_SILENT_TYPES.has(parsedData?.type)) {
|
|
814
|
+
debug$3(JSON.stringify(parsedDataLog, null, 2));
|
|
815
|
+
}
|
|
654
816
|
const _extractFullRecordBase64 = pd => {
|
|
655
817
|
if (!pd) return null;
|
|
656
818
|
// Different VOIP workers may put the payload in various fields - search all string fields
|
|
@@ -754,6 +916,9 @@ class BotiumConnectorVoip {
|
|
|
754
916
|
reject(new Error('Error: Sip Registration failed'));
|
|
755
917
|
}
|
|
756
918
|
if (parsedData && parsedData.type === 'callinfo' && parsedData.status === 'connected') {
|
|
919
|
+
// Mark connected so a later terminal error is delivered to the bot
|
|
920
|
+
// conversation instead of triggering a (now pointless) setup retry.
|
|
921
|
+
this.connected = true;
|
|
757
922
|
_info('callinfo_connected', {
|
|
758
923
|
sessionId: this.sessionId
|
|
759
924
|
});
|
|
@@ -788,6 +953,23 @@ class BotiumConnectorVoip {
|
|
|
788
953
|
});
|
|
789
954
|
}
|
|
790
955
|
if (parsedData && parsedData.type === 'error') {
|
|
956
|
+
const errMsg = parsedData.message || '';
|
|
957
|
+
// The worker reports the SIP code inside the message string
|
|
958
|
+
// ("Disconnected because of error - Reason: 487 Request Terminated")
|
|
959
|
+
// and does not set a dedicated `code` field, so detect 487 from both.
|
|
960
|
+
const is487 = parsedData.code === 487 || parsedData.code === '487' || /\b487\b/.test(errMsg) || /request terminated/i.test(errMsg);
|
|
961
|
+
// A transient '487 Request Terminated' that arrives before the call
|
|
962
|
+
// connects: retry the whole call setup instead of failing the test.
|
|
963
|
+
if (is487 && !this.connected && setup487Retries < max487Retries) {
|
|
964
|
+
_info('ws_error_msg', {
|
|
965
|
+
sessionId: this.sessionId,
|
|
966
|
+
message: parsedData.message || null,
|
|
967
|
+
code: parsedData.code || null,
|
|
968
|
+
retrying487: true
|
|
969
|
+
});
|
|
970
|
+
retryCallSetup487(errMsg);
|
|
971
|
+
return;
|
|
972
|
+
}
|
|
791
973
|
flushPendingBotMsgs('error');
|
|
792
974
|
// Ensure buffered recording is not lost on terminal worker errors.
|
|
793
975
|
this._emitBufferedFullRecordIfAny('error_buffered');
|
|
@@ -801,6 +983,46 @@ class BotiumConnectorVoip {
|
|
|
801
983
|
sendBotMsg(new Error(`Error: ${parsedData.message}`));
|
|
802
984
|
}
|
|
803
985
|
|
|
986
|
+
// Per-turn audio stream: continuous PCM chunks received during the call.
|
|
987
|
+
// The connector buffers them so _sliceTurnAudio() can extract per-turn segments.
|
|
988
|
+
if (parsedData && parsedData.type === 'audioStreamStart') {
|
|
989
|
+
this.audioStream = {
|
|
990
|
+
format: {
|
|
991
|
+
sampleRate: parsedData.sampleRate,
|
|
992
|
+
channels: parsedData.channels,
|
|
993
|
+
bitsPerSample: parsedData.bitsPerSample,
|
|
994
|
+
dataOffset: parsedData.dataOffset
|
|
995
|
+
},
|
|
996
|
+
pcmParts: [],
|
|
997
|
+
totalBytes: 0,
|
|
998
|
+
complete: false
|
|
999
|
+
};
|
|
1000
|
+
debug$3(`${this.sessionId} - audioStreamStart sampleRate=${parsedData.sampleRate} channels=${parsedData.channels} bitsPerSample=${parsedData.bitsPerSample}`);
|
|
1001
|
+
}
|
|
1002
|
+
if (parsedData && parsedData.type === 'audioStreamChunk') {
|
|
1003
|
+
if (this.audioStream && parsedData.chunk) {
|
|
1004
|
+
try {
|
|
1005
|
+
const buf = Buffer.from(parsedData.chunk, 'base64');
|
|
1006
|
+
this.audioStream.pcmParts.push(buf);
|
|
1007
|
+
this.audioStream.totalBytes += buf.length;
|
|
1008
|
+
this._maybeDetectAgentAudibleOnRecording(this._activeUserSaysVoipAgent);
|
|
1009
|
+
// Emit any per-turn audio whose playback the recording has now caught up to,
|
|
1010
|
+
// so the live transcript can show it mid-run (no UserSays latency).
|
|
1011
|
+
this._emitReadyTurnAudio('audioStreamChunk', false);
|
|
1012
|
+
} catch (e) {
|
|
1013
|
+
debug$3(`${this.sessionId} - audioStreamChunk decode error: ${e && e.message}`);
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
if (parsedData && parsedData.type === 'audioStreamEnd') {
|
|
1018
|
+
if (this.audioStream) {
|
|
1019
|
+
this.audioStream.complete = true;
|
|
1020
|
+
}
|
|
1021
|
+
debug$3(`${this.sessionId} - audioStreamEnd totalBytes=${parsedData.totalBytes}`);
|
|
1022
|
+
// Buffer is complete — cut and emit the per-turn audio now.
|
|
1023
|
+
this._flushPendingTurnAudio('audioStreamEnd');
|
|
1024
|
+
}
|
|
1025
|
+
|
|
804
1026
|
// Full record streaming support:
|
|
805
1027
|
// - some VOIP workers send the recording in chunks and an end marker
|
|
806
1028
|
if (parsedData && parsedData.type === 'fullRecordStart') {
|
|
@@ -820,11 +1042,49 @@ class BotiumConnectorVoip {
|
|
|
820
1042
|
source: 'fullRecordEnd',
|
|
821
1043
|
base64Len
|
|
822
1044
|
});
|
|
1045
|
+
// Emit per-turn audio before `this.end = true` so it is captured by the
|
|
1046
|
+
// worker before Stop() resolves (no-op if audioStreamEnd already flushed).
|
|
1047
|
+
this._flushPendingTurnAudio('fullRecordEnd');
|
|
823
1048
|
// Flush before `this.end = true` so the buffered final STT is not
|
|
824
1049
|
// dropped when Stop() clears the PSST silence timer on teardown.
|
|
825
1050
|
flushPendingBotMsgs('fullRecordEnd');
|
|
826
1051
|
this.end = true;
|
|
827
1052
|
}
|
|
1053
|
+
if (parsedData && parsedData.type === 'agentPlaybackStarted') {
|
|
1054
|
+
const playbackData = parsedData.data || {};
|
|
1055
|
+
const playedSec = playbackData.playedRecordingStartSec;
|
|
1056
|
+
const active = this._activeUserSaysVoipAgent;
|
|
1057
|
+
if (active && lodash__default["default"].isFinite(playedSec)) {
|
|
1058
|
+
active.playedRecordingStartSec = playedSec;
|
|
1059
|
+
active.playbackAtMs = playbackData.playbackAtMs;
|
|
1060
|
+
if (lodash__default["default"].isFinite(playbackData.requestedDurationMs)) {
|
|
1061
|
+
active.playbackRequestedDurationMs = playbackData.requestedDurationMs;
|
|
1062
|
+
}
|
|
1063
|
+
if (lodash__default["default"].isFinite(playbackData.digitCount)) {
|
|
1064
|
+
active.digitCount = playbackData.digitCount;
|
|
1065
|
+
}
|
|
1066
|
+
this._markReplyTrace({
|
|
1067
|
+
playedRecordingStartSec: playedSec,
|
|
1068
|
+
playbackAtMs: playbackData.playbackAtMs
|
|
1069
|
+
});
|
|
1070
|
+
const heardSec = playbackData.wireKind === 'dtmf' ? playedSec : this._applyAgentHeardRecordingStartSec(active);
|
|
1071
|
+
if (lodash__default["default"].isFinite(heardSec)) {
|
|
1072
|
+
if (playbackData.wireKind === 'dtmf') {
|
|
1073
|
+
active.heardRecordingStartSec = heardSec;
|
|
1074
|
+
this._markReplyTrace({
|
|
1075
|
+
heardRecordingStartSec: heardSec
|
|
1076
|
+
});
|
|
1077
|
+
}
|
|
1078
|
+
debug$3(`${this.sessionId} - agent audible on recording at ${heardSec}s (played=${playedSec}s)`);
|
|
1079
|
+
}
|
|
1080
|
+
// Log the heard reply-trace now that playback is audible — earlier than the
|
|
1081
|
+
// old turn-audio callback and free of the next-turn _replyTrace race. The
|
|
1082
|
+
// agent's end on the recording = played + playback length.
|
|
1083
|
+
const playbackSec = lodash__default["default"].isFinite(active.playbackRequestedDurationMs) && active.playbackRequestedDurationMs > 0 ? active.playbackRequestedDurationMs / 1000 : playbackData.wireKind === 'dtmf' ? this._dtmfPlaybackSec(active, active.digitCount || 1) : null;
|
|
1084
|
+
const agentEndSec = lodash__default["default"].isFinite(playbackSec) ? playedSec + playbackSec : undefined;
|
|
1085
|
+
this._logReplyTraceHeard(agentEndSec);
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
828
1088
|
if (parsedData && parsedData.type === 'silence') {
|
|
829
1089
|
if (lodash__default["default"].isNil(this._getIgnoreSilenceDurationAsserterLogicHook(this.convoStep))) {
|
|
830
1090
|
if (!this._hasJoinLogicHookOrRule(this.convoStep) && parsedData.data.silence.length > 0) {
|
|
@@ -834,6 +1094,36 @@ class BotiumConnectorVoip {
|
|
|
834
1094
|
}
|
|
835
1095
|
}
|
|
836
1096
|
}
|
|
1097
|
+
if (parsedData && parsedData.type === 'speech' && parsedData.event === 'onSpeechResumed') {
|
|
1098
|
+
// Positive VAD signal from the worker: the bot started speaking again after a pause.
|
|
1099
|
+
// It arrives ~1s before the resumed segment's first STT partial, so when we are
|
|
1100
|
+
// buffering finals (PSST/JOIN) we re-arm the flush timer now — keeping the buffered
|
|
1101
|
+
// fragment open to be joined instead of flushed on its own. Bounded by the same
|
|
1102
|
+
// extension budget as partial-driven re-arms to prevent infinite stranding.
|
|
1103
|
+
_info('speech_resumed', {
|
|
1104
|
+
sessionId: this.sessionId
|
|
1105
|
+
});
|
|
1106
|
+
const sttHandling = this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING];
|
|
1107
|
+
const isJoinMethod = sttHandling === 'JOIN' || sttHandling === 'PSST' || sttHandling === 'CONCAT' || this._hasJoinLogicHookOrRule(this.convoStep);
|
|
1108
|
+
if (isJoinMethod && this.botMsgs && this.botMsgs.length > 0) {
|
|
1109
|
+
const MAX_EXTENSION_MS = 60000;
|
|
1110
|
+
const now = Date.now();
|
|
1111
|
+
const withinCap = this.psstFirstRearmAt == null || now - this.psstFirstRearmAt < MAX_EXTENSION_MS;
|
|
1112
|
+
if (withinCap) {
|
|
1113
|
+
if (this.psstFirstRearmAt == null) this.psstFirstRearmAt = now;
|
|
1114
|
+
this.psstRearmCount++;
|
|
1115
|
+
_info('psst_timer_extended', {
|
|
1116
|
+
sessionId: this.sessionId,
|
|
1117
|
+
rearmCount: this.psstRearmCount,
|
|
1118
|
+
msSinceFirstRearm: now - this.psstFirstRearmAt,
|
|
1119
|
+
bufferedChunks: this.botMsgs.length,
|
|
1120
|
+
reason: 'speech_resumed',
|
|
1121
|
+
silenceDurationMs: lodash__default["default"].get(parsedData, 'data.silenceDurationMs', null)
|
|
1122
|
+
});
|
|
1123
|
+
armJoinSilenceTimer();
|
|
1124
|
+
}
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
837
1127
|
if (parsedData && parsedData.type === 'fullRecord') {
|
|
838
1128
|
// Non-chunked full record
|
|
839
1129
|
const base64 = _extractFullRecordBase64(parsedData);
|
|
@@ -953,8 +1243,13 @@ class BotiumConnectorVoip {
|
|
|
953
1243
|
messageLength: msgLen,
|
|
954
1244
|
confidence: this._getConfidenceScore(parsedData),
|
|
955
1245
|
threshold: confidenceThreshold,
|
|
956
|
-
accepted: successfulConfidenceScore
|
|
1246
|
+
accepted: successfulConfidenceScore,
|
|
1247
|
+
segmentEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.end')) ? parsedData.data.end : null,
|
|
1248
|
+
speechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.speechEndSec')) ? parsedData.data.speechEndSec : null
|
|
957
1249
|
});
|
|
1250
|
+
if (successfulConfidenceScore) {
|
|
1251
|
+
this._captureSttFinalForReplyTrace(parsedData, msgPreview);
|
|
1252
|
+
}
|
|
958
1253
|
// ORIGINAL: always emit final message immediately (ignore JOIN hooks/rules).
|
|
959
1254
|
if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'ORIGINAL') {
|
|
960
1255
|
let botMsg = {
|
|
@@ -1051,9 +1346,49 @@ class BotiumConnectorVoip {
|
|
|
1051
1346
|
}
|
|
1052
1347
|
}
|
|
1053
1348
|
});
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1349
|
+
};
|
|
1350
|
+
const retryCallSetup487 = reasonMessage => {
|
|
1351
|
+
setup487Retries++;
|
|
1352
|
+
_info('call_setup_retry_487', {
|
|
1353
|
+
sessionId: this.sessionId,
|
|
1354
|
+
attempt: setup487Retries,
|
|
1355
|
+
max: max487Retries,
|
|
1356
|
+
reason: reasonMessage || null
|
|
1357
|
+
});
|
|
1358
|
+
debug$3(`Call setup 487 retry ${setup487Retries}/${max487Retries}: ${reasonMessage}`);
|
|
1359
|
+
// Tear down the dead websocket so its stale handlers cannot fire while
|
|
1360
|
+
// the next attempt establishes a fresh worker session.
|
|
1361
|
+
try {
|
|
1362
|
+
if (this.ws) {
|
|
1363
|
+
this.ws.removeAllListeners();
|
|
1364
|
+
this.ws.terminate();
|
|
1365
|
+
}
|
|
1366
|
+
} catch (err) {
|
|
1367
|
+
debug$3(`Call setup 487 retry: websocket teardown failed: ${err && err.message}`);
|
|
1368
|
+
}
|
|
1369
|
+
this.ws = null;
|
|
1370
|
+
this.wsOpened = false;
|
|
1371
|
+
this.end = false;
|
|
1372
|
+
setTimeout(() => {
|
|
1373
|
+
establishCall().catch(err => reject(new Error('Error: ' + err)));
|
|
1374
|
+
}, this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_TIMEOUT]);
|
|
1375
|
+
};
|
|
1376
|
+
const establishCall = async () => {
|
|
1377
|
+
// Each (re)try gets a fresh worker session: new initCall -> new port ->
|
|
1378
|
+
// new websocket -> new SIP INVITE.
|
|
1379
|
+
httpInitRetries = 0;
|
|
1380
|
+
await connectHttp();
|
|
1381
|
+
if (httpInitRetries > 0) {
|
|
1382
|
+
_info('connected_after_retries', {
|
|
1383
|
+
phase: 'initCall',
|
|
1384
|
+
retries: httpInitRetries
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
wsEndpoint = computeWsEndpoint();
|
|
1388
|
+
const wsRetries = await connectWs();
|
|
1389
|
+
onWsConnected(wsRetries);
|
|
1390
|
+
};
|
|
1391
|
+
establishCall().catch(err => reject(new Error('Error: ' + err)));
|
|
1057
1392
|
});
|
|
1058
1393
|
}
|
|
1059
1394
|
async UserSays(msg) {
|
|
@@ -1063,6 +1398,10 @@ class BotiumConnectorVoip {
|
|
|
1063
1398
|
const hasDtmf = !!(msg && msg.buttons && msg.buttons.length > 0);
|
|
1064
1399
|
const dtmfMatch = msg && msg.messageText && msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
|
|
1065
1400
|
const inputType = hasDtmf || dtmfMatch ? 'dtmf' : hasText && hasVoiceMedia ? 'mixed' : hasText ? 'text' : hasVoiceMedia ? 'media' : 'unknown';
|
|
1401
|
+
// A real user turn is one that sends content (text/media/dtmf). 'unknown' turns send nothing and
|
|
1402
|
+
// surface server-side as skippable me-steps, so they must NOT consume an ordinal slot - this keeps
|
|
1403
|
+
// meTurnIndex aligned with the server's non-skippable me-step indices when placing turn audio.
|
|
1404
|
+
const meTurnIndex = inputType !== 'unknown' ? this._meTurnAudioOrdinal = (this._meTurnAudioOrdinal ?? -1) + 1 : null;
|
|
1066
1405
|
const msgPreview = hasText && msg.messageText ? String(msg.messageText).trim().substring(0, 80) : '';
|
|
1067
1406
|
_info('user_says', {
|
|
1068
1407
|
sessionId: this.sessionId,
|
|
@@ -1071,6 +1410,7 @@ class BotiumConnectorVoip {
|
|
|
1071
1410
|
messageLength: hasText && msg.messageText ? msg.messageText.length : undefined,
|
|
1072
1411
|
mediaSize: hasVoiceMedia && msg.media[0] && Buffer.isBuffer(msg.media[0].buffer) ? msg.media[0].buffer.length : undefined
|
|
1073
1412
|
});
|
|
1413
|
+
this._captureUserSaysStart(msgPreview);
|
|
1074
1414
|
// Avoid logging large buffers/base64 (can break job logs and overwhelm stdout)
|
|
1075
1415
|
try {
|
|
1076
1416
|
const safeLog = {
|
|
@@ -1105,17 +1445,45 @@ class BotiumConnectorVoip {
|
|
|
1105
1445
|
// the coach can place the agent turn on the recording timeline.
|
|
1106
1446
|
// `requestedDurationMs` is the best estimate of on-wire playback
|
|
1107
1447
|
// length (DTMF tones × digits, TTS synth output, parsed media duration).
|
|
1448
|
+
const recordingSecNow = () => this._recordingSecNow();
|
|
1108
1449
|
const stampAgentWire = (wireKind, requestedDurationMs, extras = {}) => {
|
|
1450
|
+
const wireRecordingStartSec = recordingSecNow();
|
|
1109
1451
|
msg.voipAgent = {
|
|
1110
1452
|
wireSentAtMs: Date.now(),
|
|
1111
1453
|
inputType,
|
|
1112
1454
|
wireKind,
|
|
1113
1455
|
requestedDurationMs: Math.max(0, Math.round(requestedDurationMs || 0)),
|
|
1456
|
+
...(wireRecordingStartSec != null ? {
|
|
1457
|
+
wireRecordingStartSec
|
|
1458
|
+
} : {}),
|
|
1459
|
+
// Carry the coach's reply-decision trace (set on the outgoing message before UserSays)
|
|
1460
|
+
// into voipAgent here, while we own the object and before MESSAGE_SENTTOBOT fires — so
|
|
1461
|
+
// the LIVE transcript snapshot (a deep copy taken at that event) already has it, not
|
|
1462
|
+
// just the final result. Lets the UI split "Tester reply decision" live.
|
|
1463
|
+
...(msg && msg.coachTrace && typeof msg.coachTrace === 'object' ? {
|
|
1464
|
+
coachTrace: msg.coachTrace
|
|
1465
|
+
} : {}),
|
|
1114
1466
|
...extras
|
|
1115
1467
|
};
|
|
1468
|
+
this._activeUserSaysVoipAgent = msg.voipAgent;
|
|
1469
|
+
this._captureAgentWire(msg.voipAgent, inputType);
|
|
1470
|
+
};
|
|
1471
|
+
const sendAgentWire = request => {
|
|
1472
|
+
this._sendUserSaysWs(request);
|
|
1473
|
+
this._markReplyTrace({
|
|
1474
|
+
sendAudioAtMs: Date.now()
|
|
1475
|
+
});
|
|
1476
|
+
this._logReplyTrace('wire_sent');
|
|
1477
|
+
// Finalize the wall pipeline NOW, while _replyTrace still holds this turn's data
|
|
1478
|
+
// (userSaysAtMs / ttsStartAtMs / ttsEndAtMs / wireAtMs / sendAudioAtMs are all set by
|
|
1479
|
+
// this point). The deferred turn-audio callback runs only after the agent audio is
|
|
1480
|
+
// "heard" on the recording, which nulls _replyTrace (_logReplyTraceHeard); finalizing
|
|
1481
|
+
// there would drop wallPipeline entirely and collapse the reply-delay breakdown into a
|
|
1482
|
+
// single "Audio send" bucket on the UI.
|
|
1483
|
+
this._finalizeWallPipeline(msg.voipAgent);
|
|
1116
1484
|
};
|
|
1117
1485
|
// Twilio default: 100 ms tone + 100 ms gap per digit. Drives agent-bar width only.
|
|
1118
|
-
const
|
|
1486
|
+
const DTMF_AGENT_BAR_MS_PER_DIGIT = DTMF_MS_PER_DIGIT;
|
|
1119
1487
|
if (msg && msg.buttons && msg.buttons.length > 0) {
|
|
1120
1488
|
const digits = sanitizeDtmfDigits(msg.buttons[0].payload);
|
|
1121
1489
|
if (!digits) {
|
|
@@ -1128,10 +1496,13 @@ class BotiumConnectorVoip {
|
|
|
1128
1496
|
digits,
|
|
1129
1497
|
sessionId: this.sessionId
|
|
1130
1498
|
});
|
|
1131
|
-
stampAgentWire('dtmf', digits.length *
|
|
1132
|
-
digitCount: digits.length
|
|
1499
|
+
stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
|
|
1500
|
+
digitCount: digits.length,
|
|
1501
|
+
dtmfDigits: digits
|
|
1133
1502
|
});
|
|
1134
|
-
|
|
1503
|
+
sendAgentWire(request);
|
|
1504
|
+
// Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
|
|
1505
|
+
duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
|
|
1135
1506
|
} else if (msg && msg.messageText) {
|
|
1136
1507
|
// Check for DTMF tag in messageText: <DTMF>1234</DTMF>
|
|
1137
1508
|
const dtmfMatch = msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
|
|
@@ -1152,13 +1523,14 @@ class BotiumConnectorVoip {
|
|
|
1152
1523
|
digits,
|
|
1153
1524
|
sessionId: this.sessionId
|
|
1154
1525
|
});
|
|
1155
|
-
stampAgentWire('dtmf', digits.length *
|
|
1156
|
-
digitCount: digits.length
|
|
1526
|
+
stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
|
|
1527
|
+
digitCount: digits.length,
|
|
1528
|
+
dtmfDigits: digits
|
|
1157
1529
|
});
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
if (!skipTtsForMixedInput) {
|
|
1530
|
+
sendAgentWire(request);
|
|
1531
|
+
// Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
|
|
1532
|
+
duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
|
|
1533
|
+
} else if (!skipTtsForMixedInput) {
|
|
1162
1534
|
if (!this.axiosTtsParams) {
|
|
1163
1535
|
if (!(msg.media && msg.media.length > 0 && msg.media[0].buffer)) {
|
|
1164
1536
|
return reject(new Error('TTS not configured, only audio input supported'));
|
|
@@ -1170,10 +1542,17 @@ class BotiumConnectorVoip {
|
|
|
1170
1542
|
msg.sourceData = ttsRequest;
|
|
1171
1543
|
let ttsResult = null;
|
|
1172
1544
|
const ttsStartedAt = Date.now();
|
|
1545
|
+
this._markReplyTrace({
|
|
1546
|
+
ttsStartAtMs: ttsStartedAt
|
|
1547
|
+
});
|
|
1173
1548
|
let ttsSynthMs = 0;
|
|
1174
1549
|
try {
|
|
1175
1550
|
ttsResult = await this._getTtsAudio(ttsRequest, msg.messageText);
|
|
1176
1551
|
ttsSynthMs = Date.now() - ttsStartedAt;
|
|
1552
|
+
this._markReplyTrace({
|
|
1553
|
+
ttsEndAtMs: Date.now(),
|
|
1554
|
+
ttsSynthMs
|
|
1555
|
+
});
|
|
1177
1556
|
} catch (err) {
|
|
1178
1557
|
return reject(new Error(`TTS "${msg.messageText}" failed - ${this._getAxiosErrOutput(err)}`));
|
|
1179
1558
|
}
|
|
@@ -1209,7 +1588,7 @@ class BotiumConnectorVoip {
|
|
|
1209
1588
|
ttsSynthMs,
|
|
1210
1589
|
textLength: msg.messageText ? msg.messageText.length : 0
|
|
1211
1590
|
});
|
|
1212
|
-
|
|
1591
|
+
sendAgentWire(request);
|
|
1213
1592
|
} else {
|
|
1214
1593
|
return reject(new Error('TTS failed, response is empty'));
|
|
1215
1594
|
}
|
|
@@ -1234,7 +1613,7 @@ class BotiumConnectorVoip {
|
|
|
1234
1613
|
stampAgentWire('media', 0, {
|
|
1235
1614
|
mediaUri: msg.media[0].mediaUri || null
|
|
1236
1615
|
});
|
|
1237
|
-
|
|
1616
|
+
sendAgentWire(request);
|
|
1238
1617
|
msg.attachments.push({
|
|
1239
1618
|
name: msg.media[0].mediaUri,
|
|
1240
1619
|
mimeType: msg.media[0].mimeType,
|
|
@@ -1256,16 +1635,514 @@ class BotiumConnectorVoip {
|
|
|
1256
1635
|
}
|
|
1257
1636
|
}
|
|
1258
1637
|
const requestedDurationMs = Math.max(0, Math.round((duration || 0) * 1000));
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1638
|
+
|
|
1639
|
+
// Record this turn's slice bounds and resolve immediately — the actual
|
|
1640
|
+
// slice is cut from the complete buffer at session end (see
|
|
1641
|
+
// _flushPendingTurnAudio). This keeps UserSays from waiting for the
|
|
1642
|
+
// recording to catch up with playback (no latency for TTS or DTMF). The
|
|
1643
|
+
// heard-trace telemetry (voip_reply_trace_heard) is logged from the
|
|
1644
|
+
// agentPlaybackStarted handler once playback is audible on the recording.
|
|
1645
|
+
this._recordPendingTurnAudio(msg.voipAgent, requestedDurationMs, meTurnIndex);
|
|
1646
|
+
resolve();
|
|
1263
1647
|
} catch (err) {
|
|
1264
1648
|
reject(err);
|
|
1265
1649
|
}
|
|
1266
1650
|
}, 0);
|
|
1267
1651
|
});
|
|
1268
1652
|
}
|
|
1653
|
+
_recordingSecNow() {
|
|
1654
|
+
const fmt = this.audioStream && this.audioStream.format;
|
|
1655
|
+
const bytesPerSec = fmt ? fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8) : null;
|
|
1656
|
+
if (!bytesPerSec || !this.audioStream || !(this.audioStream.totalBytes > 0)) return null;
|
|
1657
|
+
return this.audioStream.totalBytes / bytesPerSec;
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
/** Expected on-recording playback length of a DTMF turn (worker-reported, else estimated). */
|
|
1661
|
+
_dtmfPlaybackSec(voipAgent, digitCount) {
|
|
1662
|
+
const playbackMs = voipAgent && voipAgent.playbackRequestedDurationMs;
|
|
1663
|
+
if (lodash__default["default"].isFinite(playbackMs) && playbackMs > 0) return playbackMs / 1000;
|
|
1664
|
+
return dtmfPlaybackMs(digitCount) / 1000;
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
/**
|
|
1668
|
+
* Snapshot the bounds needed to slice this turn's audio later, then return.
|
|
1669
|
+
* `_lastBotTurnStartSec` is the bot-speech anchor; it is consumed here so the
|
|
1670
|
+
* next bot message sets a fresh one. The end of the agent's playback is read
|
|
1671
|
+
* from voipAgent.playedRecordingStartSec (filled asynchronously by the
|
|
1672
|
+
* agentPlaybackStarted handler) at flush time. Consecutive agent turns with no
|
|
1673
|
+
* bot message in between (anchor null) chain off the previous turn's end.
|
|
1674
|
+
*/
|
|
1675
|
+
_recordPendingTurnAudio(voipAgent, requestedDurationMs, meTurnIndex) {
|
|
1676
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1677
|
+
const botAnchorSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
|
|
1678
|
+
this._lastBotTurnStartSec = null;
|
|
1679
|
+
this._pendingTurnAudio.push({
|
|
1680
|
+
botAnchorSec,
|
|
1681
|
+
voipAgent: voipAgent || null,
|
|
1682
|
+
requestedDurationMs: Math.max(0, requestedDurationMs || 0),
|
|
1683
|
+
meTurnIndex: Number.isInteger(meTurnIndex) && meTurnIndex >= 0 ? meTurnIndex : null
|
|
1684
|
+
});
|
|
1685
|
+
_info('turn_audio_recorded', {
|
|
1686
|
+
sessionId: this.sessionId,
|
|
1687
|
+
meTurnIndex,
|
|
1688
|
+
botAnchorSec,
|
|
1689
|
+
wireKind: voipAgent && voipAgent.wireKind,
|
|
1690
|
+
pending: this._pendingTurnAudio.length
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
/**
|
|
1695
|
+
* Emit per-turn audio (turn_N.wav) for every pending turn whose playback end the
|
|
1696
|
+
* recording buffer has already reached, slicing from the live buffer and emitting a
|
|
1697
|
+
* MESSAGE_ATTACHMENT carrying meTurnIndex. Turns are processed in order (the chain
|
|
1698
|
+
* start of a follow-up turn with no bot anchor depends on the previous turn's end), so
|
|
1699
|
+
* a not-yet-ready turn stops the pass until more audio streams in. With force=true
|
|
1700
|
+
* (session end) the remainder is emitted even if the buffer is incomplete.
|
|
1701
|
+
*/
|
|
1702
|
+
_emitReadyTurnAudio(reason, force) {
|
|
1703
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1704
|
+
const pending = this._pendingTurnAudio || [];
|
|
1705
|
+
if (!pending.length) return;
|
|
1706
|
+
if (!this.audioStream || !this.audioStream.format) return; // no PCM yet
|
|
1707
|
+
const slackSec = (audioStreamIntervalMs() + 50) / 1000;
|
|
1708
|
+
const recNow = this._recordingSecNow();
|
|
1709
|
+
for (const t of pending) {
|
|
1710
|
+
if (t.emitted) continue;
|
|
1711
|
+
try {
|
|
1712
|
+
const va = t.voipAgent || {};
|
|
1713
|
+
const playbackSec = va.wireKind === 'dtmf' ? this._dtmfPlaybackSec(va, va.digitCount || 1) : lodash__default["default"].isFinite(va.playbackRequestedDurationMs) && va.playbackRequestedDurationMs > 0 ? va.playbackRequestedDurationMs / 1000 : t.requestedDurationMs / 1000;
|
|
1714
|
+
const startSec = lodash__default["default"].isFinite(t.botAnchorSec) ? t.botAnchorSec : this._turnAudioPrevEndSec;
|
|
1715
|
+
// End = where the agent's playback finished on the recording. _sliceTurnAudio
|
|
1716
|
+
// adds the configured padding on top.
|
|
1717
|
+
const anchorSec = lodash__default["default"].isFinite(va.playedRecordingStartSec) ? va.playedRecordingStartSec : lodash__default["default"].isFinite(va.heardRecordingStartSec) ? va.heardRecordingStartSec : lodash__default["default"].isFinite(va.wireRecordingStartSec) ? va.wireRecordingStartSec : null;
|
|
1718
|
+
let endSec = lodash__default["default"].isFinite(anchorSec) ? anchorSec + playbackSec : null;
|
|
1719
|
+
if (!lodash__default["default"].isFinite(endSec) && lodash__default["default"].isFinite(startSec)) endSec = startSec + playbackSec;
|
|
1720
|
+
|
|
1721
|
+
// Progressive: only emit once the recording has reached this turn's end (plus a
|
|
1722
|
+
// stream-flush slack). Stop the pass otherwise — later turns must wait their turn.
|
|
1723
|
+
if (!force) {
|
|
1724
|
+
if (!lodash__default["default"].isFinite(anchorSec) || !lodash__default["default"].isFinite(endSec)) break;
|
|
1725
|
+
if (!lodash__default["default"].isFinite(recNow) || recNow < endSec + slackSec) break;
|
|
1726
|
+
}
|
|
1727
|
+
if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) {
|
|
1728
|
+
debug$3(`${this.sessionId} - turnAudio: skip turn (start=${startSec} end=${endSec} reason=${reason})`);
|
|
1729
|
+
t.emitted = true;
|
|
1730
|
+
if (lodash__default["default"].isFinite(endSec)) this._turnAudioPrevEndSec = endSec;
|
|
1731
|
+
continue;
|
|
1732
|
+
}
|
|
1733
|
+
const audioBase64 = this._sliceTurnAudio(startSec, endSec);
|
|
1734
|
+
t.emitted = true;
|
|
1735
|
+
this._turnAudioPrevEndSec = endSec;
|
|
1736
|
+
if (!audioBase64) continue;
|
|
1737
|
+
this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
|
|
1738
|
+
this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
|
|
1739
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1740
|
+
mimeType: 'audio/wav',
|
|
1741
|
+
base64: audioBase64,
|
|
1742
|
+
meTurnIndex: t.meTurnIndex,
|
|
1743
|
+
// 0-based real-user-turn ordinal (authoritative for placement)
|
|
1744
|
+
sessionContext: {
|
|
1745
|
+
testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
|
|
1746
|
+
testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
|
|
1747
|
+
}
|
|
1748
|
+
});
|
|
1749
|
+
_info('turn_audio_emitted', {
|
|
1750
|
+
sessionId: this.sessionId,
|
|
1751
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1752
|
+
meTurnIndex: t.meTurnIndex,
|
|
1753
|
+
startSec: Number(startSec.toFixed(2)),
|
|
1754
|
+
endSec: Number(endSec.toFixed(2)),
|
|
1755
|
+
reason
|
|
1756
|
+
});
|
|
1757
|
+
} catch (err) {
|
|
1758
|
+
debug$3(`${this.sessionId} - emitReadyTurnAudio: turn slice error: ${err && err.message}`);
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
/**
|
|
1764
|
+
* Force-emit any per-turn audio not yet sent (session end). Idempotent.
|
|
1765
|
+
*/
|
|
1766
|
+
_flushPendingTurnAudio(reason) {
|
|
1767
|
+
if (this._turnAudioForceDone) return;
|
|
1768
|
+
this._turnAudioForceDone = true;
|
|
1769
|
+
const pending = this._pendingTurnAudio || [];
|
|
1770
|
+
_info('turn_audio_flush_enter', {
|
|
1771
|
+
sessionId: this.sessionId,
|
|
1772
|
+
reason,
|
|
1773
|
+
pending: pending.length,
|
|
1774
|
+
emittedAlready: pending.filter(t => t.emitted).length,
|
|
1775
|
+
hasFormat: !!(this.audioStream && this.audioStream.format),
|
|
1776
|
+
totalBytes: this.audioStream && this.audioStream.totalBytes,
|
|
1777
|
+
turnAudioEnable: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]
|
|
1778
|
+
});
|
|
1779
|
+
this._emitReadyTurnAudio(reason, true);
|
|
1780
|
+
this._emitTrailingBotAudio(reason);
|
|
1781
|
+
_info('turn_audio_flush_done', {
|
|
1782
|
+
sessionId: this.sessionId,
|
|
1783
|
+
reason,
|
|
1784
|
+
emitted: this._turnAudioCounter || 0,
|
|
1785
|
+
pending: pending.length
|
|
1786
|
+
});
|
|
1787
|
+
}
|
|
1788
|
+
|
|
1789
|
+
/**
|
|
1790
|
+
* If the call ended on a bot turn (a bot message arrived with no me-response after it),
|
|
1791
|
+
* `_lastBotTurnStartSec` was never consumed by a turn. Slice that trailing bot audio
|
|
1792
|
+
* (bot start → end of recording) and emit it as a turn clip flagged `trailingBot` so the
|
|
1793
|
+
* UI/server place it on the final bot step.
|
|
1794
|
+
*/
|
|
1795
|
+
_emitTrailingBotAudio(reason) {
|
|
1796
|
+
if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
|
|
1797
|
+
if (this._trailingBotAudioEmitted) return;
|
|
1798
|
+
const startSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
|
|
1799
|
+
if (!lodash__default["default"].isFinite(startSec)) return;
|
|
1800
|
+
if (!this.audioStream || !this.audioStream.format) return;
|
|
1801
|
+
const endSec = this._recordingSecNow();
|
|
1802
|
+
if (!lodash__default["default"].isFinite(endSec) || endSec <= startSec) return;
|
|
1803
|
+
let audioBase64 = null;
|
|
1804
|
+
try {
|
|
1805
|
+
audioBase64 = this._sliceTurnAudio(startSec, endSec);
|
|
1806
|
+
} catch (err) {
|
|
1807
|
+
debug$3(`${this.sessionId} - emitTrailingBotAudio: slice error: ${err && err.message}`);
|
|
1808
|
+
return;
|
|
1809
|
+
}
|
|
1810
|
+
if (!audioBase64) return;
|
|
1811
|
+
this._trailingBotAudioEmitted = true;
|
|
1812
|
+
this._lastBotTurnStartSec = null;
|
|
1813
|
+
this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
|
|
1814
|
+
this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
|
|
1815
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1816
|
+
mimeType: 'audio/wav',
|
|
1817
|
+
base64: audioBase64,
|
|
1818
|
+
trailingBot: true,
|
|
1819
|
+
// place on the final bot step (no me-response followed)
|
|
1820
|
+
sessionContext: {
|
|
1821
|
+
testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
|
|
1822
|
+
testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
|
|
1823
|
+
}
|
|
1824
|
+
});
|
|
1825
|
+
_info('turn_audio_trailing_emitted', {
|
|
1826
|
+
sessionId: this.sessionId,
|
|
1827
|
+
name: `turn_${this._turnAudioCounter}.wav`,
|
|
1828
|
+
startSec: Number(startSec.toFixed(2)),
|
|
1829
|
+
endSec: Number(endSec.toFixed(2)),
|
|
1830
|
+
reason
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
_agentSpeechRmsThreshold() {
|
|
1834
|
+
const raw = process.env.VOIP_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1835
|
+
const n = raw != null ? Number(raw) : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1836
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
|
|
1837
|
+
}
|
|
1838
|
+
_agentSpeechSustainedWindows() {
|
|
1839
|
+
const raw = process.env.VOIP_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1840
|
+
const n = raw != null ? parseInt(raw, 10) : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1841
|
+
return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
|
|
1842
|
+
}
|
|
1843
|
+
_audioStreamBytesPerSec() {
|
|
1844
|
+
const fmt = this.audioStream && this.audioStream.format;
|
|
1845
|
+
if (!fmt) return null;
|
|
1846
|
+
return fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8);
|
|
1847
|
+
}
|
|
1848
|
+
_pcmBufferRms(pcm, bitsPerSample) {
|
|
1849
|
+
if (!pcm || pcm.length < 2 || bitsPerSample !== 16) return 0;
|
|
1850
|
+
let sum = 0;
|
|
1851
|
+
let count = 0;
|
|
1852
|
+
for (let i = 0; i + 1 < pcm.length; i += 2) {
|
|
1853
|
+
const sample = pcm.readInt16LE(i);
|
|
1854
|
+
sum += sample * sample;
|
|
1855
|
+
count += 1;
|
|
1856
|
+
}
|
|
1857
|
+
return count > 0 ? Math.sqrt(sum / count) : 0;
|
|
1858
|
+
}
|
|
1859
|
+
_readWavPcmInfo(wavBuffer) {
|
|
1860
|
+
if (!wavBuffer || wavBuffer.length < 44) return null;
|
|
1861
|
+
if (wavBuffer.toString('ascii', 0, 4) !== 'RIFF' || wavBuffer.toString('ascii', 8, 12) !== 'WAVE') {
|
|
1862
|
+
return null;
|
|
1863
|
+
}
|
|
1864
|
+
let offset = 12;
|
|
1865
|
+
let sampleRate = null;
|
|
1866
|
+
let channels = null;
|
|
1867
|
+
let bitsPerSample = null;
|
|
1868
|
+
let dataOffset = null;
|
|
1869
|
+
let dataLength = null;
|
|
1870
|
+
while (offset + 8 <= wavBuffer.length) {
|
|
1871
|
+
const chunkId = wavBuffer.toString('ascii', offset, offset + 4);
|
|
1872
|
+
const chunkSize = wavBuffer.readUInt32LE(offset + 4);
|
|
1873
|
+
const chunkStart = offset + 8;
|
|
1874
|
+
if (chunkId === 'fmt ' && chunkSize >= 16) {
|
|
1875
|
+
channels = wavBuffer.readUInt16LE(chunkStart + 2);
|
|
1876
|
+
sampleRate = wavBuffer.readUInt32LE(chunkStart + 4);
|
|
1877
|
+
bitsPerSample = wavBuffer.readUInt16LE(chunkStart + 14);
|
|
1878
|
+
} else if (chunkId === 'data') {
|
|
1879
|
+
dataOffset = chunkStart;
|
|
1880
|
+
dataLength = chunkSize;
|
|
1881
|
+
break;
|
|
1882
|
+
}
|
|
1883
|
+
offset = chunkStart + chunkSize + chunkSize % 2;
|
|
1884
|
+
}
|
|
1885
|
+
if (!sampleRate || !channels || !bitsPerSample || dataOffset == null) return null;
|
|
1886
|
+
const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
|
|
1887
|
+
if (!bytesPerSec) return null;
|
|
1888
|
+
const pcmLength = dataLength != null ? Math.min(dataLength, wavBuffer.length - dataOffset) : wavBuffer.length - dataOffset;
|
|
1889
|
+
return {
|
|
1890
|
+
pcmOffset: dataOffset,
|
|
1891
|
+
pcmLength,
|
|
1892
|
+
bytesPerSec,
|
|
1893
|
+
bitsPerSample,
|
|
1894
|
+
sampleRate,
|
|
1895
|
+
channels
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
_findAudibleLeadInSecFromPcm(pcm, bytesPerSec, bitsPerSample) {
|
|
1899
|
+
if (!pcm || !bytesPerSec) return null;
|
|
1900
|
+
const threshold = this._agentSpeechRmsThreshold();
|
|
1901
|
+
const sustainedWindows = this._agentSpeechSustainedWindows();
|
|
1902
|
+
const windowBytes = Math.max(2, Math.floor(bytesPerSec * (AGENT_SPEECH_RMS_WINDOW_MS / 1000)));
|
|
1903
|
+
const hopBytes = Math.max(2, Math.floor(windowBytes / 2));
|
|
1904
|
+
let streak = 0;
|
|
1905
|
+
let onsetPos = null;
|
|
1906
|
+
for (let pos = 0; pos + windowBytes <= pcm.length; pos += hopBytes) {
|
|
1907
|
+
const rms = this._pcmBufferRms(pcm.subarray(pos, pos + windowBytes), bitsPerSample);
|
|
1908
|
+
if (rms >= threshold) {
|
|
1909
|
+
if (streak === 0) onsetPos = pos;
|
|
1910
|
+
streak += 1;
|
|
1911
|
+
if (streak >= sustainedWindows) {
|
|
1912
|
+
return onsetPos / bytesPerSec;
|
|
1913
|
+
}
|
|
1914
|
+
} else {
|
|
1915
|
+
streak = 0;
|
|
1916
|
+
onsetPos = null;
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
return null;
|
|
1920
|
+
}
|
|
1921
|
+
_findAudibleLeadInSecFromWavBuffer(wavBuffer) {
|
|
1922
|
+
const info = this._readWavPcmInfo(wavBuffer);
|
|
1923
|
+
if (!info) return null;
|
|
1924
|
+
const pcm = wavBuffer.subarray(info.pcmOffset, info.pcmOffset + info.pcmLength);
|
|
1925
|
+
return this._findAudibleLeadInSecFromPcm(pcm, info.bytesPerSec, info.bitsPerSample);
|
|
1926
|
+
}
|
|
1927
|
+
_findAudibleRecordingStartSecOnStream(playedSec, wireSec) {
|
|
1928
|
+
if (!lodash__default["default"].isFinite(playedSec)) return null;
|
|
1929
|
+
const bytesPerSec = this._audioStreamBytesPerSec();
|
|
1930
|
+
const stream = this.audioStream;
|
|
1931
|
+
if (!bytesPerSec || !stream || !stream.pcmParts.length) return null;
|
|
1932
|
+
const startByte = Math.max(0, Math.floor(playedSec * bytesPerSec));
|
|
1933
|
+
const pcm = Buffer.concat(stream.pcmParts);
|
|
1934
|
+
if (startByte >= pcm.length) return null;
|
|
1935
|
+
const bitsPerSample = stream.format.bitsPerSample;
|
|
1936
|
+
const leadInSec = this._findAudibleLeadInSecFromPcm(pcm.subarray(startByte), bytesPerSec, bitsPerSample);
|
|
1937
|
+
if (!lodash__default["default"].isFinite(leadInSec)) return null;
|
|
1938
|
+
let heardSec = playedSec + leadInSec;
|
|
1939
|
+
if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
|
|
1940
|
+
return heardSec;
|
|
1941
|
+
}
|
|
1942
|
+
_findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments) {
|
|
1943
|
+
if (!lodash__default["default"].isFinite(playedSec) || !lodash__default["default"].isArray(attachments)) return null;
|
|
1944
|
+
const tts = attachments.find(a => a && a.name === 'tts.wav' && a.base64);
|
|
1945
|
+
if (!tts) return null;
|
|
1946
|
+
try {
|
|
1947
|
+
const wavBuffer = Buffer.from(tts.base64, 'base64');
|
|
1948
|
+
const leadInSec = this._findAudibleLeadInSecFromWavBuffer(wavBuffer);
|
|
1949
|
+
if (!lodash__default["default"].isFinite(leadInSec)) return null;
|
|
1950
|
+
let heardSec = playedSec + leadInSec;
|
|
1951
|
+
if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
|
|
1952
|
+
return heardSec;
|
|
1953
|
+
} catch (err) {
|
|
1954
|
+
debug$3(`${this.sessionId} - TTS lead-in scan failed: ${err && err.message}`);
|
|
1955
|
+
return null;
|
|
1956
|
+
}
|
|
1957
|
+
}
|
|
1958
|
+
_resolveAgentHeardRecordingStartSec(voipAgent, attachments) {
|
|
1959
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
|
|
1960
|
+
const playedSec = voipAgent.playedRecordingStartSec;
|
|
1961
|
+
const wireSec = voipAgent.wireRecordingStartSec;
|
|
1962
|
+
const fromTts = this._findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments);
|
|
1963
|
+
const fromStream = this._findAudibleRecordingStartSecOnStream(playedSec, wireSec);
|
|
1964
|
+
const candidates = [fromTts, fromStream].filter(s => lodash__default["default"].isFinite(s));
|
|
1965
|
+
if (!candidates.length) return null;
|
|
1966
|
+
// Prefer the later onset — mixed recording can spike before clear TTS speech.
|
|
1967
|
+
return Math.max(...candidates);
|
|
1968
|
+
}
|
|
1969
|
+
_applyAgentHeardRecordingStartSec(voipAgent, attachments) {
|
|
1970
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
|
|
1971
|
+
const heardSec = this._resolveAgentHeardRecordingStartSec(voipAgent, attachments);
|
|
1972
|
+
if (!lodash__default["default"].isFinite(heardSec) || heardSec <= voipAgent.playedRecordingStartSec) {
|
|
1973
|
+
return lodash__default["default"].isFinite(voipAgent.heardRecordingStartSec) ? voipAgent.heardRecordingStartSec : null;
|
|
1974
|
+
}
|
|
1975
|
+
const prev = voipAgent.heardRecordingStartSec;
|
|
1976
|
+
if (lodash__default["default"].isFinite(prev) && prev >= heardSec) return prev;
|
|
1977
|
+
voipAgent.heardRecordingStartSec = heardSec;
|
|
1978
|
+
this._markReplyTrace({
|
|
1979
|
+
heardRecordingStartSec: heardSec
|
|
1980
|
+
});
|
|
1981
|
+
return heardSec;
|
|
1982
|
+
}
|
|
1983
|
+
_maybeDetectAgentAudibleOnRecording(voipAgent) {
|
|
1984
|
+
if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return;
|
|
1985
|
+
this._applyAgentHeardRecordingStartSec(voipAgent);
|
|
1986
|
+
}
|
|
1987
|
+
_markReplyTrace(patch) {
|
|
1988
|
+
if (!this._replyTrace || !patch) return;
|
|
1989
|
+
Object.assign(this._replyTrace, patch);
|
|
1990
|
+
}
|
|
1991
|
+
_captureSttFinalForReplyTrace(parsedData, msgPreview) {
|
|
1992
|
+
const data = parsedData && parsedData.data;
|
|
1993
|
+
const atMs = parsedData._receivedAtMs || Date.now();
|
|
1994
|
+
const recordingAtSttFinalSec = this._recordingSecNow();
|
|
1995
|
+
if (parsedData && lodash__default["default"].isFinite(recordingAtSttFinalSec)) {
|
|
1996
|
+
parsedData.recordingAtSttFinalSec = recordingAtSttFinalSec;
|
|
1997
|
+
}
|
|
1998
|
+
// Dedicated, self-documenting anchor for the downstream "STT transport" sub-phase
|
|
1999
|
+
// (receivedAtMs - finalEmittedWallMs). Unlike the generic _receivedAtMs (stamped on
|
|
2000
|
+
// every WS frame), this is set only on the accepted STT-final.
|
|
2001
|
+
if (parsedData && lodash__default["default"].isFinite(atMs)) {
|
|
2002
|
+
parsedData.sttFinalReceivedAtMs = atMs;
|
|
2003
|
+
}
|
|
2004
|
+
this._replyTrace = {
|
|
2005
|
+
sessionId: this.sessionId,
|
|
2006
|
+
botMessagePreview: msgPreview || undefined,
|
|
2007
|
+
sttFinalAtMs: atMs,
|
|
2008
|
+
sttRecordingStartSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'start')) ? data.start : null,
|
|
2009
|
+
sttRecordingEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'end')) ? data.end : null,
|
|
2010
|
+
sttSpeechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'speechEndSec')) ? data.speechEndSec : null,
|
|
2011
|
+
recordingAtSttFinalSec: lodash__default["default"].isFinite(recordingAtSttFinalSec) ? recordingAtSttFinalSec : null,
|
|
2012
|
+
queueAtMs: null,
|
|
2013
|
+
recordingAtQueueSec: null,
|
|
2014
|
+
psstTimerArmedAtMs: null,
|
|
2015
|
+
psstScheduledMs: null,
|
|
2016
|
+
psstTimerFiredAtMs: null,
|
|
2017
|
+
psstFireDelayMs: null,
|
|
2018
|
+
userSaysAtMs: null,
|
|
2019
|
+
coachWaitMs: null,
|
|
2020
|
+
ttsStartAtMs: null,
|
|
2021
|
+
ttsEndAtMs: null,
|
|
2022
|
+
ttsSynthMs: null,
|
|
2023
|
+
wireAtMs: null,
|
|
2024
|
+
wireRecordingStartSec: null,
|
|
2025
|
+
sendAudioAtMs: null,
|
|
2026
|
+
playedRecordingStartSec: null,
|
|
2027
|
+
playbackAtMs: null,
|
|
2028
|
+
heardRecordingStartSec: null,
|
|
2029
|
+
agentEndRecordingSec: null,
|
|
2030
|
+
wireKind: null,
|
|
2031
|
+
inputType: null,
|
|
2032
|
+
requestedDurationMs: null,
|
|
2033
|
+
meMessagePreview: null
|
|
2034
|
+
};
|
|
2035
|
+
}
|
|
2036
|
+
_captureBotQueuedForReplyTrace(queuedAt) {
|
|
2037
|
+
if (!this._replyTrace) return;
|
|
2038
|
+
this._replyTrace.queueAtMs = queuedAt;
|
|
2039
|
+
this._replyTrace.recordingAtQueueSec = this._recordingSecNow();
|
|
2040
|
+
}
|
|
2041
|
+
_captureUserSaysStart(msgPreview) {
|
|
2042
|
+
if (!this._replyTrace) return;
|
|
2043
|
+
const now = Date.now();
|
|
2044
|
+
this._replyTrace.userSaysAtMs = now;
|
|
2045
|
+
this._replyTrace.meMessagePreview = msgPreview || undefined;
|
|
2046
|
+
const queueAt = this._replyTrace.queueAtMs || this._lastBotSaysQueuedAt;
|
|
2047
|
+
if (lodash__default["default"].isFinite(queueAt)) {
|
|
2048
|
+
if (!this._replyTrace.queueAtMs) this._replyTrace.queueAtMs = queueAt;
|
|
2049
|
+
this._replyTrace.coachWaitMs = now - queueAt;
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
_captureAgentWire(voipAgent, inputType) {
|
|
2053
|
+
if (!this._replyTrace || !voipAgent) return;
|
|
2054
|
+
this._replyTrace.wireAtMs = voipAgent.wireSentAtMs;
|
|
2055
|
+
this._replyTrace.wireRecordingStartSec = voipAgent.wireRecordingStartSec;
|
|
2056
|
+
this._replyTrace.wireKind = voipAgent.wireKind;
|
|
2057
|
+
this._replyTrace.inputType = inputType;
|
|
2058
|
+
this._replyTrace.requestedDurationMs = voipAgent.requestedDurationMs;
|
|
2059
|
+
if (lodash__default["default"].isFinite(voipAgent.ttsSynthMs)) this._replyTrace.ttsSynthMs = voipAgent.ttsSynthMs;
|
|
2060
|
+
}
|
|
2061
|
+
_finalizeWallPipeline(voipAgent) {
|
|
2062
|
+
const t = this._replyTrace;
|
|
2063
|
+
if (!voipAgent || !t) return;
|
|
2064
|
+
voipAgent.wallPipeline = {
|
|
2065
|
+
psstScheduledMs: lodash__default["default"].isFinite(t.psstScheduledMs) ? t.psstScheduledMs : null,
|
|
2066
|
+
psstFireDelayMs: lodash__default["default"].isFinite(t.psstFireDelayMs) ? t.psstFireDelayMs : null,
|
|
2067
|
+
coachWaitMs: lodash__default["default"].isFinite(t.coachWaitMs) ? t.coachWaitMs : null,
|
|
2068
|
+
userSaysAtMs: lodash__default["default"].isFinite(t.userSaysAtMs) ? t.userSaysAtMs : null,
|
|
2069
|
+
ttsStartAtMs: lodash__default["default"].isFinite(t.ttsStartAtMs) ? t.ttsStartAtMs : null,
|
|
2070
|
+
ttsEndAtMs: lodash__default["default"].isFinite(t.ttsEndAtMs) ? t.ttsEndAtMs : null,
|
|
2071
|
+
ttsSynthMs: lodash__default["default"].isFinite(t.ttsSynthMs) ? t.ttsSynthMs : null,
|
|
2072
|
+
wireAtMs: lodash__default["default"].isFinite(t.wireAtMs) ? t.wireAtMs : null,
|
|
2073
|
+
sendAudioAtMs: lodash__default["default"].isFinite(t.sendAudioAtMs) ? t.sendAudioAtMs : null
|
|
2074
|
+
};
|
|
2075
|
+
}
|
|
2076
|
+
_replyTraceMsFromSttFinal(atMs) {
|
|
2077
|
+
const anchor = this._replyTrace && this._replyTrace.sttFinalAtMs;
|
|
2078
|
+
if (!lodash__default["default"].isFinite(anchor) || !lodash__default["default"].isFinite(atMs)) return null;
|
|
2079
|
+
return Math.round(atMs - anchor);
|
|
2080
|
+
}
|
|
2081
|
+
_replyTraceRecMs(fromSec, toSec) {
|
|
2082
|
+
if (!lodash__default["default"].isFinite(fromSec) || !lodash__default["default"].isFinite(toSec)) return null;
|
|
2083
|
+
return Math.round((toSec - fromSec) * 1000);
|
|
2084
|
+
}
|
|
2085
|
+
_logReplyTrace(trigger) {
|
|
2086
|
+
const t = this._replyTrace;
|
|
2087
|
+
if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
|
|
2088
|
+
_info('voip_reply_trace', {
|
|
2089
|
+
sessionId: t.sessionId,
|
|
2090
|
+
trigger,
|
|
2091
|
+
botPreview: t.botMessagePreview,
|
|
2092
|
+
mePreview: t.meMessagePreview,
|
|
2093
|
+
sttRecordingStartSec: t.sttRecordingStartSec,
|
|
2094
|
+
sttRecordingEndSec: t.sttRecordingEndSec,
|
|
2095
|
+
sttSpeechEndSec: t.sttSpeechEndSec,
|
|
2096
|
+
recordingAtSttFinalSec: t.recordingAtSttFinalSec,
|
|
2097
|
+
recordingAtQueueSec: t.recordingAtQueueSec,
|
|
2098
|
+
wireRecordingStartSec: t.wireRecordingStartSec,
|
|
2099
|
+
wireKind: t.wireKind,
|
|
2100
|
+
inputType: t.inputType,
|
|
2101
|
+
requestedDurationMs: t.requestedDurationMs,
|
|
2102
|
+
ttsSynthMs: t.ttsSynthMs,
|
|
2103
|
+
coachWaitMs: t.coachWaitMs,
|
|
2104
|
+
psstScheduledMs: t.psstScheduledMs,
|
|
2105
|
+
psstFireDelayMs: t.psstFireDelayMs,
|
|
2106
|
+
ms_sttFinal_to_queue: this._replyTraceMsFromSttFinal(t.queueAtMs),
|
|
2107
|
+
ms_sttFinal_to_psstFire: this._replyTraceMsFromSttFinal(t.psstTimerFiredAtMs),
|
|
2108
|
+
ms_sttFinal_to_userSays: this._replyTraceMsFromSttFinal(t.userSaysAtMs),
|
|
2109
|
+
ms_sttFinal_to_ttsStart: this._replyTraceMsFromSttFinal(t.ttsStartAtMs),
|
|
2110
|
+
ms_sttFinal_to_ttsEnd: this._replyTraceMsFromSttFinal(t.ttsEndAtMs),
|
|
2111
|
+
ms_sttFinal_to_wire: this._replyTraceMsFromSttFinal(t.wireAtMs),
|
|
2112
|
+
ms_sttFinal_to_sendAudio: this._replyTraceMsFromSttFinal(t.sendAudioAtMs),
|
|
2113
|
+
ms_userSays_to_ttsStart: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.ttsStartAtMs) ? Math.round(t.ttsStartAtMs - t.userSaysAtMs) : null,
|
|
2114
|
+
ms_userSays_to_wire: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.wireAtMs) ? Math.round(t.wireAtMs - t.userSaysAtMs) : null,
|
|
2115
|
+
ms_queue_to_userSays: t.coachWaitMs,
|
|
2116
|
+
recMs_sttEnd_to_queue: this._replyTraceRecMs(t.sttRecordingEndSec, t.recordingAtQueueSec),
|
|
2117
|
+
recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
|
|
2118
|
+
recMs_speechEnd_to_wire: this._replyTraceRecMs(t.sttSpeechEndSec, t.wireRecordingStartSec)
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
_logReplyTraceHeard(agentEndRecordingSec) {
|
|
2122
|
+
const t = this._replyTrace;
|
|
2123
|
+
if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
|
|
2124
|
+
const heardSec = t.heardRecordingStartSec;
|
|
2125
|
+
const playedSec = t.playedRecordingStartSec;
|
|
2126
|
+
if (lodash__default["default"].isFinite(agentEndRecordingSec)) {
|
|
2127
|
+
t.agentEndRecordingSec = agentEndRecordingSec;
|
|
2128
|
+
}
|
|
2129
|
+
_info('voip_reply_trace_heard', {
|
|
2130
|
+
sessionId: t.sessionId,
|
|
2131
|
+
playedRecordingStartSec: playedSec,
|
|
2132
|
+
heardRecordingStartSec: heardSec,
|
|
2133
|
+
agentEndRecordingSec: t.agentEndRecordingSec,
|
|
2134
|
+
wireRecordingStartSec: t.wireRecordingStartSec,
|
|
2135
|
+
sttRecordingEndSec: t.sttRecordingEndSec,
|
|
2136
|
+
sttSpeechEndSec: t.sttSpeechEndSec,
|
|
2137
|
+
recMs_sttEnd_to_played: this._replyTraceRecMs(t.sttRecordingEndSec, playedSec),
|
|
2138
|
+
recMs_speechEnd_to_played: this._replyTraceRecMs(t.sttSpeechEndSec, playedSec),
|
|
2139
|
+
recMs_sttEnd_to_heard: this._replyTraceRecMs(t.sttRecordingEndSec, heardSec),
|
|
2140
|
+
recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
|
|
2141
|
+
recMs_wire_to_played: this._replyTraceRecMs(t.wireRecordingStartSec, playedSec),
|
|
2142
|
+
recMs_wire_to_heard: this._replyTraceRecMs(t.wireRecordingStartSec, heardSec)
|
|
2143
|
+
});
|
|
2144
|
+
this._replyTrace = null;
|
|
2145
|
+
}
|
|
1269
2146
|
_voipWsCanSend() {
|
|
1270
2147
|
return !this.stopCalled && this.ws && this.ws.readyState === ws__default["default"].OPEN;
|
|
1271
2148
|
}
|
|
@@ -1318,6 +2195,8 @@ class BotiumConnectorVoip {
|
|
|
1318
2195
|
if (typeof this._emitBufferedFullRecordIfAny === 'function') {
|
|
1319
2196
|
this._emitBufferedFullRecordIfAny('stop_final_guard');
|
|
1320
2197
|
}
|
|
2198
|
+
// Last-resort flush in case neither audioStreamEnd nor fullRecordEnd fired.
|
|
2199
|
+
this._flushPendingTurnAudio('stop_final_guard');
|
|
1321
2200
|
}
|
|
1322
2201
|
this._emitBufferedFullRecordIfAny = null;
|
|
1323
2202
|
}
|
|
@@ -1627,6 +2506,21 @@ class BotiumConnectorVoip {
|
|
|
1627
2506
|
if (!lodash__default["default"].isFinite(parsed) || parsed <= 0) return null;
|
|
1628
2507
|
return isPsst ? Math.max(0, parsed - 500) : parsed;
|
|
1629
2508
|
}
|
|
2509
|
+
|
|
2510
|
+
// Extra grace (ms) added to the JOIN/PSST flush window when the last buffered final was cut
|
|
2511
|
+
// mid-utterance. Returns 0 when unset/invalid (feature off). See _isLastFinalNaturalEnd.
|
|
2512
|
+
_getPsstLatencyGraceMs() {
|
|
2513
|
+
const parsed = parseInt(this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS], 10);
|
|
2514
|
+
if (!lodash__default["default"].isFinite(parsed) || parsed <= 0) return 0;
|
|
2515
|
+
return parsed;
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
// A final that carries a finite silenceStartedSec ended on Azure-detected end-of-speech silence
|
|
2519
|
+
// (bot genuinely stopped) => flush at the base window, no grace. A final without it was cut
|
|
2520
|
+
// mid-utterance (forced/segmented) => a continuation is plausible => apply the latency grace.
|
|
2521
|
+
_isLastFinalNaturalEnd() {
|
|
2522
|
+
return lodash__default["default"].isFinite(lodash__default["default"].get(this.prevData, 'data.silenceStartedSec'));
|
|
2523
|
+
}
|
|
1630
2524
|
_getEffectiveJoinTimeoutMs(convoStep, botMsgs) {
|
|
1631
2525
|
const sttHandling = this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING];
|
|
1632
2526
|
const isPsst = sttHandling === 'PSST';
|
|
@@ -1662,6 +2556,100 @@ class BotiumConnectorVoip {
|
|
|
1662
2556
|
}
|
|
1663
2557
|
return null;
|
|
1664
2558
|
}
|
|
2559
|
+
|
|
2560
|
+
/**
|
|
2561
|
+
* Build a well-formed WAV Buffer from raw PCM bytes and a format descriptor.
|
|
2562
|
+
* @param {Buffer} pcm raw PCM bytes (no header)
|
|
2563
|
+
* @param {{ sampleRate: number, channels: number, bitsPerSample: number }} fmt
|
|
2564
|
+
* @returns {Buffer}
|
|
2565
|
+
*/
|
|
2566
|
+
_buildWavBuffer(pcm, fmt) {
|
|
2567
|
+
const {
|
|
2568
|
+
sampleRate,
|
|
2569
|
+
channels,
|
|
2570
|
+
bitsPerSample
|
|
2571
|
+
} = fmt;
|
|
2572
|
+
const byteRate = sampleRate * channels * (bitsPerSample / 8);
|
|
2573
|
+
const blockAlign = channels * (bitsPerSample / 8);
|
|
2574
|
+
const dataSize = pcm.length;
|
|
2575
|
+
const header = Buffer.alloc(44);
|
|
2576
|
+
header.write('RIFF', 0);
|
|
2577
|
+
header.writeUInt32LE(36 + dataSize, 4);
|
|
2578
|
+
header.write('WAVE', 8);
|
|
2579
|
+
header.write('fmt ', 12);
|
|
2580
|
+
header.writeUInt32LE(16, 16); // fmt chunk size
|
|
2581
|
+
header.writeUInt16LE(1, 20); // PCM format
|
|
2582
|
+
header.writeUInt16LE(channels, 22);
|
|
2583
|
+
header.writeUInt32LE(sampleRate, 24);
|
|
2584
|
+
header.writeUInt32LE(byteRate, 28);
|
|
2585
|
+
header.writeUInt16LE(blockAlign, 32);
|
|
2586
|
+
header.writeUInt16LE(bitsPerSample, 34);
|
|
2587
|
+
header.write('data', 36);
|
|
2588
|
+
header.writeUInt32LE(dataSize, 40);
|
|
2589
|
+
return Buffer.concat([header, pcm]);
|
|
2590
|
+
}
|
|
2591
|
+
|
|
2592
|
+
/**
|
|
2593
|
+
* Slice a segment of the continuously buffered PCM audio stream and return
|
|
2594
|
+
* it as a base64-encoded WAV string.
|
|
2595
|
+
*
|
|
2596
|
+
* @param {number} startSec start of the segment (seconds from call connect)
|
|
2597
|
+
* @param {number} endSec end of the segment (seconds from call connect)
|
|
2598
|
+
* @returns {string|null} base64 WAV or null if the stream is not ready
|
|
2599
|
+
*/
|
|
2600
|
+
_sliceTurnAudio(startSec, endSec) {
|
|
2601
|
+
const stream = this.audioStream;
|
|
2602
|
+
if (!stream || !stream.format || !stream.pcmParts || !stream.pcmParts.length) return null;
|
|
2603
|
+
if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) return null;
|
|
2604
|
+
const {
|
|
2605
|
+
sampleRate,
|
|
2606
|
+
channels,
|
|
2607
|
+
bitsPerSample
|
|
2608
|
+
} = stream.format;
|
|
2609
|
+
const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
|
|
2610
|
+
const frameBytes = channels * (bitsPerSample / 8);
|
|
2611
|
+
const offsetSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_OFFSET_MS] || 0) / 1000;
|
|
2612
|
+
const paddingSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_PADDING_MS] || 0) / 1000;
|
|
2613
|
+
const adjStart = Math.max(0, startSec + offsetSec);
|
|
2614
|
+
const adjEnd = endSec + paddingSec;
|
|
2615
|
+
|
|
2616
|
+
// Frame-align the byte boundaries.
|
|
2617
|
+
const startByte = Math.floor(adjStart * bytesPerSec / frameBytes) * frameBytes;
|
|
2618
|
+
const endByte = Math.ceil(adjEnd * bytesPerSec / frameBytes) * frameBytes;
|
|
2619
|
+
if (startByte >= stream.totalBytes) {
|
|
2620
|
+
debug$3(`${this.sessionId} - _sliceTurnAudio: startByte ${startByte} >= totalBytes ${stream.totalBytes}, skipping`);
|
|
2621
|
+
return null;
|
|
2622
|
+
}
|
|
2623
|
+
const clampedEnd = Math.min(endByte, stream.totalBytes);
|
|
2624
|
+
const sliceLen = clampedEnd - startByte;
|
|
2625
|
+
if (sliceLen <= 0) return null;
|
|
2626
|
+
|
|
2627
|
+
// Materialise only the bytes we need from the part list.
|
|
2628
|
+
const pcm = Buffer.allocUnsafe(sliceLen);
|
|
2629
|
+
let written = 0;
|
|
2630
|
+
let offset = 0;
|
|
2631
|
+
for (const part of stream.pcmParts) {
|
|
2632
|
+
const partEnd = offset + part.length;
|
|
2633
|
+
if (partEnd <= startByte) {
|
|
2634
|
+
offset += part.length;
|
|
2635
|
+
continue;
|
|
2636
|
+
}
|
|
2637
|
+
if (offset >= clampedEnd) break;
|
|
2638
|
+
const copyFrom = Math.max(0, startByte - offset);
|
|
2639
|
+
const copyTo = Math.min(part.length, clampedEnd - offset);
|
|
2640
|
+
part.copy(pcm, written, copyFrom, copyTo);
|
|
2641
|
+
written += copyTo - copyFrom;
|
|
2642
|
+
offset += part.length;
|
|
2643
|
+
}
|
|
2644
|
+
if (written === 0) return null;
|
|
2645
|
+
const slicedPcm = written < sliceLen ? pcm.slice(0, written) : pcm;
|
|
2646
|
+
const wavBuf = this._buildWavBuffer(slicedPcm, {
|
|
2647
|
+
sampleRate,
|
|
2648
|
+
channels,
|
|
2649
|
+
bitsPerSample
|
|
2650
|
+
});
|
|
2651
|
+
return wavBuf.toString('base64');
|
|
2652
|
+
}
|
|
1665
2653
|
}
|
|
1666
2654
|
var connector = BotiumConnectorVoip;
|
|
1667
2655
|
|
|
@@ -1745,6 +2733,24 @@ var botiumConnectorVoip = {
|
|
|
1745
2733
|
type: 'boolean',
|
|
1746
2734
|
required: false,
|
|
1747
2735
|
advanced: true
|
|
2736
|
+
}, {
|
|
2737
|
+
name: 'VOIP_TURN_AUDIO_ENABLE',
|
|
2738
|
+
label: 'Attach per-turn audio to each transcript message',
|
|
2739
|
+
type: 'boolean',
|
|
2740
|
+
required: false,
|
|
2741
|
+
advanced: true
|
|
2742
|
+
}, {
|
|
2743
|
+
name: 'VOIP_TURN_AUDIO_PADDING_MS',
|
|
2744
|
+
label: 'Extra milliseconds appended after each turn audio slice (absorbs STT boundary jitter)',
|
|
2745
|
+
type: 'int',
|
|
2746
|
+
required: false,
|
|
2747
|
+
advanced: true
|
|
2748
|
+
}, {
|
|
2749
|
+
name: 'VOIP_TURN_AUDIO_OFFSET_MS',
|
|
2750
|
+
label: 'Millisecond offset applied to every turn audio start time (positive = shift right)',
|
|
2751
|
+
type: 'int',
|
|
2752
|
+
required: false,
|
|
2753
|
+
advanced: true
|
|
1748
2754
|
}]
|
|
1749
2755
|
},
|
|
1750
2756
|
PluginLogicHooks: {
|