botium-connector-voip 0.0.29 → 0.0.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,6 +26,12 @@ const debug$3 = debug__default["default"]('botium-connector-voip');
26
26
  // debug = high-frequency diagnostics (DEBUG=botium-connector-voip). warn = degraded but continuing.
27
27
  // error = abort/failure. No secrets in info; STT text only as length or truncated in info.
28
28
 
29
+ /** WS frame types logged only at start/end handlers — not per-chunk (hundreds per call). */
30
+ const WS_DEBUG_SILENT_TYPES = new Set(['audioStreamChunk', 'fullRecordChunk']);
31
+ const AGENT_SPEECH_RMS_WINDOW_MS = 100;
32
+ const DEFAULT_AGENT_SPEECH_RMS_THRESHOLD = 500;
33
+ const DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS = 2;
34
+ const WS_DEBUG_BASE64_FIELD_NAMES = new Set(['chunk', 'buffer', 'base64', 'fullRecord', 'full_record', 'audio', 'audioData', 'b64_buffer']);
29
35
  const _info = (event, data) => {
30
36
  const parts = Object.entries({
31
37
  event,
@@ -39,17 +45,40 @@ const sanitizeDtmfDigits = raw => {
39
45
  if (raw == null) return '';
40
46
  return String(raw).replace(/[^0-9*#ABCDabcd]/g, '').toUpperCase();
41
47
  };
48
+
49
+ // Matches voipcall.generate_dtmf_sequence defaults (100 ms tone, 50 ms pause between digits).
50
+ const DTMF_TONE_MS = 100;
51
+ const DTMF_PAUSE_MS = 50;
52
+ const DTMF_MS_PER_DIGIT = 200;
53
+ const DEFAULT_AUDIO_STREAM_INTERVAL_MS = 250;
54
+ const audioStreamIntervalMs = () => {
55
+ const n = Number(process.env.VOIP_AUDIO_STREAM_INTERVAL_MS || process.env.AUDIO_STREAM_INTERVAL_MS);
56
+ return Number.isFinite(n) && n > 0 ? n : DEFAULT_AUDIO_STREAM_INTERVAL_MS;
57
+ };
58
+ const dtmfPlaybackMs = digitCount => {
59
+ if (!digitCount || digitCount <= 0) return 0;
60
+ return digitCount * DTMF_TONE_MS + Math.max(0, digitCount - 1) * DTMF_PAUSE_MS;
61
+ };
62
+
63
+ /** Wait before turn-audio slice so DTMF PCM is flushed through audioStream (250 ms default). */
64
+ const dtmfTurnAudioWaitMs = digitCount => {
65
+ const playbackMs = dtmfPlaybackMs(digitCount);
66
+ const streamMs = audioStreamIntervalMs();
67
+ return Math.max(digitCount * DTMF_MS_PER_DIGIT, playbackMs + streamMs + 50);
68
+ };
42
69
  const Capabilities = {
43
70
  VOIP_STT_URL_STREAM: 'VOIP_STT_URL_STREAM',
44
71
  VOIP_STT_PARAMS_STREAM: 'VOIP_STT_PARAMS_STREAM',
45
72
  VOIP_STT_METHOD_STREAM: 'VOIP_STT_METHOD_STREAM',
46
73
  VOIP_STT_BODY_STREAM: 'VOIP_STT_BODY_STREAM',
47
74
  VOIP_STT_BODY: 'VOIP_STT_BODY',
75
+ VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 'VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS',
48
76
  VOIP_STT_HEADERS: 'VOIP_STT_HEADERS',
49
77
  VOIP_STT_TIMEOUT: 'VOIP_STT_TIMEOUT',
50
78
  VOIP_STT_MESSAGE_HANDLING: 'VOIP_STT_MESSAGE_HANDLING',
51
79
  VOIP_STT_MESSAGE_HANDLING_TIMEOUT: 'VOIP_STT_MESSAGE_HANDLING_TIMEOUT',
52
80
  VOIP_STT_MESSAGE_HANDLING_TIMEOUT_SUBSEQUENT: 'VOIP_STT_MESSAGE_HANDLING_TIMEOUT_SUBSEQUENT',
81
+ VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS: 'VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS',
53
82
  VOIP_JOIN_SILENCE_DURATION_BY_SUBSTRING: 'VOIP_JOIN_SILENCE_DURATION_BY_SUBSTRING',
54
83
  VOIP_STT_DICTIONARY_REPLACEMENTS: 'VOIP_STT_DICTIONARY_REPLACEMENTS',
55
84
  VOIP_STT_MESSAGE_HANDLING_DELIMITER: 'VOIP_STT_MESSAGE_HANDLING_DELIMITER',
@@ -84,6 +113,8 @@ const Capabilities = {
84
113
  VOIP_ICE_TURN_PROTOCOL: 'VOIP_ICE_TURN_PROTOCOL',
85
114
  VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 'VOIP_WEBSOCKET_CONNECT_MAXRETRIES',
86
115
  VOIP_WEBSOCKET_CONNECT_TIMEOUT: 'VOIP_WEBSOCKET_CONNECT_TIMEOUT',
116
+ VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 'VOIP_CALL_SETUP_RETRY_487_MAXRETRIES',
117
+ VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 'VOIP_CALL_SETUP_RETRY_487_TIMEOUT',
87
118
  VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_ENABLE',
88
119
  VOIP_SILENCE_DURATION_TIMEOUT: 'VOIP_SILENCE_DURATION_TIMEOUT',
89
120
  VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: 'VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE',
@@ -92,7 +123,10 @@ const Capabilities = {
92
123
  VOIP_USE_GLOBAL_VOIP_WORKER: 'VOIP_USE_GLOBAL_VOIP_WORKER',
93
124
  VOIP_USER_INPUT_PREFER_VOICE: 'VOIP_USER_INPUT_PREFER_VOICE',
94
125
  VOIP_EMIT_SPECULATIVE_TEXT: 'VOIP_EMIT_SPECULATIVE_TEXT',
95
- VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE'
126
+ VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: 'VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE',
127
+ VOIP_TURN_AUDIO_ENABLE: 'VOIP_TURN_AUDIO_ENABLE',
128
+ VOIP_TURN_AUDIO_PADDING_MS: 'VOIP_TURN_AUDIO_PADDING_MS',
129
+ VOIP_TURN_AUDIO_OFFSET_MS: 'VOIP_TURN_AUDIO_OFFSET_MS'
96
130
  };
97
131
  const Defaults = {
98
132
  VOIP_STT_METHOD: 'POST',
@@ -104,19 +138,62 @@ const Defaults = {
104
138
  VOIP_TTS_PREFETCH_ENABLE: true,
105
139
  VOIP_STT_MESSAGE_HANDLING: 'ORIGINAL',
106
140
  VOIP_STT_MESSAGE_HANDLING_TIMEOUT: 2500,
141
+ // Extra wall-clock grace added to the JOIN/PSST flush window when the last buffered final was
142
+ // cut mid-utterance (no Azure-detected end-of-speech silence). Absorbs STT delivery latency so a
143
+ // paused-then-resumed prompt is joined instead of split. Set to 0 to disable. See _getPsstLatencyGraceMs.
144
+ VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS: 1500,
107
145
  VOIP_STT_MESSAGE_HANDLING_DELIMITER: '. ',
108
146
  VOIP_STT_MESSAGE_HANDLING_PUNCTUATION: '.!?',
109
147
  VOIP_WEBSOCKET_CONNECT_TIMEOUT: 4000,
110
148
  VOIP_WEBSOCKET_CONNECT_MAXRETRIES: 5,
149
+ // Retry the whole call setup when the SIP peer terminates the INVITE with a
150
+ // transient '487 Request Terminated' before the call connects.
151
+ VOIP_CALL_SETUP_RETRY_487_MAXRETRIES: 2,
152
+ VOIP_CALL_SETUP_RETRY_487_TIMEOUT: 2000,
111
153
  VOIP_SILENCE_DURATION_TIMEOUT: 2500,
112
154
  VOIP_SILENCE_DURATION_TIMEOUT_ENABLE: false,
113
155
  VOIP_SILENCE_DURATION_TIMEOUT_START: 1000,
114
156
  VOIP_SILENCE_DURATION_TIMEOUT_START_ENABLE: false,
115
157
  VOIP_STT_CONFIDENCE_THRESHOLD: 0.5,
158
+ VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS: 500,
116
159
  VOIP_USE_GLOBAL_VOIP_WORKER: false,
117
160
  VOIP_SIP_PROTOCOL: 'TCP',
118
161
  VOIP_USER_INPUT_PREFER_VOICE: true,
119
- VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false
162
+ VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE: false,
163
+ VOIP_TURN_AUDIO_ENABLE: true,
164
+ VOIP_TURN_AUDIO_PADDING_MS: 150,
165
+ VOIP_TURN_AUDIO_OFFSET_MS: 0
166
+ };
167
+
168
+ // Inject the Azure end-of-speech segmentation timeout into the STT body. botium-speech-processing
169
+ // applies azure.config.properties via SpeechConfig.setProperty(), so this controls how long Azure
170
+ // waits after the last word before emitting final=true. Only applied for the Azure engine, never
171
+ // overrides a value already present in the profile config, and clones so the capability object stays
172
+ // untouched. Set the capability to 0/empty to disable injection.
173
+ const injectAzureSegmentationTimeout = (body, sttParams, timeoutMs) => {
174
+ const ms = Number(timeoutMs);
175
+ if (!lodash__default["default"].isFinite(ms) || ms <= 0) return body;
176
+ const isAzure = sttParams && sttParams.stt === 'azure' || body && typeof body === 'object' && body.azure || typeof body === 'string' && body.indexOf('azure') !== -1;
177
+ if (!isAzure) return body;
178
+ let next;
179
+ if (body == null) {
180
+ next = {};
181
+ } else if (typeof body === 'string') {
182
+ try {
183
+ next = JSON.parse(body);
184
+ } catch (err) {
185
+ return body;
186
+ }
187
+ } else {
188
+ next = lodash__default["default"].cloneDeep(body);
189
+ }
190
+ if (!lodash__default["default"].isObject(next.azure)) next.azure = {};
191
+ if (!lodash__default["default"].isObject(next.azure.config)) next.azure.config = {};
192
+ if (!lodash__default["default"].isObject(next.azure.config.properties)) next.azure.config.properties = {};
193
+ if (next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs == null) {
194
+ next.azure.config.properties.Speech_SegmentationSilenceTimeoutMs = String(Math.round(ms));
195
+ }
196
+ return next;
120
197
  };
121
198
  const TTS_HTTP_AGENT = new http__default["default"].Agent({
122
199
  keepAlive: true
@@ -145,6 +222,8 @@ class BotiumConnectorVoip {
145
222
  // For debugging latency between incoming STT (bot says) and outgoing audio (sendAudio)
146
223
  this._lastBotSaysQueuedAt = null;
147
224
  this._lastBotSaysText = null;
225
+ this._replyTrace = null;
226
+ this._activeUserSaysVoipAgent = null;
148
227
  this._speculativeTurnToken = 0;
149
228
  }
150
229
  async Validate() {
@@ -193,6 +272,27 @@ class BotiumConnectorVoip {
193
272
  this.convoStep = null;
194
273
  this._lastBotSaysQueuedAt = null;
195
274
  this._lastBotSaysText = null;
275
+ this._replyTrace = null;
276
+ this._activeUserSaysVoipAgent = null;
277
+ this.audioStream = {
278
+ format: null,
279
+ pcmParts: [],
280
+ totalBytes: 0,
281
+ complete: false
282
+ };
283
+ this._turnAudioCounter = 0;
284
+ this._meTurnAudioOrdinal = -1; // 0-based ordinal of REAL user turns this session
285
+ this._lastBotTurnStartSec = null;
286
+ // Per-turn audio is NOT sliced inline (that would force UserSays to wait for the
287
+ // recording to catch up with playback). Instead each turn records a lightweight
288
+ // descriptor here; slices are cut and emitted as MESSAGE_ATTACHMENT progressively,
289
+ // as soon as the recording buffer has reached each turn's playback end (so the live
290
+ // UI can show them mid-run), with a forced flush at session end for any remainder.
291
+ // Zero added latency for TTS and DTMF turns.
292
+ this._pendingTurnAudio = [];
293
+ this._turnAudioPrevEndSec = null;
294
+ this._turnAudioForceDone = false;
295
+ this._trailingBotAudioEmitted = false;
196
296
  if (this.ttsCache) {
197
297
  this.ttsCache.clear();
198
298
  }
@@ -200,6 +300,7 @@ class BotiumConnectorVoip {
200
300
  const queuedAt = Date.now();
201
301
  this._lastBotSaysQueuedAt = queuedAt;
202
302
  this._lastBotSaysText = botMsg && botMsg.messageText ? String(botMsg.messageText) : null;
303
+ this._captureBotQueuedForReplyTrace(queuedAt);
203
304
  // Stamp the wall-clock instant at which the connector released the bot
204
305
  // utterance to botium-core's queue. Paired with `_receivedAtMs` (last
205
306
  // STT final frame) this gives the true "join silence" the connector
@@ -207,8 +308,33 @@ class BotiumConnectorVoip {
207
308
  // message up with WaitBotSays().
208
309
  if (botMsg && botMsg.sourceData) {
209
310
  const head = Array.isArray(botMsg.sourceData) ? botMsg.sourceData[0] : botMsg.sourceData;
210
- if (head && typeof head === 'object' && !('flushedAtMs' in head)) {
211
- head.flushedAtMs = queuedAt;
311
+ if (head && typeof head === 'object') {
312
+ if (!('flushedAtMs' in head)) head.flushedAtMs = queuedAt;
313
+ if (this._replyTrace) {
314
+ if (lodash__default["default"].isFinite(this._replyTrace.psstFireDelayMs)) head.psstFireDelayMs = this._replyTrace.psstFireDelayMs;
315
+ if (lodash__default["default"].isFinite(this._replyTrace.psstScheduledMs)) head.psstScheduledMs = this._replyTrace.psstScheduledMs;
316
+ }
317
+ }
318
+ }
319
+ // A turn = bot speaks first, then me responds.
320
+ // Save the bot's audio start so UserSays can slice the full exchange
321
+ // (bot + me) once me has finished speaking.
322
+ if (this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE] && botMsg && !(botMsg instanceof Error)) {
323
+ try {
324
+ const sd = botMsg.sourceData;
325
+ let startSec = null;
326
+ if (Array.isArray(sd) && sd.length > 0) {
327
+ startSec = lodash__default["default"].get(sd, '[0].data.start', null);
328
+ } else if (sd && typeof sd === 'object') {
329
+ startSec = lodash__default["default"].get(sd, 'data.start', null);
330
+ }
331
+ // Only record the FIRST bot message's start; if several bot messages
332
+ // arrive before the next UserSays they all belong to the same turn.
333
+ if (lodash__default["default"].isFinite(startSec) && this._lastBotTurnStartSec === null) {
334
+ this._lastBotTurnStartSec = startSec;
335
+ }
336
+ } catch (err) {
337
+ debug$3(`${this.sessionId} - sendBotMsg: saving turn start error: ${err && err.message}`);
212
338
  }
213
339
  }
214
340
  setTimeout(() => this.queueBotSays(botMsg), 0);
@@ -229,6 +355,10 @@ class BotiumConnectorVoip {
229
355
  botMsg.sourceData[0].silenceDuration = lodash__default["default"].isFinite(firstStart) ? firstStart : null;
230
356
  botMsg.sourceData[0].voiceDuration = lodash__default["default"].isFinite(firstStart) && lodash__default["default"].isFinite(lastEnd) ? lastEnd - firstStart : null;
231
357
  }
358
+ const lastSpeechEnd = lodash__default["default"].get(botMsgs, `[${botMsgs.length - 1}].sourceData.data.speechEndSec`, null);
359
+ if (lodash__default["default"].isFinite(lastSpeechEnd) && botMsg.sourceData[0] && botMsg.sourceData[0].data) {
360
+ botMsg.sourceData[0].data.speechEndSec = lastSpeechEnd;
361
+ }
232
362
  return botMsg;
233
363
  };
234
364
 
@@ -258,25 +388,41 @@ class BotiumConnectorVoip {
258
388
  const isJoinMethod = sttHandling === 'JOIN' || sttHandling === 'PSST' || sttHandling === 'CONCAT' || this._hasJoinLogicHookOrRule(this.convoStep);
259
389
  if (!isJoinMethod) return;
260
390
  const joinTimeoutMs = this._getEffectiveJoinTimeoutMs(this.convoStep, this.botMsgs);
391
+ // Variant-3 latency grace: when the last buffered final was cut mid-utterance (no
392
+ // Azure-detected end-of-speech silence) a continuation is plausible but its STT delivery
393
+ // lags behind the audio by ~1s. Extend the wall-clock flush window by the grace so the
394
+ // resumed segment's partial/final (or a worker speechResumed event) can re-arm this timer
395
+ // and be joined, instead of flushing the first fragment on its own. A natural-ended final
396
+ // keeps the base window and flushes fast.
397
+ const graceMs = this._getPsstLatencyGraceMs();
398
+ const graceApplied = graceMs > 0 && lodash__default["default"].isFinite(joinTimeoutMs) && joinTimeoutMs > 0 && !this._isLastFinalNaturalEnd();
399
+ const effectiveWindowMs = graceApplied ? joinTimeoutMs + graceMs : joinTimeoutMs || 0;
261
400
  if (this.silenceTimeout) {
262
401
  clearTimeout(this.silenceTimeout);
263
402
  this.silenceTimeout = null;
264
403
  }
265
404
  const bufferedAtArm = this.botMsgs.length;
266
405
  const armedAt = Date.now();
406
+ this._markReplyTrace({
407
+ psstTimerArmedAtMs: armedAt,
408
+ psstScheduledMs: effectiveWindowMs || 0
409
+ });
267
410
  _info('psst_timer_armed', {
268
411
  sessionId: this.sessionId,
269
- joinTimeoutMs: joinTimeoutMs || 0,
412
+ joinTimeoutMs: effectiveWindowMs || 0,
413
+ baseTimeoutMs: joinTimeoutMs || 0,
414
+ graceMs: graceApplied ? graceMs : 0,
415
+ graceApplied,
270
416
  bufferedChunks: bufferedAtArm,
271
417
  stopCalled: !!this.stopCalled
272
418
  });
273
- // Emit the authoritative join timeout so downstream consumers (e.g.
419
+ // Emit the authoritative flush window so downstream consumers (e.g.
274
420
  // SpeculationBuffer) can size their quiet threshold relative to it.
275
- if (this.eventEmitter && lodash__default["default"].isFinite(joinTimeoutMs) && joinTimeoutMs > 0) {
421
+ if (this.eventEmitter && lodash__default["default"].isFinite(effectiveWindowMs) && effectiveWindowMs > 0) {
276
422
  try {
277
423
  this.eventEmitter.emit('voip.psstTimerArmed', {
278
424
  sessionId: this.sessionId,
279
- joinTimeoutMs,
425
+ joinTimeoutMs: effectiveWindowMs,
280
426
  bufferedChunks: bufferedAtArm,
281
427
  armedAt
282
428
  });
@@ -287,15 +433,20 @@ class BotiumConnectorVoip {
287
433
  }
288
434
  this.silenceTimeout = setTimeout(() => {
289
435
  const fireDelay = Date.now() - armedAt;
436
+ this._markReplyTrace({
437
+ psstTimerFiredAtMs: Date.now(),
438
+ psstFireDelayMs: fireDelay
439
+ });
290
440
  if (this.botMsgs.length > 0) {
291
441
  _info('psst_timer_fired', {
292
442
  sessionId: this.sessionId,
293
443
  bufferedChunks: this.botMsgs.length,
294
444
  actualDelayMs: fireDelay,
295
- scheduledDelayMs: joinTimeoutMs || 0,
445
+ scheduledDelayMs: effectiveWindowMs || 0,
446
+ graceApplied,
296
447
  outcome: 'emit'
297
448
  });
298
- debug$3('Silence Duration Timeout (JOIN/PSST):', joinTimeoutMs, 'ms');
449
+ debug$3('Silence Duration Timeout (JOIN/PSST):', effectiveWindowMs, 'ms');
299
450
  sendBotMsg(joinBotMsg(this.botMsgs, this.joinLastPrevMsg));
300
451
  this.firstMsg = false;
301
452
  this.joinLastPrevMsg = this.botMsgs[this.botMsgs.length - 1];
@@ -308,11 +459,11 @@ class BotiumConnectorVoip {
308
459
  sessionId: this.sessionId,
309
460
  bufferedChunks: 0,
310
461
  actualDelayMs: fireDelay,
311
- scheduledDelayMs: joinTimeoutMs || 0,
462
+ scheduledDelayMs: effectiveWindowMs || 0,
312
463
  outcome: 'noop_empty_buffer'
313
464
  });
314
465
  }
315
- }, joinTimeoutMs || 0);
466
+ }, effectiveWindowMs || 0);
316
467
  };
317
468
 
318
469
  // Flush buffered STT chunks on teardown so a late final is not lost when
@@ -438,15 +589,17 @@ class BotiumConnectorVoip {
438
589
  await connectHttp(retryIndex + 1);
439
590
  }
440
591
  };
441
- await connectHttp();
442
- if (httpInitRetries > 0) {
443
- _info('connected_after_retries', {
444
- phase: 'initCall',
445
- retries: httpInitRetries
446
- });
447
- }
448
592
  return new Promise((resolve, reject) => {
449
- const wsEndpoint = `${this.caps[Capabilities.VOIP_USE_GLOBAL_VOIP_WORKER] ? process.env.BOTIUM_VOIP_WORKER_URL : this.caps[Capabilities.VOIP_WORKER_URL]}/ws/${data.port}`;
593
+ // Worker session details (port) come from connectHttp() and are recomputed
594
+ // on every (re)try, since each setup attempt gets a fresh worker session.
595
+ let wsEndpoint = null;
596
+ const computeWsEndpoint = () => `${this.caps[Capabilities.VOIP_USE_GLOBAL_VOIP_WORKER] ? process.env.BOTIUM_VOIP_WORKER_URL : this.caps[Capabilities.VOIP_WORKER_URL]}/ws/${data.port}`;
597
+ // 487-retry bookkeeping: a transient '487 Request Terminated' that arrives
598
+ // before the call connects re-runs the whole setup (fresh initCall -> ws
599
+ // -> SIP INVITE) up to max487Retries times.
600
+ let setup487Retries = 0;
601
+ let convoStepListenerAttached = false;
602
+ const max487Retries = this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_MAXRETRIES];
450
603
  const connectWs = retryIndex => {
451
604
  retryIndex = retryIndex || 0;
452
605
  return new Promise((resolve, reject) => {
@@ -480,7 +633,7 @@ class BotiumConnectorVoip {
480
633
  });
481
634
  });
482
635
  };
483
- connectWs().then(wsRetries => {
636
+ const onWsConnected = wsRetries => {
484
637
  if (wsRetries > 0) {
485
638
  _info('connected_after_retries', {
486
639
  phase: 'websocket',
@@ -494,26 +647,32 @@ class BotiumConnectorVoip {
494
647
  this.caps[Capabilities.VOIP_ICE_STUN_SERVERS] = this.caps[Capabilities.VOIP_ICE_STUN_SERVERS].split(',');
495
648
  }
496
649
  }
497
- this.eventEmitter.on('CONVO_STEP_NEXT', (container, convoStep) => {
498
- this.convoStep = convoStep;
499
- this._maybePrefetchTts(convoStep);
500
- // For PSST: send join silence duration per step to VOIP worker (controls PSST silence trigger)
501
- try {
502
- if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'PSST' && this.ws && this.ws.readyState === ws__default["default"].OPEN) {
503
- const silenceMs = this._getEffectiveJoinTimeoutMs(convoStep, this.botMsgs);
504
- if (lodash__default["default"].isFinite(silenceMs) && silenceMs > 0 && this.sessionId) {
505
- debug$3(`PSST: sending silenceDurationMs=${silenceMs} for sessionId=${this.sessionId}`);
506
- this.ws.send(JSON.stringify({
507
- METHOD: 'setSttSilenceDuration',
508
- sessionId: this.sessionId,
509
- silenceDurationMs: silenceMs
510
- }));
650
+
651
+ // Attach this once: the listener reads this.ws dynamically, so it keeps
652
+ // working across 487 setup retries that swap out the websocket.
653
+ if (!convoStepListenerAttached) {
654
+ convoStepListenerAttached = true;
655
+ this.eventEmitter.on('CONVO_STEP_NEXT', (container, convoStep) => {
656
+ this.convoStep = convoStep;
657
+ this._maybePrefetchTts(convoStep);
658
+ // For PSST: send join silence duration per step to VOIP worker (controls PSST silence trigger)
659
+ try {
660
+ if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'PSST' && this.ws && this.ws.readyState === ws__default["default"].OPEN) {
661
+ const silenceMs = this._getEffectiveJoinTimeoutMs(convoStep, this.botMsgs);
662
+ if (lodash__default["default"].isFinite(silenceMs) && silenceMs > 0 && this.sessionId) {
663
+ debug$3(`PSST: sending silenceDurationMs=${silenceMs} for sessionId=${this.sessionId}`);
664
+ this.ws.send(JSON.stringify({
665
+ METHOD: 'setSttSilenceDuration',
666
+ sessionId: this.sessionId,
667
+ silenceDurationMs: silenceMs
668
+ }));
669
+ }
511
670
  }
671
+ } catch (err) {
672
+ debug$3(`Failed sending PSST silence duration to VOIP worker: ${err.message || err}`);
512
673
  }
513
- } catch (err) {
514
- debug$3(`Failed sending PSST silence duration to VOIP worker: ${err.message || err}`);
515
- }
516
- });
674
+ });
675
+ }
517
676
  this.silence = null;
518
677
  this.msgCount = 0;
519
678
  this.sttPartialCount = 0;
@@ -561,11 +720,12 @@ class BotiumConnectorVoip {
561
720
  ICE_TURN_PROTOCOL: this.caps[Capabilities.VOIP_ICE_TURN_PROTOCOL] || 'TCP',
562
721
  MIN_SILENCE_DURATION: this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT_ENABLE] ? this.caps[Capabilities.VOIP_SILENCE_DURATION_TIMEOUT] : null,
563
722
  SDP_MEDIA_TYPE_TEXT_ENABLE: !!this.caps[Capabilities.VOIP_SDP_MEDIA_TYPE_TEXT_ENABLE],
723
+ AUDIO_STREAM: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE],
564
724
  STT_LEGACY: sttLegacy,
565
725
  STT_CONFIG: {
566
726
  stt_url: sttUrl,
567
727
  stt_params: this.caps[Capabilities.VOIP_STT_PARAMS_STREAM],
568
- stt_body: this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null
728
+ stt_body: injectAzureSegmentationTimeout(this.caps[Capabilities.VOIP_STT_BODY_STREAM] || null, this.caps[Capabilities.VOIP_STT_PARAMS_STREAM], this.caps[Capabilities.VOIP_STT_AZURE_SEGMENTATION_SILENCE_TIMEOUT_MS])
569
729
  },
570
730
  TTS_CONFIG: {
571
731
  tts_url: this.caps[Capabilities.VOIP_TTS_URL],
@@ -630,7 +790,7 @@ class BotiumConnectorVoip {
630
790
  // (fullRecord*) and hard errors so `full_record.wav` is delivered
631
791
  // on early-completion hangups. Post-Stop STT frames remain blocked.
632
792
  if (this.stopCalled) {
633
- const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error'];
793
+ const allowedPostStopTypes = ['fullRecord', 'fullRecordStart', 'fullRecordChunk', 'fullRecordEnd', 'error', 'audioStreamStart', 'audioStreamChunk', 'audioStreamEnd'];
634
794
  if (!parsedData || !allowedPostStopTypes.includes(parsedData.type)) {
635
795
  debug$3(`${this.sessionId} - Stop already called, ignoring incoming message`);
636
796
  return;
@@ -642,7 +802,7 @@ class BotiumConnectorVoip {
642
802
  if (!obj || typeof obj !== 'object') return;
643
803
  for (const key of Object.keys(obj)) {
644
804
  const val = obj[key];
645
- if (typeof val === 'string' && val.length > 500) {
805
+ if (typeof val === 'string' && val.length > 0 && (WS_DEBUG_BASE64_FIELD_NAMES.has(key) || val.length > 500)) {
646
806
  obj[key] = `<base64:${val.length}chars>`;
647
807
  } else if (val && typeof val === 'object' && !Array.isArray(val)) {
648
808
  sanitizeBase64Fields(val, `${prefix}${key}.`);
@@ -650,7 +810,9 @@ class BotiumConnectorVoip {
650
810
  }
651
811
  };
652
812
  sanitizeBase64Fields(parsedDataLog);
653
- debug$3(JSON.stringify(parsedDataLog, null, 2));
813
+ if (!WS_DEBUG_SILENT_TYPES.has(parsedData?.type)) {
814
+ debug$3(JSON.stringify(parsedDataLog, null, 2));
815
+ }
654
816
  const _extractFullRecordBase64 = pd => {
655
817
  if (!pd) return null;
656
818
  // Different VOIP workers may put the payload in various fields - search all string fields
@@ -754,6 +916,9 @@ class BotiumConnectorVoip {
754
916
  reject(new Error('Error: Sip Registration failed'));
755
917
  }
756
918
  if (parsedData && parsedData.type === 'callinfo' && parsedData.status === 'connected') {
919
+ // Mark connected so a later terminal error is delivered to the bot
920
+ // conversation instead of triggering a (now pointless) setup retry.
921
+ this.connected = true;
757
922
  _info('callinfo_connected', {
758
923
  sessionId: this.sessionId
759
924
  });
@@ -788,6 +953,23 @@ class BotiumConnectorVoip {
788
953
  });
789
954
  }
790
955
  if (parsedData && parsedData.type === 'error') {
956
+ const errMsg = parsedData.message || '';
957
+ // The worker reports the SIP code inside the message string
958
+ // ("Disconnected because of error - Reason: 487 Request Terminated")
959
+ // and does not set a dedicated `code` field, so detect 487 from both.
960
+ const is487 = parsedData.code === 487 || parsedData.code === '487' || /\b487\b/.test(errMsg) || /request terminated/i.test(errMsg);
961
+ // A transient '487 Request Terminated' that arrives before the call
962
+ // connects: retry the whole call setup instead of failing the test.
963
+ if (is487 && !this.connected && setup487Retries < max487Retries) {
964
+ _info('ws_error_msg', {
965
+ sessionId: this.sessionId,
966
+ message: parsedData.message || null,
967
+ code: parsedData.code || null,
968
+ retrying487: true
969
+ });
970
+ retryCallSetup487(errMsg);
971
+ return;
972
+ }
791
973
  flushPendingBotMsgs('error');
792
974
  // Ensure buffered recording is not lost on terminal worker errors.
793
975
  this._emitBufferedFullRecordIfAny('error_buffered');
@@ -801,6 +983,46 @@ class BotiumConnectorVoip {
801
983
  sendBotMsg(new Error(`Error: ${parsedData.message}`));
802
984
  }
803
985
 
986
+ // Per-turn audio stream: continuous PCM chunks received during the call.
987
+ // The connector buffers them so _sliceTurnAudio() can extract per-turn segments.
988
+ if (parsedData && parsedData.type === 'audioStreamStart') {
989
+ this.audioStream = {
990
+ format: {
991
+ sampleRate: parsedData.sampleRate,
992
+ channels: parsedData.channels,
993
+ bitsPerSample: parsedData.bitsPerSample,
994
+ dataOffset: parsedData.dataOffset
995
+ },
996
+ pcmParts: [],
997
+ totalBytes: 0,
998
+ complete: false
999
+ };
1000
+ debug$3(`${this.sessionId} - audioStreamStart sampleRate=${parsedData.sampleRate} channels=${parsedData.channels} bitsPerSample=${parsedData.bitsPerSample}`);
1001
+ }
1002
+ if (parsedData && parsedData.type === 'audioStreamChunk') {
1003
+ if (this.audioStream && parsedData.chunk) {
1004
+ try {
1005
+ const buf = Buffer.from(parsedData.chunk, 'base64');
1006
+ this.audioStream.pcmParts.push(buf);
1007
+ this.audioStream.totalBytes += buf.length;
1008
+ this._maybeDetectAgentAudibleOnRecording(this._activeUserSaysVoipAgent);
1009
+ // Emit any per-turn audio whose playback the recording has now caught up to,
1010
+ // so the live transcript can show it mid-run (no UserSays latency).
1011
+ this._emitReadyTurnAudio('audioStreamChunk', false);
1012
+ } catch (e) {
1013
+ debug$3(`${this.sessionId} - audioStreamChunk decode error: ${e && e.message}`);
1014
+ }
1015
+ }
1016
+ }
1017
+ if (parsedData && parsedData.type === 'audioStreamEnd') {
1018
+ if (this.audioStream) {
1019
+ this.audioStream.complete = true;
1020
+ }
1021
+ debug$3(`${this.sessionId} - audioStreamEnd totalBytes=${parsedData.totalBytes}`);
1022
+ // Buffer is complete — cut and emit the per-turn audio now.
1023
+ this._flushPendingTurnAudio('audioStreamEnd');
1024
+ }
1025
+
804
1026
  // Full record streaming support:
805
1027
  // - some VOIP workers send the recording in chunks and an end marker
806
1028
  if (parsedData && parsedData.type === 'fullRecordStart') {
@@ -820,11 +1042,49 @@ class BotiumConnectorVoip {
820
1042
  source: 'fullRecordEnd',
821
1043
  base64Len
822
1044
  });
1045
+ // Emit per-turn audio before `this.end = true` so it is captured by the
1046
+ // worker before Stop() resolves (no-op if audioStreamEnd already flushed).
1047
+ this._flushPendingTurnAudio('fullRecordEnd');
823
1048
  // Flush before `this.end = true` so the buffered final STT is not
824
1049
  // dropped when Stop() clears the PSST silence timer on teardown.
825
1050
  flushPendingBotMsgs('fullRecordEnd');
826
1051
  this.end = true;
827
1052
  }
1053
+ if (parsedData && parsedData.type === 'agentPlaybackStarted') {
1054
+ const playbackData = parsedData.data || {};
1055
+ const playedSec = playbackData.playedRecordingStartSec;
1056
+ const active = this._activeUserSaysVoipAgent;
1057
+ if (active && lodash__default["default"].isFinite(playedSec)) {
1058
+ active.playedRecordingStartSec = playedSec;
1059
+ active.playbackAtMs = playbackData.playbackAtMs;
1060
+ if (lodash__default["default"].isFinite(playbackData.requestedDurationMs)) {
1061
+ active.playbackRequestedDurationMs = playbackData.requestedDurationMs;
1062
+ }
1063
+ if (lodash__default["default"].isFinite(playbackData.digitCount)) {
1064
+ active.digitCount = playbackData.digitCount;
1065
+ }
1066
+ this._markReplyTrace({
1067
+ playedRecordingStartSec: playedSec,
1068
+ playbackAtMs: playbackData.playbackAtMs
1069
+ });
1070
+ const heardSec = playbackData.wireKind === 'dtmf' ? playedSec : this._applyAgentHeardRecordingStartSec(active);
1071
+ if (lodash__default["default"].isFinite(heardSec)) {
1072
+ if (playbackData.wireKind === 'dtmf') {
1073
+ active.heardRecordingStartSec = heardSec;
1074
+ this._markReplyTrace({
1075
+ heardRecordingStartSec: heardSec
1076
+ });
1077
+ }
1078
+ debug$3(`${this.sessionId} - agent audible on recording at ${heardSec}s (played=${playedSec}s)`);
1079
+ }
1080
+ // Log the heard reply-trace now that playback is audible — earlier than the
1081
+ // old turn-audio callback and free of the next-turn _replyTrace race. The
1082
+ // agent's end on the recording = played + playback length.
1083
+ const playbackSec = lodash__default["default"].isFinite(active.playbackRequestedDurationMs) && active.playbackRequestedDurationMs > 0 ? active.playbackRequestedDurationMs / 1000 : playbackData.wireKind === 'dtmf' ? this._dtmfPlaybackSec(active, active.digitCount || 1) : null;
1084
+ const agentEndSec = lodash__default["default"].isFinite(playbackSec) ? playedSec + playbackSec : undefined;
1085
+ this._logReplyTraceHeard(agentEndSec);
1086
+ }
1087
+ }
828
1088
  if (parsedData && parsedData.type === 'silence') {
829
1089
  if (lodash__default["default"].isNil(this._getIgnoreSilenceDurationAsserterLogicHook(this.convoStep))) {
830
1090
  if (!this._hasJoinLogicHookOrRule(this.convoStep) && parsedData.data.silence.length > 0) {
@@ -834,6 +1094,36 @@ class BotiumConnectorVoip {
834
1094
  }
835
1095
  }
836
1096
  }
1097
+ if (parsedData && parsedData.type === 'speech' && parsedData.event === 'onSpeechResumed') {
1098
+ // Positive VAD signal from the worker: the bot started speaking again after a pause.
1099
+ // It arrives ~1s before the resumed segment's first STT partial, so when we are
1100
+ // buffering finals (PSST/JOIN) we re-arm the flush timer now — keeping the buffered
1101
+ // fragment open to be joined instead of flushed on its own. Bounded by the same
1102
+ // extension budget as partial-driven re-arms to prevent infinite stranding.
1103
+ _info('speech_resumed', {
1104
+ sessionId: this.sessionId
1105
+ });
1106
+ const sttHandling = this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING];
1107
+ const isJoinMethod = sttHandling === 'JOIN' || sttHandling === 'PSST' || sttHandling === 'CONCAT' || this._hasJoinLogicHookOrRule(this.convoStep);
1108
+ if (isJoinMethod && this.botMsgs && this.botMsgs.length > 0) {
1109
+ const MAX_EXTENSION_MS = 60000;
1110
+ const now = Date.now();
1111
+ const withinCap = this.psstFirstRearmAt == null || now - this.psstFirstRearmAt < MAX_EXTENSION_MS;
1112
+ if (withinCap) {
1113
+ if (this.psstFirstRearmAt == null) this.psstFirstRearmAt = now;
1114
+ this.psstRearmCount++;
1115
+ _info('psst_timer_extended', {
1116
+ sessionId: this.sessionId,
1117
+ rearmCount: this.psstRearmCount,
1118
+ msSinceFirstRearm: now - this.psstFirstRearmAt,
1119
+ bufferedChunks: this.botMsgs.length,
1120
+ reason: 'speech_resumed',
1121
+ silenceDurationMs: lodash__default["default"].get(parsedData, 'data.silenceDurationMs', null)
1122
+ });
1123
+ armJoinSilenceTimer();
1124
+ }
1125
+ }
1126
+ }
837
1127
  if (parsedData && parsedData.type === 'fullRecord') {
838
1128
  // Non-chunked full record
839
1129
  const base64 = _extractFullRecordBase64(parsedData);
@@ -953,8 +1243,13 @@ class BotiumConnectorVoip {
953
1243
  messageLength: msgLen,
954
1244
  confidence: this._getConfidenceScore(parsedData),
955
1245
  threshold: confidenceThreshold,
956
- accepted: successfulConfidenceScore
1246
+ accepted: successfulConfidenceScore,
1247
+ segmentEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.end')) ? parsedData.data.end : null,
1248
+ speechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(parsedData, 'data.speechEndSec')) ? parsedData.data.speechEndSec : null
957
1249
  });
1250
+ if (successfulConfidenceScore) {
1251
+ this._captureSttFinalForReplyTrace(parsedData, msgPreview);
1252
+ }
958
1253
  // ORIGINAL: always emit final message immediately (ignore JOIN hooks/rules).
959
1254
  if (this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING] === 'ORIGINAL') {
960
1255
  let botMsg = {
@@ -1051,9 +1346,49 @@ class BotiumConnectorVoip {
1051
1346
  }
1052
1347
  }
1053
1348
  });
1054
- }).catch(err => {
1055
- reject(new Error('Error: ' + err));
1056
- });
1349
+ };
1350
+ const retryCallSetup487 = reasonMessage => {
1351
+ setup487Retries++;
1352
+ _info('call_setup_retry_487', {
1353
+ sessionId: this.sessionId,
1354
+ attempt: setup487Retries,
1355
+ max: max487Retries,
1356
+ reason: reasonMessage || null
1357
+ });
1358
+ debug$3(`Call setup 487 retry ${setup487Retries}/${max487Retries}: ${reasonMessage}`);
1359
+ // Tear down the dead websocket so its stale handlers cannot fire while
1360
+ // the next attempt establishes a fresh worker session.
1361
+ try {
1362
+ if (this.ws) {
1363
+ this.ws.removeAllListeners();
1364
+ this.ws.terminate();
1365
+ }
1366
+ } catch (err) {
1367
+ debug$3(`Call setup 487 retry: websocket teardown failed: ${err && err.message}`);
1368
+ }
1369
+ this.ws = null;
1370
+ this.wsOpened = false;
1371
+ this.end = false;
1372
+ setTimeout(() => {
1373
+ establishCall().catch(err => reject(new Error('Error: ' + err)));
1374
+ }, this.caps[Capabilities.VOIP_CALL_SETUP_RETRY_487_TIMEOUT]);
1375
+ };
1376
+ const establishCall = async () => {
1377
+ // Each (re)try gets a fresh worker session: new initCall -> new port ->
1378
+ // new websocket -> new SIP INVITE.
1379
+ httpInitRetries = 0;
1380
+ await connectHttp();
1381
+ if (httpInitRetries > 0) {
1382
+ _info('connected_after_retries', {
1383
+ phase: 'initCall',
1384
+ retries: httpInitRetries
1385
+ });
1386
+ }
1387
+ wsEndpoint = computeWsEndpoint();
1388
+ const wsRetries = await connectWs();
1389
+ onWsConnected(wsRetries);
1390
+ };
1391
+ establishCall().catch(err => reject(new Error('Error: ' + err)));
1057
1392
  });
1058
1393
  }
1059
1394
  async UserSays(msg) {
@@ -1063,6 +1398,10 @@ class BotiumConnectorVoip {
1063
1398
  const hasDtmf = !!(msg && msg.buttons && msg.buttons.length > 0);
1064
1399
  const dtmfMatch = msg && msg.messageText && msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
1065
1400
  const inputType = hasDtmf || dtmfMatch ? 'dtmf' : hasText && hasVoiceMedia ? 'mixed' : hasText ? 'text' : hasVoiceMedia ? 'media' : 'unknown';
1401
+ // A real user turn is one that sends content (text/media/dtmf). 'unknown' turns send nothing and
1402
+ // surface server-side as skippable me-steps, so they must NOT consume an ordinal slot - this keeps
1403
+ // meTurnIndex aligned with the server's non-skippable me-step indices when placing turn audio.
1404
+ const meTurnIndex = inputType !== 'unknown' ? this._meTurnAudioOrdinal = (this._meTurnAudioOrdinal ?? -1) + 1 : null;
1066
1405
  const msgPreview = hasText && msg.messageText ? String(msg.messageText).trim().substring(0, 80) : '';
1067
1406
  _info('user_says', {
1068
1407
  sessionId: this.sessionId,
@@ -1071,6 +1410,7 @@ class BotiumConnectorVoip {
1071
1410
  messageLength: hasText && msg.messageText ? msg.messageText.length : undefined,
1072
1411
  mediaSize: hasVoiceMedia && msg.media[0] && Buffer.isBuffer(msg.media[0].buffer) ? msg.media[0].buffer.length : undefined
1073
1412
  });
1413
+ this._captureUserSaysStart(msgPreview);
1074
1414
  // Avoid logging large buffers/base64 (can break job logs and overwhelm stdout)
1075
1415
  try {
1076
1416
  const safeLog = {
@@ -1105,17 +1445,45 @@ class BotiumConnectorVoip {
1105
1445
  // the coach can place the agent turn on the recording timeline.
1106
1446
  // `requestedDurationMs` is the best estimate of on-wire playback
1107
1447
  // length (DTMF tones × digits, TTS synth output, parsed media duration).
1448
+ const recordingSecNow = () => this._recordingSecNow();
1108
1449
  const stampAgentWire = (wireKind, requestedDurationMs, extras = {}) => {
1450
+ const wireRecordingStartSec = recordingSecNow();
1109
1451
  msg.voipAgent = {
1110
1452
  wireSentAtMs: Date.now(),
1111
1453
  inputType,
1112
1454
  wireKind,
1113
1455
  requestedDurationMs: Math.max(0, Math.round(requestedDurationMs || 0)),
1456
+ ...(wireRecordingStartSec != null ? {
1457
+ wireRecordingStartSec
1458
+ } : {}),
1459
+ // Carry the coach's reply-decision trace (set on the outgoing message before UserSays)
1460
+ // into voipAgent here, while we own the object and before MESSAGE_SENTTOBOT fires — so
1461
+ // the LIVE transcript snapshot (a deep copy taken at that event) already has it, not
1462
+ // just the final result. Lets the UI split "Tester reply decision" live.
1463
+ ...(msg && msg.coachTrace && typeof msg.coachTrace === 'object' ? {
1464
+ coachTrace: msg.coachTrace
1465
+ } : {}),
1114
1466
  ...extras
1115
1467
  };
1468
+ this._activeUserSaysVoipAgent = msg.voipAgent;
1469
+ this._captureAgentWire(msg.voipAgent, inputType);
1470
+ };
1471
+ const sendAgentWire = request => {
1472
+ this._sendUserSaysWs(request);
1473
+ this._markReplyTrace({
1474
+ sendAudioAtMs: Date.now()
1475
+ });
1476
+ this._logReplyTrace('wire_sent');
1477
+ // Finalize the wall pipeline NOW, while _replyTrace still holds this turn's data
1478
+ // (userSaysAtMs / ttsStartAtMs / ttsEndAtMs / wireAtMs / sendAudioAtMs are all set by
1479
+ // this point). The deferred turn-audio callback runs only after the agent audio is
1480
+ // "heard" on the recording, which nulls _replyTrace (_logReplyTraceHeard); finalizing
1481
+ // there would drop wallPipeline entirely and collapse the reply-delay breakdown into a
1482
+ // single "Audio send" bucket on the UI.
1483
+ this._finalizeWallPipeline(msg.voipAgent);
1116
1484
  };
1117
1485
  // Twilio default: 100 ms tone + 100 ms gap per digit. Drives agent-bar width only.
1118
- const DTMF_MS_PER_DIGIT = 200;
1486
+ const DTMF_AGENT_BAR_MS_PER_DIGIT = DTMF_MS_PER_DIGIT;
1119
1487
  if (msg && msg.buttons && msg.buttons.length > 0) {
1120
1488
  const digits = sanitizeDtmfDigits(msg.buttons[0].payload);
1121
1489
  if (!digits) {
@@ -1128,10 +1496,13 @@ class BotiumConnectorVoip {
1128
1496
  digits,
1129
1497
  sessionId: this.sessionId
1130
1498
  });
1131
- stampAgentWire('dtmf', digits.length * DTMF_MS_PER_DIGIT, {
1132
- digitCount: digits.length
1499
+ stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
1500
+ digitCount: digits.length,
1501
+ dtmfDigits: digits
1133
1502
  });
1134
- this._sendUserSaysWs(request);
1503
+ sendAgentWire(request);
1504
+ // Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
1505
+ duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
1135
1506
  } else if (msg && msg.messageText) {
1136
1507
  // Check for DTMF tag in messageText: <DTMF>1234</DTMF>
1137
1508
  const dtmfMatch = msg.messageText.match(/<DTMF>([^<]+)<\/DTMF>/i);
@@ -1152,13 +1523,14 @@ class BotiumConnectorVoip {
1152
1523
  digits,
1153
1524
  sessionId: this.sessionId
1154
1525
  });
1155
- stampAgentWire('dtmf', digits.length * DTMF_MS_PER_DIGIT, {
1156
- digitCount: digits.length
1526
+ stampAgentWire('dtmf', digits.length * DTMF_AGENT_BAR_MS_PER_DIGIT, {
1527
+ digitCount: digits.length,
1528
+ dtmfDigits: digits
1157
1529
  });
1158
- this._sendUserSaysWs(request);
1159
- return resolve();
1160
- }
1161
- if (!skipTtsForMixedInput) {
1530
+ sendAgentWire(request);
1531
+ // Wait for DTMF playback plus at least one audioStream flush before slicing turn audio.
1532
+ duration = dtmfTurnAudioWaitMs(digits.length) / 1000;
1533
+ } else if (!skipTtsForMixedInput) {
1162
1534
  if (!this.axiosTtsParams) {
1163
1535
  if (!(msg.media && msg.media.length > 0 && msg.media[0].buffer)) {
1164
1536
  return reject(new Error('TTS not configured, only audio input supported'));
@@ -1170,10 +1542,17 @@ class BotiumConnectorVoip {
1170
1542
  msg.sourceData = ttsRequest;
1171
1543
  let ttsResult = null;
1172
1544
  const ttsStartedAt = Date.now();
1545
+ this._markReplyTrace({
1546
+ ttsStartAtMs: ttsStartedAt
1547
+ });
1173
1548
  let ttsSynthMs = 0;
1174
1549
  try {
1175
1550
  ttsResult = await this._getTtsAudio(ttsRequest, msg.messageText);
1176
1551
  ttsSynthMs = Date.now() - ttsStartedAt;
1552
+ this._markReplyTrace({
1553
+ ttsEndAtMs: Date.now(),
1554
+ ttsSynthMs
1555
+ });
1177
1556
  } catch (err) {
1178
1557
  return reject(new Error(`TTS "${msg.messageText}" failed - ${this._getAxiosErrOutput(err)}`));
1179
1558
  }
@@ -1209,7 +1588,7 @@ class BotiumConnectorVoip {
1209
1588
  ttsSynthMs,
1210
1589
  textLength: msg.messageText ? msg.messageText.length : 0
1211
1590
  });
1212
- this._sendUserSaysWs(request);
1591
+ sendAgentWire(request);
1213
1592
  } else {
1214
1593
  return reject(new Error('TTS failed, response is empty'));
1215
1594
  }
@@ -1234,7 +1613,7 @@ class BotiumConnectorVoip {
1234
1613
  stampAgentWire('media', 0, {
1235
1614
  mediaUri: msg.media[0].mediaUri || null
1236
1615
  });
1237
- this._sendUserSaysWs(request);
1616
+ sendAgentWire(request);
1238
1617
  msg.attachments.push({
1239
1618
  name: msg.media[0].mediaUri,
1240
1619
  mimeType: msg.media[0].mimeType,
@@ -1256,16 +1635,514 @@ class BotiumConnectorVoip {
1256
1635
  }
1257
1636
  }
1258
1637
  const requestedDurationMs = Math.max(0, Math.round((duration || 0) * 1000));
1259
- if (requestedDurationMs <= 0) {
1260
- return resolve();
1261
- }
1262
- setTimeout(resolve, requestedDurationMs);
1638
+
1639
+ // Record this turn's slice bounds and resolve immediately — the actual
1640
+ // slice is cut from the complete buffer at session end (see
1641
+ // _flushPendingTurnAudio). This keeps UserSays from waiting for the
1642
+ // recording to catch up with playback (no latency for TTS or DTMF). The
1643
+ // heard-trace telemetry (voip_reply_trace_heard) is logged from the
1644
+ // agentPlaybackStarted handler once playback is audible on the recording.
1645
+ this._recordPendingTurnAudio(msg.voipAgent, requestedDurationMs, meTurnIndex);
1646
+ resolve();
1263
1647
  } catch (err) {
1264
1648
  reject(err);
1265
1649
  }
1266
1650
  }, 0);
1267
1651
  });
1268
1652
  }
1653
+ _recordingSecNow() {
1654
+ const fmt = this.audioStream && this.audioStream.format;
1655
+ const bytesPerSec = fmt ? fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8) : null;
1656
+ if (!bytesPerSec || !this.audioStream || !(this.audioStream.totalBytes > 0)) return null;
1657
+ return this.audioStream.totalBytes / bytesPerSec;
1658
+ }
1659
+
1660
+ /** Expected on-recording playback length of a DTMF turn (worker-reported, else estimated). */
1661
+ _dtmfPlaybackSec(voipAgent, digitCount) {
1662
+ const playbackMs = voipAgent && voipAgent.playbackRequestedDurationMs;
1663
+ if (lodash__default["default"].isFinite(playbackMs) && playbackMs > 0) return playbackMs / 1000;
1664
+ return dtmfPlaybackMs(digitCount) / 1000;
1665
+ }
1666
+
1667
+ /**
1668
+ * Snapshot the bounds needed to slice this turn's audio later, then return.
1669
+ * `_lastBotTurnStartSec` is the bot-speech anchor; it is consumed here so the
1670
+ * next bot message sets a fresh one. The end of the agent's playback is read
1671
+ * from voipAgent.playedRecordingStartSec (filled asynchronously by the
1672
+ * agentPlaybackStarted handler) at flush time. Consecutive agent turns with no
1673
+ * bot message in between (anchor null) chain off the previous turn's end.
1674
+ */
1675
+ _recordPendingTurnAudio(voipAgent, requestedDurationMs, meTurnIndex) {
1676
+ if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
1677
+ const botAnchorSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
1678
+ this._lastBotTurnStartSec = null;
1679
+ this._pendingTurnAudio.push({
1680
+ botAnchorSec,
1681
+ voipAgent: voipAgent || null,
1682
+ requestedDurationMs: Math.max(0, requestedDurationMs || 0),
1683
+ meTurnIndex: Number.isInteger(meTurnIndex) && meTurnIndex >= 0 ? meTurnIndex : null
1684
+ });
1685
+ _info('turn_audio_recorded', {
1686
+ sessionId: this.sessionId,
1687
+ meTurnIndex,
1688
+ botAnchorSec,
1689
+ wireKind: voipAgent && voipAgent.wireKind,
1690
+ pending: this._pendingTurnAudio.length
1691
+ });
1692
+ }
1693
+
1694
+ /**
1695
+ * Emit per-turn audio (turn_N.wav) for every pending turn whose playback end the
1696
+ * recording buffer has already reached, slicing from the live buffer and emitting a
1697
+ * MESSAGE_ATTACHMENT carrying meTurnIndex. Turns are processed in order (the chain
1698
+ * start of a follow-up turn with no bot anchor depends on the previous turn's end), so
1699
+ * a not-yet-ready turn stops the pass until more audio streams in. With force=true
1700
+ * (session end) the remainder is emitted even if the buffer is incomplete.
1701
+ */
1702
+ _emitReadyTurnAudio(reason, force) {
1703
+ if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
1704
+ const pending = this._pendingTurnAudio || [];
1705
+ if (!pending.length) return;
1706
+ if (!this.audioStream || !this.audioStream.format) return; // no PCM yet
1707
+ const slackSec = (audioStreamIntervalMs() + 50) / 1000;
1708
+ const recNow = this._recordingSecNow();
1709
+ for (const t of pending) {
1710
+ if (t.emitted) continue;
1711
+ try {
1712
+ const va = t.voipAgent || {};
1713
+ const playbackSec = va.wireKind === 'dtmf' ? this._dtmfPlaybackSec(va, va.digitCount || 1) : lodash__default["default"].isFinite(va.playbackRequestedDurationMs) && va.playbackRequestedDurationMs > 0 ? va.playbackRequestedDurationMs / 1000 : t.requestedDurationMs / 1000;
1714
+ const startSec = lodash__default["default"].isFinite(t.botAnchorSec) ? t.botAnchorSec : this._turnAudioPrevEndSec;
1715
+ // End = where the agent's playback finished on the recording. _sliceTurnAudio
1716
+ // adds the configured padding on top.
1717
+ const anchorSec = lodash__default["default"].isFinite(va.playedRecordingStartSec) ? va.playedRecordingStartSec : lodash__default["default"].isFinite(va.heardRecordingStartSec) ? va.heardRecordingStartSec : lodash__default["default"].isFinite(va.wireRecordingStartSec) ? va.wireRecordingStartSec : null;
1718
+ let endSec = lodash__default["default"].isFinite(anchorSec) ? anchorSec + playbackSec : null;
1719
+ if (!lodash__default["default"].isFinite(endSec) && lodash__default["default"].isFinite(startSec)) endSec = startSec + playbackSec;
1720
+
1721
+ // Progressive: only emit once the recording has reached this turn's end (plus a
1722
+ // stream-flush slack). Stop the pass otherwise — later turns must wait their turn.
1723
+ if (!force) {
1724
+ if (!lodash__default["default"].isFinite(anchorSec) || !lodash__default["default"].isFinite(endSec)) break;
1725
+ if (!lodash__default["default"].isFinite(recNow) || recNow < endSec + slackSec) break;
1726
+ }
1727
+ if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) {
1728
+ debug$3(`${this.sessionId} - turnAudio: skip turn (start=${startSec} end=${endSec} reason=${reason})`);
1729
+ t.emitted = true;
1730
+ if (lodash__default["default"].isFinite(endSec)) this._turnAudioPrevEndSec = endSec;
1731
+ continue;
1732
+ }
1733
+ const audioBase64 = this._sliceTurnAudio(startSec, endSec);
1734
+ t.emitted = true;
1735
+ this._turnAudioPrevEndSec = endSec;
1736
+ if (!audioBase64) continue;
1737
+ this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
1738
+ this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
1739
+ name: `turn_${this._turnAudioCounter}.wav`,
1740
+ mimeType: 'audio/wav',
1741
+ base64: audioBase64,
1742
+ meTurnIndex: t.meTurnIndex,
1743
+ // 0-based real-user-turn ordinal (authoritative for placement)
1744
+ sessionContext: {
1745
+ testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
1746
+ testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
1747
+ }
1748
+ });
1749
+ _info('turn_audio_emitted', {
1750
+ sessionId: this.sessionId,
1751
+ name: `turn_${this._turnAudioCounter}.wav`,
1752
+ meTurnIndex: t.meTurnIndex,
1753
+ startSec: Number(startSec.toFixed(2)),
1754
+ endSec: Number(endSec.toFixed(2)),
1755
+ reason
1756
+ });
1757
+ } catch (err) {
1758
+ debug$3(`${this.sessionId} - emitReadyTurnAudio: turn slice error: ${err && err.message}`);
1759
+ }
1760
+ }
1761
+ }
1762
+
1763
+ /**
1764
+ * Force-emit any per-turn audio not yet sent (session end). Idempotent.
1765
+ */
1766
+ _flushPendingTurnAudio(reason) {
1767
+ if (this._turnAudioForceDone) return;
1768
+ this._turnAudioForceDone = true;
1769
+ const pending = this._pendingTurnAudio || [];
1770
+ _info('turn_audio_flush_enter', {
1771
+ sessionId: this.sessionId,
1772
+ reason,
1773
+ pending: pending.length,
1774
+ emittedAlready: pending.filter(t => t.emitted).length,
1775
+ hasFormat: !!(this.audioStream && this.audioStream.format),
1776
+ totalBytes: this.audioStream && this.audioStream.totalBytes,
1777
+ turnAudioEnable: !!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]
1778
+ });
1779
+ this._emitReadyTurnAudio(reason, true);
1780
+ this._emitTrailingBotAudio(reason);
1781
+ _info('turn_audio_flush_done', {
1782
+ sessionId: this.sessionId,
1783
+ reason,
1784
+ emitted: this._turnAudioCounter || 0,
1785
+ pending: pending.length
1786
+ });
1787
+ }
1788
+
1789
+ /**
1790
+ * If the call ended on a bot turn (a bot message arrived with no me-response after it),
1791
+ * `_lastBotTurnStartSec` was never consumed by a turn. Slice that trailing bot audio
1792
+ * (bot start → end of recording) and emit it as a turn clip flagged `trailingBot` so the
1793
+ * UI/server place it on the final bot step.
1794
+ */
1795
+ _emitTrailingBotAudio(reason) {
1796
+ if (!this.caps[Capabilities.VOIP_TURN_AUDIO_ENABLE]) return;
1797
+ if (this._trailingBotAudioEmitted) return;
1798
+ const startSec = lodash__default["default"].isFinite(this._lastBotTurnStartSec) ? this._lastBotTurnStartSec : null;
1799
+ if (!lodash__default["default"].isFinite(startSec)) return;
1800
+ if (!this.audioStream || !this.audioStream.format) return;
1801
+ const endSec = this._recordingSecNow();
1802
+ if (!lodash__default["default"].isFinite(endSec) || endSec <= startSec) return;
1803
+ let audioBase64 = null;
1804
+ try {
1805
+ audioBase64 = this._sliceTurnAudio(startSec, endSec);
1806
+ } catch (err) {
1807
+ debug$3(`${this.sessionId} - emitTrailingBotAudio: slice error: ${err && err.message}`);
1808
+ return;
1809
+ }
1810
+ if (!audioBase64) return;
1811
+ this._trailingBotAudioEmitted = true;
1812
+ this._lastBotTurnStartSec = null;
1813
+ this._turnAudioCounter = (this._turnAudioCounter || 0) + 1;
1814
+ this.eventEmitter.emit('MESSAGE_ATTACHMENT', this.container, {
1815
+ name: `turn_${this._turnAudioCounter}.wav`,
1816
+ mimeType: 'audio/wav',
1817
+ base64: audioBase64,
1818
+ trailingBot: true,
1819
+ // place on the final bot step (no me-response followed)
1820
+ sessionContext: {
1821
+ testSessionId: this.caps.VOIP_TEST_SESSION_ID || null,
1822
+ testSessionJobId: this.caps.VOIP_TEST_SESSION_JOB_ID || null
1823
+ }
1824
+ });
1825
+ _info('turn_audio_trailing_emitted', {
1826
+ sessionId: this.sessionId,
1827
+ name: `turn_${this._turnAudioCounter}.wav`,
1828
+ startSec: Number(startSec.toFixed(2)),
1829
+ endSec: Number(endSec.toFixed(2)),
1830
+ reason
1831
+ });
1832
+ }
1833
+ _agentSpeechRmsThreshold() {
1834
+ const raw = process.env.VOIP_AGENT_SPEECH_RMS_THRESHOLD;
1835
+ const n = raw != null ? Number(raw) : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
1836
+ return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_RMS_THRESHOLD;
1837
+ }
1838
+ _agentSpeechSustainedWindows() {
1839
+ const raw = process.env.VOIP_AGENT_SPEECH_SUSTAINED_WINDOWS;
1840
+ const n = raw != null ? parseInt(raw, 10) : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
1841
+ return Number.isFinite(n) && n > 0 ? n : DEFAULT_AGENT_SPEECH_SUSTAINED_WINDOWS;
1842
+ }
1843
+ _audioStreamBytesPerSec() {
1844
+ const fmt = this.audioStream && this.audioStream.format;
1845
+ if (!fmt) return null;
1846
+ return fmt.sampleRate * fmt.channels * (fmt.bitsPerSample / 8);
1847
+ }
1848
+ _pcmBufferRms(pcm, bitsPerSample) {
1849
+ if (!pcm || pcm.length < 2 || bitsPerSample !== 16) return 0;
1850
+ let sum = 0;
1851
+ let count = 0;
1852
+ for (let i = 0; i + 1 < pcm.length; i += 2) {
1853
+ const sample = pcm.readInt16LE(i);
1854
+ sum += sample * sample;
1855
+ count += 1;
1856
+ }
1857
+ return count > 0 ? Math.sqrt(sum / count) : 0;
1858
+ }
1859
+ _readWavPcmInfo(wavBuffer) {
1860
+ if (!wavBuffer || wavBuffer.length < 44) return null;
1861
+ if (wavBuffer.toString('ascii', 0, 4) !== 'RIFF' || wavBuffer.toString('ascii', 8, 12) !== 'WAVE') {
1862
+ return null;
1863
+ }
1864
+ let offset = 12;
1865
+ let sampleRate = null;
1866
+ let channels = null;
1867
+ let bitsPerSample = null;
1868
+ let dataOffset = null;
1869
+ let dataLength = null;
1870
+ while (offset + 8 <= wavBuffer.length) {
1871
+ const chunkId = wavBuffer.toString('ascii', offset, offset + 4);
1872
+ const chunkSize = wavBuffer.readUInt32LE(offset + 4);
1873
+ const chunkStart = offset + 8;
1874
+ if (chunkId === 'fmt ' && chunkSize >= 16) {
1875
+ channels = wavBuffer.readUInt16LE(chunkStart + 2);
1876
+ sampleRate = wavBuffer.readUInt32LE(chunkStart + 4);
1877
+ bitsPerSample = wavBuffer.readUInt16LE(chunkStart + 14);
1878
+ } else if (chunkId === 'data') {
1879
+ dataOffset = chunkStart;
1880
+ dataLength = chunkSize;
1881
+ break;
1882
+ }
1883
+ offset = chunkStart + chunkSize + chunkSize % 2;
1884
+ }
1885
+ if (!sampleRate || !channels || !bitsPerSample || dataOffset == null) return null;
1886
+ const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
1887
+ if (!bytesPerSec) return null;
1888
+ const pcmLength = dataLength != null ? Math.min(dataLength, wavBuffer.length - dataOffset) : wavBuffer.length - dataOffset;
1889
+ return {
1890
+ pcmOffset: dataOffset,
1891
+ pcmLength,
1892
+ bytesPerSec,
1893
+ bitsPerSample,
1894
+ sampleRate,
1895
+ channels
1896
+ };
1897
+ }
1898
+ _findAudibleLeadInSecFromPcm(pcm, bytesPerSec, bitsPerSample) {
1899
+ if (!pcm || !bytesPerSec) return null;
1900
+ const threshold = this._agentSpeechRmsThreshold();
1901
+ const sustainedWindows = this._agentSpeechSustainedWindows();
1902
+ const windowBytes = Math.max(2, Math.floor(bytesPerSec * (AGENT_SPEECH_RMS_WINDOW_MS / 1000)));
1903
+ const hopBytes = Math.max(2, Math.floor(windowBytes / 2));
1904
+ let streak = 0;
1905
+ let onsetPos = null;
1906
+ for (let pos = 0; pos + windowBytes <= pcm.length; pos += hopBytes) {
1907
+ const rms = this._pcmBufferRms(pcm.subarray(pos, pos + windowBytes), bitsPerSample);
1908
+ if (rms >= threshold) {
1909
+ if (streak === 0) onsetPos = pos;
1910
+ streak += 1;
1911
+ if (streak >= sustainedWindows) {
1912
+ return onsetPos / bytesPerSec;
1913
+ }
1914
+ } else {
1915
+ streak = 0;
1916
+ onsetPos = null;
1917
+ }
1918
+ }
1919
+ return null;
1920
+ }
1921
+ _findAudibleLeadInSecFromWavBuffer(wavBuffer) {
1922
+ const info = this._readWavPcmInfo(wavBuffer);
1923
+ if (!info) return null;
1924
+ const pcm = wavBuffer.subarray(info.pcmOffset, info.pcmOffset + info.pcmLength);
1925
+ return this._findAudibleLeadInSecFromPcm(pcm, info.bytesPerSec, info.bitsPerSample);
1926
+ }
1927
+ _findAudibleRecordingStartSecOnStream(playedSec, wireSec) {
1928
+ if (!lodash__default["default"].isFinite(playedSec)) return null;
1929
+ const bytesPerSec = this._audioStreamBytesPerSec();
1930
+ const stream = this.audioStream;
1931
+ if (!bytesPerSec || !stream || !stream.pcmParts.length) return null;
1932
+ const startByte = Math.max(0, Math.floor(playedSec * bytesPerSec));
1933
+ const pcm = Buffer.concat(stream.pcmParts);
1934
+ if (startByte >= pcm.length) return null;
1935
+ const bitsPerSample = stream.format.bitsPerSample;
1936
+ const leadInSec = this._findAudibleLeadInSecFromPcm(pcm.subarray(startByte), bytesPerSec, bitsPerSample);
1937
+ if (!lodash__default["default"].isFinite(leadInSec)) return null;
1938
+ let heardSec = playedSec + leadInSec;
1939
+ if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
1940
+ return heardSec;
1941
+ }
1942
+ _findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments) {
1943
+ if (!lodash__default["default"].isFinite(playedSec) || !lodash__default["default"].isArray(attachments)) return null;
1944
+ const tts = attachments.find(a => a && a.name === 'tts.wav' && a.base64);
1945
+ if (!tts) return null;
1946
+ try {
1947
+ const wavBuffer = Buffer.from(tts.base64, 'base64');
1948
+ const leadInSec = this._findAudibleLeadInSecFromWavBuffer(wavBuffer);
1949
+ if (!lodash__default["default"].isFinite(leadInSec)) return null;
1950
+ let heardSec = playedSec + leadInSec;
1951
+ if (lodash__default["default"].isFinite(wireSec)) heardSec = Math.max(wireSec, heardSec);
1952
+ return heardSec;
1953
+ } catch (err) {
1954
+ debug$3(`${this.sessionId} - TTS lead-in scan failed: ${err && err.message}`);
1955
+ return null;
1956
+ }
1957
+ }
1958
+ _resolveAgentHeardRecordingStartSec(voipAgent, attachments) {
1959
+ if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
1960
+ const playedSec = voipAgent.playedRecordingStartSec;
1961
+ const wireSec = voipAgent.wireRecordingStartSec;
1962
+ const fromTts = this._findAudibleRecordingStartSecFromAttachments(playedSec, wireSec, attachments);
1963
+ const fromStream = this._findAudibleRecordingStartSecOnStream(playedSec, wireSec);
1964
+ const candidates = [fromTts, fromStream].filter(s => lodash__default["default"].isFinite(s));
1965
+ if (!candidates.length) return null;
1966
+ // Prefer the later onset — mixed recording can spike before clear TTS speech.
1967
+ return Math.max(...candidates);
1968
+ }
1969
+ _applyAgentHeardRecordingStartSec(voipAgent, attachments) {
1970
+ if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return null;
1971
+ const heardSec = this._resolveAgentHeardRecordingStartSec(voipAgent, attachments);
1972
+ if (!lodash__default["default"].isFinite(heardSec) || heardSec <= voipAgent.playedRecordingStartSec) {
1973
+ return lodash__default["default"].isFinite(voipAgent.heardRecordingStartSec) ? voipAgent.heardRecordingStartSec : null;
1974
+ }
1975
+ const prev = voipAgent.heardRecordingStartSec;
1976
+ if (lodash__default["default"].isFinite(prev) && prev >= heardSec) return prev;
1977
+ voipAgent.heardRecordingStartSec = heardSec;
1978
+ this._markReplyTrace({
1979
+ heardRecordingStartSec: heardSec
1980
+ });
1981
+ return heardSec;
1982
+ }
1983
+ _maybeDetectAgentAudibleOnRecording(voipAgent) {
1984
+ if (!voipAgent || !lodash__default["default"].isFinite(voipAgent.playedRecordingStartSec)) return;
1985
+ this._applyAgentHeardRecordingStartSec(voipAgent);
1986
+ }
1987
+ _markReplyTrace(patch) {
1988
+ if (!this._replyTrace || !patch) return;
1989
+ Object.assign(this._replyTrace, patch);
1990
+ }
1991
+ _captureSttFinalForReplyTrace(parsedData, msgPreview) {
1992
+ const data = parsedData && parsedData.data;
1993
+ const atMs = parsedData._receivedAtMs || Date.now();
1994
+ const recordingAtSttFinalSec = this._recordingSecNow();
1995
+ if (parsedData && lodash__default["default"].isFinite(recordingAtSttFinalSec)) {
1996
+ parsedData.recordingAtSttFinalSec = recordingAtSttFinalSec;
1997
+ }
1998
+ // Dedicated, self-documenting anchor for the downstream "STT transport" sub-phase
1999
+ // (receivedAtMs - finalEmittedWallMs). Unlike the generic _receivedAtMs (stamped on
2000
+ // every WS frame), this is set only on the accepted STT-final.
2001
+ if (parsedData && lodash__default["default"].isFinite(atMs)) {
2002
+ parsedData.sttFinalReceivedAtMs = atMs;
2003
+ }
2004
+ this._replyTrace = {
2005
+ sessionId: this.sessionId,
2006
+ botMessagePreview: msgPreview || undefined,
2007
+ sttFinalAtMs: atMs,
2008
+ sttRecordingStartSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'start')) ? data.start : null,
2009
+ sttRecordingEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'end')) ? data.end : null,
2010
+ sttSpeechEndSec: lodash__default["default"].isFinite(lodash__default["default"].get(data, 'speechEndSec')) ? data.speechEndSec : null,
2011
+ recordingAtSttFinalSec: lodash__default["default"].isFinite(recordingAtSttFinalSec) ? recordingAtSttFinalSec : null,
2012
+ queueAtMs: null,
2013
+ recordingAtQueueSec: null,
2014
+ psstTimerArmedAtMs: null,
2015
+ psstScheduledMs: null,
2016
+ psstTimerFiredAtMs: null,
2017
+ psstFireDelayMs: null,
2018
+ userSaysAtMs: null,
2019
+ coachWaitMs: null,
2020
+ ttsStartAtMs: null,
2021
+ ttsEndAtMs: null,
2022
+ ttsSynthMs: null,
2023
+ wireAtMs: null,
2024
+ wireRecordingStartSec: null,
2025
+ sendAudioAtMs: null,
2026
+ playedRecordingStartSec: null,
2027
+ playbackAtMs: null,
2028
+ heardRecordingStartSec: null,
2029
+ agentEndRecordingSec: null,
2030
+ wireKind: null,
2031
+ inputType: null,
2032
+ requestedDurationMs: null,
2033
+ meMessagePreview: null
2034
+ };
2035
+ }
2036
+ _captureBotQueuedForReplyTrace(queuedAt) {
2037
+ if (!this._replyTrace) return;
2038
+ this._replyTrace.queueAtMs = queuedAt;
2039
+ this._replyTrace.recordingAtQueueSec = this._recordingSecNow();
2040
+ }
2041
+ _captureUserSaysStart(msgPreview) {
2042
+ if (!this._replyTrace) return;
2043
+ const now = Date.now();
2044
+ this._replyTrace.userSaysAtMs = now;
2045
+ this._replyTrace.meMessagePreview = msgPreview || undefined;
2046
+ const queueAt = this._replyTrace.queueAtMs || this._lastBotSaysQueuedAt;
2047
+ if (lodash__default["default"].isFinite(queueAt)) {
2048
+ if (!this._replyTrace.queueAtMs) this._replyTrace.queueAtMs = queueAt;
2049
+ this._replyTrace.coachWaitMs = now - queueAt;
2050
+ }
2051
+ }
2052
+ _captureAgentWire(voipAgent, inputType) {
2053
+ if (!this._replyTrace || !voipAgent) return;
2054
+ this._replyTrace.wireAtMs = voipAgent.wireSentAtMs;
2055
+ this._replyTrace.wireRecordingStartSec = voipAgent.wireRecordingStartSec;
2056
+ this._replyTrace.wireKind = voipAgent.wireKind;
2057
+ this._replyTrace.inputType = inputType;
2058
+ this._replyTrace.requestedDurationMs = voipAgent.requestedDurationMs;
2059
+ if (lodash__default["default"].isFinite(voipAgent.ttsSynthMs)) this._replyTrace.ttsSynthMs = voipAgent.ttsSynthMs;
2060
+ }
2061
+ _finalizeWallPipeline(voipAgent) {
2062
+ const t = this._replyTrace;
2063
+ if (!voipAgent || !t) return;
2064
+ voipAgent.wallPipeline = {
2065
+ psstScheduledMs: lodash__default["default"].isFinite(t.psstScheduledMs) ? t.psstScheduledMs : null,
2066
+ psstFireDelayMs: lodash__default["default"].isFinite(t.psstFireDelayMs) ? t.psstFireDelayMs : null,
2067
+ coachWaitMs: lodash__default["default"].isFinite(t.coachWaitMs) ? t.coachWaitMs : null,
2068
+ userSaysAtMs: lodash__default["default"].isFinite(t.userSaysAtMs) ? t.userSaysAtMs : null,
2069
+ ttsStartAtMs: lodash__default["default"].isFinite(t.ttsStartAtMs) ? t.ttsStartAtMs : null,
2070
+ ttsEndAtMs: lodash__default["default"].isFinite(t.ttsEndAtMs) ? t.ttsEndAtMs : null,
2071
+ ttsSynthMs: lodash__default["default"].isFinite(t.ttsSynthMs) ? t.ttsSynthMs : null,
2072
+ wireAtMs: lodash__default["default"].isFinite(t.wireAtMs) ? t.wireAtMs : null,
2073
+ sendAudioAtMs: lodash__default["default"].isFinite(t.sendAudioAtMs) ? t.sendAudioAtMs : null
2074
+ };
2075
+ }
2076
+ _replyTraceMsFromSttFinal(atMs) {
2077
+ const anchor = this._replyTrace && this._replyTrace.sttFinalAtMs;
2078
+ if (!lodash__default["default"].isFinite(anchor) || !lodash__default["default"].isFinite(atMs)) return null;
2079
+ return Math.round(atMs - anchor);
2080
+ }
2081
+ _replyTraceRecMs(fromSec, toSec) {
2082
+ if (!lodash__default["default"].isFinite(fromSec) || !lodash__default["default"].isFinite(toSec)) return null;
2083
+ return Math.round((toSec - fromSec) * 1000);
2084
+ }
2085
+ _logReplyTrace(trigger) {
2086
+ const t = this._replyTrace;
2087
+ if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
2088
+ _info('voip_reply_trace', {
2089
+ sessionId: t.sessionId,
2090
+ trigger,
2091
+ botPreview: t.botMessagePreview,
2092
+ mePreview: t.meMessagePreview,
2093
+ sttRecordingStartSec: t.sttRecordingStartSec,
2094
+ sttRecordingEndSec: t.sttRecordingEndSec,
2095
+ sttSpeechEndSec: t.sttSpeechEndSec,
2096
+ recordingAtSttFinalSec: t.recordingAtSttFinalSec,
2097
+ recordingAtQueueSec: t.recordingAtQueueSec,
2098
+ wireRecordingStartSec: t.wireRecordingStartSec,
2099
+ wireKind: t.wireKind,
2100
+ inputType: t.inputType,
2101
+ requestedDurationMs: t.requestedDurationMs,
2102
+ ttsSynthMs: t.ttsSynthMs,
2103
+ coachWaitMs: t.coachWaitMs,
2104
+ psstScheduledMs: t.psstScheduledMs,
2105
+ psstFireDelayMs: t.psstFireDelayMs,
2106
+ ms_sttFinal_to_queue: this._replyTraceMsFromSttFinal(t.queueAtMs),
2107
+ ms_sttFinal_to_psstFire: this._replyTraceMsFromSttFinal(t.psstTimerFiredAtMs),
2108
+ ms_sttFinal_to_userSays: this._replyTraceMsFromSttFinal(t.userSaysAtMs),
2109
+ ms_sttFinal_to_ttsStart: this._replyTraceMsFromSttFinal(t.ttsStartAtMs),
2110
+ ms_sttFinal_to_ttsEnd: this._replyTraceMsFromSttFinal(t.ttsEndAtMs),
2111
+ ms_sttFinal_to_wire: this._replyTraceMsFromSttFinal(t.wireAtMs),
2112
+ ms_sttFinal_to_sendAudio: this._replyTraceMsFromSttFinal(t.sendAudioAtMs),
2113
+ ms_userSays_to_ttsStart: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.ttsStartAtMs) ? Math.round(t.ttsStartAtMs - t.userSaysAtMs) : null,
2114
+ ms_userSays_to_wire: lodash__default["default"].isFinite(t.userSaysAtMs) && lodash__default["default"].isFinite(t.wireAtMs) ? Math.round(t.wireAtMs - t.userSaysAtMs) : null,
2115
+ ms_queue_to_userSays: t.coachWaitMs,
2116
+ recMs_sttEnd_to_queue: this._replyTraceRecMs(t.sttRecordingEndSec, t.recordingAtQueueSec),
2117
+ recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
2118
+ recMs_speechEnd_to_wire: this._replyTraceRecMs(t.sttSpeechEndSec, t.wireRecordingStartSec)
2119
+ });
2120
+ }
2121
+ _logReplyTraceHeard(agentEndRecordingSec) {
2122
+ const t = this._replyTrace;
2123
+ if (!t || !lodash__default["default"].isFinite(t.sttFinalAtMs)) return;
2124
+ const heardSec = t.heardRecordingStartSec;
2125
+ const playedSec = t.playedRecordingStartSec;
2126
+ if (lodash__default["default"].isFinite(agentEndRecordingSec)) {
2127
+ t.agentEndRecordingSec = agentEndRecordingSec;
2128
+ }
2129
+ _info('voip_reply_trace_heard', {
2130
+ sessionId: t.sessionId,
2131
+ playedRecordingStartSec: playedSec,
2132
+ heardRecordingStartSec: heardSec,
2133
+ agentEndRecordingSec: t.agentEndRecordingSec,
2134
+ wireRecordingStartSec: t.wireRecordingStartSec,
2135
+ sttRecordingEndSec: t.sttRecordingEndSec,
2136
+ sttSpeechEndSec: t.sttSpeechEndSec,
2137
+ recMs_sttEnd_to_played: this._replyTraceRecMs(t.sttRecordingEndSec, playedSec),
2138
+ recMs_speechEnd_to_played: this._replyTraceRecMs(t.sttSpeechEndSec, playedSec),
2139
+ recMs_sttEnd_to_heard: this._replyTraceRecMs(t.sttRecordingEndSec, heardSec),
2140
+ recMs_sttEnd_to_wire: this._replyTraceRecMs(t.sttRecordingEndSec, t.wireRecordingStartSec),
2141
+ recMs_wire_to_played: this._replyTraceRecMs(t.wireRecordingStartSec, playedSec),
2142
+ recMs_wire_to_heard: this._replyTraceRecMs(t.wireRecordingStartSec, heardSec)
2143
+ });
2144
+ this._replyTrace = null;
2145
+ }
1269
2146
  _voipWsCanSend() {
1270
2147
  return !this.stopCalled && this.ws && this.ws.readyState === ws__default["default"].OPEN;
1271
2148
  }
@@ -1318,6 +2195,8 @@ class BotiumConnectorVoip {
1318
2195
  if (typeof this._emitBufferedFullRecordIfAny === 'function') {
1319
2196
  this._emitBufferedFullRecordIfAny('stop_final_guard');
1320
2197
  }
2198
+ // Last-resort flush in case neither audioStreamEnd nor fullRecordEnd fired.
2199
+ this._flushPendingTurnAudio('stop_final_guard');
1321
2200
  }
1322
2201
  this._emitBufferedFullRecordIfAny = null;
1323
2202
  }
@@ -1627,6 +2506,21 @@ class BotiumConnectorVoip {
1627
2506
  if (!lodash__default["default"].isFinite(parsed) || parsed <= 0) return null;
1628
2507
  return isPsst ? Math.max(0, parsed - 500) : parsed;
1629
2508
  }
2509
+
2510
+ // Extra grace (ms) added to the JOIN/PSST flush window when the last buffered final was cut
2511
+ // mid-utterance. Returns 0 when unset/invalid (feature off). See _isLastFinalNaturalEnd.
2512
+ _getPsstLatencyGraceMs() {
2513
+ const parsed = parseInt(this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING_LATENCY_GRACE_MS], 10);
2514
+ if (!lodash__default["default"].isFinite(parsed) || parsed <= 0) return 0;
2515
+ return parsed;
2516
+ }
2517
+
2518
+ // A final that carries a finite silenceStartedSec ended on Azure-detected end-of-speech silence
2519
+ // (bot genuinely stopped) => flush at the base window, no grace. A final without it was cut
2520
+ // mid-utterance (forced/segmented) => a continuation is plausible => apply the latency grace.
2521
+ _isLastFinalNaturalEnd() {
2522
+ return lodash__default["default"].isFinite(lodash__default["default"].get(this.prevData, 'data.silenceStartedSec'));
2523
+ }
1630
2524
  _getEffectiveJoinTimeoutMs(convoStep, botMsgs) {
1631
2525
  const sttHandling = this.caps[Capabilities.VOIP_STT_MESSAGE_HANDLING];
1632
2526
  const isPsst = sttHandling === 'PSST';
@@ -1662,6 +2556,100 @@ class BotiumConnectorVoip {
1662
2556
  }
1663
2557
  return null;
1664
2558
  }
2559
+
2560
+ /**
2561
+ * Build a well-formed WAV Buffer from raw PCM bytes and a format descriptor.
2562
+ * @param {Buffer} pcm raw PCM bytes (no header)
2563
+ * @param {{ sampleRate: number, channels: number, bitsPerSample: number }} fmt
2564
+ * @returns {Buffer}
2565
+ */
2566
+ _buildWavBuffer(pcm, fmt) {
2567
+ const {
2568
+ sampleRate,
2569
+ channels,
2570
+ bitsPerSample
2571
+ } = fmt;
2572
+ const byteRate = sampleRate * channels * (bitsPerSample / 8);
2573
+ const blockAlign = channels * (bitsPerSample / 8);
2574
+ const dataSize = pcm.length;
2575
+ const header = Buffer.alloc(44);
2576
+ header.write('RIFF', 0);
2577
+ header.writeUInt32LE(36 + dataSize, 4);
2578
+ header.write('WAVE', 8);
2579
+ header.write('fmt ', 12);
2580
+ header.writeUInt32LE(16, 16); // fmt chunk size
2581
+ header.writeUInt16LE(1, 20); // PCM format
2582
+ header.writeUInt16LE(channels, 22);
2583
+ header.writeUInt32LE(sampleRate, 24);
2584
+ header.writeUInt32LE(byteRate, 28);
2585
+ header.writeUInt16LE(blockAlign, 32);
2586
+ header.writeUInt16LE(bitsPerSample, 34);
2587
+ header.write('data', 36);
2588
+ header.writeUInt32LE(dataSize, 40);
2589
+ return Buffer.concat([header, pcm]);
2590
+ }
2591
+
2592
+ /**
2593
+ * Slice a segment of the continuously buffered PCM audio stream and return
2594
+ * it as a base64-encoded WAV string.
2595
+ *
2596
+ * @param {number} startSec start of the segment (seconds from call connect)
2597
+ * @param {number} endSec end of the segment (seconds from call connect)
2598
+ * @returns {string|null} base64 WAV or null if the stream is not ready
2599
+ */
2600
+ _sliceTurnAudio(startSec, endSec) {
2601
+ const stream = this.audioStream;
2602
+ if (!stream || !stream.format || !stream.pcmParts || !stream.pcmParts.length) return null;
2603
+ if (!lodash__default["default"].isFinite(startSec) || !lodash__default["default"].isFinite(endSec) || endSec <= startSec) return null;
2604
+ const {
2605
+ sampleRate,
2606
+ channels,
2607
+ bitsPerSample
2608
+ } = stream.format;
2609
+ const bytesPerSec = sampleRate * channels * (bitsPerSample / 8);
2610
+ const frameBytes = channels * (bitsPerSample / 8);
2611
+ const offsetSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_OFFSET_MS] || 0) / 1000;
2612
+ const paddingSec = (this.caps[Capabilities.VOIP_TURN_AUDIO_PADDING_MS] || 0) / 1000;
2613
+ const adjStart = Math.max(0, startSec + offsetSec);
2614
+ const adjEnd = endSec + paddingSec;
2615
+
2616
+ // Frame-align the byte boundaries.
2617
+ const startByte = Math.floor(adjStart * bytesPerSec / frameBytes) * frameBytes;
2618
+ const endByte = Math.ceil(adjEnd * bytesPerSec / frameBytes) * frameBytes;
2619
+ if (startByte >= stream.totalBytes) {
2620
+ debug$3(`${this.sessionId} - _sliceTurnAudio: startByte ${startByte} >= totalBytes ${stream.totalBytes}, skipping`);
2621
+ return null;
2622
+ }
2623
+ const clampedEnd = Math.min(endByte, stream.totalBytes);
2624
+ const sliceLen = clampedEnd - startByte;
2625
+ if (sliceLen <= 0) return null;
2626
+
2627
+ // Materialise only the bytes we need from the part list.
2628
+ const pcm = Buffer.allocUnsafe(sliceLen);
2629
+ let written = 0;
2630
+ let offset = 0;
2631
+ for (const part of stream.pcmParts) {
2632
+ const partEnd = offset + part.length;
2633
+ if (partEnd <= startByte) {
2634
+ offset += part.length;
2635
+ continue;
2636
+ }
2637
+ if (offset >= clampedEnd) break;
2638
+ const copyFrom = Math.max(0, startByte - offset);
2639
+ const copyTo = Math.min(part.length, clampedEnd - offset);
2640
+ part.copy(pcm, written, copyFrom, copyTo);
2641
+ written += copyTo - copyFrom;
2642
+ offset += part.length;
2643
+ }
2644
+ if (written === 0) return null;
2645
+ const slicedPcm = written < sliceLen ? pcm.slice(0, written) : pcm;
2646
+ const wavBuf = this._buildWavBuffer(slicedPcm, {
2647
+ sampleRate,
2648
+ channels,
2649
+ bitsPerSample
2650
+ });
2651
+ return wavBuf.toString('base64');
2652
+ }
1665
2653
  }
1666
2654
  var connector = BotiumConnectorVoip;
1667
2655
 
@@ -1745,6 +2733,24 @@ var botiumConnectorVoip = {
1745
2733
  type: 'boolean',
1746
2734
  required: false,
1747
2735
  advanced: true
2736
+ }, {
2737
+ name: 'VOIP_TURN_AUDIO_ENABLE',
2738
+ label: 'Attach per-turn audio to each transcript message',
2739
+ type: 'boolean',
2740
+ required: false,
2741
+ advanced: true
2742
+ }, {
2743
+ name: 'VOIP_TURN_AUDIO_PADDING_MS',
2744
+ label: 'Extra milliseconds appended after each turn audio slice (absorbs STT boundary jitter)',
2745
+ type: 'int',
2746
+ required: false,
2747
+ advanced: true
2748
+ }, {
2749
+ name: 'VOIP_TURN_AUDIO_OFFSET_MS',
2750
+ label: 'Millisecond offset applied to every turn audio start time (positive = shift right)',
2751
+ type: 'int',
2752
+ required: false,
2753
+ advanced: true
1748
2754
  }]
1749
2755
  },
1750
2756
  PluginLogicHooks: {