@openclaw/voice-call 2026.10.2 → 2026.10.5-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -3
- package/dist/.setup/{guarded-json-api-x55yqHvW.mjs → carrier-api-BYKw_fsQ.mjs} +94 -90
- package/dist/.setup/{cli-Cn1leykM.mjs → cli-dyyK8Upw.mjs} +66 -84
- package/dist/.setup/{config-CX9J-053.mjs → config-CsrGuamE.mjs} +95 -69
- package/dist/.setup/{mock-eLv_Dc4o.mjs → mock-rySvj-G6.mjs} +12 -3
- package/dist/.setup/{plivo-CpPIENu4.mjs → plivo-COfD-Kuc.mjs} +35 -89
- package/dist/.setup/{realtime-handler-Do7Sz7Gh.mjs → realtime-handler-BdAuKAC2.mjs} +749 -464
- package/dist/.setup/{response-generator-DAhbXEta.mjs → response-generator-COu0_8-d.mjs} +14 -32
- package/dist/.setup/{runtime-entry-CbSKxSKk.mjs → runtime-entry-STjFmi_B.mjs} +1053 -446
- package/dist/.setup/{telnyx-DPilutCH.mjs → telnyx-B4P0peEe.mjs} +30 -61
- package/dist/.setup/{twilio-BWFNGyVx.mjs → twilio-BzAYgH7M.mjs} +75 -90
- package/dist/.setup/{voice-mapping-B_iAUkER.mjs → voice-mapping-1lzFmzOF.mjs} +0 -10
- package/dist/cli-output-mode.js +1 -0
- package/dist/doctor-contract-api.js +19 -23
- package/dist/index.js +212 -89
- package/dist/runtime-entry.js +1 -1
- package/dist/setup-api.js +7 -6
- package/openclaw.plugin.json +210 -2
- package/package.json +4 -4
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { d as REALTIME_VOICE_END_CALL_TOOL_NAME, s as resolveVoiceCallPublicPathPrefix } from "./config-
|
|
2
|
-
import {
|
|
1
|
+
import { d as REALTIME_VOICE_END_CALL_TOOL_NAME, g as CallBriefSchema, s as resolveVoiceCallPublicPathPrefix } from "./config-CsrGuamE.mjs";
|
|
2
|
+
import { n as appendRecentTalkEventMetadata, s as canonicalizeVoiceCallMediaBase64 } from "./runtime-entry-STjFmi_B.mjs";
|
|
3
3
|
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
|
4
4
|
import { asNullableRecord, asOptionalObjectRecord, asOptionalRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
5
|
-
import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env";
|
|
6
|
-
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, buildRealtimeVoiceAgentConsultWorkingResponse, buildRealtimeVoiceAgentErrorProviderResult, calculateMulawRms, createRealtimeVoiceSessionHarness, createSpeechThresholdGate, readRealtimeVoiceConsultQuestion, readSpeakableRealtimeVoiceToolResult, resolveRealtimeVoiceBargeIn, resolveRealtimeVoiceSessionPolicy } from "openclaw/plugin-sdk/realtime-voice";
|
|
5
|
+
import { createSubsystemLogger, sleep } from "openclaw/plugin-sdk/runtime-env";
|
|
6
|
+
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ, buildRealtimeVoiceAgentConsultWorkingResponse, buildRealtimeVoiceAgentErrorProviderResult, buildRealtimeVoiceSpeakExactMessage, calculateMulawRms, createRealtimeVoiceSessionHarness, createSpeechThresholdGate, readRealtimeVoiceConsultQuestion, readSpeakableRealtimeVoiceToolResult, resolveRealtimeVoiceBargeIn, resolveRealtimeVoiceSessionPolicy } from "openclaw/plugin-sdk/realtime-voice";
|
|
7
7
|
import { normalizeWebhookPath } from "openclaw/plugin-sdk/webhook-ingress";
|
|
8
8
|
import { isFutureDateTimestampMs, resolveExpiresAtMsFromDurationMs } from "openclaw/plugin-sdk/number-runtime";
|
|
9
9
|
import { randomUUID } from "node:crypto";
|
|
@@ -114,8 +114,6 @@ var RealtimeAudioPacer = class {
|
|
|
114
114
|
clearAudio() {
|
|
115
115
|
if (this.closed) return 0;
|
|
116
116
|
const clearedAudioBytes = this.queuedAudioBytes;
|
|
117
|
-
this.clearTimer();
|
|
118
|
-
this.resetQueue();
|
|
119
117
|
this.resetPlaybackState();
|
|
120
118
|
this.params.send(this.params.serializer.serializeClear());
|
|
121
119
|
return clearedAudioBytes;
|
|
@@ -126,8 +124,6 @@ var RealtimeAudioPacer = class {
|
|
|
126
124
|
}
|
|
127
125
|
close() {
|
|
128
126
|
this.closed = true;
|
|
129
|
-
this.clearTimer();
|
|
130
|
-
this.resetQueue();
|
|
131
127
|
this.resetPlaybackState();
|
|
132
128
|
}
|
|
133
129
|
extendPlaybackSegment(itemId, durationMs) {
|
|
@@ -145,6 +141,8 @@ var RealtimeAudioPacer = class {
|
|
|
145
141
|
return segment;
|
|
146
142
|
}
|
|
147
143
|
resetPlaybackState() {
|
|
144
|
+
this.clearTimer();
|
|
145
|
+
this.resetQueue();
|
|
148
146
|
this.playbackSegments = [];
|
|
149
147
|
this.queuedAudioBytes = 0;
|
|
150
148
|
this.sentAudioMs = 0;
|
|
@@ -168,15 +166,8 @@ var RealtimeAudioPacer = class {
|
|
|
168
166
|
this.close();
|
|
169
167
|
this.params.onBackpressure?.();
|
|
170
168
|
}
|
|
171
|
-
get pendingQueueSize() {
|
|
172
|
-
return Math.max(0, this.queue.length - this.queueHead);
|
|
173
|
-
}
|
|
174
169
|
/** Take one queued item without shifting the remaining paced-audio backlog. */
|
|
175
170
|
takeNextItem() {
|
|
176
|
-
if (this.queueHead >= this.queue.length) {
|
|
177
|
-
this.resetQueue();
|
|
178
|
-
return;
|
|
179
|
-
}
|
|
180
171
|
const item = this.queue[this.queueHead];
|
|
181
172
|
this.queueHead += 1;
|
|
182
173
|
if (this.queueHead >= this.queue.length) this.resetQueue();
|
|
@@ -196,17 +187,17 @@ var RealtimeAudioPacer = class {
|
|
|
196
187
|
if (this.closed) return;
|
|
197
188
|
const now = performance.now();
|
|
198
189
|
this.streamClockMs ??= now;
|
|
199
|
-
while (this.
|
|
190
|
+
while (this.queueHead < this.queue.length && this.streamClockMs < now + LEAD_MS) {
|
|
200
191
|
const item = this.takeNextItem();
|
|
201
192
|
if (!item) break;
|
|
202
|
-
if (!(item.type === "audio" ? this.sendAudioItem(item) : this.sendMarkItem(item))) {
|
|
193
|
+
if (!(item.type === "audio" ? this.sendAudioItem(item) : this.sendMarkItem(item.name))) {
|
|
203
194
|
this.resetQueue();
|
|
204
195
|
this.queuedAudioBytes = 0;
|
|
205
196
|
this.streamClockMs = null;
|
|
206
197
|
return;
|
|
207
198
|
}
|
|
208
199
|
}
|
|
209
|
-
if (this.
|
|
200
|
+
if (this.queueHead === this.queue.length) {
|
|
210
201
|
this.streamClockMs = null;
|
|
211
202
|
return;
|
|
212
203
|
}
|
|
@@ -217,6 +208,7 @@ var RealtimeAudioPacer = class {
|
|
|
217
208
|
this.queuedAudioBytes = Math.max(0, this.queuedAudioBytes - item.chunk.length);
|
|
218
209
|
const sent = this.params.send(this.params.serializer.serializeMedia(item.chunk.toString("base64")));
|
|
219
210
|
if (sent) {
|
|
211
|
+
this.params.onAudioSent?.(item.chunk);
|
|
220
212
|
item.segment.sentMs += item.durationMs;
|
|
221
213
|
this.sentAudioMs += item.durationMs;
|
|
222
214
|
item.segment.lastSentEndMs = this.sentAudioMs;
|
|
@@ -225,17 +217,14 @@ var RealtimeAudioPacer = class {
|
|
|
225
217
|
if (!sent || item.segment.itemId === void 0 || this.sentAudioMs - this.lastPlaybackMarkMs < LEAD_MS) return sent;
|
|
226
218
|
this.lastPlaybackMarkMs = this.sentAudioMs;
|
|
227
219
|
this.playbackMarkSequence += 1;
|
|
228
|
-
return this.sendMarkItem({
|
|
229
|
-
type: "mark",
|
|
230
|
-
name: `${this.playbackMarkPrefix}-${this.playbackMarkSequence}`
|
|
231
|
-
});
|
|
220
|
+
return this.sendMarkItem(`${this.playbackMarkPrefix}-${this.playbackMarkSequence}`);
|
|
232
221
|
}
|
|
233
222
|
/** Send a queued mark frame and bind it to the playback prefix before it. */
|
|
234
|
-
sendMarkItem(
|
|
235
|
-
const sent = this.params.send(this.params.serializer.serializeMark(
|
|
223
|
+
sendMarkItem(name) {
|
|
224
|
+
const sent = this.params.send(this.params.serializer.serializeMark(name));
|
|
236
225
|
if (sent) {
|
|
237
226
|
this.markBoundaries.push({
|
|
238
|
-
name
|
|
227
|
+
name,
|
|
239
228
|
sentMs: this.sentAudioMs
|
|
240
229
|
});
|
|
241
230
|
if (this.markBoundaries.length > MAX_PENDING_MARK_BOUNDARIES) this.markBoundaries.shift();
|
|
@@ -244,6 +233,442 @@ var RealtimeAudioPacer = class {
|
|
|
244
233
|
}
|
|
245
234
|
};
|
|
246
235
|
//#endregion
|
|
236
|
+
//#region extensions/voice-call/src/webhook/realtime-call-session-control.ts
|
|
237
|
+
const OUTBOUND_GREETING_FALLBACK_MS = 3e3;
|
|
238
|
+
const REALTIME_MEDIA_INACTIVITY_TIMEOUT_MS = 3e4;
|
|
239
|
+
const REALTIME_DISCONNECT_HANGUP_GRACE_MS = 2e3;
|
|
240
|
+
const ASSISTANT_SPEECH_RMS_THRESHOLD = .035;
|
|
241
|
+
function speakOnRealtimeBridge(bridges, callId, instructions) {
|
|
242
|
+
const bridge = bridges.get(callId);
|
|
243
|
+
if (!bridge) return {
|
|
244
|
+
success: false,
|
|
245
|
+
error: "No active realtime bridge for call"
|
|
246
|
+
};
|
|
247
|
+
try {
|
|
248
|
+
bridge.triggerGreeting(instructions);
|
|
249
|
+
return { success: true };
|
|
250
|
+
} catch (error) {
|
|
251
|
+
return {
|
|
252
|
+
success: false,
|
|
253
|
+
error: formatErrorMessage(error)
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
function buildForcedConsultSpeechPrompt(result) {
|
|
258
|
+
return [
|
|
259
|
+
"Internal OpenClaw consult result is ready.",
|
|
260
|
+
"Do not call tools for this internal result.",
|
|
261
|
+
"Speak the following answer to the caller now, briefly and naturally:",
|
|
262
|
+
result
|
|
263
|
+
].join("\n");
|
|
264
|
+
}
|
|
265
|
+
function buildGreetingInstructions(baseInstructions, greeting) {
|
|
266
|
+
const trimmedGreeting = greeting?.trim();
|
|
267
|
+
if (!trimmedGreeting) return;
|
|
268
|
+
const intro = "Start the call by greeting the caller naturally. Include this greeting in your first spoken reply:";
|
|
269
|
+
return baseInstructions ? `${baseInstructions}\n\n${intro} "${trimmedGreeting}"` : `${intro} "${trimmedGreeting}"`;
|
|
270
|
+
}
|
|
271
|
+
function buildVerbatimGreetingInstructions(baseInstructions, greeting) {
|
|
272
|
+
const trimmedGreeting = greeting?.trim();
|
|
273
|
+
if (!trimmedGreeting) return;
|
|
274
|
+
const exactGreeting = [
|
|
275
|
+
"For your first spoken reply, the first words must be the exact Answer below, verbatim and in its original language, with nothing before or after it.",
|
|
276
|
+
"Then stop and listen.",
|
|
277
|
+
buildRealtimeVoiceSpeakExactMessage({
|
|
278
|
+
text: trimmedGreeting,
|
|
279
|
+
surfaceLabel: "the callee"
|
|
280
|
+
})
|
|
281
|
+
].join("\n");
|
|
282
|
+
return baseInstructions ? `${baseInstructions}\n\n${exactGreeting}` : exactGreeting;
|
|
283
|
+
}
|
|
284
|
+
function createOutboundGreetingController(params) {
|
|
285
|
+
const waitForAnsweringMachine = params.call?.direction === "outbound" && params.call.metadata?.voicemailManagedByHost && params.call.metadata?.mode !== "notify" ? () => params.call?.metadata?.answeredBy : void 0;
|
|
286
|
+
let speechMs = 0;
|
|
287
|
+
let silenceMs = 0;
|
|
288
|
+
let acknowledged = false;
|
|
289
|
+
let claimed = !params.enabled;
|
|
290
|
+
let closed = false;
|
|
291
|
+
let waitingForAmd = Boolean(waitForAnsweringMachine);
|
|
292
|
+
let machineDetected = false;
|
|
293
|
+
let readySession;
|
|
294
|
+
let timer;
|
|
295
|
+
const clearTimer = () => {
|
|
296
|
+
if (timer) {
|
|
297
|
+
clearTimeout(timer);
|
|
298
|
+
timer = void 0;
|
|
299
|
+
}
|
|
300
|
+
};
|
|
301
|
+
const triggerOpening = () => {
|
|
302
|
+
if (!closed && readySession && params.instructions && claim()) readySession.triggerGreeting(params.instructions);
|
|
303
|
+
};
|
|
304
|
+
const isBlocked = () => {
|
|
305
|
+
const answeredBy = waitForAnsweringMachine?.();
|
|
306
|
+
if (typeof answeredBy === "string" && (answeredBy.startsWith("machine_") || answeredBy === "fax")) {
|
|
307
|
+
machineDetected = true;
|
|
308
|
+
clearTimer();
|
|
309
|
+
}
|
|
310
|
+
if (machineDetected) return true;
|
|
311
|
+
if (waitingForAmd && (answeredBy === "human" || answeredBy === "unknown")) {
|
|
312
|
+
waitingForAmd = false;
|
|
313
|
+
clearTimer();
|
|
314
|
+
triggerOpening();
|
|
315
|
+
}
|
|
316
|
+
return waitingForAmd;
|
|
317
|
+
};
|
|
318
|
+
const claim = () => {
|
|
319
|
+
if (closed || isBlocked() || claimed) return false;
|
|
320
|
+
claimed = true;
|
|
321
|
+
clearTimer();
|
|
322
|
+
return true;
|
|
323
|
+
};
|
|
324
|
+
return {
|
|
325
|
+
claim,
|
|
326
|
+
isBlocked,
|
|
327
|
+
noteInputAudio(audio) {
|
|
328
|
+
if (closed || !readySession || !isBlocked() || machineDetected || acknowledged) return;
|
|
329
|
+
if (calculateMulawRms(audio) >= ASSISTANT_SPEECH_RMS_THRESHOLD) {
|
|
330
|
+
speechMs += audio.length / 8;
|
|
331
|
+
silenceMs = 0;
|
|
332
|
+
} else {
|
|
333
|
+
silenceMs += audio.length / 8;
|
|
334
|
+
if (silenceMs >= 240) speechMs = 0;
|
|
335
|
+
}
|
|
336
|
+
if (speechMs <= 3e3) return;
|
|
337
|
+
acknowledged = true;
|
|
338
|
+
const brief = CallBriefSchema.safeParse(params.call?.metadata?.brief);
|
|
339
|
+
const language = brief.success ? brief.data.language : void 0;
|
|
340
|
+
params.acknowledge?.([`Say only a short acknowledgement${language ? ` in ${JSON.stringify(language)}` : ""}, then stop and wait. Do not call tools.`, /^(es(?:[-_]|$)|spanish|español)/i.test(language ?? "") ? "Sí, un momento" : "Yes, one moment."].join("\n"));
|
|
341
|
+
},
|
|
342
|
+
close() {
|
|
343
|
+
closed = true;
|
|
344
|
+
clearTimer();
|
|
345
|
+
},
|
|
346
|
+
onReady(session) {
|
|
347
|
+
readySession = session;
|
|
348
|
+
if (closed || machineDetected) return;
|
|
349
|
+
if (isBlocked()) {
|
|
350
|
+
if (!timer && !machineDetected) {
|
|
351
|
+
timer = setTimeout(() => {
|
|
352
|
+
if (isBlocked() && !machineDetected && !closed) {
|
|
353
|
+
waitingForAmd = false;
|
|
354
|
+
triggerOpening();
|
|
355
|
+
}
|
|
356
|
+
}, params.holdOpeningMaxMs ?? 3e4);
|
|
357
|
+
timer.unref?.();
|
|
358
|
+
}
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
361
|
+
if (!params.enabled || !params.instructions || claimed) return;
|
|
362
|
+
clearTimer();
|
|
363
|
+
timer = setTimeout(() => {
|
|
364
|
+
if (claim()) session.triggerGreeting(params.instructions);
|
|
365
|
+
}, params.fallbackMs ?? OUTBOUND_GREETING_FALLBACK_MS);
|
|
366
|
+
timer.unref?.();
|
|
367
|
+
}
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
function createRealtimeCallActivityController(params) {
|
|
371
|
+
let closed = false;
|
|
372
|
+
let started = false;
|
|
373
|
+
let consultsInFlight = 0;
|
|
374
|
+
let idleTimer;
|
|
375
|
+
let mediaTimer;
|
|
376
|
+
const clearIdle = () => {
|
|
377
|
+
if (idleTimer) {
|
|
378
|
+
clearTimeout(idleTimer);
|
|
379
|
+
idleTimer = void 0;
|
|
380
|
+
}
|
|
381
|
+
};
|
|
382
|
+
const clearMedia = () => {
|
|
383
|
+
if (mediaTimer) {
|
|
384
|
+
clearTimeout(mediaTimer);
|
|
385
|
+
mediaTimer = void 0;
|
|
386
|
+
}
|
|
387
|
+
};
|
|
388
|
+
const resetIdle = () => {
|
|
389
|
+
clearIdle();
|
|
390
|
+
if (closed || !started || !params.idleHangupMs || consultsInFlight > 0 || params.isPaused?.() === true) return;
|
|
391
|
+
idleTimer = setTimeout(params.onIdle, params.idleHangupMs);
|
|
392
|
+
idleTimer.unref?.();
|
|
393
|
+
};
|
|
394
|
+
return {
|
|
395
|
+
beginConsult() {
|
|
396
|
+
consultsInFlight += 1;
|
|
397
|
+
clearIdle();
|
|
398
|
+
},
|
|
399
|
+
close() {
|
|
400
|
+
closed = true;
|
|
401
|
+
clearIdle();
|
|
402
|
+
clearMedia();
|
|
403
|
+
},
|
|
404
|
+
endConsult() {
|
|
405
|
+
consultsInFlight = Math.max(0, consultsInFlight - 1);
|
|
406
|
+
if (consultsInFlight === 0) resetIdle();
|
|
407
|
+
},
|
|
408
|
+
isPaused: () => consultsInFlight > 0,
|
|
409
|
+
noteMedia() {
|
|
410
|
+
if (closed) return;
|
|
411
|
+
clearMedia();
|
|
412
|
+
mediaTimer = setTimeout(() => {
|
|
413
|
+
params.onMediaWarning();
|
|
414
|
+
mediaTimer = setTimeout(params.onMediaTimeout, params.mediaGraceMs ?? 2e3);
|
|
415
|
+
mediaTimer.unref?.();
|
|
416
|
+
}, params.mediaInactivityMs ?? 3e4);
|
|
417
|
+
mediaTimer.unref?.();
|
|
418
|
+
},
|
|
419
|
+
noteSpeech: resetIdle,
|
|
420
|
+
start() {
|
|
421
|
+
started = true;
|
|
422
|
+
resetIdle();
|
|
423
|
+
}
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
function createRealtimeCallAudioController(params) {
|
|
427
|
+
const cancelOutputAudioForBargeIn = (source, interruptProvider, clearedAudioBytes = 0) => {
|
|
428
|
+
const outputAudioActive = params.harness.talk.outputAudioActive;
|
|
429
|
+
const pendingTelephonyAudio = params.audioPacer.hasPendingAudio();
|
|
430
|
+
if (source === "provider" && !outputAudioActive && !pendingTelephonyAudio && clearedAudioBytes === 0) return;
|
|
431
|
+
const interruptedTurnId = params.harness.talk.activeTurnId;
|
|
432
|
+
if (outputAudioActive || pendingTelephonyAudio) interruptProvider?.(true);
|
|
433
|
+
const clearedBytes = clearedAudioBytes + (source === "local" || pendingTelephonyAudio ? params.audioPacer.clearAudio() : 0);
|
|
434
|
+
console.log(`[voice-call] realtime outbound audio cleared by ${source} barge-in callId=${params.callId} providerCallId=${params.providerCallId} queuedBytes=${clearedBytes}`);
|
|
435
|
+
if (!outputAudioActive || !interruptedTurnId) return;
|
|
436
|
+
const reason = `${source}-barge-in`;
|
|
437
|
+
params.harness.finishOutputAudio(reason);
|
|
438
|
+
params.harness.talk.cancelTurn({
|
|
439
|
+
turnId: interruptedTurnId,
|
|
440
|
+
payload: {
|
|
441
|
+
callId: params.callId,
|
|
442
|
+
providerCallId: params.providerCallId,
|
|
443
|
+
reason
|
|
444
|
+
}
|
|
445
|
+
});
|
|
446
|
+
};
|
|
447
|
+
return {
|
|
448
|
+
audioSink: {
|
|
449
|
+
isOpen: params.isOpen,
|
|
450
|
+
sendAudio: (muLaw, metadata) => {
|
|
451
|
+
if (params.isBlocked?.()) return;
|
|
452
|
+
if (muLaw.length > 0 && calculateMulawRms(muLaw) >= ASSISTANT_SPEECH_RMS_THRESHOLD) params.onAudibleOutput?.();
|
|
453
|
+
params.harness.recordOutputAudio(muLaw);
|
|
454
|
+
params.audioPacer.sendAudio(muLaw, metadata);
|
|
455
|
+
},
|
|
456
|
+
getPlaybackState: () => params.audioPacer.getPlaybackState(),
|
|
457
|
+
clearAudio: (reason) => {
|
|
458
|
+
params.harness.flushOutput(() => {
|
|
459
|
+
const clearedBytes = params.audioPacer.clearAudio();
|
|
460
|
+
if (reason === "barge-in") {
|
|
461
|
+
cancelOutputAudioForBargeIn("provider", void 0, clearedBytes);
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
console.log(`[voice-call] realtime outbound audio clear requested callId=${params.callId} providerCallId=${params.providerCallId} queuedBytes=${clearedBytes}`);
|
|
465
|
+
params.harness.finishOutputAudio(reason ?? "clear");
|
|
466
|
+
});
|
|
467
|
+
},
|
|
468
|
+
sendMark: (markName, acknowledge) => {
|
|
469
|
+
params.audioPacer.sendMark(markName);
|
|
470
|
+
if (markName && acknowledge) params.pendingMarkAcks.set(markName, acknowledge);
|
|
471
|
+
}
|
|
472
|
+
},
|
|
473
|
+
cancelOutputAudioForBargeIn
|
|
474
|
+
};
|
|
475
|
+
}
|
|
476
|
+
//#endregion
|
|
477
|
+
//#region extensions/voice-call/src/webhook/realtime-host-speech.ts
|
|
478
|
+
/** One host-requested utterance, completed by audible output rather than model receipts. */
|
|
479
|
+
function createRealtimeHostSpeechController(params) {
|
|
480
|
+
let closed = false;
|
|
481
|
+
let pending;
|
|
482
|
+
let quietTimer;
|
|
483
|
+
let deadline;
|
|
484
|
+
const finish = (error) => {
|
|
485
|
+
const current = pending;
|
|
486
|
+
pending = void 0;
|
|
487
|
+
clearTimeout(quietTimer);
|
|
488
|
+
clearTimeout(deadline);
|
|
489
|
+
if (error) current?.reject(error);
|
|
490
|
+
else current?.resolve();
|
|
491
|
+
};
|
|
492
|
+
return {
|
|
493
|
+
isActive: () => Boolean(pending),
|
|
494
|
+
speak(instructions) {
|
|
495
|
+
if (closed) return Promise.reject(/* @__PURE__ */ new Error("Realtime bridge is closed"));
|
|
496
|
+
finish(/* @__PURE__ */ new Error("Host speech replaced"));
|
|
497
|
+
return new Promise((resolve, reject) => {
|
|
498
|
+
try {
|
|
499
|
+
params.interrupt();
|
|
500
|
+
if (closed) throw new Error("Realtime bridge is closed");
|
|
501
|
+
pending = {
|
|
502
|
+
resolve,
|
|
503
|
+
reject
|
|
504
|
+
};
|
|
505
|
+
deadline = setTimeout(() => finish(/* @__PURE__ */ new Error("Realtime speech did not finish within 45 seconds")), 45e3);
|
|
506
|
+
deadline.unref?.();
|
|
507
|
+
params.trigger(instructions);
|
|
508
|
+
} catch (error) {
|
|
509
|
+
const rejection = error instanceof Error ? error : new Error(String(error));
|
|
510
|
+
finish(rejection);
|
|
511
|
+
reject(rejection);
|
|
512
|
+
}
|
|
513
|
+
});
|
|
514
|
+
},
|
|
515
|
+
noteAudibleOutput() {
|
|
516
|
+
if (!pending) return;
|
|
517
|
+
clearTimeout(quietTimer);
|
|
518
|
+
quietTimer = setTimeout(() => finish(), 1500);
|
|
519
|
+
quietTimer.unref?.();
|
|
520
|
+
},
|
|
521
|
+
close() {
|
|
522
|
+
closed = true;
|
|
523
|
+
finish(/* @__PURE__ */ new Error("Realtime bridge closed during host speech"));
|
|
524
|
+
}
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
//#endregion
|
|
528
|
+
//#region extensions/voice-call/src/webhook/realtime-call-playback.ts
|
|
529
|
+
const MAX_REALTIME_WS_BUFFERED_BYTES$1 = 1048576;
|
|
530
|
+
function createRealtimeCallPlayback(params) {
|
|
531
|
+
const { ws, callRecord, callSid, adapter, harness, initialGreetingInstructions } = params;
|
|
532
|
+
const callId = callRecord.callId;
|
|
533
|
+
const checkBackpressure = (phase) => {
|
|
534
|
+
if (ws.bufferedAmount > MAX_REALTIME_WS_BUFFERED_BYTES$1) {
|
|
535
|
+
console.warn(`[voice-call] realtime outbound websocket backpressure ${phase} send callId=${callId} providerCallId=${callSid} bufferedBytes=${ws.bufferedAmount}`);
|
|
536
|
+
ws.close(1013, "Backpressure: send buffer exceeded");
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
539
|
+
return true;
|
|
540
|
+
};
|
|
541
|
+
const sendString = (message) => {
|
|
542
|
+
if (ws.readyState !== WebSocket.OPEN || !checkBackpressure("before")) return false;
|
|
543
|
+
ws.send(message);
|
|
544
|
+
return checkBackpressure("after");
|
|
545
|
+
};
|
|
546
|
+
const pendingMarkAcks = /* @__PURE__ */ new Map();
|
|
547
|
+
const audioPacer = new RealtimeAudioPacer({
|
|
548
|
+
onPlaybackReset: () => pendingMarkAcks.clear(),
|
|
549
|
+
onAudioSent: (audio) => {
|
|
550
|
+
if (calculateMulawRms(audio) >= .035) {
|
|
551
|
+
hostSpeech.noteAudibleOutput();
|
|
552
|
+
activity.noteSpeech();
|
|
553
|
+
}
|
|
554
|
+
},
|
|
555
|
+
send: sendString,
|
|
556
|
+
serializer: adapter,
|
|
557
|
+
onBackpressure: () => {
|
|
558
|
+
console.warn(`[voice-call] realtime paced audio backpressure callId=${callId} providerCallId=${callSid}`);
|
|
559
|
+
if (ws.readyState === WebSocket.OPEN) ws.close(1013, "Backpressure: paced audio queue exceeded");
|
|
560
|
+
}
|
|
561
|
+
});
|
|
562
|
+
const outboundGreeting = createOutboundGreetingController({
|
|
563
|
+
enabled: params.hostOwnsOpening,
|
|
564
|
+
instructions: initialGreetingInstructions,
|
|
565
|
+
call: callRecord,
|
|
566
|
+
holdOpeningMaxMs: params.holdOpeningMaxMs,
|
|
567
|
+
acknowledge: (instructions) => {
|
|
568
|
+
hostSpeech.speak(instructions).catch((error) => {
|
|
569
|
+
console.warn("[voice-call] AMD acknowledgement failed:", formatErrorMessage(error));
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
});
|
|
573
|
+
const interruptModelOutput = () => {
|
|
574
|
+
params.getSession()?.handleBargeIn({
|
|
575
|
+
audioPlaybackActive: true,
|
|
576
|
+
force: true
|
|
577
|
+
});
|
|
578
|
+
audioController.cancelOutputAudioForBargeIn("local");
|
|
579
|
+
};
|
|
580
|
+
const hostSpeech = createRealtimeHostSpeechController({
|
|
581
|
+
interrupt: () => {
|
|
582
|
+
interruptModelOutput();
|
|
583
|
+
},
|
|
584
|
+
trigger: (instructions) => {
|
|
585
|
+
const session = params.getSession();
|
|
586
|
+
if (!session) throw new Error("No active realtime bridge for call");
|
|
587
|
+
session.triggerGreeting(instructions);
|
|
588
|
+
}
|
|
589
|
+
});
|
|
590
|
+
const audioController = createRealtimeCallAudioController({
|
|
591
|
+
audioPacer,
|
|
592
|
+
callId,
|
|
593
|
+
harness,
|
|
594
|
+
isOpen: () => !params.isClosed() && ws.readyState === WebSocket.OPEN,
|
|
595
|
+
isBlocked: () => outboundGreeting.isBlocked() && !hostSpeech.isActive(),
|
|
596
|
+
onAudibleOutput: () => {
|
|
597
|
+
outboundGreeting.claim();
|
|
598
|
+
},
|
|
599
|
+
pendingMarkAcks,
|
|
600
|
+
providerCallId: callSid
|
|
601
|
+
});
|
|
602
|
+
const activity = createRealtimeCallActivityController({
|
|
603
|
+
idleHangupMs: params.idleHangupMs,
|
|
604
|
+
isPaused: () => outboundGreeting.isBlocked() || hostSpeech.isActive(),
|
|
605
|
+
onIdle: () => {
|
|
606
|
+
console.warn(`[voice-call] Realtime speech idle timeout callId=${callId} providerCallId=${callSid} timeoutMs=${params.idleHangupMs}`);
|
|
607
|
+
params.closeForInactivity();
|
|
608
|
+
if (ws.readyState === WebSocket.OPEN) ws.close(1e3, "Speech inactivity");
|
|
609
|
+
},
|
|
610
|
+
onMediaWarning: () => {
|
|
611
|
+
console.warn(`[voice-call] Realtime media inactive callId=${callId} providerCallId=${callSid} timeoutMs=${REALTIME_MEDIA_INACTIVITY_TIMEOUT_MS} graceMs=${REALTIME_DISCONNECT_HANGUP_GRACE_MS}`);
|
|
612
|
+
},
|
|
613
|
+
onMediaTimeout: () => {
|
|
614
|
+
params.closeForInactivity();
|
|
615
|
+
if (ws.readyState === WebSocket.OPEN) ws.close(1e3, "Media inactivity");
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
return {
|
|
619
|
+
audioPacer,
|
|
620
|
+
pendingMarkAcks,
|
|
621
|
+
outboundGreeting,
|
|
622
|
+
audioController,
|
|
623
|
+
activity,
|
|
624
|
+
hostSpeech
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
//#endregion
|
|
628
|
+
//#region extensions/voice-call/src/webhook/realtime-transcript-text.ts
|
|
629
|
+
const MAX_PARTIAL_USER_TRANSCRIPT_CHARS = 1200;
|
|
630
|
+
function limitPartialUserTranscript(text) {
|
|
631
|
+
if (text.length <= MAX_PARTIAL_USER_TRANSCRIPT_CHARS) return text;
|
|
632
|
+
const tail = sliceUtf16Safe(text, -1200);
|
|
633
|
+
return tail.replace(/^\S+\s+/, "").trimStart() || tail.trimStart();
|
|
634
|
+
}
|
|
635
|
+
function normalizeTranscriptText(text) {
|
|
636
|
+
return text.replace(/\s+/g, " ").trim();
|
|
637
|
+
}
|
|
638
|
+
function findTextOverlap(base, next) {
|
|
639
|
+
const max = Math.min(base.length, next.length);
|
|
640
|
+
for (let size = max; size > 0; size -= 1) if (base.slice(-size) === next.slice(0, size)) return size;
|
|
641
|
+
return 0;
|
|
642
|
+
}
|
|
643
|
+
function appendTranscriptText(base, fragment) {
|
|
644
|
+
const next = normalizeTranscriptText(fragment);
|
|
645
|
+
if (!next) return base ?? "";
|
|
646
|
+
const current = normalizeTranscriptText(base ?? "");
|
|
647
|
+
if (!current) return next;
|
|
648
|
+
const currentLower = current.toLowerCase();
|
|
649
|
+
const nextLower = next.toLowerCase();
|
|
650
|
+
if (currentLower === nextLower || currentLower.endsWith(nextLower)) return current;
|
|
651
|
+
if (nextLower.startsWith(currentLower)) return next;
|
|
652
|
+
const overlap = findTextOverlap(currentLower, nextLower);
|
|
653
|
+
if (overlap >= 6 || overlap >= 3 && next.length <= 12) return `${current}${next.slice(overlap)}`.trim();
|
|
654
|
+
return `${current}${/[([{"']$/.test(current) || /^[,.;:!?)]/.test(next) ? "" : " "}${next}`.trim();
|
|
655
|
+
}
|
|
656
|
+
function resolveFinalTranscriptText(params) {
|
|
657
|
+
const final = normalizeTranscriptText(params.final);
|
|
658
|
+
const rawPartial = params.rawPartial ?? "";
|
|
659
|
+
const partial = normalizeTranscriptText(params.partial ?? rawPartial);
|
|
660
|
+
if (!partial) return final;
|
|
661
|
+
if (!final) return partial;
|
|
662
|
+
const compact = (value) => value.toLowerCase().replaceAll(/\s/g, "");
|
|
663
|
+
const compactFinal = compact(final);
|
|
664
|
+
const compactRaw = compact(rawPartial);
|
|
665
|
+
const compactPartial = compact(partial);
|
|
666
|
+
if (compactFinal.startsWith(compactPartial) || compactFinal.endsWith(compactPartial)) return final;
|
|
667
|
+
if (compactPartial.endsWith(compactFinal)) return partial;
|
|
668
|
+
if (compactRaw !== compactPartial) return appendTranscriptText(partial, params.final);
|
|
669
|
+
return normalizeTranscriptText(`${rawPartial}${params.final}`);
|
|
670
|
+
}
|
|
671
|
+
//#endregion
|
|
247
672
|
//#region extensions/voice-call/src/webhook/stream-frame-adapter.ts
|
|
248
673
|
/** Parse numeric timestamps sent as numbers or integer strings. */
|
|
249
674
|
function parseTimestampMs(value) {
|
|
@@ -332,65 +757,16 @@ const STREAM_TOKEN_TTL_MS = 3e4;
|
|
|
332
757
|
const DEFAULT_HOST = "localhost:8443";
|
|
333
758
|
const MAX_REALTIME_MESSAGE_BYTES = 262144;
|
|
334
759
|
const MAX_REALTIME_WS_BUFFERED_BYTES = 1048576;
|
|
335
|
-
const REALTIME_MEDIA_INACTIVITY_TIMEOUT_MS = 3e4;
|
|
336
|
-
const REALTIME_DISCONNECT_HANGUP_GRACE_MS = 2e3;
|
|
337
760
|
const FORCED_CONSULT_FALLBACK_DELAY_MS = 200;
|
|
338
761
|
const FORCED_CONSULT_NATIVE_DEDUPE_MS = 2e3;
|
|
339
762
|
const FORCED_CONSULT_RESULT_MAX_CHARS = 1800;
|
|
340
763
|
const FORCED_CONSULT_REASON = "provider_final_transcript_without_openclaw_agent_consult";
|
|
341
764
|
const CONSULT_TRANSCRIPT_SETTLE_MS = 350;
|
|
342
765
|
const CONSULT_TRANSCRIPT_SETTLE_MAX_MS = 1e3;
|
|
343
|
-
const MAX_PARTIAL_USER_TRANSCRIPT_CHARS = 1200;
|
|
344
766
|
const RECENT_FINAL_USER_TRANSCRIPT_TTL_MS = 2e3;
|
|
345
767
|
const BARGE_IN_REQUIRED_LOUD_CHUNKS = 2;
|
|
768
|
+
const CALLER_SPEECH_RMS_THRESHOLD = .035;
|
|
346
769
|
const logger = createSubsystemLogger("voice-call/realtime");
|
|
347
|
-
function buildGreetingInstructions(baseInstructions, greeting) {
|
|
348
|
-
const trimmedGreeting = greeting?.trim();
|
|
349
|
-
if (!trimmedGreeting) return;
|
|
350
|
-
const intro = "Start the call by greeting the caller naturally. Include this greeting in your first spoken reply:";
|
|
351
|
-
return baseInstructions ? `${baseInstructions}\n\n${intro} "${trimmedGreeting}"` : `${intro} "${trimmedGreeting}"`;
|
|
352
|
-
}
|
|
353
|
-
function normalizeTranscriptText(text) {
|
|
354
|
-
return text.replace(/\s+/g, " ").trim();
|
|
355
|
-
}
|
|
356
|
-
function findTextOverlap(base, next) {
|
|
357
|
-
const max = Math.min(base.length, next.length);
|
|
358
|
-
for (let size = max; size > 0; size -= 1) if (base.slice(-size) === next.slice(0, size)) return size;
|
|
359
|
-
return 0;
|
|
360
|
-
}
|
|
361
|
-
function appendTranscriptText(base, fragment) {
|
|
362
|
-
const next = normalizeTranscriptText(fragment);
|
|
363
|
-
if (!next) return base ?? "";
|
|
364
|
-
const current = normalizeTranscriptText(base ?? "");
|
|
365
|
-
if (!current) return next;
|
|
366
|
-
const currentLower = current.toLowerCase();
|
|
367
|
-
const nextLower = next.toLowerCase();
|
|
368
|
-
if (currentLower === nextLower || currentLower.endsWith(nextLower)) return current;
|
|
369
|
-
if (nextLower.startsWith(currentLower)) return next;
|
|
370
|
-
const overlap = findTextOverlap(currentLower, nextLower);
|
|
371
|
-
if (overlap >= 6 || overlap >= 3 && next.length <= 12) return `${current}${next.slice(overlap)}`.trim();
|
|
372
|
-
return `${current}${/[([{"']$/.test(current) || /^[,.;:!?)]/.test(next) ? "" : " "}${next}`.trim();
|
|
373
|
-
}
|
|
374
|
-
function resolveFinalTranscriptText(params) {
|
|
375
|
-
const final = normalizeTranscriptText(params.final);
|
|
376
|
-
const rawPartial = params.rawPartial ?? "";
|
|
377
|
-
const partial = normalizeTranscriptText(params.partial ?? rawPartial);
|
|
378
|
-
if (!partial) return final;
|
|
379
|
-
if (!final) return partial;
|
|
380
|
-
const compact = (value) => value.toLowerCase().replaceAll(/\s/g, "");
|
|
381
|
-
const compactFinal = compact(final);
|
|
382
|
-
const compactRaw = compact(rawPartial);
|
|
383
|
-
const compactPartial = compact(partial);
|
|
384
|
-
if (compactFinal.startsWith(compactPartial) || compactFinal.endsWith(compactPartial)) return final;
|
|
385
|
-
if (compactPartial.endsWith(compactFinal)) return partial;
|
|
386
|
-
if (compactRaw !== compactPartial) return appendTranscriptText(partial, params.final);
|
|
387
|
-
return normalizeTranscriptText(`${rawPartial}${params.final}`);
|
|
388
|
-
}
|
|
389
|
-
function limitPartialUserTranscript(text) {
|
|
390
|
-
if (text.length <= MAX_PARTIAL_USER_TRANSCRIPT_CHARS) return text;
|
|
391
|
-
const tail = sliceUtf16Safe(text, -1200);
|
|
392
|
-
return tail.replace(/^\S+\s+/, "").trimStart() || tail.trimStart();
|
|
393
|
-
}
|
|
394
770
|
function withFallbackConsultQuestion(args, fallback) {
|
|
395
771
|
const providerQuestion = readRealtimeVoiceConsultQuestion(args);
|
|
396
772
|
const question = fallback?.trim();
|
|
@@ -413,13 +789,47 @@ function withFallbackConsultQuestion(args, fallback) {
|
|
|
413
789
|
question
|
|
414
790
|
};
|
|
415
791
|
}
|
|
416
|
-
function
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
792
|
+
function captureNativeConsultTranscript(state, invocationId) {
|
|
793
|
+
const previous = state.nativeConsultInvocation;
|
|
794
|
+
if (invocationId.trim() && previous?.id === invocationId) return previous.read;
|
|
795
|
+
previous?.freeze();
|
|
796
|
+
const read = () => {
|
|
797
|
+
const buffer = state.partial ?? state.recentFinal;
|
|
798
|
+
return {
|
|
799
|
+
buffer,
|
|
800
|
+
text: buffer?.text
|
|
801
|
+
};
|
|
802
|
+
};
|
|
803
|
+
const initial = read();
|
|
804
|
+
let frozen;
|
|
805
|
+
const invocation = {
|
|
806
|
+
id: invocationId,
|
|
807
|
+
read: () => frozen ?? read(),
|
|
808
|
+
freeze: () => {
|
|
809
|
+
frozen = initial.buffer?.final ? {
|
|
810
|
+
buffer: initial.buffer,
|
|
811
|
+
text: initial.buffer.text
|
|
812
|
+
} : initial;
|
|
813
|
+
}
|
|
814
|
+
};
|
|
815
|
+
state.nativeConsultInvocation = invocation;
|
|
816
|
+
return invocation.read;
|
|
817
|
+
}
|
|
818
|
+
function clearRecentFinalUserTranscript(state) {
|
|
819
|
+
clearTimeout(state.recentFinalTimer);
|
|
820
|
+
state.recentFinalTimer = void 0;
|
|
821
|
+
state.recentFinal = void 0;
|
|
822
|
+
}
|
|
823
|
+
function consumeUserTranscript(state, consumed) {
|
|
824
|
+
const buffer = consumed?.buffer;
|
|
825
|
+
const text = consumed?.text?.trim();
|
|
826
|
+
if (!state || !buffer || !text || buffer !== state.partial && buffer !== state.recentFinal || !buffer.text.toLowerCase().startsWith(text.toLowerCase())) return;
|
|
827
|
+
const remaining = buffer.text.slice(text.length).trimStart();
|
|
828
|
+
if (remaining) {
|
|
829
|
+
buffer.text = remaining;
|
|
830
|
+
buffer.raw = remaining;
|
|
831
|
+
} else if (buffer === state.partial) state.partial = void 0;
|
|
832
|
+
else clearRecentFinalUserTranscript(state);
|
|
423
833
|
}
|
|
424
834
|
async function waitForNativeConsult(state) {
|
|
425
835
|
return await Promise.race([state.promise.then((result) => ({
|
|
@@ -427,36 +837,15 @@ async function waitForNativeConsult(state) {
|
|
|
427
837
|
result
|
|
428
838
|
})), state.cancellation.then(() => ({ kind: "cancelled" }))]);
|
|
429
839
|
}
|
|
430
|
-
function appendRecentTalkEventMetadata(metadata, event) {
|
|
431
|
-
const previous = metadata ?? {};
|
|
432
|
-
const recent = Array.isArray(previous.recentTalkEvents) ? previous.recentTalkEvents : [];
|
|
433
|
-
return {
|
|
434
|
-
...previous,
|
|
435
|
-
lastTalkEventAt: event.timestamp,
|
|
436
|
-
lastTalkEventType: event.type,
|
|
437
|
-
recentTalkEvents: [...recent, {
|
|
438
|
-
id: event.id,
|
|
439
|
-
brain: event.brain,
|
|
440
|
-
mode: event.mode,
|
|
441
|
-
provider: event.provider,
|
|
442
|
-
seq: event.seq,
|
|
443
|
-
sessionId: event.sessionId,
|
|
444
|
-
timestamp: event.timestamp,
|
|
445
|
-
transport: event.transport,
|
|
446
|
-
type: event.type,
|
|
447
|
-
...event.turnId ? { turnId: event.turnId } : {},
|
|
448
|
-
...event.final !== void 0 ? { final: event.final } : {}
|
|
449
|
-
}].slice(-12)
|
|
450
|
-
};
|
|
451
|
-
}
|
|
452
840
|
var RealtimeCallHandler = class {
|
|
453
|
-
constructor(config, manager, resolveCallRegistration, servePath, streamDisconnectLifecycle, coreConfig) {
|
|
841
|
+
constructor(config, manager, resolveCallRegistration, servePath, streamDisconnectLifecycle, coreConfig, holdOpeningMaxMs) {
|
|
454
842
|
this.config = config;
|
|
455
843
|
this.manager = manager;
|
|
456
844
|
this.resolveCallRegistration = resolveCallRegistration;
|
|
457
845
|
this.servePath = servePath;
|
|
458
846
|
this.streamDisconnectLifecycle = streamDisconnectLifecycle;
|
|
459
847
|
this.coreConfig = coreConfig;
|
|
848
|
+
this.holdOpeningMaxMs = holdOpeningMaxMs;
|
|
460
849
|
this.toolHandlers = /* @__PURE__ */ new Map();
|
|
461
850
|
this.pendingStreamTokens = /* @__PURE__ */ new Map();
|
|
462
851
|
this.activeSockets = /* @__PURE__ */ new Set();
|
|
@@ -669,20 +1058,16 @@ var RealtimeCallHandler = class {
|
|
|
669
1058
|
this.toolHandlers.set(name, fn);
|
|
670
1059
|
}
|
|
671
1060
|
speak(callId, instructions) {
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
success: false,
|
|
683
|
-
error: formatErrorMessage(error)
|
|
684
|
-
};
|
|
685
|
-
}
|
|
1061
|
+
return speakOnRealtimeBridge(this.activeBridgesByCallId, callId, instructions);
|
|
1062
|
+
}
|
|
1063
|
+
playVoicemail(callId, instructions) {
|
|
1064
|
+
return this.activeTelephonyBindingsByCallId.get(callId)?.playVoicemail(instructions);
|
|
1065
|
+
}
|
|
1066
|
+
async drainCall(callId) {
|
|
1067
|
+
await this.activeBridgesByCallId.get(callId)?.close();
|
|
1068
|
+
}
|
|
1069
|
+
async prepareCarrierPlayback(callId) {
|
|
1070
|
+
await this.activeTelephonyBindingsByCallId.get(callId)?.close();
|
|
686
1071
|
}
|
|
687
1072
|
issueStreamSession(request = {}) {
|
|
688
1073
|
const meta = {
|
|
@@ -747,8 +1132,14 @@ var RealtimeCallHandler = class {
|
|
|
747
1132
|
this.trackShutdownWork(attempt, this.terminationAttempts);
|
|
748
1133
|
return attempt;
|
|
749
1134
|
};
|
|
750
|
-
const
|
|
1135
|
+
const carrierOwnsPlayback = () => Boolean(callRecord.metadata?.voicemailStatus || callRecord.metadata?.notifyStatus);
|
|
1136
|
+
const admissionAbandoned = () => this.closing || ws.readyState !== WebSocket.OPEN || carrierOwnsPlayback();
|
|
751
1137
|
const abandonAdmission = async () => {
|
|
1138
|
+
if (carrierOwnsPlayback()) {
|
|
1139
|
+
this.streamDisconnectLifecycle.retire(callSid, streamSid);
|
|
1140
|
+
ws.close(1e3, "Carrier playback owns the call");
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
752
1143
|
if (!this.activeBridgesByCallId.has(callId)) {
|
|
753
1144
|
if (this.closing || this.serverClosingSockets.has(ws)) await emitCallEnd("shutdown");
|
|
754
1145
|
else {
|
|
@@ -807,9 +1198,13 @@ var RealtimeCallHandler = class {
|
|
|
807
1198
|
const previousTelephonyBinding = this.activeTelephonyBindingsByCallId.get(callId);
|
|
808
1199
|
const { agentId, instructions, provider: realtimeProvider, providerConfig, capabilities } = registration;
|
|
809
1200
|
const { handlesAgentConsult, toolPolicy } = sessionPolicy;
|
|
810
|
-
if (handlesAgentConsult) console.warn("[voice-call] This realtime model uses native agent delegation; the
|
|
811
|
-
const initialGreetingInstructions = buildGreetingInstructions(instructions, initialGreeting);
|
|
1201
|
+
if (handlesAgentConsult) console.warn("[voice-call] This realtime model uses native agent delegation; hang-up is available through the call-scoped OpenClaw agent, while other custom realtime function tools remain unavailable.");
|
|
1202
|
+
const initialGreetingInstructions = handlesAgentConsult ? buildVerbatimGreetingInstructions(instructions, initialGreeting) : buildGreetingInstructions(instructions, initialGreeting);
|
|
1203
|
+
const isDelayedOutboundGreeting = handlesAgentConsult && callRecord.direction === "outbound" && Boolean(initialGreetingInstructions);
|
|
1204
|
+
const hostOwnsOpening = callRecord.direction === "outbound" && Boolean(initialGreetingInstructions) && (handlesAgentConsult || Boolean(callRecord.metadata?.voicemailManagedByHost) && callRecord.metadata?.mode !== "notify");
|
|
1205
|
+
const sessionInstructions = isDelayedOutboundGreeting ? initialGreetingInstructions : instructions;
|
|
812
1206
|
const harness = createRealtimeVoiceSessionHarness({
|
|
1207
|
+
forcedConsults: { questionsMatch: (left, right) => Boolean(left && right && left.trim() === right.trim()) },
|
|
813
1208
|
talk: {
|
|
814
1209
|
sessionId: `voice-call:${callId}:realtime`,
|
|
815
1210
|
mode: "realtime",
|
|
@@ -840,32 +1235,12 @@ var RealtimeCallHandler = class {
|
|
|
840
1235
|
})
|
|
841
1236
|
},
|
|
842
1237
|
onTalkEvent: (event) => {
|
|
843
|
-
this.manager.updateCallMetadata(callRecord, (metadata) => appendRecentTalkEventMetadata(metadata, event)).catch((error) => {
|
|
1238
|
+
this.manager.updateCallMetadata(callRecord, (metadata) => appendRecentTalkEventMetadata(metadata, event, "realtime")).catch((error) => {
|
|
844
1239
|
console.warn("[voice-call] Failed to update realtime call metadata:", error);
|
|
845
1240
|
});
|
|
846
1241
|
}
|
|
847
1242
|
});
|
|
848
1243
|
const providerHandlesInputAudioBargeIn = (capabilities ?? realtimeProvider.capabilities)?.handlesInputAudioBargeIn === true;
|
|
849
|
-
const cancelOutputAudioForBargeIn = (source, interruptProvider, clearedAudioBytes = 0) => {
|
|
850
|
-
const outputAudioActive = harness.talk.outputAudioActive;
|
|
851
|
-
const pendingTelephonyAudio = audioPacer.hasPendingAudio();
|
|
852
|
-
if (source === "provider" && !outputAudioActive && !pendingTelephonyAudio && clearedAudioBytes === 0) return;
|
|
853
|
-
const interruptedTurnId = harness.talk.activeTurnId;
|
|
854
|
-
interruptProvider?.(outputAudioActive || pendingTelephonyAudio);
|
|
855
|
-
const clearedBytes = clearedAudioBytes + (source === "local" || pendingTelephonyAudio ? audioPacer.clearAudio() : 0);
|
|
856
|
-
console.log(`[voice-call] realtime outbound audio cleared by ${source} barge-in callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
|
|
857
|
-
if (!outputAudioActive || !interruptedTurnId) return;
|
|
858
|
-
const reason = `${source}-barge-in`;
|
|
859
|
-
harness.finishOutputAudio(reason);
|
|
860
|
-
harness.talk.cancelTurn({
|
|
861
|
-
turnId: interruptedTurnId,
|
|
862
|
-
payload: {
|
|
863
|
-
callId,
|
|
864
|
-
providerCallId: callSid,
|
|
865
|
-
reason
|
|
866
|
-
}
|
|
867
|
-
});
|
|
868
|
-
};
|
|
869
1244
|
harness.emit({
|
|
870
1245
|
type: "session.started",
|
|
871
1246
|
payload: {
|
|
@@ -875,42 +1250,34 @@ var RealtimeCallHandler = class {
|
|
|
875
1250
|
}
|
|
876
1251
|
});
|
|
877
1252
|
console.log(`[voice-call] Realtime bridge starting for call ${callId} (providerCallId=${callSid}, initialGreeting=${initialGreetingInstructions ? "queued" : "absent"})`);
|
|
878
|
-
const sendString = (message) => {
|
|
879
|
-
if (ws.readyState !== WebSocket.OPEN) return false;
|
|
880
|
-
if (ws.bufferedAmount > MAX_REALTIME_WS_BUFFERED_BYTES) {
|
|
881
|
-
console.warn(`[voice-call] realtime outbound websocket backpressure before send callId=${callId} providerCallId=${callSid} bufferedBytes=${ws.bufferedAmount}`);
|
|
882
|
-
ws.close(1013, "Backpressure: send buffer exceeded");
|
|
883
|
-
return false;
|
|
884
|
-
}
|
|
885
|
-
ws.send(message);
|
|
886
|
-
if (ws.bufferedAmount > MAX_REALTIME_WS_BUFFERED_BYTES) {
|
|
887
|
-
console.warn(`[voice-call] realtime outbound websocket backpressure after send callId=${callId} providerCallId=${callSid} bufferedBytes=${ws.bufferedAmount}`);
|
|
888
|
-
ws.close(1013, "Backpressure: send buffer exceeded");
|
|
889
|
-
return false;
|
|
890
|
-
}
|
|
891
|
-
return true;
|
|
892
|
-
};
|
|
893
|
-
const pendingMarkAcks = /* @__PURE__ */ new Map();
|
|
894
|
-
const audioPacer = new RealtimeAudioPacer({
|
|
895
|
-
onPlaybackReset: () => pendingMarkAcks.clear(),
|
|
896
|
-
send: sendString,
|
|
897
|
-
serializer: adapter,
|
|
898
|
-
onBackpressure: () => {
|
|
899
|
-
console.warn(`[voice-call] realtime paced audio backpressure callId=${callId} providerCallId=${callSid}`);
|
|
900
|
-
if (ws.readyState === WebSocket.OPEN) ws.close(1013, "Backpressure: paced audio queue exceeded");
|
|
901
|
-
}
|
|
902
|
-
});
|
|
903
1253
|
const speechDetector = createSpeechThresholdGate({
|
|
904
|
-
rmsThreshold:
|
|
1254
|
+
rmsThreshold: CALLER_SPEECH_RMS_THRESHOLD,
|
|
905
1255
|
speechFrames: BARGE_IN_REQUIRED_LOUD_CHUNKS,
|
|
906
1256
|
silenceFrames: 12
|
|
907
1257
|
});
|
|
908
1258
|
const interruptResponseOnInputAudio = typeof providerConfig.interruptResponseOnInputAudio === "boolean" ? providerConfig.interruptResponseOnInputAudio : void 0;
|
|
909
|
-
|
|
1259
|
+
let nativeConsultOwner = void 0;
|
|
1260
|
+
const { audioPacer, pendingMarkAcks, outboundGreeting, audioController, activity, hostSpeech } = createRealtimeCallPlayback({
|
|
1261
|
+
ws,
|
|
1262
|
+
callRecord,
|
|
1263
|
+
callSid,
|
|
1264
|
+
adapter,
|
|
1265
|
+
harness,
|
|
1266
|
+
initialGreetingInstructions,
|
|
1267
|
+
hostOwnsOpening,
|
|
1268
|
+
holdOpeningMaxMs: this.holdOpeningMaxMs,
|
|
1269
|
+
idleHangupMs: this.config.idleHangupMs,
|
|
1270
|
+
getSession: () => nativeConsultOwner,
|
|
1271
|
+
isClosed: () => sessionClosed,
|
|
1272
|
+
closeForInactivity: () => {
|
|
1273
|
+
telephonyBinding.close("inactivity");
|
|
1274
|
+
}
|
|
1275
|
+
});
|
|
910
1276
|
let provisionalCloseReason;
|
|
911
1277
|
let sessionClosed = false;
|
|
912
|
-
const
|
|
913
|
-
const userTranscriptOwner =
|
|
1278
|
+
const previousTranscriptOwner = this.userTranscriptStatesByCallId.get(callId);
|
|
1279
|
+
const userTranscriptOwner = {};
|
|
1280
|
+
this.userTranscriptStatesByCallId.set(callId, userTranscriptOwner);
|
|
914
1281
|
let transcriptPersistence = Promise.resolve();
|
|
915
1282
|
let transcriptFailure;
|
|
916
1283
|
const reportTranscriptFailure = (error) => {
|
|
@@ -940,53 +1307,42 @@ var RealtimeCallHandler = class {
|
|
|
940
1307
|
capabilities,
|
|
941
1308
|
audioFormat: REALTIME_VOICE_AUDIO_FORMAT_G711_ULAW_8KHZ,
|
|
942
1309
|
interruptResponseOnInputAudio,
|
|
943
|
-
instructions: handlesAgentConsult ? `${
|
|
1310
|
+
instructions: handlesAgentConsult ? `${sessionInstructions}\n\nUse native agent delegation for OpenClaw work. To end the current call, say out loud that you are hanging up and delegate; the OpenClaw agent acts on the transcript. Other custom realtime function tools are unavailable in this session.` : sessionInstructions,
|
|
944
1311
|
tools: handlesAgentConsult ? [] : this.config.tools,
|
|
945
1312
|
...handlesAgentConsult ? { runAgentConsult: async (request) => {
|
|
946
|
-
const owner = nativeConsultOwner
|
|
1313
|
+
const owner = nativeConsultOwner;
|
|
947
1314
|
const generation = continuityGeneration;
|
|
948
1315
|
request.signal?.throwIfAborted();
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
initialGreetingInstructions,
|
|
963
|
-
triggerGreetingOnReady: Boolean(initialGreetingInstructions),
|
|
964
|
-
audioSink: {
|
|
965
|
-
isOpen: () => !sessionClosed && ws.readyState === WebSocket.OPEN,
|
|
966
|
-
sendAudio: (muLaw, metadata) => {
|
|
967
|
-
harness.recordOutputAudio(muLaw);
|
|
968
|
-
audioPacer.sendAudio(muLaw, metadata);
|
|
969
|
-
},
|
|
970
|
-
getPlaybackState: () => audioPacer.getPlaybackState(),
|
|
971
|
-
clearAudio: (reason) => {
|
|
972
|
-
harness.flushOutput(() => {
|
|
973
|
-
const clearedBytes = audioPacer.clearAudio();
|
|
974
|
-
if (reason === "barge-in") {
|
|
975
|
-
cancelOutputAudioForBargeIn("provider", void 0, clearedBytes);
|
|
976
|
-
return;
|
|
977
|
-
}
|
|
978
|
-
console.log(`[voice-call] realtime outbound audio clear requested callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
|
|
979
|
-
harness.finishOutputAudio("clear");
|
|
1316
|
+
const invocationId = randomUUID();
|
|
1317
|
+
const transcript = captureNativeConsultTranscript(userTranscriptOwner, invocationId);
|
|
1318
|
+
activity.beginConsult();
|
|
1319
|
+
try {
|
|
1320
|
+
await transcriptPersistence;
|
|
1321
|
+
if (!owner || sessionClosed || generation !== continuityGeneration || !this.isActiveBridgeOwner(callId, owner)) throw new Error("Realtime call delegation owner is no longer active");
|
|
1322
|
+
if (toolPolicy === "none") throw new Error("Agent consultation is disabled for this call");
|
|
1323
|
+
const result = await this.executeToolCall(owner, callId, invocationId, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, { question: request.prompt }, harness.ensureTurn(), harness, userTranscriptOwner, transcript, { signal: request.signal });
|
|
1324
|
+
request.signal?.throwIfAborted();
|
|
1325
|
+
if (sessionClosed || generation !== continuityGeneration || !this.isActiveBridgeOwner(callId, owner)) throw new Error("Realtime call delegation owner is no longer active");
|
|
1326
|
+
const text = readSpeakableRealtimeVoiceToolResult(result, {
|
|
1327
|
+
keys: ["text", "output"],
|
|
1328
|
+
maxChars: FORCED_CONSULT_RESULT_MAX_CHARS
|
|
980
1329
|
});
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1330
|
+
if (!text) throw new Error("Agent consultation returned no spoken answer");
|
|
1331
|
+
return { text };
|
|
1332
|
+
} finally {
|
|
1333
|
+
activity.endConsult();
|
|
985
1334
|
}
|
|
986
|
-
},
|
|
1335
|
+
} } : {},
|
|
1336
|
+
initialGreetingInstructions,
|
|
1337
|
+
triggerGreetingOnReady: Boolean(initialGreetingInstructions) && !hostOwnsOpening,
|
|
1338
|
+
audioSink: audioController.audioSink,
|
|
987
1339
|
onTranscript: (role, text, isFinal) => {
|
|
988
|
-
const owner = nativeConsultOwner
|
|
989
|
-
if (provisionalCloseReason || sessionClosed && !isFinal || !this.getUserTranscriptState(callId, userTranscriptOwner) || owner && !this.isActiveBridgeOwner(callId, owner)) return;
|
|
1340
|
+
const owner = nativeConsultOwner;
|
|
1341
|
+
if (provisionalCloseReason || outboundGreeting.isBlocked() || sessionClosed && !isFinal || !this.getUserTranscriptState(callId, userTranscriptOwner) || owner && !this.isActiveBridgeOwner(callId, owner)) return;
|
|
1342
|
+
if (text.trim()) {
|
|
1343
|
+
outboundGreeting.claim();
|
|
1344
|
+
activity.noteSpeech();
|
|
1345
|
+
}
|
|
990
1346
|
const turnId = harness.ensureTurn();
|
|
991
1347
|
const eventType = role === "assistant" ? isFinal ? "output.text.done" : "output.text.delta" : isFinal ? "transcript.done" : "transcript.delta";
|
|
992
1348
|
const payload = role === "assistant" ? { text } : {
|
|
@@ -1010,87 +1366,116 @@ var RealtimeCallHandler = class {
|
|
|
1010
1366
|
});
|
|
1011
1367
|
if (!isFinal) {
|
|
1012
1368
|
if (role === "user" && text.trim()) {
|
|
1013
|
-
const
|
|
1014
|
-
if (!
|
|
1015
|
-
|
|
1369
|
+
const state = this.getUserTranscriptState(callId, userTranscriptOwner);
|
|
1370
|
+
if (!state) return;
|
|
1371
|
+
const partial = state.partial ??= { text: "" };
|
|
1372
|
+
partial.text = limitPartialUserTranscript(appendTranscriptText(partial.text, text));
|
|
1373
|
+
partial.raw = limitPartialUserTranscript(`${partial.raw ?? ""}${text}`);
|
|
1374
|
+
partial.updatedAt = Date.now();
|
|
1375
|
+
console.log(`[voice-call] realtime input transcript callId=${callId} providerCallId=${callSid} final=false chars=${text.trim().length} aggregateChars=${partial.text.length}`);
|
|
1016
1376
|
}
|
|
1017
1377
|
return;
|
|
1018
1378
|
}
|
|
1379
|
+
let transcript = text;
|
|
1380
|
+
let consultTranscript;
|
|
1019
1381
|
if (role === "user") {
|
|
1020
1382
|
const state = this.getUserTranscriptState(callId, userTranscriptOwner);
|
|
1021
1383
|
if (!state) return;
|
|
1022
|
-
|
|
1023
|
-
partial: state.partial,
|
|
1024
|
-
rawPartial: state.
|
|
1384
|
+
transcript = resolveFinalTranscriptText({
|
|
1385
|
+
partial: state.partial?.text,
|
|
1386
|
+
rawPartial: state.partial?.raw,
|
|
1025
1387
|
final: text
|
|
1026
1388
|
});
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
transcript,
|
|
1037
|
-
isFinal: true
|
|
1389
|
+
const buffer = state.partial ?? { text: transcript };
|
|
1390
|
+
buffer.text = transcript;
|
|
1391
|
+
buffer.final = true;
|
|
1392
|
+
state.partial = void 0;
|
|
1393
|
+
clearRecentFinalUserTranscript(state);
|
|
1394
|
+
state.recentFinal = buffer;
|
|
1395
|
+
consultTranscript = {
|
|
1396
|
+
buffer,
|
|
1397
|
+
text: transcript
|
|
1038
1398
|
};
|
|
1039
|
-
const
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
transcript,
|
|
1048
|
-
userTranscriptOwner,
|
|
1049
|
-
clearAudio: () => {
|
|
1050
|
-
const clearedBytes = audioPacer.clearAudio();
|
|
1051
|
-
console.log(`[voice-call] realtime forced consult cleared outbound audio callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
|
|
1052
|
-
}
|
|
1053
|
-
});
|
|
1054
|
-
});
|
|
1055
|
-
transcriptPersistence.catch(reportTranscriptFailure);
|
|
1056
|
-
return;
|
|
1399
|
+
const timer = setTimeout(() => {
|
|
1400
|
+
if (!this.getUserTranscriptState(callId, state) || state.recentFinalTimer !== timer) return;
|
|
1401
|
+
state.recentFinal = void 0;
|
|
1402
|
+
state.recentFinalTimer = void 0;
|
|
1403
|
+
}, RECENT_FINAL_USER_TRANSCRIPT_TTL_MS);
|
|
1404
|
+
timer.unref?.();
|
|
1405
|
+
state.recentFinalTimer = timer;
|
|
1406
|
+
console.log(`[voice-call] realtime input transcript callId=${callId} providerCallId=${callSid} final=true chars=${text.trim().length} aggregateChars=${transcript.length}`);
|
|
1057
1407
|
}
|
|
1058
|
-
|
|
1059
|
-
id: `realtime
|
|
1060
|
-
type: "call.assistant-speech",
|
|
1408
|
+
const event = {
|
|
1409
|
+
id: `realtime-${role === "user" ? "speech" : "bot"}-${callSid}-${randomUUID()}`,
|
|
1061
1410
|
callId,
|
|
1062
1411
|
providerCallId: callSid,
|
|
1063
1412
|
timestamp: Date.now(),
|
|
1064
|
-
transcript
|
|
1065
|
-
|
|
1413
|
+
transcript,
|
|
1414
|
+
...role === "user" ? {
|
|
1415
|
+
type: "call.speech",
|
|
1416
|
+
isFinal: true
|
|
1417
|
+
} : { type: "call.assistant-speech" }
|
|
1418
|
+
};
|
|
1419
|
+
const generation = continuityGeneration;
|
|
1420
|
+
transcriptPersistence = this.manager.processEvent(event).then(() => {
|
|
1421
|
+
if (role !== "user" || handlesAgentConsult || sessionClosed || generation !== continuityGeneration || !this.getUserTranscriptState(callId, userTranscriptOwner) || !this.isActiveBridgeOwner(callId, session)) return;
|
|
1422
|
+
this.scheduleForcedAgentConsult({
|
|
1423
|
+
harness,
|
|
1424
|
+
session,
|
|
1425
|
+
callId,
|
|
1426
|
+
callSid,
|
|
1427
|
+
transcript,
|
|
1428
|
+
consultTranscript,
|
|
1429
|
+
userTranscriptOwner,
|
|
1430
|
+
clearAudio: () => {
|
|
1431
|
+
const clearedBytes = audioPacer.clearAudio();
|
|
1432
|
+
console.log(`[voice-call] realtime forced consult cleared outbound audio callId=${callId} providerCallId=${callSid} queuedBytes=${clearedBytes}`);
|
|
1433
|
+
},
|
|
1434
|
+
beginConsultActivity: () => activity.beginConsult(),
|
|
1435
|
+
endConsultActivity: () => activity.endConsult()
|
|
1436
|
+
});
|
|
1437
|
+
});
|
|
1066
1438
|
transcriptPersistence.catch(reportTranscriptFailure);
|
|
1067
1439
|
},
|
|
1068
1440
|
onToolCall: async (toolEvent, sessionLocal) => {
|
|
1069
1441
|
const generation = continuityGeneration;
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
const
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1442
|
+
const isConsult = toolEvent.name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME;
|
|
1443
|
+
const invocationId = toolEvent.callId || toolEvent.itemId;
|
|
1444
|
+
const transcript = isConsult ? captureNativeConsultTranscript(userTranscriptOwner, invocationId) : void 0;
|
|
1445
|
+
if (isConsult) activity.beginConsult();
|
|
1446
|
+
try {
|
|
1447
|
+
await transcriptPersistence;
|
|
1448
|
+
if (sessionClosed || generation !== continuityGeneration || !this.isActiveBridgeOwner(callId, sessionLocal)) return;
|
|
1449
|
+
const turnId = harness.ensureTurn();
|
|
1450
|
+
harness.emit({
|
|
1451
|
+
type: "tool.call",
|
|
1452
|
+
turnId,
|
|
1453
|
+
itemId: toolEvent.itemId,
|
|
1454
|
+
callId: toolEvent.callId,
|
|
1455
|
+
payload: {
|
|
1456
|
+
name: toolEvent.name,
|
|
1457
|
+
args: toolEvent.args
|
|
1458
|
+
}
|
|
1459
|
+
});
|
|
1460
|
+
console.log(`[voice-call] realtime tool call received callId=${callId} providerCallId=${callSid} tool=${toolEvent.name}`);
|
|
1461
|
+
await this.executeToolCall(sessionLocal, callId, invocationId, toolEvent.name, toolEvent.args, turnId, harness, userTranscriptOwner, transcript);
|
|
1462
|
+
} finally {
|
|
1463
|
+
if (isConsult) activity.endConsult();
|
|
1464
|
+
}
|
|
1085
1465
|
},
|
|
1086
1466
|
onEvent: (event) => {
|
|
1087
1467
|
if (event.direction === "client" && event.type === "session.continuity.reset") {
|
|
1088
1468
|
continuityGeneration += 1;
|
|
1089
1469
|
const turnId = harness.talk.activeTurnId;
|
|
1090
|
-
const owner = nativeConsultOwner
|
|
1470
|
+
const owner = nativeConsultOwner;
|
|
1091
1471
|
if (owner && this.isActiveBridgeOwner(callId, owner)) {
|
|
1092
|
-
this.
|
|
1093
|
-
|
|
1472
|
+
const state = this.getUserTranscriptState(callId, userTranscriptOwner);
|
|
1473
|
+
if (state) {
|
|
1474
|
+
state.partial = void 0;
|
|
1475
|
+
clearRecentFinalUserTranscript(state);
|
|
1476
|
+
state.nativeConsultInvocation = void 0;
|
|
1477
|
+
}
|
|
1478
|
+
this.cancelConsultSession(callId, owner, "continuity-reset");
|
|
1094
1479
|
}
|
|
1095
1480
|
harness.flushOutput(() => {
|
|
1096
1481
|
audioPacer.clearAudio();
|
|
@@ -1106,7 +1491,9 @@ var RealtimeCallHandler = class {
|
|
|
1106
1491
|
});
|
|
1107
1492
|
return;
|
|
1108
1493
|
}
|
|
1109
|
-
if (event.type === "input_audio_buffer.speech_started") {
|
|
1494
|
+
if (event.type === "input_audio_buffer.speech_started" && !outboundGreeting.isBlocked()) {
|
|
1495
|
+
outboundGreeting.claim();
|
|
1496
|
+
activity.noteSpeech();
|
|
1110
1497
|
harness.ensureTurn();
|
|
1111
1498
|
return;
|
|
1112
1499
|
}
|
|
@@ -1134,7 +1521,7 @@ var RealtimeCallHandler = class {
|
|
|
1134
1521
|
onResponseDone: (outcome) => {
|
|
1135
1522
|
if (outcome.status === "failed" || outcome.status === "incomplete") console.warn(`[voice-call] realtime response ${outcome.status}: ${outcome.message}`);
|
|
1136
1523
|
},
|
|
1137
|
-
onReady: () => {
|
|
1524
|
+
onReady: (readySession) => {
|
|
1138
1525
|
harness.emit({
|
|
1139
1526
|
type: "session.ready",
|
|
1140
1527
|
payload: {
|
|
@@ -1142,6 +1529,7 @@ var RealtimeCallHandler = class {
|
|
|
1142
1529
|
providerCallId: callSid
|
|
1143
1530
|
}
|
|
1144
1531
|
});
|
|
1532
|
+
outboundGreeting.onReady(readySession);
|
|
1145
1533
|
},
|
|
1146
1534
|
onError: (error) => {
|
|
1147
1535
|
console.error("[voice-call] realtime voice error:", error.message);
|
|
@@ -1158,7 +1546,7 @@ var RealtimeCallHandler = class {
|
|
|
1158
1546
|
payload: { reason },
|
|
1159
1547
|
final: true
|
|
1160
1548
|
});
|
|
1161
|
-
const owner = nativeConsultOwner
|
|
1549
|
+
const owner = nativeConsultOwner;
|
|
1162
1550
|
if (!owner) {
|
|
1163
1551
|
provisionalCloseReason ??= reason;
|
|
1164
1552
|
return;
|
|
@@ -1172,7 +1560,7 @@ var RealtimeCallHandler = class {
|
|
|
1172
1560
|
}
|
|
1173
1561
|
const ownsCallState = this.isActiveBridgeOwner(callId, owner);
|
|
1174
1562
|
if (reason === "completed" && !ownsCallState) return;
|
|
1175
|
-
if (ownsCallState) closeBinding(
|
|
1563
|
+
if (ownsCallState) closeBinding(reason);
|
|
1176
1564
|
this.streamDisconnectLifecycle.retire(callSid, streamSid);
|
|
1177
1565
|
if (ws.readyState === WebSocket.OPEN) ws.close(reason === "error" ? 1011 : 1e3, "Bridge disconnected");
|
|
1178
1566
|
}
|
|
@@ -1184,7 +1572,11 @@ var RealtimeCallHandler = class {
|
|
|
1184
1572
|
console.error("[voice-call] Failed to create realtime bridge:", error);
|
|
1185
1573
|
}
|
|
1186
1574
|
if (!candidate || provisionalCloseReason) {
|
|
1187
|
-
this.
|
|
1575
|
+
if (this.getUserTranscriptState(callId, userTranscriptOwner)) {
|
|
1576
|
+
clearRecentFinalUserTranscript(userTranscriptOwner);
|
|
1577
|
+
if (previousTranscriptOwner) this.userTranscriptStatesByCallId.set(callId, previousTranscriptOwner);
|
|
1578
|
+
else this.userTranscriptStatesByCallId.delete(callId);
|
|
1579
|
+
}
|
|
1188
1580
|
try {
|
|
1189
1581
|
await drainProviderClose(() => candidate?.close());
|
|
1190
1582
|
} catch (error) {
|
|
@@ -1192,14 +1584,20 @@ var RealtimeCallHandler = class {
|
|
|
1192
1584
|
}
|
|
1193
1585
|
harness.close();
|
|
1194
1586
|
audioPacer.close();
|
|
1587
|
+
outboundGreeting.close();
|
|
1588
|
+
hostSpeech.close();
|
|
1589
|
+
activity.close();
|
|
1195
1590
|
const reason = provisionalCloseReason ?? "error";
|
|
1196
1591
|
if (!this.activeBridgesByCallId.has(callId)) emitCallEnd(reason);
|
|
1197
1592
|
if (ws.readyState === WebSocket.OPEN) ws.close(reason === "error" ? 1011 : 1e3, "Failed to create realtime bridge");
|
|
1198
1593
|
return null;
|
|
1199
1594
|
}
|
|
1200
1595
|
const session = candidate;
|
|
1201
|
-
this.
|
|
1202
|
-
|
|
1596
|
+
if (this.getUserTranscriptState(callId, userTranscriptOwner) && previousTranscriptOwner) {
|
|
1597
|
+
clearTimeout(previousTranscriptOwner.recentFinalTimer);
|
|
1598
|
+
previousTranscriptOwner.recentFinalTimer = void 0;
|
|
1599
|
+
}
|
|
1600
|
+
nativeConsultOwner = session;
|
|
1203
1601
|
const localBargeIn = !(session.bridge.handlesInputAudioBargeIn ?? providerHandlesInputAudioBargeIn) && resolveRealtimeVoiceBargeIn({
|
|
1204
1602
|
configuredBargeIn: void 0,
|
|
1205
1603
|
interruptResponseOnInputAudio,
|
|
@@ -1215,12 +1613,17 @@ var RealtimeCallHandler = class {
|
|
|
1215
1613
|
const sendAudioToSession = session.sendAudio.bind(session);
|
|
1216
1614
|
session.sendAudio = (audio) => {
|
|
1217
1615
|
if (sessionClosed) return;
|
|
1616
|
+
outboundGreeting.noteInputAudio(audio);
|
|
1617
|
+
if (outboundGreeting.isBlocked() || hostSpeech.isActive()) return;
|
|
1618
|
+
const inputRms = calculateMulawRms(audio);
|
|
1619
|
+
if (inputRms >= CALLER_SPEECH_RMS_THRESHOLD) activity.noteSpeech();
|
|
1218
1620
|
if (speechDetector.accept({
|
|
1219
|
-
rms:
|
|
1621
|
+
rms: inputRms,
|
|
1220
1622
|
peak: 0
|
|
1221
1623
|
})) {
|
|
1624
|
+
outboundGreeting.claim();
|
|
1222
1625
|
console.log(`[voice-call] realtime local speech detected callId=${callId} providerCallId=${callSid}`);
|
|
1223
|
-
if (localBargeIn) cancelOutputAudioForBargeIn("local", (audioPlaybackActive) => {
|
|
1626
|
+
if (localBargeIn && !activity.isPaused()) audioController.cancelOutputAudioForBargeIn("local", (audioPlaybackActive) => {
|
|
1224
1627
|
session.handleBargeIn({ audioPlaybackActive });
|
|
1225
1628
|
});
|
|
1226
1629
|
}
|
|
@@ -1232,32 +1635,31 @@ var RealtimeCallHandler = class {
|
|
|
1232
1635
|
session.close = () => {
|
|
1233
1636
|
if (sessionClosed) return sessionClosePromise ?? Promise.resolve();
|
|
1234
1637
|
sessionClosed = true;
|
|
1638
|
+
outboundGreeting.close();
|
|
1639
|
+
hostSpeech.close();
|
|
1640
|
+
activity.close();
|
|
1235
1641
|
this.cancelConsultSession(callId, session);
|
|
1236
1642
|
audioPacer.close();
|
|
1237
1643
|
sessionClosePromise = drainProviderClose(closeSession).finally(() => {
|
|
1238
|
-
|
|
1239
|
-
this.
|
|
1644
|
+
for (const key of [callId, callSid]) if (this.activeBridgesByCallId.get(key) === session) this.activeBridgesByCallId.delete(key);
|
|
1645
|
+
if (this.getUserTranscriptState(callId, userTranscriptOwner)) {
|
|
1646
|
+
clearRecentFinalUserTranscript(userTranscriptOwner);
|
|
1647
|
+
this.userTranscriptStatesByCallId.delete(callId);
|
|
1648
|
+
}
|
|
1240
1649
|
harness.close();
|
|
1241
1650
|
});
|
|
1242
1651
|
return sessionClosePromise;
|
|
1243
1652
|
};
|
|
1244
|
-
let livenessTimer;
|
|
1245
|
-
const clearLivenessTimer = () => {
|
|
1246
|
-
if (livenessTimer) {
|
|
1247
|
-
clearTimeout(livenessTimer);
|
|
1248
|
-
livenessTimer = void 0;
|
|
1249
|
-
}
|
|
1250
|
-
};
|
|
1251
1653
|
let bindingClosed = false;
|
|
1252
1654
|
let bindingClosePromise;
|
|
1253
|
-
const closeBinding = (
|
|
1655
|
+
const closeBinding = (cause) => {
|
|
1254
1656
|
if (bindingClosed) return bindingClosePromise ?? callEndPromise ?? Promise.resolve();
|
|
1255
1657
|
bindingClosed = true;
|
|
1256
|
-
|
|
1257
|
-
const ownsCall = this.activeTelephonyBindingsByCallId.get(callId) ===
|
|
1658
|
+
activity.close();
|
|
1659
|
+
const ownsCall = this.activeTelephonyBindingsByCallId.get(callId) === telephonyBinding;
|
|
1258
1660
|
const finishClose = () => {
|
|
1259
|
-
const stillOwnsCall = this.activeTelephonyBindingsByCallId.get(callId) ===
|
|
1260
|
-
this.
|
|
1661
|
+
const stillOwnsCall = this.activeTelephonyBindingsByCallId.get(callId) === telephonyBinding;
|
|
1662
|
+
if (stillOwnsCall) this.activeTelephonyBindingsByCallId.delete(callId);
|
|
1261
1663
|
if (cause === "disconnect") this.streamDisconnectLifecycle.disconnect(callSid, streamSid);
|
|
1262
1664
|
else this.streamDisconnectLifecycle.retire(callSid, streamSid);
|
|
1263
1665
|
if (ownsCall && stillOwnsCall && cause && cause !== "disconnect") return emitCallEnd(cause);
|
|
@@ -1292,200 +1694,69 @@ var RealtimeCallHandler = class {
|
|
|
1292
1694
|
acknowledge();
|
|
1293
1695
|
}
|
|
1294
1696
|
},
|
|
1295
|
-
close: (cause) => closeBinding(
|
|
1697
|
+
close: (cause) => closeBinding(cause),
|
|
1296
1698
|
endCall: () => {
|
|
1297
|
-
closeBinding(
|
|
1699
|
+
closeBinding();
|
|
1298
1700
|
if (ws.readyState === WebSocket.OPEN) ws.close(1e3, "Call ended");
|
|
1299
1701
|
},
|
|
1300
|
-
noteMediaActivity: () =>
|
|
1301
|
-
|
|
1302
|
-
clearLivenessTimer();
|
|
1303
|
-
livenessTimer = setTimeout(() => {
|
|
1304
|
-
console.warn(`[voice-call] Realtime media inactive callId=${callId} providerCallId=${callSid} timeoutMs=${REALTIME_MEDIA_INACTIVITY_TIMEOUT_MS} graceMs=${REALTIME_DISCONNECT_HANGUP_GRACE_MS}`);
|
|
1305
|
-
livenessTimer = setTimeout(() => {
|
|
1306
|
-
telephonyBinding.close("inactivity");
|
|
1307
|
-
if (ws.readyState === WebSocket.OPEN) ws.close(1e3, "Media inactivity");
|
|
1308
|
-
}, REALTIME_DISCONNECT_HANGUP_GRACE_MS);
|
|
1309
|
-
livenessTimer.unref?.();
|
|
1310
|
-
}, REALTIME_MEDIA_INACTIVITY_TIMEOUT_MS);
|
|
1311
|
-
livenessTimer.unref?.();
|
|
1312
|
-
},
|
|
1702
|
+
noteMediaActivity: () => activity.noteMedia(),
|
|
1703
|
+
playVoicemail: (speechInstructions) => hostSpeech.speak(speechInstructions),
|
|
1313
1704
|
retire: () => {
|
|
1314
|
-
closeBinding(
|
|
1705
|
+
closeBinding();
|
|
1315
1706
|
}
|
|
1316
1707
|
};
|
|
1317
1708
|
this.activeBridgesByCallId.set(callId, session);
|
|
1318
1709
|
this.activeBridgesByCallId.set(callSid, session);
|
|
1319
1710
|
this.activeTelephonyBindingsByCallId.set(callId, telephonyBinding);
|
|
1320
1711
|
telephonyBinding.noteMediaActivity();
|
|
1712
|
+
activity.start();
|
|
1713
|
+
if (this.config.idleHangupMs) console.log(`[voice-call] Realtime speech idle monitor armed callId=${callId} providerCallId=${callSid} timeoutMs=${this.config.idleHangupMs}`);
|
|
1321
1714
|
if (previousTelephonyBinding && previousTelephonyBinding !== telephonyBinding) previousTelephonyBinding.retire();
|
|
1322
1715
|
session.connect().catch(async (error) => {
|
|
1323
1716
|
console.error("[voice-call] Failed to connect realtime bridge:", error);
|
|
1324
1717
|
try {
|
|
1325
|
-
await closeBinding(
|
|
1718
|
+
await closeBinding("error");
|
|
1326
1719
|
} catch {} finally {
|
|
1327
1720
|
ws.close(1011, "Failed to connect");
|
|
1328
1721
|
}
|
|
1329
1722
|
});
|
|
1330
1723
|
return telephonyBinding;
|
|
1331
1724
|
}
|
|
1332
|
-
beginUserTranscriptOwnerAdoption(callId) {
|
|
1333
|
-
const adoption = {
|
|
1334
|
-
owner: {},
|
|
1335
|
-
previous: this.userTranscriptStatesByCallId.get(callId)
|
|
1336
|
-
};
|
|
1337
|
-
this.userTranscriptStatesByCallId.set(callId, adoption.owner);
|
|
1338
|
-
return adoption;
|
|
1339
|
-
}
|
|
1340
|
-
commitUserTranscriptOwnerAdoption(callId, adoption) {
|
|
1341
|
-
if (this.userTranscriptStatesByCallId.get(callId) !== adoption.owner) return;
|
|
1342
|
-
if (adoption.previous?.recentFinalTimer) {
|
|
1343
|
-
clearTimeout(adoption.previous.recentFinalTimer);
|
|
1344
|
-
adoption.previous.recentFinalTimer = void 0;
|
|
1345
|
-
}
|
|
1346
|
-
}
|
|
1347
|
-
rollbackUserTranscriptOwnerAdoption(callId, adoption) {
|
|
1348
|
-
if (this.userTranscriptStatesByCallId.get(callId) !== adoption.owner) return;
|
|
1349
|
-
if (adoption.owner.recentFinalTimer) {
|
|
1350
|
-
clearTimeout(adoption.owner.recentFinalTimer);
|
|
1351
|
-
adoption.owner.recentFinalTimer = void 0;
|
|
1352
|
-
}
|
|
1353
|
-
if (adoption.previous) {
|
|
1354
|
-
this.userTranscriptStatesByCallId.set(callId, adoption.previous);
|
|
1355
|
-
return;
|
|
1356
|
-
}
|
|
1357
|
-
this.userTranscriptStatesByCallId.delete(callId);
|
|
1358
|
-
}
|
|
1359
1725
|
getUserTranscriptState(callId, owner) {
|
|
1360
1726
|
const state = this.userTranscriptStatesByCallId.get(callId);
|
|
1361
1727
|
return state === owner ? state : void 0;
|
|
1362
1728
|
}
|
|
1363
|
-
|
|
1364
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1365
|
-
if (!state) return;
|
|
1366
|
-
const next = limitPartialUserTranscript(appendTranscriptText(state.partial, text));
|
|
1367
|
-
const raw = limitPartialUserTranscript(`${state.rawPartial ?? ""}${text}`);
|
|
1368
|
-
state.partial = next;
|
|
1369
|
-
state.rawPartial = raw;
|
|
1370
|
-
state.partialUpdatedAt = Date.now();
|
|
1371
|
-
return next;
|
|
1372
|
-
}
|
|
1373
|
-
clearPartialUserTranscript(callId, owner) {
|
|
1374
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1375
|
-
if (!state) return;
|
|
1376
|
-
state.partial = void 0;
|
|
1377
|
-
state.rawPartial = void 0;
|
|
1378
|
-
state.partialUpdatedAt = void 0;
|
|
1379
|
-
}
|
|
1380
|
-
setRecentFinalUserTranscript(callId, owner, text) {
|
|
1381
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1382
|
-
if (!state) return;
|
|
1383
|
-
this.clearRecentFinalUserTranscript(callId, owner);
|
|
1384
|
-
state.recentFinal = text;
|
|
1385
|
-
const timer = setTimeout(() => {
|
|
1386
|
-
if (this.userTranscriptStatesByCallId.get(callId) !== state) return;
|
|
1387
|
-
if (state.recentFinal === text) state.recentFinal = void 0;
|
|
1388
|
-
if (state.recentFinalTimer === timer) state.recentFinalTimer = void 0;
|
|
1389
|
-
}, RECENT_FINAL_USER_TRANSCRIPT_TTL_MS);
|
|
1390
|
-
timer.unref?.();
|
|
1391
|
-
state.recentFinalTimer = timer;
|
|
1392
|
-
}
|
|
1393
|
-
clearRecentFinalUserTranscript(callId, owner) {
|
|
1394
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1395
|
-
if (!state) return;
|
|
1396
|
-
if (state.recentFinalTimer) {
|
|
1397
|
-
clearTimeout(state.recentFinalTimer);
|
|
1398
|
-
state.recentFinalTimer = void 0;
|
|
1399
|
-
}
|
|
1400
|
-
state.recentFinal = void 0;
|
|
1401
|
-
}
|
|
1402
|
-
resetUserTranscriptState(callId, owner) {
|
|
1403
|
-
this.clearPartialUserTranscript(callId, owner);
|
|
1404
|
-
this.clearRecentFinalUserTranscript(callId, owner);
|
|
1405
|
-
}
|
|
1406
|
-
clearUserTranscriptState(callId, owner) {
|
|
1407
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1408
|
-
if (!state) return;
|
|
1409
|
-
if (state.recentFinalTimer) clearTimeout(state.recentFinalTimer);
|
|
1410
|
-
this.userTranscriptStatesByCallId.delete(callId);
|
|
1411
|
-
}
|
|
1412
|
-
cancelNativeConsult(callId, owner) {
|
|
1413
|
-
const state = this.nativeConsultsInFlightByCallId.get(callId);
|
|
1414
|
-
if (!state || state.owner !== owner) return;
|
|
1415
|
-
this.nativeConsultsInFlightByCallId.delete(callId);
|
|
1416
|
-
state.cancel();
|
|
1417
|
-
}
|
|
1418
|
-
cancelForcedConsult(callId, owner) {
|
|
1419
|
-
const state = this.forcedConsultsByCallId.get(callId);
|
|
1420
|
-
if (!state || state.owner !== owner) return;
|
|
1421
|
-
state.cancelled = true;
|
|
1422
|
-
state.sendSpeechPrompt = false;
|
|
1423
|
-
state.cancel();
|
|
1424
|
-
this.forcedConsultsByCallId.delete(callId);
|
|
1425
|
-
}
|
|
1426
|
-
resetConsultSessionForContinuity(callId, owner) {
|
|
1427
|
-
const session = this.consultSessionsByCallId.get(callId);
|
|
1428
|
-
if (!session || session.owner !== owner) return;
|
|
1429
|
-
this.cancelForcedConsult(callId, owner);
|
|
1430
|
-
this.cancelNativeConsult(callId, owner);
|
|
1431
|
-
session.coordinator.clear();
|
|
1432
|
-
}
|
|
1433
|
-
cancelConsultSession(callId, owner) {
|
|
1729
|
+
cancelConsultSession(callId, owner, mode = "retire") {
|
|
1434
1730
|
const session = this.consultSessionsByCallId.get(callId);
|
|
1435
1731
|
if (!session || session.owner !== owner) return;
|
|
1436
|
-
session.coordinator.clearPending();
|
|
1437
|
-
this.
|
|
1438
|
-
|
|
1439
|
-
|
|
1732
|
+
if (mode === "retire") session.coordinator.clearPending();
|
|
1733
|
+
const forced = this.forcedConsultsByCallId.get(callId);
|
|
1734
|
+
if (forced?.owner === owner) {
|
|
1735
|
+
forced.cancelled = true;
|
|
1736
|
+
forced.sendSpeechPrompt = false;
|
|
1737
|
+
forced.cancel();
|
|
1738
|
+
this.forcedConsultsByCallId.delete(callId);
|
|
1739
|
+
}
|
|
1740
|
+
const native = this.nativeConsultsInFlightByCallId.get(callId);
|
|
1741
|
+
if (native?.owner === owner) {
|
|
1742
|
+
this.nativeConsultsInFlightByCallId.delete(callId);
|
|
1743
|
+
native.cancel();
|
|
1744
|
+
}
|
|
1745
|
+
if (mode === "continuity-reset") session.coordinator.clear();
|
|
1746
|
+
else this.consultSessionsByCallId.delete(callId);
|
|
1440
1747
|
}
|
|
1441
1748
|
isActiveBridgeOwner(callId, owner) {
|
|
1442
1749
|
return this.activeBridgesByCallId.get(callId) === owner;
|
|
1443
1750
|
}
|
|
1444
|
-
clearActiveBridgeMappings(callId, callSid, owner) {
|
|
1445
|
-
for (const key of [callId, callSid]) {
|
|
1446
|
-
if (this.activeBridgesByCallId.get(key) !== owner) continue;
|
|
1447
|
-
this.activeBridgesByCallId.delete(key);
|
|
1448
|
-
}
|
|
1449
|
-
}
|
|
1450
|
-
clearActiveTelephonyBinding(callId, binding) {
|
|
1451
|
-
if (this.activeTelephonyBindingsByCallId.get(callId) === binding) this.activeTelephonyBindingsByCallId.delete(callId);
|
|
1452
|
-
}
|
|
1453
|
-
resolveUserTranscriptContext(callId, owner) {
|
|
1454
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1455
|
-
return state?.partial ?? state?.recentFinal;
|
|
1456
|
-
}
|
|
1457
|
-
consumePartialUserTranscript(callId, owner, consumed) {
|
|
1458
|
-
const text = consumed?.trim();
|
|
1459
|
-
if (!text) return;
|
|
1460
|
-
const state = this.getUserTranscriptState(callId, owner);
|
|
1461
|
-
const current = state?.partial;
|
|
1462
|
-
if (!current) return;
|
|
1463
|
-
if (current === text) {
|
|
1464
|
-
this.clearPartialUserTranscript(callId, owner);
|
|
1465
|
-
return;
|
|
1466
|
-
}
|
|
1467
|
-
if (current.toLowerCase().startsWith(text.toLowerCase())) {
|
|
1468
|
-
const remaining = current.slice(text.length).trimStart();
|
|
1469
|
-
if (remaining) {
|
|
1470
|
-
state.partial = remaining;
|
|
1471
|
-
state.rawPartial = remaining;
|
|
1472
|
-
} else this.clearPartialUserTranscript(callId, owner);
|
|
1473
|
-
}
|
|
1474
|
-
const recent = state.recentFinal;
|
|
1475
|
-
if (!recent) return;
|
|
1476
|
-
if (recent === text || recent.toLowerCase().startsWith(text.toLowerCase())) this.clearRecentFinalUserTranscript(callId, owner);
|
|
1477
|
-
}
|
|
1478
1751
|
async waitForConsultTranscriptSettle(callId, owner, startedAt) {
|
|
1479
1752
|
const deadline = startedAt + CONSULT_TRANSCRIPT_SETTLE_MAX_MS;
|
|
1480
1753
|
while (true) {
|
|
1481
|
-
const updatedAt = this.getUserTranscriptState(callId, owner)?.
|
|
1754
|
+
const updatedAt = this.getUserTranscriptState(callId, owner)?.partial?.updatedAt;
|
|
1482
1755
|
if (!updatedAt) return;
|
|
1483
1756
|
const now = Date.now();
|
|
1484
1757
|
const quietFor = now - updatedAt;
|
|
1485
1758
|
if (quietFor >= CONSULT_TRANSCRIPT_SETTLE_MS || now >= deadline) return;
|
|
1486
|
-
await
|
|
1487
|
-
setTimeout(resolve, Math.min(CONSULT_TRANSCRIPT_SETTLE_MS - quietFor, deadline - now));
|
|
1488
|
-
});
|
|
1759
|
+
await sleep(Math.min(CONSULT_TRANSCRIPT_SETTLE_MS - quietFor, deadline - now));
|
|
1489
1760
|
}
|
|
1490
1761
|
}
|
|
1491
1762
|
scheduleForcedAgentConsult(params) {
|
|
@@ -1518,9 +1789,11 @@ var RealtimeCallHandler = class {
|
|
|
1518
1789
|
logger.debug(`[voice-call] realtime forced agent consult reason=${FORCED_CONSULT_REASON} consultPolicy=always callId=${params.callId} providerCallId=${params.callSid} chars=${params.handle.question.length}`);
|
|
1519
1790
|
console.log(`[voice-call] realtime forced agent consult starting callId=${params.callId} providerCallId=${params.callSid} chars=${params.handle.question.length}`);
|
|
1520
1791
|
params.clearAudio();
|
|
1792
|
+
params.beginConsultActivity();
|
|
1521
1793
|
const abortController = new AbortController();
|
|
1522
1794
|
const state = {
|
|
1523
1795
|
owner: params.session,
|
|
1796
|
+
handle: params.handle,
|
|
1524
1797
|
sendSpeechPrompt: true,
|
|
1525
1798
|
cancelled: false,
|
|
1526
1799
|
cancel: () => {
|
|
@@ -1551,7 +1824,7 @@ var RealtimeCallHandler = class {
|
|
|
1551
1824
|
params.session.sendUserMessage(buildForcedConsultSpeechPrompt(text));
|
|
1552
1825
|
}
|
|
1553
1826
|
console.log(`[voice-call] realtime forced agent consult completed callId=${params.callId} providerCallId=${params.callSid} elapsedMs=${Date.now() - startedAt}`);
|
|
1554
|
-
this.
|
|
1827
|
+
consumeUserTranscript(this.getUserTranscriptState(params.callId, params.userTranscriptOwner), params.consultTranscript);
|
|
1555
1828
|
} catch (error) {
|
|
1556
1829
|
if (!state.cancelled) {
|
|
1557
1830
|
const result = buildRealtimeVoiceAgentErrorProviderResult(error);
|
|
@@ -1559,6 +1832,7 @@ var RealtimeCallHandler = class {
|
|
|
1559
1832
|
(failed ? console.warn : console.log)(`[voice-call] realtime forced agent consult ${failed ? "failed" : "cancelled"} callId=${params.callId} providerCallId=${params.callSid}${failed ? ` error=${result.error}` : ""}`);
|
|
1560
1833
|
}
|
|
1561
1834
|
} finally {
|
|
1835
|
+
params.endConsultActivity();
|
|
1562
1836
|
if (!state.cancelled) {
|
|
1563
1837
|
if (this.forcedConsultsByCallId.get(params.callId) !== state) coordinator.remove(params.handle);
|
|
1564
1838
|
else setTimeout(() => {
|
|
@@ -1636,7 +1910,7 @@ var RealtimeCallHandler = class {
|
|
|
1636
1910
|
});
|
|
1637
1911
|
binding.endCall();
|
|
1638
1912
|
}
|
|
1639
|
-
async executeToolCall(bridge, callId, bridgeCallId, name, args, turnId, harness, userTranscriptOwner, delegation) {
|
|
1913
|
+
async executeToolCall(bridge, callId, bridgeCallId, name, args, turnId, harness, userTranscriptOwner, consultTranscript, delegation) {
|
|
1640
1914
|
if (name === "openclaw_end_call") {
|
|
1641
1915
|
await this.executeEndCallTool({
|
|
1642
1916
|
bridge,
|
|
@@ -1696,6 +1970,12 @@ var RealtimeCallHandler = class {
|
|
|
1696
1970
|
};
|
|
1697
1971
|
if (name === REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
|
|
1698
1972
|
if (this.activeBridgesByCallId.get(callId) !== bridge) return;
|
|
1973
|
+
const submitBusyResult = () => submitFinalToolResult({
|
|
1974
|
+
status: "busy",
|
|
1975
|
+
started: false,
|
|
1976
|
+
retryable: true,
|
|
1977
|
+
error: "OpenClaw is still consulting on a different request. This request has not started. Wait for that consult's result before you retry this request; do not poll while it is running."
|
|
1978
|
+
});
|
|
1699
1979
|
const coordinator = harness.forcedConsults;
|
|
1700
1980
|
const forcedMatch = coordinator.recordNativeConsult(args, bridgeCallId);
|
|
1701
1981
|
if (forcedMatch.kind === "none") {
|
|
@@ -1704,25 +1984,27 @@ var RealtimeCallHandler = class {
|
|
|
1704
1984
|
}
|
|
1705
1985
|
const forcedConsultState = this.forcedConsultsByCallId.get(callId);
|
|
1706
1986
|
const forcedConsult = forcedConsultState?.owner === bridge && !forcedConsultState.cancelled ? forcedConsultState : void 0;
|
|
1707
|
-
if (forcedMatch.kind === "already_delivered"
|
|
1708
|
-
|
|
1709
|
-
|
|
1987
|
+
if (forcedMatch.kind === "already_delivered") {
|
|
1988
|
+
const cancelled = coordinator.isCancelled(forcedMatch.handle);
|
|
1989
|
+
if (cancelled && forcedConsult?.handle === forcedMatch.handle) forcedConsult.sendSpeechPrompt = false;
|
|
1990
|
+
return await submitFinalToolResult(cancelled ? {
|
|
1710
1991
|
status: "cancelled",
|
|
1711
1992
|
message: "OpenClaw cancelled this consult before completion. Do not restart it."
|
|
1712
|
-
}
|
|
1713
|
-
}
|
|
1714
|
-
if (forcedConsult) {
|
|
1715
|
-
if (forcedConsult.completedAt || forcedMatch.kind === "already_delivered") return await submitFinalToolResult({
|
|
1993
|
+
} : {
|
|
1716
1994
|
status: "already_delivered",
|
|
1717
1995
|
message: "OpenClaw already delivered this consult result internally. Do not repeat it."
|
|
1718
1996
|
});
|
|
1997
|
+
}
|
|
1998
|
+
if (forcedConsult && forcedMatch.kind !== "none" && forcedMatch.handle === forcedConsult.handle) {
|
|
1719
1999
|
forcedConsult.sendSpeechPrompt = false;
|
|
1720
2000
|
const result = await forcedConsult.promise.catch(buildRealtimeVoiceAgentErrorProviderResult);
|
|
1721
2001
|
if (forcedConsult.cancelled || forcedConsult.owner !== bridge || this.forcedConsultsByCallId.get(callId) !== forcedConsult) return;
|
|
1722
2002
|
return await submitFinalToolResult(result);
|
|
1723
2003
|
}
|
|
2004
|
+
if (forcedConsult && !forcedConsult.completedAt) return await submitBusyResult();
|
|
1724
2005
|
const existingNativeConsult = this.nativeConsultsInFlightByCallId.get(callId);
|
|
1725
|
-
if (existingNativeConsult) {
|
|
2006
|
+
if (existingNativeConsult?.owner === bridge && !existingNativeConsult.cancelled) {
|
|
2007
|
+
if (!bridgeCallId.trim() || existingNativeConsult.invocationId !== bridgeCallId) return await submitBusyResult();
|
|
1726
2008
|
console.log(`[voice-call] realtime tool call sharing in-flight agent consult callId=${callId} ageMs=${Date.now() - existingNativeConsult.startedAt}`);
|
|
1727
2009
|
await submitWorkingResponse();
|
|
1728
2010
|
const outcome = await waitForNativeConsult(existingNativeConsult);
|
|
@@ -1734,6 +2016,7 @@ var RealtimeCallHandler = class {
|
|
|
1734
2016
|
const { promise: consult, resolve: completeConsult } = createDeferred();
|
|
1735
2017
|
const state = {
|
|
1736
2018
|
owner: bridge,
|
|
2019
|
+
invocationId: bridgeCallId,
|
|
1737
2020
|
startedAt,
|
|
1738
2021
|
promise: consult,
|
|
1739
2022
|
cancellation,
|
|
@@ -1754,11 +2037,12 @@ var RealtimeCallHandler = class {
|
|
|
1754
2037
|
if (state.cancelled || !this.isActiveBridgeOwner(callId, bridge)) return;
|
|
1755
2038
|
await Promise.race([this.waitForConsultTranscriptSettle(callId, userTranscriptOwner, startedAt), state.cancellation]);
|
|
1756
2039
|
if (state.cancelled || !this.isActiveBridgeOwner(callId, bridge)) return;
|
|
2040
|
+
const transcript = consultTranscript?.();
|
|
1757
2041
|
const context = {
|
|
1758
|
-
partialUserTranscript:
|
|
2042
|
+
partialUserTranscript: transcript?.text,
|
|
1759
2043
|
abortSignal: abortController.signal
|
|
1760
2044
|
};
|
|
1761
|
-
state.
|
|
2045
|
+
state.transcript = transcript;
|
|
1762
2046
|
const handlerArgs = withFallbackConsultQuestion(args, context.partialUserTranscript);
|
|
1763
2047
|
return await invokeHandler(handlerArgs, context);
|
|
1764
2048
|
} catch (error) {
|
|
@@ -1771,14 +2055,15 @@ var RealtimeCallHandler = class {
|
|
|
1771
2055
|
const result = outcome.result;
|
|
1772
2056
|
const failed = logResult(result);
|
|
1773
2057
|
await submitFinalToolResult(result);
|
|
1774
|
-
if (!failed) this.
|
|
2058
|
+
if (!failed) consumeUserTranscript(this.getUserTranscriptState(callId, userTranscriptOwner), state.transcript);
|
|
1775
2059
|
return result;
|
|
1776
2060
|
} finally {
|
|
1777
2061
|
delegation?.signal?.removeEventListener("abort", state.cancel);
|
|
1778
2062
|
if (this.nativeConsultsInFlightByCallId.get(callId) === state) this.nativeConsultsInFlightByCallId.delete(callId);
|
|
1779
2063
|
}
|
|
1780
2064
|
}
|
|
1781
|
-
const
|
|
2065
|
+
const transcript = this.getUserTranscriptState(callId, userTranscriptOwner);
|
|
2066
|
+
const result = await invokeHandler(args, { partialUserTranscript: (transcript?.partial ?? transcript?.recentFinal)?.text });
|
|
1782
2067
|
logResult(result);
|
|
1783
2068
|
return await submitFinalToolResult(result);
|
|
1784
2069
|
}
|