@ouro.bot/cli 0.1.0-alpha.880 → 0.1.0-alpha.882
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/changelog.json +12 -0
- package/deploy/unraid/sanctuary.ouro/bundle-meta.json +1 -1
- package/deploy/unraid/sanctuary.xml +1 -1
- package/dist/senses/voice/call-auth.js +248 -0
- package/dist/senses/voice/process-guards.js +51 -0
- package/dist/senses/voice/realtime-tool-schema.js +195 -0
- package/dist/senses/voice/twilio-phone-runtime.js +13 -0
- package/dist/senses/voice/twilio-phone.js +628 -317
- package/dist/senses/voice-entry.js +2 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
|
@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.DEFAULT_TWILIO_MEDIA_MAX_UTTERANCE_MS = exports.DEFAULT_TWILIO_MEDIA_MIN_SPEECH_MS = exports.DEFAULT_TWILIO_MEDIA_SILENCE_END_MS = exports.DEFAULT_TWILIO_MEDIA_SPEECH_RMS_THRESHOLD = exports.DEFAULT_TWILIO_PHONE_TRANSPORT_MODE = exports.DEFAULT_TWILIO_PHONE_PLAYBACK_MODE = exports.TWILIO_PHONE_WEBHOOK_BASE_PATH = exports.DEFAULT_TWILIO_GREETING_PREBUFFER_MS = exports.DEFAULT_TWILIO_RECORD_MAX_LENGTH_SECONDS = exports.DEFAULT_TWILIO_RECORD_TIMEOUT_SECONDS = exports.DEFAULT_TWILIO_PHONE_PORT = exports.normalizeTwilioE164PhoneNumber = void 0;
|
|
36
|
+
exports.TwilioAudioStreamJobStore = exports.TwilioOpenAIRealtimeMediaStreamSession = exports.TwilioMediaStreamSession = exports.DEFAULT_TWILIO_MEDIA_MAX_UTTERANCE_MS = exports.DEFAULT_TWILIO_MEDIA_MIN_SPEECH_MS = exports.DEFAULT_TWILIO_MEDIA_SILENCE_END_MS = exports.DEFAULT_TWILIO_MEDIA_SPEECH_RMS_THRESHOLD = exports.DEFAULT_TWILIO_PHONE_TRANSPORT_MODE = exports.DEFAULT_TWILIO_PHONE_PLAYBACK_MODE = exports.TWILIO_PHONE_WEBHOOK_BASE_PATH = exports.DEFAULT_TWILIO_GREETING_PREBUFFER_MS = exports.DEFAULT_TWILIO_RECORD_MAX_LENGTH_SECONDS = exports.DEFAULT_TWILIO_RECORD_TIMEOUT_SECONDS = exports.DEFAULT_TWILIO_PHONE_PORT = exports.normalizeTwilioE164PhoneNumber = exports.PendingVoiceCalls = void 0;
|
|
37
37
|
exports.resolveTwilioPhoneAgentRoot = resolveTwilioPhoneAgentRoot;
|
|
38
38
|
exports.normalizeTwilioPhoneBasePath = normalizeTwilioPhoneBasePath;
|
|
39
39
|
exports.normalizeTwilioPhonePlaybackMode = normalizeTwilioPhonePlaybackMode;
|
|
@@ -45,10 +45,12 @@ exports.openAISipWebhookUrl = openAISipWebhookUrl;
|
|
|
45
45
|
exports.twilioOutboundCallWebhookUrl = twilioOutboundCallWebhookUrl;
|
|
46
46
|
exports.twilioOutboundCallStatusCallbackUrl = twilioOutboundCallStatusCallbackUrl;
|
|
47
47
|
exports.twilioOutboundCallAmdCallbackUrl = twilioOutboundCallAmdCallbackUrl;
|
|
48
|
+
exports.wsAsVoiceSessionSocket = wsAsVoiceSessionSocket;
|
|
48
49
|
exports.twilioPhoneVoiceSessionKey = twilioPhoneVoiceSessionKey;
|
|
49
50
|
exports.outboundCallAnsweredPrompt = outboundCallAnsweredPrompt;
|
|
50
51
|
exports.computeOpenAIWebhookSignature = computeOpenAIWebhookSignature;
|
|
51
52
|
exports.validateOpenAIWebhookSignature = validateOpenAIWebhookSignature;
|
|
53
|
+
exports.parseToolArguments = parseToolArguments;
|
|
52
54
|
exports.computeTwilioSignature = computeTwilioSignature;
|
|
53
55
|
exports.validateTwilioSignature = validateTwilioSignature;
|
|
54
56
|
exports.twilioRecordingMediaUrl = twilioRecordingMediaUrl;
|
|
@@ -73,12 +75,16 @@ const tools_1 = require("../../repertoire/tools");
|
|
|
73
75
|
const tool_arguments_1 = require("../../repertoire/tool-arguments");
|
|
74
76
|
const config_1 = require("../../heart/config");
|
|
75
77
|
const runtime_1 = require("../../nerves/runtime");
|
|
78
|
+
const realtime_tool_schema_1 = require("./realtime-tool-schema");
|
|
76
79
|
const playback_1 = require("./playback");
|
|
77
80
|
const transcript_1 = require("./transcript");
|
|
78
81
|
const turn_1 = require("./turn");
|
|
79
82
|
const phone_1 = require("./phone");
|
|
80
83
|
const audio_playback_1 = require("./audio-playback");
|
|
81
84
|
const floor_controller_1 = require("./floor-controller");
|
|
85
|
+
const call_auth_1 = require("./call-auth");
|
|
86
|
+
var call_auth_2 = require("./call-auth");
|
|
87
|
+
Object.defineProperty(exports, "PendingVoiceCalls", { enumerable: true, get: function () { return call_auth_2.PendingVoiceCalls; } });
|
|
82
88
|
var phone_2 = require("./phone");
|
|
83
89
|
Object.defineProperty(exports, "normalizeTwilioE164PhoneNumber", { enumerable: true, get: function () { return phone_2.normalizeTwilioE164PhoneNumber; } });
|
|
84
90
|
exports.DEFAULT_TWILIO_PHONE_PORT = 18910;
|
|
@@ -251,11 +257,8 @@ function sayTwiml(message) {
|
|
|
251
257
|
function playTwiml(url) {
|
|
252
258
|
return `<Play>${escapeXml(url)}</Play>`;
|
|
253
259
|
}
|
|
254
|
-
function
|
|
255
|
-
|
|
256
|
-
if (!trimmed)
|
|
257
|
-
return "";
|
|
258
|
-
return `<Parameter name="${escapeXml(name)}" value="${escapeXml(trimmed)}" />`;
|
|
260
|
+
function tokenParameterTwiml(token) {
|
|
261
|
+
return `<Parameter name="OuroToken" value="${escapeXml(token)}" />`;
|
|
259
262
|
}
|
|
260
263
|
function websocketRouteUrl(publicBaseUrl, route) {
|
|
261
264
|
const url = new URL(route, publicBaseUrl);
|
|
@@ -266,22 +269,62 @@ function websocketRouteUrl(publicBaseUrl, route) {
|
|
|
266
269
|
url.protocol = "wss:";
|
|
267
270
|
return url.toString();
|
|
268
271
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
...customParams,
|
|
277
|
-
GreetingJobId: greetingJobId,
|
|
278
|
-
};
|
|
272
|
+
const STREAM_FAILURE_LINE = "Sorry, I couldn't connect this call. Please try again.";
|
|
273
|
+
function streamFailureTwiml() {
|
|
274
|
+
return `${sayTwiml(STREAM_FAILURE_LINE)}<Hangup />`;
|
|
275
|
+
}
|
|
276
|
+
function mediaStreamTwiml(options, basePath, token) {
|
|
277
|
+
const streamUrl = websocketRouteUrl(options.publicBaseUrl, `${basePath}/media-stream`);
|
|
278
|
+
const endedUrl = routeUrl(options.publicBaseUrl, `${basePath}/stream-ended`);
|
|
279
279
|
return [
|
|
280
|
-
`<Connect><Stream url="${escapeXml(streamUrl)}">`,
|
|
281
|
-
|
|
280
|
+
`<Connect action="${escapeXml(endedUrl)}" method="POST"><Stream url="${escapeXml(streamUrl)}">`,
|
|
281
|
+
tokenParameterTwiml(token),
|
|
282
282
|
`</Stream></Connect>`,
|
|
283
283
|
].join("");
|
|
284
284
|
}
|
|
285
|
+
/**
|
|
286
|
+
* Record an admitted call and return the TwiML that connects its media stream, carrying a
|
|
287
|
+
* single-use token. Without the Twilio auth token (the token secret) or a CallSid the call is
|
|
288
|
+
* not admitted and the caller hears the failure line.
|
|
289
|
+
*/
|
|
290
|
+
function admitMediaStreamCall(options, basePath, pending, identity) {
|
|
291
|
+
const secret = options.twilioAuthToken?.trim();
|
|
292
|
+
if (!secret || !identity.callSid) {
|
|
293
|
+
(0, runtime_1.emitNervesEvent)({
|
|
294
|
+
level: "warn",
|
|
295
|
+
component: "senses",
|
|
296
|
+
event: "senses.voice_media_stream_not_admitted",
|
|
297
|
+
message: "could not admit a Twilio call to a media stream: auth token or CallSid missing",
|
|
298
|
+
meta: { agentName: options.agentName, hasAuthToken: String(Boolean(secret)), hasCallSid: String(Boolean(identity.callSid)) },
|
|
299
|
+
});
|
|
300
|
+
return streamFailureTwiml();
|
|
301
|
+
}
|
|
302
|
+
const nonce = (0, call_auth_1.newVoiceCallNonce)();
|
|
303
|
+
pending.record(identity, nonce);
|
|
304
|
+
return mediaStreamTwiml(options, basePath, (0, call_auth_1.mintVoiceCallToken)({
|
|
305
|
+
secret,
|
|
306
|
+
purpose: "stream",
|
|
307
|
+
agentName: options.agentName,
|
|
308
|
+
callSid: identity.callSid,
|
|
309
|
+
direction: identity.direction,
|
|
310
|
+
outboundId: identity.outboundId,
|
|
311
|
+
nowMs: Date.now(),
|
|
312
|
+
nonce,
|
|
313
|
+
}));
|
|
314
|
+
}
|
|
315
|
+
function wsAsVoiceSessionSocket(ws) {
|
|
316
|
+
return {
|
|
317
|
+
send: (data) => ws.send(data),
|
|
318
|
+
close: () => {
|
|
319
|
+
if (ws.readyState === ws_1.WebSocket.OPEN || ws.readyState === ws_1.WebSocket.CONNECTING)
|
|
320
|
+
ws.close();
|
|
321
|
+
},
|
|
322
|
+
isOpen: () => ws.readyState === ws_1.WebSocket.OPEN,
|
|
323
|
+
on: (event, callback) => {
|
|
324
|
+
ws.on(event, callback);
|
|
325
|
+
},
|
|
326
|
+
};
|
|
327
|
+
}
|
|
285
328
|
/* v8 ignore start -- private SIP URI query permutations are exercised through bridge routes; the queryless helper shape is not externally reachable today @preserve */
|
|
286
329
|
function openAISipUri(options, customHeaders = {}) {
|
|
287
330
|
const projectId = options.openaiSip?.projectId?.trim();
|
|
@@ -360,13 +403,23 @@ function friendIdFromCaller(from, callSid) {
|
|
|
360
403
|
function voiceFriendId(options, from, callSid) {
|
|
361
404
|
return options.defaultFriendId?.trim() || friendIdFromCaller(from, callSid);
|
|
362
405
|
}
|
|
406
|
+
const SAFE_VOICE_FRIEND_ID = /^[A-Za-z0-9._-]{1,128}$/;
|
|
407
|
+
function isSafeVoiceFriendId(id) {
|
|
408
|
+
return SAFE_VOICE_FRIEND_ID.test(id) && id !== "." && id !== "..";
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Resolve who is on a call. `friendId` may only come from server-side state (the outbound job
|
|
412
|
+
* record), is validated as a safe id before any store read, and must match a record id exactly
|
|
413
|
+
* (the store's display-name fallback is never used for voice). Everything else resolves from the
|
|
414
|
+
* phone number, or from the call itself when there is none.
|
|
415
|
+
*/
|
|
363
416
|
async function resolveVoiceFriendContext(options, input) {
|
|
364
417
|
const agentRoot = resolveTwilioPhoneAgentRoot(options);
|
|
365
418
|
const friendStore = new friends_1.FileFriendStore(path.join(agentRoot, "friends"));
|
|
366
419
|
const explicitFriendId = input.friendId?.trim();
|
|
367
|
-
if (explicitFriendId) {
|
|
420
|
+
if (explicitFriendId && isSafeVoiceFriendId(explicitFriendId)) {
|
|
368
421
|
const existing = await friendStore.get(explicitFriendId);
|
|
369
|
-
if (existing) {
|
|
422
|
+
if (existing && existing.id === explicitFriendId) {
|
|
370
423
|
return {
|
|
371
424
|
friendId: existing.id,
|
|
372
425
|
friendStore,
|
|
@@ -376,7 +429,7 @@ async function resolveVoiceFriendContext(options, input) {
|
|
|
376
429
|
}
|
|
377
430
|
const remotePhone = (0, phone_1.normalizeTwilioE164PhoneNumber)(input.remotePhone);
|
|
378
431
|
const provider = remotePhone ? "imessage-handle" : "local";
|
|
379
|
-
const externalId = remotePhone ||
|
|
432
|
+
const externalId = remotePhone || friendIdFromCaller("", input.callSid);
|
|
380
433
|
const resolver = new friends_1.FriendResolver(friendStore, {
|
|
381
434
|
provider,
|
|
382
435
|
externalId,
|
|
@@ -386,6 +439,25 @@ async function resolveVoiceFriendContext(options, input) {
|
|
|
386
439
|
const resolved = await resolver.resolve();
|
|
387
440
|
return { friendId: resolved.friend.id, friendStore, resolved };
|
|
388
441
|
}
|
|
442
|
+
/**
|
|
443
|
+
* The friend id for an inbound call: resolved from the signed caller ID, never from
|
|
444
|
+
* `defaultFriendId`. Falls back to a caller-derived id if the friend store is unreadable.
|
|
445
|
+
*/
|
|
446
|
+
async function inboundVoiceFriendId(options, from, callSid) {
|
|
447
|
+
try {
|
|
448
|
+
return (await resolveVoiceFriendContext(options, { remotePhone: from || undefined, callSid })).friendId;
|
|
449
|
+
}
|
|
450
|
+
catch (error) {
|
|
451
|
+
(0, runtime_1.emitNervesEvent)({
|
|
452
|
+
level: "warn",
|
|
453
|
+
component: "senses",
|
|
454
|
+
event: "senses.voice_friend_resolve_error",
|
|
455
|
+
message: "could not resolve the inbound caller to a friend record",
|
|
456
|
+
meta: { agentName: options.agentName, callSid: safeSegment(callSid), error: errorMessage(error) },
|
|
457
|
+
});
|
|
458
|
+
return friendIdFromCaller(from, callSid);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
389
461
|
function phoneIdentitySegment(input) {
|
|
390
462
|
const phoneish = input.replace(/[^0-9A-Za-z]+/g, "");
|
|
391
463
|
return phoneish || safeSegment(input);
|
|
@@ -454,22 +526,22 @@ function buildNoSpeechTranscript(utteranceId) {
|
|
|
454
526
|
source: "loopback",
|
|
455
527
|
});
|
|
456
528
|
}
|
|
457
|
-
/* v8 ignore start -- ws RawData variants are provider/runtime transport shapes; session tests cover valid and invalid stream behavior @preserve */
|
|
458
529
|
function parseTwilioMediaStreamMessage(raw) {
|
|
459
|
-
const text =
|
|
460
|
-
? raw
|
|
461
|
-
:
|
|
462
|
-
?
|
|
463
|
-
:
|
|
530
|
+
const text = typeof raw === "string"
|
|
531
|
+
? raw
|
|
532
|
+
: Buffer.isBuffer(raw)
|
|
533
|
+
? raw.toString("utf8")
|
|
534
|
+
: Array.isArray(raw)
|
|
535
|
+
? Buffer.concat(raw).toString("utf8")
|
|
536
|
+
: Buffer.from(raw).toString("utf8");
|
|
464
537
|
try {
|
|
465
538
|
const parsed = JSON.parse(text);
|
|
466
539
|
return parsed && typeof parsed === "object" ? parsed : null;
|
|
467
540
|
}
|
|
468
|
-
catch {
|
|
541
|
+
catch {
|
|
469
542
|
return null;
|
|
470
543
|
}
|
|
471
544
|
}
|
|
472
|
-
/* v8 ignore stop */
|
|
473
545
|
function stringField(value) {
|
|
474
546
|
return typeof value === "string" ? value.trim() : "";
|
|
475
547
|
}
|
|
@@ -590,9 +662,9 @@ async function transcribeRecordingOrNoSpeech(options) {
|
|
|
590
662
|
throw error;
|
|
591
663
|
}
|
|
592
664
|
}
|
|
593
|
-
/* v8 ignore start -- legacy cascade Media Streams socket loop is covered by bridge-level WebSocket tests; per-event socket/error permutations are transport-runtime edges @preserve */
|
|
594
665
|
class TwilioMediaStreamSession {
|
|
595
|
-
|
|
666
|
+
socket;
|
|
667
|
+
identity;
|
|
596
668
|
options;
|
|
597
669
|
mediaGreetingJobs;
|
|
598
670
|
lifecycle;
|
|
@@ -609,6 +681,7 @@ class TwilioMediaStreamSession {
|
|
|
609
681
|
playbackGeneration = 0;
|
|
610
682
|
playbackActive = false;
|
|
611
683
|
closed = false;
|
|
684
|
+
started = false;
|
|
612
685
|
inSpeech = false;
|
|
613
686
|
currentFrames = [];
|
|
614
687
|
preRollFrames = [];
|
|
@@ -625,8 +698,9 @@ class TwilioMediaStreamSession {
|
|
|
625
698
|
silenceEndFrames;
|
|
626
699
|
minSpeechFrames;
|
|
627
700
|
maxUtteranceFrames;
|
|
628
|
-
constructor(
|
|
629
|
-
this.
|
|
701
|
+
constructor(socket, identity, options, mediaGreetingJobs, lifecycle) {
|
|
702
|
+
this.socket = socket;
|
|
703
|
+
this.identity = identity;
|
|
630
704
|
this.options = options;
|
|
631
705
|
this.mediaGreetingJobs = mediaGreetingJobs;
|
|
632
706
|
this.lifecycle = lifecycle;
|
|
@@ -636,9 +710,9 @@ class TwilioMediaStreamSession {
|
|
|
636
710
|
this.maxUtteranceFrames = frameLimitForMs(options.mediaMaxUtteranceMs ?? exports.DEFAULT_TWILIO_MEDIA_MAX_UTTERANCE_MS);
|
|
637
711
|
}
|
|
638
712
|
attach() {
|
|
639
|
-
this.
|
|
640
|
-
this.
|
|
641
|
-
this.
|
|
713
|
+
this.socket.on("message", (raw) => this.handleRawMessage(raw));
|
|
714
|
+
this.socket.on("close", () => this.close());
|
|
715
|
+
this.socket.on("error", (error) => {
|
|
642
716
|
(0, runtime_1.emitNervesEvent)({
|
|
643
717
|
level: "error",
|
|
644
718
|
component: "senses",
|
|
@@ -649,11 +723,10 @@ class TwilioMediaStreamSession {
|
|
|
649
723
|
});
|
|
650
724
|
}
|
|
651
725
|
end() {
|
|
652
|
-
|
|
653
|
-
this.ws.close();
|
|
654
|
-
}
|
|
726
|
+
this.socket.close();
|
|
655
727
|
this.close();
|
|
656
728
|
}
|
|
729
|
+
/** Feed a raw Twilio message that arrived before this session attached (the bridge holds the first `start`). */
|
|
657
730
|
handleRawMessage(raw) {
|
|
658
731
|
const message = parseTwilioMediaStreamMessage(raw);
|
|
659
732
|
if (!message) {
|
|
@@ -684,54 +757,65 @@ class TwilioMediaStreamSession {
|
|
|
684
757
|
}
|
|
685
758
|
}
|
|
686
759
|
async handleStart(start) {
|
|
760
|
+
if (this.started)
|
|
761
|
+
return;
|
|
762
|
+
this.started = true;
|
|
687
763
|
this.streamSid = stringField(start?.streamSid);
|
|
688
|
-
this.callSid =
|
|
689
|
-
|
|
690
|
-
this.
|
|
691
|
-
this.
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
this.
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
this.
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
764
|
+
this.callSid = this.identity.callSid;
|
|
765
|
+
this.direction = this.identity.direction;
|
|
766
|
+
this.outboundId = this.identity.outboundId ?? "";
|
|
767
|
+
this.from = this.identity.from;
|
|
768
|
+
this.to = this.identity.to;
|
|
769
|
+
try {
|
|
770
|
+
this.friendId = this.direction === "outbound" && this.identity.friendId && isSafeVoiceFriendId(this.identity.friendId)
|
|
771
|
+
? this.identity.friendId
|
|
772
|
+
: this.direction === "outbound"
|
|
773
|
+
? voiceFriendId(this.options, this.from, this.callSid)
|
|
774
|
+
: await inboundVoiceFriendId(this.options, this.from, this.callSid);
|
|
775
|
+
this.sessionKey = twilioPhoneVoiceSessionKey({
|
|
776
|
+
defaultFriendId: this.friendId,
|
|
777
|
+
from: this.from,
|
|
778
|
+
to: this.to,
|
|
779
|
+
callSid: this.callSid,
|
|
780
|
+
});
|
|
781
|
+
this.lifecycle?.onIdentityChange?.(this, { callSid: this.callSid, outboundId: this.outboundId });
|
|
782
|
+
this.callDir = path.join(this.options.outputDir, safeSegment(this.callSid));
|
|
783
|
+
await fs.mkdir(this.callDir, { recursive: true });
|
|
784
|
+
(0, runtime_1.emitNervesEvent)({
|
|
785
|
+
component: "senses",
|
|
786
|
+
event: "senses.voice_twilio_media_start",
|
|
787
|
+
message: "Twilio Media Stream started",
|
|
788
|
+
meta: {
|
|
789
|
+
agentName: this.options.agentName,
|
|
790
|
+
callSid: safeSegment(this.callSid),
|
|
791
|
+
streamSid: safeSegment(this.streamSid || "stream"),
|
|
792
|
+
sessionKey: this.sessionKey,
|
|
793
|
+
},
|
|
794
|
+
});
|
|
795
|
+
const greetingJobId = this.identity.greetingJobId ?? "";
|
|
796
|
+
const greetingJob = greetingJobId
|
|
797
|
+
? this.mediaGreetingJobs.get(safeSegment(this.callSid), greetingJobId)
|
|
798
|
+
: null;
|
|
799
|
+
if (greetingJob) {
|
|
800
|
+
this.enqueueGreetingJob(greetingJobId, greetingJob);
|
|
801
|
+
}
|
|
802
|
+
else {
|
|
803
|
+
this.enqueuePrompt({
|
|
804
|
+
utteranceId: `twilio-${safeSegment(this.callSid)}-connected`,
|
|
805
|
+
promptText: callConnectedPrompt({ From: this.from, To: this.to }),
|
|
806
|
+
wasBargeIn: false,
|
|
807
|
+
});
|
|
808
|
+
}
|
|
728
809
|
}
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
810
|
+
catch (error) {
|
|
811
|
+
(0, runtime_1.emitNervesEvent)({
|
|
812
|
+
level: "error",
|
|
813
|
+
component: "senses",
|
|
814
|
+
event: "senses.voice_twilio_media_start_error",
|
|
815
|
+
message: "Twilio Media Stream could not start",
|
|
816
|
+
meta: { agentName: this.options.agentName, callSid: safeSegment(this.callSid), error: errorMessage(error) },
|
|
734
817
|
});
|
|
818
|
+
this.end();
|
|
735
819
|
}
|
|
736
820
|
}
|
|
737
821
|
handleMedia(media) {
|
|
@@ -829,6 +913,7 @@ class TwilioMediaStreamSession {
|
|
|
829
913
|
});
|
|
830
914
|
this.enqueueTurn(transcript, input.wasBargeIn);
|
|
831
915
|
}
|
|
916
|
+
/* v8 ignore start -- unchanged queue error logging; turn failures are observed through the session turn tests @preserve */
|
|
832
917
|
enqueueUtterance(utterance) {
|
|
833
918
|
this.turnQueue = this.turnQueue
|
|
834
919
|
.catch(() => undefined)
|
|
@@ -886,6 +971,7 @@ class TwilioMediaStreamSession {
|
|
|
886
971
|
});
|
|
887
972
|
});
|
|
888
973
|
}
|
|
974
|
+
/* v8 ignore stop */
|
|
889
975
|
async processUtterance(utterance) {
|
|
890
976
|
await fs.mkdir(this.callDir, { recursive: true });
|
|
891
977
|
const inputPath = path.join(this.callDir, `${safeSegment(utterance.utteranceId)}.wav`);
|
|
@@ -897,11 +983,13 @@ class TwilioMediaStreamSession {
|
|
|
897
983
|
});
|
|
898
984
|
if (this.direction === "outbound" && isVoicemailMenuTranscript(transcript.text)) {
|
|
899
985
|
if (this.outboundId) {
|
|
986
|
+
/* v8 ignore start -- the voicemail status write is best-effort and must not block hanging up @preserve */
|
|
900
987
|
await updateTwilioOutboundCallJob(this.options.outputDir, this.outboundId, {
|
|
901
988
|
status: "voicemail",
|
|
902
989
|
answeredBy: "voicemail_menu",
|
|
903
990
|
transportCallSid: this.callSid,
|
|
904
991
|
}).catch(() => null);
|
|
992
|
+
/* v8 ignore stop */
|
|
905
993
|
}
|
|
906
994
|
(0, runtime_1.emitNervesEvent)({
|
|
907
995
|
component: "senses",
|
|
@@ -913,7 +1001,7 @@ class TwilioMediaStreamSession {
|
|
|
913
1001
|
outboundId: safeSegment(this.outboundId || "unknown"),
|
|
914
1002
|
},
|
|
915
1003
|
});
|
|
916
|
-
this.
|
|
1004
|
+
this.socket.close();
|
|
917
1005
|
this.close();
|
|
918
1006
|
return;
|
|
919
1007
|
}
|
|
@@ -938,19 +1026,15 @@ class TwilioMediaStreamSession {
|
|
|
938
1026
|
const generation = this.startPlayback();
|
|
939
1027
|
const turn = await (0, turn_1.runVoiceLoopbackTurn)({
|
|
940
1028
|
agentName: this.options.agentName,
|
|
941
|
-
friendId: this.friendId
|
|
942
|
-
sessionKey: this.sessionKey
|
|
943
|
-
defaultFriendId: this.options.defaultFriendId,
|
|
944
|
-
from: this.from,
|
|
945
|
-
to: this.to,
|
|
946
|
-
callSid: this.callSid,
|
|
947
|
-
}),
|
|
1029
|
+
friendId: this.friendId,
|
|
1030
|
+
sessionKey: this.sessionKey,
|
|
948
1031
|
transcript,
|
|
949
1032
|
tts: this.options.tts,
|
|
950
1033
|
runSenseTurn: this.options.runSenseTurn,
|
|
951
1034
|
onAudioChunk: (chunk) => this.sendAudioChunk(chunk, generation),
|
|
952
1035
|
voiceCall: {
|
|
953
1036
|
requestEnd: (reason) => this.requestHangupAfterPlayback(reason),
|
|
1037
|
+
/* v8 ignore next -- tool audio playback is unchanged and exercised through the Realtime and SIP tool tests @preserve */
|
|
954
1038
|
playAudio: (request) => this.playPreparedAudio(request),
|
|
955
1039
|
},
|
|
956
1040
|
});
|
|
@@ -962,11 +1046,13 @@ class TwilioMediaStreamSession {
|
|
|
962
1046
|
this.sendAudioChunk(delivery.audio, generation);
|
|
963
1047
|
}
|
|
964
1048
|
}
|
|
1049
|
+
/* v8 ignore start -- runVoiceLoopbackTurn always yields at least one delivered segment (it falls back to a whole-reply synthesis); kept as a guard @preserve */
|
|
965
1050
|
if (deliveries.length === 0) {
|
|
966
1051
|
this.playbackActive = false;
|
|
967
1052
|
this.completeHangupIfRequested("no_playback");
|
|
968
1053
|
return;
|
|
969
1054
|
}
|
|
1055
|
+
/* v8 ignore stop */
|
|
970
1056
|
this.sendMark(generation, transcript.utteranceId);
|
|
971
1057
|
if (this.hangupRequested)
|
|
972
1058
|
this.armHangupFallback();
|
|
@@ -983,6 +1069,7 @@ class TwilioMediaStreamSession {
|
|
|
983
1069
|
},
|
|
984
1070
|
});
|
|
985
1071
|
}
|
|
1072
|
+
/* v8 ignore start -- unchanged prebuffered greeting playback; the bridge greeting tests cover the path @preserve */
|
|
986
1073
|
async streamGreetingJob(jobId, job) {
|
|
987
1074
|
if (this.closed || !this.streamSid)
|
|
988
1075
|
return;
|
|
@@ -1025,6 +1112,7 @@ class TwilioMediaStreamSession {
|
|
|
1025
1112
|
},
|
|
1026
1113
|
});
|
|
1027
1114
|
}
|
|
1115
|
+
/* v8 ignore stop */
|
|
1028
1116
|
startPlayback() {
|
|
1029
1117
|
this.playbackGeneration += 1;
|
|
1030
1118
|
this.playbackActive = true;
|
|
@@ -1032,9 +1120,9 @@ class TwilioMediaStreamSession {
|
|
|
1032
1120
|
return this.playbackGeneration;
|
|
1033
1121
|
}
|
|
1034
1122
|
sendAudioChunk(chunk, generation) {
|
|
1035
|
-
if (this.closed || generation !== this.playbackGeneration || !this.streamSid || this.
|
|
1123
|
+
if (this.closed || generation !== this.playbackGeneration || !this.streamSid || !this.socket.isOpen())
|
|
1036
1124
|
return;
|
|
1037
|
-
this.
|
|
1125
|
+
this.socket.send(JSON.stringify({
|
|
1038
1126
|
event: "media",
|
|
1039
1127
|
streamSid: this.streamSid,
|
|
1040
1128
|
media: { payload: Buffer.from(chunk).toString("base64") },
|
|
@@ -1042,14 +1130,15 @@ class TwilioMediaStreamSession {
|
|
|
1042
1130
|
this.playbackBytesByGeneration.set(generation, (this.playbackBytesByGeneration.get(generation) ?? 0) + chunk.byteLength);
|
|
1043
1131
|
}
|
|
1044
1132
|
sendMark(generation, utteranceId) {
|
|
1045
|
-
if (this.closed || generation !== this.playbackGeneration || !this.streamSid || this.
|
|
1133
|
+
if (this.closed || generation !== this.playbackGeneration || !this.streamSid || !this.socket.isOpen())
|
|
1046
1134
|
return;
|
|
1047
|
-
this.
|
|
1135
|
+
this.socket.send(JSON.stringify({
|
|
1048
1136
|
event: "mark",
|
|
1049
1137
|
streamSid: this.streamSid,
|
|
1050
1138
|
mark: { name: `voice-${generation}-${safeSegment(utteranceId)}` },
|
|
1051
1139
|
}));
|
|
1052
1140
|
}
|
|
1141
|
+
/* v8 ignore start -- unchanged cascade tool-audio playback @preserve */
|
|
1053
1142
|
async playPreparedAudio(request) {
|
|
1054
1143
|
const prepared = await (0, audio_playback_1.prepareVoiceCallAudio)(request, {
|
|
1055
1144
|
agentRoot: resolveTwilioPhoneAgentRoot(this.options),
|
|
@@ -1077,13 +1166,14 @@ class TwilioMediaStreamSession {
|
|
|
1077
1166
|
});
|
|
1078
1167
|
return { label: prepared.label, durationMs: prepared.durationMs };
|
|
1079
1168
|
}
|
|
1169
|
+
/* v8 ignore stop */
|
|
1080
1170
|
interruptPlayback() {
|
|
1081
|
-
if (!this.playbackActive || !this.streamSid || this.
|
|
1171
|
+
if (!this.playbackActive || !this.streamSid || !this.socket.isOpen())
|
|
1082
1172
|
return false;
|
|
1083
1173
|
this.cancelPendingHangup("barge_in");
|
|
1084
1174
|
this.playbackGeneration += 1;
|
|
1085
1175
|
this.playbackActive = false;
|
|
1086
|
-
this.
|
|
1176
|
+
this.socket.send(JSON.stringify({ event: "clear", streamSid: this.streamSid }));
|
|
1087
1177
|
(0, runtime_1.emitNervesEvent)({
|
|
1088
1178
|
component: "senses",
|
|
1089
1179
|
event: "senses.voice_twilio_media_barge_in",
|
|
@@ -1092,6 +1182,7 @@ class TwilioMediaStreamSession {
|
|
|
1092
1182
|
});
|
|
1093
1183
|
return true;
|
|
1094
1184
|
}
|
|
1185
|
+
/* v8 ignore start -- unchanged cascade hangup coordination and timers @preserve */
|
|
1095
1186
|
requestHangupAfterPlayback(reason) {
|
|
1096
1187
|
if (this.closed)
|
|
1097
1188
|
return;
|
|
@@ -1144,6 +1235,7 @@ class TwilioMediaStreamSession {
|
|
|
1144
1235
|
}, TWILIO_MEDIA_HANGUP_FALLBACK_MS);
|
|
1145
1236
|
this.hangupFallbackTimer.unref?.();
|
|
1146
1237
|
}
|
|
1238
|
+
/* v8 ignore stop */
|
|
1147
1239
|
clearHangupFallback() {
|
|
1148
1240
|
if (!this.hangupFallbackTimer)
|
|
1149
1241
|
return;
|
|
@@ -1151,7 +1243,7 @@ class TwilioMediaStreamSession {
|
|
|
1151
1243
|
this.hangupFallbackTimer = null;
|
|
1152
1244
|
}
|
|
1153
1245
|
}
|
|
1154
|
-
|
|
1246
|
+
exports.TwilioMediaStreamSession = TwilioMediaStreamSession;
|
|
1155
1247
|
const OPENAI_REALTIME_DEFAULT_MODEL = "gpt-realtime-2";
|
|
1156
1248
|
const OPENAI_REALTIME_DEFAULT_VOICE = "cedar";
|
|
1157
1249
|
const OPENAI_REALTIME_DEFAULT_TRANSCRIPTION_MODEL = "gpt-realtime-whisper";
|
|
@@ -1253,40 +1345,89 @@ function openAISipHeaderValue(headers, name) {
|
|
|
1253
1345
|
}
|
|
1254
1346
|
return "";
|
|
1255
1347
|
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
if (!trimmed)
|
|
1259
|
-
return "";
|
|
1260
|
-
const tel = trimmed.match(/tel:([^>;]+)/i)?.[1];
|
|
1261
|
-
if (tel)
|
|
1262
|
-
return tel.trim();
|
|
1263
|
-
const sip = trimmed.match(/sip:([^@>;]+)/i)?.[1];
|
|
1264
|
-
if (sip)
|
|
1265
|
-
return sip.trim();
|
|
1266
|
-
const bracketed = trimmed.match(/<([^>]+)>/)?.[1];
|
|
1267
|
-
return bracketed?.trim() || trimmed;
|
|
1268
|
-
}
|
|
1269
|
-
function openAISipCallMetadata(event) {
|
|
1348
|
+
/* v8 ignore stop */
|
|
1349
|
+
function openAISipCallId(event) {
|
|
1270
1350
|
const data = event.data;
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
const
|
|
1351
|
+
return data && typeof data === "object" ? stringField(data.call_id) : "";
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* Admit an incoming OpenAI SIP call only if it carries a token our signed Twilio webhook minted
|
|
1355
|
+
* for this agent and call. Caller, line, direction, outbound id, friend and reason come from the
|
|
1356
|
+
* record that webhook wrote, never from SIP headers.
|
|
1357
|
+
*/
|
|
1358
|
+
function admitOpenAISipCall(options, pending, event) {
|
|
1359
|
+
const callId = openAISipCallId(event);
|
|
1360
|
+
const headers = openAISipHeaders(event.data?.sip_headers);
|
|
1361
|
+
const secret = options.twilioAuthToken?.trim();
|
|
1362
|
+
if (!secret)
|
|
1363
|
+
return { ok: false, reason: "no_secret" };
|
|
1364
|
+
if (openAISipHeaderValue(headers, "X-Ouro-Agent") !== options.agentName)
|
|
1365
|
+
return { ok: false, reason: "wrong_agent" };
|
|
1366
|
+
const callSid = openAISipHeaderValue(headers, "X-Ouro-Call-Sid");
|
|
1367
|
+
const verified = (0, call_auth_1.verifyVoiceCallToken)({
|
|
1368
|
+
secret,
|
|
1369
|
+
purpose: "sip",
|
|
1370
|
+
token: openAISipHeaderValue(headers, "X-Ouro-Call-Token"),
|
|
1371
|
+
agentName: options.agentName,
|
|
1372
|
+
callSid,
|
|
1373
|
+
nowMs: Date.now(),
|
|
1374
|
+
});
|
|
1375
|
+
if (!verified.ok)
|
|
1376
|
+
return { ok: false, reason: verified.reason };
|
|
1377
|
+
const identity = pending.consume(callSid, verified.nonce);
|
|
1378
|
+
if (!identity)
|
|
1379
|
+
return { ok: false, reason: "no_record" };
|
|
1380
|
+
if (identity.direction !== verified.direction
|
|
1381
|
+
|| (identity.outboundId ?? "") !== (verified.outboundId ?? "")
|
|
1382
|
+
|| identity.from !== (verified.from ?? "")
|
|
1383
|
+
|| identity.to !== (verified.to ?? ""))
|
|
1384
|
+
return { ok: false, reason: "record_mismatch" };
|
|
1280
1385
|
return {
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1386
|
+
ok: true,
|
|
1387
|
+
metadata: {
|
|
1388
|
+
callId,
|
|
1389
|
+
from: identity.from,
|
|
1390
|
+
to: identity.to,
|
|
1391
|
+
direction: identity.direction,
|
|
1392
|
+
outboundId: identity.outboundId ?? "",
|
|
1393
|
+
reason: identity.reason ?? "",
|
|
1394
|
+
friendId: identity.friendId ?? "",
|
|
1395
|
+
},
|
|
1288
1396
|
};
|
|
1289
1397
|
}
|
|
1398
|
+
/** Dial OpenAI SIP with a token for the signed webhook's recorded call; a call that cannot be admitted hears the failure line. */
|
|
1399
|
+
function admitSipCall(options, pending, identity) {
|
|
1400
|
+
const secret = options.twilioAuthToken?.trim();
|
|
1401
|
+
if (!secret || !identity.callSid) {
|
|
1402
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1403
|
+
level: "warn",
|
|
1404
|
+
component: "senses",
|
|
1405
|
+
event: "senses.voice_sip_call_not_admitted",
|
|
1406
|
+
message: "could not admit a Twilio call to OpenAI SIP: auth token or CallSid missing",
|
|
1407
|
+
meta: { agentName: options.agentName, hasAuthToken: String(Boolean(secret)), hasCallSid: String(Boolean(identity.callSid)) },
|
|
1408
|
+
});
|
|
1409
|
+
return streamFailureTwiml();
|
|
1410
|
+
}
|
|
1411
|
+
const nonce = (0, call_auth_1.newVoiceCallNonce)();
|
|
1412
|
+
pending.record(identity, nonce);
|
|
1413
|
+
return openAISipDialTwiml(options, {
|
|
1414
|
+
"X-Ouro-Agent": options.agentName,
|
|
1415
|
+
"X-Ouro-Call-Sid": identity.callSid,
|
|
1416
|
+
"X-Ouro-Call-Token": (0, call_auth_1.mintVoiceCallToken)({
|
|
1417
|
+
secret,
|
|
1418
|
+
purpose: "sip",
|
|
1419
|
+
agentName: options.agentName,
|
|
1420
|
+
callSid: identity.callSid,
|
|
1421
|
+
direction: identity.direction,
|
|
1422
|
+
outboundId: identity.outboundId,
|
|
1423
|
+
from: identity.from,
|
|
1424
|
+
to: identity.to,
|
|
1425
|
+
nowMs: Date.now(),
|
|
1426
|
+
nonce,
|
|
1427
|
+
}),
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
/* v8 ignore start -- unchanged SIP prompt and Realtime config helpers @preserve */
|
|
1290
1431
|
function openAISipCallConnectedPrompt(metadata, voiceStyle) {
|
|
1291
1432
|
const styleLine = voiceStyle?.trim()
|
|
1292
1433
|
? `Phone voice target for this first turn: ${voiceStyle.trim()}`
|
|
@@ -1311,15 +1452,6 @@ function openAISipCallConnectedPrompt(metadata, voiceStyle) {
|
|
|
1311
1452
|
"Respond through the voice channel as yourself. Greet the caller naturally and briefly, then invite them to speak.",
|
|
1312
1453
|
].filter(Boolean).join("\n");
|
|
1313
1454
|
}
|
|
1314
|
-
function openAISipResponseHeaders(params, extra = {}) {
|
|
1315
|
-
return {
|
|
1316
|
-
"X-Ouro-Agent": params.Agent,
|
|
1317
|
-
"X-Ouro-Direction": params.Direction,
|
|
1318
|
-
"X-Ouro-From": params.From,
|
|
1319
|
-
"X-Ouro-To": params.To,
|
|
1320
|
-
...extra,
|
|
1321
|
-
};
|
|
1322
|
-
}
|
|
1323
1455
|
function boundedNumber(value, min, max) {
|
|
1324
1456
|
if (value === undefined || !Number.isFinite(value))
|
|
1325
1457
|
return undefined;
|
|
@@ -1389,31 +1521,15 @@ async function prepareRealtimeToolContext(context) {
|
|
|
1389
1521
|
}
|
|
1390
1522
|
return selectRealtimeToolContext(context, mcpManager);
|
|
1391
1523
|
}
|
|
1392
|
-
function realtimeToolsFromSelection(selection) {
|
|
1393
|
-
return selection.ordinary.map(({ tool }) =>
|
|
1394
|
-
type: "function",
|
|
1395
|
-
name: tool.function.name,
|
|
1396
|
-
...(tool.function.description ? { description: tool.function.description } : {}),
|
|
1397
|
-
parameters: tool.function.parameters ?? { type: "object", properties: {} },
|
|
1398
|
-
}));
|
|
1399
|
-
}
|
|
1400
|
-
/* v8 ignore start -- unchanged transport route parsing @preserve */
|
|
1401
|
-
function mediaStreamRequestedConversationEngine(url) {
|
|
1402
|
-
if (!url)
|
|
1403
|
-
return undefined;
|
|
1404
|
-
try {
|
|
1405
|
-
const parsed = new URL(url, "wss://localhost");
|
|
1406
|
-
const engine = parsed.searchParams.get("engine") ?? undefined;
|
|
1407
|
-
return engine ? normalizeTwilioPhoneConversationEngine(engine) : undefined;
|
|
1408
|
-
}
|
|
1409
|
-
catch {
|
|
1410
|
-
return undefined;
|
|
1411
|
-
}
|
|
1524
|
+
function realtimeToolsFromSelection(selection, advertiser) {
|
|
1525
|
+
return advertiser.tools(selection.ordinary.map(({ tool }) => tool));
|
|
1412
1526
|
}
|
|
1413
|
-
|
|
1414
|
-
function parseToolArguments(raw, name, selection) {
|
|
1527
|
+
function parseToolArguments(raw, name, selection, advertiser) {
|
|
1415
1528
|
if (typeof raw !== "string")
|
|
1416
1529
|
throw new Error("invalid tool arguments: expected a JSON object string");
|
|
1530
|
+
if (!advertiser.isAdvertised(selection.ordinary.map(({ tool }) => tool), name)) {
|
|
1531
|
+
throw new Error(`tool ${name} is not advertised in this voice session`);
|
|
1532
|
+
}
|
|
1417
1533
|
const schema = selection.ordinary.find((definition) => definition.tool.function.name === name)?.tool.function.parameters ?? {};
|
|
1418
1534
|
const validation = (0, tool_arguments_1.validateAdvertisedToolArguments)(raw, schema);
|
|
1419
1535
|
if (!validation.ok)
|
|
@@ -1534,9 +1650,9 @@ function pcmuPayloadDurationMs(payload) {
|
|
|
1534
1650
|
return Math.max(1, Math.round(byteLength / OPENAI_REALTIME_PCMS_BYTES_PER_MS));
|
|
1535
1651
|
}
|
|
1536
1652
|
/* v8 ignore stop */
|
|
1537
|
-
/* v8 ignore start -- Twilio Media Streams Realtime bridge is a fallback transport; direct SIP is the primary low-latency path and WebSocket integration tests cover representative behavior @preserve */
|
|
1538
1653
|
class TwilioOpenAIRealtimeMediaStreamSession {
|
|
1539
|
-
|
|
1654
|
+
socket;
|
|
1655
|
+
identity;
|
|
1540
1656
|
options;
|
|
1541
1657
|
lifecycle;
|
|
1542
1658
|
streamSid = "";
|
|
@@ -1550,12 +1666,18 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1550
1666
|
sessionKey = "";
|
|
1551
1667
|
sessionPath = "";
|
|
1552
1668
|
closed = false;
|
|
1669
|
+
started = false;
|
|
1553
1670
|
openaiReady = false;
|
|
1554
1671
|
greetingSent = false;
|
|
1555
1672
|
hangupRequested = false;
|
|
1556
1673
|
pendingAudioPayloads = [];
|
|
1557
1674
|
openaiWs = null;
|
|
1558
1675
|
toolContext;
|
|
1676
|
+
advertiser;
|
|
1677
|
+
get toolAdvertiser() {
|
|
1678
|
+
this.advertiser ??= (0, realtime_tool_schema_1.createRealtimeToolAdvertiser)({ agentName: this.options.agentName, callSid: safeSegment(this.callSid) });
|
|
1679
|
+
return this.advertiser;
|
|
1680
|
+
}
|
|
1559
1681
|
friendStore;
|
|
1560
1682
|
resolvedContext;
|
|
1561
1683
|
sessionMessages = [];
|
|
@@ -1583,15 +1705,16 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1583
1705
|
callerTurnSequence = 0;
|
|
1584
1706
|
gatedResponseRequestSequence = 0;
|
|
1585
1707
|
pendingGatedResponseId = null;
|
|
1586
|
-
constructor(
|
|
1587
|
-
this.
|
|
1708
|
+
constructor(socket, identity, options, lifecycle) {
|
|
1709
|
+
this.socket = socket;
|
|
1710
|
+
this.identity = identity;
|
|
1588
1711
|
this.options = options;
|
|
1589
1712
|
this.lifecycle = lifecycle;
|
|
1590
1713
|
}
|
|
1591
1714
|
attach() {
|
|
1592
|
-
this.
|
|
1593
|
-
this.
|
|
1594
|
-
this.
|
|
1715
|
+
this.socket.on("message", (raw) => this.handleRawMessage(raw));
|
|
1716
|
+
this.socket.on("close", () => this.close());
|
|
1717
|
+
this.socket.on("error", (error) => {
|
|
1595
1718
|
(0, runtime_1.emitNervesEvent)({
|
|
1596
1719
|
level: "error",
|
|
1597
1720
|
component: "senses",
|
|
@@ -1602,11 +1725,10 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1602
1725
|
});
|
|
1603
1726
|
}
|
|
1604
1727
|
end() {
|
|
1605
|
-
|
|
1606
|
-
this.ws.close();
|
|
1607
|
-
}
|
|
1728
|
+
this.socket.close();
|
|
1608
1729
|
this.close();
|
|
1609
1730
|
}
|
|
1731
|
+
/** Feed a raw Twilio message that arrived before this session attached (the bridge holds the first `start`). */
|
|
1610
1732
|
handleRawMessage(raw) {
|
|
1611
1733
|
const message = parseTwilioMediaStreamMessage(raw);
|
|
1612
1734
|
if (!message) {
|
|
@@ -1637,54 +1759,50 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1637
1759
|
}
|
|
1638
1760
|
}
|
|
1639
1761
|
async handleStart(start) {
|
|
1762
|
+
if (this.started)
|
|
1763
|
+
return;
|
|
1764
|
+
this.started = true;
|
|
1640
1765
|
this.streamSid = stringField(start?.streamSid);
|
|
1641
|
-
this.callSid =
|
|
1642
|
-
this.direction =
|
|
1643
|
-
this.outboundId =
|
|
1644
|
-
this.outboundReason =
|
|
1645
|
-
this.initialAudio = decodeVoiceCallAudioCustomParameter(
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
this.from = customParameter(start, "Remote") || customParameter(start, "To");
|
|
1649
|
-
this.to = customParameter(start, "Line") || customParameter(start, "From");
|
|
1650
|
-
}
|
|
1651
|
-
else {
|
|
1652
|
-
this.from = customParameter(start, "From");
|
|
1653
|
-
this.to = customParameter(start, "To");
|
|
1654
|
-
}
|
|
1655
|
-
const voiceContext = await resolveVoiceFriendContext(this.options, {
|
|
1656
|
-
friendId: explicitFriendId,
|
|
1657
|
-
remotePhone: this.from || undefined,
|
|
1658
|
-
callSid: this.callSid,
|
|
1659
|
-
});
|
|
1660
|
-
this.friendId = voiceContext.friendId;
|
|
1661
|
-
this.friendStore = voiceContext.friendStore;
|
|
1662
|
-
this.resolvedContext = voiceContext.resolved;
|
|
1663
|
-
this.sessionKey = twilioPhoneVoiceSessionKey({
|
|
1664
|
-
defaultFriendId: this.friendId,
|
|
1665
|
-
from: this.from,
|
|
1666
|
-
to: this.to,
|
|
1667
|
-
callSid: this.callSid,
|
|
1668
|
-
});
|
|
1669
|
-
this.lifecycle?.onIdentityChange?.(this, { callSid: this.callSid, outboundId: this.outboundId });
|
|
1670
|
-
this.floor = new floor_controller_1.VoiceFloorController({
|
|
1671
|
-
transport: "twilio-media-stream",
|
|
1672
|
-
agentName: this.options.agentName,
|
|
1673
|
-
callId: this.callSid,
|
|
1674
|
-
});
|
|
1675
|
-
this.floorUnsubscribe = this.floor.onTransition(() => this.flushQueuedGatedRealtimeRequest());
|
|
1676
|
-
this.floor.apply({ type: "call.connected", atMs: Date.now(), callId: this.callSid });
|
|
1677
|
-
(0, runtime_1.emitNervesEvent)({
|
|
1678
|
-
component: "senses",
|
|
1679
|
-
event: "senses.voice_twilio_realtime_start",
|
|
1680
|
-
message: "Twilio OpenAI Realtime stream started",
|
|
1681
|
-
meta: {
|
|
1682
|
-
agentName: this.options.agentName,
|
|
1683
|
-
callSid: safeSegment(this.callSid),
|
|
1684
|
-
sessionKey: this.sessionKey,
|
|
1685
|
-
},
|
|
1686
|
-
});
|
|
1766
|
+
this.callSid = this.identity.callSid;
|
|
1767
|
+
this.direction = this.identity.direction;
|
|
1768
|
+
this.outboundId = this.identity.outboundId ?? "";
|
|
1769
|
+
this.outboundReason = this.identity.reason ?? "";
|
|
1770
|
+
this.initialAudio = decodeVoiceCallAudioCustomParameter(this.identity.initialAudio ?? "");
|
|
1771
|
+
this.from = this.identity.from;
|
|
1772
|
+
this.to = this.identity.to;
|
|
1687
1773
|
try {
|
|
1774
|
+
const voiceContext = await resolveVoiceFriendContext(this.options, {
|
|
1775
|
+
friendId: this.direction === "outbound" ? this.identity.friendId : undefined,
|
|
1776
|
+
remotePhone: this.from || undefined,
|
|
1777
|
+
callSid: this.callSid,
|
|
1778
|
+
});
|
|
1779
|
+
this.friendId = voiceContext.friendId;
|
|
1780
|
+
this.friendStore = voiceContext.friendStore;
|
|
1781
|
+
this.resolvedContext = voiceContext.resolved;
|
|
1782
|
+
this.sessionKey = twilioPhoneVoiceSessionKey({
|
|
1783
|
+
defaultFriendId: this.friendId,
|
|
1784
|
+
from: this.from,
|
|
1785
|
+
to: this.to,
|
|
1786
|
+
callSid: this.callSid,
|
|
1787
|
+
});
|
|
1788
|
+
this.lifecycle?.onIdentityChange?.(this, { callSid: this.callSid, outboundId: this.outboundId });
|
|
1789
|
+
this.floor = new floor_controller_1.VoiceFloorController({
|
|
1790
|
+
transport: "twilio-media-stream",
|
|
1791
|
+
agentName: this.options.agentName,
|
|
1792
|
+
callId: this.callSid,
|
|
1793
|
+
});
|
|
1794
|
+
this.floorUnsubscribe = this.floor.onTransition(() => this.flushQueuedGatedRealtimeRequest());
|
|
1795
|
+
this.floor.apply({ type: "call.connected", atMs: Date.now(), callId: this.callSid });
|
|
1796
|
+
(0, runtime_1.emitNervesEvent)({
|
|
1797
|
+
component: "senses",
|
|
1798
|
+
event: "senses.voice_twilio_realtime_start",
|
|
1799
|
+
message: "Twilio OpenAI Realtime stream started",
|
|
1800
|
+
meta: {
|
|
1801
|
+
agentName: this.options.agentName,
|
|
1802
|
+
callSid: safeSegment(this.callSid),
|
|
1803
|
+
sessionKey: this.sessionKey,
|
|
1804
|
+
},
|
|
1805
|
+
});
|
|
1688
1806
|
await this.startOpenAIRealtimeSession();
|
|
1689
1807
|
}
|
|
1690
1808
|
catch (error) {
|
|
@@ -1698,7 +1816,6 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1698
1816
|
this.end();
|
|
1699
1817
|
}
|
|
1700
1818
|
}
|
|
1701
|
-
/* v8 ignore stop */
|
|
1702
1819
|
async startOpenAIRealtimeSession() {
|
|
1703
1820
|
const realtime = this.options.openaiRealtime;
|
|
1704
1821
|
if (!realtime?.apiKey?.trim()) {
|
|
@@ -1782,7 +1899,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1782
1899
|
session: {
|
|
1783
1900
|
type: "realtime",
|
|
1784
1901
|
instructions: fullInstructions,
|
|
1785
|
-
tools: realtimeToolsFromSelection(fullContext.toolSelection),
|
|
1902
|
+
tools: realtimeToolsFromSelection(fullContext.toolSelection, this.toolAdvertiser),
|
|
1786
1903
|
tool_choice: "auto",
|
|
1787
1904
|
},
|
|
1788
1905
|
});
|
|
@@ -1813,7 +1930,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
1813
1930
|
},
|
|
1814
1931
|
output: realtimeOutputAudioConfig(realtime, { type: "audio/pcmu" }),
|
|
1815
1932
|
},
|
|
1816
|
-
tools: realtimeToolsFromSelection(toolContext.toolSelection),
|
|
1933
|
+
tools: realtimeToolsFromSelection(toolContext.toolSelection, this.toolAdvertiser),
|
|
1817
1934
|
tool_choice: "auto",
|
|
1818
1935
|
max_output_tokens: OPENAI_REALTIME_MAX_OUTPUT_TOKENS,
|
|
1819
1936
|
...(realtime.reasoningEffort ? { reasoning: { effort: realtime.reasoningEffort } } : {}),
|
|
@@ -2240,7 +2357,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
2240
2357
|
});
|
|
2241
2358
|
let output;
|
|
2242
2359
|
try {
|
|
2243
|
-
const args = parseToolArguments(event.arguments, name, this.toolContext.toolSelection);
|
|
2360
|
+
const args = parseToolArguments(event.arguments, name, this.toolContext.toolSelection, this.toolAdvertiser);
|
|
2244
2361
|
(0, runtime_1.emitNervesEvent)({
|
|
2245
2362
|
component: "senses",
|
|
2246
2363
|
event: "senses.voice_twilio_realtime_tool_start",
|
|
@@ -2460,8 +2577,9 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
2460
2577
|
});
|
|
2461
2578
|
}
|
|
2462
2579
|
}
|
|
2580
|
+
/* v8 ignore stop */
|
|
2463
2581
|
async playPreparedAudio(request, playbackOptions = {}) {
|
|
2464
|
-
if (!this.streamSid || this.
|
|
2582
|
+
if (!this.streamSid || !this.socket.isOpen()) {
|
|
2465
2583
|
throw new Error("voice call media stream is not ready");
|
|
2466
2584
|
}
|
|
2467
2585
|
const prepared = await (0, audio_playback_1.prepareVoiceCallAudio)(request, {
|
|
@@ -2472,13 +2590,13 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
2472
2590
|
if (playbackOptions.clearFirst ?? true)
|
|
2473
2591
|
this.sendTwilioClear();
|
|
2474
2592
|
for (let offset = 0; offset < prepared.audio.byteLength; offset += 160) {
|
|
2475
|
-
if (this.closed || this.
|
|
2593
|
+
if (this.closed || !this.socket.isOpen())
|
|
2476
2594
|
break;
|
|
2477
2595
|
this.sendTwilioMedia(Buffer.from(prepared.audio.subarray(offset, offset + 160)).toString("base64"));
|
|
2478
2596
|
await delay(20);
|
|
2479
2597
|
}
|
|
2480
|
-
if (!this.closed && this.
|
|
2481
|
-
this.
|
|
2598
|
+
if (!this.closed && this.socket.isOpen()) {
|
|
2599
|
+
this.socket.send(JSON.stringify({
|
|
2482
2600
|
event: "mark",
|
|
2483
2601
|
streamSid: this.streamSid,
|
|
2484
2602
|
mark: { name: `tool-audio-${Date.now()}` },
|
|
@@ -2498,37 +2616,49 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
2498
2616
|
return { label: prepared.label, durationMs: prepared.durationMs };
|
|
2499
2617
|
}
|
|
2500
2618
|
sendTwilioMedia(payload) {
|
|
2501
|
-
if (this.closed || !this.streamSid || this.
|
|
2619
|
+
if (this.closed || !this.streamSid || !this.socket.isOpen())
|
|
2502
2620
|
return;
|
|
2503
|
-
this.
|
|
2621
|
+
this.socket.send(JSON.stringify({
|
|
2504
2622
|
event: "media",
|
|
2505
2623
|
streamSid: this.streamSid,
|
|
2506
2624
|
media: { payload },
|
|
2507
2625
|
}));
|
|
2508
2626
|
}
|
|
2509
2627
|
sendTwilioMark(playback) {
|
|
2510
|
-
if (this.closed || !this.streamSid || this.
|
|
2628
|
+
if (this.closed || !this.streamSid || !this.socket.isOpen())
|
|
2511
2629
|
return;
|
|
2512
2630
|
const name = `rt-${++this.playbackMarkIndex}`;
|
|
2513
2631
|
this.playbackMarks.set(name, playback);
|
|
2514
|
-
this.
|
|
2632
|
+
this.socket.send(JSON.stringify({
|
|
2515
2633
|
event: "mark",
|
|
2516
2634
|
streamSid: this.streamSid,
|
|
2517
2635
|
mark: { name },
|
|
2518
2636
|
}));
|
|
2519
2637
|
}
|
|
2520
2638
|
sendTwilioClear() {
|
|
2521
|
-
if (this.closed || !this.streamSid || this.
|
|
2639
|
+
if (this.closed || !this.streamSid || !this.socket.isOpen())
|
|
2522
2640
|
return;
|
|
2523
|
-
this.
|
|
2641
|
+
this.socket.send(JSON.stringify({ event: "clear", streamSid: this.streamSid }));
|
|
2524
2642
|
}
|
|
2525
2643
|
appendTranscript(role, text) {
|
|
2526
2644
|
const content = text.trim();
|
|
2527
2645
|
if (!content || !this.sessionPath)
|
|
2528
2646
|
return;
|
|
2529
2647
|
this.sessionMessages.push({ role, content });
|
|
2530
|
-
|
|
2648
|
+
try {
|
|
2649
|
+
(0, context_1.saveSession)(this.sessionPath, this.sessionMessages);
|
|
2650
|
+
}
|
|
2651
|
+
catch (error) {
|
|
2652
|
+
(0, runtime_1.emitNervesEvent)({
|
|
2653
|
+
level: "error",
|
|
2654
|
+
component: "senses",
|
|
2655
|
+
event: "senses.voice_transcript_save_error",
|
|
2656
|
+
message: "voice transcript could not be saved; the call continues",
|
|
2657
|
+
meta: { agentName: this.options.agentName, callSid: safeSegment(this.callSid), error: errorMessage(error) },
|
|
2658
|
+
});
|
|
2659
|
+
}
|
|
2531
2660
|
}
|
|
2661
|
+
/* v8 ignore start -- unchanged hangup and close transport @preserve */
|
|
2532
2662
|
completeHangupIfReady(trigger) {
|
|
2533
2663
|
if (!this.hangupRequested || this.closed)
|
|
2534
2664
|
return;
|
|
@@ -2567,6 +2697,7 @@ class TwilioOpenAIRealtimeMediaStreamSession {
|
|
|
2567
2697
|
});
|
|
2568
2698
|
}
|
|
2569
2699
|
}
|
|
2700
|
+
exports.TwilioOpenAIRealtimeMediaStreamSession = TwilioOpenAIRealtimeMediaStreamSession;
|
|
2570
2701
|
/* v8 ignore stop */
|
|
2571
2702
|
/* v8 ignore start -- direct SIP control has bridge-level coverage; provider failures, bootstrap races, and AMD timers are live-network edge permutations @preserve */
|
|
2572
2703
|
class OpenAISipPhoneSession {
|
|
@@ -2586,6 +2717,11 @@ class OpenAISipPhoneSession {
|
|
|
2586
2717
|
autoResponsesSuppressedForAmd = false;
|
|
2587
2718
|
openaiWs = null;
|
|
2588
2719
|
toolContext;
|
|
2720
|
+
advertiser;
|
|
2721
|
+
get toolAdvertiser() {
|
|
2722
|
+
this.advertiser ??= (0, realtime_tool_schema_1.createRealtimeToolAdvertiser)({ agentName: this.options.agentName, callId: safeSegment(this.metadata.callId) });
|
|
2723
|
+
return this.advertiser;
|
|
2724
|
+
}
|
|
2589
2725
|
friendStore;
|
|
2590
2726
|
resolvedContext;
|
|
2591
2727
|
sessionMessages = [];
|
|
@@ -2627,7 +2763,7 @@ class OpenAISipPhoneSession {
|
|
|
2627
2763
|
if (!sip)
|
|
2628
2764
|
throw new Error("OpenAI SIP options are not configured");
|
|
2629
2765
|
const voiceContext = await resolveVoiceFriendContext(this.options, {
|
|
2630
|
-
friendId: this.metadata.friendId ||
|
|
2766
|
+
friendId: this.metadata.friendId || undefined,
|
|
2631
2767
|
remotePhone: this.metadata.from || undefined,
|
|
2632
2768
|
callSid: this.metadata.callId,
|
|
2633
2769
|
});
|
|
@@ -2753,7 +2889,7 @@ class OpenAISipPhoneSession {
|
|
|
2753
2889
|
},
|
|
2754
2890
|
output: realtimeOutputAudioConfig(realtime),
|
|
2755
2891
|
},
|
|
2756
|
-
tools: realtimeToolsFromSelection(toolContext.toolSelection),
|
|
2892
|
+
tools: realtimeToolsFromSelection(toolContext.toolSelection, this.toolAdvertiser),
|
|
2757
2893
|
tool_choice: "auto",
|
|
2758
2894
|
max_output_tokens: OPENAI_REALTIME_MAX_OUTPUT_TOKENS,
|
|
2759
2895
|
}),
|
|
@@ -2835,7 +2971,7 @@ class OpenAISipPhoneSession {
|
|
|
2835
2971
|
session: {
|
|
2836
2972
|
type: "realtime",
|
|
2837
2973
|
instructions,
|
|
2838
|
-
tools: realtimeToolsFromSelection(toolContext.toolSelection),
|
|
2974
|
+
tools: realtimeToolsFromSelection(toolContext.toolSelection, this.toolAdvertiser),
|
|
2839
2975
|
tool_choice: "auto",
|
|
2840
2976
|
...(realtime.reasoningEffort ? { reasoning: { effort: realtime.reasoningEffort } } : {}),
|
|
2841
2977
|
},
|
|
@@ -3324,7 +3460,7 @@ class OpenAISipPhoneSession {
|
|
|
3324
3460
|
});
|
|
3325
3461
|
let output;
|
|
3326
3462
|
try {
|
|
3327
|
-
const args = parseToolArguments(event.arguments, name, this.toolContext.toolSelection);
|
|
3463
|
+
const args = parseToolArguments(event.arguments, name, this.toolContext.toolSelection, this.toolAdvertiser);
|
|
3328
3464
|
(0, runtime_1.emitNervesEvent)({
|
|
3329
3465
|
component: "senses",
|
|
3330
3466
|
event: "senses.voice_openai_sip_tool_start",
|
|
@@ -3569,13 +3705,27 @@ class OpenAISipPhoneSession {
|
|
|
3569
3705
|
return;
|
|
3570
3706
|
this.openaiWs.send(JSON.stringify(event));
|
|
3571
3707
|
}
|
|
3708
|
+
/* v8 ignore stop */
|
|
3572
3709
|
appendTranscript(role, text) {
|
|
3573
3710
|
const content = text.trim();
|
|
3711
|
+
/* v8 ignore next -- transcripts only arrive after start has set the session path @preserve */
|
|
3574
3712
|
if (!content || !this.sessionPath)
|
|
3575
3713
|
return;
|
|
3576
3714
|
this.sessionMessages.push({ role, content });
|
|
3577
|
-
|
|
3715
|
+
try {
|
|
3716
|
+
(0, context_1.saveSession)(this.sessionPath, this.sessionMessages);
|
|
3717
|
+
}
|
|
3718
|
+
catch (error) {
|
|
3719
|
+
(0, runtime_1.emitNervesEvent)({
|
|
3720
|
+
level: "error",
|
|
3721
|
+
component: "senses",
|
|
3722
|
+
event: "senses.voice_transcript_save_error",
|
|
3723
|
+
message: "voice transcript could not be saved; the call continues",
|
|
3724
|
+
meta: { agentName: this.options.agentName, callId: safeSegment(this.metadata.callId), error: errorMessage(error) },
|
|
3725
|
+
});
|
|
3726
|
+
}
|
|
3578
3727
|
}
|
|
3728
|
+
/* v8 ignore start -- unchanged SIP close transport @preserve */
|
|
3579
3729
|
close(trigger) {
|
|
3580
3730
|
if (this.closed)
|
|
3581
3731
|
return;
|
|
@@ -3787,6 +3937,7 @@ class TwilioAudioStreamJobStore {
|
|
|
3787
3937
|
return `${callSid}/${jobId}`;
|
|
3788
3938
|
}
|
|
3789
3939
|
}
|
|
3940
|
+
exports.TwilioAudioStreamJobStore = TwilioAudioStreamJobStore;
|
|
3790
3941
|
function deliveredSegments(turn) {
|
|
3791
3942
|
return turn.speechSegments.map((segment) => segment.tts);
|
|
3792
3943
|
}
|
|
@@ -3913,7 +4064,7 @@ function computeTwilioSignature(input) {
|
|
|
3913
4064
|
}
|
|
3914
4065
|
function validateTwilioSignature(input) {
|
|
3915
4066
|
if (!input.authToken.trim())
|
|
3916
|
-
return
|
|
4067
|
+
return false;
|
|
3917
4068
|
if (!input.signature.trim())
|
|
3918
4069
|
return false;
|
|
3919
4070
|
const expected = Buffer.from(computeTwilioSignature(input));
|
|
@@ -4066,7 +4217,7 @@ async function createTwilioOutboundCall(request, fetchImpl = fetch) {
|
|
|
4066
4217
|
function verifyRequest(options, request, params) {
|
|
4067
4218
|
const authToken = options.twilioAuthToken?.trim();
|
|
4068
4219
|
if (!authToken)
|
|
4069
|
-
return
|
|
4220
|
+
return false;
|
|
4070
4221
|
return validateTwilioSignature({
|
|
4071
4222
|
authToken,
|
|
4072
4223
|
url: requestPublicUrl(options.publicBaseUrl, request.path),
|
|
@@ -4074,7 +4225,37 @@ function verifyRequest(options, request, params) {
|
|
|
4074
4225
|
signature: headerValue(request.headers, "x-twilio-signature"),
|
|
4075
4226
|
});
|
|
4076
4227
|
}
|
|
4077
|
-
async function
|
|
4228
|
+
async function rejectOpenAISipCallById(options, callId, reason) {
|
|
4229
|
+
(0, runtime_1.emitNervesEvent)({
|
|
4230
|
+
level: "warn",
|
|
4231
|
+
component: "senses",
|
|
4232
|
+
event: "senses.voice_openai_sip_call_refused",
|
|
4233
|
+
message: "refused an OpenAI SIP call that did not carry a valid call token",
|
|
4234
|
+
meta: { agentName: options.agentName, callId: safeSegment(callId), reason },
|
|
4235
|
+
});
|
|
4236
|
+
const sip = options.openaiSip;
|
|
4237
|
+
const apiKey = options.openaiRealtime?.apiKey?.trim();
|
|
4238
|
+
if (!sip || !apiKey)
|
|
4239
|
+
return;
|
|
4240
|
+
try {
|
|
4241
|
+
const response = await (sip.fetch ?? fetch)(openAISipCallActionUrl(sip, callId, "reject"), {
|
|
4242
|
+
method: "POST",
|
|
4243
|
+
headers: { authorization: `Bearer ${apiKey}` },
|
|
4244
|
+
});
|
|
4245
|
+
if (!response.ok)
|
|
4246
|
+
throw new Error(`OpenAI SIP call reject failed: ${response.status}`);
|
|
4247
|
+
}
|
|
4248
|
+
catch (error) {
|
|
4249
|
+
(0, runtime_1.emitNervesEvent)({
|
|
4250
|
+
level: "error",
|
|
4251
|
+
component: "senses",
|
|
4252
|
+
event: "senses.voice_openai_sip_call_reject_error",
|
|
4253
|
+
message: "OpenAI SIP call reject request failed",
|
|
4254
|
+
meta: { agentName: options.agentName, callId: safeSegment(callId), trigger: reason, error: errorMessage(error) },
|
|
4255
|
+
});
|
|
4256
|
+
}
|
|
4257
|
+
}
|
|
4258
|
+
async function handleOpenAISipWebhook(options, request, activeSipSessions, pending, recentSipWebhooks) {
|
|
4078
4259
|
const rawBody = bodyText(request.body);
|
|
4079
4260
|
const sip = options.openaiSip;
|
|
4080
4261
|
const webhookSecret = sip?.webhookSecret?.trim();
|
|
@@ -4107,26 +4288,54 @@ async function handleOpenAISipWebhook(options, request, activeSipSessions) {
|
|
|
4107
4288
|
return textResponse(400, "invalid OpenAI webhook payload");
|
|
4108
4289
|
if (event.type !== "realtime.call.incoming")
|
|
4109
4290
|
return textResponse(200, "ok");
|
|
4110
|
-
const
|
|
4111
|
-
if (!
|
|
4291
|
+
const callId = openAISipCallId(event);
|
|
4292
|
+
if (!callId)
|
|
4112
4293
|
return textResponse(400, "missing OpenAI SIP call metadata");
|
|
4113
|
-
const
|
|
4294
|
+
const webhookId = headerValue(request.headers, "webhook-id");
|
|
4295
|
+
const newCall = recentSipWebhooks.add(`call:${callId}`);
|
|
4296
|
+
const newWebhook = webhookId ? recentSipWebhooks.add(`webhook:${webhookId}`) : true;
|
|
4297
|
+
if (!newCall || !newWebhook) {
|
|
4298
|
+
(0, runtime_1.emitNervesEvent)({
|
|
4299
|
+
level: "warn",
|
|
4300
|
+
component: "senses",
|
|
4301
|
+
event: "senses.voice_openai_sip_webhook_duplicate",
|
|
4302
|
+
message: "ignored a duplicate OpenAI SIP incoming-call webhook",
|
|
4303
|
+
meta: { agentName: options.agentName, callId: safeSegment(callId) },
|
|
4304
|
+
});
|
|
4305
|
+
return textResponse(200, "ok");
|
|
4306
|
+
}
|
|
4307
|
+
const admission = admitOpenAISipCall(options, pending, event);
|
|
4308
|
+
if (!admission.ok) {
|
|
4309
|
+
await rejectOpenAISipCallById(options, callId, admission.reason);
|
|
4310
|
+
return textResponse(200, "ok");
|
|
4311
|
+
}
|
|
4312
|
+
const session = new OpenAISipPhoneSession(options, admission.metadata, activeSipSessions);
|
|
4114
4313
|
/* v8 ignore next -- async SIP session startup failures are logged inside the session; webhook request intentionally returns immediately @preserve */
|
|
4115
4314
|
void session.start().catch(() => undefined);
|
|
4116
4315
|
return textResponse(200, "ok");
|
|
4117
4316
|
}
|
|
4118
|
-
|
|
4119
|
-
async function handleIncoming(options, basePath, params, jobs) {
|
|
4317
|
+
async function handleIncoming(options, basePath, params, jobs, pending) {
|
|
4120
4318
|
const callSid = params.CallSid?.trim() || "incoming";
|
|
4121
4319
|
const safeCallSid = safeSegment(callSid);
|
|
4122
4320
|
const callDir = path.join(options.outputDir, safeCallSid);
|
|
4123
4321
|
const utteranceId = `twilio-${safeCallSid}-connected`;
|
|
4124
|
-
const
|
|
4125
|
-
const
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
callSid
|
|
4322
|
+
const from = params.From?.trim() ?? "";
|
|
4323
|
+
const to = params.To?.trim() ?? "";
|
|
4324
|
+
// Inbound callers are identified by the signed caller ID, never by defaultFriendId.
|
|
4325
|
+
const sessionKey = twilioPhoneVoiceSessionKey({ from, to, callSid });
|
|
4326
|
+
const legacyContext = async () => {
|
|
4327
|
+
const friendId = await inboundVoiceFriendId(options, from, callSid);
|
|
4328
|
+
return { friendId, sessionKey: twilioPhoneVoiceSessionKey({ defaultFriendId: friendId, from, to, callSid }) };
|
|
4329
|
+
};
|
|
4330
|
+
const admit = (engine, greetingJobId) => admitMediaStreamCall(options, basePath, pending, {
|
|
4331
|
+
callSid: params.CallSid?.trim() ?? "",
|
|
4332
|
+
agentName: options.agentName,
|
|
4333
|
+
direction: "inbound",
|
|
4334
|
+
from,
|
|
4335
|
+
to,
|
|
4336
|
+
engine,
|
|
4337
|
+
...(greetingJobId ? { greetingJobId } : {}),
|
|
4338
|
+
...(params.StirVerstat?.trim() ? { stirVerstat: params.StirVerstat.trim() } : {}),
|
|
4130
4339
|
});
|
|
4131
4340
|
(0, runtime_1.emitNervesEvent)({
|
|
4132
4341
|
component: "senses",
|
|
@@ -4141,12 +4350,15 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4141
4350
|
message: "answering Twilio call by dialing OpenAI SIP",
|
|
4142
4351
|
meta: { agentName: options.agentName, callSid: safeCallSid, sessionKey, conversationEngine: "openai-sip" },
|
|
4143
4352
|
});
|
|
4144
|
-
return xmlResponse(
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4353
|
+
return xmlResponse(admitSipCall(options, pending, {
|
|
4354
|
+
callSid: params.CallSid?.trim() ?? "",
|
|
4355
|
+
agentName: options.agentName,
|
|
4356
|
+
direction: "inbound",
|
|
4357
|
+
from,
|
|
4358
|
+
to,
|
|
4359
|
+
engine: "openai-sip",
|
|
4360
|
+
...(params.StirVerstat?.trim() ? { stirVerstat: params.StirVerstat.trim() } : {}),
|
|
4361
|
+
}));
|
|
4150
4362
|
}
|
|
4151
4363
|
if (normalizeTwilioPhoneTransportMode(options.transportMode) === "media-stream") {
|
|
4152
4364
|
if (usesOpenAIRealtimeConversationEngine(options)) {
|
|
@@ -4156,9 +4368,10 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4156
4368
|
message: "answering Twilio call with OpenAI Realtime Media Stream",
|
|
4157
4369
|
meta: { agentName: options.agentName, callSid: safeCallSid, sessionKey, conversationEngine: "openai-realtime" },
|
|
4158
4370
|
});
|
|
4159
|
-
return xmlResponse(
|
|
4371
|
+
return xmlResponse(admit("openai-realtime"));
|
|
4160
4372
|
}
|
|
4161
4373
|
try {
|
|
4374
|
+
const { friendId, sessionKey: legacySessionKey } = await legacyContext();
|
|
4162
4375
|
await fs.mkdir(callDir, { recursive: true });
|
|
4163
4376
|
const transcript = (0, transcript_1.buildVoiceTranscript)({
|
|
4164
4377
|
utteranceId,
|
|
@@ -4177,7 +4390,7 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4177
4390
|
runTurn: (onAudioChunk) => (0, turn_1.runVoiceLoopbackTurn)({
|
|
4178
4391
|
agentName: options.agentName,
|
|
4179
4392
|
friendId,
|
|
4180
|
-
sessionKey,
|
|
4393
|
+
sessionKey: legacySessionKey,
|
|
4181
4394
|
transcript,
|
|
4182
4395
|
tts: options.tts,
|
|
4183
4396
|
runSenseTurn: options.runSenseTurn,
|
|
@@ -4198,7 +4411,7 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4198
4411
|
message: "answering Twilio call with a bidirectional Media Stream",
|
|
4199
4412
|
meta: { agentName: options.agentName, callSid: safeCallSid, sessionKey, greetingJob: prebufferState },
|
|
4200
4413
|
});
|
|
4201
|
-
return xmlResponse(
|
|
4414
|
+
return xmlResponse(admit("cascade", greetingJobId));
|
|
4202
4415
|
}
|
|
4203
4416
|
catch (error) {
|
|
4204
4417
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -4208,10 +4421,11 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4208
4421
|
message: "Twilio incoming media-stream greeting turn failed",
|
|
4209
4422
|
meta: { agentName: options.agentName, callSid: safeCallSid, error: errorMessage(error), transportMode: "media-stream" },
|
|
4210
4423
|
});
|
|
4211
|
-
return xmlResponse(
|
|
4424
|
+
return xmlResponse(admit("cascade"));
|
|
4212
4425
|
}
|
|
4213
4426
|
}
|
|
4214
4427
|
try {
|
|
4428
|
+
const { friendId, sessionKey: legacySessionKey } = await legacyContext();
|
|
4215
4429
|
await fs.mkdir(callDir, { recursive: true });
|
|
4216
4430
|
if (normalizeTwilioPhonePlaybackMode(options.playbackMode) === "stream") {
|
|
4217
4431
|
const transcript = (0, transcript_1.buildVoiceTranscript)({
|
|
@@ -4231,7 +4445,7 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4231
4445
|
runTurn: (onAudioChunk) => (0, turn_1.runVoiceLoopbackTurn)({
|
|
4232
4446
|
agentName: options.agentName,
|
|
4233
4447
|
friendId,
|
|
4234
|
-
sessionKey,
|
|
4448
|
+
sessionKey: legacySessionKey,
|
|
4235
4449
|
transcript,
|
|
4236
4450
|
tts: options.tts,
|
|
4237
4451
|
runSenseTurn: options.runSenseTurn,
|
|
@@ -4246,14 +4460,6 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4246
4460
|
message: "Twilio greeting prebuffer completed",
|
|
4247
4461
|
meta: { agentName: options.agentName, callSid: safeCallSid, utteranceId, state: prebufferState },
|
|
4248
4462
|
});
|
|
4249
|
-
if (prebufferState === "failed") {
|
|
4250
|
-
return xmlResponse(recordTwiml({
|
|
4251
|
-
publicBaseUrl: options.publicBaseUrl,
|
|
4252
|
-
basePath,
|
|
4253
|
-
timeoutSeconds: options.recordTimeoutSeconds ?? exports.DEFAULT_TWILIO_RECORD_TIMEOUT_SECONDS,
|
|
4254
|
-
maxLengthSeconds: options.recordMaxLengthSeconds ?? exports.DEFAULT_TWILIO_RECORD_MAX_LENGTH_SECONDS,
|
|
4255
|
-
}));
|
|
4256
|
-
}
|
|
4257
4463
|
return xmlResponse(`${playTwiml(streamAudioUrl(options, basePath, safeCallSid, jobId))}${nextInputTwiml(options, basePath, "record")}`);
|
|
4258
4464
|
}
|
|
4259
4465
|
return await runPhonePromptTurn({
|
|
@@ -4263,7 +4469,7 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4263
4469
|
safeCallSid,
|
|
4264
4470
|
utteranceId,
|
|
4265
4471
|
friendId,
|
|
4266
|
-
sessionKey,
|
|
4472
|
+
sessionKey: legacySessionKey,
|
|
4267
4473
|
promptText: callConnectedPrompt(params),
|
|
4268
4474
|
afterPlayback: "record",
|
|
4269
4475
|
});
|
|
@@ -4284,7 +4490,7 @@ async function handleIncoming(options, basePath, params, jobs) {
|
|
|
4284
4490
|
}));
|
|
4285
4491
|
}
|
|
4286
4492
|
}
|
|
4287
|
-
async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
4493
|
+
async function handleOutgoing(options, basePath, outboundId, params, jobs, pending) {
|
|
4288
4494
|
const job = await readTwilioOutboundCallJob(options.outputDir, outboundId);
|
|
4289
4495
|
if (!job)
|
|
4290
4496
|
return textResponse(404, "outbound voice call not found");
|
|
@@ -4299,7 +4505,7 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4299
4505
|
transportCallSid: callSid,
|
|
4300
4506
|
events: [
|
|
4301
4507
|
...(job.events ?? []),
|
|
4302
|
-
{ at: new Date().toISOString(), status: nonHumanStatus, callSid,
|
|
4508
|
+
{ at: new Date().toISOString(), status: nonHumanStatus, callSid, answeredBy },
|
|
4303
4509
|
],
|
|
4304
4510
|
});
|
|
4305
4511
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -4311,7 +4517,7 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4311
4517
|
callSid: safeCallSid,
|
|
4312
4518
|
outboundId: safeSegment(job.outboundId),
|
|
4313
4519
|
status: nonHumanStatus,
|
|
4314
|
-
answeredBy: answeredBy
|
|
4520
|
+
answeredBy: String(answeredBy),
|
|
4315
4521
|
},
|
|
4316
4522
|
});
|
|
4317
4523
|
return xmlResponse("<Hangup />");
|
|
@@ -4342,15 +4548,21 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4342
4548
|
message: "Twilio outbound voice call answered",
|
|
4343
4549
|
meta: { agentName: options.agentName, callSid: safeCallSid, outboundId: safeSegment(job.outboundId), sessionKey },
|
|
4344
4550
|
});
|
|
4345
|
-
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4551
|
+
// What the stream or SIP leg will be told about this call comes from the outbound job record
|
|
4552
|
+
// and the signed webhook, held server-side; the media stream never supplies it.
|
|
4553
|
+
const outboundIdentity = (engine, greetingJobId) => ({
|
|
4554
|
+
callSid: params.CallSid?.trim() ?? "",
|
|
4555
|
+
agentName: options.agentName,
|
|
4556
|
+
direction: "outbound",
|
|
4557
|
+
from: to,
|
|
4558
|
+
to: from,
|
|
4559
|
+
outboundId: job.outboundId,
|
|
4560
|
+
friendId,
|
|
4561
|
+
reason: job.reason,
|
|
4562
|
+
...(job.initialAudio ? { initialAudio: encodeVoiceCallAudioCustomParameter(job.initialAudio) } : {}),
|
|
4563
|
+
engine,
|
|
4564
|
+
...(greetingJobId ? { greetingJobId } : {}),
|
|
4565
|
+
});
|
|
4354
4566
|
if (usesOpenAISipOutboundConversationEngine(options)) {
|
|
4355
4567
|
(0, runtime_1.emitNervesEvent)({
|
|
4356
4568
|
component: "senses",
|
|
@@ -4358,20 +4570,11 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4358
4570
|
message: "answering Twilio outbound call by dialing OpenAI SIP",
|
|
4359
4571
|
meta: { agentName: options.agentName, callSid: safeCallSid, outboundId: safeSegment(job.outboundId), sessionKey, conversationEngine: "openai-sip" },
|
|
4360
4572
|
});
|
|
4361
|
-
return xmlResponse(
|
|
4362
|
-
Agent: options.agentName,
|
|
4363
|
-
Direction: "outbound",
|
|
4364
|
-
From: to,
|
|
4365
|
-
To: from,
|
|
4366
|
-
}, {
|
|
4367
|
-
"X-Ouro-Outbound-Id": job.outboundId,
|
|
4368
|
-
"X-Ouro-Friend-Id": friendId,
|
|
4369
|
-
"X-Ouro-Reason": job.reason,
|
|
4370
|
-
})));
|
|
4573
|
+
return xmlResponse(admitSipCall(options, pending, outboundIdentity("openai-sip")));
|
|
4371
4574
|
}
|
|
4372
4575
|
if (normalizeTwilioPhoneTransportMode(options.transportMode) === "media-stream") {
|
|
4373
4576
|
if (usesOpenAIRealtimeOutboundConversationEngine(options)) {
|
|
4374
|
-
return xmlResponse(
|
|
4577
|
+
return xmlResponse(admitMediaStreamCall(options, basePath, pending, outboundIdentity("openai-realtime")));
|
|
4375
4578
|
}
|
|
4376
4579
|
try {
|
|
4377
4580
|
await fs.mkdir(callDir, { recursive: true });
|
|
@@ -4393,7 +4596,7 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4393
4596
|
byteLength: String(job.prewarmedGreeting.byteLength),
|
|
4394
4597
|
},
|
|
4395
4598
|
});
|
|
4396
|
-
return xmlResponse(
|
|
4599
|
+
return xmlResponse(admitMediaStreamCall(options, basePath, pending, outboundIdentity("cascade", greetingJobId)));
|
|
4397
4600
|
}
|
|
4398
4601
|
catch (error) {
|
|
4399
4602
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -4442,7 +4645,7 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4442
4645
|
message: "Twilio Media Stream greeting prebuffer completed",
|
|
4443
4646
|
meta: { agentName: options.agentName, callSid: safeCallSid, utteranceId, state: prebufferState, transportMode: "media-stream" },
|
|
4444
4647
|
});
|
|
4445
|
-
return xmlResponse(
|
|
4648
|
+
return xmlResponse(admitMediaStreamCall(options, basePath, pending, outboundIdentity("cascade", greetingJobId)));
|
|
4446
4649
|
}
|
|
4447
4650
|
catch (error) {
|
|
4448
4651
|
(0, runtime_1.emitNervesEvent)({
|
|
@@ -4452,9 +4655,7 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4452
4655
|
message: "Twilio outbound media-stream greeting turn failed",
|
|
4453
4656
|
meta: { agentName: options.agentName, callSid: safeCallSid, outboundId: safeSegment(job.outboundId), error: errorMessage(error), transportMode: "media-stream" },
|
|
4454
4657
|
});
|
|
4455
|
-
return xmlResponse(
|
|
4456
|
-
...streamParams,
|
|
4457
|
-
}));
|
|
4658
|
+
return xmlResponse(admitMediaStreamCall(options, basePath, pending, outboundIdentity("cascade")));
|
|
4458
4659
|
}
|
|
4459
4660
|
}
|
|
4460
4661
|
try {
|
|
@@ -4487,6 +4688,7 @@ async function handleOutgoing(options, basePath, outboundId, params, jobs) {
|
|
|
4487
4688
|
}));
|
|
4488
4689
|
}
|
|
4489
4690
|
}
|
|
4691
|
+
/* v8 ignore start -- unchanged outbound status and AMD callbacks; the AMD lookup is not part of this change @preserve */
|
|
4490
4692
|
async function handleOutgoingStatus(options, outboundId, params) {
|
|
4491
4693
|
const job = await readTwilioOutboundCallJob(options.outputDir, outboundId);
|
|
4492
4694
|
if (!job)
|
|
@@ -4564,6 +4766,7 @@ async function handleOutgoingAmdStatus(options, outboundId, params, activeMediaS
|
|
|
4564
4766
|
});
|
|
4565
4767
|
return textResponse(200, "ok");
|
|
4566
4768
|
}
|
|
4769
|
+
/* v8 ignore stop */
|
|
4567
4770
|
async function handleListen(options, basePath) {
|
|
4568
4771
|
return xmlResponse(recordTwiml({
|
|
4569
4772
|
publicBaseUrl: options.publicBaseUrl,
|
|
@@ -4590,9 +4793,14 @@ async function handleRecording(options, basePath, params, jobs) {
|
|
|
4590
4793
|
const inputPath = path.join(callDir, `${safeRecordingSid}.wav`);
|
|
4591
4794
|
const utteranceId = `twilio-${safeCallSid}-${safeRecordingSid}`;
|
|
4592
4795
|
const downloadRecording = options.downloadRecording ?? defaultTwilioRecordingDownloader;
|
|
4593
|
-
|
|
4796
|
+
// Outbound calls are placed by us (defaultFriendId stays their fallback); inbound callers are
|
|
4797
|
+
// identified by the signed caller ID only.
|
|
4798
|
+
const outboundCall = (params.Direction ?? "").trim().toLowerCase().startsWith("outbound");
|
|
4799
|
+
const friendId = outboundCall
|
|
4800
|
+
? voiceFriendId(options, recording.from, recording.callSid)
|
|
4801
|
+
: await inboundVoiceFriendId(options, recording.from, recording.callSid);
|
|
4594
4802
|
const sessionKey = twilioPhoneVoiceSessionKey({
|
|
4595
|
-
defaultFriendId: options.defaultFriendId,
|
|
4803
|
+
defaultFriendId: outboundCall ? options.defaultFriendId : friendId,
|
|
4596
4804
|
from: recording.from,
|
|
4597
4805
|
to: recording.to,
|
|
4598
4806
|
callSid: recording.callSid,
|
|
@@ -4620,7 +4828,7 @@ async function handleRecording(options, basePath, params, jobs) {
|
|
|
4620
4828
|
const audio = await downloadRecording({
|
|
4621
4829
|
recordingUrl: mediaUrl,
|
|
4622
4830
|
accountSid: options.twilioAccountSid?.trim() || undefined,
|
|
4623
|
-
authToken: options.twilioAuthToken?.trim()
|
|
4831
|
+
authToken: options.twilioAuthToken?.trim(),
|
|
4624
4832
|
});
|
|
4625
4833
|
await fs.writeFile(inputPath, audio);
|
|
4626
4834
|
const turnTranscript = await transcribeRecordingOrNoSpeech({
|
|
@@ -4647,7 +4855,7 @@ async function handleRecording(options, basePath, params, jobs) {
|
|
|
4647
4855
|
const audio = await downloadRecording({
|
|
4648
4856
|
recordingUrl: mediaUrl,
|
|
4649
4857
|
accountSid: options.twilioAccountSid?.trim() || undefined,
|
|
4650
|
-
authToken: options.twilioAuthToken?.trim()
|
|
4858
|
+
authToken: options.twilioAuthToken?.trim(),
|
|
4651
4859
|
});
|
|
4652
4860
|
await fs.writeFile(inputPath, audio);
|
|
4653
4861
|
const transcript = await transcribeRecordingOrNoSpeech({
|
|
@@ -4763,6 +4971,50 @@ async function handleAudioStream(options, basePath, requestPath, jobs) {
|
|
|
4763
4971
|
});
|
|
4764
4972
|
return streamResponse(job.stream(), job.mimeType);
|
|
4765
4973
|
}
|
|
4974
|
+
const MEDIA_STREAM_START_TIMEOUT_MS = 10_000;
|
|
4975
|
+
/**
|
|
4976
|
+
* Check a Media Stream `start` against what the signed webhook recorded. The only accepted
|
|
4977
|
+
* identity is a record consumed with a valid, unexpired, single-use token for this agent and
|
|
4978
|
+
* this CallSid; stream parameters never supply identity.
|
|
4979
|
+
*/
|
|
4980
|
+
function admitMediaStreamStart(options, pending, activeMediaStreams, start) {
|
|
4981
|
+
const callSid = stringField(start?.callSid);
|
|
4982
|
+
const refuse = (reason, rememberRejection) => {
|
|
4983
|
+
if (rememberRejection)
|
|
4984
|
+
pending.markRejected(callSid);
|
|
4985
|
+
(0, runtime_1.emitNervesEvent)({
|
|
4986
|
+
level: "warn",
|
|
4987
|
+
component: "senses",
|
|
4988
|
+
event: "senses.voice_media_stream_rejected",
|
|
4989
|
+
message: "rejected a Twilio Media Stream start",
|
|
4990
|
+
meta: { agentName: options.agentName, callSid: safeSegment(callSid || "unknown"), reason },
|
|
4991
|
+
});
|
|
4992
|
+
return null;
|
|
4993
|
+
};
|
|
4994
|
+
const secret = options.twilioAuthToken?.trim();
|
|
4995
|
+
if (!secret)
|
|
4996
|
+
return refuse("no_secret", false);
|
|
4997
|
+
const verified = (0, call_auth_1.verifyVoiceCallToken)({
|
|
4998
|
+
secret,
|
|
4999
|
+
purpose: "stream",
|
|
5000
|
+
token: customParameter(start, "OuroToken"),
|
|
5001
|
+
agentName: options.agentName,
|
|
5002
|
+
callSid,
|
|
5003
|
+
nowMs: Date.now(),
|
|
5004
|
+
});
|
|
5005
|
+
if (!verified.ok)
|
|
5006
|
+
return refuse(verified.reason, false);
|
|
5007
|
+
const identity = pending.consume(callSid, verified.nonce);
|
|
5008
|
+
if (!identity)
|
|
5009
|
+
return refuse("no_record", true);
|
|
5010
|
+
if (identity.direction !== verified.direction || (identity.outboundId ?? "") !== (verified.outboundId ?? "")) {
|
|
5011
|
+
return refuse("record_mismatch", true);
|
|
5012
|
+
}
|
|
5013
|
+
if (activeMediaStreams.reserved.has(callSid) || activeMediaStreams.byCallSid.has(callSid))
|
|
5014
|
+
return refuse("duplicate_call", false);
|
|
5015
|
+
activeMediaStreams.reserved.add(callSid);
|
|
5016
|
+
return identity;
|
|
5017
|
+
}
|
|
4766
5018
|
function createTwilioPhoneBridge(options) {
|
|
4767
5019
|
new URL(options.publicBaseUrl);
|
|
4768
5020
|
const basePath = normalizeTwilioPhoneBasePath(options.basePath);
|
|
@@ -4772,30 +5024,77 @@ function createTwilioPhoneBridge(options) {
|
|
|
4772
5024
|
const activeMediaStreams = {
|
|
4773
5025
|
byCallSid: new Map(),
|
|
4774
5026
|
byOutboundId: new Map(),
|
|
5027
|
+
reserved: new Set(),
|
|
4775
5028
|
};
|
|
4776
5029
|
const activeSipSessions = new ActiveOpenAISipSessions();
|
|
4777
|
-
|
|
5030
|
+
const pendingCalls = options.pendingVoiceCalls ?? new call_auth_1.PendingVoiceCalls();
|
|
5031
|
+
const recentSipWebhooks = new call_auth_1.RecentIds();
|
|
5032
|
+
mediaStreams.on("connection", (ws) => {
|
|
5033
|
+
// A malformed frame makes `ws` emit `error`; with no listener that is an uncaught exception,
|
|
5034
|
+
// and this socket is reachable before any start has been verified.
|
|
5035
|
+
ws.on("error", (error) => {
|
|
5036
|
+
(0, runtime_1.emitNervesEvent)({
|
|
5037
|
+
level: "warn",
|
|
5038
|
+
component: "senses",
|
|
5039
|
+
event: "senses.voice_media_stream_socket_error",
|
|
5040
|
+
message: "a Twilio Media Stream socket raised an error",
|
|
5041
|
+
meta: { agentName: options.agentName, error: error.message },
|
|
5042
|
+
});
|
|
5043
|
+
});
|
|
5044
|
+
let reservedCallSid = "";
|
|
4778
5045
|
const lifecycle = {
|
|
4779
5046
|
onIdentityChange: (activeSession, identity) => {
|
|
4780
|
-
|
|
4781
|
-
activeMediaStreams.byCallSid.set(identity.callSid, activeSession);
|
|
5047
|
+
activeMediaStreams.byCallSid.set(identity.callSid, activeSession);
|
|
4782
5048
|
if (identity.outboundId)
|
|
4783
5049
|
activeMediaStreams.byOutboundId.set(identity.outboundId, activeSession);
|
|
4784
5050
|
},
|
|
4785
5051
|
onClose: (activeSession, identity) => {
|
|
4786
|
-
|
|
4787
|
-
|
|
4788
|
-
}
|
|
5052
|
+
// The CallSid reservation and single-use token mean no other session can hold this CallSid.
|
|
5053
|
+
activeMediaStreams.byCallSid.delete(identity.callSid);
|
|
4789
5054
|
if (identity.outboundId && activeMediaStreams.byOutboundId.get(identity.outboundId) === activeSession) {
|
|
4790
5055
|
activeMediaStreams.byOutboundId.delete(identity.outboundId);
|
|
4791
5056
|
}
|
|
4792
5057
|
},
|
|
4793
5058
|
};
|
|
4794
|
-
const
|
|
4795
|
-
const
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
5059
|
+
const socket = wsAsVoiceSessionSocket(ws);
|
|
5060
|
+
const startTimer = setTimeout(() => {
|
|
5061
|
+
ws.off("message", gate);
|
|
5062
|
+
(0, runtime_1.emitNervesEvent)({
|
|
5063
|
+
level: "warn",
|
|
5064
|
+
component: "senses",
|
|
5065
|
+
event: "senses.voice_media_stream_rejected",
|
|
5066
|
+
message: "rejected a Twilio Media Stream that never sent start",
|
|
5067
|
+
meta: { agentName: options.agentName, callSid: "unknown", reason: "start_timeout" },
|
|
5068
|
+
});
|
|
5069
|
+
socket.close();
|
|
5070
|
+
}, MEDIA_STREAM_START_TIMEOUT_MS);
|
|
5071
|
+
ws.on("close", () => {
|
|
5072
|
+
clearTimeout(startTimer);
|
|
5073
|
+
if (reservedCallSid)
|
|
5074
|
+
activeMediaStreams.reserved.delete(reservedCallSid);
|
|
5075
|
+
});
|
|
5076
|
+
// Nothing is decided about a stream until its first `start` is verified; the session class
|
|
5077
|
+
// is chosen from the consumed record, and the verified start is replayed into it.
|
|
5078
|
+
const gate = (raw) => {
|
|
5079
|
+
const message = parseTwilioMediaStreamMessage(raw);
|
|
5080
|
+
if (!message || stringField(message.event) !== "start")
|
|
5081
|
+
return;
|
|
5082
|
+
clearTimeout(startTimer);
|
|
5083
|
+
ws.off("message", gate);
|
|
5084
|
+
const identity = admitMediaStreamStart(options, pendingCalls, activeMediaStreams, message.start);
|
|
5085
|
+
if (!identity) {
|
|
5086
|
+
socket.close();
|
|
5087
|
+
return;
|
|
5088
|
+
}
|
|
5089
|
+
reservedCallSid = identity.callSid;
|
|
5090
|
+
const realtime = identity.engine === "openai-realtime" || (!identity.engine && usesOpenAIRealtimeConversationEngine(options));
|
|
5091
|
+
const session = realtime
|
|
5092
|
+
? new TwilioOpenAIRealtimeMediaStreamSession(socket, identity, options, lifecycle)
|
|
5093
|
+
: new TwilioMediaStreamSession(socket, identity, options, jobs, lifecycle);
|
|
5094
|
+
session.attach();
|
|
5095
|
+
session.handleRawMessage(raw);
|
|
5096
|
+
};
|
|
5097
|
+
ws.on("message", gate);
|
|
4799
5098
|
});
|
|
4800
5099
|
return {
|
|
4801
5100
|
async handle(request) {
|
|
@@ -4819,7 +5118,7 @@ function createTwilioPhoneBridge(options) {
|
|
|
4819
5118
|
if (method !== "POST")
|
|
4820
5119
|
return textResponse(405, "method not allowed");
|
|
4821
5120
|
if (routePath === sipWebhookPath) {
|
|
4822
|
-
return handleOpenAISipWebhook(options, { ...request, path: requestPath }, activeSipSessions);
|
|
5121
|
+
return handleOpenAISipWebhook(options, { ...request, path: requestPath }, activeSipSessions, pendingCalls, recentSipWebhooks);
|
|
4823
5122
|
}
|
|
4824
5123
|
const params = formParams(bodyText(request.body));
|
|
4825
5124
|
if (!verifyRequest(options, { ...request, path: requestPath }, params)) {
|
|
@@ -4833,7 +5132,20 @@ function createTwilioPhoneBridge(options) {
|
|
|
4833
5132
|
return textResponse(403, "invalid Twilio signature");
|
|
4834
5133
|
}
|
|
4835
5134
|
if (routePath === `${basePath}/incoming`)
|
|
4836
|
-
return handleIncoming(options, basePath, params, jobs);
|
|
5135
|
+
return handleIncoming(options, basePath, params, jobs, pendingCalls);
|
|
5136
|
+
if (routePath === `${basePath}/stream-ended`) {
|
|
5137
|
+
// `<Connect action>` fires when the stream ends. A normal end just hangs up; a call whose
|
|
5138
|
+
// stream was refused (or never connected) hears the failure line instead of dead air.
|
|
5139
|
+
const callSid = params.CallSid?.trim() ?? "";
|
|
5140
|
+
const settledPending = pendingCalls.settle(callSid);
|
|
5141
|
+
(0, runtime_1.emitNervesEvent)({
|
|
5142
|
+
component: "senses",
|
|
5143
|
+
event: "senses.voice_twilio_stream_ended",
|
|
5144
|
+
message: "Twilio reported the call's media stream ended",
|
|
5145
|
+
meta: { agentName: options.agentName, callSid: safeSegment(callSid), settledPending },
|
|
5146
|
+
});
|
|
5147
|
+
return xmlResponse(settledPending ? streamFailureTwiml() : "<Hangup />");
|
|
5148
|
+
}
|
|
4837
5149
|
if (routePath.startsWith(`${basePath}/outgoing/`)) {
|
|
4838
5150
|
const outgoingRest = routePath.slice(`${basePath}/outgoing/`.length);
|
|
4839
5151
|
const [outboundIdPart, suffix] = outgoingRest.split("/");
|
|
@@ -4845,7 +5157,7 @@ function createTwilioPhoneBridge(options) {
|
|
|
4845
5157
|
if (suffix === "amd")
|
|
4846
5158
|
return handleOutgoingAmdStatus(options, outboundId, params, activeMediaStreams, activeSipSessions);
|
|
4847
5159
|
if (suffix === undefined)
|
|
4848
|
-
return handleOutgoing(options, basePath, outboundId, params, jobs);
|
|
5160
|
+
return handleOutgoing(options, basePath, outboundId, params, jobs, pendingCalls);
|
|
4849
5161
|
}
|
|
4850
5162
|
if (routePath === `${basePath}/listen`)
|
|
4851
5163
|
return handleListen(options, basePath);
|
|
@@ -4878,7 +5190,6 @@ function createTwilioPhoneBridge(options) {
|
|
|
4878
5190
|
},
|
|
4879
5191
|
};
|
|
4880
5192
|
}
|
|
4881
|
-
/* v8 ignore stop */
|
|
4882
5193
|
/* v8 ignore start -- HTTP server adapter behavior is covered through startTwilioPhoneBridgeServer smoke tests; low-level stream disconnect branches are platform-dependent @preserve */
|
|
4883
5194
|
function readRequestBody(req, limitBytes = 1_000_000) {
|
|
4884
5195
|
return new Promise((resolve, reject) => {
|