@kognitivedev/voice-media-bridge 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/CHANGELOG.md +10 -0
- package/dist/audio.d.ts +34 -0
- package/dist/audio.js +163 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +20 -0
- package/dist/openai.d.ts +62 -0
- package/dist/openai.js +130 -0
- package/dist/profiles.d.ts +38 -0
- package/dist/profiles.js +97 -0
- package/dist/routes.d.ts +51 -0
- package/dist/routes.js +98 -0
- package/package.json +39 -0
- package/src/__tests__/media.test.ts +229 -0
- package/src/audio.ts +182 -0
- package/src/index.ts +4 -0
- package/src/openai.ts +127 -0
- package/src/profiles.ts +135 -0
- package/src/routes.ts +139 -0
- package/tsconfig.json +13 -0
package/CHANGELOG.md
ADDED
package/dist/audio.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { PhoneMediaProfile } from "./profiles";
|
|
2
|
+
import type { PhoneAudioRoute } from "./routes";
|
|
3
|
+
export interface PcmSignalSummary {
|
|
4
|
+
samples: number;
|
|
5
|
+
peakAbs: number;
|
|
6
|
+
rms: number;
|
|
7
|
+
clippedSamples: number;
|
|
8
|
+
clippingRatio: number;
|
|
9
|
+
bytes: number;
|
|
10
|
+
}
|
|
11
|
+
export interface CarrierAudioOutput {
|
|
12
|
+
payload: string;
|
|
13
|
+
durationMs: number;
|
|
14
|
+
signal: PcmSignalSummary;
|
|
15
|
+
samples: Int16Array;
|
|
16
|
+
}
|
|
17
|
+
export declare const OPENAI_PCM_SAMPLE_RATE = 24000;
|
|
18
|
+
export declare function pcm16ToBase64Le(samples: Int16Array): string;
|
|
19
|
+
export declare function base64ToPcm16Le(data: string): Int16Array;
|
|
20
|
+
export declare function pcm16ToBase64Be(samples: Int16Array): string;
|
|
21
|
+
export declare function base64ToPcm16Be(data: string): Int16Array;
|
|
22
|
+
export declare function summarizePcm16Signal(samples: Int16Array, bytes?: number): PcmSignalSummary;
|
|
23
|
+
export declare function frameDurationMs(sampleCount: number, sampleRate: number): number;
|
|
24
|
+
export declare function decodeCarrierPayload(profile: PhoneMediaProfile, payload: string): Int16Array;
|
|
25
|
+
export declare function encodeCarrierPayload(profile: PhoneMediaProfile, samples: Int16Array): string;
|
|
26
|
+
export declare function summarizeCarrierPayload(profile: PhoneMediaProfile, payload: string): PcmSignalSummary;
|
|
27
|
+
export declare function toOpenAIInputAudio(profile: PhoneMediaProfile, payload: string): string;
|
|
28
|
+
export declare function fromOpenAIOutputAudio(profile: PhoneMediaProfile, delta: string): CarrierAudioOutput;
|
|
29
|
+
export declare function toAiInputAudio(route: PhoneAudioRoute, payload: string): string;
|
|
30
|
+
export declare function fromAiOutputAudio(route: PhoneAudioRoute, delta: string, sourceRate?: 8000 | 16000 | 24000): CarrierAudioOutput;
|
|
31
|
+
export declare function isLikelySpeechPayload(profile: PhoneMediaProfile, payload: string, input?: {
|
|
32
|
+
rms?: number;
|
|
33
|
+
peakAbs?: number;
|
|
34
|
+
}): boolean;
|
package/dist/audio.js
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.OPENAI_PCM_SAMPLE_RATE = void 0;
|
|
4
|
+
exports.pcm16ToBase64Le = pcm16ToBase64Le;
|
|
5
|
+
exports.base64ToPcm16Le = base64ToPcm16Le;
|
|
6
|
+
exports.pcm16ToBase64Be = pcm16ToBase64Be;
|
|
7
|
+
exports.base64ToPcm16Be = base64ToPcm16Be;
|
|
8
|
+
exports.summarizePcm16Signal = summarizePcm16Signal;
|
|
9
|
+
exports.frameDurationMs = frameDurationMs;
|
|
10
|
+
exports.decodeCarrierPayload = decodeCarrierPayload;
|
|
11
|
+
exports.encodeCarrierPayload = encodeCarrierPayload;
|
|
12
|
+
exports.summarizeCarrierPayload = summarizeCarrierPayload;
|
|
13
|
+
exports.toOpenAIInputAudio = toOpenAIInputAudio;
|
|
14
|
+
exports.fromOpenAIOutputAudio = fromOpenAIOutputAudio;
|
|
15
|
+
exports.toAiInputAudio = toAiInputAudio;
|
|
16
|
+
exports.fromAiOutputAudio = fromAiOutputAudio;
|
|
17
|
+
exports.isLikelySpeechPayload = isLikelySpeechPayload;
|
|
18
|
+
const telephony_1 = require("@kognitivedev/telephony");
|
|
19
|
+
exports.OPENAI_PCM_SAMPLE_RATE = 24000;
|
|
20
|
+
const CLIP_THRESHOLD = 32760;
|
|
21
|
+
function pcm16ToBase64Le(samples) {
|
|
22
|
+
var _a;
|
|
23
|
+
const buffer = Buffer.alloc(samples.length * 2);
|
|
24
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
25
|
+
buffer.writeInt16LE((_a = samples[index]) !== null && _a !== void 0 ? _a : 0, index * 2);
|
|
26
|
+
}
|
|
27
|
+
return buffer.toString("base64");
|
|
28
|
+
}
|
|
29
|
+
function base64ToPcm16Le(data) {
|
|
30
|
+
const buffer = Buffer.from(data, "base64");
|
|
31
|
+
const samples = new Int16Array(Math.floor(buffer.length / 2));
|
|
32
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
33
|
+
samples[index] = buffer.readInt16LE(index * 2);
|
|
34
|
+
}
|
|
35
|
+
return samples;
|
|
36
|
+
}
|
|
37
|
+
function pcm16ToBase64Be(samples) {
|
|
38
|
+
var _a;
|
|
39
|
+
const buffer = Buffer.alloc(samples.length * 2);
|
|
40
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
41
|
+
buffer.writeInt16BE((_a = samples[index]) !== null && _a !== void 0 ? _a : 0, index * 2);
|
|
42
|
+
}
|
|
43
|
+
return buffer.toString("base64");
|
|
44
|
+
}
|
|
45
|
+
function base64ToPcm16Be(data) {
|
|
46
|
+
const buffer = Buffer.from(data, "base64");
|
|
47
|
+
const samples = new Int16Array(Math.floor(buffer.length / 2));
|
|
48
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
49
|
+
samples[index] = buffer.readInt16BE(index * 2);
|
|
50
|
+
}
|
|
51
|
+
return samples;
|
|
52
|
+
}
|
|
53
|
+
function summarizePcm16Signal(samples, bytes = samples.length * 2) {
|
|
54
|
+
if (samples.length === 0) {
|
|
55
|
+
return { samples: 0, peakAbs: 0, rms: 0, clippedSamples: 0, clippingRatio: 0, bytes };
|
|
56
|
+
}
|
|
57
|
+
let peakAbs = 0;
|
|
58
|
+
let squareSum = 0;
|
|
59
|
+
let clippedSamples = 0;
|
|
60
|
+
for (const sample of samples) {
|
|
61
|
+
const abs = Math.abs(sample);
|
|
62
|
+
if (abs > peakAbs)
|
|
63
|
+
peakAbs = abs;
|
|
64
|
+
if (abs >= CLIP_THRESHOLD)
|
|
65
|
+
clippedSamples += 1;
|
|
66
|
+
squareSum += sample * sample;
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
samples: samples.length,
|
|
70
|
+
peakAbs,
|
|
71
|
+
rms: Math.round(Math.sqrt(squareSum / samples.length)),
|
|
72
|
+
clippedSamples,
|
|
73
|
+
clippingRatio: clippedSamples / samples.length,
|
|
74
|
+
bytes,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function frameDurationMs(sampleCount, sampleRate) {
|
|
78
|
+
if (sampleRate <= 0)
|
|
79
|
+
return 0;
|
|
80
|
+
return Math.round((sampleCount / sampleRate) * 1000);
|
|
81
|
+
}
|
|
82
|
+
function decodeCarrierPayload(profile, payload) {
|
|
83
|
+
if (profile.codec === "pcmu")
|
|
84
|
+
return (0, telephony_1.decodeTwilioMulawBase64ToPcm16)(payload);
|
|
85
|
+
return profile.carrierByteOrder === "le" ? base64ToPcm16Le(payload) : base64ToPcm16Be(payload);
|
|
86
|
+
}
|
|
87
|
+
function encodeCarrierPayload(profile, samples) {
|
|
88
|
+
if (profile.codec === "pcmu")
|
|
89
|
+
return (0, telephony_1.encodePcm16ToTwilioMulawBase64)(samples);
|
|
90
|
+
return profile.carrierByteOrder === "le" ? pcm16ToBase64Le(samples) : pcm16ToBase64Be(samples);
|
|
91
|
+
}
|
|
92
|
+
function summarizeCarrierPayload(profile, payload) {
|
|
93
|
+
const bytes = Buffer.byteLength(payload, "base64");
|
|
94
|
+
return summarizePcm16Signal(decodeCarrierPayload(profile, payload), bytes);
|
|
95
|
+
}
|
|
96
|
+
function toOpenAIInputAudio(profile, payload) {
|
|
97
|
+
if (profile.openAIFormat === "audio/pcmu")
|
|
98
|
+
return payload;
|
|
99
|
+
const carrierPcm = decodeCarrierPayload(profile, payload);
|
|
100
|
+
const openAIPcm = (0, telephony_1.resamplePcm16WindowedSinc)(carrierPcm, profile.carrierSampleRate, profile.openAISampleRate);
|
|
101
|
+
return pcm16ToBase64Le(openAIPcm);
|
|
102
|
+
}
|
|
103
|
+
function fromOpenAIOutputAudio(profile, delta) {
|
|
104
|
+
if (profile.openAIFormat === "audio/pcmu") {
|
|
105
|
+
const samples = decodeCarrierPayload(profile, delta);
|
|
106
|
+
return {
|
|
107
|
+
payload: delta,
|
|
108
|
+
durationMs: frameDurationMs(samples.length, profile.carrierSampleRate),
|
|
109
|
+
signal: summarizePcm16Signal(samples, Buffer.byteLength(delta, "base64")),
|
|
110
|
+
samples,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
const openAIPcm = base64ToPcm16Le(delta);
|
|
114
|
+
const carrierPcm = (0, telephony_1.resamplePcm16WindowedSinc)(openAIPcm, profile.openAISampleRate, profile.carrierSampleRate);
|
|
115
|
+
const payload = encodeCarrierPayload(profile, carrierPcm);
|
|
116
|
+
return {
|
|
117
|
+
payload,
|
|
118
|
+
durationMs: frameDurationMs(carrierPcm.length, profile.carrierSampleRate),
|
|
119
|
+
signal: summarizePcm16Signal(carrierPcm, Buffer.byteLength(payload, "base64")),
|
|
120
|
+
samples: carrierPcm,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
function toAiInputAudio(route, payload) {
|
|
124
|
+
if (route.carrier.codec === "pcmu"
|
|
125
|
+
&& route.aiInput.codec === "pcmu"
|
|
126
|
+
&& route.carrier.carrierSampleRate === route.aiInput.sampleRate) {
|
|
127
|
+
return payload;
|
|
128
|
+
}
|
|
129
|
+
const carrierPcm = decodeCarrierPayload(route.carrier, payload);
|
|
130
|
+
const aiPcm = route.carrier.carrierSampleRate === route.aiInput.sampleRate
|
|
131
|
+
? carrierPcm
|
|
132
|
+
: (0, telephony_1.resamplePcm16WindowedSinc)(carrierPcm, route.carrier.carrierSampleRate, route.aiInput.sampleRate);
|
|
133
|
+
return pcm16ToBase64Le(aiPcm);
|
|
134
|
+
}
|
|
135
|
+
function fromAiOutputAudio(route, delta, sourceRate = route.aiOutput.sampleRate) {
|
|
136
|
+
if (route.carrier.codec === "pcmu"
|
|
137
|
+
&& route.aiOutput.codec === "pcmu"
|
|
138
|
+
&& route.carrier.carrierSampleRate === route.aiOutput.sampleRate) {
|
|
139
|
+
const samples = decodeCarrierPayload(route.carrier, delta);
|
|
140
|
+
return {
|
|
141
|
+
payload: delta,
|
|
142
|
+
durationMs: frameDurationMs(samples.length, route.carrier.carrierSampleRate),
|
|
143
|
+
signal: summarizePcm16Signal(samples, Buffer.byteLength(delta, "base64")),
|
|
144
|
+
samples,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const aiPcm = base64ToPcm16Le(delta);
|
|
148
|
+
const carrierPcm = sourceRate === route.carrier.carrierSampleRate
|
|
149
|
+
? aiPcm
|
|
150
|
+
: (0, telephony_1.resamplePcm16WindowedSinc)(aiPcm, sourceRate, route.carrier.carrierSampleRate);
|
|
151
|
+
const payload = encodeCarrierPayload(route.carrier, carrierPcm);
|
|
152
|
+
return {
|
|
153
|
+
payload,
|
|
154
|
+
durationMs: frameDurationMs(carrierPcm.length, route.carrier.carrierSampleRate),
|
|
155
|
+
signal: summarizePcm16Signal(carrierPcm, Buffer.byteLength(payload, "base64")),
|
|
156
|
+
samples: carrierPcm,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
function isLikelySpeechPayload(profile, payload, input = {}) {
|
|
160
|
+
var _a, _b;
|
|
161
|
+
const signal = summarizeCarrierPayload(profile, payload);
|
|
162
|
+
return signal.rms >= ((_a = input.rms) !== null && _a !== void 0 ? _a : 400) || signal.peakAbs >= ((_b = input.peakAbs) !== null && _b !== void 0 ? _b : 2500);
|
|
163
|
+
}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./audio"), exports);
|
|
18
|
+
__exportStar(require("./openai"), exports);
|
|
19
|
+
__exportStar(require("./profiles"), exports);
|
|
20
|
+
__exportStar(require("./routes"), exports);
|
package/dist/openai.d.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { PhoneMediaProfile } from "./profiles";
|
|
2
|
+
import type { PhoneAudioRoute } from "./routes";
|
|
3
|
+
export declare function openAIRealtimeAudioFormat(profile: PhoneMediaProfile): {
|
|
4
|
+
type: string;
|
|
5
|
+
rate?: undefined;
|
|
6
|
+
} | {
|
|
7
|
+
type: string;
|
|
8
|
+
rate: 8000 | 24000;
|
|
9
|
+
};
|
|
10
|
+
export declare function realtimeAudioFormatForRoute(route: PhoneAudioRoute): {
|
|
11
|
+
type: string;
|
|
12
|
+
rate?: undefined;
|
|
13
|
+
} | {
|
|
14
|
+
type: string;
|
|
15
|
+
rate: 8000 | 16000 | 24000;
|
|
16
|
+
};
|
|
17
|
+
export declare function realtimeOutputAudioFormatForRoute(route: PhoneAudioRoute): {
|
|
18
|
+
type: string;
|
|
19
|
+
rate?: undefined;
|
|
20
|
+
} | {
|
|
21
|
+
type: string;
|
|
22
|
+
rate: 8000 | 16000 | 24000;
|
|
23
|
+
};
|
|
24
|
+
export declare function isOpenAIRealtimeAudioReady(event: unknown, profile?: PhoneMediaProfile): boolean;
|
|
25
|
+
export declare function isRealtimeAudioReadyForRoute(event: unknown, route: PhoneAudioRoute): boolean;
|
|
26
|
+
export declare function createCarrierMediaMessage(profile: PhoneMediaProfile, streamSid: string | null, payload: string): {
|
|
27
|
+
event: string;
|
|
28
|
+
media: {
|
|
29
|
+
payload: string;
|
|
30
|
+
};
|
|
31
|
+
streamSid?: undefined;
|
|
32
|
+
} | {
|
|
33
|
+
event: string;
|
|
34
|
+
streamSid: string;
|
|
35
|
+
media: {
|
|
36
|
+
payload: string;
|
|
37
|
+
};
|
|
38
|
+
} | null;
|
|
39
|
+
export declare function createCarrierMarkMessage(profile: PhoneMediaProfile, streamSid: string | null, markName: string): {
|
|
40
|
+
event: string;
|
|
41
|
+
mark: {
|
|
42
|
+
name: string;
|
|
43
|
+
};
|
|
44
|
+
streamSid?: undefined;
|
|
45
|
+
} | {
|
|
46
|
+
event: string;
|
|
47
|
+
streamSid: string;
|
|
48
|
+
mark: {
|
|
49
|
+
name: string;
|
|
50
|
+
};
|
|
51
|
+
} | null;
|
|
52
|
+
export declare function createCarrierClearMessage(profile: PhoneMediaProfile, streamSid: string | null): {
|
|
53
|
+
event: string;
|
|
54
|
+
streamSid?: undefined;
|
|
55
|
+
} | {
|
|
56
|
+
event: string;
|
|
57
|
+
streamSid: string;
|
|
58
|
+
} | null;
|
|
59
|
+
export interface CreateOpenAIPhoneRealtimeBridgeInput {
|
|
60
|
+
profile: PhoneMediaProfile;
|
|
61
|
+
}
|
|
62
|
+
export declare function createOpenAIPhoneRealtimeBridge(_input: CreateOpenAIPhoneRealtimeBridgeInput): never;
|
package/dist/openai.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.openAIRealtimeAudioFormat = openAIRealtimeAudioFormat;
|
|
4
|
+
exports.realtimeAudioFormatForRoute = realtimeAudioFormatForRoute;
|
|
5
|
+
exports.realtimeOutputAudioFormatForRoute = realtimeOutputAudioFormatForRoute;
|
|
6
|
+
exports.isOpenAIRealtimeAudioReady = isOpenAIRealtimeAudioReady;
|
|
7
|
+
exports.isRealtimeAudioReadyForRoute = isRealtimeAudioReadyForRoute;
|
|
8
|
+
exports.createCarrierMediaMessage = createCarrierMediaMessage;
|
|
9
|
+
exports.createCarrierMarkMessage = createCarrierMarkMessage;
|
|
10
|
+
exports.createCarrierClearMessage = createCarrierClearMessage;
|
|
11
|
+
exports.createOpenAIPhoneRealtimeBridge = createOpenAIPhoneRealtimeBridge;
|
|
12
|
+
const profiles_1 = require("./profiles");
|
|
13
|
+
function getRecord(value) {
|
|
14
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
15
|
+
}
|
|
16
|
+
function getString(value, fallback = "") {
|
|
17
|
+
return typeof value === "string" && value.trim() ? value.trim() : fallback;
|
|
18
|
+
}
|
|
19
|
+
function summarizeOpenAIAudioFormat(value) {
|
|
20
|
+
if (typeof value === "string")
|
|
21
|
+
return value;
|
|
22
|
+
const record = getRecord(value);
|
|
23
|
+
const type = getString(record.type, "");
|
|
24
|
+
const rate = typeof record.rate === "number" ? record.rate : undefined;
|
|
25
|
+
return type ? Object.assign({ type }, (rate ? { rate } : {})) : null;
|
|
26
|
+
}
|
|
27
|
+
function openAIRealtimeAudioFormat(profile) {
|
|
28
|
+
return profile.openAIFormat === "audio/pcmu"
|
|
29
|
+
? { type: "audio/pcmu" }
|
|
30
|
+
: { type: "audio/pcm", rate: profile.openAISampleRate };
|
|
31
|
+
}
|
|
32
|
+
function realtimeAudioFormatForRoute(route) {
|
|
33
|
+
return route.aiInput.format === "audio/pcmu"
|
|
34
|
+
? { type: "audio/pcmu" }
|
|
35
|
+
: { type: "audio/pcm", rate: route.aiInput.sampleRate };
|
|
36
|
+
}
|
|
37
|
+
function realtimeOutputAudioFormatForRoute(route) {
|
|
38
|
+
return route.aiOutput.format === "audio/pcmu"
|
|
39
|
+
? { type: "audio/pcmu" }
|
|
40
|
+
: { type: "audio/pcm", rate: route.aiOutput.sampleRate };
|
|
41
|
+
}
|
|
42
|
+
function isOpenAIRealtimeAudioReady(event, profile = (0, profiles_1.getPhoneMediaProfile)("twilio-pcmu-8k")) {
|
|
43
|
+
const record = getRecord(event);
|
|
44
|
+
if (record.type !== "session.updated")
|
|
45
|
+
return false;
|
|
46
|
+
const audio = getRecord(getRecord(record.session).audio);
|
|
47
|
+
const inputFormat = summarizeOpenAIAudioFormat(getRecord(audio.input).format);
|
|
48
|
+
const outputFormat = summarizeOpenAIAudioFormat(getRecord(audio.output).format);
|
|
49
|
+
const inputRecord = getRecord(inputFormat);
|
|
50
|
+
const outputRecord = getRecord(outputFormat);
|
|
51
|
+
const expected = openAIRealtimeAudioFormat(profile);
|
|
52
|
+
const expectedType = expected.type;
|
|
53
|
+
const expectedRate = "rate" in expected ? expected.rate : undefined;
|
|
54
|
+
const inputType = typeof inputFormat === "string" ? inputFormat : getString(inputRecord.type, "");
|
|
55
|
+
const outputType = typeof outputFormat === "string" ? outputFormat : getString(outputRecord.type, "");
|
|
56
|
+
const inputRate = typeof inputRecord.rate === "number" ? inputRecord.rate : undefined;
|
|
57
|
+
const outputRate = typeof outputRecord.rate === "number" ? outputRecord.rate : undefined;
|
|
58
|
+
if (inputType !== expectedType || outputType !== expectedType)
|
|
59
|
+
return false;
|
|
60
|
+
if (expectedRate && inputRate && inputRate !== expectedRate)
|
|
61
|
+
return false;
|
|
62
|
+
if (expectedRate && outputRate && outputRate !== expectedRate)
|
|
63
|
+
return false;
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
function isRealtimeAudioReadyForRoute(event, route) {
|
|
67
|
+
const record = getRecord(event);
|
|
68
|
+
if (record.type !== "session.updated")
|
|
69
|
+
return false;
|
|
70
|
+
const audio = getRecord(getRecord(record.session).audio);
|
|
71
|
+
const inputFormat = summarizeOpenAIAudioFormat(getRecord(audio.input).format);
|
|
72
|
+
const outputFormat = summarizeOpenAIAudioFormat(getRecord(audio.output).format);
|
|
73
|
+
const inputRecord = getRecord(inputFormat);
|
|
74
|
+
const outputRecord = getRecord(outputFormat);
|
|
75
|
+
const expectedInput = realtimeAudioFormatForRoute(route);
|
|
76
|
+
const expectedOutput = realtimeOutputAudioFormatForRoute(route);
|
|
77
|
+
const inputType = typeof inputFormat === "string" ? inputFormat : getString(inputRecord.type, "");
|
|
78
|
+
const outputType = typeof outputFormat === "string" ? outputFormat : getString(outputRecord.type, "");
|
|
79
|
+
const inputRate = typeof inputRecord.rate === "number" ? inputRecord.rate : undefined;
|
|
80
|
+
const outputRate = typeof outputRecord.rate === "number" ? outputRecord.rate : undefined;
|
|
81
|
+
if (inputType !== expectedInput.type || outputType !== expectedOutput.type)
|
|
82
|
+
return false;
|
|
83
|
+
if ("rate" in expectedInput && inputRate && inputRate !== expectedInput.rate)
|
|
84
|
+
return false;
|
|
85
|
+
if ("rate" in expectedOutput && outputRate && outputRate !== expectedOutput.rate)
|
|
86
|
+
return false;
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
function createCarrierMediaMessage(profile, streamSid, payload) {
|
|
90
|
+
if (profile.provider === "telnyx") {
|
|
91
|
+
return { event: "media", media: { payload } };
|
|
92
|
+
}
|
|
93
|
+
if (profile.provider === "sip") {
|
|
94
|
+
return { event: "media", streamSid: streamSid !== null && streamSid !== void 0 ? streamSid : "sip", media: { payload } };
|
|
95
|
+
}
|
|
96
|
+
if (!streamSid)
|
|
97
|
+
return null;
|
|
98
|
+
return {
|
|
99
|
+
event: "media",
|
|
100
|
+
streamSid,
|
|
101
|
+
media: { payload },
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function createCarrierMarkMessage(profile, streamSid, markName) {
|
|
105
|
+
if (profile.provider === "telnyx") {
|
|
106
|
+
return { event: "mark", mark: { name: markName } };
|
|
107
|
+
}
|
|
108
|
+
if (profile.provider === "sip") {
|
|
109
|
+
return { event: "mark", streamSid: streamSid !== null && streamSid !== void 0 ? streamSid : "sip", mark: { name: markName } };
|
|
110
|
+
}
|
|
111
|
+
if (!streamSid)
|
|
112
|
+
return null;
|
|
113
|
+
return {
|
|
114
|
+
event: "mark",
|
|
115
|
+
streamSid,
|
|
116
|
+
mark: { name: markName },
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function createCarrierClearMessage(profile, streamSid) {
|
|
120
|
+
if (profile.provider === "telnyx")
|
|
121
|
+
return { event: "clear" };
|
|
122
|
+
if (profile.provider === "sip")
|
|
123
|
+
return { event: "clear", streamSid: streamSid !== null && streamSid !== void 0 ? streamSid : "sip" };
|
|
124
|
+
if (!streamSid)
|
|
125
|
+
return null;
|
|
126
|
+
return { event: "clear", streamSid };
|
|
127
|
+
}
|
|
128
|
+
function createOpenAIPhoneRealtimeBridge(_input) {
|
|
129
|
+
throw new Error("createOpenAIPhoneRealtimeBridge is hosted by the backend-cloud phone runtime in this release");
|
|
130
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export type PhoneMediaProvider = "twilio" | "telnyx" | "sip";
|
|
2
|
+
export type PhoneMediaProfileId = "twilio-pcmu-8k" | "telnyx-pcmu-8k" | "telnyx-l16-16k" | "sip-pcmu-8k";
|
|
3
|
+
export type PhoneMediaCodecPreference = "auto" | "l16_16k" | "pcmu_8k";
|
|
4
|
+
export type PhoneMediaL16ByteOrder = "be" | "le";
|
|
5
|
+
export interface PhoneMediaProfile {
|
|
6
|
+
id: PhoneMediaProfileId;
|
|
7
|
+
provider: PhoneMediaProvider;
|
|
8
|
+
codec: "pcmu" | "l16";
|
|
9
|
+
carrierCodec: "PCMU" | "L16";
|
|
10
|
+
carrierSampleRate: 8000 | 16000;
|
|
11
|
+
carrierByteOrder?: PhoneMediaL16ByteOrder;
|
|
12
|
+
openAIFormat: "audio/pcmu" | "audio/pcm";
|
|
13
|
+
openAISampleRate: 8000 | 24000;
|
|
14
|
+
source: "auto" | "forced" | "fallback";
|
|
15
|
+
}
|
|
16
|
+
export interface PhoneMediaFallbackState {
|
|
17
|
+
forcePcmu?: boolean;
|
|
18
|
+
}
|
|
19
|
+
export interface ResolvePhoneMediaProfileInput {
|
|
20
|
+
provider: PhoneMediaProvider;
|
|
21
|
+
requestedCodec?: PhoneMediaCodecPreference | string | null;
|
|
22
|
+
l16ByteOrder?: PhoneMediaL16ByteOrder | string | null;
|
|
23
|
+
fallbackState?: PhoneMediaFallbackState | null;
|
|
24
|
+
}
|
|
25
|
+
export declare function getPhoneMediaProfile(id: PhoneMediaProfileId): PhoneMediaProfile;
|
|
26
|
+
export declare function normalizePhoneMediaCodecPreference(value: unknown): PhoneMediaCodecPreference;
|
|
27
|
+
export declare function normalizePhoneMediaL16ByteOrder(value: unknown): PhoneMediaL16ByteOrder;
|
|
28
|
+
export declare function resolvePhoneMediaProfile(input: ResolvePhoneMediaProfileInput): PhoneMediaProfile;
|
|
29
|
+
export declare function phoneMediaProfileMetadata(profile: PhoneMediaProfile): {
|
|
30
|
+
openAIFormat: "audio/pcmu" | "audio/pcm";
|
|
31
|
+
openAISampleRate: 8000 | 24000;
|
|
32
|
+
source: "auto" | "forced" | "fallback";
|
|
33
|
+
byteOrder?: PhoneMediaL16ByteOrder | undefined;
|
|
34
|
+
id: PhoneMediaProfileId;
|
|
35
|
+
provider: PhoneMediaProvider;
|
|
36
|
+
codec: "PCMU" | "L16";
|
|
37
|
+
sampleRate: 8000 | 16000;
|
|
38
|
+
};
|
package/dist/profiles.js
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getPhoneMediaProfile = getPhoneMediaProfile;
|
|
4
|
+
exports.normalizePhoneMediaCodecPreference = normalizePhoneMediaCodecPreference;
|
|
5
|
+
exports.normalizePhoneMediaL16ByteOrder = normalizePhoneMediaL16ByteOrder;
|
|
6
|
+
exports.resolvePhoneMediaProfile = resolvePhoneMediaProfile;
|
|
7
|
+
exports.phoneMediaProfileMetadata = phoneMediaProfileMetadata;
|
|
8
|
+
const TWILIO_PCMU_8K = {
|
|
9
|
+
id: "twilio-pcmu-8k",
|
|
10
|
+
provider: "twilio",
|
|
11
|
+
codec: "pcmu",
|
|
12
|
+
carrierCodec: "PCMU",
|
|
13
|
+
carrierSampleRate: 8000,
|
|
14
|
+
openAIFormat: "audio/pcmu",
|
|
15
|
+
openAISampleRate: 8000,
|
|
16
|
+
source: "forced",
|
|
17
|
+
};
|
|
18
|
+
const TELNYX_PCMU_8K = {
|
|
19
|
+
id: "telnyx-pcmu-8k",
|
|
20
|
+
provider: "telnyx",
|
|
21
|
+
codec: "pcmu",
|
|
22
|
+
carrierCodec: "PCMU",
|
|
23
|
+
carrierSampleRate: 8000,
|
|
24
|
+
openAIFormat: "audio/pcmu",
|
|
25
|
+
openAISampleRate: 8000,
|
|
26
|
+
source: "forced",
|
|
27
|
+
};
|
|
28
|
+
const TELNYX_L16_16K = {
|
|
29
|
+
id: "telnyx-l16-16k",
|
|
30
|
+
provider: "telnyx",
|
|
31
|
+
codec: "l16",
|
|
32
|
+
carrierCodec: "L16",
|
|
33
|
+
carrierSampleRate: 16000,
|
|
34
|
+
carrierByteOrder: "le",
|
|
35
|
+
openAIFormat: "audio/pcm",
|
|
36
|
+
openAISampleRate: 24000,
|
|
37
|
+
source: "auto",
|
|
38
|
+
};
|
|
39
|
+
const SIP_PCMU_8K = {
|
|
40
|
+
id: "sip-pcmu-8k",
|
|
41
|
+
provider: "sip",
|
|
42
|
+
codec: "pcmu",
|
|
43
|
+
carrierCodec: "PCMU",
|
|
44
|
+
carrierSampleRate: 8000,
|
|
45
|
+
openAIFormat: "audio/pcmu",
|
|
46
|
+
openAISampleRate: 8000,
|
|
47
|
+
source: "forced",
|
|
48
|
+
};
|
|
49
|
+
function getPhoneMediaProfile(id) {
|
|
50
|
+
switch (id) {
|
|
51
|
+
case "twilio-pcmu-8k":
|
|
52
|
+
return Object.assign({}, TWILIO_PCMU_8K);
|
|
53
|
+
case "telnyx-pcmu-8k":
|
|
54
|
+
return Object.assign({}, TELNYX_PCMU_8K);
|
|
55
|
+
case "telnyx-l16-16k":
|
|
56
|
+
return Object.assign({}, TELNYX_L16_16K);
|
|
57
|
+
case "sip-pcmu-8k":
|
|
58
|
+
return Object.assign({}, SIP_PCMU_8K);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function normalizePhoneMediaCodecPreference(value) {
|
|
62
|
+
if (value === "l16_16k" || value === "pcmu_8k" || value === "auto")
|
|
63
|
+
return value;
|
|
64
|
+
if (value === "L16" || value === "l16" || value === "telnyx-l16-16k")
|
|
65
|
+
return "l16_16k";
|
|
66
|
+
if (value === "PCMU" || value === "pcmu" || value === "telnyx-pcmu-8k" || value === "twilio-pcmu-8k")
|
|
67
|
+
return "pcmu_8k";
|
|
68
|
+
return "auto";
|
|
69
|
+
}
|
|
70
|
+
function normalizePhoneMediaL16ByteOrder(value) {
|
|
71
|
+
return value === "le" || value === "little" || value === "little-endian" ? "le" : "be";
|
|
72
|
+
}
|
|
73
|
+
function resolveL16ByteOrder(value, fallback) {
|
|
74
|
+
return value === undefined || value === null || value === "" ? fallback : normalizePhoneMediaL16ByteOrder(value);
|
|
75
|
+
}
|
|
76
|
+
function resolvePhoneMediaProfile(input) {
|
|
77
|
+
var _a, _b;
|
|
78
|
+
if (input.provider === "twilio")
|
|
79
|
+
return getPhoneMediaProfile("twilio-pcmu-8k");
|
|
80
|
+
if (input.provider === "sip")
|
|
81
|
+
return getPhoneMediaProfile("sip-pcmu-8k");
|
|
82
|
+
const requested = normalizePhoneMediaCodecPreference(input.requestedCodec);
|
|
83
|
+
if ((_a = input.fallbackState) === null || _a === void 0 ? void 0 : _a.forcePcmu) {
|
|
84
|
+
return Object.assign(Object.assign({}, getPhoneMediaProfile("telnyx-pcmu-8k")), { source: "fallback" });
|
|
85
|
+
}
|
|
86
|
+
if (requested === "pcmu_8k") {
|
|
87
|
+
return Object.assign(Object.assign({}, getPhoneMediaProfile("telnyx-pcmu-8k")), { source: "forced" });
|
|
88
|
+
}
|
|
89
|
+
if (requested === "l16_16k") {
|
|
90
|
+
const profile = getPhoneMediaProfile("telnyx-l16-16k");
|
|
91
|
+
return Object.assign(Object.assign({}, profile), { carrierByteOrder: resolveL16ByteOrder(input.l16ByteOrder, (_b = profile.carrierByteOrder) !== null && _b !== void 0 ? _b : "le"), source: "forced" });
|
|
92
|
+
}
|
|
93
|
+
return Object.assign(Object.assign({}, getPhoneMediaProfile("telnyx-pcmu-8k")), { source: "auto" });
|
|
94
|
+
}
|
|
95
|
+
function phoneMediaProfileMetadata(profile) {
|
|
96
|
+
return Object.assign(Object.assign({ id: profile.id, provider: profile.provider, codec: profile.carrierCodec, sampleRate: profile.carrierSampleRate }, (profile.carrierByteOrder ? { byteOrder: profile.carrierByteOrder } : {})), { openAIFormat: profile.openAIFormat, openAISampleRate: profile.openAISampleRate, source: profile.source });
|
|
97
|
+
}
|
package/dist/routes.d.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { type PhoneMediaFallbackState, type PhoneMediaL16ByteOrder, type PhoneMediaProfile, type PhoneMediaProvider } from "./profiles";
|
|
2
|
+
export type PhoneAiProvider = "openai-realtime" | "xai-realtime" | "gemini-live";
|
|
3
|
+
export type PhoneAudioCodec = "pcmu" | "pcm";
|
|
4
|
+
export type PhoneAudioFormat = "audio/pcmu" | "audio/pcm";
|
|
5
|
+
export interface PhoneAiAudioLeg {
|
|
6
|
+
codec: PhoneAudioCodec;
|
|
7
|
+
format: PhoneAudioFormat;
|
|
8
|
+
sampleRate: 8000 | 16000 | 24000;
|
|
9
|
+
mimeType: string;
|
|
10
|
+
}
|
|
11
|
+
export interface PhoneAudioRoute {
|
|
12
|
+
id: string;
|
|
13
|
+
carrierProvider: PhoneMediaProvider;
|
|
14
|
+
aiProvider: PhoneAiProvider;
|
|
15
|
+
carrier: PhoneMediaProfile;
|
|
16
|
+
aiInput: PhoneAiAudioLeg;
|
|
17
|
+
aiOutput: PhoneAiAudioLeg;
|
|
18
|
+
inputTranscoding: "none" | "decode-resample";
|
|
19
|
+
outputTranscoding: "none" | "resample-encode";
|
|
20
|
+
stable: boolean;
|
|
21
|
+
}
|
|
22
|
+
export interface ResolvePhoneAudioRouteInput {
|
|
23
|
+
carrierProvider: PhoneMediaProvider;
|
|
24
|
+
aiProvider: PhoneAiProvider | string | null | undefined;
|
|
25
|
+
requestedCodec?: string | null;
|
|
26
|
+
l16ByteOrder?: PhoneMediaL16ByteOrder | string | null;
|
|
27
|
+
fallbackState?: PhoneMediaFallbackState | null;
|
|
28
|
+
}
|
|
29
|
+
export declare function createPhoneAudioRoute(carrier: PhoneMediaProfile, aiProviderInput: PhoneAiProvider | string | null | undefined): PhoneAudioRoute;
|
|
30
|
+
export declare function resolvePhoneAudioRoute(input: ResolvePhoneAudioRouteInput): PhoneAudioRoute;
|
|
31
|
+
export declare function phoneAudioRouteMetadata(route: PhoneAudioRoute): {
|
|
32
|
+
id: string;
|
|
33
|
+
carrierProvider: PhoneMediaProvider;
|
|
34
|
+
aiProvider: PhoneAiProvider;
|
|
35
|
+
carrier: {
|
|
36
|
+
openAIFormat: "audio/pcmu" | "audio/pcm";
|
|
37
|
+
openAISampleRate: 8000 | 24000;
|
|
38
|
+
source: "auto" | "forced" | "fallback";
|
|
39
|
+
byteOrder?: PhoneMediaL16ByteOrder | undefined;
|
|
40
|
+
id: import("./profiles").PhoneMediaProfileId;
|
|
41
|
+
provider: PhoneMediaProvider;
|
|
42
|
+
codec: "PCMU" | "L16";
|
|
43
|
+
sampleRate: 8000 | 16000;
|
|
44
|
+
};
|
|
45
|
+
aiInput: PhoneAiAudioLeg;
|
|
46
|
+
aiOutput: PhoneAiAudioLeg;
|
|
47
|
+
inputTranscoding: "none" | "decode-resample";
|
|
48
|
+
outputTranscoding: "none" | "resample-encode";
|
|
49
|
+
stable: boolean;
|
|
50
|
+
};
|
|
51
|
+
export declare function getStableCarrierProfile(provider: PhoneMediaProvider): PhoneMediaProfile;
|