@charivo/realtime 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +148 -0
- package/dist/chunk-BNN3F7VN.mjs +433 -0
- package/dist/chunk-BNN3F7VN.mjs.map +1 -0
- package/dist/chunk-DTNZCAFN.mjs +522 -0
- package/dist/chunk-DTNZCAFN.mjs.map +1 -0
- package/dist/chunk-XJNT4ZTF.mjs +41 -0
- package/dist/chunk-XJNT4ZTF.mjs.map +1 -0
- package/dist/index.d.mts +88 -0
- package/dist/index.d.ts +88 -0
- package/dist/index.js +874 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +864 -0
- package/dist/index.mjs.map +1 -0
- package/dist/openai/index.d.mts +54 -0
- package/dist/openai/index.d.ts +54 -0
- package/dist/openai/index.js +476 -0
- package/dist/openai/index.js.map +1 -0
- package/dist/openai/index.mjs +4 -0
- package/dist/openai/index.mjs.map +1 -0
- package/dist/openai-agents/index.d.mts +49 -0
- package/dist/openai-agents/index.d.ts +49 -0
- package/dist/openai-agents/index.js +564 -0
- package/dist/openai-agents/index.js.map +1 -0
- package/dist/openai-agents/index.mjs +4 -0
- package/dist/openai-agents/index.mjs.map +1 -0
- package/dist/remote/index.d.mts +45 -0
- package/dist/remote/index.d.ts +45 -0
- package/dist/remote/index.js +1115 -0
- package/dist/remote/index.js.map +1 -0
- package/dist/remote/index.mjs +131 -0
- package/dist/remote/index.mjs.map +1 -0
- package/dist/types-DntfJo0o.d.mts +54 -0
- package/dist/types-DntfJo0o.d.ts +54 -0
- package/package.json +64 -0
|
@@ -0,0 +1,1115 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var core = require('@charivo/core');
|
|
4
|
+
var agentsRealtime = require('@openai/agents-realtime');
|
|
5
|
+
|
|
6
|
+
// src/remote/index.ts
|
|
7
|
+
|
|
8
|
+
// src/internal/shared.ts
|
|
9
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
10
|
+
function isRecord(value) {
|
|
11
|
+
return typeof value === "object" && value !== null;
|
|
12
|
+
}
|
|
13
|
+
function isRealtimeSessionBootstrap(value) {
|
|
14
|
+
if (!isRecord(value) || typeof value.adapter !== "string" || typeof value.transport !== "string") {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
if (value.transport === "webrtc") {
|
|
18
|
+
return typeof value.answerSdp === "string" || typeof value.clientSecret === "string";
|
|
19
|
+
}
|
|
20
|
+
if (value.transport === "websocket") {
|
|
21
|
+
return typeof value.url === "string" && typeof value.token === "string";
|
|
22
|
+
}
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
|
|
26
|
+
const controller = new AbortController();
|
|
27
|
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
28
|
+
try {
|
|
29
|
+
return await fetch(input, {
|
|
30
|
+
...init,
|
|
31
|
+
signal: controller.signal
|
|
32
|
+
});
|
|
33
|
+
} catch (error) {
|
|
34
|
+
if (isAbortError(error)) {
|
|
35
|
+
throw new Error(timeoutMessage);
|
|
36
|
+
}
|
|
37
|
+
throw error;
|
|
38
|
+
} finally {
|
|
39
|
+
clearTimeout(timeoutId);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function isAbortError(error) {
|
|
43
|
+
return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// src/openai-agents/bootstrap.ts
|
|
47
|
+
async function getOpenAIRealtimeAgentsBootstrap(options, request) {
|
|
48
|
+
if (options.sessionBootstrap) {
|
|
49
|
+
return options.sessionBootstrap(request);
|
|
50
|
+
}
|
|
51
|
+
const apiEndpoint = options.apiEndpoint;
|
|
52
|
+
if (!apiEndpoint) {
|
|
53
|
+
throw new Error(
|
|
54
|
+
"OpenAI agents realtime client requires apiEndpoint or sessionBootstrap"
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
const response = await fetchWithTimeout(
|
|
58
|
+
apiEndpoint,
|
|
59
|
+
{
|
|
60
|
+
method: "POST",
|
|
61
|
+
headers: {
|
|
62
|
+
"Content-Type": "application/json"
|
|
63
|
+
},
|
|
64
|
+
body: JSON.stringify(request)
|
|
65
|
+
},
|
|
66
|
+
`Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
|
|
67
|
+
);
|
|
68
|
+
if (!response.ok) {
|
|
69
|
+
const errorText = await response.text();
|
|
70
|
+
throw new Error(`Failed to create Realtime session: ${errorText}`);
|
|
71
|
+
}
|
|
72
|
+
const bootstrap = await response.json();
|
|
73
|
+
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
74
|
+
throw new Error("Invalid realtime session bootstrap response");
|
|
75
|
+
}
|
|
76
|
+
return bootstrap;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// src/openai-agents/lip-sync-analyzer.ts
|
|
80
|
+
var LipSyncAnalyzer = class {
|
|
81
|
+
constructor(options) {
|
|
82
|
+
this.options = options;
|
|
83
|
+
}
|
|
84
|
+
audioAnalysisStream = null;
|
|
85
|
+
audioContext = null;
|
|
86
|
+
analyser = null;
|
|
87
|
+
lipSyncInterval = null;
|
|
88
|
+
pendingAudioElementPoll = null;
|
|
89
|
+
observedAudioElement = null;
|
|
90
|
+
observedAudioElementListener = null;
|
|
91
|
+
observeAudioElement(audioElement) {
|
|
92
|
+
this.detachObservedAudioElement();
|
|
93
|
+
const tryAttachStream = () => {
|
|
94
|
+
const stream = audioElement.srcObject;
|
|
95
|
+
if (stream instanceof MediaStream) {
|
|
96
|
+
this.attachStream(stream);
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
this.observedAudioElement = audioElement;
|
|
100
|
+
this.observedAudioElementListener = tryAttachStream;
|
|
101
|
+
audioElement.addEventListener("loadedmetadata", tryAttachStream);
|
|
102
|
+
this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
|
|
103
|
+
}
|
|
104
|
+
attachStream(stream) {
|
|
105
|
+
if (this.audioAnalysisStream === stream) {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
this.stopAudioElementPolling();
|
|
109
|
+
this.stopLipSyncAnalysis();
|
|
110
|
+
this.audioAnalysisStream = stream;
|
|
111
|
+
try {
|
|
112
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
113
|
+
if (!audioContextConstructor) {
|
|
114
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
115
|
+
}
|
|
116
|
+
this.audioContext = new audioContextConstructor();
|
|
117
|
+
const source = this.audioContext.createMediaStreamSource(stream);
|
|
118
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
119
|
+
this.analyser.fftSize = 256;
|
|
120
|
+
this.analyser.smoothingTimeConstant = 0.8;
|
|
121
|
+
source.connect(this.analyser);
|
|
122
|
+
this.startLipSyncAnalysis();
|
|
123
|
+
} catch (error) {
|
|
124
|
+
this.options.onError?.(error);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
stopOutput() {
|
|
128
|
+
this.stopLipSyncAnalysis();
|
|
129
|
+
}
|
|
130
|
+
cleanup() {
|
|
131
|
+
this.stopLipSyncAnalysis();
|
|
132
|
+
this.stopAudioElementPolling();
|
|
133
|
+
this.detachObservedAudioElement();
|
|
134
|
+
if (this.audioContext) {
|
|
135
|
+
void this.audioContext.close();
|
|
136
|
+
this.audioContext = null;
|
|
137
|
+
}
|
|
138
|
+
this.audioAnalysisStream = null;
|
|
139
|
+
this.analyser = null;
|
|
140
|
+
}
|
|
141
|
+
startLipSyncAnalysis() {
|
|
142
|
+
if (!this.analyser) {
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
const bufferLength = this.analyser.frequencyBinCount;
|
|
146
|
+
const dataArray = new Uint8Array(bufferLength);
|
|
147
|
+
this.lipSyncInterval = window.setInterval(() => {
|
|
148
|
+
if (!this.analyser) {
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
this.analyser.getByteFrequencyData(dataArray);
|
|
152
|
+
let sum = 0;
|
|
153
|
+
for (let index = 0; index < bufferLength; index += 1) {
|
|
154
|
+
const normalized = dataArray[index] / 255;
|
|
155
|
+
sum += normalized * normalized;
|
|
156
|
+
}
|
|
157
|
+
const rms = Math.sqrt(sum / bufferLength);
|
|
158
|
+
this.options.onRms(Math.min(rms * 3, 1));
|
|
159
|
+
}, 1e3 / 60);
|
|
160
|
+
}
|
|
161
|
+
stopLipSyncAnalysis() {
|
|
162
|
+
if (this.lipSyncInterval) {
|
|
163
|
+
clearInterval(this.lipSyncInterval);
|
|
164
|
+
this.lipSyncInterval = null;
|
|
165
|
+
}
|
|
166
|
+
this.options.onRms(0);
|
|
167
|
+
}
|
|
168
|
+
stopAudioElementPolling() {
|
|
169
|
+
if (this.pendingAudioElementPoll) {
|
|
170
|
+
clearInterval(this.pendingAudioElementPoll);
|
|
171
|
+
this.pendingAudioElementPoll = null;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
detachObservedAudioElement() {
|
|
175
|
+
if (this.observedAudioElement && this.observedAudioElementListener) {
|
|
176
|
+
this.observedAudioElement.removeEventListener(
|
|
177
|
+
"loadedmetadata",
|
|
178
|
+
this.observedAudioElementListener
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
this.observedAudioElement = null;
|
|
182
|
+
this.observedAudioElementListener = null;
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
// src/openai-agents/session-config.ts
|
|
187
|
+
var DEFAULT_AGENT_INSTRUCTIONS = "You are a realtime voice agent controlling a Live2D character.";
|
|
188
|
+
var DEFAULT_MODEL = "gpt-realtime-mini";
|
|
189
|
+
var DEFAULT_VOICE = "marin";
|
|
190
|
+
function resolveInstructions(config) {
|
|
191
|
+
return config?.instructions ?? DEFAULT_AGENT_INSTRUCTIONS;
|
|
192
|
+
}
|
|
193
|
+
function resolveVoice(config) {
|
|
194
|
+
return config?.voice ?? DEFAULT_VOICE;
|
|
195
|
+
}
|
|
196
|
+
function toOpenAIRealtimeAgentsSessionConfig(config) {
|
|
197
|
+
return {
|
|
198
|
+
model: config?.model ?? DEFAULT_MODEL,
|
|
199
|
+
instructions: resolveInstructions(config),
|
|
200
|
+
toolChoice: config?.toolChoice ?? "auto",
|
|
201
|
+
outputModalities: ["audio"],
|
|
202
|
+
audio: {
|
|
203
|
+
output: {
|
|
204
|
+
voice: resolveVoice(config)
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// src/openai-agents/tool-schema.ts
|
|
211
|
+
function createToolSchemaOptions(parameters) {
|
|
212
|
+
const normalizedParameters = toAgentsToolParameters(parameters);
|
|
213
|
+
if (normalizedParameters.additionalProperties === false) {
|
|
214
|
+
return {
|
|
215
|
+
parameters: normalizedParameters,
|
|
216
|
+
strict: true
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
parameters: normalizedParameters,
|
|
221
|
+
strict: false
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
function toAgentsToolParameters(parameters) {
|
|
225
|
+
const baseParameters = {
|
|
226
|
+
type: "object",
|
|
227
|
+
properties: parameters.properties,
|
|
228
|
+
required: parameters.required ?? []
|
|
229
|
+
};
|
|
230
|
+
if (getAdditionalProperties(parameters) === false) {
|
|
231
|
+
return {
|
|
232
|
+
...baseParameters,
|
|
233
|
+
additionalProperties: false
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
return {
|
|
237
|
+
...baseParameters,
|
|
238
|
+
additionalProperties: true
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
function getAdditionalProperties(parameters) {
|
|
242
|
+
const value = parameters.additionalProperties;
|
|
243
|
+
return typeof value === "boolean" ? value : void 0;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// src/openai-agents/client.ts
|
|
247
|
+
var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
248
|
+
"conversation.item.input_audio_transcription.completed",
|
|
249
|
+
"error"
|
|
250
|
+
]);
|
|
251
|
+
var TOOL_RESULT_TIMEOUT_MESSAGE = "Realtime session ended before tool result was returned";
|
|
252
|
+
var OpenAIRealtimeAgentsClient = class {
|
|
253
|
+
constructor(options = {}) {
|
|
254
|
+
this.options = options;
|
|
255
|
+
}
|
|
256
|
+
session = null;
|
|
257
|
+
transport = null;
|
|
258
|
+
audioElement = null;
|
|
259
|
+
connectionWasActive = false;
|
|
260
|
+
assistant = { text: "", started: false };
|
|
261
|
+
latestHistory = [];
|
|
262
|
+
eventCallbacks = /* @__PURE__ */ new Set();
|
|
263
|
+
pendingToolCalls = /* @__PURE__ */ new Map();
|
|
264
|
+
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
265
|
+
onRms: (rms) => {
|
|
266
|
+
this.emitEvent({ type: "audio.lipsync", rms });
|
|
267
|
+
},
|
|
268
|
+
onError: (error) => {
|
|
269
|
+
console.error("Failed to setup audio analysis:", error);
|
|
270
|
+
}
|
|
271
|
+
});
|
|
272
|
+
async connect(config) {
|
|
273
|
+
try {
|
|
274
|
+
this.log("Starting OpenAI Agents Realtime WebRTC connection");
|
|
275
|
+
this.audioElement = document.createElement("audio");
|
|
276
|
+
this.audioElement.autoplay = true;
|
|
277
|
+
this.audioElement.setAttribute("playsinline", "true");
|
|
278
|
+
const agent = new agentsRealtime.RealtimeAgent({
|
|
279
|
+
name: "charivo-realtime-agent",
|
|
280
|
+
instructions: resolveInstructions(config),
|
|
281
|
+
tools: this.createProxyTools(config?.tools),
|
|
282
|
+
voice: resolveVoice(config)
|
|
283
|
+
});
|
|
284
|
+
this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
|
|
285
|
+
this.createTransportOptions(this.audioElement)
|
|
286
|
+
);
|
|
287
|
+
this.session = new agentsRealtime.RealtimeSession(agent, {
|
|
288
|
+
transport: this.transport,
|
|
289
|
+
config: toOpenAIRealtimeAgentsSessionConfig(config)
|
|
290
|
+
});
|
|
291
|
+
this.bindSessionEvents(this.session);
|
|
292
|
+
this.bindTransportEvents(this.transport);
|
|
293
|
+
this.lipSyncAnalyzer.observeAudioElement(this.audioElement);
|
|
294
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
295
|
+
adapter: core.OPENAI_REALTIME_AGENTS_ADAPTER,
|
|
296
|
+
transport: "webrtc",
|
|
297
|
+
session: config ?? {}
|
|
298
|
+
});
|
|
299
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_AGENTS_ADAPTER || bootstrap.transport !== "webrtc" || !("clientSecret" in bootstrap) || typeof bootstrap.clientSecret !== "string") {
|
|
300
|
+
throw new Error(
|
|
301
|
+
`OpenAI agents realtime client only supports ${core.OPENAI_REALTIME_AGENTS_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
await this.session.connect({
|
|
305
|
+
apiKey: bootstrap.clientSecret,
|
|
306
|
+
model: config?.model
|
|
307
|
+
});
|
|
308
|
+
this.connectionWasActive = true;
|
|
309
|
+
this.emitEvent({ type: "session.started" });
|
|
310
|
+
} catch (error) {
|
|
311
|
+
this.cleanup(error);
|
|
312
|
+
throw error;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
async disconnect() {
|
|
316
|
+
this.log("Disconnecting OpenAI Agents Realtime WebRTC");
|
|
317
|
+
this.cleanup();
|
|
318
|
+
}
|
|
319
|
+
async sendText(text) {
|
|
320
|
+
if (!this.session) {
|
|
321
|
+
throw new Error("Realtime session not active");
|
|
322
|
+
}
|
|
323
|
+
this.resetAssistantTracking();
|
|
324
|
+
this.session.sendMessage(text);
|
|
325
|
+
}
|
|
326
|
+
async sendAudio(_audio) {
|
|
327
|
+
console.warn(
|
|
328
|
+
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
async sendToolResult(callId, output) {
|
|
332
|
+
const pendingCall = this.pendingToolCalls.get(callId);
|
|
333
|
+
if (!pendingCall) {
|
|
334
|
+
throw new Error(`No pending realtime tool call for "${callId}"`);
|
|
335
|
+
}
|
|
336
|
+
this.pendingToolCalls.delete(callId);
|
|
337
|
+
pendingCall.resolve(output);
|
|
338
|
+
}
|
|
339
|
+
async interrupt() {
|
|
340
|
+
if (!this.session) {
|
|
341
|
+
throw new Error("Realtime session not active");
|
|
342
|
+
}
|
|
343
|
+
this.session.interrupt();
|
|
344
|
+
this.assistant.started = false;
|
|
345
|
+
}
|
|
346
|
+
onEvent(callback) {
|
|
347
|
+
this.eventCallbacks.add(callback);
|
|
348
|
+
}
|
|
349
|
+
bindSessionEvents(session) {
|
|
350
|
+
session.on("agent_start", () => {
|
|
351
|
+
this.ensureAssistantStarted();
|
|
352
|
+
});
|
|
353
|
+
session.on("agent_end", (_context, _agent, output) => {
|
|
354
|
+
this.finalizeAssistantResponse(output);
|
|
355
|
+
});
|
|
356
|
+
session.on("audio_start", () => {
|
|
357
|
+
this.emitEvent({ type: "audio.output.started" });
|
|
358
|
+
});
|
|
359
|
+
session.on("audio_stopped", () => {
|
|
360
|
+
this.emitEvent({ type: "audio.output.ended" });
|
|
361
|
+
});
|
|
362
|
+
session.on("history_updated", (history) => {
|
|
363
|
+
this.latestHistory = history;
|
|
364
|
+
});
|
|
365
|
+
session.on("error", ({ error }) => {
|
|
366
|
+
this.emitEvent({
|
|
367
|
+
type: "error",
|
|
368
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
369
|
+
});
|
|
370
|
+
});
|
|
371
|
+
session.on("transport_event", (event) => {
|
|
372
|
+
if (this.options.debug && DEBUG_EVENT_ALLOWLIST.has(event.type)) {
|
|
373
|
+
this.log("\u{1F4E1} [OpenAI Agents Transport Event]", event.type, event);
|
|
374
|
+
}
|
|
375
|
+
if (event.type === "conversation.item.input_audio_transcription.completed" && typeof event.transcript === "string") {
|
|
376
|
+
this.emitEvent({
|
|
377
|
+
type: "user.transcript",
|
|
378
|
+
text: event.transcript
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
bindTransportEvents(transport) {
|
|
384
|
+
transport.on("audio_transcript_delta", (event) => {
|
|
385
|
+
this.handleAssistantTranscriptDelta(event);
|
|
386
|
+
});
|
|
387
|
+
transport.on("audio_interrupted", () => {
|
|
388
|
+
this.emitEvent({ type: "audio.output.ended" });
|
|
389
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
390
|
+
});
|
|
391
|
+
transport.on("connection_change", (status) => {
|
|
392
|
+
if (status === "disconnected" && this.connectionWasActive) {
|
|
393
|
+
this.cleanupPendingToolCalls();
|
|
394
|
+
this.connectionWasActive = false;
|
|
395
|
+
this.emitEvent({ type: "session.ended" });
|
|
396
|
+
}
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
handleAssistantTranscriptDelta(event) {
|
|
400
|
+
if (!event.delta) {
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
this.ensureAssistantStarted();
|
|
404
|
+
this.assistant.text += event.delta;
|
|
405
|
+
this.emitEvent({
|
|
406
|
+
type: "assistant.text.delta",
|
|
407
|
+
text: event.delta
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
ensureAssistantStarted() {
|
|
411
|
+
if (this.assistant.started) {
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
this.assistant.started = true;
|
|
415
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
416
|
+
}
|
|
417
|
+
finalizeAssistantResponse(output) {
|
|
418
|
+
const finalText = this.getLatestAssistantText() || output || this.assistant.text;
|
|
419
|
+
this.ensureAssistantStarted();
|
|
420
|
+
if (finalText && finalText.startsWith(this.assistant.text) && finalText !== this.assistant.text) {
|
|
421
|
+
const delta = finalText.slice(this.assistant.text.length);
|
|
422
|
+
if (delta) {
|
|
423
|
+
this.emitEvent({
|
|
424
|
+
type: "assistant.text.delta",
|
|
425
|
+
text: delta
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
this.emitEvent({
|
|
430
|
+
type: "assistant.response.completed",
|
|
431
|
+
text: finalText
|
|
432
|
+
});
|
|
433
|
+
this.resetAssistantTracking();
|
|
434
|
+
}
|
|
435
|
+
getLatestAssistantText() {
|
|
436
|
+
for (let index = this.latestHistory.length - 1; index >= 0; index -= 1) {
|
|
437
|
+
const item = this.latestHistory[index];
|
|
438
|
+
if (item.type === "message" && item.role === "assistant" && Array.isArray(item.content)) {
|
|
439
|
+
const text = item.content.map((content) => {
|
|
440
|
+
if (content.type === "output_audio") {
|
|
441
|
+
return content.transcript ?? "";
|
|
442
|
+
}
|
|
443
|
+
if (content.type === "output_text") {
|
|
444
|
+
return content.text;
|
|
445
|
+
}
|
|
446
|
+
return "";
|
|
447
|
+
}).join("").trim();
|
|
448
|
+
if (text) {
|
|
449
|
+
return text;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return "";
|
|
454
|
+
}
|
|
455
|
+
createProxyTools(tools) {
|
|
456
|
+
return (tools ?? []).map((definition) => {
|
|
457
|
+
const execute = async (input, _context, details) => {
|
|
458
|
+
const toolCallItem = details?.toolCall;
|
|
459
|
+
const callId = toolCallItem?.callId ?? this.createToolCallId(definition.name);
|
|
460
|
+
return await new Promise((resolve, reject) => {
|
|
461
|
+
this.pendingToolCalls.set(callId, {
|
|
462
|
+
resolve,
|
|
463
|
+
reject
|
|
464
|
+
});
|
|
465
|
+
this.emitEvent({
|
|
466
|
+
type: "tool.call",
|
|
467
|
+
name: definition.name,
|
|
468
|
+
args: isRecord(input) ? input : {},
|
|
469
|
+
callId
|
|
470
|
+
});
|
|
471
|
+
});
|
|
472
|
+
};
|
|
473
|
+
const schemaOptions = createToolSchemaOptions(definition.parameters);
|
|
474
|
+
if (schemaOptions.strict) {
|
|
475
|
+
return agentsRealtime.tool({
|
|
476
|
+
name: definition.name,
|
|
477
|
+
description: definition.description,
|
|
478
|
+
parameters: schemaOptions.parameters,
|
|
479
|
+
strict: true,
|
|
480
|
+
needsApproval: false,
|
|
481
|
+
execute
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
return agentsRealtime.tool({
|
|
485
|
+
name: definition.name,
|
|
486
|
+
description: definition.description,
|
|
487
|
+
parameters: schemaOptions.parameters,
|
|
488
|
+
strict: false,
|
|
489
|
+
needsApproval: false,
|
|
490
|
+
execute
|
|
491
|
+
});
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
createTransportOptions(audioElement) {
|
|
495
|
+
return {
|
|
496
|
+
audioElement,
|
|
497
|
+
changePeerConnection: async (peerConnection) => {
|
|
498
|
+
peerConnection.addEventListener("track", (event) => {
|
|
499
|
+
const stream = event.streams[0];
|
|
500
|
+
if (stream) {
|
|
501
|
+
this.lipSyncAnalyzer.attachStream(stream);
|
|
502
|
+
}
|
|
503
|
+
});
|
|
504
|
+
return peerConnection;
|
|
505
|
+
}
|
|
506
|
+
};
|
|
507
|
+
}
|
|
508
|
+
cleanup(error) {
|
|
509
|
+
this.cleanupPendingToolCalls(error);
|
|
510
|
+
this.lipSyncAnalyzer.cleanup();
|
|
511
|
+
if (this.transport) {
|
|
512
|
+
this.transport.close();
|
|
513
|
+
this.transport = null;
|
|
514
|
+
}
|
|
515
|
+
if (this.session) {
|
|
516
|
+
this.session.close();
|
|
517
|
+
this.session = null;
|
|
518
|
+
}
|
|
519
|
+
if (this.audioElement) {
|
|
520
|
+
this.audioElement.srcObject = null;
|
|
521
|
+
this.audioElement = null;
|
|
522
|
+
}
|
|
523
|
+
this.latestHistory = [];
|
|
524
|
+
this.resetAssistantTracking();
|
|
525
|
+
}
|
|
526
|
+
resetAssistantTracking() {
|
|
527
|
+
this.assistant = {
|
|
528
|
+
text: "",
|
|
529
|
+
started: false
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
cleanupPendingToolCalls(error) {
|
|
533
|
+
const toolError = error instanceof Error ? error : new Error(TOOL_RESULT_TIMEOUT_MESSAGE);
|
|
534
|
+
for (const pendingCall of this.pendingToolCalls.values()) {
|
|
535
|
+
pendingCall.reject(toolError);
|
|
536
|
+
}
|
|
537
|
+
this.pendingToolCalls.clear();
|
|
538
|
+
}
|
|
539
|
+
createToolCallId(toolName) {
|
|
540
|
+
return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${toolName}-${Date.now()}`;
|
|
541
|
+
}
|
|
542
|
+
async getSessionBootstrap(request) {
|
|
543
|
+
return getOpenAIRealtimeAgentsBootstrap(this.options, request);
|
|
544
|
+
}
|
|
545
|
+
emitEvent(event) {
|
|
546
|
+
for (const callback of this.eventCallbacks) {
|
|
547
|
+
callback(event);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
log(...args) {
|
|
551
|
+
if (!this.options.debug) {
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
console.debug("[charivo/realtime/openai-agents]", ...args);
|
|
555
|
+
}
|
|
556
|
+
};
|
|
557
|
+
function createOpenAIRealtimeAgentsClient(options) {
|
|
558
|
+
return new OpenAIRealtimeAgentsClient(options);
|
|
559
|
+
}
|
|
560
|
+
var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
|
|
561
|
+
"session.created",
|
|
562
|
+
"session.updated",
|
|
563
|
+
"response.created",
|
|
564
|
+
"response.done",
|
|
565
|
+
"response.function_call_arguments.done",
|
|
566
|
+
"conversation.item.input_audio_transcription.completed",
|
|
567
|
+
"error"
|
|
568
|
+
]);
|
|
569
|
+
var OpenAIRealtimeClient = class {
|
|
570
|
+
constructor(options = {}) {
|
|
571
|
+
this.options = options;
|
|
572
|
+
}
|
|
573
|
+
pc = null;
|
|
574
|
+
dc = null;
|
|
575
|
+
audioElement = null;
|
|
576
|
+
mediaStream = null;
|
|
577
|
+
audioContext = null;
|
|
578
|
+
analyser = null;
|
|
579
|
+
lipSyncInterval = null;
|
|
580
|
+
isResponseInProgress = false;
|
|
581
|
+
cancelInFlight = false;
|
|
582
|
+
hasStartedAssistantResponse = false;
|
|
583
|
+
hasStartedAudioOutput = false;
|
|
584
|
+
assistantText = "";
|
|
585
|
+
eventCallbacks = /* @__PURE__ */ new Set();
|
|
586
|
+
async connect(config) {
|
|
587
|
+
try {
|
|
588
|
+
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
589
|
+
this.pc = new RTCPeerConnection();
|
|
590
|
+
this.audioElement = document.createElement("audio");
|
|
591
|
+
this.audioElement.autoplay = true;
|
|
592
|
+
this.pc.ontrack = (event) => {
|
|
593
|
+
if (this.audioElement) {
|
|
594
|
+
this.audioElement.srcObject = event.streams[0];
|
|
595
|
+
}
|
|
596
|
+
this.setupAudioAnalysis(event.streams[0]);
|
|
597
|
+
};
|
|
598
|
+
try {
|
|
599
|
+
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
600
|
+
audio: true
|
|
601
|
+
});
|
|
602
|
+
const audioTrack = this.mediaStream.getTracks()[0];
|
|
603
|
+
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
604
|
+
} catch {
|
|
605
|
+
throw new Error("Microphone access required for Realtime API");
|
|
606
|
+
}
|
|
607
|
+
this.dc = this.pc.createDataChannel("oai-events");
|
|
608
|
+
this.dc.onmessage = (event) => {
|
|
609
|
+
const payload = JSON.parse(event.data);
|
|
610
|
+
this.handleServerEvent(payload);
|
|
611
|
+
};
|
|
612
|
+
this.dc.onerror = () => {
|
|
613
|
+
this.emitEvent({
|
|
614
|
+
type: "error",
|
|
615
|
+
error: new Error("DataChannel error")
|
|
616
|
+
});
|
|
617
|
+
};
|
|
618
|
+
const offer = await this.pc.createOffer();
|
|
619
|
+
await this.pc.setLocalDescription(offer);
|
|
620
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
621
|
+
transport: "webrtc",
|
|
622
|
+
session: config ?? {},
|
|
623
|
+
sdpOffer: offer.sdp
|
|
624
|
+
});
|
|
625
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
626
|
+
throw new Error(
|
|
627
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
await this.pc.setRemoteDescription({
|
|
631
|
+
type: "answer",
|
|
632
|
+
sdp: bootstrap.answerSdp
|
|
633
|
+
});
|
|
634
|
+
this.emitEvent({ type: "session.started" });
|
|
635
|
+
} catch (error) {
|
|
636
|
+
this.cleanup();
|
|
637
|
+
throw error;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
async disconnect() {
|
|
641
|
+
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
642
|
+
this.cleanup();
|
|
643
|
+
}
|
|
644
|
+
async sendText(text) {
|
|
645
|
+
if (!this.dc || this.dc.readyState !== "open") {
|
|
646
|
+
throw new Error("DataChannel not ready");
|
|
647
|
+
}
|
|
648
|
+
if (this.isResponseInProgress) {
|
|
649
|
+
console.warn("\u26A0\uFE0F Response already in progress, skipping request");
|
|
650
|
+
return;
|
|
651
|
+
}
|
|
652
|
+
this.dc.send(
|
|
653
|
+
JSON.stringify({
|
|
654
|
+
type: "conversation.item.create",
|
|
655
|
+
item: {
|
|
656
|
+
type: "message",
|
|
657
|
+
role: "user",
|
|
658
|
+
content: [
|
|
659
|
+
{
|
|
660
|
+
type: "input_text",
|
|
661
|
+
text
|
|
662
|
+
}
|
|
663
|
+
]
|
|
664
|
+
}
|
|
665
|
+
})
|
|
666
|
+
);
|
|
667
|
+
this.dc.send(JSON.stringify({ type: "response.create" }));
|
|
668
|
+
this.beginResponseRequest();
|
|
669
|
+
}
|
|
670
|
+
async sendAudio(_audio) {
|
|
671
|
+
console.warn(
|
|
672
|
+
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
673
|
+
);
|
|
674
|
+
}
|
|
675
|
+
async sendToolResult(callId, output) {
|
|
676
|
+
const dc = this.requireOpenDataChannel();
|
|
677
|
+
dc.send(
|
|
678
|
+
JSON.stringify({
|
|
679
|
+
type: "conversation.item.create",
|
|
680
|
+
item: {
|
|
681
|
+
type: "function_call_output",
|
|
682
|
+
call_id: callId,
|
|
683
|
+
output: JSON.stringify(output)
|
|
684
|
+
}
|
|
685
|
+
})
|
|
686
|
+
);
|
|
687
|
+
dc.send(JSON.stringify({ type: "response.create" }));
|
|
688
|
+
this.beginResponseRequest();
|
|
689
|
+
}
|
|
690
|
+
async interrupt() {
|
|
691
|
+
const dc = this.requireOpenDataChannel();
|
|
692
|
+
if (!this.isResponseInProgress) {
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
dc.send(JSON.stringify({ type: "response.cancel" }));
|
|
696
|
+
this.isResponseInProgress = false;
|
|
697
|
+
this.cancelInFlight = true;
|
|
698
|
+
}
|
|
699
|
+
onEvent(callback) {
|
|
700
|
+
this.eventCallbacks.add(callback);
|
|
701
|
+
}
|
|
702
|
+
handleServerEvent(event) {
|
|
703
|
+
if (this.options.debug && this.shouldLogDebugEvent(event.type)) {
|
|
704
|
+
this.log("\u{1F4E1} [OpenAI Realtime Event]", event.type, event);
|
|
705
|
+
}
|
|
706
|
+
switch (event.type) {
|
|
707
|
+
case "session.created":
|
|
708
|
+
case "session.updated":
|
|
709
|
+
return;
|
|
710
|
+
case "response.audio.delta":
|
|
711
|
+
case "output_audio_buffer.started":
|
|
712
|
+
if (this.cancelInFlight) {
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
if (!this.hasStartedAudioOutput) {
|
|
716
|
+
this.hasStartedAudioOutput = true;
|
|
717
|
+
this.emitEvent({ type: "audio.output.started" });
|
|
718
|
+
}
|
|
719
|
+
return;
|
|
720
|
+
case "response.audio.done":
|
|
721
|
+
case "response.output_audio.done":
|
|
722
|
+
case "output_audio_buffer.stopped":
|
|
723
|
+
if (this.cancelInFlight) {
|
|
724
|
+
this.hasStartedAudioOutput = false;
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
this.emitEvent({ type: "audio.output.ended" });
|
|
728
|
+
this.hasStartedAudioOutput = false;
|
|
729
|
+
return;
|
|
730
|
+
case "response.done":
|
|
731
|
+
this.isResponseInProgress = false;
|
|
732
|
+
if (this.cancelInFlight) {
|
|
733
|
+
this.cancelInFlight = false;
|
|
734
|
+
this.resetResponseTracking();
|
|
735
|
+
return;
|
|
736
|
+
}
|
|
737
|
+
this.emitEvent({
|
|
738
|
+
type: "assistant.response.completed",
|
|
739
|
+
text: this.assistantText
|
|
740
|
+
});
|
|
741
|
+
this.resetResponseTracking();
|
|
742
|
+
return;
|
|
743
|
+
case "response.audio_transcript.delta":
|
|
744
|
+
case "response.output_audio_transcript.delta":
|
|
745
|
+
case "response.output_text.delta":
|
|
746
|
+
if (this.cancelInFlight || !event.delta) {
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
if (!this.hasStartedAssistantResponse) {
|
|
750
|
+
this.hasStartedAssistantResponse = true;
|
|
751
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
752
|
+
}
|
|
753
|
+
this.assistantText += event.delta;
|
|
754
|
+
this.emitEvent({
|
|
755
|
+
type: "assistant.text.delta",
|
|
756
|
+
text: event.delta
|
|
757
|
+
});
|
|
758
|
+
return;
|
|
759
|
+
case "response.output_text.done":
|
|
760
|
+
if (this.cancelInFlight || !event.text) {
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
if (!this.hasStartedAssistantResponse) {
|
|
764
|
+
this.hasStartedAssistantResponse = true;
|
|
765
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
766
|
+
}
|
|
767
|
+
if (event.text !== this.assistantText) {
|
|
768
|
+
const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
|
|
769
|
+
this.assistantText = event.text;
|
|
770
|
+
if (delta) {
|
|
771
|
+
this.emitEvent({
|
|
772
|
+
type: "assistant.text.delta",
|
|
773
|
+
text: delta
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
return;
|
|
778
|
+
case "response.output_audio_transcript.done":
|
|
779
|
+
if (this.cancelInFlight || !event.transcript) {
|
|
780
|
+
return;
|
|
781
|
+
}
|
|
782
|
+
if (!this.hasStartedAssistantResponse) {
|
|
783
|
+
this.hasStartedAssistantResponse = true;
|
|
784
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
785
|
+
}
|
|
786
|
+
if (event.transcript !== this.assistantText) {
|
|
787
|
+
const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
|
|
788
|
+
this.assistantText = event.transcript;
|
|
789
|
+
if (delta) {
|
|
790
|
+
this.emitEvent({
|
|
791
|
+
type: "assistant.text.delta",
|
|
792
|
+
text: delta
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
return;
|
|
797
|
+
case "conversation.item.input_audio_transcription.completed":
|
|
798
|
+
if (event.transcript) {
|
|
799
|
+
this.emitEvent({
|
|
800
|
+
type: "user.transcript",
|
|
801
|
+
text: event.transcript
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
return;
|
|
805
|
+
case "response.function_call_arguments.done":
|
|
806
|
+
this.handleToolCallDone(event);
|
|
807
|
+
return;
|
|
808
|
+
case "error":
|
|
809
|
+
this.isResponseInProgress = false;
|
|
810
|
+
this.cancelInFlight = false;
|
|
811
|
+
this.emitEvent({
|
|
812
|
+
type: "error",
|
|
813
|
+
error: new Error(event.error?.message || "Unknown error")
|
|
814
|
+
});
|
|
815
|
+
return;
|
|
816
|
+
default:
|
|
817
|
+
return;
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
handleToolCallDone(event) {
|
|
821
|
+
const callId = event.call_id || event.item?.call_id || event.item_id;
|
|
822
|
+
const name = event.name || event.item?.name;
|
|
823
|
+
const argsJson = event.arguments || event.item?.arguments;
|
|
824
|
+
if (!name || !argsJson) {
|
|
825
|
+
console.warn("\u26A0\uFE0F Tool call done but missing name or arguments");
|
|
826
|
+
return;
|
|
827
|
+
}
|
|
828
|
+
let args = {};
|
|
829
|
+
try {
|
|
830
|
+
const parsed = JSON.parse(argsJson);
|
|
831
|
+
if (isRecord(parsed)) {
|
|
832
|
+
args = parsed;
|
|
833
|
+
}
|
|
834
|
+
} catch (error) {
|
|
835
|
+
console.error("Failed to parse tool call args", error, argsJson);
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
this.emitEvent({
|
|
839
|
+
type: "tool.call",
|
|
840
|
+
name,
|
|
841
|
+
args,
|
|
842
|
+
callId
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
async getSessionBootstrap(request) {
|
|
846
|
+
if (this.options.sessionBootstrap) {
|
|
847
|
+
return this.options.sessionBootstrap(request);
|
|
848
|
+
}
|
|
849
|
+
const apiEndpoint = this.options.apiEndpoint;
|
|
850
|
+
if (!apiEndpoint) {
|
|
851
|
+
throw new Error(
|
|
852
|
+
"OpenAI realtime client requires apiEndpoint or sessionBootstrap"
|
|
853
|
+
);
|
|
854
|
+
}
|
|
855
|
+
const response = await fetchWithTimeout(
|
|
856
|
+
apiEndpoint,
|
|
857
|
+
{
|
|
858
|
+
method: "POST",
|
|
859
|
+
headers: {
|
|
860
|
+
"Content-Type": "application/json"
|
|
861
|
+
},
|
|
862
|
+
body: JSON.stringify(request)
|
|
863
|
+
},
|
|
864
|
+
`Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
|
|
865
|
+
);
|
|
866
|
+
if (!response.ok) {
|
|
867
|
+
const errorText = await response.text();
|
|
868
|
+
throw new Error(`Failed to create Realtime session: ${errorText}`);
|
|
869
|
+
}
|
|
870
|
+
const bootstrap = await response.json();
|
|
871
|
+
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
872
|
+
throw new Error("Invalid realtime session bootstrap response");
|
|
873
|
+
}
|
|
874
|
+
return bootstrap;
|
|
875
|
+
}
|
|
876
|
+
setupAudioAnalysis(stream) {
|
|
877
|
+
try {
|
|
878
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
879
|
+
if (!audioContextConstructor) {
|
|
880
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
881
|
+
}
|
|
882
|
+
this.audioContext = new audioContextConstructor();
|
|
883
|
+
const source = this.audioContext.createMediaStreamSource(stream);
|
|
884
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
885
|
+
this.analyser.fftSize = 256;
|
|
886
|
+
this.analyser.smoothingTimeConstant = 0.8;
|
|
887
|
+
source.connect(this.analyser);
|
|
888
|
+
this.startLipSyncAnalysis();
|
|
889
|
+
} catch (error) {
|
|
890
|
+
console.error("Failed to setup audio analysis:", error);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
startLipSyncAnalysis() {
|
|
894
|
+
if (!this.analyser) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
const bufferLength = this.analyser.frequencyBinCount;
|
|
898
|
+
const dataArray = new Uint8Array(bufferLength);
|
|
899
|
+
this.lipSyncInterval = window.setInterval(() => {
|
|
900
|
+
if (!this.analyser) {
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
this.analyser.getByteFrequencyData(dataArray);
|
|
904
|
+
let sum = 0;
|
|
905
|
+
for (let index = 0; index < bufferLength; index += 1) {
|
|
906
|
+
const normalized = dataArray[index] / 255;
|
|
907
|
+
sum += normalized * normalized;
|
|
908
|
+
}
|
|
909
|
+
const rms = Math.sqrt(sum / bufferLength);
|
|
910
|
+
this.emitEvent({
|
|
911
|
+
type: "audio.lipsync",
|
|
912
|
+
rms: Math.min(rms * 3, 1)
|
|
913
|
+
});
|
|
914
|
+
}, 1e3 / 60);
|
|
915
|
+
}
|
|
916
|
+
stopLipSyncAnalysis() {
|
|
917
|
+
if (this.lipSyncInterval) {
|
|
918
|
+
clearInterval(this.lipSyncInterval);
|
|
919
|
+
this.lipSyncInterval = null;
|
|
920
|
+
}
|
|
921
|
+
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
922
|
+
}
|
|
923
|
+
cleanup() {
|
|
924
|
+
this.stopLipSyncAnalysis();
|
|
925
|
+
if (this.dc) {
|
|
926
|
+
this.dc.close();
|
|
927
|
+
this.dc = null;
|
|
928
|
+
}
|
|
929
|
+
if (this.pc) {
|
|
930
|
+
this.pc.close();
|
|
931
|
+
this.pc = null;
|
|
932
|
+
}
|
|
933
|
+
if (this.mediaStream) {
|
|
934
|
+
this.mediaStream.getTracks().forEach((track) => track.stop());
|
|
935
|
+
this.mediaStream = null;
|
|
936
|
+
}
|
|
937
|
+
if (this.audioElement) {
|
|
938
|
+
this.audioElement.srcObject = null;
|
|
939
|
+
this.audioElement = null;
|
|
940
|
+
}
|
|
941
|
+
if (this.audioContext) {
|
|
942
|
+
this.audioContext.close();
|
|
943
|
+
this.audioContext = null;
|
|
944
|
+
}
|
|
945
|
+
this.analyser = null;
|
|
946
|
+
this.isResponseInProgress = false;
|
|
947
|
+
this.cancelInFlight = false;
|
|
948
|
+
this.resetResponseTracking();
|
|
949
|
+
}
|
|
950
|
+
beginResponseRequest() {
|
|
951
|
+
this.resetResponseTracking();
|
|
952
|
+
this.isResponseInProgress = true;
|
|
953
|
+
this.cancelInFlight = false;
|
|
954
|
+
}
|
|
955
|
+
requireOpenDataChannel() {
|
|
956
|
+
if (!this.dc || this.dc.readyState !== "open") {
|
|
957
|
+
throw new Error("DataChannel not ready");
|
|
958
|
+
}
|
|
959
|
+
return this.dc;
|
|
960
|
+
}
|
|
961
|
+
resetResponseTracking() {
|
|
962
|
+
this.assistantText = "";
|
|
963
|
+
this.hasStartedAssistantResponse = false;
|
|
964
|
+
this.hasStartedAudioOutput = false;
|
|
965
|
+
}
|
|
966
|
+
emitEvent(event) {
|
|
967
|
+
for (const callback of this.eventCallbacks) {
|
|
968
|
+
callback(event);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
log(...args) {
|
|
972
|
+
if (this.options.debug) {
|
|
973
|
+
console.log(...args);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
shouldLogDebugEvent(eventType) {
|
|
977
|
+
if (DEBUG_EVENT_ALLOWLIST2.has(eventType)) {
|
|
978
|
+
return true;
|
|
979
|
+
}
|
|
980
|
+
return eventType.startsWith("response.output_text.");
|
|
981
|
+
}
|
|
982
|
+
};
|
|
983
|
+
function createOpenAIRealtimeClient(options) {
|
|
984
|
+
return new OpenAIRealtimeClient(options);
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
// src/remote/index.ts
|
|
988
|
+
var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
|
|
989
|
+
[core.OPENAI_REALTIME_AGENTS_ADAPTER]: (options) => createOpenAIRealtimeAgentsClient({
|
|
990
|
+
debug: options.debug,
|
|
991
|
+
sessionBootstrap: options.requestBootstrap
|
|
992
|
+
}),
|
|
993
|
+
[core.OPENAI_REALTIME_ADAPTER]: (options) => createOpenAIRealtimeClient({
|
|
994
|
+
debug: options.debug,
|
|
995
|
+
sessionBootstrap: options.requestBootstrap
|
|
996
|
+
})
|
|
997
|
+
};
|
|
998
|
+
var RemoteRealtimeClient = class {
|
|
999
|
+
constructor(config = {}) {
|
|
1000
|
+
this.config = config;
|
|
1001
|
+
this.adapters = new Map(
|
|
1002
|
+
Object.entries({
|
|
1003
|
+
...DEFAULT_REMOTE_REALTIME_ADAPTERS,
|
|
1004
|
+
...config.adapters
|
|
1005
|
+
})
|
|
1006
|
+
);
|
|
1007
|
+
}
|
|
1008
|
+
transportClient = null;
|
|
1009
|
+
adapters;
|
|
1010
|
+
eventCallbacks = /* @__PURE__ */ new Set();
|
|
1011
|
+
async connect(config) {
|
|
1012
|
+
const adapterId = this.resolveAdapterId(config);
|
|
1013
|
+
const factory = this.adapters.get(adapterId);
|
|
1014
|
+
if (!factory) {
|
|
1015
|
+
throw new Error(
|
|
1016
|
+
`No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
|
|
1017
|
+
);
|
|
1018
|
+
}
|
|
1019
|
+
const transportClient = factory({
|
|
1020
|
+
debug: this.config.debug,
|
|
1021
|
+
requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
|
|
1022
|
+
});
|
|
1023
|
+
for (const callback of this.eventCallbacks) {
|
|
1024
|
+
transportClient.onEvent(callback);
|
|
1025
|
+
}
|
|
1026
|
+
this.transportClient = transportClient;
|
|
1027
|
+
try {
|
|
1028
|
+
await transportClient.connect(config);
|
|
1029
|
+
} catch (error) {
|
|
1030
|
+
this.transportClient = null;
|
|
1031
|
+
throw error;
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
async disconnect() {
|
|
1035
|
+
if (!this.transportClient) {
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
const transportClient = this.transportClient;
|
|
1039
|
+
this.transportClient = null;
|
|
1040
|
+
await transportClient.disconnect();
|
|
1041
|
+
}
|
|
1042
|
+
async sendText(text) {
|
|
1043
|
+
await this.getActiveTransportClient().sendText(text);
|
|
1044
|
+
}
|
|
1045
|
+
async sendAudio(audio) {
|
|
1046
|
+
await this.getActiveTransportClient().sendAudio(audio);
|
|
1047
|
+
}
|
|
1048
|
+
async sendToolResult(callId, output) {
|
|
1049
|
+
await this.getActiveTransportClient().sendToolResult(callId, output);
|
|
1050
|
+
}
|
|
1051
|
+
async interrupt() {
|
|
1052
|
+
await this.getActiveTransportClient().interrupt();
|
|
1053
|
+
}
|
|
1054
|
+
onEvent(callback) {
|
|
1055
|
+
this.eventCallbacks.add(callback);
|
|
1056
|
+
this.transportClient?.onEvent(callback);
|
|
1057
|
+
}
|
|
1058
|
+
getActiveTransportClient() {
|
|
1059
|
+
if (!this.transportClient) {
|
|
1060
|
+
throw new Error("Realtime session not active");
|
|
1061
|
+
}
|
|
1062
|
+
return this.transportClient;
|
|
1063
|
+
}
|
|
1064
|
+
resolveAdapterId(config) {
|
|
1065
|
+
if (this.config.resolveAdapterId) {
|
|
1066
|
+
return this.config.resolveAdapterId(config);
|
|
1067
|
+
}
|
|
1068
|
+
const transport = config?.transport ?? "webrtc";
|
|
1069
|
+
if (config?.provider === "openai" && transport === "webrtc") {
|
|
1070
|
+
return core.OPENAI_REALTIME_AGENTS_ADAPTER;
|
|
1071
|
+
}
|
|
1072
|
+
throw new Error(
|
|
1073
|
+
`No remote realtime adapter resolver for provider "${config?.provider ?? "(unspecified)"}" and transport "${transport}"`
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
async requestBootstrap(request, options) {
|
|
1077
|
+
const response = await fetchWithTimeout(
|
|
1078
|
+
this.config.apiEndpoint || "/api/realtime",
|
|
1079
|
+
{
|
|
1080
|
+
method: "POST",
|
|
1081
|
+
headers: {
|
|
1082
|
+
"Content-Type": "application/json"
|
|
1083
|
+
},
|
|
1084
|
+
body: JSON.stringify({
|
|
1085
|
+
...request,
|
|
1086
|
+
adapter: options.expectedAdapterId
|
|
1087
|
+
})
|
|
1088
|
+
},
|
|
1089
|
+
`Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
|
|
1090
|
+
);
|
|
1091
|
+
if (!response.ok) {
|
|
1092
|
+
const errorText = await response.text();
|
|
1093
|
+
throw new Error(`Failed to create Realtime session: ${errorText}`);
|
|
1094
|
+
}
|
|
1095
|
+
const bootstrap = await response.json();
|
|
1096
|
+
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
1097
|
+
throw new Error("Invalid realtime session bootstrap response");
|
|
1098
|
+
}
|
|
1099
|
+
if (bootstrap.adapter !== options.expectedAdapterId) {
|
|
1100
|
+
throw new Error(
|
|
1101
|
+
`Realtime session bootstrap adapter mismatch: expected ${options.expectedAdapterId}, received ${bootstrap.adapter}`
|
|
1102
|
+
);
|
|
1103
|
+
}
|
|
1104
|
+
return bootstrap;
|
|
1105
|
+
}
|
|
1106
|
+
};
|
|
1107
|
+
function createRemoteRealtimeClient(config) {
|
|
1108
|
+
return new RemoteRealtimeClient(config);
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
exports.DEFAULT_REMOTE_REALTIME_ADAPTERS = DEFAULT_REMOTE_REALTIME_ADAPTERS;
|
|
1112
|
+
exports.RemoteRealtimeClient = RemoteRealtimeClient;
|
|
1113
|
+
exports.createRemoteRealtimeClient = createRemoteRealtimeClient;
|
|
1114
|
+
//# sourceMappingURL=index.js.map
|
|
1115
|
+
//# sourceMappingURL=index.js.map
|