@charivo/realtime 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1115 @@
1
+ 'use strict';
2
+
3
+ var core = require('@charivo/core');
4
+ var agentsRealtime = require('@openai/agents-realtime');
5
+
6
+ // src/remote/index.ts
7
+
8
+ // src/internal/shared.ts
9
+ var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
10
+ function isRecord(value) {
11
+ return typeof value === "object" && value !== null;
12
+ }
13
+ function isRealtimeSessionBootstrap(value) {
14
+ if (!isRecord(value) || typeof value.adapter !== "string" || typeof value.transport !== "string") {
15
+ return false;
16
+ }
17
+ if (value.transport === "webrtc") {
18
+ return typeof value.answerSdp === "string" || typeof value.clientSecret === "string";
19
+ }
20
+ if (value.transport === "websocket") {
21
+ return typeof value.url === "string" && typeof value.token === "string";
22
+ }
23
+ return false;
24
+ }
25
+ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
26
+ const controller = new AbortController();
27
+ const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
28
+ try {
29
+ return await fetch(input, {
30
+ ...init,
31
+ signal: controller.signal
32
+ });
33
+ } catch (error) {
34
+ if (isAbortError(error)) {
35
+ throw new Error(timeoutMessage);
36
+ }
37
+ throw error;
38
+ } finally {
39
+ clearTimeout(timeoutId);
40
+ }
41
+ }
42
+ function isAbortError(error) {
43
+ return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
44
+ }
45
+
46
+ // src/openai-agents/bootstrap.ts
47
+ async function getOpenAIRealtimeAgentsBootstrap(options, request) {
48
+ if (options.sessionBootstrap) {
49
+ return options.sessionBootstrap(request);
50
+ }
51
+ const apiEndpoint = options.apiEndpoint;
52
+ if (!apiEndpoint) {
53
+ throw new Error(
54
+ "OpenAI agents realtime client requires apiEndpoint or sessionBootstrap"
55
+ );
56
+ }
57
+ const response = await fetchWithTimeout(
58
+ apiEndpoint,
59
+ {
60
+ method: "POST",
61
+ headers: {
62
+ "Content-Type": "application/json"
63
+ },
64
+ body: JSON.stringify(request)
65
+ },
66
+ `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
67
+ );
68
+ if (!response.ok) {
69
+ const errorText = await response.text();
70
+ throw new Error(`Failed to create Realtime session: ${errorText}`);
71
+ }
72
+ const bootstrap = await response.json();
73
+ if (!isRealtimeSessionBootstrap(bootstrap)) {
74
+ throw new Error("Invalid realtime session bootstrap response");
75
+ }
76
+ return bootstrap;
77
+ }
78
+
79
+ // src/openai-agents/lip-sync-analyzer.ts
80
+ var LipSyncAnalyzer = class {
81
+ constructor(options) {
82
+ this.options = options;
83
+ }
84
+ audioAnalysisStream = null;
85
+ audioContext = null;
86
+ analyser = null;
87
+ lipSyncInterval = null;
88
+ pendingAudioElementPoll = null;
89
+ observedAudioElement = null;
90
+ observedAudioElementListener = null;
91
+ observeAudioElement(audioElement) {
92
+ this.detachObservedAudioElement();
93
+ const tryAttachStream = () => {
94
+ const stream = audioElement.srcObject;
95
+ if (stream instanceof MediaStream) {
96
+ this.attachStream(stream);
97
+ }
98
+ };
99
+ this.observedAudioElement = audioElement;
100
+ this.observedAudioElementListener = tryAttachStream;
101
+ audioElement.addEventListener("loadedmetadata", tryAttachStream);
102
+ this.pendingAudioElementPoll = window.setInterval(tryAttachStream, 50);
103
+ }
104
+ attachStream(stream) {
105
+ if (this.audioAnalysisStream === stream) {
106
+ return;
107
+ }
108
+ this.stopAudioElementPolling();
109
+ this.stopLipSyncAnalysis();
110
+ this.audioAnalysisStream = stream;
111
+ try {
112
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
113
+ if (!audioContextConstructor) {
114
+ throw new Error("AudioContext is not supported in this browser");
115
+ }
116
+ this.audioContext = new audioContextConstructor();
117
+ const source = this.audioContext.createMediaStreamSource(stream);
118
+ this.analyser = this.audioContext.createAnalyser();
119
+ this.analyser.fftSize = 256;
120
+ this.analyser.smoothingTimeConstant = 0.8;
121
+ source.connect(this.analyser);
122
+ this.startLipSyncAnalysis();
123
+ } catch (error) {
124
+ this.options.onError?.(error);
125
+ }
126
+ }
127
+ stopOutput() {
128
+ this.stopLipSyncAnalysis();
129
+ }
130
+ cleanup() {
131
+ this.stopLipSyncAnalysis();
132
+ this.stopAudioElementPolling();
133
+ this.detachObservedAudioElement();
134
+ if (this.audioContext) {
135
+ void this.audioContext.close();
136
+ this.audioContext = null;
137
+ }
138
+ this.audioAnalysisStream = null;
139
+ this.analyser = null;
140
+ }
141
+ startLipSyncAnalysis() {
142
+ if (!this.analyser) {
143
+ return;
144
+ }
145
+ const bufferLength = this.analyser.frequencyBinCount;
146
+ const dataArray = new Uint8Array(bufferLength);
147
+ this.lipSyncInterval = window.setInterval(() => {
148
+ if (!this.analyser) {
149
+ return;
150
+ }
151
+ this.analyser.getByteFrequencyData(dataArray);
152
+ let sum = 0;
153
+ for (let index = 0; index < bufferLength; index += 1) {
154
+ const normalized = dataArray[index] / 255;
155
+ sum += normalized * normalized;
156
+ }
157
+ const rms = Math.sqrt(sum / bufferLength);
158
+ this.options.onRms(Math.min(rms * 3, 1));
159
+ }, 1e3 / 60);
160
+ }
161
+ stopLipSyncAnalysis() {
162
+ if (this.lipSyncInterval) {
163
+ clearInterval(this.lipSyncInterval);
164
+ this.lipSyncInterval = null;
165
+ }
166
+ this.options.onRms(0);
167
+ }
168
+ stopAudioElementPolling() {
169
+ if (this.pendingAudioElementPoll) {
170
+ clearInterval(this.pendingAudioElementPoll);
171
+ this.pendingAudioElementPoll = null;
172
+ }
173
+ }
174
+ detachObservedAudioElement() {
175
+ if (this.observedAudioElement && this.observedAudioElementListener) {
176
+ this.observedAudioElement.removeEventListener(
177
+ "loadedmetadata",
178
+ this.observedAudioElementListener
179
+ );
180
+ }
181
+ this.observedAudioElement = null;
182
+ this.observedAudioElementListener = null;
183
+ }
184
+ };
185
+
186
+ // src/openai-agents/session-config.ts
187
+ var DEFAULT_AGENT_INSTRUCTIONS = "You are a realtime voice agent controlling a Live2D character.";
188
+ var DEFAULT_MODEL = "gpt-realtime-mini";
189
+ var DEFAULT_VOICE = "marin";
190
+ function resolveInstructions(config) {
191
+ return config?.instructions ?? DEFAULT_AGENT_INSTRUCTIONS;
192
+ }
193
+ function resolveVoice(config) {
194
+ return config?.voice ?? DEFAULT_VOICE;
195
+ }
196
+ function toOpenAIRealtimeAgentsSessionConfig(config) {
197
+ return {
198
+ model: config?.model ?? DEFAULT_MODEL,
199
+ instructions: resolveInstructions(config),
200
+ toolChoice: config?.toolChoice ?? "auto",
201
+ outputModalities: ["audio"],
202
+ audio: {
203
+ output: {
204
+ voice: resolveVoice(config)
205
+ }
206
+ }
207
+ };
208
+ }
209
+
210
+ // src/openai-agents/tool-schema.ts
211
+ function createToolSchemaOptions(parameters) {
212
+ const normalizedParameters = toAgentsToolParameters(parameters);
213
+ if (normalizedParameters.additionalProperties === false) {
214
+ return {
215
+ parameters: normalizedParameters,
216
+ strict: true
217
+ };
218
+ }
219
+ return {
220
+ parameters: normalizedParameters,
221
+ strict: false
222
+ };
223
+ }
224
+ function toAgentsToolParameters(parameters) {
225
+ const baseParameters = {
226
+ type: "object",
227
+ properties: parameters.properties,
228
+ required: parameters.required ?? []
229
+ };
230
+ if (getAdditionalProperties(parameters) === false) {
231
+ return {
232
+ ...baseParameters,
233
+ additionalProperties: false
234
+ };
235
+ }
236
+ return {
237
+ ...baseParameters,
238
+ additionalProperties: true
239
+ };
240
+ }
241
+ function getAdditionalProperties(parameters) {
242
+ const value = parameters.additionalProperties;
243
+ return typeof value === "boolean" ? value : void 0;
244
+ }
245
+
246
+ // src/openai-agents/client.ts
247
+ var DEBUG_EVENT_ALLOWLIST = /* @__PURE__ */ new Set([
248
+ "conversation.item.input_audio_transcription.completed",
249
+ "error"
250
+ ]);
251
+ var TOOL_RESULT_TIMEOUT_MESSAGE = "Realtime session ended before tool result was returned";
252
+ var OpenAIRealtimeAgentsClient = class {
253
+ constructor(options = {}) {
254
+ this.options = options;
255
+ }
256
+ session = null;
257
+ transport = null;
258
+ audioElement = null;
259
+ connectionWasActive = false;
260
+ assistant = { text: "", started: false };
261
+ latestHistory = [];
262
+ eventCallbacks = /* @__PURE__ */ new Set();
263
+ pendingToolCalls = /* @__PURE__ */ new Map();
264
+ lipSyncAnalyzer = new LipSyncAnalyzer({
265
+ onRms: (rms) => {
266
+ this.emitEvent({ type: "audio.lipsync", rms });
267
+ },
268
+ onError: (error) => {
269
+ console.error("Failed to setup audio analysis:", error);
270
+ }
271
+ });
272
+ async connect(config) {
273
+ try {
274
+ this.log("Starting OpenAI Agents Realtime WebRTC connection");
275
+ this.audioElement = document.createElement("audio");
276
+ this.audioElement.autoplay = true;
277
+ this.audioElement.setAttribute("playsinline", "true");
278
+ const agent = new agentsRealtime.RealtimeAgent({
279
+ name: "charivo-realtime-agent",
280
+ instructions: resolveInstructions(config),
281
+ tools: this.createProxyTools(config?.tools),
282
+ voice: resolveVoice(config)
283
+ });
284
+ this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
285
+ this.createTransportOptions(this.audioElement)
286
+ );
287
+ this.session = new agentsRealtime.RealtimeSession(agent, {
288
+ transport: this.transport,
289
+ config: toOpenAIRealtimeAgentsSessionConfig(config)
290
+ });
291
+ this.bindSessionEvents(this.session);
292
+ this.bindTransportEvents(this.transport);
293
+ this.lipSyncAnalyzer.observeAudioElement(this.audioElement);
294
+ const bootstrap = await this.getSessionBootstrap({
295
+ adapter: core.OPENAI_REALTIME_AGENTS_ADAPTER,
296
+ transport: "webrtc",
297
+ session: config ?? {}
298
+ });
299
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_AGENTS_ADAPTER || bootstrap.transport !== "webrtc" || !("clientSecret" in bootstrap) || typeof bootstrap.clientSecret !== "string") {
300
+ throw new Error(
301
+ `OpenAI agents realtime client only supports ${core.OPENAI_REALTIME_AGENTS_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
302
+ );
303
+ }
304
+ await this.session.connect({
305
+ apiKey: bootstrap.clientSecret,
306
+ model: config?.model
307
+ });
308
+ this.connectionWasActive = true;
309
+ this.emitEvent({ type: "session.started" });
310
+ } catch (error) {
311
+ this.cleanup(error);
312
+ throw error;
313
+ }
314
+ }
315
+ async disconnect() {
316
+ this.log("Disconnecting OpenAI Agents Realtime WebRTC");
317
+ this.cleanup();
318
+ }
319
+ async sendText(text) {
320
+ if (!this.session) {
321
+ throw new Error("Realtime session not active");
322
+ }
323
+ this.resetAssistantTracking();
324
+ this.session.sendMessage(text);
325
+ }
326
+ async sendAudio(_audio) {
327
+ console.warn(
328
+ "sendAudio is not needed with WebRTC - audio is automatically transmitted"
329
+ );
330
+ }
331
+ async sendToolResult(callId, output) {
332
+ const pendingCall = this.pendingToolCalls.get(callId);
333
+ if (!pendingCall) {
334
+ throw new Error(`No pending realtime tool call for "${callId}"`);
335
+ }
336
+ this.pendingToolCalls.delete(callId);
337
+ pendingCall.resolve(output);
338
+ }
339
+ async interrupt() {
340
+ if (!this.session) {
341
+ throw new Error("Realtime session not active");
342
+ }
343
+ this.session.interrupt();
344
+ this.assistant.started = false;
345
+ }
346
+ onEvent(callback) {
347
+ this.eventCallbacks.add(callback);
348
+ }
349
+ bindSessionEvents(session) {
350
+ session.on("agent_start", () => {
351
+ this.ensureAssistantStarted();
352
+ });
353
+ session.on("agent_end", (_context, _agent, output) => {
354
+ this.finalizeAssistantResponse(output);
355
+ });
356
+ session.on("audio_start", () => {
357
+ this.emitEvent({ type: "audio.output.started" });
358
+ });
359
+ session.on("audio_stopped", () => {
360
+ this.emitEvent({ type: "audio.output.ended" });
361
+ });
362
+ session.on("history_updated", (history) => {
363
+ this.latestHistory = history;
364
+ });
365
+ session.on("error", ({ error }) => {
366
+ this.emitEvent({
367
+ type: "error",
368
+ error: error instanceof Error ? error : new Error(String(error))
369
+ });
370
+ });
371
+ session.on("transport_event", (event) => {
372
+ if (this.options.debug && DEBUG_EVENT_ALLOWLIST.has(event.type)) {
373
+ this.log("\u{1F4E1} [OpenAI Agents Transport Event]", event.type, event);
374
+ }
375
+ if (event.type === "conversation.item.input_audio_transcription.completed" && typeof event.transcript === "string") {
376
+ this.emitEvent({
377
+ type: "user.transcript",
378
+ text: event.transcript
379
+ });
380
+ }
381
+ });
382
+ }
383
+ bindTransportEvents(transport) {
384
+ transport.on("audio_transcript_delta", (event) => {
385
+ this.handleAssistantTranscriptDelta(event);
386
+ });
387
+ transport.on("audio_interrupted", () => {
388
+ this.emitEvent({ type: "audio.output.ended" });
389
+ this.lipSyncAnalyzer.stopOutput();
390
+ });
391
+ transport.on("connection_change", (status) => {
392
+ if (status === "disconnected" && this.connectionWasActive) {
393
+ this.cleanupPendingToolCalls();
394
+ this.connectionWasActive = false;
395
+ this.emitEvent({ type: "session.ended" });
396
+ }
397
+ });
398
+ }
399
+ handleAssistantTranscriptDelta(event) {
400
+ if (!event.delta) {
401
+ return;
402
+ }
403
+ this.ensureAssistantStarted();
404
+ this.assistant.text += event.delta;
405
+ this.emitEvent({
406
+ type: "assistant.text.delta",
407
+ text: event.delta
408
+ });
409
+ }
410
+ ensureAssistantStarted() {
411
+ if (this.assistant.started) {
412
+ return;
413
+ }
414
+ this.assistant.started = true;
415
+ this.emitEvent({ type: "assistant.response.started" });
416
+ }
417
+ finalizeAssistantResponse(output) {
418
+ const finalText = this.getLatestAssistantText() || output || this.assistant.text;
419
+ this.ensureAssistantStarted();
420
+ if (finalText && finalText.startsWith(this.assistant.text) && finalText !== this.assistant.text) {
421
+ const delta = finalText.slice(this.assistant.text.length);
422
+ if (delta) {
423
+ this.emitEvent({
424
+ type: "assistant.text.delta",
425
+ text: delta
426
+ });
427
+ }
428
+ }
429
+ this.emitEvent({
430
+ type: "assistant.response.completed",
431
+ text: finalText
432
+ });
433
+ this.resetAssistantTracking();
434
+ }
435
+ getLatestAssistantText() {
436
+ for (let index = this.latestHistory.length - 1; index >= 0; index -= 1) {
437
+ const item = this.latestHistory[index];
438
+ if (item.type === "message" && item.role === "assistant" && Array.isArray(item.content)) {
439
+ const text = item.content.map((content) => {
440
+ if (content.type === "output_audio") {
441
+ return content.transcript ?? "";
442
+ }
443
+ if (content.type === "output_text") {
444
+ return content.text;
445
+ }
446
+ return "";
447
+ }).join("").trim();
448
+ if (text) {
449
+ return text;
450
+ }
451
+ }
452
+ }
453
+ return "";
454
+ }
455
+ createProxyTools(tools) {
456
+ return (tools ?? []).map((definition) => {
457
+ const execute = async (input, _context, details) => {
458
+ const toolCallItem = details?.toolCall;
459
+ const callId = toolCallItem?.callId ?? this.createToolCallId(definition.name);
460
+ return await new Promise((resolve, reject) => {
461
+ this.pendingToolCalls.set(callId, {
462
+ resolve,
463
+ reject
464
+ });
465
+ this.emitEvent({
466
+ type: "tool.call",
467
+ name: definition.name,
468
+ args: isRecord(input) ? input : {},
469
+ callId
470
+ });
471
+ });
472
+ };
473
+ const schemaOptions = createToolSchemaOptions(definition.parameters);
474
+ if (schemaOptions.strict) {
475
+ return agentsRealtime.tool({
476
+ name: definition.name,
477
+ description: definition.description,
478
+ parameters: schemaOptions.parameters,
479
+ strict: true,
480
+ needsApproval: false,
481
+ execute
482
+ });
483
+ }
484
+ return agentsRealtime.tool({
485
+ name: definition.name,
486
+ description: definition.description,
487
+ parameters: schemaOptions.parameters,
488
+ strict: false,
489
+ needsApproval: false,
490
+ execute
491
+ });
492
+ });
493
+ }
494
+ createTransportOptions(audioElement) {
495
+ return {
496
+ audioElement,
497
+ changePeerConnection: async (peerConnection) => {
498
+ peerConnection.addEventListener("track", (event) => {
499
+ const stream = event.streams[0];
500
+ if (stream) {
501
+ this.lipSyncAnalyzer.attachStream(stream);
502
+ }
503
+ });
504
+ return peerConnection;
505
+ }
506
+ };
507
+ }
508
+ cleanup(error) {
509
+ this.cleanupPendingToolCalls(error);
510
+ this.lipSyncAnalyzer.cleanup();
511
+ if (this.transport) {
512
+ this.transport.close();
513
+ this.transport = null;
514
+ }
515
+ if (this.session) {
516
+ this.session.close();
517
+ this.session = null;
518
+ }
519
+ if (this.audioElement) {
520
+ this.audioElement.srcObject = null;
521
+ this.audioElement = null;
522
+ }
523
+ this.latestHistory = [];
524
+ this.resetAssistantTracking();
525
+ }
526
+ resetAssistantTracking() {
527
+ this.assistant = {
528
+ text: "",
529
+ started: false
530
+ };
531
+ }
532
+ cleanupPendingToolCalls(error) {
533
+ const toolError = error instanceof Error ? error : new Error(TOOL_RESULT_TIMEOUT_MESSAGE);
534
+ for (const pendingCall of this.pendingToolCalls.values()) {
535
+ pendingCall.reject(toolError);
536
+ }
537
+ this.pendingToolCalls.clear();
538
+ }
539
+ createToolCallId(toolName) {
540
+ return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${toolName}-${Date.now()}`;
541
+ }
542
+ async getSessionBootstrap(request) {
543
+ return getOpenAIRealtimeAgentsBootstrap(this.options, request);
544
+ }
545
+ emitEvent(event) {
546
+ for (const callback of this.eventCallbacks) {
547
+ callback(event);
548
+ }
549
+ }
550
+ log(...args) {
551
+ if (!this.options.debug) {
552
+ return;
553
+ }
554
+ console.debug("[charivo/realtime/openai-agents]", ...args);
555
+ }
556
+ };
557
+ function createOpenAIRealtimeAgentsClient(options) {
558
+ return new OpenAIRealtimeAgentsClient(options);
559
+ }
560
+ var DEBUG_EVENT_ALLOWLIST2 = /* @__PURE__ */ new Set([
561
+ "session.created",
562
+ "session.updated",
563
+ "response.created",
564
+ "response.done",
565
+ "response.function_call_arguments.done",
566
+ "conversation.item.input_audio_transcription.completed",
567
+ "error"
568
+ ]);
569
+ var OpenAIRealtimeClient = class {
570
+ constructor(options = {}) {
571
+ this.options = options;
572
+ }
573
+ pc = null;
574
+ dc = null;
575
+ audioElement = null;
576
+ mediaStream = null;
577
+ audioContext = null;
578
+ analyser = null;
579
+ lipSyncInterval = null;
580
+ isResponseInProgress = false;
581
+ cancelInFlight = false;
582
+ hasStartedAssistantResponse = false;
583
+ hasStartedAudioOutput = false;
584
+ assistantText = "";
585
+ eventCallbacks = /* @__PURE__ */ new Set();
586
+ async connect(config) {
587
+ try {
588
+ this.log("Starting OpenAI Realtime WebRTC connection");
589
+ this.pc = new RTCPeerConnection();
590
+ this.audioElement = document.createElement("audio");
591
+ this.audioElement.autoplay = true;
592
+ this.pc.ontrack = (event) => {
593
+ if (this.audioElement) {
594
+ this.audioElement.srcObject = event.streams[0];
595
+ }
596
+ this.setupAudioAnalysis(event.streams[0]);
597
+ };
598
+ try {
599
+ this.mediaStream = await navigator.mediaDevices.getUserMedia({
600
+ audio: true
601
+ });
602
+ const audioTrack = this.mediaStream.getTracks()[0];
603
+ this.pc.addTrack(audioTrack, this.mediaStream);
604
+ } catch {
605
+ throw new Error("Microphone access required for Realtime API");
606
+ }
607
+ this.dc = this.pc.createDataChannel("oai-events");
608
+ this.dc.onmessage = (event) => {
609
+ const payload = JSON.parse(event.data);
610
+ this.handleServerEvent(payload);
611
+ };
612
+ this.dc.onerror = () => {
613
+ this.emitEvent({
614
+ type: "error",
615
+ error: new Error("DataChannel error")
616
+ });
617
+ };
618
+ const offer = await this.pc.createOffer();
619
+ await this.pc.setLocalDescription(offer);
620
+ const bootstrap = await this.getSessionBootstrap({
621
+ transport: "webrtc",
622
+ session: config ?? {},
623
+ sdpOffer: offer.sdp
624
+ });
625
+ if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
626
+ throw new Error(
627
+ `OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
628
+ );
629
+ }
630
+ await this.pc.setRemoteDescription({
631
+ type: "answer",
632
+ sdp: bootstrap.answerSdp
633
+ });
634
+ this.emitEvent({ type: "session.started" });
635
+ } catch (error) {
636
+ this.cleanup();
637
+ throw error;
638
+ }
639
+ }
640
+ async disconnect() {
641
+ this.log("Disconnecting OpenAI Realtime WebRTC");
642
+ this.cleanup();
643
+ }
644
+ async sendText(text) {
645
+ if (!this.dc || this.dc.readyState !== "open") {
646
+ throw new Error("DataChannel not ready");
647
+ }
648
+ if (this.isResponseInProgress) {
649
+ console.warn("\u26A0\uFE0F Response already in progress, skipping request");
650
+ return;
651
+ }
652
+ this.dc.send(
653
+ JSON.stringify({
654
+ type: "conversation.item.create",
655
+ item: {
656
+ type: "message",
657
+ role: "user",
658
+ content: [
659
+ {
660
+ type: "input_text",
661
+ text
662
+ }
663
+ ]
664
+ }
665
+ })
666
+ );
667
+ this.dc.send(JSON.stringify({ type: "response.create" }));
668
+ this.beginResponseRequest();
669
+ }
670
+ async sendAudio(_audio) {
671
+ console.warn(
672
+ "sendAudio is not needed with WebRTC - audio is automatically transmitted"
673
+ );
674
+ }
675
+ async sendToolResult(callId, output) {
676
+ const dc = this.requireOpenDataChannel();
677
+ dc.send(
678
+ JSON.stringify({
679
+ type: "conversation.item.create",
680
+ item: {
681
+ type: "function_call_output",
682
+ call_id: callId,
683
+ output: JSON.stringify(output)
684
+ }
685
+ })
686
+ );
687
+ dc.send(JSON.stringify({ type: "response.create" }));
688
+ this.beginResponseRequest();
689
+ }
690
+ async interrupt() {
691
+ const dc = this.requireOpenDataChannel();
692
+ if (!this.isResponseInProgress) {
693
+ return;
694
+ }
695
+ dc.send(JSON.stringify({ type: "response.cancel" }));
696
+ this.isResponseInProgress = false;
697
+ this.cancelInFlight = true;
698
+ }
699
+ onEvent(callback) {
700
+ this.eventCallbacks.add(callback);
701
+ }
702
+ handleServerEvent(event) {
703
+ if (this.options.debug && this.shouldLogDebugEvent(event.type)) {
704
+ this.log("\u{1F4E1} [OpenAI Realtime Event]", event.type, event);
705
+ }
706
+ switch (event.type) {
707
+ case "session.created":
708
+ case "session.updated":
709
+ return;
710
+ case "response.audio.delta":
711
+ case "output_audio_buffer.started":
712
+ if (this.cancelInFlight) {
713
+ return;
714
+ }
715
+ if (!this.hasStartedAudioOutput) {
716
+ this.hasStartedAudioOutput = true;
717
+ this.emitEvent({ type: "audio.output.started" });
718
+ }
719
+ return;
720
+ case "response.audio.done":
721
+ case "response.output_audio.done":
722
+ case "output_audio_buffer.stopped":
723
+ if (this.cancelInFlight) {
724
+ this.hasStartedAudioOutput = false;
725
+ return;
726
+ }
727
+ this.emitEvent({ type: "audio.output.ended" });
728
+ this.hasStartedAudioOutput = false;
729
+ return;
730
+ case "response.done":
731
+ this.isResponseInProgress = false;
732
+ if (this.cancelInFlight) {
733
+ this.cancelInFlight = false;
734
+ this.resetResponseTracking();
735
+ return;
736
+ }
737
+ this.emitEvent({
738
+ type: "assistant.response.completed",
739
+ text: this.assistantText
740
+ });
741
+ this.resetResponseTracking();
742
+ return;
743
+ case "response.audio_transcript.delta":
744
+ case "response.output_audio_transcript.delta":
745
+ case "response.output_text.delta":
746
+ if (this.cancelInFlight || !event.delta) {
747
+ return;
748
+ }
749
+ if (!this.hasStartedAssistantResponse) {
750
+ this.hasStartedAssistantResponse = true;
751
+ this.emitEvent({ type: "assistant.response.started" });
752
+ }
753
+ this.assistantText += event.delta;
754
+ this.emitEvent({
755
+ type: "assistant.text.delta",
756
+ text: event.delta
757
+ });
758
+ return;
759
+ case "response.output_text.done":
760
+ if (this.cancelInFlight || !event.text) {
761
+ return;
762
+ }
763
+ if (!this.hasStartedAssistantResponse) {
764
+ this.hasStartedAssistantResponse = true;
765
+ this.emitEvent({ type: "assistant.response.started" });
766
+ }
767
+ if (event.text !== this.assistantText) {
768
+ const delta = event.text.startsWith(this.assistantText) ? event.text.slice(this.assistantText.length) : event.text;
769
+ this.assistantText = event.text;
770
+ if (delta) {
771
+ this.emitEvent({
772
+ type: "assistant.text.delta",
773
+ text: delta
774
+ });
775
+ }
776
+ }
777
+ return;
778
+ case "response.output_audio_transcript.done":
779
+ if (this.cancelInFlight || !event.transcript) {
780
+ return;
781
+ }
782
+ if (!this.hasStartedAssistantResponse) {
783
+ this.hasStartedAssistantResponse = true;
784
+ this.emitEvent({ type: "assistant.response.started" });
785
+ }
786
+ if (event.transcript !== this.assistantText) {
787
+ const delta = event.transcript.startsWith(this.assistantText) ? event.transcript.slice(this.assistantText.length) : event.transcript;
788
+ this.assistantText = event.transcript;
789
+ if (delta) {
790
+ this.emitEvent({
791
+ type: "assistant.text.delta",
792
+ text: delta
793
+ });
794
+ }
795
+ }
796
+ return;
797
+ case "conversation.item.input_audio_transcription.completed":
798
+ if (event.transcript) {
799
+ this.emitEvent({
800
+ type: "user.transcript",
801
+ text: event.transcript
802
+ });
803
+ }
804
+ return;
805
+ case "response.function_call_arguments.done":
806
+ this.handleToolCallDone(event);
807
+ return;
808
+ case "error":
809
+ this.isResponseInProgress = false;
810
+ this.cancelInFlight = false;
811
+ this.emitEvent({
812
+ type: "error",
813
+ error: new Error(event.error?.message || "Unknown error")
814
+ });
815
+ return;
816
+ default:
817
+ return;
818
+ }
819
+ }
820
+ handleToolCallDone(event) {
821
+ const callId = event.call_id || event.item?.call_id || event.item_id;
822
+ const name = event.name || event.item?.name;
823
+ const argsJson = event.arguments || event.item?.arguments;
824
+ if (!name || !argsJson) {
825
+ console.warn("\u26A0\uFE0F Tool call done but missing name or arguments");
826
+ return;
827
+ }
828
+ let args = {};
829
+ try {
830
+ const parsed = JSON.parse(argsJson);
831
+ if (isRecord(parsed)) {
832
+ args = parsed;
833
+ }
834
+ } catch (error) {
835
+ console.error("Failed to parse tool call args", error, argsJson);
836
+ return;
837
+ }
838
+ this.emitEvent({
839
+ type: "tool.call",
840
+ name,
841
+ args,
842
+ callId
843
+ });
844
+ }
845
+ async getSessionBootstrap(request) {
846
+ if (this.options.sessionBootstrap) {
847
+ return this.options.sessionBootstrap(request);
848
+ }
849
+ const apiEndpoint = this.options.apiEndpoint;
850
+ if (!apiEndpoint) {
851
+ throw new Error(
852
+ "OpenAI realtime client requires apiEndpoint or sessionBootstrap"
853
+ );
854
+ }
855
+ const response = await fetchWithTimeout(
856
+ apiEndpoint,
857
+ {
858
+ method: "POST",
859
+ headers: {
860
+ "Content-Type": "application/json"
861
+ },
862
+ body: JSON.stringify(request)
863
+ },
864
+ `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
865
+ );
866
+ if (!response.ok) {
867
+ const errorText = await response.text();
868
+ throw new Error(`Failed to create Realtime session: ${errorText}`);
869
+ }
870
+ const bootstrap = await response.json();
871
+ if (!isRealtimeSessionBootstrap(bootstrap)) {
872
+ throw new Error("Invalid realtime session bootstrap response");
873
+ }
874
+ return bootstrap;
875
+ }
876
+ setupAudioAnalysis(stream) {
877
+ try {
878
+ const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
879
+ if (!audioContextConstructor) {
880
+ throw new Error("AudioContext is not supported in this browser");
881
+ }
882
+ this.audioContext = new audioContextConstructor();
883
+ const source = this.audioContext.createMediaStreamSource(stream);
884
+ this.analyser = this.audioContext.createAnalyser();
885
+ this.analyser.fftSize = 256;
886
+ this.analyser.smoothingTimeConstant = 0.8;
887
+ source.connect(this.analyser);
888
+ this.startLipSyncAnalysis();
889
+ } catch (error) {
890
+ console.error("Failed to setup audio analysis:", error);
891
+ }
892
+ }
893
+ startLipSyncAnalysis() {
894
+ if (!this.analyser) {
895
+ return;
896
+ }
897
+ const bufferLength = this.analyser.frequencyBinCount;
898
+ const dataArray = new Uint8Array(bufferLength);
899
+ this.lipSyncInterval = window.setInterval(() => {
900
+ if (!this.analyser) {
901
+ return;
902
+ }
903
+ this.analyser.getByteFrequencyData(dataArray);
904
+ let sum = 0;
905
+ for (let index = 0; index < bufferLength; index += 1) {
906
+ const normalized = dataArray[index] / 255;
907
+ sum += normalized * normalized;
908
+ }
909
+ const rms = Math.sqrt(sum / bufferLength);
910
+ this.emitEvent({
911
+ type: "audio.lipsync",
912
+ rms: Math.min(rms * 3, 1)
913
+ });
914
+ }, 1e3 / 60);
915
+ }
916
+ stopLipSyncAnalysis() {
917
+ if (this.lipSyncInterval) {
918
+ clearInterval(this.lipSyncInterval);
919
+ this.lipSyncInterval = null;
920
+ }
921
+ this.emitEvent({ type: "audio.lipsync", rms: 0 });
922
+ }
923
+ cleanup() {
924
+ this.stopLipSyncAnalysis();
925
+ if (this.dc) {
926
+ this.dc.close();
927
+ this.dc = null;
928
+ }
929
+ if (this.pc) {
930
+ this.pc.close();
931
+ this.pc = null;
932
+ }
933
+ if (this.mediaStream) {
934
+ this.mediaStream.getTracks().forEach((track) => track.stop());
935
+ this.mediaStream = null;
936
+ }
937
+ if (this.audioElement) {
938
+ this.audioElement.srcObject = null;
939
+ this.audioElement = null;
940
+ }
941
+ if (this.audioContext) {
942
+ this.audioContext.close();
943
+ this.audioContext = null;
944
+ }
945
+ this.analyser = null;
946
+ this.isResponseInProgress = false;
947
+ this.cancelInFlight = false;
948
+ this.resetResponseTracking();
949
+ }
950
+ beginResponseRequest() {
951
+ this.resetResponseTracking();
952
+ this.isResponseInProgress = true;
953
+ this.cancelInFlight = false;
954
+ }
955
+ requireOpenDataChannel() {
956
+ if (!this.dc || this.dc.readyState !== "open") {
957
+ throw new Error("DataChannel not ready");
958
+ }
959
+ return this.dc;
960
+ }
961
+ resetResponseTracking() {
962
+ this.assistantText = "";
963
+ this.hasStartedAssistantResponse = false;
964
+ this.hasStartedAudioOutput = false;
965
+ }
966
+ emitEvent(event) {
967
+ for (const callback of this.eventCallbacks) {
968
+ callback(event);
969
+ }
970
+ }
971
+ log(...args) {
972
+ if (this.options.debug) {
973
+ console.log(...args);
974
+ }
975
+ }
976
+ shouldLogDebugEvent(eventType) {
977
+ if (DEBUG_EVENT_ALLOWLIST2.has(eventType)) {
978
+ return true;
979
+ }
980
+ return eventType.startsWith("response.output_text.");
981
+ }
982
+ };
983
+ function createOpenAIRealtimeClient(options) {
984
+ return new OpenAIRealtimeClient(options);
985
+ }
986
+
987
+ // src/remote/index.ts
988
+ var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
989
+ [core.OPENAI_REALTIME_AGENTS_ADAPTER]: (options) => createOpenAIRealtimeAgentsClient({
990
+ debug: options.debug,
991
+ sessionBootstrap: options.requestBootstrap
992
+ }),
993
+ [core.OPENAI_REALTIME_ADAPTER]: (options) => createOpenAIRealtimeClient({
994
+ debug: options.debug,
995
+ sessionBootstrap: options.requestBootstrap
996
+ })
997
+ };
998
+ var RemoteRealtimeClient = class {
999
+ constructor(config = {}) {
1000
+ this.config = config;
1001
+ this.adapters = new Map(
1002
+ Object.entries({
1003
+ ...DEFAULT_REMOTE_REALTIME_ADAPTERS,
1004
+ ...config.adapters
1005
+ })
1006
+ );
1007
+ }
1008
+ transportClient = null;
1009
+ adapters;
1010
+ eventCallbacks = /* @__PURE__ */ new Set();
1011
+ async connect(config) {
1012
+ const adapterId = this.resolveAdapterId(config);
1013
+ const factory = this.adapters.get(adapterId);
1014
+ if (!factory) {
1015
+ throw new Error(
1016
+ `No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
1017
+ );
1018
+ }
1019
+ const transportClient = factory({
1020
+ debug: this.config.debug,
1021
+ requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
1022
+ });
1023
+ for (const callback of this.eventCallbacks) {
1024
+ transportClient.onEvent(callback);
1025
+ }
1026
+ this.transportClient = transportClient;
1027
+ try {
1028
+ await transportClient.connect(config);
1029
+ } catch (error) {
1030
+ this.transportClient = null;
1031
+ throw error;
1032
+ }
1033
+ }
1034
+ async disconnect() {
1035
+ if (!this.transportClient) {
1036
+ return;
1037
+ }
1038
+ const transportClient = this.transportClient;
1039
+ this.transportClient = null;
1040
+ await transportClient.disconnect();
1041
+ }
1042
+ async sendText(text) {
1043
+ await this.getActiveTransportClient().sendText(text);
1044
+ }
1045
+ async sendAudio(audio) {
1046
+ await this.getActiveTransportClient().sendAudio(audio);
1047
+ }
1048
+ async sendToolResult(callId, output) {
1049
+ await this.getActiveTransportClient().sendToolResult(callId, output);
1050
+ }
1051
+ async interrupt() {
1052
+ await this.getActiveTransportClient().interrupt();
1053
+ }
1054
+ onEvent(callback) {
1055
+ this.eventCallbacks.add(callback);
1056
+ this.transportClient?.onEvent(callback);
1057
+ }
1058
+ getActiveTransportClient() {
1059
+ if (!this.transportClient) {
1060
+ throw new Error("Realtime session not active");
1061
+ }
1062
+ return this.transportClient;
1063
+ }
1064
+ resolveAdapterId(config) {
1065
+ if (this.config.resolveAdapterId) {
1066
+ return this.config.resolveAdapterId(config);
1067
+ }
1068
+ const transport = config?.transport ?? "webrtc";
1069
+ if (config?.provider === "openai" && transport === "webrtc") {
1070
+ return core.OPENAI_REALTIME_AGENTS_ADAPTER;
1071
+ }
1072
+ throw new Error(
1073
+ `No remote realtime adapter resolver for provider "${config?.provider ?? "(unspecified)"}" and transport "${transport}"`
1074
+ );
1075
+ }
1076
+ async requestBootstrap(request, options) {
1077
+ const response = await fetchWithTimeout(
1078
+ this.config.apiEndpoint || "/api/realtime",
1079
+ {
1080
+ method: "POST",
1081
+ headers: {
1082
+ "Content-Type": "application/json"
1083
+ },
1084
+ body: JSON.stringify({
1085
+ ...request,
1086
+ adapter: options.expectedAdapterId
1087
+ })
1088
+ },
1089
+ `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
1090
+ );
1091
+ if (!response.ok) {
1092
+ const errorText = await response.text();
1093
+ throw new Error(`Failed to create Realtime session: ${errorText}`);
1094
+ }
1095
+ const bootstrap = await response.json();
1096
+ if (!isRealtimeSessionBootstrap(bootstrap)) {
1097
+ throw new Error("Invalid realtime session bootstrap response");
1098
+ }
1099
+ if (bootstrap.adapter !== options.expectedAdapterId) {
1100
+ throw new Error(
1101
+ `Realtime session bootstrap adapter mismatch: expected ${options.expectedAdapterId}, received ${bootstrap.adapter}`
1102
+ );
1103
+ }
1104
+ return bootstrap;
1105
+ }
1106
+ };
1107
+ function createRemoteRealtimeClient(config) {
1108
+ return new RemoteRealtimeClient(config);
1109
+ }
1110
+
1111
+ exports.DEFAULT_REMOTE_REALTIME_ADAPTERS = DEFAULT_REMOTE_REALTIME_ADAPTERS;
1112
+ exports.RemoteRealtimeClient = RemoteRealtimeClient;
1113
+ exports.createRemoteRealtimeClient = createRemoteRealtimeClient;
1114
+ //# sourceMappingURL=index.js.map
1115
+ //# sourceMappingURL=index.js.map