@mooncompany/uplink-chat 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of @mooncompany/uplink-chat might be problematic. Click here for more details.

Files changed (158) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +185 -0
  3. package/bin/uplink.js +279 -0
  4. package/middleware/error-handler.js +69 -0
  5. package/package.json +93 -0
  6. package/public/css/agents.36b98c0f.css +1469 -0
  7. package/public/css/agents.css +1469 -0
  8. package/public/css/app.a6a7f8f5.css +2731 -0
  9. package/public/css/app.css +2731 -0
  10. package/public/css/artifacts.css +444 -0
  11. package/public/css/commands.css +55 -0
  12. package/public/css/connection.css +131 -0
  13. package/public/css/dashboard.css +233 -0
  14. package/public/css/developer.css +328 -0
  15. package/public/css/files.css +123 -0
  16. package/public/css/markdown.css +156 -0
  17. package/public/css/message-actions.css +278 -0
  18. package/public/css/mobile.css +614 -0
  19. package/public/css/panels-unified.css +483 -0
  20. package/public/css/premium.css +415 -0
  21. package/public/css/realtime.css +189 -0
  22. package/public/css/satellites.css +401 -0
  23. package/public/css/shortcuts.css +185 -0
  24. package/public/css/split-view.4def0262.css +673 -0
  25. package/public/css/split-view.css +673 -0
  26. package/public/css/theme-generator.css +391 -0
  27. package/public/css/themes.css +387 -0
  28. package/public/css/timestamps.css +54 -0
  29. package/public/css/variables.css +78 -0
  30. package/public/dist/bundle.b55050c4.js +15757 -0
  31. package/public/favicon.svg +24 -0
  32. package/public/img/agents/ada.png +0 -0
  33. package/public/img/agents/clarice.png +0 -0
  34. package/public/img/agents/dennis-nedry.png +0 -0
  35. package/public/img/agents/elliot-alderson.png +0 -0
  36. package/public/img/agents/main.png +0 -0
  37. package/public/img/agents/scotty.png +0 -0
  38. package/public/img/agents/top-flight-security.png +0 -0
  39. package/public/index.html +1083 -0
  40. package/public/js/agents-data.js +234 -0
  41. package/public/js/agents-ui.js +72 -0
  42. package/public/js/agents.js +1525 -0
  43. package/public/js/app.js +79 -0
  44. package/public/js/appearance-settings.js +111 -0
  45. package/public/js/artifacts.js +432 -0
  46. package/public/js/audio-queue.js +168 -0
  47. package/public/js/bootstrap.js +54 -0
  48. package/public/js/chat.js +1211 -0
  49. package/public/js/commands.js +581 -0
  50. package/public/js/connection-api.js +121 -0
  51. package/public/js/connection.js +1231 -0
  52. package/public/js/context-tracker.js +271 -0
  53. package/public/js/core.js +172 -0
  54. package/public/js/dashboard.js +452 -0
  55. package/public/js/developer.js +432 -0
  56. package/public/js/encryption.js +124 -0
  57. package/public/js/errors.js +122 -0
  58. package/public/js/event-bus.js +77 -0
  59. package/public/js/fetch-utils.js +171 -0
  60. package/public/js/file-handler.js +229 -0
  61. package/public/js/files.js +352 -0
  62. package/public/js/gateway-chat.js +538 -0
  63. package/public/js/logger.js +112 -0
  64. package/public/js/markdown.js +190 -0
  65. package/public/js/message-actions.js +431 -0
  66. package/public/js/message-renderer.js +288 -0
  67. package/public/js/missed-messages.js +235 -0
  68. package/public/js/mobile-debug.js +95 -0
  69. package/public/js/notifications.js +367 -0
  70. package/public/js/offline-queue.js +178 -0
  71. package/public/js/onboarding.js +543 -0
  72. package/public/js/panels.js +156 -0
  73. package/public/js/premium.js +412 -0
  74. package/public/js/realtime-voice.js +844 -0
  75. package/public/js/satellite-sync.js +256 -0
  76. package/public/js/satellite-ui.js +175 -0
  77. package/public/js/satellites.js +1516 -0
  78. package/public/js/settings.js +1087 -0
  79. package/public/js/shortcuts.js +381 -0
  80. package/public/js/split-chat.js +1234 -0
  81. package/public/js/split-resize.js +211 -0
  82. package/public/js/splitview.js +340 -0
  83. package/public/js/storage.js +408 -0
  84. package/public/js/streaming-handler.js +324 -0
  85. package/public/js/stt-settings.js +316 -0
  86. package/public/js/theme-generator.js +661 -0
  87. package/public/js/themes.js +164 -0
  88. package/public/js/timestamps.js +198 -0
  89. package/public/js/tts-settings.js +575 -0
  90. package/public/js/ui.js +267 -0
  91. package/public/js/update-notifier.js +143 -0
  92. package/public/js/utils/constants.js +165 -0
  93. package/public/js/utils/sanitize.js +93 -0
  94. package/public/js/utils/sse-parser.js +195 -0
  95. package/public/js/voice.js +883 -0
  96. package/public/manifest.json +58 -0
  97. package/public/moon_texture.jpg +0 -0
  98. package/public/sw.js +221 -0
  99. package/public/three.min.js +6 -0
  100. package/server/channel.js +529 -0
  101. package/server/chat.js +270 -0
  102. package/server/config-store.js +362 -0
  103. package/server/config.js +159 -0
  104. package/server/context.js +131 -0
  105. package/server/gateway-commands.js +211 -0
  106. package/server/gateway-proxy.js +318 -0
  107. package/server/index.js +22 -0
  108. package/server/logger.js +89 -0
  109. package/server/middleware/auth.js +188 -0
  110. package/server/middleware.js +218 -0
  111. package/server/openclaw-discover.js +308 -0
  112. package/server/premium/index.js +156 -0
  113. package/server/premium/license.js +140 -0
  114. package/server/realtime/bridge.js +837 -0
  115. package/server/realtime/index.js +349 -0
  116. package/server/realtime/tts-stream.js +446 -0
  117. package/server/routes/agents.js +564 -0
  118. package/server/routes/artifacts.js +174 -0
  119. package/server/routes/chat.js +311 -0
  120. package/server/routes/config-settings.js +345 -0
  121. package/server/routes/config.js +603 -0
  122. package/server/routes/files.js +307 -0
  123. package/server/routes/index.js +18 -0
  124. package/server/routes/media.js +451 -0
  125. package/server/routes/missed-messages.js +107 -0
  126. package/server/routes/premium.js +75 -0
  127. package/server/routes/push.js +156 -0
  128. package/server/routes/satellite.js +406 -0
  129. package/server/routes/status.js +251 -0
  130. package/server/routes/stt.js +35 -0
  131. package/server/routes/voice.js +260 -0
  132. package/server/routes/webhooks.js +203 -0
  133. package/server/routes.js +206 -0
  134. package/server/runtime-config.js +336 -0
  135. package/server/share.js +305 -0
  136. package/server/stt/faster-whisper.js +72 -0
  137. package/server/stt/groq.js +51 -0
  138. package/server/stt/index.js +196 -0
  139. package/server/stt/openai.js +49 -0
  140. package/server/sync.js +244 -0
  141. package/server/tailscale-https.js +175 -0
  142. package/server/tts.js +646 -0
  143. package/server/update-checker.js +172 -0
  144. package/server/utils/filename.js +129 -0
  145. package/server/utils.js +147 -0
  146. package/server/watchdog.js +318 -0
  147. package/server/websocket/broadcast.js +359 -0
  148. package/server/websocket/connections.js +339 -0
  149. package/server/websocket/index.js +215 -0
  150. package/server/websocket/routing.js +277 -0
  151. package/server/websocket/sync.js +102 -0
  152. package/server.js +404 -0
  153. package/utils/detect-tool-usage.js +93 -0
  154. package/utils/errors.js +158 -0
  155. package/utils/html-escape.js +84 -0
  156. package/utils/id-sanitize.js +94 -0
  157. package/utils/response.js +130 -0
  158. package/utils/with-retry.js +105 -0
@@ -0,0 +1,844 @@
1
+ // ============================================
2
+ // REALTIME VOICE MODULE
3
+ // OpenAI Realtime API WebSocket client
4
+ // Supports two modes:
5
+ // - standalone: Direct OpenAI voice (existing)
6
+ // - agent: OpenAI transcription + OpenClaw agent + TTS
7
+ // ============================================
8
+
9
+ import { UplinkLogger } from './logger.js';
10
+
11
+ // State
12
+ let ws = null;
13
+ let isActive = false;
14
+ let sessionReady = false; // True after session.created/session.updated from OpenAI
15
+ let currentMode = 'standalone'; // 'standalone' or 'agent'
16
+ let micStream = null;
17
+ let audioContext = null;
18
+ let audioWorkletNode = null;
19
+ let playbackContext = null;
20
+ let playbackQueue = [];
21
+ let isPlayingAudio = false;
22
+ let nextPlaybackTime = 0;
23
+ let sessionStartTime = null;
24
+ let timerInterval = null;
25
+ let agentName = null; // Loaded from config in agent mode
26
+ let currentUserTranscript = ''; // Buffer for user speech in agent mode
27
+ let currentAgentResponse = ''; // Buffer for agent response in agent mode
28
+ let isMicMuted = false; // Muted during TTS playback to prevent echo
29
+ let streamingMessageDiv = null; // Active streaming chat bubble for agent responses
30
+
31
+ // Constants - WebSocket
32
+ const WS_CLOSE_NORMAL = 1000;
33
+ const WS_RECONNECT_DELAY_MS = 1000;
34
+
35
+ // Constants - Audio
36
+ const SAMPLE_RATE = 24000;
37
+ const BUFFER_SIZE = 4096;
38
+ const PLAYBACK_BUFFER_DURATION = 0.1; // 100ms of buffering
39
+
40
+ // Constants - UI
41
+ const TIMER_UPDATE_INTERVAL_MS = 1000;
42
+
43
+ // ============================================
44
+ // MAIN API
45
+ // ============================================
46
+
47
+ /**
48
+ * Start a real-time voice session
49
+ * @param {string} mode - 'standalone' or 'agent' (default: 'standalone')
50
+ * @returns {Promise<boolean>} Success status
51
+ */
52
+ export async function start(mode = 'standalone') {
53
+ if (isActive) {
54
+ UplinkLogger.warn('Realtime: Session already active');
55
+ return false;
56
+ }
57
+
58
+ if (mode !== 'standalone' && mode !== 'agent') {
59
+ UplinkLogger.error(`Realtime: Invalid mode "${mode}"`);
60
+ return false;
61
+ }
62
+
63
+ currentMode = mode;
64
+
65
+ try {
66
+ // Premium check
67
+ if (window.UplinkPremium && !window.UplinkPremium.isActive()) {
68
+ window.UplinkPremium.showUpgradeModal('Real-time voice chat');
69
+ return false;
70
+ }
71
+
72
+ UplinkLogger.debug(`Realtime: Starting session in ${mode} mode`);
73
+
74
+ // Load agent config if in agent mode
75
+ if (mode === 'agent') {
76
+ await loadAgentConfig();
77
+ }
78
+
79
+ // Request mic access
80
+ if (!await initMicrophone()) {
81
+ UplinkLogger.error('Realtime: Failed to access microphone');
82
+ showError('Microphone access denied');
83
+ return false;
84
+ }
85
+
86
+ // Initialize audio playback
87
+ if (!initPlaybackContext()) {
88
+ UplinkLogger.error('Realtime: Failed to initialize audio playback');
89
+ showError('Audio playback initialization failed');
90
+ cleanup();
91
+ return false;
92
+ }
93
+
94
+ // Open WebSocket
95
+ if (!await connectWebSocket()) {
96
+ UplinkLogger.error('Realtime: Failed to connect WebSocket');
97
+ showError('Connection failed');
98
+ cleanup();
99
+ return false;
100
+ }
101
+
102
+ isActive = true;
103
+ sessionStartTime = Date.now();
104
+ updateUI();
105
+ startTimer();
106
+
107
+ // Add glow class to voice button
108
+ const voiceBtn = document.getElementById('voiceBtn');
109
+ if (voiceBtn) voiceBtn.classList.add('realtime-active');
110
+
111
+ UplinkLogger.debug('Realtime: Session started successfully');
112
+ return true;
113
+ } catch (err) {
114
+ UplinkLogger.error('Realtime: Start failed:', err);
115
+ if (window.UplinkDeveloper) {
116
+ window.UplinkDeveloper.logError(err, 'Realtime.start');
117
+ }
118
+ cleanup();
119
+ return false;
120
+ }
121
+ }
122
+
123
+ /**
124
+ * Stop the real-time voice session
125
+ * Closes WebSocket, stops mic, cleans up audio contexts
126
+ */
127
+ export function stop() {
128
+ if (!isActive) {
129
+ UplinkLogger.warn('Realtime: No active session to stop');
130
+ return;
131
+ }
132
+
133
+ UplinkLogger.debug('Realtime: Stopping session');
134
+ isActive = false;
135
+ cleanup();
136
+ updateUI();
137
+ UplinkLogger.debug('Realtime: Session stopped');
138
+ }
139
+
140
+ /**
141
+ * Check if a session is currently active
142
+ * @returns {boolean}
143
+ */
144
+ export function isSessionActive() {
145
+ return isActive;
146
+ }
147
+
148
+ /**
149
+ * Get the current mode
150
+ * @returns {string} 'standalone' or 'agent'
151
+ */
152
+ export function getMode() {
153
+ return currentMode;
154
+ }
155
+
156
+ // ============================================
157
+ // AGENT CONFIG
158
+ // ============================================
159
+
160
+ async function loadAgentConfig() {
161
+ try {
162
+ const response = await fetch('/api/config');
163
+ if (!response.ok) {
164
+ UplinkLogger.warn('Realtime: Failed to load agent config, using default name');
165
+ agentName = 'Agent';
166
+ return;
167
+ }
168
+ const config = await response.json();
169
+ agentName = config.assistantName || 'Agent';
170
+ UplinkLogger.debug(`Realtime: Agent name loaded: ${agentName}`);
171
+ } catch (err) {
172
+ UplinkLogger.error('Realtime: Config load error:', err);
173
+ agentName = 'Agent';
174
+ }
175
+ }
176
+
177
+ // ============================================
178
+ // MICROPHONE CAPTURE
179
+ // ============================================
180
+
181
+ async function initMicrophone() {
182
+ try {
183
+ micStream = await navigator.mediaDevices.getUserMedia({
184
+ audio: {
185
+ echoCancellation: true,
186
+ noiseSuppression: true,
187
+ autoGainControl: true,
188
+ sampleRate: SAMPLE_RATE
189
+ }
190
+ });
191
+
192
+ audioContext = new (window.AudioContext || window.webkitAudioContext)({
193
+ sampleRate: SAMPLE_RATE
194
+ });
195
+
196
+ const source = audioContext.createMediaStreamSource(micStream);
197
+
198
+ // Use ScriptProcessorNode for compatibility (AudioWorklet is better but requires separate file)
199
+ const processor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
200
+
201
+ processor.onaudioprocess = (e) => {
202
+ if (!isActive || !sessionReady || !ws || ws.readyState !== WebSocket.OPEN) return;
203
+
204
+ const inputData = e.inputBuffer.getChannelData(0);
205
+
206
+ // Convert Float32 to Int16 PCM
207
+ const pcm16 = new Int16Array(inputData.length);
208
+ for (let i = 0; i < inputData.length; i++) {
209
+ const s = Math.max(-1, Math.min(1, inputData[i]));
210
+ pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
211
+ }
212
+
213
+ // Encode to base64
214
+ const base64 = arrayBufferToBase64(pcm16.buffer);
215
+
216
+ // Send to server
217
+ try {
218
+ ws.send(JSON.stringify({
219
+ type: 'input_audio_buffer.append',
220
+ audio: base64
221
+ }));
222
+ } catch (err) {
223
+ UplinkLogger.error('Realtime: Failed to send audio:', err);
224
+ }
225
+ };
226
+
227
+ source.connect(processor);
228
+ processor.connect(audioContext.destination);
229
+ audioWorkletNode = processor; // Store for cleanup
230
+
231
+ UplinkLogger.debug('Realtime: Microphone initialized');
232
+ return true;
233
+ } catch (err) {
234
+ UplinkLogger.error('Realtime: Microphone init failed:', err);
235
+ return false;
236
+ }
237
+ }
238
+
239
+ // ============================================
240
+ // AUDIO PLAYBACK
241
+ // ============================================
242
+
243
+ function initPlaybackContext() {
244
+ try {
245
+ playbackContext = new (window.AudioContext || window.webkitAudioContext)({
246
+ sampleRate: SAMPLE_RATE
247
+ });
248
+
249
+ playbackQueue = [];
250
+ isPlayingAudio = false;
251
+ nextPlaybackTime = playbackContext.currentTime + PLAYBACK_BUFFER_DURATION;
252
+
253
+ UplinkLogger.debug('Realtime: Playback context initialized');
254
+ return true;
255
+ } catch (err) {
256
+ UplinkLogger.error('Realtime: Playback init failed:', err);
257
+ return false;
258
+ }
259
+ }
260
+
261
+ function playAudioChunk(base64Audio) {
262
+ if (!playbackContext || !isActive) return;
263
+
264
+ try {
265
+ // Decode base64 to PCM16
266
+ const pcmData = base64ToArrayBuffer(base64Audio);
267
+ const pcm16 = new Int16Array(pcmData);
268
+
269
+ // Convert Int16 PCM to Float32 for Web Audio API
270
+ const float32 = new Float32Array(pcm16.length);
271
+ for (let i = 0; i < pcm16.length; i++) {
272
+ float32[i] = pcm16[i] / (pcm16[i] < 0 ? 0x8000 : 0x7FFF);
273
+ }
274
+
275
+ // Create audio buffer
276
+ const audioBuffer = playbackContext.createBuffer(1, float32.length, SAMPLE_RATE);
277
+ audioBuffer.getChannelData(0).set(float32);
278
+
279
+ // Create buffer source
280
+ const source = playbackContext.createBufferSource();
281
+ source.buffer = audioBuffer;
282
+ source.connect(playbackContext.destination);
283
+
284
+ // Schedule playback with proper timing to avoid gaps
285
+ const now = playbackContext.currentTime;
286
+ const startTime = Math.max(now, nextPlaybackTime);
287
+ source.start(startTime);
288
+
289
+ // Update next playback time to queue seamlessly
290
+ nextPlaybackTime = startTime + audioBuffer.duration;
291
+
292
+ isPlayingAudio = true;
293
+
294
+ source.onended = () => {
295
+ // Check if we're at the end of the queue
296
+ if (nextPlaybackTime <= playbackContext.currentTime + PLAYBACK_BUFFER_DURATION) {
297
+ isPlayingAudio = false;
298
+ }
299
+ };
300
+
301
+ } catch (err) {
302
+ UplinkLogger.error('Realtime: Audio playback failed:', err);
303
+ }
304
+ }
305
+
306
+ // ============================================
307
+ // WEBSOCKET
308
+ // ============================================
309
+
310
+ function connectWebSocket() {
311
+ return new Promise((resolve, reject) => {
312
+ try {
313
+ // Determine WebSocket protocol based on page protocol
314
+ const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
315
+
316
+ // Add mode query parameter for agent mode
317
+ const modeParam = currentMode === 'agent' ? '?mode=agent' : '';
318
+ const wsUrl = `${protocol}//${location.host}/api/realtime${modeParam}`;
319
+
320
+ UplinkLogger.debug(`Realtime: Connecting to ${wsUrl}`);
321
+ ws = new WebSocket(wsUrl);
322
+
323
+ ws.onopen = () => {
324
+ UplinkLogger.debug('Realtime: WebSocket connected');
325
+ resolve(true);
326
+ };
327
+
328
+ ws.onmessage = (event) => {
329
+ handleWebSocketMessage(event.data);
330
+ };
331
+
332
+ ws.onerror = (error) => {
333
+ UplinkLogger.error('Realtime: WebSocket error:', error);
334
+ if (window.UplinkDeveloper) {
335
+ window.UplinkDeveloper.logError(new Error('WebSocket error'), 'Realtime WebSocket');
336
+ }
337
+ };
338
+
339
+ ws.onclose = (event) => {
340
+ UplinkLogger.debug(`Realtime: WebSocket closed (code: ${event.code})`);
341
+ if (isActive && event.code !== WS_CLOSE_NORMAL) {
342
+ showError('Connection lost');
343
+ stop();
344
+ }
345
+ };
346
+
347
+ // Timeout after 10 seconds
348
+ setTimeout(() => {
349
+ if (ws.readyState !== WebSocket.OPEN) {
350
+ ws.close();
351
+ reject(new Error('Connection timeout'));
352
+ }
353
+ }, 10000);
354
+
355
+ } catch (err) {
356
+ reject(err);
357
+ }
358
+ });
359
+ }
360
+
361
+ function handleWebSocketMessage(data) {
362
+ try {
363
+ const msg = JSON.parse(data);
364
+
365
+ // Handle bridge events (agent mode)
366
+ if (msg.type && msg.type.startsWith('bridge.')) {
367
+ handleBridgeEvent(msg);
368
+ return;
369
+ }
370
+
371
+ // Handle OpenAI Realtime events
372
+ switch (msg.type) {
373
+ case 'session.created':
374
+ case 'session.updated':
375
+ sessionReady = true;
376
+ UplinkLogger.debug('Realtime: Session ready, mic audio enabled');
377
+ break;
378
+
379
+ case 'response.audio.delta':
380
+ // Only play OpenAI audio in standalone mode
381
+ if (currentMode === 'standalone' && msg.delta) {
382
+ playAudioChunk(msg.delta);
383
+ } else if (currentMode === 'agent' && msg.delta) {
384
+ UplinkLogger.warn('Realtime: Ignoring OpenAI audio in agent mode');
385
+ }
386
+ break;
387
+
388
+ case 'response.audio.done':
389
+ UplinkLogger.debug('Realtime: Audio response complete');
390
+ if (currentMode === 'standalone') {
391
+ updateTranscript('', true); // Clear interim transcript
392
+ }
393
+ break;
394
+
395
+ case 'response.audio_transcript.delta':
396
+ // Update live transcript (standalone mode only)
397
+ if (currentMode === 'standalone' && msg.delta) {
398
+ updateTranscript(msg.delta, false);
399
+ }
400
+ break;
401
+
402
+ case 'response.audio_transcript.done':
403
+ // Finalize transcript (standalone mode only)
404
+ if (currentMode === 'standalone' && msg.transcript) {
405
+ updateTranscript(msg.transcript, true);
406
+ }
407
+ break;
408
+
409
+ case 'response.done':
410
+ UplinkLogger.debug('Realtime: Response complete');
411
+ break;
412
+
413
+ case 'error':
414
+ UplinkLogger.error('Realtime: Server error:', msg.error);
415
+ showError(msg.error?.message || 'Server error');
416
+ break;
417
+
418
+ default:
419
+ UplinkLogger.debug('Realtime: Unhandled message type:', msg.type);
420
+ }
421
+ } catch (err) {
422
+ UplinkLogger.error('Realtime: Failed to parse message:', err);
423
+ }
424
+ }
425
+
426
+ // ============================================
427
+ // BRIDGE EVENT HANDLERS (AGENT MODE)
428
+ // ============================================
429
+
430
+ function handleBridgeEvent(msg) {
431
+ switch (msg.type) {
432
+ case 'bridge.transcript':
433
+ // User's transcribed speech
434
+ if (msg.text) {
435
+ currentUserTranscript = msg.text;
436
+ // Don't show in transcript bar — it's already posted to chat
437
+
438
+ // Save to chat history if available
439
+ if (window.UplinkChat?.addMessage) {
440
+ window.UplinkChat.addMessage(msg.text, 'user');
441
+ }
442
+
443
+ // Mute mic immediately — prevents echo from TTS playback
444
+ // and stops ghost transcripts from ambient noise during processing
445
+ muteMic();
446
+
447
+ // Show waiting status while agent processes
448
+ updateVoiceStatus('waiting');
449
+ }
450
+ break;
451
+
452
+ case 'bridge.response.delta':
453
+ // Stream agent response into chat bubble
454
+ // Server sends as msg.text, not msg.delta
455
+ if (msg.text || msg.delta) {
456
+ currentAgentResponse += (msg.text || msg.delta);
457
+
458
+ // Create streaming bubble on first delta
459
+ if (!streamingMessageDiv && window.UplinkChat?.createStreamingMessage) {
460
+ streamingMessageDiv = window.UplinkChat.createStreamingMessage();
461
+ }
462
+
463
+ // Update streaming content
464
+ if (streamingMessageDiv && window.UplinkChat?.updateStreamingMessage) {
465
+ window.UplinkChat.updateStreamingMessage(streamingMessageDiv, currentAgentResponse);
466
+ }
467
+ }
468
+ break;
469
+
470
+ case 'bridge.response.done':
471
+ // Agent response complete — finalize the streaming bubble
472
+ {
473
+ const fullResponse = msg.text || currentAgentResponse;
474
+
475
+ // Finalize streaming message with full formatted content
476
+ if (streamingMessageDiv && window.UplinkChat?.finalizeSyncStream) {
477
+ window.UplinkChat.finalizeSyncStream(streamingMessageDiv, fullResponse);
478
+ streamingMessageDiv = null;
479
+ } else if (fullResponse && window.UplinkChat?.addMessage) {
480
+ // Fallback if no streaming bubble was created
481
+ window.UplinkChat.addMessage(fullResponse, 'assistant');
482
+ }
483
+
484
+ // Unmute mic after delay to let last audio chunk finish playing
485
+ // through speakers (prevents catching tail end of TTS)
486
+ // Only switch to "listening" once mic is actually back on
487
+ setTimeout(() => {
488
+ unmuteMic();
489
+ updateVoiceStatus('listening');
490
+ }, 1500);
491
+
492
+ // Reset buffers
493
+ setTimeout(() => {
494
+ currentUserTranscript = '';
495
+ currentAgentResponse = '';
496
+ }, 1000);
497
+ }
498
+ break;
499
+
500
+ case 'bridge.audio':
501
+ // TTS audio chunk from agent response
502
+ if (msg.audio) {
503
+ // Mute mic to prevent speaker→mic echo feedback
504
+ muteMic();
505
+ // Agent is speaking — show replying status (mic still muted)
506
+ updateVoiceStatus('replying');
507
+ playAudioChunk(msg.audio);
508
+ }
509
+ break;
510
+
511
+ case 'bridge.status':
512
+ // Status updates (e.g., thinking indicator)
513
+ if (msg.status === 'thinking') {
514
+ showThinkingIndicator();
515
+ // Optional: play subtle audio cue
516
+ // playThinkingCue();
517
+ } else if (msg.status === 'speaking') {
518
+ hideThinkingIndicator();
519
+ }
520
+ break;
521
+
522
+ default:
523
+ UplinkLogger.debug('Realtime: Unhandled bridge event:', msg.type);
524
+ }
525
+ }
526
+
527
+ // ============================================
528
+ // UI UPDATES
529
+ // ============================================
530
+
531
+ function updateUI() {
532
+ const indicator = document.getElementById('realtimeIndicator');
533
+ const timer = document.getElementById('realtimeTimer');
534
+ const transcript = document.getElementById('realtimeTranscript');
535
+ const badge = document.getElementById('realtimeBadge');
536
+
537
+ if (isActive) {
538
+ if (currentMode === 'agent') {
539
+ // Agent mode: voice.js handles status label ("Listening — Steven"),
540
+ // so hide the redundant realtime indicator overlay
541
+ if (indicator) {
542
+ indicator.style.display = 'none';
543
+ indicator.classList.remove('active', 'agent-mode');
544
+ }
545
+ if (timer) timer.style.display = 'none';
546
+ if (transcript) transcript.style.display = 'none';
547
+ if (badge) badge.style.display = 'none';
548
+ } else {
549
+ // Standalone mode: show the realtime indicator overlay
550
+ if (indicator) {
551
+ indicator.style.display = 'flex';
552
+ indicator.classList.add('active');
553
+ indicator.classList.remove('agent-mode');
554
+ }
555
+
556
+ if (timer) {
557
+ timer.style.display = 'block';
558
+ timer.textContent = '0:00';
559
+ }
560
+
561
+ if (transcript) {
562
+ transcript.style.display = 'block';
563
+ transcript.textContent = '';
564
+ }
565
+
566
+ if (badge) {
567
+ badge.style.display = 'none';
568
+ }
569
+ }
570
+ } else {
571
+ if (indicator) {
572
+ indicator.style.display = 'none';
573
+ indicator.classList.remove('active', 'agent-mode');
574
+ }
575
+ if (timer) {
576
+ timer.style.display = 'none';
577
+ }
578
+ if (transcript) {
579
+ transcript.style.display = 'none';
580
+ }
581
+ if (badge) {
582
+ badge.style.display = 'none';
583
+ }
584
+ stopTimer();
585
+ }
586
+ }
587
+
588
+ function startTimer() {
589
+ stopTimer(); // Clear any existing timer
590
+ timerInterval = setInterval(() => {
591
+ if (!sessionStartTime) return;
592
+
593
+ const elapsed = Math.floor((Date.now() - sessionStartTime) / TIMER_UPDATE_INTERVAL_MS);
594
+ const timer = document.getElementById('realtimeTimer');
595
+ if (timer) {
596
+ const minutes = Math.floor(elapsed / 60);
597
+ const seconds = elapsed % 60;
598
+ timer.textContent = `${minutes}:${seconds.toString().padStart(2, '0')}`;
599
+ }
600
+ }, TIMER_UPDATE_INTERVAL_MS);
601
+ }
602
+
603
+ function stopTimer() {
604
+ if (timerInterval) {
605
+ clearInterval(timerInterval);
606
+ timerInterval = null;
607
+ }
608
+ sessionStartTime = null;
609
+ }
610
+
611
+ let transcriptBuffer = '';
612
+ let transcriptTimeout = null;
613
+
614
+ /**
615
+ * Update transcript display
616
+ * @param {string} text - Text to display
617
+ * @param {boolean} isFinal - Whether this is final (clear after delay) or interim
618
+ * @param {string} role - 'user' or 'agent' (agent mode only)
619
+ */
620
+ function updateTranscript(text, isFinal, role = null) {
621
+ const transcriptEl = document.getElementById('realtimeTranscript');
622
+ if (!transcriptEl) return;
623
+
624
+ if (isFinal) {
625
+ // For agent mode, show formatted text with role
626
+ if (currentMode === 'agent' && role) {
627
+ const prefix = role === 'user' ? 'You: ' : `${agentName}: `;
628
+ transcriptEl.textContent = prefix + text;
629
+ } else {
630
+ transcriptEl.textContent = text;
631
+ }
632
+
633
+ transcriptBuffer = text;
634
+
635
+ // Auto-clear after delay if final
636
+ if (text) {
637
+ if (transcriptTimeout) clearTimeout(transcriptTimeout);
638
+ transcriptTimeout = setTimeout(() => {
639
+ transcriptBuffer = '';
640
+ transcriptEl.textContent = '';
641
+ }, 5000);
642
+ } else {
643
+ transcriptBuffer = '';
644
+ transcriptEl.textContent = '';
645
+ }
646
+ } else {
647
+ // Interim text
648
+ if (currentMode === 'agent' && role) {
649
+ const prefix = role === 'user' ? 'You: ' : `${agentName}: `;
650
+ transcriptEl.textContent = prefix + text;
651
+ } else {
652
+ transcriptBuffer = text;
653
+ transcriptEl.textContent = text;
654
+ }
655
+
656
+ // Auto-clear after 5 seconds of no updates
657
+ if (transcriptTimeout) clearTimeout(transcriptTimeout);
658
+ transcriptTimeout = setTimeout(() => {
659
+ transcriptBuffer = '';
660
+ transcriptEl.textContent = '';
661
+ }, 5000);
662
+ }
663
+ }
664
+
665
+ // ============================================
666
+ // MIC MUTE (echo prevention during TTS playback)
667
+ // ============================================
668
+
669
+ function muteMic() {
670
+ if (isMicMuted) return;
671
+ isMicMuted = true;
672
+ if (micStream) {
673
+ micStream.getAudioTracks().forEach(track => { track.enabled = false; });
674
+ }
675
+ // Visual cue: dim the button when mic is off
676
+ const voiceBtn = document.getElementById('voiceBtn');
677
+ if (voiceBtn) voiceBtn.classList.add('mic-muted');
678
+ UplinkLogger.debug('Realtime: Mic muted (TTS playing)');
679
+ }
680
+
681
+ function unmuteMic() {
682
+ if (!isMicMuted) return;
683
+ isMicMuted = false;
684
+ if (micStream) {
685
+ micStream.getAudioTracks().forEach(track => { track.enabled = true; });
686
+ }
687
+ // Restore full brightness
688
+ const voiceBtn = document.getElementById('voiceBtn');
689
+ if (voiceBtn) voiceBtn.classList.remove('mic-muted');
690
+ UplinkLogger.debug('Realtime: Mic unmuted');
691
+ }
692
+
693
+ /**
694
+ * Update the voice status label in agent mode
695
+ * @param {string} state - 'listening' | 'waiting'
696
+ */
697
+ function updateVoiceStatus(state) {
698
+ if (currentMode !== 'agent') return;
699
+
700
+ const voiceStatus = document.getElementById('voiceStatus');
701
+ if (!voiceStatus) return;
702
+
703
+ const name = agentName || 'Agent';
704
+
705
+ switch (state) {
706
+ case 'waiting':
707
+ voiceStatus.textContent = 'Waiting for reply...';
708
+ break;
709
+ case 'replying':
710
+ voiceStatus.textContent = `${name} is replying...`;
711
+ break;
712
+ case 'listening':
713
+ default:
714
+ voiceStatus.textContent = `Listening — ${name}`;
715
+ break;
716
+ }
717
+ }
718
+
719
+ function showThinkingIndicator() {
720
+ const indicator = document.getElementById('realtimeThinking');
721
+ if (indicator) {
722
+ indicator.style.display = 'block';
723
+ indicator.textContent = '💭 thinking...';
724
+ }
725
+ }
726
+
727
+ function hideThinkingIndicator() {
728
+ const indicator = document.getElementById('realtimeThinking');
729
+ if (indicator) {
730
+ indicator.style.display = 'none';
731
+ }
732
+ }
733
+
734
+ function showError(message) {
735
+ const chat = window.UplinkChat;
736
+ if (chat?.addMessage) {
737
+ chat.addMessage(message, 'system');
738
+ }
739
+ }
740
+
741
+ // ============================================
742
+ // CLEANUP
743
+ // ============================================
744
+
745
+ function cleanup() {
746
+ // Reset session state
747
+ sessionReady = false;
748
+ isMicMuted = false;
749
+ currentUserTranscript = '';
750
+ currentAgentResponse = '';
751
+ streamingMessageDiv = null;
752
+
753
+ // Remove glow class from voice button
754
+ const voiceBtn = document.getElementById('voiceBtn');
755
+ if (voiceBtn) voiceBtn.classList.remove('realtime-active');
756
+
757
+ // Stop timer
758
+ stopTimer();
759
+
760
+ // Close WebSocket
761
+ if (ws) {
762
+ if (ws.readyState === WebSocket.OPEN) {
763
+ ws.close(WS_CLOSE_NORMAL);
764
+ }
765
+ ws = null;
766
+ }
767
+
768
+ // Stop microphone
769
+ if (audioWorkletNode) {
770
+ audioWorkletNode.disconnect();
771
+ audioWorkletNode = null;
772
+ }
773
+
774
+ if (audioContext) {
775
+ audioContext.close().catch(err =>
776
+ UplinkLogger.error('Realtime: AudioContext close failed:', err)
777
+ );
778
+ audioContext = null;
779
+ }
780
+
781
+ if (micStream) {
782
+ micStream.getTracks().forEach(track => track.stop());
783
+ micStream = null;
784
+ }
785
+
786
+ // Stop playback
787
+ if (playbackContext) {
788
+ playbackContext.close().catch(err =>
789
+ UplinkLogger.error('Realtime: Playback context close failed:', err)
790
+ );
791
+ playbackContext = null;
792
+ }
793
+
794
+ playbackQueue = [];
795
+ isPlayingAudio = false;
796
+ nextPlaybackTime = 0;
797
+ transcriptBuffer = '';
798
+
799
+ if (transcriptTimeout) {
800
+ clearTimeout(transcriptTimeout);
801
+ transcriptTimeout = null;
802
+ }
803
+
804
+ hideThinkingIndicator();
805
+ }
806
+
807
+ // ============================================
808
+ // UTILITIES
809
+ // ============================================
810
+
811
+ function arrayBufferToBase64(buffer) {
812
+ const bytes = new Uint8Array(buffer);
813
+ let binary = '';
814
+ for (let i = 0; i < bytes.byteLength; i++) {
815
+ binary += String.fromCharCode(bytes[i]);
816
+ }
817
+ return btoa(binary);
818
+ }
819
+
820
+ function base64ToArrayBuffer(base64) {
821
+ const binary = atob(base64);
822
+ const bytes = new Uint8Array(binary.length);
823
+ for (let i = 0; i < binary.length; i++) {
824
+ bytes[i] = binary.charCodeAt(i);
825
+ }
826
+ return bytes.buffer;
827
+ }
828
+
829
+ // ============================================
830
+ // PUBLIC API
831
+ // ============================================
832
+
833
+ export const UplinkRealtime = {
834
+ start,
835
+ stop,
836
+ isActive: isSessionActive,
837
+ isMuted: () => isMicMuted,
838
+ getMode
839
+ };
840
+
841
+ // Backward compat: assign to window
842
+ window.UplinkRealtime = UplinkRealtime;
843
+
844
+ UplinkLogger.debug('Realtime: Module loaded');