@nextpa/chat-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +53 -0
- package/dist/constants/defaults.d.ts +6 -0
- package/dist/constants/defaults.js +69 -0
- package/dist/core/chat-manager.d.ts +17 -0
- package/dist/core/chat-manager.js +39 -0
- package/dist/core/config-manager.d.ts +14 -0
- package/dist/core/config-manager.js +75 -0
- package/dist/core/connection-manager.d.ts +68 -0
- package/dist/core/connection-manager.js +273 -0
- package/dist/core/connection-retry-handler.d.ts +11 -0
- package/dist/core/connection-retry-handler.js +33 -0
- package/dist/core/consent-manager.d.ts +15 -0
- package/dist/core/consent-manager.js +40 -0
- package/dist/core/conversation-persistence.d.ts +23 -0
- package/dist/core/conversation-persistence.js +109 -0
- package/dist/core/event-emitter.d.ts +7 -0
- package/dist/core/event-emitter.js +27 -0
- package/dist/core/greeting-gate.d.ts +16 -0
- package/dist/core/greeting-gate.js +53 -0
- package/dist/core/index.d.ts +9 -0
- package/dist/core/index.js +7 -0
- package/dist/core/link-target.d.ts +15 -0
- package/dist/core/link-target.js +64 -0
- package/dist/core/local-storage-persistence.d.ts +11 -0
- package/dist/core/local-storage-persistence.js +111 -0
- package/dist/core/navigation-manager.d.ts +24 -0
- package/dist/core/navigation-manager.js +132 -0
- package/dist/core/popover-manager.d.ts +14 -0
- package/dist/core/popover-manager.js +36 -0
- package/dist/core/session-beacon.d.ts +11 -0
- package/dist/core/session-beacon.js +60 -0
- package/dist/core/text-connection-manager.d.ts +58 -0
- package/dist/core/text-connection-manager.js +200 -0
- package/dist/core/text-mode-manager.d.ts +50 -0
- package/dist/core/text-mode-manager.js +154 -0
- package/dist/core/text-types.d.ts +20 -0
- package/dist/core/text-types.js +1 -0
- package/dist/core/text-widget-controller.d.ts +47 -0
- package/dist/core/text-widget-controller.js +278 -0
- package/dist/core/theme-manager.d.ts +10 -0
- package/dist/core/theme-manager.js +25 -0
- package/dist/core/types.d.ts +50 -0
- package/dist/core/types.js +1 -0
- package/dist/core/voice-state-manager.d.ts +85 -0
- package/dist/core/voice-state-manager.js +317 -0
- package/dist/core/widget-controller.d.ts +58 -0
- package/dist/core/widget-controller.js +533 -0
- package/dist/core/widget-core.d.ts +51 -0
- package/dist/core/widget-core.js +160 -0
- package/dist/google.d.ts +1 -0
- package/dist/google.js +12 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.js +40 -0
- package/dist/openai.d.ts +1 -0
- package/dist/openai.js +7 -0
- package/dist/services/api.d.ts +114 -0
- package/dist/services/api.js +336 -0
- package/dist/services/index.d.ts +3 -0
- package/dist/services/index.js +3 -0
- package/dist/services/page-interaction/constants.d.ts +1 -0
- package/dist/services/page-interaction/constants.js +1 -0
- package/dist/services/page-interaction/content-extractor.d.ts +2 -0
- package/dist/services/page-interaction/content-extractor.js +104 -0
- package/dist/services/page-interaction/deep-query.d.ts +8 -0
- package/dist/services/page-interaction/deep-query.js +44 -0
- package/dist/services/page-interaction/dom-executor.d.ts +7 -0
- package/dist/services/page-interaction/dom-executor.js +172 -0
- package/dist/services/page-interaction/dom-extractor.d.ts +8 -0
- package/dist/services/page-interaction/dom-extractor.js +252 -0
- package/dist/services/page-interaction/extension-bridge.d.ts +10 -0
- package/dist/services/page-interaction/extension-bridge.js +86 -0
- package/dist/services/page-interaction/iframe-navigation-manager.d.ts +28 -0
- package/dist/services/page-interaction/iframe-navigation-manager.js +154 -0
- package/dist/services/page-interaction/index.d.ts +5 -0
- package/dist/services/page-interaction/index.js +4 -0
- package/dist/services/page-interaction/page-interaction-handler.d.ts +21 -0
- package/dist/services/page-interaction/page-interaction-handler.js +129 -0
- package/dist/services/page-interaction/selector-generator.d.ts +1 -0
- package/dist/services/page-interaction/selector-generator.js +48 -0
- package/dist/services/page-interaction/types.d.ts +75 -0
- package/dist/services/page-interaction/types.js +1 -0
- package/dist/services/realtime/google/google-audio.d.ts +44 -0
- package/dist/services/realtime/google/google-audio.js +410 -0
- package/dist/services/realtime/google/google-provider.d.ts +47 -0
- package/dist/services/realtime/google/google-provider.js +465 -0
- package/dist/services/realtime/google/google-tool-mapper.d.ts +3 -0
- package/dist/services/realtime/google/google-tool-mapper.js +70 -0
- package/dist/services/realtime/google/index.d.ts +1 -0
- package/dist/services/realtime/google/index.js +1 -0
- package/dist/services/realtime/openai/index.d.ts +1 -0
- package/dist/services/realtime/openai/index.js +1 -0
- package/dist/services/realtime/openai/openai-audio-monitor.d.ts +1 -0
- package/dist/services/realtime/openai/openai-audio-monitor.js +55 -0
- package/dist/services/realtime/openai/openai-provider.d.ts +50 -0
- package/dist/services/realtime/openai/openai-provider.js +579 -0
- package/dist/services/realtime/openai/openai-tool-mapper.d.ts +15 -0
- package/dist/services/realtime/openai/openai-tool-mapper.js +14 -0
- package/dist/services/realtime/provider-factory.d.ts +4 -0
- package/dist/services/realtime/provider-factory.js +12 -0
- package/dist/services/realtime/types.d.ts +78 -0
- package/dist/services/realtime/types.js +1 -0
- package/dist/services/realtime/vertexai/index.d.ts +1 -0
- package/dist/services/realtime/vertexai/index.js +1 -0
- package/dist/services/realtime/vertexai/vertexai-provider.d.ts +53 -0
- package/dist/services/realtime/vertexai/vertexai-provider.js +561 -0
- package/dist/services/realtime/voice-session-tracker.d.ts +10 -0
- package/dist/services/realtime/voice-session-tracker.js +62 -0
- package/dist/services/signalr.d.ts +37 -0
- package/dist/services/signalr.js +145 -0
- package/dist/services/thinking-audio-pulse.d.ts +2 -0
- package/dist/services/thinking-audio-pulse.js +72 -0
- package/dist/services/thinking-audio.d.ts +2 -0
- package/dist/services/thinking-audio.js +105 -0
- package/dist/services/voice-health-watchdog.d.ts +42 -0
- package/dist/services/voice-health-watchdog.js +90 -0
- package/dist/services/webrtc-event-handler.d.ts +4 -0
- package/dist/services/webrtc-event-handler.js +42 -0
- package/dist/services/webrtc-greeting.d.ts +3 -0
- package/dist/services/webrtc-greeting.js +62 -0
- package/dist/services/webrtc-microphone.d.ts +15 -0
- package/dist/services/webrtc-microphone.js +64 -0
- package/dist/services/webrtc.d.ts +44 -0
- package/dist/services/webrtc.js +196 -0
- package/dist/types/index.d.ts +151 -0
- package/dist/types/index.js +1 -0
- package/dist/utils/audio-context.d.ts +11 -0
- package/dist/utils/audio-context.js +33 -0
- package/dist/utils/connect-chime.d.ts +11 -0
- package/dist/utils/connect-chime.js +230 -0
- package/dist/utils/consent.d.ts +4 -0
- package/dist/utils/consent.js +30 -0
- package/dist/utils/language-detection.d.ts +1 -0
- package/dist/utils/language-detection.js +17 -0
- package/dist/utils/logger.d.ts +13 -0
- package/dist/utils/logger.js +40 -0
- package/dist/utils/mic-level-analyser.d.ts +13 -0
- package/dist/utils/mic-level-analyser.js +50 -0
- package/dist/utils/proactive-events.d.ts +5 -0
- package/dist/utils/proactive-events.js +1 -0
- package/dist/utils/styles.d.ts +4 -0
- package/dist/utils/styles.js +35 -0
- package/dist/utils/theme-detection.d.ts +3 -0
- package/dist/utils/theme-detection.js +121 -0
- package/dist/utils/tool-status.d.ts +10 -0
- package/dist/utils/tool-status.js +61 -0
- package/dist/utils/ui-sounds.d.ts +3 -0
- package/dist/utils/ui-sounds.js +57 -0
- package/package.json +56 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import * as signalR from '@microsoft/signalr';
|
|
2
|
+
import { logger } from '../utils/logger';
|
|
3
|
+
export class SignalRService {
|
|
4
|
+
connection = null;
|
|
5
|
+
events = {};
|
|
6
|
+
textChatHandlers = {};
|
|
7
|
+
pageInteractionHandlers = {};
|
|
8
|
+
registrationInfo = null;
|
|
9
|
+
get isConnected() {
|
|
10
|
+
return this.connection?.state === signalR.HubConnectionState.Connected;
|
|
11
|
+
}
|
|
12
|
+
setEventHandlers(events) {
|
|
13
|
+
this.events = events;
|
|
14
|
+
}
|
|
15
|
+
setTextChatHandlers(handlers) {
|
|
16
|
+
this.textChatHandlers = handlers;
|
|
17
|
+
}
|
|
18
|
+
clearTextChatHandlers() {
|
|
19
|
+
this.textChatHandlers = {};
|
|
20
|
+
}
|
|
21
|
+
setPageInteractionHandlers(handlers) {
|
|
22
|
+
this.pageInteractionHandlers = handlers;
|
|
23
|
+
}
|
|
24
|
+
clearPageInteractionHandlers() {
|
|
25
|
+
this.pageInteractionHandlers = {};
|
|
26
|
+
}
|
|
27
|
+
getConnection() {
|
|
28
|
+
return this.connection;
|
|
29
|
+
}
|
|
30
|
+
setRegistrationInfo(clientId, tenantId, userId) {
|
|
31
|
+
this.registrationInfo = { clientId, tenantId, userId };
|
|
32
|
+
}
|
|
33
|
+
async connect(baseUrl, token) {
|
|
34
|
+
if (this.connection) {
|
|
35
|
+
await this.disconnect();
|
|
36
|
+
}
|
|
37
|
+
const hubUrl = baseUrl.endsWith('/notificationHub')
|
|
38
|
+
? baseUrl
|
|
39
|
+
: `${baseUrl.replace(/\/$/, '')}/notificationHub`;
|
|
40
|
+
logger.debug('[SignalR] Connecting to:', hubUrl);
|
|
41
|
+
this.connection = new signalR.HubConnectionBuilder()
|
|
42
|
+
.withUrl(hubUrl, {
|
|
43
|
+
accessTokenFactory: () => token,
|
|
44
|
+
})
|
|
45
|
+
.withAutomaticReconnect([2000, 5000, 10000, 30000, 60000])
|
|
46
|
+
.configureLogging(signalR.LogLevel.Information)
|
|
47
|
+
.build();
|
|
48
|
+
this.connection.serverTimeoutInMilliseconds = 60000;
|
|
49
|
+
this.connection.keepAliveIntervalInMilliseconds = 30000;
|
|
50
|
+
this.setupEventHandlers();
|
|
51
|
+
try {
|
|
52
|
+
await this.connection.start();
|
|
53
|
+
logger.debug('[SignalR] Connected');
|
|
54
|
+
await this.registerClient();
|
|
55
|
+
this.events.onConnected?.();
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
logger.error('[SignalR] Connection failed:', error);
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async disconnect() {
|
|
63
|
+
if (this.connection) {
|
|
64
|
+
await this.connection.stop();
|
|
65
|
+
this.connection = null;
|
|
66
|
+
logger.debug('[SignalR] Disconnected');
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async registerClient() {
|
|
70
|
+
if (!this.connection || !this.registrationInfo) {
|
|
71
|
+
logger.warn('[SignalR] Cannot register client: no connection or registration info');
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
await this.connection.invoke('RegisterClient', this.registrationInfo.clientId, this.registrationInfo.tenantId, this.registrationInfo.userId, 'Widget');
|
|
76
|
+
logger.debug('[SignalR] Client registered:', this.registrationInfo.clientId);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
logger.error('[SignalR] Failed to register client:', error);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
setupEventHandlers() {
|
|
83
|
+
if (!this.connection)
|
|
84
|
+
return;
|
|
85
|
+
this.connection.on('ReceiveNotification', (type, payloadJson) => {
|
|
86
|
+
logger.debug('[SignalR] ReceiveNotification:', type);
|
|
87
|
+
this.handleNotification(type, payloadJson);
|
|
88
|
+
});
|
|
89
|
+
this.connection.onreconnecting((error) => {
|
|
90
|
+
logger.debug('[SignalR] Reconnecting...', error);
|
|
91
|
+
});
|
|
92
|
+
this.connection.onreconnected(async () => {
|
|
93
|
+
logger.debug('[SignalR] Reconnected');
|
|
94
|
+
await this.registerClient();
|
|
95
|
+
this.events.onConnected?.();
|
|
96
|
+
});
|
|
97
|
+
this.connection.onclose((error) => {
|
|
98
|
+
logger.debug('[SignalR] Connection closed', error);
|
|
99
|
+
this.events.onDisconnected?.();
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
handleNotification(type, payloadJson) {
|
|
103
|
+
try {
|
|
104
|
+
const payload = JSON.parse(payloadJson);
|
|
105
|
+
switch (type) {
|
|
106
|
+
case 'conversation.stop':
|
|
107
|
+
logger.debug('[SignalR] StopConversation received');
|
|
108
|
+
this.events.onStopConversation?.();
|
|
109
|
+
break;
|
|
110
|
+
case 'navigation.openurl':
|
|
111
|
+
logger.debug('[SignalR] OpenUrl received:', payload.url);
|
|
112
|
+
this.events.onOpenUrl?.(payload.url);
|
|
113
|
+
break;
|
|
114
|
+
case 'navigation.openfile':
|
|
115
|
+
logger.debug('[SignalR] OpenFile received:', payload.url);
|
|
116
|
+
this.events.onOpenFile?.(payload.url);
|
|
117
|
+
break;
|
|
118
|
+
case 'navigation.error':
|
|
119
|
+
logger.error('[SignalR] Error received:', payload.title, payload.message);
|
|
120
|
+
this.events.onError?.(payload.title, payload.message);
|
|
121
|
+
break;
|
|
122
|
+
case 'textchat.stream_delta':
|
|
123
|
+
this.textChatHandlers.onStreamDelta?.(payload.delta, payload.messageId);
|
|
124
|
+
break;
|
|
125
|
+
case 'textchat.stream_completed':
|
|
126
|
+
logger.debug('[SignalR] TextChat stream completed, messageId:', payload.messageId);
|
|
127
|
+
this.textChatHandlers.onStreamCompleted?.(payload.fullText, payload.messageId);
|
|
128
|
+
break;
|
|
129
|
+
case 'textchat.tool_call':
|
|
130
|
+
logger.debug('[SignalR] TextChat tool call:', payload.toolName, payload.status, payload.agentLabel);
|
|
131
|
+
this.textChatHandlers.onToolCall?.(payload.toolName, payload.status, payload.result, payload.agentLabel);
|
|
132
|
+
break;
|
|
133
|
+
case 'pageinteraction.request':
|
|
134
|
+
logger.debug('[SignalR] PageInteraction request received');
|
|
135
|
+
this.pageInteractionHandlers.onPageInteractionRequest?.(payloadJson);
|
|
136
|
+
break;
|
|
137
|
+
default:
|
|
138
|
+
logger.debug('[SignalR] Unhandled notification type:', type, payload);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
logger.error('[SignalR] Failed to parse notification:', error);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
const INITIAL_DELAY_MS = 10_000;
|
|
2
|
+
const VOLUME = 0.06;
|
|
3
|
+
const BREATH_IN_S = 0.2;
|
|
4
|
+
const BREATH_HOLD_S = 0.1;
|
|
5
|
+
const BREATH_OUT_S = 0.4;
|
|
6
|
+
const PAUSE_BETWEEN_S = 4;
|
|
7
|
+
const CHORD_FREQUENCIES = [60, 50];
|
|
8
|
+
let audioContext = null;
|
|
9
|
+
let masterGain = null;
|
|
10
|
+
let isActive = false;
|
|
11
|
+
let delayTimer = null;
|
|
12
|
+
let breathTimer = null;
|
|
13
|
+
function playBreath() {
|
|
14
|
+
if (!audioContext || !masterGain || !isActive)
|
|
15
|
+
return;
|
|
16
|
+
const now = audioContext.currentTime;
|
|
17
|
+
const breathEnvelope = audioContext.createGain();
|
|
18
|
+
breathEnvelope.gain.setValueAtTime(0, now);
|
|
19
|
+
breathEnvelope.gain.linearRampToValueAtTime(1, now + BREATH_IN_S);
|
|
20
|
+
breathEnvelope.gain.setValueAtTime(1, now + BREATH_IN_S + BREATH_HOLD_S);
|
|
21
|
+
breathEnvelope.gain.linearRampToValueAtTime(0, now + BREATH_IN_S + BREATH_HOLD_S + BREATH_OUT_S);
|
|
22
|
+
const totalDuration = BREATH_IN_S + BREATH_HOLD_S + BREATH_OUT_S;
|
|
23
|
+
for (const freq of CHORD_FREQUENCIES) {
|
|
24
|
+
const osc = audioContext.createOscillator();
|
|
25
|
+
osc.type = 'sine';
|
|
26
|
+
osc.frequency.value = freq;
|
|
27
|
+
osc.connect(breathEnvelope);
|
|
28
|
+
osc.start(now);
|
|
29
|
+
osc.stop(now + totalDuration + 0.1);
|
|
30
|
+
}
|
|
31
|
+
breathEnvelope.connect(masterGain);
|
|
32
|
+
const nextBreathMs = (totalDuration + PAUSE_BETWEEN_S) * 1000;
|
|
33
|
+
breathTimer = setTimeout(() => playBreath(), nextBreathMs);
|
|
34
|
+
}
|
|
35
|
+
export function startThinkingAudio() {
|
|
36
|
+
if (isActive)
|
|
37
|
+
return;
|
|
38
|
+
isActive = true;
|
|
39
|
+
delayTimer = setTimeout(() => {
|
|
40
|
+
if (!isActive)
|
|
41
|
+
return;
|
|
42
|
+
audioContext = new AudioContext();
|
|
43
|
+
masterGain = audioContext.createGain();
|
|
44
|
+
masterGain.gain.value = VOLUME;
|
|
45
|
+
masterGain.connect(audioContext.destination);
|
|
46
|
+
if (audioContext.state === 'suspended') {
|
|
47
|
+
void audioContext.resume();
|
|
48
|
+
}
|
|
49
|
+
playBreath();
|
|
50
|
+
}, INITIAL_DELAY_MS);
|
|
51
|
+
}
|
|
52
|
+
export function stopThinkingAudio() {
|
|
53
|
+
isActive = false;
|
|
54
|
+
if (delayTimer) {
|
|
55
|
+
clearTimeout(delayTimer);
|
|
56
|
+
delayTimer = null;
|
|
57
|
+
}
|
|
58
|
+
if (breathTimer) {
|
|
59
|
+
clearTimeout(breathTimer);
|
|
60
|
+
breathTimer = null;
|
|
61
|
+
}
|
|
62
|
+
if (masterGain && audioContext) {
|
|
63
|
+
masterGain.gain.linearRampToValueAtTime(0, audioContext.currentTime + 0.3);
|
|
64
|
+
}
|
|
65
|
+
setTimeout(() => {
|
|
66
|
+
if (audioContext) {
|
|
67
|
+
void audioContext.close();
|
|
68
|
+
audioContext = null;
|
|
69
|
+
masterGain = null;
|
|
70
|
+
}
|
|
71
|
+
}, 400);
|
|
72
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// --- Timing ---
|
|
2
|
+
const INITIAL_DELAY_MS = 10_000;
|
|
3
|
+
const MIN_KEY_INTERVAL_MS = 60;
|
|
4
|
+
const MAX_KEY_INTERVAL_MS = 180;
|
|
5
|
+
const MIN_BURST_KEYS = 8;
|
|
6
|
+
const MAX_BURST_KEYS = 25;
|
|
7
|
+
const MIN_PAUSE_MS = 3_000;
|
|
8
|
+
const MAX_PAUSE_MS = 8_000;
|
|
9
|
+
// --- Sound shaping ---
|
|
10
|
+
const CLICK_DURATION_S = 0.012;
|
|
11
|
+
const CLICK_VOLUME = 0.08;
|
|
12
|
+
const NOISE_BUFFER_SIZE = 2048;
|
|
13
|
+
let audioContext = null;
|
|
14
|
+
let masterGain = null;
|
|
15
|
+
let noiseBuffer = null;
|
|
16
|
+
let isActive = false;
|
|
17
|
+
const pendingTimers = [];
|
|
18
|
+
function randomBetween(min, max) {
|
|
19
|
+
return min + Math.random() * (max - min);
|
|
20
|
+
}
|
|
21
|
+
function addTimer(callback, delayMs) {
|
|
22
|
+
const timer = setTimeout(callback, delayMs);
|
|
23
|
+
pendingTimers.push(timer);
|
|
24
|
+
}
|
|
25
|
+
function clearAllTimers() {
|
|
26
|
+
for (const timer of pendingTimers) {
|
|
27
|
+
clearTimeout(timer);
|
|
28
|
+
}
|
|
29
|
+
pendingTimers.length = 0;
|
|
30
|
+
}
|
|
31
|
+
function createNoiseBuffer(ctx) {
|
|
32
|
+
const buffer = ctx.createBuffer(1, NOISE_BUFFER_SIZE, ctx.sampleRate);
|
|
33
|
+
const data = buffer.getChannelData(0);
|
|
34
|
+
for (let i = 0; i < NOISE_BUFFER_SIZE; i++) {
|
|
35
|
+
data[i] = Math.random() * 2 - 1;
|
|
36
|
+
}
|
|
37
|
+
return buffer;
|
|
38
|
+
}
|
|
39
|
+
function playKeystroke() {
|
|
40
|
+
if (!audioContext || !masterGain || !noiseBuffer)
|
|
41
|
+
return;
|
|
42
|
+
const now = audioContext.currentTime;
|
|
43
|
+
const source = audioContext.createBufferSource();
|
|
44
|
+
source.buffer = noiseBuffer;
|
|
45
|
+
const bandpass = audioContext.createBiquadFilter();
|
|
46
|
+
bandpass.type = 'bandpass';
|
|
47
|
+
bandpass.frequency.value = 2000 + Math.random() * 2000;
|
|
48
|
+
bandpass.Q.value = 1.5 + Math.random();
|
|
49
|
+
const envelope = audioContext.createGain();
|
|
50
|
+
envelope.gain.setValueAtTime(0, now);
|
|
51
|
+
envelope.gain.linearRampToValueAtTime(0.6 + Math.random() * 0.4, now + 0.001);
|
|
52
|
+
envelope.gain.exponentialRampToValueAtTime(0.001, now + CLICK_DURATION_S);
|
|
53
|
+
source.connect(bandpass);
|
|
54
|
+
bandpass.connect(envelope);
|
|
55
|
+
envelope.connect(masterGain);
|
|
56
|
+
source.start(now);
|
|
57
|
+
source.stop(now + CLICK_DURATION_S);
|
|
58
|
+
}
|
|
59
|
+
function scheduleTypingBurst() {
|
|
60
|
+
if (!isActive)
|
|
61
|
+
return;
|
|
62
|
+
const keyCount = Math.floor(randomBetween(MIN_BURST_KEYS, MAX_BURST_KEYS));
|
|
63
|
+
let delay = 0;
|
|
64
|
+
for (let i = 0; i < keyCount; i++) {
|
|
65
|
+
const keyDelay = delay;
|
|
66
|
+
addTimer(() => {
|
|
67
|
+
if (!isActive)
|
|
68
|
+
return;
|
|
69
|
+
playKeystroke();
|
|
70
|
+
}, keyDelay);
|
|
71
|
+
delay += randomBetween(MIN_KEY_INTERVAL_MS, MAX_KEY_INTERVAL_MS);
|
|
72
|
+
}
|
|
73
|
+
const pauseAfterBurst = delay + randomBetween(MIN_PAUSE_MS, MAX_PAUSE_MS);
|
|
74
|
+
addTimer(() => scheduleTypingBurst(), pauseAfterBurst);
|
|
75
|
+
}
|
|
76
|
+
export function startThinkingAudio() {
|
|
77
|
+
if (isActive)
|
|
78
|
+
return;
|
|
79
|
+
isActive = true;
|
|
80
|
+
addTimer(() => {
|
|
81
|
+
if (!isActive)
|
|
82
|
+
return;
|
|
83
|
+
audioContext = new AudioContext();
|
|
84
|
+
masterGain = audioContext.createGain();
|
|
85
|
+
masterGain.gain.value = CLICK_VOLUME;
|
|
86
|
+
masterGain.connect(audioContext.destination);
|
|
87
|
+
noiseBuffer = createNoiseBuffer(audioContext);
|
|
88
|
+
scheduleTypingBurst();
|
|
89
|
+
}, INITIAL_DELAY_MS);
|
|
90
|
+
}
|
|
91
|
+
export function stopThinkingAudio() {
|
|
92
|
+
isActive = false;
|
|
93
|
+
clearAllTimers();
|
|
94
|
+
if (masterGain && audioContext) {
|
|
95
|
+
masterGain.gain.linearRampToValueAtTime(0, audioContext.currentTime + 0.05);
|
|
96
|
+
}
|
|
97
|
+
setTimeout(() => {
|
|
98
|
+
if (audioContext) {
|
|
99
|
+
void audioContext.close();
|
|
100
|
+
audioContext = null;
|
|
101
|
+
masterGain = null;
|
|
102
|
+
noiseBuffer = null;
|
|
103
|
+
}
|
|
104
|
+
}, 150);
|
|
105
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { RealtimeProviderClient } from './realtime/types';
|
|
2
|
+
export type VoiceUnhealthyReason = 'no-setup' | 'no-greeting-response';
|
|
3
|
+
/**
|
|
4
|
+
* Watches a freshly-connected realtime voice session and reports when it is
|
|
5
|
+
* "connected but dead": the provider socket is up, yet the session never
|
|
6
|
+
* reaches a working state. The intermittent suspended-AudioContext / lost-setup
|
|
7
|
+
* races surface exactly like this — no disconnect event fires, so the existing
|
|
8
|
+
* unexpected-disconnect reconnect path never triggers and the user sees a mute
|
|
9
|
+
* assistant + unrecognised speech until a manual reset.
|
|
10
|
+
*
|
|
11
|
+
* Two phases after connect:
|
|
12
|
+
* 1. setup — expect `sessionReady` (provider setup complete) within
|
|
13
|
+
* SETUP_TIMEOUT_MS, else trip `no-setup`.
|
|
14
|
+
* 2. greeting — only when a greeting is expected: after setup, expect a sign
|
|
15
|
+
* the model actually started responding (`assistantTranscriptDelta` OR
|
|
16
|
+
* `audioDone`) within GREETING_RESPONSE_TIMEOUT_MS, else trip
|
|
17
|
+
* `no-greeting-response`. Keyed on response *start*, not finish, so a long
|
|
18
|
+
* greeting never trips it.
|
|
19
|
+
*
|
|
20
|
+
* On a trip it calls `onUnhealthy(reason)`; the owner recovers by tearing the
|
|
21
|
+
* session down and reconnecting (like a manual reset). It calls `onHealthy()`
|
|
22
|
+
* once the session proves alive so the owner can reset its recovery budget.
|
|
23
|
+
* The watchdog never closes the connection itself — it only observes + reports.
|
|
24
|
+
*/
|
|
25
|
+
export declare class VoiceHealthWatchdog {
|
|
26
|
+
private readonly client;
|
|
27
|
+
private readonly expectGreeting;
|
|
28
|
+
private readonly onUnhealthy;
|
|
29
|
+
private readonly onHealthy;
|
|
30
|
+
private setupTimer;
|
|
31
|
+
private greetingTimer;
|
|
32
|
+
private readonly unsubscribers;
|
|
33
|
+
private settled;
|
|
34
|
+
constructor(client: RealtimeProviderClient, expectGreeting: boolean, onUnhealthy: (reason: VoiceUnhealthyReason) => void, onHealthy: () => void);
|
|
35
|
+
start(): void;
|
|
36
|
+
stop(): void;
|
|
37
|
+
private handleSessionReady;
|
|
38
|
+
private markHealthy;
|
|
39
|
+
private trip;
|
|
40
|
+
private clearTimers;
|
|
41
|
+
private clearTimer;
|
|
42
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { logger } from '../utils/logger';
|
|
2
|
+
const SETUP_TIMEOUT_MS = 6000;
|
|
3
|
+
const GREETING_RESPONSE_TIMEOUT_MS = 8000;
|
|
4
|
+
/**
|
|
5
|
+
* Watches a freshly-connected realtime voice session and reports when it is
|
|
6
|
+
* "connected but dead": the provider socket is up, yet the session never
|
|
7
|
+
* reaches a working state. The intermittent suspended-AudioContext / lost-setup
|
|
8
|
+
* races surface exactly like this — no disconnect event fires, so the existing
|
|
9
|
+
* unexpected-disconnect reconnect path never triggers and the user sees a mute
|
|
10
|
+
* assistant + unrecognised speech until a manual reset.
|
|
11
|
+
*
|
|
12
|
+
* Two phases after connect:
|
|
13
|
+
* 1. setup — expect `sessionReady` (provider setup complete) within
|
|
14
|
+
* SETUP_TIMEOUT_MS, else trip `no-setup`.
|
|
15
|
+
* 2. greeting — only when a greeting is expected: after setup, expect a sign
|
|
16
|
+
* the model actually started responding (`assistantTranscriptDelta` OR
|
|
17
|
+
* `audioDone`) within GREETING_RESPONSE_TIMEOUT_MS, else trip
|
|
18
|
+
* `no-greeting-response`. Keyed on response *start*, not finish, so a long
|
|
19
|
+
* greeting never trips it.
|
|
20
|
+
*
|
|
21
|
+
* On a trip it calls `onUnhealthy(reason)`; the owner recovers by tearing the
|
|
22
|
+
* session down and reconnecting (like a manual reset). It calls `onHealthy()`
|
|
23
|
+
* once the session proves alive so the owner can reset its recovery budget.
|
|
24
|
+
* The watchdog never closes the connection itself — it only observes + reports.
|
|
25
|
+
*/
|
|
26
|
+
export class VoiceHealthWatchdog {
|
|
27
|
+
client;
|
|
28
|
+
expectGreeting;
|
|
29
|
+
onUnhealthy;
|
|
30
|
+
onHealthy;
|
|
31
|
+
setupTimer = null;
|
|
32
|
+
greetingTimer = null;
|
|
33
|
+
unsubscribers = [];
|
|
34
|
+
settled = false;
|
|
35
|
+
constructor(client, expectGreeting, onUnhealthy, onHealthy) {
|
|
36
|
+
this.client = client;
|
|
37
|
+
this.expectGreeting = expectGreeting;
|
|
38
|
+
this.onUnhealthy = onUnhealthy;
|
|
39
|
+
this.onHealthy = onHealthy;
|
|
40
|
+
}
|
|
41
|
+
start() {
|
|
42
|
+
this.unsubscribers.push(this.client.on('sessionReady', () => this.handleSessionReady()), this.client.on('assistantTranscriptDelta', () => this.markHealthy()), this.client.on('audioDone', () => this.markHealthy()), this.client.on('connectionStateChanged', (state) => {
|
|
43
|
+
if (state === 'disconnected')
|
|
44
|
+
this.stop();
|
|
45
|
+
}));
|
|
46
|
+
this.setupTimer = setTimeout(() => this.trip('no-setup'), SETUP_TIMEOUT_MS);
|
|
47
|
+
}
|
|
48
|
+
stop() {
|
|
49
|
+
this.settled = true;
|
|
50
|
+
this.clearTimers();
|
|
51
|
+
for (const unsubscribe of this.unsubscribers)
|
|
52
|
+
unsubscribe();
|
|
53
|
+
this.unsubscribers.length = 0;
|
|
54
|
+
}
|
|
55
|
+
handleSessionReady() {
|
|
56
|
+
if (this.settled)
|
|
57
|
+
return;
|
|
58
|
+
this.clearTimer('setupTimer');
|
|
59
|
+
if (!this.expectGreeting) {
|
|
60
|
+
this.markHealthy();
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
this.greetingTimer = setTimeout(() => this.trip('no-greeting-response'), GREETING_RESPONSE_TIMEOUT_MS);
|
|
64
|
+
}
|
|
65
|
+
markHealthy() {
|
|
66
|
+
if (this.settled)
|
|
67
|
+
return;
|
|
68
|
+
logger.debug('[VoiceWatchdog] voice session healthy');
|
|
69
|
+
this.stop();
|
|
70
|
+
this.onHealthy();
|
|
71
|
+
}
|
|
72
|
+
trip(reason) {
|
|
73
|
+
if (this.settled)
|
|
74
|
+
return;
|
|
75
|
+
logger.warn('[VoiceWatchdog] voice session unhealthy:', reason, '— requesting recovery');
|
|
76
|
+
this.stop();
|
|
77
|
+
this.onUnhealthy(reason);
|
|
78
|
+
}
|
|
79
|
+
clearTimers() {
|
|
80
|
+
this.clearTimer('setupTimer');
|
|
81
|
+
this.clearTimer('greetingTimer');
|
|
82
|
+
}
|
|
83
|
+
clearTimer(which) {
|
|
84
|
+
const timer = this[which];
|
|
85
|
+
if (timer) {
|
|
86
|
+
clearTimeout(timer);
|
|
87
|
+
this[which] = null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { RealtimeProviderClient, ProviderUsage } from './realtime/types';
|
|
2
|
+
import type { ConnectionState, ConversationEvents } from '../types';
|
|
3
|
+
import type { MicrophoneManager } from './webrtc-microphone';
|
|
4
|
+
export declare function setupEventHandlers(client: RealtimeProviderClient, events: ConversationEvents, microphoneManager: MicrophoneManager, setConnectionState: (state: ConnectionState) => void, onUsageUpdated?: (usage: ProviderUsage) => void): void;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export function setupEventHandlers(client, events, microphoneManager, setConnectionState, onUsageUpdated) {
|
|
2
|
+
client.on('connectionStateChanged', setConnectionState);
|
|
3
|
+
client.on('userSpeechStarted', () => {
|
|
4
|
+
if (microphoneManager.isReady)
|
|
5
|
+
events.onUserSpeechStart?.();
|
|
6
|
+
});
|
|
7
|
+
client.on('userSpeechStopped', () => {
|
|
8
|
+
if (microphoneManager.isReady)
|
|
9
|
+
events.onUserSpeechStop?.();
|
|
10
|
+
});
|
|
11
|
+
client.on('userSpeechEnded', () => {
|
|
12
|
+
if (microphoneManager.isReady)
|
|
13
|
+
events.onUserSpeechEnd?.();
|
|
14
|
+
});
|
|
15
|
+
client.on('userAudioLevel', (level) => {
|
|
16
|
+
if (microphoneManager.isReady)
|
|
17
|
+
events.onAudioLevel?.(level);
|
|
18
|
+
});
|
|
19
|
+
client.on('userTranscriptCompleted', (transcript) => {
|
|
20
|
+
if (microphoneManager.isReady)
|
|
21
|
+
events.onTranscriptReceived?.(true, transcript);
|
|
22
|
+
});
|
|
23
|
+
client.on('userTranscriptDelta', (delta) => {
|
|
24
|
+
if (microphoneManager.isReady)
|
|
25
|
+
events.onTranscriptDelta?.(true, delta);
|
|
26
|
+
});
|
|
27
|
+
client.on('assistantTranscriptDelta', (delta) => {
|
|
28
|
+
events.onTranscriptDelta?.(false, delta);
|
|
29
|
+
});
|
|
30
|
+
client.on('assistantTranscriptDone', (transcript) => {
|
|
31
|
+
events.onTranscriptReceived?.(false, transcript);
|
|
32
|
+
});
|
|
33
|
+
client.on('functionCall', (call) => {
|
|
34
|
+
events.onFunctionCall?.(call);
|
|
35
|
+
});
|
|
36
|
+
client.on('usageUpdated', (usage) => {
|
|
37
|
+
onUsageUpdated?.(usage);
|
|
38
|
+
});
|
|
39
|
+
client.on('error', (error) => {
|
|
40
|
+
events.onError?.(error.message);
|
|
41
|
+
});
|
|
42
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { RealtimeProviderClient } from './realtime/types';
|
|
2
|
+
import type { MicrophoneManager } from './webrtc-microphone';
|
|
3
|
+
export declare function startGreetingFlow(client: RealtimeProviderClient, microphoneManager: MicrophoneManager, onMicrophoneReady: (() => void) | undefined, skipGreeting?: boolean): void;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { logger } from '../utils/logger';
|
|
2
|
+
const GREETING_TIMEOUT_MS = 30000;
|
|
3
|
+
export function startGreetingFlow(client, microphoneManager, onMicrophoneReady, skipGreeting = false) {
|
|
4
|
+
// Only bail when the connection genuinely failed. A successful connect can
|
|
5
|
+
// still report 'connecting' here for a beat: the Vertex provider resolves
|
|
6
|
+
// connect() after audio capture starts, but flips to 'connected' only in
|
|
7
|
+
// ws.onopen — which can land after this runs. requestResponse() queues
|
|
8
|
+
// internally until the session is ready, so requesting now is safe; bailing
|
|
9
|
+
// on 'connecting' drops the greeting entirely (assistant stays mute).
|
|
10
|
+
if (client.status === 'disconnected') {
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
const enableMic = () => {
|
|
14
|
+
microphoneManager.enable(client);
|
|
15
|
+
onMicrophoneReady?.();
|
|
16
|
+
};
|
|
17
|
+
if (skipGreeting) {
|
|
18
|
+
logger.debug('[WebRTC] Resumed conversation — skipping greeting, enabling microphone immediately');
|
|
19
|
+
enableMic();
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
let cleaned = false;
|
|
23
|
+
const cleanupFns = [];
|
|
24
|
+
const cleanup = () => {
|
|
25
|
+
if (cleaned)
|
|
26
|
+
return;
|
|
27
|
+
cleaned = true;
|
|
28
|
+
for (const fn of cleanupFns)
|
|
29
|
+
fn();
|
|
30
|
+
};
|
|
31
|
+
cleanupFns.push(client.on('audioDone', () => {
|
|
32
|
+
logger.debug('[WebRTC] Greeting audioDone received, monitoring audio playback...');
|
|
33
|
+
cleanup();
|
|
34
|
+
client.waitForAudioPlaybackEnd()
|
|
35
|
+
.then((reason) => {
|
|
36
|
+
logger.debug(`[WebRTC] Greeting playback complete (${reason})`);
|
|
37
|
+
enableMic();
|
|
38
|
+
})
|
|
39
|
+
.catch(() => {
|
|
40
|
+
logger.debug('[WebRTC] Greeting audio monitoring unavailable, enabling microphone');
|
|
41
|
+
enableMic();
|
|
42
|
+
});
|
|
43
|
+
}), client.on('error', (error) => {
|
|
44
|
+
if (error.code === 'conversation_already_has_active_response') {
|
|
45
|
+
logger.debug('[WebRTC] Active response already exists, waiting for greeting');
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
logger.debug('[WebRTC] Error during greeting, enabling microphone immediately');
|
|
49
|
+
cleanup();
|
|
50
|
+
enableMic();
|
|
51
|
+
}
|
|
52
|
+
}));
|
|
53
|
+
client.requestResponse();
|
|
54
|
+
logger.debug('[WebRTC] Greeting response requested');
|
|
55
|
+
setTimeout(() => {
|
|
56
|
+
cleanup();
|
|
57
|
+
if (!microphoneManager.isReady && client.status === 'connected') {
|
|
58
|
+
logger.debug('[WebRTC] Greeting timeout, enabling microphone anyway');
|
|
59
|
+
enableMic();
|
|
60
|
+
}
|
|
61
|
+
}, GREETING_TIMEOUT_MS);
|
|
62
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { RealtimeProviderClient } from './realtime/types';
|
|
2
|
+
export declare class MicrophoneManager {
|
|
3
|
+
private _isReady;
|
|
4
|
+
private pendingMuteState;
|
|
5
|
+
private audioDeviceChanged;
|
|
6
|
+
private readonly onDeviceChange;
|
|
7
|
+
get isReady(): boolean;
|
|
8
|
+
requestPermission(): Promise<'granted' | 'denied' | 'prompt'>;
|
|
9
|
+
startDeviceChangeDetection(): void;
|
|
10
|
+
stopDeviceChangeDetection(): void;
|
|
11
|
+
setMuted(muted: boolean, client: RealtimeProviderClient): void;
|
|
12
|
+
enable(client: RealtimeProviderClient): void;
|
|
13
|
+
initPendingUnmute(): void;
|
|
14
|
+
reset(): void;
|
|
15
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { logger } from '../utils/logger';
|
|
2
|
+
export class MicrophoneManager {
|
|
3
|
+
_isReady = false;
|
|
4
|
+
pendingMuteState = null;
|
|
5
|
+
audioDeviceChanged = false;
|
|
6
|
+
onDeviceChange = () => {
|
|
7
|
+
this.audioDeviceChanged = true;
|
|
8
|
+
logger.debug('[WebRTC] Audio device change detected');
|
|
9
|
+
};
|
|
10
|
+
get isReady() {
|
|
11
|
+
return this._isReady;
|
|
12
|
+
}
|
|
13
|
+
async requestPermission() {
|
|
14
|
+
try {
|
|
15
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
16
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
17
|
+
logger.debug('[WebRTC] Microphone permission granted');
|
|
18
|
+
return 'granted';
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
logger.warn('[WebRTC] Microphone permission denied:', err);
|
|
22
|
+
return 'denied';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
startDeviceChangeDetection() {
|
|
26
|
+
navigator.mediaDevices.addEventListener('devicechange', this.onDeviceChange);
|
|
27
|
+
}
|
|
28
|
+
stopDeviceChangeDetection() {
|
|
29
|
+
navigator.mediaDevices.removeEventListener('devicechange', this.onDeviceChange);
|
|
30
|
+
}
|
|
31
|
+
setMuted(muted, client) {
|
|
32
|
+
if (this.pendingMuteState !== null) {
|
|
33
|
+
this.pendingMuteState = muted;
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
if (muted || !this.audioDeviceChanged) {
|
|
37
|
+
client.mute(muted);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
this.audioDeviceChanged = false;
|
|
41
|
+
client.refreshAudioDevice()
|
|
42
|
+
.catch(() => { })
|
|
43
|
+
.then(() => client.mute(muted));
|
|
44
|
+
}
|
|
45
|
+
enable(client) {
|
|
46
|
+
if (client.status !== 'connected') {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
const shouldBeMuted = this.pendingMuteState ?? false;
|
|
50
|
+
client.mute(shouldBeMuted);
|
|
51
|
+
this.pendingMuteState = null;
|
|
52
|
+
this._isReady = true;
|
|
53
|
+
logger.debug('[WebRTC] Microphone ready');
|
|
54
|
+
}
|
|
55
|
+
initPendingUnmute() {
|
|
56
|
+
this.pendingMuteState = false;
|
|
57
|
+
}
|
|
58
|
+
reset() {
|
|
59
|
+
this.pendingMuteState = null;
|
|
60
|
+
this._isReady = false;
|
|
61
|
+
this.audioDeviceChanged = false;
|
|
62
|
+
this.stopDeviceChangeDetection();
|
|
63
|
+
}
|
|
64
|
+
}
|