@nextpa/chat-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +53 -0
- package/dist/constants/defaults.d.ts +6 -0
- package/dist/constants/defaults.js +69 -0
- package/dist/core/chat-manager.d.ts +17 -0
- package/dist/core/chat-manager.js +39 -0
- package/dist/core/config-manager.d.ts +14 -0
- package/dist/core/config-manager.js +75 -0
- package/dist/core/connection-manager.d.ts +68 -0
- package/dist/core/connection-manager.js +273 -0
- package/dist/core/connection-retry-handler.d.ts +11 -0
- package/dist/core/connection-retry-handler.js +33 -0
- package/dist/core/consent-manager.d.ts +15 -0
- package/dist/core/consent-manager.js +40 -0
- package/dist/core/conversation-persistence.d.ts +23 -0
- package/dist/core/conversation-persistence.js +109 -0
- package/dist/core/event-emitter.d.ts +7 -0
- package/dist/core/event-emitter.js +27 -0
- package/dist/core/greeting-gate.d.ts +16 -0
- package/dist/core/greeting-gate.js +53 -0
- package/dist/core/index.d.ts +9 -0
- package/dist/core/index.js +7 -0
- package/dist/core/link-target.d.ts +15 -0
- package/dist/core/link-target.js +64 -0
- package/dist/core/local-storage-persistence.d.ts +11 -0
- package/dist/core/local-storage-persistence.js +111 -0
- package/dist/core/navigation-manager.d.ts +24 -0
- package/dist/core/navigation-manager.js +132 -0
- package/dist/core/popover-manager.d.ts +14 -0
- package/dist/core/popover-manager.js +36 -0
- package/dist/core/session-beacon.d.ts +11 -0
- package/dist/core/session-beacon.js +60 -0
- package/dist/core/text-connection-manager.d.ts +58 -0
- package/dist/core/text-connection-manager.js +200 -0
- package/dist/core/text-mode-manager.d.ts +50 -0
- package/dist/core/text-mode-manager.js +154 -0
- package/dist/core/text-types.d.ts +20 -0
- package/dist/core/text-types.js +1 -0
- package/dist/core/text-widget-controller.d.ts +47 -0
- package/dist/core/text-widget-controller.js +278 -0
- package/dist/core/theme-manager.d.ts +10 -0
- package/dist/core/theme-manager.js +25 -0
- package/dist/core/types.d.ts +50 -0
- package/dist/core/types.js +1 -0
- package/dist/core/voice-state-manager.d.ts +85 -0
- package/dist/core/voice-state-manager.js +317 -0
- package/dist/core/widget-controller.d.ts +58 -0
- package/dist/core/widget-controller.js +533 -0
- package/dist/core/widget-core.d.ts +51 -0
- package/dist/core/widget-core.js +160 -0
- package/dist/google.d.ts +1 -0
- package/dist/google.js +12 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.js +40 -0
- package/dist/openai.d.ts +1 -0
- package/dist/openai.js +7 -0
- package/dist/services/api.d.ts +114 -0
- package/dist/services/api.js +336 -0
- package/dist/services/index.d.ts +3 -0
- package/dist/services/index.js +3 -0
- package/dist/services/page-interaction/constants.d.ts +1 -0
- package/dist/services/page-interaction/constants.js +1 -0
- package/dist/services/page-interaction/content-extractor.d.ts +2 -0
- package/dist/services/page-interaction/content-extractor.js +104 -0
- package/dist/services/page-interaction/deep-query.d.ts +8 -0
- package/dist/services/page-interaction/deep-query.js +44 -0
- package/dist/services/page-interaction/dom-executor.d.ts +7 -0
- package/dist/services/page-interaction/dom-executor.js +172 -0
- package/dist/services/page-interaction/dom-extractor.d.ts +8 -0
- package/dist/services/page-interaction/dom-extractor.js +252 -0
- package/dist/services/page-interaction/extension-bridge.d.ts +10 -0
- package/dist/services/page-interaction/extension-bridge.js +86 -0
- package/dist/services/page-interaction/iframe-navigation-manager.d.ts +28 -0
- package/dist/services/page-interaction/iframe-navigation-manager.js +154 -0
- package/dist/services/page-interaction/index.d.ts +5 -0
- package/dist/services/page-interaction/index.js +4 -0
- package/dist/services/page-interaction/page-interaction-handler.d.ts +21 -0
- package/dist/services/page-interaction/page-interaction-handler.js +129 -0
- package/dist/services/page-interaction/selector-generator.d.ts +1 -0
- package/dist/services/page-interaction/selector-generator.js +48 -0
- package/dist/services/page-interaction/types.d.ts +75 -0
- package/dist/services/page-interaction/types.js +1 -0
- package/dist/services/realtime/google/google-audio.d.ts +44 -0
- package/dist/services/realtime/google/google-audio.js +410 -0
- package/dist/services/realtime/google/google-provider.d.ts +47 -0
- package/dist/services/realtime/google/google-provider.js +465 -0
- package/dist/services/realtime/google/google-tool-mapper.d.ts +3 -0
- package/dist/services/realtime/google/google-tool-mapper.js +70 -0
- package/dist/services/realtime/google/index.d.ts +1 -0
- package/dist/services/realtime/google/index.js +1 -0
- package/dist/services/realtime/openai/index.d.ts +1 -0
- package/dist/services/realtime/openai/index.js +1 -0
- package/dist/services/realtime/openai/openai-audio-monitor.d.ts +1 -0
- package/dist/services/realtime/openai/openai-audio-monitor.js +55 -0
- package/dist/services/realtime/openai/openai-provider.d.ts +50 -0
- package/dist/services/realtime/openai/openai-provider.js +579 -0
- package/dist/services/realtime/openai/openai-tool-mapper.d.ts +15 -0
- package/dist/services/realtime/openai/openai-tool-mapper.js +14 -0
- package/dist/services/realtime/provider-factory.d.ts +4 -0
- package/dist/services/realtime/provider-factory.js +12 -0
- package/dist/services/realtime/types.d.ts +78 -0
- package/dist/services/realtime/types.js +1 -0
- package/dist/services/realtime/vertexai/index.d.ts +1 -0
- package/dist/services/realtime/vertexai/index.js +1 -0
- package/dist/services/realtime/vertexai/vertexai-provider.d.ts +53 -0
- package/dist/services/realtime/vertexai/vertexai-provider.js +561 -0
- package/dist/services/realtime/voice-session-tracker.d.ts +10 -0
- package/dist/services/realtime/voice-session-tracker.js +62 -0
- package/dist/services/signalr.d.ts +37 -0
- package/dist/services/signalr.js +145 -0
- package/dist/services/thinking-audio-pulse.d.ts +2 -0
- package/dist/services/thinking-audio-pulse.js +72 -0
- package/dist/services/thinking-audio.d.ts +2 -0
- package/dist/services/thinking-audio.js +105 -0
- package/dist/services/voice-health-watchdog.d.ts +42 -0
- package/dist/services/voice-health-watchdog.js +90 -0
- package/dist/services/webrtc-event-handler.d.ts +4 -0
- package/dist/services/webrtc-event-handler.js +42 -0
- package/dist/services/webrtc-greeting.d.ts +3 -0
- package/dist/services/webrtc-greeting.js +62 -0
- package/dist/services/webrtc-microphone.d.ts +15 -0
- package/dist/services/webrtc-microphone.js +64 -0
- package/dist/services/webrtc.d.ts +44 -0
- package/dist/services/webrtc.js +196 -0
- package/dist/types/index.d.ts +151 -0
- package/dist/types/index.js +1 -0
- package/dist/utils/audio-context.d.ts +11 -0
- package/dist/utils/audio-context.js +33 -0
- package/dist/utils/connect-chime.d.ts +11 -0
- package/dist/utils/connect-chime.js +230 -0
- package/dist/utils/consent.d.ts +4 -0
- package/dist/utils/consent.js +30 -0
- package/dist/utils/language-detection.d.ts +1 -0
- package/dist/utils/language-detection.js +17 -0
- package/dist/utils/logger.d.ts +13 -0
- package/dist/utils/logger.js +40 -0
- package/dist/utils/mic-level-analyser.d.ts +13 -0
- package/dist/utils/mic-level-analyser.js +50 -0
- package/dist/utils/proactive-events.d.ts +5 -0
- package/dist/utils/proactive-events.js +1 -0
- package/dist/utils/styles.d.ts +4 -0
- package/dist/utils/styles.js +35 -0
- package/dist/utils/theme-detection.d.ts +3 -0
- package/dist/utils/theme-detection.js +121 -0
- package/dist/utils/tool-status.d.ts +10 -0
- package/dist/utils/tool-status.js +61 -0
- package/dist/utils/ui-sounds.d.ts +3 -0
- package/dist/utils/ui-sounds.js +57 -0
- package/package.json +56 -0
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
import { mapToolsForGoogle } from '../google/google-tool-mapper';
|
|
2
|
+
import { GoogleAudioHandler } from '../google/google-audio';
|
|
3
|
+
import { logger } from '../../../utils/logger';
|
|
4
|
+
const AUDIO_MIME_TYPE = 'audio/pcm;rate=16000';
|
|
5
|
+
const DEFAULT_VOICE = 'Kore';
|
|
6
|
+
const MAX_TOOL_CALL_HISTORY = 5;
|
|
7
|
+
const PLAYBACK_CHECK_INTERVAL_MS = 100;
|
|
8
|
+
const PLAYBACK_SETTLED_CHECKS = 3;
|
|
9
|
+
const DEFAULT_MAX_WAIT_MS = 20000;
|
|
10
|
+
export class VertexAiRealtimeProvider {
|
|
11
|
+
ws = null;
|
|
12
|
+
audioHandler;
|
|
13
|
+
eventHandlers = new Map();
|
|
14
|
+
_status = 'disconnected';
|
|
15
|
+
pendingFunctionCalls = new Map();
|
|
16
|
+
awaitingToolResponse = false;
|
|
17
|
+
isSetupComplete = false;
|
|
18
|
+
modelResponding = false;
|
|
19
|
+
userSpeaking = false;
|
|
20
|
+
pendingCommands = [];
|
|
21
|
+
outputTranscriptBuffer = '';
|
|
22
|
+
inputTranscriptBuffer = '';
|
|
23
|
+
lastCompletedToolCalls = [];
|
|
24
|
+
sentFrames = 0;
|
|
25
|
+
droppedFrames = 0;
|
|
26
|
+
constructor() {
|
|
27
|
+
this.audioHandler = new GoogleAudioHandler();
|
|
28
|
+
}
|
|
29
|
+
get status() {
|
|
30
|
+
return this._status;
|
|
31
|
+
}
|
|
32
|
+
async connect(config) {
|
|
33
|
+
if (!config.proxyUrl) {
|
|
34
|
+
throw new Error('VertexAI provider requires proxyUrl in config');
|
|
35
|
+
}
|
|
36
|
+
if (!config.proxyBearerToken) {
|
|
37
|
+
throw new Error('VertexAI provider requires proxyBearerToken in config');
|
|
38
|
+
}
|
|
39
|
+
this._status = 'connecting';
|
|
40
|
+
this.emit('connectionStateChanged', 'connecting');
|
|
41
|
+
const wsUrl = config.proxyUrl;
|
|
42
|
+
logger.debug('[VertexAI] Connecting to proxy:', wsUrl, 'model:', config.model);
|
|
43
|
+
this.ws = new WebSocket(wsUrl, [
|
|
44
|
+
`nextpa.bearer.${config.proxyBearerToken}`,
|
|
45
|
+
'nextpa.voice.v1',
|
|
46
|
+
]);
|
|
47
|
+
this.ws.onopen = () => {
|
|
48
|
+
logger.debug('[VertexAI] WebSocket opened, sending setup');
|
|
49
|
+
this._status = 'connected';
|
|
50
|
+
this.emit('connectionStateChanged', 'connected');
|
|
51
|
+
this.sendSetupMessage(config);
|
|
52
|
+
};
|
|
53
|
+
this.ws.onmessage = async (event) => {
|
|
54
|
+
try {
|
|
55
|
+
let text;
|
|
56
|
+
if (event.data instanceof Blob) {
|
|
57
|
+
text = await event.data.text();
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
text = event.data;
|
|
61
|
+
}
|
|
62
|
+
const message = JSON.parse(text);
|
|
63
|
+
this.handleServerMessage(message);
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
logger.error('[VertexAI] Failed to parse server message:', err);
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
this.ws.onerror = (event) => {
|
|
70
|
+
logger.error('[VertexAI] WebSocket error:', event);
|
|
71
|
+
this.emit('error', { message: 'Vertex AI proxy connection error' });
|
|
72
|
+
};
|
|
73
|
+
this.ws.onclose = (event) => {
|
|
74
|
+
logger.warn('[VertexAI] WebSocket closed, code:', event.code, 'reason:', event.reason, '- modelResponding:', this.modelResponding, 'pendingToolCalls:', this.pendingFunctionCalls.size, 'lastToolCalls:', this.lastCompletedToolCalls.join(' | ') || 'none');
|
|
75
|
+
this._status = 'disconnected';
|
|
76
|
+
this.resetSessionState();
|
|
77
|
+
this.audioHandler.stop();
|
|
78
|
+
this.emit('connectionStateChanged', 'disconnected');
|
|
79
|
+
};
|
|
80
|
+
this.sentFrames = 0;
|
|
81
|
+
this.droppedFrames = 0;
|
|
82
|
+
this.audioHandler.onAudioData = (base64Pcm) => this.handleMicFrame(base64Pcm);
|
|
83
|
+
this.audioHandler.onSpeechStarted = () => {
|
|
84
|
+
if (this.userSpeaking)
|
|
85
|
+
return;
|
|
86
|
+
this.userSpeaking = true;
|
|
87
|
+
this.emit('userSpeechStarted');
|
|
88
|
+
};
|
|
89
|
+
this.audioHandler.onSpeechEnded = () => {
|
|
90
|
+
if (!this.userSpeaking)
|
|
91
|
+
return;
|
|
92
|
+
this.userSpeaking = false;
|
|
93
|
+
this.emit('userSpeechEnded');
|
|
94
|
+
};
|
|
95
|
+
this.audioHandler.onAudioLevel = (level) => this.emit('userAudioLevel', level);
|
|
96
|
+
logger.debug('[VertexAI] Starting audio capture');
|
|
97
|
+
await this.audioHandler.startCapture();
|
|
98
|
+
}
|
|
99
|
+
handleMicFrame(base64Pcm) {
|
|
100
|
+
const canSend = this.ws && this.isSetupComplete && this._status === 'connected'
|
|
101
|
+
&& this.pendingFunctionCalls.size === 0;
|
|
102
|
+
if (canSend) {
|
|
103
|
+
this.sendJson({
|
|
104
|
+
realtime_input: {
|
|
105
|
+
media_chunks: [{
|
|
106
|
+
data: base64Pcm,
|
|
107
|
+
mime_type: AUDIO_MIME_TYPE,
|
|
108
|
+
}],
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
this.sentFrames++;
|
|
112
|
+
if (this.sentFrames === 1 || this.sentFrames % 100 === 0) {
|
|
113
|
+
logger.debug('[VertexAI] Sent mic frame', this.sentFrames, '(dropped so far:', this.droppedFrames, ')');
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
this.droppedFrames++;
|
|
118
|
+
if (this.droppedFrames === 1 || this.droppedFrames % 50 === 0) {
|
|
119
|
+
logger.debug('[VertexAI] DROPPED mic frame', this.droppedFrames, 'reason:', this.describeDropReason());
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
describeDropReason() {
|
|
124
|
+
if (this.ws) {
|
|
125
|
+
if (this.isSetupComplete) {
|
|
126
|
+
if (this._status === 'connected') {
|
|
127
|
+
return 'pending-tool-calls';
|
|
128
|
+
}
|
|
129
|
+
return `status=${this._status}`;
|
|
130
|
+
}
|
|
131
|
+
return 'setup-incomplete';
|
|
132
|
+
}
|
|
133
|
+
return 'no-ws';
|
|
134
|
+
}
|
|
135
|
+
close() {
|
|
136
|
+
if (this.ws) {
|
|
137
|
+
this.ws.close();
|
|
138
|
+
this.ws = null;
|
|
139
|
+
}
|
|
140
|
+
this.audioHandler.stop();
|
|
141
|
+
this._status = 'disconnected';
|
|
142
|
+
this.resetSessionState();
|
|
143
|
+
}
|
|
144
|
+
resumeAudio() {
|
|
145
|
+
this.audioHandler.resume();
|
|
146
|
+
}
|
|
147
|
+
mute(muted) {
|
|
148
|
+
this.audioHandler.setMuted(muted);
|
|
149
|
+
}
|
|
150
|
+
interrupt() {
|
|
151
|
+
// Gemini Live has no client-side response-cancel; dropping the queued
|
|
152
|
+
// playback stops the talking immediately, which is what the user wants.
|
|
153
|
+
this.audioHandler.clearPlaybackQueue();
|
|
154
|
+
this.emit('interrupted');
|
|
155
|
+
}
|
|
156
|
+
requestResponse() {
|
|
157
|
+
if (!this.isSetupComplete) {
|
|
158
|
+
this.pendingCommands.push({ type: 'requestResponse' });
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
this.executeRequestResponse();
|
|
162
|
+
}
|
|
163
|
+
sendToolResult(callId, name, output) {
|
|
164
|
+
if (!this.isSetupComplete) {
|
|
165
|
+
this.pendingCommands.push({ type: 'sendToolResult', payload: { callId, name, output } });
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
this.executeSendToolResult(callId, name, output);
|
|
169
|
+
}
|
|
170
|
+
sendText(text) {
|
|
171
|
+
if (!this.ws)
|
|
172
|
+
return;
|
|
173
|
+
if (!this.isSetupComplete) {
|
|
174
|
+
this.pendingCommands.push({ type: 'sendText', payload: text });
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
this.sendJson({
|
|
178
|
+
client_content: {
|
|
179
|
+
turns: [{ role: 'user', parts: [{ text }] }],
|
|
180
|
+
turn_complete: true,
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
waitForAudioPlaybackEnd(timeoutMs = DEFAULT_MAX_WAIT_MS) {
|
|
185
|
+
return new Promise((resolve) => {
|
|
186
|
+
if (!this.audioHandler.isPlaybackActive) {
|
|
187
|
+
resolve('no active playback');
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
let settledCount = 0;
|
|
191
|
+
const interval = setInterval(() => {
|
|
192
|
+
if (this.audioHandler.isPlaybackActive) {
|
|
193
|
+
settledCount = 0;
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
settledCount++;
|
|
197
|
+
if (settledCount >= PLAYBACK_SETTLED_CHECKS) {
|
|
198
|
+
clearInterval(interval);
|
|
199
|
+
resolve('playback complete');
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}, PLAYBACK_CHECK_INTERVAL_MS);
|
|
203
|
+
setTimeout(() => {
|
|
204
|
+
clearInterval(interval);
|
|
205
|
+
resolve('max wait reached');
|
|
206
|
+
}, timeoutMs);
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
async refreshAudioDevice() {
|
|
210
|
+
logger.debug('[VertexAI] Refreshing audio device');
|
|
211
|
+
await this.audioHandler.restartCapture();
|
|
212
|
+
}
|
|
213
|
+
on(event, handler) {
|
|
214
|
+
if (!this.eventHandlers.has(event)) {
|
|
215
|
+
this.eventHandlers.set(event, new Set());
|
|
216
|
+
}
|
|
217
|
+
this.eventHandlers.get(event).add(handler);
|
|
218
|
+
return () => this.off(event, handler);
|
|
219
|
+
}
|
|
220
|
+
off(event, handler) {
|
|
221
|
+
this.eventHandlers.get(event)?.delete(handler);
|
|
222
|
+
}
|
|
223
|
+
sendJson(data) {
|
|
224
|
+
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
225
|
+
this.ws.send(JSON.stringify(data));
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
sendSetupMessage(config) {
|
|
229
|
+
const toolDeclarations = mapToolsForGoogle(config.tools);
|
|
230
|
+
// languageCodes pin the ASR + TTS to a specific BCP-47 locale so
|
|
231
|
+
// Gemini doesn't drift into Spanish/Hindi on quiet inputs (observed
|
|
232
|
+
// live during M3-i easybell trunk testing). Empty when the tenant
|
|
233
|
+
// has no configured locale — Gemini auto-detects then.
|
|
234
|
+
const transcriptionLanguageCodes = config.transcription?.language
|
|
235
|
+
? [config.transcription.language]
|
|
236
|
+
: undefined;
|
|
237
|
+
const setup = {
|
|
238
|
+
model: config.model,
|
|
239
|
+
generationConfig: {
|
|
240
|
+
responseModalities: ['AUDIO'],
|
|
241
|
+
speechConfig: {
|
|
242
|
+
voiceConfig: {
|
|
243
|
+
prebuiltVoiceConfig: {
|
|
244
|
+
voiceName: config.voice || DEFAULT_VOICE,
|
|
245
|
+
},
|
|
246
|
+
},
|
|
247
|
+
},
|
|
248
|
+
},
|
|
249
|
+
realtimeInputConfig: {
|
|
250
|
+
automaticActivityDetection: {
|
|
251
|
+
// LOW start sensitivity needs clearer speech to open a turn,
|
|
252
|
+
// so background noise on mobile no longer trips the VAD.
|
|
253
|
+
// (Gemini's own default is LOW; HIGH was over-eager here.)
|
|
254
|
+
startOfSpeechSensitivity: 'START_SENSITIVITY_LOW',
|
|
255
|
+
endOfSpeechSensitivity: 'END_SENSITIVITY_HIGH',
|
|
256
|
+
prefixPaddingMs: 100,
|
|
257
|
+
silenceDurationMs: 500,
|
|
258
|
+
},
|
|
259
|
+
},
|
|
260
|
+
inputAudioTranscription: transcriptionLanguageCodes
|
|
261
|
+
? { languageCodes: transcriptionLanguageCodes }
|
|
262
|
+
: {},
|
|
263
|
+
outputAudioTranscription: transcriptionLanguageCodes
|
|
264
|
+
? { languageCodes: transcriptionLanguageCodes }
|
|
265
|
+
: {},
|
|
266
|
+
};
|
|
267
|
+
if (config.instructions) {
|
|
268
|
+
setup.systemInstruction = { parts: [{ text: config.instructions }] };
|
|
269
|
+
}
|
|
270
|
+
if (toolDeclarations.length > 0) {
|
|
271
|
+
setup.tools = [{ functionDeclarations: toolDeclarations }];
|
|
272
|
+
}
|
|
273
|
+
this.sendJson({ setup });
|
|
274
|
+
}
|
|
275
|
+
emit(event, ...args) {
|
|
276
|
+
const handlers = this.eventHandlers.get(event);
|
|
277
|
+
if (!handlers)
|
|
278
|
+
return;
|
|
279
|
+
for (const handler of handlers) {
|
|
280
|
+
handler(...args);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
resetSessionState() {
|
|
284
|
+
this.ws = null;
|
|
285
|
+
this.isSetupComplete = false;
|
|
286
|
+
this.awaitingToolResponse = false;
|
|
287
|
+
this.modelResponding = false;
|
|
288
|
+
this.userSpeaking = false;
|
|
289
|
+
this.pendingFunctionCalls.clear();
|
|
290
|
+
this.pendingCommands = [];
|
|
291
|
+
this.outputTranscriptBuffer = '';
|
|
292
|
+
this.inputTranscriptBuffer = '';
|
|
293
|
+
this.lastCompletedToolCalls = [];
|
|
294
|
+
}
|
|
295
|
+
executeRequestResponse() {
|
|
296
|
+
if (!this.ws)
|
|
297
|
+
return;
|
|
298
|
+
if (this.awaitingToolResponse) {
|
|
299
|
+
logger.debug('[VertexAI] Skipping requestResponse - auto-continues after tool response');
|
|
300
|
+
this.awaitingToolResponse = false;
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
if (this.pendingFunctionCalls.size > 0 || this.modelResponding) {
|
|
304
|
+
logger.debug('[VertexAI] Skipping requestResponse - model busy');
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
this.sendJson({
|
|
308
|
+
client_content: {
|
|
309
|
+
turns: [{ role: 'user', parts: [{ text: '.' }] }],
|
|
310
|
+
turn_complete: true,
|
|
311
|
+
},
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
executeSendToolResult(callId, name, output) {
|
|
315
|
+
if (!this.ws)
|
|
316
|
+
return;
|
|
317
|
+
this.awaitingToolResponse = true;
|
|
318
|
+
const funcName = this.pendingFunctionCalls.get(callId)?.name ?? name;
|
|
319
|
+
const rawOutput = output;
|
|
320
|
+
let parsedResult;
|
|
321
|
+
try {
|
|
322
|
+
const parsed = JSON.parse(rawOutput);
|
|
323
|
+
parsedResult =
|
|
324
|
+
typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)
|
|
325
|
+
? parsed
|
|
326
|
+
: { output: parsed };
|
|
327
|
+
}
|
|
328
|
+
catch {
|
|
329
|
+
parsedResult = { output: rawOutput };
|
|
330
|
+
}
|
|
331
|
+
try {
|
|
332
|
+
this.sendJson({
|
|
333
|
+
tool_response: {
|
|
334
|
+
function_responses: [{
|
|
335
|
+
name: funcName,
|
|
336
|
+
response: parsedResult,
|
|
337
|
+
}],
|
|
338
|
+
},
|
|
339
|
+
});
|
|
340
|
+
logger.debug('[VertexAI] Tool response sent for', funcName, 'size:', rawOutput.length);
|
|
341
|
+
this.lastCompletedToolCalls.push(`${funcName}(${rawOutput.slice(0, 100)})`);
|
|
342
|
+
if (this.lastCompletedToolCalls.length > MAX_TOOL_CALL_HISTORY) {
|
|
343
|
+
this.lastCompletedToolCalls.shift();
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
catch (error) {
|
|
347
|
+
logger.error('[VertexAI] Failed to send tool response for', funcName, error);
|
|
348
|
+
this.emit('error', { message: `Tool response failed for ${funcName}: ${error}` });
|
|
349
|
+
}
|
|
350
|
+
finally {
|
|
351
|
+
this.pendingFunctionCalls.delete(callId);
|
|
352
|
+
if (this.pendingFunctionCalls.size === 0) {
|
|
353
|
+
this.modelResponding = false;
|
|
354
|
+
this.audioHandler.resumeCapture();
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
finalizeInputTranscript() {
|
|
359
|
+
if (this.inputTranscriptBuffer) {
|
|
360
|
+
this.emit('userTranscriptCompleted', this.inputTranscriptBuffer);
|
|
361
|
+
this.inputTranscriptBuffer = '';
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
handleServerMessage(message) {
|
|
365
|
+
logger.debug('[VertexAI] message keys:', Object.keys(message).join(', '));
|
|
366
|
+
if (message.setupComplete !== undefined) {
|
|
367
|
+
logger.debug('[VertexAI] Setup complete');
|
|
368
|
+
this.isSetupComplete = true;
|
|
369
|
+
this.emit('sessionReady');
|
|
370
|
+
this.flushPendingCommands();
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
const voiceActivity = (message.voiceActivity ?? message.voice_activity);
|
|
374
|
+
if (voiceActivity) {
|
|
375
|
+
this.handleVoiceActivity(voiceActivity);
|
|
376
|
+
}
|
|
377
|
+
if (message.usageMetadata) {
|
|
378
|
+
this.extractAndEmitUsage(message.usageMetadata);
|
|
379
|
+
}
|
|
380
|
+
if (message.serverContent) {
|
|
381
|
+
this.handleServerContent(message.serverContent);
|
|
382
|
+
}
|
|
383
|
+
if (message.toolCall) {
|
|
384
|
+
this.ensureSpeechStopped();
|
|
385
|
+
this.handleToolCall(message.toolCall);
|
|
386
|
+
}
|
|
387
|
+
if (message.toolCallCancellation) {
|
|
388
|
+
logger.debug('[VertexAI] Tool call cancelled, clearing', this.pendingFunctionCalls.size, 'pending calls');
|
|
389
|
+
this.pendingFunctionCalls.clear();
|
|
390
|
+
this.modelResponding = false;
|
|
391
|
+
this.audioHandler.resumeCapture();
|
|
392
|
+
this.audioHandler.clearPlaybackQueue();
|
|
393
|
+
}
|
|
394
|
+
if (message.goAway) {
|
|
395
|
+
const goAway = message.goAway;
|
|
396
|
+
const timeLeft = goAway.timeLeft;
|
|
397
|
+
logger.warn('[VertexAI] Server disconnecting soon, timeLeft:', timeLeft);
|
|
398
|
+
this.emit('serverDisconnecting', timeLeft);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
extractAndEmitUsage(metadata) {
|
|
402
|
+
logger.debug('[VertexAI] Raw usage metadata:', JSON.stringify(metadata));
|
|
403
|
+
const inputTokens = metadata.promptTokenCount ?? 0;
|
|
404
|
+
const outputTokens = metadata.responseTokenCount ?? metadata.candidatesTokenCount ?? 0;
|
|
405
|
+
const totalTokens = metadata.totalTokenCount ?? (inputTokens + outputTokens);
|
|
406
|
+
const promptDetails = metadata.promptTokensDetails;
|
|
407
|
+
const responseDetails = metadata.responseTokensDetails ??
|
|
408
|
+
metadata.candidatesTokensDetails;
|
|
409
|
+
const findModality = (details, modality) => details?.find((d) => d.modality === modality)?.tokenCount ?? undefined;
|
|
410
|
+
const providerUsage = {
|
|
411
|
+
inputTokens,
|
|
412
|
+
outputTokens,
|
|
413
|
+
totalTokens,
|
|
414
|
+
cachedTokens: metadata.cachedContentTokenCount ?? undefined,
|
|
415
|
+
audioInputTokens: findModality(promptDetails, 'AUDIO'),
|
|
416
|
+
audioOutputTokens: findModality(responseDetails, 'AUDIO'),
|
|
417
|
+
textInputTokens: findModality(promptDetails, 'TEXT'),
|
|
418
|
+
textOutputTokens: findModality(responseDetails, 'TEXT'),
|
|
419
|
+
};
|
|
420
|
+
this.emit('usageUpdated', providerUsage);
|
|
421
|
+
}
|
|
422
|
+
flushPendingCommands() {
|
|
423
|
+
const commands = this.pendingCommands;
|
|
424
|
+
this.pendingCommands = [];
|
|
425
|
+
for (const cmd of commands) {
|
|
426
|
+
switch (cmd.type) {
|
|
427
|
+
case 'requestResponse':
|
|
428
|
+
this.executeRequestResponse();
|
|
429
|
+
break;
|
|
430
|
+
case 'sendToolResult': {
|
|
431
|
+
const p = cmd.payload;
|
|
432
|
+
this.executeSendToolResult(p.callId, p.name, p.output);
|
|
433
|
+
break;
|
|
434
|
+
}
|
|
435
|
+
case 'sendText':
|
|
436
|
+
this.sendText(cmd.payload);
|
|
437
|
+
break;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
handleVoiceActivity(activity) {
|
|
442
|
+
const type = activity.voiceActivityType ?? activity.voice_activity_type;
|
|
443
|
+
if ((type === 'ACTIVITY_START') && !this.userSpeaking) {
|
|
444
|
+
logger.debug('[VertexAI] Voice activity: ACTIVITY_START');
|
|
445
|
+
this.userSpeaking = true;
|
|
446
|
+
this.emit('userSpeechStarted');
|
|
447
|
+
}
|
|
448
|
+
else if ((type === 'ACTIVITY_END') && this.userSpeaking) {
|
|
449
|
+
logger.debug('[VertexAI] Voice activity: ACTIVITY_END');
|
|
450
|
+
this.ensureSpeechStopped();
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
handleServerContent(content) {
|
|
454
|
+
const inputTranscription = content.inputTranscription;
|
|
455
|
+
const outputTranscriptionRaw = content.outputTranscription;
|
|
456
|
+
const hasContent = inputTranscription || outputTranscriptionRaw || content.modelTurn || content.turnComplete;
|
|
457
|
+
if (!hasContent && !this.modelResponding && !this.userSpeaking) {
|
|
458
|
+
logger.debug('[VertexAI] Speech activity detected (empty serverContent fallback)');
|
|
459
|
+
this.userSpeaking = true;
|
|
460
|
+
this.emit('userSpeechStarted');
|
|
461
|
+
}
|
|
462
|
+
if (inputTranscription?.text) {
|
|
463
|
+
logger.debug('[VertexAI] inputTranscription text:', inputTranscription.text.slice(0, 80));
|
|
464
|
+
if (!this.userSpeaking) {
|
|
465
|
+
this.userSpeaking = true;
|
|
466
|
+
this.emit('userSpeechStarted');
|
|
467
|
+
}
|
|
468
|
+
this.inputTranscriptBuffer += inputTranscription.text;
|
|
469
|
+
this.emit('userTranscriptDelta', inputTranscription.text);
|
|
470
|
+
}
|
|
471
|
+
if (inputTranscription?.finished) {
|
|
472
|
+
logger.debug('[VertexAI] inputTranscription finished');
|
|
473
|
+
this.finalizeInputTranscript();
|
|
474
|
+
this.userSpeaking = false;
|
|
475
|
+
this.emit('userSpeechStopped');
|
|
476
|
+
}
|
|
477
|
+
const isFirstModelResponse = !!content.modelTurn && !this.modelResponding;
|
|
478
|
+
if (content.modelTurn) {
|
|
479
|
+
this.handleModelTurn(content);
|
|
480
|
+
}
|
|
481
|
+
if (outputTranscriptionRaw?.text) {
|
|
482
|
+
if (isFirstModelResponse) {
|
|
483
|
+
this.deferAssistantTranscript(outputTranscriptionRaw.text);
|
|
484
|
+
}
|
|
485
|
+
else {
|
|
486
|
+
this.outputTranscriptBuffer += outputTranscriptionRaw.text;
|
|
487
|
+
this.emit('assistantTranscriptDelta', outputTranscriptionRaw.text);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
if (content.turnComplete) {
|
|
491
|
+
this.handleTurnComplete();
|
|
492
|
+
}
|
|
493
|
+
if (content.interrupted) {
|
|
494
|
+
this.handleInterrupted();
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
static THINKING_RENDER_DELAY_MS = 150;
|
|
498
|
+
deferAssistantTranscript(text) {
|
|
499
|
+
setTimeout(() => {
|
|
500
|
+
this.outputTranscriptBuffer += text;
|
|
501
|
+
this.emit('assistantTranscriptDelta', text);
|
|
502
|
+
}, VertexAiRealtimeProvider.THINKING_RENDER_DELAY_MS);
|
|
503
|
+
}
|
|
504
|
+
ensureSpeechStopped() {
|
|
505
|
+
if (this.inputTranscriptBuffer) {
|
|
506
|
+
this.finalizeInputTranscript();
|
|
507
|
+
}
|
|
508
|
+
this.userSpeaking = false;
|
|
509
|
+
this.emit('userSpeechStopped');
|
|
510
|
+
}
|
|
511
|
+
handleModelTurn(content) {
|
|
512
|
+
if (!this.modelResponding) {
|
|
513
|
+
logger.debug('[VertexAI] Model turn started');
|
|
514
|
+
}
|
|
515
|
+
this.modelResponding = true;
|
|
516
|
+
this.ensureSpeechStopped();
|
|
517
|
+
const modelTurn = content.modelTurn;
|
|
518
|
+
if (modelTurn?.parts) {
|
|
519
|
+
for (const part of modelTurn.parts) {
|
|
520
|
+
if (part.inlineData?.mimeType?.startsWith('audio/') && part.inlineData.data) {
|
|
521
|
+
this.audioHandler.playAudio(part.inlineData.data);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
handleTurnComplete() {
|
|
527
|
+
logger.debug('[VertexAI] Turn complete');
|
|
528
|
+
this.modelResponding = false;
|
|
529
|
+
if (this.outputTranscriptBuffer) {
|
|
530
|
+
this.emit('assistantTranscriptDone', this.outputTranscriptBuffer);
|
|
531
|
+
}
|
|
532
|
+
this.outputTranscriptBuffer = '';
|
|
533
|
+
this.emit('audioDone');
|
|
534
|
+
}
|
|
535
|
+
handleInterrupted() {
|
|
536
|
+
logger.debug('[VertexAI] Model interrupted');
|
|
537
|
+
this.modelResponding = false;
|
|
538
|
+
this.finalizeInputTranscript();
|
|
539
|
+
this.outputTranscriptBuffer = '';
|
|
540
|
+
this.audioHandler.clearPlaybackQueue();
|
|
541
|
+
this.emit('interrupted');
|
|
542
|
+
if (!this.userSpeaking) {
|
|
543
|
+
this.userSpeaking = true;
|
|
544
|
+
this.emit('userSpeechStarted');
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
handleToolCall(toolCall) {
|
|
548
|
+
if (!toolCall.functionCalls)
|
|
549
|
+
return;
|
|
550
|
+
this.modelResponding = true;
|
|
551
|
+
this.audioHandler.pauseCapture();
|
|
552
|
+
for (const fc of toolCall.functionCalls) {
|
|
553
|
+
const callId = fc.id || `call_${Date.now()}`;
|
|
554
|
+
const name = fc.name ?? '';
|
|
555
|
+
const args = JSON.stringify(fc.args || {});
|
|
556
|
+
logger.debug('[VertexAI] Tool call received:', name, 'args:', args);
|
|
557
|
+
this.pendingFunctionCalls.set(callId, { name, args });
|
|
558
|
+
this.emit('functionCall', { callId, name, arguments: args });
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { WidgetApiClient } from '../../services/api';
|
|
2
|
+
export declare class VoiceSessionTracker {
|
|
3
|
+
private sessionId;
|
|
4
|
+
private heartbeatTimer;
|
|
5
|
+
private api;
|
|
6
|
+
private active;
|
|
7
|
+
private readonly handleVisibilityChange;
|
|
8
|
+
startTracking(api: WidgetApiClient, sessionId: string): Promise<void>;
|
|
9
|
+
stopTracking(): Promise<void>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { logger } from '../../utils/logger';
|
|
2
|
+
const LOG_PREFIX = '[VoiceSessionTracker]';
|
|
3
|
+
const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
4
|
+
export class VoiceSessionTracker {
|
|
5
|
+
sessionId = null;
|
|
6
|
+
heartbeatTimer = null;
|
|
7
|
+
api = null;
|
|
8
|
+
active = false;
|
|
9
|
+
handleVisibilityChange = () => {
|
|
10
|
+
if (document.visibilityState === 'hidden' && this.active && this.sessionId) {
|
|
11
|
+
this.api?.sendVoiceSessionBeacon(this.sessionId);
|
|
12
|
+
logger.debug(`${LOG_PREFIX} Beacon sent on visibility hidden`);
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
async startTracking(api, sessionId) {
|
|
16
|
+
if (this.active) {
|
|
17
|
+
await this.stopTracking();
|
|
18
|
+
}
|
|
19
|
+
this.api = api;
|
|
20
|
+
this.sessionId = sessionId;
|
|
21
|
+
this.active = true;
|
|
22
|
+
try {
|
|
23
|
+
await api.startVoiceSession(sessionId);
|
|
24
|
+
logger.debug(`${LOG_PREFIX} Session started: ${sessionId}`);
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
logger.error(`${LOG_PREFIX} Failed to start session`, error);
|
|
28
|
+
}
|
|
29
|
+
this.heartbeatTimer = setInterval(async () => {
|
|
30
|
+
if (!this.active || !this.sessionId || !this.api)
|
|
31
|
+
return;
|
|
32
|
+
try {
|
|
33
|
+
await this.api.sendVoiceHeartbeat(this.sessionId);
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
logger.error(`${LOG_PREFIX} Heartbeat failed`, error);
|
|
37
|
+
}
|
|
38
|
+
}, HEARTBEAT_INTERVAL_MS);
|
|
39
|
+
document.addEventListener('visibilitychange', this.handleVisibilityChange);
|
|
40
|
+
}
|
|
41
|
+
async stopTracking() {
|
|
42
|
+
if (!this.active)
|
|
43
|
+
return;
|
|
44
|
+
this.active = false;
|
|
45
|
+
document.removeEventListener('visibilitychange', this.handleVisibilityChange);
|
|
46
|
+
if (this.heartbeatTimer) {
|
|
47
|
+
clearInterval(this.heartbeatTimer);
|
|
48
|
+
this.heartbeatTimer = null;
|
|
49
|
+
}
|
|
50
|
+
if (this.sessionId && this.api) {
|
|
51
|
+
try {
|
|
52
|
+
await this.api.endVoiceSession(this.sessionId);
|
|
53
|
+
logger.debug(`${LOG_PREFIX} Session ended: ${this.sessionId}`);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
logger.error(`${LOG_PREFIX} Failed to end session`, error);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
this.sessionId = null;
|
|
60
|
+
this.api = null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import * as signalR from '@microsoft/signalr';
|
|
2
|
+
export interface SignalREvents {
|
|
3
|
+
onStopConversation?: () => void;
|
|
4
|
+
onOpenUrl?: (url: string) => void;
|
|
5
|
+
onOpenFile?: (url: string) => void;
|
|
6
|
+
onError?: (title: string, message: string) => void;
|
|
7
|
+
onConnected?: () => void;
|
|
8
|
+
onDisconnected?: () => void;
|
|
9
|
+
}
|
|
10
|
+
export interface TextChatSignalRHandlers {
|
|
11
|
+
onStreamDelta?: (delta: string, messageId: string) => void;
|
|
12
|
+
onStreamCompleted?: (fullText: string, messageId: string) => void;
|
|
13
|
+
onToolCall?: (toolName: string, status: string, result?: string, agentLabel?: string) => void;
|
|
14
|
+
}
|
|
15
|
+
export interface PageInteractionSignalRHandlers {
|
|
16
|
+
onPageInteractionRequest?: (requestJson: string) => void;
|
|
17
|
+
}
|
|
18
|
+
export declare class SignalRService {
|
|
19
|
+
private connection;
|
|
20
|
+
private events;
|
|
21
|
+
private textChatHandlers;
|
|
22
|
+
private pageInteractionHandlers;
|
|
23
|
+
private registrationInfo;
|
|
24
|
+
get isConnected(): boolean;
|
|
25
|
+
setEventHandlers(events: SignalREvents): void;
|
|
26
|
+
setTextChatHandlers(handlers: TextChatSignalRHandlers): void;
|
|
27
|
+
clearTextChatHandlers(): void;
|
|
28
|
+
setPageInteractionHandlers(handlers: PageInteractionSignalRHandlers): void;
|
|
29
|
+
clearPageInteractionHandlers(): void;
|
|
30
|
+
getConnection(): signalR.HubConnection | null;
|
|
31
|
+
setRegistrationInfo(clientId: string, tenantId: string, userId: string): void;
|
|
32
|
+
connect(baseUrl: string, token: string): Promise<void>;
|
|
33
|
+
disconnect(): Promise<void>;
|
|
34
|
+
private registerClient;
|
|
35
|
+
private setupEventHandlers;
|
|
36
|
+
private handleNotification;
|
|
37
|
+
}
|