@mobileai/react-native 0.4.2 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +21 -2
  2. package/lib/module/components/AIAgent.js +216 -5
  3. package/lib/module/components/AIAgent.js.map +1 -1
  4. package/lib/module/components/AgentChatBar.js +358 -36
  5. package/lib/module/components/AgentChatBar.js.map +1 -1
  6. package/lib/module/core/AgentRuntime.js +122 -6
  7. package/lib/module/core/AgentRuntime.js.map +1 -1
  8. package/lib/module/core/systemPrompt.js +57 -0
  9. package/lib/module/core/systemPrompt.js.map +1 -1
  10. package/lib/module/index.js +8 -0
  11. package/lib/module/index.js.map +1 -1
  12. package/lib/module/providers/GeminiProvider.js +108 -85
  13. package/lib/module/providers/GeminiProvider.js.map +1 -1
  14. package/lib/module/services/AudioInputService.js +128 -0
  15. package/lib/module/services/AudioInputService.js.map +1 -0
  16. package/lib/module/services/AudioOutputService.js +154 -0
  17. package/lib/module/services/AudioOutputService.js.map +1 -0
  18. package/lib/module/services/VoiceService.js +362 -0
  19. package/lib/module/services/VoiceService.js.map +1 -0
  20. package/lib/module/utils/audioUtils.js +49 -0
  21. package/lib/module/utils/audioUtils.js.map +1 -0
  22. package/lib/module/utils/logger.js +21 -4
  23. package/lib/module/utils/logger.js.map +1 -1
  24. package/lib/typescript/babel.config.d.ts +10 -0
  25. package/lib/typescript/babel.config.d.ts.map +1 -0
  26. package/lib/typescript/eslint.config.d.mts +3 -0
  27. package/lib/typescript/eslint.config.d.mts.map +1 -0
  28. package/lib/typescript/fetch-models.d.mts +2 -0
  29. package/lib/typescript/fetch-models.d.mts.map +1 -0
  30. package/lib/typescript/list-all-models.d.mts +2 -0
  31. package/lib/typescript/list-all-models.d.mts.map +1 -0
  32. package/lib/typescript/list-models.d.mts +2 -0
  33. package/lib/typescript/list-models.d.mts.map +1 -0
  34. package/lib/typescript/src/components/AIAgent.d.ts +8 -2
  35. package/lib/typescript/src/components/AIAgent.d.ts.map +1 -1
  36. package/lib/typescript/src/components/AgentChatBar.d.ts +19 -2
  37. package/lib/typescript/src/components/AgentChatBar.d.ts.map +1 -1
  38. package/lib/typescript/src/core/AgentRuntime.d.ts +17 -1
  39. package/lib/typescript/src/core/AgentRuntime.d.ts.map +1 -1
  40. package/lib/typescript/src/core/systemPrompt.d.ts +8 -0
  41. package/lib/typescript/src/core/systemPrompt.d.ts.map +1 -1
  42. package/lib/typescript/src/core/types.d.ts +24 -1
  43. package/lib/typescript/src/core/types.d.ts.map +1 -1
  44. package/lib/typescript/src/index.d.ts +6 -1
  45. package/lib/typescript/src/index.d.ts.map +1 -1
  46. package/lib/typescript/src/providers/GeminiProvider.d.ts +22 -18
  47. package/lib/typescript/src/providers/GeminiProvider.d.ts.map +1 -1
  48. package/lib/typescript/src/services/AudioInputService.d.ts +31 -0
  49. package/lib/typescript/src/services/AudioInputService.d.ts.map +1 -0
  50. package/lib/typescript/src/services/AudioOutputService.d.ts +34 -0
  51. package/lib/typescript/src/services/AudioOutputService.d.ts.map +1 -0
  52. package/lib/typescript/src/services/VoiceService.d.ts +73 -0
  53. package/lib/typescript/src/services/VoiceService.d.ts.map +1 -0
  54. package/lib/typescript/src/utils/audioUtils.d.ts +17 -0
  55. package/lib/typescript/src/utils/audioUtils.d.ts.map +1 -0
  56. package/lib/typescript/src/utils/logger.d.ts +4 -0
  57. package/lib/typescript/src/utils/logger.d.ts.map +1 -1
  58. package/package.json +24 -8
  59. package/src/components/AIAgent.tsx +222 -3
  60. package/src/components/AgentChatBar.tsx +487 -42
  61. package/src/core/AgentRuntime.ts +131 -2
  62. package/src/core/systemPrompt.ts +62 -0
  63. package/src/core/types.ts +30 -0
  64. package/src/index.ts +16 -0
  65. package/src/providers/GeminiProvider.ts +105 -89
  66. package/src/services/AudioInputService.ts +141 -0
  67. package/src/services/AudioOutputService.ts +167 -0
  68. package/src/services/VoiceService.ts +409 -0
  69. package/src/utils/audioUtils.ts +54 -0
  70. package/src/utils/logger.ts +24 -7
@@ -0,0 +1 @@
1
+ {"version":3,"names":["logger","base64ToFloat32","GEMINI_OUTPUT_SAMPLE_RATE","AudioOutputService","audioContext","queueSourceNode","gainNode","muted","isStarted","chunkCount","constructor","config","initialize","audioApi","require","msg","error","onError","sampleRate","AudioContext","createGain","gain","value","connect","destination","createBufferQueueSource","info","message","enqueue","base64Audio","float32Data","audioBuffer","createBuffer","length","copyToChannel","enqueueBuffer","start","onPlaybackStart","debug","mute","unmute","isMuted","stop","clearBuffers","onPlaybackEnd","cleanup","close"],"sourceRoot":"../../../src","sources":["services/AudioOutputService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;AACxC,SAASC,eAAe,QAAQ,wBAAqB;;AAErD;;AAEA;AACA,MAAMC,yBAAyB,GAAG,KAAK;AASvC;;AAEA,OAAO,MAAMC,kBAAkB,CAAC;EAEtBC,YAAY,GAAQ,IAAI;EACxBC,eAAe,GAAQ,IAAI;EAC3BC,QAAQ,GAAQ,IAAI;EACpBC,KAAK,GAAG,KAAK;EACbC,SAAS,GAAG,KAAK;EACjBC,UAAU,GAAG,CAAC;EAEtBC,WAAWA,CAACC,MAAyB,GAAG,CAAC,CAAC,EAAE;IAC1C,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEA,MAAMC,UAAUA,CAAA,EAAqB;IACnC,IAAI;MACF,IAAIC,QAAa;MACjB,IAAI;QACFA,QAAQ,GAAGC,OAAO,CAAC,wBAAwB,CAAC;MAC9C,CAAC,CAAC,MAAM;QACN,MAAMC,GAAG,GACP,uGAAuG;QACzGf,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAED,GAAG,CAAC;QAChC,IAAI,CAACJ,MAAM,CAACM,OAAO,GAAGF,GAAG,CAAC;QAC1B,OAAO,KAAK;MACd;MAEA,MAAMG,UAAU,GAAG,IAAI,CAACP,MAAM,CAACO,UAAU,IAAIhB,yBAAyB;;MAEtE;MACA,IAAI,CAACE,YAAY,GAAG,IAAIS,QAAQ,CAACM,YAAY,CAAC;QAAED;MAAW,CAAC,CAAC;;MAE7D;MACA,IAAI,CAACZ,QAAQ,GAAG,IAAI,CAACF,YAAY,CAACgB,UAAU,CAAC,CAAC;MAC9C,IAAI,CAACd,QAAQ,CAACe,IAAI,CAACC,KAAK,GAAG,GAAG;MAC9B,IAAI,CAAChB,QAAQ,CAACiB,OAAO,CAAC,IAAI,CAACnB,YAAY,CAACoB,WAAW,CAAC;;MAEpD;MACA,IAAI,CAACnB,eAAe,GAAG,IAAI,CAACD,YAAY,CAACqB,uBAAuB,CAAC,CAAC;MAClE,IAAI,CAACpB,eAAe,CAACkB,OAAO,CAAC,IAAI,CAACjB,QAAQ,CAAC;MAE3CN,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,gBAAgBR,UAAU,iCAAiC,CAAC;MACvF,OAAO,IAAI;IACb,CAAC,CAAC,OAAOF,KAAU,EAAE;MACnBhB,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAE,yBAAyBA,KAAK,CAACW,OAAO,EAAE,CAAC;MACrE,IAAI,CAAChB,MAAM,CAACM,OAAO,GAAGD,KAAK,CAACW,OAAO,CAAC;MACpC,OAAO,KAAK;IACd;EACF;;EAEA;;EAEA;EACAC,OAAOA,CAACC,WAAmB,EAAQ;IACjC,IAAI,IAAI,CAACtB,KAAK,IAAI,CAAC,IAAI,CAACH,YAAY,IAAI,CAAC,IAAI,CAACC,eAAe,EAAE;IAE/D,IAAI;MACF,IAAI,CAACI,UAAU,EAAE;;MAEjB;MACA,MAAMqB,WAAW,GAAG7B,eAAe,CAAC4B,WAAW,CAAC;MAChD,MAAMX,UAAU,GAAG,IAAI,CAACP,MAAM,CAACO,UAAU,IAAIhB,yBAAyB;;MAEtE;MACA,MAAM6B,WAAW,GAAG,IAAI,CAAC3B,YAAY,CAAC4B,YAAY,CAAC,CAAC,EAAEF,WAAW,CAACG,MAAM,EAAEf,UAAU,CAAC;MACrFa,WAAW,CAACG,aAAa,CAACJ,WAAW,EAAE,CAAC,CAAC;;MAEzC;MACA,IAAI,CAACzB,eAAe,CAAC8B,aAAa,CAACJ,WAAW,CAAC;;MAE/C;MACA,IAAI,CAAC,IAAI,CAACvB,SAAS,EAAE;QACnB,IAAI,CAACH,eAAe,CAAC+B,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC5B,SAAS,GAAG,IAAI;QACrB,IAAI,CAACG,MAAM,CAAC0B,eAAe,GAAG,CAAC;QAC/BrC,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,qBAAqB,CAAC;MACnD;MAEA,IAAI,IAAI,CAACjB,UAAU,GAAG,EAAE,KAAK,CAAC,EAAE;QAC9BT,MAAM,CAACsC,KAAK,CAAC,aAAa,EAAE,iBAAiB,IAAI,CAAC7B,UAAU,EAAE,CAAC;MACjE;IACF,CAAC,CAAC,OAAOO,KAAU,EAAE;MACnBhB,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAE,kBAAkBA,KAAK,CAACW,OAAO,EAAE,CAAC;IAChE;EACF;;EAEA;;EAEAY,IAAIA,CAAA,EAAS;IACX,IAAI,CAAChC,KAAK,GAAG,IAAI;IACjB,IAAI,IAAI,CAACD,QAAQ,EAAE;MACjB,IAAI,CAACA,QAAQ,CAACe,IAAI,CAACC,KAAK,GAAG,CAAC;IAC9B;IACAtB,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC;EAC7C;EAEAc,MAAMA,CAAA,EAAS;IACb,IAAI,CAACjC,KAAK,GAAG,KAAK;IAClB,IAAI,IAAI,CAACD,QAAQ,EAAE;MACjB,IAAI,CAACA,QAAQ,CAACe,IAAI,CAACC,KAAK,GAAG,GAAG;IAChC;IACAtB,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;EAC/C;EAEA,IAAIe,OAAOA,CAAA,EAAY;IACrB,OAAO,IAAI,CAAClC,KAAK;EACnB;;EAEA;;EAEA,MAAMmC,IAAIA,CAAA,EAAkB;IAC1B,IAAI;MACF,IAAI,IAAI,CAACrC,eAAe,IAAI,IAAI,CAACG,SAAS,EAAE;QAC1C,IAAI,CAACH,eAAe,CAACqC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAACrC,eAAe,CAACsC,YAAY,CAAC,CAAC;MACrC;MACA,IAAI,CAACnC,SAAS,GAAG,KAAK;MACtB,IAAI,CAACC,UAAU,GAAG,CAAC;MACnB,IAAI,CAACE,MAAM,CAACiC,aAAa,GAAG,CAAC;MAC7B5C,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC;IAChD,CAAC,CAAC,OAAOV,KAAU,EAAE;MACnBhB,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAE,eAAeA,KAAK,CAACW,OAAO,EAAE,CAAC;IAC7D;EACF;EAEA,MAAMkB,OAAOA,CAAA,EAAkB;IAC7B,MAAM,IAAI,CAACH,IAAI,CAAC,CAAC;IACjB,IAAI;MACF,IAAI,IAAI,CAACtC,YAAY,EAAE;QACrB,MAAM,IAAI,CAACA,YAAY,CAAC0C,KAAK,CAAC,CAAC;MACjC;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEF,IAAI,CAAC1C,YAAY,GAAG,IAAI;IACxB,IAAI,CAACC,eAAe,GAAG,IAAI;IAC3B,IAAI,CAACC,QAAQ,GAAG,IAAI;EACtB;AACF","ignoreList":[]}
@@ -0,0 +1,362 @@
1
+ "use strict";
2
+
3
+ /**
4
+ * VoiceService — WebSocket connection to Gemini Live API.
5
+ *
6
+ * Handles bidirectional audio streaming between the app and Gemini:
7
+ * - Sends PCM 16kHz 16-bit audio chunks (mic input)
8
+ * - Receives PCM 24kHz 16-bit audio chunks (AI responses)
9
+ * - Receives function calls (tap, navigate, etc.) for agentic actions
10
+ * - Sends screen context (DOM text + optional screenshot) for live mode
11
+ *
12
+ * Protocol: wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent
13
+ */
14
+
15
+ import { logger } from "../utils/logger.js";
16
+
17
+ // ─── Types ─────────────────────────────────────────────────────
18
+
19
+ // ─── Constants ─────────────────────────────────────────────────
20
+
21
+ const WS_HOST = 'generativelanguage.googleapis.com';
22
+ const WS_PATH = '/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
23
+ const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
24
+ const DEFAULT_INPUT_SAMPLE_RATE = 16000;
25
+
26
+ // ─── Service ───────────────────────────────────────────────────
27
+
28
+ export class VoiceService {
29
+ ws = null;
30
+ callbacks = {};
31
+ setupComplete = false;
32
+ _status = 'disconnected';
33
+ constructor(config) {
34
+ this.config = config;
35
+ }
36
+
37
+ // ─── Connection ────────────────────────────────────────────
38
+
39
+ connect(callbacks) {
40
+ if (this.ws?.readyState === WebSocket.OPEN) {
41
+ logger.info('VoiceService', 'Already connected');
42
+ return;
43
+ }
44
+ this.callbacks = callbacks;
45
+ this.setStatus('connecting');
46
+ const model = this.config.model || DEFAULT_MODEL;
47
+ const url = `wss://${WS_HOST}${WS_PATH}?key=${this.config.apiKey}`;
48
+ logger.info('VoiceService', `Connecting to Gemini Live API (model: ${model})`);
49
+ this.ws = new WebSocket(url);
50
+ this.ws.onopen = () => {
51
+ logger.info('VoiceService', 'WebSocket connected, sending setup...');
52
+ this.sendSetup();
53
+ };
54
+ this.ws.onclose = event => {
55
+ logger.info('VoiceService', `WebSocket closed: ${event.code} ${event.reason}`);
56
+ this.setStatus('disconnected');
57
+ this.setupComplete = false;
58
+ };
59
+ this.ws.onerror = error => {
60
+ logger.error('VoiceService', `WebSocket error: ${error.message || 'Unknown'}`);
61
+ this.setStatus('error');
62
+ this.callbacks.onError?.(error.message || 'WebSocket connection error');
63
+ };
64
+ this.ws.onmessage = event => {
65
+ this.handleMessage(event);
66
+ };
67
+ }
68
+ disconnect() {
69
+ if (this.ws) {
70
+ logger.info('VoiceService', 'Disconnecting...');
71
+ this.ws.close();
72
+ this.ws = null;
73
+ this.setupComplete = false;
74
+ this.setStatus('disconnected');
75
+ }
76
+ }
77
+ get isConnected() {
78
+ return this.ws?.readyState === WebSocket.OPEN && this.setupComplete;
79
+ }
80
+ get currentStatus() {
81
+ return this._status;
82
+ }
83
+
84
+ // ─── Send Audio ────────────────────────────────────────────
85
+
86
+ /** Send PCM audio chunk (base64 encoded) to Gemini */
87
+ sendCount = 0;
88
+ sendAudio(base64Audio) {
89
+ this.sendCount++;
90
+ if (!this.isConnected) {
91
+ logger.warn('VoiceService', `sendAudio #${this.sendCount} DROPPED — not connected (ws=${this.ws?.readyState}, setup=${this.setupComplete})`);
92
+ return;
93
+ }
94
+ const message = {
95
+ realtimeInput: {
96
+ audio: {
97
+ mimeType: `audio/pcm;rate=${this.config.inputSampleRate || DEFAULT_INPUT_SAMPLE_RATE}`,
98
+ data: base64Audio
99
+ }
100
+ }
101
+ };
102
+ logger.info('VoiceService', `📤 #${this.sendCount} sending ${base64Audio.length} chars (ws=${this.ws?.readyState})`);
103
+ this.ws.send(JSON.stringify(message));
104
+ }
105
+
106
+ // ─── Send Text ─────────────────────────────────────────────
107
+
108
+ /** Send text message via realtimeInput (same channel as audio) */
109
+ sendText(text) {
110
+ if (!this.isConnected) return;
111
+ const message = {
112
+ realtimeInput: {
113
+ text
114
+ }
115
+ };
116
+ this.ws.send(JSON.stringify(message));
117
+ }
118
+
119
+ /** Send DOM tree as passive context during live conversation.
120
+ *
121
+ * Uses `clientContent` with `turnComplete: false` to inject context
122
+ * WITHOUT triggering a model response. This is the "incremental content
123
+ * updates" pattern from the Gemini docs for establishing session context.
124
+ *
125
+ * Called once at connect + after each tool call (not on a timer).
126
+ * Screenshots are handled separately via the capture_screenshot tool.
127
+ */
128
+ sendScreenContext(domText) {
129
+ if (!this.isConnected) return;
130
+ const message = {
131
+ clientContent: {
132
+ turns: [{
133
+ role: 'user',
134
+ parts: [{
135
+ text: domText
136
+ }]
137
+ }],
138
+ turnComplete: false // Passive context — don't trigger a response
139
+ }
140
+ };
141
+ this.ws.send(JSON.stringify(message));
142
+ logger.debug('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
143
+ }
144
+
145
+ // ─── Send Function Response ────────────────────────────────
146
+
147
+ /** Send function call result back to Gemini */
148
+ sendFunctionResponse(name, id, result) {
149
+ if (!this.isConnected) return;
150
+ const message = {
151
+ toolResponse: {
152
+ functionResponses: [{
153
+ name,
154
+ id,
155
+ response: result
156
+ }]
157
+ }
158
+ };
159
+ logger.info('VoiceService', `📤 Sending tool response for ${name} (id=${id})`);
160
+ this.ws.send(JSON.stringify(message));
161
+ }
162
+
163
+ // ─── Internal: Setup ───────────────────────────────────────
164
+
165
+ sendSetup() {
166
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
167
+ const model = this.config.model || DEFAULT_MODEL;
168
+ const setup = {
169
+ model: `models/${model}`,
170
+ generationConfig: {
171
+ responseModalities: ['AUDIO']
172
+ }
173
+ };
174
+ if (this.config.language) {
175
+ setup.generationConfig.speechConfig = {
176
+ languageCode: this.config.language === 'ar' ? 'ar-SA' : 'en-US'
177
+ };
178
+ }
179
+
180
+ // Add system instruction if provided
181
+ if (this.config.systemPrompt) {
182
+ setup.systemInstruction = {
183
+ parts: [{
184
+ text: this.config.systemPrompt
185
+ }]
186
+ };
187
+ }
188
+
189
+ // Add tool declarations for function calling
190
+ if (this.config.tools?.length) {
191
+ setup.tools = [{
192
+ functionDeclarations: this.config.tools.map(tool => ({
193
+ name: tool.name,
194
+ description: tool.description,
195
+ parameters: {
196
+ type: 'OBJECT',
197
+ properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => [key, {
198
+ type: param.type.toUpperCase(),
199
+ description: param.description
200
+ }])),
201
+ required: Object.entries(tool.parameters).filter(([, param]) => param.required).map(([key]) => key)
202
+ }
203
+ }))
204
+ }];
205
+ }
206
+
207
+ // Enable transcription
208
+ setup.inputAudioTranscription = {};
209
+ setup.outputAudioTranscription = {};
210
+ const setupMessage = {
211
+ setup
212
+ };
213
+ logger.info('VoiceService', `Sending setup (model: ${model}, tools: ${this.config.tools?.length || 0})`);
214
+ this.ws.send(JSON.stringify(setupMessage));
215
+ }
216
+
217
+ // ─── Internal: Message Handling ────────────────────────────
218
+
219
+ handleMessage(event) {
220
+ try {
221
+ const dataType = typeof event.data;
222
+ const dataLen = typeof event.data === 'string' ? event.data.length : event.data?.byteLength || 'unknown';
223
+ logger.info('VoiceService', `📥 WS message received: type=${dataType}, length=${dataLen}`);
224
+
225
+ // Handle binary data (could be JSON or raw PCM)
226
+ if (typeof event.data !== 'string') {
227
+ logger.info('VoiceService', '📥 Binary message — processing...');
228
+ this.handleBinaryMessage(event.data);
229
+ return;
230
+ }
231
+
232
+ // Handle JSON text messages
233
+ const message = JSON.parse(event.data);
234
+ logger.info('VoiceService', `📥 JSON message keys: ${Object.keys(message).join(', ')}`);
235
+ this.processMessage(message);
236
+ } catch (error) {
237
+ logger.error('VoiceService', `Error handling message: ${error.message}`);
238
+ }
239
+ }
240
+ handleBinaryMessage(data) {
241
+ try {
242
+ // Try to decode as JSON first
243
+ let bytes;
244
+ if (data instanceof ArrayBuffer) {
245
+ bytes = new Uint8Array(data);
246
+ } else if (data instanceof Blob) {
247
+ // Blob handling — read as ArrayBuffer
248
+ const reader = new FileReader();
249
+ reader.onload = () => {
250
+ if (reader.result instanceof ArrayBuffer) {
251
+ this.processBinaryBytes(new Uint8Array(reader.result));
252
+ }
253
+ };
254
+ reader.readAsArrayBuffer(data);
255
+ return;
256
+ } else {
257
+ return;
258
+ }
259
+ this.processBinaryBytes(bytes);
260
+ } catch (error) {
261
+ logger.error('VoiceService', `Error handling binary message: ${error.message}`);
262
+ }
263
+ }
264
+ processBinaryBytes(bytes) {
265
+ // Check if it looks like JSON (starts with '{' or '[')
266
+ const looksLikeJson = bytes.length > 0 && (bytes[0] === 123 || bytes[0] === 91);
267
+ if (looksLikeJson) {
268
+ try {
269
+ const text = new TextDecoder('utf-8').decode(bytes);
270
+ const message = JSON.parse(text);
271
+ this.processMessage(message);
272
+ } catch {
273
+ // Not JSON — treat as raw PCM audio
274
+ this.callbacks.onAudioResponse?.(this.arrayBufferToBase64(bytes.buffer));
275
+ }
276
+ } else {
277
+ // Raw PCM audio data
278
+ this.callbacks.onAudioResponse?.(this.arrayBufferToBase64(bytes.buffer));
279
+ }
280
+ }
281
+ processMessage(message) {
282
+ // Setup complete acknowledgment
283
+ if (message.setupComplete !== undefined) {
284
+ logger.info('VoiceService', '✅ Setup complete — ready for audio exchange');
285
+ this.setupComplete = true;
286
+ this.setStatus('connected');
287
+ return;
288
+ }
289
+
290
+ // Server content (audio response + transcripts)
291
+ if (message.serverContent) {
292
+ const content = message.serverContent;
293
+ logger.info('VoiceService', `📥 serverContent received — turnComplete=${content.turnComplete}, hasParts=${!!content.modelTurn?.parts}, inputTranscription=${!!content.inputTranscription}, outputTranscription=${!!content.outputTranscription}`);
294
+
295
+ // Check for turn complete
296
+ if (content.turnComplete) {
297
+ this.callbacks.onTurnComplete?.();
298
+ }
299
+
300
+ // Process model output parts
301
+ if (content.modelTurn?.parts) {
302
+ for (const part of content.modelTurn.parts) {
303
+ // Audio response
304
+ if (part.inlineData?.data) {
305
+ logger.info('VoiceService', `🔊 Audio response: ${part.inlineData.data.length} chars`);
306
+ this.callbacks.onAudioResponse?.(part.inlineData.data);
307
+ }
308
+
309
+ // Text response (transcript)
310
+ if (part.text) {
311
+ logger.info('VoiceService', `💬 Text response: "${part.text}"`);
312
+ this.callbacks.onTranscript?.(part.text, true, 'model');
313
+ }
314
+ }
315
+ }
316
+
317
+ // Input transcription (user's speech)
318
+ if (content.inputTranscription?.text) {
319
+ this.callbacks.onTranscript?.(content.inputTranscription.text, true, 'user');
320
+ }
321
+
322
+ // Output transcription (model's speech-to-text)
323
+ if (content.outputTranscription?.text) {
324
+ this.callbacks.onTranscript?.(content.outputTranscription.text, true, 'model');
325
+ }
326
+ }
327
+
328
+ // Tool calls from the model
329
+ if (message.toolCall?.functionCalls) {
330
+ for (const fn of message.toolCall.functionCalls) {
331
+ logger.info('VoiceService', `Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
332
+ this.callbacks.onToolCall?.({
333
+ name: fn.name,
334
+ args: fn.args || {},
335
+ id: fn.id
336
+ });
337
+ }
338
+ }
339
+
340
+ // Error messages
341
+ if (message.error) {
342
+ logger.error('VoiceService', `Server error: ${JSON.stringify(message.error)}`);
343
+ this.callbacks.onError?.(message.error.message || 'Server error');
344
+ }
345
+ }
346
+
347
+ // ─── Helpers ───────────────────────────────────────────────
348
+
349
+ setStatus(newStatus) {
350
+ this._status = newStatus;
351
+ this.callbacks.onStatusChange?.(newStatus);
352
+ }
353
+ arrayBufferToBase64(buffer) {
354
+ const bytes = new Uint8Array(buffer);
355
+ let binary = '';
356
+ for (let i = 0; i < bytes.byteLength; i++) {
357
+ binary += String.fromCharCode(bytes[i]);
358
+ }
359
+ return btoa(binary);
360
+ }
361
+ }
362
+ //# sourceMappingURL=VoiceService.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"names":["logger","WS_HOST","WS_PATH","DEFAULT_MODEL","DEFAULT_INPUT_SAMPLE_RATE","VoiceService","ws","callbacks","setupComplete","_status","constructor","config","connect","readyState","WebSocket","OPEN","info","setStatus","model","url","apiKey","onopen","sendSetup","onclose","event","code","reason","onerror","error","message","onError","onmessage","handleMessage","disconnect","close","isConnected","currentStatus","sendCount","sendAudio","base64Audio","warn","realtimeInput","audio","mimeType","inputSampleRate","data","length","send","JSON","stringify","sendText","text","sendScreenContext","domText","clientContent","turns","role","parts","turnComplete","debug","sendFunctionResponse","name","id","result","toolResponse","functionResponses","response","setup","generationConfig","responseModalities","language","speechConfig","languageCode","systemPrompt","systemInstruction","tools","functionDeclarations","map","tool","description","parameters","type","properties","Object","fromEntries","entries","key","param","toUpperCase","required","filter","inputAudioTranscription","outputAudioTranscription","setupMessage","dataType","dataLen","byteLength","handleBinaryMessage","parse","keys","join","processMessage","bytes","ArrayBuffer","Uint8Array","Blob","reader","FileReader","onload","processBinaryBytes","readAsArrayBuffer","looksLikeJson","TextDecoder","decode","onAudioResponse","arrayBufferToBase64","buffer","undefined","serverContent","content","modelTurn","inputTranscription","outputTranscription","onTurnComplete","part","inlineData","onTranscript","toolCall","functionCalls","fn","args","onToolCall","newStatus","onStatusChange","binary","i","String","fromCharCode","btoa"],"sourceRoot":"../../../src","sources":["services/VoiceService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;;AAGxC;;AAyBA;;AAEA,MAAMC,OAAO,GAAG,mCAAmC;AACnD,MAAMC,OAAO,GAAG,+EAA+E;AAC/F,MAAMC,aAAa,GAAG,+CAA+C;AACrE,MAAMC,yBAAyB,GAAG,KAAK;;AAEvC;;AAEA,OAAO,MAAMC,YAAY,CAAC;EAChBC,EAAE,GAAqB,IAAI;EAE3BC,SAAS,GAA0B,CAAC,CAAC;EACrCC,aAAa,GAAG,KAAK;EACrBC,OAAO,GAAgB,cAAc;EAE7CC,WAAWA,CAACC,MAA0B,EAAE;IACtC,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEAC,OAAOA,CAACL,SAAgC,EAAQ;IAC9C,IAAI,IAAI,CAACD,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;MAC1Cf,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC;MAChD;IACF;IAEA,IAAI,CAACT,SAAS,GAAGA,SAAS;IAC1B,IAAI,CAACU,SAAS,CAAC,YAAY,CAAC;IAE5B,MAAMC,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAChD,MAAMgB,GAAG,GAAG,SAASlB,OAAO,GAAGC,OAAO,QAAQ,IAAI,CAACS,MAAM,CAACS,MAAM,EAAE;IAElEpB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yCAAyCE,KAAK,GAAG,CAAC;IAC9E,IAAI,CAACZ,EAAE,GAAG,IAAIQ,SAAS,CAACK,GAAG,CAAC;IAE5B,IAAI,CAACb,EAAE,CAACe,MAAM,GAAG,MAAM;MACrBrB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,uCAAuC,CAAC;MACpE,IAAI,CAACM,SAAS,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAChB,EAAE,CAACiB,OAAO,GAAIC,KAAK,IAAK;MAC3BxB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,qBAAqBQ,KAAK,CAACC,IAAI,IAAID,KAAK,CAACE,MAAM,EAAE,CAAC;MAC9E,IAAI,CAACT,SAAS,CAAC,cAAc,CAAC;MAC9B,IAAI,CAACT,aAAa,GAAG,KAAK;IAC5B,CAAC;IAED,IAAI,CAACF,EAAE,CAACqB,OAAO,GAAIC,KAAU,IAAK;MAChC5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,oBAAoBA,KAAK,CAACC,OAAO,IAAI,SAAS,EAAE,CAAC;MAC9E,IAAI,CAACZ,SAAS,CAAC,OAAO,CAAC;MACvB,IAAI,CAACV,SAAS,CAACuB,OAAO,GAAGF,KAAK,CAACC,OAAO,IAAI,4BAA4B,CAAC;IACzE,CAAC;IAED,IAAI,CAACvB,EAAE,CAACyB,SAAS,GAAIP,KAAK,IAAK;MAC7B,IAAI,CAACQ,aAAa,CAACR,KAAK,CAAC;IAC3B,CAAC;EACH;EAEAS,UAAUA,CAAA,EAAS;IACjB,IAAI,IAAI,CAAC3B,EAAE,EAAE;MACXN,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;MAC/C,IAAI,CAACV,EAAE,CAAC4B,KAAK,CAAC,CAAC;MACf,IAAI,CAAC5B,EAAE,GAAG,IAAI;MACd,IAAI,CAACE,aAAa,GAAG,KAAK;MAC1B,IAAI,CAACS,SAAS,CAAC,cAAc,CAAC;IAChC;EACF;EAEA,IAAIkB,WAAWA,CAAA,EAAY;IACzB,OAAO,IAAI,CAAC7B,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,IAAI,IAAI,CAACP,aAAa;EACrE;EAEA,IAAI4B,aAAaA,CAAA,EAAgB;IAC/B,OAAO,IAAI,CAAC3B,OAAO;EACrB;;EAEA;;EAEA;EACQ4B,SAAS,GAAG,CAAC;EACrBC,SAASA,CAACC,WAAmB,EAAQ;IACnC,IAAI,CAACF,SAAS,EAAE;IAChB,IAAI,CAAC,IAAI,CAACF,WAAW,EAAE;MACrBnC,MAAM,CAACwC,IAAI,CAAC,cAAc,EAAE,cAAc,IAAI,CAACH,SAAS,gCAAgC,IAAI,CAAC/B,EAAE,EAAEO,UAAU,WAAW,IAAI,CAACL,aAAa,GAAG,CAAC;MAC5I;IACF;IAEA,MAAMqB,OAAO,GAAG;MACdY,aAAa,EAAE;QACbC,KAAK,EAAE;UACLC,QAAQ,EAAE,kBAAkB,IAAI,CAAChC,MAAM,CAACiC,eAAe,IAAIxC,yBAAyB,EAAE;UACtFyC,IAAI,EAAEN;QACR;MACF;IACF,CAAC;IAEDvC,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,OAAO,IAAI,CAACqB,SAAS,YAAYE,WAAW,CAACO,MAAM,cAAc,IAAI,CAACxC,EAAE,EAAEO,UAAU,GAAG,CAAC;IACpH,IAAI,CAACP,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEA;EACAqB,QAAQA,CAACC,IAAY,EAAQ;IAC3B,IAAI,CAAC,IAAI,CAAChB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdY,aAAa,EAAE;QAAEU;MAAK;IACxB,CAAC;IAED,IAAI,CAAC7C,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEuB,iBAAiBA,CAACC,OAAe,EAAQ;IACvC,IAAI,CAAC,IAAI,CAAClB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdyB,aAAa,EAAE;QACbC,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAE,MAAM;UAAEC,KAAK,EAAE,CAAC;YAAEN,IAAI,EAAEE;UAAQ,CAAC;QAAE,CAAC,CAAC;QACrDK,YAAY,EAAE,KAAK,CAAE;MACvB;IACF,CAAC;IAED,IAAI,CAACpD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;IACtC7B,MAAM,CAAC2D,KAAK,CAAC,cAAc,EAAE,2BAA2BN,OAAO,CAACP,MAAM,SAAS,CAAC;EAClF;;EAEA;;EAEA;EACAc,oBAAoBA,CAACC,IAAY,EAAEC,EAAU,EAAEC,MAAW,EAAQ;IAChE,IAAI,CAAC,IAAI,CAAC5B,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdmC,YAAY,EAAE;QACZC,iBAAiB,EAAE,CAAC;UAClBJ,IAAI;UACJC,EAAE;UACFI,QAAQ,EAAEH;QACZ,CAAC;MACH;IACF,CAAC;IAED/D,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC6C,IAAI,QAAQC,EAAE,GAAG,CAAC;IAC9E,IAAI,CAACxD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEQP,SAASA,CAAA,EAAS;IACxB,IAAI,CAAC,IAAI,CAAChB,EAAE,IAAI,IAAI,CAACA,EAAE,CAACO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;IAEvD,MAAMG,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAEhD,MAAMgE,KAAU,GAAG;MACjBjD,KAAK,EAAE,UAAUA,KAAK,EAAE;MACxBkD,gBAAgB,EAAE;QAChBC,kBAAkB,EAAE,CAAC,OAAO;MAC9B;IACF,CAAC;IAED,IAAI,IAAI,CAAC1D,MAAM,CAAC2D,QAAQ,EAAE;MACxBH,KAAK,CAACC,gBAAgB,CAACG,YAAY,GAAG;QACpCC,YAAY,EAAE,IAAI,CAAC7D,MAAM,CAAC2D,QAAQ,KAAK,IAAI,GAAG,OAAO,GAAG;MAC1D,CAAC;IACH;;IAEA;IACA,IAAI,IAAI,CAAC3D,MAAM,CAAC8D,YAAY,EAAE;MAC5BN,KAAK,CAACO,iBAAiB,GAAG;QACxBjB,KAAK,EAAE,CAAC;UAAEN,IAAI,EAAE,IAAI,CAACxC,MAAM,CAAC8D;QAAa,CAAC;MAC5C,CAAC;IACH;;IAEA;IACA,IAAI,IAAI,CAAC9D,MAAM,CAACgE,KAAK,EAAE7B,MAAM,EAAE;MAC7BqB,KAAK,CAACQ,KAAK,GAAG,CAAC;QACbC,oBAAoB,EAAE,IAAI,CAACjE,MAAM,CAACgE,KAAK,CAACE,GAAG,CAACC,IAAI,KAAK;UACnDjB,IAAI,EAAEiB,IAAI,CAACjB,IAAI;UACfkB,WAAW,EAAED,IAAI,CAACC,WAAW;UAC7BC,UAAU,EAAE;YACVC,IAAI,EAAE,QAAQ;YACdC,UAAU,EAAEC,MAAM,CAACC,WAAW,CAC5BD,MAAM,CAACE,OAAO,CAACP,IAAI,CAACE,UAAU,CAAC,CAACH,GAAG,CAAC,CAAC,CAACS,GAAG,EAAEC,KAAK,CAAC,KAAK,CACpDD,GAAG,EACH;cACEL,IAAI,EAAEM,KAAK,CAACN,IAAI,CAACO,WAAW,CAAC,CAAC;cAC9BT,WAAW,EAAEQ,KAAK,CAACR;YACrB,CAAC,CACF,CACH,CAAC;YACDU,QAAQ,EAAEN,MAAM,CAACE,OAAO,CAACP,IAAI,CAACE,UAAU,CAAC,CACtCU,MAAM,CAAC,CAAC,GAAGH,KAAK,CAAC,KAAKA,KAAK,CAACE,QAAQ,CAAC,CACrCZ,GAAG,CAAC,CAAC,CAACS,GAAG,CAAC,KAAKA,GAAG;UACvB;QACF,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACAnB,KAAK,CAACwB,uBAAuB,GAAG,CAAC,CAAC;IAClCxB,KAAK,CAACyB,wBAAwB,GAAG,CAAC,CAAC;IAEnC,MAAMC,YAAY,GAAG;MAAE1B;IAAM,CAAC;IAC9BnE,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBE,KAAK,YAAY,IAAI,CAACP,MAAM,CAACgE,KAAK,EAAE7B,MAAM,IAAI,CAAC,GAAG,CAAC;IACxG,IAAI,CAACxC,EAAE,CAACyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAAC4C,YAAY,CAAC,CAAC;EAC5C;;EAEA;;EAEQ7D,aAAaA,CAACR,KAA4B,EAAQ;IACxD,IAAI;MACF,MAAMsE,QAAQ,GAAG,OAAOtE,KAAK,CAACqB,IAAI;MAClC,MAAMkD,OAAO,GAAG,OAAOvE,KAAK,CAACqB,IAAI,KAAK,QAAQ,GAAGrB,KAAK,CAACqB,IAAI,CAACC,MAAM,GAAItB,KAAK,CAACqB,IAAI,EAAEmD,UAAU,IAAI,SAAU;MAC1GhG,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC8E,QAAQ,YAAYC,OAAO,EAAE,CAAC;;MAE1F;MACA,IAAI,OAAOvE,KAAK,CAACqB,IAAI,KAAK,QAAQ,EAAE;QAClC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mCAAmC,CAAC;QAChE,IAAI,CAACiF,mBAAmB,CAACzE,KAAK,CAACqB,IAAI,CAAC;QACpC;MACF;;MAEA;MACA,MAAMhB,OAAO,GAAGmB,IAAI,CAACkD,KAAK,CAAC1E,KAAK,CAACqB,IAAI,CAAC;MACtC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBmE,MAAM,CAACgB,IAAI,CAACtE,OAAO,CAAC,CAACuE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;MACvF,IAAI,CAACC,cAAc,CAACxE,OAAO,CAAC;IAC9B,CAAC,CAAC,OAAOD,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,2BAA2BA,KAAK,CAACC,OAAO,EAAE,CAAC;IAC1E;EACF;EAEQoE,mBAAmBA,CAACpD,IAAS,EAAQ;IAC3C,IAAI;MACF;MACA,IAAIyD,KAAiB;MACrB,IAAIzD,IAAI,YAAY0D,WAAW,EAAE;QAC/BD,KAAK,GAAG,IAAIE,UAAU,CAAC3D,IAAI,CAAC;MAC9B,CAAC,MAAM,IAAIA,IAAI,YAAY4D,IAAI,EAAE;QAC/B;QACA,MAAMC,MAAM,GAAG,IAAIC,UAAU,CAAC,CAAC;QAC/BD,MAAM,CAACE,MAAM,GAAG,MAAM;UACpB,IAAIF,MAAM,CAAC3C,MAAM,YAAYwC,WAAW,EAAE;YACxC,IAAI,CAACM,kBAAkB,CAAC,IAAIL,UAAU,CAACE,MAAM,CAAC3C,MAAM,CAAC,CAAC;UACxD;QACF,CAAC;QACD2C,MAAM,CAACI,iBAAiB,CAACjE,IAAI,CAAC;QAC9B;MACF,CAAC,MAAM;QACL;MACF;MAEA,IAAI,CAACgE,kBAAkB,CAACP,KAAK,CAAC;IAChC,CAAC,CAAC,OAAO1E,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,kCAAkCA,KAAK,CAACC,OAAO,EAAE,CAAC;IACjF;EACF;EAEQgF,kBAAkBA,CAACP,KAAiB,EAAQ;IAClD;IACA,MAAMS,aAAa,GAAGT,KAAK,CAACxD,MAAM,GAAG,CAAC,KAAKwD,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAIA,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAE/E,IAAIS,aAAa,EAAE;MACjB,IAAI;QACF,MAAM5D,IAAI,GAAG,IAAI6D,WAAW,CAAC,OAAO,CAAC,CAACC,MAAM,CAACX,KAAK,CAAC;QACnD,MAAMzE,OAAO,GAAGmB,IAAI,CAACkD,KAAK,CAAC/C,IAAI,CAAC;QAChC,IAAI,CAACkD,cAAc,CAACxE,OAAO,CAAC;MAC9B,CAAC,CAAC,MAAM;QACN;QACA,IAAI,CAACtB,SAAS,CAAC2G,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;MACzF;IACF,CAAC,MAAM;MACL;MACA,IAAI,CAAC7G,SAAS,CAAC2G,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;IACzF;EACF;EAEQf,cAAcA,CAACxE,OAAY,EAAQ;IACzC;IACA,IAAIA,OAAO,CAACrB,aAAa,KAAK6G,SAAS,EAAE;MACvCrH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,6CAA6C,CAAC;MAC1E,IAAI,CAACR,aAAa,GAAG,IAAI;MACzB,IAAI,CAACS,SAAS,CAAC,WAAW,CAAC;MAC3B;IACF;;IAEA;IACA,IAAIY,OAAO,CAACyF,aAAa,EAAE;MACzB,MAAMC,OAAO,GAAG1F,OAAO,CAACyF,aAAa;MACrCtH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,4CAA4CuG,OAAO,CAAC7D,YAAY,cAAc,CAAC,CAAC6D,OAAO,CAACC,SAAS,EAAE/D,KAAK,wBAAwB,CAAC,CAAC8D,OAAO,CAACE,kBAAkB,yBAAyB,CAAC,CAACF,OAAO,CAACG,mBAAmB,EAAE,CAAC;;MAEjP;MACA,IAAIH,OAAO,CAAC7D,YAAY,EAAE;QACxB,IAAI,CAACnD,SAAS,CAACoH,cAAc,GAAG,CAAC;MACnC;;MAEA;MACA,IAAIJ,OAAO,CAACC,SAAS,EAAE/D,KAAK,EAAE;QAC5B,KAAK,MAAMmE,IAAI,IAAIL,OAAO,CAACC,SAAS,CAAC/D,KAAK,EAAE;UAC1C;UACA,IAAImE,IAAI,CAACC,UAAU,EAAEhF,IAAI,EAAE;YACzB7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsB4G,IAAI,CAACC,UAAU,CAAChF,IAAI,CAACC,MAAM,QAAQ,CAAC;YACtF,IAAI,CAACvC,SAAS,CAAC2G,eAAe,GAAGU,IAAI,CAACC,UAAU,CAAChF,IAAI,CAAC;UACxD;;UAEA;UACA,IAAI+E,IAAI,CAACzE,IAAI,EAAE;YACbnD,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsB4G,IAAI,CAACzE,IAAI,GAAG,CAAC;YAC/D,IAAI,CAAC5C,SAAS,CAACuH,YAAY,GAAGF,IAAI,CAACzE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;UACzD;QACF;MACF;;MAEA;MACA,IAAIoE,OAAO,CAACE,kBAAkB,EAAEtE,IAAI,EAAE;QACpC,IAAI,CAAC5C,SAAS,CAACuH,YAAY,GAAGP,OAAO,CAACE,kBAAkB,CAACtE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;MAC9E;;MAEA;MACA,IAAIoE,OAAO,CAACG,mBAAmB,EAAEvE,IAAI,EAAE;QACrC,IAAI,CAAC5C,SAAS,CAACuH,YAAY,GAAGP,OAAO,CAACG,mBAAmB,CAACvE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;MAChF;IACF;;IAEA;IACA,IAAItB,OAAO,CAACkG,QAAQ,EAAEC,aAAa,EAAE;MACnC,KAAK,MAAMC,EAAE,IAAIpG,OAAO,CAACkG,QAAQ,CAACC,aAAa,EAAE;QAC/ChI,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,cAAciH,EAAE,CAACpE,IAAI,IAAIb,IAAI,CAACC,SAAS,CAACgF,EAAE,CAACC,IAAI,CAAC,GAAG,CAAC;QAChF,IAAI,CAAC3H,SAAS,CAAC4H,UAAU,GAAG;UAC1BtE,IAAI,EAAEoE,EAAE,CAACpE,IAAI;UACbqE,IAAI,EAAED,EAAE,CAACC,IAAI,IAAI,CAAC,CAAC;UACnBpE,EAAE,EAAEmE,EAAE,CAACnE;QACT,CAAC,CAAC;MACJ;IACF;;IAEA;IACA,IAAIjC,OAAO,CAACD,KAAK,EAAE;MACjB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,iBAAiBoB,IAAI,CAACC,SAAS,CAACpB,OAAO,CAACD,KAAK,CAAC,EAAE,CAAC;MAC9E,IAAI,CAACrB,SAAS,CAACuB,OAAO,GAAGD,OAAO,CAACD,KAAK,CAACC,OAAO,IAAI,cAAc,CAAC;IACnE;EACF;;EAEA;;EAEQZ,SAASA,CAACmH,SAAsB,EAAQ;IAC9C,IAAI,CAAC3H,OAAO,GAAG2H,SAAS;IACxB,IAAI,CAAC7H,SAAS,CAAC8H,cAAc,GAAGD,SAAS,CAAC;EAC5C;EAEQjB,mBAAmBA,CAACC,MAAmB,EAAU;IACvD,MAAMd,KAAK,GAAG,IAAIE,UAAU,CAACY,MAAM,CAAC;IACpC,IAAIkB,MAAM,GAAG,EAAE;IACf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGjC,KAAK,CAACN,UAAU,EAAEuC,CAAC,EAAE,EAAE;MACzCD,MAAM,IAAIE,MAAM,CAACC,YAAY,CAACnC,KAAK,CAACiC,CAAC,CAAE,CAAC;IAC1C;IACA,OAAOG,IAAI,CAACJ,MAAM,CAAC;EACrB;AACF","ignoreList":[]}
@@ -0,0 +1,49 @@
1
+ "use strict";
2
+
3
+ /**
4
+ * Audio utility functions for PCM conversion.
5
+ *
6
+ * Used by AudioInputService and AudioOutputService to convert between
7
+ * Float32 (Web Audio API) and Int16 (Gemini Live API) PCM formats.
8
+ */
9
+
10
+ /**
11
+ * Convert Float32Array PCM samples to Int16 PCM and encode as base64.
12
+ * Gemini Live API expects Int16 little-endian PCM.
13
+ */
14
+ export function float32ToInt16Base64(float32Data) {
15
+ const int16Buffer = new Int16Array(float32Data.length);
16
+ for (let i = 0; i < float32Data.length; i++) {
17
+ // Clamp to [-1, 1] and scale to Int16 range
18
+ const sample = Math.max(-1, Math.min(1, float32Data[i] || 0));
19
+ int16Buffer[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
20
+ }
21
+
22
+ // Convert Int16Array to base64
23
+ const bytes = new Uint8Array(int16Buffer.buffer);
24
+ let binary = '';
25
+ for (let i = 0; i < bytes.length; i++) {
26
+ binary += String.fromCharCode(bytes[i] || 0);
27
+ }
28
+ return typeof global.btoa === 'function' ? global.btoa(binary) : Buffer.from(binary, 'binary').toString('base64');
29
+ }
30
+
31
+ /**
32
+ * Decode base64 Int16 PCM to Float32Array.
33
+ * Used for manual decoding when decodePCMInBase64 is unavailable.
34
+ */
35
+ export function base64ToFloat32(base64) {
36
+ const binaryString = typeof global.atob === 'function' ? global.atob(base64) : Buffer.from(base64, 'base64').toString('binary');
37
+ const bytes = new Uint8Array(binaryString.length);
38
+ for (let i = 0; i < binaryString.length; i++) {
39
+ bytes[i] = binaryString.charCodeAt(i);
40
+ }
41
+ const int16Data = new Int16Array(bytes.buffer);
42
+ const float32Data = new Float32Array(int16Data.length);
43
+ for (let i = 0; i < int16Data.length; i++) {
44
+ // Scale Int16 back to Float32 [-1, 1]
45
+ float32Data[i] = (int16Data[i] || 0) / 0x8000;
46
+ }
47
+ return float32Data;
48
+ }
49
+ //# sourceMappingURL=audioUtils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"names":["float32ToInt16Base64","float32Data","int16Buffer","Int16Array","length","i","sample","Math","max","min","bytes","Uint8Array","buffer","binary","String","fromCharCode","global","btoa","Buffer","from","toString","base64ToFloat32","base64","binaryString","atob","charCodeAt","int16Data","Float32Array"],"sourceRoot":"../../../src","sources":["utils/audioUtils.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACA;AACA;AACA;AACA,OAAO,SAASA,oBAAoBA,CAACC,WAAyB,EAAU;EACtE,MAAMC,WAAW,GAAG,IAAIC,UAAU,CAACF,WAAW,CAACG,MAAM,CAAC;EACtD,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGJ,WAAW,CAACG,MAAM,EAAEC,CAAC,EAAE,EAAE;IAC3C;IACA,MAAMC,MAAM,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC,CAAC,EAAED,IAAI,CAACE,GAAG,CAAC,CAAC,EAAER,WAAW,CAACI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7DH,WAAW,CAACG,CAAC,CAAC,GAAGC,MAAM,GAAG,CAAC,GAAGA,MAAM,GAAG,MAAM,GAAGA,MAAM,GAAG,MAAM;EACjE;;EAEA;EACA,MAAMI,KAAK,GAAG,IAAIC,UAAU,CAACT,WAAW,CAACU,MAAM,CAAC;EAChD,IAAIC,MAAM,GAAG,EAAE;EACf,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGK,KAAK,CAACN,MAAM,EAAEC,CAAC,EAAE,EAAE;IACrCQ,MAAM,IAAIC,MAAM,CAACC,YAAY,CAACL,KAAK,CAACL,CAAC,CAAC,IAAI,CAAC,CAAC;EAC9C;EAEA,OAAO,OAAOW,MAAM,CAACC,IAAI,KAAK,UAAU,GACpCD,MAAM,CAACC,IAAI,CAACJ,MAAM,CAAC,GACnBK,MAAM,CAACC,IAAI,CAACN,MAAM,EAAE,QAAQ,CAAC,CAACO,QAAQ,CAAC,QAAQ,CAAC;AACtD;;AAEA;AACA;AACA;AACA;AACA,OAAO,SAASC,eAAeA,CAACC,MAAc,EAAgB;EAC5D,MAAMC,YAAY,GAAG,OAAOP,MAAM,CAACQ,IAAI,KAAK,UAAU,GAClDR,MAAM,CAACQ,IAAI,CAACF,MAAM,CAAC,GACnBJ,MAAM,CAACC,IAAI,CAACG,MAAM,EAAE,QAAQ,CAAC,CAACF,QAAQ,CAAC,QAAQ,CAAC;EAEpD,MAAMV,KAAK,GAAG,IAAIC,UAAU,CAACY,YAAY,CAACnB,MAAM,CAAC;EACjD,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGkB,YAAY,CAACnB,MAAM,EAAEC,CAAC,EAAE,EAAE;IAC5CK,KAAK,CAACL,CAAC,CAAC,GAAGkB,YAAY,CAACE,UAAU,CAACpB,CAAC,CAAC;EACvC;EAEA,MAAMqB,SAAS,GAAG,IAAIvB,UAAU,CAACO,KAAK,CAACE,MAAM,CAAC;EAC9C,MAAMX,WAAW,GAAG,IAAI0B,YAAY,CAACD,SAAS,CAACtB,MAAM,CAAC;EACtD,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGqB,SAAS,CAACtB,MAAM,EAAEC,CAAC,EAAE,EAAE;IACzC;IACAJ,WAAW,CAACI,CAAC,CAAC,GAAG,CAACqB,SAAS,CAACrB,CAAC,CAAC,IAAI,CAAC,IAAI,MAAM;EAC/C;EAEA,OAAOJ,WAAW;AACpB","ignoreList":[]}
@@ -2,14 +2,31 @@
2
2
 
3
3
  /**
4
4
  * Logger utility — prefixed console output for easy filtering.
5
+ *
6
+ * Disabled by default. Enable via `logger.setEnabled(true)` or
7
+ * pass `debug={true}` to the <AIAgent> component.
5
8
  */
6
9
  const TAG = '[AIAgent]';
10
+ let enabled = false;
7
11
  export const logger = {
8
- info: (context, ...args) => console.log(`${TAG} [${context}]`, ...args),
9
- warn: (context, ...args) => console.warn(`${TAG} [${context}]`, ...args),
10
- error: (context, ...args) => console.error(`${TAG} [${context}]`, ...args),
12
+ /** Enable or disable all SDK logging. */
13
+ setEnabled: value => {
14
+ enabled = value;
15
+ },
16
+ /** Check if logging is enabled. */
17
+ isEnabled: () => enabled,
18
+ info: (context, ...args) => {
19
+ if (enabled) console.log(`${TAG} [${context}]`, ...args);
20
+ },
21
+ warn: (context, ...args) => {
22
+ if (enabled) console.warn(`${TAG} [${context}]`, ...args);
23
+ },
24
+ error: (context, ...args) => {
25
+ // Errors always log regardless of enabled flag
26
+ console.error(`${TAG} [${context}]`, ...args);
27
+ },
11
28
  debug: (context, ...args) => {
12
- if (__DEV__) {
29
+ if (enabled && __DEV__) {
13
30
  console.log(`${TAG} [${context}] 🐛`, ...args);
14
31
  }
15
32
  }
@@ -1 +1 @@
1
- {"version":3,"names":["TAG","logger","info","context","args","console","log","warn","error","debug","__DEV__"],"sourceRoot":"../../../src","sources":["utils/logger.ts"],"mappings":";;AAAA;AACA;AACA;AACA,MAAMA,GAAG,GAAG,WAAW;AAEvB,OAAO,MAAMC,MAAM,GAAG;EACpBC,IAAI,EAAEA,CAACC,OAAe,EAAE,GAAGC,IAAW,KACpCC,OAAO,CAACC,GAAG,CAAC,GAAGN,GAAG,KAAKG,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAE7CG,IAAI,EAAEA,CAACJ,OAAe,EAAE,GAAGC,IAAW,KACpCC,OAAO,CAACE,IAAI,CAAC,GAAGP,GAAG,KAAKG,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAE9CI,KAAK,EAAEA,CAACL,OAAe,EAAE,GAAGC,IAAW,KACrCC,OAAO,CAACG,KAAK,CAAC,GAAGR,GAAG,KAAKG,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAE/CK,KAAK,EAAEA,CAACN,OAAe,EAAE,GAAGC,IAAW,KAAK;IAC1C,IAAIM,OAAO,EAAE;MACXL,OAAO,CAACC,GAAG,CAAC,GAAGN,GAAG,KAAKG,OAAO,MAAM,EAAE,GAAGC,IAAI,CAAC;IAChD;EACF;AACF,CAAC","ignoreList":[]}
1
+ {"version":3,"names":["TAG","enabled","logger","setEnabled","value","isEnabled","info","context","args","console","log","warn","error","debug","__DEV__"],"sourceRoot":"../../../src","sources":["utils/logger.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMA,GAAG,GAAG,WAAW;AAEvB,IAAIC,OAAO,GAAG,KAAK;AAEnB,OAAO,MAAMC,MAAM,GAAG;EACpB;EACAC,UAAU,EAAGC,KAAc,IAAK;IAC9BH,OAAO,GAAGG,KAAK;EACjB,CAAC;EAED;EACAC,SAAS,EAAEA,CAAA,KAAMJ,OAAO;EAExBK,IAAI,EAAEA,CAACC,OAAe,EAAE,GAAGC,IAAW,KAAK;IACzC,IAAIP,OAAO,EAAEQ,OAAO,CAACC,GAAG,CAAC,GAAGV,GAAG,KAAKO,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAC1D,CAAC;EAEDG,IAAI,EAAEA,CAACJ,OAAe,EAAE,GAAGC,IAAW,KAAK;IACzC,IAAIP,OAAO,EAAEQ,OAAO,CAACE,IAAI,CAAC,GAAGX,GAAG,KAAKO,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAC3D,CAAC;EAEDI,KAAK,EAAEA,CAACL,OAAe,EAAE,GAAGC,IAAW,KAAK;IAC1C;IACAC,OAAO,CAACG,KAAK,CAAC,GAAGZ,GAAG,KAAKO,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAC/C,CAAC;EAEDK,KAAK,EAAEA,CAACN,OAAe,EAAE,GAAGC,IAAW,KAAK;IAC1C,IAAIP,OAAO,IAAIa,OAAO,EAAE;MACtBL,OAAO,CAACC,GAAG,CAAC,GAAGV,GAAG,KAAKO,OAAO,MAAM,EAAE,GAAGC,IAAI,CAAC;IAChD;EACF;AACF,CAAC","ignoreList":[]}
@@ -0,0 +1,10 @@
1
+ export let overrides: ({
2
+ exclude: RegExp;
3
+ presets: string[];
4
+ include?: undefined;
5
+ } | {
6
+ include: RegExp;
7
+ presets: string[];
8
+ exclude?: undefined;
9
+ })[];
10
+ //# sourceMappingURL=babel.config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"babel.config.d.ts","sourceRoot":"","sources":["../../babel.config.js"],"names":[],"mappings":""}
@@ -0,0 +1,3 @@
1
+ declare const _default: import("eslint/config").Config[];
2
+ export default _default;
3
+ //# sourceMappingURL=eslint.config.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eslint.config.d.mts","sourceRoot":"","sources":["../../eslint.config.mjs"],"names":[],"mappings":""}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=fetch-models.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fetch-models.d.mts","sourceRoot":"","sources":["../../fetch-models.mjs"],"names":[],"mappings":""}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=list-all-models.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"list-all-models.d.mts","sourceRoot":"","sources":["../../list-all-models.mjs"],"names":[],"mappings":""}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=list-models.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"list-models.d.mts","sourceRoot":"","sources":["../../list-models.mjs"],"names":[],"mappings":""}
@@ -8,7 +8,7 @@
8
8
  * - Agent runtime context for useAction hooks
9
9
  */
10
10
  import React from 'react';
11
- import type { ExecutionResult, ToolDefinition, AgentStep } from '../core/types';
11
+ import type { ExecutionResult, ToolDefinition, AgentStep, TokenUsage } from '../core/types';
12
12
  interface AIAgentProps {
13
13
  /** Gemini API key */
14
14
  apiKey: string;
@@ -59,7 +59,13 @@ interface AIAgentProps {
59
59
  };
60
60
  /** Expo Router pathname (from usePathname()) */
61
61
  pathname?: string;
62
+ /** Enable voice mode (requires expo-av) */
63
+ enableVoice?: boolean;
64
+ /** Called after each step with token usage data */
65
+ onTokenUsage?: (usage: TokenUsage) => void;
66
+ /** Enable SDK debug logging (disabled by default) */
67
+ debug?: boolean;
62
68
  }
63
- export declare function AIAgent({ apiKey, model, navRef, language, maxSteps, showChatBar, children, onResult, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, }: AIAgentProps): import("react/jsx-runtime").JSX.Element;
69
+ export declare function AIAgent({ apiKey, model, navRef, language, maxSteps, showChatBar, children, onResult, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, enableVoice, onTokenUsage, debug, }: AIAgentProps): import("react/jsx-runtime").JSX.Element;
64
70
  export {};
65
71
  //# sourceMappingURL=AIAgent.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;AASf,OAAO,KAAK,EAAe,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAO7F,UAAU,YAAY;IACpB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,kBAAkB;IAClB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IAI7C,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,8BAA8B;IAC9B,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3D,6BAA6B;IAC7B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1C,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAChE,qEAAqE;IACrE,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACvE,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IACpD,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IACF,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,MAAM,CAAC,EAAE;QACP,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAC7B,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAChC,IAAI,EAAE,MAAM,IAAI,CAAC;KAClB,CAAC;IACF,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID,wBAAgB,OAAO,CAAC,EACtB,MAAM,EACN,KAA0B,EAC1B,MAAM,EACN,QAAe,EACf,QAAa,EACb,WAAkB,EAClB,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,QAAQ,GACT,EAAE,YAAY,2CA+Id"}
1
+ {"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;AAaf,OAAO,KAAK,EAA0B,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAOpH,UAAU,YAAY;IACpB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,kBAAkB;IAClB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IAI7C,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,8BAA8B;IAC9B,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3D,6BAA6B;IAC7B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1C,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAChE,qEAAqE;IACrE,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACvE,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IACpD,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IACF,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,MAAM,CAAC,EAAE;QACP,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAC7B,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAChC,IAAI,EAAE,MAAM,IAAI,CAAC;KAClB,CAAC;IACF,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mDAAmD;IACnD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAC3C,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAID,wBAAgB,OAAO,CAAC,EACtB,MAAM,EACN,KAA0B,EAC1B,MAAM,EACN,QAAe,EACf,QAAa,EACb,WAAkB,EAClB,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,WAAmB,EACnB,YAAY,EACZ,KAAa,GACd,EAAE,YAAY,2CA4Vd"}
@@ -1,15 +1,32 @@
1
1
  /**
2
2
  * AgentChatBar — Floating, draggable, compressible chat widget.
3
+ * Supports two modes: Text and Voice.
3
4
  * Does not block underlying UI natively.
4
5
  */
5
- import type { ExecutionResult } from '../core/types';
6
+ import type { ExecutionResult, AgentMode } from '../core/types';
6
7
  interface AgentChatBarProps {
7
8
  onSend: (message: string) => void;
8
9
  isThinking: boolean;
9
10
  lastResult: ExecutionResult | null;
10
11
  language: 'en' | 'ar';
11
12
  onDismiss?: () => void;
13
+ /** Available modes (default: ['text']) */
14
+ availableModes?: AgentMode[];
15
+ /** Current active mode */
16
+ mode?: AgentMode;
17
+ onModeChange?: (mode: AgentMode) => void;
18
+ /** Voice controls */
19
+ onMicToggle?: (active: boolean) => void;
20
+ onSpeakerToggle?: (muted: boolean) => void;
21
+ isMicActive?: boolean;
22
+ isSpeakerMuted?: boolean;
23
+ /** AI is currently speaking */
24
+ isAISpeaking?: boolean;
25
+ /** Voice WebSocket is connected */
26
+ isVoiceConnected?: boolean;
27
+ /** Full session cleanup (stop mic, audio, WebSocket, live mode) */
28
+ onStopSession?: () => void;
12
29
  }
13
- export declare function AgentChatBar({ onSend, isThinking, lastResult, language, onDismiss }: AgentChatBarProps): import("react/jsx-runtime").JSX.Element;
30
+ export declare function AgentChatBar({ onSend, isThinking, lastResult, language, onDismiss, availableModes, mode, onModeChange, onMicToggle, onSpeakerToggle, isMicActive, isSpeakerMuted, isAISpeaking, isVoiceConnected, onStopSession, }: AgentChatBarProps): import("react/jsx-runtime").JSX.Element;
14
31
  export {};
15
32
  //# sourceMappingURL=AgentChatBar.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"AgentChatBar.d.ts","sourceRoot":"","sources":["../../../../src/components/AgentChatBar.tsx"],"names":[],"mappings":"AAAA;;;GAGG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,UAAU,iBAAiB;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;CACxB;AAED,wBAAgB,YAAY,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,iBAAiB,2CAyGtG"}
1
+ {"version":3,"file":"AgentChatBar.d.ts","sourceRoot":"","sources":["../../../../src/components/AgentChatBar.tsx"],"names":[],"mappings":"AAAA;;;;GAIG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAIhE,UAAU,iBAAiB;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;IACvB,0CAA0C;IAC1C,cAAc,CAAC,EAAE,SAAS,EAAE,CAAC;IAC7B,0BAA0B;IAC1B,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,CAAC;IACzC,qBAAqB;IACrB,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,+BAA+B;IAC/B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mEAAmE;IACnE,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;CAC5B;AA+SD,wBAAgB,YAAY,CAAC,EAC3B,MAAM,EACN,UAAU,EACV,UAAU,EACV,QAAQ,EACR,SAAS,EACT,cAAyB,EACzB,IAAa,EACb,YAAY,EACZ,WAAW,EACX,eAAe,EACf,WAAmB,EACnB,cAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,aAAa,GACd,EAAE,iBAAiB,2CAqHnB"}