@mobileai/react-native 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -2
- package/lib/module/components/AIAgent.js +216 -5
- package/lib/module/components/AIAgent.js.map +1 -1
- package/lib/module/components/AgentChatBar.js +358 -36
- package/lib/module/components/AgentChatBar.js.map +1 -1
- package/lib/module/core/AgentRuntime.js +122 -6
- package/lib/module/core/AgentRuntime.js.map +1 -1
- package/lib/module/core/systemPrompt.js +57 -0
- package/lib/module/core/systemPrompt.js.map +1 -1
- package/lib/module/index.js +8 -0
- package/lib/module/index.js.map +1 -1
- package/lib/module/providers/GeminiProvider.js +108 -85
- package/lib/module/providers/GeminiProvider.js.map +1 -1
- package/lib/module/services/AudioInputService.js +128 -0
- package/lib/module/services/AudioInputService.js.map +1 -0
- package/lib/module/services/AudioOutputService.js +154 -0
- package/lib/module/services/AudioOutputService.js.map +1 -0
- package/lib/module/services/VoiceService.js +362 -0
- package/lib/module/services/VoiceService.js.map +1 -0
- package/lib/module/utils/audioUtils.js +49 -0
- package/lib/module/utils/audioUtils.js.map +1 -0
- package/lib/module/utils/logger.js +21 -4
- package/lib/module/utils/logger.js.map +1 -1
- package/lib/typescript/babel.config.d.ts +10 -0
- package/lib/typescript/babel.config.d.ts.map +1 -0
- package/lib/typescript/eslint.config.d.mts +3 -0
- package/lib/typescript/eslint.config.d.mts.map +1 -0
- package/lib/typescript/fetch-models.d.mts +2 -0
- package/lib/typescript/fetch-models.d.mts.map +1 -0
- package/lib/typescript/list-all-models.d.mts +2 -0
- package/lib/typescript/list-all-models.d.mts.map +1 -0
- package/lib/typescript/list-models.d.mts +2 -0
- package/lib/typescript/list-models.d.mts.map +1 -0
- package/lib/typescript/src/components/AIAgent.d.ts +8 -2
- package/lib/typescript/src/components/AIAgent.d.ts.map +1 -1
- package/lib/typescript/src/components/AgentChatBar.d.ts +19 -2
- package/lib/typescript/src/components/AgentChatBar.d.ts.map +1 -1
- package/lib/typescript/src/core/AgentRuntime.d.ts +17 -1
- package/lib/typescript/src/core/AgentRuntime.d.ts.map +1 -1
- package/lib/typescript/src/core/systemPrompt.d.ts +8 -0
- package/lib/typescript/src/core/systemPrompt.d.ts.map +1 -1
- package/lib/typescript/src/core/types.d.ts +24 -1
- package/lib/typescript/src/core/types.d.ts.map +1 -1
- package/lib/typescript/src/index.d.ts +6 -1
- package/lib/typescript/src/index.d.ts.map +1 -1
- package/lib/typescript/src/providers/GeminiProvider.d.ts +22 -18
- package/lib/typescript/src/providers/GeminiProvider.d.ts.map +1 -1
- package/lib/typescript/src/services/AudioInputService.d.ts +31 -0
- package/lib/typescript/src/services/AudioInputService.d.ts.map +1 -0
- package/lib/typescript/src/services/AudioOutputService.d.ts +34 -0
- package/lib/typescript/src/services/AudioOutputService.d.ts.map +1 -0
- package/lib/typescript/src/services/VoiceService.d.ts +73 -0
- package/lib/typescript/src/services/VoiceService.d.ts.map +1 -0
- package/lib/typescript/src/utils/audioUtils.d.ts +17 -0
- package/lib/typescript/src/utils/audioUtils.d.ts.map +1 -0
- package/lib/typescript/src/utils/logger.d.ts +4 -0
- package/lib/typescript/src/utils/logger.d.ts.map +1 -1
- package/package.json +24 -8
- package/src/components/AIAgent.tsx +222 -3
- package/src/components/AgentChatBar.tsx +487 -42
- package/src/core/AgentRuntime.ts +131 -2
- package/src/core/systemPrompt.ts +62 -0
- package/src/core/types.ts +30 -0
- package/src/index.ts +16 -0
- package/src/providers/GeminiProvider.ts +105 -89
- package/src/services/AudioInputService.ts +141 -0
- package/src/services/AudioOutputService.ts +167 -0
- package/src/services/VoiceService.ts +409 -0
- package/src/utils/audioUtils.ts +54 -0
- package/src/utils/logger.ts +24 -7
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["logger","base64ToFloat32","GEMINI_OUTPUT_SAMPLE_RATE","AudioOutputService","audioContext","queueSourceNode","gainNode","muted","isStarted","chunkCount","constructor","config","initialize","audioApi","require","msg","error","onError","sampleRate","AudioContext","createGain","gain","value","connect","destination","createBufferQueueSource","info","message","enqueue","base64Audio","float32Data","audioBuffer","createBuffer","length","copyToChannel","enqueueBuffer","start","onPlaybackStart","debug","mute","unmute","isMuted","stop","clearBuffers","onPlaybackEnd","cleanup","close"],"sourceRoot":"../../../src","sources":["services/AudioOutputService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;AACxC,SAASC,eAAe,QAAQ,wBAAqB;;AAErD;;AAEA;AACA,MAAMC,yBAAyB,GAAG,KAAK;AASvC;;AAEA,OAAO,MAAMC,kBAAkB,CAAC;EAEtBC,YAAY,GAAQ,IAAI;EACxBC,eAAe,GAAQ,IAAI;EAC3BC,QAAQ,GAAQ,IAAI;EACpBC,KAAK,GAAG,KAAK;EACbC,SAAS,GAAG,KAAK;EACjBC,UAAU,GAAG,CAAC;EAEtBC,WAAWA,CAACC,MAAyB,GAAG,CAAC,CAAC,EAAE;IAC1C,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEA,MAAMC,UAAUA,CAAA,EAAqB;IACnC,IAAI;MACF,IAAIC,QAAa;MACjB,IAAI;QACFA,QAAQ,GAAGC,OAAO,CAAC,wBAAwB,CAAC;MAC9C,CAAC,CAAC,MAAM;QACN,MAAMC,GAAG,GACP,uGAAuG;QACzGf,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAED,GAAG,CAAC;QAChC,IAAI,CAACJ,MAAM,CAACM,OAAO,GAAGF,GAAG,CAAC;QAC1B,OAAO,KAAK;MACd;MAEA,MAAMG,UAAU,GAAG,IAAI,CAACP,MAAM,CAACO,UAAU,IAAIhB,yBAAyB;;MAEtE;MACA,IAAI,CAACE,YAAY,GAAG,IAAIS,QAAQ,CAACM,YAAY,CAAC;QAAED;MAAW,CAAC,CAAC;;MAE7D;MACA,IAAI,CAACZ,QAAQ,GAAG,IAAI,CAACF,YAAY,CAACgB,UAAU,CAAC,CAAC;MAC9C,IAAI,CAACd,QAAQ,CAACe,IAAI,CAACC,KAAK,GAAG,GAAG;MAC9B,IAAI,CAAChB,QAAQ,CAACiB,OAAO,CAAC,IAAI,CAACnB,YAAY,CAACoB,WAAW,CAAC;;MAEpD;MACA,IAAI,CAACnB,eAAe,GAAG,IAAI,CAACD,YAAY,CAACqB,uBAAuB,CAAC,CAAC;MAClE,IAAI,CAACpB,eAAe,CAACkB,OAAO,CAAC,IAAI,CAACjB,QAAQ,CAAC;MAE3CN,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,gBAAgBR,UAAU,iCAAiC,CAAC;MACvF,OAAO,IAAI;IACb,CAAC,CAAC,OAAOF,KAAU,EAAE;MACnBhB,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAE,yBAAyBA,KAAK,CAACW,OAAO,EAAE,CAAC;MACrE,IAAI,CAAChB,MAAM,CAACM,OAAO,GAAGD,KAAK,CAACW,OAAO,CAAC;MACpC,OAAO,KAAK;IACd;EACF;;EAEA;;EAEA;EACAC,OAAOA,CAACC,WAAmB,EAAQ;IACjC,IAAI,IAAI,CAACtB,KAAK,IAAI,CAAC,IAAI,CAACH,YAAY,IAAI,CAAC,IAAI,CAACC,eAAe,EAAE;IAE/D,IAAI;MACF,IAAI,CAACI,UAAU,EAAE;;MAEjB;MACA,MAAMqB,WAAW,GAAG7B,eAAe,CAAC4B,WAAW,CAAC;MAChD,MAAMX,UAAU,GAAG,IAAI,CAACP,MAAM,CAACO,UAAU,IAAIhB,yBAAyB;;MAEtE;MACA,MAAM6B,WAAW,GAAG,IAAI,CAAC3B,YAAY,CAAC4B,YAAY,CAAC,CAAC,EAAEF,WAAW,CAACG,MAAM,EAAEf,UAAU,CAAC;MACrFa,WAAW,CAACG,aAAa,CAACJ,WAAW,EAAE,CAAC,CAAC;;MAEzC;MACA,IAAI,CAACzB,eAAe,CAAC8B,aAAa,CAACJ,WAAW,CAAC;;MAE/C;MACA,IAAI,CAAC,IAAI,CAACvB,SAAS,EAAE;QACnB,IAAI,CAACH,eAAe,CAAC+B,KAAK,CAAC,CAAC;QAC5B,IAAI,CAAC5B,SAAS,GAAG,IAAI;QACrB,IAAI,CAACG,MAAM,CAAC0B,eAAe,GAAG,CAAC;QAC/BrC,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,qBAAqB,CAAC;MACnD;MAEA,IAAI,IAAI,CAACjB,UAAU,GAAG,EAAE,KAAK,CAAC,EAAE;QAC9BT,MAAM,CAACsC,KAAK,CAAC,aAAa,EAAE,iBAAiB,IAAI,CAAC7B,UAAU,EAAE,CAAC;MACjE;IACF,CAAC,CAAC,OAAOO,KAAU,EAAE;MACnBhB,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAE,kBAAkBA,KAAK,CAACW,OAAO,EAAE,CAAC;IAChE;EACF;;EAEA;;EAEAY,IAAIA,CAAA,EAAS;IACX,IAAI,CAAChC,KAAK,GAAG,IAAI;IACjB,IAAI,IAAI,CAACD,QAAQ,EAAE;MACjB,IAAI,CAACA,QAAQ,CAACe,IAAI,CAACC,KAAK,GAAG,CAAC;IAC9B;IACAtB,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC;EAC7C;EAEAc,MAAMA,CAAA,EAAS;IACb,IAAI,CAACjC,KAAK,GAAG,KAAK;IAClB,IAAI,IAAI,CAACD,QAAQ,EAAE;MACjB,IAAI,CAACA,QAAQ,CAACe,IAAI,CAACC,KAAK,GAAG,GAAG;IAChC;IACAtB,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;EAC/C;EAEA,IAAIe,OAAOA,CAAA,EAAY;IACrB,OAAO,IAAI,CAAClC,KAAK;EACnB;;EAEA;;EAEA,MAAMmC,IAAIA,CAAA,EAAkB;IAC1B,IAAI;MACF,IAAI,IAAI,CAACrC,eAAe,IAAI,IAAI,CAACG,SAAS,EAAE;QAC1C,IAAI,CAACH,eAAe,CAACqC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAACrC,eAAe,CAACsC,YAAY,CAAC,CAAC;MACrC;MACA,IAAI,CAACnC,SAAS,GAAG,KAAK;MACtB,IAAI,CAACC,UAAU,GAAG,CAAC;MACnB,IAAI,CAACE,MAAM,CAACiC,aAAa,GAAG,CAAC;MAC7B5C,MAAM,CAAC0B,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC;IAChD,CAAC,CAAC,OAAOV,KAAU,EAAE;MACnBhB,MAAM,CAACgB,KAAK,CAAC,aAAa,EAAE,eAAeA,KAAK,CAACW,OAAO,EAAE,CAAC;IAC7D;EACF;EAEA,MAAMkB,OAAOA,CAAA,EAAkB;IAC7B,MAAM,IAAI,CAACH,IAAI,CAAC,CAAC;IACjB,IAAI;MACF,IAAI,IAAI,CAACtC,YAAY,EAAE;QACrB,MAAM,IAAI,CAACA,YAAY,CAAC0C,KAAK,CAAC,CAAC;MACjC;IACF,CAAC,CAAC,MAAM;MACN;IAAA;IAEF,IAAI,CAAC1C,YAAY,GAAG,IAAI;IACxB,IAAI,CAACC,eAAe,GAAG,IAAI;IAC3B,IAAI,CAACC,QAAQ,GAAG,IAAI;EACtB;AACF","ignoreList":[]}
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* VoiceService — WebSocket connection to Gemini Live API.
|
|
5
|
+
*
|
|
6
|
+
* Handles bidirectional audio streaming between the app and Gemini:
|
|
7
|
+
* - Sends PCM 16kHz 16-bit audio chunks (mic input)
|
|
8
|
+
* - Receives PCM 24kHz 16-bit audio chunks (AI responses)
|
|
9
|
+
* - Receives function calls (tap, navigate, etc.) for agentic actions
|
|
10
|
+
* - Sends screen context (DOM text + optional screenshot) for live mode
|
|
11
|
+
*
|
|
12
|
+
* Protocol: wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { logger } from "../utils/logger.js";
|
|
16
|
+
|
|
17
|
+
// ─── Types ─────────────────────────────────────────────────────
|
|
18
|
+
|
|
19
|
+
// ─── Constants ─────────────────────────────────────────────────
|
|
20
|
+
|
|
21
|
+
const WS_HOST = 'generativelanguage.googleapis.com';
|
|
22
|
+
const WS_PATH = '/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
|
|
23
|
+
const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
|
|
24
|
+
const DEFAULT_INPUT_SAMPLE_RATE = 16000;
|
|
25
|
+
|
|
26
|
+
// ─── Service ───────────────────────────────────────────────────
|
|
27
|
+
|
|
28
|
+
export class VoiceService {
|
|
29
|
+
ws = null;
|
|
30
|
+
callbacks = {};
|
|
31
|
+
setupComplete = false;
|
|
32
|
+
_status = 'disconnected';
|
|
33
|
+
constructor(config) {
|
|
34
|
+
this.config = config;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ─── Connection ────────────────────────────────────────────
|
|
38
|
+
|
|
39
|
+
connect(callbacks) {
|
|
40
|
+
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
41
|
+
logger.info('VoiceService', 'Already connected');
|
|
42
|
+
return;
|
|
43
|
+
}
|
|
44
|
+
this.callbacks = callbacks;
|
|
45
|
+
this.setStatus('connecting');
|
|
46
|
+
const model = this.config.model || DEFAULT_MODEL;
|
|
47
|
+
const url = `wss://${WS_HOST}${WS_PATH}?key=${this.config.apiKey}`;
|
|
48
|
+
logger.info('VoiceService', `Connecting to Gemini Live API (model: ${model})`);
|
|
49
|
+
this.ws = new WebSocket(url);
|
|
50
|
+
this.ws.onopen = () => {
|
|
51
|
+
logger.info('VoiceService', 'WebSocket connected, sending setup...');
|
|
52
|
+
this.sendSetup();
|
|
53
|
+
};
|
|
54
|
+
this.ws.onclose = event => {
|
|
55
|
+
logger.info('VoiceService', `WebSocket closed: ${event.code} ${event.reason}`);
|
|
56
|
+
this.setStatus('disconnected');
|
|
57
|
+
this.setupComplete = false;
|
|
58
|
+
};
|
|
59
|
+
this.ws.onerror = error => {
|
|
60
|
+
logger.error('VoiceService', `WebSocket error: ${error.message || 'Unknown'}`);
|
|
61
|
+
this.setStatus('error');
|
|
62
|
+
this.callbacks.onError?.(error.message || 'WebSocket connection error');
|
|
63
|
+
};
|
|
64
|
+
this.ws.onmessage = event => {
|
|
65
|
+
this.handleMessage(event);
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
disconnect() {
|
|
69
|
+
if (this.ws) {
|
|
70
|
+
logger.info('VoiceService', 'Disconnecting...');
|
|
71
|
+
this.ws.close();
|
|
72
|
+
this.ws = null;
|
|
73
|
+
this.setupComplete = false;
|
|
74
|
+
this.setStatus('disconnected');
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
get isConnected() {
|
|
78
|
+
return this.ws?.readyState === WebSocket.OPEN && this.setupComplete;
|
|
79
|
+
}
|
|
80
|
+
get currentStatus() {
|
|
81
|
+
return this._status;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// ─── Send Audio ────────────────────────────────────────────
|
|
85
|
+
|
|
86
|
+
/** Send PCM audio chunk (base64 encoded) to Gemini */
|
|
87
|
+
sendCount = 0;
|
|
88
|
+
sendAudio(base64Audio) {
|
|
89
|
+
this.sendCount++;
|
|
90
|
+
if (!this.isConnected) {
|
|
91
|
+
logger.warn('VoiceService', `sendAudio #${this.sendCount} DROPPED — not connected (ws=${this.ws?.readyState}, setup=${this.setupComplete})`);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const message = {
|
|
95
|
+
realtimeInput: {
|
|
96
|
+
audio: {
|
|
97
|
+
mimeType: `audio/pcm;rate=${this.config.inputSampleRate || DEFAULT_INPUT_SAMPLE_RATE}`,
|
|
98
|
+
data: base64Audio
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
logger.info('VoiceService', `📤 #${this.sendCount} sending ${base64Audio.length} chars (ws=${this.ws?.readyState})`);
|
|
103
|
+
this.ws.send(JSON.stringify(message));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ─── Send Text ─────────────────────────────────────────────
|
|
107
|
+
|
|
108
|
+
/** Send text message via realtimeInput (same channel as audio) */
|
|
109
|
+
sendText(text) {
|
|
110
|
+
if (!this.isConnected) return;
|
|
111
|
+
const message = {
|
|
112
|
+
realtimeInput: {
|
|
113
|
+
text
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
this.ws.send(JSON.stringify(message));
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Send DOM tree as passive context during live conversation.
|
|
120
|
+
*
|
|
121
|
+
* Uses `clientContent` with `turnComplete: false` to inject context
|
|
122
|
+
* WITHOUT triggering a model response. This is the "incremental content
|
|
123
|
+
* updates" pattern from the Gemini docs for establishing session context.
|
|
124
|
+
*
|
|
125
|
+
* Called once at connect + after each tool call (not on a timer).
|
|
126
|
+
* Screenshots are handled separately via the capture_screenshot tool.
|
|
127
|
+
*/
|
|
128
|
+
sendScreenContext(domText) {
|
|
129
|
+
if (!this.isConnected) return;
|
|
130
|
+
const message = {
|
|
131
|
+
clientContent: {
|
|
132
|
+
turns: [{
|
|
133
|
+
role: 'user',
|
|
134
|
+
parts: [{
|
|
135
|
+
text: domText
|
|
136
|
+
}]
|
|
137
|
+
}],
|
|
138
|
+
turnComplete: false // Passive context — don't trigger a response
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
this.ws.send(JSON.stringify(message));
|
|
142
|
+
logger.debug('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// ─── Send Function Response ────────────────────────────────
|
|
146
|
+
|
|
147
|
+
/** Send function call result back to Gemini */
|
|
148
|
+
sendFunctionResponse(name, id, result) {
|
|
149
|
+
if (!this.isConnected) return;
|
|
150
|
+
const message = {
|
|
151
|
+
toolResponse: {
|
|
152
|
+
functionResponses: [{
|
|
153
|
+
name,
|
|
154
|
+
id,
|
|
155
|
+
response: result
|
|
156
|
+
}]
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
logger.info('VoiceService', `📤 Sending tool response for ${name} (id=${id})`);
|
|
160
|
+
this.ws.send(JSON.stringify(message));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ─── Internal: Setup ───────────────────────────────────────
|
|
164
|
+
|
|
165
|
+
sendSetup() {
|
|
166
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
|
|
167
|
+
const model = this.config.model || DEFAULT_MODEL;
|
|
168
|
+
const setup = {
|
|
169
|
+
model: `models/${model}`,
|
|
170
|
+
generationConfig: {
|
|
171
|
+
responseModalities: ['AUDIO']
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
if (this.config.language) {
|
|
175
|
+
setup.generationConfig.speechConfig = {
|
|
176
|
+
languageCode: this.config.language === 'ar' ? 'ar-SA' : 'en-US'
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Add system instruction if provided
|
|
181
|
+
if (this.config.systemPrompt) {
|
|
182
|
+
setup.systemInstruction = {
|
|
183
|
+
parts: [{
|
|
184
|
+
text: this.config.systemPrompt
|
|
185
|
+
}]
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Add tool declarations for function calling
|
|
190
|
+
if (this.config.tools?.length) {
|
|
191
|
+
setup.tools = [{
|
|
192
|
+
functionDeclarations: this.config.tools.map(tool => ({
|
|
193
|
+
name: tool.name,
|
|
194
|
+
description: tool.description,
|
|
195
|
+
parameters: {
|
|
196
|
+
type: 'OBJECT',
|
|
197
|
+
properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => [key, {
|
|
198
|
+
type: param.type.toUpperCase(),
|
|
199
|
+
description: param.description
|
|
200
|
+
}])),
|
|
201
|
+
required: Object.entries(tool.parameters).filter(([, param]) => param.required).map(([key]) => key)
|
|
202
|
+
}
|
|
203
|
+
}))
|
|
204
|
+
}];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Enable transcription
|
|
208
|
+
setup.inputAudioTranscription = {};
|
|
209
|
+
setup.outputAudioTranscription = {};
|
|
210
|
+
const setupMessage = {
|
|
211
|
+
setup
|
|
212
|
+
};
|
|
213
|
+
logger.info('VoiceService', `Sending setup (model: ${model}, tools: ${this.config.tools?.length || 0})`);
|
|
214
|
+
this.ws.send(JSON.stringify(setupMessage));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ─── Internal: Message Handling ────────────────────────────
|
|
218
|
+
|
|
219
|
+
handleMessage(event) {
|
|
220
|
+
try {
|
|
221
|
+
const dataType = typeof event.data;
|
|
222
|
+
const dataLen = typeof event.data === 'string' ? event.data.length : event.data?.byteLength || 'unknown';
|
|
223
|
+
logger.info('VoiceService', `📥 WS message received: type=${dataType}, length=${dataLen}`);
|
|
224
|
+
|
|
225
|
+
// Handle binary data (could be JSON or raw PCM)
|
|
226
|
+
if (typeof event.data !== 'string') {
|
|
227
|
+
logger.info('VoiceService', '📥 Binary message — processing...');
|
|
228
|
+
this.handleBinaryMessage(event.data);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Handle JSON text messages
|
|
233
|
+
const message = JSON.parse(event.data);
|
|
234
|
+
logger.info('VoiceService', `📥 JSON message keys: ${Object.keys(message).join(', ')}`);
|
|
235
|
+
this.processMessage(message);
|
|
236
|
+
} catch (error) {
|
|
237
|
+
logger.error('VoiceService', `Error handling message: ${error.message}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
handleBinaryMessage(data) {
|
|
241
|
+
try {
|
|
242
|
+
// Try to decode as JSON first
|
|
243
|
+
let bytes;
|
|
244
|
+
if (data instanceof ArrayBuffer) {
|
|
245
|
+
bytes = new Uint8Array(data);
|
|
246
|
+
} else if (data instanceof Blob) {
|
|
247
|
+
// Blob handling — read as ArrayBuffer
|
|
248
|
+
const reader = new FileReader();
|
|
249
|
+
reader.onload = () => {
|
|
250
|
+
if (reader.result instanceof ArrayBuffer) {
|
|
251
|
+
this.processBinaryBytes(new Uint8Array(reader.result));
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
reader.readAsArrayBuffer(data);
|
|
255
|
+
return;
|
|
256
|
+
} else {
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
this.processBinaryBytes(bytes);
|
|
260
|
+
} catch (error) {
|
|
261
|
+
logger.error('VoiceService', `Error handling binary message: ${error.message}`);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
processBinaryBytes(bytes) {
|
|
265
|
+
// Check if it looks like JSON (starts with '{' or '[')
|
|
266
|
+
const looksLikeJson = bytes.length > 0 && (bytes[0] === 123 || bytes[0] === 91);
|
|
267
|
+
if (looksLikeJson) {
|
|
268
|
+
try {
|
|
269
|
+
const text = new TextDecoder('utf-8').decode(bytes);
|
|
270
|
+
const message = JSON.parse(text);
|
|
271
|
+
this.processMessage(message);
|
|
272
|
+
} catch {
|
|
273
|
+
// Not JSON — treat as raw PCM audio
|
|
274
|
+
this.callbacks.onAudioResponse?.(this.arrayBufferToBase64(bytes.buffer));
|
|
275
|
+
}
|
|
276
|
+
} else {
|
|
277
|
+
// Raw PCM audio data
|
|
278
|
+
this.callbacks.onAudioResponse?.(this.arrayBufferToBase64(bytes.buffer));
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
processMessage(message) {
|
|
282
|
+
// Setup complete acknowledgment
|
|
283
|
+
if (message.setupComplete !== undefined) {
|
|
284
|
+
logger.info('VoiceService', '✅ Setup complete — ready for audio exchange');
|
|
285
|
+
this.setupComplete = true;
|
|
286
|
+
this.setStatus('connected');
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Server content (audio response + transcripts)
|
|
291
|
+
if (message.serverContent) {
|
|
292
|
+
const content = message.serverContent;
|
|
293
|
+
logger.info('VoiceService', `📥 serverContent received — turnComplete=${content.turnComplete}, hasParts=${!!content.modelTurn?.parts}, inputTranscription=${!!content.inputTranscription}, outputTranscription=${!!content.outputTranscription}`);
|
|
294
|
+
|
|
295
|
+
// Check for turn complete
|
|
296
|
+
if (content.turnComplete) {
|
|
297
|
+
this.callbacks.onTurnComplete?.();
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Process model output parts
|
|
301
|
+
if (content.modelTurn?.parts) {
|
|
302
|
+
for (const part of content.modelTurn.parts) {
|
|
303
|
+
// Audio response
|
|
304
|
+
if (part.inlineData?.data) {
|
|
305
|
+
logger.info('VoiceService', `🔊 Audio response: ${part.inlineData.data.length} chars`);
|
|
306
|
+
this.callbacks.onAudioResponse?.(part.inlineData.data);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// Text response (transcript)
|
|
310
|
+
if (part.text) {
|
|
311
|
+
logger.info('VoiceService', `💬 Text response: "${part.text}"`);
|
|
312
|
+
this.callbacks.onTranscript?.(part.text, true, 'model');
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// Input transcription (user's speech)
|
|
318
|
+
if (content.inputTranscription?.text) {
|
|
319
|
+
this.callbacks.onTranscript?.(content.inputTranscription.text, true, 'user');
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Output transcription (model's speech-to-text)
|
|
323
|
+
if (content.outputTranscription?.text) {
|
|
324
|
+
this.callbacks.onTranscript?.(content.outputTranscription.text, true, 'model');
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Tool calls from the model
|
|
329
|
+
if (message.toolCall?.functionCalls) {
|
|
330
|
+
for (const fn of message.toolCall.functionCalls) {
|
|
331
|
+
logger.info('VoiceService', `Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
|
|
332
|
+
this.callbacks.onToolCall?.({
|
|
333
|
+
name: fn.name,
|
|
334
|
+
args: fn.args || {},
|
|
335
|
+
id: fn.id
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Error messages
|
|
341
|
+
if (message.error) {
|
|
342
|
+
logger.error('VoiceService', `Server error: ${JSON.stringify(message.error)}`);
|
|
343
|
+
this.callbacks.onError?.(message.error.message || 'Server error');
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ─── Helpers ───────────────────────────────────────────────
|
|
348
|
+
|
|
349
|
+
setStatus(newStatus) {
|
|
350
|
+
this._status = newStatus;
|
|
351
|
+
this.callbacks.onStatusChange?.(newStatus);
|
|
352
|
+
}
|
|
353
|
+
arrayBufferToBase64(buffer) {
|
|
354
|
+
const bytes = new Uint8Array(buffer);
|
|
355
|
+
let binary = '';
|
|
356
|
+
for (let i = 0; i < bytes.byteLength; i++) {
|
|
357
|
+
binary += String.fromCharCode(bytes[i]);
|
|
358
|
+
}
|
|
359
|
+
return btoa(binary);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
//# sourceMappingURL=VoiceService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["logger","WS_HOST","WS_PATH","DEFAULT_MODEL","DEFAULT_INPUT_SAMPLE_RATE","VoiceService","ws","callbacks","setupComplete","_status","constructor","config","connect","readyState","WebSocket","OPEN","info","setStatus","model","url","apiKey","onopen","sendSetup","onclose","event","code","reason","onerror","error","message","onError","onmessage","handleMessage","disconnect","close","isConnected","currentStatus","sendCount","sendAudio","base64Audio","warn","realtimeInput","audio","mimeType","inputSampleRate","data","length","send","JSON","stringify","sendText","text","sendScreenContext","domText","clientContent","turns","role","parts","turnComplete","debug","sendFunctionResponse","name","id","result","toolResponse","functionResponses","response","setup","generationConfig","responseModalities","language","speechConfig","languageCode","systemPrompt","systemInstruction","tools","functionDeclarations","map","tool","description","parameters","type","properties","Object","fromEntries","entries","key","param","toUpperCase","required","filter","inputAudioTranscription","outputAudioTranscription","setupMessage","dataType","dataLen","byteLength","handleBinaryMessage","parse","keys","join","processMessage","bytes","ArrayBuffer","Uint8Array","Blob","reader","FileReader","onload","processBinaryBytes","readAsArrayBuffer","looksLikeJson","TextDecoder","decode","onAudioResponse","arrayBufferToBase64","buffer","undefined","serverContent","content","modelTurn","inputTranscription","outputTranscription","onTurnComplete","part","inlineData","onTranscript","toolCall","functionCalls","fn","args","onToolCall","newStatus","onStatusChange","binary","i","String","fromCharCode","btoa"],"sourceRoot":"../../../src","sources":["services/VoiceService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;;AAGxC;;AAyBA;;AAEA,MAAMC,OAAO,GAAG,mCAAmC;AACnD,MAAMC,OAAO,GAAG,+EAA+E;AAC/F,MAAMC,aAAa,GAAG,+CAA+C;AACrE,MAAMC,yBAAyB,GAAG,KAAK;;AAEvC;;AAEA,OAAO,MAAMC,YAAY,CAAC;EAChBC,EAAE,GAAqB,IAAI;EAE3BC,SAAS,GAA0B,CAAC,CAAC;EACrCC,aAAa,GAAG,KAAK;EACrBC,OAAO,GAAgB,cAAc;EAE7CC,WAAWA,CAACC,MAA0B,EAAE;IACtC,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEAC,OAAOA,CAACL,SAAgC,EAAQ;IAC9C,IAAI,IAAI,CAACD,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;MAC1Cf,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC;MAChD;IACF;IAEA,IAAI,CAACT,SAAS,GAAGA,SAAS;IAC1B,IAAI,CAACU,SAAS,CAAC,YAAY,CAAC;IAE5B,MAAMC,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAChD,MAAMgB,GAAG,GAAG,SAASlB,OAAO,GAAGC,OAAO,QAAQ,IAAI,CAACS,MAAM,CAACS,MAAM,EAAE;IAElEpB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yCAAyCE,KAAK,GAAG,CAAC;IAC9E,IAAI,CAACZ,EAAE,GAAG,IAAIQ,SAAS,CAACK,GAAG,CAAC;IAE5B,IAAI,CAACb,EAAE,CAACe,MAAM,GAAG,MAAM;MACrBrB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,uCAAuC,CAAC;MACpE,IAAI,CAACM,SAAS,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAChB,EAAE,CAACiB,OAAO,GAAIC,KAAK,IAAK;MAC3BxB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,qBAAqBQ,KAAK,CAACC,IAAI,IAAID,KAAK,CAACE,MAAM,EAAE,CAAC;MAC9E,IAAI,CAACT,SAAS,CAAC,cAAc,CAAC;MAC9B,IAAI,CAACT,aAAa,GAAG,KAAK;IAC5B,CAAC;IAED,IAAI,CAACF,EAAE,CAACqB,OAAO,GAAIC,KAAU,IAAK;MAChC5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,oBAAoBA,KAAK,CAACC,OAAO,IAAI,SAAS,EAAE,CAAC;MAC9E,IAAI,CAACZ,SAAS,CAAC,OAAO,CAAC;MACvB,IAAI,CAACV,SAAS,CAACuB,OAAO,GAAGF,KAAK,CAACC,OAAO,IAAI,4BAA4B,CAAC;IACzE,CAAC;IAED,IAAI,CAACvB,EAAE,CAACyB,SAAS,GAAIP,KAAK,IAAK;MAC7B,IAAI,CAACQ,aAAa,CAACR,KAAK,CAAC;IAC3B,CAAC;EACH;EAEAS,UAAUA,CAAA,EAAS;IACjB,IAAI,IAAI,CAAC3B,EAAE,EAAE;MACXN,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;MAC/C,IAAI,CAACV,EAAE,CAAC4B,KAAK,CAAC,CAAC;MACf,IAAI,CAAC5B,EAAE,GAAG,IAAI;MACd,IAAI,CAACE,aAAa,GAAG,KAAK;MAC1B,IAAI,CAACS,SAAS,CAAC,cAAc,CAAC;IAChC;EACF;EAEA,IAAIkB,WAAWA,CAAA,EAAY;IACzB,OAAO,IAAI,CAAC7B,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,IAAI,IAAI,CAACP,aAAa;EACrE;EAEA,IAAI4B,aAAaA,CAAA,EAAgB;IAC/B,OAAO,IAAI,CAAC3B,OAAO;EACrB;;EAEA;;EAEA;EACQ4B,SAAS,GAAG,CAAC;EACrBC,SAASA,CAACC,WAAmB,EAAQ;IACnC,IAAI,CAACF,SAAS,EAAE;IAChB,IAAI,CAAC,IAAI,CAACF,WAAW,EAAE;MACrBnC,MAAM,CAACwC,IAAI,CAAC,cAAc,EAAE,cAAc,IAAI,CAACH,SAAS,gCAAgC,IAAI,CAAC/B,EAAE,EAAEO,UAAU,WAAW,IAAI,CAACL,aAAa,GAAG,CAAC;MAC5I;IACF;IAEA,MAAMqB,OAAO,GAAG;MACdY,aAAa,EAAE;QACbC,KAAK,EAAE;UACLC,QAAQ,EAAE,kBAAkB,IAAI,CAAChC,MAAM,CAACiC,eAAe,IAAIxC,yBAAyB,EAAE;UACtFyC,IAAI,EAAEN;QACR;MACF;IACF,CAAC;IAEDvC,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,OAAO,IAAI,CAACqB,SAAS,YAAYE,WAAW,CAACO,MAAM,cAAc,IAAI,CAACxC,EAAE,EAAEO,UAAU,GAAG,CAAC;IACpH,IAAI,CAACP,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEA;EACAqB,QAAQA,CAACC,IAAY,EAAQ;IAC3B,IAAI,CAAC,IAAI,CAAChB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdY,aAAa,EAAE;QAAEU;MAAK;IACxB,CAAC;IAED,IAAI,CAAC7C,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEuB,iBAAiBA,CAACC,OAAe,EAAQ;IACvC,IAAI,CAAC,IAAI,CAAClB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdyB,aAAa,EAAE;QACbC,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAE,MAAM;UAAEC,KAAK,EAAE,CAAC;YAAEN,IAAI,EAAEE;UAAQ,CAAC;QAAE,CAAC,CAAC;QACrDK,YAAY,EAAE,KAAK,CAAE;MACvB;IACF,CAAC;IAED,IAAI,CAACpD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;IACtC7B,MAAM,CAAC2D,KAAK,CAAC,cAAc,EAAE,2BAA2BN,OAAO,CAACP,MAAM,SAAS,CAAC;EAClF;;EAEA;;EAEA;EACAc,oBAAoBA,CAACC,IAAY,EAAEC,EAAU,EAAEC,MAAW,EAAQ;IAChE,IAAI,CAAC,IAAI,CAAC5B,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdmC,YAAY,EAAE;QACZC,iBAAiB,EAAE,CAAC;UAClBJ,IAAI;UACJC,EAAE;UACFI,QAAQ,EAAEH;QACZ,CAAC;MACH;IACF,CAAC;IAED/D,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC6C,IAAI,QAAQC,EAAE,GAAG,CAAC;IAC9E,IAAI,CAACxD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEQP,SAASA,CAAA,EAAS;IACxB,IAAI,CAAC,IAAI,CAAChB,EAAE,IAAI,IAAI,CAACA,EAAE,CAACO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;IAEvD,MAAMG,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAEhD,MAAMgE,KAAU,GAAG;MACjBjD,KAAK,EAAE,UAAUA,KAAK,EAAE;MACxBkD,gBAAgB,EAAE;QAChBC,kBAAkB,EAAE,CAAC,OAAO;MAC9B;IACF,CAAC;IAED,IAAI,IAAI,CAAC1D,MAAM,CAAC2D,QAAQ,EAAE;MACxBH,KAAK,CAACC,gBAAgB,CAACG,YAAY,GAAG;QACpCC,YAAY,EAAE,IAAI,CAAC7D,MAAM,CAAC2D,QAAQ,KAAK,IAAI,GAAG,OAAO,GAAG;MAC1D,CAAC;IACH;;IAEA;IACA,IAAI,IAAI,CAAC3D,MAAM,CAAC8D,YAAY,EAAE;MAC5BN,KAAK,CAACO,iBAAiB,GAAG;QACxBjB,KAAK,EAAE,CAAC;UAAEN,IAAI,EAAE,IAAI,CAACxC,MAAM,CAAC8D;QAAa,CAAC;MAC5C,CAAC;IACH;;IAEA;IACA,IAAI,IAAI,CAAC9D,MAAM,CAACgE,KAAK,EAAE7B,MAAM,EAAE;MAC7BqB,KAAK,CAACQ,KAAK,GAAG,CAAC;QACbC,oBAAoB,EAAE,IAAI,CAACjE,MAAM,CAACgE,KAAK,CAACE,GAAG,CAACC,IAAI,KAAK;UACnDjB,IAAI,EAAEiB,IAAI,CAACjB,IAAI;UACfkB,WAAW,EAAED,IAAI,CAACC,WAAW;UAC7BC,UAAU,EAAE;YACVC,IAAI,EAAE,QAAQ;YACdC,UAAU,EAAEC,MAAM,CAACC,WAAW,CAC5BD,MAAM,CAACE,OAAO,CAACP,IAAI,CAACE,UAAU,CAAC,CAACH,GAAG,CAAC,CAAC,CAACS,GAAG,EAAEC,KAAK,CAAC,KAAK,CACpDD,GAAG,EACH;cACEL,IAAI,EAAEM,KAAK,CAACN,IAAI,CAACO,WAAW,CAAC,CAAC;cAC9BT,WAAW,EAAEQ,KAAK,CAACR;YACrB,CAAC,CACF,CACH,CAAC;YACDU,QAAQ,EAAEN,MAAM,CAACE,OAAO,CAACP,IAAI,CAACE,UAAU,CAAC,CACtCU,MAAM,CAAC,CAAC,GAAGH,KAAK,CAAC,KAAKA,KAAK,CAACE,QAAQ,CAAC,CACrCZ,GAAG,CAAC,CAAC,CAACS,GAAG,CAAC,KAAKA,GAAG;UACvB;QACF,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACAnB,KAAK,CAACwB,uBAAuB,GAAG,CAAC,CAAC;IAClCxB,KAAK,CAACyB,wBAAwB,GAAG,CAAC,CAAC;IAEnC,MAAMC,YAAY,GAAG;MAAE1B;IAAM,CAAC;IAC9BnE,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBE,KAAK,YAAY,IAAI,CAACP,MAAM,CAACgE,KAAK,EAAE7B,MAAM,IAAI,CAAC,GAAG,CAAC;IACxG,IAAI,CAACxC,EAAE,CAACyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAAC4C,YAAY,CAAC,CAAC;EAC5C;;EAEA;;EAEQ7D,aAAaA,CAACR,KAA4B,EAAQ;IACxD,IAAI;MACF,MAAMsE,QAAQ,GAAG,OAAOtE,KAAK,CAACqB,IAAI;MAClC,MAAMkD,OAAO,GAAG,OAAOvE,KAAK,CAACqB,IAAI,KAAK,QAAQ,GAAGrB,KAAK,CAACqB,IAAI,CAACC,MAAM,GAAItB,KAAK,CAACqB,IAAI,EAAEmD,UAAU,IAAI,SAAU;MAC1GhG,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC8E,QAAQ,YAAYC,OAAO,EAAE,CAAC;;MAE1F;MACA,IAAI,OAAOvE,KAAK,CAACqB,IAAI,KAAK,QAAQ,EAAE;QAClC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mCAAmC,CAAC;QAChE,IAAI,CAACiF,mBAAmB,CAACzE,KAAK,CAACqB,IAAI,CAAC;QACpC;MACF;;MAEA;MACA,MAAMhB,OAAO,GAAGmB,IAAI,CAACkD,KAAK,CAAC1E,KAAK,CAACqB,IAAI,CAAC;MACtC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBmE,MAAM,CAACgB,IAAI,CAACtE,OAAO,CAAC,CAACuE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;MACvF,IAAI,CAACC,cAAc,CAACxE,OAAO,CAAC;IAC9B,CAAC,CAAC,OAAOD,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,2BAA2BA,KAAK,CAACC,OAAO,EAAE,CAAC;IAC1E;EACF;EAEQoE,mBAAmBA,CAACpD,IAAS,EAAQ;IAC3C,IAAI;MACF;MACA,IAAIyD,KAAiB;MACrB,IAAIzD,IAAI,YAAY0D,WAAW,EAAE;QAC/BD,KAAK,GAAG,IAAIE,UAAU,CAAC3D,IAAI,CAAC;MAC9B,CAAC,MAAM,IAAIA,IAAI,YAAY4D,IAAI,EAAE;QAC/B;QACA,MAAMC,MAAM,GAAG,IAAIC,UAAU,CAAC,CAAC;QAC/BD,MAAM,CAACE,MAAM,GAAG,MAAM;UACpB,IAAIF,MAAM,CAAC3C,MAAM,YAAYwC,WAAW,EAAE;YACxC,IAAI,CAACM,kBAAkB,CAAC,IAAIL,UAAU,CAACE,MAAM,CAAC3C,MAAM,CAAC,CAAC;UACxD;QACF,CAAC;QACD2C,MAAM,CAACI,iBAAiB,CAACjE,IAAI,CAAC;QAC9B;MACF,CAAC,MAAM;QACL;MACF;MAEA,IAAI,CAACgE,kBAAkB,CAACP,KAAK,CAAC;IAChC,CAAC,CAAC,OAAO1E,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,kCAAkCA,KAAK,CAACC,OAAO,EAAE,CAAC;IACjF;EACF;EAEQgF,kBAAkBA,CAACP,KAAiB,EAAQ;IAClD;IACA,MAAMS,aAAa,GAAGT,KAAK,CAACxD,MAAM,GAAG,CAAC,KAAKwD,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAIA,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAE/E,IAAIS,aAAa,EAAE;MACjB,IAAI;QACF,MAAM5D,IAAI,GAAG,IAAI6D,WAAW,CAAC,OAAO,CAAC,CAACC,MAAM,CAACX,KAAK,CAAC;QACnD,MAAMzE,OAAO,GAAGmB,IAAI,CAACkD,KAAK,CAAC/C,IAAI,CAAC;QAChC,IAAI,CAACkD,cAAc,CAACxE,OAAO,CAAC;MAC9B,CAAC,CAAC,MAAM;QACN;QACA,IAAI,CAACtB,SAAS,CAAC2G,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;MACzF;IACF,CAAC,MAAM;MACL;MACA,IAAI,CAAC7G,SAAS,CAAC2G,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;IACzF;EACF;EAEQf,cAAcA,CAACxE,OAAY,EAAQ;IACzC;IACA,IAAIA,OAAO,CAACrB,aAAa,KAAK6G,SAAS,EAAE;MACvCrH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,6CAA6C,CAAC;MAC1E,IAAI,CAACR,aAAa,GAAG,IAAI;MACzB,IAAI,CAACS,SAAS,CAAC,WAAW,CAAC;MAC3B;IACF;;IAEA;IACA,IAAIY,OAAO,CAACyF,aAAa,EAAE;MACzB,MAAMC,OAAO,GAAG1F,OAAO,CAACyF,aAAa;MACrCtH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,4CAA4CuG,OAAO,CAAC7D,YAAY,cAAc,CAAC,CAAC6D,OAAO,CAACC,SAAS,EAAE/D,KAAK,wBAAwB,CAAC,CAAC8D,OAAO,CAACE,kBAAkB,yBAAyB,CAAC,CAACF,OAAO,CAACG,mBAAmB,EAAE,CAAC;;MAEjP;MACA,IAAIH,OAAO,CAAC7D,YAAY,EAAE;QACxB,IAAI,CAACnD,SAAS,CAACoH,cAAc,GAAG,CAAC;MACnC;;MAEA;MACA,IAAIJ,OAAO,CAACC,SAAS,EAAE/D,KAAK,EAAE;QAC5B,KAAK,MAAMmE,IAAI,IAAIL,OAAO,CAACC,SAAS,CAAC/D,KAAK,EAAE;UAC1C;UACA,IAAImE,IAAI,CAACC,UAAU,EAAEhF,IAAI,EAAE;YACzB7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsB4G,IAAI,CAACC,UAAU,CAAChF,IAAI,CAACC,MAAM,QAAQ,CAAC;YACtF,IAAI,CAACvC,SAAS,CAAC2G,eAAe,GAAGU,IAAI,CAACC,UAAU,CAAChF,IAAI,CAAC;UACxD;;UAEA;UACA,IAAI+E,IAAI,CAACzE,IAAI,EAAE;YACbnD,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsB4G,IAAI,CAACzE,IAAI,GAAG,CAAC;YAC/D,IAAI,CAAC5C,SAAS,CAACuH,YAAY,GAAGF,IAAI,CAACzE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;UACzD;QACF;MACF;;MAEA;MACA,IAAIoE,OAAO,CAACE,kBAAkB,EAAEtE,IAAI,EAAE;QACpC,IAAI,CAAC5C,SAAS,CAACuH,YAAY,GAAGP,OAAO,CAACE,kBAAkB,CAACtE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;MAC9E;;MAEA;MACA,IAAIoE,OAAO,CAACG,mBAAmB,EAAEvE,IAAI,EAAE;QACrC,IAAI,CAAC5C,SAAS,CAACuH,YAAY,GAAGP,OAAO,CAACG,mBAAmB,CAACvE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;MAChF;IACF;;IAEA;IACA,IAAItB,OAAO,CAACkG,QAAQ,EAAEC,aAAa,EAAE;MACnC,KAAK,MAAMC,EAAE,IAAIpG,OAAO,CAACkG,QAAQ,CAACC,aAAa,EAAE;QAC/ChI,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,cAAciH,EAAE,CAACpE,IAAI,IAAIb,IAAI,CAACC,SAAS,CAACgF,EAAE,CAACC,IAAI,CAAC,GAAG,CAAC;QAChF,IAAI,CAAC3H,SAAS,CAAC4H,UAAU,GAAG;UAC1BtE,IAAI,EAAEoE,EAAE,CAACpE,IAAI;UACbqE,IAAI,EAAED,EAAE,CAACC,IAAI,IAAI,CAAC,CAAC;UACnBpE,EAAE,EAAEmE,EAAE,CAACnE;QACT,CAAC,CAAC;MACJ;IACF;;IAEA;IACA,IAAIjC,OAAO,CAACD,KAAK,EAAE;MACjB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,iBAAiBoB,IAAI,CAACC,SAAS,CAACpB,OAAO,CAACD,KAAK,CAAC,EAAE,CAAC;MAC9E,IAAI,CAACrB,SAAS,CAACuB,OAAO,GAAGD,OAAO,CAACD,KAAK,CAACC,OAAO,IAAI,cAAc,CAAC;IACnE;EACF;;EAEA;;EAEQZ,SAASA,CAACmH,SAAsB,EAAQ;IAC9C,IAAI,CAAC3H,OAAO,GAAG2H,SAAS;IACxB,IAAI,CAAC7H,SAAS,CAAC8H,cAAc,GAAGD,SAAS,CAAC;EAC5C;EAEQjB,mBAAmBA,CAACC,MAAmB,EAAU;IACvD,MAAMd,KAAK,GAAG,IAAIE,UAAU,CAACY,MAAM,CAAC;IACpC,IAAIkB,MAAM,GAAG,EAAE;IACf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGjC,KAAK,CAACN,UAAU,EAAEuC,CAAC,EAAE,EAAE;MACzCD,MAAM,IAAIE,MAAM,CAACC,YAAY,CAACnC,KAAK,CAACiC,CAAC,CAAE,CAAC;IAC1C;IACA,OAAOG,IAAI,CAACJ,MAAM,CAAC;EACrB;AACF","ignoreList":[]}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Audio utility functions for PCM conversion.
|
|
5
|
+
*
|
|
6
|
+
* Used by AudioInputService and AudioOutputService to convert between
|
|
7
|
+
* Float32 (Web Audio API) and Int16 (Gemini Live API) PCM formats.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Convert Float32Array PCM samples to Int16 PCM and encode as base64.
|
|
12
|
+
* Gemini Live API expects Int16 little-endian PCM.
|
|
13
|
+
*/
|
|
14
|
+
export function float32ToInt16Base64(float32Data) {
|
|
15
|
+
const int16Buffer = new Int16Array(float32Data.length);
|
|
16
|
+
for (let i = 0; i < float32Data.length; i++) {
|
|
17
|
+
// Clamp to [-1, 1] and scale to Int16 range
|
|
18
|
+
const sample = Math.max(-1, Math.min(1, float32Data[i] || 0));
|
|
19
|
+
int16Buffer[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Convert Int16Array to base64
|
|
23
|
+
const bytes = new Uint8Array(int16Buffer.buffer);
|
|
24
|
+
let binary = '';
|
|
25
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
26
|
+
binary += String.fromCharCode(bytes[i] || 0);
|
|
27
|
+
}
|
|
28
|
+
return typeof global.btoa === 'function' ? global.btoa(binary) : Buffer.from(binary, 'binary').toString('base64');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Decode base64 Int16 PCM to Float32Array.
|
|
33
|
+
* Used for manual decoding when decodePCMInBase64 is unavailable.
|
|
34
|
+
*/
|
|
35
|
+
export function base64ToFloat32(base64) {
|
|
36
|
+
const binaryString = typeof global.atob === 'function' ? global.atob(base64) : Buffer.from(base64, 'base64').toString('binary');
|
|
37
|
+
const bytes = new Uint8Array(binaryString.length);
|
|
38
|
+
for (let i = 0; i < binaryString.length; i++) {
|
|
39
|
+
bytes[i] = binaryString.charCodeAt(i);
|
|
40
|
+
}
|
|
41
|
+
const int16Data = new Int16Array(bytes.buffer);
|
|
42
|
+
const float32Data = new Float32Array(int16Data.length);
|
|
43
|
+
for (let i = 0; i < int16Data.length; i++) {
|
|
44
|
+
// Scale Int16 back to Float32 [-1, 1]
|
|
45
|
+
float32Data[i] = (int16Data[i] || 0) / 0x8000;
|
|
46
|
+
}
|
|
47
|
+
return float32Data;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=audioUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["float32ToInt16Base64","float32Data","int16Buffer","Int16Array","length","i","sample","Math","max","min","bytes","Uint8Array","buffer","binary","String","fromCharCode","global","btoa","Buffer","from","toString","base64ToFloat32","base64","binaryString","atob","charCodeAt","int16Data","Float32Array"],"sourceRoot":"../../../src","sources":["utils/audioUtils.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACA;AACA;AACA;AACA,OAAO,SAASA,oBAAoBA,CAACC,WAAyB,EAAU;EACtE,MAAMC,WAAW,GAAG,IAAIC,UAAU,CAACF,WAAW,CAACG,MAAM,CAAC;EACtD,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGJ,WAAW,CAACG,MAAM,EAAEC,CAAC,EAAE,EAAE;IAC3C;IACA,MAAMC,MAAM,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC,CAAC,EAAED,IAAI,CAACE,GAAG,CAAC,CAAC,EAAER,WAAW,CAACI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7DH,WAAW,CAACG,CAAC,CAAC,GAAGC,MAAM,GAAG,CAAC,GAAGA,MAAM,GAAG,MAAM,GAAGA,MAAM,GAAG,MAAM;EACjE;;EAEA;EACA,MAAMI,KAAK,GAAG,IAAIC,UAAU,CAACT,WAAW,CAACU,MAAM,CAAC;EAChD,IAAIC,MAAM,GAAG,EAAE;EACf,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGK,KAAK,CAACN,MAAM,EAAEC,CAAC,EAAE,EAAE;IACrCQ,MAAM,IAAIC,MAAM,CAACC,YAAY,CAACL,KAAK,CAACL,CAAC,CAAC,IAAI,CAAC,CAAC;EAC9C;EAEA,OAAO,OAAOW,MAAM,CAACC,IAAI,KAAK,UAAU,GACpCD,MAAM,CAACC,IAAI,CAACJ,MAAM,CAAC,GACnBK,MAAM,CAACC,IAAI,CAACN,MAAM,EAAE,QAAQ,CAAC,CAACO,QAAQ,CAAC,QAAQ,CAAC;AACtD;;AAEA;AACA;AACA;AACA;AACA,OAAO,SAASC,eAAeA,CAACC,MAAc,EAAgB;EAC5D,MAAMC,YAAY,GAAG,OAAOP,MAAM,CAACQ,IAAI,KAAK,UAAU,GAClDR,MAAM,CAACQ,IAAI,CAACF,MAAM,CAAC,GACnBJ,MAAM,CAACC,IAAI,CAACG,MAAM,EAAE,QAAQ,CAAC,CAACF,QAAQ,CAAC,QAAQ,CAAC;EAEpD,MAAMV,KAAK,GAAG,IAAIC,UAAU,CAACY,YAAY,CAACnB,MAAM,CAAC;EACjD,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGkB,YAAY,CAACnB,MAAM,EAAEC,CAAC,EAAE,EAAE;IAC5CK,KAAK,CAACL,CAAC,CAAC,GAAGkB,YAAY,CAACE,UAAU,CAACpB,CAAC,CAAC;EACvC;EAEA,MAAMqB,SAAS,GAAG,IAAIvB,UAAU,CAACO,KAAK,CAACE,MAAM,CAAC;EAC9C,MAAMX,WAAW,GAAG,IAAI0B,YAAY,CAACD,SAAS,CAACtB,MAAM,CAAC;EACtD,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGqB,SAAS,CAACtB,MAAM,EAAEC,CAAC,EAAE,EAAE;IACzC;IACAJ,WAAW,CAACI,CAAC,CAAC,GAAG,CAACqB,SAAS,CAACrB,CAAC,CAAC,IAAI,CAAC,IAAI,MAAM;EAC/C;EAEA,OAAOJ,WAAW;AACpB","ignoreList":[]}
|
|
@@ -2,14 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Logger utility — prefixed console output for easy filtering.
|
|
5
|
+
*
|
|
6
|
+
* Disabled by default. Enable via `logger.setEnabled(true)` or
|
|
7
|
+
* pass `debug={true}` to the <AIAgent> component.
|
|
5
8
|
*/
|
|
6
9
|
const TAG = '[AIAgent]';
|
|
10
|
+
let enabled = false;
|
|
7
11
|
export const logger = {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
12
|
+
/** Enable or disable all SDK logging. */
|
|
13
|
+
setEnabled: value => {
|
|
14
|
+
enabled = value;
|
|
15
|
+
},
|
|
16
|
+
/** Check if logging is enabled. */
|
|
17
|
+
isEnabled: () => enabled,
|
|
18
|
+
info: (context, ...args) => {
|
|
19
|
+
if (enabled) console.log(`${TAG} [${context}]`, ...args);
|
|
20
|
+
},
|
|
21
|
+
warn: (context, ...args) => {
|
|
22
|
+
if (enabled) console.warn(`${TAG} [${context}]`, ...args);
|
|
23
|
+
},
|
|
24
|
+
error: (context, ...args) => {
|
|
25
|
+
// Errors always log regardless of enabled flag
|
|
26
|
+
console.error(`${TAG} [${context}]`, ...args);
|
|
27
|
+
},
|
|
11
28
|
debug: (context, ...args) => {
|
|
12
|
-
if (__DEV__) {
|
|
29
|
+
if (enabled && __DEV__) {
|
|
13
30
|
console.log(`${TAG} [${context}] 🐛`, ...args);
|
|
14
31
|
}
|
|
15
32
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["TAG","logger","info","context","args","console","log","warn","error","debug","__DEV__"],"sourceRoot":"../../../src","sources":["utils/logger.ts"],"mappings":";;AAAA;AACA;AACA;AACA,MAAMA,GAAG,GAAG,WAAW;AAEvB,OAAO,MAAMC,MAAM,GAAG;
|
|
1
|
+
{"version":3,"names":["TAG","enabled","logger","setEnabled","value","isEnabled","info","context","args","console","log","warn","error","debug","__DEV__"],"sourceRoot":"../../../src","sources":["utils/logger.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA,MAAMA,GAAG,GAAG,WAAW;AAEvB,IAAIC,OAAO,GAAG,KAAK;AAEnB,OAAO,MAAMC,MAAM,GAAG;EACpB;EACAC,UAAU,EAAGC,KAAc,IAAK;IAC9BH,OAAO,GAAGG,KAAK;EACjB,CAAC;EAED;EACAC,SAAS,EAAEA,CAAA,KAAMJ,OAAO;EAExBK,IAAI,EAAEA,CAACC,OAAe,EAAE,GAAGC,IAAW,KAAK;IACzC,IAAIP,OAAO,EAAEQ,OAAO,CAACC,GAAG,CAAC,GAAGV,GAAG,KAAKO,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAC1D,CAAC;EAEDG,IAAI,EAAEA,CAACJ,OAAe,EAAE,GAAGC,IAAW,KAAK;IACzC,IAAIP,OAAO,EAAEQ,OAAO,CAACE,IAAI,CAAC,GAAGX,GAAG,KAAKO,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAC3D,CAAC;EAEDI,KAAK,EAAEA,CAACL,OAAe,EAAE,GAAGC,IAAW,KAAK;IAC1C;IACAC,OAAO,CAACG,KAAK,CAAC,GAAGZ,GAAG,KAAKO,OAAO,GAAG,EAAE,GAAGC,IAAI,CAAC;EAC/C,CAAC;EAEDK,KAAK,EAAEA,CAACN,OAAe,EAAE,GAAGC,IAAW,KAAK;IAC1C,IAAIP,OAAO,IAAIa,OAAO,EAAE;MACtBL,OAAO,CAACC,GAAG,CAAC,GAAGV,GAAG,KAAKO,OAAO,MAAM,EAAE,GAAGC,IAAI,CAAC;IAChD;EACF;AACF,CAAC","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"babel.config.d.ts","sourceRoot":"","sources":["../../babel.config.js"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eslint.config.d.mts","sourceRoot":"","sources":["../../eslint.config.mjs"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-models.d.mts","sourceRoot":"","sources":["../../fetch-models.mjs"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-all-models.d.mts","sourceRoot":"","sources":["../../list-all-models.mjs"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list-models.d.mts","sourceRoot":"","sources":["../../list-models.mjs"],"names":[],"mappings":""}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* - Agent runtime context for useAction hooks
|
|
9
9
|
*/
|
|
10
10
|
import React from 'react';
|
|
11
|
-
import type { ExecutionResult, ToolDefinition, AgentStep } from '../core/types';
|
|
11
|
+
import type { ExecutionResult, ToolDefinition, AgentStep, TokenUsage } from '../core/types';
|
|
12
12
|
interface AIAgentProps {
|
|
13
13
|
/** Gemini API key */
|
|
14
14
|
apiKey: string;
|
|
@@ -59,7 +59,13 @@ interface AIAgentProps {
|
|
|
59
59
|
};
|
|
60
60
|
/** Expo Router pathname (from usePathname()) */
|
|
61
61
|
pathname?: string;
|
|
62
|
+
/** Enable voice mode (requires expo-av) */
|
|
63
|
+
enableVoice?: boolean;
|
|
64
|
+
/** Called after each step with token usage data */
|
|
65
|
+
onTokenUsage?: (usage: TokenUsage) => void;
|
|
66
|
+
/** Enable SDK debug logging (disabled by default) */
|
|
67
|
+
debug?: boolean;
|
|
62
68
|
}
|
|
63
|
-
export declare function AIAgent({ apiKey, model, navRef, language, maxSteps, showChatBar, children, onResult, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, }: AIAgentProps): import("react/jsx-runtime").JSX.Element;
|
|
69
|
+
export declare function AIAgent({ apiKey, model, navRef, language, maxSteps, showChatBar, children, onResult, interactiveBlacklist, interactiveWhitelist, onBeforeStep, onAfterStep, onBeforeTask, onAfterTask, transformScreenContent, customTools, instructions, stepDelay, mcpServerUrl, router, pathname, enableVoice, onTokenUsage, debug, }: AIAgentProps): import("react/jsx-runtime").JSX.Element;
|
|
64
70
|
export {};
|
|
65
71
|
//# sourceMappingURL=AIAgent.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;AAaf,OAAO,KAAK,EAA0B,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAOpH,UAAU,YAAY;IACpB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,kBAAkB;IAClB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IAI7C,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,8BAA8B;IAC9B,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3D,6BAA6B;IAC7B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1C,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAChE,qEAAqE;IACrE,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACvE,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IACpD,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IACF,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,MAAM,CAAC,EAAE;QACP,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAC7B,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAChC,IAAI,EAAE,MAAM,IAAI,CAAC;KAClB,CAAC;IACF,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mDAAmD;IACnD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAC3C,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAID,wBAAgB,OAAO,CAAC,EACtB,MAAM,EACN,KAA0B,EAC1B,MAAM,EACN,QAAe,EACf,QAAa,EACb,WAAkB,EAClB,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,WAAmB,EACnB,YAAY,EACZ,KAAa,GACd,EAAE,YAAY,2CA4Vd"}
|
|
@@ -1,15 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* AgentChatBar — Floating, draggable, compressible chat widget.
|
|
3
|
+
* Supports two modes: Text and Voice.
|
|
3
4
|
* Does not block underlying UI natively.
|
|
4
5
|
*/
|
|
5
|
-
import type { ExecutionResult } from '../core/types';
|
|
6
|
+
import type { ExecutionResult, AgentMode } from '../core/types';
|
|
6
7
|
interface AgentChatBarProps {
|
|
7
8
|
onSend: (message: string) => void;
|
|
8
9
|
isThinking: boolean;
|
|
9
10
|
lastResult: ExecutionResult | null;
|
|
10
11
|
language: 'en' | 'ar';
|
|
11
12
|
onDismiss?: () => void;
|
|
13
|
+
/** Available modes (default: ['text']) */
|
|
14
|
+
availableModes?: AgentMode[];
|
|
15
|
+
/** Current active mode */
|
|
16
|
+
mode?: AgentMode;
|
|
17
|
+
onModeChange?: (mode: AgentMode) => void;
|
|
18
|
+
/** Voice controls */
|
|
19
|
+
onMicToggle?: (active: boolean) => void;
|
|
20
|
+
onSpeakerToggle?: (muted: boolean) => void;
|
|
21
|
+
isMicActive?: boolean;
|
|
22
|
+
isSpeakerMuted?: boolean;
|
|
23
|
+
/** AI is currently speaking */
|
|
24
|
+
isAISpeaking?: boolean;
|
|
25
|
+
/** Voice WebSocket is connected */
|
|
26
|
+
isVoiceConnected?: boolean;
|
|
27
|
+
/** Full session cleanup (stop mic, audio, WebSocket, live mode) */
|
|
28
|
+
onStopSession?: () => void;
|
|
12
29
|
}
|
|
13
|
-
export declare function AgentChatBar({ onSend, isThinking, lastResult, language, onDismiss }: AgentChatBarProps): import("react/jsx-runtime").JSX.Element;
|
|
30
|
+
export declare function AgentChatBar({ onSend, isThinking, lastResult, language, onDismiss, availableModes, mode, onModeChange, onMicToggle, onSpeakerToggle, isMicActive, isSpeakerMuted, isAISpeaking, isVoiceConnected, onStopSession, }: AgentChatBarProps): import("react/jsx-runtime").JSX.Element;
|
|
14
31
|
export {};
|
|
15
32
|
//# sourceMappingURL=AgentChatBar.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgentChatBar.d.ts","sourceRoot":"","sources":["../../../../src/components/AgentChatBar.tsx"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"AgentChatBar.d.ts","sourceRoot":"","sources":["../../../../src/components/AgentChatBar.tsx"],"names":[],"mappings":"AAAA;;;;GAIG;AAaH,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAIhE,UAAU,iBAAiB;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,eAAe,GAAG,IAAI,CAAC;IACnC,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;IACvB,0CAA0C;IAC1C,cAAc,CAAC,EAAE,SAAS,EAAE,CAAC;IAC7B,0BAA0B;IAC1B,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,CAAC;IACzC,qBAAqB;IACrB,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;IACxC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,+BAA+B;IAC/B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mCAAmC;IACnC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,mEAAmE;IACnE,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;CAC5B;AA+SD,wBAAgB,YAAY,CAAC,EAC3B,MAAM,EACN,UAAU,EACV,UAAU,EACV,QAAQ,EACR,SAAS,EACT,cAAyB,EACzB,IAAa,EACb,YAAY,EACZ,WAAW,EACX,eAAe,EACf,WAAmB,EACnB,cAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,aAAa,GACd,EAAE,iBAAiB,2CAqHnB"}
|