@mobileai/react-native 0.4.5 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/README.md +2 -13
  2. package/lib/module/components/AIAgent.js +6 -4
  3. package/lib/module/components/AIAgent.js.map +1 -1
  4. package/lib/module/core/systemPrompt.js +68 -34
  5. package/lib/module/core/systemPrompt.js.map +1 -1
  6. package/lib/module/index.js +9 -3
  7. package/lib/module/index.js.map +1 -1
  8. package/lib/module/services/VoiceService.js +67 -28
  9. package/lib/module/services/VoiceService.js.map +1 -1
  10. package/lib/typescript/src/components/AIAgent.d.ts.map +1 -1
  11. package/lib/typescript/src/core/systemPrompt.d.ts +7 -4
  12. package/lib/typescript/src/core/systemPrompt.d.ts.map +1 -1
  13. package/lib/typescript/src/index.d.ts +0 -4
  14. package/lib/typescript/src/index.d.ts.map +1 -1
  15. package/lib/typescript/src/services/VoiceService.d.ts +8 -0
  16. package/lib/typescript/src/services/VoiceService.d.ts.map +1 -1
  17. package/package.json +1 -1
  18. package/src/components/AIAgent.tsx +7 -4
  19. package/src/core/systemPrompt.ts +68 -34
  20. package/src/index.ts +8 -8
  21. package/src/services/VoiceService.ts +75 -35
package/README.md CHANGED
@@ -23,8 +23,6 @@ Wrap your navigation with `<AIAgent>`. The AI automatically understands your ent
23
23
  - 🔒 **Production-grade security** — Element gating, content masking, lifecycle hooks, human-in-the-loop confirmation.
24
24
  - 🧩 **Custom actions** — Expose any business logic (checkout, API calls) as AI-callable tools with `useAction`.
25
25
  - 🌐 **MCP bridge** — Let external AI agents (OpenClaw, Claude Desktop) control your app remotely.
26
- - 🎙️ **Live Agent (Voice Mode)** — Real-time conversational AI that sees your screen and takes action while you speak.
27
- - 🎤 **Voice Dictation** — Built-in on-device speech-to-text for the text chat mode.
28
26
  - 🌍 **Bilingual** — English and Arabic support built-in.
29
27
 
30
28
  ## 📦 Installation
@@ -39,21 +37,13 @@ No native modules required by default. Works with Expo managed workflow out of t
39
37
 
40
38
  ### Optional Native Dependencies
41
39
 
42
- If you want to use **Live Agent (Voice)**, **Text Mode Dictation**, or **Screenshots** (for image/video content), install these optional peer dependencies:
40
+ If you want to use **Screenshots** (for image/video content), install this optional peer dependency:
43
41
 
44
42
  ```bash
45
- # For Voice / Live Agent mode:
46
- npx expo install react-native-audio-api
47
-
48
- # For Text mode voice dictation (mic button):
49
- npx expo install expo-speech-recognition
50
-
51
43
  # For Screenshots (when asking AI "what do you see in this image?"):
52
44
  npx expo install react-native-view-shot
53
45
  ```
54
46
 
55
- > **Note on Permissions**: Ensure you add Microphone permissions (`NSMicrophoneUsageDescription` / `RECORD_AUDIO`) to your `app.json` or `Info.plist`/`AndroidManifest.xml` if using the audio packages.
56
-
57
47
  ## 🚀 Quick Start
58
48
 
59
49
  ```tsx
@@ -80,14 +70,13 @@ A floating chat bar appears automatically. Ask the AI to navigate, tap buttons,
80
70
 
81
71
  ### `<AIAgent>` Component
82
72
 
83
- The root provider. Wrap your app once at the top level. You can toggle between Text and Live Agent (Voice) modes.
73
+ The root provider. Wrap your app once at the top level.
84
74
 
85
75
  | Prop | Type | Default | Description |
86
76
  |------|------|---------|-------------|
87
77
  | `apiKey` | `string` | — | **Required.** Gemini API key. |
88
78
  | `model` | `string` | `'gemini-2.5-flash'` | Gemini model name. |
89
79
  | `navRef` | `NavigationContainerRef` | — | Navigation ref for auto-navigation. |
90
- | `language` | `'en' \| 'ar'` | `'en'` | UI language. Forces Gemini Voice to speak the chosen dialect natively. |
91
80
  | `maxSteps` | `number` | `10` | Max steps per task. |
92
81
  | `showChatBar` | `boolean` | `true` | Show the floating chat bar. |
93
82
  | `onResult` | `(result) => void` | — | Called when the agent finishes. |
package/lib/module/components/AIAgent.js CHANGED
@@ -165,8 +165,8 @@ export function AIAgent({
165
165
  logger.info('AIAgent', 'Creating VoiceService...');
166
166
  const runtimeTools = runtime.getTools();
167
167
  logger.info('AIAgent', `Registering ${runtimeTools.length} tools with VoiceService: ${runtimeTools.map(t => t.name).join(', ')}`);
168
- // Build the full voice system prompt (screen format + tool descriptions + guardrails)
169
- // This gives voice mode the same screen understanding as text mode
168
+ // Use voice-adapted system prompt — same core rules as text mode
169
+ // but without agent-loop directives that trigger autonomous actions
170
170
  const voicePrompt = buildVoiceSystemPrompt(language, instructions?.system);
171
171
  voiceServiceRef.current = new VoiceService({
172
172
  apiKey,
@@ -223,10 +223,12 @@ export function AIAgent({
223
223
  logger.info('AIAgent', '🎙️ Mic auto-started after connection');
224
224
  }
225
225
  });
226
- // Send initial screen context (tree) so the model knows what's on screen
226
+ // Send initial screen context so the model knows what's on screen.
227
+ // sendScreenContext uses turnComplete: false (passive context)
228
+ // so the model should NOT act on it until the user speaks.
227
229
  const initialContext = runtime.getScreenContext();
228
230
  voiceServiceRef.current?.sendScreenContext(initialContext);
229
- logger.info('AIAgent', '📡 Initial screen context sent to voice model');
231
+ logger.info('AIAgent', '📡 Initial screen context sent (passive, turnComplete=false)');
230
232
  }
231
233
  },
232
234
  onTranscript: (text, isFinal, role) => {
package/lib/module/components/AIAgent.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"names":["React","useCallback","useEffect","useMemo","useRef","useState","View","StyleSheet","AgentRuntime","GeminiProvider","AgentContext","AgentChatBar","AgentOverlay","logger","buildVoiceSystemPrompt","MCPBridge","VoiceService","AudioInputService","AudioOutputService","jsx","_jsx","jsxs","_jsxs","console","log","AIAgent","apiKey","model","navRef","language","maxSteps","showChatBar","children","onResult","interactiveBlacklist","interactiveWhitelist","onBeforeStep","onAfterStep","onBeforeTask","onAfterTask","transformScreenContent","customTools","instructions","stepDelay","mcpServerUrl","router","pathname","enableVoice","onTokenUsage","debug","setEnabled","info","rootViewRef","isThinking","setIsThinking","statusText","setStatusText","lastResult","setLastResult","mode","setMode","isMicActive","setIsMicActive","isSpeakerMuted","setIsSpeakerMuted","isAISpeaking","setIsAISpeaking","isVoiceConnected","setIsVoiceConnected","voiceServiceRef","audioInputRef","audioOutputRef","availableModes","modes","push","join","askUserResolverRef","config","onStatusUpdate","onAskUser","question","Promise","resolve","current","success","message","steps","provider","runtime","updateRefs","bridge","destroy","runtimeTools","getTools","length","map","t","name","voicePrompt","system","systemPrompt","tools","onError","err","error","initialize","then","ok","onAudioChunk","chunk","sendAudio","onPermissionDenied","warn","connect","onAudioResponse","audio","enqueue","onStatusChange","status","connected","start","initialContext","getScreenContext","sendScreenContext","onTranscript","text","isFinal","role","onToolCall","toolCall","JSON","stringify","args","result","executeTool","sendFunctionResponse","id","updatedContext","onTurnComplete","disconnect","stop","stopVoiceSession","handleSend","trim","resolver","execute","Provider","value","ref","style","styles","root","collapsable","visible","onSend","onDismiss","onModeChange","newMode","onStopSession","onMicToggle","active","onSpeakerToggle","muted","mute","unmute","create","flex"],"sourceRoot":"../../../src","sources":["components/AIAgent.tsx"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,OAAOA,KAAK,IACVC,WAAW,EACXC,SAAS,EACTC,OAAO,EACPC,MAAM,EACNC,QAAQ,QACH,OAAO;AACd,SAASC,IAAI,EAAEC,UAAU,QAAQ,cAAc;AAC/C,SAASC,YAAY,QAAQ,yBAAsB;AACnD,SAASC,cAAc,QAAQ,gCAA6B;AAC5D,SAASC,YAAY,QAAQ,uBAAoB;AACjD,SAASC,YAAY,QAAQ,mBAAgB;AAC7C,SAASC,YAAY,QAAQ,mBAAgB;AAC7C,SAASC,MAAM,QAAQ,oBAAiB;AACxC,SAASC,sBAAsB,QAAQ,yBAAsB;AAC7D,SAASC,SAAS,QAAQ,sBAAmB;AAC7C,SAASC,YAAY,QAAQ,6BAA0B;AACvD,SAASC,iBAAiB,QAAQ,kCAA+B;AACjE,SAASC,kBAAkB,QAAQ,mCAAgC;AAAC,SAAAC,GAAA,IAAAC,IAAA,EAAAC,IAAA,IAAAC,KAAA;AAGpE;AACAC,OAAO,CAACC,GAAG,CAAC,8BAA8B,CAAC;;AAE3C;;AA+DA;;AAEA,OAAO,SAASC,OAAOA,CAAC;EACtBC,MAAM;EACNC,KAAK,GAAG,kBAAkB;EAC1BC,MAAM;EACNC,QAAQ,GAAG,IAAI;EACfC,QAAQ,GAAG,EAAE;EACbC,WAAW,GAAG,IAAI;EAClBC,QAAQ;EACRC,QAAQ;EACR;EACAC,oBAAoB;EACpBC,oBAAoB;EACpBC,YAAY;EACZC,WAAW;EACXC,YAAY;EACZC,WAAW;EACXC,sBAAsB;EACtBC,WAAW;EACXC,YAAY;EACZC,SAAS;EACTC,YAAY;EACZC,MAAM;EACNC,QAAQ;EACRC,WAAW,GAAG,KAAK;EACnBC,YAAY;EACZC,KAAK,GAAG;AACI,CAAC,EAAE;EACf;EACAjD,KAAK,CAACE,SAAS,CAAC,MAAM;IACpBqB,OAAO,CAACC,GAAG,CAAC,wBAAwB,EAAEyB,KAAK,EAAE,mBAAmB,CAAC;IACjEpC,MAAM,CAACqC,UAAU,CAACD,KAAK,CAAC;IACxB,IAAIA,KAAK,EAAE;MACTpC,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,0BAA0B,CAAC;IACpD;EACF,CAAC,EAAE,CAACF,KAAK,CAAC,CAAC;EAEX,MAAMG,WAAW,GAAGhD,MAAM,CAAM,IAAI,CAAC;EACrC,MAAM,CAACiD,UAAU,EAAEC,aAAa,CAAC,GAAGjD,QAAQ,CAAC,KAAK,CAAC;EACnD,MAAM,CAACkD,UAAU,EAAEC,aAAa,CAAC,GAAGnD,QAAQ,CAAC,EAAE,CAAC;EAChD,MAAM,CAACoD,UAAU,EAAEC,aAAa,CAAC,GAAGrD,QAAQ,CAAyB,IAAI,CAAC;;EAE1E;EACA,MAAM,CAACsD,IAAI,EAAEC,OAAO,CAAC,GAAGvD,QAAQ,CAAY,MAAM,CAAC;EACnD,MAAM,CAACwD,WAAW,EAAEC,cAAc,CAAC,GAAGzD,QAAQ,CAAC,KAAK,CAAC;EACrD,MAAM,CAAC0D,cAAc,EAAEC,iBAAiB,CAAC,GAAG3D,QAAQ,CAAC,KAAK,CAAC;EAC3D,MAAM,CAAC4D,YAAY,EAAEC,eAAe,CAAC,GAAG7D,QAAQ,CAAC,KAAK,CAAC;EACvD,MAAM,CAAC8D,gBAAgB,EAAEC,mBAAmB,CAAC,GAAG/D,QAAQ,CAAC,KAAK,CAAC;EAE/D,MAAMgE,eAAe,GAAGjE,MAAM,CAAsB,IAAI,CAAC;EACzD,MAAMkE,aAAa,GAAGlE,MAAM,CAA2B,IAAI,CAAC;EAC5D,MAAMmE,cAAc,GAAGnE,MAAM,CAA4B,IAAI,CAAC;;EAE9D;EACA,MAAMoE,cAA2B,GAAGrE,OAAO,CAAC,MAAM;IAChD,MAAMsE,KAAkB,GAAG,CAAC,MAAM,CAAC;IACnC,IAAI1B,WAAW,EAAE0B,KAAK,CAACC,IAAI,CAAC,OAAO,CAAC;IACpC7D,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,oBAAoBsB,KAAK,CAACE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC9D,OAAOF,KAAK;EACd,CAAC,EAAE,CAAC1B,WAAW,CAAC,CAAC;;EAEjB;EACA,MAAM6B,kBAAkB,GAAGxE,MAAM,CAAoC,IAAI,CAAC;;EAE1E;;EAEA,MAAMyE,MAAmB,GAAG1E,OAAO,CAAC,OAAO;IACzCuB,MAAM;IACNC,KAAK;IACLE,QAAQ;IACRC,QAAQ;IACRI,oBAAoB;IACpBC,oBAAoB;IACpBC,YAAY;IACZC,WAAW;IACXC,YAAY;IACZC,WAAW;IACXC,sBAAsB;IACtBC,WAAW;IACXC,YAAY;IACZC,SAAS;IACTC,YAAY;IACZC,MAAM;IACNC,QAAQ;IACRgC,cAAc,EAAEtB,aAAa;IAC7BR,YAAY;IACZ;IACA+B,SAAS,EAAGC,QAAgB,IAAK;MAC/B,OAAO,IAAIC,OAAO,CAAUC,OAAO,IAAK;QACtCN,kBAAkB,CAACO,OAAO,GAAGD,OAAO;QACpC;QACAxB,aAAa,CAAC;UAAE0B,OAAO,EAAE,IAAI;UAAEC,OAAO,EAAE,KAAKL,QAAQ,EAAE;UAAEM,KAAK,EAAE;QAAG,CAAC,CAAC;QACrEhC,aAAa,CAAC,KAAK,CAAC;QACpBE,aAAa,CAAC,EAAE,CAAC;MACnB,CAAC,CAAC;IACJ;EACF,CAAC,CAAC,EAAE,CACF9B,MAAM,EAAEC,KAAK,EAAEE,QAAQ,EAAEC,QAAQ,EACjCI,oBAAoB,EAAEC,oBAAoB,EAC1CC,YAAY,EAAEC,WAAW,EAAEC,YAAY,EAAEC,WAAW,EACpDC,sBAAsB,EAAEC,WAAW,EAAEC,YAAY,EAAEC,SAAS,EAC5DC,YAAY,EAAEC,MAAM,EAAEC,QAAQ,EAAEE,YAAY,CAC7C,CAAC;EAEF,MAAMuC,QAAQ,GAAGpF,OAAO,CAAC,MAAM,IAAIM,cAAc,CAACiB,MAAM,EAAEC,KAAK,CAAC,EAAE,CAACD,MAAM,EAAEC,KAAK,CAAC,CAAC;EAElF,MAAM6D,OAAO,GAAGrF,OAAO,CACrB,MAAM,IAAIK,YAAY,CAAC+E,QAAQ,EAAEV,MAAM,EAAEzB,WAAW,CAAC+B,OAAO,EAAEvD,MAAM,CAAC;EACrE;EACA,CAAC2D,QAAQ,EAAEV,MAAM,CACnB,CAAC;;EAED;EACA3E,SAAS,CAAC,MAAM;IACdsF,OAAO,CAACC,UAAU,CAACrC,WAAW,CAAC+B,OAAO,EAAEvD,MAAM,CAAC;EACjD,CAAC,EAAE,CAAC4D,OAAO,EAAE5D,MAAM,CAAC,CAAC;;EAErB;;EAEA1B,SAAS,CAAC,MAAM;IACd,IAAI,CAAC0C,YAAY,EAAE;IAEnB/B,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4BP,YAAY,EAAE,CAAC;IAClE,MAAM8C,MAAM,GAAG,IAAI3E,SAAS,CAAC6B,YAAY,EAAE4C,OAAO,CAAC;IAEnD,OAAO,MAAM;MACXE,MAAM,CAACC,OAAO,CAAC,CAAC;IAClB,CAAC;EACH,CAAC,EAAE,CAAC/C,YAAY,EAAE4C,OAAO,CAAC,CAAC;;EAE3B;;EAEA;EACAtF,SAAS,CAAC,MAAM;IACd,IAAIyD,IAAI,KAAK,MAAM,EAAE;MACnB9C,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,yCAAyC,CAAC;MACjE;IACF;IAEAtC,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,oBAAoBQ,IAAI,oCAAoC,CAAC;;IAEpF;IACA,IAAI,CAACU,eAAe,CAACc,OAAO,EAAE;MAC5BtE,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,0BAA0B,CAAC;MAClD,MAAMyC,YAAY,GAAGJ,OAAO,CAACK,QAAQ,CAAC,CAAC;MACvChF,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,eAAeyC,YAAY,CAACE,MAAM,6BAA6BF,YAAY,CAACG,GAAG,CAACC,CAAC,IAAIA,CAAC,CAACC,IAAI,CAAC,CAACtB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;MACjI;MACA;MACA,MAAMuB,WAAW,GAAGpF,sBAAsB,CAACe,QAAQ,EAAEa,YAAY,EAAEyD,MAAM,CAAC;MAC1E9B,eAAe,CAACc,OAAO,GAAG,IAAInE,YAAY,CAAC;QACzCU,MAAM;QACN0E,YAAY,EAAEF,WAAW;QACzBG,KAAK,EAAET,YAAY;QACnB/D;MACF,CAAC,CAAC;MACFhB,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,8DAA8D,CAAC;IACxF;;IAEA;IACA,IAAI,CAACoB,cAAc,CAACY,OAAO,EAAE;MAC3BtE,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC;MACxDoB,cAAc,CAACY,OAAO,GAAG,IAAIjE,kBAAkB,CAAC;QAC9CoF,OAAO,EAAGC,GAAG,IAAK1F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,sBAAsBD,GAAG,EAAE;MACvE,CAAC,CAAC;MACFhC,cAAc,CAACY,OAAO,CAACsB,UAAU,CAAC,CAAC,CAACC,IAAI,CAAEC,EAAE,IAAK;QAC/C9F,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,mCAAmCwD,EAAE,EAAE,CAAC;MACjE,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,CAACrC,aAAa,CAACa,OAAO,EAAE;MAC1BtE,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,+BAA+B,CAAC;MACvDmB,aAAa,CAACa,OAAO,GAAG,IAAIlE,iBAAiB,CAAC;QAC5C;QACA2F,YAAY,EAAGC,KAAK,IAAK;UACvBhG,MAAM,CAACoC,KAAK,CAAC,SAAS,EAAE,cAAc4D,KAAK,CAACf,MAAM,QAAQ,CAAC;UAC3DzB,eAAe,CAACc,OAAO,EAAE2B,SAAS,CAACD,KAAK,CAAC;QAC3C,CAAC;QACDP,OAAO,EAAGC,GAAG,IAAK1F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,qBAAqBD,GAAG,EAAE,CAAC;QACrEQ,kBAAkB,EAAEA,CAAA,KAAMlG,MAAM,CAACmG,IAAI,CAAC,SAAS,EAAE,+BAA+B;MAClF,CAAC,CAAC;IACJ;;IAEA;IACAnG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4B,CAAC;IACpDkB,eAAe,CAACc,OAAO,CAAC8B,OAAO,CAAC;MAC9BC,eAAe,EAAGC,KAAK,IAAK;QAC1BtG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4BgE,KAAK,CAACrB,MAAM,SAAS,CAAC;QACzE5B,eAAe,CAAC,IAAI,CAAC;QACrBK,cAAc,CAACY,OAAO,EAAEiC,OAAO,CAACD,KAAK,CAAC;MACxC,CAAC;MACDE,cAAc,EAAGC,MAAM,IAAK;QAC1BzG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,iBAAiBmE,MAAM,EAAE,CAAC;QACjD,MAAMC,SAAS,GAAGD,MAAM,KAAK,WAAW;QACxClD,mBAAmB,CAACmD,SAAS,CAAC;QAC9B,IAAIA,SAAS,EAAE;UACb1G,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,iDAAiD,CAAC;UACzE;UACAmB,aAAa,CAACa,OAAO,EAAEqC,KAAK,CAAC,CAAC,CAACd,IAAI,CAAEC,EAAE,IAAK;YAC1C,IAAIA,EAAE,EAAE;cACN7C,cAAc,CAAC,IAAI,CAAC;cACpBjD,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,uCAAuC,CAAC;YACjE;UACF,CAAC,CAAC;UACF;UACA,MAAMsE,cAAc,GAAGjC,OAAO,CAACkC,gBAAgB,CAAC,CAAC;UACjDrD,eAAe,CAACc,OAAO,EAAEwC,iBAAiB,CAACF,cAAc,CAAC;UAC1D5G,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,+CAA+C,CAAC;QACzE;MACF,CAAC;MACDyE,YAAY,EAAEA,CAACC,IAAI,EAAEC,OAAO,EAAEC,IAAI,KAAK;QACrClH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,eAAe4E,IAAI,YAAYD,OAAO,OAAOD,IAAI,GAAG,CAAC;MAC9E,CAAC;MACDG,UAAU,EAAE,MAAOC,QAAQ,IAAK;QAC9BpH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,oBAAoB8E,QAAQ,CAAChC,IAAI,IAAIiC,IAAI,CAACC,SAAS,CAACF,QAAQ,CAACG,IAAI,CAAC,GAAG,CAAC;QAC7F;QACA,MAAMC,MAAM,GAAG,MAAM7C,OAAO,CAAC8C,WAAW,CAACL,QAAQ,CAAChC,IAAI,EAAEgC,QAAQ,CAACG,IAAI,CAAC;QACtEvH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,sBAAsBkF,MAAM,EAAE,CAAC;QACtDhE,eAAe,CAACc,OAAO,EAAEoD,oBAAoB,CAACN,QAAQ,CAAChC,IAAI,EAAEgC,QAAQ,CAACO,EAAE,EAAE;UAAEH;QAAO,CAAC,CAAC;;QAErF;QACA;QACA,MAAMI,cAAc,GAAGjD,OAAO,CAACkC,gBAAgB,CAAC,CAAC;QACjDrD,eAAe,CAACc,OAAO,EAAEwC,iBAAiB,CAACc,cAAc,CAAC;QAC1D5H,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gDAAgD,CAAC;MAC1E,CAAC;MACDmD,OAAO,EAAGC,GAAG,IAAK;QAChB1F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,uBAAuBD,GAAG,EAAE,CAAC;MACvD,CAAC;MACDmC,cAAc,EAAEA,CAAA,KAAM;QACpB7H,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC;QAC1Ce,eAAe,CAAC,KAAK,CAAC;MACxB;IACF,CAAC,CAAC;;IAEF;IACA,OAAO,MAAM;MACXrD,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,wCAAwCQ,IAAI,SAAS,CAAC;MAC7EU,eAAe,CAACc,OAAO,EAAEwD,UAAU,CAAC,CAAC;MACrCtE,eAAe,CAACc,OAAO,GAAG,IAAI,CAAC,CAAC;MAChCb,aAAa,CAACa,OAAO,EAAEyD,IAAI,CAAC,CAAC;MAC7B9E,cAAc,CAAC,KAAK,CAAC;MACrBI,eAAe,CAAC,KAAK,CAAC;MACtBE,mBAAmB,CAAC,KAAK,CAAC;IAC5B,CAAC;IACD;EACF,CAAC,EAAE,CAACT,IAAI,EAAEjC,MAAM,EAAE8D,OAAO,EAAE3D,QAAQ,EAAEa,YAAY,CAAC,CAAC;;EAEnD;;EAEA,MAAMmG,gBAAgB,GAAG5I,WAAW,CAAC,MAAM;IACzCY,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,6CAA6C,CAAC;IACrE;IACAmB,aAAa,CAACa,OAAO,EAAEyD,IAAI,CAAC,CAAC;IAC7B;IACArE,cAAc,CAACY,OAAO,EAAEyD,IAAI,CAAC,CAAC;IAC9B;IACAvE,eAAe,CAACc,OAAO,EAAEwD,UAAU,CAAC,CAAC;IACrCtE,eAAe,CAACc,OAAO,GAAG,IAAI;IAC9B;IACArB,cAAc,CAAC,KAAK,CAAC;IACrBI,eAAe,CAAC,KAAK,CAAC;IACtBE,mBAAmB,CAAC,KAAK,CAAC;IAC1B;IACAR,OAAO,CAAC,MAAM,CAAC;IACf/C,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC;EAC1D,CAAC,EAAE,CAACqC,OAAO,CAAC,CAAC;;EAEb;;EAEA,MAAMsD,UAAU,GAAG7I,WAAW,CAAC,MAAOoF,OAAe,IAAK;IACxD,IAAI,CAACA,OAAO,CAAC0D,IAAI,CAAC,CAAC,EAAE;IAErBlI,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,kBAAkBkC,OAAO,GAAG,CAAC;;IAEpD;IACA,IAAIT,kBAAkB,CAACO,OAAO,EAAE;MAC9B,MAAM6D,QAAQ,GAAGpE,kBAAkB,CAACO,OAAO;MAC3CP,kBAAkB,CAACO,OAAO,GAAG,IAAI;MACjC7B,aAAa,CAAC,IAAI,CAAC;MACnBE,aAAa,CAAC,2BAA2B,CAAC;MAC1CE,aAAa,CAAC,IAAI,CAAC;MACnBsF,QAAQ,CAAC3D,OAAO,CAAC;MACjB;IACF;;IAEA;IACA/B,aAAa,CAAC,IAAI,CAAC;IACnBE,aAAa,CAAC,aAAa,CAAC;IAC5BE,aAAa,CAAC,IAAI,CAAC;IAEnB,IAAI;MACF;MACA8B,OAAO,CAACC,UAAU,CAACrC,WAAW,CAAC+B,OAAO,EAAEvD,MAAM,CAAC;MAE/C,MAAMyG,MAAM,GAAG,MAAM7C,OAAO,CAACyD,OAAO,CAAC5D,OAAO,CAAC;MAE7C3B,aAAa,CAAC2E,MAAM,CAAC;MACrBpG,QAAQ,GAAGoG,MAAM,CAAC;MAElBxH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,WAAWkF,MAAM,CAACjD,OAAO,GAAG,GAAG,GAAG,GAAG,IAAIiD,MAAM,CAAChD,OAAO,EAAE,CAAC;IACnF,CAAC,CAAC,OAAOmB,KAAU,EAAE;MACnB3F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,mBAAmB,EAAEA,KAAK,CAAC;MACnD9C,aAAa,CAAC;QACZ0B,OAAO,EAAE,KAAK;QACdC,OAAO,EAAE,UAAUmB,KAAK,CAACnB,OAAO,EAAE;QAClCC,KAAK,EAAE;MACT,CAAC,CAAC;IACJ,CAAC,SAAS;MACRhC,aAAa,CAAC,KAAK,CAAC;MACpBE,aAAa,CAAC,EAAE,CAAC;IACnB;EACF,CAAC,EAAE,CAACgC,OAAO,EAAE5D,MAAM,EAAEK,QAAQ,CAAC,CAAC;;EAE/B;;EAEA,oBACEX,KAAA,CAACZ,YAAY,CAACwI,QAAQ;IAACC,KAAK,EAAE3D,OAAQ;IAAAxD,QAAA,gBACpCZ,IAAA,CAACd,IAAI;MAAC8I,GAAG,EAAEhG,WAAY;MAACiG,KAAK,EAAEC,MAAM,CAACC,IAAK;MAACC,WAAW,EAAE,KAAM;MAAAxH,QAAA,EAC5DA;IAAQ,CACL,CAAC,eAGPZ,IAAA,CAACR,YAAY;MAAC6I,OAAO,EAAEpG,UAAW;MAACE,UAAU,EAAEA;IAAW,CAAE,CAAC,EAG5DxB,WAAW,iBACVX,IAAA,CAACT,YAAY;MACX+I,MAAM,EAAEZ,UAAW;MACnBzF,UAAU,EAAEA,UAAW;MACvBI,UAAU,EAAEA,UAAW;MACvB5B,QAAQ,EAAEA,QAAS;MACnB8H,SAAS,EAAEA,CAAA,KAAMjG,aAAa,CAAC,IAAI,CAAE;MACrCc,cAAc,EAAEA,cAAe;MAC/Bb,IAAI,EAAEA,IAAK;MACXiG,YAAY,EAAGC,OAAO,IAAK;QACzBhJ,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gBAAgBQ,IAAI,MAAMkG,OAAO,EAAE,CAAC;QAC3DjG,OAAO,CAACiG,OAAO,CAAC;MAClB,CAAE;MACFhG,WAAW,EAAEA,WAAY;MACzBE,cAAc,EAAEA,cAAe;MAC/BE,YAAY,EAAEA,YAAa;MAC3B6F,aAAa,EAAEjB,gBAAiB;MAChC1E,gBAAgB,EAAEA,gBAAiB;MACnC4F,WAAW,EAAGC,MAAM,IAAK;QACvB,IAAIA,MAAM,IAAI,CAAC7F,gBAAgB,EAAE;UAC/BtD,MAAM,CAACmG,IAAI,CAAC,SAAS,EAAE,oDAAoD,CAAC;UAC5E;QACF;QACAnG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,eAAe6G,MAAM,GAAG,IAAI,GAAG,KAAK,EAAE,CAAC;QAC9DlG,cAAc,CAACkG,MAAM,CAAC;QACtB,IAAIA,MAAM,EAAE;UACVnJ,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC;UAChDmB,aAAa,CAACa,OAAO,EAAEqC,KAAK,CAAC,CAAC,CAACd,IAAI,CAAEC,EAAE,IAAK;YAC1C9F,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4BwD,EAAE,EAAE,CAAC;UAC1D,CAAC,CAAC;QACJ,CAAC,MAAM;UACL9F,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC;UAChDmB,aAAa,CAACa,OAAO,EAAEyD,IAAI,CAAC,CAAC;QAC/B;MACF,CAAE;MACFqB,eAAe,EAAGC,KAAK,IAAK;QAC1BrJ,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,mBAAmB+G,KAAK,GAAG,OAAO,GAAG,SAAS,EAAE,CAAC;QACxElG,iBAAiB,CAACkG,KAAK,CAAC;QACxB,IAAIA,KAAK,EAAE;UACT3F,cAAc,CAACY,OAAO,EAAEgF,IAAI,CAAC,CAAC;QAChC,CAAC,MAAM;UACL5F,cAAc,CAACY,OAAO,EAAEiF,MAAM,CAAC,CAAC;QAClC;MACF;IAAE,CAEH,CACF;EAAA,CACoB,CAAC;AAE5B;AAGA,MAAMd,MAAM,GAAG/I,UAAU,CAAC8J,MAAM,CAAC;EAC/Bd,IAAI,EAAE;IACJe,IAAI,EAAE;EACR;AACF,CAAC,CAAC","ignoreList":[]}
1
+ {"version":3,"names":["React","useCallback","useEffect","useMemo","useRef","useState","View","StyleSheet","AgentRuntime","GeminiProvider","AgentContext","AgentChatBar","AgentOverlay","logger","buildVoiceSystemPrompt","MCPBridge","VoiceService","AudioInputService","AudioOutputService","jsx","_jsx","jsxs","_jsxs","console","log","AIAgent","apiKey","model","navRef","language","maxSteps","showChatBar","children","onResult","interactiveBlacklist","interactiveWhitelist","onBeforeStep","onAfterStep","onBeforeTask","onAfterTask","transformScreenContent","customTools","instructions","stepDelay","mcpServerUrl","router","pathname","enableVoice","onTokenUsage","debug","setEnabled","info","rootViewRef","isThinking","setIsThinking","statusText","setStatusText","lastResult","setLastResult","mode","setMode","isMicActive","setIsMicActive","isSpeakerMuted","setIsSpeakerMuted","isAISpeaking","setIsAISpeaking","isVoiceConnected","setIsVoiceConnected","voiceServiceRef","audioInputRef","audioOutputRef","availableModes","modes","push","join","askUserResolverRef","config","onStatusUpdate","onAskUser","question","Promise","resolve","current","success","message","steps","provider","runtime","updateRefs","bridge","destroy","runtimeTools","getTools","length","map","t","name","voicePrompt","system","systemPrompt","tools","onError","err","error","initialize","then","ok","onAudioChunk","chunk","sendAudio","onPermissionDenied","warn","connect","onAudioResponse","audio","enqueue","onStatusChange","status","connected","start","initialContext","getScreenContext","sendScreenContext","onTranscript","text","isFinal","role","onToolCall","toolCall","JSON","stringify","args","result","executeTool","sendFunctionResponse","id","updatedContext","onTurnComplete","disconnect","stop","stopVoiceSession","handleSend","trim","resolver","execute","Provider","value","ref","style","styles","root","collapsable","visible","onSend","onDismiss","onModeChange","newMode","onStopSession","onMicToggle","active","onSpeakerToggle","muted","mute","unmute","create","flex"],"sourceRoot":"../../../src","sources":["components/AIAgent.tsx"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,OAAOA,KAAK,IACVC,WAAW,EACXC,SAAS,EACTC,OAAO,EACPC,MAAM,EACNC,QAAQ,QACH,OAAO;AACd,SAASC,IAAI,EAAEC,UAAU,QAAQ,cAAc;AAC/C,SAASC,YAAY,QAAQ,yBAAsB;AACnD,SAASC,cAAc,QAAQ,gCAA6B;AAC5D,SAASC,YAAY,QAAQ,uBAAoB;AACjD,SAASC,YAAY,QAAQ,mBAAgB;AAC7C,SAASC,YAAY,QAAQ,mBAAgB;AAC7C,SAASC,MAAM,QAAQ,oBAAiB;AACxC,SAASC,sBAAsB,QAAQ,yBAAsB;AAC7D,SAASC,SAAS,QAAQ,sBAAmB;AAC7C,SAASC,YAAY,QAAQ,6BAA0B;AACvD,SAASC,iBAAiB,QAAQ,kCAA+B;AACjE,SAASC,kBAAkB,QAAQ,mCAAgC;AAAC,SAAAC,GAAA,IAAAC,IAAA,EAAAC,IAAA,IAAAC,KAAA;AAGpE;AACAC,OAAO,CAACC,GAAG,CAAC,8BAA8B,CAAC;;AAE3C;;AA+DA;;AAEA,OAAO,SAASC,OAAOA,CAAC;EACtBC,MAAM;EACNC,KAAK,GAAG,kBAAkB;EAC1BC,MAAM;EACNC,QAAQ,GAAG,IAAI;EACfC,QAAQ,GAAG,EAAE;EACbC,WAAW,GAAG,IAAI;EAClBC,QAAQ;EACRC,QAAQ;EACR;EACAC,oBAAoB;EACpBC,oBAAoB;EACpBC,YAAY;EACZC,WAAW;EACXC,YAAY;EACZC,WAAW;EACXC,sBAAsB;EACtBC,WAAW;EACXC,YAAY;EACZC,SAAS;EACTC,YAAY;EACZC,MAAM;EACNC,QAAQ;EACRC,WAAW,GAAG,KAAK;EACnBC,YAAY;EACZC,KAAK,GAAG;AACI,CAAC,EAAE;EACf;EACAjD,KAAK,CAACE,SAAS,CAAC,MAAM;IACpBqB,OAAO,CAACC,GAAG,CAAC,wBAAwB,EAAEyB,KAAK,EAAE,mBAAmB,CAAC;IACjEpC,MAAM,CAACqC,UAAU,CAACD,KAAK,CAAC;IACxB,IAAIA,KAAK,EAAE;MACTpC,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,0BAA0B,CAAC;IACpD;EACF,CAAC,EAAE,CAACF,KAAK,CAAC,CAAC;EAEX,MAAMG,WAAW,GAAGhD,MAAM,CAAM,IAAI,CAAC;EACrC,MAAM,CAACiD,UAAU,EAAEC,aAAa,CAAC,GAAGjD,QAAQ,CAAC,KAAK,CAAC;EACnD,MAAM,CAACkD,UAAU,EAAEC,aAAa,CAAC,GAAGnD,QAAQ,CAAC,EAAE,CAAC;EAChD,MAAM,CAACoD,UAAU,EAAEC,aAAa,CAAC,GAAGrD,QAAQ,CAAyB,IAAI,CAAC;;EAE1E;EACA,MAAM,CAACsD,IAAI,EAAEC,OAAO,CAAC,GAAGvD,QAAQ,CAAY,MAAM,CAAC;EACnD,MAAM,CAACwD,WAAW,EAAEC,cAAc,CAAC,GAAGzD,QAAQ,CAAC,KAAK,CAAC;EACrD,MAAM,CAAC0D,cAAc,EAAEC,iBAAiB,CAAC,GAAG3D,QAAQ,CAAC,KAAK,CAAC;EAC3D,MAAM,CAAC4D,YAAY,EAAEC,eAAe,CAAC,GAAG7D,QAAQ,CAAC,KAAK,CAAC;EACvD,MAAM,CAAC8D,gBAAgB,EAAEC,mBAAmB,CAAC,GAAG/D,QAAQ,CAAC,KAAK,CAAC;EAE/D,MAAMgE,eAAe,GAAGjE,MAAM,CAAsB,IAAI,CAAC;EACzD,MAAMkE,aAAa,GAAGlE,MAAM,CAA2B,IAAI,CAAC;EAC5D,MAAMmE,cAAc,GAAGnE,MAAM,CAA4B,IAAI,CAAC;;EAE9D;EACA,MAAMoE,cAA2B,GAAGrE,OAAO,CAAC,MAAM;IAChD,MAAMsE,KAAkB,GAAG,CAAC,MAAM,CAAC;IACnC,IAAI1B,WAAW,EAAE0B,KAAK,CAACC,IAAI,CAAC,OAAO,CAAC;IACpC7D,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,oBAAoBsB,KAAK,CAACE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC9D,OAAOF,KAAK;EACd,CAAC,EAAE,CAAC1B,WAAW,CAAC,CAAC;;EAEjB;EACA,MAAM6B,kBAAkB,GAAGxE,MAAM,CAAoC,IAAI,CAAC;;EAE1E;;EAEA,MAAMyE,MAAmB,GAAG1E,OAAO,CAAC,OAAO;IACzCuB,MAAM;IACNC,KAAK;IACLE,QAAQ;IACRC,QAAQ;IACRI,oBAAoB;IACpBC,oBAAoB;IACpBC,YAAY;IACZC,WAAW;IACXC,YAAY;IACZC,WAAW;IACXC,sBAAsB;IACtBC,WAAW;IACXC,YAAY;IACZC,SAAS;IACTC,YAAY;IACZC,MAAM;IACNC,QAAQ;IACRgC,cAAc,EAAEtB,aAAa;IAC7BR,YAAY;IACZ;IACA+B,SAAS,EAAGC,QAAgB,IAAK;MAC/B,OAAO,IAAIC,OAAO,CAAUC,OAAO,IAAK;QACtCN,kBAAkB,CAACO,OAAO,GAAGD,OAAO;QACpC;QACAxB,aAAa,CAAC;UAAE0B,OAAO,EAAE,IAAI;UAAEC,OAAO,EAAE,KAAKL,QAAQ,EAAE;UAAEM,KAAK,EAAE;QAAG,CAAC,CAAC;QACrEhC,aAAa,CAAC,KAAK,CAAC;QACpBE,aAAa,CAAC,EAAE,CAAC;MACnB,CAAC,CAAC;IACJ;EACF,CAAC,CAAC,EAAE,CACF9B,MAAM,EAAEC,KAAK,EAAEE,QAAQ,EAAEC,QAAQ,EACjCI,oBAAoB,EAAEC,oBAAoB,EAC1CC,YAAY,EAAEC,WAAW,EAAEC,YAAY,EAAEC,WAAW,EACpDC,sBAAsB,EAAEC,WAAW,EAAEC,YAAY,EAAEC,SAAS,EAC5DC,YAAY,EAAEC,MAAM,EAAEC,QAAQ,EAAEE,YAAY,CAC7C,CAAC;EAEF,MAAMuC,QAAQ,GAAGpF,OAAO,CAAC,MAAM,IAAIM,cAAc,CAACiB,MAAM,EAAEC,KAAK,CAAC,EAAE,CAACD,MAAM,EAAEC,KAAK,CAAC,CAAC;EAElF,MAAM6D,OAAO,GAAGrF,OAAO,CACrB,MAAM,IAAIK,YAAY,CAAC+E,QAAQ,EAAEV,MAAM,EAAEzB,WAAW,CAAC+B,OAAO,EAAEvD,MAAM,CAAC;EACrE;EACA,CAAC2D,QAAQ,EAAEV,MAAM,CACnB,CAAC;;EAED;EACA3E,SAAS,CAAC,MAAM;IACdsF,OAAO,CAACC,UAAU,CAACrC,WAAW,CAAC+B,OAAO,EAAEvD,MAAM,CAAC;EACjD,CAAC,EAAE,CAAC4D,OAAO,EAAE5D,MAAM,CAAC,CAAC;;EAErB;;EAEA1B,SAAS,CAAC,MAAM;IACd,IAAI,CAAC0C,YAAY,EAAE;IAEnB/B,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4BP,YAAY,EAAE,CAAC;IAClE,MAAM8C,MAAM,GAAG,IAAI3E,SAAS,CAAC6B,YAAY,EAAE4C,OAAO,CAAC;IAEnD,OAAO,MAAM;MACXE,MAAM,CAACC,OAAO,CAAC,CAAC;IAClB,CAAC;EACH,CAAC,EAAE,CAAC/C,YAAY,EAAE4C,OAAO,CAAC,CAAC;;EAE3B;;EAEA;EACAtF,SAAS,CAAC,MAAM;IACd,IAAIyD,IAAI,KAAK,MAAM,EAAE;MACnB9C,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,yCAAyC,CAAC;MACjE;IACF;IAEAtC,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,oBAAoBQ,IAAI,oCAAoC,CAAC;;IAEpF;IACA,IAAI,CAACU,eAAe,CAACc,OAAO,EAAE;MAC5BtE,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,0BAA0B,CAAC;MAClD,MAAMyC,YAAY,GAAGJ,OAAO,CAACK,QAAQ,CAAC,CAAC;MACvChF,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,eAAeyC,YAAY,CAACE,MAAM,6BAA6BF,YAAY,CAACG,GAAG,CAACC,CAAC,IAAIA,CAAC,CAACC,IAAI,CAAC,CAACtB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;MACjI;MACA;MACA,MAAMuB,WAAW,GAAGpF,sBAAsB,CAACe,QAAQ,EAAEa,YAAY,EAAEyD,MAAM,CAAC;MAC1E9B,eAAe,CAACc,OAAO,GAAG,IAAInE,YAAY,CAAC;QACzCU,MAAM;QACN0E,YAAY,EAAEF,WAAW;QACzBG,KAAK,EAAET,YAAY;QACnB/D;MACF,CAAC,CAAC;MACFhB,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,8DAA8D,CAAC;IACxF;;IAEA;IACA,IAAI,CAACoB,cAAc,CAACY,OAAO,EAAE;MAC3BtE,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC;MACxDoB,cAAc,CAACY,OAAO,GAAG,IAAIjE,kBAAkB,CAAC;QAC9CoF,OAAO,EAAGC,GAAG,IAAK1F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,sBAAsBD,GAAG,EAAE;MACvE,CAAC,CAAC;MACFhC,cAAc,CAACY,OAAO,CAACsB,UAAU,CAAC,CAAC,CAACC,IAAI,CAAEC,EAAE,IAAK;QAC/C9F,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,mCAAmCwD,EAAE,EAAE,CAAC;MACjE,CAAC,CAAC;IACJ;;IAEA;IACA,IAAI,CAACrC,aAAa,CAACa,OAAO,EAAE;MAC1BtE,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,+BAA+B,CAAC;MACvDmB,aAAa,CAACa,OAAO,GAAG,IAAIlE,iBAAiB,CAAC;QAC5C;QACA2F,YAAY,EAAGC,KAAK,IAAK;UACvBhG,MAAM,CAACoC,KAAK,CAAC,SAAS,EAAE,cAAc4D,KAAK,CAACf,MAAM,QAAQ,CAAC;UAC3DzB,eAAe,CAACc,OAAO,EAAE2B,SAAS,CAACD,KAAK,CAAC;QAC3C,CAAC;QACDP,OAAO,EAAGC,GAAG,IAAK1F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,qBAAqBD,GAAG,EAAE,CAAC;QACrEQ,kBAAkB,EAAEA,CAAA,KAAMlG,MAAM,CAACmG,IAAI,CAAC,SAAS,EAAE,+BAA+B;MAClF,CAAC,CAAC;IACJ;;IAEA;IACAnG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4B,CAAC;IACpDkB,eAAe,CAACc,OAAO,CAAC8B,OAAO,CAAC;MAC9BC,eAAe,EAAGC,KAAK,IAAK;QAC1BtG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4BgE,KAAK,CAACrB,MAAM,SAAS,CAAC;QACzE5B,eAAe,CAAC,IAAI,CAAC;QACrBK,cAAc,CAACY,OAAO,EAAEiC,OAAO,CAACD,KAAK,CAAC;MACxC,CAAC;MACDE,cAAc,EAAGC,MAAM,IAAK;QAC1BzG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,iBAAiBmE,MAAM,EAAE,CAAC;QACjD,MAAMC,SAAS,GAAGD,MAAM,KAAK,WAAW;QACxClD,mBAAmB,CAACmD,SAAS,CAAC;QAC9B,IAAIA,SAAS,EAAE;UACb1G,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,iDAAiD,CAAC;UACzE;UACAmB,aAAa,CAACa,OAAO,EAAEqC,KAAK,CAAC,CAAC,CAACd,IAAI,CAAEC,EAAE,IAAK;YAC1C,IAAIA,EAAE,EAAE;cACN7C,cAAc,CAAC,IAAI,CAAC;cACpBjD,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,uCAAuC,CAAC;YACjE;UACF,CAAC,CAAC;UACF;UACA;UACA;UACA,MAAMsE,cAAc,GAAGjC,OAAO,CAACkC,gBAAgB,CAAC,CAAC;UACjDrD,eAAe,CAACc,OAAO,EAAEwC,iBAAiB,CAACF,cAAc,CAAC;UAC1D5G,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,8DAA8D,CAAC;QACxF;MACF,CAAC;MACDyE,YAAY,EAAEA,CAACC,IAAI,EAAEC,OAAO,EAAEC,IAAI,KAAK;QACrClH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,eAAe4E,IAAI,YAAYD,OAAO,OAAOD,IAAI,GAAG,CAAC;MAC9E,CAAC;MACDG,UAAU,EAAE,MAAOC,QAAQ,IAAK;QAC9BpH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,oBAAoB8E,QAAQ,CAAChC,IAAI,IAAIiC,IAAI,CAACC,SAAS,CAACF,QAAQ,CAACG,IAAI,CAAC,GAAG,CAAC;QAC7F;QACA,MAAMC,MAAM,GAAG,MAAM7C,OAAO,CAAC8C,WAAW,CAACL,QAAQ,CAAChC,IAAI,EAAEgC,QAAQ,CAACG,IAAI,CAAC;QACtEvH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,sBAAsBkF,MAAM,EAAE,CAAC;QAEtDhE,eAAe,CAACc,OAAO,EAAEoD,oBAAoB,CAACN,QAAQ,CAAChC,IAAI,EAAEgC,QAAQ,CAACO,EAAE,EAAE;UAAEH;QAAO,CAAC,CAAC;;QAErF;QACA;QACA,MAAMI,cAAc,GAAGjD,OAAO,CAACkC,gBAAgB,CAAC,CAAC;QACjDrD,eAAe,CAACc,OAAO,EAAEwC,iBAAiB,CAACc,cAAc,CAAC;QAC1D5H,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gDAAgD,CAAC;MAC1E,CAAC;MACDmD,OAAO,EAAGC,GAAG,IAAK;QAChB1F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,uBAAuBD,GAAG,EAAE,CAAC;MACvD,CAAC;MACDmC,cAAc,EAAEA,CAAA,KAAM;QACpB7H,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC;QAC1Ce,eAAe,CAAC,KAAK,CAAC;MACxB;IACF,CAAC,CAAC;;IAEF;IACA,OAAO,MAAM;MACXrD,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,wCAAwCQ,IAAI,SAAS,CAAC;MAC7EU,eAAe,CAACc,OAAO,EAAEwD,UAAU,CAAC,CAAC;MACrCtE,eAAe,CAACc,OAAO,GAAG,IAAI,CAAC,CAAC;MAChCb,aAAa,CAACa,OAAO,EAAEyD,IAAI,CAAC,CAAC;MAC7B9E,cAAc,CAAC,KAAK,CAAC;MACrBI,eAAe,CAAC,KAAK,CAAC;MACtBE,mBAAmB,CAAC,KAAK,CAAC;IAC5B,CAAC;IACD;EACF,CAAC,EAAE,CAACT,IAAI,EAAEjC,MAAM,EAAE8D,OAAO,EAAE3D,QAAQ,EAAEa,YAAY,CAAC,CAAC;;EAEnD;;EAEA,MAAMmG,gBAAgB,GAAG5I,WAAW,CAAC,MAAM;IACzCY,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,6CAA6C,CAAC;IACrE;IACAmB,aAAa,CAACa,OAAO,EAAEyD,IAAI,CAAC,CAAC;IAC7B;IACArE,cAAc,CAACY,OAAO,EAAEyD,IAAI,CAAC,CAAC;IAC9B;IACAvE,eAAe,CAACc,OAAO,EAAEwD,UAAU,CAAC,CAAC;IACrCtE,eAAe,CAACc,OAAO,GAAG,IAAI;IAC9B;IACArB,cAAc,CAAC,KAAK,CAAC;IACrBI,eAAe,CAAC,KAAK,CAAC;IACtBE,mBAAmB,CAAC,KAAK,CAAC;IAC1B;IACAR,OAAO,CAAC,MAAM,CAAC;IACf/C,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC;EAC1D,CAAC,EAAE,CAACqC,OAAO,CAAC,CAAC;;EAEb;;EAEA,MAAMsD,UAAU,GAAG7I,WAAW,CAAC,MAAOoF,OAAe,IAAK;IACxD,IAAI,CAACA,OAAO,CAAC0D,IAAI,CAAC,CAAC,EAAE;IAErBlI,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,kBAAkBkC,OAAO,GAAG,CAAC;;IAEpD;IACA,IAAIT,kBAAkB,CAACO,OAAO,EAAE;MAC9B,MAAM6D,QAAQ,GAAGpE,kBAAkB,CAACO,OAAO;MAC3CP,kBAAkB,CAACO,OAAO,GAAG,IAAI;MACjC7B,aAAa,CAAC,IAAI,CAAC;MACnBE,aAAa,CAAC,2BAA2B,CAAC;MAC1CE,aAAa,CAAC,IAAI,CAAC;MACnBsF,QAAQ,CAAC3D,OAAO,CAAC;MACjB;IACF;;IAEA;IACA/B,aAAa,CAAC,IAAI,CAAC;IACnBE,aAAa,CAAC,aAAa,CAAC;IAC5BE,aAAa,CAAC,IAAI,CAAC;IAEnB,IAAI;MACF;MACA8B,OAAO,CAACC,UAAU,CAACrC,WAAW,CAAC+B,OAAO,EAAEvD,MAAM,CAAC;MAE/C,MAAMyG,MAAM,GAAG,MAAM7C,OAAO,CAACyD,OAAO,CAAC5D,OAAO,CAAC;MAE7C3B,aAAa,CAAC2E,MAAM,CAAC;MACrBpG,QAAQ,GAAGoG,MAAM,CAAC;MAElBxH,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,WAAWkF,MAAM,CAACjD,OAAO,GAAG,GAAG,GAAG,GAAG,IAAIiD,MAAM,CAAChD,OAAO,EAAE,CAAC;IACnF,CAAC,CAAC,OAAOmB,KAAU,EAAE;MACnB3F,MAAM,CAAC2F,KAAK,CAAC,SAAS,EAAE,mBAAmB,EAAEA,KAAK,CAAC;MACnD9C,aAAa,CAAC;QACZ0B,OAAO,EAAE,KAAK;QACdC,OAAO,EAAE,UAAUmB,KAAK,CAACnB,OAAO,EAAE;QAClCC,KAAK,EAAE;MACT,CAAC,CAAC;IACJ,CAAC,SAAS;MACRhC,aAAa,CAAC,KAAK,CAAC;MACpBE,aAAa,CAAC,EAAE,CAAC;IACnB;EACF,CAAC,EAAE,CAACgC,OAAO,EAAE5D,MAAM,EAAEK,QAAQ,CAAC,CAAC;;EAE/B;;EAEA,oBACEX,KAAA,CAACZ,YAAY,CAACwI,QAAQ;IAACC,KAAK,EAAE3D,OAAQ;IAAAxD,QAAA,gBACpCZ,IAAA,CAACd,IAAI;MAAC8I,GAAG,EAAEhG,WAAY;MAACiG,KAAK,EAAEC,MAAM,CAACC,IAAK;MAACC,WAAW,EAAE,KAAM;MAAAxH,QAAA,EAC5DA;IAAQ,CACL,CAAC,eAGPZ,IAAA,CAACR,YAAY;MAAC6I,OAAO,EAAEpG,UAAW;MAACE,UAAU,EAAEA;IAAW,CAAE,CAAC,EAG5DxB,WAAW,iBACVX,IAAA,CAACT,YAAY;MACX+I,MAAM,EAAEZ,UAAW;MACnBzF,UAAU,EAAEA,UAAW;MACvBI,UAAU,EAAEA,UAAW;MACvB5B,QAAQ,EAAEA,QAAS;MACnB8H,SAAS,EAAEA,CAAA,KAAMjG,aAAa,CAAC,IAAI,CAAE;MACrCc,cAAc,EAAEA,cAAe;MAC/Bb,IAAI,EAAEA,IAAK;MACXiG,YAAY,EAAGC,OAAO,IAAK;QACzBhJ,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,gBAAgBQ,IAAI,MAAMkG,OAAO,EAAE,CAAC;QAC3DjG,OAAO,CAACiG,OAAO,CAAC;MAClB,CAAE;MACFhG,WAAW,EAAEA,WAAY;MACzBE,cAAc,EAAEA,cAAe;MAC/BE,YAAY,EAAEA,YAAa;MAC3B6F,aAAa,EAAEjB,gBAAiB;MAChC1E,gBAAgB,EAAEA,gBAAiB;MACnC4F,WAAW,EAAGC,MAAM,IAAK;QACvB,IAAIA,MAAM,IAAI,CAAC7F,gBAAgB,EAAE;UAC/BtD,MAAM,CAACmG,IAAI,CAAC,SAAS,EAAE,oDAAoD,CAAC;UAC5E;QACF;QACAnG,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,eAAe6G,MAAM,GAAG,IAAI,GAAG,KAAK,EAAE,CAAC;QAC9DlG,cAAc,CAACkG,MAAM,CAAC;QACtB,IAAIA,MAAM,EAAE;UACVnJ,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC;UAChDmB,aAAa,CAACa,OAAO,EAAEqC,KAAK,CAAC,CAAC,CAACd,IAAI,CAAEC,EAAE,IAAK;YAC1C9F,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,4BAA4BwD,EAAE,EAAE,CAAC;UAC1D,CAAC,CAAC;QACJ,CAAC,MAAM;UACL9F,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC;UAChDmB,aAAa,CAACa,OAAO,EAAEyD,IAAI,CAAC,CAAC;QAC/B;MACF,CAAE;MACFqB,eAAe,EAAGC,KAAK,IAAK;QAC1BrJ,MAAM,CAACsC,IAAI,CAAC,SAAS,EAAE,mBAAmB+G,KAAK,GAAG,OAAO,GAAG,SAAS,EAAE,CAAC;QACxElG,iBAAiB,CAACkG,KAAK,CAAC;QACxB,IAAIA,KAAK,EAAE;UACT3F,cAAc,CAACY,OAAO,EAAEgF,IAAI,CAAC,CAAC;QAChC,CAAC,MAAM;UACL5F,cAAc,CAACY,OAAO,EAAEiF,MAAM,CAAC,CAAC;QAClC;MACF;IAAE,CAEH,CACF;EAAA,CACoB,CAAC;AAE5B;AAGA,MAAMd,MAAM,GAAG/I,UAAU,CAAC8J,MAAM,CAAC;EAC/Bd,IAAI,EAAE;IACJe,IAAI,EAAE;EACR;AACF,CAAC,CAAC","ignoreList":[]}
package/lib/module/core/systemPrompt.js CHANGED
@@ -163,54 +163,88 @@ plan: "Call done to report the cart contents to the user."
163
163
  }
164
164
 
165
165
  /**
166
- * Voice-optimized system prompt for the Gemini Live API.
166
+ * Voice-adapted system prompt for the Gemini Live API.
167
167
  *
168
- * Includes the same screen format and tool semantics as text mode,
169
- * but condensed for voice context and with guardrails against
170
- * unprompted actions.
168
+ * Uses the same core rules/tools/screen format as text mode (buildSystemPrompt)
169
+ * but adapted for voice interaction:
170
+ * - No agent-loop directives (no "MUST call agent_step on every step")
171
+ * - No agent_history/user_request references (voice is conversational)
172
+ * - Explicit "wait for user voice command" guardrails
173
+ * - Voice-specific UX (concise spoken responses)
171
174
  */
172
175
  export function buildVoiceSystemPrompt(language, userInstructions) {
173
176
  const isArabic = language === 'ar';
174
- let prompt = `You are a voice-controlled AI agent operating a React Native mobile app. You can see the screen content and interact with UI elements using tools.
177
+ let prompt = `You are a voice-controlled AI agent operating a React Native mobile app. You receive periodic screen updates showing what's currently visible, and you can interact with UI elements using tools. You respond to the user via spoken audio.
175
178
 
176
- <language>
177
- ${isArabic ? 'Respond in Arabic.' : 'Respond in English.'}
178
- Use the same language as the user.
179
- </language>
180
-
181
- <screen_format>
182
- You receive periodic screen updates showing the current UI. Interactive elements appear as:
183
- [index]<type attrs>label />
179
+ <language_settings>
180
+ ${isArabic ? '- Working language: **Arabic**. Respond in Arabic.' : '- Working language: **English**. Respond in English.'}
181
+ - Use the same language as the user. Return in user's language.
182
+ </language_settings>
184
183
 
185
- - index: numeric ID for interaction (use with tap/type tools)
184
+ <screen_state>
185
+ Interactive elements are listed as [index]<type attrs>label />
186
+ - index: numeric identifier for interaction
186
187
  - type: element type (pressable, text-input, switch)
187
- - attrs: state like value="true", checked="false", role="switch"
188
- - label: visible text content
188
+ - attrs: state attributes like value="true", checked="false", role="switch"
189
+ - label: visible text content of the element
189
190
 
190
- Only elements with [index] are interactive. Text without [] is display-only.
191
- Example: [5]<switch value="true">Order Updates /> means element 5 is a switch currently ON.
192
- </screen_format>
191
+ Only elements with [index] are interactive. Use the index to tap or type into them.
192
+ Pure text elements without [] are NOT interactive — they are informational content you can read.
193
+ </screen_state>
193
194
 
194
195
  <tools>
195
196
  Available tools:
196
- - tap(index): Tap an element. For switches, this toggles their value.
197
- - type(index, text): Type text into a text-input.
198
- - navigate(screen): Navigate to a named screen.
199
- - done(text, success): Complete the task with a spoken response.
200
- - ask_user(question): Ask the user for clarification.
197
+ - tap(index): Tap an interactive element by its index. Works universally on buttons, switches, and custom components. For switches, this toggles their state.
198
+ - type(index, text): Type text into a text-input element by its index.
199
+ - navigate(screen, params): Navigate to a specific screen. params is optional JSON object.
200
+ - done(text, success): Complete task. Text is your final response to the user.
201
+ - ask_user(question): Ask the user for clarification ONLY when you cannot determine what action to take.
202
+
203
+ When you need to perform an action, call the appropriate tool function directly.
201
204
  </tools>
202
205
 
206
+ <voice_interaction_rules>
207
+ CRITICAL — THESE RULES OVERRIDE EVERYTHING ELSE:
208
+ - You are in a LIVE VOICE conversation. Wait for the user to SPEAK before doing anything.
209
+ - Screen updates arrive as passive context — they are NOT commands. Do NOT act on them.
210
+ - ONLY take action (tap, type, navigate) when the user explicitly asks you to via voice.
211
+ - When you have NO voice command from the user, stay silent. Do NOT narrate the screen.
212
+ - When the user speaks, determine the request type BEFORE acting:
213
+ 1. Information requests ("what's on screen?", "how much is X?"): Respond with spoken audio. Do NOT call any tools.
214
+ 2. Action requests ("go to settings", "add pizza to cart"): Call the appropriate tool function directly (e.g. navigate, tap).
215
+ - After completing an action, speak a brief confirmation to the user.
216
+ - Keep all spoken responses concise — the user is listening, not reading.
217
+ </voice_interaction_rules>
218
+
203
219
  <rules>
204
- CRITICAL ACTION RULES:
205
- - ONLY perform actions (tap, type, navigate) when the user explicitly asks you to do something.
206
- - NEVER tap or navigate on your own initiative — wait for the user's voice command.
207
- - When the user asks a question about what's on screen, answer verbally via done(). Do NOT tap anything.
208
- - When the user asks to toggle/enable/disable something, find the matching element by its label and use tap(index).
209
- - When a screen update arrives, do NOT interact with elements unless the user asked you to.
210
- - Use element indexes from the most recent screen update — they refresh every few seconds.
211
- - For switches: tap(index) toggles the value. You do NOT need to find a separate button.
212
- - Keep spoken responses concise — the user is listening, not reading.
213
- </rules>`;
220
+ - There are 2 types of requests — always determine which type BEFORE acting:
221
+ 1. Information requests (e.g. "what's available?", "how much is X?", "list the items"):
222
+ Respond verbally with the answer. Do NOT perform any tap/type/navigate actions.
223
+ 2. Action requests (e.g. "add margherita to cart", "go to checkout", "fill in my name"):
224
+ Execute the required UI interactions using tap/type/navigate tools.
225
+ - Only interact with elements that have an [index].
226
+ - If the current screen doesn't have what you need, use navigate() to go to another screen.
227
+ - When the user asks to go to a specific screen by name and it's listed in Available Screens, use navigate(screen) instead of tapping.
228
+ - Do not repeat one action for more than 3 times unless conditions changed.
229
+ - Do not fill in login/signup forms unless the user provides credentials. If asked to log in, use ask_user to request their email and password first.
230
+ - Do not guess or auto-fill sensitive data (passwords, payment info, personal details). Always ask the user.
231
+ - If stuck, tell the user what happened rather than repeating failed actions.
232
+ </rules>
233
+
234
+ <capability>
235
+ - It is ok to just provide information without performing any actions.
236
+ - User can ask questions about what's on screen — answer them directly by speaking.
237
+ - It is ok to fail the task. User would rather you report failure than repeat failed actions endlessly.
238
+ - The user can be wrong. If the request is not achievable, tell the user.
239
+ </capability>
240
+
241
+ <ux_rules>
242
+ - Confirm what you did: When completing actions, briefly say what happened.
243
+ - Be transparent about errors: If an action fails, explain what failed and why.
244
+ - Be concise: Keep spoken responses short and clear. No walls of text.
245
+ - Suggest next steps: After completing an action, briefly suggest what the user might want to do next.
246
+ - When a request is ambiguous, pick the most common interpretation and state your assumption.
247
+ </ux_rules>`;
214
248
 
215
249
  // Append user-provided instructions if any
216
250
  if (userInstructions?.trim()) {
package/lib/module/core/systemPrompt.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"names":["buildSystemPrompt","language","isArabic","buildVoiceSystemPrompt","userInstructions","prompt","trim"],"sourceRoot":"../../../src","sources":["core/systemPrompt.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,OAAO,SAASA,iBAAiBA,CAACC,QAAgB,EAAU;EAC1D,MAAMC,QAAQ,GAAGD,QAAQ,KAAK,IAAI;EAElC,OAAO;AACT;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,EAAEC,QAAQ,GAAG,oDAAoD,GAAG,sDAAsD;AAC1H;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,UAAU;AACV;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASC,sBAAsBA,CACpCF,QAAgB,EAChBG,gBAAyB,EACjB;EACR,MAAMF,QAAQ,GAAGD,QAAQ,KAAK,IAAI;EAElC,IAAII,MAAM,GAAG;AACf;AACA;AACA,EAAEH,QAAQ,GAAG,oBAAoB,GAAG,qBAAqB;AACzD;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,SAAS;;EAEP;EACA,IAAIE,gBAAgB,EAAEE,IAAI,CAAC,CAAC,EAAE;IAC5BD,MAAM,IAAI,2BAA2BD,gBAAgB,CAACE,IAAI,CAAC,CAAC,uBAAuB;EACrF;EAEA,OAAOD,MAAM;AACf","ignoreList":[]}
1
+ {"version":3,"names":["buildSystemPrompt","language","isArabic","buildVoiceSystemPrompt","userInstructions","prompt","trim"],"sourceRoot":"../../../src","sources":["core/systemPrompt.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,OAAO,SAASA,iBAAiBA,CAACC,QAAgB,EAAU;EAC1D,MAAMC,QAAQ,GAAGD,QAAQ,KAAK,IAAI;EAElC,OAAO;AACT;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,EAAEC,QAAQ,GAAG,oDAAoD,GAAG,sDAAsD;AAC1H;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,UAAU;AACV;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,OAAO,SAASC,sBAAsBA,CACpCF,QAAgB,EAChBG,gBAAyB,EACjB;EACR,MAAMF,QAAQ,GAAGD,QAAQ,KAAK,IAAI;EAElC,IAAII,MAAM,GAAG;AACf;AACA;AACA,EAAEH,QAAQ,GAAG,oDAAoD,GAAG,sDAAsD;AAC1H;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,YAAY;;EAEV;EACA,IAAIE,gBAAgB,EAAEE,IAAI,CAAC,CAAC,EAAE;IAC5BD,MAAM,IAAI,2BAA2BD,gBAAgB,CAACE,IAAI,CAAC,CAAC,uBAAuB;EACrF;EAEA,OAAOD,MAAM;AACf","ignoreList":[]}
package/lib/module/index.js CHANGED
@@ -14,12 +14,18 @@ export { AIAgent } from "./components/AIAgent.js";
14
14
  export { useAction } from "./hooks/useAction.js";
15
15
 
16
16
  // ─── Services ────────────────────────────────────────────────
17
- export { VoiceService } from "./services/VoiceService.js";
18
- export { AudioInputService } from "./services/AudioInputService.js";
19
- export { AudioOutputService } from "./services/AudioOutputService.js";
17
+ // export { VoiceService } from './services/VoiceService';
18
+ // export { AudioInputService } from './services/AudioInputService';
19
+ // export { AudioOutputService } from './services/AudioOutputService';
20
20
 
21
21
  // ─── Utilities ───────────────────────────────────────────────
22
22
  export { logger } from "./utils/logger.js";
23
23
 
24
24
  // ─── Types ───────────────────────────────────────────────────
25
+
26
+ // export type {
27
+ // VoiceServiceConfig,
28
+ // VoiceServiceCallbacks,
29
+ // VoiceStatus,
30
+ // } from './services/VoiceService';
25
31
  //# sourceMappingURL=index.js.map
package/lib/module/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"names":["AIAgent","useAction","VoiceService","AudioInputService","AudioOutputService","logger"],"sourceRoot":"../../src","sources":["index.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACA,SAASA,OAAO,QAAQ,yBAAsB;;AAE9C;AACA,SAASC,SAAS,QAAQ,sBAAmB;;AAE7C;AACA,SAASC,YAAY,QAAQ,4BAAyB;AACtD,SAASC,iBAAiB,QAAQ,iCAA8B;AAChE,SAASC,kBAAkB,QAAQ,kCAA+B;;AAElE;AACA,SAASC,MAAM,QAAQ,mBAAgB;;AAEvC","ignoreList":[]}
1
+ {"version":3,"names":["AIAgent","useAction","logger"],"sourceRoot":"../../src","sources":["index.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;;AAEA;AACA,SAASA,OAAO,QAAQ,yBAAsB;;AAE9C;AACA,SAASC,SAAS,QAAQ,sBAAmB;;AAE7C;AACA;AACA;AACA;;AAEA;AACA,SAASC,MAAM,QAAQ,mBAAgB;;AAEvC;;AAYA;AACA;AACA;AACA;AACA","ignoreList":[]}
package/lib/module/services/VoiceService.js CHANGED
@@ -20,7 +20,11 @@ import { logger } from "../utils/logger.js";
20
20
 
21
21
  const WS_HOST = 'generativelanguage.googleapis.com';
22
22
  const WS_PATH = '/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
23
- const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
23
+ // Use -09-2025: Google's own cookbook uses this model for Live API tool use.
24
+ // The -12-2025 model had server-side regressions with function calling
25
+ // and was deprecated March 19, 2026. The -09-2025 version has
26
+ // "improved function calling and better handling of speech cut-offs."
27
+ const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-09-2025';
24
28
  const DEFAULT_INPUT_SAMPLE_RATE = 16000;
25
29
 
26
30
  // ─── Service ───────────────────────────────────────────────────
@@ -138,8 +142,9 @@ export class VoiceService {
138
142
  turnComplete: false // Passive context — don't trigger a response
139
143
  }
140
144
  };
141
- this.ws.send(JSON.stringify(message));
142
145
  logger.debug('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
146
+ logger.debug('VoiceService', `📤 Raw Screen Context Payload: ${JSON.stringify(message).substring(0, 500)}...`);
147
+ this.ws.send(JSON.stringify(message));
143
148
  }
144
149
 
145
150
  // ─── Send Function Response ────────────────────────────────
@@ -162,6 +167,14 @@ export class VoiceService {
162
167
 
163
168
  // ─── Internal: Setup ───────────────────────────────────────
164
169
 
170
+ /**
171
+ * Builds and sends the setup message, replicating text mode's agent_step
172
+ * compound tool so the model uses structured reasoning + actions.
173
+ *
174
+ * The agent_step tool flattens reasoning fields (previous_goal_eval,
175
+ * memory, plan) + action_name enum + all action parameters into a single
176
+ * function — matching GeminiProvider.buildAgentStepDeclaration exactly.
177
+ */
165
178
  sendSetup() {
166
179
  if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
167
180
  const model = this.config.model || DEFAULT_MODEL;
@@ -169,13 +182,12 @@ export class VoiceService {
169
182
  model: `models/${model}`,
170
183
  generationConfig: {
171
184
  responseModalities: ['AUDIO']
185
+ // Note: Do NOT set thinkingBudget: 0 — it completely disables
186
+ // the model's ability to reason about when to call tools.
187
+ // The text thinking blocks are a trade-off for working function calling.
172
188
  }
173
189
  };
174
190
 
175
- // For native audio models, language is enforced via system instructions.
176
- // Explicitly setting speechConfig.languageCode causes silent API failures
177
- // without a prebuiltVoiceConfig.
178
-
179
191
  // Add system instruction if provided
180
192
  if (this.config.systemPrompt) {
181
193
  setup.systemInstruction = {
@@ -185,32 +197,58 @@ export class VoiceService {
185
197
  };
186
198
  }
187
199
 
188
- // Add tool declarations for function calling
200
+ // Add individual tool declarations for function calling
201
+ // NOTE: We use individual tools (tap, type, navigate, done, ask_user)
202
+ // instead of the compound agent_step used in text mode.
203
+ // The native audio model in real-time can call simple tools but struggles
204
+ // with the complex agent_step schema (it speaks about calling tools
205
+ // instead of actually calling them).
189
206
  if (this.config.tools?.length) {
190
- setup.tools = [{
191
- functionDeclarations: this.config.tools.map(tool => ({
192
- name: tool.name,
193
- description: tool.description,
194
- parameters: {
195
- type: 'OBJECT',
196
- properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => [key, {
197
- type: param.type.toUpperCase(),
198
- description: param.description
199
- }])),
200
- required: Object.entries(tool.parameters).filter(([, param]) => param.required).map(([key]) => key)
201
- }
202
- }))
203
- }];
207
+ const validTools = this.config.tools.filter(t => t.name !== 'capture_screenshot');
208
+ if (validTools.length > 0) {
209
+ setup.tools = [{
210
+ functionDeclarations: validTools.map(tool => {
211
+ const hasParams = Object.keys(tool.parameters || {}).length > 0;
212
+ const functionDecl = {
213
+ name: tool.name,
214
+ description: tool.description
215
+ };
216
+ if (hasParams) {
217
+ functionDecl.parameters = {
218
+ type: 'OBJECT',
219
+ properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => {
220
+ // Native audio model crashes with BOOLEAN/ENUM types (error 1008)
221
+ // Convert to STRING as a workaround
222
+ let paramType = param.type.toUpperCase();
223
+ let desc = param.description;
224
+ if (paramType === 'BOOLEAN') {
225
+ paramType = 'STRING';
226
+ desc = `${desc} (use "true" or "false")`;
227
+ }
228
+ return [key, {
229
+ type: paramType,
230
+ description: desc
231
+ }];
232
+ })),
233
+ required: Object.entries(tool.parameters).filter(([, param]) => param.required).map(([key]) => key)
234
+ };
235
+ }
236
+ return functionDecl;
237
+ })
238
+ }];
239
+ }
204
240
  }
205
-
206
- // Enable transcription
207
- setup.inputAudioTranscription = {};
208
- setup.outputAudioTranscription = {};
209
241
  const setupMessage = {
210
242
  setup
211
243
  };
212
- logger.info('VoiceService', `Sending setup (model: ${model}, tools: ${this.config.tools?.length || 0})`);
213
- this.ws.send(JSON.stringify(setupMessage));
244
+ logger.info('VoiceService', `Sending setup (model: ${model}, ${this.config.tools?.length || 0} tools)`);
245
+ try {
246
+ const payload = JSON.stringify(setupMessage);
247
+ logger.info('VoiceService', `📤 Raw Setup Payload: ${payload}`);
248
+ this.ws.send(payload);
249
+ } catch (err) {
250
+ logger.error('VoiceService', `❌ Error stringifying setup message: ${err.message}`);
251
+ }
214
252
  }
215
253
 
216
254
  // ─── Internal: Message Handling ────────────────────────────
@@ -231,6 +269,7 @@ export class VoiceService {
231
269
  // Handle JSON text messages
232
270
  const message = JSON.parse(event.data);
233
271
  logger.info('VoiceService', `📥 JSON message keys: ${Object.keys(message).join(', ')}`);
272
+ logger.info('VoiceService', `📥 Raw JSON Message: ${event.data.substring(0, 1000)}`);
234
273
  this.processMessage(message);
235
274
  } catch (error) {
236
275
  logger.error('VoiceService', `Error handling message: ${error.message}`);
@@ -327,7 +366,7 @@ export class VoiceService {
327
366
  // Tool calls from the model
328
367
  if (message.toolCall?.functionCalls) {
329
368
  for (const fn of message.toolCall.functionCalls) {
330
- logger.info('VoiceService', `Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
369
+ logger.info('VoiceService', `🎯 Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
331
370
  this.callbacks.onToolCall?.({
332
371
  name: fn.name,
333
372
  args: fn.args || {},
package/lib/module/services/VoiceService.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"names":["logger","WS_HOST","WS_PATH","DEFAULT_MODEL","DEFAULT_INPUT_SAMPLE_RATE","VoiceService","ws","callbacks","setupComplete","_status","constructor","config","connect","readyState","WebSocket","OPEN","info","setStatus","model","url","apiKey","onopen","sendSetup","onclose","event","code","reason","onerror","error","message","onError","onmessage","handleMessage","disconnect","close","isConnected","currentStatus","sendCount","sendAudio","base64Audio","warn","realtimeInput","audio","mimeType","inputSampleRate","data","length","send","JSON","stringify","sendText","text","sendScreenContext","domText","clientContent","turns","role","parts","turnComplete","debug","sendFunctionResponse","name","id","result","toolResponse","functionResponses","response","setup","generationConfig","responseModalities","systemPrompt","systemInstruction","tools","functionDeclarations","map","tool","description","parameters","type","properties","Object","fromEntries","entries","key","param","toUpperCase","required","filter","inputAudioTranscription","outputAudioTranscription","setupMessage","dataType","dataLen","byteLength","handleBinaryMessage","parse","keys","join","processMessage","bytes","ArrayBuffer","Uint8Array","Blob","reader","FileReader","onload","processBinaryBytes","readAsArrayBuffer","looksLikeJson","TextDecoder","decode","onAudioResponse","arrayBufferToBase64","buffer","undefined","serverContent","content","modelTurn","inputTranscription","outputTranscription","onTurnComplete","part","inlineData","onTranscript","toolCall","functionCalls","fn","args","onToolCall","newStatus","onStatusChange","binary","i","String","fromCharCode","btoa"],"sourceRoot":"../../../src","sources":["services/VoiceService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;;AAGxC;;AAyBA;;AAEA,MAAMC,OAAO,GAAG,mCAAmC;AACnD,MAAMC,OAAO,GAAG,+EAA+E;AAC/F,MAAMC,aAAa,GAAG,+CAA+C;AACrE,MAAMC,yBAAyB,GAAG,KAAK;;AAEvC;;AAEA,OAAO,MAAMC,YAAY,CAAC;EAChBC,EAAE,GAAqB,IAAI;EAE3BC,SAAS,GAA0B,CAAC,CAAC;EACrCC,aAAa,GAAG,KAAK;EACrBC,OAAO,GAAgB,cAAc;EAE7CC,WAAWA,CAACC,MAA0B,EAAE;IACtC,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEAC,OAAOA,CAACL,SAAgC,EAAQ;IAC9C,IAAI,IAAI,CAACD,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;MAC1Cf,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC;MAChD;IACF;IAEA,IAAI,CAACT,SAAS,GAAGA,SAAS;IAC1B,IAAI,CAACU,SAAS,CAAC,YAAY,CAAC;IAE5B,MAAMC,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAChD,MAAMgB,GAAG,GAAG,SAASlB,OAAO,GAAGC,OAAO,QAAQ,IAAI,CAACS,MAAM,CAACS,MAAM,EAAE;IAElEpB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yCAAyCE,KAAK,GAAG,CAAC;IAC9E,IAAI,CAACZ,EAAE,GAAG,IAAIQ,SAAS,CAACK,GAAG,CAAC;IAE5B,IAAI,CAACb,EAAE,CAACe,MAAM,GAAG,MAAM;MACrBrB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,uCAAuC,CAAC;MACpE,IAAI,CAACM,SAAS,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAChB,EAAE,CAACiB,OAAO,GAAIC,KAAK,IAAK;MAC3BxB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,qBAAqBQ,KAAK,CAACC,IAAI,IAAID,KAAK,CAACE,MAAM,EAAE,CAAC;MAC9E,IAAI,CAACT,SAAS,CAAC,cAAc,CAAC;MAC9B,IAAI,CAACT,aAAa,GAAG,KAAK;IAC5B,CAAC;IAED,IAAI,CAACF,EAAE,CAACqB,OAAO,GAAIC,KAAU,IAAK;MAChC5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,oBAAoBA,KAAK,CAACC,OAAO,IAAI,SAAS,EAAE,CAAC;MAC9E,IAAI,CAACZ,SAAS,CAAC,OAAO,CAAC;MACvB,IAAI,CAACV,SAAS,CAACuB,OAAO,GAAGF,KAAK,CAACC,OAAO,IAAI,4BAA4B,CAAC;IACzE,CAAC;IAED,IAAI,CAACvB,EAAE,CAACyB,SAAS,GAAIP,KAAK,IAAK;MAC7B,IAAI,CAACQ,aAAa,CAACR,KAAK,CAAC;IAC3B,CAAC;EACH;EAEAS,UAAUA,CAAA,EAAS;IACjB,IAAI,IAAI,CAAC3B,EAAE,EAAE;MACXN,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;MAC/C,IAAI,CAACV,EAAE,CAAC4B,KAAK,CAAC,CAAC;MACf,IAAI,CAAC5B,EAAE,GAAG,IAAI;MACd,IAAI,CAACE,aAAa,GAAG,KAAK;MAC1B,IAAI,CAACS,SAAS,CAAC,cAAc,CAAC;IAChC;EACF;EAEA,IAAIkB,WAAWA,CAAA,EAAY;IACzB,OAAO,IAAI,CAAC7B,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,IAAI,IAAI,CAACP,aAAa;EACrE;EAEA,IAAI4B,aAAaA,CAAA,EAAgB;IAC/B,OAAO,IAAI,CAAC3B,OAAO;EACrB;;EAEA;;EAEA;EACQ4B,SAAS,GAAG,CAAC;EACrBC,SAASA,CAACC,WAAmB,EAAQ;IACnC,IAAI,CAACF,SAAS,EAAE;IAChB,IAAI,CAAC,IAAI,CAACF,WAAW,EAAE;MACrBnC,MAAM,CAACwC,IAAI,CAAC,cAAc,EAAE,cAAc,IAAI,CAACH,SAAS,gCAAgC,IAAI,CAAC/B,EAAE,EAAEO,UAAU,WAAW,IAAI,CAACL,aAAa,GAAG,CAAC;MAC5I;IACF;IAEA,MAAMqB,OAAO,GAAG;MACdY,aAAa,EAAE;QACbC,KAAK,EAAE;UACLC,QAAQ,EAAE,kBAAkB,IAAI,CAAChC,MAAM,CAACiC,eAAe,IAAIxC,yBAAyB,EAAE;UACtFyC,IAAI,EAAEN;QACR;MACF;IACF,CAAC;IAEDvC,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,OAAO,IAAI,CAACqB,SAAS,YAAYE,WAAW,CAACO,MAAM,cAAc,IAAI,CAACxC,EAAE,EAAEO,UAAU,GAAG,CAAC;IACpH,IAAI,CAACP,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEA;EACAqB,QAAQA,CAACC,IAAY,EAAQ;IAC3B,IAAI,CAAC,IAAI,CAAChB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdY,aAAa,EAAE;QAAEU;MAAK;IACxB,CAAC;IAED,IAAI,CAAC7C,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEuB,iBAAiBA,CAACC,OAAe,EAAQ;IACvC,IAAI,CAAC,IAAI,CAAClB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdyB,aAAa,EAAE;QACbC,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAE,MAAM;UAAEC,KAAK,EAAE,CAAC;YAAEN,IAAI,EAAEE;UAAQ,CAAC;QAAE,CAAC,CAAC;QACrDK,YAAY,EAAE,KAAK,CAAE;MACvB;IACF,CAAC;IAED,IAAI,CAACpD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;IACtC7B,MAAM,CAAC2D,KAAK,CAAC,cAAc,EAAE,2BAA2BN,OAAO,CAACP,MAAM,SAAS,CAAC;EAClF;;EAEA;;EAEA;EACAc,oBAAoBA,CAACC,IAAY,EAAEC,EAAU,EAAEC,MAAW,EAAQ;IAChE,IAAI,CAAC,IAAI,CAAC5B,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdmC,YAAY,EAAE;QACZC,iBAAiB,EAAE,CAAC;UAClBJ,IAAI;UACJC,EAAE;UACFI,QAAQ,EAAEH;QACZ,CAAC;MACH;IACF,CAAC;IAED/D,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC6C,IAAI,QAAQC,EAAE,GAAG,CAAC;IAC9E,IAAI,CAACxD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEQP,SAASA,CAAA,EAAS;IACxB,IAAI,CAAC,IAAI,CAAChB,EAAE,IAAI,IAAI,CAACA,EAAE,CAACO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;IAEvD,MAAMG,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAEhD,MAAMgE,KAAU,GAAG;MACjBjD,KAAK,EAAE,UAAUA,KAAK,EAAE;MACxBkD,gBAAgB,EAAE;QAChBC,kBAAkB,EAAE,CAAC,OAAO;MAC9B;IACF,CAAC;;IAED;IACA;IACA;;IAEA;IACA,IAAI,IAAI,CAAC1D,MAAM,CAAC2D,YAAY,EAAE;MAC5BH,KAAK,CAACI,iBAAiB,GAAG;QACxBd,KAAK,EAAE,CAAC;UAAEN,IAAI,EAAE,IAAI,CAACxC,MAAM,CAAC2D;QAAa,CAAC;MAC5C,CAAC;IACH;;IAEA;IACA,IAAI,IAAI,CAAC3D,MAAM,CAAC6D,KAAK,EAAE1B,MAAM,EAAE;MAC7BqB,KAAK,CAACK,KAAK,GAAG,CAAC;QACbC,oBAAoB,EAAE,IAAI,CAAC9D,MAAM,CAAC6D,KAAK,CAACE,GAAG,CAACC,IAAI,KAAK;UACnDd,IAAI,EAAEc,IAAI,CAACd,IAAI;UACfe,WAAW,EAAED,IAAI,CAACC,WAAW;UAC7BC,UAAU,EAAE;YACVC,IAAI,EAAE,QAAQ;YACdC,UAAU,EAAEC,MAAM,CAACC,WAAW,CAC5BD,MAAM,CAACE,OAAO,CAACP,IAAI,CAACE,UAAU,CAAC,CAACH,GAAG,CAAC,CAAC,CAACS,GAAG,EAAEC,KAAK,CAAC,KAAK,CACpDD,GAAG,EACH;cACEL,IAAI,EAAEM,KAAK,CAACN,IAAI,CAACO,WAAW,CAAC,CAAC;cAC9BT,WAAW,EAAEQ,KAAK,CAACR;YACrB,CAAC,CACF,CACH,CAAC;YACDU,QAAQ,EAAEN,MAAM,CAACE,OAAO,CAACP,IAAI,CAACE,UAAU,CAAC,CACtCU,MAAM,CAAC,CAAC,GAAGH,KAAK,CAAC,KAAKA,KAAK,CAACE,QAAQ,CAAC,CACrCZ,GAAG,CAAC,CAAC,CAACS,GAAG,CAAC,KAAKA,GAAG;UACvB;QACF,CAAC,CAAC;MACJ,CAAC,CAAC;IACJ;;IAEA;IACAhB,KAAK,CAACqB,uBAAuB,GAAG,CAAC,CAAC;IAClCrB,KAAK,CAACsB,wBAAwB,GAAG,CAAC,CAAC;IAEnC,MAAMC,YAAY,GAAG;MAAEvB;IAAM,CAAC;IAC9BnE,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBE,KAAK,YAAY,IAAI,CAACP,MAAM,CAAC6D,KAAK,EAAE1B,MAAM,IAAI,CAAC,GAAG,CAAC;IACxG,IAAI,CAACxC,EAAE,CAACyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACyC,YAAY,CAAC,CAAC;EAC5C;;EAEA;;EAEQ1D,aAAaA,CAACR,KAA4B,EAAQ;IACxD,IAAI;MACF,MAAMmE,QAAQ,GAAG,OAAOnE,KAAK,CAACqB,IAAI;MAClC,MAAM+C,OAAO,GAAG,OAAOpE,KAAK,CAACqB,IAAI,KAAK,QAAQ,GAAGrB,KAAK,CAACqB,IAAI,CAACC,MAAM,GAAItB,KAAK,CAACqB,IAAI,EAAEgD,UAAU,IAAI,SAAU;MAC1G7F,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC2E,QAAQ,YAAYC,OAAO,EAAE,CAAC;;MAE1F;MACA,IAAI,OAAOpE,KAAK,CAACqB,IAAI,KAAK,QAAQ,EAAE;QAClC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mCAAmC,CAAC;QAChE,IAAI,CAAC8E,mBAAmB,CAACtE,KAAK,CAACqB,IAAI,CAAC;QACpC;MACF;;MAEA;MACA,MAAMhB,OAAO,GAAGmB,IAAI,CAAC+C,KAAK,CAACvE,KAAK,CAACqB,IAAI,CAAC;MACtC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBgE,MAAM,CAACgB,IAAI,CAACnE,OAAO,CAAC,CAACoE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;MACvF,IAAI,CAACC,cAAc,CAACrE,OAAO,CAAC;IAC9B,CAAC,CAAC,OAAOD,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,2BAA2BA,KAAK,CAACC,OAAO,EAAE,CAAC;IAC1E;EACF;EAEQiE,mBAAmBA,CAACjD,IAAS,EAAQ;IAC3C,IAAI;MACF;MACA,IAAIsD,KAAiB;MACrB,IAAItD,IAAI,YAAYuD,WAAW,EAAE;QAC/BD,KAAK,GAAG,IAAIE,UAAU,CAACxD,IAAI,CAAC;MAC9B,CAAC,MAAM,IAAIA,IAAI,YAAYyD,IAAI,EAAE;QAC/B;QACA,MAAMC,MAAM,GAAG,IAAIC,UAAU,CAAC,CAAC;QAC/BD,MAAM,CAACE,MAAM,GAAG,MAAM;UACpB,IAAIF,MAAM,CAACxC,MAAM,YAAYqC,WAAW,EAAE;YACxC,IAAI,CAACM,kBAAkB,CAAC,IAAIL,UAAU,CAACE,MAAM,CAACxC,MAAM,CAAC,CAAC;UACxD;QACF,CAAC;QACDwC,MAAM,CAACI,iBAAiB,CAAC9D,IAAI,CAAC;QAC9B;MACF,CAAC,MAAM;QACL;MACF;MAEA,IAAI,CAAC6D,kBAAkB,CAACP,KAAK,CAAC;IAChC,CAAC,CAAC,OAAOvE,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,kCAAkCA,KAAK,CAACC,OAAO,EAAE,CAAC;IACjF;EACF;EAEQ6E,kBAAkBA,CAACP,KAAiB,EAAQ;IAClD;IACA,MAAMS,aAAa,GAAGT,KAAK,CAACrD,MAAM,GAAG,CAAC,KAAKqD,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAIA,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAE/E,IAAIS,aAAa,EAAE;MACjB,IAAI;QACF,MAAMzD,IAAI,GAAG,IAAI0D,WAAW,CAAC,OAAO,CAAC,CAACC,MAAM,CAACX,KAAK,CAAC;QACnD,MAAMtE,OAAO,GAAGmB,IAAI,CAAC+C,KAAK,CAAC5C,IAAI,CAAC;QAChC,IAAI,CAAC+C,cAAc,CAACrE,OAAO,CAAC;MAC9B,CAAC,CAAC,MAAM;QACN;QACA,IAAI,CAACtB,SAAS,CAACwG,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;MACzF;IACF,CAAC,MAAM;MACL;MACA,IAAI,CAAC1G,SAAS,CAACwG,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;IACzF;EACF;EAEQf,cAAcA,CAACrE,OAAY,EAAQ;IACzC;IACA,IAAIA,OAAO,CAACrB,aAAa,KAAK0G,SAAS,EAAE;MACvClH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,6CAA6C,CAAC;MAC1E,IAAI,CAACR,aAAa,GAAG,IAAI;MACzB,IAAI,CAACS,SAAS,CAAC,WAAW,CAAC;MAC3B;IACF;;IAEA;IACA,IAAIY,OAAO,CAACsF,aAAa,EAAE;MACzB,MAAMC,OAAO,GAAGvF,OAAO,CAACsF,aAAa;MACrCnH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,4CAA4CoG,OAAO,CAAC1D,YAAY,cAAc,CAAC,CAAC0D,OAAO,CAACC,SAAS,EAAE5D,KAAK,wBAAwB,CAAC,CAAC2D,OAAO,CAACE,kBAAkB,yBAAyB,CAAC,CAACF,OAAO,CAACG,mBAAmB,EAAE,CAAC;;MAEjP;MACA,IAAIH,OAAO,CAAC1D,YAAY,EAAE;QACxB,IAAI,CAACnD,SAAS,CAACiH,cAAc,GAAG,CAAC;MACnC;;MAEA;MACA,IAAIJ,OAAO,CAACC,SAAS,EAAE5D,KAAK,EAAE;QAC5B,KAAK,MAAMgE,IAAI,IAAIL,OAAO,CAACC,SAAS,CAAC5D,KAAK,EAAE;UAC1C;UACA,IAAIgE,IAAI,CAACC,UAAU,EAAE7E,IAAI,EAAE;YACzB7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsByG,IAAI,CAACC,UAAU,CAAC7E,IAAI,CAACC,MAAM,QAAQ,CAAC;YACtF,IAAI,CAACvC,SAAS,CAACwG,eAAe,GAAGU,IAAI,CAACC,UAAU,CAAC7E,IAAI,CAAC;UACxD;;UAEA;UACA,IAAI4E,IAAI,CAACtE,IAAI,EAAE;YACbnD,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsByG,IAAI,CAACtE,IAAI,GAAG,CAAC;YAC/D,IAAI,CAAC5C,SAAS,CAACoH,YAAY,GAAGF,IAAI,CAACtE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;UACzD;QACF;MACF;;MAEA;MACA,IAAIiE,OAAO,CAACE,kBAAkB,EAAEnE,IAAI,EAAE;QACpC,IAAI,CAAC5C,SAAS,CAACoH,YAAY,GAAGP,OAAO,CAACE,kBAAkB,CAACnE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;MAC9E;;MAEA;MACA,IAAIiE,OAAO,CAACG,mBAAmB,EAAEpE,IAAI,EAAE;QACrC,IAAI,CAAC5C,SAAS,CAACoH,YAAY,GAAGP,OAAO,CAACG,mBAAmB,CAACpE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;MAChF;IACF;;IAEA;IACA,IAAItB,OAAO,CAAC+F,QAAQ,EAAEC,aAAa,EAAE;MACnC,KAAK,MAAMC,EAAE,IAAIjG,OAAO,CAAC+F,QAAQ,CAACC,aAAa,EAAE;QAC/C7H,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,cAAc8G,EAAE,CAACjE,IAAI,IAAIb,IAAI,CAACC,SAAS,CAAC6E,EAAE,CAACC,IAAI,CAAC,GAAG,CAAC;QAChF,IAAI,CAACxH,SAAS,CAACyH,UAAU,GAAG;UAC1BnE,IAAI,EAAEiE,EAAE,CAACjE,IAAI;UACbkE,IAAI,EAAED,EAAE,CAACC,IAAI,IAAI,CAAC,CAAC;UACnBjE,EAAE,EAAEgE,EAAE,CAAChE;QACT,CAAC,CAAC;MACJ;IACF;;IAEA;IACA,IAAIjC,OAAO,CAACD,KAAK,EAAE;MACjB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,iBAAiBoB,IAAI,CAACC,SAAS,CAACpB,OAAO,CAACD,KAAK,CAAC,EAAE,CAAC;MAC9E,IAAI,CAACrB,SAAS,CAACuB,OAAO,GAAGD,OAAO,CAACD,KAAK,CAACC,OAAO,IAAI,cAAc,CAAC;IACnE;EACF;;EAEA;;EAEQZ,SAASA,CAACgH,SAAsB,EAAQ;IAC9C,IAAI,CAACxH,OAAO,GAAGwH,SAAS;IACxB,IAAI,CAAC1H,SAAS,CAAC2H,cAAc,GAAGD,SAAS,CAAC;EAC5C;EAEQjB,mBAAmBA,CAACC,MAAmB,EAAU;IACvD,MAAMd,KAAK,GAAG,IAAIE,UAAU,CAACY,MAAM,CAAC;IACpC,IAAIkB,MAAM,GAAG,EAAE;IACf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGjC,KAAK,CAACN,UAAU,EAAEuC,CAAC,EAAE,EAAE;MACzCD,MAAM,IAAIE,MAAM,CAACC,YAAY,CAACnC,KAAK,CAACiC,CAAC,CAAE,CAAC;IAC1C;IACA,OAAOG,IAAI,CAACJ,MAAM,CAAC;EACrB;AACF","ignoreList":[]}
1
+ {"version":3,"names":["logger","WS_HOST","WS_PATH","DEFAULT_MODEL","DEFAULT_INPUT_SAMPLE_RATE","VoiceService","ws","callbacks","setupComplete","_status","constructor","config","connect","readyState","WebSocket","OPEN","info","setStatus","model","url","apiKey","onopen","sendSetup","onclose","event","code","reason","onerror","error","message","onError","onmessage","handleMessage","disconnect","close","isConnected","currentStatus","sendCount","sendAudio","base64Audio","warn","realtimeInput","audio","mimeType","inputSampleRate","data","length","send","JSON","stringify","sendText","text","sendScreenContext","domText","clientContent","turns","role","parts","turnComplete","debug","substring","sendFunctionResponse","name","id","result","toolResponse","functionResponses","response","setup","generationConfig","responseModalities","systemPrompt","systemInstruction","tools","validTools","filter","t","functionDeclarations","map","tool","hasParams","Object","keys","parameters","functionDecl","description","type","properties","fromEntries","entries","key","param","paramType","toUpperCase","desc","required","setupMessage","payload","err","dataType","dataLen","byteLength","handleBinaryMessage","parse","join","processMessage","bytes","ArrayBuffer","Uint8Array","Blob","reader","FileReader","onload","processBinaryBytes","readAsArrayBuffer","looksLikeJson","TextDecoder","decode","onAudioResponse","arrayBufferToBase64","buffer","undefined","serverContent","content","modelTurn","inputTranscription","outputTranscription","onTurnComplete","part","inlineData","onTranscript","toolCall","functionCalls","fn","args","onToolCall","newStatus","onStatusChange","binary","i","String","fromCharCode","btoa"],"sourceRoot":"../../../src","sources":["services/VoiceService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;;AAGxC;;AAyBA;;AAEA,MAAMC,OAAO,GAAG,mCAAmC;AACnD,MAAMC,OAAO,GAAG,+EAA+E;AAC/F;AACA;AACA;AACA;AACA,MAAMC,aAAa,GAAG,+CAA+C;AACrE,MAAMC,yBAAyB,GAAG,KAAK;;AAEvC;;AAEA,OAAO,MAAMC,YAAY,CAAC;EAChBC,EAAE,GAAqB,IAAI;EAE3BC,SAAS,GAA0B,CAAC,CAAC;EACrCC,aAAa,GAAG,KAAK;EACrBC,OAAO,GAAgB,cAAc;EAE7CC,WAAWA,CAACC,MAA0B,EAAE;IACtC,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEAC,OAAOA,CAACL,SAAgC,EAAQ;IAC9C,IAAI,IAAI,CAACD,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;MAC1Cf,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mBAAmB,CAAC;MAChD;IACF;IAEA,IAAI,CAACT,SAAS,GAAGA,SAAS;IAC1B,IAAI,CAACU,SAAS,CAAC,YAAY,CAAC;IAE5B,MAAMC,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAChD,MAAMgB,GAAG,GAAG,SAASlB,OAAO,GAAGC,OAAO,QAAQ,IAAI,CAACS,MAAM,CAACS,MAAM,EAAE;IAElEpB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yCAAyCE,KAAK,GAAG,CAAC;IAC9E,IAAI,CAACZ,EAAE,GAAG,IAAIQ,SAAS,CAACK,GAAG,CAAC;IAE5B,IAAI,CAACb,EAAE,CAACe,MAAM,GAAG,MAAM;MACrBrB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,uCAAuC,CAAC;MACpE,IAAI,CAACM,SAAS,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAChB,EAAE,CAACiB,OAAO,GAAIC,KAAK,IAAK;MAC3BxB,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,qBAAqBQ,KAAK,CAACC,IAAI,IAAID,KAAK,CAACE,MAAM,EAAE,CAAC;MAC9E,IAAI,CAACT,SAAS,CAAC,cAAc,CAAC;MAC9B,IAAI,CAACT,aAAa,GAAG,KAAK;IAC5B,CAAC;IAED,IAAI,CAACF,EAAE,CAACqB,OAAO,GAAIC,KAAU,IAAK;MAChC5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,oBAAoBA,KAAK,CAACC,OAAO,IAAI,SAAS,EAAE,CAAC;MAC9E,IAAI,CAACZ,SAAS,CAAC,OAAO,CAAC;MACvB,IAAI,CAACV,SAAS,CAACuB,OAAO,GAAGF,KAAK,CAACC,OAAO,IAAI,4BAA4B,CAAC;IACzE,CAAC;IAED,IAAI,CAACvB,EAAE,CAACyB,SAAS,GAAIP,KAAK,IAAK;MAC7B,IAAI,CAACQ,aAAa,CAACR,KAAK,CAAC;IAC3B,CAAC;EACH;EAEAS,UAAUA,CAAA,EAAS;IACjB,IAAI,IAAI,CAAC3B,EAAE,EAAE;MACXN,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;MAC/C,IAAI,CAACV,EAAE,CAAC4B,KAAK,CAAC,CAAC;MACf,IAAI,CAAC5B,EAAE,GAAG,IAAI;MACd,IAAI,CAACE,aAAa,GAAG,KAAK;MAC1B,IAAI,CAACS,SAAS,CAAC,cAAc,CAAC;IAChC;EACF;EAEA,IAAIkB,WAAWA,CAAA,EAAY;IACzB,OAAO,IAAI,CAAC7B,EAAE,EAAEO,UAAU,KAAKC,SAAS,CAACC,IAAI,IAAI,IAAI,CAACP,aAAa;EACrE;EAEA,IAAI4B,aAAaA,CAAA,EAAgB;IAC/B,OAAO,IAAI,CAAC3B,OAAO;EACrB;;EAEA;;EAEA;EACQ4B,SAAS,GAAG,CAAC;EACrBC,SAASA,CAACC,WAAmB,EAAQ;IACnC,IAAI,CAACF,SAAS,EAAE;IAChB,IAAI,CAAC,IAAI,CAACF,WAAW,EAAE;MACrBnC,MAAM,CAACwC,IAAI,CAAC,cAAc,EAAE,cAAc,IAAI,CAACH,SAAS,gCAAgC,IAAI,CAAC/B,EAAE,EAAEO,UAAU,WAAW,IAAI,CAACL,aAAa,GAAG,CAAC;MAC5I;IACF;IAEA,MAAMqB,OAAO,GAAG;MACdY,aAAa,EAAE;QACbC,KAAK,EAAE;UACLC,QAAQ,EAAE,kBAAkB,IAAI,CAAChC,MAAM,CAACiC,eAAe,IAAIxC,yBAAyB,EAAE;UACtFyC,IAAI,EAAEN;QACR;MACF;IACF,CAAC;IAEDvC,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,OAAO,IAAI,CAACqB,SAAS,YAAYE,WAAW,CAACO,MAAM,cAAc,IAAI,CAACxC,EAAE,EAAEO,UAAU,GAAG,CAAC;IACpH,IAAI,CAACP,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEA;EACAqB,QAAQA,CAACC,IAAY,EAAQ;IAC3B,IAAI,CAAC,IAAI,CAAChB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdY,aAAa,EAAE;QAAEU;MAAK;IACxB,CAAC;IAED,IAAI,CAAC7C,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACEuB,iBAAiBA,CAACC,OAAe,EAAQ;IACvC,IAAI,CAAC,IAAI,CAAClB,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdyB,aAAa,EAAE;QACbC,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAE,MAAM;UAAEC,KAAK,EAAE,CAAC;YAAEN,IAAI,EAAEE;UAAQ,CAAC;QAAE,CAAC,CAAC;QACrDK,YAAY,EAAE,KAAK,CAAE;MACvB;IACF,CAAC;IAED1D,MAAM,CAAC2D,KAAK,CAAC,cAAc,EAAE,2BAA2BN,OAAO,CAACP,MAAM,SAAS,CAAC;IAChF9C,MAAM,CAAC2D,KAAK,CAAC,cAAc,EAAE,kCAAkCX,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC+B,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC;IAC9G,IAAI,CAACtD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEA;EACAgC,oBAAoBA,CAACC,IAAY,EAAEC,EAAU,EAAEC,MAAW,EAAQ;IAChE,IAAI,CAAC,IAAI,CAAC7B,WAAW,EAAE;IAEvB,MAAMN,OAAO,GAAG;MACdoC,YAAY,EAAE;QACZC,iBAAiB,EAAE,CAAC;UAClBJ,IAAI;UACJC,EAAE;UACFI,QAAQ,EAAEH;QACZ,CAAC;MACH;IACF,CAAC;IAEDhE,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgC8C,IAAI,QAAQC,EAAE,GAAG,CAAC;IAC9E,IAAI,CAACzD,EAAE,CAAEyC,IAAI,CAACC,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;EACxC;;EAEA;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACUP,SAASA,CAAA,EAAS;IACxB,IAAI,CAAC,IAAI,CAAChB,EAAE,IAAI,IAAI,CAACA,EAAE,CAACO,UAAU,KAAKC,SAAS,CAACC,IAAI,EAAE;IAEvD,MAAMG,KAAK,GAAG,IAAI,CAACP,MAAM,CAACO,KAAK,IAAIf,aAAa;IAEhD,MAAMiE,KAAU,GAAG;MACjBlD,KAAK,EAAE,UAAUA,KAAK,EAAE;MACxBmD,gBAAgB,EAAE;QAChBC,kBAAkB,EAAE,CAAC,OAAO;QAC5B;QACA;QACA;MACF;IACF,CAAC;;IAED;IACA,IAAI,IAAI,CAAC3D,MAAM,CAAC4D,YAAY,EAAE;MAC5BH,KAAK,CAACI,iBAAiB,GAAG;QACxBf,KAAK,EAAE,CAAC;UAAEN,IAAI,EAAE,IAAI,CAACxC,MAAM,CAAC4D;QAAa,CAAC;MAC5C,CAAC;IACH;;IAEA;IACA;IACA;IACA;IACA;IACA;IACA,IAAI,IAAI,CAAC5D,MAAM,CAAC8D,KAAK,EAAE3B,MAAM,EAAE;MAC7B,MAAM4B,UAAU,GAAG,IAAI,CAAC/D,MAAM,CAAC8D,KAAK,CAACE,MAAM,CAACC,CAAC,IAAIA,CAAC,CAACd,IAAI,KAAK,oBAAoB,CAAC;MACjF,IAAIY,UAAU,CAAC5B,MAAM,GAAG,CAAC,EAAE;QACzBsB,KAAK,CAACK,KAAK,GAAG,CAAC;UACbI,oBAAoB,EAAEH,UAAU,CAACI,GAAG,CAACC,IAAI,IAAI;YAC3C,MAAMC,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACH,IAAI,CAACI,UAAU,IAAI,CAAC,CAAC,CAAC,CAACrC,MAAM,GAAG,CAAC;YAC/D,MAAMsC,YAAiB,GAAG;cACxBtB,IAAI,EAAEiB,IAAI,CAACjB,IAAI;cACfuB,WAAW,EAAEN,IAAI,CAACM;YACpB,CAAC;YAED,IAAIL,SAAS,EAAE;cACbI,YAAY,CAACD,UAAU,GAAG;gBACxBG,IAAI,EAAE,QAAQ;gBACdC,UAAU,EAAEN,MAAM,CAACO,WAAW,CAC5BP,MAAM,CAACQ,OAAO,CAACV,IAAI,CAACI,UAAU,CAAC,CAACL,GAAG,CAAC,CAAC,CAACY,GAAG,EAAEC,KAAK,CAAC,KAAK;kBACpD;kBACA;kBACA,IAAIC,SAAS,GAAGD,KAAK,CAACL,IAAI,CAACO,WAAW,CAAC,CAAC;kBACxC,IAAIC,IAAI,GAAGH,KAAK,CAACN,WAAW;kBAC5B,IAAIO,SAAS,KAAK,SAAS,EAAE;oBAC3BA,SAAS,GAAG,QAAQ;oBACpBE,IAAI,GAAG,GAAGA,IAAI,0BAA0B;kBAC1C;kBACA,OAAO,CACLJ,GAAG,EACH;oBACEJ,IAAI,EAAEM,SAAS;oBACfP,WAAW,EAAES;kBACf,CAAC,CACF;gBACH,CAAC,CACH,CAAC;gBACDC,QAAQ,EAAEd,MAAM,CAACQ,OAAO,CAACV,IAAI,CAACI,UAAU,CAAC,CACtCR,MAAM,CAAC,CAAC,GAAGgB,KAAK,CAAC,KAAKA,KAAK,CAACI,QAAQ,CAAC,CACrCjB,GAAG,CAAC,CAAC,CAACY,GAAG,CAAC,KAAKA,GAAG;cACvB,CAAC;YACH;YACA,OAAON,YAAY;UACrB,CAAC;QACH,CAAC,CAAC;MACJ;IACF;IAEA,MAAMY,YAAY,GAAG;MAAE5B;IAAM,CAAC;IAC9BpE,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBE,KAAK,KAAK,IAAI,CAACP,MAAM,CAAC8D,KAAK,EAAE3B,MAAM,IAAI,CAAC,SAAS,CAAC;IACvG,IAAI;MACF,MAAMmD,OAAO,GAAGjD,IAAI,CAACC,SAAS,CAAC+C,YAAY,CAAC;MAC5ChG,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBiF,OAAO,EAAE,CAAC;MAC/D,IAAI,CAAC3F,EAAE,CAACyC,IAAI,CAACkD,OAAO,CAAC;IACvB,CAAC,CAAC,OAAOC,GAAQ,EAAE;MACjBlG,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,uCAAuCsE,GAAG,CAACrE,OAAO,EAAE,CAAC;IACpF;EACF;;EAEA;;EAEQG,aAAaA,CAACR,KAA4B,EAAQ;IACxD,IAAI;MACF,MAAM2E,QAAQ,GAAG,OAAO3E,KAAK,CAACqB,IAAI;MAClC,MAAMuD,OAAO,GAAG,OAAO5E,KAAK,CAACqB,IAAI,KAAK,QAAQ,GAAGrB,KAAK,CAACqB,IAAI,CAACC,MAAM,GAAItB,KAAK,CAACqB,IAAI,EAAEwD,UAAU,IAAI,SAAU;MAC1GrG,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,gCAAgCmF,QAAQ,YAAYC,OAAO,EAAE,CAAC;;MAE1F;MACA,IAAI,OAAO5E,KAAK,CAACqB,IAAI,KAAK,QAAQ,EAAE;QAClC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,mCAAmC,CAAC;QAChE,IAAI,CAACsF,mBAAmB,CAAC9E,KAAK,CAACqB,IAAI,CAAC;QACpC;MACF;;MAEA;MACA,MAAMhB,OAAO,GAAGmB,IAAI,CAACuD,KAAK,CAAC/E,KAAK,CAACqB,IAAI,CAAC;MACtC7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,yBAAyBiE,MAAM,CAACC,IAAI,CAACrD,OAAO,CAAC,CAAC2E,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;MACvFxG,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,wBAAwBQ,KAAK,CAACqB,IAAI,CAACe,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;MACpF,IAAI,CAAC6C,cAAc,CAAC5E,OAAO,CAAC;IAC9B,CAAC,CAAC,OAAOD,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,2BAA2BA,KAAK,CAACC,OAAO,EAAE,CAAC;IAC1E;EACF;EAEQyE,mBAAmBA,CAACzD,IAAS,EAAQ;IAC3C,IAAI;MACF;MACA,IAAI6D,KAAiB;MACrB,IAAI7D,IAAI,YAAY8D,WAAW,EAAE;QAC/BD,KAAK,GAAG,IAAIE,UAAU,CAAC/D,IAAI,CAAC;MAC9B,CAAC,MAAM,IAAIA,IAAI,YAAYgE,IAAI,EAAE;QAC/B;QACA,MAAMC,MAAM,GAAG,IAAIC,UAAU,CAAC,CAAC;QAC/BD,MAAM,CAACE,MAAM,GAAG,MAAM;UACpB,IAAIF,MAAM,CAAC9C,MAAM,YAAY2C,WAAW,EAAE;YACxC,IAAI,CAACM,kBAAkB,CAAC,IAAIL,UAAU,CAACE,MAAM,CAAC9C,MAAM,CAAC,CAAC;UACxD;QACF,CAAC;QACD8C,MAAM,CAACI,iBAAiB,CAACrE,IAAI,CAAC;QAC9B;MACF,CAAC,MAAM;QACL;MACF;MAEA,IAAI,CAACoE,kBAAkB,CAACP,KAAK,CAAC;IAChC,CAAC,CAAC,OAAO9E,KAAU,EAAE;MACnB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,kCAAkCA,KAAK,CAACC,OAAO,EAAE,CAAC;IACjF;EACF;EAEQoF,kBAAkBA,CAACP,KAAiB,EAAQ;IAClD;IACA,MAAMS,aAAa,GAAGT,KAAK,CAAC5D,MAAM,GAAG,CAAC,KAAK4D,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAIA,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAE/E,IAAIS,aAAa,EAAE;MACjB,IAAI;QACF,MAAMhE,IAAI,GAAG,IAAIiE,WAAW,CAAC,OAAO,CAAC,CAACC,MAAM,CAACX,KAAK,CAAC;QACnD,MAAM7E,OAAO,GAAGmB,IAAI,CAACuD,KAAK,CAACpD,IAAI,CAAC;QAChC,IAAI,CAACsD,cAAc,CAAC5E,OAAO,CAAC;MAC9B,CAAC,CAAC,MAAM;QACN;QACA,IAAI,CAACtB,SAAS,CAAC+G,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;MACzF;IACF,CAAC,MAAM;MACL;MACA,IAAI,CAACjH,SAAS,CAAC+G,eAAe,GAAG,IAAI,CAACC,mBAAmB,CAACb,KAAK,CAACc,MAAqB,CAAC,CAAC;IACzF;EACF;EAEQf,cAAcA,CAAC5E,OAAY,EAAQ;IACzC;IACA,IAAIA,OAAO,CAACrB,aAAa,KAAKiH,SAAS,EAAE;MACvCzH,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,6CAA6C,CAAC;MAC1E,IAAI,CAACR,aAAa,GAAG,IAAI;MACzB,IAAI,CAACS,SAAS,CAAC,WAAW,CAAC;MAC3B;IACF;;IAEA;IACA,IAAIY,OAAO,CAAC6F,aAAa,EAAE;MACzB,MAAMC,OAAO,GAAG9F,OAAO,CAAC6F,aAAa;MACrC1H,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,4CAA4C2G,OAAO,CAACjE,YAAY,cAAc,CAAC,CAACiE,OAAO,CAACC,SAAS,EAAEnE,KAAK,wBAAwB,CAAC,CAACkE,OAAO,CAACE,kBAAkB,yBAAyB,CAAC,CAACF,OAAO,CAACG,mBAAmB,EAAE,CAAC;;MAEjP;MACA,IAAIH,OAAO,CAACjE,YAAY,EAAE;QACxB,IAAI,CAACnD,SAAS,CAACwH,cAAc,GAAG,CAAC;MACnC;;MAEA;MACA,IAAIJ,OAAO,CAACC,SAAS,EAAEnE,KAAK,EAAE;QAC5B,KAAK,MAAMuE,IAAI,IAAIL,OAAO,CAACC,SAAS,CAACnE,KAAK,EAAE;UAC1C;UACA,IAAIuE,IAAI,CAACC,UAAU,EAAEpF,IAAI,EAAE;YACzB7C,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsBgH,IAAI,CAACC,UAAU,CAACpF,IAAI,CAACC,MAAM,QAAQ,CAAC;YACtF,IAAI,CAACvC,SAAS,CAAC+G,eAAe,GAAGU,IAAI,CAACC,UAAU,CAACpF,IAAI,CAAC;UACxD;;UAEA;UACA,IAAImF,IAAI,CAAC7E,IAAI,EAAE;YACbnD,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,sBAAsBgH,IAAI,CAAC7E,IAAI,GAAG,CAAC;YAC/D,IAAI,CAAC5C,SAAS,CAAC2H,YAAY,GAAGF,IAAI,CAAC7E,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;UACzD;QACF;MACF;;MAEA;MACA,IAAIwE,OAAO,CAACE,kBAAkB,EAAE1E,IAAI,EAAE;QACpC,IAAI,CAAC5C,SAAS,CAAC2H,YAAY,GAAGP,OAAO,CAACE,kBAAkB,CAAC1E,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;MAC9E;;MAEA;MACA,IAAIwE,OAAO,CAACG,mBAAmB,EAAE3E,IAAI,EAAE;QACrC,IAAI,CAAC5C,SAAS,CAAC2H,YAAY,GAAGP,OAAO,CAACG,mBAAmB,CAAC3E,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;MAChF;IACF;;IAEA;IACA,IAAItB,OAAO,CAACsG,QAAQ,EAAEC,aAAa,EAAE;MACnC,KAAK,MAAMC,EAAE,IAAIxG,OAAO,CAACsG,QAAQ,CAACC,aAAa,EAAE;QAC/CpI,MAAM,CAACgB,IAAI,CAAC,cAAc,EAAE,iBAAiBqH,EAAE,CAACvE,IAAI,IAAId,IAAI,CAACC,SAAS,CAACoF,EAAE,CAACC,IAAI,CAAC,GAAG,CAAC;QACnF,IAAI,CAAC/H,SAAS,CAACgI,UAAU,GAAG;UAC1BzE,IAAI,EAAEuE,EAAE,CAACvE,IAAI;UACbwE,IAAI,EAAED,EAAE,CAACC,IAAI,IAAI,CAAC,CAAC;UACnBvE,EAAE,EAAEsE,EAAE,CAACtE;QACT,CAAC,CAAC;MACJ;IACF;;IAEA;IACA,IAAIlC,OAAO,CAACD,KAAK,EAAE;MACjB5B,MAAM,CAAC4B,KAAK,CAAC,cAAc,EAAE,iBAAiBoB,IAAI,CAACC,SAAS,CAACpB,OAAO,CAACD,KAAK,CAAC,EAAE,CAAC;MAC9E,IAAI,CAACrB,SAAS,CAACuB,OAAO,GAAGD,OAAO,CAACD,KAAK,CAACC,OAAO,IAAI,cAAc,CAAC;IACnE;EACF;;EAEA;;EAEQZ,SAASA,CAACuH,SAAsB,EAAQ;IAC9C,IAAI,CAAC/H,OAAO,GAAG+H,SAAS;IACxB,IAAI,CAACjI,SAAS,CAACkI,cAAc,GAAGD,SAAS,CAAC;EAC5C;EAEQjB,mBAAmBA,CAACC,MAAmB,EAAU;IACvD,MAAMd,KAAK,GAAG,IAAIE,UAAU,CAACY,MAAM,CAAC;IACpC,IAAIkB,MAAM,GAAG,EAAE;IACf,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGjC,KAAK,CAACL,UAAU,EAAEsC,CAAC,EAAE,EAAE;MACzCD,MAAM,IAAIE,MAAM,CAACC,YAAY,CAACnC,KAAK,CAACiC,CAAC,CAAE,CAAC;IAC1C;IACA,OAAOG,IAAI,CAACJ,MAAM,CAAC;EACrB;AACF","ignoreList":[]}
package/lib/typescript/src/components/AIAgent.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;AAaf,OAAO,KAAK,EAA0B,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAOpH,UAAU,YAAY;IACpB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,kBAAkB;IAClB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IAI7C,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,8BAA8B;IAC9B,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3D,6BAA6B;IAC7B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1C,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAChE,qEAAqE;IACrE,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACvE,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IACpD,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IACF,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,MAAM,CAAC,EAAE;QACP,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAC7B,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAChC,IAAI,EAAE,MAAM,IAAI,CAAC;KAClB,CAAC;IACF,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mDAAmD;IACnD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAC3C,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAID,wBAAgB,OAAO,CAAC,EACtB,MAAM,EACN,KAA0B,EAC1B,MAAM,EACN,QAAe,EACf,QAAa,EACb,WAAkB,EAClB,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,WAAmB,EACnB,YAAY,EACZ,KAAa,GACd,EAAE,YAAY,2CA4Vd"}
1
+ {"version":3,"file":"AIAgent.d.ts","sourceRoot":"","sources":["../../../../src/components/AIAgent.tsx"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAMN,MAAM,OAAO,CAAC;AAaf,OAAO,KAAK,EAA0B,eAAe,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAOpH,UAAU,YAAY;IACpB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,kBAAkB;IAClB,QAAQ,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6BAA6B;IAC7B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gCAAgC;IAChC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAC1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IAI7C,qDAAqD;IACrD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9C,8BAA8B;IAC9B,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3D,6BAA6B;IAC7B,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7D,gCAAgC;IAChC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1C,kCAAkC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAChE,qEAAqE;IACrE,sBAAsB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACvE,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI,CAAC,CAAC;IACpD,2CAA2C;IAC3C,YAAY,CAAC,EAAE;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;KAC3E,CAAC;IACF,gCAAgC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,MAAM,CAAC,EAAE;QACP,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAC7B,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;QAChC,IAAI,EAAE,MAAM,IAAI,CAAC;KAClB,CAAC;IACF,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mDAAmD;IACnD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAC3C,qDAAqD;IACrD,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAID,wBAAgB,OAAO,CAAC,EACtB,MAAM,EACN,KAA0B,EAC1B,MAAM,EACN,QAAe,EACf,QAAa,EACb,WAAkB,EAClB,QAAQ,EACR,QAAQ,EAER,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,WAAW,EACX,YAAY,EACZ,SAAS,EACT,YAAY,EACZ,MAAM,EACN,QAAQ,EACR,WAAmB,EACnB,YAAY,EACZ,KAAa,GACd,EAAE,YAAY,2CA+Vd"}
package/lib/typescript/src/core/systemPrompt.d.ts CHANGED
@@ -7,11 +7,14 @@
7
7
  */
8
8
  export declare function buildSystemPrompt(language: string): string;
9
9
  /**
10
- * Voice-optimized system prompt for the Gemini Live API.
10
+ * Voice-adapted system prompt for the Gemini Live API.
11
11
  *
12
- * Includes the same screen format and tool semantics as text mode,
13
- * but condensed for voice context and with guardrails against
14
- * unprompted actions.
12
+ * Uses the same core rules/tools/screen format as text mode (buildSystemPrompt)
13
+ * but adapted for voice interaction:
14
+ * - No agent-loop directives (no "MUST call agent_step on every step")
15
+ * - No agent_history/user_request references (voice is conversational)
16
+ * - Explicit "wait for user voice command" guardrails
17
+ * - Voice-specific UX (concise spoken responses)
15
18
  */
16
19
  export declare function buildVoiceSystemPrompt(language: string, userInstructions?: string): string;
17
20
  //# sourceMappingURL=systemPrompt.d.ts.map
package/lib/typescript/src/core/systemPrompt.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"systemPrompt.d.ts","sourceRoot":"","sources":["../../../../src/core/systemPrompt.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAyJ1D;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,EAChB,gBAAgB,CAAC,EAAE,MAAM,GACxB,MAAM,CAkDR"}
1
+ {"version":3,"file":"systemPrompt.d.ts","sourceRoot":"","sources":["../../../../src/core/systemPrompt.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAyJ1D;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,EAChB,gBAAgB,CAAC,EAAE,MAAM,GACxB,MAAM,CAiFR"}
package/lib/typescript/src/index.d.ts CHANGED
@@ -6,10 +6,6 @@
6
6
  */
7
7
  export { AIAgent } from './components/AIAgent';
8
8
  export { useAction } from './hooks/useAction';
9
- export { VoiceService } from './services/VoiceService';
10
- export { AudioInputService } from './services/AudioInputService';
11
- export { AudioOutputService } from './services/AudioOutputService';
12
9
  export { logger } from './utils/logger';
13
10
  export type { AgentConfig, AgentMode, ExecutionResult, InteractiveElement, DehydratedScreen, ToolDefinition, ActionDefinition, TokenUsage, } from './core/types';
14
- export type { VoiceServiceConfig, VoiceServiceCallbacks, VoiceStatus, } from './services/VoiceService';
15
11
  //# sourceMappingURL=index.d.ts.map
package/lib/typescript/src/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAG/C,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAG9C,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAGnE,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAGxC,YAAY,EACV,WAAW,EACX,SAAS,EACT,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,cAAc,CAAC;AAEtB,YAAY,EACV,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,GACZ,MAAM,yBAAyB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAG/C,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAQ9C,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAGxC,YAAY,EACV,WAAW,EACX,SAAS,EACT,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,EAChB,UAAU,GACX,MAAM,cAAc,CAAC"}
package/lib/typescript/src/services/VoiceService.d.ts CHANGED
@@ -62,6 +62,14 @@ export declare class VoiceService {
62
62
  sendScreenContext(domText: string): void;
63
63
  /** Send function call result back to Gemini */
64
64
  sendFunctionResponse(name: string, id: string, result: any): void;
65
+ /**
66
+ * Builds and sends the setup message, replicating text mode's agent_step
67
+ * compound tool so the model uses structured reasoning + actions.
68
+ *
69
+ * The agent_step tool flattens reasoning fields (previous_goal_eval,
70
+ * memory, plan) + action_name enum + all action parameters into a single
71
+ * function — matching GeminiProvider.buildAgentStepDeclaration exactly.
72
+ */
65
73
  private sendSetup;
66
74
  private handleMessage;
67
75
  private handleBinaryMessage;
package/lib/typescript/src/services/VoiceService.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"VoiceService.d.ts","sourceRoot":"","sources":["../../../../src/services/VoiceService.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAIpD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAChD,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACzF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,KAAK,IAAI,CAAC;IAChF,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;IAC/C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,IAAI,CAAC;CAC7B;AAED,MAAM,MAAM,WAAW,GAAG,cAAc,GAAG,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;AAWhF,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,EAAE,kBAAkB;IAMtC,OAAO,CAAC,SAAS,EAAE,qBAAqB,GAAG,IAAI;IAqC/C,UAAU,IAAI,IAAI;IAUlB,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,aAAa,IAAI,WAAW,CAE/B;IAID,sDAAsD;IACtD,OAAO,CAAC,SAAS,CAAK;IACtB,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAsBpC,kEAAkE;IAClE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU5B;;;;;;;;OAQG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAgBxC,+CAA+C;IAC/C,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI;IAmBjE,OAAO,CAAC,SAAS;IA2DjB,OAAO,CAAC,aAAa;IAsBrB,OAAO,CAAC,mBAAmB;IA0B3B,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,mBAAmB;CAQ5B"}
1
+ {"version":3,"file":"VoiceService.d.ts","sourceRoot":"","sources":["../../../../src/services/VoiceService.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAIpD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,eAAe,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAChD,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IACzF,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,KAAK,IAAI,CAAC;IAChF,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;IAC/C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,IAAI,CAAC;CAC7B;AAED,MAAM,MAAM,WAAW,GAAG,cAAc,GAAG,YAAY,GAAG,WAAW,GAAG,OAAO,CAAC;AAehF,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,EAAE,kBAAkB;IAMtC,OAAO,CAAC,SAAS,EAAE,qBAAqB,GAAG,IAAI;IAqC/C,UAAU,IAAI,IAAI;IAUlB,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED,IAAI,aAAa,IAAI,WAAW,CAE/B;IAID,sDAAsD;IACtD,OAAO,CAAC,SAAS,CAAK;IACtB,SAAS,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAsBpC,kEAAkE;IAClE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU5B;;;;;;;;OAQG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAiBxC,+CAA+C;IAC/C,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,IAAI;IAmBjE;;;;;;;OAOG;IACH,OAAO,CAAC,SAAS;IAqFjB,OAAO,CAAC,aAAa;IAuBrB,OAAO,CAAC,mBAAmB;IA0B3B,OAAO,CAAC,kBAAkB;IAmB1B,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,mBAAmB;CAQ5B"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mobileai/react-native",
3
- "version": "0.4.5",
3
+ "version": "0.4.6",
4
4
  "description": "Build autonomous AI agents for React Native and Expo apps. Provides AI-native UI traversal, tool calling, and structured reasoning.",
5
5
  "main": "./lib/module/index.js",
6
6
  "source": "./src/index.ts",
package/src/components/AIAgent.tsx CHANGED
@@ -242,8 +242,8 @@ export function AIAgent({
242
242
  logger.info('AIAgent', 'Creating VoiceService...');
243
243
  const runtimeTools = runtime.getTools();
244
244
  logger.info('AIAgent', `Registering ${runtimeTools.length} tools with VoiceService: ${runtimeTools.map(t => t.name).join(', ')}`);
245
- // Build the full voice system prompt (screen format + tool descriptions + guardrails)
246
- // This gives voice mode the same screen understanding as text mode
245
+ // Use voice-adapted system prompt — same core rules as text mode
246
+ // but without agent-loop directives that trigger autonomous actions
247
247
  const voicePrompt = buildVoiceSystemPrompt(language, instructions?.system);
248
248
  voiceServiceRef.current = new VoiceService({
249
249
  apiKey,
@@ -300,10 +300,12 @@ export function AIAgent({
300
300
  logger.info('AIAgent', '🎙️ Mic auto-started after connection');
301
301
  }
302
302
  });
303
- // Send initial screen context (tree) so the model knows what's on screen
303
+ // Send initial screen context so the model knows what's on screen.
304
+ // sendScreenContext uses turnComplete: false (passive context)
305
+ // so the model should NOT act on it until the user speaks.
304
306
  const initialContext = runtime.getScreenContext();
305
307
  voiceServiceRef.current?.sendScreenContext(initialContext);
306
- logger.info('AIAgent', '📡 Initial screen context sent to voice model');
308
+ logger.info('AIAgent', '📡 Initial screen context sent (passive, turnComplete=false)');
307
309
  }
308
310
  },
309
311
  onTranscript: (text, isFinal, role) => {
@@ -314,6 +316,7 @@ export function AIAgent({
314
316
  // Execute the tool via AgentRuntime and send result back to Gemini
315
317
  const result = await runtime.executeTool(toolCall.name, toolCall.args);
316
318
  logger.info('AIAgent', `Voice tool result: ${result}`);
319
+
317
320
  voiceServiceRef.current?.sendFunctionResponse(toolCall.name, toolCall.id, { result });
318
321
 
319
322
  // After tool execution, push updated screen context
package/src/core/systemPrompt.ts CHANGED
@@ -162,11 +162,14 @@ plan: "Call done to report the cart contents to the user."
162
162
  }
163
163
 
164
164
  /**
165
- * Voice-optimized system prompt for the Gemini Live API.
165
+ * Voice-adapted system prompt for the Gemini Live API.
166
166
  *
167
- * Includes the same screen format and tool semantics as text mode,
168
- * but condensed for voice context and with guardrails against
169
- * unprompted actions.
167
+ * Uses the same core rules/tools/screen format as text mode (buildSystemPrompt)
168
+ * but adapted for voice interaction:
169
+ * - No agent-loop directives (no "MUST call agent_step on every step")
170
+ * - No agent_history/user_request references (voice is conversational)
171
+ * - Explicit "wait for user voice command" guardrails
172
+ * - Voice-specific UX (concise spoken responses)
170
173
  */
171
174
  export function buildVoiceSystemPrompt(
172
175
  language: string,
@@ -174,46 +177,77 @@ export function buildVoiceSystemPrompt(
174
177
  ): string {
175
178
  const isArabic = language === 'ar';
176
179
 
177
- let prompt = `You are a voice-controlled AI agent operating a React Native mobile app. You can see the screen content and interact with UI elements using tools.
180
+ let prompt = `You are a voice-controlled AI agent operating a React Native mobile app. You receive periodic screen updates showing what's currently visible, and you can interact with UI elements using tools. You respond to the user via spoken audio.
178
181
 
179
- <language>
180
- ${isArabic ? 'Respond in Arabic.' : 'Respond in English.'}
181
- Use the same language as the user.
182
- </language>
183
-
184
- <screen_format>
185
- You receive periodic screen updates showing the current UI. Interactive elements appear as:
186
- [index]<type attrs>label />
182
+ <language_settings>
183
+ ${isArabic ? '- Working language: **Arabic**. Respond in Arabic.' : '- Working language: **English**. Respond in English.'}
184
+ - Use the same language as the user. Return in user's language.
185
+ </language_settings>
187
186
 
188
- - index: numeric ID for interaction (use with tap/type tools)
187
+ <screen_state>
188
+ Interactive elements are listed as [index]<type attrs>label />
189
+ - index: numeric identifier for interaction
189
190
  - type: element type (pressable, text-input, switch)
190
- - attrs: state like value="true", checked="false", role="switch"
191
- - label: visible text content
191
+ - attrs: state attributes like value="true", checked="false", role="switch"
192
+ - label: visible text content of the element
192
193
 
193
- Only elements with [index] are interactive. Text without [] is display-only.
194
- Example: [5]<switch value="true">Order Updates /> means element 5 is a switch currently ON.
195
- </screen_format>
194
+ Only elements with [index] are interactive. Use the index to tap or type into them.
195
+ Pure text elements without [] are NOT interactive — they are informational content you can read.
196
+ </screen_state>
196
197
 
197
198
  <tools>
198
199
  Available tools:
199
- - tap(index): Tap an element. For switches, this toggles their value.
200
- - type(index, text): Type text into a text-input.
201
- - navigate(screen): Navigate to a named screen.
202
- - done(text, success): Complete the task with a spoken response.
203
- - ask_user(question): Ask the user for clarification.
200
+ - tap(index): Tap an interactive element by its index. Works universally on buttons, switches, and custom components. For switches, this toggles their state.
201
+ - type(index, text): Type text into a text-input element by its index.
202
+ - navigate(screen, params): Navigate to a specific screen. params is optional JSON object.
203
+ - done(text, success): Complete task. Text is your final response to the user.
204
+ - ask_user(question): Ask the user for clarification ONLY when you cannot determine what action to take.
205
+
206
+ When you need to perform an action, call the appropriate tool function directly.
204
207
  </tools>
205
208
 
209
+ <voice_interaction_rules>
210
+ CRITICAL — THESE RULES OVERRIDE EVERYTHING ELSE:
211
+ - You are in a LIVE VOICE conversation. Wait for the user to SPEAK before doing anything.
212
+ - Screen updates arrive as passive context — they are NOT commands. Do NOT act on them.
213
+ - ONLY take action (tap, type, navigate) when the user explicitly asks you to via voice.
214
+ - When you have NO voice command from the user, stay silent. Do NOT narrate the screen.
215
+ - When the user speaks, determine the request type BEFORE acting:
216
+ 1. Information requests ("what's on screen?", "how much is X?"): Respond with spoken audio. Do NOT call any tools.
217
+ 2. Action requests ("go to settings", "add pizza to cart"): Call the appropriate tool function directly (e.g. navigate, tap).
218
+ - After completing an action, speak a brief confirmation to the user.
219
+ - Keep all spoken responses concise — the user is listening, not reading.
220
+ </voice_interaction_rules>
221
+
206
222
  <rules>
207
- CRITICAL ACTION RULES:
208
- - ONLY perform actions (tap, type, navigate) when the user explicitly asks you to do something.
209
- - NEVER tap or navigate on your own initiative — wait for the user's voice command.
210
- - When the user asks a question about what's on screen, answer verbally via done(). Do NOT tap anything.
211
- - When the user asks to toggle/enable/disable something, find the matching element by its label and use tap(index).
212
- - When a screen update arrives, do NOT interact with elements unless the user asked you to.
213
- - Use element indexes from the most recent screen update — they refresh every few seconds.
214
- - For switches: tap(index) toggles the value. You do NOT need to find a separate button.
215
- - Keep spoken responses concise — the user is listening, not reading.
216
- </rules>`;
223
+ - There are 2 types of requests — always determine which type BEFORE acting:
224
+ 1. Information requests (e.g. "what's available?", "how much is X?", "list the items"):
225
+ Respond verbally with the answer. Do NOT perform any tap/type/navigate actions.
226
+ 2. Action requests (e.g. "add margherita to cart", "go to checkout", "fill in my name"):
227
+ Execute the required UI interactions using tap/type/navigate tools.
228
+ - Only interact with elements that have an [index].
229
+ - If the current screen doesn't have what you need, use navigate() to go to another screen.
230
+ - When the user asks to go to a specific screen by name and it's listed in Available Screens, use navigate(screen) instead of tapping.
231
+ - Do not repeat one action for more than 3 times unless conditions changed.
232
+ - Do not fill in login/signup forms unless the user provides credentials. If asked to log in, use ask_user to request their email and password first.
233
+ - Do not guess or auto-fill sensitive data (passwords, payment info, personal details). Always ask the user.
234
+ - If stuck, tell the user what happened rather than repeating failed actions.
235
+ </rules>
236
+
237
+ <capability>
238
+ - It is ok to just provide information without performing any actions.
239
+ - User can ask questions about what's on screen — answer them directly by speaking.
240
+ - It is ok to fail the task. User would rather you report failure than repeat failed actions endlessly.
241
+ - The user can be wrong. If the request is not achievable, tell the user.
242
+ </capability>
243
+
244
+ <ux_rules>
245
+ - Confirm what you did: When completing actions, briefly say what happened.
246
+ - Be transparent about errors: If an action fails, explain what failed and why.
247
+ - Be concise: Keep spoken responses short and clear. No walls of text.
248
+ - Suggest next steps: After completing an action, briefly suggest what the user might want to do next.
249
+ - When a request is ambiguous, pick the most common interpretation and state your assumption.
250
+ </ux_rules>`;
217
251
 
218
252
  // Append user-provided instructions if any
219
253
  if (userInstructions?.trim()) {
package/src/index.ts CHANGED
@@ -12,9 +12,9 @@ export { AIAgent } from './components/AIAgent';
12
12
  export { useAction } from './hooks/useAction';
13
13
 
14
14
  // ─── Services ────────────────────────────────────────────────
15
- export { VoiceService } from './services/VoiceService';
16
- export { AudioInputService } from './services/AudioInputService';
17
- export { AudioOutputService } from './services/AudioOutputService';
15
+ // export { VoiceService } from './services/VoiceService';
16
+ // export { AudioInputService } from './services/AudioInputService';
17
+ // export { AudioOutputService } from './services/AudioOutputService';
18
18
 
19
19
  // ─── Utilities ───────────────────────────────────────────────
20
20
  export { logger } from './utils/logger';
@@ -31,8 +31,8 @@ export type {
31
31
  TokenUsage,
32
32
  } from './core/types';
33
33
 
34
- export type {
35
- VoiceServiceConfig,
36
- VoiceServiceCallbacks,
37
- VoiceStatus,
38
- } from './services/VoiceService';
34
+ // export type {
35
+ // VoiceServiceConfig,
36
+ // VoiceServiceCallbacks,
37
+ // VoiceStatus,
38
+ // } from './services/VoiceService';
package/src/services/VoiceService.ts CHANGED
@@ -42,7 +42,11 @@ export type VoiceStatus = 'disconnected' | 'connecting' | 'connected' | 'error';
42
42
 
43
43
  const WS_HOST = 'generativelanguage.googleapis.com';
44
44
  const WS_PATH = '/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
45
- const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
45
+ // Use -09-2025: Google's own cookbook uses this model for Live API tool use.
46
+ // The -12-2025 model had server-side regressions with function calling
47
+ // and was deprecated March 19, 2026. The -09-2025 version has
48
+ // "improved function calling and better handling of speech cut-offs."
49
+ const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-09-2025';
46
50
  const DEFAULT_INPUT_SAMPLE_RATE = 16000;
47
51
 
48
52
  // ─── Service ───────────────────────────────────────────────────
@@ -171,8 +175,9 @@ export class VoiceService {
171
175
  },
172
176
  };
173
177
 
174
- this.ws!.send(JSON.stringify(message));
175
178
  logger.debug('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
179
+ logger.debug('VoiceService', `📤 Raw Screen Context Payload: ${JSON.stringify(message).substring(0, 500)}...`);
180
+ this.ws!.send(JSON.stringify(message));
176
181
  }
177
182
 
178
183
  // ─── Send Function Response ────────────────────────────────
@@ -197,6 +202,14 @@ export class VoiceService {
197
202
 
198
203
  // ─── Internal: Setup ───────────────────────────────────────
199
204
 
205
+ /**
206
+ * Builds and sends the setup message, replicating text mode's agent_step
207
+ * compound tool so the model uses structured reasoning + actions.
208
+ *
209
+ * The agent_step tool flattens reasoning fields (previous_goal_eval,
210
+ * memory, plan) + action_name enum + all action parameters into a single
211
+ * function — matching GeminiProvider.buildAgentStepDeclaration exactly.
212
+ */
200
213
  private sendSetup(): void {
201
214
  if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
202
215
 
@@ -206,13 +219,12 @@ export class VoiceService {
206
219
  model: `models/${model}`,
207
220
  generationConfig: {
208
221
  responseModalities: ['AUDIO'],
222
+ // Note: Do NOT set thinkingBudget: 0 — it completely disables
223
+ // the model's ability to reason about when to call tools.
224
+ // The text thinking blocks are a trade-off for working function calling.
209
225
  },
210
226
  };
211
227
 
212
- // For native audio models, language is enforced via system instructions.
213
- // Explicitly setting speechConfig.languageCode causes silent API failures
214
- // without a prebuiltVoiceConfig.
215
-
216
228
  // Add system instruction if provided
217
229
  if (this.config.systemPrompt) {
218
230
  setup.systemInstruction = {
@@ -220,38 +232,65 @@ export class VoiceService {
220
232
  };
221
233
  }
222
234
 
223
- // Add tool declarations for function calling
235
+ // Add individual tool declarations for function calling
236
+ // NOTE: We use individual tools (tap, type, navigate, done, ask_user)
237
+ // instead of the compound agent_step used in text mode.
238
+ // The native audio model in real-time can call simple tools but struggles
239
+ // with the complex agent_step schema (it speaks about calling tools
240
+ // instead of actually calling them).
224
241
  if (this.config.tools?.length) {
225
- setup.tools = [{
226
- functionDeclarations: this.config.tools.map(tool => ({
227
- name: tool.name,
228
- description: tool.description,
229
- parameters: {
230
- type: 'OBJECT',
231
- properties: Object.fromEntries(
232
- Object.entries(tool.parameters).map(([key, param]) => [
233
- key,
234
- {
235
- type: param.type.toUpperCase(),
236
- description: param.description,
237
- },
238
- ])
239
- ),
240
- required: Object.entries(tool.parameters)
241
- .filter(([, param]) => param.required)
242
- .map(([key]) => key),
243
- },
244
- })),
245
- }];
242
+ const validTools = this.config.tools.filter(t => t.name !== 'capture_screenshot');
243
+ if (validTools.length > 0) {
244
+ setup.tools = [{
245
+ functionDeclarations: validTools.map(tool => {
246
+ const hasParams = Object.keys(tool.parameters || {}).length > 0;
247
+ const functionDecl: any = {
248
+ name: tool.name,
249
+ description: tool.description,
250
+ };
251
+
252
+ if (hasParams) {
253
+ functionDecl.parameters = {
254
+ type: 'OBJECT',
255
+ properties: Object.fromEntries(
256
+ Object.entries(tool.parameters).map(([key, param]) => {
257
+ // Native audio model crashes with BOOLEAN/ENUM types (error 1008)
258
+ // Convert to STRING as a workaround
259
+ let paramType = param.type.toUpperCase();
260
+ let desc = param.description;
261
+ if (paramType === 'BOOLEAN') {
262
+ paramType = 'STRING';
263
+ desc = `${desc} (use "true" or "false")`;
264
+ }
265
+ return [
266
+ key,
267
+ {
268
+ type: paramType,
269
+ description: desc,
270
+ },
271
+ ];
272
+ })
273
+ ),
274
+ required: Object.entries(tool.parameters)
275
+ .filter(([, param]) => param.required)
276
+ .map(([key]) => key),
277
+ };
278
+ }
279
+ return functionDecl;
280
+ }),
281
+ }];
282
+ }
246
283
  }
247
284
 
248
- // Enable transcription
249
- setup.inputAudioTranscription = {};
250
- setup.outputAudioTranscription = {};
251
-
252
285
  const setupMessage = { setup };
253
- logger.info('VoiceService', `Sending setup (model: ${model}, tools: ${this.config.tools?.length || 0})`);
254
- this.ws.send(JSON.stringify(setupMessage));
286
+ logger.info('VoiceService', `Sending setup (model: ${model}, ${this.config.tools?.length || 0} tools)`);
287
+ try {
288
+ const payload = JSON.stringify(setupMessage);
289
+ logger.info('VoiceService', `📤 Raw Setup Payload: ${payload}`);
290
+ this.ws.send(payload);
291
+ } catch (err: any) {
292
+ logger.error('VoiceService', `❌ Error stringifying setup message: ${err.message}`);
293
+ }
255
294
  }
256
295
 
257
296
  // ─── Internal: Message Handling ────────────────────────────
@@ -272,6 +311,7 @@ export class VoiceService {
272
311
  // Handle JSON text messages
273
312
  const message = JSON.parse(event.data);
274
313
  logger.info('VoiceService', `📥 JSON message keys: ${Object.keys(message).join(', ')}`);
314
+ logger.info('VoiceService', `📥 Raw JSON Message: ${event.data.substring(0, 1000)}`);
275
315
  this.processMessage(message);
276
316
  } catch (error: any) {
277
317
  logger.error('VoiceService', `Error handling message: ${error.message}`);
@@ -373,7 +413,7 @@ export class VoiceService {
373
413
  // Tool calls from the model
374
414
  if (message.toolCall?.functionCalls) {
375
415
  for (const fn of message.toolCall.functionCalls) {
376
- logger.info('VoiceService', `Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
416
+ logger.info('VoiceService', `🎯 Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
377
417
  this.callbacks.onToolCall?.({
378
418
  name: fn.name,
379
419
  args: fn.args || {},