@ariaflowagents/gemini-native-audio 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +104 -0
  2. package/dist/CallWorker.d.ts +56 -0
  3. package/dist/CallWorker.d.ts.map +1 -0
  4. package/dist/CallWorker.js +172 -0
  5. package/dist/CallWorker.js.map +1 -0
  6. package/dist/CapabilityCallWorker.d.ts +46 -0
  7. package/dist/CapabilityCallWorker.d.ts.map +1 -0
  8. package/dist/CapabilityCallWorker.js +319 -0
  9. package/dist/CapabilityCallWorker.js.map +1 -0
  10. package/dist/GeminiLiveSession.d.ts +86 -0
  11. package/dist/GeminiLiveSession.d.ts.map +1 -0
  12. package/dist/GeminiLiveSession.js +297 -0
  13. package/dist/GeminiLiveSession.js.map +1 -0
  14. package/dist/RealtimeCallWorker.d.ts +47 -0
  15. package/dist/RealtimeCallWorker.d.ts.map +1 -0
  16. package/dist/RealtimeCallWorker.js +55 -0
  17. package/dist/RealtimeCallWorker.js.map +1 -0
  18. package/dist/VoiceEngine.d.ts +67 -0
  19. package/dist/VoiceEngine.d.ts.map +1 -0
  20. package/dist/VoiceEngine.js +156 -0
  21. package/dist/VoiceEngine.js.map +1 -0
  22. package/dist/factories.d.ts +32 -0
  23. package/dist/factories.d.ts.map +1 -0
  24. package/dist/factories.js +43 -0
  25. package/dist/factories.js.map +1 -0
  26. package/dist/index.d.ts +13 -0
  27. package/dist/index.d.ts.map +1 -0
  28. package/dist/index.js +12 -0
  29. package/dist/index.js.map +1 -0
  30. package/dist/openai/OpenAIRealtimeClient.d.ts +51 -0
  31. package/dist/openai/OpenAIRealtimeClient.d.ts.map +1 -0
  32. package/dist/openai/OpenAIRealtimeClient.js +327 -0
  33. package/dist/openai/OpenAIRealtimeClient.js.map +1 -0
  34. package/dist/openai/index.d.ts +3 -0
  35. package/dist/openai/index.d.ts.map +1 -0
  36. package/dist/openai/index.js +2 -0
  37. package/dist/openai/index.js.map +1 -0
  38. package/dist/schema-bridge.d.ts +14 -0
  39. package/dist/schema-bridge.d.ts.map +1 -0
  40. package/dist/schema-bridge.js +20 -0
  41. package/dist/schema-bridge.js.map +1 -0
  42. package/dist/types.d.ts +150 -0
  43. package/dist/types.d.ts.map +1 -0
  44. package/dist/types.js +2 -0
  45. package/dist/types.js.map +1 -0
  46. package/package.json +44 -0
@@ -0,0 +1,297 @@
1
+ import { GoogleGenAI } from '@google/genai';
2
+ import { toolSetToGeminiDeclarations } from './schema-bridge.js';
3
+ const DEFAULT_MODEL = 'gemini-3.1-flash-live-preview';
4
+ const SAMPLE_RATE = 24000;
5
+ /**
6
+ * Thin wrapper around @google/genai ai.live.connect().
7
+ *
8
+ * Manages:
9
+ * - Connection lifecycle (connect/disconnect)
10
+ * - Audio input/output encoding (base64 PCM ↔ Uint8Array)
11
+ * - Tool call dispatch → onEvent callback
12
+ * - Session resumption via newHandle
13
+ */
14
+ export class GeminiLiveSession {
15
+ ai;
16
+ session = null;
17
+ config;
18
+ resumptionHandle;
19
+ _connected = false;
20
+ /** True during updateConfig() reconnect cycle — suppresses 'disconnected' event. */
21
+ _reconfiguring = false;
22
+ /** Mutable overrides updated by updateConfig(). Merged with config.overrides. */
23
+ runtimeOverrides;
24
+ /** Typed event listener registry for the RealtimeAudioClient interface. */
25
+ listeners = new Map();
26
+ constructor(config) {
27
+ this.config = config;
28
+ this.ai = new GoogleGenAI({ apiKey: config.gemini.apiKey });
29
+ }
30
+ get connected() {
31
+ return this._connected;
32
+ }
33
+ // ─── RealtimeAudioClient: on / off / ping ───────────────────────────────────
34
+ on(event, handler) {
35
+ if (!this.listeners.has(event)) {
36
+ this.listeners.set(event, new Set());
37
+ }
38
+ this.listeners.get(event).add(handler);
39
+ }
40
+ off(event, handler) {
41
+ this.listeners.get(event)?.delete(handler);
42
+ }
43
+ emitEvent(event, ...args) {
44
+ const handlers = this.listeners.get(event);
45
+ if (!handlers)
46
+ return;
47
+ for (const handler of handlers) {
48
+ handler(...args);
49
+ }
50
+ }
51
+ /**
52
+ * Gemini Live wraps the underlying WebSocket via @google/genai SDK.
53
+ * There is no direct ping API, so we return connection state as a health check.
54
+ */
55
+ async ping() {
56
+ return Promise.resolve(this._connected);
57
+ }
58
+ /**
59
+ * Connect to Gemini Live.
60
+ *
61
+ * @param config Optional RealtimeSessionConfig. When provided, systemInstruction
62
+ * and tools are applied as runtimeOverrides (takes precedence over constructor
63
+ * config.overrides and agent config). This satisfies the RealtimeAudioClient
64
+ * interface while preserving backward compatibility with CapabilityCallWorker,
65
+ * which calls connect() with no arguments.
66
+ */
67
+ async connect(config) {
68
+ // If a RealtimeSessionConfig is provided (RealtimeAudioClient interface call),
69
+ // store its fields as runtimeOverrides so they win over constructor config.
70
+ if (config) {
71
+ this.runtimeOverrides = {
72
+ systemInstruction: config.systemInstruction,
73
+ tools: config.tools,
74
+ };
75
+ }
76
+ const { agent, gemini } = this.config;
77
+ const model = (config?.model ?? gemini.model) ?? DEFAULT_MODEL;
78
+ // Merge overrides: runtimeOverrides (from updateConfig) take priority over
79
+ // config.overrides (from constructor), which take priority over agent config.
80
+ const activeOverrides = { ...this.config.overrides, ...this.runtimeOverrides };
81
+ // Gemini 3.1 rejects empty tools arrays — only send when declarations exist.
82
+ const declarations = activeOverrides.tools
83
+ ?? (agent.tools ? toolSetToGeminiDeclarations(agent.tools) : undefined);
84
+ const tools = declarations && declarations.length > 0
85
+ ? [{ functionDeclarations: declarations }]
86
+ : undefined;
87
+ // Resolve prompt — active overrides win, then fall back to agent config.
88
+ const promptText = activeOverrides.systemInstruction ?? (() => {
89
+ const agentPrompt = agent.prompt;
90
+ if (!agentPrompt)
91
+ return '';
92
+ if (typeof agentPrompt === 'string')
93
+ return agentPrompt;
94
+ // PromptTemplate / AgentPrompt: join sections
95
+ return agentPrompt.sections
96
+ .map(s => s.content)
97
+ .join('\n\n');
98
+ })();
99
+ const liveConfig = {
100
+ responseModalities: ['AUDIO'],
101
+ systemInstruction: { parts: [{ text: promptText }] },
102
+ tools,
103
+ outputAudioTranscription: {},
104
+ };
105
+ if (agent.voice) {
106
+ liveConfig.speechConfig = {
107
+ voiceConfig: { prebuiltVoiceConfig: { voiceName: agent.voice } },
108
+ };
109
+ }
110
+ if (this.resumptionHandle) {
111
+ liveConfig.sessionResumption = { handle: this.resumptionHandle };
112
+ }
113
+ console.log('[GeminiLiveSession] Connecting with config:', {
114
+ model,
115
+ promptLength: promptText.length,
116
+ toolCount: tools?.length ?? 0,
117
+ toolNames: tools?.[0]?.functionDeclarations?.map((d) => d.name) ?? [],
118
+ voice: agent.voice,
119
+ hasResumption: !!this.resumptionHandle,
120
+ });
121
+ this.session = await this.ai.live.connect({
122
+ model,
123
+ config: liveConfig,
124
+ callbacks: {
125
+ onopen: () => {
126
+ this._connected = true;
127
+ console.log('[GeminiLiveSession] Connection opened');
128
+ },
129
+ onmessage: (message) => {
130
+ this.handleServerMessage(message);
131
+ },
132
+ onerror: (error) => {
133
+ const errMsg = String(error);
134
+ this.config.onEvent({ type: 'error', error: errMsg });
135
+ this.emitEvent('error', errMsg);
136
+ },
137
+ onclose: () => {
138
+ this._connected = false;
139
+ console.log(`[GeminiLiveSession] Connection closed (reconfiguring=${this._reconfiguring})`);
140
+ // Don't emit 'disconnected' during a reconfigure cycle — the session
141
+ // will reconnect immediately. Only emit for genuine connection loss.
142
+ if (!this._reconfiguring) {
143
+ this.emitEvent('disconnected');
144
+ }
145
+ },
146
+ },
147
+ });
148
+ }
149
+ async disconnect() {
150
+ if (this.session) {
151
+ try {
152
+ await this.session.close();
153
+ }
154
+ catch {
155
+ // Ignore close errors
156
+ }
157
+ this._connected = false;
158
+ this.session = null;
159
+ }
160
+ }
161
+ /**
162
+ * Send a raw PCM audio frame to Gemini Live.
163
+ * Input: Uint8Array of 16-bit PCM samples at 24kHz.
164
+ */
165
+ sendAudio(frame) {
166
+ if (!this.session || !this._connected)
167
+ return;
168
+ const base64 = Buffer.from(frame).toString('base64');
169
+ this.session.sendRealtimeInput({
170
+ audio: {
171
+ data: base64,
172
+ mimeType: `audio/pcm;rate=${SAMPLE_RATE}`,
173
+ },
174
+ });
175
+ }
176
+ /**
177
+ * Send a tool response back to Gemini Live.
178
+ */
179
+ sendToolResponse(responses) {
180
+ if (!this.session)
181
+ return;
182
+ this.session.sendToolResponse({
183
+ functionResponses: responses.map(r => ({
184
+ id: r.id,
185
+ name: r.name,
186
+ response: { output: r.output },
187
+ })),
188
+ });
189
+ }
190
+ /**
191
+ * Update the session config (system prompt and/or tools) by disconnecting
192
+ * and reconnecting with the new overrides. The existing resumption handle is
193
+ * preserved so the conversation context is not lost.
194
+ *
195
+ * Accepts Partial<RealtimeSessionConfig> to satisfy the RealtimeAudioClient
196
+ * interface. Only systemInstruction and tools fields are applied; other fields
197
+ * (voice, model, audio) require a full reconnect via connect().
198
+ */
199
+ async updateConfig(config) {
200
+ this.runtimeOverrides = {
201
+ systemInstruction: config.systemInstruction,
202
+ tools: config.tools,
203
+ };
204
+ // Set reconfiguring flag to suppress the 'disconnected' event during
205
+ // the reconnect cycle. Without this, RealtimeRuntime would interpret the
206
+ // disconnect as a connection loss and terminate the session.
207
+ this._reconfiguring = true;
208
+ try {
209
+ await this.disconnect();
210
+ await this.connect();
211
+ }
212
+ finally {
213
+ this._reconfiguring = false;
214
+ }
215
+ }
216
+ /**
217
+ * Nudge the model to speak after reconfigure. Tries sendClientContent first;
218
+ * falls back to sendRealtimeInput with text for models that restrict client content.
219
+ */
220
+ requestResponse(instruction) {
221
+ if (!this.session || !this._connected)
222
+ return;
223
+ const text = instruction ?? 'Continue from the current flow state now.';
224
+ try {
225
+ if (typeof this.session.sendClientContent === 'function') {
226
+ this.session.sendClientContent({
227
+ turns: [{ role: 'user', parts: [{ text }] }],
228
+ });
229
+ return;
230
+ }
231
+ }
232
+ catch (err) {
233
+ console.error('[GeminiLiveSession] sendClientContent failed, falling back to sendRealtimeInput:', err);
234
+ }
235
+ try {
236
+ this.session.sendRealtimeInput({ text });
237
+ }
238
+ catch (err) {
239
+ console.error('[GeminiLiveSession] requestResponse failed:', err);
240
+ }
241
+ }
242
+ // --- Private ---
243
+ handleServerMessage(message) {
244
+ // Audio output
245
+ if (message.serverContent?.modelTurn?.parts) {
246
+ for (const part of message.serverContent.modelTurn.parts) {
247
+ if (part.inlineData?.data) {
248
+ const audioData = new Uint8Array(Buffer.from(part.inlineData.data, 'base64'));
249
+ this.config.onEvent({ type: 'audio', data: audioData });
250
+ this.emitEvent('audio', audioData);
251
+ }
252
+ if (part.text) {
253
+ this.config.onEvent({ type: 'transcript', text: part.text, role: 'assistant' });
254
+ this.emitEvent('transcript', part.text, 'assistant');
255
+ }
256
+ }
257
+ }
258
+ // Output audio transcription
259
+ if (message.serverContent?.outputTranscription?.text) {
260
+ const text = message.serverContent.outputTranscription.text;
261
+ this.config.onEvent({ type: 'transcript', text, role: 'assistant' });
262
+ this.emitEvent('transcript', text, 'assistant');
263
+ }
264
+ // Input transcription
265
+ if (message.serverContent?.inputTranscription?.text) {
266
+ const text = message.serverContent.inputTranscription.text;
267
+ this.config.onEvent({ type: 'transcript', text, role: 'user' });
268
+ this.emitEvent('transcript', text, 'user');
269
+ }
270
+ // Turn complete
271
+ if (message.serverContent?.turnComplete) {
272
+ this.config.onEvent({ type: 'turn-complete' });
273
+ this.emitEvent('turn-complete');
274
+ }
275
+ // Interrupted
276
+ if (message.serverContent?.interrupted) {
277
+ this.config.onEvent({ type: 'interrupted' });
278
+ this.emitEvent('interrupted');
279
+ }
280
+ // Tool calls
281
+ if (message.toolCall?.functionCalls) {
282
+ for (const fc of message.toolCall.functionCalls) {
283
+ this.config.onEvent({ type: 'tool-call', id: fc.id, name: fc.name, args: fc.args });
284
+ this.emitEvent('tool-call', fc.id, fc.name, fc.args);
285
+ }
286
+ }
287
+ // Session resumption — internal only, no RealtimeEventMap equivalent
288
+ if (message.sessionResumptionUpdate?.newHandle) {
289
+ this.resumptionHandle = message.sessionResumptionUpdate.newHandle;
290
+ this.config.onEvent({
291
+ type: 'session-resumed',
292
+ newHandle: this.resumptionHandle,
293
+ });
294
+ }
295
+ }
296
+ }
297
+ //# sourceMappingURL=GeminiLiveSession.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"GeminiLiveSession.js","sourceRoot":"","sources":["../src/GeminiLiveSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAQjE,MAAM,aAAa,GAAG,+BAA+B,CAAC;AACtD,MAAM,WAAW,GAAG,KAAK,CAAC;AAiB1B;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IACpB,EAAE,CAAmC;IACrC,OAAO,GAAQ,IAAI,CAAC;IACpB,MAAM,CAA0B;IAChC,gBAAgB,CAAU;IAC1B,UAAU,GAAG,KAAK,CAAC;IAC3B,oFAAoF;IAC5E,cAAc,GAAG,KAAK,CAAC;IAC/B,iFAAiF;IACzE,gBAAgB,CAAuE;IAC/F,2EAA2E;IACnE,SAAS,GAAmE,IAAI,GAAG,EAAE,CAAC;IAE9F,YAAY,MAA+B;QACzC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,+EAA+E;IAE/E,EAAE,CAAmC,KAAQ,EAAE,OAA4B;QACzE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,GAAG,CAAC,OAAuC,CAAC,CAAC;IAC1E,CAAC;IAED,GAAG,CAAmC,KAAQ,EAAE,OAA4B;QAC1E,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,OAAuC,CAAC,CAAC;IAC7E,CAAC;IAEO,SAAS,CACf,KAAQ,EACR,GAAG,IAAqC;QAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAI,IAAkB,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,OAAO,CAAC,MAA8B;QAC1C,+EAA+E;QAC/E,4EAA4E;QAC5E,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,gBAAgB,GAAG;gBACtB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC;QAE/D,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE/E,6EAA6E;QAC7E,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK;eACrC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,2BAA2B,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;YACnD,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,YAAY,EAAE,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC;QAEd,yEAAyE;QACzE,MAAM,UAAU,GAAG,eAAe,CAAC,iBAAiB,IAAI,CAAC,GAAG,EAAE;YAC5D,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC;YACjC,IAAI,CAAC,WAAW;gBAAE,OAAO,EAAE,CAAC;YAC5B,IAAI,OAAO,WAAW,KAAK,QAAQ;gBAAE,OAAO,WAAW,CAAC;YACxD,8CAA8C;YAC9C,OAAQ,WAAwD,CAAC,QAAQ;iBACtE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,UAAU,GAA4B;YAC1C,kBAAkB,EAAE,CAAC,OAAO,CAAC;YAC7B,iBAAiB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE;YACpD,KAAK;YACL,wBAAwB,EAAE,EAAE;SAC7B,CAAC;QAEF,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,UAAU,CAAC,YAAY,GAAG;gBACxB,WAAW,EAAE,EAAE,mBAAmB,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE;aACjE,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,UAAU,CAAC,iBAAiB,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACnE,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,6CAA6C,EAAE;YACzD,KAAK;YACL,YAAY,EAAE,UAAU,CAAC,MAAM;YAC/B,SAAS,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC7B,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE;YAC1E,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACxC,KAAK;YACL,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE;gBACT,MAAM,EAAE,GAAG,EAAE;oBACX,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;gBACvD,CAAC;gBACD,SAAS,EAAE,CAAC,OAAY,EAAE,EAAE;oBAC1B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBACpC,CAAC;gBACD,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;oBACtB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;oBACtD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAClC,CAAC;gBACD,OAAO,EAAE,GAAG,EAAE;oBACZ,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;oBACxB,OAAO,CAAC,GAAG,CAAC,wDAAwD,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;oBAC5F,qEAAqE;oBACrE,qEAAqE;oBACrE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;wBACzB,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;oBACjC,CAAC;gBACH,CAAC;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,KAAiB;QACzB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;YAC7B,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,kBAAkB,WAAW,EAAE;aAC1C;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,SAAiC;QAChD,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC5B,iBAAiB,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;aAC/B,CAAC,CAAC;SACJ,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CAAC,MAAsC;QACvD,IAAI,CAAC,gBAAgB,GAAG;YACtB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;QACF,qEAAqE;QACrE,yEAAyE;QACzE,6DAA6D;QAC7D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC9B,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,WAAoB;QAClC,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC9C,MAAM,IAAI,GAAG,WAAW,IAAI,2CAA2C,CAAC;QACxE,IAAI,CAAC;YACH,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,KAAK,UAAU,EAAE,CAAC;gBACzD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;oBAC7B,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;iBAC7C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,kFAAkF,EAAE,GAAG,CAAC,CAAC;QACzG,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,kBAAkB;IAEV,mBAAmB,CAAC,OAAY;QACtC,eAAe;QACf,IAAI,OAAO,CAAC,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;gBACzD,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC9E,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;oBACxD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBACrC,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;oBAChF,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAO,CAAC,aAAa,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,GAAW,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,IAAI,CAAC;YACpE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YACrE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAClD,CAAC;QAED,sBAAsB;QACtB,IAAI,OAAO,CAAC,aAAa,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YACpD,MAAM,IAAI,GAAW,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,IAAI,CAAC;YACnE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,CAAC,aAAa,EAAE,YAAY,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QAED,cAAc;QACd,IAAI,OAAO,CAAC,aAAa,EAAE,WAAW,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;YAC7C,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAChC,CAAC;QAED,aAAa;QACb,IAAI,OAAO,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC;YACpC,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpF,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,IAAI,OAAO,CAAC,uBAAuB,EAAE,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,uBAAuB,CAAC,SAAS,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAClB,IAAI,EAAE,iBAAiB;gBACvB,SAAS,EAAE,IAAI,CAAC,gBAAiB;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,47 @@
1
+ import type { RealtimeAudioClient } from '@ariaflowagents/core/realtime';
2
+ import type { WorkerLike, TransportSession, VoiceAgentConfig } from './types.js';
3
+ import type { RealtimeRuntime } from '@ariaflowagents/core/realtime';
4
+ /**
5
+ * Factory function that creates a RealtimeAudioClient for a given agent.
6
+ * This is the provider abstraction point: Gemini, OpenAI, or any future
7
+ * provider just needs a factory that returns a valid RealtimeAudioClient.
8
+ */
9
+ export type ModelClientFactory = (agent: VoiceAgentConfig) => RealtimeAudioClient;
10
+ export interface RealtimeCallWorkerConfig {
11
+ callId: string;
12
+ sessionId: string;
13
+ userId?: string;
14
+ agent: VoiceAgentConfig;
15
+ transport: TransportSession;
16
+ runtime: RealtimeRuntime;
17
+ /** Factory that creates the provider model client (Gemini, OpenAI, etc.). */
18
+ createModelClient: ModelClientFactory;
19
+ }
20
+ /**
21
+ * Thin, provider-agnostic wrapper that bridges VoiceEngine's WorkerLike
22
+ * interface with the core RealtimeRuntime.
23
+ *
24
+ * This worker owns ZERO orchestration logic. All semantic behavior
25
+ * (hooks, persistence, extraction, memory, capability routing) is
26
+ * handled by the OrchestrationAuthority composed inside RealtimeRuntime.
27
+ *
28
+ * The worker only owns:
29
+ * 1. Creating a model client via the injected factory
30
+ * 2. Calling runtime.startSession() with the client + transport
31
+ * 3. Exposing start()/stop() for VoiceEngine lifecycle
32
+ */
33
+ export declare class RealtimeCallWorker implements WorkerLike {
34
+ private config;
35
+ private sessionHandle;
36
+ constructor(config: RealtimeCallWorkerConfig);
37
+ get callId(): string;
38
+ /**
39
+ * Start the call: create model client via factory, delegate to RealtimeRuntime.
40
+ */
41
+ start(): Promise<void>;
42
+ /**
43
+ * Stop the call: delegate to RealtimeRuntime.
44
+ */
45
+ stop(): Promise<void>;
46
+ }
47
+ //# sourceMappingURL=RealtimeCallWorker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RealtimeCallWorker.d.ts","sourceRoot":"","sources":["../src/RealtimeCallWorker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAyB,MAAM,+BAA+B,CAAC;AAChG,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAErE;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,mBAAmB,CAAC;AAElF,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,gBAAgB,CAAC;IACxB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,OAAO,EAAE,eAAe,CAAC;IACzB,6EAA6E;IAC7E,iBAAiB,EAAE,kBAAkB,CAAC;CACvC;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,kBAAmB,YAAW,UAAU;IACnD,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,aAAa,CAAsC;gBAE/C,MAAM,EAAE,wBAAwB;IAI5C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAM5B"}
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Thin, provider-agnostic wrapper that bridges VoiceEngine's WorkerLike
3
+ * interface with the core RealtimeRuntime.
4
+ *
5
+ * This worker owns ZERO orchestration logic. All semantic behavior
6
+ * (hooks, persistence, extraction, memory, capability routing) is
7
+ * handled by the OrchestrationAuthority composed inside RealtimeRuntime.
8
+ *
9
+ * The worker only owns:
10
+ * 1. Creating a model client via the injected factory
11
+ * 2. Calling runtime.startSession() with the client + transport
12
+ * 3. Exposing start()/stop() for VoiceEngine lifecycle
13
+ */
14
+ export class RealtimeCallWorker {
15
+ config;
16
+ sessionHandle = null;
17
+ constructor(config) {
18
+ this.config = config;
19
+ }
20
+ get callId() {
21
+ return this.config.callId;
22
+ }
23
+ /**
24
+ * Start the call: create model client via factory, delegate to RealtimeRuntime.
25
+ */
26
+ async start() {
27
+ const { runtime, transport, agent, sessionId, userId, createModelClient } = this.config;
28
+ // Create the provider model client via the injected factory.
29
+ // The factory returns a RealtimeAudioClient (Gemini, OpenAI, etc.).
30
+ const modelClient = createModelClient(agent);
31
+ // Delegate to RealtimeRuntime — this is where all orchestration happens.
32
+ this.sessionHandle = await runtime.startSession({
33
+ modelClient,
34
+ transport: {
35
+ sendAudio: (data) => transport.sendAudio(data),
36
+ onAudio: (handler) => transport.onAudio(handler),
37
+ onClose: (handler) => transport.onClose(handler),
38
+ close: () => transport.close(),
39
+ },
40
+ sessionId,
41
+ userId,
42
+ agentId: agent.id,
43
+ });
44
+ }
45
+ /**
46
+ * Stop the call: delegate to RealtimeRuntime.
47
+ */
48
+ async stop() {
49
+ if (this.sessionHandle) {
50
+ await this.sessionHandle.stop();
51
+ this.sessionHandle = null;
52
+ }
53
+ }
54
+ }
55
+ //# sourceMappingURL=RealtimeCallWorker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"RealtimeCallWorker.js","sourceRoot":"","sources":["../src/RealtimeCallWorker.ts"],"names":[],"mappings":"AAsBA;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAA2B;IACjC,aAAa,GAAiC,IAAI,CAAC;IAE3D,YAAY,MAAgC;QAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAExF,6DAA6D;QAC7D,oEAAoE;QACpE,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAE7C,yEAAyE;QACzE,IAAI,CAAC,aAAa,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC;YAC9C,WAAW;YACX,SAAS,EAAE;gBACT,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC9C,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;gBAChD,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;gBAChD,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE;aAC/B;YACD,SAAS;YACT,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,EAAE;SAClB,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,67 @@
1
+ import type { AgentConfig } from '@ariaflowagents/core/types';
2
+ import type { VoiceEngineConfig, VoiceAgentConfig, AcceptCallParams, WorkerLike } from './types.js';
3
+ /**
4
+ * VoiceEngine — provider-agnostic call acceptor backed by the core
5
+ * RealtimeRuntime and OrchestrationAuthority.
6
+ *
7
+ * Supports any realtime provider (Gemini, OpenAI, custom) through the
8
+ * `createModelClient` factory config. Defaults to Gemini when `gemini`
9
+ * config is provided and no custom factory is set.
10
+ *
11
+ * Architecture:
12
+ * 1. VoiceEngine constructs a DefaultOrchestrationAuthority (core's brain)
13
+ * 2. VoiceEngine constructs a RealtimeRuntime (core's realtime facade)
14
+ * 3. acceptCall() creates a thin RealtimeCallWorker that delegates
15
+ * to RealtimeRuntime.startSession()
16
+ *
17
+ * All voice sessions get the FULL set of core semantics:
18
+ * - Hook lifecycle (onStart, onEnd, onToolResult, onHandoff, etc.)
19
+ * - Persistence with retry and onPersistenceError hook
20
+ * - Post-turn extraction
21
+ * - Memory ingestion
22
+ * - CapabilityHost routing (flows, handoffs, extraction, guardrails)
23
+ *
24
+ * Usage (Gemini — default):
25
+ * ```typescript
26
+ * const engine = new VoiceEngine({ foundation, agents, defaultAgentId, gemini });
27
+ * ```
28
+ *
29
+ * Usage (OpenAI Realtime):
30
+ * ```typescript
31
+ * const engine = new VoiceEngine({
32
+ * foundation, agents, defaultAgentId,
33
+ * createModelClient: () => new OpenAIRealtimeClient({ apiKey: '...' }),
34
+ * });
35
+ * ```
36
+ */
37
+ export declare class VoiceEngine {
38
+ private agents;
39
+ private foundation;
40
+ private config;
41
+ private activeWorkers;
42
+ private realtimeRuntime;
43
+ private modelClientFactory;
44
+ constructor(config: VoiceEngineConfig);
45
+ /**
46
+ * Accept an incoming call and create a RealtimeCallWorker.
47
+ * The worker is NOT started automatically — call `worker.start()`.
48
+ */
49
+ acceptCall(params: AcceptCallParams): Promise<WorkerLike>;
50
+ /**
51
+ * Get an active call worker by call ID.
52
+ */
53
+ getWorker(callId: string): WorkerLike | undefined;
54
+ /**
55
+ * Stop and remove a call worker.
56
+ */
57
+ endCall(callId: string): Promise<void>;
58
+ /**
59
+ * Stop all active calls.
60
+ */
61
+ shutdown(): Promise<void>;
62
+ }
63
+ /**
64
+ * Convert VoiceAgentConfig to a shape compatible with core AgentConfig.
65
+ */
66
+ export declare function voiceAgentToCoreAgent(agent: VoiceAgentConfig): AgentConfig;
67
+ //# sourceMappingURL=VoiceEngine.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VoiceEngine.d.ts","sourceRoot":"","sources":["../src/VoiceEngine.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAE9D,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAQpG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAuC;IACrD,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,aAAa,CAAiC;IACtD,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,kBAAkB,CAAmD;gBAEjE,MAAM,EAAE,iBAAiB;IAgDrC;;;OAGG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC;IA4B/D;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIjD;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ5C;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAIhC;AAID;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,CAU1E"}
@@ -0,0 +1,156 @@
1
+ import crypto from 'node:crypto';
2
+ import { DefaultOrchestrationAuthority } from '@ariaflowagents/core/orchestration';
3
+ import { RealtimeRuntime } from '@ariaflowagents/core/realtime';
4
+ import { RealtimeCallWorker } from './RealtimeCallWorker.js';
5
+ import { GeminiLiveSession } from './GeminiLiveSession.js';
6
+ /**
7
+ * VoiceEngine — provider-agnostic call acceptor backed by the core
8
+ * RealtimeRuntime and OrchestrationAuthority.
9
+ *
10
+ * Supports any realtime provider (Gemini, OpenAI, custom) through the
11
+ * `createModelClient` factory config. Defaults to Gemini when `gemini`
12
+ * config is provided and no custom factory is set.
13
+ *
14
+ * Architecture:
15
+ * 1. VoiceEngine constructs a DefaultOrchestrationAuthority (core's brain)
16
+ * 2. VoiceEngine constructs a RealtimeRuntime (core's realtime facade)
17
+ * 3. acceptCall() creates a thin RealtimeCallWorker that delegates
18
+ * to RealtimeRuntime.startSession()
19
+ *
20
+ * All voice sessions get the FULL set of core semantics:
21
+ * - Hook lifecycle (onStart, onEnd, onToolResult, onHandoff, etc.)
22
+ * - Persistence with retry and onPersistenceError hook
23
+ * - Post-turn extraction
24
+ * - Memory ingestion
25
+ * - CapabilityHost routing (flows, handoffs, extraction, guardrails)
26
+ *
27
+ * Usage (Gemini — default):
28
+ * ```typescript
29
+ * const engine = new VoiceEngine({ foundation, agents, defaultAgentId, gemini });
30
+ * ```
31
+ *
32
+ * Usage (OpenAI Realtime):
33
+ * ```typescript
34
+ * const engine = new VoiceEngine({
35
+ * foundation, agents, defaultAgentId,
36
+ * createModelClient: () => new OpenAIRealtimeClient({ apiKey: '...' }),
37
+ * });
38
+ * ```
39
+ */
40
+ export class VoiceEngine {
41
+ agents = new Map();
42
+ foundation;
43
+ config;
44
+ activeWorkers = new Map();
45
+ realtimeRuntime;
46
+ modelClientFactory;
47
+ constructor(config) {
48
+ this.config = config;
49
+ this.foundation = config.foundation;
50
+ for (const agent of config.agents) {
51
+ this.agents.set(agent.id, agent);
52
+ }
53
+ // Resolve the model client factory:
54
+ // 1. Custom factory takes priority (OpenAI, custom providers)
55
+ // 2. Fall back to Gemini factory when gemini config is provided
56
+ // 3. Error if neither is provided
57
+ if (config.createModelClient) {
58
+ this.modelClientFactory = config.createModelClient;
59
+ }
60
+ else if (config.gemini) {
61
+ const geminiConfig = config.gemini;
62
+ this.modelClientFactory = (agent) => {
63
+ return new GeminiLiveSession({
64
+ gemini: geminiConfig,
65
+ agent,
66
+ onEvent: () => { }, // Events go through the on() interface
67
+ });
68
+ };
69
+ }
70
+ else {
71
+ throw new Error('VoiceEngine: either "gemini" config or "createModelClient" factory is required');
72
+ }
73
+ // Build the core authority — this is the single brain for all voice sessions.
74
+ const authority = new DefaultOrchestrationAuthority({
75
+ agents: config.agents.map(voiceAgentToCoreAgent),
76
+ defaultAgentId: config.defaultAgentId,
77
+ hooks: config.hooks,
78
+ memoryService: config.memoryService,
79
+ memoryIngestion: config.memoryIngestion,
80
+ extractionModel: config.extractionModel,
81
+ autoRetrieveProvider: config.autoRetrieveProvider,
82
+ });
83
+ // Build the realtime facade — provider-agnostic event loop over authority.
84
+ this.realtimeRuntime = new RealtimeRuntime({
85
+ agents: config.agents.map(voiceAgentToCoreAgent),
86
+ defaultAgentId: config.defaultAgentId,
87
+ authority,
88
+ });
89
+ }
90
+ /**
91
+ * Accept an incoming call and create a RealtimeCallWorker.
92
+ * The worker is NOT started automatically — call `worker.start()`.
93
+ */
94
+ async acceptCall(params) {
95
+ const agentId = params.agentId ?? this.config.defaultAgentId;
96
+ const agent = this.agents.get(agentId);
97
+ if (!agent) {
98
+ throw new Error(`VoiceEngine: agent "${agentId}" not found`);
99
+ }
100
+ const callId = params.callId ?? crypto.randomUUID();
101
+ const sessionId = params.sessionId ?? `voice-${callId}`;
102
+ // Create a thin worker that delegates to RealtimeRuntime.
103
+ // The model client factory is provider-agnostic — same worker
104
+ // works for Gemini, OpenAI, or any future provider.
105
+ const worker = new RealtimeCallWorker({
106
+ callId,
107
+ sessionId,
108
+ userId: params.userId,
109
+ agent,
110
+ transport: params.transport,
111
+ runtime: this.realtimeRuntime,
112
+ createModelClient: this.modelClientFactory,
113
+ });
114
+ this.activeWorkers.set(callId, worker);
115
+ return worker;
116
+ }
117
+ /**
118
+ * Get an active call worker by call ID.
119
+ */
120
+ getWorker(callId) {
121
+ return this.activeWorkers.get(callId);
122
+ }
123
+ /**
124
+ * Stop and remove a call worker.
125
+ */
126
+ async endCall(callId) {
127
+ const worker = this.activeWorkers.get(callId);
128
+ if (worker) {
129
+ await worker.stop();
130
+ this.activeWorkers.delete(callId);
131
+ }
132
+ }
133
+ /**
134
+ * Stop all active calls.
135
+ */
136
+ async shutdown() {
137
+ const promises = Array.from(this.activeWorkers.keys()).map(id => this.endCall(id));
138
+ await Promise.allSettled(promises);
139
+ }
140
+ }
141
+ // ─── Helpers ────────────────────────────────────────────────────────────────
142
+ /**
143
+ * Convert VoiceAgentConfig to a shape compatible with core AgentConfig.
144
+ */
145
+ export function voiceAgentToCoreAgent(agent) {
146
+ return {
147
+ id: agent.id,
148
+ name: agent.name,
149
+ description: agent.description,
150
+ prompt: agent.prompt,
151
+ type: agent.flow ? 'flow' : 'llm',
152
+ tools: agent.tools,
153
+ ...(agent.flow ? { flow: agent.flow, initialNode: agent.initialNode ?? agent.flow.nodes[0]?.id } : {}),
154
+ };
155
+ }
156
+ //# sourceMappingURL=VoiceEngine.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VoiceEngine.js","sourceRoot":"","sources":["../src/VoiceEngine.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAMjC,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,OAAO,WAAW;IACd,MAAM,GAAG,IAAI,GAAG,EAA4B,CAAC;IAC7C,UAAU,CAAa;IACvB,MAAM,CAAoB;IAC1B,aAAa,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,eAAe,CAAkB;IACjC,kBAAkB,CAAmD;IAE7E,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QAEpC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,oCAAoC;QACpC,8DAA8D;QAC9D,gEAAgE;QAChE,kCAAkC;QAClC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7B,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACrD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;YACnC,IAAI,CAAC,kBAAkB,GAAG,CAAC,KAAuB,EAAE,EAAE;gBACpD,OAAO,IAAI,iBAAiB,CAAC;oBAC3B,MAAM,EAAE,YAAY;oBACpB,KAAK;oBACL,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,uCAAuC;iBAC3D,CAAC,CAAC;YACL,CAAC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CACb,gFAAgF,CACjF,CAAC;QACJ,CAAC;QAED,8EAA8E;QAC9E,MAAM,SAAS,GAA2B,IAAI,6BAA6B,CAAC;YAC1E,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC;YAChD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,oBAAoB,EAAE,MAAM,CAAC,oBAAoB;SAClD,CAAC,CAAC;QAEH,2EAA2E;QAC3E,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CAAC;YACzC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC;YAChD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,MAAwB;QACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,uBAAuB,OAAO,aAAa,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,MAAM,EAAE,CAAC;QAExD,0DAA0D;QAC1D,8DAA8D;QAC9D,oDAAoD;QACpD,MAAM,MAAM,GAAe,IAAI,kBAAkB,CAAC;YAChD,MAAM;YACN,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK;YACL,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,IAAI,CAAC,eAAe;YAC7B,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;SAC3C,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,MAAc;QACtB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;CACF;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAuB;IAC3D,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACjC,KAAK,EAAE,KAAK,CAAC,KAAY;QACzB,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxF,CAAC;AACnB,CAAC"}