@ariaflowagents/gemini-native-audio 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -0
- package/dist/CallWorker.d.ts +56 -0
- package/dist/CallWorker.d.ts.map +1 -0
- package/dist/CallWorker.js +172 -0
- package/dist/CallWorker.js.map +1 -0
- package/dist/CapabilityCallWorker.d.ts +46 -0
- package/dist/CapabilityCallWorker.d.ts.map +1 -0
- package/dist/CapabilityCallWorker.js +319 -0
- package/dist/CapabilityCallWorker.js.map +1 -0
- package/dist/GeminiLiveSession.d.ts +86 -0
- package/dist/GeminiLiveSession.d.ts.map +1 -0
- package/dist/GeminiLiveSession.js +297 -0
- package/dist/GeminiLiveSession.js.map +1 -0
- package/dist/RealtimeCallWorker.d.ts +47 -0
- package/dist/RealtimeCallWorker.d.ts.map +1 -0
- package/dist/RealtimeCallWorker.js +55 -0
- package/dist/RealtimeCallWorker.js.map +1 -0
- package/dist/VoiceEngine.d.ts +67 -0
- package/dist/VoiceEngine.d.ts.map +1 -0
- package/dist/VoiceEngine.js +156 -0
- package/dist/VoiceEngine.js.map +1 -0
- package/dist/factories.d.ts +32 -0
- package/dist/factories.d.ts.map +1 -0
- package/dist/factories.js +43 -0
- package/dist/factories.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/openai/OpenAIRealtimeClient.d.ts +51 -0
- package/dist/openai/OpenAIRealtimeClient.d.ts.map +1 -0
- package/dist/openai/OpenAIRealtimeClient.js +327 -0
- package/dist/openai/OpenAIRealtimeClient.js.map +1 -0
- package/dist/openai/index.d.ts +3 -0
- package/dist/openai/index.d.ts.map +1 -0
- package/dist/openai/index.js +2 -0
- package/dist/openai/index.js.map +1 -0
- package/dist/schema-bridge.d.ts +14 -0
- package/dist/schema-bridge.d.ts.map +1 -0
- package/dist/schema-bridge.js +20 -0
- package/dist/schema-bridge.js.map +1 -0
- package/dist/types.d.ts +150 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +44 -0
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
import { GoogleGenAI } from '@google/genai';
|
|
2
|
+
import { toolSetToGeminiDeclarations } from './schema-bridge.js';
|
|
3
|
+
const DEFAULT_MODEL = 'gemini-3.1-flash-live-preview';
|
|
4
|
+
const SAMPLE_RATE = 24000;
|
|
5
|
+
/**
|
|
6
|
+
* Thin wrapper around @google/genai ai.live.connect().
|
|
7
|
+
*
|
|
8
|
+
* Manages:
|
|
9
|
+
* - Connection lifecycle (connect/disconnect)
|
|
10
|
+
* - Audio input/output encoding (base64 PCM ↔ Uint8Array)
|
|
11
|
+
* - Tool call dispatch → onEvent callback
|
|
12
|
+
* - Session resumption via newHandle
|
|
13
|
+
*/
|
|
14
|
+
export class GeminiLiveSession {
|
|
15
|
+
ai;
|
|
16
|
+
session = null;
|
|
17
|
+
config;
|
|
18
|
+
resumptionHandle;
|
|
19
|
+
_connected = false;
|
|
20
|
+
/** True during updateConfig() reconnect cycle — suppresses 'disconnected' event. */
|
|
21
|
+
_reconfiguring = false;
|
|
22
|
+
/** Mutable overrides updated by updateConfig(). Merged with config.overrides. */
|
|
23
|
+
runtimeOverrides;
|
|
24
|
+
/** Typed event listener registry for the RealtimeAudioClient interface. */
|
|
25
|
+
listeners = new Map();
|
|
26
|
+
constructor(config) {
|
|
27
|
+
this.config = config;
|
|
28
|
+
this.ai = new GoogleGenAI({ apiKey: config.gemini.apiKey });
|
|
29
|
+
}
|
|
30
|
+
get connected() {
|
|
31
|
+
return this._connected;
|
|
32
|
+
}
|
|
33
|
+
// ─── RealtimeAudioClient: on / off / ping ───────────────────────────────────
|
|
34
|
+
on(event, handler) {
|
|
35
|
+
if (!this.listeners.has(event)) {
|
|
36
|
+
this.listeners.set(event, new Set());
|
|
37
|
+
}
|
|
38
|
+
this.listeners.get(event).add(handler);
|
|
39
|
+
}
|
|
40
|
+
off(event, handler) {
|
|
41
|
+
this.listeners.get(event)?.delete(handler);
|
|
42
|
+
}
|
|
43
|
+
emitEvent(event, ...args) {
|
|
44
|
+
const handlers = this.listeners.get(event);
|
|
45
|
+
if (!handlers)
|
|
46
|
+
return;
|
|
47
|
+
for (const handler of handlers) {
|
|
48
|
+
handler(...args);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Gemini Live wraps the underlying WebSocket via @google/genai SDK.
|
|
53
|
+
* There is no direct ping API, so we return connection state as a health check.
|
|
54
|
+
*/
|
|
55
|
+
async ping() {
|
|
56
|
+
return Promise.resolve(this._connected);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Connect to Gemini Live.
|
|
60
|
+
*
|
|
61
|
+
* @param config Optional RealtimeSessionConfig. When provided, systemInstruction
|
|
62
|
+
* and tools are applied as runtimeOverrides (takes precedence over constructor
|
|
63
|
+
* config.overrides and agent config). This satisfies the RealtimeAudioClient
|
|
64
|
+
* interface while preserving backward compatibility with CapabilityCallWorker,
|
|
65
|
+
* which calls connect() with no arguments.
|
|
66
|
+
*/
|
|
67
|
+
async connect(config) {
|
|
68
|
+
// If a RealtimeSessionConfig is provided (RealtimeAudioClient interface call),
|
|
69
|
+
// store its fields as runtimeOverrides so they win over constructor config.
|
|
70
|
+
if (config) {
|
|
71
|
+
this.runtimeOverrides = {
|
|
72
|
+
systemInstruction: config.systemInstruction,
|
|
73
|
+
tools: config.tools,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
const { agent, gemini } = this.config;
|
|
77
|
+
const model = (config?.model ?? gemini.model) ?? DEFAULT_MODEL;
|
|
78
|
+
// Merge overrides: runtimeOverrides (from updateConfig) take priority over
|
|
79
|
+
// config.overrides (from constructor), which take priority over agent config.
|
|
80
|
+
const activeOverrides = { ...this.config.overrides, ...this.runtimeOverrides };
|
|
81
|
+
// Gemini 3.1 rejects empty tools arrays — only send when declarations exist.
|
|
82
|
+
const declarations = activeOverrides.tools
|
|
83
|
+
?? (agent.tools ? toolSetToGeminiDeclarations(agent.tools) : undefined);
|
|
84
|
+
const tools = declarations && declarations.length > 0
|
|
85
|
+
? [{ functionDeclarations: declarations }]
|
|
86
|
+
: undefined;
|
|
87
|
+
// Resolve prompt — active overrides win, then fall back to agent config.
|
|
88
|
+
const promptText = activeOverrides.systemInstruction ?? (() => {
|
|
89
|
+
const agentPrompt = agent.prompt;
|
|
90
|
+
if (!agentPrompt)
|
|
91
|
+
return '';
|
|
92
|
+
if (typeof agentPrompt === 'string')
|
|
93
|
+
return agentPrompt;
|
|
94
|
+
// PromptTemplate / AgentPrompt: join sections
|
|
95
|
+
return agentPrompt.sections
|
|
96
|
+
.map(s => s.content)
|
|
97
|
+
.join('\n\n');
|
|
98
|
+
})();
|
|
99
|
+
const liveConfig = {
|
|
100
|
+
responseModalities: ['AUDIO'],
|
|
101
|
+
systemInstruction: { parts: [{ text: promptText }] },
|
|
102
|
+
tools,
|
|
103
|
+
outputAudioTranscription: {},
|
|
104
|
+
};
|
|
105
|
+
if (agent.voice) {
|
|
106
|
+
liveConfig.speechConfig = {
|
|
107
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: agent.voice } },
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
if (this.resumptionHandle) {
|
|
111
|
+
liveConfig.sessionResumption = { handle: this.resumptionHandle };
|
|
112
|
+
}
|
|
113
|
+
console.log('[GeminiLiveSession] Connecting with config:', {
|
|
114
|
+
model,
|
|
115
|
+
promptLength: promptText.length,
|
|
116
|
+
toolCount: tools?.length ?? 0,
|
|
117
|
+
toolNames: tools?.[0]?.functionDeclarations?.map((d) => d.name) ?? [],
|
|
118
|
+
voice: agent.voice,
|
|
119
|
+
hasResumption: !!this.resumptionHandle,
|
|
120
|
+
});
|
|
121
|
+
this.session = await this.ai.live.connect({
|
|
122
|
+
model,
|
|
123
|
+
config: liveConfig,
|
|
124
|
+
callbacks: {
|
|
125
|
+
onopen: () => {
|
|
126
|
+
this._connected = true;
|
|
127
|
+
console.log('[GeminiLiveSession] Connection opened');
|
|
128
|
+
},
|
|
129
|
+
onmessage: (message) => {
|
|
130
|
+
this.handleServerMessage(message);
|
|
131
|
+
},
|
|
132
|
+
onerror: (error) => {
|
|
133
|
+
const errMsg = String(error);
|
|
134
|
+
this.config.onEvent({ type: 'error', error: errMsg });
|
|
135
|
+
this.emitEvent('error', errMsg);
|
|
136
|
+
},
|
|
137
|
+
onclose: () => {
|
|
138
|
+
this._connected = false;
|
|
139
|
+
console.log(`[GeminiLiveSession] Connection closed (reconfiguring=${this._reconfiguring})`);
|
|
140
|
+
// Don't emit 'disconnected' during a reconfigure cycle — the session
|
|
141
|
+
// will reconnect immediately. Only emit for genuine connection loss.
|
|
142
|
+
if (!this._reconfiguring) {
|
|
143
|
+
this.emitEvent('disconnected');
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
async disconnect() {
|
|
150
|
+
if (this.session) {
|
|
151
|
+
try {
|
|
152
|
+
await this.session.close();
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
// Ignore close errors
|
|
156
|
+
}
|
|
157
|
+
this._connected = false;
|
|
158
|
+
this.session = null;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Send a raw PCM audio frame to Gemini Live.
|
|
163
|
+
* Input: Uint8Array of 16-bit PCM samples at 24kHz.
|
|
164
|
+
*/
|
|
165
|
+
sendAudio(frame) {
|
|
166
|
+
if (!this.session || !this._connected)
|
|
167
|
+
return;
|
|
168
|
+
const base64 = Buffer.from(frame).toString('base64');
|
|
169
|
+
this.session.sendRealtimeInput({
|
|
170
|
+
audio: {
|
|
171
|
+
data: base64,
|
|
172
|
+
mimeType: `audio/pcm;rate=${SAMPLE_RATE}`,
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Send a tool response back to Gemini Live.
|
|
178
|
+
*/
|
|
179
|
+
sendToolResponse(responses) {
|
|
180
|
+
if (!this.session)
|
|
181
|
+
return;
|
|
182
|
+
this.session.sendToolResponse({
|
|
183
|
+
functionResponses: responses.map(r => ({
|
|
184
|
+
id: r.id,
|
|
185
|
+
name: r.name,
|
|
186
|
+
response: { output: r.output },
|
|
187
|
+
})),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Update the session config (system prompt and/or tools) by disconnecting
|
|
192
|
+
* and reconnecting with the new overrides. The existing resumption handle is
|
|
193
|
+
* preserved so the conversation context is not lost.
|
|
194
|
+
*
|
|
195
|
+
* Accepts Partial<RealtimeSessionConfig> to satisfy the RealtimeAudioClient
|
|
196
|
+
* interface. Only systemInstruction and tools fields are applied; other fields
|
|
197
|
+
* (voice, model, audio) require a full reconnect via connect().
|
|
198
|
+
*/
|
|
199
|
+
async updateConfig(config) {
|
|
200
|
+
this.runtimeOverrides = {
|
|
201
|
+
systemInstruction: config.systemInstruction,
|
|
202
|
+
tools: config.tools,
|
|
203
|
+
};
|
|
204
|
+
// Set reconfiguring flag to suppress the 'disconnected' event during
|
|
205
|
+
// the reconnect cycle. Without this, RealtimeRuntime would interpret the
|
|
206
|
+
// disconnect as a connection loss and terminate the session.
|
|
207
|
+
this._reconfiguring = true;
|
|
208
|
+
try {
|
|
209
|
+
await this.disconnect();
|
|
210
|
+
await this.connect();
|
|
211
|
+
}
|
|
212
|
+
finally {
|
|
213
|
+
this._reconfiguring = false;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Nudge the model to speak after reconfigure. Tries sendClientContent first;
|
|
218
|
+
* falls back to sendRealtimeInput with text for models that restrict client content.
|
|
219
|
+
*/
|
|
220
|
+
requestResponse(instruction) {
|
|
221
|
+
if (!this.session || !this._connected)
|
|
222
|
+
return;
|
|
223
|
+
const text = instruction ?? 'Continue from the current flow state now.';
|
|
224
|
+
try {
|
|
225
|
+
if (typeof this.session.sendClientContent === 'function') {
|
|
226
|
+
this.session.sendClientContent({
|
|
227
|
+
turns: [{ role: 'user', parts: [{ text }] }],
|
|
228
|
+
});
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
catch (err) {
|
|
233
|
+
console.error('[GeminiLiveSession] sendClientContent failed, falling back to sendRealtimeInput:', err);
|
|
234
|
+
}
|
|
235
|
+
try {
|
|
236
|
+
this.session.sendRealtimeInput({ text });
|
|
237
|
+
}
|
|
238
|
+
catch (err) {
|
|
239
|
+
console.error('[GeminiLiveSession] requestResponse failed:', err);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
// --- Private ---
|
|
243
|
+
handleServerMessage(message) {
|
|
244
|
+
// Audio output
|
|
245
|
+
if (message.serverContent?.modelTurn?.parts) {
|
|
246
|
+
for (const part of message.serverContent.modelTurn.parts) {
|
|
247
|
+
if (part.inlineData?.data) {
|
|
248
|
+
const audioData = new Uint8Array(Buffer.from(part.inlineData.data, 'base64'));
|
|
249
|
+
this.config.onEvent({ type: 'audio', data: audioData });
|
|
250
|
+
this.emitEvent('audio', audioData);
|
|
251
|
+
}
|
|
252
|
+
if (part.text) {
|
|
253
|
+
this.config.onEvent({ type: 'transcript', text: part.text, role: 'assistant' });
|
|
254
|
+
this.emitEvent('transcript', part.text, 'assistant');
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
// Output audio transcription
|
|
259
|
+
if (message.serverContent?.outputTranscription?.text) {
|
|
260
|
+
const text = message.serverContent.outputTranscription.text;
|
|
261
|
+
this.config.onEvent({ type: 'transcript', text, role: 'assistant' });
|
|
262
|
+
this.emitEvent('transcript', text, 'assistant');
|
|
263
|
+
}
|
|
264
|
+
// Input transcription
|
|
265
|
+
if (message.serverContent?.inputTranscription?.text) {
|
|
266
|
+
const text = message.serverContent.inputTranscription.text;
|
|
267
|
+
this.config.onEvent({ type: 'transcript', text, role: 'user' });
|
|
268
|
+
this.emitEvent('transcript', text, 'user');
|
|
269
|
+
}
|
|
270
|
+
// Turn complete
|
|
271
|
+
if (message.serverContent?.turnComplete) {
|
|
272
|
+
this.config.onEvent({ type: 'turn-complete' });
|
|
273
|
+
this.emitEvent('turn-complete');
|
|
274
|
+
}
|
|
275
|
+
// Interrupted
|
|
276
|
+
if (message.serverContent?.interrupted) {
|
|
277
|
+
this.config.onEvent({ type: 'interrupted' });
|
|
278
|
+
this.emitEvent('interrupted');
|
|
279
|
+
}
|
|
280
|
+
// Tool calls
|
|
281
|
+
if (message.toolCall?.functionCalls) {
|
|
282
|
+
for (const fc of message.toolCall.functionCalls) {
|
|
283
|
+
this.config.onEvent({ type: 'tool-call', id: fc.id, name: fc.name, args: fc.args });
|
|
284
|
+
this.emitEvent('tool-call', fc.id, fc.name, fc.args);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
// Session resumption — internal only, no RealtimeEventMap equivalent
|
|
288
|
+
if (message.sessionResumptionUpdate?.newHandle) {
|
|
289
|
+
this.resumptionHandle = message.sessionResumptionUpdate.newHandle;
|
|
290
|
+
this.config.onEvent({
|
|
291
|
+
type: 'session-resumed',
|
|
292
|
+
newHandle: this.resumptionHandle,
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
//# sourceMappingURL=GeminiLiveSession.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GeminiLiveSession.js","sourceRoot":"","sources":["../src/GeminiLiveSession.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AAQjE,MAAM,aAAa,GAAG,+BAA+B,CAAC;AACtD,MAAM,WAAW,GAAG,KAAK,CAAC;AAiB1B;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IACpB,EAAE,CAAmC;IACrC,OAAO,GAAQ,IAAI,CAAC;IACpB,MAAM,CAA0B;IAChC,gBAAgB,CAAU;IAC1B,UAAU,GAAG,KAAK,CAAC;IAC3B,oFAAoF;IAC5E,cAAc,GAAG,KAAK,CAAC;IAC/B,iFAAiF;IACzE,gBAAgB,CAAuE;IAC/F,2EAA2E;IACnE,SAAS,GAAmE,IAAI,GAAG,EAAE,CAAC;IAE9F,YAAY,MAA+B;QACzC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,+EAA+E;IAE/E,EAAE,CAAmC,KAAQ,EAAE,OAA4B;QACzE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,GAAG,CAAC,OAAuC,CAAC,CAAC;IAC1E,CAAC;IAED,GAAG,CAAmC,KAAQ,EAAE,OAA4B;QAC1E,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,OAAuC,CAAC,CAAC;IAC7E,CAAC;IAEO,SAAS,CACf,KAAQ,EACR,GAAG,IAAqC;QAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAI,IAAkB,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1C,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,OAAO,CAAC,MAA8B;QAC1C,+EAA+E;QAC/E,4EAA4E;QAC5E,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,gBAAgB,GAAG;gBACtB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;gBAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,aAAa,CAAC;QAE/D,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE/E,6EAA6E;QAC7E,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK;eACrC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,2BAA2B,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;YACnD,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,YAAY,EAAE,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC;QAEd,yEAAyE;QACzE,MAAM,UAAU,GAAG,eAAe,CAAC,iBAAiB,IAAI,CAAC,GAAG,EAAE;YAC5D,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC;YACjC,IAAI,CAAC,WAAW;gBAAE,OAAO,EAAE,CAAC;YAC5B,IAAI,OAAO,WAAW,KAAK,QAAQ;gBAAE,OAAO,WAAW,CAAC;YACxD,8CAA8C;YAC9C,OAAQ,WAAwD,CAAC,QAAQ;iBACtE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;iBACnB,IAAI,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,UAAU,GAA4B;YAC1C,kBAAkB,EAAE,CAAC,OAAO,CAAC;YAC7B,iBAAiB,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE;YACpD,KAAK;YACL,wBAAwB,EAAE,EAAE;SAC7B,CAAC;QAEF,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,UAAU,CAAC,YAAY,GAAG;gBACxB,WAAW,EAAE,EAAE,mBAAmB,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE;aACjE,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,UAAU,CAAC,iBAAiB,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACnE,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,6CAA6C,EAAE;YACzD,KAAK;YACL,YAAY,EAAE,UAAU,CAAC,MAAM;YAC/B,SAAS,EAAE,KAAK,EAAE,MAAM,IAAI,CAAC;YAC7B,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE;YAC1E,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACxC,KAAK;YACL,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE;gBACT,MAAM,EAAE,GAAG,EAAE;oBACX,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;gBACvD,CAAC;gBACD,SAAS,EAAE,CAAC,OAAY,EAAE,EAAE;oBAC1B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBACpC,CAAC;gBACD,OAAO,EAAE,CAAC,KAAU,EAAE,EAAE;oBACtB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC7B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;oBACtD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAClC,CAAC;gBACD,OAAO,EAAE,GAAG,EAAE;oBACZ,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;oBACxB,OAAO,CAAC,GAAG,CAAC,wDAAwD,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;oBAC5F,qEAAqE;oBACrE,qEAAqE;oBACrE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;wBACzB,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;oBACjC,CAAC;gBACH,CAAC;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,KAAiB;QACzB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;YAC7B,KAAK,EAAE;gBACL,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,kBAAkB,WAAW,EAAE;aAC1C;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,SAAiC;QAChD,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC5B,iBAAiB,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACrC,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;aAC/B,CAAC,CAAC;SACJ,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CAAC,MAAsC;QACvD,IAAI,CAAC,gBAAgB,GAAG;YACtB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;QACF,qEAAqE;QACrE,yEAAyE;QACzE,6DAA6D;QAC7D,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC9B,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,WAAoB;QAClC,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC9C,MAAM,IAAI,GAAG,WAAW,IAAI,2CAA2C,CAAC;QACxE,IAAI,CAAC;YACH,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,KAAK,UAAU,EAAE,CAAC;gBACzD,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC;oBAC7B,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;iBAC7C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,kFAAkF,EAAE,GAAG,CAAC,CAAC;QACzG,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,GAAG,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,kBAAkB;IAEV,mBAAmB,CAAC,OAAY;QACtC,eAAe;QACf,IAAI,OAAO,CAAC,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;gBACzD,IAAI,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;oBAC1B,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC9E,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;oBACxD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;gBACrC,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;oBAChF,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAO,CAAC,aAAa,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC;YACrD,MAAM,IAAI,GAAW,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,IAAI,CAAC;YACpE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YACrE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;QAClD,CAAC;QAED,sBAAsB;QACtB,IAAI,OAAO,CAAC,aAAa,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;YACpD,MAAM,IAAI,GAAW,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,IAAI,CAAC;YACnE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,gBAAgB;QAChB,IAAI,OAAO,CAAC,aAAa,EAAE,YAAY,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QAED,cAAc;QACd,IAAI,OAAO,CAAC,aAAa,EAAE,WAAW,EAAE,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;YAC7C,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAChC,CAAC;QAED,aAAa;QACb,IAAI,OAAO,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC;YACpC,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpF,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,IAAI,OAAO,CAAC,uBAAuB,EAAE,SAAS,EAAE,CAAC;YAC/C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,uBAAuB,CAAC,SAAS,CAAC;YAClE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAClB,IAAI,EAAE,iBAAiB;gBACvB,SAAS,EAAE,IAAI,CAAC,gBAAiB;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { RealtimeAudioClient } from '@ariaflowagents/core/realtime';
|
|
2
|
+
import type { WorkerLike, TransportSession, VoiceAgentConfig } from './types.js';
|
|
3
|
+
import type { RealtimeRuntime } from '@ariaflowagents/core/realtime';
|
|
4
|
+
/**
|
|
5
|
+
* Factory function that creates a RealtimeAudioClient for a given agent.
|
|
6
|
+
* This is the provider abstraction point: Gemini, OpenAI, or any future
|
|
7
|
+
* provider just needs a factory that returns a valid RealtimeAudioClient.
|
|
8
|
+
*/
|
|
9
|
+
export type ModelClientFactory = (agent: VoiceAgentConfig) => RealtimeAudioClient;
|
|
10
|
+
export interface RealtimeCallWorkerConfig {
|
|
11
|
+
callId: string;
|
|
12
|
+
sessionId: string;
|
|
13
|
+
userId?: string;
|
|
14
|
+
agent: VoiceAgentConfig;
|
|
15
|
+
transport: TransportSession;
|
|
16
|
+
runtime: RealtimeRuntime;
|
|
17
|
+
/** Factory that creates the provider model client (Gemini, OpenAI, etc.). */
|
|
18
|
+
createModelClient: ModelClientFactory;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Thin, provider-agnostic wrapper that bridges VoiceEngine's WorkerLike
|
|
22
|
+
* interface with the core RealtimeRuntime.
|
|
23
|
+
*
|
|
24
|
+
* This worker owns ZERO orchestration logic. All semantic behavior
|
|
25
|
+
* (hooks, persistence, extraction, memory, capability routing) is
|
|
26
|
+
* handled by the OrchestrationAuthority composed inside RealtimeRuntime.
|
|
27
|
+
*
|
|
28
|
+
* The worker only owns:
|
|
29
|
+
* 1. Creating a model client via the injected factory
|
|
30
|
+
* 2. Calling runtime.startSession() with the client + transport
|
|
31
|
+
* 3. Exposing start()/stop() for VoiceEngine lifecycle
|
|
32
|
+
*/
|
|
33
|
+
export declare class RealtimeCallWorker implements WorkerLike {
|
|
34
|
+
private config;
|
|
35
|
+
private sessionHandle;
|
|
36
|
+
constructor(config: RealtimeCallWorkerConfig);
|
|
37
|
+
get callId(): string;
|
|
38
|
+
/**
|
|
39
|
+
* Start the call: create model client via factory, delegate to RealtimeRuntime.
|
|
40
|
+
*/
|
|
41
|
+
start(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Stop the call: delegate to RealtimeRuntime.
|
|
44
|
+
*/
|
|
45
|
+
stop(): Promise<void>;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=RealtimeCallWorker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RealtimeCallWorker.d.ts","sourceRoot":"","sources":["../src/RealtimeCallWorker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAyB,MAAM,+BAA+B,CAAC;AAChG,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAErE;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,KAAK,EAAE,gBAAgB,KAAK,mBAAmB,CAAC;AAElF,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,gBAAgB,CAAC;IACxB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,OAAO,EAAE,eAAe,CAAC;IACzB,6EAA6E;IAC7E,iBAAiB,EAAE,kBAAkB,CAAC;CACvC;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,kBAAmB,YAAW,UAAU;IACnD,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,aAAa,CAAsC;gBAE/C,MAAM,EAAE,wBAAwB;IAI5C,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAM5B"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thin, provider-agnostic wrapper that bridges VoiceEngine's WorkerLike
|
|
3
|
+
* interface with the core RealtimeRuntime.
|
|
4
|
+
*
|
|
5
|
+
* This worker owns ZERO orchestration logic. All semantic behavior
|
|
6
|
+
* (hooks, persistence, extraction, memory, capability routing) is
|
|
7
|
+
* handled by the OrchestrationAuthority composed inside RealtimeRuntime.
|
|
8
|
+
*
|
|
9
|
+
* The worker only owns:
|
|
10
|
+
* 1. Creating a model client via the injected factory
|
|
11
|
+
* 2. Calling runtime.startSession() with the client + transport
|
|
12
|
+
* 3. Exposing start()/stop() for VoiceEngine lifecycle
|
|
13
|
+
*/
|
|
14
|
+
export class RealtimeCallWorker {
|
|
15
|
+
config;
|
|
16
|
+
sessionHandle = null;
|
|
17
|
+
constructor(config) {
|
|
18
|
+
this.config = config;
|
|
19
|
+
}
|
|
20
|
+
get callId() {
|
|
21
|
+
return this.config.callId;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Start the call: create model client via factory, delegate to RealtimeRuntime.
|
|
25
|
+
*/
|
|
26
|
+
async start() {
|
|
27
|
+
const { runtime, transport, agent, sessionId, userId, createModelClient } = this.config;
|
|
28
|
+
// Create the provider model client via the injected factory.
|
|
29
|
+
// The factory returns a RealtimeAudioClient (Gemini, OpenAI, etc.).
|
|
30
|
+
const modelClient = createModelClient(agent);
|
|
31
|
+
// Delegate to RealtimeRuntime — this is where all orchestration happens.
|
|
32
|
+
this.sessionHandle = await runtime.startSession({
|
|
33
|
+
modelClient,
|
|
34
|
+
transport: {
|
|
35
|
+
sendAudio: (data) => transport.sendAudio(data),
|
|
36
|
+
onAudio: (handler) => transport.onAudio(handler),
|
|
37
|
+
onClose: (handler) => transport.onClose(handler),
|
|
38
|
+
close: () => transport.close(),
|
|
39
|
+
},
|
|
40
|
+
sessionId,
|
|
41
|
+
userId,
|
|
42
|
+
agentId: agent.id,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Stop the call: delegate to RealtimeRuntime.
|
|
47
|
+
*/
|
|
48
|
+
async stop() {
|
|
49
|
+
if (this.sessionHandle) {
|
|
50
|
+
await this.sessionHandle.stop();
|
|
51
|
+
this.sessionHandle = null;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=RealtimeCallWorker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RealtimeCallWorker.js","sourceRoot":"","sources":["../src/RealtimeCallWorker.ts"],"names":[],"mappings":"AAsBA;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAA2B;IACjC,aAAa,GAAiC,IAAI,CAAC;IAE3D,YAAY,MAAgC;QAC1C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAExF,6DAA6D;QAC7D,oEAAoE;QACpE,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAE7C,yEAAyE;QACzE,IAAI,CAAC,aAAa,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC;YAC9C,WAAW;YACX,SAAS,EAAE;gBACT,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC9C,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;gBAChD,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC;gBAChD,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE;aAC/B;YACD,SAAS;YACT,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,EAAE;SAClB,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { AgentConfig } from '@ariaflowagents/core/types';
|
|
2
|
+
import type { VoiceEngineConfig, VoiceAgentConfig, AcceptCallParams, WorkerLike } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* VoiceEngine — provider-agnostic call acceptor backed by the core
|
|
5
|
+
* RealtimeRuntime and OrchestrationAuthority.
|
|
6
|
+
*
|
|
7
|
+
* Supports any realtime provider (Gemini, OpenAI, custom) through the
|
|
8
|
+
* `createModelClient` factory config. Defaults to Gemini when `gemini`
|
|
9
|
+
* config is provided and no custom factory is set.
|
|
10
|
+
*
|
|
11
|
+
* Architecture:
|
|
12
|
+
* 1. VoiceEngine constructs a DefaultOrchestrationAuthority (core's brain)
|
|
13
|
+
* 2. VoiceEngine constructs a RealtimeRuntime (core's realtime facade)
|
|
14
|
+
* 3. acceptCall() creates a thin RealtimeCallWorker that delegates
|
|
15
|
+
* to RealtimeRuntime.startSession()
|
|
16
|
+
*
|
|
17
|
+
* All voice sessions get the FULL set of core semantics:
|
|
18
|
+
* - Hook lifecycle (onStart, onEnd, onToolResult, onHandoff, etc.)
|
|
19
|
+
* - Persistence with retry and onPersistenceError hook
|
|
20
|
+
* - Post-turn extraction
|
|
21
|
+
* - Memory ingestion
|
|
22
|
+
* - CapabilityHost routing (flows, handoffs, extraction, guardrails)
|
|
23
|
+
*
|
|
24
|
+
* Usage (Gemini — default):
|
|
25
|
+
* ```typescript
|
|
26
|
+
* const engine = new VoiceEngine({ foundation, agents, defaultAgentId, gemini });
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* Usage (OpenAI Realtime):
|
|
30
|
+
* ```typescript
|
|
31
|
+
* const engine = new VoiceEngine({
|
|
32
|
+
* foundation, agents, defaultAgentId,
|
|
33
|
+
* createModelClient: () => new OpenAIRealtimeClient({ apiKey: '...' }),
|
|
34
|
+
* });
|
|
35
|
+
* ```
|
|
36
|
+
*/
|
|
37
|
+
export declare class VoiceEngine {
|
|
38
|
+
private agents;
|
|
39
|
+
private foundation;
|
|
40
|
+
private config;
|
|
41
|
+
private activeWorkers;
|
|
42
|
+
private realtimeRuntime;
|
|
43
|
+
private modelClientFactory;
|
|
44
|
+
constructor(config: VoiceEngineConfig);
|
|
45
|
+
/**
|
|
46
|
+
* Accept an incoming call and create a RealtimeCallWorker.
|
|
47
|
+
* The worker is NOT started automatically — call `worker.start()`.
|
|
48
|
+
*/
|
|
49
|
+
acceptCall(params: AcceptCallParams): Promise<WorkerLike>;
|
|
50
|
+
/**
|
|
51
|
+
* Get an active call worker by call ID.
|
|
52
|
+
*/
|
|
53
|
+
getWorker(callId: string): WorkerLike | undefined;
|
|
54
|
+
/**
|
|
55
|
+
* Stop and remove a call worker.
|
|
56
|
+
*/
|
|
57
|
+
endCall(callId: string): Promise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Stop all active calls.
|
|
60
|
+
*/
|
|
61
|
+
shutdown(): Promise<void>;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Convert VoiceAgentConfig to a shape compatible with core AgentConfig.
|
|
65
|
+
*/
|
|
66
|
+
export declare function voiceAgentToCoreAgent(agent: VoiceAgentConfig): AgentConfig;
|
|
67
|
+
//# sourceMappingURL=VoiceEngine.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VoiceEngine.d.ts","sourceRoot":"","sources":["../src/VoiceEngine.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAE9D,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAQpG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAuC;IACrD,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,aAAa,CAAiC;IACtD,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,kBAAkB,CAAmD;gBAEjE,MAAM,EAAE,iBAAiB;IAgDrC;;;OAGG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC;IA4B/D;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAIjD;;OAEG;IACG,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ5C;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAIhC;AAID;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,CAU1E"}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import { DefaultOrchestrationAuthority } from '@ariaflowagents/core/orchestration';
|
|
3
|
+
import { RealtimeRuntime } from '@ariaflowagents/core/realtime';
|
|
4
|
+
import { RealtimeCallWorker } from './RealtimeCallWorker.js';
|
|
5
|
+
import { GeminiLiveSession } from './GeminiLiveSession.js';
|
|
6
|
+
/**
|
|
7
|
+
* VoiceEngine — provider-agnostic call acceptor backed by the core
|
|
8
|
+
* RealtimeRuntime and OrchestrationAuthority.
|
|
9
|
+
*
|
|
10
|
+
* Supports any realtime provider (Gemini, OpenAI, custom) through the
|
|
11
|
+
* `createModelClient` factory config. Defaults to Gemini when `gemini`
|
|
12
|
+
* config is provided and no custom factory is set.
|
|
13
|
+
*
|
|
14
|
+
* Architecture:
|
|
15
|
+
* 1. VoiceEngine constructs a DefaultOrchestrationAuthority (core's brain)
|
|
16
|
+
* 2. VoiceEngine constructs a RealtimeRuntime (core's realtime facade)
|
|
17
|
+
* 3. acceptCall() creates a thin RealtimeCallWorker that delegates
|
|
18
|
+
* to RealtimeRuntime.startSession()
|
|
19
|
+
*
|
|
20
|
+
* All voice sessions get the FULL set of core semantics:
|
|
21
|
+
* - Hook lifecycle (onStart, onEnd, onToolResult, onHandoff, etc.)
|
|
22
|
+
* - Persistence with retry and onPersistenceError hook
|
|
23
|
+
* - Post-turn extraction
|
|
24
|
+
* - Memory ingestion
|
|
25
|
+
* - CapabilityHost routing (flows, handoffs, extraction, guardrails)
|
|
26
|
+
*
|
|
27
|
+
* Usage (Gemini — default):
|
|
28
|
+
* ```typescript
|
|
29
|
+
* const engine = new VoiceEngine({ foundation, agents, defaultAgentId, gemini });
|
|
30
|
+
* ```
|
|
31
|
+
*
|
|
32
|
+
* Usage (OpenAI Realtime):
|
|
33
|
+
* ```typescript
|
|
34
|
+
* const engine = new VoiceEngine({
|
|
35
|
+
* foundation, agents, defaultAgentId,
|
|
36
|
+
* createModelClient: () => new OpenAIRealtimeClient({ apiKey: '...' }),
|
|
37
|
+
* });
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export class VoiceEngine {
|
|
41
|
+
agents = new Map();
|
|
42
|
+
foundation;
|
|
43
|
+
config;
|
|
44
|
+
activeWorkers = new Map();
|
|
45
|
+
realtimeRuntime;
|
|
46
|
+
modelClientFactory;
|
|
47
|
+
constructor(config) {
|
|
48
|
+
this.config = config;
|
|
49
|
+
this.foundation = config.foundation;
|
|
50
|
+
for (const agent of config.agents) {
|
|
51
|
+
this.agents.set(agent.id, agent);
|
|
52
|
+
}
|
|
53
|
+
// Resolve the model client factory:
|
|
54
|
+
// 1. Custom factory takes priority (OpenAI, custom providers)
|
|
55
|
+
// 2. Fall back to Gemini factory when gemini config is provided
|
|
56
|
+
// 3. Error if neither is provided
|
|
57
|
+
if (config.createModelClient) {
|
|
58
|
+
this.modelClientFactory = config.createModelClient;
|
|
59
|
+
}
|
|
60
|
+
else if (config.gemini) {
|
|
61
|
+
const geminiConfig = config.gemini;
|
|
62
|
+
this.modelClientFactory = (agent) => {
|
|
63
|
+
return new GeminiLiveSession({
|
|
64
|
+
gemini: geminiConfig,
|
|
65
|
+
agent,
|
|
66
|
+
onEvent: () => { }, // Events go through the on() interface
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
throw new Error('VoiceEngine: either "gemini" config or "createModelClient" factory is required');
|
|
72
|
+
}
|
|
73
|
+
// Build the core authority — this is the single brain for all voice sessions.
|
|
74
|
+
const authority = new DefaultOrchestrationAuthority({
|
|
75
|
+
agents: config.agents.map(voiceAgentToCoreAgent),
|
|
76
|
+
defaultAgentId: config.defaultAgentId,
|
|
77
|
+
hooks: config.hooks,
|
|
78
|
+
memoryService: config.memoryService,
|
|
79
|
+
memoryIngestion: config.memoryIngestion,
|
|
80
|
+
extractionModel: config.extractionModel,
|
|
81
|
+
autoRetrieveProvider: config.autoRetrieveProvider,
|
|
82
|
+
});
|
|
83
|
+
// Build the realtime facade — provider-agnostic event loop over authority.
|
|
84
|
+
this.realtimeRuntime = new RealtimeRuntime({
|
|
85
|
+
agents: config.agents.map(voiceAgentToCoreAgent),
|
|
86
|
+
defaultAgentId: config.defaultAgentId,
|
|
87
|
+
authority,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Accept an incoming call and create a RealtimeCallWorker.
|
|
92
|
+
* The worker is NOT started automatically — call `worker.start()`.
|
|
93
|
+
*/
|
|
94
|
+
async acceptCall(params) {
|
|
95
|
+
const agentId = params.agentId ?? this.config.defaultAgentId;
|
|
96
|
+
const agent = this.agents.get(agentId);
|
|
97
|
+
if (!agent) {
|
|
98
|
+
throw new Error(`VoiceEngine: agent "${agentId}" not found`);
|
|
99
|
+
}
|
|
100
|
+
const callId = params.callId ?? crypto.randomUUID();
|
|
101
|
+
const sessionId = params.sessionId ?? `voice-${callId}`;
|
|
102
|
+
// Create a thin worker that delegates to RealtimeRuntime.
|
|
103
|
+
// The model client factory is provider-agnostic — same worker
|
|
104
|
+
// works for Gemini, OpenAI, or any future provider.
|
|
105
|
+
const worker = new RealtimeCallWorker({
|
|
106
|
+
callId,
|
|
107
|
+
sessionId,
|
|
108
|
+
userId: params.userId,
|
|
109
|
+
agent,
|
|
110
|
+
transport: params.transport,
|
|
111
|
+
runtime: this.realtimeRuntime,
|
|
112
|
+
createModelClient: this.modelClientFactory,
|
|
113
|
+
});
|
|
114
|
+
this.activeWorkers.set(callId, worker);
|
|
115
|
+
return worker;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Get an active call worker by call ID.
|
|
119
|
+
*/
|
|
120
|
+
getWorker(callId) {
|
|
121
|
+
return this.activeWorkers.get(callId);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Stop and remove a call worker.
|
|
125
|
+
*/
|
|
126
|
+
async endCall(callId) {
|
|
127
|
+
const worker = this.activeWorkers.get(callId);
|
|
128
|
+
if (worker) {
|
|
129
|
+
await worker.stop();
|
|
130
|
+
this.activeWorkers.delete(callId);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Stop all active calls.
|
|
135
|
+
*/
|
|
136
|
+
async shutdown() {
|
|
137
|
+
const promises = Array.from(this.activeWorkers.keys()).map(id => this.endCall(id));
|
|
138
|
+
await Promise.allSettled(promises);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
142
|
+
/**
|
|
143
|
+
* Convert VoiceAgentConfig to a shape compatible with core AgentConfig.
|
|
144
|
+
*/
|
|
145
|
+
export function voiceAgentToCoreAgent(agent) {
|
|
146
|
+
return {
|
|
147
|
+
id: agent.id,
|
|
148
|
+
name: agent.name,
|
|
149
|
+
description: agent.description,
|
|
150
|
+
prompt: agent.prompt,
|
|
151
|
+
type: agent.flow ? 'flow' : 'llm',
|
|
152
|
+
tools: agent.tools,
|
|
153
|
+
...(agent.flow ? { flow: agent.flow, initialNode: agent.initialNode ?? agent.flow.nodes[0]?.id } : {}),
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
//# sourceMappingURL=VoiceEngine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VoiceEngine.js","sourceRoot":"","sources":["../src/VoiceEngine.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAMjC,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,OAAO,WAAW;IACd,MAAM,GAAG,IAAI,GAAG,EAA4B,CAAC;IAC7C,UAAU,CAAa;IACvB,MAAM,CAAoB;IAC1B,aAAa,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC9C,eAAe,CAAkB;IACjC,kBAAkB,CAAmD;IAE7E,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QAEpC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACnC,CAAC;QAED,oCAAoC;QACpC,8DAA8D;QAC9D,gEAAgE;QAChE,kCAAkC;QAClC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7B,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QACrD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;YACnC,IAAI,CAAC,kBAAkB,GAAG,CAAC,KAAuB,EAAE,EAAE;gBACpD,OAAO,IAAI,iBAAiB,CAAC;oBAC3B,MAAM,EAAE,YAAY;oBACpB,KAAK;oBACL,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,uCAAuC;iBAC3D,CAAC,CAAC;YACL,CAAC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CACb,gFAAgF,CACjF,CAAC;QACJ,CAAC;QAED,8EAA8E;QAC9E,MAAM,SAAS,GAA2B,IAAI,6BAA6B,CAAC;YAC1E,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC;YAChD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,oBAAoB,EAAE,MAAM,CAAC,oBAAoB;SAClD,CAAC,CAAC;QAEH,2EAA2E;QAC3E,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CAAC;YACzC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC;YAChD,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,MAAwB;QACvC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,uBAAuB,OAAO,aAAa,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,MAAM,EAAE,CAAC;QAExD,0DAA0D;QAC1D,8DAA8D;QAC9D,oDAAoD;QACpD,MAAM,MAAM,GAAe,IAAI,kBAAkB,CAAC;YAChD,MAAM;YACN,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK;YACL,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,IAAI,CAAC,eAAe;YAC7B,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;SAC3C,CAAC,CAAC;QAEH,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,MAAc;QACtB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;CACF;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAuB;IAC3D,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACjC,KAAK,EAAE,KAAK,CAAC,KAAY;QACzB,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxF,CAAC;AACnB,CAAC"}
|