@just-every/ensemble 0.2.78 → 0.2.80

package/dist/model_providers/gemini.js

@@ -2,9 +2,9 @@ import { GoogleGenAI, Type, FunctionCallingConfigMode, Modality, } from '@google
 import { v4 as uuidv4 } from 'uuid';
 import { BaseModelProvider } from './base_provider.js';
 import { costTracker } from '../index.js';
-import { log_llm_error, log_llm_request, log_llm_response, } from '../utils/llm_logger.js';
+import { log_llm_error, log_llm_request, log_llm_response } from '../utils/llm_logger.js';
 import { isPaused } from '../utils/pause_controller.js';
-import { appendMessageWithImage, resizeAndTruncateForGemini, } from '../utils/image_utils.js';
+import { appendMessageWithImage, resizeAndTruncateForGemini } from '../utils/image_utils.js';
 function convertParameterToGeminiFormat(param) {
     let type = Type.STRING;
     switch (param.type) {
@@ -85,8 +85,7 @@ function convertParameterToGeminiFormat(param) {
         if (param.properties && typeof param.properties === 'object') {
             result.properties = {};
             for (const [propName, propSchema] of Object.entries(param.properties)) {
-                result.properties[propName] =
-                    convertParameterToGeminiFormat(propSchema);
+                result.properties[propName] = convertParameterToGeminiFormat(propSchema);
             }
         }
         else {
@@ -161,9 +160,7 @@ async function convertToGeminiFunctionDeclarations(tools) {
             parameters: {
                 type: Type.OBJECT,
                 properties,
-                required: Array.isArray(resolvedParams?.required)
-                    ? resolvedParams.required
-                    : [],
+                required: Array.isArray(resolvedParams?.required) ? resolvedParams.required : [],
             },
         };
     }));
@@ -218,10 +215,7 @@ async function convertToGeminiContents(model, messages) {
             let args = {};
             try {
                 const parsedArgs = JSON.parse(msg.arguments || '{}');
-                args =
-                    typeof parsedArgs === 'object' && parsedArgs !== null
-                        ? parsedArgs
-                        : { value: parsedArgs };
+                args = typeof parsedArgs === 'object' && parsedArgs !== null ? parsedArgs : { value: parsedArgs };
             }
             catch (e) {
                 console.error(`Failed to parse function call arguments for ${msg.name}:`, msg.arguments, e);
@@ -264,8 +258,7 @@ async function convertToGeminiContents(model, messages) {
             contents = await appendMessageWithImage(model, contents, message, {
                 read: () => textOutput,
                 write: value => {
-                    message.parts[0].functionResponse.response.content =
-                        value;
+                    message.parts[0].functionResponse.response.content = value;
                     return message;
                 },
             }, addImagesToInput);
@@ -275,9 +268,7 @@ async function convertToGeminiContents(model, messages) {
             if (typeof msg.content === 'string') {
                 textContent = msg.content;
             }
-            else if (msg.content &&
-                typeof msg.content === 'object' &&
-                'text' in msg.content) {
+            else if (msg.content && typeof msg.content === 'object' && 'text' in msg.content) {
                 textContent = msg.content.text;
             }
             else {
@@ -326,15 +317,14 @@ export class GeminiProvider extends BaseModelProvider {
             this._client = new GoogleGenAI({
                 apiKey: apiKey,
                 vertexai: false,
+                httpOptions: { apiVersion: 'v1alpha' },
             });
         }
         return this._client;
     }
     async createEmbedding(input, model, opts) {
         try {
-            let actualModelId = model.startsWith('gemini/')
-                ? model.substring(7)
-                : model;
+            let actualModelId = model.startsWith('gemini/') ? model.substring(7) : model;
             let thinkingConfig = null;
             for (const [suffix, budget] of Object.entries(THINKING_BUDGET_CONFIGS)) {
                 if (actualModelId.endsWith(suffix)) {
@@ -355,9 +345,7 @@ export class GeminiProvider extends BaseModelProvider {
                 payload.config.thinkingConfig = thinkingConfig;
             }
             const response = await this.client.models.embedContent(payload);
-            console.log('[Gemini] Embedding response structure:', JSON.stringify(response, (key, value) => key === 'values' &&
-                Array.isArray(value) &&
-                value.length > 10
+            console.log('[Gemini] Embedding response structure:', JSON.stringify(response, (key, value) => key === 'values' && Array.isArray(value) && value.length > 10
                 ? `[${value.length} items]`
                 : value, 2));
             if (!response.embeddings || !Array.isArray(response.embeddings)) {
@@ -376,11 +364,8 @@ export class GeminiProvider extends BaseModelProvider {
                 }
                 else {
                     console.warn('[Gemini] Could not find expected "values" property in embeddings response');
-                    extractedValues =
-                        response.embeddings;
-                    dimensions = Array.isArray(extractedValues[0])
-                        ? extractedValues[0].length
-                        : 0;
+                    extractedValues = response.embeddings;
+                    dimensions = Array.isArray(extractedValues[0]) ? extractedValues[0].length : 0;
                 }
             }
             costTracker.addUsage({
@@ -396,8 +381,7 @@ export class GeminiProvider extends BaseModelProvider {
             }
             else {
                 let result;
-                if (Array.isArray(extractedValues) &&
-                    extractedValues.length >= 1) {
+                if (Array.isArray(extractedValues) && extractedValues.length >= 1) {
                     const firstValue = extractedValues[0];
                     if (Array.isArray(firstValue)) {
                         result = firstValue;
@@ -417,10 +401,7 @@ export class GeminiProvider extends BaseModelProvider {
                         adjustedResult = result.slice(0, 3072);
                     }
                     else {
-                        adjustedResult = [
-                            ...result,
-                            ...Array(3072 - result.length).fill(0),
-                        ];
+                        adjustedResult = [...result, ...Array(3072 - result.length).fill(0)];
                     }
                 }
                 return adjustedResult;
@@ -444,8 +425,7 @@ export class GeminiProvider extends BaseModelProvider {
             catch (error) {
                 attempts++;
                 const errorMsg = error instanceof Error ? error.message : String(error);
-                if (errorMsg.includes('Incomplete JSON segment') &&
-                    attempts <= maxRetries) {
+                if (errorMsg.includes('Incomplete JSON segment') && attempts <= maxRetries) {
                     console.warn(`[Gemini] Incomplete JSON segment error, retrying (${attempts}/${maxRetries})...`);
                     await new Promise(resolve => setTimeout(resolve, 1000 * attempts));
                     continue;
@@ -456,9 +436,7 @@ export class GeminiProvider extends BaseModelProvider {
     }
     async *createResponseStream(messages, model, agent) {
         const { getToolsFromAgent } = await import('../utils/agent.js');
-        const tools = agent
-            ? await getToolsFromAgent(agent)
-            : [];
+        const tools = agent ? await getToolsFromAgent(agent) : [];
         const settings = agent?.modelSettings;
         let messageId = uuidv4();
         let contentBuffer = '';
@@ -526,8 +504,7 @@ export class GeminiProvider extends BaseModelProvider {
                         if ('additionalProperties' in obj) {
                             delete obj.additionalProperties;
                         }
-                        if (obj.properties &&
-                            typeof obj.properties === 'object') {
+                        if (obj.properties && typeof obj.properties === 'object') {
                             Object.values(obj.properties).forEach(prop => {
                                 removeAdditionalProperties(prop);
                             });
@@ -559,9 +536,7 @@ export class GeminiProvider extends BaseModelProvider {
                             settings.tool_choice?.type === 'function' &&
                             settings.tool_choice?.function?.name) {
                             toolChoice = FunctionCallingConfigMode.ANY;
-                            allowedFunctionNames = [
-                                settings.tool_choice.function.name,
-                            ];
+                            allowedFunctionNames = [settings.tool_choice.function.name];
                         }
                         else if (settings.tool_choice === 'required') {
                             toolChoice = FunctionCallingConfigMode.ANY;
@@ -579,8 +554,7 @@ export class GeminiProvider extends BaseModelProvider {
                                 },
                             };
                             if (allowedFunctionNames.length > 0) {
-                                config.toolConfig.functionCallingConfig.allowedFunctionNames =
-                                    allowedFunctionNames;
+                                config.toolConfig.functionCallingConfig.allowedFunctionNames = allowedFunctionNames;
                             }
                         }
                     }
@@ -697,8 +671,7 @@ export class GeminiProvider extends BaseModelProvider {
                                 message_id: messageId,
                                 order: eventOrder++,
                             };
-                            contentBuffer +=
-                                '\n\nSearch Results:\n' + formatted + '\n';
+                            contentBuffer += '\n\nSearch Results:\n' + formatted + '\n';
                         }
                     }
                 }
@@ -743,9 +716,7 @@ export class GeminiProvider extends BaseModelProvider {
         }
         catch (error) {
             log_llm_error(requestId, error);
-            const errorMessage = error instanceof Error
-                ? error.stack || error.message
-                : String(error);
+            const errorMessage = error instanceof Error ? error.stack || error.message : String(error);
             if (errorMessage.includes('Incomplete JSON segment')) {
                 console.error('[Gemini] Stream terminated with incomplete JSON. This may indicate network issues or timeouts.');
             }
@@ -796,8 +767,7 @@ export class GeminiProvider extends BaseModelProvider {
                 },
             });
             const images = [];
-            if (response.generatedImages &&
-                response.generatedImages.length > 0) {
+            if (response.generatedImages && response.generatedImages.length > 0) {
                 for (const generatedImage of response.generatedImages) {
                     if (generatedImage.image?.imageBytes) {
                         const base64Image = `data:image/png;base64,${generatedImage.image.imageBytes}`;
@@ -879,14 +849,12 @@ export class GeminiProvider extends BaseModelProvider {
                 throw new Error('No audio generated from Gemini TTS');
             }
             const candidate = response.candidates[0];
-            if (!candidate.content.parts ||
-                candidate.content.parts.length === 0) {
+            if (!candidate.content.parts || candidate.content.parts.length === 0) {
                 throw new Error('No audio parts in Gemini TTS response');
             }
             let audioData;
             for (const part of candidate.content.parts) {
-                if (part.inlineData &&
-                    part.inlineData.mimeType?.includes('audio')) {
+                if (part.inlineData && part.inlineData.mimeType?.includes('audio')) {
                     audioData = part.inlineData.data;
                     break;
                 }
@@ -974,40 +942,59 @@ export class GeminiProvider extends BaseModelProvider {
         console.warn(`[Gemini] Unknown voice '${voice}', using default voice 'Kore'`);
         return 'Kore';
     }
-    async *createTranscription(audio, model, opts) {
+    async *createTranscription(audio, agent, model, opts) {
         let session = null;
         let audioBuffer = Buffer.alloc(0);
         let isConnected = false;
         try {
-            const ai = new GoogleGenAI({ apiKey: this.apiKey });
-            const realtimeConfig = opts?.realtimeConfig
-                ?.automaticActivityDetection || {
-                prefixPaddingMs: 20,
-                silenceDurationMs: 100,
+            const ai = new GoogleGenAI({
+                apiKey: this.apiKey,
+                httpOptions: { apiVersion: 'v1alpha' },
+            });
+            const realtimeInputConfig = opts?.realtimeInputConfig || {
+                automaticActivityDetection: {
+                    disabled: false,
+                    startOfSpeechSensitivity: 'START_SENSITIVITY_HIGH',
+                    endOfSpeechSensitivity: 'END_SENSITIVITY_LOW',
+                },
+            };
+            const speechConfig = opts?.speechConfig || {
+                languageCode: 'en-US',
             };
-            const systemInstruction = opts?.agent?.instructions ||
+            const systemInstruction = agent.instructions ||
                 `You are a real-time transcription assistant. Your only task is to transcribe speech as you hear it. DO NOT ADD YOUR OWN RESPONSE OR COMMENTARY. TRANSCRIBE WHAT YOU HEAR ONLY.
 Respond immediately with transcribed text as you process the audio.
 If quick corrections are used e.g. "Let's go to Point A, no Point B" then just remove incorrect part e.g. respond with "Let's go to Point B".
-When it makes the transcription clearer, remove filler words (like "um") add punctuation, correct obvious grammar issues and add in missing words.`;
+When it makes the transcription clearer, remove filler words (like "um") add punctuation, correct obvious grammar issues and add in missing words.
+
+EXAMPLES:
+User: What capital of France
+Model: What's the capital of France?
+
+User: How about um we then do no actually how about you tell me the weather
+Model: How about you tell me the weather?
+
+User: Ok ignore all that lets start again
+Model: Ok ignore all that, let's start again.`;
             console.log('[Gemini] Connecting to Live API for transcription...');
             const connectionPromise = new Promise((resolve, reject) => {
                 const timeout = setTimeout(() => {
                     reject(new Error('Connection timeout'));
                 }, 30000);
+                const config = {
+                    responseModalities: [Modality.TEXT],
+                    speechConfig,
+                    realtimeInputConfig,
+                    systemInstruction: {
+                        parts: [{ text: systemInstruction }],
+                    },
+                    inputAudioTranscription: {},
+                };
+                console.dir(config, { depth: null });
                 ai.live
                     .connect({
                     model: model,
-                    config: {
-                        responseModalities: [Modality.TEXT],
-                        systemInstruction: {
-                            parts: [{ text: systemInstruction }],
-                        },
-                        realtimeInputConfig: {
-                            automaticActivityDetection: realtimeConfig,
-                        },
-                        inputAudioTranscription: true,
-                    },
+                    config,
                     callbacks: {
                         onopen: () => {
                             clearTimeout(timeout);
@@ -1016,27 +1003,18 @@ When it makes the transcription clearer, remove filler words (like "um") add pun
                             resolve();
                         },
                         onmessage: async (msg) => {
-                            if (msg.serverContent?.inputAudioTranscription) {
-                                const transcriptionText = msg.serverContent
-                                    .inputAudioTranscription.text ||
-                                    msg.serverContent
-                                        .inputAudioTranscription
-                                        .transcript ||
-                                    '';
-                                if (transcriptionText) {
-                                    const previewEvent = {
-                                        type: 'transcription_preview',
-                                        timestamp: new Date().toISOString(),
-                                        text: transcriptionText,
-                                        isFinal: true,
-                                    };
-                                    transcriptEvents.push(previewEvent);
-                                    console.debug('[Gemini] Received input transcription:', transcriptionText);
-                                }
+                            if (msg.serverContent?.inputTranscription?.text) {
+                                const previewEvent = {
+                                    type: 'transcription_preview',
+                                    timestamp: new Date().toISOString(),
+                                    text: msg.serverContent.inputTranscription.text,
+                                    isFinal: true,
+                                };
+                                transcriptEvents.push(previewEvent);
+                                console.debug('[Gemini] Received input transcription:', msg.serverContent.inputTranscription.text);
                             }
                             if (msg.serverContent?.modelTurn?.parts) {
-                                for (const part of msg.serverContent
-                                    .modelTurn.parts) {
+                                for (const part of msg.serverContent.modelTurn.parts) {
                                     if (part.text && part.text.trim()) {
                                         const deltaEvent = {
                                             type: 'transcription_delta',
@@ -1058,26 +1036,34 @@ When it makes the transcription clearer, remove filler words (like "um") add pun
                             if (msg.usageMetadata) {
                                 costTracker.addUsage({
                                     model: model,
-                                    input_tokens: msg.usageMetadata
-                                        .promptTokenCount || 0,
-                                    output_tokens: msg.usageMetadata
-                                        .responseTokenCount || 0,
+                                    input_tokens: msg.usageMetadata.promptTokenCount || 0,
+                                    output_tokens: msg.usageMetadata.responseTokenCount || 0,
                                     input_modality: 'audio',
                                     output_modality: 'text',
                                     metadata: {
-                                        totalTokens: msg.usageMetadata
-                                            .totalTokenCount || 0,
+                                        totalTokens: msg.usageMetadata.totalTokenCount || 0,
                                         source: 'gemini-live-transcription',
                                     },
                                 });
                             }
                         },
                         onerror: (err) => {
-                            console.error('[Gemini] Live API error:', err);
+                            console.error('[Gemini] Live API error:', {
+                                code: err.code,
+                                reason: err.reason,
+                                wasClean: err.wasClean,
+                            });
                             connectionError = err;
                         },
-                        onclose: () => {
+                        onclose: (event) => {
                             console.log('[Gemini] Live session closed');
+                            if (event) {
+                                console.log('[Gemini] Close event details:', {
+                                    code: event.code,
+                                    reason: event.reason,
+                                    wasClean: event.wasClean,
+                                });
+                            }
                             isConnected = false;
                         },
                     },
@@ -1124,10 +1110,7 @@ When it makes the transcription clearer, remove filler words (like "um") add pun
                     if (done)
                         break;
                     if (value) {
-                        audioBuffer = Buffer.concat([
-                            audioBuffer,
-                            Buffer.from(value),
-                        ]);
+                        audioBuffer = Buffer.concat([audioBuffer, Buffer.from(value)]);
                         while (audioBuffer.length >= chunkSize) {
                             const chunk = audioBuffer.slice(0, chunkSize);
                             audioBuffer = audioBuffer.slice(chunkSize);
@@ -1170,21 +1153,34 @@ When it makes the transcription clearer, remove filler words (like "um") add pun
             const errorEvent = {
                 type: 'error',
                 timestamp: new Date().toISOString(),
-                error: error instanceof Error
-                    ? error.message
-                    : 'Transcription failed',
+                error: error instanceof Error ? error.message : 'Transcription failed',
             };
             yield errorEvent;
         }
     }
+    async createLiveSession(config, agent, model, opts) {
+        console.log(`[Gemini] Creating Live session with model ${model}`);
+        const liveModels = [
+            'gemini-2.0-flash-live-001',
+            'gemini-live-2.5-flash-preview',
+            'gemini-2.5-flash-preview-native-audio-dialog',
+            'gemini-2.5-flash-exp-native-audio-thinking-dialog',
+            'gemini-2.0-flash-exp',
+        ];
+        if (!liveModels.some(m => model.includes(m))) {
+            throw new Error(`Model ${model} does not support Live API. Supported models: ${liveModels.join(', ')}`);
+        }
+        const sessionId = uuidv4();
+        const liveSession = new GeminiLiveSession(sessionId, this.client, model, config, agent, opts);
+        await liveSession.initialize();
+        return liveSession;
+    }
 }
 function normalizeAudioSource(source) {
     if (source instanceof ReadableStream) {
         return source;
     }
-    if (typeof source === 'object' &&
-        source !== null &&
-        Symbol.asyncIterator in source) {
+    if (typeof source === 'object' && source !== null && Symbol.asyncIterator in source) {
         return new ReadableStream({
             async start(controller) {
                 try {
@@ -1214,5 +1210,397 @@ function normalizeAudioSource(source) {
     }
     throw new Error(`Unsupported audio source type: ${typeof source}`);
 }
+class GeminiLiveSession {
+    sessionId;
+    ai;
+    model;
+    config;
+    agent;
+    options;
+    session = null;
+    eventQueue = [];
+    eventResolvers = [];
+    _isActive = true;
+    sessionClosed = false;
+    messageHistory = [];
+    currentTurn = null;
+    constructor(sessionId, ai, model, config, agent, options) {
+        this.sessionId = sessionId;
+        this.ai = ai;
+        this.model = model;
+        this.config = config;
+        this.agent = agent;
+        this.options = options;
+    }
+    async initialize() {
+        const connectionPromise = new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                reject(new Error('Connection timeout'));
+            }, 30000);
+            const tools = [];
+            if (this.config.tools) {
+                for (const toolGroup of this.config.tools) {
+                    if (toolGroup.functionDeclarations) {
+                        const functionDeclarations = toolGroup.functionDeclarations.map(func => ({
+                            name: func.name,
+                            description: func.description,
+                            parameters: convertParameterToGeminiFormat(func.parameters),
+                        }));
+                        tools.push({ functionDeclarations });
+                    }
+                    if (toolGroup.codeExecution) {
+                        tools.push({ codeExecution: {} });
+                    }
+                    if (toolGroup.googleSearch) {
+                        tools.push({ googleSearch: {} });
+                    }
+                }
+            }
+            let systemInstruction = undefined;
+            if (this.agent.instructions) {
+                systemInstruction = {
+                    parts: [{ text: this.agent.instructions }],
+                };
+            }
+            const responseModalities = this.config.responseModalities[0] === 'AUDIO' ? [Modality.AUDIO] : [Modality.TEXT];
+            const config = {
+                responseModalities,
+                systemInstruction,
+                tools: tools.length > 0 ? tools : undefined,
+            };
+            if (this.config.responseModalities[0] === 'AUDIO' && this.config.speechConfig) {
+                config.speechConfig = {
+                    voiceConfig: this.config.speechConfig.voiceConfig,
+                };
+            }
+            if (this.config.realtimeInputConfig) {
+                config.realtimeInputConfig = {
+                    automaticActivityDetection: this.config.realtimeInputConfig.automaticActivityDetection
+                        ? {
+                            disabled: this.config.realtimeInputConfig.automaticActivityDetection.disabled,
+                        }
+                        : undefined,
+                };
+            }
+            if (this.config.inputAudioTranscription) {
+                config.inputAudioTranscription = true;
+            }
+            if (this.config.outputAudioTranscription) {
+                config.outputAudioTranscription = true;
+            }
+            if (this.config.enableAffectiveDialog) {
+                config.enableAffectiveDialog = true;
+            }
+            if (this.config.proactivity) {
+                config.proactivity = this.config.proactivity;
+            }
+            console.log('[Gemini] Connecting with config:', JSON.stringify(config, null, 2));
+            this.ai.live
+                .connect({
+                model: this.model,
+                config,
+                callbacks: {
+                    onopen: () => {
+                        clearTimeout(timeout);
+                        console.log('[Gemini] Live session connected');
+                        this.pushEvent({
+                            type: 'live_ready',
+                            timestamp: new Date().toISOString(),
+                        });
+                        resolve();
+                    },
+                    onmessage: (msg) => {
+                        this.handleMessage(msg);
+                    },
+                    onerror: (err) => {
+                        console.error('[Gemini] Live API error:', err);
+                        console.error('[Gemini] Error details:', JSON.stringify(err, null, 2));
+                        this.pushEvent({
+                            type: 'error',
+                            timestamp: new Date().toISOString(),
+                            error: err.message || String(err),
+                            code: err.code,
+                            recoverable: true,
+                        });
+                    },
+                    onclose: (event) => {
+                        console.log('[Gemini] Live session closed', event);
+                        if (event) {
+                            console.log('[Gemini] Close event details:', {
+                                code: event.code,
+                                reason: event.reason,
+                                wasClean: event.wasClean,
+                            });
+                        }
+                        this._isActive = false;
+                        this.sessionClosed = true;
+                        this.resolveAllWaitingEvents();
+                    },
+                },
+            })
+                .then(s => {
+                this.session = s;
+            });
+        });
+        await connectionPromise;
+    }
+    handleMessage(msg) {
+        console.log('[Gemini] Received message:', JSON.stringify(msg, null, 2));
+        if (msg.error) {
+            console.error('[Gemini] Error in message:', msg.error);
+            this.pushEvent({
+                type: 'error',
+                timestamp: new Date().toISOString(),
+                error: msg.error.message || JSON.stringify(msg.error),
+                code: msg.error.code || 'UNKNOWN_ERROR',
+                recoverable: false,
+            });
+            return;
+        }
+        if (msg.serverContent?.modelTurn?.parts) {
+            for (const part of msg.serverContent.modelTurn.parts) {
+                if (part.inlineData?.mimeType?.startsWith('audio/')) {
+                    this.pushEvent({
+                        type: 'audio_output',
+                        timestamp: new Date().toISOString(),
+                        data: part.inlineData.data,
+                        format: {
+                            sampleRate: 24000,
+                            channels: 1,
+                            encoding: 'pcm',
+                        },
+                    });
+                }
+                if (part.text) {
+                    if (!this.currentTurn || this.currentTurn.role !== 'model') {
+                        this.currentTurn = { role: 'model', text: '' };
+                        this.pushEvent({
+                            type: 'turn_start',
+                            timestamp: new Date().toISOString(),
+                            role: 'model',
+                        });
+                    }
+                    this.currentTurn.text += part.text;
+                    this.pushEvent({
+                        type: 'text_delta',
+                        timestamp: new Date().toISOString(),
+                        delta: part.text,
+                    });
+                    this.pushEvent({
+                        type: 'message_delta',
+                        timestamp: new Date().toISOString(),
+                        delta: part.text,
+                    });
+                }
+            }
+        }
+        if (msg.serverContent?.modelTurn?.parts) {
+            for (const part of msg.serverContent.modelTurn.parts) {
+                if (part.functionCall) {
+                    const toolCall = {
+                        id: uuidv4(),
+                        type: 'function',
+                        function: {
+                            name: part.functionCall.name,
+                            arguments: JSON.stringify(part.functionCall.args),
+                        },
+                    };
+                    this.pushEvent({
+                        type: 'tool_call',
+                        timestamp: new Date().toISOString(),
+                        toolCalls: [toolCall],
+                    });
+                }
+            }
+        }
+        if (msg.serverContent?.inputAudioTranscription) {
+            const text = msg.serverContent.inputAudioTranscription.text ||
+                msg.serverContent.inputAudioTranscription.transcript ||
+                '';
+            if (text) {
+                this.pushEvent({
+                    type: 'transcription_input',
+                    timestamp: new Date().toISOString(),
+                    text,
+                });
+            }
+        }
+        if (msg.serverContent?.outputTranscription) {
+            const text = msg.serverContent.outputTranscription.text || '';
+            if (text) {
+                this.pushEvent({
+                    type: 'transcription_output',
+                    timestamp: new Date().toISOString(),
+                    text,
+                });
+            }
+        }
+        if (msg.serverContent?.turnComplete) {
+            if (this.currentTurn) {
+                const message = this.currentTurn.role === 'model'
+                    ? {
+                        type: 'message',
+                        role: 'assistant',
+                        content: this.currentTurn.text,
+                        status: 'completed',
+                    }
+                    : {
+                        type: 'message',
+                        role: 'user',
+                        content: this.currentTurn.text,
+                    };
+                this.messageHistory.push(message);
+                this.pushEvent({
+                    type: 'turn_complete',
+                    timestamp: new Date().toISOString(),
+                    role: this.currentTurn.role,
+                    message,
+                });
+                this.currentTurn = null;
+            }
+        }
+        if (msg.serverContent?.interrupted) {
+            const cancelledToolCalls = [];
+            if (msg.serverContent.cancelledFunctionCalls) {
+                cancelledToolCalls.push(...msg.serverContent.cancelledFunctionCalls.map((fc) => fc.id));
+            }
+            this.pushEvent({
+                type: 'interrupted',
+                timestamp: new Date().toISOString(),
+                cancelledToolCalls,
+            });
+        }
+        if (msg.usageMetadata) {
+            const usage = msg.usageMetadata;
+            const inputTokens = usage.promptTokenCount || 0;
+            const outputTokens = usage.candidatesTokenCount || 0;
+            const totalTokens = usage.totalTokenCount || 0;
+            costTracker.addUsage({
+                model: this.model,
+                input_tokens: inputTokens,
+                output_tokens: outputTokens,
+                cached_tokens: usage.cachedContentTokenCount || 0,
+                metadata: {
+                    total_tokens: totalTokens,
+                    source: 'gemini-live',
+                },
+            });
+            const inputCost = undefined;
+            const outputCost = undefined;
+            const totalCost = undefined;
+            this.pushEvent({
+                type: 'cost_update',
+                timestamp: new Date().toISOString(),
+                usage: {
+                    inputTokens,
+                    outputTokens,
+                    totalTokens,
+                    inputCost,
+                    outputCost,
+                    totalCost,
+                },
+            });
+        }
+    }
+    async sendAudio(audio) {
+        if (!this.session || !this._isActive) {
+            console.error(`[GeminiLiveSession ${this.sessionId}] Cannot send audio - session not active`);
+            throw new Error('Session is not active');
+        }
+        console.log(`[GeminiLiveSession ${this.sessionId}] Sending audio: ${audio.data.length} chars (base64), mimeType: ${audio.mimeType}`);
+        try {
+            await this.session.sendRealtimeInput({
+                media: {
+                    mimeType: audio.mimeType,
+                    data: audio.data,
+                },
+            });
+            console.log(`[GeminiLiveSession ${this.sessionId}] Audio sent successfully`);
+        }
+        catch (error) {
+            console.error(`[GeminiLiveSession ${this.sessionId}] Error sending audio:`, error);
+            throw error;
+        }
+        const size = Math.ceil((audio.data.length * 3) / 4);
+        this.pushEvent({
+            type: 'audio_input',
+            timestamp: new Date().toISOString(),
+            size,
+        });
+    }
+    async sendText(text, role = 'user') {
+        if (!this.session || !this._isActive) {
+            throw new Error('Session is not active');
+        }
+        const message = {
+            role: role === 'assistant' ? 'model' : 'user',
+            parts: [{ text }],
+        };
+        await this.session.sendClientContent({
+            turns: [message],
+        });
+        this.pushEvent({
+            type: 'turn_start',
+            timestamp: new Date().toISOString(),
+            role: role === 'assistant' ? 'model' : 'user',
+        });
+    }
+    async sendToolResponse(toolResults) {
+        if (!this.session || !this._isActive) {
+            throw new Error('Session is not active');
+        }
+        const functionResponses = toolResults.map(result => ({
+            id: result.call_id || result.id,
+            name: result.toolCall.function.name,
+            response: result.error ? { error: result.error } : { result: result.output },
+        }));
+        await this.session.sendToolResponse({ functionResponses });
+    }
+    async *getEventStream() {
+        while (this._isActive || this.eventQueue.length > 0) {
+            if (this.eventQueue.length > 0) {
+                yield this.eventQueue.shift();
+            }
+            else {
+                const result = await new Promise(resolve => {
+                    if (this.sessionClosed && this.eventQueue.length === 0) {
+                        resolve({ done: true, value: undefined });
+                    }
+                    else {
+                        this.eventResolvers.push(resolve);
+                    }
+                });
+                if (result.done)
+                    break;
+                if (result.value)
+                    yield result.value;
+            }
+        }
+    }
+    async close() {
+        if (this.session && this._isActive) {
+            this._isActive = false;
+            await this.session.close();
+        }
+    }
+    isActive() {
+        return this._isActive;
+    }
+    pushEvent(event) {
+        if (this.eventResolvers.length > 0) {
+            const resolver = this.eventResolvers.shift();
+            resolver({ value: event, done: false });
+        }
+        else {
+            this.eventQueue.push(event);
+        }
+    }
+    resolveAllWaitingEvents() {
+        for (const resolver of this.eventResolvers) {
+            resolver({ done: true, value: undefined });
+        }
+        this.eventResolvers = [];
+    }
+}
 export const geminiProvider = new GeminiProvider();
 //# sourceMappingURL=gemini.js.map