@just-every/ensemble 0.2.79 → 0.2.81

package/dist/model_providers/openai.js

@@ -1,11 +1,11 @@
 import { BaseModelProvider } from './base_provider.js';
 import OpenAI, { toFile } from 'openai';
 import { costTracker } from '../index.js';
-import { log_llm_request, log_llm_response, log_llm_error, } from '../utils/llm_logger.js';
+import { log_llm_request, log_llm_response, log_llm_error } from '../utils/llm_logger.js';
 import { isPaused } from '../utils/pause_controller.js';
-import { appendMessageWithImage, resizeAndSplitForOpenAI, } from '../utils/image_utils.js';
-import { bufferDelta, flushBufferedDeltas, } from '../utils/delta_buffer.js';
-import { createCitationTracker, formatCitation, generateFootnotes, } from '../utils/citation_tracker.js';
+import { appendMessageWithImage, resizeAndSplitForOpenAI } from '../utils/image_utils.js';
+import { bufferDelta, flushBufferedDeltas } from '../utils/delta_buffer.js';
+import { createCitationTracker, formatCitation, generateFootnotes } from '../utils/citation_tracker.js';
 const BROWSER_WIDTH = 1024;
 const BROWSER_HEIGHT = 1536;
 function processSchemaForOpenAI(schema, originalProperties) {
@@ -47,8 +47,7 @@ function processSchemaForOpenAI(schema, originalProperties) {
                 delete schema[keyword];
             }
         });
-        const isObject = schema.type === 'object' ||
-            (schema.type === undefined && schema.properties !== undefined);
+        const isObject = schema.type === 'object' || (schema.type === undefined && schema.properties !== undefined);
         for (const key of ['anyOf', 'allOf']) {
             if (Array.isArray(schema[key])) {
                 schema[key].forEach((variantSchema) => processSchemaRecursively(variantSchema));
@@ -98,8 +97,7 @@ function processSchemaForOpenAI(schema, originalProperties) {
             delete processedSchema.required;
         }
     }
-    if (processedSchema.properties &&
-        processedSchema.additionalProperties === undefined) {
+    if (processedSchema.properties && processedSchema.additionalProperties === undefined) {
         processedSchema.additionalProperties = false;
     }
     return processedSchema;
@@ -258,9 +256,7 @@ export class OpenAIProvider extends BaseModelProvider {
                 input_tokens: inputTokens,
                 output_tokens: 0,
                 metadata: {
-                    dimensions: response.data[0]?.embedding.length ||
-                        opts?.dimensions ||
-                        1536,
+                    dimensions: response.data[0]?.embedding.length || opts?.dimensions || 1536,
                 },
             });
             if (Array.isArray(input) && input.length > 1) {
@@ -284,21 +280,17 @@ export class OpenAIProvider extends BaseModelProvider {
                 quality = 'medium';
             else if (opts?.quality === 'hd')
                 quality = 'high';
-            else if (opts?.quality === 'low' ||
-                opts?.quality === 'medium' ||
-                opts?.quality === 'high') {
+            else if (opts?.quality === 'low' || opts?.quality === 'medium' || opts?.quality === 'high') {
                 quality = opts.quality;
             }
             let size = 'auto';
             if (opts?.size === 'square' || opts?.size === '1024x1024') {
                 size = '1024x1024';
             }
-            else if (opts?.size === 'landscape' ||
-                opts?.size === '1536x1024') {
+            else if (opts?.size === 'landscape' || opts?.size === '1536x1024') {
                 size = '1536x1024';
             }
-            else if (opts?.size === 'portrait' ||
-                opts?.size === '1024x1536') {
+            else if (opts?.size === 'portrait' || opts?.size === '1024x1536') {
                 size = '1024x1536';
             }
             const background = 'auto';
@@ -307,17 +299,16 @@ export class OpenAIProvider extends BaseModelProvider {
             let response;
             if (source_images) {
                 console.log('[OpenAI] Using images.edit with source_images');
-                const imageArray = Array.isArray(source_images)
-                    ? source_images
-                    : [source_images];
+                const imageArray = Array.isArray(source_images) ? source_images : [source_images];
                 const imageFiles = [];
                 for (const sourceImg of imageArray) {
                     let imageFile;
-                    if (sourceImg.startsWith('http://') ||
-                        sourceImg.startsWith('https://')) {
+                    if (sourceImg.startsWith('http://') || sourceImg.startsWith('https://')) {
                         const imageResponse = await fetch(sourceImg);
                         const imageBuffer = await imageResponse.arrayBuffer();
-                        imageFile = await toFile(new Uint8Array(imageBuffer), `image_${imageFiles.length}.png`, { type: 'image/png' });
+                        imageFile = await toFile(new Uint8Array(imageBuffer), `image_${imageFiles.length}.png`, {
+                            type: 'image/png',
+                        });
                     }
                     else {
                         let base64Data = sourceImg;
@@ -325,7 +316,9 @@ export class OpenAIProvider extends BaseModelProvider {
                             base64Data = sourceImg.split(',')[1];
                         }
                         const binaryData = Buffer.from(base64Data, 'base64');
-                        imageFile = await toFile(new Uint8Array(binaryData), `image_${imageFiles.length}.png`, { type: 'image/png' });
+                        imageFile = await toFile(new Uint8Array(binaryData), `image_${imageFiles.length}.png`, {
+                            type: 'image/png',
+                        });
                     }
                     imageFiles.push(imageFile);
                 }
@@ -468,45 +461,36 @@ export class OpenAIProvider extends BaseModelProvider {
     }
     async *createResponseStream(messages, model, agent) {
         const { getToolsFromAgent } = await import('../utils/agent.js');
-        const tools = agent
-            ? await getToolsFromAgent(agent)
-            : [];
+        const tools = agent ? await getToolsFromAgent(agent) : [];
         const settings = agent?.modelSettings;
         let requestId;
         try {
             let input = [];
             for (const messageFull of messages) {
                 let message = { ...messageFull };
-                const originalModel = message
-                    .model;
+                const originalModel = message.model;
                 delete message.timestamp;
                 delete message.model;
                 delete message.pinned;
                 if (message.type === 'thinking') {
-                    if (model.startsWith('o') &&
-                        message.thinking_id &&
-                        model === originalModel) {
+                    if (model.startsWith('o') && message.thinking_id && model === originalModel) {
                         console.log(`[OpenAI] Processing thinking message with ID: ${message.thinking_id}`, message);
                         const match = message.thinking_id.match(/^(rs_[A-Za-z0-9]+)-(\d)$/);
                         if (match) {
                             const reasoningId = match[1];
                             const summaryIndex = parseInt(match[2], 10);
-                            const summaryText = typeof message.content === 'string'
-                                ? message.content
-                                : JSON.stringify(message.content);
+                            const summaryText = typeof message.content === 'string' ? message.content : JSON.stringify(message.content);
                             const summaryEntry = {
                                 type: 'summary_text',
                                 text: summaryText,
                             };
-                            const existingIndex = input.findIndex((item) => item.type === 'reasoning' &&
-                                item.id === reasoningId);
+                            const existingIndex = input.findIndex((item) => item.type === 'reasoning' && item.id === reasoningId);
                             if (existingIndex !== -1) {
                                 const existingItem = input[existingIndex];
                                 if (!existingItem.summary) {
                                     existingItem.summary = [];
                                 }
-                                existingItem.summary[summaryIndex] =
-                                    summaryEntry;
+                                existingItem.summary[summaryIndex] = summaryEntry;
                                 input[existingIndex] = existingItem;
                             }
                             else {
@@ -530,9 +514,7 @@ export class OpenAIProvider extends BaseModelProvider {
                     continue;
                 }
                 if (message.type === 'function_call') {
-                    if (message.id &&
-                        (!message.id.startsWith('fc_') ||
-                            model !== originalModel)) {
+                    if (message.id && (!message.id.startsWith('fc_') || model !== originalModel)) {
                         const { id, ...rest } = message;
                         message = rest;
                     }
@@ -545,12 +527,8 @@ export class OpenAIProvider extends BaseModelProvider {
                     input = await appendMessageWithImage(model, input, messageToAdd, 'output', addImagesToInput, `function call output of ${message.name}`);
                     continue;
                 }
-                if ((message.type ?? 'message') === 'message' &&
-                    'content' in message) {
-                    if ('id' in message &&
-                        message.id &&
-                        (!message.id.startsWith('msg_') ||
-                            model !== originalModel)) {
+                if ((message.type ?? 'message') === 'message' && 'content' in message) {
+                    if ('id' in message && message.id && (!message.id.startsWith('msg_') || model !== originalModel)) {
                         const { id, ...rest } = message;
                         message = rest;
                         console.log(`[OpenAI] Removed message ID: ${id} model: ${model} originalModel: ${originalModel}`);
@@ -580,11 +558,7 @@ export class OpenAIProvider extends BaseModelProvider {
                     requestParams.top_p = settings.top_p;
                 }
             }
-            const REASONING_EFFORT_CONFIGS = [
-                'low',
-                'medium',
-                'high',
-            ];
+            const REASONING_EFFORT_CONFIGS = ['low', 'medium', 'high'];
             let hasEffortSuffix = false;
             for (const effort of REASONING_EFFORT_CONFIGS) {
                 const suffix = `-${effort}`;
@@ -651,22 +625,18 @@ export class OpenAIProvider extends BaseModelProvider {
                     }
                     if (event.type === 'response.in_progress') {
                     }
-                    else if (event.type === 'response.completed' &&
-                        event.response?.usage) {
+                    else if (event.type === 'response.completed' && event.response?.usage) {
                         costTracker.addUsage({
                             model,
                             input_tokens: event.response.usage.input_tokens || 0,
                             output_tokens: event.response.usage.output_tokens || 0,
-                            cached_tokens: event.response.usage.input_tokens_details
-                                ?.cached_tokens || 0,
+                            cached_tokens: event.response.usage.input_tokens_details?.cached_tokens || 0,
                             metadata: {
-                                reasoning_tokens: event.response.usage.output_tokens_details
-                                    ?.reasoning_tokens || 0,
+                                reasoning_tokens: event.response.usage.output_tokens_details?.reasoning_tokens || 0,
                             },
                         });
                     }
-                    else if (event.type === 'response.failed' &&
-                        event.response?.error) {
+                    else if (event.type === 'response.failed' && event.response?.error) {
                         const errorInfo = event.response.error;
                         log_llm_error(requestId, errorInfo);
                         console.error(`Response ${event.response.id} failed: [${errorInfo.code}] ${errorInfo.message}`);
@@ -675,8 +645,7 @@ export class OpenAIProvider extends BaseModelProvider {
                             error: `OpenAI response  failed: [${errorInfo.code}] ${errorInfo.message}`,
                         };
                     }
-                    else if (event.type === 'response.incomplete' &&
-                        event.response?.incomplete_details) {
+                    else if (event.type === 'response.incomplete' && event.response?.incomplete_details) {
                         const reason = event.response.incomplete_details.reason;
                         log_llm_error(requestId, 'OpenAI response incomplete: ' + reason);
                         console.warn(`Response ${event.response.id} incomplete: ${reason}`);
@@ -685,8 +654,7 @@ export class OpenAIProvider extends BaseModelProvider {
                             error: 'OpenAI response incomplete: ' + reason,
                         };
                     }
-                    else if (event.type === 'response.output_item.added' &&
-                        event.item) {
+                    else if (event.type === 'response.output_item.added' && event.item) {
                         if (event.item.type === 'function_call') {
                             if (!toolCallStates.has(event.item.id)) {
                                 toolCallStates.set(event.item.id, {
@@ -704,10 +672,8 @@ export class OpenAIProvider extends BaseModelProvider {
                             }
                         }
                     }
-                    else if (event.type === 'response.output_item.done' &&
-                        event.item) {
-                        if (event.item.type === 'reasoning' &&
-                            !event.item.summary.length) {
+                    else if (event.type === 'response.output_item.done' && event.item) {
+                        if (event.item.type === 'reasoning' && !event.item.summary.length) {
                             yield {
                                 type: 'message_complete',
                                 content: '',
@@ -716,14 +682,11 @@ export class OpenAIProvider extends BaseModelProvider {
                             };
                         }
                     }
-                    else if (event.type === 'response.content_part.added' &&
-                        event.part) {
+                    else if (event.type === 'response.content_part.added' && event.part) {
                     }
-                    else if (event.type === 'response.content_part.done' &&
-                        event.part) {
+                    else if (event.type === 'response.content_part.done' && event.part) {
                     }
-                    else if (event.type === 'response.output_text.delta' &&
-                        event.delta) {
+                    else if (event.type === 'response.output_text.delta' && event.delta) {
                         const itemId = event.item_id;
                         let position = messagePositions.get(itemId) ?? 0;
                         for (const ev of bufferDelta(deltaBuffers, itemId, event.delta, content => ({
@@ -736,15 +699,12 @@ export class OpenAIProvider extends BaseModelProvider {
                         }
                         messagePositions.set(itemId, position);
                     }
-                    else if (event.type ===
-                        'response.output_text.annotation.added' &&
+                    else if (event.type === 'response.output_text.annotation.added' &&
                         event.annotation) {
                         const eventData = event;
-                        if (eventData.annotation?.type === 'url_citation' &&
-                            eventData.annotation.url) {
+                        if (eventData.annotation?.type === 'url_citation' && eventData.annotation.url) {
                             const marker = formatCitation(citationTracker, {
-                                title: eventData.annotation.title ||
-                                    eventData.annotation.url,
+                                title: eventData.annotation.title || eventData.annotation.url,
                                 url: eventData.annotation.url,
                             });
                             let position = messagePositions.get(eventData.item_id) ?? 0;
@@ -760,8 +720,7 @@ export class OpenAIProvider extends BaseModelProvider {
                             console.log('Annotation added:', eventData.annotation);
                         }
                     }
-                    else if (event.type === 'response.output_text.done' &&
-                        event.text !== undefined) {
+                    else if (event.type === 'response.output_text.done' && event.text !== undefined) {
                         const itemId = event.item_id;
                         let finalText = event.text;
                         if (citationTracker.citations.size > 0) {
@@ -775,12 +734,10 @@ export class OpenAIProvider extends BaseModelProvider {
                         };
                         messagePositions.delete(itemId);
                     }
-                    else if (event.type === 'response.refusal.delta' &&
-                        event.delta) {
+                    else if (event.type === 'response.refusal.delta' && event.delta) {
                         console.log(`Refusal delta for item ${event.item_id}: ${event.delta}`);
                     }
-                    else if (event.type === 'response.refusal.done' &&
-                        event.refusal) {
+                    else if (event.type === 'response.refusal.done' && event.refusal) {
                         log_llm_error(requestId, 'OpenAI refusal error: ' + event.refusal);
                         console.log(`Refusal done for item ${event.item_id}: ${event.refusal}`);
                         yield {
@@ -788,9 +745,7 @@ export class OpenAIProvider extends BaseModelProvider {
                             error: 'OpenAI refusal error: ' + event.refusal,
                         };
                     }
-                    else if (event.type ===
-                        'response.function_call_arguments.delta' &&
-                        event.delta) {
+                    else if (event.type === 'response.function_call_arguments.delta' && event.delta) {
                         const currentCall = toolCallStates.get(event.item_id);
                         if (currentCall) {
                             currentCall.function.arguments += event.delta;
@@ -799,8 +754,7 @@ export class OpenAIProvider extends BaseModelProvider {
                             console.warn(`Received function_call_arguments.delta for unknown item_id: ${event.item_id}`);
                         }
                     }
-                    else if (event.type ===
-                        'response.function_call_arguments.done' &&
+                    else if (event.type === 'response.function_call_arguments.done' &&
                         event.arguments !== undefined) {
                         const currentCall = toolCallStates.get(event.item_id);
                         if (currentCall) {
@@ -839,9 +793,7 @@ export class OpenAIProvider extends BaseModelProvider {
                     else if (event.type === 'response.reasoning_summary_part.done') {
                         console.log(`Reasoning summary part done for item ${event.item_id}, index ${event.summary_index}`);
                     }
-                    else if (event.type ===
-                        'response.reasoning_summary_text.delta' &&
-                        event.delta) {
+                    else if (event.type === 'response.reasoning_summary_text.delta' && event.delta) {
                         const itemId = event.item_id + '-' + event.summary_index;
                         let position = reasoningPositions.get(itemId) ?? 0;
                         reasoningAggregates.set(itemId, reasoningAggregates.get(itemId) + event.delta);
@@ -854,8 +806,7 @@ export class OpenAIProvider extends BaseModelProvider {
                         };
                         reasoningPositions.set(itemId, position);
                     }
-                    else if (event.type === 'response.reasoning_summary_text.done' &&
-                        event.text !== undefined) {
+                    else if (event.type === 'response.reasoning_summary_text.done' && event.text !== undefined) {
                         const itemId = event.item_id + '-' + event.summary_index;
                         const aggregatedThinking = event.text;
                         yield {
@@ -920,11 +871,221 @@ export class OpenAIProvider extends BaseModelProvider {
             console.error('Error in OpenAI streaming response:', error);
             yield {
                 type: 'error',
-                error: 'OpenAI streaming error: ' +
-                    (error instanceof Error ? error.stack : String(error)),
+                error: 'OpenAI streaming error: ' + (error instanceof Error ? error.stack : String(error)),
             };
         }
     }
+    async *createTranscription(audio, agent, model, opts) {
+        const transcriptionModels = ['gpt-4o-transcribe', 'gpt-4o-mini-transcribe', 'whisper-1'];
+        if (!transcriptionModels.includes(model)) {
+            throw new Error(`Model ${model} does not support transcription. Supported models: ${transcriptionModels.join(', ')}`);
+        }
+        let ws = null;
+        let isConnected = false;
+        let connectionError = null;
+        try {
+            const { WebSocket } = await import('ws');
+            const apiKey = this.apiKey || process.env.OPENAI_API_KEY;
+            if (!apiKey) {
+                throw new Error('Failed to initialize OpenAI transcription. Make sure OPENAI_API_KEY is set.');
+            }
+            const wsUrl = 'wss://api.openai.com/v1/realtime?intent=transcription';
+            ws = new WebSocket(wsUrl, {
+                headers: {
+                    Authorization: 'Bearer ' + apiKey,
+                    'OpenAI-Beta': 'realtime=v1',
+                },
+            });
+            const transcriptEvents = [];
+            const connectionPromise = new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => {
+                    reject(new Error('Connection timeout'));
+                }, 10000);
+                ws.on('open', () => {
+                    clearTimeout(timeout);
+                    console.log('[OpenAI] WebSocket connected for transcription');
+                    isConnected = true;
+                    resolve();
+                });
+                ws.on('error', error => {
+                    clearTimeout(timeout);
+                    connectionError = error;
+                    reject(error);
+                });
+            });
+            ws.on('message', (data) => {
+                try {
+                    const event = JSON.parse(data.toString());
+                    switch (event.type) {
+                        case 'transcription_session.created':
+                        case 'session.created': {
+                            const sessionUpdate = {
+                                type: 'transcription_session.update',
+                                input_audio_format: opts?.audioFormat?.encoding === 'pcm' ? 'pcm16' : 'pcm16',
+                                input_audio_transcription: {
+                                    model: model,
+                                    prompt: opts?.prompt || '',
+                                    language: opts?.language || '',
+                                },
+                                turn_detection: opts?.vad === false
+                                    ? null
+                                    : {
+                                        type: 'server_vad',
+                                        threshold: 0.5,
+                                        prefix_padding_ms: 300,
+                                        silence_duration_ms: 500,
+                                    },
+                                input_audio_noise_reduction: opts?.noiseReduction === null
+                                    ? null
+                                    : {
+                                        type: opts?.noiseReduction || 'near_field',
+                                    },
+                            };
+                            ws.send(JSON.stringify(sessionUpdate));
+                            break;
+                        }
+                        case 'conversation.item.input_audio_transcription.delta': {
+                            if (model !== 'whisper-1') {
+                                const deltaEvent = {
+                                    type: 'transcription_delta',
+                                    timestamp: new Date().toISOString(),
+                                    delta: event.delta,
+                                    partial: true,
+                                };
+                                transcriptEvents.push(deltaEvent);
+                            }
+                            break;
+                        }
+                        case 'conversation.item.input_audio_transcription.completed': {
+                            const completeText = event.transcript;
+                            if (model === 'whisper-1') {
+                                const deltaEvent = {
+                                    type: 'transcription_delta',
+                                    timestamp: new Date().toISOString(),
+                                    delta: completeText,
+                                    partial: false,
+                                };
+                                transcriptEvents.push(deltaEvent);
+                            }
+                            const turnEvent = {
+                                type: 'transcription_turn',
+                                timestamp: new Date().toISOString(),
+                                text: completeText,
+                            };
+                            transcriptEvents.push(turnEvent);
+                            break;
+                        }
+                        case 'input_audio_buffer.speech_started': {
+                            const previewEvent = {
+                                type: 'transcription_preview',
+                                timestamp: new Date().toISOString(),
+                                text: '',
+                                isFinal: false,
+                            };
+                            transcriptEvents.push(previewEvent);
+                            break;
+                        }
+                        case 'input_audio_buffer.speech_stopped': {
+                            break;
+                        }
+                        case 'error': {
+                            const errorEvent = {
+                                type: 'error',
+                                timestamp: new Date().toISOString(),
+                                error: event.error?.message || 'Unknown error',
+                            };
+                            transcriptEvents.push(errorEvent);
+                            break;
+                        }
+                    }
+                }
+                catch (error) {
+                    console.error('[OpenAI] Error processing message:', error);
+                }
+            });
+            ws.on('close', () => {
+                console.log('[OpenAI] WebSocket closed');
+                isConnected = false;
+            });
+            await connectionPromise;
+            const audioStream = normalizeAudioSource(audio);
+            const reader = audioStream.getReader();
+            try {
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done)
+                        break;
+                    if (value && ws && isConnected) {
+                        const audioEvent = {
+                            type: 'input_audio_buffer.append',
+                            audio: Buffer.from(value).toString('base64'),
+                        };
+                        ws.send(JSON.stringify(audioEvent));
+                    }
+                    if (transcriptEvents.length > 0) {
+                        const events = transcriptEvents.splice(0, transcriptEvents.length);
+                        for (const event of events) {
+                            yield event;
+                        }
+                    }
+                    if (connectionError) {
+                        throw connectionError;
+                    }
+                }
+                if (opts?.vad === false && ws && isConnected) {
+                    ws.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
+                }
+                await new Promise(resolve => setTimeout(resolve, 1000));
+                if (transcriptEvents.length > 0) {
+                    const events = transcriptEvents.splice(0, transcriptEvents.length);
+                    for (const event of events) {
+                        yield event;
+                    }
+                }
+                const completeEvent = {
+                    type: 'transcription_complete',
+                    timestamp: new Date().toISOString(),
+                };
+                yield completeEvent;
+            }
+            finally {
+                reader.releaseLock();
+                if (ws && ws.readyState === ws.OPEN) {
+                    ws.close();
+                }
+            }
+        }
+        catch (error) {
+            console.error('[OpenAI] Transcription error:', error);
+            const errorEvent = {
+                type: 'error',
+                timestamp: new Date().toISOString(),
+                error: error instanceof Error ? error.message : 'Transcription failed',
+            };
+            yield errorEvent;
+        }
+    }
+}
+function normalizeAudioSource(source) {
+    if (source instanceof ReadableStream) {
+        return source;
+    }
+    if (typeof source === 'object' && source !== null && Symbol.asyncIterator in source) {
+        return new ReadableStream({
+            async start(controller) {
+                try {
+                    for await (const chunk of source) {
+                        controller.enqueue(chunk);
+                    }
+                    controller.close();
+                }
+                catch (error) {
+                    controller.error(error);
+                }
+            },
+        });
+    }
+    throw new Error('Invalid audio source type');
 }
 export const openaiProvider = new OpenAIProvider();
 //# sourceMappingURL=openai.js.map