@crewdle/mist-connector-openai 1.0.13 → 1.0.15

package/dist/models/OpenAIFileConnector.js

@@ -22,6 +22,6 @@ export class OpenAIFileConnector {
         return uploadedFile.id;
     }
     async deleteFile(fileId) {
-        await this.client.files.del(fileId);
+        await this.client.files.delete(fileId);
     }
 }

package/dist/models/OpenAIGenerativeAIWorkerConnector.js

@@ -1,4 +1,5 @@
 import OpenAI from 'openai';
+import { GenerativeAIEngineType, GenerativeAITaskType } from '@crewdle/web-sdk-types';
 export class OpenAIGenerativeAIWorkerConnector {
     models = new Map();
     apiKey;
@@ -22,7 +23,7 @@ export class OpenAIGenerativeAIWorkerConnector {
         this.models.clear();
     }
     getEngineType() {
-        return 'openai';
+        return GenerativeAIEngineType.OpenAI;
     }
     async processJob(parameters, options) {
         if (!this.client) {
@@ -31,7 +32,7 @@ export class OpenAIGenerativeAIWorkerConnector {
         if (!options || !this.models.has(options.model.id)) {
             throw new Error('Model not initialized');
         }
-        if (options.model.outputType === 'vector') {
+        if (options.model.taskType === GenerativeAITaskType.Embeddings) {
             const response = await this.client.embeddings.create({
                 model: options.model.id,
                 input: parameters.prompt,
@@ -41,42 +42,131 @@ export class OpenAIGenerativeAIWorkerConnector {
                 throw new Error('No response data');
             }
             return {
-                type: 'prompt',
+                type: "prompt" /* GenerativeAIJobType.Prompt */,
                 output: response.data[0].embedding,
                 inputTokens: response.usage?.prompt_tokens ?? 0,
                 outputTokens: 0,
             };
         }
+        if (options.model.taskType === GenerativeAITaskType.SpeechToText) {
+            if (typeof parameters.prompt === 'string') {
+                throw new Error('Prompt must be an array of messages');
+            }
+            let file;
+            for (const message of parameters.prompt) {
+                if (message.type === 'input_audio') {
+                    file = new File([message.input_audio.data], 'audio.mp3', { type: message.input_audio.format });
+                }
+            }
+            if (!file) {
+                throw new Error('No file found');
+            }
+            console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription');
+            const response = await this.client.audio.transcriptions.create({
+                model: options.model.id,
+                file,
+                response_format: 'json',
+            });
+            console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription response');
+            return {
+                type: "prompt" /* GenerativeAIJobType.Prompt */,
+                output: response.text,
+                inputTokens: response.usage?.type === 'tokens' ? response.usage.input_tokens : 0,
+                outputTokens: response.usage?.type === 'tokens' ? response.usage.output_tokens : 0,
+            };
+        }
+        if (options.model.taskType === GenerativeAITaskType.ImageGeneration) {
+            const images = [];
+            let prompt = parameters.prompt;
+            if (typeof parameters.prompt !== 'string') {
+                for (const message of parameters.prompt) {
+                    if (message.type === 'image_url') {
+                        const parts = message.image_url.url.split(',');
+                        const data = parts[1];
+                        const type = parts[0].replace('data:', '').replace(';base64', '');
+                        const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
+                        const blob = new Blob([byteArray], { type });
+                        images.push(new File([blob], `image.${type.split('/')[1]}`, { type }));
+                    }
+                    if (message.type === 'text') {
+                        prompt = message.text;
+                    }
+                }
+            }
+            if (images.length === 0) {
+                console.log('OpenAIGenerativeAIWorkerConnector.processJob image generation');
+                const response = await this.client.images.generate({
+                    model: options.model.id,
+                    prompt,
+                });
+                console.log('OpenAIGenerativeAIWorkerConnector.processJob image generation response');
+                return {
+                    type: "prompt" /* GenerativeAIJobType.Prompt */,
+                    output: '',
+                    resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
+                    inputTokens: response.usage?.input_tokens ?? 0,
+                    outputTokens: response.usage?.output_tokens ?? 0,
+                };
+            }
+            else {
+                console.log('OpenAIGenerativeAIWorkerConnector.processJob image edit');
+                const response = await this.client.images.edit({
+                    model: options.model.id,
+                    image: images.length === 1 ? images[0] : images,
+                    prompt,
+                });
+                console.log('OpenAIGenerativeAIWorkerConnector.processJob image edit response');
+                return {
+                    type: "prompt" /* GenerativeAIJobType.Prompt */,
+                    output: '',
+                    resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
+                    inputTokens: response.usage?.input_tokens ?? 0,
+                    outputTokens: response.usage?.output_tokens ?? 0,
+                };
+            }
+        }
         const responseFormat = this.getResponseFormat(parameters);
         const tools = this.getTools(parameters);
         const messages = this.getMessages(parameters);
-        let totalInputTokens = 0;
-        let totalOutputTokens = 0;
-        let accumulatedOutput = '';
+        let inputTokens = 0;
+        let outputTokens = 0;
+        let output = '';
         while (true) {
-            const response = await this.client.chat.completions.create({
+            console.log('OpenAIGenerativeAIWorkerConnector.processJob', options.model.id);
+            const response = await this.client.responses.create({
                 model: options.model.id,
-                messages,
-                max_completion_tokens: parameters.maxTokens,
+                input: messages,
+                max_output_tokens: parameters.maxTokens,
                 temperature: parameters.temperature,
-                response_format: responseFormat,
+                text: responseFormat,
                 tools,
             });
-            const message = response.choices[0].message;
-            totalInputTokens += response.usage?.prompt_tokens ?? 0;
-            totalOutputTokens += response.usage?.completion_tokens ?? 0;
-            if (!message.tool_calls) {
-                accumulatedOutput += message.content ?? '';
-                return {
-                    type: 'prompt',
-                    output: accumulatedOutput,
-                    inputTokens: totalInputTokens,
-                    outputTokens: totalOutputTokens,
-                };
-            }
+            console.log('OpenAIGenerativeAIWorkerConnector.processJob response');
+            inputTokens += response.usage?.input_tokens ?? 0;
+            outputTokens += response.usage?.output_tokens ?? 0;
             const promises = [];
-            this.processToolCalls(promises, message.tool_calls, messages, parameters.functions);
-            await Promise.all(promises);
+            for (const content of response.output) {
+                if (content.type === 'message') {
+                    for (const message of content.content) {
+                        if (message.type === 'output_text') {
+                            output += message.text;
+                        }
+                    }
+                }
+                if (content.type === 'function_call') {
+                    promises.push(this.processToolCall(parameters, messages, content.name, content.call_id, content.arguments));
+                }
+            }
+            if (promises.length > 0) {
+                await Promise.all(promises);
+                continue;
+            }
+            return {
+                type: "prompt" /* GenerativeAIJobType.Prompt */,
+                output,
+                inputTokens,
+                outputTokens,
+            };
         }
     }
     async *processJobStream(parameters, options) {
@@ -86,138 +176,193 @@ export class OpenAIGenerativeAIWorkerConnector {
         if (!options || !this.models.has(options.model.id)) {
             throw new Error('Model not initialized');
         }
+        if (options.model.taskType === GenerativeAITaskType.SpeechToText) {
+            if (typeof parameters.prompt === 'string') {
+                throw new Error('Prompt must be an array of messages');
+            }
+            let file;
+            for (const message of parameters.prompt) {
+                if (message.type === 'input_audio') {
+                    const dataParts = message.input_audio.data.split(',');
+                    const data = dataParts[1];
+                    const type = dataParts[0].replace('data:', '').replace(';base64', '');
+                    const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
+                    const blob = new Blob([byteArray], { type });
+                    file = new File([blob], `audio.${message.input_audio.format}`, { type });
+                }
+            }
+            if (!file) {
+                throw new Error('No file found');
+            }
+            console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription');
+            const stream = await this.client.audio.transcriptions.create({
+                model: options.model.id,
+                file,
+                response_format: 'json',
+                stream: true,
+            });
+            console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription response');
+            for await (const chunk of stream) {
+                yield {
+                    type: "prompt" /* GenerativeAIJobType.Prompt */,
+                    output: chunk.type === 'transcript.text.delta' ? chunk.delta : '',
+                    inputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.input_tokens ?? 0,
+                    outputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.output_tokens ?? 0,
+                };
+            }
+            return;
+        }
+        if (options.model.taskType === GenerativeAITaskType.ImageGeneration) {
+            const images = [];
+            let prompt = parameters.prompt;
+            if (typeof parameters.prompt !== 'string') {
+                for (const message of parameters.prompt) {
+                    if (message.type === 'image_url') {
+                        const parts = message.image_url.url.split(',');
+                        const data = parts[1];
+                        const type = parts[0].replace('data:', '').replace(';base64', '');
+                        const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
+                        const blob = new Blob([byteArray], { type });
+                        images.push(new File([blob], `image.${type.split('/')[1]}`, { type }));
+                    }
+                    if (message.type === 'text') {
+                        prompt = message.text;
+                    }
+                }
+            }
+            if (images.length === 0) {
+                console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length === 0');
+                const response = await this.client.images.generate({
+                    model: options.model.id,
+                    prompt,
+                });
+                console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length === 0 response');
+                yield {
+                    type: "prompt" /* GenerativeAIJobType.Prompt */,
+                    output: '',
+                    resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
+                    inputTokens: response.usage?.input_tokens ?? 0,
+                    outputTokens: response.usage?.output_tokens ?? 0,
+                };
+                return;
+            }
+            else {
+                console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length > 0');
+                const response = await this.client.images.edit({
+                    model: options.model.id,
+                    image: images.length === 1 ? images[0] : images,
+                    prompt,
+                });
+                console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length > 0 response');
+                yield {
+                    type: "prompt" /* GenerativeAIJobType.Prompt */,
+                    output: '',
+                    resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
+                    inputTokens: response.usage?.input_tokens ?? 0,
+                    outputTokens: response.usage?.output_tokens ?? 0,
+                };
+                return;
+            }
+        }
         const responseFormat = this.getResponseFormat(parameters);
         const tools = this.getTools(parameters);
         const messages = this.getMessages(parameters);
         while (true) {
-            const stream = await this.client.chat.completions.create({
+            console.log('OpenAIGenerativeAIWorkerConnector.processJobStream', options.model.id);
+            const stream = await this.client.responses.create({
                 model: options.model.id,
-                messages,
-                max_completion_tokens: parameters.maxTokens,
+                input: messages,
+                max_output_tokens: parameters.maxTokens,
                 temperature: parameters.temperature,
-                stream: true,
-                stream_options: {
-                    include_usage: true,
-                },
+                text: responseFormat,
                 tools,
-                response_format: responseFormat,
+                stream: true,
             });
-            let message = {};
+            console.log('OpenAIGenerativeAIWorkerConnector.processJobStream response');
+            const promises = [];
             for await (const chunk of stream) {
-                if (chunk.choices[0]?.delta?.tool_calls) {
-                    message = this.messageReducer(message, chunk);
+                if (chunk.type === 'response.output_text.delta') {
+                    yield {
+                        type: "prompt" /* GenerativeAIJobType.Prompt */,
+                        output: chunk.delta,
+                        inputTokens: 0,
+                        outputTokens: 0,
+                    };
                 }
+                if (chunk.type === 'response.completed') {
+                    yield {
+                        type: "prompt" /* GenerativeAIJobType.Prompt */,
+                        output: '',
+                        inputTokens: chunk.response.usage?.input_tokens ?? 0,
+                        outputTokens: chunk.response.usage?.output_tokens ?? 0,
+                    };
+                }
+                if (chunk.type === 'response.output_item.done') {
+                    if (chunk.item.type === 'function_call') {
+                        promises.push(this.processToolCall(parameters, messages, chunk.item.name, chunk.item.call_id, chunk.item.arguments));
+                    }
+                }
+            }
+            if (promises.length > 0) {
+                await Promise.all(promises);
                 yield {
-                    type: 'prompt',
-                    output: chunk.choices[0]?.delta?.content ?? '',
-                    inputTokens: chunk.usage?.prompt_tokens ?? 0,
-                    outputTokens: chunk.usage?.completion_tokens ?? 0,
+                    type: "prompt" /* GenerativeAIJobType.Prompt */,
+                    output: '\n\n',
+                    inputTokens: 0,
+                    outputTokens: 0,
                 };
+                continue;
             }
-            if (!message.tool_calls) {
-                break;
-            }
-            const promises = [];
-            this.processToolCalls(promises, message.tool_calls, messages, parameters.functions);
-            await Promise.all(promises);
-            yield {
-                type: "prompt" /* GenerativeAIJobType.Prompt */,
-                output: '\n\n',
-                inputTokens: 0,
-                outputTokens: 0,
-            };
+            break;
         }
     }
-    processToolCalls(promises, toolCalls, messages, functions) {
+    async processToolCall(parameters, messages, toolName, toolCallId, input) {
         messages.push({
-            role: 'assistant',
-            content: null,
-            tool_calls: toolCalls,
-            refusal: null,
+            type: 'function_call',
+            arguments: input,
+            call_id: toolCallId,
+            name: toolName,
         });
-        for (const toolCall of toolCalls) {
-            if (toolCall?.function && functions) {
-                try {
-                    const func = functions.get(toolCall.function.name);
-                    if (func) {
-                        const result = func.callback(JSON.parse(toolCall.function.arguments));
-                        if (result instanceof Promise) {
-                            promises.push(result.then((res) => {
-                                messages.push({
-                                    role: 'tool',
-                                    tool_call_id: toolCall.id,
-                                    content: res ?? 'Tool does not exist',
-                                });
-                            }));
-                        }
-                        else {
-                            messages.push({
-                                role: 'tool',
-                                tool_call_id: toolCall.id,
-                                content: result ?? 'Tool does not exist',
-                            });
-                        }
-                    }
-                    else {
-                        messages.push({
-                            role: 'tool',
-                            tool_call_id: toolCall.id,
-                            content: 'Tool does not exist',
-                        });
-                    }
-                }
-                catch (error) {
-                    console.error('Error processing tool call', error);
-                    messages.push({
-                        role: 'tool',
-                        tool_call_id: toolCall.id,
-                        content: 'Error processing tool call',
-                    });
-                }
-            }
+        const func = parameters.functions?.get(toolName);
+        if (!func) {
+            messages.push({
+                type: 'function_call_output',
+                call_id: toolCallId,
+                output: 'Tool not found',
+            });
+            return;
         }
-    }
-    messageReducer(previous, item) {
-        const reduce = (acc, delta) => {
-            acc = { ...acc };
-            for (const [key, value] of Object.entries(delta)) {
-                if (acc[key] === undefined || acc[key] === null) {
-                    acc[key] = value;
-                    //  OpenAI.Chat.Completions.ChatCompletionMessageToolCall does not have a key, .index
-                    if (Array.isArray(acc[key])) {
-                        for (const arr of acc[key]) {
-                            delete arr.index;
-                        }
-                    }
-                }
-                else if (typeof acc[key] === 'string' && typeof value === 'string') {
-                    acc[key] += value;
-                }
-                else if (typeof acc[key] === 'number' && typeof value === 'number') {
-                    acc[key] = value;
-                }
-                else if (Array.isArray(acc[key]) && Array.isArray(value)) {
-                    const accArray = acc[key];
-                    for (let i = 0; i < value.length; i++) {
-                        const { index, ...chunkTool } = value[i];
-                        if (index - accArray.length > 1) {
-                            throw new Error(`Error: An array has an empty value when tool_calls are constructed. tool_calls: ${accArray}; tool: ${value}`);
-                        }
-                        accArray[index] = reduce(accArray[index], chunkTool);
-                    }
-                }
-                else if (typeof acc[key] === 'object' && typeof value === 'object') {
-                    acc[key] = reduce(acc[key], value);
-                }
+        try {
+            if (input === '') {
+                input = '{}';
+            }
+            console.log('OpenAIGenerativeAIWorkerConnector.processToolCall', toolName, input);
+            const result = func.callback(JSON.parse(input));
+            if (result instanceof Promise) {
+                const output = await result;
+                console.log('OpenAIGenerativeAIWorkerConnector.processToolCall result');
+                messages.push({
+                    type: 'function_call_output',
+                    call_id: toolCallId,
+                    output,
+                });
+            }
+            else {
+                messages.push({
+                    type: 'function_call_output',
+                    call_id: toolCallId,
+                    output: result,
+                });
             }
-            return acc;
-        };
-        const choice = item.choices[0];
-        if (!choice) {
-            // chunk contains information about usage and token counts
-            return previous;
         }
-        return reduce(previous, choice.delta);
+        catch (error) {
+            console.error('Error processing tool call', error);
+            messages.push({
+                type: 'function_call_output',
+                call_id: toolCallId,
+                output: 'Error processing tool call',
+            });
+        }
     }
     getMessages(parameters) {
         const messages = [];
@@ -237,31 +382,58 @@ export class OpenAIGenerativeAIWorkerConnector {
         }
         messages.push({
             role: 'user',
-            content: parameters.prompt,
+            content: this.getInnerMessages(parameters.prompt),
         });
         return messages;
     }
+    getInnerMessages(content) {
+        if (typeof content === 'string') {
+            return content;
+        }
+        return content.map((c) => {
+            if (c.type === 'text') {
+                return {
+                    type: 'input_text',
+                    text: c.text,
+                };
+            }
+            if (c.type === 'file') {
+                return {
+                    type: 'input_file',
+                    filename: c.file.filename,
+                    file_data: c.file.file_data,
+                };
+            }
+            if (c.type === 'image_url') {
+                return {
+                    type: 'input_image',
+                    image_url: c.image_url.url,
+                };
+            }
+            throw new Error('Invalid content type');
+        });
+    }
     getTools(parameters) {
         if (!parameters.functions || parameters.functions.size === 0) {
             return undefined;
         }
         const tools = [];
         for (const [name, func] of parameters.functions) {
-            let params;
+            let params = null;
             if (func.params) {
                 params = {
                     type: 'object',
                     properties: func.params,
                     required: Object.keys(func.params),
+                    additionalProperties: false,
                 };
             }
             tools.push({
                 type: 'function',
-                function: {
-                    name,
-                    description: func.description,
-                    parameters: params,
-                }
+                name,
+                description: func.description,
+                strict: true,
+                parameters: params,
             });
         }
         return tools;
@@ -271,33 +443,38 @@ export class OpenAIGenerativeAIWorkerConnector {
             return undefined;
         }
         if (parameters.grammar === 'json') {
-            return { type: 'json_object' };
+            return { format: { type: 'json_object' } };
         }
         if (parameters.grammar === 'if_else') {
             return {
-                type: 'json_schema',
-                json_schema: {
+                format: {
+                    type: 'json_schema',
+                    strict: true,
                     name: 'router_response',
                     schema: {
                         type: 'object',
                         properties: {
-                            'condition_result': {
+                            condition_result: {
                                 type: 'boolean',
+                                description: 'Indicates whether a certain condition is met.'
                             }
                         },
-                        required: ['result'],
-                        additionalProperties: false,
-                    }
+                        required: [
+                            'condition_result'
+                        ],
+                        additionalProperties: false
+                    },
                 }
             };
         }
         if (parameters.grammar !== 'default') {
             return {
-                type: 'json_schema',
-                json_schema: {
+                format: {
+                    type: 'json_schema',
+                    strict: true,
                     name: 'custom_response',
                     schema: parameters.grammar,
-                }
+                },
             };
         }
         return undefined;

package/dist/models/OpenAISpeechConnector.js

@@ -1,4 +1,5 @@
 import OpenAI from 'openai';
+import { GenerativeAIEngineType } from '@crewdle/web-sdk-types';
 export class OpenAISpeechConnector {
     client;
     constructor(apiKey) {
@@ -6,13 +7,13 @@ export class OpenAISpeechConnector {
             apiKey,
         });
     }
-    async speak(model, text, voice, instructions) {
+    async speak(model, text, voice, format, instructions) {
         const response = await this.client.audio.speech.create({
             model,
             voice: voice,
             input: text,
             instructions,
-            response_format: 'pcm',
+            response_format: format,
         });
         if (!response.body) {
             throw new Error('No response body');
@@ -30,6 +31,6 @@ export class OpenAISpeechConnector {
         });
     }
     getEngineType() {
-        return 'openai';
+        return GenerativeAIEngineType.OpenAI;
     }
 }

package/dist/types/models/OpenAIGenerativeAIWorkerConnector.d.ts

@@ -9,9 +9,9 @@ export declare class OpenAIGenerativeAIWorkerConnector implements IGenerativeAIW
     getEngineType(): GenerativeAIEngineType;
     processJob(parameters: GenerativeAIWorkerConnectorParameters, options?: IGenerativeAIWorkerOptions): Promise<IGenerativeAIWorkerConnectorPromptResult>;
     processJobStream(parameters: GenerativeAIWorkerConnectorParameters, options?: IGenerativeAIWorkerOptions): AsyncGenerator<IGenerativeAIWorkerConnectorPromptResult>;
-    private processToolCalls;
-    private messageReducer;
+    private processToolCall;
     private getMessages;
+    private getInnerMessages;
     private getTools;
     private getResponseFormat;
 }

package/dist/types/models/OpenAISpeechConnector.d.ts

@@ -2,6 +2,6 @@ import { GenerativeAIEngineType, IGenerativeAISpeechConnector } from '@crewdle/w
 export declare class OpenAISpeechConnector implements IGenerativeAISpeechConnector {
     private client;
     constructor(apiKey: string);
-    speak(model: string, text: string, voice: string, instructions?: string): Promise<ReadableStream>;
+    speak(model: string, text: string, voice: string, format: string, instructions?: string): Promise<ReadableStream>;
     getEngineType(): GenerativeAIEngineType;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewdle/mist-connector-openai",
-  "version": "1.0.13",
+  "version": "1.0.15",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/types/index.d.ts",
@@ -15,11 +15,11 @@
     "dist/"
   ],
   "devDependencies": {
-    "@crewdle/web-sdk-types": "^1.0.46",
+    "@crewdle/web-sdk-types": "^1.0.48",
     "@types/node": "^22.13.9",
     "typescript": "^5.8.2"
   },
   "dependencies": {
-    "openai": "^4.100.0"
+    "openai": "^5.8.2"
   }
 }