@crewdle/mist-connector-openai 1.0.19 → 1.0.21

package/dist/models/OpenAIGenerativeAIWorkerConnector.js

@@ -55,7 +55,12 @@ export class OpenAIGenerativeAIWorkerConnector {
             let file;
             for (const message of parameters.prompt) {
                 if (message.type === 'input_audio') {
-                    file = new File([message.input_audio.data], 'audio.mp3', { type: message.input_audio.format });
+                    const dataParts = message.input_audio.data.split(',');
+                    const data = dataParts[1];
+                    const type = dataParts[0].replace('data:', '').replace(';base64', '');
+                    const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
+                    const blob = new Blob([byteArray], { type });
+                    file = new File([blob], `audio.${message.input_audio.format}`, { type });
                 }
             }
             if (!file) {
@@ -72,7 +77,7 @@ export class OpenAIGenerativeAIWorkerConnector {
             console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription response');
             return {
                 type: "prompt" /* GenerativeAIJobType.Prompt */,
-                output: response.text,
+                output: (!parameters.responseFormat || parameters.responseFormat === 'json') ? response.text : JSON.stringify(response),
                 inputTokens: response.usage?.type === 'tokens' ? response.usage.input_tokens : 0,
                 outputTokens: response.usage?.type === 'tokens' ? response.usage.output_tokens : 0,
             };
@@ -127,9 +132,10 @@ export class OpenAIGenerativeAIWorkerConnector {
                 };
             }
         }
-        const responseFormat = this.getResponseFormat(parameters);
+        const responseFormat = this.getResponseFormat(parameters, options.model.id);
         const tools = this.getTools(parameters);
         const messages = this.getMessages(parameters);
+        const reasoning = this.getReasoning(parameters, options.model.id);
         let inputTokens = 0;
         let outputTokens = 0;
         let output = '';
@@ -143,6 +149,7 @@ export class OpenAIGenerativeAIWorkerConnector {
                 temperature: parameters.temperature,
                 text: responseFormat,
                 tools,
+                reasoning,
                 store: parameters.privacy === true ? false : true,
             });
             console.log('OpenAIGenerativeAIWorkerConnector.processJob response');
@@ -203,21 +210,37 @@ export class OpenAIGenerativeAIWorkerConnector {
                 throw new Error('No file found');
             }
             console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription');
-            const stream = await this.client.audio.transcriptions.create({
-                model: options.model.id,
-                file,
-                response_format: parameters.responseFormat ? parameters.responseFormat : 'json',
-                language: parameters.language,
-                timestamp_granularities: parameters.timestampGranularities ? [parameters.timestampGranularities] : undefined,
-                stream: true,
-            });
-            console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription response');
-            for await (const chunk of stream) {
+            if (!parameters.responseFormat || parameters.responseFormat === 'json') {
+                const stream = await this.client.audio.transcriptions.create({
+                    model: options.model.id,
+                    file,
+                    response_format: 'json',
+                    language: parameters.language,
+                    stream: true,
+                });
+                console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription response');
+                for await (const chunk of stream) {
+                    yield {
+                        type: "prompt" /* GenerativeAIJobType.Prompt */,
+                        output: chunk.type === 'transcript.text.delta' ? chunk.delta : '',
+                        inputTokens: chunk.type === 'transcript.text.done' ? chunk.usage?.input_tokens ?? 0 : 0,
+                        outputTokens: chunk.type === 'transcript.text.done' ? chunk.usage?.output_tokens ?? 0 : 0,
+                    };
+                }
+            }
+            else {
+                const response = await this.client.audio.transcriptions.create({
+                    model: options.model.id,
+                    file,
+                    response_format: parameters.responseFormat ? parameters.responseFormat : 'json',
+                    language: parameters.language,
+                    timestamp_granularities: parameters.timestampGranularities ? [parameters.timestampGranularities] : undefined,
+                });
                 yield {
                     type: "prompt" /* GenerativeAIJobType.Prompt */,
-                    output: chunk.type === 'transcript.text.delta' ? chunk.delta : '',
-                    inputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.input_tokens ?? 0,
-                    outputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.output_tokens ?? 0,
+                    output: JSON.stringify(response),
+                    inputTokens: 0,
+                    outputTokens: 0,
                 };
             }
             return;
@@ -274,9 +297,11 @@ export class OpenAIGenerativeAIWorkerConnector {
                 return;
             }
         }
-        const responseFormat = this.getResponseFormat(parameters);
+        const responseFormat = this.getResponseFormat(parameters, options.model.id);
         const tools = this.getTools(parameters);
         const messages = this.getMessages(parameters);
+        const reasoning = this.getReasoning(parameters, options.model.id);
+        let firstChunk = true;
         while (true) {
             console.log('OpenAIGenerativeAIWorkerConnector.processJobStream', options.model.id);
             const stream = await this.client.responses.create({
@@ -286,6 +311,7 @@ export class OpenAIGenerativeAIWorkerConnector {
                 temperature: parameters.temperature,
                 text: responseFormat,
                 tools,
+                reasoning,
                 stream: true,
                 store: parameters.privacy === true ? false : true,
             });
@@ -293,6 +319,10 @@ export class OpenAIGenerativeAIWorkerConnector {
             const promises = [];
             for await (const chunk of stream) {
                 if (chunk.type === 'response.output_text.delta') {
+                    if (firstChunk && chunk.delta.trim()) {
+                        console.log('OpenAIGenerativeAIWorkerConnector.processJobStream first chunk');
+                        firstChunk = false;
+                    }
                     yield {
                         type: "prompt" /* GenerativeAIJobType.Prompt */,
                         output: chunk.delta,
@@ -470,8 +500,22 @@ export class OpenAIGenerativeAIWorkerConnector {
         }
         return tools;
     }
-    getResponseFormat(parameters) {
-        if (!parameters.grammar) {
+    getReasoning(parameters, modelId) {
+        if (!modelId.startsWith('gpt-5')) {
+            return undefined;
+        }
+        if (!parameters.reasoning) {
+            return undefined;
+        }
+        return {
+            effort: parameters.reasoning,
+        };
+    }
+    getResponseFormat(parameters, modelId) {
+        if (!parameters.grammar || parameters.grammar === 'default') {
+            if (modelId.startsWith('gpt-5') && parameters.verbosity) {
+                return { verbosity: parameters.verbosity };
+            }
             return undefined;
         }
         if (parameters.grammar === 'json') {

package/dist/types/models/OpenAIGenerativeAIWorkerConnector.d.ts

@@ -13,5 +13,6 @@ export declare class OpenAIGenerativeAIWorkerConnector implements IGenerativeAIW
     private getMessages;
     private getInnerMessages;
     private getTools;
+    private getReasoning;
     private getResponseFormat;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewdle/mist-connector-openai",
-  "version": "1.0.19",
+  "version": "1.0.21",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/types/index.d.ts",
@@ -15,11 +15,11 @@
     "dist/"
   ],
   "devDependencies": {
-    "@crewdle/web-sdk-types": "^1.0.52",
+    "@crewdle/web-sdk-types": "^1.0.54",
     "@types/node": "^22.13.9",
     "typescript": "^5.8.2"
   },
   "dependencies": {
-    "openai": "^5.8.2"
+    "openai": "^6.1.0"
   }
 }