@crewdle/mist-connector-openai 1.0.14 → 1.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/dist/models/OpenAIGenerativeAIWorkerConnector.js +203 -4
  2. package/dist/models/OpenAISpeechConnector.js +4 -3
  3. package/dist/types/models/OpenAISpeechConnector.d.ts +1 -1
  4. package/package.json +3 -3
package/dist/models/OpenAIGenerativeAIWorkerConnector.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import OpenAI from 'openai';
2
+ import { GenerativeAIEngineType, GenerativeAITaskType } from '@crewdle/web-sdk-types';
2
3
  export class OpenAIGenerativeAIWorkerConnector {
3
4
  models = new Map();
4
5
  apiKey;
@@ -22,7 +23,7 @@ export class OpenAIGenerativeAIWorkerConnector {
22
23
  this.models.clear();
23
24
  }
24
25
  getEngineType() {
25
- return 'openai';
26
+ return GenerativeAIEngineType.OpenAI;
26
27
  }
27
28
  async processJob(parameters, options) {
28
29
  if (!this.client) {
@@ -31,7 +32,7 @@ export class OpenAIGenerativeAIWorkerConnector {
31
32
  if (!options || !this.models.has(options.model.id)) {
32
33
  throw new Error('Model not initialized');
33
34
  }
34
- if (options.model.outputType === 'vector') {
35
+ if (options.model.taskType === GenerativeAITaskType.Embeddings) {
35
36
  const response = await this.client.embeddings.create({
36
37
  model: options.model.id,
37
38
  input: parameters.prompt,
@@ -41,19 +42,98 @@ export class OpenAIGenerativeAIWorkerConnector {
41
42
  throw new Error('No response data');
42
43
  }
43
44
  return {
44
- type: 'prompt',
45
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
45
46
  output: response.data[0].embedding,
46
47
  inputTokens: response.usage?.prompt_tokens ?? 0,
47
48
  outputTokens: 0,
48
49
  };
49
50
  }
51
+ if (options.model.taskType === GenerativeAITaskType.SpeechToText) {
52
+ if (typeof parameters.prompt === 'string') {
53
+ throw new Error('Prompt must be an array of messages');
54
+ }
55
+ let file;
56
+ for (const message of parameters.prompt) {
57
+ if (message.type === 'input_audio') {
58
+ file = new File([message.input_audio.data], 'audio.mp3', { type: message.input_audio.format });
59
+ }
60
+ }
61
+ if (!file) {
62
+ throw new Error('No file found');
63
+ }
64
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription');
65
+ const response = await this.client.audio.transcriptions.create({
66
+ model: options.model.id,
67
+ file,
68
+ response_format: 'json',
69
+ });
70
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription response');
71
+ return {
72
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
73
+ output: response.text,
74
+ inputTokens: response.usage?.type === 'tokens' ? response.usage.input_tokens : 0,
75
+ outputTokens: response.usage?.type === 'tokens' ? response.usage.output_tokens : 0,
76
+ };
77
+ }
78
+ if (options.model.taskType === GenerativeAITaskType.ImageGeneration) {
79
+ const images = [];
80
+ let prompt = parameters.prompt;
81
+ if (typeof parameters.prompt !== 'string') {
82
+ for (const message of parameters.prompt) {
83
+ if (message.type === 'image_url') {
84
+ const parts = message.image_url.url.split(',');
85
+ const data = parts[1];
86
+ const type = parts[0].replace('data:', '').replace(';base64', '');
87
+ const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
88
+ const blob = new Blob([byteArray], { type });
89
+ images.push(new File([blob], `image.${type.split('/')[1]}`, { type }));
90
+ }
91
+ if (message.type === 'text') {
92
+ prompt = message.text;
93
+ }
94
+ }
95
+ }
96
+ if (images.length === 0) {
97
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image generation');
98
+ const response = await this.client.images.generate({
99
+ model: options.model.id,
100
+ prompt,
101
+ });
102
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image generation response');
103
+ return {
104
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
105
+ output: '',
106
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
107
+ inputTokens: response.usage?.input_tokens ?? 0,
108
+ outputTokens: response.usage?.output_tokens ?? 0,
109
+ };
110
+ }
111
+ else {
112
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image edit');
113
+ const response = await this.client.images.edit({
114
+ model: options.model.id,
115
+ image: images.length === 1 ? images[0] : images,
116
+ prompt,
117
+ });
118
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image edit response');
119
+ return {
120
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
121
+ output: '',
122
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
123
+ inputTokens: response.usage?.input_tokens ?? 0,
124
+ outputTokens: response.usage?.output_tokens ?? 0,
125
+ };
126
+ }
127
+ }
50
128
  const responseFormat = this.getResponseFormat(parameters);
51
129
  const tools = this.getTools(parameters);
52
130
  const messages = this.getMessages(parameters);
53
131
  let inputTokens = 0;
54
132
  let outputTokens = 0;
55
133
  let output = '';
134
+ let resultFile;
56
135
  while (true) {
136
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob', options.model.id);
57
137
  const response = await this.client.responses.create({
58
138
  model: options.model.id,
59
139
  input: messages,
@@ -62,6 +142,7 @@ export class OpenAIGenerativeAIWorkerConnector {
62
142
  text: responseFormat,
63
143
  tools,
64
144
  });
145
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob response');
65
146
  inputTokens += response.usage?.input_tokens ?? 0;
66
147
  outputTokens += response.usage?.output_tokens ?? 0;
67
148
  const promises = [];
@@ -73,6 +154,9 @@ export class OpenAIGenerativeAIWorkerConnector {
73
154
  }
74
155
  }
75
156
  }
157
+ if (content.type === 'image_generation_call' && content.result) {
158
+ resultFile = `data:image/png;base64,${content.result}`;
159
+ }
76
160
  if (content.type === 'function_call') {
77
161
  promises.push(this.processToolCall(parameters, messages, content.name, content.call_id, content.arguments));
78
162
  }
@@ -84,6 +168,7 @@ export class OpenAIGenerativeAIWorkerConnector {
84
168
  return {
85
169
  type: "prompt" /* GenerativeAIJobType.Prompt */,
86
170
  output,
171
+ resultFile,
87
172
  inputTokens,
88
173
  outputTokens,
89
174
  };
@@ -96,10 +181,99 @@ export class OpenAIGenerativeAIWorkerConnector {
96
181
  if (!options || !this.models.has(options.model.id)) {
97
182
  throw new Error('Model not initialized');
98
183
  }
184
+ if (options.model.taskType === GenerativeAITaskType.SpeechToText) {
185
+ if (typeof parameters.prompt === 'string') {
186
+ throw new Error('Prompt must be an array of messages');
187
+ }
188
+ let file;
189
+ for (const message of parameters.prompt) {
190
+ if (message.type === 'input_audio') {
191
+ const dataParts = message.input_audio.data.split(',');
192
+ const data = dataParts[1];
193
+ const type = dataParts[0].replace('data:', '').replace(';base64', '');
194
+ const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
195
+ const blob = new Blob([byteArray], { type });
196
+ file = new File([blob], `audio.${message.input_audio.format}`, { type });
197
+ }
198
+ }
199
+ if (!file) {
200
+ throw new Error('No file found');
201
+ }
202
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription');
203
+ const stream = await this.client.audio.transcriptions.create({
204
+ model: options.model.id,
205
+ file,
206
+ response_format: 'json',
207
+ stream: true,
208
+ });
209
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription response');
210
+ for await (const chunk of stream) {
211
+ yield {
212
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
213
+ output: chunk.type === 'transcript.text.delta' ? chunk.delta : '',
214
+ inputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.input_tokens ?? 0,
215
+ outputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.output_tokens ?? 0,
216
+ };
217
+ }
218
+ return;
219
+ }
220
+ if (options.model.taskType === GenerativeAITaskType.ImageGeneration) {
221
+ const images = [];
222
+ let prompt = parameters.prompt;
223
+ if (typeof parameters.prompt !== 'string') {
224
+ for (const message of parameters.prompt) {
225
+ if (message.type === 'image_url') {
226
+ const parts = message.image_url.url.split(',');
227
+ const data = parts[1];
228
+ const type = parts[0].replace('data:', '').replace(';base64', '');
229
+ const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
230
+ const blob = new Blob([byteArray], { type });
231
+ images.push(new File([blob], `image.${type.split('/')[1]}`, { type }));
232
+ }
233
+ if (message.type === 'text') {
234
+ prompt = message.text;
235
+ }
236
+ }
237
+ }
238
+ if (images.length === 0) {
239
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length === 0');
240
+ const response = await this.client.images.generate({
241
+ model: options.model.id,
242
+ prompt,
243
+ });
244
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length === 0 response');
245
+ yield {
246
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
247
+ output: '',
248
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
249
+ inputTokens: response.usage?.input_tokens ?? 0,
250
+ outputTokens: response.usage?.output_tokens ?? 0,
251
+ };
252
+ return;
253
+ }
254
+ else {
255
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length > 0');
256
+ const response = await this.client.images.edit({
257
+ model: options.model.id,
258
+ image: images.length === 1 ? images[0] : images,
259
+ prompt,
260
+ });
261
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length > 0 response');
262
+ yield {
263
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
264
+ output: '',
265
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
266
+ inputTokens: response.usage?.input_tokens ?? 0,
267
+ outputTokens: response.usage?.output_tokens ?? 0,
268
+ };
269
+ return;
270
+ }
271
+ }
99
272
  const responseFormat = this.getResponseFormat(parameters);
100
273
  const tools = this.getTools(parameters);
101
274
  const messages = this.getMessages(parameters);
102
275
  while (true) {
276
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream', options.model.id);
103
277
  const stream = await this.client.responses.create({
104
278
  model: options.model.id,
105
279
  input: messages,
@@ -109,6 +283,7 @@ export class OpenAIGenerativeAIWorkerConnector {
109
283
  tools,
110
284
  stream: true,
111
285
  });
286
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream response');
112
287
  const promises = [];
113
288
  for await (const chunk of stream) {
114
289
  if (chunk.type === 'response.output_text.delta') {
@@ -131,6 +306,15 @@ export class OpenAIGenerativeAIWorkerConnector {
131
306
  if (chunk.item.type === 'function_call') {
132
307
  promises.push(this.processToolCall(parameters, messages, chunk.item.name, chunk.item.call_id, chunk.item.arguments));
133
308
  }
309
+ if (chunk.item.type === 'image_generation_call' && chunk.item.result) {
310
+ yield {
311
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
312
+ output: '',
313
+ resultFile: `data:image/png;base64,${chunk.item.result}`,
314
+ inputTokens: 0,
315
+ outputTokens: 0,
316
+ };
317
+ }
134
318
  }
135
319
  }
136
320
  if (promises.length > 0) {
@@ -166,12 +350,15 @@ export class OpenAIGenerativeAIWorkerConnector {
166
350
  if (input === '') {
167
351
  input = '{}';
168
352
  }
353
+ console.log('OpenAIGenerativeAIWorkerConnector.processToolCall', toolName, input);
169
354
  const result = func.callback(JSON.parse(input));
170
355
  if (result instanceof Promise) {
356
+ const output = await result;
357
+ console.log('OpenAIGenerativeAIWorkerConnector.processToolCall result');
171
358
  messages.push({
172
359
  type: 'function_call_output',
173
360
  call_id: toolCallId,
174
- output: await result,
361
+ output,
175
362
  });
176
363
  }
177
364
  else {
@@ -246,6 +433,18 @@ export class OpenAIGenerativeAIWorkerConnector {
246
433
  }
247
434
  const tools = [];
248
435
  for (const [name, func] of parameters.functions) {
436
+ if (func.type === 'mcp') {
437
+ tools.push(func);
438
+ continue;
439
+ }
440
+ if (func.type === 'code_interpreter') {
441
+ tools.push(func);
442
+ continue;
443
+ }
444
+ if (func.type === 'image_generation') {
445
+ tools.push(func);
446
+ continue;
447
+ }
249
448
  let params = null;
250
449
  if (func.params) {
251
450
  params = {
package/dist/models/OpenAISpeechConnector.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import OpenAI from 'openai';
2
+ import { GenerativeAIEngineType } from '@crewdle/web-sdk-types';
2
3
  export class OpenAISpeechConnector {
3
4
  client;
4
5
  constructor(apiKey) {
@@ -6,13 +7,13 @@ export class OpenAISpeechConnector {
6
7
  apiKey,
7
8
  });
8
9
  }
9
- async speak(model, text, voice, instructions) {
10
+ async speak(model, text, voice, format, instructions) {
10
11
  const response = await this.client.audio.speech.create({
11
12
  model,
12
13
  voice: voice,
13
14
  input: text,
14
15
  instructions,
15
- response_format: 'pcm',
16
+ response_format: format,
16
17
  });
17
18
  if (!response.body) {
18
19
  throw new Error('No response body');
@@ -30,6 +31,6 @@ export class OpenAISpeechConnector {
30
31
  });
31
32
  }
32
33
  getEngineType() {
33
- return 'openai';
34
+ return GenerativeAIEngineType.OpenAI;
34
35
  }
35
36
  }
package/dist/types/models/OpenAISpeechConnector.d.ts CHANGED
@@ -2,6 +2,6 @@ import { GenerativeAIEngineType, IGenerativeAISpeechConnector } from '@crewdle/w
2
2
  export declare class OpenAISpeechConnector implements IGenerativeAISpeechConnector {
3
3
  private client;
4
4
  constructor(apiKey: string);
5
- speak(model: string, text: string, voice: string, instructions?: string): Promise<ReadableStream>;
5
+ speak(model: string, text: string, voice: string, format: string, instructions?: string): Promise<ReadableStream>;
6
6
  getEngineType(): GenerativeAIEngineType;
7
7
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crewdle/mist-connector-openai",
3
- "version": "1.0.14",
3
+ "version": "1.0.16",
4
4
  "description": "",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/types/index.d.ts",
@@ -15,11 +15,11 @@
15
15
  "dist/"
16
16
  ],
17
17
  "devDependencies": {
18
- "@crewdle/web-sdk-types": "^1.0.46",
18
+ "@crewdle/web-sdk-types": "^1.0.50",
19
19
  "@types/node": "^22.13.9",
20
20
  "typescript": "^5.8.2"
21
21
  },
22
22
  "dependencies": {
23
- "openai": "^5.5.1"
23
+ "openai": "^5.8.2"
24
24
  }
25
25
  }