@crewdle/mist-connector-openai 1.0.14 → 1.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/dist/models/OpenAIGenerativeAIWorkerConnector.js +177 -4
  2. package/dist/models/OpenAISpeechConnector.js +4 -3
  3. package/dist/types/models/OpenAISpeechConnector.d.ts +1 -1
  4. package/package.json +3 -3
package/dist/models/OpenAIGenerativeAIWorkerConnector.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import OpenAI from 'openai';
2
+ import { GenerativeAIEngineType, GenerativeAITaskType } from '@crewdle/web-sdk-types';
2
3
  export class OpenAIGenerativeAIWorkerConnector {
3
4
  models = new Map();
4
5
  apiKey;
@@ -22,7 +23,7 @@ export class OpenAIGenerativeAIWorkerConnector {
22
23
  this.models.clear();
23
24
  }
24
25
  getEngineType() {
25
- return 'openai';
26
+ return GenerativeAIEngineType.OpenAI;
26
27
  }
27
28
  async processJob(parameters, options) {
28
29
  if (!this.client) {
@@ -31,7 +32,7 @@ export class OpenAIGenerativeAIWorkerConnector {
31
32
  if (!options || !this.models.has(options.model.id)) {
32
33
  throw new Error('Model not initialized');
33
34
  }
34
- if (options.model.outputType === 'vector') {
35
+ if (options.model.taskType === GenerativeAITaskType.Embeddings) {
35
36
  const response = await this.client.embeddings.create({
36
37
  model: options.model.id,
37
38
  input: parameters.prompt,
@@ -41,12 +42,89 @@ export class OpenAIGenerativeAIWorkerConnector {
41
42
  throw new Error('No response data');
42
43
  }
43
44
  return {
44
- type: 'prompt',
45
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
45
46
  output: response.data[0].embedding,
46
47
  inputTokens: response.usage?.prompt_tokens ?? 0,
47
48
  outputTokens: 0,
48
49
  };
49
50
  }
51
+ if (options.model.taskType === GenerativeAITaskType.SpeechToText) {
52
+ if (typeof parameters.prompt === 'string') {
53
+ throw new Error('Prompt must be an array of messages');
54
+ }
55
+ let file;
56
+ for (const message of parameters.prompt) {
57
+ if (message.type === 'input_audio') {
58
+ file = new File([message.input_audio.data], 'audio.mp3', { type: message.input_audio.format });
59
+ }
60
+ }
61
+ if (!file) {
62
+ throw new Error('No file found');
63
+ }
64
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription');
65
+ const response = await this.client.audio.transcriptions.create({
66
+ model: options.model.id,
67
+ file,
68
+ response_format: 'json',
69
+ });
70
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob audio transcription response');
71
+ return {
72
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
73
+ output: response.text,
74
+ inputTokens: response.usage?.type === 'tokens' ? response.usage.input_tokens : 0,
75
+ outputTokens: response.usage?.type === 'tokens' ? response.usage.output_tokens : 0,
76
+ };
77
+ }
78
+ if (options.model.taskType === GenerativeAITaskType.ImageGeneration) {
79
+ const images = [];
80
+ let prompt = parameters.prompt;
81
+ if (typeof parameters.prompt !== 'string') {
82
+ for (const message of parameters.prompt) {
83
+ if (message.type === 'image_url') {
84
+ const parts = message.image_url.url.split(',');
85
+ const data = parts[1];
86
+ const type = parts[0].replace('data:', '').replace(';base64', '');
87
+ const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
88
+ const blob = new Blob([byteArray], { type });
89
+ images.push(new File([blob], `image.${type.split('/')[1]}`, { type }));
90
+ }
91
+ if (message.type === 'text') {
92
+ prompt = message.text;
93
+ }
94
+ }
95
+ }
96
+ if (images.length === 0) {
97
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image generation');
98
+ const response = await this.client.images.generate({
99
+ model: options.model.id,
100
+ prompt,
101
+ });
102
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image generation response');
103
+ return {
104
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
105
+ output: '',
106
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
107
+ inputTokens: response.usage?.input_tokens ?? 0,
108
+ outputTokens: response.usage?.output_tokens ?? 0,
109
+ };
110
+ }
111
+ else {
112
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image edit');
113
+ const response = await this.client.images.edit({
114
+ model: options.model.id,
115
+ image: images.length === 1 ? images[0] : images,
116
+ prompt,
117
+ });
118
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob image edit response');
119
+ return {
120
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
121
+ output: '',
122
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
123
+ inputTokens: response.usage?.input_tokens ?? 0,
124
+ outputTokens: response.usage?.output_tokens ?? 0,
125
+ };
126
+ }
127
+ }
50
128
  const responseFormat = this.getResponseFormat(parameters);
51
129
  const tools = this.getTools(parameters);
52
130
  const messages = this.getMessages(parameters);
@@ -54,6 +132,7 @@ export class OpenAIGenerativeAIWorkerConnector {
54
132
  let outputTokens = 0;
55
133
  let output = '';
56
134
  while (true) {
135
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob', options.model.id);
57
136
  const response = await this.client.responses.create({
58
137
  model: options.model.id,
59
138
  input: messages,
@@ -62,6 +141,7 @@ export class OpenAIGenerativeAIWorkerConnector {
62
141
  text: responseFormat,
63
142
  tools,
64
143
  });
144
+ console.log('OpenAIGenerativeAIWorkerConnector.processJob response');
65
145
  inputTokens += response.usage?.input_tokens ?? 0;
66
146
  outputTokens += response.usage?.output_tokens ?? 0;
67
147
  const promises = [];
@@ -96,10 +176,99 @@ export class OpenAIGenerativeAIWorkerConnector {
96
176
  if (!options || !this.models.has(options.model.id)) {
97
177
  throw new Error('Model not initialized');
98
178
  }
179
+ if (options.model.taskType === GenerativeAITaskType.SpeechToText) {
180
+ if (typeof parameters.prompt === 'string') {
181
+ throw new Error('Prompt must be an array of messages');
182
+ }
183
+ let file;
184
+ for (const message of parameters.prompt) {
185
+ if (message.type === 'input_audio') {
186
+ const dataParts = message.input_audio.data.split(',');
187
+ const data = dataParts[1];
188
+ const type = dataParts[0].replace('data:', '').replace(';base64', '');
189
+ const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
190
+ const blob = new Blob([byteArray], { type });
191
+ file = new File([blob], `audio.${message.input_audio.format}`, { type });
192
+ }
193
+ }
194
+ if (!file) {
195
+ throw new Error('No file found');
196
+ }
197
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription');
198
+ const stream = await this.client.audio.transcriptions.create({
199
+ model: options.model.id,
200
+ file,
201
+ response_format: 'json',
202
+ stream: true,
203
+ });
204
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream audio transcription response');
205
+ for await (const chunk of stream) {
206
+ yield {
207
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
208
+ output: chunk.type === 'transcript.text.delta' ? chunk.delta : '',
209
+ inputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.input_tokens ?? 0,
210
+ outputTokens: chunk.type === 'transcript.text.done' ? 0 : chunk.usage?.output_tokens ?? 0,
211
+ };
212
+ }
213
+ return;
214
+ }
215
+ if (options.model.taskType === GenerativeAITaskType.ImageGeneration) {
216
+ const images = [];
217
+ let prompt = parameters.prompt;
218
+ if (typeof parameters.prompt !== 'string') {
219
+ for (const message of parameters.prompt) {
220
+ if (message.type === 'image_url') {
221
+ const parts = message.image_url.url.split(',');
222
+ const data = parts[1];
223
+ const type = parts[0].replace('data:', '').replace(';base64', '');
224
+ const byteArray = Uint8Array.from(Buffer.from(data, 'base64'));
225
+ const blob = new Blob([byteArray], { type });
226
+ images.push(new File([blob], `image.${type.split('/')[1]}`, { type }));
227
+ }
228
+ if (message.type === 'text') {
229
+ prompt = message.text;
230
+ }
231
+ }
232
+ }
233
+ if (images.length === 0) {
234
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length === 0');
235
+ const response = await this.client.images.generate({
236
+ model: options.model.id,
237
+ prompt,
238
+ });
239
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length === 0 response');
240
+ yield {
241
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
242
+ output: '',
243
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
244
+ inputTokens: response.usage?.input_tokens ?? 0,
245
+ outputTokens: response.usage?.output_tokens ?? 0,
246
+ };
247
+ return;
248
+ }
249
+ else {
250
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length > 0');
251
+ const response = await this.client.images.edit({
252
+ model: options.model.id,
253
+ image: images.length === 1 ? images[0] : images,
254
+ prompt,
255
+ });
256
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream images.length > 0 response');
257
+ yield {
258
+ type: "prompt" /* GenerativeAIJobType.Prompt */,
259
+ output: '',
260
+ resultFile: `data:image/png;base64,${response.data?.[0]?.b64_json ?? ''}`,
261
+ inputTokens: response.usage?.input_tokens ?? 0,
262
+ outputTokens: response.usage?.output_tokens ?? 0,
263
+ };
264
+ return;
265
+ }
266
+ }
99
267
  const responseFormat = this.getResponseFormat(parameters);
100
268
  const tools = this.getTools(parameters);
101
269
  const messages = this.getMessages(parameters);
102
270
  while (true) {
271
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream', options.model.id);
103
272
  const stream = await this.client.responses.create({
104
273
  model: options.model.id,
105
274
  input: messages,
@@ -109,6 +278,7 @@ export class OpenAIGenerativeAIWorkerConnector {
109
278
  tools,
110
279
  stream: true,
111
280
  });
281
+ console.log('OpenAIGenerativeAIWorkerConnector.processJobStream response');
112
282
  const promises = [];
113
283
  for await (const chunk of stream) {
114
284
  if (chunk.type === 'response.output_text.delta') {
@@ -166,12 +336,15 @@ export class OpenAIGenerativeAIWorkerConnector {
166
336
  if (input === '') {
167
337
  input = '{}';
168
338
  }
339
+ console.log('OpenAIGenerativeAIWorkerConnector.processToolCall', toolName, input);
169
340
  const result = func.callback(JSON.parse(input));
170
341
  if (result instanceof Promise) {
342
+ const output = await result;
343
+ console.log('OpenAIGenerativeAIWorkerConnector.processToolCall result');
171
344
  messages.push({
172
345
  type: 'function_call_output',
173
346
  call_id: toolCallId,
174
- output: await result,
347
+ output,
175
348
  });
176
349
  }
177
350
  else {
package/dist/models/OpenAISpeechConnector.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import OpenAI from 'openai';
2
+ import { GenerativeAIEngineType } from '@crewdle/web-sdk-types';
2
3
  export class OpenAISpeechConnector {
3
4
  client;
4
5
  constructor(apiKey) {
@@ -6,13 +7,13 @@ export class OpenAISpeechConnector {
6
7
  apiKey,
7
8
  });
8
9
  }
9
- async speak(model, text, voice, instructions) {
10
+ async speak(model, text, voice, format, instructions) {
10
11
  const response = await this.client.audio.speech.create({
11
12
  model,
12
13
  voice: voice,
13
14
  input: text,
14
15
  instructions,
15
- response_format: 'pcm',
16
+ response_format: format,
16
17
  });
17
18
  if (!response.body) {
18
19
  throw new Error('No response body');
@@ -30,6 +31,6 @@ export class OpenAISpeechConnector {
30
31
  });
31
32
  }
32
33
  getEngineType() {
33
- return 'openai';
34
+ return GenerativeAIEngineType.OpenAI;
34
35
  }
35
36
  }
package/dist/types/models/OpenAISpeechConnector.d.ts CHANGED
@@ -2,6 +2,6 @@ import { GenerativeAIEngineType, IGenerativeAISpeechConnector } from '@crewdle/w
2
2
  export declare class OpenAISpeechConnector implements IGenerativeAISpeechConnector {
3
3
  private client;
4
4
  constructor(apiKey: string);
5
- speak(model: string, text: string, voice: string, instructions?: string): Promise<ReadableStream>;
5
+ speak(model: string, text: string, voice: string, format: string, instructions?: string): Promise<ReadableStream>;
6
6
  getEngineType(): GenerativeAIEngineType;
7
7
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crewdle/mist-connector-openai",
3
- "version": "1.0.14",
3
+ "version": "1.0.15",
4
4
  "description": "",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/types/index.d.ts",
@@ -15,11 +15,11 @@
15
15
  "dist/"
16
16
  ],
17
17
  "devDependencies": {
18
- "@crewdle/web-sdk-types": "^1.0.46",
18
+ "@crewdle/web-sdk-types": "^1.0.48",
19
19
  "@types/node": "^22.13.9",
20
20
  "typescript": "^5.8.2"
21
21
  },
22
22
  "dependencies": {
23
- "openai": "^5.5.1"
23
+ "openai": "^5.8.2"
24
24
  }
25
25
  }