@micdrop/openai 1.4.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -145,17 +145,374 @@ var OpenaiAgent = class extends Agent {
145
145
  }
146
146
  };
147
147
 
148
- // src/OpenaiSTT.ts
149
- import { Pcm16Resampler, STT } from "@micdrop/server";
148
+ // src/OpenaiRealtime.ts
149
+ import {
150
+ Pcm16Resampler,
151
+ Realtime
152
+ } from "@micdrop/server";
150
153
  import WebSocket from "ws";
151
- var DEFAULT_MODEL2 = "gpt-4o-transcribe";
152
- var DEFAULT_LANGUAGE = "en";
154
+ import z2, { toJSONSchema as toJSONSchema2 } from "zod";
155
+ var DEFAULT_MODEL2 = "gpt-realtime-2.1";
156
+ var DEFAULT_VOICE = "marin";
157
+ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
153
158
  var SAMPLE_RATE = 16e3;
154
159
  var OPENAI_SAMPLE_RATE = 24e3;
160
+ var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
155
161
  var DEFAULT_CONNECTION_TIMEOUT = 5e3;
156
- var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
157
162
  var DEFAULT_RETRY_DELAY2 = 1e3;
158
163
  var DEFAULT_MAX_RETRY = 3;
164
+ var OpenaiRealtime = class extends Realtime {
165
+ constructor(options) {
166
+ super(options);
167
+ this.connected = false;
168
+ this.everConnected = false;
169
+ this.destroyed = false;
170
+ // Events sent while the connection opens
171
+ this.pending = [];
172
+ this.retryCount = 0;
173
+ this.inputResampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
174
+ this.outputResampler = new Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
175
+ // Bumped by every interruption, so late responses can tell they are stale
176
+ this.generation = 0;
177
+ this.requested = [];
178
+ this.ignoredResponses = /* @__PURE__ */ new Set();
179
+ this.audioStartedAt = 0;
180
+ this.audioBytes = 0;
181
+ this.connect();
182
+ }
183
+ destroy() {
184
+ this.destroyed = true;
185
+ super.destroy();
186
+ if (this.reconnectTimeout) {
187
+ clearTimeout(this.reconnectTimeout);
188
+ this.reconnectTimeout = void 0;
189
+ }
190
+ this.closeSocket();
191
+ }
192
+ openTurn() {
193
+ this.inputResampler.reset();
194
+ }
195
+ appendAudio(chunk) {
196
+ const audio = this.inputResampler.process(chunk);
197
+ if (audio.length === 0) return;
198
+ this.send({
199
+ type: "input_audio_buffer.append",
200
+ audio: audio.toString("base64")
201
+ });
202
+ }
203
+ closeTurn() {
204
+ this.send({ type: "input_audio_buffer.commit" });
205
+ this.createResponse();
206
+ }
207
+ clearTurn() {
208
+ this.send({ type: "input_audio_buffer.clear" });
209
+ }
210
+ generate() {
211
+ this.createResponse();
212
+ }
213
+ speakText(text) {
214
+ this.createResponse({
215
+ conversation: "none",
216
+ instructions: `Say exactly the following text, word for word, and nothing else: ${text}`
217
+ });
218
+ }
219
+ cancelAnswer() {
220
+ this.generation++;
221
+ this.outputResampler.reset();
222
+ const response = this.response;
223
+ if (!response) return;
224
+ this.response = void 0;
225
+ this.ignoredResponses.add(response.id);
226
+ this.send({ type: "response.cancel", response_id: response.id });
227
+ if (!response.outOfBand && this.audioItemId) {
228
+ const played = Math.min(
229
+ Date.now() - this.audioStartedAt,
230
+ this.audioBytes / OPENAI_BYTES_PER_MS
231
+ );
232
+ this.send({
233
+ type: "conversation.item.truncate",
234
+ item_id: this.audioItemId,
235
+ content_index: 0,
236
+ audio_end_ms: Math.max(0, Math.floor(played))
237
+ });
238
+ }
239
+ this.audioItemId = void 0;
240
+ }
241
+ updateTools() {
242
+ if (this.connected) this.sendSessionUpdate();
243
+ }
244
+ injectMessage(role, text) {
245
+ if (!this.connected && this.everConnected) return;
246
+ this.send(this.messageItem(role, text));
247
+ }
248
+ createResponse(response) {
249
+ this.requested.push({
250
+ generation: this.generation,
251
+ outOfBand: !!response
252
+ });
253
+ this.send({ type: "response.create", ...response ? { response } : {} });
254
+ }
255
+ send(event) {
256
+ if (this.connected && this.socket?.readyState === WebSocket.OPEN) {
257
+ this.socket.send(JSON.stringify(event));
258
+ } else {
259
+ this.pending.push(event);
260
+ }
261
+ }
262
+ messageItem(role, text) {
263
+ return {
264
+ type: "conversation.item.create",
265
+ item: {
266
+ type: "message",
267
+ role,
268
+ content: [
269
+ {
270
+ type: role === "assistant" ? "output_text" : "input_text",
271
+ text
272
+ }
273
+ ]
274
+ }
275
+ };
276
+ }
277
+ sendSessionUpdate() {
278
+ this.socket?.send(
279
+ JSON.stringify({
280
+ type: "session.update",
281
+ session: {
282
+ type: "realtime",
283
+ model: this.options.model || DEFAULT_MODEL2,
284
+ instructions: this.options.systemPrompt,
285
+ output_modalities: ["audio"],
286
+ audio: {
287
+ input: {
288
+ format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
289
+ // The Micdrop client detects when the user speaks
290
+ turn_detection: null,
291
+ transcription: {
292
+ model: this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL,
293
+ ...this.options.language ? { language: this.options.language } : {}
294
+ },
295
+ noise_reduction: { type: "near_field" }
296
+ },
297
+ output: {
298
+ format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
299
+ voice: this.options.voice || DEFAULT_VOICE
300
+ }
301
+ },
302
+ tools: this.tools.map((tool) => ({
303
+ type: "function",
304
+ name: tool.name,
305
+ description: tool.description,
306
+ parameters: toJSONSchema2(tool.inputSchema || z2.object())
307
+ }))
308
+ }
309
+ })
310
+ );
311
+ }
312
+ connect() {
313
+ this.initWS().catch((error) => {
314
+ this.log("Connection error:", error);
315
+ this.reconnect();
316
+ });
317
+ }
318
+ initWS() {
319
+ return new Promise((resolve, reject) => {
320
+ const model = this.options.model || DEFAULT_MODEL2;
321
+ const socket = new WebSocket(
322
+ `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
323
+ { headers: { Authorization: `Bearer ${this.options.apiKey}` } }
324
+ );
325
+ this.socket = socket;
326
+ const timeout = setTimeout(() => {
327
+ this.log("Connection timeout");
328
+ this.closeSocket();
329
+ reject(new Error("WebSocket connection timeout"));
330
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
331
+ socket.on("open", () => {
332
+ clearTimeout(timeout);
333
+ this.log("Connection opened");
334
+ this.sendSessionUpdate();
335
+ if (this.everConnected) {
336
+ for (const message of this.conversation) {
337
+ if (message.role !== "user" && message.role !== "assistant") {
338
+ continue;
339
+ }
340
+ socket.send(
341
+ JSON.stringify(this.messageItem(message.role, message.content))
342
+ );
343
+ }
344
+ }
345
+ this.connected = true;
346
+ this.everConnected = true;
347
+ this.retryCount = 0;
348
+ const pending = this.pending;
349
+ this.pending = [];
350
+ pending.forEach((event) => this.send(event));
351
+ resolve();
352
+ });
353
+ socket.on("error", (error) => {
354
+ clearTimeout(timeout);
355
+ this.log("WebSocket error:", error);
356
+ reject(new Error("WebSocket connection error"));
357
+ });
358
+ socket.on("close", (code, reason) => {
359
+ clearTimeout(timeout);
360
+ const wasConnected = this.connected;
361
+ this.closeSocket();
362
+ if (this.destroyed) return;
363
+ this.log("Connection closed", { code, reason: reason.toString() });
364
+ if (wasConnected) this.reconnect();
365
+ });
366
+ socket.on("message", (data) => {
367
+ this.handleMessage(JSON.parse(data.toString()));
368
+ });
369
+ });
370
+ }
371
+ closeSocket() {
372
+ const socket = this.socket;
373
+ this.socket = void 0;
374
+ this.connected = false;
375
+ this.pending = [];
376
+ this.requested = [];
377
+ this.response = void 0;
378
+ if (!socket) return;
379
+ socket.removeAllListeners();
380
+ socket.on("error", () => {
381
+ });
382
+ if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
383
+ socket.close(1e3);
384
+ }
385
+ this.endAnswer();
386
+ }
387
+ reconnect() {
388
+ if (this.destroyed) return;
389
+ this.retryCount++;
390
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
391
+ this.log("Max retries reached, giving up");
392
+ this.emit("Failed");
393
+ return;
394
+ }
395
+ this.log("Reconnecting...");
396
+ this.reconnectTimeout = setTimeout(() => {
397
+ this.reconnectTimeout = void 0;
398
+ this.connect();
399
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
400
+ }
401
+ isIgnored(responseId) {
402
+ return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
403
+ }
404
+ handleMessage(message) {
405
+ switch (message.type) {
406
+ case "response.created": {
407
+ const requested = this.requested.shift() ?? {
408
+ generation: this.generation,
409
+ outOfBand: false
410
+ };
411
+ const id = message.response.id;
412
+ if (requested.generation !== this.generation) {
413
+ this.ignoredResponses.add(id);
414
+ this.socket?.send(
415
+ JSON.stringify({ type: "response.cancel", response_id: id })
416
+ );
417
+ break;
418
+ }
419
+ this.response = { id, ...requested };
420
+ this.audioItemId = void 0;
421
+ this.audioBytes = 0;
422
+ this.outputResampler.reset();
423
+ break;
424
+ }
425
+ case "response.output_audio.delta": {
426
+ if (this.isIgnored(message.response_id)) break;
427
+ const audio = Buffer.from(message.delta, "base64");
428
+ if (this.audioItemId !== message.item_id) {
429
+ this.audioItemId = message.item_id;
430
+ this.audioStartedAt = Date.now();
431
+ this.audioBytes = 0;
432
+ }
433
+ this.audioBytes += audio.length;
434
+ const output = this.outputResampler.process(audio);
435
+ if (output.length > 0) this.emitAudio(output);
436
+ break;
437
+ }
438
+ case "response.output_audio_transcript.delta":
439
+ if (this.isIgnored(message.response_id)) break;
440
+ this.addAnswerTranscript(message.delta);
441
+ break;
442
+ case "conversation.item.input_audio_transcription.completed":
443
+ this.log(`User transcript: "${message.transcript}"`);
444
+ this.addUserTranscript(message.transcript || "");
445
+ this.commitUserTranscript();
446
+ break;
447
+ case "conversation.item.input_audio_transcription.failed":
448
+ this.log("Transcription failed:", message.error);
449
+ this.commitUserTranscript();
450
+ break;
451
+ case "response.done":
452
+ this.onResponseDone(message.response);
453
+ break;
454
+ case "error":
455
+ if (message.error?.code === "response_cancel_not_active") break;
456
+ this.log("Error:", message.error);
457
+ break;
458
+ default:
459
+ break;
460
+ }
461
+ }
462
+ async onResponseDone(response) {
463
+ if (this.ignoredResponses.delete(response.id)) return;
464
+ if (this.response?.id !== response.id) return;
465
+ this.response = void 0;
466
+ const generation = this.generation;
467
+ if (response.status === "failed") {
468
+ this.log("Response failed:", response.status_details);
469
+ }
470
+ const calls = (response.output ?? []).filter(
471
+ (item) => item.type === "function_call"
472
+ );
473
+ if (calls.length === 0) {
474
+ this.endAnswer();
475
+ return;
476
+ }
477
+ let skipAnswer = true;
478
+ for (const call of calls) {
479
+ const toolCall = {
480
+ role: "tool_call",
481
+ toolCallId: call.call_id,
482
+ toolName: call.name,
483
+ parameters: call.arguments
484
+ };
485
+ const result = await this.runTool(toolCall);
486
+ if (generation !== this.generation) return;
487
+ if (!result.skipAnswer) skipAnswer = false;
488
+ this.send({
489
+ type: "conversation.item.create",
490
+ item: {
491
+ type: "function_call_output",
492
+ call_id: call.call_id,
493
+ output: JSON.stringify(result.output ?? null)
494
+ }
495
+ });
496
+ }
497
+ if (skipAnswer) {
498
+ this.endAnswer();
499
+ } else {
500
+ this.createResponse();
501
+ }
502
+ }
503
+ };
504
+
505
+ // src/OpenaiSTT.ts
506
+ import { Pcm16Resampler as Pcm16Resampler2, STT } from "@micdrop/server";
507
+ import WebSocket2 from "ws";
508
+ var DEFAULT_MODEL3 = "gpt-4o-transcribe";
509
+ var DEFAULT_LANGUAGE = "en";
510
+ var SAMPLE_RATE2 = 16e3;
511
+ var OPENAI_SAMPLE_RATE2 = 24e3;
512
+ var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
513
+ var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
514
+ var DEFAULT_RETRY_DELAY3 = 1e3;
515
+ var DEFAULT_MAX_RETRY2 = 3;
159
516
  var OpenaiSTT = class extends STT {
160
517
  constructor(options) {
161
518
  super();
@@ -163,7 +520,7 @@ var OpenaiSTT = class extends STT {
163
520
  this.retryCount = 0;
164
521
  this.audioChunksPending = [];
165
522
  // Store audio chunks to send them again if reconnecting
166
- this.resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
523
+ this.resampler = new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
167
524
  this.initPromise = this.initWS().catch((error) => {
168
525
  console.error("[OpenaiSTT] Connection error:", error);
169
526
  this.reconnect();
@@ -200,7 +557,7 @@ var OpenaiSTT = class extends STT {
200
557
  this.transcriptionTimeout = void 0;
201
558
  }
202
559
  this.socket?.removeAllListeners();
203
- if (this.socket?.readyState === WebSocket.OPEN) {
560
+ if (this.socket?.readyState === WebSocket2.OPEN) {
204
561
  this.socket?.close(1e3);
205
562
  }
206
563
  this.socket = void 0;
@@ -216,10 +573,10 @@ var OpenaiSTT = class extends STT {
216
573
  input: {
217
574
  format: {
218
575
  type: "audio/pcm",
219
- rate: OPENAI_SAMPLE_RATE
576
+ rate: OPENAI_SAMPLE_RATE2
220
577
  },
221
578
  transcription: {
222
- model: this.options.model || DEFAULT_MODEL2,
579
+ model: this.options.model || DEFAULT_MODEL3,
223
580
  language: this.options.language || DEFAULT_LANGUAGE,
224
581
  prompt: this.options.prompt || "Transcribe the incoming audio in real time."
225
582
  },
@@ -238,7 +595,7 @@ var OpenaiSTT = class extends STT {
238
595
  }
239
596
  async initWS() {
240
597
  return new Promise((resolve, reject) => {
241
- const socket = new WebSocket(
598
+ const socket = new WebSocket2(
242
599
  "wss://api.openai.com/v1/realtime?intent=transcription",
243
600
  {
244
601
  headers: {
@@ -253,7 +610,7 @@ var OpenaiSTT = class extends STT {
253
610
  socket.close();
254
611
  this.socket = void 0;
255
612
  reject(new Error("WebSocket connection timeout"));
256
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
613
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
257
614
  socket.addEventListener("open", () => {
258
615
  clearTimeout(timeout);
259
616
  this.sendSessionUpdate();
@@ -327,7 +684,7 @@ var OpenaiSTT = class extends STT {
327
684
  }
328
685
  reconnect() {
329
686
  this.retryCount++;
330
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
687
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
331
688
  this.log("Max retries reached, giving up");
332
689
  this.emit("Failed", this.audioChunksPending);
333
690
  return;
@@ -349,7 +706,7 @@ var OpenaiSTT = class extends STT {
349
706
  this.log("Reconnection error:", error);
350
707
  this.reconnect();
351
708
  });
352
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
709
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
353
710
  });
354
711
  }
355
712
  commitAudio() {
@@ -360,11 +717,11 @@ var OpenaiSTT = class extends STT {
360
717
  };
361
718
 
362
719
  // src/OpenaiTTS.ts
363
- import { Pcm16Resampler as Pcm16Resampler2, SentenceSplitter, TTS } from "@micdrop/server";
720
+ import { Pcm16Resampler as Pcm16Resampler3, SentenceSplitter, TTS } from "@micdrop/server";
364
721
  import OpenAI2 from "openai";
365
- var DEFAULT_MODEL3 = "gpt-4o-mini-tts";
366
- var DEFAULT_VOICE = "alloy";
367
- var OPENAI_SAMPLE_RATE2 = 24e3;
722
+ var DEFAULT_MODEL4 = "gpt-4o-mini-tts";
723
+ var DEFAULT_VOICE2 = "alloy";
724
+ var OPENAI_SAMPLE_RATE3 = 24e3;
368
725
  var OUTPUT_SAMPLE_RATE = 16e3;
369
726
  var MAX_IN_FLIGHT = 2;
370
727
  var OPENING_CUSHION = 300 * 32;
@@ -478,8 +835,8 @@ var OpenaiTTS = class extends TTS {
478
835
  this.abortControllers.add(controller);
479
836
  const response = this.openai.audio.speech.create(
480
837
  {
481
- model: this.options.model || DEFAULT_MODEL3,
482
- voice: this.options.voice || DEFAULT_VOICE,
838
+ model: this.options.model || DEFAULT_MODEL4,
839
+ voice: this.options.voice || DEFAULT_VOICE2,
483
840
  input: item.text,
484
841
  response_format: "pcm",
485
842
  ...this.options.instructions ? { instructions: this.options.instructions } : {},
@@ -518,8 +875,8 @@ var OpenaiTTS = class extends TTS {
518
875
  if (!response?.body) return;
519
876
  if (item.counter !== this.counter) return;
520
877
  this.log(`Synthesizing: "${item.text}"`);
521
- const resampler = new Pcm16Resampler2(
522
- OPENAI_SAMPLE_RATE2,
878
+ const resampler = new Pcm16Resampler3(
879
+ OPENAI_SAMPLE_RATE3,
523
880
  OUTPUT_SAMPLE_RATE
524
881
  );
525
882
  const reader = response.body.getReader();
@@ -533,6 +890,11 @@ var OpenaiTTS = class extends TTS {
533
890
  const output = resampler.process(Buffer.from(value));
534
891
  if (output.length > 0) this.emitAudio(output);
535
892
  }
893
+ } catch (error) {
894
+ if (!item.controller.signal.aborted) {
895
+ this.log("Error reading synthesized speech:", error);
896
+ this.emit("Failed", [item.text]);
897
+ }
536
898
  } finally {
537
899
  this.abortControllers.delete(item.controller);
538
900
  }
@@ -540,6 +902,7 @@ var OpenaiTTS = class extends TTS {
540
902
  };
541
903
  export {
542
904
  OpenaiAgent,
905
+ OpenaiRealtime,
543
906
  OpenaiSTT,
544
907
  OpenaiTTS
545
908
  };