@micdrop/openai 1.4.0 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -31,6 +31,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
31
31
  var index_exports = {};
32
32
  __export(index_exports, {
33
33
  OpenaiAgent: () => OpenaiAgent,
34
+ OpenaiRealtime: () => OpenaiRealtime,
34
35
  OpenaiSTT: () => OpenaiSTT,
35
36
  OpenaiTTS: () => OpenaiTTS
36
37
  });
@@ -183,25 +184,379 @@ var OpenaiAgent = class extends import_server.Agent {
183
184
  }
184
185
  };
185
186
 
186
- // src/OpenaiSTT.ts
187
+ // src/OpenaiRealtime.ts
187
188
  var import_server2 = require("@micdrop/server");
188
189
  var import_ws = __toESM(require("ws"));
189
- var DEFAULT_MODEL2 = "gpt-4o-transcribe";
190
- var DEFAULT_LANGUAGE = "en";
190
+ var import_zod2 = __toESM(require("zod"));
191
+ var DEFAULT_MODEL2 = "gpt-realtime-2.1";
192
+ var DEFAULT_VOICE = "marin";
193
+ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
191
194
  var SAMPLE_RATE = 16e3;
192
195
  var OPENAI_SAMPLE_RATE = 24e3;
196
+ var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
193
197
  var DEFAULT_CONNECTION_TIMEOUT = 5e3;
194
- var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
195
198
  var DEFAULT_RETRY_DELAY2 = 1e3;
196
199
  var DEFAULT_MAX_RETRY = 3;
197
- var OpenaiSTT = class extends import_server2.STT {
200
+ var OpenaiRealtime = class extends import_server2.Realtime {
201
+ constructor(options) {
202
+ super(options);
203
+ this.connected = false;
204
+ this.everConnected = false;
205
+ this.destroyed = false;
206
+ // Events sent while the connection opens
207
+ this.pending = [];
208
+ this.retryCount = 0;
209
+ this.inputResampler = new import_server2.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
210
+ this.outputResampler = new import_server2.Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
211
+ // Bumped by every interruption, so late responses can tell they are stale
212
+ this.generation = 0;
213
+ this.requested = [];
214
+ this.ignoredResponses = /* @__PURE__ */ new Set();
215
+ this.audioStartedAt = 0;
216
+ this.audioBytes = 0;
217
+ this.connect();
218
+ }
219
+ destroy() {
220
+ this.destroyed = true;
221
+ super.destroy();
222
+ if (this.reconnectTimeout) {
223
+ clearTimeout(this.reconnectTimeout);
224
+ this.reconnectTimeout = void 0;
225
+ }
226
+ this.closeSocket();
227
+ }
228
+ openTurn() {
229
+ this.inputResampler.reset();
230
+ }
231
+ appendAudio(chunk) {
232
+ const audio = this.inputResampler.process(chunk);
233
+ if (audio.length === 0) return;
234
+ this.send({
235
+ type: "input_audio_buffer.append",
236
+ audio: audio.toString("base64")
237
+ });
238
+ }
239
+ closeTurn() {
240
+ this.send({ type: "input_audio_buffer.commit" });
241
+ this.createResponse();
242
+ }
243
+ clearTurn() {
244
+ this.send({ type: "input_audio_buffer.clear" });
245
+ }
246
+ generate() {
247
+ this.createResponse();
248
+ }
249
+ speakText(text) {
250
+ this.createResponse({
251
+ conversation: "none",
252
+ instructions: `Say exactly the following text, word for word, and nothing else: ${text}`
253
+ });
254
+ }
255
+ cancelAnswer() {
256
+ this.generation++;
257
+ this.outputResampler.reset();
258
+ const response = this.response;
259
+ if (!response) return;
260
+ this.response = void 0;
261
+ this.ignoredResponses.add(response.id);
262
+ this.send({ type: "response.cancel", response_id: response.id });
263
+ if (!response.outOfBand && this.audioItemId) {
264
+ const played = Math.min(
265
+ Date.now() - this.audioStartedAt,
266
+ this.audioBytes / OPENAI_BYTES_PER_MS
267
+ );
268
+ this.send({
269
+ type: "conversation.item.truncate",
270
+ item_id: this.audioItemId,
271
+ content_index: 0,
272
+ audio_end_ms: Math.max(0, Math.floor(played))
273
+ });
274
+ }
275
+ this.audioItemId = void 0;
276
+ }
277
+ updateTools() {
278
+ if (this.connected) this.sendSessionUpdate();
279
+ }
280
+ injectMessage(role, text) {
281
+ if (!this.connected && this.everConnected) return;
282
+ this.send(this.messageItem(role, text));
283
+ }
284
+ createResponse(response) {
285
+ this.requested.push({
286
+ generation: this.generation,
287
+ outOfBand: !!response
288
+ });
289
+ this.send({ type: "response.create", ...response ? { response } : {} });
290
+ }
291
+ send(event) {
292
+ if (this.connected && this.socket?.readyState === import_ws.default.OPEN) {
293
+ this.socket.send(JSON.stringify(event));
294
+ } else {
295
+ this.pending.push(event);
296
+ }
297
+ }
298
+ messageItem(role, text) {
299
+ return {
300
+ type: "conversation.item.create",
301
+ item: {
302
+ type: "message",
303
+ role,
304
+ content: [
305
+ {
306
+ type: role === "assistant" ? "output_text" : "input_text",
307
+ text
308
+ }
309
+ ]
310
+ }
311
+ };
312
+ }
313
+ sendSessionUpdate() {
314
+ this.socket?.send(
315
+ JSON.stringify({
316
+ type: "session.update",
317
+ session: {
318
+ type: "realtime",
319
+ model: this.options.model || DEFAULT_MODEL2,
320
+ instructions: this.options.systemPrompt,
321
+ output_modalities: ["audio"],
322
+ audio: {
323
+ input: {
324
+ format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
325
+ // The Micdrop client detects when the user speaks
326
+ turn_detection: null,
327
+ transcription: {
328
+ model: this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL,
329
+ ...this.options.language ? { language: this.options.language } : {}
330
+ },
331
+ noise_reduction: { type: "near_field" }
332
+ },
333
+ output: {
334
+ format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE },
335
+ voice: this.options.voice || DEFAULT_VOICE
336
+ }
337
+ },
338
+ tools: this.tools.map((tool) => ({
339
+ type: "function",
340
+ name: tool.name,
341
+ description: tool.description,
342
+ parameters: (0, import_zod2.toJSONSchema)(tool.inputSchema || import_zod2.default.object())
343
+ }))
344
+ }
345
+ })
346
+ );
347
+ }
348
+ connect() {
349
+ this.initWS().catch((error) => {
350
+ this.log("Connection error:", error);
351
+ this.reconnect();
352
+ });
353
+ }
354
+ initWS() {
355
+ return new Promise((resolve, reject) => {
356
+ const model = this.options.model || DEFAULT_MODEL2;
357
+ const socket = new import_ws.default(
358
+ `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
359
+ { headers: { Authorization: `Bearer ${this.options.apiKey}` } }
360
+ );
361
+ this.socket = socket;
362
+ const timeout = setTimeout(() => {
363
+ this.log("Connection timeout");
364
+ this.closeSocket();
365
+ reject(new Error("WebSocket connection timeout"));
366
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
367
+ socket.on("open", () => {
368
+ clearTimeout(timeout);
369
+ this.log("Connection opened");
370
+ this.sendSessionUpdate();
371
+ if (this.everConnected) {
372
+ for (const message of this.conversation) {
373
+ if (message.role !== "user" && message.role !== "assistant") {
374
+ continue;
375
+ }
376
+ socket.send(
377
+ JSON.stringify(this.messageItem(message.role, message.content))
378
+ );
379
+ }
380
+ }
381
+ this.connected = true;
382
+ this.everConnected = true;
383
+ this.retryCount = 0;
384
+ const pending = this.pending;
385
+ this.pending = [];
386
+ pending.forEach((event) => this.send(event));
387
+ resolve();
388
+ });
389
+ socket.on("error", (error) => {
390
+ clearTimeout(timeout);
391
+ this.log("WebSocket error:", error);
392
+ reject(new Error("WebSocket connection error"));
393
+ });
394
+ socket.on("close", (code, reason) => {
395
+ clearTimeout(timeout);
396
+ const wasConnected = this.connected;
397
+ this.closeSocket();
398
+ if (this.destroyed) return;
399
+ this.log("Connection closed", { code, reason: reason.toString() });
400
+ if (wasConnected) this.reconnect();
401
+ });
402
+ socket.on("message", (data) => {
403
+ this.handleMessage(JSON.parse(data.toString()));
404
+ });
405
+ });
406
+ }
407
+ closeSocket() {
408
+ const socket = this.socket;
409
+ this.socket = void 0;
410
+ this.connected = false;
411
+ this.pending = [];
412
+ this.requested = [];
413
+ this.response = void 0;
414
+ if (!socket) return;
415
+ socket.removeAllListeners();
416
+ socket.on("error", () => {
417
+ });
418
+ if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
419
+ socket.close(1e3);
420
+ }
421
+ this.endAnswer();
422
+ }
423
+ reconnect() {
424
+ if (this.destroyed) return;
425
+ this.retryCount++;
426
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
427
+ this.log("Max retries reached, giving up");
428
+ this.emit("Failed");
429
+ return;
430
+ }
431
+ this.log("Reconnecting...");
432
+ this.reconnectTimeout = setTimeout(() => {
433
+ this.reconnectTimeout = void 0;
434
+ this.connect();
435
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
436
+ }
437
+ isIgnored(responseId) {
438
+ return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
439
+ }
440
+ handleMessage(message) {
441
+ switch (message.type) {
442
+ case "response.created": {
443
+ const requested = this.requested.shift() ?? {
444
+ generation: this.generation,
445
+ outOfBand: false
446
+ };
447
+ const id = message.response.id;
448
+ if (requested.generation !== this.generation) {
449
+ this.ignoredResponses.add(id);
450
+ this.socket?.send(
451
+ JSON.stringify({ type: "response.cancel", response_id: id })
452
+ );
453
+ break;
454
+ }
455
+ this.response = { id, ...requested };
456
+ this.audioItemId = void 0;
457
+ this.audioBytes = 0;
458
+ this.outputResampler.reset();
459
+ break;
460
+ }
461
+ case "response.output_audio.delta": {
462
+ if (this.isIgnored(message.response_id)) break;
463
+ const audio = Buffer.from(message.delta, "base64");
464
+ if (this.audioItemId !== message.item_id) {
465
+ this.audioItemId = message.item_id;
466
+ this.audioStartedAt = Date.now();
467
+ this.audioBytes = 0;
468
+ }
469
+ this.audioBytes += audio.length;
470
+ const output = this.outputResampler.process(audio);
471
+ if (output.length > 0) this.emitAudio(output);
472
+ break;
473
+ }
474
+ case "response.output_audio_transcript.delta":
475
+ if (this.isIgnored(message.response_id)) break;
476
+ this.addAnswerTranscript(message.delta);
477
+ break;
478
+ case "conversation.item.input_audio_transcription.completed":
479
+ this.log(`User transcript: "${message.transcript}"`);
480
+ this.addUserTranscript(message.transcript || "");
481
+ this.commitUserTranscript();
482
+ break;
483
+ case "conversation.item.input_audio_transcription.failed":
484
+ this.log("Transcription failed:", message.error);
485
+ this.commitUserTranscript();
486
+ break;
487
+ case "response.done":
488
+ this.onResponseDone(message.response);
489
+ break;
490
+ case "error":
491
+ if (message.error?.code === "response_cancel_not_active") break;
492
+ this.log("Error:", message.error);
493
+ break;
494
+ default:
495
+ break;
496
+ }
497
+ }
498
+ async onResponseDone(response) {
499
+ if (this.ignoredResponses.delete(response.id)) return;
500
+ if (this.response?.id !== response.id) return;
501
+ this.response = void 0;
502
+ const generation = this.generation;
503
+ if (response.status === "failed") {
504
+ this.log("Response failed:", response.status_details);
505
+ }
506
+ const calls = (response.output ?? []).filter(
507
+ (item) => item.type === "function_call"
508
+ );
509
+ if (calls.length === 0) {
510
+ this.endAnswer();
511
+ return;
512
+ }
513
+ let skipAnswer = true;
514
+ for (const call of calls) {
515
+ const toolCall = {
516
+ role: "tool_call",
517
+ toolCallId: call.call_id,
518
+ toolName: call.name,
519
+ parameters: call.arguments
520
+ };
521
+ const result = await this.runTool(toolCall);
522
+ if (generation !== this.generation) return;
523
+ if (!result.skipAnswer) skipAnswer = false;
524
+ this.send({
525
+ type: "conversation.item.create",
526
+ item: {
527
+ type: "function_call_output",
528
+ call_id: call.call_id,
529
+ output: JSON.stringify(result.output ?? null)
530
+ }
531
+ });
532
+ }
533
+ if (skipAnswer) {
534
+ this.endAnswer();
535
+ } else {
536
+ this.createResponse();
537
+ }
538
+ }
539
+ };
540
+
541
+ // src/OpenaiSTT.ts
542
+ var import_server3 = require("@micdrop/server");
543
+ var import_ws2 = __toESM(require("ws"));
544
+ var DEFAULT_MODEL3 = "gpt-4o-transcribe";
545
+ var DEFAULT_LANGUAGE = "en";
546
+ var SAMPLE_RATE2 = 16e3;
547
+ var OPENAI_SAMPLE_RATE2 = 24e3;
548
+ var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
549
+ var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
550
+ var DEFAULT_RETRY_DELAY3 = 1e3;
551
+ var DEFAULT_MAX_RETRY2 = 3;
552
+ var OpenaiSTT = class extends import_server3.STT {
198
553
  constructor(options) {
199
554
  super();
200
555
  this.options = options;
201
556
  this.retryCount = 0;
202
557
  this.audioChunksPending = [];
203
558
  // Store audio chunks to send them again if reconnecting
204
- this.resampler = new import_server2.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
559
+ this.resampler = new import_server3.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
205
560
  this.initPromise = this.initWS().catch((error) => {
206
561
  console.error("[OpenaiSTT] Connection error:", error);
207
562
  this.reconnect();
@@ -238,7 +593,7 @@ var OpenaiSTT = class extends import_server2.STT {
238
593
  this.transcriptionTimeout = void 0;
239
594
  }
240
595
  this.socket?.removeAllListeners();
241
- if (this.socket?.readyState === import_ws.default.OPEN) {
596
+ if (this.socket?.readyState === import_ws2.default.OPEN) {
242
597
  this.socket?.close(1e3);
243
598
  }
244
599
  this.socket = void 0;
@@ -254,10 +609,10 @@ var OpenaiSTT = class extends import_server2.STT {
254
609
  input: {
255
610
  format: {
256
611
  type: "audio/pcm",
257
- rate: OPENAI_SAMPLE_RATE
612
+ rate: OPENAI_SAMPLE_RATE2
258
613
  },
259
614
  transcription: {
260
- model: this.options.model || DEFAULT_MODEL2,
615
+ model: this.options.model || DEFAULT_MODEL3,
261
616
  language: this.options.language || DEFAULT_LANGUAGE,
262
617
  prompt: this.options.prompt || "Transcribe the incoming audio in real time."
263
618
  },
@@ -276,7 +631,7 @@ var OpenaiSTT = class extends import_server2.STT {
276
631
  }
277
632
  async initWS() {
278
633
  return new Promise((resolve, reject) => {
279
- const socket = new import_ws.default(
634
+ const socket = new import_ws2.default(
280
635
  "wss://api.openai.com/v1/realtime?intent=transcription",
281
636
  {
282
637
  headers: {
@@ -291,7 +646,7 @@ var OpenaiSTT = class extends import_server2.STT {
291
646
  socket.close();
292
647
  this.socket = void 0;
293
648
  reject(new Error("WebSocket connection timeout"));
294
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
649
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
295
650
  socket.addEventListener("open", () => {
296
651
  clearTimeout(timeout);
297
652
  this.sendSessionUpdate();
@@ -365,7 +720,7 @@ var OpenaiSTT = class extends import_server2.STT {
365
720
  }
366
721
  reconnect() {
367
722
  this.retryCount++;
368
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
723
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
369
724
  this.log("Max retries reached, giving up");
370
725
  this.emit("Failed", this.audioChunksPending);
371
726
  return;
@@ -387,7 +742,7 @@ var OpenaiSTT = class extends import_server2.STT {
387
742
  this.log("Reconnection error:", error);
388
743
  this.reconnect();
389
744
  });
390
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
745
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
391
746
  });
392
747
  }
393
748
  commitAudio() {
@@ -398,15 +753,15 @@ var OpenaiSTT = class extends import_server2.STT {
398
753
  };
399
754
 
400
755
  // src/OpenaiTTS.ts
401
- var import_server3 = require("@micdrop/server");
756
+ var import_server4 = require("@micdrop/server");
402
757
  var import_openai2 = __toESM(require("openai"));
403
- var DEFAULT_MODEL3 = "gpt-4o-mini-tts";
404
- var DEFAULT_VOICE = "alloy";
405
- var OPENAI_SAMPLE_RATE2 = 24e3;
758
+ var DEFAULT_MODEL4 = "gpt-4o-mini-tts";
759
+ var DEFAULT_VOICE2 = "alloy";
760
+ var OPENAI_SAMPLE_RATE3 = 24e3;
406
761
  var OUTPUT_SAMPLE_RATE = 16e3;
407
762
  var MAX_IN_FLIGHT = 2;
408
763
  var OPENING_CUSHION = 300 * 32;
409
- var OpenaiTTS = class extends import_server3.TTS {
764
+ var OpenaiTTS = class extends import_server4.TTS {
410
765
  constructor(options) {
411
766
  super();
412
767
  this.options = options;
@@ -417,7 +772,7 @@ var OpenaiTTS = class extends import_server3.TTS {
417
772
  // `counter`, which numbers the synthesis calls and must only move when there
418
773
  // is something to synthesize.
419
774
  this.generation = 0;
420
- this.splitter = new import_server3.SentenceSplitter();
775
+ this.splitter = new import_server4.SentenceSplitter();
421
776
  this.queue = [];
422
777
  // Sentences whose synthesis has not started
423
778
  this.pending = [];
@@ -516,8 +871,8 @@ var OpenaiTTS = class extends import_server3.TTS {
516
871
  this.abortControllers.add(controller);
517
872
  const response = this.openai.audio.speech.create(
518
873
  {
519
- model: this.options.model || DEFAULT_MODEL3,
520
- voice: this.options.voice || DEFAULT_VOICE,
874
+ model: this.options.model || DEFAULT_MODEL4,
875
+ voice: this.options.voice || DEFAULT_VOICE2,
521
876
  input: item.text,
522
877
  response_format: "pcm",
523
878
  ...this.options.instructions ? { instructions: this.options.instructions } : {},
@@ -556,8 +911,8 @@ var OpenaiTTS = class extends import_server3.TTS {
556
911
  if (!response?.body) return;
557
912
  if (item.counter !== this.counter) return;
558
913
  this.log(`Synthesizing: "${item.text}"`);
559
- const resampler = new import_server3.Pcm16Resampler(
560
- OPENAI_SAMPLE_RATE2,
914
+ const resampler = new import_server4.Pcm16Resampler(
915
+ OPENAI_SAMPLE_RATE3,
561
916
  OUTPUT_SAMPLE_RATE
562
917
  );
563
918
  const reader = response.body.getReader();
@@ -571,6 +926,11 @@ var OpenaiTTS = class extends import_server3.TTS {
571
926
  const output = resampler.process(Buffer.from(value));
572
927
  if (output.length > 0) this.emitAudio(output);
573
928
  }
929
+ } catch (error) {
930
+ if (!item.controller.signal.aborted) {
931
+ this.log("Error reading synthesized speech:", error);
932
+ this.emit("Failed", [item.text]);
933
+ }
574
934
  } finally {
575
935
  this.abortControllers.delete(item.controller);
576
936
  }
@@ -579,6 +939,7 @@ var OpenaiTTS = class extends import_server3.TTS {
579
939
  // Annotate the CommonJS export names for ESM import in node:
580
940
  0 && (module.exports = {
581
941
  OpenaiAgent,
942
+ OpenaiRealtime,
582
943
  OpenaiSTT,
583
944
  OpenaiTTS
584
945
  });