@micdrop/openai 1.7.1 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -6,19 +6,69 @@ var DEFAULT_MODEL = "gpt-4o";
6
6
  var DEFAULT_MAX_STEPS = 5;
7
7
  var DEFAULT_MAX_RETRIES = 3;
8
8
  var DEFAULT_RETRY_DELAY = 1e3;
9
+ var StepInterruptedError = class extends Error {
10
+ constructor(outcome, reason) {
11
+ super("Answer interrupted");
12
+ this.outcome = outcome;
13
+ this.reason = reason;
14
+ }
15
+ };
9
16
  var OpenaiAgent = class extends Agent {
10
17
  constructor(options) {
11
18
  super(options);
12
19
  this.openai = "openai" in options ? options.openai : new OpenAI({ apiKey: options.apiKey });
13
20
  }
14
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
15
- if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
16
- console.error("[OpenaiAgent] Max steps reached");
17
- return;
21
+ async generateAnswer(stream, signal) {
22
+ const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
23
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
24
+ for (let step = 0; step < maxSteps; step++) {
25
+ let outcome;
26
+ for (let tryCount = 0; !outcome; tryCount++) {
27
+ try {
28
+ outcome = await this.generateStep(stream, signal);
29
+ } catch (error) {
30
+ if (signal.aborted) return;
31
+ if (error instanceof StepInterruptedError) {
32
+ console.error("[OpenaiAgent] Answer interrupted:", error.reason);
33
+ outcome = error.outcome;
34
+ break;
35
+ }
36
+ console.error("[OpenaiAgent] Error answering:", error);
37
+ if (tryCount >= maxRetry) {
38
+ this.log("Max retries reached, giving up");
39
+ this.emit("Failed");
40
+ return;
41
+ }
42
+ await new Promise(
43
+ (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
44
+ );
45
+ if (signal.aborted) return;
46
+ }
47
+ }
48
+ if (signal.aborted) return;
49
+ if (outcome.recorded || outcome.skipAnswer) return;
50
+ if (outcome.text) {
51
+ const { message, metadata } = this.extract(outcome.text);
52
+ this.addAssistantMessage(message, metadata);
53
+ return;
54
+ }
18
55
  }
19
- const abortController = new AbortController();
20
- this.abortController = abortController;
21
- const signal = abortController.signal;
56
+ console.error("[OpenaiAgent] Max steps reached");
57
+ }
58
+ /**
59
+ * Runs one request, writing its text to the stream and running its tools.
60
+ *
61
+ * It throws the error of the request when it failed before producing
62
+ * anything, which can then be retried, and a {@link StepInterruptedError}
63
+ * once something was written or done.
64
+ */
65
+ async generateStep(stream, signal) {
66
+ const outcome = {
67
+ text: "",
68
+ recorded: false,
69
+ skipAnswer: false
70
+ };
71
+ let produced = false;
22
72
  let extracting = false;
23
73
  const extractOptions = this.getExtractOptions();
24
74
  try {
@@ -32,14 +82,14 @@ var OpenaiAgent = class extends Agent {
32
82
  },
33
83
  { signal }
34
84
  );
35
- let skipAnswer = false;
36
- let hasAnswer = false;
37
85
  for await (const event of response) {
38
- if (abortController !== this.abortController) return;
86
+ if (signal.aborted) return outcome;
39
87
  switch (event.type) {
40
88
  case "response.output_text.delta":
41
- if (hasAnswer) break;
89
+ if (outcome.recorded) break;
42
90
  this.log(`Answer chunk: "${event.delta}"`);
91
+ outcome.text += event.delta;
92
+ produced = true;
43
93
  if (extractOptions) {
44
94
  if (!extracting) {
45
95
  const startTagIndex = event.delta.indexOf(
@@ -57,15 +107,17 @@ var OpenaiAgent = class extends Agent {
57
107
  }
58
108
  stream.write(event.delta);
59
109
  break;
60
- case "response.output_text.done":
61
- if (hasAnswer) break;
110
+ case "response.output_text.done": {
111
+ if (outcome.recorded) break;
62
112
  const { message, metadata } = this.extract(event.text);
63
113
  this.addAssistantMessage(message, metadata);
64
- hasAnswer = true;
114
+ outcome.recorded = true;
65
115
  break;
116
+ }
66
117
  case "response.output_item.done":
67
118
  if (event.item.type === "function_call") {
68
119
  const toolCall = event.item;
120
+ produced = true;
69
121
  const result = await this.executeTool({
70
122
  role: "tool_call",
71
123
  toolCallId: toolCall.call_id,
@@ -73,7 +125,7 @@ var OpenaiAgent = class extends Agent {
73
125
  parameters: toolCall.arguments
74
126
  });
75
127
  if (result.skipAnswer) {
76
- skipAnswer = true;
128
+ outcome.skipAnswer = true;
77
129
  }
78
130
  }
79
131
  break;
@@ -81,23 +133,12 @@ var OpenaiAgent = class extends Agent {
81
133
  break;
82
134
  }
83
135
  }
84
- if (abortController !== this.abortController) return;
85
- if (!skipAnswer && !hasAnswer) {
86
- await this.generateAnswer(stream, stepCount + 1);
87
- }
88
- this.abortController = void 0;
136
+ return outcome;
89
137
  } catch (error) {
90
- if (error instanceof OpenAI.APIUserAbortError) return;
91
- console.error("[OpenaiAgent] Error answering:", error);
92
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
93
- await new Promise(
94
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
95
- );
96
- await this.generateAnswer(stream, stepCount, tryCount + 1);
97
- } else {
98
- this.log("Max retries reached, giving up");
99
- this.emit("Failed");
138
+ if (produced && !signal.aborted) {
139
+ throw new StepInterruptedError(outcome, error);
100
140
  }
141
+ throw error;
101
142
  }
102
143
  }
103
144
  buildInput() {
@@ -115,7 +156,8 @@ var OpenaiAgent = class extends Agent {
115
156
  type: "function_call",
116
157
  call_id: message.toolCallId,
117
158
  name: message.toolName,
118
- arguments: JSON.stringify(message.parameters)
159
+ // Already the JSON the model wrote
160
+ arguments: message.parameters
119
161
  };
120
162
  case "tool_result":
121
163
  return {
@@ -137,12 +179,6 @@ var OpenaiAgent = class extends Agent {
137
179
  parameters: toJSONSchema(tool.inputSchema || z.object())
138
180
  }));
139
181
  }
140
- cancel() {
141
- if (!this.abortController) return;
142
- this.log("Cancel");
143
- this.abortController.abort();
144
- this.abortController = void 0;
145
- }
146
182
  };
147
183
 
148
184
  // src/OpenaiClassifier.ts
@@ -248,11 +284,29 @@ function buildQuestion(name, question) {
248
284
  // src/OpenaiRealtime.ts
249
285
  import {
250
286
  Pcm16Resampler,
251
- Realtime
287
+ Realtime,
288
+ ReconnectingSocket
252
289
  } from "@micdrop/server";
253
- import WebSocket from "ws";
254
290
  import z2, { toJSONSchema as toJSONSchema2 } from "zod";
255
291
 
292
+ // src/session.ts
293
+ var sessionHandshake = {
294
+ read(data) {
295
+ const message = parseMessage(data);
296
+ if (message?.type === "error") {
297
+ throw new Error(message.error?.message || "Session refused");
298
+ }
299
+ return message?.type === "session.created" || message?.type === "transcription_session.created";
300
+ }
301
+ };
302
+ function parseMessage(data) {
303
+ try {
304
+ return JSON.parse(data.toString());
305
+ } catch {
306
+ return void 0;
307
+ }
308
+ }
309
+
256
310
  // src/transcription.ts
257
311
  function transcriptionLanguage(model, language) {
258
312
  if (!language) return {};
@@ -269,36 +323,56 @@ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
269
323
  var SAMPLE_RATE = 16e3;
270
324
  var OPENAI_SAMPLE_RATE = 24e3;
271
325
  var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
272
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
273
- var DEFAULT_RETRY_DELAY2 = 1e3;
274
- var DEFAULT_MAX_RETRY = 3;
326
+ var MAX_PENDING_EVENTS = 5e3;
327
+ var REQUEST_METADATA_KEY = "micdrop_request";
275
328
  var OpenaiRealtime = class extends Realtime {
276
329
  constructor(options) {
277
330
  super(options);
278
- this.connected = false;
279
- this.everConnected = false;
280
- this.destroyed = false;
331
+ this.connection = "connecting";
281
332
  // Events sent while the connection opens
282
333
  this.pending = [];
283
- this.retryCount = 0;
334
+ this.requestCount = 0;
284
335
  this.inputResampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
285
336
  this.outputResampler = new Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
286
337
  // Bumped by every interruption, so late responses can tell they are stale
287
338
  this.generation = 0;
288
- this.requested = [];
339
+ // Responses asked for and not created yet, by the id sent as their metadata
340
+ // and as the event id of their request, which an error names
341
+ this.requested = /* @__PURE__ */ new Map();
289
342
  this.ignoredResponses = /* @__PURE__ */ new Set();
290
343
  this.audioStartedAt = 0;
291
344
  this.audioBytes = 0;
292
- this.connect();
345
+ const model = options.model || DEFAULT_MODEL3;
346
+ this.socket = new ReconnectingSocket({
347
+ url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
348
+ clientOptions: {
349
+ headers: { Authorization: `Bearer ${options.apiKey}` }
350
+ },
351
+ connectionTimeout: options.connectionTimeout,
352
+ retryDelay: options.retryDelay,
353
+ maxRetry: options.maxRetry,
354
+ handshake: sessionHandshake,
355
+ log: (...message) => this.log(...message)
356
+ });
357
+ this.socket.on("Open", ({ reconnected }) => this.onOpen(reconnected));
358
+ this.socket.on("Close", () => {
359
+ this.connection = "reconnecting";
360
+ this.resetSession();
361
+ });
362
+ this.socket.on("Failed", () => {
363
+ this.connection = "closed";
364
+ this.pending = [];
365
+ this.resetSession();
366
+ this.emit("Failed");
367
+ });
368
+ this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
369
+ this.socket.connect();
293
370
  }
294
371
  destroy() {
295
- this.destroyed = true;
296
372
  super.destroy();
297
- if (this.reconnectTimeout) {
298
- clearTimeout(this.reconnectTimeout);
299
- this.reconnectTimeout = void 0;
300
- }
301
- this.closeSocket();
373
+ this.connection = "closed";
374
+ this.pending = [];
375
+ this.socket.destroy();
302
376
  }
303
377
  openTurn() {
304
378
  this.inputResampler.reset();
@@ -350,25 +424,49 @@ var OpenaiRealtime = class extends Realtime {
350
424
  this.audioItemId = void 0;
351
425
  }
352
426
  updateTools() {
353
- if (this.connected) this.sendSessionUpdate();
427
+ if (this.connection === "connected") this.sendSessionUpdate();
354
428
  }
355
429
  injectMessage(role, text) {
356
- if (!this.connected && this.everConnected) return;
430
+ if (this.connection === "reconnecting") return;
357
431
  this.send(this.messageItem(role, text));
358
432
  }
433
+ /**
434
+ * Asks for a response, tagged so its `response.created` tells which request
435
+ * it answers. Matching them by order alone would shift every later response
436
+ * onto the wrong request as soon as one is refused.
437
+ */
359
438
  createResponse(response) {
360
- this.requested.push({
361
- generation: this.generation,
362
- outOfBand: !!response
439
+ const id = `micdrop-response-${++this.requestCount}`;
440
+ const generation = this.generation;
441
+ const accepted = this.send({
442
+ type: "response.create",
443
+ // Errors caused by this event carry its id
444
+ event_id: id,
445
+ response: { ...response, metadata: { [REQUEST_METADATA_KEY]: id } }
363
446
  });
364
- this.send({ type: "response.create", ...response ? { response } : {} });
447
+ if (!accepted) {
448
+ queueMicrotask(() => {
449
+ if (generation === this.generation) this.endAnswer();
450
+ });
451
+ return;
452
+ }
453
+ this.requested.set(id, { generation, outOfBand: !!response });
365
454
  }
455
+ /**
456
+ * Sends an event, or holds it until the connection opens.
457
+ * @returns False when it was dropped, the connection being given up on
458
+ */
366
459
  send(event) {
367
- if (this.connected && this.socket?.readyState === WebSocket.OPEN) {
368
- this.socket.send(JSON.stringify(event));
369
- } else {
370
- this.pending.push(event);
460
+ if (this.connection === "connected") {
461
+ return this.socket.send(JSON.stringify(event));
371
462
  }
463
+ if (this.connection === "closed") return false;
464
+ if (this.pending.length >= MAX_PENDING_EVENTS) {
465
+ this.log("Too many events waiting for the connection, dropped");
466
+ return false;
467
+ }
468
+ this.pending.push(event);
469
+ return true;
372
470
  }
373
471
  messageItem(role, text) {
374
472
  return {
@@ -387,7 +485,7 @@ var OpenaiRealtime = class extends Realtime {
387
485
  }
388
486
  sendSessionUpdate() {
389
487
  const transcriptionModel = this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
390
- this.socket?.send(
488
+ this.socket.send(
391
489
  JSON.stringify({
392
490
  type: "session.update",
393
491
  session: {
@@ -424,111 +522,50 @@ var OpenaiRealtime = class extends Realtime {
424
522
  })
425
523
  );
426
524
  }
427
- connect() {
428
- this.initWS().catch((error) => {
429
- this.log("Connection error:", error);
430
- this.reconnect();
431
- });
432
- }
433
- initWS() {
434
- return new Promise((resolve, reject) => {
435
- const model = this.options.model || DEFAULT_MODEL3;
436
- const socket = new WebSocket(
437
- `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
438
- { headers: { Authorization: `Bearer ${this.options.apiKey}` } }
439
- );
440
- this.socket = socket;
441
- const timeout = setTimeout(() => {
442
- this.log("Connection timeout");
443
- this.closeSocket();
444
- reject(new Error("WebSocket connection timeout"));
445
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
446
- socket.on("open", () => {
447
- clearTimeout(timeout);
448
- this.log("Connection opened");
449
- this.sendSessionUpdate();
450
- if (this.everConnected) {
451
- for (const message of this.conversation) {
452
- if (message.role !== "user" && message.role !== "assistant") {
453
- continue;
454
- }
455
- socket.send(
456
- JSON.stringify(this.messageItem(message.role, message.content))
457
- );
458
- }
459
- }
460
- this.connected = true;
461
- this.everConnected = true;
462
- this.retryCount = 0;
463
- const pending = this.pending;
464
- this.pending = [];
465
- pending.forEach((event) => this.send(event));
466
- resolve();
467
- });
468
- socket.on("error", (error) => {
469
- clearTimeout(timeout);
470
- this.log("WebSocket error:", error);
471
- reject(new Error("WebSocket connection error"));
472
- });
473
- socket.on("close", (code, reason) => {
474
- clearTimeout(timeout);
475
- const wasConnected = this.connected;
476
- this.closeSocket();
477
- if (this.destroyed) return;
478
- this.log("Connection closed", { code, reason: reason.toString() });
479
- if (wasConnected) this.reconnect();
480
- });
481
- socket.on("message", (data) => {
482
- this.handleMessage(JSON.parse(data.toString()));
483
- });
484
- });
485
- }
486
- closeSocket() {
487
- const socket = this.socket;
488
- this.socket = void 0;
489
- this.connected = false;
525
+ /**
526
+ * Sets the session up, then sends what waited for it. A new connection
527
+ * starts from an empty conversation, so a reconnection sends it again first.
528
+ */
529
+ onOpen(reconnected) {
530
+ this.connection = "connected";
531
+ this.sendSessionUpdate();
532
+ if (reconnected) {
533
+ for (const message of this.conversation) {
534
+ if (message.role !== "user" && message.role !== "assistant") continue;
535
+ this.send(this.messageItem(message.role, message.content));
536
+ }
537
+ }
538
+ const pending = this.pending;
490
539
  this.pending = [];
491
- this.requested = [];
540
+ pending.forEach((event) => this.send(event));
541
+ }
542
+ /**
543
+ * Forgets what belonged to the session that is gone: its responses, and the
544
+ * answer in progress, which is lost with it.
545
+ */
546
+ resetSession() {
547
+ this.requested.clear();
492
548
  this.response = void 0;
493
- if (!socket) return;
494
- socket.removeAllListeners();
495
- socket.on("error", () => {
496
- });
497
- if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
498
- socket.close(1e3);
499
- }
549
+ this.ignoredResponses.clear();
550
+ this.audioItemId = void 0;
551
+ this.audioBytes = 0;
552
+ this.outputResampler.reset();
553
+ this.generation++;
500
554
  this.endAnswer();
501
555
  }
502
- reconnect() {
503
- if (this.destroyed) return;
504
- this.retryCount++;
505
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
506
- this.log("Max retries reached, giving up");
507
- this.emit("Failed");
508
- return;
509
- }
510
- this.log("Reconnecting...");
511
- this.reconnectTimeout = setTimeout(() => {
512
- this.reconnectTimeout = void 0;
513
- this.connect();
514
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
515
- }
516
556
  isIgnored(responseId) {
517
557
  return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
518
558
  }
519
559
  handleMessage(message) {
520
- switch (message.type) {
560
+ switch (message?.type) {
521
561
  case "response.created": {
522
- const requested = this.requested.shift() ?? {
523
- generation: this.generation,
524
- outOfBand: false
525
- };
526
562
  const id = message.response.id;
527
- if (requested.generation !== this.generation) {
563
+ const requestId = message.response.metadata?.[REQUEST_METADATA_KEY];
564
+ const requested = requestId ? this.requested.get(requestId) : { generation: this.generation, outOfBand: false };
565
+ if (requestId) this.requested.delete(requestId);
566
+ if (!requested || requested.generation !== this.generation) {
528
567
  this.ignoredResponses.add(id);
529
- this.socket?.send(
530
- JSON.stringify({ type: "response.cancel", response_id: id })
531
- );
568
+ this.send({ type: "response.cancel", response_id: id });
532
569
  break;
533
570
  }
534
571
  this.response = { id, ...requested };
@@ -566,10 +603,18 @@ var OpenaiRealtime = class extends Realtime {
566
603
  case "response.done":
567
604
  this.onResponseDone(message.response);
568
605
  break;
569
- case "error":
606
+ case "error": {
570
607
  if (message.error?.code === "response_cancel_not_active") break;
571
608
  this.log("Error:", message.error);
609
+ const requestId = message.error?.event_id;
610
+ const requested = this.requested.get(requestId);
611
+ if (!requested) break;
612
+ this.requested.delete(requestId);
613
+ if (requested.generation === this.generation && !this.response) {
614
+ this.endAnswer();
615
+ }
572
616
  break;
617
+ }
573
618
  default:
574
619
  break;
575
620
  }
@@ -618,70 +663,79 @@ var OpenaiRealtime = class extends Realtime {
618
663
  };
619
664
 
620
665
  // src/OpenaiSTT.ts
621
- import { Pcm16Resampler as Pcm16Resampler2, STT } from "@micdrop/server";
622
- import WebSocket2 from "ws";
666
+ import { Pcm16Resampler as Pcm16Resampler2, ReconnectingSocket as ReconnectingSocket2, STT } from "@micdrop/server";
623
667
  var DEFAULT_MODEL4 = "gpt-4o-transcribe";
624
668
  var DEFAULT_LANGUAGE = "en";
625
669
  var SAMPLE_RATE2 = 16e3;
626
670
  var OPENAI_SAMPLE_RATE2 = 24e3;
627
- var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
628
671
  var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
629
- var DEFAULT_RETRY_DELAY3 = 1e3;
630
- var DEFAULT_MAX_RETRY2 = 3;
631
672
  var OpenaiSTT = class extends STT {
632
673
  constructor(options) {
633
674
  super();
634
675
  this.options = options;
635
- this.retryCount = 0;
636
- this.destroyed = false;
637
- this.audioChunksPending = [];
638
- // Store audio chunks to send them again if reconnecting
639
- this.resampler = new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
640
- this.initPromise = this.initWS().catch((error) => {
641
- console.error("[OpenaiSTT] Connection error:", error);
642
- this.reconnect();
676
+ this.utteranceCount = 0;
677
+ // Utterances not transcribed yet, in the order they were spoken. Each one
678
+ // leaves the list with the single transcript it emits.
679
+ this.utterances = [];
680
+ // Commits sent and not acknowledged yet, in the order the API handles them
681
+ this.unacknowledged = [];
682
+ this.socket = new ReconnectingSocket2({
683
+ url: "wss://api.openai.com/v1/realtime?intent=transcription",
684
+ clientOptions: {
685
+ headers: { Authorization: `Bearer ${this.options.apiKey}` }
686
+ },
687
+ connectionTimeout: options.connectionTimeout,
688
+ retryDelay: options.retryDelay,
689
+ maxRetry: options.maxRetry,
690
+ handshake: sessionHandshake,
691
+ log: (...message) => this.log(...message)
692
+ });
693
+ this.socket.on("Open", () => this.onOpen());
694
+ this.socket.on("Close", () => {
695
+ this.unacknowledged = [];
643
696
  });
697
+ this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
698
+ this.socket.on("Failed", () => this.onFailed());
699
+ this.socket.connect();
644
700
  }
701
+ /**
702
+ * Transcribes one utterance, and emits its transcript once, possibly empty
703
+ * when none comes in time.
704
+ *
705
+ * After the connection was given up on, the next utterance tries it again.
706
+ */
645
707
  transcribe(audioStream) {
646
- this.resampler.reset();
647
- audioStream.on("data", async (chunk) => {
648
- this.audioChunksPending.push(chunk);
649
- await this.initPromise;
650
- this.sendAudioChunk(chunk);
651
- this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
708
+ const utterance = {
709
+ id: `micdrop-stt-${++this.utteranceCount}`,
710
+ status: "streaming",
711
+ chunks: [],
712
+ resampler: new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2)
713
+ };
714
+ this.utterances.push(utterance);
715
+ this.socket.connect();
716
+ audioStream.on("data", (chunk) => {
717
+ if (!this.utterances.includes(utterance)) return;
718
+ utterance.chunks.push(chunk);
719
+ this.sendAudioChunk(utterance, chunk);
652
720
  });
653
- audioStream.on("end", async () => {
654
- await this.initPromise;
655
- if (this.audioChunksPending.length === 0) return;
656
- this.commitAudio();
657
- this.transcriptionTimeout = setTimeout(() => {
658
- this.transcriptionTimeout = void 0;
659
- this.log("Transcription timeout");
660
- this.emit("Transcript", "");
661
- this.audioChunksPending.length = 0;
662
- }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
721
+ audioStream.on("end", () => {
722
+ if (!this.utterances.includes(utterance)) return;
723
+ if (utterance.chunks.length === 0) {
724
+ this.remove(utterance);
725
+ return;
726
+ }
727
+ utterance.status = "committed";
728
+ this.commitAudio(utterance);
663
729
  });
664
730
  }
665
731
  destroy() {
666
732
  super.destroy();
667
- this.destroyed = true;
668
- if (this.reconnectTimeout) {
669
- clearTimeout(this.reconnectTimeout);
670
- this.reconnectTimeout = void 0;
671
- }
672
- if (this.transcriptionTimeout) {
673
- clearTimeout(this.transcriptionTimeout);
674
- this.transcriptionTimeout = void 0;
675
- }
676
- if (this.connectionTimeout) {
677
- clearTimeout(this.connectionTimeout);
678
- this.connectionTimeout = void 0;
679
- }
680
- if (this.socket) closeSocket(this.socket);
681
- this.socket = void 0;
733
+ this.socket.destroy();
734
+ this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
735
+ this.utterances = [];
736
+ this.unacknowledged = [];
682
737
  }
683
738
  sendSessionUpdate() {
684
- if (!this.socket) return;
685
739
  const model = this.options.model || DEFAULT_MODEL4;
686
740
  this.socket.send(
687
741
  JSON.stringify({
@@ -715,56 +769,35 @@ var OpenaiSTT = class extends STT {
715
769
  })
716
770
  );
717
771
  }
718
- async initWS() {
719
- return new Promise((resolve, reject) => {
720
- const socket = new WebSocket2(
721
- "wss://api.openai.com/v1/realtime?intent=transcription",
722
- {
723
- headers: {
724
- Authorization: `Bearer ${this.options.apiKey}`
725
- }
726
- }
727
- );
728
- this.socket = socket;
729
- const timeout = setTimeout(() => {
730
- this.log("Connection timeout");
731
- closeSocket(socket);
732
- this.socket = void 0;
733
- reject(new Error("WebSocket connection timeout"));
734
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
735
- this.connectionTimeout = timeout;
736
- socket.addEventListener("open", () => {
737
- clearTimeout(timeout);
738
- this.sendSessionUpdate();
739
- this.log("Connection opened");
740
- resolve();
741
- });
742
- socket.addEventListener("error", (error) => {
743
- clearTimeout(timeout);
744
- this.log("WebSocket error:", error);
745
- reject(new Error("WebSocket connection error"));
746
- });
747
- socket.addEventListener(
748
- "close",
749
- ({ code, reason }) => {
750
- clearTimeout(timeout);
751
- this.socket?.removeAllListeners();
752
- this.socket = void 0;
753
- if (code !== 1e3) {
754
- this.reconnect();
755
- } else {
756
- this.log("Connection closed", { code, reason });
757
- }
758
- }
759
- );
760
- socket.addEventListener("message", (event) => {
761
- this.handleMessage(JSON.parse(event.data.toString()));
762
- });
763
- });
772
+ /**
773
+ * Configures the session, then sends again what a previous connection did
774
+ * not get transcribed. The OpenAI buffer of a new session is empty, so each
775
+ * utterance is replayed through a fresh resampler, which reproduces the exact
776
+ * same stream, and committed again if it was.
777
+ */
778
+ onOpen() {
779
+ this.sendSessionUpdate();
780
+ this.unacknowledged = [];
781
+ if (this.utterances.length > 0) this.log("Sending audio chunks again");
782
+ for (const utterance of this.utterances) {
783
+ if (utterance.transcript !== void 0) continue;
784
+ utterance.itemId = void 0;
785
+ utterance.resampler = new Pcm16Resampler2(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
786
+ utterance.chunks.forEach((chunk) => this.sendAudioChunk(utterance, chunk));
787
+ if (utterance.status === "committed") this.commitAudio(utterance);
788
+ }
764
789
  }
765
- sendAudioChunk(chunk) {
766
- if (!this.socket) return;
767
- const resampled = this.resampler.process(chunk);
790
+ /** Every retry failed: the audio waiting for a transcript is handed back */
791
+ onFailed() {
792
+ const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
793
+ this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
794
+ this.utterances = [];
795
+ this.unacknowledged = [];
796
+ this.emit("Failed", chunks);
797
+ }
798
+ sendAudioChunk(utterance, chunk) {
799
+ if (!this.socket.isOpen) return;
800
+ const resampled = utterance.resampler.process(chunk);
768
801
  if (resampled.length === 0) return;
769
802
  this.socket.send(
770
803
  JSON.stringify({
@@ -773,91 +806,108 @@ var OpenaiSTT = class extends STT {
773
806
  })
774
807
  );
775
808
  }
809
+ commitAudio(utterance) {
810
+ if (!this.socket.isOpen) return;
811
+ this.socket.send(
812
+ JSON.stringify({
813
+ type: "input_audio_buffer.commit",
814
+ event_id: utterance.id
815
+ })
816
+ );
817
+ this.unacknowledged.push(utterance);
818
+ this.log("Committed audio buffer");
819
+ clearTimeout(utterance.timer);
820
+ utterance.timer = setTimeout(() => {
821
+ utterance.timer = void 0;
822
+ this.log("Transcription timeout");
823
+ this.finish(utterance, "");
824
+ }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
825
+ }
826
+ /**
827
+ * Settles the transcript of an utterance, the only one it gets.
828
+ *
829
+ * The API transcribes each utterance on its own and may finish a short one
830
+ * before a long one spoken earlier, so transcripts go out in the order the
831
+ * utterances were spoken.
832
+ */
833
+ finish(utterance, transcript) {
834
+ if (!this.utterances.includes(utterance)) return;
835
+ if (utterance.transcript !== void 0) return;
836
+ clearTimeout(utterance.timer);
837
+ utterance.timer = void 0;
838
+ utterance.transcript = transcript;
839
+ while (this.utterances[0]?.transcript !== void 0) {
840
+ const first = this.utterances[0];
841
+ this.remove(first);
842
+ this.emit("Transcript", first.transcript);
843
+ }
844
+ }
845
+ remove(utterance) {
846
+ clearTimeout(utterance.timer);
847
+ utterance.timer = void 0;
848
+ this.utterances = this.utterances.filter((other) => other !== utterance);
849
+ this.unacknowledged = this.unacknowledged.filter(
850
+ (other) => other !== utterance
851
+ );
852
+ }
853
+ /**
854
+ * The utterance a transcript event is about. The API names the item, and
855
+ * the item of each utterance is known from the acknowledgement of its
856
+ * commit. A transcript of an utterance that timed out matches none.
857
+ */
858
+ findUtterance(itemId) {
859
+ if (!itemId) {
860
+ return this.utterances.find(
861
+ ({ status, transcript }) => status === "committed" && transcript === void 0
862
+ );
863
+ }
864
+ return this.utterances.find((utterance) => utterance.itemId === itemId);
865
+ }
776
866
  handleMessage(message) {
777
- switch (message.type) {
778
- // case 'input_audio_buffer.committed':
779
- // this.log('Audio buffer committed')
780
- // break
781
- // case 'input_audio_buffer.speech_started':
782
- // this.log('Speech started')
783
- // break
784
- // case 'input_audio_buffer.speech_stopped':
785
- // this.log('Speech stopped')
786
- // break
787
- // case 'conversation.item.input_audio_transcription.delta':
788
- // this.log(`Received transcript delta: "${message.delta}"`)
789
- // break
790
- case "conversation.item.input_audio_transcription.completed":
867
+ switch (message?.type) {
868
+ case "input_audio_buffer.committed": {
869
+ const utterance = this.unacknowledged.shift();
870
+ if (utterance) utterance.itemId = message.item_id;
871
+ break;
872
+ }
873
+ case "conversation.item.input_audio_transcription.completed": {
791
874
  const transcript = message.transcript || "";
792
875
  this.log(`Received completed transcript: "${transcript}"`);
793
- this.emit("Transcript", transcript);
794
- this.audioChunksPending.length = 0;
795
- if (this.transcriptionTimeout) {
796
- clearTimeout(this.transcriptionTimeout);
797
- this.transcriptionTimeout = void 0;
876
+ const utterance = this.findUtterance(message.item_id);
877
+ if (!utterance) {
878
+ this.log("Transcript of an utterance already given up on, ignored");
879
+ break;
798
880
  }
881
+ this.finish(utterance, transcript);
799
882
  break;
800
- case "session.created":
801
- this.retryCount = 0;
883
+ }
884
+ case "conversation.item.input_audio_transcription.failed": {
885
+ this.log("Transcription failed:", message.error);
886
+ const utterance = this.findUtterance(message.item_id);
887
+ if (utterance) this.finish(utterance, "");
802
888
  break;
803
- case "error":
889
+ }
890
+ case "error": {
804
891
  this.log("Error:", message.error);
892
+ const utterance = this.utterances.find(
893
+ ({ id }) => id === message.error?.event_id
894
+ );
895
+ if (utterance) this.finish(utterance, "");
805
896
  break;
897
+ }
806
898
  default:
807
899
  break;
808
900
  }
809
901
  }
810
- reconnect() {
811
- if (this.destroyed) return;
812
- this.retryCount++;
813
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
814
- this.log("Max retries reached, giving up");
815
- this.emit("Failed", this.audioChunksPending);
816
- return;
817
- }
818
- this.initPromise = new Promise((resolve) => {
819
- this.log("Reconnecting...");
820
- this.reconnectTimeout = setTimeout(() => {
821
- this.reconnectTimeout = void 0;
822
- this.initWS().then(() => {
823
- if (this.audioChunksPending.length > 0) {
824
- this.log("Sending audio chunks again");
825
- this.resampler.reset();
826
- this.audioChunksPending.forEach(
827
- (chunk) => this.sendAudioChunk(chunk)
828
- );
829
- }
830
- }).then(resolve).catch((error) => {
831
- this.log("Reconnection error:", error);
832
- this.reconnect();
833
- });
834
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
835
- });
836
- }
837
- commitAudio() {
838
- if (!this.socket) return;
839
- this.socket.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
840
- this.log("Committed audio buffer");
841
- }
842
902
  };
843
- function closeSocket(socket) {
844
- socket.removeAllListeners();
845
- socket.on("error", () => {
846
- });
847
- if (socket.readyState === WebSocket2.OPEN || socket.readyState === WebSocket2.CONNECTING) {
848
- socket.close(1e3);
849
- }
850
- }
851
903
 
852
904
  // src/OpenaiTTS.ts
853
905
  import { Pcm16Resampler as Pcm16Resampler4, SentenceSplitter, TTS } from "@micdrop/server";
854
906
  import OpenAI3 from "openai";
855
907
 
856
908
  // src/OpenaiRealtimeSpeech.ts
857
- import { Pcm16Resampler as Pcm16Resampler3 } from "@micdrop/server";
858
- import WebSocket3 from "ws";
909
+ import { Pcm16Resampler as Pcm16Resampler3, ReconnectingSocket as ReconnectingSocket3 } from "@micdrop/server";
859
910
  var DEFAULT_REASONING_EFFORT = "minimal";
860
- var DEFAULT_CONNECTION_TIMEOUT3 = 5e3;
861
911
  var MIN_SPEED = 0.25;
862
912
  var MAX_SPEED = 1.5;
863
913
  var OPENAI_SAMPLE_RATE3 = 24e3;
@@ -870,22 +920,48 @@ var OpenaiRealtimeSpeech = class {
870
920
  constructor(options, log) {
871
921
  this.options = options;
872
922
  this.log = log;
873
- this.destroyed = false;
874
923
  this.itemCount = 0;
875
924
  // Responses asked for and not finished, by their metadata id and by the id
876
- // the API gave them. Cancelled ones leave both maps, which mutes them.
925
+ // the API gave them. Finished ones leave both maps, which mutes them.
877
926
  this.items = /* @__PURE__ */ new Map();
878
927
  this.responses = /* @__PURE__ */ new Map();
879
- this.connect().catch(() => {
928
+ const { model, apiKey } = options;
929
+ this.socket = new ReconnectingSocket3({
930
+ url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
931
+ clientOptions: { headers: { Authorization: `Bearer ${apiKey}` } },
932
+ connectionTimeout: options.connectionTimeout,
933
+ handshake: sessionHandshake,
934
+ log
935
+ });
936
+ this.socket.on("Open", () => {
937
+ this.socket.send(JSON.stringify(this.sessionUpdate()));
880
938
  });
939
+ this.socket.on("Close", () => {
940
+ for (const item of [...this.items.values()]) {
941
+ if (item.status === "created") {
942
+ this.finish(item, "failed", new Error("Connection lost"));
943
+ } else if (item.status === "requested") {
944
+ item.status = "waiting";
945
+ this.request(item);
946
+ }
947
+ }
948
+ });
949
+ this.socket.on("Message", (data) => {
950
+ try {
951
+ this.handleMessage(parseMessage(data));
952
+ } catch (error) {
953
+ this.log("Error handling message", error);
954
+ }
955
+ });
956
+ this.socket.connect();
881
957
  }
882
958
  synthesize(text) {
883
959
  const item = {
884
960
  id: `micdrop-tts-${++this.itemCount}`,
885
961
  text,
962
+ status: "waiting",
886
963
  resampler: new Pcm16Resampler3(OPENAI_SAMPLE_RATE3, OUTPUT_SAMPLE_RATE),
887
- audio: [],
888
- done: false
964
+ audio: []
889
965
  };
890
966
  this.items.set(item.id, item);
891
967
  this.request(item);
@@ -895,13 +971,28 @@ var OpenaiRealtimeSpeech = class {
895
971
  };
896
972
  }
897
973
  destroy() {
898
- this.destroyed = true;
899
- this.closeSocket();
974
+ this.socket.destroy();
975
+ for (const item of [...this.items.values()]) {
976
+ this.finish(item, "failed", new Error("TTS destroyed"));
977
+ }
900
978
  }
901
- request(item) {
902
- this.connect().then(() => {
903
- if (!this.items.has(item.id)) return;
904
- this.send({
979
+ /**
980
+ * Sends the sentence once the connection is open.
981
+ *
982
+ * A Realtime session lasts an hour at most, and the connection opens again
983
+ * on its own when the API closes it. After every retry failed, the next
984
+ * sentence starts over with a full retry budget.
985
+ */
986
+ async request(item) {
987
+ this.socket.connect();
988
+ const open = await this.socket.ready();
989
+ if (item.status !== "waiting") return;
990
+ if (!open) {
991
+ this.finish(item, "failed", new Error("WebSocket connection failed"));
992
+ return;
993
+ }
994
+ const sent = this.socket.send(
995
+ JSON.stringify({
905
996
  type: "response.create",
906
997
  // Errors caused by this event carry its id
907
998
  event_id: item.id,
@@ -927,8 +1018,13 @@ var OpenaiRealtimeSpeech = class {
927
1018
  output_modalities: ["audio"],
928
1019
  metadata: { micdrop_id: item.id }
929
1020
  }
930
- });
931
- }).catch((error) => this.finish(item, error));
1021
+ })
1022
+ );
1023
+ if (sent) {
1024
+ item.status = "requested";
1025
+ } else {
1026
+ this.finish(item, "failed", new Error("WebSocket connection lost"));
1027
+ }
932
1028
  }
933
1029
  async *read(item) {
934
1030
  while (true) {
@@ -937,21 +1033,22 @@ var OpenaiRealtimeSpeech = class {
937
1033
  yield chunk;
938
1034
  continue;
939
1035
  }
940
- if (item.error) throw item.error;
941
- if (item.done) return;
1036
+ if (item.status === "failed") throw item.error;
1037
+ if (item.status === "done" || item.status === "cancelled") return;
942
1038
  await new Promise((resolve) => item.wake = resolve);
943
1039
  }
944
1040
  }
945
1041
  cancel(item) {
946
- if (item.responseId && !item.done) {
1042
+ if (item.status === "created" && item.responseId) {
947
1043
  this.send({ type: "response.cancel", response_id: item.responseId });
948
1044
  }
949
1045
  item.audio = [];
950
- this.finish(item);
1046
+ this.finish(item, "cancelled");
951
1047
  }
952
- finish(item, error) {
1048
+ /** Ends a sentence, the first end being the only one that counts */
1049
+ finish(item, status, error) {
953
1050
  if (!this.items.has(item.id)) return;
954
- item.done = true;
1051
+ item.status = status;
955
1052
  item.error = error;
956
1053
  this.items.delete(item.id);
957
1054
  if (item.responseId) this.responses.delete(item.responseId);
@@ -960,9 +1057,7 @@ var OpenaiRealtimeSpeech = class {
960
1057
  wake?.();
961
1058
  }
962
1059
  send(event) {
963
- if (this.socket?.readyState === WebSocket3.OPEN) {
964
- this.socket.send(JSON.stringify(event));
965
- }
1060
+ this.socket.send(JSON.stringify(event));
966
1061
  }
967
1062
  sessionUpdate() {
968
1063
  const { instructions, speed } = this.options;
@@ -989,80 +1084,8 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
989
1084
  }
990
1085
  };
991
1086
  }
992
- /**
993
- * Opens the session, or hands over the one already open.
994
- *
995
- * A Realtime session lasts an hour at most. A closed one is opened again by
996
- * the next sentence rather than on a timer, so an idle agent holds no
997
- * connection.
998
- */
999
- connect() {
1000
- if (this.destroyed) return Promise.reject(new Error("TTS destroyed"));
1001
- if (this.socket?.readyState === WebSocket3.OPEN) {
1002
- return Promise.resolve(this.socket);
1003
- }
1004
- if (this.connecting) return this.connecting;
1005
- const { model, apiKey } = this.options;
1006
- const connecting = new Promise((resolve, reject) => {
1007
- const socket = new WebSocket3(
1008
- `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
1009
- { headers: { Authorization: `Bearer ${apiKey}` } }
1010
- );
1011
- this.socket = socket;
1012
- const timeout = setTimeout(() => {
1013
- this.log("Connection timeout");
1014
- this.closeSocket();
1015
- reject(new Error("WebSocket connection timeout"));
1016
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT3);
1017
- socket.on("open", () => {
1018
- clearTimeout(timeout);
1019
- this.log("Connection opened");
1020
- socket.send(JSON.stringify(this.sessionUpdate()));
1021
- resolve(socket);
1022
- });
1023
- socket.on("error", (error) => {
1024
- clearTimeout(timeout);
1025
- this.log("WebSocket error:", error);
1026
- reject(new Error("WebSocket connection error"));
1027
- });
1028
- socket.on("close", (code, reason) => {
1029
- clearTimeout(timeout);
1030
- this.log("Connection closed", { code, reason: reason.toString() });
1031
- reject(new Error("WebSocket connection closed"));
1032
- this.closeSocket();
1033
- });
1034
- socket.on("message", (data) => {
1035
- try {
1036
- this.handleMessage(JSON.parse(data.toString()));
1037
- } catch (error) {
1038
- this.log("Error handling message", error);
1039
- }
1040
- });
1041
- });
1042
- this.connecting = connecting;
1043
- const settle = () => {
1044
- if (this.connecting === connecting) this.connecting = void 0;
1045
- };
1046
- connecting.then(settle, settle);
1047
- return connecting;
1048
- }
1049
- closeSocket() {
1050
- const socket = this.socket;
1051
- this.socket = void 0;
1052
- this.connecting = void 0;
1053
- for (const item of [...this.items.values()]) {
1054
- this.finish(item, new Error("Connection lost"));
1055
- }
1056
- if (!socket) return;
1057
- socket.removeAllListeners();
1058
- socket.on("error", () => {
1059
- });
1060
- if (socket.readyState === WebSocket3.OPEN || socket.readyState === WebSocket3.CONNECTING) {
1061
- socket.close(1e3);
1062
- }
1063
- }
1064
1087
  handleMessage(message) {
1065
- switch (message.type) {
1088
+ switch (message?.type) {
1066
1089
  case "response.created": {
1067
1090
  const id = message.response.id;
1068
1091
  const item = this.items.get(message.response.metadata?.micdrop_id);
@@ -1070,6 +1093,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1070
1093
  this.send({ type: "response.cancel", response_id: id });
1071
1094
  break;
1072
1095
  }
1096
+ item.status = "created";
1073
1097
  item.responseId = id;
1074
1098
  this.responses.set(id, item);
1075
1099
  break;
@@ -1101,12 +1125,13 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1101
1125
  if (response.status === "failed") {
1102
1126
  this.finish(
1103
1127
  item,
1128
+ "failed",
1104
1129
  new Error(
1105
1130
  `Response failed: ${JSON.stringify(response.status_details)}`
1106
1131
  )
1107
1132
  );
1108
1133
  } else {
1109
- this.finish(item);
1134
+ this.finish(item, "done");
1110
1135
  }
1111
1136
  break;
1112
1137
  }
@@ -1114,7 +1139,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1114
1139
  if (message.error?.code === "response_cancel_not_active") break;
1115
1140
  this.log("Error:", message.error);
1116
1141
  const item = this.items.get(message.error?.event_id);
1117
- if (item) this.finish(item, new Error(message.error?.message));
1142
+ if (item) this.finish(item, "failed", new Error(message.error?.message));
1118
1143
  break;
1119
1144
  }
1120
1145
  default:
@@ -1194,15 +1219,17 @@ var OpenaiTTS = class extends TTS {
1194
1219
  this.enqueue(counter, sentence);
1195
1220
  }
1196
1221
  });
1197
- textStream.on("error", (error) => {
1198
- this.log("Error in text stream", error);
1199
- });
1200
- textStream.on("end", () => {
1222
+ const flush = () => {
1201
1223
  if (!counter || counter !== this.counter) return;
1202
1224
  for (const sentence of this.splitter.flush()) {
1203
1225
  this.enqueue(counter, sentence);
1204
1226
  }
1227
+ };
1228
+ textStream.on("error", (error) => {
1229
+ this.log("Error in text stream", error);
1230
+ flush();
1205
1231
  });
1232
+ textStream.on("end", flush);
1206
1233
  }
1207
1234
  cancel() {
1208
1235
  this.generation++;