@micdrop/openai 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -49,19 +49,69 @@ var DEFAULT_MODEL = "gpt-4o";
49
49
  var DEFAULT_MAX_STEPS = 5;
50
50
  var DEFAULT_MAX_RETRIES = 3;
51
51
  var DEFAULT_RETRY_DELAY = 1e3;
52
+ var StepInterruptedError = class extends Error {
53
+ constructor(outcome, reason) {
54
+ super("Answer interrupted");
55
+ this.outcome = outcome;
56
+ this.reason = reason;
57
+ }
58
+ };
52
59
  var OpenaiAgent = class extends import_server.Agent {
53
60
  constructor(options) {
54
61
  super(options);
55
62
  this.openai = "openai" in options ? options.openai : new import_openai.default({ apiKey: options.apiKey });
56
63
  }
57
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
58
- if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
59
- console.error("[OpenaiAgent] Max steps reached");
60
- return;
64
+ async generateAnswer(stream, signal) {
65
+ const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
66
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
67
+ for (let step = 0; step < maxSteps; step++) {
68
+ let outcome;
69
+ for (let tryCount = 0; !outcome; tryCount++) {
70
+ try {
71
+ outcome = await this.generateStep(stream, signal);
72
+ } catch (error) {
73
+ if (signal.aborted) return;
74
+ if (error instanceof StepInterruptedError) {
75
+ console.error("[OpenaiAgent] Answer interrupted:", error.reason);
76
+ outcome = error.outcome;
77
+ break;
78
+ }
79
+ console.error("[OpenaiAgent] Error answering:", error);
80
+ if (tryCount >= maxRetry) {
81
+ this.log("Max retries reached, giving up");
82
+ this.emit("Failed");
83
+ return;
84
+ }
85
+ await new Promise(
86
+ (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
87
+ );
88
+ if (signal.aborted) return;
89
+ }
90
+ }
91
+ if (signal.aborted) return;
92
+ if (outcome.recorded || outcome.skipAnswer) return;
93
+ if (outcome.text) {
94
+ const { message, metadata } = this.extract(outcome.text);
95
+ this.addAssistantMessage(message, metadata);
96
+ return;
97
+ }
61
98
  }
62
- const abortController = new AbortController();
63
- this.abortController = abortController;
64
- const signal = abortController.signal;
99
+ console.error("[OpenaiAgent] Max steps reached");
100
+ }
101
+ /**
102
+ * Runs one request, writing its text to the stream and running its tools.
103
+ *
104
+ * It throws the error of the request when it failed before producing
105
+ * anything, which can then be retried, and a {@link StepInterruptedError}
106
+ * once something was written or done.
107
+ */
108
+ async generateStep(stream, signal) {
109
+ const outcome = {
110
+ text: "",
111
+ recorded: false,
112
+ skipAnswer: false
113
+ };
114
+ let produced = false;
65
115
  let extracting = false;
66
116
  const extractOptions = this.getExtractOptions();
67
117
  try {
@@ -75,14 +125,14 @@ var OpenaiAgent = class extends import_server.Agent {
75
125
  },
76
126
  { signal }
77
127
  );
78
- let skipAnswer = false;
79
- let hasAnswer = false;
80
128
  for await (const event of response) {
81
- if (abortController !== this.abortController) return;
129
+ if (signal.aborted) return outcome;
82
130
  switch (event.type) {
83
131
  case "response.output_text.delta":
84
- if (hasAnswer) break;
132
+ if (outcome.recorded) break;
85
133
  this.log(`Answer chunk: "${event.delta}"`);
134
+ outcome.text += event.delta;
135
+ produced = true;
86
136
  if (extractOptions) {
87
137
  if (!extracting) {
88
138
  const startTagIndex = event.delta.indexOf(
@@ -100,15 +150,17 @@ var OpenaiAgent = class extends import_server.Agent {
100
150
  }
101
151
  stream.write(event.delta);
102
152
  break;
103
- case "response.output_text.done":
104
- if (hasAnswer) break;
153
+ case "response.output_text.done": {
154
+ if (outcome.recorded) break;
105
155
  const { message, metadata } = this.extract(event.text);
106
156
  this.addAssistantMessage(message, metadata);
107
- hasAnswer = true;
157
+ outcome.recorded = true;
108
158
  break;
159
+ }
109
160
  case "response.output_item.done":
110
161
  if (event.item.type === "function_call") {
111
162
  const toolCall = event.item;
163
+ produced = true;
112
164
  const result = await this.executeTool({
113
165
  role: "tool_call",
114
166
  toolCallId: toolCall.call_id,
@@ -116,7 +168,7 @@ var OpenaiAgent = class extends import_server.Agent {
116
168
  parameters: toolCall.arguments
117
169
  });
118
170
  if (result.skipAnswer) {
119
- skipAnswer = true;
171
+ outcome.skipAnswer = true;
120
172
  }
121
173
  }
122
174
  break;
@@ -124,23 +176,12 @@ var OpenaiAgent = class extends import_server.Agent {
124
176
  break;
125
177
  }
126
178
  }
127
- if (abortController !== this.abortController) return;
128
- if (!skipAnswer && !hasAnswer) {
129
- await this.generateAnswer(stream, stepCount + 1);
130
- }
131
- this.abortController = void 0;
179
+ return outcome;
132
180
  } catch (error) {
133
- if (error instanceof import_openai.default.APIUserAbortError) return;
134
- console.error("[OpenaiAgent] Error answering:", error);
135
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
136
- await new Promise(
137
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
138
- );
139
- await this.generateAnswer(stream, stepCount, tryCount + 1);
140
- } else {
141
- this.log("Max retries reached, giving up");
142
- this.emit("Failed");
181
+ if (produced && !signal.aborted) {
182
+ throw new StepInterruptedError(outcome, error);
143
183
  }
184
+ throw error;
144
185
  }
145
186
  }
146
187
  buildInput() {
@@ -158,7 +199,8 @@ var OpenaiAgent = class extends import_server.Agent {
158
199
  type: "function_call",
159
200
  call_id: message.toolCallId,
160
201
  name: message.toolName,
161
- arguments: JSON.stringify(message.parameters)
202
+ // Already the JSON the model wrote
203
+ arguments: message.parameters
162
204
  };
163
205
  case "tool_result":
164
206
  return {
@@ -180,12 +222,6 @@ var OpenaiAgent = class extends import_server.Agent {
180
222
  parameters: (0, import_zod.toJSONSchema)(tool.inputSchema || import_zod.default.object())
181
223
  }));
182
224
  }
183
- cancel() {
184
- if (!this.abortController) return;
185
- this.log("Cancel");
186
- this.abortController.abort();
187
- this.abortController = void 0;
188
- }
189
225
  };
190
226
 
191
227
  // src/OpenaiClassifier.ts
@@ -290,9 +326,26 @@ function buildQuestion(name, question) {
290
326
 
291
327
  // src/OpenaiRealtime.ts
292
328
  var import_server3 = require("@micdrop/server");
293
- var import_ws = __toESM(require("ws"));
294
329
  var import_zod2 = __toESM(require("zod"));
295
330
 
331
+ // src/session.ts
332
+ var sessionHandshake = {
333
+ read(data) {
334
+ const message = parseMessage(data);
335
+ if (message?.type === "error") {
336
+ throw new Error(message.error?.message || "Session refused");
337
+ }
338
+ return message?.type === "session.created" || message?.type === "transcription_session.created";
339
+ }
340
+ };
341
+ function parseMessage(data) {
342
+ try {
343
+ return JSON.parse(data.toString());
344
+ } catch {
345
+ return void 0;
346
+ }
347
+ }
348
+
296
349
  // src/transcription.ts
297
350
  function transcriptionLanguage(model, language) {
298
351
  if (!language) return {};
@@ -309,36 +362,56 @@ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
309
362
  var SAMPLE_RATE = 16e3;
310
363
  var OPENAI_SAMPLE_RATE = 24e3;
311
364
  var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
312
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
313
- var DEFAULT_RETRY_DELAY2 = 1e3;
314
- var DEFAULT_MAX_RETRY = 3;
365
+ var MAX_PENDING_EVENTS = 5e3;
366
+ var REQUEST_METADATA_KEY = "micdrop_request";
315
367
  var OpenaiRealtime = class extends import_server3.Realtime {
316
368
  constructor(options) {
317
369
  super(options);
318
- this.connected = false;
319
- this.everConnected = false;
320
- this.destroyed = false;
370
+ this.connection = "connecting";
321
371
  // Events sent while the connection opens
322
372
  this.pending = [];
323
- this.retryCount = 0;
373
+ this.requestCount = 0;
324
374
  this.inputResampler = new import_server3.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
325
375
  this.outputResampler = new import_server3.Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
326
376
  // Bumped by every interruption, so late responses can tell they are stale
327
377
  this.generation = 0;
328
- this.requested = [];
378
+ // Responses asked for and not created yet, by the id sent as their metadata
379
+ // and as the event id of their request, which an error names
380
+ this.requested = /* @__PURE__ */ new Map();
329
381
  this.ignoredResponses = /* @__PURE__ */ new Set();
330
382
  this.audioStartedAt = 0;
331
383
  this.audioBytes = 0;
332
- this.connect();
384
+ const model = options.model || DEFAULT_MODEL3;
385
+ this.socket = new import_server3.ReconnectingSocket({
386
+ url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
387
+ clientOptions: {
388
+ headers: { Authorization: `Bearer ${options.apiKey}` }
389
+ },
390
+ connectionTimeout: options.connectionTimeout,
391
+ retryDelay: options.retryDelay,
392
+ maxRetry: options.maxRetry,
393
+ handshake: sessionHandshake,
394
+ log: (...message) => this.log(...message)
395
+ });
396
+ this.socket.on("Open", ({ reconnected }) => this.onOpen(reconnected));
397
+ this.socket.on("Close", () => {
398
+ this.connection = "reconnecting";
399
+ this.resetSession();
400
+ });
401
+ this.socket.on("Failed", () => {
402
+ this.connection = "closed";
403
+ this.pending = [];
404
+ this.resetSession();
405
+ this.emit("Failed");
406
+ });
407
+ this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
408
+ this.socket.connect();
333
409
  }
334
410
  destroy() {
335
- this.destroyed = true;
336
411
  super.destroy();
337
- if (this.reconnectTimeout) {
338
- clearTimeout(this.reconnectTimeout);
339
- this.reconnectTimeout = void 0;
340
- }
341
- this.closeSocket();
412
+ this.connection = "closed";
413
+ this.pending = [];
414
+ this.socket.destroy();
342
415
  }
343
416
  openTurn() {
344
417
  this.inputResampler.reset();
@@ -390,25 +463,49 @@ var OpenaiRealtime = class extends import_server3.Realtime {
390
463
  this.audioItemId = void 0;
391
464
  }
392
465
  updateTools() {
393
- if (this.connected) this.sendSessionUpdate();
466
+ if (this.connection === "connected") this.sendSessionUpdate();
394
467
  }
395
468
  injectMessage(role, text) {
396
- if (!this.connected && this.everConnected) return;
469
+ if (this.connection === "reconnecting") return;
397
470
  this.send(this.messageItem(role, text));
398
471
  }
472
+ /**
473
+ * Asks for a response, tagged so its `response.created` tells which request
474
+ * it answers. Matching them by order alone would shift every later response
475
+ * onto the wrong request as soon as one is refused.
476
+ */
399
477
  createResponse(response) {
400
- this.requested.push({
401
- generation: this.generation,
402
- outOfBand: !!response
478
+ const id = `micdrop-response-${++this.requestCount}`;
479
+ const generation = this.generation;
480
+ const accepted = this.send({
481
+ type: "response.create",
482
+ // Errors caused by this event carry its id
483
+ event_id: id,
484
+ response: { ...response, metadata: { [REQUEST_METADATA_KEY]: id } }
403
485
  });
404
- this.send({ type: "response.create", ...response ? { response } : {} });
486
+ if (!accepted) {
487
+ queueMicrotask(() => {
488
+ if (generation === this.generation) this.endAnswer();
489
+ });
490
+ return;
491
+ }
492
+ this.requested.set(id, { generation, outOfBand: !!response });
405
493
  }
494
+ /**
495
+ * Sends an event, or holds it until the connection opens.
496
+ * @returns False when it was dropped, the connection being given up on
497
+ */
406
498
  send(event) {
407
- if (this.connected && this.socket?.readyState === import_ws.default.OPEN) {
408
- this.socket.send(JSON.stringify(event));
409
- } else {
410
- this.pending.push(event);
499
+ if (this.connection === "connected") {
500
+ return this.socket.send(JSON.stringify(event));
411
501
  }
502
+ if (this.connection === "closed") return false;
503
+ if (this.pending.length >= MAX_PENDING_EVENTS) {
504
+ this.log("Too many events waiting for the connection, dropped");
505
+ return false;
506
+ }
507
+ this.pending.push(event);
508
+ return true;
412
509
  }
413
510
  messageItem(role, text) {
414
511
  return {
@@ -427,7 +524,7 @@ var OpenaiRealtime = class extends import_server3.Realtime {
427
524
  }
428
525
  sendSessionUpdate() {
429
526
  const transcriptionModel = this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
430
- this.socket?.send(
527
+ this.socket.send(
431
528
  JSON.stringify({
432
529
  type: "session.update",
433
530
  session: {
@@ -464,111 +561,50 @@ var OpenaiRealtime = class extends import_server3.Realtime {
464
561
  })
465
562
  );
466
563
  }
467
- connect() {
468
- this.initWS().catch((error) => {
469
- this.log("Connection error:", error);
470
- this.reconnect();
471
- });
472
- }
473
- initWS() {
474
- return new Promise((resolve, reject) => {
475
- const model = this.options.model || DEFAULT_MODEL3;
476
- const socket = new import_ws.default(
477
- `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
478
- { headers: { Authorization: `Bearer ${this.options.apiKey}` } }
479
- );
480
- this.socket = socket;
481
- const timeout = setTimeout(() => {
482
- this.log("Connection timeout");
483
- this.closeSocket();
484
- reject(new Error("WebSocket connection timeout"));
485
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
486
- socket.on("open", () => {
487
- clearTimeout(timeout);
488
- this.log("Connection opened");
489
- this.sendSessionUpdate();
490
- if (this.everConnected) {
491
- for (const message of this.conversation) {
492
- if (message.role !== "user" && message.role !== "assistant") {
493
- continue;
494
- }
495
- socket.send(
496
- JSON.stringify(this.messageItem(message.role, message.content))
497
- );
498
- }
499
- }
500
- this.connected = true;
501
- this.everConnected = true;
502
- this.retryCount = 0;
503
- const pending = this.pending;
504
- this.pending = [];
505
- pending.forEach((event) => this.send(event));
506
- resolve();
507
- });
508
- socket.on("error", (error) => {
509
- clearTimeout(timeout);
510
- this.log("WebSocket error:", error);
511
- reject(new Error("WebSocket connection error"));
512
- });
513
- socket.on("close", (code, reason) => {
514
- clearTimeout(timeout);
515
- const wasConnected = this.connected;
516
- this.closeSocket();
517
- if (this.destroyed) return;
518
- this.log("Connection closed", { code, reason: reason.toString() });
519
- if (wasConnected) this.reconnect();
520
- });
521
- socket.on("message", (data) => {
522
- this.handleMessage(JSON.parse(data.toString()));
523
- });
524
- });
525
- }
526
- closeSocket() {
527
- const socket = this.socket;
528
- this.socket = void 0;
529
- this.connected = false;
564
+ /**
565
+ * Sets the session up, then sends what waited for it. A new connection
566
+ * starts from an empty conversation, so a reconnection sends it again first.
567
+ */
568
+ onOpen(reconnected) {
569
+ this.connection = "connected";
570
+ this.sendSessionUpdate();
571
+ if (reconnected) {
572
+ for (const message of this.conversation) {
573
+ if (message.role !== "user" && message.role !== "assistant") continue;
574
+ this.send(this.messageItem(message.role, message.content));
575
+ }
576
+ }
577
+ const pending = this.pending;
530
578
  this.pending = [];
531
- this.requested = [];
579
+ pending.forEach((event) => this.send(event));
580
+ }
581
+ /**
582
+ * Forgets what belonged to the session that is gone: its responses, and the
583
+ * answer in progress, which is lost with it.
584
+ */
585
+ resetSession() {
586
+ this.requested.clear();
532
587
  this.response = void 0;
533
- if (!socket) return;
534
- socket.removeAllListeners();
535
- socket.on("error", () => {
536
- });
537
- if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
538
- socket.close(1e3);
539
- }
588
+ this.ignoredResponses.clear();
589
+ this.audioItemId = void 0;
590
+ this.audioBytes = 0;
591
+ this.outputResampler.reset();
592
+ this.generation++;
540
593
  this.endAnswer();
541
594
  }
542
- reconnect() {
543
- if (this.destroyed) return;
544
- this.retryCount++;
545
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
546
- this.log("Max retries reached, giving up");
547
- this.emit("Failed");
548
- return;
549
- }
550
- this.log("Reconnecting...");
551
- this.reconnectTimeout = setTimeout(() => {
552
- this.reconnectTimeout = void 0;
553
- this.connect();
554
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
555
- }
556
595
  isIgnored(responseId) {
557
596
  return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
558
597
  }
559
598
  handleMessage(message) {
560
- switch (message.type) {
599
+ switch (message?.type) {
561
600
  case "response.created": {
562
- const requested = this.requested.shift() ?? {
563
- generation: this.generation,
564
- outOfBand: false
565
- };
566
601
  const id = message.response.id;
567
- if (requested.generation !== this.generation) {
602
+ const requestId = message.response.metadata?.[REQUEST_METADATA_KEY];
603
+ const requested = requestId ? this.requested.get(requestId) : { generation: this.generation, outOfBand: false };
604
+ if (requestId) this.requested.delete(requestId);
605
+ if (!requested || requested.generation !== this.generation) {
568
606
  this.ignoredResponses.add(id);
569
- this.socket?.send(
570
- JSON.stringify({ type: "response.cancel", response_id: id })
571
- );
607
+ this.send({ type: "response.cancel", response_id: id });
572
608
  break;
573
609
  }
574
610
  this.response = { id, ...requested };
@@ -606,10 +642,18 @@ var OpenaiRealtime = class extends import_server3.Realtime {
606
642
  case "response.done":
607
643
  this.onResponseDone(message.response);
608
644
  break;
609
- case "error":
645
+ case "error": {
610
646
  if (message.error?.code === "response_cancel_not_active") break;
611
647
  this.log("Error:", message.error);
648
+ const requestId = message.error?.event_id;
649
+ const requested = this.requested.get(requestId);
650
+ if (!requested) break;
651
+ this.requested.delete(requestId);
652
+ if (requested.generation === this.generation && !this.response) {
653
+ this.endAnswer();
654
+ }
612
655
  break;
656
+ }
613
657
  default:
614
658
  break;
615
659
  }
@@ -659,69 +703,78 @@ var OpenaiRealtime = class extends import_server3.Realtime {
659
703
 
660
704
  // src/OpenaiSTT.ts
661
705
  var import_server4 = require("@micdrop/server");
662
- var import_ws2 = __toESM(require("ws"));
663
706
  var DEFAULT_MODEL4 = "gpt-4o-transcribe";
664
707
  var DEFAULT_LANGUAGE = "en";
665
708
  var SAMPLE_RATE2 = 16e3;
666
709
  var OPENAI_SAMPLE_RATE2 = 24e3;
667
- var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
668
710
  var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
669
- var DEFAULT_RETRY_DELAY3 = 1e3;
670
- var DEFAULT_MAX_RETRY2 = 3;
671
711
  var OpenaiSTT = class extends import_server4.STT {
672
712
  constructor(options) {
673
713
  super();
674
714
  this.options = options;
675
- this.retryCount = 0;
676
- this.destroyed = false;
677
- this.audioChunksPending = [];
678
- // Store audio chunks to send them again if reconnecting
679
- this.resampler = new import_server4.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
680
- this.initPromise = this.initWS().catch((error) => {
681
- console.error("[OpenaiSTT] Connection error:", error);
682
- this.reconnect();
715
+ this.utteranceCount = 0;
716
+ // Utterances not transcribed yet, in the order they were spoken. Each one
717
+ // leaves the list with the single transcript it emits.
718
+ this.utterances = [];
719
+ // Commits sent and not acknowledged yet, in the order the API handles them
720
+ this.unacknowledged = [];
721
+ this.socket = new import_server4.ReconnectingSocket({
722
+ url: "wss://api.openai.com/v1/realtime?intent=transcription",
723
+ clientOptions: {
724
+ headers: { Authorization: `Bearer ${this.options.apiKey}` }
725
+ },
726
+ connectionTimeout: options.connectionTimeout,
727
+ retryDelay: options.retryDelay,
728
+ maxRetry: options.maxRetry,
729
+ handshake: sessionHandshake,
730
+ log: (...message) => this.log(...message)
731
+ });
732
+ this.socket.on("Open", () => this.onOpen());
733
+ this.socket.on("Close", () => {
734
+ this.unacknowledged = [];
683
735
  });
736
+ this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
737
+ this.socket.on("Failed", () => this.onFailed());
738
+ this.socket.connect();
684
739
  }
740
+ /**
741
+ * Transcribes one utterance, and emits its transcript once, possibly empty
742
+ * when none comes in time.
743
+ *
744
+ * After the connection was given up on, the next utterance tries it again.
745
+ */
685
746
  transcribe(audioStream) {
686
- this.resampler.reset();
687
- audioStream.on("data", async (chunk) => {
688
- this.audioChunksPending.push(chunk);
689
- await this.initPromise;
690
- this.sendAudioChunk(chunk);
691
- this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
747
+ const utterance = {
748
+ id: `micdrop-stt-${++this.utteranceCount}`,
749
+ status: "streaming",
750
+ chunks: [],
751
+ resampler: new import_server4.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2)
752
+ };
753
+ this.utterances.push(utterance);
754
+ this.socket.connect();
755
+ audioStream.on("data", (chunk) => {
756
+ if (!this.utterances.includes(utterance)) return;
757
+ utterance.chunks.push(chunk);
758
+ this.sendAudioChunk(utterance, chunk);
692
759
  });
693
- audioStream.on("end", async () => {
694
- await this.initPromise;
695
- if (this.audioChunksPending.length === 0) return;
696
- this.commitAudio();
697
- this.transcriptionTimeout = setTimeout(() => {
698
- this.transcriptionTimeout = void 0;
699
- this.log("Transcription timeout");
700
- this.emit("Transcript", "");
701
- this.audioChunksPending.length = 0;
702
- }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
760
+ audioStream.on("end", () => {
761
+ if (!this.utterances.includes(utterance)) return;
762
+ if (utterance.chunks.length === 0) {
763
+ this.remove(utterance);
764
+ return;
765
+ }
766
+ utterance.status = "committed";
767
+ this.commitAudio(utterance);
703
768
  });
704
769
  }
705
770
  destroy() {
706
771
  super.destroy();
707
- this.destroyed = true;
708
- if (this.reconnectTimeout) {
709
- clearTimeout(this.reconnectTimeout);
710
- this.reconnectTimeout = void 0;
711
- }
712
- if (this.transcriptionTimeout) {
713
- clearTimeout(this.transcriptionTimeout);
714
- this.transcriptionTimeout = void 0;
715
- }
716
- if (this.connectionTimeout) {
717
- clearTimeout(this.connectionTimeout);
718
- this.connectionTimeout = void 0;
719
- }
720
- if (this.socket) closeSocket(this.socket);
721
- this.socket = void 0;
772
+ this.socket.destroy();
773
+ this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
774
+ this.utterances = [];
775
+ this.unacknowledged = [];
722
776
  }
723
777
  sendSessionUpdate() {
724
- if (!this.socket) return;
725
778
  const model = this.options.model || DEFAULT_MODEL4;
726
779
  this.socket.send(
727
780
  JSON.stringify({
@@ -755,56 +808,35 @@ var OpenaiSTT = class extends import_server4.STT {
755
808
  })
756
809
  );
757
810
  }
758
- async initWS() {
759
- return new Promise((resolve, reject) => {
760
- const socket = new import_ws2.default(
761
- "wss://api.openai.com/v1/realtime?intent=transcription",
762
- {
763
- headers: {
764
- Authorization: `Bearer ${this.options.apiKey}`
765
- }
766
- }
767
- );
768
- this.socket = socket;
769
- const timeout = setTimeout(() => {
770
- this.log("Connection timeout");
771
- closeSocket(socket);
772
- this.socket = void 0;
773
- reject(new Error("WebSocket connection timeout"));
774
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
775
- this.connectionTimeout = timeout;
776
- socket.addEventListener("open", () => {
777
- clearTimeout(timeout);
778
- this.sendSessionUpdate();
779
- this.log("Connection opened");
780
- resolve();
781
- });
782
- socket.addEventListener("error", (error) => {
783
- clearTimeout(timeout);
784
- this.log("WebSocket error:", error);
785
- reject(new Error("WebSocket connection error"));
786
- });
787
- socket.addEventListener(
788
- "close",
789
- ({ code, reason }) => {
790
- clearTimeout(timeout);
791
- this.socket?.removeAllListeners();
792
- this.socket = void 0;
793
- if (code !== 1e3) {
794
- this.reconnect();
795
- } else {
796
- this.log("Connection closed", { code, reason });
797
- }
798
- }
799
- );
800
- socket.addEventListener("message", (event) => {
801
- this.handleMessage(JSON.parse(event.data.toString()));
802
- });
803
- });
811
+ /**
812
+ * Configures the session, then sends again what a previous connection did
813
+ * not get transcribed. The OpenAI buffer of a new session is empty, so each
814
+ * utterance is replayed through a fresh resampler, which reproduces the exact
815
+ * same stream, and committed again if it was.
816
+ */
817
+ onOpen() {
818
+ this.sendSessionUpdate();
819
+ this.unacknowledged = [];
820
+ if (this.utterances.length > 0) this.log("Sending audio chunks again");
821
+ for (const utterance of this.utterances) {
822
+ if (utterance.transcript !== void 0) continue;
823
+ utterance.itemId = void 0;
824
+ utterance.resampler = new import_server4.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
825
+ utterance.chunks.forEach((chunk) => this.sendAudioChunk(utterance, chunk));
826
+ if (utterance.status === "committed") this.commitAudio(utterance);
827
+ }
804
828
  }
805
- sendAudioChunk(chunk) {
806
- if (!this.socket) return;
807
- const resampled = this.resampler.process(chunk);
829
+ /** Every retry failed: the audio waiting for a transcript is handed back */
830
+ onFailed() {
831
+ const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
832
+ this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
833
+ this.utterances = [];
834
+ this.unacknowledged = [];
835
+ this.emit("Failed", chunks);
836
+ }
837
+ sendAudioChunk(utterance, chunk) {
838
+ if (!this.socket.isOpen) return;
839
+ const resampled = utterance.resampler.process(chunk);
808
840
  if (resampled.length === 0) return;
809
841
  this.socket.send(
810
842
  JSON.stringify({
@@ -813,81 +845,100 @@ var OpenaiSTT = class extends import_server4.STT {
813
845
  })
814
846
  );
815
847
  }
848
+ commitAudio(utterance) {
849
+ if (!this.socket.isOpen) return;
850
+ this.socket.send(
851
+ JSON.stringify({
852
+ type: "input_audio_buffer.commit",
853
+ event_id: utterance.id
854
+ })
855
+ );
856
+ this.unacknowledged.push(utterance);
857
+ this.log("Committed audio buffer");
858
+ clearTimeout(utterance.timer);
859
+ utterance.timer = setTimeout(() => {
860
+ utterance.timer = void 0;
861
+ this.log("Transcription timeout");
862
+ this.finish(utterance, "");
863
+ }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
864
+ }
865
+ /**
866
+ * Settles the transcript of an utterance, the only one it gets.
867
+ *
868
+ * The API transcribes each utterance on its own and may finish a short one
869
+ * before a long one spoken earlier, so transcripts go out in the order the
870
+ * utterances were spoken.
871
+ */
872
+ finish(utterance, transcript) {
873
+ if (!this.utterances.includes(utterance)) return;
874
+ if (utterance.transcript !== void 0) return;
875
+ clearTimeout(utterance.timer);
876
+ utterance.timer = void 0;
877
+ utterance.transcript = transcript;
878
+ while (this.utterances[0]?.transcript !== void 0) {
879
+ const first = this.utterances[0];
880
+ this.remove(first);
881
+ this.emit("Transcript", first.transcript);
882
+ }
883
+ }
884
+ remove(utterance) {
885
+ clearTimeout(utterance.timer);
886
+ utterance.timer = void 0;
887
+ this.utterances = this.utterances.filter((other) => other !== utterance);
888
+ this.unacknowledged = this.unacknowledged.filter(
889
+ (other) => other !== utterance
890
+ );
891
+ }
892
+ /**
893
+ * The utterance a transcript event is about. The API names the item, and
894
+ * the item of each utterance is known from the acknowledgement of its
895
+ * commit. A transcript of an utterance that timed out matches none.
896
+ */
897
+ findUtterance(itemId) {
898
+ if (!itemId) {
899
+ return this.utterances.find(
900
+ ({ status, transcript }) => status === "committed" && transcript === void 0
901
+ );
902
+ }
903
+ return this.utterances.find((utterance) => utterance.itemId === itemId);
904
+ }
816
905
  handleMessage(message) {
817
- switch (message.type) {
818
- // case 'input_audio_buffer.committed':
819
- // this.log('Audio buffer committed')
820
- // break
821
- // case 'input_audio_buffer.speech_started':
822
- // this.log('Speech started')
823
- // break
824
- // case 'input_audio_buffer.speech_stopped':
825
- // this.log('Speech stopped')
826
- // break
827
- // case 'conversation.item.input_audio_transcription.delta':
828
- // this.log(`Received transcript delta: "${message.delta}"`)
829
- // break
830
- case "conversation.item.input_audio_transcription.completed":
906
+ switch (message?.type) {
907
+ case "input_audio_buffer.committed": {
908
+ const utterance = this.unacknowledged.shift();
909
+ if (utterance) utterance.itemId = message.item_id;
910
+ break;
911
+ }
912
+ case "conversation.item.input_audio_transcription.completed": {
831
913
  const transcript = message.transcript || "";
832
914
  this.log(`Received completed transcript: "${transcript}"`);
833
- this.emit("Transcript", transcript);
834
- this.audioChunksPending.length = 0;
835
- if (this.transcriptionTimeout) {
836
- clearTimeout(this.transcriptionTimeout);
837
- this.transcriptionTimeout = void 0;
915
+ const utterance = this.findUtterance(message.item_id);
916
+ if (!utterance) {
917
+ this.log("Transcript of an utterance already given up on, ignored");
918
+ break;
838
919
  }
920
+ this.finish(utterance, transcript);
839
921
  break;
840
- case "session.created":
841
- this.retryCount = 0;
922
+ }
923
+ case "conversation.item.input_audio_transcription.failed": {
924
+ this.log("Transcription failed:", message.error);
925
+ const utterance = this.findUtterance(message.item_id);
926
+ if (utterance) this.finish(utterance, "");
842
927
  break;
843
- case "error":
928
+ }
929
+ case "error": {
844
930
  this.log("Error:", message.error);
931
+ const utterance = this.utterances.find(
932
+ ({ id }) => id === message.error?.event_id
933
+ );
934
+ if (utterance) this.finish(utterance, "");
845
935
  break;
936
+ }
846
937
  default:
847
938
  break;
848
939
  }
849
940
  }
850
- reconnect() {
851
- if (this.destroyed) return;
852
- this.retryCount++;
853
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
854
- this.log("Max retries reached, giving up");
855
- this.emit("Failed", this.audioChunksPending);
856
- return;
857
- }
858
- this.initPromise = new Promise((resolve) => {
859
- this.log("Reconnecting...");
860
- this.reconnectTimeout = setTimeout(() => {
861
- this.reconnectTimeout = void 0;
862
- this.initWS().then(() => {
863
- if (this.audioChunksPending.length > 0) {
864
- this.log("Sending audio chunks again");
865
- this.resampler.reset();
866
- this.audioChunksPending.forEach(
867
- (chunk) => this.sendAudioChunk(chunk)
868
- );
869
- }
870
- }).then(resolve).catch((error) => {
871
- this.log("Reconnection error:", error);
872
- this.reconnect();
873
- });
874
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
875
- });
876
- }
877
- commitAudio() {
878
- if (!this.socket) return;
879
- this.socket.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
880
- this.log("Committed audio buffer");
881
- }
882
941
  };
883
- function closeSocket(socket) {
884
- socket.removeAllListeners();
885
- socket.on("error", () => {
886
- });
887
- if (socket.readyState === import_ws2.default.OPEN || socket.readyState === import_ws2.default.CONNECTING) {
888
- socket.close(1e3);
889
- }
890
- }
891
942
 
892
943
  // src/OpenaiTTS.ts
893
944
  var import_server6 = require("@micdrop/server");
@@ -895,9 +946,7 @@ var import_openai3 = __toESM(require("openai"));
895
946
 
896
947
  // src/OpenaiRealtimeSpeech.ts
897
948
  var import_server5 = require("@micdrop/server");
898
- var import_ws3 = __toESM(require("ws"));
899
949
  var DEFAULT_REASONING_EFFORT = "minimal";
900
- var DEFAULT_CONNECTION_TIMEOUT3 = 5e3;
901
950
  var MIN_SPEED = 0.25;
902
951
  var MAX_SPEED = 1.5;
903
952
  var OPENAI_SAMPLE_RATE3 = 24e3;
@@ -910,22 +959,48 @@ var OpenaiRealtimeSpeech = class {
910
959
  constructor(options, log) {
911
960
  this.options = options;
912
961
  this.log = log;
913
- this.destroyed = false;
914
962
  this.itemCount = 0;
915
963
  // Responses asked for and not finished, by their metadata id and by the id
916
- // the API gave them. Cancelled ones leave both maps, which mutes them.
964
+ // the API gave them. Finished ones leave both maps, which mutes them.
917
965
  this.items = /* @__PURE__ */ new Map();
918
966
  this.responses = /* @__PURE__ */ new Map();
919
- this.connect().catch(() => {
967
+ const { model, apiKey } = options;
968
+ this.socket = new import_server5.ReconnectingSocket({
969
+ url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
970
+ clientOptions: { headers: { Authorization: `Bearer ${apiKey}` } },
971
+ connectionTimeout: options.connectionTimeout,
972
+ handshake: sessionHandshake,
973
+ log
974
+ });
975
+ this.socket.on("Open", () => {
976
+ this.socket.send(JSON.stringify(this.sessionUpdate()));
920
977
  });
978
+ this.socket.on("Close", () => {
979
+ for (const item of [...this.items.values()]) {
980
+ if (item.status === "created") {
981
+ this.finish(item, "failed", new Error("Connection lost"));
982
+ } else if (item.status === "requested") {
983
+ item.status = "waiting";
984
+ this.request(item);
985
+ }
986
+ }
987
+ });
988
+ this.socket.on("Message", (data) => {
989
+ try {
990
+ this.handleMessage(parseMessage(data));
991
+ } catch (error) {
992
+ this.log("Error handling message", error);
993
+ }
994
+ });
995
+ this.socket.connect();
921
996
  }
922
997
  synthesize(text) {
923
998
  const item = {
924
999
  id: `micdrop-tts-${++this.itemCount}`,
925
1000
  text,
1001
+ status: "waiting",
926
1002
  resampler: new import_server5.Pcm16Resampler(OPENAI_SAMPLE_RATE3, OUTPUT_SAMPLE_RATE),
927
- audio: [],
928
- done: false
1003
+ audio: []
929
1004
  };
930
1005
  this.items.set(item.id, item);
931
1006
  this.request(item);
@@ -935,13 +1010,28 @@ var OpenaiRealtimeSpeech = class {
935
1010
  };
936
1011
  }
937
1012
  destroy() {
938
- this.destroyed = true;
939
- this.closeSocket();
1013
+ this.socket.destroy();
1014
+ for (const item of [...this.items.values()]) {
1015
+ this.finish(item, "failed", new Error("TTS destroyed"));
1016
+ }
940
1017
  }
941
- request(item) {
942
- this.connect().then(() => {
943
- if (!this.items.has(item.id)) return;
944
- this.send({
1018
+ /**
1019
+ * Sends the sentence once the connection is open.
1020
+ *
1021
+ * A Realtime session lasts an hour at most, and the connection opens again
1022
+ * on its own when the API closes it. After every retry failed, the next
1023
+ * sentence starts over with a full retry budget.
1024
+ */
1025
+ async request(item) {
1026
+ this.socket.connect();
1027
+ const open = await this.socket.ready();
1028
+ if (item.status !== "waiting") return;
1029
+ if (!open) {
1030
+ this.finish(item, "failed", new Error("WebSocket connection failed"));
1031
+ return;
1032
+ }
1033
+ const sent = this.socket.send(
1034
+ JSON.stringify({
945
1035
  type: "response.create",
946
1036
  // Errors caused by this event carry its id
947
1037
  event_id: item.id,
@@ -967,8 +1057,13 @@ var OpenaiRealtimeSpeech = class {
967
1057
  output_modalities: ["audio"],
968
1058
  metadata: { micdrop_id: item.id }
969
1059
  }
970
- });
971
- }).catch((error) => this.finish(item, error));
1060
+ })
1061
+ );
1062
+ if (sent) {
1063
+ item.status = "requested";
1064
+ } else {
1065
+ this.finish(item, "failed", new Error("WebSocket connection lost"));
1066
+ }
972
1067
  }
973
1068
  async *read(item) {
974
1069
  while (true) {
@@ -977,21 +1072,22 @@ var OpenaiRealtimeSpeech = class {
977
1072
  yield chunk;
978
1073
  continue;
979
1074
  }
980
- if (item.error) throw item.error;
981
- if (item.done) return;
1075
+ if (item.status === "failed") throw item.error;
1076
+ if (item.status === "done" || item.status === "cancelled") return;
982
1077
  await new Promise((resolve) => item.wake = resolve);
983
1078
  }
984
1079
  }
985
1080
  cancel(item) {
986
- if (item.responseId && !item.done) {
1081
+ if (item.status === "created" && item.responseId) {
987
1082
  this.send({ type: "response.cancel", response_id: item.responseId });
988
1083
  }
989
1084
  item.audio = [];
990
- this.finish(item);
1085
+ this.finish(item, "cancelled");
991
1086
  }
992
- finish(item, error) {
1087
+ /** Ends a sentence, the first end being the only one that counts */
1088
+ finish(item, status, error) {
993
1089
  if (!this.items.has(item.id)) return;
994
- item.done = true;
1090
+ item.status = status;
995
1091
  item.error = error;
996
1092
  this.items.delete(item.id);
997
1093
  if (item.responseId) this.responses.delete(item.responseId);
@@ -1000,9 +1096,7 @@ var OpenaiRealtimeSpeech = class {
1000
1096
  wake?.();
1001
1097
  }
1002
1098
  send(event) {
1003
- if (this.socket?.readyState === import_ws3.default.OPEN) {
1004
- this.socket.send(JSON.stringify(event));
1005
- }
1099
+ this.socket.send(JSON.stringify(event));
1006
1100
  }
1007
1101
  sessionUpdate() {
1008
1102
  const { instructions, speed } = this.options;
@@ -1029,80 +1123,8 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1029
1123
  }
1030
1124
  };
1031
1125
  }
1032
- /**
1033
- * Opens the session, or hands over the one already open.
1034
- *
1035
- * A Realtime session lasts an hour at most. A closed one is opened again by
1036
- * the next sentence rather than on a timer, so an idle agent holds no
1037
- * connection.
1038
- */
1039
- connect() {
1040
- if (this.destroyed) return Promise.reject(new Error("TTS destroyed"));
1041
- if (this.socket?.readyState === import_ws3.default.OPEN) {
1042
- return Promise.resolve(this.socket);
1043
- }
1044
- if (this.connecting) return this.connecting;
1045
- const { model, apiKey } = this.options;
1046
- const connecting = new Promise((resolve, reject) => {
1047
- const socket = new import_ws3.default(
1048
- `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
1049
- { headers: { Authorization: `Bearer ${apiKey}` } }
1050
- );
1051
- this.socket = socket;
1052
- const timeout = setTimeout(() => {
1053
- this.log("Connection timeout");
1054
- this.closeSocket();
1055
- reject(new Error("WebSocket connection timeout"));
1056
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT3);
1057
- socket.on("open", () => {
1058
- clearTimeout(timeout);
1059
- this.log("Connection opened");
1060
- socket.send(JSON.stringify(this.sessionUpdate()));
1061
- resolve(socket);
1062
- });
1063
- socket.on("error", (error) => {
1064
- clearTimeout(timeout);
1065
- this.log("WebSocket error:", error);
1066
- reject(new Error("WebSocket connection error"));
1067
- });
1068
- socket.on("close", (code, reason) => {
1069
- clearTimeout(timeout);
1070
- this.log("Connection closed", { code, reason: reason.toString() });
1071
- reject(new Error("WebSocket connection closed"));
1072
- this.closeSocket();
1073
- });
1074
- socket.on("message", (data) => {
1075
- try {
1076
- this.handleMessage(JSON.parse(data.toString()));
1077
- } catch (error) {
1078
- this.log("Error handling message", error);
1079
- }
1080
- });
1081
- });
1082
- this.connecting = connecting;
1083
- const settle = () => {
1084
- if (this.connecting === connecting) this.connecting = void 0;
1085
- };
1086
- connecting.then(settle, settle);
1087
- return connecting;
1088
- }
1089
- closeSocket() {
1090
- const socket = this.socket;
1091
- this.socket = void 0;
1092
- this.connecting = void 0;
1093
- for (const item of [...this.items.values()]) {
1094
- this.finish(item, new Error("Connection lost"));
1095
- }
1096
- if (!socket) return;
1097
- socket.removeAllListeners();
1098
- socket.on("error", () => {
1099
- });
1100
- if (socket.readyState === import_ws3.default.OPEN || socket.readyState === import_ws3.default.CONNECTING) {
1101
- socket.close(1e3);
1102
- }
1103
- }
1104
1126
  handleMessage(message) {
1105
- switch (message.type) {
1127
+ switch (message?.type) {
1106
1128
  case "response.created": {
1107
1129
  const id = message.response.id;
1108
1130
  const item = this.items.get(message.response.metadata?.micdrop_id);
@@ -1110,6 +1132,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1110
1132
  this.send({ type: "response.cancel", response_id: id });
1111
1133
  break;
1112
1134
  }
1135
+ item.status = "created";
1113
1136
  item.responseId = id;
1114
1137
  this.responses.set(id, item);
1115
1138
  break;
@@ -1141,12 +1164,13 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1141
1164
  if (response.status === "failed") {
1142
1165
  this.finish(
1143
1166
  item,
1167
+ "failed",
1144
1168
  new Error(
1145
1169
  `Response failed: ${JSON.stringify(response.status_details)}`
1146
1170
  )
1147
1171
  );
1148
1172
  } else {
1149
- this.finish(item);
1173
+ this.finish(item, "done");
1150
1174
  }
1151
1175
  break;
1152
1176
  }
@@ -1154,7 +1178,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1154
1178
  if (message.error?.code === "response_cancel_not_active") break;
1155
1179
  this.log("Error:", message.error);
1156
1180
  const item = this.items.get(message.error?.event_id);
1157
- if (item) this.finish(item, new Error(message.error?.message));
1181
+ if (item) this.finish(item, "failed", new Error(message.error?.message));
1158
1182
  break;
1159
1183
  }
1160
1184
  default:
@@ -1234,15 +1258,17 @@ var OpenaiTTS = class extends import_server6.TTS {
1234
1258
  this.enqueue(counter, sentence);
1235
1259
  }
1236
1260
  });
1237
- textStream.on("error", (error) => {
1238
- this.log("Error in text stream", error);
1239
- });
1240
- textStream.on("end", () => {
1261
+ const flush = () => {
1241
1262
  if (!counter || counter !== this.counter) return;
1242
1263
  for (const sentence of this.splitter.flush()) {
1243
1264
  this.enqueue(counter, sentence);
1244
1265
  }
1266
+ };
1267
+ textStream.on("error", (error) => {
1268
+ this.log("Error in text stream", error);
1269
+ flush();
1245
1270
  });
1271
+ textStream.on("end", flush);
1246
1272
  }
1247
1273
  cancel() {
1248
1274
  this.generation++;