@micdrop/openai 1.7.1 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -42,28 +42,128 @@ __export(index_exports, {
42
42
  module.exports = __toCommonJS(index_exports);
43
43
 
44
44
  // src/OpenaiAgent.ts
45
+ var import_server2 = require("@micdrop/server");
46
+
47
+ // src/models.ts
45
48
  var import_server = require("@micdrop/server");
49
+ function fetchAll(apiKey) {
50
+ return (0, import_server.cachedList)(["openai-raw-models"], apiKey, async () => {
51
+ const response = await fetch("https://api.openai.com/v1/models", {
52
+ headers: { Authorization: `Bearer ${apiKey}` }
53
+ });
54
+ if (!response.ok) throw new Error(`OpenAI answered ${response.status}`);
55
+ const { data } = await response.json();
56
+ return data;
57
+ });
58
+ }
59
+ var SNAPSHOT = /-(\d{4}-\d{2}-\d{2}|\d{4})$/;
60
+ function kindsOf(id) {
61
+ if (SNAPSHOT.test(id)) return [];
62
+ if (id.includes("transcribe")) return id.includes("diarize") ? [] : ["stt"];
63
+ if (id.startsWith("gpt-realtime")) {
64
+ return /translate|whisper/.test(id) ? [] : ["realtime", "tts"];
65
+ }
66
+ if (id.includes("-tts") || /^tts-/.test(id)) return ["tts"];
67
+ if (!/^(gpt-[4-9]|o\d)/.test(id)) return [];
68
+ if (/audio|image|search|live|codex|-pro\b/.test(id)) return [];
69
+ return ["agent"];
70
+ }
71
+ function listOpenaiModels(kind, options) {
72
+ return (0, import_server.cachedModels)(
73
+ ["openai", kind],
74
+ options.apiKey,
75
+ async () => (await fetchAll(options.apiKey)).filter((model) => kindsOf(model.id).includes(kind)).sort((a, b) => a.id.localeCompare(b.id)).map((model) => ({
76
+ id: model.id,
77
+ // GPT-5 and later reason, and the realtime models from gpt-realtime-2
78
+ reasoning: kind === "agent" ? /^(o\d|gpt-([5-9]|\d{2,}))/.test(model.id) : /^gpt-realtime-[2-9]/.test(model.id),
79
+ deprecated: !!model.shutdown_date,
80
+ shutdownDate: model.shutdown_date || void 0
81
+ })),
82
+ options.refresh
83
+ );
84
+ }
85
+
86
+ // src/OpenaiAgent.ts
46
87
  var import_openai = __toESM(require("openai"));
47
88
  var import_zod = __toESM(require("zod"));
48
89
  var DEFAULT_MODEL = "gpt-4o";
49
90
  var DEFAULT_MAX_STEPS = 5;
50
91
  var DEFAULT_MAX_RETRIES = 3;
51
92
  var DEFAULT_RETRY_DELAY = 1e3;
52
- var OpenaiAgent = class extends import_server.Agent {
93
+ var StepInterruptedError = class extends Error {
94
+ constructor(outcome, reason) {
95
+ super("Answer interrupted");
96
+ this.outcome = outcome;
97
+ this.reason = reason;
98
+ }
99
+ };
100
+ var OpenaiAgent = class extends import_server2.Agent {
101
+ /**
102
+ * Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
103
+ * else about them.
104
+ *
105
+ * The list is fetched once and shared by every call of the process for a
106
+ * day. `refresh` fetches it again.
107
+ */
108
+ static listModels(options) {
109
+ return listOpenaiModels("agent", options);
110
+ }
53
111
  constructor(options) {
54
112
  super(options);
55
113
  this.openai = "openai" in options ? options.openai : new import_openai.default({ apiKey: options.apiKey });
56
114
  }
57
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
58
- if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
59
- console.error("[OpenaiAgent] Max steps reached");
60
- return;
115
+ async generateAnswer(stream, signal) {
116
+ const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
117
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
118
+ for (let step = 0; step < maxSteps; step++) {
119
+ let outcome;
120
+ for (let tryCount = 0; !outcome; tryCount++) {
121
+ try {
122
+ outcome = await this.generateStep(stream, signal);
123
+ } catch (error) {
124
+ if (signal.aborted) return;
125
+ if (error instanceof StepInterruptedError) {
126
+ console.error("[OpenaiAgent] Answer interrupted:", error.reason);
127
+ outcome = error.outcome;
128
+ break;
129
+ }
130
+ console.error("[OpenaiAgent] Error answering:", error);
131
+ if (tryCount >= maxRetry) {
132
+ this.log("Max retries reached, giving up");
133
+ this.emit("Failed");
134
+ return;
135
+ }
136
+ await new Promise(
137
+ (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
138
+ );
139
+ if (signal.aborted) return;
140
+ }
141
+ }
142
+ if (signal.aborted) return;
143
+ if (outcome.recorded || outcome.skipAnswer) return;
144
+ if (outcome.text) {
145
+ const { message, metadata } = this.extract(outcome.text);
146
+ this.addAssistantMessage(message, metadata);
147
+ return;
148
+ }
61
149
  }
62
- const abortController = new AbortController();
63
- this.abortController = abortController;
64
- const signal = abortController.signal;
65
- let extracting = false;
66
- const extractOptions = this.getExtractOptions();
150
+ console.error("[OpenaiAgent] Max steps reached");
151
+ }
152
+ /**
153
+ * Runs one request, writing its text to the stream and running its tools.
154
+ *
155
+ * It throws the error of the request when it failed before producing
156
+ * anything, which can then be retried, and a {@link StepInterruptedError}
157
+ * once something was written or done.
158
+ */
159
+ async generateStep(stream, signal) {
160
+ const outcome = {
161
+ text: "",
162
+ recorded: false,
163
+ skipAnswer: false
164
+ };
165
+ let produced = false;
166
+ const write = this.answerWriter(stream);
67
167
  try {
68
168
  const response = await this.openai.responses.create(
69
169
  {
@@ -75,40 +175,27 @@ var OpenaiAgent = class extends import_server.Agent {
75
175
  },
76
176
  { signal }
77
177
  );
78
- let skipAnswer = false;
79
- let hasAnswer = false;
80
178
  for await (const event of response) {
81
- if (abortController !== this.abortController) return;
179
+ if (signal.aborted) return outcome;
82
180
  switch (event.type) {
83
181
  case "response.output_text.delta":
84
- if (hasAnswer) break;
182
+ if (outcome.recorded) break;
85
183
  this.log(`Answer chunk: "${event.delta}"`);
86
- if (extractOptions) {
87
- if (!extracting) {
88
- const startTagIndex = event.delta.indexOf(
89
- extractOptions.startTag
90
- );
91
- if (startTagIndex !== -1) {
92
- extracting = true;
93
- const messagePart = event.delta.slice(0, startTagIndex).trimEnd();
94
- stream.write(messagePart);
95
- break;
96
- }
97
- } else {
98
- break;
99
- }
100
- }
101
- stream.write(event.delta);
184
+ outcome.text += event.delta;
185
+ produced = true;
186
+ write(event.delta);
102
187
  break;
103
- case "response.output_text.done":
104
- if (hasAnswer) break;
188
+ case "response.output_text.done": {
189
+ if (outcome.recorded) break;
105
190
  const { message, metadata } = this.extract(event.text);
106
191
  this.addAssistantMessage(message, metadata);
107
- hasAnswer = true;
192
+ outcome.recorded = true;
108
193
  break;
194
+ }
109
195
  case "response.output_item.done":
110
196
  if (event.item.type === "function_call") {
111
197
  const toolCall = event.item;
198
+ produced = true;
112
199
  const result = await this.executeTool({
113
200
  role: "tool_call",
114
201
  toolCallId: toolCall.call_id,
@@ -116,7 +203,7 @@ var OpenaiAgent = class extends import_server.Agent {
116
203
  parameters: toolCall.arguments
117
204
  });
118
205
  if (result.skipAnswer) {
119
- skipAnswer = true;
206
+ outcome.skipAnswer = true;
120
207
  }
121
208
  }
122
209
  break;
@@ -124,23 +211,12 @@ var OpenaiAgent = class extends import_server.Agent {
124
211
  break;
125
212
  }
126
213
  }
127
- if (abortController !== this.abortController) return;
128
- if (!skipAnswer && !hasAnswer) {
129
- await this.generateAnswer(stream, stepCount + 1);
130
- }
131
- this.abortController = void 0;
214
+ return outcome;
132
215
  } catch (error) {
133
- if (error instanceof import_openai.default.APIUserAbortError) return;
134
- console.error("[OpenaiAgent] Error answering:", error);
135
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
136
- await new Promise(
137
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
138
- );
139
- await this.generateAnswer(stream, stepCount, tryCount + 1);
140
- } else {
141
- this.log("Max retries reached, giving up");
142
- this.emit("Failed");
216
+ if (produced && !signal.aborted) {
217
+ throw new StepInterruptedError(outcome, error);
143
218
  }
219
+ throw error;
144
220
  }
145
221
  }
146
222
  buildInput() {
@@ -158,7 +234,8 @@ var OpenaiAgent = class extends import_server.Agent {
158
234
  type: "function_call",
159
235
  call_id: message.toolCallId,
160
236
  name: message.toolName,
161
- arguments: JSON.stringify(message.parameters)
237
+ // Already the JSON the model wrote
238
+ arguments: message.parameters
162
239
  };
163
240
  case "tool_result":
164
241
  return {
@@ -180,16 +257,10 @@ var OpenaiAgent = class extends import_server.Agent {
180
257
  parameters: (0, import_zod.toJSONSchema)(tool.inputSchema || import_zod.default.object())
181
258
  }));
182
259
  }
183
- cancel() {
184
- if (!this.abortController) return;
185
- this.log("Cancel");
186
- this.abortController.abort();
187
- this.abortController = void 0;
188
- }
189
260
  };
190
261
 
191
262
  // src/OpenaiClassifier.ts
192
- var import_server2 = require("@micdrop/server");
263
+ var import_server3 = require("@micdrop/server");
193
264
  var import_openai2 = __toESM(require("openai"));
194
265
  function predicate(instructions) {
195
266
  return { type: "predicate", instructions };
@@ -202,7 +273,7 @@ function score(instructions, levels) {
202
273
  }
203
274
  var DEFAULT_MODEL2 = "gpt-6-luna";
204
275
  var DEFAULT_TIMEOUT = 3e3;
205
- var OpenaiClassifier = class extends import_server2.Classifier {
276
+ var OpenaiClassifier = class extends import_server3.Classifier {
206
277
  constructor(options) {
207
278
  super();
208
279
  this.options = options;
@@ -289,10 +360,27 @@ function buildQuestion(name, question) {
289
360
  }
290
361
 
291
362
  // src/OpenaiRealtime.ts
292
- var import_server3 = require("@micdrop/server");
293
- var import_ws = __toESM(require("ws"));
363
+ var import_server4 = require("@micdrop/server");
294
364
  var import_zod2 = __toESM(require("zod"));
295
365
 
366
+ // src/session.ts
367
+ var sessionHandshake = {
368
+ read(data) {
369
+ const message = parseMessage(data);
370
+ if (message?.type === "error") {
371
+ throw new Error(message.error?.message || "Session refused");
372
+ }
373
+ return message?.type === "session.created" || message?.type === "transcription_session.created";
374
+ }
375
+ };
376
+ function parseMessage(data) {
377
+ try {
378
+ return JSON.parse(data.toString());
379
+ } catch {
380
+ return void 0;
381
+ }
382
+ }
383
+
296
384
  // src/transcription.ts
297
385
  function transcriptionLanguage(model, language) {
298
386
  if (!language) return {};
@@ -309,36 +397,65 @@ var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
309
397
  var SAMPLE_RATE = 16e3;
310
398
  var OPENAI_SAMPLE_RATE = 24e3;
311
399
  var OPENAI_BYTES_PER_MS = OPENAI_SAMPLE_RATE * 2 / 1e3;
312
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
313
- var DEFAULT_RETRY_DELAY2 = 1e3;
314
- var DEFAULT_MAX_RETRY = 3;
315
- var OpenaiRealtime = class extends import_server3.Realtime {
400
+ var MAX_PENDING_EVENTS = 5e3;
401
+ var REQUEST_METADATA_KEY = "micdrop_request";
402
+ var OpenaiRealtime = class extends import_server4.Realtime {
316
403
  constructor(options) {
317
404
  super(options);
318
- this.connected = false;
319
- this.everConnected = false;
320
- this.destroyed = false;
405
+ this.connection = "connecting";
321
406
  // Events sent while the connection opens
322
407
  this.pending = [];
323
- this.retryCount = 0;
324
- this.inputResampler = new import_server3.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
325
- this.outputResampler = new import_server3.Pcm16Resampler(OPENAI_SAMPLE_RATE, SAMPLE_RATE);
408
+ this.requestCount = 0;
409
+ this.inputResampler = new import_server4.Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE);
326
410
  // Bumped by every interruption, so late responses can tell they are stale
327
411
  this.generation = 0;
328
- this.requested = [];
412
+ // Responses asked for and not created yet, by the id sent as their metadata
413
+ // and as the event id of their request, which an error names
414
+ this.requested = /* @__PURE__ */ new Map();
329
415
  this.ignoredResponses = /* @__PURE__ */ new Set();
330
416
  this.audioStartedAt = 0;
331
417
  this.audioBytes = 0;
332
- this.connect();
418
+ const model = options.model || DEFAULT_MODEL3;
419
+ this.socket = new import_server4.ReconnectingSocket({
420
+ url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
421
+ clientOptions: {
422
+ headers: { Authorization: `Bearer ${options.apiKey}` }
423
+ },
424
+ connectionTimeout: options.connectionTimeout,
425
+ retryDelay: options.retryDelay,
426
+ maxRetry: options.maxRetry,
427
+ handshake: sessionHandshake,
428
+ log: (...message) => this.log(...message)
429
+ });
430
+ this.socket.on("Open", ({ reconnected }) => this.onOpen(reconnected));
431
+ this.socket.on("Close", () => {
432
+ this.connection = "reconnecting";
433
+ this.resetSession();
434
+ });
435
+ this.socket.on("Failed", () => {
436
+ this.connection = "closed";
437
+ this.pending = [];
438
+ this.resetSession();
439
+ this.emit("Failed");
440
+ });
441
+ this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
442
+ this.socket.connect();
443
+ }
444
+ /**
445
+ * Lists the Realtime models, sorted from their names since OpenAI says nothing
446
+ * else about them.
447
+ *
448
+ * The list is fetched once and shared by every call of the process for a
449
+ * day. `refresh` fetches it again.
450
+ */
451
+ static listModels(options) {
452
+ return listOpenaiModels("realtime", options);
333
453
  }
334
454
  destroy() {
335
- this.destroyed = true;
336
455
  super.destroy();
337
- if (this.reconnectTimeout) {
338
- clearTimeout(this.reconnectTimeout);
339
- this.reconnectTimeout = void 0;
340
- }
341
- this.closeSocket();
456
+ this.connection = "closed";
457
+ this.pending = [];
458
+ this.socket.destroy();
342
459
  }
343
460
  openTurn() {
344
461
  this.inputResampler.reset();
@@ -367,9 +484,11 @@ var OpenaiRealtime = class extends import_server3.Realtime {
367
484
  instructions: `Say exactly the following text, word for word, and nothing else: ${text}`
368
485
  });
369
486
  }
487
+ get sampleRate() {
488
+ return OPENAI_SAMPLE_RATE;
489
+ }
370
490
  cancelAnswer() {
371
491
  this.generation++;
372
- this.outputResampler.reset();
373
492
  const response = this.response;
374
493
  if (!response) return;
375
494
  this.response = void 0;
@@ -390,25 +509,49 @@ var OpenaiRealtime = class extends import_server3.Realtime {
390
509
  this.audioItemId = void 0;
391
510
  }
392
511
  updateTools() {
393
- if (this.connected) this.sendSessionUpdate();
512
+ if (this.connection === "connected") this.sendSessionUpdate();
394
513
  }
395
514
  injectMessage(role, text) {
396
- if (!this.connected && this.everConnected) return;
515
+ if (this.connection === "reconnecting") return;
397
516
  this.send(this.messageItem(role, text));
398
517
  }
518
+ /**
519
+ * Asks for a response, tagged so its `response.created` tells which request
520
+ * it answers. Matching them by order alone would shift every later response
521
+ * onto the wrong request as soon as one is refused.
522
+ */
399
523
  createResponse(response) {
400
- this.requested.push({
401
- generation: this.generation,
402
- outOfBand: !!response
524
+ const id = `micdrop-response-${++this.requestCount}`;
525
+ const generation = this.generation;
526
+ const accepted = this.send({
527
+ type: "response.create",
528
+ // Errors caused by this event carry its id
529
+ event_id: id,
530
+ response: { ...response, metadata: { [REQUEST_METADATA_KEY]: id } }
403
531
  });
404
- this.send({ type: "response.create", ...response ? { response } : {} });
532
+ if (!accepted) {
533
+ queueMicrotask(() => {
534
+ if (generation === this.generation) this.endAnswer();
535
+ });
536
+ return;
537
+ }
538
+ this.requested.set(id, { generation, outOfBand: !!response });
405
539
  }
540
+ /**
541
+ * Sends an event, or holds it until the connection opens.
542
+ * @returns False when it was dropped, the connection being given up on
543
+ */
406
544
  send(event) {
407
- if (this.connected && this.socket?.readyState === import_ws.default.OPEN) {
408
- this.socket.send(JSON.stringify(event));
409
- } else {
410
- this.pending.push(event);
545
+ if (this.connection === "connected") {
546
+ return this.socket.send(JSON.stringify(event));
411
547
  }
548
+ if (this.connection === "closed") return false;
549
+ if (this.pending.length >= MAX_PENDING_EVENTS) {
550
+ this.log("Too many events waiting for the connection, dropped");
551
+ return false;
552
+ }
553
+ this.pending.push(event);
554
+ return true;
412
555
  }
413
556
  messageItem(role, text) {
414
557
  return {
@@ -427,13 +570,14 @@ var OpenaiRealtime = class extends import_server3.Realtime {
427
570
  }
428
571
  sendSessionUpdate() {
429
572
  const transcriptionModel = this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
430
- this.socket?.send(
573
+ this.socket.send(
431
574
  JSON.stringify({
432
575
  type: "session.update",
433
576
  session: {
434
577
  type: "realtime",
435
578
  model: this.options.model || DEFAULT_MODEL3,
436
579
  instructions: this.options.systemPrompt,
580
+ ...this.options.reasoningEffort ? { reasoning: { effort: this.options.reasoningEffort } } : {},
437
581
  output_modalities: ["audio"],
438
582
  audio: {
439
583
  input: {
@@ -464,117 +608,54 @@ var OpenaiRealtime = class extends import_server3.Realtime {
464
608
  })
465
609
  );
466
610
  }
467
- connect() {
468
- this.initWS().catch((error) => {
469
- this.log("Connection error:", error);
470
- this.reconnect();
471
- });
472
- }
473
- initWS() {
474
- return new Promise((resolve, reject) => {
475
- const model = this.options.model || DEFAULT_MODEL3;
476
- const socket = new import_ws.default(
477
- `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
478
- { headers: { Authorization: `Bearer ${this.options.apiKey}` } }
479
- );
480
- this.socket = socket;
481
- const timeout = setTimeout(() => {
482
- this.log("Connection timeout");
483
- this.closeSocket();
484
- reject(new Error("WebSocket connection timeout"));
485
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
486
- socket.on("open", () => {
487
- clearTimeout(timeout);
488
- this.log("Connection opened");
489
- this.sendSessionUpdate();
490
- if (this.everConnected) {
491
- for (const message of this.conversation) {
492
- if (message.role !== "user" && message.role !== "assistant") {
493
- continue;
494
- }
495
- socket.send(
496
- JSON.stringify(this.messageItem(message.role, message.content))
497
- );
498
- }
499
- }
500
- this.connected = true;
501
- this.everConnected = true;
502
- this.retryCount = 0;
503
- const pending = this.pending;
504
- this.pending = [];
505
- pending.forEach((event) => this.send(event));
506
- resolve();
507
- });
508
- socket.on("error", (error) => {
509
- clearTimeout(timeout);
510
- this.log("WebSocket error:", error);
511
- reject(new Error("WebSocket connection error"));
512
- });
513
- socket.on("close", (code, reason) => {
514
- clearTimeout(timeout);
515
- const wasConnected = this.connected;
516
- this.closeSocket();
517
- if (this.destroyed) return;
518
- this.log("Connection closed", { code, reason: reason.toString() });
519
- if (wasConnected) this.reconnect();
520
- });
521
- socket.on("message", (data) => {
522
- this.handleMessage(JSON.parse(data.toString()));
523
- });
524
- });
525
- }
526
- closeSocket() {
527
- const socket = this.socket;
528
- this.socket = void 0;
529
- this.connected = false;
611
+ /**
612
+ * Sets the session up, then sends what waited for it. A new connection
613
+ * starts from an empty conversation, so a reconnection sends it again first.
614
+ */
615
+ onOpen(reconnected) {
616
+ this.connection = "connected";
617
+ this.sendSessionUpdate();
618
+ if (reconnected) {
619
+ for (const message of this.conversation) {
620
+ if (message.role !== "user" && message.role !== "assistant") continue;
621
+ this.send(this.messageItem(message.role, message.content));
622
+ }
623
+ }
624
+ const pending = this.pending;
530
625
  this.pending = [];
531
- this.requested = [];
626
+ pending.forEach((event) => this.send(event));
627
+ }
628
+ /**
629
+ * Forgets what belonged to the session that is gone: its responses, and the
630
+ * answer in progress, which is lost with it.
631
+ */
632
+ resetSession() {
633
+ this.requested.clear();
532
634
  this.response = void 0;
533
- if (!socket) return;
534
- socket.removeAllListeners();
535
- socket.on("error", () => {
536
- });
537
- if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
538
- socket.close(1e3);
539
- }
635
+ this.ignoredResponses.clear();
636
+ this.audioItemId = void 0;
637
+ this.audioBytes = 0;
638
+ this.generation++;
540
639
  this.endAnswer();
541
640
  }
542
- reconnect() {
543
- if (this.destroyed) return;
544
- this.retryCount++;
545
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
546
- this.log("Max retries reached, giving up");
547
- this.emit("Failed");
548
- return;
549
- }
550
- this.log("Reconnecting...");
551
- this.reconnectTimeout = setTimeout(() => {
552
- this.reconnectTimeout = void 0;
553
- this.connect();
554
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
555
- }
556
641
  isIgnored(responseId) {
557
642
  return !responseId || this.ignoredResponses.has(responseId) || this.response?.id !== responseId;
558
643
  }
559
644
  handleMessage(message) {
560
- switch (message.type) {
645
+ switch (message?.type) {
561
646
  case "response.created": {
562
- const requested = this.requested.shift() ?? {
563
- generation: this.generation,
564
- outOfBand: false
565
- };
566
647
  const id = message.response.id;
567
- if (requested.generation !== this.generation) {
648
+ const requestId = message.response.metadata?.[REQUEST_METADATA_KEY];
649
+ const requested = requestId ? this.requested.get(requestId) : { generation: this.generation, outOfBand: false };
650
+ if (requestId) this.requested.delete(requestId);
651
+ if (!requested || requested.generation !== this.generation) {
568
652
  this.ignoredResponses.add(id);
569
- this.socket?.send(
570
- JSON.stringify({ type: "response.cancel", response_id: id })
571
- );
653
+ this.send({ type: "response.cancel", response_id: id });
572
654
  break;
573
655
  }
574
656
  this.response = { id, ...requested };
575
657
  this.audioItemId = void 0;
576
658
  this.audioBytes = 0;
577
- this.outputResampler.reset();
578
659
  break;
579
660
  }
580
661
  case "response.output_audio.delta": {
@@ -586,8 +667,7 @@ var OpenaiRealtime = class extends import_server3.Realtime {
586
667
  this.audioBytes = 0;
587
668
  }
588
669
  this.audioBytes += audio.length;
589
- const output = this.outputResampler.process(audio);
590
- if (output.length > 0) this.emitAudio(output);
670
+ if (audio.length > 0) this.emitAudio(audio);
591
671
  break;
592
672
  }
593
673
  case "response.output_audio_transcript.delta":
@@ -606,10 +686,18 @@ var OpenaiRealtime = class extends import_server3.Realtime {
606
686
  case "response.done":
607
687
  this.onResponseDone(message.response);
608
688
  break;
609
- case "error":
689
+ case "error": {
610
690
  if (message.error?.code === "response_cancel_not_active") break;
611
691
  this.log("Error:", message.error);
692
+ const requestId = message.error?.event_id;
693
+ const requested = this.requested.get(requestId);
694
+ if (!requested) break;
695
+ this.requested.delete(requestId);
696
+ if (requested.generation === this.generation && !this.response) {
697
+ this.endAnswer();
698
+ }
612
699
  break;
700
+ }
613
701
  default:
614
702
  break;
615
703
  }
@@ -658,70 +746,89 @@ var OpenaiRealtime = class extends import_server3.Realtime {
658
746
  };
659
747
 
660
748
  // src/OpenaiSTT.ts
661
- var import_server4 = require("@micdrop/server");
662
- var import_ws2 = __toESM(require("ws"));
749
+ var import_server5 = require("@micdrop/server");
663
750
  var DEFAULT_MODEL4 = "gpt-4o-transcribe";
664
751
  var DEFAULT_LANGUAGE = "en";
665
752
  var SAMPLE_RATE2 = 16e3;
666
753
  var OPENAI_SAMPLE_RATE2 = 24e3;
667
- var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
668
754
  var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
669
- var DEFAULT_RETRY_DELAY3 = 1e3;
670
- var DEFAULT_MAX_RETRY2 = 3;
671
- var OpenaiSTT = class extends import_server4.STT {
755
+ var OpenaiSTT = class extends import_server5.STT {
672
756
  constructor(options) {
673
757
  super();
674
758
  this.options = options;
675
- this.retryCount = 0;
676
- this.destroyed = false;
677
- this.audioChunksPending = [];
678
- // Store audio chunks to send them again if reconnecting
679
- this.resampler = new import_server4.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
680
- this.initPromise = this.initWS().catch((error) => {
681
- console.error("[OpenaiSTT] Connection error:", error);
682
- this.reconnect();
759
+ this.utteranceCount = 0;
760
+ // Utterances not transcribed yet, in the order they were spoken. Each one
761
+ // leaves the list with the single transcript it emits.
762
+ this.utterances = [];
763
+ // Commits sent and not acknowledged yet, in the order the API handles them
764
+ this.unacknowledged = [];
765
+ this.socket = new import_server5.ReconnectingSocket({
766
+ url: "wss://api.openai.com/v1/realtime?intent=transcription",
767
+ clientOptions: {
768
+ headers: { Authorization: `Bearer ${this.options.apiKey}` }
769
+ },
770
+ connectionTimeout: options.connectionTimeout,
771
+ retryDelay: options.retryDelay,
772
+ maxRetry: options.maxRetry,
773
+ handshake: sessionHandshake,
774
+ log: (...message) => this.log(...message)
775
+ });
776
+ this.socket.on("Open", () => this.onOpen());
777
+ this.socket.on("Close", () => {
778
+ this.unacknowledged = [];
683
779
  });
780
+ this.socket.on("Message", (data) => this.handleMessage(parseMessage(data)));
781
+ this.socket.on("Failed", () => this.onFailed());
782
+ this.socket.connect();
783
+ }
784
+ /**
785
+ * Lists the transcription models, sorted from their names since OpenAI says nothing
786
+ * else about them.
787
+ *
788
+ * The list is fetched once and shared by every call of the process for a
789
+ * day. `refresh` fetches it again.
790
+ */
791
+ static listModels(options) {
792
+ return listOpenaiModels("stt", options);
684
793
  }
794
+ /**
795
+ * Transcribes one utterance, and emits its transcript once, possibly empty
796
+ * when none comes in time.
797
+ *
798
+ * After the connection was given up on, the next utterance tries it again.
799
+ */
685
800
  transcribe(audioStream) {
686
- this.resampler.reset();
687
- audioStream.on("data", async (chunk) => {
688
- this.audioChunksPending.push(chunk);
689
- await this.initPromise;
690
- this.sendAudioChunk(chunk);
691
- this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
801
+ const utterance = {
802
+ id: `micdrop-stt-${++this.utteranceCount}`,
803
+ status: "streaming",
804
+ chunks: [],
805
+ resampler: new import_server5.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2)
806
+ };
807
+ this.utterances.push(utterance);
808
+ this.socket.connect();
809
+ audioStream.on("data", (chunk) => {
810
+ if (!this.utterances.includes(utterance)) return;
811
+ utterance.chunks.push(chunk);
812
+ this.sendAudioChunk(utterance, chunk);
692
813
  });
693
- audioStream.on("end", async () => {
694
- await this.initPromise;
695
- if (this.audioChunksPending.length === 0) return;
696
- this.commitAudio();
697
- this.transcriptionTimeout = setTimeout(() => {
698
- this.transcriptionTimeout = void 0;
699
- this.log("Transcription timeout");
700
- this.emit("Transcript", "");
701
- this.audioChunksPending.length = 0;
702
- }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
814
+ audioStream.on("end", () => {
815
+ if (!this.utterances.includes(utterance)) return;
816
+ if (utterance.chunks.length === 0) {
817
+ this.remove(utterance);
818
+ return;
819
+ }
820
+ utterance.status = "committed";
821
+ this.commitAudio(utterance);
703
822
  });
704
823
  }
705
824
  destroy() {
706
825
  super.destroy();
707
- this.destroyed = true;
708
- if (this.reconnectTimeout) {
709
- clearTimeout(this.reconnectTimeout);
710
- this.reconnectTimeout = void 0;
711
- }
712
- if (this.transcriptionTimeout) {
713
- clearTimeout(this.transcriptionTimeout);
714
- this.transcriptionTimeout = void 0;
715
- }
716
- if (this.connectionTimeout) {
717
- clearTimeout(this.connectionTimeout);
718
- this.connectionTimeout = void 0;
719
- }
720
- if (this.socket) closeSocket(this.socket);
721
- this.socket = void 0;
826
+ this.socket.destroy();
827
+ this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
828
+ this.utterances = [];
829
+ this.unacknowledged = [];
722
830
  }
723
831
  sendSessionUpdate() {
724
- if (!this.socket) return;
725
832
  const model = this.options.model || DEFAULT_MODEL4;
726
833
  this.socket.send(
727
834
  JSON.stringify({
@@ -755,56 +862,35 @@ var OpenaiSTT = class extends import_server4.STT {
755
862
  })
756
863
  );
757
864
  }
758
- async initWS() {
759
- return new Promise((resolve, reject) => {
760
- const socket = new import_ws2.default(
761
- "wss://api.openai.com/v1/realtime?intent=transcription",
762
- {
763
- headers: {
764
- Authorization: `Bearer ${this.options.apiKey}`
765
- }
766
- }
767
- );
768
- this.socket = socket;
769
- const timeout = setTimeout(() => {
770
- this.log("Connection timeout");
771
- closeSocket(socket);
772
- this.socket = void 0;
773
- reject(new Error("WebSocket connection timeout"));
774
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
775
- this.connectionTimeout = timeout;
776
- socket.addEventListener("open", () => {
777
- clearTimeout(timeout);
778
- this.sendSessionUpdate();
779
- this.log("Connection opened");
780
- resolve();
781
- });
782
- socket.addEventListener("error", (error) => {
783
- clearTimeout(timeout);
784
- this.log("WebSocket error:", error);
785
- reject(new Error("WebSocket connection error"));
786
- });
787
- socket.addEventListener(
788
- "close",
789
- ({ code, reason }) => {
790
- clearTimeout(timeout);
791
- this.socket?.removeAllListeners();
792
- this.socket = void 0;
793
- if (code !== 1e3) {
794
- this.reconnect();
795
- } else {
796
- this.log("Connection closed", { code, reason });
797
- }
798
- }
799
- );
800
- socket.addEventListener("message", (event) => {
801
- this.handleMessage(JSON.parse(event.data.toString()));
802
- });
803
- });
865
+ /**
866
+ * Configures the session, then sends again what a previous connection did
867
+ * not get transcribed. The OpenAI buffer of a new session is empty, so each
868
+ * utterance is replayed through a fresh resampler, which reproduces the exact
869
+ * same stream, and committed again if it was.
870
+ */
871
+ onOpen() {
872
+ this.sendSessionUpdate();
873
+ this.unacknowledged = [];
874
+ if (this.utterances.length > 0) this.log("Sending audio chunks again");
875
+ for (const utterance of this.utterances) {
876
+ if (utterance.transcript !== void 0) continue;
877
+ utterance.itemId = void 0;
878
+ utterance.resampler = new import_server5.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
879
+ utterance.chunks.forEach((chunk) => this.sendAudioChunk(utterance, chunk));
880
+ if (utterance.status === "committed") this.commitAudio(utterance);
881
+ }
804
882
  }
805
- sendAudioChunk(chunk) {
806
- if (!this.socket) return;
807
- const resampled = this.resampler.process(chunk);
883
+ /** Every retry failed: the audio waiting for a transcript is handed back */
884
+ onFailed() {
885
+ const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
886
+ this.utterances.forEach((utterance) => clearTimeout(utterance.timer));
887
+ this.utterances = [];
888
+ this.unacknowledged = [];
889
+ this.emit("Failed", chunks);
890
+ }
891
+ sendAudioChunk(utterance, chunk) {
892
+ if (!this.socket.isOpen) return;
893
+ const resampled = utterance.resampler.process(chunk);
808
894
  if (resampled.length === 0) return;
809
895
  this.socket.send(
810
896
  JSON.stringify({
@@ -813,95 +899,111 @@ var OpenaiSTT = class extends import_server4.STT {
813
899
  })
814
900
  );
815
901
  }
902
+ commitAudio(utterance) {
903
+ if (!this.socket.isOpen) return;
904
+ this.socket.send(
905
+ JSON.stringify({
906
+ type: "input_audio_buffer.commit",
907
+ event_id: utterance.id
908
+ })
909
+ );
910
+ this.unacknowledged.push(utterance);
911
+ this.log("Committed audio buffer");
912
+ clearTimeout(utterance.timer);
913
+ utterance.timer = setTimeout(() => {
914
+ utterance.timer = void 0;
915
+ this.log("Transcription timeout");
916
+ this.finish(utterance, "");
917
+ }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
918
+ }
919
+ /**
920
+ * Settles the transcript of an utterance, the only one it gets.
921
+ *
922
+ * The API transcribes each utterance on its own and may finish a short one
923
+ * before a long one spoken earlier, so transcripts go out in the order the
924
+ * utterances were spoken.
925
+ */
926
+ finish(utterance, transcript) {
927
+ if (!this.utterances.includes(utterance)) return;
928
+ if (utterance.transcript !== void 0) return;
929
+ clearTimeout(utterance.timer);
930
+ utterance.timer = void 0;
931
+ utterance.transcript = transcript;
932
+ while (this.utterances[0]?.transcript !== void 0) {
933
+ const first = this.utterances[0];
934
+ this.remove(first);
935
+ this.emit("Transcript", first.transcript);
936
+ }
937
+ }
938
+ remove(utterance) {
939
+ clearTimeout(utterance.timer);
940
+ utterance.timer = void 0;
941
+ this.utterances = this.utterances.filter((other) => other !== utterance);
942
+ this.unacknowledged = this.unacknowledged.filter(
943
+ (other) => other !== utterance
944
+ );
945
+ }
946
+ /**
947
+ * The utterance a transcript event is about. The API names the item, and
948
+ * the item of each utterance is known from the acknowledgement of its
949
+ * commit. A transcript of an utterance that timed out matches none.
950
+ */
951
+ findUtterance(itemId) {
952
+ if (!itemId) {
953
+ return this.utterances.find(
954
+ ({ status, transcript }) => status === "committed" && transcript === void 0
955
+ );
956
+ }
957
+ return this.utterances.find((utterance) => utterance.itemId === itemId);
958
+ }
816
959
  handleMessage(message) {
817
- switch (message.type) {
818
- // case 'input_audio_buffer.committed':
819
- // this.log('Audio buffer committed')
820
- // break
821
- // case 'input_audio_buffer.speech_started':
822
- // this.log('Speech started')
823
- // break
824
- // case 'input_audio_buffer.speech_stopped':
825
- // this.log('Speech stopped')
826
- // break
827
- // case 'conversation.item.input_audio_transcription.delta':
828
- // this.log(`Received transcript delta: "${message.delta}"`)
829
- // break
830
- case "conversation.item.input_audio_transcription.completed":
960
+ switch (message?.type) {
961
+ case "input_audio_buffer.committed": {
962
+ const utterance = this.unacknowledged.shift();
963
+ if (utterance) utterance.itemId = message.item_id;
964
+ break;
965
+ }
966
+ case "conversation.item.input_audio_transcription.completed": {
831
967
  const transcript = message.transcript || "";
832
968
  this.log(`Received completed transcript: "${transcript}"`);
833
- this.emit("Transcript", transcript);
834
- this.audioChunksPending.length = 0;
835
- if (this.transcriptionTimeout) {
836
- clearTimeout(this.transcriptionTimeout);
837
- this.transcriptionTimeout = void 0;
969
+ const utterance = this.findUtterance(message.item_id);
970
+ if (!utterance) {
971
+ this.log("Transcript of an utterance already given up on, ignored");
972
+ break;
838
973
  }
974
+ this.finish(utterance, transcript);
839
975
  break;
840
- case "session.created":
841
- this.retryCount = 0;
976
+ }
977
+ case "conversation.item.input_audio_transcription.failed": {
978
+ this.log("Transcription failed:", message.error);
979
+ const utterance = this.findUtterance(message.item_id);
980
+ if (utterance) this.finish(utterance, "");
842
981
  break;
843
- case "error":
982
+ }
983
+ case "error": {
844
984
  this.log("Error:", message.error);
985
+ const utterance = this.utterances.find(
986
+ ({ id }) => id === message.error?.event_id
987
+ );
988
+ if (utterance) this.finish(utterance, "");
845
989
  break;
990
+ }
846
991
  default:
847
992
  break;
848
993
  }
849
994
  }
850
- reconnect() {
851
- if (this.destroyed) return;
852
- this.retryCount++;
853
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
854
- this.log("Max retries reached, giving up");
855
- this.emit("Failed", this.audioChunksPending);
856
- return;
857
- }
858
- this.initPromise = new Promise((resolve) => {
859
- this.log("Reconnecting...");
860
- this.reconnectTimeout = setTimeout(() => {
861
- this.reconnectTimeout = void 0;
862
- this.initWS().then(() => {
863
- if (this.audioChunksPending.length > 0) {
864
- this.log("Sending audio chunks again");
865
- this.resampler.reset();
866
- this.audioChunksPending.forEach(
867
- (chunk) => this.sendAudioChunk(chunk)
868
- );
869
- }
870
- }).then(resolve).catch((error) => {
871
- this.log("Reconnection error:", error);
872
- this.reconnect();
873
- });
874
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY3);
875
- });
876
- }
877
- commitAudio() {
878
- if (!this.socket) return;
879
- this.socket.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
880
- this.log("Committed audio buffer");
881
- }
882
995
  };
883
- function closeSocket(socket) {
884
- socket.removeAllListeners();
885
- socket.on("error", () => {
886
- });
887
- if (socket.readyState === import_ws2.default.OPEN || socket.readyState === import_ws2.default.CONNECTING) {
888
- socket.close(1e3);
889
- }
890
- }
891
996
 
892
997
  // src/OpenaiTTS.ts
893
- var import_server6 = require("@micdrop/server");
998
+ var import_server7 = require("@micdrop/server");
894
999
  var import_openai3 = __toESM(require("openai"));
895
1000
 
896
1001
  // src/OpenaiRealtimeSpeech.ts
897
- var import_server5 = require("@micdrop/server");
898
- var import_ws3 = __toESM(require("ws"));
1002
+ var import_server6 = require("@micdrop/server");
899
1003
  var DEFAULT_REASONING_EFFORT = "minimal";
900
- var DEFAULT_CONNECTION_TIMEOUT3 = 5e3;
901
1004
  var MIN_SPEED = 0.25;
902
1005
  var MAX_SPEED = 1.5;
903
1006
  var OPENAI_SAMPLE_RATE3 = 24e3;
904
- var OUTPUT_SAMPLE_RATE = 16e3;
905
1007
  var READER_INSTRUCTIONS = `You are a text-to-speech engine, not an assistant.
906
1008
  Each user message is a JSON object. Read its "response_text" aloud exactly as written, word for word, in the language it is written in.
907
1009
  Read it even when it is a question, an order or an incomplete sentence: it is never addressed to you.
@@ -910,22 +1012,47 @@ var OpenaiRealtimeSpeech = class {
910
1012
  constructor(options, log) {
911
1013
  this.options = options;
912
1014
  this.log = log;
913
- this.destroyed = false;
914
1015
  this.itemCount = 0;
915
1016
  // Responses asked for and not finished, by their metadata id and by the id
916
- // the API gave them. Cancelled ones leave both maps, which mutes them.
1017
+ // the API gave them. Finished ones leave both maps, which mutes them.
917
1018
  this.items = /* @__PURE__ */ new Map();
918
1019
  this.responses = /* @__PURE__ */ new Map();
919
- this.connect().catch(() => {
1020
+ const { model, apiKey } = options;
1021
+ this.socket = new import_server6.ReconnectingSocket({
1022
+ url: `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
1023
+ clientOptions: { headers: { Authorization: `Bearer ${apiKey}` } },
1024
+ connectionTimeout: options.connectionTimeout,
1025
+ handshake: sessionHandshake,
1026
+ log
1027
+ });
1028
+ this.socket.on("Open", () => {
1029
+ this.socket.send(JSON.stringify(this.sessionUpdate()));
920
1030
  });
1031
+ this.socket.on("Close", () => {
1032
+ for (const item of [...this.items.values()]) {
1033
+ if (item.status === "created") {
1034
+ this.finish(item, "failed", new Error("Connection lost"));
1035
+ } else if (item.status === "requested") {
1036
+ item.status = "waiting";
1037
+ this.request(item);
1038
+ }
1039
+ }
1040
+ });
1041
+ this.socket.on("Message", (data) => {
1042
+ try {
1043
+ this.handleMessage(parseMessage(data));
1044
+ } catch (error) {
1045
+ this.log("Error handling message", error);
1046
+ }
1047
+ });
1048
+ this.socket.connect();
921
1049
  }
922
1050
  synthesize(text) {
923
1051
  const item = {
924
1052
  id: `micdrop-tts-${++this.itemCount}`,
925
1053
  text,
926
- resampler: new import_server5.Pcm16Resampler(OPENAI_SAMPLE_RATE3, OUTPUT_SAMPLE_RATE),
927
- audio: [],
928
- done: false
1054
+ status: "waiting",
1055
+ audio: []
929
1056
  };
930
1057
  this.items.set(item.id, item);
931
1058
  this.request(item);
@@ -935,13 +1062,28 @@ var OpenaiRealtimeSpeech = class {
935
1062
  };
936
1063
  }
937
1064
  destroy() {
938
- this.destroyed = true;
939
- this.closeSocket();
1065
+ this.socket.destroy();
1066
+ for (const item of [...this.items.values()]) {
1067
+ this.finish(item, "failed", new Error("TTS destroyed"));
1068
+ }
940
1069
  }
941
- request(item) {
942
- this.connect().then(() => {
943
- if (!this.items.has(item.id)) return;
944
- this.send({
1070
+ /**
1071
+ * Sends the sentence once the connection is open.
1072
+ *
1073
+ * A Realtime session lasts an hour at most, and the connection opens again
1074
+ * on its own when the API closes it. After every retry failed, the next
1075
+ * sentence starts over with a full retry budget.
1076
+ */
1077
+ async request(item) {
1078
+ this.socket.connect();
1079
+ const open = await this.socket.ready();
1080
+ if (item.status !== "waiting") return;
1081
+ if (!open) {
1082
+ this.finish(item, "failed", new Error("WebSocket connection failed"));
1083
+ return;
1084
+ }
1085
+ const sent = this.socket.send(
1086
+ JSON.stringify({
945
1087
  type: "response.create",
946
1088
  // Errors caused by this event carry its id
947
1089
  event_id: item.id,
@@ -967,8 +1109,13 @@ var OpenaiRealtimeSpeech = class {
967
1109
  output_modalities: ["audio"],
968
1110
  metadata: { micdrop_id: item.id }
969
1111
  }
970
- });
971
- }).catch((error) => this.finish(item, error));
1112
+ })
1113
+ );
1114
+ if (sent) {
1115
+ item.status = "requested";
1116
+ } else {
1117
+ this.finish(item, "failed", new Error("WebSocket connection lost"));
1118
+ }
972
1119
  }
973
1120
  async *read(item) {
974
1121
  while (true) {
@@ -977,21 +1124,22 @@ var OpenaiRealtimeSpeech = class {
977
1124
  yield chunk;
978
1125
  continue;
979
1126
  }
980
- if (item.error) throw item.error;
981
- if (item.done) return;
1127
+ if (item.status === "failed") throw item.error;
1128
+ if (item.status === "done" || item.status === "cancelled") return;
982
1129
  await new Promise((resolve) => item.wake = resolve);
983
1130
  }
984
1131
  }
985
1132
  cancel(item) {
986
- if (item.responseId && !item.done) {
1133
+ if (item.status === "created" && item.responseId) {
987
1134
  this.send({ type: "response.cancel", response_id: item.responseId });
988
1135
  }
989
1136
  item.audio = [];
990
- this.finish(item);
1137
+ this.finish(item, "cancelled");
991
1138
  }
992
- finish(item, error) {
1139
+ /** Ends a sentence, the first end being the only one that counts */
1140
+ finish(item, status, error) {
993
1141
  if (!this.items.has(item.id)) return;
994
- item.done = true;
1142
+ item.status = status;
995
1143
  item.error = error;
996
1144
  this.items.delete(item.id);
997
1145
  if (item.responseId) this.responses.delete(item.responseId);
@@ -1000,9 +1148,7 @@ var OpenaiRealtimeSpeech = class {
1000
1148
  wake?.();
1001
1149
  }
1002
1150
  send(event) {
1003
- if (this.socket?.readyState === import_ws3.default.OPEN) {
1004
- this.socket.send(JSON.stringify(event));
1005
- }
1151
+ this.socket.send(JSON.stringify(event));
1006
1152
  }
1007
1153
  sessionUpdate() {
1008
1154
  const { instructions, speed } = this.options;
@@ -1029,80 +1175,8 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1029
1175
  }
1030
1176
  };
1031
1177
  }
1032
- /**
1033
- * Opens the session, or hands over the one already open.
1034
- *
1035
- * A Realtime session lasts an hour at most. A closed one is opened again by
1036
- * the next sentence rather than on a timer, so an idle agent holds no
1037
- * connection.
1038
- */
1039
- connect() {
1040
- if (this.destroyed) return Promise.reject(new Error("TTS destroyed"));
1041
- if (this.socket?.readyState === import_ws3.default.OPEN) {
1042
- return Promise.resolve(this.socket);
1043
- }
1044
- if (this.connecting) return this.connecting;
1045
- const { model, apiKey } = this.options;
1046
- const connecting = new Promise((resolve, reject) => {
1047
- const socket = new import_ws3.default(
1048
- `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
1049
- { headers: { Authorization: `Bearer ${apiKey}` } }
1050
- );
1051
- this.socket = socket;
1052
- const timeout = setTimeout(() => {
1053
- this.log("Connection timeout");
1054
- this.closeSocket();
1055
- reject(new Error("WebSocket connection timeout"));
1056
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT3);
1057
- socket.on("open", () => {
1058
- clearTimeout(timeout);
1059
- this.log("Connection opened");
1060
- socket.send(JSON.stringify(this.sessionUpdate()));
1061
- resolve(socket);
1062
- });
1063
- socket.on("error", (error) => {
1064
- clearTimeout(timeout);
1065
- this.log("WebSocket error:", error);
1066
- reject(new Error("WebSocket connection error"));
1067
- });
1068
- socket.on("close", (code, reason) => {
1069
- clearTimeout(timeout);
1070
- this.log("Connection closed", { code, reason: reason.toString() });
1071
- reject(new Error("WebSocket connection closed"));
1072
- this.closeSocket();
1073
- });
1074
- socket.on("message", (data) => {
1075
- try {
1076
- this.handleMessage(JSON.parse(data.toString()));
1077
- } catch (error) {
1078
- this.log("Error handling message", error);
1079
- }
1080
- });
1081
- });
1082
- this.connecting = connecting;
1083
- const settle = () => {
1084
- if (this.connecting === connecting) this.connecting = void 0;
1085
- };
1086
- connecting.then(settle, settle);
1087
- return connecting;
1088
- }
1089
- closeSocket() {
1090
- const socket = this.socket;
1091
- this.socket = void 0;
1092
- this.connecting = void 0;
1093
- for (const item of [...this.items.values()]) {
1094
- this.finish(item, new Error("Connection lost"));
1095
- }
1096
- if (!socket) return;
1097
- socket.removeAllListeners();
1098
- socket.on("error", () => {
1099
- });
1100
- if (socket.readyState === import_ws3.default.OPEN || socket.readyState === import_ws3.default.CONNECTING) {
1101
- socket.close(1e3);
1102
- }
1103
- }
1104
1178
  handleMessage(message) {
1105
- switch (message.type) {
1179
+ switch (message?.type) {
1106
1180
  case "response.created": {
1107
1181
  const id = message.response.id;
1108
1182
  const item = this.items.get(message.response.metadata?.micdrop_id);
@@ -1110,6 +1184,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1110
1184
  this.send({ type: "response.cancel", response_id: id });
1111
1185
  break;
1112
1186
  }
1187
+ item.status = "created";
1113
1188
  item.responseId = id;
1114
1189
  this.responses.set(id, item);
1115
1190
  break;
@@ -1117,9 +1192,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1117
1192
  case "response.output_audio.delta": {
1118
1193
  const item = this.responses.get(message.response_id);
1119
1194
  if (!item) break;
1120
- const audio = item.resampler.process(
1121
- Buffer.from(message.delta, "base64")
1122
- );
1195
+ const audio = Buffer.from(message.delta, "base64");
1123
1196
  if (audio.length === 0) break;
1124
1197
  item.audio.push(audio);
1125
1198
  const wake = item.wake;
@@ -1141,12 +1214,13 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1141
1214
  if (response.status === "failed") {
1142
1215
  this.finish(
1143
1216
  item,
1217
+ "failed",
1144
1218
  new Error(
1145
1219
  `Response failed: ${JSON.stringify(response.status_details)}`
1146
1220
  )
1147
1221
  );
1148
1222
  } else {
1149
- this.finish(item);
1223
+ this.finish(item, "done");
1150
1224
  }
1151
1225
  break;
1152
1226
  }
@@ -1154,7 +1228,7 @@ Delivery: ${instructions}` : READER_INSTRUCTIONS,
1154
1228
  if (message.error?.code === "response_cancel_not_active") break;
1155
1229
  this.log("Error:", message.error);
1156
1230
  const item = this.items.get(message.error?.event_id);
1157
- if (item) this.finish(item, new Error(message.error?.message));
1231
+ if (item) this.finish(item, "failed", new Error(message.error?.message));
1158
1232
  break;
1159
1233
  }
1160
1234
  default:
@@ -1171,10 +1245,21 @@ var DEFAULT_MODEL5 = "gpt-4o-mini-tts";
1171
1245
  var DEFAULT_VOICE2 = "alloy";
1172
1246
  var DEFAULT_REALTIME_VOICE = "marin";
1173
1247
  var OPENAI_SAMPLE_RATE4 = 24e3;
1174
- var OUTPUT_SAMPLE_RATE2 = 16e3;
1175
1248
  var MAX_IN_FLIGHT = 2;
1176
- var OPENING_CUSHION = 300 * 32;
1177
- var OpenaiTTS = class extends import_server6.TTS {
1249
+ var OPENING_CUSHION = 300 * OPENAI_SAMPLE_RATE4 * 2 / 1e3;
1250
+ var SHARED_VOICES = [
1251
+ "alloy",
1252
+ "ash",
1253
+ "ballad",
1254
+ "coral",
1255
+ "echo",
1256
+ "sage",
1257
+ "shimmer",
1258
+ "verse"
1259
+ ];
1260
+ var SPEECH_VOICES = ["fable", "nova", "onyx"];
1261
+ var REALTIME_VOICES = ["marin", "cedar"];
1262
+ var OpenaiTTS = class extends import_server7.TTS {
1178
1263
  constructor(options) {
1179
1264
  super();
1180
1265
  this.options = options;
@@ -1185,7 +1270,7 @@ var OpenaiTTS = class extends import_server6.TTS {
1185
1270
  // `counter`, which numbers the synthesis calls and must only move when there
1186
1271
  // is something to synthesize.
1187
1272
  this.generation = 0;
1188
- this.splitter = new import_server6.SentenceSplitter();
1273
+ this.splitter = new import_server7.SentenceSplitter();
1189
1274
  this.queue = [];
1190
1275
  // Sentences whose synthesis has not started
1191
1276
  this.pending = [];
@@ -1214,6 +1299,37 @@ var OpenaiTTS = class extends import_server6.TTS {
1214
1299
  this.openai = "openai" in options ? options.openai : new import_openai3.default({ apiKey: options.apiKey });
1215
1300
  }
1216
1301
  }
1302
+ /**
1303
+ * Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
1304
+ * else about them.
1305
+ *
1306
+ * The list is fetched once and shared by every call of the process for a
1307
+ * day. `refresh` fetches it again.
1308
+ */
1309
+ static listModels(options) {
1310
+ return listOpenaiModels("tts", options);
1311
+ }
1312
+ /**
1313
+ * Lists the voices of the model: the Realtime API has no fable, nova and
1314
+ * onyx, and adds marin and cedar. OpenAI has no listing, so the list is
1315
+ * written here. Every voice reads every language, so `language` keeps them
1316
+ * all.
1317
+ */
1318
+ static async listVoices(options = {}) {
1319
+ const realtime = (options.model || DEFAULT_MODEL5).startsWith("gpt-realtime");
1320
+ const ids = [
1321
+ ...realtime ? REALTIME_VOICES : [],
1322
+ ...SHARED_VOICES,
1323
+ ...realtime ? [] : SPEECH_VOICES
1324
+ ];
1325
+ return ids.map((id) => ({
1326
+ id,
1327
+ name: id.charAt(0).toUpperCase() + id.slice(1)
1328
+ }));
1329
+ }
1330
+ get sampleRate() {
1331
+ return OPENAI_SAMPLE_RATE4;
1332
+ }
1217
1333
  speak(textStream) {
1218
1334
  const generation = ++this.generation;
1219
1335
  let counter = 0;
@@ -1234,15 +1350,17 @@ var OpenaiTTS = class extends import_server6.TTS {
1234
1350
  this.enqueue(counter, sentence);
1235
1351
  }
1236
1352
  });
1237
- textStream.on("error", (error) => {
1238
- this.log("Error in text stream", error);
1239
- });
1240
- textStream.on("end", () => {
1353
+ const flush = () => {
1241
1354
  if (!counter || counter !== this.counter) return;
1242
1355
  for (const sentence of this.splitter.flush()) {
1243
1356
  this.enqueue(counter, sentence);
1244
1357
  }
1358
+ };
1359
+ textStream.on("error", (error) => {
1360
+ this.log("Error in text stream", error);
1361
+ flush();
1245
1362
  });
1363
+ textStream.on("end", flush);
1246
1364
  }
1247
1365
  cancel() {
1248
1366
  this.generation++;
@@ -1323,16 +1441,13 @@ var OpenaiTTS = class extends import_server6.TTS {
1323
1441
  async function* read() {
1324
1442
  const { body } = await response;
1325
1443
  if (!body) return;
1326
- const resampler = new import_server6.Pcm16Resampler(
1327
- OPENAI_SAMPLE_RATE4,
1328
- OUTPUT_SAMPLE_RATE2
1329
- );
1444
+ const aligner = new import_server7.Pcm16Aligner();
1330
1445
  const reader = body.getReader();
1331
1446
  try {
1332
1447
  while (true) {
1333
1448
  const { done, value } = await reader.read();
1334
1449
  if (done) break;
1335
- const output = resampler.process(Buffer.from(value));
1450
+ const output = aligner.process(Buffer.from(value));
1336
1451
  if (output.length > 0) yield output;
1337
1452
  }
1338
1453
  } finally {