@micdrop/openai 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +28 -3
- package/dist/index.d.ts +28 -3
- package/dist/index.js +113 -38
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +113 -38
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -80,17 +80,42 @@ declare class OpenaiTTS extends TTS {
|
|
|
80
80
|
private readonly options;
|
|
81
81
|
private openai;
|
|
82
82
|
private counter;
|
|
83
|
+
private generation;
|
|
83
84
|
private buffer;
|
|
84
85
|
private queue;
|
|
86
|
+
private pending;
|
|
87
|
+
private cushion;
|
|
88
|
+
private cushionBytes;
|
|
85
89
|
private processing;
|
|
86
|
-
private
|
|
90
|
+
private abortControllers;
|
|
87
91
|
constructor(options: OpenaiTTSOptions);
|
|
88
92
|
speak(textStream: Readable): void;
|
|
89
93
|
cancel(): void;
|
|
90
94
|
private flushSentences;
|
|
91
95
|
private enqueue;
|
|
92
|
-
|
|
93
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Starts what can be started, then speaks what is ready.
|
|
98
|
+
*
|
|
99
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
100
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
101
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
102
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
103
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
104
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
105
|
+
*
|
|
106
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
107
|
+
* answer from opening one request per sentence all at once.
|
|
108
|
+
*/
|
|
109
|
+
private pump;
|
|
110
|
+
/** Speaks the started requests in the order they were queued. */
|
|
111
|
+
private drain;
|
|
112
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
113
|
+
private request;
|
|
114
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
115
|
+
private emitAudio;
|
|
116
|
+
private flushCushion;
|
|
117
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
118
|
+
private speakItem;
|
|
94
119
|
}
|
|
95
120
|
|
|
96
121
|
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
package/dist/index.d.ts
CHANGED
|
@@ -80,17 +80,42 @@ declare class OpenaiTTS extends TTS {
|
|
|
80
80
|
private readonly options;
|
|
81
81
|
private openai;
|
|
82
82
|
private counter;
|
|
83
|
+
private generation;
|
|
83
84
|
private buffer;
|
|
84
85
|
private queue;
|
|
86
|
+
private pending;
|
|
87
|
+
private cushion;
|
|
88
|
+
private cushionBytes;
|
|
85
89
|
private processing;
|
|
86
|
-
private
|
|
90
|
+
private abortControllers;
|
|
87
91
|
constructor(options: OpenaiTTSOptions);
|
|
88
92
|
speak(textStream: Readable): void;
|
|
89
93
|
cancel(): void;
|
|
90
94
|
private flushSentences;
|
|
91
95
|
private enqueue;
|
|
92
|
-
|
|
93
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Starts what can be started, then speaks what is ready.
|
|
98
|
+
*
|
|
99
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
100
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
101
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
102
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
103
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
104
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
105
|
+
*
|
|
106
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
107
|
+
* answer from opening one request per sentence all at once.
|
|
108
|
+
*/
|
|
109
|
+
private pump;
|
|
110
|
+
/** Speaks the started requests in the order they were queued. */
|
|
111
|
+
private drain;
|
|
112
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
113
|
+
private request;
|
|
114
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
115
|
+
private emitAudio;
|
|
116
|
+
private flushCushion;
|
|
117
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
118
|
+
private speakItem;
|
|
94
119
|
}
|
|
95
120
|
|
|
96
121
|
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
package/dist/index.js
CHANGED
|
@@ -448,23 +448,48 @@ var DEFAULT_MODEL3 = "gpt-4o-mini-tts";
|
|
|
448
448
|
var DEFAULT_VOICE = "alloy";
|
|
449
449
|
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
450
450
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
451
|
+
var MAX_IN_FLIGHT = 2;
|
|
452
|
+
var OPENING_CUSHION = 300 * 32;
|
|
451
453
|
var OpenaiTTS = class extends import_server3.TTS {
|
|
452
454
|
constructor(options) {
|
|
453
455
|
super();
|
|
454
456
|
this.options = options;
|
|
455
457
|
this.counter = 0;
|
|
456
458
|
// Identifies the current speak() call
|
|
459
|
+
// Bumped by every speak() and every cancel(), so a call claimed late can tell
|
|
460
|
+
// whether it is still the one that should be heard. Kept apart from
|
|
461
|
+
// `counter`, which numbers the synthesis calls and must only move when there
|
|
462
|
+
// is something to synthesize.
|
|
463
|
+
this.generation = 0;
|
|
457
464
|
this.buffer = "";
|
|
458
465
|
// Incomplete sentence waiting for more text
|
|
459
466
|
this.queue = [];
|
|
467
|
+
// Sentences whose synthesis has not started
|
|
468
|
+
this.pending = [];
|
|
469
|
+
// Requests in flight, in speaking order
|
|
470
|
+
this.cushion = [];
|
|
471
|
+
// Opening audio held back, see OPENING_CUSHION
|
|
472
|
+
this.cushionBytes = 0;
|
|
460
473
|
this.processing = false;
|
|
474
|
+
// More than one request is in flight at a time, so they are aborted as a set.
|
|
475
|
+
this.abortControllers = /* @__PURE__ */ new Set();
|
|
461
476
|
this.openai = "openai" in options ? options.openai : new import_openai2.default({ apiKey: options.apiKey });
|
|
462
477
|
}
|
|
463
478
|
speak(textStream) {
|
|
464
|
-
this.
|
|
465
|
-
|
|
466
|
-
|
|
479
|
+
const generation = ++this.generation;
|
|
480
|
+
let counter = 0;
|
|
481
|
+
const claimCall = () => {
|
|
482
|
+
if (counter) return true;
|
|
483
|
+
if (this.generation !== generation) return false;
|
|
484
|
+
this.counter++;
|
|
485
|
+
counter = this.counter;
|
|
486
|
+
this.buffer = "";
|
|
487
|
+
this.cushion = [];
|
|
488
|
+
this.cushionBytes = 0;
|
|
489
|
+
return true;
|
|
490
|
+
};
|
|
467
491
|
textStream.on("data", (chunk) => {
|
|
492
|
+
if (!claimCall()) return;
|
|
468
493
|
if (counter !== this.counter) return;
|
|
469
494
|
this.buffer += chunk.toString("utf-8");
|
|
470
495
|
this.flushSentences(counter, false);
|
|
@@ -473,17 +498,21 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
473
498
|
this.log("Error in text stream", error);
|
|
474
499
|
});
|
|
475
500
|
textStream.on("end", () => {
|
|
476
|
-
if (counter !== this.counter) return;
|
|
501
|
+
if (!counter || counter !== this.counter) return;
|
|
477
502
|
this.flushSentences(counter, true);
|
|
478
503
|
});
|
|
479
504
|
}
|
|
480
505
|
cancel() {
|
|
506
|
+
this.generation++;
|
|
481
507
|
this.log("Cancel");
|
|
482
508
|
this.counter++;
|
|
483
509
|
this.buffer = "";
|
|
484
510
|
this.queue = [];
|
|
485
|
-
this.
|
|
486
|
-
this.
|
|
511
|
+
this.pending = [];
|
|
512
|
+
this.cushion = [];
|
|
513
|
+
this.cushionBytes = 0;
|
|
514
|
+
this.abortControllers.forEach((controller) => controller.abort());
|
|
515
|
+
this.abortControllers.clear();
|
|
487
516
|
}
|
|
488
517
|
// Extract complete sentences from the buffer and enqueue them.
|
|
489
518
|
// On `end`, whatever remains in the buffer is flushed as a last sentence.
|
|
@@ -506,36 +535,88 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
506
535
|
}
|
|
507
536
|
enqueue(counter, text) {
|
|
508
537
|
this.queue.push({ counter, text });
|
|
509
|
-
this.
|
|
538
|
+
this.pump();
|
|
510
539
|
}
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
540
|
+
/**
|
|
541
|
+
* Starts what can be started, then speaks what is ready.
|
|
542
|
+
*
|
|
543
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
544
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
545
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
546
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
547
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
548
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
549
|
+
*
|
|
550
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
551
|
+
* answer from opening one request per sentence all at once.
|
|
552
|
+
*/
|
|
553
|
+
pump() {
|
|
554
|
+
while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {
|
|
515
555
|
const item = this.queue.shift();
|
|
516
556
|
if (item.counter !== this.counter) continue;
|
|
517
|
-
|
|
557
|
+
this.pending.push(this.request(item));
|
|
558
|
+
}
|
|
559
|
+
this.drain();
|
|
560
|
+
}
|
|
561
|
+
/** Speaks the started requests in the order they were queued. */
|
|
562
|
+
async drain() {
|
|
563
|
+
if (this.processing) return;
|
|
564
|
+
this.processing = true;
|
|
565
|
+
while (this.pending.length > 0) {
|
|
566
|
+
const item = this.pending.shift();
|
|
567
|
+
this.pump();
|
|
568
|
+
await this.speakItem(item);
|
|
518
569
|
}
|
|
519
570
|
this.processing = false;
|
|
520
|
-
|
|
571
|
+
this.flushCushion();
|
|
572
|
+
if (this.pending.length > 0 || this.queue.length > 0) this.pump();
|
|
521
573
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
574
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
575
|
+
request(item) {
|
|
576
|
+
const controller = new AbortController();
|
|
577
|
+
this.abortControllers.add(controller);
|
|
578
|
+
const response = this.openai.audio.speech.create(
|
|
579
|
+
{
|
|
580
|
+
model: this.options.model || DEFAULT_MODEL3,
|
|
581
|
+
voice: this.options.voice || DEFAULT_VOICE,
|
|
582
|
+
input: item.text,
|
|
583
|
+
response_format: "pcm",
|
|
584
|
+
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
585
|
+
...this.options.speed ? { speed: this.options.speed } : {}
|
|
586
|
+
},
|
|
587
|
+
{ signal: controller.signal }
|
|
588
|
+
).catch((error) => {
|
|
589
|
+
if (!controller.signal.aborted) {
|
|
590
|
+
this.log("Error synthesizing speech:", error);
|
|
591
|
+
this.emit("Failed", [item.text]);
|
|
592
|
+
}
|
|
593
|
+
return null;
|
|
594
|
+
});
|
|
595
|
+
return { ...item, controller, response };
|
|
596
|
+
}
|
|
597
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
598
|
+
emitAudio(chunk) {
|
|
599
|
+
if (this.cushionBytes >= OPENING_CUSHION) {
|
|
600
|
+
this.emit("Audio", chunk);
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
this.cushion.push(chunk);
|
|
604
|
+
this.cushionBytes += chunk.length;
|
|
605
|
+
if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion();
|
|
606
|
+
}
|
|
607
|
+
flushCushion() {
|
|
608
|
+
if (this.cushion.length === 0) return;
|
|
609
|
+
const held = this.cushion;
|
|
610
|
+
this.cushion = [];
|
|
611
|
+
for (const chunk of held) this.emit("Audio", chunk);
|
|
612
|
+
}
|
|
613
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
614
|
+
async speakItem(item) {
|
|
525
615
|
try {
|
|
526
|
-
const response = await
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
input: text,
|
|
531
|
-
response_format: "pcm",
|
|
532
|
-
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
533
|
-
...this.options.speed ? { speed: this.options.speed } : {}
|
|
534
|
-
},
|
|
535
|
-
{ signal: abortController.signal }
|
|
536
|
-
);
|
|
537
|
-
if (!response.body) return;
|
|
538
|
-
this.log(`Synthesizing: "${text}"`);
|
|
616
|
+
const response = await item.response;
|
|
617
|
+
if (!response?.body) return;
|
|
618
|
+
if (item.counter !== this.counter) return;
|
|
619
|
+
this.log(`Synthesizing: "${item.text}"`);
|
|
539
620
|
const resampler = new Pcm16Resampler(
|
|
540
621
|
OPENAI_SAMPLE_RATE2,
|
|
541
622
|
OUTPUT_SAMPLE_RATE
|
|
@@ -544,21 +625,15 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
544
625
|
while (true) {
|
|
545
626
|
const { done, value } = await reader.read();
|
|
546
627
|
if (done) break;
|
|
547
|
-
if (counter !== this.counter) {
|
|
628
|
+
if (item.counter !== this.counter) {
|
|
548
629
|
await reader.cancel();
|
|
549
630
|
break;
|
|
550
631
|
}
|
|
551
632
|
const output = resampler.process(Buffer.from(value));
|
|
552
|
-
if (output.length > 0) this.
|
|
633
|
+
if (output.length > 0) this.emitAudio(output);
|
|
553
634
|
}
|
|
554
|
-
} catch (error) {
|
|
555
|
-
if (abortController.signal.aborted) return;
|
|
556
|
-
this.log("Error synthesizing speech:", error);
|
|
557
|
-
this.emit("Failed", [text]);
|
|
558
635
|
} finally {
|
|
559
|
-
|
|
560
|
-
this.abortController = void 0;
|
|
561
|
-
}
|
|
636
|
+
this.abortControllers.delete(item.controller);
|
|
562
637
|
}
|
|
563
638
|
}
|
|
564
639
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\nexport * from './OpenaiTTS'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n let hasAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n // Ignore extra outputs, we keep the first answer only\n if (hasAnswer) break\n\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n // Keep the first answer only.\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n if (hasAnswer) break\n\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n hasAnswer = true\n // Keep reading events instead of returning here: the API streams\n // one output item at a time, so a function call the model made\n // alongside its answer is only announced after the text is done.\n // Returning dropped every tool call that came with an answer.\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again when the model called a tool without answering, so the\n // turn still produces something to say\n if (!skipAnswer && !hasAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = []\n private processing = false\n private abortController?: AbortController\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n this.counter++\n const counter = this.counter\n this.buffer = ''\n\n textStream.on('data', (chunk: Buffer) => {\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n if (counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.abortController?.abort()\n this.abortController = undefined\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.processQueue()\n }\n\n private async processQueue() {\n if (this.processing) return\n this.processing = true\n\n while (this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n await this.synthesize(item.counter, item.text)\n }\n\n this.processing = false\n // Items may have been enqueued right as we exited the loop\n if (this.queue.length > 0) this.processQueue()\n }\n\n private async synthesize(counter: number, text: string) {\n const abortController = new AbortController()\n this.abortController = abortController\n\n try {\n const response = await this.openai.audio.speech.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: abortController.signal }\n )\n\n if (!response.body) return\n this.log(`Synthesizing: \"${text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emit('Audio', output)\n }\n } catch (error) {\n if (abortController.signal.aborted) return\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [text])\n } finally {\n if (this.abortController === abortController) {\n this.abortController = undefined\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,oBAAmB;AAEnB,iBAAgC;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AACjB,UAAI,YAAY;AAGhB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AAEH,gBAAI,UAAW;AAEf,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AAMH,gBAAI,UAAW;AAEf,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,wBAAY;AAKZ;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAI9C,UAAI,CAAC,cAAc,CAAC,WAAW;AAC7B,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAA,QAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACnNA,IAAAC,iBAAoB;AAEpB,gBAAsB;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMC,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASF;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI,UAAAE;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,IAAAE,iBAAoB;AACpB,IAAAC,iBAAmB;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAOpB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EAQjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAN7B,SAAQ,UAAU;AAClB;AAAA,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B,SAAQ,aAAa;AAKnB,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,eAAAC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,SAAK;AACL,UAAM,UAAU,KAAK;AACrB,SAAK,SAAS;AAEd,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AACzB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,iBAAiB,MAAM;AAC5B,SAAK,kBAAkB;AAAA,EACzB;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAc,eAAe;AAC3B,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,MAAM,SAAS,GAAG;AAC5B,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,YAAM,KAAK,WAAW,KAAK,SAAS,KAAK,IAAI;AAAA,IAC/C;AAEA,SAAK,aAAa;AAElB,QAAI,KAAK,MAAM,SAAS,EAAG,MAAK,aAAa;AAAA,EAC/C;AAAA,EAEA,MAAc,WAAW,SAAiB,MAAc;AACtD,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AAEvB,QAAI;AACF,YAAM,WAAW,MAAM,KAAK,OAAO,MAAM,OAAO;AAAA,QAC9C;AAAA,UACE,OAAO,KAAK,QAAQ,SAASF;AAAA,UAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO;AAAA,UACP,iBAAiB;AAAA,UACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,UACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,QAC5D;AAAA,QACA,EAAE,QAAQ,gBAAgB,OAAO;AAAA,MACnC;AAEA,UAAI,CAAC,SAAS,KAAM;AACpB,WAAK,IAAI,kBAAkB,IAAI,GAAG;AAElC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,YAAY,KAAK,SAAS;AAC5B,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,KAAK,SAAS,MAAM;AAAA,MAClD;AAAA,IACF,SAAS,OAAO;AACd,UAAI,gBAAgB,OAAO,QAAS;AACpC,WAAK,IAAI,8BAA8B,KAAK;AAC5C,WAAK,KAAK,UAAU,CAAC,IAAI,CAAC;AAAA,IAC5B,UAAE;AACA,UAAI,KAAK,oBAAoB,iBAAiB;AAC5C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AACF;","names":["OpenAI","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","WebSocket","import_server","import_openai","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\nexport * from './OpenaiTTS'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n let hasAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n // Ignore extra outputs, we keep the first answer only\n if (hasAnswer) break\n\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n // Keep the first answer only.\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n if (hasAnswer) break\n\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n hasAnswer = true\n // Keep reading events instead of returning here: the API streams\n // one output item at a time, so a function call the model made\n // alongside its answer is only announced after the text is done.\n // Returning dropped every tool call that came with an answer.\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again when the model called a tool without answering, so the\n // turn still produces something to say\n if (!skipAnswer && !hasAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\nconst MAX_IN_FLIGHT = 2 // Requests asked for at once, current one included\n// Audio held back at the start of an utterance, in bytes of the output format\n// (16 bits, 16 kHz, so 32 bytes per millisecond).\n//\n// The endpoint answers with a burst of a hundred milliseconds or so, then goes\n// quiet while it generates the rest. Forwarding that burst as it arrives means\n// the browser starts playing a syllable it cannot continue, and the hole lands\n// inside the first word. Holding the opening back until there is enough of it\n// to cover the pause costs the same time either way, and spends it before the\n// first syllable rather than inside it.\nconst OPENING_CUSHION = 300 * 32\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\n/** A sentence whose synthesis has been asked for and not yet spoken. */\ninterface PendingItem extends QueueItem {\n controller: AbortController\n response: Promise<{ body: ReadableStream<Uint8Array> | null } | null>\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n // Bumped by every speak() and every cancel(), so a call claimed late can tell\n // whether it is still the one that should be heard. Kept apart from\n // `counter`, which numbers the synthesis calls and must only move when there\n // is something to synthesize.\n private generation = 0\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = [] // Sentences whose synthesis has not started\n private pending: PendingItem[] = [] // Requests in flight, in speaking order\n private cushion: Buffer[] = [] // Opening audio held back, see OPENING_CUSHION\n private cushionBytes = 0\n private processing = false\n // More than one request is in flight at a time, so they are aborted as a set.\n private abortControllers = new Set<AbortController>()\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n const generation = ++this.generation\n let counter = 0\n\n // Claiming the call is deferred until there is something to say.\n //\n // Taking the next number right away would drop the utterance still being\n // spoken, since both the queue and the streaming reader skip anything\n // stamped with an older one. A stream that never carries a word, which is\n // what an answer skipped by a tool or by onBeforeAnswer hands over, would\n // then cut the assistant off and throw away the sentences still queued.\n const claimCall = () => {\n if (counter) return true\n // Cancelled, or superseded by another speak(), before the first word.\n if (this.generation !== generation) return false\n this.counter++\n counter = this.counter\n this.buffer = ''\n this.cushion = []\n this.cushionBytes = 0\n return true\n }\n\n textStream.on('data', (chunk: Buffer) => {\n if (!claimCall()) return\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n // Nothing was ever said, so there is nothing left to flush.\n if (!counter || counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.generation++\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.pending = []\n this.cushion = []\n this.cushionBytes = 0\n this.abortControllers.forEach((controller) => controller.abort())\n this.abortControllers.clear()\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.pump()\n }\n\n /**\n * Starts what can be started, then speaks what is ready.\n *\n * The endpoint takes a few hundred milliseconds before its first byte, and\n * waiting for that between two sentences leaves a silence in the middle of\n * her voice. It is loudest at the very start, where a greeting is often one\n * short sentence: a syllable, a hole, then the rest of the answer. Asking for\n * a sentence the moment it is complete, rather than when its turn to be\n * spoken comes, hides that latency behind the audio already playing.\n *\n * Two requests at a time is enough to cover the gap, and it keeps a long\n * answer from opening one request per sentence all at once.\n */\n private pump() {\n while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n this.pending.push(this.request(item))\n }\n this.drain()\n }\n\n /** Speaks the started requests in the order they were queued. */\n private async drain() {\n if (this.processing) return\n this.processing = true\n\n while (this.pending.length > 0) {\n const item = this.pending.shift()!\n // A slot just freed, so the sentence after the next one can start now\n this.pump()\n await this.speakItem(item)\n }\n\n this.processing = false\n // Nothing left to speak: an utterance shorter than the cushion is released\n // here, since there is no longer anything coming to fill it up.\n this.flushCushion()\n // Sentences may have arrived right as we exited the loop\n if (this.pending.length > 0 || this.queue.length > 0) this.pump()\n }\n\n /** Asks for one sentence, without waiting for the answer. */\n private request(item: QueueItem): PendingItem {\n const controller = new AbortController()\n this.abortControllers.add(controller)\n\n const response = this.openai.audio.speech\n .create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: item.text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: controller.signal }\n )\n .catch((error) => {\n if (!controller.signal.aborted) {\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [item.text])\n }\n return null\n })\n\n return { ...item, controller, response }\n }\n\n /** Holds the opening of an utterance back, then lets the rest through. */\n private emitAudio(chunk: Buffer) {\n if (this.cushionBytes >= OPENING_CUSHION) {\n this.emit('Audio', chunk)\n return\n }\n this.cushion.push(chunk)\n this.cushionBytes += chunk.length\n if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion()\n }\n\n private flushCushion() {\n if (this.cushion.length === 0) return\n const held = this.cushion\n this.cushion = []\n for (const chunk of held) this.emit('Audio', chunk)\n }\n\n /** Emits one sentence, whose request was started earlier. */\n private async speakItem(item: PendingItem) {\n try {\n const response = await item.response\n if (!response?.body) return\n if (item.counter !== this.counter) return\n this.log(`Synthesizing: \"${item.text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (item.counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emitAudio(output)\n }\n } finally {\n this.abortControllers.delete(item.controller)\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,oBAAmB;AAEnB,iBAAgC;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AACjB,UAAI,YAAY;AAGhB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AAEH,gBAAI,UAAW;AAEf,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AAMH,gBAAI,UAAW;AAEf,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,wBAAY;AAKZ;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAI9C,UAAI,CAAC,cAAc,CAAC,WAAW;AAC7B,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAA,QAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACnNA,IAAAC,iBAAoB;AAEpB,gBAAsB;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMC,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASF;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI,UAAAE;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,IAAAE,iBAAoB;AACpB,IAAAC,iBAAmB;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAC3B,IAAM,gBAAgB;AAUtB,IAAM,kBAAkB,MAAM;AAavB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EAiBjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAf7B,SAAQ,UAAU;AAKlB;AAAA;AAAA;AAAA;AAAA;AAAA,SAAQ,aAAa;AACrB,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B;AAAA,SAAQ,UAAyB,CAAC;AAClC;AAAA,SAAQ,UAAoB,CAAC;AAC7B;AAAA,SAAQ,eAAe;AACvB,SAAQ,aAAa;AAErB;AAAA,SAAQ,mBAAmB,oBAAI,IAAqB;AAIlD,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,eAAAC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,UAAM,aAAa,EAAE,KAAK;AAC1B,QAAI,UAAU;AASd,UAAM,YAAY,MAAM;AACtB,UAAI,QAAS,QAAO;AAEpB,UAAI,KAAK,eAAe,WAAY,QAAO;AAC3C,WAAK;AACL,gBAAU,KAAK;AACf,WAAK,SAAS;AACd,WAAK,UAAU,CAAC;AAChB,WAAK,eAAe;AACpB,aAAO;AAAA,IACT;AAEA,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,CAAC,UAAU,EAAG;AAClB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AAEzB,UAAI,CAAC,WAAW,YAAY,KAAK,QAAS;AAC1C,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK;AACL,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,UAAU,CAAC;AAChB,SAAK,UAAU,CAAC;AAChB,SAAK,eAAe;AACpB,SAAK,iBAAiB,QAAQ,CAAC,eAAe,WAAW,MAAM,CAAC;AAChE,SAAK,iBAAiB,MAAM;AAAA,EAC9B;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeQ,OAAO;AACb,WAAO,KAAK,QAAQ,SAAS,iBAAiB,KAAK,MAAM,SAAS,GAAG;AACnE,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,QAAQ,KAAK,KAAK,QAAQ,IAAI,CAAC;AAAA,IACtC;AACA,SAAK,MAAM;AAAA,EACb;AAAA;AAAA,EAGA,MAAc,QAAQ;AACpB,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,QAAQ,SAAS,GAAG;AAC9B,YAAM,OAAO,KAAK,QAAQ,MAAM;AAEhC,WAAK,KAAK;AACV,YAAM,KAAK,UAAU,IAAI;AAAA,IAC3B;AAEA,SAAK,aAAa;AAGlB,SAAK,aAAa;AAElB,QAAI,KAAK,QAAQ,SAAS,KAAK,KAAK,MAAM,SAAS,EAAG,MAAK,KAAK;AAAA,EAClE;AAAA;AAAA,EAGQ,QAAQ,MAA8B;AAC5C,UAAM,aAAa,IAAI,gBAAgB;AACvC,SAAK,iBAAiB,IAAI,UAAU;AAEpC,UAAM,WAAW,KAAK,OAAO,MAAM,OAChC;AAAA,MACC;AAAA,QACE,OAAO,KAAK,QAAQ,SAASF;AAAA,QAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,OAAO,KAAK;AAAA,QACZ,iBAAiB;AAAA,QACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,QACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC5D;AAAA,MACA,EAAE,QAAQ,WAAW,OAAO;AAAA,IAC9B,EACC,MAAM,CAAC,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,SAAS;AAC9B,aAAK,IAAI,8BAA8B,KAAK;AAC5C,aAAK,KAAK,UAAU,CAAC,KAAK,IAAI,CAAC;AAAA,MACjC;AACA,aAAO;AAAA,IACT,CAAC;AAEH,WAAO,EAAE,GAAG,MAAM,YAAY,SAAS;AAAA,EACzC;AAAA;AAAA,EAGQ,UAAU,OAAe;AAC/B,QAAI,KAAK,gBAAgB,iBAAiB;AACxC,WAAK,KAAK,SAAS,KAAK;AACxB;AAAA,IACF;AACA,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,gBAAgB,MAAM;AAC3B,QAAI,KAAK,gBAAgB,gBAAiB,MAAK,aAAa;AAAA,EAC9D;AAAA,EAEQ,eAAe;AACrB,QAAI,KAAK,QAAQ,WAAW,EAAG;AAC/B,UAAM,OAAO,KAAK;AAClB,SAAK,UAAU,CAAC;AAChB,eAAW,SAAS,KAAM,MAAK,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA;AAAA,EAGA,MAAc,UAAU,MAAmB;AACzC,QAAI;AACF,YAAM,WAAW,MAAM,KAAK;AAC5B,UAAI,CAAC,UAAU,KAAM;AACrB,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,IAAI,kBAAkB,KAAK,IAAI,GAAG;AAEvC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,KAAK,YAAY,KAAK,SAAS;AACjC,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,UAAU,MAAM;AAAA,MAC9C;AAAA,IACF,UAAE;AACA,WAAK,iBAAiB,OAAO,KAAK,UAAU;AAAA,IAC9C;AAAA,EACF;AACF;","names":["OpenAI","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","WebSocket","import_server","import_openai","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
package/dist/index.mjs
CHANGED
|
@@ -410,23 +410,48 @@ var DEFAULT_MODEL3 = "gpt-4o-mini-tts";
|
|
|
410
410
|
var DEFAULT_VOICE = "alloy";
|
|
411
411
|
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
412
412
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
413
|
+
var MAX_IN_FLIGHT = 2;
|
|
414
|
+
var OPENING_CUSHION = 300 * 32;
|
|
413
415
|
var OpenaiTTS = class extends TTS {
|
|
414
416
|
constructor(options) {
|
|
415
417
|
super();
|
|
416
418
|
this.options = options;
|
|
417
419
|
this.counter = 0;
|
|
418
420
|
// Identifies the current speak() call
|
|
421
|
+
// Bumped by every speak() and every cancel(), so a call claimed late can tell
|
|
422
|
+
// whether it is still the one that should be heard. Kept apart from
|
|
423
|
+
// `counter`, which numbers the synthesis calls and must only move when there
|
|
424
|
+
// is something to synthesize.
|
|
425
|
+
this.generation = 0;
|
|
419
426
|
this.buffer = "";
|
|
420
427
|
// Incomplete sentence waiting for more text
|
|
421
428
|
this.queue = [];
|
|
429
|
+
// Sentences whose synthesis has not started
|
|
430
|
+
this.pending = [];
|
|
431
|
+
// Requests in flight, in speaking order
|
|
432
|
+
this.cushion = [];
|
|
433
|
+
// Opening audio held back, see OPENING_CUSHION
|
|
434
|
+
this.cushionBytes = 0;
|
|
422
435
|
this.processing = false;
|
|
436
|
+
// More than one request is in flight at a time, so they are aborted as a set.
|
|
437
|
+
this.abortControllers = /* @__PURE__ */ new Set();
|
|
423
438
|
this.openai = "openai" in options ? options.openai : new OpenAI2({ apiKey: options.apiKey });
|
|
424
439
|
}
|
|
425
440
|
speak(textStream) {
|
|
426
|
-
this.
|
|
427
|
-
|
|
428
|
-
|
|
441
|
+
const generation = ++this.generation;
|
|
442
|
+
let counter = 0;
|
|
443
|
+
const claimCall = () => {
|
|
444
|
+
if (counter) return true;
|
|
445
|
+
if (this.generation !== generation) return false;
|
|
446
|
+
this.counter++;
|
|
447
|
+
counter = this.counter;
|
|
448
|
+
this.buffer = "";
|
|
449
|
+
this.cushion = [];
|
|
450
|
+
this.cushionBytes = 0;
|
|
451
|
+
return true;
|
|
452
|
+
};
|
|
429
453
|
textStream.on("data", (chunk) => {
|
|
454
|
+
if (!claimCall()) return;
|
|
430
455
|
if (counter !== this.counter) return;
|
|
431
456
|
this.buffer += chunk.toString("utf-8");
|
|
432
457
|
this.flushSentences(counter, false);
|
|
@@ -435,17 +460,21 @@ var OpenaiTTS = class extends TTS {
|
|
|
435
460
|
this.log("Error in text stream", error);
|
|
436
461
|
});
|
|
437
462
|
textStream.on("end", () => {
|
|
438
|
-
if (counter !== this.counter) return;
|
|
463
|
+
if (!counter || counter !== this.counter) return;
|
|
439
464
|
this.flushSentences(counter, true);
|
|
440
465
|
});
|
|
441
466
|
}
|
|
442
467
|
cancel() {
|
|
468
|
+
this.generation++;
|
|
443
469
|
this.log("Cancel");
|
|
444
470
|
this.counter++;
|
|
445
471
|
this.buffer = "";
|
|
446
472
|
this.queue = [];
|
|
447
|
-
this.
|
|
448
|
-
this.
|
|
473
|
+
this.pending = [];
|
|
474
|
+
this.cushion = [];
|
|
475
|
+
this.cushionBytes = 0;
|
|
476
|
+
this.abortControllers.forEach((controller) => controller.abort());
|
|
477
|
+
this.abortControllers.clear();
|
|
449
478
|
}
|
|
450
479
|
// Extract complete sentences from the buffer and enqueue them.
|
|
451
480
|
// On `end`, whatever remains in the buffer is flushed as a last sentence.
|
|
@@ -468,36 +497,88 @@ var OpenaiTTS = class extends TTS {
|
|
|
468
497
|
}
|
|
469
498
|
enqueue(counter, text) {
|
|
470
499
|
this.queue.push({ counter, text });
|
|
471
|
-
this.
|
|
500
|
+
this.pump();
|
|
472
501
|
}
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
502
|
+
/**
|
|
503
|
+
* Starts what can be started, then speaks what is ready.
|
|
504
|
+
*
|
|
505
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
506
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
507
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
508
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
509
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
510
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
511
|
+
*
|
|
512
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
513
|
+
* answer from opening one request per sentence all at once.
|
|
514
|
+
*/
|
|
515
|
+
pump() {
|
|
516
|
+
while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {
|
|
477
517
|
const item = this.queue.shift();
|
|
478
518
|
if (item.counter !== this.counter) continue;
|
|
479
|
-
|
|
519
|
+
this.pending.push(this.request(item));
|
|
520
|
+
}
|
|
521
|
+
this.drain();
|
|
522
|
+
}
|
|
523
|
+
/** Speaks the started requests in the order they were queued. */
|
|
524
|
+
async drain() {
|
|
525
|
+
if (this.processing) return;
|
|
526
|
+
this.processing = true;
|
|
527
|
+
while (this.pending.length > 0) {
|
|
528
|
+
const item = this.pending.shift();
|
|
529
|
+
this.pump();
|
|
530
|
+
await this.speakItem(item);
|
|
480
531
|
}
|
|
481
532
|
this.processing = false;
|
|
482
|
-
|
|
533
|
+
this.flushCushion();
|
|
534
|
+
if (this.pending.length > 0 || this.queue.length > 0) this.pump();
|
|
483
535
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
536
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
537
|
+
request(item) {
|
|
538
|
+
const controller = new AbortController();
|
|
539
|
+
this.abortControllers.add(controller);
|
|
540
|
+
const response = this.openai.audio.speech.create(
|
|
541
|
+
{
|
|
542
|
+
model: this.options.model || DEFAULT_MODEL3,
|
|
543
|
+
voice: this.options.voice || DEFAULT_VOICE,
|
|
544
|
+
input: item.text,
|
|
545
|
+
response_format: "pcm",
|
|
546
|
+
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
547
|
+
...this.options.speed ? { speed: this.options.speed } : {}
|
|
548
|
+
},
|
|
549
|
+
{ signal: controller.signal }
|
|
550
|
+
).catch((error) => {
|
|
551
|
+
if (!controller.signal.aborted) {
|
|
552
|
+
this.log("Error synthesizing speech:", error);
|
|
553
|
+
this.emit("Failed", [item.text]);
|
|
554
|
+
}
|
|
555
|
+
return null;
|
|
556
|
+
});
|
|
557
|
+
return { ...item, controller, response };
|
|
558
|
+
}
|
|
559
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
560
|
+
emitAudio(chunk) {
|
|
561
|
+
if (this.cushionBytes >= OPENING_CUSHION) {
|
|
562
|
+
this.emit("Audio", chunk);
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
this.cushion.push(chunk);
|
|
566
|
+
this.cushionBytes += chunk.length;
|
|
567
|
+
if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion();
|
|
568
|
+
}
|
|
569
|
+
flushCushion() {
|
|
570
|
+
if (this.cushion.length === 0) return;
|
|
571
|
+
const held = this.cushion;
|
|
572
|
+
this.cushion = [];
|
|
573
|
+
for (const chunk of held) this.emit("Audio", chunk);
|
|
574
|
+
}
|
|
575
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
576
|
+
async speakItem(item) {
|
|
487
577
|
try {
|
|
488
|
-
const response = await
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
input: text,
|
|
493
|
-
response_format: "pcm",
|
|
494
|
-
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
495
|
-
...this.options.speed ? { speed: this.options.speed } : {}
|
|
496
|
-
},
|
|
497
|
-
{ signal: abortController.signal }
|
|
498
|
-
);
|
|
499
|
-
if (!response.body) return;
|
|
500
|
-
this.log(`Synthesizing: "${text}"`);
|
|
578
|
+
const response = await item.response;
|
|
579
|
+
if (!response?.body) return;
|
|
580
|
+
if (item.counter !== this.counter) return;
|
|
581
|
+
this.log(`Synthesizing: "${item.text}"`);
|
|
501
582
|
const resampler = new Pcm16Resampler(
|
|
502
583
|
OPENAI_SAMPLE_RATE2,
|
|
503
584
|
OUTPUT_SAMPLE_RATE
|
|
@@ -506,21 +587,15 @@ var OpenaiTTS = class extends TTS {
|
|
|
506
587
|
while (true) {
|
|
507
588
|
const { done, value } = await reader.read();
|
|
508
589
|
if (done) break;
|
|
509
|
-
if (counter !== this.counter) {
|
|
590
|
+
if (item.counter !== this.counter) {
|
|
510
591
|
await reader.cancel();
|
|
511
592
|
break;
|
|
512
593
|
}
|
|
513
594
|
const output = resampler.process(Buffer.from(value));
|
|
514
|
-
if (output.length > 0) this.
|
|
595
|
+
if (output.length > 0) this.emitAudio(output);
|
|
515
596
|
}
|
|
516
|
-
} catch (error) {
|
|
517
|
-
if (abortController.signal.aborted) return;
|
|
518
|
-
this.log("Error synthesizing speech:", error);
|
|
519
|
-
this.emit("Failed", [text]);
|
|
520
597
|
} finally {
|
|
521
|
-
|
|
522
|
-
this.abortController = void 0;
|
|
523
|
-
}
|
|
598
|
+
this.abortControllers.delete(item.controller);
|
|
524
599
|
}
|
|
525
600
|
}
|
|
526
601
|
};
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n let hasAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n // Ignore extra outputs, we keep the first answer only\n if (hasAnswer) break\n\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n // Keep the first answer only.\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n if (hasAnswer) break\n\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n hasAnswer = true\n // Keep reading events instead of returning here: the API streams\n // one output item at a time, so a function call the model made\n // alongside its answer is only announced after the text is done.\n // Returning dropped every tool call that came with an answer.\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again when the model called a tool without answering, so the\n // turn still produces something to say\n if (!skipAnswer && !hasAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = []\n private processing = false\n private abortController?: AbortController\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n this.counter++\n const counter = this.counter\n this.buffer = ''\n\n textStream.on('data', (chunk: Buffer) => {\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n if (counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.abortController?.abort()\n this.abortController = undefined\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.processQueue()\n }\n\n private async processQueue() {\n if (this.processing) return\n this.processing = true\n\n while (this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n await this.synthesize(item.counter, item.text)\n }\n\n this.processing = false\n // Items may have been enqueued right as we exited the loop\n if (this.queue.length > 0) this.processQueue()\n }\n\n private async synthesize(counter: number, text: string) {\n const abortController = new AbortController()\n this.abortController = abortController\n\n try {\n const response = await this.openai.audio.speech.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: abortController.signal }\n )\n\n if (!response.body) return\n this.log(`Synthesizing: \"${text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emit('Audio', output)\n }\n } catch (error) {\n if (abortController.signal.aborted) return\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [text])\n } finally {\n if (this.abortController === abortController) {\n this.abortController = undefined\n }\n }\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,OAAO,YAAY;AAEnB,OAAO,KAAK,oBAAoB;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AACjB,UAAI,YAAY;AAGhB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AAEH,gBAAI,UAAW;AAEf,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AAMH,gBAAI,UAAW;AAEf,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,wBAAY;AAKZ;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAI9C,UAAI,CAAC,cAAc,CAAC,WAAW;AAC7B,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACnNA,SAAS,WAAW;AAEpB,OAAO,eAAe;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMA,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,IAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASD;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,SAAS,WAAW;AACpB,OAAOC,aAAY;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAOpB,IAAM,YAAN,cAAwB,IAAI;AAAA,EAQjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAN7B,SAAQ,UAAU;AAClB;AAAA,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B,SAAQ,aAAa;AAKnB,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAIC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,SAAK;AACL,UAAM,UAAU,KAAK;AACrB,SAAK,SAAS;AAEd,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AACzB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,iBAAiB,MAAM;AAC5B,SAAK,kBAAkB;AAAA,EACzB;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAc,eAAe;AAC3B,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,MAAM,SAAS,GAAG;AAC5B,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,YAAM,KAAK,WAAW,KAAK,SAAS,KAAK,IAAI;AAAA,IAC/C;AAEA,SAAK,aAAa;AAElB,QAAI,KAAK,MAAM,SAAS,EAAG,MAAK,aAAa;AAAA,EAC/C;AAAA,EAEA,MAAc,WAAW,SAAiB,MAAc;AACtD,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AAEvB,QAAI;AACF,YAAM,WAAW,MAAM,KAAK,OAAO,MAAM,OAAO;AAAA,QAC9C;AAAA,UACE,OAAO,KAAK,QAAQ,SAASF;AAAA,UAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO;AAAA,UACP,iBAAiB;AAAA,UACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,UACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,QAC5D;AAAA,QACA,EAAE,QAAQ,gBAAgB,OAAO;AAAA,MACnC;AAEA,UAAI,CAAC,SAAS,KAAM;AACpB,WAAK,IAAI,kBAAkB,IAAI,GAAG;AAElC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,YAAY,KAAK,SAAS;AAC5B,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,KAAK,SAAS,MAAM;AAAA,MAClD;AAAA,IACF,SAAS,OAAO;AACd,UAAI,gBAAgB,OAAO,QAAS;AACpC,WAAK,IAAI,8BAA8B,KAAK;AAC5C,WAAK,KAAK,UAAU,CAAC,IAAI,CAAC;AAAA,IAC5B,UAAE;AACA,UAAI,KAAK,oBAAoB,iBAAiB;AAC5C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AACF;","names":["DEFAULT_MODEL","DEFAULT_RETRY_DELAY","OpenAI","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
|
1
|
+
{"version":3,"sources":["../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n let hasAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n // Ignore extra outputs, we keep the first answer only\n if (hasAnswer) break\n\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n // Keep the first answer only.\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n if (hasAnswer) break\n\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n hasAnswer = true\n // Keep reading events instead of returning here: the API streams\n // one output item at a time, so a function call the model made\n // alongside its answer is only announced after the text is done.\n // Returning dropped every tool call that came with an answer.\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again when the model called a tool without answering, so the\n // turn still produces something to say\n if (!skipAnswer && !hasAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\nconst MAX_IN_FLIGHT = 2 // Requests asked for at once, current one included\n// Audio held back at the start of an utterance, in bytes of the output format\n// (16 bits, 16 kHz, so 32 bytes per millisecond).\n//\n// The endpoint answers with a burst of a hundred milliseconds or so, then goes\n// quiet while it generates the rest. Forwarding that burst as it arrives means\n// the browser starts playing a syllable it cannot continue, and the hole lands\n// inside the first word. Holding the opening back until there is enough of it\n// to cover the pause costs the same time either way, and spends it before the\n// first syllable rather than inside it.\nconst OPENING_CUSHION = 300 * 32\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\n/** A sentence whose synthesis has been asked for and not yet spoken. */\ninterface PendingItem extends QueueItem {\n controller: AbortController\n response: Promise<{ body: ReadableStream<Uint8Array> | null } | null>\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n // Bumped by every speak() and every cancel(), so a call claimed late can tell\n // whether it is still the one that should be heard. Kept apart from\n // `counter`, which numbers the synthesis calls and must only move when there\n // is something to synthesize.\n private generation = 0\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = [] // Sentences whose synthesis has not started\n private pending: PendingItem[] = [] // Requests in flight, in speaking order\n private cushion: Buffer[] = [] // Opening audio held back, see OPENING_CUSHION\n private cushionBytes = 0\n private processing = false\n // More than one request is in flight at a time, so they are aborted as a set.\n private abortControllers = new Set<AbortController>()\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n const generation = ++this.generation\n let counter = 0\n\n // Claiming the call is deferred until there is something to say.\n //\n // Taking the next number right away would drop the utterance still being\n // spoken, since both the queue and the streaming reader skip anything\n // stamped with an older one. A stream that never carries a word, which is\n // what an answer skipped by a tool or by onBeforeAnswer hands over, would\n // then cut the assistant off and throw away the sentences still queued.\n const claimCall = () => {\n if (counter) return true\n // Cancelled, or superseded by another speak(), before the first word.\n if (this.generation !== generation) return false\n this.counter++\n counter = this.counter\n this.buffer = ''\n this.cushion = []\n this.cushionBytes = 0\n return true\n }\n\n textStream.on('data', (chunk: Buffer) => {\n if (!claimCall()) return\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n // Nothing was ever said, so there is nothing left to flush.\n if (!counter || counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.generation++\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.pending = []\n this.cushion = []\n this.cushionBytes = 0\n this.abortControllers.forEach((controller) => controller.abort())\n this.abortControllers.clear()\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.pump()\n }\n\n /**\n * Starts what can be started, then speaks what is ready.\n *\n * The endpoint takes a few hundred milliseconds before its first byte, and\n * waiting for that between two sentences leaves a silence in the middle of\n * her voice. It is loudest at the very start, where a greeting is often one\n * short sentence: a syllable, a hole, then the rest of the answer. Asking for\n * a sentence the moment it is complete, rather than when its turn to be\n * spoken comes, hides that latency behind the audio already playing.\n *\n * Two requests at a time is enough to cover the gap, and it keeps a long\n * answer from opening one request per sentence all at once.\n */\n private pump() {\n while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n this.pending.push(this.request(item))\n }\n this.drain()\n }\n\n /** Speaks the started requests in the order they were queued. */\n private async drain() {\n if (this.processing) return\n this.processing = true\n\n while (this.pending.length > 0) {\n const item = this.pending.shift()!\n // A slot just freed, so the sentence after the next one can start now\n this.pump()\n await this.speakItem(item)\n }\n\n this.processing = false\n // Nothing left to speak: an utterance shorter than the cushion is released\n // here, since there is no longer anything coming to fill it up.\n this.flushCushion()\n // Sentences may have arrived right as we exited the loop\n if (this.pending.length > 0 || this.queue.length > 0) this.pump()\n }\n\n /** Asks for one sentence, without waiting for the answer. */\n private request(item: QueueItem): PendingItem {\n const controller = new AbortController()\n this.abortControllers.add(controller)\n\n const response = this.openai.audio.speech\n .create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: item.text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: controller.signal }\n )\n .catch((error) => {\n if (!controller.signal.aborted) {\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [item.text])\n }\n return null\n })\n\n return { ...item, controller, response }\n }\n\n /** Holds the opening of an utterance back, then lets the rest through. */\n private emitAudio(chunk: Buffer) {\n if (this.cushionBytes >= OPENING_CUSHION) {\n this.emit('Audio', chunk)\n return\n }\n this.cushion.push(chunk)\n this.cushionBytes += chunk.length\n if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion()\n }\n\n private flushCushion() {\n if (this.cushion.length === 0) return\n const held = this.cushion\n this.cushion = []\n for (const chunk of held) this.emit('Audio', chunk)\n }\n\n /** Emits one sentence, whose request was started earlier. */\n private async speakItem(item: PendingItem) {\n try {\n const response = await item.response\n if (!response?.body) return\n if (item.counter !== this.counter) return\n this.log(`Synthesizing: \"${item.text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (item.counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emitAudio(output)\n }\n } finally {\n this.abortControllers.delete(item.controller)\n }\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,OAAO,YAAY;AAEnB,OAAO,KAAK,oBAAoB;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AACjB,UAAI,YAAY;AAGhB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AAEH,gBAAI,UAAW;AAEf,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AAMH,gBAAI,UAAW;AAEf,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,wBAAY;AAKZ;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAI9C,UAAI,CAAC,cAAc,CAAC,WAAW;AAC7B,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACnNA,SAAS,WAAW;AAEpB,OAAO,eAAe;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMA,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,IAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASD;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,SAAS,WAAW;AACpB,OAAOC,aAAY;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAC3B,IAAM,gBAAgB;AAUtB,IAAM,kBAAkB,MAAM;AAavB,IAAM,YAAN,cAAwB,IAAI;AAAA,EAiBjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAf7B,SAAQ,UAAU;AAKlB;AAAA;AAAA;AAAA;AAAA;AAAA,SAAQ,aAAa;AACrB,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B;AAAA,SAAQ,UAAyB,CAAC;AAClC;AAAA,SAAQ,UAAoB,CAAC;AAC7B;AAAA,SAAQ,eAAe;AACvB,SAAQ,aAAa;AAErB;AAAA,SAAQ,mBAAmB,oBAAI,IAAqB;AAIlD,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAIC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,UAAM,aAAa,EAAE,KAAK;AAC1B,QAAI,UAAU;AASd,UAAM,YAAY,MAAM;AACtB,UAAI,QAAS,QAAO;AAEpB,UAAI,KAAK,eAAe,WAAY,QAAO;AAC3C,WAAK;AACL,gBAAU,KAAK;AACf,WAAK,SAAS;AACd,WAAK,UAAU,CAAC;AAChB,WAAK,eAAe;AACpB,aAAO;AAAA,IACT;AAEA,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,CAAC,UAAU,EAAG;AAClB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AAEzB,UAAI,CAAC,WAAW,YAAY,KAAK,QAAS;AAC1C,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK;AACL,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,UAAU,CAAC;AAChB,SAAK,UAAU,CAAC;AAChB,SAAK,eAAe;AACpB,SAAK,iBAAiB,QAAQ,CAAC,eAAe,WAAW,MAAM,CAAC;AAChE,SAAK,iBAAiB,MAAM;AAAA,EAC9B;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeQ,OAAO;AACb,WAAO,KAAK,QAAQ,SAAS,iBAAiB,KAAK,MAAM,SAAS,GAAG;AACnE,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,QAAQ,KAAK,KAAK,QAAQ,IAAI,CAAC;AAAA,IACtC;AACA,SAAK,MAAM;AAAA,EACb;AAAA;AAAA,EAGA,MAAc,QAAQ;AACpB,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,QAAQ,SAAS,GAAG;AAC9B,YAAM,OAAO,KAAK,QAAQ,MAAM;AAEhC,WAAK,KAAK;AACV,YAAM,KAAK,UAAU,IAAI;AAAA,IAC3B;AAEA,SAAK,aAAa;AAGlB,SAAK,aAAa;AAElB,QAAI,KAAK,QAAQ,SAAS,KAAK,KAAK,MAAM,SAAS,EAAG,MAAK,KAAK;AAAA,EAClE;AAAA;AAAA,EAGQ,QAAQ,MAA8B;AAC5C,UAAM,aAAa,IAAI,gBAAgB;AACvC,SAAK,iBAAiB,IAAI,UAAU;AAEpC,UAAM,WAAW,KAAK,OAAO,MAAM,OAChC;AAAA,MACC;AAAA,QACE,OAAO,KAAK,QAAQ,SAASF;AAAA,QAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,OAAO,KAAK;AAAA,QACZ,iBAAiB;AAAA,QACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,QACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC5D;AAAA,MACA,EAAE,QAAQ,WAAW,OAAO;AAAA,IAC9B,EACC,MAAM,CAAC,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,SAAS;AAC9B,aAAK,IAAI,8BAA8B,KAAK;AAC5C,aAAK,KAAK,UAAU,CAAC,KAAK,IAAI,CAAC;AAAA,MACjC;AACA,aAAO;AAAA,IACT,CAAC;AAEH,WAAO,EAAE,GAAG,MAAM,YAAY,SAAS;AAAA,EACzC;AAAA;AAAA,EAGQ,UAAU,OAAe;AAC/B,QAAI,KAAK,gBAAgB,iBAAiB;AACxC,WAAK,KAAK,SAAS,KAAK;AACxB;AAAA,IACF;AACA,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,gBAAgB,MAAM;AAC3B,QAAI,KAAK,gBAAgB,gBAAiB,MAAK,aAAa;AAAA,EAC9D;AAAA,EAEQ,eAAe;AACrB,QAAI,KAAK,QAAQ,WAAW,EAAG;AAC/B,UAAM,OAAO,KAAK;AAClB,SAAK,UAAU,CAAC;AAChB,eAAW,SAAS,KAAM,MAAK,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA;AAAA,EAGA,MAAc,UAAU,MAAmB;AACzC,QAAI;AACF,YAAM,WAAW,MAAM,KAAK;AAC5B,UAAI,CAAC,UAAU,KAAM;AACrB,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,IAAI,kBAAkB,KAAK,IAAI,GAAG;AAEvC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,KAAK,YAAY,KAAK,SAAS;AACjC,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,UAAU,MAAM;AAAA,MAC9C;AAAA,IACF,UAAE;AACA,WAAK,iBAAiB,OAAO,KAAK,UAAU;AAAA,IAC9C;AAAA,EACF;AACF;","names":["DEFAULT_MODEL","DEFAULT_RETRY_DELAY","OpenAI","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|