@micdrop/openai 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.mts +28 -3
- package/dist/index.d.ts +28 -3
- package/dist/index.js +119 -41
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +119 -41
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -102,4 +102,4 @@ MIT
|
|
|
102
102
|
|
|
103
103
|
## Author
|
|
104
104
|
|
|
105
|
-
Originally developed for [Raconte.ai](https://www.raconte.ai) and open sourced by [
|
|
105
|
+
Originally developed for [Raconte.ai](https://www.raconte.ai), created and open sourced by [Godefroy de Compreignac](https://github.com/Godefroy)
|
package/dist/index.d.mts
CHANGED
|
@@ -80,17 +80,42 @@ declare class OpenaiTTS extends TTS {
|
|
|
80
80
|
private readonly options;
|
|
81
81
|
private openai;
|
|
82
82
|
private counter;
|
|
83
|
+
private generation;
|
|
83
84
|
private buffer;
|
|
84
85
|
private queue;
|
|
86
|
+
private pending;
|
|
87
|
+
private cushion;
|
|
88
|
+
private cushionBytes;
|
|
85
89
|
private processing;
|
|
86
|
-
private
|
|
90
|
+
private abortControllers;
|
|
87
91
|
constructor(options: OpenaiTTSOptions);
|
|
88
92
|
speak(textStream: Readable): void;
|
|
89
93
|
cancel(): void;
|
|
90
94
|
private flushSentences;
|
|
91
95
|
private enqueue;
|
|
92
|
-
|
|
93
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Starts what can be started, then speaks what is ready.
|
|
98
|
+
*
|
|
99
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
100
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
101
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
102
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
103
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
104
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
105
|
+
*
|
|
106
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
107
|
+
* answer from opening one request per sentence all at once.
|
|
108
|
+
*/
|
|
109
|
+
private pump;
|
|
110
|
+
/** Speaks the started requests in the order they were queued. */
|
|
111
|
+
private drain;
|
|
112
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
113
|
+
private request;
|
|
114
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
115
|
+
private emitAudio;
|
|
116
|
+
private flushCushion;
|
|
117
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
118
|
+
private speakItem;
|
|
94
119
|
}
|
|
95
120
|
|
|
96
121
|
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
package/dist/index.d.ts
CHANGED
|
@@ -80,17 +80,42 @@ declare class OpenaiTTS extends TTS {
|
|
|
80
80
|
private readonly options;
|
|
81
81
|
private openai;
|
|
82
82
|
private counter;
|
|
83
|
+
private generation;
|
|
83
84
|
private buffer;
|
|
84
85
|
private queue;
|
|
86
|
+
private pending;
|
|
87
|
+
private cushion;
|
|
88
|
+
private cushionBytes;
|
|
85
89
|
private processing;
|
|
86
|
-
private
|
|
90
|
+
private abortControllers;
|
|
87
91
|
constructor(options: OpenaiTTSOptions);
|
|
88
92
|
speak(textStream: Readable): void;
|
|
89
93
|
cancel(): void;
|
|
90
94
|
private flushSentences;
|
|
91
95
|
private enqueue;
|
|
92
|
-
|
|
93
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Starts what can be started, then speaks what is ready.
|
|
98
|
+
*
|
|
99
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
100
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
101
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
102
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
103
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
104
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
105
|
+
*
|
|
106
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
107
|
+
* answer from opening one request per sentence all at once.
|
|
108
|
+
*/
|
|
109
|
+
private pump;
|
|
110
|
+
/** Speaks the started requests in the order they were queued. */
|
|
111
|
+
private drain;
|
|
112
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
113
|
+
private request;
|
|
114
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
115
|
+
private emitAudio;
|
|
116
|
+
private flushCushion;
|
|
117
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
118
|
+
private speakItem;
|
|
94
119
|
}
|
|
95
120
|
|
|
96
121
|
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
package/dist/index.js
CHANGED
|
@@ -71,10 +71,12 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
71
71
|
{ signal }
|
|
72
72
|
);
|
|
73
73
|
let skipAnswer = false;
|
|
74
|
+
let hasAnswer = false;
|
|
74
75
|
for await (const event of response) {
|
|
75
76
|
if (abortController !== this.abortController) return;
|
|
76
77
|
switch (event.type) {
|
|
77
78
|
case "response.output_text.delta":
|
|
79
|
+
if (hasAnswer) break;
|
|
78
80
|
this.log(`Answer chunk: "${event.delta}"`);
|
|
79
81
|
if (extractOptions) {
|
|
80
82
|
if (!extracting) {
|
|
@@ -94,10 +96,11 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
94
96
|
stream.write(event.delta);
|
|
95
97
|
break;
|
|
96
98
|
case "response.output_text.done":
|
|
99
|
+
if (hasAnswer) break;
|
|
97
100
|
const { message, metadata } = this.extract(event.text);
|
|
98
101
|
this.addAssistantMessage(message, metadata);
|
|
99
|
-
|
|
100
|
-
|
|
102
|
+
hasAnswer = true;
|
|
103
|
+
break;
|
|
101
104
|
case "response.output_item.done":
|
|
102
105
|
if (event.item.type === "function_call") {
|
|
103
106
|
const toolCall = event.item;
|
|
@@ -117,7 +120,7 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
117
120
|
}
|
|
118
121
|
}
|
|
119
122
|
if (abortController !== this.abortController) return;
|
|
120
|
-
if (!skipAnswer) {
|
|
123
|
+
if (!skipAnswer && !hasAnswer) {
|
|
121
124
|
await this.generateAnswer(stream, stepCount + 1);
|
|
122
125
|
}
|
|
123
126
|
this.abortController = void 0;
|
|
@@ -445,23 +448,48 @@ var DEFAULT_MODEL3 = "gpt-4o-mini-tts";
|
|
|
445
448
|
var DEFAULT_VOICE = "alloy";
|
|
446
449
|
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
447
450
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
451
|
+
var MAX_IN_FLIGHT = 2;
|
|
452
|
+
var OPENING_CUSHION = 300 * 32;
|
|
448
453
|
var OpenaiTTS = class extends import_server3.TTS {
|
|
449
454
|
constructor(options) {
|
|
450
455
|
super();
|
|
451
456
|
this.options = options;
|
|
452
457
|
this.counter = 0;
|
|
453
458
|
// Identifies the current speak() call
|
|
459
|
+
// Bumped by every speak() and every cancel(), so a call claimed late can tell
|
|
460
|
+
// whether it is still the one that should be heard. Kept apart from
|
|
461
|
+
// `counter`, which numbers the synthesis calls and must only move when there
|
|
462
|
+
// is something to synthesize.
|
|
463
|
+
this.generation = 0;
|
|
454
464
|
this.buffer = "";
|
|
455
465
|
// Incomplete sentence waiting for more text
|
|
456
466
|
this.queue = [];
|
|
467
|
+
// Sentences whose synthesis has not started
|
|
468
|
+
this.pending = [];
|
|
469
|
+
// Requests in flight, in speaking order
|
|
470
|
+
this.cushion = [];
|
|
471
|
+
// Opening audio held back, see OPENING_CUSHION
|
|
472
|
+
this.cushionBytes = 0;
|
|
457
473
|
this.processing = false;
|
|
474
|
+
// More than one request is in flight at a time, so they are aborted as a set.
|
|
475
|
+
this.abortControllers = /* @__PURE__ */ new Set();
|
|
458
476
|
this.openai = "openai" in options ? options.openai : new import_openai2.default({ apiKey: options.apiKey });
|
|
459
477
|
}
|
|
460
478
|
speak(textStream) {
|
|
461
|
-
this.
|
|
462
|
-
|
|
463
|
-
|
|
479
|
+
const generation = ++this.generation;
|
|
480
|
+
let counter = 0;
|
|
481
|
+
const claimCall = () => {
|
|
482
|
+
if (counter) return true;
|
|
483
|
+
if (this.generation !== generation) return false;
|
|
484
|
+
this.counter++;
|
|
485
|
+
counter = this.counter;
|
|
486
|
+
this.buffer = "";
|
|
487
|
+
this.cushion = [];
|
|
488
|
+
this.cushionBytes = 0;
|
|
489
|
+
return true;
|
|
490
|
+
};
|
|
464
491
|
textStream.on("data", (chunk) => {
|
|
492
|
+
if (!claimCall()) return;
|
|
465
493
|
if (counter !== this.counter) return;
|
|
466
494
|
this.buffer += chunk.toString("utf-8");
|
|
467
495
|
this.flushSentences(counter, false);
|
|
@@ -470,17 +498,21 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
470
498
|
this.log("Error in text stream", error);
|
|
471
499
|
});
|
|
472
500
|
textStream.on("end", () => {
|
|
473
|
-
if (counter !== this.counter) return;
|
|
501
|
+
if (!counter || counter !== this.counter) return;
|
|
474
502
|
this.flushSentences(counter, true);
|
|
475
503
|
});
|
|
476
504
|
}
|
|
477
505
|
cancel() {
|
|
506
|
+
this.generation++;
|
|
478
507
|
this.log("Cancel");
|
|
479
508
|
this.counter++;
|
|
480
509
|
this.buffer = "";
|
|
481
510
|
this.queue = [];
|
|
482
|
-
this.
|
|
483
|
-
this.
|
|
511
|
+
this.pending = [];
|
|
512
|
+
this.cushion = [];
|
|
513
|
+
this.cushionBytes = 0;
|
|
514
|
+
this.abortControllers.forEach((controller) => controller.abort());
|
|
515
|
+
this.abortControllers.clear();
|
|
484
516
|
}
|
|
485
517
|
// Extract complete sentences from the buffer and enqueue them.
|
|
486
518
|
// On `end`, whatever remains in the buffer is flushed as a last sentence.
|
|
@@ -503,36 +535,88 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
503
535
|
}
|
|
504
536
|
enqueue(counter, text) {
|
|
505
537
|
this.queue.push({ counter, text });
|
|
506
|
-
this.
|
|
538
|
+
this.pump();
|
|
507
539
|
}
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
540
|
+
/**
|
|
541
|
+
* Starts what can be started, then speaks what is ready.
|
|
542
|
+
*
|
|
543
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
544
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
545
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
546
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
547
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
548
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
549
|
+
*
|
|
550
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
551
|
+
* answer from opening one request per sentence all at once.
|
|
552
|
+
*/
|
|
553
|
+
pump() {
|
|
554
|
+
while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {
|
|
512
555
|
const item = this.queue.shift();
|
|
513
556
|
if (item.counter !== this.counter) continue;
|
|
514
|
-
|
|
557
|
+
this.pending.push(this.request(item));
|
|
558
|
+
}
|
|
559
|
+
this.drain();
|
|
560
|
+
}
|
|
561
|
+
/** Speaks the started requests in the order they were queued. */
|
|
562
|
+
async drain() {
|
|
563
|
+
if (this.processing) return;
|
|
564
|
+
this.processing = true;
|
|
565
|
+
while (this.pending.length > 0) {
|
|
566
|
+
const item = this.pending.shift();
|
|
567
|
+
this.pump();
|
|
568
|
+
await this.speakItem(item);
|
|
515
569
|
}
|
|
516
570
|
this.processing = false;
|
|
517
|
-
|
|
571
|
+
this.flushCushion();
|
|
572
|
+
if (this.pending.length > 0 || this.queue.length > 0) this.pump();
|
|
518
573
|
}
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
574
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
575
|
+
request(item) {
|
|
576
|
+
const controller = new AbortController();
|
|
577
|
+
this.abortControllers.add(controller);
|
|
578
|
+
const response = this.openai.audio.speech.create(
|
|
579
|
+
{
|
|
580
|
+
model: this.options.model || DEFAULT_MODEL3,
|
|
581
|
+
voice: this.options.voice || DEFAULT_VOICE,
|
|
582
|
+
input: item.text,
|
|
583
|
+
response_format: "pcm",
|
|
584
|
+
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
585
|
+
...this.options.speed ? { speed: this.options.speed } : {}
|
|
586
|
+
},
|
|
587
|
+
{ signal: controller.signal }
|
|
588
|
+
).catch((error) => {
|
|
589
|
+
if (!controller.signal.aborted) {
|
|
590
|
+
this.log("Error synthesizing speech:", error);
|
|
591
|
+
this.emit("Failed", [item.text]);
|
|
592
|
+
}
|
|
593
|
+
return null;
|
|
594
|
+
});
|
|
595
|
+
return { ...item, controller, response };
|
|
596
|
+
}
|
|
597
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
598
|
+
emitAudio(chunk) {
|
|
599
|
+
if (this.cushionBytes >= OPENING_CUSHION) {
|
|
600
|
+
this.emit("Audio", chunk);
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
this.cushion.push(chunk);
|
|
604
|
+
this.cushionBytes += chunk.length;
|
|
605
|
+
if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion();
|
|
606
|
+
}
|
|
607
|
+
flushCushion() {
|
|
608
|
+
if (this.cushion.length === 0) return;
|
|
609
|
+
const held = this.cushion;
|
|
610
|
+
this.cushion = [];
|
|
611
|
+
for (const chunk of held) this.emit("Audio", chunk);
|
|
612
|
+
}
|
|
613
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
614
|
+
async speakItem(item) {
|
|
522
615
|
try {
|
|
523
|
-
const response = await
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
input: text,
|
|
528
|
-
response_format: "pcm",
|
|
529
|
-
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
530
|
-
...this.options.speed ? { speed: this.options.speed } : {}
|
|
531
|
-
},
|
|
532
|
-
{ signal: abortController.signal }
|
|
533
|
-
);
|
|
534
|
-
if (!response.body) return;
|
|
535
|
-
this.log(`Synthesizing: "${text}"`);
|
|
616
|
+
const response = await item.response;
|
|
617
|
+
if (!response?.body) return;
|
|
618
|
+
if (item.counter !== this.counter) return;
|
|
619
|
+
this.log(`Synthesizing: "${item.text}"`);
|
|
536
620
|
const resampler = new Pcm16Resampler(
|
|
537
621
|
OPENAI_SAMPLE_RATE2,
|
|
538
622
|
OUTPUT_SAMPLE_RATE
|
|
@@ -541,21 +625,15 @@ var OpenaiTTS = class extends import_server3.TTS {
|
|
|
541
625
|
while (true) {
|
|
542
626
|
const { done, value } = await reader.read();
|
|
543
627
|
if (done) break;
|
|
544
|
-
if (counter !== this.counter) {
|
|
628
|
+
if (item.counter !== this.counter) {
|
|
545
629
|
await reader.cancel();
|
|
546
630
|
break;
|
|
547
631
|
}
|
|
548
632
|
const output = resampler.process(Buffer.from(value));
|
|
549
|
-
if (output.length > 0) this.
|
|
633
|
+
if (output.length > 0) this.emitAudio(output);
|
|
550
634
|
}
|
|
551
|
-
} catch (error) {
|
|
552
|
-
if (abortController.signal.aborted) return;
|
|
553
|
-
this.log("Error synthesizing speech:", error);
|
|
554
|
-
this.emit("Failed", [text]);
|
|
555
635
|
} finally {
|
|
556
|
-
|
|
557
|
-
this.abortController = void 0;
|
|
558
|
-
}
|
|
636
|
+
this.abortControllers.delete(item.controller);
|
|
559
637
|
}
|
|
560
638
|
}
|
|
561
639
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\nexport * from './OpenaiTTS'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n this.abortController = undefined\n // Stop query now that we have a complete answer\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n return\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again in case of tool call\n if (!skipAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = []\n private processing = false\n private abortController?: AbortController\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n this.counter++\n const counter = this.counter\n this.buffer = ''\n\n textStream.on('data', (chunk: Buffer) => {\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n if (counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.abortController?.abort()\n this.abortController = undefined\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.processQueue()\n }\n\n private async processQueue() {\n if (this.processing) return\n this.processing = true\n\n while (this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n await this.synthesize(item.counter, item.text)\n }\n\n this.processing = false\n // Items may have been enqueued right as we exited the loop\n if (this.queue.length > 0) this.processQueue()\n }\n\n private async synthesize(counter: number, text: string) {\n const abortController = new AbortController()\n this.abortController = abortController\n\n try {\n const response = await this.openai.audio.speech.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: abortController.signal }\n )\n\n if (!response.body) return\n this.log(`Synthesizing: \"${text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emit('Audio', output)\n }\n } catch (error) {\n if (abortController.signal.aborted) return\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [text])\n } finally {\n if (this.abortController === abortController) {\n this.abortController = undefined\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,oBAAmB;AAEnB,iBAAgC;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AAGjB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,iBAAK,kBAAkB;AAMvB;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAG9C,UAAI,CAAC,YAAY;AACf,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAA,QAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACxMA,IAAAC,iBAAoB;AAEpB,gBAAsB;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMC,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASF;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI,UAAAE;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,IAAAE,iBAAoB;AACpB,IAAAC,iBAAmB;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAOpB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EAQjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAN7B,SAAQ,UAAU;AAClB;AAAA,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B,SAAQ,aAAa;AAKnB,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,eAAAC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,SAAK;AACL,UAAM,UAAU,KAAK;AACrB,SAAK,SAAS;AAEd,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AACzB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,iBAAiB,MAAM;AAC5B,SAAK,kBAAkB;AAAA,EACzB;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAc,eAAe;AAC3B,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,MAAM,SAAS,GAAG;AAC5B,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,YAAM,KAAK,WAAW,KAAK,SAAS,KAAK,IAAI;AAAA,IAC/C;AAEA,SAAK,aAAa;AAElB,QAAI,KAAK,MAAM,SAAS,EAAG,MAAK,aAAa;AAAA,EAC/C;AAAA,EAEA,MAAc,WAAW,SAAiB,MAAc;AACtD,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AAEvB,QAAI;AACF,YAAM,WAAW,MAAM,KAAK,OAAO,MAAM,OAAO;AAAA,QAC9C;AAAA,UACE,OAAO,KAAK,QAAQ,SAASF;AAAA,UAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO;AAAA,UACP,iBAAiB;AAAA,UACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,UACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,QAC5D;AAAA,QACA,EAAE,QAAQ,gBAAgB,OAAO;AAAA,MACnC;AAEA,UAAI,CAAC,SAAS,KAAM;AACpB,WAAK,IAAI,kBAAkB,IAAI,GAAG;AAElC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,YAAY,KAAK,SAAS;AAC5B,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,KAAK,SAAS,MAAM;AAAA,MAClD;AAAA,IACF,SAAS,OAAO;AACd,UAAI,gBAAgB,OAAO,QAAS;AACpC,WAAK,IAAI,8BAA8B,KAAK;AAC5C,WAAK,KAAK,UAAU,CAAC,IAAI,CAAC;AAAA,IAC5B,UAAE;AACA,UAAI,KAAK,oBAAoB,iBAAiB;AAC5C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AACF;","names":["OpenAI","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","WebSocket","import_server","import_openai","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["export * from './OpenaiAgent'\nexport * from './OpenaiSTT'\nexport * from './OpenaiTTS'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n let hasAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n // Ignore extra outputs, we keep the first answer only\n if (hasAnswer) break\n\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n // Keep the first answer only.\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n if (hasAnswer) break\n\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n hasAnswer = true\n // Keep reading events instead of returning here: the API streams\n // one output item at a time, so a function call the model made\n // alongside its answer is only announced after the text is done.\n // Returning dropped every tool call that came with an answer.\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again when the model called a tool without answering, so the\n // turn still produces something to say\n if (!skipAnswer && !hasAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\nconst MAX_IN_FLIGHT = 2 // Requests asked for at once, current one included\n// Audio held back at the start of an utterance, in bytes of the output format\n// (16 bits, 16 kHz, so 32 bytes per millisecond).\n//\n// The endpoint answers with a burst of a hundred milliseconds or so, then goes\n// quiet while it generates the rest. Forwarding that burst as it arrives means\n// the browser starts playing a syllable it cannot continue, and the hole lands\n// inside the first word. Holding the opening back until there is enough of it\n// to cover the pause costs the same time either way, and spends it before the\n// first syllable rather than inside it.\nconst OPENING_CUSHION = 300 * 32\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\n/** A sentence whose synthesis has been asked for and not yet spoken. */\ninterface PendingItem extends QueueItem {\n controller: AbortController\n response: Promise<{ body: ReadableStream<Uint8Array> | null } | null>\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n // Bumped by every speak() and every cancel(), so a call claimed late can tell\n // whether it is still the one that should be heard. Kept apart from\n // `counter`, which numbers the synthesis calls and must only move when there\n // is something to synthesize.\n private generation = 0\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = [] // Sentences whose synthesis has not started\n private pending: PendingItem[] = [] // Requests in flight, in speaking order\n private cushion: Buffer[] = [] // Opening audio held back, see OPENING_CUSHION\n private cushionBytes = 0\n private processing = false\n // More than one request is in flight at a time, so they are aborted as a set.\n private abortControllers = new Set<AbortController>()\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n const generation = ++this.generation\n let counter = 0\n\n // Claiming the call is deferred until there is something to say.\n //\n // Taking the next number right away would drop the utterance still being\n // spoken, since both the queue and the streaming reader skip anything\n // stamped with an older one. A stream that never carries a word, which is\n // what an answer skipped by a tool or by onBeforeAnswer hands over, would\n // then cut the assistant off and throw away the sentences still queued.\n const claimCall = () => {\n if (counter) return true\n // Cancelled, or superseded by another speak(), before the first word.\n if (this.generation !== generation) return false\n this.counter++\n counter = this.counter\n this.buffer = ''\n this.cushion = []\n this.cushionBytes = 0\n return true\n }\n\n textStream.on('data', (chunk: Buffer) => {\n if (!claimCall()) return\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n // Nothing was ever said, so there is nothing left to flush.\n if (!counter || counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.generation++\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.pending = []\n this.cushion = []\n this.cushionBytes = 0\n this.abortControllers.forEach((controller) => controller.abort())\n this.abortControllers.clear()\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.pump()\n }\n\n /**\n * Starts what can be started, then speaks what is ready.\n *\n * The endpoint takes a few hundred milliseconds before its first byte, and\n * waiting for that between two sentences leaves a silence in the middle of\n * her voice. It is loudest at the very start, where a greeting is often one\n * short sentence: a syllable, a hole, then the rest of the answer. Asking for\n * a sentence the moment it is complete, rather than when its turn to be\n * spoken comes, hides that latency behind the audio already playing.\n *\n * Two requests at a time is enough to cover the gap, and it keeps a long\n * answer from opening one request per sentence all at once.\n */\n private pump() {\n while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n this.pending.push(this.request(item))\n }\n this.drain()\n }\n\n /** Speaks the started requests in the order they were queued. */\n private async drain() {\n if (this.processing) return\n this.processing = true\n\n while (this.pending.length > 0) {\n const item = this.pending.shift()!\n // A slot just freed, so the sentence after the next one can start now\n this.pump()\n await this.speakItem(item)\n }\n\n this.processing = false\n // Nothing left to speak: an utterance shorter than the cushion is released\n // here, since there is no longer anything coming to fill it up.\n this.flushCushion()\n // Sentences may have arrived right as we exited the loop\n if (this.pending.length > 0 || this.queue.length > 0) this.pump()\n }\n\n /** Asks for one sentence, without waiting for the answer. */\n private request(item: QueueItem): PendingItem {\n const controller = new AbortController()\n this.abortControllers.add(controller)\n\n const response = this.openai.audio.speech\n .create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: item.text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: controller.signal }\n )\n .catch((error) => {\n if (!controller.signal.aborted) {\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [item.text])\n }\n return null\n })\n\n return { ...item, controller, response }\n }\n\n /** Holds the opening of an utterance back, then lets the rest through. */\n private emitAudio(chunk: Buffer) {\n if (this.cushionBytes >= OPENING_CUSHION) {\n this.emit('Audio', chunk)\n return\n }\n this.cushion.push(chunk)\n this.cushionBytes += chunk.length\n if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion()\n }\n\n private flushCushion() {\n if (this.cushion.length === 0) return\n const held = this.cushion\n this.cushion = []\n for (const chunk of held) this.emit('Audio', chunk)\n }\n\n /** Emits one sentence, whose request was started earlier. */\n private async speakItem(item: PendingItem) {\n try {\n const response = await item.response\n if (!response?.body) return\n if (item.counter !== this.counter) return\n this.log(`Synthesizing: \"${item.text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (item.counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emitAudio(output)\n }\n } finally {\n this.abortControllers.delete(item.controller)\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,oBAAmB;AAEnB,iBAAgC;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,oBAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,cAAAA,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AACjB,UAAI,YAAY;AAGhB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AAEH,gBAAI,UAAW;AAEf,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AAMH,gBAAI,UAAW;AAEf,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,wBAAY;AAKZ;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAI9C,UAAI,CAAC,cAAc,CAAC,WAAW;AAC7B,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,cAAAA,QAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACnNA,IAAAC,iBAAoB;AAEpB,gBAAsB;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMC,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASF;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI,UAAAE;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,IAAAE,iBAAoB;AACpB,IAAAC,iBAAmB;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAC3B,IAAM,gBAAgB;AAUtB,IAAM,kBAAkB,MAAM;AAavB,IAAM,YAAN,cAAwB,mBAAI;AAAA,EAiBjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAf7B,SAAQ,UAAU;AAKlB;AAAA;AAAA;AAAA;AAAA;AAAA,SAAQ,aAAa;AACrB,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B;AAAA,SAAQ,UAAyB,CAAC;AAClC;AAAA,SAAQ,UAAoB,CAAC;AAC7B;AAAA,SAAQ,eAAe;AACvB,SAAQ,aAAa;AAErB;AAAA,SAAQ,mBAAmB,oBAAI,IAAqB;AAIlD,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,eAAAC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,UAAM,aAAa,EAAE,KAAK;AAC1B,QAAI,UAAU;AASd,UAAM,YAAY,MAAM;AACtB,UAAI,QAAS,QAAO;AAEpB,UAAI,KAAK,eAAe,WAAY,QAAO;AAC3C,WAAK;AACL,gBAAU,KAAK;AACf,WAAK,SAAS;AACd,WAAK,UAAU,CAAC;AAChB,WAAK,eAAe;AACpB,aAAO;AAAA,IACT;AAEA,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,CAAC,UAAU,EAAG;AAClB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AAEzB,UAAI,CAAC,WAAW,YAAY,KAAK,QAAS;AAC1C,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK;AACL,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,UAAU,CAAC;AAChB,SAAK,UAAU,CAAC;AAChB,SAAK,eAAe;AACpB,SAAK,iBAAiB,QAAQ,CAAC,eAAe,WAAW,MAAM,CAAC;AAChE,SAAK,iBAAiB,MAAM;AAAA,EAC9B;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeQ,OAAO;AACb,WAAO,KAAK,QAAQ,SAAS,iBAAiB,KAAK,MAAM,SAAS,GAAG;AACnE,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,QAAQ,KAAK,KAAK,QAAQ,IAAI,CAAC;AAAA,IACtC;AACA,SAAK,MAAM;AAAA,EACb;AAAA;AAAA,EAGA,MAAc,QAAQ;AACpB,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,QAAQ,SAAS,GAAG;AAC9B,YAAM,OAAO,KAAK,QAAQ,MAAM;AAEhC,WAAK,KAAK;AACV,YAAM,KAAK,UAAU,IAAI;AAAA,IAC3B;AAEA,SAAK,aAAa;AAGlB,SAAK,aAAa;AAElB,QAAI,KAAK,QAAQ,SAAS,KAAK,KAAK,MAAM,SAAS,EAAG,MAAK,KAAK;AAAA,EAClE;AAAA;AAAA,EAGQ,QAAQ,MAA8B;AAC5C,UAAM,aAAa,IAAI,gBAAgB;AACvC,SAAK,iBAAiB,IAAI,UAAU;AAEpC,UAAM,WAAW,KAAK,OAAO,MAAM,OAChC;AAAA,MACC;AAAA,QACE,OAAO,KAAK,QAAQ,SAASF;AAAA,QAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,OAAO,KAAK;AAAA,QACZ,iBAAiB;AAAA,QACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,QACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC5D;AAAA,MACA,EAAE,QAAQ,WAAW,OAAO;AAAA,IAC9B,EACC,MAAM,CAAC,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,SAAS;AAC9B,aAAK,IAAI,8BAA8B,KAAK;AAC5C,aAAK,KAAK,UAAU,CAAC,KAAK,IAAI,CAAC;AAAA,MACjC;AACA,aAAO;AAAA,IACT,CAAC;AAEH,WAAO,EAAE,GAAG,MAAM,YAAY,SAAS;AAAA,EACzC;AAAA;AAAA,EAGQ,UAAU,OAAe;AAC/B,QAAI,KAAK,gBAAgB,iBAAiB;AACxC,WAAK,KAAK,SAAS,KAAK;AACxB;AAAA,IACF;AACA,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,gBAAgB,MAAM;AAC3B,QAAI,KAAK,gBAAgB,gBAAiB,MAAK,aAAa;AAAA,EAC9D;AAAA,EAEQ,eAAe;AACrB,QAAI,KAAK,QAAQ,WAAW,EAAG;AAC/B,UAAM,OAAO,KAAK;AAClB,SAAK,UAAU,CAAC;AAChB,eAAW,SAAS,KAAM,MAAK,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA;AAAA,EAGA,MAAc,UAAU,MAAmB;AACzC,QAAI;AACF,YAAM,WAAW,MAAM,KAAK;AAC5B,UAAI,CAAC,UAAU,KAAM;AACrB,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,IAAI,kBAAkB,KAAK,IAAI,GAAG;AAEvC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,KAAK,YAAY,KAAK,SAAS;AACjC,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,UAAU,MAAM;AAAA,MAC9C;AAAA,IACF,UAAE;AACA,WAAK,iBAAiB,OAAO,KAAK,UAAU;AAAA,IAC9C;AAAA,EACF;AACF;","names":["OpenAI","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","WebSocket","import_server","import_openai","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
package/dist/index.mjs
CHANGED
|
@@ -33,10 +33,12 @@ var OpenaiAgent = class extends Agent {
|
|
|
33
33
|
{ signal }
|
|
34
34
|
);
|
|
35
35
|
let skipAnswer = false;
|
|
36
|
+
let hasAnswer = false;
|
|
36
37
|
for await (const event of response) {
|
|
37
38
|
if (abortController !== this.abortController) return;
|
|
38
39
|
switch (event.type) {
|
|
39
40
|
case "response.output_text.delta":
|
|
41
|
+
if (hasAnswer) break;
|
|
40
42
|
this.log(`Answer chunk: "${event.delta}"`);
|
|
41
43
|
if (extractOptions) {
|
|
42
44
|
if (!extracting) {
|
|
@@ -56,10 +58,11 @@ var OpenaiAgent = class extends Agent {
|
|
|
56
58
|
stream.write(event.delta);
|
|
57
59
|
break;
|
|
58
60
|
case "response.output_text.done":
|
|
61
|
+
if (hasAnswer) break;
|
|
59
62
|
const { message, metadata } = this.extract(event.text);
|
|
60
63
|
this.addAssistantMessage(message, metadata);
|
|
61
|
-
|
|
62
|
-
|
|
64
|
+
hasAnswer = true;
|
|
65
|
+
break;
|
|
63
66
|
case "response.output_item.done":
|
|
64
67
|
if (event.item.type === "function_call") {
|
|
65
68
|
const toolCall = event.item;
|
|
@@ -79,7 +82,7 @@ var OpenaiAgent = class extends Agent {
|
|
|
79
82
|
}
|
|
80
83
|
}
|
|
81
84
|
if (abortController !== this.abortController) return;
|
|
82
|
-
if (!skipAnswer) {
|
|
85
|
+
if (!skipAnswer && !hasAnswer) {
|
|
83
86
|
await this.generateAnswer(stream, stepCount + 1);
|
|
84
87
|
}
|
|
85
88
|
this.abortController = void 0;
|
|
@@ -407,23 +410,48 @@ var DEFAULT_MODEL3 = "gpt-4o-mini-tts";
|
|
|
407
410
|
var DEFAULT_VOICE = "alloy";
|
|
408
411
|
var OPENAI_SAMPLE_RATE2 = 24e3;
|
|
409
412
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
413
|
+
var MAX_IN_FLIGHT = 2;
|
|
414
|
+
var OPENING_CUSHION = 300 * 32;
|
|
410
415
|
var OpenaiTTS = class extends TTS {
|
|
411
416
|
constructor(options) {
|
|
412
417
|
super();
|
|
413
418
|
this.options = options;
|
|
414
419
|
this.counter = 0;
|
|
415
420
|
// Identifies the current speak() call
|
|
421
|
+
// Bumped by every speak() and every cancel(), so a call claimed late can tell
|
|
422
|
+
// whether it is still the one that should be heard. Kept apart from
|
|
423
|
+
// `counter`, which numbers the synthesis calls and must only move when there
|
|
424
|
+
// is something to synthesize.
|
|
425
|
+
this.generation = 0;
|
|
416
426
|
this.buffer = "";
|
|
417
427
|
// Incomplete sentence waiting for more text
|
|
418
428
|
this.queue = [];
|
|
429
|
+
// Sentences whose synthesis has not started
|
|
430
|
+
this.pending = [];
|
|
431
|
+
// Requests in flight, in speaking order
|
|
432
|
+
this.cushion = [];
|
|
433
|
+
// Opening audio held back, see OPENING_CUSHION
|
|
434
|
+
this.cushionBytes = 0;
|
|
419
435
|
this.processing = false;
|
|
436
|
+
// More than one request is in flight at a time, so they are aborted as a set.
|
|
437
|
+
this.abortControllers = /* @__PURE__ */ new Set();
|
|
420
438
|
this.openai = "openai" in options ? options.openai : new OpenAI2({ apiKey: options.apiKey });
|
|
421
439
|
}
|
|
422
440
|
speak(textStream) {
|
|
423
|
-
this.
|
|
424
|
-
|
|
425
|
-
|
|
441
|
+
const generation = ++this.generation;
|
|
442
|
+
let counter = 0;
|
|
443
|
+
const claimCall = () => {
|
|
444
|
+
if (counter) return true;
|
|
445
|
+
if (this.generation !== generation) return false;
|
|
446
|
+
this.counter++;
|
|
447
|
+
counter = this.counter;
|
|
448
|
+
this.buffer = "";
|
|
449
|
+
this.cushion = [];
|
|
450
|
+
this.cushionBytes = 0;
|
|
451
|
+
return true;
|
|
452
|
+
};
|
|
426
453
|
textStream.on("data", (chunk) => {
|
|
454
|
+
if (!claimCall()) return;
|
|
427
455
|
if (counter !== this.counter) return;
|
|
428
456
|
this.buffer += chunk.toString("utf-8");
|
|
429
457
|
this.flushSentences(counter, false);
|
|
@@ -432,17 +460,21 @@ var OpenaiTTS = class extends TTS {
|
|
|
432
460
|
this.log("Error in text stream", error);
|
|
433
461
|
});
|
|
434
462
|
textStream.on("end", () => {
|
|
435
|
-
if (counter !== this.counter) return;
|
|
463
|
+
if (!counter || counter !== this.counter) return;
|
|
436
464
|
this.flushSentences(counter, true);
|
|
437
465
|
});
|
|
438
466
|
}
|
|
439
467
|
cancel() {
|
|
468
|
+
this.generation++;
|
|
440
469
|
this.log("Cancel");
|
|
441
470
|
this.counter++;
|
|
442
471
|
this.buffer = "";
|
|
443
472
|
this.queue = [];
|
|
444
|
-
this.
|
|
445
|
-
this.
|
|
473
|
+
this.pending = [];
|
|
474
|
+
this.cushion = [];
|
|
475
|
+
this.cushionBytes = 0;
|
|
476
|
+
this.abortControllers.forEach((controller) => controller.abort());
|
|
477
|
+
this.abortControllers.clear();
|
|
446
478
|
}
|
|
447
479
|
// Extract complete sentences from the buffer and enqueue them.
|
|
448
480
|
// On `end`, whatever remains in the buffer is flushed as a last sentence.
|
|
@@ -465,36 +497,88 @@ var OpenaiTTS = class extends TTS {
|
|
|
465
497
|
}
|
|
466
498
|
enqueue(counter, text) {
|
|
467
499
|
this.queue.push({ counter, text });
|
|
468
|
-
this.
|
|
500
|
+
this.pump();
|
|
469
501
|
}
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
502
|
+
/**
|
|
503
|
+
* Starts what can be started, then speaks what is ready.
|
|
504
|
+
*
|
|
505
|
+
* The endpoint takes a few hundred milliseconds before its first byte, and
|
|
506
|
+
* waiting for that between two sentences leaves a silence in the middle of
|
|
507
|
+
* her voice. It is loudest at the very start, where a greeting is often one
|
|
508
|
+
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
509
|
+
* a sentence the moment it is complete, rather than when its turn to be
|
|
510
|
+
* spoken comes, hides that latency behind the audio already playing.
|
|
511
|
+
*
|
|
512
|
+
* Two requests at a time is enough to cover the gap, and it keeps a long
|
|
513
|
+
* answer from opening one request per sentence all at once.
|
|
514
|
+
*/
|
|
515
|
+
pump() {
|
|
516
|
+
while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {
|
|
474
517
|
const item = this.queue.shift();
|
|
475
518
|
if (item.counter !== this.counter) continue;
|
|
476
|
-
|
|
519
|
+
this.pending.push(this.request(item));
|
|
520
|
+
}
|
|
521
|
+
this.drain();
|
|
522
|
+
}
|
|
523
|
+
/** Speaks the started requests in the order they were queued. */
|
|
524
|
+
async drain() {
|
|
525
|
+
if (this.processing) return;
|
|
526
|
+
this.processing = true;
|
|
527
|
+
while (this.pending.length > 0) {
|
|
528
|
+
const item = this.pending.shift();
|
|
529
|
+
this.pump();
|
|
530
|
+
await this.speakItem(item);
|
|
477
531
|
}
|
|
478
532
|
this.processing = false;
|
|
479
|
-
|
|
533
|
+
this.flushCushion();
|
|
534
|
+
if (this.pending.length > 0 || this.queue.length > 0) this.pump();
|
|
480
535
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
536
|
+
/** Asks for one sentence, without waiting for the answer. */
|
|
537
|
+
request(item) {
|
|
538
|
+
const controller = new AbortController();
|
|
539
|
+
this.abortControllers.add(controller);
|
|
540
|
+
const response = this.openai.audio.speech.create(
|
|
541
|
+
{
|
|
542
|
+
model: this.options.model || DEFAULT_MODEL3,
|
|
543
|
+
voice: this.options.voice || DEFAULT_VOICE,
|
|
544
|
+
input: item.text,
|
|
545
|
+
response_format: "pcm",
|
|
546
|
+
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
547
|
+
...this.options.speed ? { speed: this.options.speed } : {}
|
|
548
|
+
},
|
|
549
|
+
{ signal: controller.signal }
|
|
550
|
+
).catch((error) => {
|
|
551
|
+
if (!controller.signal.aborted) {
|
|
552
|
+
this.log("Error synthesizing speech:", error);
|
|
553
|
+
this.emit("Failed", [item.text]);
|
|
554
|
+
}
|
|
555
|
+
return null;
|
|
556
|
+
});
|
|
557
|
+
return { ...item, controller, response };
|
|
558
|
+
}
|
|
559
|
+
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
560
|
+
emitAudio(chunk) {
|
|
561
|
+
if (this.cushionBytes >= OPENING_CUSHION) {
|
|
562
|
+
this.emit("Audio", chunk);
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
this.cushion.push(chunk);
|
|
566
|
+
this.cushionBytes += chunk.length;
|
|
567
|
+
if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion();
|
|
568
|
+
}
|
|
569
|
+
flushCushion() {
|
|
570
|
+
if (this.cushion.length === 0) return;
|
|
571
|
+
const held = this.cushion;
|
|
572
|
+
this.cushion = [];
|
|
573
|
+
for (const chunk of held) this.emit("Audio", chunk);
|
|
574
|
+
}
|
|
575
|
+
/** Emits one sentence, whose request was started earlier. */
|
|
576
|
+
async speakItem(item) {
|
|
484
577
|
try {
|
|
485
|
-
const response = await
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
input: text,
|
|
490
|
-
response_format: "pcm",
|
|
491
|
-
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
492
|
-
...this.options.speed ? { speed: this.options.speed } : {}
|
|
493
|
-
},
|
|
494
|
-
{ signal: abortController.signal }
|
|
495
|
-
);
|
|
496
|
-
if (!response.body) return;
|
|
497
|
-
this.log(`Synthesizing: "${text}"`);
|
|
578
|
+
const response = await item.response;
|
|
579
|
+
if (!response?.body) return;
|
|
580
|
+
if (item.counter !== this.counter) return;
|
|
581
|
+
this.log(`Synthesizing: "${item.text}"`);
|
|
498
582
|
const resampler = new Pcm16Resampler(
|
|
499
583
|
OPENAI_SAMPLE_RATE2,
|
|
500
584
|
OUTPUT_SAMPLE_RATE
|
|
@@ -503,21 +587,15 @@ var OpenaiTTS = class extends TTS {
|
|
|
503
587
|
while (true) {
|
|
504
588
|
const { done, value } = await reader.read();
|
|
505
589
|
if (done) break;
|
|
506
|
-
if (counter !== this.counter) {
|
|
590
|
+
if (item.counter !== this.counter) {
|
|
507
591
|
await reader.cancel();
|
|
508
592
|
break;
|
|
509
593
|
}
|
|
510
594
|
const output = resampler.process(Buffer.from(value));
|
|
511
|
-
if (output.length > 0) this.
|
|
595
|
+
if (output.length > 0) this.emitAudio(output);
|
|
512
596
|
}
|
|
513
|
-
} catch (error) {
|
|
514
|
-
if (abortController.signal.aborted) return;
|
|
515
|
-
this.log("Error synthesizing speech:", error);
|
|
516
|
-
this.emit("Failed", [text]);
|
|
517
597
|
} finally {
|
|
518
|
-
|
|
519
|
-
this.abortController = void 0;
|
|
520
|
-
}
|
|
598
|
+
this.abortControllers.delete(item.controller);
|
|
521
599
|
}
|
|
522
600
|
}
|
|
523
601
|
};
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n this.abortController = undefined\n // Stop query now that we have a complete answer\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n return\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again in case of tool call\n if (!skipAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = []\n private processing = false\n private abortController?: AbortController\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n this.counter++\n const counter = this.counter\n this.buffer = ''\n\n textStream.on('data', (chunk: Buffer) => {\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n if (counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.abortController?.abort()\n this.abortController = undefined\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.processQueue()\n }\n\n private async processQueue() {\n if (this.processing) return\n this.processing = true\n\n while (this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n await this.synthesize(item.counter, item.text)\n }\n\n this.processing = false\n // Items may have been enqueued right as we exited the loop\n if (this.queue.length > 0) this.processQueue()\n }\n\n private async synthesize(counter: number, text: string) {\n const abortController = new AbortController()\n this.abortController = abortController\n\n try {\n const response = await this.openai.audio.speech.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: abortController.signal }\n )\n\n if (!response.body) return\n this.log(`Synthesizing: \"${text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emit('Audio', output)\n }\n } catch (error) {\n if (abortController.signal.aborted) return\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [text])\n } finally {\n if (this.abortController === abortController) {\n this.abortController = undefined\n }\n }\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,OAAO,YAAY;AAEnB,OAAO,KAAK,oBAAoB;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AAGjB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AACH,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AACH,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,iBAAK,kBAAkB;AAMvB;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAG9C,UAAI,CAAC,YAAY;AACf,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACxMA,SAAS,WAAW;AAEpB,OAAO,eAAe;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMA,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,IAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASD;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,SAAS,WAAW;AACpB,OAAOC,aAAY;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAOpB,IAAM,YAAN,cAAwB,IAAI;AAAA,EAQjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAN7B,SAAQ,UAAU;AAClB;AAAA,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B,SAAQ,aAAa;AAKnB,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAIC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,SAAK;AACL,UAAM,UAAU,KAAK;AACrB,SAAK,SAAS;AAEd,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AACzB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,iBAAiB,MAAM;AAC5B,SAAK,kBAAkB;AAAA,EACzB;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,aAAa;AAAA,EACpB;AAAA,EAEA,MAAc,eAAe;AAC3B,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,MAAM,SAAS,GAAG;AAC5B,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,YAAM,KAAK,WAAW,KAAK,SAAS,KAAK,IAAI;AAAA,IAC/C;AAEA,SAAK,aAAa;AAElB,QAAI,KAAK,MAAM,SAAS,EAAG,MAAK,aAAa;AAAA,EAC/C;AAAA,EAEA,MAAc,WAAW,SAAiB,MAAc;AACtD,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AAEvB,QAAI;AACF,YAAM,WAAW,MAAM,KAAK,OAAO,MAAM,OAAO;AAAA,QAC9C;AAAA,UACE,OAAO,KAAK,QAAQ,SAASF;AAAA,UAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO;AAAA,UACP,iBAAiB;AAAA,UACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,UACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,QAC5D;AAAA,QACA,EAAE,QAAQ,gBAAgB,OAAO;AAAA,MACnC;AAEA,UAAI,CAAC,SAAS,KAAM;AACpB,WAAK,IAAI,kBAAkB,IAAI,GAAG;AAElC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,YAAY,KAAK,SAAS;AAC5B,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,KAAK,SAAS,MAAM;AAAA,MAClD;AAAA,IACF,SAAS,OAAO;AACd,UAAI,gBAAgB,OAAO,QAAS;AACpC,WAAK,IAAI,8BAA8B,KAAK;AAC5C,WAAK,KAAK,UAAU,CAAC,IAAI,CAAC;AAAA,IAC5B,UAAE;AACA,UAAI,KAAK,oBAAoB,iBAAiB;AAC5C,aAAK,kBAAkB;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AACF;","names":["DEFAULT_MODEL","DEFAULT_RETRY_DELAY","OpenAI","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
|
1
|
+
{"version":3,"sources":["../src/OpenaiAgent.ts","../src/OpenaiSTT.ts","../src/utils/Pcm16Resampler.ts","../src/OpenaiTTS.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { PassThrough } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport type OpenaiAgentOptions = AgentOptions &\n OpenaiOptions & {\n model?: string\n settings?: Omit<\n OpenAI.Responses.ResponseCreateParamsStreaming,\n 'input' | 'model' | 'tools' | 'stream'\n >\n retryDelay?: number\n maxRetry?: number\n maxSteps?: number\n }\n\nexport type OpenaiOptions = { apiKey: string } | { openai: OpenAI }\n\nconst DEFAULT_MODEL = 'gpt-4o'\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_MAX_RETRIES = 3\nconst DEFAULT_RETRY_DELAY = 1000\n\nexport class OpenaiAgent extends Agent<OpenaiAgentOptions> {\n private openai: OpenAI\n private abortController?: AbortController\n\n constructor(options: OpenaiAgentOptions) {\n super(options)\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: PassThrough,\n stepCount = 0,\n tryCount = 0\n ): Promise<void> {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[OpenaiAgent] Max steps reached')\n return\n }\n\n const abortController = new AbortController()\n this.abortController = abortController\n const signal = abortController.signal\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n // Generate answer\n const response = await this.openai.responses.create(\n {\n model: this.options.model || DEFAULT_MODEL,\n input: this.buildInput(),\n tools: this.buildTools(),\n stream: true,\n ...this.options.settings,\n },\n { signal }\n )\n let skipAnswer = false\n let hasAnswer = false\n\n // Handle response events\n for await (const event of response) {\n if (abortController !== this.abortController) return\n switch (event.type) {\n case 'response.output_text.delta':\n // Ignore extra outputs, we keep the first answer only\n if (hasAnswer) break\n\n this.log(`Answer chunk: \"${event.delta}\"`)\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = event.delta.indexOf(\n extractOptions.startTag\n )\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = event.delta\n .slice(0, startTagIndex)\n .trimEnd()\n stream.write(messagePart)\n break\n }\n } else {\n // Extracting, don't write to stream\n break\n }\n }\n\n // Send chunk\n stream.write(event.delta)\n break\n\n case 'response.output_text.done':\n // Keep the first answer only.\n // Note: OpenAI can sometimes return multiple outputs.\n // See https://community.openai.com/t/how-to-prevent-the-api-returning-multiple-outputs/1251365/28\n // We can prompt it like this to avoid consuming too many tokens:\n // `Return one complete answer and then stop`\n if (hasAnswer) break\n\n const { message, metadata } = this.extract(event.text)\n this.addAssistantMessage(message, metadata)\n hasAnswer = true\n // Keep reading events instead of returning here: the API streams\n // one output item at a time, so a function call the model made\n // alongside its answer is only announced after the text is done.\n // Returning dropped every tool call that came with an answer.\n break\n\n case 'response.output_item.done':\n if (event.item.type === 'function_call') {\n const toolCall = event.item\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.call_id,\n toolName: toolCall.name,\n parameters: toolCall.arguments,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n break\n\n default:\n break\n }\n }\n\n if (abortController !== this.abortController) return\n\n // Query again when the model called a tool without answering, so the\n // turn still produces something to say\n if (!skipAnswer && !hasAnswer) {\n await this.generateAnswer(stream, stepCount + 1)\n }\n this.abortController = undefined\n } catch (error) {\n if (error instanceof OpenAI.APIUserAbortError) return\n console.error('[OpenaiAgent] Error answering:', error)\n\n if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildInput(): OpenAI.Responses.ResponseInput {\n return this.conversation.map((message) => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n type: 'function_call',\n call_id: message.toolCallId,\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n }\n case 'tool_result':\n return {\n type: 'function_call_output',\n call_id: message.toolCallId,\n output: message.output,\n }\n }\n })\n }\n\n private buildTools(): OpenAI.Responses.Tool[] | undefined {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n name: tool.name,\n description: tool.description,\n strict: true,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n }))\n }\n\n cancel() {\n if (!this.abortController) return\n this.log('Cancel')\n this.abortController.abort()\n this.abortController = undefined\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Real-time STT\n *\n * @see https://platform.openai.com/docs/guides/speech-to-text\n */\n\nexport interface OpenaiSTTOptions {\n apiKey: string\n model?: string\n language?: string\n prompt?: string\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-transcribe'\nconst DEFAULT_LANGUAGE = 'en'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst OPENAI_SAMPLE_RATE = 24000 // Min rate accepted by the GA Realtime API\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class OpenaiSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private retryCount = 0\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private resampler = new Pcm16Resampler(SAMPLE_RATE, OPENAI_SAMPLE_RATE)\n\n constructor(private options: OpenaiSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.initWS().catch((error) => {\n console.error('[OpenaiSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n transcribe(audioStream: Readable) {\n // New utterance: OpenAI commits and clears its buffer after each one, so\n // resample this stream independently from the previous one.\n this.resampler.reset()\n\n // Read audio stream and send to OpenAI\n audioStream.on('data', async (chunk: Buffer) => {\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.commitAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', '')\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the transcription session (GA Realtime API schema)\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n type: 'transcription',\n audio: {\n input: {\n format: {\n type: 'audio/pcm',\n rate: OPENAI_SAMPLE_RATE,\n },\n transcription: {\n model: this.options.model || DEFAULT_MODEL,\n language: this.options.language || DEFAULT_LANGUAGE,\n prompt:\n this.options.prompt ||\n 'Transcribe the incoming audio in real time.',\n },\n // Disable server-side VAD: the Micdrop client already detects\n // speech and only streams audio while the user is talking, so we\n // commit the buffer manually at the end of each utterance.\n turn_detection: null,\n noise_reduction: {\n type: 'near_field',\n },\n },\n },\n },\n })\n )\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(\n 'wss://api.openai.com/v1/realtime?intent=transcription',\n {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n }\n )\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error: any) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener(\n 'close',\n ({ code, reason }: { code: number; reason: string }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n }\n )\n\n socket.addEventListener('message', (event: any) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n\n // Resample 16kHz -> 24kHz (GA Realtime API requires a rate >= 24kHz)\n const resampled = this.resampler.process(chunk)\n if (resampled.length === 0) return\n\n this.socket.send(\n JSON.stringify({\n type: 'input_audio_buffer.append',\n audio: resampled.toString('base64'),\n })\n )\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n // case 'input_audio_buffer.committed':\n // this.log('Audio buffer committed')\n // break\n\n // case 'input_audio_buffer.speech_started':\n // this.log('Speech started')\n // break\n\n // case 'input_audio_buffer.speech_stopped':\n // this.log('Speech stopped')\n // break\n\n // case 'conversation.item.input_audio_transcription.delta':\n // this.log(`Received transcript delta: \"${message.delta}\"`)\n // break\n\n case 'conversation.item.input_audio_transcription.completed':\n const transcript = message.transcript || ''\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript)\n // Reset audio chunks and clear timeout\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n\n case 'error':\n this.log('Error:', message.error)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Send audio chunks again if reconnecting during transcription.\n // The OpenAI buffer is fresh, so replay them through a reset\n // resampler to reproduce the exact same stream.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.resampler.reset()\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n\n private commitAudio() {\n if (!this.socket) return\n this.socket.send(JSON.stringify({ type: 'input_audio_buffer.commit' }))\n this.log('Committed audio buffer')\n }\n}\n","/**\n * Streaming linear-interpolation resampler for PCM16 mono audio.\n *\n * Works in both directions (up or downsampling). It is stateful: it handles\n * arbitrary byte boundaries (a network chunk can split a 16-bit sample) and\n * keeps the fractional sample position continuous across chunks, so feeding a\n * stream chunk by chunk yields the same result as resampling it in one go.\n *\n * Used by OpenaiSTT (16kHz -> 24kHz, GA Realtime API requires >= 24kHz) and\n * OpenaiTTS (24kHz pcm output -> 16kHz expected by the Micdrop client).\n */\nexport class Pcm16Resampler {\n private readonly step: number\n private leftover: Buffer<ArrayBufferLike> = Buffer.alloc(0)\n private pos = 0 // Fractional position into the first sample of the buffer\n\n constructor(inRate: number, outRate: number) {\n this.step = inRate / outRate\n }\n\n // Reset to the initial state, to resample a new independent stream\n // (e.g. resending buffered audio after a reconnection).\n reset() {\n this.leftover = Buffer.alloc(0)\n this.pos = 0\n }\n\n process(chunk: Buffer): Buffer {\n const buf = this.leftover.length\n ? Buffer.concat([this.leftover, chunk])\n : chunk\n const samples = Math.floor(buf.length / 2)\n\n // Need at least 2 samples to interpolate\n if (samples < 2) {\n this.leftover = buf\n return Buffer.alloc(0)\n }\n\n const out: number[] = []\n let p = this.pos\n while (Math.floor(p) + 1 < samples) {\n const i = Math.floor(p)\n const frac = p - i\n const s0 = buf.readInt16LE(i * 2)\n const s1 = buf.readInt16LE((i + 1) * 2)\n out.push(Math.round(s0 + (s1 - s0) * frac))\n p += this.step\n }\n\n // Keep the last still-needed sample (and any trailing odd byte) for the\n // next chunk, and carry the fractional position relative to it.\n const consumed = Math.floor(p)\n this.pos = p - consumed\n this.leftover = buf.subarray(consumed * 2)\n\n const result = Buffer.alloc(out.length * 2)\n for (let k = 0; k < out.length; k++) {\n result.writeInt16LE(out[k], k * 2)\n }\n return result\n }\n}\n","import { TTS } from '@micdrop/server'\nimport OpenAI from 'openai'\nimport { Readable } from 'stream'\nimport { OpenaiOptions } from './OpenaiAgent'\nimport { Pcm16Resampler } from './utils/Pcm16Resampler'\n\n/**\n * OpenAI Text-to-Speech\n *\n * @see https://platform.openai.com/docs/guides/text-to-speech\n *\n * The OpenAI speech endpoint takes a complete text input (no streaming text\n * in), so the incoming text stream is buffered into sentences and each\n * sentence is synthesized as soon as it is complete. Requests are processed\n * sequentially, which keeps the emitted audio in order while still starting\n * playback as soon as the first sentence is ready.\n */\n\nexport type OpenaiTTSOptions = OpenaiOptions & {\n model?: string\n voice?: string\n // Prosody instructions (accent, emotion, speed, tone...).\n // Only works with gpt-4o-mini-tts, not tts-1 / tts-1-hd.\n instructions?: string\n // Speech speed from 0.25 to 4.0. Only works with tts-1 / tts-1-hd.\n speed?: number\n}\n\nconst DEFAULT_MODEL = 'gpt-4o-mini-tts'\nconst DEFAULT_VOICE = 'alloy'\nconst OPENAI_SAMPLE_RATE = 24000 // Rate of the pcm output from OpenAI\nconst OUTPUT_SAMPLE_RATE = 16000 // Rate expected by the Micdrop client\nconst MAX_IN_FLIGHT = 2 // Requests asked for at once, current one included\n// Audio held back at the start of an utterance, in bytes of the output format\n// (16 bits, 16 kHz, so 32 bytes per millisecond).\n//\n// The endpoint answers with a burst of a hundred milliseconds or so, then goes\n// quiet while it generates the rest. Forwarding that burst as it arrives means\n// the browser starts playing a syllable it cannot continue, and the hole lands\n// inside the first word. Holding the opening back until there is enough of it\n// to cover the pause costs the same time either way, and spends it before the\n// first syllable rather than inside it.\nconst OPENING_CUSHION = 300 * 32\n\ninterface QueueItem {\n counter: number\n text: string\n}\n\n/** A sentence whose synthesis has been asked for and not yet spoken. */\ninterface PendingItem extends QueueItem {\n controller: AbortController\n response: Promise<{ body: ReadableStream<Uint8Array> | null } | null>\n}\n\nexport class OpenaiTTS extends TTS {\n private openai: OpenAI\n private counter = 0 // Identifies the current speak() call\n // Bumped by every speak() and every cancel(), so a call claimed late can tell\n // whether it is still the one that should be heard. Kept apart from\n // `counter`, which numbers the synthesis calls and must only move when there\n // is something to synthesize.\n private generation = 0\n private buffer = '' // Incomplete sentence waiting for more text\n private queue: QueueItem[] = [] // Sentences whose synthesis has not started\n private pending: PendingItem[] = [] // Requests in flight, in speaking order\n private cushion: Buffer[] = [] // Opening audio held back, see OPENING_CUSHION\n private cushionBytes = 0\n private processing = false\n // More than one request is in flight at a time, so they are aborted as a set.\n private abortControllers = new Set<AbortController>()\n\n constructor(private readonly options: OpenaiTTSOptions) {\n super()\n this.openai =\n 'openai' in options\n ? options.openai\n : new OpenAI({ apiKey: options.apiKey })\n }\n\n speak(textStream: Readable) {\n const generation = ++this.generation\n let counter = 0\n\n // Claiming the call is deferred until there is something to say.\n //\n // Taking the next number right away would drop the utterance still being\n // spoken, since both the queue and the streaming reader skip anything\n // stamped with an older one. A stream that never carries a word, which is\n // what an answer skipped by a tool or by onBeforeAnswer hands over, would\n // then cut the assistant off and throw away the sentences still queued.\n const claimCall = () => {\n if (counter) return true\n // Cancelled, or superseded by another speak(), before the first word.\n if (this.generation !== generation) return false\n this.counter++\n counter = this.counter\n this.buffer = ''\n this.cushion = []\n this.cushionBytes = 0\n return true\n }\n\n textStream.on('data', (chunk: Buffer) => {\n if (!claimCall()) return\n if (counter !== this.counter) return\n this.buffer += chunk.toString('utf-8')\n this.flushSentences(counter, false)\n })\n\n textStream.on('error', (error) => {\n this.log('Error in text stream', error)\n })\n\n textStream.on('end', () => {\n // Nothing was ever said, so there is nothing left to flush.\n if (!counter || counter !== this.counter) return\n this.flushSentences(counter, true)\n })\n }\n\n cancel() {\n this.generation++\n this.log('Cancel')\n // Increment counter to ignore in-flight and queued work\n this.counter++\n this.buffer = ''\n this.queue = []\n this.pending = []\n this.cushion = []\n this.cushionBytes = 0\n this.abortControllers.forEach((controller) => controller.abort())\n this.abortControllers.clear()\n }\n\n // Extract complete sentences from the buffer and enqueue them.\n // On `end`, whatever remains in the buffer is flushed as a last sentence.\n private flushSentences(counter: number, end: boolean) {\n const regex = /[\\s\\S]*?[.!?…\\n]+(?=\\s|$)/g\n let match: RegExpExecArray | null\n let lastIndex = 0\n while ((match = regex.exec(this.buffer)) !== null) {\n // A sentence ending at the very end of an unfinished stream may still\n // grow, so keep it buffered until more text arrives or the stream ends.\n if (!end && regex.lastIndex === this.buffer.length) break\n const sentence = match[0].trim()\n if (sentence) this.enqueue(counter, sentence)\n lastIndex = regex.lastIndex\n }\n this.buffer = this.buffer.slice(lastIndex)\n\n if (end) {\n const rest = this.buffer.trim()\n if (rest) this.enqueue(counter, rest)\n this.buffer = ''\n }\n }\n\n private enqueue(counter: number, text: string) {\n this.queue.push({ counter, text })\n this.pump()\n }\n\n /**\n * Starts what can be started, then speaks what is ready.\n *\n * The endpoint takes a few hundred milliseconds before its first byte, and\n * waiting for that between two sentences leaves a silence in the middle of\n * her voice. It is loudest at the very start, where a greeting is often one\n * short sentence: a syllable, a hole, then the rest of the answer. Asking for\n * a sentence the moment it is complete, rather than when its turn to be\n * spoken comes, hides that latency behind the audio already playing.\n *\n * Two requests at a time is enough to cover the gap, and it keeps a long\n * answer from opening one request per sentence all at once.\n */\n private pump() {\n while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {\n const item = this.queue.shift()!\n // Skip work from a cancelled or superseded speak() call\n if (item.counter !== this.counter) continue\n this.pending.push(this.request(item))\n }\n this.drain()\n }\n\n /** Speaks the started requests in the order they were queued. */\n private async drain() {\n if (this.processing) return\n this.processing = true\n\n while (this.pending.length > 0) {\n const item = this.pending.shift()!\n // A slot just freed, so the sentence after the next one can start now\n this.pump()\n await this.speakItem(item)\n }\n\n this.processing = false\n // Nothing left to speak: an utterance shorter than the cushion is released\n // here, since there is no longer anything coming to fill it up.\n this.flushCushion()\n // Sentences may have arrived right as we exited the loop\n if (this.pending.length > 0 || this.queue.length > 0) this.pump()\n }\n\n /** Asks for one sentence, without waiting for the answer. */\n private request(item: QueueItem): PendingItem {\n const controller = new AbortController()\n this.abortControllers.add(controller)\n\n const response = this.openai.audio.speech\n .create(\n {\n model: this.options.model || DEFAULT_MODEL,\n voice: this.options.voice || DEFAULT_VOICE,\n input: item.text,\n response_format: 'pcm',\n ...(this.options.instructions\n ? { instructions: this.options.instructions }\n : {}),\n ...(this.options.speed ? { speed: this.options.speed } : {}),\n },\n { signal: controller.signal }\n )\n .catch((error) => {\n if (!controller.signal.aborted) {\n this.log('Error synthesizing speech:', error)\n this.emit('Failed', [item.text])\n }\n return null\n })\n\n return { ...item, controller, response }\n }\n\n /** Holds the opening of an utterance back, then lets the rest through. */\n private emitAudio(chunk: Buffer) {\n if (this.cushionBytes >= OPENING_CUSHION) {\n this.emit('Audio', chunk)\n return\n }\n this.cushion.push(chunk)\n this.cushionBytes += chunk.length\n if (this.cushionBytes >= OPENING_CUSHION) this.flushCushion()\n }\n\n private flushCushion() {\n if (this.cushion.length === 0) return\n const held = this.cushion\n this.cushion = []\n for (const chunk of held) this.emit('Audio', chunk)\n }\n\n /** Emits one sentence, whose request was started earlier. */\n private async speakItem(item: PendingItem) {\n try {\n const response = await item.response\n if (!response?.body) return\n if (item.counter !== this.counter) return\n this.log(`Synthesizing: \"${item.text}\"`)\n\n const resampler = new Pcm16Resampler(\n OPENAI_SAMPLE_RATE,\n OUTPUT_SAMPLE_RATE\n )\n const reader = response.body.getReader()\n\n while (true) {\n const { done, value } = await reader.read()\n if (done) break\n if (item.counter !== this.counter) {\n await reader.cancel()\n break\n }\n const output = resampler.process(Buffer.from(value))\n if (output.length > 0) this.emitAudio(output)\n }\n } finally {\n this.abortControllers.delete(item.controller)\n }\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,OAAO,YAAY;AAEnB,OAAO,KAAK,oBAAoB;AAgBhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAC5B,IAAM,sBAAsB;AAErB,IAAM,cAAN,cAA0B,MAA0B;AAAA,EAIzD,YAAY,SAA6B;AACvC,UAAM,OAAO;AACb,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAI,OAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACI;AACf,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,iCAAiC;AAC/C;AAAA,IACF;AAEA,UAAM,kBAAkB,IAAI,gBAAgB;AAC5C,SAAK,kBAAkB;AACvB,UAAM,SAAS,gBAAgB;AAG/B,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AAEF,YAAM,WAAW,MAAM,KAAK,OAAO,UAAU;AAAA,QAC3C;AAAA,UACE,OAAO,KAAK,QAAQ,SAAS;AAAA,UAC7B,OAAO,KAAK,WAAW;AAAA,UACvB,OAAO,KAAK,WAAW;AAAA,UACvB,QAAQ;AAAA,UACR,GAAG,KAAK,QAAQ;AAAA,QAClB;AAAA,QACA,EAAE,OAAO;AAAA,MACX;AACA,UAAI,aAAa;AACjB,UAAI,YAAY;AAGhB,uBAAiB,SAAS,UAAU;AAClC,YAAI,oBAAoB,KAAK,gBAAiB;AAC9C,gBAAQ,MAAM,MAAM;AAAA,UAClB,KAAK;AAEH,gBAAI,UAAW;AAEf,iBAAK,IAAI,kBAAkB,MAAM,KAAK,GAAG;AAGzC,gBAAI,gBAAgB;AAClB,kBAAI,CAAC,YAAY;AACf,sBAAM,gBAAgB,MAAM,MAAM;AAAA,kBAChC,eAAe;AAAA,gBACjB;AACA,oBAAI,kBAAkB,IAAI;AACxB,+BAAa;AACb,wBAAM,cAAc,MAAM,MACvB,MAAM,GAAG,aAAa,EACtB,QAAQ;AACX,yBAAO,MAAM,WAAW;AACxB;AAAA,gBACF;AAAA,cACF,OAAO;AAEL;AAAA,cACF;AAAA,YACF;AAGA,mBAAO,MAAM,MAAM,KAAK;AACxB;AAAA,UAEF,KAAK;AAMH,gBAAI,UAAW;AAEf,kBAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,MAAM,IAAI;AACrD,iBAAK,oBAAoB,SAAS,QAAQ;AAC1C,wBAAY;AAKZ;AAAA,UAEF,KAAK;AACH,gBAAI,MAAM,KAAK,SAAS,iBAAiB;AACvC,oBAAM,WAAW,MAAM;AACvB,oBAAM,SAAS,MAAM,KAAK,YAAY;AAAA,gBACpC,MAAM;AAAA,gBACN,YAAY,SAAS;AAAA,gBACrB,UAAU,SAAS;AAAA,gBACnB,YAAY,SAAS;AAAA,cACvB,CAAC;AACD,kBAAI,OAAO,YAAY;AACrB,6BAAa;AAAA,cACf;AAAA,YACF;AACA;AAAA,UAEF;AACE;AAAA,QACJ;AAAA,MACF;AAEA,UAAI,oBAAoB,KAAK,gBAAiB;AAI9C,UAAI,CAAC,cAAc,CAAC,WAAW;AAC7B,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AACA,WAAK,kBAAkB;AAAA,IACzB,SAAS,OAAO;AACd,UAAI,iBAAiB,OAAO,kBAAmB;AAC/C,cAAQ,MAAM,kCAAkC,KAAK;AAErD,UAAI,YAAY,KAAK,QAAQ,YAAY,sBAAsB;AAC7D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAA6C;AACnD,WAAO,KAAK,aAAa,IAAI,CAAC,YAAY;AACxC,cAAQ,QAAQ,MAAM;AAAA,QACpB,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK;AACH,iBAAO;AAAA,YACL,MAAM,QAAQ;AAAA,YACd,SAAS,QAAQ;AAAA,UACnB;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,MAAM,QAAQ;AAAA,YACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,UAC9C;AAAA,QACF,KAAK;AACH,iBAAO;AAAA,YACL,MAAM;AAAA,YACN,SAAS,QAAQ;AAAA,YACjB,QAAQ,QAAQ;AAAA,UAClB;AAAA,MACJ;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,aAAkD;AAExD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,MAAM,KAAK;AAAA,MACX,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,IACzD,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,gBAAiB;AAC3B,SAAK,IAAI,QAAQ;AACjB,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAAA,EACzB;AACF;;;ACnNA,SAAS,WAAW;AAEpB,OAAO,eAAe;;;ACSf,IAAM,iBAAN,MAAqB;AAAA;AAAA,EAK1B,YAAY,QAAgB,SAAiB;AAH7C,SAAQ,WAAoC,OAAO,MAAM,CAAC;AAC1D,SAAQ,MAAM;AAGZ,SAAK,OAAO,SAAS;AAAA,EACvB;AAAA;AAAA;AAAA,EAIA,QAAQ;AACN,SAAK,WAAW,OAAO,MAAM,CAAC;AAC9B,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,QAAQ,OAAuB;AAC7B,UAAM,MAAM,KAAK,SAAS,SACtB,OAAO,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,IACpC;AACJ,UAAM,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC;AAGzC,QAAI,UAAU,GAAG;AACf,WAAK,WAAW;AAChB,aAAO,OAAO,MAAM,CAAC;AAAA,IACvB;AAEA,UAAM,MAAgB,CAAC;AACvB,QAAI,IAAI,KAAK;AACb,WAAO,KAAK,MAAM,CAAC,IAAI,IAAI,SAAS;AAClC,YAAM,IAAI,KAAK,MAAM,CAAC;AACtB,YAAM,OAAO,IAAI;AACjB,YAAM,KAAK,IAAI,YAAY,IAAI,CAAC;AAChC,YAAM,KAAK,IAAI,aAAa,IAAI,KAAK,CAAC;AACtC,UAAI,KAAK,KAAK,MAAM,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1C,WAAK,KAAK;AAAA,IACZ;AAIA,UAAM,WAAW,KAAK,MAAM,CAAC;AAC7B,SAAK,MAAM,IAAI;AACf,SAAK,WAAW,IAAI,SAAS,WAAW,CAAC;AAEzC,UAAM,SAAS,OAAO,MAAM,IAAI,SAAS,CAAC;AAC1C,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,aAAO,aAAa,IAAI,CAAC,GAAG,IAAI,CAAC;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACF;;;ADxCA,IAAMA,iBAAgB;AACtB,IAAM,mBAAmB;AACzB,IAAM,cAAc;AACpB,IAAM,qBAAqB;AAC3B,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAM,oBAAoB;AAEnB,IAAM,YAAN,cAAwB,IAAI;AAAA,EASjC,YAAoB,SAA2B;AAC7C,UAAM;AADY;AALpB,SAAQ,aAAa;AAErB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,YAAY,IAAI,eAAe,aAAa,kBAAkB;AAMpE,SAAK,cAAc,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AAChD,cAAQ,MAAM,iCAAiC,KAAK;AACpD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,aAAuB;AAGhC,SAAK,UAAU,MAAM;AAGrB,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAC9C,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAGD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,YAAY;AAGjB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,EAAE;AAC1B,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAGlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,MAAM;AAAA,UACN,OAAO;AAAA,YACL,OAAO;AAAA,cACL,QAAQ;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,eAAe;AAAA,gBACb,OAAO,KAAK,QAAQ,SAASD;AAAA,gBAC7B,UAAU,KAAK,QAAQ,YAAY;AAAA,gBACnC,QACE,KAAK,QAAQ,UACb;AAAA,cACJ;AAAA;AAAA;AAAA;AAAA,cAIA,gBAAgB;AAAA,cAChB,iBAAiB;AAAA,gBACf,MAAM;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI;AAAA,QACjB;AAAA,QACA;AAAA,UACE,SAAS;AAAA,YACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AACA,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAe;AAC/C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO;AAAA,QACL;AAAA,QACA,CAAC,EAAE,MAAM,OAAO,MAAwC;AACtD,uBAAa,OAAO;AACpB,eAAK,QAAQ,mBAAmB;AAChC,eAAK,SAAS;AAEd,cAAI,SAAS,KAAM;AACjB,iBAAK,UAAU;AAAA,UACjB,OAAO;AACL,iBAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,UAChD;AAAA,QACF;AAAA,MACF;AAEA,aAAO,iBAAiB,WAAW,CAAC,UAAe;AACjD,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAGlB,UAAM,YAAY,KAAK,UAAU,QAAQ,KAAK;AAC9C,QAAI,UAAU,WAAW,EAAG;AAE5B,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,UAAU,SAAS,QAAQ;AAAA,MACpC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAiBpB,KAAK;AACH,cAAM,aAAa,QAAQ,cAAc;AACzC,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,UAAU;AAElC,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MAEF,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,KAAK;AAChC;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,UAAU,MAAM;AACrB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcC,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AAAA,EAEQ,cAAc;AACpB,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,4BAA4B,CAAC,CAAC;AACtE,SAAK,IAAI,wBAAwB;AAAA,EACnC;AACF;;;AEzRA,SAAS,WAAW;AACpB,OAAOC,aAAY;AA2BnB,IAAMC,iBAAgB;AACtB,IAAM,gBAAgB;AACtB,IAAMC,sBAAqB;AAC3B,IAAM,qBAAqB;AAC3B,IAAM,gBAAgB;AAUtB,IAAM,kBAAkB,MAAM;AAavB,IAAM,YAAN,cAAwB,IAAI;AAAA,EAiBjC,YAA6B,SAA2B;AACtD,UAAM;AADqB;AAf7B,SAAQ,UAAU;AAKlB;AAAA;AAAA;AAAA;AAAA;AAAA,SAAQ,aAAa;AACrB,SAAQ,SAAS;AACjB;AAAA,SAAQ,QAAqB,CAAC;AAC9B;AAAA,SAAQ,UAAyB,CAAC;AAClC;AAAA,SAAQ,UAAoB,CAAC;AAC7B;AAAA,SAAQ,eAAe;AACvB,SAAQ,aAAa;AAErB;AAAA,SAAQ,mBAAmB,oBAAI,IAAqB;AAIlD,SAAK,SACH,YAAY,UACR,QAAQ,SACR,IAAIC,QAAO,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,YAAsB;AAC1B,UAAM,aAAa,EAAE,KAAK;AAC1B,QAAI,UAAU;AASd,UAAM,YAAY,MAAM;AACtB,UAAI,QAAS,QAAO;AAEpB,UAAI,KAAK,eAAe,WAAY,QAAO;AAC3C,WAAK;AACL,gBAAU,KAAK;AACf,WAAK,SAAS;AACd,WAAK,UAAU,CAAC;AAChB,WAAK,eAAe;AACpB,aAAO;AAAA,IACT;AAEA,eAAW,GAAG,QAAQ,CAAC,UAAkB;AACvC,UAAI,CAAC,UAAU,EAAG;AAClB,UAAI,YAAY,KAAK,QAAS;AAC9B,WAAK,UAAU,MAAM,SAAS,OAAO;AACrC,WAAK,eAAe,SAAS,KAAK;AAAA,IACpC,CAAC;AAED,eAAW,GAAG,SAAS,CAAC,UAAU;AAChC,WAAK,IAAI,wBAAwB,KAAK;AAAA,IACxC,CAAC;AAED,eAAW,GAAG,OAAO,MAAM;AAEzB,UAAI,CAAC,WAAW,YAAY,KAAK,QAAS;AAC1C,WAAK,eAAe,SAAS,IAAI;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,SAAS;AACP,SAAK;AACL,SAAK,IAAI,QAAQ;AAEjB,SAAK;AACL,SAAK,SAAS;AACd,SAAK,QAAQ,CAAC;AACd,SAAK,UAAU,CAAC;AAChB,SAAK,UAAU,CAAC;AAChB,SAAK,eAAe;AACpB,SAAK,iBAAiB,QAAQ,CAAC,eAAe,WAAW,MAAM,CAAC;AAChE,SAAK,iBAAiB,MAAM;AAAA,EAC9B;AAAA;AAAA;AAAA,EAIQ,eAAe,SAAiB,KAAc;AACpD,UAAM,QAAQ;AACd,QAAI;AACJ,QAAI,YAAY;AAChB,YAAQ,QAAQ,MAAM,KAAK,KAAK,MAAM,OAAO,MAAM;AAGjD,UAAI,CAAC,OAAO,MAAM,cAAc,KAAK,OAAO,OAAQ;AACpD,YAAM,WAAW,MAAM,CAAC,EAAE,KAAK;AAC/B,UAAI,SAAU,MAAK,QAAQ,SAAS,QAAQ;AAC5C,kBAAY,MAAM;AAAA,IACpB;AACA,SAAK,SAAS,KAAK,OAAO,MAAM,SAAS;AAEzC,QAAI,KAAK;AACP,YAAM,OAAO,KAAK,OAAO,KAAK;AAC9B,UAAI,KAAM,MAAK,QAAQ,SAAS,IAAI;AACpC,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,QAAQ,SAAiB,MAAc;AAC7C,SAAK,MAAM,KAAK,EAAE,SAAS,KAAK,CAAC;AACjC,SAAK,KAAK;AAAA,EACZ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeQ,OAAO;AACb,WAAO,KAAK,QAAQ,SAAS,iBAAiB,KAAK,MAAM,SAAS,GAAG;AACnE,YAAM,OAAO,KAAK,MAAM,MAAM;AAE9B,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,QAAQ,KAAK,KAAK,QAAQ,IAAI,CAAC;AAAA,IACtC;AACA,SAAK,MAAM;AAAA,EACb;AAAA;AAAA,EAGA,MAAc,QAAQ;AACpB,QAAI,KAAK,WAAY;AACrB,SAAK,aAAa;AAElB,WAAO,KAAK,QAAQ,SAAS,GAAG;AAC9B,YAAM,OAAO,KAAK,QAAQ,MAAM;AAEhC,WAAK,KAAK;AACV,YAAM,KAAK,UAAU,IAAI;AAAA,IAC3B;AAEA,SAAK,aAAa;AAGlB,SAAK,aAAa;AAElB,QAAI,KAAK,QAAQ,SAAS,KAAK,KAAK,MAAM,SAAS,EAAG,MAAK,KAAK;AAAA,EAClE;AAAA;AAAA,EAGQ,QAAQ,MAA8B;AAC5C,UAAM,aAAa,IAAI,gBAAgB;AACvC,SAAK,iBAAiB,IAAI,UAAU;AAEpC,UAAM,WAAW,KAAK,OAAO,MAAM,OAChC;AAAA,MACC;AAAA,QACE,OAAO,KAAK,QAAQ,SAASF;AAAA,QAC7B,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,OAAO,KAAK;AAAA,QACZ,iBAAiB;AAAA,QACjB,GAAI,KAAK,QAAQ,eACb,EAAE,cAAc,KAAK,QAAQ,aAAa,IAC1C,CAAC;AAAA,QACL,GAAI,KAAK,QAAQ,QAAQ,EAAE,OAAO,KAAK,QAAQ,MAAM,IAAI,CAAC;AAAA,MAC5D;AAAA,MACA,EAAE,QAAQ,WAAW,OAAO;AAAA,IAC9B,EACC,MAAM,CAAC,UAAU;AAChB,UAAI,CAAC,WAAW,OAAO,SAAS;AAC9B,aAAK,IAAI,8BAA8B,KAAK;AAC5C,aAAK,KAAK,UAAU,CAAC,KAAK,IAAI,CAAC;AAAA,MACjC;AACA,aAAO;AAAA,IACT,CAAC;AAEH,WAAO,EAAE,GAAG,MAAM,YAAY,SAAS;AAAA,EACzC;AAAA;AAAA,EAGQ,UAAU,OAAe;AAC/B,QAAI,KAAK,gBAAgB,iBAAiB;AACxC,WAAK,KAAK,SAAS,KAAK;AACxB;AAAA,IACF;AACA,SAAK,QAAQ,KAAK,KAAK;AACvB,SAAK,gBAAgB,MAAM;AAC3B,QAAI,KAAK,gBAAgB,gBAAiB,MAAK,aAAa;AAAA,EAC9D;AAAA,EAEQ,eAAe;AACrB,QAAI,KAAK,QAAQ,WAAW,EAAG;AAC/B,UAAM,OAAO,KAAK;AAClB,SAAK,UAAU,CAAC;AAChB,eAAW,SAAS,KAAM,MAAK,KAAK,SAAS,KAAK;AAAA,EACpD;AAAA;AAAA,EAGA,MAAc,UAAU,MAAmB;AACzC,QAAI;AACF,YAAM,WAAW,MAAM,KAAK;AAC5B,UAAI,CAAC,UAAU,KAAM;AACrB,UAAI,KAAK,YAAY,KAAK,QAAS;AACnC,WAAK,IAAI,kBAAkB,KAAK,IAAI,GAAG;AAEvC,YAAM,YAAY,IAAI;AAAA,QACpBC;AAAA,QACA;AAAA,MACF;AACA,YAAM,SAAS,SAAS,KAAK,UAAU;AAEvC,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,YAAI,KAAK,YAAY,KAAK,SAAS;AACjC,gBAAM,OAAO,OAAO;AACpB;AAAA,QACF;AACA,cAAM,SAAS,UAAU,QAAQ,OAAO,KAAK,KAAK,CAAC;AACnD,YAAI,OAAO,SAAS,EAAG,MAAK,UAAU,MAAM;AAAA,MAC9C;AAAA,IACF,UAAE;AACA,WAAK,iBAAiB,OAAO,KAAK,UAAU;AAAA,IAC9C;AAAA,EACF;AACF;","names":["DEFAULT_MODEL","DEFAULT_RETRY_DELAY","OpenAI","DEFAULT_MODEL","OPENAI_SAMPLE_RATE","OpenAI"]}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@micdrop/openai",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.2",
|
|
4
4
|
"description": "OpenAI implementation for @micdrop/server",
|
|
5
|
-
"author": "
|
|
5
|
+
"author": "Godefroy de Compreignac",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"main": "./dist/index.js",
|
|
8
8
|
"module": "./dist/index.mjs",
|