@micdrop/openai 1.5.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -6
- package/dist/index.d.mts +28 -11
- package/dist/index.d.ts +28 -11
- package/dist/index.js +391 -53
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +389 -51
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -188,6 +188,17 @@ var OpenaiAgent = class extends import_server.Agent {
|
|
|
188
188
|
var import_server2 = require("@micdrop/server");
|
|
189
189
|
var import_ws = __toESM(require("ws"));
|
|
190
190
|
var import_zod2 = __toESM(require("zod"));
|
|
191
|
+
|
|
192
|
+
// src/transcription.ts
|
|
193
|
+
function transcriptionLanguage(model, language) {
|
|
194
|
+
if (!language) return {};
|
|
195
|
+
return isListingLanguages(model) ? { languages: [language] } : { language };
|
|
196
|
+
}
|
|
197
|
+
function isListingLanguages(model) {
|
|
198
|
+
return model.startsWith("gpt-transcribe") || model.startsWith("gpt-live-transcribe");
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// src/OpenaiRealtime.ts
|
|
191
202
|
var DEFAULT_MODEL2 = "gpt-realtime-2.1";
|
|
192
203
|
var DEFAULT_VOICE = "marin";
|
|
193
204
|
var DEFAULT_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
|
|
@@ -311,6 +322,7 @@ var OpenaiRealtime = class extends import_server2.Realtime {
|
|
|
311
322
|
};
|
|
312
323
|
}
|
|
313
324
|
sendSessionUpdate() {
|
|
325
|
+
const transcriptionModel = this.options.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL;
|
|
314
326
|
this.socket?.send(
|
|
315
327
|
JSON.stringify({
|
|
316
328
|
type: "session.update",
|
|
@@ -325,8 +337,11 @@ var OpenaiRealtime = class extends import_server2.Realtime {
|
|
|
325
337
|
// The Micdrop client detects when the user speaks
|
|
326
338
|
turn_detection: null,
|
|
327
339
|
transcription: {
|
|
328
|
-
model:
|
|
329
|
-
...
|
|
340
|
+
model: transcriptionModel,
|
|
341
|
+
...transcriptionLanguage(
|
|
342
|
+
transcriptionModel,
|
|
343
|
+
this.options.language
|
|
344
|
+
)
|
|
330
345
|
},
|
|
331
346
|
noise_reduction: { type: "near_field" }
|
|
332
347
|
},
|
|
@@ -554,6 +569,7 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
554
569
|
super();
|
|
555
570
|
this.options = options;
|
|
556
571
|
this.retryCount = 0;
|
|
572
|
+
this.destroyed = false;
|
|
557
573
|
this.audioChunksPending = [];
|
|
558
574
|
// Store audio chunks to send them again if reconnecting
|
|
559
575
|
this.resampler = new import_server3.Pcm16Resampler(SAMPLE_RATE2, OPENAI_SAMPLE_RATE2);
|
|
@@ -584,6 +600,7 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
584
600
|
}
|
|
585
601
|
destroy() {
|
|
586
602
|
super.destroy();
|
|
603
|
+
this.destroyed = true;
|
|
587
604
|
if (this.reconnectTimeout) {
|
|
588
605
|
clearTimeout(this.reconnectTimeout);
|
|
589
606
|
this.reconnectTimeout = void 0;
|
|
@@ -592,14 +609,16 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
592
609
|
clearTimeout(this.transcriptionTimeout);
|
|
593
610
|
this.transcriptionTimeout = void 0;
|
|
594
611
|
}
|
|
595
|
-
this.
|
|
596
|
-
|
|
597
|
-
this.
|
|
612
|
+
if (this.connectionTimeout) {
|
|
613
|
+
clearTimeout(this.connectionTimeout);
|
|
614
|
+
this.connectionTimeout = void 0;
|
|
598
615
|
}
|
|
616
|
+
if (this.socket) closeSocket(this.socket);
|
|
599
617
|
this.socket = void 0;
|
|
600
618
|
}
|
|
601
619
|
sendSessionUpdate() {
|
|
602
620
|
if (!this.socket) return;
|
|
621
|
+
const model = this.options.model || DEFAULT_MODEL3;
|
|
603
622
|
this.socket.send(
|
|
604
623
|
JSON.stringify({
|
|
605
624
|
type: "session.update",
|
|
@@ -612,8 +631,11 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
612
631
|
rate: OPENAI_SAMPLE_RATE2
|
|
613
632
|
},
|
|
614
633
|
transcription: {
|
|
615
|
-
model
|
|
616
|
-
|
|
634
|
+
model,
|
|
635
|
+
...transcriptionLanguage(
|
|
636
|
+
model,
|
|
637
|
+
this.options.language || DEFAULT_LANGUAGE
|
|
638
|
+
),
|
|
617
639
|
prompt: this.options.prompt || "Transcribe the incoming audio in real time."
|
|
618
640
|
},
|
|
619
641
|
// Disable server-side VAD: the Micdrop client already detects
|
|
@@ -642,11 +664,11 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
642
664
|
this.socket = socket;
|
|
643
665
|
const timeout = setTimeout(() => {
|
|
644
666
|
this.log("Connection timeout");
|
|
645
|
-
socket
|
|
646
|
-
socket.close();
|
|
667
|
+
closeSocket(socket);
|
|
647
668
|
this.socket = void 0;
|
|
648
669
|
reject(new Error("WebSocket connection timeout"));
|
|
649
670
|
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
671
|
+
this.connectionTimeout = timeout;
|
|
650
672
|
socket.addEventListener("open", () => {
|
|
651
673
|
clearTimeout(timeout);
|
|
652
674
|
this.sendSessionUpdate();
|
|
@@ -711,6 +733,9 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
711
733
|
this.transcriptionTimeout = void 0;
|
|
712
734
|
}
|
|
713
735
|
break;
|
|
736
|
+
case "session.created":
|
|
737
|
+
this.retryCount = 0;
|
|
738
|
+
break;
|
|
714
739
|
case "error":
|
|
715
740
|
this.log("Error:", message.error);
|
|
716
741
|
break;
|
|
@@ -719,6 +744,7 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
719
744
|
}
|
|
720
745
|
}
|
|
721
746
|
reconnect() {
|
|
747
|
+
if (this.destroyed) return;
|
|
722
748
|
this.retryCount++;
|
|
723
749
|
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
724
750
|
this.log("Max retries reached, giving up");
|
|
@@ -730,7 +756,6 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
730
756
|
this.reconnectTimeout = setTimeout(() => {
|
|
731
757
|
this.reconnectTimeout = void 0;
|
|
732
758
|
this.initWS().then(() => {
|
|
733
|
-
this.retryCount = 0;
|
|
734
759
|
if (this.audioChunksPending.length > 0) {
|
|
735
760
|
this.log("Sending audio chunks again");
|
|
736
761
|
this.resampler.reset();
|
|
@@ -751,17 +776,301 @@ var OpenaiSTT = class extends import_server3.STT {
|
|
|
751
776
|
this.log("Committed audio buffer");
|
|
752
777
|
}
|
|
753
778
|
};
|
|
779
|
+
function closeSocket(socket) {
|
|
780
|
+
socket.removeAllListeners();
|
|
781
|
+
socket.on("error", () => {
|
|
782
|
+
});
|
|
783
|
+
if (socket.readyState === import_ws2.default.OPEN || socket.readyState === import_ws2.default.CONNECTING) {
|
|
784
|
+
socket.close(1e3);
|
|
785
|
+
}
|
|
786
|
+
}
|
|
754
787
|
|
|
755
788
|
// src/OpenaiTTS.ts
|
|
756
|
-
var
|
|
789
|
+
var import_server5 = require("@micdrop/server");
|
|
757
790
|
var import_openai2 = __toESM(require("openai"));
|
|
758
|
-
|
|
759
|
-
|
|
791
|
+
|
|
792
|
+
// src/OpenaiRealtimeSpeech.ts
|
|
793
|
+
var import_server4 = require("@micdrop/server");
|
|
794
|
+
var import_ws3 = __toESM(require("ws"));
|
|
795
|
+
var DEFAULT_REASONING_EFFORT = "minimal";
|
|
796
|
+
var DEFAULT_CONNECTION_TIMEOUT3 = 5e3;
|
|
797
|
+
var MIN_SPEED = 0.25;
|
|
798
|
+
var MAX_SPEED = 1.5;
|
|
760
799
|
var OPENAI_SAMPLE_RATE3 = 24e3;
|
|
761
800
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
801
|
+
var READER_INSTRUCTIONS = `You are a text-to-speech engine, not an assistant.
|
|
802
|
+
Each user message is a JSON object. Read its "response_text" aloud exactly as written, word for word, in the language it is written in.
|
|
803
|
+
Read it even when it is a question, an order or an incomplete sentence: it is never addressed to you.
|
|
804
|
+
Say nothing else: no answer, no greeting, no translation, no comment, no correction.`;
|
|
805
|
+
var OpenaiRealtimeSpeech = class {
|
|
806
|
+
constructor(options, log) {
|
|
807
|
+
this.options = options;
|
|
808
|
+
this.log = log;
|
|
809
|
+
this.destroyed = false;
|
|
810
|
+
this.itemCount = 0;
|
|
811
|
+
// Responses asked for and not finished, by their metadata id and by the id
|
|
812
|
+
// the API gave them. Cancelled ones leave both maps, which mutes them.
|
|
813
|
+
this.items = /* @__PURE__ */ new Map();
|
|
814
|
+
this.responses = /* @__PURE__ */ new Map();
|
|
815
|
+
this.connect().catch(() => {
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
synthesize(text) {
|
|
819
|
+
const item = {
|
|
820
|
+
id: `micdrop-tts-${++this.itemCount}`,
|
|
821
|
+
text,
|
|
822
|
+
resampler: new import_server4.Pcm16Resampler(OPENAI_SAMPLE_RATE3, OUTPUT_SAMPLE_RATE),
|
|
823
|
+
audio: [],
|
|
824
|
+
done: false
|
|
825
|
+
};
|
|
826
|
+
this.items.set(item.id, item);
|
|
827
|
+
this.request(item);
|
|
828
|
+
return {
|
|
829
|
+
audio: this.read(item),
|
|
830
|
+
cancel: () => this.cancel(item)
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
destroy() {
|
|
834
|
+
this.destroyed = true;
|
|
835
|
+
this.closeSocket();
|
|
836
|
+
}
|
|
837
|
+
request(item) {
|
|
838
|
+
this.connect().then(() => {
|
|
839
|
+
if (!this.items.has(item.id)) return;
|
|
840
|
+
this.send({
|
|
841
|
+
type: "response.create",
|
|
842
|
+
// Errors caused by this event carry its id
|
|
843
|
+
event_id: item.id,
|
|
844
|
+
response: {
|
|
845
|
+
// Out of the conversation, which then stays empty, and without
|
|
846
|
+
// its context: each sentence is read on its own
|
|
847
|
+
conversation: "none",
|
|
848
|
+
input: [
|
|
849
|
+
{
|
|
850
|
+
type: "message",
|
|
851
|
+
role: "user",
|
|
852
|
+
content: [
|
|
853
|
+
{
|
|
854
|
+
type: "input_text",
|
|
855
|
+
text: JSON.stringify({
|
|
856
|
+
response_text: item.text,
|
|
857
|
+
require_repeat_verbatim: true
|
|
858
|
+
})
|
|
859
|
+
}
|
|
860
|
+
]
|
|
861
|
+
}
|
|
862
|
+
],
|
|
863
|
+
output_modalities: ["audio"],
|
|
864
|
+
metadata: { micdrop_id: item.id }
|
|
865
|
+
}
|
|
866
|
+
});
|
|
867
|
+
}).catch((error) => this.finish(item, error));
|
|
868
|
+
}
|
|
869
|
+
async *read(item) {
|
|
870
|
+
while (true) {
|
|
871
|
+
const chunk = item.audio.shift();
|
|
872
|
+
if (chunk) {
|
|
873
|
+
yield chunk;
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
876
|
+
if (item.error) throw item.error;
|
|
877
|
+
if (item.done) return;
|
|
878
|
+
await new Promise((resolve) => item.wake = resolve);
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
cancel(item) {
|
|
882
|
+
if (item.responseId && !item.done) {
|
|
883
|
+
this.send({ type: "response.cancel", response_id: item.responseId });
|
|
884
|
+
}
|
|
885
|
+
item.audio = [];
|
|
886
|
+
this.finish(item);
|
|
887
|
+
}
|
|
888
|
+
finish(item, error) {
|
|
889
|
+
if (!this.items.has(item.id)) return;
|
|
890
|
+
item.done = true;
|
|
891
|
+
item.error = error;
|
|
892
|
+
this.items.delete(item.id);
|
|
893
|
+
if (item.responseId) this.responses.delete(item.responseId);
|
|
894
|
+
const wake = item.wake;
|
|
895
|
+
item.wake = void 0;
|
|
896
|
+
wake?.();
|
|
897
|
+
}
|
|
898
|
+
send(event) {
|
|
899
|
+
if (this.socket?.readyState === import_ws3.default.OPEN) {
|
|
900
|
+
this.socket.send(JSON.stringify(event));
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
sessionUpdate() {
|
|
904
|
+
const { instructions, speed } = this.options;
|
|
905
|
+
return {
|
|
906
|
+
type: "session.update",
|
|
907
|
+
session: {
|
|
908
|
+
type: "realtime",
|
|
909
|
+
model: this.options.model,
|
|
910
|
+
instructions: instructions ? `${READER_INSTRUCTIONS}
|
|
911
|
+
Delivery: ${instructions}` : READER_INSTRUCTIONS,
|
|
912
|
+
output_modalities: ["audio"],
|
|
913
|
+
reasoning: {
|
|
914
|
+
effort: this.options.reasoningEffort || DEFAULT_REASONING_EFFORT
|
|
915
|
+
},
|
|
916
|
+
audio: {
|
|
917
|
+
// Nothing is ever heard, there is no turn to detect
|
|
918
|
+
input: { turn_detection: null },
|
|
919
|
+
output: {
|
|
920
|
+
format: { type: "audio/pcm", rate: OPENAI_SAMPLE_RATE3 },
|
|
921
|
+
voice: this.options.voice,
|
|
922
|
+
...speed ? { speed: Math.min(MAX_SPEED, Math.max(MIN_SPEED, speed)) } : {}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
/**
|
|
929
|
+
* Opens the session, or hands over the one already open.
|
|
930
|
+
*
|
|
931
|
+
* A Realtime session lasts an hour at most. A closed one is opened again by
|
|
932
|
+
* the next sentence rather than on a timer, so an idle agent holds no
|
|
933
|
+
* connection.
|
|
934
|
+
*/
|
|
935
|
+
connect() {
|
|
936
|
+
if (this.destroyed) return Promise.reject(new Error("TTS destroyed"));
|
|
937
|
+
if (this.socket?.readyState === import_ws3.default.OPEN) {
|
|
938
|
+
return Promise.resolve(this.socket);
|
|
939
|
+
}
|
|
940
|
+
if (this.connecting) return this.connecting;
|
|
941
|
+
const { model, apiKey } = this.options;
|
|
942
|
+
const connecting = new Promise((resolve, reject) => {
|
|
943
|
+
const socket = new import_ws3.default(
|
|
944
|
+
`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`,
|
|
945
|
+
{ headers: { Authorization: `Bearer ${apiKey}` } }
|
|
946
|
+
);
|
|
947
|
+
this.socket = socket;
|
|
948
|
+
const timeout = setTimeout(() => {
|
|
949
|
+
this.log("Connection timeout");
|
|
950
|
+
this.closeSocket();
|
|
951
|
+
reject(new Error("WebSocket connection timeout"));
|
|
952
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT3);
|
|
953
|
+
socket.on("open", () => {
|
|
954
|
+
clearTimeout(timeout);
|
|
955
|
+
this.log("Connection opened");
|
|
956
|
+
socket.send(JSON.stringify(this.sessionUpdate()));
|
|
957
|
+
resolve(socket);
|
|
958
|
+
});
|
|
959
|
+
socket.on("error", (error) => {
|
|
960
|
+
clearTimeout(timeout);
|
|
961
|
+
this.log("WebSocket error:", error);
|
|
962
|
+
reject(new Error("WebSocket connection error"));
|
|
963
|
+
});
|
|
964
|
+
socket.on("close", (code, reason) => {
|
|
965
|
+
clearTimeout(timeout);
|
|
966
|
+
this.log("Connection closed", { code, reason: reason.toString() });
|
|
967
|
+
reject(new Error("WebSocket connection closed"));
|
|
968
|
+
this.closeSocket();
|
|
969
|
+
});
|
|
970
|
+
socket.on("message", (data) => {
|
|
971
|
+
try {
|
|
972
|
+
this.handleMessage(JSON.parse(data.toString()));
|
|
973
|
+
} catch (error) {
|
|
974
|
+
this.log("Error handling message", error);
|
|
975
|
+
}
|
|
976
|
+
});
|
|
977
|
+
});
|
|
978
|
+
this.connecting = connecting;
|
|
979
|
+
const settle = () => {
|
|
980
|
+
if (this.connecting === connecting) this.connecting = void 0;
|
|
981
|
+
};
|
|
982
|
+
connecting.then(settle, settle);
|
|
983
|
+
return connecting;
|
|
984
|
+
}
|
|
985
|
+
closeSocket() {
|
|
986
|
+
const socket = this.socket;
|
|
987
|
+
this.socket = void 0;
|
|
988
|
+
this.connecting = void 0;
|
|
989
|
+
for (const item of [...this.items.values()]) {
|
|
990
|
+
this.finish(item, new Error("Connection lost"));
|
|
991
|
+
}
|
|
992
|
+
if (!socket) return;
|
|
993
|
+
socket.removeAllListeners();
|
|
994
|
+
socket.on("error", () => {
|
|
995
|
+
});
|
|
996
|
+
if (socket.readyState === import_ws3.default.OPEN || socket.readyState === import_ws3.default.CONNECTING) {
|
|
997
|
+
socket.close(1e3);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
handleMessage(message) {
|
|
1001
|
+
switch (message.type) {
|
|
1002
|
+
case "response.created": {
|
|
1003
|
+
const id = message.response.id;
|
|
1004
|
+
const item = this.items.get(message.response.metadata?.micdrop_id);
|
|
1005
|
+
if (!item) {
|
|
1006
|
+
this.send({ type: "response.cancel", response_id: id });
|
|
1007
|
+
break;
|
|
1008
|
+
}
|
|
1009
|
+
item.responseId = id;
|
|
1010
|
+
this.responses.set(id, item);
|
|
1011
|
+
break;
|
|
1012
|
+
}
|
|
1013
|
+
case "response.output_audio.delta": {
|
|
1014
|
+
const item = this.responses.get(message.response_id);
|
|
1015
|
+
if (!item) break;
|
|
1016
|
+
const audio = item.resampler.process(
|
|
1017
|
+
Buffer.from(message.delta, "base64")
|
|
1018
|
+
);
|
|
1019
|
+
if (audio.length === 0) break;
|
|
1020
|
+
item.audio.push(audio);
|
|
1021
|
+
const wake = item.wake;
|
|
1022
|
+
item.wake = void 0;
|
|
1023
|
+
wake?.();
|
|
1024
|
+
break;
|
|
1025
|
+
}
|
|
1026
|
+
case "response.output_audio_transcript.done": {
|
|
1027
|
+
const item = this.responses.get(message.response_id);
|
|
1028
|
+
if (item && normalize(message.transcript) !== normalize(item.text)) {
|
|
1029
|
+
this.log(`Read differently: "${message.transcript}"`);
|
|
1030
|
+
}
|
|
1031
|
+
break;
|
|
1032
|
+
}
|
|
1033
|
+
case "response.done": {
|
|
1034
|
+
const response = message.response;
|
|
1035
|
+
const item = this.responses.get(response.id);
|
|
1036
|
+
if (!item) break;
|
|
1037
|
+
if (response.status === "failed") {
|
|
1038
|
+
this.finish(
|
|
1039
|
+
item,
|
|
1040
|
+
new Error(
|
|
1041
|
+
`Response failed: ${JSON.stringify(response.status_details)}`
|
|
1042
|
+
)
|
|
1043
|
+
);
|
|
1044
|
+
} else {
|
|
1045
|
+
this.finish(item);
|
|
1046
|
+
}
|
|
1047
|
+
break;
|
|
1048
|
+
}
|
|
1049
|
+
case "error": {
|
|
1050
|
+
if (message.error?.code === "response_cancel_not_active") break;
|
|
1051
|
+
this.log("Error:", message.error);
|
|
1052
|
+
const item = this.items.get(message.error?.event_id);
|
|
1053
|
+
if (item) this.finish(item, new Error(message.error?.message));
|
|
1054
|
+
break;
|
|
1055
|
+
}
|
|
1056
|
+
default:
|
|
1057
|
+
break;
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
};
|
|
1061
|
+
function normalize(text = "") {
|
|
1062
|
+
return text.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// src/OpenaiTTS.ts
|
|
1066
|
+
var DEFAULT_MODEL4 = "gpt-4o-mini-tts";
|
|
1067
|
+
var DEFAULT_VOICE2 = "alloy";
|
|
1068
|
+
var DEFAULT_REALTIME_VOICE = "marin";
|
|
1069
|
+
var OPENAI_SAMPLE_RATE4 = 24e3;
|
|
1070
|
+
var OUTPUT_SAMPLE_RATE2 = 16e3;
|
|
762
1071
|
var MAX_IN_FLIGHT = 2;
|
|
763
1072
|
var OPENING_CUSHION = 300 * 32;
|
|
764
|
-
var OpenaiTTS = class extends
|
|
1073
|
+
var OpenaiTTS = class extends import_server5.TTS {
|
|
765
1074
|
constructor(options) {
|
|
766
1075
|
super();
|
|
767
1076
|
this.options = options;
|
|
@@ -772,7 +1081,7 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
772
1081
|
// `counter`, which numbers the synthesis calls and must only move when there
|
|
773
1082
|
// is something to synthesize.
|
|
774
1083
|
this.generation = 0;
|
|
775
|
-
this.splitter = new
|
|
1084
|
+
this.splitter = new import_server5.SentenceSplitter();
|
|
776
1085
|
this.queue = [];
|
|
777
1086
|
// Sentences whose synthesis has not started
|
|
778
1087
|
this.pending = [];
|
|
@@ -781,9 +1090,25 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
781
1090
|
// Opening audio held back, see OPENING_CUSHION
|
|
782
1091
|
this.cushionBytes = 0;
|
|
783
1092
|
this.processing = false;
|
|
784
|
-
// More than one request is in flight at a time, so they are
|
|
785
|
-
this.
|
|
786
|
-
|
|
1093
|
+
// More than one request is in flight at a time, so they are stopped as a set.
|
|
1094
|
+
this.syntheses = /* @__PURE__ */ new Set();
|
|
1095
|
+
const model = options.model || DEFAULT_MODEL4;
|
|
1096
|
+
if (model.startsWith("gpt-realtime")) {
|
|
1097
|
+
this.realtime = new OpenaiRealtimeSpeech(
|
|
1098
|
+
{
|
|
1099
|
+
apiKey: "openai" in options ? options.openai.apiKey : options.apiKey,
|
|
1100
|
+
model,
|
|
1101
|
+
voice: options.voice || DEFAULT_REALTIME_VOICE,
|
|
1102
|
+
instructions: options.instructions,
|
|
1103
|
+
speed: options.speed,
|
|
1104
|
+
reasoningEffort: options.reasoningEffort,
|
|
1105
|
+
connectionTimeout: options.connectionTimeout
|
|
1106
|
+
},
|
|
1107
|
+
(...message) => this.log(...message)
|
|
1108
|
+
);
|
|
1109
|
+
} else {
|
|
1110
|
+
this.openai = "openai" in options ? options.openai : new import_openai2.default({ apiKey: options.apiKey });
|
|
1111
|
+
}
|
|
787
1112
|
}
|
|
788
1113
|
speak(textStream) {
|
|
789
1114
|
const generation = ++this.generation;
|
|
@@ -824,8 +1149,12 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
824
1149
|
this.pending = [];
|
|
825
1150
|
this.cushion = [];
|
|
826
1151
|
this.cushionBytes = 0;
|
|
827
|
-
this.
|
|
828
|
-
this.
|
|
1152
|
+
this.syntheses.forEach((synthesis) => synthesis.cancel());
|
|
1153
|
+
this.syntheses.clear();
|
|
1154
|
+
}
|
|
1155
|
+
destroy() {
|
|
1156
|
+
super.destroy();
|
|
1157
|
+
this.realtime?.destroy();
|
|
829
1158
|
}
|
|
830
1159
|
enqueue(counter, text) {
|
|
831
1160
|
this.queue.push({ counter, text });
|
|
@@ -834,7 +1163,7 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
834
1163
|
/**
|
|
835
1164
|
* Starts what can be started, then speaks what is ready.
|
|
836
1165
|
*
|
|
837
|
-
*
|
|
1166
|
+
* A request takes a few hundred milliseconds before its first byte, and
|
|
838
1167
|
* waiting for that between two sentences leaves a silence in the middle of
|
|
839
1168
|
* her voice. It is loudest at the very start, where a greeting is often one
|
|
840
1169
|
* short sentence: a syllable, a hole, then the rest of the answer. Asking for
|
|
@@ -848,7 +1177,9 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
848
1177
|
while (this.pending.length < MAX_IN_FLIGHT && this.queue.length > 0) {
|
|
849
1178
|
const item = this.queue.shift();
|
|
850
1179
|
if (item.counter !== this.counter) continue;
|
|
851
|
-
this.
|
|
1180
|
+
const synthesis = this.realtime ? this.realtime.synthesize(item.text) : this.requestSpeech(item.text);
|
|
1181
|
+
this.syntheses.add(synthesis);
|
|
1182
|
+
this.pending.push({ ...item, synthesis });
|
|
852
1183
|
}
|
|
853
1184
|
this.drain();
|
|
854
1185
|
}
|
|
@@ -865,28 +1196,47 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
865
1196
|
this.flushCushion();
|
|
866
1197
|
if (this.pending.length > 0 || this.queue.length > 0) this.pump();
|
|
867
1198
|
}
|
|
868
|
-
/**
|
|
869
|
-
|
|
1199
|
+
/**
|
|
1200
|
+
* Asks the speech endpoint for one sentence, without waiting for the answer.
|
|
1201
|
+
*
|
|
1202
|
+
* Only for tts-1, tts-1-hd and gpt-4o-mini-tts, until January 6, 2027.
|
|
1203
|
+
*/
|
|
1204
|
+
requestSpeech(text) {
|
|
870
1205
|
const controller = new AbortController();
|
|
871
|
-
this.abortControllers.add(controller);
|
|
872
1206
|
const response = this.openai.audio.speech.create(
|
|
873
1207
|
{
|
|
874
1208
|
model: this.options.model || DEFAULT_MODEL4,
|
|
875
1209
|
voice: this.options.voice || DEFAULT_VOICE2,
|
|
876
|
-
input:
|
|
1210
|
+
input: text,
|
|
877
1211
|
response_format: "pcm",
|
|
878
1212
|
...this.options.instructions ? { instructions: this.options.instructions } : {},
|
|
879
1213
|
...this.options.speed ? { speed: this.options.speed } : {}
|
|
880
1214
|
},
|
|
881
1215
|
{ signal: controller.signal }
|
|
882
|
-
)
|
|
883
|
-
|
|
884
|
-
this.log("Error synthesizing speech:", error);
|
|
885
|
-
this.emit("Failed", [item.text]);
|
|
886
|
-
}
|
|
887
|
-
return null;
|
|
1216
|
+
);
|
|
1217
|
+
response.catch(() => {
|
|
888
1218
|
});
|
|
889
|
-
|
|
1219
|
+
async function* read() {
|
|
1220
|
+
const { body } = await response;
|
|
1221
|
+
if (!body) return;
|
|
1222
|
+
const resampler = new import_server5.Pcm16Resampler(
|
|
1223
|
+
OPENAI_SAMPLE_RATE4,
|
|
1224
|
+
OUTPUT_SAMPLE_RATE2
|
|
1225
|
+
);
|
|
1226
|
+
const reader = body.getReader();
|
|
1227
|
+
try {
|
|
1228
|
+
while (true) {
|
|
1229
|
+
const { done, value } = await reader.read();
|
|
1230
|
+
if (done) break;
|
|
1231
|
+
const output = resampler.process(Buffer.from(value));
|
|
1232
|
+
if (output.length > 0) yield output;
|
|
1233
|
+
}
|
|
1234
|
+
} finally {
|
|
1235
|
+
reader.cancel().catch(() => {
|
|
1236
|
+
});
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
return { audio: read(), cancel: () => controller.abort() };
|
|
890
1240
|
}
|
|
891
1241
|
/** Holds the opening of an utterance back, then lets the rest through. */
|
|
892
1242
|
emitAudio(chunk) {
|
|
@@ -904,35 +1254,23 @@ var OpenaiTTS = class extends import_server4.TTS {
|
|
|
904
1254
|
this.cushion = [];
|
|
905
1255
|
for (const chunk of held) this.emit("Audio", chunk);
|
|
906
1256
|
}
|
|
907
|
-
/** Emits one sentence,
|
|
1257
|
+
/** Emits one sentence as its audio arrives, from a request started earlier. */
|
|
908
1258
|
async speakItem(item) {
|
|
909
1259
|
try {
|
|
910
|
-
const response = await item.response;
|
|
911
|
-
if (!response?.body) return;
|
|
912
1260
|
if (item.counter !== this.counter) return;
|
|
913
1261
|
this.log(`Synthesizing: "${item.text}"`);
|
|
914
|
-
const
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
);
|
|
918
|
-
const reader = response.body.getReader();
|
|
919
|
-
while (true) {
|
|
920
|
-
const { done, value } = await reader.read();
|
|
921
|
-
if (done) break;
|
|
922
|
-
if (item.counter !== this.counter) {
|
|
923
|
-
await reader.cancel();
|
|
924
|
-
break;
|
|
925
|
-
}
|
|
926
|
-
const output = resampler.process(Buffer.from(value));
|
|
927
|
-
if (output.length > 0) this.emitAudio(output);
|
|
1262
|
+
for await (const chunk of item.synthesis.audio) {
|
|
1263
|
+
if (item.counter !== this.counter) break;
|
|
1264
|
+
this.emitAudio(chunk);
|
|
928
1265
|
}
|
|
929
1266
|
} catch (error) {
|
|
930
|
-
if (
|
|
931
|
-
this.log("Error
|
|
1267
|
+
if (item.counter === this.counter) {
|
|
1268
|
+
this.log("Error synthesizing speech:", error);
|
|
932
1269
|
this.emit("Failed", [item.text]);
|
|
933
1270
|
}
|
|
934
1271
|
} finally {
|
|
935
|
-
|
|
1272
|
+
item.synthesis.cancel();
|
|
1273
|
+
this.syntheses.delete(item.synthesis);
|
|
936
1274
|
}
|
|
937
1275
|
}
|
|
938
1276
|
};
|