@micdrop/gradium 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -3
- package/dist/index.d.mts +35 -27
- package/dist/index.d.ts +35 -27
- package/dist/index.js +304 -379
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +308 -371
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1,419 +1,368 @@
|
|
|
1
1
|
// src/GradiumSTT.ts
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import {
|
|
3
|
+
ReconnectingSocket,
|
|
4
|
+
STT
|
|
5
|
+
} from "@micdrop/server";
|
|
4
6
|
|
|
5
7
|
// src/types.ts
|
|
6
8
|
var DEFAULT_MODEL_NAME = "default";
|
|
7
9
|
var DEFAULT_OUTPUT_FORMAT = "pcm_16000";
|
|
8
10
|
var DEFAULT_INPUT_FORMAT = "pcm_16000";
|
|
9
11
|
var DEFAULT_REGION = "eu";
|
|
12
|
+
var DEFAULT_LANGUAGE = "any";
|
|
10
13
|
|
|
11
14
|
// src/GradiumSTT.ts
|
|
12
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
13
15
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
14
|
-
var
|
|
15
|
-
var
|
|
16
|
+
var FRAME_DURATION_MS = 80;
|
|
17
|
+
var DEFAULT_DELAY_IN_FRAMES = 10;
|
|
16
18
|
var GradiumSTT = class extends STT {
|
|
19
|
+
// Given by the server when ready
|
|
17
20
|
constructor(options) {
|
|
18
21
|
super();
|
|
19
22
|
this.options = options;
|
|
20
|
-
|
|
21
|
-
//
|
|
22
|
-
this.
|
|
23
|
-
// Accumulated text segments for the current utterance
|
|
23
|
+
// Utterances in the order their audio went out. The server answers in that
|
|
24
|
+
// order too, so every text segment belongs to the first one not flushed yet
|
|
25
|
+
this.utterances = [];
|
|
24
26
|
this.flushId = 0;
|
|
25
|
-
this.
|
|
26
|
-
|
|
27
|
-
this.
|
|
27
|
+
this.delayInFrames = DEFAULT_DELAY_IN_FRAMES;
|
|
28
|
+
/** Sends the setup, and waits for the server to accept it */
|
|
29
|
+
this.handshake = {
|
|
30
|
+
start: (socket) => {
|
|
31
|
+
socket.send(
|
|
32
|
+
JSON.stringify({
|
|
33
|
+
type: "setup",
|
|
34
|
+
model_name: this.options.modelName ?? DEFAULT_MODEL_NAME,
|
|
35
|
+
input_format: this.options.inputFormat ?? DEFAULT_INPUT_FORMAT,
|
|
36
|
+
json_config: this.buildJsonConfig()
|
|
37
|
+
})
|
|
38
|
+
);
|
|
39
|
+
this.log("Sent setup");
|
|
40
|
+
},
|
|
41
|
+
read: (data) => {
|
|
42
|
+
let message;
|
|
43
|
+
try {
|
|
44
|
+
message = JSON.parse(data.toString());
|
|
45
|
+
} catch {
|
|
46
|
+
this.log("Error parsing message", data);
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
if (message.type === "error") {
|
|
50
|
+
throw new Error(`Setup refused: ${message.message}`);
|
|
51
|
+
}
|
|
52
|
+
if (message.type !== "ready") return false;
|
|
53
|
+
this.log("Server ready");
|
|
54
|
+
if (typeof message.delay_in_frames === "number") {
|
|
55
|
+
this.delayInFrames = message.delay_in_frames;
|
|
56
|
+
}
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
// A new connection knows nothing of the utterances under way: the audio read
|
|
61
|
+
// before the first connection opened, or lost with the previous one, goes out
|
|
62
|
+
// again whole, and the server transcribes it from the start
|
|
63
|
+
this.onOpen = () => {
|
|
64
|
+
this.utterances = this.utterances.filter((utterance) => !utterance.done);
|
|
65
|
+
if (this.utterances.length === 0) return;
|
|
66
|
+
this.log("Sending audio chunks again");
|
|
67
|
+
for (const utterance of this.utterances) {
|
|
68
|
+
utterance.transcript = "";
|
|
69
|
+
if (utterance.flushId === void 0) {
|
|
70
|
+
utterance.audio.forEach((chunk) => this.sendAudioChunk(chunk));
|
|
71
|
+
} else {
|
|
72
|
+
utterance.audio.slice(0, -1).forEach((chunk) => this.sendAudioChunk(chunk));
|
|
73
|
+
this.sendEnd(utterance);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
this.onMessage = (data) => {
|
|
78
|
+
let message;
|
|
79
|
+
try {
|
|
80
|
+
message = JSON.parse(data.toString());
|
|
81
|
+
} catch {
|
|
82
|
+
this.log("Error parsing message", data);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
switch (message.type) {
|
|
86
|
+
case "text": {
|
|
87
|
+
const utterance = this.utterances[0];
|
|
88
|
+
if (typeof message.text === "string" && utterance) {
|
|
89
|
+
utterance.transcript += (utterance.transcript ? " " : "") + message.text;
|
|
90
|
+
this.log(`Received text segment: "${message.text}"`);
|
|
91
|
+
}
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
case "flushed": {
|
|
95
|
+
const index = this.utterances.findIndex(
|
|
96
|
+
(utterance) => utterance.flushId === message.flush_id
|
|
97
|
+
);
|
|
98
|
+
if (index === -1) break;
|
|
99
|
+
const flushed = this.utterances.splice(0, index + 1);
|
|
100
|
+
flushed.forEach((utterance) => this.emitTranscript(utterance));
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "end_text":
|
|
104
|
+
break;
|
|
105
|
+
case "step":
|
|
106
|
+
break;
|
|
107
|
+
case "end_of_stream":
|
|
108
|
+
this.log("End of stream");
|
|
109
|
+
break;
|
|
110
|
+
case "error":
|
|
111
|
+
this.log("Error:", message.message, message.code);
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
this.onFailed = () => {
|
|
116
|
+
const pending = this.utterances.filter((utterance) => !utterance.done);
|
|
117
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timeout));
|
|
118
|
+
this.utterances = [];
|
|
119
|
+
this.emit(
|
|
120
|
+
"Failed",
|
|
121
|
+
pending.flatMap((utterance) => utterance.audio)
|
|
122
|
+
);
|
|
123
|
+
};
|
|
124
|
+
const region = options.region ?? DEFAULT_REGION;
|
|
125
|
+
this.socket = new ReconnectingSocket({
|
|
126
|
+
url: `wss://${region}.api.gradium.ai/api/speech/asr`,
|
|
127
|
+
clientOptions: { headers: { "x-api-key": options.apiKey } },
|
|
128
|
+
connectionTimeout: options.connectionTimeout,
|
|
129
|
+
retryDelay: options.retryDelay,
|
|
130
|
+
maxRetry: options.maxRetry,
|
|
131
|
+
// The server opens the connection before checking the setup, so the
|
|
132
|
+
// connection only counts as open once it answers ready
|
|
133
|
+
handshake: this.handshake,
|
|
134
|
+
log: (...message) => this.log(...message)
|
|
135
|
+
});
|
|
136
|
+
this.socket.on("Open", this.onOpen);
|
|
137
|
+
this.socket.on("Message", this.onMessage);
|
|
138
|
+
this.socket.on("Failed", this.onFailed);
|
|
139
|
+
this.socket.connect();
|
|
28
140
|
}
|
|
29
141
|
transcribe(audioStream) {
|
|
30
|
-
let
|
|
31
|
-
audioStream.on("data",
|
|
32
|
-
if (!
|
|
33
|
-
|
|
34
|
-
this.
|
|
142
|
+
let utterance;
|
|
143
|
+
audioStream.on("data", (chunk) => {
|
|
144
|
+
if (!utterance) {
|
|
145
|
+
utterance = { audio: [], transcript: "", done: false };
|
|
146
|
+
this.utterances.push(utterance);
|
|
147
|
+
this.socket.connect();
|
|
148
|
+
}
|
|
149
|
+
if (utterance.done) return;
|
|
150
|
+
utterance.audio.push(chunk);
|
|
151
|
+
if (this.sendAudioChunk(chunk)) {
|
|
152
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
35
153
|
}
|
|
36
|
-
this.audioChunksPending.push(chunk);
|
|
37
|
-
await this.initPromise;
|
|
38
|
-
this.sendAudioChunk(chunk);
|
|
39
|
-
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
40
154
|
});
|
|
41
|
-
audioStream.on("end",
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
this.
|
|
46
|
-
this.
|
|
47
|
-
|
|
155
|
+
audioStream.on("end", () => {
|
|
156
|
+
if (!utterance || utterance.done) return;
|
|
157
|
+
const current = utterance;
|
|
158
|
+
current.audio.push(this.trailingSilence());
|
|
159
|
+
current.flushId = ++this.flushId;
|
|
160
|
+
if (this.socket.isOpen) this.sendEnd(current);
|
|
161
|
+
current.timeout = setTimeout(() => {
|
|
162
|
+
current.timeout = void 0;
|
|
48
163
|
this.log("Transcription timeout");
|
|
49
|
-
this.emitTranscript();
|
|
164
|
+
this.emitTranscript(current);
|
|
50
165
|
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
51
166
|
});
|
|
52
167
|
}
|
|
53
168
|
destroy() {
|
|
54
169
|
super.destroy();
|
|
55
|
-
this.
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
this.reconnectTimeout = void 0;
|
|
59
|
-
}
|
|
60
|
-
if (this.transcriptionTimeout) {
|
|
61
|
-
clearTimeout(this.transcriptionTimeout);
|
|
62
|
-
this.transcriptionTimeout = void 0;
|
|
63
|
-
}
|
|
64
|
-
this.socket?.removeAllListeners();
|
|
65
|
-
if (this.socket?.readyState === WebSocket.OPEN) {
|
|
66
|
-
this.socket?.close(1e3);
|
|
67
|
-
}
|
|
68
|
-
this.socket = void 0;
|
|
69
|
-
}
|
|
70
|
-
connect() {
|
|
71
|
-
return this.initWS().catch((error) => {
|
|
72
|
-
console.error("[GradiumSTT] Connection error:", error);
|
|
73
|
-
this.reconnect();
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
getEndpoint() {
|
|
77
|
-
const region = this.options.region ?? DEFAULT_REGION;
|
|
78
|
-
return `wss://${region}.api.gradium.ai/api/speech/asr`;
|
|
79
|
-
}
|
|
80
|
-
async initWS() {
|
|
81
|
-
return new Promise((resolve, reject) => {
|
|
82
|
-
const socket = new WebSocket(this.getEndpoint(), {
|
|
83
|
-
headers: {
|
|
84
|
-
"x-api-key": this.options.apiKey
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
this.socket = socket;
|
|
88
|
-
const timeout = setTimeout(() => {
|
|
89
|
-
this.log("Connection timeout");
|
|
90
|
-
socket.removeAllListeners();
|
|
91
|
-
socket.close();
|
|
92
|
-
this.socket = void 0;
|
|
93
|
-
reject(new Error("WebSocket connection timeout"));
|
|
94
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
95
|
-
socket.addEventListener("open", () => {
|
|
96
|
-
clearTimeout(timeout);
|
|
97
|
-
this.sendSetup();
|
|
98
|
-
this.log("Connection opened");
|
|
99
|
-
resolve();
|
|
100
|
-
});
|
|
101
|
-
socket.addEventListener("error", (error) => {
|
|
102
|
-
clearTimeout(timeout);
|
|
103
|
-
this.log("WebSocket error:", error);
|
|
104
|
-
reject(new Error("WebSocket connection error"));
|
|
105
|
-
});
|
|
106
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
107
|
-
clearTimeout(timeout);
|
|
108
|
-
this.socket?.removeAllListeners();
|
|
109
|
-
this.socket = void 0;
|
|
110
|
-
if (code !== 1e3) {
|
|
111
|
-
this.reconnect();
|
|
112
|
-
} else {
|
|
113
|
-
this.log("Connection closed", { code, reason });
|
|
114
|
-
}
|
|
115
|
-
});
|
|
116
|
-
socket.addEventListener("message", (event) => {
|
|
117
|
-
try {
|
|
118
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
119
|
-
} catch {
|
|
120
|
-
this.log("Error parsing message", event.data);
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
});
|
|
170
|
+
this.socket.destroy();
|
|
171
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timeout));
|
|
172
|
+
this.utterances = [];
|
|
124
173
|
}
|
|
125
174
|
buildJsonConfig() {
|
|
126
175
|
const config = { ...this.options.jsonConfig };
|
|
127
|
-
if (
|
|
128
|
-
config.language = this.options.language;
|
|
176
|
+
if (config.language === void 0) {
|
|
177
|
+
config.language = this.options.language ?? DEFAULT_LANGUAGE;
|
|
129
178
|
}
|
|
130
|
-
return
|
|
131
|
-
}
|
|
132
|
-
sendSetup() {
|
|
133
|
-
this.socket?.send(
|
|
134
|
-
JSON.stringify({
|
|
135
|
-
type: "setup",
|
|
136
|
-
model_name: this.options.modelName ?? DEFAULT_MODEL_NAME,
|
|
137
|
-
input_format: this.options.inputFormat ?? DEFAULT_INPUT_FORMAT,
|
|
138
|
-
json_config: this.buildJsonConfig()
|
|
139
|
-
})
|
|
140
|
-
);
|
|
141
|
-
this.log("Sent setup");
|
|
179
|
+
return config;
|
|
142
180
|
}
|
|
143
181
|
sendAudioChunk(chunk) {
|
|
144
|
-
this.socket
|
|
182
|
+
return this.socket.send(
|
|
145
183
|
JSON.stringify({
|
|
146
184
|
type: "audio",
|
|
147
185
|
audio: chunk.toString("base64")
|
|
148
186
|
})
|
|
149
187
|
);
|
|
150
188
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
189
|
+
/**
|
|
190
|
+
* Silence as long as the transcription lags behind, to follow the speech.
|
|
191
|
+
*
|
|
192
|
+
* The model writes each word a few frames after hearing it, and a flush only
|
|
193
|
+
* processes the audio already sent. With nothing after the last word, the
|
|
194
|
+
* server confirms the flush before writing that word, which then opens the
|
|
195
|
+
* next transcript.
|
|
196
|
+
*/
|
|
197
|
+
trailingSilence() {
|
|
198
|
+
const sampleRate = pcmSampleRate(
|
|
199
|
+
this.options.inputFormat ?? DEFAULT_INPUT_FORMAT
|
|
200
|
+
);
|
|
201
|
+
if (!sampleRate) return Buffer.alloc(0);
|
|
202
|
+
const durationMs = this.delayInFrames * FRAME_DURATION_MS;
|
|
203
|
+
const samples = Math.round(sampleRate * durationMs / 1e3);
|
|
204
|
+
return Buffer.alloc(samples * 2);
|
|
154
205
|
}
|
|
155
|
-
|
|
156
|
-
|
|
206
|
+
/** Sends the trailing silence, already the last audio chunk, and the flush */
|
|
207
|
+
sendEnd(utterance) {
|
|
208
|
+
const silence = utterance.audio[utterance.audio.length - 1];
|
|
209
|
+
if (silence.length) this.sendAudioChunk(silence);
|
|
210
|
+
this.socket.send(
|
|
211
|
+
JSON.stringify({ type: "flush", flush_id: utterance.flushId })
|
|
212
|
+
);
|
|
213
|
+
this.log(`Sent flush (flush_id=${utterance.flushId})`);
|
|
214
|
+
}
|
|
215
|
+
emitTranscript(utterance) {
|
|
216
|
+
if (utterance.done) return;
|
|
217
|
+
utterance.done = true;
|
|
218
|
+
clearTimeout(utterance.timeout);
|
|
219
|
+
utterance.timeout = void 0;
|
|
220
|
+
utterance.audio = [];
|
|
221
|
+
const transcript = utterance.transcript.replace(/\s+/g, " ").trim();
|
|
157
222
|
this.log(`Received transcript: "${transcript}"`);
|
|
158
223
|
this.emit("Transcript", transcript);
|
|
159
|
-
this.transcript = "";
|
|
160
|
-
this.audioChunksPending.length = 0;
|
|
161
|
-
if (this.transcriptionTimeout) {
|
|
162
|
-
clearTimeout(this.transcriptionTimeout);
|
|
163
|
-
this.transcriptionTimeout = void 0;
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
handleMessage(message) {
|
|
167
|
-
switch (message.type) {
|
|
168
|
-
case "ready":
|
|
169
|
-
this.log("Server ready");
|
|
170
|
-
break;
|
|
171
|
-
case "text":
|
|
172
|
-
if (typeof message.text === "string") {
|
|
173
|
-
this.transcript += (this.transcript ? " " : "") + message.text;
|
|
174
|
-
this.log(`Received text segment: "${message.text}"`);
|
|
175
|
-
}
|
|
176
|
-
break;
|
|
177
|
-
case "end_text":
|
|
178
|
-
break;
|
|
179
|
-
case "flushed":
|
|
180
|
-
if (message.flush_id === this.flushId) {
|
|
181
|
-
this.emitTranscript();
|
|
182
|
-
}
|
|
183
|
-
break;
|
|
184
|
-
case "step":
|
|
185
|
-
break;
|
|
186
|
-
case "end_of_stream":
|
|
187
|
-
this.log("End of stream");
|
|
188
|
-
break;
|
|
189
|
-
case "error":
|
|
190
|
-
this.log("Error:", message.message, message.code);
|
|
191
|
-
break;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
reconnect() {
|
|
195
|
-
if (this.destroyed) return;
|
|
196
|
-
this.retryCount++;
|
|
197
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
198
|
-
this.log("Max retries reached, giving up");
|
|
199
|
-
this.emit("Failed", this.audioChunksPending);
|
|
200
|
-
return;
|
|
201
|
-
}
|
|
202
|
-
this.initPromise = new Promise((resolve) => {
|
|
203
|
-
this.log("Reconnecting...");
|
|
204
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
205
|
-
this.reconnectTimeout = void 0;
|
|
206
|
-
this.initWS().then(() => {
|
|
207
|
-
this.retryCount = 0;
|
|
208
|
-
if (this.audioChunksPending.length > 0) {
|
|
209
|
-
this.log("Sending audio chunks again");
|
|
210
|
-
this.transcript = "";
|
|
211
|
-
this.audioChunksPending.forEach(
|
|
212
|
-
(chunk) => this.sendAudioChunk(chunk)
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
}).then(resolve).catch((error) => {
|
|
216
|
-
this.log("Reconnection error:", error);
|
|
217
|
-
this.reconnect();
|
|
218
|
-
});
|
|
219
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
220
|
-
});
|
|
221
224
|
}
|
|
222
225
|
};
|
|
226
|
+
function pcmSampleRate(format) {
|
|
227
|
+
if (format === "pcm") return 24e3;
|
|
228
|
+
const match = format.match(/^pcm_(\d+)$/);
|
|
229
|
+
return match ? Number(match[1]) : void 0;
|
|
230
|
+
}
|
|
223
231
|
|
|
224
232
|
// src/GradiumTTS.ts
|
|
225
|
-
import { TTS } from "@micdrop/server";
|
|
226
|
-
import WebSocket2 from "ws";
|
|
227
|
-
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
228
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
229
|
-
var DEFAULT_MAX_RETRY2 = 3;
|
|
233
|
+
import { ReconnectingSocket as ReconnectingSocket2, TTS } from "@micdrop/server";
|
|
230
234
|
var GradiumTTS = class extends TTS {
|
|
231
235
|
constructor(options) {
|
|
232
236
|
super();
|
|
233
237
|
this.options = options;
|
|
238
|
+
// Numbers the audio requests, and only moves when there is something to say
|
|
234
239
|
this.counter = 0;
|
|
235
240
|
// Bumped by every speak() and every cancel(), so a request claimed late can
|
|
236
|
-
// tell whether it is still the one that should be heard
|
|
237
|
-
// `counter`, which numbers the audio requests and must only move when there
|
|
238
|
-
// is something to synthesize.
|
|
241
|
+
// tell whether it is still the one that should be heard
|
|
239
242
|
this.generation = 0;
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
this.
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
243
|
+
// A new connection has no session for the request under way: its setup and
|
|
244
|
+
// its words go out again, the setup first, with the same client_req_id
|
|
245
|
+
this.onOpen = () => {
|
|
246
|
+
const request = this.request;
|
|
247
|
+
if (!request) return;
|
|
248
|
+
this.log("Sending the request again");
|
|
249
|
+
this.sendSetup(request.id);
|
|
250
|
+
if (request.words) this.sendTranscript(request.words, request.id);
|
|
251
|
+
if (request.ended) this.sendEnd(request);
|
|
252
|
+
};
|
|
253
|
+
this.onMessage = (data) => {
|
|
254
|
+
let message;
|
|
255
|
+
try {
|
|
256
|
+
message = JSON.parse(data.toString());
|
|
257
|
+
} catch {
|
|
258
|
+
this.log("Error parsing message", data);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
switch (message.type) {
|
|
262
|
+
case "ready":
|
|
263
|
+
this.log("Server ready", message.request_id ?? "");
|
|
264
|
+
break;
|
|
265
|
+
case "audio":
|
|
266
|
+
this.processAudioMessage(message);
|
|
267
|
+
break;
|
|
268
|
+
case "end_of_stream":
|
|
269
|
+
if (!this.request || message.client_req_id !== void 0 && message.client_req_id !== this.request.id)
|
|
270
|
+
return;
|
|
271
|
+
this.log("Audio ended");
|
|
272
|
+
this.request = void 0;
|
|
273
|
+
break;
|
|
274
|
+
case "error":
|
|
275
|
+
this.log("Error:", message.message, message.code);
|
|
276
|
+
if (!this.request) break;
|
|
277
|
+
this.onFailed();
|
|
278
|
+
this.socket.restart();
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
this.onFailed = () => {
|
|
283
|
+
const request = this.request;
|
|
284
|
+
this.request = void 0;
|
|
285
|
+
const text = request ? request.words + request.partial : "";
|
|
286
|
+
this.emit("Failed", text ? [text] : []);
|
|
287
|
+
};
|
|
288
|
+
const region = options.region ?? DEFAULT_REGION;
|
|
289
|
+
this.socket = new ReconnectingSocket2({
|
|
290
|
+
url: `wss://${region}.api.gradium.ai/api/speech/tts`,
|
|
291
|
+
clientOptions: { headers: { "x-api-key": options.apiKey } },
|
|
292
|
+
connectionTimeout: options.connectionTimeout,
|
|
293
|
+
retryDelay: options.retryDelay,
|
|
294
|
+
maxRetry: options.maxRetry,
|
|
295
|
+
log: (...message) => this.log(...message)
|
|
250
296
|
});
|
|
297
|
+
this.socket.on("Open", this.onOpen);
|
|
298
|
+
this.socket.on("Message", this.onMessage);
|
|
299
|
+
this.socket.on("Failed", this.onFailed);
|
|
300
|
+
this.socket.connect();
|
|
251
301
|
}
|
|
252
302
|
speak(textStream) {
|
|
253
303
|
const generation = ++this.generation;
|
|
254
|
-
let
|
|
255
|
-
let clientReqId = "";
|
|
304
|
+
let request;
|
|
256
305
|
const claimRequest = () => {
|
|
257
|
-
if (
|
|
258
|
-
if (this.generation !== generation) return
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
this.
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
return true;
|
|
306
|
+
if (request) return request;
|
|
307
|
+
if (this.generation !== generation) return;
|
|
308
|
+
request = {
|
|
309
|
+
id: (++this.counter).toString(),
|
|
310
|
+
words: "",
|
|
311
|
+
partial: "",
|
|
312
|
+
ended: false
|
|
313
|
+
};
|
|
314
|
+
this.request = request;
|
|
315
|
+
this.sendSetup(request.id);
|
|
316
|
+
this.socket.connect();
|
|
317
|
+
return request;
|
|
270
318
|
};
|
|
271
|
-
textStream.on("data",
|
|
272
|
-
|
|
273
|
-
if (
|
|
319
|
+
textStream.on("data", (chunk) => {
|
|
320
|
+
const request2 = claimRequest();
|
|
321
|
+
if (!request2 || this.request !== request2) return;
|
|
274
322
|
const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
|
|
275
|
-
this.textSent += text;
|
|
276
|
-
await this.initPromise;
|
|
277
|
-
if (counter !== this.counter) return;
|
|
278
323
|
const spaceIndex = text.lastIndexOf(" ");
|
|
279
324
|
if (spaceIndex === -1) {
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
this.sendTranscript(
|
|
283
|
-
this.textBuffer + text.slice(0, spaceIndex + 1),
|
|
284
|
-
clientReqId
|
|
285
|
-
);
|
|
286
|
-
this.textBuffer = text.slice(spaceIndex + 1);
|
|
325
|
+
request2.partial += text;
|
|
326
|
+
return;
|
|
287
327
|
}
|
|
328
|
+
const words = request2.partial + text.slice(0, spaceIndex + 1);
|
|
329
|
+
request2.partial = text.slice(spaceIndex + 1);
|
|
330
|
+
request2.words += words;
|
|
331
|
+
this.sendTranscript(words, request2.id);
|
|
288
332
|
});
|
|
289
333
|
textStream.on("error", (error) => {
|
|
290
|
-
if (!
|
|
291
|
-
this.log("Error in text stream,
|
|
292
|
-
this.
|
|
334
|
+
if (!request || this.request !== request) return;
|
|
335
|
+
this.log("Error in text stream, cancelling the audio", error);
|
|
336
|
+
this.stopRequest();
|
|
293
337
|
});
|
|
294
|
-
textStream.on("end",
|
|
295
|
-
if (!
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
if (this.textBuffer.trim()) {
|
|
299
|
-
this.sendTranscript(this.textBuffer, clientReqId);
|
|
300
|
-
this.textBuffer = "";
|
|
301
|
-
}
|
|
302
|
-
this.socket?.send(
|
|
303
|
-
JSON.stringify({
|
|
304
|
-
type: "end_of_stream",
|
|
305
|
-
client_req_id: clientReqId
|
|
306
|
-
})
|
|
307
|
-
);
|
|
308
|
-
this.log("Sent end_of_stream");
|
|
338
|
+
textStream.on("end", () => {
|
|
339
|
+
if (!request || this.request !== request) return;
|
|
340
|
+
request.ended = true;
|
|
341
|
+
this.sendEnd(request);
|
|
309
342
|
});
|
|
310
343
|
}
|
|
344
|
+
/**
|
|
345
|
+
* Stops the current utterance. Gradium can only stop a synthesis by closing
|
|
346
|
+
* its connection, so a new one opens right away.
|
|
347
|
+
*/
|
|
311
348
|
cancel() {
|
|
312
349
|
this.generation++;
|
|
313
|
-
if (!this.
|
|
350
|
+
if (!this.request) return;
|
|
314
351
|
this.log("Cancel");
|
|
315
|
-
this.
|
|
316
|
-
this.textSent = "";
|
|
317
|
-
this.textBuffer = "";
|
|
318
|
-
this.counter++;
|
|
319
|
-
if (this.reconnectTimeout) {
|
|
320
|
-
clearTimeout(this.reconnectTimeout);
|
|
321
|
-
this.reconnectTimeout = void 0;
|
|
322
|
-
}
|
|
323
|
-
const socket = this.socket;
|
|
324
|
-
this.socket = void 0;
|
|
325
|
-
if (socket) {
|
|
326
|
-
socket.removeAllListeners();
|
|
327
|
-
if (socket.readyState === WebSocket2.OPEN || socket.readyState === WebSocket2.CONNECTING) {
|
|
328
|
-
socket.close(1e3);
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
this.connect();
|
|
352
|
+
this.stopRequest();
|
|
332
353
|
}
|
|
333
354
|
destroy() {
|
|
355
|
+
this.socket.destroy();
|
|
334
356
|
super.destroy();
|
|
335
|
-
|
|
336
|
-
clearTimeout(this.reconnectTimeout);
|
|
337
|
-
this.reconnectTimeout = void 0;
|
|
338
|
-
}
|
|
339
|
-
this.socket?.removeAllListeners();
|
|
340
|
-
if (this.socket?.readyState === WebSocket2.OPEN) {
|
|
341
|
-
this.socket?.close(1e3);
|
|
342
|
-
}
|
|
343
|
-
this.socket = void 0;
|
|
344
|
-
this.isProcessing = false;
|
|
357
|
+
this.request = void 0;
|
|
345
358
|
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
async initWS() {
|
|
351
|
-
return new Promise((resolve, reject) => {
|
|
352
|
-
const socket = new WebSocket2(this.getEndpoint(), {
|
|
353
|
-
headers: {
|
|
354
|
-
"x-api-key": this.options.apiKey
|
|
355
|
-
}
|
|
356
|
-
});
|
|
357
|
-
this.socket = socket;
|
|
358
|
-
const timeout = setTimeout(() => {
|
|
359
|
-
this.log("Connection timeout");
|
|
360
|
-
socket.removeAllListeners();
|
|
361
|
-
socket.close();
|
|
362
|
-
this.socket = void 0;
|
|
363
|
-
reject(new Error("WebSocket connection timeout"));
|
|
364
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
365
|
-
socket.addEventListener("open", () => {
|
|
366
|
-
clearTimeout(timeout);
|
|
367
|
-
this.log("Connection opened");
|
|
368
|
-
resolve();
|
|
369
|
-
});
|
|
370
|
-
socket.addEventListener("error", (error) => {
|
|
371
|
-
clearTimeout(timeout);
|
|
372
|
-
this.log("WebSocket error:", error);
|
|
373
|
-
reject(new Error("WebSocket connection error"));
|
|
374
|
-
});
|
|
375
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
376
|
-
clearTimeout(timeout);
|
|
377
|
-
this.socket?.removeAllListeners();
|
|
378
|
-
this.socket = void 0;
|
|
379
|
-
if (code !== 1e3) {
|
|
380
|
-
this.reconnect();
|
|
381
|
-
} else {
|
|
382
|
-
this.log("Connection closed", { code, reason });
|
|
383
|
-
if (!this.isProcessing) {
|
|
384
|
-
this.reconnect();
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
});
|
|
388
|
-
socket.addEventListener("message", (event) => {
|
|
389
|
-
try {
|
|
390
|
-
const message = JSON.parse(event.data.toString());
|
|
391
|
-
switch (message.type) {
|
|
392
|
-
case "ready":
|
|
393
|
-
this.log("Server ready", message.request_id ?? "");
|
|
394
|
-
break;
|
|
395
|
-
case "audio":
|
|
396
|
-
this.processAudioMessage(message);
|
|
397
|
-
break;
|
|
398
|
-
case "end_of_stream":
|
|
399
|
-
if ("client_req_id" in message && message.client_req_id !== this.counter.toString())
|
|
400
|
-
return;
|
|
401
|
-
this.log("Audio ended");
|
|
402
|
-
this.isProcessing = false;
|
|
403
|
-
this.textSent = "";
|
|
404
|
-
break;
|
|
405
|
-
case "error":
|
|
406
|
-
this.log("Error:", message.message, message.code);
|
|
407
|
-
break;
|
|
408
|
-
}
|
|
409
|
-
} catch {
|
|
410
|
-
this.log("Error parsing message", event.data);
|
|
411
|
-
}
|
|
412
|
-
});
|
|
413
|
-
});
|
|
359
|
+
/** Forgets the current request, and closes the connection that carries it */
|
|
360
|
+
stopRequest() {
|
|
361
|
+
this.request = void 0;
|
|
362
|
+
this.socket.restart();
|
|
414
363
|
}
|
|
415
364
|
sendSetup(clientReqId) {
|
|
416
|
-
this.socket
|
|
365
|
+
const sent = this.socket.send(
|
|
417
366
|
JSON.stringify({
|
|
418
367
|
type: "setup",
|
|
419
368
|
voice_id: this.options.voiceId,
|
|
@@ -424,53 +373,41 @@ var GradiumTTS = class extends TTS {
|
|
|
424
373
|
client_req_id: clientReqId
|
|
425
374
|
})
|
|
426
375
|
);
|
|
427
|
-
this.log(`Sent setup (client_req_id=${clientReqId})`);
|
|
376
|
+
if (sent) this.log(`Sent setup (client_req_id=${clientReqId})`);
|
|
428
377
|
}
|
|
429
378
|
sendTranscript(text, clientReqId) {
|
|
430
|
-
this.socket
|
|
379
|
+
const sent = this.socket.send(
|
|
431
380
|
JSON.stringify({
|
|
432
381
|
type: "text",
|
|
433
382
|
text,
|
|
434
383
|
client_req_id: clientReqId
|
|
435
384
|
})
|
|
436
385
|
);
|
|
437
|
-
this.log(`Sent transcript: "${text}"`);
|
|
386
|
+
if (sent) this.log(`Sent transcript: "${text}"`);
|
|
387
|
+
}
|
|
388
|
+
/** Sends the last partial word, then the end of the text */
|
|
389
|
+
sendEnd(request) {
|
|
390
|
+
if (request.partial.trim()) {
|
|
391
|
+
this.sendTranscript(request.partial, request.id);
|
|
392
|
+
}
|
|
393
|
+
const sent = this.socket.send(
|
|
394
|
+
JSON.stringify({
|
|
395
|
+
type: "end_of_stream",
|
|
396
|
+
client_req_id: request.id
|
|
397
|
+
})
|
|
398
|
+
);
|
|
399
|
+
if (sent) this.log("Sent end_of_stream");
|
|
438
400
|
}
|
|
439
401
|
processAudioMessage(message) {
|
|
440
|
-
if (message.client_req_id !== this.
|
|
402
|
+
if (!this.request || message.client_req_id !== this.request.id) return;
|
|
441
403
|
const chunk = Buffer.from(message.audio, "base64");
|
|
442
404
|
this.log(`Received audio chunk (${chunk.length} bytes)`);
|
|
443
405
|
this.emit("Audio", chunk);
|
|
444
406
|
}
|
|
445
|
-
reconnect() {
|
|
446
|
-
this.retryCount++;
|
|
447
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
448
|
-
this.log("Max retries reached, giving up");
|
|
449
|
-
this.emit("Failed", [this.textSent]);
|
|
450
|
-
return;
|
|
451
|
-
}
|
|
452
|
-
this.initPromise = new Promise((resolve) => {
|
|
453
|
-
this.log("Reconnecting...");
|
|
454
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
455
|
-
this.reconnectTimeout = void 0;
|
|
456
|
-
this.initWS().then(() => {
|
|
457
|
-
this.retryCount = 0;
|
|
458
|
-
if (this.textSent.length > 0) {
|
|
459
|
-
const clientReqId = this.counter.toString();
|
|
460
|
-
this.log("Sending text chunks again");
|
|
461
|
-
this.sendSetup(clientReqId);
|
|
462
|
-
this.sendTranscript(this.textSent, clientReqId);
|
|
463
|
-
}
|
|
464
|
-
}).then(resolve).catch((error) => {
|
|
465
|
-
this.log("Reconnection error:", error);
|
|
466
|
-
this.reconnect();
|
|
467
|
-
});
|
|
468
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
469
|
-
});
|
|
470
|
-
}
|
|
471
407
|
};
|
|
472
408
|
export {
|
|
473
409
|
DEFAULT_INPUT_FORMAT,
|
|
410
|
+
DEFAULT_LANGUAGE,
|
|
474
411
|
DEFAULT_MODEL_NAME,
|
|
475
412
|
DEFAULT_OUTPUT_FORMAT,
|
|
476
413
|
DEFAULT_REGION,
|