@micdrop/gradium 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/dist/index.d.mts +24 -31
- package/dist/index.d.ts +24 -31
- package/dist/index.js +277 -404
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +282 -396
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.mjs
CHANGED
|
@@ -1,17 +1,8 @@
|
|
|
1
1
|
// src/GradiumSTT.ts
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import WebSocket from "ws";
|
|
7
|
-
function closeSocket(socket) {
|
|
8
|
-
socket.removeAllListeners();
|
|
9
|
-
socket.on("error", () => {
|
|
10
|
-
});
|
|
11
|
-
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
12
|
-
socket.close(1e3);
|
|
13
|
-
}
|
|
14
|
-
}
|
|
2
|
+
import {
|
|
3
|
+
ReconnectingSocket,
|
|
4
|
+
STT
|
|
5
|
+
} from "@micdrop/server";
|
|
15
6
|
|
|
16
7
|
// src/types.ts
|
|
17
8
|
var DEFAULT_MODEL_NAME = "default";
|
|
@@ -21,124 +12,164 @@ var DEFAULT_REGION = "eu";
|
|
|
21
12
|
var DEFAULT_LANGUAGE = "any";
|
|
22
13
|
|
|
23
14
|
// src/GradiumSTT.ts
|
|
24
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
25
15
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
26
|
-
var DEFAULT_RETRY_DELAY = 1e3;
|
|
27
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
28
16
|
var FRAME_DURATION_MS = 80;
|
|
29
17
|
var DEFAULT_DELAY_IN_FRAMES = 10;
|
|
30
18
|
var GradiumSTT = class extends STT {
|
|
19
|
+
// Given by the server when ready
|
|
31
20
|
constructor(options) {
|
|
32
21
|
super();
|
|
33
22
|
this.options = options;
|
|
34
|
-
|
|
35
|
-
//
|
|
36
|
-
this.
|
|
37
|
-
// Accumulated text segments for the current utterance
|
|
23
|
+
// Utterances in the order their audio went out. The server answers in that
|
|
24
|
+
// order too, so every text segment belongs to the first one not flushed yet
|
|
25
|
+
this.utterances = [];
|
|
38
26
|
this.flushId = 0;
|
|
39
27
|
this.delayInFrames = DEFAULT_DELAY_IN_FRAMES;
|
|
40
|
-
|
|
41
|
-
this.
|
|
42
|
-
|
|
43
|
-
|
|
28
|
+
/** Sends the setup, and waits for the server to accept it */
|
|
29
|
+
this.handshake = {
|
|
30
|
+
start: (socket) => {
|
|
31
|
+
socket.send(
|
|
32
|
+
JSON.stringify({
|
|
33
|
+
type: "setup",
|
|
34
|
+
model_name: this.options.modelName ?? DEFAULT_MODEL_NAME,
|
|
35
|
+
input_format: this.options.inputFormat ?? DEFAULT_INPUT_FORMAT,
|
|
36
|
+
json_config: this.buildJsonConfig()
|
|
37
|
+
})
|
|
38
|
+
);
|
|
39
|
+
this.log("Sent setup");
|
|
40
|
+
},
|
|
41
|
+
read: (data) => {
|
|
42
|
+
let message;
|
|
43
|
+
try {
|
|
44
|
+
message = JSON.parse(data.toString());
|
|
45
|
+
} catch {
|
|
46
|
+
this.log("Error parsing message", data);
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
if (message.type === "error") {
|
|
50
|
+
throw new Error(`Setup refused: ${message.message}`);
|
|
51
|
+
}
|
|
52
|
+
if (message.type !== "ready") return false;
|
|
53
|
+
this.log("Server ready");
|
|
54
|
+
if (typeof message.delay_in_frames === "number") {
|
|
55
|
+
this.delayInFrames = message.delay_in_frames;
|
|
56
|
+
}
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
// A new connection knows nothing of the utterances under way: the audio read
|
|
61
|
+
// before the first connection opened, or lost with the previous one, goes out
|
|
62
|
+
// again whole, and the server transcribes it from the start
|
|
63
|
+
this.onOpen = () => {
|
|
64
|
+
this.utterances = this.utterances.filter((utterance) => !utterance.done);
|
|
65
|
+
if (this.utterances.length === 0) return;
|
|
66
|
+
this.log("Sending audio chunks again");
|
|
67
|
+
for (const utterance of this.utterances) {
|
|
68
|
+
utterance.transcript = "";
|
|
69
|
+
if (utterance.flushId === void 0) {
|
|
70
|
+
utterance.audio.forEach((chunk) => this.sendAudioChunk(chunk));
|
|
71
|
+
} else {
|
|
72
|
+
utterance.audio.slice(0, -1).forEach((chunk) => this.sendAudioChunk(chunk));
|
|
73
|
+
this.sendEnd(utterance);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
this.onMessage = (data) => {
|
|
78
|
+
let message;
|
|
79
|
+
try {
|
|
80
|
+
message = JSON.parse(data.toString());
|
|
81
|
+
} catch {
|
|
82
|
+
this.log("Error parsing message", data);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
switch (message.type) {
|
|
86
|
+
case "text": {
|
|
87
|
+
const utterance = this.utterances[0];
|
|
88
|
+
if (typeof message.text === "string" && utterance) {
|
|
89
|
+
utterance.transcript += (utterance.transcript ? " " : "") + message.text;
|
|
90
|
+
this.log(`Received text segment: "${message.text}"`);
|
|
91
|
+
}
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
case "flushed": {
|
|
95
|
+
const index = this.utterances.findIndex(
|
|
96
|
+
(utterance) => utterance.flushId === message.flush_id
|
|
97
|
+
);
|
|
98
|
+
if (index === -1) break;
|
|
99
|
+
const flushed = this.utterances.splice(0, index + 1);
|
|
100
|
+
flushed.forEach((utterance) => this.emitTranscript(utterance));
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "end_text":
|
|
104
|
+
break;
|
|
105
|
+
case "step":
|
|
106
|
+
break;
|
|
107
|
+
case "end_of_stream":
|
|
108
|
+
this.log("End of stream");
|
|
109
|
+
break;
|
|
110
|
+
case "error":
|
|
111
|
+
this.log("Error:", message.message, message.code);
|
|
112
|
+
break;
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
this.onFailed = () => {
|
|
116
|
+
const pending = this.utterances.filter((utterance) => !utterance.done);
|
|
117
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timeout));
|
|
118
|
+
this.utterances = [];
|
|
119
|
+
this.emit(
|
|
120
|
+
"Failed",
|
|
121
|
+
pending.flatMap((utterance) => utterance.audio)
|
|
122
|
+
);
|
|
123
|
+
};
|
|
124
|
+
const region = options.region ?? DEFAULT_REGION;
|
|
125
|
+
this.socket = new ReconnectingSocket({
|
|
126
|
+
url: `wss://${region}.api.gradium.ai/api/speech/asr`,
|
|
127
|
+
clientOptions: { headers: { "x-api-key": options.apiKey } },
|
|
128
|
+
connectionTimeout: options.connectionTimeout,
|
|
129
|
+
retryDelay: options.retryDelay,
|
|
130
|
+
maxRetry: options.maxRetry,
|
|
131
|
+
// The server opens the connection before checking the setup, so the
|
|
132
|
+
// connection only counts as open once it answers ready
|
|
133
|
+
handshake: this.handshake,
|
|
134
|
+
log: (...message) => this.log(...message)
|
|
135
|
+
});
|
|
136
|
+
this.socket.on("Open", this.onOpen);
|
|
137
|
+
this.socket.on("Message", this.onMessage);
|
|
138
|
+
this.socket.on("Failed", this.onFailed);
|
|
139
|
+
this.socket.connect();
|
|
44
140
|
}
|
|
45
141
|
transcribe(audioStream) {
|
|
46
|
-
let
|
|
47
|
-
audioStream.on("data",
|
|
48
|
-
if (!
|
|
49
|
-
|
|
50
|
-
this.
|
|
142
|
+
let utterance;
|
|
143
|
+
audioStream.on("data", (chunk) => {
|
|
144
|
+
if (!utterance) {
|
|
145
|
+
utterance = { audio: [], transcript: "", done: false };
|
|
146
|
+
this.utterances.push(utterance);
|
|
147
|
+
this.socket.connect();
|
|
148
|
+
}
|
|
149
|
+
if (utterance.done) return;
|
|
150
|
+
utterance.audio.push(chunk);
|
|
151
|
+
if (this.sendAudioChunk(chunk)) {
|
|
152
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
51
153
|
}
|
|
52
|
-
this.audioChunksPending.push(chunk);
|
|
53
|
-
await this.initPromise;
|
|
54
|
-
this.sendAudioChunk(chunk);
|
|
55
|
-
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
56
154
|
});
|
|
57
|
-
audioStream.on("end",
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
this.
|
|
61
|
-
|
|
62
|
-
this.
|
|
63
|
-
|
|
64
|
-
|
|
155
|
+
audioStream.on("end", () => {
|
|
156
|
+
if (!utterance || utterance.done) return;
|
|
157
|
+
const current = utterance;
|
|
158
|
+
current.audio.push(this.trailingSilence());
|
|
159
|
+
current.flushId = ++this.flushId;
|
|
160
|
+
if (this.socket.isOpen) this.sendEnd(current);
|
|
161
|
+
current.timeout = setTimeout(() => {
|
|
162
|
+
current.timeout = void 0;
|
|
65
163
|
this.log("Transcription timeout");
|
|
66
|
-
this.emitTranscript();
|
|
164
|
+
this.emitTranscript(current);
|
|
67
165
|
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
68
166
|
});
|
|
69
167
|
}
|
|
70
168
|
destroy() {
|
|
71
169
|
super.destroy();
|
|
72
|
-
this.
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
this.reconnectTimeout = void 0;
|
|
76
|
-
}
|
|
77
|
-
if (this.transcriptionTimeout) {
|
|
78
|
-
clearTimeout(this.transcriptionTimeout);
|
|
79
|
-
this.transcriptionTimeout = void 0;
|
|
80
|
-
}
|
|
81
|
-
if (this.connectionTimeout) {
|
|
82
|
-
clearTimeout(this.connectionTimeout);
|
|
83
|
-
this.connectionTimeout = void 0;
|
|
84
|
-
}
|
|
85
|
-
if (this.socket) closeSocket(this.socket);
|
|
86
|
-
this.socket = void 0;
|
|
87
|
-
}
|
|
88
|
-
connect() {
|
|
89
|
-
return this.initWS().catch((error) => {
|
|
90
|
-
console.error("[GradiumSTT] Connection error:", error);
|
|
91
|
-
this.reconnect();
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
getEndpoint() {
|
|
95
|
-
const region = this.options.region ?? DEFAULT_REGION;
|
|
96
|
-
return `wss://${region}.api.gradium.ai/api/speech/asr`;
|
|
97
|
-
}
|
|
98
|
-
async initWS() {
|
|
99
|
-
return new Promise((resolve, reject) => {
|
|
100
|
-
const socket = new WebSocket2(this.getEndpoint(), {
|
|
101
|
-
headers: {
|
|
102
|
-
"x-api-key": this.options.apiKey
|
|
103
|
-
}
|
|
104
|
-
});
|
|
105
|
-
this.socket = socket;
|
|
106
|
-
const timeout = setTimeout(() => {
|
|
107
|
-
this.log("Connection timeout");
|
|
108
|
-
closeSocket(socket);
|
|
109
|
-
this.socket = void 0;
|
|
110
|
-
reject(new Error("WebSocket connection timeout"));
|
|
111
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
112
|
-
this.connectionTimeout = timeout;
|
|
113
|
-
socket.addEventListener("open", () => {
|
|
114
|
-
clearTimeout(timeout);
|
|
115
|
-
this.sendSetup();
|
|
116
|
-
this.log("Connection opened");
|
|
117
|
-
resolve();
|
|
118
|
-
});
|
|
119
|
-
socket.addEventListener("error", (error) => {
|
|
120
|
-
clearTimeout(timeout);
|
|
121
|
-
this.log("WebSocket error:", error);
|
|
122
|
-
reject(new Error("WebSocket connection error"));
|
|
123
|
-
});
|
|
124
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
125
|
-
clearTimeout(timeout);
|
|
126
|
-
this.socket?.removeAllListeners();
|
|
127
|
-
this.socket = void 0;
|
|
128
|
-
if (code !== 1e3) {
|
|
129
|
-
this.reconnect();
|
|
130
|
-
} else {
|
|
131
|
-
this.log("Connection closed", { code, reason });
|
|
132
|
-
}
|
|
133
|
-
});
|
|
134
|
-
socket.addEventListener("message", (event) => {
|
|
135
|
-
try {
|
|
136
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
137
|
-
} catch {
|
|
138
|
-
this.log("Error parsing message", event.data);
|
|
139
|
-
}
|
|
140
|
-
});
|
|
141
|
-
});
|
|
170
|
+
this.socket.destroy();
|
|
171
|
+
this.utterances.forEach((utterance) => clearTimeout(utterance.timeout));
|
|
172
|
+
this.utterances = [];
|
|
142
173
|
}
|
|
143
174
|
buildJsonConfig() {
|
|
144
175
|
const config = { ...this.options.jsonConfig };
|
|
@@ -147,19 +178,8 @@ var GradiumSTT = class extends STT {
|
|
|
147
178
|
}
|
|
148
179
|
return config;
|
|
149
180
|
}
|
|
150
|
-
sendSetup() {
|
|
151
|
-
this.socket?.send(
|
|
152
|
-
JSON.stringify({
|
|
153
|
-
type: "setup",
|
|
154
|
-
model_name: this.options.modelName ?? DEFAULT_MODEL_NAME,
|
|
155
|
-
input_format: this.options.inputFormat ?? DEFAULT_INPUT_FORMAT,
|
|
156
|
-
json_config: this.buildJsonConfig()
|
|
157
|
-
})
|
|
158
|
-
);
|
|
159
|
-
this.log("Sent setup");
|
|
160
|
-
}
|
|
161
181
|
sendAudioChunk(chunk) {
|
|
162
|
-
this.socket
|
|
182
|
+
return this.socket.send(
|
|
163
183
|
JSON.stringify({
|
|
164
184
|
type: "audio",
|
|
165
185
|
audio: chunk.toString("base64")
|
|
@@ -167,98 +187,40 @@ var GradiumSTT = class extends STT {
|
|
|
167
187
|
);
|
|
168
188
|
}
|
|
169
189
|
/**
|
|
170
|
-
*
|
|
190
|
+
* Silence as long as the transcription lags behind, to follow the speech.
|
|
171
191
|
*
|
|
172
192
|
* The model writes each word a few frames after hearing it, and a flush only
|
|
173
193
|
* processes the audio already sent. With nothing after the last word, the
|
|
174
194
|
* server confirms the flush before writing that word, which then opens the
|
|
175
195
|
* next transcript.
|
|
176
196
|
*/
|
|
177
|
-
|
|
197
|
+
trailingSilence() {
|
|
178
198
|
const sampleRate = pcmSampleRate(
|
|
179
199
|
this.options.inputFormat ?? DEFAULT_INPUT_FORMAT
|
|
180
200
|
);
|
|
181
|
-
if (!sampleRate) return;
|
|
201
|
+
if (!sampleRate) return Buffer.alloc(0);
|
|
182
202
|
const durationMs = this.delayInFrames * FRAME_DURATION_MS;
|
|
183
203
|
const samples = Math.round(sampleRate * durationMs / 1e3);
|
|
184
|
-
|
|
185
|
-
this.audioChunksPending.push(silence);
|
|
186
|
-
this.sendAudioChunk(silence);
|
|
187
|
-
this.log(`Sent ${durationMs}ms of silence`);
|
|
204
|
+
return Buffer.alloc(samples * 2);
|
|
188
205
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
206
|
+
/** Sends the trailing silence, already the last audio chunk, and the flush */
|
|
207
|
+
sendEnd(utterance) {
|
|
208
|
+
const silence = utterance.audio[utterance.audio.length - 1];
|
|
209
|
+
if (silence.length) this.sendAudioChunk(silence);
|
|
210
|
+
this.socket.send(
|
|
211
|
+
JSON.stringify({ type: "flush", flush_id: utterance.flushId })
|
|
212
|
+
);
|
|
213
|
+
this.log(`Sent flush (flush_id=${utterance.flushId})`);
|
|
192
214
|
}
|
|
193
|
-
emitTranscript() {
|
|
194
|
-
|
|
215
|
+
emitTranscript(utterance) {
|
|
216
|
+
if (utterance.done) return;
|
|
217
|
+
utterance.done = true;
|
|
218
|
+
clearTimeout(utterance.timeout);
|
|
219
|
+
utterance.timeout = void 0;
|
|
220
|
+
utterance.audio = [];
|
|
221
|
+
const transcript = utterance.transcript.replace(/\s+/g, " ").trim();
|
|
195
222
|
this.log(`Received transcript: "${transcript}"`);
|
|
196
223
|
this.emit("Transcript", transcript);
|
|
197
|
-
this.transcript = "";
|
|
198
|
-
this.audioChunksPending.length = 0;
|
|
199
|
-
if (this.transcriptionTimeout) {
|
|
200
|
-
clearTimeout(this.transcriptionTimeout);
|
|
201
|
-
this.transcriptionTimeout = void 0;
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
handleMessage(message) {
|
|
205
|
-
switch (message.type) {
|
|
206
|
-
case "ready":
|
|
207
|
-
this.log("Server ready");
|
|
208
|
-
this.retryCount = 0;
|
|
209
|
-
if (typeof message.delay_in_frames === "number") {
|
|
210
|
-
this.delayInFrames = message.delay_in_frames;
|
|
211
|
-
}
|
|
212
|
-
break;
|
|
213
|
-
case "text":
|
|
214
|
-
if (typeof message.text === "string") {
|
|
215
|
-
this.transcript += (this.transcript ? " " : "") + message.text;
|
|
216
|
-
this.log(`Received text segment: "${message.text}"`);
|
|
217
|
-
}
|
|
218
|
-
break;
|
|
219
|
-
case "end_text":
|
|
220
|
-
break;
|
|
221
|
-
case "flushed":
|
|
222
|
-
if (message.flush_id === this.flushId) {
|
|
223
|
-
this.emitTranscript();
|
|
224
|
-
}
|
|
225
|
-
break;
|
|
226
|
-
case "step":
|
|
227
|
-
break;
|
|
228
|
-
case "end_of_stream":
|
|
229
|
-
this.log("End of stream");
|
|
230
|
-
break;
|
|
231
|
-
case "error":
|
|
232
|
-
this.log("Error:", message.message, message.code);
|
|
233
|
-
break;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
reconnect() {
|
|
237
|
-
if (this.destroyed) return;
|
|
238
|
-
this.retryCount++;
|
|
239
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
240
|
-
this.log("Max retries reached, giving up");
|
|
241
|
-
this.emit("Failed", this.audioChunksPending);
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
this.initPromise = new Promise((resolve) => {
|
|
245
|
-
this.log("Reconnecting...");
|
|
246
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
247
|
-
this.reconnectTimeout = void 0;
|
|
248
|
-
this.initWS().then(() => {
|
|
249
|
-
if (this.audioChunksPending.length > 0) {
|
|
250
|
-
this.log("Sending audio chunks again");
|
|
251
|
-
this.transcript = "";
|
|
252
|
-
this.audioChunksPending.forEach(
|
|
253
|
-
(chunk) => this.sendAudioChunk(chunk)
|
|
254
|
-
);
|
|
255
|
-
}
|
|
256
|
-
}).then(resolve).catch((error) => {
|
|
257
|
-
this.log("Reconnection error:", error);
|
|
258
|
-
this.reconnect();
|
|
259
|
-
});
|
|
260
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY);
|
|
261
|
-
});
|
|
262
224
|
}
|
|
263
225
|
};
|
|
264
226
|
function pcmSampleRate(format) {
|
|
@@ -268,202 +230,139 @@ function pcmSampleRate(format) {
|
|
|
268
230
|
}
|
|
269
231
|
|
|
270
232
|
// src/GradiumTTS.ts
|
|
271
|
-
import { TTS } from "@micdrop/server";
|
|
272
|
-
import WebSocket3 from "ws";
|
|
273
|
-
var DEFAULT_CONNECTION_TIMEOUT2 = 5e3;
|
|
274
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
275
|
-
var DEFAULT_MAX_RETRY2 = 3;
|
|
233
|
+
import { ReconnectingSocket as ReconnectingSocket2, TTS } from "@micdrop/server";
|
|
276
234
|
var GradiumTTS = class extends TTS {
|
|
277
235
|
constructor(options) {
|
|
278
236
|
super();
|
|
279
237
|
this.options = options;
|
|
238
|
+
// Numbers the audio requests, and only moves when there is something to say
|
|
280
239
|
this.counter = 0;
|
|
281
240
|
// Bumped by every speak() and every cancel(), so a request claimed late can
|
|
282
|
-
// tell whether it is still the one that should be heard
|
|
283
|
-
// `counter`, which numbers the audio requests and must only move when there
|
|
284
|
-
// is something to synthesize.
|
|
241
|
+
// tell whether it is still the one that should be heard
|
|
285
242
|
this.generation = 0;
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
this.
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
243
|
+
// A new connection has no session for the request under way: its setup and
|
|
244
|
+
// its words go out again, the setup first, with the same client_req_id
|
|
245
|
+
this.onOpen = () => {
|
|
246
|
+
const request = this.request;
|
|
247
|
+
if (!request) return;
|
|
248
|
+
this.log("Sending the request again");
|
|
249
|
+
this.sendSetup(request.id);
|
|
250
|
+
if (request.words) this.sendTranscript(request.words, request.id);
|
|
251
|
+
if (request.ended) this.sendEnd(request);
|
|
252
|
+
};
|
|
253
|
+
this.onMessage = (data) => {
|
|
254
|
+
let message;
|
|
255
|
+
try {
|
|
256
|
+
message = JSON.parse(data.toString());
|
|
257
|
+
} catch {
|
|
258
|
+
this.log("Error parsing message", data);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
switch (message.type) {
|
|
262
|
+
case "ready":
|
|
263
|
+
this.log("Server ready", message.request_id ?? "");
|
|
264
|
+
break;
|
|
265
|
+
case "audio":
|
|
266
|
+
this.processAudioMessage(message);
|
|
267
|
+
break;
|
|
268
|
+
case "end_of_stream":
|
|
269
|
+
if (!this.request || message.client_req_id !== void 0 && message.client_req_id !== this.request.id)
|
|
270
|
+
return;
|
|
271
|
+
this.log("Audio ended");
|
|
272
|
+
this.request = void 0;
|
|
273
|
+
break;
|
|
274
|
+
case "error":
|
|
275
|
+
this.log("Error:", message.message, message.code);
|
|
276
|
+
if (!this.request) break;
|
|
277
|
+
this.onFailed();
|
|
278
|
+
this.socket.restart();
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
this.onFailed = () => {
|
|
283
|
+
const request = this.request;
|
|
284
|
+
this.request = void 0;
|
|
285
|
+
const text = request ? request.words + request.partial : "";
|
|
286
|
+
this.emit("Failed", text ? [text] : []);
|
|
287
|
+
};
|
|
288
|
+
const region = options.region ?? DEFAULT_REGION;
|
|
289
|
+
this.socket = new ReconnectingSocket2({
|
|
290
|
+
url: `wss://${region}.api.gradium.ai/api/speech/tts`,
|
|
291
|
+
clientOptions: { headers: { "x-api-key": options.apiKey } },
|
|
292
|
+
connectionTimeout: options.connectionTimeout,
|
|
293
|
+
retryDelay: options.retryDelay,
|
|
294
|
+
maxRetry: options.maxRetry,
|
|
295
|
+
log: (...message) => this.log(...message)
|
|
298
296
|
});
|
|
297
|
+
this.socket.on("Open", this.onOpen);
|
|
298
|
+
this.socket.on("Message", this.onMessage);
|
|
299
|
+
this.socket.on("Failed", this.onFailed);
|
|
300
|
+
this.socket.connect();
|
|
299
301
|
}
|
|
300
302
|
speak(textStream) {
|
|
301
303
|
const generation = ++this.generation;
|
|
302
|
-
let
|
|
303
|
-
let clientReqId = "";
|
|
304
|
+
let request;
|
|
304
305
|
const claimRequest = () => {
|
|
305
|
-
if (
|
|
306
|
-
if (this.generation !== generation) return
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
this.
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
return true;
|
|
306
|
+
if (request) return request;
|
|
307
|
+
if (this.generation !== generation) return;
|
|
308
|
+
request = {
|
|
309
|
+
id: (++this.counter).toString(),
|
|
310
|
+
words: "",
|
|
311
|
+
partial: "",
|
|
312
|
+
ended: false
|
|
313
|
+
};
|
|
314
|
+
this.request = request;
|
|
315
|
+
this.sendSetup(request.id);
|
|
316
|
+
this.socket.connect();
|
|
317
|
+
return request;
|
|
318
318
|
};
|
|
319
|
-
textStream.on("data",
|
|
320
|
-
|
|
321
|
-
if (
|
|
319
|
+
textStream.on("data", (chunk) => {
|
|
320
|
+
const request2 = claimRequest();
|
|
321
|
+
if (!request2 || this.request !== request2) return;
|
|
322
322
|
const text = chunk.toString("utf-8").replace(/[\r\n ]+/g, " ");
|
|
323
|
-
this.textSent += text;
|
|
324
|
-
await this.initPromise;
|
|
325
|
-
if (counter !== this.counter) return;
|
|
326
323
|
const spaceIndex = text.lastIndexOf(" ");
|
|
327
324
|
if (spaceIndex === -1) {
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
this.sendTranscript(
|
|
331
|
-
this.textBuffer + text.slice(0, spaceIndex + 1),
|
|
332
|
-
clientReqId
|
|
333
|
-
);
|
|
334
|
-
this.textBuffer = text.slice(spaceIndex + 1);
|
|
325
|
+
request2.partial += text;
|
|
326
|
+
return;
|
|
335
327
|
}
|
|
328
|
+
const words = request2.partial + text.slice(0, spaceIndex + 1);
|
|
329
|
+
request2.partial = text.slice(spaceIndex + 1);
|
|
330
|
+
request2.words += words;
|
|
331
|
+
this.sendTranscript(words, request2.id);
|
|
336
332
|
});
|
|
337
333
|
textStream.on("error", (error) => {
|
|
338
|
-
if (!
|
|
339
|
-
this.log("Error in text stream,
|
|
340
|
-
this.
|
|
334
|
+
if (!request || this.request !== request) return;
|
|
335
|
+
this.log("Error in text stream, cancelling the audio", error);
|
|
336
|
+
this.stopRequest();
|
|
341
337
|
});
|
|
342
|
-
textStream.on("end",
|
|
343
|
-
if (!
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
if (this.textBuffer.trim()) {
|
|
347
|
-
this.sendTranscript(this.textBuffer, clientReqId);
|
|
348
|
-
this.textBuffer = "";
|
|
349
|
-
}
|
|
350
|
-
this.socket?.send(
|
|
351
|
-
JSON.stringify({
|
|
352
|
-
type: "end_of_stream",
|
|
353
|
-
client_req_id: clientReqId
|
|
354
|
-
})
|
|
355
|
-
);
|
|
356
|
-
this.log("Sent end_of_stream");
|
|
338
|
+
textStream.on("end", () => {
|
|
339
|
+
if (!request || this.request !== request) return;
|
|
340
|
+
request.ended = true;
|
|
341
|
+
this.sendEnd(request);
|
|
357
342
|
});
|
|
358
343
|
}
|
|
344
|
+
/**
|
|
345
|
+
* Stops the current utterance. Gradium can only stop a synthesis by closing
|
|
346
|
+
* its connection, so a new one opens right away.
|
|
347
|
+
*/
|
|
359
348
|
cancel() {
|
|
360
349
|
this.generation++;
|
|
361
|
-
if (!this.
|
|
350
|
+
if (!this.request) return;
|
|
362
351
|
this.log("Cancel");
|
|
363
|
-
this.
|
|
364
|
-
this.textSent = "";
|
|
365
|
-
this.textBuffer = "";
|
|
366
|
-
this.counter++;
|
|
367
|
-
if (this.reconnectTimeout) {
|
|
368
|
-
clearTimeout(this.reconnectTimeout);
|
|
369
|
-
this.reconnectTimeout = void 0;
|
|
370
|
-
}
|
|
371
|
-
if (this.connectionTimeout) {
|
|
372
|
-
clearTimeout(this.connectionTimeout);
|
|
373
|
-
this.connectionTimeout = void 0;
|
|
374
|
-
}
|
|
375
|
-
const socket = this.socket;
|
|
376
|
-
this.socket = void 0;
|
|
377
|
-
if (socket) closeSocket(socket);
|
|
378
|
-
this.connect();
|
|
352
|
+
this.stopRequest();
|
|
379
353
|
}
|
|
380
354
|
destroy() {
|
|
381
|
-
this.
|
|
355
|
+
this.socket.destroy();
|
|
382
356
|
super.destroy();
|
|
383
|
-
|
|
384
|
-
clearTimeout(this.reconnectTimeout);
|
|
385
|
-
this.reconnectTimeout = void 0;
|
|
386
|
-
}
|
|
387
|
-
if (this.connectionTimeout) {
|
|
388
|
-
clearTimeout(this.connectionTimeout);
|
|
389
|
-
this.connectionTimeout = void 0;
|
|
390
|
-
}
|
|
391
|
-
if (this.socket) closeSocket(this.socket);
|
|
392
|
-
this.socket = void 0;
|
|
393
|
-
this.isProcessing = false;
|
|
394
|
-
}
|
|
395
|
-
getEndpoint() {
|
|
396
|
-
const region = this.options.region ?? DEFAULT_REGION;
|
|
397
|
-
return `wss://${region}.api.gradium.ai/api/speech/tts`;
|
|
357
|
+
this.request = void 0;
|
|
398
358
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
"x-api-key": this.options.apiKey
|
|
404
|
-
}
|
|
405
|
-
});
|
|
406
|
-
this.socket = socket;
|
|
407
|
-
const timeout = setTimeout(() => {
|
|
408
|
-
this.log("Connection timeout");
|
|
409
|
-
closeSocket(socket);
|
|
410
|
-
this.socket = void 0;
|
|
411
|
-
reject(new Error("WebSocket connection timeout"));
|
|
412
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT2);
|
|
413
|
-
this.connectionTimeout = timeout;
|
|
414
|
-
socket.addEventListener("open", () => {
|
|
415
|
-
clearTimeout(timeout);
|
|
416
|
-
this.log("Connection opened");
|
|
417
|
-
resolve();
|
|
418
|
-
});
|
|
419
|
-
socket.addEventListener("error", (error) => {
|
|
420
|
-
clearTimeout(timeout);
|
|
421
|
-
this.log("WebSocket error:", error);
|
|
422
|
-
reject(new Error("WebSocket connection error"));
|
|
423
|
-
});
|
|
424
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
425
|
-
clearTimeout(timeout);
|
|
426
|
-
this.socket?.removeAllListeners();
|
|
427
|
-
this.socket = void 0;
|
|
428
|
-
if (code !== 1e3) {
|
|
429
|
-
this.reconnect();
|
|
430
|
-
} else {
|
|
431
|
-
this.log("Connection closed", { code, reason });
|
|
432
|
-
if (!this.isProcessing) {
|
|
433
|
-
this.reconnect();
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
});
|
|
437
|
-
socket.addEventListener("message", (event) => {
|
|
438
|
-
try {
|
|
439
|
-
const message = JSON.parse(event.data.toString());
|
|
440
|
-
switch (message.type) {
|
|
441
|
-
case "ready":
|
|
442
|
-
this.log("Server ready", message.request_id ?? "");
|
|
443
|
-
this.retryCount = 0;
|
|
444
|
-
break;
|
|
445
|
-
case "audio":
|
|
446
|
-
this.processAudioMessage(message);
|
|
447
|
-
break;
|
|
448
|
-
case "end_of_stream":
|
|
449
|
-
if ("client_req_id" in message && message.client_req_id !== this.counter.toString())
|
|
450
|
-
return;
|
|
451
|
-
this.log("Audio ended");
|
|
452
|
-
this.isProcessing = false;
|
|
453
|
-
this.textSent = "";
|
|
454
|
-
break;
|
|
455
|
-
case "error":
|
|
456
|
-
this.log("Error:", message.message, message.code);
|
|
457
|
-
break;
|
|
458
|
-
}
|
|
459
|
-
} catch {
|
|
460
|
-
this.log("Error parsing message", event.data);
|
|
461
|
-
}
|
|
462
|
-
});
|
|
463
|
-
});
|
|
359
|
+
/** Forgets the current request, and closes the connection that carries it */
|
|
360
|
+
stopRequest() {
|
|
361
|
+
this.request = void 0;
|
|
362
|
+
this.socket.restart();
|
|
464
363
|
}
|
|
465
364
|
sendSetup(clientReqId) {
|
|
466
|
-
this.socket
|
|
365
|
+
const sent = this.socket.send(
|
|
467
366
|
JSON.stringify({
|
|
468
367
|
type: "setup",
|
|
469
368
|
voice_id: this.options.voiceId,
|
|
@@ -474,50 +373,37 @@ var GradiumTTS = class extends TTS {
|
|
|
474
373
|
client_req_id: clientReqId
|
|
475
374
|
})
|
|
476
375
|
);
|
|
477
|
-
this.log(`Sent setup (client_req_id=${clientReqId})`);
|
|
376
|
+
if (sent) this.log(`Sent setup (client_req_id=${clientReqId})`);
|
|
478
377
|
}
|
|
479
378
|
sendTranscript(text, clientReqId) {
|
|
480
|
-
this.socket
|
|
379
|
+
const sent = this.socket.send(
|
|
481
380
|
JSON.stringify({
|
|
482
381
|
type: "text",
|
|
483
382
|
text,
|
|
484
383
|
client_req_id: clientReqId
|
|
485
384
|
})
|
|
486
385
|
);
|
|
487
|
-
this.log(`Sent transcript: "${text}"`);
|
|
386
|
+
if (sent) this.log(`Sent transcript: "${text}"`);
|
|
387
|
+
}
|
|
388
|
+
/** Sends the last partial word, then the end of the text */
|
|
389
|
+
sendEnd(request) {
|
|
390
|
+
if (request.partial.trim()) {
|
|
391
|
+
this.sendTranscript(request.partial, request.id);
|
|
392
|
+
}
|
|
393
|
+
const sent = this.socket.send(
|
|
394
|
+
JSON.stringify({
|
|
395
|
+
type: "end_of_stream",
|
|
396
|
+
client_req_id: request.id
|
|
397
|
+
})
|
|
398
|
+
);
|
|
399
|
+
if (sent) this.log("Sent end_of_stream");
|
|
488
400
|
}
|
|
489
401
|
processAudioMessage(message) {
|
|
490
|
-
if (message.client_req_id !== this.
|
|
402
|
+
if (!this.request || message.client_req_id !== this.request.id) return;
|
|
491
403
|
const chunk = Buffer.from(message.audio, "base64");
|
|
492
404
|
this.log(`Received audio chunk (${chunk.length} bytes)`);
|
|
493
405
|
this.emit("Audio", chunk);
|
|
494
406
|
}
|
|
495
|
-
reconnect() {
|
|
496
|
-
if (this.destroyed) return;
|
|
497
|
-
this.retryCount++;
|
|
498
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
499
|
-
this.log("Max retries reached, giving up");
|
|
500
|
-
this.emit("Failed", [this.textSent]);
|
|
501
|
-
return;
|
|
502
|
-
}
|
|
503
|
-
this.initPromise = new Promise((resolve) => {
|
|
504
|
-
this.log("Reconnecting...");
|
|
505
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
506
|
-
this.reconnectTimeout = void 0;
|
|
507
|
-
this.initWS().then(() => {
|
|
508
|
-
if (this.textSent.length > 0) {
|
|
509
|
-
const clientReqId = this.counter.toString();
|
|
510
|
-
this.log("Sending text chunks again");
|
|
511
|
-
this.sendSetup(clientReqId);
|
|
512
|
-
this.sendTranscript(this.textSent, clientReqId);
|
|
513
|
-
}
|
|
514
|
-
}).then(resolve).catch((error) => {
|
|
515
|
-
this.log("Reconnection error:", error);
|
|
516
|
-
this.reconnect();
|
|
517
|
-
});
|
|
518
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
519
|
-
});
|
|
520
|
-
}
|
|
521
407
|
};
|
|
522
408
|
export {
|
|
523
409
|
DEFAULT_INPUT_FORMAT,
|