@micdrop/mistral 1.1.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -2
- package/dist/index.d.mts +58 -20
- package/dist/index.d.ts +58 -20
- package/dist/index.js +339 -217
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +338 -217
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -31,7 +31,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
MistralAgent: () => MistralAgent,
|
|
34
|
-
MistralSTT: () => MistralSTT
|
|
34
|
+
MistralSTT: () => MistralSTT,
|
|
35
|
+
MistralTTS: () => MistralTTS
|
|
35
36
|
});
|
|
36
37
|
module.exports = __toCommonJS(index_exports);
|
|
37
38
|
|
|
@@ -43,17 +44,67 @@ var DEFAULT_MODEL = "mistral-large-latest";
|
|
|
43
44
|
var DEFAULT_MAX_RETRY = 3;
|
|
44
45
|
var DEFAULT_MAX_STEPS = 5;
|
|
45
46
|
var DEFAULT_RETRY_DELAY = 500;
|
|
47
|
+
var StepInterruptedError = class extends Error {
|
|
48
|
+
constructor(outcome) {
|
|
49
|
+
super("Answer interrupted");
|
|
50
|
+
this.outcome = outcome;
|
|
51
|
+
}
|
|
52
|
+
};
|
|
46
53
|
var MistralAgent = class extends import_server.Agent {
|
|
47
54
|
constructor(options) {
|
|
48
55
|
super(options);
|
|
49
56
|
this.mistral = new import_mistralai.Mistral({ apiKey: options.apiKey });
|
|
50
57
|
}
|
|
51
|
-
async generateAnswer(stream,
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
58
|
+
async generateAnswer(stream, signal) {
|
|
59
|
+
const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
|
|
60
|
+
const maxRetry = this.options.maxRetry || DEFAULT_MAX_RETRY;
|
|
61
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
62
|
+
let outcome;
|
|
63
|
+
for (let tryCount = 0; !outcome; tryCount++) {
|
|
64
|
+
try {
|
|
65
|
+
outcome = await this.generateStep(stream, signal);
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (signal.aborted) return;
|
|
68
|
+
if (error instanceof StepInterruptedError) {
|
|
69
|
+
outcome = error.outcome;
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
console.error("[MistralAgent] Error:", error);
|
|
73
|
+
if (tryCount >= maxRetry) {
|
|
74
|
+
this.log("Max retries reached, giving up");
|
|
75
|
+
this.emit("Failed");
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
await new Promise(
|
|
79
|
+
(resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
|
|
80
|
+
);
|
|
81
|
+
if (signal.aborted) return;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (signal.aborted) return;
|
|
85
|
+
if (outcome.fullAnswer) {
|
|
86
|
+
const { message, metadata } = this.extract(outcome.fullAnswer);
|
|
87
|
+
this.addAssistantMessage(message, metadata);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
if (outcome.skipAnswer) {
|
|
91
|
+
this.conversation.push({
|
|
92
|
+
role: "assistant",
|
|
93
|
+
content: " "
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
55
97
|
}
|
|
56
|
-
|
|
98
|
+
console.error("[MistralAgent] Max steps reached");
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
102
|
+
*
|
|
103
|
+
* It throws a plain error when the request failed before producing
|
|
104
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
105
|
+
* once something was written or done.
|
|
106
|
+
*/
|
|
107
|
+
async generateStep(stream, signal) {
|
|
57
108
|
if (this.conversation.length === 1) {
|
|
58
109
|
this.conversation.push({
|
|
59
110
|
role: "user",
|
|
@@ -62,30 +113,36 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
62
113
|
}
|
|
63
114
|
let extracting = false;
|
|
64
115
|
const extractOptions = this.getExtractOptions();
|
|
116
|
+
const outcome = { fullAnswer: "", skipAnswer: false };
|
|
117
|
+
let produced = false;
|
|
65
118
|
try {
|
|
66
|
-
const result = await this.mistral.chat.stream(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
119
|
+
const result = await this.mistral.chat.stream(
|
|
120
|
+
{
|
|
121
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
122
|
+
messages: this.buildMessages(),
|
|
123
|
+
tools: this.buildTools(),
|
|
124
|
+
...this.options.settings
|
|
125
|
+
},
|
|
126
|
+
{ signal }
|
|
127
|
+
);
|
|
74
128
|
for await (const event of result) {
|
|
75
|
-
if (
|
|
129
|
+
if (signal.aborted) return outcome;
|
|
76
130
|
const delta = event.data.choices[0]?.delta;
|
|
77
131
|
const chunk = delta?.content;
|
|
78
132
|
const toolCalls = delta?.toolCalls;
|
|
79
133
|
if (typeof chunk === "string") {
|
|
80
134
|
if (chunk === "") continue;
|
|
81
135
|
this.log(`Answer chunk: "${chunk}"`);
|
|
82
|
-
fullAnswer += chunk;
|
|
136
|
+
outcome.fullAnswer += chunk;
|
|
137
|
+
produced = true;
|
|
83
138
|
if (extractOptions) {
|
|
84
139
|
if (!extracting) {
|
|
85
|
-
const startTagIndex = fullAnswer.indexOf(
|
|
140
|
+
const startTagIndex = outcome.fullAnswer.indexOf(
|
|
141
|
+
extractOptions.startTag
|
|
142
|
+
);
|
|
86
143
|
if (startTagIndex !== -1) {
|
|
87
144
|
extracting = true;
|
|
88
|
-
const messagePart = fullAnswer.slice(0, startTagIndex).trimEnd();
|
|
145
|
+
const messagePart = outcome.fullAnswer.slice(0, startTagIndex).trimEnd();
|
|
89
146
|
stream.write(messagePart);
|
|
90
147
|
continue;
|
|
91
148
|
}
|
|
@@ -96,42 +153,27 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
96
153
|
stream.write(chunk);
|
|
97
154
|
} else if (toolCalls?.length) {
|
|
98
155
|
for (const toolCall of toolCalls) {
|
|
156
|
+
if (signal.aborted) return outcome;
|
|
157
|
+
produced = true;
|
|
99
158
|
const result2 = await this.executeTool({
|
|
100
159
|
role: "tool_call",
|
|
101
160
|
toolCallId: toolCall.id || "",
|
|
102
161
|
toolName: toolCall.function.name,
|
|
103
|
-
|
|
162
|
+
// The SDK hands the arguments over parsed or as JSON
|
|
163
|
+
parameters: typeof toolCall.function.arguments === "string" ? toolCall.function.arguments : JSON.stringify(toolCall.function.arguments)
|
|
104
164
|
});
|
|
105
165
|
if (result2.skipAnswer) {
|
|
106
|
-
skipAnswer = true;
|
|
166
|
+
outcome.skipAnswer = true;
|
|
107
167
|
}
|
|
108
168
|
}
|
|
109
169
|
}
|
|
110
170
|
}
|
|
111
|
-
if (answerCount !== this.answerCount) return;
|
|
112
|
-
if (fullAnswer) {
|
|
113
|
-
const { message, metadata } = this.extract(fullAnswer);
|
|
114
|
-
this.addAssistantMessage(message, metadata);
|
|
115
|
-
} else if (skipAnswer) {
|
|
116
|
-
this.conversation.push({
|
|
117
|
-
role: "assistant",
|
|
118
|
-
content: " "
|
|
119
|
-
});
|
|
120
|
-
} else {
|
|
121
|
-
await this.generateAnswer(stream, stepCount + 1);
|
|
122
|
-
}
|
|
123
171
|
} catch (error) {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
(resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
|
|
128
|
-
);
|
|
129
|
-
await this.generateAnswer(stream, stepCount, tryCount + 1);
|
|
130
|
-
} else {
|
|
131
|
-
this.log("Max retries reached, giving up");
|
|
132
|
-
this.emit("Failed");
|
|
133
|
-
}
|
|
172
|
+
if (!produced || signal.aborted) throw error;
|
|
173
|
+
console.error("[MistralAgent] Error after a partial answer:", error);
|
|
174
|
+
throw new StepInterruptedError(outcome);
|
|
134
175
|
}
|
|
176
|
+
return outcome;
|
|
135
177
|
}
|
|
136
178
|
buildMessages() {
|
|
137
179
|
return this.conversation.map(
|
|
@@ -153,7 +195,8 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
153
195
|
id: message.toolCallId,
|
|
154
196
|
function: {
|
|
155
197
|
name: message.toolName,
|
|
156
|
-
|
|
198
|
+
// Already the JSON the model wrote
|
|
199
|
+
arguments: message.parameters
|
|
157
200
|
}
|
|
158
201
|
}
|
|
159
202
|
]
|
|
@@ -186,18 +229,11 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
186
229
|
}
|
|
187
230
|
}));
|
|
188
231
|
}
|
|
189
|
-
cancel() {
|
|
190
|
-
if (!this.answering) return;
|
|
191
|
-
this.log("Cancel");
|
|
192
|
-
this.answering = false;
|
|
193
|
-
this.answerCount++;
|
|
194
|
-
}
|
|
195
232
|
};
|
|
196
233
|
|
|
197
234
|
// src/MistralSTT.ts
|
|
198
235
|
var import_server2 = require("@micdrop/server");
|
|
199
|
-
var
|
|
200
|
-
var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-2602";
|
|
236
|
+
var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-3-0";
|
|
201
237
|
var DEFAULT_ENCODING = "pcm_s16le";
|
|
202
238
|
var SAMPLE_RATE = 16e3;
|
|
203
239
|
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
@@ -208,62 +244,117 @@ var MistralSTT = class extends import_server2.STT {
|
|
|
208
244
|
constructor(options) {
|
|
209
245
|
super();
|
|
210
246
|
this.options = options;
|
|
211
|
-
|
|
212
|
-
//
|
|
213
|
-
this.
|
|
214
|
-
//
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
247
|
+
// Utterances not done yet, oldest first. The server answers them in the
|
|
248
|
+
// order their audio went out, so the transcript in progress is the oldest's.
|
|
249
|
+
this.utterances = [];
|
|
250
|
+
// Flushes of utterances that timed out on this session, whose deltas and
|
|
251
|
+
// transcription.done may still arrive and must not go to the next one. Each
|
|
252
|
+
// is forgotten after another timeout, so an answer that never comes does not
|
|
253
|
+
// swallow every transcript after it.
|
|
254
|
+
this.staleFlushes = [];
|
|
255
|
+
// A new session knows nothing of the previous one: every pending utterance
|
|
256
|
+
// is sent again in full and replays its whole transcript, so the deltas it
|
|
257
|
+
// gathered start over to avoid duplicating them
|
|
258
|
+
this.onOpen = ({ reconnected }) => {
|
|
259
|
+
this.staleFlushes = [];
|
|
260
|
+
const pending = this.utterances.filter(
|
|
261
|
+
(utterance) => utterance.chunks.length > 0
|
|
262
|
+
);
|
|
263
|
+
if (pending.length === 0) return;
|
|
264
|
+
this.log(
|
|
265
|
+
reconnected ? "Sending audio chunks again" : "Sending queued audio chunks"
|
|
266
|
+
);
|
|
267
|
+
for (const utterance of pending) {
|
|
268
|
+
utterance.text = "";
|
|
269
|
+
utterance.chunks.forEach((chunk) => this.sendAudioChunk(chunk));
|
|
270
|
+
if (utterance.status === "flushing") this.flushAudio(utterance);
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
this.onMessage = (data) => {
|
|
274
|
+
let message;
|
|
275
|
+
try {
|
|
276
|
+
message = JSON.parse(data.toString());
|
|
277
|
+
} catch {
|
|
278
|
+
this.log("Ignoring a message that is not JSON");
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
switch (message.type) {
|
|
282
|
+
case "transcription.text.delta": {
|
|
283
|
+
if (typeof message.text !== "string") break;
|
|
284
|
+
this.log(`Received transcript delta: "${message.text}"`);
|
|
285
|
+
if (this.hasStaleFlush()) break;
|
|
286
|
+
const utterance = this.utterances[0];
|
|
287
|
+
if (utterance) utterance.text += message.text;
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
case "transcription.done": {
|
|
291
|
+
if (this.hasStaleFlush()) {
|
|
292
|
+
this.staleFlushes.shift();
|
|
293
|
+
this.log("Ignoring the transcript of a timed out utterance");
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
const utterance = this.utterances[0];
|
|
297
|
+
if (utterance?.status !== "flushing") {
|
|
298
|
+
this.log("Ignoring an unexpected transcription.done");
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
const transcript = typeof message.text === "string" ? message.text : utterance.text;
|
|
302
|
+
this.log(`Received completed transcript: "${transcript}"`);
|
|
303
|
+
this.complete(utterance, transcript);
|
|
304
|
+
break;
|
|
305
|
+
}
|
|
306
|
+
case "error":
|
|
307
|
+
this.log("Error:", message.error ?? message);
|
|
308
|
+
break;
|
|
309
|
+
default:
|
|
310
|
+
break;
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
this.onFailed = () => {
|
|
314
|
+
const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
|
|
315
|
+
this.utterances.forEach((utterance) => this.dropUtterance(utterance));
|
|
316
|
+
this.utterances = [];
|
|
317
|
+
this.emit("Failed", chunks);
|
|
318
|
+
};
|
|
319
|
+
this.socket = new import_server2.ReconnectingSocket({
|
|
320
|
+
url: this.getURL(),
|
|
321
|
+
clientOptions: {
|
|
322
|
+
headers: {
|
|
323
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
324
|
+
}
|
|
325
|
+
},
|
|
326
|
+
connectionTimeout: this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT,
|
|
327
|
+
retryDelay: this.options.retryDelay ?? DEFAULT_RETRY_DELAY2,
|
|
328
|
+
maxRetry: this.options.maxRetry ?? DEFAULT_MAX_RETRY2,
|
|
329
|
+
// The audio format must be set before any audio is appended
|
|
330
|
+
handshake: {
|
|
331
|
+
start: (socket) => socket.send(this.getSessionUpdate())
|
|
332
|
+
},
|
|
333
|
+
log: (...message) => this.log(...message)
|
|
334
|
+
});
|
|
335
|
+
this.socket.on("Open", this.onOpen);
|
|
336
|
+
this.socket.on("Message", this.onMessage);
|
|
337
|
+
this.socket.on("Failed", this.onFailed);
|
|
338
|
+
this.socket.connect();
|
|
218
339
|
}
|
|
219
340
|
transcribe(audioStream) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
341
|
+
this.socket.connect();
|
|
342
|
+
const utterance = { status: "streaming", chunks: [], text: "" };
|
|
343
|
+
this.utterances.push(utterance);
|
|
344
|
+
audioStream.on("data", (chunk) => {
|
|
345
|
+
if (utterance.status !== "streaming") return;
|
|
346
|
+
utterance.chunks.push(chunk);
|
|
347
|
+
if (this.sendAudioChunk(chunk)) {
|
|
348
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
225
349
|
}
|
|
226
|
-
this.audioChunksPending.push(chunk);
|
|
227
|
-
await this.initPromise;
|
|
228
|
-
this.sendAudioChunk(chunk);
|
|
229
|
-
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
230
|
-
});
|
|
231
|
-
audioStream.on("end", async () => {
|
|
232
|
-
await this.initPromise;
|
|
233
|
-
if (this.audioChunksPending.length === 0) return;
|
|
234
|
-
this.flushAudio();
|
|
235
|
-
this.transcriptionTimeout = setTimeout(() => {
|
|
236
|
-
this.transcriptionTimeout = void 0;
|
|
237
|
-
this.log("Transcription timeout");
|
|
238
|
-
this.emit("Transcript", this.transcriptDelta.trim());
|
|
239
|
-
this.transcriptDelta = "";
|
|
240
|
-
this.audioChunksPending.length = 0;
|
|
241
|
-
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
242
350
|
});
|
|
351
|
+
audioStream.on("end", () => this.endUtterance(utterance));
|
|
243
352
|
}
|
|
244
353
|
destroy() {
|
|
245
354
|
super.destroy();
|
|
246
|
-
this.
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
this.reconnectTimeout = void 0;
|
|
250
|
-
}
|
|
251
|
-
if (this.transcriptionTimeout) {
|
|
252
|
-
clearTimeout(this.transcriptionTimeout);
|
|
253
|
-
this.transcriptionTimeout = void 0;
|
|
254
|
-
}
|
|
255
|
-
if (this.connectionTimeout) {
|
|
256
|
-
clearTimeout(this.connectionTimeout);
|
|
257
|
-
this.connectionTimeout = void 0;
|
|
258
|
-
}
|
|
259
|
-
if (this.socket) closeSocket(this.socket);
|
|
260
|
-
this.socket = void 0;
|
|
261
|
-
}
|
|
262
|
-
connect() {
|
|
263
|
-
return this.initWS().catch((error) => {
|
|
264
|
-
console.error("[MistralSTT] Connection error:", error);
|
|
265
|
-
this.reconnect();
|
|
266
|
-
});
|
|
355
|
+
this.socket.destroy();
|
|
356
|
+
this.utterances.forEach((utterance) => this.dropUtterance(utterance));
|
|
357
|
+
this.utterances = [];
|
|
267
358
|
}
|
|
268
359
|
getURL() {
|
|
269
360
|
const model = this.options.model || DEFAULT_MODEL2;
|
|
@@ -271,145 +362,176 @@ var MistralSTT = class extends import_server2.STT {
|
|
|
271
362
|
model
|
|
272
363
|
)}`;
|
|
273
364
|
}
|
|
274
|
-
|
|
275
|
-
return
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
365
|
+
getSessionUpdate() {
|
|
366
|
+
return JSON.stringify({
|
|
367
|
+
type: "session.update",
|
|
368
|
+
session: {
|
|
369
|
+
audio_format: {
|
|
370
|
+
encoding: this.options.encoding || DEFAULT_ENCODING,
|
|
371
|
+
sample_rate: SAMPLE_RATE
|
|
372
|
+
},
|
|
373
|
+
...this.options.targetStreamingDelayMs !== void 0 && {
|
|
374
|
+
target_streaming_delay_ms: this.options.targetStreamingDelayMs
|
|
279
375
|
}
|
|
280
|
-
}
|
|
281
|
-
this.socket = socket;
|
|
282
|
-
const timeout = setTimeout(() => {
|
|
283
|
-
this.log("Connection timeout");
|
|
284
|
-
closeSocket(socket);
|
|
285
|
-
this.socket = void 0;
|
|
286
|
-
reject(new Error("WebSocket connection timeout"));
|
|
287
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
288
|
-
this.connectionTimeout = timeout;
|
|
289
|
-
socket.addEventListener("open", () => {
|
|
290
|
-
clearTimeout(timeout);
|
|
291
|
-
this.sendSessionUpdate();
|
|
292
|
-
this.log("Connection opened");
|
|
293
|
-
resolve();
|
|
294
|
-
});
|
|
295
|
-
socket.addEventListener("error", (error) => {
|
|
296
|
-
clearTimeout(timeout);
|
|
297
|
-
this.log("WebSocket error:", error);
|
|
298
|
-
reject(new Error("WebSocket connection error"));
|
|
299
|
-
});
|
|
300
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
301
|
-
clearTimeout(timeout);
|
|
302
|
-
this.socket?.removeAllListeners();
|
|
303
|
-
this.socket = void 0;
|
|
304
|
-
if (code !== 1e3) {
|
|
305
|
-
this.reconnect();
|
|
306
|
-
} else {
|
|
307
|
-
this.log("Connection closed", { code, reason });
|
|
308
|
-
}
|
|
309
|
-
});
|
|
310
|
-
socket.addEventListener("message", (event) => {
|
|
311
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
312
|
-
});
|
|
376
|
+
}
|
|
313
377
|
});
|
|
314
378
|
}
|
|
315
|
-
|
|
316
|
-
if (
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
)
|
|
379
|
+
endUtterance(utterance) {
|
|
380
|
+
if (utterance.status !== "streaming") return;
|
|
381
|
+
if (utterance.chunks.length === 0) {
|
|
382
|
+
this.removeUtterance(utterance);
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
utterance.status = "flushing";
|
|
386
|
+
if (this.socket.isOpen) this.flushAudio(utterance);
|
|
387
|
+
}
|
|
388
|
+
flushAudio(utterance) {
|
|
389
|
+
if (!this.socket.send(JSON.stringify({ type: "input_audio.flush" }))) {
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
this.log("Flushed audio input");
|
|
393
|
+
clearTimeout(utterance.timeout);
|
|
394
|
+
utterance.timeout = setTimeout(() => {
|
|
395
|
+
this.log("Transcription timeout");
|
|
396
|
+
this.staleFlushes.push(Date.now() + this.transcriptionTimeout);
|
|
397
|
+
this.complete(utterance, utterance.text);
|
|
398
|
+
}, this.transcriptionTimeout);
|
|
399
|
+
}
|
|
400
|
+
get transcriptionTimeout() {
|
|
401
|
+
return this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT;
|
|
402
|
+
}
|
|
403
|
+
/** Whether an answer to a timed out flush may still be on its way */
|
|
404
|
+
hasStaleFlush() {
|
|
405
|
+
const now = Date.now();
|
|
406
|
+
while (this.staleFlushes.length > 0 && this.staleFlushes[0] <= now) {
|
|
407
|
+
this.log("Forgetting a flush that was never answered");
|
|
408
|
+
this.staleFlushes.shift();
|
|
409
|
+
}
|
|
410
|
+
return this.staleFlushes.length > 0;
|
|
411
|
+
}
|
|
412
|
+
complete(utterance, transcript) {
|
|
413
|
+
if (utterance.status === "done") return;
|
|
414
|
+
this.removeUtterance(utterance);
|
|
415
|
+
this.emit("Transcript", transcript.trim());
|
|
416
|
+
}
|
|
417
|
+
removeUtterance(utterance) {
|
|
418
|
+
this.dropUtterance(utterance);
|
|
419
|
+
this.utterances = this.utterances.filter((u) => u !== utterance);
|
|
420
|
+
}
|
|
421
|
+
dropUtterance(utterance) {
|
|
422
|
+
utterance.status = "done";
|
|
423
|
+
clearTimeout(utterance.timeout);
|
|
424
|
+
utterance.timeout = void 0;
|
|
331
425
|
}
|
|
332
426
|
sendAudioChunk(chunk) {
|
|
333
|
-
|
|
334
|
-
this.socket.send(
|
|
427
|
+
return this.socket.send(
|
|
335
428
|
JSON.stringify({
|
|
336
429
|
type: "input_audio.append",
|
|
337
430
|
audio: chunk.toString("base64")
|
|
338
431
|
})
|
|
339
432
|
);
|
|
340
433
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
// src/MistralTTS.ts
|
|
437
|
+
var import_server3 = require("@micdrop/server");
|
|
438
|
+
var DEFAULT_MODEL3 = "voxtral-mini-tts-3-0";
|
|
439
|
+
var DEFAULT_VOICE_ID = "voice_3150";
|
|
440
|
+
var DEFAULT_BASE_URL = "https://api.mistral.ai";
|
|
441
|
+
var MISTRAL_SAMPLE_RATE = 24e3;
|
|
442
|
+
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
443
|
+
var FLOAT32_BYTES = 4;
|
|
444
|
+
var MistralTTS = class extends import_server3.SentenceTTS {
|
|
445
|
+
constructor(options) {
|
|
446
|
+
super();
|
|
447
|
+
this.options = options;
|
|
345
448
|
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
449
|
+
async synthesize(text, signal) {
|
|
450
|
+
const response = await this.request(text, signal);
|
|
451
|
+
if (!response.body) throw new Error("Mistral answered without a body");
|
|
452
|
+
const resampler = new import_server3.Pcm16Resampler(
|
|
453
|
+
MISTRAL_SAMPLE_RATE,
|
|
454
|
+
OUTPUT_SAMPLE_RATE
|
|
455
|
+
);
|
|
456
|
+
let leftover = Buffer.alloc(0);
|
|
457
|
+
for await (const event of readEvents(response.body, signal)) {
|
|
458
|
+
if (event.type === "error") {
|
|
459
|
+
throw new Error(
|
|
460
|
+
`Speech generation failed: ${JSON.stringify(event.error ?? event)}`
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
if (event.type !== "speech.audio.delta" || !event.audio_data) continue;
|
|
464
|
+
const data = Buffer.concat([
|
|
465
|
+
leftover,
|
|
466
|
+
Buffer.from(event.audio_data, "base64")
|
|
467
|
+
]);
|
|
468
|
+
const length = data.length - data.length % FLOAT32_BYTES;
|
|
469
|
+
leftover = data.subarray(length);
|
|
470
|
+
const samples = new Float32Array(length / FLOAT32_BYTES);
|
|
471
|
+
for (let i = 0; i < samples.length; i++) {
|
|
472
|
+
samples[i] = data.readFloatLE(i * FLOAT32_BYTES);
|
|
365
473
|
}
|
|
366
|
-
|
|
367
|
-
this.log("Error:", message.error ?? message);
|
|
368
|
-
break;
|
|
369
|
-
default:
|
|
370
|
-
break;
|
|
474
|
+
if (!this.emitAudio(resampler.process((0, import_server3.float32ToPcm16)(samples)))) break;
|
|
371
475
|
}
|
|
476
|
+
return void 0;
|
|
372
477
|
}
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}).then(resolve).catch((error) => {
|
|
395
|
-
this.log("Reconnection error:", error);
|
|
396
|
-
this.reconnect();
|
|
397
|
-
});
|
|
398
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
478
|
+
async request(text, signal) {
|
|
479
|
+
const { options } = this;
|
|
480
|
+
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
481
|
+
const response = await fetch(`${baseUrl}/v1/audio/speech`, {
|
|
482
|
+
method: "POST",
|
|
483
|
+
headers: {
|
|
484
|
+
Authorization: `Bearer ${options.apiKey}`,
|
|
485
|
+
"Content-Type": "application/json",
|
|
486
|
+
Accept: "text/event-stream"
|
|
487
|
+
},
|
|
488
|
+
signal,
|
|
489
|
+
body: JSON.stringify({
|
|
490
|
+
...options.settings,
|
|
491
|
+
model: options.model || DEFAULT_MODEL3,
|
|
492
|
+
input: text,
|
|
493
|
+
...options.refAudio ? { ref_audio: options.refAudio } : { voice_id: options.voiceId || DEFAULT_VOICE_ID },
|
|
494
|
+
// Raw samples rather than wav, so chunks can be concatenated without a
|
|
495
|
+
// header in the middle of the audio
|
|
496
|
+
response_format: "pcm",
|
|
497
|
+
stream: true
|
|
498
|
+
})
|
|
399
499
|
});
|
|
500
|
+
if (!response.ok) {
|
|
501
|
+
const details = await response.text().catch(() => "");
|
|
502
|
+
throw new Error(
|
|
503
|
+
`Mistral answered ${response.status} ${response.statusText}: ${details}`
|
|
504
|
+
);
|
|
505
|
+
}
|
|
506
|
+
return response;
|
|
400
507
|
}
|
|
401
508
|
};
|
|
402
|
-
function
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
509
|
+
async function* readEvents(body, signal) {
|
|
510
|
+
const reader = body.getReader();
|
|
511
|
+
const decoder = new TextDecoder();
|
|
512
|
+
let buffer = "";
|
|
513
|
+
try {
|
|
514
|
+
while (!signal.aborted) {
|
|
515
|
+
const { done, value } = await reader.read();
|
|
516
|
+
if (done) break;
|
|
517
|
+
buffer += decoder.decode(value, { stream: true }).replace(/\r\n/g, "\n");
|
|
518
|
+
let end;
|
|
519
|
+
while ((end = buffer.indexOf("\n\n")) >= 0) {
|
|
520
|
+
const block = buffer.slice(0, end);
|
|
521
|
+
buffer = buffer.slice(end + 2);
|
|
522
|
+
const data = block.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trimStart()).join("\n");
|
|
523
|
+
if (data) yield JSON.parse(data);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
} finally {
|
|
527
|
+
reader.cancel().catch(() => {
|
|
528
|
+
});
|
|
408
529
|
}
|
|
409
530
|
}
|
|
410
531
|
// Annotate the CommonJS export names for ESM import in node:
|
|
411
532
|
0 && (module.exports = {
|
|
412
533
|
MistralAgent,
|
|
413
|
-
MistralSTT
|
|
534
|
+
MistralSTT,
|
|
535
|
+
MistralTTS
|
|
414
536
|
});
|
|
415
537
|
//# sourceMappingURL=index.js.map
|