@micdrop/mistral 1.2.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/dist/index.d.mts +75 -18
- package/dist/index.d.ts +75 -18
- package/dist/index.js +424 -261
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +426 -257
- package/dist/index.mjs.map +1 -1
- package/package.json +8 -4
package/dist/index.mjs
CHANGED
|
@@ -2,99 +2,209 @@
|
|
|
2
2
|
import { Agent } from "@micdrop/server";
|
|
3
3
|
import { Mistral } from "@mistralai/mistralai";
|
|
4
4
|
import z, { toJSONSchema } from "zod";
|
|
5
|
-
|
|
5
|
+
|
|
6
|
+
// src/deltaText.ts
|
|
7
|
+
function textOf(content) {
|
|
8
|
+
if (typeof content === "string") return content;
|
|
9
|
+
if (!Array.isArray(content)) return void 0;
|
|
10
|
+
return content.map((chunk) => chunk.type === "text" ? chunk.text : "").join("");
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// src/models.ts
|
|
14
|
+
import { cachedList, cachedModels } from "@micdrop/server";
|
|
15
|
+
var DEFAULT_BASE_URL = "https://api.mistral.ai";
|
|
16
|
+
function fetchAll(apiKey, baseUrl) {
|
|
17
|
+
return cachedList(["mistral-raw-models", baseUrl], apiKey, async () => {
|
|
18
|
+
const response = await fetch(`${baseUrl}/v1/models`, {
|
|
19
|
+
headers: { Authorization: `Bearer ${apiKey}` }
|
|
20
|
+
});
|
|
21
|
+
if (!response.ok) throw new Error(`Mistral answered ${response.status}`);
|
|
22
|
+
const { data } = await response.json();
|
|
23
|
+
return data;
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
function listMistralModels(component, keep, options) {
|
|
27
|
+
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
28
|
+
return cachedModels(
|
|
29
|
+
["mistral", component, baseUrl],
|
|
30
|
+
options.apiKey,
|
|
31
|
+
async () => {
|
|
32
|
+
const models = [];
|
|
33
|
+
const seen = /* @__PURE__ */ new Set();
|
|
34
|
+
for (const model of await fetchAll(options.apiKey, baseUrl)) {
|
|
35
|
+
if (!keep(model) || seen.has(model.id)) continue;
|
|
36
|
+
const names = [model.id, ...model.aliases ?? []];
|
|
37
|
+
names.forEach((name) => seen.add(name));
|
|
38
|
+
const id = names.find((name) => name.endsWith("-latest")) ?? model.id;
|
|
39
|
+
models.push({
|
|
40
|
+
id,
|
|
41
|
+
name: model.name || void 0,
|
|
42
|
+
description: model.description || void 0,
|
|
43
|
+
aliases: names.filter((name) => name !== id),
|
|
44
|
+
languages: model.audio_transcription_capabilities?.languages ?? void 0,
|
|
45
|
+
reasoning: !!model.capabilities.reasoning,
|
|
46
|
+
deprecated: !!model.deprecation,
|
|
47
|
+
shutdownDate: model.deprecation?.slice(0, 10) || void 0
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return models;
|
|
51
|
+
},
|
|
52
|
+
options.refresh
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// src/MistralAgent.ts
|
|
57
|
+
var DEFAULT_MODEL = "mistral-medium-latest";
|
|
6
58
|
var DEFAULT_MAX_RETRY = 3;
|
|
7
59
|
var DEFAULT_MAX_STEPS = 5;
|
|
8
60
|
var DEFAULT_RETRY_DELAY = 500;
|
|
61
|
+
var StepInterruptedError = class extends Error {
|
|
62
|
+
constructor(outcome) {
|
|
63
|
+
super("Answer interrupted");
|
|
64
|
+
this.outcome = outcome;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
9
67
|
var MistralAgent = class extends Agent {
|
|
68
|
+
/**
|
|
69
|
+
* Lists the models that can answer a call: chat models that call tools. `reasoning` marks the ones that can reason, slower to answer when they do, each one once with its aliases.
|
|
70
|
+
*
|
|
71
|
+
* The list is fetched once and shared by every call of the process for a
|
|
72
|
+
* day. `refresh` fetches it again.
|
|
73
|
+
*/
|
|
74
|
+
static listModels(options) {
|
|
75
|
+
return listMistralModels(
|
|
76
|
+
"agent",
|
|
77
|
+
({ capabilities }) => capabilities.completion_chat && capabilities.function_calling,
|
|
78
|
+
options
|
|
79
|
+
);
|
|
80
|
+
}
|
|
10
81
|
constructor(options) {
|
|
11
82
|
super(options);
|
|
12
83
|
this.mistral = new Mistral({ apiKey: options.apiKey });
|
|
13
84
|
}
|
|
14
|
-
async generateAnswer(stream,
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
85
|
+
async generateAnswer(stream, signal) {
|
|
86
|
+
const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
|
|
87
|
+
const maxRetry = this.options.maxRetry || DEFAULT_MAX_RETRY;
|
|
88
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
89
|
+
let outcome;
|
|
90
|
+
for (let tryCount = 0; !outcome; tryCount++) {
|
|
91
|
+
try {
|
|
92
|
+
outcome = await this.generateStep(stream, signal);
|
|
93
|
+
} catch (error) {
|
|
94
|
+
if (signal.aborted) return;
|
|
95
|
+
if (error instanceof StepInterruptedError) {
|
|
96
|
+
outcome = error.outcome;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
console.error("[MistralAgent] Error:", error);
|
|
100
|
+
if (tryCount >= maxRetry) {
|
|
101
|
+
this.log("Max retries reached, giving up");
|
|
102
|
+
this.emit("Failed");
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
await new Promise(
|
|
106
|
+
(resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
|
|
107
|
+
);
|
|
108
|
+
if (signal.aborted) return;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (signal.aborted) return;
|
|
112
|
+
if (outcome.fullAnswer) {
|
|
113
|
+
const { message, metadata } = this.extract(outcome.fullAnswer);
|
|
114
|
+
this.addAssistantMessage(message, metadata);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
if (outcome.skipAnswer) {
|
|
118
|
+
this.conversation.push({
|
|
119
|
+
role: "assistant",
|
|
120
|
+
content: " "
|
|
121
|
+
});
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
if (!outcome.ranTools) {
|
|
125
|
+
this.log("Empty answer");
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
18
128
|
}
|
|
19
|
-
|
|
129
|
+
console.error("[MistralAgent] Max steps reached");
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
133
|
+
*
|
|
134
|
+
* It throws a plain error when the request failed before producing
|
|
135
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
136
|
+
* once something was written or done.
|
|
137
|
+
*/
|
|
138
|
+
async generateStep(stream, signal) {
|
|
20
139
|
if (this.conversation.length === 1) {
|
|
21
140
|
this.conversation.push({
|
|
22
141
|
role: "user",
|
|
23
142
|
content: ""
|
|
24
143
|
});
|
|
25
144
|
}
|
|
26
|
-
|
|
27
|
-
const
|
|
145
|
+
const write = this.answerWriter(stream);
|
|
146
|
+
const outcome = {
|
|
147
|
+
fullAnswer: "",
|
|
148
|
+
skipAnswer: false,
|
|
149
|
+
ranTools: false
|
|
150
|
+
};
|
|
151
|
+
let produced = false;
|
|
28
152
|
try {
|
|
29
|
-
const result = await this.mistral.chat.stream(
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
153
|
+
const result = await this.mistral.chat.stream(
|
|
154
|
+
{
|
|
155
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
156
|
+
messages: this.buildMessages(),
|
|
157
|
+
tools: this.buildTools(),
|
|
158
|
+
...this.options.settings
|
|
159
|
+
},
|
|
160
|
+
{ signal }
|
|
161
|
+
);
|
|
162
|
+
const toolCalls = [];
|
|
163
|
+
const toolCallsByIndex = /* @__PURE__ */ new Map();
|
|
37
164
|
for await (const event of result) {
|
|
38
|
-
if (
|
|
165
|
+
if (signal.aborted) return outcome;
|
|
39
166
|
const delta = event.data.choices[0]?.delta;
|
|
40
|
-
const chunk = delta?.content;
|
|
41
|
-
|
|
42
|
-
if (typeof chunk === "string") {
|
|
43
|
-
if (chunk === "") continue;
|
|
167
|
+
const chunk = textOf(delta?.content);
|
|
168
|
+
if (chunk) {
|
|
44
169
|
this.log(`Answer chunk: "${chunk}"`);
|
|
45
|
-
fullAnswer += chunk;
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
continue;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
stream.write(chunk);
|
|
60
|
-
} else if (toolCalls?.length) {
|
|
61
|
-
for (const toolCall of toolCalls) {
|
|
62
|
-
const result2 = await this.executeTool({
|
|
63
|
-
role: "tool_call",
|
|
64
|
-
toolCallId: toolCall.id || "",
|
|
65
|
-
toolName: toolCall.function.name,
|
|
66
|
-
parameters: toolCall.function.arguments
|
|
67
|
-
});
|
|
68
|
-
if (result2.skipAnswer) {
|
|
69
|
-
skipAnswer = true;
|
|
70
|
-
}
|
|
170
|
+
outcome.fullAnswer += chunk;
|
|
171
|
+
produced = true;
|
|
172
|
+
write(chunk);
|
|
173
|
+
}
|
|
174
|
+
for (const [position, fragment] of (delta?.toolCalls ?? []).entries()) {
|
|
175
|
+
const index = fragment.index ?? position;
|
|
176
|
+
let toolCall = toolCallsByIndex.get(index);
|
|
177
|
+
if (!toolCall || fragment.id && toolCall.id && fragment.id !== toolCall.id) {
|
|
178
|
+
toolCall = { id: "", name: "", arguments: "" };
|
|
179
|
+
toolCalls.push(toolCall);
|
|
180
|
+
toolCallsByIndex.set(index, toolCall);
|
|
71
181
|
}
|
|
182
|
+
if (fragment.id) toolCall.id = fragment.id;
|
|
183
|
+
if (fragment.function.name) toolCall.name = fragment.function.name;
|
|
184
|
+
const args = fragment.function.arguments;
|
|
185
|
+
toolCall.arguments += typeof args === "string" ? args : JSON.stringify(args);
|
|
72
186
|
}
|
|
73
187
|
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
188
|
+
for (const toolCall of toolCalls) {
|
|
189
|
+
if (signal.aborted) return outcome;
|
|
190
|
+
produced = true;
|
|
191
|
+
outcome.ranTools = true;
|
|
192
|
+
const result2 = await this.executeTool({
|
|
193
|
+
role: "tool_call",
|
|
194
|
+
toolCallId: toolCall.id,
|
|
195
|
+
toolName: toolCall.name,
|
|
196
|
+
parameters: toolCall.arguments || "{}"
|
|
82
197
|
});
|
|
83
|
-
|
|
84
|
-
|
|
198
|
+
if (result2.skipAnswer) {
|
|
199
|
+
outcome.skipAnswer = true;
|
|
200
|
+
}
|
|
85
201
|
}
|
|
86
202
|
} catch (error) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
(resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
|
|
91
|
-
);
|
|
92
|
-
await this.generateAnswer(stream, stepCount, tryCount + 1);
|
|
93
|
-
} else {
|
|
94
|
-
this.log("Max retries reached, giving up");
|
|
95
|
-
this.emit("Failed");
|
|
96
|
-
}
|
|
203
|
+
if (!produced || signal.aborted) throw error;
|
|
204
|
+
console.error("[MistralAgent] Error after a partial answer:", error);
|
|
205
|
+
throw new StepInterruptedError(outcome);
|
|
97
206
|
}
|
|
207
|
+
return outcome;
|
|
98
208
|
}
|
|
99
209
|
buildMessages() {
|
|
100
210
|
return this.conversation.map(
|
|
@@ -116,7 +226,8 @@ var MistralAgent = class extends Agent {
|
|
|
116
226
|
id: message.toolCallId,
|
|
117
227
|
function: {
|
|
118
228
|
name: message.toolName,
|
|
119
|
-
|
|
229
|
+
// Already the JSON the model wrote
|
|
230
|
+
arguments: message.parameters
|
|
120
231
|
}
|
|
121
232
|
}
|
|
122
233
|
]
|
|
@@ -149,17 +260,10 @@ var MistralAgent = class extends Agent {
|
|
|
149
260
|
}
|
|
150
261
|
}));
|
|
151
262
|
}
|
|
152
|
-
cancel() {
|
|
153
|
-
if (!this.answering) return;
|
|
154
|
-
this.log("Cancel");
|
|
155
|
-
this.answering = false;
|
|
156
|
-
this.answerCount++;
|
|
157
|
-
}
|
|
158
263
|
};
|
|
159
264
|
|
|
160
265
|
// src/MistralSTT.ts
|
|
161
|
-
import { STT } from "@micdrop/server";
|
|
162
|
-
import WebSocket from "ws";
|
|
266
|
+
import { ReconnectingSocket, STT } from "@micdrop/server";
|
|
163
267
|
var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-3-0";
|
|
164
268
|
var DEFAULT_ENCODING = "pcm_s16le";
|
|
165
269
|
var SAMPLE_RATE = 16e3;
|
|
@@ -171,62 +275,130 @@ var MistralSTT = class extends STT {
|
|
|
171
275
|
constructor(options) {
|
|
172
276
|
super();
|
|
173
277
|
this.options = options;
|
|
174
|
-
|
|
175
|
-
//
|
|
176
|
-
this.
|
|
177
|
-
//
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
278
|
+
// Utterances not done yet, oldest first. The server answers them in the
|
|
279
|
+
// order their audio went out, so the transcript in progress is the oldest's.
|
|
280
|
+
this.utterances = [];
|
|
281
|
+
// Flushes of utterances that timed out on this session, whose deltas and
|
|
282
|
+
// transcription.done may still arrive and must not go to the next one. Each
|
|
283
|
+
// is forgotten after another timeout, so an answer that never comes does not
|
|
284
|
+
// swallow every transcript after it.
|
|
285
|
+
this.staleFlushes = [];
|
|
286
|
+
// A new session knows nothing of the previous one: every pending utterance
|
|
287
|
+
// is sent again in full and replays its whole transcript, so the deltas it
|
|
288
|
+
// gathered start over to avoid duplicating them
|
|
289
|
+
this.onOpen = ({ reconnected }) => {
|
|
290
|
+
this.staleFlushes = [];
|
|
291
|
+
const pending = this.utterances.filter(
|
|
292
|
+
(utterance) => utterance.chunks.length > 0
|
|
293
|
+
);
|
|
294
|
+
if (pending.length === 0) return;
|
|
295
|
+
this.log(
|
|
296
|
+
reconnected ? "Sending audio chunks again" : "Sending queued audio chunks"
|
|
297
|
+
);
|
|
298
|
+
for (const utterance of pending) {
|
|
299
|
+
utterance.text = "";
|
|
300
|
+
utterance.chunks.forEach((chunk) => this.sendAudioChunk(chunk));
|
|
301
|
+
if (utterance.status === "flushing") this.flushAudio(utterance);
|
|
302
|
+
}
|
|
303
|
+
};
|
|
304
|
+
this.onMessage = (data) => {
|
|
305
|
+
let message;
|
|
306
|
+
try {
|
|
307
|
+
message = JSON.parse(data.toString());
|
|
308
|
+
} catch {
|
|
309
|
+
this.log("Ignoring a message that is not JSON");
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
switch (message.type) {
|
|
313
|
+
case "transcription.text.delta": {
|
|
314
|
+
if (typeof message.text !== "string") break;
|
|
315
|
+
this.log(`Received transcript delta: "${message.text}"`);
|
|
316
|
+
if (this.hasStaleFlush()) break;
|
|
317
|
+
const utterance = this.utterances[0];
|
|
318
|
+
if (utterance) utterance.text += message.text;
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
case "transcription.done": {
|
|
322
|
+
if (this.hasStaleFlush()) {
|
|
323
|
+
this.staleFlushes.shift();
|
|
324
|
+
this.log("Ignoring the transcript of a timed out utterance");
|
|
325
|
+
break;
|
|
326
|
+
}
|
|
327
|
+
const utterance = this.utterances[0];
|
|
328
|
+
if (utterance?.status !== "flushing") {
|
|
329
|
+
this.log("Ignoring an unexpected transcription.done");
|
|
330
|
+
break;
|
|
331
|
+
}
|
|
332
|
+
const transcript = typeof message.text === "string" ? message.text : utterance.text;
|
|
333
|
+
this.log(`Received completed transcript: "${transcript}"`);
|
|
334
|
+
this.complete(utterance, transcript);
|
|
335
|
+
break;
|
|
336
|
+
}
|
|
337
|
+
case "error":
|
|
338
|
+
this.log("Error:", message.error ?? message);
|
|
339
|
+
break;
|
|
340
|
+
default:
|
|
341
|
+
break;
|
|
342
|
+
}
|
|
343
|
+
};
|
|
344
|
+
this.onFailed = () => {
|
|
345
|
+
const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
|
|
346
|
+
this.utterances.forEach((utterance) => this.dropUtterance(utterance));
|
|
347
|
+
this.utterances = [];
|
|
348
|
+
this.emit("Failed", chunks);
|
|
349
|
+
};
|
|
350
|
+
this.socket = new ReconnectingSocket({
|
|
351
|
+
url: this.getURL(),
|
|
352
|
+
clientOptions: {
|
|
353
|
+
headers: {
|
|
354
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
355
|
+
}
|
|
356
|
+
},
|
|
357
|
+
connectionTimeout: this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT,
|
|
358
|
+
retryDelay: this.options.retryDelay ?? DEFAULT_RETRY_DELAY2,
|
|
359
|
+
maxRetry: this.options.maxRetry ?? DEFAULT_MAX_RETRY2,
|
|
360
|
+
// The audio format must be set before any audio is appended
|
|
361
|
+
handshake: {
|
|
362
|
+
start: (socket) => socket.send(this.getSessionUpdate())
|
|
363
|
+
},
|
|
364
|
+
log: (...message) => this.log(...message)
|
|
365
|
+
});
|
|
366
|
+
this.socket.on("Open", this.onOpen);
|
|
367
|
+
this.socket.on("Message", this.onMessage);
|
|
368
|
+
this.socket.on("Failed", this.onFailed);
|
|
369
|
+
this.socket.connect();
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Lists the models that transcribe as the user speaks, the realtime ones `MistralSTT` streams to, each one once with its aliases.
|
|
373
|
+
*
|
|
374
|
+
* The list is fetched once and shared by every call of the process for a
|
|
375
|
+
* day. `refresh` fetches it again.
|
|
376
|
+
*/
|
|
377
|
+
static listModels(options) {
|
|
378
|
+
return listMistralModels(
|
|
379
|
+
"stt",
|
|
380
|
+
({ capabilities }) => capabilities.audio_transcription_realtime,
|
|
381
|
+
options
|
|
382
|
+
);
|
|
181
383
|
}
|
|
182
384
|
transcribe(audioStream) {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
385
|
+
this.socket.connect();
|
|
386
|
+
const utterance = { status: "streaming", chunks: [], text: "" };
|
|
387
|
+
this.utterances.push(utterance);
|
|
388
|
+
audioStream.on("data", (chunk) => {
|
|
389
|
+
if (utterance.status !== "streaming") return;
|
|
390
|
+
utterance.chunks.push(chunk);
|
|
391
|
+
if (this.sendAudioChunk(chunk)) {
|
|
392
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
188
393
|
}
|
|
189
|
-
this.audioChunksPending.push(chunk);
|
|
190
|
-
await this.initPromise;
|
|
191
|
-
this.sendAudioChunk(chunk);
|
|
192
|
-
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
193
|
-
});
|
|
194
|
-
audioStream.on("end", async () => {
|
|
195
|
-
await this.initPromise;
|
|
196
|
-
if (this.audioChunksPending.length === 0) return;
|
|
197
|
-
this.flushAudio();
|
|
198
|
-
this.transcriptionTimeout = setTimeout(() => {
|
|
199
|
-
this.transcriptionTimeout = void 0;
|
|
200
|
-
this.log("Transcription timeout");
|
|
201
|
-
this.emit("Transcript", this.transcriptDelta.trim());
|
|
202
|
-
this.transcriptDelta = "";
|
|
203
|
-
this.audioChunksPending.length = 0;
|
|
204
|
-
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
205
394
|
});
|
|
395
|
+
audioStream.on("end", () => this.endUtterance(utterance));
|
|
206
396
|
}
|
|
207
397
|
destroy() {
|
|
208
398
|
super.destroy();
|
|
209
|
-
this.
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
this.reconnectTimeout = void 0;
|
|
213
|
-
}
|
|
214
|
-
if (this.transcriptionTimeout) {
|
|
215
|
-
clearTimeout(this.transcriptionTimeout);
|
|
216
|
-
this.transcriptionTimeout = void 0;
|
|
217
|
-
}
|
|
218
|
-
if (this.connectionTimeout) {
|
|
219
|
-
clearTimeout(this.connectionTimeout);
|
|
220
|
-
this.connectionTimeout = void 0;
|
|
221
|
-
}
|
|
222
|
-
if (this.socket) closeSocket(this.socket);
|
|
223
|
-
this.socket = void 0;
|
|
224
|
-
}
|
|
225
|
-
connect() {
|
|
226
|
-
return this.initWS().catch((error) => {
|
|
227
|
-
console.error("[MistralSTT] Connection error:", error);
|
|
228
|
-
this.reconnect();
|
|
229
|
-
});
|
|
399
|
+
this.socket.destroy();
|
|
400
|
+
this.utterances.forEach((utterance) => this.dropUtterance(utterance));
|
|
401
|
+
this.utterances = [];
|
|
230
402
|
}
|
|
231
403
|
getURL() {
|
|
232
404
|
const model = this.options.model || DEFAULT_MODEL2;
|
|
@@ -234,163 +406,131 @@ var MistralSTT = class extends STT {
|
|
|
234
406
|
model
|
|
235
407
|
)}`;
|
|
236
408
|
}
|
|
237
|
-
|
|
238
|
-
return
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
409
|
+
getSessionUpdate() {
|
|
410
|
+
return JSON.stringify({
|
|
411
|
+
type: "session.update",
|
|
412
|
+
session: {
|
|
413
|
+
audio_format: {
|
|
414
|
+
encoding: this.options.encoding || DEFAULT_ENCODING,
|
|
415
|
+
sample_rate: SAMPLE_RATE
|
|
416
|
+
},
|
|
417
|
+
...this.options.targetStreamingDelayMs !== void 0 && {
|
|
418
|
+
target_streaming_delay_ms: this.options.targetStreamingDelayMs
|
|
242
419
|
}
|
|
243
|
-
}
|
|
244
|
-
this.socket = socket;
|
|
245
|
-
const timeout = setTimeout(() => {
|
|
246
|
-
this.log("Connection timeout");
|
|
247
|
-
closeSocket(socket);
|
|
248
|
-
this.socket = void 0;
|
|
249
|
-
reject(new Error("WebSocket connection timeout"));
|
|
250
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
251
|
-
this.connectionTimeout = timeout;
|
|
252
|
-
socket.addEventListener("open", () => {
|
|
253
|
-
clearTimeout(timeout);
|
|
254
|
-
this.sendSessionUpdate();
|
|
255
|
-
this.log("Connection opened");
|
|
256
|
-
resolve();
|
|
257
|
-
});
|
|
258
|
-
socket.addEventListener("error", (error) => {
|
|
259
|
-
clearTimeout(timeout);
|
|
260
|
-
this.log("WebSocket error:", error);
|
|
261
|
-
reject(new Error("WebSocket connection error"));
|
|
262
|
-
});
|
|
263
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
264
|
-
clearTimeout(timeout);
|
|
265
|
-
this.socket?.removeAllListeners();
|
|
266
|
-
this.socket = void 0;
|
|
267
|
-
if (code !== 1e3) {
|
|
268
|
-
this.reconnect();
|
|
269
|
-
} else {
|
|
270
|
-
this.log("Connection closed", { code, reason });
|
|
271
|
-
}
|
|
272
|
-
});
|
|
273
|
-
socket.addEventListener("message", (event) => {
|
|
274
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
275
|
-
});
|
|
420
|
+
}
|
|
276
421
|
});
|
|
277
422
|
}
|
|
278
|
-
|
|
279
|
-
if (
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
)
|
|
423
|
+
endUtterance(utterance) {
|
|
424
|
+
if (utterance.status !== "streaming") return;
|
|
425
|
+
if (utterance.chunks.length === 0) {
|
|
426
|
+
this.removeUtterance(utterance);
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
utterance.status = "flushing";
|
|
430
|
+
if (this.socket.isOpen) this.flushAudio(utterance);
|
|
431
|
+
}
|
|
432
|
+
flushAudio(utterance) {
|
|
433
|
+
if (!this.socket.send(JSON.stringify({ type: "input_audio.flush" }))) {
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
this.log("Flushed audio input");
|
|
437
|
+
clearTimeout(utterance.timeout);
|
|
438
|
+
utterance.timeout = setTimeout(() => {
|
|
439
|
+
this.log("Transcription timeout");
|
|
440
|
+
this.staleFlushes.push(Date.now() + this.transcriptionTimeout);
|
|
441
|
+
this.complete(utterance, utterance.text);
|
|
442
|
+
}, this.transcriptionTimeout);
|
|
443
|
+
}
|
|
444
|
+
get transcriptionTimeout() {
|
|
445
|
+
return this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT;
|
|
446
|
+
}
|
|
447
|
+
/** Whether an answer to a timed out flush may still be on its way */
|
|
448
|
+
hasStaleFlush() {
|
|
449
|
+
const now = Date.now();
|
|
450
|
+
while (this.staleFlushes.length > 0 && this.staleFlushes[0] <= now) {
|
|
451
|
+
this.log("Forgetting a flush that was never answered");
|
|
452
|
+
this.staleFlushes.shift();
|
|
453
|
+
}
|
|
454
|
+
return this.staleFlushes.length > 0;
|
|
455
|
+
}
|
|
456
|
+
complete(utterance, transcript) {
|
|
457
|
+
if (utterance.status === "done") return;
|
|
458
|
+
this.removeUtterance(utterance);
|
|
459
|
+
this.emit("Transcript", transcript.trim());
|
|
460
|
+
}
|
|
461
|
+
removeUtterance(utterance) {
|
|
462
|
+
this.dropUtterance(utterance);
|
|
463
|
+
this.utterances = this.utterances.filter((u) => u !== utterance);
|
|
464
|
+
}
|
|
465
|
+
dropUtterance(utterance) {
|
|
466
|
+
utterance.status = "done";
|
|
467
|
+
clearTimeout(utterance.timeout);
|
|
468
|
+
utterance.timeout = void 0;
|
|
294
469
|
}
|
|
295
470
|
sendAudioChunk(chunk) {
|
|
296
|
-
|
|
297
|
-
this.socket.send(
|
|
471
|
+
return this.socket.send(
|
|
298
472
|
JSON.stringify({
|
|
299
473
|
type: "input_audio.append",
|
|
300
474
|
audio: chunk.toString("base64")
|
|
301
475
|
})
|
|
302
476
|
);
|
|
303
477
|
}
|
|
304
|
-
flushAudio() {
|
|
305
|
-
if (!this.socket) return;
|
|
306
|
-
this.socket.send(JSON.stringify({ type: "input_audio.flush" }));
|
|
307
|
-
this.log("Flushed audio input");
|
|
308
|
-
}
|
|
309
|
-
handleMessage(message) {
|
|
310
|
-
switch (message.type) {
|
|
311
|
-
case "transcription.text.delta":
|
|
312
|
-
if (typeof message.text === "string") {
|
|
313
|
-
this.transcriptDelta += message.text;
|
|
314
|
-
this.log(`Received transcript delta: "${message.text}"`);
|
|
315
|
-
}
|
|
316
|
-
break;
|
|
317
|
-
case "transcription.done": {
|
|
318
|
-
const transcript = typeof message.text === "string" ? message.text : this.transcriptDelta;
|
|
319
|
-
this.log(`Received completed transcript: "${transcript}"`);
|
|
320
|
-
this.emit("Transcript", transcript.trim());
|
|
321
|
-
this.transcriptDelta = "";
|
|
322
|
-
this.audioChunksPending.length = 0;
|
|
323
|
-
if (this.transcriptionTimeout) {
|
|
324
|
-
clearTimeout(this.transcriptionTimeout);
|
|
325
|
-
this.transcriptionTimeout = void 0;
|
|
326
|
-
}
|
|
327
|
-
break;
|
|
328
|
-
}
|
|
329
|
-
case "error":
|
|
330
|
-
this.log("Error:", message.error ?? message);
|
|
331
|
-
break;
|
|
332
|
-
default:
|
|
333
|
-
break;
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
reconnect() {
|
|
337
|
-
if (this.destroyed) return;
|
|
338
|
-
this.retryCount++;
|
|
339
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
340
|
-
this.log("Max retries reached, giving up");
|
|
341
|
-
this.emit("Failed", this.audioChunksPending);
|
|
342
|
-
return;
|
|
343
|
-
}
|
|
344
|
-
this.initPromise = new Promise((resolve) => {
|
|
345
|
-
this.log("Reconnecting...");
|
|
346
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
347
|
-
this.reconnectTimeout = void 0;
|
|
348
|
-
this.initWS().then(() => {
|
|
349
|
-
this.retryCount = 0;
|
|
350
|
-
if (this.audioChunksPending.length > 0) {
|
|
351
|
-
this.log("Sending audio chunks again");
|
|
352
|
-
this.transcriptDelta = "";
|
|
353
|
-
this.audioChunksPending.forEach(
|
|
354
|
-
(chunk) => this.sendAudioChunk(chunk)
|
|
355
|
-
);
|
|
356
|
-
}
|
|
357
|
-
}).then(resolve).catch((error) => {
|
|
358
|
-
this.log("Reconnection error:", error);
|
|
359
|
-
this.reconnect();
|
|
360
|
-
});
|
|
361
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
362
|
-
});
|
|
363
|
-
}
|
|
364
478
|
};
|
|
365
|
-
function closeSocket(socket) {
|
|
366
|
-
socket.removeAllListeners();
|
|
367
|
-
socket.on("error", () => {
|
|
368
|
-
});
|
|
369
|
-
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
370
|
-
socket.close(1e3);
|
|
371
|
-
}
|
|
372
|
-
}
|
|
373
479
|
|
|
374
480
|
// src/MistralTTS.ts
|
|
375
|
-
import {
|
|
481
|
+
import {
|
|
482
|
+
cachedVoices,
|
|
483
|
+
float32ToPcm16,
|
|
484
|
+
SentenceTTS,
|
|
485
|
+
voiceGender,
|
|
486
|
+
voicesForLanguage
|
|
487
|
+
} from "@micdrop/server";
|
|
376
488
|
var DEFAULT_MODEL3 = "voxtral-mini-tts-3-0";
|
|
377
489
|
var DEFAULT_VOICE_ID = "voice_3150";
|
|
378
|
-
var DEFAULT_BASE_URL = "https://api.mistral.ai";
|
|
379
490
|
var MISTRAL_SAMPLE_RATE = 24e3;
|
|
380
|
-
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
381
491
|
var FLOAT32_BYTES = 4;
|
|
492
|
+
var PAGE_SIZE = 100;
|
|
382
493
|
var MistralTTS = class extends SentenceTTS {
|
|
383
494
|
constructor(options) {
|
|
384
495
|
super();
|
|
385
496
|
this.options = options;
|
|
386
497
|
}
|
|
498
|
+
/**
|
|
499
|
+
* Lists the models that speak, each one once with its aliases.
|
|
500
|
+
*
|
|
501
|
+
* The list is fetched once and shared by every call of the process for a
|
|
502
|
+
* day. `refresh` fetches it again.
|
|
503
|
+
*/
|
|
504
|
+
static listModels(options) {
|
|
505
|
+
return listMistralModels(
|
|
506
|
+
"tts",
|
|
507
|
+
({ capabilities }) => capabilities.audio_speech,
|
|
508
|
+
options
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Lists the voices the account can speak with, the presets and its own.
|
|
513
|
+
*
|
|
514
|
+
* `language`, such as `fr` or `fr-FR`, keeps the voices that speak it.
|
|
515
|
+
*
|
|
516
|
+
* The list is fetched once and shared by every call of the process for a
|
|
517
|
+
* day. `refresh` fetches it again, for a voice added a moment ago.
|
|
518
|
+
*/
|
|
519
|
+
static listVoices(options) {
|
|
520
|
+
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
521
|
+
return cachedVoices(
|
|
522
|
+
["mistral", baseUrl],
|
|
523
|
+
options.apiKey,
|
|
524
|
+
() => fetchVoices(options.apiKey, baseUrl),
|
|
525
|
+
options.refresh
|
|
526
|
+
).then((voices) => voicesForLanguage(voices, options.language));
|
|
527
|
+
}
|
|
528
|
+
get sampleRate() {
|
|
529
|
+
return MISTRAL_SAMPLE_RATE;
|
|
530
|
+
}
|
|
387
531
|
async synthesize(text, signal) {
|
|
388
532
|
const response = await this.request(text, signal);
|
|
389
533
|
if (!response.body) throw new Error("Mistral answered without a body");
|
|
390
|
-
const resampler = new Pcm16Resampler(
|
|
391
|
-
MISTRAL_SAMPLE_RATE,
|
|
392
|
-
OUTPUT_SAMPLE_RATE
|
|
393
|
-
);
|
|
394
534
|
let leftover = Buffer.alloc(0);
|
|
395
535
|
for await (const event of readEvents(response.body, signal)) {
|
|
396
536
|
if (event.type === "error") {
|
|
@@ -409,7 +549,7 @@ var MistralTTS = class extends SentenceTTS {
|
|
|
409
549
|
for (let i = 0; i < samples.length; i++) {
|
|
410
550
|
samples[i] = data.readFloatLE(i * FLOAT32_BYTES);
|
|
411
551
|
}
|
|
412
|
-
if (!this.emitAudio(
|
|
552
|
+
if (!this.emitAudio(float32ToPcm16(samples))) break;
|
|
413
553
|
}
|
|
414
554
|
return void 0;
|
|
415
555
|
}
|
|
@@ -466,6 +606,35 @@ async function* readEvents(body, signal) {
|
|
|
466
606
|
});
|
|
467
607
|
}
|
|
468
608
|
}
|
|
609
|
+
async function fetchVoices(apiKey, baseUrl) {
|
|
610
|
+
const voices = [];
|
|
611
|
+
for (let offset = 0, total = 1; offset < total; offset += PAGE_SIZE) {
|
|
612
|
+
const response = await fetch(
|
|
613
|
+
`${baseUrl}/v1/audio/voices?offset=${offset}&limit=${PAGE_SIZE}`,
|
|
614
|
+
{ headers: { Authorization: `Bearer ${apiKey}` } }
|
|
615
|
+
);
|
|
616
|
+
if (!response.ok) throw new Error(`Mistral answered ${response.status}`);
|
|
617
|
+
const data = await response.json();
|
|
618
|
+
total = data.total ?? 0;
|
|
619
|
+
if (data.items.length === 0) break;
|
|
620
|
+
for (const voice of data.items) {
|
|
621
|
+
voices.push({
|
|
622
|
+
// A preset goes by its slug, a voice of the account by its id
|
|
623
|
+
id: voice.slug || voice.id,
|
|
624
|
+
name: voice.name,
|
|
625
|
+
// Mistral writes en_us where the other providers write en-US
|
|
626
|
+
languages: voice.languages?.map((code) => {
|
|
627
|
+
const [language, region] = code.split("_");
|
|
628
|
+
return region ? `${language}-${region.toUpperCase()}` : language;
|
|
629
|
+
}),
|
|
630
|
+
gender: voiceGender(voice.gender),
|
|
631
|
+
description: voice.description || void 0,
|
|
632
|
+
custom: voice.type !== "preset"
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
return voices;
|
|
637
|
+
}
|
|
469
638
|
export {
|
|
470
639
|
MistralAgent,
|
|
471
640
|
MistralSTT,
|