@micdrop/mistral 1.2.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -11
- package/dist/index.d.mts +75 -18
- package/dist/index.d.ts +75 -18
- package/dist/index.js +424 -261
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +426 -257
- package/dist/index.mjs.map +1 -1
- package/package.json +8 -4
package/dist/index.js
CHANGED
|
@@ -37,102 +37,212 @@ __export(index_exports, {
|
|
|
37
37
|
module.exports = __toCommonJS(index_exports);
|
|
38
38
|
|
|
39
39
|
// src/MistralAgent.ts
|
|
40
|
-
var
|
|
40
|
+
var import_server2 = require("@micdrop/server");
|
|
41
41
|
var import_mistralai = require("@mistralai/mistralai");
|
|
42
42
|
var import_zod = __toESM(require("zod"));
|
|
43
|
-
|
|
43
|
+
|
|
44
|
+
// src/deltaText.ts
|
|
45
|
+
function textOf(content) {
|
|
46
|
+
if (typeof content === "string") return content;
|
|
47
|
+
if (!Array.isArray(content)) return void 0;
|
|
48
|
+
return content.map((chunk) => chunk.type === "text" ? chunk.text : "").join("");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// src/models.ts
|
|
52
|
+
var import_server = require("@micdrop/server");
|
|
53
|
+
var DEFAULT_BASE_URL = "https://api.mistral.ai";
|
|
54
|
+
function fetchAll(apiKey, baseUrl) {
|
|
55
|
+
return (0, import_server.cachedList)(["mistral-raw-models", baseUrl], apiKey, async () => {
|
|
56
|
+
const response = await fetch(`${baseUrl}/v1/models`, {
|
|
57
|
+
headers: { Authorization: `Bearer ${apiKey}` }
|
|
58
|
+
});
|
|
59
|
+
if (!response.ok) throw new Error(`Mistral answered ${response.status}`);
|
|
60
|
+
const { data } = await response.json();
|
|
61
|
+
return data;
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
function listMistralModels(component, keep, options) {
|
|
65
|
+
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
66
|
+
return (0, import_server.cachedModels)(
|
|
67
|
+
["mistral", component, baseUrl],
|
|
68
|
+
options.apiKey,
|
|
69
|
+
async () => {
|
|
70
|
+
const models = [];
|
|
71
|
+
const seen = /* @__PURE__ */ new Set();
|
|
72
|
+
for (const model of await fetchAll(options.apiKey, baseUrl)) {
|
|
73
|
+
if (!keep(model) || seen.has(model.id)) continue;
|
|
74
|
+
const names = [model.id, ...model.aliases ?? []];
|
|
75
|
+
names.forEach((name) => seen.add(name));
|
|
76
|
+
const id = names.find((name) => name.endsWith("-latest")) ?? model.id;
|
|
77
|
+
models.push({
|
|
78
|
+
id,
|
|
79
|
+
name: model.name || void 0,
|
|
80
|
+
description: model.description || void 0,
|
|
81
|
+
aliases: names.filter((name) => name !== id),
|
|
82
|
+
languages: model.audio_transcription_capabilities?.languages ?? void 0,
|
|
83
|
+
reasoning: !!model.capabilities.reasoning,
|
|
84
|
+
deprecated: !!model.deprecation,
|
|
85
|
+
shutdownDate: model.deprecation?.slice(0, 10) || void 0
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return models;
|
|
89
|
+
},
|
|
90
|
+
options.refresh
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// src/MistralAgent.ts
|
|
95
|
+
var DEFAULT_MODEL = "mistral-medium-latest";
|
|
44
96
|
var DEFAULT_MAX_RETRY = 3;
|
|
45
97
|
var DEFAULT_MAX_STEPS = 5;
|
|
46
98
|
var DEFAULT_RETRY_DELAY = 500;
|
|
47
|
-
var
|
|
99
|
+
var StepInterruptedError = class extends Error {
|
|
100
|
+
constructor(outcome) {
|
|
101
|
+
super("Answer interrupted");
|
|
102
|
+
this.outcome = outcome;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
var MistralAgent = class extends import_server2.Agent {
|
|
106
|
+
/**
|
|
107
|
+
* Lists the models that can answer a call: chat models that call tools. `reasoning` marks the ones that can reason, slower to answer when they do, each one once with its aliases.
|
|
108
|
+
*
|
|
109
|
+
* The list is fetched once and shared by every call of the process for a
|
|
110
|
+
* day. `refresh` fetches it again.
|
|
111
|
+
*/
|
|
112
|
+
static listModels(options) {
|
|
113
|
+
return listMistralModels(
|
|
114
|
+
"agent",
|
|
115
|
+
({ capabilities }) => capabilities.completion_chat && capabilities.function_calling,
|
|
116
|
+
options
|
|
117
|
+
);
|
|
118
|
+
}
|
|
48
119
|
constructor(options) {
|
|
49
120
|
super(options);
|
|
50
121
|
this.mistral = new import_mistralai.Mistral({ apiKey: options.apiKey });
|
|
51
122
|
}
|
|
52
|
-
async generateAnswer(stream,
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
123
|
+
async generateAnswer(stream, signal) {
|
|
124
|
+
const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
|
|
125
|
+
const maxRetry = this.options.maxRetry || DEFAULT_MAX_RETRY;
|
|
126
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
127
|
+
let outcome;
|
|
128
|
+
for (let tryCount = 0; !outcome; tryCount++) {
|
|
129
|
+
try {
|
|
130
|
+
outcome = await this.generateStep(stream, signal);
|
|
131
|
+
} catch (error) {
|
|
132
|
+
if (signal.aborted) return;
|
|
133
|
+
if (error instanceof StepInterruptedError) {
|
|
134
|
+
outcome = error.outcome;
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
console.error("[MistralAgent] Error:", error);
|
|
138
|
+
if (tryCount >= maxRetry) {
|
|
139
|
+
this.log("Max retries reached, giving up");
|
|
140
|
+
this.emit("Failed");
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
await new Promise(
|
|
144
|
+
(resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
|
|
145
|
+
);
|
|
146
|
+
if (signal.aborted) return;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (signal.aborted) return;
|
|
150
|
+
if (outcome.fullAnswer) {
|
|
151
|
+
const { message, metadata } = this.extract(outcome.fullAnswer);
|
|
152
|
+
this.addAssistantMessage(message, metadata);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
if (outcome.skipAnswer) {
|
|
156
|
+
this.conversation.push({
|
|
157
|
+
role: "assistant",
|
|
158
|
+
content: " "
|
|
159
|
+
});
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
if (!outcome.ranTools) {
|
|
163
|
+
this.log("Empty answer");
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
56
166
|
}
|
|
57
|
-
|
|
167
|
+
console.error("[MistralAgent] Max steps reached");
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
171
|
+
*
|
|
172
|
+
* It throws a plain error when the request failed before producing
|
|
173
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
174
|
+
* once something was written or done.
|
|
175
|
+
*/
|
|
176
|
+
async generateStep(stream, signal) {
|
|
58
177
|
if (this.conversation.length === 1) {
|
|
59
178
|
this.conversation.push({
|
|
60
179
|
role: "user",
|
|
61
180
|
content: ""
|
|
62
181
|
});
|
|
63
182
|
}
|
|
64
|
-
|
|
65
|
-
const
|
|
183
|
+
const write = this.answerWriter(stream);
|
|
184
|
+
const outcome = {
|
|
185
|
+
fullAnswer: "",
|
|
186
|
+
skipAnswer: false,
|
|
187
|
+
ranTools: false
|
|
188
|
+
};
|
|
189
|
+
let produced = false;
|
|
66
190
|
try {
|
|
67
|
-
const result = await this.mistral.chat.stream(
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
191
|
+
const result = await this.mistral.chat.stream(
|
|
192
|
+
{
|
|
193
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
194
|
+
messages: this.buildMessages(),
|
|
195
|
+
tools: this.buildTools(),
|
|
196
|
+
...this.options.settings
|
|
197
|
+
},
|
|
198
|
+
{ signal }
|
|
199
|
+
);
|
|
200
|
+
const toolCalls = [];
|
|
201
|
+
const toolCallsByIndex = /* @__PURE__ */ new Map();
|
|
75
202
|
for await (const event of result) {
|
|
76
|
-
if (
|
|
203
|
+
if (signal.aborted) return outcome;
|
|
77
204
|
const delta = event.data.choices[0]?.delta;
|
|
78
|
-
const chunk = delta?.content;
|
|
79
|
-
|
|
80
|
-
if (typeof chunk === "string") {
|
|
81
|
-
if (chunk === "") continue;
|
|
205
|
+
const chunk = textOf(delta?.content);
|
|
206
|
+
if (chunk) {
|
|
82
207
|
this.log(`Answer chunk: "${chunk}"`);
|
|
83
|
-
fullAnswer += chunk;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
continue;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
stream.write(chunk);
|
|
98
|
-
} else if (toolCalls?.length) {
|
|
99
|
-
for (const toolCall of toolCalls) {
|
|
100
|
-
const result2 = await this.executeTool({
|
|
101
|
-
role: "tool_call",
|
|
102
|
-
toolCallId: toolCall.id || "",
|
|
103
|
-
toolName: toolCall.function.name,
|
|
104
|
-
parameters: toolCall.function.arguments
|
|
105
|
-
});
|
|
106
|
-
if (result2.skipAnswer) {
|
|
107
|
-
skipAnswer = true;
|
|
108
|
-
}
|
|
208
|
+
outcome.fullAnswer += chunk;
|
|
209
|
+
produced = true;
|
|
210
|
+
write(chunk);
|
|
211
|
+
}
|
|
212
|
+
for (const [position, fragment] of (delta?.toolCalls ?? []).entries()) {
|
|
213
|
+
const index = fragment.index ?? position;
|
|
214
|
+
let toolCall = toolCallsByIndex.get(index);
|
|
215
|
+
if (!toolCall || fragment.id && toolCall.id && fragment.id !== toolCall.id) {
|
|
216
|
+
toolCall = { id: "", name: "", arguments: "" };
|
|
217
|
+
toolCalls.push(toolCall);
|
|
218
|
+
toolCallsByIndex.set(index, toolCall);
|
|
109
219
|
}
|
|
220
|
+
if (fragment.id) toolCall.id = fragment.id;
|
|
221
|
+
if (fragment.function.name) toolCall.name = fragment.function.name;
|
|
222
|
+
const args = fragment.function.arguments;
|
|
223
|
+
toolCall.arguments += typeof args === "string" ? args : JSON.stringify(args);
|
|
110
224
|
}
|
|
111
225
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
226
|
+
for (const toolCall of toolCalls) {
|
|
227
|
+
if (signal.aborted) return outcome;
|
|
228
|
+
produced = true;
|
|
229
|
+
outcome.ranTools = true;
|
|
230
|
+
const result2 = await this.executeTool({
|
|
231
|
+
role: "tool_call",
|
|
232
|
+
toolCallId: toolCall.id,
|
|
233
|
+
toolName: toolCall.name,
|
|
234
|
+
parameters: toolCall.arguments || "{}"
|
|
120
235
|
});
|
|
121
|
-
|
|
122
|
-
|
|
236
|
+
if (result2.skipAnswer) {
|
|
237
|
+
outcome.skipAnswer = true;
|
|
238
|
+
}
|
|
123
239
|
}
|
|
124
240
|
} catch (error) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
(resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
|
|
129
|
-
);
|
|
130
|
-
await this.generateAnswer(stream, stepCount, tryCount + 1);
|
|
131
|
-
} else {
|
|
132
|
-
this.log("Max retries reached, giving up");
|
|
133
|
-
this.emit("Failed");
|
|
134
|
-
}
|
|
241
|
+
if (!produced || signal.aborted) throw error;
|
|
242
|
+
console.error("[MistralAgent] Error after a partial answer:", error);
|
|
243
|
+
throw new StepInterruptedError(outcome);
|
|
135
244
|
}
|
|
245
|
+
return outcome;
|
|
136
246
|
}
|
|
137
247
|
buildMessages() {
|
|
138
248
|
return this.conversation.map(
|
|
@@ -154,7 +264,8 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
154
264
|
id: message.toolCallId,
|
|
155
265
|
function: {
|
|
156
266
|
name: message.toolName,
|
|
157
|
-
|
|
267
|
+
// Already the JSON the model wrote
|
|
268
|
+
arguments: message.parameters
|
|
158
269
|
}
|
|
159
270
|
}
|
|
160
271
|
]
|
|
@@ -187,17 +298,10 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
187
298
|
}
|
|
188
299
|
}));
|
|
189
300
|
}
|
|
190
|
-
cancel() {
|
|
191
|
-
if (!this.answering) return;
|
|
192
|
-
this.log("Cancel");
|
|
193
|
-
this.answering = false;
|
|
194
|
-
this.answerCount++;
|
|
195
|
-
}
|
|
196
301
|
};
|
|
197
302
|
|
|
198
303
|
// src/MistralSTT.ts
|
|
199
|
-
var
|
|
200
|
-
var import_ws = __toESM(require("ws"));
|
|
304
|
+
var import_server3 = require("@micdrop/server");
|
|
201
305
|
var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-3-0";
|
|
202
306
|
var DEFAULT_ENCODING = "pcm_s16le";
|
|
203
307
|
var SAMPLE_RATE = 16e3;
|
|
@@ -205,66 +309,134 @@ var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
|
205
309
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
206
310
|
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
207
311
|
var DEFAULT_MAX_RETRY2 = 3;
|
|
208
|
-
var MistralSTT = class extends
|
|
312
|
+
var MistralSTT = class extends import_server3.STT {
|
|
209
313
|
constructor(options) {
|
|
210
314
|
super();
|
|
211
315
|
this.options = options;
|
|
212
|
-
|
|
213
|
-
//
|
|
214
|
-
this.
|
|
215
|
-
//
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
316
|
+
// Utterances not done yet, oldest first. The server answers them in the
|
|
317
|
+
// order their audio went out, so the transcript in progress is the oldest's.
|
|
318
|
+
this.utterances = [];
|
|
319
|
+
// Flushes of utterances that timed out on this session, whose deltas and
|
|
320
|
+
// transcription.done may still arrive and must not go to the next one. Each
|
|
321
|
+
// is forgotten after another timeout, so an answer that never comes does not
|
|
322
|
+
// swallow every transcript after it.
|
|
323
|
+
this.staleFlushes = [];
|
|
324
|
+
// A new session knows nothing of the previous one: every pending utterance
|
|
325
|
+
// is sent again in full and replays its whole transcript, so the deltas it
|
|
326
|
+
// gathered start over to avoid duplicating them
|
|
327
|
+
this.onOpen = ({ reconnected }) => {
|
|
328
|
+
this.staleFlushes = [];
|
|
329
|
+
const pending = this.utterances.filter(
|
|
330
|
+
(utterance) => utterance.chunks.length > 0
|
|
331
|
+
);
|
|
332
|
+
if (pending.length === 0) return;
|
|
333
|
+
this.log(
|
|
334
|
+
reconnected ? "Sending audio chunks again" : "Sending queued audio chunks"
|
|
335
|
+
);
|
|
336
|
+
for (const utterance of pending) {
|
|
337
|
+
utterance.text = "";
|
|
338
|
+
utterance.chunks.forEach((chunk) => this.sendAudioChunk(chunk));
|
|
339
|
+
if (utterance.status === "flushing") this.flushAudio(utterance);
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
this.onMessage = (data) => {
|
|
343
|
+
let message;
|
|
344
|
+
try {
|
|
345
|
+
message = JSON.parse(data.toString());
|
|
346
|
+
} catch {
|
|
347
|
+
this.log("Ignoring a message that is not JSON");
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
switch (message.type) {
|
|
351
|
+
case "transcription.text.delta": {
|
|
352
|
+
if (typeof message.text !== "string") break;
|
|
353
|
+
this.log(`Received transcript delta: "${message.text}"`);
|
|
354
|
+
if (this.hasStaleFlush()) break;
|
|
355
|
+
const utterance = this.utterances[0];
|
|
356
|
+
if (utterance) utterance.text += message.text;
|
|
357
|
+
break;
|
|
358
|
+
}
|
|
359
|
+
case "transcription.done": {
|
|
360
|
+
if (this.hasStaleFlush()) {
|
|
361
|
+
this.staleFlushes.shift();
|
|
362
|
+
this.log("Ignoring the transcript of a timed out utterance");
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
const utterance = this.utterances[0];
|
|
366
|
+
if (utterance?.status !== "flushing") {
|
|
367
|
+
this.log("Ignoring an unexpected transcription.done");
|
|
368
|
+
break;
|
|
369
|
+
}
|
|
370
|
+
const transcript = typeof message.text === "string" ? message.text : utterance.text;
|
|
371
|
+
this.log(`Received completed transcript: "${transcript}"`);
|
|
372
|
+
this.complete(utterance, transcript);
|
|
373
|
+
break;
|
|
374
|
+
}
|
|
375
|
+
case "error":
|
|
376
|
+
this.log("Error:", message.error ?? message);
|
|
377
|
+
break;
|
|
378
|
+
default:
|
|
379
|
+
break;
|
|
380
|
+
}
|
|
381
|
+
};
|
|
382
|
+
this.onFailed = () => {
|
|
383
|
+
const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
|
|
384
|
+
this.utterances.forEach((utterance) => this.dropUtterance(utterance));
|
|
385
|
+
this.utterances = [];
|
|
386
|
+
this.emit("Failed", chunks);
|
|
387
|
+
};
|
|
388
|
+
this.socket = new import_server3.ReconnectingSocket({
|
|
389
|
+
url: this.getURL(),
|
|
390
|
+
clientOptions: {
|
|
391
|
+
headers: {
|
|
392
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
393
|
+
}
|
|
394
|
+
},
|
|
395
|
+
connectionTimeout: this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT,
|
|
396
|
+
retryDelay: this.options.retryDelay ?? DEFAULT_RETRY_DELAY2,
|
|
397
|
+
maxRetry: this.options.maxRetry ?? DEFAULT_MAX_RETRY2,
|
|
398
|
+
// The audio format must be set before any audio is appended
|
|
399
|
+
handshake: {
|
|
400
|
+
start: (socket) => socket.send(this.getSessionUpdate())
|
|
401
|
+
},
|
|
402
|
+
log: (...message) => this.log(...message)
|
|
403
|
+
});
|
|
404
|
+
this.socket.on("Open", this.onOpen);
|
|
405
|
+
this.socket.on("Message", this.onMessage);
|
|
406
|
+
this.socket.on("Failed", this.onFailed);
|
|
407
|
+
this.socket.connect();
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Lists the models that transcribe as the user speaks, the realtime ones `MistralSTT` streams to, each one once with its aliases.
|
|
411
|
+
*
|
|
412
|
+
* The list is fetched once and shared by every call of the process for a
|
|
413
|
+
* day. `refresh` fetches it again.
|
|
414
|
+
*/
|
|
415
|
+
static listModels(options) {
|
|
416
|
+
return listMistralModels(
|
|
417
|
+
"stt",
|
|
418
|
+
({ capabilities }) => capabilities.audio_transcription_realtime,
|
|
419
|
+
options
|
|
420
|
+
);
|
|
219
421
|
}
|
|
220
422
|
transcribe(audioStream) {
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
423
|
+
this.socket.connect();
|
|
424
|
+
const utterance = { status: "streaming", chunks: [], text: "" };
|
|
425
|
+
this.utterances.push(utterance);
|
|
426
|
+
audioStream.on("data", (chunk) => {
|
|
427
|
+
if (utterance.status !== "streaming") return;
|
|
428
|
+
utterance.chunks.push(chunk);
|
|
429
|
+
if (this.sendAudioChunk(chunk)) {
|
|
430
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
226
431
|
}
|
|
227
|
-
this.audioChunksPending.push(chunk);
|
|
228
|
-
await this.initPromise;
|
|
229
|
-
this.sendAudioChunk(chunk);
|
|
230
|
-
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
231
|
-
});
|
|
232
|
-
audioStream.on("end", async () => {
|
|
233
|
-
await this.initPromise;
|
|
234
|
-
if (this.audioChunksPending.length === 0) return;
|
|
235
|
-
this.flushAudio();
|
|
236
|
-
this.transcriptionTimeout = setTimeout(() => {
|
|
237
|
-
this.transcriptionTimeout = void 0;
|
|
238
|
-
this.log("Transcription timeout");
|
|
239
|
-
this.emit("Transcript", this.transcriptDelta.trim());
|
|
240
|
-
this.transcriptDelta = "";
|
|
241
|
-
this.audioChunksPending.length = 0;
|
|
242
|
-
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
243
432
|
});
|
|
433
|
+
audioStream.on("end", () => this.endUtterance(utterance));
|
|
244
434
|
}
|
|
245
435
|
destroy() {
|
|
246
436
|
super.destroy();
|
|
247
|
-
this.
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
this.reconnectTimeout = void 0;
|
|
251
|
-
}
|
|
252
|
-
if (this.transcriptionTimeout) {
|
|
253
|
-
clearTimeout(this.transcriptionTimeout);
|
|
254
|
-
this.transcriptionTimeout = void 0;
|
|
255
|
-
}
|
|
256
|
-
if (this.connectionTimeout) {
|
|
257
|
-
clearTimeout(this.connectionTimeout);
|
|
258
|
-
this.connectionTimeout = void 0;
|
|
259
|
-
}
|
|
260
|
-
if (this.socket) closeSocket(this.socket);
|
|
261
|
-
this.socket = void 0;
|
|
262
|
-
}
|
|
263
|
-
connect() {
|
|
264
|
-
return this.initWS().catch((error) => {
|
|
265
|
-
console.error("[MistralSTT] Connection error:", error);
|
|
266
|
-
this.reconnect();
|
|
267
|
-
});
|
|
437
|
+
this.socket.destroy();
|
|
438
|
+
this.utterances.forEach((utterance) => this.dropUtterance(utterance));
|
|
439
|
+
this.utterances = [];
|
|
268
440
|
}
|
|
269
441
|
getURL() {
|
|
270
442
|
const model = this.options.model || DEFAULT_MODEL2;
|
|
@@ -272,163 +444,125 @@ var MistralSTT = class extends import_server2.STT {
|
|
|
272
444
|
model
|
|
273
445
|
)}`;
|
|
274
446
|
}
|
|
275
|
-
|
|
276
|
-
return
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
closeSocket(socket);
|
|
286
|
-
this.socket = void 0;
|
|
287
|
-
reject(new Error("WebSocket connection timeout"));
|
|
288
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
289
|
-
this.connectionTimeout = timeout;
|
|
290
|
-
socket.addEventListener("open", () => {
|
|
291
|
-
clearTimeout(timeout);
|
|
292
|
-
this.sendSessionUpdate();
|
|
293
|
-
this.log("Connection opened");
|
|
294
|
-
resolve();
|
|
295
|
-
});
|
|
296
|
-
socket.addEventListener("error", (error) => {
|
|
297
|
-
clearTimeout(timeout);
|
|
298
|
-
this.log("WebSocket error:", error);
|
|
299
|
-
reject(new Error("WebSocket connection error"));
|
|
300
|
-
});
|
|
301
|
-
socket.addEventListener("close", ({ code, reason }) => {
|
|
302
|
-
clearTimeout(timeout);
|
|
303
|
-
this.socket?.removeAllListeners();
|
|
304
|
-
this.socket = void 0;
|
|
305
|
-
if (code !== 1e3) {
|
|
306
|
-
this.reconnect();
|
|
307
|
-
} else {
|
|
308
|
-
this.log("Connection closed", { code, reason });
|
|
447
|
+
getSessionUpdate() {
|
|
448
|
+
return JSON.stringify({
|
|
449
|
+
type: "session.update",
|
|
450
|
+
session: {
|
|
451
|
+
audio_format: {
|
|
452
|
+
encoding: this.options.encoding || DEFAULT_ENCODING,
|
|
453
|
+
sample_rate: SAMPLE_RATE
|
|
454
|
+
},
|
|
455
|
+
...this.options.targetStreamingDelayMs !== void 0 && {
|
|
456
|
+
target_streaming_delay_ms: this.options.targetStreamingDelayMs
|
|
309
457
|
}
|
|
310
|
-
}
|
|
311
|
-
socket.addEventListener("message", (event) => {
|
|
312
|
-
this.handleMessage(JSON.parse(event.data.toString()));
|
|
313
|
-
});
|
|
458
|
+
}
|
|
314
459
|
});
|
|
315
460
|
}
|
|
316
|
-
|
|
317
|
-
if (
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
)
|
|
461
|
+
endUtterance(utterance) {
|
|
462
|
+
if (utterance.status !== "streaming") return;
|
|
463
|
+
if (utterance.chunks.length === 0) {
|
|
464
|
+
this.removeUtterance(utterance);
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
utterance.status = "flushing";
|
|
468
|
+
if (this.socket.isOpen) this.flushAudio(utterance);
|
|
469
|
+
}
|
|
470
|
+
flushAudio(utterance) {
|
|
471
|
+
if (!this.socket.send(JSON.stringify({ type: "input_audio.flush" }))) {
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
this.log("Flushed audio input");
|
|
475
|
+
clearTimeout(utterance.timeout);
|
|
476
|
+
utterance.timeout = setTimeout(() => {
|
|
477
|
+
this.log("Transcription timeout");
|
|
478
|
+
this.staleFlushes.push(Date.now() + this.transcriptionTimeout);
|
|
479
|
+
this.complete(utterance, utterance.text);
|
|
480
|
+
}, this.transcriptionTimeout);
|
|
481
|
+
}
|
|
482
|
+
get transcriptionTimeout() {
|
|
483
|
+
return this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT;
|
|
484
|
+
}
|
|
485
|
+
/** Whether an answer to a timed out flush may still be on its way */
|
|
486
|
+
hasStaleFlush() {
|
|
487
|
+
const now = Date.now();
|
|
488
|
+
while (this.staleFlushes.length > 0 && this.staleFlushes[0] <= now) {
|
|
489
|
+
this.log("Forgetting a flush that was never answered");
|
|
490
|
+
this.staleFlushes.shift();
|
|
491
|
+
}
|
|
492
|
+
return this.staleFlushes.length > 0;
|
|
493
|
+
}
|
|
494
|
+
complete(utterance, transcript) {
|
|
495
|
+
if (utterance.status === "done") return;
|
|
496
|
+
this.removeUtterance(utterance);
|
|
497
|
+
this.emit("Transcript", transcript.trim());
|
|
498
|
+
}
|
|
499
|
+
removeUtterance(utterance) {
|
|
500
|
+
this.dropUtterance(utterance);
|
|
501
|
+
this.utterances = this.utterances.filter((u) => u !== utterance);
|
|
502
|
+
}
|
|
503
|
+
dropUtterance(utterance) {
|
|
504
|
+
utterance.status = "done";
|
|
505
|
+
clearTimeout(utterance.timeout);
|
|
506
|
+
utterance.timeout = void 0;
|
|
332
507
|
}
|
|
333
508
|
sendAudioChunk(chunk) {
|
|
334
|
-
|
|
335
|
-
this.socket.send(
|
|
509
|
+
return this.socket.send(
|
|
336
510
|
JSON.stringify({
|
|
337
511
|
type: "input_audio.append",
|
|
338
512
|
audio: chunk.toString("base64")
|
|
339
513
|
})
|
|
340
514
|
);
|
|
341
515
|
}
|
|
342
|
-
flushAudio() {
|
|
343
|
-
if (!this.socket) return;
|
|
344
|
-
this.socket.send(JSON.stringify({ type: "input_audio.flush" }));
|
|
345
|
-
this.log("Flushed audio input");
|
|
346
|
-
}
|
|
347
|
-
handleMessage(message) {
|
|
348
|
-
switch (message.type) {
|
|
349
|
-
case "transcription.text.delta":
|
|
350
|
-
if (typeof message.text === "string") {
|
|
351
|
-
this.transcriptDelta += message.text;
|
|
352
|
-
this.log(`Received transcript delta: "${message.text}"`);
|
|
353
|
-
}
|
|
354
|
-
break;
|
|
355
|
-
case "transcription.done": {
|
|
356
|
-
const transcript = typeof message.text === "string" ? message.text : this.transcriptDelta;
|
|
357
|
-
this.log(`Received completed transcript: "${transcript}"`);
|
|
358
|
-
this.emit("Transcript", transcript.trim());
|
|
359
|
-
this.transcriptDelta = "";
|
|
360
|
-
this.audioChunksPending.length = 0;
|
|
361
|
-
if (this.transcriptionTimeout) {
|
|
362
|
-
clearTimeout(this.transcriptionTimeout);
|
|
363
|
-
this.transcriptionTimeout = void 0;
|
|
364
|
-
}
|
|
365
|
-
break;
|
|
366
|
-
}
|
|
367
|
-
case "error":
|
|
368
|
-
this.log("Error:", message.error ?? message);
|
|
369
|
-
break;
|
|
370
|
-
default:
|
|
371
|
-
break;
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
reconnect() {
|
|
375
|
-
if (this.destroyed) return;
|
|
376
|
-
this.retryCount++;
|
|
377
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
378
|
-
this.log("Max retries reached, giving up");
|
|
379
|
-
this.emit("Failed", this.audioChunksPending);
|
|
380
|
-
return;
|
|
381
|
-
}
|
|
382
|
-
this.initPromise = new Promise((resolve) => {
|
|
383
|
-
this.log("Reconnecting...");
|
|
384
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
385
|
-
this.reconnectTimeout = void 0;
|
|
386
|
-
this.initWS().then(() => {
|
|
387
|
-
this.retryCount = 0;
|
|
388
|
-
if (this.audioChunksPending.length > 0) {
|
|
389
|
-
this.log("Sending audio chunks again");
|
|
390
|
-
this.transcriptDelta = "";
|
|
391
|
-
this.audioChunksPending.forEach(
|
|
392
|
-
(chunk) => this.sendAudioChunk(chunk)
|
|
393
|
-
);
|
|
394
|
-
}
|
|
395
|
-
}).then(resolve).catch((error) => {
|
|
396
|
-
this.log("Reconnection error:", error);
|
|
397
|
-
this.reconnect();
|
|
398
|
-
});
|
|
399
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
400
|
-
});
|
|
401
|
-
}
|
|
402
516
|
};
|
|
403
|
-
function closeSocket(socket) {
|
|
404
|
-
socket.removeAllListeners();
|
|
405
|
-
socket.on("error", () => {
|
|
406
|
-
});
|
|
407
|
-
if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
|
|
408
|
-
socket.close(1e3);
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
517
|
|
|
412
518
|
// src/MistralTTS.ts
|
|
413
|
-
var
|
|
519
|
+
var import_server4 = require("@micdrop/server");
|
|
414
520
|
var DEFAULT_MODEL3 = "voxtral-mini-tts-3-0";
|
|
415
521
|
var DEFAULT_VOICE_ID = "voice_3150";
|
|
416
|
-
var DEFAULT_BASE_URL = "https://api.mistral.ai";
|
|
417
522
|
var MISTRAL_SAMPLE_RATE = 24e3;
|
|
418
|
-
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
419
523
|
var FLOAT32_BYTES = 4;
|
|
420
|
-
var
|
|
524
|
+
var PAGE_SIZE = 100;
|
|
525
|
+
var MistralTTS = class extends import_server4.SentenceTTS {
|
|
421
526
|
constructor(options) {
|
|
422
527
|
super();
|
|
423
528
|
this.options = options;
|
|
424
529
|
}
|
|
530
|
+
/**
|
|
531
|
+
* Lists the models that speak, each one once with its aliases.
|
|
532
|
+
*
|
|
533
|
+
* The list is fetched once and shared by every call of the process for a
|
|
534
|
+
* day. `refresh` fetches it again.
|
|
535
|
+
*/
|
|
536
|
+
static listModels(options) {
|
|
537
|
+
return listMistralModels(
|
|
538
|
+
"tts",
|
|
539
|
+
({ capabilities }) => capabilities.audio_speech,
|
|
540
|
+
options
|
|
541
|
+
);
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Lists the voices the account can speak with, the presets and its own.
|
|
545
|
+
*
|
|
546
|
+
* `language`, such as `fr` or `fr-FR`, keeps the voices that speak it.
|
|
547
|
+
*
|
|
548
|
+
* The list is fetched once and shared by every call of the process for a
|
|
549
|
+
* day. `refresh` fetches it again, for a voice added a moment ago.
|
|
550
|
+
*/
|
|
551
|
+
static listVoices(options) {
|
|
552
|
+
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
553
|
+
return (0, import_server4.cachedVoices)(
|
|
554
|
+
["mistral", baseUrl],
|
|
555
|
+
options.apiKey,
|
|
556
|
+
() => fetchVoices(options.apiKey, baseUrl),
|
|
557
|
+
options.refresh
|
|
558
|
+
).then((voices) => (0, import_server4.voicesForLanguage)(voices, options.language));
|
|
559
|
+
}
|
|
560
|
+
get sampleRate() {
|
|
561
|
+
return MISTRAL_SAMPLE_RATE;
|
|
562
|
+
}
|
|
425
563
|
async synthesize(text, signal) {
|
|
426
564
|
const response = await this.request(text, signal);
|
|
427
565
|
if (!response.body) throw new Error("Mistral answered without a body");
|
|
428
|
-
const resampler = new import_server3.Pcm16Resampler(
|
|
429
|
-
MISTRAL_SAMPLE_RATE,
|
|
430
|
-
OUTPUT_SAMPLE_RATE
|
|
431
|
-
);
|
|
432
566
|
let leftover = Buffer.alloc(0);
|
|
433
567
|
for await (const event of readEvents(response.body, signal)) {
|
|
434
568
|
if (event.type === "error") {
|
|
@@ -447,7 +581,7 @@ var MistralTTS = class extends import_server3.SentenceTTS {
|
|
|
447
581
|
for (let i = 0; i < samples.length; i++) {
|
|
448
582
|
samples[i] = data.readFloatLE(i * FLOAT32_BYTES);
|
|
449
583
|
}
|
|
450
|
-
if (!this.emitAudio(
|
|
584
|
+
if (!this.emitAudio((0, import_server4.float32ToPcm16)(samples))) break;
|
|
451
585
|
}
|
|
452
586
|
return void 0;
|
|
453
587
|
}
|
|
@@ -504,6 +638,35 @@ async function* readEvents(body, signal) {
|
|
|
504
638
|
});
|
|
505
639
|
}
|
|
506
640
|
}
|
|
641
|
+
async function fetchVoices(apiKey, baseUrl) {
|
|
642
|
+
const voices = [];
|
|
643
|
+
for (let offset = 0, total = 1; offset < total; offset += PAGE_SIZE) {
|
|
644
|
+
const response = await fetch(
|
|
645
|
+
`${baseUrl}/v1/audio/voices?offset=${offset}&limit=${PAGE_SIZE}`,
|
|
646
|
+
{ headers: { Authorization: `Bearer ${apiKey}` } }
|
|
647
|
+
);
|
|
648
|
+
if (!response.ok) throw new Error(`Mistral answered ${response.status}`);
|
|
649
|
+
const data = await response.json();
|
|
650
|
+
total = data.total ?? 0;
|
|
651
|
+
if (data.items.length === 0) break;
|
|
652
|
+
for (const voice of data.items) {
|
|
653
|
+
voices.push({
|
|
654
|
+
// A preset goes by its slug, a voice of the account by its id
|
|
655
|
+
id: voice.slug || voice.id,
|
|
656
|
+
name: voice.name,
|
|
657
|
+
// Mistral writes en_us where the other providers write en-US
|
|
658
|
+
languages: voice.languages?.map((code) => {
|
|
659
|
+
const [language, region] = code.split("_");
|
|
660
|
+
return region ? `${language}-${region.toUpperCase()}` : language;
|
|
661
|
+
}),
|
|
662
|
+
gender: (0, import_server4.voiceGender)(voice.gender),
|
|
663
|
+
description: voice.description || void 0,
|
|
664
|
+
custom: voice.type !== "preset"
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return voices;
|
|
669
|
+
}
|
|
507
670
|
// Annotate the CommonJS export names for ESM import in node:
|
|
508
671
|
0 && (module.exports = {
|
|
509
672
|
MistralAgent,
|