@micdrop/mistral 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -6,17 +6,67 @@ var DEFAULT_MODEL = "mistral-large-latest";
6
6
  var DEFAULT_MAX_RETRY = 3;
7
7
  var DEFAULT_MAX_STEPS = 5;
8
8
  var DEFAULT_RETRY_DELAY = 500;
9
+ var StepInterruptedError = class extends Error {
10
+ constructor(outcome) {
11
+ super("Answer interrupted");
12
+ this.outcome = outcome;
13
+ }
14
+ };
9
15
  var MistralAgent = class extends Agent {
10
16
  constructor(options) {
11
17
  super(options);
12
18
  this.mistral = new Mistral({ apiKey: options.apiKey });
13
19
  }
14
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
15
- if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
16
- console.error("[MistralAgent] Max steps reached");
17
- return;
20
+ async generateAnswer(stream, signal) {
21
+ const maxSteps = this.options.maxSteps || DEFAULT_MAX_STEPS;
22
+ const maxRetry = this.options.maxRetry || DEFAULT_MAX_RETRY;
23
+ for (let step = 0; step < maxSteps; step++) {
24
+ let outcome;
25
+ for (let tryCount = 0; !outcome; tryCount++) {
26
+ try {
27
+ outcome = await this.generateStep(stream, signal);
28
+ } catch (error) {
29
+ if (signal.aborted) return;
30
+ if (error instanceof StepInterruptedError) {
31
+ outcome = error.outcome;
32
+ break;
33
+ }
34
+ console.error("[MistralAgent] Error:", error);
35
+ if (tryCount >= maxRetry) {
36
+ this.log("Max retries reached, giving up");
37
+ this.emit("Failed");
38
+ return;
39
+ }
40
+ await new Promise(
41
+ (resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
42
+ );
43
+ if (signal.aborted) return;
44
+ }
45
+ }
46
+ if (signal.aborted) return;
47
+ if (outcome.fullAnswer) {
48
+ const { message, metadata } = this.extract(outcome.fullAnswer);
49
+ this.addAssistantMessage(message, metadata);
50
+ return;
51
+ }
52
+ if (outcome.skipAnswer) {
53
+ this.conversation.push({
54
+ role: "assistant",
55
+ content: " "
56
+ });
57
+ return;
58
+ }
18
59
  }
19
- const answerCount = this.answerCount;
60
+ console.error("[MistralAgent] Max steps reached");
61
+ }
62
+ /**
63
+ * Runs one request, writing its text to the stream and running its tools.
64
+ *
65
+ * It throws a plain error when the request failed before producing
66
+ * anything, which can then be retried, and a {@link StepInterruptedError}
67
+ * once something was written or done.
68
+ */
69
+ async generateStep(stream, signal) {
20
70
  if (this.conversation.length === 1) {
21
71
  this.conversation.push({
22
72
  role: "user",
@@ -25,30 +75,36 @@ var MistralAgent = class extends Agent {
25
75
  }
26
76
  let extracting = false;
27
77
  const extractOptions = this.getExtractOptions();
78
+ const outcome = { fullAnswer: "", skipAnswer: false };
79
+ let produced = false;
28
80
  try {
29
- const result = await this.mistral.chat.stream({
30
- model: this.options.model || DEFAULT_MODEL,
31
- messages: this.buildMessages(),
32
- tools: this.buildTools(),
33
- ...this.options.settings
34
- });
35
- let fullAnswer = "";
36
- let skipAnswer = false;
81
+ const result = await this.mistral.chat.stream(
82
+ {
83
+ model: this.options.model || DEFAULT_MODEL,
84
+ messages: this.buildMessages(),
85
+ tools: this.buildTools(),
86
+ ...this.options.settings
87
+ },
88
+ { signal }
89
+ );
37
90
  for await (const event of result) {
38
- if (answerCount !== this.answerCount) return;
91
+ if (signal.aborted) return outcome;
39
92
  const delta = event.data.choices[0]?.delta;
40
93
  const chunk = delta?.content;
41
94
  const toolCalls = delta?.toolCalls;
42
95
  if (typeof chunk === "string") {
43
96
  if (chunk === "") continue;
44
97
  this.log(`Answer chunk: "${chunk}"`);
45
- fullAnswer += chunk;
98
+ outcome.fullAnswer += chunk;
99
+ produced = true;
46
100
  if (extractOptions) {
47
101
  if (!extracting) {
48
- const startTagIndex = fullAnswer.indexOf(extractOptions.startTag);
102
+ const startTagIndex = outcome.fullAnswer.indexOf(
103
+ extractOptions.startTag
104
+ );
49
105
  if (startTagIndex !== -1) {
50
106
  extracting = true;
51
- const messagePart = fullAnswer.slice(0, startTagIndex).trimEnd();
107
+ const messagePart = outcome.fullAnswer.slice(0, startTagIndex).trimEnd();
52
108
  stream.write(messagePart);
53
109
  continue;
54
110
  }
@@ -59,42 +115,27 @@ var MistralAgent = class extends Agent {
59
115
  stream.write(chunk);
60
116
  } else if (toolCalls?.length) {
61
117
  for (const toolCall of toolCalls) {
118
+ if (signal.aborted) return outcome;
119
+ produced = true;
62
120
  const result2 = await this.executeTool({
63
121
  role: "tool_call",
64
122
  toolCallId: toolCall.id || "",
65
123
  toolName: toolCall.function.name,
66
- parameters: toolCall.function.arguments
124
+ // The SDK hands the arguments over parsed or as JSON
125
+ parameters: typeof toolCall.function.arguments === "string" ? toolCall.function.arguments : JSON.stringify(toolCall.function.arguments)
67
126
  });
68
127
  if (result2.skipAnswer) {
69
- skipAnswer = true;
128
+ outcome.skipAnswer = true;
70
129
  }
71
130
  }
72
131
  }
73
132
  }
74
- if (answerCount !== this.answerCount) return;
75
- if (fullAnswer) {
76
- const { message, metadata } = this.extract(fullAnswer);
77
- this.addAssistantMessage(message, metadata);
78
- } else if (skipAnswer) {
79
- this.conversation.push({
80
- role: "assistant",
81
- content: " "
82
- });
83
- } else {
84
- await this.generateAnswer(stream, stepCount + 1);
85
- }
86
133
  } catch (error) {
87
- console.error("[MistralAgent] Error:", error);
88
- if (tryCount < (this.options.maxRetry || DEFAULT_MAX_RETRY)) {
89
- await new Promise(
90
- (resolve) => setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)
91
- );
92
- await this.generateAnswer(stream, stepCount, tryCount + 1);
93
- } else {
94
- this.log("Max retries reached, giving up");
95
- this.emit("Failed");
96
- }
134
+ if (!produced || signal.aborted) throw error;
135
+ console.error("[MistralAgent] Error after a partial answer:", error);
136
+ throw new StepInterruptedError(outcome);
97
137
  }
138
+ return outcome;
98
139
  }
99
140
  buildMessages() {
100
141
  return this.conversation.map(
@@ -116,7 +157,8 @@ var MistralAgent = class extends Agent {
116
157
  id: message.toolCallId,
117
158
  function: {
118
159
  name: message.toolName,
119
- arguments: JSON.stringify(message.parameters)
160
+ // Already the JSON the model wrote
161
+ arguments: message.parameters
120
162
  }
121
163
  }
122
164
  ]
@@ -149,17 +191,10 @@ var MistralAgent = class extends Agent {
149
191
  }
150
192
  }));
151
193
  }
152
- cancel() {
153
- if (!this.answering) return;
154
- this.log("Cancel");
155
- this.answering = false;
156
- this.answerCount++;
157
- }
158
194
  };
159
195
 
160
196
  // src/MistralSTT.ts
161
- import { STT } from "@micdrop/server";
162
- import WebSocket from "ws";
197
+ import { ReconnectingSocket, STT } from "@micdrop/server";
163
198
  var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-3-0";
164
199
  var DEFAULT_ENCODING = "pcm_s16le";
165
200
  var SAMPLE_RATE = 16e3;
@@ -171,62 +206,117 @@ var MistralSTT = class extends STT {
171
206
  constructor(options) {
172
207
  super();
173
208
  this.options = options;
174
- this.audioChunksPending = [];
175
- // Store audio chunks to send them again if reconnecting
176
- this.transcriptDelta = "";
177
- // Accumulated transcription.text.delta chunks
178
- this.retryCount = 0;
179
- this.destroyed = false;
180
- this.initPromise = this.connect();
209
+ // Utterances not done yet, oldest first. The server answers them in the
210
+ // order their audio went out, so the transcript in progress is the oldest's.
211
+ this.utterances = [];
212
+ // Flushes of utterances that timed out on this session, whose deltas and
213
+ // transcription.done may still arrive and must not go to the next one. Each
214
+ // is forgotten after another timeout, so an answer that never comes does not
215
+ // swallow every transcript after it.
216
+ this.staleFlushes = [];
217
+ // A new session knows nothing of the previous one: every pending utterance
218
+ // is sent again in full and replays its whole transcript, so the deltas it
219
+ // gathered start over to avoid duplicating them
220
+ this.onOpen = ({ reconnected }) => {
221
+ this.staleFlushes = [];
222
+ const pending = this.utterances.filter(
223
+ (utterance) => utterance.chunks.length > 0
224
+ );
225
+ if (pending.length === 0) return;
226
+ this.log(
227
+ reconnected ? "Sending audio chunks again" : "Sending queued audio chunks"
228
+ );
229
+ for (const utterance of pending) {
230
+ utterance.text = "";
231
+ utterance.chunks.forEach((chunk) => this.sendAudioChunk(chunk));
232
+ if (utterance.status === "flushing") this.flushAudio(utterance);
233
+ }
234
+ };
235
+ this.onMessage = (data) => {
236
+ let message;
237
+ try {
238
+ message = JSON.parse(data.toString());
239
+ } catch {
240
+ this.log("Ignoring a message that is not JSON");
241
+ return;
242
+ }
243
+ switch (message.type) {
244
+ case "transcription.text.delta": {
245
+ if (typeof message.text !== "string") break;
246
+ this.log(`Received transcript delta: "${message.text}"`);
247
+ if (this.hasStaleFlush()) break;
248
+ const utterance = this.utterances[0];
249
+ if (utterance) utterance.text += message.text;
250
+ break;
251
+ }
252
+ case "transcription.done": {
253
+ if (this.hasStaleFlush()) {
254
+ this.staleFlushes.shift();
255
+ this.log("Ignoring the transcript of a timed out utterance");
256
+ break;
257
+ }
258
+ const utterance = this.utterances[0];
259
+ if (utterance?.status !== "flushing") {
260
+ this.log("Ignoring an unexpected transcription.done");
261
+ break;
262
+ }
263
+ const transcript = typeof message.text === "string" ? message.text : utterance.text;
264
+ this.log(`Received completed transcript: "${transcript}"`);
265
+ this.complete(utterance, transcript);
266
+ break;
267
+ }
268
+ case "error":
269
+ this.log("Error:", message.error ?? message);
270
+ break;
271
+ default:
272
+ break;
273
+ }
274
+ };
275
+ this.onFailed = () => {
276
+ const chunks = this.utterances.flatMap((utterance) => utterance.chunks);
277
+ this.utterances.forEach((utterance) => this.dropUtterance(utterance));
278
+ this.utterances = [];
279
+ this.emit("Failed", chunks);
280
+ };
281
+ this.socket = new ReconnectingSocket({
282
+ url: this.getURL(),
283
+ clientOptions: {
284
+ headers: {
285
+ Authorization: `Bearer ${this.options.apiKey}`
286
+ }
287
+ },
288
+ connectionTimeout: this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT,
289
+ retryDelay: this.options.retryDelay ?? DEFAULT_RETRY_DELAY2,
290
+ maxRetry: this.options.maxRetry ?? DEFAULT_MAX_RETRY2,
291
+ // The audio format must be set before any audio is appended
292
+ handshake: {
293
+ start: (socket) => socket.send(this.getSessionUpdate())
294
+ },
295
+ log: (...message) => this.log(...message)
296
+ });
297
+ this.socket.on("Open", this.onOpen);
298
+ this.socket.on("Message", this.onMessage);
299
+ this.socket.on("Failed", this.onFailed);
300
+ this.socket.connect();
181
301
  }
182
302
  transcribe(audioStream) {
183
- let started = false;
184
- audioStream.on("data", async (chunk) => {
185
- if (!started) {
186
- started = true;
187
- this.transcriptDelta = "";
303
+ this.socket.connect();
304
+ const utterance = { status: "streaming", chunks: [], text: "" };
305
+ this.utterances.push(utterance);
306
+ audioStream.on("data", (chunk) => {
307
+ if (utterance.status !== "streaming") return;
308
+ utterance.chunks.push(chunk);
309
+ if (this.sendAudioChunk(chunk)) {
310
+ this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
188
311
  }
189
- this.audioChunksPending.push(chunk);
190
- await this.initPromise;
191
- this.sendAudioChunk(chunk);
192
- this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
193
- });
194
- audioStream.on("end", async () => {
195
- await this.initPromise;
196
- if (this.audioChunksPending.length === 0) return;
197
- this.flushAudio();
198
- this.transcriptionTimeout = setTimeout(() => {
199
- this.transcriptionTimeout = void 0;
200
- this.log("Transcription timeout");
201
- this.emit("Transcript", this.transcriptDelta.trim());
202
- this.transcriptDelta = "";
203
- this.audioChunksPending.length = 0;
204
- }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
205
312
  });
313
+ audioStream.on("end", () => this.endUtterance(utterance));
206
314
  }
207
315
  destroy() {
208
316
  super.destroy();
209
- this.destroyed = true;
210
- if (this.reconnectTimeout) {
211
- clearTimeout(this.reconnectTimeout);
212
- this.reconnectTimeout = void 0;
213
- }
214
- if (this.transcriptionTimeout) {
215
- clearTimeout(this.transcriptionTimeout);
216
- this.transcriptionTimeout = void 0;
217
- }
218
- if (this.connectionTimeout) {
219
- clearTimeout(this.connectionTimeout);
220
- this.connectionTimeout = void 0;
221
- }
222
- if (this.socket) closeSocket(this.socket);
223
- this.socket = void 0;
224
- }
225
- connect() {
226
- return this.initWS().catch((error) => {
227
- console.error("[MistralSTT] Connection error:", error);
228
- this.reconnect();
229
- });
317
+ this.socket.destroy();
318
+ this.utterances.forEach((utterance) => this.dropUtterance(utterance));
319
+ this.utterances = [];
230
320
  }
231
321
  getURL() {
232
322
  const model = this.options.model || DEFAULT_MODEL2;
@@ -234,142 +324,76 @@ var MistralSTT = class extends STT {
234
324
  model
235
325
  )}`;
236
326
  }
237
- async initWS() {
238
- return new Promise((resolve, reject) => {
239
- const socket = new WebSocket(this.getURL(), {
240
- headers: {
241
- Authorization: `Bearer ${this.options.apiKey}`
242
- }
243
- });
244
- this.socket = socket;
245
- const timeout = setTimeout(() => {
246
- this.log("Connection timeout");
247
- closeSocket(socket);
248
- this.socket = void 0;
249
- reject(new Error("WebSocket connection timeout"));
250
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
251
- this.connectionTimeout = timeout;
252
- socket.addEventListener("open", () => {
253
- clearTimeout(timeout);
254
- this.sendSessionUpdate();
255
- this.log("Connection opened");
256
- resolve();
257
- });
258
- socket.addEventListener("error", (error) => {
259
- clearTimeout(timeout);
260
- this.log("WebSocket error:", error);
261
- reject(new Error("WebSocket connection error"));
262
- });
263
- socket.addEventListener("close", ({ code, reason }) => {
264
- clearTimeout(timeout);
265
- this.socket?.removeAllListeners();
266
- this.socket = void 0;
267
- if (code !== 1e3) {
268
- this.reconnect();
269
- } else {
270
- this.log("Connection closed", { code, reason });
327
+ getSessionUpdate() {
328
+ return JSON.stringify({
329
+ type: "session.update",
330
+ session: {
331
+ audio_format: {
332
+ encoding: this.options.encoding || DEFAULT_ENCODING,
333
+ sample_rate: SAMPLE_RATE
334
+ },
335
+ ...this.options.targetStreamingDelayMs !== void 0 && {
336
+ target_streaming_delay_ms: this.options.targetStreamingDelayMs
271
337
  }
272
- });
273
- socket.addEventListener("message", (event) => {
274
- this.handleMessage(JSON.parse(event.data.toString()));
275
- });
338
+ }
276
339
  });
277
340
  }
278
- sendSessionUpdate() {
279
- if (!this.socket) return;
280
- this.socket.send(
281
- JSON.stringify({
282
- type: "session.update",
283
- session: {
284
- audio_format: {
285
- encoding: this.options.encoding || DEFAULT_ENCODING,
286
- sample_rate: SAMPLE_RATE
287
- },
288
- ...this.options.targetStreamingDelayMs !== void 0 && {
289
- target_streaming_delay_ms: this.options.targetStreamingDelayMs
290
- }
291
- }
292
- })
293
- );
341
+ endUtterance(utterance) {
342
+ if (utterance.status !== "streaming") return;
343
+ if (utterance.chunks.length === 0) {
344
+ this.removeUtterance(utterance);
345
+ return;
346
+ }
347
+ utterance.status = "flushing";
348
+ if (this.socket.isOpen) this.flushAudio(utterance);
349
+ }
350
+ flushAudio(utterance) {
351
+ if (!this.socket.send(JSON.stringify({ type: "input_audio.flush" }))) {
352
+ return;
353
+ }
354
+ this.log("Flushed audio input");
355
+ clearTimeout(utterance.timeout);
356
+ utterance.timeout = setTimeout(() => {
357
+ this.log("Transcription timeout");
358
+ this.staleFlushes.push(Date.now() + this.transcriptionTimeout);
359
+ this.complete(utterance, utterance.text);
360
+ }, this.transcriptionTimeout);
361
+ }
362
+ get transcriptionTimeout() {
363
+ return this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT;
364
+ }
365
+ /** Whether an answer to a timed out flush may still be on its way */
366
+ hasStaleFlush() {
367
+ const now = Date.now();
368
+ while (this.staleFlushes.length > 0 && this.staleFlushes[0] <= now) {
369
+ this.log("Forgetting a flush that was never answered");
370
+ this.staleFlushes.shift();
371
+ }
372
+ return this.staleFlushes.length > 0;
373
+ }
374
+ complete(utterance, transcript) {
375
+ if (utterance.status === "done") return;
376
+ this.removeUtterance(utterance);
377
+ this.emit("Transcript", transcript.trim());
378
+ }
379
+ removeUtterance(utterance) {
380
+ this.dropUtterance(utterance);
381
+ this.utterances = this.utterances.filter((u) => u !== utterance);
382
+ }
383
+ dropUtterance(utterance) {
384
+ utterance.status = "done";
385
+ clearTimeout(utterance.timeout);
386
+ utterance.timeout = void 0;
294
387
  }
295
388
  sendAudioChunk(chunk) {
296
- if (!this.socket) return;
297
- this.socket.send(
389
+ return this.socket.send(
298
390
  JSON.stringify({
299
391
  type: "input_audio.append",
300
392
  audio: chunk.toString("base64")
301
393
  })
302
394
  );
303
395
  }
304
- flushAudio() {
305
- if (!this.socket) return;
306
- this.socket.send(JSON.stringify({ type: "input_audio.flush" }));
307
- this.log("Flushed audio input");
308
- }
309
- handleMessage(message) {
310
- switch (message.type) {
311
- case "transcription.text.delta":
312
- if (typeof message.text === "string") {
313
- this.transcriptDelta += message.text;
314
- this.log(`Received transcript delta: "${message.text}"`);
315
- }
316
- break;
317
- case "transcription.done": {
318
- const transcript = typeof message.text === "string" ? message.text : this.transcriptDelta;
319
- this.log(`Received completed transcript: "${transcript}"`);
320
- this.emit("Transcript", transcript.trim());
321
- this.transcriptDelta = "";
322
- this.audioChunksPending.length = 0;
323
- if (this.transcriptionTimeout) {
324
- clearTimeout(this.transcriptionTimeout);
325
- this.transcriptionTimeout = void 0;
326
- }
327
- break;
328
- }
329
- case "error":
330
- this.log("Error:", message.error ?? message);
331
- break;
332
- default:
333
- break;
334
- }
335
- }
336
- reconnect() {
337
- if (this.destroyed) return;
338
- this.retryCount++;
339
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
340
- this.log("Max retries reached, giving up");
341
- this.emit("Failed", this.audioChunksPending);
342
- return;
343
- }
344
- this.initPromise = new Promise((resolve) => {
345
- this.log("Reconnecting...");
346
- this.reconnectTimeout = setTimeout(() => {
347
- this.reconnectTimeout = void 0;
348
- this.initWS().then(() => {
349
- this.retryCount = 0;
350
- if (this.audioChunksPending.length > 0) {
351
- this.log("Sending audio chunks again");
352
- this.transcriptDelta = "";
353
- this.audioChunksPending.forEach(
354
- (chunk) => this.sendAudioChunk(chunk)
355
- );
356
- }
357
- }).then(resolve).catch((error) => {
358
- this.log("Reconnection error:", error);
359
- this.reconnect();
360
- });
361
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
362
- });
363
- }
364
396
  };
365
- function closeSocket(socket) {
366
- socket.removeAllListeners();
367
- socket.on("error", () => {
368
- });
369
- if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
370
- socket.close(1e3);
371
- }
372
- }
373
397
 
374
398
  // src/MistralTTS.ts
375
399
  import { float32ToPcm16, Pcm16Resampler, SentenceTTS } from "@micdrop/server";