@micdrop/mistral 1.0.11 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -1
- package/dist/index.d.mts +42 -3
- package/dist/index.d.ts +42 -3
- package/dist/index.js +209 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +207 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -31,6 +31,28 @@ new MicdropServer(socket, {
|
|
|
31
31
|
})
|
|
32
32
|
```
|
|
33
33
|
|
|
34
|
+
## Mistral STT (Speech-to-Text)
|
|
35
|
+
|
|
36
|
+
Real-time transcription with Voxtral.
|
|
37
|
+
|
|
38
|
+
### Usage
|
|
39
|
+
|
|
40
|
+
```typescript
|
|
41
|
+
import { MistralSTT } from '@micdrop/mistral'
|
|
42
|
+
import { MicdropServer } from '@micdrop/server'
|
|
43
|
+
|
|
44
|
+
const stt = new MistralSTT({
|
|
45
|
+
apiKey: process.env.MISTRAL_API_KEY || '',
|
|
46
|
+
model: 'voxtral-mini-transcribe-realtime-2602', // Default model
|
|
47
|
+
})
|
|
48
|
+
|
|
49
|
+
// Use with MicdropServer
|
|
50
|
+
new MicdropServer(socket, {
|
|
51
|
+
stt,
|
|
52
|
+
// ... other options
|
|
53
|
+
})
|
|
54
|
+
```
|
|
55
|
+
|
|
34
56
|
## Documentation
|
|
35
57
|
|
|
36
58
|
Read full [documentation of the Mistral integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/mistral) on the [website](https://micdrop.dev).
|
|
@@ -41,4 +63,4 @@ MIT
|
|
|
41
63
|
|
|
42
64
|
## Author
|
|
43
65
|
|
|
44
|
-
Originally developed for [Raconte.ai](https://www.raconte.ai) and open sourced by [
|
|
66
|
+
Originally developed for [Raconte.ai](https://www.raconte.ai), created and open sourced by [Godefroy de Compreignac](https://github.com/Godefroy)
|
package/dist/index.d.mts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { Agent, AgentOptions } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, STT } from '@micdrop/server';
|
|
2
2
|
import { ChatCompletionStreamRequest } from '@mistralai/mistralai/models/components';
|
|
3
|
-
import { Writable } from 'stream';
|
|
3
|
+
import { Writable, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
interface MistralAgentOptions extends AgentOptions {
|
|
6
6
|
apiKey: string;
|
|
@@ -19,4 +19,43 @@ declare class MistralAgent extends Agent<MistralAgentOptions> {
|
|
|
19
19
|
cancel(): void;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Mistral Real-time STT (Voxtral)
|
|
24
|
+
*
|
|
25
|
+
* @see https://docs.mistral.ai/studio-api/audio/speech_to_text/realtime_transcription
|
|
26
|
+
*/
|
|
27
|
+
type MistralAudioEncoding = 'pcm_s16le' | 'pcm_s32le' | 'pcm_f16le' | 'pcm_f32le' | 'pcm_mulaw' | 'pcm_alaw';
|
|
28
|
+
interface MistralSTTOptions {
|
|
29
|
+
apiKey: string;
|
|
30
|
+
model?: string;
|
|
31
|
+
encoding?: MistralAudioEncoding;
|
|
32
|
+
targetStreamingDelayMs?: number;
|
|
33
|
+
connectionTimeout?: number;
|
|
34
|
+
transcriptionTimeout?: number;
|
|
35
|
+
retryDelay?: number;
|
|
36
|
+
maxRetry?: number;
|
|
37
|
+
}
|
|
38
|
+
declare class MistralSTT extends STT {
|
|
39
|
+
private options;
|
|
40
|
+
private socket?;
|
|
41
|
+
private initPromise;
|
|
42
|
+
private reconnectTimeout?;
|
|
43
|
+
private transcriptionTimeout?;
|
|
44
|
+
private audioChunksPending;
|
|
45
|
+
private transcriptDelta;
|
|
46
|
+
private retryCount;
|
|
47
|
+
private destroyed;
|
|
48
|
+
constructor(options: MistralSTTOptions);
|
|
49
|
+
transcribe(audioStream: Readable): void;
|
|
50
|
+
destroy(): void;
|
|
51
|
+
private connect;
|
|
52
|
+
private getURL;
|
|
53
|
+
private initWS;
|
|
54
|
+
private sendSessionUpdate;
|
|
55
|
+
private sendAudioChunk;
|
|
56
|
+
private flushAudio;
|
|
57
|
+
private handleMessage;
|
|
58
|
+
private reconnect;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export { MistralAgent, type MistralAgentOptions, type MistralAudioEncoding, MistralSTT, type MistralSTTOptions };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { Agent, AgentOptions } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, STT } from '@micdrop/server';
|
|
2
2
|
import { ChatCompletionStreamRequest } from '@mistralai/mistralai/models/components';
|
|
3
|
-
import { Writable } from 'stream';
|
|
3
|
+
import { Writable, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
interface MistralAgentOptions extends AgentOptions {
|
|
6
6
|
apiKey: string;
|
|
@@ -19,4 +19,43 @@ declare class MistralAgent extends Agent<MistralAgentOptions> {
|
|
|
19
19
|
cancel(): void;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Mistral Real-time STT (Voxtral)
|
|
24
|
+
*
|
|
25
|
+
* @see https://docs.mistral.ai/studio-api/audio/speech_to_text/realtime_transcription
|
|
26
|
+
*/
|
|
27
|
+
type MistralAudioEncoding = 'pcm_s16le' | 'pcm_s32le' | 'pcm_f16le' | 'pcm_f32le' | 'pcm_mulaw' | 'pcm_alaw';
|
|
28
|
+
interface MistralSTTOptions {
|
|
29
|
+
apiKey: string;
|
|
30
|
+
model?: string;
|
|
31
|
+
encoding?: MistralAudioEncoding;
|
|
32
|
+
targetStreamingDelayMs?: number;
|
|
33
|
+
connectionTimeout?: number;
|
|
34
|
+
transcriptionTimeout?: number;
|
|
35
|
+
retryDelay?: number;
|
|
36
|
+
maxRetry?: number;
|
|
37
|
+
}
|
|
38
|
+
declare class MistralSTT extends STT {
|
|
39
|
+
private options;
|
|
40
|
+
private socket?;
|
|
41
|
+
private initPromise;
|
|
42
|
+
private reconnectTimeout?;
|
|
43
|
+
private transcriptionTimeout?;
|
|
44
|
+
private audioChunksPending;
|
|
45
|
+
private transcriptDelta;
|
|
46
|
+
private retryCount;
|
|
47
|
+
private destroyed;
|
|
48
|
+
constructor(options: MistralSTTOptions);
|
|
49
|
+
transcribe(audioStream: Readable): void;
|
|
50
|
+
destroy(): void;
|
|
51
|
+
private connect;
|
|
52
|
+
private getURL;
|
|
53
|
+
private initWS;
|
|
54
|
+
private sendSessionUpdate;
|
|
55
|
+
private sendAudioChunk;
|
|
56
|
+
private flushAudio;
|
|
57
|
+
private handleMessage;
|
|
58
|
+
private reconnect;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export { MistralAgent, type MistralAgentOptions, type MistralAudioEncoding, MistralSTT, type MistralSTTOptions };
|
package/dist/index.js
CHANGED
|
@@ -30,7 +30,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/index.ts
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
|
-
MistralAgent: () => MistralAgent
|
|
33
|
+
MistralAgent: () => MistralAgent,
|
|
34
|
+
MistralSTT: () => MistralSTT
|
|
34
35
|
});
|
|
35
36
|
module.exports = __toCommonJS(index_exports);
|
|
36
37
|
|
|
@@ -192,8 +193,214 @@ var MistralAgent = class extends import_server.Agent {
|
|
|
192
193
|
this.answerCount++;
|
|
193
194
|
}
|
|
194
195
|
};
|
|
196
|
+
|
|
197
|
+
// src/MistralSTT.ts
|
|
198
|
+
var import_server2 = require("@micdrop/server");
|
|
199
|
+
var import_ws = __toESM(require("ws"));
|
|
200
|
+
var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-2602";
|
|
201
|
+
var DEFAULT_ENCODING = "pcm_s16le";
|
|
202
|
+
var SAMPLE_RATE = 16e3;
|
|
203
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
204
|
+
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
205
|
+
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
206
|
+
var DEFAULT_MAX_RETRY2 = 3;
|
|
207
|
+
var MistralSTT = class extends import_server2.STT {
|
|
208
|
+
constructor(options) {
|
|
209
|
+
super();
|
|
210
|
+
this.options = options;
|
|
211
|
+
this.audioChunksPending = [];
|
|
212
|
+
// Store audio chunks to send them again if reconnecting
|
|
213
|
+
this.transcriptDelta = "";
|
|
214
|
+
// Accumulated transcription.text.delta chunks
|
|
215
|
+
this.retryCount = 0;
|
|
216
|
+
this.destroyed = false;
|
|
217
|
+
this.initPromise = this.connect();
|
|
218
|
+
}
|
|
219
|
+
transcribe(audioStream) {
|
|
220
|
+
let started = false;
|
|
221
|
+
audioStream.on("data", async (chunk) => {
|
|
222
|
+
if (!started) {
|
|
223
|
+
started = true;
|
|
224
|
+
this.transcriptDelta = "";
|
|
225
|
+
}
|
|
226
|
+
this.audioChunksPending.push(chunk);
|
|
227
|
+
await this.initPromise;
|
|
228
|
+
this.sendAudioChunk(chunk);
|
|
229
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
230
|
+
});
|
|
231
|
+
audioStream.on("end", async () => {
|
|
232
|
+
await this.initPromise;
|
|
233
|
+
if (this.audioChunksPending.length === 0) return;
|
|
234
|
+
this.flushAudio();
|
|
235
|
+
this.transcriptionTimeout = setTimeout(() => {
|
|
236
|
+
this.transcriptionTimeout = void 0;
|
|
237
|
+
this.log("Transcription timeout");
|
|
238
|
+
this.emit("Transcript", this.transcriptDelta.trim());
|
|
239
|
+
this.transcriptDelta = "";
|
|
240
|
+
this.audioChunksPending.length = 0;
|
|
241
|
+
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
destroy() {
|
|
245
|
+
super.destroy();
|
|
246
|
+
this.destroyed = true;
|
|
247
|
+
if (this.reconnectTimeout) {
|
|
248
|
+
clearTimeout(this.reconnectTimeout);
|
|
249
|
+
this.reconnectTimeout = void 0;
|
|
250
|
+
}
|
|
251
|
+
if (this.transcriptionTimeout) {
|
|
252
|
+
clearTimeout(this.transcriptionTimeout);
|
|
253
|
+
this.transcriptionTimeout = void 0;
|
|
254
|
+
}
|
|
255
|
+
this.socket?.removeAllListeners();
|
|
256
|
+
if (this.socket?.readyState === import_ws.default.OPEN) {
|
|
257
|
+
this.socket?.close(1e3);
|
|
258
|
+
}
|
|
259
|
+
this.socket = void 0;
|
|
260
|
+
}
|
|
261
|
+
connect() {
|
|
262
|
+
return this.initWS().catch((error) => {
|
|
263
|
+
console.error("[MistralSTT] Connection error:", error);
|
|
264
|
+
this.reconnect();
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
getURL() {
|
|
268
|
+
const model = this.options.model || DEFAULT_MODEL2;
|
|
269
|
+
return `wss://api.mistral.ai/v1/audio/transcriptions/realtime?model=${encodeURIComponent(
|
|
270
|
+
model
|
|
271
|
+
)}`;
|
|
272
|
+
}
|
|
273
|
+
async initWS() {
|
|
274
|
+
return new Promise((resolve, reject) => {
|
|
275
|
+
const socket = new import_ws.default(this.getURL(), {
|
|
276
|
+
headers: {
|
|
277
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
this.socket = socket;
|
|
281
|
+
const timeout = setTimeout(() => {
|
|
282
|
+
this.log("Connection timeout");
|
|
283
|
+
socket.removeAllListeners();
|
|
284
|
+
socket.close();
|
|
285
|
+
this.socket = void 0;
|
|
286
|
+
reject(new Error("WebSocket connection timeout"));
|
|
287
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
288
|
+
socket.addEventListener("open", () => {
|
|
289
|
+
clearTimeout(timeout);
|
|
290
|
+
this.sendSessionUpdate();
|
|
291
|
+
this.log("Connection opened");
|
|
292
|
+
resolve();
|
|
293
|
+
});
|
|
294
|
+
socket.addEventListener("error", (error) => {
|
|
295
|
+
clearTimeout(timeout);
|
|
296
|
+
this.log("WebSocket error:", error);
|
|
297
|
+
reject(new Error("WebSocket connection error"));
|
|
298
|
+
});
|
|
299
|
+
socket.addEventListener("close", ({ code, reason }) => {
|
|
300
|
+
clearTimeout(timeout);
|
|
301
|
+
this.socket?.removeAllListeners();
|
|
302
|
+
this.socket = void 0;
|
|
303
|
+
if (code !== 1e3) {
|
|
304
|
+
this.reconnect();
|
|
305
|
+
} else {
|
|
306
|
+
this.log("Connection closed", { code, reason });
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
socket.addEventListener("message", (event) => {
|
|
310
|
+
this.handleMessage(JSON.parse(event.data.toString()));
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
sendSessionUpdate() {
|
|
315
|
+
if (!this.socket) return;
|
|
316
|
+
this.socket.send(
|
|
317
|
+
JSON.stringify({
|
|
318
|
+
type: "session.update",
|
|
319
|
+
session: {
|
|
320
|
+
audio_format: {
|
|
321
|
+
encoding: this.options.encoding || DEFAULT_ENCODING,
|
|
322
|
+
sample_rate: SAMPLE_RATE
|
|
323
|
+
},
|
|
324
|
+
...this.options.targetStreamingDelayMs !== void 0 && {
|
|
325
|
+
target_streaming_delay_ms: this.options.targetStreamingDelayMs
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
})
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
sendAudioChunk(chunk) {
|
|
332
|
+
if (!this.socket) return;
|
|
333
|
+
this.socket.send(
|
|
334
|
+
JSON.stringify({
|
|
335
|
+
type: "input_audio.append",
|
|
336
|
+
audio: chunk.toString("base64")
|
|
337
|
+
})
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
flushAudio() {
|
|
341
|
+
if (!this.socket) return;
|
|
342
|
+
this.socket.send(JSON.stringify({ type: "input_audio.flush" }));
|
|
343
|
+
this.log("Flushed audio input");
|
|
344
|
+
}
|
|
345
|
+
handleMessage(message) {
|
|
346
|
+
switch (message.type) {
|
|
347
|
+
case "transcription.text.delta":
|
|
348
|
+
if (typeof message.text === "string") {
|
|
349
|
+
this.transcriptDelta += message.text;
|
|
350
|
+
this.log(`Received transcript delta: "${message.text}"`);
|
|
351
|
+
}
|
|
352
|
+
break;
|
|
353
|
+
case "transcription.done": {
|
|
354
|
+
const transcript = typeof message.text === "string" ? message.text : this.transcriptDelta;
|
|
355
|
+
this.log(`Received completed transcript: "${transcript}"`);
|
|
356
|
+
this.emit("Transcript", transcript.trim());
|
|
357
|
+
this.transcriptDelta = "";
|
|
358
|
+
this.audioChunksPending.length = 0;
|
|
359
|
+
if (this.transcriptionTimeout) {
|
|
360
|
+
clearTimeout(this.transcriptionTimeout);
|
|
361
|
+
this.transcriptionTimeout = void 0;
|
|
362
|
+
}
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
case "error":
|
|
366
|
+
this.log("Error:", message.error ?? message);
|
|
367
|
+
break;
|
|
368
|
+
default:
|
|
369
|
+
break;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
reconnect() {
|
|
373
|
+
if (this.destroyed) return;
|
|
374
|
+
this.retryCount++;
|
|
375
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
376
|
+
this.log("Max retries reached, giving up");
|
|
377
|
+
this.emit("Failed", this.audioChunksPending);
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
this.initPromise = new Promise((resolve) => {
|
|
381
|
+
this.log("Reconnecting...");
|
|
382
|
+
this.reconnectTimeout = setTimeout(() => {
|
|
383
|
+
this.reconnectTimeout = void 0;
|
|
384
|
+
this.initWS().then(() => {
|
|
385
|
+
this.retryCount = 0;
|
|
386
|
+
if (this.audioChunksPending.length > 0) {
|
|
387
|
+
this.log("Sending audio chunks again");
|
|
388
|
+
this.transcriptDelta = "";
|
|
389
|
+
this.audioChunksPending.forEach(
|
|
390
|
+
(chunk) => this.sendAudioChunk(chunk)
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
}).then(resolve).catch((error) => {
|
|
394
|
+
this.log("Reconnection error:", error);
|
|
395
|
+
this.reconnect();
|
|
396
|
+
});
|
|
397
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
};
|
|
195
401
|
// Annotate the CommonJS export names for ESM import in node:
|
|
196
402
|
0 && (module.exports = {
|
|
197
|
-
MistralAgent
|
|
403
|
+
MistralAgent,
|
|
404
|
+
MistralSTT
|
|
198
405
|
});
|
|
199
406
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/MistralAgent.ts"],"sourcesContent":["export * from './MistralAgent'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport { Mistral } from '@mistralai/mistralai'\nimport {\n ChatCompletionStreamRequest,\n ChatCompletionStreamRequestMessages,\n} from '@mistralai/mistralai/models/components'\nimport { Writable } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport interface MistralAgentOptions extends AgentOptions {\n apiKey: string\n model?: string\n settings?: Omit<ChatCompletionStreamRequest, 'messages' | 'model'>\n maxRetry?: number\n retryDelay?: number\n maxSteps?: number\n}\n\nconst DEFAULT_MODEL = 'mistral-large-latest'\nconst DEFAULT_MAX_RETRY = 3\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_RETRY_DELAY = 500\n\nexport class MistralAgent extends Agent<MistralAgentOptions> {\n private mistral: Mistral\n\n constructor(options: MistralAgentOptions) {\n super(options)\n this.mistral = new Mistral({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: Writable,\n stepCount = 0,\n tryCount = 0\n ) {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[MistralAgent] Max steps reached')\n return\n }\n\n const answerCount = this.answerCount\n\n // Hack: Mistral needs a user message if there is only a system message\n if (this.conversation.length === 1) {\n this.conversation.push({\n role: 'user',\n content: '',\n })\n }\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n const result = await this.mistral.chat.stream({\n model: this.options.model || DEFAULT_MODEL,\n messages: this.buildMessages(),\n tools: this.buildTools(),\n ...this.options.settings,\n })\n let fullAnswer = ''\n let skipAnswer = false\n\n // Handle response events\n for await (const event of result) {\n if (answerCount !== this.answerCount) return\n const delta = event.data.choices[0]?.delta\n const chunk = delta?.content\n const toolCalls = delta?.toolCalls\n\n // Text chunk\n if (typeof chunk === 'string') {\n if (chunk === '') continue\n this.log(`Answer chunk: \"${chunk}\"`)\n fullAnswer += chunk\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = fullAnswer.indexOf(extractOptions.startTag)\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = fullAnswer.slice(0, startTagIndex).trimEnd()\n stream.write(messagePart)\n continue\n }\n } else {\n // Extracting, don't write to stream\n continue\n }\n }\n stream.write(chunk)\n } else if (toolCalls?.length) {\n for (const toolCall of toolCalls) {\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.id || '',\n toolName: toolCall.function.name,\n parameters: toolCall.function.arguments as string,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n }\n }\n\n if (answerCount !== this.answerCount) return\n\n if (fullAnswer) {\n // Add full answer to conversation\n const { message, metadata } = this.extract(fullAnswer)\n this.addAssistantMessage(message, metadata)\n } else if (skipAnswer) {\n // Hack: Mistral needs an assistant message after tool calls\n this.conversation.push({\n role: 'assistant',\n content: ' ',\n })\n } else {\n // Query again in case of tool call\n await this.generateAnswer(stream, stepCount + 1)\n }\n } catch (error: any) {\n console.error('[MistralAgent] Error:', error)\n if (tryCount < (this.options.maxRetry || DEFAULT_MAX_RETRY)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildMessages(): ChatCompletionStreamRequestMessages[] {\n return this.conversation.map(\n (message): ChatCompletionStreamRequestMessages => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n role: 'assistant',\n toolCalls: [\n {\n type: 'function',\n id: message.toolCallId,\n function: {\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n },\n },\n ],\n }\n case 'tool_result':\n return {\n role: 'tool',\n toolCallId: message.toolCallId,\n name: message.toolName,\n content: [\n {\n type: 'text',\n text: message.output,\n },\n ],\n }\n }\n }\n )\n }\n\n private buildTools(): ChatCompletionStreamRequest['tools'] {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n function: {\n name: tool.name,\n description: tool.description,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n },\n }))\n }\n\n cancel() {\n if (!this.answering) return\n this.log('Cancel')\n this.answering = false\n\n // Increment counter to avoid processing messages from previous calls\n this.answerCount++\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,uBAAwB;AAMxB,iBAAgC;AAWhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAErB,IAAM,eAAN,cAA2B,oBAA2B;AAAA,EAG3D,YAAY,SAA8B;AACxC,UAAM,OAAO;AACb,SAAK,UAAU,IAAI,yBAAQ,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EACvD;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACX;AACA,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,kCAAkC;AAChD;AAAA,IACF;AAEA,UAAM,cAAc,KAAK;AAGzB,QAAI,KAAK,aAAa,WAAW,GAAG;AAClC,WAAK,aAAa,KAAK;AAAA,QACrB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,OAAO;AAAA,QAC5C,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,UAAU,KAAK,cAAc;AAAA,QAC7B,OAAO,KAAK,WAAW;AAAA,QACvB,GAAG,KAAK,QAAQ;AAAA,MAClB,CAAC;AACD,UAAI,aAAa;AACjB,UAAI,aAAa;AAGjB,uBAAiB,SAAS,QAAQ;AAChC,YAAI,gBAAgB,KAAK,YAAa;AACtC,cAAM,QAAQ,MAAM,KAAK,QAAQ,CAAC,GAAG;AACrC,cAAM,QAAQ,OAAO;AACrB,cAAM,YAAY,OAAO;AAGzB,YAAI,OAAO,UAAU,UAAU;AAC7B,cAAI,UAAU,GAAI;AAClB,eAAK,IAAI,kBAAkB,KAAK,GAAG;AACnC,wBAAc;AAGd,cAAI,gBAAgB;AAClB,gBAAI,CAAC,YAAY;AACf,oBAAM,gBAAgB,WAAW,QAAQ,eAAe,QAAQ;AAChE,kBAAI,kBAAkB,IAAI;AACxB,6BAAa;AACb,sBAAM,cAAc,WAAW,MAAM,GAAG,aAAa,EAAE,QAAQ;AAC/D,uBAAO,MAAM,WAAW;AACxB;AAAA,cACF;AAAA,YACF,OAAO;AAEL;AAAA,YACF;AAAA,UACF;AACA,iBAAO,MAAM,KAAK;AAAA,QACpB,WAAW,WAAW,QAAQ;AAC5B,qBAAW,YAAY,WAAW;AAChC,kBAAMA,UAAS,MAAM,KAAK,YAAY;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,SAAS,MAAM;AAAA,cAC3B,UAAU,SAAS,SAAS;AAAA,cAC5B,YAAY,SAAS,SAAS;AAAA,YAChC,CAAC;AACD,gBAAIA,QAAO,YAAY;AACrB,2BAAa;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,UAAI,gBAAgB,KAAK,YAAa;AAEtC,UAAI,YAAY;AAEd,cAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,UAAU;AACrD,aAAK,oBAAoB,SAAS,QAAQ;AAAA,MAC5C,WAAW,YAAY;AAErB,aAAK,aAAa,KAAK;AAAA,UACrB,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AAAA,MACH,OAAO;AAEL,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AAAA,IACF,SAAS,OAAY;AACnB,cAAQ,MAAM,yBAAyB,KAAK;AAC5C,UAAI,YAAY,KAAK,QAAQ,YAAY,oBAAoB;AAC3D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,gBAAuD;AAC7D,WAAO,KAAK,aAAa;AAAA,MACvB,CAAC,YAAiD;AAChD,gBAAQ,QAAQ,MAAM;AAAA,UACpB,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AACH,mBAAO;AAAA,cACL,MAAM,QAAQ;AAAA,cACd,SAAS,QAAQ;AAAA,YACnB;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,WAAW;AAAA,gBACT;AAAA,kBACE,MAAM;AAAA,kBACN,IAAI,QAAQ;AAAA,kBACZ,UAAU;AAAA,oBACR,MAAM,QAAQ;AAAA,oBACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,kBAC9C;AAAA,gBACF;AAAA,cACF;AAAA,YACF;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,YAAY,QAAQ;AAAA,cACpB,MAAM,QAAQ;AAAA,cACd,SAAS;AAAA,gBACP;AAAA,kBACE,MAAM;AAAA,kBACN,MAAM,QAAQ;AAAA,gBAChB;AAAA,cACF;AAAA,YACF;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAAmD;AAEzD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,UAAU;AAAA,QACR,MAAM,KAAK;AAAA,QACX,aAAa,KAAK;AAAA,QAClB,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,MACzD;AAAA,IACF,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,UAAW;AACrB,SAAK,IAAI,QAAQ;AACjB,SAAK,YAAY;AAGjB,SAAK;AAAA,EACP;AACF;","names":["result","z"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/MistralAgent.ts","../src/MistralSTT.ts"],"sourcesContent":["export * from './MistralAgent'\nexport * from './MistralSTT'\n","import { Agent, AgentOptions } from '@micdrop/server'\nimport { Mistral } from '@mistralai/mistralai'\nimport {\n ChatCompletionStreamRequest,\n ChatCompletionStreamRequestMessages,\n} from '@mistralai/mistralai/models/components'\nimport { Writable } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport interface MistralAgentOptions extends AgentOptions {\n apiKey: string\n model?: string\n settings?: Omit<ChatCompletionStreamRequest, 'messages' | 'model'>\n maxRetry?: number\n retryDelay?: number\n maxSteps?: number\n}\n\nconst DEFAULT_MODEL = 'mistral-large-latest'\nconst DEFAULT_MAX_RETRY = 3\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_RETRY_DELAY = 500\n\nexport class MistralAgent extends Agent<MistralAgentOptions> {\n private mistral: Mistral\n\n constructor(options: MistralAgentOptions) {\n super(options)\n this.mistral = new Mistral({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: Writable,\n stepCount = 0,\n tryCount = 0\n ) {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[MistralAgent] Max steps reached')\n return\n }\n\n const answerCount = this.answerCount\n\n // Hack: Mistral needs a user message if there is only a system message\n if (this.conversation.length === 1) {\n this.conversation.push({\n role: 'user',\n content: '',\n })\n }\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n const result = await this.mistral.chat.stream({\n model: this.options.model || DEFAULT_MODEL,\n messages: this.buildMessages(),\n tools: this.buildTools(),\n ...this.options.settings,\n })\n let fullAnswer = ''\n let skipAnswer = false\n\n // Handle response events\n for await (const event of result) {\n if (answerCount !== this.answerCount) return\n const delta = event.data.choices[0]?.delta\n const chunk = delta?.content\n const toolCalls = delta?.toolCalls\n\n // Text chunk\n if (typeof chunk === 'string') {\n if (chunk === '') continue\n this.log(`Answer chunk: \"${chunk}\"`)\n fullAnswer += chunk\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = fullAnswer.indexOf(extractOptions.startTag)\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = fullAnswer.slice(0, startTagIndex).trimEnd()\n stream.write(messagePart)\n continue\n }\n } else {\n // Extracting, don't write to stream\n continue\n }\n }\n stream.write(chunk)\n } else if (toolCalls?.length) {\n for (const toolCall of toolCalls) {\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.id || '',\n toolName: toolCall.function.name,\n parameters: toolCall.function.arguments as string,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n }\n }\n\n if (answerCount !== this.answerCount) return\n\n if (fullAnswer) {\n // Add full answer to conversation\n const { message, metadata } = this.extract(fullAnswer)\n this.addAssistantMessage(message, metadata)\n } else if (skipAnswer) {\n // Hack: Mistral needs an assistant message after tool calls\n this.conversation.push({\n role: 'assistant',\n content: ' ',\n })\n } else {\n // Query again in case of tool call\n await this.generateAnswer(stream, stepCount + 1)\n }\n } catch (error: any) {\n console.error('[MistralAgent] Error:', error)\n if (tryCount < (this.options.maxRetry || DEFAULT_MAX_RETRY)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildMessages(): ChatCompletionStreamRequestMessages[] {\n return this.conversation.map(\n (message): ChatCompletionStreamRequestMessages => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n role: 'assistant',\n toolCalls: [\n {\n type: 'function',\n id: message.toolCallId,\n function: {\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n },\n },\n ],\n }\n case 'tool_result':\n return {\n role: 'tool',\n toolCallId: message.toolCallId,\n name: message.toolName,\n content: [\n {\n type: 'text',\n text: message.output,\n },\n ],\n }\n }\n }\n )\n }\n\n private buildTools(): ChatCompletionStreamRequest['tools'] {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n function: {\n name: tool.name,\n description: tool.description,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n },\n }))\n }\n\n cancel() {\n if (!this.answering) return\n this.log('Cancel')\n this.answering = false\n\n // Increment counter to avoid processing messages from previous calls\n this.answerCount++\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\n\n/**\n * Mistral Real-time STT (Voxtral)\n *\n * @see https://docs.mistral.ai/studio-api/audio/speech_to_text/realtime_transcription\n */\n\nexport type MistralAudioEncoding =\n | 'pcm_s16le'\n | 'pcm_s32le'\n | 'pcm_f16le'\n | 'pcm_f32le'\n | 'pcm_mulaw'\n | 'pcm_alaw'\n\nexport interface MistralSTTOptions {\n apiKey: string\n model?: string\n encoding?: MistralAudioEncoding\n targetStreamingDelayMs?: number\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'voxtral-mini-transcribe-realtime-2602'\nconst DEFAULT_ENCODING: MistralAudioEncoding = 'pcm_s16le'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class MistralSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private transcriptDelta = '' // Accumulated transcription.text.delta chunks\n private retryCount = 0\n private destroyed = false\n\n constructor(private options: MistralSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.connect()\n }\n\n transcribe(audioStream: Readable) {\n let started = false\n\n // Read audio stream and send to Mistral\n audioStream.on('data', async (chunk: Buffer) => {\n // New utterance: a flush finalizes and resets the transcription context,\n // so start from a clean transcript for this stream, but only once it\n // carries something. Voice detection opens a stream on any noise and many\n // of them never carry a word, and clearing here rather than on entry\n // keeps those from wiping the transcript of the utterance before, whose\n // flush may still be on its way back.\n if (!started) {\n started = true\n this.transcriptDelta = ''\n }\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end: flush buffered audio so the server finalizes the\n // transcription (transcription.done). Unlike input_audio.end, a flush keeps\n // the session alive, so the same connection is reused for the next utterance.\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.flushAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', this.transcriptDelta.trim())\n this.transcriptDelta = ''\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n this.destroyed = true\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private connect(): Promise<void> {\n return this.initWS().catch((error) => {\n console.error('[MistralSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n private getURL() {\n const model = this.options.model || DEFAULT_MODEL\n return `wss://api.mistral.ai/v1/audio/transcriptions/realtime?model=${encodeURIComponent(\n model\n )}`\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(this.getURL(), {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n })\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener('close', ({ code, reason }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n })\n\n socket.addEventListener('message', (event) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the audio format. Must be sent before any audio is appended;\n // Mistral rejects format updates once audio has started.\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n audio_format: {\n encoding: this.options.encoding || DEFAULT_ENCODING,\n sample_rate: SAMPLE_RATE,\n },\n ...(this.options.targetStreamingDelayMs !== undefined && {\n target_streaming_delay_ms: this.options.targetStreamingDelayMs,\n }),\n },\n })\n )\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n this.socket.send(\n JSON.stringify({\n type: 'input_audio.append',\n audio: chunk.toString('base64'),\n })\n )\n }\n\n private flushAudio() {\n if (!this.socket) return\n // Flush buffered audio so the server processes the tail and emits a final\n // transcription.done for this utterance. We deliberately do NOT send\n // input_audio.end: that terminates the session (the server drops the\n // connection), whereas a flush finalizes the current utterance and keeps the\n // session ready for the next one.\n this.socket.send(JSON.stringify({ type: 'input_audio.flush' }))\n this.log('Flushed audio input')\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n case 'transcription.text.delta':\n if (typeof message.text === 'string') {\n this.transcriptDelta += message.text\n this.log(`Received transcript delta: \"${message.text}\"`)\n }\n break\n\n case 'transcription.done': {\n const transcript =\n typeof message.text === 'string' ? message.text : this.transcriptDelta\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript.trim())\n // Reset state and clear timeout. The session stays alive after a flush,\n // so the same connection is reused for the next utterance.\n this.transcriptDelta = ''\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n }\n\n case 'error':\n this.log('Error:', message.error ?? message)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n if (this.destroyed) return\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Resend audio chunks if reconnecting during transcription. The new\n // session is fresh and replays the whole utterance, so reset the\n // accumulated deltas to avoid duplicating them.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.transcriptDelta = ''\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,oBAAoC;AACpC,uBAAwB;AAMxB,iBAAgC;AAWhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAErB,IAAM,eAAN,cAA2B,oBAA2B;AAAA,EAG3D,YAAY,SAA8B;AACxC,UAAM,OAAO;AACb,SAAK,UAAU,IAAI,yBAAQ,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EACvD;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACX;AACA,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,kCAAkC;AAChD;AAAA,IACF;AAEA,UAAM,cAAc,KAAK;AAGzB,QAAI,KAAK,aAAa,WAAW,GAAG;AAClC,WAAK,aAAa,KAAK;AAAA,QACrB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,OAAO;AAAA,QAC5C,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,UAAU,KAAK,cAAc;AAAA,QAC7B,OAAO,KAAK,WAAW;AAAA,QACvB,GAAG,KAAK,QAAQ;AAAA,MAClB,CAAC;AACD,UAAI,aAAa;AACjB,UAAI,aAAa;AAGjB,uBAAiB,SAAS,QAAQ;AAChC,YAAI,gBAAgB,KAAK,YAAa;AACtC,cAAM,QAAQ,MAAM,KAAK,QAAQ,CAAC,GAAG;AACrC,cAAM,QAAQ,OAAO;AACrB,cAAM,YAAY,OAAO;AAGzB,YAAI,OAAO,UAAU,UAAU;AAC7B,cAAI,UAAU,GAAI;AAClB,eAAK,IAAI,kBAAkB,KAAK,GAAG;AACnC,wBAAc;AAGd,cAAI,gBAAgB;AAClB,gBAAI,CAAC,YAAY;AACf,oBAAM,gBAAgB,WAAW,QAAQ,eAAe,QAAQ;AAChE,kBAAI,kBAAkB,IAAI;AACxB,6BAAa;AACb,sBAAM,cAAc,WAAW,MAAM,GAAG,aAAa,EAAE,QAAQ;AAC/D,uBAAO,MAAM,WAAW;AACxB;AAAA,cACF;AAAA,YACF,OAAO;AAEL;AAAA,YACF;AAAA,UACF;AACA,iBAAO,MAAM,KAAK;AAAA,QACpB,WAAW,WAAW,QAAQ;AAC5B,qBAAW,YAAY,WAAW;AAChC,kBAAMA,UAAS,MAAM,KAAK,YAAY;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,SAAS,MAAM;AAAA,cAC3B,UAAU,SAAS,SAAS;AAAA,cAC5B,YAAY,SAAS,SAAS;AAAA,YAChC,CAAC;AACD,gBAAIA,QAAO,YAAY;AACrB,2BAAa;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,UAAI,gBAAgB,KAAK,YAAa;AAEtC,UAAI,YAAY;AAEd,cAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,UAAU;AACrD,aAAK,oBAAoB,SAAS,QAAQ;AAAA,MAC5C,WAAW,YAAY;AAErB,aAAK,aAAa,KAAK;AAAA,UACrB,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AAAA,MACH,OAAO;AAEL,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AAAA,IACF,SAAS,OAAY;AACnB,cAAQ,MAAM,yBAAyB,KAAK;AAC5C,UAAI,YAAY,KAAK,QAAQ,YAAY,oBAAoB;AAC3D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,gBAAuD;AAC7D,WAAO,KAAK,aAAa;AAAA,MACvB,CAAC,YAAiD;AAChD,gBAAQ,QAAQ,MAAM;AAAA,UACpB,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AACH,mBAAO;AAAA,cACL,MAAM,QAAQ;AAAA,cACd,SAAS,QAAQ;AAAA,YACnB;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,WAAW;AAAA,gBACT;AAAA,kBACE,MAAM;AAAA,kBACN,IAAI,QAAQ;AAAA,kBACZ,UAAU;AAAA,oBACR,MAAM,QAAQ;AAAA,oBACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,kBAC9C;AAAA,gBACF;AAAA,cACF;AAAA,YACF;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,YAAY,QAAQ;AAAA,cACpB,MAAM,QAAQ;AAAA,cACd,SAAS;AAAA,gBACP;AAAA,kBACE,MAAM;AAAA,kBACN,MAAM,QAAQ;AAAA,gBAChB;AAAA,cACF;AAAA,YACF;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAAmD;AAEzD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,UAAU;AAAA,QACR,MAAM,KAAK;AAAA,QACX,aAAa,KAAK;AAAA,QAClB,gBAAY,yBAAa,KAAK,eAAe,WAAAC,QAAE,OAAO,CAAC;AAAA,MACzD;AAAA,IACF,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,UAAW;AACrB,SAAK,IAAI,QAAQ;AACjB,SAAK,YAAY;AAGjB,SAAK;AAAA,EACP;AACF;;;AC5MA,IAAAC,iBAAoB;AAEpB,gBAAsB;AA2BtB,IAAMC,iBAAgB;AACtB,IAAM,mBAAyC;AAC/C,IAAM,cAAc;AACpB,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAMC,qBAAoB;AAEnB,IAAM,aAAN,cAAyB,mBAAI;AAAA,EAUlC,YAAoB,SAA4B;AAC9C,UAAM;AADY;AALpB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,kBAAkB;AAC1B;AAAA,SAAQ,aAAa;AACrB,SAAQ,YAAY;AAMlB,SAAK,cAAc,KAAK,QAAQ;AAAA,EAClC;AAAA,EAEA,WAAW,aAAuB;AAChC,QAAI,UAAU;AAGd,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAO9C,UAAI,CAAC,SAAS;AACZ,kBAAU;AACV,aAAK,kBAAkB;AAAA,MACzB;AACA,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAKD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,WAAW;AAGhB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,KAAK,gBAAgB,KAAK,CAAC;AACnD,aAAK,kBAAkB;AACvB,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,SAAK,YAAY;AACjB,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAAC,QAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,UAAyB;AAC/B,WAAO,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AACpC,cAAQ,MAAM,kCAAkC,KAAK;AACrD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEQ,SAAS;AACf,UAAM,QAAQ,KAAK,QAAQ,SAASH;AACpC,WAAO,+DAA+D;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI,UAAAG,QAAU,KAAK,OAAO,GAAG;AAAA,QAC1C,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,QAC9C;AAAA,MACF,CAAC;AACD,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAU;AAC1C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,EAAE,MAAM,OAAO,MAAM;AACrD,qBAAa,OAAO;AACpB,aAAK,QAAQ,mBAAmB;AAChC,aAAK,SAAS;AAEd,YAAI,SAAS,KAAM;AACjB,eAAK,UAAU;AAAA,QACjB,OAAO;AACL,eAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,QAChD;AAAA,MACF,CAAC;AAED,aAAO,iBAAiB,WAAW,CAAC,UAAU;AAC5C,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAIlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,cAAc;AAAA,YACZ,UAAU,KAAK,QAAQ,YAAY;AAAA,YACnC,aAAa;AAAA,UACf;AAAA,UACA,GAAI,KAAK,QAAQ,2BAA2B,UAAa;AAAA,YACvD,2BAA2B,KAAK,QAAQ;AAAA,UAC1C;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,MAAM,SAAS,QAAQ;AAAA,MAChC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,aAAa;AACnB,QAAI,CAAC,KAAK,OAAQ;AAMlB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC,CAAC;AAC9D,SAAK,IAAI,qBAAqB;AAAA,EAChC;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,YAAI,OAAO,QAAQ,SAAS,UAAU;AACpC,eAAK,mBAAmB,QAAQ;AAChC,eAAK,IAAI,+BAA+B,QAAQ,IAAI,GAAG;AAAA,QACzD;AACA;AAAA,MAEF,KAAK,sBAAsB;AACzB,cAAM,aACJ,OAAO,QAAQ,SAAS,WAAW,QAAQ,OAAO,KAAK;AACzD,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,WAAW,KAAK,CAAC;AAGzC,aAAK,kBAAkB;AACvB,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,SAAS,OAAO;AAC3C;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,QAAI,KAAK,UAAW;AACpB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAYD,qBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,kBAAkB;AACvB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AACF;","names":["result","z","import_server","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","DEFAULT_MAX_RETRY","WebSocket"]}
|
package/dist/index.mjs
CHANGED
|
@@ -156,7 +156,213 @@ var MistralAgent = class extends Agent {
|
|
|
156
156
|
this.answerCount++;
|
|
157
157
|
}
|
|
158
158
|
};
|
|
159
|
+
|
|
160
|
+
// src/MistralSTT.ts
|
|
161
|
+
import { STT } from "@micdrop/server";
|
|
162
|
+
import WebSocket from "ws";
|
|
163
|
+
var DEFAULT_MODEL2 = "voxtral-mini-transcribe-realtime-2602";
|
|
164
|
+
var DEFAULT_ENCODING = "pcm_s16le";
|
|
165
|
+
var SAMPLE_RATE = 16e3;
|
|
166
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
167
|
+
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
168
|
+
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
169
|
+
var DEFAULT_MAX_RETRY2 = 3;
|
|
170
|
+
var MistralSTT = class extends STT {
|
|
171
|
+
constructor(options) {
|
|
172
|
+
super();
|
|
173
|
+
this.options = options;
|
|
174
|
+
this.audioChunksPending = [];
|
|
175
|
+
// Store audio chunks to send them again if reconnecting
|
|
176
|
+
this.transcriptDelta = "";
|
|
177
|
+
// Accumulated transcription.text.delta chunks
|
|
178
|
+
this.retryCount = 0;
|
|
179
|
+
this.destroyed = false;
|
|
180
|
+
this.initPromise = this.connect();
|
|
181
|
+
}
|
|
182
|
+
transcribe(audioStream) {
|
|
183
|
+
let started = false;
|
|
184
|
+
audioStream.on("data", async (chunk) => {
|
|
185
|
+
if (!started) {
|
|
186
|
+
started = true;
|
|
187
|
+
this.transcriptDelta = "";
|
|
188
|
+
}
|
|
189
|
+
this.audioChunksPending.push(chunk);
|
|
190
|
+
await this.initPromise;
|
|
191
|
+
this.sendAudioChunk(chunk);
|
|
192
|
+
this.log(`Sent audio chunk (${chunk.byteLength} bytes)`);
|
|
193
|
+
});
|
|
194
|
+
audioStream.on("end", async () => {
|
|
195
|
+
await this.initPromise;
|
|
196
|
+
if (this.audioChunksPending.length === 0) return;
|
|
197
|
+
this.flushAudio();
|
|
198
|
+
this.transcriptionTimeout = setTimeout(() => {
|
|
199
|
+
this.transcriptionTimeout = void 0;
|
|
200
|
+
this.log("Transcription timeout");
|
|
201
|
+
this.emit("Transcript", this.transcriptDelta.trim());
|
|
202
|
+
this.transcriptDelta = "";
|
|
203
|
+
this.audioChunksPending.length = 0;
|
|
204
|
+
}, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
destroy() {
|
|
208
|
+
super.destroy();
|
|
209
|
+
this.destroyed = true;
|
|
210
|
+
if (this.reconnectTimeout) {
|
|
211
|
+
clearTimeout(this.reconnectTimeout);
|
|
212
|
+
this.reconnectTimeout = void 0;
|
|
213
|
+
}
|
|
214
|
+
if (this.transcriptionTimeout) {
|
|
215
|
+
clearTimeout(this.transcriptionTimeout);
|
|
216
|
+
this.transcriptionTimeout = void 0;
|
|
217
|
+
}
|
|
218
|
+
this.socket?.removeAllListeners();
|
|
219
|
+
if (this.socket?.readyState === WebSocket.OPEN) {
|
|
220
|
+
this.socket?.close(1e3);
|
|
221
|
+
}
|
|
222
|
+
this.socket = void 0;
|
|
223
|
+
}
|
|
224
|
+
connect() {
|
|
225
|
+
return this.initWS().catch((error) => {
|
|
226
|
+
console.error("[MistralSTT] Connection error:", error);
|
|
227
|
+
this.reconnect();
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
getURL() {
|
|
231
|
+
const model = this.options.model || DEFAULT_MODEL2;
|
|
232
|
+
return `wss://api.mistral.ai/v1/audio/transcriptions/realtime?model=${encodeURIComponent(
|
|
233
|
+
model
|
|
234
|
+
)}`;
|
|
235
|
+
}
|
|
236
|
+
async initWS() {
|
|
237
|
+
return new Promise((resolve, reject) => {
|
|
238
|
+
const socket = new WebSocket(this.getURL(), {
|
|
239
|
+
headers: {
|
|
240
|
+
Authorization: `Bearer ${this.options.apiKey}`
|
|
241
|
+
}
|
|
242
|
+
});
|
|
243
|
+
this.socket = socket;
|
|
244
|
+
const timeout = setTimeout(() => {
|
|
245
|
+
this.log("Connection timeout");
|
|
246
|
+
socket.removeAllListeners();
|
|
247
|
+
socket.close();
|
|
248
|
+
this.socket = void 0;
|
|
249
|
+
reject(new Error("WebSocket connection timeout"));
|
|
250
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
251
|
+
socket.addEventListener("open", () => {
|
|
252
|
+
clearTimeout(timeout);
|
|
253
|
+
this.sendSessionUpdate();
|
|
254
|
+
this.log("Connection opened");
|
|
255
|
+
resolve();
|
|
256
|
+
});
|
|
257
|
+
socket.addEventListener("error", (error) => {
|
|
258
|
+
clearTimeout(timeout);
|
|
259
|
+
this.log("WebSocket error:", error);
|
|
260
|
+
reject(new Error("WebSocket connection error"));
|
|
261
|
+
});
|
|
262
|
+
socket.addEventListener("close", ({ code, reason }) => {
|
|
263
|
+
clearTimeout(timeout);
|
|
264
|
+
this.socket?.removeAllListeners();
|
|
265
|
+
this.socket = void 0;
|
|
266
|
+
if (code !== 1e3) {
|
|
267
|
+
this.reconnect();
|
|
268
|
+
} else {
|
|
269
|
+
this.log("Connection closed", { code, reason });
|
|
270
|
+
}
|
|
271
|
+
});
|
|
272
|
+
socket.addEventListener("message", (event) => {
|
|
273
|
+
this.handleMessage(JSON.parse(event.data.toString()));
|
|
274
|
+
});
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
sendSessionUpdate() {
|
|
278
|
+
if (!this.socket) return;
|
|
279
|
+
this.socket.send(
|
|
280
|
+
JSON.stringify({
|
|
281
|
+
type: "session.update",
|
|
282
|
+
session: {
|
|
283
|
+
audio_format: {
|
|
284
|
+
encoding: this.options.encoding || DEFAULT_ENCODING,
|
|
285
|
+
sample_rate: SAMPLE_RATE
|
|
286
|
+
},
|
|
287
|
+
...this.options.targetStreamingDelayMs !== void 0 && {
|
|
288
|
+
target_streaming_delay_ms: this.options.targetStreamingDelayMs
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
})
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
sendAudioChunk(chunk) {
|
|
295
|
+
if (!this.socket) return;
|
|
296
|
+
this.socket.send(
|
|
297
|
+
JSON.stringify({
|
|
298
|
+
type: "input_audio.append",
|
|
299
|
+
audio: chunk.toString("base64")
|
|
300
|
+
})
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
flushAudio() {
|
|
304
|
+
if (!this.socket) return;
|
|
305
|
+
this.socket.send(JSON.stringify({ type: "input_audio.flush" }));
|
|
306
|
+
this.log("Flushed audio input");
|
|
307
|
+
}
|
|
308
|
+
handleMessage(message) {
|
|
309
|
+
switch (message.type) {
|
|
310
|
+
case "transcription.text.delta":
|
|
311
|
+
if (typeof message.text === "string") {
|
|
312
|
+
this.transcriptDelta += message.text;
|
|
313
|
+
this.log(`Received transcript delta: "${message.text}"`);
|
|
314
|
+
}
|
|
315
|
+
break;
|
|
316
|
+
case "transcription.done": {
|
|
317
|
+
const transcript = typeof message.text === "string" ? message.text : this.transcriptDelta;
|
|
318
|
+
this.log(`Received completed transcript: "${transcript}"`);
|
|
319
|
+
this.emit("Transcript", transcript.trim());
|
|
320
|
+
this.transcriptDelta = "";
|
|
321
|
+
this.audioChunksPending.length = 0;
|
|
322
|
+
if (this.transcriptionTimeout) {
|
|
323
|
+
clearTimeout(this.transcriptionTimeout);
|
|
324
|
+
this.transcriptionTimeout = void 0;
|
|
325
|
+
}
|
|
326
|
+
break;
|
|
327
|
+
}
|
|
328
|
+
case "error":
|
|
329
|
+
this.log("Error:", message.error ?? message);
|
|
330
|
+
break;
|
|
331
|
+
default:
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
reconnect() {
|
|
336
|
+
if (this.destroyed) return;
|
|
337
|
+
this.retryCount++;
|
|
338
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY2)) {
|
|
339
|
+
this.log("Max retries reached, giving up");
|
|
340
|
+
this.emit("Failed", this.audioChunksPending);
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
this.initPromise = new Promise((resolve) => {
|
|
344
|
+
this.log("Reconnecting...");
|
|
345
|
+
this.reconnectTimeout = setTimeout(() => {
|
|
346
|
+
this.reconnectTimeout = void 0;
|
|
347
|
+
this.initWS().then(() => {
|
|
348
|
+
this.retryCount = 0;
|
|
349
|
+
if (this.audioChunksPending.length > 0) {
|
|
350
|
+
this.log("Sending audio chunks again");
|
|
351
|
+
this.transcriptDelta = "";
|
|
352
|
+
this.audioChunksPending.forEach(
|
|
353
|
+
(chunk) => this.sendAudioChunk(chunk)
|
|
354
|
+
);
|
|
355
|
+
}
|
|
356
|
+
}).then(resolve).catch((error) => {
|
|
357
|
+
this.log("Reconnection error:", error);
|
|
358
|
+
this.reconnect();
|
|
359
|
+
});
|
|
360
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
};
|
|
159
364
|
export {
|
|
160
|
-
MistralAgent
|
|
365
|
+
MistralAgent,
|
|
366
|
+
MistralSTT
|
|
161
367
|
};
|
|
162
368
|
//# sourceMappingURL=index.mjs.map
|
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/MistralAgent.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport { Mistral } from '@mistralai/mistralai'\nimport {\n ChatCompletionStreamRequest,\n ChatCompletionStreamRequestMessages,\n} from '@mistralai/mistralai/models/components'\nimport { Writable } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport interface MistralAgentOptions extends AgentOptions {\n apiKey: string\n model?: string\n settings?: Omit<ChatCompletionStreamRequest, 'messages' | 'model'>\n maxRetry?: number\n retryDelay?: number\n maxSteps?: number\n}\n\nconst DEFAULT_MODEL = 'mistral-large-latest'\nconst DEFAULT_MAX_RETRY = 3\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_RETRY_DELAY = 500\n\nexport class MistralAgent extends Agent<MistralAgentOptions> {\n private mistral: Mistral\n\n constructor(options: MistralAgentOptions) {\n super(options)\n this.mistral = new Mistral({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: Writable,\n stepCount = 0,\n tryCount = 0\n ) {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[MistralAgent] Max steps reached')\n return\n }\n\n const answerCount = this.answerCount\n\n // Hack: Mistral needs a user message if there is only a system message\n if (this.conversation.length === 1) {\n this.conversation.push({\n role: 'user',\n content: '',\n })\n }\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n const result = await this.mistral.chat.stream({\n model: this.options.model || DEFAULT_MODEL,\n messages: this.buildMessages(),\n tools: this.buildTools(),\n ...this.options.settings,\n })\n let fullAnswer = ''\n let skipAnswer = false\n\n // Handle response events\n for await (const event of result) {\n if (answerCount !== this.answerCount) return\n const delta = event.data.choices[0]?.delta\n const chunk = delta?.content\n const toolCalls = delta?.toolCalls\n\n // Text chunk\n if (typeof chunk === 'string') {\n if (chunk === '') continue\n this.log(`Answer chunk: \"${chunk}\"`)\n fullAnswer += chunk\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = fullAnswer.indexOf(extractOptions.startTag)\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = fullAnswer.slice(0, startTagIndex).trimEnd()\n stream.write(messagePart)\n continue\n }\n } else {\n // Extracting, don't write to stream\n continue\n }\n }\n stream.write(chunk)\n } else if (toolCalls?.length) {\n for (const toolCall of toolCalls) {\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.id || '',\n toolName: toolCall.function.name,\n parameters: toolCall.function.arguments as string,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n }\n }\n\n if (answerCount !== this.answerCount) return\n\n if (fullAnswer) {\n // Add full answer to conversation\n const { message, metadata } = this.extract(fullAnswer)\n this.addAssistantMessage(message, metadata)\n } else if (skipAnswer) {\n // Hack: Mistral needs an assistant message after tool calls\n this.conversation.push({\n role: 'assistant',\n content: ' ',\n })\n } else {\n // Query again in case of tool call\n await this.generateAnswer(stream, stepCount + 1)\n }\n } catch (error: any) {\n console.error('[MistralAgent] Error:', error)\n if (tryCount < (this.options.maxRetry || DEFAULT_MAX_RETRY)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildMessages(): ChatCompletionStreamRequestMessages[] {\n return this.conversation.map(\n (message): ChatCompletionStreamRequestMessages => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n role: 'assistant',\n toolCalls: [\n {\n type: 'function',\n id: message.toolCallId,\n function: {\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n },\n },\n ],\n }\n case 'tool_result':\n return {\n role: 'tool',\n toolCallId: message.toolCallId,\n name: message.toolName,\n content: [\n {\n type: 'text',\n text: message.output,\n },\n ],\n }\n }\n }\n )\n }\n\n private buildTools(): ChatCompletionStreamRequest['tools'] {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n function: {\n name: tool.name,\n description: tool.description,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n },\n }))\n }\n\n cancel() {\n if (!this.answering) return\n this.log('Cancel')\n this.answering = false\n\n // Increment counter to avoid processing messages from previous calls\n this.answerCount++\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,SAAS,eAAe;AAMxB,OAAO,KAAK,oBAAoB;AAWhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAErB,IAAM,eAAN,cAA2B,MAA2B;AAAA,EAG3D,YAAY,SAA8B;AACxC,UAAM,OAAO;AACb,SAAK,UAAU,IAAI,QAAQ,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EACvD;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACX;AACA,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,kCAAkC;AAChD;AAAA,IACF;AAEA,UAAM,cAAc,KAAK;AAGzB,QAAI,KAAK,aAAa,WAAW,GAAG;AAClC,WAAK,aAAa,KAAK;AAAA,QACrB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,OAAO;AAAA,QAC5C,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,UAAU,KAAK,cAAc;AAAA,QAC7B,OAAO,KAAK,WAAW;AAAA,QACvB,GAAG,KAAK,QAAQ;AAAA,MAClB,CAAC;AACD,UAAI,aAAa;AACjB,UAAI,aAAa;AAGjB,uBAAiB,SAAS,QAAQ;AAChC,YAAI,gBAAgB,KAAK,YAAa;AACtC,cAAM,QAAQ,MAAM,KAAK,QAAQ,CAAC,GAAG;AACrC,cAAM,QAAQ,OAAO;AACrB,cAAM,YAAY,OAAO;AAGzB,YAAI,OAAO,UAAU,UAAU;AAC7B,cAAI,UAAU,GAAI;AAClB,eAAK,IAAI,kBAAkB,KAAK,GAAG;AACnC,wBAAc;AAGd,cAAI,gBAAgB;AAClB,gBAAI,CAAC,YAAY;AACf,oBAAM,gBAAgB,WAAW,QAAQ,eAAe,QAAQ;AAChE,kBAAI,kBAAkB,IAAI;AACxB,6BAAa;AACb,sBAAM,cAAc,WAAW,MAAM,GAAG,aAAa,EAAE,QAAQ;AAC/D,uBAAO,MAAM,WAAW;AACxB;AAAA,cACF;AAAA,YACF,OAAO;AAEL;AAAA,YACF;AAAA,UACF;AACA,iBAAO,MAAM,KAAK;AAAA,QACpB,WAAW,WAAW,QAAQ;AAC5B,qBAAW,YAAY,WAAW;AAChC,kBAAMA,UAAS,MAAM,KAAK,YAAY;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,SAAS,MAAM;AAAA,cAC3B,UAAU,SAAS,SAAS;AAAA,cAC5B,YAAY,SAAS,SAAS;AAAA,YAChC,CAAC;AACD,gBAAIA,QAAO,YAAY;AACrB,2BAAa;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,UAAI,gBAAgB,KAAK,YAAa;AAEtC,UAAI,YAAY;AAEd,cAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,UAAU;AACrD,aAAK,oBAAoB,SAAS,QAAQ;AAAA,MAC5C,WAAW,YAAY;AAErB,aAAK,aAAa,KAAK;AAAA,UACrB,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AAAA,MACH,OAAO;AAEL,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AAAA,IACF,SAAS,OAAY;AACnB,cAAQ,MAAM,yBAAyB,KAAK;AAC5C,UAAI,YAAY,KAAK,QAAQ,YAAY,oBAAoB;AAC3D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,gBAAuD;AAC7D,WAAO,KAAK,aAAa;AAAA,MACvB,CAAC,YAAiD;AAChD,gBAAQ,QAAQ,MAAM;AAAA,UACpB,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AACH,mBAAO;AAAA,cACL,MAAM,QAAQ;AAAA,cACd,SAAS,QAAQ;AAAA,YACnB;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,WAAW;AAAA,gBACT;AAAA,kBACE,MAAM;AAAA,kBACN,IAAI,QAAQ;AAAA,kBACZ,UAAU;AAAA,oBACR,MAAM,QAAQ;AAAA,oBACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,kBAC9C;AAAA,gBACF;AAAA,cACF;AAAA,YACF;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,YAAY,QAAQ;AAAA,cACpB,MAAM,QAAQ;AAAA,cACd,SAAS;AAAA,gBACP;AAAA,kBACE,MAAM;AAAA,kBACN,MAAM,QAAQ;AAAA,gBAChB;AAAA,cACF;AAAA,YACF;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAAmD;AAEzD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,UAAU;AAAA,QACR,MAAM,KAAK;AAAA,QACX,aAAa,KAAK;AAAA,QAClB,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,MACzD;AAAA,IACF,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,UAAW;AACrB,SAAK,IAAI,QAAQ;AACjB,SAAK,YAAY;AAGjB,SAAK;AAAA,EACP;AACF;","names":["result"]}
|
|
1
|
+
{"version":3,"sources":["../src/MistralAgent.ts","../src/MistralSTT.ts"],"sourcesContent":["import { Agent, AgentOptions } from '@micdrop/server'\nimport { Mistral } from '@mistralai/mistralai'\nimport {\n ChatCompletionStreamRequest,\n ChatCompletionStreamRequestMessages,\n} from '@mistralai/mistralai/models/components'\nimport { Writable } from 'stream'\nimport z, { toJSONSchema } from 'zod'\n\nexport interface MistralAgentOptions extends AgentOptions {\n apiKey: string\n model?: string\n settings?: Omit<ChatCompletionStreamRequest, 'messages' | 'model'>\n maxRetry?: number\n retryDelay?: number\n maxSteps?: number\n}\n\nconst DEFAULT_MODEL = 'mistral-large-latest'\nconst DEFAULT_MAX_RETRY = 3\nconst DEFAULT_MAX_STEPS = 5\nconst DEFAULT_RETRY_DELAY = 500\n\nexport class MistralAgent extends Agent<MistralAgentOptions> {\n private mistral: Mistral\n\n constructor(options: MistralAgentOptions) {\n super(options)\n this.mistral = new Mistral({ apiKey: options.apiKey })\n }\n\n protected async generateAnswer(\n stream: Writable,\n stepCount = 0,\n tryCount = 0\n ) {\n if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {\n console.error('[MistralAgent] Max steps reached')\n return\n }\n\n const answerCount = this.answerCount\n\n // Hack: Mistral needs a user message if there is only a system message\n if (this.conversation.length === 1) {\n this.conversation.push({\n role: 'user',\n content: '',\n })\n }\n\n // Prepare extracting\n let extracting = false\n const extractOptions = this.getExtractOptions()\n\n try {\n const result = await this.mistral.chat.stream({\n model: this.options.model || DEFAULT_MODEL,\n messages: this.buildMessages(),\n tools: this.buildTools(),\n ...this.options.settings,\n })\n let fullAnswer = ''\n let skipAnswer = false\n\n // Handle response events\n for await (const event of result) {\n if (answerCount !== this.answerCount) return\n const delta = event.data.choices[0]?.delta\n const chunk = delta?.content\n const toolCalls = delta?.toolCalls\n\n // Text chunk\n if (typeof chunk === 'string') {\n if (chunk === '') continue\n this.log(`Answer chunk: \"${chunk}\"`)\n fullAnswer += chunk\n\n // Extracting value?\n if (extractOptions) {\n if (!extracting) {\n const startTagIndex = fullAnswer.indexOf(extractOptions.startTag)\n if (startTagIndex !== -1) {\n extracting = true\n const messagePart = fullAnswer.slice(0, startTagIndex).trimEnd()\n stream.write(messagePart)\n continue\n }\n } else {\n // Extracting, don't write to stream\n continue\n }\n }\n stream.write(chunk)\n } else if (toolCalls?.length) {\n for (const toolCall of toolCalls) {\n const result = await this.executeTool({\n role: 'tool_call',\n toolCallId: toolCall.id || '',\n toolName: toolCall.function.name,\n parameters: toolCall.function.arguments as string,\n })\n if (result.skipAnswer) {\n skipAnswer = true\n }\n }\n }\n }\n\n if (answerCount !== this.answerCount) return\n\n if (fullAnswer) {\n // Add full answer to conversation\n const { message, metadata } = this.extract(fullAnswer)\n this.addAssistantMessage(message, metadata)\n } else if (skipAnswer) {\n // Hack: Mistral needs an assistant message after tool calls\n this.conversation.push({\n role: 'assistant',\n content: ' ',\n })\n } else {\n // Query again in case of tool call\n await this.generateAnswer(stream, stepCount + 1)\n }\n } catch (error: any) {\n console.error('[MistralAgent] Error:', error)\n if (tryCount < (this.options.maxRetry || DEFAULT_MAX_RETRY)) {\n await new Promise((resolve) =>\n setTimeout(resolve, this.options.retryDelay || DEFAULT_RETRY_DELAY)\n )\n await this.generateAnswer(stream, stepCount, tryCount + 1)\n } else {\n this.log('Max retries reached, giving up')\n this.emit('Failed')\n }\n }\n }\n\n private buildMessages(): ChatCompletionStreamRequestMessages[] {\n return this.conversation.map(\n (message): ChatCompletionStreamRequestMessages => {\n switch (message.role) {\n case 'user':\n case 'assistant':\n case 'system':\n return {\n role: message.role,\n content: message.content,\n }\n case 'tool_call':\n return {\n role: 'assistant',\n toolCalls: [\n {\n type: 'function',\n id: message.toolCallId,\n function: {\n name: message.toolName,\n arguments: JSON.stringify(message.parameters),\n },\n },\n ],\n }\n case 'tool_result':\n return {\n role: 'tool',\n toolCallId: message.toolCallId,\n name: message.toolName,\n content: [\n {\n type: 'text',\n text: message.output,\n },\n ],\n }\n }\n }\n )\n }\n\n private buildTools(): ChatCompletionStreamRequest['tools'] {\n // Disable tools if first message\n const enableTools = this.conversation.length > 1\n if (!enableTools) return undefined\n\n return this.tools.map((tool) => ({\n type: 'function',\n function: {\n name: tool.name,\n description: tool.description,\n parameters: toJSONSchema(tool.inputSchema || z.object()),\n },\n }))\n }\n\n cancel() {\n if (!this.answering) return\n this.log('Cancel')\n this.answering = false\n\n // Increment counter to avoid processing messages from previous calls\n this.answerCount++\n }\n}\n","import { STT } from '@micdrop/server'\nimport { Readable } from 'stream'\nimport WebSocket from 'ws'\n\n/**\n * Mistral Real-time STT (Voxtral)\n *\n * @see https://docs.mistral.ai/studio-api/audio/speech_to_text/realtime_transcription\n */\n\nexport type MistralAudioEncoding =\n | 'pcm_s16le'\n | 'pcm_s32le'\n | 'pcm_f16le'\n | 'pcm_f32le'\n | 'pcm_mulaw'\n | 'pcm_alaw'\n\nexport interface MistralSTTOptions {\n apiKey: string\n model?: string\n encoding?: MistralAudioEncoding\n targetStreamingDelayMs?: number\n connectionTimeout?: number\n transcriptionTimeout?: number\n retryDelay?: number\n maxRetry?: number\n}\n\nconst DEFAULT_MODEL = 'voxtral-mini-transcribe-realtime-2602'\nconst DEFAULT_ENCODING: MistralAudioEncoding = 'pcm_s16le'\nconst SAMPLE_RATE = 16000 // Rate of the incoming audio (Micdrop client)\nconst DEFAULT_CONNECTION_TIMEOUT = 5000\nconst DEFAULT_TRANSCRIPTION_TIMEOUT = 4000\nconst DEFAULT_RETRY_DELAY = 1000\nconst DEFAULT_MAX_RETRY = 3\n\nexport class MistralSTT extends STT {\n private socket?: WebSocket\n private initPromise: Promise<void>\n private reconnectTimeout?: NodeJS.Timeout\n private transcriptionTimeout?: NodeJS.Timeout\n private audioChunksPending: Buffer[] = [] // Store audio chunks to send them again if reconnecting\n private transcriptDelta = '' // Accumulated transcription.text.delta chunks\n private retryCount = 0\n private destroyed = false\n\n constructor(private options: MistralSTTOptions) {\n super()\n\n // Setup WebSocket connection\n this.initPromise = this.connect()\n }\n\n transcribe(audioStream: Readable) {\n let started = false\n\n // Read audio stream and send to Mistral\n audioStream.on('data', async (chunk: Buffer) => {\n // New utterance: a flush finalizes and resets the transcription context,\n // so start from a clean transcript for this stream, but only once it\n // carries something. Voice detection opens a stream on any noise and many\n // of them never carry a word, and clearing here rather than on entry\n // keeps those from wiping the transcript of the utterance before, whose\n // flush may still be on its way back.\n if (!started) {\n started = true\n this.transcriptDelta = ''\n }\n this.audioChunksPending.push(chunk)\n await this.initPromise\n this.sendAudioChunk(chunk)\n this.log(`Sent audio chunk (${chunk.byteLength} bytes)`)\n })\n\n // Handle stream end: flush buffered audio so the server finalizes the\n // transcription (transcription.done). Unlike input_audio.end, a flush keeps\n // the session alive, so the same connection is reused for the next utterance.\n audioStream.on('end', async () => {\n await this.initPromise\n if (this.audioChunksPending.length === 0) return\n this.flushAudio()\n\n // Timeout transcription if no transcript is received\n this.transcriptionTimeout = setTimeout(() => {\n this.transcriptionTimeout = undefined\n this.log('Transcription timeout')\n this.emit('Transcript', this.transcriptDelta.trim())\n this.transcriptDelta = ''\n this.audioChunksPending.length = 0\n }, this.options.transcriptionTimeout || DEFAULT_TRANSCRIPTION_TIMEOUT)\n })\n }\n\n destroy() {\n super.destroy()\n this.destroyed = true\n if (this.reconnectTimeout) {\n clearTimeout(this.reconnectTimeout)\n this.reconnectTimeout = undefined\n }\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n this.socket?.removeAllListeners()\n if (this.socket?.readyState === WebSocket.OPEN) {\n this.socket?.close(1000)\n }\n this.socket = undefined\n }\n\n private connect(): Promise<void> {\n return this.initWS().catch((error) => {\n console.error('[MistralSTT] Connection error:', error)\n this.reconnect()\n })\n }\n\n private getURL() {\n const model = this.options.model || DEFAULT_MODEL\n return `wss://api.mistral.ai/v1/audio/transcriptions/realtime?model=${encodeURIComponent(\n model\n )}`\n }\n\n private async initWS(): Promise<void> {\n return new Promise<void>((resolve, reject) => {\n const socket = new WebSocket(this.getURL(), {\n headers: {\n Authorization: `Bearer ${this.options.apiKey}`,\n },\n })\n this.socket = socket\n\n const timeout = setTimeout(() => {\n this.log('Connection timeout')\n socket.removeAllListeners()\n socket.close()\n this.socket = undefined\n reject(new Error('WebSocket connection timeout'))\n }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT)\n\n socket.addEventListener('open', () => {\n clearTimeout(timeout)\n this.sendSessionUpdate()\n this.log('Connection opened')\n resolve()\n })\n\n socket.addEventListener('error', (error) => {\n clearTimeout(timeout)\n this.log('WebSocket error:', error)\n reject(new Error('WebSocket connection error'))\n })\n\n socket.addEventListener('close', ({ code, reason }) => {\n clearTimeout(timeout)\n this.socket?.removeAllListeners()\n this.socket = undefined\n\n if (code !== 1000) {\n this.reconnect()\n } else {\n this.log('Connection closed', { code, reason })\n }\n })\n\n socket.addEventListener('message', (event) => {\n this.handleMessage(JSON.parse(event.data.toString()))\n })\n })\n }\n\n private sendSessionUpdate() {\n if (!this.socket) return\n\n // Configure the audio format. Must be sent before any audio is appended;\n // Mistral rejects format updates once audio has started.\n this.socket.send(\n JSON.stringify({\n type: 'session.update',\n session: {\n audio_format: {\n encoding: this.options.encoding || DEFAULT_ENCODING,\n sample_rate: SAMPLE_RATE,\n },\n ...(this.options.targetStreamingDelayMs !== undefined && {\n target_streaming_delay_ms: this.options.targetStreamingDelayMs,\n }),\n },\n })\n )\n }\n\n private sendAudioChunk(chunk: Buffer) {\n if (!this.socket) return\n this.socket.send(\n JSON.stringify({\n type: 'input_audio.append',\n audio: chunk.toString('base64'),\n })\n )\n }\n\n private flushAudio() {\n if (!this.socket) return\n // Flush buffered audio so the server processes the tail and emits a final\n // transcription.done for this utterance. We deliberately do NOT send\n // input_audio.end: that terminates the session (the server drops the\n // connection), whereas a flush finalizes the current utterance and keeps the\n // session ready for the next one.\n this.socket.send(JSON.stringify({ type: 'input_audio.flush' }))\n this.log('Flushed audio input')\n }\n\n private handleMessage(message: any) {\n switch (message.type) {\n case 'transcription.text.delta':\n if (typeof message.text === 'string') {\n this.transcriptDelta += message.text\n this.log(`Received transcript delta: \"${message.text}\"`)\n }\n break\n\n case 'transcription.done': {\n const transcript =\n typeof message.text === 'string' ? message.text : this.transcriptDelta\n this.log(`Received completed transcript: \"${transcript}\"`)\n this.emit('Transcript', transcript.trim())\n // Reset state and clear timeout. The session stays alive after a flush,\n // so the same connection is reused for the next utterance.\n this.transcriptDelta = ''\n this.audioChunksPending.length = 0\n if (this.transcriptionTimeout) {\n clearTimeout(this.transcriptionTimeout)\n this.transcriptionTimeout = undefined\n }\n break\n }\n\n case 'error':\n this.log('Error:', message.error ?? message)\n break\n\n default:\n break\n }\n }\n\n private reconnect() {\n if (this.destroyed) return\n this.retryCount++\n if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {\n this.log('Max retries reached, giving up')\n this.emit('Failed', this.audioChunksPending)\n return\n }\n\n this.initPromise = new Promise((resolve) => {\n this.log('Reconnecting...')\n this.reconnectTimeout = setTimeout(() => {\n this.reconnectTimeout = undefined\n this.initWS()\n .then(() => {\n this.retryCount = 0\n\n // Resend audio chunks if reconnecting during transcription. The new\n // session is fresh and replays the whole utterance, so reset the\n // accumulated deltas to avoid duplicating them.\n if (this.audioChunksPending.length > 0) {\n this.log('Sending audio chunks again')\n this.transcriptDelta = ''\n this.audioChunksPending.forEach((chunk) =>\n this.sendAudioChunk(chunk)\n )\n }\n })\n .then(resolve)\n .catch((error) => {\n this.log('Reconnection error:', error)\n this.reconnect()\n })\n }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)\n })\n }\n}\n"],"mappings":";AAAA,SAAS,aAA2B;AACpC,SAAS,eAAe;AAMxB,OAAO,KAAK,oBAAoB;AAWhC,IAAM,gBAAgB;AACtB,IAAM,oBAAoB;AAC1B,IAAM,oBAAoB;AAC1B,IAAM,sBAAsB;AAErB,IAAM,eAAN,cAA2B,MAA2B;AAAA,EAG3D,YAAY,SAA8B;AACxC,UAAM,OAAO;AACb,SAAK,UAAU,IAAI,QAAQ,EAAE,QAAQ,QAAQ,OAAO,CAAC;AAAA,EACvD;AAAA,EAEA,MAAgB,eACd,QACA,YAAY,GACZ,WAAW,GACX;AACA,QAAI,cAAc,KAAK,QAAQ,YAAY,oBAAoB;AAC7D,cAAQ,MAAM,kCAAkC;AAChD;AAAA,IACF;AAEA,UAAM,cAAc,KAAK;AAGzB,QAAI,KAAK,aAAa,WAAW,GAAG;AAClC,WAAK,aAAa,KAAK;AAAA,QACrB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAGA,QAAI,aAAa;AACjB,UAAM,iBAAiB,KAAK,kBAAkB;AAE9C,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,QAAQ,KAAK,OAAO;AAAA,QAC5C,OAAO,KAAK,QAAQ,SAAS;AAAA,QAC7B,UAAU,KAAK,cAAc;AAAA,QAC7B,OAAO,KAAK,WAAW;AAAA,QACvB,GAAG,KAAK,QAAQ;AAAA,MAClB,CAAC;AACD,UAAI,aAAa;AACjB,UAAI,aAAa;AAGjB,uBAAiB,SAAS,QAAQ;AAChC,YAAI,gBAAgB,KAAK,YAAa;AACtC,cAAM,QAAQ,MAAM,KAAK,QAAQ,CAAC,GAAG;AACrC,cAAM,QAAQ,OAAO;AACrB,cAAM,YAAY,OAAO;AAGzB,YAAI,OAAO,UAAU,UAAU;AAC7B,cAAI,UAAU,GAAI;AAClB,eAAK,IAAI,kBAAkB,KAAK,GAAG;AACnC,wBAAc;AAGd,cAAI,gBAAgB;AAClB,gBAAI,CAAC,YAAY;AACf,oBAAM,gBAAgB,WAAW,QAAQ,eAAe,QAAQ;AAChE,kBAAI,kBAAkB,IAAI;AACxB,6BAAa;AACb,sBAAM,cAAc,WAAW,MAAM,GAAG,aAAa,EAAE,QAAQ;AAC/D,uBAAO,MAAM,WAAW;AACxB;AAAA,cACF;AAAA,YACF,OAAO;AAEL;AAAA,YACF;AAAA,UACF;AACA,iBAAO,MAAM,KAAK;AAAA,QACpB,WAAW,WAAW,QAAQ;AAC5B,qBAAW,YAAY,WAAW;AAChC,kBAAMA,UAAS,MAAM,KAAK,YAAY;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,SAAS,MAAM;AAAA,cAC3B,UAAU,SAAS,SAAS;AAAA,cAC5B,YAAY,SAAS,SAAS;AAAA,YAChC,CAAC;AACD,gBAAIA,QAAO,YAAY;AACrB,2BAAa;AAAA,YACf;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,UAAI,gBAAgB,KAAK,YAAa;AAEtC,UAAI,YAAY;AAEd,cAAM,EAAE,SAAS,SAAS,IAAI,KAAK,QAAQ,UAAU;AACrD,aAAK,oBAAoB,SAAS,QAAQ;AAAA,MAC5C,WAAW,YAAY;AAErB,aAAK,aAAa,KAAK;AAAA,UACrB,MAAM;AAAA,UACN,SAAS;AAAA,QACX,CAAC;AAAA,MACH,OAAO;AAEL,cAAM,KAAK,eAAe,QAAQ,YAAY,CAAC;AAAA,MACjD;AAAA,IACF,SAAS,OAAY;AACnB,cAAQ,MAAM,yBAAyB,KAAK;AAC5C,UAAI,YAAY,KAAK,QAAQ,YAAY,oBAAoB;AAC3D,cAAM,IAAI;AAAA,UAAQ,CAAC,YACjB,WAAW,SAAS,KAAK,QAAQ,cAAc,mBAAmB;AAAA,QACpE;AACA,cAAM,KAAK,eAAe,QAAQ,WAAW,WAAW,CAAC;AAAA,MAC3D,OAAO;AACL,aAAK,IAAI,gCAAgC;AACzC,aAAK,KAAK,QAAQ;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,gBAAuD;AAC7D,WAAO,KAAK,aAAa;AAAA,MACvB,CAAC,YAAiD;AAChD,gBAAQ,QAAQ,MAAM;AAAA,UACpB,KAAK;AAAA,UACL,KAAK;AAAA,UACL,KAAK;AACH,mBAAO;AAAA,cACL,MAAM,QAAQ;AAAA,cACd,SAAS,QAAQ;AAAA,YACnB;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,WAAW;AAAA,gBACT;AAAA,kBACE,MAAM;AAAA,kBACN,IAAI,QAAQ;AAAA,kBACZ,UAAU;AAAA,oBACR,MAAM,QAAQ;AAAA,oBACd,WAAW,KAAK,UAAU,QAAQ,UAAU;AAAA,kBAC9C;AAAA,gBACF;AAAA,cACF;AAAA,YACF;AAAA,UACF,KAAK;AACH,mBAAO;AAAA,cACL,MAAM;AAAA,cACN,YAAY,QAAQ;AAAA,cACpB,MAAM,QAAQ;AAAA,cACd,SAAS;AAAA,gBACP;AAAA,kBACE,MAAM;AAAA,kBACN,MAAM,QAAQ;AAAA,gBAChB;AAAA,cACF;AAAA,YACF;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,aAAmD;AAEzD,UAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,QAAI,CAAC,YAAa,QAAO;AAEzB,WAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,MAC/B,MAAM;AAAA,MACN,UAAU;AAAA,QACR,MAAM,KAAK;AAAA,QACX,aAAa,KAAK;AAAA,QAClB,YAAY,aAAa,KAAK,eAAe,EAAE,OAAO,CAAC;AAAA,MACzD;AAAA,IACF,EAAE;AAAA,EACJ;AAAA,EAEA,SAAS;AACP,QAAI,CAAC,KAAK,UAAW;AACrB,SAAK,IAAI,QAAQ;AACjB,SAAK,YAAY;AAGjB,SAAK;AAAA,EACP;AACF;;;AC5MA,SAAS,WAAW;AAEpB,OAAO,eAAe;AA2BtB,IAAMC,iBAAgB;AACtB,IAAM,mBAAyC;AAC/C,IAAM,cAAc;AACpB,IAAM,6BAA6B;AACnC,IAAM,gCAAgC;AACtC,IAAMC,uBAAsB;AAC5B,IAAMC,qBAAoB;AAEnB,IAAM,aAAN,cAAyB,IAAI;AAAA,EAUlC,YAAoB,SAA4B;AAC9C,UAAM;AADY;AALpB,SAAQ,qBAA+B,CAAC;AACxC;AAAA,SAAQ,kBAAkB;AAC1B;AAAA,SAAQ,aAAa;AACrB,SAAQ,YAAY;AAMlB,SAAK,cAAc,KAAK,QAAQ;AAAA,EAClC;AAAA,EAEA,WAAW,aAAuB;AAChC,QAAI,UAAU;AAGd,gBAAY,GAAG,QAAQ,OAAO,UAAkB;AAO9C,UAAI,CAAC,SAAS;AACZ,kBAAU;AACV,aAAK,kBAAkB;AAAA,MACzB;AACA,WAAK,mBAAmB,KAAK,KAAK;AAClC,YAAM,KAAK;AACX,WAAK,eAAe,KAAK;AACzB,WAAK,IAAI,qBAAqB,MAAM,UAAU,SAAS;AAAA,IACzD,CAAC;AAKD,gBAAY,GAAG,OAAO,YAAY;AAChC,YAAM,KAAK;AACX,UAAI,KAAK,mBAAmB,WAAW,EAAG;AAC1C,WAAK,WAAW;AAGhB,WAAK,uBAAuB,WAAW,MAAM;AAC3C,aAAK,uBAAuB;AAC5B,aAAK,IAAI,uBAAuB;AAChC,aAAK,KAAK,cAAc,KAAK,gBAAgB,KAAK,CAAC;AACnD,aAAK,kBAAkB;AACvB,aAAK,mBAAmB,SAAS;AAAA,MACnC,GAAG,KAAK,QAAQ,wBAAwB,6BAA6B;AAAA,IACvE,CAAC;AAAA,EACH;AAAA,EAEA,UAAU;AACR,UAAM,QAAQ;AACd,SAAK,YAAY;AACjB,QAAI,KAAK,kBAAkB;AACzB,mBAAa,KAAK,gBAAgB;AAClC,WAAK,mBAAmB;AAAA,IAC1B;AACA,QAAI,KAAK,sBAAsB;AAC7B,mBAAa,KAAK,oBAAoB;AACtC,WAAK,uBAAuB;AAAA,IAC9B;AACA,SAAK,QAAQ,mBAAmB;AAChC,QAAI,KAAK,QAAQ,eAAe,UAAU,MAAM;AAC9C,WAAK,QAAQ,MAAM,GAAI;AAAA,IACzB;AACA,SAAK,SAAS;AAAA,EAChB;AAAA,EAEQ,UAAyB;AAC/B,WAAO,KAAK,OAAO,EAAE,MAAM,CAAC,UAAU;AACpC,cAAQ,MAAM,kCAAkC,KAAK;AACrD,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEQ,SAAS;AACf,UAAM,QAAQ,KAAK,QAAQ,SAASF;AACpC,WAAO,+DAA+D;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,SAAwB;AACpC,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,SAAS,IAAI,UAAU,KAAK,OAAO,GAAG;AAAA,QAC1C,SAAS;AAAA,UACP,eAAe,UAAU,KAAK,QAAQ,MAAM;AAAA,QAC9C;AAAA,MACF,CAAC;AACD,WAAK,SAAS;AAEd,YAAM,UAAU,WAAW,MAAM;AAC/B,aAAK,IAAI,oBAAoB;AAC7B,eAAO,mBAAmB;AAC1B,eAAO,MAAM;AACb,aAAK,SAAS;AACd,eAAO,IAAI,MAAM,8BAA8B,CAAC;AAAA,MAClD,GAAG,KAAK,QAAQ,qBAAqB,0BAA0B;AAE/D,aAAO,iBAAiB,QAAQ,MAAM;AACpC,qBAAa,OAAO;AACpB,aAAK,kBAAkB;AACvB,aAAK,IAAI,mBAAmB;AAC5B,gBAAQ;AAAA,MACV,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,UAAU;AAC1C,qBAAa,OAAO;AACpB,aAAK,IAAI,oBAAoB,KAAK;AAClC,eAAO,IAAI,MAAM,4BAA4B,CAAC;AAAA,MAChD,CAAC;AAED,aAAO,iBAAiB,SAAS,CAAC,EAAE,MAAM,OAAO,MAAM;AACrD,qBAAa,OAAO;AACpB,aAAK,QAAQ,mBAAmB;AAChC,aAAK,SAAS;AAEd,YAAI,SAAS,KAAM;AACjB,eAAK,UAAU;AAAA,QACjB,OAAO;AACL,eAAK,IAAI,qBAAqB,EAAE,MAAM,OAAO,CAAC;AAAA,QAChD;AAAA,MACF,CAAC;AAED,aAAO,iBAAiB,WAAW,CAAC,UAAU;AAC5C,aAAK,cAAc,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC;AAAA,MACtD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEQ,oBAAoB;AAC1B,QAAI,CAAC,KAAK,OAAQ;AAIlB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,SAAS;AAAA,UACP,cAAc;AAAA,YACZ,UAAU,KAAK,QAAQ,YAAY;AAAA,YACnC,aAAa;AAAA,UACf;AAAA,UACA,GAAI,KAAK,QAAQ,2BAA2B,UAAa;AAAA,YACvD,2BAA2B,KAAK,QAAQ;AAAA,UAC1C;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,eAAe,OAAe;AACpC,QAAI,CAAC,KAAK,OAAQ;AAClB,SAAK,OAAO;AAAA,MACV,KAAK,UAAU;AAAA,QACb,MAAM;AAAA,QACN,OAAO,MAAM,SAAS,QAAQ;AAAA,MAChC,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEQ,aAAa;AACnB,QAAI,CAAC,KAAK,OAAQ;AAMlB,SAAK,OAAO,KAAK,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC,CAAC;AAC9D,SAAK,IAAI,qBAAqB;AAAA,EAChC;AAAA,EAEQ,cAAc,SAAc;AAClC,YAAQ,QAAQ,MAAM;AAAA,MACpB,KAAK;AACH,YAAI,OAAO,QAAQ,SAAS,UAAU;AACpC,eAAK,mBAAmB,QAAQ;AAChC,eAAK,IAAI,+BAA+B,QAAQ,IAAI,GAAG;AAAA,QACzD;AACA;AAAA,MAEF,KAAK,sBAAsB;AACzB,cAAM,aACJ,OAAO,QAAQ,SAAS,WAAW,QAAQ,OAAO,KAAK;AACzD,aAAK,IAAI,mCAAmC,UAAU,GAAG;AACzD,aAAK,KAAK,cAAc,WAAW,KAAK,CAAC;AAGzC,aAAK,kBAAkB;AACvB,aAAK,mBAAmB,SAAS;AACjC,YAAI,KAAK,sBAAsB;AAC7B,uBAAa,KAAK,oBAAoB;AACtC,eAAK,uBAAuB;AAAA,QAC9B;AACA;AAAA,MACF;AAAA,MAEA,KAAK;AACH,aAAK,IAAI,UAAU,QAAQ,SAAS,OAAO;AAC3C;AAAA,MAEF;AACE;AAAA,IACJ;AAAA,EACF;AAAA,EAEQ,YAAY;AAClB,QAAI,KAAK,UAAW;AACpB,SAAK;AACL,QAAI,KAAK,cAAc,KAAK,QAAQ,YAAYE,qBAAoB;AAClE,WAAK,IAAI,gCAAgC;AACzC,WAAK,KAAK,UAAU,KAAK,kBAAkB;AAC3C;AAAA,IACF;AAEA,SAAK,cAAc,IAAI,QAAQ,CAAC,YAAY;AAC1C,WAAK,IAAI,iBAAiB;AAC1B,WAAK,mBAAmB,WAAW,MAAM;AACvC,aAAK,mBAAmB;AACxB,aAAK,OAAO,EACT,KAAK,MAAM;AACV,eAAK,aAAa;AAKlB,cAAI,KAAK,mBAAmB,SAAS,GAAG;AACtC,iBAAK,IAAI,4BAA4B;AACrC,iBAAK,kBAAkB;AACvB,iBAAK,mBAAmB;AAAA,cAAQ,CAAC,UAC/B,KAAK,eAAe,KAAK;AAAA,YAC3B;AAAA,UACF;AAAA,QACF,CAAC,EACA,KAAK,OAAO,EACZ,MAAM,CAAC,UAAU;AAChB,eAAK,IAAI,uBAAuB,KAAK;AACrC,eAAK,UAAU;AAAA,QACjB,CAAC;AAAA,MACL,GAAG,KAAK,QAAQ,cAAcD,oBAAmB;AAAA,IACnD,CAAC;AAAA,EACH;AACF;","names":["result","DEFAULT_MODEL","DEFAULT_RETRY_DELAY","DEFAULT_MAX_RETRY"]}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@micdrop/mistral",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"description": "Mistral implementation for @micdrop/server",
|
|
5
|
-
"author": "
|
|
5
|
+
"author": "Godefroy de Compreignac",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"main": "./dist/index.js",
|
|
8
8
|
"module": "./dist/index.mjs",
|
|
@@ -32,10 +32,12 @@
|
|
|
32
32
|
"voice",
|
|
33
33
|
"conversation",
|
|
34
34
|
"mistral",
|
|
35
|
+
"stt",
|
|
35
36
|
"tts"
|
|
36
37
|
],
|
|
37
38
|
"dependencies": {
|
|
38
|
-
"@mistralai/mistralai": "^1.14.1"
|
|
39
|
+
"@mistralai/mistralai": "^1.14.1",
|
|
40
|
+
"ws": "^8.19.0"
|
|
39
41
|
},
|
|
40
42
|
"peerDependencies": {
|
|
41
43
|
"@micdrop/server": "*",
|
|
@@ -43,6 +45,7 @@
|
|
|
43
45
|
},
|
|
44
46
|
"devDependencies": {
|
|
45
47
|
"@types/node": "^22.19.13",
|
|
48
|
+
"@types/ws": "^8.18.1",
|
|
46
49
|
"tsup": "^8.5.1",
|
|
47
50
|
"typescript": "^5.9.3"
|
|
48
51
|
}
|