@micdrop/mistral 1.1.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -2
- package/dist/index.d.mts +58 -20
- package/dist/index.d.ts +58 -20
- package/dist/index.js +339 -217
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +338 -217
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -64,7 +64,7 @@ See the [Agent interface](https://micdrop.dev/docs/ai-integration/custom-integra
|
|
|
64
64
|
|
|
65
65
|
## Mistral STT (Speech-to-Text)
|
|
66
66
|
|
|
67
|
-
Real-time transcription with Voxtral.
|
|
67
|
+
Real-time transcription with Voxtral Mini Transcribe Realtime 3.
|
|
68
68
|
|
|
69
69
|
### Usage with MicdropServer
|
|
70
70
|
|
|
@@ -74,7 +74,7 @@ import { MicdropServer } from '@micdrop/server'
|
|
|
74
74
|
|
|
75
75
|
const stt = new MistralSTT({
|
|
76
76
|
apiKey: process.env.MISTRAL_API_KEY || '',
|
|
77
|
-
model: 'voxtral-mini-transcribe-realtime-
|
|
77
|
+
model: 'voxtral-mini-transcribe-realtime-3-0', // Default model
|
|
78
78
|
})
|
|
79
79
|
|
|
80
80
|
// Use with MicdropServer
|
|
@@ -110,6 +110,53 @@ stt.transcribe(createReadStream('speech.pcm'))
|
|
|
110
110
|
|
|
111
111
|
See the [STT interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-stt) for the full contract.
|
|
112
112
|
|
|
113
|
+
## Mistral TTS (Text-to-Speech)
|
|
114
|
+
|
|
115
|
+
Text to speech with Voxtral Speech 3. The answer is cut into sentences, and the audio of each one is sent as Mistral streams it.
|
|
116
|
+
|
|
117
|
+
### Usage with MicdropServer
|
|
118
|
+
|
|
119
|
+
```typescript
|
|
120
|
+
import { MistralTTS } from '@micdrop/mistral'
|
|
121
|
+
import { MicdropServer } from '@micdrop/server'
|
|
122
|
+
|
|
123
|
+
const tts = new MistralTTS({
|
|
124
|
+
apiKey: process.env.MISTRAL_API_KEY || '',
|
|
125
|
+
model: 'voxtral-mini-tts-3-0', // Default model
|
|
126
|
+
voiceId: 'voice_592', // Marie, French
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
// Use with MicdropServer
|
|
130
|
+
new MicdropServer(socket, {
|
|
131
|
+
tts,
|
|
132
|
+
// ... other options
|
|
133
|
+
})
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### Usage without MicdropServer
|
|
137
|
+
|
|
138
|
+
```typescript
|
|
139
|
+
import { MistralTTS } from '@micdrop/mistral'
|
|
140
|
+
import { Readable } from 'stream'
|
|
141
|
+
|
|
142
|
+
const tts = new MistralTTS({ apiKey: process.env.MISTRAL_API_KEY || '' })
|
|
143
|
+
|
|
144
|
+
// Audio is raw PCM, 16 bits, 16 kHz, mono
|
|
145
|
+
tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
|
|
146
|
+
tts.on('Failed', (texts) => console.error('Failed:', texts))
|
|
147
|
+
|
|
148
|
+
tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Events
|
|
152
|
+
|
|
153
|
+
| Event | Payload | Description |
|
|
154
|
+
| -------- | ---------- | --------------------------------------------------------------------- |
|
|
155
|
+
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, 16 kHz, mono, ready to be played. |
|
|
156
|
+
| `Failed` | `string[]` | Synthesis gave up after an error, with the text that stayed unspoken. |
|
|
157
|
+
|
|
158
|
+
See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-tts) for the full contract.
|
|
159
|
+
|
|
113
160
|
## Documentation
|
|
114
161
|
|
|
115
162
|
Read full [documentation of the Mistral integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/mistral) on the [website](https://micdrop.dev).
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, STT } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
2
2
|
import { ChatCompletionStreamRequest } from '@mistralai/mistralai/models/components';
|
|
3
3
|
import { Writable, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -13,16 +13,23 @@ interface MistralAgentOptions extends AgentOptions {
|
|
|
13
13
|
declare class MistralAgent extends Agent<MistralAgentOptions> {
|
|
14
14
|
private mistral;
|
|
15
15
|
constructor(options: MistralAgentOptions);
|
|
16
|
-
protected generateAnswer(stream: Writable,
|
|
16
|
+
protected generateAnswer(stream: Writable, signal: AbortSignal): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
19
|
+
*
|
|
20
|
+
* It throws a plain error when the request failed before producing
|
|
21
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
22
|
+
* once something was written or done.
|
|
23
|
+
*/
|
|
24
|
+
private generateStep;
|
|
17
25
|
private buildMessages;
|
|
18
26
|
private buildTools;
|
|
19
|
-
cancel(): void;
|
|
20
27
|
}
|
|
21
28
|
|
|
22
29
|
/**
|
|
23
30
|
* Mistral Real-time STT (Voxtral)
|
|
24
31
|
*
|
|
25
|
-
* @see https://docs.mistral.ai/studio
|
|
32
|
+
* @see https://docs.mistral.ai/studio/audio/speech-to-text/overview
|
|
26
33
|
*/
|
|
27
34
|
type MistralAudioEncoding = 'pcm_s16le' | 'pcm_s32le' | 'pcm_f16le' | 'pcm_f32le' | 'pcm_mulaw' | 'pcm_alaw';
|
|
28
35
|
interface MistralSTTOptions {
|
|
@@ -30,33 +37,64 @@ interface MistralSTTOptions {
|
|
|
30
37
|
model?: string;
|
|
31
38
|
encoding?: MistralAudioEncoding;
|
|
32
39
|
targetStreamingDelayMs?: number;
|
|
40
|
+
/** How long a connection attempt may take */
|
|
33
41
|
connectionTimeout?: number;
|
|
42
|
+
/** How long an utterance waits for its transcript once it ended */
|
|
34
43
|
transcriptionTimeout?: number;
|
|
44
|
+
/** Delay between two connection attempts */
|
|
35
45
|
retryDelay?: number;
|
|
46
|
+
/** How many attempts in a row may fail before `Failed` is emitted */
|
|
36
47
|
maxRetry?: number;
|
|
37
48
|
}
|
|
38
49
|
declare class MistralSTT extends STT {
|
|
39
50
|
private options;
|
|
40
|
-
private socket
|
|
41
|
-
private
|
|
42
|
-
private
|
|
43
|
-
private connectionTimeout?;
|
|
44
|
-
private transcriptionTimeout?;
|
|
45
|
-
private audioChunksPending;
|
|
46
|
-
private transcriptDelta;
|
|
47
|
-
private retryCount;
|
|
48
|
-
private destroyed;
|
|
51
|
+
private socket;
|
|
52
|
+
private utterances;
|
|
53
|
+
private staleFlushes;
|
|
49
54
|
constructor(options: MistralSTTOptions);
|
|
50
55
|
transcribe(audioStream: Readable): void;
|
|
51
56
|
destroy(): void;
|
|
52
|
-
private connect;
|
|
53
57
|
private getURL;
|
|
54
|
-
private
|
|
55
|
-
private
|
|
56
|
-
private
|
|
58
|
+
private getSessionUpdate;
|
|
59
|
+
private onOpen;
|
|
60
|
+
private onMessage;
|
|
61
|
+
private onFailed;
|
|
62
|
+
private endUtterance;
|
|
57
63
|
private flushAudio;
|
|
58
|
-
private
|
|
59
|
-
|
|
64
|
+
private get transcriptionTimeout();
|
|
65
|
+
/** Whether an answer to a timed out flush may still be on its way */
|
|
66
|
+
private hasStaleFlush;
|
|
67
|
+
private complete;
|
|
68
|
+
private removeUtterance;
|
|
69
|
+
private dropUtterance;
|
|
70
|
+
private sendAudioChunk;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Mistral text to speech (Voxtral Speech 3)
|
|
75
|
+
*
|
|
76
|
+
* @see https://docs.mistral.ai/studio/audio/text-to-speech/overview
|
|
77
|
+
* @see https://docs.mistral.ai/api/endpoint/audio/speech
|
|
78
|
+
*
|
|
79
|
+
* The speech endpoint reads a whole text at once, so the answer is cut into
|
|
80
|
+
* sentences, and the audio of each one is emitted as the server streams it.
|
|
81
|
+
* A sentence starts playing in about half a second and is generated several
|
|
82
|
+
* times faster than it is spoken, so the next one is ready before the current
|
|
83
|
+
* one ends.
|
|
84
|
+
*/
|
|
85
|
+
interface MistralTTSOptions {
|
|
86
|
+
apiKey: string;
|
|
87
|
+
model?: string;
|
|
88
|
+
voiceId?: string;
|
|
89
|
+
refAudio?: string;
|
|
90
|
+
settings?: Record<string, unknown>;
|
|
91
|
+
baseUrl?: string;
|
|
92
|
+
}
|
|
93
|
+
declare class MistralTTS extends SentenceTTS {
|
|
94
|
+
private readonly options;
|
|
95
|
+
constructor(options: MistralTTSOptions);
|
|
96
|
+
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
97
|
+
private request;
|
|
60
98
|
}
|
|
61
99
|
|
|
62
|
-
export { MistralAgent, type MistralAgentOptions, type MistralAudioEncoding, MistralSTT, type MistralSTTOptions };
|
|
100
|
+
export { MistralAgent, type MistralAgentOptions, type MistralAudioEncoding, MistralSTT, type MistralSTTOptions, MistralTTS, type MistralTTSOptions };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, STT } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
2
2
|
import { ChatCompletionStreamRequest } from '@mistralai/mistralai/models/components';
|
|
3
3
|
import { Writable, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -13,16 +13,23 @@ interface MistralAgentOptions extends AgentOptions {
|
|
|
13
13
|
declare class MistralAgent extends Agent<MistralAgentOptions> {
|
|
14
14
|
private mistral;
|
|
15
15
|
constructor(options: MistralAgentOptions);
|
|
16
|
-
protected generateAnswer(stream: Writable,
|
|
16
|
+
protected generateAnswer(stream: Writable, signal: AbortSignal): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
19
|
+
*
|
|
20
|
+
* It throws a plain error when the request failed before producing
|
|
21
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
22
|
+
* once something was written or done.
|
|
23
|
+
*/
|
|
24
|
+
private generateStep;
|
|
17
25
|
private buildMessages;
|
|
18
26
|
private buildTools;
|
|
19
|
-
cancel(): void;
|
|
20
27
|
}
|
|
21
28
|
|
|
22
29
|
/**
|
|
23
30
|
* Mistral Real-time STT (Voxtral)
|
|
24
31
|
*
|
|
25
|
-
* @see https://docs.mistral.ai/studio
|
|
32
|
+
* @see https://docs.mistral.ai/studio/audio/speech-to-text/overview
|
|
26
33
|
*/
|
|
27
34
|
type MistralAudioEncoding = 'pcm_s16le' | 'pcm_s32le' | 'pcm_f16le' | 'pcm_f32le' | 'pcm_mulaw' | 'pcm_alaw';
|
|
28
35
|
interface MistralSTTOptions {
|
|
@@ -30,33 +37,64 @@ interface MistralSTTOptions {
|
|
|
30
37
|
model?: string;
|
|
31
38
|
encoding?: MistralAudioEncoding;
|
|
32
39
|
targetStreamingDelayMs?: number;
|
|
40
|
+
/** How long a connection attempt may take */
|
|
33
41
|
connectionTimeout?: number;
|
|
42
|
+
/** How long an utterance waits for its transcript once it ended */
|
|
34
43
|
transcriptionTimeout?: number;
|
|
44
|
+
/** Delay between two connection attempts */
|
|
35
45
|
retryDelay?: number;
|
|
46
|
+
/** How many attempts in a row may fail before `Failed` is emitted */
|
|
36
47
|
maxRetry?: number;
|
|
37
48
|
}
|
|
38
49
|
declare class MistralSTT extends STT {
|
|
39
50
|
private options;
|
|
40
|
-
private socket
|
|
41
|
-
private
|
|
42
|
-
private
|
|
43
|
-
private connectionTimeout?;
|
|
44
|
-
private transcriptionTimeout?;
|
|
45
|
-
private audioChunksPending;
|
|
46
|
-
private transcriptDelta;
|
|
47
|
-
private retryCount;
|
|
48
|
-
private destroyed;
|
|
51
|
+
private socket;
|
|
52
|
+
private utterances;
|
|
53
|
+
private staleFlushes;
|
|
49
54
|
constructor(options: MistralSTTOptions);
|
|
50
55
|
transcribe(audioStream: Readable): void;
|
|
51
56
|
destroy(): void;
|
|
52
|
-
private connect;
|
|
53
57
|
private getURL;
|
|
54
|
-
private
|
|
55
|
-
private
|
|
56
|
-
private
|
|
58
|
+
private getSessionUpdate;
|
|
59
|
+
private onOpen;
|
|
60
|
+
private onMessage;
|
|
61
|
+
private onFailed;
|
|
62
|
+
private endUtterance;
|
|
57
63
|
private flushAudio;
|
|
58
|
-
private
|
|
59
|
-
|
|
64
|
+
private get transcriptionTimeout();
|
|
65
|
+
/** Whether an answer to a timed out flush may still be on its way */
|
|
66
|
+
private hasStaleFlush;
|
|
67
|
+
private complete;
|
|
68
|
+
private removeUtterance;
|
|
69
|
+
private dropUtterance;
|
|
70
|
+
private sendAudioChunk;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Mistral text to speech (Voxtral Speech 3)
|
|
75
|
+
*
|
|
76
|
+
* @see https://docs.mistral.ai/studio/audio/text-to-speech/overview
|
|
77
|
+
* @see https://docs.mistral.ai/api/endpoint/audio/speech
|
|
78
|
+
*
|
|
79
|
+
* The speech endpoint reads a whole text at once, so the answer is cut into
|
|
80
|
+
* sentences, and the audio of each one is emitted as the server streams it.
|
|
81
|
+
* A sentence starts playing in about half a second and is generated several
|
|
82
|
+
* times faster than it is spoken, so the next one is ready before the current
|
|
83
|
+
* one ends.
|
|
84
|
+
*/
|
|
85
|
+
interface MistralTTSOptions {
|
|
86
|
+
apiKey: string;
|
|
87
|
+
model?: string;
|
|
88
|
+
voiceId?: string;
|
|
89
|
+
refAudio?: string;
|
|
90
|
+
settings?: Record<string, unknown>;
|
|
91
|
+
baseUrl?: string;
|
|
92
|
+
}
|
|
93
|
+
declare class MistralTTS extends SentenceTTS {
|
|
94
|
+
private readonly options;
|
|
95
|
+
constructor(options: MistralTTSOptions);
|
|
96
|
+
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
97
|
+
private request;
|
|
60
98
|
}
|
|
61
99
|
|
|
62
|
-
export { MistralAgent, type MistralAgentOptions, type MistralAudioEncoding, MistralSTT, type MistralSTTOptions };
|
|
100
|
+
export { MistralAgent, type MistralAgentOptions, type MistralAudioEncoding, MistralSTT, type MistralSTTOptions, MistralTTS, type MistralTTSOptions };
|