@micdrop/openai 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -12
- package/dist/index.d.mts +58 -2
- package/dist/index.d.ts +58 -2
- package/dist/index.js +142 -53
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +136 -39
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @micdrop/openai
|
|
2
2
|
|
|
3
|
-
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/
|
|
3
|
+
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/providers)
|
|
4
4
|
|
|
5
5
|
OpenAI implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
6
|
|
|
@@ -71,7 +71,7 @@ agent.answer().on('data', (chunk) => process.stdout.write(chunk))
|
|
|
71
71
|
| `EndCall` | none | The agent decided that the call is over. |
|
|
72
72
|
| `Failed` | none | The agent gave up generating an answer after its retries. |
|
|
73
73
|
|
|
74
|
-
See the [Agent interface](https://micdrop.dev/docs/
|
|
74
|
+
See the [Agent interface](https://micdrop.dev/docs/agents/custom) for the full contract.
|
|
75
75
|
|
|
76
76
|
## OpenAI STT (Speech-to-Text)
|
|
77
77
|
|
|
@@ -121,7 +121,7 @@ stt.transcribe(createReadStream('speech.pcm'))
|
|
|
121
121
|
| `Transcript` | `string` | Transcription of one utterance. The text is empty when nothing was recognized. |
|
|
122
122
|
| `Failed` | `Buffer[]` | Transcription gave up after its retries, with the audio chunks left pending. |
|
|
123
123
|
|
|
124
|
-
See the [STT interface](https://micdrop.dev/docs/
|
|
124
|
+
See the [STT interface](https://micdrop.dev/docs/speech-to-text/custom) for the full contract.
|
|
125
125
|
|
|
126
126
|
## OpenAI TTS (Text-to-Speech)
|
|
127
127
|
|
|
@@ -171,7 +171,7 @@ const tts = new OpenaiTTS({
|
|
|
171
171
|
voice: 'alloy',
|
|
172
172
|
})
|
|
173
173
|
|
|
174
|
-
// Audio is raw PCM, 16 bits,
|
|
174
|
+
// Audio is raw PCM, 16 bits, mono, at tts.sampleRate
|
|
175
175
|
tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
|
|
176
176
|
tts.on('Failed', (texts) => console.error('Failed:', texts))
|
|
177
177
|
|
|
@@ -180,12 +180,12 @@ tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
|
|
|
180
180
|
|
|
181
181
|
### Events
|
|
182
182
|
|
|
183
|
-
| Event | Payload | Description
|
|
184
|
-
| -------- | ---------- |
|
|
185
|
-
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits,
|
|
186
|
-
| `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken.
|
|
183
|
+
| Event | Payload | Description |
|
|
184
|
+
| -------- | ---------- | -------------------------------------------------------------------------------------- |
|
|
185
|
+
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, mono, at `tts.sampleRate` (24 kHz), ready to be played. |
|
|
186
|
+
| `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
|
|
187
187
|
|
|
188
|
-
See the [TTS interface](https://micdrop.dev/docs/
|
|
188
|
+
See the [TTS interface](https://micdrop.dev/docs/text-to-speech/custom) for the full contract.
|
|
189
189
|
|
|
190
190
|
> **Voices**: the Realtime API offers `alloy`, `ash`, `ballad`, `coral`, `echo`, `sage`, `shimmer`, `verse`, `marin` and `cedar`. `fable`, `nova` and `onyx` only exist on the speech endpoint.
|
|
191
191
|
|
|
@@ -193,7 +193,7 @@ See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrati
|
|
|
193
193
|
|
|
194
194
|
## OpenAI Realtime
|
|
195
195
|
|
|
196
|
-
`OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/
|
|
196
|
+
`OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/realtime) running on the OpenAI Realtime API. It hears the user and answers with its own voice, in place of a speech to text, an agent and a text to speech.
|
|
197
197
|
|
|
198
198
|
```typescript
|
|
199
199
|
import { OpenaiRealtime } from '@micdrop/openai'
|
|
@@ -205,6 +205,7 @@ const realtime = new OpenaiRealtime({
|
|
|
205
205
|
voice: 'marin', // Default voice
|
|
206
206
|
systemPrompt: 'You are a helpful assistant',
|
|
207
207
|
language: 'en', // Optional, helps the transcription of the user
|
|
208
|
+
reasoningEffort: 'minimal', // Optional, the lowest answers soonest
|
|
208
209
|
})
|
|
209
210
|
|
|
210
211
|
new MicdropServer(socket, {
|
|
@@ -213,7 +214,7 @@ new MicdropServer(socket, {
|
|
|
213
214
|
})
|
|
214
215
|
```
|
|
215
216
|
|
|
216
|
-
It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits,
|
|
217
|
+
It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits, 24 kHz, mono) and `PartialMessage` (the transcript of the answer so far).
|
|
217
218
|
|
|
218
219
|
## OpenAI Classifier
|
|
219
220
|
|
|
@@ -246,7 +247,7 @@ new MicdropServer(socket, {
|
|
|
246
247
|
|
|
247
248
|
## Documentation
|
|
248
249
|
|
|
249
|
-
Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/
|
|
250
|
+
Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/providers) on the [website](https://micdrop.dev).
|
|
250
251
|
|
|
251
252
|
## License
|
|
252
253
|
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS, MicdropVoice } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -15,6 +15,17 @@ type OpenaiOptions = {
|
|
|
15
15
|
openai: OpenAI;
|
|
16
16
|
};
|
|
17
17
|
declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
|
|
18
|
+
/**
|
|
19
|
+
* Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
|
|
20
|
+
* else about them.
|
|
21
|
+
*
|
|
22
|
+
* The list is fetched once and shared by every call of the process for a
|
|
23
|
+
* day. `refresh` fetches it again.
|
|
24
|
+
*/
|
|
25
|
+
static listModels(options: {
|
|
26
|
+
apiKey: string;
|
|
27
|
+
refresh?: boolean;
|
|
28
|
+
}): Promise<MicdropModel[]>;
|
|
18
29
|
private openai;
|
|
19
30
|
constructor(options: OpenaiAgentOptions);
|
|
20
31
|
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
@@ -148,17 +159,28 @@ interface OpenaiRealtimeOptions extends RealtimeOptions {
|
|
|
148
159
|
voice?: string;
|
|
149
160
|
language?: string;
|
|
150
161
|
transcriptionModel?: string;
|
|
162
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
|
151
163
|
connectionTimeout?: number;
|
|
152
164
|
retryDelay?: number;
|
|
153
165
|
maxRetry?: number;
|
|
154
166
|
}
|
|
155
167
|
declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
168
|
+
/**
|
|
169
|
+
* Lists the Realtime models, sorted from their names since OpenAI says nothing
|
|
170
|
+
* else about them.
|
|
171
|
+
*
|
|
172
|
+
* The list is fetched once and shared by every call of the process for a
|
|
173
|
+
* day. `refresh` fetches it again.
|
|
174
|
+
*/
|
|
175
|
+
static listModels(options: {
|
|
176
|
+
apiKey: string;
|
|
177
|
+
refresh?: boolean;
|
|
178
|
+
}): Promise<MicdropModel[]>;
|
|
156
179
|
private socket;
|
|
157
180
|
private connection;
|
|
158
181
|
private pending;
|
|
159
182
|
private requestCount;
|
|
160
183
|
private inputResampler;
|
|
161
|
-
private outputResampler;
|
|
162
184
|
private generation;
|
|
163
185
|
private requested;
|
|
164
186
|
private response?;
|
|
@@ -174,6 +196,7 @@ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
|
174
196
|
protected clearTurn(): void;
|
|
175
197
|
protected generate(): void;
|
|
176
198
|
protected speakText(text: string): void;
|
|
199
|
+
get sampleRate(): number;
|
|
177
200
|
protected cancelAnswer(): void;
|
|
178
201
|
protected updateTools(): void;
|
|
179
202
|
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
@@ -222,6 +245,17 @@ interface OpenaiSTTOptions {
|
|
|
222
245
|
}
|
|
223
246
|
declare class OpenaiSTT extends STT {
|
|
224
247
|
private options;
|
|
248
|
+
/**
|
|
249
|
+
* Lists the transcription models, sorted from their names since OpenAI says nothing
|
|
250
|
+
* else about them.
|
|
251
|
+
*
|
|
252
|
+
* The list is fetched once and shared by every call of the process for a
|
|
253
|
+
* day. `refresh` fetches it again.
|
|
254
|
+
*/
|
|
255
|
+
static listModels(options: {
|
|
256
|
+
apiKey: string;
|
|
257
|
+
refresh?: boolean;
|
|
258
|
+
}): Promise<MicdropModel[]>;
|
|
225
259
|
private socket;
|
|
226
260
|
private utteranceCount;
|
|
227
261
|
private utterances;
|
|
@@ -293,6 +327,27 @@ type OpenaiTTSOptions = OpenaiOptions & {
|
|
|
293
327
|
};
|
|
294
328
|
declare class OpenaiTTS extends TTS {
|
|
295
329
|
private readonly options;
|
|
330
|
+
/**
|
|
331
|
+
* Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
|
|
332
|
+
* else about them.
|
|
333
|
+
*
|
|
334
|
+
* The list is fetched once and shared by every call of the process for a
|
|
335
|
+
* day. `refresh` fetches it again.
|
|
336
|
+
*/
|
|
337
|
+
static listModels(options: {
|
|
338
|
+
apiKey: string;
|
|
339
|
+
refresh?: boolean;
|
|
340
|
+
}): Promise<MicdropModel[]>;
|
|
341
|
+
/**
|
|
342
|
+
* Lists the voices of the model: the Realtime API has no fable, nova and
|
|
343
|
+
* onyx, and adds marin and cedar. OpenAI has no listing, so the list is
|
|
344
|
+
* written here. Every voice reads every language, so `language` keeps them
|
|
345
|
+
* all.
|
|
346
|
+
*/
|
|
347
|
+
static listVoices(options?: {
|
|
348
|
+
model?: string;
|
|
349
|
+
language?: string;
|
|
350
|
+
}): Promise<MicdropVoice[]>;
|
|
296
351
|
private openai?;
|
|
297
352
|
private realtime?;
|
|
298
353
|
private counter;
|
|
@@ -304,6 +359,7 @@ declare class OpenaiTTS extends TTS {
|
|
|
304
359
|
private cushionBytes;
|
|
305
360
|
private processing;
|
|
306
361
|
private syntheses;
|
|
362
|
+
get sampleRate(): number;
|
|
307
363
|
constructor(options: OpenaiTTSOptions);
|
|
308
364
|
speak(textStream: Readable): void;
|
|
309
365
|
cancel(): void;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS, MicdropVoice } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -15,6 +15,17 @@ type OpenaiOptions = {
|
|
|
15
15
|
openai: OpenAI;
|
|
16
16
|
};
|
|
17
17
|
declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
|
|
18
|
+
/**
|
|
19
|
+
* Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
|
|
20
|
+
* else about them.
|
|
21
|
+
*
|
|
22
|
+
* The list is fetched once and shared by every call of the process for a
|
|
23
|
+
* day. `refresh` fetches it again.
|
|
24
|
+
*/
|
|
25
|
+
static listModels(options: {
|
|
26
|
+
apiKey: string;
|
|
27
|
+
refresh?: boolean;
|
|
28
|
+
}): Promise<MicdropModel[]>;
|
|
18
29
|
private openai;
|
|
19
30
|
constructor(options: OpenaiAgentOptions);
|
|
20
31
|
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
@@ -148,17 +159,28 @@ interface OpenaiRealtimeOptions extends RealtimeOptions {
|
|
|
148
159
|
voice?: string;
|
|
149
160
|
language?: string;
|
|
150
161
|
transcriptionModel?: string;
|
|
162
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
|
151
163
|
connectionTimeout?: number;
|
|
152
164
|
retryDelay?: number;
|
|
153
165
|
maxRetry?: number;
|
|
154
166
|
}
|
|
155
167
|
declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
168
|
+
/**
|
|
169
|
+
* Lists the Realtime models, sorted from their names since OpenAI says nothing
|
|
170
|
+
* else about them.
|
|
171
|
+
*
|
|
172
|
+
* The list is fetched once and shared by every call of the process for a
|
|
173
|
+
* day. `refresh` fetches it again.
|
|
174
|
+
*/
|
|
175
|
+
static listModels(options: {
|
|
176
|
+
apiKey: string;
|
|
177
|
+
refresh?: boolean;
|
|
178
|
+
}): Promise<MicdropModel[]>;
|
|
156
179
|
private socket;
|
|
157
180
|
private connection;
|
|
158
181
|
private pending;
|
|
159
182
|
private requestCount;
|
|
160
183
|
private inputResampler;
|
|
161
|
-
private outputResampler;
|
|
162
184
|
private generation;
|
|
163
185
|
private requested;
|
|
164
186
|
private response?;
|
|
@@ -174,6 +196,7 @@ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
|
174
196
|
protected clearTurn(): void;
|
|
175
197
|
protected generate(): void;
|
|
176
198
|
protected speakText(text: string): void;
|
|
199
|
+
get sampleRate(): number;
|
|
177
200
|
protected cancelAnswer(): void;
|
|
178
201
|
protected updateTools(): void;
|
|
179
202
|
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
@@ -222,6 +245,17 @@ interface OpenaiSTTOptions {
|
|
|
222
245
|
}
|
|
223
246
|
declare class OpenaiSTT extends STT {
|
|
224
247
|
private options;
|
|
248
|
+
/**
|
|
249
|
+
* Lists the transcription models, sorted from their names since OpenAI says nothing
|
|
250
|
+
* else about them.
|
|
251
|
+
*
|
|
252
|
+
* The list is fetched once and shared by every call of the process for a
|
|
253
|
+
* day. `refresh` fetches it again.
|
|
254
|
+
*/
|
|
255
|
+
static listModels(options: {
|
|
256
|
+
apiKey: string;
|
|
257
|
+
refresh?: boolean;
|
|
258
|
+
}): Promise<MicdropModel[]>;
|
|
225
259
|
private socket;
|
|
226
260
|
private utteranceCount;
|
|
227
261
|
private utterances;
|
|
@@ -293,6 +327,27 @@ type OpenaiTTSOptions = OpenaiOptions & {
|
|
|
293
327
|
};
|
|
294
328
|
declare class OpenaiTTS extends TTS {
|
|
295
329
|
private readonly options;
|
|
330
|
+
/**
|
|
331
|
+
* Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
|
|
332
|
+
* else about them.
|
|
333
|
+
*
|
|
334
|
+
* The list is fetched once and shared by every call of the process for a
|
|
335
|
+
* day. `refresh` fetches it again.
|
|
336
|
+
*/
|
|
337
|
+
static listModels(options: {
|
|
338
|
+
apiKey: string;
|
|
339
|
+
refresh?: boolean;
|
|
340
|
+
}): Promise<MicdropModel[]>;
|
|
341
|
+
/**
|
|
342
|
+
* Lists the voices of the model: the Realtime API has no fable, nova and
|
|
343
|
+
* onyx, and adds marin and cedar. OpenAI has no listing, so the list is
|
|
344
|
+
* written here. Every voice reads every language, so `language` keeps them
|
|
345
|
+
* all.
|
|
346
|
+
*/
|
|
347
|
+
static listVoices(options?: {
|
|
348
|
+
model?: string;
|
|
349
|
+
language?: string;
|
|
350
|
+
}): Promise<MicdropVoice[]>;
|
|
296
351
|
private openai?;
|
|
297
352
|
private realtime?;
|
|
298
353
|
private counter;
|
|
@@ -304,6 +359,7 @@ declare class OpenaiTTS extends TTS {
|
|
|
304
359
|
private cushionBytes;
|
|
305
360
|
private processing;
|
|
306
361
|
private syntheses;
|
|
362
|
+
get sampleRate(): number;
|
|
307
363
|
constructor(options: OpenaiTTSOptions);
|
|
308
364
|
speak(textStream: Readable): void;
|
|
309
365
|
cancel(): void;
|