@micdrop/openai 1.7.1 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -12
- package/dist/index.d.mts +124 -27
- package/dist/index.d.ts +124 -27
- package/dist/index.js +579 -464
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +580 -456
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @micdrop/openai
|
|
2
2
|
|
|
3
|
-
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/
|
|
3
|
+
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/providers)
|
|
4
4
|
|
|
5
5
|
OpenAI implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
6
|
|
|
@@ -71,7 +71,7 @@ agent.answer().on('data', (chunk) => process.stdout.write(chunk))
|
|
|
71
71
|
| `EndCall` | none | The agent decided that the call is over. |
|
|
72
72
|
| `Failed` | none | The agent gave up generating an answer after its retries. |
|
|
73
73
|
|
|
74
|
-
See the [Agent interface](https://micdrop.dev/docs/
|
|
74
|
+
See the [Agent interface](https://micdrop.dev/docs/agents/custom) for the full contract.
|
|
75
75
|
|
|
76
76
|
## OpenAI STT (Speech-to-Text)
|
|
77
77
|
|
|
@@ -121,7 +121,7 @@ stt.transcribe(createReadStream('speech.pcm'))
|
|
|
121
121
|
| `Transcript` | `string` | Transcription of one utterance. The text is empty when nothing was recognized. |
|
|
122
122
|
| `Failed` | `Buffer[]` | Transcription gave up after its retries, with the audio chunks left pending. |
|
|
123
123
|
|
|
124
|
-
See the [STT interface](https://micdrop.dev/docs/
|
|
124
|
+
See the [STT interface](https://micdrop.dev/docs/speech-to-text/custom) for the full contract.
|
|
125
125
|
|
|
126
126
|
## OpenAI TTS (Text-to-Speech)
|
|
127
127
|
|
|
@@ -171,7 +171,7 @@ const tts = new OpenaiTTS({
|
|
|
171
171
|
voice: 'alloy',
|
|
172
172
|
})
|
|
173
173
|
|
|
174
|
-
// Audio is raw PCM, 16 bits,
|
|
174
|
+
// Audio is raw PCM, 16 bits, mono, at tts.sampleRate
|
|
175
175
|
tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
|
|
176
176
|
tts.on('Failed', (texts) => console.error('Failed:', texts))
|
|
177
177
|
|
|
@@ -180,12 +180,12 @@ tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
|
|
|
180
180
|
|
|
181
181
|
### Events
|
|
182
182
|
|
|
183
|
-
| Event | Payload | Description
|
|
184
|
-
| -------- | ---------- |
|
|
185
|
-
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits,
|
|
186
|
-
| `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken.
|
|
183
|
+
| Event | Payload | Description |
|
|
184
|
+
| -------- | ---------- | -------------------------------------------------------------------------------------- |
|
|
185
|
+
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, mono, at `tts.sampleRate` (24 kHz), ready to be played. |
|
|
186
|
+
| `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
|
|
187
187
|
|
|
188
|
-
See the [TTS interface](https://micdrop.dev/docs/
|
|
188
|
+
See the [TTS interface](https://micdrop.dev/docs/text-to-speech/custom) for the full contract.
|
|
189
189
|
|
|
190
190
|
> **Voices**: the Realtime API offers `alloy`, `ash`, `ballad`, `coral`, `echo`, `sage`, `shimmer`, `verse`, `marin` and `cedar`. `fable`, `nova` and `onyx` only exist on the speech endpoint.
|
|
191
191
|
|
|
@@ -193,7 +193,7 @@ See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrati
|
|
|
193
193
|
|
|
194
194
|
## OpenAI Realtime
|
|
195
195
|
|
|
196
|
-
`OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/
|
|
196
|
+
`OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/realtime) running on the OpenAI Realtime API. It hears the user and answers with its own voice, in place of a speech to text, an agent and a text to speech.
|
|
197
197
|
|
|
198
198
|
```typescript
|
|
199
199
|
import { OpenaiRealtime } from '@micdrop/openai'
|
|
@@ -205,6 +205,7 @@ const realtime = new OpenaiRealtime({
|
|
|
205
205
|
voice: 'marin', // Default voice
|
|
206
206
|
systemPrompt: 'You are a helpful assistant',
|
|
207
207
|
language: 'en', // Optional, helps the transcription of the user
|
|
208
|
+
reasoningEffort: 'minimal', // Optional, the lowest answers soonest
|
|
208
209
|
})
|
|
209
210
|
|
|
210
211
|
new MicdropServer(socket, {
|
|
@@ -213,7 +214,7 @@ new MicdropServer(socket, {
|
|
|
213
214
|
})
|
|
214
215
|
```
|
|
215
216
|
|
|
216
|
-
It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits,
|
|
217
|
+
It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits, 24 kHz, mono) and `PartialMessage` (the transcript of the answer so far).
|
|
217
218
|
|
|
218
219
|
## OpenAI Classifier
|
|
219
220
|
|
|
@@ -246,7 +247,7 @@ new MicdropServer(socket, {
|
|
|
246
247
|
|
|
247
248
|
## Documentation
|
|
248
249
|
|
|
249
|
-
Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/
|
|
250
|
+
Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/providers) on the [website](https://micdrop.dev).
|
|
250
251
|
|
|
251
252
|
## License
|
|
252
253
|
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS, MicdropVoice } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -15,13 +15,30 @@ type OpenaiOptions = {
|
|
|
15
15
|
openai: OpenAI;
|
|
16
16
|
};
|
|
17
17
|
declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
|
|
18
|
+
/**
|
|
19
|
+
* Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
|
|
20
|
+
* else about them.
|
|
21
|
+
*
|
|
22
|
+
* The list is fetched once and shared by every call of the process for a
|
|
23
|
+
* day. `refresh` fetches it again.
|
|
24
|
+
*/
|
|
25
|
+
static listModels(options: {
|
|
26
|
+
apiKey: string;
|
|
27
|
+
refresh?: boolean;
|
|
28
|
+
}): Promise<MicdropModel[]>;
|
|
18
29
|
private openai;
|
|
19
|
-
private abortController?;
|
|
20
30
|
constructor(options: OpenaiAgentOptions);
|
|
21
|
-
protected generateAnswer(stream: PassThrough,
|
|
31
|
+
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
34
|
+
*
|
|
35
|
+
* It throws the error of the request when it failed before producing
|
|
36
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
37
|
+
* once something was written or done.
|
|
38
|
+
*/
|
|
39
|
+
private generateStep;
|
|
22
40
|
private buildInput;
|
|
23
41
|
private buildTools;
|
|
24
|
-
cancel(): void;
|
|
25
42
|
}
|
|
26
43
|
|
|
27
44
|
/**
|
|
@@ -142,20 +159,28 @@ interface OpenaiRealtimeOptions extends RealtimeOptions {
|
|
|
142
159
|
voice?: string;
|
|
143
160
|
language?: string;
|
|
144
161
|
transcriptionModel?: string;
|
|
162
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
|
145
163
|
connectionTimeout?: number;
|
|
146
164
|
retryDelay?: number;
|
|
147
165
|
maxRetry?: number;
|
|
148
166
|
}
|
|
149
167
|
declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
168
|
+
/**
|
|
169
|
+
* Lists the Realtime models, sorted from their names since OpenAI says nothing
|
|
170
|
+
* else about them.
|
|
171
|
+
*
|
|
172
|
+
* The list is fetched once and shared by every call of the process for a
|
|
173
|
+
* day. `refresh` fetches it again.
|
|
174
|
+
*/
|
|
175
|
+
static listModels(options: {
|
|
176
|
+
apiKey: string;
|
|
177
|
+
refresh?: boolean;
|
|
178
|
+
}): Promise<MicdropModel[]>;
|
|
179
|
+
private socket;
|
|
180
|
+
private connection;
|
|
154
181
|
private pending;
|
|
155
|
-
private
|
|
156
|
-
private retryCount;
|
|
182
|
+
private requestCount;
|
|
157
183
|
private inputResampler;
|
|
158
|
-
private outputResampler;
|
|
159
184
|
private generation;
|
|
160
185
|
private requested;
|
|
161
186
|
private response?;
|
|
@@ -171,17 +196,33 @@ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
|
171
196
|
protected clearTurn(): void;
|
|
172
197
|
protected generate(): void;
|
|
173
198
|
protected speakText(text: string): void;
|
|
199
|
+
get sampleRate(): number;
|
|
174
200
|
protected cancelAnswer(): void;
|
|
175
201
|
protected updateTools(): void;
|
|
176
202
|
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
203
|
+
/**
|
|
204
|
+
* Asks for a response, tagged so its `response.created` tells which request
|
|
205
|
+
* it answers. Matching them by order alone would shift every later response
|
|
206
|
+
* onto the wrong request as soon as one is refused.
|
|
207
|
+
*/
|
|
177
208
|
private createResponse;
|
|
209
|
+
/**
|
|
210
|
+
* Sends an event, or holds it until the connection opens.
|
|
211
|
+
* @returns False when it was dropped, the connection being given up on
|
|
212
|
+
*/
|
|
178
213
|
private send;
|
|
179
214
|
private messageItem;
|
|
180
215
|
private sendSessionUpdate;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
216
|
+
/**
|
|
217
|
+
* Sets the session up, then sends what waited for it. A new connection
|
|
218
|
+
* starts from an empty conversation, so a reconnection sends it again first.
|
|
219
|
+
*/
|
|
220
|
+
private onOpen;
|
|
221
|
+
/**
|
|
222
|
+
* Forgets what belonged to the session that is gone: its responses, and the
|
|
223
|
+
* answer in progress, which is lost with it.
|
|
224
|
+
*/
|
|
225
|
+
private resetSession;
|
|
185
226
|
private isIgnored;
|
|
186
227
|
private handleMessage;
|
|
187
228
|
private onResponseDone;
|
|
@@ -204,24 +245,58 @@ interface OpenaiSTTOptions {
|
|
|
204
245
|
}
|
|
205
246
|
declare class OpenaiSTT extends STT {
|
|
206
247
|
private options;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
248
|
+
/**
|
|
249
|
+
* Lists the transcription models, sorted from their names since OpenAI says nothing
|
|
250
|
+
* else about them.
|
|
251
|
+
*
|
|
252
|
+
* The list is fetched once and shared by every call of the process for a
|
|
253
|
+
* day. `refresh` fetches it again.
|
|
254
|
+
*/
|
|
255
|
+
static listModels(options: {
|
|
256
|
+
apiKey: string;
|
|
257
|
+
refresh?: boolean;
|
|
258
|
+
}): Promise<MicdropModel[]>;
|
|
259
|
+
private socket;
|
|
260
|
+
private utteranceCount;
|
|
261
|
+
private utterances;
|
|
262
|
+
private unacknowledged;
|
|
216
263
|
constructor(options: OpenaiSTTOptions);
|
|
264
|
+
/**
|
|
265
|
+
* Transcribes one utterance, and emits its transcript once, possibly empty
|
|
266
|
+
* when none comes in time.
|
|
267
|
+
*
|
|
268
|
+
* After the connection was given up on, the next utterance tries it again.
|
|
269
|
+
*/
|
|
217
270
|
transcribe(audioStream: Readable): void;
|
|
218
271
|
destroy(): void;
|
|
219
272
|
private sendSessionUpdate;
|
|
220
|
-
|
|
273
|
+
/**
|
|
274
|
+
* Configures the session, then sends again what a previous connection did
|
|
275
|
+
* not get transcribed. The OpenAI buffer of a new session is empty, so each
|
|
276
|
+
* utterance is replayed through a fresh resampler, which reproduces the exact
|
|
277
|
+
* same stream, and committed again if it was.
|
|
278
|
+
*/
|
|
279
|
+
private onOpen;
|
|
280
|
+
/** Every retry failed: the audio waiting for a transcript is handed back */
|
|
281
|
+
private onFailed;
|
|
221
282
|
private sendAudioChunk;
|
|
222
|
-
private handleMessage;
|
|
223
|
-
private reconnect;
|
|
224
283
|
private commitAudio;
|
|
284
|
+
/**
|
|
285
|
+
* Settles the transcript of an utterance, the only one it gets.
|
|
286
|
+
*
|
|
287
|
+
* The API transcribes each utterance on its own and may finish a short one
|
|
288
|
+
* before a long one spoken earlier, so transcripts go out in the order the
|
|
289
|
+
* utterances were spoken.
|
|
290
|
+
*/
|
|
291
|
+
private finish;
|
|
292
|
+
private remove;
|
|
293
|
+
/**
|
|
294
|
+
* The utterance a transcript event is about. The API names the item, and
|
|
295
|
+
* the item of each utterance is known from the acknowledgement of its
|
|
296
|
+
* commit. A transcript of an utterance that timed out matches none.
|
|
297
|
+
*/
|
|
298
|
+
private findUtterance;
|
|
299
|
+
private handleMessage;
|
|
225
300
|
}
|
|
226
301
|
|
|
227
302
|
/**
|
|
@@ -252,6 +327,27 @@ type OpenaiTTSOptions = OpenaiOptions & {
|
|
|
252
327
|
};
|
|
253
328
|
declare class OpenaiTTS extends TTS {
|
|
254
329
|
private readonly options;
|
|
330
|
+
/**
|
|
331
|
+
* Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
|
|
332
|
+
* else about them.
|
|
333
|
+
*
|
|
334
|
+
* The list is fetched once and shared by every call of the process for a
|
|
335
|
+
* day. `refresh` fetches it again.
|
|
336
|
+
*/
|
|
337
|
+
static listModels(options: {
|
|
338
|
+
apiKey: string;
|
|
339
|
+
refresh?: boolean;
|
|
340
|
+
}): Promise<MicdropModel[]>;
|
|
341
|
+
/**
|
|
342
|
+
* Lists the voices of the model: the Realtime API has no fable, nova and
|
|
343
|
+
* onyx, and adds marin and cedar. OpenAI has no listing, so the list is
|
|
344
|
+
* written here. Every voice reads every language, so `language` keeps them
|
|
345
|
+
* all.
|
|
346
|
+
*/
|
|
347
|
+
static listVoices(options?: {
|
|
348
|
+
model?: string;
|
|
349
|
+
language?: string;
|
|
350
|
+
}): Promise<MicdropVoice[]>;
|
|
255
351
|
private openai?;
|
|
256
352
|
private realtime?;
|
|
257
353
|
private counter;
|
|
@@ -263,6 +359,7 @@ declare class OpenaiTTS extends TTS {
|
|
|
263
359
|
private cushionBytes;
|
|
264
360
|
private processing;
|
|
265
361
|
private syntheses;
|
|
362
|
+
get sampleRate(): number;
|
|
266
363
|
constructor(options: OpenaiTTSOptions);
|
|
267
364
|
speak(textStream: Readable): void;
|
|
268
365
|
cancel(): void;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS, MicdropVoice } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -15,13 +15,30 @@ type OpenaiOptions = {
|
|
|
15
15
|
openai: OpenAI;
|
|
16
16
|
};
|
|
17
17
|
declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
|
|
18
|
+
/**
|
|
19
|
+
* Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
|
|
20
|
+
* else about them.
|
|
21
|
+
*
|
|
22
|
+
* The list is fetched once and shared by every call of the process for a
|
|
23
|
+
* day. `refresh` fetches it again.
|
|
24
|
+
*/
|
|
25
|
+
static listModels(options: {
|
|
26
|
+
apiKey: string;
|
|
27
|
+
refresh?: boolean;
|
|
28
|
+
}): Promise<MicdropModel[]>;
|
|
18
29
|
private openai;
|
|
19
|
-
private abortController?;
|
|
20
30
|
constructor(options: OpenaiAgentOptions);
|
|
21
|
-
protected generateAnswer(stream: PassThrough,
|
|
31
|
+
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Runs one request, writing its text to the stream and running its tools.
|
|
34
|
+
*
|
|
35
|
+
* It throws the error of the request when it failed before producing
|
|
36
|
+
* anything, which can then be retried, and a {@link StepInterruptedError}
|
|
37
|
+
* once something was written or done.
|
|
38
|
+
*/
|
|
39
|
+
private generateStep;
|
|
22
40
|
private buildInput;
|
|
23
41
|
private buildTools;
|
|
24
|
-
cancel(): void;
|
|
25
42
|
}
|
|
26
43
|
|
|
27
44
|
/**
|
|
@@ -142,20 +159,28 @@ interface OpenaiRealtimeOptions extends RealtimeOptions {
|
|
|
142
159
|
voice?: string;
|
|
143
160
|
language?: string;
|
|
144
161
|
transcriptionModel?: string;
|
|
162
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
|
|
145
163
|
connectionTimeout?: number;
|
|
146
164
|
retryDelay?: number;
|
|
147
165
|
maxRetry?: number;
|
|
148
166
|
}
|
|
149
167
|
declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
168
|
+
/**
|
|
169
|
+
* Lists the Realtime models, sorted from their names since OpenAI says nothing
|
|
170
|
+
* else about them.
|
|
171
|
+
*
|
|
172
|
+
* The list is fetched once and shared by every call of the process for a
|
|
173
|
+
* day. `refresh` fetches it again.
|
|
174
|
+
*/
|
|
175
|
+
static listModels(options: {
|
|
176
|
+
apiKey: string;
|
|
177
|
+
refresh?: boolean;
|
|
178
|
+
}): Promise<MicdropModel[]>;
|
|
179
|
+
private socket;
|
|
180
|
+
private connection;
|
|
154
181
|
private pending;
|
|
155
|
-
private
|
|
156
|
-
private retryCount;
|
|
182
|
+
private requestCount;
|
|
157
183
|
private inputResampler;
|
|
158
|
-
private outputResampler;
|
|
159
184
|
private generation;
|
|
160
185
|
private requested;
|
|
161
186
|
private response?;
|
|
@@ -171,17 +196,33 @@ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
|
171
196
|
protected clearTurn(): void;
|
|
172
197
|
protected generate(): void;
|
|
173
198
|
protected speakText(text: string): void;
|
|
199
|
+
get sampleRate(): number;
|
|
174
200
|
protected cancelAnswer(): void;
|
|
175
201
|
protected updateTools(): void;
|
|
176
202
|
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
203
|
+
/**
|
|
204
|
+
* Asks for a response, tagged so its `response.created` tells which request
|
|
205
|
+
* it answers. Matching them by order alone would shift every later response
|
|
206
|
+
* onto the wrong request as soon as one is refused.
|
|
207
|
+
*/
|
|
177
208
|
private createResponse;
|
|
209
|
+
/**
|
|
210
|
+
* Sends an event, or holds it until the connection opens.
|
|
211
|
+
* @returns False when it was dropped, the connection being given up on
|
|
212
|
+
*/
|
|
178
213
|
private send;
|
|
179
214
|
private messageItem;
|
|
180
215
|
private sendSessionUpdate;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
216
|
+
/**
|
|
217
|
+
* Sets the session up, then sends what waited for it. A new connection
|
|
218
|
+
* starts from an empty conversation, so a reconnection sends it again first.
|
|
219
|
+
*/
|
|
220
|
+
private onOpen;
|
|
221
|
+
/**
|
|
222
|
+
* Forgets what belonged to the session that is gone: its responses, and the
|
|
223
|
+
* answer in progress, which is lost with it.
|
|
224
|
+
*/
|
|
225
|
+
private resetSession;
|
|
185
226
|
private isIgnored;
|
|
186
227
|
private handleMessage;
|
|
187
228
|
private onResponseDone;
|
|
@@ -204,24 +245,58 @@ interface OpenaiSTTOptions {
|
|
|
204
245
|
}
|
|
205
246
|
declare class OpenaiSTT extends STT {
|
|
206
247
|
private options;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
248
|
+
/**
|
|
249
|
+
* Lists the transcription models, sorted from their names since OpenAI says nothing
|
|
250
|
+
* else about them.
|
|
251
|
+
*
|
|
252
|
+
* The list is fetched once and shared by every call of the process for a
|
|
253
|
+
* day. `refresh` fetches it again.
|
|
254
|
+
*/
|
|
255
|
+
static listModels(options: {
|
|
256
|
+
apiKey: string;
|
|
257
|
+
refresh?: boolean;
|
|
258
|
+
}): Promise<MicdropModel[]>;
|
|
259
|
+
private socket;
|
|
260
|
+
private utteranceCount;
|
|
261
|
+
private utterances;
|
|
262
|
+
private unacknowledged;
|
|
216
263
|
constructor(options: OpenaiSTTOptions);
|
|
264
|
+
/**
|
|
265
|
+
* Transcribes one utterance, and emits its transcript once, possibly empty
|
|
266
|
+
* when none comes in time.
|
|
267
|
+
*
|
|
268
|
+
* After the connection was given up on, the next utterance tries it again.
|
|
269
|
+
*/
|
|
217
270
|
transcribe(audioStream: Readable): void;
|
|
218
271
|
destroy(): void;
|
|
219
272
|
private sendSessionUpdate;
|
|
220
|
-
|
|
273
|
+
/**
|
|
274
|
+
* Configures the session, then sends again what a previous connection did
|
|
275
|
+
* not get transcribed. The OpenAI buffer of a new session is empty, so each
|
|
276
|
+
* utterance is replayed through a fresh resampler, which reproduces the exact
|
|
277
|
+
* same stream, and committed again if it was.
|
|
278
|
+
*/
|
|
279
|
+
private onOpen;
|
|
280
|
+
/** Every retry failed: the audio waiting for a transcript is handed back */
|
|
281
|
+
private onFailed;
|
|
221
282
|
private sendAudioChunk;
|
|
222
|
-
private handleMessage;
|
|
223
|
-
private reconnect;
|
|
224
283
|
private commitAudio;
|
|
284
|
+
/**
|
|
285
|
+
* Settles the transcript of an utterance, the only one it gets.
|
|
286
|
+
*
|
|
287
|
+
* The API transcribes each utterance on its own and may finish a short one
|
|
288
|
+
* before a long one spoken earlier, so transcripts go out in the order the
|
|
289
|
+
* utterances were spoken.
|
|
290
|
+
*/
|
|
291
|
+
private finish;
|
|
292
|
+
private remove;
|
|
293
|
+
/**
|
|
294
|
+
* The utterance a transcript event is about. The API names the item, and
|
|
295
|
+
* the item of each utterance is known from the acknowledgement of its
|
|
296
|
+
* commit. A transcript of an utterance that timed out matches none.
|
|
297
|
+
*/
|
|
298
|
+
private findUtterance;
|
|
299
|
+
private handleMessage;
|
|
225
300
|
}
|
|
226
301
|
|
|
227
302
|
/**
|
|
@@ -252,6 +327,27 @@ type OpenaiTTSOptions = OpenaiOptions & {
|
|
|
252
327
|
};
|
|
253
328
|
declare class OpenaiTTS extends TTS {
|
|
254
329
|
private readonly options;
|
|
330
|
+
/**
|
|
331
|
+
* Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
|
|
332
|
+
* else about them.
|
|
333
|
+
*
|
|
334
|
+
* The list is fetched once and shared by every call of the process for a
|
|
335
|
+
* day. `refresh` fetches it again.
|
|
336
|
+
*/
|
|
337
|
+
static listModels(options: {
|
|
338
|
+
apiKey: string;
|
|
339
|
+
refresh?: boolean;
|
|
340
|
+
}): Promise<MicdropModel[]>;
|
|
341
|
+
/**
|
|
342
|
+
* Lists the voices of the model: the Realtime API has no fable, nova and
|
|
343
|
+
* onyx, and adds marin and cedar. OpenAI has no listing, so the list is
|
|
344
|
+
* written here. Every voice reads every language, so `language` keeps them
|
|
345
|
+
* all.
|
|
346
|
+
*/
|
|
347
|
+
static listVoices(options?: {
|
|
348
|
+
model?: string;
|
|
349
|
+
language?: string;
|
|
350
|
+
}): Promise<MicdropVoice[]>;
|
|
255
351
|
private openai?;
|
|
256
352
|
private realtime?;
|
|
257
353
|
private counter;
|
|
@@ -263,6 +359,7 @@ declare class OpenaiTTS extends TTS {
|
|
|
263
359
|
private cushionBytes;
|
|
264
360
|
private processing;
|
|
265
361
|
private syntheses;
|
|
362
|
+
get sampleRate(): number;
|
|
266
363
|
constructor(options: OpenaiTTSOptions);
|
|
267
364
|
speak(textStream: Readable): void;
|
|
268
365
|
cancel(): void;
|