@micdrop/openai 1.7.1 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # @micdrop/openai
2
2
 
3
- [Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/ai-integration/provided-integrations/openai)
3
+ [Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/providers)
4
4
 
5
5
  OpenAI implementation for [@micdrop/server](https://micdrop.dev/docs/server).
6
6
 
@@ -71,7 +71,7 @@ agent.answer().on('data', (chunk) => process.stdout.write(chunk))
71
71
  | `EndCall` | none | The agent decided that the call is over. |
72
72
  | `Failed` | none | The agent gave up generating an answer after its retries. |
73
73
 
74
- See the [Agent interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-agent) for the full contract.
74
+ See the [Agent interface](https://micdrop.dev/docs/agents/custom) for the full contract.
75
75
 
76
76
  ## OpenAI STT (Speech-to-Text)
77
77
 
@@ -121,7 +121,7 @@ stt.transcribe(createReadStream('speech.pcm'))
121
121
  | `Transcript` | `string` | Transcription of one utterance. The text is empty when nothing was recognized. |
122
122
  | `Failed` | `Buffer[]` | Transcription gave up after its retries, with the audio chunks left pending. |
123
123
 
124
- See the [STT interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-stt) for the full contract.
124
+ See the [STT interface](https://micdrop.dev/docs/speech-to-text/custom) for the full contract.
125
125
 
126
126
  ## OpenAI TTS (Text-to-Speech)
127
127
 
@@ -171,7 +171,7 @@ const tts = new OpenaiTTS({
171
171
  voice: 'alloy',
172
172
  })
173
173
 
174
- // Audio is raw PCM, 16 bits, 16 kHz, mono
174
+ // Audio is raw PCM, 16 bits, mono, at tts.sampleRate
175
175
  tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
176
176
  tts.on('Failed', (texts) => console.error('Failed:', texts))
177
177
 
@@ -180,12 +180,12 @@ tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
180
180
 
181
181
  ### Events
182
182
 
183
- | Event | Payload | Description |
184
- | -------- | ---------- | ------------------------------------------------------------------------ |
185
- | `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, 16 kHz, mono, ready to be played. |
186
- | `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
183
+ | Event | Payload | Description |
184
+ | -------- | ---------- | -------------------------------------------------------------------------------------- |
185
+ | `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, mono, at `tts.sampleRate` (24 kHz), ready to be played. |
186
+ | `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
187
187
 
188
- See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-tts) for the full contract.
188
+ See the [TTS interface](https://micdrop.dev/docs/text-to-speech/custom) for the full contract.
189
189
 
190
190
  > **Voices**: the Realtime API offers `alloy`, `ash`, `ballad`, `coral`, `echo`, `sage`, `shimmer`, `verse`, `marin` and `cedar`. `fable`, `nova` and `onyx` only exist on the speech endpoint.
191
191
 
@@ -193,7 +193,7 @@ See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrati
193
193
 
194
194
  ## OpenAI Realtime
195
195
 
196
- `OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the OpenAI Realtime API. It hears the user and answers with its own voice, in place of a speech to text, an agent and a text to speech.
196
+ `OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/realtime) running on the OpenAI Realtime API. It hears the user and answers with its own voice, in place of a speech to text, an agent and a text to speech.
197
197
 
198
198
  ```typescript
199
199
  import { OpenaiRealtime } from '@micdrop/openai'
@@ -205,6 +205,7 @@ const realtime = new OpenaiRealtime({
205
205
  voice: 'marin', // Default voice
206
206
  systemPrompt: 'You are a helpful assistant',
207
207
  language: 'en', // Optional, helps the transcription of the user
208
+ reasoningEffort: 'minimal', // Optional, the lowest answers soonest
208
209
  })
209
210
 
210
211
  new MicdropServer(socket, {
@@ -213,7 +214,7 @@ new MicdropServer(socket, {
213
214
  })
214
215
  ```
215
216
 
216
- It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits, 16 kHz, mono) and `PartialMessage` (the transcript of the answer so far).
217
+ It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits, 24 kHz, mono) and `PartialMessage` (the transcript of the answer so far).
217
218
 
218
219
  ## OpenAI Classifier
219
220
 
@@ -246,7 +247,7 @@ new MicdropServer(socket, {
246
247
 
247
248
  ## Documentation
248
249
 
249
- Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/openai) on the [website](https://micdrop.dev).
250
+ Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/providers) on the [website](https://micdrop.dev).
250
251
 
251
252
  ## License
252
253
 
package/dist/index.d.mts CHANGED
@@ -1,4 +1,4 @@
1
- import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
1
+ import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS, MicdropVoice } from '@micdrop/server';
2
2
  import OpenAI from 'openai';
3
3
  import { PassThrough, Readable } from 'stream';
4
4
 
@@ -15,13 +15,30 @@ type OpenaiOptions = {
15
15
  openai: OpenAI;
16
16
  };
17
17
  declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
18
+ /**
19
+ * Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
20
+ * else about them.
21
+ *
22
+ * The list is fetched once and shared by every call of the process for a
23
+ * day. `refresh` fetches it again.
24
+ */
25
+ static listModels(options: {
26
+ apiKey: string;
27
+ refresh?: boolean;
28
+ }): Promise<MicdropModel[]>;
18
29
  private openai;
19
- private abortController?;
20
30
  constructor(options: OpenaiAgentOptions);
21
- protected generateAnswer(stream: PassThrough, stepCount?: number, tryCount?: number): Promise<void>;
31
+ protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
32
+ /**
33
+ * Runs one request, writing its text to the stream and running its tools.
34
+ *
35
+ * It throws the error of the request when it failed before producing
36
+ * anything, which can then be retried, and a {@link StepInterruptedError}
37
+ * once something was written or done.
38
+ */
39
+ private generateStep;
22
40
  private buildInput;
23
41
  private buildTools;
24
- cancel(): void;
25
42
  }
26
43
 
27
44
  /**
@@ -142,20 +159,28 @@ interface OpenaiRealtimeOptions extends RealtimeOptions {
142
159
  voice?: string;
143
160
  language?: string;
144
161
  transcriptionModel?: string;
162
+ reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
145
163
  connectionTimeout?: number;
146
164
  retryDelay?: number;
147
165
  maxRetry?: number;
148
166
  }
149
167
  declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
150
- private socket?;
151
- private connected;
152
- private everConnected;
153
- private destroyed;
168
+ /**
169
+ * Lists the Realtime models, sorted from their names since OpenAI says nothing
170
+ * else about them.
171
+ *
172
+ * The list is fetched once and shared by every call of the process for a
173
+ * day. `refresh` fetches it again.
174
+ */
175
+ static listModels(options: {
176
+ apiKey: string;
177
+ refresh?: boolean;
178
+ }): Promise<MicdropModel[]>;
179
+ private socket;
180
+ private connection;
154
181
  private pending;
155
- private reconnectTimeout?;
156
- private retryCount;
182
+ private requestCount;
157
183
  private inputResampler;
158
- private outputResampler;
159
184
  private generation;
160
185
  private requested;
161
186
  private response?;
@@ -171,17 +196,33 @@ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
171
196
  protected clearTurn(): void;
172
197
  protected generate(): void;
173
198
  protected speakText(text: string): void;
199
+ get sampleRate(): number;
174
200
  protected cancelAnswer(): void;
175
201
  protected updateTools(): void;
176
202
  protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
203
+ /**
204
+ * Asks for a response, tagged so its `response.created` tells which request
205
+ * it answers. Matching them by order alone would shift every later response
206
+ * onto the wrong request as soon as one is refused.
207
+ */
177
208
  private createResponse;
209
+ /**
210
+ * Sends an event, or holds it until the connection opens.
211
+ * @returns False when it was dropped, the connection being given up on
212
+ */
178
213
  private send;
179
214
  private messageItem;
180
215
  private sendSessionUpdate;
181
- private connect;
182
- private initWS;
183
- private closeSocket;
184
- private reconnect;
216
+ /**
217
+ * Sets the session up, then sends what waited for it. A new connection
218
+ * starts from an empty conversation, so a reconnection sends it again first.
219
+ */
220
+ private onOpen;
221
+ /**
222
+ * Forgets what belonged to the session that is gone: its responses, and the
223
+ * answer in progress, which is lost with it.
224
+ */
225
+ private resetSession;
185
226
  private isIgnored;
186
227
  private handleMessage;
187
228
  private onResponseDone;
@@ -204,24 +245,58 @@ interface OpenaiSTTOptions {
204
245
  }
205
246
  declare class OpenaiSTT extends STT {
206
247
  private options;
207
- private socket?;
208
- private initPromise;
209
- private reconnectTimeout?;
210
- private connectionTimeout?;
211
- private retryCount;
212
- private destroyed;
213
- private transcriptionTimeout?;
214
- private audioChunksPending;
215
- private resampler;
248
+ /**
249
+ * Lists the transcription models, sorted from their names since OpenAI says nothing
250
+ * else about them.
251
+ *
252
+ * The list is fetched once and shared by every call of the process for a
253
+ * day. `refresh` fetches it again.
254
+ */
255
+ static listModels(options: {
256
+ apiKey: string;
257
+ refresh?: boolean;
258
+ }): Promise<MicdropModel[]>;
259
+ private socket;
260
+ private utteranceCount;
261
+ private utterances;
262
+ private unacknowledged;
216
263
  constructor(options: OpenaiSTTOptions);
264
+ /**
265
+ * Transcribes one utterance, and emits its transcript once, possibly empty
266
+ * when none comes in time.
267
+ *
268
+ * After the connection was given up on, the next utterance tries it again.
269
+ */
217
270
  transcribe(audioStream: Readable): void;
218
271
  destroy(): void;
219
272
  private sendSessionUpdate;
220
- private initWS;
273
+ /**
274
+ * Configures the session, then sends again what a previous connection did
275
+ * not get transcribed. The OpenAI buffer of a new session is empty, so each
276
+ * utterance is replayed through a fresh resampler, which reproduces the exact
277
+ * same stream, and committed again if it was.
278
+ */
279
+ private onOpen;
280
+ /** Every retry failed: the audio waiting for a transcript is handed back */
281
+ private onFailed;
221
282
  private sendAudioChunk;
222
- private handleMessage;
223
- private reconnect;
224
283
  private commitAudio;
284
+ /**
285
+ * Settles the transcript of an utterance, the only one it gets.
286
+ *
287
+ * The API transcribes each utterance on its own and may finish a short one
288
+ * before a long one spoken earlier, so transcripts go out in the order the
289
+ * utterances were spoken.
290
+ */
291
+ private finish;
292
+ private remove;
293
+ /**
294
+ * The utterance a transcript event is about. The API names the item, and
295
+ * the item of each utterance is known from the acknowledgement of its
296
+ * commit. A transcript of an utterance that timed out matches none.
297
+ */
298
+ private findUtterance;
299
+ private handleMessage;
225
300
  }
226
301
 
227
302
  /**
@@ -252,6 +327,27 @@ type OpenaiTTSOptions = OpenaiOptions & {
252
327
  };
253
328
  declare class OpenaiTTS extends TTS {
254
329
  private readonly options;
330
+ /**
331
+ * Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
332
+ * else about them.
333
+ *
334
+ * The list is fetched once and shared by every call of the process for a
335
+ * day. `refresh` fetches it again.
336
+ */
337
+ static listModels(options: {
338
+ apiKey: string;
339
+ refresh?: boolean;
340
+ }): Promise<MicdropModel[]>;
341
+ /**
342
+ * Lists the voices of the model: the Realtime API has no fable, nova and
343
+ * onyx, and adds marin and cedar. OpenAI has no listing, so the list is
344
+ * written here. Every voice reads every language, so `language` keeps them
345
+ * all.
346
+ */
347
+ static listVoices(options?: {
348
+ model?: string;
349
+ language?: string;
350
+ }): Promise<MicdropVoice[]>;
255
351
  private openai?;
256
352
  private realtime?;
257
353
  private counter;
@@ -263,6 +359,7 @@ declare class OpenaiTTS extends TTS {
263
359
  private cushionBytes;
264
360
  private processing;
265
361
  private syntheses;
362
+ get sampleRate(): number;
266
363
  constructor(options: OpenaiTTSOptions);
267
364
  speak(textStream: Readable): void;
268
365
  cancel(): void;
package/dist/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
1
+ import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, TTS, MicdropVoice } from '@micdrop/server';
2
2
  import OpenAI from 'openai';
3
3
  import { PassThrough, Readable } from 'stream';
4
4
 
@@ -15,13 +15,30 @@ type OpenaiOptions = {
15
15
  openai: OpenAI;
16
16
  };
17
17
  declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
18
+ /**
19
+ * Lists the chat models that can answer a call. `reasoning` marks the ones that reason, see `settings.reasoning`, sorted from their names since OpenAI says nothing
20
+ * else about them.
21
+ *
22
+ * The list is fetched once and shared by every call of the process for a
23
+ * day. `refresh` fetches it again.
24
+ */
25
+ static listModels(options: {
26
+ apiKey: string;
27
+ refresh?: boolean;
28
+ }): Promise<MicdropModel[]>;
18
29
  private openai;
19
- private abortController?;
20
30
  constructor(options: OpenaiAgentOptions);
21
- protected generateAnswer(stream: PassThrough, stepCount?: number, tryCount?: number): Promise<void>;
31
+ protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
32
+ /**
33
+ * Runs one request, writing its text to the stream and running its tools.
34
+ *
35
+ * It throws the error of the request when it failed before producing
36
+ * anything, which can then be retried, and a {@link StepInterruptedError}
37
+ * once something was written or done.
38
+ */
39
+ private generateStep;
22
40
  private buildInput;
23
41
  private buildTools;
24
- cancel(): void;
25
42
  }
26
43
 
27
44
  /**
@@ -142,20 +159,28 @@ interface OpenaiRealtimeOptions extends RealtimeOptions {
142
159
  voice?: string;
143
160
  language?: string;
144
161
  transcriptionModel?: string;
162
+ reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
145
163
  connectionTimeout?: number;
146
164
  retryDelay?: number;
147
165
  maxRetry?: number;
148
166
  }
149
167
  declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
150
- private socket?;
151
- private connected;
152
- private everConnected;
153
- private destroyed;
168
+ /**
169
+ * Lists the Realtime models, sorted from their names since OpenAI says nothing
170
+ * else about them.
171
+ *
172
+ * The list is fetched once and shared by every call of the process for a
173
+ * day. `refresh` fetches it again.
174
+ */
175
+ static listModels(options: {
176
+ apiKey: string;
177
+ refresh?: boolean;
178
+ }): Promise<MicdropModel[]>;
179
+ private socket;
180
+ private connection;
154
181
  private pending;
155
- private reconnectTimeout?;
156
- private retryCount;
182
+ private requestCount;
157
183
  private inputResampler;
158
- private outputResampler;
159
184
  private generation;
160
185
  private requested;
161
186
  private response?;
@@ -171,17 +196,33 @@ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
171
196
  protected clearTurn(): void;
172
197
  protected generate(): void;
173
198
  protected speakText(text: string): void;
199
+ get sampleRate(): number;
174
200
  protected cancelAnswer(): void;
175
201
  protected updateTools(): void;
176
202
  protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
203
+ /**
204
+ * Asks for a response, tagged so its `response.created` tells which request
205
+ * it answers. Matching them by order alone would shift every later response
206
+ * onto the wrong request as soon as one is refused.
207
+ */
177
208
  private createResponse;
209
+ /**
210
+ * Sends an event, or holds it until the connection opens.
211
+ * @returns False when it was dropped, the connection being given up on
212
+ */
178
213
  private send;
179
214
  private messageItem;
180
215
  private sendSessionUpdate;
181
- private connect;
182
- private initWS;
183
- private closeSocket;
184
- private reconnect;
216
+ /**
217
+ * Sets the session up, then sends what waited for it. A new connection
218
+ * starts from an empty conversation, so a reconnection sends it again first.
219
+ */
220
+ private onOpen;
221
+ /**
222
+ * Forgets what belonged to the session that is gone: its responses, and the
223
+ * answer in progress, which is lost with it.
224
+ */
225
+ private resetSession;
185
226
  private isIgnored;
186
227
  private handleMessage;
187
228
  private onResponseDone;
@@ -204,24 +245,58 @@ interface OpenaiSTTOptions {
204
245
  }
205
246
  declare class OpenaiSTT extends STT {
206
247
  private options;
207
- private socket?;
208
- private initPromise;
209
- private reconnectTimeout?;
210
- private connectionTimeout?;
211
- private retryCount;
212
- private destroyed;
213
- private transcriptionTimeout?;
214
- private audioChunksPending;
215
- private resampler;
248
+ /**
249
+ * Lists the transcription models, sorted from their names since OpenAI says nothing
250
+ * else about them.
251
+ *
252
+ * The list is fetched once and shared by every call of the process for a
253
+ * day. `refresh` fetches it again.
254
+ */
255
+ static listModels(options: {
256
+ apiKey: string;
257
+ refresh?: boolean;
258
+ }): Promise<MicdropModel[]>;
259
+ private socket;
260
+ private utteranceCount;
261
+ private utterances;
262
+ private unacknowledged;
216
263
  constructor(options: OpenaiSTTOptions);
264
+ /**
265
+ * Transcribes one utterance, and emits its transcript once, possibly empty
266
+ * when none comes in time.
267
+ *
268
+ * After the connection was given up on, the next utterance tries it again.
269
+ */
217
270
  transcribe(audioStream: Readable): void;
218
271
  destroy(): void;
219
272
  private sendSessionUpdate;
220
- private initWS;
273
+ /**
274
+ * Configures the session, then sends again what a previous connection did
275
+ * not get transcribed. The OpenAI buffer of a new session is empty, so each
276
+ * utterance is replayed through a fresh resampler, which reproduces the exact
277
+ * same stream, and committed again if it was.
278
+ */
279
+ private onOpen;
280
+ /** Every retry failed: the audio waiting for a transcript is handed back */
281
+ private onFailed;
221
282
  private sendAudioChunk;
222
- private handleMessage;
223
- private reconnect;
224
283
  private commitAudio;
284
+ /**
285
+ * Settles the transcript of an utterance, the only one it gets.
286
+ *
287
+ * The API transcribes each utterance on its own and may finish a short one
288
+ * before a long one spoken earlier, so transcripts go out in the order the
289
+ * utterances were spoken.
290
+ */
291
+ private finish;
292
+ private remove;
293
+ /**
294
+ * The utterance a transcript event is about. The API names the item, and
295
+ * the item of each utterance is known from the acknowledgement of its
296
+ * commit. A transcript of an utterance that timed out matches none.
297
+ */
298
+ private findUtterance;
299
+ private handleMessage;
225
300
  }
226
301
 
227
302
  /**
@@ -252,6 +327,27 @@ type OpenaiTTSOptions = OpenaiOptions & {
252
327
  };
253
328
  declare class OpenaiTTS extends TTS {
254
329
  private readonly options;
330
+ /**
331
+ * Lists the models that speak: the speech endpoint ones and the gpt-realtime ones, sorted from their names since OpenAI says nothing
332
+ * else about them.
333
+ *
334
+ * The list is fetched once and shared by every call of the process for a
335
+ * day. `refresh` fetches it again.
336
+ */
337
+ static listModels(options: {
338
+ apiKey: string;
339
+ refresh?: boolean;
340
+ }): Promise<MicdropModel[]>;
341
+ /**
342
+ * Lists the voices of the model: the Realtime API has no fable, nova and
343
+ * onyx, and adds marin and cedar. OpenAI has no listing, so the list is
344
+ * written here. Every voice reads every language, so `language` keeps them
345
+ * all.
346
+ */
347
+ static listVoices(options?: {
348
+ model?: string;
349
+ language?: string;
350
+ }): Promise<MicdropVoice[]>;
255
351
  private openai?;
256
352
  private realtime?;
257
353
  private counter;
@@ -263,6 +359,7 @@ declare class OpenaiTTS extends TTS {
263
359
  private cushionBytes;
264
360
  private processing;
265
361
  private syntheses;
362
+ get sampleRate(): number;
266
363
  constructor(options: OpenaiTTSOptions);
267
364
  speak(textStream: Readable): void;
268
365
  cancel(): void;