@micdrop/openai 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -12,7 +12,7 @@ npm install @micdrop/openai
12
12
 
13
13
  ## OpenAI Agent
14
14
 
15
- ### Usage
15
+ ### Usage with MicdropServer
16
16
 
17
17
  ```typescript
18
18
  import { OpenaiAgent } from '@micdrop/openai'
@@ -42,9 +42,40 @@ new MicdropServer(socket, {
42
42
  })
43
43
  ```
44
44
 
45
+ ### Usage without MicdropServer
46
+
47
+ ```typescript
48
+ import { OpenaiAgent } from '@micdrop/openai'
49
+
50
+ const agent = new OpenaiAgent({
51
+ apiKey: process.env.OPENAI_API_KEY || '',
52
+ systemPrompt: 'You are a helpful assistant',
53
+ })
54
+
55
+ agent.on('Message', (message) => console.log('Message:', message))
56
+
57
+ agent.addUserMessage('Hello, what can you do?')
58
+
59
+ // The answer is a text stream, written as the model generates it
60
+ agent.answer().on('data', (chunk) => process.stdout.write(chunk))
61
+ ```
62
+
63
+ ### Events
64
+
65
+ | Event | Payload | Description |
66
+ | ----------------------- | ------------------------- | ----------------------------------------------------------------------- |
67
+ | `Message` | `MicdropConversationItem` | A message, a tool call or a tool result was added to the conversation. |
68
+ | `ToolCall` | `MicdropToolCall` | A tool declared with `emitOutput` ran, with its parameters and output. |
69
+ | `CancelLastUserMessage` | none | The last user message was dropped because it carried no intent. |
70
+ | `SkipAnswer` | none | The agent stays silent and waits for the user to finish their sentence. |
71
+ | `EndCall` | none | The agent decided that the call is over. |
72
+ | `Failed` | none | The agent gave up generating an answer after its retries. |
73
+
74
+ See the [Agent interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-agent) for the full contract.
75
+
45
76
  ## OpenAI STT (Speech-to-Text)
46
77
 
47
- ### Usage
78
+ ### Usage with MicdropServer
48
79
 
49
80
  ```typescript
50
81
  import { OpenaiSTT } from '@micdrop/openai'
@@ -63,9 +94,36 @@ new MicdropServer(socket, {
63
94
  })
64
95
  ```
65
96
 
97
+ ### Usage without MicdropServer
98
+
99
+ ```typescript
100
+ import { OpenaiSTT } from '@micdrop/openai'
101
+ import { createReadStream } from 'fs'
102
+
103
+ const stt = new OpenaiSTT({
104
+ apiKey: process.env.OPENAI_API_KEY || '',
105
+ language: 'en',
106
+ })
107
+
108
+ stt.on('Transcript', (text) => console.log('Transcript:', text))
109
+ stt.on('Failed', (chunks) => console.error('Failed:', chunks.length, 'chunks'))
110
+
111
+ // Audio is raw PCM, 16 bits, 16 kHz, mono
112
+ stt.transcribe(createReadStream('speech.pcm'))
113
+ ```
114
+
115
+ ### Events
116
+
117
+ | Event | Payload | Description |
118
+ | ------------ | ---------- | ------------------------------------------------------------------------------ |
119
+ | `Transcript` | `string` | Transcription of one utterance. The text is empty when nothing was recognized. |
120
+ | `Failed` | `Buffer[]` | Transcription gave up after its retries, with the audio chunks left pending. |
121
+
122
+ See the [STT interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-stt) for the full contract.
123
+
66
124
  ## OpenAI TTS (Text-to-Speech)
67
125
 
68
- ### Usage
126
+ ### Usage with MicdropServer
69
127
 
70
128
  ```typescript
71
129
  import { OpenaiTTS } from '@micdrop/openai'
@@ -90,8 +148,59 @@ new MicdropServer(socket, {
90
148
  })
91
149
  ```
92
150
 
151
+ ### Usage without MicdropServer
152
+
153
+ ```typescript
154
+ import { OpenaiTTS } from '@micdrop/openai'
155
+ import { Readable } from 'stream'
156
+
157
+ const tts = new OpenaiTTS({
158
+ apiKey: process.env.OPENAI_API_KEY || '',
159
+ voice: 'alloy',
160
+ })
161
+
162
+ // Audio is raw PCM, 16 bits, 16 kHz, mono
163
+ tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
164
+ tts.on('Failed', (texts) => console.error('Failed:', texts))
165
+
166
+ tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
167
+ ```
168
+
169
+ ### Events
170
+
171
+ | Event | Payload | Description |
172
+ | -------- | ---------- | ------------------------------------------------------------------------ |
173
+ | `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, 16 kHz, mono, ready to be played. |
174
+ | `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
175
+
176
+ See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-tts) for the full contract.
177
+
93
178
  > **Language**: OpenAI's speech API has no language parameter, the voice follows the language of the input text. To influence the spoken language or accent, use `instructions` (e.g. `'Speak in French'`) with the `gpt-4o-mini-tts` model.
94
179
 
180
+ ## OpenAI Realtime
181
+
182
+ `OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the OpenAI Realtime API. It hears the user and answers with its own voice, in place of a speech to text, an agent and a text to speech.
183
+
184
+ ```typescript
185
+ import { OpenaiRealtime } from '@micdrop/openai'
186
+ import { MicdropServer } from '@micdrop/server'
187
+
188
+ const realtime = new OpenaiRealtime({
189
+ apiKey: process.env.OPENAI_API_KEY || '',
190
+ model: 'gpt-realtime-2.1', // Default model
191
+ voice: 'marin', // Default voice
192
+ systemPrompt: 'You are a helpful assistant',
193
+ language: 'en', // Optional, helps the transcription of the user
194
+ })
195
+
196
+ new MicdropServer(socket, {
197
+ realtime,
198
+ generateFirstMessage: true,
199
+ })
200
+ ```
201
+
202
+ It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits, 16 kHz, mono) and `PartialMessage` (the transcript of the answer so far).
203
+
95
204
  ## Documentation
96
205
 
97
206
  Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/openai) on the [website](https://micdrop.dev).
package/dist/index.d.mts CHANGED
@@ -1,4 +1,4 @@
1
- import { Agent, AgentOptions, STT, TTS } from '@micdrop/server';
1
+ import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
2
2
  import OpenAI from 'openai';
3
3
  import { PassThrough, Readable } from 'stream';
4
4
 
@@ -24,6 +24,66 @@ declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
24
24
  cancel(): void;
25
25
  }
26
26
 
27
+ /**
28
+ * OpenAI Realtime API, a model hearing the user and answering with its voice
29
+ *
30
+ * @see https://developers.openai.com/api/docs/guides/realtime-conversations
31
+ *
32
+ * The Micdrop client already tells when the user speaks, so the server side
33
+ * turn detection is off: the audio of a turn is committed when the client
34
+ * closes it, and the answer is asked for right after.
35
+ */
36
+ interface OpenaiRealtimeOptions extends RealtimeOptions {
37
+ apiKey: string;
38
+ model?: string;
39
+ voice?: string;
40
+ language?: string;
41
+ transcriptionModel?: string;
42
+ connectionTimeout?: number;
43
+ retryDelay?: number;
44
+ maxRetry?: number;
45
+ }
46
+ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
47
+ private socket?;
48
+ private connected;
49
+ private everConnected;
50
+ private destroyed;
51
+ private pending;
52
+ private reconnectTimeout?;
53
+ private retryCount;
54
+ private inputResampler;
55
+ private outputResampler;
56
+ private generation;
57
+ private requested;
58
+ private response?;
59
+ private ignoredResponses;
60
+ private audioItemId?;
61
+ private audioStartedAt;
62
+ private audioBytes;
63
+ constructor(options: OpenaiRealtimeOptions);
64
+ destroy(): void;
65
+ protected openTurn(): void;
66
+ protected appendAudio(chunk: Buffer): void;
67
+ protected closeTurn(): void;
68
+ protected clearTurn(): void;
69
+ protected generate(): void;
70
+ protected speakText(text: string): void;
71
+ protected cancelAnswer(): void;
72
+ protected updateTools(): void;
73
+ protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
74
+ private createResponse;
75
+ private send;
76
+ private messageItem;
77
+ private sendSessionUpdate;
78
+ private connect;
79
+ private initWS;
80
+ private closeSocket;
81
+ private reconnect;
82
+ private isIgnored;
83
+ private handleMessage;
84
+ private onResponseDone;
85
+ }
86
+
27
87
  /**
28
88
  * OpenAI Real-time STT
29
89
  *
@@ -117,4 +177,4 @@ declare class OpenaiTTS extends TTS {
117
177
  private speakItem;
118
178
  }
119
179
 
120
- export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
180
+ export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiRealtime, type OpenaiRealtimeOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
package/dist/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { Agent, AgentOptions, STT, TTS } from '@micdrop/server';
1
+ import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
2
2
  import OpenAI from 'openai';
3
3
  import { PassThrough, Readable } from 'stream';
4
4
 
@@ -24,6 +24,66 @@ declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
24
24
  cancel(): void;
25
25
  }
26
26
 
27
+ /**
28
+ * OpenAI Realtime API, a model hearing the user and answering with its voice
29
+ *
30
+ * @see https://developers.openai.com/api/docs/guides/realtime-conversations
31
+ *
32
+ * The Micdrop client already tells when the user speaks, so the server side
33
+ * turn detection is off: the audio of a turn is committed when the client
34
+ * closes it, and the answer is asked for right after.
35
+ */
36
+ interface OpenaiRealtimeOptions extends RealtimeOptions {
37
+ apiKey: string;
38
+ model?: string;
39
+ voice?: string;
40
+ language?: string;
41
+ transcriptionModel?: string;
42
+ connectionTimeout?: number;
43
+ retryDelay?: number;
44
+ maxRetry?: number;
45
+ }
46
+ declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
47
+ private socket?;
48
+ private connected;
49
+ private everConnected;
50
+ private destroyed;
51
+ private pending;
52
+ private reconnectTimeout?;
53
+ private retryCount;
54
+ private inputResampler;
55
+ private outputResampler;
56
+ private generation;
57
+ private requested;
58
+ private response?;
59
+ private ignoredResponses;
60
+ private audioItemId?;
61
+ private audioStartedAt;
62
+ private audioBytes;
63
+ constructor(options: OpenaiRealtimeOptions);
64
+ destroy(): void;
65
+ protected openTurn(): void;
66
+ protected appendAudio(chunk: Buffer): void;
67
+ protected closeTurn(): void;
68
+ protected clearTurn(): void;
69
+ protected generate(): void;
70
+ protected speakText(text: string): void;
71
+ protected cancelAnswer(): void;
72
+ protected updateTools(): void;
73
+ protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
74
+ private createResponse;
75
+ private send;
76
+ private messageItem;
77
+ private sendSessionUpdate;
78
+ private connect;
79
+ private initWS;
80
+ private closeSocket;
81
+ private reconnect;
82
+ private isIgnored;
83
+ private handleMessage;
84
+ private onResponseDone;
85
+ }
86
+
27
87
  /**
28
88
  * OpenAI Real-time STT
29
89
  *
@@ -117,4 +177,4 @@ declare class OpenaiTTS extends TTS {
117
177
  private speakItem;
118
178
  }
119
179
 
120
- export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
180
+ export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiRealtime, type OpenaiRealtimeOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };