@micdrop/openai 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -3
- package/dist/index.d.mts +63 -4
- package/dist/index.d.ts +63 -4
- package/dist/index.js +383 -88
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +384 -87
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ npm install @micdrop/openai
|
|
|
12
12
|
|
|
13
13
|
## OpenAI Agent
|
|
14
14
|
|
|
15
|
-
### Usage
|
|
15
|
+
### Usage with MicdropServer
|
|
16
16
|
|
|
17
17
|
```typescript
|
|
18
18
|
import { OpenaiAgent } from '@micdrop/openai'
|
|
@@ -42,9 +42,40 @@ new MicdropServer(socket, {
|
|
|
42
42
|
})
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
+
### Usage without MicdropServer
|
|
46
|
+
|
|
47
|
+
```typescript
|
|
48
|
+
import { OpenaiAgent } from '@micdrop/openai'
|
|
49
|
+
|
|
50
|
+
const agent = new OpenaiAgent({
|
|
51
|
+
apiKey: process.env.OPENAI_API_KEY || '',
|
|
52
|
+
systemPrompt: 'You are a helpful assistant',
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
agent.on('Message', (message) => console.log('Message:', message))
|
|
56
|
+
|
|
57
|
+
agent.addUserMessage('Hello, what can you do?')
|
|
58
|
+
|
|
59
|
+
// The answer is a text stream, written as the model generates it
|
|
60
|
+
agent.answer().on('data', (chunk) => process.stdout.write(chunk))
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Events
|
|
64
|
+
|
|
65
|
+
| Event | Payload | Description |
|
|
66
|
+
| ----------------------- | ------------------------- | ----------------------------------------------------------------------- |
|
|
67
|
+
| `Message` | `MicdropConversationItem` | A message, a tool call or a tool result was added to the conversation. |
|
|
68
|
+
| `ToolCall` | `MicdropToolCall` | A tool declared with `emitOutput` ran, with its parameters and output. |
|
|
69
|
+
| `CancelLastUserMessage` | none | The last user message was dropped because it carried no intent. |
|
|
70
|
+
| `SkipAnswer` | none | The agent stays silent and waits for the user to finish their sentence. |
|
|
71
|
+
| `EndCall` | none | The agent decided that the call is over. |
|
|
72
|
+
| `Failed` | none | The agent gave up generating an answer after its retries. |
|
|
73
|
+
|
|
74
|
+
See the [Agent interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-agent) for the full contract.
|
|
75
|
+
|
|
45
76
|
## OpenAI STT (Speech-to-Text)
|
|
46
77
|
|
|
47
|
-
### Usage
|
|
78
|
+
### Usage with MicdropServer
|
|
48
79
|
|
|
49
80
|
```typescript
|
|
50
81
|
import { OpenaiSTT } from '@micdrop/openai'
|
|
@@ -63,9 +94,36 @@ new MicdropServer(socket, {
|
|
|
63
94
|
})
|
|
64
95
|
```
|
|
65
96
|
|
|
97
|
+
### Usage without MicdropServer
|
|
98
|
+
|
|
99
|
+
```typescript
|
|
100
|
+
import { OpenaiSTT } from '@micdrop/openai'
|
|
101
|
+
import { createReadStream } from 'fs'
|
|
102
|
+
|
|
103
|
+
const stt = new OpenaiSTT({
|
|
104
|
+
apiKey: process.env.OPENAI_API_KEY || '',
|
|
105
|
+
language: 'en',
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
stt.on('Transcript', (text) => console.log('Transcript:', text))
|
|
109
|
+
stt.on('Failed', (chunks) => console.error('Failed:', chunks.length, 'chunks'))
|
|
110
|
+
|
|
111
|
+
// Audio is raw PCM, 16 bits, 16 kHz, mono
|
|
112
|
+
stt.transcribe(createReadStream('speech.pcm'))
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Events
|
|
116
|
+
|
|
117
|
+
| Event | Payload | Description |
|
|
118
|
+
| ------------ | ---------- | ------------------------------------------------------------------------------ |
|
|
119
|
+
| `Transcript` | `string` | Transcription of one utterance. The text is empty when nothing was recognized. |
|
|
120
|
+
| `Failed` | `Buffer[]` | Transcription gave up after its retries, with the audio chunks left pending. |
|
|
121
|
+
|
|
122
|
+
See the [STT interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-stt) for the full contract.
|
|
123
|
+
|
|
66
124
|
## OpenAI TTS (Text-to-Speech)
|
|
67
125
|
|
|
68
|
-
### Usage
|
|
126
|
+
### Usage with MicdropServer
|
|
69
127
|
|
|
70
128
|
```typescript
|
|
71
129
|
import { OpenaiTTS } from '@micdrop/openai'
|
|
@@ -90,8 +148,59 @@ new MicdropServer(socket, {
|
|
|
90
148
|
})
|
|
91
149
|
```
|
|
92
150
|
|
|
151
|
+
### Usage without MicdropServer
|
|
152
|
+
|
|
153
|
+
```typescript
|
|
154
|
+
import { OpenaiTTS } from '@micdrop/openai'
|
|
155
|
+
import { Readable } from 'stream'
|
|
156
|
+
|
|
157
|
+
const tts = new OpenaiTTS({
|
|
158
|
+
apiKey: process.env.OPENAI_API_KEY || '',
|
|
159
|
+
voice: 'alloy',
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
// Audio is raw PCM, 16 bits, 16 kHz, mono
|
|
163
|
+
tts.on('Audio', (chunk) => console.log('Audio:', chunk.length, 'bytes'))
|
|
164
|
+
tts.on('Failed', (texts) => console.error('Failed:', texts))
|
|
165
|
+
|
|
166
|
+
tts.speak(Readable.from(['Hello! ', 'What can I do for you?']))
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Events
|
|
170
|
+
|
|
171
|
+
| Event | Payload | Description |
|
|
172
|
+
| -------- | ---------- | ------------------------------------------------------------------------ |
|
|
173
|
+
| `Audio` | `Buffer` | A chunk of audio, PCM 16 bits, 16 kHz, mono, ready to be played. |
|
|
174
|
+
| `Failed` | `string[]` | Synthesis gave up after its retries, with the text that stayed unspoken. |
|
|
175
|
+
|
|
176
|
+
See the [TTS interface](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-tts) for the full contract.
|
|
177
|
+
|
|
93
178
|
> **Language**: OpenAI's speech API has no language parameter, the voice follows the language of the input text. To influence the spoken language or accent, use `instructions` (e.g. `'Speak in French'`) with the `gpt-4o-mini-tts` model.
|
|
94
179
|
|
|
180
|
+
## OpenAI Realtime
|
|
181
|
+
|
|
182
|
+
`OpenaiRealtime` is a [realtime model](https://micdrop.dev/docs/server/realtime) running on the OpenAI Realtime API. It hears the user and answers with its own voice, in place of a speech to text, an agent and a text to speech.
|
|
183
|
+
|
|
184
|
+
```typescript
|
|
185
|
+
import { OpenaiRealtime } from '@micdrop/openai'
|
|
186
|
+
import { MicdropServer } from '@micdrop/server'
|
|
187
|
+
|
|
188
|
+
const realtime = new OpenaiRealtime({
|
|
189
|
+
apiKey: process.env.OPENAI_API_KEY || '',
|
|
190
|
+
model: 'gpt-realtime-2.1', // Default model
|
|
191
|
+
voice: 'marin', // Default voice
|
|
192
|
+
systemPrompt: 'You are a helpful assistant',
|
|
193
|
+
language: 'en', // Optional, helps the transcription of the user
|
|
194
|
+
})
|
|
195
|
+
|
|
196
|
+
new MicdropServer(socket, {
|
|
197
|
+
realtime,
|
|
198
|
+
generateFirstMessage: true,
|
|
199
|
+
})
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
It emits the events of an agent, plus `Audio` (the voice of the model, PCM 16 bits, 16 kHz, mono) and `PartialMessage` (the transcript of the answer so far).
|
|
203
|
+
|
|
95
204
|
## Documentation
|
|
96
205
|
|
|
97
206
|
Read full [documentation of the OpenAI integration for Micdrop](https://micdrop.dev/docs/ai-integration/provided-integrations/openai) on the [website](https://micdrop.dev).
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, STT, TTS } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -24,6 +24,66 @@ declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
|
|
|
24
24
|
cancel(): void;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* OpenAI Realtime API, a model hearing the user and answering with its voice
|
|
29
|
+
*
|
|
30
|
+
* @see https://developers.openai.com/api/docs/guides/realtime-conversations
|
|
31
|
+
*
|
|
32
|
+
* The Micdrop client already tells when the user speaks, so the server side
|
|
33
|
+
* turn detection is off: the audio of a turn is committed when the client
|
|
34
|
+
* closes it, and the answer is asked for right after.
|
|
35
|
+
*/
|
|
36
|
+
interface OpenaiRealtimeOptions extends RealtimeOptions {
|
|
37
|
+
apiKey: string;
|
|
38
|
+
model?: string;
|
|
39
|
+
voice?: string;
|
|
40
|
+
language?: string;
|
|
41
|
+
transcriptionModel?: string;
|
|
42
|
+
connectionTimeout?: number;
|
|
43
|
+
retryDelay?: number;
|
|
44
|
+
maxRetry?: number;
|
|
45
|
+
}
|
|
46
|
+
declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
47
|
+
private socket?;
|
|
48
|
+
private connected;
|
|
49
|
+
private everConnected;
|
|
50
|
+
private destroyed;
|
|
51
|
+
private pending;
|
|
52
|
+
private reconnectTimeout?;
|
|
53
|
+
private retryCount;
|
|
54
|
+
private inputResampler;
|
|
55
|
+
private outputResampler;
|
|
56
|
+
private generation;
|
|
57
|
+
private requested;
|
|
58
|
+
private response?;
|
|
59
|
+
private ignoredResponses;
|
|
60
|
+
private audioItemId?;
|
|
61
|
+
private audioStartedAt;
|
|
62
|
+
private audioBytes;
|
|
63
|
+
constructor(options: OpenaiRealtimeOptions);
|
|
64
|
+
destroy(): void;
|
|
65
|
+
protected openTurn(): void;
|
|
66
|
+
protected appendAudio(chunk: Buffer): void;
|
|
67
|
+
protected closeTurn(): void;
|
|
68
|
+
protected clearTurn(): void;
|
|
69
|
+
protected generate(): void;
|
|
70
|
+
protected speakText(text: string): void;
|
|
71
|
+
protected cancelAnswer(): void;
|
|
72
|
+
protected updateTools(): void;
|
|
73
|
+
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
74
|
+
private createResponse;
|
|
75
|
+
private send;
|
|
76
|
+
private messageItem;
|
|
77
|
+
private sendSessionUpdate;
|
|
78
|
+
private connect;
|
|
79
|
+
private initWS;
|
|
80
|
+
private closeSocket;
|
|
81
|
+
private reconnect;
|
|
82
|
+
private isIgnored;
|
|
83
|
+
private handleMessage;
|
|
84
|
+
private onResponseDone;
|
|
85
|
+
}
|
|
86
|
+
|
|
27
87
|
/**
|
|
28
88
|
* OpenAI Real-time STT
|
|
29
89
|
*
|
|
@@ -81,7 +141,7 @@ declare class OpenaiTTS extends TTS {
|
|
|
81
141
|
private openai;
|
|
82
142
|
private counter;
|
|
83
143
|
private generation;
|
|
84
|
-
private
|
|
144
|
+
private splitter;
|
|
85
145
|
private queue;
|
|
86
146
|
private pending;
|
|
87
147
|
private cushion;
|
|
@@ -91,7 +151,6 @@ declare class OpenaiTTS extends TTS {
|
|
|
91
151
|
constructor(options: OpenaiTTSOptions);
|
|
92
152
|
speak(textStream: Readable): void;
|
|
93
153
|
cancel(): void;
|
|
94
|
-
private flushSentences;
|
|
95
154
|
private enqueue;
|
|
96
155
|
/**
|
|
97
156
|
* Starts what can be started, then speaks what is ready.
|
|
@@ -118,4 +177,4 @@ declare class OpenaiTTS extends TTS {
|
|
|
118
177
|
private speakItem;
|
|
119
178
|
}
|
|
120
179
|
|
|
121
|
-
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
|
180
|
+
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiRealtime, type OpenaiRealtimeOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, AgentOptions, STT, TTS } from '@micdrop/server';
|
|
1
|
+
import { Agent, AgentOptions, Realtime, RealtimeOptions, STT, TTS } from '@micdrop/server';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
@@ -24,6 +24,66 @@ declare class OpenaiAgent extends Agent<OpenaiAgentOptions> {
|
|
|
24
24
|
cancel(): void;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
/**
|
|
28
|
+
* OpenAI Realtime API, a model hearing the user and answering with its voice
|
|
29
|
+
*
|
|
30
|
+
* @see https://developers.openai.com/api/docs/guides/realtime-conversations
|
|
31
|
+
*
|
|
32
|
+
* The Micdrop client already tells when the user speaks, so the server side
|
|
33
|
+
* turn detection is off: the audio of a turn is committed when the client
|
|
34
|
+
* closes it, and the answer is asked for right after.
|
|
35
|
+
*/
|
|
36
|
+
interface OpenaiRealtimeOptions extends RealtimeOptions {
|
|
37
|
+
apiKey: string;
|
|
38
|
+
model?: string;
|
|
39
|
+
voice?: string;
|
|
40
|
+
language?: string;
|
|
41
|
+
transcriptionModel?: string;
|
|
42
|
+
connectionTimeout?: number;
|
|
43
|
+
retryDelay?: number;
|
|
44
|
+
maxRetry?: number;
|
|
45
|
+
}
|
|
46
|
+
declare class OpenaiRealtime extends Realtime<OpenaiRealtimeOptions> {
|
|
47
|
+
private socket?;
|
|
48
|
+
private connected;
|
|
49
|
+
private everConnected;
|
|
50
|
+
private destroyed;
|
|
51
|
+
private pending;
|
|
52
|
+
private reconnectTimeout?;
|
|
53
|
+
private retryCount;
|
|
54
|
+
private inputResampler;
|
|
55
|
+
private outputResampler;
|
|
56
|
+
private generation;
|
|
57
|
+
private requested;
|
|
58
|
+
private response?;
|
|
59
|
+
private ignoredResponses;
|
|
60
|
+
private audioItemId?;
|
|
61
|
+
private audioStartedAt;
|
|
62
|
+
private audioBytes;
|
|
63
|
+
constructor(options: OpenaiRealtimeOptions);
|
|
64
|
+
destroy(): void;
|
|
65
|
+
protected openTurn(): void;
|
|
66
|
+
protected appendAudio(chunk: Buffer): void;
|
|
67
|
+
protected closeTurn(): void;
|
|
68
|
+
protected clearTurn(): void;
|
|
69
|
+
protected generate(): void;
|
|
70
|
+
protected speakText(text: string): void;
|
|
71
|
+
protected cancelAnswer(): void;
|
|
72
|
+
protected updateTools(): void;
|
|
73
|
+
protected injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
74
|
+
private createResponse;
|
|
75
|
+
private send;
|
|
76
|
+
private messageItem;
|
|
77
|
+
private sendSessionUpdate;
|
|
78
|
+
private connect;
|
|
79
|
+
private initWS;
|
|
80
|
+
private closeSocket;
|
|
81
|
+
private reconnect;
|
|
82
|
+
private isIgnored;
|
|
83
|
+
private handleMessage;
|
|
84
|
+
private onResponseDone;
|
|
85
|
+
}
|
|
86
|
+
|
|
27
87
|
/**
|
|
28
88
|
* OpenAI Real-time STT
|
|
29
89
|
*
|
|
@@ -81,7 +141,7 @@ declare class OpenaiTTS extends TTS {
|
|
|
81
141
|
private openai;
|
|
82
142
|
private counter;
|
|
83
143
|
private generation;
|
|
84
|
-
private
|
|
144
|
+
private splitter;
|
|
85
145
|
private queue;
|
|
86
146
|
private pending;
|
|
87
147
|
private cushion;
|
|
@@ -91,7 +151,6 @@ declare class OpenaiTTS extends TTS {
|
|
|
91
151
|
constructor(options: OpenaiTTSOptions);
|
|
92
152
|
speak(textStream: Readable): void;
|
|
93
153
|
cancel(): void;
|
|
94
|
-
private flushSentences;
|
|
95
154
|
private enqueue;
|
|
96
155
|
/**
|
|
97
156
|
* Starts what can be started, then speaks what is ready.
|
|
@@ -118,4 +177,4 @@ declare class OpenaiTTS extends TTS {
|
|
|
118
177
|
private speakItem;
|
|
119
178
|
}
|
|
120
179
|
|
|
121
|
-
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|
|
180
|
+
export { OpenaiAgent, type OpenaiAgentOptions, type OpenaiOptions, OpenaiRealtime, type OpenaiRealtimeOptions, OpenaiSTT, type OpenaiSTTOptions, OpenaiTTS, type OpenaiTTSOptions };
|