@micdrop/client 1.9.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,6 +1,12 @@
1
+ # 🖐️🎤 Micdrop: Real-Time Voice Conversations with AI
2
+
3
+ Micdrop is a set of open source Typescript packages to build real-time voice conversations with AI agents. It handles all the complexities on the browser and server side (microphone, speaker, VAD, network communication, etc) and provides ready-to-use implementations for various AI providers.
4
+
1
5
  # @micdrop/client
2
6
 
3
- A browser library for handling real-time voice conversations with microphone and speaker management.
7
+ The browser implementation of [Micdrop](../../README.md).
8
+
9
+ It is framework agnostic, you can use it with React, Vue, Angular or any other framework. See [demo-client](../../examples/demo-client/README.md) for a complete example with React.
4
10
 
5
11
  For server implementation, see [@micdrop/server](../server/README.md) package.
6
12
 
@@ -8,7 +14,7 @@ For server implementation, see [@micdrop/server](../server/README.md) package.
8
14
 
9
15
  - 🎤 Real-time microphone recording with voice activity detection (VAD)
10
16
  - 🔊 Advanced audio playback:
11
- - Streaming support via MediaSource
17
+ - Streaming support
12
18
  - Pause/resume functionality
13
19
  - Volume control
14
20
  - Device selection and testing
@@ -29,60 +35,104 @@ yarn add @micdrop/client
29
35
  pnpm add @micdrop/client
30
36
  ```
31
37
 
38
+ If you work with React, you can also install [@micdrop/react](../react/README.md) package to get a ready-to-use React hooks.
39
+
32
40
  ## Quick Start
33
41
 
42
+ ### Start a call
43
+
44
+ `Micdrop` is a singleton instance of `MicdropClient` that can be used to start, stop and manage a call.
45
+
34
46
  ```typescript
35
- import { CallClient } from '@micdrop/client'
47
+ import { Micdrop } from '@micdrop/client'
36
48
 
37
- // Create a new call handler instance
38
- const call = CallClient.getInstance<YourParamsType>({
39
- // VAD (see docs)
40
- vad: ['volume', 'silero'],
49
+ // Start a call
50
+ await Micdrop.start({
51
+ url: 'wss://your-server.com/ws',
41
52
  })
53
+ ```
42
54
 
43
- // Configure the call
44
- call.url = 'wss://your-server.com/ws'
45
- call.params = {
46
- /* your parameters (to check auth, etc) */
47
- }
55
+ ### Control the call
48
56
 
49
- // Listen for events
50
- call.on('StateChange', () => {
57
+ ```typescript
58
+ // Pause/resume
59
+ Micdrop.pause()
60
+ Micdrop.resume()
61
+
62
+ // Stop the call
63
+ await Micdrop.stop()
64
+ ```
65
+
66
+ ### Listen for events
67
+
68
+ You can listen for events to get the state of the call and handle errors.
69
+
70
+ ```typescript
71
+ Micdrop.on('StateChange', () => {
51
72
  console.log('State changed')
52
73
  })
53
- call.on('EndCall', () => {
74
+ Micdrop.on('EndCall', () => {
54
75
  console.log('Call ended by assistant')
55
76
  })
56
- call.on('Error', (error) => {
77
+ Micdrop.on('Error', (error) => {
57
78
  console.error('Error occurred:', error)
58
79
  })
80
+ ```
81
+
82
+ ### Complete Example
59
83
 
60
- // Start the call
61
- await call.start()
84
+ ```typescript
85
+ import { Micdrop } from '@micdrop/client'
86
+
87
+ // Start a call
88
+ await Micdrop.start({
89
+ // URL of the WebSocket server (using @micdrop/server)
90
+ url: 'wss://your-server.com/ws',
91
+ // Parameters (optional) to check auth or provide other data
92
+ params: {
93
+ authorization: '1234',
94
+ lang: navigator.language,
95
+ },
96
+ // Voice Activity Detection (see docs)
97
+ vad: ['volume', 'silero'],
98
+ // Disable ability for the user to interrupt the assistant when it is speaking
99
+ disableInterruption: true,
100
+ // Enable debug logging
101
+ debugLog: true,
102
+ })
103
+
104
+ // Listen for events
105
+ Micdrop.on('StateChange', (state) => {
106
+ console.log('State:', state)
107
+ })
108
+ Micdrop.on('EndCall', () => {
109
+ console.log('Call ended by assistant')
110
+ })
111
+ Micdrop.on('Error', (error) => {
112
+ console.error('Error occurred:', error)
113
+ })
62
114
 
63
115
  // Pause/resume
64
- call.pause()
65
- call.resume()
116
+ Micdrop.pause()
117
+ Micdrop.resume()
66
118
 
67
119
  // Stop the call
68
- await call.stop()
120
+ await Micdrop.stop()
69
121
  ```
70
122
 
71
123
  ## Demo
72
124
 
73
- Check out the demo implementation in the [@micdrop/demo-client](../demo-client/README.md) package. It shows:
125
+ Check out the demo implementation in the [demo-client example](../../examples/demo-client/README.md) package. It shows:
74
126
 
75
127
  - Setting up a React application with WebSocket communication
76
- - Configuring the CallClient with custom parameters
128
+ - Configuring the MicdropClient with custom parameters
77
129
  - Managing microphone input and audio playback
78
130
  - Handling conversation state and UI updates
79
131
  - Error handling patterns
80
132
 
81
- Here's a simplified version from the demo:
82
-
83
133
  ## Documentation
84
134
 
85
- - **[CallClient](./docs/CallClient.md)** - Manages WebSocket connections, audio streaming, and conversation state
135
+ - **[MicdropClient](./docs/MicdropClient.md)** - Manages WebSocket connections, audio streaming, and conversation state
86
136
 
87
137
  - **[MicRecorder](./docs/MicRecorder.md)** - Records audio from microphone with voice activity detection and speech events
88
138
 
@@ -90,14 +140,18 @@ Here's a simplified version from the demo:
90
140
 
91
141
  - **[Speaker](./docs/Speaker.md)** - Handles audio output devices and playback with analysis capabilities
92
142
 
143
+ - **[VAD](./docs/VAD.md)** - Voice Activity Detection, how to use and customize
144
+
93
145
  ## Browser Support
94
146
 
147
+ Fully supported in Chrome, Firefox, Safari and Edge.
148
+
95
149
  Requires browsers with support for:
96
150
 
97
151
  - WebSocket API
98
- - MediaRecorder API
99
152
  - Web Audio API
100
- - getUserMedia API
153
+ - MediaDevices API
154
+ - MediaRecorder API
101
155
 
102
156
  ## License
103
157
 
@@ -105,6 +159,4 @@ MIT
105
159
 
106
160
  ## Author
107
161
 
108
- Originally developed for [Raconte.ai](https://www.raconte.ai)
109
-
110
- by [Lonestone](https://www.lonestone.io) ([GitHub](https://github.com/lonestone))
162
+ Originally developed for [Raconte.ai](https://www.raconte.ai) and open sourced by [Lonestone](https://www.lonestone.io) ([GitHub](https://github.com/lonestone))
package/dist/index.d.mts CHANGED
@@ -1,5 +1,6 @@
1
1
  import EventEmitter from 'eventemitter3';
2
- import { VAD as VAD$1, Conversation as Conversation$1 } from '@micdrop/client';
2
+ import { MicdropConversation as MicdropConversation$1, VAD as VAD$1 } from '@micdrop/client';
3
+ import { Readable } from 'stream';
3
4
 
4
5
  interface AudioAnalyserEvents {
5
6
  volume: [number];
@@ -16,13 +17,11 @@ declare class AudioAnalyser extends EventEmitter<AudioAnalyserEvents> {
16
17
  private getMaxVolume;
17
18
  }
18
19
 
19
- declare let mic: Mic;
20
- declare class Mic {
20
+ declare class Mic$1 {
21
21
  analyser: AudioAnalyser;
22
- private currentDeviceId;
22
+ deviceId: string | undefined;
23
23
  private audioStream;
24
24
  private sourceNode;
25
- constructor();
26
25
  private getMicConstraints;
27
26
  /**
28
27
  * Starts the microphone
@@ -35,10 +34,26 @@ declare class Mic {
35
34
  */
36
35
  stop(): void;
37
36
  }
38
- declare global {
39
- interface Window {
40
- micdropMic: Mic;
41
- }
37
+
38
+ interface SpeakerEvents {
39
+ StartPlaying: void;
40
+ StopPlaying: void;
41
+ }
42
+ declare class Speaker$1 extends EventEmitter<SpeakerEvents> {
43
+ isPlaying: boolean;
44
+ analyser: AudioAnalyser;
45
+ deviceId: string | undefined;
46
+ devices: MediaDeviceInfo[];
47
+ private audioStream?;
48
+ private audioSourceNode?;
49
+ private audioElementDestination?;
50
+ start(): Promise<void>;
51
+ playAudio(blob: Blob): Promise<void>;
52
+ stopAudio(): Promise<void>;
53
+ changeDevice(deviceId: string): Promise<void>;
54
+ get isDefaultDevice(): boolean;
55
+ private setIsPlaying;
56
+ private setDevice;
42
57
  }
43
58
 
44
59
  interface VADEvents {
@@ -58,6 +73,7 @@ declare enum VADStatus {
58
73
  * VAD = Voice Activity Detection
59
74
  */
60
75
  declare abstract class VAD extends EventEmitter<VADEvents> {
76
+ readonly name: string;
61
77
  status: VADStatus;
62
78
  delay: number;
63
79
  /**
@@ -90,6 +106,7 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
90
106
 
91
107
  type VADConfigName = 'volume' | 'silero';
92
108
  type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
109
+ declare function getVAD(vad?: VADConfig): VAD;
93
110
 
94
111
  interface MicRecorderState {
95
112
  isStarting: boolean;
@@ -105,17 +122,17 @@ interface MicRecorderEvents {
105
122
  }
106
123
  declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
107
124
  state: MicRecorderState;
125
+ vad: VAD;
108
126
  private audioInfo;
109
- private vad;
110
127
  private recorder;
111
128
  private delayedStream;
112
129
  private speakingConfirmed;
113
130
  private queuedChunks;
114
131
  constructor(vadConfig?: VADConfig);
115
- start(stream: MediaStream): Promise<void>;
116
- mute(): void;
117
- unmute(): void;
118
- stop(): void;
132
+ start: (stream: MediaStream) => Promise<void>;
133
+ mute: () => void;
134
+ unmute: () => void;
135
+ stop: () => void;
119
136
  private startSpeakingDetection;
120
137
  private stopSpeakingDetection;
121
138
  private onStartSpeaking;
@@ -127,37 +144,6 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
127
144
  private getAudioInfo;
128
145
  }
129
146
 
130
- declare let speaker: Speaker;
131
- interface SpeakerEvents {
132
- StartPlaying: void;
133
- StopPlaying: void;
134
- }
135
- declare class Speaker extends EventEmitter<SpeakerEvents> {
136
- isPlaying: boolean;
137
- analyser: AudioAnalyser;
138
- private audioElement?;
139
- private player?;
140
- private streamingEnabled;
141
- constructor();
142
- private setupSpeakerPlayer;
143
- get isStreamingEnabled(): boolean;
144
- protected changeIsPlaying(isPlaying: boolean): void;
145
- enableStreaming(): Promise<void>;
146
- disableStreaming(): Promise<void>;
147
- canChangeDevice(): boolean;
148
- private setSinkId;
149
- changeDevice(speakerId: string): Promise<void>;
150
- pauseAudio(): void;
151
- resumeAudio(): void;
152
- playAudio(blob: Blob): Promise<void>;
153
- stopAudio(): void;
154
- }
155
- declare global {
156
- interface Window {
157
- micdropSpeaker: Speaker;
158
- }
159
- }
160
-
161
147
  declare enum LocalStorageKeys {
162
148
  MicDevice = "micdrop.micDevice",
163
149
  SpeakerDevice = "micdrop.speakerDevice",
@@ -225,45 +211,69 @@ declare class VolumeVAD extends VAD {
225
211
  resetOptions(): void;
226
212
  }
227
213
 
228
- declare enum CallClientErrorCode {
214
+ declare const Mic: Mic$1;
215
+ declare const Speaker: Speaker$1;
216
+
217
+ declare enum MicdropClientErrorCode {
218
+ MissingUrl = "MissingUrl",
229
219
  Mic = "Mic",
220
+ Connection = "Connection",
221
+ InternalServer = "InternalServer",
222
+ BadRequest = "BadRequest",
230
223
  Unauthorized = "Unauthorized",
231
- Error = "Error"
224
+ NotFound = "NotFound",
225
+ Unknown = "Unknown"
232
226
  }
233
- declare class CallClientError extends Error {
234
- code: CallClientErrorCode;
235
- constructor(code?: CallClientErrorCode);
227
+ declare class MicdropClientError extends Error {
228
+ code: MicdropClientErrorCode;
229
+ constructor(code?: MicdropClientErrorCode, message?: string);
236
230
  }
231
+ declare function getClientErrorFromWSCloseEvent(event: CloseEvent): MicdropClientError | undefined;
232
+ declare function getClientErrorFromWSCloseEventCode(code: number): MicdropClientErrorCode | undefined;
237
233
 
238
- interface CallClientEvents {
234
+ interface MicdropEvents {
239
235
  EndCall: [];
240
- Error: [CallClientError];
241
- StateChange: void;
236
+ Error: [MicdropClientError];
237
+ StateChange: [MicdropState];
242
238
  }
243
- interface CallClientOptions {
239
+ interface MicdropOptions {
240
+ url?: string;
241
+ params?: Record<string, any>;
244
242
  vad?: VADConfig;
245
243
  disableInterruption?: boolean;
244
+ debugLog?: boolean;
246
245
  }
247
- declare global {
248
- interface Window {
249
- micdropCallClient: any;
250
- }
246
+ interface MicdropState {
247
+ isStarting: boolean;
248
+ isStarted: boolean;
249
+ isPaused: boolean;
250
+ isListening: boolean;
251
+ isProcessing: boolean;
252
+ isUserSpeaking: boolean;
253
+ isAssistantSpeaking: boolean;
254
+ isMicStarted: boolean;
255
+ isMicMuted: boolean;
256
+ micDeviceId: string | undefined;
257
+ speakerDeviceId: string | undefined;
258
+ micDevices: MediaDeviceInfo[];
259
+ speakerDevices: MediaDeviceInfo[];
260
+ conversation: MicdropConversation$1;
261
+ error: MicdropClientError | undefined;
251
262
  }
252
- declare class CallClient<Params extends {}, Options extends CallClientOptions = CallClientOptions> extends EventEmitter<CallClientEvents> {
253
- private options?;
254
- static getInstance<P extends {}, O extends CallClientOptions = CallClientOptions>(options?: O): CallClient<P, O>;
255
- url?: string;
256
- params?: Params;
263
+ declare class MicdropClient extends EventEmitter<MicdropEvents> implements MicdropState {
264
+ options: MicdropOptions;
257
265
  micRecorder?: MicRecorder;
258
- vad: VAD$1;
259
- conversation: Conversation$1;
260
- debug: boolean;
266
+ conversation: MicdropConversation$1;
267
+ error: MicdropClientError | undefined;
268
+ speakerDevices: MediaDeviceInfo[];
269
+ micDevices: MediaDeviceInfo[];
261
270
  private ws?;
262
271
  private micStream?;
263
272
  private startTime;
264
273
  private _isProcessing;
265
274
  private _isPaused;
266
- private constructor();
275
+ constructor(options?: MicdropOptions);
276
+ get vad(): VAD$1 | undefined;
267
277
  get isStarted(): boolean;
268
278
  get isStarting(): boolean;
269
279
  get isPaused(): boolean;
@@ -275,65 +285,139 @@ declare class CallClient<Params extends {}, Options extends CallClientOptions =
275
285
  get isMicMuted(): boolean;
276
286
  get isUserSpeaking(): boolean;
277
287
  get isAssistantSpeaking(): boolean;
278
- start(): Promise<void>;
279
- stop(): Promise<void>;
280
- pause(): void;
281
- resume(): void;
282
- destroy(): Promise<void>;
283
- startMic(deviceId?: string, record?: boolean): Promise<void>;
288
+ get micDeviceId(): string | undefined;
289
+ get speakerDeviceId(): string | undefined;
290
+ get state(): MicdropState;
291
+ start: (options?: MicdropOptions) => Promise<void>;
292
+ stop: () => Promise<void>;
293
+ pause: () => void;
294
+ resume: () => void;
295
+ startMic: (options?: {
296
+ vad?: VADConfig;
297
+ deviceId?: string;
298
+ record?: boolean;
299
+ }) => Promise<void>;
300
+ changeMicDevice: (deviceId: string) => Promise<void>;
301
+ changeSpeakerDevice: (deviceId: string) => Promise<void>;
284
302
  private stopMic;
303
+ private updateDevices;
285
304
  private startWS;
305
+ private onWSOpen;
306
+ private onWSMessage;
307
+ private onWSClose;
308
+ private onWSError;
286
309
  private stopWS;
287
310
  private onSpeakerStartPlaying;
288
311
  private onSpeakerStopPlaying;
289
312
  private addMessage;
313
+ private setError;
290
314
  private notifyStateChange;
291
315
  private log;
292
316
  }
293
317
 
294
- declare enum CallClientCommands {
318
+ declare const Micdrop: MicdropClient;
319
+
320
+ declare class Logger {
321
+ private readonly name;
322
+ constructor(name: string);
323
+ log(...message: any[]): void;
324
+ }
325
+
326
+ interface AgentOptions {
327
+ systemPrompt: string;
328
+ }
329
+ interface AgentAnswerReturn {
330
+ message: Promise<string>;
331
+ stream: Readable;
332
+ }
333
+ interface TextPromise {
334
+ promise: Promise<string>;
335
+ resolve: (value: string) => void;
336
+ reject: (reason?: any) => void;
337
+ }
338
+ declare abstract class Agent<Options extends AgentOptions = AgentOptions> extends EventEmitter {
339
+ protected options: Options;
340
+ logger?: Logger;
341
+ conversation: MicdropConversation;
342
+ constructor(options: Options);
343
+ abstract answer(): AgentAnswerReturn;
344
+ abstract cancel(): void;
345
+ addUserMessage(text: string): void;
346
+ addAssistantMessage(text: string): void;
347
+ protected addMessage(role: 'user' | 'assistant' | 'system', text: string): void;
348
+ protected endCall(): void;
349
+ protected cancelLastUserMessage(): void;
350
+ protected cancelLastAssistantMessage(): void;
351
+ protected skipAnswer(): void;
352
+ protected createTextPromise(): TextPromise;
353
+ protected log(...message: any[]): void;
354
+ destroy(): void;
355
+ }
356
+
357
+ declare const MIME_TYPE_TO_EXTENSION: {
358
+ readonly 'audio/wav': "wav";
359
+ readonly 'audio/ogg': "ogg";
360
+ readonly 'audio/mpeg': "mp3";
361
+ readonly 'audio/webm': "webm";
362
+ readonly 'audio/mp4': "mp4";
363
+ readonly 'audio/flac': "flac";
364
+ };
365
+ interface STTEvents {
366
+ Transcript: [string];
367
+ }
368
+ declare abstract class STT extends EventEmitter<STTEvents> {
369
+ protected mimeType?: keyof typeof MIME_TYPE_TO_EXTENSION;
370
+ logger?: Logger;
371
+ transcribe(audioStream: Readable): void;
372
+ protected log(...message: any[]): void;
373
+ destroy(): void;
374
+ protected get extension(): string;
375
+ private detectMimeType;
376
+ }
377
+
378
+ declare abstract class TTS {
379
+ logger?: Logger;
380
+ abstract speak(textStream: Readable): Readable;
381
+ abstract cancel(): void;
382
+ protected log(...message: any[]): void;
383
+ destroy(): void;
384
+ }
385
+
386
+ declare enum MicdropClientCommands {
295
387
  StartSpeaking = "StartSpeaking",
296
388
  StopSpeaking = "StopSpeaking",
297
389
  Mute = "Mute"
298
390
  }
299
- declare enum CallServerCommands {
391
+ declare enum MicdropServerCommands {
300
392
  Message = "Message",
301
393
  CancelLastAssistantMessage = "CancelLastAssistantMessage",
302
394
  CancelLastUserMessage = "CancelLastUserMessage",
303
395
  SkipAnswer = "SkipAnswer",
304
- EnableSpeakerStreaming = "EnableSpeakerStreaming",
305
396
  EndCall = "EndCall"
306
397
  }
307
- interface CallConfig {
308
- systemPrompt: string;
398
+ interface MicdropConfig {
309
399
  firstMessage?: string;
310
- debugLog?: boolean;
311
- debugSaveSpeech?: boolean;
312
- disableTTS?: boolean;
313
- generateAnswer(conversation: Conversation): Promise<string | ConversationMessage>;
314
- speech2Text(audioBlob: Blob, prevMessage?: string): Promise<string>;
315
- text2Speech(text: string): Promise<ArrayBuffer | NodeJS.ReadableStream>;
316
- onMessage?(message: ConversationMessage): void;
317
- onEnd?(call: CallSummary): void;
318
- }
319
- interface CallSummary {
320
- conversation: Conversation;
400
+ generateFirstMessage?: boolean;
401
+ agent: Agent;
402
+ stt: STT;
403
+ tts: TTS;
404
+ onEnd?(call: MicdropCallSummary): void;
405
+ }
406
+ interface MicdropCallSummary {
407
+ conversation: MicdropConversation;
321
408
  duration: number;
322
409
  }
323
- type Conversation = ConversationMessage[];
324
- type AnswerCommands = {
325
- endCall?: boolean;
326
- cancelLastUserMessage?: boolean;
327
- skipAnswer?: boolean;
328
- };
329
- type AnswerMetadata = {
410
+ type MicdropConversation = MicdropConversationMessage[];
411
+ type MicdropAnswerMetadata = {
330
412
  [key: string]: any;
331
413
  };
332
- interface ConversationMessage<Data extends AnswerMetadata = AnswerMetadata> {
414
+ interface MicdropConversationMessage<Data extends MicdropAnswerMetadata = MicdropAnswerMetadata> {
333
415
  role: 'system' | 'user' | 'assistant';
334
416
  content: string;
335
- commands?: AnswerCommands;
336
417
  metadata?: Data;
337
418
  }
419
+ type DeepPartial<T> = T extends object ? {
420
+ [P in keyof T]?: DeepPartial<T[P]>;
421
+ } : T;
338
422
 
339
- export { type AnswerCommands, type AnswerMetadata, CallClient, CallClientCommands, CallClientError, CallClientErrorCode, type CallClientEvents, type CallClientOptions, type CallConfig, CallServerCommands, type CallSummary, type Conversation, type ConversationMessage, LocalStorageKeys, mic as Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, MultipleVAD, SileroVAD, type SileroVADOptions, speaker as Speaker, VAD, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions };
423
+ export { type DeepPartial, LocalStorageKeys, Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConfig, type MicdropConversation, type MicdropConversationMessage, type MicdropEvents, type MicdropOptions, MicdropServerCommands, type MicdropState, MultipleVAD, SileroVAD, type SileroVADOptions, Speaker, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD };