@micdrop/client 1.0.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -41
- package/dist/index.d.mts +130 -16
- package/dist/index.d.ts +130 -16
- package/dist/index.js +391 -96
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +387 -95
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -7,7 +7,12 @@ For server implementation, see [@micdrop/server](../server/README.md) package.
|
|
|
7
7
|
## Features
|
|
8
8
|
|
|
9
9
|
- 🎤 Real-time microphone recording with voice activity detection (VAD)
|
|
10
|
-
- 🔊
|
|
10
|
+
- 🔊 Advanced audio playback:
|
|
11
|
+
- Streaming support via MediaSource
|
|
12
|
+
- Pause/resume functionality
|
|
13
|
+
- Volume control
|
|
14
|
+
- Device selection and testing
|
|
15
|
+
- Real-time audio analysis
|
|
11
16
|
- 🌐 WebSocket-based audio streaming
|
|
12
17
|
- 📊 Audio analysis and volume monitoring
|
|
13
18
|
- 🎛️ Configurable speech detection settings
|
|
@@ -30,56 +35,33 @@ pnpm add @micdrop/client
|
|
|
30
35
|
import { CallHandler } from '@micdrop/client'
|
|
31
36
|
|
|
32
37
|
// Create a new call handler instance
|
|
33
|
-
const call =
|
|
38
|
+
const call = CallHandler.getInstance<YourParamsType>()
|
|
34
39
|
|
|
35
|
-
//
|
|
40
|
+
// Configure the call
|
|
41
|
+
call.url = 'wss://your-server.com/ws'
|
|
42
|
+
call.params = {
|
|
43
|
+
/* your parameters (to check auth, etc) */
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Listen for events
|
|
36
47
|
call.on('StateChange', () => {
|
|
37
|
-
console.log('
|
|
38
|
-
/*
|
|
39
|
-
{
|
|
40
|
-
"conversation": [
|
|
41
|
-
{ "role": "assistant", "content": "Hello!" },
|
|
42
|
-
{ "role": "user", "content": "User Message 1" },
|
|
43
|
-
{ "role": "assistant", "content": "Assistant Message 1" }
|
|
44
|
-
],
|
|
45
|
-
"isMicStarted": false,
|
|
46
|
-
"isStarted": false,
|
|
47
|
-
"isStarting": false,
|
|
48
|
-
"isWSStarted": false,
|
|
49
|
-
"isWSStarting": false
|
|
50
|
-
}
|
|
51
|
-
*/
|
|
52
|
-
|
|
53
|
-
console.log('MicRecorder state:', call.micRecorder.state)
|
|
54
|
-
/*
|
|
55
|
-
{
|
|
56
|
-
"isStarting": false,
|
|
57
|
-
"isStarted": false,
|
|
58
|
-
"isMuted": false,
|
|
59
|
-
"isSpeaking": false,
|
|
60
|
-
"threshold": -50
|
|
61
|
-
}
|
|
62
|
-
*/
|
|
48
|
+
console.log('State changed')
|
|
63
49
|
})
|
|
64
|
-
|
|
65
|
-
// Start the call
|
|
66
|
-
await call.start()
|
|
67
|
-
|
|
68
|
-
// Pause/resume microphone
|
|
69
|
-
call.pause()
|
|
70
|
-
call.resume()
|
|
71
|
-
|
|
72
|
-
// Listen for end of interview
|
|
73
50
|
call.on('EndInterview', () => {
|
|
74
51
|
console.log('Interview ended')
|
|
75
52
|
})
|
|
76
|
-
|
|
77
|
-
// Listen for errors
|
|
78
53
|
call.on('Error', (error) => {
|
|
79
54
|
console.error('Error occurred:', error)
|
|
80
55
|
})
|
|
81
56
|
|
|
82
|
-
//
|
|
57
|
+
// Start the call
|
|
58
|
+
await call.start()
|
|
59
|
+
|
|
60
|
+
// Pause/resume
|
|
61
|
+
call.pause()
|
|
62
|
+
call.resume()
|
|
63
|
+
|
|
64
|
+
// Stop the call
|
|
83
65
|
await call.stop()
|
|
84
66
|
```
|
|
85
67
|
|
package/dist/index.d.mts
CHANGED
|
@@ -6,6 +6,46 @@ declare enum LocalStorageKeys {
|
|
|
6
6
|
MicThreshold = "micdrop.micThreshold"
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
interface VADEvents {
|
|
10
|
+
StartSpeaking: void;
|
|
11
|
+
ConfirmSpeaking: void;
|
|
12
|
+
CancelSpeaking: void;
|
|
13
|
+
StopSpeaking: void;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Abstract VAD class
|
|
17
|
+
* VAD = Voice Activity Detection
|
|
18
|
+
*/
|
|
19
|
+
declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
20
|
+
delay: number;
|
|
21
|
+
protected threshold: number;
|
|
22
|
+
/**
|
|
23
|
+
* Constructor for the VAD class
|
|
24
|
+
* @param threshold - The threshold to use for the VAD
|
|
25
|
+
*/
|
|
26
|
+
constructor(threshold?: number);
|
|
27
|
+
/**
|
|
28
|
+
* Checks if the VAD is started
|
|
29
|
+
* @returns True if the VAD is started, false otherwise
|
|
30
|
+
*/
|
|
31
|
+
abstract get isStarted(): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Starts the VAD
|
|
34
|
+
* @param stream - The stream to start the VAD on
|
|
35
|
+
* @param threshold - The threshold to use for the VAD
|
|
36
|
+
*/
|
|
37
|
+
abstract start(stream: MediaStream, threshold?: number): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Stops the VAD
|
|
40
|
+
*/
|
|
41
|
+
abstract stop(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Sets the threshold for the VAD
|
|
44
|
+
* @param threshold - The threshold to use for the VAD
|
|
45
|
+
*/
|
|
46
|
+
abstract setThreshold(threshold: number): Promise<void>;
|
|
47
|
+
}
|
|
48
|
+
|
|
9
49
|
interface MicRecorderState {
|
|
10
50
|
isStarting: boolean;
|
|
11
51
|
isStarted: boolean;
|
|
@@ -22,10 +62,12 @@ interface MicRecorderEvents {
|
|
|
22
62
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
23
63
|
state: MicRecorderState;
|
|
24
64
|
private audioInfo;
|
|
25
|
-
private
|
|
65
|
+
private vad;
|
|
26
66
|
private recorder;
|
|
27
67
|
private delayedStream;
|
|
28
|
-
|
|
68
|
+
private speakingConfirmed;
|
|
69
|
+
private queuedChunks;
|
|
70
|
+
constructor(vad: VAD);
|
|
29
71
|
start(stream: MediaStream): Promise<void>;
|
|
30
72
|
mute(): void;
|
|
31
73
|
unmute(): void;
|
|
@@ -34,6 +76,8 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
34
76
|
private startSpeakingDetection;
|
|
35
77
|
private stopSpeakingDetection;
|
|
36
78
|
private onStartSpeaking;
|
|
79
|
+
private onConfirmSpeaking;
|
|
80
|
+
private onCancelSpeaking;
|
|
37
81
|
private onStopSpeaking;
|
|
38
82
|
private onDataAvailable;
|
|
39
83
|
private changeState;
|
|
@@ -55,36 +99,102 @@ declare class AudioAnalyser extends EventEmitter<AudioAnalyserEvents> {
|
|
|
55
99
|
private getMaxVolume;
|
|
56
100
|
}
|
|
57
101
|
|
|
102
|
+
declare let audioContext: AudioContext | undefined;
|
|
58
103
|
declare let micAnalyser: AudioAnalyser | undefined;
|
|
59
104
|
declare const defaultMicThreshold = -50;
|
|
60
|
-
|
|
105
|
+
/**
|
|
106
|
+
* Starts the microphone
|
|
107
|
+
* @param deviceId - The deviceId to use
|
|
108
|
+
* @returns The microphone stream
|
|
109
|
+
*/
|
|
61
110
|
declare function startMicrophone(deviceId?: string): Promise<MediaStream>;
|
|
111
|
+
/**
|
|
112
|
+
* Stops the microphone
|
|
113
|
+
*/
|
|
62
114
|
declare function stopMicrophone(): void;
|
|
63
|
-
declare function createDelayedStream(stream: MediaStream): MediaStream;
|
|
64
115
|
|
|
65
|
-
declare const
|
|
116
|
+
declare const microphone_audioContext: typeof audioContext;
|
|
66
117
|
declare const microphone_defaultMicThreshold: typeof defaultMicThreshold;
|
|
67
|
-
declare const microphone_getMicConstraints: typeof getMicConstraints;
|
|
68
118
|
declare const microphone_micAnalyser: typeof micAnalyser;
|
|
69
119
|
declare const microphone_startMicrophone: typeof startMicrophone;
|
|
70
120
|
declare const microphone_stopMicrophone: typeof stopMicrophone;
|
|
71
121
|
declare namespace microphone {
|
|
72
|
-
export {
|
|
122
|
+
export { microphone_audioContext as audioContext, microphone_defaultMicThreshold as defaultMicThreshold, microphone_micAnalyser as micAnalyser, microphone_startMicrophone as startMicrophone, microphone_stopMicrophone as stopMicrophone };
|
|
73
123
|
}
|
|
74
124
|
|
|
75
125
|
declare let speakerAnalyser: AudioAnalyser | undefined;
|
|
126
|
+
/**
|
|
127
|
+
* Checks if the speaker device can be changed
|
|
128
|
+
* @returns True if the speaker device can be changed, false otherwise
|
|
129
|
+
*/
|
|
76
130
|
declare function canChangeSpeakerDevice(): boolean;
|
|
131
|
+
/**
|
|
132
|
+
* Changes the speaker device
|
|
133
|
+
* @param speakerId - The speakerId to use
|
|
134
|
+
*/
|
|
77
135
|
declare function changeSpeakerDevice(speakerId: string): Promise<void>;
|
|
78
|
-
|
|
79
|
-
|
|
136
|
+
/**
|
|
137
|
+
* Pauses the audio
|
|
138
|
+
*/
|
|
139
|
+
declare function pauseAudio(): void;
|
|
140
|
+
/**
|
|
141
|
+
* Resumes the audio
|
|
142
|
+
*/
|
|
143
|
+
declare function resumeAudio(): void;
|
|
144
|
+
/**
|
|
145
|
+
* Plays the audio
|
|
146
|
+
* @param blob - The blob to play (adds to the queue if already playing)
|
|
147
|
+
*/
|
|
148
|
+
declare function playAudio(blob: Blob): Promise<void>;
|
|
149
|
+
/**
|
|
150
|
+
* Stops the audio
|
|
151
|
+
*/
|
|
152
|
+
declare function stopAudio(): void;
|
|
80
153
|
|
|
81
154
|
declare const speaker_canChangeSpeakerDevice: typeof canChangeSpeakerDevice;
|
|
82
155
|
declare const speaker_changeSpeakerDevice: typeof changeSpeakerDevice;
|
|
83
|
-
declare const
|
|
156
|
+
declare const speaker_pauseAudio: typeof pauseAudio;
|
|
157
|
+
declare const speaker_playAudio: typeof playAudio;
|
|
158
|
+
declare const speaker_resumeAudio: typeof resumeAudio;
|
|
84
159
|
declare const speaker_speakerAnalyser: typeof speakerAnalyser;
|
|
85
|
-
declare const
|
|
160
|
+
declare const speaker_stopAudio: typeof stopAudio;
|
|
86
161
|
declare namespace speaker {
|
|
87
|
-
export { speaker_canChangeSpeakerDevice as canChangeSpeakerDevice, speaker_changeSpeakerDevice as changeSpeakerDevice,
|
|
162
|
+
export { speaker_canChangeSpeakerDevice as canChangeSpeakerDevice, speaker_changeSpeakerDevice as changeSpeakerDevice, speaker_pauseAudio as pauseAudio, speaker_playAudio as playAudio, speaker_resumeAudio as resumeAudio, speaker_speakerAnalyser as speakerAnalyser, speaker_stopAudio as stopAudio };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Silero VAD
|
|
167
|
+
* @see https://github.com/ricky0123/vad
|
|
168
|
+
*/
|
|
169
|
+
declare class SileroVAD extends VAD {
|
|
170
|
+
private vad;
|
|
171
|
+
get isStarted(): boolean;
|
|
172
|
+
start(stream: MediaStream, threshold?: number): Promise<void>;
|
|
173
|
+
stop(): Promise<void>;
|
|
174
|
+
setThreshold(threshold: number): Promise<void>;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Volume-based Voice Activity Detection
|
|
179
|
+
* Based on the hark library implementation
|
|
180
|
+
*/
|
|
181
|
+
declare class VolumeVAD extends VAD {
|
|
182
|
+
private audioContext;
|
|
183
|
+
private analyser;
|
|
184
|
+
private sourceNode;
|
|
185
|
+
private fftBins;
|
|
186
|
+
private running;
|
|
187
|
+
private speaking;
|
|
188
|
+
private attemptSpeaking;
|
|
189
|
+
private history;
|
|
190
|
+
private speakingHistory;
|
|
191
|
+
constructor();
|
|
192
|
+
private getMaxVolume;
|
|
193
|
+
get isStarted(): boolean;
|
|
194
|
+
start(stream: MediaStream, threshold?: number): Promise<void>;
|
|
195
|
+
private startLoop;
|
|
196
|
+
stop(): Promise<void>;
|
|
197
|
+
setThreshold(threshold: number): Promise<void>;
|
|
88
198
|
}
|
|
89
199
|
|
|
90
200
|
declare enum CallHandlerErrorCode {
|
|
@@ -102,9 +212,12 @@ interface CallHandlerEvents {
|
|
|
102
212
|
Error: [CallHandlerError];
|
|
103
213
|
StateChange: void;
|
|
104
214
|
}
|
|
105
|
-
|
|
215
|
+
interface CallHandlerOptions {
|
|
216
|
+
vad?: VAD | 'volume' | 'silero';
|
|
217
|
+
}
|
|
218
|
+
declare class CallHandler<Params extends {}, Options extends CallHandlerOptions = CallHandlerOptions> extends EventEmitter<CallHandlerEvents> {
|
|
106
219
|
private static instance;
|
|
107
|
-
static getInstance<T extends {}>(): CallHandler<T>;
|
|
220
|
+
static getInstance<T extends {}, Options extends CallHandlerOptions = CallHandlerOptions>(options?: Options): CallHandler<T, Options>;
|
|
108
221
|
url?: string;
|
|
109
222
|
params?: Params;
|
|
110
223
|
micRecorder: MicRecorder;
|
|
@@ -115,6 +228,7 @@ declare class CallHandler<Params extends {}> extends EventEmitter<CallHandlerEve
|
|
|
115
228
|
private startTime;
|
|
116
229
|
private lastAudioBlob?;
|
|
117
230
|
private constructor();
|
|
231
|
+
private getVAD;
|
|
118
232
|
get isStarted(): boolean;
|
|
119
233
|
get isStarting(): boolean;
|
|
120
234
|
get isWSStarted(): boolean;
|
|
@@ -152,7 +266,7 @@ interface CallConfig {
|
|
|
152
266
|
disableTTS?: boolean;
|
|
153
267
|
generateAnswer(conversation: Conversation): Promise<string>;
|
|
154
268
|
speech2Text(blob: Blob, prompt?: string): Promise<string>;
|
|
155
|
-
text2Speech(text: string): Promise<ArrayBuffer>;
|
|
269
|
+
text2Speech(text: string): Promise<ArrayBuffer | NodeJS.ReadableStream>;
|
|
156
270
|
onMessage?(message: ConversationMessage): void;
|
|
157
271
|
onEnd?(call: CallSummary): void;
|
|
158
272
|
}
|
|
@@ -166,4 +280,4 @@ interface ConversationMessage {
|
|
|
166
280
|
content: string;
|
|
167
281
|
}
|
|
168
282
|
|
|
169
|
-
export { CallClientCommands, type CallConfig, CallHandler, CallHandlerError, CallHandlerErrorCode, type CallHandlerEvents, CallServerCommands, type CallSummary, type Conversation, type ConversationMessage, LocalStorageKeys, microphone as Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, speaker as Speaker };
|
|
283
|
+
export { CallClientCommands, type CallConfig, CallHandler, CallHandlerError, CallHandlerErrorCode, type CallHandlerEvents, type CallHandlerOptions, CallServerCommands, type CallSummary, type Conversation, type ConversationMessage, LocalStorageKeys, microphone as Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, SileroVAD, speaker as Speaker, VAD, VolumeVAD };
|
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,46 @@ declare enum LocalStorageKeys {
|
|
|
6
6
|
MicThreshold = "micdrop.micThreshold"
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
interface VADEvents {
|
|
10
|
+
StartSpeaking: void;
|
|
11
|
+
ConfirmSpeaking: void;
|
|
12
|
+
CancelSpeaking: void;
|
|
13
|
+
StopSpeaking: void;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Abstract VAD class
|
|
17
|
+
* VAD = Voice Activity Detection
|
|
18
|
+
*/
|
|
19
|
+
declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
20
|
+
delay: number;
|
|
21
|
+
protected threshold: number;
|
|
22
|
+
/**
|
|
23
|
+
* Constructor for the VAD class
|
|
24
|
+
* @param threshold - The threshold to use for the VAD
|
|
25
|
+
*/
|
|
26
|
+
constructor(threshold?: number);
|
|
27
|
+
/**
|
|
28
|
+
* Checks if the VAD is started
|
|
29
|
+
* @returns True if the VAD is started, false otherwise
|
|
30
|
+
*/
|
|
31
|
+
abstract get isStarted(): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Starts the VAD
|
|
34
|
+
* @param stream - The stream to start the VAD on
|
|
35
|
+
* @param threshold - The threshold to use for the VAD
|
|
36
|
+
*/
|
|
37
|
+
abstract start(stream: MediaStream, threshold?: number): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Stops the VAD
|
|
40
|
+
*/
|
|
41
|
+
abstract stop(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Sets the threshold for the VAD
|
|
44
|
+
* @param threshold - The threshold to use for the VAD
|
|
45
|
+
*/
|
|
46
|
+
abstract setThreshold(threshold: number): Promise<void>;
|
|
47
|
+
}
|
|
48
|
+
|
|
9
49
|
interface MicRecorderState {
|
|
10
50
|
isStarting: boolean;
|
|
11
51
|
isStarted: boolean;
|
|
@@ -22,10 +62,12 @@ interface MicRecorderEvents {
|
|
|
22
62
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
23
63
|
state: MicRecorderState;
|
|
24
64
|
private audioInfo;
|
|
25
|
-
private
|
|
65
|
+
private vad;
|
|
26
66
|
private recorder;
|
|
27
67
|
private delayedStream;
|
|
28
|
-
|
|
68
|
+
private speakingConfirmed;
|
|
69
|
+
private queuedChunks;
|
|
70
|
+
constructor(vad: VAD);
|
|
29
71
|
start(stream: MediaStream): Promise<void>;
|
|
30
72
|
mute(): void;
|
|
31
73
|
unmute(): void;
|
|
@@ -34,6 +76,8 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
|
34
76
|
private startSpeakingDetection;
|
|
35
77
|
private stopSpeakingDetection;
|
|
36
78
|
private onStartSpeaking;
|
|
79
|
+
private onConfirmSpeaking;
|
|
80
|
+
private onCancelSpeaking;
|
|
37
81
|
private onStopSpeaking;
|
|
38
82
|
private onDataAvailable;
|
|
39
83
|
private changeState;
|
|
@@ -55,36 +99,102 @@ declare class AudioAnalyser extends EventEmitter<AudioAnalyserEvents> {
|
|
|
55
99
|
private getMaxVolume;
|
|
56
100
|
}
|
|
57
101
|
|
|
102
|
+
declare let audioContext: AudioContext | undefined;
|
|
58
103
|
declare let micAnalyser: AudioAnalyser | undefined;
|
|
59
104
|
declare const defaultMicThreshold = -50;
|
|
60
|
-
|
|
105
|
+
/**
|
|
106
|
+
* Starts the microphone
|
|
107
|
+
* @param deviceId - The deviceId to use
|
|
108
|
+
* @returns The microphone stream
|
|
109
|
+
*/
|
|
61
110
|
declare function startMicrophone(deviceId?: string): Promise<MediaStream>;
|
|
111
|
+
/**
|
|
112
|
+
* Stops the microphone
|
|
113
|
+
*/
|
|
62
114
|
declare function stopMicrophone(): void;
|
|
63
|
-
declare function createDelayedStream(stream: MediaStream): MediaStream;
|
|
64
115
|
|
|
65
|
-
declare const
|
|
116
|
+
declare const microphone_audioContext: typeof audioContext;
|
|
66
117
|
declare const microphone_defaultMicThreshold: typeof defaultMicThreshold;
|
|
67
|
-
declare const microphone_getMicConstraints: typeof getMicConstraints;
|
|
68
118
|
declare const microphone_micAnalyser: typeof micAnalyser;
|
|
69
119
|
declare const microphone_startMicrophone: typeof startMicrophone;
|
|
70
120
|
declare const microphone_stopMicrophone: typeof stopMicrophone;
|
|
71
121
|
declare namespace microphone {
|
|
72
|
-
export {
|
|
122
|
+
export { microphone_audioContext as audioContext, microphone_defaultMicThreshold as defaultMicThreshold, microphone_micAnalyser as micAnalyser, microphone_startMicrophone as startMicrophone, microphone_stopMicrophone as stopMicrophone };
|
|
73
123
|
}
|
|
74
124
|
|
|
75
125
|
declare let speakerAnalyser: AudioAnalyser | undefined;
|
|
126
|
+
/**
|
|
127
|
+
* Checks if the speaker device can be changed
|
|
128
|
+
* @returns True if the speaker device can be changed, false otherwise
|
|
129
|
+
*/
|
|
76
130
|
declare function canChangeSpeakerDevice(): boolean;
|
|
131
|
+
/**
|
|
132
|
+
* Changes the speaker device
|
|
133
|
+
* @param speakerId - The speakerId to use
|
|
134
|
+
*/
|
|
77
135
|
declare function changeSpeakerDevice(speakerId: string): Promise<void>;
|
|
78
|
-
|
|
79
|
-
|
|
136
|
+
/**
|
|
137
|
+
* Pauses the audio
|
|
138
|
+
*/
|
|
139
|
+
declare function pauseAudio(): void;
|
|
140
|
+
/**
|
|
141
|
+
* Resumes the audio
|
|
142
|
+
*/
|
|
143
|
+
declare function resumeAudio(): void;
|
|
144
|
+
/**
|
|
145
|
+
* Plays the audio
|
|
146
|
+
* @param blob - The blob to play (adds to the queue if already playing)
|
|
147
|
+
*/
|
|
148
|
+
declare function playAudio(blob: Blob): Promise<void>;
|
|
149
|
+
/**
|
|
150
|
+
* Stops the audio
|
|
151
|
+
*/
|
|
152
|
+
declare function stopAudio(): void;
|
|
80
153
|
|
|
81
154
|
declare const speaker_canChangeSpeakerDevice: typeof canChangeSpeakerDevice;
|
|
82
155
|
declare const speaker_changeSpeakerDevice: typeof changeSpeakerDevice;
|
|
83
|
-
declare const
|
|
156
|
+
declare const speaker_pauseAudio: typeof pauseAudio;
|
|
157
|
+
declare const speaker_playAudio: typeof playAudio;
|
|
158
|
+
declare const speaker_resumeAudio: typeof resumeAudio;
|
|
84
159
|
declare const speaker_speakerAnalyser: typeof speakerAnalyser;
|
|
85
|
-
declare const
|
|
160
|
+
declare const speaker_stopAudio: typeof stopAudio;
|
|
86
161
|
declare namespace speaker {
|
|
87
|
-
export { speaker_canChangeSpeakerDevice as canChangeSpeakerDevice, speaker_changeSpeakerDevice as changeSpeakerDevice,
|
|
162
|
+
export { speaker_canChangeSpeakerDevice as canChangeSpeakerDevice, speaker_changeSpeakerDevice as changeSpeakerDevice, speaker_pauseAudio as pauseAudio, speaker_playAudio as playAudio, speaker_resumeAudio as resumeAudio, speaker_speakerAnalyser as speakerAnalyser, speaker_stopAudio as stopAudio };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Silero VAD
|
|
167
|
+
* @see https://github.com/ricky0123/vad
|
|
168
|
+
*/
|
|
169
|
+
declare class SileroVAD extends VAD {
|
|
170
|
+
private vad;
|
|
171
|
+
get isStarted(): boolean;
|
|
172
|
+
start(stream: MediaStream, threshold?: number): Promise<void>;
|
|
173
|
+
stop(): Promise<void>;
|
|
174
|
+
setThreshold(threshold: number): Promise<void>;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Volume-based Voice Activity Detection
|
|
179
|
+
* Based on the hark library implementation
|
|
180
|
+
*/
|
|
181
|
+
declare class VolumeVAD extends VAD {
|
|
182
|
+
private audioContext;
|
|
183
|
+
private analyser;
|
|
184
|
+
private sourceNode;
|
|
185
|
+
private fftBins;
|
|
186
|
+
private running;
|
|
187
|
+
private speaking;
|
|
188
|
+
private attemptSpeaking;
|
|
189
|
+
private history;
|
|
190
|
+
private speakingHistory;
|
|
191
|
+
constructor();
|
|
192
|
+
private getMaxVolume;
|
|
193
|
+
get isStarted(): boolean;
|
|
194
|
+
start(stream: MediaStream, threshold?: number): Promise<void>;
|
|
195
|
+
private startLoop;
|
|
196
|
+
stop(): Promise<void>;
|
|
197
|
+
setThreshold(threshold: number): Promise<void>;
|
|
88
198
|
}
|
|
89
199
|
|
|
90
200
|
declare enum CallHandlerErrorCode {
|
|
@@ -102,9 +212,12 @@ interface CallHandlerEvents {
|
|
|
102
212
|
Error: [CallHandlerError];
|
|
103
213
|
StateChange: void;
|
|
104
214
|
}
|
|
105
|
-
|
|
215
|
+
interface CallHandlerOptions {
|
|
216
|
+
vad?: VAD | 'volume' | 'silero';
|
|
217
|
+
}
|
|
218
|
+
declare class CallHandler<Params extends {}, Options extends CallHandlerOptions = CallHandlerOptions> extends EventEmitter<CallHandlerEvents> {
|
|
106
219
|
private static instance;
|
|
107
|
-
static getInstance<T extends {}>(): CallHandler<T>;
|
|
220
|
+
static getInstance<T extends {}, Options extends CallHandlerOptions = CallHandlerOptions>(options?: Options): CallHandler<T, Options>;
|
|
108
221
|
url?: string;
|
|
109
222
|
params?: Params;
|
|
110
223
|
micRecorder: MicRecorder;
|
|
@@ -115,6 +228,7 @@ declare class CallHandler<Params extends {}> extends EventEmitter<CallHandlerEve
|
|
|
115
228
|
private startTime;
|
|
116
229
|
private lastAudioBlob?;
|
|
117
230
|
private constructor();
|
|
231
|
+
private getVAD;
|
|
118
232
|
get isStarted(): boolean;
|
|
119
233
|
get isStarting(): boolean;
|
|
120
234
|
get isWSStarted(): boolean;
|
|
@@ -152,7 +266,7 @@ interface CallConfig {
|
|
|
152
266
|
disableTTS?: boolean;
|
|
153
267
|
generateAnswer(conversation: Conversation): Promise<string>;
|
|
154
268
|
speech2Text(blob: Blob, prompt?: string): Promise<string>;
|
|
155
|
-
text2Speech(text: string): Promise<ArrayBuffer>;
|
|
269
|
+
text2Speech(text: string): Promise<ArrayBuffer | NodeJS.ReadableStream>;
|
|
156
270
|
onMessage?(message: ConversationMessage): void;
|
|
157
271
|
onEnd?(call: CallSummary): void;
|
|
158
272
|
}
|
|
@@ -166,4 +280,4 @@ interface ConversationMessage {
|
|
|
166
280
|
content: string;
|
|
167
281
|
}
|
|
168
282
|
|
|
169
|
-
export { CallClientCommands, type CallConfig, CallHandler, CallHandlerError, CallHandlerErrorCode, type CallHandlerEvents, CallServerCommands, type CallSummary, type Conversation, type ConversationMessage, LocalStorageKeys, microphone as Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, speaker as Speaker };
|
|
283
|
+
export { CallClientCommands, type CallConfig, CallHandler, CallHandlerError, CallHandlerErrorCode, type CallHandlerEvents, type CallHandlerOptions, CallServerCommands, type CallSummary, type Conversation, type ConversationMessage, LocalStorageKeys, microphone as Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, SileroVAD, speaker as Speaker, VAD, VolumeVAD };
|