@micdrop/client 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +32 -8
- package/dist/index.d.ts +32 -8
- package/dist/index.js +129 -59
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +128 -59
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -79,6 +79,11 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
79
79
|
* @returns True if the VAD is started, false otherwise
|
|
80
80
|
*/
|
|
81
81
|
abstract get isStarted(): boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Checks if the VAD is paused
|
|
84
|
+
* @returns True if the VAD is paused, false otherwise
|
|
85
|
+
*/
|
|
86
|
+
abstract get isPaused(): boolean;
|
|
82
87
|
/**
|
|
83
88
|
* Starts the VAD
|
|
84
89
|
* @param stream - The stream to start the VAD on
|
|
@@ -88,6 +93,14 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
88
93
|
* Stops the VAD
|
|
89
94
|
*/
|
|
90
95
|
abstract stop(): Promise<void>;
|
|
96
|
+
/**
|
|
97
|
+
* Pauses the VAD
|
|
98
|
+
*/
|
|
99
|
+
abstract pause(): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Resumes the VAD
|
|
102
|
+
*/
|
|
103
|
+
abstract resume(): Promise<void>;
|
|
91
104
|
/**
|
|
92
105
|
* Emits an event (overrides the default implementation)
|
|
93
106
|
* @param event - The event to emit
|
|
@@ -105,11 +118,11 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
105
118
|
type VADConfigName = 'volume' | 'silero';
|
|
106
119
|
type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
|
|
107
120
|
declare function getVAD(vad?: VADConfig): VAD;
|
|
121
|
+
declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
|
|
108
122
|
|
|
109
123
|
interface MicRecorderState {
|
|
110
124
|
isStarting: boolean;
|
|
111
125
|
isStarted: boolean;
|
|
112
|
-
isMuted: boolean;
|
|
113
126
|
isSpeaking: boolean;
|
|
114
127
|
}
|
|
115
128
|
interface MicRecorderEvents {
|
|
@@ -119,19 +132,19 @@ interface MicRecorderEvents {
|
|
|
119
132
|
StateChange: [MicRecorderState];
|
|
120
133
|
}
|
|
121
134
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
135
|
+
private vadConfig?;
|
|
122
136
|
state: MicRecorderState;
|
|
123
137
|
vad: VAD;
|
|
138
|
+
private stream;
|
|
124
139
|
private source;
|
|
125
140
|
private workletNode;
|
|
126
141
|
private delayedStream;
|
|
127
|
-
private workletUrl;
|
|
128
142
|
private speakingConfirmed;
|
|
129
143
|
private queuedChunks;
|
|
130
|
-
constructor(vadConfig?: VADConfig);
|
|
144
|
+
constructor(vadConfig?: VADConfig | undefined);
|
|
131
145
|
start: (stream: MediaStream) => Promise<void>;
|
|
132
|
-
mute: () => void;
|
|
133
|
-
unmute: () => void;
|
|
134
146
|
stop: () => void;
|
|
147
|
+
changeVad: (vadConfig: VADConfig) => void;
|
|
135
148
|
private onStartSpeaking;
|
|
136
149
|
private onConfirmSpeaking;
|
|
137
150
|
private onCancelSpeaking;
|
|
@@ -154,9 +167,12 @@ declare class MultipleVAD extends VAD {
|
|
|
154
167
|
vads: VAD[];
|
|
155
168
|
constructor(vads: Array<VAD | VADConfigName>);
|
|
156
169
|
get isStarted(): boolean;
|
|
157
|
-
|
|
170
|
+
get isPaused(): boolean;
|
|
171
|
+
start(stream: MediaStream): Promise<void>;
|
|
158
172
|
private onStatusChange;
|
|
159
173
|
stop(): Promise<void>;
|
|
174
|
+
pause(): Promise<void>;
|
|
175
|
+
resume(): Promise<void>;
|
|
160
176
|
}
|
|
161
177
|
|
|
162
178
|
type SileroVADOptions = {
|
|
@@ -172,10 +188,14 @@ type SileroVADOptions = {
|
|
|
172
188
|
declare class SileroVAD extends VAD {
|
|
173
189
|
options: SileroVADOptions;
|
|
174
190
|
private vad;
|
|
191
|
+
private _isPaused;
|
|
175
192
|
constructor(options?: Partial<SileroVADOptions>);
|
|
176
193
|
get isStarted(): boolean;
|
|
194
|
+
get isPaused(): boolean;
|
|
177
195
|
start(stream: MediaStream): Promise<void>;
|
|
178
196
|
stop(): Promise<void>;
|
|
197
|
+
pause(): Promise<void>;
|
|
198
|
+
resume(): Promise<void>;
|
|
179
199
|
setOptions(options: Partial<SileroVADOptions>): void;
|
|
180
200
|
resetOptions(): void;
|
|
181
201
|
}
|
|
@@ -190,6 +210,7 @@ type VolumeVADOptions = {
|
|
|
190
210
|
*/
|
|
191
211
|
declare class VolumeVAD extends VAD {
|
|
192
212
|
options: VolumeVADOptions;
|
|
213
|
+
private _isPaused;
|
|
193
214
|
private analyser;
|
|
194
215
|
private sourceNode;
|
|
195
216
|
private fftBins;
|
|
@@ -200,9 +221,13 @@ declare class VolumeVAD extends VAD {
|
|
|
200
221
|
constructor(options?: Partial<VolumeVADOptions>);
|
|
201
222
|
private getMaxVolume;
|
|
202
223
|
get isStarted(): boolean;
|
|
224
|
+
get isPaused(): boolean;
|
|
203
225
|
start(stream: MediaStream): Promise<void>;
|
|
204
226
|
private startLoop;
|
|
227
|
+
private resetHistory;
|
|
205
228
|
stop(): Promise<void>;
|
|
229
|
+
pause(): Promise<void>;
|
|
230
|
+
resume(): Promise<void>;
|
|
206
231
|
setOptions(options: Partial<VolumeVADOptions>): void;
|
|
207
232
|
resetOptions(): void;
|
|
208
233
|
}
|
|
@@ -345,7 +370,6 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
345
370
|
startMic: (options?: {
|
|
346
371
|
vad?: VADConfig;
|
|
347
372
|
deviceId?: string;
|
|
348
|
-
record?: boolean;
|
|
349
373
|
}) => Promise<void>;
|
|
350
374
|
changeMicDevice: (deviceId: string) => Promise<void>;
|
|
351
375
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
@@ -367,4 +391,4 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
367
391
|
|
|
368
392
|
declare const Micdrop: MicdropClient;
|
|
369
393
|
|
|
370
|
-
export { type DeepPartial, LocalStorageKeys, Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropEvents, type MicdropOptions, MicdropServerCommands, type MicdropState, type MicdropToolCall, MultipleVAD, SileroVAD, type SileroVADOptions, Speaker, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD };
|
|
394
|
+
export { type DeepPartial, LocalStorageKeys, Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropEvents, type MicdropOptions, MicdropServerCommands, type MicdropState, type MicdropToolCall, MultipleVAD, SileroVAD, type SileroVADOptions, Speaker, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions, equalVADConfig, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD };
|
package/dist/index.d.ts
CHANGED
|
@@ -79,6 +79,11 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
79
79
|
* @returns True if the VAD is started, false otherwise
|
|
80
80
|
*/
|
|
81
81
|
abstract get isStarted(): boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Checks if the VAD is paused
|
|
84
|
+
* @returns True if the VAD is paused, false otherwise
|
|
85
|
+
*/
|
|
86
|
+
abstract get isPaused(): boolean;
|
|
82
87
|
/**
|
|
83
88
|
* Starts the VAD
|
|
84
89
|
* @param stream - The stream to start the VAD on
|
|
@@ -88,6 +93,14 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
88
93
|
* Stops the VAD
|
|
89
94
|
*/
|
|
90
95
|
abstract stop(): Promise<void>;
|
|
96
|
+
/**
|
|
97
|
+
* Pauses the VAD
|
|
98
|
+
*/
|
|
99
|
+
abstract pause(): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Resumes the VAD
|
|
102
|
+
*/
|
|
103
|
+
abstract resume(): Promise<void>;
|
|
91
104
|
/**
|
|
92
105
|
* Emits an event (overrides the default implementation)
|
|
93
106
|
* @param event - The event to emit
|
|
@@ -105,11 +118,11 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
|
|
|
105
118
|
type VADConfigName = 'volume' | 'silero';
|
|
106
119
|
type VADConfig = VAD | VADConfigName | Array<VAD | VADConfigName>;
|
|
107
120
|
declare function getVAD(vad?: VADConfig): VAD;
|
|
121
|
+
declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
|
|
108
122
|
|
|
109
123
|
interface MicRecorderState {
|
|
110
124
|
isStarting: boolean;
|
|
111
125
|
isStarted: boolean;
|
|
112
|
-
isMuted: boolean;
|
|
113
126
|
isSpeaking: boolean;
|
|
114
127
|
}
|
|
115
128
|
interface MicRecorderEvents {
|
|
@@ -119,19 +132,19 @@ interface MicRecorderEvents {
|
|
|
119
132
|
StateChange: [MicRecorderState];
|
|
120
133
|
}
|
|
121
134
|
declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
|
|
135
|
+
private vadConfig?;
|
|
122
136
|
state: MicRecorderState;
|
|
123
137
|
vad: VAD;
|
|
138
|
+
private stream;
|
|
124
139
|
private source;
|
|
125
140
|
private workletNode;
|
|
126
141
|
private delayedStream;
|
|
127
|
-
private workletUrl;
|
|
128
142
|
private speakingConfirmed;
|
|
129
143
|
private queuedChunks;
|
|
130
|
-
constructor(vadConfig?: VADConfig);
|
|
144
|
+
constructor(vadConfig?: VADConfig | undefined);
|
|
131
145
|
start: (stream: MediaStream) => Promise<void>;
|
|
132
|
-
mute: () => void;
|
|
133
|
-
unmute: () => void;
|
|
134
146
|
stop: () => void;
|
|
147
|
+
changeVad: (vadConfig: VADConfig) => void;
|
|
135
148
|
private onStartSpeaking;
|
|
136
149
|
private onConfirmSpeaking;
|
|
137
150
|
private onCancelSpeaking;
|
|
@@ -154,9 +167,12 @@ declare class MultipleVAD extends VAD {
|
|
|
154
167
|
vads: VAD[];
|
|
155
168
|
constructor(vads: Array<VAD | VADConfigName>);
|
|
156
169
|
get isStarted(): boolean;
|
|
157
|
-
|
|
170
|
+
get isPaused(): boolean;
|
|
171
|
+
start(stream: MediaStream): Promise<void>;
|
|
158
172
|
private onStatusChange;
|
|
159
173
|
stop(): Promise<void>;
|
|
174
|
+
pause(): Promise<void>;
|
|
175
|
+
resume(): Promise<void>;
|
|
160
176
|
}
|
|
161
177
|
|
|
162
178
|
type SileroVADOptions = {
|
|
@@ -172,10 +188,14 @@ type SileroVADOptions = {
|
|
|
172
188
|
declare class SileroVAD extends VAD {
|
|
173
189
|
options: SileroVADOptions;
|
|
174
190
|
private vad;
|
|
191
|
+
private _isPaused;
|
|
175
192
|
constructor(options?: Partial<SileroVADOptions>);
|
|
176
193
|
get isStarted(): boolean;
|
|
194
|
+
get isPaused(): boolean;
|
|
177
195
|
start(stream: MediaStream): Promise<void>;
|
|
178
196
|
stop(): Promise<void>;
|
|
197
|
+
pause(): Promise<void>;
|
|
198
|
+
resume(): Promise<void>;
|
|
179
199
|
setOptions(options: Partial<SileroVADOptions>): void;
|
|
180
200
|
resetOptions(): void;
|
|
181
201
|
}
|
|
@@ -190,6 +210,7 @@ type VolumeVADOptions = {
|
|
|
190
210
|
*/
|
|
191
211
|
declare class VolumeVAD extends VAD {
|
|
192
212
|
options: VolumeVADOptions;
|
|
213
|
+
private _isPaused;
|
|
193
214
|
private analyser;
|
|
194
215
|
private sourceNode;
|
|
195
216
|
private fftBins;
|
|
@@ -200,9 +221,13 @@ declare class VolumeVAD extends VAD {
|
|
|
200
221
|
constructor(options?: Partial<VolumeVADOptions>);
|
|
201
222
|
private getMaxVolume;
|
|
202
223
|
get isStarted(): boolean;
|
|
224
|
+
get isPaused(): boolean;
|
|
203
225
|
start(stream: MediaStream): Promise<void>;
|
|
204
226
|
private startLoop;
|
|
227
|
+
private resetHistory;
|
|
205
228
|
stop(): Promise<void>;
|
|
229
|
+
pause(): Promise<void>;
|
|
230
|
+
resume(): Promise<void>;
|
|
206
231
|
setOptions(options: Partial<VolumeVADOptions>): void;
|
|
207
232
|
resetOptions(): void;
|
|
208
233
|
}
|
|
@@ -345,7 +370,6 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
345
370
|
startMic: (options?: {
|
|
346
371
|
vad?: VADConfig;
|
|
347
372
|
deviceId?: string;
|
|
348
|
-
record?: boolean;
|
|
349
373
|
}) => Promise<void>;
|
|
350
374
|
changeMicDevice: (deviceId: string) => Promise<void>;
|
|
351
375
|
changeSpeakerDevice: (deviceId: string) => Promise<void>;
|
|
@@ -367,4 +391,4 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
|
|
|
367
391
|
|
|
368
392
|
declare const Micdrop: MicdropClient;
|
|
369
393
|
|
|
370
|
-
export { type DeepPartial, LocalStorageKeys, Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropEvents, type MicdropOptions, MicdropServerCommands, type MicdropState, type MicdropToolCall, MultipleVAD, SileroVAD, type SileroVADOptions, Speaker, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD };
|
|
394
|
+
export { type DeepPartial, LocalStorageKeys, Mic, MicRecorder, type MicRecorderEvents, type MicRecorderState, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropEvents, type MicdropOptions, MicdropServerCommands, type MicdropState, type MicdropToolCall, MultipleVAD, SileroVAD, type SileroVADOptions, Speaker, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VolumeVAD, type VolumeVADOptions, equalVADConfig, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD };
|
package/dist/index.js
CHANGED
|
@@ -35,6 +35,7 @@ __export(index_exports, {
|
|
|
35
35
|
VAD: () => VAD,
|
|
36
36
|
VADStatus: () => VADStatus,
|
|
37
37
|
VolumeVAD: () => VolumeVAD,
|
|
38
|
+
equalVADConfig: () => equalVADConfig,
|
|
38
39
|
getClientErrorFromWSCloseEvent: () => getClientErrorFromWSCloseEvent,
|
|
39
40
|
getClientErrorFromWSCloseEventCode: () => getClientErrorFromWSCloseEventCode,
|
|
40
41
|
getVAD: () => getVAD
|
|
@@ -387,9 +388,6 @@ var Speaker = class extends import_eventemitter33.EventEmitter {
|
|
|
387
388
|
// src/audio/MicRecorder.ts
|
|
388
389
|
var import_eventemitter35 = require("eventemitter3");
|
|
389
390
|
|
|
390
|
-
// raw:./pcm-processor-worklet
|
|
391
|
-
var pcm_processor_worklet_default = "const INTERVAL_MS = 100\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.audioBuffer = []\n this.isRecording = false\n this.sourceSampleRate = 16000\n this.targetSampleRate = 16000\n this.resamplingRatio = 1\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n\n // Listen for messages\n this.port.onmessage = this.handleMessage.bind(this)\n }\n\n handleMessage(event) {\n if (event.data.type === 'configure') {\n this.sourceSampleRate = event.data.sourceSampleRate\n this.targetSampleRate = event.data.targetSampleRate\n this.resamplingRatio = this.sourceSampleRate / this.targetSampleRate\n // Recalculate chunk interval for the source sample rate\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n } else if (event.data.type === 'start') {\n this.isRecording = true\n } else if (event.data.type === 'stop') {\n this.isRecording = false\n // Clear any remaining audio buffer when stopping\n this.audioBuffer.length = 0\n }\n }\n\n process(inputs) {\n const input = inputs[0]\n\n if (input && input.length > 0 && this.isRecording) {\n const inputChannel = input[0]\n\n // Store audio data in buffer\n this.audioBuffer.push(new Float32Array(inputChannel))\n\n // Check if we have enough data for a chunk\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n\n if (totalSamples >= this.CHUNK_INTERVAL_SAMPLES) {\n this.processChunk()\n }\n }\n\n return true\n }\n\n processChunk() {\n if (this.audioBuffer.length === 0) return\n\n // Calculate total samples\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n if (totalSamples === 0) return\n\n // Merge all buffers\n const mergedBuffer = new Float32Array(totalSamples)\n let offset = 0\n for (const buffer of this.audioBuffer) {\n mergedBuffer.set(buffer, offset)\n offset += buffer.length\n }\n\n // Resample to target sample rate if necessary\n const resampledBuffer = this.resample(mergedBuffer)\n\n // Convert float32 to int16 PCM\n const pcmData = this.floatToPCM16(resampledBuffer)\n\n // Clear the buffer\n this.audioBuffer.length = 0\n\n // Send PCM chunk to main thread\n this.port.postMessage({\n type: 'chunk',\n data: pcmData,\n })\n }\n\n resample(inputBuffer) {\n // If no resampling needed, return original buffer\n if (this.resamplingRatio === 1) {\n return inputBuffer\n }\n\n // Calculate output length\n const outputLength = Math.round(inputBuffer.length / this.resamplingRatio)\n const outputBuffer = new Float32Array(outputLength)\n\n // Simple linear interpolation resampling\n for (let i = 0; i < outputLength; i++) {\n const srcIndex = i * this.resamplingRatio\n const srcIndexFloor = Math.floor(srcIndex)\n const srcIndexCeil = Math.min(srcIndexFloor + 1, inputBuffer.length - 1)\n const fraction = srcIndex - srcIndexFloor\n\n // Linear interpolation\n outputBuffer[i] =\n inputBuffer[srcIndexFloor] * (1 - fraction) +\n inputBuffer[srcIndexCeil] * fraction\n }\n\n return outputBuffer\n }\n\n floatToPCM16(floatData) {\n const pcmData = new Int16Array(floatData.length)\n for (let i = 0; i < floatData.length; i++) {\n const sample = Math.max(-1, Math.min(1, floatData[i]))\n pcmData[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff\n }\n return pcmData\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
|
|
392
|
-
|
|
393
391
|
// src/audio/utils/delayedStream.ts
|
|
394
392
|
function createDelayedStream(stream, delay) {
|
|
395
393
|
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
@@ -493,7 +491,10 @@ var MultipleVAD = class extends VAD {
|
|
|
493
491
|
get isStarted() {
|
|
494
492
|
return this.vads.some((vad) => vad.isStarted);
|
|
495
493
|
}
|
|
496
|
-
|
|
494
|
+
get isPaused() {
|
|
495
|
+
return this.vads.every((vad) => vad.isPaused);
|
|
496
|
+
}
|
|
497
|
+
async start(stream) {
|
|
497
498
|
for (const vad of this.vads) {
|
|
498
499
|
if (vad.isStarted) {
|
|
499
500
|
continue;
|
|
@@ -508,6 +509,16 @@ var MultipleVAD = class extends VAD {
|
|
|
508
509
|
await vad.stop();
|
|
509
510
|
}
|
|
510
511
|
}
|
|
512
|
+
async pause() {
|
|
513
|
+
for (const vad of this.vads) {
|
|
514
|
+
await vad.pause();
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
async resume() {
|
|
518
|
+
for (const vad of this.vads) {
|
|
519
|
+
await vad.resume();
|
|
520
|
+
}
|
|
521
|
+
}
|
|
511
522
|
};
|
|
512
523
|
|
|
513
524
|
// src/audio/vad/SileroVAD.ts
|
|
@@ -522,6 +533,7 @@ var SileroVAD = class extends VAD {
|
|
|
522
533
|
constructor(options) {
|
|
523
534
|
super();
|
|
524
535
|
this.options = defaultOptions;
|
|
536
|
+
this._isPaused = false;
|
|
525
537
|
if (options) {
|
|
526
538
|
this.setOptions(options);
|
|
527
539
|
} else {
|
|
@@ -538,6 +550,9 @@ var SileroVAD = class extends VAD {
|
|
|
538
550
|
get isStarted() {
|
|
539
551
|
return !!this.vad;
|
|
540
552
|
}
|
|
553
|
+
get isPaused() {
|
|
554
|
+
return this._isPaused;
|
|
555
|
+
}
|
|
541
556
|
async start(stream) {
|
|
542
557
|
if (this.vad) return;
|
|
543
558
|
this.vad = await import_vad_web.MicVAD.new({
|
|
@@ -557,6 +572,16 @@ var SileroVAD = class extends VAD {
|
|
|
557
572
|
this.vad.destroy();
|
|
558
573
|
this.vad = void 0;
|
|
559
574
|
}
|
|
575
|
+
async pause() {
|
|
576
|
+
if (!this.vad || this._isPaused) return;
|
|
577
|
+
this._isPaused = true;
|
|
578
|
+
this.vad.pause();
|
|
579
|
+
}
|
|
580
|
+
async resume() {
|
|
581
|
+
if (!this.vad || !this._isPaused) return;
|
|
582
|
+
this._isPaused = false;
|
|
583
|
+
this.vad.start();
|
|
584
|
+
}
|
|
560
585
|
setOptions(options) {
|
|
561
586
|
this.options = { ...this.options, ...options };
|
|
562
587
|
this.vad?.setOptions(this.options);
|
|
@@ -579,6 +604,7 @@ var VolumeVAD = class extends VAD {
|
|
|
579
604
|
constructor(options) {
|
|
580
605
|
super();
|
|
581
606
|
this.options = defaultOptions2;
|
|
607
|
+
this._isPaused = false;
|
|
582
608
|
this.running = false;
|
|
583
609
|
this.speaking = false;
|
|
584
610
|
this.attemptSpeaking = false;
|
|
@@ -610,8 +636,11 @@ var VolumeVAD = class extends VAD {
|
|
|
610
636
|
get isStarted() {
|
|
611
637
|
return this.running;
|
|
612
638
|
}
|
|
639
|
+
get isPaused() {
|
|
640
|
+
return this._isPaused;
|
|
641
|
+
}
|
|
613
642
|
async start(stream) {
|
|
614
|
-
if (this.running) return;
|
|
643
|
+
if (this.running || this._isPaused) return;
|
|
615
644
|
this.analyser = audioContext.createAnalyser();
|
|
616
645
|
this.sourceNode = audioContext.createMediaStreamSource(stream);
|
|
617
646
|
this.analyser.fftSize = 512;
|
|
@@ -659,10 +688,25 @@ var VolumeVAD = class extends VAD {
|
|
|
659
688
|
this.speakingHistory.push(Number(currentVolume > this.options.threshold));
|
|
660
689
|
setTimeout(() => this.startLoop(), this.delay);
|
|
661
690
|
}
|
|
691
|
+
resetHistory() {
|
|
692
|
+
this.speaking = false;
|
|
693
|
+
this.attemptSpeaking = false;
|
|
694
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
695
|
+
this.speakingHistory[i] = 0;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
662
698
|
async stop() {
|
|
663
699
|
if (!this.running) return;
|
|
664
700
|
this.running = false;
|
|
665
|
-
this.
|
|
701
|
+
this.resetHistory();
|
|
702
|
+
switch (this.status) {
|
|
703
|
+
case 2 /* Speaking */:
|
|
704
|
+
this.emit("StopSpeaking");
|
|
705
|
+
break;
|
|
706
|
+
case 1 /* MaybeSpeaking */:
|
|
707
|
+
this.emit("CancelSpeaking");
|
|
708
|
+
break;
|
|
709
|
+
}
|
|
666
710
|
if (this.sourceNode) {
|
|
667
711
|
this.sourceNode.disconnect();
|
|
668
712
|
this.sourceNode = void 0;
|
|
@@ -673,6 +717,26 @@ var VolumeVAD = class extends VAD {
|
|
|
673
717
|
}
|
|
674
718
|
this.fftBins = void 0;
|
|
675
719
|
}
|
|
720
|
+
async pause() {
|
|
721
|
+
if (!this.running || this._isPaused) return;
|
|
722
|
+
this._isPaused = true;
|
|
723
|
+
this.running = false;
|
|
724
|
+
this.resetHistory();
|
|
725
|
+
switch (this.status) {
|
|
726
|
+
case 2 /* Speaking */:
|
|
727
|
+
this.emit("StopSpeaking");
|
|
728
|
+
break;
|
|
729
|
+
case 1 /* MaybeSpeaking */:
|
|
730
|
+
this.emit("CancelSpeaking");
|
|
731
|
+
break;
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
async resume() {
|
|
735
|
+
if (!this._isPaused) return;
|
|
736
|
+
this._isPaused = false;
|
|
737
|
+
this.running = true;
|
|
738
|
+
this.startLoop();
|
|
739
|
+
}
|
|
676
740
|
setOptions(options) {
|
|
677
741
|
if (typeof options.history === "number" && options.history < 3) {
|
|
678
742
|
throw new Error("History must be at least 3");
|
|
@@ -705,18 +769,39 @@ function getVAD(vad) {
|
|
|
705
769
|
}
|
|
706
770
|
return vad;
|
|
707
771
|
}
|
|
772
|
+
function equalVADConfig(a, b) {
|
|
773
|
+
if (a === b) return true;
|
|
774
|
+
if (Array.isArray(a) && Array.isArray(b) && a.length === b.length) {
|
|
775
|
+
return a.every((item, index) => equalVADConfig(item, b[index]));
|
|
776
|
+
}
|
|
777
|
+
return false;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
// raw:./pcm-processor-worklet
|
|
781
|
+
var pcm_processor_worklet_default = "const INTERVAL_MS = 100\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.audioBuffer = []\n this.isRecording = false\n this.sourceSampleRate = 16000\n this.targetSampleRate = 16000\n this.resamplingRatio = 1\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n\n // Listen for messages\n this.port.onmessage = this.handleMessage.bind(this)\n }\n\n handleMessage(event) {\n if (event.data.type === 'configure') {\n this.sourceSampleRate = event.data.sourceSampleRate\n this.targetSampleRate = event.data.targetSampleRate\n this.resamplingRatio = this.sourceSampleRate / this.targetSampleRate\n // Recalculate chunk interval for the source sample rate\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n } else if (event.data.type === 'start') {\n this.isRecording = true\n } else if (event.data.type === 'stop') {\n this.isRecording = false\n // Clear any remaining audio buffer when stopping\n this.audioBuffer.length = 0\n }\n }\n\n process(inputs) {\n const input = inputs[0]\n\n if (input && input.length > 0 && this.isRecording) {\n const inputChannel = input[0]\n\n // Store audio data in buffer\n this.audioBuffer.push(new Float32Array(inputChannel))\n\n // Check if we have enough data for a chunk\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n\n if (totalSamples >= this.CHUNK_INTERVAL_SAMPLES) {\n this.processChunk()\n }\n }\n\n return true\n }\n\n processChunk() {\n if (this.audioBuffer.length === 0) return\n\n // Calculate total samples\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n if (totalSamples === 0) return\n\n // Merge all buffers\n const mergedBuffer = new Float32Array(totalSamples)\n let offset = 0\n for (const buffer of this.audioBuffer) {\n mergedBuffer.set(buffer, offset)\n offset += buffer.length\n }\n\n // Resample to target sample rate if necessary\n const resampledBuffer = this.resample(mergedBuffer)\n\n // Convert float32 to int16 PCM\n const pcmData = this.floatToPCM16(resampledBuffer)\n\n // Clear the buffer\n this.audioBuffer.length = 0\n\n // Send PCM chunk to main thread\n this.port.postMessage({\n type: 'chunk',\n data: pcmData,\n })\n }\n\n resample(inputBuffer) {\n // If no resampling needed, return original buffer\n if (this.resamplingRatio === 1) {\n return inputBuffer\n }\n\n // Calculate output length\n const outputLength = Math.round(inputBuffer.length / this.resamplingRatio)\n const outputBuffer = new Float32Array(outputLength)\n\n // Simple linear interpolation resampling\n for (let i = 0; i < outputLength; i++) {\n const srcIndex = i * this.resamplingRatio\n const srcIndexFloor = Math.floor(srcIndex)\n const srcIndexCeil = Math.min(srcIndexFloor + 1, inputBuffer.length - 1)\n const fraction = srcIndex - srcIndexFloor\n\n // Linear interpolation\n outputBuffer[i] =\n inputBuffer[srcIndexFloor] * (1 - fraction) +\n inputBuffer[srcIndexCeil] * fraction\n }\n\n return outputBuffer\n }\n\n floatToPCM16(floatData) {\n const pcmData = new Int16Array(floatData.length)\n for (let i = 0; i < floatData.length; i++) {\n const sample = Math.max(-1, Math.min(1, floatData[i]))\n pcmData[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff\n }\n return pcmData\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
|
|
782
|
+
|
|
783
|
+
// src/audio/pcm-processor.ts
|
|
784
|
+
async function initPcmProcessor() {
|
|
785
|
+
if (window.micdropPcmProcessor) return;
|
|
786
|
+
window.micdropPcmProcessor = true;
|
|
787
|
+
const workletBlob = new Blob([pcm_processor_worklet_default], {
|
|
788
|
+
type: "application/javascript"
|
|
789
|
+
});
|
|
790
|
+
const workletUrl = URL.createObjectURL(workletBlob);
|
|
791
|
+
await audioContext.audioWorklet.addModule(workletUrl);
|
|
792
|
+
}
|
|
708
793
|
|
|
709
794
|
// src/audio/MicRecorder.ts
|
|
710
795
|
var SAMPLE_RATE2 = 16e3;
|
|
711
796
|
var defaultMicRecorderState = {
|
|
712
797
|
isStarting: false,
|
|
713
798
|
isStarted: false,
|
|
714
|
-
isMuted: false,
|
|
715
799
|
isSpeaking: false
|
|
716
800
|
};
|
|
717
801
|
var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
718
802
|
constructor(vadConfig) {
|
|
719
803
|
super();
|
|
804
|
+
this.vadConfig = vadConfig;
|
|
720
805
|
this.speakingConfirmed = false;
|
|
721
806
|
this.queuedChunks = [];
|
|
722
807
|
this.start = async (stream) => {
|
|
@@ -726,25 +811,22 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
726
811
|
try {
|
|
727
812
|
this.changeState({
|
|
728
813
|
isStarting: true,
|
|
729
|
-
isMuted: false,
|
|
730
814
|
isSpeaking: false
|
|
731
815
|
});
|
|
816
|
+
this.stream = stream;
|
|
732
817
|
this.speakingConfirmed = false;
|
|
733
|
-
|
|
734
|
-
stream,
|
|
735
|
-
this.vad.delay / 1e3 + 0.05
|
|
736
|
-
);
|
|
737
|
-
this.delayedStream = delayedStream;
|
|
738
|
-
await audioContext.audioWorklet.addModule(this.workletUrl);
|
|
739
|
-
this.source = audioContext.createMediaStreamSource(delayedStream);
|
|
818
|
+
await initPcmProcessor();
|
|
740
819
|
this.workletNode = new AudioWorkletNode(audioContext, "pcm-processor");
|
|
741
820
|
this.workletNode.port.onmessage = this.onWorkletMessage;
|
|
742
|
-
this.source.connect(this.workletNode);
|
|
743
821
|
this.workletNode.port.postMessage({
|
|
744
822
|
type: "configure",
|
|
745
823
|
sourceSampleRate: audioContext.sampleRate,
|
|
746
824
|
targetSampleRate: SAMPLE_RATE2
|
|
747
825
|
});
|
|
826
|
+
const delayedStream = createDelayedStream(stream, this.vad.delay / 1e3);
|
|
827
|
+
this.delayedStream = delayedStream;
|
|
828
|
+
this.source = audioContext.createMediaStreamSource(delayedStream);
|
|
829
|
+
this.source.connect(this.workletNode);
|
|
748
830
|
await this.vad.start(stream);
|
|
749
831
|
this.vad.on("StartSpeaking", this.onStartSpeaking);
|
|
750
832
|
this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
@@ -759,30 +841,13 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
759
841
|
console.error(err);
|
|
760
842
|
}
|
|
761
843
|
};
|
|
762
|
-
this.mute = () => {
|
|
763
|
-
if (this.state.isMuted) return;
|
|
764
|
-
if (this.state.isSpeaking) {
|
|
765
|
-
this.onStopSpeaking();
|
|
766
|
-
}
|
|
767
|
-
this.changeState({ isMuted: true });
|
|
768
|
-
};
|
|
769
|
-
this.unmute = () => {
|
|
770
|
-
if (!this.state.isMuted) return;
|
|
771
|
-
this.changeState({ isMuted: false });
|
|
772
|
-
if (this.vad.status === 1 /* MaybeSpeaking */) {
|
|
773
|
-
this.onStartSpeaking();
|
|
774
|
-
} else if (this.vad.status === 2 /* Speaking */) {
|
|
775
|
-
this.onStartSpeaking();
|
|
776
|
-
this.onConfirmSpeaking();
|
|
777
|
-
}
|
|
778
|
-
};
|
|
779
844
|
this.stop = () => {
|
|
780
845
|
this.changeState({
|
|
781
846
|
isStarting: false,
|
|
782
847
|
isStarted: false,
|
|
783
|
-
isMuted: false,
|
|
784
848
|
isSpeaking: false
|
|
785
849
|
});
|
|
850
|
+
this.stream = void 0;
|
|
786
851
|
try {
|
|
787
852
|
this.vad.stop();
|
|
788
853
|
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
@@ -805,32 +870,39 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
805
870
|
console.error(err);
|
|
806
871
|
}
|
|
807
872
|
};
|
|
873
|
+
this.changeVad = (vadConfig) => {
|
|
874
|
+
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
875
|
+
const stream = this.stream;
|
|
876
|
+
if (stream) {
|
|
877
|
+
this.stop();
|
|
878
|
+
}
|
|
879
|
+
this.vadConfig = vadConfig;
|
|
880
|
+
this.vad = getVAD(vadConfig);
|
|
881
|
+
if (stream) {
|
|
882
|
+
this.start(stream);
|
|
883
|
+
}
|
|
884
|
+
};
|
|
808
885
|
this.onStartSpeaking = async () => {
|
|
809
|
-
if (this.state.isMuted) return;
|
|
810
886
|
this.speakingConfirmed = false;
|
|
811
887
|
this.queuedChunks.length = 0;
|
|
812
888
|
this.workletNode?.port.postMessage({ type: "start" });
|
|
813
889
|
};
|
|
814
890
|
this.onConfirmSpeaking = async () => {
|
|
815
|
-
if (this.state.isMuted) return;
|
|
816
891
|
this.speakingConfirmed = true;
|
|
817
892
|
this.changeState({ isSpeaking: true });
|
|
818
893
|
this.emit("StartSpeaking");
|
|
819
894
|
};
|
|
820
895
|
this.onCancelSpeaking = async () => {
|
|
821
|
-
if (this.state.isMuted) return;
|
|
822
896
|
this.queuedChunks.length = 0;
|
|
823
897
|
this.workletNode?.port.postMessage({ type: "stop" });
|
|
824
898
|
};
|
|
825
899
|
this.onStopSpeaking = async () => {
|
|
826
|
-
if (this.state.isMuted) return;
|
|
827
900
|
this.speakingConfirmed = false;
|
|
828
901
|
this.changeState({ isSpeaking: false });
|
|
829
902
|
this.emit("StopSpeaking");
|
|
830
903
|
this.workletNode?.port.postMessage({ type: "stop" });
|
|
831
904
|
};
|
|
832
905
|
this.onWorkletMessage = (event) => {
|
|
833
|
-
if (this.state.isMuted) return;
|
|
834
906
|
const { type, data } = event.data;
|
|
835
907
|
if (type === "chunk") {
|
|
836
908
|
const pcmData = data;
|
|
@@ -846,10 +918,6 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
846
918
|
}
|
|
847
919
|
};
|
|
848
920
|
this.state = defaultMicRecorderState;
|
|
849
|
-
const workletBlob = new Blob([pcm_processor_worklet_default], {
|
|
850
|
-
type: "application/javascript"
|
|
851
|
-
});
|
|
852
|
-
this.workletUrl = URL.createObjectURL(workletBlob);
|
|
853
921
|
this.vad = getVAD(vadConfig);
|
|
854
922
|
}
|
|
855
923
|
changeState(state) {
|
|
@@ -953,7 +1021,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
953
1021
|
this._isProcessing = true;
|
|
954
1022
|
this._isMuted = false;
|
|
955
1023
|
this._isPaused = false;
|
|
956
|
-
|
|
1024
|
+
if (!this.micRecorder || options?.vad) {
|
|
1025
|
+
await this.startMic({ vad: options?.vad });
|
|
1026
|
+
}
|
|
957
1027
|
await this.startWS();
|
|
958
1028
|
};
|
|
959
1029
|
this.stop = async () => {
|
|
@@ -974,20 +1044,20 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
974
1044
|
Speaker2.stopAudio();
|
|
975
1045
|
};
|
|
976
1046
|
this.mute = () => {
|
|
977
|
-
this.
|
|
1047
|
+
this.vad?.pause();
|
|
978
1048
|
this._isMuted = true;
|
|
979
1049
|
this.notifyStateChange();
|
|
980
1050
|
};
|
|
981
1051
|
this.unmute = () => {
|
|
982
1052
|
if (!this.isPaused && !(this.options.disableInterruption && Speaker2.isPlaying)) {
|
|
983
|
-
this.
|
|
1053
|
+
this.vad?.resume();
|
|
984
1054
|
}
|
|
985
1055
|
this._isMuted = false;
|
|
986
1056
|
this.notifyStateChange();
|
|
987
1057
|
};
|
|
988
1058
|
this.pause = () => {
|
|
989
1059
|
if (this.isPaused) return;
|
|
990
|
-
this.
|
|
1060
|
+
this.vad?.pause();
|
|
991
1061
|
this._isPaused = true;
|
|
992
1062
|
this._isProcessing = false;
|
|
993
1063
|
this.notifyStateChange();
|
|
@@ -997,7 +1067,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
997
1067
|
this.resume = () => {
|
|
998
1068
|
if (!this.isPaused) return;
|
|
999
1069
|
if (!this.isMuted) {
|
|
1000
|
-
this.
|
|
1070
|
+
this.vad?.resume();
|
|
1001
1071
|
}
|
|
1002
1072
|
this._isPaused = false;
|
|
1003
1073
|
this.notifyStateChange();
|
|
@@ -1009,7 +1079,12 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1009
1079
|
this.options.vad = options.vad;
|
|
1010
1080
|
}
|
|
1011
1081
|
try {
|
|
1012
|
-
if (
|
|
1082
|
+
if (this.micRecorder) {
|
|
1083
|
+
this.micRecorder.stop();
|
|
1084
|
+
if (options.vad) {
|
|
1085
|
+
this.micRecorder.changeVad(options.vad);
|
|
1086
|
+
}
|
|
1087
|
+
} else {
|
|
1013
1088
|
this.micRecorder = new MicRecorder(this.options.vad);
|
|
1014
1089
|
this.micRecorder.on("StateChange", () => {
|
|
1015
1090
|
this.notifyStateChange();
|
|
@@ -1034,15 +1109,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1034
1109
|
this.notifyStateChange();
|
|
1035
1110
|
});
|
|
1036
1111
|
}
|
|
1037
|
-
const isRecorderStarted = this.micRecorder.state.isStarted;
|
|
1038
|
-
if (isRecorderStarted) {
|
|
1039
|
-
this.micRecorder?.stop();
|
|
1040
|
-
}
|
|
1041
1112
|
const micStream = await Mic2.start(options.deviceId);
|
|
1042
1113
|
this.micStream = micStream;
|
|
1043
|
-
|
|
1044
|
-
await this.micRecorder.start(micStream);
|
|
1045
|
-
}
|
|
1114
|
+
await this.micRecorder.start(micStream);
|
|
1046
1115
|
await this.updateDevices();
|
|
1047
1116
|
await Speaker2.start();
|
|
1048
1117
|
this.notifyStateChange();
|
|
@@ -1163,14 +1232,14 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1163
1232
|
this.onSpeakerStartPlaying = () => {
|
|
1164
1233
|
this.log("Speaker started");
|
|
1165
1234
|
if (this.options.disableInterruption) {
|
|
1166
|
-
this.
|
|
1235
|
+
this.vad?.pause();
|
|
1167
1236
|
}
|
|
1168
1237
|
this.notifyStateChange();
|
|
1169
1238
|
};
|
|
1170
1239
|
this.onSpeakerStopPlaying = () => {
|
|
1171
1240
|
this.log("Speaker stopped");
|
|
1172
1241
|
if (this.options.disableInterruption && !this.isMuted) {
|
|
1173
|
-
this.
|
|
1242
|
+
this.vad?.resume();
|
|
1174
1243
|
}
|
|
1175
1244
|
this.notifyStateChange();
|
|
1176
1245
|
};
|
|
@@ -1209,7 +1278,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1209
1278
|
return !!this.micStream;
|
|
1210
1279
|
}
|
|
1211
1280
|
get isMicMuted() {
|
|
1212
|
-
return this.
|
|
1281
|
+
return this.vad?.isPaused ?? false;
|
|
1213
1282
|
}
|
|
1214
1283
|
get isUserSpeaking() {
|
|
1215
1284
|
return this.micRecorder?.state.isSpeaking ?? false;
|
|
@@ -1331,6 +1400,7 @@ var Micdrop = window.micdropClient;
|
|
|
1331
1400
|
VAD,
|
|
1332
1401
|
VADStatus,
|
|
1333
1402
|
VolumeVAD,
|
|
1403
|
+
equalVADConfig,
|
|
1334
1404
|
getClientErrorFromWSCloseEvent,
|
|
1335
1405
|
getClientErrorFromWSCloseEventCode,
|
|
1336
1406
|
getVAD
|