@micdrop/client 2.2.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -24
- package/dist/index.d.mts +444 -73
- package/dist/index.d.ts +444 -73
- package/dist/index.js +983 -678
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +963 -676
- package/dist/index.mjs.map +1 -1
- package/package.json +11 -7
package/dist/index.js
CHANGED
|
@@ -20,8 +20,9 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
Mic: () => Mic,
|
|
24
|
+
MicController: () => MicController,
|
|
25
|
+
MicDriver: () => MicDriver,
|
|
25
26
|
MicRecorder: () => MicRecorder,
|
|
26
27
|
Micdrop: () => Micdrop,
|
|
27
28
|
MicdropClient: () => MicdropClient,
|
|
@@ -29,441 +30,486 @@ __export(index_exports, {
|
|
|
29
30
|
MicdropClientError: () => MicdropClientError,
|
|
30
31
|
MicdropClientErrorCode: () => MicdropClientErrorCode,
|
|
31
32
|
MicdropServerCommands: () => MicdropServerCommands,
|
|
33
|
+
MicdropStorageKeys: () => MicdropStorageKeys,
|
|
32
34
|
MultipleVAD: () => MultipleVAD,
|
|
35
|
+
Pcm16AudioStream: () => Pcm16AudioStream,
|
|
36
|
+
SILERO_FRAME_SAMPLES: () => SILERO_FRAME_SAMPLES,
|
|
37
|
+
SILERO_SAMPLE_RATE: () => SILERO_SAMPLE_RATE,
|
|
33
38
|
SileroVAD: () => SileroVAD,
|
|
34
|
-
Speaker: () =>
|
|
39
|
+
Speaker: () => Speaker,
|
|
40
|
+
SpeakerController: () => SpeakerController,
|
|
41
|
+
SpeakerDriver: () => SpeakerDriver,
|
|
35
42
|
VAD: () => VAD,
|
|
36
43
|
VADStatus: () => VADStatus,
|
|
44
|
+
VOLUME_SAMPLE_RATE: () => VOLUME_SAMPLE_RATE,
|
|
45
|
+
VolumeMeter: () => VolumeMeter,
|
|
37
46
|
VolumeVAD: () => VolumeVAD,
|
|
47
|
+
concatFloat32: () => concatFloat32,
|
|
38
48
|
equalVADConfig: () => equalVADConfig,
|
|
49
|
+
floatToPcm16: () => floatToPcm16,
|
|
39
50
|
getClientErrorFromWSCloseEvent: () => getClientErrorFromWSCloseEvent,
|
|
40
51
|
getClientErrorFromWSCloseEventCode: () => getClientErrorFromWSCloseEventCode,
|
|
41
52
|
getVAD: () => getVAD,
|
|
42
|
-
|
|
53
|
+
getVolume: () => getVolume,
|
|
54
|
+
isRecoverableError: () => isRecoverableError,
|
|
55
|
+
pcm16ToArrayBuffer: () => pcm16ToArrayBuffer,
|
|
56
|
+
pcm16ToFloat: () => pcm16ToFloat,
|
|
57
|
+
resample: () => resample,
|
|
58
|
+
setMicdropStorage: () => setMicdropStorage,
|
|
59
|
+
setSileroModelLoader: () => setSileroModelLoader,
|
|
60
|
+
storage: () => storage
|
|
43
61
|
});
|
|
44
62
|
module.exports = __toCommonJS(index_exports);
|
|
45
63
|
|
|
46
|
-
// src/audio/
|
|
64
|
+
// src/audio/Mic.ts
|
|
47
65
|
var import_eventemitter3 = require("eventemitter3");
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
66
|
+
|
|
67
|
+
// src/audio/pcm.ts
|
|
68
|
+
function resample(input, inputRate, outputRate) {
|
|
69
|
+
if (inputRate === outputRate || input.length === 0) return input;
|
|
70
|
+
const ratio = inputRate / outputRate;
|
|
71
|
+
const outputLength = Math.round(input.length / ratio);
|
|
72
|
+
const output = new Float32Array(outputLength);
|
|
73
|
+
for (let i = 0; i < outputLength; i++) {
|
|
74
|
+
const position = i * ratio;
|
|
75
|
+
const index = Math.floor(position);
|
|
76
|
+
const nextIndex = Math.min(index + 1, input.length - 1);
|
|
77
|
+
const fraction = position - index;
|
|
78
|
+
output[i] = input[index] * (1 - fraction) + input[nextIndex] * fraction;
|
|
79
|
+
}
|
|
80
|
+
return output;
|
|
81
|
+
}
|
|
82
|
+
function floatToPcm16(input) {
|
|
83
|
+
const output = new Int16Array(input.length);
|
|
84
|
+
for (let i = 0; i < input.length; i++) {
|
|
85
|
+
const sample = Math.max(-1, Math.min(1, input[i]));
|
|
86
|
+
output[i] = sample < 0 ? sample * 32768 : sample * 32767;
|
|
67
87
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
maxVolume = this.fftBins[i];
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return maxVolume;
|
|
88
|
+
return output;
|
|
89
|
+
}
|
|
90
|
+
function pcm16ToFloat(input) {
|
|
91
|
+
const output = new Float32Array(input.length);
|
|
92
|
+
for (let i = 0; i < input.length; i++) {
|
|
93
|
+
output[i] = input[i] / 32768;
|
|
78
94
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
}
|
|
91
|
-
|
|
95
|
+
return output;
|
|
96
|
+
}
|
|
97
|
+
function concatFloat32(chunks) {
|
|
98
|
+
if (chunks.length === 1) return chunks[0];
|
|
99
|
+
let length = 0;
|
|
100
|
+
for (const chunk of chunks) length += chunk.length;
|
|
101
|
+
const output = new Float32Array(length);
|
|
102
|
+
let offset = 0;
|
|
103
|
+
for (const chunk of chunks) {
|
|
104
|
+
output.set(chunk, offset);
|
|
105
|
+
offset += chunk.length;
|
|
106
|
+
}
|
|
107
|
+
return output;
|
|
92
108
|
}
|
|
93
|
-
function
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
source.disconnect();
|
|
99
|
-
source.buffer = null;
|
|
100
|
-
source.removeEventListener("ended", onEnded);
|
|
101
|
-
};
|
|
102
|
-
source.addEventListener("ended", onEnded);
|
|
103
|
-
source.start();
|
|
109
|
+
function pcm16ToArrayBuffer(pcm) {
|
|
110
|
+
return pcm.buffer.slice(
|
|
111
|
+
pcm.byteOffset,
|
|
112
|
+
pcm.byteOffset + pcm.byteLength
|
|
113
|
+
);
|
|
104
114
|
}
|
|
105
115
|
|
|
106
|
-
// src/audio/
|
|
107
|
-
var
|
|
108
|
-
|
|
109
|
-
|
|
116
|
+
// src/audio/volume.ts
|
|
117
|
+
var FFT_SIZE = 512;
|
|
118
|
+
var BIN_COUNT = FFT_SIZE / 2;
|
|
119
|
+
var FIRST_BIN = 4;
|
|
120
|
+
var SMOOTHING = 0.1;
|
|
121
|
+
var VOLUME_SAMPLE_RATE = 16e3;
|
|
122
|
+
var WINDOW = new Float32Array(FFT_SIZE);
|
|
123
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
124
|
+
const a = 2 * Math.PI * i / (FFT_SIZE - 1);
|
|
125
|
+
WINDOW[i] = 0.42 - 0.5 * Math.cos(a) + 0.08 * Math.cos(2 * a);
|
|
110
126
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
127
|
+
var REVERSED = new Uint16Array(FFT_SIZE);
|
|
128
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
129
|
+
let reversed = 0;
|
|
130
|
+
for (let bit = 1; bit < FFT_SIZE; bit <<= 1) {
|
|
131
|
+
reversed = reversed << 1 | (i & bit ? 1 : 0);
|
|
132
|
+
}
|
|
133
|
+
REVERSED[i] = reversed;
|
|
116
134
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
LocalStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
123
|
-
LocalStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
124
|
-
return LocalStorageKeys2;
|
|
125
|
-
})(LocalStorageKeys || {});
|
|
126
|
-
|
|
127
|
-
// src/audio/utils/stopStream.ts
|
|
128
|
-
function stopStream(stream) {
|
|
129
|
-
stream?.getTracks().forEach((track) => track.stop());
|
|
135
|
+
var COS = new Float32Array(FFT_SIZE / 2);
|
|
136
|
+
var SIN = new Float32Array(FFT_SIZE / 2);
|
|
137
|
+
for (let i = 0; i < FFT_SIZE / 2; i++) {
|
|
138
|
+
COS[i] = Math.cos(-2 * Math.PI * i / FFT_SIZE);
|
|
139
|
+
SIN[i] = Math.sin(-2 * Math.PI * i / FFT_SIZE);
|
|
130
140
|
}
|
|
131
|
-
|
|
132
|
-
// src/audio/Mic.ts
|
|
133
|
-
var Mic = class {
|
|
141
|
+
var VolumeMeter = class {
|
|
134
142
|
constructor() {
|
|
135
|
-
this.
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
deviceId: { ideal: deviceId },
|
|
141
|
-
sampleRate: 16e3,
|
|
142
|
-
// not working, it will follow device settings, usually 44.1kHz
|
|
143
|
-
sampleSize: 16,
|
|
144
|
-
channelCount: 1,
|
|
145
|
-
echoCancellation: true,
|
|
146
|
-
noiseSuppression: true
|
|
147
|
-
}
|
|
148
|
-
};
|
|
143
|
+
this.real = new Float32Array(FFT_SIZE);
|
|
144
|
+
this.imaginary = new Float32Array(FFT_SIZE);
|
|
145
|
+
this.magnitudes = new Float32Array(BIN_COUNT);
|
|
146
|
+
this.smoothed = new Float32Array(BIN_COUNT);
|
|
147
|
+
this.hasPrevious = false;
|
|
149
148
|
}
|
|
150
149
|
/**
|
|
151
|
-
*
|
|
152
|
-
* @param
|
|
153
|
-
*
|
|
150
|
+
* Measures a window of audio
|
|
151
|
+
* @param samples - At least 512 mono samples at 16 kHz, in the -1..1 range.
|
|
152
|
+
* Only the last 512 are looked at.
|
|
153
|
+
* @returns The level in decibels, between -Infinity (silence) and 0
|
|
154
154
|
*/
|
|
155
|
-
|
|
156
|
-
if (
|
|
157
|
-
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
|
|
155
|
+
measure(samples) {
|
|
156
|
+
if (samples.length === 0) return -Infinity;
|
|
157
|
+
const offset = Math.max(0, samples.length - FFT_SIZE);
|
|
158
|
+
const { real, imaginary } = this;
|
|
159
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
160
|
+
const sample = offset + i < samples.length ? samples[offset + i] : 0;
|
|
161
|
+
real[REVERSED[i]] = sample * WINDOW[i];
|
|
162
|
+
imaginary[REVERSED[i]] = 0;
|
|
161
163
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
164
|
+
for (let size = 2; size <= FFT_SIZE; size <<= 1) {
|
|
165
|
+
const half = size >> 1;
|
|
166
|
+
const step = FFT_SIZE / size;
|
|
167
|
+
for (let start = 0; start < FFT_SIZE; start += size) {
|
|
168
|
+
for (let i = 0; i < half; i++) {
|
|
169
|
+
const even = start + i;
|
|
170
|
+
const odd = even + half;
|
|
171
|
+
const cos = COS[i * step];
|
|
172
|
+
const sin = SIN[i * step];
|
|
173
|
+
const realOdd = real[odd] * cos - imaginary[odd] * sin;
|
|
174
|
+
const imaginaryOdd = real[odd] * sin + imaginary[odd] * cos;
|
|
175
|
+
real[odd] = real[even] - realOdd;
|
|
176
|
+
imaginary[odd] = imaginary[even] - imaginaryOdd;
|
|
177
|
+
real[even] += realOdd;
|
|
178
|
+
imaginary[even] += imaginaryOdd;
|
|
179
|
+
}
|
|
165
180
|
}
|
|
166
|
-
stopStream(this.audioStream);
|
|
167
181
|
}
|
|
168
|
-
|
|
169
|
-
this.
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (newDeviceId === "default" && deviceId) {
|
|
175
|
-
newDeviceId = deviceId;
|
|
182
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
183
|
+
this.magnitudes[k] = Math.sqrt(real[k] * real[k] + imaginary[k] * imaginary[k]) / FFT_SIZE;
|
|
184
|
+
}
|
|
185
|
+
if (this.hasPrevious) {
|
|
186
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
187
|
+
this.smoothed[k] = SMOOTHING * this.smoothed[k] + (1 - SMOOTHING) * this.magnitudes[k];
|
|
176
188
|
}
|
|
177
|
-
localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
|
|
178
|
-
this.deviceId = newDeviceId;
|
|
179
189
|
} else {
|
|
180
|
-
|
|
181
|
-
this.
|
|
190
|
+
this.smoothed.set(this.magnitudes);
|
|
191
|
+
this.hasPrevious = true;
|
|
182
192
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
193
|
+
let loudest = -Infinity;
|
|
194
|
+
for (let k = FIRST_BIN; k < BIN_COUNT; k++) {
|
|
195
|
+
const magnitude = this.smoothed[k];
|
|
196
|
+
if (magnitude === 0) continue;
|
|
197
|
+
const decibels = 20 * Math.log10(magnitude);
|
|
198
|
+
if (decibels > loudest && decibels < 0) loudest = decibels;
|
|
199
|
+
}
|
|
200
|
+
return loudest;
|
|
201
|
+
}
|
|
202
|
+
/** Forgets the previous window, for when capture stops and starts again */
|
|
203
|
+
reset() {
|
|
204
|
+
this.hasPrevious = false;
|
|
205
|
+
this.smoothed.fill(0);
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
function getVolume(samples) {
|
|
209
|
+
return new VolumeMeter().measure(samples);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// src/audio/Mic.ts
|
|
213
|
+
var VOLUME_INTERVAL = 100;
|
|
214
|
+
var VOLUME_WINDOW = 512;
|
|
215
|
+
var RATE_CHECK_DURATION = 2e3;
|
|
216
|
+
var RATE_TOLERANCE = 0.15;
|
|
217
|
+
var MicController = class extends import_eventemitter3.EventEmitter {
|
|
218
|
+
constructor() {
|
|
219
|
+
super(...arguments);
|
|
220
|
+
this.meter = new VolumeMeter();
|
|
221
|
+
this.pending = [];
|
|
222
|
+
this.pendingLength = 0;
|
|
223
|
+
this.pendingRate = VOLUME_SAMPLE_RATE;
|
|
224
|
+
this.rateCheckedAt = 0;
|
|
225
|
+
this.rateCheckSamples = 0;
|
|
226
|
+
this.rateWarned = false;
|
|
227
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
228
|
+
this.volume = -Infinity;
|
|
229
|
+
this.onFrames = (frames, sampleRate) => {
|
|
230
|
+
this.emit("Frames", frames, sampleRate);
|
|
231
|
+
this.checkRate(frames.length, sampleRate);
|
|
232
|
+
if (sampleRate !== this.pendingRate) {
|
|
233
|
+
this.pending = [];
|
|
234
|
+
this.pendingLength = 0;
|
|
235
|
+
this.pendingRate = sampleRate;
|
|
236
|
+
this.meter.reset();
|
|
237
|
+
}
|
|
238
|
+
this.pending.push(frames);
|
|
239
|
+
this.pendingLength += frames.length;
|
|
240
|
+
const interval = Math.round(VOLUME_INTERVAL / 1e3 * sampleRate);
|
|
241
|
+
if (this.pendingLength < interval) return;
|
|
242
|
+
const needed = Math.ceil(VOLUME_WINDOW * sampleRate / VOLUME_SAMPLE_RATE);
|
|
243
|
+
const gathered = concatFloat32(this.pending);
|
|
244
|
+
const window = gathered.subarray(Math.max(0, gathered.length - needed));
|
|
245
|
+
this.pending = [];
|
|
246
|
+
this.pendingLength = 0;
|
|
247
|
+
this.volume = this.meter.measure(
|
|
248
|
+
resample(window, sampleRate, VOLUME_SAMPLE_RATE)
|
|
249
|
+
);
|
|
250
|
+
this.emit("Volume", this.volume);
|
|
251
|
+
};
|
|
252
|
+
this.onDeviceChange = () => {
|
|
253
|
+
this.emit("DeviceChange");
|
|
254
|
+
};
|
|
255
|
+
this.onError = (error) => {
|
|
256
|
+
this.emit("Error", error);
|
|
257
|
+
};
|
|
187
258
|
}
|
|
188
259
|
/**
|
|
189
|
-
*
|
|
260
|
+
* Replaces the audio driver, to record with another library
|
|
261
|
+
* @param driver - The driver to record with
|
|
190
262
|
*/
|
|
191
|
-
|
|
192
|
-
if (this.
|
|
193
|
-
|
|
194
|
-
this.
|
|
263
|
+
setDriver(driver) {
|
|
264
|
+
if (this.driver === driver) return;
|
|
265
|
+
if (this.driver) {
|
|
266
|
+
this.driver.off("Frames", this.onFrames);
|
|
267
|
+
this.driver.off("DeviceChange", this.onDeviceChange);
|
|
268
|
+
this.driver.off("Error", this.onError);
|
|
195
269
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
270
|
+
this.driver = driver;
|
|
271
|
+
driver.on("Frames", this.onFrames);
|
|
272
|
+
driver.on("DeviceChange", this.onDeviceChange);
|
|
273
|
+
driver.on("Error", this.onError);
|
|
274
|
+
}
|
|
275
|
+
getDriver() {
|
|
276
|
+
if (!this.driver) {
|
|
277
|
+
throw new Error(
|
|
278
|
+
"No microphone driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Mic.setDriver()."
|
|
279
|
+
);
|
|
199
280
|
}
|
|
281
|
+
return this.driver;
|
|
282
|
+
}
|
|
283
|
+
get isStarted() {
|
|
284
|
+
return this.driver?.isStarted ?? false;
|
|
285
|
+
}
|
|
286
|
+
get deviceId() {
|
|
287
|
+
return this.driver?.deviceId;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Asks for the microphone permission and starts capturing
|
|
291
|
+
* @param deviceId - Input device to record from, defaults to the system one
|
|
292
|
+
*/
|
|
293
|
+
async start(deviceId) {
|
|
294
|
+
this.resetVolume();
|
|
295
|
+
await this.getDriver().start(deviceId);
|
|
296
|
+
}
|
|
297
|
+
/** Stops capturing and releases the microphone */
|
|
298
|
+
async stop() {
|
|
299
|
+
await this.driver?.stop();
|
|
300
|
+
this.resetVolume();
|
|
301
|
+
}
|
|
302
|
+
/** Lists the input devices the system offers */
|
|
303
|
+
async getDevices() {
|
|
304
|
+
if (!this.driver) return [];
|
|
305
|
+
return this.driver.getDevices();
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Times the captured audio against the rate it claims to be at.
|
|
309
|
+
*
|
|
310
|
+
* A microphone that announces 16 kHz while delivering 48 kHz sends speech to
|
|
311
|
+
* the server three times too slow, which comes back as a short transcript in
|
|
312
|
+
* a random language. That is invisible otherwise, so it is worth saying out
|
|
313
|
+
* loud once.
|
|
314
|
+
* @param samples - How many samples just arrived
|
|
315
|
+
* @param sampleRate - The rate they are said to be at
|
|
316
|
+
*/
|
|
317
|
+
checkRate(samples, sampleRate) {
|
|
318
|
+
if (this.rateWarned) return;
|
|
319
|
+
const now = Date.now();
|
|
320
|
+
if (this.rateCheckedAt === 0) {
|
|
321
|
+
this.rateCheckedAt = now;
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
this.rateCheckSamples += samples;
|
|
325
|
+
const elapsed = now - this.rateCheckedAt;
|
|
326
|
+
if (elapsed < RATE_CHECK_DURATION) return;
|
|
327
|
+
const delivered = this.rateCheckSamples / elapsed * 1e3;
|
|
328
|
+
this.rateCheckedAt = now;
|
|
329
|
+
this.rateCheckSamples = 0;
|
|
330
|
+
if (Math.abs(delivered - sampleRate) / sampleRate <= RATE_TOLERANCE) return;
|
|
331
|
+
this.rateWarned = true;
|
|
332
|
+
console.warn(
|
|
333
|
+
`[Micdrop] The microphone announces ${sampleRate} Hz but delivers about ${Math.round(delivered)} Hz. Speech is being sent at the wrong speed, so transcription will be poor.`
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
resetVolume() {
|
|
337
|
+
this.pending = [];
|
|
338
|
+
this.pendingLength = 0;
|
|
339
|
+
this.meter.reset();
|
|
340
|
+
this.volume = -Infinity;
|
|
341
|
+
this.rateCheckedAt = 0;
|
|
342
|
+
this.rateCheckSamples = 0;
|
|
200
343
|
}
|
|
201
344
|
};
|
|
202
345
|
|
|
203
346
|
// src/audio/Speaker.ts
|
|
204
|
-
var import_eventemitter33 = require("eventemitter3");
|
|
205
|
-
|
|
206
|
-
// src/audio/stream/AudioStream.ts
|
|
207
347
|
var import_eventemitter32 = require("eventemitter3");
|
|
208
|
-
var AudioStream = class extends import_eventemitter32.EventEmitter {
|
|
209
|
-
constructor() {
|
|
210
|
-
super();
|
|
211
|
-
this.isPlaying = false;
|
|
212
|
-
this.outputNode = audioContext.createGain();
|
|
213
|
-
}
|
|
214
|
-
setIsPlaying(isPlaying) {
|
|
215
|
-
if (this.isPlaying === isPlaying) return;
|
|
216
|
-
this.isPlaying = isPlaying;
|
|
217
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
218
|
-
}
|
|
219
|
-
};
|
|
220
348
|
|
|
221
|
-
// src/
|
|
222
|
-
var
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
349
|
+
// src/storage.ts
|
|
350
|
+
var MicdropStorageKeys = /* @__PURE__ */ ((MicdropStorageKeys2) => {
|
|
351
|
+
MicdropStorageKeys2["MicDevice"] = "micdrop.micDevice";
|
|
352
|
+
MicdropStorageKeys2["SpeakerDevice"] = "micdrop.speakerDevice";
|
|
353
|
+
MicdropStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
354
|
+
MicdropStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
355
|
+
return MicdropStorageKeys2;
|
|
356
|
+
})(MicdropStorageKeys || {});
|
|
357
|
+
var MemoryStorage = class {
|
|
227
358
|
constructor() {
|
|
228
|
-
|
|
229
|
-
this.blobQueue = [];
|
|
230
|
-
this.processing = false;
|
|
231
|
-
this.sourceNodes = [];
|
|
232
|
-
this.nextStartTime = 0;
|
|
233
|
-
this.prebuffering = true;
|
|
234
|
-
this.prebuffer = [];
|
|
235
|
-
this.prebufferDuration = 0;
|
|
359
|
+
this.values = /* @__PURE__ */ new Map();
|
|
236
360
|
}
|
|
237
|
-
|
|
238
|
-
return this.
|
|
361
|
+
getItem(key) {
|
|
362
|
+
return this.values.get(key) ?? null;
|
|
239
363
|
}
|
|
240
|
-
|
|
241
|
-
this.
|
|
242
|
-
if (this.processing) return;
|
|
243
|
-
this.processing = true;
|
|
244
|
-
try {
|
|
245
|
-
while (this.blobQueue.length > 0) {
|
|
246
|
-
const next = this.blobQueue.shift();
|
|
247
|
-
try {
|
|
248
|
-
const arrayBuffer = await next.arrayBuffer();
|
|
249
|
-
const audioBuffer = this.createAudioBuffer(arrayBuffer);
|
|
250
|
-
if (audioBuffer) this.handleBuffer(audioBuffer);
|
|
251
|
-
} catch (error) {
|
|
252
|
-
console.error("Failed to play audio chunk:", error);
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
} finally {
|
|
256
|
-
this.processing = false;
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
stopAudio() {
|
|
260
|
-
for (const node of this.sourceNodes) {
|
|
261
|
-
try {
|
|
262
|
-
node.stop();
|
|
263
|
-
node.disconnect();
|
|
264
|
-
} catch (error) {
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
this.sourceNodes = [];
|
|
268
|
-
this.blobQueue = [];
|
|
269
|
-
this.nextStartTime = 0;
|
|
270
|
-
this.resetPrebuffer();
|
|
271
|
-
this.setIsPlaying(false);
|
|
364
|
+
setItem(key, value) {
|
|
365
|
+
this.values.set(key, value);
|
|
272
366
|
}
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
handleBuffer(audioBuffer) {
|
|
276
|
-
if (!this.prebuffering) {
|
|
277
|
-
this.scheduleBuffer(audioBuffer);
|
|
278
|
-
return;
|
|
279
|
-
}
|
|
280
|
-
this.prebuffer.push(audioBuffer);
|
|
281
|
-
this.prebufferDuration += audioBuffer.duration * 1e3;
|
|
282
|
-
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
283
|
-
this.flushPrebuffer();
|
|
284
|
-
return;
|
|
285
|
-
}
|
|
286
|
-
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
287
|
-
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
367
|
+
removeItem(key) {
|
|
368
|
+
this.values.delete(key);
|
|
288
369
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
370
|
+
};
|
|
371
|
+
var current = new MemoryStorage();
|
|
372
|
+
function setMicdropStorage(storage2) {
|
|
373
|
+
current = storage2;
|
|
374
|
+
}
|
|
375
|
+
var storage = {
|
|
376
|
+
getItem: (key) => {
|
|
377
|
+
try {
|
|
378
|
+
return current.getItem(key);
|
|
379
|
+
} catch {
|
|
380
|
+
return null;
|
|
293
381
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
}
|
|
300
|
-
resetPrebuffer() {
|
|
301
|
-
if (this.quietTimer !== void 0) {
|
|
302
|
-
clearTimeout(this.quietTimer);
|
|
303
|
-
this.quietTimer = void 0;
|
|
382
|
+
},
|
|
383
|
+
setItem: (key, value) => {
|
|
384
|
+
try {
|
|
385
|
+
current.setItem(key, value);
|
|
386
|
+
} catch {
|
|
304
387
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
this.prebufferDuration = 0;
|
|
308
|
-
}
|
|
309
|
-
createAudioBuffer(arrayBuffer) {
|
|
388
|
+
},
|
|
389
|
+
removeItem: (key) => {
|
|
310
390
|
try {
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
if (numSamples === 0) {
|
|
314
|
-
return null;
|
|
315
|
-
}
|
|
316
|
-
const audioBuffer = audioContext.createBuffer(
|
|
317
|
-
CHANNELS,
|
|
318
|
-
numSamples,
|
|
319
|
-
SAMPLE_RATE
|
|
320
|
-
);
|
|
321
|
-
const channelData = audioBuffer.getChannelData(0);
|
|
322
|
-
for (let i = 0; i < numSamples; i++) {
|
|
323
|
-
channelData[i] = int16Data[i] / 32768;
|
|
324
|
-
}
|
|
325
|
-
return audioBuffer;
|
|
326
|
-
} catch (error) {
|
|
327
|
-
console.error("Error creating AudioBuffer:", error);
|
|
328
|
-
return null;
|
|
391
|
+
current.removeItem(key);
|
|
392
|
+
} catch {
|
|
329
393
|
}
|
|
330
394
|
}
|
|
331
|
-
scheduleBuffer(audioBuffer) {
|
|
332
|
-
const sourceNode = audioContext.createBufferSource();
|
|
333
|
-
sourceNode.buffer = audioBuffer;
|
|
334
|
-
sourceNode.connect(this.outputNode);
|
|
335
|
-
const now = audioContext.currentTime;
|
|
336
|
-
const startTime = Math.max(this.nextStartTime, now);
|
|
337
|
-
this.nextStartTime = startTime + audioBuffer.duration;
|
|
338
|
-
sourceNode.onended = () => {
|
|
339
|
-
const index = this.sourceNodes.indexOf(sourceNode);
|
|
340
|
-
if (index !== -1) this.sourceNodes.splice(index, 1);
|
|
341
|
-
if (this.sourceNodes.length === 0 && this.blobQueue.length === 0) {
|
|
342
|
-
this.nextStartTime = 0;
|
|
343
|
-
this.resetPrebuffer();
|
|
344
|
-
this.setIsPlaying(false);
|
|
345
|
-
}
|
|
346
|
-
};
|
|
347
|
-
this.sourceNodes.push(sourceNode);
|
|
348
|
-
sourceNode.start(startTime);
|
|
349
|
-
this.setIsPlaying(true);
|
|
350
|
-
}
|
|
351
395
|
};
|
|
352
396
|
|
|
353
397
|
// src/audio/Speaker.ts
|
|
354
|
-
var
|
|
398
|
+
var SpeakerController = class extends import_eventemitter32.EventEmitter {
|
|
355
399
|
constructor() {
|
|
356
400
|
super(...arguments);
|
|
357
|
-
|
|
358
|
-
this.
|
|
359
|
-
|
|
360
|
-
this.
|
|
401
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
402
|
+
this.volume = -Infinity;
|
|
403
|
+
this.onStartPlaying = () => this.emit("StartPlaying");
|
|
404
|
+
this.onStopPlaying = () => {
|
|
405
|
+
this.volume = -Infinity;
|
|
406
|
+
this.emit("StopPlaying");
|
|
407
|
+
};
|
|
408
|
+
this.onVolume = (volume) => {
|
|
409
|
+
this.volume = volume;
|
|
410
|
+
this.emit("Volume", volume);
|
|
411
|
+
};
|
|
361
412
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
this.
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
async playAudio(blob) {
|
|
373
|
-
this.start();
|
|
374
|
-
try {
|
|
375
|
-
await this.audioStream?.playAudio(blob);
|
|
376
|
-
} catch (error) {
|
|
377
|
-
console.error("Error playing audio blob", error, blob);
|
|
413
|
+
/**
|
|
414
|
+
* Replaces the audio driver, to play sound with another library
|
|
415
|
+
* @param driver - The driver to play with
|
|
416
|
+
*/
|
|
417
|
+
setDriver(driver) {
|
|
418
|
+
if (this.driver === driver) return;
|
|
419
|
+
if (this.driver) {
|
|
420
|
+
this.driver.off("StartPlaying", this.onStartPlaying);
|
|
421
|
+
this.driver.off("StopPlaying", this.onStopPlaying);
|
|
422
|
+
this.driver.off("Volume", this.onVolume);
|
|
378
423
|
}
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
this.
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
424
|
+
this.driver = driver;
|
|
425
|
+
driver.on("StartPlaying", this.onStartPlaying);
|
|
426
|
+
driver.on("StopPlaying", this.onStopPlaying);
|
|
427
|
+
driver.on("Volume", this.onVolume);
|
|
428
|
+
}
|
|
429
|
+
getDriver() {
|
|
430
|
+
if (!this.driver) {
|
|
431
|
+
throw new Error(
|
|
432
|
+
"No speaker driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Speaker.setDriver()."
|
|
433
|
+
);
|
|
390
434
|
}
|
|
435
|
+
return this.driver;
|
|
391
436
|
}
|
|
392
|
-
get
|
|
393
|
-
return
|
|
437
|
+
get isPlaying() {
|
|
438
|
+
return this.driver?.isPlaying ?? false;
|
|
394
439
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
this.isPlaying = isPlaying;
|
|
398
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
440
|
+
get deviceId() {
|
|
441
|
+
return this.driver?.deviceId;
|
|
399
442
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
443
|
+
/** Prepares the audio graph, called before the first chunk arrives */
|
|
444
|
+
async start() {
|
|
445
|
+
const driver = this.getDriver();
|
|
446
|
+
await driver.start();
|
|
447
|
+
const saved = storage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
448
|
+
if (saved && saved !== driver.deviceId) {
|
|
449
|
+
try {
|
|
450
|
+
await driver.changeDevice(saved);
|
|
451
|
+
} catch {
|
|
452
|
+
storage.removeItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
453
|
+
}
|
|
406
454
|
}
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Queues samples for playback, right after the ones already queued
|
|
458
|
+
* @param pcm - Mono samples as 16 bits integers, or the bytes holding them
|
|
459
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
460
|
+
*/
|
|
461
|
+
playAudio(pcm, sampleRate = 16e3) {
|
|
407
462
|
try {
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
const sinkId = !deviceId || defaultDevice ? "" : deviceId;
|
|
411
|
-
if (defaultDevice || "setSinkId" in audioContext) {
|
|
412
|
-
await audioContext.setSinkId?.(sinkId);
|
|
413
|
-
this.audioSourceNode.disconnect();
|
|
414
|
-
this.audioSourceNode.connect(audioContext.destination);
|
|
415
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
416
|
-
if (this.audioElementDestination) {
|
|
417
|
-
this.audioElementDestination.pause();
|
|
418
|
-
this.audioElementDestination.srcObject = null;
|
|
419
|
-
this.audioElementDestination = void 0;
|
|
420
|
-
}
|
|
421
|
-
} else {
|
|
422
|
-
if (!this.audioElementDestination) {
|
|
423
|
-
const mediaStreamDestination = audioContext.createMediaStreamDestination();
|
|
424
|
-
this.audioSourceNode.disconnect();
|
|
425
|
-
this.audioSourceNode.connect(mediaStreamDestination);
|
|
426
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
427
|
-
this.audioElementDestination = new Audio();
|
|
428
|
-
this.audioElementDestination.autoplay = true;
|
|
429
|
-
this.audioElementDestination.srcObject = mediaStreamDestination.stream;
|
|
430
|
-
}
|
|
431
|
-
await this.audioElementDestination.setSinkId(sinkId);
|
|
432
|
-
}
|
|
463
|
+
const samples = pcm instanceof Int16Array ? pcm : new Int16Array(pcm);
|
|
464
|
+
this.getDriver().play(samples, sampleRate);
|
|
433
465
|
} catch (error) {
|
|
434
|
-
console.error(
|
|
466
|
+
console.error("[Micdrop] Error playing audio", error);
|
|
435
467
|
}
|
|
436
468
|
}
|
|
469
|
+
/** Drops everything queued and stops playing immediately */
|
|
470
|
+
stopAudio() {
|
|
471
|
+
this.driver?.stopAudio();
|
|
472
|
+
}
|
|
473
|
+
/** Releases the audio graph */
|
|
474
|
+
async stop() {
|
|
475
|
+
await this.driver?.stop();
|
|
476
|
+
this.volume = -Infinity;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Plays through another output
|
|
480
|
+
* @param deviceId - Output device, or route on a phone
|
|
481
|
+
*/
|
|
482
|
+
async changeDevice(deviceId) {
|
|
483
|
+
await this.getDriver().changeDevice(deviceId);
|
|
484
|
+
storage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
485
|
+
}
|
|
486
|
+
/** Lists the output devices the system offers */
|
|
487
|
+
async getDevices() {
|
|
488
|
+
if (!this.driver) return [];
|
|
489
|
+
return this.driver.getDevices();
|
|
490
|
+
}
|
|
437
491
|
};
|
|
438
492
|
|
|
439
493
|
// src/audio/MicRecorder.ts
|
|
440
|
-
var
|
|
441
|
-
|
|
442
|
-
// src/audio/utils/delayedStream.ts
|
|
443
|
-
function createDelayedStream(stream, delay) {
|
|
444
|
-
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
445
|
-
const delayNode = audioContext.createDelay(delay);
|
|
446
|
-
delayNode.delayTime.value = delay;
|
|
447
|
-
const audioDestination = audioContext.createMediaStreamDestination();
|
|
448
|
-
audioSource.connect(delayNode);
|
|
449
|
-
delayNode.connect(audioDestination);
|
|
450
|
-
return audioDestination.stream;
|
|
451
|
-
}
|
|
494
|
+
var import_eventemitter34 = require("eventemitter3");
|
|
452
495
|
|
|
453
496
|
// src/audio/vad/VAD.ts
|
|
454
|
-
var
|
|
497
|
+
var import_eventemitter33 = require("eventemitter3");
|
|
455
498
|
var VADStatus = /* @__PURE__ */ ((VADStatus2) => {
|
|
456
499
|
VADStatus2[VADStatus2["Silence"] = 0] = "Silence";
|
|
457
500
|
VADStatus2[VADStatus2["MaybeSpeaking"] = 1] = "MaybeSpeaking";
|
|
458
501
|
VADStatus2[VADStatus2["Speaking"] = 2] = "Speaking";
|
|
459
502
|
return VADStatus2;
|
|
460
503
|
})(VADStatus || {});
|
|
461
|
-
var VAD = class extends
|
|
504
|
+
var VAD = class extends import_eventemitter33.EventEmitter {
|
|
462
505
|
constructor() {
|
|
463
506
|
super(...arguments);
|
|
464
507
|
this.name = this.constructor.name;
|
|
465
508
|
this.status = 0 /* Silence */;
|
|
466
|
-
//
|
|
509
|
+
// Worst case time this VAD takes to notice that someone started speaking.
|
|
510
|
+
// That much audio is kept in reserve and sent along with the turn, so the
|
|
511
|
+
// beginning of a sentence reaches the server rather than being cut off.
|
|
512
|
+
// A VAD that needs several samples to make up its mind has to say so here.
|
|
467
513
|
this.delay = 100;
|
|
468
514
|
}
|
|
469
515
|
/**
|
|
@@ -546,13 +592,13 @@ var MultipleVAD = class extends VAD {
|
|
|
546
592
|
get isPaused() {
|
|
547
593
|
return this.vads.every((vad) => vad.isPaused);
|
|
548
594
|
}
|
|
549
|
-
async start(
|
|
595
|
+
async start(mic) {
|
|
550
596
|
for (const vad of this.vads) {
|
|
551
597
|
if (vad.isStarted) {
|
|
552
598
|
continue;
|
|
553
599
|
}
|
|
554
600
|
vad.on("ChangeStatus", this.onStatusChange);
|
|
555
|
-
await vad.start(
|
|
601
|
+
await vad.start(mic);
|
|
556
602
|
}
|
|
557
603
|
}
|
|
558
604
|
async stop() {
|
|
@@ -574,87 +620,96 @@ var MultipleVAD = class extends VAD {
|
|
|
574
620
|
};
|
|
575
621
|
|
|
576
622
|
// src/audio/vad/SileroVAD.ts
|
|
577
|
-
var
|
|
623
|
+
var SILERO_SAMPLE_RATE = 16e3;
|
|
624
|
+
var SILERO_FRAME_SAMPLES = 512;
|
|
625
|
+
var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
|
|
578
626
|
var defaultOptions = {
|
|
579
627
|
positiveSpeechThreshold: 0.18,
|
|
580
628
|
negativeSpeechThreshold: 0.11,
|
|
581
629
|
minSpeechFrames: 8,
|
|
582
630
|
redemptionFrames: 20
|
|
583
631
|
};
|
|
632
|
+
var loadModel;
|
|
633
|
+
function setSileroModelLoader(loader) {
|
|
634
|
+
loadModel = loader;
|
|
635
|
+
}
|
|
584
636
|
var SileroVAD = class extends VAD {
|
|
585
637
|
constructor(options) {
|
|
586
638
|
super();
|
|
587
639
|
this.name = "SileroVAD";
|
|
588
640
|
this.options = defaultOptions;
|
|
641
|
+
// The model decides on the first window it hears speech in, so it is quick.
|
|
642
|
+
// The reserve still covers a few windows, since scoring runs asynchronously
|
|
643
|
+
// and may lag behind the audio on a busy device.
|
|
644
|
+
this.delay = 3 * MS_PER_FRAME;
|
|
589
645
|
this._isPaused = false;
|
|
646
|
+
this.buffer = [];
|
|
647
|
+
this.bufferLength = 0;
|
|
648
|
+
this.bufferRate = SILERO_SAMPLE_RATE;
|
|
649
|
+
this.speaking = false;
|
|
650
|
+
this.speechFrames = 0;
|
|
651
|
+
this.silenceFrames = 0;
|
|
652
|
+
this.processing = false;
|
|
653
|
+
this.onFrames = (frames, sampleRate) => {
|
|
654
|
+
if (this._isPaused || !this.model) return;
|
|
655
|
+
if (sampleRate !== this.bufferRate) {
|
|
656
|
+
this.buffer = [];
|
|
657
|
+
this.bufferLength = 0;
|
|
658
|
+
this.bufferRate = sampleRate;
|
|
659
|
+
}
|
|
660
|
+
this.buffer.push(frames);
|
|
661
|
+
this.bufferLength += frames.length;
|
|
662
|
+
this.drain();
|
|
663
|
+
};
|
|
590
664
|
if (options) {
|
|
591
665
|
this.setOptions(options);
|
|
592
666
|
} else {
|
|
593
|
-
const
|
|
594
|
-
"micdrop.SileroVADOptions" /* SileroVADOptions */
|
|
595
|
-
);
|
|
667
|
+
const saved = storage.getItem("micdrop.SileroVADOptions" /* SileroVADOptions */);
|
|
596
668
|
try {
|
|
597
|
-
this.setOptions(
|
|
598
|
-
} catch
|
|
669
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
670
|
+
} catch {
|
|
599
671
|
this.setOptions({});
|
|
600
672
|
}
|
|
601
673
|
}
|
|
602
674
|
}
|
|
603
675
|
get isStarted() {
|
|
604
|
-
return !!this.
|
|
676
|
+
return !!this.mic;
|
|
605
677
|
}
|
|
606
678
|
get isPaused() {
|
|
607
679
|
return this._isPaused;
|
|
608
680
|
}
|
|
609
|
-
async start(
|
|
610
|
-
if (this.
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
// CDN size limit, so every file 404s). unpkg serves it, so the ONNX wasm
|
|
621
|
-
// backend is loaded from there instead.
|
|
622
|
-
onnxWASMBasePath: "https://unpkg.com/onnxruntime-web@1.23.2/dist/",
|
|
623
|
-
onSpeechStart: () => {
|
|
624
|
-
this.emit("StartSpeaking");
|
|
625
|
-
},
|
|
626
|
-
onSpeechRealStart: () => {
|
|
627
|
-
this.emit("ConfirmSpeaking");
|
|
628
|
-
},
|
|
629
|
-
onVADMisfire: () => {
|
|
630
|
-
this.emit("CancelSpeaking");
|
|
631
|
-
},
|
|
632
|
-
onSpeechEnd: () => {
|
|
633
|
-
this.emit("StopSpeaking");
|
|
634
|
-
},
|
|
635
|
-
...this.options
|
|
636
|
-
});
|
|
637
|
-
this.vad.start();
|
|
681
|
+
async start(mic) {
|
|
682
|
+
if (this.mic) return;
|
|
683
|
+
if (!loadModel) {
|
|
684
|
+
throw new Error(
|
|
685
|
+
"No Silero model. Add `import '@micdrop/web/silero'` in a browser or `import '@micdrop/react-native/silero'` on a phone, or call setSileroModelLoader()."
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
this.model = await loadModel();
|
|
689
|
+
this.resetState();
|
|
690
|
+
this.mic = mic;
|
|
691
|
+
mic.on("Frames", this.onFrames);
|
|
638
692
|
}
|
|
639
693
|
async stop() {
|
|
640
|
-
|
|
641
|
-
this.
|
|
642
|
-
this.
|
|
694
|
+
this.mic?.off("Frames", this.onFrames);
|
|
695
|
+
this.mic = void 0;
|
|
696
|
+
this.closeOpenTurn();
|
|
697
|
+
this.resetState();
|
|
698
|
+
await this.model?.release();
|
|
699
|
+
this.model = void 0;
|
|
643
700
|
}
|
|
644
701
|
async pause() {
|
|
645
|
-
if (
|
|
702
|
+
if (this._isPaused) return;
|
|
646
703
|
this._isPaused = true;
|
|
647
|
-
this.
|
|
704
|
+
this.closeOpenTurn();
|
|
705
|
+
this.resetState();
|
|
648
706
|
}
|
|
649
707
|
async resume() {
|
|
650
|
-
if (!this.vad || !this._isPaused) return;
|
|
651
708
|
this._isPaused = false;
|
|
652
|
-
this.vad.start();
|
|
653
709
|
}
|
|
654
710
|
setOptions(options) {
|
|
655
711
|
this.options = { ...this.options, ...options };
|
|
656
|
-
|
|
657
|
-
localStorage.setItem(
|
|
712
|
+
storage.setItem(
|
|
658
713
|
"micdrop.SileroVADOptions" /* SileroVADOptions */,
|
|
659
714
|
JSON.stringify(this.options)
|
|
660
715
|
);
|
|
@@ -662,7 +717,79 @@ var SileroVAD = class extends VAD {
|
|
|
662
717
|
resetOptions() {
|
|
663
718
|
this.setOptions(defaultOptions);
|
|
664
719
|
}
|
|
720
|
+
// The model is asynchronous, so windows are scored one after the other rather
|
|
721
|
+
// than raced: its answer for a window depends on the ones before it.
|
|
722
|
+
async drain() {
|
|
723
|
+
if (this.processing) return;
|
|
724
|
+
this.processing = true;
|
|
725
|
+
try {
|
|
726
|
+
const needed = Math.ceil(
|
|
727
|
+
SILERO_FRAME_SAMPLES * this.bufferRate / SILERO_SAMPLE_RATE
|
|
728
|
+
);
|
|
729
|
+
while (this.bufferLength >= needed && this.model && !this._isPaused) {
|
|
730
|
+
const gathered = concatFloat32(this.buffer);
|
|
731
|
+
const window = gathered.subarray(0, needed);
|
|
732
|
+
const rest = gathered.subarray(needed);
|
|
733
|
+
this.buffer = rest.length ? [rest] : [];
|
|
734
|
+
this.bufferLength = rest.length;
|
|
735
|
+
const frame = resample(window, this.bufferRate, SILERO_SAMPLE_RATE);
|
|
736
|
+
const probability = await this.model.process(
|
|
737
|
+
frame.length === SILERO_FRAME_SAMPLES ? frame : fit(frame, SILERO_FRAME_SAMPLES)
|
|
738
|
+
);
|
|
739
|
+
this.onProbability(probability);
|
|
740
|
+
}
|
|
741
|
+
} catch (error) {
|
|
742
|
+
console.error("[Micdrop] Silero VAD failed", error);
|
|
743
|
+
} finally {
|
|
744
|
+
this.processing = false;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
// Ported from the reference implementation of the algorithm
|
|
748
|
+
// https://docs.vad.ricky0123.com/user-guide/algorithm/
|
|
749
|
+
onProbability(probability) {
|
|
750
|
+
const { positiveSpeechThreshold, negativeSpeechThreshold } = this.options;
|
|
751
|
+
if (probability >= positiveSpeechThreshold) {
|
|
752
|
+
this.silenceFrames = 0;
|
|
753
|
+
this.speechFrames++;
|
|
754
|
+
if (!this.speaking) {
|
|
755
|
+
this.speaking = true;
|
|
756
|
+
this.emit("StartSpeaking");
|
|
757
|
+
}
|
|
758
|
+
if (this.speechFrames === this.options.minSpeechFrames) {
|
|
759
|
+
this.emit("ConfirmSpeaking");
|
|
760
|
+
}
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
if (probability < negativeSpeechThreshold && this.speaking) {
|
|
764
|
+
this.silenceFrames++;
|
|
765
|
+
if (this.silenceFrames < this.options.redemptionFrames) return;
|
|
766
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
767
|
+
this.speaking = false;
|
|
768
|
+
this.speechFrames = 0;
|
|
769
|
+
this.silenceFrames = 0;
|
|
770
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
closeOpenTurn() {
|
|
774
|
+
if (!this.speaking) return;
|
|
775
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
776
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
777
|
+
}
|
|
778
|
+
resetState() {
|
|
779
|
+
this.model?.reset();
|
|
780
|
+
this.buffer = [];
|
|
781
|
+
this.bufferLength = 0;
|
|
782
|
+
this.speaking = false;
|
|
783
|
+
this.speechFrames = 0;
|
|
784
|
+
this.silenceFrames = 0;
|
|
785
|
+
}
|
|
665
786
|
};
|
|
787
|
+
function fit(frame, length) {
|
|
788
|
+
if (frame.length === length) return frame;
|
|
789
|
+
const fitted = new Float32Array(length);
|
|
790
|
+
fitted.set(frame.subarray(0, length));
|
|
791
|
+
return fitted;
|
|
792
|
+
}
|
|
666
793
|
|
|
667
794
|
// src/audio/vad/VolumeVAD.ts
|
|
668
795
|
var defaultOptions2 = {
|
|
@@ -674,138 +801,92 @@ var VolumeVAD = class extends VAD {
|
|
|
674
801
|
super();
|
|
675
802
|
this.name = "VolumeVAD";
|
|
676
803
|
this.options = defaultOptions2;
|
|
677
|
-
|
|
804
|
+
// The microphone reports a level about every 100 ms, and two consecutive loud
|
|
805
|
+
// ones are needed before a turn opens. Speech can therefore begin up to two
|
|
806
|
+
// reports before it is noticed, and a third is kept as a margin: without that
|
|
807
|
+
// reserve the first syllable of a sentence never reaches the server.
|
|
808
|
+
this.delay = 300;
|
|
678
809
|
this.running = false;
|
|
810
|
+
this._isPaused = false;
|
|
679
811
|
this.speaking = false;
|
|
680
812
|
this.attemptSpeaking = false;
|
|
681
813
|
this.speakingHistory = [];
|
|
814
|
+
this.onVolume = (volume) => {
|
|
815
|
+
if (!this.running) return;
|
|
816
|
+
const isLoud = volume > this.options.threshold;
|
|
817
|
+
if (isLoud) {
|
|
818
|
+
if (!this.speaking) {
|
|
819
|
+
let recentHistory = 0;
|
|
820
|
+
for (let i = Math.max(0, this.speakingHistory.length - 3); i < this.speakingHistory.length; i++) {
|
|
821
|
+
recentHistory += this.speakingHistory[i];
|
|
822
|
+
}
|
|
823
|
+
if (recentHistory >= 2) {
|
|
824
|
+
this.speaking = true;
|
|
825
|
+
this.attemptSpeaking = false;
|
|
826
|
+
this.emit("ConfirmSpeaking");
|
|
827
|
+
} else if (recentHistory === 1 && !this.attemptSpeaking) {
|
|
828
|
+
this.attemptSpeaking = true;
|
|
829
|
+
this.emit("StartSpeaking");
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
} else if (this.speaking) {
|
|
833
|
+
let totalHistory = 0;
|
|
834
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
835
|
+
totalHistory += this.speakingHistory[i];
|
|
836
|
+
}
|
|
837
|
+
if (totalHistory === 0) {
|
|
838
|
+
this.speaking = false;
|
|
839
|
+
this.emit("StopSpeaking");
|
|
840
|
+
}
|
|
841
|
+
} else if (this.attemptSpeaking) {
|
|
842
|
+
const lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
843
|
+
if (lastHistory === 0) {
|
|
844
|
+
this.attemptSpeaking = false;
|
|
845
|
+
this.emit("CancelSpeaking");
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
this.speakingHistory.shift();
|
|
849
|
+
this.speakingHistory.push(Number(isLoud));
|
|
850
|
+
};
|
|
682
851
|
if (options) {
|
|
683
852
|
this.setOptions(options);
|
|
684
853
|
} else {
|
|
685
|
-
const
|
|
686
|
-
"micdrop.VolumeVADOptions" /* VolumeVADOptions */
|
|
687
|
-
);
|
|
854
|
+
const saved = storage.getItem("micdrop.VolumeVADOptions" /* VolumeVADOptions */);
|
|
688
855
|
try {
|
|
689
|
-
this.setOptions(
|
|
690
|
-
} catch
|
|
856
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
857
|
+
} catch {
|
|
691
858
|
this.setOptions({});
|
|
692
859
|
}
|
|
693
860
|
}
|
|
694
861
|
}
|
|
695
|
-
getMaxVolume() {
|
|
696
|
-
if (!this.analyser || !this.fftBins) return -Infinity;
|
|
697
|
-
let maxVolume = -Infinity;
|
|
698
|
-
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
699
|
-
for (let i = 4; i < this.fftBins.length; i++) {
|
|
700
|
-
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
701
|
-
maxVolume = this.fftBins[i];
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
return maxVolume;
|
|
705
|
-
}
|
|
706
862
|
get isStarted() {
|
|
707
863
|
return this.running;
|
|
708
864
|
}
|
|
709
865
|
get isPaused() {
|
|
710
866
|
return this._isPaused;
|
|
711
867
|
}
|
|
712
|
-
async start(
|
|
713
|
-
if (this.
|
|
714
|
-
this.
|
|
715
|
-
|
|
716
|
-
this.
|
|
717
|
-
this.analyser.smoothingTimeConstant = 0.1;
|
|
718
|
-
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
719
|
-
this.sourceNode.connect(this.analyser);
|
|
720
|
-
this.running = true;
|
|
721
|
-
this.startLoop();
|
|
722
|
-
}
|
|
723
|
-
startLoop() {
|
|
724
|
-
if (!this.running) return;
|
|
725
|
-
const currentVolume = this.getMaxVolume();
|
|
726
|
-
if (currentVolume > this.options.threshold) {
|
|
727
|
-
if (!this.speaking) {
|
|
728
|
-
let recentHistory = 0;
|
|
729
|
-
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
730
|
-
recentHistory += this.speakingHistory[i];
|
|
731
|
-
}
|
|
732
|
-
if (recentHistory >= 2) {
|
|
733
|
-
this.speaking = true;
|
|
734
|
-
this.attemptSpeaking = false;
|
|
735
|
-
this.emit("ConfirmSpeaking");
|
|
736
|
-
} else if (recentHistory === 1) {
|
|
737
|
-
this.attemptSpeaking = true;
|
|
738
|
-
this.emit("StartSpeaking");
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
} else if (this.speaking) {
|
|
742
|
-
let totalHistory = 0;
|
|
743
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
744
|
-
totalHistory += this.speakingHistory[i];
|
|
745
|
-
}
|
|
746
|
-
if (totalHistory === 0) {
|
|
747
|
-
this.speaking = false;
|
|
748
|
-
this.emit("StopSpeaking");
|
|
749
|
-
}
|
|
750
|
-
} else if (this.attemptSpeaking) {
|
|
751
|
-
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
752
|
-
if (lastHistory === 0) {
|
|
753
|
-
this.attemptSpeaking = false;
|
|
754
|
-
this.emit("CancelSpeaking");
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
|
-
this.speakingHistory.shift();
|
|
758
|
-
this.speakingHistory.push(Number(currentVolume > this.options.threshold));
|
|
759
|
-
setTimeout(() => this.startLoop(), this.delay);
|
|
760
|
-
}
|
|
761
|
-
resetHistory() {
|
|
762
|
-
this.speaking = false;
|
|
763
|
-
this.attemptSpeaking = false;
|
|
764
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
765
|
-
this.speakingHistory[i] = 0;
|
|
766
|
-
}
|
|
868
|
+
async start(mic) {
|
|
869
|
+
if (this.mic) return;
|
|
870
|
+
this.mic = mic;
|
|
871
|
+
mic.on("Volume", this.onVolume);
|
|
872
|
+
this.running = !this._isPaused;
|
|
767
873
|
}
|
|
768
874
|
async stop() {
|
|
769
|
-
|
|
875
|
+
this.mic?.off("Volume", this.onVolume);
|
|
876
|
+
this.mic = void 0;
|
|
770
877
|
this.running = false;
|
|
771
|
-
this.
|
|
772
|
-
switch (this.status) {
|
|
773
|
-
case 2 /* Speaking */:
|
|
774
|
-
this.emit("StopSpeaking");
|
|
775
|
-
break;
|
|
776
|
-
case 1 /* MaybeSpeaking */:
|
|
777
|
-
this.emit("CancelSpeaking");
|
|
778
|
-
break;
|
|
779
|
-
}
|
|
780
|
-
if (this.sourceNode) {
|
|
781
|
-
this.sourceNode.disconnect();
|
|
782
|
-
this.sourceNode = void 0;
|
|
783
|
-
}
|
|
784
|
-
if (this.analyser) {
|
|
785
|
-
this.analyser.disconnect();
|
|
786
|
-
this.analyser = void 0;
|
|
787
|
-
}
|
|
788
|
-
this.fftBins = void 0;
|
|
878
|
+
this.reset();
|
|
789
879
|
}
|
|
790
880
|
async pause() {
|
|
791
|
-
if (
|
|
881
|
+
if (this._isPaused) return;
|
|
792
882
|
this._isPaused = true;
|
|
793
883
|
this.running = false;
|
|
794
|
-
this.
|
|
795
|
-
switch (this.status) {
|
|
796
|
-
case 2 /* Speaking */:
|
|
797
|
-
this.emit("StopSpeaking");
|
|
798
|
-
break;
|
|
799
|
-
case 1 /* MaybeSpeaking */:
|
|
800
|
-
this.emit("CancelSpeaking");
|
|
801
|
-
break;
|
|
802
|
-
}
|
|
884
|
+
this.reset();
|
|
803
885
|
}
|
|
804
886
|
async resume() {
|
|
805
887
|
if (!this._isPaused) return;
|
|
806
888
|
this._isPaused = false;
|
|
807
|
-
this.running =
|
|
808
|
-
this.startLoop();
|
|
889
|
+
this.running = !!this.mic;
|
|
809
890
|
}
|
|
810
891
|
setOptions(options) {
|
|
811
892
|
if (typeof options.history === "number" && options.history < 3) {
|
|
@@ -818,7 +899,7 @@ var VolumeVAD = class extends VAD {
|
|
|
818
899
|
while (this.speakingHistory.length < this.options.history) {
|
|
819
900
|
this.speakingHistory.push(0);
|
|
820
901
|
}
|
|
821
|
-
|
|
902
|
+
storage.setItem(
|
|
822
903
|
"micdrop.VolumeVADOptions" /* VolumeVADOptions */,
|
|
823
904
|
JSON.stringify(this.options)
|
|
824
905
|
);
|
|
@@ -826,6 +907,22 @@ var VolumeVAD = class extends VAD {
|
|
|
826
907
|
resetOptions() {
|
|
827
908
|
this.setOptions(defaultOptions2);
|
|
828
909
|
}
|
|
910
|
+
reset() {
|
|
911
|
+
const status = this.status;
|
|
912
|
+
this.speaking = false;
|
|
913
|
+
this.attemptSpeaking = false;
|
|
914
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
915
|
+
this.speakingHistory[i] = 0;
|
|
916
|
+
}
|
|
917
|
+
switch (status) {
|
|
918
|
+
case 2 /* Speaking */:
|
|
919
|
+
this.emit("StopSpeaking");
|
|
920
|
+
break;
|
|
921
|
+
case 1 /* MaybeSpeaking */:
|
|
922
|
+
this.emit("CancelSpeaking");
|
|
923
|
+
break;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
829
926
|
};
|
|
830
927
|
|
|
831
928
|
// src/audio/vad/getVAD.ts
|
|
@@ -847,34 +944,27 @@ function equalVADConfig(a, b) {
|
|
|
847
944
|
return false;
|
|
848
945
|
}
|
|
849
946
|
|
|
850
|
-
// raw:./pcm-processor-worklet
|
|
851
|
-
var pcm_processor_worklet_default = "const INTERVAL_MS = 100\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.audioBuffer = []\n this.isRecording = false\n this.sourceSampleRate = 16000\n this.targetSampleRate = 16000\n this.resamplingRatio = 1\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n\n // Listen for messages\n this.port.onmessage = this.handleMessage.bind(this)\n }\n\n handleMessage(event) {\n if (event.data.type === 'configure') {\n this.sourceSampleRate = event.data.sourceSampleRate\n this.targetSampleRate = event.data.targetSampleRate\n this.resamplingRatio = this.sourceSampleRate / this.targetSampleRate\n // Recalculate chunk interval for the source sample rate\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n } else if (event.data.type === 'start') {\n this.isRecording = true\n } else if (event.data.type === 'stop') {\n this.isRecording = false\n // Clear any remaining audio buffer when stopping\n this.audioBuffer.length = 0\n }\n }\n\n process(inputs) {\n const input = inputs[0]\n\n if (input && input.length > 0 && this.isRecording) {\n const inputChannel = input[0]\n\n // Store audio data in buffer\n this.audioBuffer.push(new Float32Array(inputChannel))\n\n // Check if we have enough data for a chunk\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n\n if (totalSamples >= this.CHUNK_INTERVAL_SAMPLES) {\n this.processChunk()\n }\n }\n\n return true\n }\n\n processChunk() {\n if (this.audioBuffer.length === 0) return\n\n // Calculate total samples\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n if (totalSamples === 0) return\n\n // Merge all buffers\n const mergedBuffer = new Float32Array(totalSamples)\n let offset = 0\n for (const buffer of this.audioBuffer) {\n mergedBuffer.set(buffer, offset)\n offset += buffer.length\n }\n\n // Resample to target sample rate if necessary\n const resampledBuffer = this.resample(mergedBuffer)\n\n // Convert float32 to int16 PCM\n const pcmData = this.floatToPCM16(resampledBuffer)\n\n // Clear the buffer\n this.audioBuffer.length = 0\n\n // Send PCM chunk to main thread\n this.port.postMessage({\n type: 'chunk',\n data: pcmData,\n })\n }\n\n resample(inputBuffer) {\n // If no resampling needed, return original buffer\n if (this.resamplingRatio === 1) {\n return inputBuffer\n }\n\n // Calculate output length\n const outputLength = Math.round(inputBuffer.length / this.resamplingRatio)\n const outputBuffer = new Float32Array(outputLength)\n\n // Simple linear interpolation resampling\n for (let i = 0; i < outputLength; i++) {\n const srcIndex = i * this.resamplingRatio\n const srcIndexFloor = Math.floor(srcIndex)\n const srcIndexCeil = Math.min(srcIndexFloor + 1, inputBuffer.length - 1)\n const fraction = srcIndex - srcIndexFloor\n\n // Linear interpolation\n outputBuffer[i] =\n inputBuffer[srcIndexFloor] * (1 - fraction) +\n inputBuffer[srcIndexCeil] * fraction\n }\n\n return outputBuffer\n }\n\n floatToPCM16(floatData) {\n const pcmData = new Int16Array(floatData.length)\n for (let i = 0; i < floatData.length; i++) {\n const sample = Math.max(-1, Math.min(1, floatData[i]))\n pcmData[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff\n }\n return pcmData\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
|
|
852
|
-
|
|
853
|
-
// src/audio/pcm-processor.ts
|
|
854
|
-
async function initPcmProcessor() {
|
|
855
|
-
if (window.micdropPcmProcessor) return;
|
|
856
|
-
window.micdropPcmProcessor = true;
|
|
857
|
-
const workletBlob = new Blob([pcm_processor_worklet_default], {
|
|
858
|
-
type: "application/javascript"
|
|
859
|
-
});
|
|
860
|
-
const workletUrl = URL.createObjectURL(workletBlob);
|
|
861
|
-
await audioContext.audioWorklet.addModule(workletUrl);
|
|
862
|
-
}
|
|
863
|
-
|
|
864
947
|
// src/audio/MicRecorder.ts
|
|
865
|
-
var
|
|
948
|
+
var SAMPLE_RATE = 16e3;
|
|
949
|
+
var CHUNK_INTERVAL = 100;
|
|
866
950
|
var defaultMicRecorderState = {
|
|
867
951
|
isStarting: false,
|
|
868
952
|
isStarted: false,
|
|
869
953
|
isSpeaking: false
|
|
870
954
|
};
|
|
871
|
-
var MicRecorder = class extends
|
|
955
|
+
var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
872
956
|
constructor(vadConfig) {
|
|
873
957
|
super();
|
|
874
958
|
this.vadConfig = vadConfig;
|
|
959
|
+
this.isRecording = false;
|
|
875
960
|
this.speakingConfirmed = false;
|
|
961
|
+
this.sourceSampleRate = SAMPLE_RATE;
|
|
962
|
+
this.reserve = [];
|
|
963
|
+
this.reserveLength = 0;
|
|
964
|
+
this.buffer = [];
|
|
965
|
+
this.bufferLength = 0;
|
|
876
966
|
this.queuedChunks = [];
|
|
877
|
-
this.start = async (
|
|
967
|
+
this.start = async (mic) => {
|
|
878
968
|
if (this.state.isStarted) {
|
|
879
969
|
throw new Error("MicRecorder is already started");
|
|
880
970
|
}
|
|
@@ -883,21 +973,10 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
883
973
|
isStarting: true,
|
|
884
974
|
isSpeaking: false
|
|
885
975
|
});
|
|
886
|
-
this.
|
|
887
|
-
this.
|
|
888
|
-
|
|
889
|
-
this.
|
|
890
|
-
this.workletNode.port.onmessage = this.onWorkletMessage;
|
|
891
|
-
this.workletNode.port.postMessage({
|
|
892
|
-
type: "configure",
|
|
893
|
-
sourceSampleRate: audioContext.sampleRate,
|
|
894
|
-
targetSampleRate: SAMPLE_RATE2
|
|
895
|
-
});
|
|
896
|
-
const delayedStream = createDelayedStream(stream, this.vad.delay / 1e3);
|
|
897
|
-
this.delayedStream = delayedStream;
|
|
898
|
-
this.source = audioContext.createMediaStreamSource(delayedStream);
|
|
899
|
-
this.source.connect(this.workletNode);
|
|
900
|
-
await this.vad.start(stream);
|
|
976
|
+
this.mic = mic;
|
|
977
|
+
this.resetBuffers();
|
|
978
|
+
mic.on("Frames", this.onFrames);
|
|
979
|
+
await this.vad.start(mic);
|
|
901
980
|
this.vad.on("StartSpeaking", this.onStartSpeaking);
|
|
902
981
|
this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
903
982
|
this.vad.on("CancelSpeaking", this.onCancelSpeaking);
|
|
@@ -917,79 +996,138 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
917
996
|
isStarted: false,
|
|
918
997
|
isSpeaking: false
|
|
919
998
|
});
|
|
920
|
-
this.stream = void 0;
|
|
921
999
|
try {
|
|
1000
|
+
this.mic?.off("Frames", this.onFrames);
|
|
1001
|
+
this.mic = void 0;
|
|
922
1002
|
this.vad.stop();
|
|
923
1003
|
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
924
1004
|
this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
925
1005
|
this.vad.off("CancelSpeaking", this.onCancelSpeaking);
|
|
926
1006
|
this.vad.off("StopSpeaking", this.onStopSpeaking);
|
|
927
|
-
if (this.workletNode) {
|
|
928
|
-
this.workletNode.disconnect();
|
|
929
|
-
this.workletNode = void 0;
|
|
930
|
-
}
|
|
931
|
-
if (this.source) {
|
|
932
|
-
this.source.disconnect();
|
|
933
|
-
this.source = void 0;
|
|
934
|
-
}
|
|
935
|
-
if (this.delayedStream) {
|
|
936
|
-
stopStream(this.delayedStream);
|
|
937
|
-
this.delayedStream = void 0;
|
|
938
|
-
}
|
|
939
1007
|
} catch (err) {
|
|
940
1008
|
console.error(err);
|
|
941
1009
|
}
|
|
1010
|
+
this.resetBuffers();
|
|
942
1011
|
};
|
|
943
1012
|
this.changeVad = (vadConfig) => {
|
|
944
1013
|
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
945
|
-
const
|
|
946
|
-
if (
|
|
1014
|
+
const mic = this.mic;
|
|
1015
|
+
if (mic) {
|
|
947
1016
|
this.stop();
|
|
948
1017
|
}
|
|
949
1018
|
this.vadConfig = vadConfig;
|
|
950
1019
|
this.vad = getVAD(vadConfig);
|
|
951
|
-
if (
|
|
952
|
-
this.start(
|
|
1020
|
+
if (mic) {
|
|
1021
|
+
this.start(mic);
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
this.onFrames = (frames, sampleRate) => {
|
|
1025
|
+
if (sampleRate !== this.sourceSampleRate) {
|
|
1026
|
+
this.flushChunks();
|
|
1027
|
+
this.sourceSampleRate = sampleRate;
|
|
1028
|
+
this.reserve = [];
|
|
1029
|
+
this.reserveLength = 0;
|
|
1030
|
+
}
|
|
1031
|
+
if (this.isRecording) {
|
|
1032
|
+
this.buffer.push(frames);
|
|
1033
|
+
this.bufferLength += frames.length;
|
|
1034
|
+
this.flushChunks();
|
|
1035
|
+
return;
|
|
1036
|
+
}
|
|
1037
|
+
this.reserve.push(frames);
|
|
1038
|
+
this.reserveLength += frames.length;
|
|
1039
|
+
const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
|
|
1040
|
+
while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
|
|
1041
|
+
this.reserveLength -= this.reserve[0].length;
|
|
1042
|
+
this.reserve.shift();
|
|
953
1043
|
}
|
|
954
1044
|
};
|
|
955
|
-
this.onStartSpeaking =
|
|
1045
|
+
this.onStartSpeaking = () => {
|
|
956
1046
|
this.speakingConfirmed = false;
|
|
957
1047
|
this.queuedChunks.length = 0;
|
|
958
|
-
this.
|
|
1048
|
+
this.isRecording = true;
|
|
1049
|
+
this.buffer = this.reserve;
|
|
1050
|
+
this.bufferLength = this.reserveLength;
|
|
1051
|
+
this.reserve = [];
|
|
1052
|
+
this.reserveLength = 0;
|
|
1053
|
+
this.flushChunks();
|
|
959
1054
|
};
|
|
960
|
-
this.onConfirmSpeaking =
|
|
1055
|
+
this.onConfirmSpeaking = () => {
|
|
961
1056
|
this.speakingConfirmed = true;
|
|
962
1057
|
this.changeState({ isSpeaking: true });
|
|
963
1058
|
this.emit("StartSpeaking");
|
|
964
|
-
|
|
965
|
-
|
|
1059
|
+
for (const chunk of this.queuedChunks) {
|
|
1060
|
+
this.emit("Chunk", chunk);
|
|
1061
|
+
}
|
|
966
1062
|
this.queuedChunks.length = 0;
|
|
967
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
968
1063
|
};
|
|
969
|
-
this.
|
|
970
|
-
this.
|
|
971
|
-
this.changeState({ isSpeaking: false });
|
|
972
|
-
this.emit("StopSpeaking");
|
|
973
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
1064
|
+
this.onCancelSpeaking = () => {
|
|
1065
|
+
this.stopRecording();
|
|
974
1066
|
};
|
|
975
|
-
this.
|
|
976
|
-
const
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
this.queuedChunks.push(pcmData);
|
|
981
|
-
return;
|
|
982
|
-
}
|
|
983
|
-
for (const chunk of this.queuedChunks) {
|
|
984
|
-
this.emit("Chunk", chunk);
|
|
985
|
-
}
|
|
986
|
-
this.queuedChunks.length = 0;
|
|
987
|
-
this.emit("Chunk", pcmData);
|
|
1067
|
+
this.onStopSpeaking = () => {
|
|
1068
|
+
const wasConfirmed = this.speakingConfirmed;
|
|
1069
|
+
this.changeState({ isSpeaking: false });
|
|
1070
|
+
if (wasConfirmed) {
|
|
1071
|
+
this.flushChunks(true);
|
|
988
1072
|
}
|
|
1073
|
+
this.stopRecording();
|
|
1074
|
+
this.emit("StopSpeaking");
|
|
989
1075
|
};
|
|
990
1076
|
this.state = defaultMicRecorderState;
|
|
991
1077
|
this.vad = getVAD(vadConfig);
|
|
992
1078
|
}
|
|
1079
|
+
stopRecording() {
|
|
1080
|
+
this.isRecording = false;
|
|
1081
|
+
this.speakingConfirmed = false;
|
|
1082
|
+
this.queuedChunks.length = 0;
|
|
1083
|
+
this.buffer = [];
|
|
1084
|
+
this.bufferLength = 0;
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Emits every complete chunk waiting in the buffer
|
|
1088
|
+
* @param final - Also emit the incomplete last chunk
|
|
1089
|
+
*/
|
|
1090
|
+
flushChunks(final = false) {
|
|
1091
|
+
const chunkLength = Math.round(
|
|
1092
|
+
CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
|
|
1093
|
+
);
|
|
1094
|
+
if (this.bufferLength === 0) return;
|
|
1095
|
+
if (this.bufferLength < chunkLength && !final) return;
|
|
1096
|
+
const merged = concatFloat32(this.buffer);
|
|
1097
|
+
let offset = 0;
|
|
1098
|
+
while (merged.length - offset >= chunkLength) {
|
|
1099
|
+
this.emitChunk(merged.subarray(offset, offset + chunkLength));
|
|
1100
|
+
offset += chunkLength;
|
|
1101
|
+
}
|
|
1102
|
+
const rest = merged.subarray(offset);
|
|
1103
|
+
if (final && rest.length > 0) {
|
|
1104
|
+
this.emitChunk(rest);
|
|
1105
|
+
this.buffer = [];
|
|
1106
|
+
this.bufferLength = 0;
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
this.buffer = rest.length ? [rest] : [];
|
|
1110
|
+
this.bufferLength = rest.length;
|
|
1111
|
+
}
|
|
1112
|
+
emitChunk(samples) {
|
|
1113
|
+
const pcm = floatToPcm16(
|
|
1114
|
+
resample(samples, this.sourceSampleRate, SAMPLE_RATE)
|
|
1115
|
+
);
|
|
1116
|
+
if (!this.speakingConfirmed) {
|
|
1117
|
+
this.queuedChunks.push(pcm);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
this.emit("Chunk", pcm);
|
|
1121
|
+
}
|
|
1122
|
+
resetBuffers() {
|
|
1123
|
+
this.isRecording = false;
|
|
1124
|
+
this.speakingConfirmed = false;
|
|
1125
|
+
this.reserve = [];
|
|
1126
|
+
this.reserveLength = 0;
|
|
1127
|
+
this.buffer = [];
|
|
1128
|
+
this.bufferLength = 0;
|
|
1129
|
+
this.queuedChunks.length = 0;
|
|
1130
|
+
}
|
|
993
1131
|
changeState(state) {
|
|
994
1132
|
const hasChanged = Object.keys(state).some(
|
|
995
1133
|
(key) => this.state[key] !== state[key]
|
|
@@ -1000,18 +1138,182 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
1000
1138
|
}
|
|
1001
1139
|
};
|
|
1002
1140
|
|
|
1141
|
+
// src/audio/Pcm16AudioStream.ts
|
|
1142
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1143
|
+
var PREBUFFER_DURATION = 100;
|
|
1144
|
+
var QUIET_FLUSH_DELAY = 600;
|
|
1145
|
+
var UTTERANCE_END_DELAY = 300;
|
|
1146
|
+
var VOLUME_INTERVAL2 = 100;
|
|
1147
|
+
var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
1148
|
+
constructor(sink) {
|
|
1149
|
+
super();
|
|
1150
|
+
this.sink = sink;
|
|
1151
|
+
this.isPlaying = false;
|
|
1152
|
+
this.sources = [];
|
|
1153
|
+
this.nextStartTime = 0;
|
|
1154
|
+
this.prebuffering = true;
|
|
1155
|
+
this.prebuffer = [];
|
|
1156
|
+
this.prebufferDuration = 0;
|
|
1157
|
+
this.volumeTimers = [];
|
|
1158
|
+
this.meter = new VolumeMeter();
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Queues samples for playback
|
|
1162
|
+
* @param pcm - Mono samples as 16 bits integers
|
|
1163
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
1164
|
+
*/
|
|
1165
|
+
playAudio(pcm, sampleRate) {
|
|
1166
|
+
if (pcm.length === 0) return;
|
|
1167
|
+
const samples = resample(
|
|
1168
|
+
pcm16ToFloat(pcm),
|
|
1169
|
+
sampleRate,
|
|
1170
|
+
this.sink.sampleRate
|
|
1171
|
+
);
|
|
1172
|
+
this.handleSamples(samples);
|
|
1173
|
+
}
|
|
1174
|
+
/** Drops everything queued and stops playing immediately */
|
|
1175
|
+
stopAudio() {
|
|
1176
|
+
for (const source of this.sources) {
|
|
1177
|
+
try {
|
|
1178
|
+
source.stop();
|
|
1179
|
+
} catch (error) {
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
this.sources = [];
|
|
1183
|
+
this.nextStartTime = 0;
|
|
1184
|
+
this.resetPrebuffer();
|
|
1185
|
+
this.setIsPlaying(false);
|
|
1186
|
+
this.clearVolumeTimers();
|
|
1187
|
+
}
|
|
1188
|
+
// Accumulate audio until enough is buffered, then start playing. Once
|
|
1189
|
+
// playing, schedule incoming samples directly.
|
|
1190
|
+
handleSamples(samples) {
|
|
1191
|
+
if (this.endTimer !== void 0) {
|
|
1192
|
+
clearTimeout(this.endTimer);
|
|
1193
|
+
this.endTimer = void 0;
|
|
1194
|
+
}
|
|
1195
|
+
if (!this.prebuffering) {
|
|
1196
|
+
this.scheduleSamples(samples);
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
this.prebuffer.push(samples);
|
|
1200
|
+
this.prebufferDuration += samples.length / this.sink.sampleRate * 1e3;
|
|
1201
|
+
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
1202
|
+
this.flushPrebuffer();
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
1205
|
+
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
1206
|
+
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
1207
|
+
}
|
|
1208
|
+
flushPrebuffer() {
|
|
1209
|
+
if (this.quietTimer !== void 0) {
|
|
1210
|
+
clearTimeout(this.quietTimer);
|
|
1211
|
+
this.quietTimer = void 0;
|
|
1212
|
+
}
|
|
1213
|
+
this.prebuffering = false;
|
|
1214
|
+
const buffers = this.prebuffer;
|
|
1215
|
+
this.prebuffer = [];
|
|
1216
|
+
this.prebufferDuration = 0;
|
|
1217
|
+
for (const samples of buffers) this.scheduleSamples(samples);
|
|
1218
|
+
}
|
|
1219
|
+
resetPrebuffer() {
|
|
1220
|
+
if (this.quietTimer !== void 0) {
|
|
1221
|
+
clearTimeout(this.quietTimer);
|
|
1222
|
+
this.quietTimer = void 0;
|
|
1223
|
+
}
|
|
1224
|
+
if (this.endTimer !== void 0) {
|
|
1225
|
+
clearTimeout(this.endTimer);
|
|
1226
|
+
this.endTimer = void 0;
|
|
1227
|
+
}
|
|
1228
|
+
this.prebuffering = true;
|
|
1229
|
+
this.prebuffer = [];
|
|
1230
|
+
this.prebufferDuration = 0;
|
|
1231
|
+
}
|
|
1232
|
+
scheduleSamples(samples) {
|
|
1233
|
+
const duration = samples.length / this.sink.sampleRate;
|
|
1234
|
+
const now = this.sink.currentTime;
|
|
1235
|
+
const startTime = Math.max(this.nextStartTime, now);
|
|
1236
|
+
this.nextStartTime = startTime + duration;
|
|
1237
|
+
const source = this.sink.schedule(samples, startTime, () => {
|
|
1238
|
+
const index = this.sources.indexOf(source);
|
|
1239
|
+
if (index !== -1) this.sources.splice(index, 1);
|
|
1240
|
+
if (this.sources.length === 0) {
|
|
1241
|
+
this.nextStartTime = 0;
|
|
1242
|
+
this.setIsPlaying(false);
|
|
1243
|
+
if (this.endTimer !== void 0) clearTimeout(this.endTimer);
|
|
1244
|
+
this.endTimer = setTimeout(() => {
|
|
1245
|
+
this.endTimer = void 0;
|
|
1246
|
+
this.resetPrebuffer();
|
|
1247
|
+
}, UTTERANCE_END_DELAY);
|
|
1248
|
+
}
|
|
1249
|
+
});
|
|
1250
|
+
this.sources.push(source);
|
|
1251
|
+
this.scheduleVolume(samples, startTime - now);
|
|
1252
|
+
this.setIsPlaying(true);
|
|
1253
|
+
}
|
|
1254
|
+
// Report the level of what is being heard rather than of what just arrived:
|
|
1255
|
+
// a burst of chunks would otherwise light up the meter all at once and leave
|
|
1256
|
+
// it dark for the rest of the sentence.
|
|
1257
|
+
scheduleVolume(samples, delay) {
|
|
1258
|
+
const step = Math.max(
|
|
1259
|
+
1,
|
|
1260
|
+
Math.round(VOLUME_INTERVAL2 / 1e3 * this.sink.sampleRate)
|
|
1261
|
+
);
|
|
1262
|
+
for (let offset = 0; offset < samples.length; offset += step) {
|
|
1263
|
+
const slice = samples.subarray(offset, offset + step);
|
|
1264
|
+
const at = delay * 1e3 + offset / this.sink.sampleRate * 1e3;
|
|
1265
|
+
const timer = setTimeout(
|
|
1266
|
+
() => {
|
|
1267
|
+
const index = this.volumeTimers.indexOf(timer);
|
|
1268
|
+
if (index !== -1) this.volumeTimers.splice(index, 1);
|
|
1269
|
+
this.emit(
|
|
1270
|
+
"Volume",
|
|
1271
|
+
this.meter.measure(
|
|
1272
|
+
resample(slice, this.sink.sampleRate, VOLUME_SAMPLE_RATE)
|
|
1273
|
+
)
|
|
1274
|
+
);
|
|
1275
|
+
},
|
|
1276
|
+
Math.max(0, at)
|
|
1277
|
+
);
|
|
1278
|
+
this.volumeTimers.push(timer);
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
clearVolumeTimers() {
|
|
1282
|
+
for (const timer of this.volumeTimers) clearTimeout(timer);
|
|
1283
|
+
this.volumeTimers = [];
|
|
1284
|
+
this.meter.reset();
|
|
1285
|
+
}
|
|
1286
|
+
setIsPlaying(isPlaying) {
|
|
1287
|
+
if (this.isPlaying === isPlaying) return;
|
|
1288
|
+
this.isPlaying = isPlaying;
|
|
1289
|
+
if (!isPlaying) {
|
|
1290
|
+
this.clearVolumeTimers();
|
|
1291
|
+
this.emit("Volume", -Infinity);
|
|
1292
|
+
}
|
|
1293
|
+
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
|
|
1297
|
+
// src/audio/types.ts
|
|
1298
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1299
|
+
var MicDriver = class extends import_eventemitter36.EventEmitter {
|
|
1300
|
+
};
|
|
1301
|
+
var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
|
|
1302
|
+
};
|
|
1303
|
+
|
|
1003
1304
|
// src/audio/index.ts
|
|
1004
|
-
|
|
1005
|
-
|
|
1305
|
+
var globalScope = globalThis;
|
|
1306
|
+
if (!globalScope.micdropMic) {
|
|
1307
|
+
globalScope.micdropMic = new MicController();
|
|
1006
1308
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
window.micdropSpeaker = new Speaker();
|
|
1309
|
+
if (!globalScope.micdropSpeaker) {
|
|
1310
|
+
globalScope.micdropSpeaker = new SpeakerController();
|
|
1010
1311
|
}
|
|
1011
|
-
var
|
|
1312
|
+
var Mic = globalScope.micdropMic;
|
|
1313
|
+
var Speaker = globalScope.micdropSpeaker;
|
|
1012
1314
|
|
|
1013
1315
|
// src/client/MicdropClient.ts
|
|
1014
|
-
var
|
|
1316
|
+
var import_eventemitter37 = require("eventemitter3");
|
|
1015
1317
|
|
|
1016
1318
|
// ../server/src/types.ts
|
|
1017
1319
|
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
@@ -1049,7 +1351,7 @@ var MicdropClientError = class extends Error {
|
|
|
1049
1351
|
}
|
|
1050
1352
|
};
|
|
1051
1353
|
function getClientErrorFromWSCloseEvent(event) {
|
|
1052
|
-
const code = getClientErrorFromWSCloseEventCode(event.code);
|
|
1354
|
+
const code = getClientErrorFromWSCloseEventCode(event.code ?? 1006);
|
|
1053
1355
|
return code && new MicdropClientError(code, event.reason);
|
|
1054
1356
|
}
|
|
1055
1357
|
function getClientErrorFromWSCloseEventCode(code) {
|
|
@@ -1079,7 +1381,7 @@ var DEFAULT_RECONNECT_OPTIONS = {
|
|
|
1079
1381
|
delayMs: 1e3,
|
|
1080
1382
|
connectionTimeout: 5e3
|
|
1081
1383
|
};
|
|
1082
|
-
var MicdropClient = class extends
|
|
1384
|
+
var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
1083
1385
|
constructor(options = {}) {
|
|
1084
1386
|
super();
|
|
1085
1387
|
this.options = options;
|
|
@@ -1104,7 +1406,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1104
1406
|
this._isReconnecting = false;
|
|
1105
1407
|
this.reconnectAttempt = 0;
|
|
1106
1408
|
if (this.reconnectTimer) {
|
|
1107
|
-
|
|
1409
|
+
clearTimeout(this.reconnectTimer);
|
|
1108
1410
|
this.reconnectTimer = void 0;
|
|
1109
1411
|
}
|
|
1110
1412
|
if (!this.micRecorder || options?.vad) {
|
|
@@ -1118,7 +1420,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1118
1420
|
this._isPaused = false;
|
|
1119
1421
|
this._isReconnecting = false;
|
|
1120
1422
|
if (this.reconnectTimer) {
|
|
1121
|
-
|
|
1423
|
+
clearTimeout(this.reconnectTimer);
|
|
1122
1424
|
this.reconnectTimer = void 0;
|
|
1123
1425
|
}
|
|
1124
1426
|
try {
|
|
@@ -1126,13 +1428,12 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1126
1428
|
} catch (error) {
|
|
1127
1429
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1128
1430
|
}
|
|
1431
|
+
Speaker.stopAudio();
|
|
1129
1432
|
try {
|
|
1130
|
-
this.
|
|
1131
|
-
this.stopMic();
|
|
1433
|
+
await this.stopMic();
|
|
1132
1434
|
} catch (error) {
|
|
1133
1435
|
console.error("[MicdropClient] Error stopping microphone", error);
|
|
1134
1436
|
}
|
|
1135
|
-
Speaker2.stopAudio();
|
|
1136
1437
|
};
|
|
1137
1438
|
this.mute = () => {
|
|
1138
1439
|
this.vad?.pause();
|
|
@@ -1140,7 +1441,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1140
1441
|
this.notifyStateChange();
|
|
1141
1442
|
};
|
|
1142
1443
|
this.unmute = () => {
|
|
1143
|
-
if (!this.isPaused && !(this.options.disableInterruption &&
|
|
1444
|
+
if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
|
|
1144
1445
|
this.vad?.resume();
|
|
1145
1446
|
}
|
|
1146
1447
|
this._isMuted = false;
|
|
@@ -1152,7 +1453,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1152
1453
|
this._isPaused = true;
|
|
1153
1454
|
this._isProcessing = false;
|
|
1154
1455
|
this.notifyStateChange();
|
|
1155
|
-
|
|
1456
|
+
Speaker.stopAudio();
|
|
1156
1457
|
this.ws?.send("Mute" /* Mute */);
|
|
1157
1458
|
};
|
|
1158
1459
|
this.resume = () => {
|
|
@@ -1180,16 +1481,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1180
1481
|
this.micRecorder.on("StateChange", () => {
|
|
1181
1482
|
this.notifyStateChange();
|
|
1182
1483
|
});
|
|
1183
|
-
this.micRecorder.on("Chunk", (
|
|
1184
|
-
this.log(
|
|
1185
|
-
this.ws?.send(
|
|
1484
|
+
this.micRecorder.on("Chunk", (chunk) => {
|
|
1485
|
+
this.log("User audio chunk", chunk.length);
|
|
1486
|
+
this.ws?.send(pcm16ToArrayBuffer(chunk));
|
|
1186
1487
|
});
|
|
1187
1488
|
this.micRecorder.on("StartSpeaking", () => {
|
|
1188
1489
|
this.log("User start speaking");
|
|
1189
1490
|
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
1190
1491
|
this._isProcessing = false;
|
|
1191
1492
|
this.notifyStateChange();
|
|
1192
|
-
|
|
1493
|
+
Speaker.stopAudio();
|
|
1193
1494
|
});
|
|
1194
1495
|
this.micRecorder.on("StopSpeaking", () => {
|
|
1195
1496
|
this.log("User stop speaking");
|
|
@@ -1200,11 +1501,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1200
1501
|
this.notifyStateChange();
|
|
1201
1502
|
});
|
|
1202
1503
|
}
|
|
1203
|
-
const
|
|
1204
|
-
|
|
1205
|
-
|
|
1504
|
+
const deviceId = options.deviceId ?? storage.getItem("micdrop.micDevice" /* MicDevice */) ?? void 0;
|
|
1505
|
+
await Mic.start(deviceId);
|
|
1506
|
+
if (Mic.deviceId) {
|
|
1507
|
+
storage.setItem("micdrop.micDevice" /* MicDevice */, Mic.deviceId);
|
|
1508
|
+
} else {
|
|
1509
|
+
storage.removeItem("micdrop.micDevice" /* MicDevice */);
|
|
1510
|
+
}
|
|
1511
|
+
await this.micRecorder.start(Mic);
|
|
1512
|
+
await Speaker.start();
|
|
1206
1513
|
await this.updateDevices();
|
|
1207
|
-
await Speaker2.start();
|
|
1208
1514
|
this.notifyStateChange();
|
|
1209
1515
|
} catch (error) {
|
|
1210
1516
|
this.setError(
|
|
@@ -1221,37 +1527,22 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1221
1527
|
await this.startMic({ deviceId });
|
|
1222
1528
|
};
|
|
1223
1529
|
this.changeSpeakerDevice = async (deviceId) => {
|
|
1224
|
-
await
|
|
1530
|
+
await Speaker.changeDevice(deviceId);
|
|
1225
1531
|
this.notifyStateChange();
|
|
1226
1532
|
};
|
|
1227
1533
|
this.updateDevices = async () => {
|
|
1228
|
-
const
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
const defaultDevicesIndexes = defaultDevices.map(
|
|
1235
|
-
(device) => devices.findIndex(
|
|
1236
|
-
(d) => d.groupId === device.groupId && d.kind === device.kind
|
|
1237
|
-
)
|
|
1238
|
-
).filter((index) => index !== -1);
|
|
1239
|
-
for (const index of defaultDevicesIndexes) {
|
|
1240
|
-
const device = devices[index];
|
|
1241
|
-
devices.splice(index, 1);
|
|
1242
|
-
devices.unshift(device);
|
|
1243
|
-
}
|
|
1244
|
-
this.micDevices = devices.filter((device) => device.kind === "audioinput");
|
|
1245
|
-
this.speakerDevices = devices.filter(
|
|
1246
|
-
(device) => device.kind === "audiooutput"
|
|
1247
|
-
);
|
|
1248
|
-
Speaker2.devices = this.speakerDevices;
|
|
1534
|
+
const [micDevices, speakerDevices] = await Promise.all([
|
|
1535
|
+
Mic.getDevices(),
|
|
1536
|
+
Speaker.getDevices()
|
|
1537
|
+
]);
|
|
1538
|
+
this.micDevices = micDevices;
|
|
1539
|
+
this.speakerDevices = speakerDevices;
|
|
1249
1540
|
this.notifyStateChange();
|
|
1250
1541
|
};
|
|
1251
1542
|
this.onWSOpen = () => {
|
|
1252
1543
|
this.log("WebSocket opened");
|
|
1253
1544
|
if (this.connectionTimer) {
|
|
1254
|
-
|
|
1545
|
+
clearTimeout(this.connectionTimer);
|
|
1255
1546
|
this.connectionTimer = void 0;
|
|
1256
1547
|
}
|
|
1257
1548
|
this._isReconnecting = false;
|
|
@@ -1266,36 +1557,36 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1266
1557
|
}
|
|
1267
1558
|
};
|
|
1268
1559
|
this.onWSMessage = (event) => {
|
|
1269
|
-
|
|
1270
|
-
if (
|
|
1560
|
+
const data = event.data;
|
|
1561
|
+
if (data instanceof ArrayBuffer) {
|
|
1562
|
+
this.log("Received audio", data.byteLength);
|
|
1271
1563
|
if (this.isPaused || this.isUserSpeaking) return;
|
|
1272
|
-
|
|
1564
|
+
Speaker.playAudio(new Int16Array(data));
|
|
1273
1565
|
this._isProcessing = false;
|
|
1274
1566
|
this.notifyStateChange();
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1567
|
+
return;
|
|
1568
|
+
}
|
|
1569
|
+
if (typeof data !== "string") {
|
|
1570
|
+
console.warn(`[MicdropClient] Unknown message type: ${data}`);
|
|
1571
|
+
return;
|
|
1572
|
+
}
|
|
1573
|
+
this.log("Received message:", data);
|
|
1574
|
+
if (data.startsWith("Message" /* Message */)) {
|
|
1278
1575
|
try {
|
|
1279
1576
|
const message = JSON.parse(
|
|
1280
|
-
|
|
1577
|
+
data.substring("Message" /* Message */.length + 1)
|
|
1281
1578
|
);
|
|
1282
1579
|
this.addMessage(message);
|
|
1283
1580
|
} catch (error) {
|
|
1284
|
-
console.error(
|
|
1285
|
-
"[MicdropClient] Error parsing message:",
|
|
1286
|
-
event.data,
|
|
1287
|
-
error
|
|
1288
|
-
);
|
|
1581
|
+
console.error("[MicdropClient] Error parsing message:", data, error);
|
|
1289
1582
|
}
|
|
1290
|
-
} else if (
|
|
1583
|
+
} else if (data === "EndCall" /* EndCall */) {
|
|
1291
1584
|
this.emit("EndCall");
|
|
1292
|
-
} else if (
|
|
1585
|
+
} else if (data === "SkipAnswer" /* SkipAnswer */) {
|
|
1293
1586
|
this._isProcessing = false;
|
|
1294
1587
|
this.notifyStateChange();
|
|
1295
|
-
} else if (
|
|
1296
|
-
const lastMessage = this.conversation.
|
|
1297
|
-
(message) => message.role === "user"
|
|
1298
|
-
);
|
|
1588
|
+
} else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
1589
|
+
const lastMessage = this.conversation.map((message) => message.role).lastIndexOf("user");
|
|
1299
1590
|
if (lastMessage !== -1) {
|
|
1300
1591
|
this.conversation = this.conversation.filter(
|
|
1301
1592
|
(_, index) => index !== lastMessage
|
|
@@ -1303,24 +1594,20 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1303
1594
|
this._isProcessing = false;
|
|
1304
1595
|
this.notifyStateChange();
|
|
1305
1596
|
}
|
|
1306
|
-
} else if (
|
|
1597
|
+
} else if (data.startsWith("ToolCall" /* ToolCall */)) {
|
|
1307
1598
|
try {
|
|
1308
1599
|
const toolCall = JSON.parse(
|
|
1309
|
-
|
|
1600
|
+
data.substring("ToolCall" /* ToolCall */.length + 1)
|
|
1310
1601
|
);
|
|
1311
1602
|
this.emit("ToolCall", toolCall);
|
|
1312
1603
|
} catch (error) {
|
|
1313
|
-
console.error(
|
|
1314
|
-
"[MicdropClient] Error parsing tool call:",
|
|
1315
|
-
event.data,
|
|
1316
|
-
error
|
|
1317
|
-
);
|
|
1604
|
+
console.error("[MicdropClient] Error parsing tool call:", data, error);
|
|
1318
1605
|
}
|
|
1319
1606
|
}
|
|
1320
1607
|
};
|
|
1321
1608
|
this.onWSClose = (event) => {
|
|
1322
|
-
this.log("WebSocket closed", event);
|
|
1323
|
-
const error = getClientErrorFromWSCloseEvent(event);
|
|
1609
|
+
this.log("WebSocket closed", event?.code, event?.reason);
|
|
1610
|
+
const error = getClientErrorFromWSCloseEvent(event ?? {});
|
|
1324
1611
|
if (error) {
|
|
1325
1612
|
this.setError(error);
|
|
1326
1613
|
}
|
|
@@ -1334,7 +1621,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1334
1621
|
this.stop();
|
|
1335
1622
|
}
|
|
1336
1623
|
};
|
|
1337
|
-
this.onWSError = (
|
|
1624
|
+
this.onWSError = () => {
|
|
1338
1625
|
this.setError(new MicdropClientError("Connection" /* Connection */));
|
|
1339
1626
|
};
|
|
1340
1627
|
this.onSpeakerStartPlaying = () => {
|
|
@@ -1357,9 +1644,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1357
1644
|
this.notifyStateChange();
|
|
1358
1645
|
}
|
|
1359
1646
|
};
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1647
|
+
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1648
|
+
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1649
|
+
Mic.on("DeviceChange", this.updateDevices);
|
|
1363
1650
|
}
|
|
1364
1651
|
get vad() {
|
|
1365
1652
|
return this.micRecorder?.vad;
|
|
@@ -1386,13 +1673,13 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1386
1673
|
return this.isMicStarted && !this.isPaused && !this.isProcessing && !this.isMicMuted && !this.isUserSpeaking && !this.isAssistantSpeaking;
|
|
1387
1674
|
}
|
|
1388
1675
|
get isWSStarted() {
|
|
1389
|
-
return this.ws?.readyState ===
|
|
1676
|
+
return this.ws?.readyState === 1;
|
|
1390
1677
|
}
|
|
1391
1678
|
get isWSStarting() {
|
|
1392
|
-
return this.ws?.readyState ===
|
|
1679
|
+
return this.ws?.readyState === 0;
|
|
1393
1680
|
}
|
|
1394
1681
|
get isMicStarted() {
|
|
1395
|
-
return
|
|
1682
|
+
return Mic.isStarted;
|
|
1396
1683
|
}
|
|
1397
1684
|
get isMicMuted() {
|
|
1398
1685
|
return this.vad?.isPaused ?? false;
|
|
@@ -1401,13 +1688,13 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1401
1688
|
return this.micRecorder?.state.isSpeaking ?? false;
|
|
1402
1689
|
}
|
|
1403
1690
|
get isAssistantSpeaking() {
|
|
1404
|
-
return
|
|
1691
|
+
return Speaker.isPlaying;
|
|
1405
1692
|
}
|
|
1406
1693
|
get micDeviceId() {
|
|
1407
|
-
return
|
|
1694
|
+
return Mic.deviceId;
|
|
1408
1695
|
}
|
|
1409
1696
|
get speakerDeviceId() {
|
|
1410
|
-
return
|
|
1697
|
+
return Speaker.deviceId;
|
|
1411
1698
|
}
|
|
1412
1699
|
get state() {
|
|
1413
1700
|
return {
|
|
@@ -1430,11 +1717,10 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1430
1717
|
speakerDevices: this.speakerDevices
|
|
1431
1718
|
};
|
|
1432
1719
|
}
|
|
1433
|
-
stopMic() {
|
|
1720
|
+
async stopMic() {
|
|
1434
1721
|
this.micRecorder?.stop();
|
|
1435
1722
|
this.micRecorder = void 0;
|
|
1436
|
-
|
|
1437
|
-
this.micStream = void 0;
|
|
1723
|
+
await Mic.stop();
|
|
1438
1724
|
this.notifyStateChange();
|
|
1439
1725
|
}
|
|
1440
1726
|
async startWS() {
|
|
@@ -1450,22 +1736,22 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1450
1736
|
throw new MicdropClientError("MissingUrl" /* MissingUrl */);
|
|
1451
1737
|
}
|
|
1452
1738
|
this.ws = new WebSocket(this.options.url);
|
|
1453
|
-
this.ws.binaryType = "
|
|
1739
|
+
this.ws.binaryType = "arraybuffer";
|
|
1454
1740
|
this.notifyStateChange();
|
|
1455
1741
|
this.ws.onopen = this.onWSOpen;
|
|
1456
1742
|
this.ws.onmessage = this.onWSMessage;
|
|
1457
1743
|
this.ws.onclose = this.onWSClose;
|
|
1458
1744
|
this.ws.onerror = this.onWSError;
|
|
1459
|
-
this.connectionTimer =
|
|
1745
|
+
this.connectionTimer = setTimeout(() => {
|
|
1460
1746
|
this.connectionTimer = void 0;
|
|
1461
|
-
if (this.ws?.readyState ===
|
|
1747
|
+
if (this.ws?.readyState === 0) {
|
|
1462
1748
|
this.log("WebSocket connection timeout");
|
|
1463
1749
|
this.ws.close();
|
|
1464
1750
|
}
|
|
1465
1751
|
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
1466
1752
|
} catch (error) {
|
|
1467
1753
|
this.setError(
|
|
1468
|
-
new MicdropClientError(
|
|
1754
|
+
error instanceof MicdropClientError ? error : new MicdropClientError(
|
|
1469
1755
|
"Connection" /* Connection */,
|
|
1470
1756
|
error?.message
|
|
1471
1757
|
)
|
|
@@ -1476,11 +1762,11 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1476
1762
|
}
|
|
1477
1763
|
stopWS() {
|
|
1478
1764
|
if (this.connectionTimer) {
|
|
1479
|
-
|
|
1765
|
+
clearTimeout(this.connectionTimer);
|
|
1480
1766
|
this.connectionTimer = void 0;
|
|
1481
1767
|
}
|
|
1482
1768
|
if (!this.ws) return;
|
|
1483
|
-
if (this.ws.readyState ===
|
|
1769
|
+
if (this.ws.readyState === 1 || this.ws.readyState === 0) {
|
|
1484
1770
|
this.ws.close();
|
|
1485
1771
|
}
|
|
1486
1772
|
this.ws = void 0;
|
|
@@ -1500,9 +1786,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1500
1786
|
this.vad?.pause?.();
|
|
1501
1787
|
}
|
|
1502
1788
|
if (this.reconnectTimer) {
|
|
1503
|
-
|
|
1789
|
+
clearTimeout(this.reconnectTimer);
|
|
1504
1790
|
}
|
|
1505
|
-
this.reconnectTimer =
|
|
1791
|
+
this.reconnectTimer = setTimeout(async () => {
|
|
1506
1792
|
try {
|
|
1507
1793
|
await this.startWS();
|
|
1508
1794
|
} catch {
|
|
@@ -1532,14 +1818,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1532
1818
|
};
|
|
1533
1819
|
|
|
1534
1820
|
// src/Micdrop.ts
|
|
1535
|
-
|
|
1536
|
-
|
|
1821
|
+
var globalScope2 = globalThis;
|
|
1822
|
+
if (!globalScope2.micdropClient) {
|
|
1823
|
+
globalScope2.micdropClient = new MicdropClient();
|
|
1537
1824
|
}
|
|
1538
|
-
var Micdrop =
|
|
1825
|
+
var Micdrop = globalScope2.micdropClient;
|
|
1539
1826
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1540
1827
|
0 && (module.exports = {
|
|
1541
|
-
LocalStorageKeys,
|
|
1542
1828
|
Mic,
|
|
1829
|
+
MicController,
|
|
1830
|
+
MicDriver,
|
|
1543
1831
|
MicRecorder,
|
|
1544
1832
|
Micdrop,
|
|
1545
1833
|
MicdropClient,
|
|
@@ -1547,16 +1835,33 @@ var Micdrop = window.micdropClient;
|
|
|
1547
1835
|
MicdropClientError,
|
|
1548
1836
|
MicdropClientErrorCode,
|
|
1549
1837
|
MicdropServerCommands,
|
|
1838
|
+
MicdropStorageKeys,
|
|
1550
1839
|
MultipleVAD,
|
|
1840
|
+
Pcm16AudioStream,
|
|
1841
|
+
SILERO_FRAME_SAMPLES,
|
|
1842
|
+
SILERO_SAMPLE_RATE,
|
|
1551
1843
|
SileroVAD,
|
|
1552
1844
|
Speaker,
|
|
1845
|
+
SpeakerController,
|
|
1846
|
+
SpeakerDriver,
|
|
1553
1847
|
VAD,
|
|
1554
1848
|
VADStatus,
|
|
1849
|
+
VOLUME_SAMPLE_RATE,
|
|
1850
|
+
VolumeMeter,
|
|
1555
1851
|
VolumeVAD,
|
|
1852
|
+
concatFloat32,
|
|
1556
1853
|
equalVADConfig,
|
|
1854
|
+
floatToPcm16,
|
|
1557
1855
|
getClientErrorFromWSCloseEvent,
|
|
1558
1856
|
getClientErrorFromWSCloseEventCode,
|
|
1559
1857
|
getVAD,
|
|
1560
|
-
|
|
1858
|
+
getVolume,
|
|
1859
|
+
isRecoverableError,
|
|
1860
|
+
pcm16ToArrayBuffer,
|
|
1861
|
+
pcm16ToFloat,
|
|
1862
|
+
resample,
|
|
1863
|
+
setMicdropStorage,
|
|
1864
|
+
setSileroModelLoader,
|
|
1865
|
+
storage
|
|
1561
1866
|
});
|
|
1562
1867
|
//# sourceMappingURL=index.js.map
|