@micdrop/client 2.2.10 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -23
- package/dist/index.d.mts +444 -73
- package/dist/index.d.ts +444 -73
- package/dist/index.js +983 -691
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +963 -689
- package/dist/index.mjs.map +1 -1
- package/package.json +10 -6
package/dist/index.js
CHANGED
|
@@ -20,8 +20,9 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
Mic: () => Mic,
|
|
24
|
+
MicController: () => MicController,
|
|
25
|
+
MicDriver: () => MicDriver,
|
|
25
26
|
MicRecorder: () => MicRecorder,
|
|
26
27
|
Micdrop: () => Micdrop,
|
|
27
28
|
MicdropClient: () => MicdropClient,
|
|
@@ -29,454 +30,486 @@ __export(index_exports, {
|
|
|
29
30
|
MicdropClientError: () => MicdropClientError,
|
|
30
31
|
MicdropClientErrorCode: () => MicdropClientErrorCode,
|
|
31
32
|
MicdropServerCommands: () => MicdropServerCommands,
|
|
33
|
+
MicdropStorageKeys: () => MicdropStorageKeys,
|
|
32
34
|
MultipleVAD: () => MultipleVAD,
|
|
35
|
+
Pcm16AudioStream: () => Pcm16AudioStream,
|
|
36
|
+
SILERO_FRAME_SAMPLES: () => SILERO_FRAME_SAMPLES,
|
|
37
|
+
SILERO_SAMPLE_RATE: () => SILERO_SAMPLE_RATE,
|
|
33
38
|
SileroVAD: () => SileroVAD,
|
|
34
|
-
Speaker: () =>
|
|
39
|
+
Speaker: () => Speaker,
|
|
40
|
+
SpeakerController: () => SpeakerController,
|
|
41
|
+
SpeakerDriver: () => SpeakerDriver,
|
|
35
42
|
VAD: () => VAD,
|
|
36
43
|
VADStatus: () => VADStatus,
|
|
44
|
+
VOLUME_SAMPLE_RATE: () => VOLUME_SAMPLE_RATE,
|
|
45
|
+
VolumeMeter: () => VolumeMeter,
|
|
37
46
|
VolumeVAD: () => VolumeVAD,
|
|
47
|
+
concatFloat32: () => concatFloat32,
|
|
38
48
|
equalVADConfig: () => equalVADConfig,
|
|
49
|
+
floatToPcm16: () => floatToPcm16,
|
|
39
50
|
getClientErrorFromWSCloseEvent: () => getClientErrorFromWSCloseEvent,
|
|
40
51
|
getClientErrorFromWSCloseEventCode: () => getClientErrorFromWSCloseEventCode,
|
|
41
52
|
getVAD: () => getVAD,
|
|
42
|
-
|
|
53
|
+
getVolume: () => getVolume,
|
|
54
|
+
isRecoverableError: () => isRecoverableError,
|
|
55
|
+
pcm16ToArrayBuffer: () => pcm16ToArrayBuffer,
|
|
56
|
+
pcm16ToFloat: () => pcm16ToFloat,
|
|
57
|
+
resample: () => resample,
|
|
58
|
+
setMicdropStorage: () => setMicdropStorage,
|
|
59
|
+
setSileroModelLoader: () => setSileroModelLoader,
|
|
60
|
+
storage: () => storage
|
|
43
61
|
});
|
|
44
62
|
module.exports = __toCommonJS(index_exports);
|
|
45
63
|
|
|
46
|
-
// src/audio/
|
|
64
|
+
// src/audio/Mic.ts
|
|
47
65
|
var import_eventemitter3 = require("eventemitter3");
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
66
|
+
|
|
67
|
+
// src/audio/pcm.ts
|
|
68
|
+
function resample(input, inputRate, outputRate) {
|
|
69
|
+
if (inputRate === outputRate || input.length === 0) return input;
|
|
70
|
+
const ratio = inputRate / outputRate;
|
|
71
|
+
const outputLength = Math.round(input.length / ratio);
|
|
72
|
+
const output = new Float32Array(outputLength);
|
|
73
|
+
for (let i = 0; i < outputLength; i++) {
|
|
74
|
+
const position = i * ratio;
|
|
75
|
+
const index = Math.floor(position);
|
|
76
|
+
const nextIndex = Math.min(index + 1, input.length - 1);
|
|
77
|
+
const fraction = position - index;
|
|
78
|
+
output[i] = input[index] * (1 - fraction) + input[nextIndex] * fraction;
|
|
79
|
+
}
|
|
80
|
+
return output;
|
|
81
|
+
}
|
|
82
|
+
function floatToPcm16(input) {
|
|
83
|
+
const output = new Int16Array(input.length);
|
|
84
|
+
for (let i = 0; i < input.length; i++) {
|
|
85
|
+
const sample = Math.max(-1, Math.min(1, input[i]));
|
|
86
|
+
output[i] = sample < 0 ? sample * 32768 : sample * 32767;
|
|
67
87
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
maxVolume = this.fftBins[i];
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return maxVolume;
|
|
88
|
+
return output;
|
|
89
|
+
}
|
|
90
|
+
function pcm16ToFloat(input) {
|
|
91
|
+
const output = new Float32Array(input.length);
|
|
92
|
+
for (let i = 0; i < input.length; i++) {
|
|
93
|
+
output[i] = input[i] / 32768;
|
|
78
94
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
// src/audio/utils/unlock.ts
|
|
82
|
-
function unlock(audioContext2) {
|
|
83
|
-
const events = ["touchstart", "touchend", "mousedown", "keydown"];
|
|
84
|
-
const unlock2 = async () => {
|
|
85
|
-
unlockSafari(audioContext2);
|
|
86
|
-
if (audioContext2.state === "suspended") {
|
|
87
|
-
await audioContext2.resume();
|
|
88
|
-
}
|
|
89
|
-
events.forEach((e) => document.body.removeEventListener(e, unlock2));
|
|
90
|
-
};
|
|
91
|
-
events.forEach((e) => document.body.addEventListener(e, unlock2, false));
|
|
95
|
+
return output;
|
|
92
96
|
}
|
|
93
|
-
function
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
97
|
+
function concatFloat32(chunks) {
|
|
98
|
+
if (chunks.length === 1) return chunks[0];
|
|
99
|
+
let length = 0;
|
|
100
|
+
for (const chunk of chunks) length += chunk.length;
|
|
101
|
+
const output = new Float32Array(length);
|
|
102
|
+
let offset = 0;
|
|
103
|
+
for (const chunk of chunks) {
|
|
104
|
+
output.set(chunk, offset);
|
|
105
|
+
offset += chunk.length;
|
|
106
|
+
}
|
|
107
|
+
return output;
|
|
108
|
+
}
|
|
109
|
+
function pcm16ToArrayBuffer(pcm) {
|
|
110
|
+
return pcm.buffer.slice(
|
|
111
|
+
pcm.byteOffset,
|
|
112
|
+
pcm.byteOffset + pcm.byteLength
|
|
113
|
+
);
|
|
104
114
|
}
|
|
105
115
|
|
|
106
|
-
// src/audio/
|
|
107
|
-
var
|
|
108
|
-
|
|
109
|
-
|
|
116
|
+
// src/audio/volume.ts
|
|
117
|
+
var FFT_SIZE = 512;
|
|
118
|
+
var BIN_COUNT = FFT_SIZE / 2;
|
|
119
|
+
var FIRST_BIN = 4;
|
|
120
|
+
var SMOOTHING = 0.1;
|
|
121
|
+
var VOLUME_SAMPLE_RATE = 16e3;
|
|
122
|
+
var WINDOW = new Float32Array(FFT_SIZE);
|
|
123
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
124
|
+
const a = 2 * Math.PI * i / (FFT_SIZE - 1);
|
|
125
|
+
WINDOW[i] = 0.42 - 0.5 * Math.cos(a) + 0.08 * Math.cos(2 * a);
|
|
110
126
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
127
|
+
var REVERSED = new Uint16Array(FFT_SIZE);
|
|
128
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
129
|
+
let reversed = 0;
|
|
130
|
+
for (let bit = 1; bit < FFT_SIZE; bit <<= 1) {
|
|
131
|
+
reversed = reversed << 1 | (i & bit ? 1 : 0);
|
|
132
|
+
}
|
|
133
|
+
REVERSED[i] = reversed;
|
|
116
134
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
LocalStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
123
|
-
LocalStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
124
|
-
return LocalStorageKeys2;
|
|
125
|
-
})(LocalStorageKeys || {});
|
|
126
|
-
|
|
127
|
-
// src/audio/utils/stopStream.ts
|
|
128
|
-
function stopStream(stream) {
|
|
129
|
-
stream?.getTracks().forEach((track) => track.stop());
|
|
135
|
+
var COS = new Float32Array(FFT_SIZE / 2);
|
|
136
|
+
var SIN = new Float32Array(FFT_SIZE / 2);
|
|
137
|
+
for (let i = 0; i < FFT_SIZE / 2; i++) {
|
|
138
|
+
COS[i] = Math.cos(-2 * Math.PI * i / FFT_SIZE);
|
|
139
|
+
SIN[i] = Math.sin(-2 * Math.PI * i / FFT_SIZE);
|
|
130
140
|
}
|
|
131
|
-
|
|
132
|
-
// src/audio/Mic.ts
|
|
133
|
-
var Mic = class {
|
|
141
|
+
var VolumeMeter = class {
|
|
134
142
|
constructor() {
|
|
135
|
-
this.
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
deviceId: { ideal: deviceId },
|
|
141
|
-
sampleRate: 16e3,
|
|
142
|
-
// not working, it will follow device settings, usually 44.1kHz
|
|
143
|
-
sampleSize: 16,
|
|
144
|
-
channelCount: 1,
|
|
145
|
-
echoCancellation: true,
|
|
146
|
-
noiseSuppression: true
|
|
147
|
-
}
|
|
148
|
-
};
|
|
143
|
+
this.real = new Float32Array(FFT_SIZE);
|
|
144
|
+
this.imaginary = new Float32Array(FFT_SIZE);
|
|
145
|
+
this.magnitudes = new Float32Array(BIN_COUNT);
|
|
146
|
+
this.smoothed = new Float32Array(BIN_COUNT);
|
|
147
|
+
this.hasPrevious = false;
|
|
149
148
|
}
|
|
150
149
|
/**
|
|
151
|
-
*
|
|
152
|
-
* @param
|
|
153
|
-
*
|
|
150
|
+
* Measures a window of audio
|
|
151
|
+
* @param samples - At least 512 mono samples at 16 kHz, in the -1..1 range.
|
|
152
|
+
* Only the last 512 are looked at.
|
|
153
|
+
* @returns The level in decibels, between -Infinity (silence) and 0
|
|
154
154
|
*/
|
|
155
|
-
|
|
156
|
-
if (
|
|
157
|
-
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
|
|
155
|
+
measure(samples) {
|
|
156
|
+
if (samples.length === 0) return -Infinity;
|
|
157
|
+
const offset = Math.max(0, samples.length - FFT_SIZE);
|
|
158
|
+
const { real, imaginary } = this;
|
|
159
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
160
|
+
const sample = offset + i < samples.length ? samples[offset + i] : 0;
|
|
161
|
+
real[REVERSED[i]] = sample * WINDOW[i];
|
|
162
|
+
imaginary[REVERSED[i]] = 0;
|
|
161
163
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
164
|
+
for (let size = 2; size <= FFT_SIZE; size <<= 1) {
|
|
165
|
+
const half = size >> 1;
|
|
166
|
+
const step = FFT_SIZE / size;
|
|
167
|
+
for (let start = 0; start < FFT_SIZE; start += size) {
|
|
168
|
+
for (let i = 0; i < half; i++) {
|
|
169
|
+
const even = start + i;
|
|
170
|
+
const odd = even + half;
|
|
171
|
+
const cos = COS[i * step];
|
|
172
|
+
const sin = SIN[i * step];
|
|
173
|
+
const realOdd = real[odd] * cos - imaginary[odd] * sin;
|
|
174
|
+
const imaginaryOdd = real[odd] * sin + imaginary[odd] * cos;
|
|
175
|
+
real[odd] = real[even] - realOdd;
|
|
176
|
+
imaginary[odd] = imaginary[even] - imaginaryOdd;
|
|
177
|
+
real[even] += realOdd;
|
|
178
|
+
imaginary[even] += imaginaryOdd;
|
|
179
|
+
}
|
|
165
180
|
}
|
|
166
|
-
stopStream(this.audioStream);
|
|
167
181
|
}
|
|
168
|
-
|
|
169
|
-
this.
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (newDeviceId === "default" && deviceId) {
|
|
175
|
-
newDeviceId = deviceId;
|
|
182
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
183
|
+
this.magnitudes[k] = Math.sqrt(real[k] * real[k] + imaginary[k] * imaginary[k]) / FFT_SIZE;
|
|
184
|
+
}
|
|
185
|
+
if (this.hasPrevious) {
|
|
186
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
187
|
+
this.smoothed[k] = SMOOTHING * this.smoothed[k] + (1 - SMOOTHING) * this.magnitudes[k];
|
|
176
188
|
}
|
|
177
|
-
localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
|
|
178
|
-
this.deviceId = newDeviceId;
|
|
179
189
|
} else {
|
|
180
|
-
|
|
181
|
-
this.
|
|
190
|
+
this.smoothed.set(this.magnitudes);
|
|
191
|
+
this.hasPrevious = true;
|
|
192
|
+
}
|
|
193
|
+
let loudest = -Infinity;
|
|
194
|
+
for (let k = FIRST_BIN; k < BIN_COUNT; k++) {
|
|
195
|
+
const magnitude = this.smoothed[k];
|
|
196
|
+
if (magnitude === 0) continue;
|
|
197
|
+
const decibels = 20 * Math.log10(magnitude);
|
|
198
|
+
if (decibels > loudest && decibels < 0) loudest = decibels;
|
|
182
199
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
200
|
+
return loudest;
|
|
201
|
+
}
|
|
202
|
+
/** Forgets the previous window, for when capture stops and starts again */
|
|
203
|
+
reset() {
|
|
204
|
+
this.hasPrevious = false;
|
|
205
|
+
this.smoothed.fill(0);
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
function getVolume(samples) {
|
|
209
|
+
return new VolumeMeter().measure(samples);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// src/audio/Mic.ts
|
|
213
|
+
var VOLUME_INTERVAL = 100;
|
|
214
|
+
var VOLUME_WINDOW = 512;
|
|
215
|
+
var RATE_CHECK_DURATION = 2e3;
|
|
216
|
+
var RATE_TOLERANCE = 0.15;
|
|
217
|
+
var MicController = class extends import_eventemitter3.EventEmitter {
|
|
218
|
+
constructor() {
|
|
219
|
+
super(...arguments);
|
|
220
|
+
this.meter = new VolumeMeter();
|
|
221
|
+
this.pending = [];
|
|
222
|
+
this.pendingLength = 0;
|
|
223
|
+
this.pendingRate = VOLUME_SAMPLE_RATE;
|
|
224
|
+
this.rateCheckedAt = 0;
|
|
225
|
+
this.rateCheckSamples = 0;
|
|
226
|
+
this.rateWarned = false;
|
|
227
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
228
|
+
this.volume = -Infinity;
|
|
229
|
+
this.onFrames = (frames, sampleRate) => {
|
|
230
|
+
this.emit("Frames", frames, sampleRate);
|
|
231
|
+
this.checkRate(frames.length, sampleRate);
|
|
232
|
+
if (sampleRate !== this.pendingRate) {
|
|
233
|
+
this.pending = [];
|
|
234
|
+
this.pendingLength = 0;
|
|
235
|
+
this.pendingRate = sampleRate;
|
|
236
|
+
this.meter.reset();
|
|
237
|
+
}
|
|
238
|
+
this.pending.push(frames);
|
|
239
|
+
this.pendingLength += frames.length;
|
|
240
|
+
const interval = Math.round(VOLUME_INTERVAL / 1e3 * sampleRate);
|
|
241
|
+
if (this.pendingLength < interval) return;
|
|
242
|
+
const needed = Math.ceil(VOLUME_WINDOW * sampleRate / VOLUME_SAMPLE_RATE);
|
|
243
|
+
const gathered = concatFloat32(this.pending);
|
|
244
|
+
const window = gathered.subarray(Math.max(0, gathered.length - needed));
|
|
245
|
+
this.pending = [];
|
|
246
|
+
this.pendingLength = 0;
|
|
247
|
+
this.volume = this.meter.measure(
|
|
248
|
+
resample(window, sampleRate, VOLUME_SAMPLE_RATE)
|
|
249
|
+
);
|
|
250
|
+
this.emit("Volume", this.volume);
|
|
251
|
+
};
|
|
252
|
+
this.onDeviceChange = () => {
|
|
253
|
+
this.emit("DeviceChange");
|
|
254
|
+
};
|
|
255
|
+
this.onError = (error) => {
|
|
256
|
+
this.emit("Error", error);
|
|
257
|
+
};
|
|
187
258
|
}
|
|
188
259
|
/**
|
|
189
|
-
*
|
|
260
|
+
* Replaces the audio driver, to record with another library
|
|
261
|
+
* @param driver - The driver to record with
|
|
190
262
|
*/
|
|
191
|
-
|
|
192
|
-
if (this.
|
|
193
|
-
|
|
194
|
-
this.
|
|
263
|
+
setDriver(driver) {
|
|
264
|
+
if (this.driver === driver) return;
|
|
265
|
+
if (this.driver) {
|
|
266
|
+
this.driver.off("Frames", this.onFrames);
|
|
267
|
+
this.driver.off("DeviceChange", this.onDeviceChange);
|
|
268
|
+
this.driver.off("Error", this.onError);
|
|
269
|
+
}
|
|
270
|
+
this.driver = driver;
|
|
271
|
+
driver.on("Frames", this.onFrames);
|
|
272
|
+
driver.on("DeviceChange", this.onDeviceChange);
|
|
273
|
+
driver.on("Error", this.onError);
|
|
274
|
+
}
|
|
275
|
+
getDriver() {
|
|
276
|
+
if (!this.driver) {
|
|
277
|
+
throw new Error(
|
|
278
|
+
"No microphone driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Mic.setDriver()."
|
|
279
|
+
);
|
|
195
280
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
281
|
+
return this.driver;
|
|
282
|
+
}
|
|
283
|
+
get isStarted() {
|
|
284
|
+
return this.driver?.isStarted ?? false;
|
|
285
|
+
}
|
|
286
|
+
get deviceId() {
|
|
287
|
+
return this.driver?.deviceId;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Asks for the microphone permission and starts capturing
|
|
291
|
+
* @param deviceId - Input device to record from, defaults to the system one
|
|
292
|
+
*/
|
|
293
|
+
async start(deviceId) {
|
|
294
|
+
this.resetVolume();
|
|
295
|
+
await this.getDriver().start(deviceId);
|
|
296
|
+
}
|
|
297
|
+
/** Stops capturing and releases the microphone */
|
|
298
|
+
async stop() {
|
|
299
|
+
await this.driver?.stop();
|
|
300
|
+
this.resetVolume();
|
|
301
|
+
}
|
|
302
|
+
/** Lists the input devices the system offers */
|
|
303
|
+
async getDevices() {
|
|
304
|
+
if (!this.driver) return [];
|
|
305
|
+
return this.driver.getDevices();
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Times the captured audio against the rate it claims to be at.
|
|
309
|
+
*
|
|
310
|
+
* A microphone that announces 16 kHz while delivering 48 kHz sends speech to
|
|
311
|
+
* the server three times too slow, which comes back as a short transcript in
|
|
312
|
+
* a random language. That is invisible otherwise, so it is worth saying out
|
|
313
|
+
* loud once.
|
|
314
|
+
* @param samples - How many samples just arrived
|
|
315
|
+
* @param sampleRate - The rate they are said to be at
|
|
316
|
+
*/
|
|
317
|
+
checkRate(samples, sampleRate) {
|
|
318
|
+
if (this.rateWarned) return;
|
|
319
|
+
const now = Date.now();
|
|
320
|
+
if (this.rateCheckedAt === 0) {
|
|
321
|
+
this.rateCheckedAt = now;
|
|
322
|
+
return;
|
|
199
323
|
}
|
|
324
|
+
this.rateCheckSamples += samples;
|
|
325
|
+
const elapsed = now - this.rateCheckedAt;
|
|
326
|
+
if (elapsed < RATE_CHECK_DURATION) return;
|
|
327
|
+
const delivered = this.rateCheckSamples / elapsed * 1e3;
|
|
328
|
+
this.rateCheckedAt = now;
|
|
329
|
+
this.rateCheckSamples = 0;
|
|
330
|
+
if (Math.abs(delivered - sampleRate) / sampleRate <= RATE_TOLERANCE) return;
|
|
331
|
+
this.rateWarned = true;
|
|
332
|
+
console.warn(
|
|
333
|
+
`[Micdrop] The microphone announces ${sampleRate} Hz but delivers about ${Math.round(delivered)} Hz. Speech is being sent at the wrong speed, so transcription will be poor.`
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
resetVolume() {
|
|
337
|
+
this.pending = [];
|
|
338
|
+
this.pendingLength = 0;
|
|
339
|
+
this.meter.reset();
|
|
340
|
+
this.volume = -Infinity;
|
|
341
|
+
this.rateCheckedAt = 0;
|
|
342
|
+
this.rateCheckSamples = 0;
|
|
200
343
|
}
|
|
201
344
|
};
|
|
202
345
|
|
|
203
346
|
// src/audio/Speaker.ts
|
|
204
|
-
var import_eventemitter33 = require("eventemitter3");
|
|
205
|
-
|
|
206
|
-
// src/audio/stream/AudioStream.ts
|
|
207
347
|
var import_eventemitter32 = require("eventemitter3");
|
|
208
|
-
var AudioStream = class extends import_eventemitter32.EventEmitter {
|
|
209
|
-
constructor() {
|
|
210
|
-
super();
|
|
211
|
-
this.isPlaying = false;
|
|
212
|
-
this.outputNode = audioContext.createGain();
|
|
213
|
-
}
|
|
214
|
-
setIsPlaying(isPlaying) {
|
|
215
|
-
if (this.isPlaying === isPlaying) return;
|
|
216
|
-
this.isPlaying = isPlaying;
|
|
217
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
218
|
-
}
|
|
219
|
-
};
|
|
220
348
|
|
|
221
|
-
// src/
|
|
222
|
-
var
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
349
|
+
// src/storage.ts
|
|
350
|
+
var MicdropStorageKeys = /* @__PURE__ */ ((MicdropStorageKeys2) => {
|
|
351
|
+
MicdropStorageKeys2["MicDevice"] = "micdrop.micDevice";
|
|
352
|
+
MicdropStorageKeys2["SpeakerDevice"] = "micdrop.speakerDevice";
|
|
353
|
+
MicdropStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
354
|
+
MicdropStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
355
|
+
return MicdropStorageKeys2;
|
|
356
|
+
})(MicdropStorageKeys || {});
|
|
357
|
+
var MemoryStorage = class {
|
|
228
358
|
constructor() {
|
|
229
|
-
|
|
230
|
-
this.blobQueue = [];
|
|
231
|
-
this.processing = false;
|
|
232
|
-
this.sourceNodes = [];
|
|
233
|
-
this.nextStartTime = 0;
|
|
234
|
-
this.prebuffering = true;
|
|
235
|
-
this.prebuffer = [];
|
|
236
|
-
this.prebufferDuration = 0;
|
|
359
|
+
this.values = /* @__PURE__ */ new Map();
|
|
237
360
|
}
|
|
238
|
-
|
|
239
|
-
return this.
|
|
361
|
+
getItem(key) {
|
|
362
|
+
return this.values.get(key) ?? null;
|
|
240
363
|
}
|
|
241
|
-
|
|
242
|
-
this.
|
|
243
|
-
if (this.processing) return;
|
|
244
|
-
this.processing = true;
|
|
245
|
-
try {
|
|
246
|
-
while (this.blobQueue.length > 0) {
|
|
247
|
-
const next = this.blobQueue.shift();
|
|
248
|
-
try {
|
|
249
|
-
const arrayBuffer = await next.arrayBuffer();
|
|
250
|
-
const audioBuffer = this.createAudioBuffer(arrayBuffer);
|
|
251
|
-
if (audioBuffer) this.handleBuffer(audioBuffer);
|
|
252
|
-
} catch (error) {
|
|
253
|
-
console.error("Failed to play audio chunk:", error);
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
} finally {
|
|
257
|
-
this.processing = false;
|
|
258
|
-
}
|
|
364
|
+
setItem(key, value) {
|
|
365
|
+
this.values.set(key, value);
|
|
259
366
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
try {
|
|
263
|
-
node.stop();
|
|
264
|
-
node.disconnect();
|
|
265
|
-
} catch (error) {
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
this.sourceNodes = [];
|
|
269
|
-
this.blobQueue = [];
|
|
270
|
-
this.nextStartTime = 0;
|
|
271
|
-
this.resetPrebuffer();
|
|
272
|
-
this.setIsPlaying(false);
|
|
367
|
+
removeItem(key) {
|
|
368
|
+
this.values.delete(key);
|
|
273
369
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
this.prebuffer.push(audioBuffer);
|
|
286
|
-
this.prebufferDuration += audioBuffer.duration * 1e3;
|
|
287
|
-
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
288
|
-
this.flushPrebuffer();
|
|
289
|
-
return;
|
|
290
|
-
}
|
|
291
|
-
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
292
|
-
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
293
|
-
}
|
|
294
|
-
flushPrebuffer() {
|
|
295
|
-
if (this.quietTimer !== void 0) {
|
|
296
|
-
clearTimeout(this.quietTimer);
|
|
297
|
-
this.quietTimer = void 0;
|
|
298
|
-
}
|
|
299
|
-
this.prebuffering = false;
|
|
300
|
-
const buffers = this.prebuffer;
|
|
301
|
-
this.prebuffer = [];
|
|
302
|
-
this.prebufferDuration = 0;
|
|
303
|
-
for (const audioBuffer of buffers) this.scheduleBuffer(audioBuffer);
|
|
304
|
-
}
|
|
305
|
-
resetPrebuffer() {
|
|
306
|
-
if (this.quietTimer !== void 0) {
|
|
307
|
-
clearTimeout(this.quietTimer);
|
|
308
|
-
this.quietTimer = void 0;
|
|
370
|
+
};
|
|
371
|
+
var current = new MemoryStorage();
|
|
372
|
+
function setMicdropStorage(storage2) {
|
|
373
|
+
current = storage2;
|
|
374
|
+
}
|
|
375
|
+
var storage = {
|
|
376
|
+
getItem: (key) => {
|
|
377
|
+
try {
|
|
378
|
+
return current.getItem(key);
|
|
379
|
+
} catch {
|
|
380
|
+
return null;
|
|
309
381
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
382
|
+
},
|
|
383
|
+
setItem: (key, value) => {
|
|
384
|
+
try {
|
|
385
|
+
current.setItem(key, value);
|
|
386
|
+
} catch {
|
|
313
387
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
this.prebufferDuration = 0;
|
|
317
|
-
}
|
|
318
|
-
createAudioBuffer(arrayBuffer) {
|
|
388
|
+
},
|
|
389
|
+
removeItem: (key) => {
|
|
319
390
|
try {
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
if (numSamples === 0) {
|
|
323
|
-
return null;
|
|
324
|
-
}
|
|
325
|
-
const audioBuffer = audioContext.createBuffer(
|
|
326
|
-
CHANNELS,
|
|
327
|
-
numSamples,
|
|
328
|
-
SAMPLE_RATE
|
|
329
|
-
);
|
|
330
|
-
const channelData = audioBuffer.getChannelData(0);
|
|
331
|
-
for (let i = 0; i < numSamples; i++) {
|
|
332
|
-
channelData[i] = int16Data[i] / 32768;
|
|
333
|
-
}
|
|
334
|
-
return audioBuffer;
|
|
335
|
-
} catch (error) {
|
|
336
|
-
console.error("Error creating AudioBuffer:", error);
|
|
337
|
-
return null;
|
|
391
|
+
current.removeItem(key);
|
|
392
|
+
} catch {
|
|
338
393
|
}
|
|
339
394
|
}
|
|
340
|
-
scheduleBuffer(audioBuffer) {
|
|
341
|
-
const sourceNode = audioContext.createBufferSource();
|
|
342
|
-
sourceNode.buffer = audioBuffer;
|
|
343
|
-
sourceNode.connect(this.outputNode);
|
|
344
|
-
const now = audioContext.currentTime;
|
|
345
|
-
const startTime = Math.max(this.nextStartTime, now);
|
|
346
|
-
this.nextStartTime = startTime + audioBuffer.duration;
|
|
347
|
-
sourceNode.onended = () => {
|
|
348
|
-
const index = this.sourceNodes.indexOf(sourceNode);
|
|
349
|
-
if (index !== -1) this.sourceNodes.splice(index, 1);
|
|
350
|
-
if (this.sourceNodes.length === 0 && this.blobQueue.length === 0) {
|
|
351
|
-
this.nextStartTime = 0;
|
|
352
|
-
this.setIsPlaying(false);
|
|
353
|
-
if (this.endTimer !== void 0) clearTimeout(this.endTimer);
|
|
354
|
-
this.endTimer = setTimeout(() => {
|
|
355
|
-
this.endTimer = void 0;
|
|
356
|
-
this.resetPrebuffer();
|
|
357
|
-
}, UTTERANCE_END_DELAY);
|
|
358
|
-
}
|
|
359
|
-
};
|
|
360
|
-
this.sourceNodes.push(sourceNode);
|
|
361
|
-
sourceNode.start(startTime);
|
|
362
|
-
this.setIsPlaying(true);
|
|
363
|
-
}
|
|
364
395
|
};
|
|
365
396
|
|
|
366
397
|
// src/audio/Speaker.ts
|
|
367
|
-
var
|
|
398
|
+
var SpeakerController = class extends import_eventemitter32.EventEmitter {
|
|
368
399
|
constructor() {
|
|
369
400
|
super(...arguments);
|
|
370
|
-
|
|
371
|
-
this.
|
|
372
|
-
|
|
373
|
-
this.
|
|
401
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
402
|
+
this.volume = -Infinity;
|
|
403
|
+
this.onStartPlaying = () => this.emit("StartPlaying");
|
|
404
|
+
this.onStopPlaying = () => {
|
|
405
|
+
this.volume = -Infinity;
|
|
406
|
+
this.emit("StopPlaying");
|
|
407
|
+
};
|
|
408
|
+
this.onVolume = (volume) => {
|
|
409
|
+
this.volume = volume;
|
|
410
|
+
this.emit("Volume", volume);
|
|
411
|
+
};
|
|
374
412
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
this.
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
async playAudio(blob) {
|
|
386
|
-
this.start();
|
|
387
|
-
try {
|
|
388
|
-
await this.audioStream?.playAudio(blob);
|
|
389
|
-
} catch (error) {
|
|
390
|
-
console.error("Error playing audio blob", error, blob);
|
|
413
|
+
/**
|
|
414
|
+
* Replaces the audio driver, to play sound with another library
|
|
415
|
+
* @param driver - The driver to play with
|
|
416
|
+
*/
|
|
417
|
+
setDriver(driver) {
|
|
418
|
+
if (this.driver === driver) return;
|
|
419
|
+
if (this.driver) {
|
|
420
|
+
this.driver.off("StartPlaying", this.onStartPlaying);
|
|
421
|
+
this.driver.off("StopPlaying", this.onStopPlaying);
|
|
422
|
+
this.driver.off("Volume", this.onVolume);
|
|
391
423
|
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
this.
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
424
|
+
this.driver = driver;
|
|
425
|
+
driver.on("StartPlaying", this.onStartPlaying);
|
|
426
|
+
driver.on("StopPlaying", this.onStopPlaying);
|
|
427
|
+
driver.on("Volume", this.onVolume);
|
|
428
|
+
}
|
|
429
|
+
getDriver() {
|
|
430
|
+
if (!this.driver) {
|
|
431
|
+
throw new Error(
|
|
432
|
+
"No speaker driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Speaker.setDriver()."
|
|
433
|
+
);
|
|
403
434
|
}
|
|
435
|
+
return this.driver;
|
|
404
436
|
}
|
|
405
|
-
get
|
|
406
|
-
return
|
|
437
|
+
get isPlaying() {
|
|
438
|
+
return this.driver?.isPlaying ?? false;
|
|
407
439
|
}
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
this.isPlaying = isPlaying;
|
|
411
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
440
|
+
get deviceId() {
|
|
441
|
+
return this.driver?.deviceId;
|
|
412
442
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
443
|
+
/** Prepares the audio graph, called before the first chunk arrives */
|
|
444
|
+
async start() {
|
|
445
|
+
const driver = this.getDriver();
|
|
446
|
+
await driver.start();
|
|
447
|
+
const saved = storage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
448
|
+
if (saved && saved !== driver.deviceId) {
|
|
449
|
+
try {
|
|
450
|
+
await driver.changeDevice(saved);
|
|
451
|
+
} catch {
|
|
452
|
+
storage.removeItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
453
|
+
}
|
|
419
454
|
}
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Queues samples for playback, right after the ones already queued
|
|
458
|
+
* @param pcm - Mono samples as 16 bits integers, or the bytes holding them
|
|
459
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
460
|
+
*/
|
|
461
|
+
playAudio(pcm, sampleRate = 16e3) {
|
|
420
462
|
try {
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
const sinkId = !deviceId || defaultDevice ? "" : deviceId;
|
|
424
|
-
if (defaultDevice || "setSinkId" in audioContext) {
|
|
425
|
-
await audioContext.setSinkId?.(sinkId);
|
|
426
|
-
this.audioSourceNode.disconnect();
|
|
427
|
-
this.audioSourceNode.connect(audioContext.destination);
|
|
428
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
429
|
-
if (this.audioElementDestination) {
|
|
430
|
-
this.audioElementDestination.pause();
|
|
431
|
-
this.audioElementDestination.srcObject = null;
|
|
432
|
-
this.audioElementDestination = void 0;
|
|
433
|
-
}
|
|
434
|
-
} else {
|
|
435
|
-
if (!this.audioElementDestination) {
|
|
436
|
-
const mediaStreamDestination = audioContext.createMediaStreamDestination();
|
|
437
|
-
this.audioSourceNode.disconnect();
|
|
438
|
-
this.audioSourceNode.connect(mediaStreamDestination);
|
|
439
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
440
|
-
this.audioElementDestination = new Audio();
|
|
441
|
-
this.audioElementDestination.autoplay = true;
|
|
442
|
-
this.audioElementDestination.srcObject = mediaStreamDestination.stream;
|
|
443
|
-
}
|
|
444
|
-
await this.audioElementDestination.setSinkId(sinkId);
|
|
445
|
-
}
|
|
463
|
+
const samples = pcm instanceof Int16Array ? pcm : new Int16Array(pcm);
|
|
464
|
+
this.getDriver().play(samples, sampleRate);
|
|
446
465
|
} catch (error) {
|
|
447
|
-
console.error(
|
|
466
|
+
console.error("[Micdrop] Error playing audio", error);
|
|
448
467
|
}
|
|
449
468
|
}
|
|
469
|
+
/** Drops everything queued and stops playing immediately */
|
|
470
|
+
stopAudio() {
|
|
471
|
+
this.driver?.stopAudio();
|
|
472
|
+
}
|
|
473
|
+
/** Releases the audio graph */
|
|
474
|
+
async stop() {
|
|
475
|
+
await this.driver?.stop();
|
|
476
|
+
this.volume = -Infinity;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Plays through another output
|
|
480
|
+
* @param deviceId - Output device, or route on a phone
|
|
481
|
+
*/
|
|
482
|
+
async changeDevice(deviceId) {
|
|
483
|
+
await this.getDriver().changeDevice(deviceId);
|
|
484
|
+
storage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
485
|
+
}
|
|
486
|
+
/** Lists the output devices the system offers */
|
|
487
|
+
async getDevices() {
|
|
488
|
+
if (!this.driver) return [];
|
|
489
|
+
return this.driver.getDevices();
|
|
490
|
+
}
|
|
450
491
|
};
|
|
451
492
|
|
|
452
493
|
// src/audio/MicRecorder.ts
|
|
453
|
-
var
|
|
454
|
-
|
|
455
|
-
// src/audio/utils/delayedStream.ts
|
|
456
|
-
function createDelayedStream(stream, delay) {
|
|
457
|
-
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
458
|
-
const delayNode = audioContext.createDelay(delay);
|
|
459
|
-
delayNode.delayTime.value = delay;
|
|
460
|
-
const audioDestination = audioContext.createMediaStreamDestination();
|
|
461
|
-
audioSource.connect(delayNode);
|
|
462
|
-
delayNode.connect(audioDestination);
|
|
463
|
-
return audioDestination.stream;
|
|
464
|
-
}
|
|
494
|
+
var import_eventemitter34 = require("eventemitter3");
|
|
465
495
|
|
|
466
496
|
// src/audio/vad/VAD.ts
|
|
467
|
-
var
|
|
497
|
+
var import_eventemitter33 = require("eventemitter3");
|
|
468
498
|
var VADStatus = /* @__PURE__ */ ((VADStatus2) => {
|
|
469
499
|
VADStatus2[VADStatus2["Silence"] = 0] = "Silence";
|
|
470
500
|
VADStatus2[VADStatus2["MaybeSpeaking"] = 1] = "MaybeSpeaking";
|
|
471
501
|
VADStatus2[VADStatus2["Speaking"] = 2] = "Speaking";
|
|
472
502
|
return VADStatus2;
|
|
473
503
|
})(VADStatus || {});
|
|
474
|
-
var VAD = class extends
|
|
504
|
+
var VAD = class extends import_eventemitter33.EventEmitter {
|
|
475
505
|
constructor() {
|
|
476
506
|
super(...arguments);
|
|
477
507
|
this.name = this.constructor.name;
|
|
478
508
|
this.status = 0 /* Silence */;
|
|
479
|
-
//
|
|
509
|
+
// Worst case time this VAD takes to notice that someone started speaking.
|
|
510
|
+
// That much audio is kept in reserve and sent along with the turn, so the
|
|
511
|
+
// beginning of a sentence reaches the server rather than being cut off.
|
|
512
|
+
// A VAD that needs several samples to make up its mind has to say so here.
|
|
480
513
|
this.delay = 100;
|
|
481
514
|
}
|
|
482
515
|
/**
|
|
@@ -559,13 +592,13 @@ var MultipleVAD = class extends VAD {
|
|
|
559
592
|
get isPaused() {
|
|
560
593
|
return this.vads.every((vad) => vad.isPaused);
|
|
561
594
|
}
|
|
562
|
-
async start(
|
|
595
|
+
async start(mic) {
|
|
563
596
|
for (const vad of this.vads) {
|
|
564
597
|
if (vad.isStarted) {
|
|
565
598
|
continue;
|
|
566
599
|
}
|
|
567
600
|
vad.on("ChangeStatus", this.onStatusChange);
|
|
568
|
-
await vad.start(
|
|
601
|
+
await vad.start(mic);
|
|
569
602
|
}
|
|
570
603
|
}
|
|
571
604
|
async stop() {
|
|
@@ -587,87 +620,96 @@ var MultipleVAD = class extends VAD {
|
|
|
587
620
|
};
|
|
588
621
|
|
|
589
622
|
// src/audio/vad/SileroVAD.ts
|
|
590
|
-
var
|
|
623
|
+
var SILERO_SAMPLE_RATE = 16e3;
|
|
624
|
+
var SILERO_FRAME_SAMPLES = 512;
|
|
625
|
+
var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
|
|
591
626
|
var defaultOptions = {
|
|
592
627
|
positiveSpeechThreshold: 0.18,
|
|
593
628
|
negativeSpeechThreshold: 0.11,
|
|
594
629
|
minSpeechFrames: 8,
|
|
595
630
|
redemptionFrames: 20
|
|
596
631
|
};
|
|
632
|
+
var loadModel;
|
|
633
|
+
function setSileroModelLoader(loader) {
|
|
634
|
+
loadModel = loader;
|
|
635
|
+
}
|
|
597
636
|
var SileroVAD = class extends VAD {
|
|
598
637
|
constructor(options) {
|
|
599
638
|
super();
|
|
600
639
|
this.name = "SileroVAD";
|
|
601
640
|
this.options = defaultOptions;
|
|
641
|
+
// The model decides on the first window it hears speech in, so it is quick.
|
|
642
|
+
// The reserve still covers a few windows, since scoring runs asynchronously
|
|
643
|
+
// and may lag behind the audio on a busy device.
|
|
644
|
+
this.delay = 3 * MS_PER_FRAME;
|
|
602
645
|
this._isPaused = false;
|
|
646
|
+
this.buffer = [];
|
|
647
|
+
this.bufferLength = 0;
|
|
648
|
+
this.bufferRate = SILERO_SAMPLE_RATE;
|
|
649
|
+
this.speaking = false;
|
|
650
|
+
this.speechFrames = 0;
|
|
651
|
+
this.silenceFrames = 0;
|
|
652
|
+
this.processing = false;
|
|
653
|
+
this.onFrames = (frames, sampleRate) => {
|
|
654
|
+
if (this._isPaused || !this.model) return;
|
|
655
|
+
if (sampleRate !== this.bufferRate) {
|
|
656
|
+
this.buffer = [];
|
|
657
|
+
this.bufferLength = 0;
|
|
658
|
+
this.bufferRate = sampleRate;
|
|
659
|
+
}
|
|
660
|
+
this.buffer.push(frames);
|
|
661
|
+
this.bufferLength += frames.length;
|
|
662
|
+
this.drain();
|
|
663
|
+
};
|
|
603
664
|
if (options) {
|
|
604
665
|
this.setOptions(options);
|
|
605
666
|
} else {
|
|
606
|
-
const
|
|
607
|
-
"micdrop.SileroVADOptions" /* SileroVADOptions */
|
|
608
|
-
);
|
|
667
|
+
const saved = storage.getItem("micdrop.SileroVADOptions" /* SileroVADOptions */);
|
|
609
668
|
try {
|
|
610
|
-
this.setOptions(
|
|
611
|
-
} catch
|
|
669
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
670
|
+
} catch {
|
|
612
671
|
this.setOptions({});
|
|
613
672
|
}
|
|
614
673
|
}
|
|
615
674
|
}
|
|
616
675
|
get isStarted() {
|
|
617
|
-
return !!this.
|
|
676
|
+
return !!this.mic;
|
|
618
677
|
}
|
|
619
678
|
get isPaused() {
|
|
620
679
|
return this._isPaused;
|
|
621
680
|
}
|
|
622
|
-
async start(
|
|
623
|
-
if (this.
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
// CDN size limit, so every file 404s). unpkg serves it, so the ONNX wasm
|
|
634
|
-
// backend is loaded from there instead.
|
|
635
|
-
onnxWASMBasePath: "https://unpkg.com/onnxruntime-web@1.23.2/dist/",
|
|
636
|
-
onSpeechStart: () => {
|
|
637
|
-
this.emit("StartSpeaking");
|
|
638
|
-
},
|
|
639
|
-
onSpeechRealStart: () => {
|
|
640
|
-
this.emit("ConfirmSpeaking");
|
|
641
|
-
},
|
|
642
|
-
onVADMisfire: () => {
|
|
643
|
-
this.emit("CancelSpeaking");
|
|
644
|
-
},
|
|
645
|
-
onSpeechEnd: () => {
|
|
646
|
-
this.emit("StopSpeaking");
|
|
647
|
-
},
|
|
648
|
-
...this.options
|
|
649
|
-
});
|
|
650
|
-
this.vad.start();
|
|
681
|
+
async start(mic) {
|
|
682
|
+
if (this.mic) return;
|
|
683
|
+
if (!loadModel) {
|
|
684
|
+
throw new Error(
|
|
685
|
+
"No Silero model. Add `import '@micdrop/web/silero'` in a browser or `import '@micdrop/react-native/silero'` on a phone, or call setSileroModelLoader()."
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
this.model = await loadModel();
|
|
689
|
+
this.resetState();
|
|
690
|
+
this.mic = mic;
|
|
691
|
+
mic.on("Frames", this.onFrames);
|
|
651
692
|
}
|
|
652
693
|
async stop() {
|
|
653
|
-
|
|
654
|
-
this.
|
|
655
|
-
this.
|
|
694
|
+
this.mic?.off("Frames", this.onFrames);
|
|
695
|
+
this.mic = void 0;
|
|
696
|
+
this.closeOpenTurn();
|
|
697
|
+
this.resetState();
|
|
698
|
+
await this.model?.release();
|
|
699
|
+
this.model = void 0;
|
|
656
700
|
}
|
|
657
701
|
async pause() {
|
|
658
|
-
if (
|
|
702
|
+
if (this._isPaused) return;
|
|
659
703
|
this._isPaused = true;
|
|
660
|
-
this.
|
|
704
|
+
this.closeOpenTurn();
|
|
705
|
+
this.resetState();
|
|
661
706
|
}
|
|
662
707
|
async resume() {
|
|
663
|
-
if (!this.vad || !this._isPaused) return;
|
|
664
708
|
this._isPaused = false;
|
|
665
|
-
this.vad.start();
|
|
666
709
|
}
|
|
667
710
|
setOptions(options) {
|
|
668
711
|
this.options = { ...this.options, ...options };
|
|
669
|
-
|
|
670
|
-
localStorage.setItem(
|
|
712
|
+
storage.setItem(
|
|
671
713
|
"micdrop.SileroVADOptions" /* SileroVADOptions */,
|
|
672
714
|
JSON.stringify(this.options)
|
|
673
715
|
);
|
|
@@ -675,7 +717,79 @@ var SileroVAD = class extends VAD {
|
|
|
675
717
|
resetOptions() {
|
|
676
718
|
this.setOptions(defaultOptions);
|
|
677
719
|
}
|
|
720
|
+
// The model is asynchronous, so windows are scored one after the other rather
|
|
721
|
+
// than raced: its answer for a window depends on the ones before it.
|
|
722
|
+
async drain() {
|
|
723
|
+
if (this.processing) return;
|
|
724
|
+
this.processing = true;
|
|
725
|
+
try {
|
|
726
|
+
const needed = Math.ceil(
|
|
727
|
+
SILERO_FRAME_SAMPLES * this.bufferRate / SILERO_SAMPLE_RATE
|
|
728
|
+
);
|
|
729
|
+
while (this.bufferLength >= needed && this.model && !this._isPaused) {
|
|
730
|
+
const gathered = concatFloat32(this.buffer);
|
|
731
|
+
const window = gathered.subarray(0, needed);
|
|
732
|
+
const rest = gathered.subarray(needed);
|
|
733
|
+
this.buffer = rest.length ? [rest] : [];
|
|
734
|
+
this.bufferLength = rest.length;
|
|
735
|
+
const frame = resample(window, this.bufferRate, SILERO_SAMPLE_RATE);
|
|
736
|
+
const probability = await this.model.process(
|
|
737
|
+
frame.length === SILERO_FRAME_SAMPLES ? frame : fit(frame, SILERO_FRAME_SAMPLES)
|
|
738
|
+
);
|
|
739
|
+
this.onProbability(probability);
|
|
740
|
+
}
|
|
741
|
+
} catch (error) {
|
|
742
|
+
console.error("[Micdrop] Silero VAD failed", error);
|
|
743
|
+
} finally {
|
|
744
|
+
this.processing = false;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
// Ported from the reference implementation of the algorithm
|
|
748
|
+
// https://docs.vad.ricky0123.com/user-guide/algorithm/
|
|
749
|
+
onProbability(probability) {
|
|
750
|
+
const { positiveSpeechThreshold, negativeSpeechThreshold } = this.options;
|
|
751
|
+
if (probability >= positiveSpeechThreshold) {
|
|
752
|
+
this.silenceFrames = 0;
|
|
753
|
+
this.speechFrames++;
|
|
754
|
+
if (!this.speaking) {
|
|
755
|
+
this.speaking = true;
|
|
756
|
+
this.emit("StartSpeaking");
|
|
757
|
+
}
|
|
758
|
+
if (this.speechFrames === this.options.minSpeechFrames) {
|
|
759
|
+
this.emit("ConfirmSpeaking");
|
|
760
|
+
}
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
if (probability < negativeSpeechThreshold && this.speaking) {
|
|
764
|
+
this.silenceFrames++;
|
|
765
|
+
if (this.silenceFrames < this.options.redemptionFrames) return;
|
|
766
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
767
|
+
this.speaking = false;
|
|
768
|
+
this.speechFrames = 0;
|
|
769
|
+
this.silenceFrames = 0;
|
|
770
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
closeOpenTurn() {
|
|
774
|
+
if (!this.speaking) return;
|
|
775
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
776
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
777
|
+
}
|
|
778
|
+
resetState() {
|
|
779
|
+
this.model?.reset();
|
|
780
|
+
this.buffer = [];
|
|
781
|
+
this.bufferLength = 0;
|
|
782
|
+
this.speaking = false;
|
|
783
|
+
this.speechFrames = 0;
|
|
784
|
+
this.silenceFrames = 0;
|
|
785
|
+
}
|
|
678
786
|
};
|
|
787
|
+
function fit(frame, length) {
|
|
788
|
+
if (frame.length === length) return frame;
|
|
789
|
+
const fitted = new Float32Array(length);
|
|
790
|
+
fitted.set(frame.subarray(0, length));
|
|
791
|
+
return fitted;
|
|
792
|
+
}
|
|
679
793
|
|
|
680
794
|
// src/audio/vad/VolumeVAD.ts
|
|
681
795
|
var defaultOptions2 = {
|
|
@@ -687,138 +801,92 @@ var VolumeVAD = class extends VAD {
|
|
|
687
801
|
super();
|
|
688
802
|
this.name = "VolumeVAD";
|
|
689
803
|
this.options = defaultOptions2;
|
|
690
|
-
|
|
804
|
+
// The microphone reports a level about every 100 ms, and two consecutive loud
|
|
805
|
+
// ones are needed before a turn opens. Speech can therefore begin up to two
|
|
806
|
+
// reports before it is noticed, and a third is kept as a margin: without that
|
|
807
|
+
// reserve the first syllable of a sentence never reaches the server.
|
|
808
|
+
this.delay = 300;
|
|
691
809
|
this.running = false;
|
|
810
|
+
this._isPaused = false;
|
|
692
811
|
this.speaking = false;
|
|
693
812
|
this.attemptSpeaking = false;
|
|
694
813
|
this.speakingHistory = [];
|
|
814
|
+
this.onVolume = (volume) => {
|
|
815
|
+
if (!this.running) return;
|
|
816
|
+
const isLoud = volume > this.options.threshold;
|
|
817
|
+
if (isLoud) {
|
|
818
|
+
if (!this.speaking) {
|
|
819
|
+
let recentHistory = 0;
|
|
820
|
+
for (let i = Math.max(0, this.speakingHistory.length - 3); i < this.speakingHistory.length; i++) {
|
|
821
|
+
recentHistory += this.speakingHistory[i];
|
|
822
|
+
}
|
|
823
|
+
if (recentHistory >= 2) {
|
|
824
|
+
this.speaking = true;
|
|
825
|
+
this.attemptSpeaking = false;
|
|
826
|
+
this.emit("ConfirmSpeaking");
|
|
827
|
+
} else if (recentHistory === 1 && !this.attemptSpeaking) {
|
|
828
|
+
this.attemptSpeaking = true;
|
|
829
|
+
this.emit("StartSpeaking");
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
} else if (this.speaking) {
|
|
833
|
+
let totalHistory = 0;
|
|
834
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
835
|
+
totalHistory += this.speakingHistory[i];
|
|
836
|
+
}
|
|
837
|
+
if (totalHistory === 0) {
|
|
838
|
+
this.speaking = false;
|
|
839
|
+
this.emit("StopSpeaking");
|
|
840
|
+
}
|
|
841
|
+
} else if (this.attemptSpeaking) {
|
|
842
|
+
const lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
843
|
+
if (lastHistory === 0) {
|
|
844
|
+
this.attemptSpeaking = false;
|
|
845
|
+
this.emit("CancelSpeaking");
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
this.speakingHistory.shift();
|
|
849
|
+
this.speakingHistory.push(Number(isLoud));
|
|
850
|
+
};
|
|
695
851
|
if (options) {
|
|
696
852
|
this.setOptions(options);
|
|
697
853
|
} else {
|
|
698
|
-
const
|
|
699
|
-
"micdrop.VolumeVADOptions" /* VolumeVADOptions */
|
|
700
|
-
);
|
|
854
|
+
const saved = storage.getItem("micdrop.VolumeVADOptions" /* VolumeVADOptions */);
|
|
701
855
|
try {
|
|
702
|
-
this.setOptions(
|
|
703
|
-
} catch
|
|
856
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
857
|
+
} catch {
|
|
704
858
|
this.setOptions({});
|
|
705
859
|
}
|
|
706
860
|
}
|
|
707
861
|
}
|
|
708
|
-
getMaxVolume() {
|
|
709
|
-
if (!this.analyser || !this.fftBins) return -Infinity;
|
|
710
|
-
let maxVolume = -Infinity;
|
|
711
|
-
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
712
|
-
for (let i = 4; i < this.fftBins.length; i++) {
|
|
713
|
-
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
714
|
-
maxVolume = this.fftBins[i];
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
return maxVolume;
|
|
718
|
-
}
|
|
719
862
|
get isStarted() {
|
|
720
863
|
return this.running;
|
|
721
864
|
}
|
|
722
865
|
get isPaused() {
|
|
723
866
|
return this._isPaused;
|
|
724
867
|
}
|
|
725
|
-
async start(
|
|
726
|
-
if (this.
|
|
727
|
-
this.
|
|
728
|
-
|
|
729
|
-
this.
|
|
730
|
-
this.analyser.smoothingTimeConstant = 0.1;
|
|
731
|
-
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
732
|
-
this.sourceNode.connect(this.analyser);
|
|
733
|
-
this.running = true;
|
|
734
|
-
this.startLoop();
|
|
735
|
-
}
|
|
736
|
-
startLoop() {
|
|
737
|
-
if (!this.running) return;
|
|
738
|
-
const currentVolume = this.getMaxVolume();
|
|
739
|
-
if (currentVolume > this.options.threshold) {
|
|
740
|
-
if (!this.speaking) {
|
|
741
|
-
let recentHistory = 0;
|
|
742
|
-
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
743
|
-
recentHistory += this.speakingHistory[i];
|
|
744
|
-
}
|
|
745
|
-
if (recentHistory >= 2) {
|
|
746
|
-
this.speaking = true;
|
|
747
|
-
this.attemptSpeaking = false;
|
|
748
|
-
this.emit("ConfirmSpeaking");
|
|
749
|
-
} else if (recentHistory === 1) {
|
|
750
|
-
this.attemptSpeaking = true;
|
|
751
|
-
this.emit("StartSpeaking");
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
} else if (this.speaking) {
|
|
755
|
-
let totalHistory = 0;
|
|
756
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
757
|
-
totalHistory += this.speakingHistory[i];
|
|
758
|
-
}
|
|
759
|
-
if (totalHistory === 0) {
|
|
760
|
-
this.speaking = false;
|
|
761
|
-
this.emit("StopSpeaking");
|
|
762
|
-
}
|
|
763
|
-
} else if (this.attemptSpeaking) {
|
|
764
|
-
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
765
|
-
if (lastHistory === 0) {
|
|
766
|
-
this.attemptSpeaking = false;
|
|
767
|
-
this.emit("CancelSpeaking");
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
this.speakingHistory.shift();
|
|
771
|
-
this.speakingHistory.push(Number(currentVolume > this.options.threshold));
|
|
772
|
-
setTimeout(() => this.startLoop(), this.delay);
|
|
773
|
-
}
|
|
774
|
-
resetHistory() {
|
|
775
|
-
this.speaking = false;
|
|
776
|
-
this.attemptSpeaking = false;
|
|
777
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
778
|
-
this.speakingHistory[i] = 0;
|
|
779
|
-
}
|
|
868
|
+
async start(mic) {
|
|
869
|
+
if (this.mic) return;
|
|
870
|
+
this.mic = mic;
|
|
871
|
+
mic.on("Volume", this.onVolume);
|
|
872
|
+
this.running = !this._isPaused;
|
|
780
873
|
}
|
|
781
874
|
async stop() {
|
|
782
|
-
|
|
875
|
+
this.mic?.off("Volume", this.onVolume);
|
|
876
|
+
this.mic = void 0;
|
|
783
877
|
this.running = false;
|
|
784
|
-
this.
|
|
785
|
-
switch (this.status) {
|
|
786
|
-
case 2 /* Speaking */:
|
|
787
|
-
this.emit("StopSpeaking");
|
|
788
|
-
break;
|
|
789
|
-
case 1 /* MaybeSpeaking */:
|
|
790
|
-
this.emit("CancelSpeaking");
|
|
791
|
-
break;
|
|
792
|
-
}
|
|
793
|
-
if (this.sourceNode) {
|
|
794
|
-
this.sourceNode.disconnect();
|
|
795
|
-
this.sourceNode = void 0;
|
|
796
|
-
}
|
|
797
|
-
if (this.analyser) {
|
|
798
|
-
this.analyser.disconnect();
|
|
799
|
-
this.analyser = void 0;
|
|
800
|
-
}
|
|
801
|
-
this.fftBins = void 0;
|
|
878
|
+
this.reset();
|
|
802
879
|
}
|
|
803
880
|
async pause() {
|
|
804
|
-
if (
|
|
881
|
+
if (this._isPaused) return;
|
|
805
882
|
this._isPaused = true;
|
|
806
883
|
this.running = false;
|
|
807
|
-
this.
|
|
808
|
-
switch (this.status) {
|
|
809
|
-
case 2 /* Speaking */:
|
|
810
|
-
this.emit("StopSpeaking");
|
|
811
|
-
break;
|
|
812
|
-
case 1 /* MaybeSpeaking */:
|
|
813
|
-
this.emit("CancelSpeaking");
|
|
814
|
-
break;
|
|
815
|
-
}
|
|
884
|
+
this.reset();
|
|
816
885
|
}
|
|
817
886
|
async resume() {
|
|
818
887
|
if (!this._isPaused) return;
|
|
819
888
|
this._isPaused = false;
|
|
820
|
-
this.running =
|
|
821
|
-
this.startLoop();
|
|
889
|
+
this.running = !!this.mic;
|
|
822
890
|
}
|
|
823
891
|
setOptions(options) {
|
|
824
892
|
if (typeof options.history === "number" && options.history < 3) {
|
|
@@ -831,7 +899,7 @@ var VolumeVAD = class extends VAD {
|
|
|
831
899
|
while (this.speakingHistory.length < this.options.history) {
|
|
832
900
|
this.speakingHistory.push(0);
|
|
833
901
|
}
|
|
834
|
-
|
|
902
|
+
storage.setItem(
|
|
835
903
|
"micdrop.VolumeVADOptions" /* VolumeVADOptions */,
|
|
836
904
|
JSON.stringify(this.options)
|
|
837
905
|
);
|
|
@@ -839,6 +907,22 @@ var VolumeVAD = class extends VAD {
|
|
|
839
907
|
resetOptions() {
|
|
840
908
|
this.setOptions(defaultOptions2);
|
|
841
909
|
}
|
|
910
|
+
reset() {
|
|
911
|
+
const status = this.status;
|
|
912
|
+
this.speaking = false;
|
|
913
|
+
this.attemptSpeaking = false;
|
|
914
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
915
|
+
this.speakingHistory[i] = 0;
|
|
916
|
+
}
|
|
917
|
+
switch (status) {
|
|
918
|
+
case 2 /* Speaking */:
|
|
919
|
+
this.emit("StopSpeaking");
|
|
920
|
+
break;
|
|
921
|
+
case 1 /* MaybeSpeaking */:
|
|
922
|
+
this.emit("CancelSpeaking");
|
|
923
|
+
break;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
842
926
|
};
|
|
843
927
|
|
|
844
928
|
// src/audio/vad/getVAD.ts
|
|
@@ -860,34 +944,27 @@ function equalVADConfig(a, b) {
|
|
|
860
944
|
return false;
|
|
861
945
|
}
|
|
862
946
|
|
|
863
|
-
// raw:./pcm-processor-worklet
|
|
864
|
-
var pcm_processor_worklet_default = "const INTERVAL_MS = 100\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.audioBuffer = []\n this.isRecording = false\n this.sourceSampleRate = 16000\n this.targetSampleRate = 16000\n this.resamplingRatio = 1\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n\n // Listen for messages\n this.port.onmessage = this.handleMessage.bind(this)\n }\n\n handleMessage(event) {\n if (event.data.type === 'configure') {\n this.sourceSampleRate = event.data.sourceSampleRate\n this.targetSampleRate = event.data.targetSampleRate\n this.resamplingRatio = this.sourceSampleRate / this.targetSampleRate\n // Recalculate chunk interval for the source sample rate\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n } else if (event.data.type === 'start') {\n this.isRecording = true\n } else if (event.data.type === 'stop') {\n this.isRecording = false\n // Clear any remaining audio buffer when stopping\n this.audioBuffer.length = 0\n }\n }\n\n process(inputs) {\n const input = inputs[0]\n\n if (input && input.length > 0 && this.isRecording) {\n const inputChannel = input[0]\n\n // Store audio data in buffer\n this.audioBuffer.push(new Float32Array(inputChannel))\n\n // Check if we have enough data for a chunk\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n\n if (totalSamples >= this.CHUNK_INTERVAL_SAMPLES) {\n this.processChunk()\n }\n }\n\n return true\n }\n\n processChunk() {\n if (this.audioBuffer.length === 0) return\n\n // Calculate total samples\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n if (totalSamples === 0) return\n\n // Merge all buffers\n const mergedBuffer = new Float32Array(totalSamples)\n let offset = 0\n for (const buffer of this.audioBuffer) {\n mergedBuffer.set(buffer, offset)\n offset += buffer.length\n }\n\n // Resample to target sample rate if necessary\n const resampledBuffer = this.resample(mergedBuffer)\n\n // Convert float32 to int16 PCM\n const pcmData = this.floatToPCM16(resampledBuffer)\n\n // Clear the buffer\n this.audioBuffer.length = 0\n\n // Send PCM chunk to main thread\n this.port.postMessage({\n type: 'chunk',\n data: pcmData,\n })\n }\n\n resample(inputBuffer) {\n // If no resampling needed, return original buffer\n if (this.resamplingRatio === 1) {\n return inputBuffer\n }\n\n // Calculate output length\n const outputLength = Math.round(inputBuffer.length / this.resamplingRatio)\n const outputBuffer = new Float32Array(outputLength)\n\n // Simple linear interpolation resampling\n for (let i = 0; i < outputLength; i++) {\n const srcIndex = i * this.resamplingRatio\n const srcIndexFloor = Math.floor(srcIndex)\n const srcIndexCeil = Math.min(srcIndexFloor + 1, inputBuffer.length - 1)\n const fraction = srcIndex - srcIndexFloor\n\n // Linear interpolation\n outputBuffer[i] =\n inputBuffer[srcIndexFloor] * (1 - fraction) +\n inputBuffer[srcIndexCeil] * fraction\n }\n\n return outputBuffer\n }\n\n floatToPCM16(floatData) {\n const pcmData = new Int16Array(floatData.length)\n for (let i = 0; i < floatData.length; i++) {\n const sample = Math.max(-1, Math.min(1, floatData[i]))\n pcmData[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff\n }\n return pcmData\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
|
|
865
|
-
|
|
866
|
-
// src/audio/pcm-processor.ts
|
|
867
|
-
async function initPcmProcessor() {
|
|
868
|
-
if (window.micdropPcmProcessor) return;
|
|
869
|
-
window.micdropPcmProcessor = true;
|
|
870
|
-
const workletBlob = new Blob([pcm_processor_worklet_default], {
|
|
871
|
-
type: "application/javascript"
|
|
872
|
-
});
|
|
873
|
-
const workletUrl = URL.createObjectURL(workletBlob);
|
|
874
|
-
await audioContext.audioWorklet.addModule(workletUrl);
|
|
875
|
-
}
|
|
876
|
-
|
|
877
947
|
// src/audio/MicRecorder.ts
|
|
878
|
-
var
|
|
948
|
+
var SAMPLE_RATE = 16e3;
|
|
949
|
+
var CHUNK_INTERVAL = 100;
|
|
879
950
|
var defaultMicRecorderState = {
|
|
880
951
|
isStarting: false,
|
|
881
952
|
isStarted: false,
|
|
882
953
|
isSpeaking: false
|
|
883
954
|
};
|
|
884
|
-
var MicRecorder = class extends
|
|
955
|
+
var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
885
956
|
constructor(vadConfig) {
|
|
886
957
|
super();
|
|
887
958
|
this.vadConfig = vadConfig;
|
|
959
|
+
this.isRecording = false;
|
|
888
960
|
this.speakingConfirmed = false;
|
|
961
|
+
this.sourceSampleRate = SAMPLE_RATE;
|
|
962
|
+
this.reserve = [];
|
|
963
|
+
this.reserveLength = 0;
|
|
964
|
+
this.buffer = [];
|
|
965
|
+
this.bufferLength = 0;
|
|
889
966
|
this.queuedChunks = [];
|
|
890
|
-
this.start = async (
|
|
967
|
+
this.start = async (mic) => {
|
|
891
968
|
if (this.state.isStarted) {
|
|
892
969
|
throw new Error("MicRecorder is already started");
|
|
893
970
|
}
|
|
@@ -896,21 +973,10 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
896
973
|
isStarting: true,
|
|
897
974
|
isSpeaking: false
|
|
898
975
|
});
|
|
899
|
-
this.
|
|
900
|
-
this.
|
|
901
|
-
|
|
902
|
-
this.
|
|
903
|
-
this.workletNode.port.onmessage = this.onWorkletMessage;
|
|
904
|
-
this.workletNode.port.postMessage({
|
|
905
|
-
type: "configure",
|
|
906
|
-
sourceSampleRate: audioContext.sampleRate,
|
|
907
|
-
targetSampleRate: SAMPLE_RATE2
|
|
908
|
-
});
|
|
909
|
-
const delayedStream = createDelayedStream(stream, this.vad.delay / 1e3);
|
|
910
|
-
this.delayedStream = delayedStream;
|
|
911
|
-
this.source = audioContext.createMediaStreamSource(delayedStream);
|
|
912
|
-
this.source.connect(this.workletNode);
|
|
913
|
-
await this.vad.start(stream);
|
|
976
|
+
this.mic = mic;
|
|
977
|
+
this.resetBuffers();
|
|
978
|
+
mic.on("Frames", this.onFrames);
|
|
979
|
+
await this.vad.start(mic);
|
|
914
980
|
this.vad.on("StartSpeaking", this.onStartSpeaking);
|
|
915
981
|
this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
916
982
|
this.vad.on("CancelSpeaking", this.onCancelSpeaking);
|
|
@@ -930,79 +996,138 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
930
996
|
isStarted: false,
|
|
931
997
|
isSpeaking: false
|
|
932
998
|
});
|
|
933
|
-
this.stream = void 0;
|
|
934
999
|
try {
|
|
1000
|
+
this.mic?.off("Frames", this.onFrames);
|
|
1001
|
+
this.mic = void 0;
|
|
935
1002
|
this.vad.stop();
|
|
936
1003
|
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
937
1004
|
this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
938
1005
|
this.vad.off("CancelSpeaking", this.onCancelSpeaking);
|
|
939
1006
|
this.vad.off("StopSpeaking", this.onStopSpeaking);
|
|
940
|
-
if (this.workletNode) {
|
|
941
|
-
this.workletNode.disconnect();
|
|
942
|
-
this.workletNode = void 0;
|
|
943
|
-
}
|
|
944
|
-
if (this.source) {
|
|
945
|
-
this.source.disconnect();
|
|
946
|
-
this.source = void 0;
|
|
947
|
-
}
|
|
948
|
-
if (this.delayedStream) {
|
|
949
|
-
stopStream(this.delayedStream);
|
|
950
|
-
this.delayedStream = void 0;
|
|
951
|
-
}
|
|
952
1007
|
} catch (err) {
|
|
953
1008
|
console.error(err);
|
|
954
1009
|
}
|
|
1010
|
+
this.resetBuffers();
|
|
955
1011
|
};
|
|
956
1012
|
this.changeVad = (vadConfig) => {
|
|
957
1013
|
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
958
|
-
const
|
|
959
|
-
if (
|
|
1014
|
+
const mic = this.mic;
|
|
1015
|
+
if (mic) {
|
|
960
1016
|
this.stop();
|
|
961
1017
|
}
|
|
962
1018
|
this.vadConfig = vadConfig;
|
|
963
1019
|
this.vad = getVAD(vadConfig);
|
|
964
|
-
if (
|
|
965
|
-
this.start(
|
|
1020
|
+
if (mic) {
|
|
1021
|
+
this.start(mic);
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
this.onFrames = (frames, sampleRate) => {
|
|
1025
|
+
if (sampleRate !== this.sourceSampleRate) {
|
|
1026
|
+
this.flushChunks();
|
|
1027
|
+
this.sourceSampleRate = sampleRate;
|
|
1028
|
+
this.reserve = [];
|
|
1029
|
+
this.reserveLength = 0;
|
|
1030
|
+
}
|
|
1031
|
+
if (this.isRecording) {
|
|
1032
|
+
this.buffer.push(frames);
|
|
1033
|
+
this.bufferLength += frames.length;
|
|
1034
|
+
this.flushChunks();
|
|
1035
|
+
return;
|
|
1036
|
+
}
|
|
1037
|
+
this.reserve.push(frames);
|
|
1038
|
+
this.reserveLength += frames.length;
|
|
1039
|
+
const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
|
|
1040
|
+
while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
|
|
1041
|
+
this.reserveLength -= this.reserve[0].length;
|
|
1042
|
+
this.reserve.shift();
|
|
966
1043
|
}
|
|
967
1044
|
};
|
|
968
|
-
this.onStartSpeaking =
|
|
1045
|
+
this.onStartSpeaking = () => {
|
|
969
1046
|
this.speakingConfirmed = false;
|
|
970
1047
|
this.queuedChunks.length = 0;
|
|
971
|
-
this.
|
|
1048
|
+
this.isRecording = true;
|
|
1049
|
+
this.buffer = this.reserve;
|
|
1050
|
+
this.bufferLength = this.reserveLength;
|
|
1051
|
+
this.reserve = [];
|
|
1052
|
+
this.reserveLength = 0;
|
|
1053
|
+
this.flushChunks();
|
|
972
1054
|
};
|
|
973
|
-
this.onConfirmSpeaking =
|
|
1055
|
+
this.onConfirmSpeaking = () => {
|
|
974
1056
|
this.speakingConfirmed = true;
|
|
975
1057
|
this.changeState({ isSpeaking: true });
|
|
976
1058
|
this.emit("StartSpeaking");
|
|
977
|
-
|
|
978
|
-
|
|
1059
|
+
for (const chunk of this.queuedChunks) {
|
|
1060
|
+
this.emit("Chunk", chunk);
|
|
1061
|
+
}
|
|
979
1062
|
this.queuedChunks.length = 0;
|
|
980
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
981
1063
|
};
|
|
982
|
-
this.
|
|
983
|
-
this.
|
|
984
|
-
this.changeState({ isSpeaking: false });
|
|
985
|
-
this.emit("StopSpeaking");
|
|
986
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
1064
|
+
this.onCancelSpeaking = () => {
|
|
1065
|
+
this.stopRecording();
|
|
987
1066
|
};
|
|
988
|
-
this.
|
|
989
|
-
const
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
this.queuedChunks.push(pcmData);
|
|
994
|
-
return;
|
|
995
|
-
}
|
|
996
|
-
for (const chunk of this.queuedChunks) {
|
|
997
|
-
this.emit("Chunk", chunk);
|
|
998
|
-
}
|
|
999
|
-
this.queuedChunks.length = 0;
|
|
1000
|
-
this.emit("Chunk", pcmData);
|
|
1067
|
+
this.onStopSpeaking = () => {
|
|
1068
|
+
const wasConfirmed = this.speakingConfirmed;
|
|
1069
|
+
this.changeState({ isSpeaking: false });
|
|
1070
|
+
if (wasConfirmed) {
|
|
1071
|
+
this.flushChunks(true);
|
|
1001
1072
|
}
|
|
1073
|
+
this.stopRecording();
|
|
1074
|
+
this.emit("StopSpeaking");
|
|
1002
1075
|
};
|
|
1003
1076
|
this.state = defaultMicRecorderState;
|
|
1004
1077
|
this.vad = getVAD(vadConfig);
|
|
1005
1078
|
}
|
|
1079
|
+
stopRecording() {
|
|
1080
|
+
this.isRecording = false;
|
|
1081
|
+
this.speakingConfirmed = false;
|
|
1082
|
+
this.queuedChunks.length = 0;
|
|
1083
|
+
this.buffer = [];
|
|
1084
|
+
this.bufferLength = 0;
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Emits every complete chunk waiting in the buffer
|
|
1088
|
+
* @param final - Also emit the incomplete last chunk
|
|
1089
|
+
*/
|
|
1090
|
+
flushChunks(final = false) {
|
|
1091
|
+
const chunkLength = Math.round(
|
|
1092
|
+
CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
|
|
1093
|
+
);
|
|
1094
|
+
if (this.bufferLength === 0) return;
|
|
1095
|
+
if (this.bufferLength < chunkLength && !final) return;
|
|
1096
|
+
const merged = concatFloat32(this.buffer);
|
|
1097
|
+
let offset = 0;
|
|
1098
|
+
while (merged.length - offset >= chunkLength) {
|
|
1099
|
+
this.emitChunk(merged.subarray(offset, offset + chunkLength));
|
|
1100
|
+
offset += chunkLength;
|
|
1101
|
+
}
|
|
1102
|
+
const rest = merged.subarray(offset);
|
|
1103
|
+
if (final && rest.length > 0) {
|
|
1104
|
+
this.emitChunk(rest);
|
|
1105
|
+
this.buffer = [];
|
|
1106
|
+
this.bufferLength = 0;
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
this.buffer = rest.length ? [rest] : [];
|
|
1110
|
+
this.bufferLength = rest.length;
|
|
1111
|
+
}
|
|
1112
|
+
emitChunk(samples) {
|
|
1113
|
+
const pcm = floatToPcm16(
|
|
1114
|
+
resample(samples, this.sourceSampleRate, SAMPLE_RATE)
|
|
1115
|
+
);
|
|
1116
|
+
if (!this.speakingConfirmed) {
|
|
1117
|
+
this.queuedChunks.push(pcm);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
this.emit("Chunk", pcm);
|
|
1121
|
+
}
|
|
1122
|
+
resetBuffers() {
|
|
1123
|
+
this.isRecording = false;
|
|
1124
|
+
this.speakingConfirmed = false;
|
|
1125
|
+
this.reserve = [];
|
|
1126
|
+
this.reserveLength = 0;
|
|
1127
|
+
this.buffer = [];
|
|
1128
|
+
this.bufferLength = 0;
|
|
1129
|
+
this.queuedChunks.length = 0;
|
|
1130
|
+
}
|
|
1006
1131
|
changeState(state) {
|
|
1007
1132
|
const hasChanged = Object.keys(state).some(
|
|
1008
1133
|
(key) => this.state[key] !== state[key]
|
|
@@ -1013,18 +1138,182 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
1013
1138
|
}
|
|
1014
1139
|
};
|
|
1015
1140
|
|
|
1141
|
+
// src/audio/Pcm16AudioStream.ts
|
|
1142
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1143
|
+
var PREBUFFER_DURATION = 100;
|
|
1144
|
+
var QUIET_FLUSH_DELAY = 600;
|
|
1145
|
+
var UTTERANCE_END_DELAY = 300;
|
|
1146
|
+
var VOLUME_INTERVAL2 = 100;
|
|
1147
|
+
var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
1148
|
+
constructor(sink) {
|
|
1149
|
+
super();
|
|
1150
|
+
this.sink = sink;
|
|
1151
|
+
this.isPlaying = false;
|
|
1152
|
+
this.sources = [];
|
|
1153
|
+
this.nextStartTime = 0;
|
|
1154
|
+
this.prebuffering = true;
|
|
1155
|
+
this.prebuffer = [];
|
|
1156
|
+
this.prebufferDuration = 0;
|
|
1157
|
+
this.volumeTimers = [];
|
|
1158
|
+
this.meter = new VolumeMeter();
|
|
1159
|
+
}
|
|
1160
|
+
/**
|
|
1161
|
+
* Queues samples for playback
|
|
1162
|
+
* @param pcm - Mono samples as 16 bits integers
|
|
1163
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
1164
|
+
*/
|
|
1165
|
+
playAudio(pcm, sampleRate) {
|
|
1166
|
+
if (pcm.length === 0) return;
|
|
1167
|
+
const samples = resample(
|
|
1168
|
+
pcm16ToFloat(pcm),
|
|
1169
|
+
sampleRate,
|
|
1170
|
+
this.sink.sampleRate
|
|
1171
|
+
);
|
|
1172
|
+
this.handleSamples(samples);
|
|
1173
|
+
}
|
|
1174
|
+
/** Drops everything queued and stops playing immediately */
|
|
1175
|
+
stopAudio() {
|
|
1176
|
+
for (const source of this.sources) {
|
|
1177
|
+
try {
|
|
1178
|
+
source.stop();
|
|
1179
|
+
} catch (error) {
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
this.sources = [];
|
|
1183
|
+
this.nextStartTime = 0;
|
|
1184
|
+
this.resetPrebuffer();
|
|
1185
|
+
this.setIsPlaying(false);
|
|
1186
|
+
this.clearVolumeTimers();
|
|
1187
|
+
}
|
|
1188
|
+
// Accumulate audio until enough is buffered, then start playing. Once
|
|
1189
|
+
// playing, schedule incoming samples directly.
|
|
1190
|
+
handleSamples(samples) {
|
|
1191
|
+
if (this.endTimer !== void 0) {
|
|
1192
|
+
clearTimeout(this.endTimer);
|
|
1193
|
+
this.endTimer = void 0;
|
|
1194
|
+
}
|
|
1195
|
+
if (!this.prebuffering) {
|
|
1196
|
+
this.scheduleSamples(samples);
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
this.prebuffer.push(samples);
|
|
1200
|
+
this.prebufferDuration += samples.length / this.sink.sampleRate * 1e3;
|
|
1201
|
+
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
1202
|
+
this.flushPrebuffer();
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
1205
|
+
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
1206
|
+
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
1207
|
+
}
|
|
1208
|
+
flushPrebuffer() {
|
|
1209
|
+
if (this.quietTimer !== void 0) {
|
|
1210
|
+
clearTimeout(this.quietTimer);
|
|
1211
|
+
this.quietTimer = void 0;
|
|
1212
|
+
}
|
|
1213
|
+
this.prebuffering = false;
|
|
1214
|
+
const buffers = this.prebuffer;
|
|
1215
|
+
this.prebuffer = [];
|
|
1216
|
+
this.prebufferDuration = 0;
|
|
1217
|
+
for (const samples of buffers) this.scheduleSamples(samples);
|
|
1218
|
+
}
|
|
1219
|
+
resetPrebuffer() {
|
|
1220
|
+
if (this.quietTimer !== void 0) {
|
|
1221
|
+
clearTimeout(this.quietTimer);
|
|
1222
|
+
this.quietTimer = void 0;
|
|
1223
|
+
}
|
|
1224
|
+
if (this.endTimer !== void 0) {
|
|
1225
|
+
clearTimeout(this.endTimer);
|
|
1226
|
+
this.endTimer = void 0;
|
|
1227
|
+
}
|
|
1228
|
+
this.prebuffering = true;
|
|
1229
|
+
this.prebuffer = [];
|
|
1230
|
+
this.prebufferDuration = 0;
|
|
1231
|
+
}
|
|
1232
|
+
scheduleSamples(samples) {
|
|
1233
|
+
const duration = samples.length / this.sink.sampleRate;
|
|
1234
|
+
const now = this.sink.currentTime;
|
|
1235
|
+
const startTime = Math.max(this.nextStartTime, now);
|
|
1236
|
+
this.nextStartTime = startTime + duration;
|
|
1237
|
+
const source = this.sink.schedule(samples, startTime, () => {
|
|
1238
|
+
const index = this.sources.indexOf(source);
|
|
1239
|
+
if (index !== -1) this.sources.splice(index, 1);
|
|
1240
|
+
if (this.sources.length === 0) {
|
|
1241
|
+
this.nextStartTime = 0;
|
|
1242
|
+
this.setIsPlaying(false);
|
|
1243
|
+
if (this.endTimer !== void 0) clearTimeout(this.endTimer);
|
|
1244
|
+
this.endTimer = setTimeout(() => {
|
|
1245
|
+
this.endTimer = void 0;
|
|
1246
|
+
this.resetPrebuffer();
|
|
1247
|
+
}, UTTERANCE_END_DELAY);
|
|
1248
|
+
}
|
|
1249
|
+
});
|
|
1250
|
+
this.sources.push(source);
|
|
1251
|
+
this.scheduleVolume(samples, startTime - now);
|
|
1252
|
+
this.setIsPlaying(true);
|
|
1253
|
+
}
|
|
1254
|
+
// Report the level of what is being heard rather than of what just arrived:
|
|
1255
|
+
// a burst of chunks would otherwise light up the meter all at once and leave
|
|
1256
|
+
// it dark for the rest of the sentence.
|
|
1257
|
+
scheduleVolume(samples, delay) {
|
|
1258
|
+
const step = Math.max(
|
|
1259
|
+
1,
|
|
1260
|
+
Math.round(VOLUME_INTERVAL2 / 1e3 * this.sink.sampleRate)
|
|
1261
|
+
);
|
|
1262
|
+
for (let offset = 0; offset < samples.length; offset += step) {
|
|
1263
|
+
const slice = samples.subarray(offset, offset + step);
|
|
1264
|
+
const at = delay * 1e3 + offset / this.sink.sampleRate * 1e3;
|
|
1265
|
+
const timer = setTimeout(
|
|
1266
|
+
() => {
|
|
1267
|
+
const index = this.volumeTimers.indexOf(timer);
|
|
1268
|
+
if (index !== -1) this.volumeTimers.splice(index, 1);
|
|
1269
|
+
this.emit(
|
|
1270
|
+
"Volume",
|
|
1271
|
+
this.meter.measure(
|
|
1272
|
+
resample(slice, this.sink.sampleRate, VOLUME_SAMPLE_RATE)
|
|
1273
|
+
)
|
|
1274
|
+
);
|
|
1275
|
+
},
|
|
1276
|
+
Math.max(0, at)
|
|
1277
|
+
);
|
|
1278
|
+
this.volumeTimers.push(timer);
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
clearVolumeTimers() {
|
|
1282
|
+
for (const timer of this.volumeTimers) clearTimeout(timer);
|
|
1283
|
+
this.volumeTimers = [];
|
|
1284
|
+
this.meter.reset();
|
|
1285
|
+
}
|
|
1286
|
+
setIsPlaying(isPlaying) {
|
|
1287
|
+
if (this.isPlaying === isPlaying) return;
|
|
1288
|
+
this.isPlaying = isPlaying;
|
|
1289
|
+
if (!isPlaying) {
|
|
1290
|
+
this.clearVolumeTimers();
|
|
1291
|
+
this.emit("Volume", -Infinity);
|
|
1292
|
+
}
|
|
1293
|
+
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
|
|
1297
|
+
// src/audio/types.ts
|
|
1298
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1299
|
+
var MicDriver = class extends import_eventemitter36.EventEmitter {
|
|
1300
|
+
};
|
|
1301
|
+
var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
|
|
1302
|
+
};
|
|
1303
|
+
|
|
1016
1304
|
// src/audio/index.ts
|
|
1017
|
-
|
|
1018
|
-
|
|
1305
|
+
var globalScope = globalThis;
|
|
1306
|
+
if (!globalScope.micdropMic) {
|
|
1307
|
+
globalScope.micdropMic = new MicController();
|
|
1019
1308
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
window.micdropSpeaker = new Speaker();
|
|
1309
|
+
if (!globalScope.micdropSpeaker) {
|
|
1310
|
+
globalScope.micdropSpeaker = new SpeakerController();
|
|
1023
1311
|
}
|
|
1024
|
-
var
|
|
1312
|
+
var Mic = globalScope.micdropMic;
|
|
1313
|
+
var Speaker = globalScope.micdropSpeaker;
|
|
1025
1314
|
|
|
1026
1315
|
// src/client/MicdropClient.ts
|
|
1027
|
-
var
|
|
1316
|
+
var import_eventemitter37 = require("eventemitter3");
|
|
1028
1317
|
|
|
1029
1318
|
// ../server/src/types.ts
|
|
1030
1319
|
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
@@ -1062,7 +1351,7 @@ var MicdropClientError = class extends Error {
|
|
|
1062
1351
|
}
|
|
1063
1352
|
};
|
|
1064
1353
|
function getClientErrorFromWSCloseEvent(event) {
|
|
1065
|
-
const code = getClientErrorFromWSCloseEventCode(event.code);
|
|
1354
|
+
const code = getClientErrorFromWSCloseEventCode(event.code ?? 1006);
|
|
1066
1355
|
return code && new MicdropClientError(code, event.reason);
|
|
1067
1356
|
}
|
|
1068
1357
|
function getClientErrorFromWSCloseEventCode(code) {
|
|
@@ -1092,7 +1381,7 @@ var DEFAULT_RECONNECT_OPTIONS = {
|
|
|
1092
1381
|
delayMs: 1e3,
|
|
1093
1382
|
connectionTimeout: 5e3
|
|
1094
1383
|
};
|
|
1095
|
-
var MicdropClient = class extends
|
|
1384
|
+
var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
1096
1385
|
constructor(options = {}) {
|
|
1097
1386
|
super();
|
|
1098
1387
|
this.options = options;
|
|
@@ -1117,7 +1406,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1117
1406
|
this._isReconnecting = false;
|
|
1118
1407
|
this.reconnectAttempt = 0;
|
|
1119
1408
|
if (this.reconnectTimer) {
|
|
1120
|
-
|
|
1409
|
+
clearTimeout(this.reconnectTimer);
|
|
1121
1410
|
this.reconnectTimer = void 0;
|
|
1122
1411
|
}
|
|
1123
1412
|
if (!this.micRecorder || options?.vad) {
|
|
@@ -1131,7 +1420,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1131
1420
|
this._isPaused = false;
|
|
1132
1421
|
this._isReconnecting = false;
|
|
1133
1422
|
if (this.reconnectTimer) {
|
|
1134
|
-
|
|
1423
|
+
clearTimeout(this.reconnectTimer);
|
|
1135
1424
|
this.reconnectTimer = void 0;
|
|
1136
1425
|
}
|
|
1137
1426
|
try {
|
|
@@ -1139,13 +1428,12 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1139
1428
|
} catch (error) {
|
|
1140
1429
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1141
1430
|
}
|
|
1431
|
+
Speaker.stopAudio();
|
|
1142
1432
|
try {
|
|
1143
|
-
this.
|
|
1144
|
-
this.stopMic();
|
|
1433
|
+
await this.stopMic();
|
|
1145
1434
|
} catch (error) {
|
|
1146
1435
|
console.error("[MicdropClient] Error stopping microphone", error);
|
|
1147
1436
|
}
|
|
1148
|
-
Speaker2.stopAudio();
|
|
1149
1437
|
};
|
|
1150
1438
|
this.mute = () => {
|
|
1151
1439
|
this.vad?.pause();
|
|
@@ -1153,7 +1441,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1153
1441
|
this.notifyStateChange();
|
|
1154
1442
|
};
|
|
1155
1443
|
this.unmute = () => {
|
|
1156
|
-
if (!this.isPaused && !(this.options.disableInterruption &&
|
|
1444
|
+
if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
|
|
1157
1445
|
this.vad?.resume();
|
|
1158
1446
|
}
|
|
1159
1447
|
this._isMuted = false;
|
|
@@ -1165,7 +1453,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1165
1453
|
this._isPaused = true;
|
|
1166
1454
|
this._isProcessing = false;
|
|
1167
1455
|
this.notifyStateChange();
|
|
1168
|
-
|
|
1456
|
+
Speaker.stopAudio();
|
|
1169
1457
|
this.ws?.send("Mute" /* Mute */);
|
|
1170
1458
|
};
|
|
1171
1459
|
this.resume = () => {
|
|
@@ -1193,16 +1481,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1193
1481
|
this.micRecorder.on("StateChange", () => {
|
|
1194
1482
|
this.notifyStateChange();
|
|
1195
1483
|
});
|
|
1196
|
-
this.micRecorder.on("Chunk", (
|
|
1197
|
-
this.log(
|
|
1198
|
-
this.ws?.send(
|
|
1484
|
+
this.micRecorder.on("Chunk", (chunk) => {
|
|
1485
|
+
this.log("User audio chunk", chunk.length);
|
|
1486
|
+
this.ws?.send(pcm16ToArrayBuffer(chunk));
|
|
1199
1487
|
});
|
|
1200
1488
|
this.micRecorder.on("StartSpeaking", () => {
|
|
1201
1489
|
this.log("User start speaking");
|
|
1202
1490
|
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
1203
1491
|
this._isProcessing = false;
|
|
1204
1492
|
this.notifyStateChange();
|
|
1205
|
-
|
|
1493
|
+
Speaker.stopAudio();
|
|
1206
1494
|
});
|
|
1207
1495
|
this.micRecorder.on("StopSpeaking", () => {
|
|
1208
1496
|
this.log("User stop speaking");
|
|
@@ -1213,11 +1501,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1213
1501
|
this.notifyStateChange();
|
|
1214
1502
|
});
|
|
1215
1503
|
}
|
|
1216
|
-
const
|
|
1217
|
-
|
|
1218
|
-
|
|
1504
|
+
const deviceId = options.deviceId ?? storage.getItem("micdrop.micDevice" /* MicDevice */) ?? void 0;
|
|
1505
|
+
await Mic.start(deviceId);
|
|
1506
|
+
if (Mic.deviceId) {
|
|
1507
|
+
storage.setItem("micdrop.micDevice" /* MicDevice */, Mic.deviceId);
|
|
1508
|
+
} else {
|
|
1509
|
+
storage.removeItem("micdrop.micDevice" /* MicDevice */);
|
|
1510
|
+
}
|
|
1511
|
+
await this.micRecorder.start(Mic);
|
|
1512
|
+
await Speaker.start();
|
|
1219
1513
|
await this.updateDevices();
|
|
1220
|
-
await Speaker2.start();
|
|
1221
1514
|
this.notifyStateChange();
|
|
1222
1515
|
} catch (error) {
|
|
1223
1516
|
this.setError(
|
|
@@ -1234,37 +1527,22 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1234
1527
|
await this.startMic({ deviceId });
|
|
1235
1528
|
};
|
|
1236
1529
|
this.changeSpeakerDevice = async (deviceId) => {
|
|
1237
|
-
await
|
|
1530
|
+
await Speaker.changeDevice(deviceId);
|
|
1238
1531
|
this.notifyStateChange();
|
|
1239
1532
|
};
|
|
1240
1533
|
this.updateDevices = async () => {
|
|
1241
|
-
const
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
const defaultDevicesIndexes = defaultDevices.map(
|
|
1248
|
-
(device) => devices.findIndex(
|
|
1249
|
-
(d) => d.groupId === device.groupId && d.kind === device.kind
|
|
1250
|
-
)
|
|
1251
|
-
).filter((index) => index !== -1);
|
|
1252
|
-
for (const index of defaultDevicesIndexes) {
|
|
1253
|
-
const device = devices[index];
|
|
1254
|
-
devices.splice(index, 1);
|
|
1255
|
-
devices.unshift(device);
|
|
1256
|
-
}
|
|
1257
|
-
this.micDevices = devices.filter((device) => device.kind === "audioinput");
|
|
1258
|
-
this.speakerDevices = devices.filter(
|
|
1259
|
-
(device) => device.kind === "audiooutput"
|
|
1260
|
-
);
|
|
1261
|
-
Speaker2.devices = this.speakerDevices;
|
|
1534
|
+
const [micDevices, speakerDevices] = await Promise.all([
|
|
1535
|
+
Mic.getDevices(),
|
|
1536
|
+
Speaker.getDevices()
|
|
1537
|
+
]);
|
|
1538
|
+
this.micDevices = micDevices;
|
|
1539
|
+
this.speakerDevices = speakerDevices;
|
|
1262
1540
|
this.notifyStateChange();
|
|
1263
1541
|
};
|
|
1264
1542
|
this.onWSOpen = () => {
|
|
1265
1543
|
this.log("WebSocket opened");
|
|
1266
1544
|
if (this.connectionTimer) {
|
|
1267
|
-
|
|
1545
|
+
clearTimeout(this.connectionTimer);
|
|
1268
1546
|
this.connectionTimer = void 0;
|
|
1269
1547
|
}
|
|
1270
1548
|
this._isReconnecting = false;
|
|
@@ -1279,36 +1557,36 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1279
1557
|
}
|
|
1280
1558
|
};
|
|
1281
1559
|
this.onWSMessage = (event) => {
|
|
1282
|
-
|
|
1283
|
-
if (
|
|
1560
|
+
const data = event.data;
|
|
1561
|
+
if (data instanceof ArrayBuffer) {
|
|
1562
|
+
this.log("Received audio", data.byteLength);
|
|
1284
1563
|
if (this.isPaused || this.isUserSpeaking) return;
|
|
1285
|
-
|
|
1564
|
+
Speaker.playAudio(new Int16Array(data));
|
|
1286
1565
|
this._isProcessing = false;
|
|
1287
1566
|
this.notifyStateChange();
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1567
|
+
return;
|
|
1568
|
+
}
|
|
1569
|
+
if (typeof data !== "string") {
|
|
1570
|
+
console.warn(`[MicdropClient] Unknown message type: ${data}`);
|
|
1571
|
+
return;
|
|
1572
|
+
}
|
|
1573
|
+
this.log("Received message:", data);
|
|
1574
|
+
if (data.startsWith("Message" /* Message */)) {
|
|
1291
1575
|
try {
|
|
1292
1576
|
const message = JSON.parse(
|
|
1293
|
-
|
|
1577
|
+
data.substring("Message" /* Message */.length + 1)
|
|
1294
1578
|
);
|
|
1295
1579
|
this.addMessage(message);
|
|
1296
1580
|
} catch (error) {
|
|
1297
|
-
console.error(
|
|
1298
|
-
"[MicdropClient] Error parsing message:",
|
|
1299
|
-
event.data,
|
|
1300
|
-
error
|
|
1301
|
-
);
|
|
1581
|
+
console.error("[MicdropClient] Error parsing message:", data, error);
|
|
1302
1582
|
}
|
|
1303
|
-
} else if (
|
|
1583
|
+
} else if (data === "EndCall" /* EndCall */) {
|
|
1304
1584
|
this.emit("EndCall");
|
|
1305
|
-
} else if (
|
|
1585
|
+
} else if (data === "SkipAnswer" /* SkipAnswer */) {
|
|
1306
1586
|
this._isProcessing = false;
|
|
1307
1587
|
this.notifyStateChange();
|
|
1308
|
-
} else if (
|
|
1309
|
-
const lastMessage = this.conversation.
|
|
1310
|
-
(message) => message.role === "user"
|
|
1311
|
-
);
|
|
1588
|
+
} else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
1589
|
+
const lastMessage = this.conversation.map((message) => message.role).lastIndexOf("user");
|
|
1312
1590
|
if (lastMessage !== -1) {
|
|
1313
1591
|
this.conversation = this.conversation.filter(
|
|
1314
1592
|
(_, index) => index !== lastMessage
|
|
@@ -1316,24 +1594,20 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1316
1594
|
this._isProcessing = false;
|
|
1317
1595
|
this.notifyStateChange();
|
|
1318
1596
|
}
|
|
1319
|
-
} else if (
|
|
1597
|
+
} else if (data.startsWith("ToolCall" /* ToolCall */)) {
|
|
1320
1598
|
try {
|
|
1321
1599
|
const toolCall = JSON.parse(
|
|
1322
|
-
|
|
1600
|
+
data.substring("ToolCall" /* ToolCall */.length + 1)
|
|
1323
1601
|
);
|
|
1324
1602
|
this.emit("ToolCall", toolCall);
|
|
1325
1603
|
} catch (error) {
|
|
1326
|
-
console.error(
|
|
1327
|
-
"[MicdropClient] Error parsing tool call:",
|
|
1328
|
-
event.data,
|
|
1329
|
-
error
|
|
1330
|
-
);
|
|
1604
|
+
console.error("[MicdropClient] Error parsing tool call:", data, error);
|
|
1331
1605
|
}
|
|
1332
1606
|
}
|
|
1333
1607
|
};
|
|
1334
1608
|
this.onWSClose = (event) => {
|
|
1335
|
-
this.log("WebSocket closed", event);
|
|
1336
|
-
const error = getClientErrorFromWSCloseEvent(event);
|
|
1609
|
+
this.log("WebSocket closed", event?.code, event?.reason);
|
|
1610
|
+
const error = getClientErrorFromWSCloseEvent(event ?? {});
|
|
1337
1611
|
if (error) {
|
|
1338
1612
|
this.setError(error);
|
|
1339
1613
|
}
|
|
@@ -1347,7 +1621,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1347
1621
|
this.stop();
|
|
1348
1622
|
}
|
|
1349
1623
|
};
|
|
1350
|
-
this.onWSError = (
|
|
1624
|
+
this.onWSError = () => {
|
|
1351
1625
|
this.setError(new MicdropClientError("Connection" /* Connection */));
|
|
1352
1626
|
};
|
|
1353
1627
|
this.onSpeakerStartPlaying = () => {
|
|
@@ -1370,9 +1644,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1370
1644
|
this.notifyStateChange();
|
|
1371
1645
|
}
|
|
1372
1646
|
};
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1647
|
+
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1648
|
+
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1649
|
+
Mic.on("DeviceChange", this.updateDevices);
|
|
1376
1650
|
}
|
|
1377
1651
|
get vad() {
|
|
1378
1652
|
return this.micRecorder?.vad;
|
|
@@ -1399,13 +1673,13 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1399
1673
|
return this.isMicStarted && !this.isPaused && !this.isProcessing && !this.isMicMuted && !this.isUserSpeaking && !this.isAssistantSpeaking;
|
|
1400
1674
|
}
|
|
1401
1675
|
get isWSStarted() {
|
|
1402
|
-
return this.ws?.readyState ===
|
|
1676
|
+
return this.ws?.readyState === 1;
|
|
1403
1677
|
}
|
|
1404
1678
|
get isWSStarting() {
|
|
1405
|
-
return this.ws?.readyState ===
|
|
1679
|
+
return this.ws?.readyState === 0;
|
|
1406
1680
|
}
|
|
1407
1681
|
get isMicStarted() {
|
|
1408
|
-
return
|
|
1682
|
+
return Mic.isStarted;
|
|
1409
1683
|
}
|
|
1410
1684
|
get isMicMuted() {
|
|
1411
1685
|
return this.vad?.isPaused ?? false;
|
|
@@ -1414,13 +1688,13 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1414
1688
|
return this.micRecorder?.state.isSpeaking ?? false;
|
|
1415
1689
|
}
|
|
1416
1690
|
get isAssistantSpeaking() {
|
|
1417
|
-
return
|
|
1691
|
+
return Speaker.isPlaying;
|
|
1418
1692
|
}
|
|
1419
1693
|
get micDeviceId() {
|
|
1420
|
-
return
|
|
1694
|
+
return Mic.deviceId;
|
|
1421
1695
|
}
|
|
1422
1696
|
get speakerDeviceId() {
|
|
1423
|
-
return
|
|
1697
|
+
return Speaker.deviceId;
|
|
1424
1698
|
}
|
|
1425
1699
|
get state() {
|
|
1426
1700
|
return {
|
|
@@ -1443,11 +1717,10 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1443
1717
|
speakerDevices: this.speakerDevices
|
|
1444
1718
|
};
|
|
1445
1719
|
}
|
|
1446
|
-
stopMic() {
|
|
1720
|
+
async stopMic() {
|
|
1447
1721
|
this.micRecorder?.stop();
|
|
1448
1722
|
this.micRecorder = void 0;
|
|
1449
|
-
|
|
1450
|
-
this.micStream = void 0;
|
|
1723
|
+
await Mic.stop();
|
|
1451
1724
|
this.notifyStateChange();
|
|
1452
1725
|
}
|
|
1453
1726
|
async startWS() {
|
|
@@ -1463,22 +1736,22 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1463
1736
|
throw new MicdropClientError("MissingUrl" /* MissingUrl */);
|
|
1464
1737
|
}
|
|
1465
1738
|
this.ws = new WebSocket(this.options.url);
|
|
1466
|
-
this.ws.binaryType = "
|
|
1739
|
+
this.ws.binaryType = "arraybuffer";
|
|
1467
1740
|
this.notifyStateChange();
|
|
1468
1741
|
this.ws.onopen = this.onWSOpen;
|
|
1469
1742
|
this.ws.onmessage = this.onWSMessage;
|
|
1470
1743
|
this.ws.onclose = this.onWSClose;
|
|
1471
1744
|
this.ws.onerror = this.onWSError;
|
|
1472
|
-
this.connectionTimer =
|
|
1745
|
+
this.connectionTimer = setTimeout(() => {
|
|
1473
1746
|
this.connectionTimer = void 0;
|
|
1474
|
-
if (this.ws?.readyState ===
|
|
1747
|
+
if (this.ws?.readyState === 0) {
|
|
1475
1748
|
this.log("WebSocket connection timeout");
|
|
1476
1749
|
this.ws.close();
|
|
1477
1750
|
}
|
|
1478
1751
|
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
1479
1752
|
} catch (error) {
|
|
1480
1753
|
this.setError(
|
|
1481
|
-
new MicdropClientError(
|
|
1754
|
+
error instanceof MicdropClientError ? error : new MicdropClientError(
|
|
1482
1755
|
"Connection" /* Connection */,
|
|
1483
1756
|
error?.message
|
|
1484
1757
|
)
|
|
@@ -1489,11 +1762,11 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1489
1762
|
}
|
|
1490
1763
|
stopWS() {
|
|
1491
1764
|
if (this.connectionTimer) {
|
|
1492
|
-
|
|
1765
|
+
clearTimeout(this.connectionTimer);
|
|
1493
1766
|
this.connectionTimer = void 0;
|
|
1494
1767
|
}
|
|
1495
1768
|
if (!this.ws) return;
|
|
1496
|
-
if (this.ws.readyState ===
|
|
1769
|
+
if (this.ws.readyState === 1 || this.ws.readyState === 0) {
|
|
1497
1770
|
this.ws.close();
|
|
1498
1771
|
}
|
|
1499
1772
|
this.ws = void 0;
|
|
@@ -1513,9 +1786,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1513
1786
|
this.vad?.pause?.();
|
|
1514
1787
|
}
|
|
1515
1788
|
if (this.reconnectTimer) {
|
|
1516
|
-
|
|
1789
|
+
clearTimeout(this.reconnectTimer);
|
|
1517
1790
|
}
|
|
1518
|
-
this.reconnectTimer =
|
|
1791
|
+
this.reconnectTimer = setTimeout(async () => {
|
|
1519
1792
|
try {
|
|
1520
1793
|
await this.startWS();
|
|
1521
1794
|
} catch {
|
|
@@ -1545,14 +1818,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1545
1818
|
};
|
|
1546
1819
|
|
|
1547
1820
|
// src/Micdrop.ts
|
|
1548
|
-
|
|
1549
|
-
|
|
1821
|
+
var globalScope2 = globalThis;
|
|
1822
|
+
if (!globalScope2.micdropClient) {
|
|
1823
|
+
globalScope2.micdropClient = new MicdropClient();
|
|
1550
1824
|
}
|
|
1551
|
-
var Micdrop =
|
|
1825
|
+
var Micdrop = globalScope2.micdropClient;
|
|
1552
1826
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1553
1827
|
0 && (module.exports = {
|
|
1554
|
-
LocalStorageKeys,
|
|
1555
1828
|
Mic,
|
|
1829
|
+
MicController,
|
|
1830
|
+
MicDriver,
|
|
1556
1831
|
MicRecorder,
|
|
1557
1832
|
Micdrop,
|
|
1558
1833
|
MicdropClient,
|
|
@@ -1560,16 +1835,33 @@ var Micdrop = window.micdropClient;
|
|
|
1560
1835
|
MicdropClientError,
|
|
1561
1836
|
MicdropClientErrorCode,
|
|
1562
1837
|
MicdropServerCommands,
|
|
1838
|
+
MicdropStorageKeys,
|
|
1563
1839
|
MultipleVAD,
|
|
1840
|
+
Pcm16AudioStream,
|
|
1841
|
+
SILERO_FRAME_SAMPLES,
|
|
1842
|
+
SILERO_SAMPLE_RATE,
|
|
1564
1843
|
SileroVAD,
|
|
1565
1844
|
Speaker,
|
|
1845
|
+
SpeakerController,
|
|
1846
|
+
SpeakerDriver,
|
|
1566
1847
|
VAD,
|
|
1567
1848
|
VADStatus,
|
|
1849
|
+
VOLUME_SAMPLE_RATE,
|
|
1850
|
+
VolumeMeter,
|
|
1568
1851
|
VolumeVAD,
|
|
1852
|
+
concatFloat32,
|
|
1569
1853
|
equalVADConfig,
|
|
1854
|
+
floatToPcm16,
|
|
1570
1855
|
getClientErrorFromWSCloseEvent,
|
|
1571
1856
|
getClientErrorFromWSCloseEventCode,
|
|
1572
1857
|
getVAD,
|
|
1573
|
-
|
|
1858
|
+
getVolume,
|
|
1859
|
+
isRecoverableError,
|
|
1860
|
+
pcm16ToArrayBuffer,
|
|
1861
|
+
pcm16ToFloat,
|
|
1862
|
+
resample,
|
|
1863
|
+
setMicdropStorage,
|
|
1864
|
+
setSileroModelLoader,
|
|
1865
|
+
storage
|
|
1574
1866
|
});
|
|
1575
1867
|
//# sourceMappingURL=index.js.map
|