@micdrop/client 2.2.10 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -23
- package/dist/index.d.mts +581 -120
- package/dist/index.d.ts +581 -120
- package/dist/index.js +1100 -696
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1079 -694
- package/dist/index.mjs.map +1 -1
- package/package.json +10 -6
package/dist/index.js
CHANGED
|
@@ -20,8 +20,10 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
|
-
|
|
24
|
-
Mic: () =>
|
|
23
|
+
DEFAULT_TURN_MAX_WAIT: () => DEFAULT_TURN_MAX_WAIT,
|
|
24
|
+
Mic: () => Mic,
|
|
25
|
+
MicController: () => MicController,
|
|
26
|
+
MicDriver: () => MicDriver,
|
|
25
27
|
MicRecorder: () => MicRecorder,
|
|
26
28
|
Micdrop: () => Micdrop,
|
|
27
29
|
MicdropClient: () => MicdropClient,
|
|
@@ -29,454 +31,486 @@ __export(index_exports, {
|
|
|
29
31
|
MicdropClientError: () => MicdropClientError,
|
|
30
32
|
MicdropClientErrorCode: () => MicdropClientErrorCode,
|
|
31
33
|
MicdropServerCommands: () => MicdropServerCommands,
|
|
34
|
+
MicdropStorageKeys: () => MicdropStorageKeys,
|
|
32
35
|
MultipleVAD: () => MultipleVAD,
|
|
36
|
+
Pcm16AudioStream: () => Pcm16AudioStream,
|
|
37
|
+
SILERO_FRAME_SAMPLES: () => SILERO_FRAME_SAMPLES,
|
|
38
|
+
SILERO_SAMPLE_RATE: () => SILERO_SAMPLE_RATE,
|
|
33
39
|
SileroVAD: () => SileroVAD,
|
|
34
|
-
Speaker: () =>
|
|
40
|
+
Speaker: () => Speaker,
|
|
41
|
+
SpeakerController: () => SpeakerController,
|
|
42
|
+
SpeakerDriver: () => SpeakerDriver,
|
|
35
43
|
VAD: () => VAD,
|
|
36
44
|
VADStatus: () => VADStatus,
|
|
45
|
+
VOLUME_SAMPLE_RATE: () => VOLUME_SAMPLE_RATE,
|
|
46
|
+
VolumeMeter: () => VolumeMeter,
|
|
37
47
|
VolumeVAD: () => VolumeVAD,
|
|
48
|
+
concatFloat32: () => concatFloat32,
|
|
38
49
|
equalVADConfig: () => equalVADConfig,
|
|
50
|
+
floatToPcm16: () => floatToPcm16,
|
|
39
51
|
getClientErrorFromWSCloseEvent: () => getClientErrorFromWSCloseEvent,
|
|
40
52
|
getClientErrorFromWSCloseEventCode: () => getClientErrorFromWSCloseEventCode,
|
|
41
53
|
getVAD: () => getVAD,
|
|
42
|
-
|
|
54
|
+
getVolume: () => getVolume,
|
|
55
|
+
isRecoverableError: () => isRecoverableError,
|
|
56
|
+
pcm16ToArrayBuffer: () => pcm16ToArrayBuffer,
|
|
57
|
+
pcm16ToFloat: () => pcm16ToFloat,
|
|
58
|
+
resample: () => resample,
|
|
59
|
+
setMicdropStorage: () => setMicdropStorage,
|
|
60
|
+
setSileroModelLoader: () => setSileroModelLoader,
|
|
61
|
+
storage: () => storage
|
|
43
62
|
});
|
|
44
63
|
module.exports = __toCommonJS(index_exports);
|
|
45
64
|
|
|
46
|
-
// src/audio/
|
|
65
|
+
// src/audio/Mic.ts
|
|
47
66
|
var import_eventemitter3 = require("eventemitter3");
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
+
|
|
68
|
+
// src/audio/pcm.ts
|
|
69
|
+
function resample(input, inputRate, outputRate) {
|
|
70
|
+
if (inputRate === outputRate || input.length === 0) return input;
|
|
71
|
+
const ratio = inputRate / outputRate;
|
|
72
|
+
const outputLength = Math.round(input.length / ratio);
|
|
73
|
+
const output = new Float32Array(outputLength);
|
|
74
|
+
for (let i = 0; i < outputLength; i++) {
|
|
75
|
+
const position = i * ratio;
|
|
76
|
+
const index = Math.floor(position);
|
|
77
|
+
const nextIndex = Math.min(index + 1, input.length - 1);
|
|
78
|
+
const fraction = position - index;
|
|
79
|
+
output[i] = input[index] * (1 - fraction) + input[nextIndex] * fraction;
|
|
80
|
+
}
|
|
81
|
+
return output;
|
|
82
|
+
}
|
|
83
|
+
function floatToPcm16(input) {
|
|
84
|
+
const output = new Int16Array(input.length);
|
|
85
|
+
for (let i = 0; i < input.length; i++) {
|
|
86
|
+
const sample = Math.max(-1, Math.min(1, input[i]));
|
|
87
|
+
output[i] = sample < 0 ? sample * 32768 : sample * 32767;
|
|
67
88
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
maxVolume = this.fftBins[i];
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return maxVolume;
|
|
89
|
+
return output;
|
|
90
|
+
}
|
|
91
|
+
function pcm16ToFloat(input) {
|
|
92
|
+
const output = new Float32Array(input.length);
|
|
93
|
+
for (let i = 0; i < input.length; i++) {
|
|
94
|
+
output[i] = input[i] / 32768;
|
|
78
95
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
// src/audio/utils/unlock.ts
|
|
82
|
-
function unlock(audioContext2) {
|
|
83
|
-
const events = ["touchstart", "touchend", "mousedown", "keydown"];
|
|
84
|
-
const unlock2 = async () => {
|
|
85
|
-
unlockSafari(audioContext2);
|
|
86
|
-
if (audioContext2.state === "suspended") {
|
|
87
|
-
await audioContext2.resume();
|
|
88
|
-
}
|
|
89
|
-
events.forEach((e) => document.body.removeEventListener(e, unlock2));
|
|
90
|
-
};
|
|
91
|
-
events.forEach((e) => document.body.addEventListener(e, unlock2, false));
|
|
96
|
+
return output;
|
|
92
97
|
}
|
|
93
|
-
function
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
98
|
+
function concatFloat32(chunks) {
|
|
99
|
+
if (chunks.length === 1) return chunks[0];
|
|
100
|
+
let length = 0;
|
|
101
|
+
for (const chunk of chunks) length += chunk.length;
|
|
102
|
+
const output = new Float32Array(length);
|
|
103
|
+
let offset = 0;
|
|
104
|
+
for (const chunk of chunks) {
|
|
105
|
+
output.set(chunk, offset);
|
|
106
|
+
offset += chunk.length;
|
|
107
|
+
}
|
|
108
|
+
return output;
|
|
109
|
+
}
|
|
110
|
+
function pcm16ToArrayBuffer(pcm) {
|
|
111
|
+
return pcm.buffer.slice(
|
|
112
|
+
pcm.byteOffset,
|
|
113
|
+
pcm.byteOffset + pcm.byteLength
|
|
114
|
+
);
|
|
104
115
|
}
|
|
105
116
|
|
|
106
|
-
// src/audio/
|
|
107
|
-
var
|
|
108
|
-
|
|
109
|
-
|
|
117
|
+
// src/audio/volume.ts
|
|
118
|
+
var FFT_SIZE = 512;
|
|
119
|
+
var BIN_COUNT = FFT_SIZE / 2;
|
|
120
|
+
var FIRST_BIN = 4;
|
|
121
|
+
var SMOOTHING = 0.1;
|
|
122
|
+
var VOLUME_SAMPLE_RATE = 16e3;
|
|
123
|
+
var WINDOW = new Float32Array(FFT_SIZE);
|
|
124
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
125
|
+
const a = 2 * Math.PI * i / (FFT_SIZE - 1);
|
|
126
|
+
WINDOW[i] = 0.42 - 0.5 * Math.cos(a) + 0.08 * Math.cos(2 * a);
|
|
110
127
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
128
|
+
var REVERSED = new Uint16Array(FFT_SIZE);
|
|
129
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
130
|
+
let reversed = 0;
|
|
131
|
+
for (let bit = 1; bit < FFT_SIZE; bit <<= 1) {
|
|
132
|
+
reversed = reversed << 1 | (i & bit ? 1 : 0);
|
|
133
|
+
}
|
|
134
|
+
REVERSED[i] = reversed;
|
|
116
135
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
LocalStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
123
|
-
LocalStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
124
|
-
return LocalStorageKeys2;
|
|
125
|
-
})(LocalStorageKeys || {});
|
|
126
|
-
|
|
127
|
-
// src/audio/utils/stopStream.ts
|
|
128
|
-
function stopStream(stream) {
|
|
129
|
-
stream?.getTracks().forEach((track) => track.stop());
|
|
136
|
+
var COS = new Float32Array(FFT_SIZE / 2);
|
|
137
|
+
var SIN = new Float32Array(FFT_SIZE / 2);
|
|
138
|
+
for (let i = 0; i < FFT_SIZE / 2; i++) {
|
|
139
|
+
COS[i] = Math.cos(-2 * Math.PI * i / FFT_SIZE);
|
|
140
|
+
SIN[i] = Math.sin(-2 * Math.PI * i / FFT_SIZE);
|
|
130
141
|
}
|
|
131
|
-
|
|
132
|
-
// src/audio/Mic.ts
|
|
133
|
-
var Mic = class {
|
|
142
|
+
var VolumeMeter = class {
|
|
134
143
|
constructor() {
|
|
135
|
-
this.
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
deviceId: { ideal: deviceId },
|
|
141
|
-
sampleRate: 16e3,
|
|
142
|
-
// not working, it will follow device settings, usually 44.1kHz
|
|
143
|
-
sampleSize: 16,
|
|
144
|
-
channelCount: 1,
|
|
145
|
-
echoCancellation: true,
|
|
146
|
-
noiseSuppression: true
|
|
147
|
-
}
|
|
148
|
-
};
|
|
144
|
+
this.real = new Float32Array(FFT_SIZE);
|
|
145
|
+
this.imaginary = new Float32Array(FFT_SIZE);
|
|
146
|
+
this.magnitudes = new Float32Array(BIN_COUNT);
|
|
147
|
+
this.smoothed = new Float32Array(BIN_COUNT);
|
|
148
|
+
this.hasPrevious = false;
|
|
149
149
|
}
|
|
150
150
|
/**
|
|
151
|
-
*
|
|
152
|
-
* @param
|
|
153
|
-
*
|
|
151
|
+
* Measures a window of audio
|
|
152
|
+
* @param samples - At least 512 mono samples at 16 kHz, in the -1..1 range.
|
|
153
|
+
* Only the last 512 are looked at.
|
|
154
|
+
* @returns The level in decibels, between -Infinity (silence) and 0
|
|
154
155
|
*/
|
|
155
|
-
|
|
156
|
-
if (
|
|
157
|
-
|
|
156
|
+
measure(samples) {
|
|
157
|
+
if (samples.length === 0) return -Infinity;
|
|
158
|
+
const offset = Math.max(0, samples.length - FFT_SIZE);
|
|
159
|
+
const { real, imaginary } = this;
|
|
160
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
161
|
+
const sample = offset + i < samples.length ? samples[offset + i] : 0;
|
|
162
|
+
real[REVERSED[i]] = sample * WINDOW[i];
|
|
163
|
+
imaginary[REVERSED[i]] = 0;
|
|
158
164
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
+
for (let size = 2; size <= FFT_SIZE; size <<= 1) {
|
|
166
|
+
const half = size >> 1;
|
|
167
|
+
const step = FFT_SIZE / size;
|
|
168
|
+
for (let start = 0; start < FFT_SIZE; start += size) {
|
|
169
|
+
for (let i = 0; i < half; i++) {
|
|
170
|
+
const even = start + i;
|
|
171
|
+
const odd = even + half;
|
|
172
|
+
const cos = COS[i * step];
|
|
173
|
+
const sin = SIN[i * step];
|
|
174
|
+
const realOdd = real[odd] * cos - imaginary[odd] * sin;
|
|
175
|
+
const imaginaryOdd = real[odd] * sin + imaginary[odd] * cos;
|
|
176
|
+
real[odd] = real[even] - realOdd;
|
|
177
|
+
imaginary[odd] = imaginary[even] - imaginaryOdd;
|
|
178
|
+
real[even] += realOdd;
|
|
179
|
+
imaginary[even] += imaginaryOdd;
|
|
180
|
+
}
|
|
165
181
|
}
|
|
166
|
-
stopStream(this.audioStream);
|
|
167
182
|
}
|
|
168
|
-
|
|
169
|
-
this.
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (newDeviceId === "default" && deviceId) {
|
|
175
|
-
newDeviceId = deviceId;
|
|
183
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
184
|
+
this.magnitudes[k] = Math.sqrt(real[k] * real[k] + imaginary[k] * imaginary[k]) / FFT_SIZE;
|
|
185
|
+
}
|
|
186
|
+
if (this.hasPrevious) {
|
|
187
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
188
|
+
this.smoothed[k] = SMOOTHING * this.smoothed[k] + (1 - SMOOTHING) * this.magnitudes[k];
|
|
176
189
|
}
|
|
177
|
-
localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
|
|
178
|
-
this.deviceId = newDeviceId;
|
|
179
190
|
} else {
|
|
180
|
-
|
|
181
|
-
this.
|
|
191
|
+
this.smoothed.set(this.magnitudes);
|
|
192
|
+
this.hasPrevious = true;
|
|
182
193
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
194
|
+
let loudest = -Infinity;
|
|
195
|
+
for (let k = FIRST_BIN; k < BIN_COUNT; k++) {
|
|
196
|
+
const magnitude = this.smoothed[k];
|
|
197
|
+
if (magnitude === 0) continue;
|
|
198
|
+
const decibels = 20 * Math.log10(magnitude);
|
|
199
|
+
if (decibels > loudest && decibels < 0) loudest = decibels;
|
|
200
|
+
}
|
|
201
|
+
return loudest;
|
|
202
|
+
}
|
|
203
|
+
/** Forgets the previous window, for when capture stops and starts again */
|
|
204
|
+
reset() {
|
|
205
|
+
this.hasPrevious = false;
|
|
206
|
+
this.smoothed.fill(0);
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
function getVolume(samples) {
|
|
210
|
+
return new VolumeMeter().measure(samples);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// src/audio/Mic.ts
|
|
214
|
+
var VOLUME_INTERVAL = 100;
|
|
215
|
+
var VOLUME_WINDOW = 512;
|
|
216
|
+
var RATE_CHECK_DURATION = 2e3;
|
|
217
|
+
var RATE_TOLERANCE = 0.15;
|
|
218
|
+
var MicController = class extends import_eventemitter3.EventEmitter {
|
|
219
|
+
constructor() {
|
|
220
|
+
super(...arguments);
|
|
221
|
+
this.meter = new VolumeMeter();
|
|
222
|
+
this.pending = [];
|
|
223
|
+
this.pendingLength = 0;
|
|
224
|
+
this.pendingRate = VOLUME_SAMPLE_RATE;
|
|
225
|
+
this.rateCheckedAt = 0;
|
|
226
|
+
this.rateCheckSamples = 0;
|
|
227
|
+
this.rateWarned = false;
|
|
228
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
229
|
+
this.volume = -Infinity;
|
|
230
|
+
this.onFrames = (frames, sampleRate) => {
|
|
231
|
+
this.emit("Frames", frames, sampleRate);
|
|
232
|
+
this.checkRate(frames.length, sampleRate);
|
|
233
|
+
if (sampleRate !== this.pendingRate) {
|
|
234
|
+
this.pending = [];
|
|
235
|
+
this.pendingLength = 0;
|
|
236
|
+
this.pendingRate = sampleRate;
|
|
237
|
+
this.meter.reset();
|
|
238
|
+
}
|
|
239
|
+
this.pending.push(frames);
|
|
240
|
+
this.pendingLength += frames.length;
|
|
241
|
+
const interval = Math.round(VOLUME_INTERVAL / 1e3 * sampleRate);
|
|
242
|
+
if (this.pendingLength < interval) return;
|
|
243
|
+
const needed = Math.ceil(VOLUME_WINDOW * sampleRate / VOLUME_SAMPLE_RATE);
|
|
244
|
+
const gathered = concatFloat32(this.pending);
|
|
245
|
+
const window = gathered.subarray(Math.max(0, gathered.length - needed));
|
|
246
|
+
this.pending = [];
|
|
247
|
+
this.pendingLength = 0;
|
|
248
|
+
this.volume = this.meter.measure(
|
|
249
|
+
resample(window, sampleRate, VOLUME_SAMPLE_RATE)
|
|
250
|
+
);
|
|
251
|
+
this.emit("Volume", this.volume);
|
|
252
|
+
};
|
|
253
|
+
this.onDeviceChange = () => {
|
|
254
|
+
this.emit("DeviceChange");
|
|
255
|
+
};
|
|
256
|
+
this.onError = (error) => {
|
|
257
|
+
this.emit("Error", error);
|
|
258
|
+
};
|
|
187
259
|
}
|
|
188
260
|
/**
|
|
189
|
-
*
|
|
261
|
+
* Replaces the audio driver, to record with another library
|
|
262
|
+
* @param driver - The driver to record with
|
|
190
263
|
*/
|
|
191
|
-
|
|
192
|
-
if (this.
|
|
193
|
-
|
|
194
|
-
this.
|
|
264
|
+
setDriver(driver) {
|
|
265
|
+
if (this.driver === driver) return;
|
|
266
|
+
if (this.driver) {
|
|
267
|
+
this.driver.off("Frames", this.onFrames);
|
|
268
|
+
this.driver.off("DeviceChange", this.onDeviceChange);
|
|
269
|
+
this.driver.off("Error", this.onError);
|
|
195
270
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
271
|
+
this.driver = driver;
|
|
272
|
+
driver.on("Frames", this.onFrames);
|
|
273
|
+
driver.on("DeviceChange", this.onDeviceChange);
|
|
274
|
+
driver.on("Error", this.onError);
|
|
275
|
+
}
|
|
276
|
+
getDriver() {
|
|
277
|
+
if (!this.driver) {
|
|
278
|
+
throw new Error(
|
|
279
|
+
"No microphone driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Mic.setDriver()."
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
return this.driver;
|
|
283
|
+
}
|
|
284
|
+
get isStarted() {
|
|
285
|
+
return this.driver?.isStarted ?? false;
|
|
286
|
+
}
|
|
287
|
+
get deviceId() {
|
|
288
|
+
return this.driver?.deviceId;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Asks for the microphone permission and starts capturing
|
|
292
|
+
* @param deviceId - Input device to record from, defaults to the system one
|
|
293
|
+
*/
|
|
294
|
+
async start(deviceId) {
|
|
295
|
+
this.resetVolume();
|
|
296
|
+
await this.getDriver().start(deviceId);
|
|
297
|
+
}
|
|
298
|
+
/** Stops capturing and releases the microphone */
|
|
299
|
+
async stop() {
|
|
300
|
+
await this.driver?.stop();
|
|
301
|
+
this.resetVolume();
|
|
302
|
+
}
|
|
303
|
+
/** Lists the input devices the system offers */
|
|
304
|
+
async getDevices() {
|
|
305
|
+
if (!this.driver) return [];
|
|
306
|
+
return this.driver.getDevices();
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Times the captured audio against the rate it claims to be at.
|
|
310
|
+
*
|
|
311
|
+
* A microphone that announces 16 kHz while delivering 48 kHz sends speech to
|
|
312
|
+
* the server three times too slow, which comes back as a short transcript in
|
|
313
|
+
* a random language. That is invisible otherwise, so it is worth saying out
|
|
314
|
+
* loud once.
|
|
315
|
+
* @param samples - How many samples just arrived
|
|
316
|
+
* @param sampleRate - The rate they are said to be at
|
|
317
|
+
*/
|
|
318
|
+
checkRate(samples, sampleRate) {
|
|
319
|
+
if (this.rateWarned) return;
|
|
320
|
+
const now = Date.now();
|
|
321
|
+
if (this.rateCheckedAt === 0) {
|
|
322
|
+
this.rateCheckedAt = now;
|
|
323
|
+
return;
|
|
199
324
|
}
|
|
325
|
+
this.rateCheckSamples += samples;
|
|
326
|
+
const elapsed = now - this.rateCheckedAt;
|
|
327
|
+
if (elapsed < RATE_CHECK_DURATION) return;
|
|
328
|
+
const delivered = this.rateCheckSamples / elapsed * 1e3;
|
|
329
|
+
this.rateCheckedAt = now;
|
|
330
|
+
this.rateCheckSamples = 0;
|
|
331
|
+
if (Math.abs(delivered - sampleRate) / sampleRate <= RATE_TOLERANCE) return;
|
|
332
|
+
this.rateWarned = true;
|
|
333
|
+
console.warn(
|
|
334
|
+
`[Micdrop] The microphone announces ${sampleRate} Hz but delivers about ${Math.round(delivered)} Hz. Speech is being sent at the wrong speed, so transcription will be poor.`
|
|
335
|
+
);
|
|
336
|
+
}
|
|
337
|
+
resetVolume() {
|
|
338
|
+
this.pending = [];
|
|
339
|
+
this.pendingLength = 0;
|
|
340
|
+
this.meter.reset();
|
|
341
|
+
this.volume = -Infinity;
|
|
342
|
+
this.rateCheckedAt = 0;
|
|
343
|
+
this.rateCheckSamples = 0;
|
|
200
344
|
}
|
|
201
345
|
};
|
|
202
346
|
|
|
203
347
|
// src/audio/Speaker.ts
|
|
204
|
-
var import_eventemitter33 = require("eventemitter3");
|
|
205
|
-
|
|
206
|
-
// src/audio/stream/AudioStream.ts
|
|
207
348
|
var import_eventemitter32 = require("eventemitter3");
|
|
208
|
-
var AudioStream = class extends import_eventemitter32.EventEmitter {
|
|
209
|
-
constructor() {
|
|
210
|
-
super();
|
|
211
|
-
this.isPlaying = false;
|
|
212
|
-
this.outputNode = audioContext.createGain();
|
|
213
|
-
}
|
|
214
|
-
setIsPlaying(isPlaying) {
|
|
215
|
-
if (this.isPlaying === isPlaying) return;
|
|
216
|
-
this.isPlaying = isPlaying;
|
|
217
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
218
|
-
}
|
|
219
|
-
};
|
|
220
349
|
|
|
221
|
-
// src/
|
|
222
|
-
var
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
350
|
+
// src/storage.ts
|
|
351
|
+
var MicdropStorageKeys = /* @__PURE__ */ ((MicdropStorageKeys2) => {
|
|
352
|
+
MicdropStorageKeys2["MicDevice"] = "micdrop.micDevice";
|
|
353
|
+
MicdropStorageKeys2["SpeakerDevice"] = "micdrop.speakerDevice";
|
|
354
|
+
MicdropStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
355
|
+
MicdropStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
356
|
+
return MicdropStorageKeys2;
|
|
357
|
+
})(MicdropStorageKeys || {});
|
|
358
|
+
var MemoryStorage = class {
|
|
228
359
|
constructor() {
|
|
229
|
-
|
|
230
|
-
this.blobQueue = [];
|
|
231
|
-
this.processing = false;
|
|
232
|
-
this.sourceNodes = [];
|
|
233
|
-
this.nextStartTime = 0;
|
|
234
|
-
this.prebuffering = true;
|
|
235
|
-
this.prebuffer = [];
|
|
236
|
-
this.prebufferDuration = 0;
|
|
360
|
+
this.values = /* @__PURE__ */ new Map();
|
|
237
361
|
}
|
|
238
|
-
|
|
239
|
-
return this.
|
|
362
|
+
getItem(key) {
|
|
363
|
+
return this.values.get(key) ?? null;
|
|
240
364
|
}
|
|
241
|
-
|
|
242
|
-
this.
|
|
243
|
-
if (this.processing) return;
|
|
244
|
-
this.processing = true;
|
|
245
|
-
try {
|
|
246
|
-
while (this.blobQueue.length > 0) {
|
|
247
|
-
const next = this.blobQueue.shift();
|
|
248
|
-
try {
|
|
249
|
-
const arrayBuffer = await next.arrayBuffer();
|
|
250
|
-
const audioBuffer = this.createAudioBuffer(arrayBuffer);
|
|
251
|
-
if (audioBuffer) this.handleBuffer(audioBuffer);
|
|
252
|
-
} catch (error) {
|
|
253
|
-
console.error("Failed to play audio chunk:", error);
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
} finally {
|
|
257
|
-
this.processing = false;
|
|
258
|
-
}
|
|
365
|
+
setItem(key, value) {
|
|
366
|
+
this.values.set(key, value);
|
|
259
367
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
try {
|
|
263
|
-
node.stop();
|
|
264
|
-
node.disconnect();
|
|
265
|
-
} catch (error) {
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
this.sourceNodes = [];
|
|
269
|
-
this.blobQueue = [];
|
|
270
|
-
this.nextStartTime = 0;
|
|
271
|
-
this.resetPrebuffer();
|
|
272
|
-
this.setIsPlaying(false);
|
|
273
|
-
}
|
|
274
|
-
// Accumulate audio until enough is buffered, then start playing. Once
|
|
275
|
-
// playing, schedule incoming buffers directly.
|
|
276
|
-
handleBuffer(audioBuffer) {
|
|
277
|
-
if (this.endTimer !== void 0) {
|
|
278
|
-
clearTimeout(this.endTimer);
|
|
279
|
-
this.endTimer = void 0;
|
|
280
|
-
}
|
|
281
|
-
if (!this.prebuffering) {
|
|
282
|
-
this.scheduleBuffer(audioBuffer);
|
|
283
|
-
return;
|
|
284
|
-
}
|
|
285
|
-
this.prebuffer.push(audioBuffer);
|
|
286
|
-
this.prebufferDuration += audioBuffer.duration * 1e3;
|
|
287
|
-
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
288
|
-
this.flushPrebuffer();
|
|
289
|
-
return;
|
|
290
|
-
}
|
|
291
|
-
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
292
|
-
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
293
|
-
}
|
|
294
|
-
flushPrebuffer() {
|
|
295
|
-
if (this.quietTimer !== void 0) {
|
|
296
|
-
clearTimeout(this.quietTimer);
|
|
297
|
-
this.quietTimer = void 0;
|
|
298
|
-
}
|
|
299
|
-
this.prebuffering = false;
|
|
300
|
-
const buffers = this.prebuffer;
|
|
301
|
-
this.prebuffer = [];
|
|
302
|
-
this.prebufferDuration = 0;
|
|
303
|
-
for (const audioBuffer of buffers) this.scheduleBuffer(audioBuffer);
|
|
368
|
+
removeItem(key) {
|
|
369
|
+
this.values.delete(key);
|
|
304
370
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
371
|
+
};
|
|
372
|
+
var current = new MemoryStorage();
|
|
373
|
+
function setMicdropStorage(storage2) {
|
|
374
|
+
current = storage2;
|
|
375
|
+
}
|
|
376
|
+
var storage = {
|
|
377
|
+
getItem: (key) => {
|
|
378
|
+
try {
|
|
379
|
+
return current.getItem(key);
|
|
380
|
+
} catch {
|
|
381
|
+
return null;
|
|
309
382
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
383
|
+
},
|
|
384
|
+
setItem: (key, value) => {
|
|
385
|
+
try {
|
|
386
|
+
current.setItem(key, value);
|
|
387
|
+
} catch {
|
|
313
388
|
}
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
this.prebufferDuration = 0;
|
|
317
|
-
}
|
|
318
|
-
createAudioBuffer(arrayBuffer) {
|
|
389
|
+
},
|
|
390
|
+
removeItem: (key) => {
|
|
319
391
|
try {
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
if (numSamples === 0) {
|
|
323
|
-
return null;
|
|
324
|
-
}
|
|
325
|
-
const audioBuffer = audioContext.createBuffer(
|
|
326
|
-
CHANNELS,
|
|
327
|
-
numSamples,
|
|
328
|
-
SAMPLE_RATE
|
|
329
|
-
);
|
|
330
|
-
const channelData = audioBuffer.getChannelData(0);
|
|
331
|
-
for (let i = 0; i < numSamples; i++) {
|
|
332
|
-
channelData[i] = int16Data[i] / 32768;
|
|
333
|
-
}
|
|
334
|
-
return audioBuffer;
|
|
335
|
-
} catch (error) {
|
|
336
|
-
console.error("Error creating AudioBuffer:", error);
|
|
337
|
-
return null;
|
|
392
|
+
current.removeItem(key);
|
|
393
|
+
} catch {
|
|
338
394
|
}
|
|
339
395
|
}
|
|
340
|
-
scheduleBuffer(audioBuffer) {
|
|
341
|
-
const sourceNode = audioContext.createBufferSource();
|
|
342
|
-
sourceNode.buffer = audioBuffer;
|
|
343
|
-
sourceNode.connect(this.outputNode);
|
|
344
|
-
const now = audioContext.currentTime;
|
|
345
|
-
const startTime = Math.max(this.nextStartTime, now);
|
|
346
|
-
this.nextStartTime = startTime + audioBuffer.duration;
|
|
347
|
-
sourceNode.onended = () => {
|
|
348
|
-
const index = this.sourceNodes.indexOf(sourceNode);
|
|
349
|
-
if (index !== -1) this.sourceNodes.splice(index, 1);
|
|
350
|
-
if (this.sourceNodes.length === 0 && this.blobQueue.length === 0) {
|
|
351
|
-
this.nextStartTime = 0;
|
|
352
|
-
this.setIsPlaying(false);
|
|
353
|
-
if (this.endTimer !== void 0) clearTimeout(this.endTimer);
|
|
354
|
-
this.endTimer = setTimeout(() => {
|
|
355
|
-
this.endTimer = void 0;
|
|
356
|
-
this.resetPrebuffer();
|
|
357
|
-
}, UTTERANCE_END_DELAY);
|
|
358
|
-
}
|
|
359
|
-
};
|
|
360
|
-
this.sourceNodes.push(sourceNode);
|
|
361
|
-
sourceNode.start(startTime);
|
|
362
|
-
this.setIsPlaying(true);
|
|
363
|
-
}
|
|
364
396
|
};
|
|
365
397
|
|
|
366
398
|
// src/audio/Speaker.ts
|
|
367
|
-
var
|
|
399
|
+
var SpeakerController = class extends import_eventemitter32.EventEmitter {
|
|
368
400
|
constructor() {
|
|
369
401
|
super(...arguments);
|
|
370
|
-
|
|
371
|
-
this.
|
|
372
|
-
|
|
373
|
-
this.
|
|
402
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
403
|
+
this.volume = -Infinity;
|
|
404
|
+
this.onStartPlaying = () => this.emit("StartPlaying");
|
|
405
|
+
this.onStopPlaying = () => {
|
|
406
|
+
this.volume = -Infinity;
|
|
407
|
+
this.emit("StopPlaying");
|
|
408
|
+
};
|
|
409
|
+
this.onVolume = (volume) => {
|
|
410
|
+
this.volume = volume;
|
|
411
|
+
this.emit("Volume", volume);
|
|
412
|
+
};
|
|
374
413
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
this.
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
async playAudio(blob) {
|
|
386
|
-
this.start();
|
|
387
|
-
try {
|
|
388
|
-
await this.audioStream?.playAudio(blob);
|
|
389
|
-
} catch (error) {
|
|
390
|
-
console.error("Error playing audio blob", error, blob);
|
|
414
|
+
/**
|
|
415
|
+
* Replaces the audio driver, to play sound with another library
|
|
416
|
+
* @param driver - The driver to play with
|
|
417
|
+
*/
|
|
418
|
+
setDriver(driver) {
|
|
419
|
+
if (this.driver === driver) return;
|
|
420
|
+
if (this.driver) {
|
|
421
|
+
this.driver.off("StartPlaying", this.onStartPlaying);
|
|
422
|
+
this.driver.off("StopPlaying", this.onStopPlaying);
|
|
423
|
+
this.driver.off("Volume", this.onVolume);
|
|
391
424
|
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
this.
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
425
|
+
this.driver = driver;
|
|
426
|
+
driver.on("StartPlaying", this.onStartPlaying);
|
|
427
|
+
driver.on("StopPlaying", this.onStopPlaying);
|
|
428
|
+
driver.on("Volume", this.onVolume);
|
|
429
|
+
}
|
|
430
|
+
getDriver() {
|
|
431
|
+
if (!this.driver) {
|
|
432
|
+
throw new Error(
|
|
433
|
+
"No speaker driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Speaker.setDriver()."
|
|
434
|
+
);
|
|
403
435
|
}
|
|
436
|
+
return this.driver;
|
|
404
437
|
}
|
|
405
|
-
get
|
|
406
|
-
return
|
|
438
|
+
get isPlaying() {
|
|
439
|
+
return this.driver?.isPlaying ?? false;
|
|
407
440
|
}
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
this.isPlaying = isPlaying;
|
|
411
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
441
|
+
get deviceId() {
|
|
442
|
+
return this.driver?.deviceId;
|
|
412
443
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
444
|
+
/** Prepares the audio graph, called before the first chunk arrives */
|
|
445
|
+
async start() {
|
|
446
|
+
const driver = this.getDriver();
|
|
447
|
+
await driver.start();
|
|
448
|
+
const saved = storage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
449
|
+
if (saved && saved !== driver.deviceId) {
|
|
450
|
+
try {
|
|
451
|
+
await driver.changeDevice(saved);
|
|
452
|
+
} catch {
|
|
453
|
+
storage.removeItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
454
|
+
}
|
|
419
455
|
}
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Queues samples for playback, right after the ones already queued
|
|
459
|
+
* @param pcm - Mono samples as 16 bits integers, or the bytes holding them
|
|
460
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
461
|
+
*/
|
|
462
|
+
playAudio(pcm, sampleRate = 16e3) {
|
|
420
463
|
try {
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
const sinkId = !deviceId || defaultDevice ? "" : deviceId;
|
|
424
|
-
if (defaultDevice || "setSinkId" in audioContext) {
|
|
425
|
-
await audioContext.setSinkId?.(sinkId);
|
|
426
|
-
this.audioSourceNode.disconnect();
|
|
427
|
-
this.audioSourceNode.connect(audioContext.destination);
|
|
428
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
429
|
-
if (this.audioElementDestination) {
|
|
430
|
-
this.audioElementDestination.pause();
|
|
431
|
-
this.audioElementDestination.srcObject = null;
|
|
432
|
-
this.audioElementDestination = void 0;
|
|
433
|
-
}
|
|
434
|
-
} else {
|
|
435
|
-
if (!this.audioElementDestination) {
|
|
436
|
-
const mediaStreamDestination = audioContext.createMediaStreamDestination();
|
|
437
|
-
this.audioSourceNode.disconnect();
|
|
438
|
-
this.audioSourceNode.connect(mediaStreamDestination);
|
|
439
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
440
|
-
this.audioElementDestination = new Audio();
|
|
441
|
-
this.audioElementDestination.autoplay = true;
|
|
442
|
-
this.audioElementDestination.srcObject = mediaStreamDestination.stream;
|
|
443
|
-
}
|
|
444
|
-
await this.audioElementDestination.setSinkId(sinkId);
|
|
445
|
-
}
|
|
464
|
+
const samples = pcm instanceof Int16Array ? pcm : new Int16Array(pcm);
|
|
465
|
+
this.getDriver().play(samples, sampleRate);
|
|
446
466
|
} catch (error) {
|
|
447
|
-
console.error(
|
|
467
|
+
console.error("[Micdrop] Error playing audio", error);
|
|
448
468
|
}
|
|
449
469
|
}
|
|
470
|
+
/** Drops everything queued and stops playing immediately */
|
|
471
|
+
stopAudio() {
|
|
472
|
+
this.driver?.stopAudio();
|
|
473
|
+
}
|
|
474
|
+
/** Releases the audio graph */
|
|
475
|
+
async stop() {
|
|
476
|
+
await this.driver?.stop();
|
|
477
|
+
this.volume = -Infinity;
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Plays through another output
|
|
481
|
+
* @param deviceId - Output device, or route on a phone
|
|
482
|
+
*/
|
|
483
|
+
async changeDevice(deviceId) {
|
|
484
|
+
await this.getDriver().changeDevice(deviceId);
|
|
485
|
+
storage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
486
|
+
}
|
|
487
|
+
/** Lists the output devices the system offers */
|
|
488
|
+
async getDevices() {
|
|
489
|
+
if (!this.driver) return [];
|
|
490
|
+
return this.driver.getDevices();
|
|
491
|
+
}
|
|
450
492
|
};
|
|
451
493
|
|
|
452
494
|
// src/audio/MicRecorder.ts
|
|
453
|
-
var
|
|
454
|
-
|
|
455
|
-
// src/audio/utils/delayedStream.ts
|
|
456
|
-
function createDelayedStream(stream, delay) {
|
|
457
|
-
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
458
|
-
const delayNode = audioContext.createDelay(delay);
|
|
459
|
-
delayNode.delayTime.value = delay;
|
|
460
|
-
const audioDestination = audioContext.createMediaStreamDestination();
|
|
461
|
-
audioSource.connect(delayNode);
|
|
462
|
-
delayNode.connect(audioDestination);
|
|
463
|
-
return audioDestination.stream;
|
|
464
|
-
}
|
|
495
|
+
var import_eventemitter34 = require("eventemitter3");
|
|
465
496
|
|
|
466
497
|
// src/audio/vad/VAD.ts
|
|
467
|
-
var
|
|
498
|
+
var import_eventemitter33 = require("eventemitter3");
|
|
468
499
|
var VADStatus = /* @__PURE__ */ ((VADStatus2) => {
|
|
469
500
|
VADStatus2[VADStatus2["Silence"] = 0] = "Silence";
|
|
470
501
|
VADStatus2[VADStatus2["MaybeSpeaking"] = 1] = "MaybeSpeaking";
|
|
471
502
|
VADStatus2[VADStatus2["Speaking"] = 2] = "Speaking";
|
|
472
503
|
return VADStatus2;
|
|
473
504
|
})(VADStatus || {});
|
|
474
|
-
var VAD = class extends
|
|
505
|
+
var VAD = class extends import_eventemitter33.EventEmitter {
|
|
475
506
|
constructor() {
|
|
476
507
|
super(...arguments);
|
|
477
508
|
this.name = this.constructor.name;
|
|
478
509
|
this.status = 0 /* Silence */;
|
|
479
|
-
//
|
|
510
|
+
// Worst case time this VAD takes to notice that someone started speaking.
|
|
511
|
+
// That much audio is kept in reserve and sent along with the turn, so the
|
|
512
|
+
// beginning of a sentence reaches the server rather than being cut off.
|
|
513
|
+
// A VAD that needs several samples to make up its mind has to say so here.
|
|
480
514
|
this.delay = 100;
|
|
481
515
|
}
|
|
482
516
|
/**
|
|
@@ -559,13 +593,13 @@ var MultipleVAD = class extends VAD {
|
|
|
559
593
|
get isPaused() {
|
|
560
594
|
return this.vads.every((vad) => vad.isPaused);
|
|
561
595
|
}
|
|
562
|
-
async start(
|
|
596
|
+
async start(mic) {
|
|
563
597
|
for (const vad of this.vads) {
|
|
564
598
|
if (vad.isStarted) {
|
|
565
599
|
continue;
|
|
566
600
|
}
|
|
567
601
|
vad.on("ChangeStatus", this.onStatusChange);
|
|
568
|
-
await vad.start(
|
|
602
|
+
await vad.start(mic);
|
|
569
603
|
}
|
|
570
604
|
}
|
|
571
605
|
async stop() {
|
|
@@ -587,87 +621,96 @@ var MultipleVAD = class extends VAD {
|
|
|
587
621
|
};
|
|
588
622
|
|
|
589
623
|
// src/audio/vad/SileroVAD.ts
|
|
590
|
-
var
|
|
624
|
+
var SILERO_SAMPLE_RATE = 16e3;
|
|
625
|
+
var SILERO_FRAME_SAMPLES = 512;
|
|
626
|
+
var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
|
|
591
627
|
var defaultOptions = {
|
|
592
628
|
positiveSpeechThreshold: 0.18,
|
|
593
629
|
negativeSpeechThreshold: 0.11,
|
|
594
|
-
minSpeechFrames:
|
|
595
|
-
redemptionFrames:
|
|
630
|
+
minSpeechFrames: 4,
|
|
631
|
+
redemptionFrames: 8
|
|
596
632
|
};
|
|
633
|
+
var loadModel;
|
|
634
|
+
function setSileroModelLoader(loader) {
|
|
635
|
+
loadModel = loader;
|
|
636
|
+
}
|
|
597
637
|
var SileroVAD = class extends VAD {
|
|
598
638
|
constructor(options) {
|
|
599
639
|
super();
|
|
600
640
|
this.name = "SileroVAD";
|
|
601
641
|
this.options = defaultOptions;
|
|
642
|
+
// The model decides on the first window it hears speech in, so it is quick.
|
|
643
|
+
// The reserve still covers a few windows, since scoring runs asynchronously
|
|
644
|
+
// and may lag behind the audio on a busy device.
|
|
645
|
+
this.delay = 3 * MS_PER_FRAME;
|
|
602
646
|
this._isPaused = false;
|
|
647
|
+
this.buffer = [];
|
|
648
|
+
this.bufferLength = 0;
|
|
649
|
+
this.bufferRate = SILERO_SAMPLE_RATE;
|
|
650
|
+
this.speaking = false;
|
|
651
|
+
this.speechFrames = 0;
|
|
652
|
+
this.silenceFrames = 0;
|
|
653
|
+
this.processing = false;
|
|
654
|
+
this.onFrames = (frames, sampleRate) => {
|
|
655
|
+
if (this._isPaused || !this.model) return;
|
|
656
|
+
if (sampleRate !== this.bufferRate) {
|
|
657
|
+
this.buffer = [];
|
|
658
|
+
this.bufferLength = 0;
|
|
659
|
+
this.bufferRate = sampleRate;
|
|
660
|
+
}
|
|
661
|
+
this.buffer.push(frames);
|
|
662
|
+
this.bufferLength += frames.length;
|
|
663
|
+
this.drain();
|
|
664
|
+
};
|
|
603
665
|
if (options) {
|
|
604
666
|
this.setOptions(options);
|
|
605
667
|
} else {
|
|
606
|
-
const
|
|
607
|
-
"micdrop.SileroVADOptions" /* SileroVADOptions */
|
|
608
|
-
);
|
|
668
|
+
const saved = storage.getItem("micdrop.SileroVADOptions" /* SileroVADOptions */);
|
|
609
669
|
try {
|
|
610
|
-
this.setOptions(
|
|
611
|
-
} catch
|
|
670
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
671
|
+
} catch {
|
|
612
672
|
this.setOptions({});
|
|
613
673
|
}
|
|
614
674
|
}
|
|
615
675
|
}
|
|
616
676
|
get isStarted() {
|
|
617
|
-
return !!this.
|
|
677
|
+
return !!this.mic;
|
|
618
678
|
}
|
|
619
679
|
get isPaused() {
|
|
620
680
|
return this._isPaused;
|
|
621
681
|
}
|
|
622
|
-
async start(
|
|
623
|
-
if (this.
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
// CDN size limit, so every file 404s). unpkg serves it, so the ONNX wasm
|
|
634
|
-
// backend is loaded from there instead.
|
|
635
|
-
onnxWASMBasePath: "https://unpkg.com/onnxruntime-web@1.23.2/dist/",
|
|
636
|
-
onSpeechStart: () => {
|
|
637
|
-
this.emit("StartSpeaking");
|
|
638
|
-
},
|
|
639
|
-
onSpeechRealStart: () => {
|
|
640
|
-
this.emit("ConfirmSpeaking");
|
|
641
|
-
},
|
|
642
|
-
onVADMisfire: () => {
|
|
643
|
-
this.emit("CancelSpeaking");
|
|
644
|
-
},
|
|
645
|
-
onSpeechEnd: () => {
|
|
646
|
-
this.emit("StopSpeaking");
|
|
647
|
-
},
|
|
648
|
-
...this.options
|
|
649
|
-
});
|
|
650
|
-
this.vad.start();
|
|
682
|
+
async start(mic) {
|
|
683
|
+
if (this.mic) return;
|
|
684
|
+
if (!loadModel) {
|
|
685
|
+
throw new Error(
|
|
686
|
+
"No Silero model. Add `import '@micdrop/web/silero'` in a browser or `import '@micdrop/react-native/silero'` on a phone, or call setSileroModelLoader()."
|
|
687
|
+
);
|
|
688
|
+
}
|
|
689
|
+
this.model = await loadModel();
|
|
690
|
+
this.resetState();
|
|
691
|
+
this.mic = mic;
|
|
692
|
+
mic.on("Frames", this.onFrames);
|
|
651
693
|
}
|
|
652
694
|
async stop() {
|
|
653
|
-
|
|
654
|
-
this.
|
|
655
|
-
this.
|
|
695
|
+
this.mic?.off("Frames", this.onFrames);
|
|
696
|
+
this.mic = void 0;
|
|
697
|
+
this.closeOpenTurn();
|
|
698
|
+
this.resetState();
|
|
699
|
+
await this.model?.release();
|
|
700
|
+
this.model = void 0;
|
|
656
701
|
}
|
|
657
702
|
async pause() {
|
|
658
|
-
if (
|
|
703
|
+
if (this._isPaused) return;
|
|
659
704
|
this._isPaused = true;
|
|
660
|
-
this.
|
|
705
|
+
this.closeOpenTurn();
|
|
706
|
+
this.resetState();
|
|
661
707
|
}
|
|
662
708
|
async resume() {
|
|
663
|
-
if (!this.vad || !this._isPaused) return;
|
|
664
709
|
this._isPaused = false;
|
|
665
|
-
this.vad.start();
|
|
666
710
|
}
|
|
667
711
|
setOptions(options) {
|
|
668
712
|
this.options = { ...this.options, ...options };
|
|
669
|
-
|
|
670
|
-
localStorage.setItem(
|
|
713
|
+
storage.setItem(
|
|
671
714
|
"micdrop.SileroVADOptions" /* SileroVADOptions */,
|
|
672
715
|
JSON.stringify(this.options)
|
|
673
716
|
);
|
|
@@ -675,7 +718,79 @@ var SileroVAD = class extends VAD {
|
|
|
675
718
|
resetOptions() {
|
|
676
719
|
this.setOptions(defaultOptions);
|
|
677
720
|
}
|
|
721
|
+
// The model is asynchronous, so windows are scored one after the other rather
|
|
722
|
+
// than raced: its answer for a window depends on the ones before it.
|
|
723
|
+
async drain() {
|
|
724
|
+
if (this.processing) return;
|
|
725
|
+
this.processing = true;
|
|
726
|
+
try {
|
|
727
|
+
const needed = Math.ceil(
|
|
728
|
+
SILERO_FRAME_SAMPLES * this.bufferRate / SILERO_SAMPLE_RATE
|
|
729
|
+
);
|
|
730
|
+
while (this.bufferLength >= needed && this.model && !this._isPaused) {
|
|
731
|
+
const gathered = concatFloat32(this.buffer);
|
|
732
|
+
const window = gathered.subarray(0, needed);
|
|
733
|
+
const rest = gathered.subarray(needed);
|
|
734
|
+
this.buffer = rest.length ? [rest] : [];
|
|
735
|
+
this.bufferLength = rest.length;
|
|
736
|
+
const frame = resample(window, this.bufferRate, SILERO_SAMPLE_RATE);
|
|
737
|
+
const probability = await this.model.process(
|
|
738
|
+
frame.length === SILERO_FRAME_SAMPLES ? frame : fit(frame, SILERO_FRAME_SAMPLES)
|
|
739
|
+
);
|
|
740
|
+
this.onProbability(probability);
|
|
741
|
+
}
|
|
742
|
+
} catch (error) {
|
|
743
|
+
console.error("[Micdrop] Silero VAD failed", error);
|
|
744
|
+
} finally {
|
|
745
|
+
this.processing = false;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
// Ported from the reference implementation of the algorithm
|
|
749
|
+
// https://docs.vad.ricky0123.com/user-guide/algorithm/
|
|
750
|
+
onProbability(probability) {
|
|
751
|
+
const { positiveSpeechThreshold, negativeSpeechThreshold } = this.options;
|
|
752
|
+
if (probability >= positiveSpeechThreshold) {
|
|
753
|
+
this.silenceFrames = 0;
|
|
754
|
+
this.speechFrames++;
|
|
755
|
+
if (!this.speaking) {
|
|
756
|
+
this.speaking = true;
|
|
757
|
+
this.emit("StartSpeaking");
|
|
758
|
+
}
|
|
759
|
+
if (this.speechFrames === this.options.minSpeechFrames) {
|
|
760
|
+
this.emit("ConfirmSpeaking");
|
|
761
|
+
}
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
if (probability < negativeSpeechThreshold && this.speaking) {
|
|
765
|
+
this.silenceFrames++;
|
|
766
|
+
if (this.silenceFrames < this.options.redemptionFrames) return;
|
|
767
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
768
|
+
this.speaking = false;
|
|
769
|
+
this.speechFrames = 0;
|
|
770
|
+
this.silenceFrames = 0;
|
|
771
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
closeOpenTurn() {
|
|
775
|
+
if (!this.speaking) return;
|
|
776
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
777
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
778
|
+
}
|
|
779
|
+
resetState() {
|
|
780
|
+
this.model?.reset();
|
|
781
|
+
this.buffer = [];
|
|
782
|
+
this.bufferLength = 0;
|
|
783
|
+
this.speaking = false;
|
|
784
|
+
this.speechFrames = 0;
|
|
785
|
+
this.silenceFrames = 0;
|
|
786
|
+
}
|
|
678
787
|
};
|
|
788
|
+
function fit(frame, length) {
|
|
789
|
+
if (frame.length === length) return frame;
|
|
790
|
+
const fitted = new Float32Array(length);
|
|
791
|
+
fitted.set(frame.subarray(0, length));
|
|
792
|
+
return fitted;
|
|
793
|
+
}
|
|
679
794
|
|
|
680
795
|
// src/audio/vad/VolumeVAD.ts
|
|
681
796
|
var defaultOptions2 = {
|
|
@@ -687,138 +802,92 @@ var VolumeVAD = class extends VAD {
|
|
|
687
802
|
super();
|
|
688
803
|
this.name = "VolumeVAD";
|
|
689
804
|
this.options = defaultOptions2;
|
|
690
|
-
|
|
805
|
+
// The microphone reports a level about every 100 ms, and two consecutive loud
|
|
806
|
+
// ones are needed before a turn opens. Speech can therefore begin up to two
|
|
807
|
+
// reports before it is noticed, and a third is kept as a margin: without that
|
|
808
|
+
// reserve the first syllable of a sentence never reaches the server.
|
|
809
|
+
this.delay = 300;
|
|
691
810
|
this.running = false;
|
|
811
|
+
this._isPaused = false;
|
|
692
812
|
this.speaking = false;
|
|
693
813
|
this.attemptSpeaking = false;
|
|
694
814
|
this.speakingHistory = [];
|
|
815
|
+
this.onVolume = (volume) => {
|
|
816
|
+
if (!this.running) return;
|
|
817
|
+
const isLoud = volume > this.options.threshold;
|
|
818
|
+
if (isLoud) {
|
|
819
|
+
if (!this.speaking) {
|
|
820
|
+
let recentHistory = 0;
|
|
821
|
+
for (let i = Math.max(0, this.speakingHistory.length - 3); i < this.speakingHistory.length; i++) {
|
|
822
|
+
recentHistory += this.speakingHistory[i];
|
|
823
|
+
}
|
|
824
|
+
if (recentHistory >= 2) {
|
|
825
|
+
this.speaking = true;
|
|
826
|
+
this.attemptSpeaking = false;
|
|
827
|
+
this.emit("ConfirmSpeaking");
|
|
828
|
+
} else if (recentHistory === 1 && !this.attemptSpeaking) {
|
|
829
|
+
this.attemptSpeaking = true;
|
|
830
|
+
this.emit("StartSpeaking");
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
} else if (this.speaking) {
|
|
834
|
+
let totalHistory = 0;
|
|
835
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
836
|
+
totalHistory += this.speakingHistory[i];
|
|
837
|
+
}
|
|
838
|
+
if (totalHistory === 0) {
|
|
839
|
+
this.speaking = false;
|
|
840
|
+
this.emit("StopSpeaking");
|
|
841
|
+
}
|
|
842
|
+
} else if (this.attemptSpeaking) {
|
|
843
|
+
const lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
844
|
+
if (lastHistory === 0) {
|
|
845
|
+
this.attemptSpeaking = false;
|
|
846
|
+
this.emit("CancelSpeaking");
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
this.speakingHistory.shift();
|
|
850
|
+
this.speakingHistory.push(Number(isLoud));
|
|
851
|
+
};
|
|
695
852
|
if (options) {
|
|
696
853
|
this.setOptions(options);
|
|
697
854
|
} else {
|
|
698
|
-
const
|
|
699
|
-
"micdrop.VolumeVADOptions" /* VolumeVADOptions */
|
|
700
|
-
);
|
|
855
|
+
const saved = storage.getItem("micdrop.VolumeVADOptions" /* VolumeVADOptions */);
|
|
701
856
|
try {
|
|
702
|
-
this.setOptions(
|
|
703
|
-
} catch
|
|
857
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
858
|
+
} catch {
|
|
704
859
|
this.setOptions({});
|
|
705
860
|
}
|
|
706
861
|
}
|
|
707
862
|
}
|
|
708
|
-
getMaxVolume() {
|
|
709
|
-
if (!this.analyser || !this.fftBins) return -Infinity;
|
|
710
|
-
let maxVolume = -Infinity;
|
|
711
|
-
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
712
|
-
for (let i = 4; i < this.fftBins.length; i++) {
|
|
713
|
-
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
714
|
-
maxVolume = this.fftBins[i];
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
return maxVolume;
|
|
718
|
-
}
|
|
719
863
|
get isStarted() {
|
|
720
864
|
return this.running;
|
|
721
865
|
}
|
|
722
866
|
get isPaused() {
|
|
723
867
|
return this._isPaused;
|
|
724
868
|
}
|
|
725
|
-
async start(
|
|
726
|
-
if (this.
|
|
727
|
-
this.
|
|
728
|
-
|
|
729
|
-
this.
|
|
730
|
-
this.analyser.smoothingTimeConstant = 0.1;
|
|
731
|
-
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
732
|
-
this.sourceNode.connect(this.analyser);
|
|
733
|
-
this.running = true;
|
|
734
|
-
this.startLoop();
|
|
735
|
-
}
|
|
736
|
-
startLoop() {
|
|
737
|
-
if (!this.running) return;
|
|
738
|
-
const currentVolume = this.getMaxVolume();
|
|
739
|
-
if (currentVolume > this.options.threshold) {
|
|
740
|
-
if (!this.speaking) {
|
|
741
|
-
let recentHistory = 0;
|
|
742
|
-
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
743
|
-
recentHistory += this.speakingHistory[i];
|
|
744
|
-
}
|
|
745
|
-
if (recentHistory >= 2) {
|
|
746
|
-
this.speaking = true;
|
|
747
|
-
this.attemptSpeaking = false;
|
|
748
|
-
this.emit("ConfirmSpeaking");
|
|
749
|
-
} else if (recentHistory === 1) {
|
|
750
|
-
this.attemptSpeaking = true;
|
|
751
|
-
this.emit("StartSpeaking");
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
} else if (this.speaking) {
|
|
755
|
-
let totalHistory = 0;
|
|
756
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
757
|
-
totalHistory += this.speakingHistory[i];
|
|
758
|
-
}
|
|
759
|
-
if (totalHistory === 0) {
|
|
760
|
-
this.speaking = false;
|
|
761
|
-
this.emit("StopSpeaking");
|
|
762
|
-
}
|
|
763
|
-
} else if (this.attemptSpeaking) {
|
|
764
|
-
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
765
|
-
if (lastHistory === 0) {
|
|
766
|
-
this.attemptSpeaking = false;
|
|
767
|
-
this.emit("CancelSpeaking");
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
this.speakingHistory.shift();
|
|
771
|
-
this.speakingHistory.push(Number(currentVolume > this.options.threshold));
|
|
772
|
-
setTimeout(() => this.startLoop(), this.delay);
|
|
773
|
-
}
|
|
774
|
-
resetHistory() {
|
|
775
|
-
this.speaking = false;
|
|
776
|
-
this.attemptSpeaking = false;
|
|
777
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
778
|
-
this.speakingHistory[i] = 0;
|
|
779
|
-
}
|
|
869
|
+
async start(mic) {
|
|
870
|
+
if (this.mic) return;
|
|
871
|
+
this.mic = mic;
|
|
872
|
+
mic.on("Volume", this.onVolume);
|
|
873
|
+
this.running = !this._isPaused;
|
|
780
874
|
}
|
|
781
875
|
async stop() {
|
|
782
|
-
|
|
876
|
+
this.mic?.off("Volume", this.onVolume);
|
|
877
|
+
this.mic = void 0;
|
|
783
878
|
this.running = false;
|
|
784
|
-
this.
|
|
785
|
-
switch (this.status) {
|
|
786
|
-
case 2 /* Speaking */:
|
|
787
|
-
this.emit("StopSpeaking");
|
|
788
|
-
break;
|
|
789
|
-
case 1 /* MaybeSpeaking */:
|
|
790
|
-
this.emit("CancelSpeaking");
|
|
791
|
-
break;
|
|
792
|
-
}
|
|
793
|
-
if (this.sourceNode) {
|
|
794
|
-
this.sourceNode.disconnect();
|
|
795
|
-
this.sourceNode = void 0;
|
|
796
|
-
}
|
|
797
|
-
if (this.analyser) {
|
|
798
|
-
this.analyser.disconnect();
|
|
799
|
-
this.analyser = void 0;
|
|
800
|
-
}
|
|
801
|
-
this.fftBins = void 0;
|
|
879
|
+
this.reset();
|
|
802
880
|
}
|
|
803
881
|
async pause() {
|
|
804
|
-
if (
|
|
882
|
+
if (this._isPaused) return;
|
|
805
883
|
this._isPaused = true;
|
|
806
884
|
this.running = false;
|
|
807
|
-
this.
|
|
808
|
-
switch (this.status) {
|
|
809
|
-
case 2 /* Speaking */:
|
|
810
|
-
this.emit("StopSpeaking");
|
|
811
|
-
break;
|
|
812
|
-
case 1 /* MaybeSpeaking */:
|
|
813
|
-
this.emit("CancelSpeaking");
|
|
814
|
-
break;
|
|
815
|
-
}
|
|
885
|
+
this.reset();
|
|
816
886
|
}
|
|
817
887
|
async resume() {
|
|
818
888
|
if (!this._isPaused) return;
|
|
819
889
|
this._isPaused = false;
|
|
820
|
-
this.running =
|
|
821
|
-
this.startLoop();
|
|
890
|
+
this.running = !!this.mic;
|
|
822
891
|
}
|
|
823
892
|
setOptions(options) {
|
|
824
893
|
if (typeof options.history === "number" && options.history < 3) {
|
|
@@ -831,7 +900,7 @@ var VolumeVAD = class extends VAD {
|
|
|
831
900
|
while (this.speakingHistory.length < this.options.history) {
|
|
832
901
|
this.speakingHistory.push(0);
|
|
833
902
|
}
|
|
834
|
-
|
|
903
|
+
storage.setItem(
|
|
835
904
|
"micdrop.VolumeVADOptions" /* VolumeVADOptions */,
|
|
836
905
|
JSON.stringify(this.options)
|
|
837
906
|
);
|
|
@@ -839,6 +908,22 @@ var VolumeVAD = class extends VAD {
|
|
|
839
908
|
resetOptions() {
|
|
840
909
|
this.setOptions(defaultOptions2);
|
|
841
910
|
}
|
|
911
|
+
reset() {
|
|
912
|
+
const status = this.status;
|
|
913
|
+
this.speaking = false;
|
|
914
|
+
this.attemptSpeaking = false;
|
|
915
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
916
|
+
this.speakingHistory[i] = 0;
|
|
917
|
+
}
|
|
918
|
+
switch (status) {
|
|
919
|
+
case 2 /* Speaking */:
|
|
920
|
+
this.emit("StopSpeaking");
|
|
921
|
+
break;
|
|
922
|
+
case 1 /* MaybeSpeaking */:
|
|
923
|
+
this.emit("CancelSpeaking");
|
|
924
|
+
break;
|
|
925
|
+
}
|
|
926
|
+
}
|
|
842
927
|
};
|
|
843
928
|
|
|
844
929
|
// src/audio/vad/getVAD.ts
|
|
@@ -860,34 +945,36 @@ function equalVADConfig(a, b) {
|
|
|
860
945
|
return false;
|
|
861
946
|
}
|
|
862
947
|
|
|
863
|
-
// raw:./pcm-processor-worklet
|
|
864
|
-
var pcm_processor_worklet_default = "const INTERVAL_MS = 100\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.audioBuffer = []\n this.isRecording = false\n this.sourceSampleRate = 16000\n this.targetSampleRate = 16000\n this.resamplingRatio = 1\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n\n // Listen for messages\n this.port.onmessage = this.handleMessage.bind(this)\n }\n\n handleMessage(event) {\n if (event.data.type === 'configure') {\n this.sourceSampleRate = event.data.sourceSampleRate\n this.targetSampleRate = event.data.targetSampleRate\n this.resamplingRatio = this.sourceSampleRate / this.targetSampleRate\n // Recalculate chunk interval for the source sample rate\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n } else if (event.data.type === 'start') {\n this.isRecording = true\n } else if (event.data.type === 'stop') {\n this.isRecording = false\n // Clear any remaining audio buffer when stopping\n this.audioBuffer.length = 0\n }\n }\n\n process(inputs) {\n const input = inputs[0]\n\n if (input && input.length > 0 && this.isRecording) {\n const inputChannel = input[0]\n\n // Store audio data in buffer\n this.audioBuffer.push(new Float32Array(inputChannel))\n\n // Check if we have enough data for a chunk\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n\n if (totalSamples >= this.CHUNK_INTERVAL_SAMPLES) {\n this.processChunk()\n }\n }\n\n return true\n }\n\n processChunk() {\n if (this.audioBuffer.length === 0) return\n\n // Calculate total samples\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n if (totalSamples === 0) return\n\n // Merge all buffers\n const mergedBuffer = new Float32Array(totalSamples)\n let offset = 0\n for (const buffer of this.audioBuffer) {\n mergedBuffer.set(buffer, offset)\n offset += buffer.length\n }\n\n // Resample to target sample rate if necessary\n const resampledBuffer = this.resample(mergedBuffer)\n\n // Convert float32 to int16 PCM\n const pcmData = this.floatToPCM16(resampledBuffer)\n\n // Clear the buffer\n this.audioBuffer.length = 0\n\n // Send PCM chunk to main thread\n this.port.postMessage({\n type: 'chunk',\n data: pcmData,\n })\n }\n\n resample(inputBuffer) {\n // If no resampling needed, return original buffer\n if (this.resamplingRatio === 1) {\n return inputBuffer\n }\n\n // Calculate output length\n const outputLength = Math.round(inputBuffer.length / this.resamplingRatio)\n const outputBuffer = new Float32Array(outputLength)\n\n // Simple linear interpolation resampling\n for (let i = 0; i < outputLength; i++) {\n const srcIndex = i * this.resamplingRatio\n const srcIndexFloor = Math.floor(srcIndex)\n const srcIndexCeil = Math.min(srcIndexFloor + 1, inputBuffer.length - 1)\n const fraction = srcIndex - srcIndexFloor\n\n // Linear interpolation\n outputBuffer[i] =\n inputBuffer[srcIndexFloor] * (1 - fraction) +\n inputBuffer[srcIndexCeil] * fraction\n }\n\n return outputBuffer\n }\n\n floatToPCM16(floatData) {\n const pcmData = new Int16Array(floatData.length)\n for (let i = 0; i < floatData.length; i++) {\n const sample = Math.max(-1, Math.min(1, floatData[i]))\n pcmData[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff\n }\n return pcmData\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
|
|
865
|
-
|
|
866
|
-
// src/audio/pcm-processor.ts
|
|
867
|
-
async function initPcmProcessor() {
|
|
868
|
-
if (window.micdropPcmProcessor) return;
|
|
869
|
-
window.micdropPcmProcessor = true;
|
|
870
|
-
const workletBlob = new Blob([pcm_processor_worklet_default], {
|
|
871
|
-
type: "application/javascript"
|
|
872
|
-
});
|
|
873
|
-
const workletUrl = URL.createObjectURL(workletBlob);
|
|
874
|
-
await audioContext.audioWorklet.addModule(workletUrl);
|
|
875
|
-
}
|
|
876
|
-
|
|
877
948
|
// src/audio/MicRecorder.ts
|
|
878
|
-
var
|
|
949
|
+
var SAMPLE_RATE = 16e3;
|
|
950
|
+
var CHUNK_INTERVAL = 100;
|
|
951
|
+
var DEFAULT_TURN_MAX_WAIT = 4e3;
|
|
879
952
|
var defaultMicRecorderState = {
|
|
880
953
|
isStarting: false,
|
|
881
954
|
isStarted: false,
|
|
882
955
|
isSpeaking: false
|
|
883
956
|
};
|
|
884
|
-
var MicRecorder = class extends
|
|
885
|
-
constructor(vadConfig) {
|
|
957
|
+
var MicRecorder = class extends import_eventemitter34.EventEmitter {
|
|
958
|
+
constructor(vadConfig, turnDetector, turnMaxWait = DEFAULT_TURN_MAX_WAIT) {
|
|
886
959
|
super();
|
|
887
960
|
this.vadConfig = vadConfig;
|
|
961
|
+
this.turnDetector = turnDetector;
|
|
962
|
+
this.turnMaxWait = turnMaxWait;
|
|
963
|
+
this.isRecording = false;
|
|
888
964
|
this.speakingConfirmed = false;
|
|
965
|
+
this.sourceSampleRate = SAMPLE_RATE;
|
|
966
|
+
this.reserve = [];
|
|
967
|
+
this.reserveLength = 0;
|
|
968
|
+
this.buffer = [];
|
|
969
|
+
this.bufferLength = 0;
|
|
889
970
|
this.queuedChunks = [];
|
|
890
|
-
|
|
971
|
+
// The turn is open between the first confirmed word and the moment the
|
|
972
|
+
// server is told to answer, which can span several pauses
|
|
973
|
+
this.turnOpen = false;
|
|
974
|
+
this.turnListening = false;
|
|
975
|
+
// Bumped whenever the turn moves on, so a late answer is dropped
|
|
976
|
+
this.turnRound = 0;
|
|
977
|
+
this.start = async (mic) => {
|
|
891
978
|
if (this.state.isStarted) {
|
|
892
979
|
throw new Error("MicRecorder is already started");
|
|
893
980
|
}
|
|
@@ -896,21 +983,10 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
896
983
|
isStarting: true,
|
|
897
984
|
isSpeaking: false
|
|
898
985
|
});
|
|
899
|
-
this.
|
|
900
|
-
this.
|
|
901
|
-
|
|
902
|
-
this.
|
|
903
|
-
this.workletNode.port.onmessage = this.onWorkletMessage;
|
|
904
|
-
this.workletNode.port.postMessage({
|
|
905
|
-
type: "configure",
|
|
906
|
-
sourceSampleRate: audioContext.sampleRate,
|
|
907
|
-
targetSampleRate: SAMPLE_RATE2
|
|
908
|
-
});
|
|
909
|
-
const delayedStream = createDelayedStream(stream, this.vad.delay / 1e3);
|
|
910
|
-
this.delayedStream = delayedStream;
|
|
911
|
-
this.source = audioContext.createMediaStreamSource(delayedStream);
|
|
912
|
-
this.source.connect(this.workletNode);
|
|
913
|
-
await this.vad.start(stream);
|
|
986
|
+
this.mic = mic;
|
|
987
|
+
this.resetBuffers();
|
|
988
|
+
mic.on("Frames", this.onFrames);
|
|
989
|
+
await this.vad.start(mic);
|
|
914
990
|
this.vad.on("StartSpeaking", this.onStartSpeaking);
|
|
915
991
|
this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
916
992
|
this.vad.on("CancelSpeaking", this.onCancelSpeaking);
|
|
@@ -930,79 +1006,213 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
930
1006
|
isStarted: false,
|
|
931
1007
|
isSpeaking: false
|
|
932
1008
|
});
|
|
933
|
-
this.
|
|
1009
|
+
this.cancelPendingClose();
|
|
934
1010
|
try {
|
|
1011
|
+
this.mic?.off("Frames", this.onFrames);
|
|
1012
|
+
this.mic = void 0;
|
|
935
1013
|
this.vad.stop();
|
|
936
1014
|
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
937
1015
|
this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
938
1016
|
this.vad.off("CancelSpeaking", this.onCancelSpeaking);
|
|
939
1017
|
this.vad.off("StopSpeaking", this.onStopSpeaking);
|
|
940
|
-
if (this.workletNode) {
|
|
941
|
-
this.workletNode.disconnect();
|
|
942
|
-
this.workletNode = void 0;
|
|
943
|
-
}
|
|
944
|
-
if (this.source) {
|
|
945
|
-
this.source.disconnect();
|
|
946
|
-
this.source = void 0;
|
|
947
|
-
}
|
|
948
|
-
if (this.delayedStream) {
|
|
949
|
-
stopStream(this.delayedStream);
|
|
950
|
-
this.delayedStream = void 0;
|
|
951
|
-
}
|
|
952
1018
|
} catch (err) {
|
|
953
1019
|
console.error(err);
|
|
954
1020
|
}
|
|
1021
|
+
this.resetBuffers();
|
|
1022
|
+
};
|
|
1023
|
+
/**
|
|
1024
|
+
* Changes what decides when a turn is over
|
|
1025
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
1026
|
+
*/
|
|
1027
|
+
this.changeTurnDetector = (turnDetector) => {
|
|
1028
|
+
if (turnDetector === this.turnDetector) return;
|
|
1029
|
+
if (this.turnOpen) {
|
|
1030
|
+
this.endTurn();
|
|
1031
|
+
}
|
|
1032
|
+
this.turnDetector = turnDetector;
|
|
955
1033
|
};
|
|
956
1034
|
this.changeVad = (vadConfig) => {
|
|
957
1035
|
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
958
|
-
const
|
|
959
|
-
if (
|
|
1036
|
+
const mic = this.mic;
|
|
1037
|
+
if (mic) {
|
|
960
1038
|
this.stop();
|
|
961
1039
|
}
|
|
962
1040
|
this.vadConfig = vadConfig;
|
|
963
1041
|
this.vad = getVAD(vadConfig);
|
|
964
|
-
if (
|
|
965
|
-
this.start(
|
|
1042
|
+
if (mic) {
|
|
1043
|
+
this.start(mic);
|
|
966
1044
|
}
|
|
967
1045
|
};
|
|
968
|
-
this.
|
|
1046
|
+
this.onFrames = (frames, sampleRate) => {
|
|
1047
|
+
if (this.turnListening) {
|
|
1048
|
+
this.turnDetector?.push(frames, sampleRate);
|
|
1049
|
+
}
|
|
1050
|
+
if (sampleRate !== this.sourceSampleRate) {
|
|
1051
|
+
this.flushChunks();
|
|
1052
|
+
this.sourceSampleRate = sampleRate;
|
|
1053
|
+
this.reserve = [];
|
|
1054
|
+
this.reserveLength = 0;
|
|
1055
|
+
}
|
|
1056
|
+
if (this.isRecording) {
|
|
1057
|
+
this.buffer.push(frames);
|
|
1058
|
+
this.bufferLength += frames.length;
|
|
1059
|
+
this.flushChunks();
|
|
1060
|
+
return;
|
|
1061
|
+
}
|
|
1062
|
+
this.reserve.push(frames);
|
|
1063
|
+
this.reserveLength += frames.length;
|
|
1064
|
+
const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
|
|
1065
|
+
while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
|
|
1066
|
+
this.reserveLength -= this.reserve[0].length;
|
|
1067
|
+
this.reserve.shift();
|
|
1068
|
+
}
|
|
1069
|
+
};
|
|
1070
|
+
this.onStartSpeaking = () => {
|
|
969
1071
|
this.speakingConfirmed = false;
|
|
970
1072
|
this.queuedChunks.length = 0;
|
|
971
|
-
this.
|
|
1073
|
+
this.isRecording = true;
|
|
1074
|
+
this.buffer = this.reserve;
|
|
1075
|
+
this.bufferLength = this.reserveLength;
|
|
1076
|
+
this.reserve = [];
|
|
1077
|
+
this.reserveLength = 0;
|
|
1078
|
+
if (this.turnDetector && !this.turnListening) {
|
|
1079
|
+
this.turnDetector.reset();
|
|
1080
|
+
this.turnListening = true;
|
|
1081
|
+
for (const frames of this.buffer) {
|
|
1082
|
+
this.turnDetector.push(frames, this.sourceSampleRate);
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
this.flushChunks();
|
|
972
1086
|
};
|
|
973
|
-
this.onConfirmSpeaking =
|
|
1087
|
+
this.onConfirmSpeaking = () => {
|
|
974
1088
|
this.speakingConfirmed = true;
|
|
975
1089
|
this.changeState({ isSpeaking: true });
|
|
976
|
-
this.
|
|
977
|
-
|
|
978
|
-
|
|
1090
|
+
this.cancelPendingClose();
|
|
1091
|
+
if (!this.turnOpen) {
|
|
1092
|
+
this.turnOpen = true;
|
|
1093
|
+
this.emit("StartSpeaking");
|
|
1094
|
+
}
|
|
1095
|
+
for (const chunk of this.queuedChunks) {
|
|
1096
|
+
this.emit("Chunk", chunk);
|
|
1097
|
+
}
|
|
979
1098
|
this.queuedChunks.length = 0;
|
|
980
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
981
1099
|
};
|
|
982
|
-
this.
|
|
983
|
-
this.
|
|
984
|
-
this.
|
|
985
|
-
|
|
986
|
-
|
|
1100
|
+
this.onCancelSpeaking = () => {
|
|
1101
|
+
this.stopRecording();
|
|
1102
|
+
if (!this.turnOpen) {
|
|
1103
|
+
this.turnListening = false;
|
|
1104
|
+
}
|
|
987
1105
|
};
|
|
988
|
-
this.
|
|
989
|
-
const
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
this.queuedChunks.push(pcmData);
|
|
994
|
-
return;
|
|
995
|
-
}
|
|
996
|
-
for (const chunk of this.queuedChunks) {
|
|
997
|
-
this.emit("Chunk", chunk);
|
|
998
|
-
}
|
|
999
|
-
this.queuedChunks.length = 0;
|
|
1000
|
-
this.emit("Chunk", pcmData);
|
|
1106
|
+
this.onStopSpeaking = () => {
|
|
1107
|
+
const wasConfirmed = this.speakingConfirmed;
|
|
1108
|
+
this.changeState({ isSpeaking: false });
|
|
1109
|
+
if (wasConfirmed) {
|
|
1110
|
+
this.flushChunks(true);
|
|
1001
1111
|
}
|
|
1112
|
+
this.stopRecording();
|
|
1113
|
+
this.closeTurn();
|
|
1002
1114
|
};
|
|
1003
1115
|
this.state = defaultMicRecorderState;
|
|
1004
1116
|
this.vad = getVAD(vadConfig);
|
|
1005
1117
|
}
|
|
1118
|
+
/**
|
|
1119
|
+
* Ends the turn, unless the detector hears an unfinished sentence.
|
|
1120
|
+
*
|
|
1121
|
+
* A microphone that is no longer listening ends it straight away: a muted or
|
|
1122
|
+
* paused call has nothing more to say, whatever the sentence sounded like.
|
|
1123
|
+
*/
|
|
1124
|
+
async closeTurn() {
|
|
1125
|
+
if (!this.turnDetector || !this.turnOpen || !this.vad.isStarted || this.vad.isPaused) {
|
|
1126
|
+
this.endTurn();
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
const round = this.turnRound;
|
|
1130
|
+
try {
|
|
1131
|
+
const { complete } = await this.turnDetector.predict();
|
|
1132
|
+
if (round !== this.turnRound) return;
|
|
1133
|
+
if (complete) {
|
|
1134
|
+
this.endTurn();
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
} catch (error) {
|
|
1138
|
+
console.error("[Micdrop] Turn detection failed", error);
|
|
1139
|
+
this.endTurn();
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
1142
|
+
this.cancelPendingClose();
|
|
1143
|
+
this.turnTimer = setTimeout(() => {
|
|
1144
|
+
this.turnTimer = void 0;
|
|
1145
|
+
this.endTurn();
|
|
1146
|
+
}, this.turnMaxWait);
|
|
1147
|
+
}
|
|
1148
|
+
endTurn() {
|
|
1149
|
+
this.cancelPendingClose();
|
|
1150
|
+
this.turnOpen = false;
|
|
1151
|
+
this.turnListening = false;
|
|
1152
|
+
this.emit("StopSpeaking");
|
|
1153
|
+
}
|
|
1154
|
+
/** Drops the pending close, both the timer and the answer still owed */
|
|
1155
|
+
cancelPendingClose() {
|
|
1156
|
+
this.turnRound++;
|
|
1157
|
+
if (this.turnTimer) {
|
|
1158
|
+
clearTimeout(this.turnTimer);
|
|
1159
|
+
this.turnTimer = void 0;
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
stopRecording() {
|
|
1163
|
+
this.isRecording = false;
|
|
1164
|
+
this.speakingConfirmed = false;
|
|
1165
|
+
this.queuedChunks.length = 0;
|
|
1166
|
+
this.buffer = [];
|
|
1167
|
+
this.bufferLength = 0;
|
|
1168
|
+
}
|
|
1169
|
+
/**
|
|
1170
|
+
* Emits every complete chunk waiting in the buffer
|
|
1171
|
+
* @param final - Also emit the incomplete last chunk
|
|
1172
|
+
*/
|
|
1173
|
+
flushChunks(final = false) {
|
|
1174
|
+
const chunkLength = Math.round(
|
|
1175
|
+
CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
|
|
1176
|
+
);
|
|
1177
|
+
if (this.bufferLength === 0) return;
|
|
1178
|
+
if (this.bufferLength < chunkLength && !final) return;
|
|
1179
|
+
const merged = concatFloat32(this.buffer);
|
|
1180
|
+
let offset = 0;
|
|
1181
|
+
while (merged.length - offset >= chunkLength) {
|
|
1182
|
+
this.emitChunk(merged.subarray(offset, offset + chunkLength));
|
|
1183
|
+
offset += chunkLength;
|
|
1184
|
+
}
|
|
1185
|
+
const rest = merged.subarray(offset);
|
|
1186
|
+
if (final && rest.length > 0) {
|
|
1187
|
+
this.emitChunk(rest);
|
|
1188
|
+
this.buffer = [];
|
|
1189
|
+
this.bufferLength = 0;
|
|
1190
|
+
return;
|
|
1191
|
+
}
|
|
1192
|
+
this.buffer = rest.length ? [rest] : [];
|
|
1193
|
+
this.bufferLength = rest.length;
|
|
1194
|
+
}
|
|
1195
|
+
emitChunk(samples) {
|
|
1196
|
+
const pcm = floatToPcm16(
|
|
1197
|
+
resample(samples, this.sourceSampleRate, SAMPLE_RATE)
|
|
1198
|
+
);
|
|
1199
|
+
if (!this.speakingConfirmed) {
|
|
1200
|
+
this.queuedChunks.push(pcm);
|
|
1201
|
+
return;
|
|
1202
|
+
}
|
|
1203
|
+
this.emit("Chunk", pcm);
|
|
1204
|
+
}
|
|
1205
|
+
resetBuffers() {
|
|
1206
|
+
this.isRecording = false;
|
|
1207
|
+
this.speakingConfirmed = false;
|
|
1208
|
+
this.turnOpen = false;
|
|
1209
|
+
this.turnListening = false;
|
|
1210
|
+
this.reserve = [];
|
|
1211
|
+
this.reserveLength = 0;
|
|
1212
|
+
this.buffer = [];
|
|
1213
|
+
this.bufferLength = 0;
|
|
1214
|
+
this.queuedChunks.length = 0;
|
|
1215
|
+
}
|
|
1006
1216
|
changeState(state) {
|
|
1007
1217
|
const hasChanged = Object.keys(state).some(
|
|
1008
1218
|
(key) => this.state[key] !== state[key]
|
|
@@ -1013,18 +1223,182 @@ var MicRecorder = class extends import_eventemitter35.EventEmitter {
|
|
|
1013
1223
|
}
|
|
1014
1224
|
};
|
|
1015
1225
|
|
|
1226
|
+
// src/audio/Pcm16AudioStream.ts
|
|
1227
|
+
var import_eventemitter35 = require("eventemitter3");
|
|
1228
|
+
var PREBUFFER_DURATION = 100;
|
|
1229
|
+
var QUIET_FLUSH_DELAY = 600;
|
|
1230
|
+
var UTTERANCE_END_DELAY = 300;
|
|
1231
|
+
var VOLUME_INTERVAL2 = 100;
|
|
1232
|
+
var Pcm16AudioStream = class extends import_eventemitter35.EventEmitter {
|
|
1233
|
+
constructor(sink) {
|
|
1234
|
+
super();
|
|
1235
|
+
this.sink = sink;
|
|
1236
|
+
this.isPlaying = false;
|
|
1237
|
+
this.sources = [];
|
|
1238
|
+
this.nextStartTime = 0;
|
|
1239
|
+
this.prebuffering = true;
|
|
1240
|
+
this.prebuffer = [];
|
|
1241
|
+
this.prebufferDuration = 0;
|
|
1242
|
+
this.volumeTimers = [];
|
|
1243
|
+
this.meter = new VolumeMeter();
|
|
1244
|
+
}
|
|
1245
|
+
/**
|
|
1246
|
+
* Queues samples for playback
|
|
1247
|
+
* @param pcm - Mono samples as 16 bits integers
|
|
1248
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
1249
|
+
*/
|
|
1250
|
+
playAudio(pcm, sampleRate) {
|
|
1251
|
+
if (pcm.length === 0) return;
|
|
1252
|
+
const samples = resample(
|
|
1253
|
+
pcm16ToFloat(pcm),
|
|
1254
|
+
sampleRate,
|
|
1255
|
+
this.sink.sampleRate
|
|
1256
|
+
);
|
|
1257
|
+
this.handleSamples(samples);
|
|
1258
|
+
}
|
|
1259
|
+
/** Drops everything queued and stops playing immediately */
|
|
1260
|
+
stopAudio() {
|
|
1261
|
+
for (const source of this.sources) {
|
|
1262
|
+
try {
|
|
1263
|
+
source.stop();
|
|
1264
|
+
} catch (error) {
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
this.sources = [];
|
|
1268
|
+
this.nextStartTime = 0;
|
|
1269
|
+
this.resetPrebuffer();
|
|
1270
|
+
this.setIsPlaying(false);
|
|
1271
|
+
this.clearVolumeTimers();
|
|
1272
|
+
}
|
|
1273
|
+
// Accumulate audio until enough is buffered, then start playing. Once
|
|
1274
|
+
// playing, schedule incoming samples directly.
|
|
1275
|
+
handleSamples(samples) {
|
|
1276
|
+
if (this.endTimer !== void 0) {
|
|
1277
|
+
clearTimeout(this.endTimer);
|
|
1278
|
+
this.endTimer = void 0;
|
|
1279
|
+
}
|
|
1280
|
+
if (!this.prebuffering) {
|
|
1281
|
+
this.scheduleSamples(samples);
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
this.prebuffer.push(samples);
|
|
1285
|
+
this.prebufferDuration += samples.length / this.sink.sampleRate * 1e3;
|
|
1286
|
+
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
1287
|
+
this.flushPrebuffer();
|
|
1288
|
+
return;
|
|
1289
|
+
}
|
|
1290
|
+
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
1291
|
+
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
1292
|
+
}
|
|
1293
|
+
flushPrebuffer() {
|
|
1294
|
+
if (this.quietTimer !== void 0) {
|
|
1295
|
+
clearTimeout(this.quietTimer);
|
|
1296
|
+
this.quietTimer = void 0;
|
|
1297
|
+
}
|
|
1298
|
+
this.prebuffering = false;
|
|
1299
|
+
const buffers = this.prebuffer;
|
|
1300
|
+
this.prebuffer = [];
|
|
1301
|
+
this.prebufferDuration = 0;
|
|
1302
|
+
for (const samples of buffers) this.scheduleSamples(samples);
|
|
1303
|
+
}
|
|
1304
|
+
resetPrebuffer() {
|
|
1305
|
+
if (this.quietTimer !== void 0) {
|
|
1306
|
+
clearTimeout(this.quietTimer);
|
|
1307
|
+
this.quietTimer = void 0;
|
|
1308
|
+
}
|
|
1309
|
+
if (this.endTimer !== void 0) {
|
|
1310
|
+
clearTimeout(this.endTimer);
|
|
1311
|
+
this.endTimer = void 0;
|
|
1312
|
+
}
|
|
1313
|
+
this.prebuffering = true;
|
|
1314
|
+
this.prebuffer = [];
|
|
1315
|
+
this.prebufferDuration = 0;
|
|
1316
|
+
}
|
|
1317
|
+
scheduleSamples(samples) {
|
|
1318
|
+
const duration = samples.length / this.sink.sampleRate;
|
|
1319
|
+
const now = this.sink.currentTime;
|
|
1320
|
+
const startTime = Math.max(this.nextStartTime, now);
|
|
1321
|
+
this.nextStartTime = startTime + duration;
|
|
1322
|
+
const source = this.sink.schedule(samples, startTime, () => {
|
|
1323
|
+
const index = this.sources.indexOf(source);
|
|
1324
|
+
if (index !== -1) this.sources.splice(index, 1);
|
|
1325
|
+
if (this.sources.length === 0) {
|
|
1326
|
+
this.nextStartTime = 0;
|
|
1327
|
+
this.setIsPlaying(false);
|
|
1328
|
+
if (this.endTimer !== void 0) clearTimeout(this.endTimer);
|
|
1329
|
+
this.endTimer = setTimeout(() => {
|
|
1330
|
+
this.endTimer = void 0;
|
|
1331
|
+
this.resetPrebuffer();
|
|
1332
|
+
}, UTTERANCE_END_DELAY);
|
|
1333
|
+
}
|
|
1334
|
+
});
|
|
1335
|
+
this.sources.push(source);
|
|
1336
|
+
this.scheduleVolume(samples, startTime - now);
|
|
1337
|
+
this.setIsPlaying(true);
|
|
1338
|
+
}
|
|
1339
|
+
// Report the level of what is being heard rather than of what just arrived:
|
|
1340
|
+
// a burst of chunks would otherwise light up the meter all at once and leave
|
|
1341
|
+
// it dark for the rest of the sentence.
|
|
1342
|
+
scheduleVolume(samples, delay) {
|
|
1343
|
+
const step = Math.max(
|
|
1344
|
+
1,
|
|
1345
|
+
Math.round(VOLUME_INTERVAL2 / 1e3 * this.sink.sampleRate)
|
|
1346
|
+
);
|
|
1347
|
+
for (let offset = 0; offset < samples.length; offset += step) {
|
|
1348
|
+
const slice = samples.subarray(offset, offset + step);
|
|
1349
|
+
const at = delay * 1e3 + offset / this.sink.sampleRate * 1e3;
|
|
1350
|
+
const timer = setTimeout(
|
|
1351
|
+
() => {
|
|
1352
|
+
const index = this.volumeTimers.indexOf(timer);
|
|
1353
|
+
if (index !== -1) this.volumeTimers.splice(index, 1);
|
|
1354
|
+
this.emit(
|
|
1355
|
+
"Volume",
|
|
1356
|
+
this.meter.measure(
|
|
1357
|
+
resample(slice, this.sink.sampleRate, VOLUME_SAMPLE_RATE)
|
|
1358
|
+
)
|
|
1359
|
+
);
|
|
1360
|
+
},
|
|
1361
|
+
Math.max(0, at)
|
|
1362
|
+
);
|
|
1363
|
+
this.volumeTimers.push(timer);
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
clearVolumeTimers() {
|
|
1367
|
+
for (const timer of this.volumeTimers) clearTimeout(timer);
|
|
1368
|
+
this.volumeTimers = [];
|
|
1369
|
+
this.meter.reset();
|
|
1370
|
+
}
|
|
1371
|
+
setIsPlaying(isPlaying) {
|
|
1372
|
+
if (this.isPlaying === isPlaying) return;
|
|
1373
|
+
this.isPlaying = isPlaying;
|
|
1374
|
+
if (!isPlaying) {
|
|
1375
|
+
this.clearVolumeTimers();
|
|
1376
|
+
this.emit("Volume", -Infinity);
|
|
1377
|
+
}
|
|
1378
|
+
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
1379
|
+
}
|
|
1380
|
+
};
|
|
1381
|
+
|
|
1382
|
+
// src/audio/types.ts
|
|
1383
|
+
var import_eventemitter36 = require("eventemitter3");
|
|
1384
|
+
var MicDriver = class extends import_eventemitter36.EventEmitter {
|
|
1385
|
+
};
|
|
1386
|
+
var SpeakerDriver = class extends import_eventemitter36.EventEmitter {
|
|
1387
|
+
};
|
|
1388
|
+
|
|
1016
1389
|
// src/audio/index.ts
|
|
1017
|
-
|
|
1018
|
-
|
|
1390
|
+
var globalScope = globalThis;
|
|
1391
|
+
if (!globalScope.micdropMic) {
|
|
1392
|
+
globalScope.micdropMic = new MicController();
|
|
1019
1393
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
window.micdropSpeaker = new Speaker();
|
|
1394
|
+
if (!globalScope.micdropSpeaker) {
|
|
1395
|
+
globalScope.micdropSpeaker = new SpeakerController();
|
|
1023
1396
|
}
|
|
1024
|
-
var
|
|
1397
|
+
var Mic = globalScope.micdropMic;
|
|
1398
|
+
var Speaker = globalScope.micdropSpeaker;
|
|
1025
1399
|
|
|
1026
1400
|
// src/client/MicdropClient.ts
|
|
1027
|
-
var
|
|
1401
|
+
var import_eventemitter37 = require("eventemitter3");
|
|
1028
1402
|
|
|
1029
1403
|
// ../server/src/types.ts
|
|
1030
1404
|
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
@@ -1062,7 +1436,7 @@ var MicdropClientError = class extends Error {
|
|
|
1062
1436
|
}
|
|
1063
1437
|
};
|
|
1064
1438
|
function getClientErrorFromWSCloseEvent(event) {
|
|
1065
|
-
const code = getClientErrorFromWSCloseEventCode(event.code);
|
|
1439
|
+
const code = getClientErrorFromWSCloseEventCode(event.code ?? 1006);
|
|
1066
1440
|
return code && new MicdropClientError(code, event.reason);
|
|
1067
1441
|
}
|
|
1068
1442
|
function getClientErrorFromWSCloseEventCode(code) {
|
|
@@ -1092,7 +1466,7 @@ var DEFAULT_RECONNECT_OPTIONS = {
|
|
|
1092
1466
|
delayMs: 1e3,
|
|
1093
1467
|
connectionTimeout: 5e3
|
|
1094
1468
|
};
|
|
1095
|
-
var MicdropClient = class extends
|
|
1469
|
+
var MicdropClient = class extends import_eventemitter37.EventEmitter {
|
|
1096
1470
|
constructor(options = {}) {
|
|
1097
1471
|
super();
|
|
1098
1472
|
this.options = options;
|
|
@@ -1117,7 +1491,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1117
1491
|
this._isReconnecting = false;
|
|
1118
1492
|
this.reconnectAttempt = 0;
|
|
1119
1493
|
if (this.reconnectTimer) {
|
|
1120
|
-
|
|
1494
|
+
clearTimeout(this.reconnectTimer);
|
|
1121
1495
|
this.reconnectTimer = void 0;
|
|
1122
1496
|
}
|
|
1123
1497
|
if (!this.micRecorder || options?.vad) {
|
|
@@ -1131,7 +1505,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1131
1505
|
this._isPaused = false;
|
|
1132
1506
|
this._isReconnecting = false;
|
|
1133
1507
|
if (this.reconnectTimer) {
|
|
1134
|
-
|
|
1508
|
+
clearTimeout(this.reconnectTimer);
|
|
1135
1509
|
this.reconnectTimer = void 0;
|
|
1136
1510
|
}
|
|
1137
1511
|
try {
|
|
@@ -1139,13 +1513,12 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1139
1513
|
} catch (error) {
|
|
1140
1514
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1141
1515
|
}
|
|
1516
|
+
Speaker.stopAudio();
|
|
1142
1517
|
try {
|
|
1143
|
-
this.
|
|
1144
|
-
this.stopMic();
|
|
1518
|
+
await this.stopMic();
|
|
1145
1519
|
} catch (error) {
|
|
1146
1520
|
console.error("[MicdropClient] Error stopping microphone", error);
|
|
1147
1521
|
}
|
|
1148
|
-
Speaker2.stopAudio();
|
|
1149
1522
|
};
|
|
1150
1523
|
this.mute = () => {
|
|
1151
1524
|
this.vad?.pause();
|
|
@@ -1153,7 +1526,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1153
1526
|
this.notifyStateChange();
|
|
1154
1527
|
};
|
|
1155
1528
|
this.unmute = () => {
|
|
1156
|
-
if (!this.isPaused && !(this.options.disableInterruption &&
|
|
1529
|
+
if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
|
|
1157
1530
|
this.vad?.resume();
|
|
1158
1531
|
}
|
|
1159
1532
|
this._isMuted = false;
|
|
@@ -1165,7 +1538,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1165
1538
|
this._isPaused = true;
|
|
1166
1539
|
this._isProcessing = false;
|
|
1167
1540
|
this.notifyStateChange();
|
|
1168
|
-
|
|
1541
|
+
Speaker.stopAudio();
|
|
1169
1542
|
this.ws?.send("Mute" /* Mute */);
|
|
1170
1543
|
};
|
|
1171
1544
|
this.resume = () => {
|
|
@@ -1189,20 +1562,24 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1189
1562
|
this.micRecorder.changeVad(options.vad);
|
|
1190
1563
|
}
|
|
1191
1564
|
} else {
|
|
1192
|
-
this.micRecorder = new MicRecorder(
|
|
1565
|
+
this.micRecorder = new MicRecorder(
|
|
1566
|
+
this.options.vad,
|
|
1567
|
+
this.options.turnDetector,
|
|
1568
|
+
this.options.turnMaxWait
|
|
1569
|
+
);
|
|
1193
1570
|
this.micRecorder.on("StateChange", () => {
|
|
1194
1571
|
this.notifyStateChange();
|
|
1195
1572
|
});
|
|
1196
|
-
this.micRecorder.on("Chunk", (
|
|
1197
|
-
this.log(
|
|
1198
|
-
this.ws?.send(
|
|
1573
|
+
this.micRecorder.on("Chunk", (chunk) => {
|
|
1574
|
+
this.log("User audio chunk", chunk.length);
|
|
1575
|
+
this.ws?.send(pcm16ToArrayBuffer(chunk));
|
|
1199
1576
|
});
|
|
1200
1577
|
this.micRecorder.on("StartSpeaking", () => {
|
|
1201
1578
|
this.log("User start speaking");
|
|
1202
1579
|
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
1203
1580
|
this._isProcessing = false;
|
|
1204
1581
|
this.notifyStateChange();
|
|
1205
|
-
|
|
1582
|
+
Speaker.stopAudio();
|
|
1206
1583
|
});
|
|
1207
1584
|
this.micRecorder.on("StopSpeaking", () => {
|
|
1208
1585
|
this.log("User stop speaking");
|
|
@@ -1213,11 +1590,16 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1213
1590
|
this.notifyStateChange();
|
|
1214
1591
|
});
|
|
1215
1592
|
}
|
|
1216
|
-
const
|
|
1217
|
-
|
|
1218
|
-
|
|
1593
|
+
const deviceId = options.deviceId ?? storage.getItem("micdrop.micDevice" /* MicDevice */) ?? void 0;
|
|
1594
|
+
await Mic.start(deviceId);
|
|
1595
|
+
if (Mic.deviceId) {
|
|
1596
|
+
storage.setItem("micdrop.micDevice" /* MicDevice */, Mic.deviceId);
|
|
1597
|
+
} else {
|
|
1598
|
+
storage.removeItem("micdrop.micDevice" /* MicDevice */);
|
|
1599
|
+
}
|
|
1600
|
+
await this.micRecorder.start(Mic);
|
|
1601
|
+
await Speaker.start();
|
|
1219
1602
|
await this.updateDevices();
|
|
1220
|
-
await Speaker2.start();
|
|
1221
1603
|
this.notifyStateChange();
|
|
1222
1604
|
} catch (error) {
|
|
1223
1605
|
this.setError(
|
|
@@ -1230,41 +1612,48 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1230
1612
|
throw error;
|
|
1231
1613
|
}
|
|
1232
1614
|
};
|
|
1615
|
+
/**
|
|
1616
|
+
* Changes what decides when a turn is over, during a call or before one
|
|
1617
|
+
* @param turnDetector - The new detector, or nothing to leave it to the VAD
|
|
1618
|
+
*/
|
|
1619
|
+
this.setTurnDetector = (turnDetector) => {
|
|
1620
|
+
this.options.turnDetector = turnDetector;
|
|
1621
|
+
this.micRecorder?.changeTurnDetector(turnDetector);
|
|
1622
|
+
};
|
|
1623
|
+
/**
|
|
1624
|
+
* Says how long a held turn waits for the rest of the sentence.
|
|
1625
|
+
*
|
|
1626
|
+
* The way out of a wrong verdict: a detector can hear an unfinished sentence
|
|
1627
|
+
* where there is none, and the speaker who never comes back still gets an
|
|
1628
|
+
* answer once this runs out.
|
|
1629
|
+
* @param milliseconds - How long to wait past the detector asking to
|
|
1630
|
+
*/
|
|
1631
|
+
this.setTurnMaxWait = (milliseconds) => {
|
|
1632
|
+
this.options.turnMaxWait = milliseconds;
|
|
1633
|
+
if (this.micRecorder) {
|
|
1634
|
+
this.micRecorder.turnMaxWait = milliseconds;
|
|
1635
|
+
}
|
|
1636
|
+
};
|
|
1233
1637
|
this.changeMicDevice = async (deviceId) => {
|
|
1234
1638
|
await this.startMic({ deviceId });
|
|
1235
1639
|
};
|
|
1236
1640
|
this.changeSpeakerDevice = async (deviceId) => {
|
|
1237
|
-
await
|
|
1641
|
+
await Speaker.changeDevice(deviceId);
|
|
1238
1642
|
this.notifyStateChange();
|
|
1239
1643
|
};
|
|
1240
1644
|
this.updateDevices = async () => {
|
|
1241
|
-
const
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
const defaultDevicesIndexes = defaultDevices.map(
|
|
1248
|
-
(device) => devices.findIndex(
|
|
1249
|
-
(d) => d.groupId === device.groupId && d.kind === device.kind
|
|
1250
|
-
)
|
|
1251
|
-
).filter((index) => index !== -1);
|
|
1252
|
-
for (const index of defaultDevicesIndexes) {
|
|
1253
|
-
const device = devices[index];
|
|
1254
|
-
devices.splice(index, 1);
|
|
1255
|
-
devices.unshift(device);
|
|
1256
|
-
}
|
|
1257
|
-
this.micDevices = devices.filter((device) => device.kind === "audioinput");
|
|
1258
|
-
this.speakerDevices = devices.filter(
|
|
1259
|
-
(device) => device.kind === "audiooutput"
|
|
1260
|
-
);
|
|
1261
|
-
Speaker2.devices = this.speakerDevices;
|
|
1645
|
+
const [micDevices, speakerDevices] = await Promise.all([
|
|
1646
|
+
Mic.getDevices(),
|
|
1647
|
+
Speaker.getDevices()
|
|
1648
|
+
]);
|
|
1649
|
+
this.micDevices = micDevices;
|
|
1650
|
+
this.speakerDevices = speakerDevices;
|
|
1262
1651
|
this.notifyStateChange();
|
|
1263
1652
|
};
|
|
1264
1653
|
this.onWSOpen = () => {
|
|
1265
1654
|
this.log("WebSocket opened");
|
|
1266
1655
|
if (this.connectionTimer) {
|
|
1267
|
-
|
|
1656
|
+
clearTimeout(this.connectionTimer);
|
|
1268
1657
|
this.connectionTimer = void 0;
|
|
1269
1658
|
}
|
|
1270
1659
|
this._isReconnecting = false;
|
|
@@ -1279,36 +1668,36 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1279
1668
|
}
|
|
1280
1669
|
};
|
|
1281
1670
|
this.onWSMessage = (event) => {
|
|
1282
|
-
|
|
1283
|
-
if (
|
|
1671
|
+
const data = event.data;
|
|
1672
|
+
if (data instanceof ArrayBuffer) {
|
|
1673
|
+
this.log("Received audio", data.byteLength);
|
|
1284
1674
|
if (this.isPaused || this.isUserSpeaking) return;
|
|
1285
|
-
|
|
1675
|
+
Speaker.playAudio(new Int16Array(data));
|
|
1286
1676
|
this._isProcessing = false;
|
|
1287
1677
|
this.notifyStateChange();
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1678
|
+
return;
|
|
1679
|
+
}
|
|
1680
|
+
if (typeof data !== "string") {
|
|
1681
|
+
console.warn(`[MicdropClient] Unknown message type: ${data}`);
|
|
1682
|
+
return;
|
|
1683
|
+
}
|
|
1684
|
+
this.log("Received message:", data);
|
|
1685
|
+
if (data.startsWith("Message" /* Message */)) {
|
|
1291
1686
|
try {
|
|
1292
1687
|
const message = JSON.parse(
|
|
1293
|
-
|
|
1688
|
+
data.substring("Message" /* Message */.length + 1)
|
|
1294
1689
|
);
|
|
1295
1690
|
this.addMessage(message);
|
|
1296
1691
|
} catch (error) {
|
|
1297
|
-
console.error(
|
|
1298
|
-
"[MicdropClient] Error parsing message:",
|
|
1299
|
-
event.data,
|
|
1300
|
-
error
|
|
1301
|
-
);
|
|
1692
|
+
console.error("[MicdropClient] Error parsing message:", data, error);
|
|
1302
1693
|
}
|
|
1303
|
-
} else if (
|
|
1694
|
+
} else if (data === "EndCall" /* EndCall */) {
|
|
1304
1695
|
this.emit("EndCall");
|
|
1305
|
-
} else if (
|
|
1696
|
+
} else if (data === "SkipAnswer" /* SkipAnswer */) {
|
|
1306
1697
|
this._isProcessing = false;
|
|
1307
1698
|
this.notifyStateChange();
|
|
1308
|
-
} else if (
|
|
1309
|
-
const lastMessage = this.conversation.
|
|
1310
|
-
(message) => message.role === "user"
|
|
1311
|
-
);
|
|
1699
|
+
} else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
1700
|
+
const lastMessage = this.conversation.map((message) => message.role).lastIndexOf("user");
|
|
1312
1701
|
if (lastMessage !== -1) {
|
|
1313
1702
|
this.conversation = this.conversation.filter(
|
|
1314
1703
|
(_, index) => index !== lastMessage
|
|
@@ -1316,24 +1705,20 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1316
1705
|
this._isProcessing = false;
|
|
1317
1706
|
this.notifyStateChange();
|
|
1318
1707
|
}
|
|
1319
|
-
} else if (
|
|
1708
|
+
} else if (data.startsWith("ToolCall" /* ToolCall */)) {
|
|
1320
1709
|
try {
|
|
1321
1710
|
const toolCall = JSON.parse(
|
|
1322
|
-
|
|
1711
|
+
data.substring("ToolCall" /* ToolCall */.length + 1)
|
|
1323
1712
|
);
|
|
1324
1713
|
this.emit("ToolCall", toolCall);
|
|
1325
1714
|
} catch (error) {
|
|
1326
|
-
console.error(
|
|
1327
|
-
"[MicdropClient] Error parsing tool call:",
|
|
1328
|
-
event.data,
|
|
1329
|
-
error
|
|
1330
|
-
);
|
|
1715
|
+
console.error("[MicdropClient] Error parsing tool call:", data, error);
|
|
1331
1716
|
}
|
|
1332
1717
|
}
|
|
1333
1718
|
};
|
|
1334
1719
|
this.onWSClose = (event) => {
|
|
1335
|
-
this.log("WebSocket closed", event);
|
|
1336
|
-
const error = getClientErrorFromWSCloseEvent(event);
|
|
1720
|
+
this.log("WebSocket closed", event?.code, event?.reason);
|
|
1721
|
+
const error = getClientErrorFromWSCloseEvent(event ?? {});
|
|
1337
1722
|
if (error) {
|
|
1338
1723
|
this.setError(error);
|
|
1339
1724
|
}
|
|
@@ -1347,7 +1732,7 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1347
1732
|
this.stop();
|
|
1348
1733
|
}
|
|
1349
1734
|
};
|
|
1350
|
-
this.onWSError = (
|
|
1735
|
+
this.onWSError = () => {
|
|
1351
1736
|
this.setError(new MicdropClientError("Connection" /* Connection */));
|
|
1352
1737
|
};
|
|
1353
1738
|
this.onSpeakerStartPlaying = () => {
|
|
@@ -1370,9 +1755,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1370
1755
|
this.notifyStateChange();
|
|
1371
1756
|
}
|
|
1372
1757
|
};
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1758
|
+
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1759
|
+
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1760
|
+
Mic.on("DeviceChange", this.updateDevices);
|
|
1376
1761
|
}
|
|
1377
1762
|
get vad() {
|
|
1378
1763
|
return this.micRecorder?.vad;
|
|
@@ -1399,13 +1784,13 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1399
1784
|
return this.isMicStarted && !this.isPaused && !this.isProcessing && !this.isMicMuted && !this.isUserSpeaking && !this.isAssistantSpeaking;
|
|
1400
1785
|
}
|
|
1401
1786
|
get isWSStarted() {
|
|
1402
|
-
return this.ws?.readyState ===
|
|
1787
|
+
return this.ws?.readyState === 1;
|
|
1403
1788
|
}
|
|
1404
1789
|
get isWSStarting() {
|
|
1405
|
-
return this.ws?.readyState ===
|
|
1790
|
+
return this.ws?.readyState === 0;
|
|
1406
1791
|
}
|
|
1407
1792
|
get isMicStarted() {
|
|
1408
|
-
return
|
|
1793
|
+
return Mic.isStarted;
|
|
1409
1794
|
}
|
|
1410
1795
|
get isMicMuted() {
|
|
1411
1796
|
return this.vad?.isPaused ?? false;
|
|
@@ -1414,13 +1799,13 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1414
1799
|
return this.micRecorder?.state.isSpeaking ?? false;
|
|
1415
1800
|
}
|
|
1416
1801
|
get isAssistantSpeaking() {
|
|
1417
|
-
return
|
|
1802
|
+
return Speaker.isPlaying;
|
|
1418
1803
|
}
|
|
1419
1804
|
get micDeviceId() {
|
|
1420
|
-
return
|
|
1805
|
+
return Mic.deviceId;
|
|
1421
1806
|
}
|
|
1422
1807
|
get speakerDeviceId() {
|
|
1423
|
-
return
|
|
1808
|
+
return Speaker.deviceId;
|
|
1424
1809
|
}
|
|
1425
1810
|
get state() {
|
|
1426
1811
|
return {
|
|
@@ -1443,11 +1828,10 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1443
1828
|
speakerDevices: this.speakerDevices
|
|
1444
1829
|
};
|
|
1445
1830
|
}
|
|
1446
|
-
stopMic() {
|
|
1831
|
+
async stopMic() {
|
|
1447
1832
|
this.micRecorder?.stop();
|
|
1448
1833
|
this.micRecorder = void 0;
|
|
1449
|
-
|
|
1450
|
-
this.micStream = void 0;
|
|
1834
|
+
await Mic.stop();
|
|
1451
1835
|
this.notifyStateChange();
|
|
1452
1836
|
}
|
|
1453
1837
|
async startWS() {
|
|
@@ -1463,22 +1847,22 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1463
1847
|
throw new MicdropClientError("MissingUrl" /* MissingUrl */);
|
|
1464
1848
|
}
|
|
1465
1849
|
this.ws = new WebSocket(this.options.url);
|
|
1466
|
-
this.ws.binaryType = "
|
|
1850
|
+
this.ws.binaryType = "arraybuffer";
|
|
1467
1851
|
this.notifyStateChange();
|
|
1468
1852
|
this.ws.onopen = this.onWSOpen;
|
|
1469
1853
|
this.ws.onmessage = this.onWSMessage;
|
|
1470
1854
|
this.ws.onclose = this.onWSClose;
|
|
1471
1855
|
this.ws.onerror = this.onWSError;
|
|
1472
|
-
this.connectionTimer =
|
|
1856
|
+
this.connectionTimer = setTimeout(() => {
|
|
1473
1857
|
this.connectionTimer = void 0;
|
|
1474
|
-
if (this.ws?.readyState ===
|
|
1858
|
+
if (this.ws?.readyState === 0) {
|
|
1475
1859
|
this.log("WebSocket connection timeout");
|
|
1476
1860
|
this.ws.close();
|
|
1477
1861
|
}
|
|
1478
1862
|
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
1479
1863
|
} catch (error) {
|
|
1480
1864
|
this.setError(
|
|
1481
|
-
new MicdropClientError(
|
|
1865
|
+
error instanceof MicdropClientError ? error : new MicdropClientError(
|
|
1482
1866
|
"Connection" /* Connection */,
|
|
1483
1867
|
error?.message
|
|
1484
1868
|
)
|
|
@@ -1489,11 +1873,11 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1489
1873
|
}
|
|
1490
1874
|
stopWS() {
|
|
1491
1875
|
if (this.connectionTimer) {
|
|
1492
|
-
|
|
1876
|
+
clearTimeout(this.connectionTimer);
|
|
1493
1877
|
this.connectionTimer = void 0;
|
|
1494
1878
|
}
|
|
1495
1879
|
if (!this.ws) return;
|
|
1496
|
-
if (this.ws.readyState ===
|
|
1880
|
+
if (this.ws.readyState === 1 || this.ws.readyState === 0) {
|
|
1497
1881
|
this.ws.close();
|
|
1498
1882
|
}
|
|
1499
1883
|
this.ws = void 0;
|
|
@@ -1513,9 +1897,9 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1513
1897
|
this.vad?.pause?.();
|
|
1514
1898
|
}
|
|
1515
1899
|
if (this.reconnectTimer) {
|
|
1516
|
-
|
|
1900
|
+
clearTimeout(this.reconnectTimer);
|
|
1517
1901
|
}
|
|
1518
|
-
this.reconnectTimer =
|
|
1902
|
+
this.reconnectTimer = setTimeout(async () => {
|
|
1519
1903
|
try {
|
|
1520
1904
|
await this.startWS();
|
|
1521
1905
|
} catch {
|
|
@@ -1545,14 +1929,17 @@ var MicdropClient = class extends import_eventemitter36.EventEmitter {
|
|
|
1545
1929
|
};
|
|
1546
1930
|
|
|
1547
1931
|
// src/Micdrop.ts
|
|
1548
|
-
|
|
1549
|
-
|
|
1932
|
+
var globalScope2 = globalThis;
|
|
1933
|
+
if (!globalScope2.micdropClient) {
|
|
1934
|
+
globalScope2.micdropClient = new MicdropClient();
|
|
1550
1935
|
}
|
|
1551
|
-
var Micdrop =
|
|
1936
|
+
var Micdrop = globalScope2.micdropClient;
|
|
1552
1937
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1553
1938
|
0 && (module.exports = {
|
|
1554
|
-
|
|
1939
|
+
DEFAULT_TURN_MAX_WAIT,
|
|
1555
1940
|
Mic,
|
|
1941
|
+
MicController,
|
|
1942
|
+
MicDriver,
|
|
1556
1943
|
MicRecorder,
|
|
1557
1944
|
Micdrop,
|
|
1558
1945
|
MicdropClient,
|
|
@@ -1560,16 +1947,33 @@ var Micdrop = window.micdropClient;
|
|
|
1560
1947
|
MicdropClientError,
|
|
1561
1948
|
MicdropClientErrorCode,
|
|
1562
1949
|
MicdropServerCommands,
|
|
1950
|
+
MicdropStorageKeys,
|
|
1563
1951
|
MultipleVAD,
|
|
1952
|
+
Pcm16AudioStream,
|
|
1953
|
+
SILERO_FRAME_SAMPLES,
|
|
1954
|
+
SILERO_SAMPLE_RATE,
|
|
1564
1955
|
SileroVAD,
|
|
1565
1956
|
Speaker,
|
|
1957
|
+
SpeakerController,
|
|
1958
|
+
SpeakerDriver,
|
|
1566
1959
|
VAD,
|
|
1567
1960
|
VADStatus,
|
|
1961
|
+
VOLUME_SAMPLE_RATE,
|
|
1962
|
+
VolumeMeter,
|
|
1568
1963
|
VolumeVAD,
|
|
1964
|
+
concatFloat32,
|
|
1569
1965
|
equalVADConfig,
|
|
1966
|
+
floatToPcm16,
|
|
1570
1967
|
getClientErrorFromWSCloseEvent,
|
|
1571
1968
|
getClientErrorFromWSCloseEventCode,
|
|
1572
1969
|
getVAD,
|
|
1573
|
-
|
|
1970
|
+
getVolume,
|
|
1971
|
+
isRecoverableError,
|
|
1972
|
+
pcm16ToArrayBuffer,
|
|
1973
|
+
pcm16ToFloat,
|
|
1974
|
+
resample,
|
|
1975
|
+
setMicdropStorage,
|
|
1976
|
+
setSileroModelLoader,
|
|
1977
|
+
storage
|
|
1574
1978
|
});
|
|
1575
1979
|
//# sourceMappingURL=index.js.map
|