@micdrop/client 2.2.9 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -24
- package/dist/index.d.mts +444 -73
- package/dist/index.d.ts +444 -73
- package/dist/index.js +983 -678
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +963 -676
- package/dist/index.mjs.map +1 -1
- package/package.json +11 -7
package/dist/index.mjs
CHANGED
|
@@ -1,424 +1,452 @@
|
|
|
1
|
-
// src/audio/
|
|
1
|
+
// src/audio/Mic.ts
|
|
2
2
|
import { EventEmitter } from "eventemitter3";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
3
|
+
|
|
4
|
+
// src/audio/pcm.ts
|
|
5
|
+
function resample(input, inputRate, outputRate) {
|
|
6
|
+
if (inputRate === outputRate || input.length === 0) return input;
|
|
7
|
+
const ratio = inputRate / outputRate;
|
|
8
|
+
const outputLength = Math.round(input.length / ratio);
|
|
9
|
+
const output = new Float32Array(outputLength);
|
|
10
|
+
for (let i = 0; i < outputLength; i++) {
|
|
11
|
+
const position = i * ratio;
|
|
12
|
+
const index = Math.floor(position);
|
|
13
|
+
const nextIndex = Math.min(index + 1, input.length - 1);
|
|
14
|
+
const fraction = position - index;
|
|
15
|
+
output[i] = input[index] * (1 - fraction) + input[nextIndex] * fraction;
|
|
16
|
+
}
|
|
17
|
+
return output;
|
|
18
|
+
}
|
|
19
|
+
function floatToPcm16(input) {
|
|
20
|
+
const output = new Int16Array(input.length);
|
|
21
|
+
for (let i = 0; i < input.length; i++) {
|
|
22
|
+
const sample = Math.max(-1, Math.min(1, input[i]));
|
|
23
|
+
output[i] = sample < 0 ? sample * 32768 : sample * 32767;
|
|
22
24
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
maxVolume = this.fftBins[i];
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
return maxVolume;
|
|
25
|
+
return output;
|
|
26
|
+
}
|
|
27
|
+
function pcm16ToFloat(input) {
|
|
28
|
+
const output = new Float32Array(input.length);
|
|
29
|
+
for (let i = 0; i < input.length; i++) {
|
|
30
|
+
output[i] = input[i] / 32768;
|
|
33
31
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
|
|
32
|
+
return output;
|
|
33
|
+
}
|
|
34
|
+
function concatFloat32(chunks) {
|
|
35
|
+
if (chunks.length === 1) return chunks[0];
|
|
36
|
+
let length = 0;
|
|
37
|
+
for (const chunk of chunks) length += chunk.length;
|
|
38
|
+
const output = new Float32Array(length);
|
|
39
|
+
let offset = 0;
|
|
40
|
+
for (const chunk of chunks) {
|
|
41
|
+
output.set(chunk, offset);
|
|
42
|
+
offset += chunk.length;
|
|
43
|
+
}
|
|
44
|
+
return output;
|
|
47
45
|
}
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
source.disconnect();
|
|
54
|
-
source.buffer = null;
|
|
55
|
-
source.removeEventListener("ended", onEnded);
|
|
56
|
-
};
|
|
57
|
-
source.addEventListener("ended", onEnded);
|
|
58
|
-
source.start();
|
|
46
|
+
function pcm16ToArrayBuffer(pcm) {
|
|
47
|
+
return pcm.buffer.slice(
|
|
48
|
+
pcm.byteOffset,
|
|
49
|
+
pcm.byteOffset + pcm.byteLength
|
|
50
|
+
);
|
|
59
51
|
}
|
|
60
52
|
|
|
61
|
-
// src/audio/
|
|
62
|
-
var
|
|
63
|
-
|
|
64
|
-
|
|
53
|
+
// src/audio/volume.ts
|
|
54
|
+
var FFT_SIZE = 512;
|
|
55
|
+
var BIN_COUNT = FFT_SIZE / 2;
|
|
56
|
+
var FIRST_BIN = 4;
|
|
57
|
+
var SMOOTHING = 0.1;
|
|
58
|
+
var VOLUME_SAMPLE_RATE = 16e3;
|
|
59
|
+
var WINDOW = new Float32Array(FFT_SIZE);
|
|
60
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
61
|
+
const a = 2 * Math.PI * i / (FFT_SIZE - 1);
|
|
62
|
+
WINDOW[i] = 0.42 - 0.5 * Math.cos(a) + 0.08 * Math.cos(2 * a);
|
|
65
63
|
}
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
64
|
+
var REVERSED = new Uint16Array(FFT_SIZE);
|
|
65
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
66
|
+
let reversed = 0;
|
|
67
|
+
for (let bit = 1; bit < FFT_SIZE; bit <<= 1) {
|
|
68
|
+
reversed = reversed << 1 | (i & bit ? 1 : 0);
|
|
69
|
+
}
|
|
70
|
+
REVERSED[i] = reversed;
|
|
71
71
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
LocalStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
78
|
-
LocalStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
79
|
-
return LocalStorageKeys2;
|
|
80
|
-
})(LocalStorageKeys || {});
|
|
81
|
-
|
|
82
|
-
// src/audio/utils/stopStream.ts
|
|
83
|
-
function stopStream(stream) {
|
|
84
|
-
stream?.getTracks().forEach((track) => track.stop());
|
|
72
|
+
var COS = new Float32Array(FFT_SIZE / 2);
|
|
73
|
+
var SIN = new Float32Array(FFT_SIZE / 2);
|
|
74
|
+
for (let i = 0; i < FFT_SIZE / 2; i++) {
|
|
75
|
+
COS[i] = Math.cos(-2 * Math.PI * i / FFT_SIZE);
|
|
76
|
+
SIN[i] = Math.sin(-2 * Math.PI * i / FFT_SIZE);
|
|
85
77
|
}
|
|
86
|
-
|
|
87
|
-
// src/audio/Mic.ts
|
|
88
|
-
var Mic = class {
|
|
78
|
+
var VolumeMeter = class {
|
|
89
79
|
constructor() {
|
|
90
|
-
this.
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
deviceId: { ideal: deviceId },
|
|
96
|
-
sampleRate: 16e3,
|
|
97
|
-
// not working, it will follow device settings, usually 44.1kHz
|
|
98
|
-
sampleSize: 16,
|
|
99
|
-
channelCount: 1,
|
|
100
|
-
echoCancellation: true,
|
|
101
|
-
noiseSuppression: true
|
|
102
|
-
}
|
|
103
|
-
};
|
|
80
|
+
this.real = new Float32Array(FFT_SIZE);
|
|
81
|
+
this.imaginary = new Float32Array(FFT_SIZE);
|
|
82
|
+
this.magnitudes = new Float32Array(BIN_COUNT);
|
|
83
|
+
this.smoothed = new Float32Array(BIN_COUNT);
|
|
84
|
+
this.hasPrevious = false;
|
|
104
85
|
}
|
|
105
86
|
/**
|
|
106
|
-
*
|
|
107
|
-
* @param
|
|
108
|
-
*
|
|
87
|
+
* Measures a window of audio
|
|
88
|
+
* @param samples - At least 512 mono samples at 16 kHz, in the -1..1 range.
|
|
89
|
+
* Only the last 512 are looked at.
|
|
90
|
+
* @returns The level in decibels, between -Infinity (silence) and 0
|
|
109
91
|
*/
|
|
110
|
-
|
|
111
|
-
if (
|
|
112
|
-
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
|
|
92
|
+
measure(samples) {
|
|
93
|
+
if (samples.length === 0) return -Infinity;
|
|
94
|
+
const offset = Math.max(0, samples.length - FFT_SIZE);
|
|
95
|
+
const { real, imaginary } = this;
|
|
96
|
+
for (let i = 0; i < FFT_SIZE; i++) {
|
|
97
|
+
const sample = offset + i < samples.length ? samples[offset + i] : 0;
|
|
98
|
+
real[REVERSED[i]] = sample * WINDOW[i];
|
|
99
|
+
imaginary[REVERSED[i]] = 0;
|
|
116
100
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
101
|
+
for (let size = 2; size <= FFT_SIZE; size <<= 1) {
|
|
102
|
+
const half = size >> 1;
|
|
103
|
+
const step = FFT_SIZE / size;
|
|
104
|
+
for (let start = 0; start < FFT_SIZE; start += size) {
|
|
105
|
+
for (let i = 0; i < half; i++) {
|
|
106
|
+
const even = start + i;
|
|
107
|
+
const odd = even + half;
|
|
108
|
+
const cos = COS[i * step];
|
|
109
|
+
const sin = SIN[i * step];
|
|
110
|
+
const realOdd = real[odd] * cos - imaginary[odd] * sin;
|
|
111
|
+
const imaginaryOdd = real[odd] * sin + imaginary[odd] * cos;
|
|
112
|
+
real[odd] = real[even] - realOdd;
|
|
113
|
+
imaginary[odd] = imaginary[even] - imaginaryOdd;
|
|
114
|
+
real[even] += realOdd;
|
|
115
|
+
imaginary[even] += imaginaryOdd;
|
|
116
|
+
}
|
|
120
117
|
}
|
|
121
|
-
stopStream(this.audioStream);
|
|
122
118
|
}
|
|
123
|
-
|
|
124
|
-
this.
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
if (newDeviceId === "default" && deviceId) {
|
|
130
|
-
newDeviceId = deviceId;
|
|
119
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
120
|
+
this.magnitudes[k] = Math.sqrt(real[k] * real[k] + imaginary[k] * imaginary[k]) / FFT_SIZE;
|
|
121
|
+
}
|
|
122
|
+
if (this.hasPrevious) {
|
|
123
|
+
for (let k = 0; k < BIN_COUNT; k++) {
|
|
124
|
+
this.smoothed[k] = SMOOTHING * this.smoothed[k] + (1 - SMOOTHING) * this.magnitudes[k];
|
|
131
125
|
}
|
|
132
|
-
localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
|
|
133
|
-
this.deviceId = newDeviceId;
|
|
134
126
|
} else {
|
|
135
|
-
|
|
136
|
-
this.
|
|
127
|
+
this.smoothed.set(this.magnitudes);
|
|
128
|
+
this.hasPrevious = true;
|
|
137
129
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
130
|
+
let loudest = -Infinity;
|
|
131
|
+
for (let k = FIRST_BIN; k < BIN_COUNT; k++) {
|
|
132
|
+
const magnitude = this.smoothed[k];
|
|
133
|
+
if (magnitude === 0) continue;
|
|
134
|
+
const decibels = 20 * Math.log10(magnitude);
|
|
135
|
+
if (decibels > loudest && decibels < 0) loudest = decibels;
|
|
136
|
+
}
|
|
137
|
+
return loudest;
|
|
138
|
+
}
|
|
139
|
+
/** Forgets the previous window, for when capture stops and starts again */
|
|
140
|
+
reset() {
|
|
141
|
+
this.hasPrevious = false;
|
|
142
|
+
this.smoothed.fill(0);
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
function getVolume(samples) {
|
|
146
|
+
return new VolumeMeter().measure(samples);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// src/audio/Mic.ts
|
|
150
|
+
var VOLUME_INTERVAL = 100;
|
|
151
|
+
var VOLUME_WINDOW = 512;
|
|
152
|
+
var RATE_CHECK_DURATION = 2e3;
|
|
153
|
+
var RATE_TOLERANCE = 0.15;
|
|
154
|
+
var MicController = class extends EventEmitter {
|
|
155
|
+
constructor() {
|
|
156
|
+
super(...arguments);
|
|
157
|
+
this.meter = new VolumeMeter();
|
|
158
|
+
this.pending = [];
|
|
159
|
+
this.pendingLength = 0;
|
|
160
|
+
this.pendingRate = VOLUME_SAMPLE_RATE;
|
|
161
|
+
this.rateCheckedAt = 0;
|
|
162
|
+
this.rateCheckSamples = 0;
|
|
163
|
+
this.rateWarned = false;
|
|
164
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
165
|
+
this.volume = -Infinity;
|
|
166
|
+
this.onFrames = (frames, sampleRate) => {
|
|
167
|
+
this.emit("Frames", frames, sampleRate);
|
|
168
|
+
this.checkRate(frames.length, sampleRate);
|
|
169
|
+
if (sampleRate !== this.pendingRate) {
|
|
170
|
+
this.pending = [];
|
|
171
|
+
this.pendingLength = 0;
|
|
172
|
+
this.pendingRate = sampleRate;
|
|
173
|
+
this.meter.reset();
|
|
174
|
+
}
|
|
175
|
+
this.pending.push(frames);
|
|
176
|
+
this.pendingLength += frames.length;
|
|
177
|
+
const interval = Math.round(VOLUME_INTERVAL / 1e3 * sampleRate);
|
|
178
|
+
if (this.pendingLength < interval) return;
|
|
179
|
+
const needed = Math.ceil(VOLUME_WINDOW * sampleRate / VOLUME_SAMPLE_RATE);
|
|
180
|
+
const gathered = concatFloat32(this.pending);
|
|
181
|
+
const window = gathered.subarray(Math.max(0, gathered.length - needed));
|
|
182
|
+
this.pending = [];
|
|
183
|
+
this.pendingLength = 0;
|
|
184
|
+
this.volume = this.meter.measure(
|
|
185
|
+
resample(window, sampleRate, VOLUME_SAMPLE_RATE)
|
|
186
|
+
);
|
|
187
|
+
this.emit("Volume", this.volume);
|
|
188
|
+
};
|
|
189
|
+
this.onDeviceChange = () => {
|
|
190
|
+
this.emit("DeviceChange");
|
|
191
|
+
};
|
|
192
|
+
this.onError = (error) => {
|
|
193
|
+
this.emit("Error", error);
|
|
194
|
+
};
|
|
142
195
|
}
|
|
143
196
|
/**
|
|
144
|
-
*
|
|
197
|
+
* Replaces the audio driver, to record with another library
|
|
198
|
+
* @param driver - The driver to record with
|
|
145
199
|
*/
|
|
146
|
-
|
|
147
|
-
if (this.
|
|
148
|
-
|
|
149
|
-
this.
|
|
200
|
+
setDriver(driver) {
|
|
201
|
+
if (this.driver === driver) return;
|
|
202
|
+
if (this.driver) {
|
|
203
|
+
this.driver.off("Frames", this.onFrames);
|
|
204
|
+
this.driver.off("DeviceChange", this.onDeviceChange);
|
|
205
|
+
this.driver.off("Error", this.onError);
|
|
150
206
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
207
|
+
this.driver = driver;
|
|
208
|
+
driver.on("Frames", this.onFrames);
|
|
209
|
+
driver.on("DeviceChange", this.onDeviceChange);
|
|
210
|
+
driver.on("Error", this.onError);
|
|
211
|
+
}
|
|
212
|
+
getDriver() {
|
|
213
|
+
if (!this.driver) {
|
|
214
|
+
throw new Error(
|
|
215
|
+
"No microphone driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Mic.setDriver()."
|
|
216
|
+
);
|
|
154
217
|
}
|
|
218
|
+
return this.driver;
|
|
219
|
+
}
|
|
220
|
+
get isStarted() {
|
|
221
|
+
return this.driver?.isStarted ?? false;
|
|
222
|
+
}
|
|
223
|
+
get deviceId() {
|
|
224
|
+
return this.driver?.deviceId;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Asks for the microphone permission and starts capturing
|
|
228
|
+
* @param deviceId - Input device to record from, defaults to the system one
|
|
229
|
+
*/
|
|
230
|
+
async start(deviceId) {
|
|
231
|
+
this.resetVolume();
|
|
232
|
+
await this.getDriver().start(deviceId);
|
|
233
|
+
}
|
|
234
|
+
/** Stops capturing and releases the microphone */
|
|
235
|
+
async stop() {
|
|
236
|
+
await this.driver?.stop();
|
|
237
|
+
this.resetVolume();
|
|
238
|
+
}
|
|
239
|
+
/** Lists the input devices the system offers */
|
|
240
|
+
async getDevices() {
|
|
241
|
+
if (!this.driver) return [];
|
|
242
|
+
return this.driver.getDevices();
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Times the captured audio against the rate it claims to be at.
|
|
246
|
+
*
|
|
247
|
+
* A microphone that announces 16 kHz while delivering 48 kHz sends speech to
|
|
248
|
+
* the server three times too slow, which comes back as a short transcript in
|
|
249
|
+
* a random language. That is invisible otherwise, so it is worth saying out
|
|
250
|
+
* loud once.
|
|
251
|
+
* @param samples - How many samples just arrived
|
|
252
|
+
* @param sampleRate - The rate they are said to be at
|
|
253
|
+
*/
|
|
254
|
+
checkRate(samples, sampleRate) {
|
|
255
|
+
if (this.rateWarned) return;
|
|
256
|
+
const now = Date.now();
|
|
257
|
+
if (this.rateCheckedAt === 0) {
|
|
258
|
+
this.rateCheckedAt = now;
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
this.rateCheckSamples += samples;
|
|
262
|
+
const elapsed = now - this.rateCheckedAt;
|
|
263
|
+
if (elapsed < RATE_CHECK_DURATION) return;
|
|
264
|
+
const delivered = this.rateCheckSamples / elapsed * 1e3;
|
|
265
|
+
this.rateCheckedAt = now;
|
|
266
|
+
this.rateCheckSamples = 0;
|
|
267
|
+
if (Math.abs(delivered - sampleRate) / sampleRate <= RATE_TOLERANCE) return;
|
|
268
|
+
this.rateWarned = true;
|
|
269
|
+
console.warn(
|
|
270
|
+
`[Micdrop] The microphone announces ${sampleRate} Hz but delivers about ${Math.round(delivered)} Hz. Speech is being sent at the wrong speed, so transcription will be poor.`
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
resetVolume() {
|
|
274
|
+
this.pending = [];
|
|
275
|
+
this.pendingLength = 0;
|
|
276
|
+
this.meter.reset();
|
|
277
|
+
this.volume = -Infinity;
|
|
278
|
+
this.rateCheckedAt = 0;
|
|
279
|
+
this.rateCheckSamples = 0;
|
|
155
280
|
}
|
|
156
281
|
};
|
|
157
282
|
|
|
158
283
|
// src/audio/Speaker.ts
|
|
159
|
-
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
160
|
-
|
|
161
|
-
// src/audio/stream/AudioStream.ts
|
|
162
284
|
import { EventEmitter as EventEmitter2 } from "eventemitter3";
|
|
163
|
-
var AudioStream = class extends EventEmitter2 {
|
|
164
|
-
constructor() {
|
|
165
|
-
super();
|
|
166
|
-
this.isPlaying = false;
|
|
167
|
-
this.outputNode = audioContext.createGain();
|
|
168
|
-
}
|
|
169
|
-
setIsPlaying(isPlaying) {
|
|
170
|
-
if (this.isPlaying === isPlaying) return;
|
|
171
|
-
this.isPlaying = isPlaying;
|
|
172
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
173
|
-
}
|
|
174
|
-
};
|
|
175
285
|
|
|
176
|
-
// src/
|
|
177
|
-
var
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
286
|
+
// src/storage.ts
|
|
287
|
+
var MicdropStorageKeys = /* @__PURE__ */ ((MicdropStorageKeys2) => {
|
|
288
|
+
MicdropStorageKeys2["MicDevice"] = "micdrop.micDevice";
|
|
289
|
+
MicdropStorageKeys2["SpeakerDevice"] = "micdrop.speakerDevice";
|
|
290
|
+
MicdropStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
|
|
291
|
+
MicdropStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
|
|
292
|
+
return MicdropStorageKeys2;
|
|
293
|
+
})(MicdropStorageKeys || {});
|
|
294
|
+
var MemoryStorage = class {
|
|
182
295
|
constructor() {
|
|
183
|
-
|
|
184
|
-
this.blobQueue = [];
|
|
185
|
-
this.processing = false;
|
|
186
|
-
this.sourceNodes = [];
|
|
187
|
-
this.nextStartTime = 0;
|
|
188
|
-
this.prebuffering = true;
|
|
189
|
-
this.prebuffer = [];
|
|
190
|
-
this.prebufferDuration = 0;
|
|
296
|
+
this.values = /* @__PURE__ */ new Map();
|
|
191
297
|
}
|
|
192
|
-
|
|
193
|
-
return this.
|
|
298
|
+
getItem(key) {
|
|
299
|
+
return this.values.get(key) ?? null;
|
|
194
300
|
}
|
|
195
|
-
|
|
196
|
-
this.
|
|
197
|
-
if (this.processing) return;
|
|
198
|
-
this.processing = true;
|
|
199
|
-
try {
|
|
200
|
-
while (this.blobQueue.length > 0) {
|
|
201
|
-
const next = this.blobQueue.shift();
|
|
202
|
-
try {
|
|
203
|
-
const arrayBuffer = await next.arrayBuffer();
|
|
204
|
-
const audioBuffer = this.createAudioBuffer(arrayBuffer);
|
|
205
|
-
if (audioBuffer) this.handleBuffer(audioBuffer);
|
|
206
|
-
} catch (error) {
|
|
207
|
-
console.error("Failed to play audio chunk:", error);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
} finally {
|
|
211
|
-
this.processing = false;
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
stopAudio() {
|
|
215
|
-
for (const node of this.sourceNodes) {
|
|
216
|
-
try {
|
|
217
|
-
node.stop();
|
|
218
|
-
node.disconnect();
|
|
219
|
-
} catch (error) {
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
this.sourceNodes = [];
|
|
223
|
-
this.blobQueue = [];
|
|
224
|
-
this.nextStartTime = 0;
|
|
225
|
-
this.resetPrebuffer();
|
|
226
|
-
this.setIsPlaying(false);
|
|
301
|
+
setItem(key, value) {
|
|
302
|
+
this.values.set(key, value);
|
|
227
303
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
handleBuffer(audioBuffer) {
|
|
231
|
-
if (!this.prebuffering) {
|
|
232
|
-
this.scheduleBuffer(audioBuffer);
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
this.prebuffer.push(audioBuffer);
|
|
236
|
-
this.prebufferDuration += audioBuffer.duration * 1e3;
|
|
237
|
-
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
238
|
-
this.flushPrebuffer();
|
|
239
|
-
return;
|
|
240
|
-
}
|
|
241
|
-
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
242
|
-
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
304
|
+
removeItem(key) {
|
|
305
|
+
this.values.delete(key);
|
|
243
306
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
307
|
+
};
|
|
308
|
+
var current = new MemoryStorage();
|
|
309
|
+
function setMicdropStorage(storage2) {
|
|
310
|
+
current = storage2;
|
|
311
|
+
}
|
|
312
|
+
var storage = {
|
|
313
|
+
getItem: (key) => {
|
|
314
|
+
try {
|
|
315
|
+
return current.getItem(key);
|
|
316
|
+
} catch {
|
|
317
|
+
return null;
|
|
248
318
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
}
|
|
255
|
-
resetPrebuffer() {
|
|
256
|
-
if (this.quietTimer !== void 0) {
|
|
257
|
-
clearTimeout(this.quietTimer);
|
|
258
|
-
this.quietTimer = void 0;
|
|
319
|
+
},
|
|
320
|
+
setItem: (key, value) => {
|
|
321
|
+
try {
|
|
322
|
+
current.setItem(key, value);
|
|
323
|
+
} catch {
|
|
259
324
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
this.prebufferDuration = 0;
|
|
263
|
-
}
|
|
264
|
-
createAudioBuffer(arrayBuffer) {
|
|
325
|
+
},
|
|
326
|
+
removeItem: (key) => {
|
|
265
327
|
try {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
if (numSamples === 0) {
|
|
269
|
-
return null;
|
|
270
|
-
}
|
|
271
|
-
const audioBuffer = audioContext.createBuffer(
|
|
272
|
-
CHANNELS,
|
|
273
|
-
numSamples,
|
|
274
|
-
SAMPLE_RATE
|
|
275
|
-
);
|
|
276
|
-
const channelData = audioBuffer.getChannelData(0);
|
|
277
|
-
for (let i = 0; i < numSamples; i++) {
|
|
278
|
-
channelData[i] = int16Data[i] / 32768;
|
|
279
|
-
}
|
|
280
|
-
return audioBuffer;
|
|
281
|
-
} catch (error) {
|
|
282
|
-
console.error("Error creating AudioBuffer:", error);
|
|
283
|
-
return null;
|
|
328
|
+
current.removeItem(key);
|
|
329
|
+
} catch {
|
|
284
330
|
}
|
|
285
331
|
}
|
|
286
|
-
scheduleBuffer(audioBuffer) {
|
|
287
|
-
const sourceNode = audioContext.createBufferSource();
|
|
288
|
-
sourceNode.buffer = audioBuffer;
|
|
289
|
-
sourceNode.connect(this.outputNode);
|
|
290
|
-
const now = audioContext.currentTime;
|
|
291
|
-
const startTime = Math.max(this.nextStartTime, now);
|
|
292
|
-
this.nextStartTime = startTime + audioBuffer.duration;
|
|
293
|
-
sourceNode.onended = () => {
|
|
294
|
-
const index = this.sourceNodes.indexOf(sourceNode);
|
|
295
|
-
if (index !== -1) this.sourceNodes.splice(index, 1);
|
|
296
|
-
if (this.sourceNodes.length === 0 && this.blobQueue.length === 0) {
|
|
297
|
-
this.nextStartTime = 0;
|
|
298
|
-
this.resetPrebuffer();
|
|
299
|
-
this.setIsPlaying(false);
|
|
300
|
-
}
|
|
301
|
-
};
|
|
302
|
-
this.sourceNodes.push(sourceNode);
|
|
303
|
-
sourceNode.start(startTime);
|
|
304
|
-
this.setIsPlaying(true);
|
|
305
|
-
}
|
|
306
332
|
};
|
|
307
333
|
|
|
308
334
|
// src/audio/Speaker.ts
|
|
309
|
-
var
|
|
335
|
+
var SpeakerController = class extends EventEmitter2 {
|
|
310
336
|
constructor() {
|
|
311
337
|
super(...arguments);
|
|
312
|
-
|
|
313
|
-
this.
|
|
314
|
-
|
|
315
|
-
this.
|
|
338
|
+
/** Last level measured, in decibels, between -Infinity and 0 */
|
|
339
|
+
this.volume = -Infinity;
|
|
340
|
+
this.onStartPlaying = () => this.emit("StartPlaying");
|
|
341
|
+
this.onStopPlaying = () => {
|
|
342
|
+
this.volume = -Infinity;
|
|
343
|
+
this.emit("StopPlaying");
|
|
344
|
+
};
|
|
345
|
+
this.onVolume = (volume) => {
|
|
346
|
+
this.volume = volume;
|
|
347
|
+
this.emit("Volume", volume);
|
|
348
|
+
};
|
|
316
349
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
this.
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
async playAudio(blob) {
|
|
328
|
-
this.start();
|
|
329
|
-
try {
|
|
330
|
-
await this.audioStream?.playAudio(blob);
|
|
331
|
-
} catch (error) {
|
|
332
|
-
console.error("Error playing audio blob", error, blob);
|
|
350
|
+
/**
|
|
351
|
+
* Replaces the audio driver, to play sound with another library
|
|
352
|
+
* @param driver - The driver to play with
|
|
353
|
+
*/
|
|
354
|
+
setDriver(driver) {
|
|
355
|
+
if (this.driver === driver) return;
|
|
356
|
+
if (this.driver) {
|
|
357
|
+
this.driver.off("StartPlaying", this.onStartPlaying);
|
|
358
|
+
this.driver.off("StopPlaying", this.onStopPlaying);
|
|
359
|
+
this.driver.off("Volume", this.onVolume);
|
|
333
360
|
}
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
this.
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
361
|
+
this.driver = driver;
|
|
362
|
+
driver.on("StartPlaying", this.onStartPlaying);
|
|
363
|
+
driver.on("StopPlaying", this.onStopPlaying);
|
|
364
|
+
driver.on("Volume", this.onVolume);
|
|
365
|
+
}
|
|
366
|
+
getDriver() {
|
|
367
|
+
if (!this.driver) {
|
|
368
|
+
throw new Error(
|
|
369
|
+
"No speaker driver. Import @micdrop/web in a browser or @micdrop/react-native on a phone, or call Speaker.setDriver()."
|
|
370
|
+
);
|
|
345
371
|
}
|
|
372
|
+
return this.driver;
|
|
346
373
|
}
|
|
347
|
-
get
|
|
348
|
-
return
|
|
374
|
+
get isPlaying() {
|
|
375
|
+
return this.driver?.isPlaying ?? false;
|
|
349
376
|
}
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
this.isPlaying = isPlaying;
|
|
353
|
-
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
377
|
+
get deviceId() {
|
|
378
|
+
return this.driver?.deviceId;
|
|
354
379
|
}
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
380
|
+
/** Prepares the audio graph, called before the first chunk arrives */
|
|
381
|
+
async start() {
|
|
382
|
+
const driver = this.getDriver();
|
|
383
|
+
await driver.start();
|
|
384
|
+
const saved = storage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
385
|
+
if (saved && saved !== driver.deviceId) {
|
|
386
|
+
try {
|
|
387
|
+
await driver.changeDevice(saved);
|
|
388
|
+
} catch {
|
|
389
|
+
storage.removeItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
390
|
+
}
|
|
361
391
|
}
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Queues samples for playback, right after the ones already queued
|
|
395
|
+
* @param pcm - Mono samples as 16 bits integers, or the bytes holding them
|
|
396
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
397
|
+
*/
|
|
398
|
+
playAudio(pcm, sampleRate = 16e3) {
|
|
362
399
|
try {
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
const sinkId = !deviceId || defaultDevice ? "" : deviceId;
|
|
366
|
-
if (defaultDevice || "setSinkId" in audioContext) {
|
|
367
|
-
await audioContext.setSinkId?.(sinkId);
|
|
368
|
-
this.audioSourceNode.disconnect();
|
|
369
|
-
this.audioSourceNode.connect(audioContext.destination);
|
|
370
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
371
|
-
if (this.audioElementDestination) {
|
|
372
|
-
this.audioElementDestination.pause();
|
|
373
|
-
this.audioElementDestination.srcObject = null;
|
|
374
|
-
this.audioElementDestination = void 0;
|
|
375
|
-
}
|
|
376
|
-
} else {
|
|
377
|
-
if (!this.audioElementDestination) {
|
|
378
|
-
const mediaStreamDestination = audioContext.createMediaStreamDestination();
|
|
379
|
-
this.audioSourceNode.disconnect();
|
|
380
|
-
this.audioSourceNode.connect(mediaStreamDestination);
|
|
381
|
-
this.audioSourceNode.connect(this.analyser.node);
|
|
382
|
-
this.audioElementDestination = new Audio();
|
|
383
|
-
this.audioElementDestination.autoplay = true;
|
|
384
|
-
this.audioElementDestination.srcObject = mediaStreamDestination.stream;
|
|
385
|
-
}
|
|
386
|
-
await this.audioElementDestination.setSinkId(sinkId);
|
|
387
|
-
}
|
|
400
|
+
const samples = pcm instanceof Int16Array ? pcm : new Int16Array(pcm);
|
|
401
|
+
this.getDriver().play(samples, sampleRate);
|
|
388
402
|
} catch (error) {
|
|
389
|
-
console.error(
|
|
403
|
+
console.error("[Micdrop] Error playing audio", error);
|
|
390
404
|
}
|
|
391
405
|
}
|
|
406
|
+
/** Drops everything queued and stops playing immediately */
|
|
407
|
+
stopAudio() {
|
|
408
|
+
this.driver?.stopAudio();
|
|
409
|
+
}
|
|
410
|
+
/** Releases the audio graph */
|
|
411
|
+
async stop() {
|
|
412
|
+
await this.driver?.stop();
|
|
413
|
+
this.volume = -Infinity;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Plays through another output
|
|
417
|
+
* @param deviceId - Output device, or route on a phone
|
|
418
|
+
*/
|
|
419
|
+
async changeDevice(deviceId) {
|
|
420
|
+
await this.getDriver().changeDevice(deviceId);
|
|
421
|
+
storage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, deviceId);
|
|
422
|
+
}
|
|
423
|
+
/** Lists the output devices the system offers */
|
|
424
|
+
async getDevices() {
|
|
425
|
+
if (!this.driver) return [];
|
|
426
|
+
return this.driver.getDevices();
|
|
427
|
+
}
|
|
392
428
|
};
|
|
393
429
|
|
|
394
430
|
// src/audio/MicRecorder.ts
|
|
395
|
-
import { EventEmitter as
|
|
396
|
-
|
|
397
|
-
// src/audio/utils/delayedStream.ts
|
|
398
|
-
function createDelayedStream(stream, delay) {
|
|
399
|
-
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
400
|
-
const delayNode = audioContext.createDelay(delay);
|
|
401
|
-
delayNode.delayTime.value = delay;
|
|
402
|
-
const audioDestination = audioContext.createMediaStreamDestination();
|
|
403
|
-
audioSource.connect(delayNode);
|
|
404
|
-
delayNode.connect(audioDestination);
|
|
405
|
-
return audioDestination.stream;
|
|
406
|
-
}
|
|
431
|
+
import { EventEmitter as EventEmitter4 } from "eventemitter3";
|
|
407
432
|
|
|
408
433
|
// src/audio/vad/VAD.ts
|
|
409
|
-
import { EventEmitter as
|
|
434
|
+
import { EventEmitter as EventEmitter3 } from "eventemitter3";
|
|
410
435
|
var VADStatus = /* @__PURE__ */ ((VADStatus2) => {
|
|
411
436
|
VADStatus2[VADStatus2["Silence"] = 0] = "Silence";
|
|
412
437
|
VADStatus2[VADStatus2["MaybeSpeaking"] = 1] = "MaybeSpeaking";
|
|
413
438
|
VADStatus2[VADStatus2["Speaking"] = 2] = "Speaking";
|
|
414
439
|
return VADStatus2;
|
|
415
440
|
})(VADStatus || {});
|
|
416
|
-
var VAD = class extends
|
|
441
|
+
var VAD = class extends EventEmitter3 {
|
|
417
442
|
constructor() {
|
|
418
443
|
super(...arguments);
|
|
419
444
|
this.name = this.constructor.name;
|
|
420
445
|
this.status = 0 /* Silence */;
|
|
421
|
-
//
|
|
446
|
+
// Worst case time this VAD takes to notice that someone started speaking.
|
|
447
|
+
// That much audio is kept in reserve and sent along with the turn, so the
|
|
448
|
+
// beginning of a sentence reaches the server rather than being cut off.
|
|
449
|
+
// A VAD that needs several samples to make up its mind has to say so here.
|
|
422
450
|
this.delay = 100;
|
|
423
451
|
}
|
|
424
452
|
/**
|
|
@@ -501,13 +529,13 @@ var MultipleVAD = class extends VAD {
|
|
|
501
529
|
get isPaused() {
|
|
502
530
|
return this.vads.every((vad) => vad.isPaused);
|
|
503
531
|
}
|
|
504
|
-
async start(
|
|
532
|
+
async start(mic) {
|
|
505
533
|
for (const vad of this.vads) {
|
|
506
534
|
if (vad.isStarted) {
|
|
507
535
|
continue;
|
|
508
536
|
}
|
|
509
537
|
vad.on("ChangeStatus", this.onStatusChange);
|
|
510
|
-
await vad.start(
|
|
538
|
+
await vad.start(mic);
|
|
511
539
|
}
|
|
512
540
|
}
|
|
513
541
|
async stop() {
|
|
@@ -529,87 +557,96 @@ var MultipleVAD = class extends VAD {
|
|
|
529
557
|
};
|
|
530
558
|
|
|
531
559
|
// src/audio/vad/SileroVAD.ts
|
|
532
|
-
|
|
560
|
+
var SILERO_SAMPLE_RATE = 16e3;
|
|
561
|
+
var SILERO_FRAME_SAMPLES = 512;
|
|
562
|
+
var MS_PER_FRAME = SILERO_FRAME_SAMPLES / SILERO_SAMPLE_RATE * 1e3;
|
|
533
563
|
var defaultOptions = {
|
|
534
564
|
positiveSpeechThreshold: 0.18,
|
|
535
565
|
negativeSpeechThreshold: 0.11,
|
|
536
566
|
minSpeechFrames: 8,
|
|
537
567
|
redemptionFrames: 20
|
|
538
568
|
};
|
|
569
|
+
var loadModel;
|
|
570
|
+
function setSileroModelLoader(loader) {
|
|
571
|
+
loadModel = loader;
|
|
572
|
+
}
|
|
539
573
|
var SileroVAD = class extends VAD {
|
|
540
574
|
constructor(options) {
|
|
541
575
|
super();
|
|
542
576
|
this.name = "SileroVAD";
|
|
543
577
|
this.options = defaultOptions;
|
|
578
|
+
// The model decides on the first window it hears speech in, so it is quick.
|
|
579
|
+
// The reserve still covers a few windows, since scoring runs asynchronously
|
|
580
|
+
// and may lag behind the audio on a busy device.
|
|
581
|
+
this.delay = 3 * MS_PER_FRAME;
|
|
544
582
|
this._isPaused = false;
|
|
583
|
+
this.buffer = [];
|
|
584
|
+
this.bufferLength = 0;
|
|
585
|
+
this.bufferRate = SILERO_SAMPLE_RATE;
|
|
586
|
+
this.speaking = false;
|
|
587
|
+
this.speechFrames = 0;
|
|
588
|
+
this.silenceFrames = 0;
|
|
589
|
+
this.processing = false;
|
|
590
|
+
this.onFrames = (frames, sampleRate) => {
|
|
591
|
+
if (this._isPaused || !this.model) return;
|
|
592
|
+
if (sampleRate !== this.bufferRate) {
|
|
593
|
+
this.buffer = [];
|
|
594
|
+
this.bufferLength = 0;
|
|
595
|
+
this.bufferRate = sampleRate;
|
|
596
|
+
}
|
|
597
|
+
this.buffer.push(frames);
|
|
598
|
+
this.bufferLength += frames.length;
|
|
599
|
+
this.drain();
|
|
600
|
+
};
|
|
545
601
|
if (options) {
|
|
546
602
|
this.setOptions(options);
|
|
547
603
|
} else {
|
|
548
|
-
const
|
|
549
|
-
"micdrop.SileroVADOptions" /* SileroVADOptions */
|
|
550
|
-
);
|
|
604
|
+
const saved = storage.getItem("micdrop.SileroVADOptions" /* SileroVADOptions */);
|
|
551
605
|
try {
|
|
552
|
-
this.setOptions(
|
|
553
|
-
} catch
|
|
606
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
607
|
+
} catch {
|
|
554
608
|
this.setOptions({});
|
|
555
609
|
}
|
|
556
610
|
}
|
|
557
611
|
}
|
|
558
612
|
get isStarted() {
|
|
559
|
-
return !!this.
|
|
613
|
+
return !!this.mic;
|
|
560
614
|
}
|
|
561
615
|
get isPaused() {
|
|
562
616
|
return this._isPaused;
|
|
563
617
|
}
|
|
564
|
-
async start(
|
|
565
|
-
if (this.
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
// CDN size limit, so every file 404s). unpkg serves it, so the ONNX wasm
|
|
576
|
-
// backend is loaded from there instead.
|
|
577
|
-
onnxWASMBasePath: "https://unpkg.com/onnxruntime-web@1.23.2/dist/",
|
|
578
|
-
onSpeechStart: () => {
|
|
579
|
-
this.emit("StartSpeaking");
|
|
580
|
-
},
|
|
581
|
-
onSpeechRealStart: () => {
|
|
582
|
-
this.emit("ConfirmSpeaking");
|
|
583
|
-
},
|
|
584
|
-
onVADMisfire: () => {
|
|
585
|
-
this.emit("CancelSpeaking");
|
|
586
|
-
},
|
|
587
|
-
onSpeechEnd: () => {
|
|
588
|
-
this.emit("StopSpeaking");
|
|
589
|
-
},
|
|
590
|
-
...this.options
|
|
591
|
-
});
|
|
592
|
-
this.vad.start();
|
|
618
|
+
async start(mic) {
|
|
619
|
+
if (this.mic) return;
|
|
620
|
+
if (!loadModel) {
|
|
621
|
+
throw new Error(
|
|
622
|
+
"No Silero model. Add `import '@micdrop/web/silero'` in a browser or `import '@micdrop/react-native/silero'` on a phone, or call setSileroModelLoader()."
|
|
623
|
+
);
|
|
624
|
+
}
|
|
625
|
+
this.model = await loadModel();
|
|
626
|
+
this.resetState();
|
|
627
|
+
this.mic = mic;
|
|
628
|
+
mic.on("Frames", this.onFrames);
|
|
593
629
|
}
|
|
594
630
|
async stop() {
|
|
595
|
-
|
|
596
|
-
this.
|
|
597
|
-
this.
|
|
631
|
+
this.mic?.off("Frames", this.onFrames);
|
|
632
|
+
this.mic = void 0;
|
|
633
|
+
this.closeOpenTurn();
|
|
634
|
+
this.resetState();
|
|
635
|
+
await this.model?.release();
|
|
636
|
+
this.model = void 0;
|
|
598
637
|
}
|
|
599
638
|
async pause() {
|
|
600
|
-
if (
|
|
639
|
+
if (this._isPaused) return;
|
|
601
640
|
this._isPaused = true;
|
|
602
|
-
this.
|
|
641
|
+
this.closeOpenTurn();
|
|
642
|
+
this.resetState();
|
|
603
643
|
}
|
|
604
644
|
async resume() {
|
|
605
|
-
if (!this.vad || !this._isPaused) return;
|
|
606
645
|
this._isPaused = false;
|
|
607
|
-
this.vad.start();
|
|
608
646
|
}
|
|
609
647
|
setOptions(options) {
|
|
610
648
|
this.options = { ...this.options, ...options };
|
|
611
|
-
|
|
612
|
-
localStorage.setItem(
|
|
649
|
+
storage.setItem(
|
|
613
650
|
"micdrop.SileroVADOptions" /* SileroVADOptions */,
|
|
614
651
|
JSON.stringify(this.options)
|
|
615
652
|
);
|
|
@@ -617,7 +654,79 @@ var SileroVAD = class extends VAD {
|
|
|
617
654
|
resetOptions() {
|
|
618
655
|
this.setOptions(defaultOptions);
|
|
619
656
|
}
|
|
657
|
+
// The model is asynchronous, so windows are scored one after the other rather
|
|
658
|
+
// than raced: its answer for a window depends on the ones before it.
|
|
659
|
+
async drain() {
|
|
660
|
+
if (this.processing) return;
|
|
661
|
+
this.processing = true;
|
|
662
|
+
try {
|
|
663
|
+
const needed = Math.ceil(
|
|
664
|
+
SILERO_FRAME_SAMPLES * this.bufferRate / SILERO_SAMPLE_RATE
|
|
665
|
+
);
|
|
666
|
+
while (this.bufferLength >= needed && this.model && !this._isPaused) {
|
|
667
|
+
const gathered = concatFloat32(this.buffer);
|
|
668
|
+
const window = gathered.subarray(0, needed);
|
|
669
|
+
const rest = gathered.subarray(needed);
|
|
670
|
+
this.buffer = rest.length ? [rest] : [];
|
|
671
|
+
this.bufferLength = rest.length;
|
|
672
|
+
const frame = resample(window, this.bufferRate, SILERO_SAMPLE_RATE);
|
|
673
|
+
const probability = await this.model.process(
|
|
674
|
+
frame.length === SILERO_FRAME_SAMPLES ? frame : fit(frame, SILERO_FRAME_SAMPLES)
|
|
675
|
+
);
|
|
676
|
+
this.onProbability(probability);
|
|
677
|
+
}
|
|
678
|
+
} catch (error) {
|
|
679
|
+
console.error("[Micdrop] Silero VAD failed", error);
|
|
680
|
+
} finally {
|
|
681
|
+
this.processing = false;
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
// Ported from the reference implementation of the algorithm
|
|
685
|
+
// https://docs.vad.ricky0123.com/user-guide/algorithm/
|
|
686
|
+
onProbability(probability) {
|
|
687
|
+
const { positiveSpeechThreshold, negativeSpeechThreshold } = this.options;
|
|
688
|
+
if (probability >= positiveSpeechThreshold) {
|
|
689
|
+
this.silenceFrames = 0;
|
|
690
|
+
this.speechFrames++;
|
|
691
|
+
if (!this.speaking) {
|
|
692
|
+
this.speaking = true;
|
|
693
|
+
this.emit("StartSpeaking");
|
|
694
|
+
}
|
|
695
|
+
if (this.speechFrames === this.options.minSpeechFrames) {
|
|
696
|
+
this.emit("ConfirmSpeaking");
|
|
697
|
+
}
|
|
698
|
+
return;
|
|
699
|
+
}
|
|
700
|
+
if (probability < negativeSpeechThreshold && this.speaking) {
|
|
701
|
+
this.silenceFrames++;
|
|
702
|
+
if (this.silenceFrames < this.options.redemptionFrames) return;
|
|
703
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
704
|
+
this.speaking = false;
|
|
705
|
+
this.speechFrames = 0;
|
|
706
|
+
this.silenceFrames = 0;
|
|
707
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
closeOpenTurn() {
|
|
711
|
+
if (!this.speaking) return;
|
|
712
|
+
const confirmed = this.speechFrames >= this.options.minSpeechFrames;
|
|
713
|
+
this.emit(confirmed ? "StopSpeaking" : "CancelSpeaking");
|
|
714
|
+
}
|
|
715
|
+
resetState() {
|
|
716
|
+
this.model?.reset();
|
|
717
|
+
this.buffer = [];
|
|
718
|
+
this.bufferLength = 0;
|
|
719
|
+
this.speaking = false;
|
|
720
|
+
this.speechFrames = 0;
|
|
721
|
+
this.silenceFrames = 0;
|
|
722
|
+
}
|
|
620
723
|
};
|
|
724
|
+
function fit(frame, length) {
|
|
725
|
+
if (frame.length === length) return frame;
|
|
726
|
+
const fitted = new Float32Array(length);
|
|
727
|
+
fitted.set(frame.subarray(0, length));
|
|
728
|
+
return fitted;
|
|
729
|
+
}
|
|
621
730
|
|
|
622
731
|
// src/audio/vad/VolumeVAD.ts
|
|
623
732
|
var defaultOptions2 = {
|
|
@@ -629,138 +738,92 @@ var VolumeVAD = class extends VAD {
|
|
|
629
738
|
super();
|
|
630
739
|
this.name = "VolumeVAD";
|
|
631
740
|
this.options = defaultOptions2;
|
|
632
|
-
|
|
741
|
+
// The microphone reports a level about every 100 ms, and two consecutive loud
|
|
742
|
+
// ones are needed before a turn opens. Speech can therefore begin up to two
|
|
743
|
+
// reports before it is noticed, and a third is kept as a margin: without that
|
|
744
|
+
// reserve the first syllable of a sentence never reaches the server.
|
|
745
|
+
this.delay = 300;
|
|
633
746
|
this.running = false;
|
|
747
|
+
this._isPaused = false;
|
|
634
748
|
this.speaking = false;
|
|
635
749
|
this.attemptSpeaking = false;
|
|
636
750
|
this.speakingHistory = [];
|
|
751
|
+
this.onVolume = (volume) => {
|
|
752
|
+
if (!this.running) return;
|
|
753
|
+
const isLoud = volume > this.options.threshold;
|
|
754
|
+
if (isLoud) {
|
|
755
|
+
if (!this.speaking) {
|
|
756
|
+
let recentHistory = 0;
|
|
757
|
+
for (let i = Math.max(0, this.speakingHistory.length - 3); i < this.speakingHistory.length; i++) {
|
|
758
|
+
recentHistory += this.speakingHistory[i];
|
|
759
|
+
}
|
|
760
|
+
if (recentHistory >= 2) {
|
|
761
|
+
this.speaking = true;
|
|
762
|
+
this.attemptSpeaking = false;
|
|
763
|
+
this.emit("ConfirmSpeaking");
|
|
764
|
+
} else if (recentHistory === 1 && !this.attemptSpeaking) {
|
|
765
|
+
this.attemptSpeaking = true;
|
|
766
|
+
this.emit("StartSpeaking");
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
} else if (this.speaking) {
|
|
770
|
+
let totalHistory = 0;
|
|
771
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
772
|
+
totalHistory += this.speakingHistory[i];
|
|
773
|
+
}
|
|
774
|
+
if (totalHistory === 0) {
|
|
775
|
+
this.speaking = false;
|
|
776
|
+
this.emit("StopSpeaking");
|
|
777
|
+
}
|
|
778
|
+
} else if (this.attemptSpeaking) {
|
|
779
|
+
const lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
780
|
+
if (lastHistory === 0) {
|
|
781
|
+
this.attemptSpeaking = false;
|
|
782
|
+
this.emit("CancelSpeaking");
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
this.speakingHistory.shift();
|
|
786
|
+
this.speakingHistory.push(Number(isLoud));
|
|
787
|
+
};
|
|
637
788
|
if (options) {
|
|
638
789
|
this.setOptions(options);
|
|
639
790
|
} else {
|
|
640
|
-
const
|
|
641
|
-
"micdrop.VolumeVADOptions" /* VolumeVADOptions */
|
|
642
|
-
);
|
|
791
|
+
const saved = storage.getItem("micdrop.VolumeVADOptions" /* VolumeVADOptions */);
|
|
643
792
|
try {
|
|
644
|
-
this.setOptions(
|
|
645
|
-
} catch
|
|
793
|
+
this.setOptions(saved ? JSON.parse(saved) : {});
|
|
794
|
+
} catch {
|
|
646
795
|
this.setOptions({});
|
|
647
796
|
}
|
|
648
797
|
}
|
|
649
798
|
}
|
|
650
|
-
getMaxVolume() {
|
|
651
|
-
if (!this.analyser || !this.fftBins) return -Infinity;
|
|
652
|
-
let maxVolume = -Infinity;
|
|
653
|
-
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
654
|
-
for (let i = 4; i < this.fftBins.length; i++) {
|
|
655
|
-
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
656
|
-
maxVolume = this.fftBins[i];
|
|
657
|
-
}
|
|
658
|
-
}
|
|
659
|
-
return maxVolume;
|
|
660
|
-
}
|
|
661
799
|
get isStarted() {
|
|
662
800
|
return this.running;
|
|
663
801
|
}
|
|
664
802
|
get isPaused() {
|
|
665
803
|
return this._isPaused;
|
|
666
804
|
}
|
|
667
|
-
async start(
|
|
668
|
-
if (this.
|
|
669
|
-
this.
|
|
670
|
-
|
|
671
|
-
this.
|
|
672
|
-
this.analyser.smoothingTimeConstant = 0.1;
|
|
673
|
-
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
674
|
-
this.sourceNode.connect(this.analyser);
|
|
675
|
-
this.running = true;
|
|
676
|
-
this.startLoop();
|
|
677
|
-
}
|
|
678
|
-
startLoop() {
|
|
679
|
-
if (!this.running) return;
|
|
680
|
-
const currentVolume = this.getMaxVolume();
|
|
681
|
-
if (currentVolume > this.options.threshold) {
|
|
682
|
-
if (!this.speaking) {
|
|
683
|
-
let recentHistory = 0;
|
|
684
|
-
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
685
|
-
recentHistory += this.speakingHistory[i];
|
|
686
|
-
}
|
|
687
|
-
if (recentHistory >= 2) {
|
|
688
|
-
this.speaking = true;
|
|
689
|
-
this.attemptSpeaking = false;
|
|
690
|
-
this.emit("ConfirmSpeaking");
|
|
691
|
-
} else if (recentHistory === 1) {
|
|
692
|
-
this.attemptSpeaking = true;
|
|
693
|
-
this.emit("StartSpeaking");
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
} else if (this.speaking) {
|
|
697
|
-
let totalHistory = 0;
|
|
698
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
699
|
-
totalHistory += this.speakingHistory[i];
|
|
700
|
-
}
|
|
701
|
-
if (totalHistory === 0) {
|
|
702
|
-
this.speaking = false;
|
|
703
|
-
this.emit("StopSpeaking");
|
|
704
|
-
}
|
|
705
|
-
} else if (this.attemptSpeaking) {
|
|
706
|
-
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
707
|
-
if (lastHistory === 0) {
|
|
708
|
-
this.attemptSpeaking = false;
|
|
709
|
-
this.emit("CancelSpeaking");
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
this.speakingHistory.shift();
|
|
713
|
-
this.speakingHistory.push(Number(currentVolume > this.options.threshold));
|
|
714
|
-
setTimeout(() => this.startLoop(), this.delay);
|
|
715
|
-
}
|
|
716
|
-
resetHistory() {
|
|
717
|
-
this.speaking = false;
|
|
718
|
-
this.attemptSpeaking = false;
|
|
719
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
720
|
-
this.speakingHistory[i] = 0;
|
|
721
|
-
}
|
|
805
|
+
async start(mic) {
|
|
806
|
+
if (this.mic) return;
|
|
807
|
+
this.mic = mic;
|
|
808
|
+
mic.on("Volume", this.onVolume);
|
|
809
|
+
this.running = !this._isPaused;
|
|
722
810
|
}
|
|
723
811
|
async stop() {
|
|
724
|
-
|
|
812
|
+
this.mic?.off("Volume", this.onVolume);
|
|
813
|
+
this.mic = void 0;
|
|
725
814
|
this.running = false;
|
|
726
|
-
this.
|
|
727
|
-
switch (this.status) {
|
|
728
|
-
case 2 /* Speaking */:
|
|
729
|
-
this.emit("StopSpeaking");
|
|
730
|
-
break;
|
|
731
|
-
case 1 /* MaybeSpeaking */:
|
|
732
|
-
this.emit("CancelSpeaking");
|
|
733
|
-
break;
|
|
734
|
-
}
|
|
735
|
-
if (this.sourceNode) {
|
|
736
|
-
this.sourceNode.disconnect();
|
|
737
|
-
this.sourceNode = void 0;
|
|
738
|
-
}
|
|
739
|
-
if (this.analyser) {
|
|
740
|
-
this.analyser.disconnect();
|
|
741
|
-
this.analyser = void 0;
|
|
742
|
-
}
|
|
743
|
-
this.fftBins = void 0;
|
|
815
|
+
this.reset();
|
|
744
816
|
}
|
|
745
817
|
async pause() {
|
|
746
|
-
if (
|
|
818
|
+
if (this._isPaused) return;
|
|
747
819
|
this._isPaused = true;
|
|
748
820
|
this.running = false;
|
|
749
|
-
this.
|
|
750
|
-
switch (this.status) {
|
|
751
|
-
case 2 /* Speaking */:
|
|
752
|
-
this.emit("StopSpeaking");
|
|
753
|
-
break;
|
|
754
|
-
case 1 /* MaybeSpeaking */:
|
|
755
|
-
this.emit("CancelSpeaking");
|
|
756
|
-
break;
|
|
757
|
-
}
|
|
821
|
+
this.reset();
|
|
758
822
|
}
|
|
759
823
|
async resume() {
|
|
760
824
|
if (!this._isPaused) return;
|
|
761
825
|
this._isPaused = false;
|
|
762
|
-
this.running =
|
|
763
|
-
this.startLoop();
|
|
826
|
+
this.running = !!this.mic;
|
|
764
827
|
}
|
|
765
828
|
setOptions(options) {
|
|
766
829
|
if (typeof options.history === "number" && options.history < 3) {
|
|
@@ -773,7 +836,7 @@ var VolumeVAD = class extends VAD {
|
|
|
773
836
|
while (this.speakingHistory.length < this.options.history) {
|
|
774
837
|
this.speakingHistory.push(0);
|
|
775
838
|
}
|
|
776
|
-
|
|
839
|
+
storage.setItem(
|
|
777
840
|
"micdrop.VolumeVADOptions" /* VolumeVADOptions */,
|
|
778
841
|
JSON.stringify(this.options)
|
|
779
842
|
);
|
|
@@ -781,6 +844,22 @@ var VolumeVAD = class extends VAD {
|
|
|
781
844
|
resetOptions() {
|
|
782
845
|
this.setOptions(defaultOptions2);
|
|
783
846
|
}
|
|
847
|
+
reset() {
|
|
848
|
+
const status = this.status;
|
|
849
|
+
this.speaking = false;
|
|
850
|
+
this.attemptSpeaking = false;
|
|
851
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
852
|
+
this.speakingHistory[i] = 0;
|
|
853
|
+
}
|
|
854
|
+
switch (status) {
|
|
855
|
+
case 2 /* Speaking */:
|
|
856
|
+
this.emit("StopSpeaking");
|
|
857
|
+
break;
|
|
858
|
+
case 1 /* MaybeSpeaking */:
|
|
859
|
+
this.emit("CancelSpeaking");
|
|
860
|
+
break;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
784
863
|
};
|
|
785
864
|
|
|
786
865
|
// src/audio/vad/getVAD.ts
|
|
@@ -802,34 +881,27 @@ function equalVADConfig(a, b) {
|
|
|
802
881
|
return false;
|
|
803
882
|
}
|
|
804
883
|
|
|
805
|
-
// raw:./pcm-processor-worklet
|
|
806
|
-
var pcm_processor_worklet_default = "const INTERVAL_MS = 100\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.audioBuffer = []\n this.isRecording = false\n this.sourceSampleRate = 16000\n this.targetSampleRate = 16000\n this.resamplingRatio = 1\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n\n // Listen for messages\n this.port.onmessage = this.handleMessage.bind(this)\n }\n\n handleMessage(event) {\n if (event.data.type === 'configure') {\n this.sourceSampleRate = event.data.sourceSampleRate\n this.targetSampleRate = event.data.targetSampleRate\n this.resamplingRatio = this.sourceSampleRate / this.targetSampleRate\n // Recalculate chunk interval for the source sample rate\n this.CHUNK_INTERVAL_SAMPLES = Math.round(\n (this.sourceSampleRate * INTERVAL_MS) / 1000\n )\n } else if (event.data.type === 'start') {\n this.isRecording = true\n } else if (event.data.type === 'stop') {\n this.isRecording = false\n // Clear any remaining audio buffer when stopping\n this.audioBuffer.length = 0\n }\n }\n\n process(inputs) {\n const input = inputs[0]\n\n if (input && input.length > 0 && this.isRecording) {\n const inputChannel = input[0]\n\n // Store audio data in buffer\n this.audioBuffer.push(new Float32Array(inputChannel))\n\n // Check if we have enough data for a chunk\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n\n if (totalSamples >= this.CHUNK_INTERVAL_SAMPLES) {\n this.processChunk()\n }\n }\n\n return true\n }\n\n processChunk() {\n if (this.audioBuffer.length === 0) return\n\n // Calculate total samples\n const totalSamples = this.audioBuffer.reduce(\n (sum, buffer) => sum + buffer.length,\n 0\n )\n if (totalSamples === 0) return\n\n // Merge all buffers\n const mergedBuffer = new Float32Array(totalSamples)\n let offset = 0\n for (const buffer of this.audioBuffer) {\n mergedBuffer.set(buffer, offset)\n offset += buffer.length\n }\n\n // Resample to target sample rate if necessary\n const resampledBuffer = this.resample(mergedBuffer)\n\n // Convert float32 to int16 PCM\n const pcmData = this.floatToPCM16(resampledBuffer)\n\n // Clear the buffer\n this.audioBuffer.length = 0\n\n // Send PCM chunk to main thread\n this.port.postMessage({\n type: 'chunk',\n data: pcmData,\n })\n }\n\n resample(inputBuffer) {\n // If no resampling needed, return original buffer\n if (this.resamplingRatio === 1) {\n return inputBuffer\n }\n\n // Calculate output length\n const outputLength = Math.round(inputBuffer.length / this.resamplingRatio)\n const outputBuffer = new Float32Array(outputLength)\n\n // Simple linear interpolation resampling\n for (let i = 0; i < outputLength; i++) {\n const srcIndex = i * this.resamplingRatio\n const srcIndexFloor = Math.floor(srcIndex)\n const srcIndexCeil = Math.min(srcIndexFloor + 1, inputBuffer.length - 1)\n const fraction = srcIndex - srcIndexFloor\n\n // Linear interpolation\n outputBuffer[i] =\n inputBuffer[srcIndexFloor] * (1 - fraction) +\n inputBuffer[srcIndexCeil] * fraction\n }\n\n return outputBuffer\n }\n\n floatToPCM16(floatData) {\n const pcmData = new Int16Array(floatData.length)\n for (let i = 0; i < floatData.length; i++) {\n const sample = Math.max(-1, Math.min(1, floatData[i]))\n pcmData[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff\n }\n return pcmData\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
|
|
807
|
-
|
|
808
|
-
// src/audio/pcm-processor.ts
|
|
809
|
-
async function initPcmProcessor() {
|
|
810
|
-
if (window.micdropPcmProcessor) return;
|
|
811
|
-
window.micdropPcmProcessor = true;
|
|
812
|
-
const workletBlob = new Blob([pcm_processor_worklet_default], {
|
|
813
|
-
type: "application/javascript"
|
|
814
|
-
});
|
|
815
|
-
const workletUrl = URL.createObjectURL(workletBlob);
|
|
816
|
-
await audioContext.audioWorklet.addModule(workletUrl);
|
|
817
|
-
}
|
|
818
|
-
|
|
819
884
|
// src/audio/MicRecorder.ts
|
|
820
|
-
var
|
|
885
|
+
var SAMPLE_RATE = 16e3;
|
|
886
|
+
var CHUNK_INTERVAL = 100;
|
|
821
887
|
var defaultMicRecorderState = {
|
|
822
888
|
isStarting: false,
|
|
823
889
|
isStarted: false,
|
|
824
890
|
isSpeaking: false
|
|
825
891
|
};
|
|
826
|
-
var MicRecorder = class extends
|
|
892
|
+
var MicRecorder = class extends EventEmitter4 {
|
|
827
893
|
constructor(vadConfig) {
|
|
828
894
|
super();
|
|
829
895
|
this.vadConfig = vadConfig;
|
|
896
|
+
this.isRecording = false;
|
|
830
897
|
this.speakingConfirmed = false;
|
|
898
|
+
this.sourceSampleRate = SAMPLE_RATE;
|
|
899
|
+
this.reserve = [];
|
|
900
|
+
this.reserveLength = 0;
|
|
901
|
+
this.buffer = [];
|
|
902
|
+
this.bufferLength = 0;
|
|
831
903
|
this.queuedChunks = [];
|
|
832
|
-
this.start = async (
|
|
904
|
+
this.start = async (mic) => {
|
|
833
905
|
if (this.state.isStarted) {
|
|
834
906
|
throw new Error("MicRecorder is already started");
|
|
835
907
|
}
|
|
@@ -838,21 +910,10 @@ var MicRecorder = class extends EventEmitter5 {
|
|
|
838
910
|
isStarting: true,
|
|
839
911
|
isSpeaking: false
|
|
840
912
|
});
|
|
841
|
-
this.
|
|
842
|
-
this.
|
|
843
|
-
|
|
844
|
-
this.
|
|
845
|
-
this.workletNode.port.onmessage = this.onWorkletMessage;
|
|
846
|
-
this.workletNode.port.postMessage({
|
|
847
|
-
type: "configure",
|
|
848
|
-
sourceSampleRate: audioContext.sampleRate,
|
|
849
|
-
targetSampleRate: SAMPLE_RATE2
|
|
850
|
-
});
|
|
851
|
-
const delayedStream = createDelayedStream(stream, this.vad.delay / 1e3);
|
|
852
|
-
this.delayedStream = delayedStream;
|
|
853
|
-
this.source = audioContext.createMediaStreamSource(delayedStream);
|
|
854
|
-
this.source.connect(this.workletNode);
|
|
855
|
-
await this.vad.start(stream);
|
|
913
|
+
this.mic = mic;
|
|
914
|
+
this.resetBuffers();
|
|
915
|
+
mic.on("Frames", this.onFrames);
|
|
916
|
+
await this.vad.start(mic);
|
|
856
917
|
this.vad.on("StartSpeaking", this.onStartSpeaking);
|
|
857
918
|
this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
858
919
|
this.vad.on("CancelSpeaking", this.onCancelSpeaking);
|
|
@@ -872,79 +933,138 @@ var MicRecorder = class extends EventEmitter5 {
|
|
|
872
933
|
isStarted: false,
|
|
873
934
|
isSpeaking: false
|
|
874
935
|
});
|
|
875
|
-
this.stream = void 0;
|
|
876
936
|
try {
|
|
937
|
+
this.mic?.off("Frames", this.onFrames);
|
|
938
|
+
this.mic = void 0;
|
|
877
939
|
this.vad.stop();
|
|
878
940
|
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
879
941
|
this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
880
942
|
this.vad.off("CancelSpeaking", this.onCancelSpeaking);
|
|
881
943
|
this.vad.off("StopSpeaking", this.onStopSpeaking);
|
|
882
|
-
if (this.workletNode) {
|
|
883
|
-
this.workletNode.disconnect();
|
|
884
|
-
this.workletNode = void 0;
|
|
885
|
-
}
|
|
886
|
-
if (this.source) {
|
|
887
|
-
this.source.disconnect();
|
|
888
|
-
this.source = void 0;
|
|
889
|
-
}
|
|
890
|
-
if (this.delayedStream) {
|
|
891
|
-
stopStream(this.delayedStream);
|
|
892
|
-
this.delayedStream = void 0;
|
|
893
|
-
}
|
|
894
944
|
} catch (err) {
|
|
895
945
|
console.error(err);
|
|
896
946
|
}
|
|
947
|
+
this.resetBuffers();
|
|
897
948
|
};
|
|
898
949
|
this.changeVad = (vadConfig) => {
|
|
899
950
|
if (equalVADConfig(vadConfig, this.vadConfig)) return;
|
|
900
|
-
const
|
|
901
|
-
if (
|
|
951
|
+
const mic = this.mic;
|
|
952
|
+
if (mic) {
|
|
902
953
|
this.stop();
|
|
903
954
|
}
|
|
904
955
|
this.vadConfig = vadConfig;
|
|
905
956
|
this.vad = getVAD(vadConfig);
|
|
906
|
-
if (
|
|
907
|
-
this.start(
|
|
957
|
+
if (mic) {
|
|
958
|
+
this.start(mic);
|
|
959
|
+
}
|
|
960
|
+
};
|
|
961
|
+
this.onFrames = (frames, sampleRate) => {
|
|
962
|
+
if (sampleRate !== this.sourceSampleRate) {
|
|
963
|
+
this.flushChunks();
|
|
964
|
+
this.sourceSampleRate = sampleRate;
|
|
965
|
+
this.reserve = [];
|
|
966
|
+
this.reserveLength = 0;
|
|
967
|
+
}
|
|
968
|
+
if (this.isRecording) {
|
|
969
|
+
this.buffer.push(frames);
|
|
970
|
+
this.bufferLength += frames.length;
|
|
971
|
+
this.flushChunks();
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
this.reserve.push(frames);
|
|
975
|
+
this.reserveLength += frames.length;
|
|
976
|
+
const reserveMax = Math.round(this.vad.delay / 1e3 * sampleRate);
|
|
977
|
+
while (this.reserve.length > 1 && this.reserveLength - this.reserve[0].length >= reserveMax) {
|
|
978
|
+
this.reserveLength -= this.reserve[0].length;
|
|
979
|
+
this.reserve.shift();
|
|
908
980
|
}
|
|
909
981
|
};
|
|
910
|
-
this.onStartSpeaking =
|
|
982
|
+
this.onStartSpeaking = () => {
|
|
911
983
|
this.speakingConfirmed = false;
|
|
912
984
|
this.queuedChunks.length = 0;
|
|
913
|
-
this.
|
|
985
|
+
this.isRecording = true;
|
|
986
|
+
this.buffer = this.reserve;
|
|
987
|
+
this.bufferLength = this.reserveLength;
|
|
988
|
+
this.reserve = [];
|
|
989
|
+
this.reserveLength = 0;
|
|
990
|
+
this.flushChunks();
|
|
914
991
|
};
|
|
915
|
-
this.onConfirmSpeaking =
|
|
992
|
+
this.onConfirmSpeaking = () => {
|
|
916
993
|
this.speakingConfirmed = true;
|
|
917
994
|
this.changeState({ isSpeaking: true });
|
|
918
995
|
this.emit("StartSpeaking");
|
|
919
|
-
|
|
920
|
-
|
|
996
|
+
for (const chunk of this.queuedChunks) {
|
|
997
|
+
this.emit("Chunk", chunk);
|
|
998
|
+
}
|
|
921
999
|
this.queuedChunks.length = 0;
|
|
922
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
923
1000
|
};
|
|
924
|
-
this.
|
|
925
|
-
this.
|
|
926
|
-
this.changeState({ isSpeaking: false });
|
|
927
|
-
this.emit("StopSpeaking");
|
|
928
|
-
this.workletNode?.port.postMessage({ type: "stop" });
|
|
1001
|
+
this.onCancelSpeaking = () => {
|
|
1002
|
+
this.stopRecording();
|
|
929
1003
|
};
|
|
930
|
-
this.
|
|
931
|
-
const
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
this.queuedChunks.push(pcmData);
|
|
936
|
-
return;
|
|
937
|
-
}
|
|
938
|
-
for (const chunk of this.queuedChunks) {
|
|
939
|
-
this.emit("Chunk", chunk);
|
|
940
|
-
}
|
|
941
|
-
this.queuedChunks.length = 0;
|
|
942
|
-
this.emit("Chunk", pcmData);
|
|
1004
|
+
this.onStopSpeaking = () => {
|
|
1005
|
+
const wasConfirmed = this.speakingConfirmed;
|
|
1006
|
+
this.changeState({ isSpeaking: false });
|
|
1007
|
+
if (wasConfirmed) {
|
|
1008
|
+
this.flushChunks(true);
|
|
943
1009
|
}
|
|
1010
|
+
this.stopRecording();
|
|
1011
|
+
this.emit("StopSpeaking");
|
|
944
1012
|
};
|
|
945
1013
|
this.state = defaultMicRecorderState;
|
|
946
1014
|
this.vad = getVAD(vadConfig);
|
|
947
1015
|
}
|
|
1016
|
+
stopRecording() {
|
|
1017
|
+
this.isRecording = false;
|
|
1018
|
+
this.speakingConfirmed = false;
|
|
1019
|
+
this.queuedChunks.length = 0;
|
|
1020
|
+
this.buffer = [];
|
|
1021
|
+
this.bufferLength = 0;
|
|
1022
|
+
}
|
|
1023
|
+
/**
|
|
1024
|
+
* Emits every complete chunk waiting in the buffer
|
|
1025
|
+
* @param final - Also emit the incomplete last chunk
|
|
1026
|
+
*/
|
|
1027
|
+
flushChunks(final = false) {
|
|
1028
|
+
const chunkLength = Math.round(
|
|
1029
|
+
CHUNK_INTERVAL / 1e3 * this.sourceSampleRate
|
|
1030
|
+
);
|
|
1031
|
+
if (this.bufferLength === 0) return;
|
|
1032
|
+
if (this.bufferLength < chunkLength && !final) return;
|
|
1033
|
+
const merged = concatFloat32(this.buffer);
|
|
1034
|
+
let offset = 0;
|
|
1035
|
+
while (merged.length - offset >= chunkLength) {
|
|
1036
|
+
this.emitChunk(merged.subarray(offset, offset + chunkLength));
|
|
1037
|
+
offset += chunkLength;
|
|
1038
|
+
}
|
|
1039
|
+
const rest = merged.subarray(offset);
|
|
1040
|
+
if (final && rest.length > 0) {
|
|
1041
|
+
this.emitChunk(rest);
|
|
1042
|
+
this.buffer = [];
|
|
1043
|
+
this.bufferLength = 0;
|
|
1044
|
+
return;
|
|
1045
|
+
}
|
|
1046
|
+
this.buffer = rest.length ? [rest] : [];
|
|
1047
|
+
this.bufferLength = rest.length;
|
|
1048
|
+
}
|
|
1049
|
+
emitChunk(samples) {
|
|
1050
|
+
const pcm = floatToPcm16(
|
|
1051
|
+
resample(samples, this.sourceSampleRate, SAMPLE_RATE)
|
|
1052
|
+
);
|
|
1053
|
+
if (!this.speakingConfirmed) {
|
|
1054
|
+
this.queuedChunks.push(pcm);
|
|
1055
|
+
return;
|
|
1056
|
+
}
|
|
1057
|
+
this.emit("Chunk", pcm);
|
|
1058
|
+
}
|
|
1059
|
+
resetBuffers() {
|
|
1060
|
+
this.isRecording = false;
|
|
1061
|
+
this.speakingConfirmed = false;
|
|
1062
|
+
this.reserve = [];
|
|
1063
|
+
this.reserveLength = 0;
|
|
1064
|
+
this.buffer = [];
|
|
1065
|
+
this.bufferLength = 0;
|
|
1066
|
+
this.queuedChunks.length = 0;
|
|
1067
|
+
}
|
|
948
1068
|
changeState(state) {
|
|
949
1069
|
const hasChanged = Object.keys(state).some(
|
|
950
1070
|
(key) => this.state[key] !== state[key]
|
|
@@ -955,18 +1075,182 @@ var MicRecorder = class extends EventEmitter5 {
|
|
|
955
1075
|
}
|
|
956
1076
|
};
|
|
957
1077
|
|
|
1078
|
+
// src/audio/Pcm16AudioStream.ts
|
|
1079
|
+
import { EventEmitter as EventEmitter5 } from "eventemitter3";
|
|
1080
|
+
var PREBUFFER_DURATION = 100;
|
|
1081
|
+
var QUIET_FLUSH_DELAY = 600;
|
|
1082
|
+
var UTTERANCE_END_DELAY = 300;
|
|
1083
|
+
var VOLUME_INTERVAL2 = 100;
|
|
1084
|
+
var Pcm16AudioStream = class extends EventEmitter5 {
|
|
1085
|
+
constructor(sink) {
|
|
1086
|
+
super();
|
|
1087
|
+
this.sink = sink;
|
|
1088
|
+
this.isPlaying = false;
|
|
1089
|
+
this.sources = [];
|
|
1090
|
+
this.nextStartTime = 0;
|
|
1091
|
+
this.prebuffering = true;
|
|
1092
|
+
this.prebuffer = [];
|
|
1093
|
+
this.prebufferDuration = 0;
|
|
1094
|
+
this.volumeTimers = [];
|
|
1095
|
+
this.meter = new VolumeMeter();
|
|
1096
|
+
}
|
|
1097
|
+
/**
|
|
1098
|
+
* Queues samples for playback
|
|
1099
|
+
* @param pcm - Mono samples as 16 bits integers
|
|
1100
|
+
* @param sampleRate - Sample rate of `pcm`, in Hz
|
|
1101
|
+
*/
|
|
1102
|
+
playAudio(pcm, sampleRate) {
|
|
1103
|
+
if (pcm.length === 0) return;
|
|
1104
|
+
const samples = resample(
|
|
1105
|
+
pcm16ToFloat(pcm),
|
|
1106
|
+
sampleRate,
|
|
1107
|
+
this.sink.sampleRate
|
|
1108
|
+
);
|
|
1109
|
+
this.handleSamples(samples);
|
|
1110
|
+
}
|
|
1111
|
+
/** Drops everything queued and stops playing immediately */
|
|
1112
|
+
stopAudio() {
|
|
1113
|
+
for (const source of this.sources) {
|
|
1114
|
+
try {
|
|
1115
|
+
source.stop();
|
|
1116
|
+
} catch (error) {
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
this.sources = [];
|
|
1120
|
+
this.nextStartTime = 0;
|
|
1121
|
+
this.resetPrebuffer();
|
|
1122
|
+
this.setIsPlaying(false);
|
|
1123
|
+
this.clearVolumeTimers();
|
|
1124
|
+
}
|
|
1125
|
+
// Accumulate audio until enough is buffered, then start playing. Once
|
|
1126
|
+
// playing, schedule incoming samples directly.
|
|
1127
|
+
handleSamples(samples) {
|
|
1128
|
+
if (this.endTimer !== void 0) {
|
|
1129
|
+
clearTimeout(this.endTimer);
|
|
1130
|
+
this.endTimer = void 0;
|
|
1131
|
+
}
|
|
1132
|
+
if (!this.prebuffering) {
|
|
1133
|
+
this.scheduleSamples(samples);
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
1136
|
+
this.prebuffer.push(samples);
|
|
1137
|
+
this.prebufferDuration += samples.length / this.sink.sampleRate * 1e3;
|
|
1138
|
+
if (this.prebufferDuration >= PREBUFFER_DURATION) {
|
|
1139
|
+
this.flushPrebuffer();
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
1142
|
+
if (this.quietTimer !== void 0) clearTimeout(this.quietTimer);
|
|
1143
|
+
this.quietTimer = setTimeout(() => this.flushPrebuffer(), QUIET_FLUSH_DELAY);
|
|
1144
|
+
}
|
|
1145
|
+
flushPrebuffer() {
|
|
1146
|
+
if (this.quietTimer !== void 0) {
|
|
1147
|
+
clearTimeout(this.quietTimer);
|
|
1148
|
+
this.quietTimer = void 0;
|
|
1149
|
+
}
|
|
1150
|
+
this.prebuffering = false;
|
|
1151
|
+
const buffers = this.prebuffer;
|
|
1152
|
+
this.prebuffer = [];
|
|
1153
|
+
this.prebufferDuration = 0;
|
|
1154
|
+
for (const samples of buffers) this.scheduleSamples(samples);
|
|
1155
|
+
}
|
|
1156
|
+
resetPrebuffer() {
|
|
1157
|
+
if (this.quietTimer !== void 0) {
|
|
1158
|
+
clearTimeout(this.quietTimer);
|
|
1159
|
+
this.quietTimer = void 0;
|
|
1160
|
+
}
|
|
1161
|
+
if (this.endTimer !== void 0) {
|
|
1162
|
+
clearTimeout(this.endTimer);
|
|
1163
|
+
this.endTimer = void 0;
|
|
1164
|
+
}
|
|
1165
|
+
this.prebuffering = true;
|
|
1166
|
+
this.prebuffer = [];
|
|
1167
|
+
this.prebufferDuration = 0;
|
|
1168
|
+
}
|
|
1169
|
+
scheduleSamples(samples) {
|
|
1170
|
+
const duration = samples.length / this.sink.sampleRate;
|
|
1171
|
+
const now = this.sink.currentTime;
|
|
1172
|
+
const startTime = Math.max(this.nextStartTime, now);
|
|
1173
|
+
this.nextStartTime = startTime + duration;
|
|
1174
|
+
const source = this.sink.schedule(samples, startTime, () => {
|
|
1175
|
+
const index = this.sources.indexOf(source);
|
|
1176
|
+
if (index !== -1) this.sources.splice(index, 1);
|
|
1177
|
+
if (this.sources.length === 0) {
|
|
1178
|
+
this.nextStartTime = 0;
|
|
1179
|
+
this.setIsPlaying(false);
|
|
1180
|
+
if (this.endTimer !== void 0) clearTimeout(this.endTimer);
|
|
1181
|
+
this.endTimer = setTimeout(() => {
|
|
1182
|
+
this.endTimer = void 0;
|
|
1183
|
+
this.resetPrebuffer();
|
|
1184
|
+
}, UTTERANCE_END_DELAY);
|
|
1185
|
+
}
|
|
1186
|
+
});
|
|
1187
|
+
this.sources.push(source);
|
|
1188
|
+
this.scheduleVolume(samples, startTime - now);
|
|
1189
|
+
this.setIsPlaying(true);
|
|
1190
|
+
}
|
|
1191
|
+
// Report the level of what is being heard rather than of what just arrived:
|
|
1192
|
+
// a burst of chunks would otherwise light up the meter all at once and leave
|
|
1193
|
+
// it dark for the rest of the sentence.
|
|
1194
|
+
scheduleVolume(samples, delay) {
|
|
1195
|
+
const step = Math.max(
|
|
1196
|
+
1,
|
|
1197
|
+
Math.round(VOLUME_INTERVAL2 / 1e3 * this.sink.sampleRate)
|
|
1198
|
+
);
|
|
1199
|
+
for (let offset = 0; offset < samples.length; offset += step) {
|
|
1200
|
+
const slice = samples.subarray(offset, offset + step);
|
|
1201
|
+
const at = delay * 1e3 + offset / this.sink.sampleRate * 1e3;
|
|
1202
|
+
const timer = setTimeout(
|
|
1203
|
+
() => {
|
|
1204
|
+
const index = this.volumeTimers.indexOf(timer);
|
|
1205
|
+
if (index !== -1) this.volumeTimers.splice(index, 1);
|
|
1206
|
+
this.emit(
|
|
1207
|
+
"Volume",
|
|
1208
|
+
this.meter.measure(
|
|
1209
|
+
resample(slice, this.sink.sampleRate, VOLUME_SAMPLE_RATE)
|
|
1210
|
+
)
|
|
1211
|
+
);
|
|
1212
|
+
},
|
|
1213
|
+
Math.max(0, at)
|
|
1214
|
+
);
|
|
1215
|
+
this.volumeTimers.push(timer);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
clearVolumeTimers() {
|
|
1219
|
+
for (const timer of this.volumeTimers) clearTimeout(timer);
|
|
1220
|
+
this.volumeTimers = [];
|
|
1221
|
+
this.meter.reset();
|
|
1222
|
+
}
|
|
1223
|
+
setIsPlaying(isPlaying) {
|
|
1224
|
+
if (this.isPlaying === isPlaying) return;
|
|
1225
|
+
this.isPlaying = isPlaying;
|
|
1226
|
+
if (!isPlaying) {
|
|
1227
|
+
this.clearVolumeTimers();
|
|
1228
|
+
this.emit("Volume", -Infinity);
|
|
1229
|
+
}
|
|
1230
|
+
this.emit(isPlaying ? "StartPlaying" : "StopPlaying");
|
|
1231
|
+
}
|
|
1232
|
+
};
|
|
1233
|
+
|
|
1234
|
+
// src/audio/types.ts
|
|
1235
|
+
import { EventEmitter as EventEmitter6 } from "eventemitter3";
|
|
1236
|
+
var MicDriver = class extends EventEmitter6 {
|
|
1237
|
+
};
|
|
1238
|
+
var SpeakerDriver = class extends EventEmitter6 {
|
|
1239
|
+
};
|
|
1240
|
+
|
|
958
1241
|
// src/audio/index.ts
|
|
959
|
-
|
|
960
|
-
|
|
1242
|
+
var globalScope = globalThis;
|
|
1243
|
+
if (!globalScope.micdropMic) {
|
|
1244
|
+
globalScope.micdropMic = new MicController();
|
|
961
1245
|
}
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
window.micdropSpeaker = new Speaker();
|
|
1246
|
+
if (!globalScope.micdropSpeaker) {
|
|
1247
|
+
globalScope.micdropSpeaker = new SpeakerController();
|
|
965
1248
|
}
|
|
966
|
-
var
|
|
1249
|
+
var Mic = globalScope.micdropMic;
|
|
1250
|
+
var Speaker = globalScope.micdropSpeaker;
|
|
967
1251
|
|
|
968
1252
|
// src/client/MicdropClient.ts
|
|
969
|
-
import { EventEmitter as
|
|
1253
|
+
import { EventEmitter as EventEmitter7 } from "eventemitter3";
|
|
970
1254
|
|
|
971
1255
|
// ../server/src/types.ts
|
|
972
1256
|
var MicdropClientCommands = /* @__PURE__ */ ((MicdropClientCommands2) => {
|
|
@@ -1004,7 +1288,7 @@ var MicdropClientError = class extends Error {
|
|
|
1004
1288
|
}
|
|
1005
1289
|
};
|
|
1006
1290
|
function getClientErrorFromWSCloseEvent(event) {
|
|
1007
|
-
const code = getClientErrorFromWSCloseEventCode(event.code);
|
|
1291
|
+
const code = getClientErrorFromWSCloseEventCode(event.code ?? 1006);
|
|
1008
1292
|
return code && new MicdropClientError(code, event.reason);
|
|
1009
1293
|
}
|
|
1010
1294
|
function getClientErrorFromWSCloseEventCode(code) {
|
|
@@ -1034,7 +1318,7 @@ var DEFAULT_RECONNECT_OPTIONS = {
|
|
|
1034
1318
|
delayMs: 1e3,
|
|
1035
1319
|
connectionTimeout: 5e3
|
|
1036
1320
|
};
|
|
1037
|
-
var MicdropClient = class extends
|
|
1321
|
+
var MicdropClient = class extends EventEmitter7 {
|
|
1038
1322
|
constructor(options = {}) {
|
|
1039
1323
|
super();
|
|
1040
1324
|
this.options = options;
|
|
@@ -1059,7 +1343,7 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1059
1343
|
this._isReconnecting = false;
|
|
1060
1344
|
this.reconnectAttempt = 0;
|
|
1061
1345
|
if (this.reconnectTimer) {
|
|
1062
|
-
|
|
1346
|
+
clearTimeout(this.reconnectTimer);
|
|
1063
1347
|
this.reconnectTimer = void 0;
|
|
1064
1348
|
}
|
|
1065
1349
|
if (!this.micRecorder || options?.vad) {
|
|
@@ -1073,7 +1357,7 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1073
1357
|
this._isPaused = false;
|
|
1074
1358
|
this._isReconnecting = false;
|
|
1075
1359
|
if (this.reconnectTimer) {
|
|
1076
|
-
|
|
1360
|
+
clearTimeout(this.reconnectTimer);
|
|
1077
1361
|
this.reconnectTimer = void 0;
|
|
1078
1362
|
}
|
|
1079
1363
|
try {
|
|
@@ -1081,13 +1365,12 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1081
1365
|
} catch (error) {
|
|
1082
1366
|
console.error("[MicdropClient] Error stopping WebSocket", error);
|
|
1083
1367
|
}
|
|
1368
|
+
Speaker.stopAudio();
|
|
1084
1369
|
try {
|
|
1085
|
-
this.
|
|
1086
|
-
this.stopMic();
|
|
1370
|
+
await this.stopMic();
|
|
1087
1371
|
} catch (error) {
|
|
1088
1372
|
console.error("[MicdropClient] Error stopping microphone", error);
|
|
1089
1373
|
}
|
|
1090
|
-
Speaker2.stopAudio();
|
|
1091
1374
|
};
|
|
1092
1375
|
this.mute = () => {
|
|
1093
1376
|
this.vad?.pause();
|
|
@@ -1095,7 +1378,7 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1095
1378
|
this.notifyStateChange();
|
|
1096
1379
|
};
|
|
1097
1380
|
this.unmute = () => {
|
|
1098
|
-
if (!this.isPaused && !(this.options.disableInterruption &&
|
|
1381
|
+
if (!this.isPaused && !(this.options.disableInterruption && Speaker.isPlaying)) {
|
|
1099
1382
|
this.vad?.resume();
|
|
1100
1383
|
}
|
|
1101
1384
|
this._isMuted = false;
|
|
@@ -1107,7 +1390,7 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1107
1390
|
this._isPaused = true;
|
|
1108
1391
|
this._isProcessing = false;
|
|
1109
1392
|
this.notifyStateChange();
|
|
1110
|
-
|
|
1393
|
+
Speaker.stopAudio();
|
|
1111
1394
|
this.ws?.send("Mute" /* Mute */);
|
|
1112
1395
|
};
|
|
1113
1396
|
this.resume = () => {
|
|
@@ -1135,16 +1418,16 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1135
1418
|
this.micRecorder.on("StateChange", () => {
|
|
1136
1419
|
this.notifyStateChange();
|
|
1137
1420
|
});
|
|
1138
|
-
this.micRecorder.on("Chunk", (
|
|
1139
|
-
this.log(
|
|
1140
|
-
this.ws?.send(
|
|
1421
|
+
this.micRecorder.on("Chunk", (chunk) => {
|
|
1422
|
+
this.log("User audio chunk", chunk.length);
|
|
1423
|
+
this.ws?.send(pcm16ToArrayBuffer(chunk));
|
|
1141
1424
|
});
|
|
1142
1425
|
this.micRecorder.on("StartSpeaking", () => {
|
|
1143
1426
|
this.log("User start speaking");
|
|
1144
1427
|
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
1145
1428
|
this._isProcessing = false;
|
|
1146
1429
|
this.notifyStateChange();
|
|
1147
|
-
|
|
1430
|
+
Speaker.stopAudio();
|
|
1148
1431
|
});
|
|
1149
1432
|
this.micRecorder.on("StopSpeaking", () => {
|
|
1150
1433
|
this.log("User stop speaking");
|
|
@@ -1155,11 +1438,16 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1155
1438
|
this.notifyStateChange();
|
|
1156
1439
|
});
|
|
1157
1440
|
}
|
|
1158
|
-
const
|
|
1159
|
-
|
|
1160
|
-
|
|
1441
|
+
const deviceId = options.deviceId ?? storage.getItem("micdrop.micDevice" /* MicDevice */) ?? void 0;
|
|
1442
|
+
await Mic.start(deviceId);
|
|
1443
|
+
if (Mic.deviceId) {
|
|
1444
|
+
storage.setItem("micdrop.micDevice" /* MicDevice */, Mic.deviceId);
|
|
1445
|
+
} else {
|
|
1446
|
+
storage.removeItem("micdrop.micDevice" /* MicDevice */);
|
|
1447
|
+
}
|
|
1448
|
+
await this.micRecorder.start(Mic);
|
|
1449
|
+
await Speaker.start();
|
|
1161
1450
|
await this.updateDevices();
|
|
1162
|
-
await Speaker2.start();
|
|
1163
1451
|
this.notifyStateChange();
|
|
1164
1452
|
} catch (error) {
|
|
1165
1453
|
this.setError(
|
|
@@ -1176,37 +1464,22 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1176
1464
|
await this.startMic({ deviceId });
|
|
1177
1465
|
};
|
|
1178
1466
|
this.changeSpeakerDevice = async (deviceId) => {
|
|
1179
|
-
await
|
|
1467
|
+
await Speaker.changeDevice(deviceId);
|
|
1180
1468
|
this.notifyStateChange();
|
|
1181
1469
|
};
|
|
1182
1470
|
this.updateDevices = async () => {
|
|
1183
|
-
const
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
const defaultDevicesIndexes = defaultDevices.map(
|
|
1190
|
-
(device) => devices.findIndex(
|
|
1191
|
-
(d) => d.groupId === device.groupId && d.kind === device.kind
|
|
1192
|
-
)
|
|
1193
|
-
).filter((index) => index !== -1);
|
|
1194
|
-
for (const index of defaultDevicesIndexes) {
|
|
1195
|
-
const device = devices[index];
|
|
1196
|
-
devices.splice(index, 1);
|
|
1197
|
-
devices.unshift(device);
|
|
1198
|
-
}
|
|
1199
|
-
this.micDevices = devices.filter((device) => device.kind === "audioinput");
|
|
1200
|
-
this.speakerDevices = devices.filter(
|
|
1201
|
-
(device) => device.kind === "audiooutput"
|
|
1202
|
-
);
|
|
1203
|
-
Speaker2.devices = this.speakerDevices;
|
|
1471
|
+
const [micDevices, speakerDevices] = await Promise.all([
|
|
1472
|
+
Mic.getDevices(),
|
|
1473
|
+
Speaker.getDevices()
|
|
1474
|
+
]);
|
|
1475
|
+
this.micDevices = micDevices;
|
|
1476
|
+
this.speakerDevices = speakerDevices;
|
|
1204
1477
|
this.notifyStateChange();
|
|
1205
1478
|
};
|
|
1206
1479
|
this.onWSOpen = () => {
|
|
1207
1480
|
this.log("WebSocket opened");
|
|
1208
1481
|
if (this.connectionTimer) {
|
|
1209
|
-
|
|
1482
|
+
clearTimeout(this.connectionTimer);
|
|
1210
1483
|
this.connectionTimer = void 0;
|
|
1211
1484
|
}
|
|
1212
1485
|
this._isReconnecting = false;
|
|
@@ -1221,36 +1494,36 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1221
1494
|
}
|
|
1222
1495
|
};
|
|
1223
1496
|
this.onWSMessage = (event) => {
|
|
1224
|
-
|
|
1225
|
-
if (
|
|
1497
|
+
const data = event.data;
|
|
1498
|
+
if (data instanceof ArrayBuffer) {
|
|
1499
|
+
this.log("Received audio", data.byteLength);
|
|
1226
1500
|
if (this.isPaused || this.isUserSpeaking) return;
|
|
1227
|
-
|
|
1501
|
+
Speaker.playAudio(new Int16Array(data));
|
|
1228
1502
|
this._isProcessing = false;
|
|
1229
1503
|
this.notifyStateChange();
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1504
|
+
return;
|
|
1505
|
+
}
|
|
1506
|
+
if (typeof data !== "string") {
|
|
1507
|
+
console.warn(`[MicdropClient] Unknown message type: ${data}`);
|
|
1508
|
+
return;
|
|
1509
|
+
}
|
|
1510
|
+
this.log("Received message:", data);
|
|
1511
|
+
if (data.startsWith("Message" /* Message */)) {
|
|
1233
1512
|
try {
|
|
1234
1513
|
const message = JSON.parse(
|
|
1235
|
-
|
|
1514
|
+
data.substring("Message" /* Message */.length + 1)
|
|
1236
1515
|
);
|
|
1237
1516
|
this.addMessage(message);
|
|
1238
1517
|
} catch (error) {
|
|
1239
|
-
console.error(
|
|
1240
|
-
"[MicdropClient] Error parsing message:",
|
|
1241
|
-
event.data,
|
|
1242
|
-
error
|
|
1243
|
-
);
|
|
1518
|
+
console.error("[MicdropClient] Error parsing message:", data, error);
|
|
1244
1519
|
}
|
|
1245
|
-
} else if (
|
|
1520
|
+
} else if (data === "EndCall" /* EndCall */) {
|
|
1246
1521
|
this.emit("EndCall");
|
|
1247
|
-
} else if (
|
|
1522
|
+
} else if (data === "SkipAnswer" /* SkipAnswer */) {
|
|
1248
1523
|
this._isProcessing = false;
|
|
1249
1524
|
this.notifyStateChange();
|
|
1250
|
-
} else if (
|
|
1251
|
-
const lastMessage = this.conversation.
|
|
1252
|
-
(message) => message.role === "user"
|
|
1253
|
-
);
|
|
1525
|
+
} else if (data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
1526
|
+
const lastMessage = this.conversation.map((message) => message.role).lastIndexOf("user");
|
|
1254
1527
|
if (lastMessage !== -1) {
|
|
1255
1528
|
this.conversation = this.conversation.filter(
|
|
1256
1529
|
(_, index) => index !== lastMessage
|
|
@@ -1258,24 +1531,20 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1258
1531
|
this._isProcessing = false;
|
|
1259
1532
|
this.notifyStateChange();
|
|
1260
1533
|
}
|
|
1261
|
-
} else if (
|
|
1534
|
+
} else if (data.startsWith("ToolCall" /* ToolCall */)) {
|
|
1262
1535
|
try {
|
|
1263
1536
|
const toolCall = JSON.parse(
|
|
1264
|
-
|
|
1537
|
+
data.substring("ToolCall" /* ToolCall */.length + 1)
|
|
1265
1538
|
);
|
|
1266
1539
|
this.emit("ToolCall", toolCall);
|
|
1267
1540
|
} catch (error) {
|
|
1268
|
-
console.error(
|
|
1269
|
-
"[MicdropClient] Error parsing tool call:",
|
|
1270
|
-
event.data,
|
|
1271
|
-
error
|
|
1272
|
-
);
|
|
1541
|
+
console.error("[MicdropClient] Error parsing tool call:", data, error);
|
|
1273
1542
|
}
|
|
1274
1543
|
}
|
|
1275
1544
|
};
|
|
1276
1545
|
this.onWSClose = (event) => {
|
|
1277
|
-
this.log("WebSocket closed", event);
|
|
1278
|
-
const error = getClientErrorFromWSCloseEvent(event);
|
|
1546
|
+
this.log("WebSocket closed", event?.code, event?.reason);
|
|
1547
|
+
const error = getClientErrorFromWSCloseEvent(event ?? {});
|
|
1279
1548
|
if (error) {
|
|
1280
1549
|
this.setError(error);
|
|
1281
1550
|
}
|
|
@@ -1289,7 +1558,7 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1289
1558
|
this.stop();
|
|
1290
1559
|
}
|
|
1291
1560
|
};
|
|
1292
|
-
this.onWSError = (
|
|
1561
|
+
this.onWSError = () => {
|
|
1293
1562
|
this.setError(new MicdropClientError("Connection" /* Connection */));
|
|
1294
1563
|
};
|
|
1295
1564
|
this.onSpeakerStartPlaying = () => {
|
|
@@ -1312,9 +1581,9 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1312
1581
|
this.notifyStateChange();
|
|
1313
1582
|
}
|
|
1314
1583
|
};
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1584
|
+
Speaker.on("StartPlaying", this.onSpeakerStartPlaying);
|
|
1585
|
+
Speaker.on("StopPlaying", this.onSpeakerStopPlaying);
|
|
1586
|
+
Mic.on("DeviceChange", this.updateDevices);
|
|
1318
1587
|
}
|
|
1319
1588
|
get vad() {
|
|
1320
1589
|
return this.micRecorder?.vad;
|
|
@@ -1341,13 +1610,13 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1341
1610
|
return this.isMicStarted && !this.isPaused && !this.isProcessing && !this.isMicMuted && !this.isUserSpeaking && !this.isAssistantSpeaking;
|
|
1342
1611
|
}
|
|
1343
1612
|
get isWSStarted() {
|
|
1344
|
-
return this.ws?.readyState ===
|
|
1613
|
+
return this.ws?.readyState === 1;
|
|
1345
1614
|
}
|
|
1346
1615
|
get isWSStarting() {
|
|
1347
|
-
return this.ws?.readyState ===
|
|
1616
|
+
return this.ws?.readyState === 0;
|
|
1348
1617
|
}
|
|
1349
1618
|
get isMicStarted() {
|
|
1350
|
-
return
|
|
1619
|
+
return Mic.isStarted;
|
|
1351
1620
|
}
|
|
1352
1621
|
get isMicMuted() {
|
|
1353
1622
|
return this.vad?.isPaused ?? false;
|
|
@@ -1356,13 +1625,13 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1356
1625
|
return this.micRecorder?.state.isSpeaking ?? false;
|
|
1357
1626
|
}
|
|
1358
1627
|
get isAssistantSpeaking() {
|
|
1359
|
-
return
|
|
1628
|
+
return Speaker.isPlaying;
|
|
1360
1629
|
}
|
|
1361
1630
|
get micDeviceId() {
|
|
1362
|
-
return
|
|
1631
|
+
return Mic.deviceId;
|
|
1363
1632
|
}
|
|
1364
1633
|
get speakerDeviceId() {
|
|
1365
|
-
return
|
|
1634
|
+
return Speaker.deviceId;
|
|
1366
1635
|
}
|
|
1367
1636
|
get state() {
|
|
1368
1637
|
return {
|
|
@@ -1385,11 +1654,10 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1385
1654
|
speakerDevices: this.speakerDevices
|
|
1386
1655
|
};
|
|
1387
1656
|
}
|
|
1388
|
-
stopMic() {
|
|
1657
|
+
async stopMic() {
|
|
1389
1658
|
this.micRecorder?.stop();
|
|
1390
1659
|
this.micRecorder = void 0;
|
|
1391
|
-
|
|
1392
|
-
this.micStream = void 0;
|
|
1660
|
+
await Mic.stop();
|
|
1393
1661
|
this.notifyStateChange();
|
|
1394
1662
|
}
|
|
1395
1663
|
async startWS() {
|
|
@@ -1405,22 +1673,22 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1405
1673
|
throw new MicdropClientError("MissingUrl" /* MissingUrl */);
|
|
1406
1674
|
}
|
|
1407
1675
|
this.ws = new WebSocket(this.options.url);
|
|
1408
|
-
this.ws.binaryType = "
|
|
1676
|
+
this.ws.binaryType = "arraybuffer";
|
|
1409
1677
|
this.notifyStateChange();
|
|
1410
1678
|
this.ws.onopen = this.onWSOpen;
|
|
1411
1679
|
this.ws.onmessage = this.onWSMessage;
|
|
1412
1680
|
this.ws.onclose = this.onWSClose;
|
|
1413
1681
|
this.ws.onerror = this.onWSError;
|
|
1414
|
-
this.connectionTimer =
|
|
1682
|
+
this.connectionTimer = setTimeout(() => {
|
|
1415
1683
|
this.connectionTimer = void 0;
|
|
1416
|
-
if (this.ws?.readyState ===
|
|
1684
|
+
if (this.ws?.readyState === 0) {
|
|
1417
1685
|
this.log("WebSocket connection timeout");
|
|
1418
1686
|
this.ws.close();
|
|
1419
1687
|
}
|
|
1420
1688
|
}, this.options.reconnect?.connectionTimeout ?? DEFAULT_RECONNECT_OPTIONS.connectionTimeout);
|
|
1421
1689
|
} catch (error) {
|
|
1422
1690
|
this.setError(
|
|
1423
|
-
new MicdropClientError(
|
|
1691
|
+
error instanceof MicdropClientError ? error : new MicdropClientError(
|
|
1424
1692
|
"Connection" /* Connection */,
|
|
1425
1693
|
error?.message
|
|
1426
1694
|
)
|
|
@@ -1431,11 +1699,11 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1431
1699
|
}
|
|
1432
1700
|
stopWS() {
|
|
1433
1701
|
if (this.connectionTimer) {
|
|
1434
|
-
|
|
1702
|
+
clearTimeout(this.connectionTimer);
|
|
1435
1703
|
this.connectionTimer = void 0;
|
|
1436
1704
|
}
|
|
1437
1705
|
if (!this.ws) return;
|
|
1438
|
-
if (this.ws.readyState ===
|
|
1706
|
+
if (this.ws.readyState === 1 || this.ws.readyState === 0) {
|
|
1439
1707
|
this.ws.close();
|
|
1440
1708
|
}
|
|
1441
1709
|
this.ws = void 0;
|
|
@@ -1455,9 +1723,9 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1455
1723
|
this.vad?.pause?.();
|
|
1456
1724
|
}
|
|
1457
1725
|
if (this.reconnectTimer) {
|
|
1458
|
-
|
|
1726
|
+
clearTimeout(this.reconnectTimer);
|
|
1459
1727
|
}
|
|
1460
|
-
this.reconnectTimer =
|
|
1728
|
+
this.reconnectTimer = setTimeout(async () => {
|
|
1461
1729
|
try {
|
|
1462
1730
|
await this.startWS();
|
|
1463
1731
|
} catch {
|
|
@@ -1487,13 +1755,15 @@ var MicdropClient = class extends EventEmitter6 {
|
|
|
1487
1755
|
};
|
|
1488
1756
|
|
|
1489
1757
|
// src/Micdrop.ts
|
|
1490
|
-
|
|
1491
|
-
|
|
1758
|
+
var globalScope2 = globalThis;
|
|
1759
|
+
if (!globalScope2.micdropClient) {
|
|
1760
|
+
globalScope2.micdropClient = new MicdropClient();
|
|
1492
1761
|
}
|
|
1493
|
-
var Micdrop =
|
|
1762
|
+
var Micdrop = globalScope2.micdropClient;
|
|
1494
1763
|
export {
|
|
1495
|
-
|
|
1496
|
-
|
|
1764
|
+
Mic,
|
|
1765
|
+
MicController,
|
|
1766
|
+
MicDriver,
|
|
1497
1767
|
MicRecorder,
|
|
1498
1768
|
Micdrop,
|
|
1499
1769
|
MicdropClient,
|
|
@@ -1501,16 +1771,33 @@ export {
|
|
|
1501
1771
|
MicdropClientError,
|
|
1502
1772
|
MicdropClientErrorCode,
|
|
1503
1773
|
MicdropServerCommands,
|
|
1774
|
+
MicdropStorageKeys,
|
|
1504
1775
|
MultipleVAD,
|
|
1776
|
+
Pcm16AudioStream,
|
|
1777
|
+
SILERO_FRAME_SAMPLES,
|
|
1778
|
+
SILERO_SAMPLE_RATE,
|
|
1505
1779
|
SileroVAD,
|
|
1506
|
-
|
|
1780
|
+
Speaker,
|
|
1781
|
+
SpeakerController,
|
|
1782
|
+
SpeakerDriver,
|
|
1507
1783
|
VAD,
|
|
1508
1784
|
VADStatus,
|
|
1785
|
+
VOLUME_SAMPLE_RATE,
|
|
1786
|
+
VolumeMeter,
|
|
1509
1787
|
VolumeVAD,
|
|
1788
|
+
concatFloat32,
|
|
1510
1789
|
equalVADConfig,
|
|
1790
|
+
floatToPcm16,
|
|
1511
1791
|
getClientErrorFromWSCloseEvent,
|
|
1512
1792
|
getClientErrorFromWSCloseEventCode,
|
|
1513
1793
|
getVAD,
|
|
1514
|
-
|
|
1794
|
+
getVolume,
|
|
1795
|
+
isRecoverableError,
|
|
1796
|
+
pcm16ToArrayBuffer,
|
|
1797
|
+
pcm16ToFloat,
|
|
1798
|
+
resample,
|
|
1799
|
+
setMicdropStorage,
|
|
1800
|
+
setSileroModelLoader,
|
|
1801
|
+
storage
|
|
1515
1802
|
};
|
|
1516
1803
|
//# sourceMappingURL=index.mjs.map
|