@micdrop/client 1.0.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -41
- package/dist/index.d.mts +130 -16
- package/dist/index.d.ts +130 -16
- package/dist/index.js +391 -96
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +387 -95
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -1
package/dist/index.js
CHANGED
|
@@ -38,7 +38,10 @@ __export(index_exports, {
|
|
|
38
38
|
LocalStorageKeys: () => LocalStorageKeys,
|
|
39
39
|
Mic: () => microphone_exports,
|
|
40
40
|
MicRecorder: () => MicRecorder,
|
|
41
|
-
|
|
41
|
+
SileroVAD: () => SileroVAD,
|
|
42
|
+
Speaker: () => speaker_exports,
|
|
43
|
+
VAD: () => VAD,
|
|
44
|
+
VolumeVAD: () => VolumeVAD
|
|
42
45
|
});
|
|
43
46
|
module.exports = __toCommonJS(index_exports);
|
|
44
47
|
|
|
@@ -52,14 +55,12 @@ var LocalStorageKeys = /* @__PURE__ */ ((LocalStorageKeys2) => {
|
|
|
52
55
|
|
|
53
56
|
// src/audio/MicRecorder.ts
|
|
54
57
|
var import_eventemitter32 = __toESM(require("eventemitter3"));
|
|
55
|
-
var import_hark = __toESM(require("hark"));
|
|
56
58
|
|
|
57
59
|
// src/audio/microphone.ts
|
|
58
60
|
var microphone_exports = {};
|
|
59
61
|
__export(microphone_exports, {
|
|
60
|
-
|
|
62
|
+
audioContext: () => audioContext,
|
|
61
63
|
defaultMicThreshold: () => defaultMicThreshold,
|
|
62
|
-
getMicConstraints: () => getMicConstraints,
|
|
63
64
|
micAnalyser: () => micAnalyser,
|
|
64
65
|
startMicrophone: () => startMicrophone,
|
|
65
66
|
stopMicrophone: () => stopMicrophone
|
|
@@ -176,13 +177,15 @@ function stopMicrophone() {
|
|
|
176
177
|
audioStream = void 0;
|
|
177
178
|
}
|
|
178
179
|
}
|
|
179
|
-
|
|
180
|
+
|
|
181
|
+
// src/audio/createDelayedStream.ts
|
|
182
|
+
function createDelayedStream(stream, delay) {
|
|
180
183
|
if (!audioContext) {
|
|
181
184
|
throw new Error("AudioContext not initialized");
|
|
182
185
|
}
|
|
183
186
|
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
184
|
-
const delayNode = audioContext.createDelay();
|
|
185
|
-
delayNode.delayTime.value =
|
|
187
|
+
const delayNode = audioContext.createDelay(delay);
|
|
188
|
+
delayNode.delayTime.value = delay;
|
|
186
189
|
const audioDestination = audioContext.createMediaStreamDestination();
|
|
187
190
|
audioSource.connect(delayNode);
|
|
188
191
|
delayNode.connect(audioDestination);
|
|
@@ -190,7 +193,7 @@ function createDelayedStream(stream) {
|
|
|
190
193
|
}
|
|
191
194
|
|
|
192
195
|
// src/audio/MicRecorder.ts
|
|
193
|
-
var timeSlice =
|
|
196
|
+
var timeSlice = 100;
|
|
194
197
|
var defaultMicRecorderState = {
|
|
195
198
|
isStarting: false,
|
|
196
199
|
isStarted: false,
|
|
@@ -199,22 +202,36 @@ var defaultMicRecorderState = {
|
|
|
199
202
|
threshold: defaultMicThreshold
|
|
200
203
|
};
|
|
201
204
|
var MicRecorder = class extends import_eventemitter32.default {
|
|
202
|
-
constructor() {
|
|
205
|
+
constructor(vad) {
|
|
203
206
|
super();
|
|
204
207
|
this.audioInfo = this.getAudioInfo();
|
|
208
|
+
this.speakingConfirmed = false;
|
|
209
|
+
this.queuedChunks = [];
|
|
205
210
|
this.onStartSpeaking = (async () => {
|
|
206
211
|
if (!this.recorder || this.state.isMuted) return;
|
|
207
212
|
this.recorder.start(timeSlice);
|
|
213
|
+
this.queuedChunks.length = 0;
|
|
214
|
+
}).bind(this);
|
|
215
|
+
this.onConfirmSpeaking = (async () => {
|
|
208
216
|
this.emit("StartSpeaking");
|
|
209
217
|
this.changeState({ isSpeaking: true });
|
|
218
|
+
this.speakingConfirmed = true;
|
|
219
|
+
}).bind(this);
|
|
220
|
+
this.onCancelSpeaking = (async () => {
|
|
221
|
+
if (!this.recorder) return;
|
|
222
|
+
this.recorder.stop();
|
|
223
|
+
this.queuedChunks.length = 0;
|
|
210
224
|
}).bind(this);
|
|
211
225
|
this.onStopSpeaking = (async () => {
|
|
212
|
-
if (!this.recorder
|
|
226
|
+
if (!this.recorder) return;
|
|
213
227
|
this.recorder.stop();
|
|
214
228
|
this.changeState({ isSpeaking: false });
|
|
229
|
+
this.emit("StopSpeaking");
|
|
230
|
+
this.speakingConfirmed = false;
|
|
215
231
|
}).bind(this);
|
|
216
232
|
const threshold = parseFloat(localStorage.getItem("micdrop.micThreshold" /* MicThreshold */) || "0") || defaultMicThreshold;
|
|
217
233
|
this.state = { ...defaultMicRecorderState, threshold };
|
|
234
|
+
this.vad = vad;
|
|
218
235
|
}
|
|
219
236
|
async start(stream) {
|
|
220
237
|
if (this.state.isStarted) {
|
|
@@ -226,15 +243,19 @@ var MicRecorder = class extends import_eventemitter32.default {
|
|
|
226
243
|
isMuted: false,
|
|
227
244
|
isSpeaking: false
|
|
228
245
|
});
|
|
229
|
-
const delayedStream = createDelayedStream(
|
|
246
|
+
const delayedStream = createDelayedStream(
|
|
247
|
+
stream,
|
|
248
|
+
this.vad.delay / 1e3 + 0.05
|
|
249
|
+
);
|
|
230
250
|
this.delayedStream = delayedStream;
|
|
231
|
-
if (
|
|
232
|
-
this.recorder
|
|
233
|
-
mimeType: this.audioInfo.mimeType,
|
|
234
|
-
audioBitsPerSecond: 128e3
|
|
235
|
-
});
|
|
236
|
-
this.recorder.ondataavailable = this.onDataAvailable.bind(this);
|
|
251
|
+
if (this.recorder) {
|
|
252
|
+
this.recorder.stop();
|
|
237
253
|
}
|
|
254
|
+
this.recorder = new window.MediaRecorder(delayedStream, {
|
|
255
|
+
mimeType: this.audioInfo.mimeType,
|
|
256
|
+
audioBitsPerSecond: 128e3
|
|
257
|
+
});
|
|
258
|
+
this.recorder.ondataavailable = this.onDataAvailable.bind(this);
|
|
238
259
|
await this.startSpeakingDetection(stream);
|
|
239
260
|
this.changeState({
|
|
240
261
|
isStarting: false,
|
|
@@ -276,30 +297,35 @@ var MicRecorder = class extends import_eventemitter32.default {
|
|
|
276
297
|
setThreshold(threshold) {
|
|
277
298
|
this.changeState({ threshold });
|
|
278
299
|
localStorage.setItem("micdrop.micThreshold" /* MicThreshold */, threshold.toString());
|
|
279
|
-
this.
|
|
300
|
+
this.vad.setThreshold(threshold);
|
|
280
301
|
}
|
|
281
302
|
async startSpeakingDetection(stream) {
|
|
282
|
-
if (this.
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
this.
|
|
289
|
-
this.
|
|
290
|
-
}
|
|
291
|
-
stopSpeakingDetection() {
|
|
292
|
-
|
|
293
|
-
this.
|
|
294
|
-
this.
|
|
295
|
-
this.
|
|
296
|
-
this.
|
|
303
|
+
if (this.vad.isStarted) {
|
|
304
|
+
this.vad.stop();
|
|
305
|
+
}
|
|
306
|
+
this.vad.on("StartSpeaking", this.onStartSpeaking);
|
|
307
|
+
this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
308
|
+
this.vad.on("CancelSpeaking", this.onCancelSpeaking);
|
|
309
|
+
this.vad.on("StopSpeaking", this.onStopSpeaking);
|
|
310
|
+
await this.vad.start(stream, this.state.threshold);
|
|
311
|
+
}
|
|
312
|
+
async stopSpeakingDetection() {
|
|
313
|
+
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
314
|
+
this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
315
|
+
this.vad.off("CancelSpeaking", this.onCancelSpeaking);
|
|
316
|
+
this.vad.off("StopSpeaking", this.onStopSpeaking);
|
|
317
|
+
await this.vad.stop();
|
|
297
318
|
}
|
|
298
319
|
async onDataAvailable(blobEvent) {
|
|
299
|
-
this.
|
|
300
|
-
|
|
301
|
-
|
|
320
|
+
if (!this.speakingConfirmed) {
|
|
321
|
+
this.queuedChunks.push(blobEvent.data);
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
for (const chunk of this.queuedChunks) {
|
|
325
|
+
this.emit("Chunk", chunk);
|
|
302
326
|
}
|
|
327
|
+
this.queuedChunks.length = 0;
|
|
328
|
+
this.emit("Chunk", blobEvent.data);
|
|
303
329
|
}
|
|
304
330
|
changeState(state) {
|
|
305
331
|
const hasChanged = Object.keys(state).some(
|
|
@@ -326,9 +352,11 @@ var speaker_exports = {};
|
|
|
326
352
|
__export(speaker_exports, {
|
|
327
353
|
canChangeSpeakerDevice: () => canChangeSpeakerDevice,
|
|
328
354
|
changeSpeakerDevice: () => changeSpeakerDevice,
|
|
329
|
-
|
|
355
|
+
pauseAudio: () => pauseAudio,
|
|
356
|
+
playAudio: () => playAudio,
|
|
357
|
+
resumeAudio: () => resumeAudio,
|
|
330
358
|
speakerAnalyser: () => speakerAnalyser,
|
|
331
|
-
|
|
359
|
+
stopAudio: () => stopAudio
|
|
332
360
|
});
|
|
333
361
|
|
|
334
362
|
// src/audio/unlock.ts
|
|
@@ -362,88 +390,344 @@ function unlockSafari(audioContext3) {
|
|
|
362
390
|
// src/audio/speaker.ts
|
|
363
391
|
var audioElement;
|
|
364
392
|
var audioContext2;
|
|
365
|
-
var
|
|
393
|
+
var mediaSource;
|
|
394
|
+
var sourceBuffer;
|
|
395
|
+
var currentMediaStream;
|
|
396
|
+
var isProcessingQueue = false;
|
|
397
|
+
var pendingBlobs = [];
|
|
366
398
|
var speakerAnalyser;
|
|
367
399
|
function init2() {
|
|
368
400
|
if (audioContext2) return;
|
|
369
401
|
audioContext2 = new (window.AudioContext || window.webkitAudioContext)();
|
|
370
402
|
speakerAnalyser = new AudioAnalyser(audioContext2);
|
|
371
|
-
speakerAnalyser.node.connect(audioContext2.destination);
|
|
372
403
|
unlock(audioContext2);
|
|
373
404
|
}
|
|
374
405
|
init2();
|
|
375
|
-
function
|
|
376
|
-
return window.HTMLMediaElement && "sinkId" in window.HTMLMediaElement.prototype;
|
|
377
|
-
}
|
|
378
|
-
function setSinkId(speakerId) {
|
|
379
|
-
if (speakerId && canChangeSpeakerDevice()) {
|
|
380
|
-
return audioElement?.setSinkId(speakerId);
|
|
381
|
-
}
|
|
382
|
-
}
|
|
383
|
-
async function attachAudioElement(speakerId) {
|
|
406
|
+
async function setupAudioElement(speakerId) {
|
|
384
407
|
if (!audioContext2 || !speakerAnalyser) return;
|
|
385
408
|
if (!speakerId) {
|
|
386
409
|
speakerId = localStorage.getItem("micdrop.speakerDevice" /* SpeakerDevice */) || void 0;
|
|
387
410
|
}
|
|
388
|
-
if (!
|
|
389
|
-
|
|
390
|
-
|
|
411
|
+
if (!audioElement) {
|
|
412
|
+
audioElement = new Audio();
|
|
413
|
+
audioElement.autoplay = true;
|
|
414
|
+
mediaSource = new MediaSource();
|
|
415
|
+
if (!mediaSource) {
|
|
416
|
+
throw new Error("Failed to create MediaSource");
|
|
417
|
+
}
|
|
418
|
+
audioElement.src = URL.createObjectURL(mediaSource);
|
|
419
|
+
const ms = mediaSource;
|
|
420
|
+
ms.addEventListener("sourceopen", () => {
|
|
421
|
+
sourceBuffer = ms.addSourceBuffer("audio/mpeg");
|
|
422
|
+
sourceBuffer.addEventListener("updateend", () => {
|
|
423
|
+
processQueue();
|
|
424
|
+
});
|
|
425
|
+
if (pendingBlobs.length > 0) {
|
|
426
|
+
processQueue();
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
audioElement.addEventListener("timeupdate", function onTimeUpdate() {
|
|
430
|
+
if (!audioContext2 || !speakerAnalyser || !audioElement) return;
|
|
431
|
+
try {
|
|
432
|
+
const mediaStream = audioElement.captureStream();
|
|
433
|
+
if (mediaStream.getAudioTracks().length === 0) return;
|
|
434
|
+
currentMediaStream = mediaStream;
|
|
435
|
+
const sourceNode2 = audioContext2.createMediaStreamSource(mediaStream);
|
|
436
|
+
sourceNode2.connect(speakerAnalyser.node);
|
|
437
|
+
audioElement.removeEventListener("timeupdate", onTimeUpdate);
|
|
438
|
+
} catch (error) {
|
|
439
|
+
console.error("Error creating media stream source:", error);
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
if (speakerId) {
|
|
444
|
+
try {
|
|
445
|
+
await setSinkId(speakerId);
|
|
446
|
+
} catch (error) {
|
|
447
|
+
console.error(`Error setting Audio Output to ${speakerId}`, error);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
return audioElement;
|
|
451
|
+
}
|
|
452
|
+
async function processQueue() {
|
|
453
|
+
if (!sourceBuffer || !mediaSource || pendingBlobs.length === 0) return;
|
|
454
|
+
if (isProcessingQueue || sourceBuffer.updating) return;
|
|
455
|
+
isProcessingQueue = true;
|
|
456
|
+
try {
|
|
457
|
+
await playNextBlob();
|
|
458
|
+
} finally {
|
|
459
|
+
isProcessingQueue = false;
|
|
460
|
+
if (pendingBlobs.length > 0 && !sourceBuffer.updating) {
|
|
461
|
+
processQueue();
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
async function playNextBlob() {
|
|
466
|
+
if (!sourceBuffer || !mediaSource || pendingBlobs.length === 0) return;
|
|
467
|
+
if (sourceBuffer.updating) return;
|
|
468
|
+
if (mediaSource.readyState !== "open") {
|
|
469
|
+
console.warn("MediaSource is no longer open, resetting audio pipeline");
|
|
470
|
+
stopAudio();
|
|
471
|
+
await setupAudioElement();
|
|
391
472
|
return;
|
|
392
473
|
}
|
|
393
|
-
|
|
474
|
+
const blob = pendingBlobs.shift();
|
|
394
475
|
try {
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
476
|
+
if (sourceBuffer && Array.from(mediaSource.sourceBuffers).includes(sourceBuffer)) {
|
|
477
|
+
const arrayBuffer = await blob.arrayBuffer();
|
|
478
|
+
sourceBuffer.appendBuffer(arrayBuffer);
|
|
479
|
+
} else {
|
|
480
|
+
console.warn("SourceBuffer is no longer valid, resetting audio pipeline");
|
|
481
|
+
stopAudio();
|
|
482
|
+
await setupAudioElement();
|
|
483
|
+
}
|
|
401
484
|
} catch (error) {
|
|
402
|
-
console.error(
|
|
403
|
-
|
|
485
|
+
console.error("Error appending buffer", error);
|
|
486
|
+
if (error instanceof Error && error.name === "InvalidStateError") {
|
|
487
|
+
console.warn("Resetting audio pipeline due to invalid state");
|
|
488
|
+
stopAudio();
|
|
489
|
+
await setupAudioElement();
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
function canChangeSpeakerDevice() {
|
|
494
|
+
return window.HTMLMediaElement && "sinkId" in window.HTMLMediaElement.prototype;
|
|
495
|
+
}
|
|
496
|
+
async function setSinkId(speakerId) {
|
|
497
|
+
if (speakerId && canChangeSpeakerDevice() && audioElement) {
|
|
498
|
+
return audioElement.setSinkId(speakerId);
|
|
404
499
|
}
|
|
405
500
|
}
|
|
406
501
|
async function changeSpeakerDevice(speakerId) {
|
|
407
502
|
if (!canChangeSpeakerDevice()) return;
|
|
408
503
|
try {
|
|
409
|
-
await
|
|
504
|
+
await setupAudioElement(speakerId);
|
|
410
505
|
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, speakerId);
|
|
411
506
|
} catch (error) {
|
|
412
507
|
console.error(`Error setting Audio Output to ${speakerId}`, error);
|
|
413
508
|
}
|
|
414
509
|
}
|
|
415
|
-
|
|
510
|
+
function pauseAudio() {
|
|
511
|
+
if (audioElement) {
|
|
512
|
+
audioElement.pause();
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
function resumeAudio() {
|
|
516
|
+
if (audioElement) {
|
|
517
|
+
audioElement.play();
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async function playAudio(blob) {
|
|
416
521
|
if (!audioContext2 || !speakerAnalyser) return;
|
|
522
|
+
pendingBlobs.push(blob);
|
|
523
|
+
if (!audioElement) {
|
|
524
|
+
await setupAudioElement();
|
|
525
|
+
}
|
|
526
|
+
if (!audioElement) return;
|
|
417
527
|
try {
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
};
|
|
425
|
-
sourceNode2.buffer = await audioContext2.decodeAudioData(
|
|
426
|
-
await blob.arrayBuffer()
|
|
427
|
-
);
|
|
428
|
-
sourceNode2.connect(speakerAnalyser.node);
|
|
429
|
-
sourceNode2.start();
|
|
528
|
+
if (audioElement.paused) {
|
|
529
|
+
audioElement.play();
|
|
530
|
+
}
|
|
531
|
+
if (sourceBuffer && !sourceBuffer.updating && !isProcessingQueue) {
|
|
532
|
+
processQueue();
|
|
533
|
+
}
|
|
430
534
|
} catch (error) {
|
|
431
535
|
console.error("Error playing audio blob", error, blob);
|
|
432
536
|
}
|
|
433
537
|
}
|
|
434
|
-
function
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
538
|
+
function stopAudio() {
|
|
539
|
+
pendingBlobs.length = 0;
|
|
540
|
+
if (mediaSource) {
|
|
541
|
+
try {
|
|
542
|
+
if (mediaSource.readyState === "open") {
|
|
543
|
+
mediaSource.endOfStream();
|
|
544
|
+
}
|
|
545
|
+
if (sourceBuffer && mediaSource.readyState !== "closed") {
|
|
546
|
+
mediaSource.removeSourceBuffer(sourceBuffer);
|
|
547
|
+
}
|
|
548
|
+
} catch (error) {
|
|
549
|
+
console.error("Error cleaning up media source", error);
|
|
550
|
+
}
|
|
551
|
+
mediaSource = void 0;
|
|
552
|
+
sourceBuffer = void 0;
|
|
553
|
+
}
|
|
554
|
+
if (audioElement) {
|
|
555
|
+
audioElement.pause();
|
|
556
|
+
audioElement.src = "";
|
|
557
|
+
audioElement = void 0;
|
|
558
|
+
}
|
|
559
|
+
if (currentMediaStream) {
|
|
560
|
+
currentMediaStream.getTracks().forEach((track) => track.stop());
|
|
561
|
+
currentMediaStream = void 0;
|
|
440
562
|
}
|
|
441
|
-
audioBlobSourceNode.disconnect();
|
|
442
|
-
audioBlobSourceNode = void 0;
|
|
443
563
|
}
|
|
444
564
|
|
|
445
|
-
// src/
|
|
565
|
+
// src/audio/vad/SileroVAD.ts
|
|
566
|
+
var import_src = require("@ricky0123/vad-web/src");
|
|
567
|
+
|
|
568
|
+
// src/audio/vad/VAD.ts
|
|
446
569
|
var import_eventemitter33 = __toESM(require("eventemitter3"));
|
|
570
|
+
var VAD = class extends import_eventemitter33.default {
|
|
571
|
+
/**
|
|
572
|
+
* Constructor for the VAD class
|
|
573
|
+
* @param threshold - The threshold to use for the VAD
|
|
574
|
+
*/
|
|
575
|
+
constructor(threshold = 0) {
|
|
576
|
+
super();
|
|
577
|
+
// Max delay to detect speech, can be used to record delayed stream
|
|
578
|
+
this.delay = 100;
|
|
579
|
+
this.threshold = threshold;
|
|
580
|
+
}
|
|
581
|
+
};
|
|
582
|
+
|
|
583
|
+
// src/audio/vad/SileroVAD.ts
|
|
584
|
+
var SileroVAD = class extends VAD {
|
|
585
|
+
get isStarted() {
|
|
586
|
+
return !!this.vad;
|
|
587
|
+
}
|
|
588
|
+
async start(stream, threshold) {
|
|
589
|
+
if (this.vad) return;
|
|
590
|
+
if (threshold !== void 0) {
|
|
591
|
+
this.threshold = threshold;
|
|
592
|
+
}
|
|
593
|
+
const speechThreshold = Math.min(
|
|
594
|
+
Math.max((this.threshold + 100) / 100, 0),
|
|
595
|
+
1
|
|
596
|
+
);
|
|
597
|
+
this.vad = await import_src.MicVAD.new({
|
|
598
|
+
stream,
|
|
599
|
+
onSpeechStart: () => this.emit("StartSpeaking"),
|
|
600
|
+
onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
|
|
601
|
+
onVADMisfire: () => this.emit("CancelSpeaking"),
|
|
602
|
+
onSpeechEnd: () => this.emit("StopSpeaking"),
|
|
603
|
+
positiveSpeechThreshold: speechThreshold,
|
|
604
|
+
negativeSpeechThreshold: speechThreshold * 0.7,
|
|
605
|
+
// Adjust negative threshold proportionally
|
|
606
|
+
redemptionFrames: 5
|
|
607
|
+
});
|
|
608
|
+
this.vad.start();
|
|
609
|
+
}
|
|
610
|
+
async stop() {
|
|
611
|
+
if (!this.vad) return;
|
|
612
|
+
this.vad.pause();
|
|
613
|
+
this.vad = void 0;
|
|
614
|
+
}
|
|
615
|
+
async setThreshold(threshold) {
|
|
616
|
+
if (threshold === this.threshold) return;
|
|
617
|
+
if (this.vad) {
|
|
618
|
+
const stream = this.vad["stream"];
|
|
619
|
+
await this.stop();
|
|
620
|
+
await this.start(stream, threshold);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
|
|
625
|
+
// src/audio/vad/VolumeVAD.ts
|
|
626
|
+
var VolumeVAD = class extends VAD {
|
|
627
|
+
constructor() {
|
|
628
|
+
super();
|
|
629
|
+
this.running = false;
|
|
630
|
+
this.speaking = false;
|
|
631
|
+
this.attemptSpeaking = false;
|
|
632
|
+
this.history = 10;
|
|
633
|
+
this.speakingHistory = [];
|
|
634
|
+
for (let i = 0; i < this.history; i++) {
|
|
635
|
+
this.speakingHistory.push(0);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
getMaxVolume() {
|
|
639
|
+
if (!this.analyser || !this.fftBins) return -Infinity;
|
|
640
|
+
let maxVolume = -Infinity;
|
|
641
|
+
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
642
|
+
for (let i = 4; i < this.fftBins.length; i++) {
|
|
643
|
+
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
644
|
+
maxVolume = this.fftBins[i];
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
return maxVolume;
|
|
648
|
+
}
|
|
649
|
+
get isStarted() {
|
|
650
|
+
return this.running;
|
|
651
|
+
}
|
|
652
|
+
async start(stream, threshold) {
|
|
653
|
+
if (this.running) return;
|
|
654
|
+
if (threshold !== void 0) {
|
|
655
|
+
this.threshold = threshold;
|
|
656
|
+
}
|
|
657
|
+
this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
|
658
|
+
this.analyser = this.audioContext.createAnalyser();
|
|
659
|
+
this.sourceNode = this.audioContext.createMediaStreamSource(stream);
|
|
660
|
+
this.analyser.fftSize = 512;
|
|
661
|
+
this.analyser.smoothingTimeConstant = 0.1;
|
|
662
|
+
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
663
|
+
this.sourceNode.connect(this.analyser);
|
|
664
|
+
this.running = true;
|
|
665
|
+
this.startLoop();
|
|
666
|
+
}
|
|
667
|
+
startLoop() {
|
|
668
|
+
if (!this.running) return;
|
|
669
|
+
const currentVolume = this.getMaxVolume();
|
|
670
|
+
if (currentVolume > this.threshold) {
|
|
671
|
+
if (!this.speaking) {
|
|
672
|
+
let recentHistory = 0;
|
|
673
|
+
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
674
|
+
recentHistory += this.speakingHistory[i];
|
|
675
|
+
}
|
|
676
|
+
if (recentHistory >= 2) {
|
|
677
|
+
this.speaking = true;
|
|
678
|
+
this.attemptSpeaking = false;
|
|
679
|
+
this.emit("ConfirmSpeaking");
|
|
680
|
+
} else if (recentHistory === 1) {
|
|
681
|
+
this.attemptSpeaking = true;
|
|
682
|
+
this.emit("StartSpeaking");
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
} else if (this.speaking) {
|
|
686
|
+
let totalHistory = 0;
|
|
687
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
688
|
+
totalHistory += this.speakingHistory[i];
|
|
689
|
+
}
|
|
690
|
+
if (totalHistory === 0) {
|
|
691
|
+
this.speaking = false;
|
|
692
|
+
this.emit("StopSpeaking");
|
|
693
|
+
}
|
|
694
|
+
} else if (this.attemptSpeaking) {
|
|
695
|
+
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
696
|
+
if (lastHistory === 0) {
|
|
697
|
+
this.attemptSpeaking = false;
|
|
698
|
+
this.emit("CancelSpeaking");
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
this.speakingHistory.shift();
|
|
702
|
+
this.speakingHistory.push(Number(currentVolume > this.threshold));
|
|
703
|
+
setTimeout(() => this.startLoop(), this.delay);
|
|
704
|
+
}
|
|
705
|
+
async stop() {
|
|
706
|
+
if (!this.running) return;
|
|
707
|
+
this.running = false;
|
|
708
|
+
this.speaking = false;
|
|
709
|
+
if (this.sourceNode) {
|
|
710
|
+
this.sourceNode.disconnect();
|
|
711
|
+
this.sourceNode = void 0;
|
|
712
|
+
}
|
|
713
|
+
if (this.analyser) {
|
|
714
|
+
this.analyser.disconnect();
|
|
715
|
+
this.analyser = void 0;
|
|
716
|
+
}
|
|
717
|
+
if (this.audioContext) {
|
|
718
|
+
await this.audioContext.close();
|
|
719
|
+
this.audioContext = void 0;
|
|
720
|
+
}
|
|
721
|
+
this.fftBins = void 0;
|
|
722
|
+
}
|
|
723
|
+
async setThreshold(threshold) {
|
|
724
|
+
if (threshold === this.threshold) return;
|
|
725
|
+
this.threshold = threshold;
|
|
726
|
+
}
|
|
727
|
+
};
|
|
728
|
+
|
|
729
|
+
// src/call/CallHandler.ts
|
|
730
|
+
var import_eventemitter34 = __toESM(require("eventemitter3"));
|
|
447
731
|
|
|
448
732
|
// src/call/CallHandlerError.ts
|
|
449
733
|
var CallHandlerErrorCode = /* @__PURE__ */ ((CallHandlerErrorCode2) => {
|
|
@@ -460,36 +744,42 @@ var CallHandlerError = class extends Error {
|
|
|
460
744
|
};
|
|
461
745
|
|
|
462
746
|
// src/call/CallHandler.ts
|
|
463
|
-
var CallHandler = class _CallHandler extends
|
|
464
|
-
constructor() {
|
|
747
|
+
var CallHandler = class _CallHandler extends import_eventemitter34.default {
|
|
748
|
+
constructor(options) {
|
|
465
749
|
super();
|
|
466
|
-
this.micRecorder = new MicRecorder();
|
|
467
750
|
this.conversation = [];
|
|
468
751
|
this.debug = false;
|
|
469
752
|
this.startTime = 0;
|
|
753
|
+
this.micRecorder = new MicRecorder(this.getVAD(options?.vad));
|
|
470
754
|
this.micRecorder.on("StateChange", () => {
|
|
471
755
|
this.notifyStateChange();
|
|
472
756
|
});
|
|
473
757
|
this.micRecorder.on("Chunk", (blob) => {
|
|
474
|
-
this.log(`[Mic]
|
|
758
|
+
this.log(`[Mic] Received chunk`, blob);
|
|
475
759
|
this.ws?.send(blob);
|
|
476
760
|
});
|
|
477
761
|
this.micRecorder.on("StartSpeaking", () => {
|
|
478
|
-
this.log("[Mic]
|
|
762
|
+
this.log("[Mic] Start speaking");
|
|
479
763
|
this.ws?.send("startSpeaking" /* StartSpeaking */);
|
|
480
|
-
|
|
764
|
+
stopAudio();
|
|
481
765
|
});
|
|
482
766
|
this.micRecorder.on("StopSpeaking", () => {
|
|
483
|
-
this.log("[Mic]
|
|
767
|
+
this.log("[Mic] Stop speaking");
|
|
484
768
|
this.ws?.send("stopSpeaking" /* StopSpeaking */);
|
|
485
769
|
});
|
|
486
770
|
}
|
|
487
|
-
static getInstance() {
|
|
771
|
+
static getInstance(options) {
|
|
488
772
|
if (!_CallHandler.instance) {
|
|
489
|
-
_CallHandler.instance = new _CallHandler();
|
|
773
|
+
_CallHandler.instance = new _CallHandler(options);
|
|
490
774
|
}
|
|
491
775
|
return _CallHandler.instance;
|
|
492
776
|
}
|
|
777
|
+
getVAD(vad) {
|
|
778
|
+
if (!vad) return new VolumeVAD();
|
|
779
|
+
if (vad === "volume") return new VolumeVAD();
|
|
780
|
+
if (vad === "silero") return new SileroVAD();
|
|
781
|
+
return vad;
|
|
782
|
+
}
|
|
493
783
|
get isStarted() {
|
|
494
784
|
return this.isWSStarted && this.micRecorder.state.isStarted;
|
|
495
785
|
}
|
|
@@ -527,11 +817,13 @@ var CallHandler = class _CallHandler extends import_eventemitter33.default {
|
|
|
527
817
|
pause() {
|
|
528
818
|
this.micRecorder.mute();
|
|
529
819
|
this.notifyStateChange();
|
|
820
|
+
pauseAudio();
|
|
530
821
|
}
|
|
531
822
|
resume() {
|
|
532
823
|
if (this.lastAudioBlob) {
|
|
533
|
-
|
|
824
|
+
playAudio(this.lastAudioBlob);
|
|
534
825
|
}
|
|
826
|
+
resumeAudio();
|
|
535
827
|
this.micRecorder.unmute();
|
|
536
828
|
this.notifyStateChange();
|
|
537
829
|
}
|
|
@@ -580,7 +872,7 @@ var CallHandler = class _CallHandler extends import_eventemitter33.default {
|
|
|
580
872
|
this.ws.onmessage = (event) => {
|
|
581
873
|
this.log("[WS]", event.data);
|
|
582
874
|
if (event.data instanceof Blob) {
|
|
583
|
-
|
|
875
|
+
playAudio(event.data);
|
|
584
876
|
this.lastAudioBlob = event.data;
|
|
585
877
|
} else if (typeof event.data !== "string") {
|
|
586
878
|
console.warn(`[WS] Unknown message type: ${event.data}`);
|
|
@@ -665,6 +957,9 @@ var CallServerCommands = /* @__PURE__ */ ((CallServerCommands2) => {
|
|
|
665
957
|
LocalStorageKeys,
|
|
666
958
|
Mic,
|
|
667
959
|
MicRecorder,
|
|
668
|
-
|
|
960
|
+
SileroVAD,
|
|
961
|
+
Speaker,
|
|
962
|
+
VAD,
|
|
963
|
+
VolumeVAD
|
|
669
964
|
});
|
|
670
965
|
//# sourceMappingURL=index.js.map
|