@micdrop/client 1.4.1 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1,21 +1,3 @@
1
- var __defProp = Object.defineProperty;
2
- var __export = (target, all) => {
3
- for (var name in all)
4
- __defProp(target, name, { get: all[name], enumerable: true });
5
- };
6
-
7
- // src/audio/MicRecorder.ts
8
- import EventEmitter2 from "eventemitter3";
9
-
10
- // src/audio/microphone.ts
11
- var microphone_exports = {};
12
- __export(microphone_exports, {
13
- defaultMicThreshold: () => defaultMicThreshold,
14
- micAnalyser: () => micAnalyser,
15
- startMicrophone: () => startMicrophone,
16
- stopMicrophone: () => stopMicrophone
17
- });
18
-
19
1
  // src/audio/utils/AudioAnalyser.ts
20
2
  import EventEmitter from "eventemitter3";
21
3
  var AudioAnalyser = class extends EventEmitter {
@@ -51,8 +33,42 @@ var AudioAnalyser = class extends EventEmitter {
51
33
  }
52
34
  };
53
35
 
36
+ // src/audio/utils/unlock.ts
37
+ function unlock(audioContext2) {
38
+ const events = ["touchstart", "touchend", "mousedown", "keydown"];
39
+ const unlock2 = async () => {
40
+ unlockSafari(audioContext2);
41
+ if (audioContext2.state === "suspended") {
42
+ await audioContext2.resume();
43
+ }
44
+ events.forEach((e) => document.body.removeEventListener(e, unlock2));
45
+ };
46
+ events.forEach((e) => document.body.addEventListener(e, unlock2, false));
47
+ }
48
+ function unlockSafari(audioContext2) {
49
+ const source = audioContext2.createBufferSource();
50
+ source.buffer = audioContext2.createBuffer(1, 1, 22050);
51
+ source.connect(audioContext2.destination);
52
+ const onEnded = () => {
53
+ source.disconnect();
54
+ source.buffer = null;
55
+ source.removeEventListener("ended", onEnded);
56
+ };
57
+ source.addEventListener("ended", onEnded);
58
+ source.start();
59
+ }
60
+
54
61
  // src/audio/utils/audioContext.ts
55
- var audioContext = new (window.AudioContext || window.webkitAudioContext)();
62
+ var audioContext;
63
+ function createAudioContext() {
64
+ return new (window.AudioContext || window.webkitAudioContext)();
65
+ }
66
+ if (window.micdropAudioContext) {
67
+ audioContext = window.micdropAudioContext;
68
+ } else {
69
+ window.micdropAudioContext = audioContext = createAudioContext();
70
+ unlock(audioContext);
71
+ }
56
72
 
57
73
  // src/audio/utils/localStorage.ts
58
74
  var LocalStorageKeys = /* @__PURE__ */ ((LocalStorageKeys2) => {
@@ -67,71 +83,86 @@ function stopStream(stream) {
67
83
  stream?.getTracks().forEach((track) => track.stop());
68
84
  }
69
85
 
70
- // src/audio/microphone.ts
71
- var currentDeviceId;
72
- var audioStream;
73
- var sourceNode;
74
- var micAnalyser = new AudioAnalyser(audioContext);
86
+ // src/audio/mic.ts
87
+ var mic;
75
88
  var defaultMicThreshold = -50;
76
- function getMicConstraints(deviceId) {
77
- return {
78
- audio: {
79
- deviceId: { ideal: deviceId },
80
- sampleRate: 16e3,
81
- // not working, it will follow device settings, usually 44.1kHz
82
- sampleSize: 16,
83
- channelCount: 1,
84
- echoCancellation: true,
85
- noiseSuppression: true
86
- }
87
- };
88
- }
89
- async function startMicrophone(deviceId) {
90
- if (!audioContext || !micAnalyser) {
91
- throw new Error("AudioContext not initialized");
92
- }
93
- if (!deviceId) {
94
- deviceId = localStorage.getItem("micdrop.micDevice" /* MicDevice */) || void 0;
89
+ var Mic = class {
90
+ constructor() {
91
+ this.defaultThreshold = defaultMicThreshold;
92
+ this.analyser = new AudioAnalyser(audioContext);
93
+ }
94
+ getMicConstraints(deviceId) {
95
+ return {
96
+ audio: {
97
+ deviceId: { ideal: deviceId },
98
+ sampleRate: 16e3,
99
+ // not working, it will follow device settings, usually 44.1kHz
100
+ sampleSize: 16,
101
+ channelCount: 1,
102
+ echoCancellation: true,
103
+ noiseSuppression: true
104
+ }
105
+ };
95
106
  }
96
- if (audioStream) {
97
- if (currentDeviceId === deviceId) {
98
- return audioStream;
107
+ /**
108
+ * Starts the microphone
109
+ * @param deviceId - The deviceId to use
110
+ * @returns The microphone stream
111
+ */
112
+ async start(deviceId) {
113
+ if (!audioContext || !this.analyser) {
114
+ throw new Error("AudioContext not initialized");
99
115
  }
100
- stopStream(audioStream);
101
- }
102
- audioStream = await navigator.mediaDevices.getUserMedia(
103
- getMicConstraints(deviceId)
104
- );
105
- const track = audioStream.getTracks()[0];
106
- const newDeviceId = track && track.getCapabilities && track.getCapabilities().deviceId;
107
- if (newDeviceId) {
108
- localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
109
- currentDeviceId = newDeviceId;
110
- } else {
111
- localStorage.removeItem("micdrop.micDevice" /* MicDevice */);
112
- currentDeviceId = void 0;
113
- }
114
- if (sourceNode) sourceNode.disconnect();
115
- sourceNode = audioContext.createMediaStreamSource(audioStream);
116
- sourceNode.connect(micAnalyser.node);
117
- return audioStream;
118
- }
119
- function stopMicrophone() {
120
- if (sourceNode) {
121
- sourceNode.disconnect();
122
- sourceNode = void 0;
116
+ if (!deviceId) {
117
+ deviceId = localStorage.getItem("micdrop.micDevice" /* MicDevice */) || void 0;
118
+ }
119
+ if (this.audioStream) {
120
+ if (this.currentDeviceId === deviceId) {
121
+ return this.audioStream;
122
+ }
123
+ stopStream(this.audioStream);
124
+ }
125
+ this.audioStream = await navigator.mediaDevices.getUserMedia(
126
+ this.getMicConstraints(deviceId)
127
+ );
128
+ const track = this.audioStream.getTracks()[0];
129
+ const newDeviceId = track && track.getCapabilities && track.getCapabilities().deviceId;
130
+ if (newDeviceId) {
131
+ localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
132
+ this.currentDeviceId = newDeviceId;
133
+ } else {
134
+ localStorage.removeItem("micdrop.micDevice" /* MicDevice */);
135
+ this.currentDeviceId = void 0;
136
+ }
137
+ if (this.sourceNode) this.sourceNode.disconnect();
138
+ this.sourceNode = audioContext.createMediaStreamSource(this.audioStream);
139
+ this.sourceNode.connect(this.analyser.node);
140
+ return this.audioStream;
123
141
  }
124
- if (audioStream) {
125
- stopStream(audioStream);
126
- audioStream = void 0;
142
+ /**
143
+ * Stops the microphone
144
+ */
145
+ stop() {
146
+ if (this.sourceNode) {
147
+ this.sourceNode.disconnect();
148
+ this.sourceNode = void 0;
149
+ }
150
+ if (this.audioStream) {
151
+ stopStream(this.audioStream);
152
+ this.audioStream = void 0;
153
+ }
127
154
  }
155
+ };
156
+ if (!window.micdropMic) {
157
+ window.micdropMic = new Mic();
128
158
  }
159
+ mic = window.micdropMic;
129
160
 
130
- // src/audio/utils/createDelayedStream.ts
161
+ // src/audio/MicRecorder.ts
162
+ import EventEmitter3 from "eventemitter3";
163
+
164
+ // src/audio/utils/delayedStream.ts
131
165
  function createDelayedStream(stream, delay) {
132
- if (!audioContext) {
133
- throw new Error("AudioContext not initialized");
134
- }
135
166
  const audioSource = audioContext.createMediaStreamSource(stream);
136
167
  const delayNode = audioContext.createDelay(delay);
137
168
  delayNode.delayTime.value = delay;
@@ -141,6 +172,166 @@ function createDelayedStream(stream, delay) {
141
172
  return audioDestination.stream;
142
173
  }
143
174
 
175
+ // src/audio/vad/SileroVAD.ts
176
+ import { MicVAD } from "@ricky0123/vad-web/src";
177
+
178
+ // src/audio/vad/VAD.ts
179
+ import EventEmitter2 from "eventemitter3";
180
+ var VAD = class extends EventEmitter2 {
181
+ /**
182
+ * Constructor for the VAD class
183
+ * @param threshold - The threshold to use for the VAD
184
+ */
185
+ constructor(threshold = 0) {
186
+ super();
187
+ // Max delay to detect speech, can be used to record delayed stream
188
+ this.delay = 100;
189
+ this.threshold = threshold;
190
+ }
191
+ };
192
+
193
+ // src/audio/vad/SileroVAD.ts
194
+ var SileroVAD = class extends VAD {
195
+ get isStarted() {
196
+ return !!this.vad;
197
+ }
198
+ async start(stream, threshold) {
199
+ if (this.vad) return;
200
+ if (threshold !== void 0) {
201
+ this.threshold = threshold;
202
+ }
203
+ const speechThreshold = Math.min(
204
+ Math.max((this.threshold + 100) / 100, 0),
205
+ 1
206
+ );
207
+ this.vad = await MicVAD.new({
208
+ stream,
209
+ model: "v5",
210
+ onSpeechStart: () => this.emit("StartSpeaking"),
211
+ onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
212
+ onVADMisfire: () => this.emit("CancelSpeaking"),
213
+ onSpeechEnd: () => this.emit("StopSpeaking"),
214
+ positiveSpeechThreshold: speechThreshold,
215
+ negativeSpeechThreshold: speechThreshold * 0.7,
216
+ minSpeechFrames: 3,
217
+ redemptionFrames: 9
218
+ });
219
+ this.vad.start();
220
+ }
221
+ async stop() {
222
+ if (!this.vad) return;
223
+ this.vad.destroy();
224
+ this.vad = void 0;
225
+ }
226
+ async setThreshold(threshold) {
227
+ if (threshold === this.threshold) return;
228
+ if (this.vad) {
229
+ const stream = this.vad["stream"];
230
+ await this.stop();
231
+ await this.start(stream, threshold);
232
+ }
233
+ }
234
+ };
235
+
236
+ // src/audio/vad/VolumeVAD.ts
237
+ var VolumeVAD = class extends VAD {
238
+ constructor() {
239
+ super();
240
+ this.running = false;
241
+ this.speaking = false;
242
+ this.attemptSpeaking = false;
243
+ this.history = 10;
244
+ this.speakingHistory = [];
245
+ for (let i = 0; i < this.history; i++) {
246
+ this.speakingHistory.push(0);
247
+ }
248
+ }
249
+ getMaxVolume() {
250
+ if (!this.analyser || !this.fftBins) return -Infinity;
251
+ let maxVolume = -Infinity;
252
+ this.analyser.getFloatFrequencyData(this.fftBins);
253
+ for (let i = 4; i < this.fftBins.length; i++) {
254
+ if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
255
+ maxVolume = this.fftBins[i];
256
+ }
257
+ }
258
+ return maxVolume;
259
+ }
260
+ get isStarted() {
261
+ return this.running;
262
+ }
263
+ async start(stream, threshold) {
264
+ if (this.running) return;
265
+ if (threshold !== void 0) {
266
+ this.threshold = threshold;
267
+ }
268
+ this.analyser = audioContext.createAnalyser();
269
+ this.sourceNode = audioContext.createMediaStreamSource(stream);
270
+ this.analyser.fftSize = 512;
271
+ this.analyser.smoothingTimeConstant = 0.1;
272
+ this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
273
+ this.sourceNode.connect(this.analyser);
274
+ this.running = true;
275
+ this.startLoop();
276
+ }
277
+ startLoop() {
278
+ if (!this.running) return;
279
+ const currentVolume = this.getMaxVolume();
280
+ if (currentVolume > this.threshold) {
281
+ if (!this.speaking) {
282
+ let recentHistory = 0;
283
+ for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
284
+ recentHistory += this.speakingHistory[i];
285
+ }
286
+ if (recentHistory >= 2) {
287
+ this.speaking = true;
288
+ this.attemptSpeaking = false;
289
+ this.emit("ConfirmSpeaking");
290
+ } else if (recentHistory === 1) {
291
+ this.attemptSpeaking = true;
292
+ this.emit("StartSpeaking");
293
+ }
294
+ }
295
+ } else if (this.speaking) {
296
+ let totalHistory = 0;
297
+ for (let i = 0; i < this.speakingHistory.length; i++) {
298
+ totalHistory += this.speakingHistory[i];
299
+ }
300
+ if (totalHistory === 0) {
301
+ this.speaking = false;
302
+ this.emit("StopSpeaking");
303
+ }
304
+ } else if (this.attemptSpeaking) {
305
+ let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
306
+ if (lastHistory === 0) {
307
+ this.attemptSpeaking = false;
308
+ this.emit("CancelSpeaking");
309
+ }
310
+ }
311
+ this.speakingHistory.shift();
312
+ this.speakingHistory.push(Number(currentVolume > this.threshold));
313
+ setTimeout(() => this.startLoop(), this.delay);
314
+ }
315
+ async stop() {
316
+ if (!this.running) return;
317
+ this.running = false;
318
+ this.speaking = false;
319
+ if (this.sourceNode) {
320
+ this.sourceNode.disconnect();
321
+ this.sourceNode = void 0;
322
+ }
323
+ if (this.analyser) {
324
+ this.analyser.disconnect();
325
+ this.analyser = void 0;
326
+ }
327
+ this.fftBins = void 0;
328
+ }
329
+ async setThreshold(threshold) {
330
+ if (threshold === this.threshold) return;
331
+ this.threshold = threshold;
332
+ }
333
+ };
334
+
144
335
  // src/audio/MicRecorder.ts
145
336
  var timeSlice = 100;
146
337
  var defaultMicRecorderState = {
@@ -150,9 +341,10 @@ var defaultMicRecorderState = {
150
341
  isSpeaking: false,
151
342
  threshold: defaultMicThreshold
152
343
  };
153
- var MicRecorder = class extends EventEmitter2 {
154
- constructor(vad) {
344
+ var MicRecorder = class extends EventEmitter3 {
345
+ constructor(vadConfig) {
155
346
  super();
347
+ this.vadConfig = vadConfig;
156
348
  this.audioInfo = this.getAudioInfo();
157
349
  this.speakingConfirmed = false;
158
350
  this.queuedChunks = [];
@@ -193,7 +385,6 @@ var MicRecorder = class extends EventEmitter2 {
193
385
  }).bind(this);
194
386
  const threshold = parseFloat(localStorage.getItem("micdrop.micThreshold" /* MicThreshold */) || "0") || defaultMicThreshold;
195
387
  this.state = { ...defaultMicRecorderState, threshold };
196
- this.vad = vad;
197
388
  }
198
389
  async start(stream) {
199
390
  if (this.state.isStarted) {
@@ -205,6 +396,9 @@ var MicRecorder = class extends EventEmitter2 {
205
396
  isMuted: false,
206
397
  isSpeaking: false
207
398
  });
399
+ if (!this.vad) {
400
+ this.vad = this.initVAD(this.vadConfig);
401
+ }
208
402
  const delayedStream = createDelayedStream(
209
403
  stream,
210
404
  this.vad.delay / 1e3 + 0.05
@@ -259,9 +453,18 @@ var MicRecorder = class extends EventEmitter2 {
259
453
  setThreshold(threshold) {
260
454
  this.changeState({ threshold });
261
455
  localStorage.setItem("micdrop.micThreshold" /* MicThreshold */, threshold.toString());
262
- this.vad.setThreshold(threshold);
456
+ this.vad?.setThreshold(threshold);
457
+ }
458
+ initVAD(vad) {
459
+ if (!vad) return new VolumeVAD();
460
+ if (vad === "volume") return new VolumeVAD();
461
+ if (vad === "silero") return new SileroVAD();
462
+ return vad;
263
463
  }
264
464
  async startSpeakingDetection(stream) {
465
+ if (!this.vad) {
466
+ throw new Error("VAD not initialized");
467
+ }
265
468
  if (this.vad.isStarted) {
266
469
  this.vad.stop();
267
470
  }
@@ -272,6 +475,9 @@ var MicRecorder = class extends EventEmitter2 {
272
475
  await this.vad.start(stream, this.state.threshold);
273
476
  }
274
477
  async stopSpeakingDetection() {
478
+ if (!this.vad) {
479
+ throw new Error("VAD not initialized");
480
+ }
275
481
  this.vad.off("StartSpeaking", this.onStartSpeaking);
276
482
  this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
277
483
  this.vad.off("CancelSpeaking", this.onCancelSpeaking);
@@ -309,21 +515,6 @@ var MicRecorder = class extends EventEmitter2 {
309
515
  }
310
516
  };
311
517
 
312
- // src/audio/speaker.ts
313
- var speaker_exports = {};
314
- __export(speaker_exports, {
315
- canChangeSpeakerDevice: () => canChangeSpeakerDevice,
316
- changeSpeakerDevice: () => changeSpeakerDevice,
317
- disableStreaming: () => disableStreaming,
318
- enableStreaming: () => enableStreaming,
319
- isStreamingEnabled: () => isStreamingEnabled,
320
- pauseAudio: () => pauseAudio,
321
- playAudio: () => playAudio,
322
- resumeAudio: () => resumeAudio,
323
- speakerAnalyser: () => speakerAnalyser,
324
- stopAudio: () => stopAudio
325
- });
326
-
327
518
  // src/audio/player/SpeakerPlayer.ts
328
519
  var SpeakerPlayer = class {
329
520
  constructor(outputNode) {
@@ -514,7 +705,7 @@ var SpeakerMediaSourcePlayer = class extends SpeakerPlayer {
514
705
  this.audioElement = void 0;
515
706
  }
516
707
  if (this.mediaStream) {
517
- this.mediaStream.getTracks().forEach((track) => track.stop());
708
+ stopStream(this.mediaStream);
518
709
  this.mediaStream = void 0;
519
710
  }
520
711
  this.mediaSource = void 0;
@@ -579,280 +770,94 @@ var SpeakerSequencePlayer = class extends SpeakerPlayer {
579
770
  }
580
771
  };
581
772
 
582
- // src/audio/utils/unlock.ts
583
- function unlock(audioContext2) {
584
- const events = ["touchstart", "touchend", "mousedown", "keydown"];
585
- events.forEach((e) => document.body.addEventListener(e, unlock2, false));
586
- async function unlock2() {
587
- unlockSafari(audioContext2);
588
- if (audioContext2.state === "suspended") {
589
- await audioContext2.resume();
773
+ // src/audio/speaker.ts
774
+ var speaker;
775
+ var Speaker = class {
776
+ constructor() {
777
+ this.streamingEnabled = false;
778
+ this.analyser = new AudioAnalyser(audioContext);
779
+ this.audioElement = new Audio();
780
+ this.audioElement.autoplay = true;
781
+ const speakerId = localStorage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
782
+ if (speakerId) {
783
+ this.setSinkId(speakerId).catch((error) => {
784
+ console.error("Failed to set sink ID:", error);
785
+ });
590
786
  }
591
- clean();
592
- }
593
- function clean() {
594
- events.forEach((e) => document.body.removeEventListener(e, unlock2));
787
+ this.setupSpeakerPlayer();
788
+ const destinationNode = audioContext.createMediaStreamDestination();
789
+ this.analyser.node.connect(destinationNode);
790
+ this.audioElement.srcObject = destinationNode.stream;
595
791
  }
596
- }
597
- function unlockSafari(audioContext2) {
598
- const source = audioContext2.createBufferSource();
599
- source.buffer = audioContext2.createBuffer(1, 1, 22050);
600
- source.connect(audioContext2.destination);
601
- const onEnded = () => {
602
- source.disconnect();
603
- source.buffer = null;
604
- source.removeEventListener("ended", onEnded);
605
- };
606
- source.addEventListener("ended", onEnded);
607
- source.start();
608
- }
609
-
610
- // src/audio/speaker.ts
611
- var audioElement;
612
- var player;
613
- var streamingEnabled = false;
614
- var speakerAnalyser = new AudioAnalyser(audioContext);
615
- async function init() {
616
- unlock(audioContext);
617
- audioElement = new Audio();
618
- audioElement.autoplay = true;
619
- const speakerId = localStorage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
620
- if (speakerId) {
792
+ async setupSpeakerPlayer() {
621
793
  try {
622
- await setSinkId(speakerId);
794
+ this.player?.destroy();
795
+ this.player = this.streamingEnabled ? SpeakerMediaSourcePlayer.isCompatible ? new SpeakerMediaSourcePlayer(this.analyser.node) : new SpeakerConcatPlayer(this.analyser.node) : new SpeakerSequencePlayer(this.analyser.node);
796
+ await this.player?.setup();
623
797
  } catch (error) {
624
- console.error("Failed to set sink ID:", error);
625
- }
626
- }
627
- await setupSpeakerPlayer();
628
- const destinationNode = audioContext.createMediaStreamDestination();
629
- speakerAnalyser.node.connect(destinationNode);
630
- audioElement.srcObject = destinationNode.stream;
631
- }
632
- init();
633
- async function setupSpeakerPlayer() {
634
- try {
635
- player?.destroy();
636
- player = streamingEnabled ? SpeakerMediaSourcePlayer.isCompatible ? new SpeakerMediaSourcePlayer(speakerAnalyser.node) : new SpeakerConcatPlayer(speakerAnalyser.node) : new SpeakerSequencePlayer(speakerAnalyser.node);
637
- await player?.setup();
638
- } catch (error) {
639
- console.error("Error setting up speaker player", error);
640
- }
641
- }
642
- async function isStreamingEnabled() {
643
- return streamingEnabled;
644
- }
645
- async function enableStreaming() {
646
- if (streamingEnabled) return;
647
- streamingEnabled = true;
648
- await setupSpeakerPlayer();
649
- }
650
- async function disableStreaming() {
651
- if (!streamingEnabled) return;
652
- streamingEnabled = false;
653
- await setupSpeakerPlayer();
654
- }
655
- function canChangeSpeakerDevice() {
656
- return window.HTMLMediaElement && "sinkId" in window.HTMLMediaElement.prototype;
657
- }
658
- async function setSinkId(speakerId) {
659
- if (speakerId && canChangeSpeakerDevice() && audioElement) {
660
- return audioElement.setSinkId(speakerId);
661
- }
662
- }
663
- async function changeSpeakerDevice(speakerId) {
664
- if (!canChangeSpeakerDevice()) return;
665
- try {
666
- await setSinkId(speakerId);
667
- localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, speakerId);
668
- } catch (error) {
669
- console.error(`Error setting Audio Output to ${speakerId}`, error);
670
- }
671
- }
672
- function pauseAudio() {
673
- if (audioElement) {
674
- audioElement.pause();
675
- }
676
- }
677
- function resumeAudio() {
678
- if (audioElement) {
679
- audioElement.play();
680
- }
681
- }
682
- async function playAudio(blob) {
683
- if (!audioElement || !player) return;
684
- try {
685
- player.addBlob(blob);
686
- if (audioElement.paused) {
687
- audioElement.play();
798
+ console.error("Error setting up speaker player", error);
688
799
  }
689
- } catch (error) {
690
- console.error("Error playing audio blob", error, blob);
691
800
  }
692
- }
693
- function stopAudio() {
694
- audioElement?.pause();
695
- player?.stop();
696
- }
697
-
698
- // src/audio/vad/SileroVAD.ts
699
- import { MicVAD } from "@ricky0123/vad-web/src";
700
-
701
- // src/audio/vad/VAD.ts
702
- import EventEmitter3 from "eventemitter3";
703
- var VAD = class extends EventEmitter3 {
704
- /**
705
- * Constructor for the VAD class
706
- * @param threshold - The threshold to use for the VAD
707
- */
708
- constructor(threshold = 0) {
709
- super();
710
- // Max delay to detect speech, can be used to record delayed stream
711
- this.delay = 100;
712
- this.threshold = threshold;
801
+ get isStreamingEnabled() {
802
+ return this.streamingEnabled;
713
803
  }
714
- };
715
-
716
- // src/audio/vad/SileroVAD.ts
717
- var SileroVAD = class extends VAD {
718
- get isStarted() {
719
- return !!this.vad;
804
+ async enableStreaming() {
805
+ if (this.streamingEnabled) return;
806
+ this.streamingEnabled = true;
807
+ await this.setupSpeakerPlayer();
720
808
  }
721
- async start(stream, threshold) {
722
- if (this.vad) return;
723
- if (threshold !== void 0) {
724
- this.threshold = threshold;
725
- }
726
- const speechThreshold = Math.min(
727
- Math.max((this.threshold + 100) / 100, 0),
728
- 1
729
- );
730
- this.vad = await MicVAD.new({
731
- stream,
732
- onSpeechStart: () => this.emit("StartSpeaking"),
733
- onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
734
- onVADMisfire: () => this.emit("CancelSpeaking"),
735
- onSpeechEnd: () => this.emit("StopSpeaking"),
736
- positiveSpeechThreshold: speechThreshold,
737
- negativeSpeechThreshold: speechThreshold * 0.7,
738
- // Adjust negative threshold proportionally
739
- redemptionFrames: 5
740
- });
741
- this.vad.start();
809
+ async disableStreaming() {
810
+ if (!this.streamingEnabled) return;
811
+ this.streamingEnabled = false;
812
+ await this.setupSpeakerPlayer();
742
813
  }
743
- async stop() {
744
- if (!this.vad) return;
745
- this.vad.pause();
746
- this.vad = void 0;
814
+ canChangeDevice() {
815
+ return window.HTMLMediaElement && "sinkId" in window.HTMLMediaElement.prototype;
747
816
  }
748
- async setThreshold(threshold) {
749
- if (threshold === this.threshold) return;
750
- if (this.vad) {
751
- const stream = this.vad["stream"];
752
- await this.stop();
753
- await this.start(stream, threshold);
817
+ async setSinkId(speakerId) {
818
+ if (speakerId && this.canChangeDevice() && this.audioElement) {
819
+ return this.audioElement.setSinkId(speakerId);
754
820
  }
755
821
  }
756
- };
757
-
758
- // src/audio/vad/VolumeVAD.ts
759
- var VolumeVAD = class extends VAD {
760
- constructor() {
761
- super();
762
- this.running = false;
763
- this.speaking = false;
764
- this.attemptSpeaking = false;
765
- this.history = 10;
766
- this.speakingHistory = [];
767
- for (let i = 0; i < this.history; i++) {
768
- this.speakingHistory.push(0);
822
+ async changeDevice(speakerId) {
823
+ if (!this.canChangeDevice()) return;
824
+ try {
825
+ await this.setSinkId(speakerId);
826
+ localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, speakerId);
827
+ } catch (error) {
828
+ console.error(`Error setting Audio Output to ${speakerId}`, error);
769
829
  }
770
830
  }
771
- getMaxVolume() {
772
- if (!this.analyser || !this.fftBins) return -Infinity;
773
- let maxVolume = -Infinity;
774
- this.analyser.getFloatFrequencyData(this.fftBins);
775
- for (let i = 4; i < this.fftBins.length; i++) {
776
- if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
777
- maxVolume = this.fftBins[i];
778
- }
831
+ pauseAudio() {
832
+ if (this.audioElement) {
833
+ this.audioElement.pause();
779
834
  }
780
- return maxVolume;
781
- }
782
- get isStarted() {
783
- return this.running;
784
835
  }
785
- async start(stream, threshold) {
786
- if (this.running) return;
787
- if (threshold !== void 0) {
788
- this.threshold = threshold;
836
+ resumeAudio() {
837
+ if (this.audioElement) {
838
+ this.audioElement.play();
789
839
  }
790
- this.analyser = audioContext.createAnalyser();
791
- this.sourceNode = audioContext.createMediaStreamSource(stream);
792
- this.analyser.fftSize = 512;
793
- this.analyser.smoothingTimeConstant = 0.1;
794
- this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
795
- this.sourceNode.connect(this.analyser);
796
- this.running = true;
797
- this.startLoop();
798
840
  }
799
- startLoop() {
800
- if (!this.running) return;
801
- const currentVolume = this.getMaxVolume();
802
- if (currentVolume > this.threshold) {
803
- if (!this.speaking) {
804
- let recentHistory = 0;
805
- for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
806
- recentHistory += this.speakingHistory[i];
807
- }
808
- if (recentHistory >= 2) {
809
- this.speaking = true;
810
- this.attemptSpeaking = false;
811
- this.emit("ConfirmSpeaking");
812
- } else if (recentHistory === 1) {
813
- this.attemptSpeaking = true;
814
- this.emit("StartSpeaking");
815
- }
816
- }
817
- } else if (this.speaking) {
818
- let totalHistory = 0;
819
- for (let i = 0; i < this.speakingHistory.length; i++) {
820
- totalHistory += this.speakingHistory[i];
821
- }
822
- if (totalHistory === 0) {
823
- this.speaking = false;
824
- this.emit("StopSpeaking");
825
- }
826
- } else if (this.attemptSpeaking) {
827
- let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
828
- if (lastHistory === 0) {
829
- this.attemptSpeaking = false;
830
- this.emit("CancelSpeaking");
841
+ async playAudio(blob) {
842
+ if (!this.audioElement || !this.player) return;
843
+ try {
844
+ this.player.addBlob(blob);
845
+ if (this.audioElement.paused) {
846
+ this.audioElement.play();
831
847
  }
848
+ } catch (error) {
849
+ console.error("Error playing audio blob", error, blob);
832
850
  }
833
- this.speakingHistory.shift();
834
- this.speakingHistory.push(Number(currentVolume > this.threshold));
835
- setTimeout(() => this.startLoop(), this.delay);
836
- }
837
- async stop() {
838
- if (!this.running) return;
839
- this.running = false;
840
- this.speaking = false;
841
- if (this.sourceNode) {
842
- this.sourceNode.disconnect();
843
- this.sourceNode = void 0;
844
- }
845
- if (this.analyser) {
846
- this.analyser.disconnect();
847
- this.analyser = void 0;
848
- }
849
- this.fftBins = void 0;
850
851
  }
851
- async setThreshold(threshold) {
852
- if (threshold === this.threshold) return;
853
- this.threshold = threshold;
852
+ stopAudio() {
853
+ this.audioElement?.pause();
854
+ this.player?.stop();
854
855
  }
855
856
  };
857
+ if (!window.micdropSpeaker) {
858
+ window.micdropSpeaker = new Speaker();
859
+ }
860
+ speaker = window.micdropSpeaker;
856
861
 
857
862
  // src/call/CallHandler.ts
858
863
  import EventEmitter4 from "eventemitter3";
@@ -875,49 +880,23 @@ var CallHandlerError = class extends Error {
875
880
  var CallHandler = class _CallHandler extends EventEmitter4 {
876
881
  constructor(options) {
877
882
  super();
883
+ this.options = options;
878
884
  this.conversation = [];
879
885
  this.debug = false;
880
886
  this.startTime = 0;
881
887
  this._isProcessing = false;
882
- this.micRecorder = new MicRecorder(this.getVAD(options?.vad));
883
- this.micRecorder.on("StateChange", () => {
884
- this.notifyStateChange();
885
- });
886
- this.micRecorder.on("Chunk", (blob) => {
887
- this.log(`[Mic] Received chunk`, blob);
888
- this.ws?.send(blob);
889
- });
890
- this.micRecorder.on("StartSpeaking", () => {
891
- this.log("[Mic] Start speaking");
892
- this.ws?.send("startSpeaking" /* StartSpeaking */);
893
- this._isProcessing = false;
894
- this.notifyStateChange();
895
- stopAudio();
896
- });
897
- this.micRecorder.on("StopSpeaking", () => {
898
- this.log("[Mic] Stop speaking");
899
- this.ws?.send("stopSpeaking" /* StopSpeaking */);
900
- this._isProcessing = true;
901
- this.notifyStateChange();
902
- });
903
888
  }
904
889
  static getInstance(options) {
905
- if (!_CallHandler.instance) {
906
- _CallHandler.instance = new _CallHandler(options);
890
+ if (!window.micdropCallHandler) {
891
+ window.micdropCallHandler = new _CallHandler(options);
907
892
  }
908
- return _CallHandler.instance;
909
- }
910
- getVAD(vad) {
911
- if (!vad) return new VolumeVAD();
912
- if (vad === "volume") return new VolumeVAD();
913
- if (vad === "silero") return new SileroVAD();
914
- return vad;
893
+ return window.micdropCallHandler;
915
894
  }
916
895
  get isStarted() {
917
- return this.isWSStarted && this.micRecorder.state.isStarted;
896
+ return this.isWSStarted && this.micRecorder?.state.isStarted || false;
918
897
  }
919
898
  get isStarting() {
920
- return this.isWSStarting || this.micRecorder.state.isStarting;
899
+ return this.isWSStarting || this.micRecorder?.state.isStarting || false;
921
900
  }
922
901
  get isWSStarted() {
923
902
  return this.ws?.readyState === WebSocket.OPEN;
@@ -928,6 +907,15 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
928
907
  get isMicStarted() {
929
908
  return !!this.micStream;
930
909
  }
910
+ get isMicMuted() {
911
+ return this.micRecorder?.state.isMuted ?? false;
912
+ }
913
+ get isMicSpeaking() {
914
+ return this.micRecorder?.state.isSpeaking ?? false;
915
+ }
916
+ get micThreshold() {
917
+ return this.micRecorder?.state.threshold ?? mic.defaultThreshold;
918
+ }
931
919
  get isProcessing() {
932
920
  return this._isProcessing;
933
921
  }
@@ -945,32 +933,57 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
945
933
  console.error("[CallHandler] stop WS", error);
946
934
  }
947
935
  try {
948
- this.micRecorder.stop();
936
+ this.micRecorder?.stop();
949
937
  this.stopMic();
950
938
  } catch (error) {
951
939
  console.error("[CallHandler] stop mic", error);
952
940
  }
953
941
  }
954
942
  pause() {
955
- this.micRecorder.mute();
943
+ this.micRecorder?.mute();
944
+ this._isProcessing = false;
956
945
  this.notifyStateChange();
957
- pauseAudio();
946
+ speaker.pauseAudio();
947
+ this.ws?.send("Mute" /* Mute */);
958
948
  }
959
949
  resume() {
960
950
  if (this.lastAudioBlob) {
961
- playAudio(this.lastAudioBlob);
951
+ speaker.playAudio(this.lastAudioBlob);
962
952
  }
963
- resumeAudio();
964
- this.micRecorder.unmute();
953
+ speaker.resumeAudio();
954
+ this.micRecorder?.unmute();
965
955
  this.notifyStateChange();
966
956
  }
967
957
  async startMic(deviceId, record = true) {
968
958
  try {
959
+ if (!this.micRecorder) {
960
+ this.micRecorder = new MicRecorder(this.options?.vad);
961
+ this.micRecorder.on("StateChange", () => {
962
+ this.notifyStateChange();
963
+ });
964
+ this.micRecorder.on("Chunk", (blob) => {
965
+ this.log(`[Mic] Received chunk`, blob);
966
+ this.ws?.send(blob);
967
+ });
968
+ this.micRecorder.on("StartSpeaking", () => {
969
+ this.log("[Mic] Start speaking");
970
+ this.ws?.send("StartSpeaking" /* StartSpeaking */);
971
+ this._isProcessing = false;
972
+ this.notifyStateChange();
973
+ speaker.stopAudio();
974
+ });
975
+ this.micRecorder.on("StopSpeaking", () => {
976
+ this.log("[Mic] Stop speaking");
977
+ this.ws?.send("StopSpeaking" /* StopSpeaking */);
978
+ this._isProcessing = true;
979
+ this.notifyStateChange();
980
+ });
981
+ }
969
982
  const isRecorderStarted = this.micRecorder.state.isStarted;
970
983
  if (isRecorderStarted) {
971
- this.micRecorder.stop();
984
+ this.micRecorder?.stop();
972
985
  }
973
- this.micStream = await startMicrophone(deviceId);
986
+ this.micStream = await mic.start(deviceId);
974
987
  this.notifyStateChange();
975
988
  if (isRecorderStarted || record) {
976
989
  await this.micRecorder.start(this.micStream);
@@ -982,7 +995,9 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
982
995
  }
983
996
  }
984
997
  stopMic() {
985
- stopMicrophone();
998
+ this.micRecorder?.stop();
999
+ this.micRecorder = void 0;
1000
+ mic.stop();
986
1001
  this.micStream = void 0;
987
1002
  this.notifyStateChange();
988
1003
  }
@@ -1009,34 +1024,44 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
1009
1024
  this.ws.onmessage = (event) => {
1010
1025
  this.log("[WS]", event.data);
1011
1026
  if (event.data instanceof Blob) {
1012
- playAudio(event.data);
1027
+ speaker.playAudio(event.data);
1013
1028
  this.lastAudioBlob = event.data;
1014
1029
  this._isProcessing = false;
1015
1030
  this.notifyStateChange();
1016
1031
  } else if (typeof event.data !== "string") {
1017
1032
  console.warn(`[WS] Unknown message type: ${event.data}`);
1018
- } else if (event.data.startsWith("message" /* Message */)) {
1033
+ } else if (event.data.startsWith("Message" /* Message */)) {
1019
1034
  const message = JSON.parse(
1020
- event.data.substring("message" /* Message */.length + 1)
1035
+ event.data.substring("Message" /* Message */.length + 1)
1021
1036
  );
1022
1037
  this.addMessage(message);
1023
- } else if (event.data === "endInterview" /* EndInterview */) {
1038
+ } else if (event.data === "EndCall" /* EndCall */) {
1024
1039
  setTimeout(() => {
1025
- this.emit("EndInterview");
1040
+ this.emit("EndCall");
1026
1041
  }, 2e3);
1027
- } else if (event.data === "cancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
1042
+ } else if (event.data === "CancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
1028
1043
  const lastMessage = this.conversation[this.conversation.length - 1];
1029
1044
  if (lastMessage?.role === "assistant") {
1030
1045
  this.conversation = this.conversation.slice(0, -1);
1031
1046
  this.notifyStateChange();
1032
1047
  }
1033
- } else if (event.data === "enableSpeakerStreaming" /* EnableSpeakerStreaming */) {
1034
- enableStreaming();
1048
+ } else if (event.data === "SkipAnswer" /* SkipAnswer */) {
1049
+ this._isProcessing = false;
1050
+ this.notifyStateChange();
1051
+ } else if (event.data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
1052
+ const lastMessage = this.conversation[this.conversation.length - 1];
1053
+ if (lastMessage?.role === "user") {
1054
+ this.conversation = this.conversation.slice(0, -1);
1055
+ this._isProcessing = false;
1056
+ this.notifyStateChange();
1057
+ }
1058
+ } else if (event.data === "EnableSpeakerStreaming" /* EnableSpeakerStreaming */) {
1059
+ speaker.enableStreaming();
1035
1060
  }
1036
1061
  };
1037
1062
  this.ws.onclose = (event) => {
1038
1063
  this.log("[WS] Closed", event);
1039
- this.notifyStateChange();
1064
+ this.stop();
1040
1065
  if (event.code >= 1001 && event.code <= 1011 && event.code !== 1005) {
1041
1066
  this.emit("Error", new CallHandlerError());
1042
1067
  } else if (event.code === 4401) {
@@ -1074,16 +1099,18 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
1074
1099
 
1075
1100
  // ../server/src/types.ts
1076
1101
  var CallClientCommands = /* @__PURE__ */ ((CallClientCommands2) => {
1077
- CallClientCommands2["StartSpeaking"] = "startSpeaking";
1078
- CallClientCommands2["StopSpeaking"] = "stopSpeaking";
1079
- CallClientCommands2["Mute"] = "mute";
1102
+ CallClientCommands2["StartSpeaking"] = "StartSpeaking";
1103
+ CallClientCommands2["StopSpeaking"] = "StopSpeaking";
1104
+ CallClientCommands2["Mute"] = "Mute";
1080
1105
  return CallClientCommands2;
1081
1106
  })(CallClientCommands || {});
1082
1107
  var CallServerCommands = /* @__PURE__ */ ((CallServerCommands2) => {
1083
- CallServerCommands2["Message"] = "message";
1084
- CallServerCommands2["CancelLastAssistantMessage"] = "cancelLastAssistantMessage";
1085
- CallServerCommands2["EnableSpeakerStreaming"] = "enableSpeakerStreaming";
1086
- CallServerCommands2["EndInterview"] = "endInterview";
1108
+ CallServerCommands2["Message"] = "Message";
1109
+ CallServerCommands2["CancelLastAssistantMessage"] = "CancelLastAssistantMessage";
1110
+ CallServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
1111
+ CallServerCommands2["SkipAnswer"] = "SkipAnswer";
1112
+ CallServerCommands2["EnableSpeakerStreaming"] = "EnableSpeakerStreaming";
1113
+ CallServerCommands2["EndCall"] = "EndCall";
1087
1114
  return CallServerCommands2;
1088
1115
  })(CallServerCommands || {});
1089
1116
  export {
@@ -1093,10 +1120,10 @@ export {
1093
1120
  CallHandlerErrorCode,
1094
1121
  CallServerCommands,
1095
1122
  LocalStorageKeys,
1096
- microphone_exports as Mic,
1123
+ mic as Mic,
1097
1124
  MicRecorder,
1098
1125
  SileroVAD,
1099
- speaker_exports as Speaker,
1126
+ speaker as Speaker,
1100
1127
  VAD,
1101
1128
  VolumeVAD
1102
1129
  };