@micdrop/client 1.5.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -74,7 +74,8 @@ if (window.micdropAudioContext) {
74
74
  var LocalStorageKeys = /* @__PURE__ */ ((LocalStorageKeys2) => {
75
75
  LocalStorageKeys2["MicDevice"] = "micdrop.micDevice";
76
76
  LocalStorageKeys2["SpeakerDevice"] = "micdrop.speakerDevice";
77
- LocalStorageKeys2["MicThreshold"] = "micdrop.micThreshold";
77
+ LocalStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
78
+ LocalStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
78
79
  return LocalStorageKeys2;
79
80
  })(LocalStorageKeys || {});
80
81
 
@@ -85,10 +86,8 @@ function stopStream(stream) {
85
86
 
86
87
  // src/audio/mic.ts
87
88
  var mic;
88
- var defaultMicThreshold = -50;
89
89
  var Mic = class {
90
90
  constructor() {
91
- this.defaultThreshold = defaultMicThreshold;
92
91
  this.analyser = new AudioAnalyser(audioContext);
93
92
  }
94
93
  getMicConstraints(deviceId) {
@@ -172,38 +171,103 @@ function createDelayedStream(stream, delay) {
172
171
  return audioDestination.stream;
173
172
  }
174
173
 
175
- // src/audio/vad/SileroVAD.ts
176
- import { MicVAD } from "@ricky0123/vad-web/src";
177
-
178
174
  // src/audio/vad/VAD.ts
179
175
  import EventEmitter2 from "eventemitter3";
180
176
  var VAD = class extends EventEmitter2 {
181
- /**
182
- * Constructor for the VAD class
183
- * @param threshold - The threshold to use for the VAD
184
- */
185
- constructor(threshold = 0) {
186
- super();
177
+ constructor() {
178
+ super(...arguments);
187
179
  // Max delay to detect speech, can be used to record delayed stream
188
180
  this.delay = 100;
189
- this.threshold = threshold;
181
+ }
182
+ };
183
+
184
+ // src/audio/vad/MultipleVAD.ts
185
+ var MultipleVAD = class extends VAD {
186
+ constructor(vads) {
187
+ super();
188
+ this.startSpeakingCount = 0;
189
+ this.confirmSpeakingCount = 0;
190
+ this.lastEventName = "StopSpeaking";
191
+ this.vads = vads.map((vad) => getVAD(vad));
192
+ }
193
+ emit(eventName, ..._) {
194
+ this.lastEventName = eventName;
195
+ return super.emit(eventName);
196
+ }
197
+ get isStarted() {
198
+ return this.vads.some((vad) => vad.isStarted);
199
+ }
200
+ async start(stream, threshold) {
201
+ for (const vad of this.vads) {
202
+ if (vad.isStarted) {
203
+ continue;
204
+ }
205
+ vad.on("StartSpeaking", () => {
206
+ this.startSpeakingCount++;
207
+ if ((this.lastEventName === "StopSpeaking" || this.lastEventName === "CancelSpeaking") && this.startSpeakingCount === this.vads.length) {
208
+ this.emit("StartSpeaking");
209
+ }
210
+ });
211
+ vad.on("ConfirmSpeaking", () => {
212
+ this.confirmSpeakingCount++;
213
+ if (this.lastEventName === "StartSpeaking" && this.confirmSpeakingCount === this.vads.length) {
214
+ this.emit("ConfirmSpeaking");
215
+ }
216
+ });
217
+ vad.on("CancelSpeaking", () => {
218
+ this.startSpeakingCount--;
219
+ if (this.lastEventName === "StartSpeaking") {
220
+ this.emit("CancelSpeaking");
221
+ }
222
+ });
223
+ vad.on("StopSpeaking", () => {
224
+ this.startSpeakingCount--;
225
+ this.confirmSpeakingCount--;
226
+ if (this.lastEventName === "ConfirmSpeaking") {
227
+ this.emit("StopSpeaking");
228
+ }
229
+ });
230
+ await vad.start(stream);
231
+ }
232
+ }
233
+ async stop() {
234
+ for (const vad of this.vads) {
235
+ await vad.stop();
236
+ vad.removeAllListeners();
237
+ }
190
238
  }
191
239
  };
192
240
 
193
241
  // src/audio/vad/SileroVAD.ts
242
+ import { MicVAD } from "@ricky0123/vad-web/src";
243
+ var defaultOptions = {
244
+ positiveSpeechThreshold: 0.23,
245
+ negativeSpeechThreshold: 0.16,
246
+ minSpeechFrames: 8,
247
+ redemptionFrames: 20
248
+ };
194
249
  var SileroVAD = class extends VAD {
250
+ constructor(options) {
251
+ super();
252
+ this.options = defaultOptions;
253
+ if (options) {
254
+ this.setOptions(options);
255
+ } else {
256
+ const savedOptions = localStorage.getItem(
257
+ "micdrop.SileroVADOptions" /* SileroVADOptions */
258
+ );
259
+ try {
260
+ this.setOptions(savedOptions ? JSON.parse(savedOptions) : {});
261
+ } catch (error) {
262
+ this.setOptions({});
263
+ }
264
+ }
265
+ }
195
266
  get isStarted() {
196
267
  return !!this.vad;
197
268
  }
198
- async start(stream, threshold) {
269
+ async start(stream) {
199
270
  if (this.vad) return;
200
- if (threshold !== void 0) {
201
- this.threshold = threshold;
202
- }
203
- const speechThreshold = Math.min(
204
- Math.max((this.threshold + 100) / 100, 0),
205
- 1
206
- );
207
271
  this.vad = await MicVAD.new({
208
272
  stream,
209
273
  model: "v5",
@@ -211,10 +275,7 @@ var SileroVAD = class extends VAD {
211
275
  onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
212
276
  onVADMisfire: () => this.emit("CancelSpeaking"),
213
277
  onSpeechEnd: () => this.emit("StopSpeaking"),
214
- positiveSpeechThreshold: speechThreshold,
215
- negativeSpeechThreshold: speechThreshold * 0.7,
216
- minSpeechFrames: 3,
217
- redemptionFrames: 9
278
+ ...this.options
218
279
  });
219
280
  this.vad.start();
220
281
  }
@@ -223,27 +284,43 @@ var SileroVAD = class extends VAD {
223
284
  this.vad.destroy();
224
285
  this.vad = void 0;
225
286
  }
226
- async setThreshold(threshold) {
227
- if (threshold === this.threshold) return;
228
- if (this.vad) {
229
- const stream = this.vad["stream"];
230
- await this.stop();
231
- await this.start(stream, threshold);
232
- }
287
+ setOptions(options) {
288
+ this.options = { ...this.options, ...options };
289
+ this.vad?.setOptions(this.options);
290
+ localStorage.setItem(
291
+ "micdrop.SileroVADOptions" /* SileroVADOptions */,
292
+ JSON.stringify(this.options)
293
+ );
294
+ }
295
+ resetOptions() {
296
+ this.setOptions(defaultOptions);
233
297
  }
234
298
  };
235
299
 
236
300
  // src/audio/vad/VolumeVAD.ts
301
+ var defaultOptions2 = {
302
+ history: 5,
303
+ threshold: -55
304
+ };
237
305
  var VolumeVAD = class extends VAD {
238
- constructor() {
306
+ constructor(options) {
239
307
  super();
308
+ this.options = defaultOptions2;
240
309
  this.running = false;
241
310
  this.speaking = false;
242
311
  this.attemptSpeaking = false;
243
- this.history = 10;
244
312
  this.speakingHistory = [];
245
- for (let i = 0; i < this.history; i++) {
246
- this.speakingHistory.push(0);
313
+ if (options) {
314
+ this.setOptions(options);
315
+ } else {
316
+ const savedOptions = localStorage.getItem(
317
+ "micdrop.VolumeVADOptions" /* VolumeVADOptions */
318
+ );
319
+ try {
320
+ this.setOptions(savedOptions ? JSON.parse(savedOptions) : {});
321
+ } catch (error) {
322
+ this.setOptions({});
323
+ }
247
324
  }
248
325
  }
249
326
  getMaxVolume() {
@@ -260,11 +337,8 @@ var VolumeVAD = class extends VAD {
260
337
  get isStarted() {
261
338
  return this.running;
262
339
  }
263
- async start(stream, threshold) {
340
+ async start(stream) {
264
341
  if (this.running) return;
265
- if (threshold !== void 0) {
266
- this.threshold = threshold;
267
- }
268
342
  this.analyser = audioContext.createAnalyser();
269
343
  this.sourceNode = audioContext.createMediaStreamSource(stream);
270
344
  this.analyser.fftSize = 512;
@@ -277,7 +351,7 @@ var VolumeVAD = class extends VAD {
277
351
  startLoop() {
278
352
  if (!this.running) return;
279
353
  const currentVolume = this.getMaxVolume();
280
- if (currentVolume > this.threshold) {
354
+ if (currentVolume > this.options.threshold) {
281
355
  if (!this.speaking) {
282
356
  let recentHistory = 0;
283
357
  for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
@@ -309,7 +383,7 @@ var VolumeVAD = class extends VAD {
309
383
  }
310
384
  }
311
385
  this.speakingHistory.shift();
312
- this.speakingHistory.push(Number(currentVolume > this.threshold));
386
+ this.speakingHistory.push(Number(currentVolume > this.options.threshold));
313
387
  setTimeout(() => this.startLoop(), this.delay);
314
388
  }
315
389
  async stop() {
@@ -326,25 +400,49 @@ var VolumeVAD = class extends VAD {
326
400
  }
327
401
  this.fftBins = void 0;
328
402
  }
329
- async setThreshold(threshold) {
330
- if (threshold === this.threshold) return;
331
- this.threshold = threshold;
403
+ setOptions(options) {
404
+ if (typeof options.history === "number" && options.history < 3) {
405
+ throw new Error("History must be at least 3");
406
+ }
407
+ this.options = { ...this.options, ...options };
408
+ while (this.speakingHistory.length > this.options.history) {
409
+ this.speakingHistory.shift();
410
+ }
411
+ while (this.speakingHistory.length < this.options.history) {
412
+ this.speakingHistory.push(0);
413
+ }
414
+ localStorage.setItem(
415
+ "micdrop.VolumeVADOptions" /* VolumeVADOptions */,
416
+ JSON.stringify(this.options)
417
+ );
418
+ }
419
+ resetOptions() {
420
+ this.setOptions(defaultOptions2);
332
421
  }
333
422
  };
334
423
 
424
+ // src/audio/vad/getVAD.ts
425
+ function getVAD(vad) {
426
+ if (!vad) return new VolumeVAD();
427
+ if (vad === "volume") return new VolumeVAD();
428
+ if (vad === "silero") return new SileroVAD();
429
+ if (Array.isArray(vad)) {
430
+ return new MultipleVAD(vad);
431
+ }
432
+ return vad;
433
+ }
434
+
335
435
  // src/audio/MicRecorder.ts
336
436
  var timeSlice = 100;
337
437
  var defaultMicRecorderState = {
338
438
  isStarting: false,
339
439
  isStarted: false,
340
440
  isMuted: false,
341
- isSpeaking: false,
342
- threshold: defaultMicThreshold
441
+ isSpeaking: false
343
442
  };
344
443
  var MicRecorder = class extends EventEmitter3 {
345
444
  constructor(vadConfig) {
346
445
  super();
347
- this.vadConfig = vadConfig;
348
446
  this.audioInfo = this.getAudioInfo();
349
447
  this.speakingConfirmed = false;
350
448
  this.queuedChunks = [];
@@ -383,8 +481,8 @@ var MicRecorder = class extends EventEmitter3 {
383
481
  this.emit("StopSpeaking");
384
482
  this.speakingConfirmed = false;
385
483
  }).bind(this);
386
- const threshold = parseFloat(localStorage.getItem("micdrop.micThreshold" /* MicThreshold */) || "0") || defaultMicThreshold;
387
- this.state = { ...defaultMicRecorderState, threshold };
484
+ this.state = defaultMicRecorderState;
485
+ this.vad = getVAD(vadConfig);
388
486
  }
389
487
  async start(stream) {
390
488
  if (this.state.isStarted) {
@@ -396,9 +494,6 @@ var MicRecorder = class extends EventEmitter3 {
396
494
  isMuted: false,
397
495
  isSpeaking: false
398
496
  });
399
- if (!this.vad) {
400
- this.vad = this.initVAD(this.vadConfig);
401
- }
402
497
  const delayedStream = createDelayedStream(
403
498
  stream,
404
499
  this.vad.delay / 1e3 + 0.05
@@ -450,17 +545,6 @@ var MicRecorder = class extends EventEmitter3 {
450
545
  console.error(err);
451
546
  }
452
547
  }
453
- setThreshold(threshold) {
454
- this.changeState({ threshold });
455
- localStorage.setItem("micdrop.micThreshold" /* MicThreshold */, threshold.toString());
456
- this.vad?.setThreshold(threshold);
457
- }
458
- initVAD(vad) {
459
- if (!vad) return new VolumeVAD();
460
- if (vad === "volume") return new VolumeVAD();
461
- if (vad === "silero") return new SileroVAD();
462
- return vad;
463
- }
464
548
  async startSpeakingDetection(stream) {
465
549
  if (!this.vad) {
466
550
  throw new Error("VAD not initialized");
@@ -472,7 +556,7 @@ var MicRecorder = class extends EventEmitter3 {
472
556
  this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
473
557
  this.vad.on("CancelSpeaking", this.onCancelSpeaking);
474
558
  this.vad.on("StopSpeaking", this.onStopSpeaking);
475
- await this.vad.start(stream, this.state.threshold);
559
+ await this.vad.start(stream);
476
560
  }
477
561
  async stopSpeakingDetection() {
478
562
  if (!this.vad) {
@@ -885,6 +969,7 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
885
969
  this.debug = false;
886
970
  this.startTime = 0;
887
971
  this._isProcessing = false;
972
+ this.vad = getVAD(this.options?.vad);
888
973
  }
889
974
  static getInstance(options) {
890
975
  if (!window.micdropCallHandler) {
@@ -913,9 +998,6 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
913
998
  get isMicSpeaking() {
914
999
  return this.micRecorder?.state.isSpeaking ?? false;
915
1000
  }
916
- get micThreshold() {
917
- return this.micRecorder?.state.threshold ?? mic.defaultThreshold;
918
- }
919
1001
  get isProcessing() {
920
1002
  return this._isProcessing;
921
1003
  }
@@ -944,7 +1026,7 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
944
1026
  this._isProcessing = false;
945
1027
  this.notifyStateChange();
946
1028
  speaker.pauseAudio();
947
- this.ws?.send("mute" /* Mute */);
1029
+ this.ws?.send("Mute" /* Mute */);
948
1030
  }
949
1031
  resume() {
950
1032
  if (this.lastAudioBlob) {
@@ -957,7 +1039,7 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
957
1039
  async startMic(deviceId, record = true) {
958
1040
  try {
959
1041
  if (!this.micRecorder) {
960
- this.micRecorder = new MicRecorder(this.options?.vad);
1042
+ this.micRecorder = new MicRecorder(this.vad);
961
1043
  this.micRecorder.on("StateChange", () => {
962
1044
  this.notifyStateChange();
963
1045
  });
@@ -967,14 +1049,14 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
967
1049
  });
968
1050
  this.micRecorder.on("StartSpeaking", () => {
969
1051
  this.log("[Mic] Start speaking");
970
- this.ws?.send("startSpeaking" /* StartSpeaking */);
1052
+ this.ws?.send("StartSpeaking" /* StartSpeaking */);
971
1053
  this._isProcessing = false;
972
1054
  this.notifyStateChange();
973
1055
  speaker.stopAudio();
974
1056
  });
975
1057
  this.micRecorder.on("StopSpeaking", () => {
976
1058
  this.log("[Mic] Stop speaking");
977
- this.ws?.send("stopSpeaking" /* StopSpeaking */);
1059
+ this.ws?.send("StopSpeaking" /* StopSpeaking */);
978
1060
  this._isProcessing = true;
979
1061
  this.notifyStateChange();
980
1062
  });
@@ -1030,28 +1112,38 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
1030
1112
  this.notifyStateChange();
1031
1113
  } else if (typeof event.data !== "string") {
1032
1114
  console.warn(`[WS] Unknown message type: ${event.data}`);
1033
- } else if (event.data.startsWith("message" /* Message */)) {
1115
+ } else if (event.data.startsWith("Message" /* Message */)) {
1034
1116
  const message = JSON.parse(
1035
- event.data.substring("message" /* Message */.length + 1)
1117
+ event.data.substring("Message" /* Message */.length + 1)
1036
1118
  );
1037
1119
  this.addMessage(message);
1038
- } else if (event.data === "endInterview" /* EndInterview */) {
1120
+ } else if (event.data === "EndCall" /* EndCall */) {
1039
1121
  setTimeout(() => {
1040
- this.emit("EndInterview");
1122
+ this.emit("EndCall");
1041
1123
  }, 2e3);
1042
- } else if (event.data === "cancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
1124
+ } else if (event.data === "CancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
1043
1125
  const lastMessage = this.conversation[this.conversation.length - 1];
1044
1126
  if (lastMessage?.role === "assistant") {
1045
1127
  this.conversation = this.conversation.slice(0, -1);
1046
1128
  this.notifyStateChange();
1047
1129
  }
1048
- } else if (event.data === "enableSpeakerStreaming" /* EnableSpeakerStreaming */) {
1130
+ } else if (event.data === "SkipAnswer" /* SkipAnswer */) {
1131
+ this._isProcessing = false;
1132
+ this.notifyStateChange();
1133
+ } else if (event.data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
1134
+ const lastMessage = this.conversation[this.conversation.length - 1];
1135
+ if (lastMessage?.role === "user") {
1136
+ this.conversation = this.conversation.slice(0, -1);
1137
+ this._isProcessing = false;
1138
+ this.notifyStateChange();
1139
+ }
1140
+ } else if (event.data === "EnableSpeakerStreaming" /* EnableSpeakerStreaming */) {
1049
1141
  speaker.enableStreaming();
1050
1142
  }
1051
1143
  };
1052
1144
  this.ws.onclose = (event) => {
1053
1145
  this.log("[WS] Closed", event);
1054
- this.notifyStateChange();
1146
+ this.stop();
1055
1147
  if (event.code >= 1001 && event.code <= 1011 && event.code !== 1005) {
1056
1148
  this.emit("Error", new CallHandlerError());
1057
1149
  } else if (event.code === 4401) {
@@ -1089,16 +1181,18 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
1089
1181
 
1090
1182
  // ../server/src/types.ts
1091
1183
  var CallClientCommands = /* @__PURE__ */ ((CallClientCommands2) => {
1092
- CallClientCommands2["StartSpeaking"] = "startSpeaking";
1093
- CallClientCommands2["StopSpeaking"] = "stopSpeaking";
1094
- CallClientCommands2["Mute"] = "mute";
1184
+ CallClientCommands2["StartSpeaking"] = "StartSpeaking";
1185
+ CallClientCommands2["StopSpeaking"] = "StopSpeaking";
1186
+ CallClientCommands2["Mute"] = "Mute";
1095
1187
  return CallClientCommands2;
1096
1188
  })(CallClientCommands || {});
1097
1189
  var CallServerCommands = /* @__PURE__ */ ((CallServerCommands2) => {
1098
- CallServerCommands2["Message"] = "message";
1099
- CallServerCommands2["CancelLastAssistantMessage"] = "cancelLastAssistantMessage";
1100
- CallServerCommands2["EnableSpeakerStreaming"] = "enableSpeakerStreaming";
1101
- CallServerCommands2["EndInterview"] = "endInterview";
1190
+ CallServerCommands2["Message"] = "Message";
1191
+ CallServerCommands2["CancelLastAssistantMessage"] = "CancelLastAssistantMessage";
1192
+ CallServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
1193
+ CallServerCommands2["SkipAnswer"] = "SkipAnswer";
1194
+ CallServerCommands2["EnableSpeakerStreaming"] = "EnableSpeakerStreaming";
1195
+ CallServerCommands2["EndCall"] = "EndCall";
1102
1196
  return CallServerCommands2;
1103
1197
  })(CallServerCommands || {});
1104
1198
  export {
@@ -1110,6 +1204,7 @@ export {
1110
1204
  LocalStorageKeys,
1111
1205
  mic as Mic,
1112
1206
  MicRecorder,
1207
+ MultipleVAD,
1113
1208
  SileroVAD,
1114
1209
  speaker as Speaker,
1115
1210
  VAD,