@micdrop/client 1.5.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -38,6 +38,7 @@ __export(index_exports, {
38
38
  LocalStorageKeys: () => LocalStorageKeys,
39
39
  Mic: () => mic,
40
40
  MicRecorder: () => MicRecorder,
41
+ MultipleVAD: () => MultipleVAD,
41
42
  SileroVAD: () => SileroVAD,
42
43
  Speaker: () => speaker,
43
44
  VAD: () => VAD,
@@ -121,7 +122,8 @@ if (window.micdropAudioContext) {
121
122
  var LocalStorageKeys = /* @__PURE__ */ ((LocalStorageKeys2) => {
122
123
  LocalStorageKeys2["MicDevice"] = "micdrop.micDevice";
123
124
  LocalStorageKeys2["SpeakerDevice"] = "micdrop.speakerDevice";
124
- LocalStorageKeys2["MicThreshold"] = "micdrop.micThreshold";
125
+ LocalStorageKeys2["VolumeVADOptions"] = "micdrop.VolumeVADOptions";
126
+ LocalStorageKeys2["SileroVADOptions"] = "micdrop.SileroVADOptions";
125
127
  return LocalStorageKeys2;
126
128
  })(LocalStorageKeys || {});
127
129
 
@@ -132,10 +134,8 @@ function stopStream(stream) {
132
134
 
133
135
  // src/audio/mic.ts
134
136
  var mic;
135
- var defaultMicThreshold = -50;
136
137
  var Mic = class {
137
138
  constructor() {
138
- this.defaultThreshold = defaultMicThreshold;
139
139
  this.analyser = new AudioAnalyser(audioContext);
140
140
  }
141
141
  getMicConstraints(deviceId) {
@@ -219,38 +219,103 @@ function createDelayedStream(stream, delay) {
219
219
  return audioDestination.stream;
220
220
  }
221
221
 
222
- // src/audio/vad/SileroVAD.ts
223
- var import_src = require("@ricky0123/vad-web/src");
224
-
225
222
  // src/audio/vad/VAD.ts
226
223
  var import_eventemitter32 = __toESM(require("eventemitter3"));
227
224
  var VAD = class extends import_eventemitter32.default {
228
- /**
229
- * Constructor for the VAD class
230
- * @param threshold - The threshold to use for the VAD
231
- */
232
- constructor(threshold = 0) {
233
- super();
225
+ constructor() {
226
+ super(...arguments);
234
227
  // Max delay to detect speech, can be used to record delayed stream
235
228
  this.delay = 100;
236
- this.threshold = threshold;
229
+ }
230
+ };
231
+
232
+ // src/audio/vad/MultipleVAD.ts
233
+ var MultipleVAD = class extends VAD {
234
+ constructor(vads) {
235
+ super();
236
+ this.startSpeakingCount = 0;
237
+ this.confirmSpeakingCount = 0;
238
+ this.lastEventName = "StopSpeaking";
239
+ this.vads = vads.map((vad) => getVAD(vad));
240
+ }
241
+ emit(eventName, ..._) {
242
+ this.lastEventName = eventName;
243
+ return super.emit(eventName);
244
+ }
245
+ get isStarted() {
246
+ return this.vads.some((vad) => vad.isStarted);
247
+ }
248
+ async start(stream, threshold) {
249
+ for (const vad of this.vads) {
250
+ if (vad.isStarted) {
251
+ continue;
252
+ }
253
+ vad.on("StartSpeaking", () => {
254
+ this.startSpeakingCount++;
255
+ if ((this.lastEventName === "StopSpeaking" || this.lastEventName === "CancelSpeaking") && this.startSpeakingCount === this.vads.length) {
256
+ this.emit("StartSpeaking");
257
+ }
258
+ });
259
+ vad.on("ConfirmSpeaking", () => {
260
+ this.confirmSpeakingCount++;
261
+ if (this.lastEventName === "StartSpeaking" && this.confirmSpeakingCount === this.vads.length) {
262
+ this.emit("ConfirmSpeaking");
263
+ }
264
+ });
265
+ vad.on("CancelSpeaking", () => {
266
+ this.startSpeakingCount--;
267
+ if (this.lastEventName === "StartSpeaking") {
268
+ this.emit("CancelSpeaking");
269
+ }
270
+ });
271
+ vad.on("StopSpeaking", () => {
272
+ this.startSpeakingCount--;
273
+ this.confirmSpeakingCount--;
274
+ if (this.lastEventName === "ConfirmSpeaking") {
275
+ this.emit("StopSpeaking");
276
+ }
277
+ });
278
+ await vad.start(stream);
279
+ }
280
+ }
281
+ async stop() {
282
+ for (const vad of this.vads) {
283
+ await vad.stop();
284
+ vad.removeAllListeners();
285
+ }
237
286
  }
238
287
  };
239
288
 
240
289
  // src/audio/vad/SileroVAD.ts
290
+ var import_src = require("@ricky0123/vad-web/src");
291
+ var defaultOptions = {
292
+ positiveSpeechThreshold: 0.23,
293
+ negativeSpeechThreshold: 0.16,
294
+ minSpeechFrames: 8,
295
+ redemptionFrames: 20
296
+ };
241
297
  var SileroVAD = class extends VAD {
298
+ constructor(options) {
299
+ super();
300
+ this.options = defaultOptions;
301
+ if (options) {
302
+ this.setOptions(options);
303
+ } else {
304
+ const savedOptions = localStorage.getItem(
305
+ "micdrop.SileroVADOptions" /* SileroVADOptions */
306
+ );
307
+ try {
308
+ this.setOptions(savedOptions ? JSON.parse(savedOptions) : {});
309
+ } catch (error) {
310
+ this.setOptions({});
311
+ }
312
+ }
313
+ }
242
314
  get isStarted() {
243
315
  return !!this.vad;
244
316
  }
245
- async start(stream, threshold) {
317
+ async start(stream) {
246
318
  if (this.vad) return;
247
- if (threshold !== void 0) {
248
- this.threshold = threshold;
249
- }
250
- const speechThreshold = Math.min(
251
- Math.max((this.threshold + 100) / 100, 0),
252
- 1
253
- );
254
319
  this.vad = await import_src.MicVAD.new({
255
320
  stream,
256
321
  model: "v5",
@@ -258,10 +323,7 @@ var SileroVAD = class extends VAD {
258
323
  onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
259
324
  onVADMisfire: () => this.emit("CancelSpeaking"),
260
325
  onSpeechEnd: () => this.emit("StopSpeaking"),
261
- positiveSpeechThreshold: speechThreshold,
262
- negativeSpeechThreshold: speechThreshold * 0.7,
263
- minSpeechFrames: 3,
264
- redemptionFrames: 9
326
+ ...this.options
265
327
  });
266
328
  this.vad.start();
267
329
  }
@@ -270,27 +332,43 @@ var SileroVAD = class extends VAD {
270
332
  this.vad.destroy();
271
333
  this.vad = void 0;
272
334
  }
273
- async setThreshold(threshold) {
274
- if (threshold === this.threshold) return;
275
- if (this.vad) {
276
- const stream = this.vad["stream"];
277
- await this.stop();
278
- await this.start(stream, threshold);
279
- }
335
+ setOptions(options) {
336
+ this.options = { ...this.options, ...options };
337
+ this.vad?.setOptions(this.options);
338
+ localStorage.setItem(
339
+ "micdrop.SileroVADOptions" /* SileroVADOptions */,
340
+ JSON.stringify(this.options)
341
+ );
342
+ }
343
+ resetOptions() {
344
+ this.setOptions(defaultOptions);
280
345
  }
281
346
  };
282
347
 
283
348
  // src/audio/vad/VolumeVAD.ts
349
+ var defaultOptions2 = {
350
+ history: 5,
351
+ threshold: -55
352
+ };
284
353
  var VolumeVAD = class extends VAD {
285
- constructor() {
354
+ constructor(options) {
286
355
  super();
356
+ this.options = defaultOptions2;
287
357
  this.running = false;
288
358
  this.speaking = false;
289
359
  this.attemptSpeaking = false;
290
- this.history = 10;
291
360
  this.speakingHistory = [];
292
- for (let i = 0; i < this.history; i++) {
293
- this.speakingHistory.push(0);
361
+ if (options) {
362
+ this.setOptions(options);
363
+ } else {
364
+ const savedOptions = localStorage.getItem(
365
+ "micdrop.VolumeVADOptions" /* VolumeVADOptions */
366
+ );
367
+ try {
368
+ this.setOptions(savedOptions ? JSON.parse(savedOptions) : {});
369
+ } catch (error) {
370
+ this.setOptions({});
371
+ }
294
372
  }
295
373
  }
296
374
  getMaxVolume() {
@@ -307,11 +385,8 @@ var VolumeVAD = class extends VAD {
307
385
  get isStarted() {
308
386
  return this.running;
309
387
  }
310
- async start(stream, threshold) {
388
+ async start(stream) {
311
389
  if (this.running) return;
312
- if (threshold !== void 0) {
313
- this.threshold = threshold;
314
- }
315
390
  this.analyser = audioContext.createAnalyser();
316
391
  this.sourceNode = audioContext.createMediaStreamSource(stream);
317
392
  this.analyser.fftSize = 512;
@@ -324,7 +399,7 @@ var VolumeVAD = class extends VAD {
324
399
  startLoop() {
325
400
  if (!this.running) return;
326
401
  const currentVolume = this.getMaxVolume();
327
- if (currentVolume > this.threshold) {
402
+ if (currentVolume > this.options.threshold) {
328
403
  if (!this.speaking) {
329
404
  let recentHistory = 0;
330
405
  for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
@@ -356,7 +431,7 @@ var VolumeVAD = class extends VAD {
356
431
  }
357
432
  }
358
433
  this.speakingHistory.shift();
359
- this.speakingHistory.push(Number(currentVolume > this.threshold));
434
+ this.speakingHistory.push(Number(currentVolume > this.options.threshold));
360
435
  setTimeout(() => this.startLoop(), this.delay);
361
436
  }
362
437
  async stop() {
@@ -373,25 +448,49 @@ var VolumeVAD = class extends VAD {
373
448
  }
374
449
  this.fftBins = void 0;
375
450
  }
376
- async setThreshold(threshold) {
377
- if (threshold === this.threshold) return;
378
- this.threshold = threshold;
451
+ setOptions(options) {
452
+ if (typeof options.history === "number" && options.history < 3) {
453
+ throw new Error("History must be at least 3");
454
+ }
455
+ this.options = { ...this.options, ...options };
456
+ while (this.speakingHistory.length > this.options.history) {
457
+ this.speakingHistory.shift();
458
+ }
459
+ while (this.speakingHistory.length < this.options.history) {
460
+ this.speakingHistory.push(0);
461
+ }
462
+ localStorage.setItem(
463
+ "micdrop.VolumeVADOptions" /* VolumeVADOptions */,
464
+ JSON.stringify(this.options)
465
+ );
466
+ }
467
+ resetOptions() {
468
+ this.setOptions(defaultOptions2);
379
469
  }
380
470
  };
381
471
 
472
+ // src/audio/vad/getVAD.ts
473
+ function getVAD(vad) {
474
+ if (!vad) return new VolumeVAD();
475
+ if (vad === "volume") return new VolumeVAD();
476
+ if (vad === "silero") return new SileroVAD();
477
+ if (Array.isArray(vad)) {
478
+ return new MultipleVAD(vad);
479
+ }
480
+ return vad;
481
+ }
482
+
382
483
  // src/audio/MicRecorder.ts
383
484
  var timeSlice = 100;
384
485
  var defaultMicRecorderState = {
385
486
  isStarting: false,
386
487
  isStarted: false,
387
488
  isMuted: false,
388
- isSpeaking: false,
389
- threshold: defaultMicThreshold
489
+ isSpeaking: false
390
490
  };
391
491
  var MicRecorder = class extends import_eventemitter33.default {
392
492
  constructor(vadConfig) {
393
493
  super();
394
- this.vadConfig = vadConfig;
395
494
  this.audioInfo = this.getAudioInfo();
396
495
  this.speakingConfirmed = false;
397
496
  this.queuedChunks = [];
@@ -430,8 +529,8 @@ var MicRecorder = class extends import_eventemitter33.default {
430
529
  this.emit("StopSpeaking");
431
530
  this.speakingConfirmed = false;
432
531
  }).bind(this);
433
- const threshold = parseFloat(localStorage.getItem("micdrop.micThreshold" /* MicThreshold */) || "0") || defaultMicThreshold;
434
- this.state = { ...defaultMicRecorderState, threshold };
532
+ this.state = defaultMicRecorderState;
533
+ this.vad = getVAD(vadConfig);
435
534
  }
436
535
  async start(stream) {
437
536
  if (this.state.isStarted) {
@@ -443,9 +542,6 @@ var MicRecorder = class extends import_eventemitter33.default {
443
542
  isMuted: false,
444
543
  isSpeaking: false
445
544
  });
446
- if (!this.vad) {
447
- this.vad = this.initVAD(this.vadConfig);
448
- }
449
545
  const delayedStream = createDelayedStream(
450
546
  stream,
451
547
  this.vad.delay / 1e3 + 0.05
@@ -497,17 +593,6 @@ var MicRecorder = class extends import_eventemitter33.default {
497
593
  console.error(err);
498
594
  }
499
595
  }
500
- setThreshold(threshold) {
501
- this.changeState({ threshold });
502
- localStorage.setItem("micdrop.micThreshold" /* MicThreshold */, threshold.toString());
503
- this.vad?.setThreshold(threshold);
504
- }
505
- initVAD(vad) {
506
- if (!vad) return new VolumeVAD();
507
- if (vad === "volume") return new VolumeVAD();
508
- if (vad === "silero") return new SileroVAD();
509
- return vad;
510
- }
511
596
  async startSpeakingDetection(stream) {
512
597
  if (!this.vad) {
513
598
  throw new Error("VAD not initialized");
@@ -519,7 +604,7 @@ var MicRecorder = class extends import_eventemitter33.default {
519
604
  this.vad.on("ConfirmSpeaking", this.onConfirmSpeaking);
520
605
  this.vad.on("CancelSpeaking", this.onCancelSpeaking);
521
606
  this.vad.on("StopSpeaking", this.onStopSpeaking);
522
- await this.vad.start(stream, this.state.threshold);
607
+ await this.vad.start(stream);
523
608
  }
524
609
  async stopSpeakingDetection() {
525
610
  if (!this.vad) {
@@ -932,6 +1017,7 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
932
1017
  this.debug = false;
933
1018
  this.startTime = 0;
934
1019
  this._isProcessing = false;
1020
+ this.vad = getVAD(this.options?.vad);
935
1021
  }
936
1022
  static getInstance(options) {
937
1023
  if (!window.micdropCallHandler) {
@@ -960,9 +1046,6 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
960
1046
  get isMicSpeaking() {
961
1047
  return this.micRecorder?.state.isSpeaking ?? false;
962
1048
  }
963
- get micThreshold() {
964
- return this.micRecorder?.state.threshold ?? mic.defaultThreshold;
965
- }
966
1049
  get isProcessing() {
967
1050
  return this._isProcessing;
968
1051
  }
@@ -991,7 +1074,7 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
991
1074
  this._isProcessing = false;
992
1075
  this.notifyStateChange();
993
1076
  speaker.pauseAudio();
994
- this.ws?.send("mute" /* Mute */);
1077
+ this.ws?.send("Mute" /* Mute */);
995
1078
  }
996
1079
  resume() {
997
1080
  if (this.lastAudioBlob) {
@@ -1004,7 +1087,7 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
1004
1087
  async startMic(deviceId, record = true) {
1005
1088
  try {
1006
1089
  if (!this.micRecorder) {
1007
- this.micRecorder = new MicRecorder(this.options?.vad);
1090
+ this.micRecorder = new MicRecorder(this.vad);
1008
1091
  this.micRecorder.on("StateChange", () => {
1009
1092
  this.notifyStateChange();
1010
1093
  });
@@ -1014,14 +1097,14 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
1014
1097
  });
1015
1098
  this.micRecorder.on("StartSpeaking", () => {
1016
1099
  this.log("[Mic] Start speaking");
1017
- this.ws?.send("startSpeaking" /* StartSpeaking */);
1100
+ this.ws?.send("StartSpeaking" /* StartSpeaking */);
1018
1101
  this._isProcessing = false;
1019
1102
  this.notifyStateChange();
1020
1103
  speaker.stopAudio();
1021
1104
  });
1022
1105
  this.micRecorder.on("StopSpeaking", () => {
1023
1106
  this.log("[Mic] Stop speaking");
1024
- this.ws?.send("stopSpeaking" /* StopSpeaking */);
1107
+ this.ws?.send("StopSpeaking" /* StopSpeaking */);
1025
1108
  this._isProcessing = true;
1026
1109
  this.notifyStateChange();
1027
1110
  });
@@ -1077,28 +1160,38 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
1077
1160
  this.notifyStateChange();
1078
1161
  } else if (typeof event.data !== "string") {
1079
1162
  console.warn(`[WS] Unknown message type: ${event.data}`);
1080
- } else if (event.data.startsWith("message" /* Message */)) {
1163
+ } else if (event.data.startsWith("Message" /* Message */)) {
1081
1164
  const message = JSON.parse(
1082
- event.data.substring("message" /* Message */.length + 1)
1165
+ event.data.substring("Message" /* Message */.length + 1)
1083
1166
  );
1084
1167
  this.addMessage(message);
1085
- } else if (event.data === "endInterview" /* EndInterview */) {
1168
+ } else if (event.data === "EndCall" /* EndCall */) {
1086
1169
  setTimeout(() => {
1087
- this.emit("EndInterview");
1170
+ this.emit("EndCall");
1088
1171
  }, 2e3);
1089
- } else if (event.data === "cancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
1172
+ } else if (event.data === "CancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
1090
1173
  const lastMessage = this.conversation[this.conversation.length - 1];
1091
1174
  if (lastMessage?.role === "assistant") {
1092
1175
  this.conversation = this.conversation.slice(0, -1);
1093
1176
  this.notifyStateChange();
1094
1177
  }
1095
- } else if (event.data === "enableSpeakerStreaming" /* EnableSpeakerStreaming */) {
1178
+ } else if (event.data === "SkipAnswer" /* SkipAnswer */) {
1179
+ this._isProcessing = false;
1180
+ this.notifyStateChange();
1181
+ } else if (event.data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
1182
+ const lastMessage = this.conversation[this.conversation.length - 1];
1183
+ if (lastMessage?.role === "user") {
1184
+ this.conversation = this.conversation.slice(0, -1);
1185
+ this._isProcessing = false;
1186
+ this.notifyStateChange();
1187
+ }
1188
+ } else if (event.data === "EnableSpeakerStreaming" /* EnableSpeakerStreaming */) {
1096
1189
  speaker.enableStreaming();
1097
1190
  }
1098
1191
  };
1099
1192
  this.ws.onclose = (event) => {
1100
1193
  this.log("[WS] Closed", event);
1101
- this.notifyStateChange();
1194
+ this.stop();
1102
1195
  if (event.code >= 1001 && event.code <= 1011 && event.code !== 1005) {
1103
1196
  this.emit("Error", new CallHandlerError());
1104
1197
  } else if (event.code === 4401) {
@@ -1136,16 +1229,18 @@ var CallHandler = class _CallHandler extends import_eventemitter34.default {
1136
1229
 
1137
1230
  // ../server/src/types.ts
1138
1231
  var CallClientCommands = /* @__PURE__ */ ((CallClientCommands2) => {
1139
- CallClientCommands2["StartSpeaking"] = "startSpeaking";
1140
- CallClientCommands2["StopSpeaking"] = "stopSpeaking";
1141
- CallClientCommands2["Mute"] = "mute";
1232
+ CallClientCommands2["StartSpeaking"] = "StartSpeaking";
1233
+ CallClientCommands2["StopSpeaking"] = "StopSpeaking";
1234
+ CallClientCommands2["Mute"] = "Mute";
1142
1235
  return CallClientCommands2;
1143
1236
  })(CallClientCommands || {});
1144
1237
  var CallServerCommands = /* @__PURE__ */ ((CallServerCommands2) => {
1145
- CallServerCommands2["Message"] = "message";
1146
- CallServerCommands2["CancelLastAssistantMessage"] = "cancelLastAssistantMessage";
1147
- CallServerCommands2["EnableSpeakerStreaming"] = "enableSpeakerStreaming";
1148
- CallServerCommands2["EndInterview"] = "endInterview";
1238
+ CallServerCommands2["Message"] = "Message";
1239
+ CallServerCommands2["CancelLastAssistantMessage"] = "CancelLastAssistantMessage";
1240
+ CallServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
1241
+ CallServerCommands2["SkipAnswer"] = "SkipAnswer";
1242
+ CallServerCommands2["EnableSpeakerStreaming"] = "EnableSpeakerStreaming";
1243
+ CallServerCommands2["EndCall"] = "EndCall";
1149
1244
  return CallServerCommands2;
1150
1245
  })(CallServerCommands || {});
1151
1246
  // Annotate the CommonJS export names for ESM import in node:
@@ -1158,6 +1253,7 @@ var CallServerCommands = /* @__PURE__ */ ((CallServerCommands2) => {
1158
1253
  LocalStorageKeys,
1159
1254
  Mic,
1160
1255
  MicRecorder,
1256
+ MultipleVAD,
1161
1257
  SileroVAD,
1162
1258
  Speaker,
1163
1259
  VAD,