@micdrop/client 1.4.1 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.d.mts +99 -95
- package/dist/index.d.ts +99 -95
- package/dist/index.js +433 -400
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +433 -406
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1,21 +1,3 @@
|
|
|
1
|
-
var __defProp = Object.defineProperty;
|
|
2
|
-
var __export = (target, all) => {
|
|
3
|
-
for (var name in all)
|
|
4
|
-
__defProp(target, name, { get: all[name], enumerable: true });
|
|
5
|
-
};
|
|
6
|
-
|
|
7
|
-
// src/audio/MicRecorder.ts
|
|
8
|
-
import EventEmitter2 from "eventemitter3";
|
|
9
|
-
|
|
10
|
-
// src/audio/microphone.ts
|
|
11
|
-
var microphone_exports = {};
|
|
12
|
-
__export(microphone_exports, {
|
|
13
|
-
defaultMicThreshold: () => defaultMicThreshold,
|
|
14
|
-
micAnalyser: () => micAnalyser,
|
|
15
|
-
startMicrophone: () => startMicrophone,
|
|
16
|
-
stopMicrophone: () => stopMicrophone
|
|
17
|
-
});
|
|
18
|
-
|
|
19
1
|
// src/audio/utils/AudioAnalyser.ts
|
|
20
2
|
import EventEmitter from "eventemitter3";
|
|
21
3
|
var AudioAnalyser = class extends EventEmitter {
|
|
@@ -51,8 +33,42 @@ var AudioAnalyser = class extends EventEmitter {
|
|
|
51
33
|
}
|
|
52
34
|
};
|
|
53
35
|
|
|
36
|
+
// src/audio/utils/unlock.ts
|
|
37
|
+
function unlock(audioContext2) {
|
|
38
|
+
const events = ["touchstart", "touchend", "mousedown", "keydown"];
|
|
39
|
+
const unlock2 = async () => {
|
|
40
|
+
unlockSafari(audioContext2);
|
|
41
|
+
if (audioContext2.state === "suspended") {
|
|
42
|
+
await audioContext2.resume();
|
|
43
|
+
}
|
|
44
|
+
events.forEach((e) => document.body.removeEventListener(e, unlock2));
|
|
45
|
+
};
|
|
46
|
+
events.forEach((e) => document.body.addEventListener(e, unlock2, false));
|
|
47
|
+
}
|
|
48
|
+
function unlockSafari(audioContext2) {
|
|
49
|
+
const source = audioContext2.createBufferSource();
|
|
50
|
+
source.buffer = audioContext2.createBuffer(1, 1, 22050);
|
|
51
|
+
source.connect(audioContext2.destination);
|
|
52
|
+
const onEnded = () => {
|
|
53
|
+
source.disconnect();
|
|
54
|
+
source.buffer = null;
|
|
55
|
+
source.removeEventListener("ended", onEnded);
|
|
56
|
+
};
|
|
57
|
+
source.addEventListener("ended", onEnded);
|
|
58
|
+
source.start();
|
|
59
|
+
}
|
|
60
|
+
|
|
54
61
|
// src/audio/utils/audioContext.ts
|
|
55
|
-
var audioContext
|
|
62
|
+
var audioContext;
|
|
63
|
+
function createAudioContext() {
|
|
64
|
+
return new (window.AudioContext || window.webkitAudioContext)();
|
|
65
|
+
}
|
|
66
|
+
if (window.micdropAudioContext) {
|
|
67
|
+
audioContext = window.micdropAudioContext;
|
|
68
|
+
} else {
|
|
69
|
+
window.micdropAudioContext = audioContext = createAudioContext();
|
|
70
|
+
unlock(audioContext);
|
|
71
|
+
}
|
|
56
72
|
|
|
57
73
|
// src/audio/utils/localStorage.ts
|
|
58
74
|
var LocalStorageKeys = /* @__PURE__ */ ((LocalStorageKeys2) => {
|
|
@@ -67,71 +83,86 @@ function stopStream(stream) {
|
|
|
67
83
|
stream?.getTracks().forEach((track) => track.stop());
|
|
68
84
|
}
|
|
69
85
|
|
|
70
|
-
// src/audio/
|
|
71
|
-
var
|
|
72
|
-
var audioStream;
|
|
73
|
-
var sourceNode;
|
|
74
|
-
var micAnalyser = new AudioAnalyser(audioContext);
|
|
86
|
+
// src/audio/mic.ts
|
|
87
|
+
var mic;
|
|
75
88
|
var defaultMicThreshold = -50;
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
if (!deviceId) {
|
|
94
|
-
deviceId = localStorage.getItem("micdrop.micDevice" /* MicDevice */) || void 0;
|
|
89
|
+
var Mic = class {
|
|
90
|
+
constructor() {
|
|
91
|
+
this.defaultThreshold = defaultMicThreshold;
|
|
92
|
+
this.analyser = new AudioAnalyser(audioContext);
|
|
93
|
+
}
|
|
94
|
+
getMicConstraints(deviceId) {
|
|
95
|
+
return {
|
|
96
|
+
audio: {
|
|
97
|
+
deviceId: { ideal: deviceId },
|
|
98
|
+
sampleRate: 16e3,
|
|
99
|
+
// not working, it will follow device settings, usually 44.1kHz
|
|
100
|
+
sampleSize: 16,
|
|
101
|
+
channelCount: 1,
|
|
102
|
+
echoCancellation: true,
|
|
103
|
+
noiseSuppression: true
|
|
104
|
+
}
|
|
105
|
+
};
|
|
95
106
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
107
|
+
/**
|
|
108
|
+
* Starts the microphone
|
|
109
|
+
* @param deviceId - The deviceId to use
|
|
110
|
+
* @returns The microphone stream
|
|
111
|
+
*/
|
|
112
|
+
async start(deviceId) {
|
|
113
|
+
if (!audioContext || !this.analyser) {
|
|
114
|
+
throw new Error("AudioContext not initialized");
|
|
99
115
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
sourceNode.disconnect();
|
|
122
|
-
sourceNode =
|
|
116
|
+
if (!deviceId) {
|
|
117
|
+
deviceId = localStorage.getItem("micdrop.micDevice" /* MicDevice */) || void 0;
|
|
118
|
+
}
|
|
119
|
+
if (this.audioStream) {
|
|
120
|
+
if (this.currentDeviceId === deviceId) {
|
|
121
|
+
return this.audioStream;
|
|
122
|
+
}
|
|
123
|
+
stopStream(this.audioStream);
|
|
124
|
+
}
|
|
125
|
+
this.audioStream = await navigator.mediaDevices.getUserMedia(
|
|
126
|
+
this.getMicConstraints(deviceId)
|
|
127
|
+
);
|
|
128
|
+
const track = this.audioStream.getTracks()[0];
|
|
129
|
+
const newDeviceId = track && track.getCapabilities && track.getCapabilities().deviceId;
|
|
130
|
+
if (newDeviceId) {
|
|
131
|
+
localStorage.setItem("micdrop.micDevice" /* MicDevice */, newDeviceId);
|
|
132
|
+
this.currentDeviceId = newDeviceId;
|
|
133
|
+
} else {
|
|
134
|
+
localStorage.removeItem("micdrop.micDevice" /* MicDevice */);
|
|
135
|
+
this.currentDeviceId = void 0;
|
|
136
|
+
}
|
|
137
|
+
if (this.sourceNode) this.sourceNode.disconnect();
|
|
138
|
+
this.sourceNode = audioContext.createMediaStreamSource(this.audioStream);
|
|
139
|
+
this.sourceNode.connect(this.analyser.node);
|
|
140
|
+
return this.audioStream;
|
|
123
141
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
142
|
+
/**
|
|
143
|
+
* Stops the microphone
|
|
144
|
+
*/
|
|
145
|
+
stop() {
|
|
146
|
+
if (this.sourceNode) {
|
|
147
|
+
this.sourceNode.disconnect();
|
|
148
|
+
this.sourceNode = void 0;
|
|
149
|
+
}
|
|
150
|
+
if (this.audioStream) {
|
|
151
|
+
stopStream(this.audioStream);
|
|
152
|
+
this.audioStream = void 0;
|
|
153
|
+
}
|
|
127
154
|
}
|
|
155
|
+
};
|
|
156
|
+
if (!window.micdropMic) {
|
|
157
|
+
window.micdropMic = new Mic();
|
|
128
158
|
}
|
|
159
|
+
mic = window.micdropMic;
|
|
129
160
|
|
|
130
|
-
// src/audio/
|
|
161
|
+
// src/audio/MicRecorder.ts
|
|
162
|
+
import EventEmitter3 from "eventemitter3";
|
|
163
|
+
|
|
164
|
+
// src/audio/utils/delayedStream.ts
|
|
131
165
|
function createDelayedStream(stream, delay) {
|
|
132
|
-
if (!audioContext) {
|
|
133
|
-
throw new Error("AudioContext not initialized");
|
|
134
|
-
}
|
|
135
166
|
const audioSource = audioContext.createMediaStreamSource(stream);
|
|
136
167
|
const delayNode = audioContext.createDelay(delay);
|
|
137
168
|
delayNode.delayTime.value = delay;
|
|
@@ -141,6 +172,166 @@ function createDelayedStream(stream, delay) {
|
|
|
141
172
|
return audioDestination.stream;
|
|
142
173
|
}
|
|
143
174
|
|
|
175
|
+
// src/audio/vad/SileroVAD.ts
|
|
176
|
+
import { MicVAD } from "@ricky0123/vad-web/src";
|
|
177
|
+
|
|
178
|
+
// src/audio/vad/VAD.ts
|
|
179
|
+
import EventEmitter2 from "eventemitter3";
|
|
180
|
+
var VAD = class extends EventEmitter2 {
|
|
181
|
+
/**
|
|
182
|
+
* Constructor for the VAD class
|
|
183
|
+
* @param threshold - The threshold to use for the VAD
|
|
184
|
+
*/
|
|
185
|
+
constructor(threshold = 0) {
|
|
186
|
+
super();
|
|
187
|
+
// Max delay to detect speech, can be used to record delayed stream
|
|
188
|
+
this.delay = 100;
|
|
189
|
+
this.threshold = threshold;
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
// src/audio/vad/SileroVAD.ts
|
|
194
|
+
var SileroVAD = class extends VAD {
|
|
195
|
+
get isStarted() {
|
|
196
|
+
return !!this.vad;
|
|
197
|
+
}
|
|
198
|
+
async start(stream, threshold) {
|
|
199
|
+
if (this.vad) return;
|
|
200
|
+
if (threshold !== void 0) {
|
|
201
|
+
this.threshold = threshold;
|
|
202
|
+
}
|
|
203
|
+
const speechThreshold = Math.min(
|
|
204
|
+
Math.max((this.threshold + 100) / 100, 0),
|
|
205
|
+
1
|
|
206
|
+
);
|
|
207
|
+
this.vad = await MicVAD.new({
|
|
208
|
+
stream,
|
|
209
|
+
model: "v5",
|
|
210
|
+
onSpeechStart: () => this.emit("StartSpeaking"),
|
|
211
|
+
onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
|
|
212
|
+
onVADMisfire: () => this.emit("CancelSpeaking"),
|
|
213
|
+
onSpeechEnd: () => this.emit("StopSpeaking"),
|
|
214
|
+
positiveSpeechThreshold: speechThreshold,
|
|
215
|
+
negativeSpeechThreshold: speechThreshold * 0.7,
|
|
216
|
+
minSpeechFrames: 3,
|
|
217
|
+
redemptionFrames: 9
|
|
218
|
+
});
|
|
219
|
+
this.vad.start();
|
|
220
|
+
}
|
|
221
|
+
async stop() {
|
|
222
|
+
if (!this.vad) return;
|
|
223
|
+
this.vad.destroy();
|
|
224
|
+
this.vad = void 0;
|
|
225
|
+
}
|
|
226
|
+
async setThreshold(threshold) {
|
|
227
|
+
if (threshold === this.threshold) return;
|
|
228
|
+
if (this.vad) {
|
|
229
|
+
const stream = this.vad["stream"];
|
|
230
|
+
await this.stop();
|
|
231
|
+
await this.start(stream, threshold);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
// src/audio/vad/VolumeVAD.ts
|
|
237
|
+
var VolumeVAD = class extends VAD {
|
|
238
|
+
constructor() {
|
|
239
|
+
super();
|
|
240
|
+
this.running = false;
|
|
241
|
+
this.speaking = false;
|
|
242
|
+
this.attemptSpeaking = false;
|
|
243
|
+
this.history = 10;
|
|
244
|
+
this.speakingHistory = [];
|
|
245
|
+
for (let i = 0; i < this.history; i++) {
|
|
246
|
+
this.speakingHistory.push(0);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
getMaxVolume() {
|
|
250
|
+
if (!this.analyser || !this.fftBins) return -Infinity;
|
|
251
|
+
let maxVolume = -Infinity;
|
|
252
|
+
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
253
|
+
for (let i = 4; i < this.fftBins.length; i++) {
|
|
254
|
+
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
255
|
+
maxVolume = this.fftBins[i];
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return maxVolume;
|
|
259
|
+
}
|
|
260
|
+
get isStarted() {
|
|
261
|
+
return this.running;
|
|
262
|
+
}
|
|
263
|
+
async start(stream, threshold) {
|
|
264
|
+
if (this.running) return;
|
|
265
|
+
if (threshold !== void 0) {
|
|
266
|
+
this.threshold = threshold;
|
|
267
|
+
}
|
|
268
|
+
this.analyser = audioContext.createAnalyser();
|
|
269
|
+
this.sourceNode = audioContext.createMediaStreamSource(stream);
|
|
270
|
+
this.analyser.fftSize = 512;
|
|
271
|
+
this.analyser.smoothingTimeConstant = 0.1;
|
|
272
|
+
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
273
|
+
this.sourceNode.connect(this.analyser);
|
|
274
|
+
this.running = true;
|
|
275
|
+
this.startLoop();
|
|
276
|
+
}
|
|
277
|
+
startLoop() {
|
|
278
|
+
if (!this.running) return;
|
|
279
|
+
const currentVolume = this.getMaxVolume();
|
|
280
|
+
if (currentVolume > this.threshold) {
|
|
281
|
+
if (!this.speaking) {
|
|
282
|
+
let recentHistory = 0;
|
|
283
|
+
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
284
|
+
recentHistory += this.speakingHistory[i];
|
|
285
|
+
}
|
|
286
|
+
if (recentHistory >= 2) {
|
|
287
|
+
this.speaking = true;
|
|
288
|
+
this.attemptSpeaking = false;
|
|
289
|
+
this.emit("ConfirmSpeaking");
|
|
290
|
+
} else if (recentHistory === 1) {
|
|
291
|
+
this.attemptSpeaking = true;
|
|
292
|
+
this.emit("StartSpeaking");
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
} else if (this.speaking) {
|
|
296
|
+
let totalHistory = 0;
|
|
297
|
+
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
298
|
+
totalHistory += this.speakingHistory[i];
|
|
299
|
+
}
|
|
300
|
+
if (totalHistory === 0) {
|
|
301
|
+
this.speaking = false;
|
|
302
|
+
this.emit("StopSpeaking");
|
|
303
|
+
}
|
|
304
|
+
} else if (this.attemptSpeaking) {
|
|
305
|
+
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
306
|
+
if (lastHistory === 0) {
|
|
307
|
+
this.attemptSpeaking = false;
|
|
308
|
+
this.emit("CancelSpeaking");
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
this.speakingHistory.shift();
|
|
312
|
+
this.speakingHistory.push(Number(currentVolume > this.threshold));
|
|
313
|
+
setTimeout(() => this.startLoop(), this.delay);
|
|
314
|
+
}
|
|
315
|
+
async stop() {
|
|
316
|
+
if (!this.running) return;
|
|
317
|
+
this.running = false;
|
|
318
|
+
this.speaking = false;
|
|
319
|
+
if (this.sourceNode) {
|
|
320
|
+
this.sourceNode.disconnect();
|
|
321
|
+
this.sourceNode = void 0;
|
|
322
|
+
}
|
|
323
|
+
if (this.analyser) {
|
|
324
|
+
this.analyser.disconnect();
|
|
325
|
+
this.analyser = void 0;
|
|
326
|
+
}
|
|
327
|
+
this.fftBins = void 0;
|
|
328
|
+
}
|
|
329
|
+
async setThreshold(threshold) {
|
|
330
|
+
if (threshold === this.threshold) return;
|
|
331
|
+
this.threshold = threshold;
|
|
332
|
+
}
|
|
333
|
+
};
|
|
334
|
+
|
|
144
335
|
// src/audio/MicRecorder.ts
|
|
145
336
|
var timeSlice = 100;
|
|
146
337
|
var defaultMicRecorderState = {
|
|
@@ -150,9 +341,10 @@ var defaultMicRecorderState = {
|
|
|
150
341
|
isSpeaking: false,
|
|
151
342
|
threshold: defaultMicThreshold
|
|
152
343
|
};
|
|
153
|
-
var MicRecorder = class extends
|
|
154
|
-
constructor(
|
|
344
|
+
var MicRecorder = class extends EventEmitter3 {
|
|
345
|
+
constructor(vadConfig) {
|
|
155
346
|
super();
|
|
347
|
+
this.vadConfig = vadConfig;
|
|
156
348
|
this.audioInfo = this.getAudioInfo();
|
|
157
349
|
this.speakingConfirmed = false;
|
|
158
350
|
this.queuedChunks = [];
|
|
@@ -193,7 +385,6 @@ var MicRecorder = class extends EventEmitter2 {
|
|
|
193
385
|
}).bind(this);
|
|
194
386
|
const threshold = parseFloat(localStorage.getItem("micdrop.micThreshold" /* MicThreshold */) || "0") || defaultMicThreshold;
|
|
195
387
|
this.state = { ...defaultMicRecorderState, threshold };
|
|
196
|
-
this.vad = vad;
|
|
197
388
|
}
|
|
198
389
|
async start(stream) {
|
|
199
390
|
if (this.state.isStarted) {
|
|
@@ -205,6 +396,9 @@ var MicRecorder = class extends EventEmitter2 {
|
|
|
205
396
|
isMuted: false,
|
|
206
397
|
isSpeaking: false
|
|
207
398
|
});
|
|
399
|
+
if (!this.vad) {
|
|
400
|
+
this.vad = this.initVAD(this.vadConfig);
|
|
401
|
+
}
|
|
208
402
|
const delayedStream = createDelayedStream(
|
|
209
403
|
stream,
|
|
210
404
|
this.vad.delay / 1e3 + 0.05
|
|
@@ -259,9 +453,18 @@ var MicRecorder = class extends EventEmitter2 {
|
|
|
259
453
|
setThreshold(threshold) {
|
|
260
454
|
this.changeState({ threshold });
|
|
261
455
|
localStorage.setItem("micdrop.micThreshold" /* MicThreshold */, threshold.toString());
|
|
262
|
-
this.vad
|
|
456
|
+
this.vad?.setThreshold(threshold);
|
|
457
|
+
}
|
|
458
|
+
initVAD(vad) {
|
|
459
|
+
if (!vad) return new VolumeVAD();
|
|
460
|
+
if (vad === "volume") return new VolumeVAD();
|
|
461
|
+
if (vad === "silero") return new SileroVAD();
|
|
462
|
+
return vad;
|
|
263
463
|
}
|
|
264
464
|
async startSpeakingDetection(stream) {
|
|
465
|
+
if (!this.vad) {
|
|
466
|
+
throw new Error("VAD not initialized");
|
|
467
|
+
}
|
|
265
468
|
if (this.vad.isStarted) {
|
|
266
469
|
this.vad.stop();
|
|
267
470
|
}
|
|
@@ -272,6 +475,9 @@ var MicRecorder = class extends EventEmitter2 {
|
|
|
272
475
|
await this.vad.start(stream, this.state.threshold);
|
|
273
476
|
}
|
|
274
477
|
async stopSpeakingDetection() {
|
|
478
|
+
if (!this.vad) {
|
|
479
|
+
throw new Error("VAD not initialized");
|
|
480
|
+
}
|
|
275
481
|
this.vad.off("StartSpeaking", this.onStartSpeaking);
|
|
276
482
|
this.vad.off("ConfirmSpeaking", this.onConfirmSpeaking);
|
|
277
483
|
this.vad.off("CancelSpeaking", this.onCancelSpeaking);
|
|
@@ -309,21 +515,6 @@ var MicRecorder = class extends EventEmitter2 {
|
|
|
309
515
|
}
|
|
310
516
|
};
|
|
311
517
|
|
|
312
|
-
// src/audio/speaker.ts
|
|
313
|
-
var speaker_exports = {};
|
|
314
|
-
__export(speaker_exports, {
|
|
315
|
-
canChangeSpeakerDevice: () => canChangeSpeakerDevice,
|
|
316
|
-
changeSpeakerDevice: () => changeSpeakerDevice,
|
|
317
|
-
disableStreaming: () => disableStreaming,
|
|
318
|
-
enableStreaming: () => enableStreaming,
|
|
319
|
-
isStreamingEnabled: () => isStreamingEnabled,
|
|
320
|
-
pauseAudio: () => pauseAudio,
|
|
321
|
-
playAudio: () => playAudio,
|
|
322
|
-
resumeAudio: () => resumeAudio,
|
|
323
|
-
speakerAnalyser: () => speakerAnalyser,
|
|
324
|
-
stopAudio: () => stopAudio
|
|
325
|
-
});
|
|
326
|
-
|
|
327
518
|
// src/audio/player/SpeakerPlayer.ts
|
|
328
519
|
var SpeakerPlayer = class {
|
|
329
520
|
constructor(outputNode) {
|
|
@@ -514,7 +705,7 @@ var SpeakerMediaSourcePlayer = class extends SpeakerPlayer {
|
|
|
514
705
|
this.audioElement = void 0;
|
|
515
706
|
}
|
|
516
707
|
if (this.mediaStream) {
|
|
517
|
-
this.mediaStream
|
|
708
|
+
stopStream(this.mediaStream);
|
|
518
709
|
this.mediaStream = void 0;
|
|
519
710
|
}
|
|
520
711
|
this.mediaSource = void 0;
|
|
@@ -579,280 +770,94 @@ var SpeakerSequencePlayer = class extends SpeakerPlayer {
|
|
|
579
770
|
}
|
|
580
771
|
};
|
|
581
772
|
|
|
582
|
-
// src/audio/
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
773
|
+
// src/audio/speaker.ts
|
|
774
|
+
var speaker;
|
|
775
|
+
var Speaker = class {
|
|
776
|
+
constructor() {
|
|
777
|
+
this.streamingEnabled = false;
|
|
778
|
+
this.analyser = new AudioAnalyser(audioContext);
|
|
779
|
+
this.audioElement = new Audio();
|
|
780
|
+
this.audioElement.autoplay = true;
|
|
781
|
+
const speakerId = localStorage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
782
|
+
if (speakerId) {
|
|
783
|
+
this.setSinkId(speakerId).catch((error) => {
|
|
784
|
+
console.error("Failed to set sink ID:", error);
|
|
785
|
+
});
|
|
590
786
|
}
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
787
|
+
this.setupSpeakerPlayer();
|
|
788
|
+
const destinationNode = audioContext.createMediaStreamDestination();
|
|
789
|
+
this.analyser.node.connect(destinationNode);
|
|
790
|
+
this.audioElement.srcObject = destinationNode.stream;
|
|
595
791
|
}
|
|
596
|
-
|
|
597
|
-
function unlockSafari(audioContext2) {
|
|
598
|
-
const source = audioContext2.createBufferSource();
|
|
599
|
-
source.buffer = audioContext2.createBuffer(1, 1, 22050);
|
|
600
|
-
source.connect(audioContext2.destination);
|
|
601
|
-
const onEnded = () => {
|
|
602
|
-
source.disconnect();
|
|
603
|
-
source.buffer = null;
|
|
604
|
-
source.removeEventListener("ended", onEnded);
|
|
605
|
-
};
|
|
606
|
-
source.addEventListener("ended", onEnded);
|
|
607
|
-
source.start();
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
// src/audio/speaker.ts
|
|
611
|
-
var audioElement;
|
|
612
|
-
var player;
|
|
613
|
-
var streamingEnabled = false;
|
|
614
|
-
var speakerAnalyser = new AudioAnalyser(audioContext);
|
|
615
|
-
async function init() {
|
|
616
|
-
unlock(audioContext);
|
|
617
|
-
audioElement = new Audio();
|
|
618
|
-
audioElement.autoplay = true;
|
|
619
|
-
const speakerId = localStorage.getItem("micdrop.speakerDevice" /* SpeakerDevice */);
|
|
620
|
-
if (speakerId) {
|
|
792
|
+
async setupSpeakerPlayer() {
|
|
621
793
|
try {
|
|
622
|
-
|
|
794
|
+
this.player?.destroy();
|
|
795
|
+
this.player = this.streamingEnabled ? SpeakerMediaSourcePlayer.isCompatible ? new SpeakerMediaSourcePlayer(this.analyser.node) : new SpeakerConcatPlayer(this.analyser.node) : new SpeakerSequencePlayer(this.analyser.node);
|
|
796
|
+
await this.player?.setup();
|
|
623
797
|
} catch (error) {
|
|
624
|
-
console.error("
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
await setupSpeakerPlayer();
|
|
628
|
-
const destinationNode = audioContext.createMediaStreamDestination();
|
|
629
|
-
speakerAnalyser.node.connect(destinationNode);
|
|
630
|
-
audioElement.srcObject = destinationNode.stream;
|
|
631
|
-
}
|
|
632
|
-
init();
|
|
633
|
-
async function setupSpeakerPlayer() {
|
|
634
|
-
try {
|
|
635
|
-
player?.destroy();
|
|
636
|
-
player = streamingEnabled ? SpeakerMediaSourcePlayer.isCompatible ? new SpeakerMediaSourcePlayer(speakerAnalyser.node) : new SpeakerConcatPlayer(speakerAnalyser.node) : new SpeakerSequencePlayer(speakerAnalyser.node);
|
|
637
|
-
await player?.setup();
|
|
638
|
-
} catch (error) {
|
|
639
|
-
console.error("Error setting up speaker player", error);
|
|
640
|
-
}
|
|
641
|
-
}
|
|
642
|
-
async function isStreamingEnabled() {
|
|
643
|
-
return streamingEnabled;
|
|
644
|
-
}
|
|
645
|
-
async function enableStreaming() {
|
|
646
|
-
if (streamingEnabled) return;
|
|
647
|
-
streamingEnabled = true;
|
|
648
|
-
await setupSpeakerPlayer();
|
|
649
|
-
}
|
|
650
|
-
async function disableStreaming() {
|
|
651
|
-
if (!streamingEnabled) return;
|
|
652
|
-
streamingEnabled = false;
|
|
653
|
-
await setupSpeakerPlayer();
|
|
654
|
-
}
|
|
655
|
-
function canChangeSpeakerDevice() {
|
|
656
|
-
return window.HTMLMediaElement && "sinkId" in window.HTMLMediaElement.prototype;
|
|
657
|
-
}
|
|
658
|
-
async function setSinkId(speakerId) {
|
|
659
|
-
if (speakerId && canChangeSpeakerDevice() && audioElement) {
|
|
660
|
-
return audioElement.setSinkId(speakerId);
|
|
661
|
-
}
|
|
662
|
-
}
|
|
663
|
-
async function changeSpeakerDevice(speakerId) {
|
|
664
|
-
if (!canChangeSpeakerDevice()) return;
|
|
665
|
-
try {
|
|
666
|
-
await setSinkId(speakerId);
|
|
667
|
-
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, speakerId);
|
|
668
|
-
} catch (error) {
|
|
669
|
-
console.error(`Error setting Audio Output to ${speakerId}`, error);
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
function pauseAudio() {
|
|
673
|
-
if (audioElement) {
|
|
674
|
-
audioElement.pause();
|
|
675
|
-
}
|
|
676
|
-
}
|
|
677
|
-
function resumeAudio() {
|
|
678
|
-
if (audioElement) {
|
|
679
|
-
audioElement.play();
|
|
680
|
-
}
|
|
681
|
-
}
|
|
682
|
-
async function playAudio(blob) {
|
|
683
|
-
if (!audioElement || !player) return;
|
|
684
|
-
try {
|
|
685
|
-
player.addBlob(blob);
|
|
686
|
-
if (audioElement.paused) {
|
|
687
|
-
audioElement.play();
|
|
798
|
+
console.error("Error setting up speaker player", error);
|
|
688
799
|
}
|
|
689
|
-
} catch (error) {
|
|
690
|
-
console.error("Error playing audio blob", error, blob);
|
|
691
800
|
}
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
audioElement?.pause();
|
|
695
|
-
player?.stop();
|
|
696
|
-
}
|
|
697
|
-
|
|
698
|
-
// src/audio/vad/SileroVAD.ts
|
|
699
|
-
import { MicVAD } from "@ricky0123/vad-web/src";
|
|
700
|
-
|
|
701
|
-
// src/audio/vad/VAD.ts
|
|
702
|
-
import EventEmitter3 from "eventemitter3";
|
|
703
|
-
var VAD = class extends EventEmitter3 {
|
|
704
|
-
/**
|
|
705
|
-
* Constructor for the VAD class
|
|
706
|
-
* @param threshold - The threshold to use for the VAD
|
|
707
|
-
*/
|
|
708
|
-
constructor(threshold = 0) {
|
|
709
|
-
super();
|
|
710
|
-
// Max delay to detect speech, can be used to record delayed stream
|
|
711
|
-
this.delay = 100;
|
|
712
|
-
this.threshold = threshold;
|
|
801
|
+
get isStreamingEnabled() {
|
|
802
|
+
return this.streamingEnabled;
|
|
713
803
|
}
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
get isStarted() {
|
|
719
|
-
return !!this.vad;
|
|
804
|
+
async enableStreaming() {
|
|
805
|
+
if (this.streamingEnabled) return;
|
|
806
|
+
this.streamingEnabled = true;
|
|
807
|
+
await this.setupSpeakerPlayer();
|
|
720
808
|
}
|
|
721
|
-
async
|
|
722
|
-
if (this.
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
}
|
|
726
|
-
const speechThreshold = Math.min(
|
|
727
|
-
Math.max((this.threshold + 100) / 100, 0),
|
|
728
|
-
1
|
|
729
|
-
);
|
|
730
|
-
this.vad = await MicVAD.new({
|
|
731
|
-
stream,
|
|
732
|
-
onSpeechStart: () => this.emit("StartSpeaking"),
|
|
733
|
-
onSpeechRealStart: () => this.emit("ConfirmSpeaking"),
|
|
734
|
-
onVADMisfire: () => this.emit("CancelSpeaking"),
|
|
735
|
-
onSpeechEnd: () => this.emit("StopSpeaking"),
|
|
736
|
-
positiveSpeechThreshold: speechThreshold,
|
|
737
|
-
negativeSpeechThreshold: speechThreshold * 0.7,
|
|
738
|
-
// Adjust negative threshold proportionally
|
|
739
|
-
redemptionFrames: 5
|
|
740
|
-
});
|
|
741
|
-
this.vad.start();
|
|
809
|
+
async disableStreaming() {
|
|
810
|
+
if (!this.streamingEnabled) return;
|
|
811
|
+
this.streamingEnabled = false;
|
|
812
|
+
await this.setupSpeakerPlayer();
|
|
742
813
|
}
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
this.vad.pause();
|
|
746
|
-
this.vad = void 0;
|
|
814
|
+
canChangeDevice() {
|
|
815
|
+
return window.HTMLMediaElement && "sinkId" in window.HTMLMediaElement.prototype;
|
|
747
816
|
}
|
|
748
|
-
async
|
|
749
|
-
if (
|
|
750
|
-
|
|
751
|
-
const stream = this.vad["stream"];
|
|
752
|
-
await this.stop();
|
|
753
|
-
await this.start(stream, threshold);
|
|
817
|
+
async setSinkId(speakerId) {
|
|
818
|
+
if (speakerId && this.canChangeDevice() && this.audioElement) {
|
|
819
|
+
return this.audioElement.setSinkId(speakerId);
|
|
754
820
|
}
|
|
755
821
|
}
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
this.speaking = false;
|
|
764
|
-
this.attemptSpeaking = false;
|
|
765
|
-
this.history = 10;
|
|
766
|
-
this.speakingHistory = [];
|
|
767
|
-
for (let i = 0; i < this.history; i++) {
|
|
768
|
-
this.speakingHistory.push(0);
|
|
822
|
+
async changeDevice(speakerId) {
|
|
823
|
+
if (!this.canChangeDevice()) return;
|
|
824
|
+
try {
|
|
825
|
+
await this.setSinkId(speakerId);
|
|
826
|
+
localStorage.setItem("micdrop.speakerDevice" /* SpeakerDevice */, speakerId);
|
|
827
|
+
} catch (error) {
|
|
828
|
+
console.error(`Error setting Audio Output to ${speakerId}`, error);
|
|
769
829
|
}
|
|
770
830
|
}
|
|
771
|
-
|
|
772
|
-
if (
|
|
773
|
-
|
|
774
|
-
this.analyser.getFloatFrequencyData(this.fftBins);
|
|
775
|
-
for (let i = 4; i < this.fftBins.length; i++) {
|
|
776
|
-
if (this.fftBins[i] > maxVolume && this.fftBins[i] < 0) {
|
|
777
|
-
maxVolume = this.fftBins[i];
|
|
778
|
-
}
|
|
831
|
+
pauseAudio() {
|
|
832
|
+
if (this.audioElement) {
|
|
833
|
+
this.audioElement.pause();
|
|
779
834
|
}
|
|
780
|
-
return maxVolume;
|
|
781
|
-
}
|
|
782
|
-
get isStarted() {
|
|
783
|
-
return this.running;
|
|
784
835
|
}
|
|
785
|
-
|
|
786
|
-
if (this.
|
|
787
|
-
|
|
788
|
-
this.threshold = threshold;
|
|
836
|
+
resumeAudio() {
|
|
837
|
+
if (this.audioElement) {
|
|
838
|
+
this.audioElement.play();
|
|
789
839
|
}
|
|
790
|
-
this.analyser = audioContext.createAnalyser();
|
|
791
|
-
this.sourceNode = audioContext.createMediaStreamSource(stream);
|
|
792
|
-
this.analyser.fftSize = 512;
|
|
793
|
-
this.analyser.smoothingTimeConstant = 0.1;
|
|
794
|
-
this.fftBins = new Float32Array(this.analyser.frequencyBinCount);
|
|
795
|
-
this.sourceNode.connect(this.analyser);
|
|
796
|
-
this.running = true;
|
|
797
|
-
this.startLoop();
|
|
798
840
|
}
|
|
799
|
-
|
|
800
|
-
if (!this.
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
if (
|
|
804
|
-
|
|
805
|
-
for (let i = this.speakingHistory.length - 3; i < this.speakingHistory.length; i++) {
|
|
806
|
-
recentHistory += this.speakingHistory[i];
|
|
807
|
-
}
|
|
808
|
-
if (recentHistory >= 2) {
|
|
809
|
-
this.speaking = true;
|
|
810
|
-
this.attemptSpeaking = false;
|
|
811
|
-
this.emit("ConfirmSpeaking");
|
|
812
|
-
} else if (recentHistory === 1) {
|
|
813
|
-
this.attemptSpeaking = true;
|
|
814
|
-
this.emit("StartSpeaking");
|
|
815
|
-
}
|
|
816
|
-
}
|
|
817
|
-
} else if (this.speaking) {
|
|
818
|
-
let totalHistory = 0;
|
|
819
|
-
for (let i = 0; i < this.speakingHistory.length; i++) {
|
|
820
|
-
totalHistory += this.speakingHistory[i];
|
|
821
|
-
}
|
|
822
|
-
if (totalHistory === 0) {
|
|
823
|
-
this.speaking = false;
|
|
824
|
-
this.emit("StopSpeaking");
|
|
825
|
-
}
|
|
826
|
-
} else if (this.attemptSpeaking) {
|
|
827
|
-
let lastHistory = this.speakingHistory[this.speakingHistory.length - 1];
|
|
828
|
-
if (lastHistory === 0) {
|
|
829
|
-
this.attemptSpeaking = false;
|
|
830
|
-
this.emit("CancelSpeaking");
|
|
841
|
+
async playAudio(blob) {
|
|
842
|
+
if (!this.audioElement || !this.player) return;
|
|
843
|
+
try {
|
|
844
|
+
this.player.addBlob(blob);
|
|
845
|
+
if (this.audioElement.paused) {
|
|
846
|
+
this.audioElement.play();
|
|
831
847
|
}
|
|
848
|
+
} catch (error) {
|
|
849
|
+
console.error("Error playing audio blob", error, blob);
|
|
832
850
|
}
|
|
833
|
-
this.speakingHistory.shift();
|
|
834
|
-
this.speakingHistory.push(Number(currentVolume > this.threshold));
|
|
835
|
-
setTimeout(() => this.startLoop(), this.delay);
|
|
836
|
-
}
|
|
837
|
-
async stop() {
|
|
838
|
-
if (!this.running) return;
|
|
839
|
-
this.running = false;
|
|
840
|
-
this.speaking = false;
|
|
841
|
-
if (this.sourceNode) {
|
|
842
|
-
this.sourceNode.disconnect();
|
|
843
|
-
this.sourceNode = void 0;
|
|
844
|
-
}
|
|
845
|
-
if (this.analyser) {
|
|
846
|
-
this.analyser.disconnect();
|
|
847
|
-
this.analyser = void 0;
|
|
848
|
-
}
|
|
849
|
-
this.fftBins = void 0;
|
|
850
851
|
}
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
this.
|
|
852
|
+
stopAudio() {
|
|
853
|
+
this.audioElement?.pause();
|
|
854
|
+
this.player?.stop();
|
|
854
855
|
}
|
|
855
856
|
};
|
|
857
|
+
if (!window.micdropSpeaker) {
|
|
858
|
+
window.micdropSpeaker = new Speaker();
|
|
859
|
+
}
|
|
860
|
+
speaker = window.micdropSpeaker;
|
|
856
861
|
|
|
857
862
|
// src/call/CallHandler.ts
|
|
858
863
|
import EventEmitter4 from "eventemitter3";
|
|
@@ -875,49 +880,23 @@ var CallHandlerError = class extends Error {
|
|
|
875
880
|
var CallHandler = class _CallHandler extends EventEmitter4 {
|
|
876
881
|
constructor(options) {
|
|
877
882
|
super();
|
|
883
|
+
this.options = options;
|
|
878
884
|
this.conversation = [];
|
|
879
885
|
this.debug = false;
|
|
880
886
|
this.startTime = 0;
|
|
881
887
|
this._isProcessing = false;
|
|
882
|
-
this.micRecorder = new MicRecorder(this.getVAD(options?.vad));
|
|
883
|
-
this.micRecorder.on("StateChange", () => {
|
|
884
|
-
this.notifyStateChange();
|
|
885
|
-
});
|
|
886
|
-
this.micRecorder.on("Chunk", (blob) => {
|
|
887
|
-
this.log(`[Mic] Received chunk`, blob);
|
|
888
|
-
this.ws?.send(blob);
|
|
889
|
-
});
|
|
890
|
-
this.micRecorder.on("StartSpeaking", () => {
|
|
891
|
-
this.log("[Mic] Start speaking");
|
|
892
|
-
this.ws?.send("startSpeaking" /* StartSpeaking */);
|
|
893
|
-
this._isProcessing = false;
|
|
894
|
-
this.notifyStateChange();
|
|
895
|
-
stopAudio();
|
|
896
|
-
});
|
|
897
|
-
this.micRecorder.on("StopSpeaking", () => {
|
|
898
|
-
this.log("[Mic] Stop speaking");
|
|
899
|
-
this.ws?.send("stopSpeaking" /* StopSpeaking */);
|
|
900
|
-
this._isProcessing = true;
|
|
901
|
-
this.notifyStateChange();
|
|
902
|
-
});
|
|
903
888
|
}
|
|
904
889
|
static getInstance(options) {
|
|
905
|
-
if (!
|
|
906
|
-
|
|
890
|
+
if (!window.micdropCallHandler) {
|
|
891
|
+
window.micdropCallHandler = new _CallHandler(options);
|
|
907
892
|
}
|
|
908
|
-
return
|
|
909
|
-
}
|
|
910
|
-
getVAD(vad) {
|
|
911
|
-
if (!vad) return new VolumeVAD();
|
|
912
|
-
if (vad === "volume") return new VolumeVAD();
|
|
913
|
-
if (vad === "silero") return new SileroVAD();
|
|
914
|
-
return vad;
|
|
893
|
+
return window.micdropCallHandler;
|
|
915
894
|
}
|
|
916
895
|
get isStarted() {
|
|
917
|
-
return this.isWSStarted && this.micRecorder
|
|
896
|
+
return this.isWSStarted && this.micRecorder?.state.isStarted || false;
|
|
918
897
|
}
|
|
919
898
|
get isStarting() {
|
|
920
|
-
return this.isWSStarting || this.micRecorder
|
|
899
|
+
return this.isWSStarting || this.micRecorder?.state.isStarting || false;
|
|
921
900
|
}
|
|
922
901
|
get isWSStarted() {
|
|
923
902
|
return this.ws?.readyState === WebSocket.OPEN;
|
|
@@ -928,6 +907,15 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
|
|
|
928
907
|
get isMicStarted() {
|
|
929
908
|
return !!this.micStream;
|
|
930
909
|
}
|
|
910
|
+
get isMicMuted() {
|
|
911
|
+
return this.micRecorder?.state.isMuted ?? false;
|
|
912
|
+
}
|
|
913
|
+
get isMicSpeaking() {
|
|
914
|
+
return this.micRecorder?.state.isSpeaking ?? false;
|
|
915
|
+
}
|
|
916
|
+
get micThreshold() {
|
|
917
|
+
return this.micRecorder?.state.threshold ?? mic.defaultThreshold;
|
|
918
|
+
}
|
|
931
919
|
get isProcessing() {
|
|
932
920
|
return this._isProcessing;
|
|
933
921
|
}
|
|
@@ -945,32 +933,57 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
|
|
|
945
933
|
console.error("[CallHandler] stop WS", error);
|
|
946
934
|
}
|
|
947
935
|
try {
|
|
948
|
-
this.micRecorder
|
|
936
|
+
this.micRecorder?.stop();
|
|
949
937
|
this.stopMic();
|
|
950
938
|
} catch (error) {
|
|
951
939
|
console.error("[CallHandler] stop mic", error);
|
|
952
940
|
}
|
|
953
941
|
}
|
|
954
942
|
pause() {
|
|
955
|
-
this.micRecorder
|
|
943
|
+
this.micRecorder?.mute();
|
|
944
|
+
this._isProcessing = false;
|
|
956
945
|
this.notifyStateChange();
|
|
957
|
-
pauseAudio();
|
|
946
|
+
speaker.pauseAudio();
|
|
947
|
+
this.ws?.send("Mute" /* Mute */);
|
|
958
948
|
}
|
|
959
949
|
resume() {
|
|
960
950
|
if (this.lastAudioBlob) {
|
|
961
|
-
playAudio(this.lastAudioBlob);
|
|
951
|
+
speaker.playAudio(this.lastAudioBlob);
|
|
962
952
|
}
|
|
963
|
-
resumeAudio();
|
|
964
|
-
this.micRecorder
|
|
953
|
+
speaker.resumeAudio();
|
|
954
|
+
this.micRecorder?.unmute();
|
|
965
955
|
this.notifyStateChange();
|
|
966
956
|
}
|
|
967
957
|
async startMic(deviceId, record = true) {
|
|
968
958
|
try {
|
|
959
|
+
if (!this.micRecorder) {
|
|
960
|
+
this.micRecorder = new MicRecorder(this.options?.vad);
|
|
961
|
+
this.micRecorder.on("StateChange", () => {
|
|
962
|
+
this.notifyStateChange();
|
|
963
|
+
});
|
|
964
|
+
this.micRecorder.on("Chunk", (blob) => {
|
|
965
|
+
this.log(`[Mic] Received chunk`, blob);
|
|
966
|
+
this.ws?.send(blob);
|
|
967
|
+
});
|
|
968
|
+
this.micRecorder.on("StartSpeaking", () => {
|
|
969
|
+
this.log("[Mic] Start speaking");
|
|
970
|
+
this.ws?.send("StartSpeaking" /* StartSpeaking */);
|
|
971
|
+
this._isProcessing = false;
|
|
972
|
+
this.notifyStateChange();
|
|
973
|
+
speaker.stopAudio();
|
|
974
|
+
});
|
|
975
|
+
this.micRecorder.on("StopSpeaking", () => {
|
|
976
|
+
this.log("[Mic] Stop speaking");
|
|
977
|
+
this.ws?.send("StopSpeaking" /* StopSpeaking */);
|
|
978
|
+
this._isProcessing = true;
|
|
979
|
+
this.notifyStateChange();
|
|
980
|
+
});
|
|
981
|
+
}
|
|
969
982
|
const isRecorderStarted = this.micRecorder.state.isStarted;
|
|
970
983
|
if (isRecorderStarted) {
|
|
971
|
-
this.micRecorder
|
|
984
|
+
this.micRecorder?.stop();
|
|
972
985
|
}
|
|
973
|
-
this.micStream = await
|
|
986
|
+
this.micStream = await mic.start(deviceId);
|
|
974
987
|
this.notifyStateChange();
|
|
975
988
|
if (isRecorderStarted || record) {
|
|
976
989
|
await this.micRecorder.start(this.micStream);
|
|
@@ -982,7 +995,9 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
|
|
|
982
995
|
}
|
|
983
996
|
}
|
|
984
997
|
stopMic() {
|
|
985
|
-
|
|
998
|
+
this.micRecorder?.stop();
|
|
999
|
+
this.micRecorder = void 0;
|
|
1000
|
+
mic.stop();
|
|
986
1001
|
this.micStream = void 0;
|
|
987
1002
|
this.notifyStateChange();
|
|
988
1003
|
}
|
|
@@ -1009,34 +1024,44 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
|
|
|
1009
1024
|
this.ws.onmessage = (event) => {
|
|
1010
1025
|
this.log("[WS]", event.data);
|
|
1011
1026
|
if (event.data instanceof Blob) {
|
|
1012
|
-
playAudio(event.data);
|
|
1027
|
+
speaker.playAudio(event.data);
|
|
1013
1028
|
this.lastAudioBlob = event.data;
|
|
1014
1029
|
this._isProcessing = false;
|
|
1015
1030
|
this.notifyStateChange();
|
|
1016
1031
|
} else if (typeof event.data !== "string") {
|
|
1017
1032
|
console.warn(`[WS] Unknown message type: ${event.data}`);
|
|
1018
|
-
} else if (event.data.startsWith("
|
|
1033
|
+
} else if (event.data.startsWith("Message" /* Message */)) {
|
|
1019
1034
|
const message = JSON.parse(
|
|
1020
|
-
event.data.substring("
|
|
1035
|
+
event.data.substring("Message" /* Message */.length + 1)
|
|
1021
1036
|
);
|
|
1022
1037
|
this.addMessage(message);
|
|
1023
|
-
} else if (event.data === "
|
|
1038
|
+
} else if (event.data === "EndCall" /* EndCall */) {
|
|
1024
1039
|
setTimeout(() => {
|
|
1025
|
-
this.emit("
|
|
1040
|
+
this.emit("EndCall");
|
|
1026
1041
|
}, 2e3);
|
|
1027
|
-
} else if (event.data === "
|
|
1042
|
+
} else if (event.data === "CancelLastAssistantMessage" /* CancelLastAssistantMessage */) {
|
|
1028
1043
|
const lastMessage = this.conversation[this.conversation.length - 1];
|
|
1029
1044
|
if (lastMessage?.role === "assistant") {
|
|
1030
1045
|
this.conversation = this.conversation.slice(0, -1);
|
|
1031
1046
|
this.notifyStateChange();
|
|
1032
1047
|
}
|
|
1033
|
-
} else if (event.data === "
|
|
1034
|
-
|
|
1048
|
+
} else if (event.data === "SkipAnswer" /* SkipAnswer */) {
|
|
1049
|
+
this._isProcessing = false;
|
|
1050
|
+
this.notifyStateChange();
|
|
1051
|
+
} else if (event.data === "CancelLastUserMessage" /* CancelLastUserMessage */) {
|
|
1052
|
+
const lastMessage = this.conversation[this.conversation.length - 1];
|
|
1053
|
+
if (lastMessage?.role === "user") {
|
|
1054
|
+
this.conversation = this.conversation.slice(0, -1);
|
|
1055
|
+
this._isProcessing = false;
|
|
1056
|
+
this.notifyStateChange();
|
|
1057
|
+
}
|
|
1058
|
+
} else if (event.data === "EnableSpeakerStreaming" /* EnableSpeakerStreaming */) {
|
|
1059
|
+
speaker.enableStreaming();
|
|
1035
1060
|
}
|
|
1036
1061
|
};
|
|
1037
1062
|
this.ws.onclose = (event) => {
|
|
1038
1063
|
this.log("[WS] Closed", event);
|
|
1039
|
-
this.
|
|
1064
|
+
this.stop();
|
|
1040
1065
|
if (event.code >= 1001 && event.code <= 1011 && event.code !== 1005) {
|
|
1041
1066
|
this.emit("Error", new CallHandlerError());
|
|
1042
1067
|
} else if (event.code === 4401) {
|
|
@@ -1074,16 +1099,18 @@ var CallHandler = class _CallHandler extends EventEmitter4 {
|
|
|
1074
1099
|
|
|
1075
1100
|
// ../server/src/types.ts
|
|
1076
1101
|
var CallClientCommands = /* @__PURE__ */ ((CallClientCommands2) => {
|
|
1077
|
-
CallClientCommands2["StartSpeaking"] = "
|
|
1078
|
-
CallClientCommands2["StopSpeaking"] = "
|
|
1079
|
-
CallClientCommands2["Mute"] = "
|
|
1102
|
+
CallClientCommands2["StartSpeaking"] = "StartSpeaking";
|
|
1103
|
+
CallClientCommands2["StopSpeaking"] = "StopSpeaking";
|
|
1104
|
+
CallClientCommands2["Mute"] = "Mute";
|
|
1080
1105
|
return CallClientCommands2;
|
|
1081
1106
|
})(CallClientCommands || {});
|
|
1082
1107
|
var CallServerCommands = /* @__PURE__ */ ((CallServerCommands2) => {
|
|
1083
|
-
CallServerCommands2["Message"] = "
|
|
1084
|
-
CallServerCommands2["CancelLastAssistantMessage"] = "
|
|
1085
|
-
CallServerCommands2["
|
|
1086
|
-
CallServerCommands2["
|
|
1108
|
+
CallServerCommands2["Message"] = "Message";
|
|
1109
|
+
CallServerCommands2["CancelLastAssistantMessage"] = "CancelLastAssistantMessage";
|
|
1110
|
+
CallServerCommands2["CancelLastUserMessage"] = "CancelLastUserMessage";
|
|
1111
|
+
CallServerCommands2["SkipAnswer"] = "SkipAnswer";
|
|
1112
|
+
CallServerCommands2["EnableSpeakerStreaming"] = "EnableSpeakerStreaming";
|
|
1113
|
+
CallServerCommands2["EndCall"] = "EndCall";
|
|
1087
1114
|
return CallServerCommands2;
|
|
1088
1115
|
})(CallServerCommands || {});
|
|
1089
1116
|
export {
|
|
@@ -1093,10 +1120,10 @@ export {
|
|
|
1093
1120
|
CallHandlerErrorCode,
|
|
1094
1121
|
CallServerCommands,
|
|
1095
1122
|
LocalStorageKeys,
|
|
1096
|
-
|
|
1123
|
+
mic as Mic,
|
|
1097
1124
|
MicRecorder,
|
|
1098
1125
|
SileroVAD,
|
|
1099
|
-
|
|
1126
|
+
speaker as Speaker,
|
|
1100
1127
|
VAD,
|
|
1101
1128
|
VolumeVAD
|
|
1102
1129
|
};
|