@micdrop/web 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,73 @@
1
+ # 🖐️🎤 Micdrop: Real-Time Voice Conversations with AI
2
+
3
+ [Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/client) | [Basic example](../../examples/basic) | [Demo](../../examples/advanced)
4
+
5
+ Micdrop is a set of open source Typescript packages to build real-time voice conversations with AI agents. It handles all the complexities on the client and server side (microphone, speaker, VAD, network communication, etc) and provides ready-to-use implementations for various AI providers.
6
+
7
+ # @micdrop/web
8
+
9
+ The browser side of [Micdrop](https://micdrop.dev). It records and plays with the Web Audio API, and re-exports the whole [`@micdrop/client`](../client) API, so this is the only package a web app installs.
10
+
11
+ It is framework agnostic, you can use it with React, Vue, Angular or any other framework. See the [advanced demo](../../examples/advanced) for a complete example with React.
12
+
13
+ For server implementation, see [@micdrop/server](https://micdrop.dev/docs/server).
14
+
15
+ ## Features
16
+
17
+ - 🎤 Real-time microphone recording and playback
18
+ - 🗣️ Voice activity detection, on volume or with the Silero model
19
+ - 🔊 Device selection and testing
20
+ - 🔌 Full state and events for UI integration
21
+ - 🌐 WebSocket audio streaming
22
+
23
+ ## Installation
24
+
25
+ ```bash
26
+ npm install @micdrop/web
27
+ ```
28
+
29
+ That's it 🎉 No configuration, no asset to copy.
30
+
31
+ ## Usage
32
+
33
+ ```ts
34
+ import { Micdrop } from '@micdrop/web'
35
+
36
+ await Micdrop.start({ url: 'wss://example.com/call' })
37
+
38
+ Micdrop.on('StateChange', (state) => {
39
+ console.log(state.isAssistantSpeaking, state.conversation)
40
+ })
41
+ ```
42
+
43
+ ## Coming from @micdrop/client v2
44
+
45
+ The browser code moved here so the same call can run on a phone. Change the import and the names below, everything else is unchanged. The full guide is on the website: [v2 to v3](https://micdrop.dev/docs/migration/v2-to-v3).
46
+
47
+ ```diff
48
+ -import { Micdrop } from '@micdrop/client'
49
+ +import { Micdrop } from '@micdrop/web'
50
+ ```
51
+
52
+ - Devices are `{ id, label, kind }` rather than `MediaDeviceInfo`, so `device.deviceId` becomes `device.id`
53
+ - `Speaker.playAudio()` takes PCM16 or an `ArrayBuffer` rather than a `Blob`
54
+ - `Mic.analyser` and `Speaker.analyser` are gone, the level comes from `Mic.on('Volume')` and `Speaker.on('Volume')`. For a raw `AnalyserNode`, connect one to `getSpeakerOutput()`.
55
+
56
+ ## Browser specifics
57
+
58
+ - `import '@micdrop/web/silero'` adds the Silero voice detection, and only then does the ONNX runtime enter your bundle
59
+ - `setSileroOptions({ model })` serves the Silero model from your own domain rather than from a CDN
60
+ - `getSpeakerOutput()` gives the node the assistant voice comes out of, to analyse or process it
61
+ - `audioContext` is the shared `AudioContext`
62
+
63
+ ## Documentation
64
+
65
+ Read the full [client documentation](https://micdrop.dev/docs/client) on the [website](https://micdrop.dev).
66
+
67
+ ## License
68
+
69
+ MIT
70
+
71
+ ## Author
72
+
73
+ Originally developed for [Raconte.ai](https://www.raconte.ai), created and open sourced by [Godefroy de Compreignac](https://github.com/Godefroy)
@@ -0,0 +1,85 @@
1
+ import { AudioSink, ScheduledAudio, MicDriver, MicdropDevice, SpeakerDriver } from '@micdrop/client';
2
+ export * from '@micdrop/client';
3
+
4
+ declare let audioContext: AudioContext;
5
+ declare function createAudioContext(): AudioContext;
6
+ declare global {
7
+ interface Window {
8
+ micdropAudioContext: AudioContext;
9
+ }
10
+ }
11
+
12
+ /**
13
+ * Plays samples through the Web Audio graph, on the timeline given by the
14
+ * caller so consecutive chunks are heard without a gap.
15
+ */
16
+ declare class WebAudioSink implements AudioSink {
17
+ readonly output: GainNode;
18
+ constructor();
19
+ get sampleRate(): number;
20
+ get currentTime(): number;
21
+ schedule(samples: Float32Array, when: number, onEnded: () => void): ScheduledAudio;
22
+ }
23
+
24
+ /**
25
+ * Records the microphone with the Web Audio API.
26
+ *
27
+ * An audio worklet delivers the samples at a steady pace, and everything that
28
+ * happens to them afterwards lives in `@micdrop/client`.
29
+ */
30
+ declare class WebMic extends MicDriver {
31
+ private stream;
32
+ private source;
33
+ private worklet;
34
+ private _deviceId;
35
+ get isStarted(): boolean;
36
+ get deviceId(): string | undefined;
37
+ start(deviceId?: string): Promise<void>;
38
+ stop(): Promise<void>;
39
+ getDevices(): Promise<MicdropDevice[]>;
40
+ private onWorkletMessage;
41
+ private onDeviceChange;
42
+ }
43
+ /**
44
+ * Lists the devices of one kind, the default one first
45
+ * @param kind - Which devices to list
46
+ */
47
+ declare function listDevices(kind: 'audioinput' | 'audiooutput'): Promise<MicdropDevice[]>;
48
+
49
+ /**
50
+ * Plays the assistant voice with the Web Audio API.
51
+ */
52
+ declare class WebSpeaker extends SpeakerDriver {
53
+ private sink;
54
+ private stream;
55
+ private _deviceId;
56
+ private element;
57
+ get isPlaying(): boolean;
58
+ get deviceId(): string | undefined;
59
+ /** The node the sound comes out of, to analyse or process it */
60
+ get output(): AudioNode | undefined;
61
+ start(): Promise<void>;
62
+ play(pcm: Int16Array, sampleRate: number): void;
63
+ stopAudio(): void;
64
+ stop(): Promise<void>;
65
+ changeDevice(deviceId: string): Promise<void>;
66
+ getDevices(): Promise<MicdropDevice[]>;
67
+ }
68
+
69
+ /**
70
+ * The node the assistant voice comes out of, to analyse or process it.
71
+ *
72
+ * Connect an AnalyserNode to it to read the envelope on your own frames, which
73
+ * is finer than the ten levels a second the `Volume` event reports.
74
+ * @returns The node, once the call has started
75
+ */
76
+ declare function getSpeakerOutput(): AudioNode | undefined;
77
+
78
+ /**
79
+ * Records and plays with the Web Audio API, and keeps settings in
80
+ * `localStorage`. Called when the package is imported, so a page has nothing
81
+ * to wire.
82
+ */
83
+ declare function registerWebAudio(): void;
84
+
85
+ export { WebAudioSink, WebMic, WebSpeaker, audioContext, createAudioContext, getSpeakerOutput, listDevices, registerWebAudio };
@@ -0,0 +1,85 @@
1
+ import { AudioSink, ScheduledAudio, MicDriver, MicdropDevice, SpeakerDriver } from '@micdrop/client';
2
+ export * from '@micdrop/client';
3
+
4
+ declare let audioContext: AudioContext;
5
+ declare function createAudioContext(): AudioContext;
6
+ declare global {
7
+ interface Window {
8
+ micdropAudioContext: AudioContext;
9
+ }
10
+ }
11
+
12
+ /**
13
+ * Plays samples through the Web Audio graph, on the timeline given by the
14
+ * caller so consecutive chunks are heard without a gap.
15
+ */
16
+ declare class WebAudioSink implements AudioSink {
17
+ readonly output: GainNode;
18
+ constructor();
19
+ get sampleRate(): number;
20
+ get currentTime(): number;
21
+ schedule(samples: Float32Array, when: number, onEnded: () => void): ScheduledAudio;
22
+ }
23
+
24
+ /**
25
+ * Records the microphone with the Web Audio API.
26
+ *
27
+ * An audio worklet delivers the samples at a steady pace, and everything that
28
+ * happens to them afterwards lives in `@micdrop/client`.
29
+ */
30
+ declare class WebMic extends MicDriver {
31
+ private stream;
32
+ private source;
33
+ private worklet;
34
+ private _deviceId;
35
+ get isStarted(): boolean;
36
+ get deviceId(): string | undefined;
37
+ start(deviceId?: string): Promise<void>;
38
+ stop(): Promise<void>;
39
+ getDevices(): Promise<MicdropDevice[]>;
40
+ private onWorkletMessage;
41
+ private onDeviceChange;
42
+ }
43
+ /**
44
+ * Lists the devices of one kind, the default one first
45
+ * @param kind - Which devices to list
46
+ */
47
+ declare function listDevices(kind: 'audioinput' | 'audiooutput'): Promise<MicdropDevice[]>;
48
+
49
+ /**
50
+ * Plays the assistant voice with the Web Audio API.
51
+ */
52
+ declare class WebSpeaker extends SpeakerDriver {
53
+ private sink;
54
+ private stream;
55
+ private _deviceId;
56
+ private element;
57
+ get isPlaying(): boolean;
58
+ get deviceId(): string | undefined;
59
+ /** The node the sound comes out of, to analyse or process it */
60
+ get output(): AudioNode | undefined;
61
+ start(): Promise<void>;
62
+ play(pcm: Int16Array, sampleRate: number): void;
63
+ stopAudio(): void;
64
+ stop(): Promise<void>;
65
+ changeDevice(deviceId: string): Promise<void>;
66
+ getDevices(): Promise<MicdropDevice[]>;
67
+ }
68
+
69
+ /**
70
+ * The node the assistant voice comes out of, to analyse or process it.
71
+ *
72
+ * Connect an AnalyserNode to it to read the envelope on your own frames, which
73
+ * is finer than the ten levels a second the `Volume` event reports.
74
+ * @returns The node, once the call has started
75
+ */
76
+ declare function getSpeakerOutput(): AudioNode | undefined;
77
+
78
+ /**
79
+ * Records and plays with the Web Audio API, and keeps settings in
80
+ * `localStorage`. Called when the package is imported, so a page has nothing
81
+ * to wire.
82
+ */
83
+ declare function registerWebAudio(): void;
84
+
85
+ export { WebAudioSink, WebMic, WebSpeaker, audioContext, createAudioContext, getSpeakerOutput, listDevices, registerWebAudio };
package/dist/index.js ADDED
@@ -0,0 +1,323 @@
1
+ "use strict";
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
6
+ var __export = (target, all) => {
7
+ for (var name in all)
8
+ __defProp(target, name, { get: all[name], enumerable: true });
9
+ };
10
+ var __copyProps = (to, from, except, desc) => {
11
+ if (from && typeof from === "object" || typeof from === "function") {
12
+ for (let key of __getOwnPropNames(from))
13
+ if (!__hasOwnProp.call(to, key) && key !== except)
14
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
15
+ }
16
+ return to;
17
+ };
18
+ var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
19
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
20
+
21
+ // src/index.ts
22
+ var index_exports = {};
23
+ __export(index_exports, {
24
+ WebAudioSink: () => WebAudioSink,
25
+ WebMic: () => WebMic,
26
+ WebSpeaker: () => WebSpeaker,
27
+ audioContext: () => audioContext,
28
+ createAudioContext: () => createAudioContext,
29
+ getSpeakerOutput: () => getSpeakerOutput,
30
+ listDevices: () => listDevices,
31
+ registerWebAudio: () => registerWebAudio
32
+ });
33
+ module.exports = __toCommonJS(index_exports);
34
+ var import_client3 = require("@micdrop/client");
35
+
36
+ // src/audio/WebMic.ts
37
+ var import_client = require("@micdrop/client");
38
+
39
+ // src/audio/unlock.ts
40
+ function unlock(audioContext2) {
41
+ const events = ["touchstart", "touchend", "mousedown", "keydown"];
42
+ const unlock2 = async () => {
43
+ unlockSafari(audioContext2);
44
+ if (audioContext2.state === "suspended") {
45
+ await audioContext2.resume();
46
+ }
47
+ events.forEach((e) => document.body.removeEventListener(e, unlock2));
48
+ };
49
+ events.forEach((e) => document.body.addEventListener(e, unlock2, false));
50
+ }
51
+ function unlockSafari(audioContext2) {
52
+ const source = audioContext2.createBufferSource();
53
+ source.buffer = audioContext2.createBuffer(1, 1, 22050);
54
+ source.connect(audioContext2.destination);
55
+ const onEnded = () => {
56
+ source.disconnect();
57
+ source.buffer = null;
58
+ source.removeEventListener("ended", onEnded);
59
+ };
60
+ source.addEventListener("ended", onEnded);
61
+ source.start();
62
+ }
63
+
64
+ // src/audio/audioContext.ts
65
+ var audioContext;
66
+ function createAudioContext() {
67
+ return new (window.AudioContext || window.webkitAudioContext)();
68
+ }
69
+ if (window.micdropAudioContext) {
70
+ audioContext = window.micdropAudioContext;
71
+ } else {
72
+ window.micdropAudioContext = audioContext = createAudioContext();
73
+ unlock(audioContext);
74
+ }
75
+
76
+ // raw:./pcm-processor-worklet
77
+ var pcm_processor_worklet_default = "// Hands the microphone signal to the main thread, continuously.\n//\n// Deciding whether someone is speaking, keeping a reserve of audio and cutting\n// it into chunks all happen in @micdrop/client, so this only has to deliver the\n// samples at a steady pace.\n\nconst INTERVAL_MS = 20\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.buffers = []\n this.length = 0\n // `sampleRate` is provided by the worklet scope\n this.frameLength = Math.round((sampleRate * INTERVAL_MS) / 1000)\n }\n\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n\n this.buffers.push(new Float32Array(input[0]))\n this.length += input[0].length\n if (this.length < this.frameLength) return true\n\n const frames = new Float32Array(this.length)\n let offset = 0\n for (const buffer of this.buffers) {\n frames.set(buffer, offset)\n offset += buffer.length\n }\n this.buffers.length = 0\n this.length = 0\n\n this.port.postMessage({ type: 'frames', frames, sampleRate }, [\n frames.buffer,\n ])\n return true\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
78
+
79
+ // src/audio/pcm-processor.ts
80
+ async function initPcmProcessor() {
81
+ if (window.micdropPcmProcessor) return;
82
+ window.micdropPcmProcessor = true;
83
+ const workletBlob = new Blob([pcm_processor_worklet_default], {
84
+ type: "application/javascript"
85
+ });
86
+ const workletUrl = URL.createObjectURL(workletBlob);
87
+ await audioContext.audioWorklet.addModule(workletUrl);
88
+ }
89
+
90
+ // src/audio/stopStream.ts
91
+ function stopStream(stream) {
92
+ stream?.getTracks().forEach((track) => track.stop());
93
+ }
94
+
95
+ // src/audio/WebMic.ts
96
+ var WebMic = class extends import_client.MicDriver {
97
+ constructor() {
98
+ super(...arguments);
99
+ this.onWorkletMessage = (event) => {
100
+ const { type, frames, sampleRate } = event.data;
101
+ if (type !== "frames") return;
102
+ this.emit("Frames", frames, sampleRate);
103
+ };
104
+ this.onDeviceChange = () => {
105
+ this.emit("DeviceChange");
106
+ };
107
+ }
108
+ get isStarted() {
109
+ return !!this.stream;
110
+ }
111
+ get deviceId() {
112
+ return this._deviceId;
113
+ }
114
+ async start(deviceId) {
115
+ if (this.stream) {
116
+ if (this._deviceId === deviceId) return;
117
+ await this.stop();
118
+ }
119
+ this.stream = await navigator.mediaDevices.getUserMedia({
120
+ audio: {
121
+ deviceId: { ideal: deviceId },
122
+ sampleRate: 16e3,
123
+ // not working, it will follow device settings, usually 44.1kHz
124
+ sampleSize: 16,
125
+ channelCount: 1,
126
+ echoCancellation: true,
127
+ noiseSuppression: true
128
+ }
129
+ });
130
+ const track = this.stream.getTracks()[0];
131
+ const settled = track?.getCapabilities?.().deviceId;
132
+ this._deviceId = settled && settled !== "default" ? settled : deviceId ?? settled;
133
+ await initPcmProcessor();
134
+ this.worklet = new AudioWorkletNode(audioContext, "pcm-processor");
135
+ this.worklet.port.onmessage = this.onWorkletMessage;
136
+ this.source = audioContext.createMediaStreamSource(this.stream);
137
+ this.source.connect(this.worklet);
138
+ navigator.mediaDevices.addEventListener("devicechange", this.onDeviceChange);
139
+ }
140
+ async stop() {
141
+ navigator.mediaDevices.removeEventListener(
142
+ "devicechange",
143
+ this.onDeviceChange
144
+ );
145
+ if (this.worklet) {
146
+ this.worklet.port.onmessage = null;
147
+ this.worklet.disconnect();
148
+ this.worklet = void 0;
149
+ }
150
+ if (this.source) {
151
+ this.source.disconnect();
152
+ this.source = void 0;
153
+ }
154
+ if (this.stream) {
155
+ stopStream(this.stream);
156
+ this.stream = void 0;
157
+ }
158
+ }
159
+ async getDevices() {
160
+ return listDevices("audioinput");
161
+ }
162
+ };
163
+ async function listDevices(kind) {
164
+ const all = await navigator.mediaDevices.enumerateDevices();
165
+ if (all.length === 0) return [];
166
+ const defaults = all.filter((device) => device.deviceId === "default");
167
+ const devices = all.filter(
168
+ (device) => device.deviceId !== "default" && device.kind === kind
169
+ );
170
+ for (const fallback of defaults) {
171
+ const index = devices.findIndex(
172
+ (device) => device.groupId === fallback.groupId
173
+ );
174
+ if (index > 0) devices.unshift(...devices.splice(index, 1));
175
+ }
176
+ return devices.map((device) => ({
177
+ id: device.deviceId,
178
+ label: device.label || (kind === "audioinput" ? "Microphone" : "Speaker"),
179
+ kind
180
+ }));
181
+ }
182
+
183
+ // src/audio/WebSpeaker.ts
184
+ var import_client2 = require("@micdrop/client");
185
+
186
+ // src/audio/WebAudioSink.ts
187
+ var WebAudioSink = class {
188
+ constructor() {
189
+ this.output = audioContext.createGain();
190
+ }
191
+ get sampleRate() {
192
+ return audioContext.sampleRate;
193
+ }
194
+ get currentTime() {
195
+ return audioContext.currentTime;
196
+ }
197
+ schedule(samples, when, onEnded) {
198
+ const buffer = audioContext.createBuffer(
199
+ 1,
200
+ samples.length,
201
+ audioContext.sampleRate
202
+ );
203
+ buffer.copyToChannel(samples, 0);
204
+ const source = audioContext.createBufferSource();
205
+ source.buffer = buffer;
206
+ source.connect(this.output);
207
+ source.onended = onEnded;
208
+ source.start(when);
209
+ let stopped = false;
210
+ return {
211
+ stop() {
212
+ if (stopped) return;
213
+ stopped = true;
214
+ source.onended = null;
215
+ try {
216
+ source.stop();
217
+ } catch {
218
+ }
219
+ source.disconnect();
220
+ }
221
+ };
222
+ }
223
+ };
224
+
225
+ // src/audio/WebSpeaker.ts
226
+ var WebSpeaker = class extends import_client2.SpeakerDriver {
227
+ get isPlaying() {
228
+ return this.stream?.isPlaying ?? false;
229
+ }
230
+ get deviceId() {
231
+ return this._deviceId;
232
+ }
233
+ /** The node the sound comes out of, to analyse or process it */
234
+ get output() {
235
+ return this.sink?.output;
236
+ }
237
+ async start() {
238
+ if (this.stream) return;
239
+ this.sink = new WebAudioSink();
240
+ this.stream = new import_client2.Pcm16AudioStream(this.sink);
241
+ this.stream.on("StartPlaying", () => this.emit("StartPlaying"));
242
+ this.stream.on("StopPlaying", () => this.emit("StopPlaying"));
243
+ this.stream.on("Volume", (volume) => this.emit("Volume", volume));
244
+ this.sink.output.connect(audioContext.destination);
245
+ }
246
+ play(pcm, sampleRate) {
247
+ this.stream?.playAudio(pcm, sampleRate);
248
+ }
249
+ stopAudio() {
250
+ this.stream?.stopAudio();
251
+ }
252
+ async stop() {
253
+ this.stream?.stopAudio();
254
+ this.stream?.removeAllListeners();
255
+ this.stream = void 0;
256
+ this.sink?.output.disconnect();
257
+ this.sink = void 0;
258
+ if (this.element) {
259
+ this.element.pause();
260
+ this.element.srcObject = null;
261
+ this.element = void 0;
262
+ }
263
+ }
264
+ async changeDevice(deviceId) {
265
+ const sink = this.sink;
266
+ if (!sink) {
267
+ this._deviceId = deviceId;
268
+ return;
269
+ }
270
+ const devices = await this.getDevices();
271
+ const isDefault = devices.length === 0 || devices[0].id === deviceId;
272
+ if (isDefault || "setSinkId" in audioContext) {
273
+ await audioContext.setSinkId?.(isDefault ? "" : deviceId);
274
+ sink.output.disconnect();
275
+ sink.output.connect(audioContext.destination);
276
+ if (this.element) {
277
+ this.element.pause();
278
+ this.element.srcObject = null;
279
+ this.element = void 0;
280
+ }
281
+ } else {
282
+ if (!this.element) {
283
+ const destination = audioContext.createMediaStreamDestination();
284
+ sink.output.disconnect();
285
+ sink.output.connect(destination);
286
+ this.element = new Audio();
287
+ this.element.autoplay = true;
288
+ this.element.srcObject = destination.stream;
289
+ }
290
+ await this.element.setSinkId(deviceId);
291
+ }
292
+ this._deviceId = deviceId;
293
+ }
294
+ async getDevices() {
295
+ return listDevices("audiooutput");
296
+ }
297
+ };
298
+
299
+ // src/index.ts
300
+ __reExport(index_exports, require("@micdrop/client"), module.exports);
301
+ function getSpeakerOutput() {
302
+ const driver = import_client3.Speaker.getDriver();
303
+ return driver instanceof WebSpeaker ? driver.output : void 0;
304
+ }
305
+ function registerWebAudio() {
306
+ import_client3.Mic.setDriver(new WebMic());
307
+ import_client3.Speaker.setDriver(new WebSpeaker());
308
+ (0, import_client3.setMicdropStorage)(localStorage);
309
+ }
310
+ registerWebAudio();
311
+ // Annotate the CommonJS export names for ESM import in node:
312
+ 0 && (module.exports = {
313
+ WebAudioSink,
314
+ WebMic,
315
+ WebSpeaker,
316
+ audioContext,
317
+ createAudioContext,
318
+ getSpeakerOutput,
319
+ listDevices,
320
+ registerWebAudio,
321
+ ...require("@micdrop/client")
322
+ });
323
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/index.ts","../src/audio/WebMic.ts","../src/audio/unlock.ts","../src/audio/audioContext.ts","raw:./pcm-processor-worklet","../src/audio/pcm-processor.ts","../src/audio/stopStream.ts","../src/audio/WebSpeaker.ts","../src/audio/WebAudioSink.ts"],"sourcesContent":["import { Mic, setMicdropStorage, Speaker } from '@micdrop/client'\nimport { WebMic } from './audio/WebMic'\nimport { WebSpeaker } from './audio/WebSpeaker'\n\n/**\n * The node the assistant voice comes out of, to analyse or process it.\n *\n * Connect an AnalyserNode to it to read the envelope on your own frames, which\n * is finer than the ten levels a second the `Volume` event reports.\n * @returns The node, once the call has started\n */\nexport function getSpeakerOutput(): AudioNode | undefined {\n const driver = Speaker.getDriver()\n return driver instanceof WebSpeaker ? driver.output : undefined\n}\n\nexport * from '@micdrop/client'\nexport * from './audio/audioContext'\nexport * from './audio/WebAudioSink'\nexport * from './audio/WebMic'\nexport * from './audio/WebSpeaker'\n\n/**\n * Records and plays with the Web Audio API, and keeps settings in\n * `localStorage`. Called when the package is imported, so a page has nothing\n * to wire.\n */\nexport function registerWebAudio() {\n Mic.setDriver(new WebMic())\n Speaker.setDriver(new WebSpeaker())\n setMicdropStorage(localStorage)\n}\n\nregisterWebAudio()\n","import { MicDriver, MicdropDevice } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { initPcmProcessor } from './pcm-processor'\nimport { stopStream } from './stopStream'\n\n/**\n * Records the microphone with the Web Audio API.\n *\n * An audio worklet delivers the samples at a steady pace, and everything that\n * happens to them afterwards lives in `@micdrop/client`.\n */\nexport class WebMic extends MicDriver {\n private stream: MediaStream | undefined\n private source: MediaStreamAudioSourceNode | undefined\n private worklet: AudioWorkletNode | undefined\n private _deviceId: string | undefined\n\n get isStarted(): boolean {\n return !!this.stream\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n async start(deviceId?: string): Promise<void> {\n if (this.stream) {\n // Same device, keep recording\n if (this._deviceId === deviceId) return\n await this.stop()\n }\n\n this.stream = await navigator.mediaDevices.getUserMedia({\n audio: {\n deviceId: { ideal: deviceId },\n sampleRate: 16000, // not working, it will follow device settings, usually 44.1kHz\n sampleSize: 16,\n channelCount: 1,\n echoCancellation: true,\n noiseSuppression: true,\n },\n })\n\n // Remember which device the browser settled on\n const track = this.stream.getTracks()[0]\n const settled = track?.getCapabilities?.().deviceId\n this._deviceId =\n settled && settled !== 'default' ? settled : (deviceId ?? settled)\n\n await initPcmProcessor()\n this.worklet = new AudioWorkletNode(audioContext, 'pcm-processor')\n this.worklet.port.onmessage = this.onWorkletMessage\n\n this.source = audioContext.createMediaStreamSource(this.stream)\n this.source.connect(this.worklet)\n\n // A browser only reveals the device labels once a stream is running, and\n // Firefox lists nothing at all before that\n navigator.mediaDevices.addEventListener('devicechange', this.onDeviceChange)\n }\n\n async stop(): Promise<void> {\n navigator.mediaDevices.removeEventListener(\n 'devicechange',\n this.onDeviceChange\n )\n if (this.worklet) {\n this.worklet.port.onmessage = null\n this.worklet.disconnect()\n this.worklet = undefined\n }\n if (this.source) {\n this.source.disconnect()\n this.source = undefined\n }\n if (this.stream) {\n stopStream(this.stream)\n this.stream = undefined\n }\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audioinput')\n }\n\n private onWorkletMessage = (event: MessageEvent) => {\n const { type, frames, sampleRate } = event.data\n if (type !== 'frames') return\n this.emit('Frames', frames as Float32Array, sampleRate as number)\n }\n\n private onDeviceChange = () => {\n this.emit('DeviceChange')\n }\n}\n\n/**\n * Lists the devices of one kind, the default one first\n * @param kind - Which devices to list\n */\nexport async function listDevices(\n kind: 'audioinput' | 'audiooutput'\n): Promise<MicdropDevice[]> {\n const all = await navigator.mediaDevices.enumerateDevices()\n if (all.length === 0) return []\n\n // A browser lists the default device twice, once under the id \"default\" and\n // once under its real id. Keep the real one, and put it first.\n const defaults = all.filter((device) => device.deviceId === 'default')\n const devices = all.filter(\n (device) => device.deviceId !== 'default' && device.kind === kind\n )\n for (const fallback of defaults) {\n const index = devices.findIndex(\n (device) => device.groupId === fallback.groupId\n )\n if (index > 0) devices.unshift(...devices.splice(index, 1))\n }\n\n return devices.map((device) => ({\n id: device.deviceId,\n label: device.label || (kind === 'audioinput' ? 'Microphone' : 'Speaker'),\n kind,\n }))\n}\n","// Inspired https://www.mattmontag.com/web/unlock-web-audio-in-safari-for-ios-and-macos\nexport function unlock(audioContext: AudioContext): void {\n const events = ['touchstart', 'touchend', 'mousedown', 'keydown']\n\n const unlock = async () => {\n unlockSafari(audioContext)\n if (audioContext.state === 'suspended') {\n await audioContext.resume()\n }\n // Clean events\n events.forEach((e) => document.body.removeEventListener(e, unlock))\n }\n\n // Add events\n events.forEach((e) => document.body.addEventListener(e, unlock, false))\n}\n\n// safari/ios hack - inspired by https://codepen.io/kslstn/pen/pagLqL\n// unlock audioContext - call when interactive (button click)\nfunction unlockSafari(audioContext: AudioContext): void {\n // create empty buffer and play it to unlock audioContext when interactive\n const source = audioContext.createBufferSource()\n source.buffer = audioContext.createBuffer(1, 1, 22050)\n source.connect(audioContext.destination)\n const onEnded = () => {\n source.disconnect()\n source.buffer = null\n source.removeEventListener('ended', onEnded)\n }\n source.addEventListener('ended', onEnded)\n source.start()\n}\n","import { unlock } from './unlock'\n\nexport let audioContext: AudioContext\n\nexport function createAudioContext() {\n return new (window.AudioContext || (window as any).webkitAudioContext)()\n}\n\ndeclare global {\n interface Window {\n micdropAudioContext: AudioContext\n }\n}\n\nif (window.micdropAudioContext) {\n audioContext = window.micdropAudioContext\n} else {\n window.micdropAudioContext = audioContext = createAudioContext()\n\n // Hack to unlock audio context on some browsers\n unlock(audioContext)\n}\n","// Hands the microphone signal to the main thread, continuously.\n//\n// Deciding whether someone is speaking, keeping a reserve of audio and cutting\n// it into chunks all happen in @micdrop/client, so this only has to deliver the\n// samples at a steady pace.\n\nconst INTERVAL_MS = 20\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.buffers = []\n this.length = 0\n // `sampleRate` is provided by the worklet scope\n this.frameLength = Math.round((sampleRate * INTERVAL_MS) / 1000)\n }\n\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n\n this.buffers.push(new Float32Array(input[0]))\n this.length += input[0].length\n if (this.length < this.frameLength) return true\n\n const frames = new Float32Array(this.length)\n let offset = 0\n for (const buffer of this.buffers) {\n frames.set(buffer, offset)\n offset += buffer.length\n }\n this.buffers.length = 0\n this.length = 0\n\n this.port.postMessage({ type: 'frames', frames, sampleRate }, [\n frames.buffer,\n ])\n return true\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n","import pcmProcessorWorklet from './pcm-processor-worklet?raw'\nimport { audioContext } from './audioContext'\n\ndeclare global {\n interface Window {\n micdropPcmProcessor: boolean\n }\n}\n\nexport async function initPcmProcessor() {\n if (window.micdropPcmProcessor) return\n window.micdropPcmProcessor = true\n\n // Init worklet URL\n const workletBlob = new Blob([pcmProcessorWorklet], {\n type: 'application/javascript',\n })\n const workletUrl = URL.createObjectURL(workletBlob)\n\n // Load the worklet module\n await audioContext.audioWorklet.addModule(workletUrl)\n}\n","/**\n * Stops a stream\n * @param stream - The stream to stop\n */\nexport function stopStream(stream?: MediaStream) {\n stream?.getTracks().forEach((track) => track.stop())\n}\n","import { MicdropDevice, Pcm16AudioStream, SpeakerDriver } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { listDevices } from './WebMic'\nimport { WebAudioSink } from './WebAudioSink'\n\n/**\n * Plays the assistant voice with the Web Audio API.\n */\nexport class WebSpeaker extends SpeakerDriver {\n private sink: WebAudioSink | undefined\n private stream: Pcm16AudioStream | undefined\n private _deviceId: string | undefined\n // Firefox cannot pick an output on an audio context, so a non default device\n // is played back through an audio element instead\n private element: HTMLAudioElement | undefined\n\n get isPlaying(): boolean {\n return this.stream?.isPlaying ?? false\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n /** The node the sound comes out of, to analyse or process it */\n get output(): AudioNode | undefined {\n return this.sink?.output\n }\n\n async start(): Promise<void> {\n if (this.stream) return\n\n this.sink = new WebAudioSink()\n this.stream = new Pcm16AudioStream(this.sink)\n this.stream.on('StartPlaying', () => this.emit('StartPlaying'))\n this.stream.on('StopPlaying', () => this.emit('StopPlaying'))\n this.stream.on('Volume', (volume) => this.emit('Volume', volume))\n\n this.sink.output.connect(audioContext.destination)\n }\n\n play(pcm: Int16Array, sampleRate: number): void {\n this.stream?.playAudio(pcm, sampleRate)\n }\n\n stopAudio(): void {\n this.stream?.stopAudio()\n }\n\n async stop(): Promise<void> {\n this.stream?.stopAudio()\n this.stream?.removeAllListeners()\n this.stream = undefined\n this.sink?.output.disconnect()\n this.sink = undefined\n if (this.element) {\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n }\n }\n\n async changeDevice(deviceId: string): Promise<void> {\n const sink = this.sink\n if (!sink) {\n this._deviceId = deviceId\n return\n }\n\n const devices = await this.getDevices()\n const isDefault = devices.length === 0 || devices[0].id === deviceId\n\n if (isDefault || 'setSinkId' in audioContext) {\n // Playing through the audio context is the preferred way, it is the only\n // one that keeps echo cancellation working\n await (audioContext as any).setSinkId?.(isDefault ? '' : deviceId)\n sink.output.disconnect()\n sink.output.connect(audioContext.destination)\n\n if (this.element) {\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n }\n } else {\n // Firefox: stream the sound back into an audio element, which does accept\n // an output device\n if (!this.element) {\n const destination = audioContext.createMediaStreamDestination()\n sink.output.disconnect()\n sink.output.connect(destination)\n\n this.element = new Audio()\n this.element.autoplay = true\n this.element.srcObject = destination.stream\n }\n await this.element.setSinkId(deviceId)\n }\n\n this._deviceId = deviceId\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audiooutput')\n }\n}\n","import { AudioSink, ScheduledAudio } from '@micdrop/client'\nimport { audioContext } from './audioContext'\n\n/**\n * Plays samples through the Web Audio graph, on the timeline given by the\n * caller so consecutive chunks are heard without a gap.\n */\nexport class WebAudioSink implements AudioSink {\n public readonly output: GainNode\n\n constructor() {\n this.output = audioContext.createGain()\n }\n\n get sampleRate(): number {\n return audioContext.sampleRate\n }\n\n get currentTime(): number {\n return audioContext.currentTime\n }\n\n schedule(\n samples: Float32Array,\n when: number,\n onEnded: () => void\n ): ScheduledAudio {\n const buffer = audioContext.createBuffer(\n 1,\n samples.length,\n audioContext.sampleRate\n )\n buffer.copyToChannel(samples as Float32Array<ArrayBuffer>, 0)\n\n const source = audioContext.createBufferSource()\n source.buffer = buffer\n source.connect(this.output)\n source.onended = onEnded\n source.start(when)\n\n let stopped = false\n return {\n stop() {\n if (stopped) return\n stopped = true\n source.onended = null\n try {\n source.stop()\n } catch {\n // Already finished\n }\n source.disconnect()\n },\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAAA,iBAAgD;;;ACAhD,oBAAyC;;;ACClC,SAAS,OAAOC,eAAkC;AACvD,QAAM,SAAS,CAAC,cAAc,YAAY,aAAa,SAAS;AAEhE,QAAMC,UAAS,YAAY;AACzB,iBAAaD,aAAY;AACzB,QAAIA,cAAa,UAAU,aAAa;AACtC,YAAMA,cAAa,OAAO;AAAA,IAC5B;AAEA,WAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,oBAAoB,GAAGC,OAAM,CAAC;AAAA,EACpE;AAGA,SAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,iBAAiB,GAAGA,SAAQ,KAAK,CAAC;AACxE;AAIA,SAAS,aAAaD,eAAkC;AAEtD,QAAM,SAASA,cAAa,mBAAmB;AAC/C,SAAO,SAASA,cAAa,aAAa,GAAG,GAAG,KAAK;AACrD,SAAO,QAAQA,cAAa,WAAW;AACvC,QAAM,UAAU,MAAM;AACpB,WAAO,WAAW;AAClB,WAAO,SAAS;AAChB,WAAO,oBAAoB,SAAS,OAAO;AAAA,EAC7C;AACA,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO,MAAM;AACf;;;AC7BO,IAAI;AAEJ,SAAS,qBAAqB;AACnC,SAAO,KAAK,OAAO,gBAAiB,OAAe,oBAAoB;AACzE;AAQA,IAAI,OAAO,qBAAqB;AAC9B,iBAAe,OAAO;AACxB,OAAO;AACL,SAAO,sBAAsB,eAAe,mBAAmB;AAG/D,SAAO,YAAY;AACrB;;;ACrBA;;;ACSA,eAAsB,mBAAmB;AACvC,MAAI,OAAO,oBAAqB;AAChC,SAAO,sBAAsB;AAG7B,QAAM,cAAc,IAAI,KAAK,CAAC,6BAAmB,GAAG;AAAA,IAClD,MAAM;AAAA,EACR,CAAC;AACD,QAAM,aAAa,IAAI,gBAAgB,WAAW;AAGlD,QAAM,aAAa,aAAa,UAAU,UAAU;AACtD;;;ACjBO,SAAS,WAAW,QAAsB;AAC/C,UAAQ,UAAU,EAAE,QAAQ,CAAC,UAAU,MAAM,KAAK,CAAC;AACrD;;;ALKO,IAAM,SAAN,cAAqB,wBAAU;AAAA,EAA/B;AAAA;AA0EL,SAAQ,mBAAmB,CAAC,UAAwB;AAClD,YAAM,EAAE,MAAM,QAAQ,WAAW,IAAI,MAAM;AAC3C,UAAI,SAAS,SAAU;AACvB,WAAK,KAAK,UAAU,QAAwB,UAAoB;AAAA,IAClE;AAEA,SAAQ,iBAAiB,MAAM;AAC7B,WAAK,KAAK,cAAc;AAAA,IAC1B;AAAA;AAAA,EA5EA,IAAI,YAAqB;AACvB,WAAO,CAAC,CAAC,KAAK;AAAA,EAChB;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,UAAkC;AAC5C,QAAI,KAAK,QAAQ;AAEf,UAAI,KAAK,cAAc,SAAU;AACjC,YAAM,KAAK,KAAK;AAAA,IAClB;AAEA,SAAK,SAAS,MAAM,UAAU,aAAa,aAAa;AAAA,MACtD,OAAO;AAAA,QACL,UAAU,EAAE,OAAO,SAAS;AAAA,QAC5B,YAAY;AAAA;AAAA,QACZ,YAAY;AAAA,QACZ,cAAc;AAAA,QACd,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,MACpB;AAAA,IACF,CAAC;AAGD,UAAM,QAAQ,KAAK,OAAO,UAAU,EAAE,CAAC;AACvC,UAAM,UAAU,OAAO,kBAAkB,EAAE;AAC3C,SAAK,YACH,WAAW,YAAY,YAAY,UAAW,YAAY;AAE5D,UAAM,iBAAiB;AACvB,SAAK,UAAU,IAAI,iBAAiB,cAAc,eAAe;AACjE,SAAK,QAAQ,KAAK,YAAY,KAAK;AAEnC,SAAK,SAAS,aAAa,wBAAwB,KAAK,MAAM;AAC9D,SAAK,OAAO,QAAQ,KAAK,OAAO;AAIhC,cAAU,aAAa,iBAAiB,gBAAgB,KAAK,cAAc;AAAA,EAC7E;AAAA,EAEA,MAAM,OAAsB;AAC1B,cAAU,aAAa;AAAA,MACrB;AAAA,MACA,KAAK;AAAA,IACP;AACA,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,KAAK,YAAY;AAC9B,WAAK,QAAQ,WAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,WAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AACA,QAAI,KAAK,QAAQ;AACf,iBAAW,KAAK,MAAM;AACtB,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,YAAY;AAAA,EACjC;AAWF;AAMA,eAAsB,YACpB,MAC0B;AAC1B,QAAM,MAAM,MAAM,UAAU,aAAa,iBAAiB;AAC1D,MAAI,IAAI,WAAW,EAAG,QAAO,CAAC;AAI9B,QAAM,WAAW,IAAI,OAAO,CAAC,WAAW,OAAO,aAAa,SAAS;AACrE,QAAM,UAAU,IAAI;AAAA,IAClB,CAAC,WAAW,OAAO,aAAa,aAAa,OAAO,SAAS;AAAA,EAC/D;AACA,aAAW,YAAY,UAAU;AAC/B,UAAM,QAAQ,QAAQ;AAAA,MACpB,CAAC,WAAW,OAAO,YAAY,SAAS;AAAA,IAC1C;AACA,QAAI,QAAQ,EAAG,SAAQ,QAAQ,GAAG,QAAQ,OAAO,OAAO,CAAC,CAAC;AAAA,EAC5D;AAEA,SAAO,QAAQ,IAAI,CAAC,YAAY;AAAA,IAC9B,IAAI,OAAO;AAAA,IACX,OAAO,OAAO,UAAU,SAAS,eAAe,eAAe;AAAA,IAC/D;AAAA,EACF,EAAE;AACJ;;;AM5HA,IAAAE,iBAA+D;;;ACOxD,IAAM,eAAN,MAAwC;AAAA,EAG7C,cAAc;AACZ,SAAK,SAAS,aAAa,WAAW;AAAA,EACxC;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,SACE,SACA,MACA,SACgB;AAChB,UAAM,SAAS,aAAa;AAAA,MAC1B;AAAA,MACA,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AACA,WAAO,cAAc,SAAsC,CAAC;AAE5D,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,KAAK,MAAM;AAC1B,WAAO,UAAU;AACjB,WAAO,MAAM,IAAI;AAEjB,QAAI,UAAU;AACd,WAAO;AAAA,MACL,OAAO;AACL,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,UAAU;AACjB,YAAI;AACF,iBAAO,KAAK;AAAA,QACd,QAAQ;AAAA,QAER;AACA,eAAO,WAAW;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;;;AD/CO,IAAM,aAAN,cAAyB,6BAAc;AAAA,EAQ5C,IAAI,YAAqB;AACvB,WAAO,KAAK,QAAQ,aAAa;AAAA,EACnC;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAgC;AAClC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,MAAM,QAAuB;AAC3B,QAAI,KAAK,OAAQ;AAEjB,SAAK,OAAO,IAAI,aAAa;AAC7B,SAAK,SAAS,IAAI,gCAAiB,KAAK,IAAI;AAC5C,SAAK,OAAO,GAAG,gBAAgB,MAAM,KAAK,KAAK,cAAc,CAAC;AAC9D,SAAK,OAAO,GAAG,eAAe,MAAM,KAAK,KAAK,aAAa,CAAC;AAC5D,SAAK,OAAO,GAAG,UAAU,CAAC,WAAW,KAAK,KAAK,UAAU,MAAM,CAAC;AAEhE,SAAK,KAAK,OAAO,QAAQ,aAAa,WAAW;AAAA,EACnD;AAAA,EAEA,KAAK,KAAiB,YAA0B;AAC9C,SAAK,QAAQ,UAAU,KAAK,UAAU;AAAA,EACxC;AAAA,EAEA,YAAkB;AAChB,SAAK,QAAQ,UAAU;AAAA,EACzB;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,QAAQ,UAAU;AACvB,SAAK,QAAQ,mBAAmB;AAChC,SAAK,SAAS;AACd,SAAK,MAAM,OAAO,WAAW;AAC7B,SAAK,OAAO;AACZ,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,MAAM;AACnB,WAAK,QAAQ,YAAY;AACzB,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,UAAiC;AAClD,UAAM,OAAO,KAAK;AAClB,QAAI,CAAC,MAAM;AACT,WAAK,YAAY;AACjB;AAAA,IACF;AAEA,UAAM,UAAU,MAAM,KAAK,WAAW;AACtC,UAAM,YAAY,QAAQ,WAAW,KAAK,QAAQ,CAAC,EAAE,OAAO;AAE5D,QAAI,aAAa,eAAe,cAAc;AAG5C,YAAO,aAAqB,YAAY,YAAY,KAAK,QAAQ;AACjE,WAAK,OAAO,WAAW;AACvB,WAAK,OAAO,QAAQ,aAAa,WAAW;AAE5C,UAAI,KAAK,SAAS;AAChB,aAAK,QAAQ,MAAM;AACnB,aAAK,QAAQ,YAAY;AACzB,aAAK,UAAU;AAAA,MACjB;AAAA,IACF,OAAO;AAGL,UAAI,CAAC,KAAK,SAAS;AACjB,cAAM,cAAc,aAAa,6BAA6B;AAC9D,aAAK,OAAO,WAAW;AACvB,aAAK,OAAO,QAAQ,WAAW;AAE/B,aAAK,UAAU,IAAI,MAAM;AACzB,aAAK,QAAQ,WAAW;AACxB,aAAK,QAAQ,YAAY,YAAY;AAAA,MACvC;AACA,YAAM,KAAK,QAAQ,UAAU,QAAQ;AAAA,IACvC;AAEA,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,aAAa;AAAA,EAClC;AACF;;;APzFA,0BAAc,4BAhBd;AAWO,SAAS,mBAA0C;AACxD,QAAM,SAAS,uBAAQ,UAAU;AACjC,SAAO,kBAAkB,aAAa,OAAO,SAAS;AACxD;AAaO,SAAS,mBAAmB;AACjC,qBAAI,UAAU,IAAI,OAAO,CAAC;AAC1B,yBAAQ,UAAU,IAAI,WAAW,CAAC;AAClC,wCAAkB,YAAY;AAChC;AAEA,iBAAiB;","names":["import_client","audioContext","unlock","import_client"]}
package/dist/index.mjs ADDED
@@ -0,0 +1,289 @@
1
+ // src/index.ts
2
+ import { Mic, setMicdropStorage, Speaker } from "@micdrop/client";
3
+
4
+ // src/audio/WebMic.ts
5
+ import { MicDriver } from "@micdrop/client";
6
+
7
+ // src/audio/unlock.ts
8
+ function unlock(audioContext2) {
9
+ const events = ["touchstart", "touchend", "mousedown", "keydown"];
10
+ const unlock2 = async () => {
11
+ unlockSafari(audioContext2);
12
+ if (audioContext2.state === "suspended") {
13
+ await audioContext2.resume();
14
+ }
15
+ events.forEach((e) => document.body.removeEventListener(e, unlock2));
16
+ };
17
+ events.forEach((e) => document.body.addEventListener(e, unlock2, false));
18
+ }
19
+ function unlockSafari(audioContext2) {
20
+ const source = audioContext2.createBufferSource();
21
+ source.buffer = audioContext2.createBuffer(1, 1, 22050);
22
+ source.connect(audioContext2.destination);
23
+ const onEnded = () => {
24
+ source.disconnect();
25
+ source.buffer = null;
26
+ source.removeEventListener("ended", onEnded);
27
+ };
28
+ source.addEventListener("ended", onEnded);
29
+ source.start();
30
+ }
31
+
32
+ // src/audio/audioContext.ts
33
+ var audioContext;
34
+ function createAudioContext() {
35
+ return new (window.AudioContext || window.webkitAudioContext)();
36
+ }
37
+ if (window.micdropAudioContext) {
38
+ audioContext = window.micdropAudioContext;
39
+ } else {
40
+ window.micdropAudioContext = audioContext = createAudioContext();
41
+ unlock(audioContext);
42
+ }
43
+
44
+ // raw:./pcm-processor-worklet
45
+ var pcm_processor_worklet_default = "// Hands the microphone signal to the main thread, continuously.\n//\n// Deciding whether someone is speaking, keeping a reserve of audio and cutting\n// it into chunks all happen in @micdrop/client, so this only has to deliver the\n// samples at a steady pace.\n\nconst INTERVAL_MS = 20\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.buffers = []\n this.length = 0\n // `sampleRate` is provided by the worklet scope\n this.frameLength = Math.round((sampleRate * INTERVAL_MS) / 1000)\n }\n\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n\n this.buffers.push(new Float32Array(input[0]))\n this.length += input[0].length\n if (this.length < this.frameLength) return true\n\n const frames = new Float32Array(this.length)\n let offset = 0\n for (const buffer of this.buffers) {\n frames.set(buffer, offset)\n offset += buffer.length\n }\n this.buffers.length = 0\n this.length = 0\n\n this.port.postMessage({ type: 'frames', frames, sampleRate }, [\n frames.buffer,\n ])\n return true\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n";
46
+
47
+ // src/audio/pcm-processor.ts
48
+ async function initPcmProcessor() {
49
+ if (window.micdropPcmProcessor) return;
50
+ window.micdropPcmProcessor = true;
51
+ const workletBlob = new Blob([pcm_processor_worklet_default], {
52
+ type: "application/javascript"
53
+ });
54
+ const workletUrl = URL.createObjectURL(workletBlob);
55
+ await audioContext.audioWorklet.addModule(workletUrl);
56
+ }
57
+
58
+ // src/audio/stopStream.ts
59
+ function stopStream(stream) {
60
+ stream?.getTracks().forEach((track) => track.stop());
61
+ }
62
+
63
+ // src/audio/WebMic.ts
64
+ var WebMic = class extends MicDriver {
65
+ constructor() {
66
+ super(...arguments);
67
+ this.onWorkletMessage = (event) => {
68
+ const { type, frames, sampleRate } = event.data;
69
+ if (type !== "frames") return;
70
+ this.emit("Frames", frames, sampleRate);
71
+ };
72
+ this.onDeviceChange = () => {
73
+ this.emit("DeviceChange");
74
+ };
75
+ }
76
+ get isStarted() {
77
+ return !!this.stream;
78
+ }
79
+ get deviceId() {
80
+ return this._deviceId;
81
+ }
82
+ async start(deviceId) {
83
+ if (this.stream) {
84
+ if (this._deviceId === deviceId) return;
85
+ await this.stop();
86
+ }
87
+ this.stream = await navigator.mediaDevices.getUserMedia({
88
+ audio: {
89
+ deviceId: { ideal: deviceId },
90
+ sampleRate: 16e3,
91
+ // not working, it will follow device settings, usually 44.1kHz
92
+ sampleSize: 16,
93
+ channelCount: 1,
94
+ echoCancellation: true,
95
+ noiseSuppression: true
96
+ }
97
+ });
98
+ const track = this.stream.getTracks()[0];
99
+ const settled = track?.getCapabilities?.().deviceId;
100
+ this._deviceId = settled && settled !== "default" ? settled : deviceId ?? settled;
101
+ await initPcmProcessor();
102
+ this.worklet = new AudioWorkletNode(audioContext, "pcm-processor");
103
+ this.worklet.port.onmessage = this.onWorkletMessage;
104
+ this.source = audioContext.createMediaStreamSource(this.stream);
105
+ this.source.connect(this.worklet);
106
+ navigator.mediaDevices.addEventListener("devicechange", this.onDeviceChange);
107
+ }
108
+ async stop() {
109
+ navigator.mediaDevices.removeEventListener(
110
+ "devicechange",
111
+ this.onDeviceChange
112
+ );
113
+ if (this.worklet) {
114
+ this.worklet.port.onmessage = null;
115
+ this.worklet.disconnect();
116
+ this.worklet = void 0;
117
+ }
118
+ if (this.source) {
119
+ this.source.disconnect();
120
+ this.source = void 0;
121
+ }
122
+ if (this.stream) {
123
+ stopStream(this.stream);
124
+ this.stream = void 0;
125
+ }
126
+ }
127
+ async getDevices() {
128
+ return listDevices("audioinput");
129
+ }
130
+ };
131
+ async function listDevices(kind) {
132
+ const all = await navigator.mediaDevices.enumerateDevices();
133
+ if (all.length === 0) return [];
134
+ const defaults = all.filter((device) => device.deviceId === "default");
135
+ const devices = all.filter(
136
+ (device) => device.deviceId !== "default" && device.kind === kind
137
+ );
138
+ for (const fallback of defaults) {
139
+ const index = devices.findIndex(
140
+ (device) => device.groupId === fallback.groupId
141
+ );
142
+ if (index > 0) devices.unshift(...devices.splice(index, 1));
143
+ }
144
+ return devices.map((device) => ({
145
+ id: device.deviceId,
146
+ label: device.label || (kind === "audioinput" ? "Microphone" : "Speaker"),
147
+ kind
148
+ }));
149
+ }
150
+
151
+ // src/audio/WebSpeaker.ts
152
+ import { Pcm16AudioStream, SpeakerDriver } from "@micdrop/client";
153
+
154
+ // src/audio/WebAudioSink.ts
155
+ var WebAudioSink = class {
156
+ constructor() {
157
+ this.output = audioContext.createGain();
158
+ }
159
+ get sampleRate() {
160
+ return audioContext.sampleRate;
161
+ }
162
+ get currentTime() {
163
+ return audioContext.currentTime;
164
+ }
165
+ schedule(samples, when, onEnded) {
166
+ const buffer = audioContext.createBuffer(
167
+ 1,
168
+ samples.length,
169
+ audioContext.sampleRate
170
+ );
171
+ buffer.copyToChannel(samples, 0);
172
+ const source = audioContext.createBufferSource();
173
+ source.buffer = buffer;
174
+ source.connect(this.output);
175
+ source.onended = onEnded;
176
+ source.start(when);
177
+ let stopped = false;
178
+ return {
179
+ stop() {
180
+ if (stopped) return;
181
+ stopped = true;
182
+ source.onended = null;
183
+ try {
184
+ source.stop();
185
+ } catch {
186
+ }
187
+ source.disconnect();
188
+ }
189
+ };
190
+ }
191
+ };
192
+
193
+ // src/audio/WebSpeaker.ts
194
+ var WebSpeaker = class extends SpeakerDriver {
195
+ get isPlaying() {
196
+ return this.stream?.isPlaying ?? false;
197
+ }
198
+ get deviceId() {
199
+ return this._deviceId;
200
+ }
201
+ /** The node the sound comes out of, to analyse or process it */
202
+ get output() {
203
+ return this.sink?.output;
204
+ }
205
+ async start() {
206
+ if (this.stream) return;
207
+ this.sink = new WebAudioSink();
208
+ this.stream = new Pcm16AudioStream(this.sink);
209
+ this.stream.on("StartPlaying", () => this.emit("StartPlaying"));
210
+ this.stream.on("StopPlaying", () => this.emit("StopPlaying"));
211
+ this.stream.on("Volume", (volume) => this.emit("Volume", volume));
212
+ this.sink.output.connect(audioContext.destination);
213
+ }
214
+ play(pcm, sampleRate) {
215
+ this.stream?.playAudio(pcm, sampleRate);
216
+ }
217
+ stopAudio() {
218
+ this.stream?.stopAudio();
219
+ }
220
+ async stop() {
221
+ this.stream?.stopAudio();
222
+ this.stream?.removeAllListeners();
223
+ this.stream = void 0;
224
+ this.sink?.output.disconnect();
225
+ this.sink = void 0;
226
+ if (this.element) {
227
+ this.element.pause();
228
+ this.element.srcObject = null;
229
+ this.element = void 0;
230
+ }
231
+ }
232
+ async changeDevice(deviceId) {
233
+ const sink = this.sink;
234
+ if (!sink) {
235
+ this._deviceId = deviceId;
236
+ return;
237
+ }
238
+ const devices = await this.getDevices();
239
+ const isDefault = devices.length === 0 || devices[0].id === deviceId;
240
+ if (isDefault || "setSinkId" in audioContext) {
241
+ await audioContext.setSinkId?.(isDefault ? "" : deviceId);
242
+ sink.output.disconnect();
243
+ sink.output.connect(audioContext.destination);
244
+ if (this.element) {
245
+ this.element.pause();
246
+ this.element.srcObject = null;
247
+ this.element = void 0;
248
+ }
249
+ } else {
250
+ if (!this.element) {
251
+ const destination = audioContext.createMediaStreamDestination();
252
+ sink.output.disconnect();
253
+ sink.output.connect(destination);
254
+ this.element = new Audio();
255
+ this.element.autoplay = true;
256
+ this.element.srcObject = destination.stream;
257
+ }
258
+ await this.element.setSinkId(deviceId);
259
+ }
260
+ this._deviceId = deviceId;
261
+ }
262
+ async getDevices() {
263
+ return listDevices("audiooutput");
264
+ }
265
+ };
266
+
267
+ // src/index.ts
268
+ export * from "@micdrop/client";
269
+ function getSpeakerOutput() {
270
+ const driver = Speaker.getDriver();
271
+ return driver instanceof WebSpeaker ? driver.output : void 0;
272
+ }
273
+ function registerWebAudio() {
274
+ Mic.setDriver(new WebMic());
275
+ Speaker.setDriver(new WebSpeaker());
276
+ setMicdropStorage(localStorage);
277
+ }
278
+ registerWebAudio();
279
+ export {
280
+ WebAudioSink,
281
+ WebMic,
282
+ WebSpeaker,
283
+ audioContext,
284
+ createAudioContext,
285
+ getSpeakerOutput,
286
+ listDevices,
287
+ registerWebAudio
288
+ };
289
+ //# sourceMappingURL=index.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/index.ts","../src/audio/WebMic.ts","../src/audio/unlock.ts","../src/audio/audioContext.ts","raw:./pcm-processor-worklet","../src/audio/pcm-processor.ts","../src/audio/stopStream.ts","../src/audio/WebSpeaker.ts","../src/audio/WebAudioSink.ts"],"sourcesContent":["import { Mic, setMicdropStorage, Speaker } from '@micdrop/client'\nimport { WebMic } from './audio/WebMic'\nimport { WebSpeaker } from './audio/WebSpeaker'\n\n/**\n * The node the assistant voice comes out of, to analyse or process it.\n *\n * Connect an AnalyserNode to it to read the envelope on your own frames, which\n * is finer than the ten levels a second the `Volume` event reports.\n * @returns The node, once the call has started\n */\nexport function getSpeakerOutput(): AudioNode | undefined {\n const driver = Speaker.getDriver()\n return driver instanceof WebSpeaker ? driver.output : undefined\n}\n\nexport * from '@micdrop/client'\nexport * from './audio/audioContext'\nexport * from './audio/WebAudioSink'\nexport * from './audio/WebMic'\nexport * from './audio/WebSpeaker'\n\n/**\n * Records and plays with the Web Audio API, and keeps settings in\n * `localStorage`. Called when the package is imported, so a page has nothing\n * to wire.\n */\nexport function registerWebAudio() {\n Mic.setDriver(new WebMic())\n Speaker.setDriver(new WebSpeaker())\n setMicdropStorage(localStorage)\n}\n\nregisterWebAudio()\n","import { MicDriver, MicdropDevice } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { initPcmProcessor } from './pcm-processor'\nimport { stopStream } from './stopStream'\n\n/**\n * Records the microphone with the Web Audio API.\n *\n * An audio worklet delivers the samples at a steady pace, and everything that\n * happens to them afterwards lives in `@micdrop/client`.\n */\nexport class WebMic extends MicDriver {\n private stream: MediaStream | undefined\n private source: MediaStreamAudioSourceNode | undefined\n private worklet: AudioWorkletNode | undefined\n private _deviceId: string | undefined\n\n get isStarted(): boolean {\n return !!this.stream\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n async start(deviceId?: string): Promise<void> {\n if (this.stream) {\n // Same device, keep recording\n if (this._deviceId === deviceId) return\n await this.stop()\n }\n\n this.stream = await navigator.mediaDevices.getUserMedia({\n audio: {\n deviceId: { ideal: deviceId },\n sampleRate: 16000, // not working, it will follow device settings, usually 44.1kHz\n sampleSize: 16,\n channelCount: 1,\n echoCancellation: true,\n noiseSuppression: true,\n },\n })\n\n // Remember which device the browser settled on\n const track = this.stream.getTracks()[0]\n const settled = track?.getCapabilities?.().deviceId\n this._deviceId =\n settled && settled !== 'default' ? settled : (deviceId ?? settled)\n\n await initPcmProcessor()\n this.worklet = new AudioWorkletNode(audioContext, 'pcm-processor')\n this.worklet.port.onmessage = this.onWorkletMessage\n\n this.source = audioContext.createMediaStreamSource(this.stream)\n this.source.connect(this.worklet)\n\n // A browser only reveals the device labels once a stream is running, and\n // Firefox lists nothing at all before that\n navigator.mediaDevices.addEventListener('devicechange', this.onDeviceChange)\n }\n\n async stop(): Promise<void> {\n navigator.mediaDevices.removeEventListener(\n 'devicechange',\n this.onDeviceChange\n )\n if (this.worklet) {\n this.worklet.port.onmessage = null\n this.worklet.disconnect()\n this.worklet = undefined\n }\n if (this.source) {\n this.source.disconnect()\n this.source = undefined\n }\n if (this.stream) {\n stopStream(this.stream)\n this.stream = undefined\n }\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audioinput')\n }\n\n private onWorkletMessage = (event: MessageEvent) => {\n const { type, frames, sampleRate } = event.data\n if (type !== 'frames') return\n this.emit('Frames', frames as Float32Array, sampleRate as number)\n }\n\n private onDeviceChange = () => {\n this.emit('DeviceChange')\n }\n}\n\n/**\n * Lists the devices of one kind, the default one first\n * @param kind - Which devices to list\n */\nexport async function listDevices(\n kind: 'audioinput' | 'audiooutput'\n): Promise<MicdropDevice[]> {\n const all = await navigator.mediaDevices.enumerateDevices()\n if (all.length === 0) return []\n\n // A browser lists the default device twice, once under the id \"default\" and\n // once under its real id. Keep the real one, and put it first.\n const defaults = all.filter((device) => device.deviceId === 'default')\n const devices = all.filter(\n (device) => device.deviceId !== 'default' && device.kind === kind\n )\n for (const fallback of defaults) {\n const index = devices.findIndex(\n (device) => device.groupId === fallback.groupId\n )\n if (index > 0) devices.unshift(...devices.splice(index, 1))\n }\n\n return devices.map((device) => ({\n id: device.deviceId,\n label: device.label || (kind === 'audioinput' ? 'Microphone' : 'Speaker'),\n kind,\n }))\n}\n","// Inspired https://www.mattmontag.com/web/unlock-web-audio-in-safari-for-ios-and-macos\nexport function unlock(audioContext: AudioContext): void {\n const events = ['touchstart', 'touchend', 'mousedown', 'keydown']\n\n const unlock = async () => {\n unlockSafari(audioContext)\n if (audioContext.state === 'suspended') {\n await audioContext.resume()\n }\n // Clean events\n events.forEach((e) => document.body.removeEventListener(e, unlock))\n }\n\n // Add events\n events.forEach((e) => document.body.addEventListener(e, unlock, false))\n}\n\n// safari/ios hack - inspired by https://codepen.io/kslstn/pen/pagLqL\n// unlock audioContext - call when interactive (button click)\nfunction unlockSafari(audioContext: AudioContext): void {\n // create empty buffer and play it to unlock audioContext when interactive\n const source = audioContext.createBufferSource()\n source.buffer = audioContext.createBuffer(1, 1, 22050)\n source.connect(audioContext.destination)\n const onEnded = () => {\n source.disconnect()\n source.buffer = null\n source.removeEventListener('ended', onEnded)\n }\n source.addEventListener('ended', onEnded)\n source.start()\n}\n","import { unlock } from './unlock'\n\nexport let audioContext: AudioContext\n\nexport function createAudioContext() {\n return new (window.AudioContext || (window as any).webkitAudioContext)()\n}\n\ndeclare global {\n interface Window {\n micdropAudioContext: AudioContext\n }\n}\n\nif (window.micdropAudioContext) {\n audioContext = window.micdropAudioContext\n} else {\n window.micdropAudioContext = audioContext = createAudioContext()\n\n // Hack to unlock audio context on some browsers\n unlock(audioContext)\n}\n","// Hands the microphone signal to the main thread, continuously.\n//\n// Deciding whether someone is speaking, keeping a reserve of audio and cutting\n// it into chunks all happen in @micdrop/client, so this only has to deliver the\n// samples at a steady pace.\n\nconst INTERVAL_MS = 20\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.buffers = []\n this.length = 0\n // `sampleRate` is provided by the worklet scope\n this.frameLength = Math.round((sampleRate * INTERVAL_MS) / 1000)\n }\n\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n\n this.buffers.push(new Float32Array(input[0]))\n this.length += input[0].length\n if (this.length < this.frameLength) return true\n\n const frames = new Float32Array(this.length)\n let offset = 0\n for (const buffer of this.buffers) {\n frames.set(buffer, offset)\n offset += buffer.length\n }\n this.buffers.length = 0\n this.length = 0\n\n this.port.postMessage({ type: 'frames', frames, sampleRate }, [\n frames.buffer,\n ])\n return true\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n","import pcmProcessorWorklet from './pcm-processor-worklet?raw'\nimport { audioContext } from './audioContext'\n\ndeclare global {\n interface Window {\n micdropPcmProcessor: boolean\n }\n}\n\nexport async function initPcmProcessor() {\n if (window.micdropPcmProcessor) return\n window.micdropPcmProcessor = true\n\n // Init worklet URL\n const workletBlob = new Blob([pcmProcessorWorklet], {\n type: 'application/javascript',\n })\n const workletUrl = URL.createObjectURL(workletBlob)\n\n // Load the worklet module\n await audioContext.audioWorklet.addModule(workletUrl)\n}\n","/**\n * Stops a stream\n * @param stream - The stream to stop\n */\nexport function stopStream(stream?: MediaStream) {\n stream?.getTracks().forEach((track) => track.stop())\n}\n","import { MicdropDevice, Pcm16AudioStream, SpeakerDriver } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { listDevices } from './WebMic'\nimport { WebAudioSink } from './WebAudioSink'\n\n/**\n * Plays the assistant voice with the Web Audio API.\n */\nexport class WebSpeaker extends SpeakerDriver {\n private sink: WebAudioSink | undefined\n private stream: Pcm16AudioStream | undefined\n private _deviceId: string | undefined\n // Firefox cannot pick an output on an audio context, so a non default device\n // is played back through an audio element instead\n private element: HTMLAudioElement | undefined\n\n get isPlaying(): boolean {\n return this.stream?.isPlaying ?? false\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n /** The node the sound comes out of, to analyse or process it */\n get output(): AudioNode | undefined {\n return this.sink?.output\n }\n\n async start(): Promise<void> {\n if (this.stream) return\n\n this.sink = new WebAudioSink()\n this.stream = new Pcm16AudioStream(this.sink)\n this.stream.on('StartPlaying', () => this.emit('StartPlaying'))\n this.stream.on('StopPlaying', () => this.emit('StopPlaying'))\n this.stream.on('Volume', (volume) => this.emit('Volume', volume))\n\n this.sink.output.connect(audioContext.destination)\n }\n\n play(pcm: Int16Array, sampleRate: number): void {\n this.stream?.playAudio(pcm, sampleRate)\n }\n\n stopAudio(): void {\n this.stream?.stopAudio()\n }\n\n async stop(): Promise<void> {\n this.stream?.stopAudio()\n this.stream?.removeAllListeners()\n this.stream = undefined\n this.sink?.output.disconnect()\n this.sink = undefined\n if (this.element) {\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n }\n }\n\n async changeDevice(deviceId: string): Promise<void> {\n const sink = this.sink\n if (!sink) {\n this._deviceId = deviceId\n return\n }\n\n const devices = await this.getDevices()\n const isDefault = devices.length === 0 || devices[0].id === deviceId\n\n if (isDefault || 'setSinkId' in audioContext) {\n // Playing through the audio context is the preferred way, it is the only\n // one that keeps echo cancellation working\n await (audioContext as any).setSinkId?.(isDefault ? '' : deviceId)\n sink.output.disconnect()\n sink.output.connect(audioContext.destination)\n\n if (this.element) {\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n }\n } else {\n // Firefox: stream the sound back into an audio element, which does accept\n // an output device\n if (!this.element) {\n const destination = audioContext.createMediaStreamDestination()\n sink.output.disconnect()\n sink.output.connect(destination)\n\n this.element = new Audio()\n this.element.autoplay = true\n this.element.srcObject = destination.stream\n }\n await this.element.setSinkId(deviceId)\n }\n\n this._deviceId = deviceId\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audiooutput')\n }\n}\n","import { AudioSink, ScheduledAudio } from '@micdrop/client'\nimport { audioContext } from './audioContext'\n\n/**\n * Plays samples through the Web Audio graph, on the timeline given by the\n * caller so consecutive chunks are heard without a gap.\n */\nexport class WebAudioSink implements AudioSink {\n public readonly output: GainNode\n\n constructor() {\n this.output = audioContext.createGain()\n }\n\n get sampleRate(): number {\n return audioContext.sampleRate\n }\n\n get currentTime(): number {\n return audioContext.currentTime\n }\n\n schedule(\n samples: Float32Array,\n when: number,\n onEnded: () => void\n ): ScheduledAudio {\n const buffer = audioContext.createBuffer(\n 1,\n samples.length,\n audioContext.sampleRate\n )\n buffer.copyToChannel(samples as Float32Array<ArrayBuffer>, 0)\n\n const source = audioContext.createBufferSource()\n source.buffer = buffer\n source.connect(this.output)\n source.onended = onEnded\n source.start(when)\n\n let stopped = false\n return {\n stop() {\n if (stopped) return\n stopped = true\n source.onended = null\n try {\n source.stop()\n } catch {\n // Already finished\n }\n source.disconnect()\n },\n }\n }\n}\n"],"mappings":";AAAA,SAAS,KAAK,mBAAmB,eAAe;;;ACAhD,SAAS,iBAAgC;;;ACClC,SAAS,OAAOA,eAAkC;AACvD,QAAM,SAAS,CAAC,cAAc,YAAY,aAAa,SAAS;AAEhE,QAAMC,UAAS,YAAY;AACzB,iBAAaD,aAAY;AACzB,QAAIA,cAAa,UAAU,aAAa;AACtC,YAAMA,cAAa,OAAO;AAAA,IAC5B;AAEA,WAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,oBAAoB,GAAGC,OAAM,CAAC;AAAA,EACpE;AAGA,SAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,iBAAiB,GAAGA,SAAQ,KAAK,CAAC;AACxE;AAIA,SAAS,aAAaD,eAAkC;AAEtD,QAAM,SAASA,cAAa,mBAAmB;AAC/C,SAAO,SAASA,cAAa,aAAa,GAAG,GAAG,KAAK;AACrD,SAAO,QAAQA,cAAa,WAAW;AACvC,QAAM,UAAU,MAAM;AACpB,WAAO,WAAW;AAClB,WAAO,SAAS;AAChB,WAAO,oBAAoB,SAAS,OAAO;AAAA,EAC7C;AACA,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO,MAAM;AACf;;;AC7BO,IAAI;AAEJ,SAAS,qBAAqB;AACnC,SAAO,KAAK,OAAO,gBAAiB,OAAe,oBAAoB;AACzE;AAQA,IAAI,OAAO,qBAAqB;AAC9B,iBAAe,OAAO;AACxB,OAAO;AACL,SAAO,sBAAsB,eAAe,mBAAmB;AAG/D,SAAO,YAAY;AACrB;;;ACrBA;;;ACSA,eAAsB,mBAAmB;AACvC,MAAI,OAAO,oBAAqB;AAChC,SAAO,sBAAsB;AAG7B,QAAM,cAAc,IAAI,KAAK,CAAC,6BAAmB,GAAG;AAAA,IAClD,MAAM;AAAA,EACR,CAAC;AACD,QAAM,aAAa,IAAI,gBAAgB,WAAW;AAGlD,QAAM,aAAa,aAAa,UAAU,UAAU;AACtD;;;ACjBO,SAAS,WAAW,QAAsB;AAC/C,UAAQ,UAAU,EAAE,QAAQ,CAAC,UAAU,MAAM,KAAK,CAAC;AACrD;;;ALKO,IAAM,SAAN,cAAqB,UAAU;AAAA,EAA/B;AAAA;AA0EL,SAAQ,mBAAmB,CAAC,UAAwB;AAClD,YAAM,EAAE,MAAM,QAAQ,WAAW,IAAI,MAAM;AAC3C,UAAI,SAAS,SAAU;AACvB,WAAK,KAAK,UAAU,QAAwB,UAAoB;AAAA,IAClE;AAEA,SAAQ,iBAAiB,MAAM;AAC7B,WAAK,KAAK,cAAc;AAAA,IAC1B;AAAA;AAAA,EA5EA,IAAI,YAAqB;AACvB,WAAO,CAAC,CAAC,KAAK;AAAA,EAChB;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,MAAM,UAAkC;AAC5C,QAAI,KAAK,QAAQ;AAEf,UAAI,KAAK,cAAc,SAAU;AACjC,YAAM,KAAK,KAAK;AAAA,IAClB;AAEA,SAAK,SAAS,MAAM,UAAU,aAAa,aAAa;AAAA,MACtD,OAAO;AAAA,QACL,UAAU,EAAE,OAAO,SAAS;AAAA,QAC5B,YAAY;AAAA;AAAA,QACZ,YAAY;AAAA,QACZ,cAAc;AAAA,QACd,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,MACpB;AAAA,IACF,CAAC;AAGD,UAAM,QAAQ,KAAK,OAAO,UAAU,EAAE,CAAC;AACvC,UAAM,UAAU,OAAO,kBAAkB,EAAE;AAC3C,SAAK,YACH,WAAW,YAAY,YAAY,UAAW,YAAY;AAE5D,UAAM,iBAAiB;AACvB,SAAK,UAAU,IAAI,iBAAiB,cAAc,eAAe;AACjE,SAAK,QAAQ,KAAK,YAAY,KAAK;AAEnC,SAAK,SAAS,aAAa,wBAAwB,KAAK,MAAM;AAC9D,SAAK,OAAO,QAAQ,KAAK,OAAO;AAIhC,cAAU,aAAa,iBAAiB,gBAAgB,KAAK,cAAc;AAAA,EAC7E;AAAA,EAEA,MAAM,OAAsB;AAC1B,cAAU,aAAa;AAAA,MACrB;AAAA,MACA,KAAK;AAAA,IACP;AACA,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,KAAK,YAAY;AAC9B,WAAK,QAAQ,WAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,WAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AACA,QAAI,KAAK,QAAQ;AACf,iBAAW,KAAK,MAAM;AACtB,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,YAAY;AAAA,EACjC;AAWF;AAMA,eAAsB,YACpB,MAC0B;AAC1B,QAAM,MAAM,MAAM,UAAU,aAAa,iBAAiB;AAC1D,MAAI,IAAI,WAAW,EAAG,QAAO,CAAC;AAI9B,QAAM,WAAW,IAAI,OAAO,CAAC,WAAW,OAAO,aAAa,SAAS;AACrE,QAAM,UAAU,IAAI;AAAA,IAClB,CAAC,WAAW,OAAO,aAAa,aAAa,OAAO,SAAS;AAAA,EAC/D;AACA,aAAW,YAAY,UAAU;AAC/B,UAAM,QAAQ,QAAQ;AAAA,MACpB,CAAC,WAAW,OAAO,YAAY,SAAS;AAAA,IAC1C;AACA,QAAI,QAAQ,EAAG,SAAQ,QAAQ,GAAG,QAAQ,OAAO,OAAO,CAAC,CAAC;AAAA,EAC5D;AAEA,SAAO,QAAQ,IAAI,CAAC,YAAY;AAAA,IAC9B,IAAI,OAAO;AAAA,IACX,OAAO,OAAO,UAAU,SAAS,eAAe,eAAe;AAAA,IAC/D;AAAA,EACF,EAAE;AACJ;;;AM5HA,SAAwB,kBAAkB,qBAAqB;;;ACOxD,IAAM,eAAN,MAAwC;AAAA,EAG7C,cAAc;AACZ,SAAK,SAAS,aAAa,WAAW;AAAA,EACxC;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,SACE,SACA,MACA,SACgB;AAChB,UAAM,SAAS,aAAa;AAAA,MAC1B;AAAA,MACA,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AACA,WAAO,cAAc,SAAsC,CAAC;AAE5D,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,KAAK,MAAM;AAC1B,WAAO,UAAU;AACjB,WAAO,MAAM,IAAI;AAEjB,QAAI,UAAU;AACd,WAAO;AAAA,MACL,OAAO;AACL,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,UAAU;AACjB,YAAI;AACF,iBAAO,KAAK;AAAA,QACd,QAAQ;AAAA,QAER;AACA,eAAO,WAAW;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;;;AD/CO,IAAM,aAAN,cAAyB,cAAc;AAAA,EAQ5C,IAAI,YAAqB;AACvB,WAAO,KAAK,QAAQ,aAAa;AAAA,EACnC;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAgC;AAClC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,MAAM,QAAuB;AAC3B,QAAI,KAAK,OAAQ;AAEjB,SAAK,OAAO,IAAI,aAAa;AAC7B,SAAK,SAAS,IAAI,iBAAiB,KAAK,IAAI;AAC5C,SAAK,OAAO,GAAG,gBAAgB,MAAM,KAAK,KAAK,cAAc,CAAC;AAC9D,SAAK,OAAO,GAAG,eAAe,MAAM,KAAK,KAAK,aAAa,CAAC;AAC5D,SAAK,OAAO,GAAG,UAAU,CAAC,WAAW,KAAK,KAAK,UAAU,MAAM,CAAC;AAEhE,SAAK,KAAK,OAAO,QAAQ,aAAa,WAAW;AAAA,EACnD;AAAA,EAEA,KAAK,KAAiB,YAA0B;AAC9C,SAAK,QAAQ,UAAU,KAAK,UAAU;AAAA,EACxC;AAAA,EAEA,YAAkB;AAChB,SAAK,QAAQ,UAAU;AAAA,EACzB;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,QAAQ,UAAU;AACvB,SAAK,QAAQ,mBAAmB;AAChC,SAAK,SAAS;AACd,SAAK,MAAM,OAAO,WAAW;AAC7B,SAAK,OAAO;AACZ,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,MAAM;AACnB,WAAK,QAAQ,YAAY;AACzB,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,UAAiC;AAClD,UAAM,OAAO,KAAK;AAClB,QAAI,CAAC,MAAM;AACT,WAAK,YAAY;AACjB;AAAA,IACF;AAEA,UAAM,UAAU,MAAM,KAAK,WAAW;AACtC,UAAM,YAAY,QAAQ,WAAW,KAAK,QAAQ,CAAC,EAAE,OAAO;AAE5D,QAAI,aAAa,eAAe,cAAc;AAG5C,YAAO,aAAqB,YAAY,YAAY,KAAK,QAAQ;AACjE,WAAK,OAAO,WAAW;AACvB,WAAK,OAAO,QAAQ,aAAa,WAAW;AAE5C,UAAI,KAAK,SAAS;AAChB,aAAK,QAAQ,MAAM;AACnB,aAAK,QAAQ,YAAY;AACzB,aAAK,UAAU;AAAA,MACjB;AAAA,IACF,OAAO;AAGL,UAAI,CAAC,KAAK,SAAS;AACjB,cAAM,cAAc,aAAa,6BAA6B;AAC9D,aAAK,OAAO,WAAW;AACvB,aAAK,OAAO,QAAQ,WAAW;AAE/B,aAAK,UAAU,IAAI,MAAM;AACzB,aAAK,QAAQ,WAAW;AACxB,aAAK,QAAQ,YAAY,YAAY;AAAA,MACvC;AACA,YAAM,KAAK,QAAQ,UAAU,QAAQ;AAAA,IACvC;AAEA,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,aAAa;AAAA,EAClC;AACF;;;APzFA,cAAc;AALP,SAAS,mBAA0C;AACxD,QAAM,SAAS,QAAQ,UAAU;AACjC,SAAO,kBAAkB,aAAa,OAAO,SAAS;AACxD;AAaO,SAAS,mBAAmB;AACjC,MAAI,UAAU,IAAI,OAAO,CAAC;AAC1B,UAAQ,UAAU,IAAI,WAAW,CAAC;AAClC,oBAAkB,YAAY;AAChC;AAEA,iBAAiB;","names":["audioContext","unlock"]}
@@ -0,0 +1,36 @@
1
+ import { SileroModel } from '@micdrop/client';
2
+
3
+ /** Where the model is fetched from, when the app does not provide it */
4
+ declare const DEFAULT_SILERO_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx";
5
+ interface OnnxSileroOptions {
6
+ /** Address of the Silero v5 model, or the bytes of it */
7
+ model?: string | ArrayBuffer;
8
+ /** Where the ONNX runtime loads its WebAssembly files from */
9
+ wasmPath?: string;
10
+ }
11
+ /**
12
+ * Runs the Silero model in the browser, on the ONNX WebAssembly runtime.
13
+ *
14
+ * The state machine that turns its answers into turns lives in
15
+ * `@micdrop/client`, this only scores windows of audio.
16
+ */
17
+ declare class OnnxSileroModel implements SileroModel {
18
+ private options;
19
+ private session;
20
+ private state;
21
+ private sampleRate;
22
+ constructor(options?: OnnxSileroOptions);
23
+ load(): Promise<void>;
24
+ process(frame: Float32Array): Promise<number>;
25
+ reset(): void;
26
+ release(): Promise<void>;
27
+ }
28
+
29
+ /**
30
+ * Says where the Silero model comes from, to serve it from your own domain
31
+ * rather than from a CDN, or to hand over its bytes directly.
32
+ * @param next - Address or bytes of the model, and of the ONNX runtime
33
+ */
34
+ declare function setSileroOptions(next: OnnxSileroOptions): void;
35
+
36
+ export { DEFAULT_SILERO_MODEL_URL, OnnxSileroModel, type OnnxSileroOptions, setSileroOptions };
@@ -0,0 +1,36 @@
1
+ import { SileroModel } from '@micdrop/client';
2
+
3
+ /** Where the model is fetched from, when the app does not provide it */
4
+ declare const DEFAULT_SILERO_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx";
5
+ interface OnnxSileroOptions {
6
+ /** Address of the Silero v5 model, or the bytes of it */
7
+ model?: string | ArrayBuffer;
8
+ /** Where the ONNX runtime loads its WebAssembly files from */
9
+ wasmPath?: string;
10
+ }
11
+ /**
12
+ * Runs the Silero model in the browser, on the ONNX WebAssembly runtime.
13
+ *
14
+ * The state machine that turns its answers into turns lives in
15
+ * `@micdrop/client`, this only scores windows of audio.
16
+ */
17
+ declare class OnnxSileroModel implements SileroModel {
18
+ private options;
19
+ private session;
20
+ private state;
21
+ private sampleRate;
22
+ constructor(options?: OnnxSileroOptions);
23
+ load(): Promise<void>;
24
+ process(frame: Float32Array): Promise<number>;
25
+ reset(): void;
26
+ release(): Promise<void>;
27
+ }
28
+
29
+ /**
30
+ * Says where the Silero model comes from, to serve it from your own domain
31
+ * rather than from a CDN, or to hand over its bytes directly.
32
+ * @param next - Address or bytes of the model, and of the ONNX runtime
33
+ */
34
+ declare function setSileroOptions(next: OnnxSileroOptions): void;
35
+
36
+ export { DEFAULT_SILERO_MODEL_URL, OnnxSileroModel, type OnnxSileroOptions, setSileroOptions };
package/dist/silero.js ADDED
@@ -0,0 +1,103 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/silero.ts
31
+ var silero_exports = {};
32
+ __export(silero_exports, {
33
+ DEFAULT_SILERO_MODEL_URL: () => DEFAULT_SILERO_MODEL_URL,
34
+ OnnxSileroModel: () => OnnxSileroModel,
35
+ setSileroOptions: () => setSileroOptions
36
+ });
37
+ module.exports = __toCommonJS(silero_exports);
38
+ var import_client2 = require("@micdrop/client");
39
+
40
+ // src/vad/OnnxSileroModel.ts
41
+ var import_client = require("@micdrop/client");
42
+ var ort = __toESM(require("onnxruntime-web"));
43
+ var DEFAULT_SILERO_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx";
44
+ var DEFAULT_WASM_PATH = "https://unpkg.com/onnxruntime-web@1.23.2/dist/";
45
+ var loading;
46
+ var OnnxSileroModel = class {
47
+ constructor(options2 = {}) {
48
+ this.options = options2;
49
+ this.state = newState();
50
+ this.sampleRate = new ort.Tensor("int64", [BigInt(import_client.SILERO_SAMPLE_RATE)]);
51
+ }
52
+ async load() {
53
+ if (this.session) return;
54
+ ort.env.wasm.wasmPaths = this.options.wasmPath ?? DEFAULT_WASM_PATH;
55
+ const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL;
56
+ let bytes;
57
+ if (typeof source === "string") {
58
+ loading = loading ?? fetch(source).then((response) => response.arrayBuffer());
59
+ bytes = await loading;
60
+ } else {
61
+ bytes = source;
62
+ }
63
+ this.session = await ort.InferenceSession.create(bytes);
64
+ }
65
+ async process(frame) {
66
+ if (!this.session) await this.load();
67
+ const output = await this.session.run({
68
+ input: new ort.Tensor("float32", frame, [1, frame.length]),
69
+ state: this.state,
70
+ sr: this.sampleRate
71
+ });
72
+ this.state = output.stateN;
73
+ return output.output.data[0];
74
+ }
75
+ reset() {
76
+ this.state = newState();
77
+ }
78
+ async release() {
79
+ await this.session?.release();
80
+ this.session = void 0;
81
+ }
82
+ };
83
+ function newState() {
84
+ return new ort.Tensor("float32", new Float32Array(2 * 128), [2, 1, 128]);
85
+ }
86
+
87
+ // src/silero.ts
88
+ var options = {};
89
+ function setSileroOptions(next) {
90
+ options = next;
91
+ }
92
+ (0, import_client2.setSileroModelLoader)(async () => {
93
+ const model = new OnnxSileroModel(options);
94
+ await model.load();
95
+ return model;
96
+ });
97
+ // Annotate the CommonJS export names for ESM import in node:
98
+ 0 && (module.exports = {
99
+ DEFAULT_SILERO_MODEL_URL,
100
+ OnnxSileroModel,
101
+ setSileroOptions
102
+ });
103
+ //# sourceMappingURL=silero.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/silero.ts","../src/vad/OnnxSileroModel.ts"],"sourcesContent":["import { setSileroModelLoader } from '@micdrop/client'\nimport { OnnxSileroModel, OnnxSileroOptions } from './vad/OnnxSileroModel'\n\nexport * from './vad/OnnxSileroModel'\n\nlet options: OnnxSileroOptions = {}\n\n/**\n * Says where the Silero model comes from, to serve it from your own domain\n * rather than from a CDN, or to hand over its bytes directly.\n * @param next - Address or bytes of the model, and of the ONNX runtime\n */\nexport function setSileroOptions(next: OnnxSileroOptions) {\n options = next\n}\n\n// The ONNX runtime weighs more than the rest of Micdrop put together, so it\n// only enters the bundle of an app that imports this file\nsetSileroModelLoader(async () => {\n const model = new OnnxSileroModel(options)\n await model.load()\n return model\n})\n","import { SileroModel, SILERO_SAMPLE_RATE } from '@micdrop/client'\nimport * as ort from 'onnxruntime-web'\n\n/** Where the model is fetched from, when the app does not provide it */\nexport const DEFAULT_SILERO_MODEL_URL =\n 'https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx'\n\n// onnxruntime-web@1.23 is not hosted on jsdelivr (its dist exceeds the CDN size\n// limit, so every file 404s). unpkg serves it, so the wasm backend comes from\n// there instead.\nconst DEFAULT_WASM_PATH = 'https://unpkg.com/onnxruntime-web@1.23.2/dist/'\n\nexport interface OnnxSileroOptions {\n /** Address of the Silero v5 model, or the bytes of it */\n model?: string | ArrayBuffer\n /** Where the ONNX runtime loads its WebAssembly files from */\n wasmPath?: string\n}\n\nlet loading: Promise<ArrayBuffer> | undefined\n\n/**\n * Runs the Silero model in the browser, on the ONNX WebAssembly runtime.\n *\n * The state machine that turns its answers into turns lives in\n * `@micdrop/client`, this only scores windows of audio.\n */\nexport class OnnxSileroModel implements SileroModel {\n private session: ort.InferenceSession | undefined\n private state: ort.Tensor\n private sampleRate: ort.Tensor\n\n constructor(private options: OnnxSileroOptions = {}) {\n this.state = newState()\n this.sampleRate = new ort.Tensor('int64', [BigInt(SILERO_SAMPLE_RATE)])\n }\n\n async load() {\n if (this.session) return\n\n ort.env.wasm.wasmPaths = this.options.wasmPath ?? DEFAULT_WASM_PATH\n\n const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL\n let bytes: ArrayBuffer\n if (typeof source === 'string') {\n // Fetched once per page, the model is a couple of megabytes\n loading =\n loading ?? fetch(source).then((response) => response.arrayBuffer())\n bytes = await loading\n } else {\n bytes = source\n }\n\n this.session = await ort.InferenceSession.create(bytes)\n }\n\n async process(frame: Float32Array): Promise<number> {\n if (!this.session) await this.load()\n\n const output = await this.session!.run({\n input: new ort.Tensor('float32', frame, [1, frame.length]),\n state: this.state,\n sr: this.sampleRate,\n })\n\n this.state = output.stateN as ort.Tensor\n return (output.output.data as Float32Array)[0]\n }\n\n reset() {\n this.state = newState()\n }\n\n async release() {\n await this.session?.release()\n this.session = undefined\n }\n}\n\n/** The memory the model carries from one window to the next, emptied */\nfunction newState(): ort.Tensor {\n return new ort.Tensor('float32', new Float32Array(2 * 128), [2, 1, 128])\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAAA,iBAAqC;;;ACArC,oBAAgD;AAChD,UAAqB;AAGd,IAAM,2BACX;AAKF,IAAM,oBAAoB;AAS1B,IAAI;AAQG,IAAM,kBAAN,MAA6C;AAAA,EAKlD,YAAoBC,WAA6B,CAAC,GAAG;AAAjC,mBAAAA;AAClB,SAAK,QAAQ,SAAS;AACtB,SAAK,aAAa,IAAQ,WAAO,SAAS,CAAC,OAAO,gCAAkB,CAAC,CAAC;AAAA,EACxE;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,KAAK,QAAS;AAElB,IAAI,QAAI,KAAK,YAAY,KAAK,QAAQ,YAAY;AAElD,UAAM,SAAS,KAAK,QAAQ,SAAS;AACrC,QAAI;AACJ,QAAI,OAAO,WAAW,UAAU;AAE9B,gBACE,WAAW,MAAM,MAAM,EAAE,KAAK,CAAC,aAAa,SAAS,YAAY,CAAC;AACpE,cAAQ,MAAM;AAAA,IAChB,OAAO;AACL,cAAQ;AAAA,IACV;AAEA,SAAK,UAAU,MAAU,qBAAiB,OAAO,KAAK;AAAA,EACxD;AAAA,EAEA,MAAM,QAAQ,OAAsC;AAClD,QAAI,CAAC,KAAK,QAAS,OAAM,KAAK,KAAK;AAEnC,UAAM,SAAS,MAAM,KAAK,QAAS,IAAI;AAAA,MACrC,OAAO,IAAQ,WAAO,WAAW,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC;AAAA,MACzD,OAAO,KAAK;AAAA,MACZ,IAAI,KAAK;AAAA,IACX,CAAC;AAED,SAAK,QAAQ,OAAO;AACpB,WAAQ,OAAO,OAAO,KAAsB,CAAC;AAAA,EAC/C;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,MAAM,UAAU;AACd,UAAM,KAAK,SAAS,QAAQ;AAC5B,SAAK,UAAU;AAAA,EACjB;AACF;AAGA,SAAS,WAAuB;AAC9B,SAAO,IAAQ,WAAO,WAAW,IAAI,aAAa,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;AACzE;;;AD7EA,IAAI,UAA6B,CAAC;AAO3B,SAAS,iBAAiB,MAAyB;AACxD,YAAU;AACZ;AAAA,IAIA,qCAAqB,YAAY;AAC/B,QAAM,QAAQ,IAAI,gBAAgB,OAAO;AACzC,QAAM,MAAM,KAAK;AACjB,SAAO;AACT,CAAC;","names":["import_client","options"]}
@@ -0,0 +1,66 @@
1
+ // src/silero.ts
2
+ import { setSileroModelLoader } from "@micdrop/client";
3
+
4
+ // src/vad/OnnxSileroModel.ts
5
+ import { SILERO_SAMPLE_RATE } from "@micdrop/client";
6
+ import * as ort from "onnxruntime-web";
7
+ var DEFAULT_SILERO_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx";
8
+ var DEFAULT_WASM_PATH = "https://unpkg.com/onnxruntime-web@1.23.2/dist/";
9
+ var loading;
10
+ var OnnxSileroModel = class {
11
+ constructor(options2 = {}) {
12
+ this.options = options2;
13
+ this.state = newState();
14
+ this.sampleRate = new ort.Tensor("int64", [BigInt(SILERO_SAMPLE_RATE)]);
15
+ }
16
+ async load() {
17
+ if (this.session) return;
18
+ ort.env.wasm.wasmPaths = this.options.wasmPath ?? DEFAULT_WASM_PATH;
19
+ const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL;
20
+ let bytes;
21
+ if (typeof source === "string") {
22
+ loading = loading ?? fetch(source).then((response) => response.arrayBuffer());
23
+ bytes = await loading;
24
+ } else {
25
+ bytes = source;
26
+ }
27
+ this.session = await ort.InferenceSession.create(bytes);
28
+ }
29
+ async process(frame) {
30
+ if (!this.session) await this.load();
31
+ const output = await this.session.run({
32
+ input: new ort.Tensor("float32", frame, [1, frame.length]),
33
+ state: this.state,
34
+ sr: this.sampleRate
35
+ });
36
+ this.state = output.stateN;
37
+ return output.output.data[0];
38
+ }
39
+ reset() {
40
+ this.state = newState();
41
+ }
42
+ async release() {
43
+ await this.session?.release();
44
+ this.session = void 0;
45
+ }
46
+ };
47
+ function newState() {
48
+ return new ort.Tensor("float32", new Float32Array(2 * 128), [2, 1, 128]);
49
+ }
50
+
51
+ // src/silero.ts
52
+ var options = {};
53
+ function setSileroOptions(next) {
54
+ options = next;
55
+ }
56
+ setSileroModelLoader(async () => {
57
+ const model = new OnnxSileroModel(options);
58
+ await model.load();
59
+ return model;
60
+ });
61
+ export {
62
+ DEFAULT_SILERO_MODEL_URL,
63
+ OnnxSileroModel,
64
+ setSileroOptions
65
+ };
66
+ //# sourceMappingURL=silero.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/silero.ts","../src/vad/OnnxSileroModel.ts"],"sourcesContent":["import { setSileroModelLoader } from '@micdrop/client'\nimport { OnnxSileroModel, OnnxSileroOptions } from './vad/OnnxSileroModel'\n\nexport * from './vad/OnnxSileroModel'\n\nlet options: OnnxSileroOptions = {}\n\n/**\n * Says where the Silero model comes from, to serve it from your own domain\n * rather than from a CDN, or to hand over its bytes directly.\n * @param next - Address or bytes of the model, and of the ONNX runtime\n */\nexport function setSileroOptions(next: OnnxSileroOptions) {\n options = next\n}\n\n// The ONNX runtime weighs more than the rest of Micdrop put together, so it\n// only enters the bundle of an app that imports this file\nsetSileroModelLoader(async () => {\n const model = new OnnxSileroModel(options)\n await model.load()\n return model\n})\n","import { SileroModel, SILERO_SAMPLE_RATE } from '@micdrop/client'\nimport * as ort from 'onnxruntime-web'\n\n/** Where the model is fetched from, when the app does not provide it */\nexport const DEFAULT_SILERO_MODEL_URL =\n 'https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx'\n\n// onnxruntime-web@1.23 is not hosted on jsdelivr (its dist exceeds the CDN size\n// limit, so every file 404s). unpkg serves it, so the wasm backend comes from\n// there instead.\nconst DEFAULT_WASM_PATH = 'https://unpkg.com/onnxruntime-web@1.23.2/dist/'\n\nexport interface OnnxSileroOptions {\n /** Address of the Silero v5 model, or the bytes of it */\n model?: string | ArrayBuffer\n /** Where the ONNX runtime loads its WebAssembly files from */\n wasmPath?: string\n}\n\nlet loading: Promise<ArrayBuffer> | undefined\n\n/**\n * Runs the Silero model in the browser, on the ONNX WebAssembly runtime.\n *\n * The state machine that turns its answers into turns lives in\n * `@micdrop/client`, this only scores windows of audio.\n */\nexport class OnnxSileroModel implements SileroModel {\n private session: ort.InferenceSession | undefined\n private state: ort.Tensor\n private sampleRate: ort.Tensor\n\n constructor(private options: OnnxSileroOptions = {}) {\n this.state = newState()\n this.sampleRate = new ort.Tensor('int64', [BigInt(SILERO_SAMPLE_RATE)])\n }\n\n async load() {\n if (this.session) return\n\n ort.env.wasm.wasmPaths = this.options.wasmPath ?? DEFAULT_WASM_PATH\n\n const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL\n let bytes: ArrayBuffer\n if (typeof source === 'string') {\n // Fetched once per page, the model is a couple of megabytes\n loading =\n loading ?? fetch(source).then((response) => response.arrayBuffer())\n bytes = await loading\n } else {\n bytes = source\n }\n\n this.session = await ort.InferenceSession.create(bytes)\n }\n\n async process(frame: Float32Array): Promise<number> {\n if (!this.session) await this.load()\n\n const output = await this.session!.run({\n input: new ort.Tensor('float32', frame, [1, frame.length]),\n state: this.state,\n sr: this.sampleRate,\n })\n\n this.state = output.stateN as ort.Tensor\n return (output.output.data as Float32Array)[0]\n }\n\n reset() {\n this.state = newState()\n }\n\n async release() {\n await this.session?.release()\n this.session = undefined\n }\n}\n\n/** The memory the model carries from one window to the next, emptied */\nfunction newState(): ort.Tensor {\n return new ort.Tensor('float32', new Float32Array(2 * 128), [2, 1, 128])\n}\n"],"mappings":";AAAA,SAAS,4BAA4B;;;ACArC,SAAsB,0BAA0B;AAChD,YAAY,SAAS;AAGd,IAAM,2BACX;AAKF,IAAM,oBAAoB;AAS1B,IAAI;AAQG,IAAM,kBAAN,MAA6C;AAAA,EAKlD,YAAoBA,WAA6B,CAAC,GAAG;AAAjC,mBAAAA;AAClB,SAAK,QAAQ,SAAS;AACtB,SAAK,aAAa,IAAQ,WAAO,SAAS,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAAA,EACxE;AAAA,EAEA,MAAM,OAAO;AACX,QAAI,KAAK,QAAS;AAElB,IAAI,QAAI,KAAK,YAAY,KAAK,QAAQ,YAAY;AAElD,UAAM,SAAS,KAAK,QAAQ,SAAS;AACrC,QAAI;AACJ,QAAI,OAAO,WAAW,UAAU;AAE9B,gBACE,WAAW,MAAM,MAAM,EAAE,KAAK,CAAC,aAAa,SAAS,YAAY,CAAC;AACpE,cAAQ,MAAM;AAAA,IAChB,OAAO;AACL,cAAQ;AAAA,IACV;AAEA,SAAK,UAAU,MAAU,qBAAiB,OAAO,KAAK;AAAA,EACxD;AAAA,EAEA,MAAM,QAAQ,OAAsC;AAClD,QAAI,CAAC,KAAK,QAAS,OAAM,KAAK,KAAK;AAEnC,UAAM,SAAS,MAAM,KAAK,QAAS,IAAI;AAAA,MACrC,OAAO,IAAQ,WAAO,WAAW,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC;AAAA,MACzD,OAAO,KAAK;AAAA,MACZ,IAAI,KAAK;AAAA,IACX,CAAC;AAED,SAAK,QAAQ,OAAO;AACpB,WAAQ,OAAO,OAAO,KAAsB,CAAC;AAAA,EAC/C;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,MAAM,UAAU;AACd,UAAM,KAAK,SAAS,QAAQ;AAC5B,SAAK,UAAU;AAAA,EACjB;AACF;AAGA,SAAS,WAAuB;AAC9B,SAAO,IAAQ,WAAO,WAAW,IAAI,aAAa,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;AACzE;;;AD7EA,IAAI,UAA6B,CAAC;AAO3B,SAAS,iBAAiB,MAAyB;AACxD,YAAU;AACZ;AAIA,qBAAqB,YAAY;AAC/B,QAAM,QAAQ,IAAI,gBAAgB,OAAO;AACzC,QAAM,MAAM,KAAK;AACjB,SAAO;AACT,CAAC;","names":["options"]}
package/package.json ADDED
@@ -0,0 +1,56 @@
1
+ {
2
+ "name": "@micdrop/web",
3
+ "version": "1.0.0",
4
+ "description": "\ud83d\udd90\ufe0f\ud83c\udfa4 Micdrop: real-time voice conversations with AI, in the browser",
5
+ "author": "Godefroy de Compreignac",
6
+ "license": "MIT",
7
+ "main": "dist/index.js",
8
+ "module": "dist/index.mjs",
9
+ "types": "dist/index.d.ts",
10
+ "scripts": {
11
+ "build": "tsup",
12
+ "dev": "tsup --watch",
13
+ "clean": "rm -rf dist",
14
+ "typecheck": "tsc --noEmit",
15
+ "prepublishOnly": "npm run build"
16
+ },
17
+ "files": [
18
+ "dist"
19
+ ],
20
+ "repository": {
21
+ "type": "git",
22
+ "url": "git+https://github.com/Godefroy/micdrop.git",
23
+ "directory": "packages/web"
24
+ },
25
+ "publishConfig": {
26
+ "access": "public"
27
+ },
28
+ "keywords": [
29
+ "microphone",
30
+ "audio",
31
+ "browser",
32
+ "voice",
33
+ "conversation"
34
+ ],
35
+ "dependencies": {
36
+ "@micdrop/client": "^3.0.0",
37
+ "onnxruntime-web": "^1.23.2"
38
+ },
39
+ "devDependencies": {
40
+ "esbuild-raw-plugin": "^0.3.1",
41
+ "tsup": "^8.5.1",
42
+ "typescript": "^5.9.3"
43
+ },
44
+ "exports": {
45
+ ".": {
46
+ "types": "./dist/index.d.ts",
47
+ "import": "./dist/index.mjs",
48
+ "require": "./dist/index.js"
49
+ },
50
+ "./silero": {
51
+ "types": "./dist/silero.d.ts",
52
+ "import": "./dist/silero.mjs",
53
+ "require": "./dist/silero.js"
54
+ }
55
+ }
56
+ }