@micdrop/web 1.3.1 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +26 -7
- package/dist/index.d.ts +26 -7
- package/dist/index.js +255 -127
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +255 -127
- package/dist/index.mjs.map +1 -1
- package/dist/silero.d.mts +3 -1
- package/dist/silero.d.ts +3 -1
- package/dist/silero.js +26 -21
- package/dist/silero.js.map +1 -1
- package/dist/silero.mjs +31 -22
- package/dist/silero.mjs.map +1 -1
- package/dist/vite.js +1 -1
- package/dist/vite.js.map +1 -1
- package/package.json +3 -3
package/dist/index.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts","../src/audio/WebMic.ts","../src/audio/unlock.ts","../src/audio/audioContext.ts","raw:./pcm-processor-worklet","../src/audio/pcm-processor.ts","../src/audio/stopStream.ts","../src/audio/WebSpeaker.ts","../src/audio/WebAudioSink.ts"],"sourcesContent":["import { Mic, setMicdropStorage, Speaker } from '@micdrop/client'\nimport { WebMic } from './audio/WebMic'\nimport { WebSpeaker } from './audio/WebSpeaker'\n\n/**\n * The node the assistant voice comes out of, to analyse or process it.\n *\n * Connect an AnalyserNode to it to read the envelope on your own frames, which\n * is finer than the ten levels a second the `Volume` event reports.\n * @returns The node, once the call has started\n */\nexport function getSpeakerOutput(): AudioNode | undefined {\n const driver = Speaker.getDriver()\n return driver instanceof WebSpeaker ? driver.output : undefined\n}\n\nexport * from '@micdrop/client'\nexport * from './audio/audioContext'\nexport * from './audio/WebAudioSink'\nexport * from './audio/WebMic'\nexport * from './audio/WebSpeaker'\n\n/**\n * Records and plays with the Web Audio API, and keeps settings in\n * `localStorage`. Called when the package is imported, so a page has nothing\n * to wire.\n */\nexport function registerWebAudio() {\n Mic.setDriver(new WebMic())\n Speaker.setDriver(new WebSpeaker())\n setMicdropStorage(localStorage)\n}\n\nregisterWebAudio()\n","import { MicDriver, MicdropDevice } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { initPcmProcessor } from './pcm-processor'\nimport { stopStream } from './stopStream'\n\n/**\n * How long Firefox needs to let go of a microphone it released.\n *\n * A microphone requested again before that comes back without its processing:\n * no echo cancellation, no noise suppression and a raw gain some 30 dB higher,\n * while its settings still report them on. The room then sounds like speech,\n * and the assistant hears its own voice. No event tells when the device is\n * free, so a new stream waits for this delay to pass.\n */\nconst FIREFOX_RELEASE_DELAY = 15000 // ms\n\nconst isFirefox =\n typeof navigator !== 'undefined' && /Firefox\\//.test(navigator.userAgent)\n\n// The last stream released, outside the class: the browser holds the device\n// whichever WebMic opened it\nlet lastRelease: { at: number; deviceId: string | undefined } | undefined\n\n/**\n * Records the microphone with the Web Audio API.\n *\n * An audio worklet delivers the samples at a steady pace, and everything that\n * happens to them afterwards lives in `@micdrop/client`.\n */\nexport class WebMic extends MicDriver {\n private stream: MediaStream | undefined\n private source: MediaStreamAudioSourceNode | undefined\n private worklet: AudioWorkletNode | undefined\n private _deviceId: string | undefined\n\n get isStarted(): boolean {\n return !!this.stream\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n // Asking for echo cancellation is not getting it: some browsers and devices\n // do without, and the settings of the track say so\n get echoCancellation(): boolean | undefined {\n const track = this.stream?.getAudioTracks()[0]\n const setting = track?.getSettings?.().echoCancellation as unknown\n if (setting === undefined) return undefined\n return setting !== false\n }\n\n async start(deviceId?: string): Promise<void> {\n if (this.stream) {\n // Same device, keep recording\n if (this._deviceId === deviceId) return\n await this.stop()\n }\n\n await waitForRelease(deviceId)\n\n this.stream = await navigator.mediaDevices.getUserMedia({\n audio: {\n deviceId: { ideal: deviceId },\n sampleRate: 16000, // not working, it will follow device settings, usually 44.1kHz\n sampleSize: 16,\n channelCount: 1,\n echoCancellation: true,\n noiseSuppression: true,\n },\n })\n\n // Remember which device the browser settled on\n const track = this.stream.getTracks()[0]\n const settled = track?.getCapabilities?.().deviceId\n this._deviceId =\n settled && settled !== 'default' ? settled : (deviceId ?? settled)\n\n await initPcmProcessor()\n this.worklet = new AudioWorkletNode(audioContext, 'pcm-processor')\n this.worklet.port.onmessage = this.onWorkletMessage\n\n this.source = audioContext.createMediaStreamSource(this.stream)\n this.source.connect(this.worklet)\n\n // A browser only reveals the device labels once a stream is running, and\n // Firefox lists nothing at all before that\n navigator.mediaDevices.addEventListener('devicechange', this.onDeviceChange)\n }\n\n async stop(): Promise<void> {\n navigator.mediaDevices.removeEventListener(\n 'devicechange',\n this.onDeviceChange\n )\n if (this.worklet) {\n this.worklet.port.onmessage = null\n this.worklet.disconnect()\n this.worklet = undefined\n }\n if (this.source) {\n this.source.disconnect()\n this.source = undefined\n }\n if (this.stream) {\n stopStream(this.stream)\n this.stream = undefined\n lastRelease = { at: Date.now(), deviceId: this._deviceId }\n }\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audioinput')\n }\n\n private onWorkletMessage = (event: MessageEvent) => {\n const { type, frames, sampleRate } = event.data\n if (type !== 'frames') return\n this.emit('Frames', frames as Float32Array, sampleRate as number)\n }\n\n private onDeviceChange = () => {\n this.emit('DeviceChange')\n }\n}\n\n/**\n * Waits until Firefox let go of the microphone, see FIREFOX_RELEASE_DELAY\n * @param deviceId - The device about to be opened\n */\nasync function waitForRelease(deviceId?: string) {\n if (!isFirefox || !lastRelease) return\n if (deviceId && lastRelease.deviceId && deviceId !== lastRelease.deviceId) {\n return\n }\n const remaining = lastRelease.at + FIREFOX_RELEASE_DELAY - Date.now()\n if (remaining > 0) {\n await new Promise((resolve) => setTimeout(resolve, remaining))\n }\n}\n\n/**\n * Lists the devices of one kind, the default one first\n * @param kind - Which devices to list\n */\nexport async function listDevices(\n kind: 'audioinput' | 'audiooutput'\n): Promise<MicdropDevice[]> {\n const all = await navigator.mediaDevices.enumerateDevices()\n if (all.length === 0) return []\n\n // A browser lists the default device twice, once under the id \"default\" and\n // once under its real id. Keep the real one, and put it first.\n const defaults = all.filter((device) => device.deviceId === 'default')\n const devices = all.filter(\n (device) => device.deviceId !== 'default' && device.kind === kind\n )\n for (const fallback of defaults) {\n const index = devices.findIndex(\n (device) => device.groupId === fallback.groupId\n )\n if (index > 0) devices.unshift(...devices.splice(index, 1))\n }\n\n return devices.map((device) => ({\n id: device.deviceId,\n label: device.label || (kind === 'audioinput' ? 'Microphone' : 'Speaker'),\n kind,\n }))\n}\n","// Inspired https://www.mattmontag.com/web/unlock-web-audio-in-safari-for-ios-and-macos\nexport function unlock(audioContext: AudioContext): void {\n const events = ['touchstart', 'touchend', 'mousedown', 'keydown']\n\n const unlock = async () => {\n unlockSafari(audioContext)\n if (audioContext.state === 'suspended') {\n await audioContext.resume()\n }\n // Clean events\n events.forEach((e) => document.body.removeEventListener(e, unlock))\n }\n\n // Add events\n events.forEach((e) => document.body.addEventListener(e, unlock, false))\n}\n\n// safari/ios hack - inspired by https://codepen.io/kslstn/pen/pagLqL\n// unlock audioContext - call when interactive (button click)\nfunction unlockSafari(audioContext: AudioContext): void {\n // create empty buffer and play it to unlock audioContext when interactive\n const source = audioContext.createBufferSource()\n source.buffer = audioContext.createBuffer(1, 1, 22050)\n source.connect(audioContext.destination)\n const onEnded = () => {\n source.disconnect()\n source.buffer = null\n source.removeEventListener('ended', onEnded)\n }\n source.addEventListener('ended', onEnded)\n source.start()\n}\n","import { unlock } from './unlock'\n\nexport let audioContext: AudioContext\n\nexport function createAudioContext() {\n return new (window.AudioContext || (window as any).webkitAudioContext)()\n}\n\ndeclare global {\n interface Window {\n micdropAudioContext: AudioContext\n }\n}\n\nif (window.micdropAudioContext) {\n audioContext = window.micdropAudioContext\n} else {\n window.micdropAudioContext = audioContext = createAudioContext()\n\n // Hack to unlock audio context on some browsers\n unlock(audioContext)\n}\n","// Hands the microphone signal to the main thread, continuously.\n//\n// Deciding whether someone is speaking, keeping a reserve of audio and cutting\n// it into chunks all happen in @micdrop/client, so this only has to deliver the\n// samples at a steady pace.\n\nconst INTERVAL_MS = 20\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.buffers = []\n this.length = 0\n // `sampleRate` is provided by the worklet scope\n this.frameLength = Math.round((sampleRate * INTERVAL_MS) / 1000)\n }\n\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n\n this.buffers.push(new Float32Array(input[0]))\n this.length += input[0].length\n if (this.length < this.frameLength) return true\n\n const frames = new Float32Array(this.length)\n let offset = 0\n for (const buffer of this.buffers) {\n frames.set(buffer, offset)\n offset += buffer.length\n }\n this.buffers.length = 0\n this.length = 0\n\n this.port.postMessage({ type: 'frames', frames, sampleRate }, [\n frames.buffer,\n ])\n return true\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n","import pcmProcessorWorklet from './pcm-processor-worklet?raw'\nimport { audioContext } from './audioContext'\n\ndeclare global {\n interface Window {\n micdropPcmProcessor: boolean\n }\n}\n\nexport async function initPcmProcessor() {\n if (window.micdropPcmProcessor) return\n window.micdropPcmProcessor = true\n\n // Init worklet URL\n const workletBlob = new Blob([pcmProcessorWorklet], {\n type: 'application/javascript',\n })\n const workletUrl = URL.createObjectURL(workletBlob)\n\n // Load the worklet module\n await audioContext.audioWorklet.addModule(workletUrl)\n}\n","/**\n * Stops a stream\n * @param stream - The stream to stop\n */\nexport function stopStream(stream?: MediaStream) {\n stream?.getTracks().forEach((track) => track.stop())\n}\n","import { MicdropDevice, Pcm16AudioStream, SpeakerDriver } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { listDevices } from './WebMic'\nimport { WebAudioSink } from './WebAudioSink'\n\n// A Bluetooth headset switches to its call profile as its microphone opens,\n// and Firefox mangles what it plays through that switch: the first answer of a\n// call comes out all at once. The speaker starts right after the microphone, so\n// it plays silence that long and holds the first answer until then.\nconst FIREFOX_WARM_UP = 2000 // ms\n\n/**\n * Plays the assistant voice with the Web Audio API.\n */\nexport class WebSpeaker extends SpeakerDriver {\n private sink: WebAudioSink | undefined\n private stream: Pcm16AudioStream | undefined\n private _deviceId: string | undefined\n // Firefox cannot pick an output on an audio context, so a non default device\n // is played back through an audio element instead. iOS reports echo\n // cancellation on and still hears an audio context, but not an element.\n private element: HTMLAudioElement | undefined\n private isEchoFallback = false\n\n get isPlaying(): boolean {\n return this.stream?.isPlaying ?? false\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n /** The node the sound comes out of, to analyse or process it */\n get output(): AudioNode | undefined {\n return this.sink?.output\n }\n\n async start(): Promise<void> {\n if (this.stream) return\n\n this.sink = new WebAudioSink()\n this.stream = new Pcm16AudioStream(this.sink)\n if (navigator.userAgent.includes('Firefox')) {\n this.stream.warmUp(FIREFOX_WARM_UP)\n }\n this.stream.on('StartPlaying', () => this.emit('StartPlaying'))\n this.stream.on('StopPlaying', () => this.emit('StopPlaying'))\n this.stream.on('Volume', (volume) => this.emit('Volume', volume))\n\n this.sink.output.connect(audioContext.destination)\n }\n\n play(pcm: Int16Array, sampleRate: number): void {\n this.stream?.playAudio(pcm, sampleRate)\n }\n\n stopAudio(): void {\n this.stream?.stopAudio()\n }\n\n async stop(): Promise<void> {\n this.stream?.destroy()\n this.stream = undefined\n this.sink?.output.disconnect()\n this.sink = undefined\n this.isEchoFallback = false\n if (this.element) {\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n }\n }\n\n /**\n * Moves the sound to an audio element, while it plays. The audio context\n * keeps playing until the element is known to be allowed to.\n */\n async useEchoFallback(): Promise<boolean> {\n const sink = this.sink\n if (!sink || this.element) return false\n\n const destination = audioContext.createMediaStreamDestination()\n const element = new Audio()\n element.srcObject = destination.stream\n try {\n await element.play()\n } catch (error) {\n console.error('[Micdrop] Playing through an element failed', error)\n element.srcObject = null\n return false\n }\n\n // The call ended in the meantime\n if (this.sink !== sink || this.element) {\n element.pause()\n element.srcObject = null\n return false\n }\n sink.output.disconnect(audioContext.destination)\n sink.output.connect(destination)\n this.element = element\n this.isEchoFallback = true\n return true\n }\n\n resetEchoFallback(): void {\n if (!this.isEchoFallback || !this.sink || !this.element) return\n this.isEchoFallback = false\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n this.sink.output.disconnect()\n this.sink.output.connect(audioContext.destination)\n }\n\n async changeDevice(deviceId: string): Promise<void> {\n const sink = this.sink\n if (!sink) {\n this._deviceId = deviceId\n return\n }\n\n const devices = await this.getDevices()\n const isDefault = devices.length === 0 || devices[0].id === deviceId\n\n if (!this.isEchoFallback && (isDefault || 'setSinkId' in audioContext)) {\n // Playing through the audio context is the preferred way, it is the only\n // one that keeps echo cancellation working\n await (audioContext as any).setSinkId?.(isDefault ? '' : deviceId)\n sink.output.disconnect()\n sink.output.connect(audioContext.destination)\n\n if (this.element) {\n this.element.pause()\n this.element.srcObject = null\n this.element = undefined\n }\n } else {\n // Firefox: stream the sound back into an audio element, which does accept\n // an output device\n if (!this.element) {\n const destination = audioContext.createMediaStreamDestination()\n sink.output.disconnect()\n sink.output.connect(destination)\n\n this.element = new Audio()\n this.element.autoplay = true\n this.element.srcObject = destination.stream\n }\n await this.element.setSinkId?.(deviceId)\n }\n\n this._deviceId = deviceId\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audiooutput')\n }\n}\n","import { AudioSink, ScheduledAudio } from '@micdrop/client'\nimport { audioContext } from './audioContext'\n\n/**\n * Plays samples through the Web Audio graph, on the timeline given by the\n * caller so consecutive chunks are heard without a gap.\n */\nexport class WebAudioSink implements AudioSink {\n public readonly output: GainNode\n\n constructor() {\n this.output = audioContext.createGain()\n }\n\n get sampleRate(): number {\n return audioContext.sampleRate\n }\n\n get currentTime(): number {\n return audioContext.currentTime\n }\n\n schedule(\n samples: Float32Array,\n when: number,\n onEnded: () => void\n ): ScheduledAudio {\n const buffer = audioContext.createBuffer(\n 1,\n samples.length,\n audioContext.sampleRate\n )\n buffer.copyToChannel(samples as Float32Array<ArrayBuffer>, 0)\n\n const source = audioContext.createBufferSource()\n source.buffer = buffer\n source.connect(this.output)\n source.onended = onEnded\n source.start(when)\n\n let stopped = false\n return {\n stop() {\n if (stopped) return\n stopped = true\n source.onended = null\n try {\n source.stop()\n } catch {\n // Already finished\n }\n source.disconnect()\n },\n }\n }\n}\n"],"mappings":";AAAA,SAAS,KAAK,mBAAmB,eAAe;;;ACAhD,SAAS,iBAAgC;;;ACClC,SAAS,OAAOA,eAAkC;AACvD,QAAM,SAAS,CAAC,cAAc,YAAY,aAAa,SAAS;AAEhE,QAAMC,UAAS,YAAY;AACzB,iBAAaD,aAAY;AACzB,QAAIA,cAAa,UAAU,aAAa;AACtC,YAAMA,cAAa,OAAO;AAAA,IAC5B;AAEA,WAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,oBAAoB,GAAGC,OAAM,CAAC;AAAA,EACpE;AAGA,SAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,iBAAiB,GAAGA,SAAQ,KAAK,CAAC;AACxE;AAIA,SAAS,aAAaD,eAAkC;AAEtD,QAAM,SAASA,cAAa,mBAAmB;AAC/C,SAAO,SAASA,cAAa,aAAa,GAAG,GAAG,KAAK;AACrD,SAAO,QAAQA,cAAa,WAAW;AACvC,QAAM,UAAU,MAAM;AACpB,WAAO,WAAW;AAClB,WAAO,SAAS;AAChB,WAAO,oBAAoB,SAAS,OAAO;AAAA,EAC7C;AACA,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO,MAAM;AACf;;;AC7BO,IAAI;AAEJ,SAAS,qBAAqB;AACnC,SAAO,KAAK,OAAO,gBAAiB,OAAe,oBAAoB;AACzE;AAQA,IAAI,OAAO,qBAAqB;AAC9B,iBAAe,OAAO;AACxB,OAAO;AACL,SAAO,sBAAsB,eAAe,mBAAmB;AAG/D,SAAO,YAAY;AACrB;;;ACrBA;;;ACSA,eAAsB,mBAAmB;AACvC,MAAI,OAAO,oBAAqB;AAChC,SAAO,sBAAsB;AAG7B,QAAM,cAAc,IAAI,KAAK,CAAC,6BAAmB,GAAG;AAAA,IAClD,MAAM;AAAA,EACR,CAAC;AACD,QAAM,aAAa,IAAI,gBAAgB,WAAW;AAGlD,QAAM,aAAa,aAAa,UAAU,UAAU;AACtD;;;ACjBO,SAAS,WAAW,QAAsB;AAC/C,UAAQ,UAAU,EAAE,QAAQ,CAAC,UAAU,MAAM,KAAK,CAAC;AACrD;;;ALQA,IAAM,wBAAwB;AAE9B,IAAM,YACJ,OAAO,cAAc,eAAe,YAAY,KAAK,UAAU,SAAS;AAI1E,IAAI;AAQG,IAAM,SAAN,cAAqB,UAAU;AAAA,EAA/B;AAAA;AAsFL,SAAQ,mBAAmB,CAAC,UAAwB;AAClD,YAAM,EAAE,MAAM,QAAQ,WAAW,IAAI,MAAM;AAC3C,UAAI,SAAS,SAAU;AACvB,WAAK,KAAK,UAAU,QAAwB,UAAoB;AAAA,IAClE;AAEA,SAAQ,iBAAiB,MAAM;AAC7B,WAAK,KAAK,cAAc;AAAA,IAC1B;AAAA;AAAA,EAxFA,IAAI,YAAqB;AACvB,WAAO,CAAC,CAAC,KAAK;AAAA,EAChB;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA,EAIA,IAAI,mBAAwC;AAC1C,UAAM,QAAQ,KAAK,QAAQ,eAAe,EAAE,CAAC;AAC7C,UAAM,UAAU,OAAO,cAAc,EAAE;AACvC,QAAI,YAAY,OAAW,QAAO;AAClC,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAM,MAAM,UAAkC;AAC5C,QAAI,KAAK,QAAQ;AAEf,UAAI,KAAK,cAAc,SAAU;AACjC,YAAM,KAAK,KAAK;AAAA,IAClB;AAEA,UAAM,eAAe,QAAQ;AAE7B,SAAK,SAAS,MAAM,UAAU,aAAa,aAAa;AAAA,MACtD,OAAO;AAAA,QACL,UAAU,EAAE,OAAO,SAAS;AAAA,QAC5B,YAAY;AAAA;AAAA,QACZ,YAAY;AAAA,QACZ,cAAc;AAAA,QACd,kBAAkB;AAAA,QAClB,kBAAkB;AAAA,MACpB;AAAA,IACF,CAAC;AAGD,UAAM,QAAQ,KAAK,OAAO,UAAU,EAAE,CAAC;AACvC,UAAM,UAAU,OAAO,kBAAkB,EAAE;AAC3C,SAAK,YACH,WAAW,YAAY,YAAY,UAAW,YAAY;AAE5D,UAAM,iBAAiB;AACvB,SAAK,UAAU,IAAI,iBAAiB,cAAc,eAAe;AACjE,SAAK,QAAQ,KAAK,YAAY,KAAK;AAEnC,SAAK,SAAS,aAAa,wBAAwB,KAAK,MAAM;AAC9D,SAAK,OAAO,QAAQ,KAAK,OAAO;AAIhC,cAAU,aAAa,iBAAiB,gBAAgB,KAAK,cAAc;AAAA,EAC7E;AAAA,EAEA,MAAM,OAAsB;AAC1B,cAAU,aAAa;AAAA,MACrB;AAAA,MACA,KAAK;AAAA,IACP;AACA,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,KAAK,YAAY;AAC9B,WAAK,QAAQ,WAAW;AACxB,WAAK,UAAU;AAAA,IACjB;AACA,QAAI,KAAK,QAAQ;AACf,WAAK,OAAO,WAAW;AACvB,WAAK,SAAS;AAAA,IAChB;AACA,QAAI,KAAK,QAAQ;AACf,iBAAW,KAAK,MAAM;AACtB,WAAK,SAAS;AACd,oBAAc,EAAE,IAAI,KAAK,IAAI,GAAG,UAAU,KAAK,UAAU;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,YAAY;AAAA,EACjC;AAWF;AAMA,eAAe,eAAe,UAAmB;AAC/C,MAAI,CAAC,aAAa,CAAC,YAAa;AAChC,MAAI,YAAY,YAAY,YAAY,aAAa,YAAY,UAAU;AACzE;AAAA,EACF;AACA,QAAM,YAAY,YAAY,KAAK,wBAAwB,KAAK,IAAI;AACpE,MAAI,YAAY,GAAG;AACjB,UAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,SAAS,CAAC;AAAA,EAC/D;AACF;AAMA,eAAsB,YACpB,MAC0B;AAC1B,QAAM,MAAM,MAAM,UAAU,aAAa,iBAAiB;AAC1D,MAAI,IAAI,WAAW,EAAG,QAAO,CAAC;AAI9B,QAAM,WAAW,IAAI,OAAO,CAAC,WAAW,OAAO,aAAa,SAAS;AACrE,QAAM,UAAU,IAAI;AAAA,IAClB,CAAC,WAAW,OAAO,aAAa,aAAa,OAAO,SAAS;AAAA,EAC/D;AACA,aAAW,YAAY,UAAU;AAC/B,UAAM,QAAQ,QAAQ;AAAA,MACpB,CAAC,WAAW,OAAO,YAAY,SAAS;AAAA,IAC1C;AACA,QAAI,QAAQ,EAAG,SAAQ,QAAQ,GAAG,QAAQ,OAAO,OAAO,CAAC,CAAC;AAAA,EAC5D;AAEA,SAAO,QAAQ,IAAI,CAAC,YAAY;AAAA,IAC9B,IAAI,OAAO;AAAA,IACX,OAAO,OAAO,UAAU,SAAS,eAAe,eAAe;AAAA,IAC/D;AAAA,EACF,EAAE;AACJ;;;AMzKA,SAAwB,kBAAkB,qBAAqB;;;ACOxD,IAAM,eAAN,MAAwC;AAAA,EAG7C,cAAc;AACZ,SAAK,SAAS,aAAa,WAAW;AAAA,EACxC;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,SACE,SACA,MACA,SACgB;AAChB,UAAM,SAAS,aAAa;AAAA,MAC1B;AAAA,MACA,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AACA,WAAO,cAAc,SAAsC,CAAC;AAE5D,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,KAAK,MAAM;AAC1B,WAAO,UAAU;AACjB,WAAO,MAAM,IAAI;AAEjB,QAAI,UAAU;AACd,WAAO;AAAA,MACL,OAAO;AACL,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,UAAU;AACjB,YAAI;AACF,iBAAO,KAAK;AAAA,QACd,QAAQ;AAAA,QAER;AACA,eAAO,WAAW;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;;;AD9CA,IAAM,kBAAkB;AAKjB,IAAM,aAAN,cAAyB,cAAc;AAAA,EAAvC;AAAA;AAQL,SAAQ,iBAAiB;AAAA;AAAA,EAEzB,IAAI,YAAqB;AACvB,WAAO,KAAK,QAAQ,aAAa;AAAA,EACnC;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAgC;AAClC,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA,EAEA,MAAM,QAAuB;AAC3B,QAAI,KAAK,OAAQ;AAEjB,SAAK,OAAO,IAAI,aAAa;AAC7B,SAAK,SAAS,IAAI,iBAAiB,KAAK,IAAI;AAC5C,QAAI,UAAU,UAAU,SAAS,SAAS,GAAG;AAC3C,WAAK,OAAO,OAAO,eAAe;AAAA,IACpC;AACA,SAAK,OAAO,GAAG,gBAAgB,MAAM,KAAK,KAAK,cAAc,CAAC;AAC9D,SAAK,OAAO,GAAG,eAAe,MAAM,KAAK,KAAK,aAAa,CAAC;AAC5D,SAAK,OAAO,GAAG,UAAU,CAAC,WAAW,KAAK,KAAK,UAAU,MAAM,CAAC;AAEhE,SAAK,KAAK,OAAO,QAAQ,aAAa,WAAW;AAAA,EACnD;AAAA,EAEA,KAAK,KAAiB,YAA0B;AAC9C,SAAK,QAAQ,UAAU,KAAK,UAAU;AAAA,EACxC;AAAA,EAEA,YAAkB;AAChB,SAAK,QAAQ,UAAU;AAAA,EACzB;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,QAAQ,QAAQ;AACrB,SAAK,SAAS;AACd,SAAK,MAAM,OAAO,WAAW;AAC7B,SAAK,OAAO;AACZ,SAAK,iBAAiB;AACtB,QAAI,KAAK,SAAS;AAChB,WAAK,QAAQ,MAAM;AACnB,WAAK,QAAQ,YAAY;AACzB,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,kBAAoC;AACxC,UAAM,OAAO,KAAK;AAClB,QAAI,CAAC,QAAQ,KAAK,QAAS,QAAO;AAElC,UAAM,cAAc,aAAa,6BAA6B;AAC9D,UAAM,UAAU,IAAI,MAAM;AAC1B,YAAQ,YAAY,YAAY;AAChC,QAAI;AACF,YAAM,QAAQ,KAAK;AAAA,IACrB,SAAS,OAAO;AACd,cAAQ,MAAM,+CAA+C,KAAK;AAClE,cAAQ,YAAY;AACpB,aAAO;AAAA,IACT;AAGA,QAAI,KAAK,SAAS,QAAQ,KAAK,SAAS;AACtC,cAAQ,MAAM;AACd,cAAQ,YAAY;AACpB,aAAO;AAAA,IACT;AACA,SAAK,OAAO,WAAW,aAAa,WAAW;AAC/C,SAAK,OAAO,QAAQ,WAAW;AAC/B,SAAK,UAAU;AACf,SAAK,iBAAiB;AACtB,WAAO;AAAA,EACT;AAAA,EAEA,oBAA0B;AACxB,QAAI,CAAC,KAAK,kBAAkB,CAAC,KAAK,QAAQ,CAAC,KAAK,QAAS;AACzD,SAAK,iBAAiB;AACtB,SAAK,QAAQ,MAAM;AACnB,SAAK,QAAQ,YAAY;AACzB,SAAK,UAAU;AACf,SAAK,KAAK,OAAO,WAAW;AAC5B,SAAK,KAAK,OAAO,QAAQ,aAAa,WAAW;AAAA,EACnD;AAAA,EAEA,MAAM,aAAa,UAAiC;AAClD,UAAM,OAAO,KAAK;AAClB,QAAI,CAAC,MAAM;AACT,WAAK,YAAY;AACjB;AAAA,IACF;AAEA,UAAM,UAAU,MAAM,KAAK,WAAW;AACtC,UAAM,YAAY,QAAQ,WAAW,KAAK,QAAQ,CAAC,EAAE,OAAO;AAE5D,QAAI,CAAC,KAAK,mBAAmB,aAAa,eAAe,eAAe;AAGtE,YAAO,aAAqB,YAAY,YAAY,KAAK,QAAQ;AACjE,WAAK,OAAO,WAAW;AACvB,WAAK,OAAO,QAAQ,aAAa,WAAW;AAE5C,UAAI,KAAK,SAAS;AAChB,aAAK,QAAQ,MAAM;AACnB,aAAK,QAAQ,YAAY;AACzB,aAAK,UAAU;AAAA,MACjB;AAAA,IACF,OAAO;AAGL,UAAI,CAAC,KAAK,SAAS;AACjB,cAAM,cAAc,aAAa,6BAA6B;AAC9D,aAAK,OAAO,WAAW;AACvB,aAAK,OAAO,QAAQ,WAAW;AAE/B,aAAK,UAAU,IAAI,MAAM;AACzB,aAAK,QAAQ,WAAW;AACxB,aAAK,QAAQ,YAAY,YAAY;AAAA,MACvC;AACA,YAAM,KAAK,QAAQ,YAAY,QAAQ;AAAA,IACzC;AAEA,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,aAAa;AAAA,EAClC;AACF;;;AP9IA,cAAc;AALP,SAAS,mBAA0C;AACxD,QAAM,SAAS,QAAQ,UAAU;AACjC,SAAO,kBAAkB,aAAa,OAAO,SAAS;AACxD;AAaO,SAAS,mBAAmB;AACjC,MAAI,UAAU,IAAI,OAAO,CAAC;AAC1B,UAAQ,UAAU,IAAI,WAAW,CAAC;AAClC,oBAAkB,YAAY;AAChC;AAEA,iBAAiB;","names":["audioContext","unlock"]}
|
|
1
|
+
{"version":3,"sources":["../src/index.ts","../src/audio/WebMic.ts","../src/audio/unlock.ts","../src/audio/audioContext.ts","raw:./pcm-processor-worklet","../src/audio/pcm-processor.ts","../src/audio/stopStream.ts","../src/audio/WebSpeaker.ts","../src/audio/WebAudioSink.ts"],"sourcesContent":["import { Mic, setMicdropStorage, Speaker } from '@micdrop/client'\nimport { WebMic } from './audio/WebMic'\nimport { WebSpeaker } from './audio/WebSpeaker'\n\n/**\n * The node the assistant voice comes out of, to analyse or process it.\n *\n * Connect an AnalyserNode to it to read the envelope on your own frames, which\n * is finer than the ten levels a second the `Volume` event reports.\n * @returns The node, once the call has started\n */\nexport function getSpeakerOutput(): AudioNode | undefined {\n const driver = Speaker.getDriver()\n return driver instanceof WebSpeaker ? driver.output : undefined\n}\n\nexport * from '@micdrop/client'\nexport * from './audio/audioContext'\nexport * from './audio/WebAudioSink'\nexport * from './audio/WebMic'\nexport * from './audio/WebSpeaker'\n\n/**\n * Records and plays with the Web Audio API, and keeps settings in\n * `localStorage`. Called when the package is imported, so a page has nothing\n * to wire.\n */\nexport function registerWebAudio() {\n Mic.setDriver(new WebMic())\n Speaker.setDriver(new WebSpeaker())\n setMicdropStorage(localStorage)\n}\n\nregisterWebAudio()\n","import { MicDriver, MicdropDevice } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { initPcmProcessor } from './pcm-processor'\nimport { stopStream } from './stopStream'\n\n/**\n * How long Firefox needs to let go of a microphone it released.\n *\n * A microphone requested again before that comes back without its processing:\n * no echo cancellation, no noise suppression and a raw gain some 30 dB higher,\n * while its settings still report them on. The room then sounds like speech,\n * and the assistant hears its own voice. No event tells when the device is\n * free, so a new stream waits for this delay to pass.\n */\nconst FIREFOX_RELEASE_DELAY = 15000 // ms\n\nconst isFirefox =\n typeof navigator !== 'undefined' && /Firefox\\//.test(navigator.userAgent)\n\n// The last stream released, outside the class: the browser holds the device\n// whichever WebMic opened it\nlet lastRelease: { at: number; deviceId: string | undefined } | undefined\n\ntype MicState =\n | { status: 'stopped' }\n | { status: 'starting'; requested: string | undefined }\n | {\n status: 'started'\n /** The device asked for, undefined for the system one */\n requested: string | undefined\n stream: MediaStream\n source: MediaStreamAudioSourceNode\n worklet: AudioWorkletNode\n }\n\n/**\n * Records the microphone with the Web Audio API.\n *\n * An audio worklet delivers the samples at a steady pace, and everything that\n * happens to them afterwards lives in `@micdrop/client`.\n *\n * Starts and stops take turns, so two starts never hold two streams. A stop or\n * a start for another device aborts the start under way, which lets go of\n * whatever it obtained, and the last call wins.\n */\nexport class WebMic extends MicDriver {\n private state: MicState = { status: 'stopped' }\n // The device the browser settled on last, kept after a stop\n private _deviceId: string | undefined\n // The start under way or waiting for its turn, which the next call aborts\n private attempt:\n | {\n requested: string | undefined\n controller: AbortController\n done: Promise<void>\n }\n | undefined\n private queue: Promise<void> = Promise.resolve()\n\n get isStarted(): boolean {\n return this.state.status === 'started'\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n // Asking for echo cancellation is not getting it: some browsers and devices\n // do without, and the settings of the track say so\n get echoCancellation(): boolean | undefined {\n if (this.state.status !== 'started') return undefined\n const track = this.state.stream.getAudioTracks()[0]\n const setting = track?.getSettings?.().echoCancellation as unknown\n if (setting === undefined) return undefined\n return setting !== false\n }\n\n start(deviceId?: string): Promise<void> {\n // The same request is already on its way\n if (this.attempt && this.attempt.requested === deviceId) {\n return this.attempt.done\n }\n\n this.attempt?.controller.abort()\n const controller = new AbortController()\n const done = this.enqueue(() => this.open(deviceId, controller.signal))\n const attempt = { requested: deviceId, controller, done }\n this.attempt = attempt\n const clear = () => {\n if (this.attempt === attempt) this.attempt = undefined\n }\n done.then(clear, clear)\n return done\n }\n\n async stop(): Promise<void> {\n this.attempt?.controller.abort()\n this.attempt = undefined\n // Resolves once a start under way has let go of its stream too\n await this.enqueue(async () => this.close())\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audioinput')\n }\n\n private enqueue(operation: () => Promise<void>): Promise<void> {\n const done = this.queue.then(operation)\n this.queue = done.catch(() => undefined)\n return done\n }\n\n private async open(requested: string | undefined, signal: AbortSignal) {\n if (signal.aborted) return\n\n const state = this.state\n if (state.status === 'started') {\n // Same device, keep recording. A device asked for by its real id is the\n // one the browser settled on when the system one was asked for.\n if (requested === state.requested || requested === this._deviceId) return\n this.close()\n }\n\n this.state = { status: 'starting', requested }\n let stream: MediaStream | undefined\n let settled: string | undefined\n try {\n await waitForRelease(requested, signal)\n if (signal.aborted) return\n\n stream = await navigator.mediaDevices.getUserMedia({\n audio: {\n deviceId: { ideal: requested },\n sampleRate: 16000, // not working, it will follow device settings, usually 44.1kHz\n sampleSize: 16,\n channelCount: 1,\n echoCancellation: true,\n noiseSuppression: true,\n },\n })\n\n // Remember which device the browser settled on\n const track = stream.getTracks()[0]\n const capability = track?.getCapabilities?.().deviceId\n settled =\n capability && capability !== 'default'\n ? capability\n : (requested ?? capability)\n if (signal.aborted) return\n\n await initPcmProcessor()\n if (signal.aborted) return\n\n const worklet = new AudioWorkletNode(audioContext, 'pcm-processor')\n const source = audioContext.createMediaStreamSource(stream)\n worklet.port.onmessage = this.onWorkletMessage\n source.connect(worklet)\n\n // A browser only reveals the device labels once a stream is running, and\n // Firefox lists nothing at all before that\n navigator.mediaDevices.addEventListener(\n 'devicechange',\n this.onDeviceChange\n )\n\n this._deviceId = settled\n this.state = { status: 'started', requested, stream, source, worklet }\n stream = undefined\n } finally {\n // Aborted or failed: nothing obtained on the way stays open\n if (stream) releaseStream(stream, settled)\n if (this.state.status === 'starting') this.state = { status: 'stopped' }\n }\n }\n\n /** Tears the running microphone down, if there is one */\n private close() {\n const state = this.state\n if (state.status !== 'started') return\n this.state = { status: 'stopped' }\n\n navigator.mediaDevices.removeEventListener(\n 'devicechange',\n this.onDeviceChange\n )\n state.worklet.port.onmessage = null\n state.worklet.disconnect()\n state.source.disconnect()\n releaseStream(state.stream, this._deviceId)\n }\n\n private onWorkletMessage = (event: MessageEvent) => {\n const { type, frames, sampleRate } = event.data\n if (type !== 'frames') return\n this.emit('Frames', frames as Float32Array, sampleRate as number)\n }\n\n private onDeviceChange = () => {\n this.emit('DeviceChange')\n }\n}\n\n/**\n * Stops a stream, and notes when its device was let go\n * @param stream - The stream to stop\n * @param deviceId - The device it recorded from\n */\nfunction releaseStream(stream: MediaStream, deviceId: string | undefined) {\n stopStream(stream)\n lastRelease = { at: Date.now(), deviceId }\n}\n\n/**\n * Waits until Firefox let go of the microphone, see FIREFOX_RELEASE_DELAY\n * @param deviceId - The device about to be opened\n * @param signal - Ends the wait early, when the start is aborted\n */\nasync function waitForRelease(\n deviceId: string | undefined,\n signal: AbortSignal\n) {\n if (!isFirefox || !lastRelease) return\n if (deviceId && lastRelease.deviceId && deviceId !== lastRelease.deviceId) {\n return\n }\n const remaining = lastRelease.at + FIREFOX_RELEASE_DELAY - Date.now()\n if (remaining <= 0 || signal.aborted) return\n await new Promise<void>((resolve) => {\n const done = () => {\n clearTimeout(timer)\n signal.removeEventListener('abort', done)\n resolve()\n }\n const timer = setTimeout(done, remaining)\n signal.addEventListener('abort', done)\n })\n}\n\n/**\n * Lists the devices of one kind, the default one first\n * @param kind - Which devices to list\n */\nexport async function listDevices(\n kind: 'audioinput' | 'audiooutput'\n): Promise<MicdropDevice[]> {\n const all = await navigator.mediaDevices.enumerateDevices()\n if (all.length === 0) return []\n\n // A browser lists the default device twice, once under the id \"default\" and\n // once under its real id. Keep the real one, and put it first.\n const defaults = all.filter((device) => device.deviceId === 'default')\n const devices = all.filter(\n (device) => device.deviceId !== 'default' && device.kind === kind\n )\n for (const fallback of defaults) {\n const index = devices.findIndex(\n (device) => device.groupId === fallback.groupId\n )\n if (index > 0) devices.unshift(...devices.splice(index, 1))\n }\n\n return devices.map((device) => ({\n id: device.deviceId,\n label: device.label || (kind === 'audioinput' ? 'Microphone' : 'Speaker'),\n kind,\n }))\n}\n","// Inspired https://www.mattmontag.com/web/unlock-web-audio-in-safari-for-ios-and-macos\nexport function unlock(audioContext: AudioContext): void {\n const events = ['touchstart', 'touchend', 'mousedown', 'keydown']\n\n const unlock = async () => {\n unlockSafari(audioContext)\n if (audioContext.state === 'suspended') {\n await audioContext.resume()\n }\n // Clean events\n events.forEach((e) => document.body.removeEventListener(e, unlock))\n }\n\n // Add events\n events.forEach((e) => document.body.addEventListener(e, unlock, false))\n}\n\n// safari/ios hack - inspired by https://codepen.io/kslstn/pen/pagLqL\n// unlock audioContext - call when interactive (button click)\nfunction unlockSafari(audioContext: AudioContext): void {\n // create empty buffer and play it to unlock audioContext when interactive\n const source = audioContext.createBufferSource()\n source.buffer = audioContext.createBuffer(1, 1, 22050)\n source.connect(audioContext.destination)\n const onEnded = () => {\n source.disconnect()\n source.buffer = null\n source.removeEventListener('ended', onEnded)\n }\n source.addEventListener('ended', onEnded)\n source.start()\n}\n","import { unlock } from './unlock'\n\nexport let audioContext: AudioContext\n\nexport function createAudioContext() {\n return new (window.AudioContext || (window as any).webkitAudioContext)()\n}\n\ndeclare global {\n interface Window {\n micdropAudioContext: AudioContext\n }\n}\n\nif (window.micdropAudioContext) {\n audioContext = window.micdropAudioContext\n} else {\n window.micdropAudioContext = audioContext = createAudioContext()\n\n // Hack to unlock audio context on some browsers\n unlock(audioContext)\n}\n","// Hands the microphone signal to the main thread, continuously.\n//\n// Deciding whether someone is speaking, keeping a reserve of audio and cutting\n// it into chunks all happen in @micdrop/client, so this only has to deliver the\n// samples at a steady pace.\n\nconst INTERVAL_MS = 20\n\nclass PCMProcessorWorklet extends AudioWorkletProcessor {\n constructor() {\n super()\n this.buffers = []\n this.length = 0\n // `sampleRate` is provided by the worklet scope\n this.frameLength = Math.round((sampleRate * INTERVAL_MS) / 1000)\n }\n\n process(inputs) {\n const input = inputs[0]\n if (!input || input.length === 0) return true\n\n this.buffers.push(new Float32Array(input[0]))\n this.length += input[0].length\n if (this.length < this.frameLength) return true\n\n const frames = new Float32Array(this.length)\n let offset = 0\n for (const buffer of this.buffers) {\n frames.set(buffer, offset)\n offset += buffer.length\n }\n this.buffers.length = 0\n this.length = 0\n\n this.port.postMessage({ type: 'frames', frames, sampleRate }, [\n frames.buffer,\n ])\n return true\n }\n}\n\nregisterProcessor('pcm-processor', PCMProcessorWorklet)\n","import pcmProcessorWorklet from './pcm-processor-worklet?raw'\nimport { audioContext } from './audioContext'\n\ndeclare global {\n interface Window {\n micdropPcmProcessor?: Promise<void>\n }\n}\n\n/**\n * Loads the worklet into the audio context, once per page. A second caller\n * waits for the same load rather than building a node before it exists, and a\n * failed load is forgotten so the next microphone tries again.\n */\nexport function initPcmProcessor(): Promise<void> {\n if (!window.micdropPcmProcessor) {\n const workletBlob = new Blob([pcmProcessorWorklet], {\n type: 'application/javascript',\n })\n const workletUrl = URL.createObjectURL(workletBlob)\n const loading = audioContext.audioWorklet.addModule(workletUrl)\n window.micdropPcmProcessor = loading\n loading.catch(() => {\n if (window.micdropPcmProcessor === loading) {\n window.micdropPcmProcessor = undefined\n }\n })\n }\n return window.micdropPcmProcessor\n}\n","/**\n * Stops a stream\n * @param stream - The stream to stop\n */\nexport function stopStream(stream?: MediaStream) {\n stream?.getTracks().forEach((track) => track.stop())\n}\n","import { MicdropDevice, Pcm16AudioStream, SpeakerDriver } from '@micdrop/client'\nimport { audioContext } from './audioContext'\nimport { listDevices } from './WebMic'\nimport { WebAudioSink } from './WebAudioSink'\n\n// A Bluetooth headset switches to its call profile as its microphone opens,\n// and Firefox mangles what it plays through that switch: the first answer of a\n// call comes out all at once. The speaker starts right after the microphone, so\n// it plays silence that long and holds the first answer until then. The\n// microphone opens again at each call, while the speaker stays started, so the\n// warm-up comes back with every start.\nconst FIREFOX_WARM_UP = 2000 // ms\nconst IS_FIREFOX =\n typeof navigator !== 'undefined' && navigator.userAgent.includes('Firefox')\n\n/**\n * Where the sound goes. The audio context is the preferred way, it is the only\n * one that keeps echo cancellation working. Firefox cannot pick an output on\n * an audio context, so a non default device is played back through an audio\n * element instead. iOS reports echo cancellation on and still hears an audio\n * context, but not an element, which is the echo fallback.\n */\ntype SpeakerOutput =\n | { kind: 'context' }\n | {\n kind: 'element'\n reason: 'device' | 'echoFallback'\n element: HTMLAudioElement\n destination: MediaStreamAudioDestinationNode\n }\n\ntype SpeakerState =\n | { status: 'stopped' }\n | {\n status: 'started'\n sink: WebAudioSink\n stream: Pcm16AudioStream\n output: SpeakerOutput\n }\n\n/**\n * Plays the assistant voice with the Web Audio API.\n *\n * The output device is remembered across calls: one chosen before `start()`\n * applies as soon as it starts, and the echo fallback plays through it too.\n */\nexport class WebSpeaker extends SpeakerDriver {\n private state: SpeakerState = { status: 'stopped' }\n // The output asked for last, undefined for the system one\n private _deviceId: string | undefined\n // Device changes take turns, so the last one asked for is the one heard\n private routing: Promise<void> = Promise.resolve()\n\n get isPlaying(): boolean {\n return this.state.status === 'started' && this.state.stream.isPlaying\n }\n\n get deviceId(): string | undefined {\n return this._deviceId\n }\n\n /** The node the sound comes out of, to analyse or process it */\n get output(): AudioNode | undefined {\n return this.state.status === 'started' ? this.state.sink.output : undefined\n }\n\n /**\n * Prepares the audio graph, once, and plays through the switch of a headset\n * whose microphone just opened, at every call\n */\n async start(): Promise<void> {\n if (this.state.status === 'started') {\n if (IS_FIREFOX) this.state.stream.warmUp(FIREFOX_WARM_UP)\n return\n }\n\n const sink = new WebAudioSink()\n const stream = new Pcm16AudioStream(sink)\n if (IS_FIREFOX) stream.warmUp(FIREFOX_WARM_UP)\n stream.on('StartPlaying', () => this.emit('StartPlaying'))\n stream.on('StopPlaying', () => this.emit('StopPlaying'))\n stream.on('Volume', (volume) => this.emit('Volume', volume))\n\n sink.output.connect(audioContext.destination)\n this.state = {\n status: 'started',\n sink,\n stream,\n output: { kind: 'context' },\n }\n\n // Play through the output chosen while stopped, or during the last call\n const deviceId = this._deviceId\n if (deviceId) {\n try {\n await this.route(deviceId)\n } catch (error) {\n console.error(\n '[Micdrop] Playing through the chosen output failed',\n error\n )\n // Forgotten, so the caller sees the system output in use\n if (this._deviceId === deviceId) this._deviceId = undefined\n }\n }\n }\n\n play(pcm: Int16Array, sampleRate: number): void {\n if (this.state.status !== 'started') return\n this.state.stream.playAudio(pcm, sampleRate)\n }\n\n stopAudio(): void {\n if (this.state.status !== 'started') return\n this.state.stream.stopAudio()\n }\n\n async stop(): Promise<void> {\n const state = this.state\n if (state.status !== 'started') return\n this.state = { status: 'stopped' }\n state.stream.destroy()\n state.sink.output.disconnect()\n closeOutput(state.output)\n }\n\n /**\n * Moves the sound to an audio element, while it plays. The audio context\n * keeps playing until the element is known to be allowed to.\n */\n async useEchoFallback(): Promise<boolean> {\n const state = this.state\n if (state.status !== 'started' || state.output.kind !== 'context') {\n return false\n }\n\n const output = createElementOutput('echoFallback')\n output.element.srcObject = output.destination.stream\n try {\n await output.element.play()\n // Through the device chosen, rather than the system one\n if (this._deviceId) await output.element.setSinkId?.(this._deviceId)\n } catch (error) {\n console.error('[Micdrop] Playing through an element failed', error)\n closeOutput(output)\n return false\n }\n\n // The call ended, or the output changed, in the meantime\n if (this.state !== state || state.output.kind !== 'context') {\n closeOutput(output)\n return false\n }\n setOutput(state, output)\n return true\n }\n\n resetEchoFallback(): void {\n const state = this.state\n if (\n state.status !== 'started' ||\n state.output.kind !== 'element' ||\n state.output.reason !== 'echoFallback'\n ) {\n return\n }\n setOutput(state, { kind: 'context' })\n\n // Firefox needs an element of its own to play through a chosen device\n const deviceId = this._deviceId\n if (deviceId) {\n this.route(deviceId).catch((error) =>\n console.error(\n '[Micdrop] Playing through the chosen output failed',\n error\n )\n )\n }\n }\n\n async changeDevice(deviceId: string): Promise<void> {\n await this.route(deviceId)\n this._deviceId = deviceId\n }\n\n async getDevices(): Promise<MicdropDevice[]> {\n return listDevices('audiooutput')\n }\n\n /**\n * Plays through a device, after the changes asked for before\n * @param deviceId - The output device\n */\n private route(deviceId: string): Promise<void> {\n const done = this.routing.then(() => this.applyDevice(deviceId))\n this.routing = done.catch(() => undefined)\n return done\n }\n\n private async applyDevice(deviceId: string): Promise<void> {\n // Stopped: start() applies it\n const state = this.state\n if (state.status !== 'started') return\n\n const devices = await this.getDevices()\n const isDefault = devices.length === 0 || devices[0].id === deviceId\n // Stopped, or started again, while the devices were listed\n if (this.state !== state) return\n\n const output = state.output\n if (output.kind === 'element' && output.reason === 'echoFallback') {\n // The element stays, it only plays through the new device\n await output.element.setSinkId?.(isDefault ? '' : deviceId)\n return\n }\n\n if (isDefault || 'setSinkId' in audioContext) {\n await (audioContext as any).setSinkId?.(isDefault ? '' : deviceId)\n if (this.state !== state) return\n // The echo fallback took over meanwhile, and keeps the sound\n const current = state.output\n if (current.kind === 'element' && current.reason === 'echoFallback') {\n await current.element.setSinkId?.(isDefault ? '' : deviceId)\n return\n }\n setOutput(state, { kind: 'context' })\n return\n }\n\n // Firefox: stream the sound back into an audio element, which does accept\n // an output device\n let element = output.kind === 'element' ? output.element : undefined\n if (!element) {\n const next = createElementOutput('device')\n next.element.autoplay = true\n next.element.srcObject = next.destination.stream\n setOutput(state, next)\n element = next.element\n }\n await element.setSinkId?.(deviceId)\n }\n}\n\n/**\n * Sends the sound of a running speaker somewhere else\n * @param state - The running speaker\n * @param output - Where the sound goes from now on\n */\nfunction setOutput(\n state: Extract<SpeakerState, { status: 'started' }>,\n output: SpeakerOutput\n) {\n const previous = state.output\n if (previous === output) return\n if (previous.kind === 'context' && output.kind === 'context') return\n // Only this connection, an analyser plugged on the output stays plugged\n state.sink.output.disconnect(destinationOf(previous))\n state.sink.output.connect(destinationOf(output))\n state.output = output\n closeOutput(previous)\n}\n\nfunction destinationOf(output: SpeakerOutput): AudioNode {\n return output.kind === 'context'\n ? audioContext.destination\n : output.destination\n}\n\n/** An audio element fed from the audio context, not playing yet */\nfunction createElementOutput(\n reason: 'device' | 'echoFallback'\n): Extract<SpeakerOutput, { kind: 'element' }> {\n return {\n kind: 'element',\n reason,\n element: new Audio(),\n destination: audioContext.createMediaStreamDestination(),\n }\n}\n\n/** Silences and frees an output, once nothing plays through it any more */\nfunction closeOutput(output: SpeakerOutput) {\n if (output.kind !== 'element') return\n output.element.pause()\n output.element.srcObject = null\n}\n","import { AudioSink, ScheduledAudio } from '@micdrop/client'\nimport { audioContext } from './audioContext'\n\n/**\n * Plays samples through the Web Audio graph, on the timeline given by the\n * caller so consecutive chunks are heard without a gap.\n */\nexport class WebAudioSink implements AudioSink {\n public readonly output: GainNode\n\n constructor() {\n this.output = audioContext.createGain()\n }\n\n get sampleRate(): number {\n return audioContext.sampleRate\n }\n\n get currentTime(): number {\n return audioContext.currentTime\n }\n\n schedule(\n samples: Float32Array,\n when: number,\n onEnded: () => void\n ): ScheduledAudio {\n const buffer = audioContext.createBuffer(\n 1,\n samples.length,\n audioContext.sampleRate\n )\n buffer.copyToChannel(samples as Float32Array<ArrayBuffer>, 0)\n\n const source = audioContext.createBufferSource()\n source.buffer = buffer\n source.connect(this.output)\n source.onended = onEnded\n source.start(when)\n\n let stopped = false\n return {\n stop() {\n if (stopped) return\n stopped = true\n source.onended = null\n try {\n source.stop()\n } catch {\n // Already finished\n }\n source.disconnect()\n },\n }\n }\n}\n"],"mappings":";AAAA,SAAS,KAAK,mBAAmB,eAAe;;;ACAhD,SAAS,iBAAgC;;;ACClC,SAAS,OAAOA,eAAkC;AACvD,QAAM,SAAS,CAAC,cAAc,YAAY,aAAa,SAAS;AAEhE,QAAMC,UAAS,YAAY;AACzB,iBAAaD,aAAY;AACzB,QAAIA,cAAa,UAAU,aAAa;AACtC,YAAMA,cAAa,OAAO;AAAA,IAC5B;AAEA,WAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,oBAAoB,GAAGC,OAAM,CAAC;AAAA,EACpE;AAGA,SAAO,QAAQ,CAAC,MAAM,SAAS,KAAK,iBAAiB,GAAGA,SAAQ,KAAK,CAAC;AACxE;AAIA,SAAS,aAAaD,eAAkC;AAEtD,QAAM,SAASA,cAAa,mBAAmB;AAC/C,SAAO,SAASA,cAAa,aAAa,GAAG,GAAG,KAAK;AACrD,SAAO,QAAQA,cAAa,WAAW;AACvC,QAAM,UAAU,MAAM;AACpB,WAAO,WAAW;AAClB,WAAO,SAAS;AAChB,WAAO,oBAAoB,SAAS,OAAO;AAAA,EAC7C;AACA,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO,MAAM;AACf;;;AC7BO,IAAI;AAEJ,SAAS,qBAAqB;AACnC,SAAO,KAAK,OAAO,gBAAiB,OAAe,oBAAoB;AACzE;AAQA,IAAI,OAAO,qBAAqB;AAC9B,iBAAe,OAAO;AACxB,OAAO;AACL,SAAO,sBAAsB,eAAe,mBAAmB;AAG/D,SAAO,YAAY;AACrB;;;ACrBA;;;ACcO,SAAS,mBAAkC;AAChD,MAAI,CAAC,OAAO,qBAAqB;AAC/B,UAAM,cAAc,IAAI,KAAK,CAAC,6BAAmB,GAAG;AAAA,MAClD,MAAM;AAAA,IACR,CAAC;AACD,UAAM,aAAa,IAAI,gBAAgB,WAAW;AAClD,UAAM,UAAU,aAAa,aAAa,UAAU,UAAU;AAC9D,WAAO,sBAAsB;AAC7B,YAAQ,MAAM,MAAM;AAClB,UAAI,OAAO,wBAAwB,SAAS;AAC1C,eAAO,sBAAsB;AAAA,MAC/B;AAAA,IACF,CAAC;AAAA,EACH;AACA,SAAO,OAAO;AAChB;;;ACzBO,SAAS,WAAW,QAAsB;AAC/C,UAAQ,UAAU,EAAE,QAAQ,CAAC,UAAU,MAAM,KAAK,CAAC;AACrD;;;ALQA,IAAM,wBAAwB;AAE9B,IAAM,YACJ,OAAO,cAAc,eAAe,YAAY,KAAK,UAAU,SAAS;AAI1E,IAAI;AAwBG,IAAM,SAAN,cAAqB,UAAU;AAAA,EAA/B;AAAA;AACL,SAAQ,QAAkB,EAAE,QAAQ,UAAU;AAW9C,SAAQ,QAAuB,QAAQ,QAAQ;AAsI/C,SAAQ,mBAAmB,CAAC,UAAwB;AAClD,YAAM,EAAE,MAAM,QAAQ,WAAW,IAAI,MAAM;AAC3C,UAAI,SAAS,SAAU;AACvB,WAAK,KAAK,UAAU,QAAwB,UAAoB;AAAA,IAClE;AAEA,SAAQ,iBAAiB,MAAM;AAC7B,WAAK,KAAK,cAAc;AAAA,IAC1B;AAAA;AAAA,EA5IA,IAAI,YAAqB;AACvB,WAAO,KAAK,MAAM,WAAW;AAAA,EAC/B;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA,EAIA,IAAI,mBAAwC;AAC1C,QAAI,KAAK,MAAM,WAAW,UAAW,QAAO;AAC5C,UAAM,QAAQ,KAAK,MAAM,OAAO,eAAe,EAAE,CAAC;AAClD,UAAM,UAAU,OAAO,cAAc,EAAE;AACvC,QAAI,YAAY,OAAW,QAAO;AAClC,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAM,UAAkC;AAEtC,QAAI,KAAK,WAAW,KAAK,QAAQ,cAAc,UAAU;AACvD,aAAO,KAAK,QAAQ;AAAA,IACtB;AAEA,SAAK,SAAS,WAAW,MAAM;AAC/B,UAAM,aAAa,IAAI,gBAAgB;AACvC,UAAM,OAAO,KAAK,QAAQ,MAAM,KAAK,KAAK,UAAU,WAAW,MAAM,CAAC;AACtE,UAAM,UAAU,EAAE,WAAW,UAAU,YAAY,KAAK;AACxD,SAAK,UAAU;AACf,UAAM,QAAQ,MAAM;AAClB,UAAI,KAAK,YAAY,QAAS,MAAK,UAAU;AAAA,IAC/C;AACA,SAAK,KAAK,OAAO,KAAK;AACtB,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAsB;AAC1B,SAAK,SAAS,WAAW,MAAM;AAC/B,SAAK,UAAU;AAEf,UAAM,KAAK,QAAQ,YAAY,KAAK,MAAM,CAAC;AAAA,EAC7C;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,YAAY;AAAA,EACjC;AAAA,EAEQ,QAAQ,WAA+C;AAC7D,UAAM,OAAO,KAAK,MAAM,KAAK,SAAS;AACtC,SAAK,QAAQ,KAAK,MAAM,MAAM,MAAS;AACvC,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,KAAK,WAA+B,QAAqB;AACrE,QAAI,OAAO,QAAS;AAEpB,UAAM,QAAQ,KAAK;AACnB,QAAI,MAAM,WAAW,WAAW;AAG9B,UAAI,cAAc,MAAM,aAAa,cAAc,KAAK,UAAW;AACnE,WAAK,MAAM;AAAA,IACb;AAEA,SAAK,QAAQ,EAAE,QAAQ,YAAY,UAAU;AAC7C,QAAI;AACJ,QAAI;AACJ,QAAI;AACF,YAAM,eAAe,WAAW,MAAM;AACtC,UAAI,OAAO,QAAS;AAEpB,eAAS,MAAM,UAAU,aAAa,aAAa;AAAA,QACjD,OAAO;AAAA,UACL,UAAU,EAAE,OAAO,UAAU;AAAA,UAC7B,YAAY;AAAA;AAAA,UACZ,YAAY;AAAA,UACZ,cAAc;AAAA,UACd,kBAAkB;AAAA,UAClB,kBAAkB;AAAA,QACpB;AAAA,MACF,CAAC;AAGD,YAAM,QAAQ,OAAO,UAAU,EAAE,CAAC;AAClC,YAAM,aAAa,OAAO,kBAAkB,EAAE;AAC9C,gBACE,cAAc,eAAe,YACzB,aACC,aAAa;AACpB,UAAI,OAAO,QAAS;AAEpB,YAAM,iBAAiB;AACvB,UAAI,OAAO,QAAS;AAEpB,YAAM,UAAU,IAAI,iBAAiB,cAAc,eAAe;AAClE,YAAM,SAAS,aAAa,wBAAwB,MAAM;AAC1D,cAAQ,KAAK,YAAY,KAAK;AAC9B,aAAO,QAAQ,OAAO;AAItB,gBAAU,aAAa;AAAA,QACrB;AAAA,QACA,KAAK;AAAA,MACP;AAEA,WAAK,YAAY;AACjB,WAAK,QAAQ,EAAE,QAAQ,WAAW,WAAW,QAAQ,QAAQ,QAAQ;AACrE,eAAS;AAAA,IACX,UAAE;AAEA,UAAI,OAAQ,eAAc,QAAQ,OAAO;AACzC,UAAI,KAAK,MAAM,WAAW,WAAY,MAAK,QAAQ,EAAE,QAAQ,UAAU;AAAA,IACzE;AAAA,EACF;AAAA;AAAA,EAGQ,QAAQ;AACd,UAAM,QAAQ,KAAK;AACnB,QAAI,MAAM,WAAW,UAAW;AAChC,SAAK,QAAQ,EAAE,QAAQ,UAAU;AAEjC,cAAU,aAAa;AAAA,MACrB;AAAA,MACA,KAAK;AAAA,IACP;AACA,UAAM,QAAQ,KAAK,YAAY;AAC/B,UAAM,QAAQ,WAAW;AACzB,UAAM,OAAO,WAAW;AACxB,kBAAc,MAAM,QAAQ,KAAK,SAAS;AAAA,EAC5C;AAWF;AAOA,SAAS,cAAc,QAAqB,UAA8B;AACxE,aAAW,MAAM;AACjB,gBAAc,EAAE,IAAI,KAAK,IAAI,GAAG,SAAS;AAC3C;AAOA,eAAe,eACb,UACA,QACA;AACA,MAAI,CAAC,aAAa,CAAC,YAAa;AAChC,MAAI,YAAY,YAAY,YAAY,aAAa,YAAY,UAAU;AACzE;AAAA,EACF;AACA,QAAM,YAAY,YAAY,KAAK,wBAAwB,KAAK,IAAI;AACpE,MAAI,aAAa,KAAK,OAAO,QAAS;AACtC,QAAM,IAAI,QAAc,CAAC,YAAY;AACnC,UAAM,OAAO,MAAM;AACjB,mBAAa,KAAK;AAClB,aAAO,oBAAoB,SAAS,IAAI;AACxC,cAAQ;AAAA,IACV;AACA,UAAM,QAAQ,WAAW,MAAM,SAAS;AACxC,WAAO,iBAAiB,SAAS,IAAI;AAAA,EACvC,CAAC;AACH;AAMA,eAAsB,YACpB,MAC0B;AAC1B,QAAM,MAAM,MAAM,UAAU,aAAa,iBAAiB;AAC1D,MAAI,IAAI,WAAW,EAAG,QAAO,CAAC;AAI9B,QAAM,WAAW,IAAI,OAAO,CAAC,WAAW,OAAO,aAAa,SAAS;AACrE,QAAM,UAAU,IAAI;AAAA,IAClB,CAAC,WAAW,OAAO,aAAa,aAAa,OAAO,SAAS;AAAA,EAC/D;AACA,aAAW,YAAY,UAAU;AAC/B,UAAM,QAAQ,QAAQ;AAAA,MACpB,CAAC,WAAW,OAAO,YAAY,SAAS;AAAA,IAC1C;AACA,QAAI,QAAQ,EAAG,SAAQ,QAAQ,GAAG,QAAQ,OAAO,OAAO,CAAC,CAAC;AAAA,EAC5D;AAEA,SAAO,QAAQ,IAAI,CAAC,YAAY;AAAA,IAC9B,IAAI,OAAO;AAAA,IACX,OAAO,OAAO,UAAU,SAAS,eAAe,eAAe;AAAA,IAC/D;AAAA,EACF,EAAE;AACJ;;;AM1QA,SAAwB,kBAAkB,qBAAqB;;;ACOxD,IAAM,eAAN,MAAwC;AAAA,EAG7C,cAAc;AACZ,SAAK,SAAS,aAAa,WAAW;AAAA,EACxC;AAAA,EAEA,IAAI,aAAqB;AACvB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,aAAa;AAAA,EACtB;AAAA,EAEA,SACE,SACA,MACA,SACgB;AAChB,UAAM,SAAS,aAAa;AAAA,MAC1B;AAAA,MACA,QAAQ;AAAA,MACR,aAAa;AAAA,IACf;AACA,WAAO,cAAc,SAAsC,CAAC;AAE5D,UAAM,SAAS,aAAa,mBAAmB;AAC/C,WAAO,SAAS;AAChB,WAAO,QAAQ,KAAK,MAAM;AAC1B,WAAO,UAAU;AACjB,WAAO,MAAM,IAAI;AAEjB,QAAI,UAAU;AACd,WAAO;AAAA,MACL,OAAO;AACL,YAAI,QAAS;AACb,kBAAU;AACV,eAAO,UAAU;AACjB,YAAI;AACF,iBAAO,KAAK;AAAA,QACd,QAAQ;AAAA,QAER;AACA,eAAO,WAAW;AAAA,MACpB;AAAA,IACF;AAAA,EACF;AACF;;;AD5CA,IAAM,kBAAkB;AACxB,IAAM,aACJ,OAAO,cAAc,eAAe,UAAU,UAAU,SAAS,SAAS;AAiCrE,IAAM,aAAN,cAAyB,cAAc;AAAA,EAAvC;AAAA;AACL,SAAQ,QAAsB,EAAE,QAAQ,UAAU;AAIlD;AAAA,SAAQ,UAAyB,QAAQ,QAAQ;AAAA;AAAA,EAEjD,IAAI,YAAqB;AACvB,WAAO,KAAK,MAAM,WAAW,aAAa,KAAK,MAAM,OAAO;AAAA,EAC9D;AAAA,EAEA,IAAI,WAA+B;AACjC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,IAAI,SAAgC;AAClC,WAAO,KAAK,MAAM,WAAW,YAAY,KAAK,MAAM,KAAK,SAAS;AAAA,EACpE;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,QAAuB;AAC3B,QAAI,KAAK,MAAM,WAAW,WAAW;AACnC,UAAI,WAAY,MAAK,MAAM,OAAO,OAAO,eAAe;AACxD;AAAA,IACF;AAEA,UAAM,OAAO,IAAI,aAAa;AAC9B,UAAM,SAAS,IAAI,iBAAiB,IAAI;AACxC,QAAI,WAAY,QAAO,OAAO,eAAe;AAC7C,WAAO,GAAG,gBAAgB,MAAM,KAAK,KAAK,cAAc,CAAC;AACzD,WAAO,GAAG,eAAe,MAAM,KAAK,KAAK,aAAa,CAAC;AACvD,WAAO,GAAG,UAAU,CAAC,WAAW,KAAK,KAAK,UAAU,MAAM,CAAC;AAE3D,SAAK,OAAO,QAAQ,aAAa,WAAW;AAC5C,SAAK,QAAQ;AAAA,MACX,QAAQ;AAAA,MACR;AAAA,MACA;AAAA,MACA,QAAQ,EAAE,MAAM,UAAU;AAAA,IAC5B;AAGA,UAAM,WAAW,KAAK;AACtB,QAAI,UAAU;AACZ,UAAI;AACF,cAAM,KAAK,MAAM,QAAQ;AAAA,MAC3B,SAAS,OAAO;AACd,gBAAQ;AAAA,UACN;AAAA,UACA;AAAA,QACF;AAEA,YAAI,KAAK,cAAc,SAAU,MAAK,YAAY;AAAA,MACpD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,KAAK,KAAiB,YAA0B;AAC9C,QAAI,KAAK,MAAM,WAAW,UAAW;AACrC,SAAK,MAAM,OAAO,UAAU,KAAK,UAAU;AAAA,EAC7C;AAAA,EAEA,YAAkB;AAChB,QAAI,KAAK,MAAM,WAAW,UAAW;AACrC,SAAK,MAAM,OAAO,UAAU;AAAA,EAC9B;AAAA,EAEA,MAAM,OAAsB;AAC1B,UAAM,QAAQ,KAAK;AACnB,QAAI,MAAM,WAAW,UAAW;AAChC,SAAK,QAAQ,EAAE,QAAQ,UAAU;AACjC,UAAM,OAAO,QAAQ;AACrB,UAAM,KAAK,OAAO,WAAW;AAC7B,gBAAY,MAAM,MAAM;AAAA,EAC1B;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,kBAAoC;AACxC,UAAM,QAAQ,KAAK;AACnB,QAAI,MAAM,WAAW,aAAa,MAAM,OAAO,SAAS,WAAW;AACjE,aAAO;AAAA,IACT;AAEA,UAAM,SAAS,oBAAoB,cAAc;AACjD,WAAO,QAAQ,YAAY,OAAO,YAAY;AAC9C,QAAI;AACF,YAAM,OAAO,QAAQ,KAAK;AAE1B,UAAI,KAAK,UAAW,OAAM,OAAO,QAAQ,YAAY,KAAK,SAAS;AAAA,IACrE,SAAS,OAAO;AACd,cAAQ,MAAM,+CAA+C,KAAK;AAClE,kBAAY,MAAM;AAClB,aAAO;AAAA,IACT;AAGA,QAAI,KAAK,UAAU,SAAS,MAAM,OAAO,SAAS,WAAW;AAC3D,kBAAY,MAAM;AAClB,aAAO;AAAA,IACT;AACA,cAAU,OAAO,MAAM;AACvB,WAAO;AAAA,EACT;AAAA,EAEA,oBAA0B;AACxB,UAAM,QAAQ,KAAK;AACnB,QACE,MAAM,WAAW,aACjB,MAAM,OAAO,SAAS,aACtB,MAAM,OAAO,WAAW,gBACxB;AACA;AAAA,IACF;AACA,cAAU,OAAO,EAAE,MAAM,UAAU,CAAC;AAGpC,UAAM,WAAW,KAAK;AACtB,QAAI,UAAU;AACZ,WAAK,MAAM,QAAQ,EAAE;AAAA,QAAM,CAAC,UAC1B,QAAQ;AAAA,UACN;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa,UAAiC;AAClD,UAAM,KAAK,MAAM,QAAQ;AACzB,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAAuC;AAC3C,WAAO,YAAY,aAAa;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,MAAM,UAAiC;AAC7C,UAAM,OAAO,KAAK,QAAQ,KAAK,MAAM,KAAK,YAAY,QAAQ,CAAC;AAC/D,SAAK,UAAU,KAAK,MAAM,MAAM,MAAS;AACzC,WAAO;AAAA,EACT;AAAA,EAEA,MAAc,YAAY,UAAiC;AAEzD,UAAM,QAAQ,KAAK;AACnB,QAAI,MAAM,WAAW,UAAW;AAEhC,UAAM,UAAU,MAAM,KAAK,WAAW;AACtC,UAAM,YAAY,QAAQ,WAAW,KAAK,QAAQ,CAAC,EAAE,OAAO;AAE5D,QAAI,KAAK,UAAU,MAAO;AAE1B,UAAM,SAAS,MAAM;AACrB,QAAI,OAAO,SAAS,aAAa,OAAO,WAAW,gBAAgB;AAEjE,YAAM,OAAO,QAAQ,YAAY,YAAY,KAAK,QAAQ;AAC1D;AAAA,IACF;AAEA,QAAI,aAAa,eAAe,cAAc;AAC5C,YAAO,aAAqB,YAAY,YAAY,KAAK,QAAQ;AACjE,UAAI,KAAK,UAAU,MAAO;AAE1B,YAAM,UAAU,MAAM;AACtB,UAAI,QAAQ,SAAS,aAAa,QAAQ,WAAW,gBAAgB;AACnE,cAAM,QAAQ,QAAQ,YAAY,YAAY,KAAK,QAAQ;AAC3D;AAAA,MACF;AACA,gBAAU,OAAO,EAAE,MAAM,UAAU,CAAC;AACpC;AAAA,IACF;AAIA,QAAI,UAAU,OAAO,SAAS,YAAY,OAAO,UAAU;AAC3D,QAAI,CAAC,SAAS;AACZ,YAAM,OAAO,oBAAoB,QAAQ;AACzC,WAAK,QAAQ,WAAW;AACxB,WAAK,QAAQ,YAAY,KAAK,YAAY;AAC1C,gBAAU,OAAO,IAAI;AACrB,gBAAU,KAAK;AAAA,IACjB;AACA,UAAM,QAAQ,YAAY,QAAQ;AAAA,EACpC;AACF;AAOA,SAAS,UACP,OACA,QACA;AACA,QAAM,WAAW,MAAM;AACvB,MAAI,aAAa,OAAQ;AACzB,MAAI,SAAS,SAAS,aAAa,OAAO,SAAS,UAAW;AAE9D,QAAM,KAAK,OAAO,WAAW,cAAc,QAAQ,CAAC;AACpD,QAAM,KAAK,OAAO,QAAQ,cAAc,MAAM,CAAC;AAC/C,QAAM,SAAS;AACf,cAAY,QAAQ;AACtB;AAEA,SAAS,cAAc,QAAkC;AACvD,SAAO,OAAO,SAAS,YACnB,aAAa,cACb,OAAO;AACb;AAGA,SAAS,oBACP,QAC6C;AAC7C,SAAO;AAAA,IACL,MAAM;AAAA,IACN;AAAA,IACA,SAAS,IAAI,MAAM;AAAA,IACnB,aAAa,aAAa,6BAA6B;AAAA,EACzD;AACF;AAGA,SAAS,YAAY,QAAuB;AAC1C,MAAI,OAAO,SAAS,UAAW;AAC/B,SAAO,QAAQ,MAAM;AACrB,SAAO,QAAQ,YAAY;AAC7B;;;AP7QA,cAAc;AALP,SAAS,mBAA0C;AACxD,QAAM,SAAS,QAAQ,UAAU;AACjC,SAAO,kBAAkB,aAAa,OAAO,SAAS;AACxD;AAaO,SAAS,mBAAmB;AACjC,MAAI,UAAU,IAAI,OAAO,CAAC;AAC1B,UAAQ,UAAU,IAAI,WAAW,CAAC;AAClC,oBAAkB,YAAY;AAChC;AAEA,iBAAiB;","names":["audioContext","unlock"]}
|
package/dist/silero.d.mts
CHANGED
|
@@ -16,7 +16,8 @@ interface OnnxSileroOptions {
|
|
|
16
16
|
* Runs the Silero model in the browser, on the ONNX WebAssembly runtime.
|
|
17
17
|
*
|
|
18
18
|
* The state machine that turns its answers into turns lives in
|
|
19
|
-
* `@micdrop/client`, this only scores windows of audio.
|
|
19
|
+
* `@micdrop/client`, this only scores windows of audio. Once released, the
|
|
20
|
+
* model is done with: create a new one to load it again.
|
|
20
21
|
*/
|
|
21
22
|
declare class OnnxSileroModel implements SileroModel {
|
|
22
23
|
private options;
|
|
@@ -25,6 +26,7 @@ declare class OnnxSileroModel implements SileroModel {
|
|
|
25
26
|
private sampleRate;
|
|
26
27
|
constructor(options?: OnnxSileroOptions);
|
|
27
28
|
load(): Promise<void>;
|
|
29
|
+
private createSession;
|
|
28
30
|
process(frame: Float32Array): Promise<number>;
|
|
29
31
|
reset(): void;
|
|
30
32
|
release(): Promise<void>;
|
package/dist/silero.d.ts
CHANGED
|
@@ -16,7 +16,8 @@ interface OnnxSileroOptions {
|
|
|
16
16
|
* Runs the Silero model in the browser, on the ONNX WebAssembly runtime.
|
|
17
17
|
*
|
|
18
18
|
* The state machine that turns its answers into turns lives in
|
|
19
|
-
* `@micdrop/client`, this only scores windows of audio.
|
|
19
|
+
* `@micdrop/client`, this only scores windows of audio. Once released, the
|
|
20
|
+
* model is done with: create a new one to load it again.
|
|
20
21
|
*/
|
|
21
22
|
declare class OnnxSileroModel implements SileroModel {
|
|
22
23
|
private options;
|
|
@@ -25,6 +26,7 @@ declare class OnnxSileroModel implements SileroModel {
|
|
|
25
26
|
private sampleRate;
|
|
26
27
|
constructor(options?: OnnxSileroOptions);
|
|
27
28
|
load(): Promise<void>;
|
|
29
|
+
private createSession;
|
|
28
30
|
process(frame: Float32Array): Promise<number>;
|
|
29
31
|
reset(): void;
|
|
30
32
|
release(): Promise<void>;
|
package/dist/silero.js
CHANGED
|
@@ -55,48 +55,53 @@ function queueOnnxRun(run) {
|
|
|
55
55
|
|
|
56
56
|
// src/vad/OnnxSileroModel.ts
|
|
57
57
|
var DEFAULT_SILERO_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx";
|
|
58
|
-
var DEFAULT_WASM_PATH =
|
|
59
|
-
var
|
|
58
|
+
var DEFAULT_WASM_PATH = `https://unpkg.com/onnxruntime-web@${ort.env.versions.web}/dist/`;
|
|
59
|
+
var fetchModel = (0, import_client.createDownloadCache)(async (url) => {
|
|
60
|
+
const response = await fetch(url);
|
|
61
|
+
if (!response.ok) {
|
|
62
|
+
throw new Error(
|
|
63
|
+
`Could not load the Silero model from ${url} (${response.status})`
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
return response.arrayBuffer();
|
|
67
|
+
});
|
|
60
68
|
var OnnxSileroModel = class {
|
|
61
69
|
constructor(options2 = {}) {
|
|
62
70
|
this.options = options2;
|
|
71
|
+
this.session = new import_client.LazySession("Silero", () => this.createSession());
|
|
63
72
|
this.state = newState();
|
|
64
73
|
this.sampleRate = new ort.Tensor("int64", [BigInt(import_client.SILERO_SAMPLE_RATE)]);
|
|
65
74
|
}
|
|
66
75
|
async load() {
|
|
67
|
-
|
|
76
|
+
await this.session.get();
|
|
77
|
+
}
|
|
78
|
+
async createSession() {
|
|
68
79
|
const wasmPath = this.options.wasmPath ?? DEFAULT_WASM_PATH;
|
|
69
80
|
if (wasmPath !== false) {
|
|
70
81
|
ort.env.wasm.wasmPaths = wasmPath;
|
|
71
82
|
}
|
|
72
83
|
const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL;
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
loading = loading ?? fetch(source).then((response) => response.arrayBuffer());
|
|
76
|
-
bytes = await loading;
|
|
77
|
-
} else {
|
|
78
|
-
bytes = source;
|
|
79
|
-
}
|
|
80
|
-
this.session = await ort.InferenceSession.create(bytes);
|
|
84
|
+
const bytes = typeof source === "string" ? await fetchModel(source) : source;
|
|
85
|
+
return ort.InferenceSession.create(bytes);
|
|
81
86
|
}
|
|
82
87
|
async process(frame) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
88
|
+
return this.session.use(
|
|
89
|
+
(session) => queueOnnxRun(async () => {
|
|
90
|
+
const output = await session.run({
|
|
91
|
+
input: new ort.Tensor("float32", frame, [1, frame.length]),
|
|
92
|
+
state: this.state,
|
|
93
|
+
sr: this.sampleRate
|
|
94
|
+
});
|
|
95
|
+
this.state = output.stateN;
|
|
96
|
+
return output.output.data[0];
|
|
89
97
|
})
|
|
90
98
|
);
|
|
91
|
-
this.state = output.stateN;
|
|
92
|
-
return output.output.data[0];
|
|
93
99
|
}
|
|
94
100
|
reset() {
|
|
95
101
|
this.state = newState();
|
|
96
102
|
}
|
|
97
103
|
async release() {
|
|
98
|
-
await this.session
|
|
99
|
-
this.session = void 0;
|
|
104
|
+
await this.session.release();
|
|
100
105
|
}
|
|
101
106
|
};
|
|
102
107
|
function newState() {
|
package/dist/silero.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/silero.ts","../src/vad/OnnxSileroModel.ts","../src/vad/onnxQueue.ts"],"sourcesContent":["import { setSileroModelLoader } from '@micdrop/client'\nimport { OnnxSileroModel, OnnxSileroOptions } from './vad/OnnxSileroModel'\n\nexport * from './vad/OnnxSileroModel'\nexport * from './vad/onnxQueue'\n\nlet options: OnnxSileroOptions = {}\n\n/**\n * Says where the Silero model comes from, to serve it from your own domain\n * rather than from a CDN, or to hand over its bytes directly. Merged into the\n * options already set, so it keeps what the `@micdrop/web/vite` plugin chose.\n * @param next - Address or bytes of the model, and of the ONNX runtime\n */\nexport function setSileroOptions(next: OnnxSileroOptions) {\n options = { ...options, ...next }\n}\n\n// The ONNX runtime weighs more than the rest of Micdrop put together, so it\n// only enters the bundle of an app that imports this file\nsetSileroModelLoader(async () => {\n const model = new OnnxSileroModel(options)\n await model.load()\n return model\n})\n","import {
|
|
1
|
+
{"version":3,"sources":["../src/silero.ts","../src/vad/OnnxSileroModel.ts","../src/vad/onnxQueue.ts"],"sourcesContent":["import { setSileroModelLoader } from '@micdrop/client'\nimport { OnnxSileroModel, OnnxSileroOptions } from './vad/OnnxSileroModel'\n\nexport * from './vad/OnnxSileroModel'\nexport * from './vad/onnxQueue'\n\nlet options: OnnxSileroOptions = {}\n\n/**\n * Says where the Silero model comes from, to serve it from your own domain\n * rather than from a CDN, or to hand over its bytes directly. Merged into the\n * options already set, so it keeps what the `@micdrop/web/vite` plugin chose.\n * @param next - Address or bytes of the model, and of the ONNX runtime\n */\nexport function setSileroOptions(next: OnnxSileroOptions) {\n options = { ...options, ...next }\n}\n\n// The ONNX runtime weighs more than the rest of Micdrop put together, so it\n// only enters the bundle of an app that imports this file\nsetSileroModelLoader(async () => {\n const model = new OnnxSileroModel(options)\n await model.load()\n return model\n})\n","import {\n createDownloadCache,\n LazySession,\n SileroModel,\n SILERO_SAMPLE_RATE,\n} from '@micdrop/client'\nimport * as ort from 'onnxruntime-web'\nimport { queueOnnxRun } from './onnxQueue'\n\n/** Where the model is fetched from, when the app does not provide it */\nexport const DEFAULT_SILERO_MODEL_URL =\n 'https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx'\n\n// onnxruntime-web is not hosted on jsdelivr (its dist exceeds the CDN size\n// limit, so every file 404s). unpkg serves it, so the wasm backend comes from\n// there instead, at the version of the installed runtime: its JavaScript only\n// loads the WebAssembly files of the same release.\nconst DEFAULT_WASM_PATH = `https://unpkg.com/onnxruntime-web@${ort.env.versions.web}/dist/`\n\nexport interface OnnxSileroOptions {\n /** Address of the Silero v5 model, or the bytes of it */\n model?: string | ArrayBuffer\n /**\n * Where the ONNX runtime loads its WebAssembly files from. `false` leaves\n * them to the runtime, which finds them next to itself once a bundler has\n * emitted them, as the `@micdrop/web/vite` plugin does.\n */\n wasmPath?: string | false\n}\n\n// A couple of megabytes for each address, fetched once per page\nconst fetchModel = createDownloadCache(async (url): Promise<ArrayBuffer> => {\n const response = await fetch(url)\n if (!response.ok) {\n throw new Error(\n `Could not load the Silero model from ${url} (${response.status})`\n )\n }\n return response.arrayBuffer()\n})\n\n/**\n * Runs the Silero model in the browser, on the ONNX WebAssembly runtime.\n *\n * The state machine that turns its answers into turns lives in\n * `@micdrop/client`, this only scores windows of audio. Once released, the\n * model is done with: create a new one to load it again.\n */\nexport class OnnxSileroModel implements SileroModel {\n private session = new LazySession('Silero', () => this.createSession())\n private state: ort.Tensor\n private sampleRate: ort.Tensor\n\n constructor(private options: OnnxSileroOptions = {}) {\n this.state = newState()\n this.sampleRate = new ort.Tensor('int64', [BigInt(SILERO_SAMPLE_RATE)])\n }\n\n async load() {\n await this.session.get()\n }\n\n private async createSession(): Promise<ort.InferenceSession> {\n const wasmPath = this.options.wasmPath ?? DEFAULT_WASM_PATH\n if (wasmPath !== false) {\n ort.env.wasm.wasmPaths = wasmPath\n }\n\n const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL\n const bytes = typeof source === 'string' ? await fetchModel(source) : source\n return ort.InferenceSession.create(bytes)\n }\n\n async process(frame: Float32Array): Promise<number> {\n return this.session.use((session) =>\n queueOnnxRun(async () => {\n const output = await session.run({\n input: new ort.Tensor('float32', frame, [1, frame.length]),\n state: this.state,\n sr: this.sampleRate,\n })\n // Carried over before the next window takes its turn, which reads it\n this.state = output.stateN as ort.Tensor\n return (output.output.data as Float32Array)[0]\n })\n )\n }\n\n reset() {\n this.state = newState()\n }\n\n async release() {\n await this.session.release()\n }\n}\n\n/** The memory the model carries from one window to the next, emptied */\nfunction newState(): ort.Tensor {\n return new ort.Tensor('float32', new Float32Array(2 * 128), [2, 1, 128])\n}\n","/**\n * Runs one inference at a time, whichever model asks.\n *\n * The ONNX runtime keeps a single active session for the whole WebAssembly\n * module, so two models cannot be in flight together. A model running on the\n * graphics card yields while it waits for the device, and a detector scoring\n * its next window on top of it makes both fail, one with `Session already\n * started` and the other with `Session mismatch`.\n *\n * The queue lives on the global scope so every Micdrop model takes the same\n * turns, `@micdrop/smart-turn` included, which ships separately and holds the\n * same few lines.\n */\nconst globalScope = globalThis as typeof globalThis & {\n micdropOnnxQueue?: Promise<unknown>\n}\n\n/**\n * Waits for the models ahead in the queue, then runs\n * @param run - The inference to run once the runtime is free\n */\nexport function queueOnnxRun<T>(run: () => Promise<T>): Promise<T> {\n const started = (globalScope.micdropOnnxQueue ?? Promise.resolve()).then(\n run,\n run\n )\n // A failed run still lets the next one through\n globalScope.micdropOnnxQueue = started.catch(() => undefined)\n return started\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAAAA,iBAAqC;;;ACArC,oBAKO;AACP,UAAqB;;;ACOrB,IAAM,cAAc;AAQb,SAAS,aAAgB,KAAmC;AACjE,QAAM,WAAW,YAAY,oBAAoB,QAAQ,QAAQ,GAAG;AAAA,IAClE;AAAA,IACA;AAAA,EACF;AAEA,cAAY,mBAAmB,QAAQ,MAAM,MAAM,MAAS;AAC5D,SAAO;AACT;;;ADnBO,IAAM,2BACX;AAMF,IAAM,oBAAoB,qCAAyC,QAAI,SAAS,GAAG;AAcnF,IAAM,iBAAa,mCAAoB,OAAO,QAA8B;AAC1E,QAAM,WAAW,MAAM,MAAM,GAAG;AAChC,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI;AAAA,MACR,wCAAwC,GAAG,KAAK,SAAS,MAAM;AAAA,IACjE;AAAA,EACF;AACA,SAAO,SAAS,YAAY;AAC9B,CAAC;AASM,IAAM,kBAAN,MAA6C;AAAA,EAKlD,YAAoBC,WAA6B,CAAC,GAAG;AAAjC,mBAAAA;AAJpB,SAAQ,UAAU,IAAI,0BAAY,UAAU,MAAM,KAAK,cAAc,CAAC;AAKpE,SAAK,QAAQ,SAAS;AACtB,SAAK,aAAa,IAAQ,WAAO,SAAS,CAAC,OAAO,gCAAkB,CAAC,CAAC;AAAA,EACxE;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,KAAK,QAAQ,IAAI;AAAA,EACzB;AAAA,EAEA,MAAc,gBAA+C;AAC3D,UAAM,WAAW,KAAK,QAAQ,YAAY;AAC1C,QAAI,aAAa,OAAO;AACtB,MAAI,QAAI,KAAK,YAAY;AAAA,IAC3B;AAEA,UAAM,SAAS,KAAK,QAAQ,SAAS;AACrC,UAAM,QAAQ,OAAO,WAAW,WAAW,MAAM,WAAW,MAAM,IAAI;AACtE,WAAW,qBAAiB,OAAO,KAAK;AAAA,EAC1C;AAAA,EAEA,MAAM,QAAQ,OAAsC;AAClD,WAAO,KAAK,QAAQ;AAAA,MAAI,CAAC,YACvB,aAAa,YAAY;AACvB,cAAM,SAAS,MAAM,QAAQ,IAAI;AAAA,UAC/B,OAAO,IAAQ,WAAO,WAAW,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC;AAAA,UACzD,OAAO,KAAK;AAAA,UACZ,IAAI,KAAK;AAAA,QACX,CAAC;AAED,aAAK,QAAQ,OAAO;AACpB,eAAQ,OAAO,OAAO,KAAsB,CAAC;AAAA,MAC/C,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,MAAM,UAAU;AACd,UAAM,KAAK,QAAQ,QAAQ;AAAA,EAC7B;AACF;AAGA,SAAS,WAAuB;AAC9B,SAAO,IAAQ,WAAO,WAAW,IAAI,aAAa,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;AACzE;;;AD9FA,IAAI,UAA6B,CAAC;AAQ3B,SAAS,iBAAiB,MAAyB;AACxD,YAAU,EAAE,GAAG,SAAS,GAAG,KAAK;AAClC;AAAA,IAIA,qCAAqB,YAAY;AAC/B,QAAM,QAAQ,IAAI,gBAAgB,OAAO;AACzC,QAAM,MAAM,KAAK;AACjB,SAAO;AACT,CAAC;","names":["import_client","options"]}
|
package/dist/silero.mjs
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
import { setSileroModelLoader } from "@micdrop/client";
|
|
3
3
|
|
|
4
4
|
// src/vad/OnnxSileroModel.ts
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
createDownloadCache,
|
|
7
|
+
LazySession,
|
|
8
|
+
SILERO_SAMPLE_RATE
|
|
9
|
+
} from "@micdrop/client";
|
|
6
10
|
import * as ort from "onnxruntime-web";
|
|
7
11
|
|
|
8
12
|
// src/vad/onnxQueue.ts
|
|
@@ -18,48 +22,53 @@ function queueOnnxRun(run) {
|
|
|
18
22
|
|
|
19
23
|
// src/vad/OnnxSileroModel.ts
|
|
20
24
|
var DEFAULT_SILERO_MODEL_URL = "https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx";
|
|
21
|
-
var DEFAULT_WASM_PATH =
|
|
22
|
-
var
|
|
25
|
+
var DEFAULT_WASM_PATH = `https://unpkg.com/onnxruntime-web@${ort.env.versions.web}/dist/`;
|
|
26
|
+
var fetchModel = createDownloadCache(async (url) => {
|
|
27
|
+
const response = await fetch(url);
|
|
28
|
+
if (!response.ok) {
|
|
29
|
+
throw new Error(
|
|
30
|
+
`Could not load the Silero model from ${url} (${response.status})`
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
return response.arrayBuffer();
|
|
34
|
+
});
|
|
23
35
|
var OnnxSileroModel = class {
|
|
24
36
|
constructor(options2 = {}) {
|
|
25
37
|
this.options = options2;
|
|
38
|
+
this.session = new LazySession("Silero", () => this.createSession());
|
|
26
39
|
this.state = newState();
|
|
27
40
|
this.sampleRate = new ort.Tensor("int64", [BigInt(SILERO_SAMPLE_RATE)]);
|
|
28
41
|
}
|
|
29
42
|
async load() {
|
|
30
|
-
|
|
43
|
+
await this.session.get();
|
|
44
|
+
}
|
|
45
|
+
async createSession() {
|
|
31
46
|
const wasmPath = this.options.wasmPath ?? DEFAULT_WASM_PATH;
|
|
32
47
|
if (wasmPath !== false) {
|
|
33
48
|
ort.env.wasm.wasmPaths = wasmPath;
|
|
34
49
|
}
|
|
35
50
|
const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL;
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
loading = loading ?? fetch(source).then((response) => response.arrayBuffer());
|
|
39
|
-
bytes = await loading;
|
|
40
|
-
} else {
|
|
41
|
-
bytes = source;
|
|
42
|
-
}
|
|
43
|
-
this.session = await ort.InferenceSession.create(bytes);
|
|
51
|
+
const bytes = typeof source === "string" ? await fetchModel(source) : source;
|
|
52
|
+
return ort.InferenceSession.create(bytes);
|
|
44
53
|
}
|
|
45
54
|
async process(frame) {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
return this.session.use(
|
|
56
|
+
(session) => queueOnnxRun(async () => {
|
|
57
|
+
const output = await session.run({
|
|
58
|
+
input: new ort.Tensor("float32", frame, [1, frame.length]),
|
|
59
|
+
state: this.state,
|
|
60
|
+
sr: this.sampleRate
|
|
61
|
+
});
|
|
62
|
+
this.state = output.stateN;
|
|
63
|
+
return output.output.data[0];
|
|
52
64
|
})
|
|
53
65
|
);
|
|
54
|
-
this.state = output.stateN;
|
|
55
|
-
return output.output.data[0];
|
|
56
66
|
}
|
|
57
67
|
reset() {
|
|
58
68
|
this.state = newState();
|
|
59
69
|
}
|
|
60
70
|
async release() {
|
|
61
|
-
await this.session
|
|
62
|
-
this.session = void 0;
|
|
71
|
+
await this.session.release();
|
|
63
72
|
}
|
|
64
73
|
};
|
|
65
74
|
function newState() {
|
package/dist/silero.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/silero.ts","../src/vad/OnnxSileroModel.ts","../src/vad/onnxQueue.ts"],"sourcesContent":["import { setSileroModelLoader } from '@micdrop/client'\nimport { OnnxSileroModel, OnnxSileroOptions } from './vad/OnnxSileroModel'\n\nexport * from './vad/OnnxSileroModel'\nexport * from './vad/onnxQueue'\n\nlet options: OnnxSileroOptions = {}\n\n/**\n * Says where the Silero model comes from, to serve it from your own domain\n * rather than from a CDN, or to hand over its bytes directly. Merged into the\n * options already set, so it keeps what the `@micdrop/web/vite` plugin chose.\n * @param next - Address or bytes of the model, and of the ONNX runtime\n */\nexport function setSileroOptions(next: OnnxSileroOptions) {\n options = { ...options, ...next }\n}\n\n// The ONNX runtime weighs more than the rest of Micdrop put together, so it\n// only enters the bundle of an app that imports this file\nsetSileroModelLoader(async () => {\n const model = new OnnxSileroModel(options)\n await model.load()\n return model\n})\n","import {
|
|
1
|
+
{"version":3,"sources":["../src/silero.ts","../src/vad/OnnxSileroModel.ts","../src/vad/onnxQueue.ts"],"sourcesContent":["import { setSileroModelLoader } from '@micdrop/client'\nimport { OnnxSileroModel, OnnxSileroOptions } from './vad/OnnxSileroModel'\n\nexport * from './vad/OnnxSileroModel'\nexport * from './vad/onnxQueue'\n\nlet options: OnnxSileroOptions = {}\n\n/**\n * Says where the Silero model comes from, to serve it from your own domain\n * rather than from a CDN, or to hand over its bytes directly. Merged into the\n * options already set, so it keeps what the `@micdrop/web/vite` plugin chose.\n * @param next - Address or bytes of the model, and of the ONNX runtime\n */\nexport function setSileroOptions(next: OnnxSileroOptions) {\n options = { ...options, ...next }\n}\n\n// The ONNX runtime weighs more than the rest of Micdrop put together, so it\n// only enters the bundle of an app that imports this file\nsetSileroModelLoader(async () => {\n const model = new OnnxSileroModel(options)\n await model.load()\n return model\n})\n","import {\n createDownloadCache,\n LazySession,\n SileroModel,\n SILERO_SAMPLE_RATE,\n} from '@micdrop/client'\nimport * as ort from 'onnxruntime-web'\nimport { queueOnnxRun } from './onnxQueue'\n\n/** Where the model is fetched from, when the app does not provide it */\nexport const DEFAULT_SILERO_MODEL_URL =\n 'https://cdn.jsdelivr.net/npm/@ricky0123/vad-web@0.0.30/dist/silero_vad_v5.onnx'\n\n// onnxruntime-web is not hosted on jsdelivr (its dist exceeds the CDN size\n// limit, so every file 404s). unpkg serves it, so the wasm backend comes from\n// there instead, at the version of the installed runtime: its JavaScript only\n// loads the WebAssembly files of the same release.\nconst DEFAULT_WASM_PATH = `https://unpkg.com/onnxruntime-web@${ort.env.versions.web}/dist/`\n\nexport interface OnnxSileroOptions {\n /** Address of the Silero v5 model, or the bytes of it */\n model?: string | ArrayBuffer\n /**\n * Where the ONNX runtime loads its WebAssembly files from. `false` leaves\n * them to the runtime, which finds them next to itself once a bundler has\n * emitted them, as the `@micdrop/web/vite` plugin does.\n */\n wasmPath?: string | false\n}\n\n// A couple of megabytes for each address, fetched once per page\nconst fetchModel = createDownloadCache(async (url): Promise<ArrayBuffer> => {\n const response = await fetch(url)\n if (!response.ok) {\n throw new Error(\n `Could not load the Silero model from ${url} (${response.status})`\n )\n }\n return response.arrayBuffer()\n})\n\n/**\n * Runs the Silero model in the browser, on the ONNX WebAssembly runtime.\n *\n * The state machine that turns its answers into turns lives in\n * `@micdrop/client`, this only scores windows of audio. Once released, the\n * model is done with: create a new one to load it again.\n */\nexport class OnnxSileroModel implements SileroModel {\n private session = new LazySession('Silero', () => this.createSession())\n private state: ort.Tensor\n private sampleRate: ort.Tensor\n\n constructor(private options: OnnxSileroOptions = {}) {\n this.state = newState()\n this.sampleRate = new ort.Tensor('int64', [BigInt(SILERO_SAMPLE_RATE)])\n }\n\n async load() {\n await this.session.get()\n }\n\n private async createSession(): Promise<ort.InferenceSession> {\n const wasmPath = this.options.wasmPath ?? DEFAULT_WASM_PATH\n if (wasmPath !== false) {\n ort.env.wasm.wasmPaths = wasmPath\n }\n\n const source = this.options.model ?? DEFAULT_SILERO_MODEL_URL\n const bytes = typeof source === 'string' ? await fetchModel(source) : source\n return ort.InferenceSession.create(bytes)\n }\n\n async process(frame: Float32Array): Promise<number> {\n return this.session.use((session) =>\n queueOnnxRun(async () => {\n const output = await session.run({\n input: new ort.Tensor('float32', frame, [1, frame.length]),\n state: this.state,\n sr: this.sampleRate,\n })\n // Carried over before the next window takes its turn, which reads it\n this.state = output.stateN as ort.Tensor\n return (output.output.data as Float32Array)[0]\n })\n )\n }\n\n reset() {\n this.state = newState()\n }\n\n async release() {\n await this.session.release()\n }\n}\n\n/** The memory the model carries from one window to the next, emptied */\nfunction newState(): ort.Tensor {\n return new ort.Tensor('float32', new Float32Array(2 * 128), [2, 1, 128])\n}\n","/**\n * Runs one inference at a time, whichever model asks.\n *\n * The ONNX runtime keeps a single active session for the whole WebAssembly\n * module, so two models cannot be in flight together. A model running on the\n * graphics card yields while it waits for the device, and a detector scoring\n * its next window on top of it makes both fail, one with `Session already\n * started` and the other with `Session mismatch`.\n *\n * The queue lives on the global scope so every Micdrop model takes the same\n * turns, `@micdrop/smart-turn` included, which ships separately and holds the\n * same few lines.\n */\nconst globalScope = globalThis as typeof globalThis & {\n micdropOnnxQueue?: Promise<unknown>\n}\n\n/**\n * Waits for the models ahead in the queue, then runs\n * @param run - The inference to run once the runtime is free\n */\nexport function queueOnnxRun<T>(run: () => Promise<T>): Promise<T> {\n const started = (globalScope.micdropOnnxQueue ?? Promise.resolve()).then(\n run,\n run\n )\n // A failed run still lets the next one through\n globalScope.micdropOnnxQueue = started.catch(() => undefined)\n return started\n}\n"],"mappings":";AAAA,SAAS,4BAA4B;;;ACArC;AAAA,EACE;AAAA,EACA;AAAA,EAEA;AAAA,OACK;AACP,YAAY,SAAS;;;ACOrB,IAAM,cAAc;AAQb,SAAS,aAAgB,KAAmC;AACjE,QAAM,WAAW,YAAY,oBAAoB,QAAQ,QAAQ,GAAG;AAAA,IAClE;AAAA,IACA;AAAA,EACF;AAEA,cAAY,mBAAmB,QAAQ,MAAM,MAAM,MAAS;AAC5D,SAAO;AACT;;;ADnBO,IAAM,2BACX;AAMF,IAAM,oBAAoB,qCAAyC,QAAI,SAAS,GAAG;AAcnF,IAAM,aAAa,oBAAoB,OAAO,QAA8B;AAC1E,QAAM,WAAW,MAAM,MAAM,GAAG;AAChC,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI;AAAA,MACR,wCAAwC,GAAG,KAAK,SAAS,MAAM;AAAA,IACjE;AAAA,EACF;AACA,SAAO,SAAS,YAAY;AAC9B,CAAC;AASM,IAAM,kBAAN,MAA6C;AAAA,EAKlD,YAAoBA,WAA6B,CAAC,GAAG;AAAjC,mBAAAA;AAJpB,SAAQ,UAAU,IAAI,YAAY,UAAU,MAAM,KAAK,cAAc,CAAC;AAKpE,SAAK,QAAQ,SAAS;AACtB,SAAK,aAAa,IAAQ,WAAO,SAAS,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAAA,EACxE;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,KAAK,QAAQ,IAAI;AAAA,EACzB;AAAA,EAEA,MAAc,gBAA+C;AAC3D,UAAM,WAAW,KAAK,QAAQ,YAAY;AAC1C,QAAI,aAAa,OAAO;AACtB,MAAI,QAAI,KAAK,YAAY;AAAA,IAC3B;AAEA,UAAM,SAAS,KAAK,QAAQ,SAAS;AACrC,UAAM,QAAQ,OAAO,WAAW,WAAW,MAAM,WAAW,MAAM,IAAI;AACtE,WAAW,qBAAiB,OAAO,KAAK;AAAA,EAC1C;AAAA,EAEA,MAAM,QAAQ,OAAsC;AAClD,WAAO,KAAK,QAAQ;AAAA,MAAI,CAAC,YACvB,aAAa,YAAY;AACvB,cAAM,SAAS,MAAM,QAAQ,IAAI;AAAA,UAC/B,OAAO,IAAQ,WAAO,WAAW,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC;AAAA,UACzD,OAAO,KAAK;AAAA,UACZ,IAAI,KAAK;AAAA,QACX,CAAC;AAED,aAAK,QAAQ,OAAO;AACpB,eAAQ,OAAO,OAAO,KAAsB,CAAC;AAAA,MAC/C,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,MAAM,UAAU;AACd,UAAM,KAAK,QAAQ,QAAQ;AAAA,EAC7B;AACF;AAGA,SAAS,WAAuB;AAC9B,SAAO,IAAQ,WAAO,WAAW,IAAI,aAAa,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;AACzE;;;AD9FA,IAAI,UAA6B,CAAC;AAQ3B,SAAS,iBAAiB,MAAyB;AACxD,YAAU,EAAE,GAAG,SAAS,GAAG,KAAK;AAClC;AAIA,qBAAqB,YAAY;AAC/B,QAAM,QAAQ,IAAI,gBAAgB,OAAO;AACzC,QAAM,MAAM,KAAK;AACjB,SAAO;AACT,CAAC;","names":["options"]}
|
package/dist/vite.js
CHANGED
|
@@ -34,7 +34,7 @@ __export(vite_exports, {
|
|
|
34
34
|
});
|
|
35
35
|
module.exports = __toCommonJS(vite_exports);
|
|
36
36
|
|
|
37
|
-
// ../../node_modules/.pnpm/tsup@8.5.
|
|
37
|
+
// ../../node_modules/.pnpm/tsup@8.5.1_jiti@2.7.0_postcss@8.5.32_tsx@4.23.15_typescript@5.9.3_yaml@2.9.1/node_modules/tsup/assets/cjs_shims.js
|
|
38
38
|
var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.tagName.toUpperCase() === "SCRIPT" ? document.currentScript.src : new URL("main.js", document.baseURI).href;
|
|
39
39
|
var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
40
40
|
|
package/dist/vite.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/vite.ts","../../../node_modules/.pnpm/tsup@8.5.
|
|
1
|
+
{"version":3,"sources":["../src/vite.ts","../../../node_modules/.pnpm/tsup@8.5.1_jiti@2.7.0_postcss@8.5.32_tsx@4.23.15_typescript@5.9.3_yaml@2.9.1/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["import { createRequire } from 'node:module'\nimport path from 'node:path'\nimport process from 'node:process'\nimport { fileURLToPath } from 'node:url'\nimport type { Plugin } from 'vite'\n\nconst SILERO_ID = '@micdrop/web/silero'\nconst SMART_TURN_ID = '@micdrop/smart-turn/web'\nconst VIRTUAL_SILERO = '\\0micdrop:silero'\nconst VIRTUAL_SMART_TURN = '\\0micdrop:smart-turn'\n\n/**\n * Bundles the files the voice models load at runtime, so a call never depends\n * on a public CDN being reachable.\n *\n * Without it, `@micdrop/web/silero` fetches the Silero model from jsDelivr and\n * the ONNX runtime from unpkg, which an outage, a firewall or an ad blocker\n * can keep from loading. With it, the model ships with the app and the ONNX\n * runtime finds its WebAssembly next to itself, both emitted by Vite.\n * `@micdrop/smart-turn/web` gets the same runtime, its checkpoint stays where\n * `setSmartTurnOptions` points.\n *\n * Silero only runs on WebAssembly, so the runtime is the WebAssembly build,\n * half the size of the full one. Smart Turn runs on the graphics card when\n * the browser offers it, so the full build is kept when `@micdrop/smart-turn`\n * is installed. Every import of the runtime gets the same build, so an app\n * never loads two of them.\n *\n * @example\n * import { micdrop } from '@micdrop/web/vite'\n *\n * export default defineConfig({ plugins: [micdrop()] })\n */\nexport function micdrop(options: MicdropViteOptions = {}): Plugin {\n const sileroModel = fileURLToPath(\n new URL('../assets/silero_vad_v5.onnx', import.meta.url)\n ).replace(/\\\\/g, '/')\n\n return {\n name: 'micdrop',\n enforce: 'pre',\n\n config(config) {\n const webgpu =\n options.webgpu ??\n isInstalled(\n '@micdrop/smart-turn',\n path.resolve(config.root ?? process.cwd())\n )\n return {\n resolve: {\n alias: webgpu\n ? []\n : [\n {\n find: /^onnxruntime-web$/,\n replacement: 'onnxruntime-web/wasm',\n },\n ],\n },\n optimizeDeps: {\n // Pre-bundling moves the runtime away from its WebAssembly files,\n // which it looks up relative to its own address\n exclude: ['onnxruntime-web', 'onnxruntime-web/wasm'],\n },\n }\n },\n\n // The app keeps importing `@micdrop/web/silero`, and gets a module that\n // sets the bundled options before handing over the real one. Imports from\n // dependencies and from the wrappers themselves go through untouched.\n resolveId(id, importer) {\n if (!importer || importer.startsWith('\\0')) return null\n if (importer.includes('/node_modules/')) return null\n if (id === SILERO_ID) return VIRTUAL_SILERO\n if (id === SMART_TURN_ID) return VIRTUAL_SMART_TURN\n return null\n },\n\n load(id) {\n if (id === VIRTUAL_SILERO) {\n return [\n `import { setSileroOptions } from '${SILERO_ID}'`,\n `import model from ${JSON.stringify(`${sileroModel}?url`)}`,\n `setSileroOptions({ model, wasmPath: false })`,\n `export * from '${SILERO_ID}'`,\n ].join('\\n')\n }\n if (id === VIRTUAL_SMART_TURN) {\n return [\n `import { setSmartTurnOptions } from '${SMART_TURN_ID}'`,\n `setSmartTurnOptions({ wasmPath: false })`,\n `export * from '${SMART_TURN_ID}'`,\n ].join('\\n')\n }\n return null\n },\n }\n}\n\nexport interface MicdropViteOptions {\n /**\n * Keeps the build of the ONNX runtime that runs on the graphics card, twice\n * the size of the WebAssembly one. Only Smart Turn uses it, so it defaults\n * to whether `@micdrop/smart-turn` is installed.\n */\n webgpu?: boolean\n}\n\n/** Whether the app at `root` can import the package */\nfunction isInstalled(name: string, root: string): boolean {\n try {\n createRequire(path.join(root, 'package.json')).resolve(name)\n return true\n } catch {\n return false\n }\n}\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () => \n typeof document === \"undefined\" \n ? new URL(`file:${__filename}`).href \n : (document.currentScript && document.currentScript.tagName.toUpperCase() === 'SCRIPT') \n ? document.currentScript.src \n : new URL(\"main.js\", document.baseURI).href;\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,QAAQ,YAAY,MAAM,WAC1E,SAAS,cAAc,MACvB,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEtC,IAAM,gBAAgC,iCAAiB;;;ADZ9D,yBAA8B;AAC9B,uBAAiB;AACjB,0BAAoB;AACpB,sBAA8B;AAG9B,IAAM,YAAY;AAClB,IAAM,gBAAgB;AACtB,IAAM,iBAAiB;AACvB,IAAM,qBAAqB;AAwBpB,SAAS,QAAQ,UAA8B,CAAC,GAAW;AAChE,QAAM,kBAAc;AAAA,IAClB,IAAI,IAAI,gCAAgC,aAAe;AAAA,EACzD,EAAE,QAAQ,OAAO,GAAG;AAEpB,SAAO;AAAA,IACL,MAAM;AAAA,IACN,SAAS;AAAA,IAET,OAAO,QAAQ;AACb,YAAM,SACJ,QAAQ,UACR;AAAA,QACE;AAAA,QACA,iBAAAA,QAAK,QAAQ,OAAO,QAAQ,oBAAAC,QAAQ,IAAI,CAAC;AAAA,MAC3C;AACF,aAAO;AAAA,QACL,SAAS;AAAA,UACP,OAAO,SACH,CAAC,IACD;AAAA,YACE;AAAA,cACE,MAAM;AAAA,cACN,aAAa;AAAA,YACf;AAAA,UACF;AAAA,QACN;AAAA,QACA,cAAc;AAAA;AAAA;AAAA,UAGZ,SAAS,CAAC,mBAAmB,sBAAsB;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA;AAAA;AAAA;AAAA,IAKA,UAAU,IAAI,UAAU;AACtB,UAAI,CAAC,YAAY,SAAS,WAAW,IAAI,EAAG,QAAO;AACnD,UAAI,SAAS,SAAS,gBAAgB,EAAG,QAAO;AAChD,UAAI,OAAO,UAAW,QAAO;AAC7B,UAAI,OAAO,cAAe,QAAO;AACjC,aAAO;AAAA,IACT;AAAA,IAEA,KAAK,IAAI;AACP,UAAI,OAAO,gBAAgB;AACzB,eAAO;AAAA,UACL,qCAAqC,SAAS;AAAA,UAC9C,qBAAqB,KAAK,UAAU,GAAG,WAAW,MAAM,CAAC;AAAA,UACzD;AAAA,UACA,kBAAkB,SAAS;AAAA,QAC7B,EAAE,KAAK,IAAI;AAAA,MACb;AACA,UAAI,OAAO,oBAAoB;AAC7B,eAAO;AAAA,UACL,wCAAwC,aAAa;AAAA,UACrD;AAAA,UACA,kBAAkB,aAAa;AAAA,QACjC,EAAE,KAAK,IAAI;AAAA,MACb;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAYA,SAAS,YAAY,MAAc,MAAuB;AACxD,MAAI;AACF,0CAAc,iBAAAD,QAAK,KAAK,MAAM,cAAc,CAAC,EAAE,QAAQ,IAAI;AAC3D,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":["path","process"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@micdrop/web",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.1",
|
|
4
4
|
"description": "🖐️🎤 Micdrop: real-time voice conversations with AI, in the browser",
|
|
5
5
|
"author": "Godefroy de Compreignac",
|
|
6
6
|
"license": "MIT",
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"conversation"
|
|
35
35
|
],
|
|
36
36
|
"dependencies": {
|
|
37
|
-
"@micdrop/client": "^3.
|
|
37
|
+
"@micdrop/client": "^3.6.0",
|
|
38
38
|
"onnxruntime-web": "^1.31.1"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"esbuild-raw-plugin": "^0.3.1",
|
|
51
51
|
"tsup": "^8.5.1",
|
|
52
52
|
"typescript": "^5.9.3",
|
|
53
|
-
"vite": "^
|
|
53
|
+
"vite": "^8.3.5"
|
|
54
54
|
},
|
|
55
55
|
"exports": {
|
|
56
56
|
".": {
|