@tensamin/audio 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/chunk-FS635GMR.mjs +47 -0
- package/dist/chunk-HFSKQ33X.mjs +38 -0
- package/{src/vad/vad-state.ts → dist/chunk-JJASCVEW.mjs} +21 -33
- package/dist/chunk-OZ7KMC4S.mjs +46 -0
- package/dist/chunk-QU7E5HBA.mjs +106 -0
- package/dist/chunk-SDTOKWM2.mjs +39 -0
- package/{src/vad/vad-node.ts → dist/chunk-UMU2KIB6.mjs} +10 -20
- package/dist/chunk-WBQAMGXK.mjs +0 -0
- package/dist/context/audio-context.d.mts +32 -0
- package/dist/context/audio-context.d.ts +32 -0
- package/dist/context/audio-context.js +75 -0
- package/dist/context/audio-context.mjs +16 -0
- package/dist/extensibility/plugins.d.mts +9 -0
- package/dist/extensibility/plugins.d.ts +9 -0
- package/dist/extensibility/plugins.js +180 -0
- package/dist/extensibility/plugins.mjs +14 -0
- package/dist/index.d.mts +10 -216
- package/dist/index.d.ts +10 -216
- package/dist/index.js +11 -10
- package/dist/index.mjs +29 -352
- package/dist/livekit/integration.d.mts +11 -0
- package/dist/livekit/integration.d.ts +11 -0
- package/dist/livekit/integration.js +368 -0
- package/dist/livekit/integration.mjs +12 -0
- package/dist/noise-suppression/rnnoise-node.d.mts +10 -0
- package/dist/noise-suppression/rnnoise-node.d.ts +10 -0
- package/dist/noise-suppression/rnnoise-node.js +73 -0
- package/dist/noise-suppression/rnnoise-node.mjs +6 -0
- package/dist/pipeline/audio-pipeline.d.mts +6 -0
- package/dist/pipeline/audio-pipeline.d.ts +6 -0
- package/dist/pipeline/audio-pipeline.js +335 -0
- package/dist/pipeline/audio-pipeline.mjs +11 -0
- package/dist/types.d.mts +155 -0
- package/dist/types.d.ts +155 -0
- package/dist/types.js +18 -0
- package/dist/types.mjs +1 -0
- package/dist/vad/vad-node.d.mts +9 -0
- package/dist/vad/vad-node.d.ts +9 -0
- package/dist/vad/vad-node.js +92 -0
- package/dist/vad/vad-node.mjs +6 -0
- package/dist/vad/vad-state.d.mts +15 -0
- package/dist/vad/vad-state.d.ts +15 -0
- package/dist/vad/vad-state.js +83 -0
- package/dist/vad/vad-state.mjs +6 -0
- package/package.json +8 -5
- package/.github/workflows/publish.yml +0 -29
- package/bun.lock +0 -258
- package/src/context/audio-context.ts +0 -69
- package/src/extensibility/plugins.ts +0 -45
- package/src/index.ts +0 -8
- package/src/livekit/integration.ts +0 -61
- package/src/noise-suppression/rnnoise-node.ts +0 -62
- package/src/pipeline/audio-pipeline.ts +0 -154
- package/src/types.ts +0 -167
- package/tsconfig.json +0 -46
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/extensibility/plugins.ts
|
|
31
|
+
var plugins_exports = {};
|
|
32
|
+
__export(plugins_exports, {
|
|
33
|
+
getNoiseSuppressionPlugin: () => getNoiseSuppressionPlugin,
|
|
34
|
+
getVADPlugin: () => getVADPlugin,
|
|
35
|
+
registerNoiseSuppressionPlugin: () => registerNoiseSuppressionPlugin,
|
|
36
|
+
registerVADPlugin: () => registerVADPlugin
|
|
37
|
+
});
|
|
38
|
+
module.exports = __toCommonJS(plugins_exports);
|
|
39
|
+
|
|
40
|
+
// src/noise-suppression/rnnoise-node.ts
|
|
41
|
+
var RNNoisePlugin = class {
|
|
42
|
+
name = "rnnoise-ns";
|
|
43
|
+
wasmBuffer = null;
|
|
44
|
+
async createNode(context, config) {
|
|
45
|
+
const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
|
|
46
|
+
if (!config?.enabled) {
|
|
47
|
+
const pass = context.createGain();
|
|
48
|
+
return pass;
|
|
49
|
+
}
|
|
50
|
+
if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
|
|
51
|
+
throw new Error(
|
|
52
|
+
"RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets and provide the URLs."
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
if (!this.wasmBuffer) {
|
|
56
|
+
this.wasmBuffer = await loadRnnoise({
|
|
57
|
+
url: config.wasmUrl,
|
|
58
|
+
simdUrl: config.simdUrl
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
const workletUrl = config.workletUrl;
|
|
62
|
+
try {
|
|
63
|
+
await context.audioWorklet.addModule(workletUrl);
|
|
64
|
+
} catch (e) {
|
|
65
|
+
console.warn("Failed to add RNNoise worklet module:", e);
|
|
66
|
+
}
|
|
67
|
+
const node = new RnnoiseWorkletNode(context, {
|
|
68
|
+
wasmBinary: this.wasmBuffer,
|
|
69
|
+
maxChannels: 1
|
|
70
|
+
// Mono for now
|
|
71
|
+
});
|
|
72
|
+
return node;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
// src/vad/vad-node.ts
|
|
77
|
+
var energyVadWorkletCode = `
|
|
78
|
+
class EnergyVadProcessor extends AudioWorkletProcessor {
|
|
79
|
+
constructor() {
|
|
80
|
+
super();
|
|
81
|
+
this.smoothing = 0.95;
|
|
82
|
+
this.energy = 0;
|
|
83
|
+
this.noiseFloor = 0.001;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
process(inputs, outputs, parameters) {
|
|
87
|
+
const input = inputs[0];
|
|
88
|
+
if (!input || !input.length) return true;
|
|
89
|
+
const channel = input[0];
|
|
90
|
+
|
|
91
|
+
// Calculate RMS
|
|
92
|
+
let sum = 0;
|
|
93
|
+
for (let i = 0; i < channel.length; i++) {
|
|
94
|
+
sum += channel[i] * channel[i];
|
|
95
|
+
}
|
|
96
|
+
const rms = Math.sqrt(sum / channel.length);
|
|
97
|
+
|
|
98
|
+
// Simple adaptive noise floor (very basic)
|
|
99
|
+
if (rms < this.noiseFloor) {
|
|
100
|
+
this.noiseFloor = this.noiseFloor * 0.99 + rms * 0.01;
|
|
101
|
+
} else {
|
|
102
|
+
this.noiseFloor = this.noiseFloor * 0.999 + rms * 0.001;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Calculate "probability" based on SNR
|
|
106
|
+
// This is a heuristic mapping from energy to 0-1
|
|
107
|
+
const snr = rms / (this.noiseFloor + 1e-6);
|
|
108
|
+
const probability = Math.min(1, Math.max(0, (snr - 1.5) / 10)); // Arbitrary scaling
|
|
109
|
+
|
|
110
|
+
this.port.postMessage({ probability });
|
|
111
|
+
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
registerProcessor('energy-vad-processor', EnergyVadProcessor);
|
|
116
|
+
`;
|
|
117
|
+
var EnergyVADPlugin = class {
|
|
118
|
+
name = "energy-vad";
|
|
119
|
+
async createNode(context, config, onDecision) {
|
|
120
|
+
const blob = new Blob([energyVadWorkletCode], {
|
|
121
|
+
type: "application/javascript"
|
|
122
|
+
});
|
|
123
|
+
const url = URL.createObjectURL(blob);
|
|
124
|
+
try {
|
|
125
|
+
await context.audioWorklet.addModule(url);
|
|
126
|
+
} catch (e) {
|
|
127
|
+
console.warn("Failed to add Energy VAD worklet:", e);
|
|
128
|
+
throw e;
|
|
129
|
+
} finally {
|
|
130
|
+
URL.revokeObjectURL(url);
|
|
131
|
+
}
|
|
132
|
+
const node = new AudioWorkletNode(context, "energy-vad-processor");
|
|
133
|
+
node.port.onmessage = (event) => {
|
|
134
|
+
const { probability } = event.data;
|
|
135
|
+
onDecision(probability);
|
|
136
|
+
};
|
|
137
|
+
return node;
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
// src/extensibility/plugins.ts
|
|
142
|
+
var nsPlugins = /* @__PURE__ */ new Map();
|
|
143
|
+
var vadPlugins = /* @__PURE__ */ new Map();
|
|
144
|
+
var defaultNs = new RNNoisePlugin();
|
|
145
|
+
nsPlugins.set(defaultNs.name, defaultNs);
|
|
146
|
+
var defaultVad = new EnergyVADPlugin();
|
|
147
|
+
vadPlugins.set(defaultVad.name, defaultVad);
|
|
148
|
+
function registerNoiseSuppressionPlugin(plugin) {
|
|
149
|
+
nsPlugins.set(plugin.name, plugin);
|
|
150
|
+
}
|
|
151
|
+
function registerVADPlugin(plugin) {
|
|
152
|
+
vadPlugins.set(plugin.name, plugin);
|
|
153
|
+
}
|
|
154
|
+
function getNoiseSuppressionPlugin(name) {
|
|
155
|
+
if (!name) return defaultNs;
|
|
156
|
+
const plugin = nsPlugins.get(name);
|
|
157
|
+
if (!plugin) {
|
|
158
|
+
console.warn(
|
|
159
|
+
`Noise suppression plugin '${name}' not found, falling back to default.`
|
|
160
|
+
);
|
|
161
|
+
return defaultNs;
|
|
162
|
+
}
|
|
163
|
+
return plugin;
|
|
164
|
+
}
|
|
165
|
+
function getVADPlugin(name) {
|
|
166
|
+
if (!name) return defaultVad;
|
|
167
|
+
const plugin = vadPlugins.get(name);
|
|
168
|
+
if (!plugin) {
|
|
169
|
+
console.warn(`VAD plugin '${name}' not found, falling back to default.`);
|
|
170
|
+
return defaultVad;
|
|
171
|
+
}
|
|
172
|
+
return plugin;
|
|
173
|
+
}
|
|
174
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
175
|
+
0 && (module.exports = {
|
|
176
|
+
getNoiseSuppressionPlugin,
|
|
177
|
+
getVADPlugin,
|
|
178
|
+
registerNoiseSuppressionPlugin,
|
|
179
|
+
registerVADPlugin
|
|
180
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getNoiseSuppressionPlugin,
|
|
3
|
+
getVADPlugin,
|
|
4
|
+
registerNoiseSuppressionPlugin,
|
|
5
|
+
registerVADPlugin
|
|
6
|
+
} from "../chunk-FS635GMR.mjs";
|
|
7
|
+
import "../chunk-SDTOKWM2.mjs";
|
|
8
|
+
import "../chunk-UMU2KIB6.mjs";
|
|
9
|
+
export {
|
|
10
|
+
getNoiseSuppressionPlugin,
|
|
11
|
+
getVADPlugin,
|
|
12
|
+
registerNoiseSuppressionPlugin,
|
|
13
|
+
registerVADPlugin
|
|
14
|
+
};
|
package/dist/index.d.mts
CHANGED
|
@@ -1,216 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
noiseSuppression?: {
|
|
12
|
-
enabled: boolean;
|
|
13
|
-
/**
|
|
14
|
-
* Path or URL to the RNNoise WASM binary.
|
|
15
|
-
* If not provided, the default from @sapphi-red/web-noise-suppressor will be used (if bundler supports it).
|
|
16
|
-
*/
|
|
17
|
-
wasmUrl?: string;
|
|
18
|
-
/**
|
|
19
|
-
* Path or URL to the RNNoise worklet script.
|
|
20
|
-
*/
|
|
21
|
-
workletUrl?: string;
|
|
22
|
-
/**
|
|
23
|
-
* Plugin name to use. Defaults to 'rnnoise-ns'.
|
|
24
|
-
*/
|
|
25
|
-
pluginName?: string;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* Voice Activity Detection (VAD) configuration.
|
|
29
|
-
*/
|
|
30
|
-
vad?: {
|
|
31
|
-
enabled: boolean;
|
|
32
|
-
/**
|
|
33
|
-
* Plugin name to use. Defaults to 'rnnoise-vad' or 'energy-vad'.
|
|
34
|
-
*/
|
|
35
|
-
pluginName?: string;
|
|
36
|
-
/**
|
|
37
|
-
* Probability threshold for speech onset (0-1).
|
|
38
|
-
* Default: 0.5
|
|
39
|
-
*/
|
|
40
|
-
startThreshold?: number;
|
|
41
|
-
/**
|
|
42
|
-
* Probability threshold for speech offset (0-1).
|
|
43
|
-
* Default: 0.4
|
|
44
|
-
*/
|
|
45
|
-
stopThreshold?: number;
|
|
46
|
-
/**
|
|
47
|
-
* Time in ms to wait after speech stops before considering it silent.
|
|
48
|
-
* Default: 300ms
|
|
49
|
-
*/
|
|
50
|
-
hangoverMs?: number;
|
|
51
|
-
/**
|
|
52
|
-
* Time in ms of audio to buffer before speech onset to avoid cutting the start.
|
|
53
|
-
* Default: 200ms
|
|
54
|
-
*/
|
|
55
|
-
preRollMs?: number;
|
|
56
|
-
};
|
|
57
|
-
/**
|
|
58
|
-
* Output gain and muting configuration.
|
|
59
|
-
*/
|
|
60
|
-
output?: {
|
|
61
|
-
/**
|
|
62
|
-
* Gain to apply when speaking (0-1+). Default: 1.0
|
|
63
|
-
*/
|
|
64
|
-
speechGain?: number;
|
|
65
|
-
/**
|
|
66
|
-
* Gain to apply when silent (0-1). Default: 0.0 (mute)
|
|
67
|
-
*/
|
|
68
|
-
silenceGain?: number;
|
|
69
|
-
/**
|
|
70
|
-
* Time in seconds to ramp gain changes. Default: 0.02
|
|
71
|
-
*/
|
|
72
|
-
gainRampTime?: number;
|
|
73
|
-
};
|
|
74
|
-
/**
|
|
75
|
-
* LiveKit integration configuration.
|
|
76
|
-
*/
|
|
77
|
-
livekit?: {
|
|
78
|
-
/**
|
|
79
|
-
* Whether to call track.mute()/unmute() on the LocalAudioTrack based on VAD.
|
|
80
|
-
* This saves bandwidth but has more signaling overhead.
|
|
81
|
-
* Default: false (uses gain gating only)
|
|
82
|
-
*/
|
|
83
|
-
manageTrackMute?: boolean;
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
/**
|
|
87
|
-
* Represents the state of Voice Activity Detection.
|
|
88
|
-
*/
|
|
89
|
-
interface VADState {
|
|
90
|
-
/**
|
|
91
|
-
* Whether speech is currently detected (after hysteresis).
|
|
92
|
-
*/
|
|
93
|
-
isSpeaking: boolean;
|
|
94
|
-
/**
|
|
95
|
-
* Raw probability of speech from the VAD model (0-1).
|
|
96
|
-
*/
|
|
97
|
-
probability: number;
|
|
98
|
-
/**
|
|
99
|
-
* Current state enum.
|
|
100
|
-
*/
|
|
101
|
-
state: "silent" | "speech_starting" | "speaking" | "speech_ending";
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* Events emitted by the audio pipeline.
|
|
105
|
-
*/
|
|
106
|
-
type AudioPipelineEvents = {
|
|
107
|
-
vadChange: VADState;
|
|
108
|
-
error: Error;
|
|
109
|
-
};
|
|
110
|
-
/**
|
|
111
|
-
* Handle to a running audio processing pipeline.
|
|
112
|
-
*/
|
|
113
|
-
interface AudioPipelineHandle {
|
|
114
|
-
/**
|
|
115
|
-
* The processed MediaStreamTrack.
|
|
116
|
-
*/
|
|
117
|
-
readonly processedTrack: MediaStreamTrack;
|
|
118
|
-
/**
|
|
119
|
-
* Event emitter for VAD state and errors.
|
|
120
|
-
*/
|
|
121
|
-
readonly events: Emitter<AudioPipelineEvents>;
|
|
122
|
-
/**
|
|
123
|
-
* Current VAD state.
|
|
124
|
-
*/
|
|
125
|
-
readonly state: VADState;
|
|
126
|
-
/**
|
|
127
|
-
* Update configuration at runtime.
|
|
128
|
-
*/
|
|
129
|
-
setConfig(config: Partial<AudioProcessingConfig>): void;
|
|
130
|
-
/**
|
|
131
|
-
* Stop processing and release resources.
|
|
132
|
-
*/
|
|
133
|
-
dispose(): void;
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* Interface for a Noise Suppression Plugin.
|
|
137
|
-
*/
|
|
138
|
-
interface NoiseSuppressionPlugin {
|
|
139
|
-
name: string;
|
|
140
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["noiseSuppression"]): Promise<AudioNode>;
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Interface for a VAD Plugin.
|
|
144
|
-
*/
|
|
145
|
-
interface VADPlugin {
|
|
146
|
-
name: string;
|
|
147
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["vad"], onDecision: (probability: number) => void): Promise<AudioNode>;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
/**
|
|
151
|
-
* Manages a shared AudioContext for the application.
|
|
152
|
-
*/
|
|
153
|
-
/**
|
|
154
|
-
* Gets the shared AudioContext, creating it if necessary.
|
|
155
|
-
* @param options Optional AudioContextOptions
|
|
156
|
-
*/
|
|
157
|
-
declare function getAudioContext(options?: AudioContextOptions): AudioContext;
|
|
158
|
-
/**
|
|
159
|
-
* Registers a pipeline usage. Keeps track of active users.
|
|
160
|
-
*/
|
|
161
|
-
declare function registerPipeline(): void;
|
|
162
|
-
/**
|
|
163
|
-
* Unregisters a pipeline usage.
|
|
164
|
-
* Optionally closes the context if no pipelines are active (not implemented by default to avoid churn).
|
|
165
|
-
*/
|
|
166
|
-
declare function unregisterPipeline(): void;
|
|
167
|
-
/**
|
|
168
|
-
* Resumes the shared AudioContext.
|
|
169
|
-
* Should be called in response to a user gesture.
|
|
170
|
-
*/
|
|
171
|
-
declare function resumeAudioContext(): Promise<void>;
|
|
172
|
-
/**
|
|
173
|
-
* Suspends the shared AudioContext.
|
|
174
|
-
*/
|
|
175
|
-
declare function suspendAudioContext(): Promise<void>;
|
|
176
|
-
/**
|
|
177
|
-
* Closes the shared AudioContext and releases resources.
|
|
178
|
-
*/
|
|
179
|
-
declare function closeAudioContext(): Promise<void>;
|
|
180
|
-
|
|
181
|
-
declare function createAudioPipeline(sourceTrack: MediaStreamTrack, config?: AudioProcessingConfig): Promise<AudioPipelineHandle>;
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Attaches the audio processing pipeline to a LiveKit LocalAudioTrack.
|
|
185
|
-
* This replaces the underlying MediaStreamTrack with the processed one.
|
|
186
|
-
*/
|
|
187
|
-
declare function attachProcessingToTrack(track: LocalAudioTrack, config?: AudioProcessingConfig): Promise<AudioPipelineHandle>;
|
|
188
|
-
|
|
189
|
-
declare function registerNoiseSuppressionPlugin(plugin: NoiseSuppressionPlugin): void;
|
|
190
|
-
declare function registerVADPlugin(plugin: VADPlugin): void;
|
|
191
|
-
declare function getNoiseSuppressionPlugin(name?: string): NoiseSuppressionPlugin;
|
|
192
|
-
declare function getVADPlugin(name?: string): VADPlugin;
|
|
193
|
-
|
|
194
|
-
declare class RNNoisePlugin implements NoiseSuppressionPlugin {
|
|
195
|
-
name: string;
|
|
196
|
-
private wasmBuffer;
|
|
197
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["noiseSuppression"]): Promise<AudioNode>;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
declare class EnergyVADPlugin implements VADPlugin {
|
|
201
|
-
name: string;
|
|
202
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["vad"], onDecision: (probability: number) => void): Promise<AudioNode>;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
declare class VADStateMachine {
|
|
206
|
-
private config;
|
|
207
|
-
private currentState;
|
|
208
|
-
private lastSpeechTime;
|
|
209
|
-
private speechStartTime;
|
|
210
|
-
private frameDurationMs;
|
|
211
|
-
constructor(config: AudioProcessingConfig["vad"]);
|
|
212
|
-
updateConfig(config: Partial<AudioProcessingConfig["vad"]>): void;
|
|
213
|
-
processFrame(probability: number, timestamp: number): VADState;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
export { type AudioPipelineEvents, type AudioPipelineHandle, type AudioProcessingConfig, EnergyVADPlugin, type NoiseSuppressionPlugin, RNNoisePlugin, type VADPlugin, type VADState, VADStateMachine, attachProcessingToTrack, closeAudioContext, createAudioPipeline, getAudioContext, getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerPipeline, registerVADPlugin, resumeAudioContext, suspendAudioContext, unregisterPipeline };
|
|
1
|
+
export { AudioPipelineEvents, AudioPipelineHandle, AudioProcessingConfig, NoiseSuppressionPlugin, VADPlugin, VADState } from './types.mjs';
|
|
2
|
+
export { closeAudioContext, getAudioContext, registerPipeline, resumeAudioContext, suspendAudioContext, unregisterPipeline } from './context/audio-context.mjs';
|
|
3
|
+
export { createAudioPipeline } from './pipeline/audio-pipeline.mjs';
|
|
4
|
+
export { attachProcessingToTrack } from './livekit/integration.mjs';
|
|
5
|
+
export { getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerVADPlugin } from './extensibility/plugins.mjs';
|
|
6
|
+
export { RNNoisePlugin } from './noise-suppression/rnnoise-node.mjs';
|
|
7
|
+
export { EnergyVADPlugin } from './vad/vad-node.mjs';
|
|
8
|
+
export { VADStateMachine } from './vad/vad-state.mjs';
|
|
9
|
+
import 'mitt';
|
|
10
|
+
import 'livekit-client';
|
package/dist/index.d.ts
CHANGED
|
@@ -1,216 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
noiseSuppression?: {
|
|
12
|
-
enabled: boolean;
|
|
13
|
-
/**
|
|
14
|
-
* Path or URL to the RNNoise WASM binary.
|
|
15
|
-
* If not provided, the default from @sapphi-red/web-noise-suppressor will be used (if bundler supports it).
|
|
16
|
-
*/
|
|
17
|
-
wasmUrl?: string;
|
|
18
|
-
/**
|
|
19
|
-
* Path or URL to the RNNoise worklet script.
|
|
20
|
-
*/
|
|
21
|
-
workletUrl?: string;
|
|
22
|
-
/**
|
|
23
|
-
* Plugin name to use. Defaults to 'rnnoise-ns'.
|
|
24
|
-
*/
|
|
25
|
-
pluginName?: string;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* Voice Activity Detection (VAD) configuration.
|
|
29
|
-
*/
|
|
30
|
-
vad?: {
|
|
31
|
-
enabled: boolean;
|
|
32
|
-
/**
|
|
33
|
-
* Plugin name to use. Defaults to 'rnnoise-vad' or 'energy-vad'.
|
|
34
|
-
*/
|
|
35
|
-
pluginName?: string;
|
|
36
|
-
/**
|
|
37
|
-
* Probability threshold for speech onset (0-1).
|
|
38
|
-
* Default: 0.5
|
|
39
|
-
*/
|
|
40
|
-
startThreshold?: number;
|
|
41
|
-
/**
|
|
42
|
-
* Probability threshold for speech offset (0-1).
|
|
43
|
-
* Default: 0.4
|
|
44
|
-
*/
|
|
45
|
-
stopThreshold?: number;
|
|
46
|
-
/**
|
|
47
|
-
* Time in ms to wait after speech stops before considering it silent.
|
|
48
|
-
* Default: 300ms
|
|
49
|
-
*/
|
|
50
|
-
hangoverMs?: number;
|
|
51
|
-
/**
|
|
52
|
-
* Time in ms of audio to buffer before speech onset to avoid cutting the start.
|
|
53
|
-
* Default: 200ms
|
|
54
|
-
*/
|
|
55
|
-
preRollMs?: number;
|
|
56
|
-
};
|
|
57
|
-
/**
|
|
58
|
-
* Output gain and muting configuration.
|
|
59
|
-
*/
|
|
60
|
-
output?: {
|
|
61
|
-
/**
|
|
62
|
-
* Gain to apply when speaking (0-1+). Default: 1.0
|
|
63
|
-
*/
|
|
64
|
-
speechGain?: number;
|
|
65
|
-
/**
|
|
66
|
-
* Gain to apply when silent (0-1). Default: 0.0 (mute)
|
|
67
|
-
*/
|
|
68
|
-
silenceGain?: number;
|
|
69
|
-
/**
|
|
70
|
-
* Time in seconds to ramp gain changes. Default: 0.02
|
|
71
|
-
*/
|
|
72
|
-
gainRampTime?: number;
|
|
73
|
-
};
|
|
74
|
-
/**
|
|
75
|
-
* LiveKit integration configuration.
|
|
76
|
-
*/
|
|
77
|
-
livekit?: {
|
|
78
|
-
/**
|
|
79
|
-
* Whether to call track.mute()/unmute() on the LocalAudioTrack based on VAD.
|
|
80
|
-
* This saves bandwidth but has more signaling overhead.
|
|
81
|
-
* Default: false (uses gain gating only)
|
|
82
|
-
*/
|
|
83
|
-
manageTrackMute?: boolean;
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
/**
|
|
87
|
-
* Represents the state of Voice Activity Detection.
|
|
88
|
-
*/
|
|
89
|
-
interface VADState {
|
|
90
|
-
/**
|
|
91
|
-
* Whether speech is currently detected (after hysteresis).
|
|
92
|
-
*/
|
|
93
|
-
isSpeaking: boolean;
|
|
94
|
-
/**
|
|
95
|
-
* Raw probability of speech from the VAD model (0-1).
|
|
96
|
-
*/
|
|
97
|
-
probability: number;
|
|
98
|
-
/**
|
|
99
|
-
* Current state enum.
|
|
100
|
-
*/
|
|
101
|
-
state: "silent" | "speech_starting" | "speaking" | "speech_ending";
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* Events emitted by the audio pipeline.
|
|
105
|
-
*/
|
|
106
|
-
type AudioPipelineEvents = {
|
|
107
|
-
vadChange: VADState;
|
|
108
|
-
error: Error;
|
|
109
|
-
};
|
|
110
|
-
/**
|
|
111
|
-
* Handle to a running audio processing pipeline.
|
|
112
|
-
*/
|
|
113
|
-
interface AudioPipelineHandle {
|
|
114
|
-
/**
|
|
115
|
-
* The processed MediaStreamTrack.
|
|
116
|
-
*/
|
|
117
|
-
readonly processedTrack: MediaStreamTrack;
|
|
118
|
-
/**
|
|
119
|
-
* Event emitter for VAD state and errors.
|
|
120
|
-
*/
|
|
121
|
-
readonly events: Emitter<AudioPipelineEvents>;
|
|
122
|
-
/**
|
|
123
|
-
* Current VAD state.
|
|
124
|
-
*/
|
|
125
|
-
readonly state: VADState;
|
|
126
|
-
/**
|
|
127
|
-
* Update configuration at runtime.
|
|
128
|
-
*/
|
|
129
|
-
setConfig(config: Partial<AudioProcessingConfig>): void;
|
|
130
|
-
/**
|
|
131
|
-
* Stop processing and release resources.
|
|
132
|
-
*/
|
|
133
|
-
dispose(): void;
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* Interface for a Noise Suppression Plugin.
|
|
137
|
-
*/
|
|
138
|
-
interface NoiseSuppressionPlugin {
|
|
139
|
-
name: string;
|
|
140
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["noiseSuppression"]): Promise<AudioNode>;
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Interface for a VAD Plugin.
|
|
144
|
-
*/
|
|
145
|
-
interface VADPlugin {
|
|
146
|
-
name: string;
|
|
147
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["vad"], onDecision: (probability: number) => void): Promise<AudioNode>;
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
/**
|
|
151
|
-
* Manages a shared AudioContext for the application.
|
|
152
|
-
*/
|
|
153
|
-
/**
|
|
154
|
-
* Gets the shared AudioContext, creating it if necessary.
|
|
155
|
-
* @param options Optional AudioContextOptions
|
|
156
|
-
*/
|
|
157
|
-
declare function getAudioContext(options?: AudioContextOptions): AudioContext;
|
|
158
|
-
/**
|
|
159
|
-
* Registers a pipeline usage. Keeps track of active users.
|
|
160
|
-
*/
|
|
161
|
-
declare function registerPipeline(): void;
|
|
162
|
-
/**
|
|
163
|
-
* Unregisters a pipeline usage.
|
|
164
|
-
* Optionally closes the context if no pipelines are active (not implemented by default to avoid churn).
|
|
165
|
-
*/
|
|
166
|
-
declare function unregisterPipeline(): void;
|
|
167
|
-
/**
|
|
168
|
-
* Resumes the shared AudioContext.
|
|
169
|
-
* Should be called in response to a user gesture.
|
|
170
|
-
*/
|
|
171
|
-
declare function resumeAudioContext(): Promise<void>;
|
|
172
|
-
/**
|
|
173
|
-
* Suspends the shared AudioContext.
|
|
174
|
-
*/
|
|
175
|
-
declare function suspendAudioContext(): Promise<void>;
|
|
176
|
-
/**
|
|
177
|
-
* Closes the shared AudioContext and releases resources.
|
|
178
|
-
*/
|
|
179
|
-
declare function closeAudioContext(): Promise<void>;
|
|
180
|
-
|
|
181
|
-
declare function createAudioPipeline(sourceTrack: MediaStreamTrack, config?: AudioProcessingConfig): Promise<AudioPipelineHandle>;
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Attaches the audio processing pipeline to a LiveKit LocalAudioTrack.
|
|
185
|
-
* This replaces the underlying MediaStreamTrack with the processed one.
|
|
186
|
-
*/
|
|
187
|
-
declare function attachProcessingToTrack(track: LocalAudioTrack, config?: AudioProcessingConfig): Promise<AudioPipelineHandle>;
|
|
188
|
-
|
|
189
|
-
declare function registerNoiseSuppressionPlugin(plugin: NoiseSuppressionPlugin): void;
|
|
190
|
-
declare function registerVADPlugin(plugin: VADPlugin): void;
|
|
191
|
-
declare function getNoiseSuppressionPlugin(name?: string): NoiseSuppressionPlugin;
|
|
192
|
-
declare function getVADPlugin(name?: string): VADPlugin;
|
|
193
|
-
|
|
194
|
-
declare class RNNoisePlugin implements NoiseSuppressionPlugin {
|
|
195
|
-
name: string;
|
|
196
|
-
private wasmBuffer;
|
|
197
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["noiseSuppression"]): Promise<AudioNode>;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
declare class EnergyVADPlugin implements VADPlugin {
|
|
201
|
-
name: string;
|
|
202
|
-
createNode(context: AudioContext, config: AudioProcessingConfig["vad"], onDecision: (probability: number) => void): Promise<AudioNode>;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
declare class VADStateMachine {
|
|
206
|
-
private config;
|
|
207
|
-
private currentState;
|
|
208
|
-
private lastSpeechTime;
|
|
209
|
-
private speechStartTime;
|
|
210
|
-
private frameDurationMs;
|
|
211
|
-
constructor(config: AudioProcessingConfig["vad"]);
|
|
212
|
-
updateConfig(config: Partial<AudioProcessingConfig["vad"]>): void;
|
|
213
|
-
processFrame(probability: number, timestamp: number): VADState;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
export { type AudioPipelineEvents, type AudioPipelineHandle, type AudioProcessingConfig, EnergyVADPlugin, type NoiseSuppressionPlugin, RNNoisePlugin, type VADPlugin, type VADState, VADStateMachine, attachProcessingToTrack, closeAudioContext, createAudioPipeline, getAudioContext, getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerPipeline, registerVADPlugin, resumeAudioContext, suspendAudioContext, unregisterPipeline };
|
|
1
|
+
export { AudioPipelineEvents, AudioPipelineHandle, AudioProcessingConfig, NoiseSuppressionPlugin, VADPlugin, VADState } from './types.js';
|
|
2
|
+
export { closeAudioContext, getAudioContext, registerPipeline, resumeAudioContext, suspendAudioContext, unregisterPipeline } from './context/audio-context.js';
|
|
3
|
+
export { createAudioPipeline } from './pipeline/audio-pipeline.js';
|
|
4
|
+
export { attachProcessingToTrack } from './livekit/integration.js';
|
|
5
|
+
export { getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerVADPlugin } from './extensibility/plugins.js';
|
|
6
|
+
export { RNNoisePlugin } from './noise-suppression/rnnoise-node.js';
|
|
7
|
+
export { EnergyVADPlugin } from './vad/vad-node.js';
|
|
8
|
+
export { VADStateMachine } from './vad/vad-state.js';
|
|
9
|
+
import 'mitt';
|
|
10
|
+
import 'livekit-client';
|
package/dist/index.js
CHANGED
|
@@ -90,32 +90,33 @@ async function closeAudioContext() {
|
|
|
90
90
|
var import_mitt = __toESM(require("mitt"));
|
|
91
91
|
|
|
92
92
|
// src/noise-suppression/rnnoise-node.ts
|
|
93
|
-
var import_web_noise_suppressor = require("@sapphi-red/web-noise-suppressor");
|
|
94
|
-
var DEFAULT_WASM_URL = "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/rnnoise.wasm";
|
|
95
|
-
var DEFAULT_SIMD_WASM_URL = "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/rnnoise_simd.wasm";
|
|
96
|
-
var DEFAULT_WORKLET_URL = "https://unpkg.com/@sapphi-red/web-noise-suppressor@0.3.5/dist/noise-suppressor-worklet.min.js";
|
|
97
93
|
var RNNoisePlugin = class {
|
|
98
94
|
name = "rnnoise-ns";
|
|
99
95
|
wasmBuffer = null;
|
|
100
96
|
async createNode(context, config) {
|
|
97
|
+
const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
|
|
101
98
|
if (!config?.enabled) {
|
|
102
99
|
const pass = context.createGain();
|
|
103
100
|
return pass;
|
|
104
101
|
}
|
|
102
|
+
if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
"RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets and provide the URLs."
|
|
105
|
+
);
|
|
106
|
+
}
|
|
105
107
|
if (!this.wasmBuffer) {
|
|
106
|
-
this.wasmBuffer = await
|
|
107
|
-
url: config.wasmUrl
|
|
108
|
-
simdUrl:
|
|
109
|
-
// We should probably allow config for this too, but for now default is fine.
|
|
108
|
+
this.wasmBuffer = await loadRnnoise({
|
|
109
|
+
url: config.wasmUrl,
|
|
110
|
+
simdUrl: config.simdUrl
|
|
110
111
|
});
|
|
111
112
|
}
|
|
112
|
-
const workletUrl = config.workletUrl
|
|
113
|
+
const workletUrl = config.workletUrl;
|
|
113
114
|
try {
|
|
114
115
|
await context.audioWorklet.addModule(workletUrl);
|
|
115
116
|
} catch (e) {
|
|
116
117
|
console.warn("Failed to add RNNoise worklet module:", e);
|
|
117
118
|
}
|
|
118
|
-
const node = new
|
|
119
|
+
const node = new RnnoiseWorkletNode(context, {
|
|
119
120
|
wasmBinary: this.wasmBuffer,
|
|
120
121
|
maxChannels: 1
|
|
121
122
|
// Mono for now
|