@tensamin/audio 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -2
- package/dist/chunk-FS635GMR.mjs +47 -0
- package/dist/chunk-HFSKQ33X.mjs +38 -0
- package/{src/vad/vad-state.ts → dist/chunk-JJASCVEW.mjs} +21 -33
- package/dist/chunk-OZ7KMC4S.mjs +46 -0
- package/dist/chunk-QU7E5HBA.mjs +106 -0
- package/dist/chunk-SDTOKWM2.mjs +39 -0
- package/{src/vad/vad-node.ts → dist/chunk-UMU2KIB6.mjs} +10 -20
- package/dist/chunk-WBQAMGXK.mjs +0 -0
- package/dist/context/audio-context.d.mts +32 -0
- package/dist/context/audio-context.d.ts +32 -0
- package/dist/context/audio-context.js +75 -0
- package/dist/context/audio-context.mjs +16 -0
- package/dist/extensibility/plugins.d.mts +9 -0
- package/dist/extensibility/plugins.d.ts +9 -0
- package/dist/extensibility/plugins.js +180 -0
- package/dist/extensibility/plugins.mjs +14 -0
- package/dist/index.d.mts +10 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +419 -0
- package/dist/index.mjs +47 -0
- package/dist/livekit/integration.d.mts +11 -0
- package/dist/livekit/integration.d.ts +11 -0
- package/dist/livekit/integration.js +368 -0
- package/dist/livekit/integration.mjs +12 -0
- package/dist/noise-suppression/rnnoise-node.d.mts +10 -0
- package/dist/noise-suppression/rnnoise-node.d.ts +10 -0
- package/dist/noise-suppression/rnnoise-node.js +73 -0
- package/dist/noise-suppression/rnnoise-node.mjs +6 -0
- package/dist/pipeline/audio-pipeline.d.mts +6 -0
- package/dist/pipeline/audio-pipeline.d.ts +6 -0
- package/dist/pipeline/audio-pipeline.js +335 -0
- package/dist/pipeline/audio-pipeline.mjs +11 -0
- package/dist/types.d.mts +155 -0
- package/dist/types.d.ts +155 -0
- package/dist/types.js +18 -0
- package/dist/types.mjs +1 -0
- package/dist/vad/vad-node.d.mts +9 -0
- package/dist/vad/vad-node.d.ts +9 -0
- package/dist/vad/vad-node.js +92 -0
- package/dist/vad/vad-node.mjs +6 -0
- package/dist/vad/vad-state.d.mts +15 -0
- package/dist/vad/vad-state.d.ts +15 -0
- package/dist/vad/vad-state.js +83 -0
- package/dist/vad/vad-state.mjs +6 -0
- package/package.json +11 -14
- package/.github/workflows/publish.yml +0 -23
- package/src/context/audio-context.ts +0 -69
- package/src/extensibility/plugins.ts +0 -45
- package/src/index.ts +0 -8
- package/src/livekit/integration.ts +0 -61
- package/src/noise-suppression/rnnoise-node.ts +0 -62
- package/src/pipeline/audio-pipeline.ts +0 -154
- package/src/types.ts +0 -167
- package/tsconfig.json +0 -29
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/extensibility/plugins.ts
|
|
31
|
+
var plugins_exports = {};
|
|
32
|
+
__export(plugins_exports, {
|
|
33
|
+
getNoiseSuppressionPlugin: () => getNoiseSuppressionPlugin,
|
|
34
|
+
getVADPlugin: () => getVADPlugin,
|
|
35
|
+
registerNoiseSuppressionPlugin: () => registerNoiseSuppressionPlugin,
|
|
36
|
+
registerVADPlugin: () => registerVADPlugin
|
|
37
|
+
});
|
|
38
|
+
module.exports = __toCommonJS(plugins_exports);
|
|
39
|
+
|
|
40
|
+
// src/noise-suppression/rnnoise-node.ts
|
|
41
|
+
var RNNoisePlugin = class {
|
|
42
|
+
name = "rnnoise-ns";
|
|
43
|
+
wasmBuffer = null;
|
|
44
|
+
async createNode(context, config) {
|
|
45
|
+
const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
|
|
46
|
+
if (!config?.enabled) {
|
|
47
|
+
const pass = context.createGain();
|
|
48
|
+
return pass;
|
|
49
|
+
}
|
|
50
|
+
if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
|
|
51
|
+
throw new Error(
|
|
52
|
+
"RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets and provide the URLs."
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
if (!this.wasmBuffer) {
|
|
56
|
+
this.wasmBuffer = await loadRnnoise({
|
|
57
|
+
url: config.wasmUrl,
|
|
58
|
+
simdUrl: config.simdUrl
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
const workletUrl = config.workletUrl;
|
|
62
|
+
try {
|
|
63
|
+
await context.audioWorklet.addModule(workletUrl);
|
|
64
|
+
} catch (e) {
|
|
65
|
+
console.warn("Failed to add RNNoise worklet module:", e);
|
|
66
|
+
}
|
|
67
|
+
const node = new RnnoiseWorkletNode(context, {
|
|
68
|
+
wasmBinary: this.wasmBuffer,
|
|
69
|
+
maxChannels: 1
|
|
70
|
+
// Mono for now
|
|
71
|
+
});
|
|
72
|
+
return node;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
// src/vad/vad-node.ts
|
|
77
|
+
var energyVadWorkletCode = `
|
|
78
|
+
class EnergyVadProcessor extends AudioWorkletProcessor {
|
|
79
|
+
constructor() {
|
|
80
|
+
super();
|
|
81
|
+
this.smoothing = 0.95;
|
|
82
|
+
this.energy = 0;
|
|
83
|
+
this.noiseFloor = 0.001;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
process(inputs, outputs, parameters) {
|
|
87
|
+
const input = inputs[0];
|
|
88
|
+
if (!input || !input.length) return true;
|
|
89
|
+
const channel = input[0];
|
|
90
|
+
|
|
91
|
+
// Calculate RMS
|
|
92
|
+
let sum = 0;
|
|
93
|
+
for (let i = 0; i < channel.length; i++) {
|
|
94
|
+
sum += channel[i] * channel[i];
|
|
95
|
+
}
|
|
96
|
+
const rms = Math.sqrt(sum / channel.length);
|
|
97
|
+
|
|
98
|
+
// Simple adaptive noise floor (very basic)
|
|
99
|
+
if (rms < this.noiseFloor) {
|
|
100
|
+
this.noiseFloor = this.noiseFloor * 0.99 + rms * 0.01;
|
|
101
|
+
} else {
|
|
102
|
+
this.noiseFloor = this.noiseFloor * 0.999 + rms * 0.001;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Calculate "probability" based on SNR
|
|
106
|
+
// This is a heuristic mapping from energy to 0-1
|
|
107
|
+
const snr = rms / (this.noiseFloor + 1e-6);
|
|
108
|
+
const probability = Math.min(1, Math.max(0, (snr - 1.5) / 10)); // Arbitrary scaling
|
|
109
|
+
|
|
110
|
+
this.port.postMessage({ probability });
|
|
111
|
+
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
registerProcessor('energy-vad-processor', EnergyVadProcessor);
|
|
116
|
+
`;
|
|
117
|
+
var EnergyVADPlugin = class {
|
|
118
|
+
name = "energy-vad";
|
|
119
|
+
async createNode(context, config, onDecision) {
|
|
120
|
+
const blob = new Blob([energyVadWorkletCode], {
|
|
121
|
+
type: "application/javascript"
|
|
122
|
+
});
|
|
123
|
+
const url = URL.createObjectURL(blob);
|
|
124
|
+
try {
|
|
125
|
+
await context.audioWorklet.addModule(url);
|
|
126
|
+
} catch (e) {
|
|
127
|
+
console.warn("Failed to add Energy VAD worklet:", e);
|
|
128
|
+
throw e;
|
|
129
|
+
} finally {
|
|
130
|
+
URL.revokeObjectURL(url);
|
|
131
|
+
}
|
|
132
|
+
const node = new AudioWorkletNode(context, "energy-vad-processor");
|
|
133
|
+
node.port.onmessage = (event) => {
|
|
134
|
+
const { probability } = event.data;
|
|
135
|
+
onDecision(probability);
|
|
136
|
+
};
|
|
137
|
+
return node;
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
// src/extensibility/plugins.ts
|
|
142
|
+
var nsPlugins = /* @__PURE__ */ new Map();
|
|
143
|
+
var vadPlugins = /* @__PURE__ */ new Map();
|
|
144
|
+
var defaultNs = new RNNoisePlugin();
|
|
145
|
+
nsPlugins.set(defaultNs.name, defaultNs);
|
|
146
|
+
var defaultVad = new EnergyVADPlugin();
|
|
147
|
+
vadPlugins.set(defaultVad.name, defaultVad);
|
|
148
|
+
function registerNoiseSuppressionPlugin(plugin) {
|
|
149
|
+
nsPlugins.set(plugin.name, plugin);
|
|
150
|
+
}
|
|
151
|
+
function registerVADPlugin(plugin) {
|
|
152
|
+
vadPlugins.set(plugin.name, plugin);
|
|
153
|
+
}
|
|
154
|
+
function getNoiseSuppressionPlugin(name) {
|
|
155
|
+
if (!name) return defaultNs;
|
|
156
|
+
const plugin = nsPlugins.get(name);
|
|
157
|
+
if (!plugin) {
|
|
158
|
+
console.warn(
|
|
159
|
+
`Noise suppression plugin '${name}' not found, falling back to default.`
|
|
160
|
+
);
|
|
161
|
+
return defaultNs;
|
|
162
|
+
}
|
|
163
|
+
return plugin;
|
|
164
|
+
}
|
|
165
|
+
function getVADPlugin(name) {
|
|
166
|
+
if (!name) return defaultVad;
|
|
167
|
+
const plugin = vadPlugins.get(name);
|
|
168
|
+
if (!plugin) {
|
|
169
|
+
console.warn(`VAD plugin '${name}' not found, falling back to default.`);
|
|
170
|
+
return defaultVad;
|
|
171
|
+
}
|
|
172
|
+
return plugin;
|
|
173
|
+
}
|
|
174
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
175
|
+
0 && (module.exports = {
|
|
176
|
+
getNoiseSuppressionPlugin,
|
|
177
|
+
getVADPlugin,
|
|
178
|
+
registerNoiseSuppressionPlugin,
|
|
179
|
+
registerVADPlugin
|
|
180
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getNoiseSuppressionPlugin,
|
|
3
|
+
getVADPlugin,
|
|
4
|
+
registerNoiseSuppressionPlugin,
|
|
5
|
+
registerVADPlugin
|
|
6
|
+
} from "../chunk-FS635GMR.mjs";
|
|
7
|
+
import "../chunk-SDTOKWM2.mjs";
|
|
8
|
+
import "../chunk-UMU2KIB6.mjs";
|
|
9
|
+
export {
|
|
10
|
+
getNoiseSuppressionPlugin,
|
|
11
|
+
getVADPlugin,
|
|
12
|
+
registerNoiseSuppressionPlugin,
|
|
13
|
+
registerVADPlugin
|
|
14
|
+
};
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { AudioPipelineEvents, AudioPipelineHandle, AudioProcessingConfig, NoiseSuppressionPlugin, VADPlugin, VADState } from './types.mjs';
|
|
2
|
+
export { closeAudioContext, getAudioContext, registerPipeline, resumeAudioContext, suspendAudioContext, unregisterPipeline } from './context/audio-context.mjs';
|
|
3
|
+
export { createAudioPipeline } from './pipeline/audio-pipeline.mjs';
|
|
4
|
+
export { attachProcessingToTrack } from './livekit/integration.mjs';
|
|
5
|
+
export { getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerVADPlugin } from './extensibility/plugins.mjs';
|
|
6
|
+
export { RNNoisePlugin } from './noise-suppression/rnnoise-node.mjs';
|
|
7
|
+
export { EnergyVADPlugin } from './vad/vad-node.mjs';
|
|
8
|
+
export { VADStateMachine } from './vad/vad-state.mjs';
|
|
9
|
+
import 'mitt';
|
|
10
|
+
import 'livekit-client';
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { AudioPipelineEvents, AudioPipelineHandle, AudioProcessingConfig, NoiseSuppressionPlugin, VADPlugin, VADState } from './types.js';
|
|
2
|
+
export { closeAudioContext, getAudioContext, registerPipeline, resumeAudioContext, suspendAudioContext, unregisterPipeline } from './context/audio-context.js';
|
|
3
|
+
export { createAudioPipeline } from './pipeline/audio-pipeline.js';
|
|
4
|
+
export { attachProcessingToTrack } from './livekit/integration.js';
|
|
5
|
+
export { getNoiseSuppressionPlugin, getVADPlugin, registerNoiseSuppressionPlugin, registerVADPlugin } from './extensibility/plugins.js';
|
|
6
|
+
export { RNNoisePlugin } from './noise-suppression/rnnoise-node.js';
|
|
7
|
+
export { EnergyVADPlugin } from './vad/vad-node.js';
|
|
8
|
+
export { VADStateMachine } from './vad/vad-state.js';
|
|
9
|
+
import 'mitt';
|
|
10
|
+
import 'livekit-client';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/index.ts
|
|
31
|
+
var index_exports = {};
|
|
32
|
+
__export(index_exports, {
|
|
33
|
+
EnergyVADPlugin: () => EnergyVADPlugin,
|
|
34
|
+
RNNoisePlugin: () => RNNoisePlugin,
|
|
35
|
+
VADStateMachine: () => VADStateMachine,
|
|
36
|
+
attachProcessingToTrack: () => attachProcessingToTrack,
|
|
37
|
+
closeAudioContext: () => closeAudioContext,
|
|
38
|
+
createAudioPipeline: () => createAudioPipeline,
|
|
39
|
+
getAudioContext: () => getAudioContext,
|
|
40
|
+
getNoiseSuppressionPlugin: () => getNoiseSuppressionPlugin,
|
|
41
|
+
getVADPlugin: () => getVADPlugin,
|
|
42
|
+
registerNoiseSuppressionPlugin: () => registerNoiseSuppressionPlugin,
|
|
43
|
+
registerPipeline: () => registerPipeline,
|
|
44
|
+
registerVADPlugin: () => registerVADPlugin,
|
|
45
|
+
resumeAudioContext: () => resumeAudioContext,
|
|
46
|
+
suspendAudioContext: () => suspendAudioContext,
|
|
47
|
+
unregisterPipeline: () => unregisterPipeline
|
|
48
|
+
});
|
|
49
|
+
module.exports = __toCommonJS(index_exports);
|
|
50
|
+
|
|
51
|
+
// src/context/audio-context.ts
|
|
52
|
+
var sharedContext = null;
|
|
53
|
+
var activePipelines = 0;
|
|
54
|
+
function getAudioContext(options) {
|
|
55
|
+
if (typeof window === "undefined" || typeof AudioContext === "undefined") {
|
|
56
|
+
throw new Error(
|
|
57
|
+
"AudioContext is not supported in this environment (browser only)."
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
if (!sharedContext || sharedContext.state === "closed") {
|
|
61
|
+
sharedContext = new AudioContext(options);
|
|
62
|
+
}
|
|
63
|
+
return sharedContext;
|
|
64
|
+
}
|
|
65
|
+
function registerPipeline() {
|
|
66
|
+
activePipelines++;
|
|
67
|
+
}
|
|
68
|
+
function unregisterPipeline() {
|
|
69
|
+
activePipelines = Math.max(0, activePipelines - 1);
|
|
70
|
+
}
|
|
71
|
+
async function resumeAudioContext() {
|
|
72
|
+
if (sharedContext && sharedContext.state === "suspended") {
|
|
73
|
+
await sharedContext.resume();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
async function suspendAudioContext() {
|
|
77
|
+
if (sharedContext && sharedContext.state === "running") {
|
|
78
|
+
await sharedContext.suspend();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async function closeAudioContext() {
|
|
82
|
+
if (sharedContext && sharedContext.state !== "closed") {
|
|
83
|
+
await sharedContext.close();
|
|
84
|
+
}
|
|
85
|
+
sharedContext = null;
|
|
86
|
+
activePipelines = 0;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// src/pipeline/audio-pipeline.ts
|
|
90
|
+
var import_mitt = __toESM(require("mitt"));
|
|
91
|
+
|
|
92
|
+
// src/noise-suppression/rnnoise-node.ts
|
|
93
|
+
var RNNoisePlugin = class {
|
|
94
|
+
name = "rnnoise-ns";
|
|
95
|
+
wasmBuffer = null;
|
|
96
|
+
async createNode(context, config) {
|
|
97
|
+
const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
|
|
98
|
+
if (!config?.enabled) {
|
|
99
|
+
const pass = context.createGain();
|
|
100
|
+
return pass;
|
|
101
|
+
}
|
|
102
|
+
if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
"RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets and provide the URLs."
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
if (!this.wasmBuffer) {
|
|
108
|
+
this.wasmBuffer = await loadRnnoise({
|
|
109
|
+
url: config.wasmUrl,
|
|
110
|
+
simdUrl: config.simdUrl
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
const workletUrl = config.workletUrl;
|
|
114
|
+
try {
|
|
115
|
+
await context.audioWorklet.addModule(workletUrl);
|
|
116
|
+
} catch (e) {
|
|
117
|
+
console.warn("Failed to add RNNoise worklet module:", e);
|
|
118
|
+
}
|
|
119
|
+
const node = new RnnoiseWorkletNode(context, {
|
|
120
|
+
wasmBinary: this.wasmBuffer,
|
|
121
|
+
maxChannels: 1
|
|
122
|
+
// Mono for now
|
|
123
|
+
});
|
|
124
|
+
return node;
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
// src/vad/vad-node.ts
|
|
129
|
+
var energyVadWorkletCode = `
|
|
130
|
+
class EnergyVadProcessor extends AudioWorkletProcessor {
|
|
131
|
+
constructor() {
|
|
132
|
+
super();
|
|
133
|
+
this.smoothing = 0.95;
|
|
134
|
+
this.energy = 0;
|
|
135
|
+
this.noiseFloor = 0.001;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
process(inputs, outputs, parameters) {
|
|
139
|
+
const input = inputs[0];
|
|
140
|
+
if (!input || !input.length) return true;
|
|
141
|
+
const channel = input[0];
|
|
142
|
+
|
|
143
|
+
// Calculate RMS
|
|
144
|
+
let sum = 0;
|
|
145
|
+
for (let i = 0; i < channel.length; i++) {
|
|
146
|
+
sum += channel[i] * channel[i];
|
|
147
|
+
}
|
|
148
|
+
const rms = Math.sqrt(sum / channel.length);
|
|
149
|
+
|
|
150
|
+
// Simple adaptive noise floor (very basic)
|
|
151
|
+
if (rms < this.noiseFloor) {
|
|
152
|
+
this.noiseFloor = this.noiseFloor * 0.99 + rms * 0.01;
|
|
153
|
+
} else {
|
|
154
|
+
this.noiseFloor = this.noiseFloor * 0.999 + rms * 0.001;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Calculate "probability" based on SNR
|
|
158
|
+
// This is a heuristic mapping from energy to 0-1
|
|
159
|
+
const snr = rms / (this.noiseFloor + 1e-6);
|
|
160
|
+
const probability = Math.min(1, Math.max(0, (snr - 1.5) / 10)); // Arbitrary scaling
|
|
161
|
+
|
|
162
|
+
this.port.postMessage({ probability });
|
|
163
|
+
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
registerProcessor('energy-vad-processor', EnergyVadProcessor);
|
|
168
|
+
`;
|
|
169
|
+
var EnergyVADPlugin = class {
|
|
170
|
+
name = "energy-vad";
|
|
171
|
+
async createNode(context, config, onDecision) {
|
|
172
|
+
const blob = new Blob([energyVadWorkletCode], {
|
|
173
|
+
type: "application/javascript"
|
|
174
|
+
});
|
|
175
|
+
const url = URL.createObjectURL(blob);
|
|
176
|
+
try {
|
|
177
|
+
await context.audioWorklet.addModule(url);
|
|
178
|
+
} catch (e) {
|
|
179
|
+
console.warn("Failed to add Energy VAD worklet:", e);
|
|
180
|
+
throw e;
|
|
181
|
+
} finally {
|
|
182
|
+
URL.revokeObjectURL(url);
|
|
183
|
+
}
|
|
184
|
+
const node = new AudioWorkletNode(context, "energy-vad-processor");
|
|
185
|
+
node.port.onmessage = (event) => {
|
|
186
|
+
const { probability } = event.data;
|
|
187
|
+
onDecision(probability);
|
|
188
|
+
};
|
|
189
|
+
return node;
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
|
|
193
|
+
// src/extensibility/plugins.ts
|
|
194
|
+
var nsPlugins = /* @__PURE__ */ new Map();
|
|
195
|
+
var vadPlugins = /* @__PURE__ */ new Map();
|
|
196
|
+
var defaultNs = new RNNoisePlugin();
|
|
197
|
+
nsPlugins.set(defaultNs.name, defaultNs);
|
|
198
|
+
var defaultVad = new EnergyVADPlugin();
|
|
199
|
+
vadPlugins.set(defaultVad.name, defaultVad);
|
|
200
|
+
function registerNoiseSuppressionPlugin(plugin) {
|
|
201
|
+
nsPlugins.set(plugin.name, plugin);
|
|
202
|
+
}
|
|
203
|
+
function registerVADPlugin(plugin) {
|
|
204
|
+
vadPlugins.set(plugin.name, plugin);
|
|
205
|
+
}
|
|
206
|
+
function getNoiseSuppressionPlugin(name) {
|
|
207
|
+
if (!name) return defaultNs;
|
|
208
|
+
const plugin = nsPlugins.get(name);
|
|
209
|
+
if (!plugin) {
|
|
210
|
+
console.warn(
|
|
211
|
+
`Noise suppression plugin '${name}' not found, falling back to default.`
|
|
212
|
+
);
|
|
213
|
+
return defaultNs;
|
|
214
|
+
}
|
|
215
|
+
return plugin;
|
|
216
|
+
}
|
|
217
|
+
function getVADPlugin(name) {
|
|
218
|
+
if (!name) return defaultVad;
|
|
219
|
+
const plugin = vadPlugins.get(name);
|
|
220
|
+
if (!plugin) {
|
|
221
|
+
console.warn(`VAD plugin '${name}' not found, falling back to default.`);
|
|
222
|
+
return defaultVad;
|
|
223
|
+
}
|
|
224
|
+
return plugin;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// src/vad/vad-state.ts
|
|
228
|
+
var VADStateMachine = class {
|
|
229
|
+
config;
|
|
230
|
+
currentState = "silent";
|
|
231
|
+
lastSpeechTime = 0;
|
|
232
|
+
speechStartTime = 0;
|
|
233
|
+
frameDurationMs = 20;
|
|
234
|
+
// Assumed frame duration, updated by calls
|
|
235
|
+
constructor(config) {
|
|
236
|
+
this.config = {
|
|
237
|
+
enabled: config?.enabled ?? true,
|
|
238
|
+
pluginName: config?.pluginName ?? "energy-vad",
|
|
239
|
+
startThreshold: config?.startThreshold ?? 0.5,
|
|
240
|
+
stopThreshold: config?.stopThreshold ?? 0.4,
|
|
241
|
+
hangoverMs: config?.hangoverMs ?? 300,
|
|
242
|
+
preRollMs: config?.preRollMs ?? 200
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
updateConfig(config) {
|
|
246
|
+
this.config = { ...this.config, ...config };
|
|
247
|
+
}
|
|
248
|
+
processFrame(probability, timestamp) {
|
|
249
|
+
const { startThreshold, stopThreshold, hangoverMs } = this.config;
|
|
250
|
+
let newState = this.currentState;
|
|
251
|
+
if (this.currentState === "silent" || this.currentState === "speech_ending") {
|
|
252
|
+
if (probability >= startThreshold) {
|
|
253
|
+
newState = "speech_starting";
|
|
254
|
+
this.speechStartTime = timestamp;
|
|
255
|
+
this.lastSpeechTime = timestamp;
|
|
256
|
+
} else {
|
|
257
|
+
newState = "silent";
|
|
258
|
+
}
|
|
259
|
+
} else if (this.currentState === "speech_starting" || this.currentState === "speaking") {
|
|
260
|
+
if (probability >= stopThreshold) {
|
|
261
|
+
newState = "speaking";
|
|
262
|
+
this.lastSpeechTime = timestamp;
|
|
263
|
+
} else {
|
|
264
|
+
const timeSinceSpeech = timestamp - this.lastSpeechTime;
|
|
265
|
+
if (timeSinceSpeech < hangoverMs) {
|
|
266
|
+
newState = "speaking";
|
|
267
|
+
} else {
|
|
268
|
+
newState = "speech_ending";
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (newState === "speech_starting") newState = "speaking";
|
|
273
|
+
if (newState === "speech_ending") newState = "silent";
|
|
274
|
+
this.currentState = newState;
|
|
275
|
+
return {
|
|
276
|
+
isSpeaking: newState === "speaking",
|
|
277
|
+
probability,
|
|
278
|
+
state: newState
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
// src/pipeline/audio-pipeline.ts
|
|
284
|
+
async function createAudioPipeline(sourceTrack, config = {}) {
|
|
285
|
+
const context = getAudioContext();
|
|
286
|
+
registerPipeline();
|
|
287
|
+
const fullConfig = {
|
|
288
|
+
noiseSuppression: { enabled: true, ...config.noiseSuppression },
|
|
289
|
+
vad: { enabled: true, ...config.vad },
|
|
290
|
+
output: {
|
|
291
|
+
speechGain: 1,
|
|
292
|
+
silenceGain: 0,
|
|
293
|
+
gainRampTime: 0.02,
|
|
294
|
+
...config.output
|
|
295
|
+
},
|
|
296
|
+
livekit: { manageTrackMute: false, ...config.livekit }
|
|
297
|
+
};
|
|
298
|
+
const sourceStream = new MediaStream([sourceTrack]);
|
|
299
|
+
const sourceNode = context.createMediaStreamSource(sourceStream);
|
|
300
|
+
const nsPlugin = getNoiseSuppressionPlugin(
|
|
301
|
+
fullConfig.noiseSuppression?.pluginName
|
|
302
|
+
);
|
|
303
|
+
const nsNode = await nsPlugin.createNode(
|
|
304
|
+
context,
|
|
305
|
+
fullConfig.noiseSuppression
|
|
306
|
+
);
|
|
307
|
+
const vadPlugin = getVADPlugin(fullConfig.vad?.pluginName);
|
|
308
|
+
const vadStateMachine = new VADStateMachine(fullConfig.vad);
|
|
309
|
+
const emitter = (0, import_mitt.default)();
|
|
310
|
+
const vadNode = await vadPlugin.createNode(
|
|
311
|
+
context,
|
|
312
|
+
fullConfig.vad,
|
|
313
|
+
(prob) => {
|
|
314
|
+
const timestamp = context.currentTime * 1e3;
|
|
315
|
+
const newState = vadStateMachine.processFrame(prob, timestamp);
|
|
316
|
+
if (newState.state !== lastVadState.state || Math.abs(newState.probability - lastVadState.probability) > 0.1) {
|
|
317
|
+
emitter.emit("vadChange", newState);
|
|
318
|
+
lastVadState = newState;
|
|
319
|
+
updateGain(newState);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
);
|
|
323
|
+
let lastVadState = {
|
|
324
|
+
isSpeaking: false,
|
|
325
|
+
probability: 0,
|
|
326
|
+
state: "silent"
|
|
327
|
+
};
|
|
328
|
+
const splitter = context.createGain();
|
|
329
|
+
sourceNode.connect(nsNode);
|
|
330
|
+
nsNode.connect(splitter);
|
|
331
|
+
splitter.connect(vadNode);
|
|
332
|
+
const delayNode = context.createDelay(1);
|
|
333
|
+
const preRollSeconds = (fullConfig.vad?.preRollMs ?? 200) / 1e3;
|
|
334
|
+
delayNode.delayTime.value = preRollSeconds;
|
|
335
|
+
const gainNode = context.createGain();
|
|
336
|
+
gainNode.gain.value = fullConfig.output?.silenceGain ?? 0;
|
|
337
|
+
const destination = context.createMediaStreamDestination();
|
|
338
|
+
splitter.connect(delayNode);
|
|
339
|
+
delayNode.connect(gainNode);
|
|
340
|
+
gainNode.connect(destination);
|
|
341
|
+
function updateGain(state) {
|
|
342
|
+
const { speechGain, silenceGain, gainRampTime } = fullConfig.output;
|
|
343
|
+
const targetGain = state.isSpeaking ? speechGain ?? 1 : silenceGain ?? 0;
|
|
344
|
+
const now = context.currentTime;
|
|
345
|
+
gainNode.gain.setTargetAtTime(targetGain, now, gainRampTime ?? 0.02);
|
|
346
|
+
}
|
|
347
|
+
function dispose() {
|
|
348
|
+
sourceNode.disconnect();
|
|
349
|
+
nsNode.disconnect();
|
|
350
|
+
splitter.disconnect();
|
|
351
|
+
vadNode.disconnect();
|
|
352
|
+
delayNode.disconnect();
|
|
353
|
+
gainNode.disconnect();
|
|
354
|
+
destination.stream.getTracks().forEach((t) => t.stop());
|
|
355
|
+
unregisterPipeline();
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
processedTrack: destination.stream.getAudioTracks()[0],
|
|
359
|
+
events: emitter,
|
|
360
|
+
get state() {
|
|
361
|
+
return lastVadState;
|
|
362
|
+
},
|
|
363
|
+
setConfig: (newConfig) => {
|
|
364
|
+
if (newConfig.vad) {
|
|
365
|
+
vadStateMachine.updateConfig(newConfig.vad);
|
|
366
|
+
}
|
|
367
|
+
},
|
|
368
|
+
dispose
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// src/livekit/integration.ts
|
|
373
|
+
async function attachProcessingToTrack(track, config = {}) {
|
|
374
|
+
const originalTrack = track.mediaStreamTrack;
|
|
375
|
+
const pipeline = await createAudioPipeline(originalTrack, config);
|
|
376
|
+
await track.replaceTrack(pipeline.processedTrack);
|
|
377
|
+
if (config.livekit?.manageTrackMute) {
|
|
378
|
+
let isVadMuted = false;
|
|
379
|
+
pipeline.events.on("vadChange", async (state) => {
|
|
380
|
+
if (state.isSpeaking) {
|
|
381
|
+
if (isVadMuted) {
|
|
382
|
+
await track.unmute();
|
|
383
|
+
isVadMuted = false;
|
|
384
|
+
}
|
|
385
|
+
} else {
|
|
386
|
+
if (!track.isMuted) {
|
|
387
|
+
await track.mute();
|
|
388
|
+
isVadMuted = true;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
const originalDispose = pipeline.dispose;
|
|
394
|
+
pipeline.dispose = () => {
|
|
395
|
+
if (originalTrack.readyState === "live") {
|
|
396
|
+
track.replaceTrack(originalTrack).catch(console.error);
|
|
397
|
+
}
|
|
398
|
+
originalDispose();
|
|
399
|
+
};
|
|
400
|
+
return pipeline;
|
|
401
|
+
}
|
|
402
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
403
|
+
0 && (module.exports = {
|
|
404
|
+
EnergyVADPlugin,
|
|
405
|
+
RNNoisePlugin,
|
|
406
|
+
VADStateMachine,
|
|
407
|
+
attachProcessingToTrack,
|
|
408
|
+
closeAudioContext,
|
|
409
|
+
createAudioPipeline,
|
|
410
|
+
getAudioContext,
|
|
411
|
+
getNoiseSuppressionPlugin,
|
|
412
|
+
getVADPlugin,
|
|
413
|
+
registerNoiseSuppressionPlugin,
|
|
414
|
+
registerPipeline,
|
|
415
|
+
registerVADPlugin,
|
|
416
|
+
resumeAudioContext,
|
|
417
|
+
suspendAudioContext,
|
|
418
|
+
unregisterPipeline
|
|
419
|
+
});
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import "./chunk-WBQAMGXK.mjs";
|
|
2
|
+
import {
|
|
3
|
+
attachProcessingToTrack
|
|
4
|
+
} from "./chunk-HFSKQ33X.mjs";
|
|
5
|
+
import {
|
|
6
|
+
createAudioPipeline
|
|
7
|
+
} from "./chunk-QU7E5HBA.mjs";
|
|
8
|
+
import {
|
|
9
|
+
VADStateMachine
|
|
10
|
+
} from "./chunk-JJASCVEW.mjs";
|
|
11
|
+
import {
|
|
12
|
+
closeAudioContext,
|
|
13
|
+
getAudioContext,
|
|
14
|
+
registerPipeline,
|
|
15
|
+
resumeAudioContext,
|
|
16
|
+
suspendAudioContext,
|
|
17
|
+
unregisterPipeline
|
|
18
|
+
} from "./chunk-OZ7KMC4S.mjs";
|
|
19
|
+
import {
|
|
20
|
+
getNoiseSuppressionPlugin,
|
|
21
|
+
getVADPlugin,
|
|
22
|
+
registerNoiseSuppressionPlugin,
|
|
23
|
+
registerVADPlugin
|
|
24
|
+
} from "./chunk-FS635GMR.mjs";
|
|
25
|
+
import {
|
|
26
|
+
RNNoisePlugin
|
|
27
|
+
} from "./chunk-SDTOKWM2.mjs";
|
|
28
|
+
import {
|
|
29
|
+
EnergyVADPlugin
|
|
30
|
+
} from "./chunk-UMU2KIB6.mjs";
|
|
31
|
+
export {
|
|
32
|
+
EnergyVADPlugin,
|
|
33
|
+
RNNoisePlugin,
|
|
34
|
+
VADStateMachine,
|
|
35
|
+
attachProcessingToTrack,
|
|
36
|
+
closeAudioContext,
|
|
37
|
+
createAudioPipeline,
|
|
38
|
+
getAudioContext,
|
|
39
|
+
getNoiseSuppressionPlugin,
|
|
40
|
+
getVADPlugin,
|
|
41
|
+
registerNoiseSuppressionPlugin,
|
|
42
|
+
registerPipeline,
|
|
43
|
+
registerVADPlugin,
|
|
44
|
+
resumeAudioContext,
|
|
45
|
+
suspendAudioContext,
|
|
46
|
+
unregisterPipeline
|
|
47
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { LocalAudioTrack } from 'livekit-client';
|
|
2
|
+
import { AudioProcessingConfig, AudioPipelineHandle } from '../types.mjs';
|
|
3
|
+
import 'mitt';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Attaches the audio processing pipeline to a LiveKit LocalAudioTrack.
|
|
7
|
+
* This replaces the underlying MediaStreamTrack with the processed one.
|
|
8
|
+
*/
|
|
9
|
+
declare function attachProcessingToTrack(track: LocalAudioTrack, config?: AudioProcessingConfig): Promise<AudioPipelineHandle>;
|
|
10
|
+
|
|
11
|
+
export { attachProcessingToTrack };
|