@tensamin/audio 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +218 -30
- package/dist/chunk-AHBRT4RD.mjs +307 -0
- package/dist/chunk-ERJVV5JR.mjs +91 -0
- package/dist/chunk-N553RHTI.mjs +93 -0
- package/dist/chunk-NMHKX64G.mjs +118 -0
- package/dist/chunk-XO6B3D4A.mjs +67 -0
- package/dist/{chunk-FS635GMR.mjs → chunk-YOSTLLCS.mjs} +2 -2
- package/dist/extensibility/plugins.js +110 -32
- package/dist/extensibility/plugins.mjs +3 -3
- package/dist/index.js +463 -97
- package/dist/index.mjs +6 -6
- package/dist/livekit/integration.js +463 -97
- package/dist/livekit/integration.mjs +6 -6
- package/dist/noise-suppression/rnnoise-node.js +42 -14
- package/dist/noise-suppression/rnnoise-node.mjs +1 -1
- package/dist/pipeline/audio-pipeline.js +396 -83
- package/dist/pipeline/audio-pipeline.mjs +5 -5
- package/dist/types.d.mts +118 -10
- package/dist/types.d.ts +118 -10
- package/dist/vad/vad-node.js +68 -18
- package/dist/vad/vad-node.mjs +1 -1
- package/dist/vad/vad-state.d.mts +1 -0
- package/dist/vad/vad-state.d.ts +1 -0
- package/dist/vad/vad-state.js +42 -8
- package/dist/vad/vad-state.mjs +1 -1
- package/package.json +1 -1
- package/dist/chunk-HFSKQ33X.mjs +0 -38
- package/dist/chunk-JJASCVEW.mjs +0 -59
- package/dist/chunk-QU7E5HBA.mjs +0 -106
- package/dist/chunk-SDTOKWM2.mjs +0 -39
- package/dist/chunk-UMU2KIB6.mjs +0 -68
package/dist/chunk-JJASCVEW.mjs
DELETED
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
// src/vad/vad-state.ts
|
|
2
|
-
var VADStateMachine = class {
|
|
3
|
-
config;
|
|
4
|
-
currentState = "silent";
|
|
5
|
-
lastSpeechTime = 0;
|
|
6
|
-
speechStartTime = 0;
|
|
7
|
-
frameDurationMs = 20;
|
|
8
|
-
// Assumed frame duration, updated by calls
|
|
9
|
-
constructor(config) {
|
|
10
|
-
this.config = {
|
|
11
|
-
enabled: config?.enabled ?? true,
|
|
12
|
-
pluginName: config?.pluginName ?? "energy-vad",
|
|
13
|
-
startThreshold: config?.startThreshold ?? 0.5,
|
|
14
|
-
stopThreshold: config?.stopThreshold ?? 0.4,
|
|
15
|
-
hangoverMs: config?.hangoverMs ?? 300,
|
|
16
|
-
preRollMs: config?.preRollMs ?? 200
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
|
-
updateConfig(config) {
|
|
20
|
-
this.config = { ...this.config, ...config };
|
|
21
|
-
}
|
|
22
|
-
processFrame(probability, timestamp) {
|
|
23
|
-
const { startThreshold, stopThreshold, hangoverMs } = this.config;
|
|
24
|
-
let newState = this.currentState;
|
|
25
|
-
if (this.currentState === "silent" || this.currentState === "speech_ending") {
|
|
26
|
-
if (probability >= startThreshold) {
|
|
27
|
-
newState = "speech_starting";
|
|
28
|
-
this.speechStartTime = timestamp;
|
|
29
|
-
this.lastSpeechTime = timestamp;
|
|
30
|
-
} else {
|
|
31
|
-
newState = "silent";
|
|
32
|
-
}
|
|
33
|
-
} else if (this.currentState === "speech_starting" || this.currentState === "speaking") {
|
|
34
|
-
if (probability >= stopThreshold) {
|
|
35
|
-
newState = "speaking";
|
|
36
|
-
this.lastSpeechTime = timestamp;
|
|
37
|
-
} else {
|
|
38
|
-
const timeSinceSpeech = timestamp - this.lastSpeechTime;
|
|
39
|
-
if (timeSinceSpeech < hangoverMs) {
|
|
40
|
-
newState = "speaking";
|
|
41
|
-
} else {
|
|
42
|
-
newState = "speech_ending";
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
if (newState === "speech_starting") newState = "speaking";
|
|
47
|
-
if (newState === "speech_ending") newState = "silent";
|
|
48
|
-
this.currentState = newState;
|
|
49
|
-
return {
|
|
50
|
-
isSpeaking: newState === "speaking",
|
|
51
|
-
probability,
|
|
52
|
-
state: newState
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
};
|
|
56
|
-
|
|
57
|
-
export {
|
|
58
|
-
VADStateMachine
|
|
59
|
-
};
|
package/dist/chunk-QU7E5HBA.mjs
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
VADStateMachine
|
|
3
|
-
} from "./chunk-JJASCVEW.mjs";
|
|
4
|
-
import {
|
|
5
|
-
getAudioContext,
|
|
6
|
-
registerPipeline,
|
|
7
|
-
unregisterPipeline
|
|
8
|
-
} from "./chunk-OZ7KMC4S.mjs";
|
|
9
|
-
import {
|
|
10
|
-
getNoiseSuppressionPlugin,
|
|
11
|
-
getVADPlugin
|
|
12
|
-
} from "./chunk-FS635GMR.mjs";
|
|
13
|
-
|
|
14
|
-
// src/pipeline/audio-pipeline.ts
|
|
15
|
-
import mitt from "mitt";
|
|
16
|
-
async function createAudioPipeline(sourceTrack, config = {}) {
|
|
17
|
-
const context = getAudioContext();
|
|
18
|
-
registerPipeline();
|
|
19
|
-
const fullConfig = {
|
|
20
|
-
noiseSuppression: { enabled: true, ...config.noiseSuppression },
|
|
21
|
-
vad: { enabled: true, ...config.vad },
|
|
22
|
-
output: {
|
|
23
|
-
speechGain: 1,
|
|
24
|
-
silenceGain: 0,
|
|
25
|
-
gainRampTime: 0.02,
|
|
26
|
-
...config.output
|
|
27
|
-
},
|
|
28
|
-
livekit: { manageTrackMute: false, ...config.livekit }
|
|
29
|
-
};
|
|
30
|
-
const sourceStream = new MediaStream([sourceTrack]);
|
|
31
|
-
const sourceNode = context.createMediaStreamSource(sourceStream);
|
|
32
|
-
const nsPlugin = getNoiseSuppressionPlugin(
|
|
33
|
-
fullConfig.noiseSuppression?.pluginName
|
|
34
|
-
);
|
|
35
|
-
const nsNode = await nsPlugin.createNode(
|
|
36
|
-
context,
|
|
37
|
-
fullConfig.noiseSuppression
|
|
38
|
-
);
|
|
39
|
-
const vadPlugin = getVADPlugin(fullConfig.vad?.pluginName);
|
|
40
|
-
const vadStateMachine = new VADStateMachine(fullConfig.vad);
|
|
41
|
-
const emitter = mitt();
|
|
42
|
-
const vadNode = await vadPlugin.createNode(
|
|
43
|
-
context,
|
|
44
|
-
fullConfig.vad,
|
|
45
|
-
(prob) => {
|
|
46
|
-
const timestamp = context.currentTime * 1e3;
|
|
47
|
-
const newState = vadStateMachine.processFrame(prob, timestamp);
|
|
48
|
-
if (newState.state !== lastVadState.state || Math.abs(newState.probability - lastVadState.probability) > 0.1) {
|
|
49
|
-
emitter.emit("vadChange", newState);
|
|
50
|
-
lastVadState = newState;
|
|
51
|
-
updateGain(newState);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
);
|
|
55
|
-
let lastVadState = {
|
|
56
|
-
isSpeaking: false,
|
|
57
|
-
probability: 0,
|
|
58
|
-
state: "silent"
|
|
59
|
-
};
|
|
60
|
-
const splitter = context.createGain();
|
|
61
|
-
sourceNode.connect(nsNode);
|
|
62
|
-
nsNode.connect(splitter);
|
|
63
|
-
splitter.connect(vadNode);
|
|
64
|
-
const delayNode = context.createDelay(1);
|
|
65
|
-
const preRollSeconds = (fullConfig.vad?.preRollMs ?? 200) / 1e3;
|
|
66
|
-
delayNode.delayTime.value = preRollSeconds;
|
|
67
|
-
const gainNode = context.createGain();
|
|
68
|
-
gainNode.gain.value = fullConfig.output?.silenceGain ?? 0;
|
|
69
|
-
const destination = context.createMediaStreamDestination();
|
|
70
|
-
splitter.connect(delayNode);
|
|
71
|
-
delayNode.connect(gainNode);
|
|
72
|
-
gainNode.connect(destination);
|
|
73
|
-
function updateGain(state) {
|
|
74
|
-
const { speechGain, silenceGain, gainRampTime } = fullConfig.output;
|
|
75
|
-
const targetGain = state.isSpeaking ? speechGain ?? 1 : silenceGain ?? 0;
|
|
76
|
-
const now = context.currentTime;
|
|
77
|
-
gainNode.gain.setTargetAtTime(targetGain, now, gainRampTime ?? 0.02);
|
|
78
|
-
}
|
|
79
|
-
function dispose() {
|
|
80
|
-
sourceNode.disconnect();
|
|
81
|
-
nsNode.disconnect();
|
|
82
|
-
splitter.disconnect();
|
|
83
|
-
vadNode.disconnect();
|
|
84
|
-
delayNode.disconnect();
|
|
85
|
-
gainNode.disconnect();
|
|
86
|
-
destination.stream.getTracks().forEach((t) => t.stop());
|
|
87
|
-
unregisterPipeline();
|
|
88
|
-
}
|
|
89
|
-
return {
|
|
90
|
-
processedTrack: destination.stream.getAudioTracks()[0],
|
|
91
|
-
events: emitter,
|
|
92
|
-
get state() {
|
|
93
|
-
return lastVadState;
|
|
94
|
-
},
|
|
95
|
-
setConfig: (newConfig) => {
|
|
96
|
-
if (newConfig.vad) {
|
|
97
|
-
vadStateMachine.updateConfig(newConfig.vad);
|
|
98
|
-
}
|
|
99
|
-
},
|
|
100
|
-
dispose
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
export {
|
|
105
|
-
createAudioPipeline
|
|
106
|
-
};
|
package/dist/chunk-SDTOKWM2.mjs
DELETED
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
// src/noise-suppression/rnnoise-node.ts
|
|
2
|
-
var RNNoisePlugin = class {
|
|
3
|
-
name = "rnnoise-ns";
|
|
4
|
-
wasmBuffer = null;
|
|
5
|
-
async createNode(context, config) {
|
|
6
|
-
const { loadRnnoise, RnnoiseWorkletNode } = await import("@sapphi-red/web-noise-suppressor");
|
|
7
|
-
if (!config?.enabled) {
|
|
8
|
-
const pass = context.createGain();
|
|
9
|
-
return pass;
|
|
10
|
-
}
|
|
11
|
-
if (!config?.wasmUrl || !config?.simdUrl || !config?.workletUrl) {
|
|
12
|
-
throw new Error(
|
|
13
|
-
"RNNoisePlugin requires 'wasmUrl', 'simdUrl', and 'workletUrl' to be configured. Please download the assets and provide the URLs."
|
|
14
|
-
);
|
|
15
|
-
}
|
|
16
|
-
if (!this.wasmBuffer) {
|
|
17
|
-
this.wasmBuffer = await loadRnnoise({
|
|
18
|
-
url: config.wasmUrl,
|
|
19
|
-
simdUrl: config.simdUrl
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
const workletUrl = config.workletUrl;
|
|
23
|
-
try {
|
|
24
|
-
await context.audioWorklet.addModule(workletUrl);
|
|
25
|
-
} catch (e) {
|
|
26
|
-
console.warn("Failed to add RNNoise worklet module:", e);
|
|
27
|
-
}
|
|
28
|
-
const node = new RnnoiseWorkletNode(context, {
|
|
29
|
-
wasmBinary: this.wasmBuffer,
|
|
30
|
-
maxChannels: 1
|
|
31
|
-
// Mono for now
|
|
32
|
-
});
|
|
33
|
-
return node;
|
|
34
|
-
}
|
|
35
|
-
};
|
|
36
|
-
|
|
37
|
-
export {
|
|
38
|
-
RNNoisePlugin
|
|
39
|
-
};
|
package/dist/chunk-UMU2KIB6.mjs
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
// src/vad/vad-node.ts
|
|
2
|
-
var energyVadWorkletCode = `
|
|
3
|
-
class EnergyVadProcessor extends AudioWorkletProcessor {
|
|
4
|
-
constructor() {
|
|
5
|
-
super();
|
|
6
|
-
this.smoothing = 0.95;
|
|
7
|
-
this.energy = 0;
|
|
8
|
-
this.noiseFloor = 0.001;
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
process(inputs, outputs, parameters) {
|
|
12
|
-
const input = inputs[0];
|
|
13
|
-
if (!input || !input.length) return true;
|
|
14
|
-
const channel = input[0];
|
|
15
|
-
|
|
16
|
-
// Calculate RMS
|
|
17
|
-
let sum = 0;
|
|
18
|
-
for (let i = 0; i < channel.length; i++) {
|
|
19
|
-
sum += channel[i] * channel[i];
|
|
20
|
-
}
|
|
21
|
-
const rms = Math.sqrt(sum / channel.length);
|
|
22
|
-
|
|
23
|
-
// Simple adaptive noise floor (very basic)
|
|
24
|
-
if (rms < this.noiseFloor) {
|
|
25
|
-
this.noiseFloor = this.noiseFloor * 0.99 + rms * 0.01;
|
|
26
|
-
} else {
|
|
27
|
-
this.noiseFloor = this.noiseFloor * 0.999 + rms * 0.001;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// Calculate "probability" based on SNR
|
|
31
|
-
// This is a heuristic mapping from energy to 0-1
|
|
32
|
-
const snr = rms / (this.noiseFloor + 1e-6);
|
|
33
|
-
const probability = Math.min(1, Math.max(0, (snr - 1.5) / 10)); // Arbitrary scaling
|
|
34
|
-
|
|
35
|
-
this.port.postMessage({ probability });
|
|
36
|
-
|
|
37
|
-
return true;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
registerProcessor('energy-vad-processor', EnergyVadProcessor);
|
|
41
|
-
`;
|
|
42
|
-
var EnergyVADPlugin = class {
|
|
43
|
-
name = "energy-vad";
|
|
44
|
-
async createNode(context, config, onDecision) {
|
|
45
|
-
const blob = new Blob([energyVadWorkletCode], {
|
|
46
|
-
type: "application/javascript"
|
|
47
|
-
});
|
|
48
|
-
const url = URL.createObjectURL(blob);
|
|
49
|
-
try {
|
|
50
|
-
await context.audioWorklet.addModule(url);
|
|
51
|
-
} catch (e) {
|
|
52
|
-
console.warn("Failed to add Energy VAD worklet:", e);
|
|
53
|
-
throw e;
|
|
54
|
-
} finally {
|
|
55
|
-
URL.revokeObjectURL(url);
|
|
56
|
-
}
|
|
57
|
-
const node = new AudioWorkletNode(context, "energy-vad-processor");
|
|
58
|
-
node.port.onmessage = (event) => {
|
|
59
|
-
const { probability } = event.data;
|
|
60
|
-
onDecision(probability);
|
|
61
|
-
};
|
|
62
|
-
return node;
|
|
63
|
-
}
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
export {
|
|
67
|
-
EnergyVADPlugin
|
|
68
|
-
};
|