@reins-ai/voice 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +81 -2
- package/dist/barge-in.d.ts +42 -0
- package/dist/barge-in.d.ts.map +1 -0
- package/dist/barge-in.js +78 -0
- package/dist/barge-in.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +100 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +32 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +54 -0
- package/dist/config.js.map +1 -0
- package/dist/download.d.ts +18 -0
- package/dist/download.d.ts.map +1 -0
- package/dist/download.js +37 -0
- package/dist/download.js.map +1 -0
- package/dist/ear.d.ts +158 -0
- package/dist/ear.d.ts.map +1 -0
- package/dist/ear.js +387 -0
- package/dist/ear.js.map +1 -0
- package/dist/log.d.ts +6 -0
- package/dist/log.d.ts.map +1 -0
- package/dist/log.js +13 -0
- package/dist/log.js.map +1 -0
- package/dist/mic.d.ts +19 -0
- package/dist/mic.d.ts.map +1 -0
- package/dist/mic.js +43 -0
- package/dist/mic.js.map +1 -0
- package/dist/piper.d.ts +38 -0
- package/dist/piper.d.ts.map +1 -0
- package/dist/piper.js +90 -0
- package/dist/piper.js.map +1 -0
- package/dist/server.d.ts +21 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +92 -0
- package/dist/server.js.map +1 -0
- package/dist/setup.d.ts +15 -0
- package/dist/setup.d.ts.map +1 -0
- package/dist/setup.js +73 -0
- package/dist/setup.js.map +1 -0
- package/dist/speak.d.ts +74 -0
- package/dist/speak.d.ts.map +1 -0
- package/dist/speak.js +188 -0
- package/dist/speak.js.map +1 -0
- package/dist/speech-check.d.ts +27 -0
- package/dist/speech-check.d.ts.map +1 -0
- package/dist/speech-check.js +51 -0
- package/dist/speech-check.js.map +1 -0
- package/dist/stdout-guard.d.ts +2 -0
- package/dist/stdout-guard.d.ts.map +1 -0
- package/dist/stdout-guard.js +8 -0
- package/dist/stdout-guard.js.map +1 -0
- package/dist/stt.d.ts +31 -0
- package/dist/stt.d.ts.map +1 -0
- package/dist/stt.js +86 -0
- package/dist/stt.js.map +1 -0
- package/dist/turn.d.ts +28 -0
- package/dist/turn.d.ts.map +1 -0
- package/dist/turn.js +60 -0
- package/dist/turn.js.map +1 -0
- package/dist/vad.d.ts +53 -0
- package/dist/vad.d.ts.map +1 -0
- package/dist/vad.js +120 -0
- package/dist/vad.js.map +1 -0
- package/package.json +31 -4
package/dist/speak.js
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { chooseVoice, ensurePiper, ensureVoice, loadCatalog } from './piper.js';
|
|
3
|
+
// On Windows the text and the language go through environment variables, never inside the script, so nothing
|
|
4
|
+
// is interpreted.
|
|
5
|
+
const WINDOWS_SCRIPT = [
|
|
6
|
+
'Add-Type -AssemblyName System.Speech',
|
|
7
|
+
'$s = New-Object System.Speech.Synthesis.SpeechSynthesizer',
|
|
8
|
+
"$v = $s.GetInstalledVoices() | Where-Object { $_.Enabled -and $_.VoiceInfo.Culture.TwoLetterISOLanguageName -eq $env:SANORIVA_LANG } | Select-Object -First 1",
|
|
9
|
+
'if ($v) { $s.SelectVoice($v.VoiceInfo.Name) }',
|
|
10
|
+
'$s.Speak($env:SANORIVA_TEXT)',
|
|
11
|
+
'$s.Dispose()',
|
|
12
|
+
].join('; ');
|
|
13
|
+
/** How to read a text out loud with the local system voice. `lang` is a 2-letter code (es, en…). */
|
|
14
|
+
export function speakCommand(platform, text, lang) {
|
|
15
|
+
if (platform === 'win32') {
|
|
16
|
+
return {
|
|
17
|
+
command: 'powershell',
|
|
18
|
+
args: ['-NoProfile', '-NonInteractive', '-Command', WINDOWS_SCRIPT],
|
|
19
|
+
env: { SANORIVA_TEXT: text, SANORIVA_LANG: lang ?? '' },
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (platform === 'darwin')
|
|
23
|
+
return { command: 'say', args: [text] };
|
|
24
|
+
return { command: 'espeak-ng', args: [...(lang ? ['-v', lang] : []), text] };
|
|
25
|
+
}
|
|
26
|
+
const runCommand = ({ command, args, env }, signal) => new Promise((resolve, reject) => {
|
|
27
|
+
const child = spawn(command, args, { env: { ...process.env, ...env }, stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true });
|
|
28
|
+
// Cut off (the user talked): it stops and counts as finished.
|
|
29
|
+
signal?.addEventListener('abort', () => (child.kill(), resolve()), { once: true });
|
|
30
|
+
let stderr = '';
|
|
31
|
+
child.stderr.on('data', (chunk) => (stderr += chunk));
|
|
32
|
+
child.on('error', (error) => reject(new Error(`No puedo hablar (${command}): ${error.message}`)));
|
|
33
|
+
child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(stderr.trim() || `${command} exited with code ${code}`))));
|
|
34
|
+
});
|
|
35
|
+
/** The system voice (Windows, macOS or espeak-ng): always available, but it sounds robotic. */
|
|
36
|
+
export function createSpeaker({ lang, platform = process.platform, run = runCommand } = {}) {
|
|
37
|
+
return (text, language, signal) => run(speakCommand(platform, text, (language ?? lang)?.slice(0, 2)), signal);
|
|
38
|
+
}
|
|
39
|
+
/** Piper generates the audio (16-bit PCM) and ffplay plays it while it is generated. */
|
|
40
|
+
export function piperCommands({ piper, model, sampleRate, ffplay }) {
|
|
41
|
+
return {
|
|
42
|
+
synth: { command: piper, args: ['--model', model, '--output-raw'] },
|
|
43
|
+
player: {
|
|
44
|
+
command: ffplay,
|
|
45
|
+
// -probesize/-analyzeduration make it start at once, without analysing the audio first (~0.4 s saved).
|
|
46
|
+
args: ['-hide_banner', '-loglevel', 'error', '-nodisp', '-autoexit', '-probesize', '32', '-analyzeduration', '0', '-f', 's16le', '-ar', String(sampleRate), '-ch_layout', 'mono', '-'],
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/** ffplay starts playing roughly this long after receiving the first audio. */
|
|
51
|
+
const PLAYER_DELAY_MS = 300;
|
|
52
|
+
/**
|
|
53
|
+
* The level of each 30 ms chunk of 16-bit PCM audio, with the time it will play. Piper generates faster than
|
|
54
|
+
* real time, so the time comes from the position in the audio, not from when it arrives.
|
|
55
|
+
*/
|
|
56
|
+
export function pcmLevels(sampleRate, onLevel) {
|
|
57
|
+
const frame = Math.round((sampleRate * 30) / 1000);
|
|
58
|
+
let leftover = Buffer.alloc(0);
|
|
59
|
+
let start = 0;
|
|
60
|
+
let index = 0;
|
|
61
|
+
return (chunk, now = Date.now()) => {
|
|
62
|
+
if (!start)
|
|
63
|
+
start = now + PLAYER_DELAY_MS;
|
|
64
|
+
const bytes = Buffer.concat([leftover, chunk]);
|
|
65
|
+
let offset = 0;
|
|
66
|
+
for (; offset + frame * 2 <= bytes.length; offset += frame * 2) {
|
|
67
|
+
let sum = 0;
|
|
68
|
+
for (let i = 0; i < frame; i++)
|
|
69
|
+
sum += (bytes.readInt16LE(offset + i * 2) / 32_768) ** 2;
|
|
70
|
+
onLevel(Math.sqrt(sum / frame), start + (index * frame * 1000) / sampleRate, index === 0);
|
|
71
|
+
index++;
|
|
72
|
+
}
|
|
73
|
+
leftover = bytes.subarray(offset);
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Runs Piper → ffplay with the text and resolves when playback ends (so the echo window ends at the right
|
|
78
|
+
* time) or when it is cut off.
|
|
79
|
+
*/
|
|
80
|
+
function playWithPiper(commands, text, { sampleRate, onLevel, signal }) {
|
|
81
|
+
return new Promise((resolve, reject) => {
|
|
82
|
+
const synth = spawn(commands.synth.command, commands.synth.args, { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true });
|
|
83
|
+
const player = spawn(commands.player.command, commands.player.args, { stdio: ['pipe', 'ignore', 'pipe'], windowsHide: true });
|
|
84
|
+
let stderr = '';
|
|
85
|
+
synth.stderr.on('data', () => { }); // Piper reports its progress here
|
|
86
|
+
player.stderr.on('data', (chunk) => (stderr += chunk));
|
|
87
|
+
const fail = (error) => {
|
|
88
|
+
synth.kill();
|
|
89
|
+
player.kill();
|
|
90
|
+
reject(error);
|
|
91
|
+
};
|
|
92
|
+
synth.on('error', (error) => fail(new Error(`No puedo arrancar Piper: ${error.message}`)));
|
|
93
|
+
player.on('error', (error) => fail(new Error(`Cannot play the audio (${commands.player.command}): ${error.message}`)));
|
|
94
|
+
synth.stdout.pipe(player.stdin);
|
|
95
|
+
if (onLevel)
|
|
96
|
+
synth.stdout.on('data', pcmLevels(sampleRate, onLevel));
|
|
97
|
+
player.stdin.on('error', () => { }); // ffplay closing early must not crash the process
|
|
98
|
+
// Cut off (the user talked): both processes stop and it counts as finished.
|
|
99
|
+
signal?.addEventListener('abort', () => (synth.kill(), player.kill(), resolve()), { once: true });
|
|
100
|
+
player.on('close', (code) => (code === 0 ? resolve() : reject(new Error(stderr.trim() || `ffplay exited with code ${code}`))));
|
|
101
|
+
// Piper reads one sentence per line from stdin.
|
|
102
|
+
synth.stdin.end(`${text.replace(/\s+/g, ' ').trim()}\n`);
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Natural voices with Piper. Each language's voice comes from the official Piper catalog and is downloaded
|
|
107
|
+
* the first time it is needed, verified with its hash. Piper itself is also downloaded the first time.
|
|
108
|
+
*/
|
|
109
|
+
export function createPiperSpeaker({ dataDir, ffplay, voice, language, fetchImpl, onLevel }) {
|
|
110
|
+
let piper;
|
|
111
|
+
let catalog;
|
|
112
|
+
const voices = new Map();
|
|
113
|
+
const getPiper = () => (piper ??= ensurePiper(dataDir, { fetchImpl }));
|
|
114
|
+
const getCatalog = () => (catalog ??= loadCatalog(dataDir, fetchImpl));
|
|
115
|
+
const voiceFor = async (lang) => {
|
|
116
|
+
const all = await getCatalog();
|
|
117
|
+
const chosen = voice ? all[voice] : chooseVoice(all, lang ?? language ?? 'en');
|
|
118
|
+
if (!chosen)
|
|
119
|
+
throw new Error(voice ? `Voice ${voice} is not in the Piper catalog` : `Piper has no voice for "${lang}"`);
|
|
120
|
+
if (!voices.has(chosen.key))
|
|
121
|
+
voices.set(chosen.key, ensureVoice(dataDir, chosen, fetchImpl));
|
|
122
|
+
return voices.get(chosen.key);
|
|
123
|
+
};
|
|
124
|
+
const speak = async (text, lang, signal) => {
|
|
125
|
+
const [exe, { model, sampleRate }] = await Promise.all([getPiper(), voiceFor(lang)]);
|
|
126
|
+
if (signal?.aborted)
|
|
127
|
+
return;
|
|
128
|
+
await playWithPiper(piperCommands({ piper: exe, model, sampleRate, ffplay }), text, { sampleRate, ...(onLevel ? { onLevel } : {}), ...(signal ? { signal } : {}) });
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* Starts the downloads at startup so the first sentence does not wait: Piper and its catalog, and the voice
|
|
132
|
+
* only with a fixed language or voice (otherwise the voice for the language the AI speaks is downloaded
|
|
133
|
+
* then).
|
|
134
|
+
*/
|
|
135
|
+
/** Downloads Piper, its catalog and the voice for a language now (for --setup). */
|
|
136
|
+
speak.prepare = async (lang) => {
|
|
137
|
+
await Promise.all([getPiper(), voiceFor(lang)]);
|
|
138
|
+
};
|
|
139
|
+
speak.warm = () => {
|
|
140
|
+
void getPiper().catch(() => { });
|
|
141
|
+
void getCatalog().catch(() => { });
|
|
142
|
+
if (language || voice)
|
|
143
|
+
void voiceFor(language).catch(() => { });
|
|
144
|
+
};
|
|
145
|
+
return speak;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Speaks without blocking: each sentence waits for the previous one, but the caller goes on at once (the AI
|
|
149
|
+
* can act on the page while it plays). Resolves, when that sentence ends, with whether it played whole
|
|
150
|
+
* (`false`: it was cut off).
|
|
151
|
+
*/
|
|
152
|
+
export function speechQueue(speaker, onError = () => { }) {
|
|
153
|
+
let playing = Promise.resolve();
|
|
154
|
+
let current = new AbortController();
|
|
155
|
+
const say = (text, language) => {
|
|
156
|
+
const signal = current.signal;
|
|
157
|
+
const run = playing
|
|
158
|
+
.then(async () => {
|
|
159
|
+
if (!signal.aborted)
|
|
160
|
+
await speaker(text, language, signal);
|
|
161
|
+
})
|
|
162
|
+
.catch(onError)
|
|
163
|
+
.then(() => !signal.aborted);
|
|
164
|
+
playing = run.then(() => undefined);
|
|
165
|
+
return run;
|
|
166
|
+
};
|
|
167
|
+
/**
|
|
168
|
+
* The user talked: what is playing stops and what was waiting is dropped. Anything requested afterwards
|
|
169
|
+
* plays.
|
|
170
|
+
*/
|
|
171
|
+
say.cancel = () => {
|
|
172
|
+
current.abort();
|
|
173
|
+
current = new AbortController();
|
|
174
|
+
};
|
|
175
|
+
return say;
|
|
176
|
+
}
|
|
177
|
+
/** A short soft tone, generated by ffplay with no files: "I heard you, I'm thinking". */
|
|
178
|
+
export function chimeCommand(ffplay) {
|
|
179
|
+
return {
|
|
180
|
+
command: ffplay,
|
|
181
|
+
args: ['-hide_banner', '-loglevel', 'error', '-nodisp', '-autoexit', '-f', 'lavfi', '-i', 'sine=frequency=660:duration=0.15', '-af', 'volume=0.2,afade=t=out:st=0.05:d=0.1'],
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/** Plays the tone without waiting for it to end; a failure does not matter, since it is only a cue. */
|
|
185
|
+
export function createChime(ffplay, run = runCommand) {
|
|
186
|
+
return () => void run(chimeCommand(ffplay)).catch(() => { });
|
|
187
|
+
}
|
|
188
|
+
//# sourceMappingURL=speak.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speak.js","sourceRoot":"","sources":["../src/speak.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAG3C,OAAO,EAAgB,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAQ9F,6GAA6G;AAC7G,kBAAkB;AAClB,MAAM,cAAc,GAAG;IACrB,sCAAsC;IACtC,2DAA2D;IAC3D,+JAA+J;IAC/J,+CAA+C;IAC/C,8BAA8B;IAC9B,cAAc;CACf,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,oGAAoG;AACpG,MAAM,UAAU,YAAY,CAAC,QAAyB,EAAE,IAAY,EAAE,IAAa;IACjF,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;QACzB,OAAO;YACL,OAAO,EAAE,YAAY;YACrB,IAAI,EAAE,CAAC,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,cAAc,CAAC;YACnE,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,EAAE;SACxD,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;IACnE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;AAC/E,CAAC;AAID,MAAM,UAAU,GAAe,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,CAChE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IACpI,8DAA8D;IAC1D,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC;IACtD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,OAAO,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAClG,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,qBAAqB,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AACpI,CAAC,CAAC,CAAC;AAQL,+FAA+F;AAC/F,MAAM,UAAU,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,UAAU,KAAsE,EAAE;IACzJ,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAChH,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,aAAa,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAwE;IACtI,OAAO;QACL,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,cAAc,CAAC,EAAE;QACnE,MAAM,EAAE;YACN,OAAO,EAAE,MAAM;YACrB,uGAAuG;YACjG,IAAI,EAAE,CAAC,cAAc,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC;SACvL;KACF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,UAAkB,EAAE,OAA4D;IACxG,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;IACnD,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,CAAC,KAAa,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC,KAAK;YAAE,KAAK,GAAG,GAAG,GAAG,eAAe,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAC/C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,MAAM,GAAG,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/D,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE;gBAAE,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;YACzF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,KAAK,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC;YAC1F,KAAK,EAAE,CAAC;QACV,CAAC;QACD,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CACpB,QAA0C,EAC1C,IAAY,EACZ,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAA+G;IAE5I,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QACzH,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9H,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,kCAAkC;QACrE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,CAAC,KAAY,EAAE,EAAE;YAC5B,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC;QACF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,4BAA4B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,0BAA0B,QAAQ,CAAC,MAAM,CAAC,OAAO,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACvH,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,OAAO;YAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,kDAAkD;QAC1F,4EAA4E;QACxE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAClG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,2BAA2B,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACnI,gDAAgD;QAC5C,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC;AAcD;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAuB;IAC9G,IAAI,KAAkC,CAAC;IACvC,IAAI,OAAqC,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0D,CAAC;IACjF,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,CAAC,OAAO,KAAK,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAEvE,MAAM,QAAQ,GAAG,KAAK,EAAE,IAAwB,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,MAAM,UAAU,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,KAAK,8BAA8B,CAAC,CAAC,CAAC,2BAA2B,IAAI,GAAG,CAAC,CAAC;QACxH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;QAC7F,OAAO,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC;IACjC,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,KAAK,EAAE,IAAY,EAAE,IAAa,EAAE,MAAoB,EAAE,EAAE;QACxE,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrF,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO;QAC5B,MAAM,aAAa,CAAC,aAAa,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACtK,CAAC,CAAC;IACF;;;;OAIG;IACH,mFAAmF;IACnF,KAAK,CAAC,OAAO,GAAG,KAAK,EAAE,IAAa,EAAE,EAAE;QACtC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC;IACF,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE;QAChB,KAAK,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAChC,KAAK,UAAU,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAClC,IAAI,QAAQ,IAAI,KAAK;YAAE,KAAK,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC;IACF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,OAAgB,EAAE,UAAkC,GAAG,EAAE,GAAE,CAAC;IACtF,IAAI,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/C,IAAI,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,QAAiB,EAAoB,EAAE;QAChE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,MAAM,GAAG,GAAG,OAAO;aAChB,IAAI,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,OAAO;gBAAE,MAAM,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC7D,CAAC,CAAC;aACD,KAAK,CAAC,OAAO,CAAC;aACd,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpC,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF;;;OAGG;IACH,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE;QAChB,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAClC,CAAC,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,YAAY,CAAC,MAAc;IACzC,OAAO;QACL,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,CAAC,cAAc,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,kCAAkC,EAAE,KAAK,EAAE,sCAAsC,CAAC;KAC7K,CAAC;AACJ,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,MAAkB,UAAU;IACtE,OAAO,GAAG,EAAE,CAAC,KAAK,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AAC9D,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type Fetch } from './download.js';
|
|
2
|
+
/** Silero VAD v5 (MIT), pinned version from the official repository. About 2 MB. */
|
|
3
|
+
export declare const SILERO: {
|
|
4
|
+
url: string;
|
|
5
|
+
sha256: string;
|
|
6
|
+
size: number;
|
|
7
|
+
};
|
|
8
|
+
/** A sentence has real voice if Silero hears voice for at least this long. */
|
|
9
|
+
export declare const MIN_SPEECH_MS = 250;
|
|
10
|
+
/**
|
|
11
|
+
* Whether there is human voice in this audio. Works for any language: it checks whether someone talks, not
|
|
12
|
+
* what is said.
|
|
13
|
+
*/
|
|
14
|
+
export interface SpeechCheck {
|
|
15
|
+
hasSpeech(audio: Float32Array): Promise<boolean>;
|
|
16
|
+
}
|
|
17
|
+
/** Milliseconds of voice, from the voice probability of each block. */
|
|
18
|
+
export declare function speechMs(probabilities: number[]): number;
|
|
19
|
+
/**
|
|
20
|
+
* Silero VAD: a small model that tells human voice from noise. Every sentence goes through it before Whisper,
|
|
21
|
+
* and noise is not transcribed (Whisper would make something up). Downloaded once to cacheDir.
|
|
22
|
+
*/
|
|
23
|
+
export declare function createSileroCheck({ cacheDir, fetchImpl }: {
|
|
24
|
+
cacheDir: string;
|
|
25
|
+
fetchImpl?: Fetch;
|
|
26
|
+
}): Promise<SpeechCheck>;
|
|
27
|
+
//# sourceMappingURL=speech-check.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speech-check.d.ts","sourceRoot":"","sources":["../src/speech-check.ts"],"names":[],"mappings":"AAIA,OAAO,EAAY,KAAK,KAAK,EAAE,MAAM,eAAe,CAAC;AAErD,oFAAoF;AACpF,eAAO,MAAM,MAAM;;;;CAIlB,CAAC;AAQF,8EAA8E;AAC9E,eAAO,MAAM,aAAa,MAAM,CAAC;AAEjC;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAClD;AAED,uEAAuE;AACvE,wBAAgB,QAAQ,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CAExD;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,KAAK,CAAA;CAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAyB9H"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { InferenceSession, Tensor } from 'onnxruntime-node';
|
|
3
|
+
import { download } from './download.js';
|
|
4
|
+
/** Silero VAD v5 (MIT), pinned version from the official repository. About 2 MB. */
|
|
5
|
+
export const SILERO = {
|
|
6
|
+
url: 'https://huggingface.co/onnx-community/silero-vad/resolve/e71cae966052b992a7eca6b17738916ce0eca4ec/onnx/model.onnx',
|
|
7
|
+
sha256: 'a4a068cd6cf1ea8355b84327595838ca748ec29a25bc91fc82e6c299ccdc5808',
|
|
8
|
+
size: 2243022,
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Silero works on blocks of 512 samples (32 ms at 16 kHz), with 64 samples of context from the previous
|
|
12
|
+
* block.
|
|
13
|
+
*/
|
|
14
|
+
const CHUNK = 512;
|
|
15
|
+
const CONTEXT = 64;
|
|
16
|
+
/** A sentence has real voice if Silero hears voice for at least this long. */
|
|
17
|
+
export const MIN_SPEECH_MS = 250;
|
|
18
|
+
/** Milliseconds of voice, from the voice probability of each block. */
|
|
19
|
+
export function speechMs(probabilities) {
|
|
20
|
+
return probabilities.filter((p) => p > 0.5).length * ((CHUNK / 16_000) * 1000);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Silero VAD: a small model that tells human voice from noise. Every sentence goes through it before Whisper,
|
|
24
|
+
* and noise is not transcribed (Whisper would make something up). Downloaded once to cacheDir.
|
|
25
|
+
*/
|
|
26
|
+
export async function createSileroCheck({ cacheDir, fetchImpl }) {
|
|
27
|
+
const model = await download(SILERO.url, join(cacheDir, 'silero-vad', 'model.onnx'), SILERO, fetchImpl);
|
|
28
|
+
const session = await InferenceSession.create(model);
|
|
29
|
+
const sr = new Tensor('int64', BigInt64Array.from([16000n]), []);
|
|
30
|
+
return {
|
|
31
|
+
async hasSpeech(audio) {
|
|
32
|
+
let state = new Tensor('float32', new Float32Array(2 * 128), [2, 1, 128]);
|
|
33
|
+
let context = new Float32Array(CONTEXT);
|
|
34
|
+
const probabilities = [];
|
|
35
|
+
// Stops as soon as there is enough voice: each step waits its turn in ONNX Runtime behind Whisper or Smart
|
|
36
|
+
// Turn, so going through the whole sentence could take tens of seconds.
|
|
37
|
+
for (let i = 0; i + CHUNK <= audio.length && speechMs(probabilities) < MIN_SPEECH_MS; i += CHUNK) {
|
|
38
|
+
const chunk = audio.subarray(i, i + CHUNK);
|
|
39
|
+
const input = new Float32Array(CONTEXT + CHUNK);
|
|
40
|
+
input.set(context, 0);
|
|
41
|
+
input.set(chunk, CONTEXT);
|
|
42
|
+
context = chunk.slice(CHUNK - CONTEXT);
|
|
43
|
+
const out = await session.run({ input: new Tensor('float32', input, [1, input.length]), state, sr });
|
|
44
|
+
state = out['stateN'];
|
|
45
|
+
probabilities.push(out['output'].data[0]);
|
|
46
|
+
}
|
|
47
|
+
return speechMs(probabilities) >= MIN_SPEECH_MS;
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=speech-check.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"speech-check.js","sourceRoot":"","sources":["../src/speech-check.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE5D,OAAO,EAAE,QAAQ,EAAc,MAAM,eAAe,CAAC;AAErD,oFAAoF;AACpF,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,GAAG,EAAE,mHAAmH;IACxH,MAAM,EAAE,kEAAkE;IAC1E,IAAI,EAAE,OAAO;CACd,CAAC;AAEF;;;GAGG;AACH,MAAM,KAAK,GAAG,GAAG,CAAC;AAClB,MAAM,OAAO,GAAG,EAAE,CAAC;AACnB,8EAA8E;AAC9E,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAUjC,uEAAuE;AACvE,MAAM,UAAU,QAAQ,CAAC,aAAuB;IAC9C,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAA2C;IACtG,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IACxG,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAElE,OAAO;QACL,KAAK,CAAC,SAAS,CAAC,KAAK;YACnB,IAAI,KAAK,GAAW,IAAI,MAAM,CAAC,SAAS,EAAE,IAAI,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAClF,IAAI,OAAO,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;YACxC,MAAM,aAAa,GAAa,EAAE,CAAC;YACzC,2GAA2G;YAC3G,wEAAwE;YAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,aAAa,CAAC,GAAG,aAAa,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;gBACjG,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;gBAC3C,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC;gBAChD,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtB,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBAC1B,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC;gBACvC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrG,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAW,CAAC;gBAChC,aAAa,CAAC,IAAI,CAAE,GAAG,CAAC,QAAQ,CAAY,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC;QAClD,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stdout-guard.d.ts","sourceRoot":"","sources":["../src/stdout-guard.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// stdout is the MCP channel, so every log from any library goes to stderr. This module is imported first in
|
|
2
|
+
// cli.ts so it applies before anything else loads.
|
|
3
|
+
console.log = console.error;
|
|
4
|
+
console.info = console.error;
|
|
5
|
+
console.warn = console.error;
|
|
6
|
+
console.debug = console.error;
|
|
7
|
+
export {};
|
|
8
|
+
//# sourceMappingURL=stdout-guard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stdout-guard.js","sourceRoot":"","sources":["../src/stdout-guard.ts"],"names":[],"mappings":"AAAA,4GAA4G;AAC5G,mDAAmD;AACnD,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC;AAC5B,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC;AAC7B,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC;AAC7B,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC"}
|
package/dist/stt.d.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { Tensor } from '@huggingface/transformers';
|
|
2
|
+
import { type Log } from './log.js';
|
|
3
|
+
/** Speech to text. */
|
|
4
|
+
export interface Transcriber {
|
|
5
|
+
transcribe(audio: Float32Array): Promise<string>;
|
|
6
|
+
}
|
|
7
|
+
/** small: much more accurate than base in Spanish (≈3 s per sentence on CPU, 241 MB). */
|
|
8
|
+
export declare const DEFAULT_WHISPER_MODEL = "onnx-community/whisper-small";
|
|
9
|
+
export interface WhisperOptions {
|
|
10
|
+
model?: string;
|
|
11
|
+
/** A 2-letter code to fix the language; without it, the language of each sentence is detected. */
|
|
12
|
+
language?: string;
|
|
13
|
+
cacheDir: string;
|
|
14
|
+
/**
|
|
15
|
+
* `cpu`, or `dml` (the graphics card on Windows, through DirectML: ~25 % faster). If dml fails, cpu is
|
|
16
|
+
* used.
|
|
17
|
+
*/
|
|
18
|
+
device?: WhisperDevice;
|
|
19
|
+
log?: Log;
|
|
20
|
+
}
|
|
21
|
+
export type WhisperDevice = 'cpu' | 'dml';
|
|
22
|
+
/** On Windows the graphics card (DirectML ships with ONNX Runtime); elsewhere the CPU. */
|
|
23
|
+
export declare const defaultDevice: (platform?: NodeJS.Platform) => WhisperDevice;
|
|
24
|
+
/**
|
|
25
|
+
* The most likely language after <|startoftranscript|>, as the original Whisper does. transformers.js does
|
|
26
|
+
* not detect it (without a language it assumes English and "transcribes" by translating), so it is done here.
|
|
27
|
+
*/
|
|
28
|
+
export declare function mostLikelyLanguage(logits: Pick<Tensor, 'dims' | 'data'>, langToId: Record<string, number>): string;
|
|
29
|
+
/** Local Whisper with transformers.js. The model is downloaded once; afterwards everything works offline. */
|
|
30
|
+
export declare function createWhisper(options: WhisperOptions): Promise<Transcriber>;
|
|
31
|
+
//# sourceMappingURL=stt.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../src/stt.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0D,MAAM,EAAE,MAAM,2BAA2B,CAAC;AAE3G,OAAO,EAAE,KAAK,GAAG,EAAU,MAAM,UAAU,CAAC;AAE5C,sBAAsB;AACtB,MAAM,WAAW,WAAW;IAC1B,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAClD;AAED,yFAAyF;AACzF,eAAO,MAAM,qBAAqB,iCAAiC,CAAC;AAEpE,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kGAAkG;IAClG,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,GAAG,CAAC,EAAE,GAAG,CAAC;CACX;AAED,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,KAAK,CAAC;AAE1C,0FAA0F;AAC1F,eAAO,MAAM,aAAa,GAAI,WAAU,MAAM,CAAC,QAA2B,KAAG,aAAuD,CAAC;AAOrI;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAUlH;AAED,6GAA6G;AAC7G,wBAAsB,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAmBjF"}
|
package/dist/stt.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { env, pipeline, Tensor } from '@huggingface/transformers';
|
|
2
|
+
import { silent } from './log.js';
|
|
3
|
+
/** small: much more accurate than base in Spanish (≈3 s per sentence on CPU, 241 MB). */
|
|
4
|
+
export const DEFAULT_WHISPER_MODEL = 'onnx-community/whisper-small';
|
|
5
|
+
/** On Windows the graphics card (DirectML ships with ONNX Runtime); elsewhere the CPU. */
|
|
6
|
+
export const defaultDevice = (platform = process.platform) => (platform === 'win32' ? 'dml' : 'cpu');
|
|
7
|
+
/**
|
|
8
|
+
* The most likely language after <|startoftranscript|>, as the original Whisper does. transformers.js does
|
|
9
|
+
* not detect it (without a language it assumes English and "transcribes" by translating), so it is done here.
|
|
10
|
+
*/
|
|
11
|
+
export function mostLikelyLanguage(logits, langToId) {
|
|
12
|
+
const vocab = logits.dims.at(-1);
|
|
13
|
+
const data = logits.data;
|
|
14
|
+
const row = data.subarray(data.length - vocab);
|
|
15
|
+
let best = 'en';
|
|
16
|
+
let score = -Infinity;
|
|
17
|
+
for (const [token, id] of Object.entries(langToId)) {
|
|
18
|
+
if (row[id] > score)
|
|
19
|
+
[best, score] = [token.slice(2, -2), row[id]];
|
|
20
|
+
}
|
|
21
|
+
return best;
|
|
22
|
+
}
|
|
23
|
+
/** Local Whisper with transformers.js. The model is downloaded once; afterwards everything works offline. */
|
|
24
|
+
export async function createWhisper(options) {
|
|
25
|
+
const log = options.log ?? silent;
|
|
26
|
+
const device = options.device ?? 'cpu';
|
|
27
|
+
let transcriber;
|
|
28
|
+
let used = device;
|
|
29
|
+
try {
|
|
30
|
+
transcriber = await loadWhisper(options, device, log);
|
|
31
|
+
}
|
|
32
|
+
catch (error) {
|
|
33
|
+
if (device === 'cpu')
|
|
34
|
+
throw error;
|
|
35
|
+
log(`[whisper] cannot use ${device} (${error.message}); using the CPU`);
|
|
36
|
+
transcriber = await loadWhisper(options, 'cpu', log);
|
|
37
|
+
used = 'cpu';
|
|
38
|
+
}
|
|
39
|
+
// The first sentence on the graphics card is much slower (it compiles its kernels), so that cost is paid at
|
|
40
|
+
// startup.
|
|
41
|
+
const started = Date.now();
|
|
42
|
+
await transcriber.transcribe(new Float32Array(16_000));
|
|
43
|
+
log(`[whisper] ready on ${used} (${Date.now() - started} ms warm-up)`);
|
|
44
|
+
return transcriber;
|
|
45
|
+
}
|
|
46
|
+
async function loadWhisper({ model = DEFAULT_WHISPER_MODEL, language, cacheDir }, device, log) {
|
|
47
|
+
env.cacheDir = cacheDir;
|
|
48
|
+
const asr = (await pipeline('automatic-speech-recognition', model, { dtype: 'q8', device }));
|
|
49
|
+
const config = asr.model.generation_config;
|
|
50
|
+
const start = () => new Tensor('int64', BigInt64Array.from([BigInt(config.decoder_start_token_id)]), [1, 1]);
|
|
51
|
+
const transcribeIn = async (audio, lang) => {
|
|
52
|
+
const started = Date.now();
|
|
53
|
+
const output = await asr(audio, { task: 'transcribe', language: lang });
|
|
54
|
+
log(`[whisper] ${lang} · ${Date.now() - started} ms`);
|
|
55
|
+
return (Array.isArray(output) ? output.map((o) => o.text).join(' ') : output.text).trim();
|
|
56
|
+
};
|
|
57
|
+
// A fixed language is not detected.
|
|
58
|
+
if (language)
|
|
59
|
+
return { transcribe: (audio) => transcribeIn(audio, language) };
|
|
60
|
+
// Detects and transcribes with a single encoder pass (the slowest part). Uses an internal transformers.js
|
|
61
|
+
// function (version pinned in package.json); without it, the public path encodes twice.
|
|
62
|
+
const model_ = asr.model;
|
|
63
|
+
return {
|
|
64
|
+
async transcribe(audio) {
|
|
65
|
+
const started = Date.now();
|
|
66
|
+
const { input_features } = (await asr.processor(audio));
|
|
67
|
+
if (!model_._prepare_encoder_decoder_kwargs_for_generation) {
|
|
68
|
+
const { logits } = await model_({ input_features, decoder_input_ids: start() });
|
|
69
|
+
return transcribeIn(audio, mostLikelyLanguage(logits, config.lang_to_id));
|
|
70
|
+
}
|
|
71
|
+
const { encoder_outputs } = await model_._prepare_encoder_decoder_kwargs_for_generation({
|
|
72
|
+
inputs_tensor: input_features,
|
|
73
|
+
model_inputs: { input_features },
|
|
74
|
+
model_input_name: 'input_features',
|
|
75
|
+
generation_config: asr.model.generation_config,
|
|
76
|
+
});
|
|
77
|
+
const { logits } = await model_({ encoder_outputs, decoder_input_ids: start() });
|
|
78
|
+
const lang = mostLikelyLanguage(logits, config.lang_to_id);
|
|
79
|
+
const detected = Date.now();
|
|
80
|
+
const ids = await model_.generate({ inputs: input_features, encoder_outputs, language: lang, task: 'transcribe', max_new_tokens: 200 });
|
|
81
|
+
log(`[whisper] idioma detectado: ${lang} (${detected - started} ms) · transcribir ${Date.now() - detected} ms`);
|
|
82
|
+
return (asr.tokenizer.batch_decode(ids, { skip_special_tokens: true })[0] ?? '').trim();
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=stt.js.map
|
package/dist/stt.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stt.js","sourceRoot":"","sources":["../src/stt.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,2BAA2B,CAAC;AAE3G,OAAO,EAAY,MAAM,EAAE,MAAM,UAAU,CAAC;AAO5C,yFAAyF;AACzF,MAAM,CAAC,MAAM,qBAAqB,GAAG,8BAA8B,CAAC;AAiBpE,0FAA0F;AAC1F,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,WAA4B,OAAO,CAAC,QAAQ,EAAiB,EAAE,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAOrI;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAqC,EAAE,QAAgC;IACxG,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAoB,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IAC/C,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,QAAQ,CAAC;IACtB,KAAK,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnD,IAAI,GAAG,CAAC,EAAE,CAAE,GAAG,KAAK;YAAE,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6GAA6G;AAC7G,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAuB;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,MAAM,CAAC;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,IAAI,WAAwB,CAAC;IAC7B,IAAI,IAAI,GAAG,MAAM,CAAC;IAClB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,KAAK,KAAK;YAAE,MAAM,KAAK,CAAC;QAClC,GAAG,CAAC,wBAAwB,MAAM,KAAM,KAAe,CAAC,OAAO,kBAAkB,CAAC,CAAC;QACnF,WAAW,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QACrD,IAAI,GAAG,KAAK,CAAC;IACf,CAAC;IACH,4GAA4G;IAC5G,WAAW;IACT,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,WAAW,CAAC,UAAU,CAAC,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACvD,GAAG,CAAC,sBAAsB,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,cAAc,CAAC,CAAC;IACvE,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,EAAE,KAAK,GAAG,qBAAqB,EAAE,QAAQ,EAAE,QAAQ,EAAkB,EAAE,MAAqB,EAAE,GAAQ;IAC/H,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACxB,MAAM,GAAG,GAAG,CAAC,MAAM,QAAQ,CAAC,8BAA8B,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAuC,CAAC;IACnI,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,iBAAuD,CAAC;IACjF,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE7G,MAAM,YAAY,GAAG,KAAK,EAAE,KAAmB,EAAE,IAAY,EAAE,EAAE;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACxE,GAAG,CAAC,aAAa,IAAI,MAAM,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,KAAK,CAAC,CAAC;QACtD,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5F,CAAC,CAAC;IAEJ,oCAAoC;IAClC,IAAI,QAAQ;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;IAEhF,0GAA0G;IAC1G,wFAAwF;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,KAIlB,CAAC;IACF,OAAO;QACL,KAAK,CAAC,UAAU,CAAC,KAAK;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,MAAM,EAAE,cAAc,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAA+B,CAAC;YACtF,IAAI,CAAC,MAAM,CAAC,8CAA8C,EAAE,CAAC;gBAC3D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBAChF,OAAO,YAAY,CAAC,KAAK,EAAE,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5E,CAAC;YACD,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,8CAA8C,CAAC;gBACtF,aAAa,EAAE,cAAc;gBAC7B,YAAY,EAAE,EAAE,cAAc,EAAE;gBAChC,gBAAgB,EAAE,gBAAgB;gBAClC,iBAAiB,EAAE,GAAG,CAAC,KAAK,CAAC,iBAAiB;aAC/C,CAAC,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,EAAE,eAAe,EAAE,iBAAiB,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACjF,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;YAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;YACxI,GAAG,CAAC,+BAA+B,IAAI,KAAK,QAAQ,GAAG,OAAO,sBAAsB,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC;YAChH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1F,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/turn.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type Fetch } from './download.js';
|
|
2
|
+
/** Smart Turn v3.2 (BSD-2, Pipecat), pinned version from the official repository. About 8 MB, CPU. */
|
|
3
|
+
export declare const SMART_TURN: {
|
|
4
|
+
url: string;
|
|
5
|
+
sha256: string;
|
|
6
|
+
size: number;
|
|
7
|
+
};
|
|
8
|
+
/** Above this probability the user has finished talking (Smart Turn's own threshold). */
|
|
9
|
+
export declare const TURN_COMPLETE = 0.5;
|
|
10
|
+
/**
|
|
11
|
+
* Whether the user has finished their turn or is only pausing. It looks at intonation, not words, so it works
|
|
12
|
+
* for 23 languages.
|
|
13
|
+
*/
|
|
14
|
+
export interface TurnCheck {
|
|
15
|
+
/** Probability (0 to 1) that the user has finished. */
|
|
16
|
+
complete(audio: Float32Array): Promise<number>;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Prepared as Smart Turn expects: the last 8 s (zero-padded in front when shorter), normalised to mean 0 and
|
|
20
|
+
* variance 1, as the Whisper feature extractor does with do_normalize.
|
|
21
|
+
*/
|
|
22
|
+
export declare function prepareTurnAudio(audio: Float32Array): Float32Array;
|
|
23
|
+
/** Smart Turn: downloaded once to cacheDir and run locally in a few tens of milliseconds. */
|
|
24
|
+
export declare function createTurnCheck({ cacheDir, fetchImpl }: {
|
|
25
|
+
cacheDir: string;
|
|
26
|
+
fetchImpl?: Fetch;
|
|
27
|
+
}): Promise<TurnCheck>;
|
|
28
|
+
//# sourceMappingURL=turn.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"turn.d.ts","sourceRoot":"","sources":["../src/turn.ts"],"names":[],"mappings":"AAKA,OAAO,EAAY,KAAK,KAAK,EAAE,MAAM,eAAe,CAAC;AAErD,sGAAsG;AACtG,eAAO,MAAM,UAAU;;;;CAItB,CAAC;AAKF,yFAAyF;AACzF,eAAO,MAAM,aAAa,MAAM,CAAC;AAEjC;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,uDAAuD;IACvD,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAChD;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,YAAY,GAAG,YAAY,CAYlE;AAED,6FAA6F;AAC7F,wBAAsB,eAAe,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,KAAK,CAAA;CAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAuB1H"}
|
package/dist/turn.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { WhisperFeatureExtractor } from '@huggingface/transformers';
|
|
3
|
+
import { InferenceSession, Tensor } from 'onnxruntime-node';
|
|
4
|
+
import { download } from './download.js';
|
|
5
|
+
/** Smart Turn v3.2 (BSD-2, Pipecat), pinned version from the official repository. About 8 MB, CPU. */
|
|
6
|
+
export const SMART_TURN = {
|
|
7
|
+
url: 'https://huggingface.co/pipecat-ai/smart-turn-v3/resolve/f766f81d3cfdf7737ac64aad813d91bbfd56bf93/smart-turn-v3.2-cpu.onnx',
|
|
8
|
+
sha256: '2bb026316b14a660486a75b1733cd3fbab8c2fd0314dc9af7be49f8cca967e4f',
|
|
9
|
+
size: 8679182,
|
|
10
|
+
};
|
|
11
|
+
/** Smart Turn looks at the last 8 s of the turn, at 16 kHz. */
|
|
12
|
+
const SECONDS = 8;
|
|
13
|
+
const SAMPLES = SECONDS * 16_000;
|
|
14
|
+
/** Above this probability the user has finished talking (Smart Turn's own threshold). */
|
|
15
|
+
export const TURN_COMPLETE = 0.5;
|
|
16
|
+
/**
|
|
17
|
+
* Prepared as Smart Turn expects: the last 8 s (zero-padded in front when shorter), normalised to mean 0 and
|
|
18
|
+
* variance 1, as the Whisper feature extractor does with do_normalize.
|
|
19
|
+
*/
|
|
20
|
+
export function prepareTurnAudio(audio) {
|
|
21
|
+
const out = new Float32Array(SAMPLES);
|
|
22
|
+
const tail = audio.length > SAMPLES ? audio.subarray(audio.length - SAMPLES) : audio;
|
|
23
|
+
out.set(tail, SAMPLES - tail.length);
|
|
24
|
+
let mean = 0;
|
|
25
|
+
for (const s of out)
|
|
26
|
+
mean += s;
|
|
27
|
+
mean /= out.length;
|
|
28
|
+
let variance = 0;
|
|
29
|
+
for (const s of out)
|
|
30
|
+
variance += (s - mean) ** 2;
|
|
31
|
+
const scale = 1 / Math.sqrt(variance / out.length + 1e-7);
|
|
32
|
+
for (let i = 0; i < out.length; i++)
|
|
33
|
+
out[i] = (out[i] - mean) * scale;
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
/** Smart Turn: downloaded once to cacheDir and run locally in a few tens of milliseconds. */
|
|
37
|
+
export async function createTurnCheck({ cacheDir, fetchImpl }) {
|
|
38
|
+
const model = await download(SMART_TURN.url, join(cacheDir, 'smart-turn', 'smart-turn-v3.2-cpu.onnx'), SMART_TURN, fetchImpl);
|
|
39
|
+
const session = await InferenceSession.create(model);
|
|
40
|
+
// The same Whisper log-mel (80 bands), for 8 s instead of 30.
|
|
41
|
+
const features = new WhisperFeatureExtractor({
|
|
42
|
+
feature_size: 80,
|
|
43
|
+
sampling_rate: 16_000,
|
|
44
|
+
hop_length: 160,
|
|
45
|
+
n_fft: 400,
|
|
46
|
+
chunk_length: SECONDS,
|
|
47
|
+
n_samples: SAMPLES,
|
|
48
|
+
nb_max_frames: SAMPLES / 160,
|
|
49
|
+
});
|
|
50
|
+
const input = session.inputNames[0];
|
|
51
|
+
const output = session.outputNames[0];
|
|
52
|
+
return {
|
|
53
|
+
async complete(audio) {
|
|
54
|
+
const { input_features } = (await features(prepareTurnAudio(audio)));
|
|
55
|
+
const result = await session.run({ [input]: new Tensor('float32', input_features.data, input_features.dims) });
|
|
56
|
+
return result[output].data[0];
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=turn.js.map
|
package/dist/turn.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"turn.js","sourceRoot":"","sources":["../src/turn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE5D,OAAO,EAAE,QAAQ,EAAc,MAAM,eAAe,CAAC;AAErD,sGAAsG;AACtG,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,GAAG,EAAE,2HAA2H;IAChI,MAAM,EAAE,kEAAkE;IAC1E,IAAI,EAAE,OAAO;CACd,CAAC;AAEF,+DAA+D;AAC/D,MAAM,OAAO,GAAG,CAAC,CAAC;AAClB,MAAM,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AACjC,yFAAyF;AACzF,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAWjC;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAmB;IAClD,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACrF,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,GAAG;QAAE,IAAI,IAAI,CAAC,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC;IACnB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,GAAG;QAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;IACvE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAE,QAAQ,EAAE,SAAS,EAA2C;IACpG,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,0BAA0B,CAAC,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAC9H,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvD,8DAA8D;IAC5D,MAAM,QAAQ,GAAG,IAAI,uBAAuB,CAAC;QAC3C,YAAY,EAAE,EAAE;QAChB,aAAa,EAAE,MAAM;QACrB,UAAU,EAAE,GAAG;QACf,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,OAAO;QACrB,SAAS,EAAE,OAAO;QAClB,aAAa,EAAE,OAAO,GAAG,GAAG;KAC7B,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC;IAEvC,OAAO;QACL,KAAK,CAAC,QAAQ,CAAC,KAAK;YAClB,MAAM,EAAE,cAAc,EAAE,GAAG,CAAC,MAAM,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAA+D,CAAC;YACnI,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,IAAI,MAAM,CAAC,SAAS,EAAE,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/G,OAAQ,MAAM,CAAC,MAAM,CAAY,CAAC,IAAI,CAAC,CAAC,CAAW,CAAC;QACtD,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/vad.d.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/** The detector works on 30 ms blocks. */
|
|
2
|
+
export declare const FRAME_SAMPLES: number;
|
|
3
|
+
export interface VadOptions {
|
|
4
|
+
/** Consecutive voice blocks to decide the user started talking (90 ms). */
|
|
5
|
+
startFrames?: number;
|
|
6
|
+
/**
|
|
7
|
+
* Consecutive silent blocks to decide they finished (1.5 s, so short pauses while talking do not split the
|
|
8
|
+
* sentence).
|
|
9
|
+
*/
|
|
10
|
+
endFrames?: number;
|
|
11
|
+
/** Blocks kept from before the start, so the first syllable is not cut (0.3 s). */
|
|
12
|
+
preRollFrames?: number;
|
|
13
|
+
/** A sentence lasts at most 20 s. */
|
|
14
|
+
maxFrames?: number;
|
|
15
|
+
/** Minimum energy threshold, and how many times the background noise the voice must exceed. */
|
|
16
|
+
minThreshold?: number;
|
|
17
|
+
noiseFactor?: number;
|
|
18
|
+
}
|
|
19
|
+
export type VadState = 'waiting' | 'speaking' | 'done';
|
|
20
|
+
/**
|
|
21
|
+
* Energy-based voice detection. The room noise level lasts the whole conversation (it is not reset on every
|
|
22
|
+
* sentence): it falls fast with silence and rises very slowly, so voice is never mistaken for noise.
|
|
23
|
+
*/
|
|
24
|
+
export declare class VoiceDetector {
|
|
25
|
+
private readonly o;
|
|
26
|
+
private state;
|
|
27
|
+
private noise;
|
|
28
|
+
private calibrated;
|
|
29
|
+
private loud;
|
|
30
|
+
private quiet;
|
|
31
|
+
private voicedFrames;
|
|
32
|
+
private readonly preRoll;
|
|
33
|
+
private frames;
|
|
34
|
+
constructor(options?: VadOptions);
|
|
35
|
+
get threshold(): number;
|
|
36
|
+
get current(): VadState;
|
|
37
|
+
/** Consecutive silent blocks since the last voice of the current sentence. */
|
|
38
|
+
get quietFrames(): number;
|
|
39
|
+
/** How much of the sentence had real voice (above the threshold), in milliseconds. */
|
|
40
|
+
get voicedMs(): number;
|
|
41
|
+
/**
|
|
42
|
+
* `learnNoise` is false while the AI talks: its echo is not room noise, and learning it would make the user
|
|
43
|
+
* inaudible.
|
|
44
|
+
*/
|
|
45
|
+
push(frame: Float32Array, learnNoise?: boolean): VadState;
|
|
46
|
+
/** The whole sentence, including the chunk from before the start. */
|
|
47
|
+
utterance(): Float32Array;
|
|
48
|
+
/** Keeps only the last blocks of the current sentence (what came before was the AI's voice). */
|
|
49
|
+
keepLast(frames: number): void;
|
|
50
|
+
/** On to the next sentence. The learned noise level is kept. */
|
|
51
|
+
next(): void;
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=vad.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":"AAEA,0CAA0C;AAC1C,eAAO,MAAM,aAAa,QAA4B,CAAC;AAEvD,MAAM,WAAW,UAAU;IACzB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mFAAmF;IACnF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qCAAqC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+FAA+F;IAC/F,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;AAevD;;;GAGG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAuB;IACzC,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,KAAK,CAAK;IAClB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,MAAM,CAAsB;gBAExB,OAAO,GAAE,UAAe;IAYpC,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,OAAO,IAAI,QAAQ,CAEtB;IAED,8EAA8E;IAC9E,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,sFAAsF;IACtF,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED;;;OAGG;IACH,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,UAAU,UAAO,GAAG,QAAQ;IAiCtD,qEAAqE;IACrE,SAAS,IAAI,YAAY;IAUzB,gGAAgG;IAChG,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAM9B,gEAAgE;IAChE,IAAI,IAAI,IAAI;CAOb"}
|