linkgravity 1.4.1 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -19
- package/bin/cli.js +27 -3
- package/bin/platforms.js +18 -1
- package/bin/setup.js +43 -0
- package/package.json +4 -2
- package/requirements.txt +1 -0
- package/src/cogs/general_cog.py +0 -21
- package/src/cogs/voice_cog.py +81 -7
- package/src/config.py +15 -2
- package/src/core/logger.py +6 -0
- package/src/core/platform_health.py +28 -0
- package/src/handlers/thread_reply.py +1 -1
- package/src/main.py +33 -7
- package/src/main_discord.py +3 -0
- package/src/main_slack.py +252 -0
- package/src/main_telegram.py +3 -0
- package/src/messengers/base.py +5 -0
- package/src/messengers/discord_adapter.py +19 -1
- package/src/messengers/slack_adapter.py +492 -0
- package/src/services/audio_service.py +6 -1
- package/src/services/streaming.py +6 -5
- package/voice-service/index.js +14 -1001
package/voice-service/index.js
CHANGED
|
@@ -1,1032 +1,45 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
const originalLog = console.log;
|
|
4
|
-
const originalError = console.error;
|
|
5
|
-
|
|
6
|
-
function getTimestamp() {
|
|
7
|
-
const now = new Date();
|
|
8
|
-
const offset = now.getTimezoneOffset() * 60000;
|
|
9
|
-
const localTime = new Date(now.getTime() - offset);
|
|
10
|
-
return localTime.toISOString().replace('T', ' ').substring(0, 19);
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
console.log = function (...args) {
|
|
14
|
-
originalLog(`${getTimestamp()} INFO Voice:`, ...args);
|
|
15
|
-
};
|
|
16
|
-
console.error = function (...args) {
|
|
17
|
-
originalError(`${getTimestamp()} ERROR Voice:`, ...args);
|
|
18
|
-
};
|
|
19
|
-
const {
|
|
20
|
-
joinVoiceChannel,
|
|
21
|
-
createAudioPlayer,
|
|
22
|
-
createAudioResource,
|
|
23
|
-
AudioPlayerStatus,
|
|
24
|
-
EndBehaviorType,
|
|
25
|
-
VoiceConnectionStatus,
|
|
26
|
-
} = require('@discordjs/voice');
|
|
1
|
+
require('./logger'); // side-effect: timestamps console.log/error - must load before anything else logs
|
|
27
2
|
|
|
3
|
+
const { Client, GatewayIntentBits, Events } = require('discord.js');
|
|
28
4
|
const express = require('express');
|
|
29
|
-
const axios = require('axios');
|
|
30
|
-
const os = require('os');
|
|
31
|
-
const prism = require('prism-media');
|
|
32
|
-
const fs = require('fs');
|
|
33
|
-
const path = require('path');
|
|
34
|
-
const { Readable } = require('stream');
|
|
35
|
-
const { spawn } = require('child_process');
|
|
36
|
-
const ffmpegPath = require('ffmpeg-static');
|
|
37
5
|
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
if (fs.existsSync(aglJsonPath)) {
|
|
42
|
-
aglConfig = JSON.parse(fs.readFileSync(aglJsonPath, 'utf8'));
|
|
43
|
-
}
|
|
44
|
-
} catch (e) {
|
|
45
|
-
console.error('Failed to load lgy.json:', e.message);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
process.env.DISCORD_TOKEN =
|
|
49
|
-
aglConfig.discord_token || aglConfig.DISCORD_TOKEN || process.env.DISCORD_TOKEN;
|
|
6
|
+
const { aglConfig } = require('./config');
|
|
7
|
+
const state = require('./state');
|
|
8
|
+
const { registerRoutes } = require('./routes');
|
|
50
9
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
process.env.HTTPS_PROXY = '';
|
|
55
|
-
process.env.PYTHON_HOST = '';
|
|
10
|
+
if (aglConfig.voice_threshold) {
|
|
11
|
+
state.runtime.vadThreshold = parseInt(aglConfig.voice_threshold) || 3000;
|
|
12
|
+
}
|
|
56
13
|
|
|
57
14
|
const app = express();
|
|
58
15
|
app.use(express.json({ limit: '50mb' }));
|
|
59
16
|
|
|
60
|
-
let vadThreshold = 3000;
|
|
61
|
-
if (aglConfig.voice_threshold) {
|
|
62
|
-
vadThreshold = parseInt(aglConfig.voice_threshold) || 3000;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
17
|
const client = new Client({
|
|
66
18
|
intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates],
|
|
67
19
|
});
|
|
68
20
|
|
|
69
|
-
const connections = new Map();
|
|
70
|
-
const players = new Map();
|
|
71
|
-
|
|
72
|
-
function stereoToMono(buffer) {
|
|
73
|
-
// Discord voice receive is always 48kHz STEREO (see e.g. the
|
|
74
|
-
// official discordjs/voice-examples recorder, which decodes with
|
|
75
|
-
// channels: 2) - everything downstream here (WAV writing via
|
|
76
|
-
// createWavHeader, Rustpotter's config) was built assuming MONO
|
|
77
|
-
// input, so this converts the interleaved L/R stream to mono right
|
|
78
|
-
// after decoding, in one place, rather than decoding as channels: 1
|
|
79
|
-
// and having the native Opus decoder silently misinterpret an
|
|
80
|
-
// actually-stereo stream as mono.
|
|
81
|
-
const samples = buffer.length >> 2; // 2 bytes/sample * 2 channels
|
|
82
|
-
const mono = Buffer.alloc(samples * 2);
|
|
83
|
-
for (let i = 0; i < samples; i++) {
|
|
84
|
-
const l = buffer.readInt16LE(i * 4);
|
|
85
|
-
const r = buffer.readInt16LE(i * 4 + 2);
|
|
86
|
-
mono.writeInt16LE((l + r) >> 1, i * 2);
|
|
87
|
-
}
|
|
88
|
-
return mono;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
function createWavHeader(dataLength, sampleRate = 48000, channels = 1, bitDepth = 16) {
|
|
92
|
-
const buffer = Buffer.alloc(44);
|
|
93
|
-
buffer.write('RIFF', 0);
|
|
94
|
-
buffer.writeUInt32LE(36 + dataLength, 4);
|
|
95
|
-
buffer.write('WAVE', 8);
|
|
96
|
-
buffer.write('fmt ', 12);
|
|
97
|
-
buffer.writeUInt32LE(16, 16);
|
|
98
|
-
buffer.writeUInt16LE(1, 20);
|
|
99
|
-
buffer.writeUInt16LE(channels, 22);
|
|
100
|
-
buffer.writeUInt32LE(sampleRate, 24);
|
|
101
|
-
buffer.writeUInt32LE(sampleRate * channels * (bitDepth / 8), 28);
|
|
102
|
-
buffer.writeUInt16LE(channels * (bitDepth / 8), 32);
|
|
103
|
-
buffer.writeUInt16LE(bitDepth, 34);
|
|
104
|
-
buffer.write('data', 36);
|
|
105
|
-
buffer.writeUInt32LE(dataLength, 40);
|
|
106
|
-
return buffer;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
21
|
client.once(Events.ClientReady, () => {
|
|
110
22
|
console.log(`🎤 Node.js Voice Microservice is online as ${client.user.tag}`);
|
|
111
23
|
});
|
|
112
24
|
|
|
113
|
-
|
|
114
|
-
// This is the same unofficial endpoint Python's SpeechRecognition
|
|
115
|
-
// library (recognize_google) has used for years: audio encoded as FLAC,
|
|
116
|
-
// POSTed to Google's v2 speech API with the "chromium" key that library
|
|
117
|
-
// ships as its default. It's not a secret - it's the same publicly
|
|
118
|
-
// documented key referenced all over SpeechRecognition's own source and
|
|
119
|
-
// years of writeups (search "AIzaSyBOti4mM-6x9WDnZIjIeyEU21OpBXqWBgw
|
|
120
|
-
// google speech api" if you want to verify that yourself). It's still an
|
|
121
|
-
// unofficial/reverse-engineered API that Google could change or rate-
|
|
122
|
-
// limit at any time - same risk the project already had via Python, just
|
|
123
|
-
// no longer duplicated in two languages.
|
|
124
|
-
//
|
|
125
|
-
// Requires ffmpeg to do the WAV -> FLAC conversion - already a
|
|
126
|
-
// dependency of this package (ffmpeg-static bundles the binary per
|
|
127
|
-
// platform), so nothing extra to install.
|
|
128
|
-
const GOOGLE_STT_KEY = 'AIzaSyBOti4mM-6x9WDnZIjIeyEU21OpBXqWBgw';
|
|
129
|
-
|
|
130
|
-
function flacEncode(wavBuffer) {
|
|
131
|
-
return new Promise((resolve, reject) => {
|
|
132
|
-
const ff = spawn(ffmpegPath, [
|
|
133
|
-
'-hide_banner',
|
|
134
|
-
'-loglevel',
|
|
135
|
-
'error',
|
|
136
|
-
'-i',
|
|
137
|
-
'pipe:0',
|
|
138
|
-
'-f',
|
|
139
|
-
'flac',
|
|
140
|
-
'pipe:1',
|
|
141
|
-
]);
|
|
142
|
-
const out = [];
|
|
143
|
-
ff.stdout.on('data', (d) => out.push(d));
|
|
144
|
-
ff.stderr.on('data', () => {}); // -loglevel error already keeps this quiet in the normal case
|
|
145
|
-
ff.on('error', (err) => reject(new Error(`ffmpeg-static failed to run (${err.message})`)));
|
|
146
|
-
ff.on('close', (code) => {
|
|
147
|
-
if (code !== 0) return reject(new Error(`ffmpeg exited with code ${code}`));
|
|
148
|
-
resolve(Buffer.concat(out));
|
|
149
|
-
});
|
|
150
|
-
ff.stdin.write(wavBuffer);
|
|
151
|
-
ff.stdin.end();
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
async function googleSTT(wavBuffer, lang = 'ko-KR') {
|
|
156
|
-
let flacBuffer;
|
|
157
|
-
try {
|
|
158
|
-
flacBuffer = await flacEncode(wavBuffer);
|
|
159
|
-
} catch (err) {
|
|
160
|
-
console.error('[STT] FLAC encode failed:', err.message);
|
|
161
|
-
return null;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
let res;
|
|
165
|
-
try {
|
|
166
|
-
res = await fetch(
|
|
167
|
-
`https://www.google.com/speech-api/v2/recognize?output=json&client=chromium&lang=${encodeURIComponent(lang)}&key=${GOOGLE_STT_KEY}`,
|
|
168
|
-
{
|
|
169
|
-
method: 'POST',
|
|
170
|
-
headers: { 'Content-Type': 'audio/x-flac; rate=48000' },
|
|
171
|
-
body: flacBuffer,
|
|
172
|
-
},
|
|
173
|
-
);
|
|
174
|
-
} catch (err) {
|
|
175
|
-
console.error('[STT] Request to Google STT failed:', err.message);
|
|
176
|
-
return null;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
const raw = await res.text();
|
|
180
|
-
// Response is newline-delimited JSON, one object per line, e.g.:
|
|
181
|
-
// {"result":[]}
|
|
182
|
-
// {"result":[{"alternative":[{"transcript":"...","confidence":0.9}],"final":true}],"result_index":0}
|
|
183
|
-
for (const line of raw.trim().split('\n')) {
|
|
184
|
-
if (!line) continue;
|
|
185
|
-
try {
|
|
186
|
-
const obj = JSON.parse(line);
|
|
187
|
-
const transcript = obj.result?.[0]?.alternative?.[0]?.transcript;
|
|
188
|
-
if (transcript) return transcript.trim();
|
|
189
|
-
} catch (e) {
|
|
190
|
-
// not JSON / partial line - ignore
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
return null;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
const audioQueues = new Map();
|
|
197
|
-
const isPlaying = new Map();
|
|
198
|
-
|
|
199
|
-
function interruptTTS(guildId) {
|
|
200
|
-
const player = players.get(guildId);
|
|
201
|
-
let interrupted = false;
|
|
202
|
-
|
|
203
|
-
if (audioQueues.has(guildId)) {
|
|
204
|
-
// Queue now holds in-memory audio Buffers (see /play), not
|
|
205
|
-
// filepaths, so there's nothing on disk to clean up here.
|
|
206
|
-
audioQueues.set(guildId, []);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
if (player && player.state.status !== AudioPlayerStatus.Idle) {
|
|
210
|
-
player.stop();
|
|
211
|
-
console.log(`[VAD] Interrupted TTS in guild ${guildId}`);
|
|
212
|
-
interrupted = true;
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
isPlaying.set(guildId, false);
|
|
216
|
-
return interrupted;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
const activeStreams = new Map();
|
|
220
|
-
|
|
221
|
-
// user_id -> true, while that user is recording wake-word samples (see
|
|
222
|
-
// /enroll_start, /enroll_stop). Utterances from these users get routed
|
|
223
|
-
// to /enroll_sample instead of the normal wake-check/STT pipeline.
|
|
224
|
-
const enrollingUsers = new Set();
|
|
225
|
-
|
|
226
|
-
// guild_id -> ms epoch until which we're in the "awake, no need to repeat
|
|
227
|
-
// the wake word" window (set by Python via /set_active whenever it
|
|
228
|
-
// starts/renews that countdown - see VoiceCog._extend_active_window).
|
|
229
|
-
const activeUntil = new Map();
|
|
230
|
-
|
|
231
|
-
function isGuildActive(guildId) {
|
|
232
|
-
return Date.now() < (activeUntil.get(guildId) || 0);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
// --- Wake-word detection (Rustpotter, in-process) ---
|
|
236
|
-
// Runs entirely in this Node process - no Python round trip. Each
|
|
237
|
-
// enrolled user gets their own Rustpotter instance (built from the .rpw
|
|
238
|
-
// reference file this same process produces via /build_wakeword, called
|
|
239
|
-
// by EnrollmentManager._commit_enrollment once all samples are
|
|
240
|
-
// confirmed), cached here and fed audio directly as it streams in from
|
|
241
|
-
// Discord.
|
|
242
|
-
const WAKE_REF_DIR = path.join(os.homedir(), '.gemini', 'linkgravity', 'wake_refs');
|
|
243
|
-
|
|
244
|
-
let rustpotterModPromise = null;
|
|
245
|
-
function loadRustpotterModule() {
|
|
246
|
-
if (!rustpotterModPromise) {
|
|
247
|
-
rustpotterModPromise = (async () => {
|
|
248
|
-
// Bare "rustpotter-web" doesn't resolve under Node's ESM
|
|
249
|
-
// loader (the package only declares a "module" field, which
|
|
250
|
-
// is a bundler-only convention Node doesn't read) - the
|
|
251
|
-
// actual entry file has to be named explicitly.
|
|
252
|
-
//
|
|
253
|
-
// Using the full "rustpotter-web" package here (not the
|
|
254
|
-
// "-slim" variant this used to be) because it's the only one
|
|
255
|
-
// of the two that also exposes WakewordRefCreator - the
|
|
256
|
-
// builder API used by /build_wakeword below to create a new
|
|
257
|
-
// .rpw reference directly from wav samples, in-process, with
|
|
258
|
-
// no external binary. The wasm binary is ~6% bigger
|
|
259
|
-
// (810KB vs 761KB) which is irrelevant for a Node backend
|
|
260
|
-
// (this distinction only matters for browser bundle size,
|
|
261
|
-
// which is the slim variant's actual purpose).
|
|
262
|
-
const mod = await import('rustpotter-web/rustpotter_wasm.js');
|
|
263
|
-
const wasmPath = require.resolve('rustpotter-web/rustpotter_wasm_bg.wasm');
|
|
264
|
-
mod.initSync(fs.readFileSync(wasmPath));
|
|
265
|
-
return mod;
|
|
266
|
-
})();
|
|
267
|
-
}
|
|
268
|
-
return rustpotterModPromise;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
// userId -> { rustpotter, samplesPerFrame, residual: Int16Array }
|
|
272
|
-
const detectorCache = new Map();
|
|
273
|
-
|
|
274
|
-
// Wake-word confirm cutoff - must stay well above ~0.05 (rustpotter's countdown never finalizes if noise/silence clears it too); 0.4 chosen after live use kept narrowly missing genuine hits just under 0.5.
|
|
275
|
-
const WAKE_MATCH_THRESHOLD = 0.4;
|
|
276
|
-
|
|
277
|
-
async function getDetectorForUser(userId) {
|
|
278
|
-
if (detectorCache.has(userId)) return detectorCache.get(userId);
|
|
279
|
-
|
|
280
|
-
const userDir = path.join(WAKE_REF_DIR, userId);
|
|
281
|
-
if (!fs.existsSync(userDir)) return null; // not enrolled
|
|
282
|
-
|
|
283
|
-
const rpwFile = fs.readdirSync(userDir).find((f) => f.endsWith('.rpw'));
|
|
284
|
-
if (!rpwFile) return null; // samples exist but .rpw build hasn't happened/failed - see _commit_enrollment
|
|
285
|
-
|
|
286
|
-
const mod = await loadRustpotterModule();
|
|
287
|
-
const config = mod.RustpotterConfig.new();
|
|
288
|
-
config.setSampleRate(48000);
|
|
289
|
-
config.setSampleFormat(mod.SampleFormat.i16);
|
|
290
|
-
config.setChannels(1);
|
|
291
|
-
// See WAKE_MATCH_THRESHOLD's comment above - this MUST be a real,
|
|
292
|
-
// meaningful cutoff (not near-zero) for rustpotter's internal
|
|
293
|
-
// confirm-after-N-more-frames logic to ever finalize a detection.
|
|
294
|
-
config.setThreshold(WAKE_MATCH_THRESHOLD);
|
|
295
|
-
config.setAveragedThreshold(0);
|
|
296
|
-
// Default is 1 - accepting a match the very first time it's ever
|
|
297
|
-
// the leading candidate, even for just one internal frame. Raised
|
|
298
|
-
// to 4 (was 3) so a candidate has to keep winning for a few frames
|
|
299
|
-
// running before it's trusted - this matters more now that
|
|
300
|
-
// scoreMode is back to Max (see below), which is more permissive
|
|
301
|
-
// per-frame than Median was, so this is the main compensating knob
|
|
302
|
-
// against one-off spurious spikes from unrelated speech.
|
|
303
|
-
config.setMinScores(4);
|
|
304
|
-
// Max: score is whichever of the 5 enrolled samples matches best.
|
|
305
|
-
// Median (tried between the two calibration notes below) requires
|
|
306
|
-
// the middle-ranked sample to also score well, which in practice
|
|
307
|
-
// means all 5 enrollment recordings need fairly consistent
|
|
308
|
-
// tone/pace/delivery - real speech isn't that consistent, so Median
|
|
309
|
-
// made real matches with naturally varied enrollment recordings
|
|
310
|
-
// score worse, not just false positives. Max lets a genuinely
|
|
311
|
-
// varied set of 5 recordings (calm, rushed, questioning, etc.) each
|
|
312
|
-
// cover a different real-world delivery, at the cost of also being
|
|
313
|
-
// easier to trip with something that merely resembles ONE of the 5
|
|
314
|
-
// by chance - minScores(4) above and the STT jamo cross-check in
|
|
315
|
-
// VoiceCog.handle_stt_input (tightened for short wake words
|
|
316
|
-
// specifically) are what compensate for that on the other two axes
|
|
317
|
-
// instead. Still needs field-tuning against real usage logs - see
|
|
318
|
-
// README's known-issues section.
|
|
319
|
-
config.setScoreMode(mod.ScoreMode.max);
|
|
320
|
-
|
|
321
|
-
const rustpotter = mod.Rustpotter.new(config);
|
|
322
|
-
rustpotter.addWakeword(rpwFile, fs.readFileSync(path.join(userDir, rpwFile)));
|
|
323
|
-
|
|
324
|
-
// Second, diagnostic-only instance fed the exact same audio in
|
|
325
|
-
// parallel, purely so "no match" cases still show a real number in
|
|
326
|
-
// the logs. It NEVER gates wake behavior - only entry.rustpotter
|
|
327
|
-
// above does that. Why a separate instance instead of reading a
|
|
328
|
-
// low score off the real one: rustpotter's own scoring
|
|
329
|
-
// (wakeword.run_detection in the Rust source) filters out any
|
|
330
|
-
// frame that doesn't already clear `threshold` before it's even
|
|
331
|
-
// tracked internally, so there is no bound API that exposes a
|
|
332
|
-
// sub-threshold score - process*() returns undefined for those,
|
|
333
|
-
// full stop. Naively lowering the REAL detector's threshold to see
|
|
334
|
-
// more doesn't work either (that's exactly the bug from the
|
|
335
|
-
// local-wake migration: near-zero threshold means background noise
|
|
336
|
-
// clears it on almost every frame, which keeps resetting
|
|
337
|
-
// detection_countdown before it can ever reach 0, so confirmation
|
|
338
|
-
// never completes for anyone, real wake word included).
|
|
339
|
-
//
|
|
340
|
-
// This instance sidesteps that by setting eager+minScores(1), so it
|
|
341
|
-
// finalizes and hands back a real score the very first time ANY
|
|
342
|
-
// frame clears its own (near-zero) threshold, instead of waiting on
|
|
343
|
-
// the countdown - the exact same escape hatch that would break
|
|
344
|
-
// gating on the real detector is safe to lean on here because nothing
|
|
345
|
-
// downstream ever treats this instance's output as a wake trigger.
|
|
346
|
-
// scoreMode is kept identical to the real detector so the number
|
|
347
|
-
// itself is a genuine, comparable similarity score - just captured
|
|
348
|
-
// more eagerly, not computed differently.
|
|
349
|
-
const diagConfig = mod.RustpotterConfig.new();
|
|
350
|
-
diagConfig.setSampleRate(48000);
|
|
351
|
-
diagConfig.setSampleFormat(mod.SampleFormat.i16);
|
|
352
|
-
diagConfig.setChannels(1);
|
|
353
|
-
diagConfig.setThreshold(0.01);
|
|
354
|
-
diagConfig.setAveragedThreshold(0);
|
|
355
|
-
diagConfig.setMinScores(1);
|
|
356
|
-
diagConfig.setEager(true);
|
|
357
|
-
diagConfig.setScoreMode(mod.ScoreMode.max);
|
|
358
|
-
const diagRustpotter = mod.Rustpotter.new(diagConfig);
|
|
359
|
-
diagRustpotter.addWakeword(rpwFile, fs.readFileSync(path.join(userDir, rpwFile)));
|
|
360
|
-
|
|
361
|
-
const entry = {
|
|
362
|
-
rustpotter,
|
|
363
|
-
samplesPerFrame: rustpotter.getSamplesPerFrame(),
|
|
364
|
-
residual: new Int16Array(0),
|
|
365
|
-
diag: {
|
|
366
|
-
rustpotter: diagRustpotter,
|
|
367
|
-
samplesPerFrame: diagRustpotter.getSamplesPerFrame(),
|
|
368
|
-
residual: new Int16Array(0),
|
|
369
|
-
},
|
|
370
|
-
};
|
|
371
|
-
console.log(
|
|
372
|
-
`[Wake] Loaded detector for ${userId} from ${rpwFile}: samplesPerFrame=${entry.samplesPerFrame}`,
|
|
373
|
-
);
|
|
374
|
-
detectorCache.set(userId, entry);
|
|
375
|
-
return entry;
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
// Feeds one Discord PCM chunk to a detector, exactly once per sample and
|
|
379
|
-
// frame-aligned via a residual carryover - NOT by periodically
|
|
380
|
-
// re-snapshotting a growing buffer, since Rustpotter's internal window
|
|
381
|
-
// expects a genuinely continuous stream. Returns a RustpotterDetection
|
|
382
|
-
// if any complete frame in this chunk triggered one, else null.
|
|
383
|
-
//
|
|
384
|
-
// Frames are fed back-to-back with NO external overlap - rustpotter
|
|
385
|
-
// internally extracts multiple overlapping 10ms-shifted MFCCs from each
|
|
386
|
-
// ~30ms buffer passed in (confirmed against its Rust source), so the
|
|
387
|
-
// caller just needs to keep the stream continuous, not overlap it.
|
|
388
|
-
function feedPCMToDetector(entry, chunk) {
|
|
389
|
-
const incoming = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
|
|
390
|
-
let combined = incoming;
|
|
391
|
-
if (entry.residual.length) {
|
|
392
|
-
combined = new Int16Array(entry.residual.length + incoming.length);
|
|
393
|
-
combined.set(entry.residual, 0);
|
|
394
|
-
combined.set(incoming, entry.residual.length);
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
let offset = 0;
|
|
398
|
-
let detection = null;
|
|
399
|
-
while (combined.length - offset >= entry.samplesPerFrame) {
|
|
400
|
-
const frame = combined.subarray(offset, offset + entry.samplesPerFrame);
|
|
401
|
-
const result = entry.rustpotter.processI16(frame);
|
|
402
|
-
if (result) detection = result;
|
|
403
|
-
offset += entry.samplesPerFrame;
|
|
404
|
-
}
|
|
405
|
-
entry.residual = combined.subarray(offset);
|
|
406
|
-
return detection;
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
function setupReceiver(connection, guildId) {
|
|
410
|
-
const receiver = connection.receiver;
|
|
411
|
-
|
|
412
|
-
receiver.speaking.removeAllListeners('start');
|
|
413
|
-
|
|
414
|
-
receiver.speaking.on('start', (userId) => {
|
|
415
|
-
if (client.user.id === userId) return;
|
|
416
|
-
|
|
417
|
-
if (activeStreams.get(userId)) {
|
|
418
|
-
return;
|
|
419
|
-
}
|
|
420
|
-
activeStreams.set(userId, true);
|
|
421
|
-
|
|
422
|
-
let hasInterrupted = false;
|
|
423
|
-
|
|
424
|
-
const opusStream = receiver.subscribe(userId, {
|
|
425
|
-
end: {
|
|
426
|
-
behavior: EndBehaviorType.Manual,
|
|
427
|
-
},
|
|
428
|
-
});
|
|
429
|
-
const pcmStream = opusStream.pipe(
|
|
430
|
-
new prism.opus.Decoder({ rate: 48000, channels: 2, frameSize: 960 }),
|
|
431
|
-
);
|
|
432
|
-
|
|
433
|
-
// A single corrupted/dropped Opus packet (packet loss, a bad
|
|
434
|
-
// DAVE re-encrypt, whatever) throws inside prism-media's
|
|
435
|
-
// decoder. Streams turn a thrown _transform error into an
|
|
436
|
-
// 'error' event - with no listener here, Node's default for an
|
|
437
|
-
// unhandled 'error' event is to crash the ENTIRE process, not
|
|
438
|
-
// just this one user's utterance. Handling it here keeps voice
|
|
439
|
-
// alive for everyone else (and for this user's next utterance).
|
|
440
|
-
opusStream.on('error', (err) => {
|
|
441
|
-
console.error(`[Voice] Opus stream error for ${userId}:`, err.message);
|
|
442
|
-
forceEndStream();
|
|
443
|
-
});
|
|
444
|
-
pcmStream.on('error', (err) => {
|
|
445
|
-
console.error(
|
|
446
|
-
`[Voice] Opus decode error for ${userId} (bad/corrupted packet):`,
|
|
447
|
-
err.message,
|
|
448
|
-
);
|
|
449
|
-
forceEndStream();
|
|
450
|
-
});
|
|
451
|
-
|
|
452
|
-
const chunks = [];
|
|
453
|
-
|
|
454
|
-
let hasEnded = false;
|
|
455
|
-
|
|
456
|
-
const forceEndStream = () => {
|
|
457
|
-
if (hasEnded) return;
|
|
458
|
-
try {
|
|
459
|
-
opusStream.destroy();
|
|
460
|
-
} catch (e) {}
|
|
461
|
-
try {
|
|
462
|
-
pcmStream.destroy();
|
|
463
|
-
} catch (e) {}
|
|
464
|
-
pcmStream.emit('end');
|
|
465
|
-
};
|
|
466
|
-
|
|
467
|
-
const maxDurationTimer = setTimeout(forceEndStream, 30000);
|
|
468
|
-
|
|
469
|
-
// --- Wake-word gating (Rustpotter, in-process) ---
|
|
470
|
-
// Runs entirely inside this Node process now - no HTTP round
|
|
471
|
-
// trip, no Python involvement. Detector instances are cached per
|
|
472
|
-
// Discord user (see getDetectorForUser) and fed every PCM chunk
|
|
473
|
-
// as it streams in below, frame-aligned via feedPCMToDetector's
|
|
474
|
-
// residual carryover - not via periodic re-snapshotting like the
|
|
475
|
-
// old /wake_check design, since Rustpotter expects each sample
|
|
476
|
-
// fed exactly once, in order.
|
|
477
|
-
let wakeConfirmed = false;
|
|
478
|
-
let matchedWakeWord = null;
|
|
479
|
-
let detectorEntry = null;
|
|
480
|
-
let bestWakeScore = 0;
|
|
481
|
-
let bestWakeScoreName = null;
|
|
482
|
-
let bestDiagScore = 0;
|
|
483
|
-
let bestDiagScoreName = null;
|
|
484
|
-
|
|
485
|
-
if (!enrollingUsers.has(userId) && !isGuildActive(guildId)) {
|
|
486
|
-
getDetectorForUser(userId)
|
|
487
|
-
.then((entry) => {
|
|
488
|
-
if (entry) {
|
|
489
|
-
entry.rustpotter.reset();
|
|
490
|
-
entry.residual = new Int16Array(0);
|
|
491
|
-
entry.diag.rustpotter.reset();
|
|
492
|
-
entry.diag.residual = new Int16Array(0);
|
|
493
|
-
detectorEntry = entry;
|
|
494
|
-
}
|
|
495
|
-
})
|
|
496
|
-
.catch((err) =>
|
|
497
|
-
console.error(`[Wake] Failed to load detector for ${userId}:`, err.message),
|
|
498
|
-
);
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
// --- Live "listening..." indicator ---
|
|
502
|
-
// Only runs during the active/awake window (bounded, default
|
|
503
|
-
// 60s) - NOT for every VAD-detected sound like the old version,
|
|
504
|
-
// which is what made it expensive before. STT here reuses the
|
|
505
|
-
// same googleSTT() call that runs at utterance end anyway, just
|
|
506
|
-
// invoked earlier/more often on the growing buffer for live
|
|
507
|
-
// feedback.
|
|
508
|
-
const PARTIAL_INTERVAL_MS = 1500;
|
|
509
|
-
const PARTIAL_MIN_NEW_BYTES = 24000;
|
|
510
|
-
let lastPartialLength = 0;
|
|
511
|
-
let partialSent = false;
|
|
512
|
-
const partialTimer = setInterval(async () => {
|
|
513
|
-
if (hasEnded || !isSpeaking || enrollingUsers.has(userId)) return;
|
|
514
|
-
if (!isGuildActive(guildId)) return;
|
|
515
|
-
|
|
516
|
-
const currentLength = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
517
|
-
if (currentLength - lastPartialLength < PARTIAL_MIN_NEW_BYTES) return;
|
|
518
|
-
lastPartialLength = currentLength;
|
|
519
|
-
|
|
520
|
-
const windowPcm = Buffer.concat(chunks);
|
|
521
|
-
const wavHeader = createWavHeader(windowPcm.length);
|
|
522
|
-
const wavBuffer = Buffer.concat([wavHeader, windowPcm]);
|
|
523
|
-
|
|
524
|
-
const text = await googleSTT(wavBuffer);
|
|
525
|
-
partialSent = true;
|
|
526
|
-
fetch('http://127.0.0.1:18080/stt_partial', {
|
|
527
|
-
method: 'POST',
|
|
528
|
-
headers: { 'Content-Type': 'application/json' },
|
|
529
|
-
body: JSON.stringify({ guild_id: guildId, text }),
|
|
530
|
-
}).catch((err) =>
|
|
531
|
-
console.error(`[STT] Failed to send partial text to Python:`, err.message),
|
|
532
|
-
);
|
|
533
|
-
}, PARTIAL_INTERVAL_MS);
|
|
534
|
-
|
|
535
|
-
let bgNoiseRMS = 500;
|
|
536
|
-
let isSpeaking = false;
|
|
537
|
-
let silenceBytes = 0;
|
|
538
|
-
let silenceTimer = null;
|
|
539
|
-
|
|
540
|
-
pcmStream.on('data', (rawChunk) => {
|
|
541
|
-
if (hasEnded) return;
|
|
542
|
-
const chunk = stereoToMono(rawChunk); // see stereoToMono's comment - decoder gives real stereo now
|
|
543
|
-
|
|
544
|
-
let sumSquare = 0;
|
|
545
|
-
for (let i = 0; i < chunk.length; i += 2) {
|
|
546
|
-
const sample = chunk.readInt16LE(i);
|
|
547
|
-
sumSquare += sample * sample;
|
|
548
|
-
}
|
|
549
|
-
const rms = Math.sqrt(sumSquare / (chunk.length / 2));
|
|
550
|
-
|
|
551
|
-
const isBotPlaying = isPlaying.get(guildId) || false;
|
|
552
|
-
|
|
553
|
-
if (!hasInterrupted) {
|
|
554
|
-
const dynamicThreshold = isBotPlaying ? vadThreshold * 3 : vadThreshold;
|
|
555
|
-
if (rms > dynamicThreshold) {
|
|
556
|
-
if (interruptTTS(guildId)) {
|
|
557
|
-
console.log(
|
|
558
|
-
`[VAD] Loud voice detected (${Math.round(rms)}), interrupting TTS (Threshold: ${dynamicThreshold})`,
|
|
559
|
-
);
|
|
560
|
-
hasInterrupted = true;
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
|
-
|
|
565
|
-
if (isPlaying.get(guildId)) {
|
|
566
|
-
return;
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
chunks.push(chunk);
|
|
570
|
-
|
|
571
|
-
if (detectorEntry) {
|
|
572
|
-
const detection = feedPCMToDetector(detectorEntry, chunk);
|
|
573
|
-
if (detection && detection.getScore() > bestWakeScore) {
|
|
574
|
-
bestWakeScore = detection.getScore();
|
|
575
|
-
bestWakeScoreName = detection.getName();
|
|
576
|
-
}
|
|
577
|
-
const diagDetection = feedPCMToDetector(detectorEntry.diag, chunk);
|
|
578
|
-
if (diagDetection && diagDetection.getScore() > bestDiagScore) {
|
|
579
|
-
bestDiagScore = diagDetection.getScore();
|
|
580
|
-
bestDiagScoreName = diagDetection.getName();
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
if (!isSpeaking) {
|
|
585
|
-
bgNoiseRMS = bgNoiseRMS * 0.98 + rms * 0.02;
|
|
586
|
-
bgNoiseRMS = Math.max(50, Math.min(bgNoiseRMS, 3000));
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
const threshold = Math.max(bgNoiseRMS * 2.0, 800);
|
|
590
|
-
|
|
591
|
-
if (rms > threshold) {
|
|
592
|
-
isSpeaking = true;
|
|
593
|
-
silenceBytes = 0;
|
|
594
|
-
} else {
|
|
595
|
-
if (isSpeaking) {
|
|
596
|
-
silenceBytes += chunk.length;
|
|
597
|
-
if (silenceBytes >= 76800) {
|
|
598
|
-
forceEndStream();
|
|
599
|
-
return;
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
}
|
|
603
|
-
|
|
604
|
-
if (isSpeaking) {
|
|
605
|
-
if (silenceTimer) clearTimeout(silenceTimer);
|
|
606
|
-
silenceTimer = setTimeout(() => forceEndStream(), 800);
|
|
607
|
-
}
|
|
608
|
-
});
|
|
609
|
-
|
|
610
|
-
pcmStream.on('end', async () => {
|
|
611
|
-
if (hasEnded) return;
|
|
612
|
-
hasEnded = true;
|
|
613
|
-
clearTimeout(maxDurationTimer);
|
|
614
|
-
clearInterval(partialTimer);
|
|
615
|
-
if (silenceTimer) clearTimeout(silenceTimer);
|
|
616
|
-
|
|
617
|
-
activeStreams.delete(userId);
|
|
618
|
-
|
|
619
|
-
if (detectorEntry) {
|
|
620
|
-
// rustpotter needs MORE frames fed after a candidate
|
|
621
|
-
// match before it finalizes one (detection_countdown
|
|
622
|
-
// counts down from max_mfcc_frames/2 before confirming
|
|
623
|
-
// or discarding a partial match - see the Rust source's
|
|
624
|
-
// detector.rs). Live capture stops the instant the user
|
|
625
|
-
// stops talking, so without this, a genuine match
|
|
626
|
-
// candidate never gets the chance to finish counting
|
|
627
|
-
// down and is silently discarded - this is exactly why
|
|
628
|
-
// rustpotter-cli's own `test` command pads its input
|
|
629
|
-
// with 100 extra silent frames before processing (see
|
|
630
|
-
// its test.rs), and why testing the identical captured
|
|
631
|
-
// audio through the CLI scored well while this always
|
|
632
|
-
// failed live. Same fix here: flush the residual plus
|
|
633
|
-
// a few seconds of silence through the same detector
|
|
634
|
-
// before deciding pass/fail.
|
|
635
|
-
const paddingBuffer = Buffer.alloc(detectorEntry.samplesPerFrame * 100 * 2);
|
|
636
|
-
const paddingDetection = feedPCMToDetector(detectorEntry, paddingBuffer);
|
|
637
|
-
if (paddingDetection && paddingDetection.getScore() > bestWakeScore) {
|
|
638
|
-
bestWakeScore = paddingDetection.getScore();
|
|
639
|
-
bestWakeScoreName = paddingDetection.getName();
|
|
640
|
-
}
|
|
641
|
-
const diagPaddingBuffer = Buffer.alloc(
|
|
642
|
-
detectorEntry.diag.samplesPerFrame * 100 * 2,
|
|
643
|
-
);
|
|
644
|
-
const diagPaddingDetection = feedPCMToDetector(
|
|
645
|
-
detectorEntry.diag,
|
|
646
|
-
diagPaddingBuffer,
|
|
647
|
-
);
|
|
648
|
-
if (diagPaddingDetection && diagPaddingDetection.getScore() > bestDiagScore) {
|
|
649
|
-
bestDiagScore = diagPaddingDetection.getScore();
|
|
650
|
-
bestDiagScoreName = diagPaddingDetection.getName();
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
// Real pass/fail uses bestWakeScore; bestDiagScore is a separate, much looser detector shown only for "how close" - not on the same scale, not comparable to WAKE_MATCH_THRESHOLD.
|
|
654
|
-
wakeConfirmed = bestWakeScore >= WAKE_MATCH_THRESHOLD;
|
|
655
|
-
matchedWakeWord = wakeConfirmed ? bestWakeScoreName : null;
|
|
656
|
-
console.log(
|
|
657
|
-
wakeConfirmed
|
|
658
|
-
? `[Wake] ${userId}: CONFIRMED (score ${bestWakeScore.toFixed(3)} for ` +
|
|
659
|
-
`"${bestWakeScoreName}", threshold ${WAKE_MATCH_THRESHOLD})`
|
|
660
|
-
: `[Wake] ${userId}: no match (score ${bestWakeScore.toFixed(3)}, ` +
|
|
661
|
-
`threshold ${WAKE_MATCH_THRESHOLD}; diagnostic-only closeness ` +
|
|
662
|
-
`${bestDiagScore.toFixed(3)} for "${bestDiagScoreName ?? 'n/a'}" - ` +
|
|
663
|
-
`different scoring config, not directly comparable to the threshold)`,
|
|
664
|
-
);
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
const pcmBuffer = Buffer.concat(chunks);
|
|
668
|
-
|
|
669
|
-
// Enrollment mode: this utterance is a wake-word reference
|
|
670
|
-
// sample, not a command - hand it straight to Python and skip
|
|
671
|
-
// wake-check/STT/active-window logic entirely.
|
|
672
|
-
if (enrollingUsers.has(userId)) {
|
|
673
|
-
if (pcmBuffer.length < 4000) return; // too short to be a real sample
|
|
674
|
-
const wavHeader = createWavHeader(pcmBuffer.length);
|
|
675
|
-
const wavBuffer = Buffer.concat([wavHeader, pcmBuffer]);
|
|
676
|
-
try {
|
|
677
|
-
await fetch(
|
|
678
|
-
`http://127.0.0.1:18080/enroll_sample?user_id=${encodeURIComponent(userId)}`,
|
|
679
|
-
{
|
|
680
|
-
method: 'POST',
|
|
681
|
-
headers: { 'Content-Type': 'application/octet-stream' },
|
|
682
|
-
body: wavBuffer,
|
|
683
|
-
},
|
|
684
|
-
);
|
|
685
|
-
} catch (err) {
|
|
686
|
-
console.error(`[Enroll] Failed to send sample to Python:`, err.message);
|
|
687
|
-
}
|
|
688
|
-
return;
|
|
689
|
-
}
|
|
690
|
-
|
|
691
|
-
// Was 24000 (250ms) - cut off short Korean replies ("네"/"어"/"응"); noise is filtered upstream by isSpeaking's RMS/sustain check, not by duration.
|
|
692
|
-
if (pcmBuffer.length < 9600) {
|
|
693
|
-
if (partialSent) {
|
|
694
|
-
fetch('http://127.0.0.1:18080/stt_partial_cancel', {
|
|
695
|
-
method: 'POST',
|
|
696
|
-
headers: { 'Content-Type': 'application/json' },
|
|
697
|
-
body: JSON.stringify({ guild_id: guildId }),
|
|
698
|
-
}).catch(() => {});
|
|
699
|
-
}
|
|
700
|
-
return;
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
// Whether this utterance is worth transcribing at all:
|
|
704
|
-
// - isGuildActive(): already awake, no need to repeat the
|
|
705
|
-
// wake word (this is also what makes the live "listening"
|
|
706
|
-
// indicator above meaningful - same window).
|
|
707
|
-
// - wakeConfirmed: our in-process Rustpotter detector matched this
|
|
708
|
-
// user's voice against their enrolled samples.
|
|
709
|
-
// There is deliberately no "unenrolled users always get
|
|
710
|
-
// transcribed" fallback anymore - that existed only to feed
|
|
711
|
-
// the Python side's old text-similarity wake-word matching,
|
|
712
|
-
// which has been removed (it was exactly the always-on
|
|
713
|
-
// recognition overhead this Rustpotter migration was meant to
|
|
714
|
-
// get rid of). An unenrolled user simply can't wake the bot
|
|
715
|
-
// by voice until they run /sound.
|
|
716
|
-
const shouldTranscribe = isGuildActive(guildId) || wakeConfirmed;
|
|
717
|
-
|
|
718
|
-
if (!shouldTranscribe) {
|
|
719
|
-
if (partialSent) {
|
|
720
|
-
fetch('http://127.0.0.1:18080/stt_partial_cancel', {
|
|
721
|
-
method: 'POST',
|
|
722
|
-
headers: { 'Content-Type': 'application/json' },
|
|
723
|
-
body: JSON.stringify({ guild_id: guildId }),
|
|
724
|
-
}).catch(() => {});
|
|
725
|
-
}
|
|
726
|
-
return;
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
const wavHeader = createWavHeader(pcmBuffer.length);
|
|
730
|
-
const wavBuffer = Buffer.concat([wavHeader, pcmBuffer]);
|
|
731
|
-
const text = await googleSTT(wavBuffer);
|
|
732
|
-
|
|
733
|
-
try {
|
|
734
|
-
await fetch('http://127.0.0.1:18080/stt_input', {
|
|
735
|
-
method: 'POST',
|
|
736
|
-
headers: { 'Content-Type': 'application/json' },
|
|
737
|
-
body: JSON.stringify({
|
|
738
|
-
user_id: userId,
|
|
739
|
-
guild_id: guildId,
|
|
740
|
-
text,
|
|
741
|
-
wake_confirmed: wakeConfirmed,
|
|
742
|
-
matched_wake_word: matchedWakeWord,
|
|
743
|
-
}),
|
|
744
|
-
});
|
|
745
|
-
} catch (err) {
|
|
746
|
-
console.error(`[STT] Failed to send recognized text to Python:`, err.message);
|
|
747
|
-
}
|
|
748
|
-
});
|
|
749
|
-
});
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
app.get('/health', (req, res) => {
|
|
753
|
-
res.json({ ready: client.isReady() });
|
|
754
|
-
});
|
|
755
|
-
|
|
756
|
-
app.post('/join', async (req, res) => {
|
|
757
|
-
const { guild_id, channel_id } = req.body;
|
|
758
|
-
try {
|
|
759
|
-
const guild = client.guilds.cache.get(guild_id);
|
|
760
|
-
if (!guild) return res.status(404).json({ error: 'Guild not found' });
|
|
761
|
-
|
|
762
|
-
let connection = joinVoiceChannel({
|
|
763
|
-
channelId: channel_id,
|
|
764
|
-
guildId: guild_id,
|
|
765
|
-
adapterCreator: guild.voiceAdapterCreator,
|
|
766
|
-
selfDeaf: false,
|
|
767
|
-
selfMute: false,
|
|
768
|
-
});
|
|
769
|
-
|
|
770
|
-
connections.set(guild_id, connection);
|
|
771
|
-
|
|
772
|
-
setupReceiver(connection, guild_id);
|
|
773
|
-
|
|
774
|
-
connection.removeAllListeners(VoiceConnectionStatus.Ready);
|
|
775
|
-
connection.on(VoiceConnectionStatus.Ready, () => {
|
|
776
|
-
console.log(`[Voice] Connected to ${channel_id} in ${guild_id}`);
|
|
777
|
-
});
|
|
778
|
-
|
|
779
|
-
res.json({ success: true });
|
|
780
|
-
} catch (e) {
|
|
781
|
-
console.error(e);
|
|
782
|
-
res.status(500).json({ error: e.message });
|
|
783
|
-
}
|
|
784
|
-
});
|
|
785
|
-
|
|
786
|
-
app.post('/leave', (req, res) => {
|
|
787
|
-
const { guild_id } = req.body;
|
|
788
|
-
const connection = connections.get(guild_id);
|
|
789
|
-
if (!connection) {
|
|
790
|
-
return res.status(404).json({ error: 'Not connected' });
|
|
791
|
-
}
|
|
792
|
-
|
|
793
|
-
const player = players.get(guild_id);
|
|
794
|
-
if (player) {
|
|
795
|
-
try {
|
|
796
|
-
player.stop(true);
|
|
797
|
-
} catch (e) {
|
|
798
|
-
// already stopped/destroyed - fine
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
connection.destroy();
|
|
802
|
-
|
|
803
|
-
// Every one of these is per-guild state that used to survive a
|
|
804
|
-
// /leave untouched. Most critically: if isPlaying was still true
|
|
805
|
-
// (e.g. TTS got cut off mid-playback by the destroy() above, or
|
|
806
|
-
// was never cleanly resolved), the NEXT /join's audio would hit
|
|
807
|
-
// `if (isPlaying.get(guildId)) return;` at the very top of the PCM
|
|
808
|
-
// data handler and get silently dropped forever - STT and wake
|
|
809
|
-
// detection both stop working, with no error, until the whole bot
|
|
810
|
-
// restarts. Same idea for a stale `players` entry: playNextInQueue
|
|
811
|
-
// reuses whatever's cached instead of creating a fresh one, so a
|
|
812
|
-
// leftover player from the destroyed connection could end up
|
|
813
|
-
// "subscribed" to nothing and never reach Idle, which is exactly
|
|
814
|
-
// what a permanently-on "speaking" indicator on the next join looks
|
|
815
|
-
// like from the outside.
|
|
816
|
-
connections.delete(guild_id);
|
|
817
|
-
players.delete(guild_id);
|
|
818
|
-
audioQueues.delete(guild_id);
|
|
819
|
-
isPlaying.delete(guild_id);
|
|
820
|
-
activeUntil.delete(guild_id);
|
|
821
|
-
suppressNotifyMap.delete(guild_id);
|
|
822
|
-
|
|
823
|
-
res.json({ success: true });
|
|
824
|
-
});
|
|
825
|
-
|
|
826
|
-
// Tracks whether the audio that just finished playing for a guild
|
|
827
|
-
// should be treated as a real conversational turn (extends the "stay
|
|
828
|
-
// awake" window) or not (e.g. wake-word enrollment sample playback -
|
|
829
|
-
// see EnrollmentManager._play_audio's suppress_active_window). Set
|
|
830
|
-
// whenever an item is shifted off the queue to play; read once the
|
|
831
|
-
// queue drains and playback is fully idle again.
|
|
832
|
-
const suppressNotifyMap = new Map();
|
|
833
|
-
|
|
834
|
-
async function notifyTtsFinished(guild_id) {
|
|
835
|
-
try {
|
|
836
|
-
await fetch('http://127.0.0.1:18080/tts_finished', {
|
|
837
|
-
method: 'POST',
|
|
838
|
-
headers: { 'Content-Type': 'application/json' },
|
|
839
|
-
body: JSON.stringify({ guild_id }),
|
|
840
|
-
});
|
|
841
|
-
} catch (err) {
|
|
842
|
-
console.error(`[TTS] Failed to notify Python of playback completion:`, err.message);
|
|
843
|
-
}
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
function playNextInQueue(guild_id) {
|
|
847
|
-
const queue = audioQueues.get(guild_id) || [];
|
|
848
|
-
if (queue.length === 0) {
|
|
849
|
-
isPlaying.set(guild_id, false);
|
|
850
|
-
if (!suppressNotifyMap.get(guild_id)) {
|
|
851
|
-
notifyTtsFinished(guild_id);
|
|
852
|
-
}
|
|
853
|
-
return;
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
const connection = connections.get(guild_id);
|
|
857
|
-
if (!connection) {
|
|
858
|
-
isPlaying.set(guild_id, false);
|
|
859
|
-
return;
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
isPlaying.set(guild_id, true);
|
|
863
|
-
const item = queue.shift(); // { buffer, suppressActiveWindow }
|
|
864
|
-
suppressNotifyMap.set(guild_id, item.suppressActiveWindow);
|
|
865
|
-
|
|
866
|
-
try {
|
|
867
|
-
let player = players.get(guild_id);
|
|
868
|
-
if (!player) {
|
|
869
|
-
player = createAudioPlayer();
|
|
870
|
-
players.set(guild_id, player);
|
|
871
|
-
connection.subscribe(player);
|
|
872
|
-
|
|
873
|
-
player.on(AudioPlayerStatus.Idle, () => {
|
|
874
|
-
playNextInQueue(guild_id);
|
|
875
|
-
});
|
|
876
|
-
|
|
877
|
-
player.on('error', (error) => {
|
|
878
|
-
console.error(`[TTS] AudioPlayer Error:`, error.message);
|
|
879
|
-
playNextInQueue(guild_id);
|
|
880
|
-
});
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
const resource = createAudioResource(Readable.from(item.buffer));
|
|
884
|
-
player.play(resource);
|
|
885
|
-
} catch (e) {
|
|
886
|
-
console.error(`[TTS] Error playing queued audio:`, e);
|
|
887
|
-
playNextInQueue(guild_id);
|
|
888
|
-
}
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
app.post('/play', express.raw({ type: 'application/octet-stream', limit: '20mb' }), (req, res) => {
|
|
892
|
-
const guild_id = req.query.guild_id;
|
|
893
|
-
const connection = connections.get(guild_id);
|
|
894
|
-
if (!connection) return res.status(404).json({ error: 'Not connected' });
|
|
895
|
-
|
|
896
|
-
if (!audioQueues.has(guild_id)) {
|
|
897
|
-
audioQueues.set(guild_id, []);
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
audioQueues.get(guild_id).push({
|
|
901
|
-
buffer: req.body, // req.body is a Buffer here
|
|
902
|
-
suppressActiveWindow: req.query.suppress_active_window === 'true',
|
|
903
|
-
});
|
|
904
|
-
|
|
905
|
-
if (!isPlaying.get(guild_id)) {
|
|
906
|
-
playNextInQueue(guild_id);
|
|
907
|
-
}
|
|
908
|
-
|
|
909
|
-
res.json({ success: true, queued: true });
|
|
910
|
-
});
|
|
911
|
-
|
|
912
|
-
app.post('/interrupt', (req, res) => {
|
|
913
|
-
const { guild_id } = req.body;
|
|
914
|
-
// Same mechanism the VAD loud-voice check uses (interruptTTS) - this
|
|
915
|
-
// just gives Python a way to trigger it directly, for the case where
|
|
916
|
-
// a new recognized utterance should cut off whatever's currently
|
|
917
|
-
// playing regardless of how loud it was (see VoiceCog.handle_stt_input,
|
|
918
|
-
// which calls this before starting a new turn whenever a previous
|
|
919
|
-
// one was still in flight).
|
|
920
|
-
interruptTTS(guild_id);
|
|
921
|
-
res.json({ success: true });
|
|
922
|
-
});
|
|
923
|
-
|
|
924
|
-
app.post('/invalidate_detector', (req, res) => {
|
|
925
|
-
// Called by EnrollmentManager._commit_enrollment right after a
|
|
926
|
-
// NEW .rpw is successfully built. Without this, detectorCache (keyed
|
|
927
|
-
// only by user_id, loaded once and cached forever) keeps serving
|
|
928
|
-
// whatever detector - built from an OLDER recording, possibly for a
|
|
929
|
-
// completely different word - was cached the first time this user
|
|
930
|
-
// was ever checked, no matter how many times they re-enroll. That's
|
|
931
|
-
// enough on its own to make every wake attempt score exactly 0
|
|
932
|
-
// forever: it's not comparing against the word that was just said.
|
|
933
|
-
const { user_id } = req.body;
|
|
934
|
-
const deleted = detectorCache.delete(user_id);
|
|
935
|
-
console.log(`[Wake] Invalidated cached detector for ${user_id} (was cached: ${deleted})`);
|
|
936
|
-
res.json({ success: true, was_cached: deleted });
|
|
937
|
-
});
|
|
938
|
-
|
|
939
|
-
app.post('/build_wakeword', async (req, res) => {
|
|
940
|
-
// Builds a .rpw wake-word reference directly from the accepted
|
|
941
|
-
// enrollment wav samples, entirely in-process via rustpotter-web's
|
|
942
|
-
// WakewordRefCreator - see EnrollmentManager._build_rustpotter_reference
|
|
943
|
-
// in cogs/voice/enrollment.py, which used to shell out to a
|
|
944
|
-
// separately-downloaded rustpotter-cli binary for this exact step.
|
|
945
|
-
// That meant an extra install-time download (GitHub Releases API,
|
|
946
|
-
// OS/arch guessing, no checksum verification) just to run a build
|
|
947
|
-
// command whose only real job was calling the same builder API this
|
|
948
|
-
// now calls directly. Nothing else about .rpw files changes: the
|
|
949
|
-
// hot-path detector (getDetectorForUser) still just loads the bytes
|
|
950
|
-
// this returns, the same as it always did.
|
|
951
|
-
try {
|
|
952
|
-
const { name, samples } = req.body;
|
|
953
|
-
if (!name || !Array.isArray(samples) || samples.length === 0) {
|
|
954
|
-
return res
|
|
955
|
-
.status(400)
|
|
956
|
-
.json({ error: 'name and at least one sample (wav bytes) are required' });
|
|
957
|
-
}
|
|
958
|
-
|
|
959
|
-
const mod = await loadRustpotterModule();
|
|
960
|
-
const creator = mod.WakewordRefCreator.new(name);
|
|
961
|
-
try {
|
|
962
|
-
for (const sample of samples) {
|
|
963
|
-
const buf = Buffer.from(sample.data_base64, 'base64');
|
|
964
|
-
creator.addFile(sample.filename || `${name}.wav`, buf);
|
|
965
|
-
}
|
|
966
|
-
const rpwBytes = creator.saveToBytes();
|
|
967
|
-
console.log(
|
|
968
|
-
`[Wake] Built .rpw for '${name}' from ${samples.length} sample(s) via WakewordRefCreator`,
|
|
969
|
-
);
|
|
970
|
-
res.json({ success: true, rpw_base64: Buffer.from(rpwBytes).toString('base64') });
|
|
971
|
-
} finally {
|
|
972
|
-
creator.free();
|
|
973
|
-
}
|
|
974
|
-
} catch (e) {
|
|
975
|
-
console.error(`[Wake] Failed to build wakeword reference:`, e);
|
|
976
|
-
res.status(500).json({ error: e.message || String(e) });
|
|
977
|
-
}
|
|
978
|
-
});
|
|
979
|
-
|
|
980
|
-
app.post('/set_config', (req, res) => {
|
|
981
|
-
const { voice_threshold } = req.body;
|
|
982
|
-
if (voice_threshold) {
|
|
983
|
-
vadThreshold = voice_threshold;
|
|
984
|
-
console.log(`[Config] Updated VAD threshold to ${vadThreshold}`);
|
|
985
|
-
}
|
|
986
|
-
res.json({ success: true });
|
|
987
|
-
});
|
|
988
|
-
|
|
989
|
-
app.post('/enroll_start', (req, res) => {
|
|
990
|
-
const { user_id } = req.body;
|
|
991
|
-
if (!user_id) return res.status(400).json({ error: 'user_id required' });
|
|
992
|
-
enrollingUsers.add(user_id);
|
|
993
|
-
res.json({ success: true });
|
|
994
|
-
});
|
|
995
|
-
|
|
996
|
-
app.post('/enroll_stop', (req, res) => {
|
|
997
|
-
const { user_id } = req.body;
|
|
998
|
-
if (!user_id) return res.status(400).json({ error: 'user_id required' });
|
|
999
|
-
enrollingUsers.delete(user_id);
|
|
1000
|
-
res.json({ success: true });
|
|
1001
|
-
});
|
|
1002
|
-
|
|
1003
|
-
app.post('/set_active', (req, res) => {
|
|
1004
|
-
const { guild_id, active_until } = req.body;
|
|
1005
|
-
if (!guild_id || !active_until)
|
|
1006
|
-
return res.status(400).json({ error: 'guild_id and active_until required' });
|
|
1007
|
-
activeUntil.set(guild_id, active_until);
|
|
1008
|
-
res.json({ success: true });
|
|
1009
|
-
});
|
|
25
|
+
registerRoutes(app, client);
|
|
1010
26
|
|
|
1011
27
|
process.on('unhandledRejection', (reason) => {
|
|
1012
28
|
console.error('Unhandled promise rejection (voice service stays alive):', reason);
|
|
1013
29
|
});
|
|
1014
30
|
|
|
1015
31
|
process.on('uncaughtException', (err) => {
|
|
1016
|
-
//
|
|
1017
|
-
// process with no trace of why - which is what made the previous
|
|
1018
|
-
// "voice service just disappeared mid-enrollment" reports
|
|
1019
|
-
// undiagnosable. Node still exits after this (an uncaughtException
|
|
1020
|
-
// means something is in an unknown state - continuing risks worse
|
|
1021
|
-
// corruption than restarting), but now the cause is on record.
|
|
32
|
+
// Logs the cause before exiting - an unhandled sync error used to kill the process with no trace.
|
|
1022
33
|
console.error('Uncaught exception - voice service is exiting:', err);
|
|
1023
34
|
process.exit(1);
|
|
1024
35
|
});
|
|
1025
36
|
|
|
1026
37
|
// Without this, SIGTERM (lgy stop/restart) kills the process mid-connection and Discord never gets a clean leave.
|
|
1027
38
|
function shutdownGracefully() {
|
|
1028
|
-
console.log(
|
|
1029
|
-
|
|
39
|
+
console.log(
|
|
40
|
+
`[Shutdown] Disconnecting from ${state.connections.size} active voice connection(s)...`,
|
|
41
|
+
);
|
|
42
|
+
for (const connection of state.connections.values()) {
|
|
1030
43
|
try {
|
|
1031
44
|
connection.destroy();
|
|
1032
45
|
} catch (e) {
|