linkgravity 1.4.1 → 1.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1032 +1,45 @@
1
- const { Client, GatewayIntentBits, Events } = require('discord.js');
2
-
3
- const originalLog = console.log;
4
- const originalError = console.error;
5
-
6
- function getTimestamp() {
7
- const now = new Date();
8
- const offset = now.getTimezoneOffset() * 60000;
9
- const localTime = new Date(now.getTime() - offset);
10
- return localTime.toISOString().replace('T', ' ').substring(0, 19);
11
- }
12
-
13
- console.log = function (...args) {
14
- originalLog(`${getTimestamp()} INFO Voice:`, ...args);
15
- };
16
- console.error = function (...args) {
17
- originalError(`${getTimestamp()} ERROR Voice:`, ...args);
18
- };
19
- const {
20
- joinVoiceChannel,
21
- createAudioPlayer,
22
- createAudioResource,
23
- AudioPlayerStatus,
24
- EndBehaviorType,
25
- VoiceConnectionStatus,
26
- } = require('@discordjs/voice');
1
+ require('./logger'); // side-effect: timestamps console.log/error - must load before anything else logs
27
2
 
3
+ const { Client, GatewayIntentBits, Events } = require('discord.js');
28
4
  const express = require('express');
29
- const axios = require('axios');
30
- const os = require('os');
31
- const prism = require('prism-media');
32
- const fs = require('fs');
33
- const path = require('path');
34
- const { Readable } = require('stream');
35
- const { spawn } = require('child_process');
36
- const ffmpegPath = require('ffmpeg-static');
37
5
 
38
- const aglJsonPath = path.join(os.homedir(), '.gemini', 'linkgravity', 'lgy.json');
39
- let aglConfig = {};
40
- try {
41
- if (fs.existsSync(aglJsonPath)) {
42
- aglConfig = JSON.parse(fs.readFileSync(aglJsonPath, 'utf8'));
43
- }
44
- } catch (e) {
45
- console.error('Failed to load lgy.json:', e.message);
46
- }
47
-
48
- process.env.DISCORD_TOKEN =
49
- aglConfig.discord_token || aglConfig.DISCORD_TOKEN || process.env.DISCORD_TOKEN;
6
+ const { aglConfig } = require('./config');
7
+ const state = require('./state');
8
+ const { registerRoutes } = require('./routes');
50
9
 
51
- process.env.http_proxy = '';
52
- process.env.https_proxy = '';
53
- process.env.HTTP_PROXY = '';
54
- process.env.HTTPS_PROXY = '';
55
- process.env.PYTHON_HOST = '';
10
+ if (aglConfig.voice_threshold) {
11
+ state.runtime.vadThreshold = parseInt(aglConfig.voice_threshold) || 3000;
12
+ }
56
13
 
57
14
  const app = express();
58
15
  app.use(express.json({ limit: '50mb' }));
59
16
 
60
- let vadThreshold = 3000;
61
- if (aglConfig.voice_threshold) {
62
- vadThreshold = parseInt(aglConfig.voice_threshold) || 3000;
63
- }
64
-
65
17
  const client = new Client({
66
18
  intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates],
67
19
  });
68
20
 
69
- const connections = new Map();
70
- const players = new Map();
71
-
72
- function stereoToMono(buffer) {
73
- // Discord voice receive is always 48kHz STEREO (see e.g. the
74
- // official discordjs/voice-examples recorder, which decodes with
75
- // channels: 2) - everything downstream here (WAV writing via
76
- // createWavHeader, Rustpotter's config) was built assuming MONO
77
- // input, so this converts the interleaved L/R stream to mono right
78
- // after decoding, in one place, rather than decoding as channels: 1
79
- // and having the native Opus decoder silently misinterpret an
80
- // actually-stereo stream as mono.
81
- const samples = buffer.length >> 2; // 2 bytes/sample * 2 channels
82
- const mono = Buffer.alloc(samples * 2);
83
- for (let i = 0; i < samples; i++) {
84
- const l = buffer.readInt16LE(i * 4);
85
- const r = buffer.readInt16LE(i * 4 + 2);
86
- mono.writeInt16LE((l + r) >> 1, i * 2);
87
- }
88
- return mono;
89
- }
90
-
91
- function createWavHeader(dataLength, sampleRate = 48000, channels = 1, bitDepth = 16) {
92
- const buffer = Buffer.alloc(44);
93
- buffer.write('RIFF', 0);
94
- buffer.writeUInt32LE(36 + dataLength, 4);
95
- buffer.write('WAVE', 8);
96
- buffer.write('fmt ', 12);
97
- buffer.writeUInt32LE(16, 16);
98
- buffer.writeUInt16LE(1, 20);
99
- buffer.writeUInt16LE(channels, 22);
100
- buffer.writeUInt32LE(sampleRate, 24);
101
- buffer.writeUInt32LE(sampleRate * channels * (bitDepth / 8), 28);
102
- buffer.writeUInt16LE(channels * (bitDepth / 8), 32);
103
- buffer.writeUInt16LE(bitDepth, 34);
104
- buffer.write('data', 36);
105
- buffer.writeUInt32LE(dataLength, 40);
106
- return buffer;
107
- }
108
-
109
21
  client.once(Events.ClientReady, () => {
110
22
  console.log(`🎤 Node.js Voice Microservice is online as ${client.user.tag}`);
111
23
  });
112
24
 
113
- // --- STT, done directly in Node ---
114
- // This is the same unofficial endpoint Python's SpeechRecognition
115
- // library (recognize_google) has used for years: audio encoded as FLAC,
116
- // POSTed to Google's v2 speech API with the "chromium" key that library
117
- // ships as its default. It's not a secret - it's the same publicly
118
- // documented key referenced all over SpeechRecognition's own source and
119
- // years of writeups (search "AIzaSyBOti4mM-6x9WDnZIjIeyEU21OpBXqWBgw
120
- // google speech api" if you want to verify that yourself). It's still an
121
- // unofficial/reverse-engineered API that Google could change or rate-
122
- // limit at any time - same risk the project already had via Python, just
123
- // no longer duplicated in two languages.
124
- //
125
- // Requires ffmpeg to do the WAV -> FLAC conversion - already a
126
- // dependency of this package (ffmpeg-static bundles the binary per
127
- // platform), so nothing extra to install.
128
- const GOOGLE_STT_KEY = 'AIzaSyBOti4mM-6x9WDnZIjIeyEU21OpBXqWBgw';
129
-
130
- function flacEncode(wavBuffer) {
131
- return new Promise((resolve, reject) => {
132
- const ff = spawn(ffmpegPath, [
133
- '-hide_banner',
134
- '-loglevel',
135
- 'error',
136
- '-i',
137
- 'pipe:0',
138
- '-f',
139
- 'flac',
140
- 'pipe:1',
141
- ]);
142
- const out = [];
143
- ff.stdout.on('data', (d) => out.push(d));
144
- ff.stderr.on('data', () => {}); // -loglevel error already keeps this quiet in the normal case
145
- ff.on('error', (err) => reject(new Error(`ffmpeg-static failed to run (${err.message})`)));
146
- ff.on('close', (code) => {
147
- if (code !== 0) return reject(new Error(`ffmpeg exited with code ${code}`));
148
- resolve(Buffer.concat(out));
149
- });
150
- ff.stdin.write(wavBuffer);
151
- ff.stdin.end();
152
- });
153
- }
154
-
155
- async function googleSTT(wavBuffer, lang = 'ko-KR') {
156
- let flacBuffer;
157
- try {
158
- flacBuffer = await flacEncode(wavBuffer);
159
- } catch (err) {
160
- console.error('[STT] FLAC encode failed:', err.message);
161
- return null;
162
- }
163
-
164
- let res;
165
- try {
166
- res = await fetch(
167
- `https://www.google.com/speech-api/v2/recognize?output=json&client=chromium&lang=${encodeURIComponent(lang)}&key=${GOOGLE_STT_KEY}`,
168
- {
169
- method: 'POST',
170
- headers: { 'Content-Type': 'audio/x-flac; rate=48000' },
171
- body: flacBuffer,
172
- },
173
- );
174
- } catch (err) {
175
- console.error('[STT] Request to Google STT failed:', err.message);
176
- return null;
177
- }
178
-
179
- const raw = await res.text();
180
- // Response is newline-delimited JSON, one object per line, e.g.:
181
- // {"result":[]}
182
- // {"result":[{"alternative":[{"transcript":"...","confidence":0.9}],"final":true}],"result_index":0}
183
- for (const line of raw.trim().split('\n')) {
184
- if (!line) continue;
185
- try {
186
- const obj = JSON.parse(line);
187
- const transcript = obj.result?.[0]?.alternative?.[0]?.transcript;
188
- if (transcript) return transcript.trim();
189
- } catch (e) {
190
- // not JSON / partial line - ignore
191
- }
192
- }
193
- return null;
194
- }
195
-
196
- const audioQueues = new Map();
197
- const isPlaying = new Map();
198
-
199
- function interruptTTS(guildId) {
200
- const player = players.get(guildId);
201
- let interrupted = false;
202
-
203
- if (audioQueues.has(guildId)) {
204
- // Queue now holds in-memory audio Buffers (see /play), not
205
- // filepaths, so there's nothing on disk to clean up here.
206
- audioQueues.set(guildId, []);
207
- }
208
-
209
- if (player && player.state.status !== AudioPlayerStatus.Idle) {
210
- player.stop();
211
- console.log(`[VAD] Interrupted TTS in guild ${guildId}`);
212
- interrupted = true;
213
- }
214
-
215
- isPlaying.set(guildId, false);
216
- return interrupted;
217
- }
218
-
219
- const activeStreams = new Map();
220
-
221
- // user_id -> true, while that user is recording wake-word samples (see
222
- // /enroll_start, /enroll_stop). Utterances from these users get routed
223
- // to /enroll_sample instead of the normal wake-check/STT pipeline.
224
- const enrollingUsers = new Set();
225
-
226
- // guild_id -> ms epoch until which we're in the "awake, no need to repeat
227
- // the wake word" window (set by Python via /set_active whenever it
228
- // starts/renews that countdown - see VoiceCog._extend_active_window).
229
- const activeUntil = new Map();
230
-
231
- function isGuildActive(guildId) {
232
- return Date.now() < (activeUntil.get(guildId) || 0);
233
- }
234
-
235
- // --- Wake-word detection (Rustpotter, in-process) ---
236
- // Runs entirely in this Node process - no Python round trip. Each
237
- // enrolled user gets their own Rustpotter instance (built from the .rpw
238
- // reference file this same process produces via /build_wakeword, called
239
- // by EnrollmentManager._commit_enrollment once all samples are
240
- // confirmed), cached here and fed audio directly as it streams in from
241
- // Discord.
242
- const WAKE_REF_DIR = path.join(os.homedir(), '.gemini', 'linkgravity', 'wake_refs');
243
-
244
- let rustpotterModPromise = null;
245
- function loadRustpotterModule() {
246
- if (!rustpotterModPromise) {
247
- rustpotterModPromise = (async () => {
248
- // Bare "rustpotter-web" doesn't resolve under Node's ESM
249
- // loader (the package only declares a "module" field, which
250
- // is a bundler-only convention Node doesn't read) - the
251
- // actual entry file has to be named explicitly.
252
- //
253
- // Using the full "rustpotter-web" package here (not the
254
- // "-slim" variant this used to be) because it's the only one
255
- // of the two that also exposes WakewordRefCreator - the
256
- // builder API used by /build_wakeword below to create a new
257
- // .rpw reference directly from wav samples, in-process, with
258
- // no external binary. The wasm binary is ~6% bigger
259
- // (810KB vs 761KB) which is irrelevant for a Node backend
260
- // (this distinction only matters for browser bundle size,
261
- // which is the slim variant's actual purpose).
262
- const mod = await import('rustpotter-web/rustpotter_wasm.js');
263
- const wasmPath = require.resolve('rustpotter-web/rustpotter_wasm_bg.wasm');
264
- mod.initSync(fs.readFileSync(wasmPath));
265
- return mod;
266
- })();
267
- }
268
- return rustpotterModPromise;
269
- }
270
-
271
- // userId -> { rustpotter, samplesPerFrame, residual: Int16Array }
272
- const detectorCache = new Map();
273
-
274
- // Wake-word confirm cutoff - must stay well above ~0.05 (rustpotter's countdown never finalizes if noise/silence clears it too); 0.4 chosen after live use kept narrowly missing genuine hits just under 0.5.
275
- const WAKE_MATCH_THRESHOLD = 0.4;
276
-
277
- async function getDetectorForUser(userId) {
278
- if (detectorCache.has(userId)) return detectorCache.get(userId);
279
-
280
- const userDir = path.join(WAKE_REF_DIR, userId);
281
- if (!fs.existsSync(userDir)) return null; // not enrolled
282
-
283
- const rpwFile = fs.readdirSync(userDir).find((f) => f.endsWith('.rpw'));
284
- if (!rpwFile) return null; // samples exist but .rpw build hasn't happened/failed - see _commit_enrollment
285
-
286
- const mod = await loadRustpotterModule();
287
- const config = mod.RustpotterConfig.new();
288
- config.setSampleRate(48000);
289
- config.setSampleFormat(mod.SampleFormat.i16);
290
- config.setChannels(1);
291
- // See WAKE_MATCH_THRESHOLD's comment above - this MUST be a real,
292
- // meaningful cutoff (not near-zero) for rustpotter's internal
293
- // confirm-after-N-more-frames logic to ever finalize a detection.
294
- config.setThreshold(WAKE_MATCH_THRESHOLD);
295
- config.setAveragedThreshold(0);
296
- // Default is 1 - accepting a match the very first time it's ever
297
- // the leading candidate, even for just one internal frame. Raised
298
- // to 4 (was 3) so a candidate has to keep winning for a few frames
299
- // running before it's trusted - this matters more now that
300
- // scoreMode is back to Max (see below), which is more permissive
301
- // per-frame than Median was, so this is the main compensating knob
302
- // against one-off spurious spikes from unrelated speech.
303
- config.setMinScores(4);
304
- // Max: score is whichever of the 5 enrolled samples matches best.
305
- // Median (tried between the two calibration notes below) requires
306
- // the middle-ranked sample to also score well, which in practice
307
- // means all 5 enrollment recordings need fairly consistent
308
- // tone/pace/delivery - real speech isn't that consistent, so Median
309
- // made real matches with naturally varied enrollment recordings
310
- // score worse, not just false positives. Max lets a genuinely
311
- // varied set of 5 recordings (calm, rushed, questioning, etc.) each
312
- // cover a different real-world delivery, at the cost of also being
313
- // easier to trip with something that merely resembles ONE of the 5
314
- // by chance - minScores(4) above and the STT jamo cross-check in
315
- // VoiceCog.handle_stt_input (tightened for short wake words
316
- // specifically) are what compensate for that on the other two axes
317
- // instead. Still needs field-tuning against real usage logs - see
318
- // README's known-issues section.
319
- config.setScoreMode(mod.ScoreMode.max);
320
-
321
- const rustpotter = mod.Rustpotter.new(config);
322
- rustpotter.addWakeword(rpwFile, fs.readFileSync(path.join(userDir, rpwFile)));
323
-
324
- // Second, diagnostic-only instance fed the exact same audio in
325
- // parallel, purely so "no match" cases still show a real number in
326
- // the logs. It NEVER gates wake behavior - only entry.rustpotter
327
- // above does that. Why a separate instance instead of reading a
328
- // low score off the real one: rustpotter's own scoring
329
- // (wakeword.run_detection in the Rust source) filters out any
330
- // frame that doesn't already clear `threshold` before it's even
331
- // tracked internally, so there is no bound API that exposes a
332
- // sub-threshold score - process*() returns undefined for those,
333
- // full stop. Naively lowering the REAL detector's threshold to see
334
- // more doesn't work either (that's exactly the bug from the
335
- // local-wake migration: near-zero threshold means background noise
336
- // clears it on almost every frame, which keeps resetting
337
- // detection_countdown before it can ever reach 0, so confirmation
338
- // never completes for anyone, real wake word included).
339
- //
340
- // This instance sidesteps that by setting eager+minScores(1), so it
341
- // finalizes and hands back a real score the very first time ANY
342
- // frame clears its own (near-zero) threshold, instead of waiting on
343
- // the countdown - the exact same escape hatch that would break
344
- // gating on the real detector is safe to lean on here because nothing
345
- // downstream ever treats this instance's output as a wake trigger.
346
- // scoreMode is kept identical to the real detector so the number
347
- // itself is a genuine, comparable similarity score - just captured
348
- // more eagerly, not computed differently.
349
- const diagConfig = mod.RustpotterConfig.new();
350
- diagConfig.setSampleRate(48000);
351
- diagConfig.setSampleFormat(mod.SampleFormat.i16);
352
- diagConfig.setChannels(1);
353
- diagConfig.setThreshold(0.01);
354
- diagConfig.setAveragedThreshold(0);
355
- diagConfig.setMinScores(1);
356
- diagConfig.setEager(true);
357
- diagConfig.setScoreMode(mod.ScoreMode.max);
358
- const diagRustpotter = mod.Rustpotter.new(diagConfig);
359
- diagRustpotter.addWakeword(rpwFile, fs.readFileSync(path.join(userDir, rpwFile)));
360
-
361
- const entry = {
362
- rustpotter,
363
- samplesPerFrame: rustpotter.getSamplesPerFrame(),
364
- residual: new Int16Array(0),
365
- diag: {
366
- rustpotter: diagRustpotter,
367
- samplesPerFrame: diagRustpotter.getSamplesPerFrame(),
368
- residual: new Int16Array(0),
369
- },
370
- };
371
- console.log(
372
- `[Wake] Loaded detector for ${userId} from ${rpwFile}: samplesPerFrame=${entry.samplesPerFrame}`,
373
- );
374
- detectorCache.set(userId, entry);
375
- return entry;
376
- }
377
-
378
- // Feeds one Discord PCM chunk to a detector, exactly once per sample and
379
- // frame-aligned via a residual carryover - NOT by periodically
380
- // re-snapshotting a growing buffer, since Rustpotter's internal window
381
- // expects a genuinely continuous stream. Returns a RustpotterDetection
382
- // if any complete frame in this chunk triggered one, else null.
383
- //
384
- // Frames are fed back-to-back with NO external overlap - rustpotter
385
- // internally extracts multiple overlapping 10ms-shifted MFCCs from each
386
- // ~30ms buffer passed in (confirmed against its Rust source), so the
387
- // caller just needs to keep the stream continuous, not overlap it.
388
- function feedPCMToDetector(entry, chunk) {
389
- const incoming = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
390
- let combined = incoming;
391
- if (entry.residual.length) {
392
- combined = new Int16Array(entry.residual.length + incoming.length);
393
- combined.set(entry.residual, 0);
394
- combined.set(incoming, entry.residual.length);
395
- }
396
-
397
- let offset = 0;
398
- let detection = null;
399
- while (combined.length - offset >= entry.samplesPerFrame) {
400
- const frame = combined.subarray(offset, offset + entry.samplesPerFrame);
401
- const result = entry.rustpotter.processI16(frame);
402
- if (result) detection = result;
403
- offset += entry.samplesPerFrame;
404
- }
405
- entry.residual = combined.subarray(offset);
406
- return detection;
407
- }
408
-
409
- function setupReceiver(connection, guildId) {
410
- const receiver = connection.receiver;
411
-
412
- receiver.speaking.removeAllListeners('start');
413
-
414
- receiver.speaking.on('start', (userId) => {
415
- if (client.user.id === userId) return;
416
-
417
- if (activeStreams.get(userId)) {
418
- return;
419
- }
420
- activeStreams.set(userId, true);
421
-
422
- let hasInterrupted = false;
423
-
424
- const opusStream = receiver.subscribe(userId, {
425
- end: {
426
- behavior: EndBehaviorType.Manual,
427
- },
428
- });
429
- const pcmStream = opusStream.pipe(
430
- new prism.opus.Decoder({ rate: 48000, channels: 2, frameSize: 960 }),
431
- );
432
-
433
- // A single corrupted/dropped Opus packet (packet loss, a bad
434
- // DAVE re-encrypt, whatever) throws inside prism-media's
435
- // decoder. Streams turn a thrown _transform error into an
436
- // 'error' event - with no listener here, Node's default for an
437
- // unhandled 'error' event is to crash the ENTIRE process, not
438
- // just this one user's utterance. Handling it here keeps voice
439
- // alive for everyone else (and for this user's next utterance).
440
- opusStream.on('error', (err) => {
441
- console.error(`[Voice] Opus stream error for ${userId}:`, err.message);
442
- forceEndStream();
443
- });
444
- pcmStream.on('error', (err) => {
445
- console.error(
446
- `[Voice] Opus decode error for ${userId} (bad/corrupted packet):`,
447
- err.message,
448
- );
449
- forceEndStream();
450
- });
451
-
452
- const chunks = [];
453
-
454
- let hasEnded = false;
455
-
456
- const forceEndStream = () => {
457
- if (hasEnded) return;
458
- try {
459
- opusStream.destroy();
460
- } catch (e) {}
461
- try {
462
- pcmStream.destroy();
463
- } catch (e) {}
464
- pcmStream.emit('end');
465
- };
466
-
467
- const maxDurationTimer = setTimeout(forceEndStream, 30000);
468
-
469
- // --- Wake-word gating (Rustpotter, in-process) ---
470
- // Runs entirely inside this Node process now - no HTTP round
471
- // trip, no Python involvement. Detector instances are cached per
472
- // Discord user (see getDetectorForUser) and fed every PCM chunk
473
- // as it streams in below, frame-aligned via feedPCMToDetector's
474
- // residual carryover - not via periodic re-snapshotting like the
475
- // old /wake_check design, since Rustpotter expects each sample
476
- // fed exactly once, in order.
477
- let wakeConfirmed = false;
478
- let matchedWakeWord = null;
479
- let detectorEntry = null;
480
- let bestWakeScore = 0;
481
- let bestWakeScoreName = null;
482
- let bestDiagScore = 0;
483
- let bestDiagScoreName = null;
484
-
485
- if (!enrollingUsers.has(userId) && !isGuildActive(guildId)) {
486
- getDetectorForUser(userId)
487
- .then((entry) => {
488
- if (entry) {
489
- entry.rustpotter.reset();
490
- entry.residual = new Int16Array(0);
491
- entry.diag.rustpotter.reset();
492
- entry.diag.residual = new Int16Array(0);
493
- detectorEntry = entry;
494
- }
495
- })
496
- .catch((err) =>
497
- console.error(`[Wake] Failed to load detector for ${userId}:`, err.message),
498
- );
499
- }
500
-
501
- // --- Live "listening..." indicator ---
502
- // Only runs during the active/awake window (bounded, default
503
- // 60s) - NOT for every VAD-detected sound like the old version,
504
- // which is what made it expensive before. STT here reuses the
505
- // same googleSTT() call that runs at utterance end anyway, just
506
- // invoked earlier/more often on the growing buffer for live
507
- // feedback.
508
- const PARTIAL_INTERVAL_MS = 1500;
509
- const PARTIAL_MIN_NEW_BYTES = 24000;
510
- let lastPartialLength = 0;
511
- let partialSent = false;
512
- const partialTimer = setInterval(async () => {
513
- if (hasEnded || !isSpeaking || enrollingUsers.has(userId)) return;
514
- if (!isGuildActive(guildId)) return;
515
-
516
- const currentLength = chunks.reduce((sum, c) => sum + c.length, 0);
517
- if (currentLength - lastPartialLength < PARTIAL_MIN_NEW_BYTES) return;
518
- lastPartialLength = currentLength;
519
-
520
- const windowPcm = Buffer.concat(chunks);
521
- const wavHeader = createWavHeader(windowPcm.length);
522
- const wavBuffer = Buffer.concat([wavHeader, windowPcm]);
523
-
524
- const text = await googleSTT(wavBuffer);
525
- partialSent = true;
526
- fetch('http://127.0.0.1:18080/stt_partial', {
527
- method: 'POST',
528
- headers: { 'Content-Type': 'application/json' },
529
- body: JSON.stringify({ guild_id: guildId, text }),
530
- }).catch((err) =>
531
- console.error(`[STT] Failed to send partial text to Python:`, err.message),
532
- );
533
- }, PARTIAL_INTERVAL_MS);
534
-
535
- let bgNoiseRMS = 500;
536
- let isSpeaking = false;
537
- let silenceBytes = 0;
538
- let silenceTimer = null;
539
-
540
- pcmStream.on('data', (rawChunk) => {
541
- if (hasEnded) return;
542
- const chunk = stereoToMono(rawChunk); // see stereoToMono's comment - decoder gives real stereo now
543
-
544
- let sumSquare = 0;
545
- for (let i = 0; i < chunk.length; i += 2) {
546
- const sample = chunk.readInt16LE(i);
547
- sumSquare += sample * sample;
548
- }
549
- const rms = Math.sqrt(sumSquare / (chunk.length / 2));
550
-
551
- const isBotPlaying = isPlaying.get(guildId) || false;
552
-
553
- if (!hasInterrupted) {
554
- const dynamicThreshold = isBotPlaying ? vadThreshold * 3 : vadThreshold;
555
- if (rms > dynamicThreshold) {
556
- if (interruptTTS(guildId)) {
557
- console.log(
558
- `[VAD] Loud voice detected (${Math.round(rms)}), interrupting TTS (Threshold: ${dynamicThreshold})`,
559
- );
560
- hasInterrupted = true;
561
- }
562
- }
563
- }
564
-
565
- if (isPlaying.get(guildId)) {
566
- return;
567
- }
568
-
569
- chunks.push(chunk);
570
-
571
- if (detectorEntry) {
572
- const detection = feedPCMToDetector(detectorEntry, chunk);
573
- if (detection && detection.getScore() > bestWakeScore) {
574
- bestWakeScore = detection.getScore();
575
- bestWakeScoreName = detection.getName();
576
- }
577
- const diagDetection = feedPCMToDetector(detectorEntry.diag, chunk);
578
- if (diagDetection && diagDetection.getScore() > bestDiagScore) {
579
- bestDiagScore = diagDetection.getScore();
580
- bestDiagScoreName = diagDetection.getName();
581
- }
582
- }
583
-
584
- if (!isSpeaking) {
585
- bgNoiseRMS = bgNoiseRMS * 0.98 + rms * 0.02;
586
- bgNoiseRMS = Math.max(50, Math.min(bgNoiseRMS, 3000));
587
- }
588
-
589
- const threshold = Math.max(bgNoiseRMS * 2.0, 800);
590
-
591
- if (rms > threshold) {
592
- isSpeaking = true;
593
- silenceBytes = 0;
594
- } else {
595
- if (isSpeaking) {
596
- silenceBytes += chunk.length;
597
- if (silenceBytes >= 76800) {
598
- forceEndStream();
599
- return;
600
- }
601
- }
602
- }
603
-
604
- if (isSpeaking) {
605
- if (silenceTimer) clearTimeout(silenceTimer);
606
- silenceTimer = setTimeout(() => forceEndStream(), 800);
607
- }
608
- });
609
-
610
- pcmStream.on('end', async () => {
611
- if (hasEnded) return;
612
- hasEnded = true;
613
- clearTimeout(maxDurationTimer);
614
- clearInterval(partialTimer);
615
- if (silenceTimer) clearTimeout(silenceTimer);
616
-
617
- activeStreams.delete(userId);
618
-
619
- if (detectorEntry) {
620
- // rustpotter needs MORE frames fed after a candidate
621
- // match before it finalizes one (detection_countdown
622
- // counts down from max_mfcc_frames/2 before confirming
623
- // or discarding a partial match - see the Rust source's
624
- // detector.rs). Live capture stops the instant the user
625
- // stops talking, so without this, a genuine match
626
- // candidate never gets the chance to finish counting
627
- // down and is silently discarded - this is exactly why
628
- // rustpotter-cli's own `test` command pads its input
629
- // with 100 extra silent frames before processing (see
630
- // its test.rs), and why testing the identical captured
631
- // audio through the CLI scored well while this always
632
- // failed live. Same fix here: flush the residual plus
633
- // a few seconds of silence through the same detector
634
- // before deciding pass/fail.
635
- const paddingBuffer = Buffer.alloc(detectorEntry.samplesPerFrame * 100 * 2);
636
- const paddingDetection = feedPCMToDetector(detectorEntry, paddingBuffer);
637
- if (paddingDetection && paddingDetection.getScore() > bestWakeScore) {
638
- bestWakeScore = paddingDetection.getScore();
639
- bestWakeScoreName = paddingDetection.getName();
640
- }
641
- const diagPaddingBuffer = Buffer.alloc(
642
- detectorEntry.diag.samplesPerFrame * 100 * 2,
643
- );
644
- const diagPaddingDetection = feedPCMToDetector(
645
- detectorEntry.diag,
646
- diagPaddingBuffer,
647
- );
648
- if (diagPaddingDetection && diagPaddingDetection.getScore() > bestDiagScore) {
649
- bestDiagScore = diagPaddingDetection.getScore();
650
- bestDiagScoreName = diagPaddingDetection.getName();
651
- }
652
-
653
- // Real pass/fail uses bestWakeScore; bestDiagScore is a separate, much looser detector shown only for "how close" - not on the same scale, not comparable to WAKE_MATCH_THRESHOLD.
654
- wakeConfirmed = bestWakeScore >= WAKE_MATCH_THRESHOLD;
655
- matchedWakeWord = wakeConfirmed ? bestWakeScoreName : null;
656
- console.log(
657
- wakeConfirmed
658
- ? `[Wake] ${userId}: CONFIRMED (score ${bestWakeScore.toFixed(3)} for ` +
659
- `"${bestWakeScoreName}", threshold ${WAKE_MATCH_THRESHOLD})`
660
- : `[Wake] ${userId}: no match (score ${bestWakeScore.toFixed(3)}, ` +
661
- `threshold ${WAKE_MATCH_THRESHOLD}; diagnostic-only closeness ` +
662
- `${bestDiagScore.toFixed(3)} for "${bestDiagScoreName ?? 'n/a'}" - ` +
663
- `different scoring config, not directly comparable to the threshold)`,
664
- );
665
- }
666
-
667
- const pcmBuffer = Buffer.concat(chunks);
668
-
669
- // Enrollment mode: this utterance is a wake-word reference
670
- // sample, not a command - hand it straight to Python and skip
671
- // wake-check/STT/active-window logic entirely.
672
- if (enrollingUsers.has(userId)) {
673
- if (pcmBuffer.length < 4000) return; // too short to be a real sample
674
- const wavHeader = createWavHeader(pcmBuffer.length);
675
- const wavBuffer = Buffer.concat([wavHeader, pcmBuffer]);
676
- try {
677
- await fetch(
678
- `http://127.0.0.1:18080/enroll_sample?user_id=${encodeURIComponent(userId)}`,
679
- {
680
- method: 'POST',
681
- headers: { 'Content-Type': 'application/octet-stream' },
682
- body: wavBuffer,
683
- },
684
- );
685
- } catch (err) {
686
- console.error(`[Enroll] Failed to send sample to Python:`, err.message);
687
- }
688
- return;
689
- }
690
-
691
- // Was 24000 (250ms) - cut off short Korean replies ("네"/"어"/"응"); noise is filtered upstream by isSpeaking's RMS/sustain check, not by duration.
692
- if (pcmBuffer.length < 9600) {
693
- if (partialSent) {
694
- fetch('http://127.0.0.1:18080/stt_partial_cancel', {
695
- method: 'POST',
696
- headers: { 'Content-Type': 'application/json' },
697
- body: JSON.stringify({ guild_id: guildId }),
698
- }).catch(() => {});
699
- }
700
- return;
701
- }
702
-
703
- // Whether this utterance is worth transcribing at all:
704
- // - isGuildActive(): already awake, no need to repeat the
705
- // wake word (this is also what makes the live "listening"
706
- // indicator above meaningful - same window).
707
- // - wakeConfirmed: our in-process Rustpotter detector matched this
708
- // user's voice against their enrolled samples.
709
- // There is deliberately no "unenrolled users always get
710
- // transcribed" fallback anymore - that existed only to feed
711
- // the Python side's old text-similarity wake-word matching,
712
- // which has been removed (it was exactly the always-on
713
- // recognition overhead this Rustpotter migration was meant to
714
- // get rid of). An unenrolled user simply can't wake the bot
715
- // by voice until they run /sound.
716
- const shouldTranscribe = isGuildActive(guildId) || wakeConfirmed;
717
-
718
- if (!shouldTranscribe) {
719
- if (partialSent) {
720
- fetch('http://127.0.0.1:18080/stt_partial_cancel', {
721
- method: 'POST',
722
- headers: { 'Content-Type': 'application/json' },
723
- body: JSON.stringify({ guild_id: guildId }),
724
- }).catch(() => {});
725
- }
726
- return;
727
- }
728
-
729
- const wavHeader = createWavHeader(pcmBuffer.length);
730
- const wavBuffer = Buffer.concat([wavHeader, pcmBuffer]);
731
- const text = await googleSTT(wavBuffer);
732
-
733
- try {
734
- await fetch('http://127.0.0.1:18080/stt_input', {
735
- method: 'POST',
736
- headers: { 'Content-Type': 'application/json' },
737
- body: JSON.stringify({
738
- user_id: userId,
739
- guild_id: guildId,
740
- text,
741
- wake_confirmed: wakeConfirmed,
742
- matched_wake_word: matchedWakeWord,
743
- }),
744
- });
745
- } catch (err) {
746
- console.error(`[STT] Failed to send recognized text to Python:`, err.message);
747
- }
748
- });
749
- });
750
- }
751
-
752
- app.get('/health', (req, res) => {
753
- res.json({ ready: client.isReady() });
754
- });
755
-
756
- app.post('/join', async (req, res) => {
757
- const { guild_id, channel_id } = req.body;
758
- try {
759
- const guild = client.guilds.cache.get(guild_id);
760
- if (!guild) return res.status(404).json({ error: 'Guild not found' });
761
-
762
- let connection = joinVoiceChannel({
763
- channelId: channel_id,
764
- guildId: guild_id,
765
- adapterCreator: guild.voiceAdapterCreator,
766
- selfDeaf: false,
767
- selfMute: false,
768
- });
769
-
770
- connections.set(guild_id, connection);
771
-
772
- setupReceiver(connection, guild_id);
773
-
774
- connection.removeAllListeners(VoiceConnectionStatus.Ready);
775
- connection.on(VoiceConnectionStatus.Ready, () => {
776
- console.log(`[Voice] Connected to ${channel_id} in ${guild_id}`);
777
- });
778
-
779
- res.json({ success: true });
780
- } catch (e) {
781
- console.error(e);
782
- res.status(500).json({ error: e.message });
783
- }
784
- });
785
-
786
- app.post('/leave', (req, res) => {
787
- const { guild_id } = req.body;
788
- const connection = connections.get(guild_id);
789
- if (!connection) {
790
- return res.status(404).json({ error: 'Not connected' });
791
- }
792
-
793
- const player = players.get(guild_id);
794
- if (player) {
795
- try {
796
- player.stop(true);
797
- } catch (e) {
798
- // already stopped/destroyed - fine
799
- }
800
- }
801
- connection.destroy();
802
-
803
- // Every one of these is per-guild state that used to survive a
804
- // /leave untouched. Most critically: if isPlaying was still true
805
- // (e.g. TTS got cut off mid-playback by the destroy() above, or
806
- // was never cleanly resolved), the NEXT /join's audio would hit
807
- // `if (isPlaying.get(guildId)) return;` at the very top of the PCM
808
- // data handler and get silently dropped forever - STT and wake
809
- // detection both stop working, with no error, until the whole bot
810
- // restarts. Same idea for a stale `players` entry: playNextInQueue
811
- // reuses whatever's cached instead of creating a fresh one, so a
812
- // leftover player from the destroyed connection could end up
813
- // "subscribed" to nothing and never reach Idle, which is exactly
814
- // what a permanently-on "speaking" indicator on the next join looks
815
- // like from the outside.
816
- connections.delete(guild_id);
817
- players.delete(guild_id);
818
- audioQueues.delete(guild_id);
819
- isPlaying.delete(guild_id);
820
- activeUntil.delete(guild_id);
821
- suppressNotifyMap.delete(guild_id);
822
-
823
- res.json({ success: true });
824
- });
825
-
826
- // Tracks whether the audio that just finished playing for a guild
827
- // should be treated as a real conversational turn (extends the "stay
828
- // awake" window) or not (e.g. wake-word enrollment sample playback -
829
- // see EnrollmentManager._play_audio's suppress_active_window). Set
830
- // whenever an item is shifted off the queue to play; read once the
831
- // queue drains and playback is fully idle again.
832
- const suppressNotifyMap = new Map();
833
-
834
- async function notifyTtsFinished(guild_id) {
835
- try {
836
- await fetch('http://127.0.0.1:18080/tts_finished', {
837
- method: 'POST',
838
- headers: { 'Content-Type': 'application/json' },
839
- body: JSON.stringify({ guild_id }),
840
- });
841
- } catch (err) {
842
- console.error(`[TTS] Failed to notify Python of playback completion:`, err.message);
843
- }
844
- }
845
-
846
- function playNextInQueue(guild_id) {
847
- const queue = audioQueues.get(guild_id) || [];
848
- if (queue.length === 0) {
849
- isPlaying.set(guild_id, false);
850
- if (!suppressNotifyMap.get(guild_id)) {
851
- notifyTtsFinished(guild_id);
852
- }
853
- return;
854
- }
855
-
856
- const connection = connections.get(guild_id);
857
- if (!connection) {
858
- isPlaying.set(guild_id, false);
859
- return;
860
- }
861
-
862
- isPlaying.set(guild_id, true);
863
- const item = queue.shift(); // { buffer, suppressActiveWindow }
864
- suppressNotifyMap.set(guild_id, item.suppressActiveWindow);
865
-
866
- try {
867
- let player = players.get(guild_id);
868
- if (!player) {
869
- player = createAudioPlayer();
870
- players.set(guild_id, player);
871
- connection.subscribe(player);
872
-
873
- player.on(AudioPlayerStatus.Idle, () => {
874
- playNextInQueue(guild_id);
875
- });
876
-
877
- player.on('error', (error) => {
878
- console.error(`[TTS] AudioPlayer Error:`, error.message);
879
- playNextInQueue(guild_id);
880
- });
881
- }
882
-
883
- const resource = createAudioResource(Readable.from(item.buffer));
884
- player.play(resource);
885
- } catch (e) {
886
- console.error(`[TTS] Error playing queued audio:`, e);
887
- playNextInQueue(guild_id);
888
- }
889
- }
890
-
891
- app.post('/play', express.raw({ type: 'application/octet-stream', limit: '20mb' }), (req, res) => {
892
- const guild_id = req.query.guild_id;
893
- const connection = connections.get(guild_id);
894
- if (!connection) return res.status(404).json({ error: 'Not connected' });
895
-
896
- if (!audioQueues.has(guild_id)) {
897
- audioQueues.set(guild_id, []);
898
- }
899
-
900
- audioQueues.get(guild_id).push({
901
- buffer: req.body, // req.body is a Buffer here
902
- suppressActiveWindow: req.query.suppress_active_window === 'true',
903
- });
904
-
905
- if (!isPlaying.get(guild_id)) {
906
- playNextInQueue(guild_id);
907
- }
908
-
909
- res.json({ success: true, queued: true });
910
- });
911
-
912
- app.post('/interrupt', (req, res) => {
913
- const { guild_id } = req.body;
914
- // Same mechanism the VAD loud-voice check uses (interruptTTS) - this
915
- // just gives Python a way to trigger it directly, for the case where
916
- // a new recognized utterance should cut off whatever's currently
917
- // playing regardless of how loud it was (see VoiceCog.handle_stt_input,
918
- // which calls this before starting a new turn whenever a previous
919
- // one was still in flight).
920
- interruptTTS(guild_id);
921
- res.json({ success: true });
922
- });
923
-
924
- app.post('/invalidate_detector', (req, res) => {
925
- // Called by EnrollmentManager._commit_enrollment right after a
926
- // NEW .rpw is successfully built. Without this, detectorCache (keyed
927
- // only by user_id, loaded once and cached forever) keeps serving
928
- // whatever detector - built from an OLDER recording, possibly for a
929
- // completely different word - was cached the first time this user
930
- // was ever checked, no matter how many times they re-enroll. That's
931
- // enough on its own to make every wake attempt score exactly 0
932
- // forever: it's not comparing against the word that was just said.
933
- const { user_id } = req.body;
934
- const deleted = detectorCache.delete(user_id);
935
- console.log(`[Wake] Invalidated cached detector for ${user_id} (was cached: ${deleted})`);
936
- res.json({ success: true, was_cached: deleted });
937
- });
938
-
939
- app.post('/build_wakeword', async (req, res) => {
940
- // Builds a .rpw wake-word reference directly from the accepted
941
- // enrollment wav samples, entirely in-process via rustpotter-web's
942
- // WakewordRefCreator - see EnrollmentManager._build_rustpotter_reference
943
- // in cogs/voice/enrollment.py, which used to shell out to a
944
- // separately-downloaded rustpotter-cli binary for this exact step.
945
- // That meant an extra install-time download (GitHub Releases API,
946
- // OS/arch guessing, no checksum verification) just to run a build
947
- // command whose only real job was calling the same builder API this
948
- // now calls directly. Nothing else about .rpw files changes: the
949
- // hot-path detector (getDetectorForUser) still just loads the bytes
950
- // this returns, the same as it always did.
951
- try {
952
- const { name, samples } = req.body;
953
- if (!name || !Array.isArray(samples) || samples.length === 0) {
954
- return res
955
- .status(400)
956
- .json({ error: 'name and at least one sample (wav bytes) are required' });
957
- }
958
-
959
- const mod = await loadRustpotterModule();
960
- const creator = mod.WakewordRefCreator.new(name);
961
- try {
962
- for (const sample of samples) {
963
- const buf = Buffer.from(sample.data_base64, 'base64');
964
- creator.addFile(sample.filename || `${name}.wav`, buf);
965
- }
966
- const rpwBytes = creator.saveToBytes();
967
- console.log(
968
- `[Wake] Built .rpw for '${name}' from ${samples.length} sample(s) via WakewordRefCreator`,
969
- );
970
- res.json({ success: true, rpw_base64: Buffer.from(rpwBytes).toString('base64') });
971
- } finally {
972
- creator.free();
973
- }
974
- } catch (e) {
975
- console.error(`[Wake] Failed to build wakeword reference:`, e);
976
- res.status(500).json({ error: e.message || String(e) });
977
- }
978
- });
979
-
980
- app.post('/set_config', (req, res) => {
981
- const { voice_threshold } = req.body;
982
- if (voice_threshold) {
983
- vadThreshold = voice_threshold;
984
- console.log(`[Config] Updated VAD threshold to ${vadThreshold}`);
985
- }
986
- res.json({ success: true });
987
- });
988
-
989
- app.post('/enroll_start', (req, res) => {
990
- const { user_id } = req.body;
991
- if (!user_id) return res.status(400).json({ error: 'user_id required' });
992
- enrollingUsers.add(user_id);
993
- res.json({ success: true });
994
- });
995
-
996
- app.post('/enroll_stop', (req, res) => {
997
- const { user_id } = req.body;
998
- if (!user_id) return res.status(400).json({ error: 'user_id required' });
999
- enrollingUsers.delete(user_id);
1000
- res.json({ success: true });
1001
- });
1002
-
1003
- app.post('/set_active', (req, res) => {
1004
- const { guild_id, active_until } = req.body;
1005
- if (!guild_id || !active_until)
1006
- return res.status(400).json({ error: 'guild_id and active_until required' });
1007
- activeUntil.set(guild_id, active_until);
1008
- res.json({ success: true });
1009
- });
25
+ registerRoutes(app, client);
1010
26
 
1011
27
  process.on('unhandledRejection', (reason) => {
1012
28
  console.error('Unhandled promise rejection (voice service stays alive):', reason);
1013
29
  });
1014
30
 
1015
31
  process.on('uncaughtException', (err) => {
1016
- // Without this handler, an uncaught synchronous error kills the
1017
- // process with no trace of why - which is what made the previous
1018
- // "voice service just disappeared mid-enrollment" reports
1019
- // undiagnosable. Node still exits after this (an uncaughtException
1020
- // means something is in an unknown state - continuing risks worse
1021
- // corruption than restarting), but now the cause is on record.
32
+ // Logs the cause before exiting - an unhandled sync error used to kill the process with no trace.
1022
33
  console.error('Uncaught exception - voice service is exiting:', err);
1023
34
  process.exit(1);
1024
35
  });
1025
36
 
1026
37
  // Without this, SIGTERM (lgy stop/restart) kills the process mid-connection and Discord never gets a clean leave.
1027
38
  function shutdownGracefully() {
1028
- console.log(`[Shutdown] Disconnecting from ${connections.size} active voice connection(s)...`);
1029
- for (const connection of connections.values()) {
39
+ console.log(
40
+ `[Shutdown] Disconnecting from ${state.connections.size} active voice connection(s)...`,
41
+ );
42
+ for (const connection of state.connections.values()) {
1030
43
  try {
1031
44
  connection.destroy();
1032
45
  } catch (e) {