@gotcos/glasses-server 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cos-profile.example.json +7 -0
- package/.env.example +44 -0
- package/CHANGELOG.md +25 -0
- package/LICENSE +21 -0
- package/README.md +78 -0
- package/bin/cli.cjs +203 -0
- package/package.json +53 -0
- package/server/env.ts +26 -0
- package/server/index.ts +211 -0
- package/server/lib/archive-budget.ts +65 -0
- package/server/lib/archive.ts +414 -0
- package/server/lib/atomic-fs.ts +50 -0
- package/server/lib/audio-enhance.ts +87 -0
- package/server/lib/claude-bridge.ts +682 -0
- package/server/lib/claude-circuit.ts +52 -0
- package/server/lib/claude-run-ledger.ts +279 -0
- package/server/lib/codex-bridge.ts +476 -0
- package/server/lib/codex-engine-sessions.ts +140 -0
- package/server/lib/codex-run-ledger.ts +298 -0
- package/server/lib/context-builder.ts +210 -0
- package/server/lib/conversation.ts +587 -0
- package/server/lib/data-dir.ts +20 -0
- package/server/lib/display-bus.ts +21 -0
- package/server/lib/display-format.ts +23 -0
- package/server/lib/fuzzy-correct.ts +286 -0
- package/server/lib/hallucination-filter.ts +469 -0
- package/server/lib/local-day.ts +13 -0
- package/server/lib/model-router.ts +38 -0
- package/server/lib/openai-key.ts +155 -0
- package/server/lib/openai-whisper-budget.ts +170 -0
- package/server/lib/profile.ts +94 -0
- package/server/lib/python-bridge.ts +84 -0
- package/server/lib/response-cache.ts +138 -0
- package/server/lib/session-cache-writer.ts +266 -0
- package/server/lib/session-log.ts +162 -0
- package/server/lib/speaker-embeddings.ts +578 -0
- package/server/lib/telegram-notify.ts +85 -0
- package/server/lib/token-audit.ts +50 -0
- package/server/lib/transcribe-audio.ts +187 -0
- package/server/lib/utils.ts +5 -0
- package/server/lib/vad-silero.ts +179 -0
- package/server/lib/whisper-local.ts +697 -0
- package/server/routes/diag.ts +115 -0
- package/server/routes/display.ts +65 -0
- package/server/routes/health.ts +128 -0
- package/server/routes/openai-compat.ts +446 -0
- package/server/routes/openai-key.ts +121 -0
- package/server/routes/query.ts +121 -0
- package/server/routes/transcribe-stream.ts +1090 -0
- package/server/routes/transcribe.ts +55 -0
- package/shared/model-preference.ts +81 -0
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
// Speaker embedding extraction and verification using sherpa-onnx
|
|
2
|
+
// Wraps ECAPA-TDNN model for voiceprint-based speaker classification.
|
|
3
|
+
// Falls back gracefully if model is missing — amplitude classification continues.
|
|
4
|
+
//
|
|
5
|
+
// Phase 1: Auto-enrollment — high-confidence matches add G2-mic embeddings
|
|
6
|
+
// Phase 4: Calendar priming — scoped search to expected meeting attendees
|
|
7
|
+
|
|
8
|
+
import { resolve } from 'node:path'
|
|
9
|
+
import { errMsg } from './utils.js'
|
|
10
|
+
import { getOwnerSpeakerLabel } from './profile.js'
|
|
11
|
+
import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync } from 'node:fs'
|
|
12
|
+
import { fileURLToPath } from 'node:url'
|
|
13
|
+
|
|
14
|
+
// sherpa-onnx-node is CJS — use createRequire for ESM compat
|
|
15
|
+
import { createRequire } from 'node:module'
|
|
16
|
+
const require = createRequire(import.meta.url)
|
|
17
|
+
|
|
18
|
+
const __dirname = fileURLToPath(new URL('.', import.meta.url))
|
|
19
|
+
|
|
20
|
+
const MODEL_PATH = resolve(__dirname, '..', 'models',
|
|
21
|
+
'3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx')
|
|
22
|
+
import { DATA_DIR } from './data-dir.js'
|
|
23
|
+
const PROFILES_PATH = resolve(DATA_DIR, 'voice-profiles.json')
|
|
24
|
+
const CALIBRATION_LOG = resolve(DATA_DIR, 'speaker-calibration.jsonl')
|
|
25
|
+
|
|
26
|
+
// Thresholds
|
|
27
|
+
const VERIFY_THRESHOLD = 0.65
|
|
28
|
+
const SEARCH_THRESHOLD = 0.55
|
|
29
|
+
const AUTO_ENROLL_THRESHOLD = 0.88 // High bar — must be very confident before auto-enrolling
|
|
30
|
+
const AUTO_ENROLL_CONSENSUS = 2 // Must match N times in same session before enrolling
|
|
31
|
+
const MAX_EMBEDDINGS_PER_SPEAKER = 20 // FIFO cap — oldest drops when full
|
|
32
|
+
const SAMPLE_RATE = 16000
|
|
33
|
+
|
|
34
|
+
// Module-level state — sherpa-onnx-node is CJS with no TS types (SDK v0.0.7 interop)
|
|
35
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
36
|
+
let extractor: any = null
|
|
37
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
38
|
+
let manager: any = null
|
|
39
|
+
let initialized = false
|
|
40
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
41
|
+
let sherpaOnnx: any = null
|
|
42
|
+
|
|
43
|
+
// In-memory profile store cache — avoids re-reading 7MB+ JSON on every audio chunk
|
|
44
|
+
let _cachedProfileStore: ProfileStore | null = null
|
|
45
|
+
|
|
46
|
+
// Auto-enrollment session tracking: sessionId → { speakerName → matchCount }
|
|
47
|
+
const autoEnrollSessions = new Map<string, Map<string, number>>()
|
|
48
|
+
// Track which speakers already auto-enrolled this session
|
|
49
|
+
const autoEnrolledThisSession = new Map<string, Set<string>>()
|
|
50
|
+
|
|
51
|
+
interface VoiceProfile {
|
|
52
|
+
name: string
|
|
53
|
+
embeddings: number[][] // multiple enrollments for robustness
|
|
54
|
+
sources?: string[] // provenance: 'manual' | 'fireflies' | 'auto:sessionId'
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
interface ProfileStore {
|
|
58
|
+
profiles: VoiceProfile[]
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Initialize speaker embedding system. Returns false if model missing (graceful degradation). */
|
|
62
|
+
export function initSpeakerEmbeddings(): boolean {
|
|
63
|
+
if (initialized) return extractor !== null
|
|
64
|
+
|
|
65
|
+
initialized = true
|
|
66
|
+
|
|
67
|
+
if (!existsSync(MODEL_PATH)) {
|
|
68
|
+
console.log('[speaker] Model not found at', MODEL_PATH, '— embedding disabled, using amplitude fallback')
|
|
69
|
+
return false
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
sherpaOnnx = require('sherpa-onnx-node')
|
|
74
|
+
|
|
75
|
+
extractor = new sherpaOnnx.SpeakerEmbeddingExtractor({
|
|
76
|
+
model: MODEL_PATH,
|
|
77
|
+
numThreads: 2,
|
|
78
|
+
provider: 'cpu',
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
manager = new sherpaOnnx.SpeakerEmbeddingManager(extractor.dim)
|
|
82
|
+
console.log(`[speaker] Initialized: ${extractor.dim}-dim embeddings`)
|
|
83
|
+
|
|
84
|
+
// Ensure data directory exists
|
|
85
|
+
if (!existsSync(DATA_DIR)) {
|
|
86
|
+
mkdirSync(DATA_DIR, { recursive: true })
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Load persisted profiles
|
|
90
|
+
loadProfiles()
|
|
91
|
+
|
|
92
|
+
return true
|
|
93
|
+
} catch (err: unknown) {
|
|
94
|
+
console.error('[speaker] Init failed:', errMsg(err))
|
|
95
|
+
extractor = null
|
|
96
|
+
manager = null
|
|
97
|
+
return false
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Check if a speaker is enrolled */
|
|
102
|
+
export function isEnrolled(name: string): boolean {
|
|
103
|
+
if (!manager) return false
|
|
104
|
+
return manager.contains(name)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Get all enrolled speaker names */
|
|
108
|
+
export function getAllSpeakerNames(): string[] {
|
|
109
|
+
if (!manager) return []
|
|
110
|
+
return manager.getAllSpeakerNames()
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Enroll a speaker from WAV audio buffer */
|
|
114
|
+
export function enrollSpeaker(name: string, wavBuffer: Buffer): { success: boolean; dim: number; error?: string } {
|
|
115
|
+
if (!extractor || !manager) {
|
|
116
|
+
return { success: false, dim: 0, error: 'Speaker embeddings not initialized' }
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
try {
|
|
120
|
+
const embedding = extractEmbeddingFromWav(wavBuffer)
|
|
121
|
+
if (!embedding) {
|
|
122
|
+
return { success: false, dim: 0, error: 'Could not extract embedding — audio too short or silent' }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return enrollEmbedding(name, embedding, 'manual')
|
|
126
|
+
} catch (err: unknown) {
|
|
127
|
+
console.error('[speaker] Enrollment error:', errMsg(err))
|
|
128
|
+
return { success: false, dim: 0, error: errMsg(err) }
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Enroll a raw embedding directly (used by trainer and auto-enroll).
|
|
133
|
+
* skipDedupCheck: when true, bypass the 0.95 similarity dedup gate.
|
|
134
|
+
* The trainer's greedy diversity selector already ensures embeddings are diverse,
|
|
135
|
+
* so the dedup gate is redundant and too restrictive for batch training
|
|
136
|
+
* (Fireflies audio conditions are uniform enough that even "most diverse"
|
|
137
|
+
* embeddings can exceed 0.95 similarity). */
|
|
138
|
+
export function enrollEmbedding(name: string, embedding: Float32Array, source: string = 'manual', skipDedupCheck: boolean = false): { success: boolean; dim: number; error?: string } {
|
|
139
|
+
if (!extractor || !manager) {
|
|
140
|
+
return { success: false, dim: 0, error: 'Speaker embeddings not initialized' }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Check diversity: skip if too similar to an existing embedding for this speaker
|
|
144
|
+
const store = loadProfileStore()
|
|
145
|
+
const profile = store.profiles.find(p => p.name === name)
|
|
146
|
+
if (profile) {
|
|
147
|
+
// Dedup check: skip if too similar to existing (unless bypassed by trainer)
|
|
148
|
+
if (!skipDedupCheck) {
|
|
149
|
+
for (const existing of profile.embeddings) {
|
|
150
|
+
const sim = rawCosineSimilarity(embedding, new Float32Array(existing))
|
|
151
|
+
if (sim > 0.95) {
|
|
152
|
+
return { success: false, dim: extractor.dim, error: 'Too similar to existing embedding (>0.95)' }
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// FIFO cap: if at max, drop oldest before adding (always enforced)
|
|
158
|
+
if (profile.embeddings.length >= MAX_EMBEDDINGS_PER_SPEAKER) {
|
|
159
|
+
console.log(`[speaker] Profile cap reached for "${name}" (${profile.embeddings.length}/${MAX_EMBEDDINGS_PER_SPEAKER}) — dropping oldest`)
|
|
160
|
+
profile.embeddings.shift()
|
|
161
|
+
profile.sources?.shift()
|
|
162
|
+
rebuildSpeakerInManager(name, profile.embeddings)
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Add to manager — if manager rejects (internal dedup), force via remove+readd
|
|
167
|
+
let added = manager.add({ name, v: embedding })
|
|
168
|
+
if (!added) {
|
|
169
|
+
// Manager's internal dedup rejected it — force-add by rebuilding
|
|
170
|
+
// Persist first so we have the complete embedding list
|
|
171
|
+
persistProfile(name, embedding, source)
|
|
172
|
+
const updatedStore = loadProfileStore()
|
|
173
|
+
const updatedProfile = updatedStore.profiles.find(p => p.name === name)
|
|
174
|
+
if (updatedProfile) {
|
|
175
|
+
rebuildSpeakerInManager(name, updatedProfile.embeddings)
|
|
176
|
+
console.log(`[speaker] Force-enrolled "${name}" via rebuild (${updatedProfile.embeddings.length} total, source: ${source})`)
|
|
177
|
+
return { success: true, dim: extractor.dim }
|
|
178
|
+
}
|
|
179
|
+
return { success: false, dim: extractor.dim, error: 'Manager rejected and rebuild failed' }
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Persist to disk with provenance
|
|
183
|
+
persistProfile(name, embedding, source)
|
|
184
|
+
|
|
185
|
+
console.log(`[speaker] Enrolled "${name}" (${extractor.dim}-dim, source: ${source})`)
|
|
186
|
+
return { success: true, dim: extractor.dim }
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Identify speaker from WAV audio buffer.
|
|
190
|
+
* Phase 4: optionally scope search to expected attendees for fewer false positives. */
|
|
191
|
+
export function identifySpeaker(
|
|
192
|
+
wavBuffer: Buffer,
|
|
193
|
+
expectedSpeakers?: string[],
|
|
194
|
+
): { speaker: string; similarity: number } | null {
|
|
195
|
+
if (!extractor || !manager) return null
|
|
196
|
+
|
|
197
|
+
try {
|
|
198
|
+
const embedding = extractEmbeddingFromWav(wavBuffer)
|
|
199
|
+
if (!embedding) return null
|
|
200
|
+
|
|
201
|
+
// If the wearer is enrolled, verify against them first (they're wearing the glasses)
|
|
202
|
+
const owner = getOwnerSpeakerLabel()
|
|
203
|
+
if (manager.contains(owner)) {
|
|
204
|
+
const isOwner = manager.verify({ name: owner, v: embedding, threshold: VERIFY_THRESHOLD })
|
|
205
|
+
if (isOwner) {
|
|
206
|
+
const similarity = computeCosineSimilarity(embedding, owner)
|
|
207
|
+
logCalibration(owner, similarity, true)
|
|
208
|
+
return { speaker: owner, similarity }
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Phase 4: scoped search — try expected attendees first
|
|
213
|
+
if (expectedSpeakers && expectedSpeakers.length > 0) {
|
|
214
|
+
for (const name of expectedSpeakers) {
|
|
215
|
+
if (name === owner) continue // wearer already checked
|
|
216
|
+
if (!manager.contains(name)) continue
|
|
217
|
+
const matches = manager.verify({ name, v: embedding, threshold: SEARCH_THRESHOLD })
|
|
218
|
+
if (matches) {
|
|
219
|
+
const similarity = computeCosineSimilarity(embedding, name)
|
|
220
|
+
logCalibration(name, similarity, true)
|
|
221
|
+
return { speaker: name, similarity }
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Full search across all speakers (fallback or no expected speakers)
|
|
227
|
+
const found = manager.search({ v: embedding, threshold: SEARCH_THRESHOLD })
|
|
228
|
+
if (found && found.length > 0) {
|
|
229
|
+
const similarity = computeCosineSimilarity(embedding, found)
|
|
230
|
+
logCalibration(found, similarity, true)
|
|
231
|
+
return { speaker: found, similarity }
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// No match — external speaker
|
|
235
|
+
logCalibration('Ext', 0, false)
|
|
236
|
+
return { speaker: 'Ext', similarity: 0 }
|
|
237
|
+
} catch (err: unknown) {
|
|
238
|
+
console.error('[speaker] Identification error:', errMsg(err))
|
|
239
|
+
return null
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Auto-enroll from a high-confidence match during live meetings.
|
|
244
|
+
* Requires consensus: N matches above threshold in the same session before enrolling.
|
|
245
|
+
* Rate limited: max 1 auto-enrollment per speaker per session. */
|
|
246
|
+
export function autoEnroll(
|
|
247
|
+
name: string,
|
|
248
|
+
wavBuffer: Buffer,
|
|
249
|
+
similarity: number,
|
|
250
|
+
sessionId: string,
|
|
251
|
+
): { enrolled: boolean; reason: string } {
|
|
252
|
+
if (!extractor || !manager) return { enrolled: false, reason: 'not initialized' }
|
|
253
|
+
if (name === getOwnerSpeakerLabel() || name === 'Ext') return { enrolled: false, reason: 'skip owner/Ext' }
|
|
254
|
+
if (similarity < AUTO_ENROLL_THRESHOLD) return { enrolled: false, reason: `similarity ${similarity.toFixed(3)} < ${AUTO_ENROLL_THRESHOLD}` }
|
|
255
|
+
|
|
256
|
+
// Check if already auto-enrolled this session
|
|
257
|
+
if (!autoEnrolledThisSession.has(sessionId)) {
|
|
258
|
+
autoEnrolledThisSession.set(sessionId, new Set())
|
|
259
|
+
}
|
|
260
|
+
if (autoEnrolledThisSession.get(sessionId)!.has(name)) {
|
|
261
|
+
return { enrolled: false, reason: 'already enrolled this session' }
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Consensus gate: track match count per speaker per session
|
|
265
|
+
if (!autoEnrollSessions.has(sessionId)) {
|
|
266
|
+
autoEnrollSessions.set(sessionId, new Map())
|
|
267
|
+
}
|
|
268
|
+
const sessionCounts = autoEnrollSessions.get(sessionId)!
|
|
269
|
+
const count = (sessionCounts.get(name) ?? 0) + 1
|
|
270
|
+
sessionCounts.set(name, count)
|
|
271
|
+
|
|
272
|
+
if (count < AUTO_ENROLL_CONSENSUS) {
|
|
273
|
+
return { enrolled: false, reason: `consensus ${count}/${AUTO_ENROLL_CONSENSUS}` }
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// Extract embedding and enroll
|
|
277
|
+
const embedding = extractEmbeddingFromWav(wavBuffer)
|
|
278
|
+
if (!embedding) return { enrolled: false, reason: 'extraction failed' }
|
|
279
|
+
|
|
280
|
+
const result = enrollEmbedding(name, embedding, `auto:${sessionId}`)
|
|
281
|
+
if (result.success) {
|
|
282
|
+
autoEnrolledThisSession.get(sessionId)!.add(name)
|
|
283
|
+
logCalibration(name, similarity, true, 'auto-enrolled')
|
|
284
|
+
console.log(`[speaker] Auto-enrolled "${name}" (sim: ${similarity.toFixed(3)}, session: ${sessionId})`)
|
|
285
|
+
return { enrolled: true, reason: 'success' }
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
return { enrolled: false, reason: result.error ?? 'enrollment failed' }
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Clear auto-enrollment session state (call when meeting ends) */
|
|
292
|
+
export function clearAutoEnrollSession(sessionId: string): void {
|
|
293
|
+
autoEnrollSessions.delete(sessionId)
|
|
294
|
+
autoEnrolledThisSession.delete(sessionId)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Clear all embeddings for a speaker (used by fresh training mode) */
|
|
298
|
+
export function clearSpeakerEmbeddings(name: string): boolean {
|
|
299
|
+
const store = loadProfileStore()
|
|
300
|
+
const profile = store.profiles.find(p => p.name === name)
|
|
301
|
+
if (!profile || profile.embeddings.length === 0) return false
|
|
302
|
+
|
|
303
|
+
console.log(`[speaker] Clearing ${profile.embeddings.length} embeddings for "${name}"`)
|
|
304
|
+
profile.embeddings = []
|
|
305
|
+
profile.sources = []
|
|
306
|
+
|
|
307
|
+
// Remove from manager
|
|
308
|
+
if (manager && manager.contains(name)) {
|
|
309
|
+
try { manager.remove(name) } catch { /* ignore */ }
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Persist
|
|
313
|
+
writeFileSync(PROFILES_PATH, JSON.stringify(store, null, 2))
|
|
314
|
+
invalidateProfileCache()
|
|
315
|
+
return true
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** Get embedding count for a speaker */
|
|
319
|
+
export function getEmbeddingCount(name: string): number {
|
|
320
|
+
const store = loadProfileStore()
|
|
321
|
+
const profile = store.profiles.find(p => p.name === name)
|
|
322
|
+
return profile?.embeddings.length ?? 0
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** Extract raw embedding from WAV buffer */
|
|
326
|
+
export function extractEmbedding(wavBuffer: Buffer): Float32Array | null {
|
|
327
|
+
return extractEmbeddingFromWav(wavBuffer)
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** Check if the embedding system is available */
|
|
331
|
+
export function isEmbeddingAvailable(): boolean {
|
|
332
|
+
return extractor !== null && manager !== null
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/** Compute actual cosine similarity between two raw embedding vectors */
|
|
336
|
+
export function rawCosineSimilarity(a: Float32Array, b: Float32Array): number {
|
|
337
|
+
if (a.length !== b.length) return 0
|
|
338
|
+
let dot = 0, normA = 0, normB = 0
|
|
339
|
+
for (let i = 0; i < a.length; i++) {
|
|
340
|
+
dot += a[i] * b[i]
|
|
341
|
+
normA += a[i] * a[i]
|
|
342
|
+
normB += b[i] * b[i]
|
|
343
|
+
}
|
|
344
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB)
|
|
345
|
+
return denom > 0 ? dot / denom : 0
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// ── Internal helpers ───────────────────────────────────────────
|
|
349
|
+
|
|
350
|
+
function extractEmbeddingFromWav(wavBuffer: Buffer): Float32Array | null {
|
|
351
|
+
if (!extractor) return null
|
|
352
|
+
|
|
353
|
+
try {
|
|
354
|
+
// Parse WAV header to get to PCM data
|
|
355
|
+
const samples = wavBufferToFloat32(wavBuffer)
|
|
356
|
+
if (!samples || samples.length < SAMPLE_RATE * 0.5) {
|
|
357
|
+
// Need at least 0.5s of audio
|
|
358
|
+
return null
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
const stream = extractor.createStream()
|
|
362
|
+
stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE })
|
|
363
|
+
stream.inputFinished()
|
|
364
|
+
|
|
365
|
+
if (!extractor.isReady(stream)) {
|
|
366
|
+
return null
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return extractor.compute(stream)
|
|
370
|
+
} catch (err: unknown) {
|
|
371
|
+
console.error('[speaker] Embedding extraction error:', errMsg(err))
|
|
372
|
+
return null
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Convert WAV buffer (16-bit PCM) to Float32Array normalized to [-1, 1] */
|
|
377
|
+
function wavBufferToFloat32(wavBuffer: Buffer): Float32Array | null {
|
|
378
|
+
// WAV header is 44 bytes for standard PCM
|
|
379
|
+
if (wavBuffer.length < 44) return null
|
|
380
|
+
|
|
381
|
+
// Verify RIFF header
|
|
382
|
+
const riff = wavBuffer.toString('ascii', 0, 4)
|
|
383
|
+
if (riff !== 'RIFF') return null
|
|
384
|
+
|
|
385
|
+
// Find data chunk offset — standard WAV has data at offset 44,
|
|
386
|
+
// but some encoders add extra chunks. Search for 'data' marker.
|
|
387
|
+
let dataOffset = 12
|
|
388
|
+
while (dataOffset < wavBuffer.length - 8) {
|
|
389
|
+
const chunkId = wavBuffer.toString('ascii', dataOffset, dataOffset + 4)
|
|
390
|
+
const chunkSize = wavBuffer.readUInt32LE(dataOffset + 4)
|
|
391
|
+
if (chunkId === 'data') {
|
|
392
|
+
dataOffset += 8
|
|
393
|
+
break
|
|
394
|
+
}
|
|
395
|
+
dataOffset += 8 + chunkSize
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
if (dataOffset >= wavBuffer.length) return null
|
|
399
|
+
|
|
400
|
+
const pcmData = wavBuffer.subarray(dataOffset)
|
|
401
|
+
const numSamples = Math.floor(pcmData.length / 2)
|
|
402
|
+
const float32 = new Float32Array(numSamples)
|
|
403
|
+
|
|
404
|
+
for (let i = 0; i < numSamples; i++) {
|
|
405
|
+
const sample = pcmData.readInt16LE(i * 2)
|
|
406
|
+
float32[i] = sample / 32768.0
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
return float32
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** Compute cosine similarity between an embedding and a stored speaker (approximate via binary search) */
|
|
413
|
+
function computeCosineSimilarity(_embedding: Float32Array, _name: string): number {
|
|
414
|
+
// The sherpa-onnx manager doesn't expose raw stored embeddings,
|
|
415
|
+
// so we use verify with decreasing thresholds to estimate similarity
|
|
416
|
+
if (!manager) return 0
|
|
417
|
+
|
|
418
|
+
// Binary search for similarity threshold
|
|
419
|
+
let lo = 0, hi = 1
|
|
420
|
+
for (let i = 0; i < 10; i++) {
|
|
421
|
+
const mid = (lo + hi) / 2
|
|
422
|
+
const matches = manager.verify({ name: _name, v: _embedding, threshold: mid })
|
|
423
|
+
if (matches) {
|
|
424
|
+
lo = mid
|
|
425
|
+
} else {
|
|
426
|
+
hi = mid
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
return (lo + hi) / 2
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** Log calibration data for threshold tuning */
|
|
433
|
+
function logCalibration(speaker: string, similarity: number, matched: boolean, event?: string): void {
|
|
434
|
+
try {
|
|
435
|
+
const entry: Record<string, string | number | boolean> = {
|
|
436
|
+
ts: new Date().toISOString(),
|
|
437
|
+
speaker,
|
|
438
|
+
similarity: Math.round(similarity * 1000) / 1000,
|
|
439
|
+
matched,
|
|
440
|
+
}
|
|
441
|
+
if (event) entry.event = event
|
|
442
|
+
appendFileSync(CALIBRATION_LOG, JSON.stringify(entry) + '\n')
|
|
443
|
+
} catch { /* non-critical */ }
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/** Load profile store from disk (cached in memory, invalidated on write) */
|
|
447
|
+
function loadProfileStore(): ProfileStore {
|
|
448
|
+
if (_cachedProfileStore) return _cachedProfileStore
|
|
449
|
+
if (existsSync(PROFILES_PATH)) {
|
|
450
|
+
_cachedProfileStore = JSON.parse(readFileSync(PROFILES_PATH, 'utf-8'))
|
|
451
|
+
return _cachedProfileStore!
|
|
452
|
+
}
|
|
453
|
+
return { profiles: [] }
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/** Invalidate the in-memory profile store cache (call after any write to PROFILES_PATH) */
|
|
457
|
+
function invalidateProfileCache(): void {
|
|
458
|
+
_cachedProfileStore = null
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** Persist a speaker profile to disk with provenance tracking */
|
|
462
|
+
function persistProfile(name: string, embedding: Float32Array, source: string = 'manual'): void {
|
|
463
|
+
try {
|
|
464
|
+
const store = loadProfileStore()
|
|
465
|
+
|
|
466
|
+
let profile = store.profiles.find(p => p.name === name)
|
|
467
|
+
if (!profile) {
|
|
468
|
+
profile = { name, embeddings: [], sources: [] }
|
|
469
|
+
store.profiles.push(profile)
|
|
470
|
+
}
|
|
471
|
+
if (!profile.sources) profile.sources = []
|
|
472
|
+
profile.embeddings.push(Array.from(embedding))
|
|
473
|
+
profile.sources.push(source)
|
|
474
|
+
|
|
475
|
+
writeFileSync(PROFILES_PATH, JSON.stringify(store, null, 2))
|
|
476
|
+
invalidateProfileCache()
|
|
477
|
+
} catch (err: unknown) {
|
|
478
|
+
console.error('[speaker] Profile persist error:', errMsg(err))
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** Compute centroid (average) of multiple embeddings.
|
|
483
|
+
* The centroid captures the speaker's average voice across different acoustic
|
|
484
|
+
* conditions (meetings, mics, energy levels). More robust than any single embedding. */
|
|
485
|
+
function computeCentroid(embeddings: number[][]): Float32Array {
|
|
486
|
+
const dim = embeddings[0].length
|
|
487
|
+
const centroid = new Float32Array(dim)
|
|
488
|
+
for (const emb of embeddings) {
|
|
489
|
+
for (let i = 0; i < dim; i++) {
|
|
490
|
+
centroid[i] += emb[i]
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
// Average
|
|
494
|
+
for (let i = 0; i < dim; i++) {
|
|
495
|
+
centroid[i] /= embeddings.length
|
|
496
|
+
}
|
|
497
|
+
// L2 normalize (important for cosine similarity)
|
|
498
|
+
let norm = 0
|
|
499
|
+
for (let i = 0; i < dim; i++) norm += centroid[i] * centroid[i]
|
|
500
|
+
norm = Math.sqrt(norm)
|
|
501
|
+
if (norm > 0) {
|
|
502
|
+
for (let i = 0; i < dim; i++) centroid[i] /= norm
|
|
503
|
+
}
|
|
504
|
+
return centroid
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** Rebuild a speaker in the manager using the centroid of all stored embeddings.
|
|
508
|
+
* The sherpa-onnx manager only supports 1 embedding per speaker name —
|
|
509
|
+
* add() returns false for duplicates. So we compute a centroid from all
|
|
510
|
+
* diverse embeddings and register that single representative vector. */
|
|
511
|
+
function rebuildSpeakerInManager(name: string, embeddings: number[][]): void {
|
|
512
|
+
if (!manager) return
|
|
513
|
+
try {
|
|
514
|
+
// Remove existing entry
|
|
515
|
+
if (manager.contains(name)) {
|
|
516
|
+
manager.remove(name)
|
|
517
|
+
}
|
|
518
|
+
if (embeddings.length === 0) return
|
|
519
|
+
|
|
520
|
+
// Register centroid of all embeddings
|
|
521
|
+
const centroid = computeCentroid(embeddings)
|
|
522
|
+
const added = manager.add({ name, v: centroid })
|
|
523
|
+
if (added) {
|
|
524
|
+
console.log(`[speaker] Registered centroid for "${name}" (${embeddings.length} source embeddings)`)
|
|
525
|
+
} else {
|
|
526
|
+
console.error(`[speaker] Failed to register centroid for "${name}"`)
|
|
527
|
+
}
|
|
528
|
+
} catch (err: unknown) {
|
|
529
|
+
console.error(`[speaker] Rebuild failed for "${name}":`, errMsg(err))
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/** Save full profile store to disk (used by trainer for bulk updates) */
|
|
534
|
+
export function saveProfileStore(store: ProfileStore): void {
|
|
535
|
+
writeFileSync(PROFILES_PATH, JSON.stringify(store, null, 2))
|
|
536
|
+
invalidateProfileCache()
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Rebuild all profiles in manager from a store (used after bulk training) */
|
|
540
|
+
export function rebuildAllProfiles(store: ProfileStore): void {
|
|
541
|
+
if (!manager) return
|
|
542
|
+
// Clear manager completely
|
|
543
|
+
for (const name of getAllSpeakerNames()) {
|
|
544
|
+
try { manager.remove(name) } catch { /* ignore */ }
|
|
545
|
+
}
|
|
546
|
+
// Re-add using centroids
|
|
547
|
+
let loaded = 0
|
|
548
|
+
for (const profile of store.profiles) {
|
|
549
|
+
if (profile.embeddings.length === 0) continue
|
|
550
|
+
rebuildSpeakerInManager(profile.name, profile.embeddings)
|
|
551
|
+
loaded++
|
|
552
|
+
}
|
|
553
|
+
console.log(`[speaker] Rebuilt manager: ${loaded} speakers (centroid mode)`)
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/** Load persisted profiles into manager */
|
|
557
|
+
function loadProfiles(): void {
|
|
558
|
+
if (!manager || !existsSync(PROFILES_PATH)) return
|
|
559
|
+
|
|
560
|
+
try {
|
|
561
|
+
const store: ProfileStore = JSON.parse(readFileSync(PROFILES_PATH, 'utf-8'))
|
|
562
|
+
let loaded = 0
|
|
563
|
+
|
|
564
|
+
for (const profile of store.profiles) {
|
|
565
|
+
if (profile.embeddings.length === 0) continue
|
|
566
|
+
// Register centroid — manager only supports 1 embedding per speaker
|
|
567
|
+
rebuildSpeakerInManager(profile.name, profile.embeddings)
|
|
568
|
+
loaded++
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
if (loaded > 0) {
|
|
572
|
+
const names = manager.getAllSpeakerNames()
|
|
573
|
+
console.log(`[speaker] Loaded ${loaded} speakers (centroid mode): ${names.join(', ')}`)
|
|
574
|
+
}
|
|
575
|
+
} catch (err: unknown) {
|
|
576
|
+
console.error('[speaker] Profile load error:', errMsg(err))
|
|
577
|
+
}
|
|
578
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// Telegram notifier — sends session activity via Telegram bot
|
|
2
|
+
// Reads config from COS scripts .telegram_config.json
|
|
3
|
+
// Provides both security alerts (session start/end) and conversation logging
|
|
4
|
+
|
|
5
|
+
import { readFileSync } from 'node:fs'
|
|
6
|
+
import { resolve } from 'node:path'
|
|
7
|
+
import { COS_SCRIPTS_DIR } from './python-bridge.js'
|
|
8
|
+
|
|
9
|
+
interface TelegramConfig {
|
|
10
|
+
bot_token: string
|
|
11
|
+
chat_id: number
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
let config: TelegramConfig | null = null
|
|
15
|
+
|
|
16
|
+
function loadConfig(): TelegramConfig | null {
|
|
17
|
+
if (config) return config
|
|
18
|
+
if (!COS_SCRIPTS_DIR) return null
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
const raw = readFileSync(resolve(COS_SCRIPTS_DIR, '.telegram_config.json'), 'utf-8')
|
|
22
|
+
const parsed = JSON.parse(raw)
|
|
23
|
+
if (parsed.bot_token && parsed.chat_id) {
|
|
24
|
+
config = { bot_token: parsed.bot_token, chat_id: parsed.chat_id }
|
|
25
|
+
return config
|
|
26
|
+
}
|
|
27
|
+
} catch { /* config not available */ }
|
|
28
|
+
|
|
29
|
+
return null
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function sendTelegram(text: string): Promise<void> {
|
|
33
|
+
const cfg = loadConfig()
|
|
34
|
+
if (!cfg) return
|
|
35
|
+
|
|
36
|
+
try {
|
|
37
|
+
await fetch(`https://api.telegram.org/bot${cfg.bot_token}/sendMessage`, {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
headers: { 'Content-Type': 'application/json' },
|
|
40
|
+
body: JSON.stringify({
|
|
41
|
+
chat_id: cfg.chat_id,
|
|
42
|
+
text,
|
|
43
|
+
parse_mode: 'HTML',
|
|
44
|
+
disable_notification: true,
|
|
45
|
+
}),
|
|
46
|
+
})
|
|
47
|
+
} catch {
|
|
48
|
+
// Silently fail — don't break the glasses experience for a notification
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function truncate(text: string, max: number): string {
|
|
53
|
+
if (text.length <= max) return text
|
|
54
|
+
return text.slice(0, max) + '...'
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function escapeHtml(text: string): string {
|
|
58
|
+
return text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function notifySessionStart(sessionId: string, firstQuery: string): void {
|
|
62
|
+
const time = new Date().toLocaleTimeString('en-US', { hour: 'numeric', minute: '2-digit' })
|
|
63
|
+
const msg = `🟢 <b>COS Glasses session started</b>\n` +
|
|
64
|
+
`Session: <code>${sessionId}</code>\n` +
|
|
65
|
+
`Time: ${time}\n\n` +
|
|
66
|
+
`<b>First query:</b>\n${escapeHtml(truncate(firstQuery, 200))}`
|
|
67
|
+
sendTelegram(msg)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function notifySessionEnd(sessionId: string, exchangeCount: number, durationMin: number): void {
|
|
71
|
+
const time = new Date().toLocaleTimeString('en-US', { hour: 'numeric', minute: '2-digit' })
|
|
72
|
+
const msg = `🔴 <b>COS Glasses session ended</b>\n` +
|
|
73
|
+
`Session: <code>${sessionId}</code>\n` +
|
|
74
|
+
`Time: ${time}\n` +
|
|
75
|
+
`Exchanges: ${exchangeCount} messages\n` +
|
|
76
|
+
`Duration: ${durationMin}m`
|
|
77
|
+
sendTelegram(msg)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function notifyExchange(sessionId: string, query: string, response: string): void {
|
|
81
|
+
const msg = `👓 <b>${escapeHtml(truncate(query, 100))}</b>\n\n` +
|
|
82
|
+
`${escapeHtml(truncate(response, 500))}\n\n` +
|
|
83
|
+
`<i>Session ${sessionId}</i>`
|
|
84
|
+
sendTelegram(msg)
|
|
85
|
+
}
|