@gotcos/glasses-server 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/.cos-profile.example.json +7 -0
  2. package/.env.example +44 -0
  3. package/CHANGELOG.md +25 -0
  4. package/LICENSE +21 -0
  5. package/README.md +78 -0
  6. package/bin/cli.cjs +203 -0
  7. package/package.json +53 -0
  8. package/server/env.ts +26 -0
  9. package/server/index.ts +211 -0
  10. package/server/lib/archive-budget.ts +65 -0
  11. package/server/lib/archive.ts +414 -0
  12. package/server/lib/atomic-fs.ts +50 -0
  13. package/server/lib/audio-enhance.ts +87 -0
  14. package/server/lib/claude-bridge.ts +682 -0
  15. package/server/lib/claude-circuit.ts +52 -0
  16. package/server/lib/claude-run-ledger.ts +279 -0
  17. package/server/lib/codex-bridge.ts +476 -0
  18. package/server/lib/codex-engine-sessions.ts +140 -0
  19. package/server/lib/codex-run-ledger.ts +298 -0
  20. package/server/lib/context-builder.ts +210 -0
  21. package/server/lib/conversation.ts +587 -0
  22. package/server/lib/data-dir.ts +20 -0
  23. package/server/lib/display-bus.ts +21 -0
  24. package/server/lib/display-format.ts +23 -0
  25. package/server/lib/fuzzy-correct.ts +286 -0
  26. package/server/lib/hallucination-filter.ts +469 -0
  27. package/server/lib/local-day.ts +13 -0
  28. package/server/lib/model-router.ts +38 -0
  29. package/server/lib/openai-key.ts +155 -0
  30. package/server/lib/openai-whisper-budget.ts +170 -0
  31. package/server/lib/profile.ts +94 -0
  32. package/server/lib/python-bridge.ts +84 -0
  33. package/server/lib/response-cache.ts +138 -0
  34. package/server/lib/session-cache-writer.ts +266 -0
  35. package/server/lib/session-log.ts +162 -0
  36. package/server/lib/speaker-embeddings.ts +578 -0
  37. package/server/lib/telegram-notify.ts +85 -0
  38. package/server/lib/token-audit.ts +50 -0
  39. package/server/lib/transcribe-audio.ts +187 -0
  40. package/server/lib/utils.ts +5 -0
  41. package/server/lib/vad-silero.ts +179 -0
  42. package/server/lib/whisper-local.ts +697 -0
  43. package/server/routes/diag.ts +115 -0
  44. package/server/routes/display.ts +65 -0
  45. package/server/routes/health.ts +128 -0
  46. package/server/routes/openai-compat.ts +446 -0
  47. package/server/routes/openai-key.ts +121 -0
  48. package/server/routes/query.ts +121 -0
  49. package/server/routes/transcribe-stream.ts +1090 -0
  50. package/server/routes/transcribe.ts +55 -0
  51. package/shared/model-preference.ts +81 -0
@@ -0,0 +1,578 @@
1
+ // Speaker embedding extraction and verification using sherpa-onnx
2
+ // Wraps ECAPA-TDNN model for voiceprint-based speaker classification.
3
+ // Falls back gracefully if model is missing — amplitude classification continues.
4
+ //
5
+ // Phase 1: Auto-enrollment — high-confidence matches add G2-mic embeddings
6
+ // Phase 4: Calendar priming — scoped search to expected meeting attendees
7
+
8
+ import { resolve } from 'node:path'
9
+ import { errMsg } from './utils.js'
10
+ import { getOwnerSpeakerLabel } from './profile.js'
11
+ import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync } from 'node:fs'
12
+ import { fileURLToPath } from 'node:url'
13
+
14
+ // sherpa-onnx-node is CJS — use createRequire for ESM compat
15
+ import { createRequire } from 'node:module'
16
+ const require = createRequire(import.meta.url)
17
+
18
+ const __dirname = fileURLToPath(new URL('.', import.meta.url))
19
+
20
+ const MODEL_PATH = resolve(__dirname, '..', 'models',
21
+ '3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx')
22
+ import { DATA_DIR } from './data-dir.js'
23
+ const PROFILES_PATH = resolve(DATA_DIR, 'voice-profiles.json')
24
+ const CALIBRATION_LOG = resolve(DATA_DIR, 'speaker-calibration.jsonl')
25
+
26
+ // Thresholds
27
+ const VERIFY_THRESHOLD = 0.65
28
+ const SEARCH_THRESHOLD = 0.55
29
+ const AUTO_ENROLL_THRESHOLD = 0.88 // High bar — must be very confident before auto-enrolling
30
+ const AUTO_ENROLL_CONSENSUS = 2 // Must match N times in same session before enrolling
31
+ const MAX_EMBEDDINGS_PER_SPEAKER = 20 // FIFO cap — oldest drops when full
32
+ const SAMPLE_RATE = 16000
33
+
34
+ // Module-level state — sherpa-onnx-node is CJS with no TS types (SDK v0.0.7 interop)
35
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
36
+ let extractor: any = null
37
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
38
+ let manager: any = null
39
+ let initialized = false
40
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
41
+ let sherpaOnnx: any = null
42
+
43
+ // In-memory profile store cache — avoids re-reading 7MB+ JSON on every audio chunk
44
+ let _cachedProfileStore: ProfileStore | null = null
45
+
46
+ // Auto-enrollment session tracking: sessionId → { speakerName → matchCount }
47
+ const autoEnrollSessions = new Map<string, Map<string, number>>()
48
+ // Track which speakers already auto-enrolled this session
49
+ const autoEnrolledThisSession = new Map<string, Set<string>>()
50
+
51
+ interface VoiceProfile {
52
+ name: string
53
+ embeddings: number[][] // multiple enrollments for robustness
54
+ sources?: string[] // provenance: 'manual' | 'fireflies' | 'auto:sessionId'
55
+ }
56
+
57
+ interface ProfileStore {
58
+ profiles: VoiceProfile[]
59
+ }
60
+
61
+ /** Initialize speaker embedding system. Returns false if model missing (graceful degradation). */
62
+ export function initSpeakerEmbeddings(): boolean {
63
+ if (initialized) return extractor !== null
64
+
65
+ initialized = true
66
+
67
+ if (!existsSync(MODEL_PATH)) {
68
+ console.log('[speaker] Model not found at', MODEL_PATH, '— embedding disabled, using amplitude fallback')
69
+ return false
70
+ }
71
+
72
+ try {
73
+ sherpaOnnx = require('sherpa-onnx-node')
74
+
75
+ extractor = new sherpaOnnx.SpeakerEmbeddingExtractor({
76
+ model: MODEL_PATH,
77
+ numThreads: 2,
78
+ provider: 'cpu',
79
+ })
80
+
81
+ manager = new sherpaOnnx.SpeakerEmbeddingManager(extractor.dim)
82
+ console.log(`[speaker] Initialized: ${extractor.dim}-dim embeddings`)
83
+
84
+ // Ensure data directory exists
85
+ if (!existsSync(DATA_DIR)) {
86
+ mkdirSync(DATA_DIR, { recursive: true })
87
+ }
88
+
89
+ // Load persisted profiles
90
+ loadProfiles()
91
+
92
+ return true
93
+ } catch (err: unknown) {
94
+ console.error('[speaker] Init failed:', errMsg(err))
95
+ extractor = null
96
+ manager = null
97
+ return false
98
+ }
99
+ }
100
+
101
+ /** Check if a speaker is enrolled */
102
+ export function isEnrolled(name: string): boolean {
103
+ if (!manager) return false
104
+ return manager.contains(name)
105
+ }
106
+
107
+ /** Get all enrolled speaker names */
108
+ export function getAllSpeakerNames(): string[] {
109
+ if (!manager) return []
110
+ return manager.getAllSpeakerNames()
111
+ }
112
+
113
+ /** Enroll a speaker from WAV audio buffer */
114
+ export function enrollSpeaker(name: string, wavBuffer: Buffer): { success: boolean; dim: number; error?: string } {
115
+ if (!extractor || !manager) {
116
+ return { success: false, dim: 0, error: 'Speaker embeddings not initialized' }
117
+ }
118
+
119
+ try {
120
+ const embedding = extractEmbeddingFromWav(wavBuffer)
121
+ if (!embedding) {
122
+ return { success: false, dim: 0, error: 'Could not extract embedding — audio too short or silent' }
123
+ }
124
+
125
+ return enrollEmbedding(name, embedding, 'manual')
126
+ } catch (err: unknown) {
127
+ console.error('[speaker] Enrollment error:', errMsg(err))
128
+ return { success: false, dim: 0, error: errMsg(err) }
129
+ }
130
+ }
131
+
132
+ /** Enroll a raw embedding directly (used by trainer and auto-enroll).
133
+ * skipDedupCheck: when true, bypass the 0.95 similarity dedup gate.
134
+ * The trainer's greedy diversity selector already ensures embeddings are diverse,
135
+ * so the dedup gate is redundant and too restrictive for batch training
136
+ * (Fireflies audio conditions are uniform enough that even "most diverse"
137
+ * embeddings can exceed 0.95 similarity). */
138
+ export function enrollEmbedding(name: string, embedding: Float32Array, source: string = 'manual', skipDedupCheck: boolean = false): { success: boolean; dim: number; error?: string } {
139
+ if (!extractor || !manager) {
140
+ return { success: false, dim: 0, error: 'Speaker embeddings not initialized' }
141
+ }
142
+
143
+ // Check diversity: skip if too similar to an existing embedding for this speaker
144
+ const store = loadProfileStore()
145
+ const profile = store.profiles.find(p => p.name === name)
146
+ if (profile) {
147
+ // Dedup check: skip if too similar to existing (unless bypassed by trainer)
148
+ if (!skipDedupCheck) {
149
+ for (const existing of profile.embeddings) {
150
+ const sim = rawCosineSimilarity(embedding, new Float32Array(existing))
151
+ if (sim > 0.95) {
152
+ return { success: false, dim: extractor.dim, error: 'Too similar to existing embedding (>0.95)' }
153
+ }
154
+ }
155
+ }
156
+
157
+ // FIFO cap: if at max, drop oldest before adding (always enforced)
158
+ if (profile.embeddings.length >= MAX_EMBEDDINGS_PER_SPEAKER) {
159
+ console.log(`[speaker] Profile cap reached for "${name}" (${profile.embeddings.length}/${MAX_EMBEDDINGS_PER_SPEAKER}) — dropping oldest`)
160
+ profile.embeddings.shift()
161
+ profile.sources?.shift()
162
+ rebuildSpeakerInManager(name, profile.embeddings)
163
+ }
164
+ }
165
+
166
+ // Add to manager — if manager rejects (internal dedup), force via remove+readd
167
+ let added = manager.add({ name, v: embedding })
168
+ if (!added) {
169
+ // Manager's internal dedup rejected it — force-add by rebuilding
170
+ // Persist first so we have the complete embedding list
171
+ persistProfile(name, embedding, source)
172
+ const updatedStore = loadProfileStore()
173
+ const updatedProfile = updatedStore.profiles.find(p => p.name === name)
174
+ if (updatedProfile) {
175
+ rebuildSpeakerInManager(name, updatedProfile.embeddings)
176
+ console.log(`[speaker] Force-enrolled "${name}" via rebuild (${updatedProfile.embeddings.length} total, source: ${source})`)
177
+ return { success: true, dim: extractor.dim }
178
+ }
179
+ return { success: false, dim: extractor.dim, error: 'Manager rejected and rebuild failed' }
180
+ }
181
+
182
+ // Persist to disk with provenance
183
+ persistProfile(name, embedding, source)
184
+
185
+ console.log(`[speaker] Enrolled "${name}" (${extractor.dim}-dim, source: ${source})`)
186
+ return { success: true, dim: extractor.dim }
187
+ }
188
+
189
+ /** Identify speaker from WAV audio buffer.
190
+ * Phase 4: optionally scope search to expected attendees for fewer false positives. */
191
+ export function identifySpeaker(
192
+ wavBuffer: Buffer,
193
+ expectedSpeakers?: string[],
194
+ ): { speaker: string; similarity: number } | null {
195
+ if (!extractor || !manager) return null
196
+
197
+ try {
198
+ const embedding = extractEmbeddingFromWav(wavBuffer)
199
+ if (!embedding) return null
200
+
201
+ // If the wearer is enrolled, verify against them first (they're wearing the glasses)
202
+ const owner = getOwnerSpeakerLabel()
203
+ if (manager.contains(owner)) {
204
+ const isOwner = manager.verify({ name: owner, v: embedding, threshold: VERIFY_THRESHOLD })
205
+ if (isOwner) {
206
+ const similarity = computeCosineSimilarity(embedding, owner)
207
+ logCalibration(owner, similarity, true)
208
+ return { speaker: owner, similarity }
209
+ }
210
+ }
211
+
212
+ // Phase 4: scoped search — try expected attendees first
213
+ if (expectedSpeakers && expectedSpeakers.length > 0) {
214
+ for (const name of expectedSpeakers) {
215
+ if (name === owner) continue // wearer already checked
216
+ if (!manager.contains(name)) continue
217
+ const matches = manager.verify({ name, v: embedding, threshold: SEARCH_THRESHOLD })
218
+ if (matches) {
219
+ const similarity = computeCosineSimilarity(embedding, name)
220
+ logCalibration(name, similarity, true)
221
+ return { speaker: name, similarity }
222
+ }
223
+ }
224
+ }
225
+
226
+ // Full search across all speakers (fallback or no expected speakers)
227
+ const found = manager.search({ v: embedding, threshold: SEARCH_THRESHOLD })
228
+ if (found && found.length > 0) {
229
+ const similarity = computeCosineSimilarity(embedding, found)
230
+ logCalibration(found, similarity, true)
231
+ return { speaker: found, similarity }
232
+ }
233
+
234
+ // No match — external speaker
235
+ logCalibration('Ext', 0, false)
236
+ return { speaker: 'Ext', similarity: 0 }
237
+ } catch (err: unknown) {
238
+ console.error('[speaker] Identification error:', errMsg(err))
239
+ return null
240
+ }
241
+ }
242
+
243
+ /** Auto-enroll from a high-confidence match during live meetings.
244
+ * Requires consensus: N matches above threshold in the same session before enrolling.
245
+ * Rate limited: max 1 auto-enrollment per speaker per session. */
246
+ export function autoEnroll(
247
+ name: string,
248
+ wavBuffer: Buffer,
249
+ similarity: number,
250
+ sessionId: string,
251
+ ): { enrolled: boolean; reason: string } {
252
+ if (!extractor || !manager) return { enrolled: false, reason: 'not initialized' }
253
+ if (name === getOwnerSpeakerLabel() || name === 'Ext') return { enrolled: false, reason: 'skip owner/Ext' }
254
+ if (similarity < AUTO_ENROLL_THRESHOLD) return { enrolled: false, reason: `similarity ${similarity.toFixed(3)} < ${AUTO_ENROLL_THRESHOLD}` }
255
+
256
+ // Check if already auto-enrolled this session
257
+ if (!autoEnrolledThisSession.has(sessionId)) {
258
+ autoEnrolledThisSession.set(sessionId, new Set())
259
+ }
260
+ if (autoEnrolledThisSession.get(sessionId)!.has(name)) {
261
+ return { enrolled: false, reason: 'already enrolled this session' }
262
+ }
263
+
264
+ // Consensus gate: track match count per speaker per session
265
+ if (!autoEnrollSessions.has(sessionId)) {
266
+ autoEnrollSessions.set(sessionId, new Map())
267
+ }
268
+ const sessionCounts = autoEnrollSessions.get(sessionId)!
269
+ const count = (sessionCounts.get(name) ?? 0) + 1
270
+ sessionCounts.set(name, count)
271
+
272
+ if (count < AUTO_ENROLL_CONSENSUS) {
273
+ return { enrolled: false, reason: `consensus ${count}/${AUTO_ENROLL_CONSENSUS}` }
274
+ }
275
+
276
+ // Extract embedding and enroll
277
+ const embedding = extractEmbeddingFromWav(wavBuffer)
278
+ if (!embedding) return { enrolled: false, reason: 'extraction failed' }
279
+
280
+ const result = enrollEmbedding(name, embedding, `auto:${sessionId}`)
281
+ if (result.success) {
282
+ autoEnrolledThisSession.get(sessionId)!.add(name)
283
+ logCalibration(name, similarity, true, 'auto-enrolled')
284
+ console.log(`[speaker] Auto-enrolled "${name}" (sim: ${similarity.toFixed(3)}, session: ${sessionId})`)
285
+ return { enrolled: true, reason: 'success' }
286
+ }
287
+
288
+ return { enrolled: false, reason: result.error ?? 'enrollment failed' }
289
+ }
290
+
291
+ /** Clear auto-enrollment session state (call when meeting ends) */
292
+ export function clearAutoEnrollSession(sessionId: string): void {
293
+ autoEnrollSessions.delete(sessionId)
294
+ autoEnrolledThisSession.delete(sessionId)
295
+ }
296
+
297
+ /** Clear all embeddings for a speaker (used by fresh training mode) */
298
+ export function clearSpeakerEmbeddings(name: string): boolean {
299
+ const store = loadProfileStore()
300
+ const profile = store.profiles.find(p => p.name === name)
301
+ if (!profile || profile.embeddings.length === 0) return false
302
+
303
+ console.log(`[speaker] Clearing ${profile.embeddings.length} embeddings for "${name}"`)
304
+ profile.embeddings = []
305
+ profile.sources = []
306
+
307
+ // Remove from manager
308
+ if (manager && manager.contains(name)) {
309
+ try { manager.remove(name) } catch { /* ignore */ }
310
+ }
311
+
312
+ // Persist
313
+ writeFileSync(PROFILES_PATH, JSON.stringify(store, null, 2))
314
+ invalidateProfileCache()
315
+ return true
316
+ }
317
+
318
+ /** Get embedding count for a speaker */
319
+ export function getEmbeddingCount(name: string): number {
320
+ const store = loadProfileStore()
321
+ const profile = store.profiles.find(p => p.name === name)
322
+ return profile?.embeddings.length ?? 0
323
+ }
324
+
325
+ /** Extract raw embedding from WAV buffer */
326
+ export function extractEmbedding(wavBuffer: Buffer): Float32Array | null {
327
+ return extractEmbeddingFromWav(wavBuffer)
328
+ }
329
+
330
+ /** Check if the embedding system is available */
331
+ export function isEmbeddingAvailable(): boolean {
332
+ return extractor !== null && manager !== null
333
+ }
334
+
335
+ /** Compute actual cosine similarity between two raw embedding vectors */
336
+ export function rawCosineSimilarity(a: Float32Array, b: Float32Array): number {
337
+ if (a.length !== b.length) return 0
338
+ let dot = 0, normA = 0, normB = 0
339
+ for (let i = 0; i < a.length; i++) {
340
+ dot += a[i] * b[i]
341
+ normA += a[i] * a[i]
342
+ normB += b[i] * b[i]
343
+ }
344
+ const denom = Math.sqrt(normA) * Math.sqrt(normB)
345
+ return denom > 0 ? dot / denom : 0
346
+ }
347
+
348
+ // ── Internal helpers ───────────────────────────────────────────
349
+
350
+ function extractEmbeddingFromWav(wavBuffer: Buffer): Float32Array | null {
351
+ if (!extractor) return null
352
+
353
+ try {
354
+ // Parse WAV header to get to PCM data
355
+ const samples = wavBufferToFloat32(wavBuffer)
356
+ if (!samples || samples.length < SAMPLE_RATE * 0.5) {
357
+ // Need at least 0.5s of audio
358
+ return null
359
+ }
360
+
361
+ const stream = extractor.createStream()
362
+ stream.acceptWaveform({ samples, sampleRate: SAMPLE_RATE })
363
+ stream.inputFinished()
364
+
365
+ if (!extractor.isReady(stream)) {
366
+ return null
367
+ }
368
+
369
+ return extractor.compute(stream)
370
+ } catch (err: unknown) {
371
+ console.error('[speaker] Embedding extraction error:', errMsg(err))
372
+ return null
373
+ }
374
+ }
375
+
376
+ /** Convert WAV buffer (16-bit PCM) to Float32Array normalized to [-1, 1] */
377
+ function wavBufferToFloat32(wavBuffer: Buffer): Float32Array | null {
378
+ // WAV header is 44 bytes for standard PCM
379
+ if (wavBuffer.length < 44) return null
380
+
381
+ // Verify RIFF header
382
+ const riff = wavBuffer.toString('ascii', 0, 4)
383
+ if (riff !== 'RIFF') return null
384
+
385
+ // Find data chunk offset — standard WAV has data at offset 44,
386
+ // but some encoders add extra chunks. Search for 'data' marker.
387
+ let dataOffset = 12
388
+ while (dataOffset < wavBuffer.length - 8) {
389
+ const chunkId = wavBuffer.toString('ascii', dataOffset, dataOffset + 4)
390
+ const chunkSize = wavBuffer.readUInt32LE(dataOffset + 4)
391
+ if (chunkId === 'data') {
392
+ dataOffset += 8
393
+ break
394
+ }
395
+ dataOffset += 8 + chunkSize
396
+ }
397
+
398
+ if (dataOffset >= wavBuffer.length) return null
399
+
400
+ const pcmData = wavBuffer.subarray(dataOffset)
401
+ const numSamples = Math.floor(pcmData.length / 2)
402
+ const float32 = new Float32Array(numSamples)
403
+
404
+ for (let i = 0; i < numSamples; i++) {
405
+ const sample = pcmData.readInt16LE(i * 2)
406
+ float32[i] = sample / 32768.0
407
+ }
408
+
409
+ return float32
410
+ }
411
+
412
+ /** Compute cosine similarity between an embedding and a stored speaker (approximate via binary search) */
413
+ function computeCosineSimilarity(_embedding: Float32Array, _name: string): number {
414
+ // The sherpa-onnx manager doesn't expose raw stored embeddings,
415
+ // so we use verify with decreasing thresholds to estimate similarity
416
+ if (!manager) return 0
417
+
418
+ // Binary search for similarity threshold
419
+ let lo = 0, hi = 1
420
+ for (let i = 0; i < 10; i++) {
421
+ const mid = (lo + hi) / 2
422
+ const matches = manager.verify({ name: _name, v: _embedding, threshold: mid })
423
+ if (matches) {
424
+ lo = mid
425
+ } else {
426
+ hi = mid
427
+ }
428
+ }
429
+ return (lo + hi) / 2
430
+ }
431
+
432
+ /** Log calibration data for threshold tuning */
433
+ function logCalibration(speaker: string, similarity: number, matched: boolean, event?: string): void {
434
+ try {
435
+ const entry: Record<string, string | number | boolean> = {
436
+ ts: new Date().toISOString(),
437
+ speaker,
438
+ similarity: Math.round(similarity * 1000) / 1000,
439
+ matched,
440
+ }
441
+ if (event) entry.event = event
442
+ appendFileSync(CALIBRATION_LOG, JSON.stringify(entry) + '\n')
443
+ } catch { /* non-critical */ }
444
+ }
445
+
446
+ /** Load profile store from disk (cached in memory, invalidated on write) */
447
+ function loadProfileStore(): ProfileStore {
448
+ if (_cachedProfileStore) return _cachedProfileStore
449
+ if (existsSync(PROFILES_PATH)) {
450
+ _cachedProfileStore = JSON.parse(readFileSync(PROFILES_PATH, 'utf-8'))
451
+ return _cachedProfileStore!
452
+ }
453
+ return { profiles: [] }
454
+ }
455
+
456
+ /** Invalidate the in-memory profile store cache (call after any write to PROFILES_PATH) */
457
+ function invalidateProfileCache(): void {
458
+ _cachedProfileStore = null
459
+ }
460
+
461
+ /** Persist a speaker profile to disk with provenance tracking */
462
+ function persistProfile(name: string, embedding: Float32Array, source: string = 'manual'): void {
463
+ try {
464
+ const store = loadProfileStore()
465
+
466
+ let profile = store.profiles.find(p => p.name === name)
467
+ if (!profile) {
468
+ profile = { name, embeddings: [], sources: [] }
469
+ store.profiles.push(profile)
470
+ }
471
+ if (!profile.sources) profile.sources = []
472
+ profile.embeddings.push(Array.from(embedding))
473
+ profile.sources.push(source)
474
+
475
+ writeFileSync(PROFILES_PATH, JSON.stringify(store, null, 2))
476
+ invalidateProfileCache()
477
+ } catch (err: unknown) {
478
+ console.error('[speaker] Profile persist error:', errMsg(err))
479
+ }
480
+ }
481
+
482
+ /** Compute centroid (average) of multiple embeddings.
483
+ * The centroid captures the speaker's average voice across different acoustic
484
+ * conditions (meetings, mics, energy levels). More robust than any single embedding. */
485
+ function computeCentroid(embeddings: number[][]): Float32Array {
486
+ const dim = embeddings[0].length
487
+ const centroid = new Float32Array(dim)
488
+ for (const emb of embeddings) {
489
+ for (let i = 0; i < dim; i++) {
490
+ centroid[i] += emb[i]
491
+ }
492
+ }
493
+ // Average
494
+ for (let i = 0; i < dim; i++) {
495
+ centroid[i] /= embeddings.length
496
+ }
497
+ // L2 normalize (important for cosine similarity)
498
+ let norm = 0
499
+ for (let i = 0; i < dim; i++) norm += centroid[i] * centroid[i]
500
+ norm = Math.sqrt(norm)
501
+ if (norm > 0) {
502
+ for (let i = 0; i < dim; i++) centroid[i] /= norm
503
+ }
504
+ return centroid
505
+ }
506
+
507
+ /** Rebuild a speaker in the manager using the centroid of all stored embeddings.
508
+ * The sherpa-onnx manager only supports 1 embedding per speaker name —
509
+ * add() returns false for duplicates. So we compute a centroid from all
510
+ * diverse embeddings and register that single representative vector. */
511
+ function rebuildSpeakerInManager(name: string, embeddings: number[][]): void {
512
+ if (!manager) return
513
+ try {
514
+ // Remove existing entry
515
+ if (manager.contains(name)) {
516
+ manager.remove(name)
517
+ }
518
+ if (embeddings.length === 0) return
519
+
520
+ // Register centroid of all embeddings
521
+ const centroid = computeCentroid(embeddings)
522
+ const added = manager.add({ name, v: centroid })
523
+ if (added) {
524
+ console.log(`[speaker] Registered centroid for "${name}" (${embeddings.length} source embeddings)`)
525
+ } else {
526
+ console.error(`[speaker] Failed to register centroid for "${name}"`)
527
+ }
528
+ } catch (err: unknown) {
529
+ console.error(`[speaker] Rebuild failed for "${name}":`, errMsg(err))
530
+ }
531
+ }
532
+
533
+ /** Save full profile store to disk (used by trainer for bulk updates) */
534
+ export function saveProfileStore(store: ProfileStore): void {
535
+ writeFileSync(PROFILES_PATH, JSON.stringify(store, null, 2))
536
+ invalidateProfileCache()
537
+ }
538
+
539
+ /** Rebuild all profiles in manager from a store (used after bulk training) */
540
+ export function rebuildAllProfiles(store: ProfileStore): void {
541
+ if (!manager) return
542
+ // Clear manager completely
543
+ for (const name of getAllSpeakerNames()) {
544
+ try { manager.remove(name) } catch { /* ignore */ }
545
+ }
546
+ // Re-add using centroids
547
+ let loaded = 0
548
+ for (const profile of store.profiles) {
549
+ if (profile.embeddings.length === 0) continue
550
+ rebuildSpeakerInManager(profile.name, profile.embeddings)
551
+ loaded++
552
+ }
553
+ console.log(`[speaker] Rebuilt manager: ${loaded} speakers (centroid mode)`)
554
+ }
555
+
556
+ /** Load persisted profiles into manager */
557
+ function loadProfiles(): void {
558
+ if (!manager || !existsSync(PROFILES_PATH)) return
559
+
560
+ try {
561
+ const store: ProfileStore = JSON.parse(readFileSync(PROFILES_PATH, 'utf-8'))
562
+ let loaded = 0
563
+
564
+ for (const profile of store.profiles) {
565
+ if (profile.embeddings.length === 0) continue
566
+ // Register centroid — manager only supports 1 embedding per speaker
567
+ rebuildSpeakerInManager(profile.name, profile.embeddings)
568
+ loaded++
569
+ }
570
+
571
+ if (loaded > 0) {
572
+ const names = manager.getAllSpeakerNames()
573
+ console.log(`[speaker] Loaded ${loaded} speakers (centroid mode): ${names.join(', ')}`)
574
+ }
575
+ } catch (err: unknown) {
576
+ console.error('[speaker] Profile load error:', errMsg(err))
577
+ }
578
+ }
@@ -0,0 +1,85 @@
1
+ // Telegram notifier — sends session activity via Telegram bot
2
+ // Reads config from COS scripts .telegram_config.json
3
+ // Provides both security alerts (session start/end) and conversation logging
4
+
5
+ import { readFileSync } from 'node:fs'
6
+ import { resolve } from 'node:path'
7
+ import { COS_SCRIPTS_DIR } from './python-bridge.js'
8
+
9
+ interface TelegramConfig {
10
+ bot_token: string
11
+ chat_id: number
12
+ }
13
+
14
+ let config: TelegramConfig | null = null
15
+
16
+ function loadConfig(): TelegramConfig | null {
17
+ if (config) return config
18
+ if (!COS_SCRIPTS_DIR) return null
19
+
20
+ try {
21
+ const raw = readFileSync(resolve(COS_SCRIPTS_DIR, '.telegram_config.json'), 'utf-8')
22
+ const parsed = JSON.parse(raw)
23
+ if (parsed.bot_token && parsed.chat_id) {
24
+ config = { bot_token: parsed.bot_token, chat_id: parsed.chat_id }
25
+ return config
26
+ }
27
+ } catch { /* config not available */ }
28
+
29
+ return null
30
+ }
31
+
32
+ async function sendTelegram(text: string): Promise<void> {
33
+ const cfg = loadConfig()
34
+ if (!cfg) return
35
+
36
+ try {
37
+ await fetch(`https://api.telegram.org/bot${cfg.bot_token}/sendMessage`, {
38
+ method: 'POST',
39
+ headers: { 'Content-Type': 'application/json' },
40
+ body: JSON.stringify({
41
+ chat_id: cfg.chat_id,
42
+ text,
43
+ parse_mode: 'HTML',
44
+ disable_notification: true,
45
+ }),
46
+ })
47
+ } catch {
48
+ // Silently fail — don't break the glasses experience for a notification
49
+ }
50
+ }
51
+
52
+ function truncate(text: string, max: number): string {
53
+ if (text.length <= max) return text
54
+ return text.slice(0, max) + '...'
55
+ }
56
+
57
+ function escapeHtml(text: string): string {
58
+ return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
59
+ }
60
+
61
+ export function notifySessionStart(sessionId: string, firstQuery: string): void {
62
+ const time = new Date().toLocaleTimeString('en-US', { hour: 'numeric', minute: '2-digit' })
63
+ const msg = `🟢 <b>COS Glasses session started</b>\n` +
64
+ `Session: <code>${sessionId}</code>\n` +
65
+ `Time: ${time}\n\n` +
66
+ `<b>First query:</b>\n${escapeHtml(truncate(firstQuery, 200))}`
67
+ sendTelegram(msg)
68
+ }
69
+
70
+ export function notifySessionEnd(sessionId: string, exchangeCount: number, durationMin: number): void {
71
+ const time = new Date().toLocaleTimeString('en-US', { hour: 'numeric', minute: '2-digit' })
72
+ const msg = `🔴 <b>COS Glasses session ended</b>\n` +
73
+ `Session: <code>${sessionId}</code>\n` +
74
+ `Time: ${time}\n` +
75
+ `Exchanges: ${exchangeCount} messages\n` +
76
+ `Duration: ${durationMin}m`
77
+ sendTelegram(msg)
78
+ }
79
+
80
+ export function notifyExchange(sessionId: string, query: string, response: string): void {
81
+ const msg = `👓 <b>${escapeHtml(truncate(query, 100))}</b>\n\n` +
82
+ `${escapeHtml(truncate(response, 500))}\n\n` +
83
+ `<i>Session ${sessionId}</i>`
84
+ sendTelegram(msg)
85
+ }