@gotcos/glasses-server 6.21.14 → 6.21.18

package/server/lib/speaker-embeddings.ts

@@ -1,4 +1,5 @@
 // Speaker embedding extraction and verification using sherpa-onnx
+import { chooseEviction, tierBreakdown } from './embedding-eviction.js'
 // Wraps ECAPA-TDNN model for voiceprint-based speaker classification.
 // Falls back gracefully if model is missing — amplitude classification continues.
 //
@@ -15,6 +16,8 @@ import {
   mergeProfilesInStore,
   profileSimilarity,
   describeRepairs,
+  alignedSources,
+  dropEmbeddingAt,
   dropOldestEmbedding,
   hasRepairs,
   loadVoiceProfileStore,
@@ -87,7 +90,11 @@ const VERIFY_THRESHOLD = 0.65
 const SEARCH_THRESHOLD = 0.55
 const AUTO_ENROLL_THRESHOLD = 0.88    // High bar — must be very confident before auto-enrolling
 const AUTO_ENROLL_CONSENSUS = 2       // Must match N times in same session before enrolling
-const MAX_EMBEDDINGS_PER_SPEAKER = 20 // FIFO cap — oldest drops when full
+// Raised 20 -> 40 on 2026-08-06. Measured, not guessed: search latency is 1 us
+// at 20, 40 AND 80 samples per speaker (77 speakers, sherpa SpeakerEmbeddingManager),
+// so the old cap defended nothing — while 61 of 77 profiles sat AT it, meaning
+// every correction cost a sample. 20 extra slots per speaker is ~1.2 MB.
+const MAX_EMBEDDINGS_PER_SPEAKER = 40
 const SAMPLE_RATE = 16000
 
 // Module-level state — sherpa-onnx-node is CJS with no TS types (SDK v0.0.7 interop)
@@ -308,8 +315,24 @@ export function enrollEmbedding(name: string, embedding: Float32Array, source: s
     // `sources?.shift()` no-opped whenever sources was undefined or short,
     // permanently offsetting provenance from the samples it described.
     if (profile.embeddings.length >= MAX_EMBEDDINGS_PER_SPEAKER) {
-      console.log(`[speaker] Profile cap reached for "${name}" (${profile.embeddings.length}/${MAX_EMBEDDINGS_PER_SPEAKER}) — dropping oldest`)
-      dropOldestEmbedding(profile)
+      // Weakest PROVENANCE goes, not the oldest sample. Age is the wrong axis:
+      // four profiles at cap would lose their only human-supplied sample to
+      // FIFO while unverified attendee-metadata samples sat untouched.
+      // alignedSources is defence-in-depth here, not load-bearing: loadProfileStore
+      // already pads sources[] to match embeddings[], so a ragged array cannot
+      // reach this line (mutation-verified — swapping it for profile.sources is
+      // unobservable through this path). It is kept for any future caller that
+      // builds a profile without going through the loader, and is unit-tested
+      // directly in voice-profile-store.test.ts.
+      const choice = chooseEviction(alignedSources(profile), source, MAX_EMBEDDINGS_PER_SPEAKER)
+      const { droppedSource } = choice
+        ? dropEmbeddingAt(profile, choice.index)
+        : dropOldestEmbedding(profile)
+      console.log(
+        `[speaker] Profile full for "${name}" (${MAX_EMBEDDINGS_PER_SPEAKER}) — `
+        + `${choice ? choice.reason : 'no provenance available, dropping the oldest'} `
+        + `[dropped ${droppedSource ?? 'unknown'}, incoming ${source}]`,
+      )
       rebuildSpeakerInManager(name, profile.embeddings)
     }
   }
@@ -342,7 +365,7 @@ export function enrollEmbedding(name: string, embedding: Float32Array, source: s
 export function identifySpeaker(
   wavBuffer: Buffer,
   expectedSpeakers?: string[],
-): { speaker: string; similarity: number } | null {
+): { speaker: string; similarity: number; embedding?: Float32Array } | null {
   if (!extractor || !manager) return null
 
   try {
@@ -356,7 +379,7 @@ export function identifySpeaker(
       if (isOwner) {
         const similarity = computeCosineSimilarity(embedding, owner)
         logCalibration(owner, similarity, true)
-        return { speaker: owner, similarity }
+        return { speaker: owner, similarity, embedding }
       }
     }
 
@@ -369,7 +392,7 @@ export function identifySpeaker(
         if (matches) {
           const similarity = computeCosineSimilarity(embedding, name)
           logCalibration(name, similarity, true)
-          return { speaker: name, similarity }
+          return { speaker: name, similarity, embedding }
         }
       }
     }
@@ -379,12 +402,14 @@ export function identifySpeaker(
     if (found && found.length > 0) {
       const similarity = computeCosineSimilarity(embedding, found)
       logCalibration(found, similarity, true)
-      return { speaker: found, similarity }
+      return { speaker: found, similarity, embedding }
     }
 
-    // No match — external speaker
+    // No match — external speaker. The embedding still goes back: an
+    // unidentified voice's vector is the most valuable thing to retain, because
+    // naming it later is exactly the correction that has no other evidence.
     logCalibration('Ext', 0, false)
-    return { speaker: 'Ext', similarity: 0 }
+    return { speaker: 'Ext', similarity: 0, embedding }
   } catch (err: unknown) {
     console.error('[speaker] Identification error:', errMsg(err))
     return null
@@ -665,6 +690,35 @@ export function speakerReadiness(
   return state === 'active' ? 'ready' : 'unavailable'
 }
 
+/**
+ * Provenance composition across every stored profile, plus the profiles with no
+ * human-verified sample at all.
+ *
+ * Surfaced because the number that mattered here was invisible: the owner's own
+ * profile — the one driving owner detection — was 10 attendee-metadata samples,
+ * 9 identifier-labelled ones and 1 unlabelled, with nothing a human had ever
+ * confirmed. Nothing in any status output said so.
+ */
+export function profileProvenanceSummary(): {
+  profiles: number
+  atCap: number
+  cap: number
+  tiers: Record<string, number>
+  noHumanSample: string[]
+} {
+  const store = loadProfileStore()
+  const tiers: Record<string, number> = {}
+  const noHumanSample: string[] = []
+  let atCap = 0
+  for (const p of store.profiles) {
+    const breakdown = tierBreakdown(alignedSources(p))
+    for (const [k, v] of Object.entries(breakdown)) tiers[k] = (tiers[k] ?? 0) + v
+    if (p.embeddings.length >= MAX_EMBEDDINGS_PER_SPEAKER) atCap++
+    if (breakdown.human === 0) noHumanSample.push(p.name)
+  }
+  return { profiles: store.profiles.length, atCap, cap: MAX_EMBEDDINGS_PER_SPEAKER, tiers, noHumanSample }
+}
+
 /** Compute actual cosine similarity between two raw embedding vectors */
 export function rawCosineSimilarity(a: Float32Array, b: Float32Array): number {
   if (a.length !== b.length) return 0

package/server/lib/training-audio-provenance.ts

@@ -0,0 +1,70 @@
+// Tracing a stored voice sample back to the meeting that produced it.
+//
+// WHY THIS IS NEEDED. De-attribution has to undo more than a label. When a voice
+// is falsely attributed to someone, the identifier may ALSO have auto-enrolled
+// those segments into that person's profile — so the wrong voice is now part of
+// what the system thinks they sound like, and it will keep matching. Removing the
+// label without removing the samples fixes the transcript and leaves the profile
+// poisoned. That is the mechanism behind the phantom "Erick Hernandez": three
+// mislabelled seeds compounded to eighteen samples through self-training.
+//
+// WHAT CAN AND CANNOT BE TRACED. Provenance strings differ in whether they name
+// a session:
+//
+//   auto:<sessionId>           traceable — autoEnroll stamps it
+//   correction:<sessionId>     traceable — the relabel ledger stamps it
+//   g2-training:<sessionId>    traceable ONLY from 2026-08-06 onward (see below)
+//   g2-training                NOT traceable — every sample written before that
+//   fireflies / manual / …     NOT traceable — no session concept
+//
+// `train-g2` used to stamp a bare `g2-training`, discarding which meeting each
+// sample came from even though the source WAV filename carries it
+// (`meeting_1785190805524_uzxmn4_chunk327_sim0.57.wav`). Samples written before
+// the stamp landed cannot be retracted per-meeting, and de-attribution reports
+// that count rather than implying a clean sweep.
+
+/** Session id embedded in a training-audio WAV name, or null if absent. */
+export function sessionIdFromTrainingWav(filename: string): string | null {
+  // Session ids contain underscores (`meeting_1785190805524_uzxmn4`), so the
+  // split has to be on the `_chunk` marker, not on the last underscore.
+  const at = filename.indexOf('_chunk')
+  if (at <= 0) return null
+  const id = filename.slice(0, at)
+  return /^[A-Za-z0-9:_-]{3,96}$/.test(id) ? id : null
+}
+
+/** Provenance to record for a sample trained out of `filename`. */
+export function trainingSourceFor(filename: string): string {
+  const id = sessionIdFromTrainingWav(filename)
+  return id ? `g2-training:${id}` : 'g2-training'
+}
+
+/**
+ * Did this stored sample come from `sessionId`?
+ *
+ * Exact match on the session part — a prefix test would let
+ * `auto:meeting_123_extra` match session `meeting_123` and retract a different
+ * meeting's evidence.
+ */
+export function isSampleFromSession(source: string | undefined | null, sessionId: string): boolean {
+  const s = String(source ?? '')
+  if (!sessionId) return false
+  for (const prefix of ['auto:', 'correction:', 'g2-training:']) {
+    if (s.startsWith(prefix) && s.slice(prefix.length) === sessionId) return true
+  }
+  return false
+}
+
+/**
+ * Samples that belong to this speaker but cannot be tied to any meeting.
+ *
+ * Reported so a de-attribution is honest about its reach: "retracted 4, and 9
+ * older samples cannot be traced to a meeting" is actionable, while silence
+ * implies the profile was fully cleaned.
+ */
+export function untraceableSampleCount(sources: Array<string | undefined | null>): number {
+  return sources.filter(s => {
+    const str = String(s ?? '')
+    return !str.includes(':')
+  }).length
+}

package/server/lib/voice-profile-store.ts

@@ -69,7 +69,10 @@ export function describeRepairs(r: StoreRepairs): string {
   return parts.join(', ')
 }
 
-const UNKNOWN_SOURCE = 'unknown'
+/** Placeholder provenance for a sample whose source was never recorded.
+ *  Exported so callers and tests reference the same value the store writes;
+ *  3 samples in the live store carry it. */
+export const UNKNOWN_SOURCE = 'unknown'
 
 function isUsableEmbedding(row: unknown): row is number[] {
   return Array.isArray(row) && row.length > 0 && row.every(v => typeof v === 'number' && Number.isFinite(v))
@@ -317,6 +320,38 @@ export function dropOldestEmbedding(profile: VoiceProfile): { droppedSource: str
   return { droppedSource }
 }
 
+/**
+ * Drop the sample at `index`, keeping sources[] in lockstep.
+ *
+ * Alignment happens BEFORE the splice: a sources[] shorter than embeddings[]
+ * would otherwise make index N refer to two different samples in the two
+ * arrays, permanently offsetting provenance from the samples it describes —
+ * the same class of bug as the old inline `sources?.shift()`.
+ */
+export function dropEmbeddingAt(profile: VoiceProfile, index: number): { droppedSource: string | null } {
+  if (!Number.isInteger(index) || index < 0 || index >= profile.embeddings.length) {
+    return { droppedSource: null }
+  }
+  if (!Array.isArray(profile.sources)) profile.sources = []
+  while (profile.sources.length < profile.embeddings.length) profile.sources.push(UNKNOWN_SOURCE)
+  profile.embeddings.splice(index, 1)
+  const droppedSource = profile.sources.splice(index, 1)[0] ?? null
+  profile.sources.length = profile.embeddings.length
+  return { droppedSource }
+}
+
+/**
+ * Provenance of each stored sample, index-aligned with embeddings[].
+ *
+ * Holes and a short array both read as undefined, which the eviction policy
+ * classifies as `unknown` — the weakest tier. That is the safe direction: a
+ * sample whose provenance was lost must never inherit the protection given to
+ * one a human supplied.
+ */
+export function alignedSources(profile: VoiceProfile): Array<string | undefined> {
+  return Array.from({ length: profile.embeddings.length }, (_, i) => profile.sources?.[i])
+}
+
 /** Append a sample to both arrays together. */
 export function appendEmbedding(profile: VoiceProfile, embedding: number[], source: string): void {
   if (!Array.isArray(profile.sources)) profile.sources = []

package/server/routes/health.ts

@@ -6,7 +6,10 @@ import { serverMetrics } from '../lib/server-metrics.js'
 import { getServerInstanceId } from '../lib/server-instance-id.js'
 import { localFirstMeetingsCapability } from '../lib/local-first-meetings-contract.js'
 import { isSileroAvailable } from '../lib/vad-silero.js'
-import { speakerModelState, speakerReadiness } from '../lib/speaker-embeddings.js'
+import { profileProvenanceSummary, speakerModelState, speakerReadiness } from '../lib/speaker-embeddings.js'
+import { chunkEmbeddingStoreStats } from '../lib/chunk-embedding-store.js'
+import { correctionStoreStats } from '../lib/meeting-corrections.js'
+import { meetingAudioStats } from '../lib/meeting-audio-archive.js'
 import { getAvailableCliSessionId } from '../lib/claude-bridge.js'
 import {
   isWhisperLocalAvailable,
@@ -116,6 +119,17 @@ healthRouter.get('/health', async (_req, res) => {
   const speakerId = speakerModelState()
   checks.speaker_id = speakerId.state
 
+  // Visible so the correction loop can be seen banking evidence rather than
+  // trusted to be. Counts only — no session ids on an unauthenticated surface.
+  const chunkEmbeddings = chunkEmbeddingStoreStats()
+  // `pending` is the number that matters here: an intent that never closed means
+  // some meeting's files may be half-rewritten.
+  const speakerCorrections = correctionStoreStats()
+  const reviewAudio = meetingAudioStats()
+  // `noHumanSample` is the one to read: a profile with no human-verified sample
+  // is trained entirely on labels the system chose for itself.
+  const voiceProvenance = speakerId.state === 'active' ? profileProvenanceSummary() : null
+
   // Health is unauthenticated. Publish only availability; the actual CLI
   // session id is a resumable runtime handle and belongs on authenticated
   // query/debug surfaces.
@@ -232,6 +246,10 @@ healthRouter.get('/health', async (_req, res) => {
     cursor_models,
     meeting_sync,
     unsaved_captures,
+    chunk_embeddings: chunkEmbeddings,
+    speaker_corrections: speakerCorrections,
+    review_audio: reviewAudio,
+    ...(voiceProvenance ? { voice_provenance: voiceProvenance } : {}),
     capabilities: {
       transcription: {
         ...transcription,