@gotcos/glasses-server 6.21.22 → 6.21.23

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 6.21.23
+
+- `GET /meeting/:sessionId/embeddings` — why each chunk was labelled the way it
+  was. Reads the per-chunk embeddings the pipeline has retained since 6.21.15
+  and scores each one against every enrolled profile now, returning the top
+  matches and the margin between the best two. Until this route that store had
+  **no production reader at all**: the data was collected for weeks and never
+  looked at.
+
+  The margin is the point. It separates "missed by 0.02 against one profile"
+  from "equidistant between three" — a fixable near-miss versus a genuinely
+  ambiguous voice — and the review panel cannot tell those apart today. On a
+  face-mounted microphone that distinction is most of the available signal.
+
+  Read-only, and deliberately does NOT return the raw 192-float vectors: ~1 KB
+  of base64 per chunk that means nothing to a reader. Whole-session reads are
+  capped (default 50, max 400) because each chunk is scored against every
+  profile. A retained-but-absent chunk is reported in `missing` rather than
+  dropped, and `retained:false` stays distinguishable from "scored, no match".
+
 ## 6.21.22
 
 - The meeting Turbo preview is ON by default. `COS_WHISPER_MEETING_PREVIEW` is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.21.22",
+  "version": "6.21.23",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/lib/chunk-embedding-diagnostics.ts

@@ -0,0 +1,146 @@
+// Why a chunk was labelled the way it was.
+//
+// `chunk-embedding-store.ts` has retained a per-chunk embedding for every
+// identified chunk, for 14 days, since 6.21.15 — and until now NOTHING in
+// production read it back. The reader helpers existed; no route called them.
+//
+// That store is the difference between a reviewer (human or agent) who can only
+// restate what the panel already shows and one who can answer the question that
+// actually matters on a face-mounted microphone: not "who is this", but "how
+// close did we get, and to WHAT". A row that missed its match by 0.02 against
+// one profile is a very different problem from a row sitting equidistant
+// between three, and the panel cannot tell them apart today.
+//
+// WHAT THIS DELIBERATELY DOES NOT RETURN: the raw 192-float vectors. They are
+// ~1 KB of base64 each, they mean nothing to a reader, and a 400-chunk meeting
+// would be 400 KB of noise. Similarity against each enrolled profile is the
+// diagnostic; the vector is just how it is computed.
+
+import {
+  chunkEmbeddingsForIndices,
+  readChunkEmbeddings,
+  type ChunkEmbeddingRow,
+} from './chunk-embedding-store.js'
+import { rawCosineSimilarity, readVoiceProfiles } from './speaker-embeddings.js'
+
+/** Profiles scored per chunk. More than this is noise in a review context. */
+const TOP_MATCHES = 5
+
+export interface ProfileMatch {
+  speaker: string
+  /** Best cosine against any embedding held for that profile. */
+  similarity: number
+  /** How many embeddings that profile holds, so a strong score against a
+   *  1-sample profile is not read as equal to one against 20. */
+  embeddings: number
+}
+
+export interface ChunkDiagnostic {
+  chunk: number
+  /** The label the identifier chose live, which is what a correction corrects. */
+  chosen: string
+  /** The score it chose on, as recorded at capture time. */
+  chosenSimilarity: number
+  /** Best-scoring profiles NOW, recomputed against the current store — which
+   *  can differ from capture time if the profile has been trained since. */
+  matches: ProfileMatch[]
+  /** Gap between the top two current matches. A small margin means the choice
+   *  was nearly a coin flip, and that is invisible in the panel today. */
+  margin: number | null
+}
+
+export interface ChunkDiagnosticsResult {
+  sessionId: string
+  /** False when the store holds nothing for this session — aged out past the
+   *  14-day TTL, captured before 6.21.15, or embeddings disabled. Distinct from
+   *  an empty result set so a caller never reads "no data" as "no match". */
+  retained: boolean
+  chunks: ChunkDiagnostic[]
+  /** Chunks asked for that the store does not hold. */
+  missing: number[]
+  /** Profiles the scores were computed against, so a reader can see the
+   *  candidate pool rather than assume it. */
+  profileCount: number
+}
+
+/**
+ * Score one embedding against every enrolled profile.
+ *
+ * Best-of rather than mean: a profile holds up to 20 embeddings spanning
+ * different rooms and microphones, and averaging them buries the one recorded
+ * in conditions like these.
+ */
+function scoreAgainstProfiles(embedding: Float32Array): ProfileMatch[] {
+  const store = readVoiceProfiles()
+  const scored: ProfileMatch[] = []
+  for (const profile of store.profiles) {
+    let best = -1
+    for (const candidate of profile.embeddings) {
+      // The profile store holds plain number[]; the chunk store holds
+      // Float32Array. Convert at the boundary rather than widening the
+      // similarity function, which is on the live identification hot path.
+      const value = rawCosineSimilarity(embedding, new Float32Array(candidate))
+      if (value > best) best = value
+    }
+    if (best > -1) {
+      scored.push({
+        speaker: profile.name,
+        similarity: Number(best.toFixed(4)),
+        embeddings: profile.embeddings.length,
+      })
+    }
+  }
+  scored.sort((a, b) => b.similarity - a.similarity)
+  return scored.slice(0, TOP_MATCHES)
+}
+
+function toDiagnostic(row: ChunkEmbeddingRow): ChunkDiagnostic {
+  const matches = scoreAgainstProfiles(row.embedding)
+  return {
+    chunk: row.i,
+    chosen: row.speaker,
+    chosenSimilarity: Number(row.similarity.toFixed(4)),
+    matches,
+    margin: matches.length >= 2
+      ? Number((matches[0].similarity - matches[1].similarity).toFixed(4))
+      : null,
+  }
+}
+
+/**
+ * Diagnostics for specific chunks, or for the whole session when `indices` is
+ * empty.
+ *
+ * Bounded by `limit` because a long meeting holds hundreds of chunks and each
+ * one is scored against every profile — a whole-session call on a 400-chunk
+ * meeting with 77 profiles is 30,000 cosine comparisons. Fine to ask for, worth
+ * capping by default.
+ */
+export function chunkDiagnostics(
+  sessionId: string,
+  indices: number[] = [],
+  limit = 50,
+): ChunkDiagnosticsResult {
+  const profileCount = readVoiceProfiles().profiles.length
+
+  if (indices.length > 0) {
+    const rows = chunkEmbeddingsForIndices(sessionId, indices)
+    const found = new Set(rows.map(r => r.i))
+    return {
+      sessionId,
+      retained: readChunkEmbeddings(sessionId).rows.length > 0,
+      chunks: rows.slice(0, limit).map(toDiagnostic),
+      missing: indices.filter(i => !found.has(i)),
+      profileCount,
+    }
+  }
+
+  const all = readChunkEmbeddings(sessionId).rows
+  return {
+    sessionId,
+    retained: all.length > 0,
+    chunks: all.slice(0, limit).map(toDiagnostic),
+    missing: [],
+    profileCount,
+  }
+}

package/server/routes/meeting.ts

@@ -11,6 +11,8 @@ import { durableAtomicWriteFileSync } from '../lib/atomic-fs.js'
 import { appendCorrection, pendingCorrections } from '../lib/meeting-corrections.js'
 import { isSampleFromSession, untraceableSampleCount } from '../lib/training-audio-provenance.js'
 import { sendAudioFile } from '../lib/send-audio.js'
+import { chunkDiagnostics } from '../lib/chunk-embedding-diagnostics.js'
+import { errMsg } from '../lib/utils.js'
 import {
   extAudioChunkPath,
   listExtAudioChunks,
@@ -1182,6 +1184,57 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
     sendAudioFile(res, path)
   })
 
+  /**
+   * Why each chunk was labelled the way it was.
+   *
+   * Reads the per-chunk embeddings the pipeline has retained since 6.21.15 and
+   * scores each one against every enrolled profile RIGHT NOW. Until this route
+   * that store had no production reader at all — the data was collected and
+   * never looked at.
+   *
+   * This is what lets a reviewer distinguish "missed by 0.02 against one
+   * profile" from "equidistant between three", which is the difference between
+   * a fixable near-miss and a genuinely ambiguous voice. On a face-mounted
+   * microphone that distinction is most of the signal.
+   *
+   * Read-only. It scores and reports; it changes no profile and no meeting.
+   *
+   *   ?chunks=4,17,23   specific chunks (omit for the whole session)
+   *   ?limit=50         cap, because each chunk is scored against every profile
+   */
+  router.get('/meeting/:sessionId/embeddings', (req, res) => {
+    res.set('Cache-Control', 'private, no-store')
+    const sessionId = String(req.params.sessionId ?? '')
+    if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
+      res.status(400).json({ error: 'Invalid sessionId', reason: 'invalid_session_id' })
+      return
+    }
+
+    const rawChunks = String(req.query.chunks ?? '').trim()
+    const indices: number[] = []
+    if (rawChunks) {
+      for (const part of rawChunks.split(',')) {
+        const value = Number(part.trim())
+        // Reject the whole request rather than silently scoring a subset: a
+        // caller asking about chunk 17 must not get an answer about chunk 4.
+        if (!Number.isInteger(value) || value < 0) {
+          res.status(400).json({ error: `Invalid chunk index "${part.trim()}"`, reason: 'invalid_chunk_index' })
+          return
+        }
+        indices.push(value)
+      }
+    }
+
+    const rawLimit = Number(req.query.limit ?? 50)
+    const limit = Number.isInteger(rawLimit) && rawLimit > 0 ? Math.min(rawLimit, 400) : 50
+
+    try {
+      res.json(chunkDiagnostics(sessionId, indices, limit))
+    } catch (error) {
+      res.status(500).json({ error: errMsg(error), reason: 'diagnostics_failed' })
+    }
+  })
+
   /** What audio a meeting still has, so the panel can show play buttons only
    *  where they will work. */
   router.get('/meeting/:sessionId/audio', (req, res) => {