@gotcos/glasses-server 6.21.22 → 6.21.24

package/CHANGELOG.md

@@ -1,3 +1,50 @@
+## 6.21.24
+
+- A leaked recording session can no longer block every restart. The maintenance
+  drain gate counted `sessions.size`, so a session whose phone dropped
+  mid-recording without sending a close pinned that count at 1 indefinitely and
+  Install, Repair, Restart and Update Server each drained it, timed out after
+  60s, and failed — with no user-visible reason. Hit twice on 2026-08-06
+  (6.21.20 and 6.21.22); the second phantom had been silent 54 minutes and the
+  only way through was to finalize it as a real meeting, which it was not.
+
+  The gate now counts only sessions active within the last 30 minutes. A stale
+  one is reported in status as `staleTranscriptionSessions` with its session id,
+  silent duration and chunk count, so a blocked operator can see the cause —
+  and `/api/meeting/orphans` reporting 0 can no longer coexist silently with a
+  held lock, since the two read different stores.
+
+  **Nothing is reaped.** The session, its chunks and its recoverability are
+  untouched; only its claim on the restart gate expires. The 30-minute window
+  sits far above the reasons a live recording legitimately goes quiet (a
+  backgrounded phone buffering to IndexedDB, a network drop) and far below the
+  4h durable-chunk retention.
+
+  Note for anyone reading the health payload: `oldestWorkStartedAt: null` does
+  NOT indicate a leak. `recording_session` reaches the gate through
+  `extraActiveByKind`, which never contributes a timestamp, so every recording
+  session reports null — healthy or not. Only `lastActivityAt` separates them.
+
+## 6.21.23
+
+- `GET /meeting/:sessionId/embeddings` — why each chunk was labelled the way it
+  was. Reads the per-chunk embeddings the pipeline has retained since 6.21.15
+  and scores each one against every enrolled profile now, returning the top
+  matches and the margin between the best two. Until this route that store had
+  **no production reader at all**: the data was collected for weeks and never
+  looked at.
+
+  The margin is the point. It separates "missed by 0.02 against one profile"
+  from "equidistant between three" — a fixable near-miss versus a genuinely
+  ambiguous voice — and the review panel cannot tell those apart today. On a
+  face-mounted microphone that distinction is most of the available signal.
+
+  Read-only, and deliberately does NOT return the raw 192-float vectors: ~1 KB
+  of base64 per chunk that means nothing to a reader. Whole-session reads are
+  capped (default 50, max 400) because each chunk is scored against every
+  profile. A retained-but-absent chunk is reported in `missing` rather than
+  dropped, and `retained:false` stays distinguishable from "scored, no match".
+
 ## 6.21.22
 
 - The meeting Turbo preview is ON by default. `COS_WHISPER_MEETING_PREVIEW` is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.21.22",
+  "version": "6.21.24",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/lib/chunk-embedding-diagnostics.ts

@@ -0,0 +1,146 @@
+// Why a chunk was labelled the way it was.
+//
+// `chunk-embedding-store.ts` has retained a per-chunk embedding for every
+// identified chunk, for 14 days, since 6.21.15 — and until now NOTHING in
+// production read it back. The reader helpers existed; no route called them.
+//
+// That store is the difference between a reviewer (human or agent) who can only
+// restate what the panel already shows and one who can answer the question that
+// actually matters on a face-mounted microphone: not "who is this", but "how
+// close did we get, and to WHAT". A row that missed its match by 0.02 against
+// one profile is a very different problem from a row sitting equidistant
+// between three, and the panel cannot tell them apart today.
+//
+// WHAT THIS DELIBERATELY DOES NOT RETURN: the raw 192-float vectors. They are
+// ~1 KB of base64 each, they mean nothing to a reader, and a 400-chunk meeting
+// would be 400 KB of noise. Similarity against each enrolled profile is the
+// diagnostic; the vector is just how it is computed.
+
+import {
+  chunkEmbeddingsForIndices,
+  readChunkEmbeddings,
+  type ChunkEmbeddingRow,
+} from './chunk-embedding-store.js'
+import { rawCosineSimilarity, readVoiceProfiles } from './speaker-embeddings.js'
+
+/** Profiles scored per chunk. More than this is noise in a review context. */
+const TOP_MATCHES = 5
+
+export interface ProfileMatch {
+  speaker: string
+  /** Best cosine against any embedding held for that profile. */
+  similarity: number
+  /** How many embeddings that profile holds, so a strong score against a
+   *  1-sample profile is not read as equal to one against 20. */
+  embeddings: number
+}
+
+export interface ChunkDiagnostic {
+  chunk: number
+  /** The label the identifier chose live, which is what a correction corrects. */
+  chosen: string
+  /** The score it chose on, as recorded at capture time. */
+  chosenSimilarity: number
+  /** Best-scoring profiles NOW, recomputed against the current store — which
+   *  can differ from capture time if the profile has been trained since. */
+  matches: ProfileMatch[]
+  /** Gap between the top two current matches. A small margin means the choice
+   *  was nearly a coin flip, and that is invisible in the panel today. */
+  margin: number | null
+}
+
+export interface ChunkDiagnosticsResult {
+  sessionId: string
+  /** False when the store holds nothing for this session — aged out past the
+   *  14-day TTL, captured before 6.21.15, or embeddings disabled. Distinct from
+   *  an empty result set so a caller never reads "no data" as "no match". */
+  retained: boolean
+  chunks: ChunkDiagnostic[]
+  /** Chunks asked for that the store does not hold. */
+  missing: number[]
+  /** Profiles the scores were computed against, so a reader can see the
+   *  candidate pool rather than assume it. */
+  profileCount: number
+}
+
+/**
+ * Score one embedding against every enrolled profile.
+ *
+ * Best-of rather than mean: a profile holds up to 20 embeddings spanning
+ * different rooms and microphones, and averaging them buries the one recorded
+ * in conditions like these.
+ */
+function scoreAgainstProfiles(embedding: Float32Array): ProfileMatch[] {
+  const store = readVoiceProfiles()
+  const scored: ProfileMatch[] = []
+  for (const profile of store.profiles) {
+    let best = -1
+    for (const candidate of profile.embeddings) {
+      // The profile store holds plain number[]; the chunk store holds
+      // Float32Array. Convert at the boundary rather than widening the
+      // similarity function, which is on the live identification hot path.
+      const value = rawCosineSimilarity(embedding, new Float32Array(candidate))
+      if (value > best) best = value
+    }
+    if (best > -1) {
+      scored.push({
+        speaker: profile.name,
+        similarity: Number(best.toFixed(4)),
+        embeddings: profile.embeddings.length,
+      })
+    }
+  }
+  scored.sort((a, b) => b.similarity - a.similarity)
+  return scored.slice(0, TOP_MATCHES)
+}
+
+function toDiagnostic(row: ChunkEmbeddingRow): ChunkDiagnostic {
+  const matches = scoreAgainstProfiles(row.embedding)
+  return {
+    chunk: row.i,
+    chosen: row.speaker,
+    chosenSimilarity: Number(row.similarity.toFixed(4)),
+    matches,
+    margin: matches.length >= 2
+      ? Number((matches[0].similarity - matches[1].similarity).toFixed(4))
+      : null,
+  }
+}
+
+/**
+ * Diagnostics for specific chunks, or for the whole session when `indices` is
+ * empty.
+ *
+ * Bounded by `limit` because a long meeting holds hundreds of chunks and each
+ * one is scored against every profile — a whole-session call on a 400-chunk
+ * meeting with 77 profiles is 30,000 cosine comparisons. Fine to ask for, worth
+ * capping by default.
+ */
+export function chunkDiagnostics(
+  sessionId: string,
+  indices: number[] = [],
+  limit = 50,
+): ChunkDiagnosticsResult {
+  const profileCount = readVoiceProfiles().profiles.length
+
+  if (indices.length > 0) {
+    const rows = chunkEmbeddingsForIndices(sessionId, indices)
+    const found = new Set(rows.map(r => r.i))
+    return {
+      sessionId,
+      retained: readChunkEmbeddings(sessionId).rows.length > 0,
+      chunks: rows.slice(0, limit).map(toDiagnostic),
+      missing: indices.filter(i => !found.has(i)),
+      profileCount,
+    }
+  }
+
+  const all = readChunkEmbeddings(sessionId).rows
+  return {
+    sessionId,
+    retained: all.length > 0,
+    chunks: all.slice(0, limit).map(toDiagnostic),
+    missing: [],
+    profileCount,
+  }
+}

package/server/routes/maintenance.ts

@@ -15,7 +15,7 @@ import { getQueryJobRuntimeHealth } from '../lib/query-job-runtime.js'
 import { serverMetrics } from '../lib/server-metrics.js'
 import { getServerInstanceId } from '../lib/server-instance-id.js'
 import { getWhisperHealth } from '../lib/whisper-local.js'
-import { getActiveTranscriptionSessionCount } from './transcribe-stream.js'
+import { getActiveTranscriptionSessionCount, getTranscriptionSessionLiveness } from './transcribe-stream.js'
 
 export const maintenanceRouter = Router()
 
@@ -71,15 +71,21 @@ function operationCredentials(req: Request): MaintenanceOperationCredentials {
 function statusSnapshot(credentials: MaintenanceOperationCredentials = {}) {
   const jobs = getQueryJobRuntimeHealth()
   const activeTranscriptionSessions = getActiveTranscriptionSessionCount()
+  // Only sessions still plausibly recording may hold the drain gate. A leaked
+  // session — phone dropped mid-recording, no close ever sent — otherwise pins
+  // this at 1 forever and every Install/Repair/Restart/Update drains, times out
+  // and fails with nothing to show the user. The stale ones are reported below
+  // rather than counted, and are never deleted here.
+  const sessionLiveness = getTranscriptionSessionLiveness()
   const managed = managedRuntimeCapability()
   const tracked = maintenanceLifecycle.snapshot(credentials, {
-    recording_session: activeTranscriptionSessions,
+    recording_session: sessionLiveness.live,
   })
   const untrackedDurableRuns = Math.max(0, jobs.activeRuns - (tracked.activeByKind.durable_query ?? 0))
   const lifecycle = untrackedDurableRuns > 0
     ? maintenanceLifecycle.snapshot(credentials, {
       durable_query_runtime: untrackedDurableRuns,
-      recording_session: activeTranscriptionSessions,
+      recording_session: sessionLiveness.live,
     })
     : tracked
   return {
@@ -91,6 +97,13 @@ function statusSnapshot(credentials: MaintenanceOperationCredentials = {}) {
     bootId: serverMetrics.bootId,
     activeJobs: jobs.activeRuns,
     activeTranscriptionSessions,
+    // Split out so a blocked operator can SEE why, and so "0 orphans" can never
+    // again coexist with a held lock: the orphans endpoint reads the quarantine
+    // directory, this reads the in-memory session map, and they are different
+    // stores. A stale session is surfaced here and blocks nothing.
+    liveTranscriptionSessions: sessionLiveness.live,
+    staleTranscriptionSessions: sessionLiveness.stale,
+    staleTranscriptionSessionDetail: sessionLiveness.staleSessions,
     shuttingDown: jobs.shuttingDown,
     durableStoreState: jobs.store.state,
     lifecycle,

package/server/routes/meeting.ts

@@ -11,6 +11,8 @@ import { durableAtomicWriteFileSync } from '../lib/atomic-fs.js'
 import { appendCorrection, pendingCorrections } from '../lib/meeting-corrections.js'
 import { isSampleFromSession, untraceableSampleCount } from '../lib/training-audio-provenance.js'
 import { sendAudioFile } from '../lib/send-audio.js'
+import { chunkDiagnostics } from '../lib/chunk-embedding-diagnostics.js'
+import { errMsg } from '../lib/utils.js'
 import {
   extAudioChunkPath,
   listExtAudioChunks,
@@ -1182,6 +1184,57 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
     sendAudioFile(res, path)
   })
 
+  /**
+   * Why each chunk was labelled the way it was.
+   *
+   * Reads the per-chunk embeddings the pipeline has retained since 6.21.15 and
+   * scores each one against every enrolled profile RIGHT NOW. Until this route
+   * that store had no production reader at all — the data was collected and
+   * never looked at.
+   *
+   * This is what lets a reviewer distinguish "missed by 0.02 against one
+   * profile" from "equidistant between three", which is the difference between
+   * a fixable near-miss and a genuinely ambiguous voice. On a face-mounted
+   * microphone that distinction is most of the signal.
+   *
+   * Read-only. It scores and reports; it changes no profile and no meeting.
+   *
+   *   ?chunks=4,17,23   specific chunks (omit for the whole session)
+   *   ?limit=50         cap, because each chunk is scored against every profile
+   */
+  router.get('/meeting/:sessionId/embeddings', (req, res) => {
+    res.set('Cache-Control', 'private, no-store')
+    const sessionId = String(req.params.sessionId ?? '')
+    if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
+      res.status(400).json({ error: 'Invalid sessionId', reason: 'invalid_session_id' })
+      return
+    }
+
+    const rawChunks = String(req.query.chunks ?? '').trim()
+    const indices: number[] = []
+    if (rawChunks) {
+      for (const part of rawChunks.split(',')) {
+        const value = Number(part.trim())
+        // Reject the whole request rather than silently scoring a subset: a
+        // caller asking about chunk 17 must not get an answer about chunk 4.
+        if (!Number.isInteger(value) || value < 0) {
+          res.status(400).json({ error: `Invalid chunk index "${part.trim()}"`, reason: 'invalid_chunk_index' })
+          return
+        }
+        indices.push(value)
+      }
+    }
+
+    const rawLimit = Number(req.query.limit ?? 50)
+    const limit = Number.isInteger(rawLimit) && rawLimit > 0 ? Math.min(rawLimit, 400) : 50
+
+    try {
+      res.json(chunkDiagnostics(sessionId, indices, limit))
+    } catch (error) {
+      res.status(500).json({ error: errMsg(error), reason: 'diagnostics_failed' })
+    }
+  })
+
   /** What audio a meeting still has, so the panel can show play buttons only
    *  where they will work. */
   router.get('/meeting/:sessionId/audio', (req, res) => {

package/server/routes/transcribe-stream.ts

@@ -342,6 +342,76 @@ export function getActiveTranscriptionSessionCount(): number {
   return sessions.size
 }
 
+/**
+ * How long a session may go silent before it stops BLOCKING a restart.
+ *
+ * Not a retention policy and not a reap: the session, its chunks and its
+ * recoverability are untouched. This decides one thing — whether it still
+ * counts toward the maintenance drain gate.
+ *
+ * Sized against the reasons a LIVE recording legitimately goes quiet: the phone
+ * backgrounded and buffering to IndexedDB, a network drop, a long pause. Those
+ * run to minutes. It sits deliberately far below
+ * LOCAL_FIRST_MEETING_IDLE_RETENTION_MS (4h), which governs how long chunks stay
+ * recoverable and is far too long to hold a restart on.
+ */
+export const RECORDING_SESSION_STALE_MS = 30 * 60 * 1000
+
+export interface TranscriptionSessionLiveness {
+  /** Sessions still plausibly recording. These block a restart. */
+  live: number
+  /** Sessions silent past the threshold. Surfaced, NOT counted, NOT deleted. */
+  stale: number
+  /** Which ones, least-silent first, so an operator can act on a name. */
+  staleSessions: Array<{ sessionId: string; silentForMs: number; chunks: number }>
+}
+
+/**
+ * Split active sessions into live and stale.
+ *
+ * WHY THIS EXISTS. The maintenance gate counts `sessions.size` and blocks every
+ * restart while it is non-zero. A session that leaks — the phone drops
+ * mid-recording and never sends a close — pins that count at 1 indefinitely, so
+ * Install, Repair, Restart and Update Server all drain, time out and fail with
+ * no user-visible reason. Observed twice on 2026-08-06 (servers 6.21.20 and
+ * 6.21.22); the second phantom had been silent 54 minutes and the only way
+ * through was to finalize it as a real meeting, which it was not.
+ *
+ * THE DISCRIMINATOR IS `lastActivityAt`, NOT `oldestWorkStartedAt`. The obvious
+ * reading — "activeTotal >= 1 with oldestWorkStartedAt null means leaked" — is
+ * WRONG and would strand healthy recordings. `recording_session` reaches the
+ * gate through `extraActiveByKind` (routes/maintenance.ts), and the snapshot's
+ * `oldestStartedAtMs` loop walks only tracked `work` entries, so an extra count
+ * never contributes a timestamp. EVERY recording session therefore reports
+ * `oldestWorkStartedAt: null`, healthy or leaked. Only the session's own
+ * `lastActivityAt` tells them apart.
+ */
+/**
+ * The live session map, for tests only.
+ *
+ * Exposed because the liveness split is pure logic over `lastActivityAt` but
+ * the map is module-private, and a test that cannot seed it can only assert the
+ * empty case. An earlier draft of the liveness test guarded on this symbol
+ * existing and silently no-opped four of its six cases — passing green while
+ * exercising nothing. Never guard a test on its own seam; make the seam real.
+ */
+export const __sessionsForTests = sessions
+
+export function getTranscriptionSessionLiveness(now = Date.now()): TranscriptionSessionLiveness {
+  let live = 0
+  const staleSessions: TranscriptionSessionLiveness['staleSessions'] = []
+  for (const [sessionId, session] of sessions) {
+    const silentForMs = now - session.lastActivityAt
+    if (silentForMs >= RECORDING_SESSION_STALE_MS) {
+      staleSessions.push({ sessionId, silentForMs, chunks: session.chunks.length })
+    } else {
+      live++
+    }
+  }
+  staleSessions.sort((a, b) => a.silentForMs - b.silentForMs)
+  return { live, stale: staleSessions.length, staleSessions }
+}
+
 // Incremental chunk persistence — survive server restarts
 const CHUNK_PERSIST_DIR = dataPath('active-sessions')
 ensurePrivateDirectory(CHUNK_PERSIST_DIR)