@gotcos/glasses-server 6.16.9 → 6.17.0

package/CHANGELOG.md

@@ -1,3 +1,34 @@
+## 6.17.0
+
+- **Speaker diarization is now a bolt-on any install can enable.** The ~26 MB
+  voiceprint model stays out of the npm package, but the loader no longer looks
+  only inside the package (where a managed install has no copy, and where a
+  hand-placed one is destroyed by the next update). Resolution order:
+  `COS_SPEAKER_MODEL_PATH` → `~/.cos-glasses/models/` → bundled `server/models/`
+  (source checkouts). Put the model in `~/.cos-glasses/models/` and restart.
+- **A bad model file can no longer take the server down.** onnxruntime aborts
+  the process on a malformed or mismatched model rather than throwing, which
+  under a KeepAlive LaunchAgent became a permanent restart loop that killed
+  queries, meetings, and transcription over an optional feature. The file is now
+  screened structurally and loaded in a throwaway child process first, so a
+  truncated download or an HTML error page saved as `.onnx` disables diarization
+  and nothing else.
+- **`/api/health` reports `speaker_id`** as `active` / `unavailable` / `error`,
+  so a server running without voiceprints is no longer indistinguishable from
+  one doing real diarization. `error` means a model is installed but the runtime
+  rejected it.
+- **Profile config is read from the data home.** `.cos-profile.json` is now
+  loaded from `~/.cos-glasses/` when present, ahead of the package-root copy.
+  Managed installs previously resolved it inside the generation directory, where
+  no profile exists and every field silently took its default — losing
+  transcription vocabulary, whisper corrections, and the wearer label.
+- **`/api/voice/enroll` and `/api/voice/status` no longer hardcode `MU`.** Both
+  use `owner_speaker_label` (default `Me`), matching what identification already
+  used. **Upgrade note:** an install that enrolled under the old default holds a
+  profile named `MU`; `/api/voice/status` will report `enrolled: false` until you
+  set `owner_speaker_label` to `MU`. Do that rather than re-enrolling, which
+  splits one voice across two profiles.
+
 ## 6.16.9
 
 - **Message-era reset visible to a running server.** `currentMessageEraState`

package/README.md

@@ -163,6 +163,55 @@ Telegram activity export is disabled by default even when a private COS
 pipeline contains `.telegram_config.json`; enable it only with the explicit
 `COS_TELEGRAM_NOTIFICATIONS=1` opt-in.
 
+## Speaker diarization (opt-in)
+
+Without a voiceprint model this server does not classify speakers at all — it
+passes through whatever label the client sends (`Unknown` when the client sends
+nothing; the COS companion sends its own wearer/`Ext` labels). Named
+per-speaker diarization needs a ~26 MB voiceprint model that is deliberately
+**not** shipped in the npm package, so it is a bolt-on:
+
+```bash
+mkdir -p ~/.cos-glasses/models
+# put 3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx there, then restart
+```
+
+The server searches, in order: `COS_SPEAKER_MODEL_PATH` (explicit full path to
+the `.onnx`), `~/.cos-glasses/models/`, then a bundled `server/models/` copy
+(source checkouts only). **Use the data home, not the installed package** —
+anything inside the package is destroyed by the next update, while
+`~/.cos-glasses/` survives.
+
+Use an **absolute** path if you set `COS_SPEAKER_MODEL_PATH` — a relative one
+resolves against the working directory, which under the managed LaunchAgent is
+the installed package.
+
+Verify with `/api/health` → `speaker_id`:
+
+| Value | Meaning |
+|---|---|
+| `active` | model loaded, diarization running |
+| `unavailable` | no model found — labels come from the client |
+| `error` | a model is present but the runtime rejected it (see the startup log) |
+
+The model is read once at startup, so restart after adding it. A corrupt or
+mismatched `.onnx` is screened and probed in a child process first, so a bad
+download disables diarization instead of taking the server down — but it does
+mean a wrong file fails silently apart from that log line.
+
+The wearer's label comes from `owner_speaker_label` in
+`~/.cos-glasses/.cos-profile.json` (default `Me`); set it to match the profile
+name you enrol under. Train voices via `/api/voice/enroll?name=…` (the default
+name is `owner_speaker_label`); profiles persist in
+`~/.cos-glasses/data/voice-profiles.json`.
+
+**Upgrading from an older server:** before this release the enrollment default
+was hardcoded to `MU`, so an existing install may hold a profile under that name
+while `owner_speaker_label` resolves to `Me`. `/api/voice/status` will then
+report `enrolled: false`. Set `owner_speaker_label` to `MU` to keep the existing
+voiceprints rather than re-enrolling, which would split the same voice across two
+profiles.
+
 ## HQ dictation
 
 Prompt dictation defaults to HQ. The phone owns the preference: **Fast mode

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.16.9",
-  "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
+  "version": "6.17.0",
+  "description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {
     "glasses-server": "bin/cli.cjs",

package/server/lib/profile.ts

@@ -1,16 +1,38 @@
 // Profile loader — reads user identity from .cos-profile.json (gitignored)
 // Falls back to generic defaults for users who haven't configured a profile
 
-import { readFileSync } from 'node:fs'
+import { existsSync, readFileSync } from 'node:fs'
+import { homedir } from 'node:os'
 import { resolve } from 'node:path'
 import { atomicWriteFileSync } from './atomic-fs.js'
 
 const APP_ROOT = resolve(import.meta.dirname, '../..')
+
+/** The profile in the data home. Survives updates; the APP_ROOT copy does not. */
+export function homeProfilePath(): string {
+  return resolve(homedir(), '.cos-glasses', '.cos-profile.json')
+}
+
 // Single canonical path — used by BOTH the reader and the writer so a glossary
 // PUT can never write to a different file than the cache reads from. Lazy +
 // env-overridable (COS_PROFILE_PATH) so tests can target a temp file.
+//
+// APP_ROOT is the INSTALLED PACKAGE root. For a managed install that is inside
+// the generation directory, which an update replaces wholesale — so a profile
+// there is destroyed on every upgrade, and a fresh managed install has no
+// profile at all (loadProfile() catches to {} and every field silently takes
+// its default). Relying on COS_PROFILE_PATH alone is not enough either: the
+// launcher rebuilds its environment from the release manifest rather than the
+// existing plist, so an operator-set value is dropped by the next
+// install/update/repair. Prefer the data home whenever a profile lives there.
+//
+// Order: explicit override → ~/.cos-glasses/.cos-profile.json → package root.
 function profilePath(): string {
-  return process.env.COS_PROFILE_PATH || resolve(APP_ROOT, '.cos-profile.json')
+  const override = process.env.COS_PROFILE_PATH?.trim()
+  if (override) return resolve(override)
+  const home = homeProfilePath()
+  if (existsSync(home)) return home
+  return resolve(APP_ROOT, '.cos-profile.json')
 }
 
 let profileCache: Record<string, unknown> | null = null

package/server/lib/speaker-embeddings.ts

@@ -8,18 +8,62 @@
 import { resolve } from 'node:path'
 import { errMsg } from './utils.js'
 import { getOwnerSpeakerLabel } from './profile.js'
-import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync } from 'node:fs'
+import { readFileSync, writeFileSync, existsSync, appendFileSync, mkdirSync, statSync, openSync, readSync, closeSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { spawnSync } from 'node:child_process'
 import { fileURLToPath } from 'node:url'
 
+/** A real voiceprint model is ~26 MB; anything this small is a bad download. */
+const MIN_MODEL_BYTES = 1_000_000
+const PROBE_TIMEOUT_MS = 30_000
+
 // sherpa-onnx-node is CJS — use createRequire for ESM compat
 import { createRequire } from 'node:module'
 const require = createRequire(import.meta.url)
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url))
 
-const MODEL_PATH = resolve(__dirname, '..', 'models',
-  '3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx')
 import { DATA_DIR } from './data-dir.js'
+
+export const SPEAKER_MODEL_FILENAME = '3dspeaker_speech_eres2net_sv_en_voxceleb_16k.onnx'
+
+/** Where the voiceprint model may live, in priority order:
+ *
+ *  1. COS_SPEAKER_MODEL_PATH — explicit override (full path to the .onnx).
+ *  2. ~/.cos-glasses/models/ — the bolt-on location. The model is ~26 MB and is
+ *     deliberately NOT in the npm tarball, so a managed install has no bundled
+ *     copy. The data home survives generation swaps; anything inside the
+ *     installed package does not, and a model dropped there is destroyed by the
+ *     next update.
+ *  3. server/models/         — bundled, which only exists in a source checkout.
+ *
+ *  Anchored on homedir() rather than DATA_DIR/'..' on purpose: path.resolve is
+ *  purely lexical, so deriving a sibling of a relocated COS_DATA_DIR could point
+ *  the "durable" candidate at an unwritable root, or — if COS_DATA_DIR were ever
+ *  set inside the package — collapse it back onto the very directory an update
+ *  destroys, silently reintroducing the bug this ordering exists to fix.
+ *
+ *  Diarization is opt-in by design: with no model the system stays on amplitude
+ *  fallback (wearer vs Ext) rather than failing. speakerModelState() exists so
+ *  that choice is VISIBLE in /api/health instead of silently degrading.
+ */
+export function speakerModelCandidates(): string[] {
+  const override = process.env.COS_SPEAKER_MODEL_PATH?.trim()
+  return [
+    ...(override ? [resolve(override)] : []),
+    resolve(homedir(), '.cos-glasses', 'models', SPEAKER_MODEL_FILENAME),
+    resolve(__dirname, '..', 'models', SPEAKER_MODEL_FILENAME),
+  ]
+}
+
+function resolveSpeakerModelPath(): string | null {
+  // isFile, not existsSync: a directory passes an existence check and would be
+  // handed to the native loader as a model.
+  return speakerModelCandidates().find(p => {
+    try { return statSync(p).isFile() } catch { return false }
+  }) ?? null
+}
+
 const PROFILES_PATH = resolve(DATA_DIR, 'voice-profiles.json')
 const CALIBRATION_LOG = resolve(DATA_DIR, 'speaker-calibration.jsonl')
 
@@ -58,14 +102,109 @@ interface ProfileStore {
   profiles: VoiceProfile[]
 }
 
+/** Cheap structural screen for a downloaded model.
+ *
+ *  Catches the common bad downloads — an HTML error/redirect page saved as
+ *  .onnx, or a truncated transfer — before the file reaches the native runtime.
+ *  ONNX is protobuf: field 1 (ir_version, varint) encodes as a leading 0x08.
+ *  This is a screen, not validation; probeModelSafely() is the real gate. */
+function looksLikeOnnxModel(path: string): boolean {
+  try {
+    if (statSync(path).size < MIN_MODEL_BYTES) return false
+    const head = Buffer.alloc(1)
+    const fd = openSync(path, 'r')
+    try { readSync(fd, head, 0, 1, 0) } finally { closeSync(fd) }
+    return head[0] === 0x08
+  } catch { return false }
+}
+
+/** Load the model in a throwaway child process first.
+ *
+ *  onnxruntime does not throw on a malformed or mismatched model — it calls
+ *  std::terminate, so the process dies with SIGABRT (or SIGSEGV for a valid-but-
+ *  wrong model). No try/catch can intercept that. In a managed install the
+ *  LaunchAgent has KeepAlive, so the death becomes a permanent restart loop that
+ *  takes down queries, meetings, and transcription — the whole server, over an
+ *  optional feature.
+ *
+ *  Absorbing that crash in a child keeps a bad file a diarization problem
+ *  instead of an outage. Cost is one short-lived process, once, and only when a
+ *  model is actually present. */
+function probeModelSafely(modelPath: string): { ok: true } | { ok: false; reason: string } {
+  const script =
+    "const{SpeakerEmbeddingExtractor}=require('sherpa-onnx-node');" +
+    "new SpeakerEmbeddingExtractor({model:process.argv[1],numThreads:1,provider:'cpu'});"
+  const probe = spawnSync(process.execPath, ['-e', script, modelPath], {
+    cwd: resolve(__dirname, '..', '..'), // package root, so sherpa-onnx-node resolves
+    timeout: PROBE_TIMEOUT_MS,
+    stdio: ['ignore', 'ignore', 'pipe'],
+  })
+  if (probe.signal) return { ok: false, reason: `native runtime aborted (${probe.signal})` }
+  if (probe.error) return { ok: false, reason: probe.error.message }
+  if (probe.status !== 0) {
+    const stderr = String(probe.stderr ?? '').trim().split('\n').pop() ?? ''
+    return { ok: false, reason: stderr || `probe exited ${probe.status}` }
+  }
+  return { ok: true }
+}
+
 /** Initialize speaker embedding system. Returns false if model missing (graceful degradation). */
 export function initSpeakerEmbeddings(): boolean {
   if (initialized) return extractor !== null
 
   initialized = true
 
-  if (!existsSync(MODEL_PATH)) {
-    console.log('[speaker] Model not found at', MODEL_PATH, '— embedding disabled, using amplitude fallback')
+  const override = process.env.COS_SPEAKER_MODEL_PATH?.trim()
+  if (override) {
+    const overridePath = resolve(override)
+    let usable = false
+    try { usable = statSync(overridePath).isFile() } catch { usable = false }
+    if (!usable) {
+      // Falling through silently would leave the operator believing an override
+      // they mistyped (or pointed at a directory) is in effect.
+      console.warn(
+        `[speaker] COS_SPEAKER_MODEL_PATH=${overridePath} is not a readable file — ignoring it`,
+        'and falling back to the remaining candidates.',
+      )
+    }
+    if (override !== overridePath) {
+      // Relative paths resolve against cwd, which under launchd is the installed
+      // package — a location the next update deletes.
+      console.warn(
+        `[speaker] COS_SPEAKER_MODEL_PATH is relative; resolved against the working directory to ${overridePath}.`,
+        'Use an absolute path so it cannot move with the process.',
+      )
+    }
+  }
+
+  const modelPath = resolveSpeakerModelPath()
+  if (!modelPath) {
+    const boltOn = resolve(homedir(), '.cos-glasses', 'models')
+    console.log(
+      '[speaker] Voiceprint model not found — embedding disabled, speaker labels come from the client.',
+      `Searched: ${speakerModelCandidates().join(', ')}.`,
+      // Name the durable directory outright. An ordinal ("the second path")
+      // shifts with the override and pointed users at the package copy, which
+      // the next update deletes.
+      `To enable diarization put ${SPEAKER_MODEL_FILENAME} in ${boltOn}/ and restart.`,
+    )
+    return false
+  }
+
+  if (!looksLikeOnnxModel(modelPath)) {
+    console.error(
+      `[speaker] ${modelPath} does not look like an ONNX model (too small, or not protobuf) —`,
+      'embedding disabled. A partial download or an HTML error page saved as .onnx does this.',
+    )
+    return false
+  }
+
+  const probe = probeModelSafely(modelPath)
+  if (!probe.ok) {
+    console.error(
+      `[speaker] ${modelPath} failed to load — embedding disabled. Reason: ${probe.reason}.`,
+      'The server is otherwise unaffected; replace the model file and restart.',
+    )
     return false
   }
 
@@ -73,7 +212,7 @@ export function initSpeakerEmbeddings(): boolean {
     sherpaOnnx = require('sherpa-onnx-node')
 
     extractor = new sherpaOnnx.SpeakerEmbeddingExtractor({
-      model: MODEL_PATH,
+      model: modelPath,
       numThreads: 2,
       provider: 'cpu',
     })
@@ -332,6 +471,30 @@ export function isEmbeddingAvailable(): boolean {
   return extractor !== null && manager !== null
 }
 
+/** Reported on /api/health so an amplitude fallback is never mistaken for real
+ *  diarization.
+ *
+ *  `state` is the RUNTIME truth (isEmbeddingAvailable), not "is a model file on
+ *  disk". Those diverge in both directions: a model deleted after a successful
+ *  load leaves diarization working from memory, and a model present alongside a
+ *  broken/ABI-mismatched sherpa-onnx never loads at all. `error` distinguishes
+ *  that second case — a model is installed but the runtime rejected it — from a
+ *  simply unconfigured install, which otherwise look identical to an operator.
+ *
+ *  Declared after the module state it reads: hoisting it above `extractor`
+ *  would make any import-time caller throw a ReferenceError on an
+ *  unauthenticated endpoint. */
+export function speakerModelState(): {
+  state: 'active' | 'unavailable' | 'error'
+  path: string | null
+  searched: string[]
+} {
+  const path = resolveSpeakerModelPath()
+  const running = isEmbeddingAvailable()
+  const state = running ? 'active' : (path && initialized ? 'error' : 'unavailable')
+  return { state, path, searched: speakerModelCandidates() }
+}
+
 /** Compute actual cosine similarity between two raw embedding vectors */
 export function rawCosineSimilarity(a: Float32Array, b: Float32Array): number {
   if (a.length !== b.length) return 0

package/server/routes/health.ts

@@ -7,6 +7,7 @@ import { serverMetrics } from '../lib/server-metrics.js'
 import { getServerInstanceId } from '../lib/server-instance-id.js'
 import { localFirstMeetingsCapability } from '../lib/local-first-meetings-contract.js'
 import { isSileroAvailable } from '../lib/vad-silero.js'
+import { speakerModelState } from '../lib/speaker-embeddings.js'
 import { getAvailableCliSessionId } from '../lib/claude-bridge.js'
 import {
   isWhisperLocalAvailable,
@@ -176,6 +177,12 @@ healthRouter.get('/health', async (_req, res) => {
 
   checks.silero_vad = isSileroAvailable() ? 'active' : 'disabled'
 
+  // Speaker diarization is opt-in (the ~26 MB voiceprint model ships outside the
+  // npm tarball), so publish its state rather than letting the amplitude
+  // fallback masquerade as working diarization. Availability only — the resolved
+  // path is a local filesystem detail and health is unauthenticated.
+  checks.speaker_id = speakerModelState().state
+
   // Health is unauthenticated. Publish only availability; the actual CLI
   // session id is a resumable runtime handle and belongs on authenticated
   // query/debug surfaces.

package/server/routes/voice.ts

@@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url'
 import { enrollSpeaker, isEnrolled, getAllSpeakerNames, identifySpeaker, extractEmbedding, enrollEmbedding, rawCosineSimilarity, getEmbeddingCount } from '../lib/speaker-embeddings.js'
 import { statSync } from 'node:fs'
 import { trainFromFireflies, getTrainingStatus } from '../lib/speaker-trainer.js'
+import { getOwnerSpeakerLabel } from '../lib/profile.js'
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url))
 const AUDIO_SAVE_DIR = resolve(__dirname, '..', 'data', 'training-audio')
@@ -18,7 +19,10 @@ export const voiceRouter = Router()
 // POST /api/voice/enroll — accept WAV audio, extract embedding, store as profile
 voiceRouter.post('/voice/enroll', async (req, res) => {
   try {
-    const name = (req.query.name as string) || 'MU'
+    // Default to the configured wearer label ('Me' unless owner_speaker_label
+    // is set). Hardcoding one user's initials here enrolled every other install
+    // under a stranger's name.
+    const name = (req.query.name as string) || getOwnerSpeakerLabel()
 
     // Collect raw audio body
     const buffers: Buffer[] = []
@@ -38,10 +42,12 @@ voiceRouter.post('/voice/enroll', async (req, res) => {
   }
 })
 
-// GET /api/voice/status — is MU enrolled?
+// GET /api/voice/status — is the wearer enrolled?
 voiceRouter.get('/voice/status', (_req, res) => {
+  const owner = getOwnerSpeakerLabel()
   res.json({
-    enrolled: isEnrolled('MU'),
+    owner,
+    enrolled: isEnrolled(owner),
     speakers: getAllSpeakerNames(),
   })
 })