@gotcos/glasses-server 6.60.0 → 6.61.1

package/.env.example

@@ -159,6 +159,14 @@ BIND_HOST=0.0.0.0
 # changing committed transcription or meeting audio.
 # COS_WHISPER_PREVIEW_MODEL=auto    # auto | small.en | turbo | off
 
+# Who spoke when (6.61.0). Nemotron 3 Diarization separates the voices in each
+# chunk and at save; the voiceprint names them. It needs FluidAudio's
+# fluidaudiocli and its models ON THIS MAC: the server always passes --models
+# and never lets the CLI download. Missing either, labels stay on the voiceprint.
+# COS_DIARIZER=nemotron            # nemotron | embedding (voiceprint only, the 6.60 labels)
+# COS_NEMOTRON_CLI=~/.cos-glasses/bin/fluidaudiocli
+# COS_NEMOTRON_MODELS=~/.cos-glasses/models/nemotron-3-diarization   # holds Nemotron3Diarizer_fast32.mlmodelc + learnable_sil_emb.bin
+
 # Spoken reply playback defaults to local Kokoro on Apple silicon Macs. The
 # first run creates a private venv and downloads the model. Local mode fails
 # closed: it never sends text to OpenAI. Set openai_primary only when cloud TTS

package/CHANGELOG.md

@@ -1,3 +1,54 @@
+## 6.61.1
+
+### Fixes 6.61.0 labelling other people as the wearer: a split track is named only by a confident voiceprint, never by the glasses' default label
+
+- Miles, 2026-10-02 10:19, in a live meeting on 6.61.0: "MU is being assigned to loudest speaker vs my actual voice signature ... I saw Jeremy Sokolic voice showing up as MU." In that session (`meeting_1790953219564_yhwyxs`, read from a copy), 40 turns read MU. 25 were genuine (voiceprint at 0.65 or more, Even Hub `self`). 11 had similarity 0 in two-track chunks where Even Hub heard mostly someone else ("Okay.", "I can send a message. Yeah.", "to have as we play into calendar."): a short second track took the client's label, and the client's label is the glasses' default wearer label (`MU`), not evidence of who spoke. 2 more ("Of course. All right." at 0.612, and 0.625) were a track's own voiceprint check returning the wearer from the full search, which accepts any profile from 0.55, below the 0.65 at which the voiceprint verifies the wearer.
+- **The client's label never names a track.** In a split chunk, a short track that is not the dominant one is `Ext`. The client's label is still what 6.60.0 used it for: a whole chunk no voiceprint could run on (under 2 s, or no voiceprint model). A one-track chunk is labelled exactly as 6.60.0 labelled the chunk.
+- **The wearer needs owner-verify confidence in a split chunk.** A track named the wearer (`owner_speaker_label`) by its own voiceprint check, or a short dominant track that would take the whole chunk's voiceprint, keeps the wearer's name only at `OWNER_VERIFY_SIMILARITY` (0.65, now one constant shared with the voiceprint's owner verify); between 0.55 and 0.65 it is `Ext`. Any other name keeps the search threshold, as before. The two 0.665 and 0.716 turns where Even Hub heard mostly someone else are above the floor and stay, as reviewed.
+- **The final pass applies the same rule.** Its channel map was already built only from chunks at 0.65 or more. Its per-chunk fallback (the live turn a word sat in, else the chunk's label) now treats a wearer name in a split chunk below 0.65 as `Ext`, so a 6.61.0 turn like these is never carried into a final transcript.
+- **Logged and counted.** Every chunk where a would-be wearer name was refused logs one line, `[diarizer] owner-guard chunk #N refused=K client_label=X weak_owner=Y`; the chunk's `diarizer.ownerGuard` lists the reasons, and `readiness.diarizer.ownerGuard` counts them since start.
+- **Replayed on the field session (read-only copies of its 195 Nemotron chunks and their audio, the real CLI and the real voiceprint):** all 28 genuine wearer turns (0.65 or more, Even Hub `self`) stay MU; 0 of the 11 similarity-0 MU turns and 0 of the 2 weak ones (0.612, 0.625) are MU any more; the 2 borderline turns stay MU; the guard refused 13 client labels and 2 weak wearer matches; no chunk fell back.
+- Tests: the field chunks rebuilt from their records ("Okay." with Even Hub self 19/121, "I can send a message. Yeah." at self 3/125 with Kyle keeping his own track, "Of course. All right." at 0.612), the wearer staying the wearer in a one-track chunk (0.703) and a split one (0.716), a weak wearer match refused on both paths (the whole-chunk voiceprint a short dominant turn takes, and a track's own check), the client label never naming a track, the owner-guard log line and count, the route passing `owner_speaker_label`, and the final pass refusing a 6.61.0 similarity-0 MU turn while keeping a confident one. The mutation gate adds a case for each guard and retargets the three 6.61.0 cases the guard replaced: baseline 169 tests in 11 files, 104 of 104 killed. Full suite 5,723 passed, 2 skipped.
+- No Install hooks and no COS Control change.
+
+## 6.61.0
+
+### Nemotron is the default diarizer, for real: it splits two-voice chunks on the HUD during the call and in the final transcript, and the voiceprint names each voice
+
+- Miles, 2026-10-02 07:10: "I want nemotron to become the default as it's more accurate. We should see it labeling in the HUD (During the call) and in the final transcript." This folds in the unpublished 6.60.1, which made `COS_DIARIZER` default to `nemotron` but never ran Nemotron: every live chunk was still labelled by the whole-chunk voiceprint, and its `[diarizer] tracks=nemotron` log line was false. That line is gone; every `[diarizer]` line now says what ran.
+- **What it does, and what it does not claim.** Nemotron separates the voices inside a chunk and across a meeting; it names nobody. The voiceprint still decides every name, with its thresholds unchanged. What changes is that a chunk holding two voices is no longer given one label, and a stretch the voiceprint could not name can take the name of the voice Nemotron heard there. This release does not claim the labels are more accurate: the replay below measures how rarely Nemotron disagrees with labels the voiceprint was already sure of, which is stability, not accuracy (see the disclosure there).
+- The evidence it builds on is the read-only canary of 2026-10-01 (`operations/personal/wk40_2026/nemotron3_canary/`): FluidAudio v0.17.4, preset `fast32`, about 155x realtime warm; on chunks the voiceprint is sure of, Nemotron's dominant channel maps back to the same name in 29 of 29 conclusive meetings (mean 0.959); 45.6% of chunks hold two voices the voiceprint labels as one. The canary's alignment rules are reused as written: chunkEntries joined to `chunk_NNNN.wav` by chunkIndex with `audioSha256` checked, each WAV at its real `elapsed` with only the overlapping tail trimmed, the raw 10 ms probabilities scored at 0.5, 8 channels.
+- **Live, on the HUD.** Each chunk runs `fluidaudiocli nemotron3-diarize` on the chunk WAV the server has just written, started before the voiceprint and Whisper so it runs in their shadow, on the Neural Engine (`--compute-units ane`, so the GPU stays Whisper's; its probabilities matched `all` frame for frame at 0.5 on real chunks) at a lower CPU priority. A channel active for at least 0.5 s is a track. Whisper's word clocks place the words on the tracks when the ASR gives them (the iPhone path); the live whisper-server path returns compact JSON without clocks (its verbose JSON can crash the native server), so there the text is laid over the frames where Nemotron hears speech in proportion to word length, a turn change moves to the nearest phrase end within two words, and a one-word flicker inside another voice's turn is absorbed. The voiceprint names each track: one track takes the chunk's voiceprint; with two or more, a track with at least 2 s of its own audio (the same floor as a whole chunk) is named by `identifySpeaker` on that audio, its calibration rows tagged `nemotron-track`; a shorter one takes the chunk's voiceprint when it is the dominant voice, otherwise the client's label when that is a real name different from the dominant one, otherwise `Ext`. A name is never invented.
+- **The chunk response (the contract with COS Glasses 6.9.567).** `speaker` stays and is now the dominant voice's name (by voiced time among the tracks that carry the words; two channels can carry one person, and a louder voice with no words in the chunk is not who said it), `text` is unchanged, and an optional `segments: [{ speaker, text, startSec, endSec, similarity }]` lists the turns in time order; their texts joined with one space equal `text`. A replayed upload of a canonical chunk returns the same segments, and the `transcript_chunk` display event carries them. The segments are stored on the chunk, with `diarizer: { engine, tracks, timing, ms }` and, when the dominant name differs from the whole-chunk voiceprint, `voiceprintSpeaker` and `voiceprintSimilarity`. The saved sidecar's `speakers` counts segment speakers too. The whole-chunk voiceprint still runs first and still banks the chunk embedding, auto-enrols and saves training and Ext audio exactly as before. When boot recovery strips a hallucination from a stored chunk, its turns are refitted to the new text, or dropped when the words no longer line up, so they always join to the text.
+- **The fallback-experience rule.** Live never stalls or loses a chunk; every fallback keeps today's voiceprint label and gives no `segments`.
+  - The model is warmed in the background at server start (a silent second; the first Neural Engine compile took 15 to 35 s); until then chunks are `warming`. A warm-up tries three times (5 s and 30 s apart); a temp directory it cannot make is a failed attempt with a reason, never a warm-up stuck on `warming`.
+  - Each run has a 1.5 s budget and is killed with SIGKILL at the deadline. At most two run at once; a chunk that finds both busy is `busy`, never queued.
+  - **The breaker.** Three failed runs in a row (budget kills or crashes) stop spawning: every chunk takes the voiceprint at once, with no 1.5 s wait, and the server logs `[diarizer] breaker open: ...`. After a rest of 2 minutes one fresh warm-up decides; each further trip doubles the rest (2, 4, 8, 16, 32, then 60 minutes at most), and a clean chunk resets it. A warm-up that gives up rests on the same schedule; a missing CLI or missing models are re-checked every 2 minutes without spawning anything.
+  - A missing CLI or missing models (any install without FluidAudio) is logged once at warm-up, and chunks carry no per-chunk `diarizer` stamp. Run failures (timeout, crash, unreadable output, busy) are logged per chunk with their reason; state reasons (`warming`, `cooling`, `warmup_failed`) are counted per chunk and logged once per change of state.
+- **The final transcript.** When a meeting is saved, the finalization job runs one Nemotron pass over the whole meeting after the HQ batch is persisted, while the raw chunk audio still exists, and before the job reaches the ops handoff. One run over the timeline keeps the channels consistent for the whole call. A channel takes a name only from chunks the voiceprint was sure of (similarity at least 0.65, the review panel's own floor, a real name, one-sided by Even Hub, or by Nemotron itself when a chunk has no Even Hub histogram), with at least 3 such chunks and 80% of them agreeing; several channels may map to one person, and a channel two people share stays unmapped. Following the canary's retro plan, a room with more than four identities or all eight channels in use (a channel counts with half a second of speech, not a blip) gets no meeting-wide map. Every word then takes its channel's name, or keeps its per-chunk name (the live Nemotron turn it sat in, else its existing label). The timeline is built asynchronously in steps, so a long meeting does not freeze live uploads; a timeline far longer than the audio it holds (more than 1.5 times, plus 10 minutes) is a bad clock and the pass is skipped as `too_long` before anything is allocated.
+- **Only labels change.** The saved transcript is relabelled in place, word by word, against the relabelled speaker words (a 12-word window, then a three-word anchor past anything cleaning removed). A line splits where its speaker changes and is never merged with its neighbour, so a pass that changes no label reproduces the transcript byte for byte, writes nothing and moves no revision (`no_change`). Nothing is written unless the result has exactly the same words in the same order. Each changed batch word keeps `originalSpeaker`, a changed streaming chunk keeps `originalSpeaker` and gains `finalSegments`, and the sidecar keeps the whole pre-relabel transcript under `diarization.originalTranscript` with the channel map and its evidence. If the markdown write fails the sidecar is put back byte for byte; if even that fails the pass reports `sidecar_restore_failed` and says so in the log.
+- **A person's edit always wins.** A meeting with a human correction (`correctionRevision` above 0) is never relabelled. The pass re-reads both files just before writing; if a relabel, a deattribution or a summary landed during its CLI run, it stands down (`changed_during_pass`) and writes nothing. Relabel and deattribute answer 409 `meeting_finalizing` (retryable) while a finalization job exists for the meeting, as held naming already did. A second pass over an applied meeting is a no-op. If the pass fails in any way the meeting stays exactly as saved, and the job goes on.
+- **Voice profiles stay safe.** Since a chunk's label can now be its dominant turn's name, naming a voice no longer enrols a chunk whose own whole-chunk voiceprint named a third person, or a chunk Nemotron split into more than one turn; the response says how many it refused (`otherVoiceSkipped`, present only when it happened). A wrong existing label is still how a new person first appears (the Milo case).
+- **When COS sync sees it.** The relabel completes before the meeting is visible as final: the job is still in `batch_pending` when it runs, and `sync_meetings.py --g2-only --g2-import-file` runs from the ops handoff only after it returns. Sync reads speakers from the markdown transcript and, for a batch meeting, from `batchSegments[].speakerWords`, both already relabelled; `batchTranscript` is kept equal to the canonical text. No sync change is needed. With early sync on, the claim made at save is marked HQ pending and the final import replaces its transcript, as it already did for HQ.
+- **Offline by construction.** The server always passes `--models <dir>`; that path in FluidAudio has no network call, while the CLI run without it downloads from Hugging Face and deletes a cache whose weights marker it does not recognise. The directory is `COS_NEMOTRON_MODELS`, else `~/.cos-glasses/models/nemotron-3-diarization`, which the warm-up links to FluidAudio's own cache (`~/Library/Application Support/FluidAudio/Models/nemotron-3-diarization`, bundle `monolithic/v2`). An incomplete explicit directory is never replaced by another copy. With no complete local copy the CLI is never run. Probed 2026-10-02 under `sandbox-exec` with all network denied: the CLI ran from the linked directory and from the cache. The CLI has no version command, so the warm-up logs its sha256 once and whether it is the verified v0.17.4 build (`readiness.diarizer.cliKnown`).
+- **The CLI gets nothing it does not need.** It runs with PATH, HOME, TMPDIR, USER, LOGNAME and the locale only: no API token, provider key or OpenAI key reaches it (verified warm with only those on 2026-10-02).
+- **Nothing outlives the server.** Runs and temp directories in flight are tracked. `gracefulShutdown` and the process exit hook kill every CLI run and delete every temp directory (the final pass's holds the whole meeting's audio). The next start sweeps `cos-nemotron-*` directories older than a minute and kills CLI runs of ours that were orphaned.
+- **Honest health, codes only.** `/api/health` is public (no token) and the server listens on the LAN, so `readiness.diarizer` carries codes and counts only: never a speaker name, a path, a meeting title or raw error text (QA blocker; the channel map and raw errors now stay in `server.log`). It reports `requested`; `active`, which is `nemotron` only while the model is warm, the breaker is closed and most of the last five chunks really were labelled by Nemotron (otherwise `warming` or `embedding`), with the `fallback` reason; `recent`; `breaker` (open, trips, next retry); the preset, budget and lane; the warm-up; `cliKnown`; the per-chunk outcome counts by reason and the last chunk; and the final passes as `{ status, reasonCode, mode, crowded, channels, mappedChannels, wordsRelabeled, wordsTotal, excludedChunks, ms, at }` with counts keyed by status and code from a fixed set. A Nemotron fallback never degrades `readiness.status`. Every final pass logs what it applied or why it left the meeting alone, with any chunks left off the timeline (missing, hash mismatch, unreadable); a pass the save path declines because the batch sidecar was not written is counted and logged too (`metadata_not_persisted`).
+- **Settings.** `COS_DIARIZER` (default `nemotron`; `embedding`, `eres2net` or `voiceprint` turns Nemotron off entirely: no warm-up, no run, the 6.60 labels), `COS_NEMOTRON_CLI` (default `~/.cos-glasses/bin/fluidaudiocli`) and `COS_NEMOTRON_MODELS`, all in `managed-runtime-contract.json` and `.env.example`. To turn it off, put `COS_DIARIZER=embedding` in `~/.cos-glasses/.env`.
+- **After updating.** No Install hooks and no COS Control change. The first start compiles the model for the Neural Engine in the background (about 30 s on the M3 Ultra); chunks use the voiceprint until `readiness.diarizer.active` reads `nemotron`. No LLM is called anywhere in transcription.
+- **Real-data replay (read-only, 2026-10-02).** Clones of three meetings from the canary list, `COS_DATA_DIR` pointed at a scratch copy so not even the calibration log touched the store, the live path chunk by chunk exactly as the route runs it (Nemotron started first, the whole-chunk voiceprint, Whisper large-v3 through a private whisper-server with the live server's arguments, then the track names), then the final pass on a sidecar carrying those live turns. The live replay ran on `dacd8be`-era code, with the mutation gate alongside part of it, so the times are a busy-Mac upper bound.
+  - **Disclosure: what the agreement numbers measure.** They measure how rarely Nemotron disagrees with a label the voiceprint was already sure of, not whether a label is right. The live path gives a one-track chunk the stored voiceprint label by construction, so its agreement is high by design on those chunks; the final pass maps channels from the very chunks it is then scored on. Nobody listened to the changed turns. The independent check available here is the last number in each final line: final word labels against the per-track voiceprint names of the live turns.
+  - Silas and Miles, Thu 10:01 (505 chunks with text): Nemotron labelled 502 (3 `no_speech`); 114 chunks (22.7%) gained a second speaker, 1.31 segments per chunk; on 270 high-confidence chunks the dominant name matched the stored label 270 times. Final: channels {0: Silas, 1: MU}, 2,201 of 8,484 words relabelled (988 Ext to Silas, 455 Ext to MU, 609 turn splits); majority agreement 1.000 on 132 one-sided and 241 high-confidence chunks; final words against the voiceprint-named live tracks 0.983 (4,589 words).
+  - Ryan and Miles, Thu 16:03 (853): 852 labelled (1 `no_speech`); 289 (33.9%) gained a second speaker, 1.49 segments per chunk; dominant agreement 0.995 on 659. Final: {0: Ryan, 1: MU, 2: Ryan}, 4,432 of 19,140 words relabelled; majority agreement 0.993 one-sided (274), 0.974 high-confidence (650); against live tracks 0.974 (12,969 words).
+  - Five people, Wed 07:31 (753): 747 labelled (6 `no_speech`); 175 (23.4%) gained a second speaker, 1.32 per chunk; dominant agreement 0.995 on 365. Final: crowded (5 identities), so no meeting-wide map; the live turns relabelled 2,977 of 12,226 words; majority agreement 0.995 one-sided (183), 0.980 high-confidence (346); against live tracks 0.999.
+  - Latency with Whisper load, all 2,111 chunks: Nemotron's own run p50 140 to 141 ms, p95 149 to 157 ms, max 196, 302 and 680 ms; what the chunk waited beyond Whisper (the end of Nemotron plus naming the tracks) p50 0 ms, p95 89 to 114 ms, max 129, 142 and 213 ms; no chunk timed out or found the lane busy. The final pass took 19 s, 30 s and 28 s for 52, 86 and 80 minutes of audio.
+  - The final pass re-run on the QA-fixed code, on the stored sidecars without live turns: 10:01 applied ({0: Silas, 1: MU}, 1,935 of 8,484 words, one-sided 1.000, high-confidence 0.996); 16:03 applied ({0: Ryan, 1: MU, 2: Ryan}, 4,234 of 19,140, 0.993 and 0.974); 07:31 crowded with no live turns to apply, so `no_change`: nothing written.
+  - The rule that the chunk `speaker` is the voice carrying the words came after the replay; it changes none of the 2,101 replayed chunks' names (every replayed `speaker` carried words).
+  - The replay found a real bug before release: the first final pass regenerated the transcript from the relabelled words and cleaned it again, which changed which URL-only lines cleaning dropped, so the word check refused every batch meeting. Fixed by relabelling in place.
+- **QA round 1** (the Skeptic and the Ghost Hunter, on `dacd8be`). Fixed: the public health leak (blocker); the final pass overwriting a correction or summary made during its run; rename enrolment able to take a chunk the voiceprint gave to someone else; the CLI inheriting the server's tokens; health reading `nemotron` while every chunk fell back; no breaker on repeated budget kills; `no_change` that almost never fired because lines were merged; no cap on a timeline stretched by a bad `elapsed`; temp audio and CLI runs left behind by an exit; a warm-up that could stick on `warming`; recovery leaving turns that no longer joined to the text; untestable guards (a seam for the writer and the relabeller now makes the word check reachable); missing telemetry; constants defined twice. 28 of the new regression tests fail on `dacd8be` and pass here.
+- Tests: the backend choice and the models directory (offline, linking, an incomplete override); the CLI run (the arguments, the minimal environment, the frame check both ways, exit codes, SIGKILL at the budget, a missing binary, children and temp dirs killed and removed, the startup sweep); word-to-track mapping with clocks and estimated, phrase snapping, flickers, tracks, exclusive audio; naming (one track, two long tracks, a short dominant and a short other with the client label, a repeated or unknown client label, an unnamed track, word clocks over estimates, the voice that carries the words, punctuation-only tokens and merged turns joining to the text exactly, turns refitted after recovery); the runtime (a cold model, busy under a 300-chunk backlog, the breaker and its backoff, the real defaults pinned, health while chunks fall back, a warm-up whose temp dir cannot be made, the warm-up failure and identity lines, outcomes counted and logged, a missing CLI never stamped); the route (segments in the response and the HUD event with every legacy field intact, stored on the chunk, replayed, a fallback answering like 6.60, the voiceprint by choice leaving the record as 6.60 wrote it; recovery refitting turns); the final pass (consistent tracks with original labels and line breaks kept, a crashed or timed-out CLI changing no byte, words cleaning changed or removed kept exactly with labels in step, a relabel or summary during the run surviving, `no_change` writing nothing, `too_long`, a failed markdown write restoring the sidecar and a failed restore reported, the word check reached through the seam, the log lines, the timeout formula pinned, blips not crowding a room, an exit mid-pass leaving no temp audio, a human correction, the voiceprint by choice, a streaming meeting, a crowded room, the timeline's hash check, trim and refusal of a non-regular file, the map's floors, a declined pass counted); the save path (the pass runs on the saved files while the audio exists and before the job leaves `batch_pending`; a failed or throwing pass leaves the meeting byte-identical; relabel and deattribute refused while finalizing and taken after); rename enrolment refusing another voice or a two-voice chunk; unauthenticated `/api/health` after a final pass carrying no name, path or raw error; `readiness.diarizer`; the runtime contract keys. The suite runs with `COS_DIARIZER=embedding` so no test can spawn the real CLI.
+- `node server/scripts/nemotron-mutation-gate.mjs` (repository only, not in the package) runs from a green baseline, matches each target exactly once, and each mutant must fail the test named for it: baseline 160 tests in 11 files, 96 of 96 killed. In QA round 1 it caught a mutant that dropped unmatched words and hid behind a line the pass left whole; the test now inserts a word into a line the pass splits. `server/scripts/nemotron-replay.ts` is the replay above (repository only).
+
 ## 6.60.0
 
 ### A question that starts while you are away waits up to 10 minutes for the glasses you are wearing

package/README.md

@@ -472,6 +472,9 @@ past the installed hook's wait; `0` turns the away hold off),
 `COS_PERMISSION_BROKER_AWAY_RECENT_CLIENT_S` (6.60.0: default 1800; the away hold applies
 only when a `hold=1` client reported its wearer present within it; under 30 means the 30 s
 live window),
+`COS_DIARIZER` (6.61.0: default `nemotron`; `embedding` keeps the voiceprint only),
+`COS_NEMOTRON_CLI` and `COS_NEMOTRON_MODELS` (6.61.0: the FluidAudio CLI and its
+local model directory; see Speaker diarization),
 `COS_MEDIA_ROOT` (optional image/video store location; default
 `~/.cos-glasses/data/media`), and `COS_VIDEO_UPLOAD_V2=1` (private 6.27.3+
 resumable-video canary, managed by COS Control 0.5.20). The V2 canary retains
@@ -578,6 +581,67 @@ report `enrolled: false`. Set `owner_speaker_label` to `MU` to keep the existing
 voiceprints rather than re-enrolling, which would split the same voice across two
 profiles.
 
+### Who spoke when: Nemotron (6.61.0, the default)
+
+The voiceprint names a person but gives a chunk one label, and about 45% of
+meeting chunks hold two voices. Since 6.61.0 the server separates the voices
+with [Nemotron 3 Diarization](https://huggingface.co/FluidInference/nemotron-3-diarization-coreml)
+(FluidAudio's `fluidaudiocli`, preset `fast32`, Neural Engine) and lets the
+voiceprint name each one. Nemotron never names anyone itself, and the
+voiceprint's thresholds are unchanged: what changes is that a two-voice chunk
+is split into turns instead of carrying one label.
+
+- **Live, on the HUD.** Each chunk runs Nemotron in parallel with Whisper, inside
+  a 1.5 s budget, at most two at once. The chunk response keeps `speaker` (now
+  the dominant voice's name) and `text`, and adds
+  `segments: [{ speaker, text, startSec, endSec, similarity }]` in time order;
+  their texts joined with one space equal `text`. A track with at least 2 s of
+  its own audio is named by the voiceprint on that audio; a shorter one takes the
+  chunk's voiceprint when it is the dominant voice, otherwise it is `Ext`. The
+  glasses' default wearer label never names a track, and in a split chunk the
+  wearer is named only at owner-verify confidence (0.65; since 6.61.1). The
+  segments are stored on the chunk, so the saved meeting keeps them.
+- **The final transcript.** When a meeting is saved, one Nemotron pass over the
+  whole meeting maps each channel to a name from the chunks the voiceprint was
+  sure of (similarity 0.65 or more, one-sided), and relabels the transcript word
+  by word before the meeting reaches COS sync. A channel two people share stays
+  unmapped and keeps the per-chunk names. Only labels change: lines split where
+  the speaker changes and are never merged, nothing is written unless every
+  word survives in order, changed words keep `originalSpeaker`, and the sidecar
+  keeps the pre-relabel transcript under `diarization.originalTranscript`. A
+  meeting with a human speaker correction is never relabelled; an edit made
+  while the pass runs makes it stand down, and relabel or deattribute answer
+  409 `meeting_finalizing` until the meeting has finished finalizing.
+- **Never in the way.** Until the model is warm (the first Neural Engine compile
+  takes 15 to 35 s, started in the background at server start), and on any
+  timeout, crash, busy lane, missing CLI or missing model, a chunk keeps the
+  voiceprint label exactly as before, and the server logs the reason. Three
+  failed runs in a row open a breaker: no CLI is spawned and every chunk takes
+  the voiceprint at once, for 2 minutes, doubling per trip up to an hour, then
+  one fresh warm-up decides. A failed final pass leaves the meeting exactly as
+  saved. The CLI runs with a minimal environment (no tokens or keys), and no run
+  or temp file outlives the server. No model call is ever made.
+
+**Offline.** The server always passes `--models <dir>` to the CLI, a directory
+holding `Nemotron3Diarizer_fast32.mlmodelc` and `learnable_sil_emb.bin`, and that
+path in FluidAudio never downloads. Without `--models` the CLI fetches from
+Hugging Face, so the server never runs it that way. The directory is
+`COS_NEMOTRON_MODELS`, else `~/.cos-glasses/models/nemotron-3-diarization`, which
+the server links to FluidAudio's own cache
+(`~/Library/Application Support/FluidAudio/Models/nemotron-3-diarization`, left
+there by running the CLI once by hand). The CLI is `COS_NEMOTRON_CLI`, else
+`~/.cos-glasses/bin/fluidaudiocli`. Missing either, the server stays on the
+voiceprint and says so.
+
+`COS_DIARIZER=embedding` (or `voiceprint`, `eres2net`) turns Nemotron off: no
+warm-up, no run, the 6.60 labels. `/api/health` `readiness.diarizer` reports
+`active` (`nemotron` only while warm, the breaker is closed and most recent
+chunks really were labelled by Nemotron; otherwise `warming` or `embedding`),
+the `fallback` reason, the recent outcomes, the breaker, the per-chunk counts,
+the warm-up and the final passes. `/api/health` is public, so this carries codes
+and counts only, never a speaker name, path or error text; those stay in
+`server.log`. A Nemotron fallback never degrades `readiness.status`.
+
 ## HQ dictation
 
 Prompt dictation defaults to HQ. The phone owns the preference: **Fast mode

package/managed-runtime-contract.json

@@ -39,7 +39,10 @@
     "COS_PERMISSION_BROKER_AWAY_RECENT_CLIENT_S",
     "COS_LENS_GIST_ENGINE",
     "COS_LENS_GIST_MODEL",
-    "COS_LENS_GIST_DAILY_CAP"
+    "COS_LENS_GIST_DAILY_CAP",
+    "COS_DIARIZER",
+    "COS_NEMOTRON_CLI",
+    "COS_NEMOTRON_MODELS"
   ],
   "maintenance": {
     "scope": "cross_boot",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.60.0",
+  "version": "6.61.1",
   "description": "COS Glasses: self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, Cursor Agent CLI, or local Ollama",
   "type": "module",
   "bin": {
@@ -46,6 +46,8 @@
     "!server/scripts/work-intake-mutation-gate.mjs",
     "!server/scripts/work-progress-mutation-gate.mjs",
     "!server/scripts/away-hold-mutation-gate.mjs",
+    "!server/scripts/nemotron-mutation-gate.mjs",
+    "!server/scripts/nemotron-replay.ts",
     "!server/scripts/mutation-gate.mjs",
     "!server/lib/meeting-engine/__fixtures__/**",
     "!server/lib/__fixtures__/**",

package/server/index.ts

@@ -116,6 +116,8 @@ import { startLocalTtsServer, stopLocalTtsServer } from './lib/tts-local.js'
 import { initSileroVAD } from './lib/vad-silero.js'
 import { initSessionCache } from './lib/session-cache-writer.js'
 import { initSpeakerEmbeddings } from './lib/speaker-embeddings.js'
+import { startNemotronWarmup } from './lib/nemotron-live.js'
+import { killNemotronChildren } from './lib/nemotron-cli.js'
 import { logActiveSessionsOnShutdown, startAutoSnapshot } from './lib/conversation.js'
 import { getMediaStore } from './lib/media-store.js'
 import { listenRequiredServers, type RequiredListener } from './lib/listener-startup.js'
@@ -1000,6 +1002,12 @@ async function gracefulShutdown(): Promise<void> {
     console.error('[live-cues] shutdown termination failed:', error)
   }
   try { logActiveSessionsOnShutdown() } catch { /* best-effort flush */ }
+  // 6.61.0: no Nemotron run may outlive the server, and no temp directory may
+  // keep a meeting's audio (the module's exit hook repeats this for any exit).
+  try {
+    const nemotron = killNemotronChildren()
+    if (nemotron.killed || nemotron.removed) console.log(`[diarizer] shutdown: killed ${nemotron.killed} Nemotron run(s), removed ${nemotron.removed} temp dir(s)`)
+  } catch { /* best effort */ }
   stopCodexModelCatalogRefresh()
   stopWhisperServer()
   await stopWhisperPreviewServer({ final: true })
@@ -1162,6 +1170,9 @@ listenRequiredServers(listeners).then(() => {
     // Initialize Silero VAD (silence trimming before Whisper) — fails soft if model absent
     const vadOk = initSileroVAD()
     console.log(`[startup] Silero VAD: ${vadOk ? 'active' : 'disabled (model not found)'}`)
+    // 6.61.0: warm Nemotron in the background (the first Neural Engine compile
+    // takes 15-30 s). Chunks use the voiceprint until it is warm; never awaited.
+    startNemotronWarmup()
   }
 
   // Durable recovery starts with the admitted runtime by default. A literal

package/server/lib/diarizer-backend.ts

@@ -0,0 +1,163 @@
+// Which model assigns who-spoke-when.
+//
+// Voiceprints (ERes2Net) name a person. They do not separate two people inside
+// one chunk. Nemotron 3 Diarization does that, then the voiceprint names the
+// track. COS_DIARIZER selects the track model. The default is Nemotron. If its
+// CLI or its local models are missing, the server stays on the voiceprint path
+// and says so.
+//
+// OFFLINE BY CONSTRUCTION. The FluidAudio CLI downloads its models from Hugging
+// Face whenever it is run WITHOUT `--models`, and it deletes and re-downloads a
+// cache whose weights marker it does not recognise. A meeting must never wait on
+// that, so the server always passes `--models <dir>`, a local directory holding
+// the compiled bundle and the silence embedding. That code path in FluidAudio
+// (Nemotron3Models.load(directory:)) has no network call at all. When no complete
+// local directory exists the server does not run the CLI.
+
+import { execFileSync } from 'node:child_process'
+import { chmodSync, existsSync, lstatSync, mkdirSync, readlinkSync, statSync, symlinkSync, unlinkSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+export type DiarizerMode = 'nemotron' | 'embedding'
+
+/** Why Nemotron cannot run at all on this install. Per-chunk reasons live in nemotron-live.ts. */
+export type DiarizerStaticFallback = 'cli_missing' | 'models_missing'
+
+export interface DiarizerChoice {
+  requested: DiarizerMode
+  active: DiarizerMode
+  /** Why Nemotron was not used. Null when the request is what is running. */
+  fallback: DiarizerStaticFallback | null
+}
+
+/** The canary's preset: 32-frame chunks, about 155x realtime warm on an M3 Ultra. */
+export const NEMOTRON_VARIANT = 'fast32'
+/** The FluidAudio v0.17.4 cache keeps the compiled preset under this subdirectory. */
+export const NEMOTRON_BUNDLE = 'monolithic/v2'
+export const NEMOTRON_MODEL_DIRNAME = `Nemotron3Diarizer_${NEMOTRON_VARIANT}.mlmodelc`
+export const NEMOTRON_SILENCE_FILE = 'learnable_sil_emb.bin'
+
+const EMBEDDING_ALIASES = new Set(['embedding', 'eres2net', 'voiceprint', 'legacy'])
+
+export function requestedDiarizer(env: NodeJS.ProcessEnv = process.env): DiarizerMode {
+  const raw = (env.COS_DIARIZER ?? 'nemotron').trim().toLowerCase()
+  return EMBEDDING_ALIASES.has(raw) ? 'embedding' : 'nemotron'
+}
+
+/** First existing binary. Never returns a path that is not on disk. */
+export function nemotronCliPath(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string | null {
+  if (typeof env.COS_NEMOTRON_CLI === 'string') {
+    return env.COS_NEMOTRON_CLI.length > 0 && existsSync(env.COS_NEMOTRON_CLI) ? env.COS_NEMOTRON_CLI : null
+  }
+  const installed = join(home, '.cos-glasses', 'bin', 'fluidaudiocli')
+  if (existsSync(installed)) return installed
+  // Health reads this on every request; on an install with no CLI that was a
+  // synchronous `which` per request (QA 6.61.0 N5). Cached for a minute per PATH.
+  const key = env.PATH ?? ''
+  const now = Date.now()
+  if (whichCache && whichCache.key === key && now - whichCache.at < WHICH_CACHE_MS) {
+    return whichCache.value && existsSync(whichCache.value) ? whichCache.value : null
+  }
+  let value: string | null = null
+  try {
+    const found = execFileSync('which', ['fluidaudiocli'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 2_000 }).trim()
+    value = found && existsSync(found) ? found : null
+  } catch {
+    value = null
+  }
+  whichCache = { key, at: now, value }
+  return value
+}
+
+const WHICH_CACHE_MS = 60_000
+let whichCache: { key: string; at: number; value: string | null } | null = null
+
+/** The directory passed to `--models`. COS_NEMOTRON_MODELS overrides it. */
+export function nemotronModelsDir(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string {
+  const override = env.COS_NEMOTRON_MODELS?.trim()
+  return override ? override : join(home, '.cos-glasses', 'models', 'nemotron-3-diarization')
+}
+
+/** Where FluidAudio's own downloader leaves the models (`fluidaudiocli` run once without `--models`). */
+export function fluidAudioCacheDir(home: string = homedir()): string {
+  return join(home, 'Library', 'Application Support', 'FluidAudio', 'Models', 'nemotron-3-diarization')
+}
+
+function isFile(path: string): boolean {
+  try { return statSync(path).isFile() } catch { return false }
+}
+
+/** A compiled bundle is complete once its coremldata.bin is on disk (FluidAudio's own test). */
+export function modelsDirComplete(dir: string): boolean {
+  return isFile(join(dir, NEMOTRON_MODEL_DIRNAME, 'coremldata.bin')) && isFile(join(dir, NEMOTRON_SILENCE_FILE))
+}
+
+function cacheComplete(home: string): boolean {
+  const cache = fluidAudioCacheDir(home)
+  return isFile(join(cache, NEMOTRON_BUNDLE, NEMOTRON_MODEL_DIRNAME, 'coremldata.bin'))
+    && isFile(join(cache, NEMOTRON_SILENCE_FILE))
+}
+
+export interface NemotronModelSource {
+  dir: string
+  /** env: COS_NEMOTRON_MODELS. cos: ~/.cos-glasses/models already complete.
+   *  fluidaudio-cache: complete only after prepareNemotronModelsDir links it. */
+  source: 'env' | 'cos' | 'fluidaudio-cache'
+}
+
+/** Read-only: where the models are, or null. Never creates anything (health calls this). */
+export function nemotronModelSource(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): NemotronModelSource | null {
+  const dir = nemotronModelsDir(env, home)
+  if (env.COS_NEMOTRON_MODELS?.trim()) {
+    // An explicit override is the answer, complete or not. Falling through to a
+    // different copy would run weights the operator did not choose.
+    return modelsDirComplete(dir) ? { dir, source: 'env' } : null
+  }
+  if (modelsDirComplete(dir)) return { dir, source: 'cos' }
+  if (cacheComplete(home)) return { dir, source: 'fluidaudio-cache' }
+  return null
+}
+
+/**
+ * Make `--models` point at a complete local copy, or return null.
+ *
+ * FluidAudio keeps the bundle one level below the silence embedding, while
+ * `--models` wants both in one directory, so the COS directory holds two symlinks
+ * into the FluidAudio cache. Called by the warm-up, never by a health read.
+ */
+export function prepareNemotronModelsDir(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): string | null {
+  const found = nemotronModelSource(env, home)
+  if (!found) return null
+  if (found.source !== 'fluidaudio-cache') return found.dir
+  const cache = fluidAudioCacheDir(home)
+  const links: Array<[string, string]> = [
+    [join(found.dir, NEMOTRON_MODEL_DIRNAME), join(cache, NEMOTRON_BUNDLE, NEMOTRON_MODEL_DIRNAME)],
+    [join(found.dir, NEMOTRON_SILENCE_FILE), join(cache, NEMOTRON_SILENCE_FILE)],
+  ]
+  try {
+    mkdirSync(found.dir, { recursive: true, mode: 0o700 })
+    try { chmodSync(found.dir, 0o700) } catch { /* best effort */ }
+    for (const [link, target] of links) {
+      let current: string | null = null
+      try {
+        if (lstatSync(link).isSymbolicLink()) current = readlinkSync(link)
+        else continue // a real file or directory someone placed there wins
+      } catch { /* absent */ }
+      if (current === target) continue
+      if (current !== null) unlinkSync(link)
+      symlinkSync(target, link)
+    }
+  } catch {
+    return null
+  }
+  return modelsDirComplete(found.dir) ? found.dir : null
+}
+
+export function activeDiarizer(env: NodeJS.ProcessEnv = process.env, home: string = homedir()): DiarizerChoice {
+  const requested = requestedDiarizer(env)
+  if (requested === 'embedding') return { requested, active: 'embedding', fallback: null }
+  if (!nemotronCliPath(env, home)) return { requested, active: 'embedding', fallback: 'cli_missing' }
+  if (!nemotronModelSource(env, home)) return { requested, active: 'embedding', fallback: 'models_missing' }
+  return { requested, active: 'nemotron', fallback: null }
+}

package/server/lib/meeting-relabel-enrolment.ts

@@ -96,6 +96,10 @@ export interface EnrolmentReport {
   /** Candidates rejected as a different voice. `clusterSkipped === attempted`
    *  means the bucket had no dominant voice at all and nothing was written. */
   clusterSkipped: number
+  /** 6.61.0: candidates refused before clustering because the chunk is not one
+   *  voice: its own capture-time voiceprint named a third person, or Nemotron
+   *  split it into more than one turn. */
+  otherVoiceSkipped?: number
   skipped: EnrolmentSkipReason
 }
 
@@ -230,16 +234,35 @@ export function enrolNamedVoice(input: EnrolNamedVoiceInput): EnrolmentReport {
   if (read.missing) return { ...IDLE, skipped: missingReason(sidecar) }
 
   const wanted = new Set(rawIndices)
-  const candidates = read.rows.filter(r => wanted.has(r.i) && r.embedding && r.embedding.length > 0)
-  if (candidates.length === 0) return { ...IDLE, skipped: 'no_embeddings' }
+  const found = read.rows.filter(r => wanted.has(r.i) && r.embedding && r.embedding.length > 0)
+  if (found.length === 0) return { ...IDLE, skipped: 'no_embeddings' }
 
-  const attempted = candidates.length
+  // 6.61.0 (QA W3). Since Nemotron, a chunk's sidecar `speaker` is the dominant
+  // TURN's name, while the embedding row holds the WHOLE chunk's voice and the
+  // label the voiceprint gave that whole chunk. A rename of the turn's label must
+  // not teach the profile a chunk whose own voiceprint named someone else, or a
+  // chunk Nemotron heard as more than one voice: either would enrol the wrong
+  // person. `from` stays allowed (a wrong label is how a new person first shows up).
+  const chunksByIndex = new Map<number, Record<string, unknown>>()
+  for (const entry of Array.isArray(sidecar.chunkEntries) ? sidecar.chunkEntries as Array<Record<string, unknown>> : []) {
+    if (typeof entry?.chunkIndex === 'number' && entry.chunk && typeof entry.chunk === 'object') {
+      chunksByIndex.set(entry.chunkIndex, entry.chunk as Record<string, unknown>)
+    }
+  }
+  const candidates = found.filter(r => !isAnotherVoice(r.speaker, input.from, to) && !isMultiVoiceChunk(chunksByIndex.get(r.i)))
+  // Reported only when it happened, so the response is the 6.60 shape otherwise.
+  const otherVoice = found.length - candidates.length > 0 ? { otherVoiceSkipped: found.length - candidates.length } : {}
+  if (candidates.length === 0) {
+    return { enrolled: 0, attempted: found.length, created: false, clusterSkipped: 0, ...otherVoice, skipped: null }
+  }
+
+  const attempted = found.length
   const cluster = dominantCoherentCluster(candidates.map(r => r.embedding))
-  const clusterSkipped = attempted - cluster.members.length
+  const clusterSkipped = candidates.length - cluster.members.length
   // Every candidate disagreed with every other one. There is no voice here to
   // learn, only a bucket of strangers sharing one placeholder label.
   if (cluster.members.length === 0) {
-    return { enrolled: 0, attempted, created: false, clusterSkipped, skipped: null }
+    return { enrolled: 0, attempted, created: false, clusterSkipped, ...otherVoice, skipped: null }
   }
 
   // Diversity trims the coherent group to a bounded, spread-out sample. Those
@@ -261,12 +284,26 @@ export function enrolNamedVoice(input: EnrolNamedVoiceInput): EnrolmentReport {
     }
   } catch {
     return {
-      enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped,
+      enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped, ...otherVoice,
       skipped: 'store_unavailable',
     }
   }
 
   return {
-    enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped, skipped: null,
+    enrolled, attempted, created: !existedBefore && enrolled > 0, clusterSkipped, ...otherVoice, skipped: null,
   }
 }
+
+/** The row's own capture-time voiceprint named a real person other than the two in this rename. */
+export function isAnotherVoice(rowSpeaker: unknown, from: string, to: string): boolean {
+  if (typeof rowSpeaker !== 'string' || !rowSpeaker.trim()) return false
+  if (isPlaceholderLabel(rowSpeaker)) return false
+  return rowSpeaker !== from && rowSpeaker !== to
+}
+
+/** Nemotron heard more than one turn in this chunk, live or in the final pass. */
+export function isMultiVoiceChunk(chunk: Record<string, unknown> | undefined): boolean {
+  if (!chunk) return false
+  const turns = (value: unknown): number => Array.isArray(value) ? value.length : 0
+  return turns(chunk.segments) > 1 || turns(chunk.finalSegments) > 1
+}

package/server/lib/meeting-store.ts

@@ -379,7 +379,9 @@ export class MeetingStore {
       canonicalProvider: canonicalProvider(input.chunks),
       providers,
       providerCandidates: input.providerCandidates ?? {},
-      speakers: [...new Set(input.chunks.map(chunk => chunk.speaker).filter(speaker => speaker && speaker !== 'Ext'))],
+      // 6.61.0: a chunk can hold two named voices; its segments count too.
+      speakers: [...new Set(input.chunks.flatMap(chunk => [chunk.speaker, ...(chunk.segments ?? []).map(segment => segment.speaker)])
+        .filter(speaker => speaker && speaker !== 'Ext'))],
       chunks: input.chunks,
       chunkEntries: input.chunkEntries ?? input.chunks.map((chunk, chunkIndex) => ({ chunkIndex, chunk })),
       transferIntegrity: input.transferIntegrity ?? null,