@gotcos/glasses-server 6.60.0 → 6.61.0

package/server/routes/transcribe-stream.ts

@@ -40,6 +40,8 @@ import { transcribeLocal, applyCorrections, type WhisperWord } from '../lib/whis
 import { enhanceAudio } from '../lib/audio-enhance.js'
 import { trimSilence, isSileroAvailable } from '../lib/vad-silero.js'
 import { identifySpeaker, isEmbeddingAvailable, autoEnroll, AUTO_ENROLL_CANDIDATE_SIMILARITY } from '../lib/speaker-embeddings.js'
+import { nemotronLive, resolveLiveLabels, resplitSegments, type ChunkSpeakerSegment } from '../lib/nemotron-live.js'
+import { VOICEPRINT_MIN_AUDIO_SEC } from '../lib/voiceprint-floor.js'
 import {
   assertOpenAIWhisperBudget,
   recordOpenAIWhisperUsage,
@@ -266,6 +268,23 @@ export interface TranscriptChunk {
   audioSha256?: string
   canonical?: boolean
   evenHubSpeakerRole?: EvenSpeakerRoleHistogram
+  /** 6.61.0: who spoke when inside this chunk, in time order. Present when
+   *  Nemotron separated the chunk's voices; `speaker` is then the dominant name. */
+  segments?: ChunkSpeakerSegment[]
+  /** 6.61.0: which path labelled the chunk. Absent when COS_DIARIZER selects the voiceprint. */
+  diarizer?: ChunkDiarizerInfo
+}
+
+export interface ChunkDiarizerInfo {
+  engine: 'nemotron' | 'voiceprint'
+  /** Why Nemotron did not label this chunk although it was requested. */
+  fallback?: string
+  /** The whole-chunk voiceprint label, kept when the dominant track's name differs. */
+  voiceprintSpeaker?: string
+  voiceprintSimilarity?: number
+  tracks?: number
+  timing?: 'words' | 'estimated'
+  ms?: number
 }
 
 export interface ProviderCandidateRecord {
@@ -345,6 +364,9 @@ interface StreamChunkCompletionResponse {
   backend?: string
   asrProvider?: string
   fallbackReason?: string
+  /** 6.61.0, additive: the chunk's speaker turns in time order. Their texts
+   *  joined with one space equal `text`; `speaker` stays the dominant name. */
+  segments?: ChunkSpeakerSegment[]
 }
 
 const closedSessionRecords = new Map<string, ClosedTranscriptSession>()
@@ -772,6 +794,12 @@ function recoverSessions(): void {
                 const stripped = stripInlineHallucinations(chunk.text, data.sessionId)
                 if (stripped !== chunk.text) {
                   chunk.text = stripped
+                  // 6.61.0: the stored turns must still join to the text.
+                  if (chunk.segments) {
+                    const refit = resplitSegments(chunk.segments, stripped)
+                    if (refit?.length) chunk.segments = refit
+                    else delete chunk.segments
+                  }
                   cleaned++
                 }
               }
@@ -1837,6 +1865,7 @@ function canonicalChunkResponse(
     backend: existing.backend,
     asrProvider: existing.asrProvider,
     fallbackReason: existing.fallbackReason,
+    ...(existing.segments?.length ? { segments: existing.segments } : {}),
   }
 }
 
@@ -1882,7 +1911,7 @@ async function transcribeWithServerWhisper(audioBuffer: Buffer, whisperAudio: Bu
 function identifyChunkSpeaker(audioBuffer: Buffer, sessionId: string, chunkIndex: number, clientSpeaker: string): { speaker: string; similarity: number } {
   const expectedSpeakers = undefined
   const audioDurationSec = Math.max(0, (audioBuffer.length - 44)) / 32000
-  if (!isEmbeddingAvailable() || audioDurationSec < 2.0) return { speaker: clientSpeaker, similarity: 0 }
+  if (!isEmbeddingAvailable() || audioDurationSec < VOICEPRINT_MIN_AUDIO_SEC) return { speaker: clientSpeaker, similarity: 0 }
 
   const tEmb = performance.now()
   const embeddingResult = identifySpeaker(audioBuffer, expectedSpeakers)
@@ -2033,6 +2062,15 @@ async function processStreamChunk(opts: {
   recordReceivedChunk(session, chunkIndex)
   persistSessionRequired(sessionId)
 
+  // 6.61.0: Nemotron reads the durable WAV just written, in parallel with
+  // Whisper, on the Neural Engine, inside its own budget. It is started before
+  // the synchronous voiceprint so the process is already running while that
+  // blocks. diarize() never throws; a fallback returns a reason.
+  const diarizeStartedAt = performance.now()
+  const diarizePromise = nemotronLive.diarize(
+    resolve(SESSION_AUDIO_DIR, sessionId, `chunk_${String(chunkIndex).padStart(4, '0')}.wav`),
+  )
+
   const pcmData = audioBuffer.subarray(44)
   let sumSq = 0
   const nSamples = Math.floor(pcmData.length / 2)
@@ -2090,14 +2128,14 @@ async function processStreamChunk(opts: {
     sanitized = sanitizeStreamTranscript(sessionId, session, rawText, isQuiet)
   }
 
-  const { speaker, similarity } = await speakerPromise
+  const voiceprint = await speakerPromise
   if (evenHubSpeakerRole) {
     console.log(formatEvenRoleAgreement({
       chunkIndex,
       even: evenHubSpeakerRole,
       amp: clientSpeaker,
-      emb: speaker,
-      similarity,
+      emb: voiceprint.speaker,
+      similarity: voiceprint.similarity,
     }))
   }
   // Client time is authoritative for live network jitter and deferred replay.
@@ -2106,6 +2144,26 @@ async function processStreamChunk(opts: {
     : Date.now() - session.startTime
   const trimmedText = sanitized.text
 
+  // Nemotron separates the voices; the voiceprint names each one. Any fallback
+  // keeps today's whole-chunk voiceprint label. Only chunks with text count.
+  const labels = trimmedText
+    ? await resolveLiveLabels({
+        diarize: diarizePromise,
+        startedAt: diarizeStartedAt,
+        text: trimmedText,
+        words,
+        audio: audioBuffer,
+        voiceprint,
+        clientSpeaker,
+        identify: wav => {
+          const named = identifySpeaker(wav, undefined, { event: 'nemotron-track' })
+          return named ? { speaker: named.speaker, similarity: named.similarity } : null
+        },
+      })
+    : { speaker: voiceprint.speaker, similarity: voiceprint.similarity }
+  if (labels.record) nemotronLive.record({ sessionId, chunkIndex }, labels.record)
+  const { speaker, similarity } = labels
+
   if (!trimmedText) {
     console.log(`[hallucination] Filtered (${sanitized.fallbackReason || fallbackReason || 'empty'}): "${rawText.slice(0, 60)}"`)
     if (candidate && candidateRecordKey && session.providerCandidates?.[candidateRecordKey]) {
@@ -2145,6 +2203,8 @@ async function processStreamChunk(opts: {
     audioSha256,
     canonical: true,
     evenHubSpeakerRole,
+    ...(labels.segments?.length ? { segments: labels.segments } : {}),
+    ...(labels.diarizer ? { diarizer: labels.diarizer } : {}),
   }
   const finalExisting = session.chunks[chunkIndex]
   if (finalExisting?.text) {
@@ -2174,7 +2234,10 @@ async function processStreamChunk(opts: {
 
   scheduleSessionProgressiveHq(sessionId)
 
-  emitDisplay({ type: 'transcript_chunk', data: { text: trimmedText, speaker, chunkIndex, elapsed, sessionId } })
+  emitDisplay({
+    type: 'transcript_chunk',
+    data: { text: trimmedText, speaker, chunkIndex, elapsed, sessionId, ...(chunk.segments ? { segments: chunk.segments } : {}) },
+  })
   // Live Cues feed — fire-and-forget, NEVER awaited: transcription must not
   // block on a cue, and no LLM runs on this path. .catch() is required — a
   // bare `void` on a rejecting promise is an unhandled rejection, which Node