@gotcos/glasses-server 6.71.14 → 6.73.0

package/server/lib/whisper-local.ts

@@ -13,7 +13,8 @@ import { writeFileSync, unlinkSync, existsSync, readFileSync, statSync } from 'n
 import { basename, join } from 'node:path'
 import { homedir } from 'node:os'
 import crypto from 'node:crypto'
-import { batchTimestampCanaryEnabled, remapBatchWords } from './whisper-batch-timing.js'
+import { batchTimestampsEnabled, parseNativeVadTimeline, remapBatchWords } from './whisper-batch-timing.js'
+import { readSttSettings, type FrontEndId } from './stt-settings.js'
 import { getVocabulary, getOwnerName, getWhisperCorrections } from './profile.js'
 import { stripBrandUrls } from './hallucination-filter.js'
 import {
@@ -70,21 +71,34 @@ export interface HighQualityTranscriptionCapability {
 export interface HighQualityTranscriptionResult {
   text: string
   words?: WhisperWord[]
+  /** Sub-word tokens merged into spoken words (6.72.0), NOT name-corrected.
+   *  Clocks are only meaningful when the decode ran with timestamps. */
+  spokenWords?: WhisperWord[]
+  /** True when this decode ran with timestamps and the VAD remap. */
+  timestamps?: boolean
+  /** Where this decode's own VAD heard speech (seconds, input clock). Timestamp mode only. */
+  speechSpans?: Array<[number, number]>
   model: 'large-v3' | 'turbo'
   backend: 'whisper-cli' | 'whisper-server'
   actualQuality: 'hq' | 'fast'
   degradationReason?: HighQualityUnavailableReason
 }
 
-let highQualityCheckpointFingerprint: string | null = null
-let highQualityCheckpointTimingMode: boolean | null = null
+const highQualityCheckpointFingerprints = new Map<string, string>()
 
 /** Stable, path-free cache identity for progressive meeting HQ checkpoints.
  * Model/configuration is immutable for the lifetime of one server process, so
- * cache this instead of stat'ing multi-GB model files on every health poll. */
-export function getHighQualityCheckpointFingerprint(): string {
-  const timingMode = batchTimestampCanaryEnabled()
-  if (highQualityCheckpointFingerprint && highQualityCheckpointTimingMode === timingMode) return highQualityCheckpointFingerprint
+ * cache this instead of stat'ing multi-GB model files on every health poll.
+ * 6.72.0: the timing mode and the ASR front end are part of the identity, so a
+ * checkpoint decoded under one setting is never reused under another. With
+ * timestamps off and the full front end the identity is the pre-6.72 one, byte
+ * for byte. */
+export function getHighQualityCheckpointFingerprint(opts: { timestamps?: boolean; frontEnd?: FrontEndId } = {}): string {
+  const timingMode = opts.timestamps ?? batchTimestampsEnabled()
+  const frontEnd: FrontEndId = opts.frontEnd ?? readSttSettings().frontEnd
+  const cacheKey = `${timingMode ? 1 : 0}|${frontEnd}`
+  const cached = highQualityCheckpointFingerprints.get(cacheKey)
+  if (cached) return cached
   const identity = (path: string): Record<string, number | boolean> => {
     try {
       const stat = statSync(path)
@@ -93,9 +107,9 @@ export function getHighQualityCheckpointFingerprint(): string {
       return { present: false, size: 0, mtimeMs: 0 }
     }
   }
-  highQualityCheckpointFingerprint = crypto.createHash('sha256').update(JSON.stringify({
+  const fingerprint = crypto.createHash('sha256').update(JSON.stringify({
     schema: 2,
-    // Keep the existing legacy identity byte-for-byte when the canary is off.
+    // Keep the existing legacy identity byte-for-byte when timestamps are off.
     ...(timingMode ? { batchTiming: 'native-vad-timestamps-v1' } : {}),
     serverVersion: process.env.COS_SERVER_VERSION?.trim() || 'development',
     whisperCli: identity(WHISPER_CLI),
@@ -104,12 +118,12 @@ export function getHighQualityCheckpointFingerprint(): string {
     beam: 5,
     bestOf: 5,
     vadEnabled: hqCliVadEnabled('batch'),
-    enhancement: 'audio-enhance-v1',
+    enhancement: frontEnd === 'full' ? 'audio-enhance-v1' : 'audio-enhance-highpass-v1',
     prompt: buildWhisperPrompt(),
     corrections: getWhisperCorrections(),
   })).digest('hex')
-  highQualityCheckpointTimingMode = timingMode
-  return highQualityCheckpointFingerprint
+  highQualityCheckpointFingerprints.set(cacheKey, fingerprint)
+  return fingerprint
 }
 
 export function classifyHighQualityTranscriptionCapability(input: {
@@ -882,7 +896,7 @@ async function reconcileWhisperServerHealth(): Promise<boolean> {
  * Batch/save path only. Never uses whisper-server `verbose_json` (the live
  * VAD-empty crash vector). Special tokens like `[_BEG_]` / `<|...|>` are dropped.
  */
-export function parseWhisperCliFullJson(raw: string): { text: string; words: WhisperWord[] } {
+export function parseWhisperCliFullJson(raw: string): { text: string; words: WhisperWord[]; spokenWords: WhisperWord[] } {
   const data = JSON.parse(raw) as {
     transcription?: Array<{
       text?: unknown
@@ -896,6 +910,7 @@ export function parseWhisperCliFullJson(raw: string): { text: string; words: Whi
   const segments = Array.isArray(data.transcription) ? data.transcription : []
   const texts: string[] = []
   const words: WhisperWord[] = []
+  const spokenWords: WhisperWord[] = []
 
   for (const segment of segments) {
     if (typeof segment.text === 'string' && segment.text.trim()) {
@@ -909,18 +924,30 @@ export function parseWhisperCliFullJson(raw: string): { text: string; words: Whi
       const fromMs = typeof token.offsets?.from === 'number' ? token.offsets.from : Number(token.offsets?.from)
       const toMs = typeof token.offsets?.to === 'number' ? token.offsets.to : Number(token.offsets?.to)
       if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) continue
-      words.push({
+      const word: WhisperWord = {
         word: tokenText,
         start: fromMs / 1000,
         end: toMs / 1000,
         probability: typeof token.p === 'number' && Number.isFinite(token.p) ? token.p : 0,
-      })
+      }
+      words.push(word)
+      // A token without a leading space continues the previous word ("I" + "'m").
+      const rawText = typeof token.text === 'string' ? token.text : ''
+      const last = spokenWords.at(-1)
+      if (last && !/^\s/.test(rawText) && !/^[\s]*$/.test(rawText)) {
+        last.word += tokenText
+        last.end = Math.max(last.end, word.end)
+        last.probability = Math.min(last.probability, word.probability)
+      } else {
+        spokenWords.push({ ...word })
+      }
     }
   }
 
   return {
     text: texts.join(' ').replace(/\s+/g, ' ').trim(),
     words,
+    spokenWords,
   }
 }
 
@@ -949,6 +976,8 @@ export async function transcribeHighQuality(
     threads?: number
     backgroundCpu?: boolean
     signal?: AbortSignal
+    /** Meeting batch only: timestamps on/off for this decode (the job's snapshot). */
+    timestamps?: boolean
   } = {},
 ): Promise<HighQualityTranscriptionResult> {
   if (!cliAvailable) {
@@ -972,7 +1001,9 @@ export async function transcribeHighQuality(
   // Word clocks on post-meeting CPU polish, and on any decode that asks for
   // them (the prompt tail guard). Other live decodes stay compact JSON.
   const captureWords = opts.priority === 'batch' || opts.words === true
-  const timestampCanary = opts.priority === 'batch' && batchTimestampCanaryEnabled()
+  // 6.72.0: on by default for meeting batch (HQ setting "large-v3"); the job
+  // passes its own snapshot so a settings change never lands mid-meeting.
+  const timestampCanary = opts.priority === 'batch' && (opts.timestamps ?? batchTimestampsEnabled())
   let nativeDiagnostics = ''
 
   const modelPath = resolveBatchModel()
@@ -1144,6 +1175,8 @@ export async function transcribeHighQuality(
 
     let finalText = text
     let words: WhisperWord[] | undefined
+    let spokenWords: WhisperWord[] | undefined
+    let speechSpans: Array<[number, number]> | undefined
     if (captureWords) {
       try {
         if (existsSync(jsonPath)) {
@@ -1157,6 +1190,11 @@ export async function transcribeHighQuality(
           words = timed.length > 0
             ? timed.map(w => ({ ...w, word: applyCorrections(w.word) }))
             : []
+          // Spoken words stay uncorrected: the resolver joins them and corrects the text once.
+          spokenWords = timestampCanary ? remapBatchWords(parsed.spokenWords, nativeDiagnostics, useVad) : parsed.spokenWords
+          if (timestampCanary && useVad && parsed.words.length) {
+            speechSpans = parseNativeVadTimeline(nativeDiagnostics).map(s => [s.originalStart, s.originalEnd] as [number, number])
+          }
         } else if (timestampCanary) {
           throw new Error('HQ timestamp candidate did not produce CLI JSON')
         }
@@ -1193,7 +1231,10 @@ export async function transcribeHighQuality(
             ? 'large_v3_model_missing' as const
             : 'hq_disabled' as const,
         }
-    return words ? { text: corrected, words, ...metadata } : { text: corrected, ...metadata }
+    const timing = isBatch ? { timestamps: timestampCanary } : {}
+    return words
+      ? { text: corrected, words, ...(spokenWords ? { spokenWords } : {}), ...(speechSpans ? { speechSpans } : {}), ...timing, ...metadata }
+      : { text: corrected, ...timing, ...metadata }
   } finally {
     try { unlinkSync(tmpWav) } catch { /* cleanup */ }
     if (captureWords) {

package/server/routes/meeting.ts

@@ -66,6 +66,7 @@ import {
 import {
   canDeletePendingBatchAudio,
   persistBatchDecisionSidecar,
+  persistSttSidecar,
   replaceMeetingTranscriptAtomic,
 } from '../lib/meeting-batch-persistence.js'
 import {
@@ -77,6 +78,8 @@ import {
 import {
   enqueueSerializedHqWork,
   runMeetingBatchPipeline,
+  recoveryHqJob,
+  summarizeStt,
   segmentTranscriptChunks,
   stopProgressiveHqSession,
   transcribeSegments,
@@ -2178,9 +2181,9 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
       // decoder serialization — that lives in the queue tail. Chain onto it,
       // or two back-to-back recoveries plus a live post-meeting batch would
       // run parallel 16-thread large-v3 decoders on the user's Mac.
-      const results = await enqueueSerializedHqWork(
-        () => transcribeSegments(quarantineDir, segments, entries),
-      )
+      const hqJob = recoveryHqJob(quarantineDir, segments, entries)
+      const plan = hqJob.plan
+      const results = await enqueueSerializedHqWork(hqJob.run)
       // Batch whisper over a dead session has no streaming baseline to
       // compare against (evaluateBatchQuality needs one), but the inline
       // hallucination filter needs none — run the same two-pass the boot
@@ -2240,6 +2243,9 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
         })),
         transferIntegrity: null,
       })
+      try { persistSttSidecar(saved.sidecarPath, results, summarizeStt(plan, results, segments.length)) } catch (error) {
+        console.warn(`[meeting/orphans] ${sessionId}: STT hypotheses not saved: ${error instanceof Error ? error.message : error}`)
+      }
       markRecovered(quarantineDir, saved.filename)
       console.log(
         `[meeting/orphans] Recovered ${sessionId} → ${saved.filename} `

package/server/routes/prompt-drafts.ts

@@ -40,6 +40,8 @@ import { applyCorrections } from '../lib/whisper-local.js'
 import { applyFuzzyCorrections } from '../lib/fuzzy-correct.js'
 import { getAllSpeakerNames } from '../lib/speaker-embeddings.js'
 import { transcribeWhisperPreview } from '../lib/whisper-preview.js'
+import { currentDictationPlan, runParakeetDictation } from '../lib/dictation-stt.js'
+import { dictationSlotForMode, legacyDictationKey, type DictationPlan } from '../lib/stt-settings.js'
 import { autoCleanDictation, AUTOCLEAN_MAX_CHARS } from '../lib/dictation-clean.js'
 import { getVocabulary } from '../lib/profile.js'
 import { createBreaker } from '../lib/claude-circuit.js'
@@ -240,14 +242,25 @@ async function sendDraftError(res: Response, draftId: string, err: any): Promise
   res.status(err.status ?? 500).json({ error: err.message })
 }
 
-function modeQualityKey(draftId: string, chunkIndex: number, mode: 'hq' | 'fast', hash: string): string {
-  return `${draftId}:${chunkIndex}:${mode}:${hash}`
+// 6.73.0: the setting key is part of every decode key, so a decode made under
+// one STT selection is never shared with or reused for another.
+function modeQualityKey(draftId: string, chunkIndex: number, mode: 'hq' | 'fast', hash: string, sttKey: string): string {
+  return `${draftId}:${chunkIndex}:${mode}:${hash}:${sttKey}`
 }
 
-async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffer, mode: 'hq' | 'fast', purpose: 'warm' | 'final'): Promise<string> {
+/** The setting key a stored record was made under (pre-6.73 records: the 6.72.0 defaults). */
+export function recordSttKey(record: PromptDraftTranscriptRecord): string {
+  return record.sttKey ?? legacyDictationKey(record.requestedMode)
+}
+
+function planFor(mode: 'hq' | 'fast'): DictationPlan {
+  return currentDictationPlan(dictationSlotForMode(mode))
+}
+
+async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffer, mode: 'hq' | 'fast', purpose: 'warm' | 'final', plan: DictationPlan = planFor(mode)): Promise<string> {
   const hash = audioHash(audio)
-  const purposeKey = `${draftId}:${chunkIndex}:${purpose}:${mode}:${hash}`
-  const sharedKey = modeQualityKey(draftId, chunkIndex, mode, hash)
+  const purposeKey = `${draftId}:${chunkIndex}:${purpose}:${mode}:${hash}:${plan.key}`
+  const sharedKey = modeQualityKey(draftId, chunkIndex, mode, hash, plan.key)
 
   // HQ warm and HQ finalize must share one decode (plan step 8a).
   const existingShared = modeQualityJobs.get(sharedKey)
@@ -276,6 +289,9 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
           backend: warmMatches ? warm!.backend : 'shared-inflight',
           degraded: warmMatches ? warm!.degraded : false,
           ...(warmMatches && warm!.degradationReason ? { degradationReason: warm!.degradationReason } : {}),
+          sttKey: plan.key,
+          ...(warmMatches && warm!.engine ? { engine: warm!.engine } : {}),
+          ...(warmMatches && warm!.engineReason ? { engineReason: warm!.engineReason } : {}),
         }
         if (
           !cachedFinal
@@ -298,7 +314,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
     try {
       // Speculative warm is always local-only. Finalize may use automatic cloud fallback.
       const policy = purpose === 'warm' ? 'local-only' as const : 'automatic' as const
-      const result = await transcribeAudioBuffer(audio, { mode, policy })
+      const result = await transcribeAudioBuffer(audio, { mode, policy, dictationPlan: plan })
       if (!isCurrentChunk(draftId, chunkIndex, audio)) return ''
       // 6.45.4 — the sentence Whisper invents after the speaker stops. Never trims
       // audio; drops a trailing sentence only on the junk lexicon or on silence
@@ -313,9 +329,11 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
         text, hash, requestedMode: result.requestedMode, actualQuality: result.actualQuality,
         backend: result.backend, degraded: result.degraded,
         ...(result.degradationReason ? { degradationReason: result.degradationReason } : {}),
+        sttKey: result.dictationKey, engine: result.engine,
+        ...(result.engineReason ? { engineReason: result.engineReason } : {}),
       }
       await markPromptDraftChunkTranscript(draftId, chunkIndex, record, purpose)
-      console.log(`[prompt-draft] chunk ${draftId}/${chunkIndex}: ${result.elapsedMs.toFixed(1)}ms | ${result.backend} | ${purpose}/${mode} | ${text.length} chars`)
+      console.log(`[prompt-draft] chunk ${draftId}/${chunkIndex}: ${result.elapsedMs.toFixed(1)}ms | ${result.backend}${result.engineReason ? ` (${result.engineReason})` : ''} | ${purpose}/${mode} | ${text.length} chars`)
       return text
     } catch (err) {
       if (err instanceof NoSpeechDetectedError) {
@@ -324,7 +342,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
         // recognized failure shape instead of persisting either the artifact
         // or an empty gap. Other no-speech results retain existing behavior.
         if (mode === 'hq' && isCaptionCreditHallucination(err.rawText)) {
-          const fastJob = modeQualityJobs.get(modeQualityKey(draftId, chunkIndex, 'fast', hash))
+          const fastJob = [...modeQualityJobs.entries()].find(([key]) => key.startsWith(`${draftId}:${chunkIndex}:fast:${hash}:`))?.[1]
           if (fastJob) {
             try { await fastJob } catch { /* normal no-speech path below */ }
           }
@@ -342,6 +360,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
                 requestedMode: 'hq',
                 degraded: true,
                 degradationReason: 'hq_caption_credit_hallucination',
+                sttKey: plan.key,
               }, 'final')
             }
             console.warn(`[prompt-draft] dropped HQ caption-credit hallucination ${draftId}/${chunkIndex}; retained Fast text`)
@@ -349,7 +368,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
           }
         }
         await markPromptDraftChunkTranscript(draftId, chunkIndex, {
-          text: '', hash, requestedMode: mode, actualQuality: mode, backend: 'no-speech', degraded: false,
+          text: '', hash, requestedMode: mode, actualQuality: mode, backend: 'no-speech', degraded: false, sttKey: plan.key,
         }, purpose)
         return ''
       }
@@ -364,12 +383,15 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
   return job
 }
 
-function isReusableTranscript(
+export function isReusableTranscript(
   cached: PromptDraftTranscriptRecord | undefined,
   hash: string,
   mode: 'hq' | 'fast',
+  sttKey: string,
 ): boolean {
   if (!cached || cached.hash !== hash || cached.requestedMode !== mode) return false
+  // 6.73.0: a decode made under a different STT selection is decoded again.
+  if (recordSttKey(cached) !== sttKey) return false
   if (cached.backend.startsWith('legacy')) return false
   if (mode === 'fast') return cached.actualQuality === 'fast' || cached.actualQuality === 'cloud'
   // mode === 'hq'
@@ -385,12 +407,14 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
   if (!meta) throw Object.assign(new Error('draft not found'), { status: 404 })
   const texts: string[] = []
   const transcriptRecords: PromptDraftTranscriptRecord[] = []
+  // One STT snapshot for the whole finalize: a settings change mid-finalize never mixes selections.
+  const plan = planFor(mode)
   for (const chunk of readPromptDraftChunks(draftId)) {
     try {
       const hash = audioHash(chunk.audioBuffer)
       // Await in-flight HQ warm before deciding cache miss (plan step 8b belt).
       if (mode === 'hq') {
-        const inflight = modeQualityJobs.get(modeQualityKey(draftId, chunk.chunkIndex, 'hq', hash))
+        const inflight = modeQualityJobs.get(modeQualityKey(draftId, chunk.chunkIndex, 'hq', hash, plan.key))
         if (inflight) {
           try { await inflight } catch { /* finalize may retry with automatic below */ }
         }
@@ -401,8 +425,8 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
       // When Settings HQ is available, never treat a degraded turbo warm as final
       // — that left finished chats on Fast text while REVIEW looked "HQ ready".
       // Legacy records stay excluded (reconstructed provenance).
-      const reusable = isReusableTranscript(cached, hash, mode)
-      const raw = reusable ? cached!.text : await transcribeChunk(draftId, chunk.chunkIndex, chunk.audioBuffer, mode, 'final')
+      const reusable = isReusableTranscript(cached, hash, mode, plan.key)
+      const raw = reusable ? cached!.text : await transcribeChunk(draftId, chunk.chunkIndex, chunk.audioBuffer, mode, 'final', plan)
       const text = sanitizeTranscript(draftId, raw, !reusable)
       if (text.trim()) {
         texts.push(text.trim())
@@ -433,6 +457,8 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
   const degraded = mode === 'hq' && (actualQuality !== 'hq' || transcriptRecords.some(record => record.degraded))
   const backends = [...new Set(transcriptRecords.map(record => record.backend))]
   const degradationReason = transcriptRecords.find(record => record.degradationReason)?.degradationReason
+  const engines = [...new Set(transcriptRecords.map(record => record.engine ?? 'whisper'))]
+  const engineReason = transcriptRecords.find(record => record.engineReason)?.engineReason
   return {
     draftId,
     text: finalText,
@@ -445,6 +471,9 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
     degraded,
     backend: backends.length === 1 ? backends[0] : 'mixed',
     ...(degradationReason ? { degradationReason } : {}),
+    // 6.73.0, additive.
+    engine: engines.length === 1 ? engines[0] : engines.length === 0 ? plan.engine : 'mixed',
+    ...(engineReason ? { engineReason } : {}),
   }
 }
 
@@ -513,15 +542,24 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/peek', async (req, res) => {
     peekTail = peekTail.then(async () => {
       lease.setPhase('active')
       try {
-        const result = await transcribeWhisperPreview(audio)
+        // 6.73.0: the live preview follows the Preview model. Parakeet runs on the
+        // shared one-at-a-time preview lane; any miss is the 6.72.0 Whisper peek.
+        const plan = currentDictationPlan('preview')
+        let raw: string | null = null
+        let engine: 'whisper' | 'parakeet-ultra' = 'whisper'
+        if (plan.engine === 'parakeet-ultra') {
+          const outcome = await runParakeetDictation(audio, plan, 'preview')
+          if (outcome.ok) { raw = applyCorrections(outcome.text); engine = 'parakeet-ultra' }
+        }
+        if (raw === null) raw = (await transcribeWhisperPreview(audio)).text
         // 6.45.4: the same whole-sentence filler rule as the commit path, so an
         // invented closing line never paints on the lens only to vanish later.
-        const text = sanitizeTranscript(draftId, guardPromptTail({ text: result.text, speech: SPEECH_UNKNOWN }).text, false)
+        const text = sanitizeTranscript(draftId, guardPromptTail({ text: raw, speech: SPEECH_UNKNOWN }).text, false)
         if (!text) return
         if (!loadPromptDraftMeta(draftId)) return
         emitDisplay({
           type: 'prompt_transcript',
-          data: { draftId, chunkIndex, text, provisional: true, peekGen },
+          data: { draftId, chunkIndex, text, provisional: true, peekGen, engine },
         })
       } catch (err: any) {
         if (!(err instanceof NoSpeechDetectedError)) {
@@ -575,9 +613,10 @@ promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
       // transcript only after rechecking that this exact audio still owns the
       // chunk index; a retry/replacement must never paint stale words.
       if (!isCurrentChunk(req.params.draftId, chunkIndex, audio)) return
+      const warm = loadPromptDraftMeta(req.params.draftId)?.warmTranscripts?.[String(chunkIndex)]
       emitDisplay({
         type: 'prompt_transcript',
-        data: { draftId: req.params.draftId, chunkIndex, text },
+        data: { draftId: req.params.draftId, chunkIndex, text, ...(warm?.requestedMode === 'fast' && warm.engine ? { engine: warm.engine } : {}) },
       })
     }).catch(err => {
       console.warn(`[prompt-draft] warm transcription failed ${req.params.draftId}/${chunkIndex}: ${err.message}`)

package/server/routes/stt-settings.ts

@@ -0,0 +1,85 @@
+// Speech-to-text settings (6.72.0). Token-protected by the /api middleware.
+//
+//   GET  /api/stt-settings                    the settings, every option with availability, Parakeet status
+//   PUT  /api/stt-settings                    partial body: {"hq":"parakeet-ultra"}, {"resolver":false}, ...
+//   POST /api/stt-settings/parakeet/install   install the pinned Parakeet CLI and models (202; poll GET)
+//        body {} | {"localSource":"/abs/path"} | {"localSource":"...","allowDownload":true}
+//   DELETE /api/stt-settings/parakeet/install cancel a running install (nothing partial is kept)
+//
+// Clients find the route with GET: a 404 means an older server, so they hide the switcher.
+// The contract is shared with COS Control and the phone (see README "Speech-to-text settings").
+
+import { Router } from 'express'
+import { errMsg } from '../lib/utils.js'
+import { cancelParakeetInstall, installParakeet, parakeetInstalled, parakeetInstallRunning, type InstallOptions } from '../lib/parakeet.js'
+import { currentSttCapabilities, ensurePreviewWarm, parakeetStatus } from '../lib/stt-runtime.js'
+import {
+  applySettingsPatch, readLastSttJob, readSttSettings, settingsView, SttSettingsError, writeSttSettings,
+  type SttCapabilities,
+} from '../lib/stt-settings.js'
+
+export interface SttSettingsRouterDeps {
+  recordingLive: () => boolean
+  capabilities?: () => SttCapabilities
+  install?: (options: InstallOptions) => Promise<void>
+  onChanged?: () => void
+}
+
+export function createSttSettingsRouter(deps: SttSettingsRouterDeps): Router {
+  const router = Router()
+  const caps = deps.capabilities ?? currentSttCapabilities
+  const install = deps.install ?? installParakeet
+  const view = () => settingsView(readSttSettings(), caps(), parakeetStatus(deps.recordingLive()), readLastSttJob())
+
+  router.get('/stt-settings', (_req, res) => {
+    try { res.json(view()) } catch (err) { res.status(500).json({ error: errMsg(err) }) }
+  })
+
+  router.put('/stt-settings', (req, res) => {
+    try {
+      const next = applySettingsPatch(readSttSettings(), req.body, caps())
+      writeSttSettings(next)
+      ensurePreviewWarm()
+      deps.onChanged?.()
+      res.json(view())
+    } catch (err) {
+      if (err instanceof SttSettingsError) return void res.status(400).json({ error: err.message })
+      res.status(500).json({ error: errMsg(err) })
+    }
+  })
+
+  router.post('/stt-settings/parakeet/install', (req, res) => {
+    try {
+      if (deps.recordingLive()) return void res.status(409).json({ error: 'A recording is live. Install Parakeet after it is saved.' })
+      if (parakeetInstalled().installed && !parakeetInstallRunning()) return void res.status(200).json({ started: false, ...view() })
+      const body = (req.body ?? {}) as { localSource?: unknown; allowDownload?: unknown }
+      const raw = body.localSource
+      if (raw !== undefined && (typeof raw !== 'string' || !raw.startsWith('/') || raw.length > 1024)) {
+        return void res.status(400).json({ error: 'localSource must be an absolute path.' })
+      }
+      if (body.allowDownload !== undefined && typeof body.allowDownload !== 'boolean') return void res.status(400).json({ error: 'allowDownload must be true or false.' })
+      // Only bytes whose SHA-256 matches the pinned manifest are ever copied from it.
+      // A named localSource that lacks a file fails plainly unless allowDownload is true.
+      const options: InstallOptions = raw ? { localSources: [raw], requireLocal: body.allowDownload !== true } : {}
+      if (!parakeetInstallRunning()) {
+        void install(options).then(() => { ensurePreviewWarm(); deps.onChanged?.() }).catch(error => {
+          console.warn(`[stt] Parakeet install failed: ${errMsg(error)}`)
+        })
+      }
+      res.status(202).json({ started: true, ...view() })
+    } catch (err) {
+      res.status(500).json({ error: errMsg(err) })
+    }
+  })
+
+  router.delete('/stt-settings/parakeet/install', (_req, res) => {
+    try {
+      const cancelled = cancelParakeetInstall()
+      res.status(cancelled ? 202 : 200).json({ cancelled, ...view() })
+    } catch (err) {
+      res.status(500).json({ error: errMsg(err) })
+    }
+  })
+
+  return router
+}

package/server/routes/transcribe-stream.ts

@@ -37,7 +37,6 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
 import { emitDisplay } from '../lib/display-bus.js'
 import { errMsg } from '../lib/utils.js'
 import { transcribeLocal, applyCorrections, type WhisperWord } from '../lib/whisper-local.js'
-import { enhanceAudio } from '../lib/audio-enhance.js'
 import { trimSilence, isSileroAvailable } from '../lib/vad-silero.js'
 import { identifySpeaker, isEmbeddingAvailable, autoEnroll, AUTO_ENROLL_CANDIDATE_SIMILARITY } from '../lib/speaker-embeddings.js'
 import { nemotronLive, resolveLiveLabels, resplitSegments, type ChunkSpeakerSegment } from '../lib/nemotron-live.js'
@@ -56,7 +55,9 @@ import {
   streamSilenceDropReason,
   isVocabEchoOnly,
 } from '../lib/hallucination-filter.js'
-import { transcribeWhisperMeetingPreview } from '../lib/whisper-preview.js'
+import { transcribeMeetingPreview } from '../lib/stt-runtime.js'
+import { decideQuiet, asrFrontEnd, silenceDropsAllowed } from '../lib/stt-live.js'
+import { readSttSettings } from '../lib/stt-settings.js'
 import { dataPath } from '../lib/data-dir.js'
 import { ageHours, partitionExpiredAudio } from '../lib/audio-retention.js'
 import {
@@ -148,8 +149,12 @@ function ensurePrivateDirectory(path: string): void {
 
 // Silence-hallucination drops (2026-05-29, v5.9.73). Contract in streamSilenceDropReason:
 //   brand-URL-only -> dropped ALWAYS (vocab-seeded, never real speech).
-//   any-URL-only / repeated-thank-you -> dropped only when isQuiet (rms<150), so a real
-//   softly-spoken utterance or a clearly-dictated third-party URL is never dropped.
+//   any-URL-only / repeated-thank-you / quiet vocab echo -> dropped only when the chunk
+//   is RMS-quiet (rms<150), so a real softly-spoken utterance or a clearly-dictated
+//   third-party URL is never dropped. 6.72.0 keeps these TEXT drops on the RMS test on
+//   purpose: the Silero quiet gate (under 0.5 s of speech) also fires on a chunk holding
+//   one short real name, and dropping that as a "vocab echo" would delete real speech.
+//   The Silero gate decides only the prompt and the front end.
 // Rollback: COS_WHISPER_STRIP_BRAND_URLS=0 (URL drops), COS_WHISPER_THANKYOU_FILTER=0.
 const STRIP_BRAND_URLS = process.env.COS_WHISPER_STRIP_BRAND_URLS !== '0'
 const THANKYOU_FILTER = process.env.COS_WHISPER_THANKYOU_FILTER !== '0'
@@ -2219,16 +2224,16 @@ async function processStreamChunk(opts: {
     resolve(SESSION_AUDIO_DIR, sessionId, `chunk_${String(chunkIndex).padStart(4, '0')}.wav`),
   )
 
-  const pcmData = audioBuffer.subarray(44)
-  let sumSq = 0
-  const nSamples = Math.floor(pcmData.length / 2)
-  for (let i = 0; i < nSamples; i++) {
-    const s = pcmData.readInt16LE(i * 2)
-    sumSq += s * s
-  }
-  const rms = Math.sqrt(sumSq / Math.max(1, nSamples))
-  const isQuiet = rms < 150
-  const whisperAudio = isQuiet ? audioBuffer : await enhanceAudio(audioBuffer)
+  // 6.72.0: the quiet gate and the ASR front end follow the STT settings, read
+  // per chunk so a change applies from the next chunk. Defaults: Silero speech
+  // seconds (RMS 150 fired on 1% of G2 chunks) and highpass only (the full
+  // chain lifted faint background speech over the VAD). Playback audio is
+  // separate and unchanged.
+  const stt = readSttSettings()
+  const quiet = decideQuiet(audioBuffer, stt.quietGate)
+  const rms = quiet.rms
+  const isQuiet = quiet.isQuiet
+  const whisperAudio = isQuiet ? audioBuffer : await asrFrontEnd(audioBuffer, stt.frontEnd)
   const whisperContext = recentWhisperContext(sessionId, session)
   console.log(`[perf] whisperContext: ${whisperContext.length}b | session.chunks: ${session.chunks.length}`)
 
@@ -2265,7 +2270,9 @@ async function processStreamChunk(opts: {
   // uncorrected for those sources. Idempotent for the local path.
   rawText = applyCorrections(rawText)
 
-  let sanitized = sanitizeStreamTranscript(sessionId, session, rawText, isQuiet)
+  // Text drops stay on the pre-6.72 RMS test (see the contract above).
+  const dropQuiet = silenceDropsAllowed(quiet)
+  let sanitized = sanitizeStreamTranscript(sessionId, session, rawText, dropQuiet)
   if (candidate && (!sanitized.text || sanitized.fallbackReason)) {
     fallbackReason = sanitized.fallbackReason || 'empty_candidate'
     const result = await transcribeWithServerWhisper(audioBuffer, whisperAudio, whisperContext, isQuiet)
@@ -2273,7 +2280,7 @@ async function processStreamChunk(opts: {
     words = result.words
     backend = result.backend
     asrProvider = 'server-whisper'
-    sanitized = sanitizeStreamTranscript(sessionId, session, rawText, isQuiet)
+    sanitized = sanitizeStreamTranscript(sessionId, session, rawText, dropQuiet)
   }
 
   const voiceprint = await speakerPromise
@@ -2393,7 +2400,7 @@ async function processStreamChunk(opts: {
   // throws on by default. All gates live inside the feed.
   void feedLiveCueTranscript(sessionId, trimmedText, Boolean(fallbackReason)).catch(() => {})
 
-  console.log(`[perf] TOTAL request: ${(performance.now() - tReq).toFixed(1)}ms | chunk #${chunkIndex} | ${audioBuffer.length}b | rms=${Math.round(rms)} q=${isQuiet ? 1 : 0} | ${asrProvider} | "${trimmedText.slice(0, 50)}"`)
+  console.log(`[perf] TOTAL request: ${(performance.now() - tReq).toFixed(1)}ms | chunk #${chunkIndex} | ${audioBuffer.length}b | rms=${Math.round(rms)} q=${isQuiet ? 1 : 0} gate=${quiet.gate}${quiet.speechSec === null ? '' : ` speech=${quiet.speechSec.toFixed(2)}s`} fe=${stt.frontEnd} | ${asrProvider} | "${trimmedText.slice(0, 50)}"`)
   return canonicalChunkResponse(chunk, sessionId, chunkIndex)
 }
 
@@ -2420,13 +2427,15 @@ function sendStreamError(res: { status: (code: number) => { json: (body: unknown
 }
 
 /**
- * Default-off meeting preview canary.
+ * Meeting preview (on unless COS_WHISPER_MEETING_PREVIEW=0).
  *
  * This route deliberately owns no meeting session, recovery ledger, speaker
  * attribution, or persistence. It accepts a pinned copy of the still-open
- * phrase and returns provisional Turbo text. Any contention or failure is a
- * 204 cosmetic miss; the canonical /transcribe-stream upload remains the only
- * durable and speaker-attributed result.
+ * phrase and returns provisional text from the preview engine the STT settings
+ * choose (6.72.0: Whisper Turbo, or Parakeet Ultra when selected, warm, and the
+ * client lists `parakeet-ultra` in `previewModels`; every other client gets
+ * Turbo). Any contention or failure is a 204 cosmetic miss; the canonical
+ * /transcribe-stream upload remains the only durable and speaker-attributed result.
  */
 transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
   try {
@@ -2459,7 +2468,10 @@ transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
 
     meetingPreviewBusy = true
     try {
-      const result = await transcribeWhisperMeetingPreview(audio)
+      // 6.72.0: the models this client accepts. Absent (every phone before 6.10.639): Turbo only.
+      const rawModels = typeof req.query.previewModels === 'string' ? req.query.previewModels.slice(0, 200) : ''
+      const accepted = rawModels.split(',').map(m => m.trim()).filter(Boolean).slice(0, 8)
+      const result = await transcribeMeetingPreview(audio, accepted)
       if (!result) return void res.status(204).send()
       const cleaned = stripInlineHallucinationsOneShot(result.text.trim()).trim()
       if (!cleaned || sharedIsFullHallucination(cleaned)) {