@gotcos/glasses-server 6.71.14 → 6.72.0

package/server/routes/stt-settings.ts

@@ -0,0 +1,85 @@
+// Speech-to-text settings (6.72.0). Token-protected by the /api middleware.
+//
+//   GET  /api/stt-settings                    the settings, every option with availability, Parakeet status
+//   PUT  /api/stt-settings                    partial body: {"hq":"parakeet-ultra"}, {"resolver":false}, ...
+//   POST /api/stt-settings/parakeet/install   install the pinned Parakeet CLI and models (202; poll GET)
+//        body {} | {"localSource":"/abs/path"} | {"localSource":"...","allowDownload":true}
+//   DELETE /api/stt-settings/parakeet/install cancel a running install (nothing partial is kept)
+//
+// Clients find the route with GET: a 404 means an older server, so they hide the switcher.
+// The contract is shared with COS Control and the phone (see README "Speech-to-text settings").
+
+import { Router } from 'express'
+import { errMsg } from '../lib/utils.js'
+import { cancelParakeetInstall, installParakeet, parakeetInstalled, parakeetInstallRunning, type InstallOptions } from '../lib/parakeet.js'
+import { currentSttCapabilities, ensurePreviewWarm, parakeetStatus } from '../lib/stt-runtime.js'
+import {
+  applySettingsPatch, readLastSttJob, readSttSettings, settingsView, SttSettingsError, writeSttSettings,
+  type SttCapabilities,
+} from '../lib/stt-settings.js'
+
+export interface SttSettingsRouterDeps {
+  recordingLive: () => boolean
+  capabilities?: () => SttCapabilities
+  install?: (options: InstallOptions) => Promise<void>
+  onChanged?: () => void
+}
+
+export function createSttSettingsRouter(deps: SttSettingsRouterDeps): Router {
+  const router = Router()
+  const caps = deps.capabilities ?? currentSttCapabilities
+  const install = deps.install ?? installParakeet
+  const view = () => settingsView(readSttSettings(), caps(), parakeetStatus(deps.recordingLive()), readLastSttJob())
+
+  router.get('/stt-settings', (_req, res) => {
+    try { res.json(view()) } catch (err) { res.status(500).json({ error: errMsg(err) }) }
+  })
+
+  router.put('/stt-settings', (req, res) => {
+    try {
+      const next = applySettingsPatch(readSttSettings(), req.body, caps())
+      writeSttSettings(next)
+      ensurePreviewWarm()
+      deps.onChanged?.()
+      res.json(view())
+    } catch (err) {
+      if (err instanceof SttSettingsError) return void res.status(400).json({ error: err.message })
+      res.status(500).json({ error: errMsg(err) })
+    }
+  })
+
+  router.post('/stt-settings/parakeet/install', (req, res) => {
+    try {
+      if (deps.recordingLive()) return void res.status(409).json({ error: 'A recording is live. Install Parakeet after it is saved.' })
+      if (parakeetInstalled().installed && !parakeetInstallRunning()) return void res.status(200).json({ started: false, ...view() })
+      const body = (req.body ?? {}) as { localSource?: unknown; allowDownload?: unknown }
+      const raw = body.localSource
+      if (raw !== undefined && (typeof raw !== 'string' || !raw.startsWith('/') || raw.length > 1024)) {
+        return void res.status(400).json({ error: 'localSource must be an absolute path.' })
+      }
+      if (body.allowDownload !== undefined && typeof body.allowDownload !== 'boolean') return void res.status(400).json({ error: 'allowDownload must be true or false.' })
+      // Only bytes whose SHA-256 matches the pinned manifest are ever copied from it.
+      // A named localSource that lacks a file fails plainly unless allowDownload is true.
+      const options: InstallOptions = raw ? { localSources: [raw], requireLocal: body.allowDownload !== true } : {}
+      if (!parakeetInstallRunning()) {
+        void install(options).then(() => { ensurePreviewWarm(); deps.onChanged?.() }).catch(error => {
+          console.warn(`[stt] Parakeet install failed: ${errMsg(error)}`)
+        })
+      }
+      res.status(202).json({ started: true, ...view() })
+    } catch (err) {
+      res.status(500).json({ error: errMsg(err) })
+    }
+  })
+
+  router.delete('/stt-settings/parakeet/install', (_req, res) => {
+    try {
+      const cancelled = cancelParakeetInstall()
+      res.status(cancelled ? 202 : 200).json({ cancelled, ...view() })
+    } catch (err) {
+      res.status(500).json({ error: errMsg(err) })
+    }
+  })
+
+  return router
+}

package/server/routes/transcribe-stream.ts

@@ -37,7 +37,6 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
 import { emitDisplay } from '../lib/display-bus.js'
 import { errMsg } from '../lib/utils.js'
 import { transcribeLocal, applyCorrections, type WhisperWord } from '../lib/whisper-local.js'
-import { enhanceAudio } from '../lib/audio-enhance.js'
 import { trimSilence, isSileroAvailable } from '../lib/vad-silero.js'
 import { identifySpeaker, isEmbeddingAvailable, autoEnroll, AUTO_ENROLL_CANDIDATE_SIMILARITY } from '../lib/speaker-embeddings.js'
 import { nemotronLive, resolveLiveLabels, resplitSegments, type ChunkSpeakerSegment } from '../lib/nemotron-live.js'
@@ -56,7 +55,9 @@ import {
   streamSilenceDropReason,
   isVocabEchoOnly,
 } from '../lib/hallucination-filter.js'
-import { transcribeWhisperMeetingPreview } from '../lib/whisper-preview.js'
+import { transcribeMeetingPreview } from '../lib/stt-runtime.js'
+import { decideQuiet, asrFrontEnd, silenceDropsAllowed } from '../lib/stt-live.js'
+import { readSttSettings } from '../lib/stt-settings.js'
 import { dataPath } from '../lib/data-dir.js'
 import { ageHours, partitionExpiredAudio } from '../lib/audio-retention.js'
 import {
@@ -148,8 +149,12 @@ function ensurePrivateDirectory(path: string): void {
 
 // Silence-hallucination drops (2026-05-29, v5.9.73). Contract in streamSilenceDropReason:
 //   brand-URL-only -> dropped ALWAYS (vocab-seeded, never real speech).
-//   any-URL-only / repeated-thank-you -> dropped only when isQuiet (rms<150), so a real
-//   softly-spoken utterance or a clearly-dictated third-party URL is never dropped.
+//   any-URL-only / repeated-thank-you / quiet vocab echo -> dropped only when the chunk
+//   is RMS-quiet (rms<150), so a real softly-spoken utterance or a clearly-dictated
+//   third-party URL is never dropped. 6.72.0 keeps these TEXT drops on the RMS test on
+//   purpose: the Silero quiet gate (under 0.5 s of speech) also fires on a chunk holding
+//   one short real name, and dropping that as a "vocab echo" would delete real speech.
+//   The Silero gate decides only the prompt and the front end.
 // Rollback: COS_WHISPER_STRIP_BRAND_URLS=0 (URL drops), COS_WHISPER_THANKYOU_FILTER=0.
 const STRIP_BRAND_URLS = process.env.COS_WHISPER_STRIP_BRAND_URLS !== '0'
 const THANKYOU_FILTER = process.env.COS_WHISPER_THANKYOU_FILTER !== '0'
@@ -2219,16 +2224,16 @@ async function processStreamChunk(opts: {
     resolve(SESSION_AUDIO_DIR, sessionId, `chunk_${String(chunkIndex).padStart(4, '0')}.wav`),
   )
 
-  const pcmData = audioBuffer.subarray(44)
-  let sumSq = 0
-  const nSamples = Math.floor(pcmData.length / 2)
-  for (let i = 0; i < nSamples; i++) {
-    const s = pcmData.readInt16LE(i * 2)
-    sumSq += s * s
-  }
-  const rms = Math.sqrt(sumSq / Math.max(1, nSamples))
-  const isQuiet = rms < 150
-  const whisperAudio = isQuiet ? audioBuffer : await enhanceAudio(audioBuffer)
+  // 6.72.0: the quiet gate and the ASR front end follow the STT settings, read
+  // per chunk so a change applies from the next chunk. Defaults: Silero speech
+  // seconds (RMS 150 fired on 1% of G2 chunks) and highpass only (the full
+  // chain lifted faint background speech over the VAD). Playback audio is
+  // separate and unchanged.
+  const stt = readSttSettings()
+  const quiet = decideQuiet(audioBuffer, stt.quietGate)
+  const rms = quiet.rms
+  const isQuiet = quiet.isQuiet
+  const whisperAudio = isQuiet ? audioBuffer : await asrFrontEnd(audioBuffer, stt.frontEnd)
   const whisperContext = recentWhisperContext(sessionId, session)
   console.log(`[perf] whisperContext: ${whisperContext.length}b | session.chunks: ${session.chunks.length}`)
 
@@ -2265,7 +2270,9 @@ async function processStreamChunk(opts: {
   // uncorrected for those sources. Idempotent for the local path.
   rawText = applyCorrections(rawText)
 
-  let sanitized = sanitizeStreamTranscript(sessionId, session, rawText, isQuiet)
+  // Text drops stay on the pre-6.72 RMS test (see the contract above).
+  const dropQuiet = silenceDropsAllowed(quiet)
+  let sanitized = sanitizeStreamTranscript(sessionId, session, rawText, dropQuiet)
   if (candidate && (!sanitized.text || sanitized.fallbackReason)) {
     fallbackReason = sanitized.fallbackReason || 'empty_candidate'
     const result = await transcribeWithServerWhisper(audioBuffer, whisperAudio, whisperContext, isQuiet)
@@ -2273,7 +2280,7 @@ async function processStreamChunk(opts: {
     words = result.words
     backend = result.backend
     asrProvider = 'server-whisper'
-    sanitized = sanitizeStreamTranscript(sessionId, session, rawText, isQuiet)
+    sanitized = sanitizeStreamTranscript(sessionId, session, rawText, dropQuiet)
   }
 
   const voiceprint = await speakerPromise
@@ -2393,7 +2400,7 @@ async function processStreamChunk(opts: {
   // throws on by default. All gates live inside the feed.
   void feedLiveCueTranscript(sessionId, trimmedText, Boolean(fallbackReason)).catch(() => {})
 
-  console.log(`[perf] TOTAL request: ${(performance.now() - tReq).toFixed(1)}ms | chunk #${chunkIndex} | ${audioBuffer.length}b | rms=${Math.round(rms)} q=${isQuiet ? 1 : 0} | ${asrProvider} | "${trimmedText.slice(0, 50)}"`)
+  console.log(`[perf] TOTAL request: ${(performance.now() - tReq).toFixed(1)}ms | chunk #${chunkIndex} | ${audioBuffer.length}b | rms=${Math.round(rms)} q=${isQuiet ? 1 : 0} gate=${quiet.gate}${quiet.speechSec === null ? '' : ` speech=${quiet.speechSec.toFixed(2)}s`} fe=${stt.frontEnd} | ${asrProvider} | "${trimmedText.slice(0, 50)}"`)
   return canonicalChunkResponse(chunk, sessionId, chunkIndex)
 }
 
@@ -2420,13 +2427,15 @@ function sendStreamError(res: { status: (code: number) => { json: (body: unknown
 }
 
 /**
- * Default-off meeting preview canary.
+ * Meeting preview (on unless COS_WHISPER_MEETING_PREVIEW=0).
  *
  * This route deliberately owns no meeting session, recovery ledger, speaker
  * attribution, or persistence. It accepts a pinned copy of the still-open
- * phrase and returns provisional Turbo text. Any contention or failure is a
- * 204 cosmetic miss; the canonical /transcribe-stream upload remains the only
- * durable and speaker-attributed result.
+ * phrase and returns provisional text from the preview engine the STT settings
+ * choose (6.72.0: Whisper Turbo, or Parakeet Ultra when selected, warm, and the
+ * client lists `parakeet-ultra` in `previewModels`; every other client gets
+ * Turbo). Any contention or failure is a 204 cosmetic miss; the canonical
+ * /transcribe-stream upload remains the only durable and speaker-attributed result.
  */
 transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
   try {
@@ -2459,7 +2468,10 @@ transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
 
     meetingPreviewBusy = true
     try {
-      const result = await transcribeWhisperMeetingPreview(audio)
+      // 6.72.0: the models this client accepts. Absent (every phone before 6.10.639): Turbo only.
+      const rawModels = typeof req.query.previewModels === 'string' ? req.query.previewModels.slice(0, 200) : ''
+      const accepted = rawModels.split(',').map(m => m.trim()).filter(Boolean).slice(0, 8)
+      const result = await transcribeMeetingPreview(audio, accepted)
       if (!result) return void res.status(204).send()
       const cleaned = stripInlineHallucinationsOneShot(result.text.trim()).trim()
       if (!cleaned || sharedIsFullHallucination(cleaned)) {