@gotcos/glasses-server 6.14.0 → 6.15.0

package/server/lib/whisper-local.ts

@@ -8,7 +8,7 @@
 
 import { spawn, execFileSync } from 'node:child_process'
 import type { ChildProcess } from 'node:child_process'
-import { writeFileSync, unlinkSync, existsSync } from 'node:fs'
+import { writeFileSync, unlinkSync, existsSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { homedir } from 'node:os'
 import crypto from 'node:crypto'
@@ -42,9 +42,8 @@ export interface WhisperSegment {
   words?: WhisperWord[]
 }
 
-interface WhisperVerboseResponse {
-  text: string
-  segments?: WhisperSegment[]
+interface WhisperJsonResponse {
+  text?: unknown
 }
 
 // Resolve whisper.cpp binaries across Homebrew prefixes (Apple Silicon
@@ -144,6 +143,7 @@ let cliAvailable = false
 let serverAvailable = false
 let serverProcess: ReturnType<typeof spawn> | null = null
 const ownedServerChildren = new Set<ChildProcess>()
+const ownedHqChildren = new Set<ReturnType<typeof spawn>>()
 
 // Circuit breaker: track consecutive server failures to detect hung process
 let serverConsecutiveFailures = 0
@@ -389,7 +389,9 @@ async function startWhisperServerAttempt(preflightCompleted = false): Promise<vo
   console.log('[whisper-local] Starting whisper-server...')
 
   const child = spawn(WHISPER_SERVER, serverArgs, {
-    stdio: ['ignore', 'pipe', 'pipe'],
+    // whisper-server writes per-inference diagnostics. Unread pipes eventually
+    // fill and block the daemon, so the supervisor must not leave them buffered.
+    stdio: 'ignore',
     detached: false,  // Dies with parent
   })
   serverProcess = child
@@ -455,6 +457,10 @@ export function stopWhisperServer(): void {
     serverAvailable = false
     console.log('[whisper-local] whisper-server stopped')
   }
+  for (const proc of ownedHqChildren) {
+    try { proc.kill('SIGKILL') } catch { /* already exited */ }
+  }
+  ownedHqChildren.clear()
 }
 
 export function isWhisperLocalAvailable(): boolean {
@@ -516,6 +522,54 @@ async function reconcileWhisperServerHealth(): Promise<boolean> {
   return serverHealthProbe
 }
 
+/**
+ * Parse whisper-cli `-ojf` JSON into plain text + timed words.
+ *
+ * Batch/save path only. Never uses whisper-server `verbose_json` (the live
+ * VAD-empty crash vector). Special tokens like `[_BEG_]` / `<|...|>` are dropped.
+ */
+export function parseWhisperCliFullJson(raw: string): { text: string; words: WhisperWord[] } {
+  const data = JSON.parse(raw) as {
+    transcription?: Array<{
+      text?: unknown
+      tokens?: Array<{
+        text?: unknown
+        p?: unknown
+        offsets?: { from?: unknown; to?: unknown }
+      }>
+    }>
+  }
+  const segments = Array.isArray(data.transcription) ? data.transcription : []
+  const texts: string[] = []
+  const words: WhisperWord[] = []
+
+  for (const segment of segments) {
+    if (typeof segment.text === 'string' && segment.text.trim()) {
+      texts.push(segment.text.trim())
+    }
+    for (const token of Array.isArray(segment.tokens) ? segment.tokens : []) {
+      const tokenText = typeof token.text === 'string' ? token.text.trim() : ''
+      if (!tokenText) continue
+      if (tokenText.startsWith('[') && tokenText.endsWith(']')) continue
+      if (tokenText.startsWith('<|') && tokenText.endsWith('|>')) continue
+      const fromMs = typeof token.offsets?.from === 'number' ? token.offsets.from : Number(token.offsets?.from)
+      const toMs = typeof token.offsets?.to === 'number' ? token.offsets.to : Number(token.offsets?.to)
+      if (!Number.isFinite(fromMs) || !Number.isFinite(toMs)) continue
+      words.push({
+        word: tokenText,
+        start: fromMs / 1000,
+        end: toMs / 1000,
+        probability: typeof token.p === 'number' && Number.isFinite(token.p) ? token.p : 0,
+      })
+    }
+  }
+
+  return {
+    text: texts.join(' ').replace(/\s+/g, ' ').trim(),
+    words,
+  }
+}
+
 /**
  * High-quality transcription for batch/post-meeting use.
  * Uses the full Whisper large-v3 (32-layer) decoder + Silero VAD + beam search.
@@ -523,7 +577,11 @@ async function reconcileWhisperServerHealth(): Promise<boolean> {
  * Falls back to turbo weights if large-v3 not on disk or COS_BATCH_LARGE_V3=0.
  * Falls back to transcribeLocal if whisper-cli unavailable entirely.
  */
-export async function transcribeHighQuality(audioBuffer: Buffer, context?: string): Promise<{ text: string; words?: WhisperWord[] }> {
+export async function transcribeHighQuality(
+  audioBuffer: Buffer,
+  context?: string,
+  opts: { priority?: 'interactive' | 'batch' } = {},
+): Promise<{ text: string; words?: WhisperWord[] }> {
   if (!cliAvailable) {
     // Fall back to server (no beam search available via HTTP API)
     return transcribeLocal(audioBuffer, context)
@@ -532,6 +590,10 @@ export async function transcribeHighQuality(audioBuffer: Buffer, context?: strin
   const start = Date.now()
   const id = crypto.randomUUID().slice(0, 8)
   const tmpWav = join('/tmp', `cos-whisper-hq-${id}.wav`)
+  const outBase = join('/tmp', `cos-whisper-hq-${id}`)
+  const jsonPath = `${outBase}.json`
+  // Word clocks only on post-meeting CPU polish. Live stays compact JSON.
+  const captureBatchWords = opts.priority === 'batch'
 
   const modelPath = resolveBatchModel()
   const useLargeV3 = modelPath === BATCH_MODEL_LARGE_V3
@@ -541,18 +603,22 @@ export async function transcribeHighQuality(audioBuffer: Buffer, context?: strin
     writeFileSync(tmpWav, audioBuffer)
 
     const text = await new Promise<string>((resolve, reject) => {
+      const isolateBatchFromLiveMetal = opts.priority === 'batch'
       const args = [
         '-m', modelPath,
         '-f', tmpWav,
-        '-t', '16',           // Use more threads for batch (no real-time pressure)
+        '-t', isolateBatchFromLiveMetal ? '8' : '16',
         '-l', 'en',
-        '-fa',                // Flash attention — Metal win, same flag streaming uses
+        ...(isolateBatchFromLiveMetal ? ['-ng'] : ['-fa']),
         '-bs', '5',           // Beam search width 5 (default disabled)
         '-bo', '5',           // Best-of-5 candidates (default 2)
         '--no-timestamps',
         '-np',
         '--prompt', buildPrompt(context),
       ]
+      if (captureBatchWords) {
+        args.push('-ojf', '-of', outBase)
+      }
       if (useVad) {
         // Same VAD model the streaming path uses. Strips silence windows
         // before the decoder sees them — prevents the silence-hallucination
@@ -562,6 +628,7 @@ export async function transcribeHighQuality(audioBuffer: Buffer, context?: strin
       const proc = spawn(WHISPER_CLI, args, {
         stdio: ['ignore', 'pipe', 'pipe'],
       })
+      ownedHqChildren.add(proc)
 
       let stdout = ''
       let stderr = ''
@@ -573,13 +640,24 @@ export async function transcribeHighQuality(audioBuffer: Buffer, context?: strin
       // multiplier × beam-search overhead = ~240s safety ceiling for HQ.
       // Turbo retains the old 60s ceiling.
       const timeoutMs = useLargeV3 ? 240_000 : 60_000
+      let timedOut = false
+      let forceKill: ReturnType<typeof setTimeout> | null = null
       const timeout = setTimeout(() => {
-        proc.kill('SIGTERM')
-        reject(new Error(`whisper-cli HQ timeout (${timeoutMs / 1000}s, model=${useLargeV3 ? 'large-v3' : 'turbo'})`))
+        timedOut = true
+        try { proc.kill('SIGTERM') } catch { /* already exited */ }
+        forceKill = setTimeout(() => {
+          try { proc.kill('SIGKILL') } catch { /* already exited */ }
+        }, 2_000)
       }, timeoutMs)
 
       proc.on('close', (code) => {
+        ownedHqChildren.delete(proc)
         clearTimeout(timeout)
+        if (forceKill) clearTimeout(forceKill)
+        if (timedOut) {
+          reject(new Error(`whisper-cli HQ timeout (${timeoutMs / 1000}s, model=${useLargeV3 ? 'large-v3' : 'turbo'})`))
+          return
+        }
         if (code !== 0) {
           reject(new Error(`whisper-cli HQ exit ${code}: ${stderr.trim().slice(0, 200)}`))
           return
@@ -588,18 +666,47 @@ export async function transcribeHighQuality(audioBuffer: Buffer, context?: strin
       })
 
       proc.on('error', (err) => {
+        ownedHqChildren.delete(proc)
         clearTimeout(timeout)
+        if (forceKill) clearTimeout(forceKill)
         reject(new Error(`whisper-cli HQ spawn error: ${err.message}`))
       })
     })
 
-    const corrected = applyCorrections(text)
+    let finalText = text
+    let words: WhisperWord[] | undefined
+    if (captureBatchWords) {
+      try {
+        if (existsSync(jsonPath)) {
+          const parsed = parseWhisperCliFullJson(readFileSync(jsonPath, 'utf8'))
+          if (parsed.text) finalText = parsed.text
+          words = parsed.words.length > 0
+            ? parsed.words.map(w => ({ ...w, word: applyCorrections(w.word) }))
+            : []
+        }
+      } catch (err) {
+        console.warn(
+          `[whisper-hq] Batch word JSON parse failed; keeping text-only polish: ` +
+          `${err instanceof Error ? err.message : String(err)}`,
+        )
+      }
+    }
+
+    const corrected = applyCorrections(finalText)
     const elapsed = Date.now() - start
     const modelTag = useLargeV3 ? 'large-v3' : 'turbo'
-    console.log(`[whisper-hq] Batch transcribed in ${elapsed}ms (${modelTag}${useVad ? '+vad' : ''}): "${corrected.slice(0, 80)}${corrected.length > 80 ? '...' : ''}"`)
-    return { text: corrected }
+    console.log(
+      `[whisper-hq] Batch transcribed in ${elapsed}ms ` +
+      `(${modelTag}${useVad ? '+vad' : ''}${captureBatchWords ? '+words' : ''}` +
+      `${words ? `, ${words.length} words` : ''}): ` +
+      `"${corrected.slice(0, 80)}${corrected.length > 80 ? '...' : ''}"`,
+    )
+    return words ? { text: corrected, words } : { text: corrected }
   } finally {
     try { unlinkSync(tmpWav) } catch { /* cleanup */ }
+    if (captureBatchWords) {
+      try { unlinkSync(jsonPath) } catch { /* cleanup */ }
+    }
   }
 }
 
@@ -626,14 +733,17 @@ function buildPrompt(context?: string, isQuiet?: boolean): string {
 
 /**
  * Transcribe via whisper-server (persistent daemon, ~50-100ms).
- * Returns text + optional word-level timestamps from DTW alignment.
+ *
+ * Compact JSON intentionally avoids whisper.cpp's verbose_json language field,
+ * which can receive a null C string after VAD returns no speech and crash the
+ * native server before an HTTP response exists.
  */
 async function transcribeViaServer(audioBuffer: Buffer, context?: string, isQuiet?: boolean): Promise<{ text: string; words?: WhisperWord[] }> {
   const formData = new FormData()
   // Convert Buffer to Uint8Array to satisfy Blob's BlobPart type constraint
   const blob = new Blob([new Uint8Array(audioBuffer)], { type: 'audio/wav' })
   formData.append('file', blob, 'recording.wav')
-  formData.append('response_format', 'verbose_json')  // includes segments[].words[] with DTW
+  formData.append('response_format', 'json')
   formData.append('prompt', buildPrompt(context, isQuiet))
   // Anti-hallucination handled by client-side filter + context filtering.
   // Whisper-level entropy/logprob thresholds were too aggressive — silently dropped
@@ -650,20 +760,11 @@ async function transcribeViaServer(audioBuffer: Buffer, context?: string, isQuie
     throw new Error(`whisper-server ${response.status}: ${await response.text()}`)
   }
 
-  const result = await response.json() as WhisperVerboseResponse
-  const text = result.text?.trim() || ''
-
-  // Extract word-level timestamps from DTW-aligned segments (defensive — may be absent)
-  let words: WhisperWord[] | undefined
-  if (result.segments && result.segments.length > 0) {
-    const extracted = result.segments.flatMap(s => {
-      if (!s.words || !Array.isArray(s.words)) return []
-      return s.words.filter(w => typeof w.start === 'number' && typeof w.end === 'number')
-    })
-    if (extracted.length > 0) words = extracted
+  const result = await response.json() as WhisperJsonResponse
+  if (typeof result.text !== 'string') {
+    throw new Error('whisper-server returned invalid compact JSON: missing string text')
   }
-
-  return { text, words }
+  return { text: result.text.trim() }
 }
 
 /**

package/server/routes/health.ts

@@ -24,6 +24,7 @@ import { CLI_DEBUG_CAPABILITY } from '../lib/cli-debug-view.js'
 import { managedRuntimeCapability, managedServerVersion } from '../lib/managed-runtime.js'
 import { getServerGenerationId } from '../lib/managed-runtime.js'
 import { maintenanceLifecycle } from '../lib/maintenance-lifecycle.js'
+import { getLocalTtsHealth, refreshLocalTtsHealth } from '../lib/tts-local.js'
 
 export const healthRouter = Router()
 
@@ -57,6 +58,7 @@ function durableQueryJobStatus() {
 }
 
 healthRouter.get('/health', async (_req, res) => {
+  await refreshLocalTtsHealth()
   const checks: Record<string, string | number | boolean> = {
     status: 'ok',
     mode: COS_MODE ? 'cos' : 'standalone',
@@ -150,10 +152,11 @@ healthRouter.get('/health', async (_req, res) => {
   const transcription = getTranscriptionPolicySnapshot()
   const recovery = managedRuntimeCapability()
   const maintenance = maintenanceLifecycle.snapshot()
+  const tts_local = getLocalTtsHealth()
   const features = {
     claude: claudeAvailable,
     codex: codexAvailable,
-    voice: keyStatus.hasKey,
+    voice: keyStatus.hasKey || tts_local.ready,
     cos_pipeline: COS_MODE,
     whisper: isWhisperLocalAvailable(),
     promptRecovery: true,
@@ -167,15 +170,17 @@ healthRouter.get('/health', async (_req, res) => {
     transcriptionPolicy: transcription.mode,
   }
   const voice = {
+    available: keyStatus.hasKey || tts_local.ready,
     hasKey: keyStatus.hasKey,
     keySource: keyStatus.source,
+    localReady: tts_local.ready,
+    engine: tts_local.engine,
   }
 
   // Whisper-server health + cloud budget — exposed so glasses + dashboards can
   // see whether we're at risk of falling to cloud and how much budget remains.
   const whisper_health = getWhisperHealth()
   const openai_whisper_budget = getOpenAIWhisperBudgetState()
-
   const codex_models = getCodexModelCatalogSnapshot()
   res.json({
     ...checks,
@@ -187,6 +192,7 @@ healthRouter.get('/health', async (_req, res) => {
     voice,
     whisper_health,
     openai_whisper_budget,
+    tts_local,
     codex_models,
     capabilities: {
       transcription,