@gotcos/glasses-server 6.18.7 → 6.19.0

package/server/lib/unsaved-audio-quarantine.ts

@@ -0,0 +1,314 @@
+// Quarantine for unsaved meeting audio.
+//
+// Until 6.19.0 the session-audio purge (boot sweep + 60s interval) DELETED any
+// session-audio directory that was not in the in-memory sessions map and did
+// not carry a fresh save-preserved marker. An offline meeting whose deferred
+// save never landed therefore lost its full-fidelity audio within a minute of
+// the server no longer tracking the session — two real meetings were destroyed
+// this way on 2026-08-01. The only surviving fragments were in ext-audio,
+// which is a speaker-enrollment store (unrecognized-speaker chunks, hard
+// capped), not meeting audio.
+//
+// The rule now: audio evidence is moved here, never deleted in place. A
+// quarantined capture is surfaced on /api/health (unsaved_captures) and can be
+// driven to a durable scribe via POST /api/meeting/orphans/:sessionId/recover.
+// Quarantine expires on a retention clock (default 72h) — long enough to
+// survive a weekend away from the Mac, bounded enough not to grow forever.
+
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs'
+import { basename, join, resolve } from 'node:path'
+import { dataPath } from './data-dir.js'
+
+export const QUARANTINE_MANIFEST = '_quarantine.json'
+export const RECOVERED_RECEIPT = '_recovered.json'
+const CHUNK_WAV_RE = /^chunk_\d{4}\.wav$/
+
+const DEFAULT_RETENTION_HOURS = 72
+const MIN_RETENTION_HOURS = 1
+const MAX_RETENTION_HOURS = 720
+
+export interface QuarantineManifest {
+  schemaVersion: 1
+  sessionId: string
+  quarantinedAt: string
+  reason: string
+  chunkFiles: number
+  bytes: number
+}
+
+export interface UnsavedCapture {
+  sessionId: string
+  dirName: string
+  quarantinedAt: string | null
+  ageHours: number | null
+  chunkFiles: number
+  bytes: number
+  reason: string | null
+  expiresAt: string | null
+  recovered: boolean
+}
+
+export function unsavedAudioRoot(): string {
+  return dataPath('unsaved-audio')
+}
+
+export function unsavedAudioRetentionMs(env: NodeJS.ProcessEnv = process.env): number {
+  // An EMPTY value means unset, not zero: Number('') === 0, which would clamp
+  // to the 1h minimum and silently collapse the 72h safety net — on the one
+  // knob whose failure mode is losing the audio this module exists to keep.
+  const rawText = (env.COS_UNSAVED_AUDIO_RETENTION_HOURS ?? '').trim()
+  if (rawText === '') return DEFAULT_RETENTION_HOURS * 60 * 60 * 1000
+  const raw = Number(rawText)
+  const hours = Number.isFinite(raw)
+    ? Math.min(MAX_RETENTION_HOURS, Math.max(MIN_RETENTION_HOURS, raw))
+    : DEFAULT_RETENTION_HOURS
+  return hours * 60 * 60 * 1000
+}
+
+export function countChunkWavs(dirPath: string): number {
+  try {
+    return readdirSync(dirPath).filter(name => CHUNK_WAV_RE.test(name)).length
+  } catch {
+    return 0
+  }
+}
+
+function dirBytes(dirPath: string): number {
+  let total = 0
+  try {
+    for (const name of readdirSync(dirPath)) {
+      try { total += statSync(join(dirPath, name)).size } catch { /* skip */ }
+    }
+  } catch { /* empty */ }
+  return total
+}
+
+/** Move one session-audio dir into quarantine instead of deleting it.
+ *  Returns the quarantine path, or null when the move could not be made
+ *  (in which case the SOURCE IS LEFT IN PLACE — never deleted on failure). */
+export function quarantineSessionAudio(
+  sourceDir: string,
+  reason: string,
+  root: string = unsavedAudioRoot(),
+): string | null {
+  const sessionId = basename(sourceDir)
+  try {
+    if (!existsSync(sourceDir)) return null
+    mkdirSync(root, { recursive: true, mode: 0o700 })
+    let target = resolve(root, sessionId)
+    if (existsSync(target)) target = resolve(root, `${sessionId}.${Date.now()}`)
+    renameSync(sourceDir, target)
+    const manifest: QuarantineManifest = {
+      schemaVersion: 1,
+      sessionId,
+      quarantinedAt: new Date().toISOString(),
+      reason,
+      chunkFiles: countChunkWavs(target),
+      bytes: dirBytes(target),
+    }
+    try {
+      writeFileSync(resolve(target, QUARANTINE_MANIFEST), `${JSON.stringify(manifest)}\n`, {
+        encoding: 'utf8',
+        mode: 0o600,
+      })
+    } catch { /* manifest is observability; the audio move already succeeded */ }
+    return target
+  } catch {
+    // Rename failed (cross-device, permissions, race). Leave the source alone —
+    // a skipped purge is recoverable, a deleted capture is not.
+    return null
+  }
+}
+
+export type OrphanSweepAction =
+  | { dir: string; action: 'kept_live' }
+  | { dir: string; action: 'kept_preserved' }
+  | { dir: string; action: 'quarantined'; target: string }
+  | { dir: string; action: 'quarantine_failed' }
+  | { dir: string; action: 'deleted_empty' }
+
+/** Shared sweep for the boot path and the 60s interval. A directory that
+ *  still holds chunk audio is quarantined; only chunk-less directories are
+ *  deleted. A failed quarantine move keeps the source in place. */
+export function sweepOrphanedSessionAudio(
+  sessionAudioRoot: string,
+  opts: {
+    isLive: (sessionId: string) => boolean
+    hasFreshPreservedMarker: (dirPath: string) => boolean
+    reason: string
+    quarantineRoot?: string
+  },
+): OrphanSweepAction[] {
+  const actions: OrphanSweepAction[] = []
+  let entries: string[] = []
+  try {
+    entries = readdirSync(sessionAudioRoot)
+  } catch {
+    return actions
+  }
+  for (const dir of entries) {
+    const dirPath = resolve(sessionAudioRoot, dir)
+    try {
+      if (!statSync(dirPath).isDirectory()) continue
+    } catch { continue }
+    if (opts.isLive(dir)) { actions.push({ dir, action: 'kept_live' }); continue }
+    if (opts.hasFreshPreservedMarker(dirPath)) { actions.push({ dir, action: 'kept_preserved' }); continue }
+    if (countChunkWavs(dirPath) > 0) {
+      const target = quarantineSessionAudio(dirPath, opts.reason, opts.quarantineRoot)
+      actions.push(target
+        ? { dir, action: 'quarantined', target }
+        : { dir, action: 'quarantine_failed' })
+      continue
+    }
+    try {
+      rmSync(dirPath, { recursive: true, force: true })
+      actions.push({ dir, action: 'deleted_empty' })
+    } catch { /* next sweep retries */ }
+  }
+  return actions
+}
+
+function readManifest(dirPath: string): QuarantineManifest | null {
+  try {
+    const raw = JSON.parse(readFileSync(resolve(dirPath, QUARANTINE_MANIFEST), 'utf8')) as QuarantineManifest
+    if (raw?.schemaVersion !== 1 || typeof raw.sessionId !== 'string') return null
+    return raw
+  } catch {
+    return null
+  }
+}
+
+function quarantinedAtMs(dirPath: string, manifest: QuarantineManifest | null): number | null {
+  if (manifest) {
+    const parsed = Date.parse(manifest.quarantinedAt)
+    if (Number.isFinite(parsed)) return parsed
+  }
+  try {
+    return statSync(dirPath).mtimeMs
+  } catch {
+    return null
+  }
+}
+
+/** Everything currently in quarantine, newest first. Recovered captures are
+ *  flagged (they linger until the retention clock clears them) so health can
+ *  exclude them from the actionable count. */
+export function listUnsavedCaptures(
+  root: string = unsavedAudioRoot(),
+  retentionMs: number = unsavedAudioRetentionMs(),
+): UnsavedCapture[] {
+  const captures: UnsavedCapture[] = []
+  let entries: string[] = []
+  try {
+    entries = readdirSync(root)
+  } catch {
+    return captures
+  }
+  for (const dir of entries) {
+    const dirPath = resolve(root, dir)
+    try {
+      if (!statSync(dirPath).isDirectory()) continue
+    } catch { continue }
+    const manifest = readManifest(dirPath)
+    const atMs = quarantinedAtMs(dirPath, manifest)
+    captures.push({
+      sessionId: manifest?.sessionId ?? dir.replace(/\.\d+$/, ''),
+      dirName: dir,
+      quarantinedAt: atMs != null ? new Date(atMs).toISOString() : null,
+      ageHours: atMs != null ? Math.round(((Date.now() - atMs) / 3_600_000) * 10) / 10 : null,
+      chunkFiles: manifest?.chunkFiles ?? countChunkWavs(dirPath),
+      bytes: manifest?.bytes ?? dirBytes(dirPath),
+      reason: manifest?.reason ?? null,
+      expiresAt: atMs != null ? new Date(atMs + retentionMs).toISOString() : null,
+      recovered: existsSync(resolve(dirPath, RECOVERED_RECEIPT)),
+    })
+  }
+  captures.sort((a, b) => (b.quarantinedAt ?? '').localeCompare(a.quarantinedAt ?? ''))
+  return captures
+}
+
+/** Retention: quarantined captures past the clock are removed. This is the
+ *  ONLY place quarantined audio is ever deleted. */
+export function purgeExpiredQuarantine(
+  root: string = unsavedAudioRoot(),
+  retentionMs: number = unsavedAudioRetentionMs(),
+): string[] {
+  const purged: string[] = []
+  for (const capture of listUnsavedCaptures(root, retentionMs)) {
+    // Never delete a directory a recovery is actively decoding — a capture
+    // sitting exactly at the retention boundary would otherwise lose its
+    // audio mid-run, in the module whose thesis is "never delete evidence".
+    if (isRecoveryActive(capture.dirName)) continue
+    const atMs = capture.quarantinedAt ? Date.parse(capture.quarantinedAt) : NaN
+    if (!Number.isFinite(atMs)) continue
+    if (Date.now() - atMs > retentionMs) {
+      try {
+        rmSync(resolve(root, capture.dirName), { recursive: true, force: true })
+        purged.push(capture.dirName)
+      } catch { /* next sweep retries */ }
+    }
+  }
+  return purged
+}
+
+/** Resolve the quarantine dir for a sessionId (exact dir or timestamp-suffixed). */
+export function findQuarantineDir(
+  sessionId: string,
+  root: string = unsavedAudioRoot(),
+): string | null {
+  const exact = resolve(root, sessionId)
+  if (existsSync(exact)) return exact
+  try {
+    for (const dir of readdirSync(root)) {
+      if (dir === sessionId || dir.startsWith(`${sessionId}.`)) {
+        const dirPath = resolve(root, dir)
+        if (statSync(dirPath).isDirectory()) return dirPath
+      }
+    }
+  } catch { /* fall through */ }
+  return null
+}
+
+// ── Active recovery registry ─────────────────────────────────────────────
+// In-memory, single-process (matches the sessions map's own scope). Two
+// consumers: purgeExpiredQuarantine skips dirs being recovered (a capture at
+// the retention boundary must not be deleted mid-decode), and meeting_sync
+// renders an active row so COS Control warns BEFORE committing an Update
+// Server drain into a 20-90 minute recovery it cannot see.
+
+const activeRecoveries = new Map<string, { sessionId: string; dirPath: string; startedAt: number }>()
+
+export function registerActiveRecovery(sessionId: string, dirPath: string): void {
+  activeRecoveries.set(basename(dirPath), { sessionId, dirPath, startedAt: Date.now() })
+}
+
+export function clearActiveRecovery(dirPath: string): void {
+  activeRecoveries.delete(basename(dirPath))
+}
+
+export function listActiveRecoveries(): Array<{ sessionId: string; dirPath: string; startedAt: number }> {
+  return [...activeRecoveries.values()]
+}
+
+export function isRecoveryActive(dirName: string): boolean {
+  return activeRecoveries.has(dirName)
+}
+
+export function markRecovered(dirPath: string, savedFilename: string): void {
+  try {
+    writeFileSync(
+      resolve(dirPath, RECOVERED_RECEIPT),
+      `${JSON.stringify({ schemaVersion: 1, recoveredAt: new Date().toISOString(), savedFilename })}\n`,
+      { encoding: 'utf8', mode: 0o600 },
+    )
+  } catch { /* receipt is best-effort; findBySessionId remains the true guard */ }
+}

package/server/lib/whisper-local.ts

@@ -137,6 +137,12 @@ const BATCH_LARGE_V3_ENABLED = process.env.COS_BATCH_LARGE_V3 !== '0'
 const VAD_MODEL_PATH = join(process.env.HOME ?? homedir(), '.local/share/whisper-models/ggml-silero-v5.1.2.bin')
 const VAD_ENABLED = process.env.COS_WHISPER_VAD !== '0'
 
+/** Batch meeting HQ keeps CLI --vad. Interactive/prompt HQ must not — VAD was
+ *  measured dropping leading speech on compose ("device just for your awareness"). */
+export function hqCliVadEnabled(priority?: 'interactive' | 'batch'): boolean {
+  return priority === 'batch' && VAD_ENABLED && existsSync(VAD_MODEL_PATH)
+}
+
 /** Pick the batch model path. Prefer large-v3 when enabled + on disk; fall
  *  back to turbo otherwise. Logged once per process so we know which decoder
  *  actually ran when reviewing a meeting later. */
@@ -732,7 +738,6 @@ export async function transcribeHighQuality(
 
   const modelPath = resolveBatchModel()
   const useLargeV3 = modelPath === BATCH_MODEL_LARGE_V3
-  const useVad = VAD_ENABLED && existsSync(VAD_MODEL_PATH)
 
   try {
     writeFileSync(tmpWav, audioBuffer)
@@ -741,6 +746,9 @@ export async function transcribeHighQuality(
     // that is explicitly OUT of batch device policy. Only the long post-meeting
     // batch is admission-controlled against live ASR.
     const isBatch = opts.priority === 'batch'
+    // A0 (2026-07-30): CLI --vad on interactive/prompt HQ can drop leading speech
+    // (measured: "device just for your awareness"). Keep VAD for meeting batch only.
+    const useVad = hqCliVadEnabled(opts.priority)
     const decision: { device: 'metal' | 'cpu'; reason: string; metalEnabled: boolean } = isBatch
       ? (opts.forceCpu
           ? { device: 'cpu', reason: 'preempt_retry', metalEnabled: batchHqMetalEnabled() }
@@ -774,9 +782,8 @@ export async function transcribeHighQuality(
         args.push('-ojf', '-of', outBase)
       }
       if (useVad) {
-        // Same VAD model the streaming path uses. Strips silence windows
-        // before the decoder sees them — prevents the silence-hallucination
-        // failure mode even with large-v3's more permissive decoder.
+        // Batch-only: strips silence windows before decode. Interactive/prompt HQ
+        // omits this — VAD was measured dropping real leading speech on compose.
         args.push('--vad', '--vad-model', VAD_MODEL_PATH)
       }
       const proc = spawn(WHISPER_CLI, args, {
@@ -1076,8 +1083,14 @@ export function resetDecoderCaches(): void {
  * auto-restart the server process in the background and throw immediately so the
  * caller can use cloud while the server recovers (~20s model load).
  */
-export async function transcribeLocal(audioBuffer: Buffer, context?: string, isQuiet?: boolean): Promise<{ text: string; backend: 'server' | 'cli'; words?: WhisperWord[] }> {
+export async function transcribeLocal(
+  audioBuffer: Buffer,
+  context?: string,
+  isQuiet?: boolean,
+  opts?: { affectsCircuit?: boolean },
+): Promise<{ text: string; backend: 'server' | 'cli'; words?: WhisperWord[] }> {
   const start = Date.now()
+  const affectsCircuit = opts?.affectsCircuit !== false
 
   if (!serverAvailable) {
     await reconcileWhisperServerHealth()
@@ -1094,34 +1107,38 @@ export async function transcribeLocal(audioBuffer: Buffer, context?: string, isQ
       const text = applyCorrections(result.text)
       const words = result.words?.map(w => ({ ...w, word: applyCorrections(w.word) }))
       const elapsed = Date.now() - start
-      // Reset circuit breaker on success
-      if (serverConsecutiveFailures > 0) {
+      // Reset circuit breaker on success (skip for peek / non-circuit callers)
+      if (affectsCircuit && serverConsecutiveFailures > 0) {
         console.log(`[whisper-local] Server recovered after ${serverConsecutiveFailures} consecutive failure(s)`)
         serverConsecutiveFailures = 0
       }
       console.log(`[whisper-local] Server transcribed in ${elapsed}ms (${words?.length ?? 0} words): "${text.slice(0, 80)}${text.length > 80 ? '...' : ''}"`)
       return { text, backend: 'server', words }
     } catch (err: any) {
-      serverConsecutiveFailures++
-      const isTimeout = err.message.includes('timeout') || err.message.includes('aborted')
-      const isDead = err.message.includes('ECONNREFUSED') || err.message.includes('fetch failed')
-
-      if (isDead) {
-        serverAvailable = false
-        console.error(`[whisper-local] Server DEAD (ECONNREFUSED) — marked unavailable. Consecutive failures: ${serverConsecutiveFailures}`)
-      } else if (isTimeout) {
-        // Server process exists but is hung — mark unavailable so we stop trying
-        serverAvailable = false
-        console.error(`[whisper-local] Server HUNG (timeout) — marked unavailable. Consecutive failures: ${serverConsecutiveFailures}`)
-      }
+      if (affectsCircuit) {
+        serverConsecutiveFailures++
+        const isTimeout = err.message.includes('timeout') || err.message.includes('aborted')
+        const isDead = err.message.includes('ECONNREFUSED') || err.message.includes('fetch failed')
+
+        if (isDead) {
+          serverAvailable = false
+          console.error(`[whisper-local] Server DEAD (ECONNREFUSED) — marked unavailable. Consecutive failures: ${serverConsecutiveFailures}`)
+        } else if (isTimeout) {
+          // Server process exists but is hung — mark unavailable so we stop trying
+          serverAvailable = false
+          console.error(`[whisper-local] Server HUNG (timeout) — marked unavailable. Consecutive failures: ${serverConsecutiveFailures}`)
+        }
 
-      // Circuit breaker: auto-restart after threshold
-      if (serverConsecutiveFailures >= SERVER_FAILURE_THRESHOLD && !serverRestarting) {
-        console.error(`[whisper-local] ⚠ CIRCUIT BREAKER OPEN — ${serverConsecutiveFailures} consecutive failures. Auto-restarting server...`)
-        // Non-blocking restart in background
-        void restartWhisperServer()
-      } else if (serverConsecutiveFailures < SERVER_FAILURE_THRESHOLD) {
-        console.warn(`[whisper-local] Server failed (${serverConsecutiveFailures}/${SERVER_FAILURE_THRESHOLD} before restart): ${err.message}`)
+        // Circuit breaker: auto-restart after threshold
+        if (serverConsecutiveFailures >= SERVER_FAILURE_THRESHOLD && !serverRestarting) {
+          console.error(`[whisper-local] ⚠ CIRCUIT BREAKER OPEN — ${serverConsecutiveFailures} consecutive failures. Auto-restarting server...`)
+          // Non-blocking restart in background
+          void restartWhisperServer()
+        } else if (serverConsecutiveFailures < SERVER_FAILURE_THRESHOLD) {
+          console.warn(`[whisper-local] Server failed (${serverConsecutiveFailures}/${SERVER_FAILURE_THRESHOLD} before restart): ${err.message}`)
+        }
+      } else {
+        console.warn(`[whisper-local] Peek/non-circuit server failure (breaker untouched): ${err.message}`)
       }
 
       // Throw to let caller fall to OpenAI cloud (1-3s) — much faster than CLI cold-start (11s)
@@ -1132,10 +1149,12 @@ export async function transcribeLocal(audioBuffer: Buffer, context?: string, isQ
   // Server not available — still count toward circuit breaker so auto-restart can fire.
   // Without this, the counter stalls after the first failure marks serverAvailable=false
   // and subsequent calls never increment, so restart never triggers.
-  serverConsecutiveFailures++
-  if (serverConsecutiveFailures >= SERVER_FAILURE_THRESHOLD && !serverRestarting) {
-    console.error(`[whisper-local] CIRCUIT BREAKER OPEN — ${serverConsecutiveFailures} consecutive failures (server unavailable). Auto-restarting...`)
-    void restartWhisperServer()
+  if (affectsCircuit) {
+    serverConsecutiveFailures++
+    if (serverConsecutiveFailures >= SERVER_FAILURE_THRESHOLD && !serverRestarting) {
+      console.error(`[whisper-local] CIRCUIT BREAKER OPEN — ${serverConsecutiveFailures} consecutive failures (server unavailable). Auto-restarting...`)
+      void restartWhisperServer()
+    }
   }
 
   // Throw so the caller applies the configured recovery policy. CLI is

package/server/routes/health.ts

@@ -38,6 +38,7 @@ import { maintenanceLifecycle } from '../lib/maintenance-lifecycle.js'
 import { getLocalTtsHealth, refreshLocalTtsHealth } from '../lib/tts-local.js'
 import { liveCuesCapability } from '../lib/live-cues-capability.js'
 import { getMeetingSyncSnapshot } from '../lib/meeting-batch-progress.js'
+import { listUnsavedCaptures } from '../lib/unsaved-audio-quarantine.js'
 
 export const healthRouter = Router()
 
@@ -261,6 +262,20 @@ healthRouter.get('/health', async (_req, res) => {
   const cursorSnapshot = getCursorModelCatalogSnapshot()
   const { agentBinary: _cursorAgentBinary, ...cursor_models } = cursorSnapshot
   const meeting_sync = getMeetingSyncSnapshot()
+  // Quarantined unsaved captures (6.19.0). Compact on this unauthenticated
+  // surface — same exposure level as meeting_sync's meetingIds. Full detail
+  // plus the recover action live on the authenticated /api/meeting/orphans.
+  const unsavedList = listUnsavedCaptures()
+  const unsaved_captures = {
+    count: unsavedList.filter(item => !item.recovered).length,
+    items: unsavedList.slice(0, 10).map(item => ({
+      sessionId: item.sessionId,
+      ageHours: item.ageHours,
+      chunkFiles: item.chunkFiles,
+      expiresAt: item.expiresAt,
+      recovered: item.recovered,
+    })),
+  }
   res.json({
     ...checks,
     server_version: managedServerVersion(),
@@ -276,6 +291,7 @@ healthRouter.get('/health', async (_req, res) => {
     codex_models,
     cursor_models,
     meeting_sync,
+    unsaved_captures,
     capabilities: {
       transcription: { ...transcription, hq: transcriptionHq },
       recovery,