@gotcos/glasses-server 6.18.7 → 6.19.0

package/server/routes/meeting.ts

@@ -2,7 +2,8 @@
 // the standalone public meeting store. The live transcript and chunk metadata
 // are durable before the session is closed; batch improvement runs afterward.
 
-import { rmSync } from 'node:fs'
+import { readdirSync, rmSync, statSync, unlinkSync } from 'node:fs'
+import { resolve } from 'node:path'
 import { Router } from 'express'
 import { emitDisplay } from '../lib/display-bus.js'
 import { cleanTranscriptLines } from '../lib/hallucination-filter.js'
@@ -17,7 +18,29 @@ import {
   persistBatchDecisionSidecar,
   replaceMeetingTranscriptAtomic,
 } from '../lib/meeting-batch-persistence.js'
-import { runMeetingBatchPipeline } from '../lib/meeting-batch-transcribe.js'
+import {
+  BATCH_PENDING_MARKER,
+  clearMeetingBatchProgress,
+  readMeetingBatchProgress,
+  writeMeetingBatchTerminal,
+} from '../lib/meeting-batch-progress.js'
+import {
+  enqueueSerializedHqWork,
+  runMeetingBatchPipeline,
+  segmentTranscriptChunks,
+  transcribeSegments,
+} from '../lib/meeting-batch-transcribe.js'
+import {
+  clearActiveRecovery,
+  findQuarantineDir,
+  listUnsavedCaptures,
+  markRecovered,
+  registerActiveRecovery,
+} from '../lib/unsaved-audio-quarantine.js'
+import {
+  clearSessionHallucinationState,
+  stripInlineHallucinations,
+} from '../lib/hallucination-filter.js'
 import {
   selectBatchTranscriptForPersistence,
   type BatchTranscription,
@@ -376,9 +399,255 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
     }
   })
 
+  // ── Unsaved-capture recovery (6.19.0) ─────────────────────────────────
+  // Surface-only by decision (Miles, 2026-08-02): the server NEVER drives
+  // recovery on its own. It lists what the quarantine holds, and one
+  // authenticated POST drives one capture to a durable scribe.
+  const recoveringOrphans = new Set<string>()
+
+  router.get('/meeting/orphans', (_req, res) => {
+    res.set('Cache-Control', 'private, no-store')
+    const items = listUnsavedCaptures()
+    // In-flight recovery progress: transcribeSegments writes its progress file
+    // into the quarantine dir, invisible to meeting_sync (different root) —
+    // surface it here so a long recovery is not a black box.
+    const recoveringProgress: Record<string, { segmentsDone: number; segmentsTotal: number } | null> = {}
+    for (const id of recoveringOrphans) {
+      const dir = findQuarantineDir(id)
+      const progress = dir ? readMeetingBatchProgress(dir) : null
+      recoveringProgress[id] = progress
+        ? { segmentsDone: progress.segmentsDone, segmentsTotal: progress.segmentsTotal }
+        : null
+    }
+    res.json({
+      count: items.filter(item => !item.recovered).length,
+      recovering: [...recoveringOrphans],
+      recoveringProgress,
+      items,
+    })
+  })
+
+  router.post('/meeting/orphans/:sessionId/recover', (req, res) => {
+    const sessionId = String(req.params.sessionId ?? '')
+    if (!/^[A-Za-z0-9:_-]{3,96}$/.test(sessionId)) {
+      res.status(400).json({ error: 'Invalid sessionId', reason: 'invalid_session_id' })
+      return
+    }
+    const body = (req.body ?? {}) as Record<string, unknown>
+    if (body.title !== undefined && typeof body.title !== 'string') {
+      res.status(400).json({ error: 'Invalid title', reason: 'invalid_title' })
+      return
+    }
+    if (body.domain !== undefined && typeof body.domain !== 'string') {
+      res.status(400).json({ error: 'Invalid domain', reason: 'invalid_domain' })
+      return
+    }
+    // Idempotent: a capture that already reached a durable save replays its
+    // receipt — same contract as POST /meeting/save.
+    const alreadySaved = store.findBySessionId(sessionId)
+    if (alreadySaved) {
+      // Stamp the quarantine receipt too, or a saved-but-quarantined capture
+      // (e.g. save succeeded but the pending handoff failed, marker expired,
+      // sweep quarantined the leftovers — the meeting IS saved) counts as
+      // "unsaved" on health forever, a false alarm that trains the user to
+      // ignore the one channel built to report real losses.
+      const staleQuarantine = findQuarantineDir(sessionId)
+      if (staleQuarantine) markRecovered(staleQuarantine, alreadySaved.filename)
+      res.set('Cache-Control', 'private, no-store')
+      res.json({ accepted: false, alreadySaved: true, receipt: publicSaveResponse(alreadySaved, true) })
+      return
+    }
+    const quarantineDir = findQuarantineDir(sessionId)
+    if (!quarantineDir) {
+      res.status(404).json({ error: 'No quarantined audio for this session', reason: 'orphan_not_found' })
+      return
+    }
+    if (recoveringOrphans.has(sessionId)) {
+      res.status(409).json({ error: 'Recovery already in progress', reason: 'recovery_in_progress' })
+      return
+    }
+    const capture = synthesizeEntriesFromChunkWavs(quarantineDir)
+    const entries = capture.entries
+    if (entries.length === 0) {
+      res.status(422).json({ error: 'Quarantined directory holds no readable chunk audio', reason: 'no_chunk_audio' })
+      return
+    }
+
+    // Acquire BEFORE marking the session as recovering: if a drain gate makes
+    // acquire throw, nothing must linger in recoveringOrphans — a leaked entry
+    // would 409 every retry for the exact capture this route exists to save.
+    let lease: MaintenanceWorkLease
+    try {
+      lease = acquireMaintenanceWork('orphan_recovery', { phase: 'queued' })
+    } catch {
+      res.status(503).json({
+        error: 'Server is draining for maintenance — retry after the update completes',
+        reason: 'maintenance_drain',
+      })
+      return
+    }
+    recoveringOrphans.add(sessionId)
+    // Registry drives two protections: meeting_sync renders this as an active
+    // row (COS Control warns before an Update Server drain walks into a
+    // 20-90 min decode), and purgeExpiredQuarantine will not delete the dir
+    // mid-run even at the retention boundary.
+    registerActiveRecovery(sessionId, quarantineDir)
+    res.status(202).set('Cache-Control', 'private, no-store').json({
+      accepted: true,
+      sessionId,
+      chunkFiles: entries.length,
+      note: 'Recovery runs in the background behind the HQ decoder queue — expect minutes for a long meeting '
+        + '(watch recoveringProgress on GET /api/meeting/orphans). Speakers are labeled Unknown: no live ASR ever '
+        + 'ran for this capture. The capture leaves the unsaved count once its scribe is durable.',
+    })
+
+    const task = Promise.resolve().then(async () => {
+      lease.setPhase('active')
+      const startedAt = Date.now()
+      const segments = segmentTranscriptChunks(entries)
+      // transcribeSegments carries the batch contract (enhancement, Metal
+      // preemption discard + one CPU retry, overlap stripping) but NOT the
+      // decoder serialization — that lives in the queue tail. Chain onto it,
+      // or two back-to-back recoveries plus a live post-meeting batch would
+      // run parallel 16-thread large-v3 decoders on the user's Mac.
+      const results = await enqueueSerializedHqWork(
+        () => transcribeSegments(quarantineDir, segments, entries),
+      )
+      // Batch whisper over a dead session has no streaming baseline to
+      // compare against (evaluateBatchQuality needs one), but the inline
+      // hallucination filter needs none — run the same two-pass the boot
+      // session-recovery path uses: pass 1 builds the frequency blocklist
+      // across all segments, pass 2 strips with the final blocklist.
+      for (const item of results) {
+        if (item.text) stripInlineHallucinations(item.text, sessionId)
+      }
+      for (const item of results) {
+        if (item.text) item.text = stripInlineHallucinations(item.text, sessionId)
+      }
+      clearSessionHallucinationState(sessionId)
+      const transcript = cleanFinalTranscript(results.map(item => item.text).join(' '))
+      if (!transcript.trim()) {
+        throw new Error('recovery produced an empty transcript')
+      }
+      const recoveredChunks = results.map(item => ({
+        text: item.text,
+        speaker: 'Unknown',
+        elapsed: item.segment.startElapsed,
+        similarity: 0,
+        words: item.words,
+        canonical: true,
+      }))
+      const saved = store.save({
+        sessionId,
+        title: typeof body.title === 'string' && body.title.trim() ? body.title : undefined,
+        domain: typeof body.domain === 'string' && body.domain.trim() ? body.domain : undefined,
+        transcript,
+        startTime: capture.startTime,
+        durationMs: capture.durationMs,
+        chunks: recoveredChunks,
+        chunkEntries: recoveredChunks.map((chunk, position) => ({
+          chunkIndex: results[position]?.segment.startChunkIdx ?? position,
+          chunk,
+        })),
+        transferIntegrity: null,
+      })
+      markRecovered(quarantineDir, saved.filename)
+      console.log(
+        `[meeting/orphans] Recovered ${sessionId} → ${saved.filename} `
+        + `(${entries.length} chunks, ${Math.round((Date.now() - startedAt) / 1000)}s)`,
+      )
+      try {
+        emit({
+          type: 'recording_stop',
+          data: {
+            sessionId,
+            filename: saved.filename,
+            durationMin: saved.durationMin,
+            domain: saved.domain,
+          },
+        })
+      } catch { /* display is best-effort */ }
+      if (cosOpsPipelineConfigured()) {
+        await handoffMeetingToOperations(saved.filepath)
+      }
+    }).catch(error => {
+      // The quarantined audio is untouched on failure — retry stays possible
+      // until the retention clock clears it.
+      console.error(
+        `[meeting/orphans] Recovery failed for ${sessionId}: `
+        + `${error instanceof Error ? error.message : String(error)}`,
+      )
+    }).finally(() => {
+      // transcribeSegments wrote progress + a pending marker into the
+      // quarantine dir; the batch pipeline's own finally never runs here, so
+      // clean them up or the dir carries stale work files until retention.
+      try { clearMeetingBatchProgress(quarantineDir) } catch { /* best-effort */ }
+      try { unlinkSync(resolve(quarantineDir, BATCH_PENDING_MARKER)) } catch { /* best-effort */ }
+      clearActiveRecovery(quarantineDir)
+      recoveringOrphans.delete(sessionId)
+      lease.release()
+    })
+    scheduleBackground(task)
+  })
+
   return router
 }
 
+const CHUNK_WAV_NAME = /^chunk_(\d{4})\.wav$/
+const WAV_HEADER_BYTES = 44
+const PCM_BYTES_PER_MS = 32 // 16 kHz mono 16-bit
+
+interface RecoveredCapture {
+  entries: IndexedTranscriptChunk[]
+  /** Meeting start ≈ the earliest chunk's write time (chunk files keep their
+   *  original mtimes across renames — the pending-batch cleanup relies on the
+   *  same property). Falls back to now-minus-duration when mtimes are unusable. */
+  startTime: number
+  durationMs: number
+}
+
+/** Rebuild a chunk timeline for a dead session from its WAV files alone:
+ *  index from the filename, duration from PCM byte length, elapsed cumulative.
+ *  Text is empty — recovery exists precisely because no ASR ever ran. */
+function synthesizeEntriesFromChunkWavs(audioDir: string): RecoveredCapture {
+  const rows: Array<{ chunkIndex: number; durationMs: number; mtimeMs: number }> = []
+  try {
+    for (const name of readdirSync(audioDir)) {
+      const match = CHUNK_WAV_NAME.exec(name)
+      if (!match) continue
+      try {
+        const stats = statSync(resolve(audioDir, name))
+        rows.push({
+          chunkIndex: Number(match[1]),
+          durationMs: Math.max(0, Math.round((stats.size - WAV_HEADER_BYTES) / PCM_BYTES_PER_MS)),
+          mtimeMs: stats.mtimeMs,
+        })
+      } catch { /* unreadable chunk — skip */ }
+    }
+  } catch {
+    return { entries: [], startTime: Date.now(), durationMs: 0 }
+  }
+  rows.sort((a, b) => a.chunkIndex - b.chunkIndex)
+  let elapsed = 0
+  const entries = rows.map(row => {
+    const entry: IndexedTranscriptChunk = {
+      chunkIndex: row.chunkIndex,
+      chunk: { text: '', speaker: 'Unknown', elapsed, similarity: 0 },
+    }
+    elapsed += row.durationMs
+    return entry
+  })
+  const durationMs = elapsed
+  const earliestMtime = rows.reduce(
+    (minimum, row) => (Number.isFinite(row.mtimeMs) && row.mtimeMs > 0 ? Math.min(minimum, row.mtimeMs) : minimum),
+    Number.POSITIVE_INFINITY,
+  )
+  const startTime = Number.isFinite(earliestMtime) && earliestMtime !== Number.POSITIVE_INFINITY
+    ? Math.round(earliestMtime)
+    : Date.now() - durationMs
+  return { entries, startTime, durationMs }
+}
+
 async function finalizeBatch(options: {
   audioDir: string
   entries: IndexedTranscriptChunk[]
@@ -431,6 +700,16 @@ async function finalizeBatch(options: {
     rmSync(options.audioDir, { recursive: true, force: true })
   } else {
     console.warn('[meeting/save] Pending raw audio retained for bounded cleanup')
+    // The batch reached a terminal outcome but its WAVs stay behind. Record it
+    // so meeting_sync reports "retained", not perpetual active work. Rejected
+    // and pipeline-failed runs already wrote their terminal inside runBatch;
+    // this covers the accepted-but-not-fully-persisted case.
+    if (result.transcriptionQuality === 'batch') {
+      writeMeetingBatchTerminal(options.audioDir, {
+        outcome: 'accepted',
+        reason: transcriptApplied ? 'metadata_persist_failed' : 'transcript_apply_failed',
+      })
+    }
   }
 }
 

package/server/routes/prompt-drafts.ts

@@ -61,6 +61,9 @@ const modeQualityJobs = new Map<string, Promise<string>>()
 const finalizeJobs = new Map<string, Promise<any>>()
 let warmTail: Promise<void> = Promise.resolve()
 let hqWarmTail: Promise<void> = Promise.resolve()
+/** Peek decodes — drop-while-busy; never chained onto warmTail (HOL vs HQ). */
+let peekTail: Promise<void> = Promise.resolve()
+let peekBusy = false
 
 /** Speculative HQ warm while speaking. Set COS_HQ_SPECULATIVE_WARM=0 to restore Fast-only warm. */
 function speculativeHqWarmEnabled(): boolean {
@@ -372,6 +375,80 @@ promptDraftsRouter.post('/prompt-drafts/start', (req, res) => {
   }
 })
 
+/**
+ * Provisional Turbo peek — cosmetic lens UX only.
+ * Does NOT save chunks, advance the recovery ledger, or write warm/final transcripts.
+ * Drop-while-busy: overlapping peeks return 204 without queueing behind HQ warm.
+ */
+promptDraftsRouter.post('/prompt-drafts/:draftId/peek', async (req, res) => {
+  try {
+    const meta = loadPromptDraftMeta(req.params.draftId)
+    if (!meta) return void res.status(404).json({ error: 'draft not found' })
+
+    const rawIndex = Array.isArray(req.query.chunkIndex) ? req.query.chunkIndex[0] : req.query.chunkIndex
+    const chunkIndex = Number(rawIndex)
+    if (!Number.isInteger(chunkIndex) || chunkIndex < 0) {
+      return void res.status(400).json({ error: 'chunkIndex required' })
+    }
+    const rawGen = Array.isArray(req.query.peekGen) ? req.query.peekGen[0] : req.query.peekGen
+    const peekGen = Number(rawGen)
+    if (!Number.isInteger(peekGen) || peekGen < 0) {
+      return void res.status(400).json({ error: 'peekGen required' })
+    }
+
+    if (peekBusy) {
+      return void res.status(204).send()
+    }
+
+    const audio = await readRawBody(req)
+    if (!audio.length) return void res.status(400).json({ error: 'empty audio' })
+    if (audio.length > MAX_CHUNK_BYTES) return void res.status(413).json({ error: 'audio chunk too large' })
+
+    const lease = acquireMaintenanceWork('prompt_draft_peek', {
+      allowDuringDrain: true,
+      phase: 'queued',
+    })
+
+    peekBusy = true
+    const draftId = req.params.draftId
+    peekTail = peekTail.then(async () => {
+      lease.setPhase('active')
+      try {
+        const result = await transcribeAudioBuffer(audio, {
+          mode: 'fast',
+          policy: 'local-only',
+          affectsCircuit: false,
+        })
+        const text = sanitizeTranscript(draftId, result.text, false)
+        if (!text) return
+        if (!loadPromptDraftMeta(draftId)) return
+        emitDisplay({
+          type: 'prompt_transcript',
+          data: { draftId, chunkIndex, text, provisional: true, peekGen },
+        })
+      } catch (err: any) {
+        if (!(err instanceof NoSpeechDetectedError)) {
+          console.warn(`[prompt-draft] peek failed ${draftId}/${chunkIndex}: ${err?.message ?? err}`)
+        }
+      } finally {
+        peekBusy = false
+        lease.release()
+      }
+    }).catch(() => {
+      peekBusy = false
+      lease.release()
+    })
+
+    res.json({ draftId, chunkIndex, peekGen, accepted: true })
+  } catch (err: any) {
+    if (err instanceof MaintenanceLifecycleError) {
+      if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))
+      return void res.status(err.status).json({ ...maintenanceErrorPayload(err), draftPreserved: true })
+    }
+    res.status(err.status ?? (err.message === 'draft not found' ? 404 : 500)).json({ error: err.message })
+  }
+})
+
 promptDraftsRouter.post('/prompt-drafts/:draftId/chunks', async (req, res) => {
   let maintenanceLease: MaintenanceWorkLease | undefined
   try {

package/server/routes/transcribe-stream.ts

@@ -35,6 +35,12 @@ import {
   isVocabEchoOnly,
 } from '../lib/hallucination-filter.js'
 import { dataPath } from '../lib/data-dir.js'
+import {
+  countChunkWavs,
+  purgeExpiredQuarantine,
+  quarantineSessionAudio,
+  sweepOrphanedSessionAudio,
+} from '../lib/unsaved-audio-quarantine.js'
 import { durableAtomicWriteFileSync } from '../lib/atomic-fs.js'
 import {
   LOCAL_FIRST_MEETING_IDLE_RETENTION_MS,
@@ -612,16 +618,28 @@ function recoverSessions(): void {
         }
       } catch { /* skip corrupt files */ }
     }
-    // Remove orphaned session-audio dirs with no matching recovered session
+    // Orphaned session-audio dirs with no matching recovered session: audio
+    // evidence is QUARANTINED, never deleted (6.19.0 — two meetings were
+    // destroyed here on 2026-08-01 when their deferred saves never landed).
+    // Only chunk-less dirs are removed.
     try {
       if (existsSync(SESSION_AUDIO_DIR)) {
-        for (const dir of readdirSync(SESSION_AUDIO_DIR)) {
-          if (!recoveredIds.has(dir) && !hasFreshPreservedAudioMarker(resolve(SESSION_AUDIO_DIR, dir))) {
-            rmSync(resolve(SESSION_AUDIO_DIR, dir), { recursive: true, force: true })
-            console.log(`[session-recovery] Cleaned orphaned session-audio: ${dir}`)
+        const actions = sweepOrphanedSessionAudio(SESSION_AUDIO_DIR, {
+          isLive: id => recoveredIds.has(id),
+          hasFreshPreservedMarker: hasFreshPreservedAudioMarker,
+          reason: 'boot_sweep_unsaved',
+        })
+        for (const entry of actions) {
+          if (entry.action === 'quarantined') {
+            console.warn(`[session-recovery] Quarantined unsaved session-audio: ${entry.dir} → ${entry.target}`)
+          } else if (entry.action === 'deleted_empty') {
+            console.log(`[session-recovery] Cleaned empty session-audio: ${entry.dir}`)
+          } else if (entry.action === 'quarantine_failed') {
+            console.error(`[session-recovery] Quarantine move failed, source retained: ${entry.dir}`)
           }
         }
       }
+      purgeExpiredQuarantine()
     } catch {}
   } catch { /* non-critical */ }
 }
@@ -658,13 +676,24 @@ setInterval(() => {
       closeTranscriptSession(id, 'expired')
     }
   }
-  // Purge orphaned session-audio dirs (no matching active session)
+  // Orphaned session-audio dirs (no matching active session): quarantine any
+  // dir still holding chunk audio; delete only chunk-less dirs. Quarantine
+  // itself expires on the unsaved-audio retention clock, the ONLY place
+  // quarantined audio is ever deleted.
   try {
-    for (const dir of readdirSync(SESSION_AUDIO_DIR)) {
-      if (!sessions.has(dir) && !hasFreshPreservedAudioMarker(resolve(SESSION_AUDIO_DIR, dir))) {
-        rmSync(resolve(SESSION_AUDIO_DIR, dir), { recursive: true, force: true })
+    const actions = sweepOrphanedSessionAudio(SESSION_AUDIO_DIR, {
+      isLive: id => sessions.has(id),
+      hasFreshPreservedMarker: hasFreshPreservedAudioMarker,
+      reason: 'idle_expiry_unsaved',
+    })
+    for (const entry of actions) {
+      if (entry.action === 'quarantined') {
+        console.warn(`[cleanup] Quarantined unsaved session-audio: ${entry.dir} → ${entry.target}`)
+      } else if (entry.action === 'quarantine_failed') {
+        console.error(`[cleanup] Quarantine move failed, source retained: ${entry.dir}`)
       }
     }
+    purgeExpiredQuarantine()
   } catch {}
   // Purge stale pending-batch dirs. HQ large-v3 on long meetings + Control
   // restart can exceed 2h (2026-07-27: two sessions purged before batch).
@@ -1022,7 +1051,7 @@ function closeTranscriptSession(
     maxChunkIndex,
     reason,
   })
-  finishClosingTranscriptSession(sessionId, options)
+  finishClosingTranscriptSession(sessionId, { ...options, closeReason: reason })
 }
 
 /** Delete session after save */
@@ -1030,7 +1059,10 @@ export function deleteSession(sessionId: string, options: { preserveAudio?: bool
   closeTranscriptSession(sessionId, 'saved', options)
 }
 
-function finishClosingTranscriptSession(sessionId: string, options: { preserveAudio?: boolean }): void {
+function finishClosingTranscriptSession(
+  sessionId: string,
+  options: { preserveAudio?: boolean; closeReason?: ClosedTranscriptSession['reason'] },
+): void {
   sessions.delete(sessionId)
   sessionAudioBytes.delete(sessionId)
   sessionAudioWrites.delete(sessionId)
@@ -1049,8 +1081,28 @@ function finishClosingTranscriptSession(sessionId: string, options: { preserveAu
       writeFileSync(marker, String(Date.now()), { encoding: 'utf8', mode: 0o600 })
       chmodSync(marker, 0o600)
     } catch {}
-  } else {
-    try { rmSync(audioDir, { recursive: true, force: true }) } catch {}
+  } else if (options.closeReason === 'saved') {
+    // Saved sessions moved their audio to pending-batch (or explicitly
+    // preserved it above); a leftover dir here is residue, safe to delete.
+    // UNLESS chunks are still present — hasAudio() returns false on a
+    // transient statSync error, which makes preserveAudio false while the
+    // move never ran. Chunk-bearing evidence goes to quarantine, never rmSync.
+    if (countChunkWavs(audioDir) > 0) {
+      const target = quarantineSessionAudio(audioDir, 'close_saved_residual_chunks')
+      if (target) console.warn(`[transcribe-stream] Saved-close left chunk audio behind; quarantined: ${sessionId} → ${target}`)
+    } else {
+      try { rmSync(audioDir, { recursive: true, force: true }) } catch {}
+    }
+  } else if (existsSync(audioDir)) {
+    // Any non-saved close (expired, error, …) with chunk audio still on disk
+    // is an unsaved capture. Quarantine it — never delete evidence (6.19.0).
+    if (countChunkWavs(audioDir) > 0) {
+      const target = quarantineSessionAudio(audioDir, `close_${options.closeReason ?? 'unknown'}`)
+      if (target) console.warn(`[transcribe-stream] Quarantined unsaved session-audio on close: ${sessionId} → ${target}`)
+      else console.error(`[transcribe-stream] Quarantine move failed on close, source retained: ${sessionId}`)
+    } else {
+      try { rmSync(audioDir, { recursive: true, force: true }) } catch {}
+    }
   }
 }