@gotcos/glasses-server 6.71.12 → 6.71.14

package/server/routes/transcribe-stream.ts

@@ -77,8 +77,16 @@ import {
   listUnsavedCaptures,
   purgeExpiredQuarantine,
   quarantineSessionAudio,
+  readCaptureDisposition,
   sweepOrphanedSessionAudio,
 } from '../lib/unsaved-audio-quarantine.js'
+import {
+  STALE_ACTIVE_SESSION_DIR_NAME,
+  moveActiveSessionFileAside,
+  purgeExpiredStaleActiveSessionFiles,
+  sweepStaleActiveSessionFiles,
+  type StaleActiveSessionMove,
+} from '../lib/stale-active-session-files.js'
 import { durableAtomicWriteFileSync } from '../lib/atomic-fs.js'
 import { saveTrainingAudioSample } from '../lib/training-audio-save.js'
 import {
@@ -92,6 +100,7 @@ import {
   acquireMaintenanceWork,
   maintenanceAdmissionsOpen,
   maintenanceLifecycle,
+  onMaintenanceAdmissionsOpen,
   type MaintenanceWorkLease,
 } from '../lib/maintenance-lifecycle.js'
 import { feedLiveCueTranscript } from '../lib/live-cues-engine.js'
@@ -542,6 +551,8 @@ export function getTranscriptionSessionLiveness(now = Date.now()): Transcription
 // Incremental chunk persistence — survive server restarts
 const CHUNK_PERSIST_DIR = dataPath('active-sessions')
 ensurePrivateDirectory(CHUNK_PERSIST_DIR)
+/** 6.71.14: where an active-session file the server no longer owns is moved, never deleted in place. */
+const STALE_ACTIVE_SESSION_DIR = dataPath(STALE_ACTIVE_SESSION_DIR_NAME)
 
 function readClosedSessions(): Record<string, ClosedTranscriptSession> {
   if (!existsSync(CLOSED_SESSIONS_FILE)) return {}
@@ -691,6 +702,71 @@ function persistSessionBestEffort(sessionId: string): void {
   try { persistSessionRequired(sessionId) } catch { /* recovery cleanup is non-admission work */ }
 }
 
+/**
+ * 6.71.14: one pass of the stale active-session sweep, against the real session
+ * map. Exported so a test drives the exact wiring the minute interval runs.
+ */
+export function sweepStaleActiveSessionFilesNow(now: number): StaleActiveSessionMove[] {
+  const moves = sweepStaleActiveSessionFiles({
+    dir: CHUNK_PERSIST_DIR,
+    targetDir: STALE_ACTIVE_SESSION_DIR,
+    isLive: id => sessions.has(id),
+    now,
+    idleRetentionMs: LOCAL_FIRST_MEETING_IDLE_RETENTION_MS,
+  })
+  for (const move of moves) {
+    if (move.target) {
+      staleMoveFailuresLogged.delete(move.sessionId)
+      console.warn(
+        `[cleanup] Moved stale active-session file aside: ${move.sessionId} `
+        + `(${move.reason}, idle ${Math.round((move.idleMs ?? 0) / 60_000)}m, not live) \u2192 ${move.target}`,
+      )
+    } else {
+      logStaleMoveFailure(move.sessionId, `[cleanup] Could not move stale active-session file aside for ${move.sessionId} (${move.reason}); it stays in active-sessions`)
+    }
+  }
+  const purged = purgeExpiredStaleActiveSessionFiles(STALE_ACTIVE_SESSION_DIR, now)
+  if (purged > 0) console.log(`[cleanup] Purged ${purged} stale active-session file(s) past retention`)
+  return moves
+}
+
+/** A failed move repeats every minute; say so once per session id, not every tick. */
+const staleMoveFailuresLogged = new Set<string>()
+function logStaleMoveFailure(sessionId: string, message: string): void {
+  if (staleMoveFailuresLogged.has(sessionId)) return
+  staleMoveFailuresLogged.add(sessionId)
+  if (staleMoveFailuresLogged.size > 500) staleMoveFailuresLogged.delete(staleMoveFailuresLogged.values().next().value as string)
+  console.error(message)
+}
+
+/**
+ * 6.71.14: is this active-session file the leftover of a capture that is already
+ * decoded (recovered or receipted no_speech), rather than a session that RESUMED
+ * under the same id after that capture was quarantined?
+ *
+ * A late chunk is accepted (the phone deletes its copy on a 410), so a session can
+ * restart under an id whose older quarantine dir is already receipted. Its file is
+ * then newer than the quarantine and its own audio sits in session-audio/<id>.
+ * Only a file that is no newer than the quarantine AND has no chunk audio of its
+ * own is the leftover; anything else is adopted as 6.71.13 would, so a meeting is
+ * never split across a restart.
+ */
+function isTerminalCaptureFile(sessionId: string, recordedLastActivityAt: unknown, filePath: string): boolean {
+  const disposition = readCaptureDisposition(sessionId)
+  if (!disposition || (disposition.kind !== 'no_speech' && disposition.kind !== 'recovered')) return false
+  const terminalAtIso = disposition.quarantinedAt ?? disposition.receiptedAt
+  const terminalAt = terminalAtIso ? Date.parse(terminalAtIso) : NaN
+  if (!Number.isFinite(terminalAt)) return false
+  let fileActivity = typeof recordedLastActivityAt === 'number' && Number.isFinite(recordedLastActivityAt)
+    ? recordedLastActivityAt
+    : NaN
+  if (!Number.isFinite(fileActivity)) {
+    try { fileActivity = statSync(filePath).mtimeMs } catch { return false }
+  }
+  if (terminalAt < fileActivity) return false
+  return countChunkWavs(resolve(SESSION_AUDIO_DIR, sessionId)) === 0
+}
+
 /** Recover sessions from disk on server restart */
 function recoverSessions(): void {
   try {
@@ -701,6 +777,27 @@ function recoverSessions(): void {
       if (!file.endsWith('.json')) continue
       try {
         const data = JSON.parse(readFileSync(resolve(CHUNK_PERSIST_DIR, file), 'utf-8'))
+        // 6.71.14: recovery can now run after boot (deferred behind a closed
+        // maintenance gate). A session already in memory owns its file; the disk
+        // copy must never overwrite it.
+        if (typeof data.sessionId === 'string' && sessions.has(data.sessionId)) continue
+        // A capture whose quarantined audio was already decoded to a terminal
+        // outcome (a saved recovery or a no_speech receipt) is done. Adopting its
+        // file would bring back a live session for it; tombstoning would delete the
+        // file. Move it aside instead. Plain `quarantined` (never decoded) is NOT
+        // done: a crash between a chunk's first WAV write and its first
+        // active-session persist quarantines the audio, and the phone's retry then
+        // starts the session again. That file is adopted exactly as before.
+        if (typeof data.sessionId === 'string' && isTerminalCaptureFile(data.sessionId, data.lastActivityAt, resolve(CHUNK_PERSIST_DIR, file))) {
+          const terminal = readCaptureDisposition(data.sessionId)!
+          const target = moveActiveSessionFileAside(
+            resolve(CHUNK_PERSIST_DIR, file), data.sessionId, STALE_ACTIVE_SESSION_DIR, 'terminal_capture', Date.now(),
+            { lastActivityAt: typeof data.lastActivityAt === 'number' ? data.lastActivityAt : null },
+          )
+          if (target) console.warn(`[session-recovery] ${data.sessionId} was already ${terminal.kind}; moved its active-session file aside: ${target}`)
+          else logStaleMoveFailure(data.sessionId, `[session-recovery] Could not move the active-session file aside for ${data.sessionId} (already ${terminal.kind}); left in place, not adopted`)
+          continue
+        }
         // Reconstruct the (sparse) chunk array. New format keeps original
         // indices via chunksIndexed; legacy format stored a dense `chunks`.
         const indexed: Array<{ i: number; c: TranscriptChunk }> | null =
@@ -859,7 +956,9 @@ function recoverSessions(): void {
     try {
       if (existsSync(SESSION_AUDIO_DIR)) {
         const actions = sweepOrphanedSessionAudio(SESSION_AUDIO_DIR, {
-          isLive: id => recoveredIds.has(id),
+          // `|| sessions.has`: deferred recovery (6.71.14) can run while sessions
+          // are live in memory; their audio is not orphaned.
+          isLive: id => recoveredIds.has(id) || sessions.has(id),
           hasFreshPreservedMarker: hasFreshPreservedAudioMarker,
           reason: 'boot_sweep_unsaved',
         })
@@ -896,9 +995,40 @@ export function isSessionDeleted(sessionId: string): boolean {
 // A committed cross-boot maintenance operation owns durable state until the
 // trusted controller adopts the candidate. Boot recovery must not mutate or
 // promote sessions while that gate is closed.
-if (maintenanceAdmissionsOpen()) {
+//
+// 6.71.14: and once that gate opens, recovery MUST still run. Until 6.71.13 this was
+// a module-scope `if` evaluated once at import: a boot behind a closed gate (every
+// managed update) skipped it for the life of the process. Persisted sessions were
+// never adopted or tombstoned, and the closed-session tombstones were never
+// restored. On 2026-10-12 that left `active-sessions/<id>.json` for a receipted
+// capture on disk for 17 hours, reported as live by COS Control and `missing` by
+// the status route.
+let durableSessionRecoveryRan = false
+function runDurableSessionRecovery(trigger: 'boot' | 'admissions_open'): void {
+  if (durableSessionRecoveryRan) return
+  durableSessionRecoveryRan = true
   recoverClosedSessions()
   recoverSessions()
+  if (trigger === 'admissions_open') {
+    console.log(`[session-recovery] Ran deferred durable recovery after the maintenance gate opened (${sessions.size} live session(s))`)
+  }
+}
+if (maintenanceAdmissionsOpen()) {
+  runDurableSessionRecovery('boot')
+} else {
+  // Delivered on every later open; the flag above makes every delivery after the
+  // first a no-op, so disk is never re-read over a live session map.
+  onMaintenanceAdmissionsOpen(() => runDurableSessionRecovery('admissions_open'))
+}
+
+/**
+ * 6.71.14: has durable session recovery run in this process yet? Until it has, the
+ * in-memory map does not yet hold the persisted sessions, so "not in memory" is not
+ * evidence of anything. The status route answers 503 `recovery_pending` instead of
+ * `missing` while this is false.
+ */
+export function durableSessionRecoveryHasRun(): boolean {
+  return durableSessionRecoveryRan
 }
 
 /**
@@ -1011,6 +1141,10 @@ function promoteStrandedCapture(sessionId: string, lastActivityAt: number): void
 // Auto-cleanup sessions idle for the advertised retention horizon.
 setInterval(() => {
   if (!maintenanceAdmissionsOpen()) return
+  // 6.71.14: the sweeps below judge a session dead by its absence from the
+  // in-memory map. Before recovery has adopted the persisted sessions that map is
+  // empty by construction, so sweeping then would quarantine a live capture.
+  if (!durableSessionRecoveryRan) return
   // A capture whose phone went away must still become a meeting. This loop used
   // to close every idle session as 'expired', which quarantined its audio and
   // produced NO meeting — recoverable for 72h, but only by whoever thought to
@@ -1053,6 +1187,15 @@ setInterval(() => {
     // log, and nothing to reason about, because any throw in this block vanished.
     console.error(`[cleanup] Orphan-audio sweep failed: ${error instanceof Error ? error.message : error}`)
   }
+  // 6.71.14: its own try, for the same reason as auto-recovery below. An
+  // active-session file whose session is not live and has been idle past the
+  // retention horizon is moved aside; COS Control reads that directory directly
+  // and would otherwise show it as a live recording indefinitely.
+  try {
+    sweepStaleActiveSessionFilesNow(Date.now())
+  } catch (error) {
+    console.error(`[cleanup] Stale active-session sweep failed: ${error instanceof Error ? error.message : error}`)
+  }
   // DELIBERATELY ITS OWN TRY. This used to sit inside the block above, one line after
   // purgeExpiredQuarantine, so a throw anywhere in that sweep meant auto-recovery
   // silently never ran — which is exactly what happened in production on 6.23.1

package/shared/meeting-ask.ts

@@ -170,6 +170,19 @@ export interface MeetingAskExchange {
     files: MeetingAskFile[];
     error?: string;
     retryOf?: string;
+    /** Server 6.71.13 (capabilities.factCheckScope): what a completed Fact check covered. Named-claim checks and follow-ups carry none. */
+    factCheckRange?: MeetingAskFactCheckRange;
+}
+/**
+ * What one Fact check covered, in meeting-elapsed ms (the clock of throughMs and turn.elapsedMs). The first
+ * check covers the whole meeting (full); each later one covers only what is new since the last completed
+ * check's throughMs. A check that found nothing new still completes with claims 0, so the next one moves on.
+ */
+export interface MeetingAskFactCheckRange {
+    fromMs: number;
+    throughMs: number;
+    full: boolean;
+    claims: number;
 }
 export interface MeetingAskCapabilities {
     protocolVersion: 2;
@@ -179,6 +192,8 @@ export interface MeetingAskCapabilities {
     publicTopics?: true;
     /** Explicit recent-claim Fact check CTA; transport remains action=ask. */
     factCheck?: true;
+    /** 6.71.13: the server checks the whole meeting first, then only what is new, and says which in factCheckRange. */
+    factCheckScope?: 1;
     /** 6.71.1: POST /api/meeting-ask/files/media turns uploaded media into meeting files. */
     phoneFiles?: true;
     /** 6.71.1: config carries cosSearch and PUT config {cosSearch} changes it. */