@gotcos/glasses-server 6.23.0 → 6.23.1

package/CHANGELOG.md

@@ -1,3 +1,40 @@
+## 6.23.1
+
+Closes the hole 6.23.0 left open, plus a lockfile version that 6.23.0 shipped out of
+sync with package.json.
+
+- **A restart used to re-open the bug.** 6.23.0 saves a stranded capture at the
+  4-hour cutoff, but only while the server stays up. `recoverSessions()` refuses to
+  load any session already past that cutoff at boot — it tombstones it — so a
+  restart, a COS Control update, or a crash at the wrong moment meant the sweeper
+  never saw the session and its audio landed in quarantine with no meeting. Not
+  hypothetical: `meeting_1786237535593` (139 chunks, 31 MB, `idle_expiry_unsaved`)
+  arrived there that way.
+- **Quarantined audio now recovers itself.** The same 60-second tick picks ONE
+  unrecovered capture with chunks and asks the real
+  `POST /api/meeting/orphans/:id/recover` to turn it into a meeting. One at a time
+  because that route runs a full batch transcription — real GPU work, minutes for a
+  long capture — and a parallel backlog would starve a live recording. Oldest first,
+  since it is closest to the 72-hour purge.
+- **It gives up rather than looping.** Three attempts per capture, then it stops and
+  says so. A capture with unreadable chunks would otherwise be retried every 60
+  seconds for three days. The audio stays quarantined and recoverable by hand, which
+  beats a retry loop that never converges. A 409 from a manual recovery does not burn
+  the budget.
+- Recovered captures are titled "Recovered capture (audio only)", distinct from a
+  promoted session's "Auto-saved capture", because a quarantine recovery has no live
+  ASR and every speaker comes back Unknown. The library should say which is which
+  without opening the file.
+- **`package-lock.json` was still on 6.22.1 while package.json said 6.23.0.** Caught
+  by the repo's own `launcher-contract` test, which I did not re-run after bumping the
+  version. The published 6.23.0 code is unaffected; the lockfile is now aligned and
+  the suite runs after the bump, not before it.
+
+Verified live before this release: a backdated synthetic session was recovered at
+boot, drafted by the sweeper within 60s, and promoted to a meeting at the cutoff
+about 40s later, with the draft cleared and the domain inferred rather than
+hardcoded. 14 new tests here, full suite 1425 serially.
+
 ## 6.23.0
 
 A recording whose phone goes away now becomes a meeting on its own. Miles: "we end

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.23.0",
+  "version": "6.23.1",
   "description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/lib/quarantine-auto-recover.ts

@@ -0,0 +1,148 @@
+// Quarantined audio that nobody asked to recover still has to become a meeting.
+//
+// THE HOLE THIS CLOSES. 6.23.0 made a stranded session save itself at the 4-hour
+// cutoff, but that only works while the server stays up. `recoverSessions()`
+// refuses to load any session already past the cutoff at boot — it tombstones it —
+// so a restart, a COS Control update, or a crash at the wrong moment means the
+// sweeper never sees the session, its audio dir is orphaned, and it lands in
+// quarantine with no meeting. That is exactly the old behavior, reached by a
+// different door, and it is not hypothetical: `meeting_1786237535593` (139 chunks,
+// 31 MB, reason `idle_expiry_unsaved`) came through it.
+//
+// Quarantine already keeps the audio for 72 hours and `POST
+// /api/meeting/orphans/:id/recover` already turns it into a meeting. The only thing
+// missing was that a human had to notice and press it. This picks one per sweep.
+//
+// WHY ONE AT A TIME. Recovery runs a full batch transcription over every chunk WAV
+// — real GPU and CPU work, minutes for a long capture. Recovering a backlog in
+// parallel would starve a live recording, so the sweep takes the oldest and leaves
+// the rest for the next tick. The route's own `shouldAbort` already yields to a live
+// recording once started.
+//
+// WHY THE ATTEMPT LEDGER. A capture that cannot be recovered — unreadable chunks, a
+// codec the batch path rejects — would otherwise be retried every 60 seconds for 72
+// hours, burning the machine and drowning the log. After a few failures it is left
+// alone: the audio is still preserved and still recoverable by hand, which is
+// strictly better than a retry loop that never converges.
+
+import type { UnsavedCapture } from './unsaved-audio-quarantine.js'
+
+/** Attempts per capture before the sweep stops trying on its own. */
+export const MAX_AUTO_RECOVER_ATTEMPTS = 3
+
+/**
+ * Title for a capture the sweep recovers.
+ *
+ * Distinct from the promote title on purpose. A promoted session carried a live ASR
+ * transcript with speaker labels; a quarantine recovery has neither, because no live
+ * ASR ever ran on it — every speaker comes back Unknown. The user should be able to
+ * tell those two apart in the library without opening them.
+ */
+export const AUTO_RECOVER_TITLE = 'Recovered capture (audio only)'
+
+export interface AutoRecoverState {
+  /** sessionId → attempts already made this process lifetime. */
+  attempts: Map<string, number>
+  /** Recoveries currently running, from the route's own set. */
+  inFlight: ReadonlySet<string>
+}
+
+/**
+ * Which quarantined capture, if any, should the sweep recover next?
+ *
+ * Returns null when there is nothing to do — the common case — so the caller does no
+ * work on a quiet tick.
+ */
+export function pickQuarantineToRecover(
+  items: readonly UnsavedCapture[],
+  state: AutoRecoverState,
+): UnsavedCapture | null {
+  const eligible = items.filter(item => {
+    // Already a meeting. Recovering again would duplicate it.
+    if (item.recovered) return false
+    // Nothing to transcribe: a chunk-less dir is residue, not evidence.
+    if (item.chunkFiles <= 0) return false
+    // Another recovery owns this one.
+    if (state.inFlight.has(item.sessionId)) return false
+    return (state.attempts.get(item.sessionId) ?? 0) < MAX_AUTO_RECOVER_ATTEMPTS
+  })
+  if (eligible.length === 0) return null
+  // Oldest first: it is closest to the 72-hour purge, so it has the least time left.
+  // `ageHours` can be null when the marker is unreadable — treat that as oldest
+  // rather than newest, because an unreadable marker is itself a sign of an old dir.
+  return [...eligible].sort((a, b) => (b.ageHours ?? Number.MAX_SAFE_INTEGER)
+    - (a.ageHours ?? Number.MAX_SAFE_INTEGER))[0] ?? null
+}
+
+/** Record an attempt. Called BEFORE the request, so a hang still counts. */
+export function noteRecoverAttempt(state: AutoRecoverState, sessionId: string): void {
+  state.attempts.set(sessionId, (state.attempts.get(sessionId) ?? 0) + 1)
+}
+
+/** Clear the ledger for a capture that succeeded, so a later re-quarantine is fresh. */
+export function clearRecoverAttempts(state: AutoRecoverState, sessionId: string): void {
+  state.attempts.delete(sessionId)
+}
+
+export function autoRecoverExhausted(state: AutoRecoverState, sessionId: string): boolean {
+  return (state.attempts.get(sessionId) ?? 0) >= MAX_AUTO_RECOVER_ATTEMPTS
+}
+
+export interface RecoverRequestResult {
+  ok: boolean
+  status: number
+  filename?: string
+  reason?: string
+}
+
+export interface RecoverRequestOptions {
+  port: number
+  token: string
+  fetchImpl?: typeof fetch
+  timeoutMs?: number
+}
+
+/**
+ * Ask the real recover route to turn a quarantined capture into a meeting.
+ *
+ * Loopback for the same reason promote is: that route owns the recovering-set
+ * guard, the batch transcription, the `shouldAbort` yield to a live recording, the
+ * receipt and the `markRecovered` stamp. Reimplementing any of it here would fork
+ * the path a user's own button press takes.
+ *
+ * A generous default timeout: batch transcription of a long capture is minutes of
+ * real work, and aborting early would leave the route running with no one reading
+ * the result.
+ */
+export async function requestQuarantineRecovery(
+  sessionId: string,
+  options: RecoverRequestOptions,
+): Promise<RecoverRequestResult> {
+  if (!options.token) return { ok: false, status: 0, reason: 'no_token' }
+  const doFetch = options.fetchImpl ?? fetch
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 1_800_000)
+  try {
+    const res = await doFetch(
+      `http://127.0.0.1:${options.port}/api/meeting/orphans/${encodeURIComponent(sessionId)}/recover`,
+      {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', 'X-Cos-Token': options.token },
+        body: JSON.stringify({ title: AUTO_RECOVER_TITLE }),
+        signal: controller.signal,
+      },
+    )
+    let payload: Record<string, unknown> = {}
+    try { payload = await res.json() as Record<string, unknown> } catch {}
+    return {
+      ok: res.ok,
+      status: res.status,
+      filename: typeof payload.filename === 'string' ? payload.filename : undefined,
+      reason: typeof payload.reason === 'string' ? payload.reason : undefined,
+    }
+  } catch (error: any) {
+    return { ok: false, status: 0, reason: error?.name === 'AbortError' ? 'timeout' : 'request_failed' }
+  } finally {
+    clearTimeout(timer)
+  }
+}

package/server/routes/transcribe-stream.ts

@@ -23,6 +23,14 @@ import {
   writeStrandedDraft,
 } from '../lib/stranded-session-actions.js'
 import { getSessionHeartbeat } from '../lib/session-heartbeats.js'
+import {
+  MAX_AUTO_RECOVER_ATTEMPTS,
+  autoRecoverExhausted,
+  clearRecoverAttempts,
+  noteRecoverAttempt,
+  pickQuarantineToRecover,
+  requestQuarantineRecovery,
+} from '../lib/quarantine-auto-recover.js'
 import { TranscriptionUnavailableError } from '../lib/transcribe-audio.js'
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url))
@@ -56,6 +64,7 @@ import {
 import { archiveSessionAudio, runMeetingAudioRetention } from '../lib/meeting-audio-archive.js'
 import {
   countChunkWavs,
+  listUnsavedCaptures,
   purgeExpiredQuarantine,
   quarantineSessionAudio,
   sweepOrphanedSessionAudio,
@@ -836,6 +845,57 @@ if (maintenanceAdmissionsOpen()) {
   recoverSessions()
 }
 
+/**
+ * Quarantined audio has no live session, so it needs its own pass.
+ *
+ * This closes the hole 6.23.0 left open: `recoverSessions()` tombstones any session
+ * already past the 4-hour cutoff at boot, so a restart while a capture is stranded
+ * means the promote above never runs and the audio lands here instead — preserved
+ * for 72 hours, but not a meeting, and only recoverable by hand. One per tick.
+ */
+const autoRecoverState = {
+  attempts: new Map<string, number>(),
+  inFlight: new Set<string>(),
+}
+
+function autoRecoverOneQuarantinedCapture(): void {
+  const token = process.env.COS_API_TOKEN ?? ''
+  if (!token) return
+  const picked = pickQuarantineToRecover(listUnsavedCaptures(), autoRecoverState)
+  if (!picked) return
+  const sessionId = picked.sessionId
+  autoRecoverState.inFlight.add(sessionId)
+  noteRecoverAttempt(autoRecoverState, sessionId)
+  void requestQuarantineRecovery(sessionId, {
+    port: parseInt(process.env.PORT ?? '3141', 10),
+    token,
+  })
+    .then(result => {
+      if (result.ok) {
+        console.warn(
+          `[quarantine] Auto-recovered ${sessionId} \u2192 ${result.filename ?? 'saved'} `
+          + `(${picked.chunkFiles} chunks, speakers unlabeled — no live ASR ran on it)`,
+        )
+        clearRecoverAttempts(autoRecoverState, sessionId)
+        return
+      }
+      // 409 means a manual recovery already owns it, which is not a failure and
+      // should not burn the budget.
+      if (result.status === 409) {
+        clearRecoverAttempts(autoRecoverState, sessionId)
+        return
+      }
+      const attempts = autoRecoverState.attempts.get(sessionId) ?? 0
+      console.error(
+        `[quarantine] Auto-recovery failed for ${sessionId} `
+        + `(${result.status} ${result.reason ?? ''}), attempt ${attempts}/${MAX_AUTO_RECOVER_ATTEMPTS}`
+        + `${autoRecoverExhausted(autoRecoverState, sessionId)
+          ? ' — giving up, audio stays quarantined and recoverable by hand' : ''}`,
+      )
+    })
+    .finally(() => autoRecoverState.inFlight.delete(sessionId))
+}
+
 /** Sessions with an auto-save in flight, so a later tick cannot start a second. */
 const promotingStranded = new Set<string>()
 
@@ -931,6 +991,7 @@ setInterval(() => {
       }
     }
     purgeExpiredQuarantine()
+    autoRecoverOneQuarantinedCapture()
   } catch {}
   // Purge stale pending-batch dirs. HQ large-v3 on long meetings + Control
   // restart can exceed 2h (2026-07-27: two sessions purged before batch).