@gotcos/glasses-server 6.21.23 → 6.21.24

package/CHANGELOG.md

@@ -1,3 +1,30 @@
+## 6.21.24
+
+- A leaked recording session can no longer block every restart. The maintenance
+  drain gate counted `sessions.size`, so a session whose phone dropped
+  mid-recording without sending a close pinned that count at 1 indefinitely and
+  Install, Repair, Restart and Update Server each drained it, timed out after
+  60s, and failed — with no user-visible reason. Hit twice on 2026-08-06
+  (6.21.20 and 6.21.22); the second phantom had been silent 54 minutes and the
+  only way through was to finalize it as a real meeting, which it was not.
+
+  The gate now counts only sessions active within the last 30 minutes. A stale
+  one is reported in status as `staleTranscriptionSessions` with its session id,
+  silent duration and chunk count, so a blocked operator can see the cause —
+  and `/api/meeting/orphans` reporting 0 can no longer coexist silently with a
+  held lock, since the two read different stores.
+
+  **Nothing is reaped.** The session, its chunks and its recoverability are
+  untouched; only its claim on the restart gate expires. The 30-minute window
+  sits far above the reasons a live recording legitimately goes quiet (a
+  backgrounded phone buffering to IndexedDB, a network drop) and far below the
+  4h durable-chunk retention.
+
+  Note for anyone reading the health payload: `oldestWorkStartedAt: null` does
+  NOT indicate a leak. `recording_session` reaches the gate through
+  `extraActiveByKind`, which never contributes a timestamp, so every recording
+  session reports null — healthy or not. Only `lastActivityAt` separates them.
+
 ## 6.21.23
 
 - `GET /meeting/:sessionId/embeddings` — why each chunk was labelled the way it

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.21.23",
+  "version": "6.21.24",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/routes/maintenance.ts

@@ -15,7 +15,7 @@ import { getQueryJobRuntimeHealth } from '../lib/query-job-runtime.js'
 import { serverMetrics } from '../lib/server-metrics.js'
 import { getServerInstanceId } from '../lib/server-instance-id.js'
 import { getWhisperHealth } from '../lib/whisper-local.js'
-import { getActiveTranscriptionSessionCount } from './transcribe-stream.js'
+import { getActiveTranscriptionSessionCount, getTranscriptionSessionLiveness } from './transcribe-stream.js'
 
 export const maintenanceRouter = Router()
 
@@ -71,15 +71,21 @@ function operationCredentials(req: Request): MaintenanceOperationCredentials {
 function statusSnapshot(credentials: MaintenanceOperationCredentials = {}) {
   const jobs = getQueryJobRuntimeHealth()
   const activeTranscriptionSessions = getActiveTranscriptionSessionCount()
+  // Only sessions still plausibly recording may hold the drain gate. A leaked
+  // session — phone dropped mid-recording, no close ever sent — otherwise pins
+  // this at 1 forever and every Install/Repair/Restart/Update drains, times out
+  // and fails with nothing to show the user. The stale ones are reported below
+  // rather than counted, and are never deleted here.
+  const sessionLiveness = getTranscriptionSessionLiveness()
   const managed = managedRuntimeCapability()
   const tracked = maintenanceLifecycle.snapshot(credentials, {
-    recording_session: activeTranscriptionSessions,
+    recording_session: sessionLiveness.live,
   })
   const untrackedDurableRuns = Math.max(0, jobs.activeRuns - (tracked.activeByKind.durable_query ?? 0))
   const lifecycle = untrackedDurableRuns > 0
     ? maintenanceLifecycle.snapshot(credentials, {
       durable_query_runtime: untrackedDurableRuns,
-      recording_session: activeTranscriptionSessions,
+      recording_session: sessionLiveness.live,
     })
     : tracked
   return {
@@ -91,6 +97,13 @@ function statusSnapshot(credentials: MaintenanceOperationCredentials = {}) {
     bootId: serverMetrics.bootId,
     activeJobs: jobs.activeRuns,
     activeTranscriptionSessions,
+    // Split out so a blocked operator can SEE why, and so "0 orphans" can never
+    // again coexist with a held lock: the orphans endpoint reads the quarantine
+    // directory, this reads the in-memory session map, and they are different
+    // stores. A stale session is surfaced here and blocks nothing.
+    liveTranscriptionSessions: sessionLiveness.live,
+    staleTranscriptionSessions: sessionLiveness.stale,
+    staleTranscriptionSessionDetail: sessionLiveness.staleSessions,
     shuttingDown: jobs.shuttingDown,
     durableStoreState: jobs.store.state,
     lifecycle,

package/server/routes/transcribe-stream.ts

@@ -342,6 +342,76 @@ export function getActiveTranscriptionSessionCount(): number {
   return sessions.size
 }
 
+/**
+ * How long a session may go silent before it stops BLOCKING a restart.
+ *
+ * Not a retention policy and not a reap: the session, its chunks and its
+ * recoverability are untouched. This decides one thing — whether it still
+ * counts toward the maintenance drain gate.
+ *
+ * Sized against the reasons a LIVE recording legitimately goes quiet: the phone
+ * backgrounded and buffering to IndexedDB, a network drop, a long pause. Those
+ * run to minutes. It sits deliberately far below
+ * LOCAL_FIRST_MEETING_IDLE_RETENTION_MS (4h), which governs how long chunks stay
+ * recoverable and is far too long to hold a restart on.
+ */
+export const RECORDING_SESSION_STALE_MS = 30 * 60 * 1000
+
+export interface TranscriptionSessionLiveness {
+  /** Sessions still plausibly recording. These block a restart. */
+  live: number
+  /** Sessions silent past the threshold. Surfaced, NOT counted, NOT deleted. */
+  stale: number
+  /** Which ones, least-silent first, so an operator can act on a name. */
+  staleSessions: Array<{ sessionId: string; silentForMs: number; chunks: number }>
+}
+
+/**
+ * Split active sessions into live and stale.
+ *
+ * WHY THIS EXISTS. The maintenance gate counts `sessions.size` and blocks every
+ * restart while it is non-zero. A session that leaks — the phone drops
+ * mid-recording and never sends a close — pins that count at 1 indefinitely, so
+ * Install, Repair, Restart and Update Server all drain, time out and fail with
+ * no user-visible reason. Observed twice on 2026-08-06 (servers 6.21.20 and
+ * 6.21.22); the second phantom had been silent 54 minutes and the only way
+ * through was to finalize it as a real meeting, which it was not.
+ *
+ * THE DISCRIMINATOR IS `lastActivityAt`, NOT `oldestWorkStartedAt`. The obvious
+ * reading — "activeTotal >= 1 with oldestWorkStartedAt null means leaked" — is
+ * WRONG and would strand healthy recordings. `recording_session` reaches the
+ * gate through `extraActiveByKind` (routes/maintenance.ts), and the snapshot's
+ * `oldestStartedAtMs` loop walks only tracked `work` entries, so an extra count
+ * never contributes a timestamp. EVERY recording session therefore reports
+ * `oldestWorkStartedAt: null`, healthy or leaked. Only the session's own
+ * `lastActivityAt` tells them apart.
+ */
+/**
+ * The live session map, for tests only.
+ *
+ * Exposed because the liveness split is pure logic over `lastActivityAt` but
+ * the map is module-private, and a test that cannot seed it can only assert the
+ * empty case. An earlier draft of the liveness test guarded on this symbol
+ * existing and silently no-opped four of its six cases — passing green while
+ * exercising nothing. Never guard a test on its own seam; make the seam real.
+ */
+export const __sessionsForTests = sessions
+
+export function getTranscriptionSessionLiveness(now = Date.now()): TranscriptionSessionLiveness {
+  let live = 0
+  const staleSessions: TranscriptionSessionLiveness['staleSessions'] = []
+  for (const [sessionId, session] of sessions) {
+    const silentForMs = now - session.lastActivityAt
+    if (silentForMs >= RECORDING_SESSION_STALE_MS) {
+      staleSessions.push({ sessionId, silentForMs, chunks: session.chunks.length })
+    } else {
+      live++
+    }
+  }
+  staleSessions.sort((a, b) => a.silentForMs - b.silentForMs)
+  return { live, stale: staleSessions.length, staleSessions }
+}
+
 // Incremental chunk persistence — survive server restarts
 const CHUNK_PERSIST_DIR = dataPath('active-sessions')
 ensurePrivateDirectory(CHUNK_PERSIST_DIR)