@gotcos/glasses-server 6.21.19 → 6.21.21

package/CHANGELOG.md

@@ -1,3 +1,48 @@
+## 6.21.21
+
+- One transient process probe no longer disables local Whisper for the whole
+  server lifetime. On 2026-08-06 the `/bin/ps` probe inside the startup
+  preflight failed exactly once, during a generation changeover while the
+  previous generation's Python child was shutting down. Startup caught it, set
+  state `failed`, and stopped — whisper never launched, port 8178 refused for
+  ~30 minutes across a live recording, and only a manual restart recovered it.
+  Boot calls `startWhisperServer()` once, and the only other recovery path is a
+  circuit breaker that first needs three failed transcriptions, so nothing ever
+  retried the thing that had actually failed.
+
+  Both preflight probes now retry three times with backoff. `lsof` exit 1 means
+  "no matches" and is still passed straight through, so the common path costs
+  exactly one probe. Preflight still fails CLOSED after the retries: it exists
+  to prove no orphaned whisper owns port 8178, and spawning without that proof
+  risks two owners.
+
+- A surviving probe failure now names its own cause. The field failure logged
+  only `Command failed: /bin/ps …` with empty stderr — identical output for a
+  timeout kill, a non-zero exit, and a failed fork — which cost an
+  investigation and still did not settle the mechanism (measured, this probe
+  runs in ~45ms against a 2s timeout, so "it timed out" was never established).
+  Errors now carry per-attempt elapsed time, `killed`, `signal`, `code`, and
+  stderr, deduplicated so all three attempts survive the 240-character bound.
+
+## 6.21.20
+
+- Audio playback works at all. Every play button in the Control speaker review
+  returned a 404 and made no sound, on every default install, since playback
+  shipped. `res.sendFile` delegates to `send`, which defaults to
+  `dotfiles: 'ignore'` and — with no `root` set — applies that policy to the
+  WHOLE absolute path rather than to anything the request supplied. The default
+  data home is `~/.cos-glasses/data`, and `.cos-glasses` is a dot component, so
+  the file was found, confirmed to exist, then refused on the way out the door.
+  All three audio routes were affected: meeting chunks, speaker-profile samples,
+  and ext-audio samples.
+
+  The tests could not have caught this. They point `COS_DATA_DIR` at
+  `mktemp -d` — `/var/folders/...` — which cannot contain a dot component, so
+  the suite was structurally incapable of reproducing a default install and
+  stayed green while the feature was dead. The three routes now share
+  `sendAudioFile`, and `send-audio.test.ts` serves from a dot-directory on
+  purpose, over a real listener, asserting on the returned bytes.
+
 ## 6.21.19
 
 - Review playback falls back to ext-audio. The 7-day archive introduced in

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.21.19",
+  "version": "6.21.21",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/lib/send-audio.ts

@@ -0,0 +1,42 @@
+// Serving a retained WAV back to a reviewer.
+//
+// WHY THIS EXISTS AT ALL, because `res.sendFile(path)` looks like it would do.
+//
+// `send` (what Express delegates to) defaults to `dotfiles: 'ignore'`, and when
+// no `root` option is given it applies that policy to the ENTIRE absolute path,
+// not to the part a request supplied:
+//
+//   parts = normalize(path).split(sep)      // send/index.js — the whole path
+//   if (containsDotFile(parts)) ... error(404)
+//
+// The COS data home is `~/.cos-glasses/data` by default. `.cos-glasses` is a
+// dot component, so every audio route 404'd on every default install — the play
+// button rendered, the fetch failed, and nothing was heard. Verified directly
+// against the installed module: `{}` -> 404, `{dotfiles:'allow'}` -> resolves.
+//
+// The tests never caught it because they point `COS_DATA_DIR` at
+// `mktemp -d` (`/var/folders/...`), which structurally CANNOT contain a dot
+// component. The suite was green and the feature was dead. `send-audio.test.ts`
+// serves from a dot-directory on purpose.
+//
+// 'allow' is safe here rather than merely convenient: the dot component is ours
+// (the data home), and nothing a caller supplies can introduce one. Every route
+// resolves its path through a validator first — `sessionId` and speaker names
+// match `[A-Za-z0-9:_-]`, chunk indices are integers, and the archive helpers
+// re-check containment before returning. The path handed to this function is
+// already known to exist and to live under the data dir.
+
+import type { Response } from 'express'
+
+/**
+ * Send a WAV that has already been resolved and containment-checked.
+ *
+ * Callers must have verified the file exists; a missing file here surfaces as
+ * send's own 404 HTML rather than the route's JSON, which is exactly the
+ * confusing failure this module documents.
+ */
+export function sendAudioFile(res: Response, path: string): void {
+  res.type('audio/wav')
+  // See the file header: without this, any data home under a dot-directory 404s.
+  res.sendFile(path, { dotfiles: 'allow' })
+}

package/server/lib/whisper-local.ts

@@ -283,6 +283,23 @@ const WHISPER_SERVER_PORT = 8178
 const WHISPER_SERVER_URL = `http://127.0.0.1:${WHISPER_SERVER_PORT}`
 const PROCESS_PROBE_TIMEOUT_MS = 2_000
 const PROCESS_PROBE_MAX_BUFFER = 1024 * 1024
+// A process probe failing ONCE must not disable local Whisper for the whole
+// server lifetime. On 2026-08-06 the `/bin/ps` probe in the startup preflight
+// failed exactly once, during the 6.21.20 generation changeover while the
+// previous generation's Python child was still shutting down.
+// `startWhisperServer` caught it, set state 'failed', and never tried again:
+// whisper-server never launched, port 8178 refused for ~30 minutes across a
+// live recording, and only a manual restart recovered it. Boot calls
+// `startWhisperServer()` exactly once, and the only other recovery path is a
+// circuit breaker that first needs three failed transcriptions — so nothing
+// retried the thing that had actually failed.
+//
+// Three attempts at 150/300ms covers a transient blip while still failing in
+// well under a second when the probe is genuinely broken. Preflight must keep
+// failing CLOSED after that: it exists to prove no orphaned whisper owns port
+// 8178, and spawning without that proof risks two owners.
+const PROCESS_PROBE_ATTEMPTS = 3
+const PROCESS_PROBE_RETRY_BASE_MS = 150
 const PS_BIN = '/bin/ps'
 const LSOF_BIN = existsSync('/usr/sbin/lsof') ? '/usr/sbin/lsof' : 'lsof'
 
@@ -338,14 +355,90 @@ function runProcessProbe(file: string, args: string[]): Promise<string> {
   })
 }
 
+/**
+ * Describe a probe failure precisely enough that the NEXT one diagnoses itself.
+ *
+ * The 2026-08-06 failure logged only `Command failed: /bin/ps ...` with empty
+ * stderr, which is what execFile emits for a non-zero exit, a timeout kill, and
+ * a fork failure alike. That single ambiguous string cost an investigation and
+ * still did not settle the mechanism: measured, this probe runs in ~45ms against
+ * a 2s timeout, so "it timed out" was never established. Record the fields that
+ * discriminate — signal and `killed` mean the timeout fired, a numeric `code`
+ * means ps itself exited non-zero, `EAGAIN`/`ENOMEM` means the fork failed —
+ * plus how long it actually took.
+ */
+function describeProbeFailure(error: any, elapsedMs: number, attempt: number): string {
+  const parts = [`attempt ${attempt}`, `${elapsedMs}ms`]
+  if (error?.killed) parts.push('killed=true')
+  if (error?.signal) parts.push(`signal=${error.signal}`)
+  if (error?.code !== undefined && error?.code !== null) parts.push(`code=${error.code}`)
+  const stderr = String(error?.stderr ?? '').trim()
+  if (stderr) parts.push(`stderr=${stderr.slice(0, 80)}`)
+  return parts.join(' ')
+}
+
+/**
+ * Fold per-attempt details into one line that survives `boundedError`'s 240
+ * characters.
+ *
+ * Repeating an identical 48-character "Command failed: /bin/ps …" once per
+ * attempt spent most of the budget and truncated attempt 3 — the most recent
+ * and most diagnostic one. A test caught that. So the message is emitted ONCE
+ * when every attempt failed the same way, and per-attempt only when they
+ * genuinely differ, which is itself a signal worth seeing.
+ */
+function summarizeProbeFailures(attempts: Array<{ message: string; detail: string }>): string {
+  const distinct = [...new Set(attempts.map(a => a.message))]
+  if (distinct.length === 1) {
+    return `${distinct[0]} [${attempts.map(a => a.detail).join('; ')}]`
+  }
+  return attempts.map(a => `${a.message} (${a.detail})`).join(' | ')
+}
+
+/**
+ * Run a process probe, retrying a transient failure.
+ *
+ * `isExpectedFailure` short-circuits retries for a failure that is a normal
+ * result rather than a fault — `lsof` exits 1 to mean "no matches", and
+ * retrying that would triple the cost of the common case.
+ */
+async function runProcessProbeRetrying(
+  bin: string,
+  args: string[],
+  isExpectedFailure: (error: any) => boolean = () => false,
+): Promise<string> {
+  const failures: Array<{ message: string; detail: string }> = []
+  for (let attempt = 1; attempt <= PROCESS_PROBE_ATTEMPTS; attempt++) {
+    const startedAt = Date.now()
+    try {
+      return await runProcessProbe(bin, args)
+    } catch (error: any) {
+      if (isExpectedFailure(error)) throw error
+      const detail = describeProbeFailure(error, Date.now() - startedAt, attempt)
+      failures.push({ message: boundedError(error), detail })
+      if (attempt < PROCESS_PROBE_ATTEMPTS) {
+        console.warn(
+          `[whisper-local] ${bin} probe failed, retrying (${attempt}/${PROCESS_PROBE_ATTEMPTS}): ${detail}`,
+        )
+        await sleep(PROCESS_PROBE_RETRY_BASE_MS * attempt)
+      }
+    }
+  }
+  // Every attempt is reported, not just the last: a probe that fails three
+  // different ways is a different problem from one that fails identically.
+  throw new Error(summarizeProbeFailures(failures))
+}
+
 async function listProcesses(): Promise<ProcessEntry[]> {
   // `command=` includes arguments, which lets us distinguish this COS-owned
   // port/model signature from unrelated whisper-server instances.
   let output: string
   try {
-    output = await runProcessProbe(PS_BIN, ['-axww', '-o', 'pid=,ppid=,command='])
+    output = await runProcessProbeRetrying(PS_BIN, ['-axww', '-o', 'pid=,ppid=,command='])
   } catch (error) {
-    throw new Error(`unable to inspect process table: ${boundedError(error)}`)
+    throw new Error(
+      `unable to inspect process table after ${PROCESS_PROBE_ATTEMPTS} attempts: ${boundedError(error)}`,
+    )
   }
   return output.split('\n').flatMap(line => {
     const match = line.match(/^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/)
@@ -354,17 +447,25 @@ async function listProcesses(): Promise<ProcessEntry[]> {
   })
 }
 
+/** lsof exits 1 for "no matches" — a normal answer, not a fault. */
+const isLsofNoMatches = (err: any): boolean => err?.code === 1 || err?.status === 1
+
 async function listeningPids(): Promise<number[]> {
   try {
-    const output = await runProcessProbe(
+    // Retries for the same reason as the process-table probe: this also fails
+    // preflight CLOSED, so one transient blip here would equally disable
+    // whisper for the server's whole lifetime. "No matches" is passed through
+    // untouched so the common case still costs exactly one probe.
+    const output = await runProcessProbeRetrying(
       LSOF_BIN,
       ['-nP', `-iTCP:${WHISPER_SERVER_PORT}`, '-sTCP:LISTEN', '-t'],
+      isLsofNoMatches,
     )
     return output.split(/\s+/).map(Number).filter(pid => Number.isInteger(pid) && pid > 0)
   } catch (err: any) {
     // lsof uses exit 1 for "no matches". Anything else means we could not
     // prove the port state, so startup must fail closed.
-    if (err?.code === 1 || err?.status === 1) return []
+    if (isLsofNoMatches(err)) return []
     throw new Error(`unable to inspect whisper-server port ${WHISPER_SERVER_PORT}: ${boundedError(err)}`)
   }
 }

package/server/routes/meeting.ts

@@ -10,6 +10,7 @@ import { cleanTranscriptLines } from '../lib/hallucination-filter.js'
 import { durableAtomicWriteFileSync } from '../lib/atomic-fs.js'
 import { appendCorrection, pendingCorrections } from '../lib/meeting-corrections.js'
 import { isSampleFromSession, untraceableSampleCount } from '../lib/training-audio-provenance.js'
+import { sendAudioFile } from '../lib/send-audio.js'
 import {
   extAudioChunkPath,
   listExtAudioChunks,
@@ -1178,8 +1179,7 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
       })
       return
     }
-    res.type('audio/wav')
-    res.sendFile(path)
+    sendAudioFile(res, path)
   })
 
   /** What audio a meeting still has, so the panel can show play buttons only

package/server/routes/voice.ts

@@ -11,6 +11,7 @@ import { getOwnerSpeakerLabel } from '../lib/profile.js'
 import { dataPath } from '../lib/data-dir.js'
 import { purgeSpeakerCalibrationRows, relabelSpeakerCalibrationRows } from '../lib/speaker-calibration-log.js'
 import { trainingSourceFor } from '../lib/training-audio-provenance.js'
+import { sendAudioFile } from '../lib/send-audio.js'
 
 // These MUST match the writer in transcribe-stream.ts, which saves under
 // dataPath(). They previously resolved relative to __dirname — i.e. inside the
@@ -471,8 +472,7 @@ voiceRouter.get('/voice/profiles/:name/sample', (req, res) => {
     })
     return
   }
-  res.type('audio/wav')
-  res.sendFile(wav)
+  sendAudioFile(res, wav)
 })
 
 // GET /api/voice/ext-audio/:sessionId/sample — hear an unidentified voice.
@@ -492,8 +492,7 @@ voiceRouter.get('/voice/ext-audio/:sessionId/sample', (req, res) => {
     })
     return
   }
-  res.type('audio/wav')
-  res.sendFile(wav)
+  sendAudioFile(res, wav)
 })
 
 // GET /api/voice/profiles — enrolled people with sample counts and provenance.