@gotcos/glasses-server 6.36.26 → 6.36.27

package/CHANGELOG.md

@@ -1,3 +1,41 @@
+## 6.36.27
+
+The sidecar renders one request at a time. The server finally acts like it.
+
+WHAT BROKE. Chunking (6.36.25) split a reply into 9 segments and /prepare
+pre-warmed all 9 at once. The synthesis timeout was armed when a request was
+ISSUED, so it ran while that request sat in the sidecar's queue. Measured on a
+6,781-character reply: renders took ~2.6s each, but segment 5 spent 11.5s of its
+12,000ms budget waiting for a turn. On device it tipped over, the pre-warm
+returned 502, and iOS surfaced it as NotSupportedError. Five of nine segments
+played.
+
+The 12,000ms constant was not wrong when it was written -- its own comment says
+it exists to "bound hung sidecar so local_first can fall back before session TTL
+(~60s)", from an era when a reply was ONE render. Chunking changed the input and
+nothing re-derived the limit.
+
+- New render gate in tts-local.ts. One render reaches the sidecar at a time, so
+  the synthesis timeout now bounds RENDER, which is what it always claimed to
+  bound. Queue wait is governed separately.
+- Queue wait has its own DERIVED ceiling: one synthesis budget per render ahead,
+  plus one budget of headroom. The headroom is not slack -- without it the first
+  waiter's ceiling expires in a dead heat with the holder's own timeout, and a
+  request that was about to be served is rejected in the same tick. Found by the
+  test, not by reasoning.
+- /play outranks /prepare pre-warm. A user is waiting on the first and nobody is
+  waiting on the second; without priority, segment 5's playback request queues
+  behind pre-warms for 6, 7 and 8 -- work not needed for minutes. Priority never
+  reorders playback against itself.
+- /api/health tts_local now reports renderQueueDepth. This was not observable on
+  2026-08-23 and that cost an evening.
+
+Every guard mutation-verified, including one that only counting could catch:
+dropping the waiter's detach left all ten behaviour tests green while leaking an
+abort listener per queued render (45 of them on a 46-segment reply).
+
+217 files, 3042 tests.
+
 ## 6.36.26
 
 Deadline hardening for the segmented TTS path shipped in 6.36.25. Three constants

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.36.26",
+  "version": "6.36.27",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/lib/tts-local.ts

@@ -269,6 +269,11 @@ export function getLocalTtsHealth(): {
   starting: boolean
   error: string | null
   lastFallbackToOpenAI: { at: string; reason: string } | null
+  /** Renders in flight plus queued. A reply is chunked into up to MAX_CHUNKS
+   *  segments that all pre-warm at once, and the sidecar renders one at a time,
+   *  so this is the number that explains a slow or failing segment. It was not
+   *  observable on 2026-08-23 and that cost an evening. */
+  renderQueueDepth: number
 } {
   return {
     ready: serverAvailable,
@@ -282,6 +287,7 @@ export function getLocalTtsHealth(): {
     starting: serverStarting,
     error: lastError,
     lastFallbackToOpenAI,
+    renderQueueDepth: localRenderQueueDepth(),
   }
 }
 
@@ -390,15 +396,170 @@ export function stopLocalTtsServer(): void {
 }
 
 /** Synthesize via local OpenAI-shaped speech endpoint. Returns full audio Buffer. */
+
+// ── render gate ──────────────────────────────────────────────────────────────
+//
+// The Kokoro sidecar renders ONE request at a time behind its own lock. Issuing
+// N requests concurrently therefore does not make them finish any sooner; it
+// only makes each one's timeout run while it waits its turn.
+//
+// That is what broke playback on 2026-08-23. Chunking turned a reply into 9
+// renders, all fired at once by /prepare's pre-warm. Measured on a 6,781-char
+// reply: each render took ~2.6s, but segment 5's request spent 11.5s of its
+// 12,000ms budget QUEUED. On a slightly busier machine it tipped over, returned
+// 502, and iOS surfaced it as NotSupportedError. Five of nine segments played.
+//
+// The gate makes that queue explicit on our side, which buys two things:
+//
+//   1. The synthesis timeout starts when a request ACQUIRES the gate, so it
+//      bounds RENDER time -- what it was always described as bounding. Waiting
+//      for a turn is governed separately, by a derived ceiling (below).
+//   2. Playback jumps ahead of pre-warm. A /play request has a user waiting on
+//      it; a /prepare pre-warm does not. Without priority, segment 5's playback
+//      request queues behind the pre-warms for 6, 7 and 8 -- work that is not
+//      needed until minutes later.
+//
+// Priority never reorders playback against itself: a priority waiter is placed
+// after other priority waiters and before every background one.
+
+type RenderWaiter = {
+  priority: boolean
+  resolve: () => void
+  reject: (err: Error) => void
+  signal?: AbortSignal
+  detach?: () => void
+}
+
+let gateBusy = false
+const gateWaiters: RenderWaiter[] = []
+
+/** In-flight plus queued renders. Exposed for /api/health and tests. */
+export function localRenderQueueDepth(): number {
+  return gateWaiters.length + (gateBusy ? 1 : 0)
+}
+
+/** Test-only: drop all queued waiters and free the gate. */
+export function __resetRenderGate(): void {
+  for (const w of gateWaiters.splice(0)) {
+    w.detach?.()
+    w.reject(makeAbortError('render gate reset'))
+  }
+  gateBusy = false
+}
+
+function makeAbortError(message: string): Error {
+  const err = new Error(message)
+  err.name = 'AbortError'
+  return err
+}
+
+function releaseRenderGate(): void {
+  const next = gateWaiters.shift()
+  if (!next) {
+    gateBusy = false
+    return
+  }
+  // The gate stays held; ownership passes straight to `next`. Setting
+  // gateBusy = false here would let a later arrival barge in ahead of a
+  // waiter that has already been promised the slot.
+  next.detach?.()
+  next.resolve()
+}
+
+function acquireRenderGate(priority: boolean, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.reject(makeAbortError('local TTS request aborted'))
+  if (!gateBusy) {
+    gateBusy = true
+    return Promise.resolve()
+  }
+  return new Promise<void>((resolve, reject) => {
+    const waiter: RenderWaiter = { priority, resolve, reject, signal }
+
+    // DERIVED ceiling on queue wait, so a wedged sidecar cannot hang callers
+    // forever. Everyone ahead of us is individually bounded by the synthesis
+    // timeout, so the longest legitimate wait is that timeout once per render
+    // ahead of us -- the one in flight plus everyone already queued.
+    //
+    // The +1 is not slack, it is a race fix found by the test below. Without it
+    // the first waiter's ceiling is exactly one render budget, which expires in
+    // a dead heat with the holder's OWN timeout: the holder times out, releases
+    // the gate, and the waiter that was about to be served is rejected in the
+    // same tick. One extra budget of headroom means the queue always outlives
+    // the thing it is waiting for.
+    const ahead = gateWaiters.length + 1
+    const waitCeilingMs = (ahead + 1) * effectiveSynthTimeoutMs()
+    const timer = setTimeout(() => {
+      const at = gateWaiters.indexOf(waiter)
+      if (at >= 0) gateWaiters.splice(at, 1)
+      waiter.detach?.()
+      const err = new Error(
+        `local TTS queue wait exceeded ${waitCeilingMs}ms behind ${ahead} render(s)`,
+      )
+      err.name = 'TimeoutError'
+      reject(err)
+    }, waitCeilingMs)
+    if (typeof timer === 'object' && timer && 'unref' in timer) {
+      ;(timer as { unref: () => void }).unref()
+    }
+
+    const onAbort = () => {
+      const at = gateWaiters.indexOf(waiter)
+      if (at >= 0) gateWaiters.splice(at, 1)
+      waiter.detach?.()
+      reject(makeAbortError('local TTS request aborted'))
+    }
+    waiter.detach = () => {
+      clearTimeout(timer)
+      signal?.removeEventListener('abort', onAbort)
+    }
+    signal?.addEventListener('abort', onAbort, { once: true })
+
+    if (priority) {
+      const firstBackground = gateWaiters.findIndex((w) => !w.priority)
+      if (firstBackground < 0) gateWaiters.push(waiter)
+      else gateWaiters.splice(firstBackground, 0, waiter)
+    } else {
+      gateWaiters.push(waiter)
+    }
+  })
+}
+
+function effectiveSynthTimeoutMs(): number {
+  return Number.isFinite(LOCAL_TTS_SYNTH_TIMEOUT_MS) && LOCAL_TTS_SYNTH_TIMEOUT_MS > 0
+    ? LOCAL_TTS_SYNTH_TIMEOUT_MS
+    : 12_000
+}
+
 export async function synthesizeLocalTts(opts: {
   text: string
   voice: string
   format: string
   signal?: AbortSignal
+  /** True for a request a user is waiting on (/play). False/undefined for
+   *  background pre-warm, which yields the sidecar to playback. */
+  priority?: boolean
 }): Promise<Buffer> {
-  const timeoutMs = Number.isFinite(LOCAL_TTS_SYNTH_TIMEOUT_MS) && LOCAL_TTS_SYNTH_TIMEOUT_MS > 0
-    ? LOCAL_TTS_SYNTH_TIMEOUT_MS
-    : 12_000
+  // Wait for the sidecar BEFORE starting the synthesis clock. The timeout
+  // below is a render budget; it was previously charged for queue time too,
+  // which is the whole reason segment 5 of 9 died at 12s while rendering in
+  // 2.6s. See the render gate above.
+  await acquireRenderGate(opts.priority === true, opts.signal)
+  try {
+    return await synthesizeLocalTtsRender(opts)
+  } finally {
+    releaseRenderGate()
+  }
+}
+
+/** The actual sidecar call. Never invoke directly -- it assumes the caller
+ *  holds the render gate, and the timeout it starts is a RENDER budget. */
+async function synthesizeLocalTtsRender(opts: {
+  text: string
+  voice: string
+  format: string
+  signal?: AbortSignal
+}): Promise<Buffer> {
+  const timeoutMs = effectiveSynthTimeoutMs()
   const timeoutSignal = AbortSignal.timeout(timeoutMs)
   const signal =
     opts.signal && typeof AbortSignal.any === 'function'

package/server/routes/tts.ts

@@ -454,6 +454,9 @@ async function generateLocalIntoCache(
   voice: string,
   format: string,
   signal?: AbortSignal,
+  // A user is waiting on a /play render; nobody is waiting on a pre-warm.
+  // Defaults to background so a new caller cannot accidentally starve playback.
+  priority = false,
 ): Promise<{ ok: true } | { ok: false; status: number; message: string }> {
   // A memory/latency bound, NOT OpenAI's 4096. Kokoro reads what it is handed;
   // the old shared cap truncated local speech at ~3-4 pages for no reason that
@@ -471,7 +474,7 @@ async function generateLocalIntoCache(
   }
   try {
     // Local path ignores COS_VOICE_INSTRUCTIONS / per-request instructions.
-    const bytes = await synthesizeLocalTts({ text: spoken, voice, format, signal })
+    const bytes = await synthesizeLocalTts({ text: spoken, voice, format, signal, priority })
     if (!bytes.length) {
       abortEntry(hash)
       return { ok: false, status: 502, message: 'local TTS returned empty body' }
@@ -496,10 +499,11 @@ async function generateIntoCache(
   format: string,
   instructions: string,
   signal?: AbortSignal,
+  priority = false,
 ): Promise<{ ok: true } | { ok: false; status: number; message: string }> {
   if (getCached(hash)) return { ok: true }
   if (decision.backend === 'local') {
-    return generateLocalIntoCache(hash, text, decision.backendVoice, format, signal)
+    return generateLocalIntoCache(hash, text, decision.backendVoice, format, signal, priority)
   }
   return generateOpenAIIntoCache(
     hash,
@@ -520,6 +524,9 @@ async function generateWithFallback(opts: {
   enginePreference?: TtsEnginePreference | null
   signal?: AbortSignal
   sessionId?: string
+  /** True when a user is waiting (a /play cold miss). Background pre-warm
+   *  leaves it false so playback can take the sidecar ahead of it. */
+  priority?: boolean
 }): Promise<{ ok: true; hash: string } | { ok: false; status: number; message: string }> {
   const enginePreference = opts.enginePreference ?? null
   const preferOpenAI = enginePreference === 'openai'
@@ -553,6 +560,7 @@ async function generateWithFallback(opts: {
     opts.format,
     opts.instructions,
     opts.signal,
+    opts.priority === true,
   )
   if (primary.ok) {
     if (softEscapedToOpenAI) {
@@ -586,6 +594,7 @@ async function generateWithFallback(opts: {
       opts.format,
       opts.instructions,
       opts.signal,
+      opts.priority === true,
     )
     if (localResult.ok) {
       if (opts.sessionId) rebindSessionHash(opts.sessionId, localHash)
@@ -616,6 +625,7 @@ async function generateWithFallback(opts: {
       opts.format,
       opts.instructions,
       opts.signal,
+      opts.priority === true,
     )
     if (openaiResult.ok) {
       announceKokoroFallbackToOpenAI(failReason)
@@ -803,6 +813,7 @@ ttsRouter.post('/tts/stream', async (req, res) => {
         voice: decision.backendVoice,
         format: requestedFormat,
         signal: upstreamController.signal,
+        priority: true,
       })
       if (upstreamController.signal.aborted) return
       res.writeHead(200, {
@@ -1112,6 +1123,8 @@ ttsRouter.get('/tts/play/:session', async (req, res) => {
     enginePreference,
     signal: upstreamController.signal,
     sessionId: req.params.session,
+    // /play cold miss: the listener is waiting on this render right now.
+    priority: true,
   })
 
   if (!result.ok) {