@gotcos/glasses-server 6.14.1 → 6.15.0

package/.env.example

@@ -70,6 +70,17 @@ BIND_HOST=0.0.0.0
 # COS_OPENAI_WHISPER_FALLBACK=1
 # OPENAI_API_KEY=sk-...
 
+# Spoken reply playback defaults to local Kokoro on Apple silicon Macs. The
+# first run creates a private venv and downloads the model. Local mode fails
+# closed: it never sends text to OpenAI. Set openai_primary only when cloud TTS
+# is desired and OPENAI_API_KEY is configured.
+# COS_TTS_ENGINE=local_first       # local_first | openai_primary
+# COS_TTS_LOCAL_PORT=8179
+# COS_TTS_LOCAL_TIMEOUT_MS=12000
+# COS_TTS_KOKORO_VOICE=am_echo
+# COS_TTS_LOCAL_DISABLE=1          # disable the local sidecar explicitly
+# COS_TTS_PRONUNCIATIONS_JSON={"Exampleco":{"local":"[Exampleco](/ɪgzˈæmpəlkoʊ/)","openai":"ig-ZAM-pul-co"}}
+
 # ── FULL COS PIPELINE (optional — leave unset for standalone) ────────────
 # Power users running the COS Starter Kit can point the glasses at their
 # pipeline to inherit live tasks/calendar/people context. Omit for standalone.

package/CHANGELOG.md

@@ -1,3 +1,30 @@
+## 6.15.0
+
+- Add local-first spoken reply playback through a Mac-owned Kokoro sidecar on
+  Apple silicon. The existing OpenAI TTS path remains available as an explicit
+  engine choice and as the bounded fallback for `local_first` only when a key
+  and budget are available.
+- Preserve cancellation across the local synthesis boundary so abandoned
+  requests return 499 and can never become accidental cloud fallbacks. An
+  explicitly selected local engine fails closed when Kokoro is unavailable.
+- Keep cache entries and resumable sessions engine-aware, expose additive
+  `tts_local` and voice-engine health, and announce a real Kokoro-to-OpenAI
+  fallback only after cloud audio succeeds.
+- Bootstrap the pinned private Python environment asynchronously on first run,
+  avoiding an API event-loop stall while dependencies install. The local
+  sidecar is limited to Apple silicon macOS; other hosts retain their existing
+  OpenAI behavior.
+- Add opt-in, public-safe pronunciation overrides through
+  `COS_TTS_PRONUNCIATIONS_JSON`. The published package contains no personal
+  pronunciation dictionary or machine-specific path.
+
+## 6.14.2
+
+- Restore timed words on post-meeting CPU polish only: whisper-cli `-ojf`
+  token offsets feed speaker-word mapping after save. Live ASR stays on
+  compact `json` (no `verbose_json`); VAD-empty CLI windows return empty
+  transcription safely without reintroducing the live daemon crash.
+
 ## 6.14.1
 
 - Keep real-time `large-v3-turbo` stable on VAD-empty audio by using

package/README.md

@@ -11,8 +11,8 @@ API key is pasted into the phone for chat.
 npx --yes @gotcos/glasses-server@latest
 ```
 
-For the optional COS Control macOS menu bar app, prepare dependencies without
-leaving a foreground server running:
+For the optional COS Control macOS menu bar app, run the same non-mutating
+readiness check it uses before guided installation:
 
 ```bash
 npx --yes @gotcos/glasses-server@latest --prepare-only
@@ -21,9 +21,13 @@ npx --yes @gotcos/glasses-server@latest --prepare-only
 COS Control then installs the same npm package as a launchd-managed runtime.
 The original foreground command remains supported and unchanged.
 
-The launcher checks Node, finds your CLI, checks voice and image processing,
-downloads the local voice model when needed, writes `~/.cos-glasses/.env`, and
-starts the server on `0.0.0.0:3141`. On boot it prints
+Normal server start checks Node, finds your CLI, checks voice and image
+processing, writes `~/.cos-glasses/.env`, and starts the server on
+`0.0.0.0:3141`. Optional Whisper and Kokoro models are provisioned when their
+local services start; `--prepare-only` intentionally does not download or
+install optional models, write COS configuration, or start a listener. It may
+invoke an installed agent CLI's read-only version/auth probe, and that CLI may
+maintain its own user cache. On boot the server prints
 an **API token** — paste that into the COS Glasses app. Only one COS Glasses
 server may run on a Mac at a time; a second `npx` or source runner exits before
 opening ports or touching shared conversation/media state. Version 6.6.0 also
@@ -41,7 +45,10 @@ without silently losing completed replies.
   _or_ **Codex CLI** (GPT Frontier/Balanced) — https://developers.openai.com/codex/, then `codex login`
 - **Even G2 glasses** + the **COS Glasses** app from the Even Hub
 - `brew install whisper-cpp` for free local voice (the launcher can download the model)
-- _Optional:_ `brew install ffmpeg` for phone/output image attachments (text chat remains available without it)
+- _Optional:_ `brew install python@3.13 ffmpeg espeak-ng` for local Kokoro
+  spoken replies on Apple silicon (Python 3.11-3.13 is supported). `ffmpeg`
+  also enables phone/output image attachments; text chat remains available
+  without these optional dependencies.
 - _Optional:_ **Tailscale** so your phone reaches your Mac from anywhere
 
 > No `ANTHROPIC_API_KEY` is needed — chat runs through your installed CLI, billed
@@ -99,6 +106,11 @@ range is the exact Tailscale/CGNAT allocation (`100.64.0.0/10`), not all of
   Whisper fallback is optional and requires both the exact
   `COS_OPENAI_WHISPER_FALLBACK=1` opt-in and a configured key; a key alone never
   uploads audio.
+- Local-first spoken reply playback through Kokoro on Apple silicon. The first
+  use creates a private Python environment and downloads its model without
+  blocking the API. Selecting Local fails closed; `local_first` can fall back
+  to OpenAI TTS only when a key and budget are available. `/api/health`
+  reports the independent `tts_local` state and current engine.
 - Tasks / calendar / people context **if** you run the
   [COS Starter Kit](https://www.gotcos.com) (`COS_SCRIPTS_DIR`); otherwise it is
   glasses + AI only
@@ -108,7 +120,11 @@ range is the exact Tailscale/CGNAT allocation (`100.64.0.0/10`), not all of
 Config lives at `~/.cos-glasses/.env` (created on first run). Every key is
 optional except an installed CLI. Highlights: `BIND_HOST`, `PORT`,
 `COS_API_TOKEN` (auto if unset), `COS_OPENAI_WHISPER_FALLBACK=1` plus
-`OPENAI_API_KEY` (explicit cloud voice fallback),
+`OPENAI_API_KEY` (explicit cloud transcription/TTS fallback),
+`COS_TTS_ENGINE` (`local_first` or `openai_primary`),
+`COS_TTS_KOKORO_VOICE` (local voice id),
+`COS_TTS_LOCAL_DISABLE=1` (disable the sidecar), and
+`COS_TTS_PRONUNCIATIONS_JSON` (optional local/cloud pronunciation overrides),
 `COS_SCRIPTS_DIR` (full pipeline), `COS_DURABLE_QUERY_JOBS=1` (build 204+
 server-owned query recovery), and `COS_MEDIA_ROOT` (optional image-store
 location; default `~/.cos-glasses/data/media`). Your name + transcription vocabulary live in
@@ -149,6 +165,12 @@ BIND_HOST=0.0.0.0 npm run start:server
   confirm `/api/health` reports `features.whisper: true`. A typed retryable 503
   keeps compatible prompt/meeting audio available for retry instead of silently
   sending it to OpenAI.
+- *Local spoken replies unavailable?* — on Apple silicon, install
+  `python@3.13 ffmpeg espeak-ng`, restart the server, and wait for the
+  first-run Kokoro model download. Confirm `/api/health` reports
+  `tts_local.ready: true`.
+  Selecting Local never falls back to cloud; set `COS_TTS_ENGINE=openai_primary`
+  only when OpenAI playback is intentionally configured.
 - *Photos unavailable?* — install `ffmpeg`, restart the server, and confirm `/api/health` reports `features.mediaProcessingReady: true`.
 - *Prompt recovery unavailable?* — update with `npx --yes @gotcos/glasses-server@latest`, then confirm `/api/health` reports `features.promptRecovery: true`.
 - *Durable query recovery unavailable?* — build 204+ requires server 6.10.0+ and

package/bin/cli.cjs

@@ -47,6 +47,7 @@ if (process.argv.includes('--help') || process.argv.includes('-h')) {
   console.log('    - Node.js 20.11+')
   console.log('    - Claude Code CLI (not Claude Desktop) or Codex CLI')
   console.log('    - Even G2 smart glasses + the COS Glasses app (Even Hub)')
+  console.log('    - Optional local Kokoro voice: Apple silicon + Python 3.11-3.13')
   console.log('')
   console.log('  No API key is needed for chat — it runs through your installed CLI.')
   console.log('  Config persists at ~/.cos-glasses/.env')
@@ -175,13 +176,25 @@ try {
   process.exit(1)
 }
 
-// Controller probes are deliberately read-only. They verify Node, agent auth,
-// and the packaged runtime without creating config, downloading models,
-// changing permissions, or starting a listener.
+// Controller probes do not mutate COS state. They verify Node, agent auth,
+// and the packaged runtime without creating COS config, downloading Kokoro
+// models, installing Kokoro packages, changing COS permissions, or starting a
+// listener. An invoked agent CLI may still maintain its own user cache.
 if (process.argv.includes('--prepare-only')) {
+  if (process.platform === 'darwin' && process.arch === 'arm64') {
+    const python = ['python3.13', 'python3.12', 'python3.11']
+      .map((command) => ({ command, version: getCliVersion(command, '--version') }))
+      .find((candidate) => candidate.version)
+    if (python) {
+      console.log(green('  ✓') + ` Local voice Python ${python.version.replace(/^Python\s+/i, '')}`)
+    } else {
+      console.log(yellow('  ⚠') + ' Local Kokoro voice needs Python 3.11-3.13')
+      console.log('    Install: ' + bold('brew install python@3.13 ffmpeg espeak-ng'))
+    }
+  }
   console.log('')
   console.log(green('  ✓ Non-mutating readiness check complete'))
-  console.log('    COS Control can perform guided installation without hidden setup side effects.')
+  console.log('    This checks prerequisites only; normal server start provisions optional models.')
   console.log('')
   process.exit(0)
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.14.1",
+  "version": "6.15.0",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by your local Claude Code or Codex CLI",
   "type": "module",
   "bin": {
@@ -10,7 +10,7 @@
   "scripts": {
     "start": "node bin/cli.cjs",
     "start:server": "node --import tsx/esm server/index.ts",
-    "test": "vitest run",
+    "test": "vitest run --maxWorkers=1",
     "typecheck": "tsc --noEmit"
   },
   "keywords": [

package/server/index.ts

@@ -44,6 +44,7 @@ import {
   stopCodexModelCatalogRefresh,
 } from './lib/codex-model-catalog.js'
 import { startWhisperServer, stopWhisperServer } from './lib/whisper-local.js'
+import { startLocalTtsServer, stopLocalTtsServer } from './lib/tts-local.js'
 import { initSileroVAD } from './lib/vad-silero.js'
 import { initSessionCache } from './lib/session-cache-writer.js'
 import { initSpeakerEmbeddings } from './lib/speaker-embeddings.js'
@@ -266,6 +267,7 @@ async function gracefulShutdown(): Promise<void> {
   try { logActiveSessionsOnShutdown() } catch { /* best-effort flush */ }
   stopCodexModelCatalogRefresh()
   stopWhisperServer()
+  stopLocalTtsServer()
   clearTimeout(forceExit)
   process.exit(0)
 }
@@ -385,6 +387,7 @@ listenRequiredServers(listeners).then(() => {
     }
     // Start local whisper-server (model stays in RAM for ~50ms transcription)
     startWhisperServer().catch(err => console.error('[startup] Whisper server error:', err))
+    startLocalTtsServer().catch(err => console.error('[startup] Local TTS server error:', err))
     // Initialize speaker embeddings (voiceprint-based diarization) — fails soft if model absent
     const embeddingOk = initSpeakerEmbeddings()
     console.log(`[startup] Speaker embeddings: ${embeddingOk ? 'active' : 'disabled (model not found)'}`)

package/server/lib/tts-cache.ts

@@ -67,6 +67,10 @@ interface SessionEntry {
   text: string
   voice: string
   format: string
+  /** One-shot OpenAI opt-in for this session (play cold-miss must honor it). */
+  preferOpenAI?: boolean
+  /** Settings forced Local/Kokoro — do not auto-escape to OpenAI on play miss. */
+  forceLocal?: boolean
   expiresAt: number
 }
 
@@ -114,8 +118,18 @@ let totalDiskBytes = 0  // on-disk bytes (sum of all sidecar sizeBytes)
  *  a different voice for the same text correctly misses (and gets its own
  *  entry). Hash is sha256 truncated to 16 hex chars — collision probability is
  *  ~negligible for our scale. */
-export function hashKey(text: string, voice: string, format: string): string {
-  return createHash('sha256').update(`${voice}\0${format}\0${text}`).digest('hex').slice(0, 16)
+export function hashKey(
+  engine: string,
+  mappedVoice: string,
+  format: string,
+  text: string,
+  instructions?: string,
+): string {
+  let material = `${engine}\0${mappedVoice}\0${format}\0${text}`
+  if (engine === 'openai' && instructions && instructions.trim().length > 0) {
+    material += `\0${instructions.trim()}`
+  }
+  return createHash('sha256').update(material).digest('hex').slice(0, 16)
 }
 
 function ensureDiskDir(): void {
@@ -512,6 +526,18 @@ export function peekSession(uuid: string): SessionEntry | null {
   return s
 }
 
+/** Rebind a live session to a new cache hash (openai -> local fallback). */
+export function rebindSessionHash(uuid: string, hash: string): boolean {
+  const s = sessions.get(uuid)
+  if (!s) return false
+  if (s.expiresAt < Date.now()) {
+    sessions.delete(uuid)
+    return false
+  }
+  s.hash = hash
+  return true
+}
+
 /** Strict one-shot lookup — kept for callers that want to invalidate the
  *  session immediately on first read. Not used by /play in v5.9.4+. */
 export function consumeSession(uuid: string): SessionEntry | null {

package/server/lib/tts-engine.ts

@@ -0,0 +1,230 @@
+// Local TTS engine mode + OpenAI voice → Kokoro preset map.
+// Default product mode: local_first (Kokoro). OpenAI is opt-in escape hatch.
+// Modes: openai | openai_primary | local_first | local
+// Bare "auto" → local_first.
+// Per-request engine: 'local' | 'openai' forces that backend (Settings picker).
+
+export type TtsEngineMode = 'openai' | 'openai_primary' | 'local_first' | 'local'
+export type TtsBackend = 'openai' | 'local'
+export type TtsEnginePreference = 'local' | 'openai'
+
+/** Cache/engine tag written into hashKey — distinguishes OpenAI audio from Kokoro. */
+export type TtsEngineTag = 'openai' | 'kokoro'
+
+export const OPENAI_VOICE_OPTIONS = [
+  { id: 'echo', label: 'Echo (default)' },
+  { id: 'alloy', label: 'Alloy' },
+  { id: 'nova', label: 'Nova' },
+  { id: 'sage', label: 'Sage' },
+  { id: 'onyx', label: 'Onyx' },
+  { id: 'shimmer', label: 'Shimmer' },
+  { id: 'fable', label: 'Fable' },
+  { id: 'ash', label: 'Ash' },
+  { id: 'coral', label: 'Coral' },
+] as const
+
+/** American English Kokoro presets verified on disk (mlx-community/Kokoro-82M-bf16). */
+export const KOKORO_VOICE_OPTIONS = [
+  { id: 'am_echo', label: 'Echo (am_echo) · default' },
+  { id: 'am_michael', label: 'Michael (am_michael)' },
+  { id: 'am_fenrir', label: 'Fenrir (am_fenrir)' },
+  { id: 'am_puck', label: 'Puck (am_puck)' },
+  { id: 'am_onyx', label: 'Onyx (am_onyx)' },
+  { id: 'am_eric', label: 'Eric (am_eric)' },
+  { id: 'am_liam', label: 'Liam (am_liam)' },
+  { id: 'am_adam', label: 'Adam (am_adam)' },
+  { id: 'am_santa', label: 'Santa (am_santa)' },
+  { id: 'af_heart', label: 'Heart (af_heart)' },
+  { id: 'af_bella', label: 'Bella (af_bella)' },
+  { id: 'af_nova', label: 'Nova (af_nova)' },
+  { id: 'af_sarah', label: 'Sarah (af_sarah)' },
+  { id: 'af_sky', label: 'Sky (af_sky)' },
+  { id: 'af_nicole', label: 'Nicole (af_nicole)' },
+  { id: 'af_jessica', label: 'Jessica (af_jessica)' },
+  { id: 'af_alloy', label: 'Alloy (af_alloy)' },
+  { id: 'af_aoede', label: 'Aoede (af_aoede)' },
+  { id: 'af_kore', label: 'Kore (af_kore)' },
+  { id: 'af_river', label: 'River (af_river)' },
+] as const
+
+export const KOKORO_EN_US_VOICES = KOKORO_VOICE_OPTIONS.map((v) => v.id)
+
+// OpenAI id → Kokoro when Settings still sends an OpenAI id on the local path.
+const VOICE_MAP: Record<string, string> = {
+  echo: 'am_echo',
+  alloy: 'af_heart',
+  fable: 'af_bella',
+  onyx: 'am_onyx',
+  nova: 'af_nova',
+  shimmer: 'af_sky',
+  ash: 'am_fenrir',
+  sage: 'af_sarah',
+  coral: 'af_jessica',
+}
+
+const OPENAI_VOICE_IDS = new Set(OPENAI_VOICE_OPTIONS.map((v) => v.id))
+const KOKORO_VOICE_IDS = new Set(KOKORO_EN_US_VOICES)
+
+export function isOpenAIVoiceId(voice: string): boolean {
+  return OPENAI_VOICE_IDS.has(voice as typeof OPENAI_VOICE_OPTIONS[number]['id'])
+}
+
+/** Kokoro / Misaki voice file ids look like am_echo, af_heart, bm_george. */
+export function isKokoroVoiceId(voice: string): boolean {
+  return /^[a-z]{2}_[a-z0-9]+$/i.test(voice)
+}
+
+export function getTtsEngineMode(): TtsEngineMode {
+  const raw = (process.env.COS_TTS_ENGINE || 'local_first').trim().toLowerCase()
+  if (raw === 'auto') return 'local_first'
+  if (raw === 'openai' || raw === 'openai_primary' || raw === 'local_first' || raw === 'local') {
+    return raw
+  }
+  console.warn(`[tts-engine] unknown COS_TTS_ENGINE=${raw}; using local_first`)
+  return 'local_first'
+}
+
+/** Resolve which Kokoro preset to synthesize for a local request. */
+export function resolveLocalVoice(requested: string): string {
+  const v = (requested || '').trim()
+  if (v && isKokoroVoiceId(v)) {
+    const lower = v.toLowerCase()
+    // Accept any on-disk-shaped id; known list is the Settings catalog.
+    return lower
+  }
+  // Env pin only when mapping legacy OpenAI ids (not when client sent Kokoro).
+  const pinned = (process.env.COS_TTS_KOKORO_VOICE || '').trim()
+  if (pinned && isKokoroVoiceId(pinned)) return pinned.toLowerCase()
+  return VOICE_MAP[v] ?? VOICE_MAP.echo
+}
+
+/** @deprecated use resolveLocalVoice — kept for tests / call sites. */
+export function mapOpenAIVoiceToLocal(openaiVoice: string): string {
+  return resolveLocalVoice(openaiVoice)
+}
+
+export function localEngineTag(): TtsEngineTag {
+  const pinned = (process.env.COS_TTS_LOCAL_ENGINE || 'kokoro').trim().toLowerCase()
+  return pinned === 'kokoro' ? 'kokoro' : 'kokoro'
+}
+
+export interface TtsRouteDecision {
+  backend: TtsBackend
+  engineTag: TtsEngineTag
+  /** Voice id sent to the chosen backend. */
+  backendVoice: string
+  /** Voice id from the client (OpenAI or Kokoro). */
+  openaiVoice: string
+}
+
+/**
+ * Pick initial backend for a request.
+ *
+ * enginePreference:
+ *   - 'openai' → cloud (Settings “OpenAI”)
+ *   - 'local' → Kokoro (Settings “Local”)
+ *   - omitted → daemon COS_TTS_ENGINE (local_first default)
+ */
+export function decideInitialBackend(opts: {
+  openaiVoice: string
+  openaiKeyPresent: boolean
+  openaiBudgetOk: boolean
+  localReady: boolean
+  preferOpenAI?: boolean
+  enginePreference?: TtsEnginePreference | null
+}): TtsRouteDecision {
+  const requested = opts.openaiVoice
+  const localVoice = resolveLocalVoice(requested)
+  const openaiVoice = isOpenAIVoiceId(requested) ? requested : 'echo'
+  const local: TtsRouteDecision = {
+    backend: 'local',
+    engineTag: localEngineTag(),
+    backendVoice: localVoice,
+    openaiVoice: requested,
+  }
+  const openai: TtsRouteDecision = {
+    backend: 'openai',
+    engineTag: 'openai',
+    backendVoice: openaiVoice,
+    openaiVoice: requested,
+  }
+
+  const preferOpenAI =
+    opts.preferOpenAI === true || opts.enginePreference === 'openai'
+  const forceLocal = opts.enginePreference === 'local'
+
+  if (forceLocal) {
+    if (!opts.localReady) {
+      throw new Error('Local TTS selected but Kokoro sidecar is not ready')
+    }
+    return local
+  }
+
+  if (preferOpenAI) {
+    if (opts.openaiKeyPresent && opts.openaiBudgetOk) return openai
+    if (opts.localReady) {
+      console.warn('[tts-engine] OpenAI requested but unavailable; using local')
+      return local
+    }
+    throw new Error('OpenAI TTS requested but key/budget unavailable and local TTS not ready')
+  }
+
+  // Legacy: bare Kokoro voice id implies local even without enginePreference.
+  if (isKokoroVoiceId(requested) && opts.localReady) return local
+
+  const mode = getTtsEngineMode()
+
+  if (mode === 'openai') {
+    if (!opts.openaiKeyPresent) {
+      throw new Error('COS_TTS_ENGINE=openai but OPENAI_API_KEY is missing')
+    }
+    if (!opts.openaiBudgetOk) {
+      throw new Error('COS_TTS_ENGINE=openai but daily OpenAI TTS budget is exhausted')
+    }
+    return openai
+  }
+
+  if (mode === 'local') {
+    if (!opts.localReady) {
+      throw new Error('COS_TTS_ENGINE=local but local TTS sidecar is not ready')
+    }
+    return local
+  }
+
+  if (mode === 'local_first') {
+    if (opts.localReady) return local
+    if (opts.openaiKeyPresent && opts.openaiBudgetOk) return openai
+    throw new Error('local TTS unavailable and OpenAI key/budget not usable')
+  }
+
+  // openai_primary
+  if (opts.openaiKeyPresent && opts.openaiBudgetOk) return openai
+  if (opts.localReady) return local
+  if (!opts.openaiKeyPresent) {
+    throw new Error('OpenAI TTS key missing and local TTS sidecar is not ready')
+  }
+  throw new Error('OpenAI TTS budget exhausted and local TTS sidecar is not ready')
+}
+
+/** After an OpenAI failure, can we fall back to local under this mode? */
+export function canFallbackToLocal(
+  mode: TtsEngineMode,
+  localReady: boolean,
+  preferOpenAI = false,
+): boolean {
+  if (!localReady) return false
+  if (preferOpenAI) return true
+  return mode === 'openai_primary' || mode === 'local_first'
+}
+
+/** After a local failure, can we fall back to OpenAI under this mode? */
+export function canFallbackToOpenAI(
+  mode: TtsEngineMode,
+  openaiKeyPresent: boolean,
+  openaiBudgetOk: boolean,
+  forceLocal = false,
+): boolean {
+  if (forceLocal) return false
+  return openaiKeyPresent && openaiBudgetOk && mode === 'local_first'
+}
+