@gotcos/glasses-server 6.15.5 → 6.16.0

package/.env.example

@@ -70,11 +70,15 @@ BIND_HOST=0.0.0.0
 
 # ── VOICE (optional) ────────────────────────────────────────────────────
 # Local transcription is FREE via whisper.cpp (brew install whisper-cpp; the
-# model auto-downloads on first run). Voice is local-only by default. Merely
+# real-time turbo model auto-downloads on first run). Full HQ dictation also
+# needs ggml-large-v3.bin as documented in README. Voice is local-only by default. Merely
 # configuring a key never uploads audio. To allow OpenAI Whisper only after a
 # local failure, set BOTH the exact opt-in and a key:
 # COS_OPENAI_WHISPER_FALLBACK=1
 # OPENAI_API_KEY=sk-...
+# COS_HQ_SPECULATIVE_WARM=0        # disable background HQ warm
+# COS_BATCH_LARGE_V3=0             # explicitly use turbo instead of full HQ
+# COS_HQ_BEAM_INTERACTIVE=2        # interactive only; meetings stay at beam 5
 
 # Spoken reply playback defaults to local Kokoro on Apple silicon Macs. The
 # first run creates a private venv and downloads the model. Local mode fails

package/CHANGELOG.md

@@ -1,3 +1,21 @@
+## 6.16.0
+
+- **Truthful HQ results.** An HQ request is reported as HQ only when the full
+  local large-v3 decoder actually ran. Turbo, real-time server, long-audio, and
+  decode-error fallbacks now retain the requested mode while returning their
+  actual quality, backend, degradation flag, and bounded reason code.
+- **HQ capability health.** `/api/health` and `/api/models` add a path-free
+  `capabilities.transcription.hq` block with availability, model, backend, and
+  a user-safe missing-prerequisite reason. Generic Whisper liveness no longer
+  implies that large-v3 HQ is installed.
+- **Phone-visible fallback telemetry.** One-shot transcription and prompt-draft
+  finalize responses expose the same additive quality fields. Draft finalize
+  aggregates the records it actually used and reuses a successful degraded
+  warm result instead of paying for an identical second turbo decode.
+- **Default unchanged.** Absent an explicit Fast request, prompt dictation still
+  requests HQ. `COS_HQ_SPECULATIVE_WARM=0` remains the immediate warm-path
+  rollback, and meeting batch beam/isolation behavior is unchanged.
+
 ## 6.15.5
 
 - **Speculative HQ warm (no EHPK).** While a prompt-draft chunk is acknowledged,

package/README.md

@@ -106,6 +106,10 @@ range is the exact Tailscale/CGNAT allocation (`100.64.0.0/10`), not all of
   Whisper fallback is optional and requires both the exact
   `COS_OPENAI_WHISPER_FALLBACK=1` opt-in and a configured key; a key alone never
   uploads audio.
+- HQ prompt dictation is requested by default; the phone's **Fast mode** switch
+  opts into turbo. Server 6.16.0 reports whether full local large-v3 actually
+  ran, and compatible companions alert once if an HQ request used Fast or
+  Cloud instead of silently claiming HQ.
 - Local-first spoken reply playback through Kokoro on Apple silicon. The first
   use creates a private Python environment and downloads its model without
   blocking the API. Selecting Local fails closed; `local_first` can fall back
@@ -137,6 +141,32 @@ Telegram activity export is disabled by default even when a private COS
 pipeline contains `.telegram_config.json`; enable it only with the explicit
 `COS_TELEGRAM_NOTIFICATIONS=1` opt-in.
 
+## HQ dictation
+
+Prompt dictation defaults to HQ. The phone owns the preference: **Fast mode
+OFF** requests HQ, and **Fast mode ON** requests turbo. The Mac performs all
+decoding; the phone does not run Whisper.
+
+The first server start downloads the real-time turbo model. True HQ additionally
+requires the full `ggml-large-v3.bin` model (about 3.1 GB):
+
+```bash
+mkdir -p "$HOME/.local/share/whisper-models"
+curl -fL --progress-bar \
+  https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin \
+  -o "$HOME/.local/share/whisper-models/ggml-large-v3.bin.partial"
+mv "$HOME/.local/share/whisper-models/ggml-large-v3.bin.partial" \
+  "$HOME/.local/share/whisper-models/ggml-large-v3.bin"
+```
+
+Restart the server, then confirm
+`capabilities.transcription.hq.hqAvailable: true` at `/api/health`. The response
+does not expose local paths. If the CLI or model is unavailable, dictation stays
+usable on Fast and reports the downgrade truthfully. Set
+`COS_HQ_SPECULATIVE_WARM=0` to disable background HQ warm immediately; set
+`COS_BATCH_LARGE_V3=0` to explicitly use turbo. Interactive HQ uses beam 2 by
+default (`COS_HQ_BEAM_INTERACTIVE`); meeting batch remains beam 5.
+
 ## Run from source
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.15.5",
+  "version": "6.16.0",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by your local Claude Code or Codex CLI",
   "type": "module",
   "bin": {

package/server/lib/prompt-draft-store.ts

@@ -20,6 +20,7 @@ export interface PromptDraftTranscriptRecord {
   actualQuality: 'hq' | 'fast' | 'cloud'
   backend: string
   degraded: boolean
+  degradationReason?: string
   acceptedDegraded?: boolean
 }
 

package/server/lib/transcribe-audio.ts

@@ -34,6 +34,7 @@ export interface TranscribeAudioResult {
   requestedMode: TranscribeMode
   actualQuality: 'hq' | 'fast' | 'cloud'
   degraded: boolean
+  degradationReason?: string
   elapsedMs: number
   audioBytes: number
 }
@@ -160,6 +161,7 @@ export async function transcribeAudioBuffer(
   let text: string
   let backend: string
   let actualQuality: 'hq' | 'fast' | 'cloud'
+  let degradationReason: string | undefined = effectiveMode !== requestedMode ? 'audio_too_long' : undefined
   const tStart = performance.now()
 
   if (effectiveMode === 'hq') {
@@ -169,10 +171,16 @@ export async function transcribeAudioBuffer(
       const enhanced = await enhanceAudio(audioBuffer, { profile: enhanceProfile })
       const result = await transcribeHighQuality(enhanced)
       text = result.text
-      backend = enhanceProfile === 'light' ? 'hq-large-v3-light' : 'hq-large-v3'
-      actualQuality = 'hq'
+      actualQuality = result.actualQuality
+      if (result.actualQuality === 'hq') {
+        backend = enhanceProfile === 'light' ? 'hq-large-v3-light' : 'hq-large-v3'
+      } else {
+        backend = result.backend === 'whisper-cli' ? 'fast-cli-turbo' : 'fast-local-server'
+        degradationReason = result.degradationReason ?? 'hq_unavailable'
+      }
     } catch (hqErr: any) {
       console.warn(`[transcribe] HQ path failed, falling back to fast: ${hqErr.message}`)
+      degradationReason = 'hq_decode_failed'
       try {
         const result = await transcribeLocal(audioBuffer)
         text = result.text
@@ -268,6 +276,7 @@ export async function transcribeAudioBuffer(
     requestedMode,
     actualQuality,
     degraded: requestedMode === 'hq' && actualQuality !== 'hq',
+    ...(degradationReason ? { degradationReason } : {}),
     elapsedMs,
     audioBytes: audioBuffer.length,
   }

package/server/lib/whisper-local.ts

@@ -42,6 +42,49 @@ export interface WhisperSegment {
   words?: WhisperWord[]
 }
 
+export type HighQualityUnavailableReason =
+  | 'hq_disabled'
+  | 'whisper_cli_missing'
+  | 'turbo_model_missing'
+  | 'large_v3_model_missing'
+
+export interface HighQualityTranscriptionCapability {
+  hqAvailable: boolean
+  model: 'large-v3' | null
+  backend: 'whisper-cli' | null
+  reason: HighQualityUnavailableReason | null
+}
+
+export interface HighQualityTranscriptionResult {
+  text: string
+  words?: WhisperWord[]
+  model: 'large-v3' | 'turbo'
+  backend: 'whisper-cli' | 'whisper-server'
+  actualQuality: 'hq' | 'fast'
+  degradationReason?: HighQualityUnavailableReason
+}
+
+export function classifyHighQualityTranscriptionCapability(input: {
+  enabled: boolean
+  cliPresent: boolean
+  turboReady: boolean
+  largeV3Present: boolean
+}): HighQualityTranscriptionCapability {
+  if (!input.enabled) {
+    return { hqAvailable: false, model: null, backend: null, reason: 'hq_disabled' }
+  }
+  if (!input.cliPresent) {
+    return { hqAvailable: false, model: null, backend: null, reason: 'whisper_cli_missing' }
+  }
+  if (!input.turboReady) {
+    return { hqAvailable: false, model: null, backend: null, reason: 'turbo_model_missing' }
+  }
+  if (!input.largeV3Present) {
+    return { hqAvailable: false, model: null, backend: null, reason: 'large_v3_model_missing' }
+  }
+  return { hqAvailable: true, model: 'large-v3', backend: 'whisper-cli', reason: null }
+}
+
 interface WhisperJsonResponse {
   text?: unknown
 }
@@ -554,6 +597,20 @@ export function getWhisperHealth(): {
   }
 }
 
+/** Public, path-free truth about whether an HQ request can actually run the
+ * full large-v3 decoder. Keep this separate from generic Whisper liveness: the
+ * persistent turbo server may be healthy while HQ weights or the CLI are not. */
+export function getHighQualityTranscriptionCapability(): HighQualityTranscriptionCapability {
+  // cliAvailable also proves the turbo model exists. That fallback is part of
+  // the current decoder contract and is initialized once at process startup.
+  return classifyHighQualityTranscriptionCapability({
+    enabled: BATCH_LARGE_V3_ENABLED,
+    cliPresent: existsSync(WHISPER_CLI),
+    turboReady: cliAvailable,
+    largeV3Present: existsSync(BATCH_MODEL_LARGE_V3),
+  })
+}
+
 /**
  * Reconcile a cached unavailable flag with the daemon's live health endpoint.
  * Only successful inference resets the failure count: /health can be responsive
@@ -641,10 +698,18 @@ export async function transcribeHighQuality(
   audioBuffer: Buffer,
   context?: string,
   opts: { priority?: 'interactive' | 'batch' } = {},
-): Promise<{ text: string; words?: WhisperWord[] }> {
+): Promise<HighQualityTranscriptionResult> {
   if (!cliAvailable) {
     // Fall back to server (no beam search available via HTTP API)
-    return transcribeLocal(audioBuffer, context)
+    const fallback = await transcribeLocal(audioBuffer, context)
+    return {
+      text: fallback.text,
+      words: fallback.words,
+      model: 'turbo',
+      backend: fallback.backend === 'server' ? 'whisper-server' : 'whisper-cli',
+      actualQuality: 'fast',
+      degradationReason: existsSync(WHISPER_CLI) ? 'turbo_model_missing' : 'whisper_cli_missing',
+    }
   }
 
   const start = Date.now()
@@ -769,7 +834,17 @@ export async function transcribeHighQuality(
       `${words ? `, ${words.length} words` : ''}): ` +
       `"${corrected.slice(0, 80)}${corrected.length > 80 ? '...' : ''}"`,
     )
-    return words ? { text: corrected, words } : { text: corrected }
+    const metadata = useLargeV3
+      ? { model: 'large-v3' as const, backend: 'whisper-cli' as const, actualQuality: 'hq' as const }
+      : {
+          model: 'turbo' as const,
+          backend: 'whisper-cli' as const,
+          actualQuality: 'fast' as const,
+          degradationReason: BATCH_LARGE_V3_ENABLED
+            ? 'large_v3_model_missing' as const
+            : 'hq_disabled' as const,
+        }
+    return words ? { text: corrected, words, ...metadata } : { text: corrected, ...metadata }
   } finally {
     try { unlinkSync(tmpWav) } catch { /* cleanup */ }
     if (captureBatchWords) {

package/server/routes/health.ts

@@ -8,7 +8,11 @@ import { getServerInstanceId } from '../lib/server-instance-id.js'
 import { localFirstMeetingsCapability } from '../lib/local-first-meetings-contract.js'
 import { isSileroAvailable } from '../lib/vad-silero.js'
 import { getAvailableCliSessionId } from '../lib/claude-bridge.js'
-import { isWhisperLocalAvailable, getWhisperHealth } from '../lib/whisper-local.js'
+import {
+  isWhisperLocalAvailable,
+  getWhisperHealth,
+  getHighQualityTranscriptionCapability,
+} from '../lib/whisper-local.js'
 import { getOpenAIWhisperBudgetState } from '../lib/openai-whisper-budget.js'
 import { getKeyStatus } from '../lib/openai-key.js'
 import {
@@ -150,6 +154,7 @@ healthRouter.get('/health', async (_req, res) => {
   const durableJobs = durableQueryJobStatus()
   const localFirstMeetings = localFirstMeetingsCapability(getServerInstanceId())
   const transcription = getTranscriptionPolicySnapshot()
+  const transcriptionHq = getHighQualityTranscriptionCapability()
   const recovery = managedRuntimeCapability()
   const maintenance = maintenanceLifecycle.snapshot()
   const tts_local = getLocalTtsHealth()
@@ -213,7 +218,7 @@ healthRouter.get('/health', async (_req, res) => {
     tts_local,
     codex_models,
     capabilities: {
-      transcription,
+      transcription: { ...transcription, hq: transcriptionHq },
       recovery,
       maintenance: {
         state: maintenance.state,
@@ -242,6 +247,7 @@ healthRouter.get('/models', async (req, res) => {
   const durableJobs = durableQueryJobStatus()
   const localFirstMeetings = localFirstMeetingsCapability(getServerInstanceId())
   const transcription = getTranscriptionPolicySnapshot()
+  const transcriptionHq = getHighQualityTranscriptionCapability()
   res.json({
     ...catalog,
     serverInstanceId: getServerInstanceId(),
@@ -250,7 +256,7 @@ healthRouter.get('/models', async (req, res) => {
         enabled: durableJobs.enabled,
         protocolVersion: durableJobs.protocolVersion,
       },
-      transcription,
+      transcription: { ...transcription, hq: transcriptionHq },
       cliDebug: CLI_DEBUG_CAPABILITY,
       recovery: managedRuntimeCapability(),
       ...(localFirstMeetings ? { localFirstMeetings } : {}),

package/server/routes/prompt-drafts.ts

@@ -199,6 +199,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
             actualQuality: warm?.actualQuality ?? (mode === 'hq' ? 'hq' : 'fast'),
             backend: warm?.backend ?? 'shared-inflight',
             degraded: warm?.degraded ?? false,
+            ...(warm?.degradationReason ? { degradationReason: warm.degradationReason } : {}),
           }, 'final')
         }
       }
@@ -221,6 +222,7 @@ async function transcribeChunk(draftId: string, chunkIndex: number, audio: Buffe
       const record: PromptDraftTranscriptRecord = {
         text, hash, requestedMode: result.requestedMode, actualQuality: result.actualQuality,
         backend: result.backend, degraded: result.degraded,
+        ...(result.degradationReason ? { degradationReason: result.degradationReason } : {}),
       }
       await markPromptDraftChunkTranscript(draftId, chunkIndex, record, purpose)
       console.log(`[prompt-draft] chunk ${draftId}/${chunkIndex}: ${result.elapsedMs.toFixed(1)}ms | ${result.backend} | ${purpose}/${mode} | ${text.length} chars`)
@@ -247,6 +249,7 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
   const meta = loadPromptDraftMeta(draftId)
   if (!meta) throw Object.assign(new Error('draft not found'), { status: 404 })
   const texts: string[] = []
+  const transcriptRecords: PromptDraftTranscriptRecord[] = []
   for (const chunk of readPromptDraftChunks(draftId)) {
     try {
       const hash = audioHash(chunk.audioBuffer)
@@ -259,10 +262,27 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
       }
       const current = loadPromptDraftMeta(draftId)
       const cached = current?.finalTranscripts?.[String(chunk.chunkIndex)] ?? current?.warmTranscripts?.[String(chunk.chunkIndex)]
-      const reusable = Boolean(cached && cached.hash === hash && (mode === 'fast' ? cached.actualQuality === 'fast' : cached.actualQuality === 'hq'))
+      // Reuse the exact requested-mode decode even when HQ truthfully degraded
+      // to turbo. Finalize's automatic policy would make the same local choice
+      // again after a successful turbo result, so a second decode adds latency
+      // without improving quality. Legacy records stay excluded because their
+      // decoder provenance was reconstructed during migration.
+      const reusable = Boolean(
+        cached
+        && cached.hash === hash
+        && cached.requestedMode === mode
+        && !cached.backend.startsWith('legacy'),
+      )
       const raw = reusable ? cached!.text : await transcribeChunk(draftId, chunk.chunkIndex, chunk.audioBuffer, mode, 'final')
       const text = sanitizeTranscript(draftId, raw, !reusable)
-      if (text.trim()) texts.push(text.trim())
+      if (text.trim()) {
+        texts.push(text.trim())
+        const latest = loadPromptDraftMeta(draftId)
+        const used = latest?.finalTranscripts?.[String(chunk.chunkIndex)]
+          ?? latest?.warmTranscripts?.[String(chunk.chunkIndex)]
+          ?? cached
+        if (used && used.hash === hash) transcriptRecords.push(used)
+      }
     } catch (err) {
       if (err instanceof NoSpeechDetectedError) continue
       throw err
@@ -275,7 +295,28 @@ async function finalizeDraft(draftId: string, mode: 'hq' | 'fast', autoClean: Au
   }
   const finalText = await cleanOutboundDictation(text, { ...autoClean, signal })
   const finalized = await markPromptDraftFinalized(draftId, finalText)
-  return { draftId, text: finalText, recovered: true, chunkCount: finalized.receivedChunkIndexes.length, missingChunks: getMissingChunkIndexes(finalized), expiresAt: finalized.expiresAt }
+  const qualities = transcriptRecords.map(record => record.actualQuality)
+  const actualQuality: 'hq' | 'fast' | 'cloud' = qualities.includes('cloud')
+    ? 'cloud'
+    : qualities.length > 0 && qualities.every(quality => quality === 'hq')
+      ? 'hq'
+      : 'fast'
+  const degraded = mode === 'hq' && (actualQuality !== 'hq' || transcriptRecords.some(record => record.degraded))
+  const backends = [...new Set(transcriptRecords.map(record => record.backend))]
+  const degradationReason = transcriptRecords.find(record => record.degradationReason)?.degradationReason
+  return {
+    draftId,
+    text: finalText,
+    recovered: true,
+    chunkCount: finalized.receivedChunkIndexes.length,
+    missingChunks: getMissingChunkIndexes(finalized),
+    expiresAt: finalized.expiresAt,
+    requestedMode: mode,
+    actualQuality,
+    degraded,
+    backend: backends.length === 1 ? backends[0] : 'mixed',
+    ...(degradationReason ? { degradationReason } : {}),
+  }
 }
 
 const prunedAtBoot = maintenanceAdmissionsOpen() ? prunePromptDrafts() : 0

package/server/routes/transcribe.ts

@@ -44,7 +44,15 @@ transcribeRouter.post('/transcribe', async (req, res) => {
 
     const result = await transcribeAudioBuffer(audioBuffer, { mode: resolveMode(req) })
     console.log(`[perf] /transcribe: ${result.elapsedMs.toFixed(1)}ms | mode=${result.mode} | ${result.backend} | ${result.audioBytes}b | ${result.text.length} chars`)
-    res.json({ text: result.text, backend: result.backend, mode: result.mode })
+    res.json({
+      text: result.text,
+      backend: result.backend,
+      mode: result.mode,
+      requestedMode: result.requestedMode,
+      actualQuality: result.actualQuality,
+      degraded: result.degraded,
+      ...(result.degradationReason ? { degradationReason: result.degradationReason } : {}),
+    })
   } catch (err: any) {
     if (err instanceof MaintenanceLifecycleError) {
       if (err.retryAfterSeconds != null) res.setHeader('Retry-After', String(err.retryAfterSeconds))