@gotcos/glasses-server 6.1.0

package/server/lib/token-audit.ts

@@ -0,0 +1,50 @@
+// Token audit logger — zero-cost file I/O tracking for all claude -p calls.
+// Both Python (COS scripts) and TypeScript (G2 glasses) write to the same JSONL.
+// No LLM calls. Pure file append.
+
+import { appendFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { homedir } from 'node:os'
+
+import { COS_SCRIPTS_DIR } from './python-bridge.js'
+
+// Shared JSONL file — same path as Python's llm_client.py uses
+// Uses COS_SCRIPTS_DIR when available (server mode), falls back to user home
+const AUDIT_FILE = COS_SCRIPTS_DIR
+  ? resolve(COS_SCRIPTS_DIR, '.cos_token_audit.jsonl')
+  : resolve(homedir(), '.cos-glasses', 'token-audit.jsonl')
+
+// Rough char-to-token ratio (1 token ~ 4 chars for English)
+const CHARS_PER_TOKEN = 4
+
+function estimateTokens(chars: number): number {
+  return Math.max(1, Math.floor(chars / CHARS_PER_TOKEN))
+}
+
+export interface TokenAuditEntry {
+  source: string       // "g2-voice", "g2-prewarm", "g2-archive", "g2-query"
+  model: string        // "opus", "sonnet", "haiku"
+  inputChars: number
+  outputChars: number
+  durationMs: number
+  caller: string       // "voice_query", "prewarm", "chat_summary", "day_summary"
+}
+
+export function logTokenAudit(entry: TokenAuditEntry): void {
+  const record = {
+    ts: new Date().toISOString(),
+    source: entry.source,
+    model: entry.model,
+    input_chars: entry.inputChars,
+    output_chars: entry.outputChars,
+    est_input_tokens: estimateTokens(entry.inputChars),
+    est_output_tokens: estimateTokens(entry.outputChars),
+    duration_ms: entry.durationMs,
+    caller: entry.caller,
+  }
+  try {
+    appendFileSync(AUDIT_FILE, JSON.stringify(record) + '\n')
+  } catch {
+    // Never let logging break the actual call
+  }
+}

package/server/lib/transcribe-audio.ts

@@ -0,0 +1,187 @@
+import {
+  isWhisperLocalAvailable,
+  transcribeLocal,
+  transcribeHighQuality,
+  getWhisperBackend,
+} from './whisper-local.js'
+import { getVocabulary, getOwnerName } from './profile.js'
+import { applyFuzzyCorrections } from './fuzzy-correct.js'
+import { getAllSpeakerNames } from './speaker-embeddings.js'
+import {
+  assertOpenAIWhisperBudget,
+  recordOpenAIWhisperUsage,
+  estimateAudioSeconds,
+  OpenAIWhisperBudgetExhaustedError,
+} from './openai-whisper-budget.js'
+import { enhanceAudio } from './audio-enhance.js'
+import {
+  stripInlineHallucinationsOneShot,
+  isFullHallucination,
+} from './hallucination-filter.js'
+import { getOpenAIKey } from './openai-key.js'
+
+export { OpenAIWhisperBudgetExhaustedError, estimateAudioSeconds }
+
+export type TranscribeMode = 'hq' | 'fast'
+
+export interface TranscribeAudioResult {
+  text: string
+  backend: string
+  mode: TranscribeMode
+  elapsedMs: number
+  audioBytes: number
+}
+
+export class NoSpeechDetectedError extends Error {
+  readonly reason = 'no_speech'
+
+  constructor(readonly rawText = '') {
+    super('No speech detected')
+    this.name = 'NoSpeechDetectedError'
+  }
+}
+
+// HQ clips > this fall back to fast mode — large-v3 latency scales roughly linearly
+// with audio length and beam-search × best-of amplifies that. 60s is the user-perceived
+// ceiling (anything longer is a dictation, not a query — use meetings instead).
+const HQ_MAX_SECONDS = 60
+
+/** Transcribe via OpenAI Whisper API (cloud fallback).
+ *  Budget-gated: throws OpenAIWhisperBudgetExhaustedError if today's $5 cap is spent.
+ *  Ledger only ticks on SUCCESSFUL responses so retries that never reach the API
+ *  aren't double-counted. */
+async function transcribeCloud(audioBuffer: Buffer): Promise<string> {
+  assertOpenAIWhisperBudget()
+
+  const key = getOpenAIKey()
+  const audioSeconds = estimateAudioSeconds(audioBuffer)
+
+  // Detect format from magic bytes.
+  const isWav = audioBuffer.length >= 4 && audioBuffer.toString('ascii', 0, 4) === 'RIFF'
+  const filename = isWav ? 'recording.wav' : 'recording.webm'
+  const mimeType = isWav ? 'audio/wav' : 'audio/webm'
+
+  const boundary = '----COS' + Date.now()
+  const header = `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename}"\r\nContent-Type: ${mimeType}\r\n\r\n`
+  const modelPart = `\r\n--${boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\nwhisper-1`
+  const languagePart = `\r\n--${boundary}\r\nContent-Disposition: form-data; name="language"\r\n\r\nen`
+  const vocab = getVocabulary()
+  const cloudPrompt = vocab.length > 0
+    ? [getOwnerName(), ...vocab].join(', ')
+    : `${getOwnerName()}, COS Glasses, Even G2`
+  const promptPart = `\r\n--${boundary}\r\nContent-Disposition: form-data; name="prompt"\r\n\r\n${cloudPrompt}`
+  const footer = `\r\n--${boundary}--\r\n`
+
+  const body = Buffer.concat([
+    Buffer.from(header),
+    audioBuffer,
+    Buffer.from(modelPart),
+    Buffer.from(languagePart),
+    Buffer.from(promptPart),
+    Buffer.from(footer),
+  ])
+
+  const response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${key}`,
+      'Content-Type': `multipart/form-data; boundary=${boundary}`,
+    },
+    body,
+  })
+
+  if (!response.ok) {
+    const errText = await response.text()
+    throw new Error(`Whisper API ${response.status}: ${errText.slice(0, 200)}`)
+  }
+
+  const result = await response.json() as { text: string }
+  recordOpenAIWhisperUsage(audioSeconds)
+  return result.text
+}
+
+export function resolveTranscribeMode(raw: unknown): TranscribeMode {
+  return String(raw ?? '').toLowerCase() === 'fast' ? 'fast' : 'hq'
+}
+
+export async function transcribeAudioBuffer(audioBuffer: Buffer, opts: { mode?: TranscribeMode } = {}): Promise<TranscribeAudioResult> {
+  const requestedMode = opts.mode ?? 'hq'
+  const audioSeconds = estimateAudioSeconds(audioBuffer)
+  const effectiveMode: TranscribeMode =
+    requestedMode === 'hq' && audioSeconds > HQ_MAX_SECONDS ? 'fast' : requestedMode
+
+  if (effectiveMode !== requestedMode) {
+    console.log(`[transcribe] mode downgrade: hq → fast (audio ${audioSeconds.toFixed(1)}s > cap ${HQ_MAX_SECONDS}s)`)
+  }
+
+  let text: string
+  let backend: string
+  const tStart = performance.now()
+
+  if (effectiveMode === 'hq' && isWhisperLocalAvailable()) {
+    try {
+      const enhanced = await enhanceAudio(audioBuffer)
+      const result = await transcribeHighQuality(enhanced)
+      text = result.text
+      backend = 'hq-large-v3'
+    } catch (hqErr: any) {
+      console.warn(`[transcribe] HQ path failed, falling back to fast: ${hqErr.message}`)
+      try {
+        const result = await transcribeLocal(audioBuffer)
+        text = result.text
+        backend = `fast-local-${result.backend}`
+      } catch (localErr: any) {
+        console.warn(`[transcribe] Fast local also failed, falling back to cloud: ${localErr.message}`)
+        text = await transcribeCloud(audioBuffer)
+        backend = 'cloud'
+      }
+    }
+  } else if (effectiveMode === 'fast' && isWhisperLocalAvailable()) {
+    try {
+      const result = await transcribeLocal(audioBuffer)
+      text = result.text
+      backend = `fast-local-${result.backend}`
+    } catch (localErr: any) {
+      console.warn(`[transcribe] Local whisper failed (${getWhisperBackend()}), falling back to cloud: ${localErr.message}`)
+      text = await transcribeCloud(audioBuffer)
+      backend = 'cloud'
+    }
+  } else {
+    text = await transcribeCloud(audioBuffer)
+    backend = 'cloud'
+  }
+
+  if (text && text.length > 0) {
+    try {
+      text = stripInlineHallucinationsOneShot(text)
+    } catch (stripErr: any) {
+      console.warn(`[transcribe] Hallucination strip failed (non-fatal): ${stripErr.message}`)
+    }
+  }
+
+  if (text && text.length > 0) {
+    try {
+      const fuzzyTargets = [...getAllSpeakerNames(), ...getVocabulary()]
+      const { text: corrected, replacements } = applyFuzzyCorrections(text, fuzzyTargets)
+      if (replacements > 0) {
+        console.log(`[transcribe] Fuzzy corrected ${replacements} word(s)`)
+        text = corrected
+      }
+    } catch (fuzzyErr: any) {
+      console.warn(`[transcribe] Fuzzy correction failed (non-fatal): ${fuzzyErr.message}`)
+    }
+  }
+
+  const elapsedMs = performance.now() - tStart
+  if (!text || isFullHallucination(text)) {
+    throw new NoSpeechDetectedError(text || '')
+  }
+
+  return {
+    text: text.trim(),
+    backend,
+    mode: effectiveMode,
+    elapsedMs,
+    audioBytes: audioBuffer.length,
+  }
+}

package/server/lib/utils.ts

@@ -0,0 +1,5 @@
+/** Safely extract error message from unknown catch value */
+export function errMsg(err: unknown): string {
+  if (err instanceof Error) return err.message
+  return String(err)
+}

package/server/lib/vad-silero.ts

@@ -0,0 +1,179 @@
+// Silero VAD — trims silence from audio chunks before Whisper inference.
+// Uses sherpa-onnx-node's built-in Silero VAD (same package as speaker embeddings).
+// Graceful fallback: if model missing or init fails, returns audio unchanged.
+//
+// Architecture: client keeps RMS-based chunking (decides WHEN to flush).
+// Server receives chunk, runs Silero VAD to extract speech segments,
+// trims silence edges, sends only speech to Whisper.
+
+import { resolve } from 'node:path'
+import { existsSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+
+// sherpa-onnx-node is CJS — use createRequire for ESM compat
+import { createRequire } from 'node:module'
+const require = createRequire(import.meta.url)
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url))
+const MODEL_PATH = resolve(__dirname, '..', 'models', 'silero_vad.onnx')
+
+const SAMPLE_RATE = 16000
+const BYTES_PER_SAMPLE = 2  // 16-bit PCM
+const WAV_HEADER_SIZE = 44
+// Minimum samples for Silero VAD to produce meaningful results (512 = one window)
+const MIN_SAMPLES = 512
+
+// Module-level state — single Vad instance reused via reset() (~11ms vs 45ms per-call)
+let sherpaOnnx: any = null
+let available = false
+let vadInstance: any = null
+
+export interface TrimResult {
+  trimmedWav: Buffer
+  speechRatio: number
+  segments: Array<{ startSample: number; sampleCount: number }>
+}
+
+/** Initialize Silero VAD. Returns true if model loaded, false otherwise. */
+export function initSileroVAD(): boolean {
+  if (!existsSync(MODEL_PATH)) {
+    console.log('[silero-vad] Model not found at', MODEL_PATH, '— VAD disabled, audio passes through untrimmed')
+    return false
+  }
+
+  try {
+    sherpaOnnx = require('sherpa-onnx-node')
+    // Create the reusable Vad instance (kept alive for server lifetime)
+    // bufferSizeInSeconds: 10 — chunks are max 6s, no need for 60s buffer
+    vadInstance = new sherpaOnnx.Vad(
+      {
+        sileroVad: { model: MODEL_PATH, threshold: 0.5, minSilenceDuration: 0.3, minSpeechDuration: 0.25, windowSize: 512 },
+        sampleRate: SAMPLE_RATE,
+        numThreads: 1,
+        provider: 'cpu',
+      },
+      10,  // bufferSizeInSeconds — chunks are max 6s
+    )
+    vadInstance.reset()  // verify it works
+    available = true
+    console.log('[silero-vad] Initialized: Silero VAD active (reused instance, bufferSize=10s)')
+    return true
+  } catch (err: any) {
+    console.error('[silero-vad] Init failed:', err.message)
+    available = false
+    return false
+  }
+}
+
+/** Check if Silero VAD is available */
+export function isSileroAvailable(): boolean {
+  return available
+}
+
+/**
+ * Trim silence from a WAV buffer using Silero VAD.
+ * Returns the trimmed WAV + speech ratio for logging.
+ *
+ * Graceful: if anything fails, returns original buffer unchanged.
+ */
+export function trimSilence(wavBuffer: Buffer): TrimResult {
+  const fallback: TrimResult = { trimmedWav: wavBuffer, speechRatio: 0.0, segments: [] }
+
+  if (!available || !vadInstance) return fallback
+
+  try {
+    // Parse WAV: extract raw PCM from after the 44-byte header
+    if (wavBuffer.length <= WAV_HEADER_SIZE + MIN_SAMPLES * BYTES_PER_SAMPLE) {
+      // Audio too short for VAD — return unchanged
+      return fallback
+    }
+
+    // Safe Int16Array extraction — slice guarantees alignment
+    const pcmBytes = wavBuffer.buffer.slice(
+      wavBuffer.byteOffset + WAV_HEADER_SIZE,
+      wavBuffer.byteOffset + wavBuffer.length,
+    )
+    const int16 = new Int16Array(pcmBytes)
+
+    if (int16.length < MIN_SAMPLES) return fallback
+
+    // Convert Int16 → Float32 (Silero expects [-1, 1] range)
+    const float32 = new Float32Array(int16.length)
+    for (let i = 0; i < int16.length; i++) {
+      float32[i] = int16[i] / 32768
+    }
+
+    // Reuse module-level Vad instance — reset() clears state between calls
+    // Benchmark: 11ms avg vs 45ms with per-call construction
+    vadInstance.reset()
+    vadInstance.acceptWaveform(float32)
+    vadInstance.flush()
+
+    // Collect speech segments
+    const speechSegments: Array<{ startSample: number; samples: Float32Array }> = []
+    let totalSpeechSamples = 0
+
+    while (!vadInstance.isEmpty()) {
+      const segment = vadInstance.front()
+      speechSegments.push({ startSample: segment.start, samples: segment.samples })
+      totalSpeechSamples += segment.samples.length
+      vadInstance.pop()
+    }
+
+    const speechRatio = int16.length > 0 ? totalSpeechSamples / int16.length : 0.0
+
+    if (speechSegments.length === 0 || totalSpeechSamples === 0) {
+      // No speech detected — return original unchanged
+      return { trimmedWav: wavBuffer, speechRatio: 0.0, segments: [] }
+    }
+
+    // Concatenate speech segments into a single Float32Array
+    const speechFloat32 = new Float32Array(totalSpeechSamples)
+    let offset = 0
+    const segmentInfo: TrimResult['segments'] = []
+    for (const seg of speechSegments) {
+      speechFloat32.set(seg.samples, offset)
+      segmentInfo.push({ startSample: seg.startSample, sampleCount: seg.samples.length })
+      offset += seg.samples.length
+    }
+
+    // Convert Float32 back to Int16 PCM
+    const speechInt16 = new Int16Array(totalSpeechSamples)
+    for (let i = 0; i < totalSpeechSamples; i++) {
+      // Clamp to [-1, 1] then scale to Int16 range
+      const clamped = Math.max(-1, Math.min(1, speechFloat32[i]))
+      speechInt16[i] = clamped < 0 ? clamped * 32768 : clamped * 32767
+    }
+
+    const pcmLength = speechInt16.length * BYTES_PER_SAMPLE
+    const trimmedWav = Buffer.alloc(WAV_HEADER_SIZE + pcmLength)
+
+    // Write WAV header (same format as audio-pipeline.ts pcmToWav)
+    trimmedWav.write('RIFF', 0)
+    trimmedWav.writeUInt32LE(WAV_HEADER_SIZE - 8 + pcmLength, 4)
+    trimmedWav.write('WAVE', 8)
+    trimmedWav.write('fmt ', 12)
+    trimmedWav.writeUInt32LE(16, 16)          // sub-chunk size
+    trimmedWav.writeUInt16LE(1, 20)           // PCM format
+    trimmedWav.writeUInt16LE(1, 22)           // mono
+    trimmedWav.writeUInt32LE(SAMPLE_RATE, 24)
+    trimmedWav.writeUInt32LE(SAMPLE_RATE * BYTES_PER_SAMPLE, 28)  // byte rate
+    trimmedWav.writeUInt16LE(BYTES_PER_SAMPLE, 32)                // block align
+    trimmedWav.writeUInt16LE(16, 34)          // bits per sample
+    trimmedWav.write('data', 36)
+    trimmedWav.writeUInt32LE(pcmLength, 40)
+
+    // Copy PCM data after header
+    Buffer.from(speechInt16.buffer).copy(trimmedWav, WAV_HEADER_SIZE)
+
+    // Guard: if trimmed result is too small for Whisper, fall back to original
+    if (trimmedWav.length < 100) {
+      return { trimmedWav: wavBuffer, speechRatio, segments: segmentInfo }
+    }
+
+    return { trimmedWav, speechRatio, segments: segmentInfo }
+  } catch (err: any) {
+    console.error('[silero-vad] trimSilence error:', err.message)
+    return fallback
+  }
+}