@gotcos/glasses-server 6.7.0 → 6.9.0

package/CHANGELOG.md

@@ -1,5 +1,55 @@
 # Changelog
 
+## 6.9.0
+
+Live recoverable prompt transcription for COS Glasses builds 200+.
+
+- **Words appear while speaking.** After each audio chunk is durably acknowledged,
+  its sanitized fast/local transcript is published on the existing authenticated,
+  replayable display stream as `prompt_transcript`; the phone/G2 client can fill
+  the Listening body without adding another recorder, polling loop, or ASR job.
+- **Recovery remains authoritative.** The event is optional presentation state.
+  Stored WAV chunks, final HQ transcription, glossary cleanup, editing, retry,
+  and send behavior remain unchanged and continue even if no display client is
+  connected.
+- **Stale retries cannot repaint.** The server rechecks the exact draft, chunk
+  index, and audio bytes after warm transcription. Replaced audio never emits
+  its obsolete words, while client-side draft scoping, ordering, and replay
+  deduplication handle reconnects safely.
+- **Public boundary retained.** This release adds no private COS paths, personal
+  data, LaunchAgent controls, remote restart authority, or machine-management
+  endpoints.
+
+## 6.8.0
+
+Public-safe meeting finalization for COS Glasses build 199.
+
+- **Authenticated meeting save.** `POST /api/meeting/save` finalizes an existing
+  `transcribe-stream` session without adding coaching, private classification,
+  personal paths, or COS-only enrichment to the public package. Lost-chunk gaps,
+  original client timing, provider evidence, and sparse raw-audio indices remain
+  intact through deferred iPhone replay and save.
+- **Durable standalone archive.** Canonical markdown and structured sidecars are
+  published atomically under `dataPath('recordings', 'YYYY-MM')`. Directories are
+  `0700`, files are `0600`, filenames are path-safe and session-unique, and an
+  fsync-backed sidecar-first/markdown-last commit keeps incomplete pairs hidden.
+- **Review on the current client.** Authenticated `GET /api/meetings`, literal
+  `GET /api/meetings/detail`, and the build199-compatible dynamic detail route
+  list and read standalone recordings after process/package restarts. Traversal,
+  unsafe filenames, symlinked roots/months/files, absolute-path disclosure, and
+  cross-domain detail mismatches fail closed.
+- **Transcript-quality bouncer.** Post-meeting batch text must preserve at least
+  50% live coverage, provide independent evidence when no live baseline exists,
+  and avoid repeated long segments/sentences/prefixes. Mixed timestamp coverage
+  falls back to complete batch text instead of dropping text-only segments.
+- **Recovery evidence wins.** Canonical streaming text remains untouched when a
+  batch is rejected or cannot be applied. Pending WAVs are deleted only after
+  accepted text and its sidecar decision are both durable; every other outcome
+  retains audio for bounded two-hour cleanup. HQ batch decoders serialize and
+  refresh their cleanup lease while queued or active.
+- **Capability detection.** `/api/health` now advertises
+  `features.meetingFinalization` for compatible clients.
+
 ## 6.7.0
 
 Durable prompt recovery and self-healing local transcription for COS Glasses

package/README.md

@@ -64,7 +64,9 @@ The built-in IP allowlist blocks public-internet traffic regardless.
 - Send phone photos with queued prompts, and review assistant-selected generated,
   research, or explicitly used email images in Messages and on the G2 lens
 - Recover long voice prompts after phone, network, or server interruptions. Audio
-  chunks are saved before transcription and retained locally for 72 hours.
+  chunks are saved before transcription and retained locally for 72 hours. On
+  compatible app builds, their warm transcript also appears live while speaking;
+  final HQ transcription remains authoritative.
 - Live voice capture + transcription during meetings
 - Local whisper.cpp transcription (free) with OpenAI fallback (optional)
 - Tasks / calendar / people context **if** you run the

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.7.0",
+  "version": "6.9.0",
   "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by your local Claude Code or Codex CLI",
   "type": "module",
   "bin": {

package/server/index.ts

@@ -18,6 +18,8 @@ import { queryRouter } from './routes/query.js'
 import { transcribeRouter } from './routes/transcribe.js'
 import { displayRouter } from './routes/display.js'
 import { transcribeStreamRouter } from './routes/transcribe-stream.js'
+import { meetingRouter } from './routes/meeting.js'
+import { meetingsRouter } from './routes/meetings.js'
 import { openaiCompatRouter } from './routes/openai-compat.js'
 import { openaiKeyRouter } from './routes/openai-key.js'
 import { messageRefRouter } from './routes/message-ref.js'
@@ -141,6 +143,8 @@ app.use('/api', queryRouter)
 app.use('/api', transcribeRouter)
 app.use('/api', displayRouter)
 app.use('/api', transcribeStreamRouter)
+app.use('/api', meetingRouter)
+app.use('/api', meetingsRouter)
 app.use('/api', openaiKeyRouter)
 // v6.3.0 — Message History, cross-day 'reference message N', and history
 // recovery for public npx users (previously full-COS-server only).

package/server/lib/atomic-fs.ts

@@ -7,7 +7,20 @@
 // Use for sessions.json, archive/*.json, and any other durable JSON we can't
 // afford to lose.
 
-import { writeFileSync, renameSync, existsSync, readFileSync } from 'node:fs'
+import {
+  closeSync,
+  constants,
+  existsSync,
+  fchmodSync,
+  fsyncSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs'
+import { randomBytes } from 'node:crypto'
+import { basename, dirname, join } from 'node:path'
 
 export function atomicWriteFileSync(path: string, data: string | Buffer, options: { mode?: number } = {}): void {
   const tmp = `${path}.tmp`
@@ -15,6 +28,69 @@ export function atomicWriteFileSync(path: string, data: string | Buffer, options
   renameSync(tmp, path)
 }
 
+/**
+ * Publish private durable state without ever exposing a partially-written file.
+ *
+ * The existing `atomicWriteFileSync` intentionally remains lightweight for
+ * high-frequency caches. Meeting finalization uses this stronger variant:
+ * bytes and file metadata are fsync'd before rename, the destination mode is
+ * forced after publish, and the containing directory is fsync'd where the
+ * platform supports it. A randomized exclusive temp name also prevents two
+ * independent writers from sharing `<path>.tmp`.
+ */
+export function durableAtomicWriteFileSync(
+  path: string,
+  data: string | Buffer,
+  options: { mode?: number } = {},
+): void {
+  const mode = options.mode ?? 0o600
+  const dir = dirname(path)
+  const tmp = join(
+    dir,
+    `.${basename(path)}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`,
+  )
+  let fd: number | null = null
+
+  try {
+    const noFollow = constants.O_NOFOLLOW ?? 0
+    fd = openSync(
+      tmp,
+      constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | noFollow,
+      mode,
+    )
+    writeFileSync(fd, data)
+    fchmodSync(fd, mode)
+    fsyncSync(fd)
+    closeSync(fd)
+    fd = null
+
+    // The exclusive temp inode already has its final private mode. Apply chmod
+    // before the commit so no failure can be reported after rename succeeded.
+    renameSync(tmp, path)
+
+    // Persist the directory entry as well as the file contents. Some virtual
+    // filesystems do not support directory fsync; the already-fsync'd file and
+    // atomic rename still provide the strongest available behavior there.
+    let dirFd: number | null = null
+    try {
+      dirFd = openSync(dir, constants.O_RDONLY)
+      fsyncSync(dirFd)
+    } catch {
+      // Best available durability on filesystems that reject directory fsync.
+    } finally {
+      if (dirFd !== null) {
+        try { closeSync(dirFd) } catch { /* commit already succeeded */ }
+      }
+    }
+  } catch (error) {
+    if (fd !== null) {
+      try { closeSync(fd) } catch { /* already closed */ }
+    }
+    try { unlinkSync(tmp) } catch { /* publish may already have renamed it */ }
+    throw error
+  }
+}
+
 /**
  * Read + JSON.parse with explicit missing-vs-corrupt distinction.
  * - File missing: returns { status: 'missing' } — caller starts fresh silently

package/server/lib/batch-transcript-quality.ts

@@ -0,0 +1,243 @@
+import type { WhisperWord } from './whisper-local.js'
+
+export interface BatchSegment {
+  startChunkIdx: number
+  endChunkIdx: number
+  startElapsed: number
+  endElapsed: number
+  speakers: string[]
+}
+
+export interface BatchResult {
+  segment: BatchSegment
+  text: string
+  words: WhisperWord[]
+  speakerWords: Array<{ word: string; start: number; end: number; speaker: string }>
+}
+
+export interface BatchTranscription {
+  transcriptionQuality: 'batch' | 'streaming'
+  batchTranscript?: string
+  batchSegments?: BatchResult[]
+  qualityReport?: BatchQualityReport
+}
+
+export interface BatchTranscriptSelection {
+  text: string
+  source: 'speaker-words' | 'batch-text'
+}
+
+export type BatchQualityReason =
+  | 'accepted'
+  | 'insufficient-coverage'
+  | 'insufficient-batch-evidence'
+  | 'repetitive-output'
+
+export interface BatchQualityReport {
+  accepted: boolean
+  reason: BatchQualityReason
+  batchWordCount: number
+  streamingWordCount: number
+  coverageRatio: number
+  timedWordCount: number
+  timedWordRatio: number
+  maxRepeatedUnitCount: number
+  duplicateWordRatio: number
+  repeatedUnit?: string
+}
+
+interface RepetitionStats {
+  maxRepeatedUnitCount: number
+  duplicateWordRatio: number
+  repeatedUnit?: string
+}
+
+function countWords(text: string): number {
+  return text.trim().split(/\s+/).filter(Boolean).length
+}
+
+/**
+ * Speaker timestamps are optional on fallback transcription backends. Use the
+ * attributed form only when every non-empty segment is represented; otherwise
+ * persist the complete batch text so text-only segments cannot disappear.
+ */
+export function selectBatchTranscriptForPersistence(
+  batchTranscript: string,
+  batchSegments: readonly Pick<BatchResult, 'text' | 'speakerWords'>[] | undefined,
+): BatchTranscriptSelection {
+  const nonEmpty = (batchSegments ?? []).filter(segment => countWords(segment.text) > 0)
+  const completeSpeakerCoverage = nonEmpty.length > 0 && nonEmpty.every(segment => {
+    const textWords = countWords(segment.text)
+    return segment.speakerWords.length >= Math.max(1, Math.floor(textWords * 0.75))
+  })
+
+  if (!completeSpeakerCoverage) return { text: batchTranscript, source: 'batch-text' }
+
+  const lines: string[] = []
+  for (const segment of nonEmpty) {
+    let currentSpeaker = ''
+    let currentWords: string[] = []
+    for (const speakerWord of segment.speakerWords) {
+      const word = speakerWord.word.trim()
+      if (!word) continue
+      const speaker = speakerWord.speaker || 'Ext'
+      if (speaker !== currentSpeaker) {
+        if (currentWords.length > 0) lines.push(`[${currentSpeaker}]: ${currentWords.join(' ')}`)
+        currentSpeaker = speaker
+        currentWords = [word]
+      } else {
+        currentWords.push(word)
+      }
+    }
+    if (currentWords.length > 0) lines.push(`[${currentSpeaker}]: ${currentWords.join(' ')}`)
+  }
+
+  return lines.length > 0
+    ? { text: lines.join('\n'), source: 'speaker-words' }
+    : { text: batchTranscript, source: 'batch-text' }
+}
+
+function normalizeQualityUnit(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    // A repeated template with only a changing index/date/amount is still the
+    // same unit and must not manufacture apparent transcript coverage.
+    .replace(/\b\d+\b/g, '<n>')
+}
+
+function toLongQualityUnit(raw: string): { normalized: string; sample: string; wordCount: number } | null {
+  const sample = raw.replace(/\s+/g, ' ').trim()
+  const normalized = normalizeQualityUnit(sample)
+  const wordCount = countWords(normalized)
+  // Exempt short acknowledgements and ordinary meeting refrains. The observed
+  // failure repeated a complete, long prompted sentence dozens of times.
+  if (wordCount < 8 || normalized.length < 40) return null
+  return { normalized, sample, wordCount }
+}
+
+function toLongPrefixUnit(raw: string): { normalized: string; sample: string; wordCount: number } | null {
+  const unit = toLongQualityUnit(raw)
+  if (!unit) return null
+  const prefixWords = unit.normalized.split(/\s+/).slice(0, 12)
+  if (prefixWords.length < 12) return null
+  return { normalized: prefixWords.join(' '), sample: unit.sample, wordCount: prefixWords.length }
+}
+
+function measureRepetition(
+  units: readonly { normalized: string; sample: string; wordCount: number }[],
+  batchWordCount: number,
+): RepetitionStats {
+  const counts = new Map<string, { count: number; wordCount: number; sample: string }>()
+  for (const unit of units) {
+    const current = counts.get(unit.normalized)
+    if (current) current.count += 1
+    else counts.set(unit.normalized, { count: 1, wordCount: unit.wordCount, sample: unit.sample })
+  }
+
+  let maxRepeatedUnitCount = 0
+  let duplicateWords = 0
+  let repeatedUnit: string | undefined
+  for (const entry of counts.values()) {
+    if (entry.count <= 1) continue
+    duplicateWords += (entry.count - 1) * entry.wordCount
+    if (entry.count > maxRepeatedUnitCount) {
+      maxRepeatedUnitCount = entry.count
+      repeatedUnit = entry.sample.slice(0, 160)
+    }
+  }
+
+  return {
+    maxRepeatedUnitCount,
+    duplicateWordRatio: batchWordCount > 0 ? Math.min(1, duplicateWords / batchWordCount) : 0,
+    ...(repeatedUnit ? { repeatedUnit } : {}),
+  }
+}
+
+/**
+ * Deterministic bouncer for a post-meeting transcript candidate. Coverage by
+ * itself is unsafe: Whisper can repeat one prompted sentence many times and
+ * appear to recover the whole meeting. Measure long exact units at segment,
+ * sentence, and prefix granularity while leaving short conversational repeats
+ * alone.
+ */
+export function evaluateBatchQuality(
+  batchResults: readonly Pick<BatchResult, 'text' | 'words'>[],
+  streamingWordCount: number,
+): BatchQualityReport {
+  const batchText = batchResults.map(result => result.text).join(' ')
+  const batchWordCount = countWords(batchText)
+  const safeStreamingWordCount = Math.max(0, streamingWordCount)
+  const coverageRatio = safeStreamingWordCount > 0
+    ? batchWordCount / safeStreamingWordCount
+    : (batchWordCount > 0 ? 1 : 0)
+  const timedWordCount = batchResults.reduce((sum, result) => sum + (result.words?.length ?? 0), 0)
+  const timedWordRatio = batchWordCount > 0 ? timedWordCount / batchWordCount : 0
+  const nonEmptySegmentCount = batchResults.filter(result => countWords(result.text) > 0).length
+
+  const segmentUnits = batchResults
+    .map(result => toLongQualityUnit(result.text))
+    .filter((unit): unit is NonNullable<typeof unit> => unit !== null)
+  const segmentStats = measureRepetition(segmentUnits, batchWordCount)
+
+  const sentenceUnits: Array<{ normalized: string; sample: string; wordCount: number }> = []
+  for (const result of batchResults) {
+    for (const sentence of result.text.match(/[^.!?\n]+(?:[.!?]+|$)/g) ?? []) {
+      const unit = toLongQualityUnit(sentence)
+      if (unit) sentenceUnits.push(unit)
+    }
+  }
+  const sentenceStats = measureRepetition(sentenceUnits, batchWordCount)
+
+  // Keep segment and sentence prefix populations separate. A one-sentence
+  // segment otherwise counts twice and can turn two valid repeats into four.
+  const segmentPrefixStats = measureRepetition(
+    batchResults
+      .map(result => toLongPrefixUnit(result.text))
+      .filter((unit): unit is NonNullable<typeof unit> => unit !== null),
+    batchWordCount,
+  )
+  const sentencePrefixStats = measureRepetition(
+    batchResults.flatMap(result => (
+      result.text.match(/[^.!?\n]+(?:[.!?]+|$)/g) ?? []
+    ).map(sentence => toLongPrefixUnit(sentence)))
+      .filter((unit): unit is NonNullable<typeof unit> => unit !== null),
+    batchWordCount,
+  )
+
+  const repetition = [segmentStats, sentenceStats, segmentPrefixStats, sentencePrefixStats]
+    .reduce((strongest, candidate) => (
+      candidate.duplicateWordRatio > strongest.duplicateWordRatio ? candidate : strongest
+    ))
+
+  const repetitiveOutput = (
+    repetition.maxRepeatedUnitCount >= 4 && repetition.duplicateWordRatio >= 0.08
+  ) || (
+    repetition.maxRepeatedUnitCount >= 8 && repetition.duplicateWordRatio >= 0.03
+  )
+
+  const reason: BatchQualityReason = safeStreamingWordCount === 0 && (
+    batchWordCount < 50 || nonEmptySegmentCount < 2
+  )
+    ? 'insufficient-batch-evidence'
+    : coverageRatio < 0.5
+      ? 'insufficient-coverage'
+      : repetitiveOutput
+        ? 'repetitive-output'
+        : 'accepted'
+
+  return {
+    accepted: reason === 'accepted',
+    reason,
+    batchWordCount,
+    streamingWordCount: safeStreamingWordCount,
+    coverageRatio: Number(coverageRatio.toFixed(4)),
+    timedWordCount,
+    timedWordRatio: Number(timedWordRatio.toFixed(4)),
+    maxRepeatedUnitCount: repetition.maxRepeatedUnitCount,
+    duplicateWordRatio: Number(repetition.duplicateWordRatio.toFixed(4)),
+    ...(repetition.repeatedUnit ? { repeatedUnit: repetition.repeatedUnit } : {}),
+  }
+}

package/server/lib/display-bus.ts

@@ -8,7 +8,7 @@ const bus = new EventEmitter()
 bus.setMaxListeners(20) // Multiple glasses clients
 
 export interface DisplayEvent {
-  type: 'chunk' | 'done' | 'error' | 'tool_status' | 'start' | 'session_restore' | 'transcript_chunk' | 'recording_start' | 'recording_stop' | 'coaching_nudge'
+  type: 'chunk' | 'done' | 'error' | 'tool_status' | 'start' | 'session_restore' | 'transcript_chunk' | 'prompt_transcript' | 'recording_start' | 'recording_stop' | 'coaching_nudge'
   data: Record<string, unknown>
 }
 

package/server/lib/meeting-batch-persistence.ts

@@ -0,0 +1,53 @@
+import { existsSync, readFileSync } from 'node:fs'
+import { durableAtomicWriteFileSync } from './atomic-fs.js'
+import type { BatchTranscription } from './batch-transcript-quality.js'
+
+/** Replace only the final canonical transcript section. Missing markers fail closed. */
+export function replaceMeetingTranscriptAtomic(meetingPath: string, transcript: string): boolean {
+  const content = readFileSync(meetingPath, 'utf8')
+  if (!/## Transcript\n\n[\s\S]*$/.test(content)) return false
+  const updated = content
+    .replace(/\| \*\*Transcription quality\*\* \| [^|\n]+ \|/i, '| **Transcription quality** | batch |')
+    .replace(/## Transcript\n\n[\s\S]*$/, `## Transcript\n\n${transcript}\n`)
+  durableAtomicWriteFileSync(meetingPath, updated, { mode: 0o600 })
+  return true
+}
+
+/** Persist the batch decision without ever storing a rejected candidate as canonical. */
+export function persistBatchDecisionSidecar(
+  sidecarPath: string,
+  batchResult: BatchTranscription,
+  batchApplied: boolean,
+): boolean {
+  if (!existsSync(sidecarPath)) return false
+  const parsed = JSON.parse(readFileSync(sidecarPath, 'utf8')) as unknown
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return false
+  const sidecar = parsed as Record<string, unknown>
+  sidecar.transcriptionQuality = batchApplied ? 'batch' : 'streaming'
+  sidecar.batchApplied = batchApplied
+  if (batchResult.qualityReport) sidecar.batchQualityReport = batchResult.qualityReport
+
+  if (batchApplied) {
+    sidecar.batchTranscript = batchResult.batchTranscript
+    sidecar.batchSegments = batchResult.batchSegments?.map(result => ({
+      startChunkIdx: result.segment.startChunkIdx,
+      endChunkIdx: result.segment.endChunkIdx,
+      startElapsed: result.segment.startElapsed,
+      endElapsed: result.segment.endElapsed,
+      text: result.text,
+      words: result.words,
+      speakerWords: result.speakerWords,
+    }))
+  } else {
+    delete sidecar.batchTranscript
+    delete sidecar.batchSegments
+  }
+
+  durableAtomicWriteFileSync(sidecarPath, JSON.stringify(sidecar, null, 2), { mode: 0o600 })
+  return true
+}
+
+/** Raw audio is disposable only after canonical text and decision metadata are durable. */
+export function canDeletePendingBatchAudio(batchApplied: boolean, metadataPersisted: boolean): boolean {
+  return batchApplied && metadataPersisted
+}