@gotcos/glasses-server 6.73.5 → 6.73.14

package/server/routes/transcribe-stream.ts

@@ -1,3 +1,6 @@
+import { MacParakeetUnavailableError, transcribeMacParakeet } from '../lib/mac-recording-asr.js'
+import { getMeetingStore } from '../lib/meeting-store.js'
+import { MAC_AUDIO, MAC_CAPTURE_ID, persistMacChunkReceipt, macError, registerMacRecording, requireMacRecording, readMacRecording, validateMacWav } from '../lib/mac-recording.js'
 import { AUTO_ENROLL_THRESHOLD } from '../lib/voiceprint-floor.js'
 import { readSpeakerPolicy, policyLabel, type SpeakerMode } from '../lib/meeting-speaker-policy.js'
 // POST /api/transcribe-stream — Streaming transcription for continuous meeting capture
@@ -19,6 +22,7 @@ import {
   clearStrandedDraft,
   listStrandedDrafts,
   releaseStrandedState,
+  restoresAtBoot,
   shouldCloseAfterFailedPromote,
   sweepStrandedSessions,
   promoteStrandedSession,
@@ -277,7 +281,7 @@ export interface TranscriptChunk {
   /** Capture-time policy survives batch and final passes; absent means legacy Standard. */
   speakerPolicy?: SpeakerMode
   words?: WhisperWord[] // word-level timestamps from DTW alignment
-  asrProvider?: 'server-whisper' | 'iphone-whisperkit-beta'
+  asrProvider?: 'server-whisper' | 'server-parakeet' | 'iphone-whisperkit-beta'
   backend?: string
   model?: string
   mode?: string
@@ -286,6 +290,7 @@ export interface TranscriptChunk {
   audioSha256?: string
   canonical?: boolean
   evenHubSpeakerRole?: EvenSpeakerRoleHistogram
+  audioDurationMs?: number
   /** 6.61.0: who spoke when inside this chunk, in time order. Present when
    *  Nemotron separated the chunk's voices; `speaker` is then the dominant name. */
   segments?: ChunkSpeakerSegment[]
@@ -365,6 +370,8 @@ interface EmptyTranscriptCompletion {
   elapsed: number
   backend?: string
   asrProvider?: string
+  model?: string
+  latencyMs?: number
   fallbackReason?: string
   audioSha256?: string
   canonical: false
@@ -381,6 +388,8 @@ interface StreamChunkCompletionResponse {
   canonical: boolean
   backend?: string
   asrProvider?: string
+  model?: string
+  latencyMs?: number
   fallbackReason?: string
   /** 6.61.0, additive: the chunk's speaker turns in time order. Their texts
    *  joined with one space equal `text`; `speaker` stays the dominant name. */
@@ -820,7 +829,7 @@ function recoverSessions(): void {
         if (data.sessionId && (hasChunks || Array.isArray(data.receivedIndices) || Number.isFinite(lastActivityAt))) {
           // Active retention is idle-based. Long meetings are not purged merely
           // because their original start time is old.
-          if (Date.now() - lastActivityAt < LOCAL_FIRST_MEETING_IDLE_RETENTION_MS) {
+          if (restoresAtBoot(data.sessionId, Date.now() - lastActivityAt, LOCAL_FIRST_MEETING_IDLE_RETENTION_MS)) {
             const chunks: TranscriptChunk[] = []
             if (indexed) {
               for (const e of indexed) {
@@ -1520,7 +1529,11 @@ export function getSessionChunkEntries(sessionId: string): IndexedTranscriptChun
           elapsed: empty.elapsed,
           similarity: 0,
           backend: empty.backend,
-          asrProvider: empty.asrProvider === 'iphone-whisperkit-beta'
+          ...(empty.model ? { model: empty.model } : {}),
+          ...(empty.latencyMs !== undefined ? { latencyMs: empty.latencyMs } : {}),
+          asrProvider: empty.asrProvider === 'server-parakeet'
+            ? 'server-parakeet'
+            : empty.asrProvider === 'iphone-whisperkit-beta'
             ? 'iphone-whisperkit-beta'
             : empty.asrProvider === 'server-whisper'
               ? 'server-whisper'
@@ -1547,6 +1560,9 @@ export function getSessionAsrCompletedIndices(sessionId: string): number[] {
 }
 
 function scheduleSessionProgressiveHq(sessionId: string): void {
+  // Progressive checkpoints are Whisper-only and may not bypass a Mac's
+  // explicit Parakeet choice, even if the separate HQ canary is enabled.
+  if (MAC_CAPTURE_ID.test(sessionId) && readSttSettings().hq === 'parakeet-ultra') return
   // Preserve the default-off contract at the route boundary. In addition to
   // avoiding any cache work, this keeps legacy/local-first transcription
   // independent of the progressive HQ capability surface when the canary is
@@ -2016,6 +2032,8 @@ function canonicalChunkResponse(
     canonical: true,
     backend: existing.backend,
     asrProvider: existing.asrProvider,
+    ...(existing.model ? { model: existing.model } : {}),
+    ...(existing.latencyMs !== undefined ? { latencyMs: existing.latencyMs } : {}),
     fallbackReason: existing.fallbackReason,
     ...(existing.segments?.length ? { segments: existing.segments } : {}),
   }
@@ -2072,7 +2090,7 @@ function identifyChunkSpeaker(audioBuffer: Buffer, sessionId: string, chunkIndex
   if (!isEmbeddingAvailable() || audioDurationSec < VOICEPRINT_MIN_AUDIO_SEC) return policyLabel({ speaker: clientSpeaker, similarity: 0 }, policy)
 
   const tEmb = performance.now()
-  const embeddingResult = identifySpeaker(audioBuffer, expectedSpeakers, { minSimilarity: policy === 'high-confidence' ? AUTO_ENROLL_THRESHOLD : undefined })
+  const embeddingResult = identifySpeaker(audioBuffer, expectedSpeakers, { minSimilarity: policy === 'high-confidence' ? AUTO_ENROLL_THRESHOLD : undefined, ...(MAC_CAPTURE_ID.test(sessionId) ? { prioritizeOwner: false, event: 'mac-system-audio' } : {}) })
   console.log(`[perf] identifySpeaker: ${(performance.now() - tEmb).toFixed(1)}ms`)
   if (!embeddingResult) return policyLabel({ speaker: clientSpeaker, similarity: 0 }, policy)
 
@@ -2080,8 +2098,9 @@ function identifyChunkSpeaker(audioBuffer: Buffer, sessionId: string, chunkIndex
   // it exists: identifySpeaker computes it per chunk and nothing else keeps it,
   // so a correction made later has no acoustic evidence without this write.
   // Deliberately includes 'Ext' — an unidentified voice is the case with no
-  // other way to be trained.
-  if (embeddingResult.embedding) {
+  // other way to be trained. Mac playback is recognition-only: never bank
+  // its voiceprints or create auxiliary training-audio copies.
+  if (embeddingResult.embedding && !MAC_CAPTURE_ID.test(sessionId)) {
     appendChunkEmbedding(sessionId, {
       i: chunkIndex,
       speaker: policyLabel(embeddingResult, policy).speaker,
@@ -2117,7 +2136,7 @@ function identifyChunkSpeaker(audioBuffer: Buffer, sessionId: string, chunkIndex
     }
   }
 
-  if (speaker === 'Ext' && audioBuffer.length >= 16000) {
+  if (speaker === 'Ext' && audioBuffer.length >= 16000 && !MAC_CAPTURE_ID.test(sessionId)) {
     const extSessionDir = resolve(EXT_AUDIO_DIR, sessionId)
     ensurePrivateDirectory(extSessionDir)
     const extCount = extAudioCounts.get(sessionId) ?? 0
@@ -2154,6 +2173,7 @@ async function processStreamChunk(opts: {
   clientElapsed?: number
   /** Original client recording start, applied only before canonical chunks. */
   startTimeOverride?: number
+  audioDurationMs?: number
   evenHubSpeakerRole?: EvenSpeakerRoleHistogram
 }): Promise<StreamChunkCompletionResponse> {
   const { sessionId, chunkIndex, clientSpeaker, audioBuffer, candidate } = opts
@@ -2215,7 +2235,7 @@ async function processStreamChunk(opts: {
 
   // Pin identity policy at acceptance, before persistence, ASR or diarization
   // yields. A later loosening applies to the next chunk, never this audio.
-  const capturePolicy = readSpeakerPolicy(sessionId).mode
+  const capturePolicy = MAC_CAPTURE_ID.test(sessionId) ? (requireMacRecording(sessionId), 'high-confidence' as const) : readSpeakerPolicy(sessionId).mode
   await persistRawSessionAudioChunk(sessionId, chunkIndex, audioBuffer)
   // Commit the received-index ledger only after the canonical raw WAV is
   // durable. A failure is typed non-2xx and a retry remains safe.
@@ -2250,7 +2270,7 @@ async function processStreamChunk(opts: {
   let rawText = ''
   let words: WhisperWord[] | undefined
   let backend = 'candidate'
-  let asrProvider: 'server-whisper' | 'iphone-whisperkit-beta' = 'server-whisper'
+  let asrProvider: 'server-whisper' | 'server-parakeet' | 'iphone-whisperkit-beta' = 'server-whisper'
   let model: string | undefined
   let mode: string | undefined
   let latencyMs: number | undefined
@@ -2263,6 +2283,16 @@ async function processStreamChunk(opts: {
     model = candidate.model
     mode = candidate.mode
     latencyMs = candidate.latencyMs
+  } else if (MAC_CAPTURE_ID.test(sessionId) && stt.hq === 'parakeet-ultra') {
+    // Use the stored choice directly. effectiveStt/preview intentionally fall
+    // back when unavailable, which is not an acceptable canonical Mac commit.
+    const result = await transcribeMacParakeet(whisperAudio, quiet.speechSec)
+    rawText = result.text
+    words = result.words
+    backend = result.backend
+    asrProvider = result.asrProvider
+    model = result.model
+    latencyMs = result.latencyMs
   } else {
     const t0 = performance.now()
     const result = await transcribeWithServerWhisper(audioBuffer, whisperAudio, whisperContext, isQuiet)
@@ -2343,6 +2373,8 @@ async function processStreamChunk(opts: {
       elapsed,
       backend,
       asrProvider,
+      ...(model ? { model } : {}),
+      ...(latencyMs !== undefined ? { latencyMs } : {}),
       fallbackReason: sanitized.fallbackReason || fallbackReason,
       audioSha256,
       canonical: false,
@@ -2358,6 +2390,7 @@ async function processStreamChunk(opts: {
   const chunk: TranscriptChunk = {
     text: trimmedText,
     speakerPolicy: capturePolicy,
+    ...(opts.audioDurationMs !== undefined ? { audioDurationMs: opts.audioDurationMs } : {}),
     speaker,
     elapsed,
     similarity,
@@ -2417,6 +2450,9 @@ async function processStreamChunk(opts: {
 }
 
 function sendStreamError(res: { status: (code: number) => { json: (body: unknown) => unknown } }, err: unknown): unknown {
+  if (err instanceof MacParakeetUnavailableError) {
+    return res.status(err.status).json({ error: err.message, reason: err.reason, retryable: true, model: err.model })
+  }
   if (err instanceof TranscriptionUnavailableError) {
     console.warn(`[transcribe-stream] ${err.message}`)
     return res.status(err.status).json({
@@ -2507,12 +2543,82 @@ transcribeStreamRouter.post('/transcribe-stream/preview', async (req, res) => {
   }
 })
 
+/** Unlike the received ledger, only these canonical completions acknowledge ASR.
+ * A native stop supplies the total captured chunks, including its last tail. */
+export function assertMacCaptureComplete(sessionId: string, expectedChunkCount: unknown): void {
+  if (!Number.isInteger(expectedChunkCount) || Number(expectedChunkCount) < 1 || Number(expectedChunkCount) > 10000) {
+    throw macError(400, 'mac_chunk_count_required', 'Mac save requires the total captured chunk count')
+  }
+  const session = sessions.get(sessionId)
+  const expected = Number(expectedChunkCount)
+  const missingIndices: number[] = []
+  for (let index = 0; index < expected; index++) {
+    if (!session?.chunks[index] && !session?.emptyCompletions?.[String(index)]) missingIndices.push(index)
+  }
+  if (missingIndices.length || (session?.maxChunkIndex ?? -1) >= expected) {
+    throw Object.assign(macError(409, 'mac_capture_incomplete', 'Mac audio is not fully acknowledged; retry pending chunks before saving'), { missingIndices })
+  }
+}
+
+// Manual Mac pilot: independent from iPhone candidate and cue rollout gates.
+transcribeStreamRouter.get('/mac-recording/capabilities', (_req, res) => {
+  res.set('Cache-Control', 'private, no-store').json({ available: true, ...MAC_AUDIO, serverInstanceId: getServerInstanceId() })
+})
+function requireMacPin(req: { get(name: string): string | undefined }): string {
+  const pin = req.get('X-COS-Server-Instance')?.trim()
+  if (!pin) throw macError(400, 'server_identity_pin_required', 'Server identity pin required')
+  return assertPinnedServerIdentity(pin, undefined)
+}
+transcribeStreamRouter.post('/mac-recording/start', (req, res) => {
+  let lease: MaintenanceWorkLease | undefined
+  try {
+    const serverInstanceId = requireMacPin(req)
+    if (typeof req.body?.captureId !== 'string' || !MAC_CAPTURE_ID.test(req.body.captureId)) throw macError(400, 'invalid_capture_id', 'Invalid Mac capture ID')
+    if (isSessionDeleted(req.body.captureId) || getMeetingStore().findBySessionId(req.body.captureId)) throw macError(410, 'session_closed', 'Capture is closed')
+    // An existing non-Mac session may not acquire new source metadata.
+    if (sessions.has(req.body.captureId) && !readMacRecording(req.body.captureId)) throw macError(409, 'mac_capture_conflict', 'Capture ID already exists')
+    lease = acquireRecordingChunkLease({ allowDuringDrain: sessions.has(req.body.captureId) })
+    const { recording, replayed } = registerMacRecording(req.body)
+    const session = getSession(recording.sessionId)
+    if (!session.chunks.some(Boolean)) session.startTime = recording.startTime
+    persistSessionRequired(recording.sessionId)
+    res.json({ ...MAC_AUDIO, ...recording, replayed, serverInstanceId })
+  } catch (error) { sendStreamError(res, error) } finally { lease?.release() }
+})
+const macChunkLocks = new Set<string>()
+transcribeStreamRouter.post('/mac-recording/:captureId/chunks', async (req, res) => {
+  let lease: MaintenanceWorkLease | undefined
+  let locked = false
+  const id = String(req.params.captureId ?? '')
+  try {
+    requireMacPin(req)
+    const recording = requireMacRecording(id)
+    if (isSessionDeleted(id)) throw macError(410, 'session_closed', 'Capture is closed')
+    const chunkIndex = Number(req.query.chunkIndex)
+    validateChunkIndex(chunkIndex)
+    const elapsed = Number(req.query.elapsed)
+    if (!Number.isFinite(elapsed) || elapsed < 0 || elapsed > 24 * 60 * 60 * 1000) throw macError(400, 'invalid_elapsed', 'Valid capture elapsed time required')
+    if (macChunkLocks.has(id)) throw macError(409, 'mac_chunk_in_progress', 'Another chunk is processing; retry the same chunk after its receipt')
+    macChunkLocks.add(id); locked = true
+    lease = acquireRecordingChunkLease({ allowDuringDrain: sessions.has(id) })
+    const audio = await readBoundedRawBody(req, MAC_AUDIO.maxChunkBytes)
+    const duration = validateMacWav(audio)
+    if (elapsed < duration) throw macError(400, 'invalid_elapsed', 'Chunk end precedes its duration')
+    const session = getSession(id)
+    persistMacChunkReceipt(id, { chunkIndex, endElapsed: elapsed, durationMs: duration, audioSha256: sha256Hex(audio) })
+    const prior = session.chunks[chunkIndex] ?? session.emptyCompletions?.[String(chunkIndex)]
+    if (prior?.audioSha256 && prior.audioSha256 !== sha256Hex(audio)) throw macError(409, 'mac_chunk_conflict', 'Chunk index already holds different audio')
+    res.json(await processStreamChunk({ sessionId: id, chunkIndex, clientSpeaker: 'Ext', audioBuffer: audio, clientElapsed: elapsed - duration, audioDurationMs: duration, startTimeOverride: recording.startTime }))
+  } catch (error) { sendStreamError(res, error) } finally { if (locked) macChunkLocks.delete(id); lease?.release() }
+})
+
 transcribeStreamRouter.post('/transcribe-stream', async (req, res) => {
   let maintenanceLease: MaintenanceWorkLease | undefined
   try {
     // Reject a wrong Mac before consuming or persisting any upload bytes.
     assertPinnedServerIdentity(req.get('X-COS-Server-Instance'), req.query.serverInstanceId)
     const sessionId = (req.query.sessionId as string) || `g2_${Date.now()}`
+    if (sessionId.startsWith('mac_')) throw macError(400, 'mac_transport_required', 'Use the Mac recording transport for Mac captures')
     maintenanceLease = acquireRecordingChunkLease({
       allowDuringDrain: sessions.has(sessionId),
     })

package/server/routes/transcribe.ts

@@ -20,6 +20,11 @@ import {
 } from '../lib/maintenance-lifecycle.js'
 
 export const transcribeRouter = Router()
+// A Mac client checks this before transferring microphone audio. Older servers must not
+// silently ignore the requested engine and use a configured cloud fallback.
+transcribeRouter.get('/transcribe/capabilities', (_req, res) => {
+  res.json({ desktopDictationVersion: 1, requiredEngines: ['parakeet-ultra'], localOnly: true, dictionary: 'cos' })
+})
 
 function resolveMode(req: { body?: { mode?: string }; query?: { mode?: string | string[] } }) {
   return resolveTranscribeMode(
@@ -34,8 +39,18 @@ transcribeRouter.post('/transcribe', async (req, res) => {
   try {
     maintenanceLease = acquireMaintenanceWork('one_shot_transcription')
     const chunks: Buffer[] = []
+    const requiredEngine = req.query.requiredEngine
+    if (requiredEngine !== undefined && requiredEngine !== 'parakeet-ultra') {
+      return res.status(400).json({ error: 'invalid_required_engine' })
+    }
+    let size = 0
     for await (const chunk of req) {
-      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+      const part = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+      size += part.length
+      if (size > (requiredEngine ? 3_840_044 : 25 * 1024 * 1024)) {
+        return res.status(413).json({ error: 'audio_too_large' })
+      }
+      chunks.push(part)
     }
     const audioBuffer = Buffer.concat(chunks)
 
@@ -43,7 +58,10 @@ transcribeRouter.post('/transcribe', async (req, res) => {
       return res.status(400).json({ error: 'audio too short' })
     }
 
-    const result = await transcribeAudioBuffer(audioBuffer, { mode: resolveMode(req) })
+    const result = await transcribeAudioBuffer(audioBuffer, {
+      mode: resolveMode(req),
+      ...(requiredEngine ? { requiredEngine, policy: 'local-only' as const, affectsCircuit: false } : {}),
+    })
     console.log(`[perf] /transcribe: ${result.elapsedMs.toFixed(1)}ms | mode=${result.mode} | ${result.backend}${result.engineReason ? ` (${result.engineReason})` : ''} | ${result.audioBytes}b | ${result.text.length} chars`)
     res.json({
       text: result.text,

package/server/routes/work-chat.ts

@@ -1,11 +1,12 @@
 import { Router } from 'express'
-import { parseWorkChatInput, parseWorkChatAnswer, WORK_CHAT_SYSTEM } from '../lib/work-chat.js'
+import { parseWorkChatInput, parseWorkChatAnswer, WORK_CHAT_SYSTEM, WORK_CHAT_PLANNERS, WORK_CHAT_EFFORTS, WORK_CHAT_DEFAULT_EFFORT,
+  WORK_CHAT_OUTPUT_TOKENS, WORK_CHAT_THINKING_TOKENS, WORK_CHAT_DEADLINE_MS, WORK_CHAT_MAX_ASSIGNMENTS, WORK_CHAT_MAX_ITEMS } from '../lib/work-chat.js'
 import { runMeetingAskModel } from '../lib/meeting-ask-runner.js'
 import { acquireMaintenanceWork, maintenanceAdmissionsOpen } from '../lib/maintenance-lifecycle.js'
 export function createWorkChatRouter(deps = {run:runMeetingAskModel, admissions:maintenanceAdmissionsOpen, lease:()=>acquireMaintenanceWork('work_chat')}) {
   const router=Router(); let running=false
   router.use('/work/chat', (_req,res,next)=>{res.set('Cache-Control','private, no-store');next()})
-  router.get('/work/chat',(_req,res)=>res.json({protocolVersion:1,planner:'sonnet',automaticProviders:['claude','codex'],maxAssignments:3}))
+  router.get('/work/chat',(_req,res)=>res.json({protocolVersion:1,planner:'sonnet',planners:WORK_CHAT_PLANNERS,effort:WORK_CHAT_DEFAULT_EFFORT,efforts:WORK_CHAT_EFFORTS,automaticProviders:['claude','codex'],maxAssignments:WORK_CHAT_MAX_ASSIGNMENTS,contextItems:true,maxContextItems:WORK_CHAT_MAX_ITEMS}))
   router.post('/work/chat',async(req,res)=>{
     let input
     try { if (Buffer.byteLength(JSON.stringify(req.body))>100000) throw new Error(); input=parseWorkChatInput(req.body) }
@@ -17,9 +18,12 @@ export function createWorkChatRouter(deps = {run:runMeetingAskModel, admissions:
     res.on('close',abort)
     try {
       lease=deps.lease()
-      const out=await deps.run({engine:'claude',model:'sonnet'},JSON.stringify(input),WORK_CHAT_SYSTEM,controller.signal,undefined,{caller:'work_chat_'+input.mode})
+      // Planner and effort select how the model runs; they are not evidence, so they never reach the prompt.
+      const {planner='sonnet',effort=WORK_CHAT_DEFAULT_EFFORT,...prompt}=input
+      const out=await deps.run({engine:'claude',model:planner},JSON.stringify(prompt),WORK_CHAT_SYSTEM,controller.signal,undefined,{caller:'work_chat_'+input.mode},
+        {effort,maxOutputTokens:WORK_CHAT_OUTPUT_TOKENS,maxThinkingTokens:WORK_CHAT_THINKING_TOKENS[effort],deadlineMs:WORK_CHAT_DEADLINE_MS})
       const answer=parseWorkChatAnswer(out.text,input)
-      if (!res.destroyed) return res.json({protocolVersion:1,...answer,model:out.model??'claude:sonnet (requested)'})
+      if (!res.destroyed) return res.json({protocolVersion:1,...answer,model:out.model??`claude:${planner} (requested)`,effort})
     } catch {if(!res.destroyed) return res.status(503).json({error:'work_chat_unavailable',message:'No assignment was dispatched. Your saved request can be retried.'})}
     finally {res.off('close',abort);lease?.release();running=false}
   })

package/server/routes/work-search.ts

@@ -8,7 +8,7 @@
 import { Router } from 'express'
 import { listBoard } from '../lib/task-store.js'
 import { isSafeDomainName } from '../lib/domains.js'
-import { queryLength, sharedWorkSearchJevClient, selectCandidates, unavailable, workSearchEnabled, WorkSearcher, WORK_SEARCH_LIMITS,
+import { queryLength, sharedWorkSearchDecisionClient, selectCandidates, unavailable, workSearchEnabled, WorkSearcher, WORK_SEARCH_LIMITS,
   WORK_SEARCH_SCOPES, type WorkSearchResult, type WorkSearchScope } from '../lib/work-search.js'
 
 export interface WorkSearchRouteDependencies {
@@ -23,7 +23,7 @@ const TASK_ID = /^[a-f0-9]{12}$/
 export const WORK_SEARCH_LOG_EVERY_MS = 10 * 60_000
 
 export function createWorkSearchRouter(overrides: Partial<WorkSearchRouteDependencies> = {}): Router {
-  const deps: WorkSearchRouteDependencies = { searcher: new WorkSearcher(sharedWorkSearchJevClient()), list: () => listBoard(),
+  const deps: WorkSearchRouteDependencies = { searcher: new WorkSearcher(sharedWorkSearchDecisionClient()), list: () => listBoard(),
     enabled: () => workSearchEnabled(), ...overrides }
   const router = Router()
   const logged = new Map<string, number>()