@gotcos/glasses-server 6.15.3 → 6.16.0

package/server/lib/codex-bridge.ts

@@ -61,6 +61,7 @@ import {
   type MediaAttachmentRef,
 } from '../../shared/media-attachment.js'
 import { terminalProviderAuthFailure } from './provider-terminal-error.js'
+import { terminateProviderProcess } from './provider-process-lifecycle.js'
 
 const INACTIVITY_MS = 180_000
 const WALL_MAX_MS = 900_000
@@ -361,12 +362,14 @@ export async function callCodexStreaming(
     stdio: ['pipe', 'pipe', 'pipe'],
     env,
     cwd: codexCwd,
+    detached: true,
   })
 
   let fullText = ''
   let stderr = ''
   let buffer = ''
   let finalized = false
+  let terminationRequested = false
   let terminalTextError: string | null = null
   let lastActivity = Date.now()
   const emittedBlocks = new Set<string>()
@@ -571,10 +574,43 @@ export async function callCodexStreaming(
     }
   }
 
-  function handleAbort() {
+  function abandonLostDurableOwnership(message: string) {
     if (finalized) return
-    proc.kill('SIGTERM')
-    finalizeError('codex-bridge: client disconnected before Codex completed.', null, 'client_disconnected')
+    finalized = true
+    cleanup()
+    cleanupImages()
+    outputImagePublisher?.cleanup()
+    removeExchange(sid, pendingUserExchange)
+    clearEngineSessionBestEffort('provider_ownership_lost')
+    finishRunBestEffort({
+      status: 'failed',
+      startedAtMs: startTime,
+      error: message,
+      exitCode: null,
+    })
+  }
+
+  async function terminateForTerminal(
+    reason: string,
+    onClosed: (result: Awaited<ReturnType<typeof terminateProviderProcess>>) => void | Promise<void>,
+  ) {
+    if (finalized || terminationRequested) return
+    terminationRequested = true
+    cleanup()
+    const result = await terminateProviderProcess(proc)
+    if (!result.closed) {
+      console.error(`[codex-bridge] provider did not close after SIGKILL (${reason}); retaining lifecycle ownership`)
+      return
+    }
+    await onClosed(result)
+  }
+
+  function handleAbort() {
+    void terminateForTerminal('client disconnect', result => finalizeError(
+        'codex-bridge: client disconnected before Codex completed.',
+        result.code,
+        'client_disconnected',
+      ))
   }
 
   const heartbeat = setInterval(() => {
@@ -583,28 +619,30 @@ export async function callCodexStreaming(
   }, HEARTBEAT_INTERVAL_MS)
 
   let inactivityTimer = setTimeout(() => {
-    proc.kill('SIGTERM')
     const elapsed = Math.round((Date.now() - startTime) / 1000)
-    finalizeError(`No output for ${INACTIVITY_MS / 1000}s (${elapsed}s total). Codex process killed.`)
+    void terminateForTerminal('inactivity timeout', result => finalizeError(
+      `No output for ${INACTIVITY_MS / 1000}s (${elapsed}s total). Codex process killed.`,
+      result.code,
+    ))
   }, INACTIVITY_MS)
 
   function resetInactivity() {
     lastActivity = Date.now()
     clearTimeout(inactivityTimer)
     inactivityTimer = setTimeout(() => {
-      proc.kill('SIGTERM')
       const elapsed = Math.round((Date.now() - startTime) / 1000)
-      finalizeError(`No output for ${INACTIVITY_MS / 1000}s (${elapsed}s total). Codex process killed.`)
+      void terminateForTerminal('inactivity timeout', result => finalizeError(
+        `No output for ${INACTIVITY_MS / 1000}s (${elapsed}s total). Codex process killed.`,
+        result.code,
+      ))
     }, INACTIVITY_MS)
   }
 
   const wallTimer = setTimeout(() => {
-    proc.kill('SIGTERM')
-    if (fullText) {
-      void finalize(fullText)
-    } else {
-      finalizeError(`Wall clock limit reached (${WALL_MAX_MS / 1000}s). Codex process killed.`)
-    }
+    void terminateForTerminal('wall timeout', result => {
+      if (fullText) return finalize(fullText)
+      return finalizeError(`Wall clock limit reached (${WALL_MAX_MS / 1000}s). Codex process killed.`, result.code)
+    })
   }, WALL_MAX_MS)
 
   function handleEvent(event: any) {
@@ -677,6 +715,7 @@ export async function callCodexStreaming(
   })
 
   proc.on('close', (code) => {
+    if (terminationRequested) return
     if (buffer.trim()) {
       try { handleEvent(JSON.parse(buffer.trim())) } catch { /* ignore */ }
     }
@@ -696,9 +735,11 @@ export async function callCodexStreaming(
   })
 
   proc.on('error', (err) => {
+    if (terminationRequested) return
     finalizeError(`codex-bridge: ${err.message}`)
   })
   proc.stdin.on('error', (err) => {
+    if (terminationRequested) return
     finalizeError(`codex-bridge: stdin failed — ${err.message}`)
   })
 
@@ -719,18 +760,8 @@ export async function callCodexStreaming(
       generation: options?.generation,
     })
     if (providerOwned === false) {
-      proc.kill('SIGTERM')
-      finalized = true
-      cleanup()
-      cleanupImages()
-      outputImagePublisher?.cleanup()
-      removeExchange(sid, pendingUserExchange)
-      clearEngineSessionBestEffort('provider_ownership_lost')
-      finishRunBestEffort({
-        status: 'failed',
-        startedAtMs: startTime,
-        error: 'codex-bridge: durable provider ownership was lost.',
-        exitCode: null,
+      await terminateForTerminal('provider ownership lost', () => {
+        abandonLostDurableOwnership('codex-bridge: durable provider ownership was lost.')
       })
       return sid
     }
@@ -740,8 +771,10 @@ export async function callCodexStreaming(
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err)
     if (!finalized) {
-      proc.kill('SIGTERM')
-      await finalizeError(`codex-bridge: provider start failed — ${message}`)
+      await terminateForTerminal('provider start failure', result => finalizeError(
+        `codex-bridge: provider start failed — ${message}`,
+        result.code,
+      ))
     }
   }
 

package/server/lib/maintenance-lifecycle.ts

@@ -310,6 +310,7 @@ export class MaintenanceLifecycle {
   private blockedGateReason: BlockedGateReason | null = null
   private blockedGateVersion: number | null = null
   private readonly work = new Map<string, WorkEntry>()
+  private readonly admissionsOpenListeners = new Set<() => void>()
 
   constructor(options: MaintenanceLifecycleOptions = {}) {
     this.path = options.path ?? process.env.COS_MAINTENANCE_GATE_PATH?.trim() ?? DEFAULT_GATE_PATH
@@ -356,11 +357,44 @@ export class MaintenanceLifecycle {
     try {
       removeGate(this.path)
       this.gate = null
+      this.notifyAdmissionsOpen()
     } catch {
       this.blockedGateReason = 'invalid_schema'
     }
   }
 
+  /**
+   * Register deferred runtime initialization that is safe only after the
+   * durable maintenance gate opens. Delivery is asynchronous so the release
+   * response is never held open by model/cache initialization. The callback is
+   * also delivered for an already-open lifecycle, closing the boot/release
+   * race without forcing index.ts to poll.
+   */
+  onAdmissionsOpen(listener: () => void): () => void {
+    this.expireSameBootGateIfPermitted()
+    this.admissionsOpenListeners.add(listener)
+    if (!this.gate && !this.blockedGateReason) this.deliverAdmissionsOpen(listener)
+    return () => { this.admissionsOpenListeners.delete(listener) }
+  }
+
+  private deliverAdmissionsOpen(listener: () => void): void {
+    queueMicrotask(() => {
+      if (!this.admissionsOpenListeners.has(listener)) return
+      // State may have closed again between scheduling and microtask delivery.
+      // Never start admitted runtime from a stale accepting notification.
+      this.expireSameBootGateIfPermitted()
+      if (this.gate || this.blockedGateReason) return
+      try { listener() } catch (error) {
+        console.error('[maintenance] admissions-open listener failed:', error)
+      }
+    })
+  }
+
+  private notifyAdmissionsOpen(): void {
+    if (this.gate || this.blockedGateReason) return
+    for (const listener of this.admissionsOpenListeners) this.deliverAdmissionsOpen(listener)
+  }
+
   private credentialsMatch(credentials: MaintenanceOperationCredentials): {
     leaseMatches: boolean
     operationMatches: boolean
@@ -615,6 +649,7 @@ export class MaintenanceLifecycle {
       )
     }
     this.gate = null
+    this.notifyAdmissionsOpen()
   }
 
   cancelDrain(identity: MaintenanceOperationIdentity, credentials: MaintenanceOperationCredentials): void {
@@ -641,6 +676,7 @@ export class MaintenanceLifecycle {
       )
     }
     this.gate = null
+    this.notifyAdmissionsOpen()
   }
 
   snapshot(credentials: MaintenanceOperationCredentials = {}, extraActiveByKind: Record<string, number> = {}) {
@@ -741,6 +777,10 @@ export function maintenanceAdmissionsOpen(): boolean {
   return maintenanceLifecycle.snapshot().admissionsOpen
 }
 
+export function onMaintenanceAdmissionsOpen(listener: () => void): () => void {
+  return maintenanceLifecycle.onAdmissionsOpen(listener)
+}
+
 export function maintenanceErrorPayload(error: MaintenanceLifecycleError) {
   return {
     error: error.code,

package/server/lib/prompt-draft-store.ts

@@ -20,6 +20,7 @@ export interface PromptDraftTranscriptRecord {
   actualQuality: 'hq' | 'fast' | 'cloud'
   backend: string
   degraded: boolean
+  degradationReason?: string
   acceptedDegraded?: boolean
 }
 

package/server/lib/provider-process-lifecycle.ts

@@ -0,0 +1,139 @@
+import type { ChildProcess } from 'node:child_process'
+
+export interface ProviderTerminationResult {
+  closed: boolean
+  escalated: boolean
+  code: number | null
+  signal: NodeJS.Signals | null
+}
+
+interface ProviderTerminationOptions {
+  termGraceMs?: number
+  killWaitMs?: number
+}
+
+/** A detached provider's process group can outlive its CLI leader. ESRCH is
+ * the only proof that no member remains; EPERM and unknown probe failures are
+ * treated as alive so lifecycle ownership fails closed. */
+function providerGroupAlive(pid: number | undefined): boolean {
+  if (!pid || process.platform === 'win32') return false
+  try {
+    process.kill(-pid, 0)
+    return true
+  } catch (error: any) {
+    return error?.code !== 'ESRCH'
+  }
+}
+
+/** Signal the detached provider process group so tool subprocesses cannot be
+ * orphaned behind the CLI wrapper. Direct-child signaling is a safe fallback
+ * for a process that failed before its process group was established. */
+function signalProviderTree(proc: ChildProcess, signal: NodeJS.Signals): void {
+  const pid = proc.pid
+  if (pid && process.platform !== 'win32') {
+    try {
+      process.kill(-pid, signal)
+      return
+    } catch {
+      // Fall through to the direct child. close/error remains authoritative.
+    }
+  }
+  try { proc.kill(signal) } catch { /* process already terminal */ }
+}
+
+/**
+ * Terminate provider work and resolve only after Node observes leader close
+ * and the detached process group no longer exists.
+ * A caller must retain its maintenance lease when `closed` is false: releasing
+ * without a close event could let Control restart while a tool is still alive.
+ */
+export function terminateProviderProcess(
+  proc: ChildProcess,
+  options: ProviderTerminationOptions = {},
+): Promise<ProviderTerminationResult> {
+  const termGraceMs = Math.max(10, options.termGraceMs ?? 2_000)
+  const killWaitMs = Math.max(10, options.killWaitMs ?? 2_000)
+  const groupPid = proc.pid
+  const initiallyClosed = proc.exitCode !== null || proc.signalCode !== null
+
+  if (initiallyClosed && !providerGroupAlive(groupPid)) {
+    return Promise.resolve({
+      closed: true,
+      escalated: false,
+      code: proc.exitCode,
+      signal: proc.signalCode,
+    })
+  }
+
+  return new Promise(resolve => {
+    let settled = false
+    let escalated = false
+    let leaderClosed = initiallyClosed
+    let leaderCode = proc.exitCode
+    let leaderSignal = proc.signalCode
+    let escalationTimer: ReturnType<typeof setTimeout> | undefined
+    let terminalTimer: ReturnType<typeof setTimeout> | undefined
+
+    const finish = (result: ProviderTerminationResult) => {
+      if (settled) return
+      settled = true
+      if (escalationTimer) clearTimeout(escalationTimer)
+      if (terminalTimer) clearTimeout(terminalTimer)
+      proc.removeListener('close', onClose)
+      proc.removeListener('error', onError)
+      resolve(result)
+    }
+    const finishIfTreeClosed = (): boolean => {
+      if (!leaderClosed || providerGroupAlive(groupPid)) return false
+      finish({
+        closed: true,
+        escalated,
+        code: leaderCode,
+        signal: leaderSignal,
+      })
+      return true
+    }
+    const onClose = (code: number | null, signal: NodeJS.Signals | null) => {
+      leaderClosed = true
+      leaderCode = code
+      leaderSignal = signal
+      finishIfTreeClosed()
+    }
+    const onError = () => {
+      // A spawn failure has no live process to retain. Running children still
+      // produce close after an error, so wait for that authoritative event.
+      if (!proc.pid) {
+        leaderClosed = true
+        leaderCode = null
+        leaderSignal = null
+        finishIfTreeClosed()
+      }
+    }
+
+    proc.once('close', onClose)
+    proc.once('error', onError)
+    try { proc.stdin?.destroy() } catch { /* best effort */ }
+    signalProviderTree(proc, 'SIGTERM')
+
+    escalationTimer = setTimeout(() => {
+      if (finishIfTreeClosed()) return
+      escalated = true
+      signalProviderTree(proc, 'SIGKILL')
+      const deadline = Date.now() + killWaitMs
+      const pollForTreeExit = () => {
+        if (finishIfTreeClosed()) return
+        if (Date.now() < deadline) {
+          terminalTimer = setTimeout(pollForTreeExit, 25)
+          return
+        }
+        finish({
+          closed: false,
+          escalated: true,
+          code: leaderCode,
+          signal: leaderSignal,
+        })
+      }
+      terminalTimer = setTimeout(pollForTreeExit, 25)
+    }, termGraceMs)
+  })
+}

package/server/lib/provider-proof.ts

@@ -1,5 +1,6 @@
 import { spawn } from 'node:child_process'
 import { cosBrainDir } from './launch-dir.js'
+import { terminateProviderProcess } from './provider-process-lifecycle.js'
 
 export type ProofProvider = 'claude' | 'codex'
 
@@ -21,6 +22,7 @@ interface ProcessResult {
   stdout: string
   stderr: string
   timedOut: boolean
+  aborted: boolean
 }
 
 function runBounded(
@@ -28,15 +30,20 @@ function runBounded(
   args: string[],
   input: string,
   timeoutMs = 120_000,
+  signal?: AbortSignal,
 ): Promise<ProcessResult> {
   return new Promise((resolvePromise) => {
+    if (signal?.aborted) {
+      resolvePromise({ code: null, stdout: '', stderr: '', timedOut: false, aborted: true })
+      return
+    }
     const env = { ...process.env }
     delete env.CLAUDECODE
     const child = spawn(command, args, {
       cwd: cosBrainDir() ?? process.cwd(),
       env,
       stdio: ['pipe', 'pipe', 'pipe'],
-      detached: false,
+      detached: true,
     })
     let stdout = ''
     let stderr = ''
@@ -48,15 +55,25 @@ function runBounded(
       if (settled) return
       settled = true
       clearTimeout(timer)
+      signal?.removeEventListener('abort', abort)
       resolvePromise(result)
     }
     const timer = setTimeout(() => {
-      try { child.kill('SIGKILL') } catch { /* already exited */ }
-      finish({ code: null, stdout, stderr, timedOut: true })
+      void terminateProviderProcess(child, { termGraceMs: 50 }).then(result => {
+        if (result.closed) finish({ code: result.code, stdout, stderr, timedOut: true, aborted: false })
+        else console.error('[provider-proof] timed-out provider did not close after SIGKILL; retaining request ownership')
+      })
     }, timeoutMs)
     timer.unref?.()
-    child.once('error', err => finish({ code: null, stdout, stderr: err.message, timedOut: false }))
-    child.once('close', code => finish({ code, stdout, stderr, timedOut: false }))
+    const abort = () => {
+      void terminateProviderProcess(child).then(result => {
+        if (result.closed) finish({ code: result.code, stdout, stderr, timedOut: false, aborted: true })
+        else console.error('[provider-proof] canceled provider did not close after SIGKILL; retaining request ownership')
+      })
+    }
+    child.once('error', err => finish({ code: null, stdout, stderr: err.message, timedOut: false, aborted: false }))
+    child.once('close', code => finish({ code, stdout, stderr, timedOut: false, aborted: false }))
+    signal?.addEventListener('abort', abort, { once: true })
     child.stdin.on('error', () => { /* close/error is authoritative */ })
     child.stdin.end(input)
   })
@@ -94,12 +111,13 @@ export function codexProofText(stdout: string): string {
 }
 
 function safeProofError(result: ProcessResult): string {
+  if (result.aborted) return 'provider proof canceled'
   if (result.timedOut) return 'provider proof timed out'
   if (result.code !== 0) return `provider process exited ${result.code ?? 'before launch'}`
   return 'provider returned no valid proof response'
 }
 
-async function executeProof(provider: ProofProvider): Promise<ProviderProofResult> {
+async function executeProof(provider: ProofProvider, signal?: AbortSignal): Promise<ProviderProofResult> {
   const started = Date.now()
   const result = provider === 'claude'
     ? await runBounded('claude', [
@@ -110,7 +128,7 @@ async function executeProof(provider: ProofProvider): Promise<ProviderProofResul
       '--allowedTools', '',
       '--system-prompt', PROOF_PROMPT,
       PROOF_PROMPT,
-    ], '')
+    ], '', 120_000, signal)
     : await runBounded('codex', [
       'exec',
       '--sandbox', 'read-only',
@@ -119,7 +137,7 @@ async function executeProof(provider: ProofProvider): Promise<ProviderProofResul
       '--cd', cosBrainDir() ?? process.cwd(),
       '--ephemeral',
       '-',
-    ], PROOF_PROMPT)
+    ], PROOF_PROMPT, 120_000, signal)
   const text = provider === 'claude'
     ? claudeProofText(result.stdout)
     : codexProofText(result.stdout)
@@ -134,12 +152,12 @@ async function executeProof(provider: ProofProvider): Promise<ProviderProofResul
 }
 
 /** Actual no-tool model turn, cached only after success for this server boot. */
-export async function runProviderProof(provider: ProofProvider): Promise<ProviderProofResult> {
+export async function runProviderProof(provider: ProofProvider, signal?: AbortSignal): Promise<ProviderProofResult> {
   const cached = successCache.get(provider)
   if (cached) return { ...cached, cached: true }
   const existing = inFlight.get(provider)
   if (existing) return existing
-  const operation = executeProof(provider).then(result => {
+  const operation = executeProof(provider, signal).then(result => {
     if (result.ok) successCache.set(provider, result)
     return result
   }).finally(() => {

package/server/lib/transcribe-audio.ts

@@ -34,6 +34,7 @@ export interface TranscribeAudioResult {
   requestedMode: TranscribeMode
   actualQuality: 'hq' | 'fast' | 'cloud'
   degraded: boolean
+  degradationReason?: string
   elapsedMs: number
   audioBytes: number
 }
@@ -62,6 +63,12 @@ export class NoSpeechDetectedError extends Error {
 // ceiling (anything longer is a dictation, not a query — use meetings instead).
 const HQ_MAX_SECONDS = 60
 
+/** Short interactive clips use light enhance (highpass only). Override via env. */
+function hqEnhanceLightMaxSeconds(): number {
+  const value = Number.parseInt(process.env.COS_HQ_ENHANCE_LIGHT_MAX_SEC || '15', 10)
+  return Number.isFinite(value) && value >= 0 ? value : 15
+}
+
 function unavailableAfterLocalFailure(): TranscriptionUnavailableError | null {
   const fallback = getTranscriptionPolicySnapshot()
   if (fallback.openaiFallbackReady) return null
@@ -154,17 +161,26 @@ export async function transcribeAudioBuffer(
   let text: string
   let backend: string
   let actualQuality: 'hq' | 'fast' | 'cloud'
+  let degradationReason: string | undefined = effectiveMode !== requestedMode ? 'audio_too_long' : undefined
   const tStart = performance.now()
 
   if (effectiveMode === 'hq') {
     try {
-      const enhanced = await enhanceAudio(audioBuffer)
+      const lightMax = hqEnhanceLightMaxSeconds()
+      const enhanceProfile = audioSeconds < lightMax ? 'light' as const : 'full' as const
+      const enhanced = await enhanceAudio(audioBuffer, { profile: enhanceProfile })
       const result = await transcribeHighQuality(enhanced)
       text = result.text
-      backend = 'hq-large-v3'
-      actualQuality = 'hq'
+      actualQuality = result.actualQuality
+      if (result.actualQuality === 'hq') {
+        backend = enhanceProfile === 'light' ? 'hq-large-v3-light' : 'hq-large-v3'
+      } else {
+        backend = result.backend === 'whisper-cli' ? 'fast-cli-turbo' : 'fast-local-server'
+        degradationReason = result.degradationReason ?? 'hq_unavailable'
+      }
     } catch (hqErr: any) {
       console.warn(`[transcribe] HQ path failed, falling back to fast: ${hqErr.message}`)
+      degradationReason = 'hq_decode_failed'
       try {
         const result = await transcribeLocal(audioBuffer)
         text = result.text
@@ -260,6 +276,7 @@ export async function transcribeAudioBuffer(
     requestedMode,
     actualQuality,
     degraded: requestedMode === 'hq' && actualQuality !== 'hq',
+    ...(degradationReason ? { degradationReason } : {}),
     elapsedMs,
     audioBytes: audioBuffer.length,
   }