@gotcos/glasses-server 6.12.7 → 6.13.0

package/server/lib/managed-runtime.ts

@@ -0,0 +1,44 @@
+export const MANAGED_RUNTIME_CONTRACT_VERSION = 2
+
+export interface ManagedRuntimeCapability {
+  status: boolean
+  restartWhisper: boolean
+  restartServer: boolean
+  maintenanceDrain: boolean
+  lifecycleProof: boolean
+  managed: boolean
+  contractVersion: number
+}
+
+export function isManagedRuntime(): boolean {
+  return process.env.COS_MANAGED === '1'
+}
+
+export function managedRuntimeCapability(): ManagedRuntimeCapability {
+  const managed = isManagedRuntime()
+  return {
+    status: managed,
+    // Whisper lifecycle is private to the local controller. It is never
+    // exposed as a network-reachable mutation capability.
+    restartWhisper: false,
+    // Server restart is performed by the trusted local helper through launchd,
+    // never by an HTTP endpoint. This flag tells clients that managed recovery
+    // exists without widening the network attack surface.
+    restartServer: managed,
+    maintenanceDrain: managed,
+    lifecycleProof: managed,
+    managed,
+    contractVersion: MANAGED_RUNTIME_CONTRACT_VERSION,
+  }
+}
+
+export function managedServerVersion(): string | null {
+  const value = process.env.COS_SERVER_VERSION?.trim()
+  return value || null
+}
+
+/** Deployment generation expected by the trusted local controller. */
+export function getServerGenerationId(): string | null {
+  const explicit = process.env.COS_SERVER_GENERATION_ID?.trim()
+  return explicit && /^[A-Za-z0-9._:-]{1,160}$/.test(explicit) ? explicit : null
+}

package/server/lib/query-job-coordinator.ts

@@ -16,6 +16,7 @@ import {
   type QueryJobSnapshot,
   type QueryJobStoreHealth,
 } from './query-job-types.js'
+import type { MaintenanceWorkLease } from './maintenance-lifecycle.js'
 
 const CLIENT_JOB_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
 
@@ -68,6 +69,9 @@ export interface QueryJobCoordinatorOptions {
   /** Idempotent projection of a terminal journal record into canonical
    * conversation state. The journal remains authoritative if this fails. */
   projectTerminal?: (job: QueryJobSnapshot, request: QueryJobRequest) => void | Promise<void>
+  /** Acquired synchronously before serialized admission and retained through
+   * the provider terminal, so maintenance proof covers queued transitions. */
+  acquireMaintenanceWork?: () => MaintenanceWorkLease
 }
 
 interface ActiveRun {
@@ -76,6 +80,7 @@ interface ActiveRun {
   request: QueryJobRequest
   controller: AbortController
   release?: () => void
+  maintenanceLease?: MaintenanceWorkLease
   released: boolean
   callbackTail: Promise<void>
   partialText: string
@@ -114,6 +119,7 @@ export class QueryJobCoordinator {
   private readonly partialFlushChars: number
   private readonly providerTimeoutMs: number
   private readonly active = new Map<string, ActiveRun>()
+  private readonly admittedMaintenance = new Map<string, MaintenanceWorkLease>()
   private admissionTail: Promise<void> = Promise.resolve()
   private shuttingDown = false
   private callbackPersistenceFailures = 0
@@ -148,6 +154,7 @@ export class QueryJobCoordinator {
    * two simultaneous retries without a sessionId cannot allocate two sessions
    * and conflict solely because the client had not learned the first one. */
   submit(raw: unknown): Promise<QueryJobAdmissionResult> {
+    const maintenanceLease = this.options.acquireMaintenanceWork?.()
     let resolve!: (value: QueryJobAdmissionResult) => void
     let reject!: (reason?: unknown) => void
     const result = new Promise<QueryJobAdmissionResult>((res, rej) => {
@@ -160,8 +167,14 @@ export class QueryJobCoordinator {
         const normalized = await this.assignSession(raw)
         const admission = await this.store.admit(normalized)
         resolve(admission)
-        if (admission.created) queueMicrotask(() => { void this.execute(admission.job.jobId) })
+        if (admission.created) {
+          if (maintenanceLease) this.admittedMaintenance.set(admission.job.jobId, maintenanceLease)
+          queueMicrotask(() => { void this.execute(admission.job.jobId) })
+        } else {
+          maintenanceLease?.release()
+        }
       } catch (error) {
+        maintenanceLease?.release()
         reject(error)
       }
     })
@@ -186,14 +199,30 @@ export class QueryJobCoordinator {
   }
 
   private async execute(jobId: string): Promise<void> {
+    const maintenanceLease = this.admittedMaintenance.get(jobId)
+    this.admittedMaintenance.delete(jobId)
     const starting = await this.store.markStarting(jobId).catch(() => null)
-    if (!starting?.applied) return
-    const execution = await this.store.getExecution(jobId)
+    if (!starting?.applied) {
+      maintenanceLease?.release()
+      return
+    }
+    maintenanceLease?.setPhase('active')
+    let execution
+    try {
+      execution = await this.store.getExecution(jobId)
+    } catch {
+      maintenanceLease?.release()
+      return
+    }
     let release: (() => void) | undefined
     try {
       release = await this.options.acquireSessionLock?.(execution.request.sessionId)
     } catch (error) {
-      await this.store.fail(jobId, error)
+      try {
+        await this.store.fail(jobId, error)
+      } finally {
+        maintenanceLease?.release()
+      }
       return
     }
 
@@ -207,6 +236,7 @@ export class QueryJobCoordinator {
       request: execution.request,
       controller: new AbortController(),
       release,
+      maintenanceLease,
       released: false,
       callbackTail: Promise.resolve(),
       partialText: '',
@@ -536,6 +566,7 @@ export class QueryJobCoordinator {
     if (!active.released) {
       active.released = true
       active.release?.()
+      active.maintenanceLease?.release()
     }
   }
 
@@ -548,6 +579,7 @@ export class QueryJobCoordinator {
     if (!active.released) {
       active.released = true
       active.release?.()
+      active.maintenanceLease?.release()
     }
   }
 

package/server/lib/query-job-runtime.ts

@@ -26,6 +26,7 @@ import {
   type QueryJobRequest,
   type QueryJobSnapshot,
 } from './query-job-types.js'
+import { acquireMaintenanceWork } from './maintenance-lifecycle.js'
 
 const TOOL_STATUS_MESSAGES: Record<string, string> = {
   WebSearch: 'Searching web...',
@@ -161,6 +162,12 @@ const runner: QueryJobRunner = async ({ jobId, turnId, request, signal, callback
           resolvedAttachments.refs,
           metadata?.outputAttachments,
         )
+        // Acquire before the durable terminal callback can release the main
+        // query lease. Attachment association is a post-terminal write and
+        // must not create a zero-count maintenance proof gap.
+        const attachmentLease = resolvedAttachments.ids.length > 0
+          ? acquireMaintenanceWork('query_attachment_write', { allowDuringDrain: true })
+          : undefined
         const linkage = {
           provider: providerFor(model),
           resolvedModel: model,
@@ -171,34 +178,38 @@ const runner: QueryJobRunner = async ({ jobId, turnId, request, signal, callback
         } as const
         // Publish compatibility completion only after the durable terminal is
         // fsynced. Display subscribers can disappear without owning this job.
-        const terminalOwned = await callbacks.onDone({
-          text: fullText,
-          attachments,
-          outputImageStats: metadata?.outputImageStats,
-          ...linkage,
-        })
-        if (!terminalOwned) return
-        if (resolvedAttachments.ids.length > 0) {
-          await getMediaStore().associate(resolvedAttachments.ids, {
+        try {
+          const terminalOwned = await callbacks.onDone({
+            text: fullText,
+            attachments,
+            outputImageStats: metadata?.outputImageStats,
+            ...linkage,
+          })
+          if (!terminalOwned) return
+          if (resolvedAttachments.ids.length > 0) {
+            await getMediaStore().associate(resolvedAttachments.ids, {
+              sessionId: request.sessionId,
+              ...(request.globalMsgNum ? { globalMsgNum: request.globalMsgNum } : {}),
+            }).catch(error => console.error('[query-jobs] attachment association failed:', error))
+          }
+          const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
+          emitDisplay({ type: 'done', data: {
+            jobId,
+            clientJobId: request.clientJobId,
+            generation: request.generation,
+            turnId,
+            messageEra: request.messageEra,
+            globalMsgNum: request.globalMsgNum,
+            text: fullText,
             sessionId: request.sessionId,
-            ...(request.globalMsgNum ? { globalMsgNum: request.globalMsgNum } : {}),
-          }).catch(error => console.error('[query-jobs] attachment association failed:', error))
+            model,
+            cliSessionId,
+            ...runMetadata,
+            ...(attachments.length > 0 ? { attachments } : {}),
+          } })
+        } finally {
+          attachmentLease?.release()
         }
-        const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
-        emitDisplay({ type: 'done', data: {
-          jobId,
-          clientJobId: request.clientJobId,
-          generation: request.generation,
-          turnId,
-          messageEra: request.messageEra,
-          globalMsgNum: request.globalMsgNum,
-          text: fullText,
-          sessionId: request.sessionId,
-          model,
-          cliSessionId,
-          ...runMetadata,
-          ...(attachments.length > 0 ? { attachments } : {}),
-        } })
       },
       onError: async error => {
         const terminalOwned = await callbacks.onError(error)
@@ -239,6 +250,7 @@ export const queryJobStore = new QueryJobStore({
 export const queryJobCoordinator = new QueryJobCoordinator(queryJobStore, runner, {
   projectTerminal: projectPublicConversationTerminal,
   acquireSessionLock: acquireModelSessionRunLock,
+  acquireMaintenanceWork: () => acquireMaintenanceWork('durable_query', { phase: 'queued' }),
 })
 
 export function initQueryJobRuntime() {

package/server/routes/health.ts

@@ -21,6 +21,9 @@ import { durableQueryJobsCapability } from '../lib/query-job-feature.js'
 import { getQueryJobRuntimeHealth } from '../lib/query-job-runtime.js'
 import { getTranscriptionPolicySnapshot } from '../lib/transcription-policy.js'
 import { CLI_DEBUG_CAPABILITY } from '../lib/cli-debug-view.js'
+import { managedRuntimeCapability, managedServerVersion } from '../lib/managed-runtime.js'
+import { getServerGenerationId } from '../lib/managed-runtime.js'
+import { maintenanceLifecycle } from '../lib/maintenance-lifecycle.js'
 
 export const healthRouter = Router()
 
@@ -145,12 +148,8 @@ healthRouter.get('/health', async (_req, res) => {
   const durableJobs = durableQueryJobStatus()
   const localFirstMeetings = localFirstMeetingsCapability(getServerInstanceId())
   const transcription = getTranscriptionPolicySnapshot()
-  const recovery = {
-    status: false,
-    restartWhisper: false,
-    restartServer: false,
-    managed: false,
-  }
+  const recovery = managedRuntimeCapability()
+  const maintenance = maintenanceLifecycle.snapshot()
   const features = {
     claude: claudeAvailable,
     codex: codexAvailable,
@@ -180,6 +179,10 @@ healthRouter.get('/health', async (_req, res) => {
   const codex_models = getCodexModelCatalogSnapshot()
   res.json({
     ...checks,
+    server_version: managedServerVersion(),
+    server_instance_id: getServerInstanceId(),
+    boot_id: serverMetrics.bootId,
+    generation_id: getServerGenerationId(),
     features,
     voice,
     whisper_health,
@@ -188,6 +191,11 @@ healthRouter.get('/health', async (_req, res) => {
     capabilities: {
       transcription,
       recovery,
+      maintenance: {
+        state: maintenance.state,
+        admissionsOpen: maintenance.admissionsOpen,
+        carriedAcrossBoot: maintenance.operation?.carriedAcrossBoot ?? false,
+      },
       cliDebug: CLI_DEBUG_CAPABILITY,
       ...(localFirstMeetings ? { localFirstMeetings } : {}),
     },
@@ -220,12 +228,7 @@ healthRouter.get('/models', async (req, res) => {
       },
       transcription,
       cliDebug: CLI_DEBUG_CAPABILITY,
-      recovery: {
-        status: false,
-        restartWhisper: false,
-        restartServer: false,
-        managed: false,
-      },
+      recovery: managedRuntimeCapability(),
       ...(localFirstMeetings ? { localFirstMeetings } : {}),
     },
   })

package/server/routes/maintenance.ts

@@ -0,0 +1,160 @@
+import { Router, type Request, type Response } from 'express'
+import {
+  MaintenanceLifecycleError,
+  maintenanceErrorPayload,
+  maintenanceLifecycle,
+  type MaintenanceDrainRequest,
+  type MaintenanceOperationCredentials,
+  type MaintenanceOperationIdentity,
+  type MaintenanceOperationKind,
+  type MaintenanceOperationScope,
+  type MaintenancePostcondition,
+} from '../lib/maintenance-lifecycle.js'
+import { managedRuntimeCapability, managedServerVersion, getServerGenerationId } from '../lib/managed-runtime.js'
+import { getQueryJobRuntimeHealth } from '../lib/query-job-runtime.js'
+import { serverMetrics } from '../lib/server-metrics.js'
+import { getServerInstanceId } from '../lib/server-instance-id.js'
+import { getWhisperHealth } from '../lib/whisper-local.js'
+import { getActiveTranscriptionSessionCount } from './transcribe-stream.js'
+
+export const maintenanceRouter = Router()
+
+function isLoopback(req: Request): boolean {
+  const address = req.socket.remoteAddress ?? ''
+  return address === '::1' || address === '127.0.0.1' || address.startsWith('127.')
+    || address.startsWith('::ffff:127.')
+}
+
+function bodyRecord(body: unknown): Record<string, unknown> {
+  return body && typeof body === 'object' ? body as Record<string, unknown> : {}
+}
+
+function operationIdentity(body: unknown): MaintenanceOperationIdentity {
+  const value = bodyRecord(body)
+  return {
+    serverInstanceId: typeof value.serverInstanceId === 'string' ? value.serverInstanceId : '',
+    bootId: typeof value.bootId === 'string' ? value.bootId : '',
+    generationId: typeof value.generationId === 'string' ? value.generationId : '',
+    operationId: typeof value.operationId === 'string' ? value.operationId : '',
+  }
+}
+
+function drainRequest(body: unknown): MaintenanceDrainRequest {
+  const value = bodyRecord(body)
+  return {
+    ...operationIdentity(value),
+    operationKind: value.operationKind as MaintenanceOperationKind,
+    scope: value.scope as MaintenanceOperationScope,
+    postcondition: value.postcondition as MaintenancePostcondition,
+    nonceSha256: typeof value.nonceSha256 === 'string' ? value.nonceSha256 : '',
+    authorizedSuccessorGenerations: Array.isArray(value.authorizedSuccessorGenerations)
+      ? value.authorizedSuccessorGenerations.filter((item): item is string => typeof item === 'string')
+      : [],
+    ...(typeof value.ttlMs === 'number' ? { ttlMs: value.ttlMs } : {}),
+  }
+}
+
+function operationCredentials(req: Request): MaintenanceOperationCredentials {
+  return {
+    leaseId: typeof req.headers['x-cos-maintenance-lease'] === 'string'
+      ? req.headers['x-cos-maintenance-lease']
+      : undefined,
+    operationId: typeof req.headers['x-cos-maintenance-operation'] === 'string'
+      ? req.headers['x-cos-maintenance-operation']
+      : undefined,
+    nonce: typeof req.headers['x-cos-maintenance-nonce'] === 'string'
+      ? req.headers['x-cos-maintenance-nonce']
+      : undefined,
+  }
+}
+
+function statusSnapshot(credentials: MaintenanceOperationCredentials = {}) {
+  const jobs = getQueryJobRuntimeHealth()
+  const activeTranscriptionSessions = getActiveTranscriptionSessionCount()
+  const managed = managedRuntimeCapability()
+  const tracked = maintenanceLifecycle.snapshot(credentials, {
+    recording_session: activeTranscriptionSessions,
+  })
+  const untrackedDurableRuns = Math.max(0, jobs.activeRuns - (tracked.activeByKind.durable_query ?? 0))
+  const lifecycle = untrackedDurableRuns > 0
+    ? maintenanceLifecycle.snapshot(credentials, {
+      durable_query_runtime: untrackedDurableRuns,
+      recording_session: activeTranscriptionSessions,
+    })
+    : tracked
+  return {
+    contractVersion: managed.contractVersion,
+    managed: managed.managed,
+    serverVersion: managedServerVersion(),
+    generationId: getServerGenerationId(),
+    serverInstanceId: getServerInstanceId(),
+    bootId: serverMetrics.bootId,
+    activeJobs: jobs.activeRuns,
+    activeTranscriptionSessions,
+    shuttingDown: jobs.shuttingDown,
+    durableStoreState: jobs.store.state,
+    lifecycle,
+    safeToRestart: lifecycle.safeToRestart && !jobs.shuttingDown,
+    whisper: getWhisperHealth(),
+  }
+}
+
+function sendLifecycleError(res: Response, error: unknown) {
+  if (error instanceof MaintenanceLifecycleError) {
+    if (error.retryAfterSeconds != null) res.setHeader('Retry-After', String(error.retryAfterSeconds))
+    res.status(error.status).json(maintenanceErrorPayload(error))
+    return
+  }
+  res.status(500).json({ error: 'maintenance_internal_error', retryable: false })
+}
+
+function requireLoopback(req: Request, res: Response): boolean {
+  if (isLoopback(req)) return true
+  res.status(403).json({ error: 'loopback_required', retryable: false })
+  return false
+}
+
+maintenanceRouter.get('/maintenance/status', (req, res) => {
+  res.json(statusSnapshot(operationCredentials(req)))
+})
+
+maintenanceRouter.post('/maintenance/drain', (req, res) => {
+  if (!requireLoopback(req, res)) return
+  try {
+    const leaseId = maintenanceLifecycle.beginDrain(drainRequest(req.body))
+    res.json({ leaseId, ...statusSnapshot() })
+  } catch (error) {
+    sendLifecycleError(res, error)
+  }
+})
+
+maintenanceRouter.post('/maintenance/drain/adopt', (req, res) => {
+  if (!requireLoopback(req, res)) return
+  const credentials = operationCredentials(req)
+  try {
+    maintenanceLifecycle.adoptDrain(operationIdentity(req.body), credentials)
+    res.json({ adopted: true, ...statusSnapshot(credentials) })
+  } catch (error) {
+    sendLifecycleError(res, error)
+  }
+})
+
+maintenanceRouter.post('/maintenance/drain/release', (req, res) => {
+  if (!requireLoopback(req, res)) return
+  try {
+    maintenanceLifecycle.releaseDrain(operationIdentity(req.body), operationCredentials(req))
+    res.json({ released: true, ...statusSnapshot() })
+  } catch (error) {
+    sendLifecycleError(res, error)
+  }
+})
+
+maintenanceRouter.post('/maintenance/drain/cancel', (req, res) => {
+  if (!requireLoopback(req, res)) return
+  try {
+    maintenanceLifecycle.cancelDrain(operationIdentity(req.body), operationCredentials(req))
+    res.json({ canceled: true, ...statusSnapshot() })
+  } catch (error) {
+    sendLifecycleError(res, error)
+  }
+})

package/server/routes/meeting.ts

@@ -40,6 +40,7 @@ import {
   type TranscriptGapReport,
 } from './transcribe-stream.js'
 import { getServerInstanceId } from '../lib/server-instance-id.js'
+import { acquireMaintenanceWork, type MaintenanceWorkLease } from '../lib/maintenance-lifecycle.js'
 
 interface MeetingSessionSource {
   getTranscript(sessionId: string): string | null
@@ -177,6 +178,7 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
 
   router.post('/meeting/save', async (req, res) => {
     let lockedSessionId: string | null = null
+    let maintenanceLease: MaintenanceWorkLease | undefined
     try {
       const body = req.body as Record<string, unknown> | undefined
       const sessionId = typeof body?.sessionId === 'string' ? body.sessionId : ''
@@ -219,6 +221,10 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
         res.status(409).json({ error: 'Meeting save already in progress', reason: 'save_in_progress' })
         return
       }
+      // Saving/finalizing an already-recording session is a drain
+      // continuation. It must remain admitted so existing audio can reach a
+      // durable terminal while all genuinely new work is closed.
+      maintenanceLease = acquireMaintenanceWork('meeting_save', { allowDuringDrain: true })
       savingSessions.add(sessionId)
       lockedSessionId = sessionId
 
@@ -300,20 +306,30 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
       res.json(publicSaveResponse(saved))
 
       if (audioWritesReady && pendingAudioDir && chunkEntries.length > 0) {
-        const task = finalizeBatch({
-          audioDir: pendingAudioDir,
-          entries: chunkEntries.map(entry => ({ ...entry, chunk: { ...entry.chunk } })),
-          streamingWordCount: countWords(transcript),
-          meetingPath: saved.filepath,
-          sidecarPath: saved.sidecarPath,
-          runBatch,
+        // Acquire before the foreground save lease can leave scope: there is
+        // no zero-count proof gap between pending-audio handoff and the queued
+        // background finalizer.
+        const batchLease = acquireMaintenanceWork('meeting_batch_finalization', {
+          allowDuringDrain: true,
+          phase: 'queued',
+        })
+        const task = Promise.resolve().then(() => {
+          batchLease.setPhase('active')
+          return finalizeBatch({
+            audioDir: pendingAudioDir,
+            entries: chunkEntries.map(entry => ({ ...entry, chunk: { ...entry.chunk } })),
+            streamingWordCount: countWords(transcript),
+            meetingPath: saved.filepath,
+            sidecarPath: saved.sidecarPath,
+            runBatch,
+          })
         }).catch(error => {
           // Raw audio deliberately remains for the existing two-hour cleanup.
           console.error(
             `[meeting/save] Batch finalization failed for ${sessionId}: `
             + `${error instanceof Error ? error.message : String(error)}`,
           )
-        })
+        }).finally(() => batchLease.release())
         scheduleBackground(task)
       }
     } catch (error) {
@@ -324,6 +340,7 @@ export function createMeetingRouter(deps: MeetingRouteDependencies = {}): Router
       console.error('[meeting/save] Finalization failed:', error)
       res.status(500).json({ error: 'Meeting save failed', reason: 'meeting_save_error' })
     } finally {
+      maintenanceLease?.release()
       if (lockedSessionId) savingSessions.delete(lockedSessionId)
     }
   })