@gotcos/glasses-server 6.9.0 → 6.11.0

package/server/lib/codex-bridge.ts

@@ -51,6 +51,7 @@ import {
 } from './codex-run-ledger.js'
 import { codexActivityPreviewLines } from './activity-preview.js'
 import {
+  collectRunOutputImagesBounded,
   createRunOutputImagePublisher,
   isRunOutputImagePublisherCommand,
   type RunOutputImageCollectionStats,
@@ -283,7 +284,12 @@ export async function callCodexStreaming(
     }
   }
   let codexThreadId: string | undefined = engineSession?.codexThreadId
-  callbacks.onStart?.(model, sid, undefined, { codexRunId: run.runId, codexThreadId })
+  callbacks.onStart?.(model, sid, undefined, {
+    codexRunId: run.runId,
+    codexThreadId,
+    clientJobId: options?.clientJobId,
+    generation: options?.generation,
+  })
 
   let phase: Phase = 'context'
   let systemPrompt: string
@@ -312,7 +318,18 @@ export async function callCodexStreaming(
   const isFirstQuery = isNewSession(sid)
   const photoPrefix = imagePaths.length === 1 ? '[Photo]' : imagePaths.length > 1 ? `[${imagePaths.length} Photos]` : ''
   const historyQuery = photoPrefix ? `${photoPrefix} ${query || 'What do you see?'}` : query
-  const pendingUserExchange = addExchange(sid, 'user', historyQuery, globalMsgNum)
+  const exchangeProvenance = {
+    clientJobId: options?.clientJobId,
+    generation: options?.generation,
+  }
+  const pendingUserExchange = addExchange(
+    sid,
+    'user',
+    historyQuery,
+    globalMsgNum,
+    undefined,
+    exchangeProvenance,
+  )
 
   let fullQuery: string
   if (imagePaths.length === 1) {
@@ -363,6 +380,41 @@ export async function callCodexStreaming(
     options?.abortSignal?.removeEventListener('abort', handleAbort)
   }
 
+  function clearEngineSessionBestEffort(reason: string) {
+    if (!engineSession) return
+    try {
+      clearCodexEngineSession(sid, model)
+    } catch (error) {
+      console.error(`[codex-bridge] engine session clear failed (${reason}):`, error)
+    }
+  }
+
+  function saveEngineSessionBestEffort() {
+    if (!persistentCodexSession || !codexThreadId) return
+    try {
+      const saved = saveCodexEngineSession({
+        cosSessionId: sid,
+        model,
+        codexThreadId,
+        cwd: codexCwd,
+        trustMode: codexTrustMode,
+      })
+      updateCodexRun(run.runId, { codexThreadId, expiresAt: saved.expiresAt })
+    } catch (error) {
+      // The resumable-thread cache is an optimization. Its filesystem failure
+      // must never suppress the durable query terminal callback.
+      console.error('[codex-bridge] engine session save failed:', error)
+    }
+  }
+
+  function finishRunBestEffort(input: Parameters<typeof finishCodexRun>[1]) {
+    try {
+      finishCodexRun(run.runId, input)
+    } catch (error) {
+      console.error('[codex-bridge] run ledger finalization failed:', error)
+    }
+  }
+
   function emitText(text: string) {
     if (!text || emittedBlocks.has(text)) return
     emittedBlocks.add(text)
@@ -377,7 +429,49 @@ export async function callCodexStreaming(
     cleanup()
     cleanupImages()
 
-    const assistantExchange = addExchange(sid, 'assistant', text, globalMsgNum)
+    // The coordinator persists the final provider text before conversation
+    // mutation, condensation, or output-image normalization can stall/crash.
+    try {
+      const answerOwned = await callbacks.onAnswerReady?.(text)
+      if (answerOwned === false) {
+        outputImagePublisher?.cleanup()
+        removeExchange(sid, pendingUserExchange)
+        clearEngineSessionBestEffort('answer_ownership_lost')
+        finishRunBestEffort({
+          status: 'failed',
+          startedAtMs: startTime,
+          error: 'codex-bridge: durable answer ownership was lost.',
+          exitCode: null,
+        })
+        return
+      }
+    } catch (error) {
+      console.error('[codex-bridge] durable answer barrier failed:', error)
+      outputImagePublisher?.cleanup()
+      removeExchange(sid, pendingUserExchange)
+      clearEngineSessionBestEffort('answer_barrier')
+      finishRunBestEffort({
+        status: 'failed',
+        startedAtMs: startTime,
+        error: 'codex-bridge: durable answer persistence failed.',
+        exitCode: null,
+      })
+      try {
+        await callbacks.onError('codex-bridge: durable answer persistence failed.')
+      } catch (callbackError) {
+        console.error('[codex-bridge] durable barrier error callback failed:', callbackError)
+      }
+      return
+    }
+
+    const assistantExchange = addExchange(
+      sid,
+      'assistant',
+      text,
+      globalMsgNum,
+      undefined,
+      exchangeProvenance,
+    )
     if (imagePaths.length > 0) {
       replaceLastExchangeWithSummary(sid, query, text, imagePaths.length)
     }
@@ -389,7 +483,9 @@ export async function callCodexStreaming(
       const preparingHeartbeat = setInterval(() => callbacks.onToolStatus?.('Preparing images...'), HEARTBEAT_INTERVAL_MS)
       preparingHeartbeat.unref?.()
       try {
-        outputAttachments = await outputImagePublisher.collect()
+        outputAttachments = await collectRunOutputImagesBounded(outputImagePublisher, {
+          signal: options?.abortSignal,
+        })
       } catch (err) {
         console.error('[codex-bridge] output image collection failed:', err)
       } finally {
@@ -416,29 +512,27 @@ export async function callCodexStreaming(
       durationMs: totalMs,
       caller: options?.lightweight ? 'voice_query' : 'full_query',
     })
-    if (persistentCodexSession && codexThreadId) {
-      const saved = saveCodexEngineSession({
-        cosSessionId: sid,
-        model,
-        codexThreadId,
-        cwd: codexCwd,
-        trustMode: codexTrustMode,
-      })
-      updateCodexRun(run.runId, { codexThreadId, expiresAt: saved.expiresAt })
-    }
+    saveEngineSessionBestEffort()
 
-    finishCodexRun(run.runId, {
+    finishRunBestEffort({
       status: 'completed',
       startedAtMs: startTime,
       output: text,
       exitCode: 0,
     })
-    callbacks.onDone(text, model, undefined, {
-      codexRunId: run.runId,
-      codexThreadId,
-      ...(outputAttachments.length > 0 ? { outputAttachments } : {}),
-      ...(outputImageStats && outputImageStats.published > 0 ? { outputImageStats } : {}),
-    })
+    try {
+      const terminalOwned = await callbacks.onDone(text, model, undefined, {
+        codexRunId: run.runId,
+        codexThreadId,
+        clientJobId: options?.clientJobId,
+        generation: options?.generation,
+        ...(outputAttachments.length > 0 ? { outputAttachments } : {}),
+        ...(outputImageStats && outputImageStats.published > 0 ? { outputImageStats } : {}),
+      })
+      if (terminalOwned === false) return
+    } catch (error) {
+      console.error('[codex-bridge] terminal completion callback failed:', error)
+    }
 
     if (isFirstQuery) {
       notifySessionStart(sid, query)
@@ -447,23 +541,25 @@ export async function callCodexStreaming(
     notifyExchange(sid, query, text)
   }
 
-  function finalizeError(msg: string, exitCode?: number | null, status: Exclude<CodexRunStatus, 'running'> = 'failed') {
+  async function finalizeError(msg: string, exitCode?: number | null, status: Exclude<CodexRunStatus, 'running'> = 'failed') {
     if (finalized) return
     finalized = true
     cleanup()
     cleanupImages()
     outputImagePublisher?.cleanup()
     removeExchange(sid, pendingUserExchange)
-    if (engineSession) {
-      clearCodexEngineSession(sid, model)
-    }
-    finishCodexRun(run.runId, {
+    clearEngineSessionBestEffort('provider_error')
+    finishRunBestEffort({
       status,
       startedAtMs: startTime,
       error: msg,
       exitCode,
     })
-    callbacks.onError(safeCodexUserError(msg))
+    try {
+      await callbacks.onError(safeCodexUserError(msg))
+    } catch (error) {
+      console.error('[codex-bridge] terminal error callback failed:', error)
+    }
   }
 
   function handleAbort() {
@@ -580,11 +676,38 @@ export async function callCodexStreaming(
   }
 
   try {
+    const providerOwned = await callbacks.onProviderProcess?.({
+      provider: 'codex',
+      runId: run.runId,
+      pid: proc.pid,
+      clientJobId: options?.clientJobId,
+      generation: options?.generation,
+    })
+    if (providerOwned === false) {
+      proc.kill('SIGTERM')
+      finalized = true
+      cleanup()
+      cleanupImages()
+      outputImagePublisher?.cleanup()
+      removeExchange(sid, pendingUserExchange)
+      clearEngineSessionBestEffort('provider_ownership_lost')
+      finishRunBestEffort({
+        status: 'failed',
+        startedAtMs: startTime,
+        error: 'codex-bridge: durable provider ownership was lost.',
+        exitCode: null,
+      })
+      return sid
+    }
+    if (finalized) return sid
     proc.stdin.write(prompt)
     proc.stdin.end()
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err)
-    finalizeError(`codex-bridge: stdin failed — ${message}`)
+    if (!finalized) {
+      proc.kill('SIGTERM')
+      await finalizeError(`codex-bridge: provider start failed — ${message}`)
+    }
   }
 
   return sid

package/server/lib/conversation.ts

@@ -10,7 +10,7 @@ import { notifySessionStart, notifySessionEnd } from './telegram-notify.js'
 import { appendToArchive, type SessionToArchive } from './archive.js'
 import { updateGlassesSessionCache, scheduleCacheUpdate, setSessionProvider } from './session-cache-writer.js'
 import { logSessionEnd, buildSessionLogEntry, writeSessionLog } from './session-log.js'
-import { atomicWriteFileSync, loadJsonOrQuarantine } from './atomic-fs.js'
+import { atomicWriteFileSync, durableAtomicWriteFileSync, loadJsonOrQuarantine } from './atomic-fs.js'
 import { localDay } from './local-day.js'
 import { normalizeModelPreference, type ModelPreference } from '../../shared/model-preference.js'
 import { parseMediaAttachmentRefs, type MediaAttachmentRef } from '../../shared/media-attachment.js'
@@ -22,12 +22,32 @@ export interface Exchange {
   content: string
   timestamp: number
   globalMsgNum?: number  // Client's global message number for this Q&A pair
+  /** Durable query provenance. Legacy exchanges omit both fields. */
+  clientJobId?: string
+  generation?: number
   /** Public attachment refs may live on either half of a Q&A pair: request
    * photos on the user exchange, model-published images on the assistant
    * exchange. Bytes, filesystem paths, and capabilities never persist here. */
   attachments?: MediaAttachmentRef[]
 }
 
+export interface ExchangeJobProvenance {
+  clientJobId?: string
+  generation?: number
+}
+
+/** A durable job generation is the exact reconciliation/removal identity.
+ * Requiring both fields prevents a stale generation from mutating its retry. */
+export interface ExchangeJobIdentity {
+  clientJobId: string
+  generation: number
+}
+
+export interface ReconciledExchange {
+  exchange: Exchange
+  created: boolean
+}
+
 interface Session {
   id: string
   exchanges: Exchange[]
@@ -58,6 +78,46 @@ interface SessionsFile {
   savedAt: string
 }
 
+const MAX_CLIENT_JOB_ID_LENGTH = 128
+const SAFE_CLIENT_JOB_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/
+
+function validClientJobId(value: unknown): value is string {
+  return typeof value === 'string'
+    && value.length > 0
+    && value.length <= MAX_CLIENT_JOB_ID_LENGTH
+    && SAFE_CLIENT_JOB_ID_RE.test(value)
+}
+
+function validGeneration(value: unknown): value is number {
+  return typeof value === 'number'
+    && Number.isSafeInteger(value)
+    && value > 0
+}
+
+function normalizedExchangeProvenance(
+  provenance: ExchangeJobProvenance | undefined,
+): ExchangeJobProvenance {
+  if (!provenance || !validClientJobId(provenance.clientJobId)) return {}
+  return {
+    clientJobId: provenance.clientJobId,
+    ...(validGeneration(provenance.generation) ? { generation: provenance.generation } : {}),
+  }
+}
+
+function validateLoadedExchangeProvenance(exchange: Exchange): void {
+  if (!validClientJobId(exchange.clientJobId)) {
+    delete exchange.clientJobId
+    delete exchange.generation
+    return
+  }
+  if (!validGeneration(exchange.generation)) delete exchange.generation
+}
+
+function matchesJobIdentity(exchange: Exchange, identity: ExchangeJobIdentity): boolean {
+  return exchange.clientJobId === identity.clientJobId
+    && exchange.generation === identity.generation
+}
+
 function loadFromDisk(): void {
   const result = loadJsonOrQuarantine<SessionsFile>(SESSION_FILE)
   if (result.status === 'missing') return // fresh start
@@ -81,6 +141,7 @@ function loadFromDisk(): void {
     // Persistence boundary: malformed refs are dropped without sacrificing
     // the surrounding exchange or the rest of the recovered session.
     for (const exchange of session.exchanges) {
+      validateLoadedExchangeProvenance(exchange)
       if ('attachments' in exchange && exchange.attachments !== undefined) {
         const refs = parseMediaAttachmentRefs(exchange.attachments)
         if (refs.length > 0) exchange.attachments = refs
@@ -121,6 +182,29 @@ function saveToDisk(): void {
   }
 }
 
+/** Force the current in-memory conversation projection across an fsync-backed
+ * durability boundary. Durable query jobs call this only at terminal/recovery
+ * boundaries; the existing 500ms coalesced writer remains the normal hot path.
+ * Unlike saveToDisk(), failures are surfaced to the caller so it can suppress
+ * a compatibility notification and leave boot reconciliation to repair it. */
+export function flushConversationToDisk(): void {
+  if (saveTimer) {
+    clearTimeout(saveTimer)
+    saveTimer = null
+  }
+  try {
+    mkdirSync(dirname(SESSION_FILE), { recursive: true })
+    const data: SessionsFile = {
+      sessions: Object.fromEntries(sessions),
+      savedAt: new Date().toISOString(),
+    }
+    durableAtomicWriteFileSync(SESSION_FILE, JSON.stringify(data, null, 2), { mode: 0o600 })
+  } catch (err) {
+    console.error('[conversation] Failed durable terminal flush:', err)
+    throw err
+  }
+}
+
 // Restore on module load
 loadFromDisk()
 
@@ -316,6 +400,7 @@ export function addExchange(
   content: string,
   globalMsgNum?: number,
   attachments?: MediaAttachmentRef[],
+  provenance?: ExchangeJobProvenance,
 ): Exchange {
   let session = sessions.get(sessionId)
   if (!session) {
@@ -328,6 +413,7 @@ export function addExchange(
     content,
     timestamp: Date.now(),
     globalMsgNum,
+    ...normalizedExchangeProvenance(provenance),
     ...(attachments && attachments.length > 0 ? { attachments } : {}),
   }
   session.exchanges.push(exchange)
@@ -377,6 +463,113 @@ export function removeExchange(sessionId: string, exchange: Exchange): boolean {
   return true
 }
 
+/** Return the exchanges owned by one exact durable job generation. The
+ * returned objects are the live exchange objects, matching getHistory's
+ * existing shallow-copy behavior. */
+export function findExchangesByJobIdentity(
+  sessionId: string,
+  identity: ExchangeJobIdentity,
+): Exchange[] {
+  const session = sessions.get(sessionId)
+  if (!session || !validClientJobId(identity.clientJobId) || !validGeneration(identity.generation)) return []
+  return session.exchanges.filter(exchange => matchesJobIdentity(exchange, identity))
+}
+
+/** Remove only exchanges owned by one exact job generation. A role filter is
+ * used by failed runs to roll back their pending user turn without touching a
+ * completed assistant exchange or a newer retry generation. */
+export function removeExchangesByJobIdentity(
+  sessionId: string,
+  identity: ExchangeJobIdentity,
+  role?: Exchange['role'],
+): number {
+  const session = sessions.get(sessionId)
+  if (!session || !validClientJobId(identity.clientJobId) || !validGeneration(identity.generation)) return 0
+
+  const before = session.exchanges.length
+  session.exchanges = session.exchanges.filter(exchange => (
+    !matchesJobIdentity(exchange, identity) || (role !== undefined && exchange.role !== role)
+  ))
+  const removed = before - session.exchanges.length
+  if (removed === 0) return 0
+
+  session.lastActivity = Date.now()
+  scheduleSave()
+  scheduleCacheUpdate()
+  return removed
+}
+
+/** Idempotently create or update one side of a durable job exchange pair.
+ * Duplicate same-role rows for the exact identity are collapsed in the same
+ * synchronous mutation before the existing coalesced save/cache path runs.
+ * Other jobs, generations, and the opposite role are never changed. */
+export function reconcileExchangeByJobIdentity(
+  sessionId: string,
+  identity: ExchangeJobIdentity,
+  role: Exchange['role'],
+  content: string,
+  globalMsgNum?: number,
+  attachments?: MediaAttachmentRef[],
+): ReconciledExchange {
+  if (!validClientJobId(identity.clientJobId) || !validGeneration(identity.generation)) {
+    throw new Error('conversation: invalid durable job identity')
+  }
+
+  let session = sessions.get(sessionId)
+  if (!session) {
+    const now = Date.now()
+    session = {
+      id: sessionId,
+      exchanges: [],
+      lastActivity: now,
+      createdAt: now,
+      modelPreference: null,
+      contextBreaks: [],
+    }
+    sessions.set(sessionId, session)
+  }
+
+  const matchingIndexes: number[] = []
+  for (let i = 0; i < session.exchanges.length; i++) {
+    const exchange = session.exchanges[i]
+    if (exchange.role === role && matchesJobIdentity(exchange, identity)) matchingIndexes.push(i)
+  }
+
+  const validatedAttachments = parseMediaAttachmentRefs(attachments)
+  let exchange: Exchange
+  const created = matchingIndexes.length === 0
+  if (created) {
+    exchange = {
+      role,
+      content,
+      timestamp: Date.now(),
+      globalMsgNum,
+      clientJobId: identity.clientJobId,
+      generation: identity.generation,
+      ...(validatedAttachments.length > 0 ? { attachments: validatedAttachments } : {}),
+    }
+    session.exchanges.push(exchange)
+  } else {
+    exchange = session.exchanges[matchingIndexes[0]]
+    exchange.content = content
+    exchange.globalMsgNum = globalMsgNum
+    if (validatedAttachments.length > 0) exchange.attachments = validatedAttachments
+    else delete exchange.attachments
+
+    // Remove from the end so earlier indexes stay stable; retain the oldest
+    // row's timestamp and position to preserve conversation ordering.
+    for (let i = matchingIndexes.length - 1; i >= 1; i--) {
+      session.exchanges.splice(matchingIndexes[i], 1)
+    }
+  }
+
+  while (session.exchanges.length > MAX_EXCHANGES) session.exchanges.shift()
+  session.lastActivity = Date.now()
+  scheduleSave()
+  scheduleCacheUpdate()
+  return { exchange, created }
+}
+
 /** Clear a session — archive + log BEFORE deleting from the live Map.
  *  Async so callers can `await` archive completion before considering the
  *  session durable. If archive fails the session stays in the Map (retryable

package/server/lib/local-first-meetings-contract.ts

@@ -0,0 +1,53 @@
+export const LOCAL_FIRST_MEETINGS_PROTOCOL_VERSION = 1 as const
+export const LOCAL_FIRST_MEETING_IDLE_RETENTION_MS = 4 * 60 * 60 * 1000
+
+export interface LocalFirstMeetingsCapability {
+  protocolVersion: typeof LOCAL_FIRST_MEETINGS_PROTOCOL_VERSION
+  serverInstanceId: string
+  idempotentSave: true
+  sessionStatus: true
+  retentionMs: number
+}
+
+export type MeetingSessionState = 'active' | 'closed' | 'saved' | 'missing'
+export type IndexRange = [start: number, end: number]
+
+export function localFirstMeetingsCapability(serverInstanceId: string | null): LocalFirstMeetingsCapability | null {
+  if (!serverInstanceId) return null
+  return {
+    protocolVersion: LOCAL_FIRST_MEETINGS_PROTOCOL_VERSION,
+    serverInstanceId,
+    idempotentSave: true,
+    sessionStatus: true,
+    retentionMs: LOCAL_FIRST_MEETING_IDLE_RETENTION_MS,
+  }
+}
+
+/** Exact, compact representation of a sparse received-index ledger. */
+export function compressIndexRanges(indices: readonly number[]): IndexRange[] {
+  const sorted = Array.from(new Set(
+    indices.filter(value => Number.isInteger(value) && value >= 0),
+  )).sort((a, b) => a - b)
+  if (sorted.length === 0) return []
+
+  const ranges: IndexRange[] = []
+  let start = sorted[0]
+  let end = start
+  for (let index = 1; index < sorted.length; index++) {
+    const value = sorted[index]
+    if (value === end + 1) {
+      end = value
+      continue
+    }
+    ranges.push([start, end])
+    start = value
+    end = value
+  }
+  ranges.push([start, end])
+  return ranges
+}
+
+export function retainedUntilIso(lastActivityAt: number | null): string | null {
+  if (lastActivityAt == null || !Number.isFinite(lastActivityAt)) return null
+  return new Date(lastActivityAt + LOCAL_FIRST_MEETING_IDLE_RETENTION_MS).toISOString()
+}

package/server/lib/model-router.ts

@@ -16,7 +16,7 @@ import type { ModelImageInput } from './model-image-input.js'
 // mutating the same conversation/CLI session concurrently.
 const sessionRunTails = new Map<string, Promise<void>>()
 
-async function acquireSessionRunLock(sessionId: string): Promise<() => void> {
+export async function acquireModelSessionRunLock(sessionId: string): Promise<() => void> {
   const previous = sessionRunTails.get(sessionId) ?? Promise.resolve()
   let openGate!: () => void
   const gate = new Promise<void>(resolve => { openGate = resolve })
@@ -47,7 +47,7 @@ export async function callModelStreaming(
   options?: CallOptions,
 ): Promise<string> {
   const sid = getOrCreateSession(sessionId)
-  const release = await acquireSessionRunLock(sid)
+  const release = options?.sessionLockHeld ? (() => {}) : await acquireModelSessionRunLock(sid)
   if (options?.abortSignal?.aborted) {
     release()
     throw new Error('model-router: request aborted before the model run started.')
@@ -70,16 +70,16 @@ export async function callModelStreaming(
   }
   const lockedCallbacks: StreamCallbacks = {
     ...callbacks,
-    onDone: (fullText, completedModel, cliSessionId, metadata) => {
+    onDone: async (fullText, completedModel, cliSessionId, metadata) => {
       try {
-        callbacks.onDone(fullText, completedModel, cliSessionId, metadata)
+        return await callbacks.onDone(fullText, completedModel, cliSessionId, metadata)
       } finally {
         releaseTerminal()
       }
     },
-    onError: (error) => {
+    onError: async (error) => {
       try {
-        callbacks.onError(error)
+        await callbacks.onError(error)
       } finally {
         releaseTerminal()
       }