@gotcos/glasses-server 6.2.1 → 6.5.0

package/server/routes/query.ts

@@ -5,7 +5,10 @@ import { Router } from 'express'
 import { callModelStreaming } from '../lib/model-router.js'
 import { emitDisplay } from '../lib/display-bus.js'
 import { errMsg } from '../lib/utils.js'
-import { normalizeModelPreference } from '../../shared/model-preference.js'
+import { normalizeEffortPreference, normalizeModelPreference } from '../../shared/model-preference.js'
+import { QueryAttachmentError, resolveQueryAttachments } from '../lib/query-attachments.js'
+import { getMediaStore } from '../lib/media-store.js'
+import { mergeMediaAttachmentRefs } from '../../shared/media-attachment.js'
 
 const TOOL_STATUS_MESSAGES: Record<string, string> = {
   WebSearch: 'Searching web...',
@@ -16,28 +19,35 @@ const TOOL_STATUS_MESSAGES: Record<string, string> = {
 export const queryRouter = Router()
 
 queryRouter.post('/query', async (req, res) => {
-  const { query, sessionId, model, image, images, reference, globalMsgNum } = req.body
+  const { query, sessionId, model, effort, reference, globalMsgNum } = req.body
+  const activityToolMode = req.body.activityToolMode === 'off' || req.body.activityToolMode === 'preview'
+    ? req.body.activityToolMode
+    : 'status'
 
-  // Normalize: accept `images` array or legacy `image` string
-  let validImages: string[] | undefined
-  if (Array.isArray(images) && images.length > 0) {
-    // Filter to valid non-empty strings, cap at 5
-    validImages = images.filter((img: unknown) => typeof img === 'string' && img.length > 0).slice(0, 5)
-    if (validImages.length === 0) validImages = undefined
-  } else if (typeof image === 'string' && image.length > 0) {
-    // Backward compat: wrap single image as array
-    validImages = [image]
+  // Resolve durable attachment ids and legacy base64 images through one
+  // validation/normalization path before opening SSE.
+  let resolvedAttachments
+  try {
+    resolvedAttachments = await resolveQueryAttachments(req.body)
+  } catch (err) {
+    if (err instanceof QueryAttachmentError) {
+      return res.status(err.status).json({ error: err.code, detail: err.message })
+    }
+    return res.status(500).json({ error: errMsg(err) })
   }
+  const imageInputs = resolvedAttachments.inputs.length > 0 ? resolvedAttachments.inputs : undefined
+  const attachmentRefs = resolvedAttachments.refs
 
   const resolvedQuery = typeof query === 'string' ? query : ''
 
   // Vision queries can have an empty query (default to "describe what you see")
-  if ((!resolvedQuery || typeof resolvedQuery !== 'string') && !validImages) {
+  if ((!resolvedQuery || typeof resolvedQuery !== 'string') && !imageInputs) {
     return res.status(400).json({ error: 'query string or image required' })
   }
 
   // Validate model if provided
   const validModel = normalizeModelPreference(model)
+  const validEffort = normalizeEffortPreference(effort)
 
   // Validate globalMsgNum if provided
   const validGlobalMsgNum = typeof globalMsgNum === 'number' && globalMsgNum > 0
@@ -80,15 +90,40 @@ queryRouter.post('/query', async (req, res) => {
       },
       onToolStatus: (toolName) => {
         if (!done) {
-          const message = TOOL_STATUS_MESSAGES[toolName] ?? (/\s|\.{3}$/.test(toolName) ? toolName : `Using ${toolName}...`)
+          const message = activityToolMode === 'off'
+            ? 'Processing...'
+            : TOOL_STATUS_MESSAGES[toolName] ?? (/\s|\.{3}$/.test(toolName) ? toolName : `Using ${toolName}...`)
           res.write(`event: tool_status\ndata: ${JSON.stringify({ message })}\n\n`)
           emitDisplay({ type: 'tool_status', data: { message } })
         }
       },
+      // Activity lines stay on this authenticated request stream. The global
+      // display stream is intentionally unauthenticated for Even Hub recovery,
+      // so observable command/output text must never be broadcast there.
+      ...(activityToolMode === 'preview' ? {
+        onActivityLine: (line: { kind: 'input' | 'output'; text: string }) => {
+          if (!done) {
+            res.write(`event: activity_line\ndata: ${JSON.stringify(line)}\n\n`)
+          }
+        },
+      } : {}),
       onDone: (fullText, model, cliSessionId, metadata) => {
+        // Durable association is independent of the SSE socket. Backgrounding
+        // the phone cannot leave request media reserved until expiry.
+        if (resolvedAttachments.ids.length > 0) {
+          getMediaStore().associate(resolvedAttachments.ids, {
+            sessionId: sid,
+            ...(validGlobalMsgNum ? { globalMsgNum: validGlobalMsgNum } : {}),
+          }).catch((err) => console.error('[query] attachment association failed:', err))
+        }
         if (!done) {
           done = true
-          const payload = { text: fullText, sessionId: sid, model, cliSessionId, ...metadata }
+          const attachments = mergeMediaAttachmentRefs(attachmentRefs, metadata?.outputAttachments)
+          const { outputAttachments: _outputAttachments, ...runMetadata } = metadata ?? {}
+          const payload = {
+            text: fullText, sessionId: sid, model, cliSessionId, ...runMetadata,
+            ...(attachments.length > 0 ? { attachments } : {}),
+          }
           res.write(`event: done\ndata: ${JSON.stringify(payload)}\n\n`)
           emitDisplay({ type: 'done', data: payload })
           res.end()
@@ -102,13 +137,13 @@ queryRouter.post('/query', async (req, res) => {
           res.end()
         }
       },
-    }, validModel, validImages,
+    }, validModel, imageInputs,
       // Pass reference if provided (for "recall message N" feature)
       reference && typeof reference === 'object' && reference.query && reference.response
         ? { query: String(reference.query), response: String(reference.response) }
         : undefined,
       validGlobalMsgNum,
-      { abortSignal: abortController.signal },
+      { abortSignal: abortController.signal, effort: validEffort },
     )
   } catch (err: unknown) {
     if (!done) {

package/server/routes/sessions.ts

@@ -0,0 +1,299 @@
+// Session endpoints — recent list, full history, existence check, client-format messages, context breaks, end
+import { Router } from 'express'
+import { readFileSync } from 'fs'
+import { join } from 'path'
+import { getRecentSessions, getHistory, sessionExists, addContextBreak, endSession, getSessionRaw, getActiveSessions } from '../lib/conversation.js'
+import { buildSessionLogEntry, writeSessionLog } from '../lib/session-log.js'
+import { getArchiveDayMessages } from '../lib/archive.js'
+import { localDay } from '../lib/local-day.js'
+import { clearCodexEngineSession } from '../lib/codex-engine-sessions.js'
+import { mergeMediaAttachmentRefs, type MediaAttachmentRef } from '../../shared/media-attachment.js'
+
+export const sessionsRouter = Router()
+
+sessionsRouter.get('/sessions/recent', (_req, res) => {
+  const sessions = getRecentSessions(24 * 60 * 60_000)
+  res.json({ sessions })
+})
+
+// HEAD /api/sessions/:id — lightweight existence check for restore validation
+sessionsRouter.head('/sessions/:id', (req, res) => {
+  res.status(sessionExists(req.params.id) ? 200 : 404).end()
+})
+
+// GET /api/sessions/:id/history — full exchange list for session resume
+sessionsRouter.get('/sessions/:id/history', (req, res) => {
+  const exchanges = getHistory(req.params.id)
+  if (exchanges.length === 0) {
+    res.status(404).json({ error: 'Session not found or empty' })
+    return
+  }
+  res.json({ exchanges })
+})
+
+// POST /api/sessions/:id/context-break — insert a context break (prompt history gate)
+sessionsRouter.post('/sessions/:id/context-break', (req, res) => {
+  const ok = addContextBreak(req.params.id)
+  if (!ok) {
+    res.status(404).json({ error: 'Session not found' })
+    return
+  }
+  res.json({ ok: true })
+})
+
+// GET /api/sessions/:id/messages — client-compatible format (paired Q&A)
+sessionsRouter.get('/sessions/:id/messages', (req, res) => {
+  const exchanges = getHistory(req.params.id)
+  if (exchanges.length === 0) {
+    res.status(404).json({ error: 'Session not found or empty' })
+    return
+  }
+
+  // Pair user+assistant exchanges into client message format. Request refs on
+  // the user turn and output refs on the assistant turn become one safe list.
+  const messages: Array<{
+    query: string
+    text: string
+    timestamp: number
+    no?: number
+    sessionId: string
+    attachments?: MediaAttachmentRef[]
+  }> = []
+  for (let i = 0; i < exchanges.length; i++) {
+    const ex = exchanges[i]
+    if (ex.role === 'user') {
+      const next = exchanges[i + 1]
+      if (next && next.role === 'assistant') {
+        const attachments = mergeMediaAttachmentRefs(ex.attachments, next.attachments)
+        messages.push({
+          query: ex.content,
+          text: next.content,
+          timestamp: next.timestamp,
+          sessionId: req.params.id,
+          ...((ex.globalMsgNum ?? next.globalMsgNum) != null ? { no: ex.globalMsgNum ?? next.globalMsgNum } : {}),
+          ...(attachments.length > 0 ? { attachments } : {}),
+        })
+        i++ // skip the assistant exchange
+      }
+    }
+  }
+
+  res.json({ messages })
+})
+
+// POST /api/sessions/:id/end — Explicitly end a session (archive + log + notify)
+// Called by client on "new session", "clear session", or app backgrounding.
+// Prevents data loss — session gets logged to .glasses_sessions.jsonl immediately
+// instead of waiting for the 2hr TTL expiry.
+// POST /api/sessions/lookup — batch resolve timestamps to session IDs
+// Used to retroactively stamp messages that predate the sessionId feature
+sessionsRouter.post('/sessions/lookup', (req, res) => {
+  const { timestamps } = req.body as { timestamps: number[] }
+  if (!timestamps || !Array.isArray(timestamps)) {
+    res.status(400).json({ error: 'timestamps[] required' })
+    return
+  }
+
+  // Build time ranges from BOTH JSONL history AND live server sessions
+  const logPath = join(process.env.COS_SCRIPTS_DIR || '', '.glasses_sessions.jsonl')
+  const sessionRanges: Array<{ sid: string; start: number; end: number }> = []
+
+  // 1. Live server sessions (always current — no snapshot lag)
+  const recentSessions = getRecentSessions(24 * 60 * 60_000)
+  for (const rs of recentSessions) {
+    const raw = getSessionRaw(rs.id)
+    if (raw) {
+      sessionRanges.push({ sid: raw.id, start: raw.createdAt, end: Date.now() }) // extends to NOW
+    }
+  }
+
+  // 2. JSONL history (ended sessions + snapshots)
+  try {
+    const lines = readFileSync(logPath, 'utf-8').trim().split('\n')
+    for (const line of lines) {
+      const d = JSON.parse(line)
+      if (d.session_id && d.created_at) {
+        const start = new Date(d.created_at).getTime()
+        const end = d.ended_at ? new Date(d.ended_at).getTime() : start + 7200_000
+        // Live sessions take priority. For JSONL, keep the widest (latest) time range per session.
+        const existing = sessionRanges.find(s => s.sid === d.session_id)
+        if (!existing) {
+          sessionRanges.push({ sid: d.session_id, start, end })
+        } else if (end > existing.end && !getSessionRaw(d.session_id)) {
+          // Widen the JSONL range (but don't overwrite live sessions which extend to NOW)
+          existing.end = end
+        }
+      }
+    }
+  } catch { /* no log file */ }
+
+  // Match each timestamp to a session (live sessions checked first, then JSONL)
+  const results: Record<number, string | null> = {}
+  for (const ts of timestamps) {
+    let match: string | null = null
+    for (const s of sessionRanges) {
+      if (ts >= s.start && ts <= s.end) {
+        match = s.sid
+        break
+      }
+    }
+    results[ts] = match
+  }
+
+  res.json({ results, sessionsScanned: sessionRanges.length })
+})
+
+sessionsRouter.post('/sessions/:id/end', async (req, res) => {
+  try {
+    const result = await endSession(req.params.id)
+    if (!result) {
+      res.status(404).json({ error: 'Session not found' })
+      return
+    }
+    // When logged === false the archive write failed — we keep the session
+    // in the Map for the next mirror to retry, but signal 503 so the client
+    // knows NOT to wipe local messages (they're still the user's only copy).
+    if (!result.logged && result.exchangeCount > 0) {
+      res.status(503).json({
+        ok: false,
+        error: 'Archive write failed — session retained for retry',
+        exchange_count: result.exchangeCount,
+        duration_minutes: result.durationMin,
+      })
+      return
+    }
+    clearCodexEngineSession(req.params.id)
+    res.json({
+      ok: true,
+      logged: result.logged,
+      exchange_count: result.exchangeCount,
+      duration_minutes: result.durationMin,
+    })
+  } catch (err) {
+    console.error('[sessions] /end unexpected error:', err)
+    res.status(500).json({ error: String(err) })
+  }
+})
+
+// POST /api/sessions/:id/snapshot — write live session to .glasses_sessions.jsonl WITHOUT ending it
+// Enables M3 Ultra TUI to read current glasses conversation while session is still active
+sessionsRouter.post('/sessions/:id/snapshot', (req, res) => {
+  const session = getSessionRaw(req.params.id)
+  if (!session) {
+    res.status(404).json({ error: 'Session not found' })
+    return
+  }
+
+  const entry = buildSessionLogEntry({
+    id: session.id,
+    exchanges: session.exchanges,
+    createdAt: session.createdAt,
+    lastActivity: session.lastActivity,
+    modelPreference: session.modelPreference,
+    endReason: 'explicit_end', // marker — will be overwritten when session actually ends
+    slug: `[LIVE] ${(session.exchanges.find(e => e.role === 'user')?.content ?? '').slice(0, 50)}`,
+  })
+
+  const logged = writeSessionLog(entry)
+  res.json({
+    ok: true,
+    logged,
+    session_id: session.id,
+    message_count: entry.total_message_count,
+    messages_logged: entry.messages.length,
+  })
+})
+
+// GET /api/sessions/today/live-chats — live session chat summaries for today (not yet archived).
+// Date compare is LOCAL time so users chatting in CDT/PST late evening still see their
+// session under "today" instead of "tomorrow UTC".
+// Each summary includes `sessionId` so the client can drill down via
+// `/api/sessions/:id/messages` — index=-1 is a sentinel and is NOT a valid archive chat index.
+sessionsRouter.get('/sessions/today/live-chats', (_req, res) => {
+  const todayDate = localDay()
+  const liveSessions = getActiveSessions()
+  const chats: Array<{ index: number; summary: string; exchangeCount: number; startedAt: number; isLive: boolean; sessionId: string }> = []
+
+  for (const session of liveSessions) {
+    const sessionDay = localDay(session.lastActivity)
+    if (sessionDay !== todayDate) continue
+    if (session.exchanges.length === 0) continue
+
+    const firstQuery = session.exchanges.find(e => e.role === 'user')?.content ?? ''
+    const summary = firstQuery.length > 57 ? firstQuery.slice(0, 54) + '...' : firstQuery || 'Live session'
+
+    chats.push({
+      index: -1,
+      summary: `[LIVE] ${summary}`,
+      exchangeCount: session.exchanges.length,
+      startedAt: session.createdAt,
+      isLive: true,
+      sessionId: session.id,
+    })
+  }
+
+  res.json({ chats })
+})
+
+// GET /api/sessions/today/all-messages — merged view of today's archived + live session messages.
+// Dedup key is `sessionId|timestamp` (was bare timestamp, which collided on NTP skew or
+// same-ms adds). `sessionId` is always known for live exchanges; archive messages fall
+// back to the archived chat's sessionId via getArchiveDayMessages.
+sessionsRouter.get('/sessions/today/all-messages', (_req, res) => {
+  const todayDate = localDay()
+
+  const archivedMessages = getArchiveDayMessages(todayDate).map(m => ({
+    ...m,
+    source: 'archive' as const,
+  }))
+
+  const liveMessages: Array<{
+    query: string
+    text: string
+    timestamp: number
+    chatIndex: number
+    sessionId: string
+    source: 'live'
+    no?: number
+    attachments?: MediaAttachmentRef[]
+  }> = []
+  const liveSessions = getActiveSessions()
+  for (const session of liveSessions) {
+    const sessionDay = localDay(session.lastActivity)
+    if (sessionDay !== todayDate) continue
+    for (let i = 0; i < session.exchanges.length; i++) {
+      const ex = session.exchanges[i]
+      if (ex.role === 'user') {
+        const next = session.exchanges[i + 1]
+        if (next && next.role === 'assistant') {
+          const attachments = mergeMediaAttachmentRefs(ex.attachments, next.attachments)
+          liveMessages.push({
+            query: ex.content,
+            text: next.content,
+            timestamp: next.timestamp,
+            chatIndex: -1,
+            sessionId: session.id,
+            source: 'live',
+            ...((ex.globalMsgNum ?? next.globalMsgNum) != null ? { no: ex.globalMsgNum ?? next.globalMsgNum } : {}),
+            ...(attachments.length > 0 ? { attachments } : {}),
+          })
+          i++
+        }
+      }
+    }
+  }
+
+  // Merge, dedup by (sessionId, timestamp), sort chronologically.
+  const seen = new Set<string>()
+  const keyOf = (m: { sessionId?: string; timestamp: number }) => `${m.sessionId ?? ''}|${m.timestamp}`
+  const merged = [...archivedMessages, ...liveMessages]
+    .filter(m => {
+      const k = keyOf(m as any)
+      if (seen.has(k)) return false
+      seen.add(k)
+      return true
+    })
+    .sort((a, b) => a.timestamp - b.timestamp)
+
+  res.json({ messages: merged, date: todayDate })
+})

package/shared/media-attachment.ts

@@ -0,0 +1,126 @@
+// Media attachment contract — the ONE shape both the browser client and the
+// server exchange for image attachments (Release A of the image-attachments
+// plan). The public ref deliberately carries NO storage path, URL, token,
+// base64, checksum, or internal lifecycle state — those live only in the
+// server media index. Anything that persists or transmits an attachment
+// persists THIS shape (or just the id) and nothing else.
+
+export type MediaKind = 'user_photo' | 'traffic_frame' | 'generated_visual'
+
+export type MediaMime = 'image/jpeg' | 'image/png'
+
+export interface MediaAttachmentRef {
+  id: string
+  kind: MediaKind
+  mime: MediaMime
+  width: number
+  height: number
+  createdAt: string
+  label?: string
+  capturedAt?: string
+  expiresAt?: string
+}
+
+/** Hard cap on attachments per prompt — mirrored by upload validation,
+ *  query resolution, and the phone composer. */
+export const MAX_ATTACHMENTS_PER_PROMPT = 5
+
+// ── Media IDs ────────────────────────────────────────────────────────────────
+// One strict generated format, one strict validator. The id builds filesystem
+// paths on the server, so the validator rejects anything that isn't exactly
+// `m_` + 24 lowercase hex chars — no path characters can ever pass.
+
+export const MEDIA_ID_RE = /^m_[a-f0-9]{24}$/
+
+export function isValidMediaId(id: unknown): id is string {
+  return typeof id === 'string' && MEDIA_ID_RE.test(id)
+}
+
+const VALID_KINDS: ReadonlySet<string> = new Set(['user_photo', 'traffic_frame', 'generated_visual'])
+const VALID_MIMES: ReadonlySet<string> = new Set(['image/jpeg', 'image/png'])
+const MAX_LABEL_LEN = 120
+// ISO-8601 subset — what `new Date().toISOString()` emits.
+const ISO_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/
+
+function isIsoTimestamp(v: unknown): v is string {
+  return typeof v === 'string' && ISO_RE.test(v) && Number.isFinite(new Date(v).getTime())
+}
+
+function isDimension(v: unknown): v is number {
+  return typeof v === 'number' && Number.isInteger(v) && v > 0 && v <= 65_535
+}
+
+/** Validate an UNTRUSTED value into a MediaAttachmentRef, or null.
+ *  TypeScript types alone are not validation — every persistence and API
+ *  boundary that accepts a ref from outside must run it through here.
+ *  Returns a fresh object containing only the known fields (drops extras). */
+export function parseMediaAttachmentRef(raw: unknown): MediaAttachmentRef | null {
+  if (!raw || typeof raw !== 'object') return null
+  const r = raw as Record<string, unknown>
+  if (!isValidMediaId(r.id)) return null
+  if (typeof r.kind !== 'string' || !VALID_KINDS.has(r.kind)) return null
+  if (typeof r.mime !== 'string' || !VALID_MIMES.has(r.mime)) return null
+  if (!isDimension(r.width) || !isDimension(r.height)) return null
+  if (!isIsoTimestamp(r.createdAt)) return null
+  const ref: MediaAttachmentRef = {
+    id: r.id,
+    kind: r.kind as MediaKind,
+    mime: r.mime as MediaMime,
+    width: r.width,
+    height: r.height,
+    createdAt: r.createdAt,
+  }
+  if (typeof r.label === 'string' && r.label.length > 0) {
+    ref.label = r.label.slice(0, MAX_LABEL_LEN)
+  }
+  if (isIsoTimestamp(r.capturedAt)) ref.capturedAt = r.capturedAt
+  if (isIsoTimestamp(r.expiresAt)) ref.expiresAt = r.expiresAt
+  return ref
+}
+
+/** Validate an untrusted array of refs, dropping only the invalid entries
+ *  (a bad ref must never take the whole conversation record with it). */
+export function parseMediaAttachmentRefs(raw: unknown): MediaAttachmentRef[] {
+  if (!Array.isArray(raw)) return []
+  const out: MediaAttachmentRef[] = []
+  for (const item of raw) {
+    const ref = parseMediaAttachmentRef(item)
+    if (ref) out.push(ref)
+    if (out.length >= MAX_ATTACHMENTS_PER_PROMPT) break
+  }
+  return out
+}
+
+/** Validate, merge, and de-duplicate attachment refs from multiple untrusted
+ *  exchange surfaces. A completed Q&A pair can carry request refs on the user
+ *  turn and generated/research refs on the assistant turn; readers should see
+ *  one bounded list without trusting either persisted shape. First occurrence
+ *  wins so the request-side ref remains stable when the server echoes it back
+ *  in completion metadata. */
+export function mergeMediaAttachmentRefs(...sources: unknown[]): MediaAttachmentRef[] {
+  const out: MediaAttachmentRef[] = []
+  const seen = new Set<string>()
+  for (const source of sources) {
+    if (!Array.isArray(source)) continue
+    for (const item of source) {
+      const ref = parseMediaAttachmentRef(item)
+      if (!ref || seen.has(ref.id)) continue
+      seen.add(ref.id)
+      out.push(ref)
+      if (out.length >= MAX_ATTACHMENTS_PER_PROMPT) return out
+    }
+  }
+  return out
+}
+
+/** Validate an untrusted list of media IDs (dedup, cap, strict format). */
+export function parseMediaIdList(raw: unknown): string[] {
+  if (!Array.isArray(raw)) return []
+  const out: string[] = []
+  for (const item of raw) {
+    if (!isValidMediaId(item) || out.includes(item)) continue
+    out.push(item)
+    if (out.length >= MAX_ATTACHMENTS_PER_PROMPT) break
+  }
+  return out
+}