@gotcos/glasses-server 6.37.3 → 6.38.0

package/CHANGELOG.md

@@ -1,3 +1,35 @@
+## 6.38.0
+
+Six months of archived conversation you can finally search.
+
+NEW: GET /api/archive/search?q=&from=&to=&limit=. The archive already held every
+day's conversations -- 175 day files spanning six months on a real install -- with
+no way to find anything in them.
+
+The scan never calls JSON.parse. Day sizes are wildly skewed: the median day is
+36 KB, but the largest measured is 343 MB of agent transcript, and materialising
+that one day costs 1.2 GB heap / 2.3 GB RSS. On a process that also runs the
+wearer's live session that is not affordable, so days are scanned as raw bytes
+through a stream. A hit is therefore attributed to a DATE plus surrounding text,
+not to a chat; open /archive/:date/chats for structure. A full 90-day scan
+measures at 2.4 s, so there is no index to build and nothing that can drift.
+
+FIXED: GET /api/archive no longer parses 1.2 GB to list days. It built {date,
+summary, chatCount, exchangeCount} by parsing EVERY day file and discarding the
+bodies -- one request away from a multi-gigabyte spike. It now reads a sidecar
+index keyed by each day's (size, mtimeMs). Measured on the real corpus: cold build
+2,208 ms at 248 MB RSS for all 175 days, warm read 1 ms. Both halves of the key
+matter -- size alone misses an in-place edit that preserves length.
+
+listArchiveDates() is deleted rather than deprecated. It had one caller, and an
+uncalled landmine is still a landmine.
+
+Route order is load-bearing: /archive/search is registered ABOVE /archive/:date,
+because Express matches in order and the date validator would otherwise reject it
+as :date === "search". That is the same trap that has always made
+/api/archive/dates look like an empty archive. Older servers still exhibit it, so
+clients should read a 400 "Invalid date" from the search path as "route absent".
+
 ## 6.37.3
 
 Speaker ID works out of the box. A silent capture stops retrying forever. Live

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.37.3",
-  "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
+  "version": "6.38.0",
+  "description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {
     "glasses-server": "bin/cli.cjs",

package/server/lib/archive-index.ts

@@ -0,0 +1,186 @@
+// A sidecar index of per-day archive summaries.
+//
+// WHY THIS EXISTS. `listArchiveDates()` produces {date, summary, chatCount,
+// exchangeCount} by JSON.parsing EVERY day file and discarding the bodies. On the
+// real corpus that is 175 files / 1.2 GB, and the single largest day (2026-07-30,
+// 343 MB) measures at 1.2 GB heap / 2.3 GB RSS to materialise on its own. That is
+// one GET away from a multi-gigabyte spike on the same process that runs the
+// wearer's live glasses session.
+//
+// The index holds exactly the fields that listing needs, keyed by the day's own
+// (size, mtimeMs). A day whose bytes have not changed is never reopened, so the
+// steady-state cost of a listing is one readdir plus one small JSON read.
+//
+// INVALIDATION IS THE WHOLE CONTRACT. size+mtimeMs is what makes this honest: an
+// archive that is appended to changes both, so a stale entry cannot survive a
+// write. Entries for vanished days are dropped rather than kept, because a listing
+// that names a file nobody can open is worse than one that omits it.
+import { createReadStream, existsSync, readFileSync, readdirSync, statSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { atomicWriteFileSync } from './atomic-fs.js'
+
+export const ARCHIVE_INDEX_SCHEMA = 1
+
+export interface ArchiveIndexEntry {
+  date: string
+  size: number
+  mtimeMs: number
+  chatCount: number
+  exchangeCount: number
+  summary: string | null
+}
+
+interface ArchiveIndexFile {
+  schemaVersion: number
+  entries: Record<string, ArchiveIndexEntry>
+}
+
+const DATE_RE = /^\d{4}-\d{2}-\d{2}$/
+/** Chats carry exactly one `exchangeCount` each and the key appears nowhere else
+ *  in the shape. Verified against parsed truth on six real days (8/28/93/47/11/90
+ *  chats) before this was relied on. */
+const EXCHANGE_COUNT_RE = /"exchangeCount":\s*(\d+)/g
+const CARRY_CHARS = 64
+
+export function emptyIndex(): ArchiveIndexFile {
+  return { schemaVersion: ARCHIVE_INDEX_SCHEMA, entries: {} }
+}
+
+export function readIndexFile(indexPath: string): ArchiveIndexFile {
+  try {
+    if (!existsSync(indexPath)) return emptyIndex()
+    const parsed = JSON.parse(readFileSync(indexPath, 'utf8')) as ArchiveIndexFile
+    // A schema bump invalidates wholesale rather than trying to migrate: the index
+    // is a cache, and rebuilding it costs one scan.
+    if (parsed?.schemaVersion !== ARCHIVE_INDEX_SCHEMA || typeof parsed.entries !== 'object') {
+      return emptyIndex()
+    }
+    return { schemaVersion: ARCHIVE_INDEX_SCHEMA, entries: parsed.entries ?? {} }
+  } catch {
+    return emptyIndex() // a corrupt cache must never take the listing down
+  }
+}
+
+/** Pull the top-level `summary` string without materialising the day. It sits in
+ *  the opening object, so the first chunk is enough; a day that hides it deeper
+ *  simply reports null rather than costing a full read. */
+export function extractSummary(head: string): string | null {
+  const m = /"summary"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(head)
+  if (!m) return null
+  try {
+    return JSON.parse(`"${m[1]}"`) as string
+  } catch {
+    return null
+  }
+}
+
+/** Stream one day and count chats + exchanges. Never parses. */
+export async function summariseDayFile(filePath: string): Promise<{ chatCount: number; exchangeCount: number; summary: string | null }> {
+  let chatCount = 0
+  let exchangeCount = 0
+  let summary: string | null = null
+  let carry = ''
+  let first = true
+
+  await new Promise<void>((done, fail) => {
+    const stream = createReadStream(filePath, { encoding: 'utf8', highWaterMark: 1 << 20 })
+    stream.on('data', (chunk: string | Buffer) => {
+      const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8')
+      const hay = carry + text
+      if (first) {
+        summary = extractSummary(hay)
+        first = false
+      }
+      EXCHANGE_COUNT_RE.lastIndex = 0
+      let m: RegExpExecArray | null
+      while ((m = EXCHANGE_COUNT_RE.exec(hay)) !== null) {
+        chatCount++
+        exchangeCount += Number(m[1])
+      }
+      // Carry enough to reassemble a key/value split across the boundary. Matches
+      // inside the carry are not double-counted because the carry is short and the
+      // next pass starts from it, not from the whole previous chunk.
+      carry = hay.slice(Math.max(0, hay.length - CARRY_CHARS))
+    })
+    stream.on('error', fail)
+    stream.on('end', () => done())
+  })
+
+  return { chatCount, exchangeCount, summary }
+}
+
+export interface RefreshResult {
+  entries: ArchiveIndexEntry[]
+  rebuilt: string[]
+  dropped: string[]
+  fromCache: number
+}
+
+/**
+ * Bring the index in line with what is on disk and return the listing, newest
+ * first. Only days whose (size, mtimeMs) changed are reopened.
+ */
+export async function refreshArchiveIndex(dir: string, indexPath: string): Promise<RefreshResult> {
+  const file = readIndexFile(indexPath)
+  const rebuilt: string[] = []
+  const dropped: string[] = []
+  let fromCache = 0
+
+  let present: string[] = []
+  try {
+    present = readdirSync(dir)
+      .filter(f => f.endsWith('.json'))
+      .map(f => f.slice(0, -'.json'.length))
+      .filter(d => DATE_RE.test(d))
+  } catch {
+    return { entries: [], rebuilt, dropped, fromCache }
+  }
+  const presentSet = new Set(present)
+
+  for (const date of Object.keys(file.entries)) {
+    if (!presentSet.has(date)) {
+      delete file.entries[date]
+      dropped.push(date)
+    }
+  }
+
+  for (const date of present) {
+    const filePath = resolve(dir, `${date}.json`)
+    let st
+    try {
+      st = statSync(filePath)
+    } catch {
+      continue
+    }
+    const cached = file.entries[date]
+    if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) {
+      fromCache++
+      continue
+    }
+    try {
+      const s = await summariseDayFile(filePath)
+      file.entries[date] = {
+        date,
+        size: st.size,
+        mtimeMs: st.mtimeMs,
+        chatCount: s.chatCount,
+        exchangeCount: s.exchangeCount,
+        summary: s.summary,
+      }
+      rebuilt.push(date)
+    } catch {
+      continue // an unreadable day must not abort the listing
+    }
+  }
+
+  if (rebuilt.length > 0 || dropped.length > 0) {
+    try {
+      atomicWriteFileSync(indexPath, `${JSON.stringify(file)}\n`, { mode: 0o600 })
+    } catch {
+      // A cache that cannot persist still serves this call correctly.
+    }
+  }
+
+  const entries = Object.values(file.entries).sort((a, b) => (a.date < b.date ? 1 : a.date > b.date ? -1 : 0))
+  return { entries, rebuilt, dropped, fromCache }
+}

package/server/lib/archive-search.ts

@@ -0,0 +1,195 @@
+// Literal text search across the daily conversation archive.
+//
+// WHY THIS NEVER CALLS JSON.parse. The archive is 175 day files spanning six
+// months, and their sizes are wildly skewed: the median day is 36 KB but
+// 2026-07-30 is 327 MB of agent transcript (283,356 exchanges). Parsing that
+// into JS objects would cost multiple GB of heap on a server that also runs the
+// wearer's live glasses session. A day file is therefore scanned as raw bytes
+// and never materialised.
+//
+// The cost of that choice is attribution: a hit reports its DATE and a text
+// snippet, not a chat index. Callers open the day through the existing
+// /archive/:date/chats routes for structure. Day-level attribution is what a
+// stream can give safely, and the existing routes already cover the rest.
+//
+// Measured on the real corpus: a full literal scan of a 90-day window is ~2.4s,
+// so this needs no index, no background build, and nothing that can drift out of
+// sync with the archive itself.
+import { createReadStream } from 'node:fs'
+import { stat } from 'node:fs/promises'
+import { resolve } from 'node:path'
+
+export const MIN_QUERY_CHARS = 2
+export const DEFAULT_LIMIT = 50
+export const MAX_LIMIT = 200
+export const SNIPPET_RADIUS = 120
+/** Per-day snippet cap. A term appearing 40,000 times in one 327 MB day must not
+ *  return 40,000 snippets; the count still reports the true total. */
+export const MAX_SNIPPETS_PER_DAY = 3
+const CHUNK_BYTES = 1 << 20
+
+export interface ArchiveSearchHit {
+  date: string
+  matches: number
+  snippets: string[]
+}
+
+export interface ArchiveSearchResult {
+  hits: ArchiveSearchHit[]
+  scannedDays: number
+  bytesScanned: number
+  truncated: boolean
+}
+
+/** YYYY-MM-DD, the archive's own filename contract. Lexical order IS date order,
+ *  which is what makes the range filter a string compare. */
+export function isArchiveDate(value: unknown): value is string {
+  return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value)
+}
+
+export function datesInRange(dates: string[], from?: string, to?: string): string[] {
+  return dates
+    .filter(isArchiveDate)
+    .filter(d => (from ? d >= from : true) && (to ? d <= to : true))
+    .sort()
+    .reverse()
+}
+
+function cleanSnippet(raw: string): string {
+  // The scan runs over JSON source, so a snippet arrives carrying escape
+  // sequences and structural punctuation. Unescape the common ones and collapse
+  // whitespace so the caller can render it as prose.
+  return raw
+    .replace(/\\n/g, ' ')
+    .replace(/\\t/g, ' ')
+    .replace(/\\"/g, '"')
+    .replace(/\\\\/g, '\\')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+/**
+ * Count matches in `hayLower` but cut snippets from `hayOriginal`.
+ *
+ * Matching has to happen on the lowercased text for case-insensitivity, but a
+ * snippet cut from that text renders as all-lowercase prose -- "chelsie owes the
+ * sales-vs-education split" -- which reads like broken data in a UI. The offsets
+ * line up because toLowerCase() is length-preserving for effectively all real
+ * archive text; where it is NOT (a handful of exotic codepoints expand), the
+ * length check catches the drift and falls back to the lowered slice rather than
+ * cutting at a wrong offset.
+ */
+function collectSnippets(
+  hayLower: string,
+  hayOriginal: string,
+  needle: string,
+  out: string[],
+  cap: number,
+): number {
+  const aligned = hayLower.length === hayOriginal.length
+  let found = 0
+  let i = hayLower.indexOf(needle)
+  while (i !== -1) {
+    found++
+    if (out.length < cap) {
+      const start = Math.max(0, i - SNIPPET_RADIUS)
+      const end = Math.min(hayLower.length, i + needle.length + SNIPPET_RADIUS)
+      const snippet = cleanSnippet((aligned ? hayOriginal : hayLower).slice(start, end))
+      if (snippet) out.push(snippet)
+    }
+    i = hayLower.indexOf(needle, i + needle.length)
+  }
+  return found
+}
+
+/**
+ * Scan ONE day file. Returns null when the term never appears.
+ *
+ * The overlap is the whole reason this is a separate function with its own test.
+ * A 1 MB chunk boundary can fall in the middle of the search term, and a naive
+ * per-chunk indexOf silently misses it — the failure mode is a search that works
+ * in every test fixture and quietly loses hits on real multi-megabyte days.
+ * Each chunk is therefore prefixed with the last (needle.length - 1) characters
+ * of the previous one, and matches inside that carried prefix are not counted
+ * twice because the search resumes past the needle.
+ */
+export async function searchArchiveFile(
+  filePath: string,
+  needleLower: string,
+): Promise<{ matches: number; snippets: string[]; bytes: number } | null> {
+  let matches = 0
+  const snippets: string[] = []
+  let bytes = 0
+  let carry = ''
+  const overlap = Math.max(0, needleLower.length - 1)
+
+  await new Promise<void>((resolveDone, rejectDone) => {
+    const stream = createReadStream(filePath, { encoding: 'utf8', highWaterMark: CHUNK_BYTES })
+    stream.on('data', (chunk: string | Buffer) => {
+      const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8')
+      bytes += Buffer.byteLength(text, 'utf8')
+      const hay = carry + text
+      matches += collectSnippets(hay.toLowerCase(), hay, needleLower, snippets, MAX_SNIPPETS_PER_DAY)
+      carry = hay.slice(Math.max(0, hay.length - overlap))
+    })
+    stream.on('error', rejectDone)
+    stream.on('end', () => resolveDone())
+  })
+
+  return matches > 0 ? { matches, snippets, bytes } : null
+}
+
+/**
+ * Search a set of archive days, newest first, stopping once `limit` days have
+ * matched. Days are scanned in order so an early-exit favours recent history,
+ * which is what a person browsing "what did I say about X" actually wants.
+ */
+export async function searchArchive(opts: {
+  dir: string
+  dates: string[]
+  query: string
+  from?: string
+  to?: string
+  limit?: number
+}): Promise<ArchiveSearchResult> {
+  const query = (opts.query ?? '').trim()
+  if (query.length < MIN_QUERY_CHARS) {
+    throw new Error(`query must be at least ${MIN_QUERY_CHARS} characters`)
+  }
+  const limit = Math.min(Math.max(1, opts.limit ?? DEFAULT_LIMIT), MAX_LIMIT)
+  const needleLower = query.toLowerCase()
+  const candidates = datesInRange(opts.dates, opts.from, opts.to)
+
+  const hits: ArchiveSearchHit[] = []
+  let scannedDays = 0
+  let bytesScanned = 0
+  let truncated = false
+
+  for (const date of candidates) {
+    if (hits.length >= limit) {
+      truncated = true
+      break
+    }
+    const filePath = resolve(opts.dir, `${date}.json`)
+    try {
+      await stat(filePath)
+    } catch {
+      continue // listed but absent: a rotation mid-scan is not an error
+    }
+    scannedDays++
+    try {
+      const found = await searchArchiveFile(filePath, needleLower)
+      if (found) {
+        bytesScanned += found.bytes
+        hits.push({ date, matches: found.matches, snippets: found.snippets })
+      } else {
+        // still counts toward bytes so callers can report honest scan cost
+        bytesScanned += (await stat(filePath)).size
+      }
+    } catch {
+      continue // an unreadable day must not abort the whole search
+    }
+  }
+
+  return { hits, scannedDays, bytesScanned, truncated }
+}

package/server/lib/archive.ts

@@ -94,7 +94,7 @@ export function loadArchive(date: string): DailyArchive | null {
   }
   if (result.status === 'missing') return null
   // Defense: a valid-JSON but wrong-shape day file (no chats[]) would make the
-  // readers throw 500 AND drop listArchiveDates into its catch → the whole
+  // readers throw 500 AND drop the archive listing into its catch → the whole
   // Message History list vanishes on one bad file. Coerce to an empty day.
   const data = result.data
   if (data && !Array.isArray(data.chats)) data.chats = []
@@ -367,30 +367,51 @@ export async function appendToArchive(
 // ── Query functions ─────────────────────────────────────────
 
 /** List all archive dates with summaries */
-export function listArchiveDates(): ArchiveDateSummary[] {
+/** Sidecar index location. Deliberately a SIBLING of the archive directory, not
+ *  a file inside it: anything living in that directory has to be excluded by
+ *  every readdir filter forever, and one missed filter turns the cache into a
+ *  phantom "day". */
+export function archiveIndexPath(): string {
+  return dataPath('archive-index.json')
+}
+
+/** The archive directory, for readers that must NOT materialise a day file. */
+export function archiveDir(): string {
+  return ensureArchiveDir()
+}
+
+/**
+ * Date strings only, straight from readdir — no file is opened.
+ *
+ * The richer per-day summaries (chat and exchange counts) come from the sidecar
+ * index in archive-index.ts. The listing this replaced reached them by
+ * JSON.parsing EVERY day file. Measured on the real corpus that is 1.2 GB across
+ * 175 files, and the single largest day (2026-07-30, 343 MB) costs 1.2 GB heap /
+ * 2.3 GB RSS to parse on its own. Anything that only needs to know WHICH days
+ * exist must use this instead, on a server that is also running the wearer's live
+ * session.
+ */
+export function listArchiveDateStrings(): string[] {
   try {
     ensureArchiveDir()
-    const files = readdirSync(ARCHIVE_DIR)
+    return readdirSync(ARCHIVE_DIR)
       .filter(f => f.endsWith('.json'))
+      .map(f => f.slice(0, -'.json'.length))
+      .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
       .sort()
-      .reverse() // newest first
-
-    return files.map(f => {
-      const date = f.replace('.json', '')
-      const archive = loadArchive(date)
-      if (!archive) return null
-      return {
-        date: archive.date,
-        summary: archive.summary,
-        chatCount: archive.chats.length,
-        exchangeCount: archive.chats.reduce((sum, c) => sum + c.exchangeCount, 0),
-      }
-    }).filter(Boolean) as ArchiveDateSummary[]
+      .reverse()
   } catch {
     return []
   }
 }
 
+// listArchiveDates() REMOVED (2026-08-26). It produced its summaries by
+// JSON.parsing every day file -- 175 files / 1.2 GB on the real corpus, and 1.2 GB
+// heap / 2.3 GB RSS for the single largest day alone -- on the process that also
+// runs the wearer's live session. The /api/archive route now reads the sidecar
+// index (archive-index.ts) instead. Deleted rather than deprecated: an uncalled
+// landmine is still a landmine, and the next caller would not know.
+
 /** Get chat summaries for a specific day */
 export function getArchiveChats(date: string): ArchiveChatSummary[] {
   const archive = loadArchive(date)

package/server/routes/archive.ts

@@ -1,7 +1,9 @@
 // Archive endpoints — daily conversation archive for glasses history browser
 import { Router } from 'express'
-import { listArchiveDates, loadArchive, getArchiveChats, getArchiveDayMessages, appendToArchive } from '../lib/archive.js'
+import { listArchiveDateStrings, archiveDir, archiveIndexPath, loadArchive, getArchiveChats, getArchiveDayMessages, appendToArchive } from '../lib/archive.js'
 import { getArchiveChatMessagesNumbered } from './message-ref.js'
+import { searchArchive, MAX_LIMIT, DEFAULT_LIMIT } from '../lib/archive-search.js'
+import { refreshArchiveIndex } from '../lib/archive-index.js'
 import { getActiveSessions } from '../lib/conversation.js'
 
 export const archiveRouter = Router()
@@ -21,9 +23,19 @@ archiveRouter.param('date', (req, res, next, date) => {
 })
 
 // GET /api/archive — list all archive dates with summaries
-archiveRouter.get('/archive', (_req, res) => {
-  const archives = listArchiveDates()
-  res.json({ archives })
+archiveRouter.get('/archive', async (_req, res) => {
+  // Index-backed. The previous implementation parsed every day file to reach four
+  // summary fields; see archive-index.ts for the measurements that killed it.
+  const { entries, rebuilt, fromCache } = await refreshArchiveIndex(archiveDir(), archiveIndexPath())
+  res.json({
+    archives: entries.map(e => ({
+      date: e.date,
+      summary: e.summary,
+      chatCount: e.chatCount,
+      exchangeCount: e.exchangeCount,
+    })),
+    index: { rebuilt: rebuilt.length, fromCache },
+  })
 })
 
 // POST /api/archive/now — snapshot active sessions into today's archive (non-destructive)
@@ -44,6 +56,46 @@ archiveRouter.post('/archive/now', async (_req, res) => {
   res.json({ archived, date: todayDate })
 })
 
+// GET /api/archive/search — literal text search across archived days.
+//
+// REGISTRATION ORDER IS LOAD-BEARING. It must sit ABOVE /archive/:date: Express
+// matches in order, so declared after it this path arrives as :date === 'search'
+// and the param validator rejects it with 400 "Invalid date". That is exactly why
+// /api/archive/dates has always looked like an empty archive.
+//
+// The scan never parses a day file — see archive-search.ts for why (one real day
+// costs 1.2 GB of heap to materialise). Hits are attributed to a DATE plus text
+// snippets; callers open /archive/:date/chats for structure.
+archiveRouter.get('/archive/search', async (req, res) => {
+  const q = typeof req.query.q === 'string' ? req.query.q : ''
+  const from = typeof req.query.from === 'string' ? req.query.from : undefined
+  const to = typeof req.query.to === 'string' ? req.query.to : undefined
+  const rawLimit = Number.parseInt(String(req.query.limit ?? ''), 10)
+  const limit = Number.isFinite(rawLimit) ? rawLimit : DEFAULT_LIMIT
+
+  for (const [name, value] of [['from', from], ['to', to]] as const) {
+    if (value !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(value)) {
+      res.status(400).json({ error: `Invalid ${name}`, expected: 'YYYY-MM-DD' })
+      return
+    }
+  }
+
+  const started = Date.now()
+  try {
+    const result = await searchArchive({
+      dir: archiveDir(),
+      dates: listArchiveDateStrings(),
+      query: q,
+      from,
+      to,
+      limit: Math.min(limit, MAX_LIMIT),
+    })
+    res.json({ query: q.trim(), ...result, elapsedMs: Date.now() - started })
+  } catch (error) {
+    res.status(400).json({ error: (error as Error).message })
+  }
+})
+
 // GET /api/archive/:date — full daily archive
 archiveRouter.get('/archive/:date', (req, res) => {
   const archive = loadArchive(req.params.date)