@gotcos/glasses-server 6.37.3 → 6.38.1

package/CHANGELOG.md

@@ -1,3 +1,55 @@
+## 6.38.1
+
+The conversation archive was quietly storing the same conversations over and over.
+
+FIXED: `appendToArchive` blind-appended, and the daily mirror re-archives every
+still-resident prior-day session at boot and every 24h without evicting it -- so
+each restart added another copy. Measured on a real install: 1.268 GB of archive
+of which 99.3% was duplicate; one 69 MB day file held a single conversation
+2,388 times. This also inflated the archive index's chat counts and archive
+search's match counts (both from 6.38.0) by up to ~2,200x on affected days.
+
+The merge now upserts on (sessionId, startedAt), replaces a chat that has grown,
+returns without rewriting when nothing changed, and self-heals a file written by
+the old code the next time it is touched.
+
+NEW: `server/scripts/repair-archive-duplicates.ts` -- a dry-run-by-default repair
+for day files the mirror no longer touches. Backs up every file before writing,
+refuses while the server is running, verifies by unique-chat count (never file
+size), and is idempotent.
+
+## 6.38.0
+
+Six months of archived conversation you can finally search.
+
+NEW: GET /api/archive/search?q=&from=&to=&limit=. The archive already held every
+day's conversations -- 175 day files spanning six months on a real install -- with
+no way to find anything in them.
+
+The scan never calls JSON.parse. Day sizes are wildly skewed: the median day is
+36 KB, but the largest measured is 343 MB of agent transcript, and materialising
+that one day costs 1.2 GB heap / 2.3 GB RSS. On a process that also runs the
+wearer's live session that is not affordable, so days are scanned as raw bytes
+through a stream. A hit is therefore attributed to a DATE plus surrounding text,
+not to a chat; open /archive/:date/chats for structure. A full 90-day scan
+measures at 2.4 s, so there is no index to build and nothing that can drift.
+
+FIXED: GET /api/archive no longer parses 1.2 GB to list days. It built {date,
+summary, chatCount, exchangeCount} by parsing EVERY day file and discarding the
+bodies -- one request away from a multi-gigabyte spike. It now reads a sidecar
+index keyed by each day's (size, mtimeMs). Measured on the real corpus: cold build
+2,208 ms at 248 MB RSS for all 175 days, warm read 1 ms. Both halves of the key
+matter -- size alone misses an in-place edit that preserves length.
+
+listArchiveDates() is deleted rather than deprecated. It had one caller, and an
+uncalled landmine is still a landmine.
+
+Route order is load-bearing: /archive/search is registered ABOVE /archive/:date,
+because Express matches in order and the date validator would otherwise reject it
+as :date === "search". That is the same trap that has always made
+/api/archive/dates look like an empty archive. Older servers still exhibit it, so
+clients should read a 400 "Invalid date" from the search path as "route absent".
+
 ## 6.37.3
 
 Speaker ID works out of the box. A silent capture stops retrying forever. Live

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.37.3",
-  "description": "COS Glasses — self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
+  "version": "6.38.1",
+  "description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {
     "glasses-server": "bin/cli.cjs",

package/server/lib/archive-index.ts

@@ -0,0 +1,186 @@
+// A sidecar index of per-day archive summaries.
+//
+// WHY THIS EXISTS. `listArchiveDates()` produces {date, summary, chatCount,
+// exchangeCount} by JSON.parsing EVERY day file and discarding the bodies. On the
+// real corpus that is 175 files / 1.2 GB, and the single largest day (2026-07-30,
+// 343 MB) measures at 1.2 GB heap / 2.3 GB RSS to materialise on its own. That is
+// one GET away from a multi-gigabyte spike on the same process that runs the
+// wearer's live glasses session.
+//
+// The index holds exactly the fields that listing needs, keyed by the day's own
+// (size, mtimeMs). A day whose bytes have not changed is never reopened, so the
+// steady-state cost of a listing is one readdir plus one small JSON read.
+//
+// INVALIDATION IS THE WHOLE CONTRACT. size+mtimeMs is what makes this honest: an
+// archive that is appended to changes both, so a stale entry cannot survive a
+// write. Entries for vanished days are dropped rather than kept, because a listing
+// that names a file nobody can open is worse than one that omits it.
+import { createReadStream, existsSync, readFileSync, readdirSync, statSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { atomicWriteFileSync } from './atomic-fs.js'
+
+export const ARCHIVE_INDEX_SCHEMA = 1
+
+export interface ArchiveIndexEntry {
+  date: string
+  size: number
+  mtimeMs: number
+  chatCount: number
+  exchangeCount: number
+  summary: string | null
+}
+
+interface ArchiveIndexFile {
+  schemaVersion: number
+  entries: Record<string, ArchiveIndexEntry>
+}
+
+const DATE_RE = /^\d{4}-\d{2}-\d{2}$/
+/** Chats carry exactly one `exchangeCount` each and the key appears nowhere else
+ *  in the shape. Verified against parsed truth on six real days (8/28/93/47/11/90
+ *  chats) before this was relied on. */
+const EXCHANGE_COUNT_RE = /"exchangeCount":\s*(\d+)/g
+const CARRY_CHARS = 64
+
+export function emptyIndex(): ArchiveIndexFile {
+  return { schemaVersion: ARCHIVE_INDEX_SCHEMA, entries: {} }
+}
+
+export function readIndexFile(indexPath: string): ArchiveIndexFile {
+  try {
+    if (!existsSync(indexPath)) return emptyIndex()
+    const parsed = JSON.parse(readFileSync(indexPath, 'utf8')) as ArchiveIndexFile
+    // A schema bump invalidates wholesale rather than trying to migrate: the index
+    // is a cache, and rebuilding it costs one scan.
+    if (parsed?.schemaVersion !== ARCHIVE_INDEX_SCHEMA || typeof parsed.entries !== 'object') {
+      return emptyIndex()
+    }
+    return { schemaVersion: ARCHIVE_INDEX_SCHEMA, entries: parsed.entries ?? {} }
+  } catch {
+    return emptyIndex() // a corrupt cache must never take the listing down
+  }
+}
+
+/** Pull the top-level `summary` string without materialising the day. It sits in
+ *  the opening object, so the first chunk is enough; a day that hides it deeper
+ *  simply reports null rather than costing a full read. */
+export function extractSummary(head: string): string | null {
+  const m = /"summary"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(head)
+  if (!m) return null
+  try {
+    return JSON.parse(`"${m[1]}"`) as string
+  } catch {
+    return null
+  }
+}
+
+/** Stream one day and count chats + exchanges. Never parses. */
+export async function summariseDayFile(filePath: string): Promise<{ chatCount: number; exchangeCount: number; summary: string | null }> {
+  let chatCount = 0
+  let exchangeCount = 0
+  let summary: string | null = null
+  let carry = ''
+  let first = true
+
+  await new Promise<void>((done, fail) => {
+    const stream = createReadStream(filePath, { encoding: 'utf8', highWaterMark: 1 << 20 })
+    stream.on('data', (chunk: string | Buffer) => {
+      const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8')
+      const hay = carry + text
+      if (first) {
+        summary = extractSummary(hay)
+        first = false
+      }
+      EXCHANGE_COUNT_RE.lastIndex = 0
+      let m: RegExpExecArray | null
+      while ((m = EXCHANGE_COUNT_RE.exec(hay)) !== null) {
+        chatCount++
+        exchangeCount += Number(m[1])
+      }
+      // Carry enough to reassemble a key/value split across the boundary. Matches
+      // inside the carry are not double-counted because the carry is short and the
+      // next pass starts from it, not from the whole previous chunk.
+      carry = hay.slice(Math.max(0, hay.length - CARRY_CHARS))
+    })
+    stream.on('error', fail)
+    stream.on('end', () => done())
+  })
+
+  return { chatCount, exchangeCount, summary }
+}
+
+export interface RefreshResult {
+  entries: ArchiveIndexEntry[]
+  rebuilt: string[]
+  dropped: string[]
+  fromCache: number
+}
+
+/**
+ * Bring the index in line with what is on disk and return the listing, newest
+ * first. Only days whose (size, mtimeMs) changed are reopened.
+ */
+export async function refreshArchiveIndex(dir: string, indexPath: string): Promise<RefreshResult> {
+  const file = readIndexFile(indexPath)
+  const rebuilt: string[] = []
+  const dropped: string[] = []
+  let fromCache = 0
+
+  let present: string[] = []
+  try {
+    present = readdirSync(dir)
+      .filter(f => f.endsWith('.json'))
+      .map(f => f.slice(0, -'.json'.length))
+      .filter(d => DATE_RE.test(d))
+  } catch {
+    return { entries: [], rebuilt, dropped, fromCache }
+  }
+  const presentSet = new Set(present)
+
+  for (const date of Object.keys(file.entries)) {
+    if (!presentSet.has(date)) {
+      delete file.entries[date]
+      dropped.push(date)
+    }
+  }
+
+  for (const date of present) {
+    const filePath = resolve(dir, `${date}.json`)
+    let st
+    try {
+      st = statSync(filePath)
+    } catch {
+      continue
+    }
+    const cached = file.entries[date]
+    if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) {
+      fromCache++
+      continue
+    }
+    try {
+      const s = await summariseDayFile(filePath)
+      file.entries[date] = {
+        date,
+        size: st.size,
+        mtimeMs: st.mtimeMs,
+        chatCount: s.chatCount,
+        exchangeCount: s.exchangeCount,
+        summary: s.summary,
+      }
+      rebuilt.push(date)
+    } catch {
+      continue // an unreadable day must not abort the listing
+    }
+  }
+
+  if (rebuilt.length > 0 || dropped.length > 0) {
+    try {
+      atomicWriteFileSync(indexPath, `${JSON.stringify(file)}\n`, { mode: 0o600 })
+    } catch {
+      // A cache that cannot persist still serves this call correctly.
+    }
+  }
+
+  const entries = Object.values(file.entries).sort((a, b) => (a.date < b.date ? 1 : a.date > b.date ? -1 : 0))
+  return { entries, rebuilt, dropped, fromCache }
+}

package/server/lib/archive-search.ts

@@ -0,0 +1,195 @@
+// Literal text search across the daily conversation archive.
+//
+// WHY THIS NEVER CALLS JSON.parse. The archive is 175 day files spanning six
+// months, and their sizes are wildly skewed: the median day is 36 KB but
+// 2026-07-30 is 327 MB of agent transcript (283,356 exchanges). Parsing that
+// into JS objects would cost multiple GB of heap on a server that also runs the
+// wearer's live glasses session. A day file is therefore scanned as raw bytes
+// and never materialised.
+//
+// The cost of that choice is attribution: a hit reports its DATE and a text
+// snippet, not a chat index. Callers open the day through the existing
+// /archive/:date/chats routes for structure. Day-level attribution is what a
+// stream can give safely, and the existing routes already cover the rest.
+//
+// Measured on the real corpus: a full literal scan of a 90-day window is ~2.4s,
+// so this needs no index, no background build, and nothing that can drift out of
+// sync with the archive itself.
+import { createReadStream } from 'node:fs'
+import { stat } from 'node:fs/promises'
+import { resolve } from 'node:path'
+
+export const MIN_QUERY_CHARS = 2
+export const DEFAULT_LIMIT = 50
+export const MAX_LIMIT = 200
+export const SNIPPET_RADIUS = 120
+/** Per-day snippet cap. A term appearing 40,000 times in one 327 MB day must not
+ *  return 40,000 snippets; the count still reports the true total. */
+export const MAX_SNIPPETS_PER_DAY = 3
+const CHUNK_BYTES = 1 << 20
+
+export interface ArchiveSearchHit {
+  date: string
+  matches: number
+  snippets: string[]
+}
+
+export interface ArchiveSearchResult {
+  hits: ArchiveSearchHit[]
+  scannedDays: number
+  bytesScanned: number
+  truncated: boolean
+}
+
+/** YYYY-MM-DD, the archive's own filename contract. Lexical order IS date order,
+ *  which is what makes the range filter a string compare. */
+export function isArchiveDate(value: unknown): value is string {
+  return typeof value === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value)
+}
+
+export function datesInRange(dates: string[], from?: string, to?: string): string[] {
+  return dates
+    .filter(isArchiveDate)
+    .filter(d => (from ? d >= from : true) && (to ? d <= to : true))
+    .sort()
+    .reverse()
+}
+
+function cleanSnippet(raw: string): string {
+  // The scan runs over JSON source, so a snippet arrives carrying escape
+  // sequences and structural punctuation. Unescape the common ones and collapse
+  // whitespace so the caller can render it as prose.
+  return raw
+    .replace(/\\n/g, ' ')
+    .replace(/\\t/g, ' ')
+    .replace(/\\"/g, '"')
+    .replace(/\\\\/g, '\\')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+/**
+ * Count matches in `hayLower` but cut snippets from `hayOriginal`.
+ *
+ * Matching has to happen on the lowercased text for case-insensitivity, but a
+ * snippet cut from that text renders as all-lowercase prose -- "chelsie owes the
+ * sales-vs-education split" -- which reads like broken data in a UI. The offsets
+ * line up because toLowerCase() is length-preserving for effectively all real
+ * archive text; where it is NOT (a handful of exotic codepoints expand), the
+ * length check catches the drift and falls back to the lowered slice rather than
+ * cutting at a wrong offset.
+ */
+function collectSnippets(
+  hayLower: string,
+  hayOriginal: string,
+  needle: string,
+  out: string[],
+  cap: number,
+): number {
+  const aligned = hayLower.length === hayOriginal.length
+  let found = 0
+  let i = hayLower.indexOf(needle)
+  while (i !== -1) {
+    found++
+    if (out.length < cap) {
+      const start = Math.max(0, i - SNIPPET_RADIUS)
+      const end = Math.min(hayLower.length, i + needle.length + SNIPPET_RADIUS)
+      const snippet = cleanSnippet((aligned ? hayOriginal : hayLower).slice(start, end))
+      if (snippet) out.push(snippet)
+    }
+    i = hayLower.indexOf(needle, i + needle.length)
+  }
+  return found
+}
+
+/**
+ * Scan ONE day file. Returns null when the term never appears.
+ *
+ * The overlap is the whole reason this is a separate function with its own test.
+ * A 1 MB chunk boundary can fall in the middle of the search term, and a naive
+ * per-chunk indexOf silently misses it — the failure mode is a search that works
+ * in every test fixture and quietly loses hits on real multi-megabyte days.
+ * Each chunk is therefore prefixed with the last (needle.length - 1) characters
+ * of the previous one, and matches inside that carried prefix are not counted
+ * twice because the search resumes past the needle.
+ */
+export async function searchArchiveFile(
+  filePath: string,
+  needleLower: string,
+): Promise<{ matches: number; snippets: string[]; bytes: number } | null> {
+  let matches = 0
+  const snippets: string[] = []
+  let bytes = 0
+  let carry = ''
+  const overlap = Math.max(0, needleLower.length - 1)
+
+  await new Promise<void>((resolveDone, rejectDone) => {
+    const stream = createReadStream(filePath, { encoding: 'utf8', highWaterMark: CHUNK_BYTES })
+    stream.on('data', (chunk: string | Buffer) => {
+      const text = typeof chunk === 'string' ? chunk : chunk.toString('utf8')
+      bytes += Buffer.byteLength(text, 'utf8')
+      const hay = carry + text
+      matches += collectSnippets(hay.toLowerCase(), hay, needleLower, snippets, MAX_SNIPPETS_PER_DAY)
+      carry = hay.slice(Math.max(0, hay.length - overlap))
+    })
+    stream.on('error', rejectDone)
+    stream.on('end', () => resolveDone())
+  })
+
+  return matches > 0 ? { matches, snippets, bytes } : null
+}
+
+/**
+ * Search a set of archive days, newest first, stopping once `limit` days have
+ * matched. Days are scanned in order so an early-exit favours recent history,
+ * which is what a person browsing "what did I say about X" actually wants.
+ */
+export async function searchArchive(opts: {
+  dir: string
+  dates: string[]
+  query: string
+  from?: string
+  to?: string
+  limit?: number
+}): Promise<ArchiveSearchResult> {
+  const query = (opts.query ?? '').trim()
+  if (query.length < MIN_QUERY_CHARS) {
+    throw new Error(`query must be at least ${MIN_QUERY_CHARS} characters`)
+  }
+  const limit = Math.min(Math.max(1, opts.limit ?? DEFAULT_LIMIT), MAX_LIMIT)
+  const needleLower = query.toLowerCase()
+  const candidates = datesInRange(opts.dates, opts.from, opts.to)
+
+  const hits: ArchiveSearchHit[] = []
+  let scannedDays = 0
+  let bytesScanned = 0
+  let truncated = false
+
+  for (const date of candidates) {
+    if (hits.length >= limit) {
+      truncated = true
+      break
+    }
+    const filePath = resolve(opts.dir, `${date}.json`)
+    try {
+      await stat(filePath)
+    } catch {
+      continue // listed but absent: a rotation mid-scan is not an error
+    }
+    scannedDays++
+    try {
+      const found = await searchArchiveFile(filePath, needleLower)
+      if (found) {
+        bytesScanned += found.bytes
+        hits.push({ date, matches: found.matches, snippets: found.snippets })
+      } else {
+        // still counts toward bytes so callers can report honest scan cost
+        bytesScanned += (await stat(filePath)).size
+      }
+    } catch {
+      continue // an unreadable day must not abort the whole search
+    }
+  }
+
+  return { hits, scannedDays, bytesScanned, truncated }
+}

package/server/lib/archive.ts

@@ -94,7 +94,7 @@ export function loadArchive(date: string): DailyArchive | null {
   }
   if (result.status === 'missing') return null
   // Defense: a valid-JSON but wrong-shape day file (no chats[]) would make the
-  // readers throw 500 AND drop listArchiveDates into its catch → the whole
+  // readers throw 500 AND drop the archive listing into its catch → the whole
   // Message History list vanishes on one bad file. Coerce to an empty day.
   const data = result.data
   if (data && !Array.isArray(data.chats)) data.chats = []
@@ -343,12 +343,59 @@ export async function appendToArchive(
     }
 
     if (existing) {
-      // Merge: re-number chat IDs
-      const nextId = existing.chats.length
-      for (let i = 0; i < newChats.length; i++) {
-        newChats[i].id = nextId + i
+      // UPSERT, never blind-append.
+      //
+      // `runDailyArchiveMirror` walks every session still resident in memory, skips
+      // only TODAY's, and archives the rest -- at boot and every 24h -- WITHOUT
+      // evicting them from the map. This merge used to be
+      // `existing.chats.push(...newChats)`, so a session that stayed resident gained
+      // one more copy of itself in its day file on every single restart, forever.
+      //
+      // Measured on the live corpus before this fix: 1.28 GB across 176 day files, of
+      // which ~1.26 GB (98%) was duplicates, 31 files affected. 2026-07-30.json was
+      // 343 MB holding 4,421 chats belonging to exactly TWO sessions -- ~2,210 copies
+      // of roughly 0.16 MB of real content. 2026-07-28.json was 69 MB of ONE
+      // conversation, 2,388 times over. That is why the archive index and archive
+      // search (both shipped in 6.38.0) reported inflated counts, and why a day file
+      // ever reached a size that costs 1.2 GB of heap to parse.
+      //
+      // IDENTITY IS (sessionId, startedAt). `startedAt` is the chat's first exchange
+      // timestamp, so it survives re-archiving. `id` does NOT -- it is renumbered on
+      // every merge, which is precisely why the old code could never recognise a chat
+      // it had already written. Verified against the real corpus: this key collapses
+      // 2026-07-30 from 4,421 chats to 2, and leaves an unaffected day (2026-08-17,
+      // 12 chats) at exactly 12 -- so it does not over-merge distinct conversations.
+      const keyOf = (c: ArchivedChat): string => `${c.sessionId}:${c.startedAt}`
+      const byKey = new Map<string, ArchivedChat>()
+      for (const chat of existing.chats) {
+        const key = keyOf(chat)
+        const prior = byKey.get(key)
+        // Self-heal: a file written before this fix already contains duplicates.
+        // Keep the most complete copy rather than the first one encountered.
+        if (!prior || chat.exchangeCount > prior.exchangeCount) byKey.set(key, chat)
       }
-      existing.chats.push(...newChats)
+      const hadDuplicates = byKey.size !== existing.chats.length
+
+      let changed = false
+      for (const incoming of newChats) {
+        const key = keyOf(incoming)
+        const prior = byKey.get(key)
+        // REPLACE rather than skip: a session archived yesterday with 5 exchanges that
+        // now holds 8 must update, not be discarded as "already seen".
+        if (!prior || incoming.exchangeCount > prior.exchangeCount) {
+          byKey.set(key, incoming)
+          changed = true
+        }
+      }
+
+      // Nothing new and nothing to repair: return WITHOUT rewriting. Re-serialising a
+      // large day file to produce identical bytes is pure cost on the process that
+      // also records the wearer's live session, and it is the common case at boot.
+      if (!changed && !hadDuplicates) return
+
+      const merged = [...byKey.values()].sort((a, b) => a.startedAt - b.startedAt)
+      merged.forEach((chat, i) => { chat.id = i })
+      existing.chats = merged
       existing.summary = await generateDaySummary(existing.chats, opts.skipLLM)
       existing.archivedAt = new Date().toISOString()
       saveArchive(existing)
@@ -367,30 +414,51 @@ export async function appendToArchive(
 // ── Query functions ─────────────────────────────────────────
 
 /** List all archive dates with summaries */
-export function listArchiveDates(): ArchiveDateSummary[] {
+/** Sidecar index location. Deliberately a SIBLING of the archive directory, not
+ *  a file inside it: anything living in that directory has to be excluded by
+ *  every readdir filter forever, and one missed filter turns the cache into a
+ *  phantom "day". */
+export function archiveIndexPath(): string {
+  return dataPath('archive-index.json')
+}
+
+/** The archive directory, for readers that must NOT materialise a day file. */
+export function archiveDir(): string {
+  return ensureArchiveDir()
+}
+
+/**
+ * Date strings only, straight from readdir — no file is opened.
+ *
+ * The richer per-day summaries (chat and exchange counts) come from the sidecar
+ * index in archive-index.ts. The listing this replaced reached them by
+ * JSON.parsing EVERY day file. Measured on the real corpus that is 1.2 GB across
+ * 175 files, and the single largest day (2026-07-30, 343 MB) costs 1.2 GB heap /
+ * 2.3 GB RSS to parse on its own. Anything that only needs to know WHICH days
+ * exist must use this instead, on a server that is also running the wearer's live
+ * session.
+ */
+export function listArchiveDateStrings(): string[] {
   try {
     ensureArchiveDir()
-    const files = readdirSync(ARCHIVE_DIR)
+    return readdirSync(ARCHIVE_DIR)
       .filter(f => f.endsWith('.json'))
+      .map(f => f.slice(0, -'.json'.length))
+      .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
       .sort()
-      .reverse() // newest first
-
-    return files.map(f => {
-      const date = f.replace('.json', '')
-      const archive = loadArchive(date)
-      if (!archive) return null
-      return {
-        date: archive.date,
-        summary: archive.summary,
-        chatCount: archive.chats.length,
-        exchangeCount: archive.chats.reduce((sum, c) => sum + c.exchangeCount, 0),
-      }
-    }).filter(Boolean) as ArchiveDateSummary[]
+      .reverse()
   } catch {
     return []
   }
 }
 
+// listArchiveDates() REMOVED (2026-08-26). It produced its summaries by
+// JSON.parsing every day file -- 175 files / 1.2 GB on the real corpus, and 1.2 GB
+// heap / 2.3 GB RSS for the single largest day alone -- on the process that also
+// runs the wearer's live session. The /api/archive route now reads the sidecar
+// index (archive-index.ts) instead. Deleted rather than deprecated: an uncalled
+// landmine is still a landmine, and the next caller would not know.
+
 /** Get chat summaries for a specific day */
 export function getArchiveChats(date: string): ArchiveChatSummary[] {
   const archive = loadArchive(date)

package/server/routes/archive.ts

@@ -1,7 +1,9 @@
 // Archive endpoints — daily conversation archive for glasses history browser
 import { Router } from 'express'
-import { listArchiveDates, loadArchive, getArchiveChats, getArchiveDayMessages, appendToArchive } from '../lib/archive.js'
+import { listArchiveDateStrings, archiveDir, archiveIndexPath, loadArchive, getArchiveChats, getArchiveDayMessages, appendToArchive } from '../lib/archive.js'
 import { getArchiveChatMessagesNumbered } from './message-ref.js'
+import { searchArchive, MAX_LIMIT, DEFAULT_LIMIT } from '../lib/archive-search.js'
+import { refreshArchiveIndex } from '../lib/archive-index.js'
 import { getActiveSessions } from '../lib/conversation.js'
 
 export const archiveRouter = Router()
@@ -21,9 +23,19 @@ archiveRouter.param('date', (req, res, next, date) => {
 })
 
 // GET /api/archive — list all archive dates with summaries
-archiveRouter.get('/archive', (_req, res) => {
-  const archives = listArchiveDates()
-  res.json({ archives })
+archiveRouter.get('/archive', async (_req, res) => {
+  // Index-backed. The previous implementation parsed every day file to reach four
+  // summary fields; see archive-index.ts for the measurements that killed it.
+  const { entries, rebuilt, fromCache } = await refreshArchiveIndex(archiveDir(), archiveIndexPath())
+  res.json({
+    archives: entries.map(e => ({
+      date: e.date,
+      summary: e.summary,
+      chatCount: e.chatCount,
+      exchangeCount: e.exchangeCount,
+    })),
+    index: { rebuilt: rebuilt.length, fromCache },
+  })
 })
 
 // POST /api/archive/now — snapshot active sessions into today's archive (non-destructive)
@@ -44,6 +56,46 @@ archiveRouter.post('/archive/now', async (_req, res) => {
   res.json({ archived, date: todayDate })
 })
 
+// GET /api/archive/search — literal text search across archived days.
+//
+// REGISTRATION ORDER IS LOAD-BEARING. It must sit ABOVE /archive/:date: Express
+// matches in order, so declared after it this path arrives as :date === 'search'
+// and the param validator rejects it with 400 "Invalid date". That is exactly why
+// /api/archive/dates has always looked like an empty archive.
+//
+// The scan never parses a day file — see archive-search.ts for why (one real day
+// costs 1.2 GB of heap to materialise). Hits are attributed to a DATE plus text
+// snippets; callers open /archive/:date/chats for structure.
+archiveRouter.get('/archive/search', async (req, res) => {
+  const q = typeof req.query.q === 'string' ? req.query.q : ''
+  const from = typeof req.query.from === 'string' ? req.query.from : undefined
+  const to = typeof req.query.to === 'string' ? req.query.to : undefined
+  const rawLimit = Number.parseInt(String(req.query.limit ?? ''), 10)
+  const limit = Number.isFinite(rawLimit) ? rawLimit : DEFAULT_LIMIT
+
+  for (const [name, value] of [['from', from], ['to', to]] as const) {
+    if (value !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(value)) {
+      res.status(400).json({ error: `Invalid ${name}`, expected: 'YYYY-MM-DD' })
+      return
+    }
+  }
+
+  const started = Date.now()
+  try {
+    const result = await searchArchive({
+      dir: archiveDir(),
+      dates: listArchiveDateStrings(),
+      query: q,
+      from,
+      to,
+      limit: Math.min(limit, MAX_LIMIT),
+    })
+    res.json({ query: q.trim(), ...result, elapsedMs: Date.now() - started })
+  } catch (error) {
+    res.status(400).json({ error: (error as Error).message })
+  }
+})
+
 // GET /api/archive/:date — full daily archive
 archiveRouter.get('/archive/:date', (req, res) => {
   const archive = loadArchive(req.params.date)

package/server/scripts/repair-archive-duplicates.ts

@@ -0,0 +1,181 @@
+#!/usr/bin/env tsx
+// Repair day files written by the pre-upsert archive merge.
+//
+//   npx tsx server/scripts/repair-archive-duplicates.ts            # dry run, writes nothing
+//   npx tsx server/scripts/repair-archive-duplicates.ts --apply    # rewrites, after backing up
+//
+// WHAT WENT WRONG. `runDailyArchiveMirror` re-archives every session still
+// resident in memory, skipping only today's, at boot and every 24h -- without
+// evicting it. `appendToArchive` merged with a blind `existing.chats.push(...)`.
+// So a session that stayed resident gained one more copy of itself in its day
+// file on every restart. Measured before the fix: 1.28 GB across 176 day files,
+// ~1.26 GB of it duplicates. One 69 MB file held ONE conversation 2,388 times.
+//
+// The upsert in archive.ts fixes new writes AND self-heals a file the next time
+// it is touched -- so most affected days repair themselves once the mirror
+// revisits them. This script exists for the remainder: days whose sessions have
+// since been evicted, which nothing will ever touch again.
+//
+// THIS SCRIPT IMPORTS NOTHING FROM THE SERVER. `archive.ts` runs
+// checkYesterdayArchive() at module scope, so importing it would start archive
+// work while we are rewriting the archive. The atomic write below is inlined for
+// the same reason. Nothing here has an effect until --apply.
+
+import { execFileSync } from 'node:child_process'
+import { copyFileSync, existsSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+
+const APPLY = process.argv.includes('--apply')
+const DATE_RE = /^\d{4}-\d{2}-\d{2}$/
+
+interface Chat {
+  id: number
+  sessionId: string
+  startedAt: number
+  exchangeCount: number
+  [k: string]: unknown
+}
+interface Day { date: string; summary: string; chats: Chat[]; archivedAt: string; [k: string]: unknown }
+
+function archiveDirPath(): string {
+  const base = process.env.COS_DATA_DIR ?? join(homedir(), '.cos-glasses', 'data')
+  return resolve(base, 'archive')
+}
+
+/** Identity of a chat. `startedAt` is its first exchange's timestamp, so it
+ *  survives re-archiving. `id` does NOT -- it is renumbered on every merge,
+ *  which is exactly why the old code could never see a duplicate. */
+const keyOf = (c: Chat): string => `${c.sessionId}:${c.startedAt}`
+
+/** Collapse duplicates, keeping the most complete copy of each chat. Pure. */
+export function dedupeChats(chats: Chat[]): { kept: Chat[]; removed: number } {
+  const byKey = new Map<string, Chat>()
+  for (const chat of chats) {
+    const prior = byKey.get(keyOf(chat))
+    if (!prior || (chat.exchangeCount ?? 0) > (prior.exchangeCount ?? 0)) byKey.set(keyOf(chat), chat)
+  }
+  const kept = [...byKey.values()].sort((a, b) => a.startedAt - b.startedAt)
+  kept.forEach((c, i) => { c.id = i })
+  return { kept, removed: chats.length - kept.length }
+}
+
+function atomicWrite(path: string, data: string): void {
+  // Inlined rather than imported: see the header note about module-scope effects.
+  const tmp = `${path}.repair-tmp`
+  writeFileSync(tmp, data, { encoding: 'utf8', mode: 0o600 })
+  renameSync(tmp, path)
+}
+
+function main(): void {
+  const dir = archiveDirPath()
+  if (!existsSync(dir)) {
+    console.error(`No archive directory at ${dir}`)
+    process.exit(2)
+  }
+
+  // A concurrent appendToArchive would race this rewrite. The in-process archive
+  // lock cannot be taken from outside the server, so the only safe answer is to
+  // refuse while it is up rather than to hope the window is small.
+  // Only the DEFAULT data dir is at risk: that is the one the running server writes
+  // to. Pointed at a scratch copy, there is nothing to race, and refusing there
+  // would block the very rehearsal this script deserves before it touches real data.
+  const isLiveDataDir = process.env.COS_DATA_DIR === undefined
+  if (APPLY && isLiveDataDir && serverIsUp()) {
+    console.error('The COS server is listening on 127.0.0.1:3141.')
+    console.error('Stop it through COS Control before repairing, then re-run.')
+    console.error('Refusing to rewrite archive files while the server may write to them.')
+    process.exit(3)
+  }
+
+  const files = readdirSync(dir)
+    .filter(f => f.endsWith('.json') && DATE_RE.test(f.slice(0, -5)))
+    .sort()
+
+  let affected = 0
+  let chatsBefore = 0
+  let chatsAfter = 0
+  let bytesBefore = 0
+  let bytesAfter = 0
+
+  for (const file of files) {
+    const path = join(dir, file)
+    const size = statSync(path).size
+
+    let day: Day
+    try {
+      day = JSON.parse(readFileSync(path, 'utf8')) as Day
+    } catch (err) {
+      console.error(`  SKIP ${file} — unreadable: ${(err as Error).message.slice(0, 80)}`)
+      continue
+    }
+    if (!Array.isArray(day.chats) || day.chats.length === 0) continue
+
+    const { kept, removed } = dedupeChats(day.chats)
+    if (removed === 0) continue
+
+    affected++
+    chatsBefore += day.chats.length
+    chatsAfter += kept.length
+    bytesBefore += size
+
+    const before = day.chats.length
+    day.chats = kept
+    const serialised = `${JSON.stringify(day, null, 2)}\n`
+    bytesAfter += Buffer.byteLength(serialised, 'utf8')
+
+    console.log(
+      `  ${APPLY ? 'REPAIR' : 'would repair'} ${file}  ` +
+      `${(size / 1e6).toFixed(1)} MB → ${(Buffer.byteLength(serialised, 'utf8') / 1e6).toFixed(1)} MB  ` +
+      `chats ${before} → ${kept.length}  (-${removed})`,
+    )
+
+    if (APPLY) {
+      // Back up BEFORE writing. This is user conversation history; a bad rewrite
+      // with no copy is unrecoverable.
+      const backup = `${path}.bak-${Date.now()}`
+      copyFileSync(path, backup)
+      atomicWrite(path, serialised)
+
+      // Verify by UNIQUE CHAT COUNT, never by file size -- size is the metric the
+      // bug distorted, so shrinkage proves nothing about correctness.
+      const reread = JSON.parse(readFileSync(path, 'utf8')) as Day
+      const uniq = new Set(reread.chats.map(keyOf)).size
+      if (reread.chats.length !== kept.length || uniq !== kept.length) {
+        console.error(`  FAILED verification on ${file}; original preserved at ${backup}`)
+        process.exit(4)
+      }
+    }
+  }
+
+  const summary = {
+    mode: APPLY ? 'applied' : 'dry-run',
+    filesScanned: files.length,
+    filesAffected: affected,
+    chats: { before: chatsBefore, after: chatsAfter, removed: chatsBefore - chatsAfter },
+    bytes: { before: bytesBefore, after: bytesAfter, reclaimed: bytesBefore - bytesAfter },
+  }
+  console.log('')
+  console.log(JSON.stringify(summary, null, 2))
+  if (!APPLY && affected > 0) {
+    console.log('')
+    console.log('Nothing was written. Re-run with --apply to repair (each file is backed up first).')
+  }
+}
+
+function serverIsUp(): boolean {
+  try {
+    const out = execFileSync('/usr/sbin/lsof', ['-ti', ':3141'], { encoding: 'utf8', timeout: 5000 })
+    return out.trim().length > 0
+  } catch {
+    return false // lsof missing or nothing listening — do not block on an inconclusive probe
+  }
+}
+
+// Run ONLY when invoked directly. Importing this file (a test, or any tooling)
+// must not execute a repair or call process.exit -- the same module-scope hazard
+// this script refuses to inherit from archive.ts.
+const invokedDirectly = process.argv[1] !== undefined
+  && import.meta.url === pathToFileURL(process.argv[1]).href
+if (invokedDirectly) main()