@gotcos/glasses-server 6.38.0 → 6.38.1

package/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 6.38.1
+
+The conversation archive was quietly storing the same conversations over and over.
+
+FIXED: `appendToArchive` blind-appended, and the daily mirror re-archives every
+still-resident prior-day session at boot and every 24h without evicting it -- so
+each restart added another copy. Measured on a real install: 1.268 GB of archive
+of which 99.3% was duplicate; one 69 MB day file held a single conversation
+2,388 times. This also inflated the archive index's chat counts and archive
+search's match counts (both from 6.38.0) by up to ~2,200x on affected days.
+
+The merge now upserts on (sessionId, startedAt), replaces a chat that has grown,
+returns without rewriting when nothing changed, and self-heals a file written by
+the old code the next time it is touched.
+
+NEW: `server/scripts/repair-archive-duplicates.ts` -- a dry-run-by-default repair
+for day files the mirror no longer touches. Backs up every file before writing,
+refuses while the server is running, verifies by unique-chat count (never file
+size), and is idempotent.
+
 ## 6.38.0
 
 Six months of archived conversation you can finally search.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gotcos/glasses-server",
-  "version": "6.38.0",
+  "version": "6.38.1",
   "description": "COS Glasses \u2014 self-hosted AI heads-up-display server for Even G2 smart glasses, powered by Claude Code, Codex, or Cursor Agent CLI",
   "type": "module",
   "bin": {

package/server/lib/archive.ts

@@ -343,12 +343,59 @@ export async function appendToArchive(
     }
 
     if (existing) {
-      // Merge: re-number chat IDs
-      const nextId = existing.chats.length
-      for (let i = 0; i < newChats.length; i++) {
-        newChats[i].id = nextId + i
+      // UPSERT, never blind-append.
+      //
+      // `runDailyArchiveMirror` walks every session still resident in memory, skips
+      // only TODAY's, and archives the rest -- at boot and every 24h -- WITHOUT
+      // evicting them from the map. This merge used to be
+      // `existing.chats.push(...newChats)`, so a session that stayed resident gained
+      // one more copy of itself in its day file on every single restart, forever.
+      //
+      // Measured on the live corpus before this fix: 1.28 GB across 176 day files, of
+      // which ~1.26 GB (98%) was duplicates, 31 files affected. 2026-07-30.json was
+      // 343 MB holding 4,421 chats belonging to exactly TWO sessions -- ~2,210 copies
+      // of roughly 0.16 MB of real content. 2026-07-28.json was 69 MB of ONE
+      // conversation, 2,388 times over. That is why the archive index and archive
+      // search (both shipped in 6.38.0) reported inflated counts, and why a day file
+      // ever reached a size that costs 1.2 GB of heap to parse.
+      //
+      // IDENTITY IS (sessionId, startedAt). `startedAt` is the chat's first exchange
+      // timestamp, so it survives re-archiving. `id` does NOT -- it is renumbered on
+      // every merge, which is precisely why the old code could never recognise a chat
+      // it had already written. Verified against the real corpus: this key collapses
+      // 2026-07-30 from 4,421 chats to 2, and leaves an unaffected day (2026-08-17,
+      // 12 chats) at exactly 12 -- so it does not over-merge distinct conversations.
+      const keyOf = (c: ArchivedChat): string => `${c.sessionId}:${c.startedAt}`
+      const byKey = new Map<string, ArchivedChat>()
+      for (const chat of existing.chats) {
+        const key = keyOf(chat)
+        const prior = byKey.get(key)
+        // Self-heal: a file written before this fix already contains duplicates.
+        // Keep the most complete copy rather than the first one encountered.
+        if (!prior || chat.exchangeCount > prior.exchangeCount) byKey.set(key, chat)
       }
-      existing.chats.push(...newChats)
+      const hadDuplicates = byKey.size !== existing.chats.length
+
+      let changed = false
+      for (const incoming of newChats) {
+        const key = keyOf(incoming)
+        const prior = byKey.get(key)
+        // REPLACE rather than skip: a session archived yesterday with 5 exchanges that
+        // now holds 8 must update, not be discarded as "already seen".
+        if (!prior || incoming.exchangeCount > prior.exchangeCount) {
+          byKey.set(key, incoming)
+          changed = true
+        }
+      }
+
+      // Nothing new and nothing to repair: return WITHOUT rewriting. Re-serialising a
+      // large day file to produce identical bytes is pure cost on the process that
+      // also records the wearer's live session, and it is the common case at boot.
+      if (!changed && !hadDuplicates) return
+
+      const merged = [...byKey.values()].sort((a, b) => a.startedAt - b.startedAt)
+      merged.forEach((chat, i) => { chat.id = i })
+      existing.chats = merged
       existing.summary = await generateDaySummary(existing.chats, opts.skipLLM)
       existing.archivedAt = new Date().toISOString()
       saveArchive(existing)

package/server/scripts/repair-archive-duplicates.ts

@@ -0,0 +1,181 @@
+#!/usr/bin/env tsx
+// Repair day files written by the pre-upsert archive merge.
+//
+//   npx tsx server/scripts/repair-archive-duplicates.ts            # dry run, writes nothing
+//   npx tsx server/scripts/repair-archive-duplicates.ts --apply    # rewrites, after backing up
+//
+// WHAT WENT WRONG. `runDailyArchiveMirror` re-archives every session still
+// resident in memory, skipping only today's, at boot and every 24h -- without
+// evicting it. `appendToArchive` merged with a blind `existing.chats.push(...)`.
+// So a session that stayed resident gained one more copy of itself in its day
+// file on every restart. Measured before the fix: 1.28 GB across 176 day files,
+// ~1.26 GB of it duplicates. One 69 MB file held ONE conversation 2,388 times.
+//
+// The upsert in archive.ts fixes new writes AND self-heals a file the next time
+// it is touched -- so most affected days repair themselves once the mirror
+// revisits them. This script exists for the remainder: days whose sessions have
+// since been evicted, which nothing will ever touch again.
+//
+// THIS SCRIPT IMPORTS NOTHING FROM THE SERVER. `archive.ts` runs
+// checkYesterdayArchive() at module scope, so importing it would start archive
+// work while we are rewriting the archive. The atomic write below is inlined for
+// the same reason. Nothing here has an effect until --apply.
+
+import { execFileSync } from 'node:child_process'
+import { copyFileSync, existsSync, readFileSync, readdirSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+
+const APPLY = process.argv.includes('--apply')
+const DATE_RE = /^\d{4}-\d{2}-\d{2}$/
+
+interface Chat {
+  id: number
+  sessionId: string
+  startedAt: number
+  exchangeCount: number
+  [k: string]: unknown
+}
+interface Day { date: string; summary: string; chats: Chat[]; archivedAt: string; [k: string]: unknown }
+
+function archiveDirPath(): string {
+  const base = process.env.COS_DATA_DIR ?? join(homedir(), '.cos-glasses', 'data')
+  return resolve(base, 'archive')
+}
+
+/** Identity of a chat. `startedAt` is its first exchange's timestamp, so it
+ *  survives re-archiving. `id` does NOT -- it is renumbered on every merge,
+ *  which is exactly why the old code could never see a duplicate. */
+const keyOf = (c: Chat): string => `${c.sessionId}:${c.startedAt}`
+
+/** Collapse duplicates, keeping the most complete copy of each chat. Pure. */
+export function dedupeChats(chats: Chat[]): { kept: Chat[]; removed: number } {
+  const byKey = new Map<string, Chat>()
+  for (const chat of chats) {
+    const prior = byKey.get(keyOf(chat))
+    if (!prior || (chat.exchangeCount ?? 0) > (prior.exchangeCount ?? 0)) byKey.set(keyOf(chat), chat)
+  }
+  const kept = [...byKey.values()].sort((a, b) => a.startedAt - b.startedAt)
+  kept.forEach((c, i) => { c.id = i })
+  return { kept, removed: chats.length - kept.length }
+}
+
+function atomicWrite(path: string, data: string): void {
+  // Inlined rather than imported: see the header note about module-scope effects.
+  const tmp = `${path}.repair-tmp`
+  writeFileSync(tmp, data, { encoding: 'utf8', mode: 0o600 })
+  renameSync(tmp, path)
+}
+
+function main(): void {
+  const dir = archiveDirPath()
+  if (!existsSync(dir)) {
+    console.error(`No archive directory at ${dir}`)
+    process.exit(2)
+  }
+
+  // A concurrent appendToArchive would race this rewrite. The in-process archive
+  // lock cannot be taken from outside the server, so the only safe answer is to
+  // refuse while it is up rather than to hope the window is small.
+  // Only the DEFAULT data dir is at risk: that is the one the running server writes
+  // to. Pointed at a scratch copy, there is nothing to race, and refusing there
+  // would block the very rehearsal this script deserves before it touches real data.
+  const isLiveDataDir = process.env.COS_DATA_DIR === undefined
+  if (APPLY && isLiveDataDir && serverIsUp()) {
+    console.error('The COS server is listening on 127.0.0.1:3141.')
+    console.error('Stop it through COS Control before repairing, then re-run.')
+    console.error('Refusing to rewrite archive files while the server may write to them.')
+    process.exit(3)
+  }
+
+  const files = readdirSync(dir)
+    .filter(f => f.endsWith('.json') && DATE_RE.test(f.slice(0, -5)))
+    .sort()
+
+  let affected = 0
+  let chatsBefore = 0
+  let chatsAfter = 0
+  let bytesBefore = 0
+  let bytesAfter = 0
+
+  for (const file of files) {
+    const path = join(dir, file)
+    const size = statSync(path).size
+
+    let day: Day
+    try {
+      day = JSON.parse(readFileSync(path, 'utf8')) as Day
+    } catch (err) {
+      console.error(`  SKIP ${file} — unreadable: ${(err as Error).message.slice(0, 80)}`)
+      continue
+    }
+    if (!Array.isArray(day.chats) || day.chats.length === 0) continue
+
+    const { kept, removed } = dedupeChats(day.chats)
+    if (removed === 0) continue
+
+    affected++
+    chatsBefore += day.chats.length
+    chatsAfter += kept.length
+    bytesBefore += size
+
+    const before = day.chats.length
+    day.chats = kept
+    const serialised = `${JSON.stringify(day, null, 2)}\n`
+    bytesAfter += Buffer.byteLength(serialised, 'utf8')
+
+    console.log(
+      `  ${APPLY ? 'REPAIR' : 'would repair'} ${file}  ` +
+      `${(size / 1e6).toFixed(1)} MB → ${(Buffer.byteLength(serialised, 'utf8') / 1e6).toFixed(1)} MB  ` +
+      `chats ${before} → ${kept.length}  (-${removed})`,
+    )
+
+    if (APPLY) {
+      // Back up BEFORE writing. This is user conversation history; a bad rewrite
+      // with no copy is unrecoverable.
+      const backup = `${path}.bak-${Date.now()}`
+      copyFileSync(path, backup)
+      atomicWrite(path, serialised)
+
+      // Verify by UNIQUE CHAT COUNT, never by file size -- size is the metric the
+      // bug distorted, so shrinkage proves nothing about correctness.
+      const reread = JSON.parse(readFileSync(path, 'utf8')) as Day
+      const uniq = new Set(reread.chats.map(keyOf)).size
+      if (reread.chats.length !== kept.length || uniq !== kept.length) {
+        console.error(`  FAILED verification on ${file}; original preserved at ${backup}`)
+        process.exit(4)
+      }
+    }
+  }
+
+  const summary = {
+    mode: APPLY ? 'applied' : 'dry-run',
+    filesScanned: files.length,
+    filesAffected: affected,
+    chats: { before: chatsBefore, after: chatsAfter, removed: chatsBefore - chatsAfter },
+    bytes: { before: bytesBefore, after: bytesAfter, reclaimed: bytesBefore - bytesAfter },
+  }
+  console.log('')
+  console.log(JSON.stringify(summary, null, 2))
+  if (!APPLY && affected > 0) {
+    console.log('')
+    console.log('Nothing was written. Re-run with --apply to repair (each file is backed up first).')
+  }
+}
+
+function serverIsUp(): boolean {
+  try {
+    const out = execFileSync('/usr/sbin/lsof', ['-ti', ':3141'], { encoding: 'utf8', timeout: 5000 })
+    return out.trim().length > 0
+  } catch {
+    return false // lsof missing or nothing listening — do not block on an inconclusive probe
+  }
+}
+
+// Run ONLY when invoked directly. Importing this file (a test, or any tooling)
+// must not execute a repair or call process.exit -- the same module-scope hazard
+// this script refuses to inherit from archive.ts.
+const invokedDirectly = process.argv[1] !== undefined
+  && import.meta.url === pathToFileURL(process.argv[1]).href
+if (invokedDirectly) main()