@llmnesia/mcp 0.2.3 → 0.2.4

package/README.md

@@ -140,6 +140,30 @@ the [Chrome Web Store](https://chromewebstore.google.com/detail/llmnesia/leekfgb
 
 If nothing has written to the corpus yet, search results will be empty.
 
+### Local storage and compaction
+
+Conversation files are stored as losslessly compressed JSON. Search uses a
+contentless FTS5 token index: the canonical text is not duplicated inside the
+database, and result snippets are loaded from the canonical conversation only
+after SQLite has ranked the matches. No conversation or searchable token is
+dropped to save space.
+
+Current builds discard a clean browser-to-corpus transport payload after both
+the canonical conversation and active index have accepted it. Malformed input
+is kept under `inbox/quarantine/` for inspection. Older releases archived every
+successful payload and may also have left versioned indexes behind. Preview the
+exact recoverable space without writing anything:
+
+```bash
+npx -y @llmnesia/mcp@latest compact --dry-run
+```
+
+To apply it, fully quit connected desktop AI/MCP clients, then run `compact`
+without `--dry-run`. It asks for confirmation, verifies every compressed
+conversation by reading it back before removing the legacy JSON, builds the
+current index before deleting older derived versions, and never removes
+quarantined input.
+
 ## Tools
 
 Once registered, your MCP client gains seven tools over the corpus:
@@ -217,6 +241,8 @@ llmnesia-mcp ingest <…>    # ingest an NDJSON backup file
 llmnesia-mcp drain         # ingest everything in <corpus>/inbox
 llmnesia-mcp watch         # ingest new inbox deltas live (Ctrl+C to stop)
 llmnesia-mcp stats         # corpus summary
+llmnesia-mcp compact --dry-run # exact read-only storage/savings report
+llmnesia-mcp compact       # confirmed lossless migration + redundant cleanup
 llmnesia-mcp reindex       # drop + rebuild the search index from the corpus
 llmnesia-mcp search <q>    # one-shot search from the terminal
 llmnesia-mcp --version     # print the installed version

package/dist/{chunk-6H2TI5IW.js → chunk-JSNSTAFD.js}

@@ -38,7 +38,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/version.ts
-var VERSION = true ? "0.2.3" : "0.0.0-dev";
+var VERSION = true ? "0.2.4" : "0.0.0-dev";
 
 export {
   __require,

package/dist/{chunk-G42RV536.js → chunk-NSVHWVEK.js}

@@ -3,13 +3,13 @@ import { createRequire as __llmnesiaCreateRequire } from "node:module";
 const require = __llmnesiaCreateRequire(import.meta.url);
 import {
   VERSION
-} from "./chunk-6H2TI5IW.js";
+} from "./chunk-JSNSTAFD.js";
 
 // src/config.ts
 import { readFileSync } from "fs";
 import { homedir } from "os";
 import { join, resolve } from "path";
-var CORPUS_FORMAT_VERSION = 1;
+var CORPUS_FORMAT_VERSION = 2;
 function nativeHostConfigPath() {
   return join(homedir(), ".llmnesia", "config.json");
 }
@@ -39,7 +39,7 @@ function expandHome(p) {
 import { join as join2 } from "path";
 import { readdir } from "fs/promises";
 import { createHash } from "crypto";
-var INDEX_FORMAT_VERSION = 2;
+var INDEX_FORMAT_VERSION = 3;
 function indexDbFilename(version) {
   return version === 1 ? "llmnesia.db" : `llmnesia-v${version}.db`;
 }
@@ -69,6 +69,10 @@ var CorpusPaths = class {
   get processedDir() {
     return join2(this.inboxDir, "processed");
   }
+  /** Input that was readable but not clean enough to discard automatically. */
+  get quarantineDir() {
+    return join2(this.inboxDir, "quarantine");
+  }
   get conversationsDir() {
     return join2(this.root, "conversations");
   }
@@ -87,6 +91,14 @@ var CorpusPaths = class {
   }
   /** Absolute path to the JSON file backing a given docId. */
   conversationFile(platform, docId) {
+    return this.compressedConversationFile(platform, docId);
+  }
+  /** Canonical v2 path. Compression is lossless and independently readable. */
+  compressedConversationFile(platform, docId) {
+    return join2(this.platformDir(platform), `${slugifyDocId(docId)}.json.gz`);
+  }
+  /** v1 path accepted until confirmed/resumable compaction migrates it. */
+  legacyConversationFile(platform, docId) {
     return join2(this.platformDir(platform), `${slugifyDocId(docId)}.json`);
   }
   /**
@@ -114,8 +126,10 @@ function safeSegment(value) {
 }
 
 // src/corpus.ts
-import { promises as fs } from "fs";
+import { promises as fs, readFileSync as readFileSync2 } from "fs";
 import { dirname, join as join3 } from "path";
+import { promisify } from "util";
+import { gzip as gzipCallback, gunzip as gunzipCallback, gunzipSync } from "zlib";
 
 // src/serialize.ts
 function canonicalizeConversation(record) {
@@ -212,6 +226,8 @@ function assignIfDefined(target, key, value) {
 // src/corpus.ts
 var DIR_MODE = 448;
 var FILE_MODE = 384;
+var gzip = promisify(gzipCallback);
+var gunzip = promisify(gunzipCallback);
 var Corpus = class {
   paths;
   // Serialize writes within this process so concurrent upserts (inbox drain +
@@ -226,12 +242,14 @@ var Corpus = class {
     await fs.mkdir(this.paths.conversationsDir, { recursive: true, mode: DIR_MODE });
     await fs.mkdir(this.paths.memoriesDir, { recursive: true, mode: DIR_MODE });
     await fs.mkdir(this.paths.processedDir, { recursive: true, mode: DIR_MODE });
+    await fs.mkdir(this.paths.quarantineDir, { recursive: true, mode: DIR_MODE });
     await fs.mkdir(this.paths.indexDir, { recursive: true, mode: DIR_MODE });
     await hardenDir(this.paths.root);
     await hardenDir(this.paths.conversationsDir);
     await hardenDir(this.paths.memoriesDir);
     await hardenDir(this.paths.inboxDir);
     await hardenDir(this.paths.processedDir);
+    await hardenDir(this.paths.quarantineDir);
     await hardenDir(this.paths.indexDir);
     try {
       await fs.access(this.paths.metaFile);
@@ -275,8 +293,9 @@ var Corpus = class {
     return this.enqueueWrite(async () => {
       const canonical = canonicalizeConversation(record);
       const { conversation } = canonical;
-      const file = this.paths.conversationFile(conversation.platform, conversation.docId);
-      const existing = await readJsonIfExists(file);
+      const file = this.paths.compressedConversationFile(conversation.platform, conversation.docId);
+      const legacyFile = this.paths.legacyConversationFile(conversation.platform, conversation.docId);
+      const existing = await this.readBestConversation(conversation.platform, conversation.docId);
       if (existing) {
         const existingChangedAt = existing.conversation.corpusChangedAt ?? existing.conversation.updatedAt ?? 0;
         const incomingChangedAt = conversation.corpusChangedAt ?? conversation.updatedAt ?? 0;
@@ -286,8 +305,7 @@ var Corpus = class {
           return { docId: conversation.docId, outcome: "skipped" };
         }
       }
-      const serialized = `${JSON.stringify(canonical, null, 2)}
-`;
+      const serialized = serializeConversation(canonical);
       if (existing) {
         const existingSerialized = `${JSON.stringify(canonicalizeConversation(existing), null, 2)}
 `;
@@ -296,22 +314,39 @@ var Corpus = class {
         }
       }
       await fs.mkdir(dirname(file), { recursive: true, mode: DIR_MODE });
-      await writeFileAtomic(file, serialized);
+      await writeCompressedConversationAtomic(file, serialized);
+      await fs.rm(legacyFile, { force: true });
       return { docId: conversation.docId, outcome: existing ? "updated" : "created" };
     });
   }
   /** Read a conversation by docId, scanning platform dirs for the slug file. */
   async get(docId) {
-    const filename = `${slugifyDocId(docId)}.json`;
     for (const platform of await this.listPlatforms()) {
-      const candidate = join3(this.paths.platformDir(platform), filename);
-      const record = await readJsonIfExists(candidate);
+      const record = await this.readBestConversation(platform, docId);
       if (record && record.conversation.docId === docId) {
         return record;
       }
     }
     return null;
   }
+  /** Direct form used by search snippet hydration, avoiding a platform scan. */
+  async getFromPlatform(platform, docId) {
+    const record = await this.readBestConversation(platform, docId);
+    return record?.conversation.docId === docId ? record : null;
+  }
+  /**
+   * Synchronous twin used after SQLite has ranked a small result set. The
+   * search connection is synchronous already (`node:sqlite` DatabaseSync), and
+   * reading at most `limit` local gzip files keeps the public retriever API
+   * stable while avoiding a second raw-text store inside FTS5.
+   */
+  getFromPlatformSync(platform, docId) {
+    const record = readBestConversationFilesSync(
+      this.paths.compressedConversationFile(platform, docId),
+      this.paths.legacyConversationFile(platform, docId)
+    );
+    return record?.conversation.docId === docId ? record : null;
+  }
   /**
    * Write an agent-authored memory, keyed on memoryId.
    *
@@ -385,17 +420,108 @@ var Corpus = class {
       } catch {
         continue;
       }
+      const stems = /* @__PURE__ */ new Set();
       for (const name of files) {
-        if (!name.endsWith(".json")) {
-          continue;
-        }
-        const record = await readJsonIfExists(join3(dir, name));
+        if (name.endsWith(".json.gz")) stems.add(name.slice(0, -".json.gz".length));
+        else if (name.endsWith(".json")) stems.add(name.slice(0, -".json".length));
+      }
+      for (const stem of [...stems].sort()) {
+        const record = await readBestConversationFiles(
+          join3(dir, `${stem}.json.gz`),
+          join3(dir, `${stem}.json`)
+        );
         if (record) {
           yield record;
         }
       }
     }
   }
+  /** Byte/count breakdown used by `stats` and `compact --dry-run`. */
+  async conversationStorageStats() {
+    const stats = {
+      legacyFiles: 0,
+      legacyBytes: 0,
+      compressedFiles: 0,
+      compressedBytes: 0
+    };
+    for (const platform of await this.listPlatforms()) {
+      const dir = this.paths.platformDir(platform);
+      let entries;
+      try {
+        entries = await fs.readdir(dir, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const entry of entries) {
+        if (!entry.isFile()) continue;
+        const file = join3(dir, entry.name);
+        if (entry.name.endsWith(".json.gz")) {
+          stats.compressedFiles += 1;
+          stats.compressedBytes += (await fs.stat(file)).size;
+        } else if (entry.name.endsWith(".json")) {
+          stats.legacyFiles += 1;
+          stats.legacyBytes += (await fs.stat(file)).size;
+        }
+      }
+    }
+    return stats;
+  }
+  /**
+   * Convert every legacy conversation independently. Each candidate is written,
+   * read back, and byte-compared in canonical JSON form before its plaintext
+   * source is removed, making the operation resumable after any interruption.
+   */
+  async migrateLegacyConversations(options = {}) {
+    const dryRun = options.dryRun === true;
+    const before = await this.conversationStorageStats();
+    const report = {
+      ...before,
+      dryRun,
+      candidates: before.legacyFiles,
+      migrated: 0,
+      legacyRemoved: 0,
+      projectedCompressedBytes: before.compressedBytes,
+      failures: []
+    };
+    for (const platform of await this.listPlatforms()) {
+      const dir = this.paths.platformDir(platform);
+      let names;
+      try {
+        names = (await fs.readdir(dir)).filter((name) => name.endsWith(".json")).sort();
+      } catch {
+        continue;
+      }
+      for (const name of names) {
+        const legacyFile = join3(dir, name);
+        const compressedFile = join3(dir, `${name}.gz`);
+        try {
+          const legacy = await readPlainConversationIfExists(legacyFile);
+          if (!legacy) throw new Error("legacy JSON could not be parsed");
+          const compressed = await readCompressedConversationIfExists(compressedFile);
+          const winner = pickNewerConversation(compressed, legacy) ?? legacy;
+          const serialized = serializeConversation(canonicalizeConversation(winner));
+          const compressedBytes = await gzip(Buffer.from(serialized, "utf8"), { level: 9 });
+          report.projectedCompressedBytes -= await fileSizeIfExists(compressedFile);
+          report.projectedCompressedBytes += compressedBytes.byteLength;
+          if (!dryRun) {
+            await writeVerifiedCompressedConversationAtomic(compressedFile, compressedBytes, serialized);
+            await fs.rm(legacyFile);
+          }
+          report.migrated += 1;
+          report.legacyRemoved += 1;
+        } catch (error) {
+          report.failures.push({
+            file: legacyFile,
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
+      }
+    }
+    if (!dryRun && report.failures.length === 0) {
+      await this.writeMeta({ formatVersion: CORPUS_FORMAT_VERSION });
+    }
+    return report;
+  }
   async stats() {
     const byPlatform = {};
     let conversationCount = 0;
@@ -409,9 +535,13 @@ var Corpus = class {
     return { conversationCount, messageCount, byPlatform };
   }
   /** Refresh meta.json's conversationCount / lastIngestAt after an ingest run. */
-  async touchIngestMeta() {
+  async touchIngestMeta(receipt) {
     const { conversationCount } = await this.stats();
-    await this.writeMeta({ lastIngestAt: Date.now(), conversationCount });
+    await this.writeMeta({
+      lastIngestAt: Date.now(),
+      conversationCount,
+      ...receipt ? { lastIngestReceipt: receipt } : {}
+    });
   }
   enqueueWrite(fn) {
     const run = this.writeChain.then(fn, fn);
@@ -421,6 +551,12 @@ var Corpus = class {
     );
     return run;
   }
+  readBestConversation(platform, docId) {
+    return readBestConversationFiles(
+      this.paths.compressedConversationFile(platform, docId),
+      this.paths.legacyConversationFile(platform, docId)
+    );
+  }
 };
 async function readJsonIfExists(file) {
   try {
@@ -430,9 +566,83 @@ async function readJsonIfExists(file) {
     return null;
   }
 }
+function serializeConversation(record) {
+  return `${JSON.stringify(canonicalizeConversation(record), null, 2)}
+`;
+}
+async function readPlainConversationIfExists(file) {
+  return readJsonIfExists(file);
+}
+async function readCompressedConversationIfExists(file) {
+  try {
+    const raw = await fs.readFile(file);
+    return JSON.parse((await gunzip(raw)).toString("utf8"));
+  } catch {
+    return null;
+  }
+}
+async function readBestConversationFiles(compressedFile, legacyFile) {
+  const [compressed, legacy] = await Promise.all([
+    readCompressedConversationIfExists(compressedFile),
+    readPlainConversationIfExists(legacyFile)
+  ]);
+  return pickNewerConversation(compressed, legacy);
+}
+function readBestConversationFilesSync(compressedFile, legacyFile) {
+  let compressed = null;
+  let legacy = null;
+  try {
+    compressed = JSON.parse(gunzipSync(readFileSync2(compressedFile)).toString("utf8"));
+  } catch {
+  }
+  try {
+    legacy = JSON.parse(readFileSync2(legacyFile, "utf8"));
+  } catch {
+  }
+  return pickNewerConversation(compressed, legacy);
+}
+function pickNewerConversation(a, b) {
+  if (!a) return b;
+  if (!b) return a;
+  const aChanged = a.conversation.corpusChangedAt ?? a.conversation.updatedAt ?? 0;
+  const bChanged = b.conversation.corpusChangedAt ?? b.conversation.updatedAt ?? 0;
+  if (aChanged !== bChanged) return aChanged > bChanged ? a : b;
+  const aUpdated = a.conversation.updatedAt ?? 0;
+  const bUpdated = b.conversation.updatedAt ?? 0;
+  return aUpdated >= bUpdated ? a : b;
+}
+async function writeCompressedConversationAtomic(file, serialized) {
+  const compressed = await gzip(Buffer.from(serialized, "utf8"), { level: 9 });
+  await writeFileAtomic(file, compressed);
+}
+async function writeVerifiedCompressedConversationAtomic(file, compressed, serialized) {
+  const tmp = `${file}.migration-${process.pid}-${Date.now()}`;
+  try {
+    await fs.writeFile(tmp, compressed, { mode: FILE_MODE });
+    const verified = await readCompressedConversationIfExists(tmp);
+    if (!verified || serializeConversation(canonicalizeConversation(verified)) !== serialized) {
+      throw new Error("compressed read-back verification failed");
+    }
+    await fs.rename(tmp, file);
+  } catch (error) {
+    await fs.rm(tmp, { force: true });
+    throw error;
+  }
+}
+async function fileSizeIfExists(file) {
+  try {
+    return (await fs.stat(file)).size;
+  } catch {
+    return 0;
+  }
+}
 async function writeFileAtomic(file, contents) {
   const tmp = `${file}.tmp-${process.pid}-${Date.now()}`;
-  await fs.writeFile(tmp, contents, { encoding: "utf8", mode: FILE_MODE });
+  if (typeof contents === "string") {
+    await fs.writeFile(tmp, contents, { encoding: "utf8", mode: FILE_MODE });
+  } else {
+    await fs.writeFile(tmp, contents, { mode: FILE_MODE });
+  }
   await fs.rename(tmp, file);
 }
 async function hardenDir(dir) {
@@ -533,23 +743,46 @@ async function drainInbox(corpus, index, options = {}) {
   await corpus.init();
   const inbox = corpus.paths.inboxDir;
   const entries = await pendingInboxFiles(corpus);
-  const total = { ...EMPTY_REPORT, files: 0, failed: 0 };
+  const total = {
+    ...EMPTY_REPORT,
+    files: 0,
+    failed: 0,
+    discarded: 0,
+    quarantined: 0
+  };
   for (const name of entries) {
     const src = join4(inbox, name);
     try {
       const report = await ingestFile(corpus, src, index);
       accumulate(total, report);
       total.files += 1;
-      await moveToProcessed(corpus, src, name);
+      if (report.malformedLines > 0) {
+        await moveToQuarantine(corpus, src, name);
+        total.quarantined += 1;
+      } else {
+        await fs2.unlink(src);
+        total.discarded += 1;
+      }
     } catch (error) {
       if (await stillPending(src)) {
         total.failed += 1;
+        if (isCorruptDeltaError(error)) {
+          await moveToQuarantine(corpus, src, name);
+          total.quarantined += 1;
+        }
         options.onFileError?.(name, error);
       }
     }
   }
   if (total.files > 0) {
-    await corpus.touchIngestMeta();
+    await corpus.touchIngestMeta({
+      files: total.files,
+      conversations: total.conversations,
+      messages: total.messages,
+      malformedLines: total.malformedLines,
+      discarded: total.discarded,
+      quarantined: total.quarantined
+    });
   }
   return total;
 }
@@ -561,16 +794,23 @@ async function stillPending(src) {
     return false;
   }
 }
-async function moveToProcessed(corpus, src, name) {
-  const dest = join4(corpus.paths.processedDir, name);
+async function moveToQuarantine(corpus, src, name) {
+  const dest = join4(corpus.paths.quarantineDir, name);
   try {
     await fs2.rename(src, dest);
   } catch {
-    const alt = join4(corpus.paths.processedDir, `${Date.now()}-${name}`);
+    const alt = join4(corpus.paths.quarantineDir, `${Date.now()}-${name}`);
     await fs2.copyFile(src, alt);
     await fs2.unlink(src);
   }
 }
+function isCorruptDeltaError(error) {
+  if (!error || typeof error !== "object" || !("code" in error)) {
+    return false;
+  }
+  const code = String(error.code ?? "");
+  return code === "Z_DATA_ERROR" || code === "Z_BUF_ERROR" || code.startsWith("ERR_ZLIB_");
+}
 function isNdjson(name) {
   const lower = name.toLowerCase();
   return lower.endsWith(".jsonl") || lower.endsWith(".ndjson") || lower.endsWith(".jsonl.gz") || lower.endsWith(".ndjson.gz");
@@ -703,6 +943,7 @@ CREATE TABLE IF NOT EXISTS index_meta (
 var DROP_DERIVED_SQL = `
 DROP TABLE IF EXISTS conversations;
 DROP TABLE IF EXISTS messages_fts;
+DROP TABLE IF EXISTS message_rows;
 DROP TABLE IF EXISTS memories;
 DROP TABLE IF EXISTS memories_fts;
 `;
@@ -728,11 +969,18 @@ CREATE TABLE IF NOT EXISTS conversations (
 CREATE INDEX IF NOT EXISTS idx_conversations_platform ON conversations(platform);
 CREATE INDEX IF NOT EXISTS idx_conversations_createdAt ON conversations(createdAt);
 
+CREATE TABLE IF NOT EXISTS message_rows (
+  rowid INTEGER PRIMARY KEY,
+  docId TEXT NOT NULL,
+  msgIndex INTEGER NOT NULL,
+  role TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_message_rows_docId ON message_rows(docId);
+
 CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
-  docId UNINDEXED,
-  msgIndex UNINDEXED,
-  role UNINDEXED,
   text,
+  content = '',
+  contentless_delete = 1,
   tokenize = 'porter unicode61'
 );
 
@@ -760,7 +1008,10 @@ var SearchIndex = class {
   stmts;
   /** Whether the on-disk index was a stale format version when it was opened. */
   staleOnOpen;
-  constructor(dbPath) {
+  /** Canonical source used only to hydrate the few snippets returned by rank. */
+  corpus;
+  constructor(dbPath, corpus) {
+    this.corpus = corpus;
     hardenDbFile(dbPath);
     this.db = new (loadSqlite()).DatabaseSync(dbPath);
     this.db.exec("PRAGMA journal_mode = WAL");
@@ -771,12 +1022,19 @@ var SearchIndex = class {
       this.db.exec(DROP_DERIVED_SQL);
     }
     this.db.exec(SCHEMA_SQL);
+    this.db.exec("PRAGMA temp_store = MEMORY");
+    this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.snippet_fts USING fts5(
+      text, tokenize = 'porter unicode61'
+    )`);
     this.setMeta("format_version", String(INDEX_FORMAT_VERSION));
     this.stmts = {
-      deleteFts: this.db.prepare("DELETE FROM messages_fts WHERE docId = ?"),
-      insertFts: this.db.prepare(
-        "INSERT INTO messages_fts (docId, msgIndex, role, text) VALUES (?, ?, ?, ?)"
+      selectFtsRows: this.db.prepare("SELECT rowid FROM message_rows WHERE docId = ?"),
+      deleteFts: this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?"),
+      deleteMessageRows: this.db.prepare("DELETE FROM message_rows WHERE docId = ?"),
+      insertMessageRow: this.db.prepare(
+        "INSERT INTO message_rows (docId, msgIndex, role) VALUES (?, ?, ?)"
       ),
+      insertFts: this.db.prepare("INSERT INTO messages_fts (rowid, text) VALUES (?, ?)"),
       deleteConversation: this.db.prepare("DELETE FROM conversations WHERE docId = ?"),
       insertConversation: this.db.prepare(
         `INSERT INTO conversations
@@ -792,6 +1050,12 @@ var SearchIndex = class {
         `INSERT INTO memories
            (memoryId, kind, label, text, source, pinned, enabled, createdAt, updatedAt)
          VALUES (@memoryId, @kind, @label, @text, @source, @pinned, @enabled, @createdAt, @updatedAt)`
+      ),
+      clearSnippetFts: this.db.prepare("DELETE FROM temp.snippet_fts"),
+      insertSnippetFts: this.db.prepare("INSERT INTO temp.snippet_fts (text) VALUES (?)"),
+      renderSnippet: this.db.prepare(
+        `SELECT snippet(snippet_fts, 0, '[', ']', '\u2026', ${SNIPPET_TOKENS}) AS snippet
+         FROM temp.snippet_fts WHERE snippet_fts MATCH ?`
       )
     };
   }
@@ -855,7 +1119,7 @@ var SearchIndex = class {
   }
   removeConversation(docId) {
     this.transaction(() => {
-      this.stmts.deleteFts.run(docId);
+      this.deleteFtsRows(docId);
       this.stmts.deleteConversation.run(docId);
     });
   }
@@ -865,7 +1129,8 @@ var SearchIndex = class {
    * reads, which the sync {@link transaction} helper cannot hold.
    */
   async rebuildFrom(corpus) {
-    this.db.exec("DELETE FROM messages_fts; DELETE FROM conversations;");
+    this.corpus = corpus;
+    this.db.exec("DELETE FROM messages_fts; DELETE FROM message_rows; DELETE FROM conversations;");
     this.db.exec("DELETE FROM memories_fts; DELETE FROM memories;");
     let conversations = 0;
     let memories = 0;
@@ -890,15 +1155,15 @@ var SearchIndex = class {
   // transaction() helper for single upserts, or rebuildFrom's manual one).
   writeRecord(rec) {
     const { conversation, messages } = rec;
-    this.stmts.deleteFts.run(conversation.docId);
+    this.deleteFtsRows(conversation.docId);
     this.stmts.deleteConversation.run(conversation.docId);
     const titleText = [conversation.title, conversation.preview].filter(Boolean).join("\n");
     if (titleText.trim()) {
-      this.stmts.insertFts.run(conversation.docId, TITLE_ROW_INDEX, TITLE_ROLE, titleText);
+      this.insertFtsRow(conversation.docId, TITLE_ROW_INDEX, TITLE_ROLE, titleText);
     }
     for (const message of messages) {
       if (message.text.trim()) {
-        this.stmts.insertFts.run(conversation.docId, message.msgIndex, message.role, message.text);
+        this.insertFtsRow(conversation.docId, message.msgIndex, message.role, message.text);
       }
     }
     this.stmts.insertConversation.run({
@@ -915,6 +1180,17 @@ var SearchIndex = class {
       accountLabel: conversation.accountLabel ?? null
     });
   }
+  insertFtsRow(docId, msgIndex, role, text) {
+    const inserted = this.stmts.insertMessageRow.run(docId, msgIndex, role);
+    this.stmts.insertFts.run(inserted.lastInsertRowid, text);
+  }
+  deleteFtsRows(docId) {
+    const rows = this.stmts.selectFtsRows.all(docId);
+    for (const row of rows) {
+      this.stmts.deleteFts.run(row.rowid);
+    }
+    this.stmts.deleteMessageRows.run(docId);
+  }
   // Replace a memory's index rows. Caller supplies the transaction, as with
   // writeRecord. Memories live in their own tables rather than in messages_fts:
   // a memory is not a turn in a conversation, and mixing them would let a
@@ -968,9 +1244,9 @@ var SearchIndex = class {
         c.docId, c.platform, c.title, c.url, c.createdAt, c.updatedAt,
         c.messageCount, c.preview, c.pinned, c.accountId, c.accountLabel,
         m.msgIndex AS msgIndex, m.role AS role,
-        snippet(messages_fts, 3, '[', ']', '\u2026', ${SNIPPET_TOKENS}) AS snippet,
         bm25(messages_fts) AS bm25
-      FROM messages_fts m
+      FROM messages_fts
+      JOIN message_rows m ON m.rowid = messages_fts.rowid
       JOIN conversations c ON c.docId = m.docId
       WHERE ${clauses.join(" AND ")}
       ORDER BY bm25
@@ -996,7 +1272,7 @@ var SearchIndex = class {
         pinned: row.pinned === 1,
         // BM25 is negative (more negative = better); expose a positive relevance.
         score: -row.bm25,
-        snippet: row.snippet,
+        snippet: this.hydrateSnippet(row, match),
         matchRole: normalizeMatchRole(row.role),
         matchMsgIndex: row.msgIndex
       };
@@ -1009,6 +1285,25 @@ var SearchIndex = class {
     }
     return hits;
   }
+  /** Reclaim free pages and merge FTS segments after a confirmed compaction. */
+  compact() {
+    this.db.exec("INSERT INTO messages_fts(messages_fts) VALUES('optimize')");
+    this.db.exec("INSERT INTO memories_fts(memories_fts) VALUES('optimize')");
+    this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+    this.db.exec("VACUUM");
+  }
+  hydrateSnippet(row, match) {
+    const record = this.corpus?.getFromPlatformSync(row.platform, row.docId);
+    const source = row.msgIndex === TITLE_ROW_INDEX ? [record?.conversation.title ?? row.title, record?.conversation.preview ?? row.preview].filter(Boolean).join("\n") : record?.messages.find((message) => message.msgIndex === row.msgIndex)?.text ?? row.preview;
+    this.stmts.clearSnippetFts.run();
+    try {
+      this.stmts.insertSnippetFts.run(source);
+      const rendered = this.stmts.renderSnippet.get(match);
+      return rendered?.snippet ?? fallbackSnippet(source, SNIPPET_TOKENS);
+    } finally {
+      this.stmts.clearSnippetFts.run();
+    }
+  }
   close() {
     this.db.close();
   }
@@ -1022,7 +1317,7 @@ var SearchIndex = class {
 };
 async function openSearchIndex(corpus, options = {}) {
   await corpus.init();
-  const index = new SearchIndex(corpus.paths.indexDbFile);
+  const index = new SearchIndex(corpus.paths.indexDbFile, corpus);
   if (options.autoBuild !== false && (index.needsRebuild() || index.count() === 0 && index.countMemories() === 0)) {
     await index.rebuildFrom(corpus);
   }
@@ -1034,7 +1329,7 @@ async function deleteIndexDb(dbFile) {
   }
 }
 function toMatchQuery(query, mode = "any") {
-  const terms = query.split(/[^\p{L}\p{N}]+/u).filter((t) => t.length > 0);
+  const terms = queryTerms(query);
   if (terms.length === 0) {
     return null;
   }
@@ -1043,6 +1338,17 @@ function toMatchQuery(query, mode = "any") {
   }
   return terms.map((t) => `"${t}"`).join(mode === "all" ? " AND " : " OR ");
 }
+function queryTerms(query) {
+  return query.split(/[^\p{L}\p{N}]+/u).filter((term) => term.length > 0);
+}
+function fallbackSnippet(source, maxTokens) {
+  const collapsed = source.replace(/\s+/g, " ").trim();
+  if (!collapsed) return "";
+  const words = collapsed.split(" ");
+  const width = Math.max(1, maxTokens);
+  const end = Math.min(words.length, width);
+  return `${words.slice(0, end).join(" ")}${end < words.length ? "\u2026" : ""}`;
+}
 function escapeLike(value) {
   return value.replace(/[\\%_]/g, "\\$&");
 }

package/dist/{cli-77JECNWA.js → cli-ZEA7WTVW.js}

@@ -14,13 +14,13 @@ import {
   runtimeBinDir,
   staleIndexFiles,
   uninstallRuntime
-} from "./chunk-G42RV536.js";
+} from "./chunk-NSVHWVEK.js";
 import {
   VERSION,
   __commonJS,
   __require,
   __toESM
-} from "./chunk-6H2TI5IW.js";
+} from "./chunk-JSNSTAFD.js";
 
 // ../node_modules/isexe/windows.js
 var require_windows = __commonJS({
@@ -28,7 +28,7 @@ var require_windows = __commonJS({
     "use strict";
     module.exports = isexe;
     isexe.sync = sync;
-    var fs = __require("fs");
+    var fs2 = __require("fs");
     function checkPathExt(path, options) {
       var pathext = options.pathExt !== void 0 ? options.pathExt : process.env.PATHEXT;
       if (!pathext) {
@@ -53,12 +53,12 @@ var require_windows = __commonJS({
       return checkPathExt(path, options);
     }
     function isexe(path, options, cb) {
-      fs.stat(path, function(er, stat6) {
+      fs2.stat(path, function(er, stat6) {
         cb(er, er ? false : checkStat(stat6, path, options));
       });
     }
     function sync(path, options) {
-      return checkStat(fs.statSync(path), path, options);
+      return checkStat(fs2.statSync(path), path, options);
     }
   }
 });
@@ -69,14 +69,14 @@ var require_mode = __commonJS({
     "use strict";
     module.exports = isexe;
     isexe.sync = sync;
-    var fs = __require("fs");
+    var fs2 = __require("fs");
     function isexe(path, options, cb) {
-      fs.stat(path, function(er, stat6) {
+      fs2.stat(path, function(er, stat6) {
         cb(er, er ? false : checkStat(stat6, options));
       });
     }
     function sync(path, options) {
-      return checkStat(fs.statSync(path), options);
+      return checkStat(fs2.statSync(path), options);
     }
     function checkStat(stat6, options) {
       return stat6.isFile() && checkMode(stat6, options);
@@ -101,7 +101,7 @@ var require_mode = __commonJS({
 var require_isexe = __commonJS({
   "../node_modules/isexe/index.js"(exports, module) {
     "use strict";
-    var fs = __require("fs");
+    var fs2 = __require("fs");
     var core;
     if (process.platform === "win32" || global.TESTING_WINDOWS) {
       core = require_windows();
@@ -366,16 +366,16 @@ var require_shebang_command = __commonJS({
 var require_readShebang = __commonJS({
   "../node_modules/cross-spawn/lib/util/readShebang.js"(exports, module) {
     "use strict";
-    var fs = __require("fs");
+    var fs2 = __require("fs");
     var shebangCommand = require_shebang_command();
     function readShebang(command) {
       const size = 150;
       const buffer = Buffer.alloc(size);
       let fd;
       try {
-        fd = fs.openSync(command, "r");
-        fs.readSync(fd, buffer, 0, size, 0);
-        fs.closeSync(fd);
+        fd = fs2.openSync(command, "r");
+        fs2.readSync(fd, buffer, 0, size, 0);
+        fs2.closeSync(fd);
       } catch (e) {
       }
       return shebangCommand(buffer.toString());
@@ -525,7 +525,7 @@ var require_cross_spawn = __commonJS({
 
 // src/cli.ts
 import { createInterface } from "readline/promises";
-import { join as join6 } from "path";
+import { join as join7 } from "path";
 
 // ../packages/native-host/install.mjs
 import { execFile } from "child_process";
@@ -824,7 +824,7 @@ var OPTION_ALIASES = {
   h: "help",
   y: "yes"
 };
-var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["force", "yes", "version", "help"]);
+var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["force", "yes", "version", "help", "dry-run"]);
 var VALUE_OPTIONS = /* @__PURE__ */ new Set(["corpus", "limit", "platform", "title", "from", "to"]);
 function parseArgs(argv) {
   const positionals = [];
@@ -3749,6 +3749,157 @@ function formatDoctorReport(report) {
   return report.checks.map((c) => `${icon[c.status]} ${c.name}: ${c.detail}`).join("\n");
 }
 
+// src/storageMaintenance.ts
+import { promises as fs } from "fs";
+import { join as join6 } from "path";
+async function inspectCorpusStorage(corpus) {
+  const conversations = await corpus.conversationStorageStats();
+  const processed = await directoryFiles(corpus.paths.processedDir);
+  const quarantine = await directoryFiles(corpus.paths.quarantineDir);
+  const activeNames = [
+    corpus.paths.indexDbFile,
+    `${corpus.paths.indexDbFile}-wal`,
+    `${corpus.paths.indexDbFile}-shm`
+  ];
+  const activeIndex = await explicitFiles(activeNames);
+  const staleNames = await staleIndexFiles(corpus.paths.indexDir);
+  const stalePaths = staleNames.flatMap((name) => {
+    const base = join6(corpus.paths.indexDir, name);
+    return [base, `${base}-wal`, `${base}-shm`];
+  });
+  const staleIndexes = await explicitFiles(stalePaths);
+  const root = await directoryFiles(corpus.paths.root);
+  const classifiedBytes = conversations.legacyBytes + conversations.compressedBytes + processed.bytes + quarantine.bytes + activeIndex.bytes + staleIndexes.bytes;
+  const classifiedFiles = conversations.legacyFiles + conversations.compressedFiles + processed.files + quarantine.files + activeIndex.files + staleIndexes.files;
+  const other = {
+    files: Math.max(0, root.files - classifiedFiles),
+    bytes: Math.max(0, root.bytes - classifiedBytes)
+  };
+  return {
+    root: corpus.paths.root,
+    conversations,
+    processed,
+    quarantine,
+    activeIndex,
+    staleIndexes,
+    other,
+    totalBytes: root.bytes,
+    immediatelyReclaimableBytes: processed.bytes + staleIndexes.bytes
+  };
+}
+async function compactCorpus(corpus, options = {}) {
+  const dryRun = options.dryRun === true;
+  const before = await inspectCorpusStorage(corpus);
+  if (dryRun) {
+    const compression = await corpus.migrateLegacyConversations({ dryRun: true });
+    const compressionSavings = Math.max(
+      0,
+      compression.legacyBytes + compression.compressedBytes - compression.projectedCompressedBytes
+    );
+    return {
+      dryRun: true,
+      before,
+      compression,
+      processedRemoved: before.processed,
+      staleIndexesRemoved: before.staleIndexes,
+      after: null,
+      reclaimedBytes: before.immediatelyReclaimableBytes + compressionSavings
+    };
+  }
+  await corpus.init();
+  const processedRemoved = await removeDirectoryFiles(corpus.paths.processedDir);
+  const index = await openSearchIndex(corpus);
+  try {
+    index.compact();
+  } finally {
+    index.close();
+  }
+  const appliedCompression = await corpus.migrateLegacyConversations();
+  if (appliedCompression.failures.length > 0) {
+    throw new Error(
+      `Corpus compression left ${appliedCompression.failures.length} legacy file(s) untouched; stale indexes were preserved. Run compact again after fixing the reported files.`
+    );
+  }
+  const staleIndexesRemoved = await removeStaleIndexes(corpus);
+  const after = await inspectCorpusStorage(corpus);
+  return {
+    dryRun: false,
+    before,
+    compression: appliedCompression,
+    processedRemoved,
+    staleIndexesRemoved,
+    after,
+    reclaimedBytes: Math.max(0, before.totalBytes - after.totalBytes)
+  };
+}
+async function removeStaleIndexes(corpus) {
+  const names = await staleIndexFiles(corpus.paths.indexDir);
+  const paths2 = names.flatMap((name) => {
+    const base = join6(corpus.paths.indexDir, name);
+    return [base, `${base}-wal`, `${base}-shm`];
+  });
+  const before = await explicitFiles(paths2);
+  for (const name of names) {
+    await deleteIndexDb(join6(corpus.paths.indexDir, name));
+  }
+  return before;
+}
+async function removeDirectoryFiles(dir) {
+  const before = await directoryFiles(dir);
+  await removeFilesRecursively(dir);
+  return before;
+}
+async function removeFilesRecursively(dir) {
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const path = join6(dir, entry.name);
+    if (entry.isDirectory()) await removeFilesRecursively(path);
+    else if (entry.isFile()) await fs.rm(path, { force: true });
+  }
+}
+async function explicitFiles(files) {
+  let count = 0;
+  let bytes = 0;
+  for (const file of files) {
+    try {
+      const stat6 = await fs.stat(file);
+      if (stat6.isFile()) {
+        count += 1;
+        bytes += stat6.size;
+      }
+    } catch {
+    }
+  }
+  return { files: count, bytes };
+}
+async function directoryFiles(dir) {
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return { files: 0, bytes: 0 };
+  }
+  let files = 0;
+  let bytes = 0;
+  for (const entry of entries) {
+    const path = join6(dir, entry.name);
+    if (entry.isDirectory()) {
+      const nested = await directoryFiles(path);
+      files += nested.files;
+      bytes += nested.bytes;
+    } else if (entry.isFile()) {
+      files += 1;
+      bytes += (await fs.stat(path)).size;
+    }
+  }
+  return { files, bytes };
+}
+
 // src/cli.ts
 var USAGE = `llmnesia-mcp ${VERSION}
 
@@ -3765,9 +3916,12 @@ Setup:
 Power-user / debugging:
   init                 Create the corpus directory skeleton
   ingest <file...>     Ingest one or more NDJSON backup files into the corpus
-  drain                Ingest every file in <corpus>/inbox, then archive them
+  drain                Ingest every file in <corpus>/inbox; discard clean
+                       payloads and quarantine malformed ones
   watch                Watch <corpus>/inbox and ingest new deltas live
   stats                Print corpus counts (conversations, messages, platforms)
+  compact              Losslessly compress conversations and remove redundant
+                       processed batches/stale indexes (confirmation required)
   reindex              Drop and rebuild the search index from the corpus
   search <query>       Full-text search the corpus (BM25)
 
@@ -3780,6 +3934,7 @@ Options:
   --to <iso|ms>        search: only conversations created on/before this date
   --force              install: overwrite a differing corpus root in config.json
   --yes, -y            uninstall: skip its confirmation prompt
+  --dry-run             compact: calculate exact savings without writing
   --version, -v        print the version and exit
 `;
 function parseDate(value) {
@@ -3984,7 +4139,7 @@ async function main() {
 `)
         });
         process.stdout.write(
-          `Drained ${report.files} inbox file(s): ${report.conversations} conversation(s) \u2014 ${report.created} created, ${report.updated} updated, ${report.skipped} unchanged` + (report.failed > 0 ? `, ${report.failed} file(s) failed` : "") + `
+          `Drained ${report.files} inbox file(s): ${report.conversations} conversation(s) \u2014 ${report.created} created, ${report.updated} updated, ${report.skipped} unchanged` + (report.quarantined > 0 ? `, ${report.quarantined} quarantined` : "") + (report.failed > 0 ? `, ${report.failed} file(s) failed` : "") + `
 `
         );
       } finally {
@@ -4012,6 +4167,7 @@ async function main() {
     }
     case "stats": {
       const stats = await corpus.stats();
+      const storage = await inspectCorpusStorage(corpus);
       const platforms = Object.entries(stats.byPlatform).sort((a, b) => b[1] - a[1]).map(([p, n]) => `    ${p}: ${n}`).join("\n");
       process.stdout.write(
         `Corpus: ${root}
@@ -4019,14 +4175,42 @@ async function main() {
   messages:      ${stats.messageCount}
 ` + (platforms ? `  by platform:
 ${platforms}
-` : "")
+` : "") + `  storage:       ${formatBytes(storage.totalBytes)}
+    conversations: ${formatBytes(storage.conversations.legacyBytes + storage.conversations.compressedBytes)} (${storage.conversations.compressedFiles} compressed, ${storage.conversations.legacyFiles} legacy)
+    active index:  ${formatBytes(storage.activeIndex.bytes)}
+    processed:     ${formatBytes(storage.processed.bytes)} (${storage.processed.files} reclaimable files)
+    stale indexes: ${formatBytes(storage.staleIndexes.bytes)}
+    quarantine:    ${formatBytes(storage.quarantine.bytes)}
+`
+      );
+      return 0;
+    }
+    case "compact": {
+      const dryRun = flags.has("dry-run");
+      const preview = await compactCorpus(corpus, { dryRun: true });
+      process.stdout.write(formatCompactPreview(preview));
+      if (dryRun) return 0;
+      if (!flags.has("yes")) {
+        const ok = await promptConfirm(
+          "Compact this corpus now? Close desktop AI/MCP clients first. Clean processed sync payloads and stale derived indexes will be removed; canonical conversations are migrated losslessly with read-back verification."
+        );
+        if (!ok) {
+          process.stdout.write("Cancelled.\n");
+          return 0;
+        }
+      }
+      const result = await compactCorpus(corpus);
+      process.stdout.write(
+        `Compaction complete: reclaimed ${formatBytes(result.reclaimedBytes)}; ${result.compression.migrated} conversation(s) compressed, ${result.processedRemoved.files} processed file(s) removed, ${result.staleIndexesRemoved.files} stale index file(s) removed.
+Corpus now uses ${formatBytes(result.after?.totalBytes ?? 0)}.
+`
       );
       return 0;
     }
     case "reindex": {
       await corpus.init();
       await deleteIndexDb(corpus.paths.indexDbFile);
-      const index = new SearchIndex(corpus.paths.indexDbFile);
+      const index = new SearchIndex(corpus.paths.indexDbFile, corpus);
       try {
         const { conversations } = await index.rebuildFrom(corpus);
         process.stdout.write(`Rebuilt index: ${conversations} conversation(s)
@@ -4036,7 +4220,7 @@ ${platforms}
       }
       const stale = await staleIndexFiles(corpus.paths.indexDir);
       for (const name of stale) {
-        await deleteIndexDb(join6(corpus.paths.indexDir, name));
+        await deleteIndexDb(join7(corpus.paths.indexDir, name));
       }
       if (stale.length > 0) {
         process.stdout.write(
@@ -4092,6 +4276,31 @@ ${USAGE}`);
 function collapse(text) {
   return text.replace(/\s+/g, " ").trim();
 }
+function formatBytes(bytes) {
+  if (bytes < 1024) return `${bytes} B`;
+  const units = ["KiB", "MiB", "GiB", "TiB"];
+  let value = bytes / 1024;
+  let unit = units[0];
+  for (let i = 1; i < units.length && value >= 1024; i += 1) {
+    value /= 1024;
+    unit = units[i];
+  }
+  return `${value.toFixed(value >= 10 ? 1 : 2)} ${unit}`;
+}
+function formatCompactPreview(result) {
+  const { before, compression } = result;
+  const compressionSavings = Math.max(
+    0,
+    compression.legacyBytes + compression.compressedBytes - compression.projectedCompressedBytes
+  );
+  return `Compaction preview for ${before.root}:
+  current size:             ${formatBytes(before.totalBytes)}
+  processed sync payloads:  ${formatBytes(before.processed.bytes)} (${before.processed.files} files)
+  stale indexes:            ${formatBytes(before.staleIndexes.bytes)} (${before.staleIndexes.files} files)
+  lossless compression:     ${formatBytes(compressionSavings)} (${compression.candidates} legacy conversations)
+  estimated reclaimable:    ${formatBytes(result.reclaimedBytes)}
+`;
+}
 export {
   formatInstallSuccess,
   main,

package/dist/cli.js

@@ -3,7 +3,7 @@ import { createRequire as __llmnesiaCreateRequire } from "node:module";
 const require = __llmnesiaCreateRequire(import.meta.url);
 import {
   VERSION
-} from "./chunk-6H2TI5IW.js";
+} from "./chunk-JSNSTAFD.js";
 
 // src/nodeDownload.ts
 var NODE_DOWNLOAD_URLS = {
@@ -141,7 +141,7 @@ async function bootstrap() {
     await guideUnsupportedNodeVersion(process.versions.node);
     return 1;
   }
-  const { main } = await import("./cli-77JECNWA.js");
+  const { main } = await import("./cli-ZEA7WTVW.js");
   return main();
 }
 bootstrap().then((code) => {

package/dist/mcp.js

@@ -9,13 +9,13 @@ import {
   pendingInboxFiles,
   resolveCorpusDir,
   scheduleSelfUpdate
-} from "./chunk-G42RV536.js";
+} from "./chunk-NSVHWVEK.js";
 import {
   VERSION,
   __commonJS,
   __export,
   __toESM
-} from "./chunk-6H2TI5IW.js";
+} from "./chunk-JSNSTAFD.js";
 
 // ../node_modules/ajv/dist/compile/codegen/code.js
 var require_code = __commonJS({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llmnesia/mcp",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "private": false,
   "type": "module",
   "comment:mcpName": "The MCP Registry's ownership marker: it must equal server.json's `name`, and the namespace must match how you authenticate with mcp-publisher. `com.llmnesia/...` requires DNS auth (a TXT record on llmnesia.com) — see docs/MCP_INSTALLER_BOARD.md.",

package/server.json

@@ -3,13 +3,13 @@
   "name": "com.llmnesia/mcp",
   "title": "LLMnesia",
   "description": "Search your AI chat history from Claude, Cursor, Codex, and other MCP clients.",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "packages": [
     {
       "registryType": "npm",
       "registryBaseUrl": "https://registry.npmjs.org",
       "identifier": "@llmnesia/mcp",
-      "version": "0.2.3",
+      "version": "0.2.4",
       "transport": {
         "type": "stdio"
       }