@llmnesia/mcp 0.2.2 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -140,6 +140,30 @@ the [Chrome Web Store](https://chromewebstore.google.com/detail/llmnesia/leekfgb
140
140
 
141
141
  If nothing has written to the corpus yet, search results will be empty.
142
142
 
143
+ ### Local storage and compaction
144
+
145
+ Conversation files are stored as losslessly compressed JSON. Search uses a
146
+ contentless FTS5 token index: the canonical text is not duplicated inside the
147
+ database, and result snippets are loaded from the canonical conversation only
148
+ after SQLite has ranked the matches. No conversation or searchable token is
149
+ dropped to save space.
150
+
151
+ Current builds discard a clean browser-to-corpus transport payload after both
152
+ the canonical conversation and active index have accepted it. Malformed input
153
+ is kept under `inbox/quarantine/` for inspection. Older releases archived every
154
+ successful payload and may also have left versioned indexes behind. Preview the
155
+ exact recoverable space without writing anything:
156
+
157
+ ```bash
158
+ npx -y @llmnesia/mcp@latest compact --dry-run
159
+ ```
160
+
161
+ To apply it, fully quit connected desktop AI/MCP clients, then run `compact`
162
+ without `--dry-run`. It asks for confirmation, verifies every compressed
163
+ conversation by reading it back before removing the legacy JSON, builds the
164
+ current index before deleting older derived versions, and never removes
165
+ quarantined input.
166
+
143
167
  ## Tools
144
168
 
145
169
  Once registered, your MCP client gains seven tools over the corpus:
@@ -217,6 +241,8 @@ llmnesia-mcp ingest <…> # ingest an NDJSON backup file
217
241
  llmnesia-mcp drain # ingest everything in <corpus>/inbox
218
242
  llmnesia-mcp watch # ingest new inbox deltas live (Ctrl+C to stop)
219
243
  llmnesia-mcp stats # corpus summary
244
+ llmnesia-mcp compact --dry-run # exact read-only storage/savings report
245
+ llmnesia-mcp compact # confirmed lossless migration + redundant cleanup
220
246
  llmnesia-mcp reindex # drop + rebuild the search index from the corpus
221
247
  llmnesia-mcp search <q> # one-shot search from the terminal
222
248
  llmnesia-mcp --version # print the installed version
@@ -38,7 +38,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
38
38
  ));
39
39
 
40
40
  // src/version.ts
41
- var VERSION = true ? "0.2.2" : "0.0.0-dev";
41
+ var VERSION = true ? "0.2.4" : "0.0.0-dev";
42
42
 
43
43
  export {
44
44
  __require,
@@ -3,13 +3,13 @@ import { createRequire as __llmnesiaCreateRequire } from "node:module";
3
3
  const require = __llmnesiaCreateRequire(import.meta.url);
4
4
  import {
5
5
  VERSION
6
- } from "./chunk-5ZRNGUJL.js";
6
+ } from "./chunk-JSNSTAFD.js";
7
7
 
8
8
  // src/config.ts
9
9
  import { readFileSync } from "fs";
10
10
  import { homedir } from "os";
11
11
  import { join, resolve } from "path";
12
- var CORPUS_FORMAT_VERSION = 1;
12
+ var CORPUS_FORMAT_VERSION = 2;
13
13
  function nativeHostConfigPath() {
14
14
  return join(homedir(), ".llmnesia", "config.json");
15
15
  }
@@ -39,7 +39,7 @@ function expandHome(p) {
39
39
  import { join as join2 } from "path";
40
40
  import { readdir } from "fs/promises";
41
41
  import { createHash } from "crypto";
42
- var INDEX_FORMAT_VERSION = 2;
42
+ var INDEX_FORMAT_VERSION = 3;
43
43
  function indexDbFilename(version) {
44
44
  return version === 1 ? "llmnesia.db" : `llmnesia-v${version}.db`;
45
45
  }
@@ -69,6 +69,10 @@ var CorpusPaths = class {
69
69
  get processedDir() {
70
70
  return join2(this.inboxDir, "processed");
71
71
  }
72
+ /** Input that was readable but not clean enough to discard automatically. */
73
+ get quarantineDir() {
74
+ return join2(this.inboxDir, "quarantine");
75
+ }
72
76
  get conversationsDir() {
73
77
  return join2(this.root, "conversations");
74
78
  }
@@ -87,6 +91,14 @@ var CorpusPaths = class {
87
91
  }
88
92
  /** Absolute path to the JSON file backing a given docId. */
89
93
  conversationFile(platform, docId) {
94
+ return this.compressedConversationFile(platform, docId);
95
+ }
96
+ /** Canonical v2 path. Compression is lossless and independently readable. */
97
+ compressedConversationFile(platform, docId) {
98
+ return join2(this.platformDir(platform), `${slugifyDocId(docId)}.json.gz`);
99
+ }
100
+ /** v1 path accepted until confirmed/resumable compaction migrates it. */
101
+ legacyConversationFile(platform, docId) {
90
102
  return join2(this.platformDir(platform), `${slugifyDocId(docId)}.json`);
91
103
  }
92
104
  /**
@@ -114,8 +126,10 @@ function safeSegment(value) {
114
126
  }
115
127
 
116
128
  // src/corpus.ts
117
- import { promises as fs } from "fs";
129
+ import { promises as fs, readFileSync as readFileSync2 } from "fs";
118
130
  import { dirname, join as join3 } from "path";
131
+ import { promisify } from "util";
132
+ import { gzip as gzipCallback, gunzip as gunzipCallback, gunzipSync } from "zlib";
119
133
 
120
134
  // src/serialize.ts
121
135
  function canonicalizeConversation(record) {
@@ -212,6 +226,8 @@ function assignIfDefined(target, key, value) {
212
226
  // src/corpus.ts
213
227
  var DIR_MODE = 448;
214
228
  var FILE_MODE = 384;
229
+ var gzip = promisify(gzipCallback);
230
+ var gunzip = promisify(gunzipCallback);
215
231
  var Corpus = class {
216
232
  paths;
217
233
  // Serialize writes within this process so concurrent upserts (inbox drain +
@@ -226,12 +242,14 @@ var Corpus = class {
226
242
  await fs.mkdir(this.paths.conversationsDir, { recursive: true, mode: DIR_MODE });
227
243
  await fs.mkdir(this.paths.memoriesDir, { recursive: true, mode: DIR_MODE });
228
244
  await fs.mkdir(this.paths.processedDir, { recursive: true, mode: DIR_MODE });
245
+ await fs.mkdir(this.paths.quarantineDir, { recursive: true, mode: DIR_MODE });
229
246
  await fs.mkdir(this.paths.indexDir, { recursive: true, mode: DIR_MODE });
230
247
  await hardenDir(this.paths.root);
231
248
  await hardenDir(this.paths.conversationsDir);
232
249
  await hardenDir(this.paths.memoriesDir);
233
250
  await hardenDir(this.paths.inboxDir);
234
251
  await hardenDir(this.paths.processedDir);
252
+ await hardenDir(this.paths.quarantineDir);
235
253
  await hardenDir(this.paths.indexDir);
236
254
  try {
237
255
  await fs.access(this.paths.metaFile);
@@ -275,8 +293,9 @@ var Corpus = class {
275
293
  return this.enqueueWrite(async () => {
276
294
  const canonical = canonicalizeConversation(record);
277
295
  const { conversation } = canonical;
278
- const file = this.paths.conversationFile(conversation.platform, conversation.docId);
279
- const existing = await readJsonIfExists(file);
296
+ const file = this.paths.compressedConversationFile(conversation.platform, conversation.docId);
297
+ const legacyFile = this.paths.legacyConversationFile(conversation.platform, conversation.docId);
298
+ const existing = await this.readBestConversation(conversation.platform, conversation.docId);
280
299
  if (existing) {
281
300
  const existingChangedAt = existing.conversation.corpusChangedAt ?? existing.conversation.updatedAt ?? 0;
282
301
  const incomingChangedAt = conversation.corpusChangedAt ?? conversation.updatedAt ?? 0;
@@ -286,8 +305,7 @@ var Corpus = class {
286
305
  return { docId: conversation.docId, outcome: "skipped" };
287
306
  }
288
307
  }
289
- const serialized = `${JSON.stringify(canonical, null, 2)}
290
- `;
308
+ const serialized = serializeConversation(canonical);
291
309
  if (existing) {
292
310
  const existingSerialized = `${JSON.stringify(canonicalizeConversation(existing), null, 2)}
293
311
  `;
@@ -296,22 +314,39 @@ var Corpus = class {
296
314
  }
297
315
  }
298
316
  await fs.mkdir(dirname(file), { recursive: true, mode: DIR_MODE });
299
- await writeFileAtomic(file, serialized);
317
+ await writeCompressedConversationAtomic(file, serialized);
318
+ await fs.rm(legacyFile, { force: true });
300
319
  return { docId: conversation.docId, outcome: existing ? "updated" : "created" };
301
320
  });
302
321
  }
303
322
  /** Read a conversation by docId, scanning platform dirs for the slug file. */
304
323
  async get(docId) {
305
- const filename = `${slugifyDocId(docId)}.json`;
306
324
  for (const platform of await this.listPlatforms()) {
307
- const candidate = join3(this.paths.platformDir(platform), filename);
308
- const record = await readJsonIfExists(candidate);
325
+ const record = await this.readBestConversation(platform, docId);
309
326
  if (record && record.conversation.docId === docId) {
310
327
  return record;
311
328
  }
312
329
  }
313
330
  return null;
314
331
  }
332
+ /** Direct form used by search snippet hydration, avoiding a platform scan. */
333
+ async getFromPlatform(platform, docId) {
334
+ const record = await this.readBestConversation(platform, docId);
335
+ return record?.conversation.docId === docId ? record : null;
336
+ }
337
+ /**
338
+ * Synchronous twin used after SQLite has ranked a small result set. The
339
+ * search connection is synchronous already (`node:sqlite` DatabaseSync), and
340
+ * reading at most `limit` local gzip files keeps the public retriever API
341
+ * stable while avoiding a second raw-text store inside FTS5.
342
+ */
343
+ getFromPlatformSync(platform, docId) {
344
+ const record = readBestConversationFilesSync(
345
+ this.paths.compressedConversationFile(platform, docId),
346
+ this.paths.legacyConversationFile(platform, docId)
347
+ );
348
+ return record?.conversation.docId === docId ? record : null;
349
+ }
315
350
  /**
316
351
  * Write an agent-authored memory, keyed on memoryId.
317
352
  *
@@ -385,17 +420,108 @@ var Corpus = class {
385
420
  } catch {
386
421
  continue;
387
422
  }
423
+ const stems = /* @__PURE__ */ new Set();
388
424
  for (const name of files) {
389
- if (!name.endsWith(".json")) {
390
- continue;
391
- }
392
- const record = await readJsonIfExists(join3(dir, name));
425
+ if (name.endsWith(".json.gz")) stems.add(name.slice(0, -".json.gz".length));
426
+ else if (name.endsWith(".json")) stems.add(name.slice(0, -".json".length));
427
+ }
428
+ for (const stem of [...stems].sort()) {
429
+ const record = await readBestConversationFiles(
430
+ join3(dir, `${stem}.json.gz`),
431
+ join3(dir, `${stem}.json`)
432
+ );
393
433
  if (record) {
394
434
  yield record;
395
435
  }
396
436
  }
397
437
  }
398
438
  }
439
+ /** Byte/count breakdown used by `stats` and `compact --dry-run`. */
440
+ async conversationStorageStats() {
441
+ const stats = {
442
+ legacyFiles: 0,
443
+ legacyBytes: 0,
444
+ compressedFiles: 0,
445
+ compressedBytes: 0
446
+ };
447
+ for (const platform of await this.listPlatforms()) {
448
+ const dir = this.paths.platformDir(platform);
449
+ let entries;
450
+ try {
451
+ entries = await fs.readdir(dir, { withFileTypes: true });
452
+ } catch {
453
+ continue;
454
+ }
455
+ for (const entry of entries) {
456
+ if (!entry.isFile()) continue;
457
+ const file = join3(dir, entry.name);
458
+ if (entry.name.endsWith(".json.gz")) {
459
+ stats.compressedFiles += 1;
460
+ stats.compressedBytes += (await fs.stat(file)).size;
461
+ } else if (entry.name.endsWith(".json")) {
462
+ stats.legacyFiles += 1;
463
+ stats.legacyBytes += (await fs.stat(file)).size;
464
+ }
465
+ }
466
+ }
467
+ return stats;
468
+ }
469
+ /**
470
+ * Convert every legacy conversation independently. Each candidate is written,
471
+ * read back, and byte-compared in canonical JSON form before its plaintext
472
+ * source is removed, making the operation resumable after any interruption.
473
+ */
474
+ async migrateLegacyConversations(options = {}) {
475
+ const dryRun = options.dryRun === true;
476
+ const before = await this.conversationStorageStats();
477
+ const report = {
478
+ ...before,
479
+ dryRun,
480
+ candidates: before.legacyFiles,
481
+ migrated: 0,
482
+ legacyRemoved: 0,
483
+ projectedCompressedBytes: before.compressedBytes,
484
+ failures: []
485
+ };
486
+ for (const platform of await this.listPlatforms()) {
487
+ const dir = this.paths.platformDir(platform);
488
+ let names;
489
+ try {
490
+ names = (await fs.readdir(dir)).filter((name) => name.endsWith(".json")).sort();
491
+ } catch {
492
+ continue;
493
+ }
494
+ for (const name of names) {
495
+ const legacyFile = join3(dir, name);
496
+ const compressedFile = join3(dir, `${name}.gz`);
497
+ try {
498
+ const legacy = await readPlainConversationIfExists(legacyFile);
499
+ if (!legacy) throw new Error("legacy JSON could not be parsed");
500
+ const compressed = await readCompressedConversationIfExists(compressedFile);
501
+ const winner = pickNewerConversation(compressed, legacy) ?? legacy;
502
+ const serialized = serializeConversation(canonicalizeConversation(winner));
503
+ const compressedBytes = await gzip(Buffer.from(serialized, "utf8"), { level: 9 });
504
+ report.projectedCompressedBytes -= await fileSizeIfExists(compressedFile);
505
+ report.projectedCompressedBytes += compressedBytes.byteLength;
506
+ if (!dryRun) {
507
+ await writeVerifiedCompressedConversationAtomic(compressedFile, compressedBytes, serialized);
508
+ await fs.rm(legacyFile);
509
+ }
510
+ report.migrated += 1;
511
+ report.legacyRemoved += 1;
512
+ } catch (error) {
513
+ report.failures.push({
514
+ file: legacyFile,
515
+ error: error instanceof Error ? error.message : String(error)
516
+ });
517
+ }
518
+ }
519
+ }
520
+ if (!dryRun && report.failures.length === 0) {
521
+ await this.writeMeta({ formatVersion: CORPUS_FORMAT_VERSION });
522
+ }
523
+ return report;
524
+ }
399
525
  async stats() {
400
526
  const byPlatform = {};
401
527
  let conversationCount = 0;
@@ -409,9 +535,13 @@ var Corpus = class {
409
535
  return { conversationCount, messageCount, byPlatform };
410
536
  }
411
537
  /** Refresh meta.json's conversationCount / lastIngestAt after an ingest run. */
412
- async touchIngestMeta() {
538
+ async touchIngestMeta(receipt) {
413
539
  const { conversationCount } = await this.stats();
414
- await this.writeMeta({ lastIngestAt: Date.now(), conversationCount });
540
+ await this.writeMeta({
541
+ lastIngestAt: Date.now(),
542
+ conversationCount,
543
+ ...receipt ? { lastIngestReceipt: receipt } : {}
544
+ });
415
545
  }
416
546
  enqueueWrite(fn) {
417
547
  const run = this.writeChain.then(fn, fn);
@@ -421,6 +551,12 @@ var Corpus = class {
421
551
  );
422
552
  return run;
423
553
  }
554
+ readBestConversation(platform, docId) {
555
+ return readBestConversationFiles(
556
+ this.paths.compressedConversationFile(platform, docId),
557
+ this.paths.legacyConversationFile(platform, docId)
558
+ );
559
+ }
424
560
  };
425
561
  async function readJsonIfExists(file) {
426
562
  try {
@@ -430,9 +566,83 @@ async function readJsonIfExists(file) {
430
566
  return null;
431
567
  }
432
568
  }
569
+ function serializeConversation(record) {
570
+ return `${JSON.stringify(canonicalizeConversation(record), null, 2)}
571
+ `;
572
+ }
573
+ async function readPlainConversationIfExists(file) {
574
+ return readJsonIfExists(file);
575
+ }
576
+ async function readCompressedConversationIfExists(file) {
577
+ try {
578
+ const raw = await fs.readFile(file);
579
+ return JSON.parse((await gunzip(raw)).toString("utf8"));
580
+ } catch {
581
+ return null;
582
+ }
583
+ }
584
+ async function readBestConversationFiles(compressedFile, legacyFile) {
585
+ const [compressed, legacy] = await Promise.all([
586
+ readCompressedConversationIfExists(compressedFile),
587
+ readPlainConversationIfExists(legacyFile)
588
+ ]);
589
+ return pickNewerConversation(compressed, legacy);
590
+ }
591
+ function readBestConversationFilesSync(compressedFile, legacyFile) {
592
+ let compressed = null;
593
+ let legacy = null;
594
+ try {
595
+ compressed = JSON.parse(gunzipSync(readFileSync2(compressedFile)).toString("utf8"));
596
+ } catch {
597
+ }
598
+ try {
599
+ legacy = JSON.parse(readFileSync2(legacyFile, "utf8"));
600
+ } catch {
601
+ }
602
+ return pickNewerConversation(compressed, legacy);
603
+ }
604
+ function pickNewerConversation(a, b) {
605
+ if (!a) return b;
606
+ if (!b) return a;
607
+ const aChanged = a.conversation.corpusChangedAt ?? a.conversation.updatedAt ?? 0;
608
+ const bChanged = b.conversation.corpusChangedAt ?? b.conversation.updatedAt ?? 0;
609
+ if (aChanged !== bChanged) return aChanged > bChanged ? a : b;
610
+ const aUpdated = a.conversation.updatedAt ?? 0;
611
+ const bUpdated = b.conversation.updatedAt ?? 0;
612
+ return aUpdated >= bUpdated ? a : b;
613
+ }
614
+ async function writeCompressedConversationAtomic(file, serialized) {
615
+ const compressed = await gzip(Buffer.from(serialized, "utf8"), { level: 9 });
616
+ await writeFileAtomic(file, compressed);
617
+ }
618
+ async function writeVerifiedCompressedConversationAtomic(file, compressed, serialized) {
619
+ const tmp = `${file}.migration-${process.pid}-${Date.now()}`;
620
+ try {
621
+ await fs.writeFile(tmp, compressed, { mode: FILE_MODE });
622
+ const verified = await readCompressedConversationIfExists(tmp);
623
+ if (!verified || serializeConversation(canonicalizeConversation(verified)) !== serialized) {
624
+ throw new Error("compressed read-back verification failed");
625
+ }
626
+ await fs.rename(tmp, file);
627
+ } catch (error) {
628
+ await fs.rm(tmp, { force: true });
629
+ throw error;
630
+ }
631
+ }
632
+ async function fileSizeIfExists(file) {
633
+ try {
634
+ return (await fs.stat(file)).size;
635
+ } catch {
636
+ return 0;
637
+ }
638
+ }
433
639
  async function writeFileAtomic(file, contents) {
434
640
  const tmp = `${file}.tmp-${process.pid}-${Date.now()}`;
435
- await fs.writeFile(tmp, contents, { encoding: "utf8", mode: FILE_MODE });
641
+ if (typeof contents === "string") {
642
+ await fs.writeFile(tmp, contents, { encoding: "utf8", mode: FILE_MODE });
643
+ } else {
644
+ await fs.writeFile(tmp, contents, { mode: FILE_MODE });
645
+ }
436
646
  await fs.rename(tmp, file);
437
647
  }
438
648
  async function hardenDir(dir) {
@@ -533,23 +743,46 @@ async function drainInbox(corpus, index, options = {}) {
533
743
  await corpus.init();
534
744
  const inbox = corpus.paths.inboxDir;
535
745
  const entries = await pendingInboxFiles(corpus);
536
- const total = { ...EMPTY_REPORT, files: 0, failed: 0 };
746
+ const total = {
747
+ ...EMPTY_REPORT,
748
+ files: 0,
749
+ failed: 0,
750
+ discarded: 0,
751
+ quarantined: 0
752
+ };
537
753
  for (const name of entries) {
538
754
  const src = join4(inbox, name);
539
755
  try {
540
756
  const report = await ingestFile(corpus, src, index);
541
757
  accumulate(total, report);
542
758
  total.files += 1;
543
- await moveToProcessed(corpus, src, name);
759
+ if (report.malformedLines > 0) {
760
+ await moveToQuarantine(corpus, src, name);
761
+ total.quarantined += 1;
762
+ } else {
763
+ await fs2.unlink(src);
764
+ total.discarded += 1;
765
+ }
544
766
  } catch (error) {
545
767
  if (await stillPending(src)) {
546
768
  total.failed += 1;
769
+ if (isCorruptDeltaError(error)) {
770
+ await moveToQuarantine(corpus, src, name);
771
+ total.quarantined += 1;
772
+ }
547
773
  options.onFileError?.(name, error);
548
774
  }
549
775
  }
550
776
  }
551
777
  if (total.files > 0) {
552
- await corpus.touchIngestMeta();
778
+ await corpus.touchIngestMeta({
779
+ files: total.files,
780
+ conversations: total.conversations,
781
+ messages: total.messages,
782
+ malformedLines: total.malformedLines,
783
+ discarded: total.discarded,
784
+ quarantined: total.quarantined
785
+ });
553
786
  }
554
787
  return total;
555
788
  }
@@ -561,16 +794,23 @@ async function stillPending(src) {
561
794
  return false;
562
795
  }
563
796
  }
564
- async function moveToProcessed(corpus, src, name) {
565
- const dest = join4(corpus.paths.processedDir, name);
797
+ async function moveToQuarantine(corpus, src, name) {
798
+ const dest = join4(corpus.paths.quarantineDir, name);
566
799
  try {
567
800
  await fs2.rename(src, dest);
568
801
  } catch {
569
- const alt = join4(corpus.paths.processedDir, `${Date.now()}-${name}`);
802
+ const alt = join4(corpus.paths.quarantineDir, `${Date.now()}-${name}`);
570
803
  await fs2.copyFile(src, alt);
571
804
  await fs2.unlink(src);
572
805
  }
573
806
  }
807
+ function isCorruptDeltaError(error) {
808
+ if (!error || typeof error !== "object" || !("code" in error)) {
809
+ return false;
810
+ }
811
+ const code = String(error.code ?? "");
812
+ return code === "Z_DATA_ERROR" || code === "Z_BUF_ERROR" || code.startsWith("ERR_ZLIB_");
813
+ }
574
814
  function isNdjson(name) {
575
815
  const lower = name.toLowerCase();
576
816
  return lower.endsWith(".jsonl") || lower.endsWith(".ndjson") || lower.endsWith(".jsonl.gz") || lower.endsWith(".ndjson.gz");
@@ -703,6 +943,7 @@ CREATE TABLE IF NOT EXISTS index_meta (
703
943
  var DROP_DERIVED_SQL = `
704
944
  DROP TABLE IF EXISTS conversations;
705
945
  DROP TABLE IF EXISTS messages_fts;
946
+ DROP TABLE IF EXISTS message_rows;
706
947
  DROP TABLE IF EXISTS memories;
707
948
  DROP TABLE IF EXISTS memories_fts;
708
949
  `;
@@ -728,11 +969,18 @@ CREATE TABLE IF NOT EXISTS conversations (
728
969
  CREATE INDEX IF NOT EXISTS idx_conversations_platform ON conversations(platform);
729
970
  CREATE INDEX IF NOT EXISTS idx_conversations_createdAt ON conversations(createdAt);
730
971
 
972
+ CREATE TABLE IF NOT EXISTS message_rows (
973
+ rowid INTEGER PRIMARY KEY,
974
+ docId TEXT NOT NULL,
975
+ msgIndex INTEGER NOT NULL,
976
+ role TEXT NOT NULL
977
+ );
978
+ CREATE INDEX IF NOT EXISTS idx_message_rows_docId ON message_rows(docId);
979
+
731
980
  CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
732
- docId UNINDEXED,
733
- msgIndex UNINDEXED,
734
- role UNINDEXED,
735
981
  text,
982
+ content = '',
983
+ contentless_delete = 1,
736
984
  tokenize = 'porter unicode61'
737
985
  );
738
986
 
@@ -760,7 +1008,10 @@ var SearchIndex = class {
760
1008
  stmts;
761
1009
  /** Whether the on-disk index was a stale format version when it was opened. */
762
1010
  staleOnOpen;
763
- constructor(dbPath) {
1011
+ /** Canonical source used only to hydrate the few snippets returned by rank. */
1012
+ corpus;
1013
+ constructor(dbPath, corpus) {
1014
+ this.corpus = corpus;
764
1015
  hardenDbFile(dbPath);
765
1016
  this.db = new (loadSqlite()).DatabaseSync(dbPath);
766
1017
  this.db.exec("PRAGMA journal_mode = WAL");
@@ -771,12 +1022,19 @@ var SearchIndex = class {
771
1022
  this.db.exec(DROP_DERIVED_SQL);
772
1023
  }
773
1024
  this.db.exec(SCHEMA_SQL);
1025
+ this.db.exec("PRAGMA temp_store = MEMORY");
1026
+ this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS temp.snippet_fts USING fts5(
1027
+ text, tokenize = 'porter unicode61'
1028
+ )`);
774
1029
  this.setMeta("format_version", String(INDEX_FORMAT_VERSION));
775
1030
  this.stmts = {
776
- deleteFts: this.db.prepare("DELETE FROM messages_fts WHERE docId = ?"),
777
- insertFts: this.db.prepare(
778
- "INSERT INTO messages_fts (docId, msgIndex, role, text) VALUES (?, ?, ?, ?)"
1031
+ selectFtsRows: this.db.prepare("SELECT rowid FROM message_rows WHERE docId = ?"),
1032
+ deleteFts: this.db.prepare("DELETE FROM messages_fts WHERE rowid = ?"),
1033
+ deleteMessageRows: this.db.prepare("DELETE FROM message_rows WHERE docId = ?"),
1034
+ insertMessageRow: this.db.prepare(
1035
+ "INSERT INTO message_rows (docId, msgIndex, role) VALUES (?, ?, ?)"
779
1036
  ),
1037
+ insertFts: this.db.prepare("INSERT INTO messages_fts (rowid, text) VALUES (?, ?)"),
780
1038
  deleteConversation: this.db.prepare("DELETE FROM conversations WHERE docId = ?"),
781
1039
  insertConversation: this.db.prepare(
782
1040
  `INSERT INTO conversations
@@ -792,6 +1050,12 @@ var SearchIndex = class {
792
1050
  `INSERT INTO memories
793
1051
  (memoryId, kind, label, text, source, pinned, enabled, createdAt, updatedAt)
794
1052
  VALUES (@memoryId, @kind, @label, @text, @source, @pinned, @enabled, @createdAt, @updatedAt)`
1053
+ ),
1054
+ clearSnippetFts: this.db.prepare("DELETE FROM temp.snippet_fts"),
1055
+ insertSnippetFts: this.db.prepare("INSERT INTO temp.snippet_fts (text) VALUES (?)"),
1056
+ renderSnippet: this.db.prepare(
1057
+ `SELECT snippet(snippet_fts, 0, '[', ']', '\u2026', ${SNIPPET_TOKENS}) AS snippet
1058
+ FROM temp.snippet_fts WHERE snippet_fts MATCH ?`
795
1059
  )
796
1060
  };
797
1061
  }
@@ -855,7 +1119,7 @@ var SearchIndex = class {
855
1119
  }
856
1120
  removeConversation(docId) {
857
1121
  this.transaction(() => {
858
- this.stmts.deleteFts.run(docId);
1122
+ this.deleteFtsRows(docId);
859
1123
  this.stmts.deleteConversation.run(docId);
860
1124
  });
861
1125
  }
@@ -865,7 +1129,8 @@ var SearchIndex = class {
865
1129
  * reads, which the sync {@link transaction} helper cannot hold.
866
1130
  */
867
1131
  async rebuildFrom(corpus) {
868
- this.db.exec("DELETE FROM messages_fts; DELETE FROM conversations;");
1132
+ this.corpus = corpus;
1133
+ this.db.exec("DELETE FROM messages_fts; DELETE FROM message_rows; DELETE FROM conversations;");
869
1134
  this.db.exec("DELETE FROM memories_fts; DELETE FROM memories;");
870
1135
  let conversations = 0;
871
1136
  let memories = 0;
@@ -890,15 +1155,15 @@ var SearchIndex = class {
890
1155
  // transaction() helper for single upserts, or rebuildFrom's manual one).
891
1156
  writeRecord(rec) {
892
1157
  const { conversation, messages } = rec;
893
- this.stmts.deleteFts.run(conversation.docId);
1158
+ this.deleteFtsRows(conversation.docId);
894
1159
  this.stmts.deleteConversation.run(conversation.docId);
895
1160
  const titleText = [conversation.title, conversation.preview].filter(Boolean).join("\n");
896
1161
  if (titleText.trim()) {
897
- this.stmts.insertFts.run(conversation.docId, TITLE_ROW_INDEX, TITLE_ROLE, titleText);
1162
+ this.insertFtsRow(conversation.docId, TITLE_ROW_INDEX, TITLE_ROLE, titleText);
898
1163
  }
899
1164
  for (const message of messages) {
900
1165
  if (message.text.trim()) {
901
- this.stmts.insertFts.run(conversation.docId, message.msgIndex, message.role, message.text);
1166
+ this.insertFtsRow(conversation.docId, message.msgIndex, message.role, message.text);
902
1167
  }
903
1168
  }
904
1169
  this.stmts.insertConversation.run({
@@ -915,6 +1180,17 @@ var SearchIndex = class {
915
1180
  accountLabel: conversation.accountLabel ?? null
916
1181
  });
917
1182
  }
1183
+ insertFtsRow(docId, msgIndex, role, text) {
1184
+ const inserted = this.stmts.insertMessageRow.run(docId, msgIndex, role);
1185
+ this.stmts.insertFts.run(inserted.lastInsertRowid, text);
1186
+ }
1187
+ deleteFtsRows(docId) {
1188
+ const rows = this.stmts.selectFtsRows.all(docId);
1189
+ for (const row of rows) {
1190
+ this.stmts.deleteFts.run(row.rowid);
1191
+ }
1192
+ this.stmts.deleteMessageRows.run(docId);
1193
+ }
918
1194
  // Replace a memory's index rows. Caller supplies the transaction, as with
919
1195
  // writeRecord. Memories live in their own tables rather than in messages_fts:
920
1196
  // a memory is not a turn in a conversation, and mixing them would let a
@@ -968,9 +1244,9 @@ var SearchIndex = class {
968
1244
  c.docId, c.platform, c.title, c.url, c.createdAt, c.updatedAt,
969
1245
  c.messageCount, c.preview, c.pinned, c.accountId, c.accountLabel,
970
1246
  m.msgIndex AS msgIndex, m.role AS role,
971
- snippet(messages_fts, 3, '[', ']', '\u2026', ${SNIPPET_TOKENS}) AS snippet,
972
1247
  bm25(messages_fts) AS bm25
973
- FROM messages_fts m
1248
+ FROM messages_fts
1249
+ JOIN message_rows m ON m.rowid = messages_fts.rowid
974
1250
  JOIN conversations c ON c.docId = m.docId
975
1251
  WHERE ${clauses.join(" AND ")}
976
1252
  ORDER BY bm25
@@ -996,7 +1272,7 @@ var SearchIndex = class {
996
1272
  pinned: row.pinned === 1,
997
1273
  // BM25 is negative (more negative = better); expose a positive relevance.
998
1274
  score: -row.bm25,
999
- snippet: row.snippet,
1275
+ snippet: this.hydrateSnippet(row, match),
1000
1276
  matchRole: normalizeMatchRole(row.role),
1001
1277
  matchMsgIndex: row.msgIndex
1002
1278
  };
@@ -1009,6 +1285,25 @@ var SearchIndex = class {
1009
1285
  }
1010
1286
  return hits;
1011
1287
  }
1288
+ /** Reclaim free pages and merge FTS segments after a confirmed compaction. */
1289
+ compact() {
1290
+ this.db.exec("INSERT INTO messages_fts(messages_fts) VALUES('optimize')");
1291
+ this.db.exec("INSERT INTO memories_fts(memories_fts) VALUES('optimize')");
1292
+ this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
1293
+ this.db.exec("VACUUM");
1294
+ }
1295
+ hydrateSnippet(row, match) {
1296
+ const record = this.corpus?.getFromPlatformSync(row.platform, row.docId);
1297
+ const source = row.msgIndex === TITLE_ROW_INDEX ? [record?.conversation.title ?? row.title, record?.conversation.preview ?? row.preview].filter(Boolean).join("\n") : record?.messages.find((message) => message.msgIndex === row.msgIndex)?.text ?? row.preview;
1298
+ this.stmts.clearSnippetFts.run();
1299
+ try {
1300
+ this.stmts.insertSnippetFts.run(source);
1301
+ const rendered = this.stmts.renderSnippet.get(match);
1302
+ return rendered?.snippet ?? fallbackSnippet(source, SNIPPET_TOKENS);
1303
+ } finally {
1304
+ this.stmts.clearSnippetFts.run();
1305
+ }
1306
+ }
1012
1307
  close() {
1013
1308
  this.db.close();
1014
1309
  }
@@ -1022,7 +1317,7 @@ var SearchIndex = class {
1022
1317
  };
1023
1318
  async function openSearchIndex(corpus, options = {}) {
1024
1319
  await corpus.init();
1025
- const index = new SearchIndex(corpus.paths.indexDbFile);
1320
+ const index = new SearchIndex(corpus.paths.indexDbFile, corpus);
1026
1321
  if (options.autoBuild !== false && (index.needsRebuild() || index.count() === 0 && index.countMemories() === 0)) {
1027
1322
  await index.rebuildFrom(corpus);
1028
1323
  }
@@ -1034,7 +1329,7 @@ async function deleteIndexDb(dbFile) {
1034
1329
  }
1035
1330
  }
1036
1331
  function toMatchQuery(query, mode = "any") {
1037
- const terms = query.split(/[^\p{L}\p{N}]+/u).filter((t) => t.length > 0);
1332
+ const terms = queryTerms(query);
1038
1333
  if (terms.length === 0) {
1039
1334
  return null;
1040
1335
  }
@@ -1043,6 +1338,17 @@ function toMatchQuery(query, mode = "any") {
1043
1338
  }
1044
1339
  return terms.map((t) => `"${t}"`).join(mode === "all" ? " AND " : " OR ");
1045
1340
  }
1341
+ function queryTerms(query) {
1342
+ return query.split(/[^\p{L}\p{N}]+/u).filter((term) => term.length > 0);
1343
+ }
1344
+ function fallbackSnippet(source, maxTokens) {
1345
+ const collapsed = source.replace(/\s+/g, " ").trim();
1346
+ if (!collapsed) return "";
1347
+ const words = collapsed.split(" ");
1348
+ const width = Math.max(1, maxTokens);
1349
+ const end = Math.min(words.length, width);
1350
+ return `${words.slice(0, end).join(" ")}${end < words.length ? "\u2026" : ""}`;
1351
+ }
1046
1352
  function escapeLike(value) {
1047
1353
  return value.replace(/[\\%_]/g, "\\$&");
1048
1354
  }
@@ -14,13 +14,13 @@ import {
14
14
  runtimeBinDir,
15
15
  staleIndexFiles,
16
16
  uninstallRuntime
17
- } from "./chunk-LPGVZ7XY.js";
17
+ } from "./chunk-NSVHWVEK.js";
18
18
  import {
19
19
  VERSION,
20
20
  __commonJS,
21
21
  __require,
22
22
  __toESM
23
- } from "./chunk-5ZRNGUJL.js";
23
+ } from "./chunk-JSNSTAFD.js";
24
24
 
25
25
  // ../node_modules/isexe/windows.js
26
26
  var require_windows = __commonJS({
@@ -28,7 +28,7 @@ var require_windows = __commonJS({
28
28
  "use strict";
29
29
  module.exports = isexe;
30
30
  isexe.sync = sync;
31
- var fs = __require("fs");
31
+ var fs2 = __require("fs");
32
32
  function checkPathExt(path, options) {
33
33
  var pathext = options.pathExt !== void 0 ? options.pathExt : process.env.PATHEXT;
34
34
  if (!pathext) {
@@ -53,12 +53,12 @@ var require_windows = __commonJS({
53
53
  return checkPathExt(path, options);
54
54
  }
55
55
  function isexe(path, options, cb) {
56
- fs.stat(path, function(er, stat6) {
56
+ fs2.stat(path, function(er, stat6) {
57
57
  cb(er, er ? false : checkStat(stat6, path, options));
58
58
  });
59
59
  }
60
60
  function sync(path, options) {
61
- return checkStat(fs.statSync(path), path, options);
61
+ return checkStat(fs2.statSync(path), path, options);
62
62
  }
63
63
  }
64
64
  });
@@ -69,14 +69,14 @@ var require_mode = __commonJS({
69
69
  "use strict";
70
70
  module.exports = isexe;
71
71
  isexe.sync = sync;
72
- var fs = __require("fs");
72
+ var fs2 = __require("fs");
73
73
  function isexe(path, options, cb) {
74
- fs.stat(path, function(er, stat6) {
74
+ fs2.stat(path, function(er, stat6) {
75
75
  cb(er, er ? false : checkStat(stat6, options));
76
76
  });
77
77
  }
78
78
  function sync(path, options) {
79
- return checkStat(fs.statSync(path), options);
79
+ return checkStat(fs2.statSync(path), options);
80
80
  }
81
81
  function checkStat(stat6, options) {
82
82
  return stat6.isFile() && checkMode(stat6, options);
@@ -101,7 +101,7 @@ var require_mode = __commonJS({
101
101
  var require_isexe = __commonJS({
102
102
  "../node_modules/isexe/index.js"(exports, module) {
103
103
  "use strict";
104
- var fs = __require("fs");
104
+ var fs2 = __require("fs");
105
105
  var core;
106
106
  if (process.platform === "win32" || global.TESTING_WINDOWS) {
107
107
  core = require_windows();
@@ -366,16 +366,16 @@ var require_shebang_command = __commonJS({
366
366
  var require_readShebang = __commonJS({
367
367
  "../node_modules/cross-spawn/lib/util/readShebang.js"(exports, module) {
368
368
  "use strict";
369
- var fs = __require("fs");
369
+ var fs2 = __require("fs");
370
370
  var shebangCommand = require_shebang_command();
371
371
  function readShebang(command) {
372
372
  const size = 150;
373
373
  const buffer = Buffer.alloc(size);
374
374
  let fd;
375
375
  try {
376
- fd = fs.openSync(command, "r");
377
- fs.readSync(fd, buffer, 0, size, 0);
378
- fs.closeSync(fd);
376
+ fd = fs2.openSync(command, "r");
377
+ fs2.readSync(fd, buffer, 0, size, 0);
378
+ fs2.closeSync(fd);
379
379
  } catch (e) {
380
380
  }
381
381
  return shebangCommand(buffer.toString());
@@ -525,7 +525,7 @@ var require_cross_spawn = __commonJS({
525
525
 
526
526
  // src/cli.ts
527
527
  import { createInterface } from "readline/promises";
528
- import { join as join6 } from "path";
528
+ import { join as join7 } from "path";
529
529
 
530
530
  // ../packages/native-host/install.mjs
531
531
  import { execFile } from "child_process";
@@ -824,7 +824,7 @@ var OPTION_ALIASES = {
824
824
  h: "help",
825
825
  y: "yes"
826
826
  };
827
- var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["force", "yes", "version", "help"]);
827
+ var BOOLEAN_FLAGS = /* @__PURE__ */ new Set(["force", "yes", "version", "help", "dry-run"]);
828
828
  var VALUE_OPTIONS = /* @__PURE__ */ new Set(["corpus", "limit", "platform", "title", "from", "to"]);
829
829
  function parseArgs(argv) {
830
830
  const positionals = [];
@@ -3749,6 +3749,157 @@ function formatDoctorReport(report) {
3749
3749
  return report.checks.map((c) => `${icon[c.status]} ${c.name}: ${c.detail}`).join("\n");
3750
3750
  }
3751
3751
 
3752
+ // src/storageMaintenance.ts
3753
+ import { promises as fs } from "fs";
3754
+ import { join as join6 } from "path";
3755
+ async function inspectCorpusStorage(corpus) {
3756
+ const conversations = await corpus.conversationStorageStats();
3757
+ const processed = await directoryFiles(corpus.paths.processedDir);
3758
+ const quarantine = await directoryFiles(corpus.paths.quarantineDir);
3759
+ const activeNames = [
3760
+ corpus.paths.indexDbFile,
3761
+ `${corpus.paths.indexDbFile}-wal`,
3762
+ `${corpus.paths.indexDbFile}-shm`
3763
+ ];
3764
+ const activeIndex = await explicitFiles(activeNames);
3765
+ const staleNames = await staleIndexFiles(corpus.paths.indexDir);
3766
+ const stalePaths = staleNames.flatMap((name) => {
3767
+ const base = join6(corpus.paths.indexDir, name);
3768
+ return [base, `${base}-wal`, `${base}-shm`];
3769
+ });
3770
+ const staleIndexes = await explicitFiles(stalePaths);
3771
+ const root = await directoryFiles(corpus.paths.root);
3772
+ const classifiedBytes = conversations.legacyBytes + conversations.compressedBytes + processed.bytes + quarantine.bytes + activeIndex.bytes + staleIndexes.bytes;
3773
+ const classifiedFiles = conversations.legacyFiles + conversations.compressedFiles + processed.files + quarantine.files + activeIndex.files + staleIndexes.files;
3774
+ const other = {
3775
+ files: Math.max(0, root.files - classifiedFiles),
3776
+ bytes: Math.max(0, root.bytes - classifiedBytes)
3777
+ };
3778
+ return {
3779
+ root: corpus.paths.root,
3780
+ conversations,
3781
+ processed,
3782
+ quarantine,
3783
+ activeIndex,
3784
+ staleIndexes,
3785
+ other,
3786
+ totalBytes: root.bytes,
3787
+ immediatelyReclaimableBytes: processed.bytes + staleIndexes.bytes
3788
+ };
3789
+ }
3790
+ async function compactCorpus(corpus, options = {}) {
3791
+ const dryRun = options.dryRun === true;
3792
+ const before = await inspectCorpusStorage(corpus);
3793
+ if (dryRun) {
3794
+ const compression = await corpus.migrateLegacyConversations({ dryRun: true });
3795
+ const compressionSavings = Math.max(
3796
+ 0,
3797
+ compression.legacyBytes + compression.compressedBytes - compression.projectedCompressedBytes
3798
+ );
3799
+ return {
3800
+ dryRun: true,
3801
+ before,
3802
+ compression,
3803
+ processedRemoved: before.processed,
3804
+ staleIndexesRemoved: before.staleIndexes,
3805
+ after: null,
3806
+ reclaimedBytes: before.immediatelyReclaimableBytes + compressionSavings
3807
+ };
3808
+ }
3809
+ await corpus.init();
3810
+ const processedRemoved = await removeDirectoryFiles(corpus.paths.processedDir);
3811
+ const index = await openSearchIndex(corpus);
3812
+ try {
3813
+ index.compact();
3814
+ } finally {
3815
+ index.close();
3816
+ }
3817
+ const appliedCompression = await corpus.migrateLegacyConversations();
3818
+ if (appliedCompression.failures.length > 0) {
3819
+ throw new Error(
3820
+ `Corpus compression left ${appliedCompression.failures.length} legacy file(s) untouched; stale indexes were preserved. Run compact again after fixing the reported files.`
3821
+ );
3822
+ }
3823
+ const staleIndexesRemoved = await removeStaleIndexes(corpus);
3824
+ const after = await inspectCorpusStorage(corpus);
3825
+ return {
3826
+ dryRun: false,
3827
+ before,
3828
+ compression: appliedCompression,
3829
+ processedRemoved,
3830
+ staleIndexesRemoved,
3831
+ after,
3832
+ reclaimedBytes: Math.max(0, before.totalBytes - after.totalBytes)
3833
+ };
3834
+ }
3835
+ async function removeStaleIndexes(corpus) {
3836
+ const names = await staleIndexFiles(corpus.paths.indexDir);
3837
+ const paths2 = names.flatMap((name) => {
3838
+ const base = join6(corpus.paths.indexDir, name);
3839
+ return [base, `${base}-wal`, `${base}-shm`];
3840
+ });
3841
+ const before = await explicitFiles(paths2);
3842
+ for (const name of names) {
3843
+ await deleteIndexDb(join6(corpus.paths.indexDir, name));
3844
+ }
3845
+ return before;
3846
+ }
3847
+ async function removeDirectoryFiles(dir) {
3848
+ const before = await directoryFiles(dir);
3849
+ await removeFilesRecursively(dir);
3850
+ return before;
3851
+ }
3852
+ async function removeFilesRecursively(dir) {
3853
+ let entries;
3854
+ try {
3855
+ entries = await fs.readdir(dir, { withFileTypes: true });
3856
+ } catch {
3857
+ return;
3858
+ }
3859
+ for (const entry of entries) {
3860
+ const path = join6(dir, entry.name);
3861
+ if (entry.isDirectory()) await removeFilesRecursively(path);
3862
+ else if (entry.isFile()) await fs.rm(path, { force: true });
3863
+ }
3864
+ }
3865
+ async function explicitFiles(files) {
3866
+ let count = 0;
3867
+ let bytes = 0;
3868
+ for (const file of files) {
3869
+ try {
3870
+ const stat6 = await fs.stat(file);
3871
+ if (stat6.isFile()) {
3872
+ count += 1;
3873
+ bytes += stat6.size;
3874
+ }
3875
+ } catch {
3876
+ }
3877
+ }
3878
+ return { files: count, bytes };
3879
+ }
3880
+ async function directoryFiles(dir) {
3881
+ let entries;
3882
+ try {
3883
+ entries = await fs.readdir(dir, { withFileTypes: true });
3884
+ } catch {
3885
+ return { files: 0, bytes: 0 };
3886
+ }
3887
+ let files = 0;
3888
+ let bytes = 0;
3889
+ for (const entry of entries) {
3890
+ const path = join6(dir, entry.name);
3891
+ if (entry.isDirectory()) {
3892
+ const nested = await directoryFiles(path);
3893
+ files += nested.files;
3894
+ bytes += nested.bytes;
3895
+ } else if (entry.isFile()) {
3896
+ files += 1;
3897
+ bytes += (await fs.stat(path)).size;
3898
+ }
3899
+ }
3900
+ return { files, bytes };
3901
+ }
3902
+
3752
3903
  // src/cli.ts
3753
3904
  var USAGE = `llmnesia-mcp ${VERSION}
3754
3905
 
@@ -3765,9 +3916,12 @@ Setup:
3765
3916
  Power-user / debugging:
3766
3917
  init Create the corpus directory skeleton
3767
3918
  ingest <file...> Ingest one or more NDJSON backup files into the corpus
3768
- drain Ingest every file in <corpus>/inbox, then archive them
3919
+ drain Ingest every file in <corpus>/inbox; discard clean
3920
+ payloads and quarantine malformed ones
3769
3921
  watch Watch <corpus>/inbox and ingest new deltas live
3770
3922
  stats Print corpus counts (conversations, messages, platforms)
3923
+ compact Losslessly compress conversations and remove redundant
3924
+ processed batches/stale indexes (confirmation required)
3771
3925
  reindex Drop and rebuild the search index from the corpus
3772
3926
  search <query> Full-text search the corpus (BM25)
3773
3927
 
@@ -3780,6 +3934,7 @@ Options:
3780
3934
  --to <iso|ms> search: only conversations created on/before this date
3781
3935
  --force install: overwrite a differing corpus root in config.json
3782
3936
  --yes, -y uninstall: skip its confirmation prompt
3937
+ --dry-run compact: calculate exact savings without writing
3783
3938
  --version, -v print the version and exit
3784
3939
  `;
3785
3940
  function parseDate(value) {
@@ -3984,7 +4139,7 @@ async function main() {
3984
4139
  `)
3985
4140
  });
3986
4141
  process.stdout.write(
3987
- `Drained ${report.files} inbox file(s): ${report.conversations} conversation(s) \u2014 ${report.created} created, ${report.updated} updated, ${report.skipped} unchanged` + (report.failed > 0 ? `, ${report.failed} file(s) failed` : "") + `
4142
+ `Drained ${report.files} inbox file(s): ${report.conversations} conversation(s) \u2014 ${report.created} created, ${report.updated} updated, ${report.skipped} unchanged` + (report.quarantined > 0 ? `, ${report.quarantined} quarantined` : "") + (report.failed > 0 ? `, ${report.failed} file(s) failed` : "") + `
3988
4143
  `
3989
4144
  );
3990
4145
  } finally {
@@ -4012,6 +4167,7 @@ async function main() {
4012
4167
  }
4013
4168
  case "stats": {
4014
4169
  const stats = await corpus.stats();
4170
+ const storage = await inspectCorpusStorage(corpus);
4015
4171
  const platforms = Object.entries(stats.byPlatform).sort((a, b) => b[1] - a[1]).map(([p, n]) => ` ${p}: ${n}`).join("\n");
4016
4172
  process.stdout.write(
4017
4173
  `Corpus: ${root}
@@ -4019,14 +4175,42 @@ async function main() {
4019
4175
  messages: ${stats.messageCount}
4020
4176
  ` + (platforms ? ` by platform:
4021
4177
  ${platforms}
4022
- ` : "")
4178
+ ` : "") + ` storage: ${formatBytes(storage.totalBytes)}
4179
+ conversations: ${formatBytes(storage.conversations.legacyBytes + storage.conversations.compressedBytes)} (${storage.conversations.compressedFiles} compressed, ${storage.conversations.legacyFiles} legacy)
4180
+ active index: ${formatBytes(storage.activeIndex.bytes)}
4181
+ processed: ${formatBytes(storage.processed.bytes)} (${storage.processed.files} reclaimable files)
4182
+ stale indexes: ${formatBytes(storage.staleIndexes.bytes)}
4183
+ quarantine: ${formatBytes(storage.quarantine.bytes)}
4184
+ `
4185
+ );
4186
+ return 0;
4187
+ }
4188
+ case "compact": {
4189
+ const dryRun = flags.has("dry-run");
4190
+ const preview = await compactCorpus(corpus, { dryRun: true });
4191
+ process.stdout.write(formatCompactPreview(preview));
4192
+ if (dryRun) return 0;
4193
+ if (!flags.has("yes")) {
4194
+ const ok = await promptConfirm(
4195
+ "Compact this corpus now? Close desktop AI/MCP clients first. Clean processed sync payloads and stale derived indexes will be removed; canonical conversations are migrated losslessly with read-back verification."
4196
+ );
4197
+ if (!ok) {
4198
+ process.stdout.write("Cancelled.\n");
4199
+ return 0;
4200
+ }
4201
+ }
4202
+ const result = await compactCorpus(corpus);
4203
+ process.stdout.write(
4204
+ `Compaction complete: reclaimed ${formatBytes(result.reclaimedBytes)}; ${result.compression.migrated} conversation(s) compressed, ${result.processedRemoved.files} processed file(s) removed, ${result.staleIndexesRemoved.files} stale index file(s) removed.
4205
+ Corpus now uses ${formatBytes(result.after?.totalBytes ?? 0)}.
4206
+ `
4023
4207
  );
4024
4208
  return 0;
4025
4209
  }
4026
4210
  case "reindex": {
4027
4211
  await corpus.init();
4028
4212
  await deleteIndexDb(corpus.paths.indexDbFile);
4029
- const index = new SearchIndex(corpus.paths.indexDbFile);
4213
+ const index = new SearchIndex(corpus.paths.indexDbFile, corpus);
4030
4214
  try {
4031
4215
  const { conversations } = await index.rebuildFrom(corpus);
4032
4216
  process.stdout.write(`Rebuilt index: ${conversations} conversation(s)
@@ -4036,7 +4220,7 @@ ${platforms}
4036
4220
  }
4037
4221
  const stale = await staleIndexFiles(corpus.paths.indexDir);
4038
4222
  for (const name of stale) {
4039
- await deleteIndexDb(join6(corpus.paths.indexDir, name));
4223
+ await deleteIndexDb(join7(corpus.paths.indexDir, name));
4040
4224
  }
4041
4225
  if (stale.length > 0) {
4042
4226
  process.stdout.write(
@@ -4092,6 +4276,31 @@ ${USAGE}`);
4092
4276
  function collapse(text) {
4093
4277
  return text.replace(/\s+/g, " ").trim();
4094
4278
  }
4279
+ function formatBytes(bytes) {
4280
+ if (bytes < 1024) return `${bytes} B`;
4281
+ const units = ["KiB", "MiB", "GiB", "TiB"];
4282
+ let value = bytes / 1024;
4283
+ let unit = units[0];
4284
+ for (let i = 1; i < units.length && value >= 1024; i += 1) {
4285
+ value /= 1024;
4286
+ unit = units[i];
4287
+ }
4288
+ return `${value.toFixed(value >= 10 ? 1 : 2)} ${unit}`;
4289
+ }
4290
+ function formatCompactPreview(result) {
4291
+ const { before, compression } = result;
4292
+ const compressionSavings = Math.max(
4293
+ 0,
4294
+ compression.legacyBytes + compression.compressedBytes - compression.projectedCompressedBytes
4295
+ );
4296
+ return `Compaction preview for ${before.root}:
4297
+ current size: ${formatBytes(before.totalBytes)}
4298
+ processed sync payloads: ${formatBytes(before.processed.bytes)} (${before.processed.files} files)
4299
+ stale indexes: ${formatBytes(before.staleIndexes.bytes)} (${before.staleIndexes.files} files)
4300
+ lossless compression: ${formatBytes(compressionSavings)} (${compression.candidates} legacy conversations)
4301
+ estimated reclaimable: ${formatBytes(result.reclaimedBytes)}
4302
+ `;
4303
+ }
4095
4304
  export {
4096
4305
  formatInstallSuccess,
4097
4306
  main,
package/dist/cli.js CHANGED
@@ -3,7 +3,7 @@ import { createRequire as __llmnesiaCreateRequire } from "node:module";
3
3
  const require = __llmnesiaCreateRequire(import.meta.url);
4
4
  import {
5
5
  VERSION
6
- } from "./chunk-5ZRNGUJL.js";
6
+ } from "./chunk-JSNSTAFD.js";
7
7
 
8
8
  // src/nodeDownload.ts
9
9
  var NODE_DOWNLOAD_URLS = {
@@ -141,7 +141,7 @@ async function bootstrap() {
141
141
  await guideUnsupportedNodeVersion(process.versions.node);
142
142
  return 1;
143
143
  }
144
- const { main } = await import("./cli-SLGWH5Y2.js");
144
+ const { main } = await import("./cli-ZEA7WTVW.js");
145
145
  return main();
146
146
  }
147
147
  bootstrap().then((code) => {
Binary file
package/dist/mcp.js CHANGED
@@ -9,13 +9,13 @@ import {
9
9
  pendingInboxFiles,
10
10
  resolveCorpusDir,
11
11
  scheduleSelfUpdate
12
- } from "./chunk-LPGVZ7XY.js";
12
+ } from "./chunk-NSVHWVEK.js";
13
13
  import {
14
14
  VERSION,
15
15
  __commonJS,
16
16
  __export,
17
17
  __toESM
18
- } from "./chunk-5ZRNGUJL.js";
18
+ } from "./chunk-JSNSTAFD.js";
19
19
 
20
20
  // ../node_modules/ajv/dist/compile/codegen/code.js
21
21
  var require_code = __commonJS({
@@ -21496,7 +21496,7 @@ var CompanionService = class _CompanionService {
21496
21496
  newestConversationAt: this.index.listRecent(1)[0]?.updatedAt ?? null
21497
21497
  };
21498
21498
  if (hoursSinceIngest === null && status.conversationCount === 0) {
21499
- status.staleWarning = "This corpus is empty and has never received browser data, so searches here will return nothing until it is set up. The LLMnesia browser extension is what fills it: install the extension from https://llmnesia.com, then run `npx -y @llmnesia/mcp@latest install` in a terminal to connect it.";
21499
+ status.staleWarning = "This corpus is empty and has never received browser data, so searches here will return nothing until setup finishes. The browser extension is what fills it; the connector on its own cannot. If the extension is not installed, get it from https://llmnesia.com. If it is installed, check that it is version 0.3.0 or newer, since older versions have no desktop connection at all; that both steps under Full settings \u2192 Connect desktop AI are complete, including `npx -y @llmnesia/mcp@latest install`; and that the browser has been fully quit and reopened since automatic sync was enabled.";
21500
21500
  } else if (hoursSinceIngest === null) {
21501
21501
  status.staleWarning = "No browser data has ever reached this corpus. Anything here arrived another way; the extension's native sync may not be installed or permitted.";
21502
21502
  } else if (hoursSinceIngest >= STALE_AFTER_HOURS) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@llmnesia/mcp",
3
- "version": "0.2.2",
3
+ "version": "0.2.4",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "comment:mcpName": "The MCP Registry's ownership marker: it must equal server.json's `name`, and the namespace must match how you authenticate with mcp-publisher. `com.llmnesia/...` requires DNS auth (a TXT record on llmnesia.com) — see docs/MCP_INSTALLER_BOARD.md.",
package/server.json CHANGED
@@ -3,13 +3,13 @@
3
3
  "name": "com.llmnesia/mcp",
4
4
  "title": "LLMnesia",
5
5
  "description": "Search your AI chat history from Claude, Cursor, Codex, and other MCP clients.",
6
- "version": "0.2.2",
6
+ "version": "0.2.4",
7
7
  "packages": [
8
8
  {
9
9
  "registryType": "npm",
10
10
  "registryBaseUrl": "https://registry.npmjs.org",
11
11
  "identifier": "@llmnesia/mcp",
12
- "version": "0.2.2",
12
+ "version": "0.2.4",
13
13
  "transport": {
14
14
  "type": "stdio"
15
15
  }