opencode-mempalace-persistence 2.7.2 → 2.8.0

package/README.md

@@ -209,6 +209,10 @@ Next time you ask
 
 Every turn (question + answer) is saved as a drawer in MemPalace. Mining runs with `--mode convos` (default `exchange` extraction: one drawer per exchange pair, verbatim, no paraphrasing). Exports are grouped one wing per project (official multi-project pattern: `bot-oc` sessions land in wing `bot-oc`, never leaking across projects). Only completed turns are exported (in-flight replies are revisited by the next sync). The model additionally records KG facts (decisions, milestones, preferences) during conversation and at each checkpoint via MCP tools.
 
+### Message-level dedup
+
+Each opencode message is exported **exactly once ever**: exported message IDs are tracked in `sync_state.json` (retained 90 days / 200k entries) and skipped on later runs. This kills the main duplicate source mempalace's file-level dedup cannot catch — repeated boilerplate (e.g. system prompts re-sent every turn) landing in different export files. (`mempalace dedup` only compares drawers from the *same* source file, so it can't fix that either.)
+
 ### Backfill existing sessions
 
 To mine the full opencode history once (e.g. on first install):

package/dist/index.js

@@ -26,6 +26,9 @@ const LOG_FILE = "/tmp/opencode-mempalace.log";
 const MAX_INJECT_CHARS = 900;
 const MAX_SEARCH_RESULTS = 3;
 const MAX_WAKEUP_CHARS = 1500;
+// Message-ID retention for export dedup: age + size caps (see commitExportedIds).
+const MINED_IDS_MAX_AGE_MS = 90 * 24 * 3600 * 1000;
+const MINED_IDS_MAX_ENTRIES = 200000;
 // Official MemPalace hook cadence: AI checkpoint every N human messages.
 const DEFAULT_SAVE_INTERVAL = 15;
 function log(msg) {
@@ -419,11 +422,53 @@ function readSyncState() {
     try {
         const raw = JSON.parse(readFileSync(STATE_FILE, "utf-8"));
         if (typeof raw?.last_sync_ms === "number") {
-            return { last_sync_ms: raw.last_sync_ms, wings: raw.wings && typeof raw.wings === "object" ? raw.wings : {} };
+            return {
+                last_sync_ms: raw.last_sync_ms,
+                wings: raw.wings && typeof raw.wings === "object" ? raw.wings : {},
+                mined_ids: raw.mined_ids && typeof raw.mined_ids === "object" ? raw.mined_ids : {},
+            };
         }
     }
     catch { }
-    return { last_sync_ms: 0, wings: {} };
+    return { last_sync_ms: 0, wings: {}, mined_ids: {} };
+}
+// Message IDs already exported in a previous run. An ID older than every
+// cursor can never be selected again (queries use time_created > cursor),
+// so the set is pruned to stay small.
+function loadMinedIds() {
+    try {
+        const raw = (readSyncState().mined_ids || {});
+        return new Map(Object.entries(raw).filter(([, ts]) => typeof ts === "number"));
+    }
+    catch {
+        return new Map();
+    }
+}
+function commitExportedIds(byWing) {
+    try {
+        const st = readSyncState();
+        const merged = { ...(st.mined_ids || {}) };
+        for (const ids of byWing.values()) {
+            for (const [mid, ts] of ids) {
+                if (typeof ts === "number")
+                    merged[mid] = ts;
+            }
+        }
+        // Retention by AGE (90d) and SIZE (200k newest) — NOT by cursor.
+        // Cursors move backward on incomplete clamps and stall on failures;
+        // cursor-based pruning dropped IDs that future exports reselect,
+        // silently disabling the filter (seen live: set always empty).
+        const cutoff = Date.now() - MINED_IDS_MAX_AGE_MS;
+        let entries = Object.entries(merged).filter(([, ts]) => typeof ts === "number" && ts >= cutoff);
+        if (entries.length > MINED_IDS_MAX_ENTRIES) {
+            entries = entries.sort((a, b) => b[1] - a[1]).slice(0, MINED_IDS_MAX_ENTRIES);
+        }
+        st.mined_ids = Object.fromEntries(entries);
+        writeFileSync(STATE_FILE, JSON.stringify(st));
+    }
+    catch (e) {
+        log("mined-ids write err: " + String(e));
+    }
 }
 // Per-wing cursors (see PR #1524 follow-up): a global cursor stalls
 // forever when one wing keeps failing while others succeed. Each wing
@@ -474,15 +519,21 @@ print(json.dumps(rows))
         sessionsArr = JSON.parse(sessions);
     }
     catch {
-        return { wings: new Map(), now: Date.now() };
+        return { wings: new Map(), now: Date.now(), exportedIds: new Map() };
     }
     if (!sessionsArr || sessionsArr.length === 0)
-        return { wings: new Map(), now: Date.now() };
+        return { wings: new Map(), now: Date.now(), exportedIds: new Map() };
     const now = Date.now();
     // Never advance the cursor past an in-flight reply: anything skipped
     // as incomplete is revisited by the next sync (idle/exit/startup).
     let cursor = now;
     const wings = new Map();
+    // Already-mined IDs loaded ONCE per export (state file can be MBs).
+    const seen = loadMinedIds();
+    // Message IDs written to export files, PER WING. Recorded only when
+    // that wing mines successfully — recording another wing's IDs early
+    // would skip its content forever on failure.
+    const exportedByWing = new Map();
     mkdirSync(OUT_DIR, { recursive: true, mode: 0o700 });
     for (const sess of sessionsArr) {
         const [sessId, title, , directory] = sess;
@@ -531,7 +582,7 @@ for (mid, mts, mdata_raw) in rows:
         try:
             pdata = json.loads(pdata_raw)
             if pdata.get("type") == "text" and pdata.get("text","").strip():
-                texts.append({"role": role, "text": pdata.get("text").strip(), "ts": mts})
+                texts.append({"mid": mid, "role": role, "text": pdata.get("text").strip(), "ts": mts})
         except: pass
 db.close()
 print(json.dumps({"texts": texts, "incomplete": incomplete}))
@@ -540,7 +591,11 @@ print(json.dumps({"texts": texts, "incomplete": incomplete}))
         let incompleteTs = [];
         try {
             const parsed = JSON.parse(msgs);
-            msgList = parsed.texts;
+            // Message-level dedup: each message is exported exactly once ever.
+            // Repeated boilerplate (system prompts re-sent every turn) across
+            // overlapping windows was the main duplicate source mempalace's
+            // file-level dedup cannot catch (different files, same paragraph).
+            msgList = (parsed.texts || []).filter((m) => m && m.mid && !seen.has(m.mid));
             incompleteTs = parsed.incomplete || [];
         }
         catch {
@@ -572,11 +627,18 @@ print(json.dumps({"texts": texts, "incomplete": incomplete}))
         mkdirSync(wingDir, { recursive: true, mode: 0o700 });
         const fname = `sync_${prefix}_${contentHash}.txt`;
         writeFileSync(join(wingDir, fname), content + "\n", { mode: 0o600 });
+        if (!exportedByWing.has(wing))
+            exportedByWing.set(wing, new Map());
+        const wingIds = exportedByWing.get(wing);
+        for (const m of msgList) {
+            if (m && m.mid && typeof m.ts === "number")
+                wingIds.set(m.mid, m.ts);
+        }
         if (!wings.has(wing))
             wings.set(wing, []);
         wings.get(wing).push(join(wingDir, fname));
     }
-    return { wings, now: cursor };
+    return { wings, now: cursor, exportedIds: exportedByWing };
 }
 function markSynced(now, wing) {
     try {
@@ -645,7 +707,7 @@ function doDbSync() {
     const cursorFor = backfillRequested()
         ? (_wing) => 0
         : (wing) => (wing ? getLastSync(wing) : getLastSync());
-    const { wings, now } = exportNewSessions(cursorFor);
+    const { wings, now, exportedIds } = exportNewSessions(cursorFor);
     if (wings.size === 0)
         return;
     miningLock = true;
@@ -725,7 +787,10 @@ function doDbSync() {
             wingDrawers.set(wing, parseDrawers(stdout));
             // Per-wing cursor: this wing's progress is banked even if a later
             // wing fails — the counter never stalls on one slow wing again.
+            // Only THIS wing's message IDs are recorded: other wings' content
+            // is not filed yet, recording it would skip it forever on failure.
             markSynced(now, wing);
+            commitExportedIds(new Map([[wing, exportedIds.get(wing) || new Map()]]));
             for (const f of files) {
                 try {
                     unlinkSync(f);
@@ -757,7 +822,7 @@ function exitSync() {
         const bin = resolveBin();
         if (!bin)
             return;
-        const { wings, now } = exportNewSessions((wing) => (wing ? getLastSync(wing) : getLastSync()));
+        const { wings, now, exportedIds } = exportNewSessions((wing) => (wing ? getLastSync(wing) : getLastSync()));
         if (wings.size === 0)
             return;
         const deadline = Date.now() + EXIT_BUDGET_MS;
@@ -779,6 +844,7 @@ function exitSync() {
                 return;
             }
             markSynced(now, wing);
+            commitExportedIds(new Map([[wing, exportedIds.get(wing) || new Map()]]));
             done.push(wing);
         }
         for (const wing of done) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-mempalace-persistence",
-  "version": "2.7.2",
+  "version": "2.8.0",
   "description": "OpenCode plugin — auto-sync conversations to MemPalace memory in real-time. No forced wings, KG extraction via MCP tools.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",