opencode-mempalace-persistence 2.5.1 → 2.5.3

package/README.md

@@ -177,7 +177,8 @@ The model responds
   → Model records new KG facts via MCP tools (only when something new emerged)
 
 Session goes idle / process exits
-  → Background mine of everything new since last sync
+  → Background mine of everything new since last sync (per-wing cursors:
+  each wing advances independently, so one slow wing never stalls the rest)
   → TUI toast confirms what was mined (disable with `"toasts": false`)
 
 Every MemPalace call — plugin searches, model MCP calls (search, diary,

package/dist/index.d.ts

@@ -1,4 +1,11 @@
 declare const _default: ({ client }: any) => Promise<{
+    tool: {
+        mempalace_sync: {
+            description: string;
+            args: {};
+            execute(args: Record<string, never>, context: import("@opencode-ai/plugin").ToolContext): Promise<import("@opencode-ai/plugin").ToolResult>;
+        };
+    };
     "chat.message": (input: {
         sessionID: string;
         agent?: string;

package/dist/index.js

@@ -4,6 +4,7 @@ import { homedir } from "os";
 import { join, dirname } from "path";
 import { createHash } from "crypto";
 import { fileURLToPath } from "url";
+import { tool } from "@opencode-ai/plugin";
 const HOME = homedir();
 const MEMPALACE_BIN = join(HOME, ".local/bin/mempalace");
 const OPENCODE_DB = join(HOME, ".local/share/opencode/opencode.db");
@@ -82,21 +83,35 @@ function toastsEnabled() {
     catch { }
     return true;
 }
-// Messages arrived after the sync cursor: still waiting for the next run.
-function countPendingMessages(sinceMs) {
+// Messages arrived after each wing's sync cursor: still waiting for the
+// next run. Per-wing, so one slow wing never masks the others.
+function countPendingMessages() {
     try {
+        const st = readSyncState();
         const out = runPython(`
-import sqlite3
+import sqlite3, json, re
 db = sqlite3.connect(${JSON.stringify(OPENCODE_DB)})
-n = db.execute("SELECT COUNT(*) FROM message WHERE time_created > ${sinceMs}").fetchone()[0]
+cursors = json.loads(${JSON.stringify(JSON.stringify(st.wings || {}))})
+default = ${st.last_sync_ms || 0}
+rows = db.execute("""
+  SELECT s.directory, m.time_created FROM message m
+  INNER JOIN session s ON s.id = m.session_id
+""").fetchall()
 db.close()
-print(n)
+by = {}
+for (directory, mts) in rows:
+    base = ((directory or "").rstrip("/").split("/") or ["global"])[-1] or "global"
+    wing = re.sub("[^a-zA-Z0-9_-]", "_", base)[:40] or "global"
+    if mts > cursors.get(wing, default):
+        by[wing] = by.get(wing, 0) + 1
+print(json.dumps(by))
 `);
-        const n = parseInt(out.trim(), 10);
-        return isNaN(n) ? 0 : n;
+        const byWing = JSON.parse(out);
+        const total = Object.values(byWing).reduce((a, b) => a + b, 0);
+        return { total, byWing };
     }
     catch {
-        return 0;
+        return { total: 0, byWing: {} };
     }
 }
 let cachedName = undefined;
@@ -147,7 +162,52 @@ function countPendingSuffix() {
     const pending = countPendingFiles();
     return pending > 0 ? `, ${pending} file(s) waiting to mine` : ", queue empty";
 }
-// TUI toast client (set by the factory). Fire-and-forget: headless runs
+// On-demand sync snapshot for the /memory-toast command: same facts as
+// the startup toast, plus live-mine detection. Fires a toast AND returns
+// the text (visible in transcript too).
+function liveMiners() {
+    const found = [];
+    let dir = [];
+    try {
+        dir = readdirSync(join(HOME, ".mempalace/locks"));
+    }
+    catch {
+        return found;
+    }
+    for (const f of dir) {
+        if (!f.endsWith(".lock"))
+            continue;
+        try {
+            const content = readFileSync(join(HOME, ".mempalace/locks", f), "utf-8");
+            const pid = parseInt((content.match(/(\d+)/) || [])[1] || "", 10);
+            if (!pid)
+                continue;
+            try {
+                process.kill(pid, 0);
+                found.push(`PID ${pid} (${f.replace("mine_palace_", "").replace(".lock", "").slice(0, 8)}…)`);
+            }
+            catch { }
+        }
+        catch { }
+    }
+    return found;
+}
+function syncSnapshot() {
+    const st = readSyncState();
+    const pending = countPendingFiles();
+    const miners = liveMiners();
+    const agoMin = st.last_sync_ms > 0 ? Math.round((Date.now() - st.last_sync_ms) / 60000) : -1;
+    const syncAge = agoMin < 0 ? "never" : agoMin === 0 ? "<1 min ago" : `${agoMin} min ago`;
+    const mining = miners.length > 0 ? `mining NOW (${miners.join(", ")})` : "no mine running";
+    const text = `MemPalace sync — plugin v${pluginVersion()}\n` +
+        `Last sync: ${syncAge}\n` +
+        `Backlog: ${pending} file(s) waiting\n` +
+        `Status: ${mining}`;
+    const toastMsg = miners.length > 0
+        ? `mining now (${miners.length}), ${pending} file(s) waiting, last sync ${syncAge}`
+        : `idle, ${pending} file(s) waiting, last sync ${syncAge}`;
+    return { text, toastMsg };
+}
 // (`opencode run`, no TUI attached) must never break on this.
 let tuiClient = null;
 // Throttle for routine skip notices (busy palace): at most one toast
@@ -379,15 +439,26 @@ function summarizeToolCall(tool, args, out) {
     const answered = (out || "").replace(/\s+/g, " ").slice(0, 80) || "(empty)";
     return `${short} · asked: ${asked} → ${answered}`.slice(0, 220);
 }
-function getLastSync() {
-    if (!existsSync(STATE_FILE))
-        return 0;
+function readSyncState() {
     try {
-        return JSON.parse(readFileSync(STATE_FILE, "utf-8")).last_sync_ms || 0;
+        const raw = JSON.parse(readFileSync(STATE_FILE, "utf-8"));
+        if (typeof raw?.last_sync_ms === "number") {
+            return { last_sync_ms: raw.last_sync_ms, wings: raw.wings && typeof raw.wings === "object" ? raw.wings : {} };
+        }
     }
-    catch {
+    catch { }
+    return { last_sync_ms: 0, wings: {} };
+}
+// Per-wing cursors (see PR #1524 follow-up): a global cursor stalls
+// forever when one wing keeps failing while others succeed. Each wing
+// advances independently; last_sync_ms stays the min for compatibility.
+function getLastSync(wing) {
+    if (!existsSync(STATE_FILE))
         return 0;
-    }
+    const st = readSyncState();
+    if (wing && st.wings && typeof st.wings[wing] === "number")
+        return st.wings[wing];
+    return st.last_sync_ms || 0;
 }
 function dbSync() {
     if (miningLock)
@@ -402,11 +473,12 @@ function dbSync() {
 function backfillRequested() {
     return !!process.env.OPENCODE_MEMPALACE_BACKFILL;
 }
-// Export all sessions with new messages since `sinceMs` as flat transcripts,
-// grouped by project wing (official multi-project pattern: one wing per
-// project, so memories never leak across projects). Filenames embed a
-// content hash, so re-exports are naturally idempotent.
-function exportNewSessions(sinceMs) {
+// Export sessions with new messages as flat transcripts, grouped by
+// project wing. cursorFor(wing) gives each wing its own cursor (null =
+// discovery floor: sessions with anything newer anywhere). Filenames
+// embed a content hash, so re-exports are naturally idempotent.
+function exportNewSessions(cursorFor) {
+    const sinceMs = cursorFor(null);
     const sessions = runPython(`
 import sqlite3, json
 db = sqlite3.connect(${JSON.stringify(OPENCODE_DB)})
@@ -442,12 +514,13 @@ print(json.dumps(rows))
             .replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 40) || "global";
         const label = (title || "").replace(/[^a-zA-Z0-9 _-]/g, "_") || (sessId || "").slice(0, 12);
         const prefix = `${new Date().toISOString().slice(0, 10)}_${label.slice(0, 30)}_${(sessId || "").slice(0, 8)}`;
+        const wingSince = cursorFor(wing);
         const msgs = runPython(`
 import sqlite3, json
 db = sqlite3.connect(${JSON.stringify(OPENCODE_DB)})
 rows = db.execute("""
   SELECT m.id, m.time_created, m.data FROM message m
-  WHERE m.session_id = ${JSON.stringify(sessId)} AND m.time_created > ${sinceMs}
+  WHERE m.session_id = ${JSON.stringify(sessId)} AND m.time_created > ${wingSince}
   ORDER BY m.time_created
 """).fetchall()
 texts = []
@@ -460,11 +533,24 @@ for (mid, mts, mdata_raw) in rows:
     # is created when a reply STARTS, parts stream in afterwards, and
     # finish is set only on completion. Exporting mid-reply would
     # snapshot partial parts while the cursor advances past the message
-    # timestamp — losing the rest of the reply forever. So assistant
-    # messages without finish are skipped and revisited next sync.
+    # timestamp — losing the rest of the reply forever. So unfinished
+    # replies are skipped and revisited next sync — BUT only while
+    # recently active. A reply with no new parts for a while is dead
+    # (killed session, crashed run): treating it as perpetually
+    # in-flight would pin the cursor forever (seen live: a stillborn
+    # message froze sync for 7h). Dead replies are exported as-is.
+    now_ms = int(__import__("time").time() * 1000)
+    STALE_PART_MS = 30 * 60 * 1000
+    STALE_EMPTY_MS = 10 * 60 * 1000
     if role == "assistant" and not mdata.get("finish"):
-        incomplete.append(mts)
-        continue
+        max_part = db.execute("SELECT MAX(time_created) FROM part WHERE message_id = ?", (mid,)).fetchone()[0]
+        if max_part is None:
+            alive = (now_ms - mts) < STALE_EMPTY_MS
+        else:
+            alive = (now_ms - max_part) < STALE_PART_MS
+        if alive:
+            incomplete.append(mts)
+            continue
     for (pdata_raw,) in db.execute("SELECT data FROM part WHERE message_id = ? ORDER BY time_created", (mid,)).fetchall():
         try:
             pdata = json.loads(pdata_raw)
@@ -516,8 +602,23 @@ print(json.dumps({"texts": texts, "incomplete": incomplete}))
     }
     return { wings, now: cursor };
 }
-function markSynced(now) {
-    writeFileSync(STATE_FILE, JSON.stringify({ last_sync_ms: now }));
+function markSynced(now, wing) {
+    try {
+        const st = readSyncState();
+        if (wing) {
+            st.wings = st.wings || {};
+            st.wings[wing] = now;
+            const vals = Object.values(st.wings);
+            st.last_sync_ms = vals.length > 0 ? Math.min(...vals) : now;
+        }
+        else {
+            st.last_sync_ms = now;
+        }
+        writeFileSync(STATE_FILE, JSON.stringify(st));
+    }
+    catch (e) {
+        log("state write err: " + String(e));
+    }
     lastSyncTs = Date.now();
 }
 // Default `exchange` extraction: one drawer per exchange pair, verbatim,
@@ -531,15 +632,18 @@ function mineArgs(wingDir, wing) {
     return ["mine", wingDir, "--mode", "convos", "--agent", "opencode", "--wing", wing];
 }
 function cleanupExport(wings) {
-    for (const files of wings.values()) {
-        for (const f of files) {
-            try {
-                unlinkSync(f);
+    // Wipe whole wing dirs: a successful mine filed everything in them,
+    // including orphan files from previously failed runs.
+    for (const wing of wings.keys()) {
+        try {
+            for (const f of readdirSync(join(OUT_DIR, wing))) {
+                try {
+                    unlinkSync(join(OUT_DIR, wing, f));
+                }
+                catch { }
             }
-            catch { }
         }
-    }
-    for (const wing of wings.keys()) {
+        catch { }
         try {
             rmdirSync(join(OUT_DIR, wing));
         }
@@ -562,7 +666,10 @@ function doDbSync() {
     const sinceMs = backfillRequested() ? 0 : getLastSync();
     if (backfillRequested())
         log("backfill requested: exporting full history");
-    const { wings, now } = exportNewSessions(sinceMs);
+    const cursorFor = backfillRequested()
+        ? (_wing) => 0
+        : (wing) => (wing ? getLastSync(wing) : getLastSync());
+    const { wings, now } = exportNewSessions(cursorFor);
     if (wings.size === 0)
         return;
     miningLock = true;
@@ -583,20 +690,16 @@ function doDbSync() {
     const mineNext = (i, attempt = 0) => {
         if (i >= entries.length) {
             miningLock = false;
-            // Advance state only on full success: on failure the same
-            // content-hashed files are re-exported and retried at the next
-            // sync (mine is idempotent).
-            markSynced(now);
             cleanupExport(wings);
             log("mine done");
             const names = [...wings.keys()].join(", ");
             const totalDrawers = [...wingDrawers.values()].reduce((a, b) => a + b, 0);
             const detail = totalDrawers > 0 ? ` (${totalDrawers} drawers)` : "";
             // Anything that arrived while this mine was running stays pending.
-            const remaining = countPendingMessages(now);
-            const tail = remaining > 0 ? `, ${remaining} message(s) still waiting` : ", queue empty";
+            const remaining = countPendingMessages();
+            const tail = remaining.total > 0 ? `, ${remaining.total} message(s) still waiting` : ", queue empty";
             toast("success", "MemPalace", `mined ${wingCount(wings)} session(s) → ${names}${detail}${tail}`);
-            ilog("mine", { outcome: "ok", sessions: wingCount(wings), wings: [...wings.keys()], drawers: totalDrawers, remaining });
+            ilog("mine", { outcome: "ok", sessions: wingCount(wings), wings: [...wings.keys()], drawers: totalDrawers, remaining: remaining.total });
             return;
         }
         const [wing, files] = entries[i];
@@ -644,6 +747,19 @@ function doDbSync() {
             }
             log(`mined wing ${wing} (${files.length} sessions)`);
             wingDrawers.set(wing, parseDrawers(stdout));
+            // Per-wing cursor: this wing's progress is banked even if a later
+            // wing fails — the counter never stalls on one slow wing again.
+            markSynced(now, wing);
+            for (const f of files) {
+                try {
+                    unlinkSync(f);
+                }
+                catch { }
+            }
+            try {
+                rmdirSync(join(OUT_DIR, wing));
+            }
+            catch { }
             // Truthful progress: one toast per completed wing (an exact % is
             // impossible — the mine CLI is a black box with ~4s startup cost
             // per invocation, so per-file mines would only add overhead).
@@ -665,10 +781,11 @@ function exitSync() {
         const bin = resolveBin();
         if (!bin)
             return;
-        const { wings, now } = exportNewSessions(getLastSync());
+        const { wings, now } = exportNewSessions((wing) => (wing ? getLastSync(wing) : getLastSync()));
         if (wings.size === 0)
             return;
         const deadline = Date.now() + EXIT_BUDGET_MS;
+        const done = [];
         for (const [wing] of wings) {
             const remaining = deadline - Date.now();
             if (remaining <= 0) {
@@ -681,12 +798,29 @@ function exitSync() {
                 timeout: Math.min(EXIT_WING_TIMEOUT_MS, remaining),
             });
             if (res.error || res.status !== 0) {
-                errLog(`exit mine err (${wing}): ${String(res.error || res.status)}`);
+                const why = res.error?.message || res.signal || res.status;
+                errLog(`exit mine err (${wing}): ${String(why)}`);
                 return;
             }
+            markSynced(now, wing);
+            done.push(wing);
+        }
+        for (const wing of done) {
+            for (const f of wings.get(wing) || []) {
+                try {
+                    unlinkSync(f);
+                }
+                catch { }
+            }
+            try {
+                rmdirSync(join(OUT_DIR, wing));
+            }
+            catch { }
         }
-        markSynced(now);
-        cleanupExport(wings);
+        try {
+            rmdirSync(OUT_DIR);
+        }
+        catch { }
         log("exit save done");
     }
     catch (e) {
@@ -727,6 +861,18 @@ export default (async ({ client }) => {
     process.once("SIGHUP", onExit);
     process.once("exit", onExit);
     return {
+        tool: {
+            mempalace_sync: tool({
+                description: "Show live MemPalace sync state (backlog, last sync, running mines) as a TUI toast and text. Use when the user asks how mining is going.",
+                args: {},
+                async execute() {
+                    const snap = syncSnapshot();
+                    toast("info", "MemPalace", snap.toastMsg);
+                    ilog("status", { via: "tool" });
+                    return snap.text;
+                },
+            }),
+        },
         "chat.message": async (input, output) => {
             const role = output.message.role;
             if (role !== "user")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-mempalace-persistence",
-  "version": "2.5.1",
+  "version": "2.5.3",
   "description": "OpenCode plugin — auto-sync conversations to MemPalace memory in real-time. No forced wings, KG extraction via MCP tools.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",