@pi-unipi/memory 2.4.1 → 2.5.0

package/README.md

@@ -2,7 +2,7 @@
 
 Persistent memory that survives across sessions. Stores facts, preferences, and decisions with semantic vector search, so the agent remembers what you told it last week.
 
-**Primary backend: [MemPalace](https://github.com/mempalace/mempalace)** — auto-installed via `uv` on first load, with one-way auto-migration of any existing legacy memories. If MemPalace or `uv` is unavailable, the package transparently falls back to the bundled SQLite + sqlite-vec store, so memory never hard-fails.
+**Primary backend: [MemPalace](https://github.com/mempalace/mempalace)** — auto-installed via `uv` on first load, with verified, resumable migration of existing legacy memories. If MemPalace or `uv` is unavailable, the package transparently falls back to the bundled SQLite + sqlite-vec store, so memory never hard-fails.
 
 Two storage tiers: MemPalace (or SQLite) for vector similarity search, markdown files for a durable human-readable copy you can edit by hand. Project-scoped memories stay separate per codebase, global memories are accessible everywhere.
 
@@ -76,7 +76,7 @@ Memory has no configuration file. Storage paths are fixed:
 ```
 ~/.unipi/memory/                 # UniPi memory root (legacy + markdown tier)
 ├── .mempalace-install           # Cached MemPalace venv detection
-├── .mempalace-migrated          # One-way migration completion flag
+├── .mempalace-migrated          # Versioned migration verification state
 ├── global/
 │   ├── memory.db              # Global vector DB (SQLite fallback)
 │   └── *.md                   # Global memory files
@@ -94,14 +94,18 @@ On first load, the memory package:
    `uv tool install mempalace` once (caches the venv python path in
    `~/.unipi/memory/.mempalace-install`).
 2. Pings the bridge to confirm the palace is usable.
-3. If `~/.unipi/memory/.mempalace-migrated` is absent, performs a one-way
-   read-only migration of every legacy memory (SQLite rows + markdown files
-   across all projects) into MemPalace drawers, then writes the flag.
-   Migration is idempotent (deterministic drawer IDs) and never deletes or
-   mutates legacy files.
-
-Each memory operation invokes a bundled Python bridge
-(`bridge/mempalace_bridge.py`) once via `spawnSync` (~0.5s per call). The
+3. Fingerprints the durable SQLite and markdown sources. If they differ from
+   the verified state in `~/.unipi/memory/.mempalace-migrated`, performs an
+   idempotent read-only migration into MemPalace drawers. Unchanged drawers are
+   skipped, new or changed memories are upserted, and the versioned state is
+   written only after every discovered record is verified in MemPalace.
+   Failed or partial migrations remain unmarked and retry on a later session.
+   Legacy files are never deleted or mutated.
+
+Each memory operation invokes the packaged Python bridge
+(`bridge/mempalace_bridge.py`) once via `spawnSync` (~0.5s per call). Both the
+standalone memory package and the all-in-one umbrella tarball ship and resolve
+this bridge. The
 first MemPalace use on a machine also downloads the default ONNX embedding
 model (~80MB, cached at `~/.cache/chroma/onnx_models/`).
 
@@ -109,7 +113,7 @@ model (~80MB, cached at `~/.cache/chroma/onnx_models/`).
 
 ```bash
 rm ~/.unipi/memory/.mempalace-install    # re-detect MemPalace next session
-rm ~/.unipi/memory/.mempalace-migrated   # re-run one-way migration next session
+rm ~/.unipi/memory/.mempalace-migrated   # force a full verified migration pass next session
 ```
 
 ### Backend override

package/bridge/mempalace_bridge.py

@@ -245,7 +245,13 @@ def parse_markdown_memory(project: str, path: Path) -> dict[str, Any] | None:
     if parsed is None:
         return None
     fm, body = parsed
-    mid = safe_id_part(str(fm.get("id") or path.stem))
+    # Explicit IDs are authoritative and must not be normalized: UniPi's TS
+    # store permits leading/trailing underscores. Legacy files lack `id`, so
+    # retain their established filename normalization.
+    explicit_id = fm.get("id")
+    mid = str(explicit_id) if explicit_id is not None else safe_id_part(path.stem)
+    if not mid:
+        mid = "unknown"
     return {
         "project": str(fm.get("project") or project),
         "id": mid,
@@ -487,17 +493,91 @@ class Bridge:
         return synced
 
     def migrate(self, source_dir: str, project_filter: list[str] | None = None) -> dict[str, Any]:
+        """Idempotently import and then verify every discovered UniPi record.
+
+        Migration callers must not infer success merely from a bridge process
+        exiting cleanly. Return explicit discovery/failure/verification counts
+        so UniPi only writes its completion marker after full verification.
+        """
         records = discover_legacy_memories(Path(source_dir), project_filter)
         imported = 0
+        skipped = 0
+        failed = 0
+        errors: list[str] = []
         by_project: dict[str, int] = {}
+        expected = {(rec["project"], rec["id"]) for rec in records}
+
+        # Read once and skip unchanged records. Without this, adding one new
+        # markdown memory would re-embed every historical drawer on catch-up.
+        got = self.collection.get()
+        docs = got.documents if hasattr(got, "documents") else got.get("documents", [])
+        metas = got.metadatas if hasattr(got, "metadatas") else got.get("metadatas", [])
+        existing_docs: dict[tuple[str, str], str] = {}
+        for doc, meta in zip(docs, metas):
+            existing = record_from_doc(doc, meta)
+            if not existing:
+                continue
+            existing_docs[(existing["project"], existing["id"])] = doc
+
         for rec in records:
+            key = (rec["project"], rec["id"])
+            expected_doc = build_document(
+                rec["title"], rec["content"], rec["tags"], rec["project"],
+                rec.get("created", ""), rec.get("updated", ""), rec["type"], rec["id"],
+            )
+            if existing_docs.get(key) == expected_doc:
+                skipped += 1
+                continue
             try:
                 self._upsert_one(rec)
                 imported += 1
+                existing_docs[key] = expected_doc
                 by_project[rec["project"]] = by_project.get(rec["project"], 0) + 1
-            except Exception:
-                pass
-        return {"imported": imported, "projects": by_project}
+            except Exception as exc:
+                failed += 1
+                if len(errors) < 20:
+                    errors.append(
+                        f"{rec['project']}/{rec['id']}: {type(exc).__name__}: {exc}"
+                    )
+
+        # Re-read after writes and verify exact durable documents, not just
+        # optimistic in-memory bookkeeping or collection counts. A palace may
+        # also contain drawers created by other harnesses/import recipes.
+        verified_get = self.collection.get()
+        verified_docs = (
+            verified_get.documents
+            if hasattr(verified_get, "documents")
+            else verified_get.get("documents", [])
+        )
+        verified_metas = (
+            verified_get.metadatas
+            if hasattr(verified_get, "metadatas")
+            else verified_get.get("metadatas", [])
+        )
+        persisted: dict[tuple[str, str], str] = {}
+        for doc, meta in zip(verified_docs, verified_metas):
+            persisted_rec = record_from_doc(doc, meta)
+            if persisted_rec:
+                persisted[(persisted_rec["project"], persisted_rec["id"])] = doc
+        verified = 0
+        for rec in records:
+            expected_doc = build_document(
+                rec["title"], rec["content"], rec["tags"], rec["project"],
+                rec.get("created", ""), rec.get("updated", ""), rec["type"], rec["id"],
+            )
+            if persisted.get((rec["project"], rec["id"])) == expected_doc:
+                verified += 1
+
+        return {
+            "discovered": len(records),
+            "imported": imported,
+            "updated": imported,
+            "skipped": skipped,
+            "failed": failed,
+            "verified": verified,
+            "projects": by_project,
+            "errors": errors,
+        }
 
 
 # ---------------------------------------------------------------------------

package/mempalace.ts

@@ -12,6 +12,8 @@
  */
 
 import { spawnSync, spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createRequire } from "node:module";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import * as os from "node:os";
@@ -27,11 +29,74 @@ const MIGRATED_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-mi
 const PING_VERIFIED_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-ping-verified");
 const PING_VERIFIED_TTL_MS = 24 * 60 * 60 * 1000; // 24h
 
-/** Path to the bundled bridge script. */
-const BRIDGE_PATH = path.join(dirname(fileURLToPath(import.meta.url)), "bridge", "mempalace_bridge.py");
+/** Migration marker schema. Increment when migration semantics change. */
+export const MIGRATION_STATE_VERSION = 2;
+
+export interface MigrationResult {
+  discovered: number;
+  imported: number;
+  updated: number;
+  skipped: number;
+  failed: number;
+  verified: number;
+  errors?: string[];
+}
+
+export interface MigrationState {
+  version: number;
+  completedAt: string;
+  sourceFingerprint: string;
+  result: MigrationResult;
+}
+
+let cachedBridgePath: string | null | undefined;
+
+function isReadableFile(candidate: string): boolean {
+  try {
+    return fs.statSync(candidate).isFile() && fs.accessSync(candidate, fs.constants.R_OK) === undefined;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Resolve the Python bridge in both supported layouts:
+ *
+ * - standalone @pi-unipi/memory: <package>/bridge/mempalace_bridge.py
+ * - bundled @pi-unipi/unipi:     <umbrella>/packages/memory/bridge/...
+ *
+ * The explicit environment override is useful for custom packagers. The
+ * package-resolution fallback handles npm layouts where dependencies are not
+ * hoisted beside the umbrella package.
+ */
+export function resolveMempalaceBridgePath(moduleUrl = import.meta.url): string | null {
+  const moduleDir = path.dirname(fileURLToPath(moduleUrl));
+  const candidates: string[] = [];
+  if (process.env.UNIPI_MEMPALACE_BRIDGE) {
+    candidates.push(path.resolve(process.env.UNIPI_MEMPALACE_BRIDGE));
+  }
+  candidates.push(
+    path.join(moduleDir, "bridge", "mempalace_bridge.py"),
+    path.join(moduleDir, "..", "memory", "bridge", "mempalace_bridge.py"),
+  );
+
+  try {
+    const require = createRequire(moduleUrl);
+    const memoryPackage = require.resolve("@pi-unipi/memory/package.json");
+    candidates.push(path.join(path.dirname(memoryPackage), "bridge", "mempalace_bridge.py"));
+  } catch { /* standalone/source layout may not expose package resolution */ }
+
+  return candidates.find(isReadableFile) ?? null;
+}
+
+function getBridgePath(): string | null {
+  if (cachedBridgePath === undefined) cachedBridgePath = resolveMempalaceBridgePath();
+  return cachedBridgePath;
+}
 
-function dirname(p: string): string {
-  return path.dirname(p);
+/** Clear bridge discovery cache (primarily for recovery/tests). */
+export function invalidateBridgePathCache(): void {
+  cachedBridgePath = undefined;
 }
 
 export interface BridgeResponse<T> {
@@ -196,22 +261,99 @@ export function invalidatePingVerified(): void {
   try { if (fs.existsSync(PING_VERIFIED_FLAG)) fs.unlinkSync(PING_VERIFIED_FLAG); } catch { /* ignore */ }
 }
 
-/** Has the one-way legacy migration been completed? */
-export function isMigrated(): boolean {
-  return fs.existsSync(MIGRATED_FLAG);
+/**
+ * Fingerprint all durable legacy sources. This makes migration catch-up
+ * automatic for existing installations instead of treating a years-old
+ * timestamp flag as permanently complete.
+ */
+export function getMemorySourceFingerprint(
+  sourceDir = path.join(os.homedir(), ".unipi", "memory"),
+): string {
+  const hash = createHash("sha256");
+  if (!fs.existsSync(sourceDir)) return hash.update("missing").digest("hex");
+
+  const visit = (dir: string): void => {
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true })
+        .sort((a, b) => a.name.localeCompare(b.name));
+    } catch {
+      hash.update(`unreadable:${dir}`);
+      return;
+    }
+    for (const entry of entries) {
+      if (entry.name.startsWith(".")) continue;
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        visit(full);
+        continue;
+      }
+      if (!entry.isFile() || (entry.name !== "memory.db" && !entry.name.endsWith(".md"))) continue;
+      try {
+        const stat = fs.statSync(full);
+        hash.update(`${path.relative(sourceDir, full)}\0${stat.size}\0${stat.mtimeMs}\n`);
+      } catch {
+        hash.update(`unreadable:${path.relative(sourceDir, full)}\n`);
+      }
+    }
+  };
+  visit(sourceDir);
+  return hash.digest("hex");
 }
 
-/** Mark the one-way legacy migration complete. */
-export function markMigrated(): void {
+/** Read a verified v2 migration state. Legacy timestamp markers return null. */
+export function readMigrationState(flagPath = MIGRATED_FLAG): MigrationState | null {
   try {
-    fs.mkdirSync(path.dirname(MIGRATED_FLAG), { recursive: true });
-    fs.writeFileSync(MIGRATED_FLAG, new Date().toISOString(), "utf-8");
-  } catch { /* ignore */ }
+    const parsed = JSON.parse(fs.readFileSync(flagPath, "utf-8")) as MigrationState;
+    if (
+      parsed?.version !== MIGRATION_STATE_VERSION ||
+      typeof parsed.completedAt !== "string" ||
+      typeof parsed.sourceFingerprint !== "string" ||
+      !parsed.result ||
+      parsed.result.failed !== 0 ||
+      parsed.result.verified !== parsed.result.discovered
+    ) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+/** Is the palace verified against the current durable source set? */
+export function isMigrated(
+  sourceFingerprint = getMemorySourceFingerprint(),
+  flagPath = MIGRATED_FLAG,
+): boolean {
+  return readMigrationState(flagPath)?.sourceFingerprint === sourceFingerprint;
+}
+
+/** Mark migration complete only after the caller has verified every record. */
+export function markMigrated(
+  sourceFingerprint: string,
+  result: MigrationResult,
+  flagPath = MIGRATED_FLAG,
+): boolean {
+  if (result.failed !== 0 || result.verified !== result.discovered) return false;
+  try {
+    fs.mkdirSync(path.dirname(flagPath), { recursive: true });
+    const state: MigrationState = {
+      version: MIGRATION_STATE_VERSION,
+      completedAt: new Date().toISOString(),
+      sourceFingerprint,
+      result,
+    };
+    const temp = `${flagPath}.${process.pid}.tmp`;
+    fs.writeFileSync(temp, JSON.stringify(state, null, 2), "utf-8");
+    fs.renameSync(temp, flagPath);
+    return true;
+  } catch {
+    return false;
+  }
 }
 
 /** Force re-migration by clearing the flag. */
-export function clearMigratedFlag(): void {
-  try { if (fs.existsSync(MIGRATED_FLAG)) fs.unlinkSync(MIGRATED_FLAG); } catch { /* ignore */ }
+export function clearMigratedFlag(flagPath = MIGRATED_FLAG): void {
+  try { if (fs.existsSync(flagPath)) fs.unlinkSync(flagPath); } catch { /* ignore */ }
 }
 
 /**
@@ -223,7 +365,10 @@ export function runBridge<T = unknown>(
   palace: string,
   cmd: string,
   args: Record<string, unknown> = {},
+  timeoutMs = 60_000,
 ): T | null {
+  const bridgePath = getBridgePath();
+  if (!bridgePath) return null;
   let argsJson: string;
   try {
     argsJson = JSON.stringify(args);
@@ -232,9 +377,9 @@ export function runBridge<T = unknown>(
   }
   let res;
   try {
-    res = spawnSync(install.python, [BRIDGE_PATH, palace, cmd, argsJson], {
+    res = spawnSync(install.python, [bridgePath, palace, cmd, argsJson], {
       encoding: "utf-8",
-      timeout: 60_000,
+      timeout: timeoutMs,
       maxBuffer: 64 * 1024 * 1024,
     });
   } catch {
@@ -266,8 +411,14 @@ export function runBridgeAsync<T = unknown>(
   palace: string,
   cmd: string,
   args: Record<string, unknown> = {},
+  timeoutMs = 60_000,
 ): Promise<T | null> {
   return new Promise((resolve) => {
+    const bridgePath = getBridgePath();
+    if (!bridgePath) {
+      resolve(null);
+      return;
+    }
     let argsJson: string;
     try {
       argsJson = JSON.stringify(args);
@@ -278,7 +429,7 @@ export function runBridgeAsync<T = unknown>(
 
     let child;
     try {
-      child = spawn(install.python, [BRIDGE_PATH, palace, cmd, argsJson], {
+      child = spawn(install.python, [bridgePath, palace, cmd, argsJson], {
         stdio: ["ignore", "pipe", "ignore"],
       });
     } catch {
@@ -298,7 +449,7 @@ export function runBridgeAsync<T = unknown>(
     const timer = setTimeout(() => {
       try { child.kill(); } catch { /* already gone */ }
       finish(null);
-    }, 60_000);
+    }, timeoutMs);
     // Do not hold the process open purely for a background bridge call.
     timer.unref?.();
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/memory",
-  "version": "2.4.1",
+  "version": "2.5.0",
   "description": "Persistent cross-session memory with MemPalace backend (auto-installed) and SQLite fallback for Pi coding agent",
   "type": "module",
   "main": "index.ts",
@@ -43,8 +43,8 @@
     "better-sqlite3": "^12.9.0",
     "sqlite-vec": "^0.1.9",
     "js-yaml": "^4.1.0",
-    "@pi-unipi/core": "2.4.1",
-    "@pi-unipi/info-screen": "2.4.1"
+    "@pi-unipi/core": "2.5.0",
+    "@pi-unipi/info-screen": "2.5.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",

package/storage.ts

@@ -20,6 +20,7 @@ import {
   runBridgeAsync,
   isMigrated,
   markMigrated,
+  getMemorySourceFingerprint,
   isPingVerified,
   markPingVerified,
   invalidatePingVerified,
@@ -29,6 +30,7 @@ import {
   type MempalaceSearchResult,
   type MempalaceListItem,
   type MempalaceListItemAll,
+  type MigrationResult,
 } from "./mempalace.js";
 
 export type MemoryBackend = "mempalace" | "sqlite";
@@ -102,6 +104,7 @@ export interface SearchResult {
 
 /** Memory file frontmatter */
 interface MemoryFrontmatter {
+  id?: string;
   title: string;
   tags: string[];
   project: string;
@@ -201,7 +204,7 @@ export function parseMemoryContent(content: string): MemoryRecord | null {
   const frontmatter = yaml.load(frontmatterStr) as MemoryFrontmatter;
 
   return {
-    id: "",
+    id: frontmatter.id || "",
     title: frontmatter.title,
     content: body.trim(),
     tags: frontmatter.tags || [],
@@ -217,6 +220,7 @@ export function parseMemoryContent(content: string): MemoryRecord | null {
  */
 export function writeMemoryFile(filePath: string, record: MemoryRecord): void {
   const frontmatter: MemoryFrontmatter = {
+    id: record.id,
     title: record.title,
     tags: record.tags,
     project: record.project,
@@ -383,16 +387,28 @@ export class MemoryStorage {
     this.mempalaceInstall = install;
     this.backend = "mempalace";
 
-    // One-way auto-migration of legacy memories (idempotent).
-    if (!isMigrated()) {
+    // Idempotent migration + automatic catch-up. The source fingerprint turns
+    // the old one-shot timestamp into a resumable state: new/changed markdown
+    // or SQLite sources trigger another verified upsert pass. Never mark a
+    // failed/partial run complete; it will retry on a later session.
+    const sourceFingerprint = getMemorySourceFingerprint(getMemoryBaseDir());
+    if (!isMigrated(sourceFingerprint)) {
       try {
-        runBridge(install, this.palacePath, "migrate", {
+        // First migrations can embed thousands of records. Give the bridge a
+        // practical bounded window rather than the normal per-operation 60s.
+        const result = runBridge<MigrationResult>(install, this.palacePath, "migrate", {
           source_dir: getMemoryBaseDir(),
-        });
-        markMigrated();
+        }, 15 * 60_000);
+        if (
+          result &&
+          result.failed === 0 &&
+          result.verified === result.discovered
+        ) {
+          markMigrated(sourceFingerprint, result);
+        }
       } catch {
-        // Migration failed — palace still usable for new stores; legacy
-        // memories can be re-migrated later. Do not block.
+        // Palace remains available for current writes; durable markdown/SQLite
+        // sources are untouched and migration retries because no state is set.
       }
     }
 
@@ -629,8 +645,9 @@ export class MemoryStorage {
       const record = parseMemoryFile(filePath);
       if (!record) continue;
 
-      // Generate ID from title (same logic as store())
-      const id = record.title.toLowerCase().replace(/[^a-z0-9]+/g, "_");
+      // New files preserve the authoritative store ID. Legacy files have no
+      // `id` frontmatter, so retain the historical title-derived fallback.
+      const id = record.id || record.title.toLowerCase().replace(/[^a-z0-9]+/g, "_");
 
       if (existingIds.has(id)) continue; // Already in DB