@pi-unipi/memory 2.20.3 → 2.20.5

package/README.md

@@ -2,7 +2,7 @@
 
 Persistent memory that survives across sessions. Stores facts, preferences, and decisions with semantic vector search, so the agent remembers what you told it last week.
 
-**Primary backend: [MemPalace](https://github.com/mempalace/mempalace)** — auto-installed via `uv` on first load, with verified, resumable migration of existing legacy memories. If MemPalace or `uv` is unavailable, the package transparently falls back to the bundled SQLite + sqlite-vec store, so memory never hard-fails.
+**Backend: [MemPalace](https://github.com/mempalace/mempalace)** — auto-installed via `uv` on first load, with verified, resumable, incremental migration of existing markdown memories. Detection, ping, and migration run off the startup path, so loading the package never blocks the first prompt.
 
 Two storage tiers: MemPalace (or SQLite) for vector similarity search, markdown files for a durable human-readable copy you can edit by hand. Project-scoped memories stay separate per codebase, global memories are accessible everywhere.
 
@@ -74,46 +74,56 @@ Examples:
 Memory has no configuration file. Storage paths are fixed:
 
 ```
-~/.unipi/memory/                 # UniPi memory root (legacy + markdown tier)
+~/.unipi/memory/                 # UniPi memory root (markdown tier)
 ├── .mempalace-install           # Cached MemPalace venv detection
-├── .mempalace-migrated          # Versioned migration verification state
+├── .mempalace-migrated          # Legacy migration marker (seeds the ledger once)
+├── .mempalace-ledger.json       # Record-level sync ledger (project/id -> content hash)
+├── .mempalace-ping-verified     # Recent-ping cache (skips the cold-start ping)
 ├── global/
-│   ├── memory.db              # Global vector DB (SQLite fallback)
-│   └── *.md                   # Global memory files
+│   └── *.md                   # Global memory files (durable, human-readable)
 └── <project_name>/
-    ├── memory.db              # Project vector DB (SQLite fallback)
-    └── *.md                   # Project memory files
+    └── *.md                   # Project memory files (durable, human-readable)
 
-~/.mempalace/palace/             # MemPalace palace (primary backend)
+~/.mempalace/palace/             # MemPalace palace (vector backend)
 ```
 
 ## MemPalace backend
 
-On first load, the memory package:
+MemPalace is the sole vector backend; the markdown files are the durable,
+human-readable tier and the migration source. On load, the memory package:
 1. Detects MemPalace; if missing and `uv` is available, runs
    `uv tool install mempalace` once (caches the venv python path in
    `~/.unipi/memory/.mempalace-install`).
-2. Pings the bridge to confirm the palace is usable.
-3. Fingerprints the durable SQLite and markdown sources. If they differ from
-   the verified state in `~/.unipi/memory/.mempalace-migrated`, performs an
-   idempotent read-only migration into MemPalace drawers. Unchanged drawers are
-   skipped, new or changed memories are upserted, and the versioned state is
-   written only after every discovered record is verified in MemPalace.
-   Failed or partial migrations remain unmarked and retry on a later session.
+2. Marks the backend active immediately, then does everything else
+   **off the startup path** so time-to-first-input is never blocked: a
+   background task pings the bridge (skipped when recently ping-verified) and
+   runs an incremental catch-up.
+3. Catch-up is driven by a record-level ledger
+   (`~/.unipi/memory/.mempalace-ledger.json`) mapping `project/id` to the
+   sha256 of the markdown bytes last confirmed in the palace. Only records whose
+   bytes differ from the ledger are upserted, so an ordinary write never
+   triggers a full re-scan. `store()` updates the ledger only after a confirmed
+   upsert; a pre-existing `.mempalace-migrated` marker seeds the ledger once.
    Legacy files are never deleted or mutated.
 
-Each memory operation invokes the packaged Python bridge
-(`bridge/mempalace_bridge.py`) once via `spawnSync` (~0.5s per call). Both the
-standalone memory package and the all-in-one umbrella tarball ship and resolve
-this bridge. The
-first MemPalace use on a machine also downloads the default ONNX embedding
-model (~80MB, cached at `~/.cache/chroma/onnx_models/`).
+Records contended by a running MemPalace daemon's mine lock are recorded as
+**deferred** (never as failures) and retried with exponential backoff, so the
+catch-up always converges instead of re-running every boot. When a daemon is
+reachable and actively mining, the background catch-up stands down for the
+session rather than fighting the lock.
+
+Memory operations invoke the packaged Python bridge
+(`bridge/mempalace_bridge.py`); startup-path work uses the async, non-blocking
+variant. Both the standalone memory package and the all-in-one umbrella tarball
+ship and resolve this bridge. The first MemPalace use on a machine also
+downloads the default ONNX embedding model (~80MB, cached at
+`~/.cache/chroma/onnx_models/`).
 
 ### Forcing re-detection / re-migration
 
 ```bash
 rm ~/.unipi/memory/.mempalace-install    # re-detect MemPalace next session
-rm ~/.unipi/memory/.mempalace-migrated   # force a full verified migration pass next session
+rm ~/.unipi/memory/.mempalace-ledger.json  # force a full verified catch-up pass next session
 ```
 
 ### Backend override

package/bridge/mempalace_bridge.py

@@ -46,6 +46,15 @@ except Exception:  # pragma: no cover
 MEMORY_TYPES = {"preference", "decision", "pattern", "summary"}
 MIGRATION_AGENT = "unipi-memory-bridge"
 
+# MemPalace's per-palace mine lock is non-blocking: when a daemon or another
+# writer holds it, upserts raise MineAlreadyRunning (message "... is held by
+# PID ..."). That is transient contention, not a data error.
+_TRANSIENT_LOCK_RE = re.compile(r"MineAlreadyRunning|is held by", re.IGNORECASE)
+
+
+def _is_transient_lock_error(exc: Exception) -> bool:
+    return bool(_TRANSIENT_LOCK_RE.search(f"{type(exc).__name__}: {exc}"))
+
 
 # ---------------------------------------------------------------------------
 # ID + URI helpers (mirror mempalace.ids when available, else deterministic fallback)
@@ -311,8 +320,12 @@ def discover_legacy_memories(source_dir: Path, project_filter: list[str] | None
             rec = parse_markdown_memory(project, md_path)
             if rec:
                 by_key[(rec["project"], rec["id"])] = rec
-        for rec in load_sqlite_memories(project, project_dir / "memory.db"):
-            by_key.setdefault((rec["project"], rec["id"]), rec)
+        # Markdown is the sole durable source now (the SQLite fallback was
+        # removed). Discovering legacy memory.db here would surface records the
+        # TS ledger scanner cannot see, so `--only` targeted retries could not
+        # name them and the delta would never converge. `load_sqlite_memories`
+        # is retained for any explicit one-off recovery use, but is no longer
+        # part of the automatic catch-up corpus.
     return sorted(by_key.values(), key=lambda r: (r["project"], r["type"], r["title"], r["id"]))
 
 
@@ -492,18 +505,39 @@ class Bridge:
             synced += 1
         return synced
 
-    def migrate(self, source_dir: str, project_filter: list[str] | None = None) -> dict[str, Any]:
+    def migrate(
+        self,
+        source_dir: str,
+        project_filter: list[str] | None = None,
+        only: list[str] | None = None,
+    ) -> dict[str, Any]:
         """Idempotently import and then verify every discovered UniPi record.
 
         Migration callers must not infer success merely from a bridge process
         exiting cleanly. Return explicit discovery/failure/verification counts
         so UniPi only writes its completion marker after full verification.
+
+        Failures are split into two classes:
+        - ``deferred``: transient palace-lock contention (``MineAlreadyRunning``)
+          while a daemon/other writer holds the mine lock. The record is
+          untouched and simply needs retrying later; it is NOT a data error.
+        - ``failed``: a genuine per-record error (malformed record, backend
+          fault). The completion marker must never advance over these.
+
+        ``only`` optionally restricts the pass to a set of ``"project/id"`` keys
+        (targeted retry of previously deferred records) so a catch-up does not
+        re-sweep the entire corpus.
         """
         records = discover_legacy_memories(Path(source_dir), project_filter)
+        if only:
+            wanted = set(only)
+            records = [r for r in records if f"{r['project']}/{r['id']}" in wanted]
         imported = 0
         skipped = 0
         failed = 0
+        deferred = 0
         errors: list[str] = []
+        deferred_keys: list[str] = []
         by_project: dict[str, int] = {}
         expected = {(rec["project"], rec["id"]) for rec in records}
 
@@ -534,11 +568,19 @@ class Bridge:
                 existing_docs[key] = expected_doc
                 by_project[rec["project"]] = by_project.get(rec["project"], 0) + 1
             except Exception as exc:
-                failed += 1
-                if len(errors) < 20:
-                    errors.append(
-                        f"{rec['project']}/{rec['id']}: {type(exc).__name__}: {exc}"
-                    )
+                if _is_transient_lock_error(exc):
+                    # Palace held by a mine/daemon: the record is untouched and
+                    # just needs a later retry. Record the key, do not count it
+                    # as a data failure (which would block the completion marker
+                    # forever whenever a daemon is running).
+                    deferred += 1
+                    deferred_keys.append(f"{rec['project']}/{rec['id']}")
+                else:
+                    failed += 1
+                    if len(errors) < 20:
+                        errors.append(
+                            f"{rec['project']}/{rec['id']}: {type(exc).__name__}: {exc}"
+                        )
 
         # Re-read after writes and verify exact durable documents, not just
         # optimistic in-memory bookkeeping or collection counts. A palace may
@@ -560,6 +602,7 @@ class Bridge:
             if persisted_rec:
                 persisted[(persisted_rec["project"], persisted_rec["id"])] = doc
         verified = 0
+        verified_keys: list[str] = []
         for rec in records:
             expected_doc = build_document(
                 rec["title"], rec["content"], rec["tags"], rec["project"],
@@ -567,6 +610,7 @@ class Bridge:
             )
             if persisted.get((rec["project"], rec["id"])) == expected_doc:
                 verified += 1
+                verified_keys.append(f"{rec['project']}/{rec['id']}")
 
         return {
             "discovered": len(records),
@@ -574,9 +618,12 @@ class Bridge:
             "updated": imported,
             "skipped": skipped,
             "failed": failed,
+            "deferred": deferred,
             "verified": verified,
             "projects": by_project,
             "errors": errors,
+            "deferred_keys": deferred_keys[:200],
+            "verified_keys": verified_keys,
         }
 
 
@@ -617,7 +664,7 @@ def main(argv: list[str]) -> int:
         "has_title": lambda: bridge.has_title(args["wing"], args["title"]),
         "find_similar": lambda: bridge.find_similar(args["wing"], args["title"], float(args.get("threshold", 0.6))),
         "sync_orphaned": lambda: bridge.sync_orphaned(args["project_dir"], args["wing"]),
-        "migrate": lambda: bridge.migrate(args["source_dir"], args.get("projects")),
+        "migrate": lambda: bridge.migrate(args["source_dir"], args.get("projects"), args.get("only")),
     }
     handler = handlers.get(cmd)
     if handler is None:

package/mempalace.ts

@@ -25,6 +25,8 @@ export const DEFAULT_PALACE = path.join(os.homedir(), ".mempalace", "palace");
 
 const INSTALL_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-install");
 const MIGRATED_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-migrated");
+/** Record-level sync ledger (L2): supersedes the size+mtime fingerprint gate. */
+const LEDGER_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-ledger.json");
 /** Flag written after a successful ping, so subsequent sessions can skip
  *  the ~0.5s Python cold-start sanity check. Stale after PING_VERIFIED_TTL_MS. */
 const PING_VERIFIED_FLAG = path.join(os.homedir(), ".unipi", "memory", ".mempalace-ping-verified");
@@ -39,8 +41,14 @@ export interface MigrationResult {
   updated: number;
   skipped: number;
   failed: number;
+  /** Records left untouched by transient palace-lock contention (retryable). */
+  deferred?: number;
   verified: number;
   errors?: string[];
+  /** "project/id" keys deferred by lock contention, for a targeted retry. */
+  deferredKeys?: string[];
+  /** "project/id" keys confirmed durable in the palace after the run. */
+  verifiedKeys?: string[];
 }
 
 export interface MigrationState {
@@ -48,6 +56,60 @@ export interface MigrationState {
   completedAt: string;
   sourceFingerprint: string;
   result: MigrationResult;
+  /** Keys still awaiting a retry after transient lock contention. */
+  deferredKeys?: string[];
+  /** Consecutive deferred-retry rounds, for exponential backoff. */
+  attempts?: number;
+  /** Epoch ms; do not retry deferred keys before this. */
+  retryAfter?: number;
+}
+
+/** Deferred-retry backoff schedule: 15m → 1h → 6h → 24h (capped). */
+const DEFERRAL_BACKOFF_MS = [15 * 60_000, 60 * 60_000, 6 * 60 * 60_000, 24 * 60 * 60_000];
+
+function deferralBackoffMs(attempts: number): number {
+  const idx = Math.min(Math.max(attempts, 1), DEFERRAL_BACKOFF_MS.length) - 1;
+  return DEFERRAL_BACKOFF_MS[idx] ?? DEFERRAL_BACKOFF_MS[DEFERRAL_BACKOFF_MS.length - 1]!;
+}
+
+/** A migrate outcome is complete when nothing genuinely failed and every
+ *  discovered record is either verified or transiently deferred. */
+export function isMigrationComplete(result: MigrationResult): boolean {
+  return result.failed === 0 && result.verified + (result.deferred ?? 0) === result.discovered;
+}
+
+/**
+ * Normalize the raw bridge migrate payload (snake_case `deferred_keys`) into a
+ * `MigrationResult`. Returns null for a missing/malformed payload so callers
+ * never advance the marker on garbage.
+ */
+export function normalizeMigrationResult(raw: unknown): MigrationResult | null {
+  if (!raw || typeof raw !== "object") return null;
+  const r = raw as Record<string, unknown>;
+  const num = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) ? v : 0);
+  if (typeof r.discovered !== "number") return null;
+  const keys = Array.isArray(r.deferredKeys)
+    ? r.deferredKeys
+    : Array.isArray(r.deferred_keys)
+      ? r.deferred_keys
+      : [];
+  const vkeysRaw = Array.isArray(r.verifiedKeys)
+    ? r.verifiedKeys
+    : Array.isArray(r.verified_keys)
+      ? r.verified_keys
+      : [];
+  return {
+    discovered: num(r.discovered),
+    imported: num(r.imported),
+    updated: num(r.updated),
+    skipped: num(r.skipped),
+    failed: num(r.failed),
+    deferred: num(r.deferred),
+    verified: num(r.verified),
+    errors: Array.isArray(r.errors) ? (r.errors as string[]) : undefined,
+    deferredKeys: keys.filter((k): k is string => typeof k === "string"),
+    verifiedKeys: vkeysRaw.filter((k): k is string => typeof k === "string"),
+  };
 }
 
 let cachedBridgePath: string | null | undefined;
@@ -101,6 +163,27 @@ export interface BridgeResponse<T> {
   error?: string;
 }
 
+/**
+ * Full outcome of a bridge call. `result` is the parsed value (which may be a
+ * legitimate `null`, e.g. a "not found" lookup). `ok` distinguishes a
+ * successful call from a failure; `transient` marks failures that are just
+ * MemPalace palace-lock contention (retryable), not real backend errors.
+ */
+export interface BridgeOutcome<T> {
+  ok: boolean;
+  result: T | null;
+  error?: string;
+  transient: boolean;
+}
+
+/** MemPalace's non-blocking mine lock surfaces as MineAlreadyRunning / "is held by". */
+const TRANSIENT_BRIDGE_ERROR = /MineAlreadyRunning|is held by/i;
+
+/** True when a bridge error is transient palace-lock contention, not a real fault. */
+export function isTransientBridgeError(error: string | undefined | null): boolean {
+  return typeof error === "string" && TRANSIENT_BRIDGE_ERROR.test(error);
+}
+
 export interface MempalaceRecord {
   id: string;
   title: string;
@@ -291,7 +374,327 @@ export function getMemorySourceFingerprint(
   return hash.digest("hex");
 }
 
-/** Read a verified v2 migration state. Legacy timestamp markers return null. */
+// ── Daemon awareness (L3) ───────────────────────────────────────────────────
+//
+// MemPalace can run as a long-lived daemon that holds the per-palace mine lock
+// while it mines. That lock is exactly what makes our direct-bridge upserts
+// defer (L1). The daemon exposes an HTTP control API, but it has NO idempotent
+// record-upsert job — its only generic write (`mcp_tool` -> tool_add_drawer)
+// uses a CONTENT-addressed drawer id, whereas our bridge uses a deterministic
+// SOURCE-URI-addressed id. Routing our writes through the daemon would fork the
+// id scheme and duplicate drawers, so we must NOT do that. Instead we detect a
+// reachable daemon and, when it is actively mining, skip the direct catch-up
+// this session and let the L1/L2 backoff ride it out — avoiding the lock fight
+// rather than joining it. The direct bridge stays the sole write path because
+// it alone produces the correct idempotent ids. When no daemon is running
+// (per-call / MCP-less mode) nothing changes.
+
+export interface DaemonStatus {
+  /** A daemon endpoint for this palace is reachable and healthy. */
+  reachable: boolean;
+  /** The daemon is currently running a job (holds the mine lock). */
+  busy: boolean;
+}
+
+/** Replicate the daemon's palace_key: sha256(realpath(palace))[:24] (normcase
+ *  is a no-op on POSIX). */
+function palaceKey(palacePath: string): string {
+  let canonical: string;
+  try {
+    canonical = fs.realpathSync(palacePath);
+  } catch {
+    canonical = path.resolve(palacePath);
+  }
+  return createHash("sha256").update(canonical).digest("hex").slice(0, 24);
+}
+
+function daemonStateDir(palacePath: string): string {
+  const root = process.env.MEMPALACE_DAEMON_STATE_ROOT
+    ? path.resolve(os.homedir(), process.env.MEMPALACE_DAEMON_STATE_ROOT.replace(/^~(?=$|\/)/, os.homedir()))
+    : path.join(os.homedir(), ".mempalace", "daemon");
+  return path.join(root, palaceKey(palacePath));
+}
+
+/**
+ * Probe for a reachable MemPalace daemon for `palacePath` via its endpoint.json
+ * + token + /health. Never throws; returns `{reachable:false}` when there is no
+ * daemon, the endpoint is stale, or the probe errors/times out. `busy` reflects
+ * an in-flight job (active_job_id) — i.e. the mine lock is likely held.
+ */
+export async function probeDaemon(palacePath: string, timeoutMs = 300): Promise<DaemonStatus> {
+  const down: DaemonStatus = { reachable: false, busy: false };
+  try {
+    const dir = daemonStateDir(palacePath);
+    const endpointRaw = fs.readFileSync(path.join(dir, "endpoint.json"), "utf-8");
+    const token = fs.readFileSync(path.join(dir, "token"), "utf-8").trim();
+    const endpoint = JSON.parse(endpointRaw) as { host?: string; port?: number };
+    if (!endpoint.host || !endpoint.port || !token) return down;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    timer.unref?.();
+    try {
+      const resp = await fetch(`http://${endpoint.host}:${String(endpoint.port)}/health`, {
+        headers: { Authorization: `Bearer ${token}` },
+        signal: controller.signal,
+      });
+      if (!resp.ok) return down;
+      const health = (await resp.json()) as { ok?: boolean; active_job_id?: unknown };
+      if (!health.ok) return down;
+      return { reachable: true, busy: health.active_job_id != null };
+    } finally {
+      clearTimeout(timer);
+    }
+  } catch {
+    return down;
+  }
+}
+
+// ── Record-level sync ledger (L2) ────────────────────────────────────────────
+//
+// The old size+mtime fingerprint invalidated the whole migration marker on any
+// memory write (store() rewrites the .md every time), forcing a full re-sweep.
+// The ledger instead tracks, per record, the content hash last confirmed durable
+// in the palace. Catch-up then touches only records whose file differs from the
+// ledger (out-of-band edits or writes whose palace upsert was deferred), never
+// the whole corpus.
+
+export const LEDGER_VERSION = 1;
+
+export interface LedgerEntry {
+  /** sha256 of the exact .md bytes last confirmed in the palace. */
+  hash: string;
+}
+
+export interface Ledger {
+  version: number;
+  /** "project/id" -> entry. */
+  entries: Record<string, LedgerEntry>;
+  /** Keys deferred by transient lock contention, awaiting a targeted retry. */
+  deferredKeys: string[];
+  /** Keys that hit a genuine (non-transient) failure; retried with backoff too. */
+  failedKeys: string[];
+  /** Consecutive contended/failed rounds, for exponential backoff. */
+  attempts: number;
+  /** Epoch ms; do not retry deferred/failed keys before this. */
+  retryAfter?: number;
+  updatedAt: string;
+}
+
+/** A record discovered on disk: its key and the hash of its current bytes. */
+export interface ScannedRecord {
+  key: string;
+  project: string;
+  id: string;
+  hash: string;
+}
+
+function emptyLedger(): Ledger {
+  return { version: LEDGER_VERSION, entries: {}, deferredKeys: [], failedKeys: [], attempts: 0, updatedAt: new Date(0).toISOString() };
+}
+
+/** sha256 of raw file bytes — the scanner and store() hash the same bytes so
+ *  the ledger never drifts through a parse/serialize round-trip. */
+export function hashBytes(text: string): string {
+  return createHash("sha256").update(text, "utf-8").digest("hex");
+}
+
+export function readLedger(flagPath = LEDGER_FLAG): Ledger {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(flagPath, "utf-8")) as Partial<Ledger>;
+    if (parsed?.version !== LEDGER_VERSION || !parsed.entries || typeof parsed.entries !== "object") {
+      return emptyLedger();
+    }
+    return {
+      version: LEDGER_VERSION,
+      entries: parsed.entries as Record<string, LedgerEntry>,
+      deferredKeys: Array.isArray(parsed.deferredKeys) ? parsed.deferredKeys : [],
+      failedKeys: Array.isArray(parsed.failedKeys) ? parsed.failedKeys : [],
+      attempts: typeof parsed.attempts === "number" ? parsed.attempts : 0,
+      retryAfter: typeof parsed.retryAfter === "number" ? parsed.retryAfter : undefined,
+      updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : new Date(0).toISOString(),
+    };
+  } catch {
+    return emptyLedger();
+  }
+}
+
+export function writeLedger(ledger: Ledger, flagPath = LEDGER_FLAG): boolean {
+  try {
+    fs.mkdirSync(path.dirname(flagPath), { recursive: true });
+    const temp = `${flagPath}.${process.pid}.tmp`;
+    fs.writeFileSync(temp, JSON.stringify({ ...ledger, version: LEDGER_VERSION }, null, 2), "utf-8");
+    fs.renameSync(temp, flagPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Scan the durable markdown tier and return one record per `.md` file with the
+ * hash of its exact bytes. Only markdown is a migration source now (the SQLite
+ * fallback was removed), so `memory.db` is deliberately ignored — its churn was
+ * a major cause of needless re-sweeps.
+ */
+export function scanMemorySources(sourceDir = path.join(os.homedir(), ".unipi", "memory")): ScannedRecord[] {
+  const out: ScannedRecord[] = [];
+  if (!fs.existsSync(sourceDir)) return out;
+  let projects: fs.Dirent[];
+  try {
+    projects = fs.readdirSync(sourceDir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+  for (const projEntry of projects) {
+    if (!projEntry.isDirectory() || projEntry.name.startsWith(".")) continue;
+    const project = projEntry.name;
+    const projDir = path.join(sourceDir, project);
+    let files: fs.Dirent[];
+    try {
+      files = fs.readdirSync(projDir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const f of files) {
+      if (!f.isFile() || f.name.startsWith(".") || !f.name.endsWith(".md")) continue;
+      const full = path.join(projDir, f.name);
+      let text: string;
+      try {
+        text = fs.readFileSync(full, "utf-8");
+      } catch {
+        continue;
+      }
+      // Key must match the bridge exactly (parse_markdown_memory):
+      //  - id: explicit frontmatter `id`, else the filename stem normalized
+      //    ([^A-Za-z0-9]+ -> _, trimmed, lowercased).
+      //  - project: frontmatter `project` if present, else the directory name.
+      // A mismatch would make targeted `--only` retries silently no-op.
+      const fmMatch = /^---\n([\s\S]*?)\n---/.exec(text);
+      const fm = fmMatch ? fmMatch[1]! : "";
+      const readField = (name: string): string => {
+        const m = new RegExp(`(^|\\n)${name}:\\s*(.+)`).exec(fm);
+        return m ? m[2]!.trim().replace(/^["']|["']$/g, "") : "";
+      };
+      const explicitId = readField("id");
+      const id = explicitId
+        || (f.name.replace(/\.md$/, "").replace(/[^A-Za-z0-9]+/g, "_").replace(/^_+|_+$/g, "").toLowerCase() || "unknown");
+      const recProject = readField("project") || project;
+      out.push({ key: `${recProject}/${id}`, project: recProject, id, hash: hashBytes(text) });
+    }
+  }
+  return out;
+}
+
+/**
+ * Compute which record keys still need a migrate pass: those whose current file
+ * hash differs from the ledger, plus any deferred/failed keys whose backoff has
+ * elapsed. Returns the keys and whether the ledger has never been populated
+ * (→ a first full pass is warranted).
+ */
+export function ledgerDelta(
+  scanned: ScannedRecord[],
+  ledger: Ledger,
+  now = Date.now(),
+): { keys: string[]; firstRun: boolean } {
+  const firstRun = Object.keys(ledger.entries).length === 0;
+  const due = new Set<string>();
+  for (const rec of scanned) {
+    if (ledger.entries[rec.key]?.hash !== rec.hash) due.add(rec.key);
+  }
+  const backoffElapsed = typeof ledger.retryAfter !== "number" || now >= ledger.retryAfter;
+  if (backoffElapsed) {
+    for (const k of ledger.deferredKeys) due.add(k);
+    for (const k of ledger.failedKeys) due.add(k);
+  }
+  return { keys: [...due], firstRun };
+}
+
+/**
+ * Fold a migrate result into the ledger: advance the content hash for every
+ * verified key, and re-track deferred/failed keys (with backoff) so they retry
+ * later. `scannedByKey` gives the on-disk hash to record for a verified key.
+ */
+export function applyMigrationToLedger(
+  ledger: Ledger,
+  result: MigrationResult,
+  scannedByKey: Map<string, string>,
+  now = Date.now(),
+): Ledger {
+  const entries = { ...ledger.entries };
+  for (const key of result.verifiedKeys ?? []) {
+    const hash = scannedByKey.get(key);
+    if (hash) entries[key] = { hash };
+  }
+  const deferredKeys = [...new Set(result.deferredKeys ?? [])];
+  // Genuine failures are surfaced via error strings; derive their keys from the
+  // error prefix "project/id: ..." so they, too, are retried (not silently
+  // dropped) but never recorded as synced.
+  const failedKeys = [...new Set((result.errors ?? [])
+    .map((e) => e.split(":")[0]?.trim())
+    .filter((k): k is string => !!k && k.includes("/")))];
+  const hadBacklog = deferredKeys.length > 0 || failedKeys.length > 0;
+  const attempts = hadBacklog ? ledger.attempts + 1 : 0;
+  return {
+    version: LEDGER_VERSION,
+    entries,
+    deferredKeys,
+    failedKeys,
+    attempts,
+    ...(hadBacklog ? { retryAfter: now + deferralBackoffMs(attempts) } : {}),
+    updatedAt: new Date(now).toISOString(),
+  };
+}
+
+/** Record a single successful store() upsert in the ledger. */
+export function ledgerRecordStore(key: string, fileText: string, flagPath = LEDGER_FLAG): void {
+  const ledger = readLedger(flagPath);
+  ledger.entries[key] = { hash: hashBytes(fileText) };
+  // A fresh successful write clears any pending retry state for this key.
+  ledger.deferredKeys = ledger.deferredKeys.filter((k) => k !== key);
+  ledger.failedKeys = ledger.failedKeys.filter((k) => k !== key);
+  ledger.updatedAt = new Date().toISOString();
+  writeLedger(ledger, flagPath);
+}
+
+/**
+ * One-time bootstrap: if there is no ledger yet but a completed legacy
+ * `.mempalace-migrated` marker exists for the current corpus, seed the ledger
+ * from the current on-disk hashes so we do not re-migrate everything once.
+ * Deferred keys from the old marker are carried over for targeted retry.
+ */
+export function bootstrapLedgerFromLegacyMarker(
+  sourceDir = path.join(os.homedir(), ".unipi", "memory"),
+  ledgerPath = LEDGER_FLAG,
+  markerPath = MIGRATED_FLAG,
+): Ledger | null {
+  if (fs.existsSync(ledgerPath)) return null;
+  const marker = readMigrationState(markerPath);
+  if (!marker) return null;
+  const scanned = scanMemorySources(sourceDir);
+  const deferred = new Set(marker.deferredKeys ?? []);
+  const entries: Record<string, LedgerEntry> = {};
+  for (const rec of scanned) {
+    // A record known-deferred under the old marker is NOT yet durable — leave
+    // it out of entries so the delta re-attempts it.
+    if (deferred.has(rec.key)) continue;
+    entries[rec.key] = { hash: rec.hash };
+  }
+  const ledger: Ledger = {
+    version: LEDGER_VERSION,
+    entries,
+    deferredKeys: [...deferred],
+    failedKeys: [],
+    attempts: marker.attempts ?? 0,
+    ...(marker.retryAfter ? { retryAfter: marker.retryAfter } : {}),
+    updatedAt: new Date().toISOString(),
+  };
+  writeLedger(ledger, ledgerPath);
+  return ledger;
+}
+
+/** Read a verified migration state. Legacy timestamp markers, wrong-version,
+ *  malformed, or non-complete states all return null (→ treated as not
+ *  migrated). A state with transiently-deferred records is still complete. */
 export function readMigrationState(flagPath = MIGRATED_FLAG): MigrationState | null {
   try {
     const parsed = JSON.parse(fs.readFileSync(flagPath, "utf-8")) as MigrationState;
@@ -300,8 +703,7 @@ export function readMigrationState(flagPath = MIGRATED_FLAG): MigrationState | n
       typeof parsed.completedAt !== "string" ||
       typeof parsed.sourceFingerprint !== "string" ||
       !parsed.result ||
-      parsed.result.failed !== 0 ||
-      parsed.result.verified !== parsed.result.discovered
+      !isMigrationComplete(parsed.result)
     ) return null;
     return parsed;
   } catch {
@@ -309,6 +711,46 @@ export function readMigrationState(flagPath = MIGRATED_FLAG): MigrationState | n
   }
 }
 
+/**
+ * The deferred "project/id" keys due for a targeted retry now, or null when
+ * there is nothing to retry (no complete marker for this fingerprint, no
+ * deferred keys, or the backoff window has not elapsed).
+ */
+export function deferredRetryDue(
+  sourceFingerprint = getMemorySourceFingerprint(),
+  flagPath = MIGRATED_FLAG,
+  now = Date.now(),
+): string[] | null {
+  const state = readMigrationState(flagPath);
+  if (!state || state.sourceFingerprint !== sourceFingerprint) return null;
+  const keys = state.deferredKeys ?? [];
+  if (keys.length === 0) return null;
+  if (typeof state.retryAfter === "number" && now < state.retryAfter) return null;
+  return keys;
+}
+
+/**
+ * Push the deferred-retry schedule forward without changing completion — used
+ * when a targeted retry could not be recorded as complete (e.g. it surfaced a
+ * genuine failure) so we do not re-attempt it on every boot.
+ */
+export function bumpDeferredRetry(
+  sourceFingerprint: string,
+  flagPath = MIGRATED_FLAG,
+  now = Date.now(),
+): void {
+  const state = readMigrationState(flagPath);
+  if (!state || state.sourceFingerprint !== sourceFingerprint) return;
+  const attempts = (state.attempts ?? 0) + 1;
+  const next: MigrationState = { ...state, attempts, retryAfter: now + deferralBackoffMs(attempts) };
+  try {
+    fs.mkdirSync(path.dirname(flagPath), { recursive: true });
+    const temp = `${flagPath}.${process.pid}.tmp`;
+    fs.writeFileSync(temp, JSON.stringify(next, null, 2), "utf-8");
+    fs.renameSync(temp, flagPath);
+  } catch { /* best effort */ }
+}
+
 /** Is the palace verified against the current durable source set? */
 export function isMigrated(
   sourceFingerprint = getMemorySourceFingerprint(),
@@ -317,20 +759,35 @@ export function isMigrated(
   return readMigrationState(flagPath)?.sourceFingerprint === sourceFingerprint;
 }
 
-/** Mark migration complete only after the caller has verified every record. */
+/**
+ * Mark migration complete. Accepts a run where every discovered record is
+ * verified or transiently deferred (lock contention) and nothing genuinely
+ * failed. When records are deferred, persist their keys plus an exponential
+ * backoff `retryAfter` so a later session retries only those keys instead of
+ * re-sweeping the corpus. A genuine failure (`failed > 0`) or an incomplete
+ * run is refused, so the marker never advances over lost data.
+ */
 export function markMigrated(
   sourceFingerprint: string,
   result: MigrationResult,
   flagPath = MIGRATED_FLAG,
+  now = Date.now(),
 ): boolean {
-  if (result.failed !== 0 || result.verified !== result.discovered) return false;
+  if (!isMigrationComplete(result)) return false;
+  const deferred = result.deferred ?? 0;
+  const prev = readMigrationState(flagPath);
+  const prevAttempts = prev?.sourceFingerprint === sourceFingerprint ? prev.attempts ?? 0 : 0;
+  const attempts = deferred > 0 ? prevAttempts + 1 : 0;
   try {
     fs.mkdirSync(path.dirname(flagPath), { recursive: true });
     const state: MigrationState = {
       version: MIGRATION_STATE_VERSION,
-      completedAt: new Date().toISOString(),
+      completedAt: new Date(now).toISOString(),
       sourceFingerprint,
       result,
+      deferredKeys: deferred > 0 ? (result.deferredKeys ?? []) : [],
+      attempts,
+      ...(deferred > 0 ? { retryAfter: now + deferralBackoffMs(attempts) } : {}),
     };
     const temp = `${flagPath}.${process.pid}.tmp`;
     fs.writeFileSync(temp, JSON.stringify(state, null, 2), "utf-8");
@@ -342,23 +799,30 @@ export function markMigrated(
 }
 
 /**
- * Run one bridge command synchronously. Returns the parsed result, or null
- * on any failure (timeout, non-zero exit, bad JSON, ok=false).
+ * Run one bridge command synchronously, returning the full outcome so callers
+ * can tell a successful `null` result (e.g. "not found") apart from a failure,
+ * and transient palace-lock contention apart from a real backend error.
  */
-export function runBridge<T = unknown>(
+export function runBridgeOutcome<T = unknown>(
   install: MempalaceInstall,
   palace: string,
   cmd: string,
   args: Record<string, unknown> = {},
   timeoutMs = 60_000,
-): T | null {
+): BridgeOutcome<T> {
+  const fail = (error?: string): BridgeOutcome<T> => ({
+    ok: false,
+    result: null,
+    error,
+    transient: isTransientBridgeError(error),
+  });
   const bridgePath = getBridgePath();
-  if (!bridgePath) return null;
+  if (!bridgePath) return fail("bridge script not found");
   let argsJson: string;
   try {
     argsJson = JSON.stringify(args);
   } catch {
-    return null;
+    return fail("args not serializable");
   }
   let res;
   try {
@@ -367,21 +831,38 @@ export function runBridge<T = unknown>(
       timeout: timeoutMs,
       maxBuffer: 64 * 1024 * 1024,
     });
-  } catch {
-    return null;
-  }
-  if (res.error || res.status !== 0) {
-    return null;
+  } catch (err) {
+    return fail(err instanceof Error ? err.message : String(err));
   }
+  if (res.error) return fail(res.error.message);
   const out = (res.stdout || "").trim();
-  if (!out) return null;
-  try {
-    const parsed = JSON.parse(out) as BridgeResponse<T>;
-    if (!parsed.ok) return null;
-    return (parsed.result ?? null) as T | null;
-  } catch {
-    return null;
+  // A non-zero exit may still carry a structured {ok:false,error} on stdout
+  // (the bridge prints that then exits 1) — parse it so we can classify.
+  if (out) {
+    try {
+      const parsed = JSON.parse(out) as BridgeResponse<T>;
+      if (parsed.ok) return { ok: true, result: (parsed.result ?? null) as T | null, transient: false };
+      return fail(parsed.error);
+    } catch {
+      return fail("bad json from bridge");
+    }
   }
+  return fail(res.status === 0 ? "empty output" : `bridge exited ${String(res.status)}`);
+}
+
+/**
+ * Run one bridge command synchronously. Returns the parsed result, or null
+ * on any failure. Prefer {@link runBridgeOutcome} when you need to distinguish
+ * a "not found" null from a failure, or transient contention from a real error.
+ */
+export function runBridge<T = unknown>(
+  install: MempalaceInstall,
+  palace: string,
+  cmd: string,
+  args: Record<string, unknown> = {},
+  timeoutMs = 60_000,
+): T | null {
+  return runBridgeOutcome<T>(install, palace, cmd, args, timeoutMs).result;
 }
 
 /**
@@ -398,17 +879,30 @@ export function runBridgeAsync<T = unknown>(
   args: Record<string, unknown> = {},
   timeoutMs = 60_000,
 ): Promise<T | null> {
+  return runBridgeAsyncOutcome<T>(install, palace, cmd, args, timeoutMs).then((o) => o.result);
+}
+
+/** Async twin of {@link runBridgeOutcome}. Never rejects. */
+export function runBridgeAsyncOutcome<T = unknown>(
+  install: MempalaceInstall,
+  palace: string,
+  cmd: string,
+  args: Record<string, unknown> = {},
+  timeoutMs = 60_000,
+): Promise<BridgeOutcome<T>> {
   return new Promise((resolve) => {
+    const fail = (error?: string): void =>
+      resolve({ ok: false, result: null, error, transient: isTransientBridgeError(error) });
     const bridgePath = getBridgePath();
     if (!bridgePath) {
-      resolve(null);
+      fail("bridge script not found");
       return;
     }
     let argsJson: string;
     try {
       argsJson = JSON.stringify(args);
     } catch {
-      resolve(null);
+      fail("args not serializable");
       return;
     }
 
@@ -417,39 +911,49 @@ export function runBridgeAsync<T = unknown>(
       child = spawn(install.python, [bridgePath, palace, cmd, argsJson], {
         stdio: ["ignore", "pipe", "ignore"],
       });
-    } catch {
-      resolve(null);
+    } catch (err) {
+      fail(err instanceof Error ? err.message : String(err));
       return;
     }
+    // Fire-and-forget callers (e.g. the L0 background migrate) must never keep
+    // the process alive; the promise still resolves on close for awaiters.
+    child.unref?.();
 
     let out = "";
     let settled = false;
-    const finish = (value: T | null) => {
+    const finish = (outcome: BridgeOutcome<T>): void => {
       if (settled) return;
       settled = true;
       clearTimeout(timer);
-      resolve(value);
+      resolve(outcome);
     };
 
     const timer = setTimeout(() => {
       try { child.kill(); } catch { /* already gone */ }
-      finish(null);
+      finish({ ok: false, result: null, error: "bridge timed out", transient: false });
     }, timeoutMs);
     // Do not hold the process open purely for a background bridge call.
     timer.unref?.();
 
     child.stdout?.setEncoding("utf-8");
     child.stdout?.on("data", (chunk) => { out += chunk; });
-    child.on("error", () => finish(null));
-    child.on("close", (code) => {
-      if (code !== 0) return finish(null);
+    child.on("error", (err) => finish({ ok: false, result: null, error: err.message, transient: false }));
+    child.on("close", () => {
       const trimmed = out.trim();
-      if (!trimmed) return finish(null);
+      // A non-zero exit still carries {ok:false,error} on stdout; parse first.
+      if (!trimmed) {
+        finish({ ok: false, result: null, error: "empty output", transient: false });
+        return;
+      }
       try {
         const parsed = JSON.parse(trimmed) as BridgeResponse<T>;
-        finish(parsed.ok ? ((parsed.result ?? null) as T | null) : null);
+        if (parsed.ok) {
+          finish({ ok: true, result: (parsed.result ?? null) as T | null, transient: false });
+        } else {
+          finish({ ok: false, result: null, error: parsed.error, transient: isTransientBridgeError(parsed.error) });
+        }
       } catch {
-        finish(null);
+        finish({ ok: false, result: null, error: "bad json from bridge", transient: false });
       }
     });
   });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/memory",
-  "version": "2.20.3",
+  "version": "2.20.5",
   "description": "Persistent cross-session memory with MemPalace backend (auto-installed) and SQLite fallback for Pi coding agent",
   "type": "module",
   "main": "index.ts",
@@ -39,8 +39,8 @@
     "README.md"
   ],
   "dependencies": {
-    "@pi-unipi/core": "2.20.3",
-    "@pi-unipi/info-screen": "2.20.3",
+    "@pi-unipi/core": "2.20.5",
+    "@pi-unipi/info-screen": "2.20.5",
     "js-yaml": "^4.1.0"
   },
   "peerDependencies": {

package/storage.ts

@@ -15,9 +15,17 @@ import {
   ensureMempalace,
   runBridge,
   runBridgeAsync,
-  isMigrated,
-  markMigrated,
-  getMemorySourceFingerprint,
+  runBridgeOutcome,
+  runBridgeAsyncOutcome,
+  normalizeMigrationResult,
+  probeDaemon,
+  readLedger,
+  writeLedger,
+  scanMemorySources,
+  ledgerDelta,
+  applyMigrationToLedger,
+  ledgerRecordStore,
+  bootstrapLedgerFromLegacyMarker,
   isPingVerified,
   markPingVerified,
   invalidatePingVerified,
@@ -27,7 +35,6 @@ import {
   type MempalaceSearchResult,
   type MempalaceListItem,
   type MempalaceListItemAll,
-  type MigrationResult,
 } from "./mempalace.js";
 
 
@@ -198,6 +205,8 @@ export class MemoryStorage {
   private scopeDir: string;
   private mempalaceInstall: MempalaceInstall | null = null;
   private palacePath: string = DEFAULT_PALACE;
+  /** Single-flight guard for the background ping+migrate task. */
+  private bgVerifyStarted = false;
 
   constructor(projectName: string) {
     this.projectName = projectName;
@@ -210,30 +219,34 @@ export class MemoryStorage {
   }
 
   /**
-   * Run a MemPalace bridge command, invalidating the ping-verified flag
-   * when the call fails. This ensures a palace that breaks after the
-   * startup ping-skip gets re-verified on the next session.
+   * Run a MemPalace bridge command. Invalidates the ping-verified flag only on
+   * a genuine backend failure (bad process, corrupt palace, protocol error) so
+   * a broken palace gets re-verified next session.
+   *
+   * L4: a successful call that legitimately returns `null` (e.g. a "not found"
+   * lookup) and transient palace-lock contention (`MineAlreadyRunning`, common
+   * while the daemon mines) must NOT invalidate the flag — doing so forced a
+   * ~0.5s cold-start ping on every boot.
    */
   private memPalaceCall<T>(cmd: string, args: Record<string, unknown> = {}): T | null {
     const install = this.mempalaceInstall;
     if (!install) return null;
-    const result = runBridge<T>(install, this.palacePath, cmd, args);
-    if (result === null) {
-      // Backend didn't respond — force a real ping next session.
+    const outcome = runBridgeOutcome<T>(install, this.palacePath, cmd, args);
+    if (!outcome.ok && !outcome.transient) {
       invalidatePingVerified();
     }
-    return result;
+    return outcome.result;
   }
 
   /** Async twin of memPalaceCall, for paths that must not block the UI. */
   private async memPalaceCallAsync<T>(cmd: string, args: Record<string, unknown> = {}): Promise<T | null> {
     const install = this.mempalaceInstall;
     if (!install) return null;
-    const result = await runBridgeAsync<T>(install, this.palacePath, cmd, args);
-    if (result === null) {
+    const outcome = await runBridgeAsyncOutcome<T>(install, this.palacePath, cmd, args);
+    if (!outcome.ok && !outcome.transient) {
       invalidatePingVerified();
     }
-    return result;
+    return outcome.result;
   }
 
   /**
@@ -249,71 +262,94 @@ export class MemoryStorage {
   }
 
   /**
-   * Initialize storage. Tries MemPalace first (auto-install + one-way
-   * auto-migration of legacy memories). Throws if MemPalace is unavailable.
+   * Initialize storage. Requires the MemPalace backend to be installable
+   * (auto-install via uv); throws only when it is genuinely unavailable.
+   *
+   * L0: this is deliberately NON-BLOCKING. The synchronous path does only the
+   * cheap install check (a cached flag on normal boots) and then hands the
+   * ~0.5s ping and the potentially multi-second migrate/catch-up to a
+   * fire-and-forget background task. `mempalaceInstall` is set optimistically
+   * so tools work immediately; the migrate no longer sits on `session_start`
+   * and never blocks time-to-first-input.
    */
   init(): void {
     if (!fs.existsSync(this.scopeDir)) {
       fs.mkdirSync(this.scopeDir, { recursive: true });
     }
 
-    if (!this.tryInitMempalace()) {
+    let install: MempalaceInstall | null;
+    try {
+      install = ensureMempalace();
+    } catch {
+      install = null;
+    }
+    if (!install) {
       throw new Error("MemPalace backend unavailable. Ensure uv is installed.");
     }
+
+    // Optimistic: mark the backend active now so memory tools are usable from
+    // the first turn. A palace that turns out to be broken simply returns null
+    // from bridge calls (and re-verifies next session) rather than blocking.
+    this.mempalaceInstall = install;
+
+    // Ping + migration/catch-up run off the startup path.
+    void this.backgroundVerifyAndMigrate(install);
   }
 
   /**
-   * Attempt to initialize the MemPalace backend. Returns true on success.
-   * Handles auto-install and one-way auto-migration of legacy memories.
-   * Never throws — any failure returns false so init() can throw a clear error.
+   * Verify the palace is reachable (ping) and run the idempotent migration
+   * catch-up — entirely off the synchronous startup path. Single-flight per
+   * instance. Never throws into the caller.
    */
-  private tryInitMempalace(): boolean {
-    let install: MempalaceInstall | null;
-    try {
-      install = ensureMempalace();
-    } catch {
-      return false;
-    }
-    if (!install) return false;
+  private async backgroundVerifyAndMigrate(install: MempalaceInstall): Promise<void> {
+    if (this.bgVerifyStarted) return;
+    this.bgVerifyStarted = true;
 
-    // Sanity ping — if the palace/bridge is broken, fall back.
-    // Skip the ~0.5s Python cold-start when we ping-verified recently;
-    // the flag is invalidated on any backend failure so a broken palace
-    // is re-checked on the next session.
+    // Sanity ping — skip the ~0.5s Python cold-start when we ping-verified
+    // recently. A failed ping leaves the flag unset so it re-checks next
+    // session; it does not disable the optimistically-active backend.
     if (!isPingVerified()) {
-      const ok = runBridge<string>(install, this.palacePath, "ping");
-      if (ok !== "pong") return false;
+      const ok = await runBridgeAsync<string>(install, this.palacePath, "ping");
+      if (ok !== "pong") return;
       markPingVerified();
     }
 
-    this.mempalaceInstall = install;
-
-    // Idempotent migration + automatic catch-up. The source fingerprint turns
-    // the old one-shot timestamp into a resumable state: new/changed markdown
-    // or SQLite sources trigger another verified upsert pass. Never mark a
-    // failed/partial run complete; it will retry on a later session.
-    const sourceFingerprint = getMemorySourceFingerprint(getMemoryBaseDir());
-    if (!isMigrated(sourceFingerprint)) {
-      try {
-        // First migrations can embed thousands of records. Give the bridge a
-        // practical bounded window rather than the normal per-operation 60s.
-        const result = runBridge<MigrationResult>(install, this.palacePath, "migrate", {
-          source_dir: getMemoryBaseDir(),
-        }, 15 * 60_000);
-        if (
-          result &&
-          result.failed === 0 &&
-          result.verified === result.discovered
-        ) {
-          markMigrated(sourceFingerprint, result);
-        }
-      } catch {
-        // Palace remains available for current writes; durable markdown/SQLite
-        // sources are untouched and migration retries because no state is set.
-      }
+    // Incremental catch-up driven by the record-level ledger (L2). Ordinary
+    // store() writes update the ledger inline, so the only work here is
+    // out-of-band .md edits and records whose earlier upsert was deferred by
+    // lock contention. The whole-corpus fingerprint sweep is gone.
+    const source = getMemoryBaseDir();
+    try {
+      // One-time: seed the ledger from a completed legacy marker so existing
+      // installs do not re-migrate everything on the first ledger-era boot.
+      bootstrapLedgerFromLegacyMarker(source);
+
+      const ledger = readLedger();
+      const scanned = scanMemorySources(source);
+      const scannedByKey = new Map(scanned.map((r) => [r.key, r.hash]));
+      const { keys, firstRun } = ledgerDelta(scanned, ledger);
+      if (keys.length === 0) return; // fully in sync
+
+      // L3: if a MemPalace daemon is actively mining, it holds the palace lock,
+      // so a direct catch-up would just defer every contended upsert. Skip this
+      // session and let the L1/L2 backoff retry once the lock frees, rather than
+      // fighting the daemon. (There is no idempotent record-upsert daemon job to
+      // route through — its generic write path uses an incompatible drawer id.)
+      const daemon = await probeDaemon(this.palacePath);
+      if (daemon.reachable && daemon.busy) return;
+
+      // First ever ledger population runs a full pass (source_dir only);
+      // otherwise target just the delta so we never re-sweep the corpus.
+      const args: Record<string, unknown> = firstRun
+        ? { source_dir: source }
+        : { source_dir: source, only: keys };
+      const raw = await runBridgeAsync<unknown>(install, this.palacePath, "migrate", args, 15 * 60_000);
+      const result = normalizeMigrationResult(raw);
+      if (result) writeLedger(applyMigrationToLedger(ledger, result, scannedByKey));
+    } catch {
+      // Palace remains available for current writes; durable markdown sources
+      // are untouched and catch-up retries on a later session.
     }
-
-    return true;
   }
 
 
@@ -352,7 +388,7 @@ export class MemoryStorage {
    * consistent and durable as a fallback source.
    */
   private storeMempalace(record: MemoryRecord): void {
-    this.memPalaceCall("store", {
+    const stored = this.memPalaceCall("store", {
       record: {
         id: record.id,
         title: record.title,
@@ -366,14 +402,25 @@ export class MemoryStorage {
       },
     });
     // Markdown tier (durable human copy + fallback source).
+    let mdPath: string | null = null;
     try {
-      const mdPath = path.join(this.scopeDir, `${record.id}.md`);
+      mdPath = path.join(this.scopeDir, `${record.id}.md`);
       const dir = path.dirname(mdPath);
       if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
       writeMemoryFile(mdPath, record);
     } catch {
       // Palace write succeeded; markdown is best-effort.
     }
+    // L2: only record the ledger when the palace upsert actually succeeded
+    // (the bridge returns an object on success, null on transient/failed). A
+    // skipped ledger write means the incremental catch-up re-attempts this
+    // record later — we never claim an unsynced record as synced.
+    if (stored !== null && mdPath !== null) {
+      try {
+        const bytes = fs.readFileSync(mdPath, "utf-8");
+        ledgerRecordStore(`${record.project}/${record.id}`, bytes);
+      } catch { /* ledger is best-effort; delta will catch it next boot */ }
+    }
   }
 
   /**