@pi-unipi/memory 2.2.1 → 2.3.0

package/index.ts

@@ -25,7 +25,8 @@ import {
   MemoryStorage,
   getProjectName,
   searchAllProjects,
-  listAllProjects,
+  listAllProjectsCachedAsync,
+  invalidateAllProjectsCache,
 } from "./storage.js";
 import { registerMemoryTools, MEMORY_TOOLS, GLOBAL_SEARCH_ALIAS } from "./tools.js";
 import { registerMemoryCommands } from "./commands.js";
@@ -37,6 +38,26 @@ const VERSION = getPackageVersion(dirname(fileURLToPath(import.meta.url)));
 /** Storage instance for current project */
 let projectStorage: MemoryStorage | null = null;
 
+/**
+ * Whether orphaned-file sync still owes this session a run.
+ *
+ * The sync spawns a Python bridge (~0.5s). Running it in session_start delayed
+ * every startup for a job that only matters once memory is actually used, so it
+ * is deferred to the first storage access instead.
+ */
+let orphanSyncPending = false;
+
+/** Run the deferred orphaned-file sync exactly once per session. */
+function ensureOrphanSync(storage: MemoryStorage): void {
+  if (!orphanSyncPending) return;
+  orphanSyncPending = false;
+  try {
+    storage.syncOrphanedFiles();
+  } catch {
+    // Sync failure must not break the tool call that triggered it.
+  }
+}
+
 /**
  * Get storage for the current project.
  */
@@ -45,6 +66,8 @@ function getStorage(): MemoryStorage {
     // Fallback: create new instance (shouldn't happen after session_start)
     return new MemoryStorage("unknown");
   }
+  // Any real use of memory picks up markdown files added out of band.
+  ensureOrphanSync(projectStorage);
   return projectStorage;
 }
 
@@ -56,7 +79,12 @@ export default function (pi: ExtensionAPI) {
   // Register tools and commands
   registerMemoryTools(pi, getStorage, {
     onRecall: () => { recallDone = true; },
-    onStore: () => { storeDone = true; },
+    onStore: () => {
+      storeDone = true;
+      // Fires on store and delete; drop the cached cross-project counts so
+      // the info overlay reflects the write.
+      invalidateAllProjectsCache();
+    },
   });
   registerMemoryCommands(pi, getStorage);
 
@@ -71,11 +99,11 @@ export default function (pi: ExtensionAPI) {
     projectStorage = new MemoryStorage(projectName);
     try {
       projectStorage.init();
-      
-      // Sync any orphaned markdown files into the database
-      const synced = projectStorage.syncOrphanedFiles();
-      // Removed console.warn — orphaned file sync is informational only.
-      // Visible via memory tool list or info-screen memory group.
+
+      // Orphaned markdown files are synced lazily on first storage access
+      // (see ensureOrphanSync) — the Python bridge spawn is too slow to run
+      // on the startup path, and nothing reads the result until memory is used.
+      orphanSyncPending = true;
     } catch (_err) {
       // Memory init failure — running without memory. Silent startup.
       projectStorage = null;
@@ -135,8 +163,10 @@ export default function (pi: ExtensionAPI) {
           let projectMemories: Array<{ id: string; title: string; type: string }> = [];
           let allMemories: Array<{ project: string; id: string; title: string; type: string }> = [];
           try {
-            projectMemories = projectStorage.listAll();
-            allMemories = listAllProjects();
+            // Async twins: the MemPalace backend spawns Python, which would
+            // otherwise block the UI while the overlay is open.
+            projectMemories = await projectStorage.listAllAsync();
+            allMemories = await listAllProjectsCachedAsync();
           } catch (_err) {
             // Info panel data unavailable — shows empty values.
           }
@@ -162,23 +192,31 @@ export default function (pi: ExtensionAPI) {
       });
     }
 
-    // Show memory status in UI
+    // Show memory status in UI.
+    //
+    // Both counts come from the Python MemPalace bridge (~1.5s combined) and
+    // only produce a status-bar string, so they are resolved after startup and
+    // the status is filled in when they land. Blocking session_start on them
+    // delayed the whole extension chain.
     if (ctx.hasUI) {
-      let projectCount = 0;
-      let projectCountAll = 0;
-      try {
-        projectCount = projectStorage?.listAll()?.length ?? 0;
-        projectCountAll = listAllProjects().length;
-      } catch (_err) {
-        // Count unavailable — status bar shows 0.
-      }
       const mempalaceActive = projectStorage?.isMempalace() ?? false;
       const backendIcon = mempalaceActive ? "🧠" : (isEmbeddingReady() ? "⚡" : "📝");
       const warn = hasModelChanged() ? " ⚠" : "";
-      ctx.ui.setStatus(
-        "unipi-memory",
-        `${backendIcon} mem ${projectCount}p/${projectCountAll}all${warn}`
-      );
+      const setStatus = (counts: string) =>
+        ctx.ui.setStatus("unipi-memory", `${backendIcon} mem ${counts}${warn}`);
+
+      setStatus("…");
+      void (async () => {
+        let projectCount = 0;
+        let projectCountAll = 0;
+        try {
+          projectCount = (await projectStorage?.listAllAsync())?.length ?? 0;
+          projectCountAll = (await listAllProjectsCachedAsync()).length;
+        } catch (_err) {
+          // Count unavailable — status bar shows 0.
+        }
+        setStatus(`${projectCount}p/${projectCountAll}all`);
+      })();
     }
   });
 

package/mempalace.ts

@@ -11,7 +11,7 @@
  * hard-fail because the backend is missing.
  */
 
-import { spawnSync } from "node:child_process";
+import { spawnSync, spawn } from "node:child_process";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import * as os from "node:os";
@@ -254,6 +254,71 @@ export function runBridge<T = unknown>(
   }
 }
 
+/**
+ * Async variant of runBridge that does not block the event loop.
+ *
+ * spawnSync freezes the process for the whole Python round-trip (~0.5-1.1s).
+ * Use this from any path that runs while the UI is live — startup status,
+ * background refreshes — so keystrokes stay responsive.
+ */
+export function runBridgeAsync<T = unknown>(
+  install: MempalaceInstall,
+  palace: string,
+  cmd: string,
+  args: Record<string, unknown> = {},
+): Promise<T | null> {
+  return new Promise((resolve) => {
+    let argsJson: string;
+    try {
+      argsJson = JSON.stringify(args);
+    } catch {
+      resolve(null);
+      return;
+    }
+
+    let child;
+    try {
+      child = spawn(install.python, [BRIDGE_PATH, palace, cmd, argsJson], {
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+    } catch {
+      resolve(null);
+      return;
+    }
+
+    let out = "";
+    let settled = false;
+    const finish = (value: T | null) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(value);
+    };
+
+    const timer = setTimeout(() => {
+      try { child.kill(); } catch { /* already gone */ }
+      finish(null);
+    }, 60_000);
+    // Do not hold the process open purely for a background bridge call.
+    timer.unref?.();
+
+    child.stdout?.setEncoding("utf-8");
+    child.stdout?.on("data", (chunk) => { out += chunk; });
+    child.on("error", () => finish(null));
+    child.on("close", (code) => {
+      if (code !== 0) return finish(null);
+      const trimmed = out.trim();
+      if (!trimmed) return finish(null);
+      try {
+        const parsed = JSON.parse(trimmed) as BridgeResponse<T>;
+        finish(parsed.ok ? ((parsed.result ?? null) as T | null) : null);
+      } catch {
+        finish(null);
+      }
+    });
+  });
+}
+
 /** Ping the bridge — returns true if the backend is alive. */
 export function ping(install: MempalaceInstall, palace: string): boolean {
   return runBridge<string>(install, palace, "ping") === "pong";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/memory",
-  "version": "2.2.1",
+  "version": "2.3.0",
   "description": "Persistent cross-session memory with MemPalace backend (auto-installed) and SQLite fallback for Pi coding agent",
   "type": "module",
   "main": "index.ts",
@@ -43,8 +43,8 @@
     "better-sqlite3": "^12.9.0",
     "sqlite-vec": "^0.1.9",
     "js-yaml": "^4.1.0",
-    "@pi-unipi/core": "2.2.0",
-    "@pi-unipi/info-screen": "2.2.1"
+    "@pi-unipi/core": "2.3.0",
+    "@pi-unipi/info-screen": "2.3.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "^0.80.0",

package/storage.ts

@@ -17,6 +17,7 @@ import { randomUUID } from "node:crypto";
 import {
   ensureMempalace,
   runBridge,
+  runBridgeAsync,
   isMigrated,
   markMigrated,
   isPingVerified,
@@ -281,6 +282,32 @@ export class MemoryStorage {
     return result;
   }
 
+  /** Async twin of memPalaceCall, for paths that must not block the UI. */
+  private async memPalaceCallAsync<T>(cmd: string, args: Record<string, unknown> = {}): Promise<T | null> {
+    const install = this.mempalaceInstall;
+    if (!install) return null;
+    const result = await runBridgeAsync<T>(install, this.palacePath, cmd, args);
+    if (result === null) {
+      invalidatePingVerified();
+    }
+    return result;
+  }
+
+  /**
+   * Async twin of listAll().
+   *
+   * The SQLite path is already fast and stays synchronous; only the MemPalace
+   * path (a Python spawn) actually needs to yield.
+   */
+  async listAllAsync(): Promise<Array<{ id: string; title: string; type: string }>> {
+    if (this.isMempalace()) {
+      return (await this.memPalaceCallAsync<MempalaceListItem[]>("list", {
+        wing: this.projectName,
+      })) ?? [];
+    }
+    return this.listAll();
+  }
+
   /**
    * Initialize storage. Tries MemPalace first (auto-install + one-way
    * auto-migration of legacy memories); falls back to SQLite if MemPalace
@@ -1043,16 +1070,57 @@ export function searchAllProjects(
     .slice(0, limit);
 }
 
+/** Result shape shared by listAllProjects and its cached wrapper. */
+type AllProjectsEntry = { project: string; id: string; title: string; type: string };
+
+/**
+ * Cached view of listAllProjects().
+ *
+ * The uncached call spawns a Python MemPalace bridge (~1.1s). It backs two
+ * display-only counters in the info overlay, so a slightly stale number is
+ * strictly better than a 1.1s stall on every startup.
+ */
+let allProjectsCache: { at: number; value: AllProjectsEntry[] } | null = null;
+
+/** How long a cached cross-project listing stays valid. */
+const ALL_PROJECTS_TTL_MS = 60_000;
+
+/** Drop the cached cross-project listing (call after storing/deleting). */
+export function invalidateAllProjectsCache(): void {
+  allProjectsCache = null;
+}
+
+/**
+ * Async twin of listAllProjectsCached(), for UI paths.
+ *
+ * On a cache miss the MemPalace path spawns Python; doing that synchronously
+ * froze the UI for ~1.1s. Only the bridge call is async — the SQLite fallback
+ * is fast enough to run inline.
+ */
+export async function listAllProjectsCachedAsync(): Promise<AllProjectsEntry[]> {
+  const now = Date.now();
+  if (allProjectsCache && now - allProjectsCache.at < ALL_PROJECTS_TTL_MS) {
+    return allProjectsCache.value;
+  }
+
+  const install = ensureMempalace();
+  let value: AllProjectsEntry[];
+  if (install) {
+    const items = (await runBridgeAsync<MempalaceListItemAll[]>(install, DEFAULT_PALACE, "list_all", {})) ?? [];
+    value = items.map((m) => ({ project: m.project, id: m.id, title: m.title, type: m.type }));
+  } else {
+    value = listAllProjects();
+  }
+
+  allProjectsCache = { at: now, value };
+  return value;
+}
+
 /**
  * List memories from ALL projects.
  * Returns memories with project name prefix.
  */
-export function listAllProjects(): Array<{
-  project: string;
-  id: string;
-  title: string;
-  type: string;
-}> {
+export function listAllProjects(): AllProjectsEntry[] {
   // MemPalace global path: list all drawers across wings.
   const install = ensureMempalace();
   if (install) {
@@ -1067,12 +1135,7 @@ export function listAllProjects(): Array<{
 
   // SQLite fallback: iterate project directories.
   const projectDirs = getAllProjectDirs();
-  const allMemories: Array<{
-    project: string;
-    id: string;
-    title: string;
-    type: string;
-  }> = [];
+  const allMemories: AllProjectsEntry[] = [];
 
   for (const { name: projectName, dir } of projectDirs) {
     const dbPath = path.join(dir, MEMORY_DB_NAME);