kyp-mem 0.7.1 → 0.7.3

package/bin/cli.mjs

@@ -3,8 +3,9 @@
 import { spawnSync } from "child_process";
 import { appendFileSync, mkdirSync } from "fs";
 import { homedir } from "os";
-import { delimiter, dirname, join, resolve } from "path";
+import { delimiter, dirname, resolve, join } from "path";
 import { fileURLToPath } from "url";
+import { ensureVenv, resolvePython, venvDir } from "./python-env.mjs";
 
 const args = process.argv.slice(2);
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -21,34 +22,6 @@ function run(command, cmdArgs, stdio = "ignore") {
   return spawnSync(command, cmdArgs, { stdio, env });
 }
 
-function pythonCandidates() {
-  if (process.env.KYP_MEM_PYTHON) {
-    return [[process.env.KYP_MEM_PYTHON, []]];
-  }
-
-  const candidates = [
-    ["python3", []],
-    ["python", []],
-  ];
-
-  if (process.platform === "win32") {
-    candidates.unshift(["py", ["-3"]]);
-  }
-
-  return candidates;
-}
-
-function findPython() {
-  for (const [command, prefixArgs] of pythonCandidates()) {
-    const result = run(command, [...prefixArgs, "--version"]);
-    if (result.status === 0) {
-      return [command, prefixArgs];
-    }
-  }
-
-  return null;
-}
-
 // --- Hook fast path (pure Node, no Python startup) ---
 if (args[0] === "hook") {
   const hookType = args[1];
@@ -116,7 +89,8 @@ if (args[0] === "hook") {
   }
 
   if (hookType === "stop") {
-    const py = findPython();
+    // Bootstrap is fine here — the Stop hook is not latency-critical.
+    const py = resolvePython();
     if (py) {
       const [cmd, pre] = py;
       const r = run(cmd, [...pre, "-m", "kyp_mem.hooks", "stop"], "inherit");
@@ -126,7 +100,8 @@ if (args[0] === "hook") {
   }
 
   if (hookType === "session-start") {
-    const py = findPython();
+    // Good place to self-heal the venv at the start of a session.
+    const py = resolvePython();
     if (py) {
       const [cmd, pre] = py;
       const r = run(cmd, [...pre, "-m", "kyp_mem.cli", "hook", "session-start"], "inherit");
@@ -139,7 +114,19 @@ if (args[0] === "hook") {
   process.exit(1);
 }
 
-const python = findPython();
+// --- doctor: (re)provision the managed venv, then run the Python health check ---
+// Done in Node so it self-heals even when the venv is too broken to run Python.
+if (args[0] === "doctor") {
+  console.log("  Checking kyp-mem Python environment...");
+  if (!ensureVenv({ stdio: "inherit", force: true })) {
+    console.error("  \x1b[31m✗\x1b[0m Could not build the environment. Is Python 3.10+ installed?");
+    process.exit(1);
+  }
+  console.log(`  \x1b[32m✓\x1b[0m Environment ready (${venvDir()})`);
+  // fall through to run `kyp_mem.cli doctor` for the full health report
+}
+
+const python = resolvePython();
 
 if (python) {
   const [command, prefixArgs] = python;

package/bin/install.mjs

@@ -3,11 +3,8 @@
 import { spawnSync } from "child_process";
 import { mkdirSync } from "fs";
 import { homedir } from "os";
-import { fileURLToPath } from "url";
-import { dirname, join, resolve } from "path";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const root = resolve(__dirname, "..");
+import { join } from "path";
+import { ensureVenv, findSystemPython, resolvePython, venvDir } from "./python-env.mjs";
 
 const G = "\x1b[32m";
 const Y = "\x1b[33m";
@@ -17,74 +14,33 @@ const R = "\x1b[0m";
 
 function run(command, args, options = {}) {
   return spawnSync(command, args, {
-    cwd: root,
     stdio: options.stdio ?? "ignore",
-    env: {
-      ...process.env,
-      PIP_DISABLE_PIP_VERSION_CHECK: "1",
-    },
+    env: { ...process.env, PIP_DISABLE_PIP_VERSION_CHECK: "1" },
   });
 }
 
-function pythonCandidates() {
-  if (process.env.KYP_MEM_PYTHON) {
-    return [[process.env.KYP_MEM_PYTHON, []]];
-  }
-
-  const candidates = [
-    ["python3", []],
-    ["python", []],
-  ];
-
-  if (process.platform === "win32") {
-    candidates.unshift(["py", ["-3"]]);
-  }
-
-  return candidates;
-}
-
-function findPython() {
-  for (const [command, prefixArgs] of pythonCandidates()) {
-    const result = run(command, [...prefixArgs, "--version"]);
-    if (result.status === 0) {
-      return [command, prefixArgs];
-    }
-  }
-
-  return null;
-}
-
 if (process.env.KYP_MEM_SKIP_PYTHON_INSTALL === "1") {
   process.exit(0);
 }
 
-const python = findPython();
-
-if (!python) {
+if (!findSystemPython()) {
   console.log(`  ${Y}!${R} Python 3 was not found.`);
-  console.log(`  ${Y}!${R} Install Python 3.10+ and run: python3 -m pip install --user .`);
+  console.log(`  ${Y}!${R} Install Python 3.10+ and re-run: ${C}npm rebuild kyp-mem${R}`);
   process.exit(0);
 }
 
-const [pythonCommand, pythonPrefixArgs] = python;
+// Step 1: Provision kyp-mem's own virtualenv with all dependencies.
+// A dedicated venv works even when the system Python is externally managed
+// (PEP 668), so users never have to create or manage one themselves.
+console.log(`  Setting up kyp-mem Python environment...`);
 
-// Step 1: Install Python package
-console.log(`  Installing kyp-mem Python package...`);
-
-const pipResult = run(
-  pythonCommand,
-  [...pythonPrefixArgs, "-m", "pip", "install", "--user", "."],
-  { stdio: "inherit" },
-);
-
-if (pipResult.status !== 0) {
-  console.log(`  ${Y}!${R} Could not auto-install the Python package.`);
-  console.log(`  ${Y}!${R} Run manually from ${root}:`);
-  console.log("    python3 -m pip install --user .");
+if (!ensureVenv({ stdio: "inherit", force: true })) {
+  console.log(`  ${Y}!${R} Could not provision the Python environment automatically.`);
+  console.log(`  ${Y}!${R} kyp-mem will retry on first run, or run it now: ${C}kyp-mem doctor${R}`);
   process.exit(0);
 }
 
-console.log(`  ${G}✓${R} Python package installed`);
+console.log(`  ${G}✓${R} Python environment ready ${D}(${venvDir()})${R}`);
 
 // Step 2: Create default vault directory
 const vaultDir = join(homedir(), ".kyp-mem", "vault");
@@ -95,12 +51,12 @@ try {
   console.log(`  ${Y}!${R} Could not create vault at ${vaultDir}`);
 }
 
+const [py, pre] = resolvePython({ allowBootstrap: false });
+
 // Step 3: Register MCP server with Claude Code (global)
-const setupResult = run(
-  pythonCommand,
-  [...pythonPrefixArgs, "-m", "kyp_mem.cli", "setup-claude", "--global"],
-  { stdio: "inherit" },
-);
+const setupResult = run(py, [...pre, "-m", "kyp_mem.cli", "setup-claude", "--global"], {
+  stdio: "inherit",
+});
 
 if (setupResult.status === 0) {
   console.log(`  ${G}✓${R} MCP server registered with Claude Code`);
@@ -109,11 +65,9 @@ if (setupResult.status === 0) {
 }
 
 // Step 4: Install hooks (global)
-const hooksResult = run(
-  pythonCommand,
-  [...pythonPrefixArgs, "-m", "kyp_mem.cli", "install-hooks", "--global"],
-  { stdio: "inherit" },
-);
+const hooksResult = run(py, [...pre, "-m", "kyp_mem.cli", "install-hooks", "--global"], {
+  stdio: "inherit",
+});
 
 if (hooksResult.status === 0) {
   console.log(`  ${G}✓${R} Session capture hooks installed`);

package/bin/python-env.mjs

@@ -0,0 +1,115 @@
+// Shared Python environment management for kyp-mem.
+//
+// kyp-mem ships a Python backend, but modern system interpreters (Homebrew,
+// recent Debian/Ubuntu) are "externally managed" (PEP 668), so installing
+// dependencies into them with pip is blocked. Rather than ask every user to
+// create and manage a virtualenv, kyp-mem owns one: it is created at install
+// time, auto-detected at runtime, and lazily rebuilt if it ever goes missing.
+// Users never have to create, activate, or even know about it.
+
+import { spawnSync } from "child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import { homedir } from "os";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const projectRoot = resolve(__dirname, "..");
+const isWin = process.platform === "win32";
+
+export function packageVersion() {
+  try {
+    const pkg = JSON.parse(readFileSync(join(projectRoot, "package.json"), "utf8"));
+    return pkg.version || "0";
+  } catch (_) {
+    return "0";
+  }
+}
+
+export function venvDir() {
+  return join(homedir(), ".kyp-mem", "venv");
+}
+
+export function venvPython() {
+  return isWin
+    ? join(venvDir(), "Scripts", "python.exe")
+    : join(venvDir(), "bin", "python");
+}
+
+// Records the package version the venv was last provisioned for, so a kyp-mem
+// upgrade transparently reinstalls dependencies on next run.
+function stampFile() {
+  return join(venvDir(), ".kyp-installed");
+}
+
+function run(command, cmdArgs, stdio = "ignore") {
+  return spawnSync(command, cmdArgs, {
+    stdio,
+    env: { ...process.env, PIP_DISABLE_PIP_VERSION_CHECK: "1" },
+  });
+}
+
+// Find a system Python to build the venv from (or to fall back to).
+export function findSystemPython() {
+  const candidates = [];
+  if (process.env.KYP_MEM_PYTHON) candidates.push([process.env.KYP_MEM_PYTHON, []]);
+  if (isWin) candidates.push(["py", ["-3"]]);
+  candidates.push(["python3", []], ["python", []]);
+
+  for (const [command, prefixArgs] of candidates) {
+    const result = run(command, [...prefixArgs, "--version"]);
+    if (result.status === 0) return [command, prefixArgs];
+  }
+  return null;
+}
+
+// The venv exists and was provisioned for the current package version.
+export function venvReady() {
+  if (!existsSync(venvPython())) return false;
+  try {
+    return readFileSync(stampFile(), "utf8").trim() === packageVersion();
+  } catch (_) {
+    return false;
+  }
+}
+
+// Create the venv (if needed) and install kyp-mem + its dependencies into it.
+// Returns true on success. `stdio` controls pip/venv output; pass "inherit"
+// during `npm install` so users see progress, "ignore" for runtime bootstrap.
+export function ensureVenv({ stdio = "ignore", force = false } = {}) {
+  if (!force && venvReady()) return true;
+
+  const sys = findSystemPython();
+  if (!sys) return false;
+  const [cmd, pre] = sys;
+
+  if (!existsSync(venvPython())) {
+    mkdirSync(venvDir(), { recursive: true });
+    const created = run(cmd, [...pre, "-m", "venv", venvDir()], stdio);
+    if (created.status !== 0 || !existsSync(venvPython())) return false;
+  }
+
+  const py = venvPython();
+  run(py, ["-m", "pip", "install", "--upgrade", "pip"], stdio);
+  const installed = run(py, ["-m", "pip", "install", projectRoot], stdio);
+  if (installed.status !== 0) return false;
+
+  try {
+    writeFileSync(stampFile(), packageVersion());
+  } catch (_) {}
+  return true;
+}
+
+// Resolve the Python interpreter kyp-mem should run with.
+//   1. KYP_MEM_PYTHON, if set, is an explicit override (power-user escape hatch).
+//   2. The managed venv — bootstrapped on demand when `allowBootstrap`.
+//   3. Whatever venv/system Python exists, even if not fully provisioned.
+// `allowBootstrap: false` skips the (slow) install step for latency-sensitive
+// callers so they never block on a pip run.
+export function resolvePython({ allowBootstrap = true } = {}) {
+  if (process.env.KYP_MEM_PYTHON) return [process.env.KYP_MEM_PYTHON, []];
+  if (venvReady()) return [venvPython(), []];
+  if (allowBootstrap && ensureVenv()) return [venvPython(), []];
+  if (existsSync(venvPython())) return [venvPython(), []];
+  return findSystemPython();
+}

package/kyp_mem/cli.py

@@ -191,11 +191,29 @@ def _run_setup_claude(global_config: bool = False):
     print()
 
 
+def _is_in_venv(bin_path: Path) -> bool:
+    import sys
+    if sys.prefix == sys.base_prefix:
+        return False
+    try:
+        bin_path.resolve().relative_to(Path(sys.prefix).resolve())
+        return True
+    except ValueError:
+        return False
+
+
 def _get_mcp_command() -> tuple[str, list[str]]:
     kyp_mem_bin = shutil.which("kyp-mem")
     npx_bin = shutil.which("npx")
 
     if kyp_mem_bin and "_npx" not in Path(kyp_mem_bin).parts:
+        bin_path = Path(kyp_mem_bin)
+        if _is_in_venv(bin_path):
+            print(f"  {Y}Warning:{R} kyp-mem found inside a virtual env — this path won't work outside it.")
+            if npx_bin:
+                print(f"  {D}Using npx for a stable path instead.{R}")
+                return npx_bin, ["-y", "kyp-mem", "serve"]
+            print(f"  {D}Install globally: npm install -g kyp-mem{R}")
         return kyp_mem_bin, ["serve"]
     if npx_bin:
         return npx_bin, ["-y", "kyp-mem", "serve"]
@@ -326,7 +344,7 @@ def _run_uninstall(purge: bool = False):
         print()
 
 
-
+def _run_install_hooks(global_config: bool = False, remove: bool = False):
     mcp_command, _ = _get_mcp_command()
 
     if global_config:

package/kyp_mem/vault.py

@@ -233,10 +233,14 @@ class Vault:
 
     def _sync_vector_db(self):
         mem = get_session_memory()
-        for path, note in self.index.notes.items():
-            if "/Sessions/" in path or path.startswith("Sessions/"):
-                folder = note.folder
-                mem.upsert_session(path, folder, note.content)
+        if mem is None:
+            return
+        items = {
+            path: (note.folder, note.content)
+            for path, note in self.index.notes.items()
+            if "/Sessions/" in path or path.startswith("Sessions/")
+        }
+        mem.sync_sessions(items)
 
     def _load_all(self):
         notes = {}

package/kyp_mem/vector.py

@@ -1,43 +1,212 @@
+import sys
+import shutil
+import hashlib
 import chromadb
 from pathlib import Path
+from contextlib import contextmanager
+
+
+def _content_hash(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+try:
+    import fcntl  # POSIX (macOS/Linux)
+except ImportError:  # pragma: no cover - Windows fallback
+    fcntl = None
+
+
+def _log(msg: str):
+    print(f"[kyp-mem vector] {msg}", file=sys.stderr)
+
+
+def _clear_chroma_cache():
+    """Drop Chroma's process-wide PersistentClient cache so the next
+    PersistentClient(path=...) re-reads from disk instead of returning a stale
+    cached instance."""
+    try:
+        from chromadb.api.shared_system_client import SharedSystemClient
+        SharedSystemClient.clear_system_cache()
+    except Exception:
+        pass
+
 
 class SessionMemory:
+    """Semantic session store backed by ChromaDB.
+
+    Several processes touch the same on-disk Chroma directory at once: the web
+    UI, the MCP server, and the short-lived Claude Code hooks. ChromaDB's
+    PersistentClient is not built for concurrent multi-process writes, so
+    interleaved writes can corrupt the HNSW segment's pickle on disk. We guard
+    against that two ways:
+
+      1. A cross-process file lock serializes writes (and isolates them from
+         reads) so concurrent processes don't clobber each other.
+      2. If the index is already corrupt, we detect it and rebuild from the
+         markdown vault, which is the source of truth.
+    """
+
     def __init__(self, vault_path: str):
         self.db_path = Path(vault_path).parent / "chroma"
         self.db_path.mkdir(parents=True, exist_ok=True)
+        self._lock_path = self.db_path / ".kyp.lock"
+        self._open()
+        self._heal_if_corrupt()
+
+    # --- connection / recovery -------------------------------------------------
+
+    def _open(self):
         self.client = chromadb.PersistentClient(path=str(self.db_path))
         self.collection = self.client.get_or_create_collection(name="sessions")
 
+    def _rebuild(self):
+        """Drop the corrupted index and start fresh.
+
+        Safe because every session is re-embedded from its markdown note on the
+        next sync (see Vault._sync_vector_db).
+
+        We first try dropping the collection through Chroma's API (which orphans
+        the bad segment and creates a clean one). If that fails, we wipe the
+        directory on disk. Either way we must clear Chroma's process-wide client
+        cache: PersistentClient instances are cached by path, so without this a
+        re-created client would keep pointing at the deleted files (surfacing as
+        "attempt to write a readonly database")."""
+        _log("index appears corrupt — rebuilding chroma store from the vault")
+        try:
+            self.client.delete_collection(name="sessions")
+            self.collection = self.client.get_or_create_collection(name="sessions")
+            return
+        except Exception as e:
+            _log(f"in-place collection reset failed ({e!r}); wiping store on disk")
+
+        self.client = None
+        self.collection = None
+        _clear_chroma_cache()
+        shutil.rmtree(self.db_path, ignore_errors=True)
+        self.db_path.mkdir(parents=True, exist_ok=True)
+        self._open()
+
+    def _heal_if_corrupt(self):
+        """Force the write/compaction path that surfaces a corrupt segment.
+
+        Corruption only throws when Chroma deserializes the HNSW segment during
+        a write. We run a sentinel upsert+delete under the lock; if that raises,
+        we rebuild before any real sync runs, so the rebuilt store fills cleanly
+        in one pass."""
+        sentinel = "__kyp_healthcheck__"
+        try:
+            with self._locked(write=True):
+                # upsert+delete exercises the write/compaction path; the query
+                # forces the HNSW segment to load (the read path). Between them
+                # they surface both ways a corrupt segment manifests.
+                self.collection.upsert(documents=["ok"], ids=[sentinel])
+                self.collection.query(query_texts=["ok"], n_results=1)
+                self.collection.delete(ids=[sentinel])
+        except Exception as e:
+            _log(f"health check failed: {e!r}")
+            with self._locked(write=True):
+                self._rebuild()
+
+    # --- locking ---------------------------------------------------------------
+
+    @contextmanager
+    def _locked(self, write: bool):
+        if fcntl is None:
+            yield
+            return
+        mode = fcntl.LOCK_EX if write else fcntl.LOCK_SH
+        with open(self._lock_path, "a+") as lf:
+            fcntl.flock(lf, mode)
+            try:
+                yield
+            finally:
+                fcntl.flock(lf, fcntl.LOCK_UN)
+
+    # --- operations ------------------------------------------------------------
+
+    def _write_with_recovery(self, op):
+        """Run a write op under the lock. If it fails (e.g. a corrupt segment
+        slipped past the init health check), rebuild the store once and retry.
+
+        ``op`` must be self-contained — after a rebuild the collection is empty,
+        so an op that derives its work from the current collection state
+        naturally repopulates everything on the retry."""
+        for attempt in (1, 2):
+            try:
+                with self._locked(write=True):
+                    op()
+                return
+            except Exception as e:
+                if attempt == 1:
+                    _log(f"write failed ({e!r}); rebuilding and retrying")
+                    with self._locked(write=True):
+                        self._rebuild()
+                else:
+                    _log(f"write failed after rebuild: {e!r}")
+
     def upsert_session(self, path: str, project: str, content: str):
-        self.collection.upsert(
-            documents=[content],
-            metadatas=[{"project": project}],
-            ids=[path]
-        )
+        meta = {"project": project, "hash": _content_hash(content)}
+
+        def op():
+            self.collection.upsert(documents=[content], metadatas=[meta], ids=[path])
+
+        self._write_with_recovery(op)
+
+    def sync_sessions(self, items: dict):
+        """Reconcile the store with the current set of session notes.
+
+        ``items`` maps note path -> (project, content). New and changed notes
+        (by content hash) are (re)embedded, deleted notes are pruned, and
+        unchanged notes are skipped so we don't re-embed the whole corpus on
+        every Vault init/refresh."""
+        desired = {p: (proj, c, _content_hash(c)) for p, (proj, c) in items.items()}
+
+        def op():
+            existing = self.collection.get(include=["metadatas"])
+            existing_hash = {
+                i: (m or {}).get("hash")
+                for i, m in zip(existing["ids"], existing["metadatas"])
+            }
+            up_ids, up_docs, up_meta = [], [], []
+            for p, (proj, c, h) in desired.items():
+                if existing_hash.get(p) != h:
+                    up_ids.append(p)
+                    up_docs.append(c)
+                    up_meta.append({"project": proj, "hash": h})
+            stale = [i for i in existing_hash if i not in desired]
+            if up_ids:
+                self.collection.upsert(documents=up_docs, metadatas=up_meta, ids=up_ids)
+            if stale:
+                self.collection.delete(ids=stale)
+
+        self._write_with_recovery(op)
 
     def delete_session(self, path: str):
         try:
-            self.collection.delete(ids=[path])
+            with self._locked(write=True):
+                self.collection.delete(ids=[path])
         except Exception:
             pass
 
     def search_sessions(self, query: str, project: str = None, n_results: int = 5):
         where = {"project": project} if project else None
         try:
-            results = self.collection.query(
-                query_texts=[query],
-                n_results=n_results,
-                where=where
-            )
-            return results
+            with self._locked(write=False):
+                return self.collection.query(
+                    query_texts=[query],
+                    n_results=n_results,
+                    where=where,
+                )
         except Exception:
             return {"ids": [], "documents": [], "metadatas": [], "distances": []}
 
+
 session_memory = None
 
+
 def init_vector_db(vault_path: str):
     global session_memory
     session_memory = SessionMemory(vault_path)
 
+
 def get_session_memory():
     return session_memory

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kyp-mem",
-  "version": "0.7.1",
+  "version": "0.7.3",
   "description": "Know Your Project — Persistent & Session level knowledge base for AI agents. MCP-powered with wikilinks, backlinks, auto-learning, and neon web UI.",
   "bin": {
     "kyp-mem": "bin/cli.mjs"

package/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "kyp-mem"
-version = "0.7.1"
+version = "0.7.3"
 description = "Know Your Project — Persistent knowledge base for AI agents. MCP-powered with wikilinks, backlinks, auto-learning, and neon web UI."
 readme = "README.md"
 license = {text = "MIT"}