@aeriondyseti/vector-memory-mcp 2.4.4-dev.1 → 2.5.0-dev.1

package/README.md

@@ -117,13 +117,54 @@ Assistant: [calls search_memories with history_only: true, history_before/after
 
 ---
 
+## Storage Model
+
+All memories live in a single global database (`~/.vector-memory/memories.db`)
+shared by every project. Each memory is tagged with the project (working
+directory) it was stored from:
+
+- **Searches default to all projects** — results carry their project path, and
+  hits from the current project rank slightly higher. Use `scope: "project"`
+  to restrict a search to the current repo.
+- **Waypoints are per-project** and resolved automatically from the working
+  directory.
+- A repo-local database is still available via `--db-file` or
+  `VECTOR_MEMORY_DB_PATH` (note: keep the db on local disk — WAL mode
+  misbehaves on network filesystems like NFS home directories).
+
+### Migrating repo-local databases
+
+Projects that used the old per-repo `.vector-memory/` layout can be imported
+into the global store:
+
+```bash
+# Import the current repo's .vector-memory/memories.db
+bunx @aeriondyseti/vector-memory-mcp consolidate
+
+# Scan a whole directory tree and import every repo-local db found
+bunx @aeriondyseti/vector-memory-mcp consolidate ~/Development --recursive
+
+# Preview without writing (prints planned imports and ID re-keys)
+bunx @aeriondyseti/vector-memory-mcp consolidate --dry-run
+```
+
+Consolidation tags every imported memory with its repo's path, preserves
+embeddings and usefulness stats, deduplicates by ID, re-keys waypoints to
+their per-project IDs (remapping references), and backs up the global db
+first. `--archive` renames the source `.vector-memory/` to
+`.vector-memory.migrated/` after a successful import; `--force` skips the
+live-server check.
+
+---
+
 ## Configuration
 
 CLI flags:
 
 | Flag | Alias | Default | Description |
 |------|-------|---------|-------------|
-| `--db-file <path>` | `-d` | `.vector-memory/memories.db` | Database location (relative to cwd) |
+| `--db-file <path>` | `-d` | `~/.vector-memory/memories.db` | Database location (global store) |
+| `--project <path>` | | *(cwd)* | Project identity used to tag memories |
 | `--port <number>` | `-p` | `3271` | HTTP server port |
 | `--no-http` | | *(HTTP enabled)* | Disable HTTP/SSE transport |
 | `--enable-history` | | *(disabled)* | Enable conversation history indexing |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.4.4-dev.1",
+  "version": "2.5.0-dev.1",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "server/index.ts",

package/server/config/index.ts

@@ -1,6 +1,7 @@
 import arg from "arg";
 import { homedir } from "os";
 import { isAbsolute, join } from "path";
+import { normalizeProject } from "../core/project";
 import packageJson from "../../package.json" with { type: "json" };
 
 export const VERSION = packageJson.version;
@@ -23,6 +24,8 @@ export interface ConversationHistoryConfig {
 
 export interface Config {
   dbPath: string;
+  /** Canonical project identifier — normalized absolute path of the project root. */
+  project: string;
   embeddingModel: string;
   embeddingDimension: number;
   httpPort: number;
@@ -35,6 +38,7 @@ export interface Config {
 
 export interface ConfigOverrides {
   dbPath?: string;
+  project?: string;
   httpPort?: number;
   enableHttp?: boolean;
   pluginMode?: boolean;
@@ -44,8 +48,10 @@ export interface ConfigOverrides {
   historyWeight?: number;
 }
 
-// Defaults - always use repo-local .vector-memory folder
-const DEFAULT_DB_PATH = join(process.cwd(), ".vector-memory", "memories.db");
+// Defaults — single global store shared by all projects. Memories are tagged
+// with the project (cwd) they came from. Use --db-file / VECTOR_MEMORY_DB_PATH
+// for a repo-local database.
+const DEFAULT_DB_PATH = join(homedir(), ".vector-memory", "memories.db");
 const DEFAULT_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
 const DEFAULT_EMBEDDING_DIMENSION = 384;
 const DEFAULT_HTTP_PORT = 3271;
@@ -69,6 +75,7 @@ export function loadConfig(overrides: ConfigOverrides = {}): Config {
       ?? process.env.VECTOR_MEMORY_DB_PATH
       ?? DEFAULT_DB_PATH
     ),
+    project: normalizeProject(overrides.project ?? process.cwd()),
     embeddingModel: DEFAULT_EMBEDDING_MODEL,
     embeddingDimension: DEFAULT_EMBEDDING_DIMENSION,
     httpPort:
@@ -99,6 +106,7 @@ export function parseCliArgs(argv: string[]): ConfigOverrides {
   const args = arg(
     {
       "--db-file": String,
+      "--project": String,
       "--port": Number,
       "--no-http": Boolean,
       "--plugin": Boolean,
@@ -115,6 +123,7 @@ export function parseCliArgs(argv: string[]): ConfigOverrides {
 
   return {
     dbPath: args["--db-file"],
+    project: args["--project"],
     httpPort: args["--port"],
     enableHttp: args["--no-http"] ? false : undefined,
     pluginMode: args["--plugin"] ?? undefined,

package/server/core/connection.ts

@@ -1,24 +1,130 @@
 import { Database } from "bun:sqlite";
-import { existsSync, mkdirSync } from "fs";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, unlinkSync, writeSync } from "fs";
 import { dirname } from "path";
 import { removeVec0Tables, runMigrations } from "./migrations";
 
+/** How long a starting process will wait for a concurrent vec0 cleanup to finish. */
+const CLEANUP_LOCK_WAIT_MS = 15_000;
+const CLEANUP_LOCK_POLL_MS = 250;
+
+/**
+ * Check (read-only) whether the database still contains legacy vec0 virtual
+ * table entries. The destructive cleanup must only run when this is true —
+ * it rewrites sqlite_master via the sqlite3 CLI, which is unsafe while other
+ * connections hold the database open.
+ */
+function hasVec0Tables(dbPath: string): boolean {
+  let db: Database | null = null;
+  try {
+    db = new Database(dbPath, { readonly: true });
+    const row = db
+      .prepare("SELECT 1 FROM sqlite_master WHERE sql LIKE '%vec0%' LIMIT 1")
+      .get();
+    return row != null;
+  } catch {
+    // Unreadable / empty file — let the normal open path surface real errors
+    return false;
+  } finally {
+    db?.close();
+  }
+}
+
+function lockPid(lockPath: string): number | null {
+  try {
+    const pid = parseInt(readFileSync(lockPath, "utf8"), 10);
+    return Number.isFinite(pid) ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Run the vec0 cleanup under an exclusive advisory lock so that N processes
+ * starting against the same database never run the sqlite_master rewrite
+ * concurrently. Losers wait for the winner, then re-probe (the winner's
+ * cleanup makes the probe false).
+ */
+function guardedVec0Cleanup(dbPath: string): void {
+  const lockPath = `${dbPath}.vec0-cleanup.lock`;
+  const deadline = Date.now() + CLEANUP_LOCK_WAIT_MS;
+
+  while (true) {
+    try {
+      const fd = openSync(lockPath, "wx");
+      try {
+        writeSync(fd, String(process.pid));
+        // Re-probe under the lock — another process may have cleaned up
+        // between our first probe and lock acquisition.
+        if (hasVec0Tables(dbPath)) {
+          removeVec0Tables(dbPath);
+        }
+      } finally {
+        closeSync(fd);
+        try {
+          unlinkSync(lockPath);
+        } catch {
+          // already gone — fine
+        }
+      }
+      return;
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+
+      // Lock held — if the holder died, clear the stale lock and retry.
+      const pid = lockPid(lockPath);
+      if (pid !== null && !isProcessAlive(pid)) {
+        try {
+          unlinkSync(lockPath);
+        } catch {
+          // raced with another process clearing it — fine
+        }
+        continue;
+      }
+
+      if (Date.now() > deadline) {
+        throw new Error(
+          `vec0 cleanup lock held too long (${lockPath}) — remove it manually if no other server is starting`,
+        );
+      }
+      Bun.sleepSync(CLEANUP_LOCK_POLL_MS);
+    }
+  }
+}
+
 /**
  * Open (or create) a SQLite database at the given path
  * and run schema migrations.
+ *
+ * Safe for multiple concurrent processes sharing one database file:
+ * the legacy vec0 cleanup only runs when a read-only probe finds vec0
+ * entries (never for healthy databases) and is serialized by an exclusive
+ * lock; migrations are user_version-gated inside an immediate transaction.
  */
 export function connectToDatabase(dbPath: string): Database {
   mkdirSync(dirname(dbPath), { recursive: true });
 
   // Remove orphaned vec0 virtual table entries before bun:sqlite opens the
   // database. bun:sqlite cannot modify sqlite_master, so this uses the
-  // sqlite3 CLI while no other connection holds a lock.
-  if (existsSync(dbPath)) {
-    removeVec0Tables(dbPath);
+  // sqlite3 CLI — gated behind a read-only probe and an exclusive lock.
+  if (existsSync(dbPath) && hasVec0Tables(dbPath)) {
+    guardedVec0Cleanup(dbPath);
   }
 
   const db = new Database(dbPath);
 
+  // busy_timeout FIRST: it is per-connection and needs no lock, while the
+  // WAL switch takes an exclusive lock — without the timeout, concurrent
+  // processes opening a fresh db race it and fail with SQLITE_BUSY.
+  db.exec("PRAGMA busy_timeout=5000");
   // WAL mode for concurrent read performance
   db.exec("PRAGMA journal_mode=WAL");