@aeriondyseti/vector-memory-mcp 2.4.4 → 2.5.0-dev.2

package/README.md

@@ -117,13 +117,54 @@ Assistant: [calls search_memories with history_only: true, history_before/after
 
 ---
 
+## Storage Model
+
+All memories live in a single global database (`~/.vector-memory/memories.db`)
+shared by every project. Each memory is tagged with the project (working
+directory) it was stored from:
+
+- **Searches default to all projects** — results carry their project path, and
+  hits from the current project rank slightly higher. Use `scope: "project"`
+  to restrict a search to the current repo.
+- **Waypoints are per-project** and resolved automatically from the working
+  directory.
+- A repo-local database is still available via `--db-file` or
+  `VECTOR_MEMORY_DB_PATH` (note: keep the db on local disk — WAL mode
+  misbehaves on network filesystems like NFS home directories).
+
+### Migrating repo-local databases
+
+Projects that used the old per-repo `.vector-memory/` layout can be imported
+into the global store:
+
+```bash
+# Import the current repo's .vector-memory/memories.db
+bunx @aeriondyseti/vector-memory-mcp consolidate
+
+# Scan a whole directory tree and import every repo-local db found
+bunx @aeriondyseti/vector-memory-mcp consolidate ~/Development --recursive
+
+# Preview without writing (prints planned imports and ID re-keys)
+bunx @aeriondyseti/vector-memory-mcp consolidate --dry-run
+```
+
+Consolidation tags every imported memory with its repo's path, preserves
+embeddings and usefulness stats, deduplicates by ID, re-keys waypoints to
+their per-project IDs (remapping references), and backs up the global db
+first. `--archive` renames the source `.vector-memory/` to
+`.vector-memory.migrated/` after a successful import; `--force` skips the
+live-server check.
+
+---
+
 ## Configuration
 
 CLI flags:
 
 | Flag | Alias | Default | Description |
 |------|-------|---------|-------------|
-| `--db-file <path>` | `-d` | `.vector-memory/memories.db` | Database location (relative to cwd) |
+| `--db-file <path>` | `-d` | `~/.vector-memory/memories.db` | Database location (global store) |
+| `--project <path>` | | *(cwd)* | Project identity used to tag memories |
 | `--port <number>` | `-p` | `3271` | HTTP server port |
 | `--no-http` | | *(HTTP enabled)* | Disable HTTP/SSE transport |
 | `--enable-history` | | *(disabled)* | Enable conversation history indexing |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.4.4",
+  "version": "2.5.0-dev.2",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "server/index.ts",
@@ -9,6 +9,8 @@
   },
   "files": [
     "server",
+    "scripts/lancedb-extract.ts",
+    "scripts/warmup.ts",
     "README.md",
     "LICENSE"
   ],

package/scripts/lancedb-extract.ts

@@ -0,0 +1,181 @@
+#!/usr/bin/env bun
+/**
+ * Standalone LanceDB data extractor — runs in a child process so that
+ * @lancedb/lancedb native bindings never coexist with bun:sqlite's
+ * extension loading in the same process.
+ *
+ * Usage: bun scripts/lancedb-extract.ts <lance-db-path>
+ * Output: JSON on stdout — { memories: Row[], conversations: Row[] }
+ */
+
+const source = process.argv[2];
+if (!source) {
+  console.error("Usage: bun scripts/lancedb-extract.ts <lance-db-path>");
+  process.exit(1);
+}
+
+// Arrow TimeUnit enum → divisor to convert to milliseconds.
+// 0=SECOND, 1=MILLISECOND, 2=MICROSECOND, 3=NANOSECOND
+// Negative divisor = multiply (seconds → ms needs ×1000).
+const TIME_UNIT_TO_MS_DIVISOR: Record<number, bigint> = {
+  0: -1000n,   // seconds → ms (multiply by 1000)
+  1: 1n,       // ms → no conversion
+  2: 1000n,    // μs → ms
+  3: 1000000n, // ns → ms
+};
+
+function buildTimestampDivisors(schema: any): Map<string, bigint> {
+  const map = new Map<string, bigint>();
+  for (const field of schema.fields) {
+    if (field.type.typeId === 10) {
+      map.set(field.name, TIME_UNIT_TO_MS_DIVISOR[field.type.unit] ?? 1n);
+    }
+  }
+  return map;
+}
+
+function columnValue(batch: any, colName: string, rowIdx: number): unknown {
+  const col = batch.getChild(colName);
+  if (!col) return undefined;
+  try {
+    return col.get(rowIdx);
+  } catch {
+    // Arrow's getter can throw on BigInt timestamps exceeding MAX_SAFE_INTEGER;
+    // fall back to the raw typed array.
+    let offset = rowIdx;
+    for (const data of col.data) {
+      if (offset < data.length) {
+        return (data.values instanceof BigInt64Array || data.values instanceof BigUint64Array)
+          ? data.values[offset]
+          : null;
+      }
+      offset -= data.length;
+    }
+    return null;
+  }
+}
+
+function toEpochMs(value: unknown, divisor: bigint = 1n): number {
+  if (value == null) return Date.now();
+  if (value instanceof Date) return value.getTime();
+  if (typeof value === "bigint") {
+    if (divisor < 0n) return Number(value * -divisor);  // seconds → ms
+    if (divisor === 1n) return Number(value);
+    return Number(value / divisor);
+  }
+  if (typeof value === "number") {
+    if (divisor < 0n) return value * Number(-divisor);
+    if (divisor === 1n) return value;
+    return Math.floor(value / Number(divisor));
+  }
+  return Date.now();
+}
+
+function toFloatArray(vec: unknown): number[] {
+  if (Array.isArray(vec)) return vec;
+  if (vec instanceof Float32Array) return Array.from(vec);
+  if (vec && typeof (vec as any).toArray === "function") {
+    return Array.from((vec as any).toArray());
+  }
+  if (ArrayBuffer.isView(vec)) {
+    const view = vec as DataView;
+    return Array.from(new Float32Array(view.buffer, view.byteOffset, view.byteLength / 4));
+  }
+  return [];
+}
+
+const BATCH_SIZE = 100;
+const lancedb = await import("@lancedb/lancedb");
+const db = await lancedb.connect(source);
+const tableNames = await db.tableNames();
+console.error(`Found tables: ${tableNames.join(", ")}`);
+
+const result: { memories: any[]; conversations: any[] } = {
+  memories: [],
+  conversations: [],
+};
+
+if (tableNames.includes("memories")) {
+  const table = await db.openTable("memories");
+  const total = await table.countRows();
+  console.error(`Reading ${total} memories...`);
+
+  // Paginated scan — query().toArrow() without offset/limit returns
+  // non-deterministic results that can duplicate some rows and skip others.
+  const schemaSample = await table.query().limit(1).toArrow();
+  const tsDivisors = buildTimestampDivisors(schemaSample.schema);
+  const seen = new Map<string, any>();
+
+  for (let offset = 0; offset < total; offset += BATCH_SIZE) {
+    const arrowTable = await table.query().offset(offset).limit(BATCH_SIZE).toArrow();
+    for (const batch of arrowTable.batches) {
+      for (let i = 0; i < batch.numRows; i++) {
+        const id = columnValue(batch, "id", i) as string;
+        const content = columnValue(batch, "content", i) as string;
+        const lastAccessed = columnValue(batch, "last_accessed", i);
+        const accessedMs = lastAccessed != null ? toEpochMs(lastAccessed, tsDivisors.get("last_accessed")) : null;
+        // Deduplicate by ID: prefer most recently accessed, then longest content.
+        const existing = seen.get(id);
+        if (existing) {
+          const existingAccess = existing.last_accessed ?? 0;
+          const newAccess = accessedMs ?? 0;
+          if (newAccess < existingAccess) continue;
+          if (newAccess === existingAccess && content.length <= existing.content.length) continue;
+        }
+        seen.set(id, {
+          id,
+          content,
+          metadata: columnValue(batch, "metadata", i) ?? "{}",
+          vector: toFloatArray(columnValue(batch, "vector", i)),
+          created_at: toEpochMs(columnValue(batch, "created_at", i), tsDivisors.get("created_at")),
+          updated_at: toEpochMs(columnValue(batch, "updated_at", i), tsDivisors.get("updated_at")),
+          last_accessed: accessedMs,
+          superseded_by: columnValue(batch, "superseded_by", i) ?? null,
+          usefulness: columnValue(batch, "usefulness", i) ?? 0,
+          access_count: columnValue(batch, "access_count", i) ?? 0,
+        });
+      }
+    }
+  }
+  result.memories = [...seen.values()];
+  console.error(`  ${result.memories.length} unique memories read (${total} rows scanned)`);
+}
+
+if (tableNames.includes("conversation_history")) {
+  const table = await db.openTable("conversation_history");
+  const total = await table.countRows();
+  console.error(`Reading ${total} conversation chunks...`);
+
+  const schemaSample = await table.query().limit(1).toArrow();
+  const tsDivisors = buildTimestampDivisors(schemaSample.schema);
+  const seen = new Map<string, any>();
+
+  for (let offset = 0; offset < total; offset += BATCH_SIZE) {
+    const arrowTable = await table.query().offset(offset).limit(BATCH_SIZE).toArrow();
+    for (const batch of arrowTable.batches) {
+      for (let i = 0; i < batch.numRows; i++) {
+        const id = columnValue(batch, "id", i) as string;
+        const content = columnValue(batch, "content", i) as string;
+        const existing = seen.get(id);
+        if (existing && existing.content.length >= content.length) continue;
+        seen.set(id, {
+          id,
+          content,
+          metadata: columnValue(batch, "metadata", i) ?? "{}",
+          vector: toFloatArray(columnValue(batch, "vector", i)),
+          created_at: toEpochMs(columnValue(batch, "created_at", i), tsDivisors.get("created_at")),
+          session_id: columnValue(batch, "session_id", i),
+          role: columnValue(batch, "role", i),
+          message_index_start: columnValue(batch, "message_index_start", i) ?? 0,
+          message_index_end: columnValue(batch, "message_index_end", i) ?? 0,
+          project: columnValue(batch, "project", i) ?? "",
+        });
+      }
+    }
+  }
+  result.conversations = [...seen.values()];
+  console.error(`  ${result.conversations.length} unique conversation chunks read (${total} rows scanned)`);
+}
+
+await db.close?.();
+process.stdout.write(JSON.stringify(result));

package/scripts/warmup.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env bun
+
+/**
+ * Warmup script to pre-download ML models and verify dependencies
+ * This runs during installation to ensure everything is ready to use
+ */
+
+import { config } from "../server/config/index";
+import { EmbeddingsService } from "../server/core/embeddings.service";
+
+async function warmup(): Promise<void> {
+  console.log("🔥 Warming up vector-memory-mcp...");
+  console.log();
+
+  try {
+    // Check native dependencies
+    console.log("✓ Checking native dependencies...");
+    try {
+      await import("onnxruntime-node");
+      console.log("  ✓ onnxruntime-node loaded");
+    } catch (e) {
+      console.error("  ✗ onnxruntime-node failed:", (e as Error).message);
+      process.exit(1);
+    }
+
+    console.log();
+
+    // Initialize embeddings service to download model
+    console.log("📥 Downloading ML model (this may take a minute)...");
+    console.log(`   Model: ${config.embeddingModel}`);
+    console.log();
+
+    const embeddings = new EmbeddingsService(
+      config.embeddingModel,
+      config.embeddingDimension
+    );
+
+    // Trigger model download by generating a test embedding
+    const startTime = Date.now();
+    await embeddings.embed("warmup test");
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+    console.log();
+    console.log(`✅ Warmup complete! (${duration}s)`);
+    console.log();
+    console.log("Ready to use! Configure your MCP client and restart to get started.");
+    console.log();
+  } catch (error) {
+    console.error();
+    console.error("❌ Warmup failed:", error);
+    console.error();
+    console.error("This is not a critical error - the server will download models on first run.");
+    console.error("You can try running 'vector-memory-mcp warmup' manually later.");
+    process.exit(0); // Exit successfully to not block installation
+  }
+}
+
+// Only run if this is the main module
+if (import.meta.url === `file://${process.argv[1]}`) {
+  warmup();
+}
+
+export { warmup };

package/server/config/index.ts

@@ -1,6 +1,7 @@
 import arg from "arg";
 import { homedir } from "os";
 import { isAbsolute, join } from "path";
+import { normalizeProject } from "../core/project";
 import packageJson from "../../package.json" with { type: "json" };
 
 export const VERSION = packageJson.version;
@@ -23,6 +24,8 @@ export interface ConversationHistoryConfig {
 
 export interface Config {
   dbPath: string;
+  /** Canonical project identifier — normalized absolute path of the project root. */
+  project: string;
   embeddingModel: string;
   embeddingDimension: number;
   httpPort: number;
@@ -35,6 +38,7 @@ export interface Config {
 
 export interface ConfigOverrides {
   dbPath?: string;
+  project?: string;
   httpPort?: number;
   enableHttp?: boolean;
   pluginMode?: boolean;
@@ -44,8 +48,10 @@ export interface ConfigOverrides {
   historyWeight?: number;
 }
 
-// Defaults - always use repo-local .vector-memory folder
-const DEFAULT_DB_PATH = join(process.cwd(), ".vector-memory", "memories.db");
+// Defaults — single global store shared by all projects. Memories are tagged
+// with the project (cwd) they came from. Use --db-file / VECTOR_MEMORY_DB_PATH
+// for a repo-local database.
+const DEFAULT_DB_PATH = join(homedir(), ".vector-memory", "memories.db");
 const DEFAULT_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
 const DEFAULT_EMBEDDING_DIMENSION = 384;
 const DEFAULT_HTTP_PORT = 3271;
@@ -69,6 +75,7 @@ export function loadConfig(overrides: ConfigOverrides = {}): Config {
       ?? process.env.VECTOR_MEMORY_DB_PATH
       ?? DEFAULT_DB_PATH
     ),
+    project: normalizeProject(overrides.project ?? process.cwd()),
     embeddingModel: DEFAULT_EMBEDDING_MODEL,
     embeddingDimension: DEFAULT_EMBEDDING_DIMENSION,
     httpPort:
@@ -99,6 +106,7 @@ export function parseCliArgs(argv: string[]): ConfigOverrides {
   const args = arg(
     {
       "--db-file": String,
+      "--project": String,
       "--port": Number,
       "--no-http": Boolean,
       "--plugin": Boolean,
@@ -115,6 +123,7 @@ export function parseCliArgs(argv: string[]): ConfigOverrides {
 
   return {
     dbPath: args["--db-file"],
+    project: args["--project"],
     httpPort: args["--port"],
     enableHttp: args["--no-http"] ? false : undefined,
     pluginMode: args["--plugin"] ?? undefined,

package/server/core/connection.ts

@@ -1,24 +1,180 @@
 import { Database } from "bun:sqlite";
-import { existsSync, mkdirSync } from "fs";
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeSync,
+} from "fs";
 import { dirname } from "path";
 import { removeVec0Tables, runMigrations } from "./migrations";
 
+/** How long a starting process will wait for a concurrent vec0 cleanup to finish. */
+const CLEANUP_LOCK_WAIT_MS = 15_000;
+const CLEANUP_LOCK_POLL_MS = 250;
+
+/**
+ * Check (read-only) whether the database still contains legacy vec0 virtual
+ * table entries. The destructive cleanup must only run when this is true —
+ * it rewrites sqlite_master via the sqlite3 CLI, which is unsafe while other
+ * connections hold the database open.
+ */
+function hasVec0Tables(dbPath: string): boolean {
+  let db: Database | null = null;
+  try {
+    db = new Database(dbPath, { readonly: true });
+    const row = db
+      .prepare("SELECT 1 FROM sqlite_master WHERE sql LIKE '%vec0%' LIMIT 1")
+      .get();
+    return row != null;
+  } catch {
+    // Unreadable / empty file — let the normal open path surface real errors
+    return false;
+  } finally {
+    db?.close();
+  }
+}
+
+function lockPid(lockPath: string): number | null {
+  try {
+    const pid = parseInt(readFileSync(lockPath, "utf8"), 10);
+    return Number.isFinite(pid) ? pid : null;
+  } catch {
+    return null;
+  }
+}
+
+function isProcessAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Run the vec0 cleanup under an exclusive advisory lock so that N processes
+ * starting against the same database never run the sqlite_master rewrite
+ * concurrently. Losers wait for the winner, then re-probe (the winner's
+ * cleanup makes the probe false).
+ */
+function guardedVec0Cleanup(dbPath: string): void {
+  const lockPath = `${dbPath}.vec0-cleanup.lock`;
+  const deadline = Date.now() + CLEANUP_LOCK_WAIT_MS;
+
+  while (true) {
+    try {
+      const fd = openSync(lockPath, "wx");
+      try {
+        writeSync(fd, String(process.pid));
+        // Re-probe under the lock — another process may have cleaned up
+        // between our first probe and lock acquisition.
+        if (hasVec0Tables(dbPath)) {
+          removeVec0Tables(dbPath);
+        }
+      } finally {
+        closeSync(fd);
+        try {
+          unlinkSync(lockPath);
+        } catch {
+          // already gone — fine
+        }
+      }
+      return;
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+
+      // Lock held — if the holder died, clear the stale lock and retry.
+      const pid = lockPid(lockPath);
+      if (pid !== null && !isProcessAlive(pid)) {
+        try {
+          unlinkSync(lockPath);
+        } catch {
+          // raced with another process clearing it — fine
+        }
+        continue;
+      }
+
+      if (Date.now() > deadline) {
+        throw new Error(
+          `vec0 cleanup lock held too long (${lockPath}) — remove it manually if no other server is starting`,
+        );
+      }
+      Bun.sleepSync(CLEANUP_LOCK_POLL_MS);
+    }
+  }
+}
+
 /**
  * Open (or create) a SQLite database at the given path
  * and run schema migrations.
+ *
+ * Safe for multiple concurrent processes sharing one database file:
+ * the legacy vec0 cleanup only runs when a read-only probe finds vec0
+ * entries (never for healthy databases) and is serialized by an exclusive
+ * lock; migrations are user_version-gated inside an immediate transaction.
+ */
+/**
+ * Legacy LanceDB installs used the db path as a *directory*
+ * (e.g. ~/.vector-memory/memories.db/memories.lance). SQLite needs a file
+ * there, so move the directory aside instead of dying with SQLITE_CANTOPEN.
+ * Returns the path the directory was moved to, or null if nothing was done.
  */
+export function relocateLegacyLanceDir(dbPath: string): string | null {
+  if (!existsSync(dbPath) || !statSync(dbPath).isDirectory()) return null;
+
+  const entries = readdirSync(dbPath);
+  const isLance = entries.some(
+    (e) => e.endsWith(".lance") || e === "_versions" || e === "_indices",
+  );
+  if (!isLance) {
+    throw new Error(
+      `Database path ${dbPath} is a directory, not a SQLite file. ` +
+        "Move or remove it, or point --db-file at a different location.",
+    );
+  }
+
+  let target = `${dbPath}.lancedb`;
+  for (let n = 1; existsSync(target); n++) {
+    target = `${dbPath}.lancedb.${n}`;
+  }
+  try {
+    renameSync(dbPath, target);
+  } catch (err) {
+    // A concurrently starting process won the rename — nothing left to move.
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw err;
+  }
+  console.error(
+    `[vector-memory-mcp] Found a legacy LanceDB store at ${dbPath} — ` +
+      `moved it to ${target}. A fresh SQLite database will be created.`,
+  );
+  return target;
+}
+
 export function connectToDatabase(dbPath: string): Database {
   mkdirSync(dirname(dbPath), { recursive: true });
+  relocateLegacyLanceDir(dbPath);
 
   // Remove orphaned vec0 virtual table entries before bun:sqlite opens the
   // database. bun:sqlite cannot modify sqlite_master, so this uses the
-  // sqlite3 CLI while no other connection holds a lock.
-  if (existsSync(dbPath)) {
-    removeVec0Tables(dbPath);
+  // sqlite3 CLI — gated behind a read-only probe and an exclusive lock.
+  if (existsSync(dbPath) && hasVec0Tables(dbPath)) {
+    guardedVec0Cleanup(dbPath);
   }
 
   const db = new Database(dbPath);
 
+  // busy_timeout FIRST: it is per-connection and needs no lock, while the
+  // WAL switch takes an exclusive lock — without the timeout, concurrent
+  // processes opening a fresh db race it and fail with SQLITE_BUSY.
+  db.exec("PRAGMA busy_timeout=5000");
   // WAL mode for concurrent read performance
   db.exec("PRAGMA journal_mode=WAL");