@aeriondyseti/vector-memory-mcp 2.5.0-dev.1 → 2.5.0-dev.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.5.0-dev.1",
+  "version": "2.5.0-dev.2",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "server/index.ts",
@@ -9,6 +9,8 @@
   },
   "files": [
     "server",
+    "scripts/lancedb-extract.ts",
+    "scripts/warmup.ts",
     "README.md",
     "LICENSE"
   ],

package/scripts/lancedb-extract.ts

@@ -0,0 +1,181 @@
+#!/usr/bin/env bun
+/**
+ * Standalone LanceDB data extractor — runs in a child process so that
+ * @lancedb/lancedb native bindings never coexist with bun:sqlite's
+ * extension loading in the same process.
+ *
+ * Usage: bun scripts/lancedb-extract.ts <lance-db-path>
+ * Output: JSON on stdout — { memories: Row[], conversations: Row[] }
+ */
+
+const source = process.argv[2];
+if (!source) {
+  console.error("Usage: bun scripts/lancedb-extract.ts <lance-db-path>");
+  process.exit(1);
+}
+
+// Arrow TimeUnit enum → divisor to convert to milliseconds.
+// 0=SECOND, 1=MILLISECOND, 2=MICROSECOND, 3=NANOSECOND
+// Negative divisor = multiply (seconds → ms needs ×1000).
+const TIME_UNIT_TO_MS_DIVISOR: Record<number, bigint> = {
+  0: -1000n,   // seconds → ms (multiply by 1000)
+  1: 1n,       // ms → no conversion
+  2: 1000n,    // μs → ms
+  3: 1000000n, // ns → ms
+};
+
+function buildTimestampDivisors(schema: any): Map<string, bigint> {
+  const map = new Map<string, bigint>();
+  for (const field of schema.fields) {
+    if (field.type.typeId === 10) {
+      map.set(field.name, TIME_UNIT_TO_MS_DIVISOR[field.type.unit] ?? 1n);
+    }
+  }
+  return map;
+}
+
+function columnValue(batch: any, colName: string, rowIdx: number): unknown {
+  const col = batch.getChild(colName);
+  if (!col) return undefined;
+  try {
+    return col.get(rowIdx);
+  } catch {
+    // Arrow's getter can throw on BigInt timestamps exceeding MAX_SAFE_INTEGER;
+    // fall back to the raw typed array.
+    let offset = rowIdx;
+    for (const data of col.data) {
+      if (offset < data.length) {
+        return (data.values instanceof BigInt64Array || data.values instanceof BigUint64Array)
+          ? data.values[offset]
+          : null;
+      }
+      offset -= data.length;
+    }
+    return null;
+  }
+}
+
+function toEpochMs(value: unknown, divisor: bigint = 1n): number {
+  if (value == null) return Date.now();
+  if (value instanceof Date) return value.getTime();
+  if (typeof value === "bigint") {
+    if (divisor < 0n) return Number(value * -divisor);  // seconds → ms
+    if (divisor === 1n) return Number(value);
+    return Number(value / divisor);
+  }
+  if (typeof value === "number") {
+    if (divisor < 0n) return value * Number(-divisor);
+    if (divisor === 1n) return value;
+    return Math.floor(value / Number(divisor));
+  }
+  return Date.now();
+}
+
+function toFloatArray(vec: unknown): number[] {
+  if (Array.isArray(vec)) return vec;
+  if (vec instanceof Float32Array) return Array.from(vec);
+  if (vec && typeof (vec as any).toArray === "function") {
+    return Array.from((vec as any).toArray());
+  }
+  if (ArrayBuffer.isView(vec)) {
+    const view = vec as DataView;
+    return Array.from(new Float32Array(view.buffer, view.byteOffset, view.byteLength / 4));
+  }
+  return [];
+}
+
+const BATCH_SIZE = 100;
+const lancedb = await import("@lancedb/lancedb");
+const db = await lancedb.connect(source);
+const tableNames = await db.tableNames();
+console.error(`Found tables: ${tableNames.join(", ")}`);
+
+const result: { memories: any[]; conversations: any[] } = {
+  memories: [],
+  conversations: [],
+};
+
+if (tableNames.includes("memories")) {
+  const table = await db.openTable("memories");
+  const total = await table.countRows();
+  console.error(`Reading ${total} memories...`);
+
+  // Paginated scan — query().toArrow() without offset/limit returns
+  // non-deterministic results that can duplicate some rows and skip others.
+  const schemaSample = await table.query().limit(1).toArrow();
+  const tsDivisors = buildTimestampDivisors(schemaSample.schema);
+  const seen = new Map<string, any>();
+
+  for (let offset = 0; offset < total; offset += BATCH_SIZE) {
+    const arrowTable = await table.query().offset(offset).limit(BATCH_SIZE).toArrow();
+    for (const batch of arrowTable.batches) {
+      for (let i = 0; i < batch.numRows; i++) {
+        const id = columnValue(batch, "id", i) as string;
+        const content = columnValue(batch, "content", i) as string;
+        const lastAccessed = columnValue(batch, "last_accessed", i);
+        const accessedMs = lastAccessed != null ? toEpochMs(lastAccessed, tsDivisors.get("last_accessed")) : null;
+        // Deduplicate by ID: prefer most recently accessed, then longest content.
+        const existing = seen.get(id);
+        if (existing) {
+          const existingAccess = existing.last_accessed ?? 0;
+          const newAccess = accessedMs ?? 0;
+          if (newAccess < existingAccess) continue;
+          if (newAccess === existingAccess && content.length <= existing.content.length) continue;
+        }
+        seen.set(id, {
+          id,
+          content,
+          metadata: columnValue(batch, "metadata", i) ?? "{}",
+          vector: toFloatArray(columnValue(batch, "vector", i)),
+          created_at: toEpochMs(columnValue(batch, "created_at", i), tsDivisors.get("created_at")),
+          updated_at: toEpochMs(columnValue(batch, "updated_at", i), tsDivisors.get("updated_at")),
+          last_accessed: accessedMs,
+          superseded_by: columnValue(batch, "superseded_by", i) ?? null,
+          usefulness: columnValue(batch, "usefulness", i) ?? 0,
+          access_count: columnValue(batch, "access_count", i) ?? 0,
+        });
+      }
+    }
+  }
+  result.memories = [...seen.values()];
+  console.error(`  ${result.memories.length} unique memories read (${total} rows scanned)`);
+}
+
+if (tableNames.includes("conversation_history")) {
+  const table = await db.openTable("conversation_history");
+  const total = await table.countRows();
+  console.error(`Reading ${total} conversation chunks...`);
+
+  const schemaSample = await table.query().limit(1).toArrow();
+  const tsDivisors = buildTimestampDivisors(schemaSample.schema);
+  const seen = new Map<string, any>();
+
+  for (let offset = 0; offset < total; offset += BATCH_SIZE) {
+    const arrowTable = await table.query().offset(offset).limit(BATCH_SIZE).toArrow();
+    for (const batch of arrowTable.batches) {
+      for (let i = 0; i < batch.numRows; i++) {
+        const id = columnValue(batch, "id", i) as string;
+        const content = columnValue(batch, "content", i) as string;
+        const existing = seen.get(id);
+        if (existing && existing.content.length >= content.length) continue;
+        seen.set(id, {
+          id,
+          content,
+          metadata: columnValue(batch, "metadata", i) ?? "{}",
+          vector: toFloatArray(columnValue(batch, "vector", i)),
+          created_at: toEpochMs(columnValue(batch, "created_at", i), tsDivisors.get("created_at")),
+          session_id: columnValue(batch, "session_id", i),
+          role: columnValue(batch, "role", i),
+          message_index_start: columnValue(batch, "message_index_start", i) ?? 0,
+          message_index_end: columnValue(batch, "message_index_end", i) ?? 0,
+          project: columnValue(batch, "project", i) ?? "",
+        });
+      }
+    }
+  }
+  result.conversations = [...seen.values()];
+  console.error(`  ${result.conversations.length} unique conversation chunks read (${total} rows scanned)`);
+}
+
+await db.close?.();
+process.stdout.write(JSON.stringify(result));

package/scripts/warmup.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env bun
+
+/**
+ * Warmup script to pre-download ML models and verify dependencies
+ * This runs during installation to ensure everything is ready to use
+ */
+
+import { config } from "../server/config/index";
+import { EmbeddingsService } from "../server/core/embeddings.service";
+
+async function warmup(): Promise<void> {
+  console.log("🔥 Warming up vector-memory-mcp...");
+  console.log();
+
+  try {
+    // Check native dependencies
+    console.log("✓ Checking native dependencies...");
+    try {
+      await import("onnxruntime-node");
+      console.log("  ✓ onnxruntime-node loaded");
+    } catch (e) {
+      console.error("  ✗ onnxruntime-node failed:", (e as Error).message);
+      process.exit(1);
+    }
+
+    console.log();
+
+    // Initialize embeddings service to download model
+    console.log("📥 Downloading ML model (this may take a minute)...");
+    console.log(`   Model: ${config.embeddingModel}`);
+    console.log();
+
+    const embeddings = new EmbeddingsService(
+      config.embeddingModel,
+      config.embeddingDimension
+    );
+
+    // Trigger model download by generating a test embedding
+    const startTime = Date.now();
+    await embeddings.embed("warmup test");
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+
+    console.log();
+    console.log(`✅ Warmup complete! (${duration}s)`);
+    console.log();
+    console.log("Ready to use! Configure your MCP client and restart to get started.");
+    console.log();
+  } catch (error) {
+    console.error();
+    console.error("❌ Warmup failed:", error);
+    console.error();
+    console.error("This is not a critical error - the server will download models on first run.");
+    console.error("You can try running 'vector-memory-mcp warmup' manually later.");
+    process.exit(0); // Exit successfully to not block installation
+  }
+}
+
+// Only run if this is the main module
+if (import.meta.url === `file://${process.argv[1]}`) {
+  warmup();
+}
+
+export { warmup };

package/server/core/connection.ts

@@ -1,5 +1,16 @@
 import { Database } from "bun:sqlite";
-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, unlinkSync, writeSync } from "fs";
+import {
+  closeSync,
+  existsSync,
+  mkdirSync,
+  openSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeSync,
+} from "fs";
 import { dirname } from "path";
 import { removeVec0Tables, runMigrations } from "./migrations";
 
@@ -109,8 +120,47 @@ function guardedVec0Cleanup(dbPath: string): void {
  * entries (never for healthy databases) and is serialized by an exclusive
  * lock; migrations are user_version-gated inside an immediate transaction.
  */
+/**
+ * Legacy LanceDB installs used the db path as a *directory*
+ * (e.g. ~/.vector-memory/memories.db/memories.lance). SQLite needs a file
+ * there, so move the directory aside instead of dying with SQLITE_CANTOPEN.
+ * Returns the path the directory was moved to, or null if nothing was done.
+ */
+export function relocateLegacyLanceDir(dbPath: string): string | null {
+  if (!existsSync(dbPath) || !statSync(dbPath).isDirectory()) return null;
+
+  const entries = readdirSync(dbPath);
+  const isLance = entries.some(
+    (e) => e.endsWith(".lance") || e === "_versions" || e === "_indices",
+  );
+  if (!isLance) {
+    throw new Error(
+      `Database path ${dbPath} is a directory, not a SQLite file. ` +
+        "Move or remove it, or point --db-file at a different location.",
+    );
+  }
+
+  let target = `${dbPath}.lancedb`;
+  for (let n = 1; existsSync(target); n++) {
+    target = `${dbPath}.lancedb.${n}`;
+  }
+  try {
+    renameSync(dbPath, target);
+  } catch (err) {
+    // A concurrently starting process won the rename — nothing left to move.
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw err;
+  }
+  console.error(
+    `[vector-memory-mcp] Found a legacy LanceDB store at ${dbPath} — ` +
+      `moved it to ${target}. A fresh SQLite database will be created.`,
+  );
+  return target;
+}
+
 export function connectToDatabase(dbPath: string): Database {
   mkdirSync(dirname(dbPath), { recursive: true });
+  relocateLegacyLanceDir(dbPath);
 
   // Remove orphaned vec0 virtual table entries before bun:sqlite opens the
   // database. bun:sqlite cannot modify sqlite_master, so this uses the

package/server/core/consolidation.service.ts

@@ -1,9 +1,17 @@
 import { Database } from "bun:sqlite";
 import { createHash, randomUUID } from "crypto";
-import { copyFileSync, existsSync, readdirSync, readFileSync, renameSync } from "fs";
+import {
+  copyFileSync,
+  existsSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+} from "fs";
 import { readFile } from "fs/promises";
 import { homedir } from "os";
-import { dirname, join } from "path";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
 import type { EmbeddingsService } from "./embeddings.service";
 import { normalizeProject } from "./project";
 import { safeParseJsonObject, serializeVector } from "./sqlite-utils";
@@ -58,6 +66,70 @@ interface SourceMemoryRow {
   vector: Buffer | null;
 }
 
+interface SourceConversationRow {
+  id: string;
+  content: string;
+  metadata: string;
+  created_at: number;
+  session_id: string;
+  role: string;
+  message_index_start: number;
+  message_index_end: number;
+  project: string;
+  vector: Buffer | null;
+}
+
+/**
+ * LanceDB-era repo stores used `.vector-memory/memories.db` as a *directory*.
+ * Extraction shells out because @lancedb/lancedb's native bindings cannot
+ * coexist with bun:sqlite in one process.
+ */
+function isLanceDir(entries: string[]): boolean {
+  return entries.some(
+    (e) => e.endsWith(".lance") || e === "_versions" || e === "_indices",
+  );
+}
+
+async function extractLanceData(path: string): Promise<{
+  memories: Array<Omit<SourceMemoryRow, "vector"> & { vector: number[] }>;
+  conversations: Array<
+    Omit<SourceConversationRow, "vector"> & { vector: number[] }
+  >;
+}> {
+  const script = resolve(
+    dirname(fileURLToPath(import.meta.url)),
+    "..",
+    "..",
+    "scripts",
+    "lancedb-extract.ts",
+  );
+  if (!existsSync(script)) {
+    throw new Error(`LanceDB extract script not found at ${script}`);
+  }
+  const proc = Bun.spawn([process.execPath, script, path], {
+    stdout: "pipe",
+    stderr: "inherit",
+  });
+  const output = await new Response(proc.stdout).text();
+  const exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    throw new Error(`LanceDB extraction failed (exit code ${exitCode})`);
+  }
+  return JSON.parse(output);
+}
+
+/**
+ * Old schema versions stored vectors in vec0 virtual tables, which need the
+ * sqlite-vec extension to query. Sources are opened read-only without it, so
+ * treat those vectors as unreadable — rows are re-embedded on import.
+ */
+function vecTableReadable(db: Database, name: string): boolean {
+  const row = db
+    .prepare("SELECT sql FROM sqlite_master WHERE name = ?")
+    .get(name) as { sql: string | null } | null;
+  return row != null && !(row.sql ?? "").includes("vec0");
+}
+
 /** Mirrors MemoryService.waypointId — must stay byte-identical. */
 function waypointIdFor(project: string): string {
   const normalized = project.trim().toLowerCase();
@@ -219,6 +291,18 @@ export class ConsolidationService {
       errors: [],
     };
 
+    // LanceDB-era stores are directories, not SQLite files
+    if (statSync(sourceDbPath).isDirectory()) {
+      await this.consolidateLanceSource(
+        sourceDbPath,
+        project,
+        importBatch,
+        options,
+        report,
+      );
+      return report;
+    }
+
     let source: Database;
     try {
       source = new Database(sourceDbPath, { readonly: true });
@@ -242,6 +326,68 @@ export class ConsolidationService {
     return report;
   }
 
+  private async consolidateLanceSource(
+    sourceDbPath: string,
+    project: string,
+    importBatch: string,
+    options: ConsolidationOptions,
+    report: SourceReport,
+  ): Promise<void> {
+    const entries = readdirSync(sourceDbPath);
+    if (entries.length === 0) return; // failed init left an empty dir — nothing to import
+    if (!isLanceDir(entries)) {
+      report.errors.push(
+        `source is a directory but not a LanceDB store: ${sourceDbPath}`,
+      );
+      return;
+    }
+
+    let data: Awaited<ReturnType<typeof extractLanceData>>;
+    try {
+      data = await extractLanceData(sourceDbPath);
+    } catch (e) {
+      report.errors.push(
+        `LanceDB extraction failed: ${e instanceof Error ? e.message : String(e)}`,
+      );
+      return;
+    }
+
+    const expectedBytes = this.embeddings.dimension * 4;
+    const toBuffer = (vector: number[]): Buffer | null => {
+      const buf = vector.length > 0 ? serializeVector(vector) : null;
+      // Wrong-dimension vectors (model change) are dropped → re-embedded
+      return buf && buf.byteLength === expectedBytes ? buf : null;
+    };
+
+    const memoryRows: SourceMemoryRow[] = data.memories.map((m) => ({
+      ...m,
+      vector: toBuffer(m.vector),
+    }));
+    const conversationRows: SourceConversationRow[] = data.conversations.map(
+      (c) => ({ ...c, vector: toBuffer(c.vector) }),
+    );
+
+    try {
+      await this.processMemoryRows(
+        memoryRows,
+        project,
+        importBatch,
+        options,
+        report,
+      );
+      this.processConversationRows(
+        conversationRows,
+        project,
+        importBatch,
+        options,
+        report,
+      );
+      await this.importIndexState(null, sourceDbPath, project, options, report);
+    } catch (e) {
+      report.errors.push(e instanceof Error ? e.message : String(e));
+    }
+  }
+
   // ── Memories ────────────────────────────────────────────────────────
 
   private async importMemories(
@@ -253,12 +399,25 @@ export class ConsolidationService {
   ): Promise<void> {
     if (!tableExists(source, "memories")) return;
 
-    const rows = source
-      .prepare(
-        `SELECT m.*, v.vector FROM memories m
-         LEFT JOIN memories_vec v ON m.id = v.id`,
-      )
-      .all() as SourceMemoryRow[];
+    const rows = (
+      vecTableReadable(source, "memories_vec")
+        ? source.prepare(
+            `SELECT m.*, v.vector FROM memories m
+             LEFT JOIN memories_vec v ON m.id = v.id`,
+          )
+        : source.prepare("SELECT m.*, NULL AS vector FROM memories m")
+    ).all() as SourceMemoryRow[];
+
+    await this.processMemoryRows(rows, project, importBatch, options, report);
+  }
+
+  private async processMemoryRows(
+    rows: SourceMemoryRow[],
+    project: string,
+    importBatch: string,
+    options: ConsolidationOptions,
+    report: SourceReport,
+  ): Promise<void> {
     if (rows.length === 0) return;
 
     const targetGet = this.target.prepare(
@@ -316,6 +475,12 @@ export class ConsolidationService {
       }
     }
 
+    if (options.dryRun) {
+      report.memoriesImported = toImport.length;
+      this.collectUnresolved(rows, sourceIds, report);
+      return;
+    }
+
     // Pre-compute embeddings for rows whose vectors are missing or have the
     // wrong dimension (model change) — outside any transaction.
     const expectedBytes = this.embeddings.dimension * 4;
@@ -329,12 +494,6 @@ export class ConsolidationService {
       new Array(this.embeddings.dimension).fill(0),
     );
 
-    if (options.dryRun) {
-      report.memoriesImported = toImport.length;
-      this.collectUnresolved(rows, sourceIds, report);
-      return;
-    }
-
     const insertMain = this.target.prepare(
       `INSERT INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed, project)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
@@ -472,23 +631,27 @@ export class ConsolidationService {
   ): void {
     if (!tableExists(source, "conversation_history")) return;
 
-    const rows = source
-      .prepare(
-        `SELECT c.*, v.vector FROM conversation_history c
-         LEFT JOIN conversation_history_vec v ON c.id = v.id`,
-      )
-      .all() as Array<{
-      id: string;
-      content: string;
-      metadata: string;
-      created_at: number;
-      session_id: string;
-      role: string;
-      message_index_start: number;
-      message_index_end: number;
-      project: string;
-      vector: Buffer | null;
-    }>;
+    const rows = (
+      vecTableReadable(source, "conversation_history_vec")
+        ? source.prepare(
+            `SELECT c.*, v.vector FROM conversation_history c
+             LEFT JOIN conversation_history_vec v ON c.id = v.id`,
+          )
+        : source.prepare(
+            "SELECT c.*, NULL AS vector FROM conversation_history c",
+          )
+    ).all() as SourceConversationRow[];
+
+    this.processConversationRows(rows, project, importBatch, options, report);
+  }
+
+  private processConversationRows(
+    rows: SourceConversationRow[],
+    project: string,
+    importBatch: string,
+    options: ConsolidationOptions,
+    report: SourceReport,
+  ): void {
     if (rows.length === 0) return;
 
     const existsStmt = this.target.prepare(
@@ -550,7 +713,7 @@ export class ConsolidationService {
   // ── Conversation index state ────────────────────────────────────────
 
   private async importIndexState(
-    source: Database,
+    source: Database | null,
     sourceDbPath: string,
     project: string,
     options: ConsolidationOptions,
@@ -569,7 +732,7 @@ export class ConsolidationService {
     };
 
     const entries: StateRow[] = [];
-    if (tableExists(source, "conversation_index_state")) {
+    if (source && tableExists(source, "conversation_index_state")) {
       entries.push(
         ...(source
           .prepare("SELECT * FROM conversation_index_state")