@aeriondyseti/vector-memory-mcp 2.3.0-rc.1 → 2.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.3.0-rc.1",
+  "version": "2.3.0",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "server/index.ts",

package/server/core/migrations.ts

@@ -1,4 +1,6 @@
 import type { Database } from "bun:sqlite";
+import type { EmbeddingsService } from "./embeddings.service.js";
+import { serializeVector } from "./sqlite-utils.js";
 
 /**
  * Pre-migration step: remove vec0 virtual table entries from sqlite_master
@@ -113,3 +115,91 @@ export function runMigrations(db: Database): void {
   db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_role ON conversation_history(role)`);
   db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_created_at ON conversation_history(created_at)`);
 }
+
+/**
+ * Backfill missing vectors in memories_vec and conversation_history_vec.
+ *
+ * After the vec0-to-BLOB migration, existing rows may lack vector embeddings.
+ * This re-embeds their content and inserts into the _vec tables.
+ * Idempotent: skips rows that already have vectors. Fast no-op when fully backfilled.
+ */
+export async function backfillVectors(
+  db: Database,
+  embeddings: EmbeddingsService,
+): Promise<void> {
+  // ── Memories ──────────────────────────────────────────────────────
+  // Catch both missing rows (v.id IS NULL) and corrupt 0-byte BLOBs
+  const missingMemories = db
+    .prepare(
+      `SELECT m.id, m.content, json_extract(m.metadata, '$.type') AS type
+       FROM memories m
+       LEFT JOIN memories_vec v ON m.id = v.id
+       WHERE v.id IS NULL OR length(v.vector) = 0`,
+    )
+    .all() as Array<{ id: string; content: string; type: string | null }>;
+
+  if (missingMemories.length > 0) {
+    console.error(
+      `[vector-memory-mcp] Backfilling vectors for ${missingMemories.length} memories...`,
+    );
+
+    const insertVec = db.prepare(
+      "INSERT OR REPLACE INTO memories_vec (id, vector) VALUES (?, ?)",
+    );
+
+    const zeroVector = serializeVector(
+      new Array(embeddings.dimension).fill(0),
+    );
+
+    for (const row of missingMemories) {
+      // Waypoints use a zero vector (not semantically searched)
+      const blob =
+        row.type === "waypoint"
+          ? zeroVector
+          : serializeVector(await embeddings.embed(row.content));
+
+      insertVec.run(row.id, blob);
+    }
+
+    console.error(
+      `[vector-memory-mcp] Backfilled ${missingMemories.length} memory vectors`,
+    );
+  }
+
+  // ── Conversation history ──────────────────────────────────────────
+  const missingConvos = db
+    .prepare(
+      `SELECT c.id, c.content
+       FROM conversation_history c
+       LEFT JOIN conversation_history_vec v ON c.id = v.id
+       WHERE v.id IS NULL OR length(v.vector) = 0`,
+    )
+    .all() as Array<{ id: string; content: string }>;
+
+  if (missingConvos.length > 0) {
+    console.error(
+      `[vector-memory-mcp] Backfilling vectors for ${missingConvos.length} conversation chunks...`,
+    );
+
+    const insertConvoVec = db.prepare(
+      "INSERT OR REPLACE INTO conversation_history_vec (id, vector) VALUES (?, ?)",
+    );
+
+    for (let i = 0; i < missingConvos.length; i++) {
+      const row = missingConvos[i];
+      const vec = serializeVector(await embeddings.embed(row.content));
+      insertConvoVec.run(row.id, vec);
+
+      // Log progress every 100 chunks
+      if ((i + 1) % 100 === 0) {
+        console.error(
+          `[vector-memory-mcp]   ...${i + 1}/${missingConvos.length} conversation chunks`,
+        );
+      }
+    }
+
+    console.error(
+      `[vector-memory-mcp] Backfilled ${missingConvos.length} conversation vectors`,
+    );
+  }
+}

package/server/index.ts

@@ -2,6 +2,7 @@
 
 import { loadConfig, parseCliArgs } from "./config/index.js";
 import { connectToDatabase } from "./core/connection.js";
+import { backfillVectors } from "./core/migrations.js";
 import { MemoryRepository } from "./core/memory.repository.js";
 import { ConversationRepository } from "./core/conversation.repository.js";
 import { EmbeddingsService } from "./core/embeddings.service.js";
@@ -9,26 +10,6 @@ import { MemoryService } from "./core/memory.service.js";
 import { ConversationHistoryService } from "./core/conversation.service.js";
 import { startServer } from "./transports/mcp/server.js";
 import { startHttpServer } from "./transports/http/server.js";
-import { isLanceDbDirectory, migrate, formatMigrationSummary } from "./migration.js";
-
-async function runMigrate(args: string[]): Promise<void> {
-  const overrides = parseCliArgs(args.slice(1)); // skip "migrate"
-  const config = loadConfig(overrides);
-
-  const source = config.dbPath;
-  const target = source.endsWith(".sqlite") ? source.replace(/\.sqlite$/, "-migrated.sqlite") : source + ".sqlite";
-
-  if (!isLanceDbDirectory(source)) {
-    console.error(
-      `[vector-memory-mcp] No LanceDB data found at ${source}\n` +
-      `  Nothing to migrate. The server will create a fresh SQLite database on startup.`
-    );
-    return;
-  }
-
-  const result = await migrate({ source, target });
-  console.error(formatMigrationSummary(source, target, result));
-}
 
 async function main(): Promise<void> {
   const args = process.argv.slice(2);
@@ -40,27 +21,10 @@ async function main(): Promise<void> {
     return;
   }
 
-  // Check for migrate command
-  if (args[0] === "migrate") {
-    await runMigrate(args);
-    return;
-  }
-
   // Parse CLI args and load config
   const overrides = parseCliArgs(args);
   const config = loadConfig(overrides);
 
-  // Detect legacy LanceDB data and warn
-  if (isLanceDbDirectory(config.dbPath)) {
-    console.error(
-      `[vector-memory-mcp] ⚠️  Legacy LanceDB data detected at ${config.dbPath}\n` +
-      `  Your data must be migrated to the new SQLite format.\n` +
-      `  Run: vector-memory-mcp migrate\n` +
-      `  Or:  bun run server/index.ts migrate\n`
-    );
-    process.exit(1);
-  }
-
   // Initialize database
   const db = connectToDatabase(config.dbPath);
 
@@ -69,6 +33,9 @@ async function main(): Promise<void> {
   const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
   const memoryService = new MemoryService(repository, embeddings);
 
+  // Backfill any missing vectors (e.g. after vec0-to-BLOB migration)
+  await backfillVectors(db, embeddings);
+
   if (config.pluginMode) {
     console.error("[vector-memory-mcp] Running in plugin mode");
   }

package/server/transports/http/server.ts

@@ -8,7 +8,7 @@ import type { Config } from "../../config/index.js";
 import { isDeleted } from "../../core/memory.js";
 import { createMcpRoutes } from "./mcp-transport.js";
 import type { Memory, SearchIntent } from "../../core/memory.js";
-import { MigrationService } from "../../core/migration.service.js";
+
 
 /**
  * Check if a port is available by attempting to bind to it
@@ -245,34 +245,6 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
     }
   });
 
-  // Migrate from external memory database
-  app.post("/migrate", async (c) => {
-    try {
-      const body = await c.req.json().catch(() => null);
-      if (!body || typeof body !== "object") {
-        return c.json({ error: "Invalid or missing JSON body" }, 400);
-      }
-      const source = body.source;
-
-      if (!source || typeof source !== "string") {
-        return c.json({ error: "Missing or invalid 'source' field" }, 400);
-      }
-
-      const repository = memoryService.getRepository();
-      const migrationService = new MigrationService(
-        repository,
-        memoryService.getEmbeddings(),
-        repository.getDb(),
-      );
-
-      const result = await migrationService.migrate(source);
-      return c.json(result);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      return c.json({ error: message }, 500);
-    }
-  });
-
   // Get single memory
   app.get("/memories/:id", async (c) => {
     try {

package/server/transports/mcp/resources.ts

@@ -1,152 +1,11 @@
-const MIGRATE_GUIDE = `# Migrating External Memory Databases
-
-The vector-memory-mcp server exposes a \`POST /migrate\` HTTP endpoint that imports
-memories from other database formats into the running instance. All imported
-content is re-embedded with the server's current embedding model to guarantee
-consistency.
-
-## Endpoint
-
-\`\`\`
-POST http://<host>:<port>/migrate
-Content-Type: application/json
-
-{ "source": "/absolute/path/to/source/database" }
-\`\`\`
-
-## Discovering the Server Port
-
-The HTTP server writes a lockfile at \`.vector-memory/server.lock\` in the
-project's working directory. Read it to discover the current port:
-
-\`\`\`json
-{ "port": 3271, "pid": 12345 }
-\`\`\`
-
-## Supported Source Formats
-
-The endpoint auto-detects the source format from the path provided.
-
-### 1. LanceDB Directory
-Provide the path to a LanceDB data directory (contains \`.lance\` files or
-\`_versions\`/\`_indices\` subdirectories). Both memories and conversation
-history are imported.
-
-\`\`\`json
-{ "source": "/path/to/project/.vector-memory" }
-\`\`\`
-
-### 2. Own SQLite (Current or Older Schema)
-Provide the path to a \`.db\` file that was created by any version of
-vector-memory-mcp. The migrator handles missing columns (e.g. \`usefulness\`,
-\`access_count\`) by using sensible defaults. Both memories and conversation
-history are imported.
-
-\`\`\`json
-{ "source": "/path/to/old-project/.vector-memory/memories.db" }
-\`\`\`
-
-### 3. CCCMemory SQLite
-Provide the path to a CCCMemory database. The migrator extracts from the
-\`decisions\`, \`mistakes\`, \`methodologies\`, \`research_findings\`,
-\`solution_patterns\`, and \`working_memory\` tables. Each record is tagged
-with \`source_type: "cccmemory"\` and the appropriate \`memory_type\` in
-metadata.
-
-\`\`\`json
-{ "source": "/path/to/cccmemory.db" }
-\`\`\`
-
-### 4. MCP Memory Service SQLite
-Provide the path to an mcp-memory-service database. Memories with
-\`deleted_at IS NULL\` are imported. Tags and memory type are preserved in
-metadata.
-
-\`\`\`json
-{ "source": "/path/to/mcp-memory-service.db" }
-\`\`\`
-
-### 5. MIF JSON (Shodh Memory Interchange Format)
-Provide the path to a \`.json\` file exported from Shodh Memory. The file must
-contain a top-level \`memories\` array. Memory type, tags, entities, and source
-metadata are preserved.
-
-\`\`\`json
-{ "source": "/path/to/export.mif.json" }
-\`\`\`
-
-## Response
-
-The endpoint returns a JSON summary upon completion:
-
-\`\`\`json
-{
-  "source": "/path/to/source",
-  "format": "own-sqlite",
-  "memoriesImported": 142,
-  "memoriesSkipped": 3,
-  "conversationsImported": 0,
-  "conversationsSkipped": 0,
-  "errors": [],
-  "durationMs": 8320
-}
-\`\`\`
-
-- **memoriesImported**: Number of new memories written to the database.
-- **memoriesSkipped**: Records skipped because a memory with the same ID
-  already exists (safe for idempotent re-runs).
-- **conversationsImported / conversationsSkipped**: Same, for conversation
-  history chunks (LanceDB and own-sqlite formats only).
-- **errors**: Per-record errors that did not abort the migration.
-- **durationMs**: Wall-clock time for the entire operation.
-
-## Important Notes
-
-- **Re-embedding**: All content is re-embedded regardless of the source format.
-  This ensures vector consistency with the server's current model but means the
-  operation can take time for large databases (~50ms per record).
-- **Idempotent**: Running the same migration twice is safe. Duplicate IDs are
-  skipped.
-- **Non-destructive**: The source database is opened read-only and is never
-  modified.
-- **Batched writes**: Records are inserted in batches of 100 within
-  transactions. If the process is interrupted, already-committed batches are
-  durable.
-- **Error isolation**: A single bad record does not abort the migration. Check
-  the \`errors\` array in the response for any per-record failures.
-
-## Workflow Example
-
-1. Locate the source database file or directory.
-2. Read \`.vector-memory/server.lock\` to get the port.
-3. Send the migrate request:
-   \`\`\`bash
-   curl -X POST http://127.0.0.1:3271/migrate \\
-     -H "Content-Type: application/json" \\
-     -d '{"source": "/path/to/old/memories.db"}'
-   \`\`\`
-4. Inspect the response summary.
-5. Verify imported memories with a search:
-   \`\`\`bash
-   curl -X POST http://127.0.0.1:3271/search \\
-     -H "Content-Type: application/json" \\
-     -d '{"query": "test query", "limit": 5}'
-   \`\`\`
-`;
-
-export const resources = [
-  {
-    uri: "vector-memory://guides/migrate",
-    name: "Migration Guide",
-    description:
-      "How to use the POST /migrate HTTP endpoint to import memories from external database formats (LanceDB, older SQLite, CCCMemory, MCP Memory Service, MIF JSON) into the running vector-memory instance.",
-    mimeType: "text/markdown",
-  },
-];
-
-const RESOURCE_CONTENT: Record<string, string> = {
-  "vector-memory://guides/migrate": MIGRATE_GUIDE,
-};
+export const resources: Array<{
+  uri: string;
+  name: string;
+  description: string;
+  mimeType: string;
+}> = [];
+
+const RESOURCE_CONTENT: Record<string, string> = {};
 
 export function readResource(uri: string): {
   contents: Array<{ uri: string; mimeType: string; text: string }>;

package/scripts/migrate-from-lancedb.ts

@@ -1,56 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Standalone migration script: LanceDB → SQLite (sqlite-vec)
- *
- * This is a thin wrapper around server/migration.ts for direct invocation.
- * The preferred way to migrate is `vector-memory-mcp migrate`.
- *
- * Usage:
- *   bun scripts/migrate-from-lancedb.ts [--source <lancedb-dir>] [--target <sqlite-file>]
- *
- * Defaults:
- *   --source  .vector-memory/memories.db        (the old LanceDB directory)
- *   --target  .vector-memory/memories.db.sqlite  (new SQLite file)
- *
- * @deprecated Use `vector-memory-mcp migrate` instead. This script will be
- *   removed in the next major version.
- */
-
-import { migrate, formatMigrationSummary } from "../server/migration.js";
-
-function parseArgs(): { source: string; target: string } {
-  const args = process.argv.slice(2);
-  let source = ".vector-memory/memories.db";
-  let target = ".vector-memory/memories.db.sqlite";
-
-  for (let i = 0; i < args.length; i++) {
-    if (args[i] === "--source" && args[i + 1]) source = args[++i];
-    else if (args[i] === "--target" && args[i + 1]) target = args[++i];
-    else if (args[i] === "--help" || args[i] === "-h") {
-      console.log(`
-Usage: bun scripts/migrate-from-lancedb.ts [options]
-
-  Prefer: vector-memory-mcp migrate
-
-Options:
-  --source <path>  LanceDB directory (default: .vector-memory/memories.db)
-  --target <path>  SQLite output file (default: .vector-memory/memories.db.sqlite)
-  --help           Show this help
-`);
-      process.exit(0);
-    }
-  }
-
-  return { source, target };
-}
-
-async function main() {
-  const { source, target } = parseArgs();
-  const result = await migrate({ source, target });
-  console.error(formatMigrationSummary(source, target, result));
-}
-
-main().catch((err) => {
-  console.error("❌ Migration failed:", err.message ?? err);
-  process.exit(1);
-});

package/server/migration.ts

@@ -1,203 +0,0 @@
-/**
- * LanceDB -> SQLite migration logic.
- *
- * Reads LanceDB data in a child process (scripts/lancedb-extract.ts) to avoid
- * a native symbol collision between @lancedb/lancedb and bun:sqlite.
- * The extracted JSON is then written to SQLite in-process.
- *
- * @deprecated Will be removed in the next major version once LanceDB
- *   support is dropped.
- */
-
-import { existsSync, statSync } from "fs";
-import { resolve, dirname } from "path";
-import { fileURLToPath } from "url";
-import { connectToDatabase } from "./core/connection.js";
-import { serializeVector } from "./core/sqlite-utils.js";
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-// ── Detection ───────────────────────────────────────────────────────
-
-export function isLanceDbDirectory(dbPath: string): boolean {
-  return existsSync(dbPath) && statSync(dbPath).isDirectory();
-}
-
-// ── Types ───────────────────────────────────────────────────────────
-
-export interface MigrateOptions {
-  source: string;
-  target: string;
-}
-
-export interface MigrateResult {
-  memoriesMigrated: number;
-  conversationChunksMigrated: number;
-  outputSizeMB: string;
-}
-
-interface ExtractedData {
-  memories: Array<{
-    id: string;
-    content: string;
-    metadata: string;
-    vector: number[];
-    created_at: number;
-    updated_at: number;
-    last_accessed: number | null;
-    superseded_by: string | null;
-    usefulness: number;
-    access_count: number;
-  }>;
-  conversations: Array<{
-    id: string;
-    content: string;
-    metadata: string;
-    vector: number[];
-    created_at: number;
-    session_id: string;
-    role: string;
-    message_index_start: number;
-    message_index_end: number;
-    project: string;
-  }>;
-}
-
-// ── Migration ───────────────────────────────────────────────────────
-
-export async function migrate(opts: MigrateOptions): Promise<MigrateResult> {
-  const { source, target } = opts;
-
-  if (!existsSync(source)) {
-    throw new Error(`Source not found: ${source}`);
-  }
-  if (!statSync(source).isDirectory()) {
-    throw new Error(`Source is not a directory (expected LanceDB): ${source}`);
-  }
-  if (existsSync(target)) {
-    throw new Error(
-      `Target already exists: ${target}\n   Delete it first or choose a different target path.`
-    );
-  }
-
-  console.error(`📂 Source (LanceDB): ${source}`);
-  console.error(`📄 Target (SQLite):  ${target}`);
-  console.error();
-
-  // Phase 1: Extract data from LanceDB in a subprocess.
-  // This avoids a native symbol collision between @lancedb/lancedb and bun:sqlite.
-  const extractScript = resolve(__dirname, "..", "scripts", "lancedb-extract.ts");
-  const proc = Bun.spawn(["bun", extractScript, source], {
-    stdout: "pipe",
-    stderr: "inherit",
-  });
-
-  const output = await new Response(proc.stdout).text();
-  const exitCode = await proc.exited;
-
-  if (exitCode !== 0) {
-    throw new Error(`LanceDB extraction failed (exit code ${exitCode})`);
-  }
-
-  const data: ExtractedData = JSON.parse(output);
-
-  // Phase 2: Write to SQLite (no LanceDB in this process).
-  const sqliteDb = connectToDatabase(target);
-
-  let memoriesMigrated = 0;
-  let conversationChunksMigrated = 0;
-
-  if (data.memories.length > 0) {
-    console.error(`\n🧠 Writing ${data.memories.length} memories to SQLite...`);
-
-    const insertMain = sqliteDb.prepare(
-      `INSERT OR REPLACE INTO memories
-        (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
-    );
-    const deleteVec = sqliteDb.prepare(`DELETE FROM memories_vec WHERE id = ?`);
-    const insertVec = sqliteDb.prepare(
-      `INSERT INTO memories_vec (id, vector) VALUES (?, ?)`
-    );
-    const insertFts = sqliteDb.prepare(
-      `INSERT OR REPLACE INTO memories_fts (id, content) VALUES (?, ?)`
-    );
-
-    const tx = sqliteDb.transaction(() => {
-      for (const row of data.memories) {
-        insertMain.run(
-          row.id, row.content, row.metadata,
-          row.created_at, row.updated_at,
-          row.superseded_by, row.usefulness,
-          row.access_count, row.last_accessed,
-        );
-        if (row.vector.length > 0) {
-          deleteVec.run(row.id);
-          insertVec.run(row.id, serializeVector(row.vector));
-        }
-        insertFts.run(row.id, row.content);
-      }
-    });
-    tx();
-    memoriesMigrated = data.memories.length;
-    console.error(`   ✅ ${memoriesMigrated} memories migrated`);
-  }
-
-  if (data.conversations.length > 0) {
-    console.error(`\n💬 Writing ${data.conversations.length} conversation chunks to SQLite...`);
-
-    const insertMain = sqliteDb.prepare(
-      `INSERT OR REPLACE INTO conversation_history
-        (id, content, metadata, created_at, session_id, role, message_index_start, message_index_end, project)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
-    );
-    const deleteVec = sqliteDb.prepare(`DELETE FROM conversation_history_vec WHERE id = ?`);
-    const insertVec = sqliteDb.prepare(
-      `INSERT INTO conversation_history_vec (id, vector) VALUES (?, ?)`
-    );
-    const insertFts = sqliteDb.prepare(
-      `INSERT OR REPLACE INTO conversation_history_fts (id, content) VALUES (?, ?)`
-    );
-
-    const tx = sqliteDb.transaction(() => {
-      for (const row of data.conversations) {
-        insertMain.run(
-          row.id, row.content, row.metadata,
-          row.created_at, row.session_id, row.role,
-          row.message_index_start, row.message_index_end, row.project,
-        );
-        if (row.vector.length > 0) {
-          deleteVec.run(row.id);
-          insertVec.run(row.id, serializeVector(row.vector));
-        }
-        insertFts.run(row.id, row.content);
-      }
-    });
-    tx();
-    conversationChunksMigrated = data.conversations.length;
-    console.error(`   ✅ ${conversationChunksMigrated} conversation chunks migrated`);
-  }
-
-  sqliteDb.close();
-
-  const { size } = statSync(target);
-  const outputSizeMB = (size / 1024 / 1024).toFixed(2);
-
-  return { memoriesMigrated, conversationChunksMigrated, outputSizeMB };
-}
-
-export function formatMigrationSummary(
-  source: string,
-  target: string,
-  result: MigrateResult,
-): string {
-  return `
-✅ Migration complete! (${result.outputSizeMB} MB)
-   ${result.memoriesMigrated} memories, ${result.conversationChunksMigrated} conversation chunks
-
-Next steps:
-   1. Backup:   mv "${source}" "${source}.lance-backup"
-   2. Activate: mv "${target}" "${source}"
-   3. Restart your MCP server
-`;
-}