@aeriondyseti/vector-memory-mcp 1.1.0-dev.6 → 2.0.0-rc

package/src/db/memory.repository.ts

@@ -1,7 +1,12 @@
-import * as lancedb from "@lancedb/lancedb";
-import { type Table } from "@lancedb/lancedb";
-import { TABLE_NAME, memorySchema } from "./schema.js";
-import { arrowTimestampToDate, arrowVectorToArray, createFtsMutex, createRerankerMutex, escapeSql, safeParseJsonObject } from "./lancedb-utils.js";
+import type { Database } from "bun:sqlite";
+import {
+  serializeVector,
+  deserializeVector,
+  safeParseJsonObject,
+  sanitizeFtsQuery,
+  hybridRRF,
+  topByRRF,
+} from "./sqlite-utils.js";
 import {
   type Memory,
   type HybridRow,
@@ -9,206 +14,213 @@ import {
 } from "../types/memory.js";
 
 export class MemoryRepository {
-  // Mutex for schema migration - runs once per instance to add missing columns
-  private migrationPromise: Promise<void> | null = null;
-
-  // FTS index mutex — once created, the promise is never cleared (index persists in LanceDB)
-  private ensureFtsIndex: () => Promise<void>;
-
-  // Cached reranker — k=60 is constant, no need to recreate per search
-  private getReranker = createRerankerMutex();
-
-  constructor(private db: lancedb.Connection) {
-    this.ensureFtsIndex = createFtsMutex(() => this.getTable());
-  }
-
-  private async getTable() {
-    const names = await this.db.tableNames();
-    if (names.includes(TABLE_NAME)) {
-      const table = await this.db.openTable(TABLE_NAME);
-      await this.ensureMigration(table);
-      return table;
-    }
-    // Create with empty data to initialize schema
-    return await this.db.createTable(TABLE_NAME, [], { schema: memorySchema });
-  }
-
-  /**
-   * Ensures schema migration has run. Uses a mutex pattern identical to ensureFtsIndex.
-   * Adds columns introduced after the initial schema (usefulness, access_count, last_accessed).
-   */
-  private ensureMigration(table: Table): Promise<void> {
-    if (this.migrationPromise) {
-      return this.migrationPromise;
-    }
-
-    this.migrationPromise = this.migrateSchemaIfNeeded(table).catch((error) => {
-      this.migrationPromise = null;
-      throw error;
-    });
-
-    return this.migrationPromise;
-  }
-
-  /**
-   * Inspects the existing table schema and adds any missing columns with safe defaults.
-   * This handles databases created before the hybrid memory system was introduced.
-   */
-  private async migrateSchemaIfNeeded(table: Table): Promise<void> {
-    const schema = await table.schema();
-    const existingFields = new Set(schema.fields.map((f) => f.name));
-
-    const migrations: { name: string; valueSql: string }[] = [];
+  constructor(private db: Database) {}
 
-    if (!existingFields.has("usefulness")) {
-      migrations.push({ name: "usefulness", valueSql: "cast(0.0 as float)" });
-    }
-    if (!existingFields.has("access_count")) {
-      migrations.push({ name: "access_count", valueSql: "cast(0 as int)" });
-    }
-    if (!existingFields.has("last_accessed")) {
-      migrations.push({ name: "last_accessed", valueSql: "cast(NULL as timestamp)" });
-    }
-
-    if (migrations.length > 0) {
-      await table.addColumns(migrations);
-    }
-  }
+  // ---------------------------------------------------------------------------
+  // Row mapping
+  // ---------------------------------------------------------------------------
 
   /**
-   * Converts a raw LanceDB row to a Memory object.
+   * Converts a raw SQLite row from the `memories` table to a Memory object.
+   * Vector is fetched separately when needed; pass it in if available.
    */
-  private rowToMemory(row: Record<string, unknown>): Memory {
+  private rowToMemory(
+    row: Record<string, unknown>,
+    embedding: number[] = [],
+  ): Memory {
     return {
       id: row.id as string,
       content: row.content as string,
-      embedding: arrowVectorToArray(row.vector),
-      metadata: safeParseJsonObject(row.metadata as string),
-      createdAt: arrowTimestampToDate(row.created_at),
-      updatedAt: arrowTimestampToDate(row.updated_at),
-      supersededBy: row.superseded_by as string | null,
+      embedding,
+      metadata: safeParseJsonObject(row.metadata),
+      createdAt: new Date(row.created_at as number),
+      updatedAt: new Date(row.updated_at as number),
+      supersededBy: (row.superseded_by as string) ?? null,
       usefulness: (row.usefulness as number) ?? 0,
       accessCount: (row.access_count as number) ?? 0,
-      lastAccessed: row.last_accessed
-        ? arrowTimestampToDate(row.last_accessed)
-        : null,
+      lastAccessed:
+        row.last_accessed != null
+          ? new Date(row.last_accessed as number)
+          : null,
     };
   }
 
-  async insert(memory: Memory): Promise<void> {
-    const table = await this.getTable();
-    await table.add([
-      {
-        id: memory.id,
-        vector: memory.embedding,
-        content: memory.content,
-        metadata: JSON.stringify(memory.metadata),
-        created_at: memory.createdAt.getTime(),
-        updated_at: memory.updatedAt.getTime(),
-        superseded_by: memory.supersededBy,
-        usefulness: memory.usefulness,
-        access_count: memory.accessCount,
-        last_accessed: memory.lastAccessed?.getTime() ?? null,
-      },
-    ]);
+  /**
+   * Fetch the embedding vector for a memory id from the vec0 table.
+   */
+  private getEmbedding(id: string): number[] {
+    const row = this.db
+      .prepare("SELECT vector FROM memories_vec WHERE id = ?")
+      .get(id) as { vector: Buffer } | null;
+    return row ? deserializeVector(row.vector) : [];
   }
 
-  async upsert(memory: Memory): Promise<void> {
-    const table = await this.getTable();
-    const existing = await table.query().where(`id = '${escapeSql(memory.id)}'`).limit(1).toArray();
+  // ---------------------------------------------------------------------------
+  // Public API
+  // ---------------------------------------------------------------------------
 
-    if (existing.length === 0) {
-      return await this.insert(memory);
-    }
+  async insert(memory: Memory): Promise<void> {
+    const tx = this.db.transaction(() => {
+      this.db
+        .prepare(
+          `INSERT INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        )
+        .run(
+          memory.id,
+          memory.content,
+          JSON.stringify(memory.metadata),
+          memory.createdAt.getTime(),
+          memory.updatedAt.getTime(),
+          memory.supersededBy,
+          memory.usefulness,
+          memory.accessCount,
+          memory.lastAccessed?.getTime() ?? null,
+        );
+
+      this.db
+        .prepare("INSERT INTO memories_vec (id, vector) VALUES (?, ?)")
+        .run(memory.id, serializeVector(memory.embedding));
+
+      this.db
+        .prepare("INSERT INTO memories_fts (id, content) VALUES (?, ?)")
+        .run(memory.id, memory.content);
+    });
 
-    await table.update({
-      where: `id = '${escapeSql(memory.id)}'`,
-      values: {
-        vector: memory.embedding,
-        content: memory.content,
-        metadata: JSON.stringify(memory.metadata),
-        created_at: memory.createdAt.getTime(),
-        updated_at: memory.updatedAt.getTime(),
-        superseded_by: memory.supersededBy,
-        usefulness: memory.usefulness,
-        access_count: memory.accessCount,
-        last_accessed: memory.lastAccessed?.getTime() ?? null,
-      },
+    tx();
+  }
+
+  async upsert(memory: Memory): Promise<void> {
+    const tx = this.db.transaction(() => {
+      // Main table supports INSERT OR REPLACE
+      this.db
+        .prepare(
+          `INSERT OR REPLACE INTO memories (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        )
+        .run(
+          memory.id,
+          memory.content,
+          JSON.stringify(memory.metadata),
+          memory.createdAt.getTime(),
+          memory.updatedAt.getTime(),
+          memory.supersededBy,
+          memory.usefulness,
+          memory.accessCount,
+          memory.lastAccessed?.getTime() ?? null,
+        );
+
+      // vec0 virtual tables don't support REPLACE — delete then insert
+      this.db.prepare("DELETE FROM memories_vec WHERE id = ?").run(memory.id);
+      this.db
+        .prepare("INSERT INTO memories_vec (id, vector) VALUES (?, ?)")
+        .run(memory.id, serializeVector(memory.embedding));
+
+      // fts5 virtual tables don't support REPLACE — delete then insert
+      this.db.prepare("DELETE FROM memories_fts WHERE id = ?").run(memory.id);
+      this.db
+        .prepare("INSERT INTO memories_fts (id, content) VALUES (?, ?)")
+        .run(memory.id, memory.content);
     });
+
+    tx();
   }
 
   async findById(id: string): Promise<Memory | null> {
-    const table = await this.getTable();
-    const results = await table.query().where(`id = '${escapeSql(id)}'`).limit(1).toArray();
+    const row = this.db
+      .prepare("SELECT * FROM memories WHERE id = ?")
+      .get(id) as Record<string, unknown> | null;
 
-    if (results.length === 0) {
-      return null;
-    }
+    if (!row) return null;
 
-    return this.rowToMemory(results[0] as Record<string, unknown>);
+    const embedding = this.getEmbedding(id);
+    return this.rowToMemory(row, embedding);
   }
 
   async findByIds(ids: string[]): Promise<Memory[]> {
     if (ids.length === 0) return [];
-    const table = await this.getTable();
-    const inList = ids.map((id) => `'${escapeSql(id)}'`).join(", ");
-    const results = await table
-      .query()
-      .where(`id IN (${inList})`)
-      .toArray();
-    return results.map((row) => this.rowToMemory(row as Record<string, unknown>));
-  }
 
-  async markDeleted(id: string): Promise<boolean> {
-    const table = await this.getTable();
-
-    // Verify existence first to match previous behavior (return false if not found)
-    const existing = await table.query().where(`id = '${escapeSql(id)}'`).limit(1).toArray();
-    if (existing.length === 0) {
-      return false;
-    }
+    const placeholders = ids.map(() => "?").join(", ");
+    const rows = this.db
+      .prepare(`SELECT * FROM memories WHERE id IN (${placeholders})`)
+      .all(...ids) as Array<Record<string, unknown>>;
 
-    const now = Date.now();
-    await table.update({
-      where: `id = '${escapeSql(id)}'`,
-      values: {
-        superseded_by: DELETED_TOMBSTONE,
-        updated_at: now,
-      },
+    return rows.map((row) => {
+      const embedding = this.getEmbedding(row.id as string);
+      return this.rowToMemory(row, embedding);
     });
+  }
+
+  async markDeleted(id: string): Promise<boolean> {
+    const result = this.db
+      .prepare(
+        "UPDATE memories SET superseded_by = ?, updated_at = ? WHERE id = ?",
+      )
+      .run(DELETED_TOMBSTONE, Date.now(), id);
 
-    return true;
+    return result.changes > 0;
   }
 
   /**
-   * Performs hybrid search combining vector similarity and full-text search.
-   * Uses RRF (Reciprocal Rank Fusion) to combine rankings from both search methods.
-   *
-   * @param embedding - Query embedding vector
-   * @param query - Text query for full-text search
-   * @param limit - Maximum number of results to return
-   * @returns Array of HybridRow containing full Memory data plus RRF score
+   * Hybrid search combining vector KNN and FTS5, fused with Reciprocal Rank Fusion.
    */
-  async findHybrid(embedding: number[], query: string, limit: number): Promise<HybridRow[]> {
-    await this.ensureFtsIndex();
-
-    const table = await this.getTable();
-    const reranker = await this.getReranker();
-
-    const results = await table
-      .query()
-      .nearestTo(embedding)
-      .fullTextSearch(query)
-      .rerank(reranker)
-      .limit(limit)
-      .toArray();
-
-    return results.map((row) => {
-      const memory = this.rowToMemory(row as Record<string, unknown>);
-      return {
+  async findHybrid(
+    embedding: number[],
+    query: string,
+    limit: number,
+  ): Promise<HybridRow[]> {
+    const candidateLimit = limit * 3;
+    const vecBuf = serializeVector(embedding);
+
+    // Vector KNN search
+    const vectorResults = this.db
+      .prepare(
+        "SELECT id, distance FROM memories_vec WHERE vector MATCH ? AND k = ? ORDER BY distance",
+      )
+      .all(vecBuf, candidateLimit) as Array<{ id: string; distance: number }>;
+
+    // Full-text search
+    const ftsQuery = sanitizeFtsQuery(query);
+    const ftsResults = this.db
+      .prepare(
+        "SELECT id FROM memories_fts WHERE memories_fts MATCH ? LIMIT ?",
+      )
+      .all(ftsQuery, candidateLimit) as Array<{ id: string }>;
+
+    // Compute RRF scores and pick top ids
+    const rrfScores = hybridRRF(vectorResults, ftsResults);
+    const topIds = topByRRF(rrfScores, limit);
+
+    if (topIds.length === 0) return [];
+
+    // Fetch full rows for the winning ids (service layer handles deleted filtering)
+    const placeholders = topIds.map(() => "?").join(", ");
+    const rows = this.db
+      .prepare(
+        `SELECT * FROM memories WHERE id IN (${placeholders})`,
+      )
+      .all(...topIds) as Array<Record<string, unknown>>;
+
+    // Build a lookup for quick access
+    const rowMap = new Map<string, Record<string, unknown>>();
+    for (const row of rows) {
+      rowMap.set(row.id as string, row);
+    }
+
+    // Return results in RRF-ranked order, skipping any that were deleted
+    const results: HybridRow[] = [];
+    for (const id of topIds) {
+      const row = rowMap.get(id);
+      if (!row) continue; // deleted or missing
+
+      const memEmbedding = this.getEmbedding(id);
+      const memory = this.rowToMemory(row, memEmbedding);
+      results.push({
         ...memory,
-        rrfScore: (row._relevance_score as number) ?? 0,
-      };
-    });
+        rrfScore: rrfScores.get(id) ?? 0,
+      });
+    }
+
+    return results;
   }
 }

package/src/db/migrations.ts

@@ -0,0 +1,70 @@
+import type { Database } from "bun:sqlite";
+
+/**
+ * Run all schema migrations. Safe to call on every startup (uses IF NOT EXISTS).
+ */
+export function runMigrations(db: Database): void {
+  // -- Memories --
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS memories (
+      id            TEXT PRIMARY KEY,
+      content       TEXT NOT NULL,
+      metadata      TEXT NOT NULL DEFAULT '{}',
+      created_at    INTEGER NOT NULL,
+      updated_at    INTEGER NOT NULL,
+      superseded_by TEXT,
+      usefulness    REAL NOT NULL DEFAULT 0.0,
+      access_count  INTEGER NOT NULL DEFAULT 0,
+      last_accessed INTEGER
+    )
+  `);
+
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS memories_vec USING vec0(
+      id TEXT PRIMARY KEY,
+      vector float[384]
+    )
+  `);
+
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+      id UNINDEXED,
+      content
+    )
+  `);
+
+  // -- Conversation History --
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS conversation_history (
+      id                  TEXT PRIMARY KEY,
+      content             TEXT NOT NULL,
+      metadata            TEXT NOT NULL DEFAULT '{}',
+      created_at          INTEGER NOT NULL,
+      session_id          TEXT NOT NULL,
+      role                TEXT NOT NULL,
+      message_index_start INTEGER NOT NULL,
+      message_index_end   INTEGER NOT NULL,
+      project             TEXT NOT NULL
+    )
+  `);
+
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_vec USING vec0(
+      id TEXT PRIMARY KEY,
+      vector float[384]
+    )
+  `);
+
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_fts USING fts5(
+      id UNINDEXED,
+      content
+    )
+  `);
+
+  // -- Indexes --
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_session_id ON conversation_history(session_id)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_project ON conversation_history(project)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_role ON conversation_history(role)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_created_at ON conversation_history(created_at)`);
+}

package/src/db/sqlite-utils.ts

@@ -0,0 +1,78 @@
+import type { Database } from "bun:sqlite";
+
+/** RRF constant matching the previous LanceDB reranker default */
+export const RRF_K = 60;
+
+/**
+ * Serialize a number[] embedding to the raw float32 bytes sqlite-vec expects.
+ */
+export function serializeVector(vec: number[]): Buffer {
+  return Buffer.from(new Float32Array(vec).buffer);
+}
+
+/**
+ * Deserialize raw float32 bytes from sqlite-vec back to number[].
+ */
+export function deserializeVector(buf: Buffer): number[] {
+  return Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
+}
+
+/**
+ * Sanitize a user query for FTS5 by quoting each token as a literal.
+ * Prevents FTS5 syntax errors from special characters like AND, OR, *, etc.
+ */
+export function sanitizeFtsQuery(query: string): string {
+  const tokens = query.trim().split(/\s+/).filter(Boolean);
+  if (tokens.length === 0) return '""';
+  return tokens.map(t => `"${t.replace(/"/g, '""')}"`).join(" ");
+}
+
+/**
+ * Compute hybrid RRF scores from two ranked result lists.
+ * Returns a map of id -> combined RRF score, sorted descending.
+ */
+export function hybridRRF(
+  vectorResults: Array<{ id: string }>,
+  ftsResults: Array<{ id: string }>,
+  k: number = RRF_K
+): Map<string, number> {
+  const scores = new Map<string, number>();
+
+  vectorResults.forEach((r, i) => {
+    const rank = i + 1;
+    scores.set(r.id, (scores.get(r.id) ?? 0) + 1 / (k + rank));
+  });
+
+  ftsResults.forEach((r, i) => {
+    const rank = i + 1;
+    scores.set(r.id, (scores.get(r.id) ?? 0) + 1 / (k + rank));
+  });
+
+  return scores;
+}
+
+/**
+ * Sort ids by RRF score descending and return top N.
+ */
+export function topByRRF(scores: Map<string, number>, limit: number): string[] {
+  return [...scores.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, limit)
+    .map(([id]) => id);
+}
+
+/**
+ * Safely parse a JSON string, returning an empty object on failure.
+ * Ported from lancedb-utils.ts.
+ */
+export function safeParseJsonObject(raw: unknown): Record<string, unknown> {
+  if (typeof raw !== "string") return {};
+  try {
+    const parsed = JSON.parse(raw);
+    return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
+      ? parsed
+      : {};
+  } catch {
+    return {};
+  }
+}

package/src/http/server.ts

@@ -1,16 +1,14 @@
 import { Hono } from "hono";
 import { cors } from "hono/cors";
-import { serve as nodeServe } from "@hono/node-server";
 import { createServer } from "net";
+import { writeFileSync, mkdirSync, unlinkSync } from "fs";
+import { join } from "path";
 import type { MemoryService } from "../services/memory.service.js";
 import type { Config } from "../config/index.js";
 import { isDeleted } from "../types/memory.js";
 import { createMcpRoutes } from "./mcp-transport.js";
 import type { Memory, SearchIntent } from "../types/memory.js";
 
-// Detect runtime
-const isBun = typeof globalThis.Bun !== "undefined";
-
 /**
  * Check if a port is available by attempting to bind to it
  */
@@ -56,6 +54,31 @@ async function findAvailablePort(
   });
 }
 
+/**
+ * Write a lockfile so hooks can discover which port this server bound to.
+ * Written atomically after the HTTP server successfully binds.
+ */
+function writeLockfile(port: number): void {
+  const dir = join(process.cwd(), ".vector-memory");
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(
+    join(dir, "server.lock"),
+    JSON.stringify({ port, pid: process.pid }),
+    "utf8"
+  );
+}
+
+/**
+ * Remove the lockfile on clean shutdown so stale files don't linger.
+ */
+export function removeLockfile(): void {
+  try {
+    unlinkSync(join(process.cwd(), ".vector-memory", "server.lock"));
+  } catch {
+    // already gone — fine
+  }
+}
+
 export interface HttpServerOptions {
   memoryService: MemoryService;
   config: Config;
@@ -246,37 +269,19 @@ export async function startHttpServer(
   // Find an available port (uses configured port if available, otherwise picks a random one)
   const actualPort = await findAvailablePort(config.httpPort, config.httpHost);
 
-  if (isBun) {
-    // Use Bun's native server
-    const server = Bun.serve({
-      port: actualPort,
-      hostname: config.httpHost,
-      fetch: app.fetch,
-    });
-
-    console.error(
-      `[vector-memory-mcp] HTTP server listening on http://${config.httpHost}:${actualPort}`
-    );
-
-    return {
-      stop: () => server.stop(),
-      port: actualPort,
-    };
-  } else {
-    // Use Node.js server via @hono/node-server
-    const server = nodeServe({
-      fetch: app.fetch,
-      port: actualPort,
-      hostname: config.httpHost,
-    });
+  const server = Bun.serve({
+    port: actualPort,
+    hostname: config.httpHost,
+    fetch: app.fetch,
+  });
 
-    console.error(
-      `[vector-memory-mcp] HTTP server listening on http://${config.httpHost}:${actualPort}`
-    );
+  writeLockfile(actualPort);
+  console.error(
+    `[vector-memory-mcp] HTTP server listening on http://${config.httpHost}:${actualPort}`
+  );
 
-    return {
-      stop: () => server.close(),
-      port: actualPort,
-    };
-  }
+  return {
+    stop: () => { removeLockfile(); server.stop(); },
+    port: actualPort,
+  };
 }