@aeriondyseti/vector-memory-mcp 0.9.0-dev.3 → 0.9.0-dev.5

package/README.md

@@ -23,28 +23,21 @@ A local-first MCP server that provides vector-based memory storage. Uses local e
 
 ### Prerequisites
 
-- [Bun](https://bun.sh/) 1.0+ (recommended) or [Node.js](https://nodejs.org/) 20+
+- [Bun](https://bun.sh/) 1.0+
 - An MCP-compatible client (Claude Code, Claude Desktop, etc.)
 
 ### Install
 
-**With Bun (recommended):**
 ```bash
 bun install -g @aeriondyseti/vector-memory-mcp
 ```
 
-**With npm/Node.js:**
-```bash
-npm install -g @aeriondyseti/vector-memory-mcp
-```
-
 > First install downloads ML models (~90MB). This may take a minute.
 
 ### Configure
 
 Add to your MCP client config (e.g., `~/.claude/settings.json`):
 
-**With Bun:**
 ```json
 {
   "mcpServers": {
@@ -57,19 +50,6 @@ Add to your MCP client config (e.g., `~/.claude/settings.json`):
 }
 ```
 
-**With Node.js:**
-```json
-{
-  "mcpServers": {
-    "vector-memory": {
-      "type": "stdio",
-      "command": "npx",
-      "args": ["-y", "@aeriondyseti/vector-memory-mcp"]
-    }
-  }
-}
-```
-
 ### Use
 
 Restart your MCP client. You now have access to:

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "0.9.0-dev.3",
+  "version": "0.9.0-dev.5",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
-  "main": "dist/src/index.js",
+  "main": "src/index.ts",
   "bin": {
-    "vector-memory-mcp": "dist/src/index.js"
+    "vector-memory-mcp": "src/index.ts"
   },
   "files": [
-    "dist",
+    "src",
     "scripts",
     "hooks",
     "README.md",
@@ -24,11 +24,9 @@
   },
   "homepage": "https://github.com/AerionDyseti/vector-memory-mcp#readme",
   "scripts": {
-    "start": "node dist/src/index.js",
-    "dev": "bun run build && node dist/src/index.js",
-    "dev:watch": "bun run build && bun --watch run src/index.ts",
-    "build": "tsc",
-    "prebuild": "rm -rf dist",
+    "start": "bun run src/index.ts",
+    "dev": "bun --watch run src/index.ts",
+    "build": "bun run typecheck",
     "typecheck": "bunx tsc --noEmit",
     "test": "bun run scripts/test-runner.ts",
     "test:raw": "bun test --preload ./tests/preload.ts",
@@ -39,8 +37,7 @@
     "postinstall": "bun run scripts/warmup.ts",
     "publish:check": "bun run scripts/publish.ts --dry-run",
     "publish:npm": "bun run scripts/publish.ts",
-    "publish:dev": "npm publish --access public --tag dev",
-    "prepublishOnly": "bun run build"
+    "publish:dev": "npm publish --access public --tag dev"
   },
   "keywords": [
     "mcp",
@@ -51,7 +48,6 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "@hono/node-server": "^1.19.7",
     "@huggingface/transformers": "^3.8.0",
     "@lancedb/lancedb": "^0.22.3",
     "@modelcontextprotocol/sdk": "^1.0.0",

package/src/config/index.ts

@@ -0,0 +1,75 @@
+import arg from "arg";
+import { join } from "path";
+
+export type TransportMode = "stdio" | "http" | "both";
+
+export interface Config {
+  dbPath: string;
+  embeddingModel: string;
+  embeddingDimension: number;
+  httpPort: number;
+  httpHost: string;
+  enableHttp: boolean;
+  transportMode: TransportMode;
+}
+
+export interface ConfigOverrides {
+  dbPath?: string;
+  httpPort?: number;
+  enableHttp?: boolean;
+  transportMode?: TransportMode;
+}
+
+// Defaults - always use repo-local .vector-memory folder
+const DEFAULT_DB_PATH = join(process.cwd(), ".vector-memory", "memories.db");
+const DEFAULT_EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
+const DEFAULT_EMBEDDING_DIMENSION = 384;
+const DEFAULT_HTTP_PORT = 3271;
+const DEFAULT_HTTP_HOST = "127.0.0.1";
+
+function resolvePath(path: string): string {
+  return path.startsWith("/") ? path : join(process.cwd(), path);
+}
+
+export function loadConfig(overrides: ConfigOverrides = {}): Config {
+  const transportMode = overrides.transportMode ?? "stdio";
+  // HTTP enabled by default (needed for hooks), can disable with --no-http
+  const enableHttp = overrides.enableHttp ?? true;
+
+  return {
+    dbPath: resolvePath(overrides.dbPath ?? DEFAULT_DB_PATH),
+    embeddingModel: DEFAULT_EMBEDDING_MODEL,
+    embeddingDimension: DEFAULT_EMBEDDING_DIMENSION,
+    httpPort: overrides.httpPort ?? DEFAULT_HTTP_PORT,
+    httpHost: DEFAULT_HTTP_HOST,
+    enableHttp,
+    transportMode,
+  };
+}
+
+/**
+ * Parse CLI arguments into config overrides.
+ */
+export function parseCliArgs(argv: string[]): ConfigOverrides {
+  const args = arg(
+    {
+      "--db-file": String,
+      "--port": Number,
+      "--no-http": Boolean,
+
+      // Aliases
+      "-d": "--db-file",
+      "-p": "--port",
+    },
+    { argv, permissive: true }
+  );
+
+  return {
+    dbPath: args["--db-file"],
+    httpPort: args["--port"],
+    enableHttp: args["--no-http"] ? false : undefined,
+  };
+}
+
+// Default config for imports that don't use CLI args
+export const config = loadConfig();

package/src/db/connection.ts

@@ -0,0 +1,11 @@
+import * as lancedb from "@lancedb/lancedb";
+import { mkdirSync } from "fs";
+import { dirname } from "path";
+
+export async function connectToDatabase(dbPath: string): Promise<lancedb.Connection> {
+  // Ensure directory exists
+  mkdirSync(dirname(dbPath), { recursive: true });
+
+  const db = await lancedb.connect(dbPath);
+  return db;
+}

package/src/db/memory.repository.ts

@@ -0,0 +1,205 @@
+import * as lancedb from "@lancedb/lancedb";
+import { Index, rerankers, type Table } from "@lancedb/lancedb";
+import { TABLE_NAME, memorySchema } from "./schema.js";
+import {
+  type Memory,
+  type HybridRow,
+  DELETED_TOMBSTONE,
+} from "../types/memory.js";
+
+export class MemoryRepository {
+  // Mutex for FTS index creation - ensures only one index creation runs at a time
+  // Once set, this promise is never cleared (FTS index persists in the database)
+  private ftsIndexPromise: Promise<void> | null = null;
+
+  constructor(private db: lancedb.Connection) { }
+
+  private async getTable() {
+    const names = await this.db.tableNames();
+    if (names.includes(TABLE_NAME)) {
+      return await this.db.openTable(TABLE_NAME);
+    }
+    // Create with empty data to initialize schema
+    return await this.db.createTable(TABLE_NAME, [], { schema: memorySchema });
+  }
+
+  /**
+   * Ensures the FTS index exists on the content column.
+   * Uses a mutex pattern to prevent concurrent index creation.
+   * The key insight: we must capture the promise BEFORE any await.
+   */
+  private ensureFtsIndex(): Promise<void> {
+    // If there's already a pending or completed index creation, return that promise
+    if (this.ftsIndexPromise) {
+      return this.ftsIndexPromise;
+    }
+
+    // Synchronously set the promise BEFORE any await
+    // This is critical for proper mutex behavior in JS async code
+    this.ftsIndexPromise = this.createFtsIndexIfNeeded().catch((error) => {
+      // Reset on error so the next call can retry
+      this.ftsIndexPromise = null;
+      throw error;
+    });
+
+    return this.ftsIndexPromise;
+  }
+
+  /**
+   * Creates the FTS index if it doesn't already exist.
+   * Gets its own table reference to ensure consistent index state.
+   */
+  private async createFtsIndexIfNeeded(): Promise<void> {
+    const table = await this.getTable();
+    const indices = await table.listIndices();
+    const hasFtsIndex = indices.some(
+      (idx) => idx.columns.includes("content") && idx.indexType === "FTS"
+    );
+
+    if (!hasFtsIndex) {
+      await table.createIndex("content", {
+        config: Index.fts(),
+      });
+      // Wait for the index to be fully created and available
+      await table.waitForIndex(["content_idx"], 30);
+    }
+  }
+
+  /**
+   * Converts a raw LanceDB row to a Memory object.
+   */
+  private rowToMemory(row: Record<string, unknown>): Memory {
+    // Handle Arrow Vector type conversion
+    // LanceDB returns an Arrow Vector object which is iterable but not an array
+    const vectorData = row.vector as unknown;
+    const embedding = Array.isArray(vectorData)
+      ? vectorData
+      : Array.from(vectorData as Iterable<number>) as number[];
+
+    return {
+      id: row.id as string,
+      content: row.content as string,
+      embedding,
+      metadata: JSON.parse(row.metadata as string),
+      createdAt: new Date(row.created_at as number),
+      updatedAt: new Date(row.updated_at as number),
+      supersededBy: row.superseded_by as string | null,
+      usefulness: (row.usefulness as number) ?? 0,
+      accessCount: (row.access_count as number) ?? 0,
+      lastAccessed: row.last_accessed
+        ? new Date(row.last_accessed as number)
+        : null,
+    };
+  }
+
+  async insert(memory: Memory): Promise<void> {
+    const table = await this.getTable();
+    await table.add([
+      {
+        id: memory.id,
+        vector: memory.embedding,
+        content: memory.content,
+        metadata: JSON.stringify(memory.metadata),
+        created_at: memory.createdAt.getTime(),
+        updated_at: memory.updatedAt.getTime(),
+        superseded_by: memory.supersededBy,
+        usefulness: memory.usefulness,
+        access_count: memory.accessCount,
+        last_accessed: memory.lastAccessed?.getTime() ?? null,
+      },
+    ]);
+  }
+
+  async upsert(memory: Memory): Promise<void> {
+    const table = await this.getTable();
+    const existing = await table.query().where(`id = '${memory.id}'`).limit(1).toArray();
+
+    if (existing.length === 0) {
+      return await this.insert(memory);
+    }
+
+    await table.update({
+      where: `id = '${memory.id}'`,
+      values: {
+        vector: memory.embedding,
+        content: memory.content,
+        metadata: JSON.stringify(memory.metadata),
+        created_at: memory.createdAt.getTime(),
+        updated_at: memory.updatedAt.getTime(),
+        superseded_by: memory.supersededBy,
+        usefulness: memory.usefulness,
+        access_count: memory.accessCount,
+        last_accessed: memory.lastAccessed?.getTime() ?? null,
+      },
+    });
+  }
+
+  async findById(id: string): Promise<Memory | null> {
+    const table = await this.getTable();
+    const results = await table.query().where(`id = '${id}'`).limit(1).toArray();
+
+    if (results.length === 0) {
+      return null;
+    }
+
+    return this.rowToMemory(results[0] as Record<string, unknown>);
+  }
+
+  async markDeleted(id: string): Promise<boolean> {
+    const table = await this.getTable();
+
+    // Verify existence first to match previous behavior (return false if not found)
+    const existing = await table.query().where(`id = '${id}'`).limit(1).toArray();
+    if (existing.length === 0) {
+      return false;
+    }
+
+    const now = Date.now();
+    await table.update({
+      where: `id = '${id}'`,
+      values: {
+        superseded_by: DELETED_TOMBSTONE,
+        updated_at: now,
+      },
+    });
+
+    return true;
+  }
+
+  /**
+   * Performs hybrid search combining vector similarity and full-text search.
+   * Uses RRF (Reciprocal Rank Fusion) to combine rankings from both search methods.
+   *
+   * @param embedding - Query embedding vector
+   * @param query - Text query for full-text search
+   * @param limit - Maximum number of results to return
+   * @returns Array of HybridRow containing full Memory data plus RRF score
+   */
+  async findHybrid(embedding: number[], query: string, limit: number): Promise<HybridRow[]> {
+    // Ensure FTS index exists (with mutex to prevent concurrent creation)
+    // This must happen BEFORE getTable to ensure proper mutex behavior
+    await this.ensureFtsIndex();
+
+    const table = await this.getTable();
+
+    // Create RRF reranker with k=60 (standard parameter)
+    const reranker = await rerankers.RRFReranker.create(60);
+
+    // Perform hybrid search: combine vector search and full-text search
+    const results = await table
+      .query()
+      .nearestTo(embedding)
+      .fullTextSearch(query)
+      .rerank(reranker)
+      .limit(limit)
+      .toArray();
+
+    return results.map((row) => {
+      const memory = this.rowToMemory(row as Record<string, unknown>);
+      return {
+        ...memory,
+        rrfScore: (row._relevance_score as number) ?? 0,
+      };
+    });
+  }
+}

package/src/db/schema.ts

@@ -0,0 +1,42 @@
+import {
+  Schema,
+  Field,
+  FixedSizeList,
+  Float32,
+  Utf8,
+  Timestamp,
+  TimeUnit,
+  Int32,
+} from "apache-arrow";
+
+export const TABLE_NAME = "memories";
+
+export const memorySchema = new Schema([
+  new Field("id", new Utf8(), false),
+  new Field(
+    "vector",
+    new FixedSizeList(384, new Field("item", new Float32())),
+    false
+  ),
+  new Field("content", new Utf8(), false),
+  new Field("metadata", new Utf8(), false), // JSON string
+  new Field(
+    "created_at",
+    new Timestamp(TimeUnit.MILLISECOND),
+    false
+  ),
+  new Field(
+    "updated_at",
+    new Timestamp(TimeUnit.MILLISECOND),
+    false
+  ),
+  new Field("superseded_by", new Utf8(), true), // Nullable
+  new Field("usefulness", new Float32(), false),
+  new Field("access_count", new Int32(), false),
+  new Field(
+    "last_accessed",
+    new Timestamp(TimeUnit.MILLISECOND),
+    true
+  ),
+]);
+

package/src/http/mcp-transport.ts

@@ -0,0 +1,255 @@
+/**
+ * MCP HTTP Transport Handler
+ *
+ * Provides StreamableHTTP transport for MCP over HTTP.
+ * and other HTTP-based MCP clients to connect to the memory server.
+ *
+ * This implementation handles the MCP protocol directly using Hono's streaming
+ * capabilities, since StreamableHTTPServerTransport expects Node.js req/res objects.
+ */
+
+import { Hono } from "hono";
+import { streamSSE } from "hono/streaming";
+import { randomUUID } from "node:crypto";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+  type JSONRPCMessage,
+  type JSONRPCRequest,
+  type JSONRPCNotification,
+} from "@modelcontextprotocol/sdk/types.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+
+import { tools } from "../mcp/tools.js";
+import { handleToolCall } from "../mcp/handlers.js";
+import type { MemoryService } from "../services/memory.service.js";
+
+interface Session {
+  server: Server;
+  serverTransport: InstanceType<typeof InMemoryTransport>;
+  clientTransport: InstanceType<typeof InMemoryTransport>;
+  pendingResponses: Map<string | number, (response: JSONRPCMessage) => void>;
+  sseClients: Set<(message: JSONRPCMessage) => void>;
+}
+
+/**
+ * Creates MCP routes for a Hono app.
+ *
+ * Uses InMemoryTransport internally and bridges to HTTP/SSE manually,
+ * since StreamableHTTPServerTransport requires Node.js req/res objects.
+ */
+export function createMcpRoutes(memoryService: MemoryService): Hono {
+  const app = new Hono();
+
+  // Store active sessions by session ID
+  const sessions: Map<string, Session> = new Map();
+
+  /**
+   * Creates a new MCP server instance configured with memory tools.
+   */
+  async function createSession(): Promise<Session> {
+    const server = new Server(
+      { name: "vector-memory-mcp", version: "0.6.0" },
+      { capabilities: { tools: {} } }
+    );
+
+    server.setRequestHandler(ListToolsRequestSchema, async () => {
+      return { tools };
+    });
+
+    server.setRequestHandler(CallToolRequestSchema, async (request) => {
+      const { name, arguments: args } = request.params;
+      return handleToolCall(name, args, memoryService);
+    });
+
+    // Create linked in-memory transports
+    const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+
+    // Connect server to its transport
+    await server.connect(serverTransport);
+
+    const session: Session = {
+      server,
+      serverTransport,
+      clientTransport,
+      pendingResponses: new Map(),
+      sseClients: new Set(),
+    };
+
+    // Handle messages from server (responses and notifications)
+    clientTransport.onmessage = (message: JSONRPCMessage) => {
+      // Check if this is a response to a pending request
+      if ("id" in message && message.id !== undefined) {
+        const resolver = session.pendingResponses.get(message.id);
+        if (resolver) {
+          resolver(message);
+          session.pendingResponses.delete(message.id);
+          return;
+        }
+      }
+
+      // Otherwise, broadcast to SSE clients (notifications)
+      for (const sendToClient of session.sseClients) {
+        sendToClient(message);
+      }
+    };
+
+    return session;
+  }
+
+  /**
+   * Handle POST requests - session initialization and message handling
+   */
+  app.post("/mcp", async (c) => {
+    const sessionId = c.req.header("mcp-session-id");
+    const body = await c.req.json();
+
+    let session: Session | undefined;
+    let newSessionId: string | undefined;
+
+    if (sessionId && sessions.has(sessionId)) {
+      // Reuse existing session
+      session = sessions.get(sessionId)!;
+    } else if (isInitializeRequest(body)) {
+      // New session initialization
+      newSessionId = randomUUID();
+      session = await createSession();
+      sessions.set(newSessionId, session);
+      console.error(`[vector-memory-mcp] MCP session initialized: ${newSessionId}`);
+    } else {
+      // Invalid request - no session ID and not an initialize request
+      return c.json(
+        {
+          jsonrpc: "2.0",
+          error: {
+            code: -32000,
+            message: "Invalid session. Send initialize request without session ID to start.",
+          },
+          id: body.id ?? null,
+        },
+        400
+      );
+    }
+
+    // Send message to server and wait for response
+    const response = await sendAndWaitForResponse(session, body);
+
+    // Include session ID header for new sessions
+    if (newSessionId) {
+      c.header("mcp-session-id", newSessionId);
+    }
+
+    return c.json(response);
+  });
+
+  /**
+   * Handle GET requests - SSE stream for server-to-client notifications
+   */
+  app.get("/mcp", async (c) => {
+    const sessionId = c.req.header("mcp-session-id");
+
+    if (!sessionId || !sessions.has(sessionId)) {
+      return c.json(
+        {
+          jsonrpc: "2.0",
+          error: { code: -32000, message: "Invalid or missing session ID" },
+          id: null,
+        },
+        400
+      );
+    }
+
+    const session = sessions.get(sessionId)!;
+
+    return streamSSE(c, async (stream) => {
+      // Register this SSE client
+      const sendMessage = (message: JSONRPCMessage) => {
+        stream.writeSSE({
+          data: JSON.stringify(message),
+          event: "message",
+        });
+      };
+
+      session.sseClients.add(sendMessage);
+
+      // Keep connection open
+      try {
+        // Send a ping every 30 seconds to keep connection alive
+        while (true) {
+          await stream.sleep(30000);
+          await stream.writeSSE({ event: "ping", data: "" });
+        }
+      } finally {
+        session.sseClients.delete(sendMessage);
+      }
+    });
+  });
+
+  /**
+   * Handle DELETE requests - session termination
+   */
+  app.delete("/mcp", async (c) => {
+    const sessionId = c.req.header("mcp-session-id");
+
+    if (!sessionId || !sessions.has(sessionId)) {
+      return c.json(
+        {
+          jsonrpc: "2.0",
+          error: { code: -32000, message: "Invalid or missing session ID" },
+          id: null,
+        },
+        400
+      );
+    }
+
+    const session = sessions.get(sessionId)!;
+
+    // Close transports
+    await session.clientTransport.close();
+    await session.serverTransport.close();
+    await session.server.close();
+
+    sessions.delete(sessionId);
+    console.error(`[vector-memory-mcp] MCP session closed: ${sessionId}`);
+
+    return c.json({ success: true });
+  });
+
+  return app;
+}
+
+/**
+ * Send a message to the server and wait for its response.
+ */
+async function sendAndWaitForResponse(
+  session: Session,
+  message: JSONRPCRequest | JSONRPCNotification
+): Promise<JSONRPCMessage> {
+  return new Promise((resolve) => {
+    // Register response handler for requests (messages with id)
+    if ("id" in message && message.id !== undefined) {
+      session.pendingResponses.set(message.id, resolve);
+    }
+
+    // Send message to server
+    session.clientTransport.send(message);
+
+    // For notifications (no id), resolve immediately with empty response
+    if (!("id" in message) || message.id === undefined) {
+      resolve({ jsonrpc: "2.0" } as JSONRPCMessage);
+    }
+  });
+}
+
+/**
+ * Check if a message is an initialize request.
+ */
+function isInitializeRequest(body: unknown): boolean {
+  return (
+    typeof body === "object" &&
+    body !== null &&
+    "method" in body &&
+    (body as { method: string }).method === "initialize"
+  );
+}