@aeriondyseti/vector-memory-mcp 1.1.0-dev.2 → 1.1.0-dev.3

package/src/db/conversation.repository.ts

@@ -0,0 +1,120 @@
+import * as lancedb from "@lancedb/lancedb";
+import { type Table } from "@lancedb/lancedb";
+import {
+  CONVERSATION_TABLE_NAME,
+  conversationSchema,
+} from "./conversation.schema.js";
+import type {
+  ConversationHybridRow,
+  HistoryFilters,
+} from "../types/conversation.js";
+import {
+  getOrCreateTable,
+  createFtsMutex,
+  createRerankerMutex,
+  escapeSql,
+  safeParseJsonObject,
+} from "./lancedb-utils.js";
+
+export class ConversationRepository {
+  private tablePromise: Promise<Table> | null = null;
+
+  // FTS index mutex — recreated after data mutations to force re-check
+  private ensureFtsIndex = createFtsMutex(() => this.getTable());
+
+  // Cached reranker — k=60 is constant, no need to recreate per search
+  private getReranker = createRerankerMutex();
+
+  constructor(private db: lancedb.Connection) {}
+
+  private getTable(): Promise<Table> {
+    if (!this.tablePromise) {
+      this.tablePromise = getOrCreateTable(
+        this.db,
+        CONVERSATION_TABLE_NAME,
+        conversationSchema,
+      ).catch((err) => {
+        this.tablePromise = null;
+        throw err;
+      });
+    }
+    return this.tablePromise;
+  }
+
+  private rowToConversationHybridRow(
+    row: Record<string, unknown>
+  ): ConversationHybridRow {
+    const metadata = safeParseJsonObject(row.metadata as string);
+    return {
+      id: row.id as string,
+      content: row.content as string,
+      metadata,
+      createdAt: new Date(row.created_at as number),
+      rrfScore: (row._relevance_score as number) ?? 0,
+    };
+  }
+
+  async insertBatch(
+    rows: Array<{
+      id: string;
+      vector: number[];
+      content: string;
+      metadata: string;
+      created_at: number;
+      session_id: string;
+      role: string;
+      message_index_start: number;
+      message_index_end: number;
+      project: string;
+    }>
+  ): Promise<void> {
+    if (rows.length === 0) return;
+    const table = await this.getTable();
+    await table.add(rows);
+    // Reset FTS mutex so index existence is re-verified after new data
+    this.ensureFtsIndex = createFtsMutex(() => this.getTable());
+  }
+
+  async deleteBySessionId(sessionId: string): Promise<void> {
+    const table = await this.getTable();
+    await table.delete(`session_id = '${escapeSql(sessionId)}'`);
+    this.ensureFtsIndex = createFtsMutex(() => this.getTable());
+  }
+
+  async findHybrid(
+    embedding: number[],
+    query: string,
+    limit: number,
+    filters?: HistoryFilters
+  ): Promise<ConversationHybridRow[]> {
+    await this.ensureFtsIndex();
+    const table = await this.getTable();
+    const reranker = await this.getReranker();
+
+    let queryBuilder = table
+      .query()
+      .nearestTo(embedding)
+      .fullTextSearch(query)
+      .rerank(reranker);
+
+    const conditions: string[] = [];
+    if (filters?.sessionId)
+      conditions.push(`session_id = '${escapeSql(filters.sessionId)}'`);
+    if (filters?.role) conditions.push(`role = '${escapeSql(filters.role)}'`);
+    if (filters?.project)
+      conditions.push(`project = '${escapeSql(filters.project)}'`);
+    if (filters?.after)
+      conditions.push(`created_at > ${filters.after.getTime()}`);
+    if (filters?.before)
+      conditions.push(`created_at < ${filters.before.getTime()}`);
+
+    if (conditions.length > 0) {
+      queryBuilder = queryBuilder.where(conditions.join(" AND "));
+    }
+
+    const results = await queryBuilder.limit(limit).toArray();
+    return results.map((row) =>
+      this.rowToConversationHybridRow(row as Record<string, unknown>)
+    );
+  }
+}

package/src/db/conversation.schema.ts

@@ -0,0 +1,33 @@
+import {
+  Schema,
+  Field,
+  FixedSizeList,
+  Float32,
+  Utf8,
+  Timestamp,
+  TimeUnit,
+  Int32,
+} from "apache-arrow";
+
+export const CONVERSATION_TABLE_NAME = "conversation_history";
+
+export const conversationSchema = new Schema([
+  new Field("id", new Utf8(), false),
+  new Field(
+    "vector",
+    new FixedSizeList(384, new Field("item", new Float32(), false)),
+    false
+  ),
+  new Field("content", new Utf8(), false),
+  new Field("metadata", new Utf8(), false), // JSON string
+  new Field(
+    "created_at",
+    new Timestamp(TimeUnit.MILLISECOND),
+    false
+  ),
+  new Field("session_id", new Utf8(), false),
+  new Field("role", new Utf8(), false),
+  new Field("message_index_start", new Int32(), false),
+  new Field("message_index_end", new Int32(), false),
+  new Field("project", new Utf8(), false),
+]);

package/src/db/lancedb-utils.ts

@@ -3,13 +3,20 @@ import { Index, rerankers, type Table } from "@lancedb/lancedb";
 import type { Schema } from "apache-arrow";
 
 /**
- * Escapes a string for use in LanceDB/DataFusion SQL WHERE clauses.
- * Doubles single quotes to prevent SQL injection (standard SQL escaping).
+ * Escape a string value for safe interpolation into LanceDB/DataFusion SQL WHERE clauses.
+ *
+ * DataFusion uses ANSI SQL string literal rules:
+ * - String literals are delimited by single quotes
+ * - Single quotes within strings are escaped by doubling: ' -> ''
+ * - Backslashes are NOT escape characters (treated literally)
  */
-export function escapeLanceDbString(value: string): string {
+export function escapeSql(value: string): string {
   return value.replace(/'/g, "''");
 }
 
+/** Default k parameter for Reciprocal Rank Fusion reranking. */
+export const RRF_K = 60;
+
 /**
  * Converts LanceDB's Arrow Vector type to a plain number[].
  * LanceDB returns an Arrow Vector object which is iterable but not an array.
@@ -20,6 +27,17 @@ export function arrowVectorToArray(value: unknown): number[] {
     : (Array.from(value as Iterable<number>) as number[]);
 }
 
+/**
+ * Safely parse a JSON string into an object, returning an empty object on failure.
+ */
+export function safeParseJsonObject(raw: string): Record<string, unknown> {
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return {};
+  }
+}
+
 /**
  * Opens an existing table or creates it with the given schema.
  * Does NOT cache — callers should cache the returned Table if desired.
@@ -29,11 +47,21 @@ export async function getOrCreateTable(
   name: string,
   schema: Schema
 ): Promise<Table> {
-  const names = await db.tableNames();
-  if (names.includes(name)) {
+  try {
+    return await db.openTable(name);
+  } catch (err: unknown) {
+    // Only proceed to create if the table was not found
+    const message = err instanceof Error ? err.message : String(err);
+    if (!message.includes("was not found") && !message.includes("does not exist")) {
+      throw err;
+    }
+  }
+  try {
+    return await db.createTable(name, [], { schema });
+  } catch {
+    // Another caller may have created it concurrently
     return await db.openTable(name);
   }
-  return await db.createTable(name, [], { schema });
 }
 
 /**
@@ -81,7 +109,7 @@ export function createFtsMutex(
  * Same pattern as createFtsMutex: create once, cache forever, reset on error.
  */
 export function createRerankerMutex(
-  k: number = 60
+  k: number = RRF_K
 ): () => Promise<rerankers.RRFReranker> {
   let promise: Promise<rerankers.RRFReranker> | null = null;
 

package/src/db/memory.repository.ts

@@ -1,7 +1,7 @@
 import * as lancedb from "@lancedb/lancedb";
 import { type Table } from "@lancedb/lancedb";
 import { TABLE_NAME, memorySchema } from "./schema.js";
-import { arrowVectorToArray, createFtsMutex, createRerankerMutex, escapeLanceDbString } from "./lancedb-utils.js";
+import { arrowVectorToArray, createFtsMutex, createRerankerMutex, escapeSql, safeParseJsonObject } from "./lancedb-utils.js";
 import {
   type Memory,
   type HybridRow,
@@ -83,7 +83,7 @@ export class MemoryRepository {
       id: row.id as string,
       content: row.content as string,
       embedding: arrowVectorToArray(row.vector),
-      metadata: JSON.parse(row.metadata as string),
+      metadata: safeParseJsonObject(row.metadata as string),
       createdAt: new Date(row.created_at as number),
       updatedAt: new Date(row.updated_at as number),
       supersededBy: row.superseded_by as string | null,
@@ -115,14 +115,14 @@ export class MemoryRepository {
 
   async upsert(memory: Memory): Promise<void> {
     const table = await this.getTable();
-    const existing = await table.query().where(`id = '${escapeLanceDbString(memory.id)}'`).limit(1).toArray();
+    const existing = await table.query().where(`id = '${escapeSql(memory.id)}'`).limit(1).toArray();
 
     if (existing.length === 0) {
       return await this.insert(memory);
     }
 
     await table.update({
-      where: `id = '${escapeLanceDbString(memory.id)}'`,
+      where: `id = '${escapeSql(memory.id)}'`,
       values: {
         vector: memory.embedding,
         content: memory.content,
@@ -139,7 +139,7 @@ export class MemoryRepository {
 
   async findById(id: string): Promise<Memory | null> {
     const table = await this.getTable();
-    const results = await table.query().where(`id = '${escapeLanceDbString(id)}'`).limit(1).toArray();
+    const results = await table.query().where(`id = '${escapeSql(id)}'`).limit(1).toArray();
 
     if (results.length === 0) {
       return null;
@@ -152,14 +152,14 @@ export class MemoryRepository {
     const table = await this.getTable();
 
     // Verify existence first to match previous behavior (return false if not found)
-    const existing = await table.query().where(`id = '${escapeLanceDbString(id)}'`).limit(1).toArray();
+    const existing = await table.query().where(`id = '${escapeSql(id)}'`).limit(1).toArray();
     if (existing.length === 0) {
       return false;
     }
 
     const now = Date.now();
     await table.update({
-      where: `id = '${escapeLanceDbString(id)}'`,
+      where: `id = '${escapeSql(id)}'`,
       values: {
         superseded_by: DELETED_TOMBSTONE,
         updated_at: now,

package/src/http/server.ts

@@ -85,6 +85,7 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
         dbPath: config.dbPath,
         embeddingModel: config.embeddingModel,
         embeddingDimension: config.embeddingDimension,
+        historyEnabled: config.conversationHistory.enabled,
       },
     });
   });
@@ -101,16 +102,17 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
         return c.json({ error: "Missing or invalid 'query' field" }, 400);
       }
 
-      const memories = await memoryService.search(query, intent, limit);
+      const results = await memoryService.search(query, intent, limit);
 
       return c.json({
-        memories: memories.map((m) => ({
-          id: m.id,
-          content: m.content,
-          metadata: m.metadata,
-          createdAt: m.createdAt.toISOString(),
+        results: results.map((r) => ({
+          id: r.id,
+          content: r.content,
+          metadata: r.metadata,
+          source: r.source,
+          createdAt: r.createdAt.toISOString(),
         })),
-        count: memories.length,
+        count: results.length,
       });
     } catch (error) {
       const message = error instanceof Error ? error.message : "Unknown error";
@@ -193,6 +195,28 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
     }
   });
 
+  // Index conversations (trigger incremental indexing)
+  app.post("/index-conversations", async (c) => {
+    try {
+      const conversationService = memoryService.getConversationService();
+      if (!conversationService) {
+        return c.json({ error: "Conversation history indexing is not enabled" }, 400);
+      }
+
+      const body = await c.req.json().catch(() => ({}));
+      const since = body.since ? new Date(body.since as string) : undefined;
+      const result = await conversationService.indexConversations(
+        body.path as string | undefined,
+        since
+      );
+
+      return c.json(result);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Unknown error";
+      return c.json({ error: message }, 500);
+    }
+  });
+
   // Get single memory
   app.get("/memories/:id", async (c) => {
     try {

package/src/index.ts

@@ -3,10 +3,10 @@
 import { loadConfig, parseCliArgs } from "./config/index.js";
 import { connectToDatabase } from "./db/connection.js";
 import { MemoryRepository } from "./db/memory.repository.js";
+import { ConversationRepository } from "./db/conversation.repository.js";
 import { EmbeddingsService } from "./services/embeddings.service.js";
 import { MemoryService } from "./services/memory.service.js";
-import { ConversationHistoryRepository } from "./db/conversation-history.repository.js";
-import { ConversationHistoryService } from "./services/conversation-history.service.js";
+import { ConversationHistoryService } from "./services/conversation.service.js";
 import { startServer } from "./mcp/server.js";
 import { startHttpServer } from "./http/server.js";
 
@@ -32,18 +32,17 @@ async function main(): Promise<void> {
   const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
   const memoryService = new MemoryService(repository, embeddings);
 
-  // Wire conversation history if enabled
+  // Conditionally initialize conversation history indexing
   if (config.conversationHistory.enabled) {
-    const historyRepo = new ConversationHistoryRepository(db);
-    const historyService = new ConversationHistoryService(
-      historyRepo,
+    const conversationRepository = new ConversationRepository(db);
+    const conversationService = new ConversationHistoryService(
+      conversationRepository,
       embeddings,
-      config.conversationHistory.sessionPath,
-    );
-    memoryService.setConversationHistory(
-      historyService,
-      config.conversationHistory.historyWeight,
+      config.conversationHistory,
+      config.dbPath
     );
+    memoryService.setConversationService(conversationService);
+    console.error("[vector-memory-mcp] Conversation history indexing enabled");
   }
 
   // Track cleanup functions