@aeriondyseti/vector-memory-mcp 1.0.2-dev.0 → 1.1.0-dev.2

package/src/db/lancedb-utils.ts

@@ -0,0 +1,97 @@
+import * as lancedb from "@lancedb/lancedb";
+import { Index, rerankers, type Table } from "@lancedb/lancedb";
+import type { Schema } from "apache-arrow";
+
+/**
+ * Escapes a string for use in LanceDB/DataFusion SQL WHERE clauses.
+ * Doubles single quotes to prevent SQL injection (standard SQL escaping).
+ */
+export function escapeLanceDbString(value: string): string {
+  return value.replace(/'/g, "''");
+}
+
+/**
+ * Converts LanceDB's Arrow Vector type to a plain number[].
+ * LanceDB returns an Arrow Vector object which is iterable but not an array.
+ */
+export function arrowVectorToArray(value: unknown): number[] {
+  return Array.isArray(value)
+    ? value
+    : (Array.from(value as Iterable<number>) as number[]);
+}
+
+/**
+ * Opens an existing table or creates it with the given schema.
+ * Does NOT cache — callers should cache the returned Table if desired.
+ */
+export async function getOrCreateTable(
+  db: lancedb.Connection,
+  name: string,
+  schema: Schema
+): Promise<Table> {
+  const names = await db.tableNames();
+  if (names.includes(name)) {
+    return await db.openTable(name);
+  }
+  return await db.createTable(name, [], { schema });
+}
+
+/**
+ * Creates a mutex-guarded function that ensures an FTS index exists on a table's content column.
+ *
+ * Once the FTS index is confirmed/created, the promise is retained for the lifetime of the
+ * caller — the index persists in LanceDB, so re-checking is unnecessary. On error, the
+ * mutex resets so the next call can retry.
+ *
+ * The key design constraint: the promise must be captured synchronously (before any await)
+ * to prevent concurrent callers from racing past the guard.
+ */
+export function createFtsMutex(
+  getTable: () => Promise<Table>
+): () => Promise<void> {
+  let promise: Promise<void> | null = null;
+
+  return () => {
+    if (promise) return promise;
+
+    promise = (async () => {
+      const table = await getTable();
+      const indices = await table.listIndices();
+      const hasFtsIndex = indices.some(
+        (idx) => idx.columns.includes("content") && idx.indexType === "FTS"
+      );
+
+      if (!hasFtsIndex) {
+        await table.createIndex("content", {
+          config: Index.fts(),
+        });
+        await table.waitForIndex(["content_idx"], 30);
+      }
+    })().catch((error) => {
+      promise = null;
+      throw error;
+    });
+
+    return promise;
+  };
+}
+
+/**
+ * Creates a promise-mutex for RRFReranker instantiation.
+ * Same pattern as createFtsMutex: create once, cache forever, reset on error.
+ */
+export function createRerankerMutex(
+  k: number = 60
+): () => Promise<rerankers.RRFReranker> {
+  let promise: Promise<rerankers.RRFReranker> | null = null;
+
+  return () => {
+    if (!promise) {
+      promise = rerankers.RRFReranker.create(k).catch((e) => {
+        promise = null;
+        throw e;
+      });
+    }
+    return promise;
+  };
+}

package/src/db/memory.repository.ts

@@ -1,6 +1,7 @@
 import * as lancedb from "@lancedb/lancedb";
-import { Index, rerankers, type Table } from "@lancedb/lancedb";
+import { type Table } from "@lancedb/lancedb";
 import { TABLE_NAME, memorySchema } from "./schema.js";
+import { arrowVectorToArray, createFtsMutex, createRerankerMutex, escapeLanceDbString } from "./lancedb-utils.js";
 import {
   type Memory,
   type HybridRow,
@@ -8,14 +9,18 @@ import {
 } from "../types/memory.js";
 
 export class MemoryRepository {
-  // Mutex for FTS index creation - ensures only one index creation runs at a time
-  // Once set, this promise is never cleared (FTS index persists in the database)
-  private ftsIndexPromise: Promise<void> | null = null;
-
   // Mutex for schema migration - runs once per instance to add missing columns
   private migrationPromise: Promise<void> | null = null;
 
-  constructor(private db: lancedb.Connection) { }
+  // FTS index mutex — once created, the promise is never cleared (index persists in LanceDB)
+  private ensureFtsIndex: () => Promise<void>;
+
+  // Cached reranker — k=60 is constant, no need to recreate per search
+  private getReranker = createRerankerMutex();
+
+  constructor(private db: lancedb.Connection) {
+    this.ensureFtsIndex = createFtsMutex(() => this.getTable());
+  }
 
   private async getTable() {
     const names = await this.db.tableNames();
@@ -70,63 +75,14 @@ export class MemoryRepository {
     }
   }
 
-  /**
-   * Ensures the FTS index exists on the content column.
-   * Uses a mutex pattern to prevent concurrent index creation.
-   * The key insight: we must capture the promise BEFORE any await.
-   */
-  private ensureFtsIndex(): Promise<void> {
-    // If there's already a pending or completed index creation, return that promise
-    if (this.ftsIndexPromise) {
-      return this.ftsIndexPromise;
-    }
-
-    // Synchronously set the promise BEFORE any await
-    // This is critical for proper mutex behavior in JS async code
-    this.ftsIndexPromise = this.createFtsIndexIfNeeded().catch((error) => {
-      // Reset on error so the next call can retry
-      this.ftsIndexPromise = null;
-      throw error;
-    });
-
-    return this.ftsIndexPromise;
-  }
-
-  /**
-   * Creates the FTS index if it doesn't already exist.
-   * Gets its own table reference to ensure consistent index state.
-   */
-  private async createFtsIndexIfNeeded(): Promise<void> {
-    const table = await this.getTable();
-    const indices = await table.listIndices();
-    const hasFtsIndex = indices.some(
-      (idx) => idx.columns.includes("content") && idx.indexType === "FTS"
-    );
-
-    if (!hasFtsIndex) {
-      await table.createIndex("content", {
-        config: Index.fts(),
-      });
-      // Wait for the index to be fully created and available
-      await table.waitForIndex(["content_idx"], 30);
-    }
-  }
-
   /**
    * Converts a raw LanceDB row to a Memory object.
    */
   private rowToMemory(row: Record<string, unknown>): Memory {
-    // Handle Arrow Vector type conversion
-    // LanceDB returns an Arrow Vector object which is iterable but not an array
-    const vectorData = row.vector as unknown;
-    const embedding = Array.isArray(vectorData)
-      ? vectorData
-      : Array.from(vectorData as Iterable<number>) as number[];
-
     return {
       id: row.id as string,
       content: row.content as string,
-      embedding,
+      embedding: arrowVectorToArray(row.vector),
       metadata: JSON.parse(row.metadata as string),
       createdAt: new Date(row.created_at as number),
       updatedAt: new Date(row.updated_at as number),
@@ -159,14 +115,14 @@ export class MemoryRepository {
 
   async upsert(memory: Memory): Promise<void> {
     const table = await this.getTable();
-    const existing = await table.query().where(`id = '${memory.id}'`).limit(1).toArray();
+    const existing = await table.query().where(`id = '${escapeLanceDbString(memory.id)}'`).limit(1).toArray();
 
     if (existing.length === 0) {
       return await this.insert(memory);
     }
 
     await table.update({
-      where: `id = '${memory.id}'`,
+      where: `id = '${escapeLanceDbString(memory.id)}'`,
       values: {
         vector: memory.embedding,
         content: memory.content,
@@ -183,7 +139,7 @@ export class MemoryRepository {
 
   async findById(id: string): Promise<Memory | null> {
     const table = await this.getTable();
-    const results = await table.query().where(`id = '${id}'`).limit(1).toArray();
+    const results = await table.query().where(`id = '${escapeLanceDbString(id)}'`).limit(1).toArray();
 
     if (results.length === 0) {
       return null;
@@ -196,14 +152,14 @@ export class MemoryRepository {
     const table = await this.getTable();
 
     // Verify existence first to match previous behavior (return false if not found)
-    const existing = await table.query().where(`id = '${id}'`).limit(1).toArray();
+    const existing = await table.query().where(`id = '${escapeLanceDbString(id)}'`).limit(1).toArray();
     if (existing.length === 0) {
       return false;
     }
 
     const now = Date.now();
     await table.update({
-      where: `id = '${id}'`,
+      where: `id = '${escapeLanceDbString(id)}'`,
       values: {
         superseded_by: DELETED_TOMBSTONE,
         updated_at: now,
@@ -223,16 +179,11 @@ export class MemoryRepository {
    * @returns Array of HybridRow containing full Memory data plus RRF score
    */
   async findHybrid(embedding: number[], query: string, limit: number): Promise<HybridRow[]> {
-    // Ensure FTS index exists (with mutex to prevent concurrent creation)
-    // This must happen BEFORE getTable to ensure proper mutex behavior
     await this.ensureFtsIndex();
 
     const table = await this.getTable();
+    const reranker = await this.getReranker();
 
-    // Create RRF reranker with k=60 (standard parameter)
-    const reranker = await rerankers.RRFReranker.create(60);
-
-    // Perform hybrid search: combine vector search and full-text search
     const results = await table
       .query()
       .nearestTo(embedding)

package/src/db/schema.ts

@@ -9,34 +9,30 @@ import {
   Int32,
 } from "apache-arrow";
 
-export const TABLE_NAME = "memories";
+// Shared field helpers — used by both memory and conversation history schemas
+export const EMBEDDING_DIMENSION = 384;
 
-export const memorySchema = new Schema([
-  new Field("id", new Utf8(), false),
+export const vectorField = () =>
   new Field(
     "vector",
-    new FixedSizeList(384, new Field("item", new Float32())),
+    new FixedSizeList(EMBEDDING_DIMENSION, new Field("item", new Float32())),
     false
-  ),
+  );
+
+export const timestampField = (name: string, nullable = false) =>
+  new Field(name, new Timestamp(TimeUnit.MILLISECOND), nullable);
+
+export const TABLE_NAME = "memories";
+
+export const memorySchema = new Schema([
+  new Field("id", new Utf8(), false),
+  vectorField(),
   new Field("content", new Utf8(), false),
   new Field("metadata", new Utf8(), false), // JSON string
-  new Field(
-    "created_at",
-    new Timestamp(TimeUnit.MILLISECOND),
-    false
-  ),
-  new Field(
-    "updated_at",
-    new Timestamp(TimeUnit.MILLISECOND),
-    false
-  ),
+  timestampField("created_at"),
+  timestampField("updated_at"),
   new Field("superseded_by", new Utf8(), true), // Nullable
   new Field("usefulness", new Float32(), false),
   new Field("access_count", new Int32(), false),
-  new Field(
-    "last_accessed",
-    new Timestamp(TimeUnit.MILLISECOND),
-    true
-  ),
+  timestampField("last_accessed", true),
 ]);
-

package/src/index.ts

@@ -5,6 +5,8 @@ import { connectToDatabase } from "./db/connection.js";
 import { MemoryRepository } from "./db/memory.repository.js";
 import { EmbeddingsService } from "./services/embeddings.service.js";
 import { MemoryService } from "./services/memory.service.js";
+import { ConversationHistoryRepository } from "./db/conversation-history.repository.js";
+import { ConversationHistoryService } from "./services/conversation-history.service.js";
 import { startServer } from "./mcp/server.js";
 import { startHttpServer } from "./http/server.js";
 
@@ -30,6 +32,20 @@ async function main(): Promise<void> {
   const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
   const memoryService = new MemoryService(repository, embeddings);
 
+  // Wire conversation history if enabled
+  if (config.conversationHistory.enabled) {
+    const historyRepo = new ConversationHistoryRepository(db);
+    const historyService = new ConversationHistoryService(
+      historyRepo,
+      embeddings,
+      config.conversationHistory.sessionPath,
+    );
+    memoryService.setConversationHistory(
+      historyService,
+      config.conversationHistory.historyWeight,
+    );
+  }
+
   // Track cleanup functions
   let httpStop: (() => void) | null = null;
 

package/src/mcp/handlers.ts

@@ -1,6 +1,27 @@
 import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
 import type { MemoryService } from "../services/memory.service.js";
+import type { ConversationHistoryService } from "../services/conversation-history.service.js";
 import type { SearchIntent } from "../types/memory.js";
+import type { SearchResult } from "../types/conversation-history.js";
+
+/**
+ * Guard: returns the ConversationHistoryService or an error CallToolResult.
+ */
+function requireHistoryService(
+  service: MemoryService,
+): ConversationHistoryService | CallToolResult {
+  const historyService = service.getConversationHistory();
+  if (!historyService) {
+    return {
+      content: [{
+        type: "text",
+        text: "Conversation history indexing is not enabled. Set conversationHistory.enabled = true in config.",
+      }],
+      isError: true,
+    };
+  }
+  return historyService;
+}
 
 export async function handleStoreMemories(
   args: Record<string, unknown> | undefined,
@@ -103,9 +124,50 @@ export async function handleSearchMemories(
 ): Promise<CallToolResult> {
   const query = args?.query as string;
   const intent = (args?.intent as SearchIntent) ?? "fact_check";
-  const _reasonForSearch = args?.reason_for_search as string; // Logged but not used in logic
   const limit = (args?.limit as number) ?? 10;
   const includeDeleted = (args?.include_deleted as boolean) ?? false;
+  const includeHistory = (args?.include_history as boolean) ?? false;
+  const historyOnly = (args?.history_only as boolean) ?? false;
+
+  if (includeHistory && historyOnly) {
+    return {
+      content: [{
+        type: "text",
+        text: "Cannot set both include_history and history_only to true. Use history_only for conversation history only, or include_history to merge with memories.",
+      }],
+      isError: true,
+    };
+  }
+
+  // History-only: search only conversation history via the history service
+  if (historyOnly) {
+    const result = requireHistoryService(service);
+    if ("content" in result) return result;
+    const historyResults = await result.search(query, limit);
+    if (historyResults.length === 0) {
+      return {
+        content: [{ type: "text", text: "No conversation history found matching your query." }],
+      };
+    }
+    return {
+      content: [{ type: "text", text: historyResults.map((r) => formatSearchResult(r)).join("\n\n---\n\n") }],
+    };
+  }
+
+  // Unified search: merge memories + history
+  if (includeHistory) {
+    const results = await service.searchUnified(query, intent, limit, includeDeleted);
+    if (results.length === 0) {
+      return {
+        content: [{ type: "text", text: "No results found matching your query." }],
+      };
+    }
+    return {
+      content: [{ type: "text", text: results.map((r) => formatSearchResult(r, includeDeleted)).join("\n\n---\n\n") }],
+    };
+  }
+
+  // Default: memory-only search (original behavior)
   const memories = await service.search(query, intent, limit, includeDeleted);
 
   if (memories.length === 0) {
@@ -130,36 +192,61 @@ export async function handleSearchMemories(
   };
 }
 
+function formatMemoryDetail(
+  memoryId: string,
+  memory: Awaited<ReturnType<MemoryService["get"]>>
+): string {
+  if (!memory) {
+    return `Memory ${memoryId} not found`;
+  }
+
+  let result = `ID: ${memory.id}\nContent: ${memory.content}`;
+  if (Object.keys(memory.metadata).length > 0) {
+    result += `\nMetadata: ${JSON.stringify(memory.metadata)}`;
+  }
+  result += `\nCreated: ${memory.createdAt.toISOString()}`;
+  result += `\nUpdated: ${memory.updatedAt.toISOString()}`;
+  if (memory.supersededBy) {
+    result += `\nSuperseded by: ${memory.supersededBy}`;
+  }
+  return result;
+}
+
+/**
+ * Format a unified SearchResult (memory or history) for display.
+ * TODO: The default memory-only search path in handleSearchMemories formats results inline
+ * with similar logic but without the "Source:" label. Consolidating would add "Source: memory"
+ * to existing output, which may break consumers that parse it. (#5)
+ */
+function formatSearchResult(result: SearchResult, includeDeleted: boolean = false): string {
+  if (result.source === "memory") {
+    let text = `ID: ${result.id}\nSource: memory\nContent: ${result.content}`;
+    if (Object.keys(result.metadata).length > 0) {
+      text += `\nMetadata: ${JSON.stringify(result.metadata)}`;
+    }
+    if (includeDeleted && result.supersededBy) {
+      text += `\n[DELETED]`;
+    }
+    return text;
+  }
+  // conversation_history
+  let text = `ID: ${result.id}\nSource: conversation_history\nSession: ${result.sessionId}\nRole: ${result.role}\nTimestamp: ${result.timestamp.toISOString()}\nContent: ${result.content}`;
+  if (Object.keys(result.metadata).length > 0) {
+    text += `\nMetadata: ${JSON.stringify(result.metadata)}`;
+  }
+  return text;
+}
+
 export async function handleGetMemories(
   args: Record<string, unknown> | undefined,
   service: MemoryService
 ): Promise<CallToolResult> {
   const ids = args?.ids as string[];
 
-  const format = (
-    memoryId: string,
-    memory: Awaited<ReturnType<MemoryService["get"]>>
-  ) => {
-    if (!memory) {
-      return `Memory ${memoryId} not found`;
-    }
-
-    let result = `ID: ${memory.id}\nContent: ${memory.content}`;
-    if (Object.keys(memory.metadata).length > 0) {
-      result += `\nMetadata: ${JSON.stringify(memory.metadata)}`;
-    }
-    result += `\nCreated: ${memory.createdAt.toISOString()}`;
-    result += `\nUpdated: ${memory.updatedAt.toISOString()}`;
-    if (memory.supersededBy) {
-      result += `\nSuperseded by: ${memory.supersededBy}`;
-    }
-    return result;
-  };
-
   const blocks: string[] = [];
   for (const id of ids) {
     const memory = await service.get(id);
-    blocks.push(format(id, memory));
+    blocks.push(formatMemoryDetail(id, memory));
   }
 
   return {
@@ -248,6 +335,69 @@ export async function handleGetCheckpoint(
   };
 }
 
+export async function handleIndexConversations(
+  args: Record<string, unknown> | undefined,
+  service: MemoryService
+): Promise<CallToolResult> {
+  const result = requireHistoryService(service);
+  if ("content" in result) return result;
+
+  const path = args?.path as string | undefined;
+  const summary = await result.indexConversations(path);
+
+  return {
+    content: [{
+      type: "text",
+      text: `Indexing complete.\n- Sessions discovered: ${summary.sessionsDiscovered}\n- Sessions indexed: ${summary.sessionsIndexed}\n- Sessions skipped (unchanged): ${summary.sessionsSkipped}\n- Messages indexed: ${summary.messagesIndexed}`,
+    }],
+  };
+}
+
+export async function handleListIndexedSessions(
+  _args: Record<string, unknown> | undefined,
+  service: MemoryService
+): Promise<CallToolResult> {
+  const result = requireHistoryService(service);
+  if ("content" in result) return result;
+
+  const sessions = await result.listIndexedSessions();
+
+  if (sessions.length === 0) {
+    return {
+      content: [{ type: "text", text: "No indexed sessions found. Run index_conversations first." }],
+    };
+  }
+
+  const lines = sessions.map((s) => {
+    let line = `Session: ${s.sessionId}\n  Messages: ${s.messageCount}\n  First: ${s.firstMessageAt.toISOString()}\n  Last: ${s.lastMessageAt.toISOString()}\n  Indexed: ${s.indexedAt.toISOString()}`;
+    if (s.project) line += `\n  Project: ${s.project}`;
+    if (s.gitBranch) line += `\n  Branch: ${s.gitBranch}`;
+    return line;
+  });
+
+  return {
+    content: [{ type: "text", text: `${sessions.length} indexed session(s):\n\n${lines.join("\n\n")}` }],
+  };
+}
+
+export async function handleReindexSession(
+  args: Record<string, unknown> | undefined,
+  service: MemoryService
+): Promise<CallToolResult> {
+  const result = requireHistoryService(service);
+  if ("content" in result) return result;
+
+  const sessionId = args?.session_id as string;
+  const summary = await result.reindexSession(sessionId);
+
+  return {
+    content: [{
+      type: "text",
+      text: `Reindex complete for session ${sessionId}.\n- Messages indexed: ${summary.messagesIndexed}`,
+    }],
+  };
+}
+
 export async function handleToolCall(
   name: string,
   args: Record<string, unknown> | undefined,
@@ -270,6 +420,12 @@ export async function handleToolCall(
       return handleStoreCheckpoint(args, service);
     case "get_checkpoint":
       return handleGetCheckpoint(args, service);
+    case "index_conversations":
+      return handleIndexConversations(args, service);
+    case "list_indexed_sessions":
+      return handleListIndexedSessions(args, service);
+    case "reindex_session":
+      return handleReindexSession(args, service);
     default:
       return {
         content: [{ type: "text", text: `Unknown tool: ${name}` }],

package/src/mcp/tools.ts

@@ -54,7 +54,7 @@ For long content (>1000 chars), provide embedding_text with a searchable summary
     },
     required: ["memories"],
   },
-};;
+};
 
 export const deleteMemoriesTool: Tool = {
   name: "delete_memories",
@@ -74,7 +74,7 @@ export const deleteMemoriesTool: Tool = {
     },
     required: ["ids"],
   },
-};;
+};
 
 
 const updateMemoriesTool: Tool = {
@@ -167,6 +167,20 @@ When in doubt, search. Missing context is costlier than an extra query.`,
         description: "Include soft-deleted memories in results (default: false). Useful for recovering prior information.",
         default: false,
       },
+      include_history: {
+        type: "boolean",
+        description:
+          "Include conversation history results alongside memories (default: false). " +
+          "Requires conversation history indexing to be enabled. History results are weighted lower than explicit memories.",
+        default: false,
+      },
+      history_only: {
+        type: "boolean",
+        description:
+          "Search only conversation history, excluding explicit memories (default: false). " +
+          "Requires conversation history indexing to be enabled.",
+        default: false,
+      },
     },
     required: ["query", "intent", "reason_for_search"],
   },
@@ -187,7 +201,7 @@ export const getMemoriesTool: Tool = {
     },
     required: ["ids"],
   },
-};;
+};
 
 export const reportMemoryUsefulnessTool: Tool = {
   name: "report_memory_usefulness",
@@ -272,6 +286,52 @@ export const getCheckpointTool: Tool = {
   },
 };
 
+export const indexConversationsTool: Tool = {
+  name: "index_conversations",
+  description:
+    "Index Claude Code session logs for searchable conversation history. " +
+    "Scans for JSONL session files, detects new/changed sessions, and indexes text messages. " +
+    "Skips unchanged files. Run periodically or after significant work sessions.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      path: {
+        type: "string",
+        description:
+          "Directory containing session JSONL files. If omitted, auto-detects the Claude Code sessions directory.",
+      },
+    },
+  },
+};
+
+export const listIndexedSessionsTool: Tool = {
+  name: "list_indexed_sessions",
+  description:
+    "List all conversation sessions that have been indexed. " +
+    "Shows session ID, message count, time range, and associated project/branch.",
+  inputSchema: {
+    type: "object",
+    properties: {},
+  },
+};
+
+export const reindexSessionTool: Tool = {
+  name: "reindex_session",
+  description:
+    "Force a full re-index of a specific session. Deletes all existing entries for the session and re-parses from scratch. " +
+    "Use when session data appears corrupted or after parser improvements.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      session_id: {
+        type: "string",
+        description: "The session ID to reindex.",
+      },
+    },
+    required: ["session_id"],
+  },
+};
+
 export const tools: Tool[] = [
   storeMemoriesTool,
   updateMemoriesTool,
@@ -281,4 +341,7 @@ export const tools: Tool[] = [
   reportMemoryUsefulnessTool,
   storeCheckpointTool,
   getCheckpointTool,
+  indexConversationsTool,
+  listIndexedSessionsTool,
+  reindexSessionTool,
 ];