@synth-coder/memhub 0.1.6 → 0.2.1

package/src/services/memory-service.ts

@@ -1,7 +1,3 @@
-/**
- * Memory Service - Business logic for memory operations
- */
-
 import { randomUUID } from 'crypto';
 import type {
   Memory,
@@ -28,6 +24,19 @@ import type {
 import { ErrorCode } from '../contracts/types.js';
 import { MarkdownStorage, StorageError } from '../storage/markdown-storage.js';
 
+/** Minimal interface required from VectorIndex (avoids static import of native module) */
+interface IVectorIndex {
+  upsert(memory: Memory, vector: number[]): Promise<void>;
+  delete(id: string): Promise<void>;
+  search(vector: number[], limit?: number): Promise<Array<{ id: string; _distance: number }>>;
+}
+
+/** Minimal interface required from EmbeddingService */
+interface IEmbeddingService {
+  embedMemory(title: string, content: string): Promise<number[]>;
+  embed(text: string): Promise<number[]>;
+}
+
 /**
  * Custom error for service operations
  */
@@ -47,6 +56,12 @@ export class ServiceError extends Error {
  */
 export interface MemoryServiceConfig {
   storagePath: string;
+  /**
+   * Enable vector semantic search via LanceDB + local ONNX model.
+   * Set to false in unit tests to avoid loading the model.
+   * @default true
+   */
+  vectorSearch?: boolean;
 }
 
 /**
@@ -54,17 +69,107 @@ export interface MemoryServiceConfig {
  */
 export class MemoryService {
   private readonly storage: MarkdownStorage;
+  private readonly vectorIndex: IVectorIndex | null;
+  private readonly embedding: IEmbeddingService | null;
+  private readonly vectorSearchEnabled: boolean;
 
   constructor(config: MemoryServiceConfig) {
     this.storage = new MarkdownStorage({ storagePath: config.storagePath });
+    this.vectorSearchEnabled = config.vectorSearch !== false;
+
+    if (this.vectorSearchEnabled) {
+      // Lazily resolved at runtime — do not use top-level static imports so that
+      // native modules (onnxruntime-node, sharp) are never loaded when vectorSearch=false.
+      let resolvedVectorIndex: IVectorIndex | null = null;
+      let resolvedEmbedding: IEmbeddingService | null = null;
+
+      // Kick off async initialisation without blocking the constructor.
+      // The proxy objects below delegate to the real instances once ready.
+      const storagePath = config.storagePath;
+      const initPromise = (async () => {
+        const [{ VectorIndex }, { EmbeddingService }] = await Promise.all([
+          import('../storage/vector-index.js'),
+          import('./embedding-service.js'),
+        ]);
+        resolvedVectorIndex = new VectorIndex(storagePath);
+        resolvedEmbedding = EmbeddingService.getInstance();
+      })();
+
+      // Lightweight proxy that waits for init before delegating
+      this.vectorIndex = {
+        upsert: async (memory, vector) => {
+          await initPromise;
+          return resolvedVectorIndex!.upsert(memory, vector);
+        },
+        delete: async (id) => {
+          await initPromise;
+          return resolvedVectorIndex!.delete(id);
+        },
+        search: async (vector, limit) => {
+          await initPromise;
+          return resolvedVectorIndex!.search(vector, limit);
+        },
+      };
+
+      this.embedding = {
+        embedMemory: async (title, content) => {
+          await initPromise;
+          return resolvedEmbedding!.embedMemory(title, content);
+        },
+        embed: async (text) => {
+          await initPromise;
+          return resolvedEmbedding!.embed(text);
+        },
+      };
+    } else {
+      this.vectorIndex = null;
+      this.embedding = null;
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // Internal helpers
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Asynchronously embeds a memory and upserts it into the vector index.
+   * Fire-and-forget: failures are logged but do not propagate.
+   */
+  private scheduleVectorUpsert(memory: Memory): void {
+    if (!this.vectorIndex || !this.embedding) return;
+
+    const vectorIndex = this.vectorIndex;
+    const embedding = this.embedding;
+
+    // Intentionally not awaited
+    embedding
+      .embedMemory(memory.title, memory.content)
+      .then(vec => vectorIndex.upsert(memory, vec))
+      .catch(err => {
+        // Non-fatal: Markdown file is the source of truth
+        console.error('[MemHub] Vector upsert failed (non-fatal):', err);
+      });
+  }
+
+  /**
+   * Removes a memory from the vector index.
+   * Called synchronously (awaited) on delete.
+   */
+  private async removeFromVectorIndex(id: string): Promise<void> {
+    if (!this.vectorIndex) return;
+    try {
+      await this.vectorIndex.delete(id);
+    } catch (err) {
+      console.error('[MemHub] Vector delete failed (non-fatal):', err);
+    }
   }
 
+  // ---------------------------------------------------------------------------
+  // CRUD operations
+  // ---------------------------------------------------------------------------
+
   /**
    * Creates a new memory entry
-   *
-   * @param input - Create memory input
-   * @returns Create result with ID, file path, and memory object
-   * @throws ServiceError if creation fails
    */
   async create(input: CreateMemoryInput): Promise<CreateResult> {
     const now = new Date().toISOString();
@@ -83,11 +188,8 @@ export class MemoryService {
 
     try {
       const filePath = await this.storage.write(memory);
-      return {
-        id,
-        filePath,
-        memory,
-      };
+      this.scheduleVectorUpsert(memory);
+      return { id, filePath, memory };
     } catch (error) {
       throw new ServiceError(
         `Failed to create memory: ${error instanceof Error ? error.message : 'Unknown error'}`,
@@ -98,10 +200,6 @@ export class MemoryService {
 
   /**
    * Reads a memory by ID
-   *
-   * @param input - Read memory input
-   * @returns Memory object
-   * @throws ServiceError if memory not found
    */
   async read(input: ReadMemoryInput): Promise<{ memory: Memory }> {
     try {
@@ -120,13 +218,8 @@ export class MemoryService {
 
   /**
    * Updates an existing memory
-   *
-   * @param input - Update memory input
-   * @returns Updated memory object
-   * @throws ServiceError if memory not found
    */
   async update(input: UpdateMemoryInput): Promise<UpdateResult> {
-    // First read the existing memory
     let existing: Memory;
     try {
       existing = await this.storage.read(input.id);
@@ -140,7 +233,6 @@ export class MemoryService {
       );
     }
 
-    // Apply updates
     const updated: Memory = {
       ...existing,
       updatedAt: new Date().toISOString(),
@@ -153,6 +245,7 @@ export class MemoryService {
 
     try {
       await this.storage.write(updated);
+      this.scheduleVectorUpsert(updated);
       return { memory: updated };
     } catch (error) {
       throw new ServiceError(
@@ -164,18 +257,12 @@ export class MemoryService {
 
   /**
    * Deletes a memory by ID
-   *
-   * @param input - Delete memory input
-   * @returns Delete result
-   * @throws ServiceError if memory not found
    */
   async delete(input: DeleteMemoryInput): Promise<DeleteResult> {
     try {
       const filePath = await this.storage.delete(input.id);
-      return {
-        success: true,
-        filePath,
-      };
+      await this.removeFromVectorIndex(input.id);
+      return { success: true, filePath };
     } catch (error) {
       if (error instanceof StorageError && error.message.includes('not found')) {
         throw new ServiceError(`Memory not found: ${input.id}`, ErrorCode.NOT_FOUND);
@@ -187,17 +274,17 @@ export class MemoryService {
     }
   }
 
+  // ---------------------------------------------------------------------------
+  // List / Search
+  // ---------------------------------------------------------------------------
+
   /**
    * Lists memories with filtering and pagination
-   *
-   * @param input - List memory input
-   * @returns List result with memories, total count, and hasMore flag
    */
   async list(input: ListMemoryInput): Promise<ListResult> {
     try {
       const files = await this.storage.list();
 
-      // Parse all files into memories
       let memories: Memory[] = [];
       for (const file of files) {
         try {
@@ -206,31 +293,25 @@ export class MemoryService {
           );
           memories.push(memory);
         } catch {
-          // Skip invalid files
           continue;
         }
       }
 
-      // Apply filters
       if (input.category) {
         memories = memories.filter(m => m.category === input.category);
       }
-
       if (input.tags && input.tags.length > 0) {
         memories = memories.filter(m =>
           input.tags!.every(tag => m.tags.includes(tag))
         );
       }
-
       if (input.fromDate) {
         memories = memories.filter(m => m.createdAt >= input.fromDate!);
       }
-
       if (input.toDate) {
         memories = memories.filter(m => m.createdAt <= input.toDate!);
       }
 
-      // Sort
       const sortBy: SortField = input.sortBy ?? 'createdAt';
       const sortOrder: SortOrder = input.sortOrder ?? 'desc';
 
@@ -253,18 +334,15 @@ export class MemoryService {
         return sortOrder === 'asc' ? comparison : -comparison;
       });
 
-      // Apply pagination
       const total = memories.length;
       const limit = input.limit ?? 20;
       const offset = input.offset ?? 0;
-
       const paginatedMemories = memories.slice(offset, offset + limit);
-      const hasMore = offset + limit < total;
 
       return {
         memories: paginatedMemories,
         total,
-        hasMore,
+        hasMore: offset + limit < total,
       };
     } catch (error) {
       throw new ServiceError(
@@ -275,116 +353,148 @@ export class MemoryService {
   }
 
   /**
-   * Searches memories by query
-   *
-   * @param input - Search memory input
-   * @returns Search results with scores and matches
+   * Searches memories by query.
+   * Uses vector semantic search when available, falls back to keyword search.
    */
   async search(input: SearchMemoryInput): Promise<{ results: SearchResult[]; total: number }> {
-    try {
-      const listResult = await this.list({
-        category: input.category,
-        tags: input.tags,
-        limit: 1000, // Get all for search
-      });
-
-      const query = input.query.toLowerCase();
-      const keywords = query.split(/\s+/).filter(k => k.length > 0);
-
-      const results: SearchResult[] = [];
+    // --- Vector semantic search path ---
+    if (this.vectorSearchEnabled && this.vectorIndex && this.embedding) {
+      try {
+        const queryVec = await this.embedding.embed(input.query);
+        const vectorResults = await this.vectorIndex.search(
+          queryVec,
+          input.limit ?? 10
+        );
 
-      for (const memory of listResult.memories) {
-        let score = 0;
-        const matches: string[] = [];
-
-        // Search in title (higher weight)
-        const titleLower = memory.title.toLowerCase();
-        if (titleLower.includes(query)) {
-          score += 10;
-          matches.push(memory.title);
-        } else {
-          // Check individual keywords in title
-          for (const keyword of keywords) {
-            if (titleLower.includes(keyword)) {
-              score += 5;
-              if (!matches.includes(memory.title)) {
-                matches.push(memory.title);
-              }
+        const results: SearchResult[] = [];
+        for (const vr of vectorResults) {
+          try {
+            const { memory } = await this.read({ id: vr.id });
+
+            // Apply metadata filters
+            if (input.category && memory.category !== input.category) continue;
+            if (
+              input.tags &&
+              input.tags.length > 0 &&
+              !input.tags.every(t => memory.tags.includes(t))
+            ) {
+              continue;
             }
+
+            // Convert cosine distance (0‥2) → similarity score (0‥1)
+            const score = Math.max(0, 1 - vr._distance / 2);
+            results.push({ memory, score, matches: [memory.title] });
+          } catch {
+            // Memory in index but missing on disk — skip
           }
         }
 
-        // Search in content
-        const contentLower = memory.content.toLowerCase();
-        if (contentLower.includes(query)) {
-          score += 3;
-          // Extract matching snippet
-          const index = contentLower.indexOf(query);
-          const start = Math.max(0, index - 50);
-          const end = Math.min(contentLower.length, index + query.length + 50);
-          const snippet = memory.content.slice(start, end);
-          matches.push(snippet);
-        } else {
-          // Check individual keywords in content
-          for (const keyword of keywords) {
-            if (contentLower.includes(keyword)) {
-              score += 1;
-              const index = contentLower.indexOf(keyword);
-              const start = Math.max(0, index - 30);
-              const end = Math.min(contentLower.length, index + keyword.length + 30);
-              const snippet = memory.content.slice(start, end);
-              if (!matches.some(m => m.includes(snippet))) {
-                matches.push(snippet);
-              }
-            }
+        return { results, total: results.length };
+      } catch (err) {
+        // Fall through to keyword search on vector failure
+        console.error('[MemHub] Vector search failed, falling back to keyword search:', err);
+      }
+    }
+
+    // --- Keyword search fallback ---
+    return this.keywordSearch(input);
+  }
+
+  /**
+   * Legacy keyword-based search (used as fallback when vector search is unavailable).
+   */
+  private async keywordSearch(
+    input: SearchMemoryInput
+  ): Promise<{ results: SearchResult[]; total: number }> {
+    const listResult = await this.list({
+      category: input.category,
+      tags: input.tags,
+      limit: 1000,
+    });
+
+    const query = input.query.toLowerCase();
+    const keywords = query.split(/\s+/).filter(k => k.length > 0);
+    const results: SearchResult[] = [];
+
+    for (const memory of listResult.memories) {
+      let score = 0;
+      const matches: string[] = [];
+
+      const titleLower = memory.title.toLowerCase();
+      if (titleLower.includes(query)) {
+        score += 10;
+        matches.push(memory.title);
+      } else {
+        for (const keyword of keywords) {
+          if (titleLower.includes(keyword)) {
+            score += 5;
+            if (!matches.includes(memory.title)) matches.push(memory.title);
           }
         }
+      }
 
-        // Search in tags
-        for (const tag of memory.tags) {
-          if (tag.toLowerCase().includes(query) || keywords.some(k => tag.toLowerCase().includes(k))) {
-            score += 2;
-            matches.push(`Tag: ${tag}`);
+      const contentLower = memory.content.toLowerCase();
+      if (contentLower.includes(query)) {
+        score += 3;
+        const index = contentLower.indexOf(query);
+        const start = Math.max(0, index - 50);
+        const end = Math.min(contentLower.length, index + query.length + 50);
+        matches.push(memory.content.slice(start, end));
+      } else {
+        for (const keyword of keywords) {
+          if (contentLower.includes(keyword)) {
+            score += 1;
+            const index = contentLower.indexOf(keyword);
+            const start = Math.max(0, index - 30);
+            const end = Math.min(contentLower.length, index + keyword.length + 30);
+            const snippet = memory.content.slice(start, end);
+            if (!matches.some(m => m.includes(snippet))) matches.push(snippet);
           }
         }
+      }
 
-        if (score > 0) {
-          results.push({
-            memory,
-            score: Math.min(score / 20, 1), // Normalize to 0-1
-            matches: matches.slice(0, 3), // Limit matches
-          });
+      for (const tag of memory.tags) {
+        if (
+          tag.toLowerCase().includes(query) ||
+          keywords.some(k => tag.toLowerCase().includes(k))
+        ) {
+          score += 2;
+          matches.push(`Tag: ${tag}`);
         }
       }
 
-      // Sort by score descending
-      results.sort((a, b) => b.score - a.score);
-
-      // Apply limit
-      const limit = input.limit ?? 10;
-      const limitedResults = results.slice(0, limit);
-
-      return {
-        results: limitedResults,
-        total: results.length,
-      };
-    } catch (error) {
-      throw new ServiceError(
-        `Failed to search memories: ${error instanceof Error ? error.message : 'Unknown error'}`,
-        ErrorCode.STORAGE_ERROR
-      );
+      if (score > 0) {
+        results.push({
+          memory,
+          score: Math.min(score / 20, 1),
+          matches: matches.slice(0, 3),
+        });
+      }
     }
+
+    results.sort((a, b) => b.score - a.score);
+    const limit = input.limit ?? 10;
+    return { results: results.slice(0, limit), total: results.length };
   }
 
+  // ---------------------------------------------------------------------------
+  // MCP unified tools
+  // ---------------------------------------------------------------------------
+
   /**
-   * memory_load unified read API (STM-first)
+   * memory_load — unified read API.
+   *
+   * Requires either `id` (exact lookup) or `query` (semantic search).
+   * Calling without either returns an empty result.
    */
   async memoryLoad(input: MemoryLoadInput): Promise<MemoryLoadOutput> {
+    // By-ID lookup
     if (input.id) {
       const { memory } = await this.read({ id: input.id });
       return { items: [memory], total: 1 };
     }
 
+    // Semantic / keyword search
     if (input.query) {
       const searched = await this.search({
         query: input.query,
@@ -392,37 +502,16 @@ export class MemoryService {
         tags: input.tags,
         limit: input.limit,
       });
-      let items = searched.results.map(r => r.memory);
-      if (input.sessionId) {
-        items = items.filter(m => m.sessionId === input.sessionId);
-      }
+      const items = searched.results.map(r => r.memory);
       return { items, total: items.length };
     }
 
-    const listResult = await this.list({
-      category: input.category,
-      tags: input.tags,
-      limit: input.limit ?? 20,
-      sortBy: 'updatedAt',
-      sortOrder: 'desc',
-    });
-
-    let items = [...listResult.memories];
-
-    if (input.sessionId) {
-      items = items.filter(m => m.sessionId === input.sessionId);
-    }
-
-    if (input.date) {
-      const date = input.date;
-      items = items.filter(m => m.createdAt.startsWith(date));
-    }
-
-    return { items, total: items.length };
+    // No id and no query — return empty (not supported)
+    return { items: [], total: 0 };
   }
 
   /**
-   * memory_update unified write API (append/upsert)
+   * memory_update — unified write API (append/upsert)
    */
   async memoryUpdate(input: MemoryUpdateInput): Promise<MemoryUpdateOutput> {
     const now = new Date().toISOString();
@@ -445,6 +534,8 @@ export class MemoryService {
       };
 
       const filePath = await this.storage.write(updatedMemory);
+      this.scheduleVectorUpsert(updatedMemory);
+
       return {
         id: updatedMemory.id,
         sessionId,
@@ -470,6 +561,8 @@ export class MemoryService {
     };
 
     const filePath = await this.storage.write(createdMemory);
+    this.scheduleVectorUpsert(createdMemory);
+
     return {
       id,
       sessionId,
@@ -480,23 +573,18 @@ export class MemoryService {
     };
   }
 
-  /**
-   * Gets all unique categories
-   *
-   * @returns Array of category names
-   */
+  // ---------------------------------------------------------------------------
+  // Metadata helpers
+  // ---------------------------------------------------------------------------
+
   async getCategories(): Promise<GetCategoriesOutput> {
     try {
       const listResult = await this.list({ limit: 1000 });
       const categories = new Set<string>();
-
       for (const memory of listResult.memories) {
         categories.add(memory.category);
       }
-
-      return {
-        categories: Array.from(categories).sort(),
-      };
+      return { categories: Array.from(categories).sort() };
     } catch (error) {
       throw new ServiceError(
         `Failed to get categories: ${error instanceof Error ? error.message : 'Unknown error'}`,
@@ -505,25 +593,16 @@ export class MemoryService {
     }
   }
 
-  /**
-   * Gets all unique tags
-   *
-   * @returns Array of tag names
-   */
   async getTags(): Promise<GetTagsOutput> {
     try {
       const listResult = await this.list({ limit: 1000 });
       const tags = new Set<string>();
-
       for (const memory of listResult.memories) {
         for (const tag of memory.tags) {
           tags.add(tag);
         }
       }
-
-      return {
-        tags: Array.from(tags).sort(),
-      };
+      return { tags: Array.from(tags).sort() };
     } catch (error) {
       throw new ServiceError(
         `Failed to get tags: ${error instanceof Error ? error.message : 'Unknown error'}`,
@@ -532,12 +611,6 @@ export class MemoryService {
     }
   }
 
-  /**
-   * Extracts ID from file content
-   *
-   * @param content - File content
-   * @returns ID string
-   */
   private extractIdFromContent(content: string): string {
     const match = content.match(/id:\s*"?([^"\n]+)"?/);
     if (!match) {