@danielsimonjr/memory-mcp 0.47.1 → 9.8.0

package/dist/search/FuzzySearch.js

@@ -2,25 +2,123 @@
  * Fuzzy Search
  *
  * Search with typo tolerance using Levenshtein distance similarity.
+ * Uses workerpool for parallel processing on large datasets.
  *
  * @module search/FuzzySearch
  */
-import { levenshteinDistance } from '../utils/levenshtein.js';
+import { levenshteinDistance } from '../utils/index.js';
 import { SEARCH_LIMITS } from '../utils/constants.js';
 import { SearchFilterChain } from './SearchFilterChain.js';
+import workerpool from '@danielsimonjr/workerpool';
+import { fileURLToPath } from 'url';
+import { dirname, join, sep } from 'path';
 /**
  * Default fuzzy search similarity threshold (70% match required).
  * Lower values are more permissive (more typos tolerated).
  * Higher values are stricter (fewer typos tolerated).
  */
 export const DEFAULT_FUZZY_THRESHOLD = 0.7;
+/**
+ * Phase 4 Sprint 3: Maximum cache size to prevent memory bloat.
+ */
+const FUZZY_CACHE_MAX_SIZE = 100;
+/**
+ * Phase 4 Sprint 3: Cache TTL in milliseconds (5 minutes).
+ */
+const FUZZY_CACHE_TTL_MS = 5 * 60 * 1000;
+/**
+ * Phase 7 Sprint 3: Minimum number of entities to activate worker pool.
+ */
+const WORKER_MIN_ENTITIES = 500;
+/**
+ * Phase 7 Sprint 3: Maximum threshold for worker pool activation.
+ * Higher thresholds have fewer matches, so single-threaded is faster.
+ */
+const WORKER_MAX_THRESHOLD = 0.8;
 /**
  * Performs fuzzy search with configurable similarity threshold.
  */
 export class FuzzySearch {
     storage;
-    constructor(storage) {
+    /**
+     * Phase 4 Sprint 3: Result cache for fuzzy search.
+     * Maps cache key -> cached entity names.
+     */
+    fuzzyResultCache = new Map();
+    /**
+     * Phase 8: Worker pool using workerpool library.
+     * Initialized lazily when needed.
+     */
+    workerPool = null;
+    /**
+     * Phase 7 Sprint 3: Path to the worker script.
+     */
+    workerPath;
+    /**
+     * Phase 8: Whether to use worker pool for parallel processing.
+     * Can be disabled for testing or when workers are not available.
+     */
+    useWorkerPool;
+    constructor(storage, options = {}) {
         this.storage = storage;
+        this.useWorkerPool = options.useWorkerPool ?? true;
+        // Calculate worker path using ESM module resolution
+        const currentFileUrl = import.meta.url;
+        const currentDir = dirname(fileURLToPath(currentFileUrl));
+        // Check if we're running from src/ (during tests) or dist/ (production)
+        const isRunningFromSrc = currentDir.includes(`${sep}src${sep}`);
+        if (isRunningFromSrc) {
+            // During tests, worker is in dist/workers/ relative to project root
+            const projectRoot = join(currentDir, '..', '..');
+            this.workerPath = join(projectRoot, 'dist', 'workers', 'levenshteinWorker.js');
+        }
+        else {
+            // In production, worker is in dist/workers/ relative to current dist/search/
+            this.workerPath = join(currentDir, '..', 'workers', 'levenshteinWorker.js');
+        }
+    }
+    /**
+     * Phase 4 Sprint 3: Generate cache key for fuzzy search parameters.
+     */
+    generateCacheKey(query, threshold, tags, minImportance, maxImportance, offset, limit) {
+        return JSON.stringify({
+            q: query.toLowerCase(),
+            t: threshold,
+            tags: tags?.sort().join(',') ?? '',
+            min: minImportance,
+            max: maxImportance,
+            off: offset,
+            lim: limit,
+        });
+    }
+    /**
+     * Phase 4 Sprint 3: Clear the fuzzy search cache.
+     */
+    clearCache() {
+        this.fuzzyResultCache.clear();
+    }
+    /**
+     * Phase 4 Sprint 3: Invalidate stale cache entries.
+     */
+    cleanupCache() {
+        const now = Date.now();
+        const entries = Array.from(this.fuzzyResultCache.entries());
+        // Remove expired entries
+        for (const [key, entry] of entries) {
+            if (now - entry.timestamp > FUZZY_CACHE_TTL_MS) {
+                this.fuzzyResultCache.delete(key);
+            }
+        }
+        // If still over limit, remove oldest entries
+        if (this.fuzzyResultCache.size > FUZZY_CACHE_MAX_SIZE) {
+            const sortedEntries = entries
+                .filter(([k]) => this.fuzzyResultCache.has(k))
+                .sort((a, b) => a[1].timestamp - b[1].timestamp);
+            const toRemove = sortedEntries.slice(0, this.fuzzyResultCache.size - FUZZY_CACHE_MAX_SIZE);
+            for (const [key] of toRemove) {
+                this.fuzzyResultCache.delete(key);
+            }
+        }
     }
     /**
      * Fuzzy search for entities with typo tolerance and pagination.
@@ -28,6 +126,8 @@ export class FuzzySearch {
      * Uses Levenshtein distance to calculate similarity between strings.
      * Matches if similarity >= threshold (0.0 to 1.0).
      *
+     * Phase 4 Sprint 3: Implements result caching for repeated queries.
+     *
      * @param query - Search query
      * @param threshold - Similarity threshold (0.0 to 1.0), default DEFAULT_FUZZY_THRESHOLD
      * @param tags - Optional tags filter
@@ -39,25 +139,51 @@ export class FuzzySearch {
      */
     async fuzzySearch(query, threshold = DEFAULT_FUZZY_THRESHOLD, tags, minImportance, maxImportance, offset = 0, limit = SEARCH_LIMITS.DEFAULT) {
         const graph = await this.storage.loadGraph();
-        // First filter by fuzzy text match (search-specific)
-        const fuzzyMatched = graph.entities.filter(e => {
-            return (this.isFuzzyMatch(e.name, query, threshold) ||
-                this.isFuzzyMatch(e.entityType, query, threshold) ||
-                e.observations.some(o => 
-                // For observations, split into words and check each word
-                o
-                    .toLowerCase()
-                    .split(/\s+/)
-                    .some(word => this.isFuzzyMatch(word, query, threshold)) ||
-                    // Also check if the observation contains the query
-                    this.isFuzzyMatch(o, query, threshold)));
-        });
+        const queryLower = query.toLowerCase();
+        // Phase 4 Sprint 3: Generate cache key and check cache
+        const cacheKey = this.generateCacheKey(query, threshold, tags, minImportance, maxImportance, offset, limit);
+        const cached = this.fuzzyResultCache.get(cacheKey);
+        // Check if cache is valid (entity count hasn't changed)
+        if (cached && cached.entityCount === graph.entities.length) {
+            const now = Date.now();
+            if (now - cached.timestamp < FUZZY_CACHE_TTL_MS) {
+                // Return cached results
+                const cachedNameSet = new Set(cached.entityNames);
+                const cachedEntities = graph.entities.filter(e => cachedNameSet.has(e.name));
+                const cachedEntityNames = new Set(cached.entityNames);
+                const cachedRelations = graph.relations.filter(r => cachedEntityNames.has(r.from) && cachedEntityNames.has(r.to));
+                return { entities: cachedEntities, relations: cachedRelations };
+            }
+        }
+        // Phase 7 Sprint 3: Use worker pool for large graphs with low thresholds
+        // Phase 8: Respect useWorkerPool flag for testing
+        const shouldUseWorkers = this.useWorkerPool &&
+            graph.entities.length >= WORKER_MIN_ENTITIES &&
+            threshold < WORKER_MAX_THRESHOLD;
+        let fuzzyMatched;
+        if (shouldUseWorkers) {
+            fuzzyMatched = await this.searchWithWorkers(query, threshold, graph.entities);
+        }
+        else {
+            // Perform single-threaded fuzzy search
+            fuzzyMatched = this.performFuzzyMatch(graph.entities, queryLower, threshold);
+        }
         // Apply tag and importance filters using SearchFilterChain
         const filters = { tags, minImportance, maxImportance };
         const filteredEntities = SearchFilterChain.applyFilters(fuzzyMatched, filters);
         // Apply pagination using SearchFilterChain
         const pagination = SearchFilterChain.validatePagination(offset, limit);
         const paginatedEntities = SearchFilterChain.paginate(filteredEntities, pagination);
+        // Phase 4 Sprint 3: Cache the results
+        this.fuzzyResultCache.set(cacheKey, {
+            entityNames: paginatedEntities.map(e => e.name),
+            entityCount: graph.entities.length,
+            timestamp: Date.now(),
+        });
+        // Cleanup old cache entries periodically
+        if (this.fuzzyResultCache.size > FUZZY_CACHE_MAX_SIZE / 2) {
+            this.cleanupCache();
+        }
         const filteredEntityNames = new Set(paginatedEntities.map(e => e.name));
         const filteredRelations = graph.relations.filter(r => filteredEntityNames.has(r.from) && filteredEntityNames.has(r.to));
         return {
@@ -66,21 +192,42 @@ export class FuzzySearch {
         };
     }
     /**
-     * Check if two strings match with fuzzy logic.
+     * Phase 4 Sprint 3: Perform the actual fuzzy matching logic.
+     * Extracted from fuzzySearch for cleaner code structure.
+     */
+    performFuzzyMatch(entities, queryLower, threshold) {
+        return entities.filter(e => {
+            const lowercased = this.storage.getLowercased(e.name);
+            // Check name match (use pre-computed lowercase)
+            const nameLower = lowercased?.name ?? e.name.toLowerCase();
+            if (this.isFuzzyMatchLower(nameLower, queryLower, threshold))
+                return true;
+            // Check type match (use pre-computed lowercase)
+            const typeLower = lowercased?.entityType ?? e.entityType.toLowerCase();
+            if (this.isFuzzyMatchLower(typeLower, queryLower, threshold))
+                return true;
+            // Check observations (use pre-computed lowercase array)
+            const obsLower = lowercased?.observations ?? e.observations.map(o => o.toLowerCase());
+            return obsLower.some(o => 
+            // For observations, split into words and check each word
+            o
+                .split(/\s+/)
+                .some(word => this.isFuzzyMatchLower(word, queryLower, threshold)) ||
+                // Also check if the observation contains the query
+                this.isFuzzyMatchLower(o, queryLower, threshold));
+        });
+    }
+    /**
+     * Check if two already-lowercase strings match with fuzzy logic.
      *
-     * Returns true if:
-     * - Strings are identical
-     * - One contains the other
-     * - Levenshtein similarity >= threshold
+     * OPTIMIZED: Skips toLowerCase() calls when strings are already lowercase.
      *
-     * @param str1 - First string
-     * @param str2 - Second string
+     * @param s1 - First string (already lowercase)
+     * @param s2 - Second string (already lowercase)
      * @param threshold - Similarity threshold (0.0 to 1.0)
      * @returns True if strings match fuzzily
      */
-    isFuzzyMatch(str1, str2, threshold = 0.7) {
-        const s1 = str1.toLowerCase();
-        const s2 = str2.toLowerCase();
+    isFuzzyMatchLower(s1, s2, threshold = 0.7) {
         // Exact match
         if (s1 === s2)
             return true;
@@ -93,4 +240,73 @@ export class FuzzySearch {
         const similarity = 1 - distance / maxLength;
         return similarity >= threshold;
     }
+    /**
+     * Phase 8: Perform fuzzy search using workerpool for parallel processing.
+     *
+     * Splits entities into chunks and processes them in parallel using worker threads.
+     * Falls back to single-threaded search if worker execution fails.
+     *
+     * @param query - Search query
+     * @param threshold - Similarity threshold
+     * @param entities - Entities to search
+     * @returns Array of matched entities
+     */
+    async searchWithWorkers(query, threshold, entities) {
+        try {
+            // Initialize worker pool lazily using workerpool
+            if (!this.workerPool) {
+                // Enable ESM module support for Node.js 20+
+                // The 'type: module' option is needed for ESM workers but may not be in @types/node
+                const workerThreadOpts = { type: 'module' };
+                this.workerPool = workerpool.pool(this.workerPath, {
+                    maxWorkers: Math.max(1, workerpool.cpus - 1),
+                    workerType: 'thread',
+                    workerThreadOpts,
+                });
+            }
+            // Split entities into chunks based on CPU count
+            const numWorkers = Math.max(1, workerpool.cpus - 1);
+            const chunkSize = Math.ceil(entities.length / numWorkers);
+            const chunks = [];
+            for (let i = 0; i < entities.length; i += chunkSize) {
+                chunks.push(entities.slice(i, i + chunkSize));
+            }
+            // Prepare worker inputs with lowercased data
+            const workerInputs = chunks.map(chunk => ({
+                query,
+                threshold,
+                entities: chunk.map(e => ({
+                    name: e.name,
+                    nameLower: e.name.toLowerCase(),
+                    observations: e.observations.map(o => o.toLowerCase()),
+                })),
+            }));
+            // Execute all chunks in parallel using workerpool with timeout
+            const WORKER_TIMEOUT_MS = 30000; // 30 seconds
+            const results = await Promise.all(workerInputs.map(input => this.workerPool.exec('searchEntities', [input])
+                .timeout(WORKER_TIMEOUT_MS)));
+            // Flatten results and extract matched entity names
+            const matchedNames = new Set(results.flat().map(r => r.name));
+            // Return entities that matched
+            return entities.filter(e => matchedNames.has(e.name));
+        }
+        catch (error) {
+            // Worker execution failed - fall back to single-threaded mode
+            console.warn(`Worker pool execution failed, falling back to single-threaded fuzzy search: ${error instanceof Error ? error.message : String(error)}`);
+            // Use the existing single-threaded implementation
+            const queryLower = query.toLowerCase();
+            return this.performFuzzyMatch(entities, queryLower, threshold);
+        }
+    }
+    /**
+     * Phase 8: Shutdown the worker pool and clean up resources.
+     *
+     * Should be called when FuzzySearch is no longer needed.
+     */
+    async shutdown() {
+        if (this.workerPool) {
+            await this.workerPool.terminate();
+            this.workerPool = null;
+        }
+    }
 }

package/dist/search/QueryCostEstimator.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Query Cost Estimator
+ *
+ * Phase 10 Sprint 4: Estimates the cost of different search methods
+ * and recommends the optimal method based on query characteristics
+ * and graph size.
+ *
+ * @module search/QueryCostEstimator
+ */
+import type { SearchMethod, QueryCostEstimate, QueryCostEstimatorOptions } from '../types/index.js';
+/**
+ * Phase 10 Sprint 4: Estimates search query costs and recommends optimal methods.
+ *
+ * Analyzes query characteristics and graph size to estimate execution time
+ * and recommend the most appropriate search method.
+ *
+ * @example
+ * ```typescript
+ * const estimator = new QueryCostEstimator();
+ *
+ * // Get estimate for a specific method
+ * const estimate = estimator.estimateMethod('ranked', 'test query', 1000);
+ *
+ * // Get the recommended method for a query
+ * const recommendation = estimator.recommendMethod('test query', 1000);
+ *
+ * // Get estimates for all methods
+ * const allEstimates = estimator.estimateAllMethods('test query', 1000);
+ * ```
+ */
+export declare class QueryCostEstimator {
+    private options;
+    /**
+     * Create a new QueryCostEstimator instance.
+     *
+     * @param options - Optional configuration overrides
+     */
+    constructor(options?: QueryCostEstimatorOptions);
+    /**
+     * Estimate the cost of a specific search method.
+     *
+     * @param method - The search method to estimate
+     * @param query - The search query
+     * @param entityCount - Number of entities in the graph
+     * @returns Cost estimate for the method
+     */
+    estimateMethod(method: SearchMethod, query: string, entityCount: number): QueryCostEstimate;
+    /**
+     * Internal method to estimate without triggering recursion.
+     * @private
+     */
+    private estimateMethodInternal;
+    /**
+     * Get just the recommended method without full estimate (avoids recursion).
+     * @private
+     */
+    private getRecommendedMethodOnly;
+    /**
+     * Get estimates for all available search methods.
+     *
+     * @param query - The search query
+     * @param entityCount - Number of entities in the graph
+     * @returns Array of estimates for all methods
+     */
+    estimateAllMethods(query: string, entityCount: number): QueryCostEstimate[];
+    /**
+     * Recommend the best search method for a query.
+     *
+     * @param query - The search query
+     * @param entityCount - Number of entities in the graph
+     * @param preferredMethods - Optional array of methods to consider (default: all)
+     * @returns The recommended method and reason
+     */
+    recommendMethod(query: string, entityCount: number, preferredMethods?: SearchMethod[]): {
+        method: SearchMethod;
+        reason: string;
+        estimate: QueryCostEstimate;
+    };
+    /**
+     * Get the base time per entity for a search method.
+     * @private
+     */
+    private getBaseTimeForMethod;
+    /**
+     * Calculate a complexity factor based on query characteristics.
+     * @private
+     */
+    private getQueryComplexityFactor;
+    /**
+     * Get the complexity level based on entity count.
+     * @private
+     */
+    private getComplexity;
+    /**
+     * Generate a human-readable recommendation.
+     * @private
+     */
+    private getRecommendation;
+    /**
+     * Score a method based on query characteristics and graph size.
+     * Higher score = better fit.
+     * @private
+     */
+    private scoreMethod;
+    /**
+     * Get a human-readable reason for why a method was selected.
+     * @private
+     */
+    private getSelectionReason;
+}
+//# sourceMappingURL=QueryCostEstimator.d.ts.map

package/dist/search/QueryCostEstimator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"QueryCostEstimator.d.ts","sourceRoot":"","sources":["../../src/search/QueryCostEstimator.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,yBAAyB,EAC1B,MAAM,mBAAmB,CAAC;AAe3B;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,OAAO,CAAsC;IAErD;;;;OAIG;gBACS,OAAO,CAAC,EAAE,yBAAyB;IAI/C;;;;;;;OAOG;IACH,cAAc,CACZ,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,GAClB,iBAAiB;IAMpB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAsB9B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAsBhC;;;;;;OAMG;IACH,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,iBAAiB,EAAE;IAQ3E;;;;;;;OAOG;IACH,eAAe,CACb,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,gBAAgB,CAAC,EAAE,YAAY,EAAE,GAChC;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,iBAAiB,CAAA;KAAE;IAuBxE;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAe5B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAyChC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAUrB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IA2BzB;;;;OAIG;IACH,OAAO,CAAC,WAAW;IAwFnB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;CAsC3B"}