@danielsimonjr/memory-mcp 0.47.1 → 9.8.0

package/dist/utils/taskScheduler.js

@@ -0,0 +1,466 @@
+/**
+ * Task Scheduler
+ *
+ * Advanced task scheduling utilities using workerpool.
+ * Phase 8 Sprint 4: Priority queues, concurrency control, progress tracking.
+ *
+ * @module utils/taskScheduler
+ */
+import workerpool from '@danielsimonjr/workerpool';
+// ==================== Types ====================
+/**
+ * Task priority levels.
+ * Higher priority tasks are executed first.
+ */
+export var TaskPriority;
+(function (TaskPriority) {
+    TaskPriority[TaskPriority["LOW"] = 0] = "LOW";
+    TaskPriority[TaskPriority["NORMAL"] = 1] = "NORMAL";
+    TaskPriority[TaskPriority["HIGH"] = 2] = "HIGH";
+    TaskPriority[TaskPriority["CRITICAL"] = 3] = "CRITICAL";
+})(TaskPriority || (TaskPriority = {}));
+/**
+ * Task status in the queue.
+ */
+export var TaskStatus;
+(function (TaskStatus) {
+    TaskStatus["PENDING"] = "pending";
+    TaskStatus["RUNNING"] = "running";
+    TaskStatus["COMPLETED"] = "completed";
+    TaskStatus["FAILED"] = "failed";
+    TaskStatus["CANCELLED"] = "cancelled";
+})(TaskStatus || (TaskStatus = {}));
+/**
+ * Priority Task Queue with advanced scheduling.
+ *
+ * Features:
+ * - Priority-based execution (CRITICAL > HIGH > NORMAL > LOW)
+ * - Configurable concurrency limits
+ * - Progress tracking
+ * - Graceful error handling
+ * - Task cancellation
+ *
+ * @example
+ * ```typescript
+ * const queue = new TaskQueue({ concurrency: 4 });
+ *
+ * // Add tasks with different priorities
+ * queue.enqueue({
+ *   id: 'task1',
+ *   priority: TaskPriority.HIGH,
+ *   fn: (x: number) => x * 2,
+ *   input: 5,
+ * });
+ *
+ * // Process all tasks
+ * const results = await queue.processAll();
+ * ```
+ */
+export class TaskQueue {
+    queue = [];
+    running = new Map();
+    completed = [];
+    pool = null;
+    concurrency;
+    defaultTimeout;
+    isProcessing = false;
+    totalExecutionTime = 0;
+    totalProcessed = 0;
+    useWorkerPool;
+    constructor(options = {}) {
+        this.concurrency = options.concurrency ?? Math.max(1, workerpool.cpus - 1);
+        this.defaultTimeout = options.timeout ?? 30000;
+        this.useWorkerPool = options.useWorkerPool ?? true;
+    }
+    /**
+     * Get or create the worker pool.
+     */
+    getPool() {
+        if (!this.pool) {
+            this.pool = workerpool.pool({
+                maxWorkers: this.concurrency,
+                workerType: 'thread',
+            });
+        }
+        return this.pool;
+    }
+    /**
+     * Add a task to the queue.
+     *
+     * @param task - Task to add
+     * @returns Promise that resolves when the task completes
+     */
+    enqueue(task) {
+        return new Promise((resolve, reject) => {
+            const queuedTask = {
+                ...task,
+                status: TaskStatus.PENDING,
+                addedAt: Date.now(),
+                resolve: resolve,
+                reject,
+            };
+            // Insert based on priority (higher priority first)
+            const insertIndex = this.queue.findIndex(t => t.priority < task.priority);
+            if (insertIndex === -1) {
+                this.queue.push(queuedTask);
+            }
+            else {
+                this.queue.splice(insertIndex, 0, queuedTask);
+            }
+            // Start processing if not already running
+            if (!this.isProcessing) {
+                this.processNext();
+            }
+        });
+    }
+    /**
+     * Process the next task in the queue.
+     */
+    async processNext() {
+        if (this.running.size >= this.concurrency || this.queue.length === 0) {
+            if (this.running.size === 0 && this.queue.length === 0) {
+                this.isProcessing = false;
+            }
+            return;
+        }
+        this.isProcessing = true;
+        const task = this.queue.shift();
+        if (!task)
+            return;
+        task.status = TaskStatus.RUNNING;
+        this.running.set(task.id, task);
+        const startTime = Date.now();
+        try {
+            // Execute task - try worker pool first, fall back to direct execution
+            let result;
+            if (this.useWorkerPool) {
+                try {
+                    const pool = this.getPool();
+                    const fnString = task.fn.toString();
+                    const timeout = task.timeout ?? this.defaultTimeout;
+                    result = await pool
+                        .exec((input, fnStr) => {
+                        // eslint-disable-next-line no-new-func
+                        const fn = new Function('return ' + fnStr)();
+                        return fn(input);
+                    }, [task.input, fnString])
+                        .timeout(timeout);
+                }
+                catch {
+                    // Fall back to direct execution
+                    result = await Promise.resolve(task.fn(task.input));
+                }
+            }
+            else {
+                // Direct execution without worker pool
+                result = await Promise.resolve(task.fn(task.input));
+            }
+            const endTime = Date.now();
+            const duration = endTime - startTime;
+            const taskResult = {
+                id: task.id,
+                status: TaskStatus.COMPLETED,
+                result,
+                duration,
+                startedAt: startTime,
+                completedAt: endTime,
+            };
+            this.totalExecutionTime += duration;
+            this.totalProcessed++;
+            this.completed.push(taskResult);
+            this.running.delete(task.id);
+            task.resolve(taskResult);
+        }
+        catch (error) {
+            const endTime = Date.now();
+            const duration = endTime - startTime;
+            const taskResult = {
+                id: task.id,
+                status: TaskStatus.FAILED,
+                error: error instanceof Error ? error : new Error(String(error)),
+                duration,
+                startedAt: startTime,
+                completedAt: endTime,
+            };
+            this.totalProcessed++;
+            this.completed.push(taskResult);
+            this.running.delete(task.id);
+            task.resolve(taskResult);
+        }
+        // Process next task
+        this.processNext();
+    }
+    /**
+     * Cancel a pending task.
+     *
+     * @param taskId - ID of the task to cancel
+     * @returns True if task was cancelled, false if not found or already running
+     */
+    cancel(taskId) {
+        const index = this.queue.findIndex(t => t.id === taskId);
+        if (index === -1)
+            return false;
+        const task = this.queue.splice(index, 1)[0];
+        task.status = TaskStatus.CANCELLED;
+        const result = {
+            id: task.id,
+            status: TaskStatus.CANCELLED,
+            duration: 0,
+            startedAt: Date.now(),
+            completedAt: Date.now(),
+        };
+        task.resolve(result);
+        return true;
+    }
+    /**
+     * Wait for all tasks to complete.
+     *
+     * @returns Array of all task results
+     */
+    async drain() {
+        // Wait for queue to empty and all running tasks to complete
+        while (this.queue.length > 0 || this.running.size > 0) {
+            await new Promise(resolve => setTimeout(resolve, 10));
+        }
+        return [...this.completed];
+    }
+    /**
+     * Get queue statistics.
+     */
+    getStats() {
+        return {
+            pending: this.queue.length,
+            running: this.running.size,
+            completed: this.completed.filter(r => r.status === TaskStatus.COMPLETED).length,
+            failed: this.completed.filter(r => r.status === TaskStatus.FAILED).length,
+            averageExecutionTime: this.totalProcessed > 0 ? this.totalExecutionTime / this.totalProcessed : 0,
+            totalProcessed: this.totalProcessed,
+        };
+    }
+    /**
+     * Clear all completed results.
+     */
+    clearCompleted() {
+        this.completed = [];
+    }
+    /**
+     * Shutdown the task queue and release resources.
+     */
+    async shutdown() {
+        // Cancel all pending tasks
+        for (const task of this.queue) {
+            task.status = TaskStatus.CANCELLED;
+            task.resolve({
+                id: task.id,
+                status: TaskStatus.CANCELLED,
+                duration: 0,
+                startedAt: Date.now(),
+                completedAt: Date.now(),
+            });
+        }
+        this.queue = [];
+        // Terminate worker pool
+        if (this.pool) {
+            await this.pool.terminate();
+            this.pool = null;
+        }
+        this.isProcessing = false;
+    }
+}
+// ==================== Batch Processing Utilities ====================
+/**
+ * Process items in parallel batches with progress tracking.
+ *
+ * Unlike parallelMap, this provides:
+ * - Progress callbacks
+ * - Configurable concurrency
+ * - Error handling options
+ * - Task-level timeouts
+ *
+ * @template T - Input item type
+ * @template R - Output item type
+ * @param items - Array of items to process
+ * @param fn - Processing function (must be serializable)
+ * @param options - Batch processing options
+ * @returns Array of results (or errors if stopOnError is false)
+ *
+ * @example
+ * ```typescript
+ * const results = await batchProcess(
+ *   urls,
+ *   (url: string) => fetch(url).then(r => r.json()),
+ *   {
+ *     concurrency: 5,
+ *     timeout: 10000,
+ *     onProgress: ({ completed, total, percentage }) => {
+ *       console.log(`Progress: ${percentage.toFixed(1)}%`);
+ *     },
+ *   }
+ * );
+ * ```
+ */
+export async function batchProcess(items, fn, options = {}) {
+    const { concurrency = Math.max(1, workerpool.cpus - 1), timeout = 30000, onProgress, stopOnError = false, } = options;
+    const results = [];
+    let completed = 0;
+    const total = items.length;
+    // Process in batches respecting concurrency
+    for (let i = 0; i < items.length; i += concurrency) {
+        const batch = items.slice(i, i + concurrency);
+        const batchPromises = batch.map(async (item, batchIndex) => {
+            const itemIndex = i + batchIndex;
+            try {
+                // Execute with timeout
+                const result = await Promise.race([
+                    Promise.resolve(fn(item)),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Task timeout')), timeout)),
+                ]);
+                results[itemIndex] = { success: true, result };
+            }
+            catch (error) {
+                const err = error instanceof Error ? error : new Error(String(error));
+                results[itemIndex] = { success: false, error: err };
+                if (stopOnError) {
+                    throw err;
+                }
+            }
+            finally {
+                completed++;
+                if (onProgress) {
+                    onProgress({
+                        completed,
+                        total,
+                        percentage: (completed / total) * 100,
+                        currentTaskId: `item-${itemIndex}`,
+                    });
+                }
+            }
+        });
+        await Promise.all(batchPromises);
+    }
+    return results;
+}
+/**
+ * Execute tasks with rate limiting.
+ *
+ * Ensures tasks don't exceed a specified rate (tasks per second).
+ *
+ * @template T - Input item type
+ * @template R - Output item type
+ * @param items - Items to process
+ * @param fn - Processing function
+ * @param rateLimit - Maximum tasks per second
+ * @returns Array of results
+ *
+ * @example
+ * ```typescript
+ * // Process max 10 items per second
+ * const results = await rateLimitedProcess(
+ *   items,
+ *   (item) => processItem(item),
+ *   10
+ * );
+ * ```
+ */
+export async function rateLimitedProcess(items, fn, rateLimit) {
+    const results = [];
+    const minInterval = 1000 / rateLimit;
+    let lastExecutionTime = 0;
+    for (const item of items) {
+        // Calculate wait time
+        const now = Date.now();
+        const timeSinceLast = now - lastExecutionTime;
+        if (timeSinceLast < minInterval) {
+            await new Promise(resolve => setTimeout(resolve, minInterval - timeSinceLast));
+        }
+        lastExecutionTime = Date.now();
+        const result = await fn(item);
+        results.push(result);
+    }
+    return results;
+}
+/**
+ * Retry a function with exponential backoff.
+ *
+ * @template T - Return type
+ * @param fn - Function to retry
+ * @param options - Retry options
+ * @returns Result of the function
+ *
+ * @example
+ * ```typescript
+ * const result = await withRetry(
+ *   () => fetchData(),
+ *   { maxRetries: 3, baseDelay: 1000 }
+ * );
+ * ```
+ */
+export async function withRetry(fn, options = {}) {
+    const { maxRetries = 3, baseDelay = 1000, maxDelay = 30000, onRetry } = options;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            if (attempt < maxRetries) {
+                const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
+                if (onRetry) {
+                    onRetry(lastError, attempt + 1);
+                }
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+    }
+    throw lastError;
+}
+/**
+ * Create a debounced version of a function.
+ *
+ * @template T - Function arguments type
+ * @template R - Return type
+ * @param fn - Function to debounce
+ * @param delay - Delay in milliseconds
+ * @returns Debounced function
+ */
+export function debounce(fn, delay) {
+    let timeoutId = null;
+    let pendingResolve = null;
+    return (...args) => {
+        return new Promise(resolve => {
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
+            pendingResolve = resolve;
+            timeoutId = setTimeout(() => {
+                const result = fn(...args);
+                if (pendingResolve) {
+                    pendingResolve(result);
+                }
+                timeoutId = null;
+                pendingResolve = null;
+            }, delay);
+        });
+    };
+}
+/**
+ * Create a throttled version of a function.
+ *
+ * @template T - Function arguments type
+ * @template R - Return type
+ * @param fn - Function to throttle
+ * @param limit - Minimum time between calls in milliseconds
+ * @returns Throttled function
+ */
+export function throttle(fn, limit) {
+    let lastCall = 0;
+    return (...args) => {
+        const now = Date.now();
+        if (now - lastCall >= limit) {
+            lastCall = now;
+            return fn(...args);
+        }
+        return undefined;
+    };
+}

package/dist/workers/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Workers Module
+ *
+ * Worker thread utilities for parallel processing.
+ * Phase 8: Uses workerpool library for worker management.
+ *
+ * @module workers
+ */
+export type { Pool, PoolStats } from '@danielsimonjr/workerpool';
+export type { WorkerInput, MatchResult } from './levenshteinWorker.js';
+export { levenshteinDistance, similarity, searchEntities } from './levenshteinWorker.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/workers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/workers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAGjE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/workers/index.js

@@ -0,0 +1,9 @@
+/**
+ * Workers Module
+ *
+ * Worker thread utilities for parallel processing.
+ * Phase 8: Uses workerpool library for worker management.
+ *
+ * @module workers
+ */
+export { levenshteinDistance, similarity, searchEntities } from './levenshteinWorker.js';

package/dist/workers/levenshteinWorker.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Levenshtein Worker
+ *
+ * Worker thread for calculating Levenshtein distances in parallel.
+ * Uses workerpool for worker management.
+ *
+ * @module workers/levenshteinWorker
+ */
+/**
+ * Input data structure for the worker.
+ */
+export interface WorkerInput {
+    /** Search query string */
+    query: string;
+    /** Array of entities to search */
+    entities: Array<{
+        name: string;
+        nameLower: string;
+        observations: string[];
+    }>;
+    /** Similarity threshold (0.0 to 1.0) */
+    threshold: number;
+}
+/**
+ * Match result returned by the worker.
+ */
+export interface MatchResult {
+    /** Entity name that matched */
+    name: string;
+    /** Similarity score (0.0 to 1.0) */
+    score: number;
+    /** Where the match occurred */
+    matchedIn: 'name' | 'observation';
+}
+/**
+ * Calculate Levenshtein distance between two strings.
+ *
+ * Uses dynamic programming matrix for efficient computation.
+ *
+ * @param s1 - First string
+ * @param s2 - Second string
+ * @returns Levenshtein distance (number of edits)
+ */
+export declare function levenshteinDistance(s1: string, s2: string): number;
+/**
+ * Calculate similarity score between two strings.
+ *
+ * @param s1 - First string
+ * @param s2 - Second string
+ * @returns Similarity score (0.0 to 1.0, where 1.0 is identical)
+ */
+export declare function similarity(s1: string, s2: string): number;
+/**
+ * Search entities for fuzzy matches.
+ *
+ * @param data - Worker input containing query, entities, and threshold
+ * @returns Array of match results
+ */
+export declare function searchEntities(data: WorkerInput): MatchResult[];
+//# sourceMappingURL=levenshteinWorker.d.ts.map

package/dist/workers/levenshteinWorker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"levenshteinWorker.d.ts","sourceRoot":"","sources":["../../src/workers/levenshteinWorker.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,kCAAkC;IAClC,QAAQ,EAAE,KAAK,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC,CAAC;IACH,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,SAAS,EAAE,MAAM,GAAG,aAAa,CAAC;CACnC;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAgClE;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAWzD;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,WAAW,GAAG,WAAW,EAAE,CAwB/D"}

package/dist/workers/levenshteinWorker.js

@@ -0,0 +1,98 @@
+/**
+ * Levenshtein Worker
+ *
+ * Worker thread for calculating Levenshtein distances in parallel.
+ * Uses workerpool for worker management.
+ *
+ * @module workers/levenshteinWorker
+ */
+import workerpool from '@danielsimonjr/workerpool';
+/**
+ * Calculate Levenshtein distance between two strings.
+ *
+ * Uses dynamic programming matrix for efficient computation.
+ *
+ * @param s1 - First string
+ * @param s2 - Second string
+ * @returns Levenshtein distance (number of edits)
+ */
+export function levenshteinDistance(s1, s2) {
+    const len1 = s1.length;
+    const len2 = s2.length;
+    if (len1 === 0)
+        return len2;
+    if (len2 === 0)
+        return len1;
+    const matrix = [];
+    // Initialize first column
+    for (let i = 0; i <= len1; i++) {
+        matrix[i] = [i];
+    }
+    // Initialize first row
+    for (let j = 0; j <= len2; j++) {
+        matrix[0][j] = j;
+    }
+    // Fill matrix
+    for (let i = 1; i <= len1; i++) {
+        for (let j = 1; j <= len2; j++) {
+            const cost = s1[i - 1] === s2[j - 1] ? 0 : 1;
+            matrix[i][j] = Math.min(matrix[i - 1][j] + 1, // deletion
+            matrix[i][j - 1] + 1, // insertion
+            matrix[i - 1][j - 1] + cost // substitution
+            );
+        }
+    }
+    return matrix[len1][len2];
+}
+/**
+ * Calculate similarity score between two strings.
+ *
+ * @param s1 - First string
+ * @param s2 - Second string
+ * @returns Similarity score (0.0 to 1.0, where 1.0 is identical)
+ */
+export function similarity(s1, s2) {
+    // Exact match
+    if (s1 === s2)
+        return 1.0;
+    // One contains the other
+    if (s1.includes(s2) || s2.includes(s1))
+        return 1.0;
+    // Calculate Levenshtein-based similarity
+    const distance = levenshteinDistance(s1, s2);
+    const maxLength = Math.max(s1.length, s2.length);
+    return 1 - distance / maxLength;
+}
+/**
+ * Search entities for fuzzy matches.
+ *
+ * @param data - Worker input containing query, entities, and threshold
+ * @returns Array of match results
+ */
+export function searchEntities(data) {
+    const { query, entities, threshold } = data;
+    const queryLower = query.toLowerCase();
+    const results = [];
+    for (const entity of entities) {
+        // Check name similarity
+        const nameScore = similarity(queryLower, entity.nameLower);
+        if (nameScore >= threshold) {
+            results.push({ name: entity.name, score: nameScore, matchedIn: 'name' });
+            continue;
+        }
+        // Check observations
+        for (const obs of entity.observations) {
+            const obsScore = similarity(queryLower, obs);
+            if (obsScore >= threshold) {
+                results.push({ name: entity.name, score: obsScore, matchedIn: 'observation' });
+                break;
+            }
+        }
+    }
+    return results;
+}
+// Register worker methods with workerpool
+// Cast to satisfy workerpool's generic type signature
+workerpool.worker({
+    searchEntities: searchEntities,
+});

package/package.json

@@ -1,8 +1,11 @@
 {
   "name": "@danielsimonjr/memory-mcp",
-  "version": "0.47.1",
-  "description": "Enhanced MCP memory server with hierarchies, compression, archiving, and 45 advanced tools",
+  "version": "9.8.0",
+  "description": "Enhanced MCP memory server with hierarchies, compression, archiving, graph algorithms, semantic search, and 55 advanced tools",
   "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "author": "Daniel Simon Jr. (https://github.com/danielsimonjr)",
   "homepage": "https://github.com/danielsimonjr/memory-mcp",
   "bugs": "https://github.com/danielsimonjr/memory-mcp/issues",
@@ -20,7 +23,9 @@
     "tags",
     "export",
     "graphml",
-    "analytics"
+    "analytics",
+    "semantic-search",
+    "embeddings"
   ],
   "type": "module",
   "main": "./dist/index.js",
@@ -66,15 +71,23 @@
     "prepare": "npm run build",
     "watch": "tsc --watch",
     "test": "vitest run --coverage",
-    "typecheck": "tsc --noEmit --noUnusedLocals --noUnusedParameters --strict"
+    "typecheck": "tsc --noEmit --noUnusedLocals --noUnusedParameters --strict",
+    "clean": "shx rm -rf dist",
+    "docs:deps": "npx tsx tools/create-dependency-graph/create-dependency-graph.ts"
   },
   "dependencies": {
+    "@danielsimonjr/workerpool": "^10.0.1",
     "@modelcontextprotocol/sdk": "^1.21.1",
+    "async-mutex": "^0.5.0",
+    "better-sqlite3": "^11.7.0",
     "zod": "^4.1.13"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.12",
+    "@types/js-yaml": "^4.0.9",
     "@types/node": "^22",
     "@vitest/coverage-v8": "^4.0.13",
+    "js-yaml": "^4.1.1",
     "shx": "^0.4.0",
     "typescript": "^5.6.2",
     "vitest": "^4.0.13"