@danielsimonjr/memory-mcp 12.5.2 → 12.7.0

package/dist/utils/taskScheduler.js

@@ -0,0 +1,466 @@
+/**
+ * Task Scheduler
+ *
+ * Advanced task scheduling utilities using workerpool.
+ * Phase 8 Sprint 4: Priority queues, concurrency control, progress tracking.
+ *
+ * @module utils/taskScheduler
+ */
+import workerpool from '@danielsimonjr/workerpool/modern';
+// ==================== Types ====================
+/**
+ * Task priority levels.
+ * Higher priority tasks are executed first.
+ */
+export var TaskPriority;
+(function (TaskPriority) {
+    TaskPriority[TaskPriority["LOW"] = 0] = "LOW";
+    TaskPriority[TaskPriority["NORMAL"] = 1] = "NORMAL";
+    TaskPriority[TaskPriority["HIGH"] = 2] = "HIGH";
+    TaskPriority[TaskPriority["CRITICAL"] = 3] = "CRITICAL";
+})(TaskPriority || (TaskPriority = {}));
+/**
+ * Task status in the queue.
+ */
+export var TaskStatus;
+(function (TaskStatus) {
+    TaskStatus["PENDING"] = "pending";
+    TaskStatus["RUNNING"] = "running";
+    TaskStatus["COMPLETED"] = "completed";
+    TaskStatus["FAILED"] = "failed";
+    TaskStatus["CANCELLED"] = "cancelled";
+})(TaskStatus || (TaskStatus = {}));
+/**
+ * Priority Task Queue with advanced scheduling.
+ *
+ * Features:
+ * - Priority-based execution (CRITICAL > HIGH > NORMAL > LOW)
+ * - Configurable concurrency limits
+ * - Progress tracking
+ * - Graceful error handling
+ * - Task cancellation
+ *
+ * @example
+ * ```typescript
+ * const queue = new TaskQueue({ concurrency: 4 });
+ *
+ * // Add tasks with different priorities
+ * queue.enqueue({
+ *   id: 'task1',
+ *   priority: TaskPriority.HIGH,
+ *   fn: (x: number) => x * 2,
+ *   input: 5,
+ * });
+ *
+ * // Process all tasks
+ * const results = await queue.processAll();
+ * ```
+ */
+export class TaskQueue {
+    queue = [];
+    running = new Map();
+    completed = [];
+    pool = null;
+    concurrency;
+    defaultTimeout;
+    isProcessing = false;
+    totalExecutionTime = 0;
+    totalProcessed = 0;
+    useWorkerPool;
+    constructor(options = {}) {
+        this.concurrency = options.concurrency ?? Math.max(1, workerpool.cpus - 1);
+        this.defaultTimeout = options.timeout ?? 30000;
+        this.useWorkerPool = options.useWorkerPool ?? true;
+    }
+    /**
+     * Get or create the worker pool.
+     */
+    getPool() {
+        if (!this.pool) {
+            this.pool = workerpool.pool({
+                maxWorkers: this.concurrency,
+                workerType: 'thread',
+            });
+        }
+        return this.pool;
+    }
+    /**
+     * Add a task to the queue.
+     *
+     * @param task - Task to add
+     * @returns Promise that resolves when the task completes
+     */
+    enqueue(task) {
+        return new Promise((resolve, reject) => {
+            const queuedTask = {
+                ...task,
+                status: TaskStatus.PENDING,
+                addedAt: Date.now(),
+                resolve: resolve,
+                reject,
+            };
+            // Insert based on priority (higher priority first)
+            const insertIndex = this.queue.findIndex(t => t.priority < task.priority);
+            if (insertIndex === -1) {
+                this.queue.push(queuedTask);
+            }
+            else {
+                this.queue.splice(insertIndex, 0, queuedTask);
+            }
+            // Start processing if not already running
+            if (!this.isProcessing) {
+                this.processNext();
+            }
+        });
+    }
+    /**
+     * Process the next task in the queue.
+     */
+    async processNext() {
+        if (this.running.size >= this.concurrency || this.queue.length === 0) {
+            if (this.running.size === 0 && this.queue.length === 0) {
+                this.isProcessing = false;
+            }
+            return;
+        }
+        this.isProcessing = true;
+        const task = this.queue.shift();
+        if (!task)
+            return;
+        task.status = TaskStatus.RUNNING;
+        this.running.set(task.id, task);
+        const startTime = Date.now();
+        try {
+            // Execute task - try worker pool first, fall back to direct execution
+            let result;
+            if (this.useWorkerPool) {
+                try {
+                    const pool = this.getPool();
+                    const fnString = task.fn.toString();
+                    const timeout = task.timeout ?? this.defaultTimeout;
+                    result = await pool
+                        .exec((input, fnStr) => {
+                        // eslint-disable-next-line no-new-func
+                        const fn = new Function('return ' + fnStr)();
+                        return fn(input);
+                    }, [task.input, fnString])
+                        .timeout(timeout);
+                }
+                catch {
+                    // Fall back to direct execution
+                    result = await Promise.resolve(task.fn(task.input));
+                }
+            }
+            else {
+                // Direct execution without worker pool
+                result = await Promise.resolve(task.fn(task.input));
+            }
+            const endTime = Date.now();
+            const duration = endTime - startTime;
+            const taskResult = {
+                id: task.id,
+                status: TaskStatus.COMPLETED,
+                result,
+                duration,
+                startedAt: startTime,
+                completedAt: endTime,
+            };
+            this.totalExecutionTime += duration;
+            this.totalProcessed++;
+            this.completed.push(taskResult);
+            this.running.delete(task.id);
+            task.resolve(taskResult);
+        }
+        catch (error) {
+            const endTime = Date.now();
+            const duration = endTime - startTime;
+            const taskResult = {
+                id: task.id,
+                status: TaskStatus.FAILED,
+                error: error instanceof Error ? error : new Error(String(error)),
+                duration,
+                startedAt: startTime,
+                completedAt: endTime,
+            };
+            this.totalProcessed++;
+            this.completed.push(taskResult);
+            this.running.delete(task.id);
+            task.resolve(taskResult);
+        }
+        // Process next task
+        this.processNext();
+    }
+    /**
+     * Cancel a pending task.
+     *
+     * @param taskId - ID of the task to cancel
+     * @returns True if task was cancelled, false if not found or already running
+     */
+    cancel(taskId) {
+        const index = this.queue.findIndex(t => t.id === taskId);
+        if (index === -1)
+            return false;
+        const task = this.queue.splice(index, 1)[0];
+        task.status = TaskStatus.CANCELLED;
+        const result = {
+            id: task.id,
+            status: TaskStatus.CANCELLED,
+            duration: 0,
+            startedAt: Date.now(),
+            completedAt: Date.now(),
+        };
+        task.resolve(result);
+        return true;
+    }
+    /**
+     * Wait for all tasks to complete.
+     *
+     * @returns Array of all task results
+     */
+    async drain() {
+        // Wait for queue to empty and all running tasks to complete
+        while (this.queue.length > 0 || this.running.size > 0) {
+            await new Promise(resolve => setTimeout(resolve, 10));
+        }
+        return [...this.completed];
+    }
+    /**
+     * Get queue statistics.
+     */
+    getStats() {
+        return {
+            pending: this.queue.length,
+            running: this.running.size,
+            completed: this.completed.filter(r => r.status === TaskStatus.COMPLETED).length,
+            failed: this.completed.filter(r => r.status === TaskStatus.FAILED).length,
+            averageExecutionTime: this.totalProcessed > 0 ? this.totalExecutionTime / this.totalProcessed : 0,
+            totalProcessed: this.totalProcessed,
+        };
+    }
+    /**
+     * Clear all completed results.
+     */
+    clearCompleted() {
+        this.completed = [];
+    }
+    /**
+     * Shutdown the task queue and release resources.
+     */
+    async shutdown() {
+        // Cancel all pending tasks
+        for (const task of this.queue) {
+            task.status = TaskStatus.CANCELLED;
+            task.resolve({
+                id: task.id,
+                status: TaskStatus.CANCELLED,
+                duration: 0,
+                startedAt: Date.now(),
+                completedAt: Date.now(),
+            });
+        }
+        this.queue = [];
+        // Terminate worker pool
+        if (this.pool) {
+            await this.pool.terminate();
+            this.pool = null;
+        }
+        this.isProcessing = false;
+    }
+}
+// ==================== Batch Processing Utilities ====================
+/**
+ * Process items in parallel batches with progress tracking.
+ *
+ * Unlike parallelMap, this provides:
+ * - Progress callbacks
+ * - Configurable concurrency
+ * - Error handling options
+ * - Task-level timeouts
+ *
+ * @template T - Input item type
+ * @template R - Output item type
+ * @param items - Array of items to process
+ * @param fn - Processing function (must be serializable)
+ * @param options - Batch processing options
+ * @returns Array of results (or errors if stopOnError is false)
+ *
+ * @example
+ * ```typescript
+ * const results = await batchProcess(
+ *   urls,
+ *   (url: string) => fetch(url).then(r => r.json()),
+ *   {
+ *     concurrency: 5,
+ *     timeout: 10000,
+ *     onProgress: ({ completed, total, percentage }) => {
+ *       console.log(`Progress: ${percentage.toFixed(1)}%`);
+ *     },
+ *   }
+ * );
+ * ```
+ */
+export async function batchProcess(items, fn, options = {}) {
+    const { concurrency = Math.max(1, workerpool.cpus - 1), timeout = 30000, onProgress, stopOnError = false, } = options;
+    const results = [];
+    let completed = 0;
+    const total = items.length;
+    // Process in batches respecting concurrency
+    for (let i = 0; i < items.length; i += concurrency) {
+        const batch = items.slice(i, i + concurrency);
+        const batchPromises = batch.map(async (item, batchIndex) => {
+            const itemIndex = i + batchIndex;
+            try {
+                // Execute with timeout
+                const result = await Promise.race([
+                    Promise.resolve(fn(item)),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Task timeout')), timeout)),
+                ]);
+                results[itemIndex] = { success: true, result };
+            }
+            catch (error) {
+                const err = error instanceof Error ? error : new Error(String(error));
+                results[itemIndex] = { success: false, error: err };
+                if (stopOnError) {
+                    throw err;
+                }
+            }
+            finally {
+                completed++;
+                if (onProgress) {
+                    onProgress({
+                        completed,
+                        total,
+                        percentage: (completed / total) * 100,
+                        currentTaskId: `item-${itemIndex}`,
+                    });
+                }
+            }
+        });
+        await Promise.all(batchPromises);
+    }
+    return results;
+}
+/**
+ * Execute tasks with rate limiting.
+ *
+ * Ensures tasks don't exceed a specified rate (tasks per second).
+ *
+ * @template T - Input item type
+ * @template R - Output item type
+ * @param items - Items to process
+ * @param fn - Processing function
+ * @param rateLimit - Maximum tasks per second
+ * @returns Array of results
+ *
+ * @example
+ * ```typescript
+ * // Process max 10 items per second
+ * const results = await rateLimitedProcess(
+ *   items,
+ *   (item) => processItem(item),
+ *   10
+ * );
+ * ```
+ */
+export async function rateLimitedProcess(items, fn, rateLimit) {
+    const results = [];
+    const minInterval = 1000 / rateLimit;
+    let lastExecutionTime = 0;
+    for (const item of items) {
+        // Calculate wait time
+        const now = Date.now();
+        const timeSinceLast = now - lastExecutionTime;
+        if (timeSinceLast < minInterval) {
+            await new Promise(resolve => setTimeout(resolve, minInterval - timeSinceLast));
+        }
+        lastExecutionTime = Date.now();
+        const result = await fn(item);
+        results.push(result);
+    }
+    return results;
+}
+/**
+ * Retry a function with exponential backoff.
+ *
+ * @template T - Return type
+ * @param fn - Function to retry
+ * @param options - Retry options
+ * @returns Result of the function
+ *
+ * @example
+ * ```typescript
+ * const result = await withRetry(
+ *   () => fetchData(),
+ *   { maxRetries: 3, baseDelay: 1000 }
+ * );
+ * ```
+ */
+export async function withRetry(fn, options = {}) {
+    const { maxRetries = 3, baseDelay = 1000, maxDelay = 30000, onRetry } = options;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error instanceof Error ? error : new Error(String(error));
+            if (attempt < maxRetries) {
+                const delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
+                if (onRetry) {
+                    onRetry(lastError, attempt + 1);
+                }
+                await new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+    }
+    throw lastError;
+}
+/**
+ * Create a debounced version of a function.
+ *
+ * @template T - Function arguments type
+ * @template R - Return type
+ * @param fn - Function to debounce
+ * @param delay - Delay in milliseconds
+ * @returns Debounced function
+ */
+export function debounce(fn, delay) {
+    let timeoutId = null;
+    let pendingResolve = null;
+    return (...args) => {
+        return new Promise(resolve => {
+            if (timeoutId) {
+                clearTimeout(timeoutId);
+            }
+            pendingResolve = resolve;
+            timeoutId = setTimeout(() => {
+                const result = fn(...args);
+                if (pendingResolve) {
+                    pendingResolve(result);
+                }
+                timeoutId = null;
+                pendingResolve = null;
+            }, delay);
+        });
+    };
+}
+/**
+ * Create a throttled version of a function.
+ *
+ * @template T - Function arguments type
+ * @template R - Return type
+ * @param fn - Function to throttle
+ * @param limit - Minimum time between calls in milliseconds
+ * @returns Throttled function
+ */
+export function throttle(fn, limit) {
+    let lastCall = 0;
+    return (...args) => {
+        const now = Date.now();
+        if (now - lastCall >= limit) {
+            lastCall = now;
+            return fn(...args);
+        }
+        return undefined;
+    };
+}

package/dist/workers/WorkerPool.d.ts

@@ -0,0 +1,81 @@
+/**
+ * WorkerPool
+ *
+ * Manages a pool of worker threads for parallel task execution.
+ *
+ * @module workers/WorkerPool
+ */
+/**
+ * Worker pool configuration options.
+ */
+interface PoolOptions {
+    /** Maximum number of concurrent workers (default: CPU count - 1) */
+    maxWorkers?: number;
+    /** Absolute path to the worker script */
+    workerPath: string;
+}
+/**
+ * Statistics about the worker pool.
+ */
+export interface PoolStats {
+    /** Number of currently active workers */
+    activeWorkers: number;
+    /** Number of tasks waiting in queue */
+    queueSize: number;
+    /** Maximum number of concurrent workers */
+    maxWorkers: number;
+}
+/**
+ * Worker pool for parallel task execution.
+ *
+ * Features:
+ * - Limits concurrent workers to maxWorkers
+ * - Queues tasks when all workers are busy
+ * - Automatically terminates workers after task completion
+ * - Supports batch execution with executeAll()
+ */
+export declare class WorkerPool<TInput, TOutput> {
+    private taskQueue;
+    private activeWorkers;
+    private readonly maxWorkers;
+    private readonly workerPath;
+    constructor(options: PoolOptions);
+    /**
+     * Execute a single task in a worker thread.
+     *
+     * @param data - Input data for the worker
+     * @returns Promise that resolves with the worker's output
+     */
+    execute(data: TInput): Promise<TOutput>;
+    /**
+     * Execute multiple tasks in parallel.
+     *
+     * @param items - Array of input data for workers
+     * @returns Promise that resolves with array of outputs
+     */
+    executeAll(items: TInput[]): Promise<TOutput[]>;
+    /**
+     * Process the task queue, spawning workers as needed.
+     */
+    private processQueue;
+    /**
+     * Run a single task in a worker thread.
+     *
+     * @param task - Task to execute
+     */
+    private runTask;
+    /**
+     * Wait for all active workers to complete.
+     *
+     * @returns Promise that resolves when all workers are done
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Get current statistics about the worker pool.
+     *
+     * @returns Pool statistics
+     */
+    getStats(): PoolStats;
+}
+export {};
+//# sourceMappingURL=WorkerPool.d.ts.map

package/dist/workers/WorkerPool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WorkerPool.d.ts","sourceRoot":"","sources":["../../src/workers/WorkerPool.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAiBH;;GAEG;AACH,UAAU,WAAW;IACnB,oEAAoE;IACpE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;GAQG;AACH,qBAAa,UAAU,CAAC,MAAM,EAAE,OAAO;IACrC,OAAO,CAAC,SAAS,CAAqC;IACtD,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,OAAO,EAAE,WAAW;IAKhC;;;;;OAKG;IACG,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO7C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAKrD;;OAEG;IACH,OAAO,CAAC,YAAY;IASpB;;;;OAIG;IACH,OAAO,CAAC,OAAO;IAsCf;;;;OAIG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAM/B;;;;OAIG;IACH,QAAQ,IAAI,SAAS;CAOtB"}

package/dist/workers/WorkerPool.js

@@ -0,0 +1,121 @@
+/**
+ * WorkerPool
+ *
+ * Manages a pool of worker threads for parallel task execution.
+ *
+ * @module workers/WorkerPool
+ */
+import { Worker } from 'worker_threads';
+import { cpus } from 'os';
+/**
+ * Worker pool for parallel task execution.
+ *
+ * Features:
+ * - Limits concurrent workers to maxWorkers
+ * - Queues tasks when all workers are busy
+ * - Automatically terminates workers after task completion
+ * - Supports batch execution with executeAll()
+ */
+export class WorkerPool {
+    taskQueue = [];
+    activeWorkers = 0;
+    maxWorkers;
+    workerPath;
+    constructor(options) {
+        this.maxWorkers = options.maxWorkers ?? Math.max(1, cpus().length - 1);
+        this.workerPath = options.workerPath;
+    }
+    /**
+     * Execute a single task in a worker thread.
+     *
+     * @param data - Input data for the worker
+     * @returns Promise that resolves with the worker's output
+     */
+    async execute(data) {
+        return new Promise((resolve, reject) => {
+            this.taskQueue.push({ data, resolve, reject });
+            this.processQueue();
+        });
+    }
+    /**
+     * Execute multiple tasks in parallel.
+     *
+     * @param items - Array of input data for workers
+     * @returns Promise that resolves with array of outputs
+     */
+    async executeAll(items) {
+        const promises = items.map(item => this.execute(item));
+        return Promise.all(promises);
+    }
+    /**
+     * Process the task queue, spawning workers as needed.
+     */
+    processQueue() {
+        while (this.taskQueue.length > 0 && this.activeWorkers < this.maxWorkers) {
+            const task = this.taskQueue.shift();
+            if (task) {
+                this.runTask(task);
+            }
+        }
+    }
+    /**
+     * Run a single task in a worker thread.
+     *
+     * @param task - Task to execute
+     */
+    runTask(task) {
+        this.activeWorkers++;
+        let taskCompleted = false;
+        const worker = new Worker(this.workerPath, {
+            workerData: task.data,
+        });
+        worker.on('message', (result) => {
+            if (!taskCompleted) {
+                taskCompleted = true;
+                task.resolve(result);
+                this.activeWorkers--;
+                worker.terminate();
+                this.processQueue();
+            }
+        });
+        worker.on('error', (error) => {
+            if (!taskCompleted) {
+                taskCompleted = true;
+                task.reject(error);
+                this.activeWorkers--;
+                worker.terminate();
+                this.processQueue();
+            }
+        });
+        worker.on('exit', (code) => {
+            if (!taskCompleted && code !== 0) {
+                taskCompleted = true;
+                task.reject(new Error(`Worker stopped with exit code ${code}`));
+                this.activeWorkers--;
+                this.processQueue();
+            }
+        });
+    }
+    /**
+     * Wait for all active workers to complete.
+     *
+     * @returns Promise that resolves when all workers are done
+     */
+    async shutdown() {
+        while (this.activeWorkers > 0) {
+            await new Promise(resolve => setTimeout(resolve, 10));
+        }
+    }
+    /**
+     * Get current statistics about the worker pool.
+     *
+     * @returns Pool statistics
+     */
+    getStats() {
+        return {
+            activeWorkers: this.activeWorkers,
+            queueSize: this.taskQueue.length,
+            maxWorkers: this.maxWorkers,
+        };
+    }
+}

package/dist/workers/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Workers Module
+ *
+ * Worker thread utilities for parallel processing.
+ * Phase 8: Uses workerpool library for worker management.
+ *
+ * @module workers
+ */
+export type { Pool, PoolStats } from '@danielsimonjr/workerpool/modern';
+export type { WorkerInput, MatchResult } from './levenshteinWorker.js';
+export { levenshteinDistance, similarity, searchEntities } from './levenshteinWorker.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/workers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/workers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kCAAkC,CAAC;AAGxE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/workers/index.js

@@ -0,0 +1,9 @@
+/**
+ * Workers Module
+ *
+ * Worker thread utilities for parallel processing.
+ * Phase 8: Uses workerpool library for worker management.
+ *
+ * @module workers
+ */
+export { levenshteinDistance, similarity, searchEntities } from './levenshteinWorker.js';