@danielsimonjr/memory-mcp 11.0.1 → 11.1.1

package/dist/search/IncrementalIndexer.js

@@ -1,342 +0,0 @@
-/**
- * Incremental Indexer
- *
- * Phase 12 Sprint 5: Queues embedding index updates and batch-processes them
- * efficiently using flush thresholds and timer-based flushing.
- *
- * @module search/IncrementalIndexer
- */
-/**
- * Default indexer options.
- */
-export const DEFAULT_INDEXER_OPTIONS = {
-    flushThreshold: 50,
-    flushIntervalMs: 5000,
-    embeddingMode: 'document',
-};
-/**
- * Incremental indexer for embedding vectors.
- *
- * Queues index updates and batch-processes them efficiently:
- * - Auto-flush when queue reaches threshold
- * - Timer-based flush for time-sensitive updates
- * - Supports create, update, and delete operations
- * - Graceful shutdown with final flush
- *
- * @example
- * ```typescript
- * const indexer = new IncrementalIndexer(embeddingService, vectorStore, {
- *   flushThreshold: 100,
- *   flushIntervalMs: 10000,
- * });
- *
- * // Queue operations
- * indexer.queueCreate('entity1', 'Entity text content');
- * indexer.queueUpdate('entity2', 'Updated text content');
- * indexer.queueDelete('entity3');
- *
- * // Manual flush
- * const result = await indexer.flush();
- * console.log(`Processed ${result.processed} operations`);
- *
- * // Graceful shutdown
- * await indexer.shutdown();
- * ```
- */
-export class IncrementalIndexer {
-    embeddingService;
-    vectorStore;
-    options;
-    onProgress;
-    queue = [];
-    flushTimer = null;
-    isFlushing = false;
-    isShutdown = false;
-    /**
-     * Create a new incremental indexer.
-     *
-     * @param embeddingService - Service for generating embeddings
-     * @param vectorStore - Store for embedding vectors
-     * @param options - Indexer configuration options
-     */
-    constructor(embeddingService, vectorStore, options) {
-        this.embeddingService = embeddingService;
-        this.vectorStore = vectorStore;
-        this.options = { ...DEFAULT_INDEXER_OPTIONS, ...options };
-        this.onProgress = options?.onProgress;
-        // Start the flush timer
-        this.startFlushTimer();
-    }
-    /**
-     * Start the timer-based flush interval.
-     */
-    startFlushTimer() {
-        if (this.flushTimer || this.isShutdown) {
-            return;
-        }
-        this.flushTimer = setInterval(async () => {
-            if (this.queue.length > 0 && !this.isFlushing) {
-                await this.flush();
-            }
-        }, this.options.flushIntervalMs);
-    }
-    /**
-     * Stop the flush timer.
-     */
-    stopFlushTimer() {
-        if (this.flushTimer) {
-            clearInterval(this.flushTimer);
-            this.flushTimer = null;
-        }
-    }
-    /**
-     * Queue a create operation.
-     *
-     * @param entityName - Name of the entity to create
-     * @param text - Text content to embed
-     */
-    queueCreate(entityName, text) {
-        if (this.isShutdown) {
-            throw new Error('Indexer is shutdown');
-        }
-        // Remove any existing operations for this entity
-        this.removeFromQueue(entityName);
-        this.queue.push({
-            type: 'create',
-            entityName,
-            text,
-            queuedAt: Date.now(),
-        });
-        this.checkAutoFlush();
-    }
-    /**
-     * Queue an update operation.
-     *
-     * @param entityName - Name of the entity to update
-     * @param text - Updated text content to embed
-     */
-    queueUpdate(entityName, text) {
-        if (this.isShutdown) {
-            throw new Error('Indexer is shutdown');
-        }
-        // Remove any existing operations for this entity
-        this.removeFromQueue(entityName);
-        this.queue.push({
-            type: 'update',
-            entityName,
-            text,
-            queuedAt: Date.now(),
-        });
-        this.checkAutoFlush();
-    }
-    /**
-     * Queue a delete operation.
-     *
-     * @param entityName - Name of the entity to delete
-     */
-    queueDelete(entityName) {
-        if (this.isShutdown) {
-            throw new Error('Indexer is shutdown');
-        }
-        // Remove any existing operations for this entity
-        this.removeFromQueue(entityName);
-        this.queue.push({
-            type: 'delete',
-            entityName,
-            queuedAt: Date.now(),
-        });
-        this.checkAutoFlush();
-    }
-    /**
-     * Remove all queued operations for an entity.
-     *
-     * @param entityName - Entity name to remove from queue
-     */
-    removeFromQueue(entityName) {
-        this.queue = this.queue.filter(op => op.entityName !== entityName);
-    }
-    /**
-     * Check if auto-flush threshold is reached.
-     */
-    checkAutoFlush() {
-        if (this.queue.length >= this.options.flushThreshold && !this.isFlushing) {
-            // Use setImmediate to avoid blocking
-            setImmediate(() => this.flush());
-        }
-    }
-    /**
-     * Flush the queue and process all pending operations.
-     *
-     * @returns Result of the flush operation
-     */
-    async flush() {
-        if (this.isFlushing) {
-            return {
-                processed: 0,
-                succeeded: 0,
-                failed: 0,
-                errors: [],
-                durationMs: 0,
-            };
-        }
-        this.isFlushing = true;
-        const startTime = Date.now();
-        const errors = [];
-        let succeeded = 0;
-        let failed = 0;
-        // Take the current queue
-        const operations = [...this.queue];
-        this.queue = [];
-        try {
-            // Separate operations by type
-            const createOps = operations.filter(op => op.type === 'create');
-            const updateOps = operations.filter(op => op.type === 'update');
-            const deleteOps = operations.filter(op => op.type === 'delete');
-            // Process deletes first (fast, O(1))
-            for (const op of deleteOps) {
-                try {
-                    this.vectorStore.remove(op.entityName);
-                    succeeded++;
-                }
-                catch (error) {
-                    failed++;
-                    errors.push({
-                        entityName: op.entityName,
-                        error: error instanceof Error ? error.message : String(error),
-                    });
-                }
-            }
-            // Batch process creates and updates together
-            const embedOps = [...createOps, ...updateOps];
-            if (embedOps.length > 0) {
-                const texts = embedOps.map(op => op.text);
-                const entityNames = embedOps.map(op => op.entityName);
-                try {
-                    // Check if the embedding service has the batch with progress method
-                    let embeddings;
-                    if (this.onProgress && 'embedBatchWithProgress' in this.embeddingService) {
-                        embeddings = await this.embeddingService
-                            .embedBatchWithProgress(texts, this.options.embeddingMode, this.onProgress);
-                    }
-                    else {
-                        embeddings = await this.embeddingService.embedBatch(texts, this.options.embeddingMode);
-                    }
-                    // Store each embedding
-                    for (let i = 0; i < embeddings.length; i++) {
-                        try {
-                            this.vectorStore.add(entityNames[i], embeddings[i]);
-                            succeeded++;
-                        }
-                        catch (error) {
-                            failed++;
-                            errors.push({
-                                entityName: entityNames[i],
-                                error: error instanceof Error ? error.message : String(error),
-                            });
-                        }
-                    }
-                }
-                catch (error) {
-                    // Batch embedding failed, count all as failed
-                    failed += embedOps.length;
-                    for (const op of embedOps) {
-                        errors.push({
-                            entityName: op.entityName,
-                            error: error instanceof Error ? error.message : String(error),
-                        });
-                    }
-                }
-            }
-        }
-        finally {
-            this.isFlushing = false;
-        }
-        return {
-            processed: operations.length,
-            succeeded,
-            failed,
-            errors,
-            durationMs: Date.now() - startTime,
-        };
-    }
-    /**
-     * Get the current queue size.
-     *
-     * @returns Number of operations in the queue
-     */
-    getQueueSize() {
-        return this.queue.length;
-    }
-    /**
-     * Get the current queue contents (for debugging/monitoring).
-     *
-     * @returns Copy of the current queue
-     */
-    getQueue() {
-        return [...this.queue];
-    }
-    /**
-     * Check if the indexer is currently flushing.
-     *
-     * @returns True if flushing
-     */
-    isBusy() {
-        return this.isFlushing;
-    }
-    /**
-     * Check if the indexer is shutdown.
-     *
-     * @returns True if shutdown
-     */
-    isShutdownComplete() {
-        return this.isShutdown;
-    }
-    /**
-     * Graceful shutdown with final flush.
-     *
-     * Stops the timer and flushes any remaining operations.
-     *
-     * @returns Result of the final flush
-     */
-    async shutdown() {
-        this.isShutdown = true;
-        this.stopFlushTimer();
-        // Wait for any in-progress flush to complete
-        while (this.isFlushing) {
-            await new Promise(resolve => setTimeout(resolve, 10));
-        }
-        // Final flush
-        return this.flush();
-    }
-    /**
-     * Update indexer options.
-     *
-     * Note: Changes to flushIntervalMs will take effect on the next interval.
-     *
-     * @param options - New options to apply
-     */
-    updateOptions(options) {
-        const { onProgress, ...rest } = options;
-        if (onProgress !== undefined) {
-            this.onProgress = onProgress;
-        }
-        if (Object.keys(rest).length > 0) {
-            this.options = { ...this.options, ...rest };
-            // Restart timer if interval changed
-            if (rest.flushIntervalMs !== undefined) {
-                this.stopFlushTimer();
-                this.startFlushTimer();
-            }
-        }
-    }
-    /**
-     * Clear the queue without processing.
-     *
-     * @returns Number of operations cleared
-     */
-    clearQueue() {
-        const count = this.queue.length;
-        this.queue = [];
-        return count;
-    }
-}

package/dist/search/OptimizedInvertedIndex.d.ts

@@ -1,163 +0,0 @@
-/**
- * Optimized Inverted Index
- *
- * Memory-efficient inverted index using integer IDs and Uint32Array
- * for fast multi-term intersection queries.
- *
- * Phase 12 Sprint 3: Search Algorithm Optimization
- *
- * @module search/OptimizedInvertedIndex
- */
-/**
- * Statistics about memory usage.
- */
-export interface IndexMemoryUsage {
-    /** Total bytes used by posting lists */
-    postingListBytes: number;
-    /** Total bytes used by ID map */
-    idMapBytes: number;
-    /** Total bytes used by term index */
-    termIndexBytes: number;
-    /** Total estimated memory usage in bytes */
-    totalBytes: number;
-    /** Number of unique terms */
-    termCount: number;
-    /** Number of documents indexed */
-    documentCount: number;
-}
-/**
- * Result from a posting list lookup.
- */
-export interface PostingListResult {
-    /** Term that was looked up */
-    term: string;
-    /** Document IDs containing the term (sorted) */
-    docIds: Uint32Array;
-}
-/**
- * Optimized Inverted Index using integer document IDs.
- *
- * Memory Optimizations:
- * 1. Uses integer IDs instead of string entity names
- * 2. Stores posting lists as Uint32Array (4 bytes per ID vs ~20+ bytes per string)
- * 3. Maintains sorted posting lists for efficient intersection
- *
- * Performance Optimizations:
- * 1. Sorted array intersection is O(n+m) where n,m are posting list lengths
- * 2. Early termination when one list is exhausted
- * 3. Binary search available for unbalanced list sizes
- *
- * @example
- * ```typescript
- * const index = new OptimizedInvertedIndex();
- * index.addDocument('entity1', ['machine', 'learning', 'ai']);
- * index.addDocument('entity2', ['deep', 'learning', 'neural']);
- *
- * // Find documents containing both 'machine' AND 'learning'
- * const results = index.intersect(['machine', 'learning']);
- * console.log(results); // ['entity1']
- * ```
- */
-export declare class OptimizedInvertedIndex {
-    /** Map from entity name to integer ID */
-    private entityToId;
-    /** Map from integer ID to entity name */
-    private idToEntity;
-    /** Next available ID */
-    private nextId;
-    /** Inverted index: term -> sorted array of document IDs */
-    private postingLists;
-    /** Temporary posting lists (before finalization) */
-    private tempPostingLists;
-    /** Whether the index is finalized (posting lists converted to Uint32Array) */
-    private finalized;
-    /**
-     * Add a document to the index.
-     *
-     * @param entityName - Unique document identifier
-     * @param terms - Array of terms in the document (should be lowercase)
-     */
-    addDocument(entityName: string, terms: string[]): void;
-    /**
-     * Remove a document from the index.
-     *
-     * @param entityName - Document to remove
-     * @returns True if document was found and removed
-     */
-    removeDocument(entityName: string): boolean;
-    /**
-     * Finalize the index by converting posting lists to Uint32Array.
-     *
-     * This should be called after bulk indexing for optimal memory usage.
-     * The index can still be updated after finalization, but it will
-     * temporarily use more memory during updates.
-     */
-    finalize(): void;
-    /**
-     * Convert finalized index back to mutable format.
-     */
-    private unfinalize;
-    /**
-     * Get posting list for a term.
-     *
-     * @param term - Term to look up
-     * @returns Posting list result or null if term not found
-     */
-    getPostingList(term: string): PostingListResult | null;
-    /**
-     * Perform intersection of posting lists for multiple terms.
-     *
-     * Returns entity names that contain ALL specified terms.
-     *
-     * @param terms - Array of terms to intersect
-     * @returns Array of entity names containing all terms
-     */
-    intersect(terms: string[]): string[];
-    /**
-     * Perform union of posting lists for multiple terms.
-     *
-     * Returns entity names that contain ANY of the specified terms.
-     *
-     * @param terms - Array of terms to union
-     * @returns Array of entity names containing any term
-     */
-    union(terms: string[]): string[];
-    /**
-     * Get entities containing a single term.
-     *
-     * @param term - Term to search for
-     * @returns Array of entity names containing the term
-     */
-    search(term: string): string[];
-    /**
-     * Intersect two sorted Uint32Arrays.
-     *
-     * Uses merge-style intersection which is O(n+m).
-     */
-    private intersectTwo;
-    /**
-     * Get memory usage statistics.
-     */
-    getMemoryUsage(): IndexMemoryUsage;
-    /**
-     * Clear the entire index.
-     */
-    clear(): void;
-    /**
-     * Get the number of documents in the index.
-     */
-    get documentCount(): number;
-    /**
-     * Get the number of unique terms in the index.
-     */
-    get termCount(): number;
-    /**
-     * Check if an entity is indexed.
-     */
-    hasDocument(entityName: string): boolean;
-    /**
-     * Check if a term exists in the index.
-     */
-    hasTerm(term: string): boolean;
-}
-//# sourceMappingURL=OptimizedInvertedIndex.d.ts.map

package/dist/search/OptimizedInvertedIndex.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"OptimizedInvertedIndex.d.ts","sourceRoot":"","sources":["../../src/search/OptimizedInvertedIndex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,wCAAwC;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,qCAAqC;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,MAAM,EAAE,WAAW,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,sBAAsB;IACjC,yCAAyC;IACzC,OAAO,CAAC,UAAU,CAAkC;IAEpD,yCAAyC;IACzC,OAAO,CAAC,UAAU,CAAkC;IAEpD,wBAAwB;IACxB,OAAO,CAAC,MAAM,CAAa;IAE3B,2DAA2D;IAC3D,OAAO,CAAC,YAAY,CAAuC;IAE3D,oDAAoD;IACpD,OAAO,CAAC,gBAAgB,CAAoC;IAE5D,8EAA8E;IAC9E,OAAO,CAAC,SAAS,CAAkB;IAEnC;;;;;OAKG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI;IAiCtD;;;;;OAKG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IA6B3C;;;;;;OAMG;IACH,QAAQ,IAAI,IAAI;IAiBhB;;OAEG;IACH,OAAO,CAAC,UAAU;IAalB;;;;;OAKG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI;IActD;;;;;;;OAOG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAqCpC;;;;;;;OAOG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAyBhC;;;;;OAKG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAY9B;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAoBpB;;OAEG;IACH,cAAc,IAAI,gBAAgB;IAwClC;;OAEG;IACH,KAAK,IAAI,IAAI;IASb;;OAEG;IACH,IAAI,aAAa,IAAI,MAAM,CAE1B;IAED;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAItB;IAED;;OAEG;IACH,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAIxC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;CAK/B"}