@danielsimonjr/memory-mcp 0.47.1 → 9.8.0

package/dist/search/TFIDFEventSync.js

@@ -0,0 +1,133 @@
+/**
+ * TF-IDF Event Sync
+ *
+ * Phase 10 Sprint 3: Hooks TFIDFIndexManager to graph events for automatic
+ * incremental index updates when entities change.
+ *
+ * @module search/TFIDFEventSync
+ */
+/**
+ * Phase 10 Sprint 3: Synchronizes TF-IDF index with graph changes via events.
+ *
+ * Listens to graph events and triggers incremental index updates automatically.
+ * More efficient than rebuilding the entire index on every change.
+ *
+ * @example
+ * ```typescript
+ * const storage = new GraphStorage('/data/memory.jsonl');
+ * const indexManager = new TFIDFIndexManager('/data');
+ *
+ * // Load or build index
+ * await indexManager.loadIndex();
+ *
+ * // Enable automatic sync
+ * const sync = new TFIDFEventSync(indexManager, storage.events, storage);
+ * sync.enable();
+ *
+ * // Now entities added to storage will automatically update the index
+ * await storage.appendEntity({ name: 'New', entityType: 'test', observations: [] });
+ *
+ * // Disable when done
+ * sync.disable();
+ * ```
+ */
+export class TFIDFEventSync {
+    indexManager;
+    eventEmitter;
+    storage;
+    unsubscribers = [];
+    enabled = false;
+    /**
+     * Create a new TFIDFEventSync instance.
+     *
+     * @param indexManager - TFIDFIndexManager to sync
+     * @param eventEmitter - GraphEventEmitter to listen to
+     * @param storage - Storage to fetch entity data from (for updates)
+     */
+    constructor(indexManager, eventEmitter, storage) {
+        this.indexManager = indexManager;
+        this.eventEmitter = eventEmitter;
+        this.storage = storage;
+    }
+    /**
+     * Enable automatic index synchronization.
+     *
+     * Subscribes to entity:created, entity:updated, and entity:deleted events.
+     */
+    enable() {
+        if (this.enabled) {
+            return;
+        }
+        // Subscribe to entity events
+        this.unsubscribers.push(this.eventEmitter.on('entity:created', this.handleEntityCreated.bind(this)));
+        this.unsubscribers.push(this.eventEmitter.on('entity:updated', this.handleEntityUpdated.bind(this)));
+        this.unsubscribers.push(this.eventEmitter.on('entity:deleted', this.handleEntityDeleted.bind(this)));
+        this.enabled = true;
+    }
+    /**
+     * Disable automatic index synchronization.
+     *
+     * Unsubscribes from all events.
+     */
+    disable() {
+        if (!this.enabled) {
+            return;
+        }
+        for (const unsubscribe of this.unsubscribers) {
+            unsubscribe();
+        }
+        this.unsubscribers = [];
+        this.enabled = false;
+    }
+    /**
+     * Check if synchronization is enabled.
+     *
+     * @returns True if enabled
+     */
+    isEnabled() {
+        return this.enabled;
+    }
+    /**
+     * Handle entity:created event.
+     * @private
+     */
+    handleEntityCreated(event) {
+        if (!this.indexManager.isInitialized()) {
+            return;
+        }
+        this.indexManager.addDocument({
+            name: event.entity.name,
+            entityType: event.entity.entityType,
+            observations: event.entity.observations,
+        });
+    }
+    /**
+     * Handle entity:updated event.
+     * @private
+     */
+    async handleEntityUpdated(event) {
+        if (!this.indexManager.isInitialized()) {
+            return;
+        }
+        // Fetch the current entity state
+        const graph = await this.storage.loadGraph();
+        const entity = graph.entities.find(e => e.name === event.entityName);
+        if (entity) {
+            this.indexManager.updateDocument({
+                name: entity.name,
+                entityType: entity.entityType,
+                observations: entity.observations,
+            });
+        }
+    }
+    /**
+     * Handle entity:deleted event.
+     * @private
+     */
+    handleEntityDeleted(event) {
+        if (!this.indexManager.isInitialized()) {
+            return;
+        }
+        this.indexManager.removeDocument(event.entityName);
+    }
+}

package/dist/search/TFIDFIndexManager.d.ts

@@ -0,0 +1,151 @@
+/**
+ * TF-IDF Index Manager
+ *
+ * Manages pre-calculated TF-IDF indexes for fast ranked search.
+ * Handles index building, incremental updates, and persistence.
+ *
+ * @module search/TFIDFIndexManager
+ */
+import type { TFIDFIndex, KnowledgeGraph, ReadonlyKnowledgeGraph } from '../types/index.js';
+/**
+ * Manages TF-IDF index lifecycle: building, updating, and persistence.
+ */
+export declare class TFIDFIndexManager {
+    private indexPath;
+    private index;
+    constructor(storageDir: string);
+    /**
+     * Build a complete TF-IDF index from a knowledge graph.
+     *
+     * @param graph - Knowledge graph to index
+     * @returns Newly built TF-IDF index
+     */
+    buildIndex(graph: ReadonlyKnowledgeGraph): Promise<TFIDFIndex>;
+    /**
+     * Update the index incrementally when entities change.
+     *
+     * More efficient than rebuilding the entire index.
+     *
+     * @param graph - Updated knowledge graph
+     * @param changedEntityNames - Names of entities that changed
+     */
+    updateIndex(graph: ReadonlyKnowledgeGraph, changedEntityNames: Set<string>): Promise<TFIDFIndex>;
+    /**
+     * Load index from disk.
+     *
+     * @returns Loaded index or null if not found
+     */
+    loadIndex(): Promise<TFIDFIndex | null>;
+    /**
+     * Save index to disk.
+     *
+     * @param index - Index to save (uses cached index if not provided)
+     */
+    saveIndex(index?: TFIDFIndex): Promise<void>;
+    /**
+     * Get the current cached index.
+     *
+     * @returns Cached index or null if not loaded
+     */
+    getIndex(): TFIDFIndex | null;
+    /**
+     * Clear the cached index and delete from disk.
+     */
+    clearIndex(): Promise<void>;
+    /**
+     * Check if the index needs rebuilding based on graph state.
+     *
+     * @param graph - Current knowledge graph
+     * @returns True if index should be rebuilt
+     */
+    needsRebuild(graph: KnowledgeGraph): boolean;
+    /**
+     * Phase 10 Sprint 3: Add a single document to the index incrementally.
+     *
+     * More efficient than rebuilding the entire index for single entity additions.
+     * Updates TF for the new document and recalculates IDF for affected terms.
+     *
+     * @param entity - The entity to add
+     *
+     * @example
+     * ```typescript
+     * const indexManager = new TFIDFIndexManager('/data');
+     * await indexManager.loadIndex();
+     *
+     * // Add new entity
+     * indexManager.addDocument({
+     *   name: 'NewEntity',
+     *   entityType: 'person',
+     *   observations: ['Software engineer']
+     * });
+     * ```
+     */
+    addDocument(entity: {
+        name: string;
+        entityType: string;
+        observations: string[];
+    }): void;
+    /**
+     * Phase 10 Sprint 3: Remove a single document from the index incrementally.
+     *
+     * More efficient than rebuilding the entire index for single entity deletions.
+     * Recalculates IDF for terms that were in the removed document.
+     *
+     * @param entityName - Name of the entity to remove
+     *
+     * @example
+     * ```typescript
+     * indexManager.removeDocument('DeletedEntity');
+     * ```
+     */
+    removeDocument(entityName: string): void;
+    /**
+     * Phase 10 Sprint 3: Update a single document in the index incrementally.
+     *
+     * More efficient than rebuilding the entire index for single entity updates.
+     * Handles both term changes and observation updates.
+     *
+     * @param entity - The updated entity
+     *
+     * @example
+     * ```typescript
+     * indexManager.updateDocument({
+     *   name: 'ExistingEntity',
+     *   entityType: 'person',
+     *   observations: ['Updated observations']
+     * });
+     * ```
+     */
+    updateDocument(entity: {
+        name: string;
+        entityType: string;
+        observations: string[];
+    }): void;
+    /**
+     * Phase 10 Sprint 3: Recalculate IDF scores for a set of terms.
+     *
+     * @param terms - Set of terms to recalculate IDF for
+     * @private
+     */
+    private recalculateIDFForTerms;
+    /**
+     * Phase 10 Sprint 3: Recalculate IDF scores for ALL terms in the index.
+     *
+     * Called when the total document count changes (add/remove document).
+     * @private
+     */
+    private recalculateAllIDF;
+    /**
+     * Phase 10 Sprint 3: Check if the index is loaded/initialized.
+     *
+     * @returns True if index is available
+     */
+    isInitialized(): boolean;
+    /**
+     * Phase 10 Sprint 3: Get the number of documents in the index.
+     *
+     * @returns Document count or 0 if not initialized
+     */
+    getDocumentCount(): number;
+}
+//# sourceMappingURL=TFIDFIndexManager.d.ts.map

package/dist/search/TFIDFIndexManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TFIDFIndexManager.d.ts","sourceRoot":"","sources":["../../src/search/TFIDFIndexManager.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAkB,cAAc,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAgB5G;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,KAAK,CAA2B;gBAE5B,UAAU,EAAE,MAAM;IAI9B;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,sBAAsB,GAAG,OAAO,CAAC,UAAU,CAAC;IAgDpE;;;;;;;OAOG;IACG,WAAW,CAAC,KAAK,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAgEtG;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAmB7C;;;;OAIG;IACG,SAAS,CAAC,KAAK,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;;;OAIG;IACH,QAAQ,IAAI,UAAU,GAAG,IAAI;IAI7B;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IASjC;;;;;OAKG;IACH,YAAY,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO;IAsB5C;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,WAAW,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IA+BvF;;;;;;;;;;;;OAYG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAqBxC;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI;IAgD1F;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;IAkC9B;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IA8BzB;;;;OAIG;IACH,aAAa,IAAI,OAAO;IAIxB;;;;OAIG;IACH,gBAAgB,IAAI,MAAM;CAG3B"}

package/dist/search/TFIDFIndexManager.js

@@ -8,7 +8,7 @@
  */
 import * as fs from 'fs/promises';
 import * as path from 'path';
-import { calculateIDF, tokenize } from '../utils/tfidf.js';
+import { calculateIDFFromTokenSets, tokenize } from '../utils/index.js';
 const INDEX_VERSION = '1.0';
 const INDEX_FILENAME = 'tfidf-index.json';
 /**
@@ -28,18 +28,17 @@ export class TFIDFIndexManager {
      */
     async buildIndex(graph) {
         const documents = new Map();
-        const allDocumentTexts = [];
-        const allTokens = [];
-        // Build document vectors
+        const allTokenSets = [];
+        // Build document vectors - tokenize once per document
         for (const entity of graph.entities) {
             const documentText = [
                 entity.name,
                 entity.entityType,
                 ...entity.observations,
             ].join(' ');
-            allDocumentTexts.push(documentText);
             const tokens = tokenize(documentText);
-            allTokens.push(tokens);
+            const tokenSet = new Set(tokens);
+            allTokenSets.push(tokenSet);
             // Calculate term frequencies
             const termFreq = {};
             for (const term of tokens) {
@@ -51,11 +50,11 @@ export class TFIDFIndexManager {
                 documentText,
             });
         }
-        // Calculate IDF for all terms
+        // Calculate IDF for all terms using pre-tokenized sets (O(1) lookup per document)
         const idf = new Map();
-        const allTerms = new Set(allTokens.flat());
+        const allTerms = new Set(allTokenSets.flatMap(s => Array.from(s)));
         for (const term of allTerms) {
-            const idfScore = calculateIDF(term, allDocumentTexts);
+            const idfScore = calculateIDFFromTokenSets(term, allTokenSets);
             idf.set(term, idfScore);
         }
         this.index = {
@@ -80,8 +79,7 @@ export class TFIDFIndexManager {
             return this.buildIndex(graph);
         }
         // Rebuild document vectors for changed entities
-        const allDocumentTexts = [];
-        const allTokens = [];
+        const allTokenSets = [];
         const updatedDocuments = new Map(this.index.documents);
         // Remove deleted entities
         for (const entityName of changedEntityNames) {
@@ -90,16 +88,16 @@ export class TFIDFIndexManager {
                 updatedDocuments.delete(entityName);
             }
         }
-        // Update/add changed entities
+        // Update/add changed entities - tokenize once per document
         for (const entity of graph.entities) {
             const documentText = [
                 entity.name,
                 entity.entityType,
                 ...entity.observations,
             ].join(' ');
-            allDocumentTexts.push(documentText);
             const tokens = tokenize(documentText);
-            allTokens.push(tokens);
+            const tokenSet = new Set(tokens);
+            allTokenSets.push(tokenSet);
             if (changedEntityNames.has(entity.name)) {
                 // Calculate term frequencies for changed entity
                 const termFreq = {};
@@ -113,11 +111,11 @@ export class TFIDFIndexManager {
                 });
             }
         }
-        // Recalculate IDF (need all documents for accurate IDF)
+        // Recalculate IDF using pre-tokenized sets (O(1) lookup per document)
         const idf = new Map();
-        const allTerms = new Set(allTokens.flat());
+        const allTerms = new Set(allTokenSets.flatMap(s => Array.from(s)));
         for (const term of allTerms) {
-            const idfScore = calculateIDF(term, allDocumentTexts);
+            const idfScore = calculateIDFFromTokenSets(term, allTokenSets);
             idf.set(term, idfScore);
         }
         this.index = {
@@ -214,4 +212,221 @@ export class TFIDFIndexManager {
         }
         return false;
     }
+    // ==================== Phase 10 Sprint 3: Incremental Index Updates ====================
+    /**
+     * Phase 10 Sprint 3: Add a single document to the index incrementally.
+     *
+     * More efficient than rebuilding the entire index for single entity additions.
+     * Updates TF for the new document and recalculates IDF for affected terms.
+     *
+     * @param entity - The entity to add
+     *
+     * @example
+     * ```typescript
+     * const indexManager = new TFIDFIndexManager('/data');
+     * await indexManager.loadIndex();
+     *
+     * // Add new entity
+     * indexManager.addDocument({
+     *   name: 'NewEntity',
+     *   entityType: 'person',
+     *   observations: ['Software engineer']
+     * });
+     * ```
+     */
+    addDocument(entity) {
+        if (!this.index) {
+            // Can't add to non-existent index
+            return;
+        }
+        // Build document text and tokens
+        const documentText = [entity.name, entity.entityType, ...entity.observations].join(' ');
+        const tokens = tokenize(documentText);
+        // Calculate term frequencies
+        const termFreq = {};
+        for (const term of tokens) {
+            termFreq[term] = (termFreq[term] || 0) + 1;
+        }
+        // Add to documents map
+        this.index.documents.set(entity.name, {
+            entityName: entity.name,
+            terms: termFreq,
+            documentText,
+        });
+        // Update IDF for ALL terms because N changed (total document count)
+        // IDF = log(N/df), and N has increased
+        this.recalculateAllIDF();
+        // Update timestamp
+        this.index.lastUpdated = new Date().toISOString();
+    }
+    /**
+     * Phase 10 Sprint 3: Remove a single document from the index incrementally.
+     *
+     * More efficient than rebuilding the entire index for single entity deletions.
+     * Recalculates IDF for terms that were in the removed document.
+     *
+     * @param entityName - Name of the entity to remove
+     *
+     * @example
+     * ```typescript
+     * indexManager.removeDocument('DeletedEntity');
+     * ```
+     */
+    removeDocument(entityName) {
+        if (!this.index) {
+            return;
+        }
+        const document = this.index.documents.get(entityName);
+        if (!document) {
+            return;
+        }
+        // Remove from documents map
+        this.index.documents.delete(entityName);
+        // Update IDF for ALL terms because N changed (total document count)
+        // IDF = log(N/df), and N has decreased
+        this.recalculateAllIDF();
+        // Update timestamp
+        this.index.lastUpdated = new Date().toISOString();
+    }
+    /**
+     * Phase 10 Sprint 3: Update a single document in the index incrementally.
+     *
+     * More efficient than rebuilding the entire index for single entity updates.
+     * Handles both term changes and observation updates.
+     *
+     * @param entity - The updated entity
+     *
+     * @example
+     * ```typescript
+     * indexManager.updateDocument({
+     *   name: 'ExistingEntity',
+     *   entityType: 'person',
+     *   observations: ['Updated observations']
+     * });
+     * ```
+     */
+    updateDocument(entity) {
+        if (!this.index) {
+            return;
+        }
+        const oldDocument = this.index.documents.get(entity.name);
+        const oldTerms = oldDocument ? new Set(Object.keys(oldDocument.terms)) : new Set();
+        // Build new document
+        const documentText = [entity.name, entity.entityType, ...entity.observations].join(' ');
+        const tokens = tokenize(documentText);
+        const newTerms = new Set(tokens);
+        // Calculate term frequencies
+        const termFreq = {};
+        for (const term of tokens) {
+            termFreq[term] = (termFreq[term] || 0) + 1;
+        }
+        // Update documents map
+        this.index.documents.set(entity.name, {
+            entityName: entity.name,
+            terms: termFreq,
+            documentText,
+        });
+        // Find terms that changed (added or removed)
+        const changedTerms = new Set();
+        for (const term of oldTerms) {
+            if (!newTerms.has(term)) {
+                changedTerms.add(term);
+            }
+        }
+        for (const term of newTerms) {
+            if (!oldTerms.has(term)) {
+                changedTerms.add(term);
+            }
+        }
+        // Recalculate IDF for changed terms
+        if (changedTerms.size > 0) {
+            this.recalculateIDFForTerms(changedTerms);
+        }
+        // Update timestamp
+        this.index.lastUpdated = new Date().toISOString();
+    }
+    /**
+     * Phase 10 Sprint 3: Recalculate IDF scores for a set of terms.
+     *
+     * @param terms - Set of terms to recalculate IDF for
+     * @private
+     */
+    recalculateIDFForTerms(terms) {
+        if (!this.index) {
+            return;
+        }
+        const totalDocs = this.index.documents.size;
+        if (totalDocs === 0) {
+            // No documents, clear all IDF for these terms
+            for (const term of terms) {
+                this.index.idf.delete(term);
+            }
+            return;
+        }
+        // Count documents containing each term
+        for (const term of terms) {
+            let docCount = 0;
+            for (const doc of this.index.documents.values()) {
+                if (term in doc.terms) {
+                    docCount++;
+                }
+            }
+            if (docCount > 0) {
+                // IDF = log(N / df) where N = total docs, df = doc frequency
+                const idfScore = Math.log(totalDocs / docCount);
+                this.index.idf.set(term, idfScore);
+            }
+            else {
+                // Term no longer exists in any document
+                this.index.idf.delete(term);
+            }
+        }
+    }
+    /**
+     * Phase 10 Sprint 3: Recalculate IDF scores for ALL terms in the index.
+     *
+     * Called when the total document count changes (add/remove document).
+     * @private
+     */
+    recalculateAllIDF() {
+        if (!this.index) {
+            return;
+        }
+        const totalDocs = this.index.documents.size;
+        if (totalDocs === 0) {
+            // No documents, clear all IDF
+            this.index.idf.clear();
+            return;
+        }
+        // Build term -> document count map
+        const termDocCounts = new Map();
+        for (const doc of this.index.documents.values()) {
+            for (const term of Object.keys(doc.terms)) {
+                termDocCounts.set(term, (termDocCounts.get(term) ?? 0) + 1);
+            }
+        }
+        // Clear old IDF and recalculate
+        this.index.idf.clear();
+        for (const [term, docCount] of termDocCounts) {
+            // IDF = log(N / df) where N = total docs, df = doc frequency
+            const idfScore = Math.log(totalDocs / docCount);
+            this.index.idf.set(term, idfScore);
+        }
+    }
+    /**
+     * Phase 10 Sprint 3: Check if the index is loaded/initialized.
+     *
+     * @returns True if index is available
+     */
+    isInitialized() {
+        return this.index !== null;
+    }
+    /**
+     * Phase 10 Sprint 3: Get the number of documents in the index.
+     *
+     * @returns Document count or 0 if not initialized
+     */
+    getDocumentCount() {
+        return this.index?.documents.size ?? 0;
+    }
 }