@yamo/memory-mesh 2.1.2 → 2.2.0

package/lib/memory/memory-mesh.js

@@ -15,12 +15,16 @@
 
 import { fileURLToPath } from 'url';
 import fs from "fs";
+import crypto from "crypto";
 import { LanceDBClient } from "../lancedb/client.js";
 import { getConfig } from "../lancedb/config.js";
 import { getEmbeddingDimension } from "../lancedb/schema.js";
 import { handleError, StorageError, QueryError } from "../lancedb/errors.js";
 import EmbeddingFactory from "../embeddings/factory.js";
 import { Scrubber } from "../scrubber/scrubber.js";
+import { KeywordSearch } from "../search/keyword-search.js";
+import { YamoEmitter } from "../yamo/emitter.js";
+import { LLMClient } from "../llm/client.js";
 
 /**
  * MemoryMesh class for managing vector memory storage
@@ -28,16 +32,40 @@ import { Scrubber } from "../scrubber/scrubber.js";
 class MemoryMesh {
   /**
    * Create a new MemoryMesh instance
+   * @param {Object} [options={}] - Configuration options
+   * @param {boolean} [options.enableYamo=true] - Enable YAMO block emission
+   * @param {boolean} [options.enableLLM=true] - Enable LLM for reflections
+   * @param {string} [options.agentId='default'] - Agent identifier for YAMO blocks
+   * @param {string} [options.llmProvider] - LLM provider (openai, anthropic, ollama)
+   * @param {string} [options.llmApiKey] - LLM API key
+   * @param {string} [options.llmModel] - LLM model name
    */
-  constructor() {
+  constructor(options = {}) {
     this.client = null;
     this.config = null;
     this.embeddingFactory = new EmbeddingFactory();
+    this.keywordSearch = new KeywordSearch();
     this.isInitialized = false;
     this.vectorDimension = 384; // Will be set during init()
 
+    // YAMO and LLM support
+    this.enableYamo = options.enableYamo !== false;  // Default: true
+    this.enableLLM = options.enableLLM !== false;    // Default: true
+    this.agentId = options.agentId || 'default';
+    this.yamoTable = null;  // Will be initialized in init()
+    this.llmClient = null;
+
+    // Initialize LLM client if enabled
+    if (this.enableLLM) {
+      this.llmClient = new LLMClient({
+        provider: options.llmProvider,
+        apiKey: options.llmApiKey,
+        model: options.llmModel
+      });
+    }
+
     // Scrubber for Layer 0 sanitization
-    this.scrubber = new Scrubber({ 
+    this.scrubber = new Scrubber({
       enabled: true,
       chunking: {
         minTokens: 1 // Allow short memories
@@ -221,11 +249,37 @@ class MemoryMesh {
       this.embeddingFactory.configure(embeddingConfigs);
       await this.embeddingFactory.init();
 
+      // Hydrate Keyword Search (In-Memory)
+      // Note: This is efficient for small datasets (< 10k).
+      // For larger, we should persist the inverted index or use LanceDB FTS.
+      if (this.client) {
+        try {
+          const allRecords = await this.client.getAll({ limit: 10000 });
+          this.keywordSearch.load(allRecords);
+        } catch (e) {
+          // Ignore if table doesn't exist yet
+        }
+      }
+
+      // Initialize YAMO blocks table if enabled
+      if (this.enableYamo && this.client && this.client.db) {
+        try {
+          const { createYamoTable } = await import('../yamo/schema.js');
+          this.yamoTable = await createYamoTable(this.client.db, 'yamo_blocks');
+          if (process.env.YAMO_DEBUG === 'true') {
+            console.error('[MemoryMesh] YAMO blocks table initialized');
+          }
+        } catch (e) {
+          // Log warning but don't fail initialization
+          console.warn('[MemoryMesh] Failed to initialize YAMO table:', e instanceof Error ? e.message : String(e));
+        }
+      }
+
       this.isInitialized = true;
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.init' });
+      throw e;
     }
   }
 
@@ -238,6 +292,10 @@ class MemoryMesh {
   async add(content, metadata = {}) {
     await this.init();
 
+    // Default to 'event' if no type provided
+    const type = metadata.type || 'event';
+    const enrichedMetadata = { ...metadata, type };
+
     try {
       // Layer 0: Scrubber Sanitization
       let processedContent = content;
@@ -272,7 +330,7 @@ class MemoryMesh {
 
       // Validate and sanitize inputs (legacy check)
       const sanitizedContent = this._sanitizeContent(processedContent);
-      const sanitizedMetadata = this._validateMetadata({ ...metadata, ...scrubbedMetadata });
+      const sanitizedMetadata = this._validateMetadata({ ...enrichedMetadata, ...scrubbedMetadata });
 
       // Generate ID
       const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
@@ -293,6 +351,25 @@ class MemoryMesh {
       if (!this.client) throw new Error('Database client not initialized');
       const result = await this.client.add(record);
 
+      // Add to Keyword Search
+      this.keywordSearch.add(record.id, record.content, sanitizedMetadata);
+
+      // Emit YAMO block for retain operation (async, non-blocking)
+      if (this.enableYamo) {
+        // Fire and forget - don't await
+        this._emitYamoBlock('retain', result.id, YamoEmitter.buildRetainBlock({
+          content: sanitizedContent,
+          metadata: sanitizedMetadata,
+          id: result.id,
+          agentId: this.agentId,
+          memoryType: sanitizedMetadata.type || 'event'
+        })).catch(err => {
+          if (process.env.YAMO_DEBUG === 'true') {
+            console.error('[MemoryMesh] YAMO emission failed in add():', err);
+          }
+        });
+      }
+
       return {
         id: result.id,
         content: sanitizedContent,
@@ -303,7 +380,142 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.add' });
+      throw e;
+    }
+  }
+
+  /**
+   * Reflect on recent memories to generate insights (enhanced with LLM + YAMO)
+   * @param {Object} options
+   * @param {string} [options.topic] - Topic to search for
+   * @param {number} [options.lookback=10] - Number of memories to consider
+   * @param {boolean} [options.generate=true] - Whether to generate reflection via LLM
+   * @returns {Promise<Object>} Reflection result with YAMO block
+   */
+  async reflect(options = {}) {
+    await this.init();
+
+    const lookback = options.lookback || 10;
+    const topic = options.topic;
+    const generate = options.generate !== false;
+
+    // Gather memories
+    let memories = [];
+    if (topic) {
+      memories = await this.search(topic, { limit: lookback });
+    } else {
+      const all = await this.getAll();
+      memories = all
+        .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
+        .slice(0, lookback);
+    }
+
+    const prompt = `Review these memories. Synthesize a high-level "belief" or "observation".`;
+
+    // Check if LLM generation is requested and available
+    if (!generate || !this.enableLLM || !this.llmClient) {
+      // Return prompt-only mode (backward compatible)
+      return {
+        topic,
+        count: memories.length,
+        context: memories.map(m => ({
+          content: m.content,
+          type: m.metadata?.type || 'event',
+          id: m.id
+        })),
+        prompt
+      };
+    }
+
+    // Generate reflection via LLM
+    let reflection = null;
+    let confidence = 0;
+
+    try {
+      const result = await this.llmClient.reflect(prompt, memories);
+      reflection = result.reflection;
+      confidence = result.confidence;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.warn(`[MemoryMesh] LLM reflection failed: ${errorMessage}`);
+      // Fall back to simple aggregation
+      reflection = `Aggregated from ${memories.length} memories on topic: ${topic || 'general'}`;
+      confidence = 0.5;
+    }
+
+    // Store reflection to memory
+    const reflectionId = `reflect_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+    await this.add(reflection, {
+      type: 'reflection',
+      topic: topic || 'general',
+      source_memory_count: memories.length,
+      confidence,
+      generated_at: new Date().toISOString()
+    });
+
+    // Emit YAMO block if enabled
+    let yamoBlock = null;
+    if (this.enableYamo) {
+      yamoBlock = YamoEmitter.buildReflectBlock({
+        topic: topic || 'general',
+        memoryCount: memories.length,
+        agentId: this.agentId,
+        reflection,
+        confidence
+      });
+
+      await this._emitYamoBlock('reflect', reflectionId, yamoBlock);
+    }
+
+    return {
+      id: reflectionId,
+      topic: topic || 'general',
+      reflection,
+      confidence,
+      sourceMemoryCount: memories.length,
+      yamoBlock,
+      createdAt: new Date().toISOString()
+    };
+  }
+
+  /**
+   * Emit a YAMO block to the YAMO blocks table
+   * @private
+   * @param {string} operationType - 'retain', 'recall', 'reflect'
+   * @param {string|undefined} memoryId - Associated memory ID (undefined for recall)
+   * @param {string} yamoText - The YAMO block text
+   */
+  async _emitYamoBlock(operationType, memoryId, yamoText) {
+    if (!this.yamoTable) {
+      if (process.env.YAMO_DEBUG === 'true') {
+        console.warn('[MemoryMesh] YAMO table not initialized, skipping emission');
+      }
+      return;
+    }
+
+    const yamoId = `yamo_${operationType}_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+
+    try {
+      await this.yamoTable.add([{
+        id: yamoId,
+        agent_id: this.agentId,
+        operation_type: operationType,
+        yamo_text: yamoText,
+        timestamp: new Date(),
+        block_hash: null,  // Future: blockchain anchoring
+        prev_hash: null,
+        metadata: JSON.stringify({
+          memory_id: memoryId || null,
+          timestamp: new Date().toISOString()
+        })
+      }]);
+
+      if (process.env.YAMO_DEBUG === 'true') {
+        console.log(`[MemoryMesh] YAMO block emitted: ${yamoId}`);
+      }
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.error(`[MemoryMesh] Failed to emit YAMO block: ${errorMessage}`);
     }
   }
 
@@ -377,7 +589,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.addBatch', count: entries.length });
+      throw e;
     }
   }
 
@@ -411,34 +623,83 @@ class MemoryMesh {
       // Generate embedding using EmbeddingFactory
       const vector = await this.embeddingFactory.embed(query);
 
-      // Perform semantic search
+      // 1. Vector Search
       if (!this.client) throw new Error('Database client not initialized');
-      const results = await this.client.search(vector, {
-        limit,
+      const vectorResults = await this.client.search(vector, {
+        limit: limit * 2, // Fetch more for re-ranking
         metric: 'cosine',
         filter
       });
 
-      // Format results
-      const formattedResults = results.map(result => ({
-        id: result.id,
-        content: result.content,
-        metadata: result.metadata,
-        score: result.score,
-        created_at: result.created_at
-      }));
+      // 2. Keyword Search
+      const keywordResults = this.keywordSearch.search(query, { limit: limit * 2 });
+
+      // 3. Reciprocal Rank Fusion (RRF)
+      const k = 60; // RRF constant
+      const scores = new Map(); // id -> score
+      const docMap = new Map(); // id -> doc
+
+      // Process Vector Results
+      vectorResults.forEach((doc, rank) => {
+        const rrf = 1 / (k + rank + 1);
+        scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
+        docMap.set(doc.id, doc);
+      });
+
+      // Process Keyword Results
+      keywordResults.forEach((doc, rank) => {
+        const rrf = 1 / (k + rank + 1);
+        scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
+        
+        if (!docMap.has(doc.id)) {
+           // Add keyword-only match
+           docMap.set(doc.id, {
+             id: doc.id,
+             content: doc.content,
+             metadata: doc.metadata,
+             score: 0, // Base score, will be overwritten
+             created_at: new Date().toISOString() // Approximate or missing
+           });
+        }
+      });
+
+      // Sort by RRF score
+      const mergedResults = Array.from(scores.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([id, score]) => {
+          const doc = docMap.get(id);
+          if (doc) return { ...doc, score }; 
+          return null; 
+        })
+        .filter(d => d !== null);
 
       // Cache the result (unless disabled)
       if (useCache) {
         const cacheKey = this._generateCacheKey(query, { limit, filter });
-        this._cacheResult(cacheKey, formattedResults);
+        this._cacheResult(cacheKey, mergedResults);
+      }
+
+      // Emit YAMO block for recall operation (async, non-blocking)
+      if (this.enableYamo) {
+        this._emitYamoBlock('recall', undefined, YamoEmitter.buildRecallBlock({
+          query,
+          resultCount: mergedResults.length,
+          limit,
+          agentId: this.agentId,
+          searchType: 'hybrid'
+        })).catch(err => {
+          if (process.env.YAMO_DEBUG === 'true') {
+            console.error('[MemoryMesh] YAMO emission failed in search():', err);
+          }
+        });
       }
 
-      return formattedResults;
+      return mergedResults;
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.search', query });
+      throw e;
     }
   }
 
@@ -469,7 +730,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.get', id });
+      throw e;
     }
   }
 
@@ -486,7 +747,63 @@ class MemoryMesh {
       return await this.client.getAll(options);
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.getAll' });
+      throw e;
+    }
+  }
+
+  /**
+   * Get YAMO blocks for this agent (audit trail)
+   * @param {Object} options - Query options
+   * @param {string} [options.operationType] - Filter by operation type ('retain', 'recall', 'reflect')
+   * @param {number} [options.limit=10] - Max results to return
+   * @returns {Promise<Array>} List of YAMO blocks
+   */
+  async getYamoLog(options = {}) {
+    if (!this.yamoTable) {
+      return [];
+    }
+
+    const limit = options.limit || 10;
+    const operationType = options.operationType;
+
+    try {
+      // Use search with empty vector to get all records, then filter
+      // This avoids using the protected execute() method
+      const allResults = [];
+
+      // Build query manually using the LanceDB table
+      // @ts-ignore - LanceDB types may not match exactly
+      const table = this.yamoTable;
+
+      // Get all records and filter
+      // @ts-ignore
+      const records = await table.query().limit(limit * 2).toArrow();
+
+      // Process Arrow table
+      for (const row of records) {
+        const opType = row.operationType;
+        if (!operationType || opType === operationType) {
+          allResults.push({
+            id: row.id,
+            agentId: row.agentId,
+            operationType: row.operationType,
+            yamoText: row.yamoText,
+            timestamp: row.timestamp,
+            blockHash: row.blockHash,
+            metadata: row.metadata ? JSON.parse(row.metadata) : null
+          });
+
+          if (allResults.length >= limit) {
+            break;
+          }
+        }
+      }
+
+      return allResults;
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      console.error('[MemoryMesh] Failed to get YAMO log:', errorMessage);
+      return [];
     }
   }
 
@@ -548,7 +865,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.update', id });
+      throw e;
     }
   }
 
@@ -564,6 +881,9 @@ class MemoryMesh {
       if (!this.client) throw new Error('Database client not initialized');
       const result = await this.client.delete(id);
 
+      // Remove from Keyword Search
+      this.keywordSearch.remove(id);
+
       return {
         deleted: result.id,
         success: result.success
@@ -572,7 +892,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.delete', id });
+      throw e;
     }
   }
 
@@ -599,7 +919,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.stats' });
+      throw e;
     }
   }
 
@@ -803,7 +1123,7 @@ async function run() {
 
   try {
     // Route to appropriate action
-    if (action === 'ingest') {
+    if (action === 'ingest' || action === 'store') {
       // Validate required fields
       if (!input.content) {
         console.error('❌ Error: "content" field is required for ingest action');
@@ -866,6 +1186,36 @@ ${jsonResult}
       console.log(`[MemoryMesh] Deleted record ${result.deleted}`);
       console.log(JSON.stringify({ status: "ok", ...result }));
 
+    } else if (action === 'export') {
+      const records = await mesh.getAll({ limit: input.limit || 10000 });
+      console.log(JSON.stringify({ status: "ok", count: records.length, records }));
+
+    } else if (action === 'reflect') {
+      // Enhanced reflect with LLM support
+      const enableLLM = input.llm !== false;  // Default true
+      const result = await mesh.reflect({
+        topic: input.topic,
+        lookback: input.limit || 10,
+        generate: enableLLM
+      });
+
+      if (result.reflection) {
+        // New format with LLM-generated reflection
+        console.log(JSON.stringify({
+          status: "ok",
+          reflection: result.reflection,
+          confidence: result.confidence,
+          id: result.id,
+          topic: result.topic,
+          sourceMemoryCount: result.sourceMemoryCount,
+          yamoBlock: result.yamoBlock,
+          createdAt: result.createdAt
+        }));
+      } else {
+        // Old format for backward compatibility (prompt-only mode)
+        console.log(JSON.stringify({ status: "ok", ...result }));
+      }
+
     } else if (action === 'stats') {
       const stats = await mesh.stats();
       console.log('[MemoryMesh] Database Statistics:');
@@ -896,8 +1246,8 @@ ${jsonResult}
   }
 }
 
-// Export for testing
-export { MemoryMesh };
+// Export for testing and CLI usage
+export { MemoryMesh, run };
 export default MemoryMesh;
 
 // Run CLI if called directly

package/lib/search/keyword-search.js

@@ -0,0 +1,144 @@
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+
+export class KeywordSearch {
+  constructor() {
+    this.index = new Map(); // token -> Map<docId, tf>
+    this.docLengths = new Map(); // docId -> length
+    this.idf = new Map(); // token -> idf value
+    this.docs = new Map(); // docId -> content (optional, for snippet)
+    this.isDirty = false;
+  }
+
+  /**
+   * Tokenize text into normalized terms
+   * @param {string} text 
+   * @returns {string[]} tokens
+   */
+  tokenize(text) {
+    if (!text) return [];
+    return text.toLowerCase()
+      .replace(/[^\w\s]/g, '') // Remove punctuation
+      .split(/\s+/)
+      .filter(t => t.length > 2) // Filter stopwords/short
+      .map(t => t.substring(0, 20)); // Truncate
+  }
+
+  /**
+   * Add a document to the index
+   * @param {string} id 
+   * @param {string} content 
+   * @param {Object} [metadata]
+   */
+  add(id, content, metadata = {}) {
+    const tokens = this.tokenize(content);
+    const termFreqs = new Map();
+
+    tokens.forEach(t => {
+      termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+    });
+
+    this.docLengths.set(id, tokens.length);
+    this.docs.set(id, { content, metadata });
+
+    // Update index
+    for (const [token, freq] of termFreqs.entries()) {
+      if (!this.index.has(token)) {
+        this.index.set(token, new Map());
+      }
+      this.index.get(token).set(id, freq);
+    }
+
+    this.isDirty = true;
+  }
+
+  /**
+   * Remove a document
+   * @param {string} id 
+   */
+  remove(id) {
+    this.docLengths.delete(id);
+    this.docs.delete(id);
+    
+    // This is expensive O(Vocab), but okay for small scale
+    for (const docMap of this.index.values()) {
+      docMap.delete(id);
+    }
+    this.isDirty = true;
+  }
+
+  /**
+   * Recalculate IDF scores
+   */
+  _computeStats() {
+    if (!this.isDirty) return;
+    
+    const N = this.docLengths.size;
+    this.idf.clear();
+
+    for (const [token, docMap] of this.index.entries()) {
+      const df = docMap.size;
+      // Standard IDF: log(N / (df + 1)) + 1
+      const idf = Math.log(N / (df + 1)) + 1;
+      this.idf.set(token, idf);
+    }
+
+    this.isDirty = false;
+  }
+
+  /**
+   * Search for query terms
+   * @param {string} query
+   * @param {Object} options
+   * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+   */
+  search(query, options = {}) {
+    this._computeStats();
+    
+    const tokens = this.tokenize(query);
+    const scores = new Map(); // docId -> score
+    const matches = new Map(); // docId -> matched tokens
+
+    const limit = options.limit || 10;
+
+    for (const token of tokens) {
+      const docMap = this.index.get(token);
+      if (!docMap) continue;
+
+      const idf = this.idf.get(token) || 0;
+
+      for (const [docId, tf] of docMap.entries()) {
+        // TF-IDF Score
+        // Score = tf * idf * (normalization?)
+        // Simple variant:
+        const score = tf * idf;
+        
+        scores.set(docId, (scores.get(docId) || 0) + score);
+        
+        if (!matches.has(docId)) matches.set(docId, []);
+        matches.get(docId).push(token);
+      }
+    }
+
+    // Convert to array and sort
+    return Array.from(scores.entries())
+      .map(([id, score]) => ({
+        id,
+        score,
+        matches: matches.get(id) || [],
+        ...this.docs.get(id)
+      }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+
+  /**
+   * Bulk load records
+   * @param {Array} records 
+   */
+  load(records) {
+    records.forEach(r => this.add(r.id, r.content, r.metadata));
+  }
+}