@yamo/memory-mesh 2.1.2 → 2.1.3

package/bin/memory_mesh.js

@@ -1,69 +1,14 @@
 #!/usr/bin/env node
 
 /**
- * MemoryMesh CLI Adapter
- * Provides a portable interface for skills to interact with the MemoryMesh system.
- * 
- * Usage:
- *   node tools/memory_mesh.js store <content_string_or_json> [metadata_json]
- *   node tools/memory_mesh.js search <query_string> [limit]
+ * MemoryMesh CLI Entry Point
+ * Delegates to the core CLI handler in lib/memory/memory-mesh.js
  */
 
-import { MemoryMesh } from '../lib/memory/memory-mesh.js';
-import path from 'path';
+import { run } from '../lib/memory/memory-mesh.js';
 
-// Parse arguments
-const args = process.argv.slice(2);
-const command = args[0];
-
-async function main() {
-    try {
-        const mesh = new MemoryMesh();
-        // Wait for initialization if necessary (MemoryMesh constructor usually doesn't await, 
-        // but operations might need internal init. We assume standard usage.)
-
-        if (command === 'store') {
-            const content = args[1];
-            let metadata = {};
-            if (args[2]) {
-                try {
-                    metadata = JSON.parse(args[2]);
-                } catch (e) {
-                    console.error(JSON.stringify({ error: "Invalid metadata JSON" }));
-                    process.exit(1);
-                }
-            }
-            
-            if (!content) {
-                console.error(JSON.stringify({ error: "Content required for store command" }));
-                process.exit(1);
-            }
-
-            const result = await mesh.add(content, metadata);
-            console.log(JSON.stringify({ success: true, id: result.id, message: "Memory stored successfully" }));
-            
-        } else if (command === 'search') {
-            const query = args[1];
-            const limit = parseInt(args[2]) || 5;
-            
-            if (!query) {
-                console.error(JSON.stringify({ error: "Query required for search command" }));
-                process.exit(1);
-            }
-
-            const results = await mesh.search(query, { limit });
-            console.log(JSON.stringify({ success: true, results: results }));
-            
-        } else {
-            console.error(JSON.stringify({ error: `Unknown command: ${command}` }));
-            console.error("Usage: node tools/memory_mesh.js [store|search] ...");
-            process.exit(1);
-        }
-
-    } catch (error) {
-        console.error(JSON.stringify({ error: error.message }));
-        process.exit(1);
-    }
-}
-
-main();
+// Execute the main CLI handler
+run().catch(err => {
+  console.error(`❌ Fatal Error: ${err.message}`);
+  process.exit(1);
+});

package/lib/embeddings/factory.js

@@ -7,10 +7,11 @@ import EmbeddingService from "./service.js";
 import { ConfigurationError, EmbeddingError } from "../lancedb/errors.js";
 
 class EmbeddingFactory {
-  constructor() {
+  constructor(ServiceClass = EmbeddingService) {
     this.primaryService = null;
     this.fallbackServices = [];
     this.configured = false;
+    this.ServiceClass = ServiceClass;
   }
 
   /**
@@ -22,10 +23,10 @@ class EmbeddingFactory {
     // Sort by priority (lower = higher priority)
     configs.sort((a, b) => a.priority - b.priority);
 
-    this.primaryService = new EmbeddingService(configs[0]);
+    this.primaryService = new this.ServiceClass(configs[0]);
 
     if (configs.length > 1) {
-      this.fallbackServices = configs.slice(1).map(c => new EmbeddingService(c));
+      this.fallbackServices = configs.slice(1).map(c => new this.ServiceClass(c));
     }
 
     this.configured = true;

package/lib/lancedb/client.js

@@ -28,6 +28,7 @@ class LanceDBClient {
    * @param {number} [config.maxRetries] - Maximum connection retries (default: 3)
    * @param {number} [config.retryDelay] - Delay between retries in ms (default: 1000)
    * @param {number} [config.vectorDimension] - Vector dimension for embeddings (default: 384)
+   * @param {Object} [config.driver] - LanceDB driver instance (for testing)
    */
   constructor(config = {}) {
     this.uri = (config && config.uri) || process.env.LANCEDB_URI || './data/lancedb';
@@ -35,6 +36,7 @@ class LanceDBClient {
     this.maxRetries = (config && config.maxRetries) || 3;
     this.retryDelay = (config && config.retryDelay) || 1000;
     this.vectorDimension = (config && config.vectorDimension) || DEFAULT_VECTOR_DIMENSION;
+    this.driver = (config && config.driver) || lancedb;
 
     // Connection state
     this.db = null;
@@ -66,7 +68,7 @@ class LanceDBClient {
         }
 
         // Connect to database
-        this.db = await lancedb.connect(this.uri);
+        this.db = await this.driver.connect(this.uri);
 
         // Initialize table with dynamic dimension (creates if doesn't exist, opens if it does)
         this.table = await createMemoryTableWithDimension(this.db, this.tableName, this.vectorDimension);

package/lib/memory/memory-mesh.js

@@ -21,6 +21,7 @@ import { getEmbeddingDimension } from "../lancedb/schema.js";
 import { handleError, StorageError, QueryError } from "../lancedb/errors.js";
 import EmbeddingFactory from "../embeddings/factory.js";
 import { Scrubber } from "../scrubber/scrubber.js";
+import { KeywordSearch } from "../search/keyword-search.js";
 
 /**
  * MemoryMesh class for managing vector memory storage
@@ -33,6 +34,7 @@ class MemoryMesh {
     this.client = null;
     this.config = null;
     this.embeddingFactory = new EmbeddingFactory();
+    this.keywordSearch = new KeywordSearch();
     this.isInitialized = false;
     this.vectorDimension = 384; // Will be set during init()
 
@@ -221,11 +223,23 @@ class MemoryMesh {
       this.embeddingFactory.configure(embeddingConfigs);
       await this.embeddingFactory.init();
 
+      // Hydrate Keyword Search (In-Memory)
+      // Note: This is efficient for small datasets (< 10k).
+      // For larger, we should persist the inverted index or use LanceDB FTS.
+      if (this.client) {
+        try {
+          const allRecords = await this.client.getAll({ limit: 10000 });
+          this.keywordSearch.load(allRecords);
+        } catch (e) {
+          // Ignore if table doesn't exist yet
+        }
+      }
+
       this.isInitialized = true;
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.init' });
+      throw e;
     }
   }
 
@@ -238,6 +252,10 @@ class MemoryMesh {
   async add(content, metadata = {}) {
     await this.init();
 
+    // Default to 'event' if no type provided
+    const type = metadata.type || 'event';
+    const enrichedMetadata = { ...metadata, type };
+
     try {
       // Layer 0: Scrubber Sanitization
       let processedContent = content;
@@ -272,7 +290,7 @@ class MemoryMesh {
 
       // Validate and sanitize inputs (legacy check)
       const sanitizedContent = this._sanitizeContent(processedContent);
-      const sanitizedMetadata = this._validateMetadata({ ...metadata, ...scrubbedMetadata });
+      const sanitizedMetadata = this._validateMetadata({ ...enrichedMetadata, ...scrubbedMetadata });
 
       // Generate ID
       const id = `mem_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
@@ -293,6 +311,9 @@ class MemoryMesh {
       if (!this.client) throw new Error('Database client not initialized');
       const result = await this.client.add(record);
 
+      // Add to Keyword Search
+      this.keywordSearch.add(record.id, record.content, sanitizedMetadata);
+
       return {
         id: result.id,
         content: sanitizedContent,
@@ -303,10 +324,42 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.add' });
+      throw e;
     }
   }
 
+  /**
+   * Reflect on recent memories to generate insights
+   * @param {Object} options 
+   * @returns {Promise<Object>} Reflection prompt and context
+   */
+  async reflect(options = {}) {
+    await this.init();
+    const lookback = options.lookback || 10;
+    const topic = options.topic;
+
+    let memories = [];
+    if (topic) {
+      memories = await this.search(topic, { limit: lookback });
+    } else {
+      const all = await this.getAll();
+      memories = all
+        .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime())
+        .slice(0, lookback);
+    }
+
+    return {
+      topic,
+      count: memories.length,
+      context: memories.map(m => ({ 
+        content: m.content, 
+        type: m.metadata?.type || 'event',
+        id: m.id 
+      })),
+      prompt: `Review these memories. Synthesize a high-level "belief" or "observation".`
+    };
+  }
+
   /**
    * Add multiple memory entries in batch for efficiency
    * @param {Array<{content: string, metadata?: Object}>} entries - Array of entries to add
@@ -377,7 +430,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.addBatch', count: entries.length });
+      throw e;
     }
   }
 
@@ -411,34 +464,68 @@ class MemoryMesh {
       // Generate embedding using EmbeddingFactory
       const vector = await this.embeddingFactory.embed(query);
 
-      // Perform semantic search
+      // 1. Vector Search
       if (!this.client) throw new Error('Database client not initialized');
-      const results = await this.client.search(vector, {
-        limit,
+      const vectorResults = await this.client.search(vector, {
+        limit: limit * 2, // Fetch more for re-ranking
         metric: 'cosine',
         filter
       });
 
-      // Format results
-      const formattedResults = results.map(result => ({
-        id: result.id,
-        content: result.content,
-        metadata: result.metadata,
-        score: result.score,
-        created_at: result.created_at
-      }));
+      // 2. Keyword Search
+      const keywordResults = this.keywordSearch.search(query, { limit: limit * 2 });
+
+      // 3. Reciprocal Rank Fusion (RRF)
+      const k = 60; // RRF constant
+      const scores = new Map(); // id -> score
+      const docMap = new Map(); // id -> doc
+
+      // Process Vector Results
+      vectorResults.forEach((doc, rank) => {
+        const rrf = 1 / (k + rank + 1);
+        scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
+        docMap.set(doc.id, doc);
+      });
+
+      // Process Keyword Results
+      keywordResults.forEach((doc, rank) => {
+        const rrf = 1 / (k + rank + 1);
+        scores.set(doc.id, (scores.get(doc.id) || 0) + rrf);
+        
+        if (!docMap.has(doc.id)) {
+           // Add keyword-only match
+           docMap.set(doc.id, {
+             id: doc.id,
+             content: doc.content,
+             metadata: doc.metadata,
+             score: 0, // Base score, will be overwritten
+             created_at: new Date().toISOString() // Approximate or missing
+           });
+        }
+      });
+
+      // Sort by RRF score
+      const mergedResults = Array.from(scores.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, limit)
+        .map(([id, score]) => {
+          const doc = docMap.get(id);
+          if (doc) return { ...doc, score }; 
+          return null; 
+        })
+        .filter(d => d !== null);
 
       // Cache the result (unless disabled)
       if (useCache) {
         const cacheKey = this._generateCacheKey(query, { limit, filter });
-        this._cacheResult(cacheKey, formattedResults);
+        this._cacheResult(cacheKey, mergedResults);
       }
 
-      return formattedResults;
+      return mergedResults;
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.search', query });
+      throw e;
     }
   }
 
@@ -469,7 +556,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.get', id });
+      throw e;
     }
   }
 
@@ -486,7 +573,7 @@ class MemoryMesh {
       return await this.client.getAll(options);
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.getAll' });
+      throw e;
     }
   }
 
@@ -548,7 +635,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.update', id });
+      throw e;
     }
   }
 
@@ -564,6 +651,9 @@ class MemoryMesh {
       if (!this.client) throw new Error('Database client not initialized');
       const result = await this.client.delete(id);
 
+      // Remove from Keyword Search
+      this.keywordSearch.remove(id);
+
       return {
         deleted: result.id,
         success: result.success
@@ -572,7 +662,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.delete', id });
+      throw e;
     }
   }
 
@@ -599,7 +689,7 @@ class MemoryMesh {
 
     } catch (error) {
       const e = error instanceof Error ? error : new Error(String(error));
-      throw handleError(e, { context: 'MemoryMesh.stats' });
+      throw e;
     }
   }
 
@@ -803,7 +893,7 @@ async function run() {
 
   try {
     // Route to appropriate action
-    if (action === 'ingest') {
+    if (action === 'ingest' || action === 'store') {
       // Validate required fields
       if (!input.content) {
         console.error('❌ Error: "content" field is required for ingest action');
@@ -866,6 +956,14 @@ ${jsonResult}
       console.log(`[MemoryMesh] Deleted record ${result.deleted}`);
       console.log(JSON.stringify({ status: "ok", ...result }));
 
+    } else if (action === 'export') {
+      const records = await mesh.getAll({ limit: input.limit || 10000 });
+      console.log(JSON.stringify({ status: "ok", count: records.length, records }));
+
+    } else if (action === 'reflect') {
+      const result = await mesh.reflect({ topic: input.topic, lookback: input.limit });
+      console.log(JSON.stringify({ status: "ok", ...result }));
+
     } else if (action === 'stats') {
       const stats = await mesh.stats();
       console.log('[MemoryMesh] Database Statistics:');
@@ -896,8 +994,8 @@ ${jsonResult}
   }
 }
 
-// Export for testing
-export { MemoryMesh };
+// Export for testing and CLI usage
+export { MemoryMesh, run };
 export default MemoryMesh;
 
 // Run CLI if called directly

package/lib/search/keyword-search.js

@@ -0,0 +1,144 @@
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+
+export class KeywordSearch {
+  constructor() {
+    this.index = new Map(); // token -> Map<docId, tf>
+    this.docLengths = new Map(); // docId -> length
+    this.idf = new Map(); // token -> idf value
+    this.docs = new Map(); // docId -> content (optional, for snippet)
+    this.isDirty = false;
+  }
+
+  /**
+   * Tokenize text into normalized terms
+   * @param {string} text 
+   * @returns {string[]} tokens
+   */
+  tokenize(text) {
+    if (!text) return [];
+    return text.toLowerCase()
+      .replace(/[^\w\s]/g, '') // Remove punctuation
+      .split(/\s+/)
+      .filter(t => t.length > 2) // Filter stopwords/short
+      .map(t => t.substring(0, 20)); // Truncate
+  }
+
+  /**
+   * Add a document to the index
+   * @param {string} id 
+   * @param {string} content 
+   * @param {Object} [metadata]
+   */
+  add(id, content, metadata = {}) {
+    const tokens = this.tokenize(content);
+    const termFreqs = new Map();
+
+    tokens.forEach(t => {
+      termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+    });
+
+    this.docLengths.set(id, tokens.length);
+    this.docs.set(id, { content, metadata });
+
+    // Update index
+    for (const [token, freq] of termFreqs.entries()) {
+      if (!this.index.has(token)) {
+        this.index.set(token, new Map());
+      }
+      this.index.get(token).set(id, freq);
+    }
+
+    this.isDirty = true;
+  }
+
+  /**
+   * Remove a document
+   * @param {string} id 
+   */
+  remove(id) {
+    this.docLengths.delete(id);
+    this.docs.delete(id);
+    
+    // This is expensive O(Vocab), but okay for small scale
+    for (const docMap of this.index.values()) {
+      docMap.delete(id);
+    }
+    this.isDirty = true;
+  }
+
+  /**
+   * Recalculate IDF scores
+   */
+  _computeStats() {
+    if (!this.isDirty) return;
+    
+    const N = this.docLengths.size;
+    this.idf.clear();
+
+    for (const [token, docMap] of this.index.entries()) {
+      const df = docMap.size;
+      // Standard IDF: log(N / (df + 1)) + 1
+      const idf = Math.log(N / (df + 1)) + 1;
+      this.idf.set(token, idf);
+    }
+
+    this.isDirty = false;
+  }
+
+  /**
+   * Search for query terms
+   * @param {string} query 
+   * @param {Object} options 
+   * @returns {Array<{id: string, score: number, matches: string[]}>}
+   */
+  search(query, options = {}) {
+    this._computeStats();
+    
+    const tokens = this.tokenize(query);
+    const scores = new Map(); // docId -> score
+    const matches = new Map(); // docId -> matched tokens
+
+    const limit = options.limit || 10;
+
+    for (const token of tokens) {
+      const docMap = this.index.get(token);
+      if (!docMap) continue;
+
+      const idf = this.idf.get(token) || 0;
+
+      for (const [docId, tf] of docMap.entries()) {
+        // TF-IDF Score
+        // Score = tf * idf * (normalization?)
+        // Simple variant:
+        const score = tf * idf;
+        
+        scores.set(docId, (scores.get(docId) || 0) + score);
+        
+        if (!matches.has(docId)) matches.set(docId, []);
+        matches.get(docId).push(token);
+      }
+    }
+
+    // Convert to array and sort
+    return Array.from(scores.entries())
+      .map(([id, score]) => ({
+        id,
+        score,
+        matches: matches.get(id) || [],
+        ...this.docs.get(id)
+      }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+
+  /**
+   * Bulk load records
+   * @param {Array} records 
+   */
+  load(records) {
+    records.forEach(r => this.add(r.id, r.content, r.metadata));
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yamo/memory-mesh",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "description": "Portable semantic memory system with Layer 0 Scrubber for YAMO agents",
   "type": "module",
   "main": "lib/memory/index.js",
@@ -17,7 +17,7 @@
     "index.d.ts"
   ],
   "scripts": {
-    "test": "npm run type-check",
+    "test": "npm run type-check && node --test test/*.test.js",
     "type-check": "tsc --noEmit"
   },
   "dependencies": {

package/skills/SKILL.md

@@ -1,4 +1,4 @@
 ---
 name: yamo-super
-description: For all code improvement or creation tasks, strictly adhere to the logic, agents, and constraints defined in ~/.claude/skills/yamo-super/skill-super.yamo or ~/.gemini/skills/yamo-super/skill-super.yamo.
+description: A comprehensive AI coding workflow system that orchestrates test-driven development, systematic debugging, collaborative planning, git worktree isolation, and dual-mode execution (subagent-driven or batched) with mandatory TDD red-green-refactor cycles, root-cause analysis, two-stage code review, and branch lifecycle management.For all code improvement or creation tasks, strictly adhere to the logic, agents, and constraints defined in skill-super.yamo.
 ---