bus-agent 2.3.0

package/lib/memory.js

@@ -0,0 +1,667 @@
+/**
+ * CoCo Memory Layer — Agent Memory with Vector Search
+ *
+ * Dual-mode search:
+ *   1. TF-IDF keyword search (built-in, zero deps, always available)
+ *   2. Vector similarity search (optional — requires Ollama or OpenAI)
+ *
+ * Storage:
+ *   .bus/memory/{agent}/
+ *     memories.jsonl   ← Append-only log (JSONL)
+ *     index.json       ← Inverted TF-IDF index
+ *     vectors.json     ← Flat vector store for ANN search
+ *     config.json      ← Embedding provider config
+ */
+const fs = require('fs');
+const path = require('path');
+
+const MEMORY_DIR = 'memory';
+
+// ═══════════════════════════════════════════════════════
+//  Flat Vector Index — pure JS, no native deps
+// ═══════════════════════════════════════════════════════
+
+class FlatVectorIndex {
+  constructor(storeDir) {
+    this.storeDir = storeDir;
+    this.vectors = {};      // { id: [0.1, 0.2, ...] }
+    this.dimension = 0;
+    this._load();
+  }
+
+  /**
+   * Add a vector to the index
+   */
+  add(id, vector) {
+    if (this.dimension === 0 && vector.length > 0) this.dimension = vector.length;
+    this.vectors[id] = vector;
+    this._save();
+  }
+
+  /**
+   * Remove a vector
+   */
+  remove(id) {
+    delete this.vectors[id];
+    this._save();
+  }
+
+  /**
+   * Search by query vector — brute-force cosine similarity
+   * O(n) per search — fine for <10k vectors
+   */
+  search(queryVector, { limit = 10, threshold = 0.0 } = {}) {
+    const ids = Object.keys(this.vectors);
+    if (ids.length === 0 || queryVector.length === 0) return [];
+
+    const scored = [];
+    for (const id of ids) {
+      const vec = this.vectors[id];
+      if (!vec || vec.length !== queryVector.length) continue;
+      const sim = this._cosineSimilarity(queryVector, vec);
+      if (sim >= threshold) {
+        scored.push({ id, score: Math.round(sim * 1000) / 1000 });
+      }
+    }
+
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, limit);
+  }
+
+  /**
+   * Get vector count
+   */
+  get size() {
+    return Object.keys(this.vectors).length;
+  }
+
+  /**
+   * Clear all vectors
+   */
+  clear() {
+    this.vectors = {};
+    this.dimension = 0;
+    this._save();
+  }
+
+  _cosineSimilarity(a, b) {
+    let dot = 0, magA = 0, magB = 0;
+    for (let i = 0; i < a.length; i++) {
+      dot += a[i] * b[i];
+      magA += a[i] * a[i];
+      magB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(magA) * Math.sqrt(magB);
+    return denom === 0 ? 0 : dot / denom;
+  }
+
+  _save() {
+    const data = {
+      dimension: this.dimension,
+      count: Object.keys(this.vectors).length,
+      vectors: this.vectors,
+    };
+    fs.writeFileSync(path.join(this.storeDir, 'vectors.json'), JSON.stringify(data), 'utf-8');
+  }
+
+  _load() {
+    const p = path.join(this.storeDir, 'vectors.json');
+    if (fs.existsSync(p)) {
+      try {
+        const data = JSON.parse(fs.readFileSync(p, 'utf-8'));
+        this.dimension = data.dimension || 0;
+        this.vectors = data.vectors || {};
+      } catch { this.vectors = {}; }
+    }
+  }
+}
+
+// ═══════════════════════════════════════════════════════
+//  Embedding Engine — Ollama / OpenAI / Custom
+// ═══════════════════════════════════════════════════════
+
+class EmbeddingEngine {
+  constructor(config) {
+    this.config = config || {};
+  }
+
+  /**
+   * Generate embedding for a single text
+   */
+  async embed(text) {
+    const provider = this.config.provider || 'keyword';
+    switch (provider) {
+      case 'ollama': return this._ollamaEmbed(text);
+      case 'openai': return this._openaiEmbed(text);
+      case 'custom': return this._customEmbed(text);
+      default: return null;
+    }
+  }
+
+  /**
+   * Generate embeddings for multiple texts (batched)
+   */
+  async embedBatch(texts) {
+    if (texts.length === 0) return [];
+    const provider = this.config.provider || 'keyword';
+
+    switch (provider) {
+      case 'ollama': return this._ollamaEmbedBatch(texts);
+      case 'openai': return this._openaiEmbedBatch(texts);
+      case 'custom': return this._customEmbedBatch(texts);
+      default: return texts.map(() => null);
+    }
+  }
+
+  async _ollamaEmbed(text) {
+    const base = this.config.endpoint || 'http://localhost:11434';
+    // Ollama 0.3+ uses /api/embed, older uses /api/embeddings
+    const url = base.replace(/\/+$/, '') + '/api/embed';
+    const model = this.config.model || 'nomic-embed-text';
+
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ model, input: text }),
+    });
+
+    if (!res.ok) throw new Error(`Ollama embedding error: ${res.status}`);
+    const data = await res.json();
+    return data.embeddings && data.embeddings[0];
+  }
+
+  async _ollamaEmbedBatch(texts) {
+    const base = this.config.endpoint || 'http://localhost:11434';
+    const url = base.replace(/\/+$/, '') + '/api/embed';
+    const model = this.config.model || 'nomic-embed-text';
+
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ model, input: texts }),
+    });
+
+    if (!res.ok) throw new Error(`Ollama embedding error: ${res.status}`);
+    const data = await res.json();
+    return data.embeddings;
+  }
+
+  async _openaiEmbed(text) {
+    const url = this.config.endpoint || 'https://api.openai.com/v1/embeddings';
+    const model = this.config.model || 'text-embedding-3-small';
+    const key = this.config.api_key;
+
+    if (!key) throw new Error('OpenAI API key required');
+
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Authorization': `Bearer ${key}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ input: text, model }),
+    });
+
+    if (!res.ok) throw new Error(`OpenAI embedding error: ${res.status}`);
+    const data = await res.json();
+    return data.data[0].embedding;
+  }
+
+  async _openaiEmbedBatch(texts) {
+    const url = this.config.endpoint || 'https://api.openai.com/v1/embeddings';
+    const model = this.config.model || 'text-embedding-3-small';
+    const key = this.config.api_key;
+
+    if (!key) throw new Error('OpenAI API key required');
+
+    const res = await fetch(url, {
+      method: 'POST',
+      headers: { 'Authorization': `Bearer ${key}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ input: texts, model }),
+    });
+
+    if (!res.ok) throw new Error(`OpenAI embedding error: ${res.status}`);
+    const data = await res.json();
+    return data.data.map(d => d.embedding);
+  }
+
+  async _customEmbed(text) {
+    if (!this.config.endpoint) throw new Error('Custom embedding endpoint required');
+
+    const res = await fetch(this.config.endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', ...(this.config.headers || {}) },
+      body: JSON.stringify({ text, model: this.config.model }),
+    });
+
+    if (!res.ok) throw new Error(`Custom embedding error: ${res.status}`);
+    return await res.json();
+  }
+
+  async _customEmbedBatch(texts) {
+    const results = [];
+    for (const text of texts) {
+      results.push(await this._customEmbed(text));
+    }
+    return results;
+  }
+}
+
+// ═══════════════════════════════════════════════════════
+//  MemoryStore — main API
+// ═══════════════════════════════════════════════════════
+
+class MemoryStore {
+  constructor(busDir) {
+    this.busDir = busDir;
+    this.baseDir = path.join(busDir, MEMORY_DIR);
+    this._ensureDir();
+    this._config = this._loadConfig();
+    this.embedder = new EmbeddingEngine(this._config);
+  }
+
+  // ── Public API ──
+
+  /**
+   * Store a memory entry for an agent
+   * Auto-embeds if embedding provider is configured
+   */
+  async store(agent, opts = {}) {
+    const { key, content, namespace = 'default', metadata = {}, ttl } = opts;
+    if (!content) throw new Error('content is required');
+
+    const agentDir = this._agentDir(agent);
+    this._ensureAgentDir(agent);
+
+    const entry = {
+      id: `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+      key: key || null,
+      content: typeof content === 'string' ? content : JSON.stringify(content),
+      namespace,
+      metadata: { ...metadata, stored_at: new Date().toISOString() },
+      ttl: ttl || null,
+      tokens: this._tokenize(content),
+    };
+
+    // Append to JSONL
+    const line = JSON.stringify(entry) + '\n';
+    fs.appendFileSync(path.join(agentDir, 'memories.jsonl'), line, 'utf-8');
+
+    // Update TF-IDF index
+    this._updateIndex(agent, entry);
+
+    // Generate & store vector embedding (if provider configured)
+    if (this._config.provider !== 'keyword') {
+      try {
+        const vector = await this.embedder.embed(entry.content);
+        if (vector && Array.isArray(vector)) {
+          const vi = this._getVectorIndex(agent);
+          vi.add(entry.id, vector);
+        }
+      } catch (err) {
+        console.warn(`  ⚠️  Embedding failed for "${entry.content.substring(0, 40)}...": ${err.message}`);
+      }
+    }
+
+    return entry.id;
+  }
+
+  /**
+   * Search memories — uses vector search if configured, TF-IDF fallback
+   */
+  async search(agent, query, opts = {}) {
+    const { limit = 10, namespace, threshold = 0.0 } = opts;
+    const entries = this._loadMemories(agent, namespace);
+    if (entries.length === 0) return [];
+
+    // Try vector search first (if provider configured and we have vectors)
+    if (this._config.provider !== 'keyword') {
+      try {
+        const queryVector = await this.embedder.embed(query);
+        if (queryVector && Array.isArray(queryVector)) {
+          const vi = this._getVectorIndex(agent);
+          if (vi.size > 0) {
+            const vecResults = vi.search(queryVector, { limit: limit * 2, threshold });
+
+            // Map vector results back to full entries
+            const entryMap = {};
+            for (const e of entries) entryMap[e.id] = e;
+
+            const results = [];
+            for (const vr of vecResults) {
+              const entry = entryMap[vr.id];
+              if (entry) {
+                results.push({ ...entry, score: vr.score, search_type: 'vector' });
+                if (results.length >= limit) break;
+              }
+            }
+
+            if (results.length > 0) return results;
+          }
+        }
+      } catch {
+        // Fall through to TF-IDF
+      }
+    }
+
+    // TF-IDF fallback
+    const queryTokens = this._tokenize(query);
+    const scored = [];
+
+    for (const entry of entries) {
+      let score = this._cosineSimilarity(queryTokens, entry.tokens || {});
+
+      // Recency boost: 30-day half-life
+      const ageHours = (Date.now() - new Date(entry.metadata.stored_at).getTime()) / 3600000;
+      const recencyBoost = Math.max(0, 1 - (ageHours / 720));
+      score = score * 0.7 + recencyBoost * 0.3;
+
+      if (score >= threshold) {
+        scored.push({ ...entry, score: Math.round(score * 1000) / 1000, search_type: 'keyword' });
+      }
+    }
+
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, limit);
+  }
+
+  /**
+   * Recall a memory by ID or key
+   */
+  recall(agent, identifier) {
+    const entries = this._loadMemories(agent);
+    return entries.find(e => e.id === identifier || e.key === identifier) || null;
+  }
+
+  /**
+   * List memories for an agent
+   */
+  list(agent, opts = {}) {
+    const { namespace, limit = 50, offset = 0 } = opts;
+    let entries = this._loadMemories(agent, namespace);
+    entries.sort((a, b) => new Date(b.metadata.stored_at) - new Date(a.metadata.stored_at));
+    return entries.slice(offset, offset + limit);
+  }
+
+  /**
+   * Forget/delete a memory by ID
+   */
+  forget(agent, id) {
+    const agentDir = this._agentDir(agent);
+    const memFile = path.join(agentDir, 'memories.jsonl');
+    if (!fs.existsSync(memFile)) return false;
+
+    const lines = fs.readFileSync(memFile, 'utf-8').split('\n').filter(Boolean);
+    const filtered = lines.filter(line => {
+      try { const e = JSON.parse(line); return e.id !== id; } catch { return true; }
+    });
+
+    if (lines.length === filtered.length) return false;
+    fs.writeFileSync(memFile, filtered.join('\n') + '\n', 'utf-8');
+    this._rebuildIndex(agent);
+
+    // Remove from vector index
+    const vi = this._getVectorIndex(agent);
+    vi.remove(id);
+
+    return true;
+  }
+
+  /**
+   * Get memory stats
+   */
+  stats(agent) {
+    const entries = this._loadMemories(agent);
+    const byNamespace = {};
+    let totalSize = 0;
+    for (const e of entries) {
+      byNamespace[e.namespace] = (byNamespace[e.namespace] || 0) + 1;
+      totalSize += (e.content || '').length;
+    }
+    const vi = this._getVectorIndex(agent);
+    return {
+      total: entries.length,
+      namespaces: byNamespace,
+      size_bytes: totalSize,
+      vectors: vi.size,
+      vector_dimension: vi.dimension || 0,
+      embed_provider: this._config.provider || 'keyword',
+      oldest: entries.length > 0 ? entries.reduce((a, b) =>
+        new Date(a.metadata.stored_at) < new Date(b.metadata.stored_at) ? a : b
+      ).metadata.stored_at : null,
+      newest: entries.length > 0 ? entries.reduce((a, b) =>
+        new Date(a.metadata.stored_at) > new Date(b.metadata.stored_at) ? a : b
+      ).metadata.stored_at : null,
+    };
+  }
+
+  /**
+   * Archive old messages → memory
+   */
+  async archive(agent, opts = {}) {
+    const { maxAgeDays = 7, deleteAfter = true } = opts;
+    const msgsDir = path.join(this.busDir, 'messages', agent);
+    if (!fs.existsSync(msgsDir)) return { archived: 0 };
+
+    const cutoff = Date.now() - (maxAgeDays * 86400000);
+    const files = fs.readdirSync(msgsDir).filter(f => f.endsWith('.json'));
+    let archived = 0;
+
+    for (const f of files) {
+      try {
+        const msg = JSON.parse(fs.readFileSync(path.join(msgsDir, f), 'utf-8'));
+        const msgTime = new Date(msg.timestamp).getTime();
+        if (msgTime < cutoff) {
+          await this.store(agent, {
+            content: `[${msg.from}] ${msg.message}`,
+            namespace: 'archive',
+            metadata: {
+              original_id: msg.id, from: msg.from, to: msg.to,
+              original_timestamp: msg.timestamp, archived_at: new Date().toISOString(),
+            },
+          });
+          archived++;
+          if (deleteAfter) fs.unlinkSync(path.join(msgsDir, f));
+        }
+      } catch {}
+    }
+
+    return { archived };
+  }
+
+  /**
+   * Clear memories for an agent
+   */
+  clear(agent, namespace) {
+    const agentDir = this._agentDir(agent);
+    const memFile = path.join(agentDir, 'memories.jsonl');
+    if (!fs.existsSync(memFile)) return 0;
+
+    if (namespace) {
+      const lines = fs.readFileSync(memFile, 'utf-8').split('\n').filter(Boolean);
+      const filtered = lines.filter(line => {
+        try { const e = JSON.parse(line); return e.namespace !== namespace; } catch { return true; }
+      });
+      const removed = lines.length - filtered.length;
+      fs.writeFileSync(memFile, filtered.join('\n') + '\n', 'utf-8');
+      this._rebuildIndex(agent);
+      // Vectors are stored separately in vectors.json — clear them entirely since
+      // we can't easily match which vectors to remove per entry
+      this._getVectorIndex(agent).clear();
+      return removed;
+    }
+
+    const count = fs.readFileSync(memFile, 'utf-8').split('\n').filter(Boolean).length;
+    fs.writeFileSync(memFile, '', 'utf-8');
+    const idxPath = path.join(agentDir, 'index.json');
+    if (fs.existsSync(idxPath)) fs.unlinkSync(idxPath);
+    const vi = this._getVectorIndex(agent);
+    vi.clear();
+    return count;
+  }
+
+  /**
+   * Rebuild vector index for an agent from scratch
+   */
+  async rebuildVectors(agent) {
+    const entries = this._loadMemories(agent);
+    const vi = this._getVectorIndex(agent);
+    vi.clear();
+
+    if (this._config.provider === 'keyword') {
+      return { rebuilt: 0, reason: 'No embedding provider configured. Use memory configure first.' };
+    }
+
+    let success = 0;
+    let failed = 0;
+
+    for (const entry of entries) {
+      try {
+        const vector = await this.embedder.embed(entry.content);
+        if (vector && Array.isArray(vector)) {
+          vi.add(entry.id, vector);
+          success++;
+        }
+      } catch {
+        failed++;
+      }
+    }
+
+    return { rebuilt: success, failed };
+  }
+
+  // ── Embedding Configuration ──
+
+  configure(opts = {}) {
+    const cfg = { ...this._config, ...opts };
+    if (opts.provider) cfg.provider = opts.provider;
+    if (opts.api_key) cfg.api_key = opts.api_key;
+    if (opts.endpoint) cfg.endpoint = opts.endpoint;
+    if (opts.model) cfg.model = opts.model;
+    this._config = cfg;
+    this._saveConfig();
+    this.embedder = new EmbeddingEngine(cfg);
+    return cfg;
+  }
+
+  getConfig() {
+    return { ...this._config };
+  }
+
+  // ── Internals ──
+
+  _agentDir(agent) {
+    return path.join(this.baseDir, agent);
+  }
+
+  _ensureDir() {
+    if (!fs.existsSync(this.baseDir)) fs.mkdirSync(this.baseDir, { recursive: true });
+  }
+
+  _ensureAgentDir(agent) {
+    const dir = this._agentDir(agent);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  }
+
+  _getVectorIndex(agent) {
+    const dir = this._agentDir(agent);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    if (!this._vectorIndices) this._vectorIndices = {};
+    if (!this._vectorIndices[agent]) {
+      this._vectorIndices[agent] = new FlatVectorIndex(dir);
+    }
+    return this._vectorIndices[agent];
+  }
+
+  _loadMemories(agent, namespace) {
+    const memFile = path.join(this._agentDir(agent), 'memories.jsonl');
+    if (!fs.existsSync(memFile)) return [];
+
+    const lines = fs.readFileSync(memFile, 'utf-8').split('\n').filter(Boolean);
+    const entries = [];
+    for (const line of lines) {
+      try {
+        const entry = JSON.parse(line);
+        if (entry.ttl) {
+          const age = Date.now() - new Date(entry.metadata.stored_at).getTime();
+          if (age > entry.ttl * 1000) continue;
+        }
+        if (namespace && entry.namespace !== namespace) continue;
+        entries.push(entry);
+      } catch {}
+    }
+    return entries;
+  }
+
+  _tokenize(text) {
+    if (!text) return {};
+    const str = typeof text === 'string' ? text : JSON.stringify(text);
+    const tokens = str.toLowerCase()
+      .replace(/[^a-z0-9\u0E00-\u0E7F\s]/g, ' ')
+      .split(/\s+/)
+      .filter(t => t.length > 1 && t.length < 50);
+
+    const tf = {};
+    for (const t of tokens) tf[t] = (tf[t] || 0) + 1;
+    const len = tokens.length || 1;
+    for (const t of Object.keys(tf)) tf[t] /= len;
+    return tf;
+  }
+
+  _cosineSimilarity(a, b) {
+    const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
+    let dot = 0, magA = 0, magB = 0;
+    for (const k of keys) {
+      const va = a[k] || 0;
+      const vb = b[k] || 0;
+      dot += va * vb;
+      magA += va * va;
+      magB += vb * vb;
+    }
+    const denom = Math.sqrt(magA) * Math.sqrt(magB);
+    return denom === 0 ? 0 : dot / denom;
+  }
+
+  _updateIndex(agent, entry) {
+    const idxPath = path.join(this._agentDir(agent), 'index.json');
+    let index = {};
+    if (fs.existsSync(idxPath)) {
+      try { index = JSON.parse(fs.readFileSync(idxPath, 'utf-8')); } catch {}
+    }
+    for (const [token, weight] of Object.entries(entry.tokens || {})) {
+      if (!index[token]) index[token] = [];
+      index[token].push({ id: entry.id, weight });
+      if (index[token].length > 100) index[token] = index[token].slice(-100);
+    }
+    if (!index._namespaces) index._namespaces = {};
+    index._namespaces[entry.namespace] = (index._namespaces[entry.namespace] || 0) + 1;
+    index._total = (index._total || 0) + 1;
+    fs.writeFileSync(idxPath, JSON.stringify(index), 'utf-8');
+  }
+
+  _rebuildIndex(agent) {
+    const entries = this._loadMemories(agent);
+    const idxPath = path.join(this._agentDir(agent), 'index.json');
+    const index = { _namespaces: {}, _total: entries.length };
+    for (const entry of entries) {
+      index._namespaces[entry.namespace] = (index._namespaces[entry.namespace] || 0) + 1;
+      for (const [token, weight] of Object.entries(entry.tokens || {})) {
+        if (!index[token]) index[token] = [];
+        index[token].push({ id: entry.id, weight });
+      }
+    }
+    fs.writeFileSync(idxPath, JSON.stringify(index), 'utf-8');
+  }
+
+  _loadConfig() {
+    const cfgPath = path.join(this.baseDir, 'config.json');
+    if (fs.existsSync(cfgPath)) {
+      try { return JSON.parse(fs.readFileSync(cfgPath, 'utf-8')); } catch {}
+    }
+    return { provider: 'keyword', model: null, endpoint: null };
+  }
+
+  _saveConfig() {
+    const cfgPath = path.join(this.baseDir, 'config.json');
+    fs.writeFileSync(cfgPath, JSON.stringify(this._config, null, 2), 'utf-8');
+  }
+}
+
+module.exports = { MemoryStore, EmbeddingEngine, FlatVectorIndex };