@caoscompanybr/merlin 3.5.0

package/.merlin-core/core/alkimia/memory/thoughts-provider.js

@@ -0,0 +1,603 @@
+/**
+ * Thoughts Provider -- Bracket-aware memory provider for the ALKIMIA engine.
+ *
+ * Implements the provider interface expected by MemoryBridge.registerProvider().
+ * Uses keyword matching + recency + type affinity scoring to return relevant
+ * memory hints from the thoughts/ directory index.
+ *
+ * Inspired by Agno's Agentic RAG pattern: the agent receives only the most
+ * relevant knowledge at the right time, not everything upfront.
+ *
+ * @module core/alkimia/memory/thoughts-provider
+ * @version 3.0.0
+ */
+
+/* global AbortController */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { estimateTokens } = require("../utils/tokens");
+const { buildIndex, computeKeywordOverlap } = require("./thoughts-indexer");
+const { DocumentSharder, LoadStrategy } = require("./document-sharder");
+const {
+  sanitizeId,
+  sanitizeContent,
+} = require("../security/content-sanitizer");
+
+const INDEX_CACHE_TTL_MS = 60_000; // 60 seconds
+const MAX_INDEX_ENTRIES = 500; // H8 fix: Cap index size to prevent OOM
+
+// Type affinity by bracket: which document types are most useful at each level
+const TYPE_AFFINITY = {
+  MODERATE: {
+    handoff: 0.6,
+    plan: 0.8,
+    research: 0.7,
+    decision: 0.9,
+    reminder: 0.85,
+    objective: 0.9,
+    learning: 0.85,
+    process: 0.9,
+    feedback: 0.7,
+  },
+  DEPLETED: {
+    handoff: 0.9,
+    plan: 0.7,
+    research: 0.5,
+    decision: 0.8,
+    reminder: 0.8,
+    objective: 0.85,
+    learning: 0.85,
+    process: 0.9,
+    feedback: 0.6,
+  },
+  CRITICAL: {
+    handoff: 1.0,
+    plan: 0.6,
+    research: 0.3,
+    decision: 0.7,
+    reminder: 0.95,
+    objective: 0.9,
+    learning: 0.9,
+    process: 0.95,
+    feedback: 0.5,
+  },
+};
+
+const SHARD_THRESHOLD_TOKENS = 500;
+
+class ThoughtsProvider {
+  constructor(thoughtsPath) {
+    this._thoughtsPath = thoughtsPath;
+    this._indexCache = null;
+    this._indexCacheTime = 0;
+    this._sharder = new DocumentSharder({ maxChunkTokens: 200 });
+  }
+
+  /**
+   * Get relevant memory hints for the current agent and bracket.
+   *
+   * @param {string} agentId - Active agent identifier
+   * @param {string} bracket - Current context bracket (MODERATE|DEPLETED|CRITICAL)
+   * @param {number} tokenBudget - Maximum tokens to return
+   * @returns {Array<{content: string, source: string, relevance: number, tokens: number}>}
+   */
+  async getMemories(agentId, bracket, tokenBudget) {
+    const index = this._loadIndex();
+    if (!index || !index.entries || index.entries.length === 0) return [];
+
+    // Filter out superseded entries (entries whose citationId is listed in
+    // another entry's `supersedes` frontmatter field)
+    const supersededIds = new Set();
+    for (const entry of index.entries) {
+      if (entry.supersedes) {
+        const ids = Array.isArray(entry.supersedes)
+          ? entry.supersedes
+          : [entry.supersedes];
+        ids.forEach((id) => supersededIds.add(id));
+      }
+    }
+    const nonSuperseded =
+      supersededIds.size > 0
+        ? index.entries.filter((e) => !supersededIds.has(e.citationId))
+        : index.entries;
+
+    // Filter concluded/terminal-status entries
+    const terminalStatuses = [
+      "concluded",
+      "achieved",
+      "abandoned",
+      "completed",
+      "complete",
+      "expired",
+      "done",
+      "milestone_complete",
+      "deprecated",
+    ];
+    const activeEntries = nonSuperseded.filter((entry) => {
+      // Honor the indexer's lifecycle classification (e.g. TTL-expired lessons
+      // are terminal-by-age without a status change).
+      if (entry.lifecycle === "terminal") return false;
+      const status = entry.frontmatter?.status;
+      if (!status) return true; // no status = active
+      return !terminalStatuses.includes(status.toLowerCase());
+    });
+
+    // Score each entry
+    const scored = activeEntries.map((entry) => ({
+      entry,
+      score: this._scoreEntry(entry, agentId, bracket),
+    }));
+
+    // Sort by score descending
+    scored.sort((a, b) => b.score - a.score);
+
+    // Build hints within token budget
+    const hints = [];
+    let tokensUsed = 0;
+
+    for (const { entry, score } of scored) {
+      if (score < 0.1) break; // skip irrelevant entries
+      if (tokensUsed >= tokenBudget) break;
+
+      const content = this._formatHint(
+        entry,
+        bracket,
+        tokenBudget - tokensUsed,
+      );
+      const tokens = estimateTokens(content);
+
+      if (tokensUsed + tokens > tokenBudget) continue;
+
+      hints.push({
+        content,
+        source: entry.type,
+        relevance: Math.min(score, 1.0),
+        tokens,
+        citationId: entry.citationId,
+        keywords: entry.keywords,
+        domain: entry.domain,
+      });
+      tokensUsed += tokens;
+    }
+
+    // Detect contradictions among selected hints and prepend warnings
+    const contradictions = this._detectContradictions(hints);
+    if (contradictions.length > 0 && tokensUsed < tokenBudget) {
+      for (const warning of contradictions) {
+        const warningTokens = estimateTokens(warning.content);
+        if (tokensUsed + warningTokens > tokenBudget) break;
+        hints.unshift(warning);
+        tokensUsed += warningTokens;
+      }
+    }
+
+    return hints;
+  }
+
+  /**
+   * Score a single index entry for relevance.
+   *
+   * Formula: keyword_match * 0.5 + recency * 0.2 + type_affinity * 0.3
+   */
+  _scoreEntry(entry, agentId, bracket) {
+    // 1. Keyword match score (agent ID as keyword proxy)
+    const agentKeywords = this._agentKeywords(agentId);
+    let matchCount = 0;
+    for (const kw of agentKeywords) {
+      if (entry.keywords.some((ek) => ek.includes(kw) || kw.includes(ek))) {
+        matchCount++;
+      }
+    }
+    const keywordScore =
+      agentKeywords.length > 0
+        ? Math.min(matchCount / agentKeywords.length, 1.0)
+        : 0.3; // default relevance when no agent context
+
+    // 2. Recency score: 1.0 for today, decays to 0.1 at 30+ days
+    const daysSince = this._daysSince(entry.date);
+    const recencyScore = Math.max(0.1, 1.0 - daysSince / 30);
+
+    // 3. Type affinity per bracket
+    const affinityMap = TYPE_AFFINITY[bracket] || TYPE_AFFINITY.MODERATE;
+    const typeScore = affinityMap[entry.type] || 0.5;
+
+    return keywordScore * 0.5 + recencyScore * 0.2 + typeScore * 0.3;
+  }
+
+  /**
+   * Format a hint for injection into the alkimia output.
+   * Adjusts detail level based on bracket. Uses document sharding for large docs.
+   */
+  _formatHint(entry, bracket, remainingBudget) {
+    const safeTitle = sanitizeId(entry.title || "");
+    const citationId =
+      entry.citationId || `${entry.type}:${entry.date}:unknown`;
+    const tier = this._assignPriorityTier(entry);
+    const header = `[[memory:${citationId}]] [TIER-${tier}] [${entry.type.toUpperCase()}] ${safeTitle} (${entry.date})`;
+
+    if (bracket === "CRITICAL" || remainingBudget < 100) {
+      return `${header} → ${entry.path}`;
+    }
+
+    // For large documents, use sharding
+    const sharded = this._shardIfNeeded(entry, bracket, remainingBudget);
+    if (sharded)
+      return `${header}\n${sanitizeContent(sharded, { maxLength: 1500 })}\n  File: ${entry.path}`;
+
+    if (bracket === "DEPLETED") {
+      const safeSummary = sanitizeContent(entry.summary || "", {
+        maxLength: 500,
+      });
+      return `${header}\n  ${safeSummary}\n  File: ${entry.path}`;
+    }
+
+    // MODERATE: title + summary + sections
+    const safeSummary = sanitizeContent(entry.summary || "", {
+      maxLength: 800,
+    });
+    const sectionList =
+      entry.sections.length > 0
+        ? `\n  Sections: ${entry.sections
+            .slice(0, 5)
+            .map((s) => sanitizeContent(s, { maxLength: 100 }))
+            .join(", ")}`
+        : "";
+    return `${header}\n  ${safeSummary}${sectionList}\n  File: ${entry.path}`;
+  }
+
+  /**
+   * Shard large documents and load bracket-appropriate chunks.
+   * Returns null for small documents (below threshold).
+   */
+  _shardIfNeeded(entry, bracket, remainingBudget) {
+    // Only shard if we can read the full file and it's large
+    const fullPath = path.resolve(this._thoughtsPath, "..", entry.path);
+    // H7 fix: Validate path stays within project to prevent traversal
+    const projectRoot = path.resolve(this._thoughtsPath, "..", "..");
+    if (!fullPath.startsWith(projectRoot + path.sep)) return null;
+
+    let content;
+    try {
+      content = fs.readFileSync(fullPath, "utf8");
+    } catch {
+      return null;
+    }
+
+    const totalTokens = estimateTokens(content);
+    if (totalTokens < SHARD_THRESHOLD_TOKENS) return null;
+
+    const { chunks } = this._sharder.shard(content, { title: entry.title });
+    if (chunks.length === 0) return null;
+
+    const strategy =
+      bracket === "DEPLETED" ? LoadStrategy.SUMMARY : LoadStrategy.RELEVANT;
+    const keywords = entry.keywords || [];
+    const loaded = this._sharder.loadByStrategy(chunks, strategy, {
+      keywords,
+      maxTokens: Math.min(remainingBudget - 50, totalTokens),
+    });
+
+    if (loaded.length === 0) return null;
+    return loaded.map((c) => `  ${c.content.split("\n")[0]}`).join("\n");
+  }
+
+  /**
+   * Detect potential contradictions among selected hints.
+   * Compares entries of the same type and domain with ≥60% keyword overlap.
+   *
+   * @param {Array} hints - Selected hint objects with citationId, keywords, domain
+   * @returns {Array<{content: string, source: string, relevance: number, tokens: number}>}
+   */
+  _detectContradictions(hints) {
+    const warnings = [];
+    const OVERLAP_THRESHOLD = 0.6;
+
+    for (let i = 0; i < hints.length; i++) {
+      for (let j = i + 1; j < hints.length; j++) {
+        const a = hints[i];
+        const b = hints[j];
+
+        // Only compare same-type entries
+        if (a.source !== b.source) continue;
+
+        // Only compare same-domain entries (skip if no domain)
+        if (!a.domain || !b.domain || a.domain !== b.domain) continue;
+
+        // Check keyword overlap (Jaccard similarity)
+        const kw1 = a.keywords || [];
+        const kw2 = b.keywords || [];
+        if (kw1.length === 0 || kw2.length === 0) continue;
+
+        const overlap = computeKeywordOverlap(kw1, kw2);
+        if (overlap >= OVERLAP_THRESHOLD) {
+          const content =
+            `⚠ CONTRADICTION CANDIDATE: [[memory:${a.citationId}]] and [[memory:${b.citationId}]] ` +
+            `share ${(overlap * 100).toFixed(0)}% keyword overlap in domain "${a.domain}". ` +
+            `Verify which is current before relying on either.`;
+          warnings.push({
+            content,
+            source: "contradiction-detector",
+            relevance: 1.0,
+            tokens: estimateTokens(content),
+          });
+        }
+      }
+    }
+
+    return warnings;
+  }
+
+  /**
+   * Derive search keywords from agent ID.
+   */
+  _agentKeywords(agentId) {
+    if (!agentId) return [];
+    const id =
+      typeof agentId === "object"
+        ? agentId.id || agentId.name || ""
+        : String(agentId);
+    // Split compound IDs like "merlin-master" → ["merlin", "master"]
+    return id
+      .toLowerCase()
+      .split(/[-_]/)
+      .filter((t) => t.length > 2);
+  }
+
+  /**
+   * Calculate days since a date string (YYYY-MM-DD).
+   */
+  _daysSince(dateStr) {
+    if (!dateStr || dateStr === "unknown") return 30;
+    try {
+      const then = new Date(dateStr);
+      const now = new Date();
+      return Math.max(0, Math.floor((now - then) / 86_400_000));
+    } catch {
+      return 30;
+    }
+  }
+
+  /**
+   * Load index with caching (60s TTL).
+   */
+  _loadIndex() {
+    const now = Date.now();
+    if (this._indexCache && now - this._indexCacheTime < INDEX_CACHE_TTL_MS) {
+      return this._indexCache;
+    }
+
+    // Try loading pre-built INDEX.json first
+    const jsonPath = path.join(this._thoughtsPath, "shared", "INDEX.json");
+    try {
+      if (fs.existsSync(jsonPath)) {
+        const data = JSON.parse(fs.readFileSync(jsonPath, "utf8"));
+        this._indexCache = data;
+        this._indexCacheTime = now;
+        return data;
+      }
+    } catch {
+      // Fall through to live build
+    }
+
+    // Build index on-the-fly if INDEX.json doesn't exist
+    try {
+      let data = buildIndex(this._thoughtsPath);
+      // H8 fix: Cap index entries to prevent OOM on large knowledge bases
+      if (
+        data &&
+        Array.isArray(data.entries) &&
+        data.entries.length > MAX_INDEX_ENTRIES
+      ) {
+        data.entries = data.entries.slice(0, MAX_INDEX_ENTRIES);
+      }
+      this._indexCache = data;
+      this._indexCacheTime = now;
+      return data;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Search memories by keyword query across all sessions and types.
+   * Unlike getMemories(), this ignores bracket affinity and recency decay
+   * to provide explicit cross-session search capability.
+   *
+   * @param {string} query - Search query text
+   * @param {Object} [options]
+   * @param {string[]} [options.types] - Filter by document types (e.g., ["research", "learning"])
+   * @param {string} [options.since] - ISO date string filter (e.g., "2026-01-01")
+   * @param {number} [options.limit=10] - Max results to return
+   * @returns {Array<{citationId, type, title, date, path, score, keywords}>}
+   */
+  searchMemories(query, options = {}) {
+    const index = this._loadIndex();
+    if (!index || !index.entries || index.entries.length === 0) return [];
+
+    const limit = options.limit || 10;
+    const queryTokens = (query || "")
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((t) => t.length > 2);
+    if (queryTokens.length === 0) return [];
+
+    let entries = index.entries;
+
+    // Filter by types
+    if (options.types && options.types.length > 0) {
+      const typeSet = new Set(options.types);
+      entries = entries.filter((e) => typeSet.has(e.type));
+    }
+
+    // Filter by date
+    if (options.since) {
+      const sinceDate = new Date(options.since);
+      entries = entries.filter((e) => {
+        if (!e.date || e.date === "unknown") return false;
+        return new Date(e.date) >= sinceDate;
+      });
+    }
+
+    // Score by keyword match only (no bracket affinity, no recency)
+    const scored = entries.map((entry) => {
+      let matchCount = 0;
+      for (const token of queryTokens) {
+        if (
+          entry.keywords.some((kw) => kw.includes(token) || token.includes(kw))
+        ) {
+          matchCount++;
+        }
+      }
+      return {
+        citationId: entry.citationId,
+        type: entry.type,
+        title: entry.title,
+        date: entry.date,
+        path: entry.path,
+        score: queryTokens.length > 0 ? matchCount / queryTokens.length : 0,
+        keywords: entry.keywords,
+      };
+    });
+
+    // Filter zero-score, sort descending, limit
+    return scored
+      .filter((s) => s.score > 0)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+
+  /**
+   * Hybrid semantic search (v3.4). Blends the dashboard's local-embedding cosine
+   * hits with the keyword score, and falls back to keyword-only if the dashboard
+   * is unreachable — so the zero-dependency core never hard-depends on it.
+   *
+   * @param {string} query
+   * @param {Object} [options]
+   * @param {number} [options.limit=10]
+   * @param {number} [options.vectorWeight=0.6]
+   * @param {number} [options.textWeight=0.4]
+   * @param {number} [options.timeout=1500] - ms before falling back to keyword
+   * @returns {Promise<Array<{citationId,type,title,date,path,score,keywords}>>}
+   */
+  async searchMemoriesSemantic(query, options = {}) {
+    const limit = options.limit || 10;
+    const keyword = this.searchMemories(query, options);
+    const base = process.env.MERLIN_DASHBOARD_URL || "http://127.0.0.1:3939";
+    try {
+      const ctrl = new AbortController();
+      const timer = setTimeout(() => ctrl.abort(), options.timeout || 1500);
+      const res = await fetch(
+        `${base}/api/memory/search?q=${encodeURIComponent(query)}&k=${limit}`,
+        { signal: ctrl.signal },
+      );
+      clearTimeout(timer);
+      if (!res.ok) return keyword;
+      const body = await res.json();
+      const hits = Array.isArray(body.data) ? body.data : [];
+      if (hits.length === 0) return keyword;
+
+      const index = this._loadIndex();
+      const byId = new Map(
+        (index?.entries || []).map((e) => [e.citationId, e]),
+      );
+      const kwById = new Map(keyword.map((s) => [s.citationId, s.score]));
+      const vw = options.vectorWeight ?? 0.6;
+      const tw = options.textWeight ?? 0.4;
+      const merged = new Map();
+      for (const h of hits) {
+        const e = byId.get(h.citationId);
+        if (!e) continue;
+        merged.set(h.citationId, {
+          citationId: e.citationId,
+          type: e.type,
+          title: e.title,
+          date: e.date,
+          path: e.path,
+          keywords: e.keywords,
+          score: vw * (h.score || 0) + tw * (kwById.get(h.citationId) || 0),
+          semantic: true,
+        });
+      }
+      // Keyword-only matches the vector index missed still count (weighted).
+      for (const s of keyword) {
+        if (!merged.has(s.citationId)) {
+          merged.set(s.citationId, { ...s, score: tw * s.score });
+        }
+      }
+      return [...merged.values()]
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    } catch {
+      return keyword; // dashboard down / timeout → keyword fallback
+    }
+  }
+
+  /**
+   * Knowledge-First Check.
+   *
+   * Quickly checks if existing project knowledge is relevant to a query
+   * before launching a full research phase. Returns found entries and
+   * a confidence score.
+   *
+   * @param {string} query - Research query or task description
+   * @param {Object} [options]
+   * @param {number} [options.limit=3] - Max entries to return
+   * @param {number} [options.threshold=0.7] - Confidence threshold for "found"
+   * @returns {{found: boolean, entries: Array, confidence: number}}
+   */
+  hasRelevantKnowledge(query, options = {}) {
+    const limit = options.limit || 3;
+    const threshold = options.threshold || 0.7;
+
+    const results = this.searchMemories(query, { limit });
+    if (results.length === 0) {
+      return { found: false, entries: [], confidence: 0 };
+    }
+
+    const topScore = results[0].score;
+    return {
+      found: topScore >= threshold,
+      entries: results,
+      confidence: topScore,
+    };
+  }
+
+  /**
+   * Assign information priority tier to a memory entry.
+   *
+   * Maps entry types to Constitution XII priority tiers:
+   * - Tier 2: Project-local knowledge (learnings, decisions, handoffs, plans)
+   * - Tier 4: Web-sourced research
+   * - Tier 3: Everything else
+   *
+   * @param {Object} entry - Index entry with type field
+   * @returns {number} Priority tier (1-5)
+   */
+  _assignPriorityTier(entry) {
+    switch (entry.type) {
+      case "learning":
+      case "decision":
+      case "handoff":
+      case "plan":
+      case "objective":
+      case "reminder":
+      case "process":
+      case "feedback":
+        return 2; // Project-local knowledge
+      case "research":
+        return entry.keywords?.includes("web") ? 4 : 3;
+      default:
+        return 3;
+    }
+  }
+
+  clearCache() {
+    this._indexCache = null;
+    this._indexCacheTime = 0;
+  }
+}
+
+module.exports = { ThoughtsProvider };