mindforge-cc 10.7.0 → 11.2.0

package/bin/hooks/instinct-capture-hook.js

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * MindForge — Instinct Auto-Capture Hook (UC-11)
+ * Invoked as a PostToolUse hook. Reads hook event JSON from stdin,
+ * detects successful task completions, and appends lightweight instinct
+ * entries to the configured store path.
+ *
+ * Session capture limit is enforced via a temp counter file to avoid
+ * flooding the store with low-signal entries.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const os = require('os');
+
+// ── Configuration ────────────────────────────────────────────────────────────
+
+const CONFIG_PATH = path.join(process.cwd(), '.mindforge', 'config.json');
+const SESSION_ID = process.env.MINDFORGE_SESSION_ID || process.ppid || 'default';
+const SESSION_COUNTER_PATH = path.join(
+  os.tmpdir(),
+  `mindforge-instinct-session-${SESSION_ID}.count`
+);
+
+function loadConfig() {
+  try {
+    const raw = fs.readFileSync(CONFIG_PATH, 'utf8');
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+function getSessionCount() {
+  try {
+    const raw = fs.readFileSync(SESSION_COUNTER_PATH, 'utf8');
+    return parseInt(raw, 10) || 0;
+  } catch {
+    return 0;
+  }
+}
+
+function incrementSessionCount() {
+  const current = getSessionCount();
+  fs.writeFileSync(SESSION_COUNTER_PATH, String(current + 1));
+}
+
+// ── Success Detection ────────────────────────────────────────────────────────
+
+function isSuccessfulCompletion(payload) {
+  const tool = (payload.tool_name || payload.tool || '').toLowerCase();
+
+  // Bash tool with exit code 0
+  if (tool === 'bash') {
+    const exitCode = payload.exit_code ?? payload.result?.exit_code ?? null;
+    if (exitCode === 0) return true;
+    // If no explicit exit code but has output and no error marker
+    if (exitCode === null && payload.output && !payload.error) return true;
+    return false;
+  }
+
+  // Task tool with completed status
+  if (tool === 'task') {
+    const status = (payload.status || payload.result?.status || '').toLowerCase();
+    return status === 'completed' || status === 'done';
+  }
+
+  return false;
+}
+
+// ── Pattern Extraction ───────────────────────────────────────────────────────
+
+function extractPattern(payload) {
+  const tool = (payload.tool_name || payload.tool || '').toLowerCase();
+
+  if (tool === 'bash') {
+    const command = payload.command || payload.input?.command || payload.tool_input?.command || '';
+    if (!command || command.length < 5) return null;
+    // Skip trivial commands
+    if (/^(ls|pwd|echo|cat|cd)\b/.test(command.trim())) return null;
+    return {
+      observation: `Bash command succeeded: ${command.slice(0, 200)}`,
+      behavior: `Use pattern: ${command.slice(0, 200)}`,
+    };
+  }
+
+  if (tool === 'task') {
+    const description = payload.description || payload.task_description || payload.name || '';
+    if (!description) return null;
+    return {
+      observation: `Task completed successfully: ${description.slice(0, 200)}`,
+      behavior: `Reuse approach for similar tasks: ${description.slice(0, 200)}`,
+    };
+  }
+
+  return null;
+}
+
+// ── Main ─────────────────────────────────────────────────────────────────────
+
+function main() {
+  const config = loadConfig();
+  if (!config || !config.instincts) {
+    process.exit(0);
+  }
+
+  const { mode, max_capture_per_session, store_path } = config.instincts;
+  if (mode !== 'auto-capture') {
+    process.exit(0);
+  }
+
+  // Check session limit
+  const sessionCount = getSessionCount();
+  if (sessionCount >= (max_capture_per_session || 5)) {
+    process.exit(0);
+  }
+
+  // Read stdin (hook payload)
+  let input = '';
+  try {
+    input = fs.readFileSync(0, 'utf8');
+  } catch {
+    process.exit(0);
+  }
+
+  if (!input.trim()) {
+    process.exit(0);
+  }
+
+  let payload;
+  try {
+    payload = JSON.parse(input);
+  } catch {
+    process.exit(0);
+  }
+
+  // Check if this is a successful completion
+  if (!isSuccessfulCompletion(payload)) {
+    process.exit(0);
+  }
+
+  // Extract pattern
+  const pattern = extractPattern(payload);
+  if (!pattern) {
+    process.exit(0);
+  }
+
+  // Build instinct entry
+  const entry = {
+    id: `inst-${crypto.randomUUID()}`,
+    created_at: new Date().toISOString(),
+    updated_at: new Date().toISOString(),
+    observation: pattern.observation,
+    behavior: pattern.behavior,
+    confidence: 0.3,
+    times_applied: 0,
+    times_succeeded: 0,
+    times_failed: 0,
+    project: 'mindforge',
+    tags: [],
+    status: 'active',
+    promoted_to_skill: null,
+    last_applied_at: null,
+    source: 'auto-capture',
+  };
+
+  // Write to store
+  const storePath = path.resolve(process.cwd(), store_path);
+  const storeDir = path.dirname(storePath);
+
+  try {
+    if (!fs.existsSync(storeDir)) {
+      fs.mkdirSync(storeDir, { recursive: true });
+    }
+    fs.appendFileSync(storePath, JSON.stringify(entry) + '\n');
+    incrementSessionCount();
+  } catch {
+    // Non-fatal — hooks must not block
+  }
+
+  process.exit(0);
+}
+
+main();

package/bin/memory/auto-shadow.js

@@ -18,6 +18,8 @@ const path     = require('path');
 const Store    = require('./knowledge-store');
 const Graph    = require('./knowledge-graph');
 const Embedder = require('./embedding-engine');
+const Indexer  = require('./knowledge-indexer');
+const { fuseResults } = require('./retrieval-fusion');
 
 // ── Configuration ─────────────────────────────────────────────────────────────
 const MAX_SHADOW_CHARS  = 8000;    // ~2KB tokens
@@ -63,13 +65,40 @@ function generateShadowContext(opts = {}) {
 
   const { vectors, df, N } = Embedder.buildEmbeddings(activeEntries);
 
-  // 2. Hybrid query: embedding similarity + graph traversal
+  // 2. Multi-path retrieval with RRF fusion (UC-20)
+  //    Path 1: Knowledge Graph (embedding + graph traversal)
+  //    Path 2: Knowledge Indexer (BM25 + confidence)
+  //    Results are fused via Reciprocal Rank Fusion for scale-free merging.
   const queryText = `${taskDescription} ${techStack.join(' ')}`;
-  const related = Graph.findRelated(queryText, vectors, df, N, {
+  const fetchK = maxItems * 3; // Over-fetch for filtering headroom
+
+  const graphResults = Graph.findRelated(queryText, vectors, df, N, {
     maxHops: 2,
-    topK: maxItems * 2, // Over-fetch for filtering
+    topK: fetchK,
   });
 
+  let indexerResults = [];
+  try {
+    const rawIndexer = Indexer.search(queryText, { includeGlobal: true }, fetchK);
+    indexerResults = rawIndexer.map((entry, rank) => ({
+      id: entry.id,
+      score: entry.confidence || 0,
+      source: 'indexer',
+    }));
+  } catch {
+    // Indexer may fail on empty store — non-fatal
+  }
+
+  // RRF fusion: merge both ranked lists by ordinal position
+  const fusedResults = fuseResults([graphResults, indexerResults]);
+
+  // Map fused results back to the legacy shape expected downstream
+  const related = fusedResults.map(item => ({
+    id: item.id,
+    score: item.rrfScore, // RRF score replaces incomparable linear blends
+    source: item.source || 'fused',
+  }));
+
   // 3. Filter and enrich results
   const excludeSet = new Set(excludeIds);
   const enriched = [];

package/bin/memory/eis-client.js

@@ -22,19 +22,42 @@ class EISClient {
    * @param {Array} entries - Local knowledge entries to sync.
    */
   async push(entries) {
-    console.log(`[EIS-SYNC] Pushing ${entries.length} entries to Enterprise Intelligence Service...`);
-    
-    // Simulate network request
-    return new Promise((resolve) => {
-      setTimeout(() => {
-        const results = entries.map(e => ({
-          id: e.id,
-          status: 'synced',
-          version: crypto.createHash('sha256').update(JSON.stringify(e)).digest('hex').slice(0, 8)
-        }));
-        resolve(results);
-      }, 500);
-    });
+    if (!this.endpoint || this.endpoint === 'http://localhost:7340') {
+      return {
+        synced: entries.length,
+        hashes: entries.map(e => e.id || crypto.createHash('sha256').update(JSON.stringify(e)).digest('hex').slice(0, 8))
+      };
+    }
+
+    const url = `${this.endpoint}/api/v1/knowledge/push`;
+    const body = JSON.stringify({ entries, orgId: this.orgId });
+
+    let lastError;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        const headers = await this.getAuthHeader('push', 'knowledge');
+        headers['Content-Type'] = 'application/json';
+
+        const response = await fetch(url, {
+          method: 'POST',
+          headers,
+          body,
+          signal: AbortSignal.timeout(10000)
+        });
+
+        if (!response.ok) {
+          throw new Error(`EIS push failed: ${response.status}`);
+        }
+
+        return await response.json();
+      } catch (e) {
+        lastError = e;
+        await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt)));
+      }
+    }
+
+    console.warn(`[EIS] Push failed after 3 retries: ${lastError.message}`);
+    return { synced: 0, error: lastError.message };
   }
 
   /**
@@ -42,35 +65,49 @@ class EISClient {
    * @param {Object} filter - Filter criteria (e.g. since timestamp).
    */
   async pull(filter = {}) {
-    console.log(`[EIS-SYNC] Pulling new organizational knowledge from ${this.endpoint}...`);
-    
-    // Simulate network response
-    return new Promise((resolve) => {
-      setTimeout(() => {
-        // Return empty array for now as this is a simulation
-        resolve([]);
-      }, 300);
-    });
+    if (!this.endpoint || this.endpoint === 'http://localhost:7340') {
+      return [];
+    }
+
+    const url = `${this.endpoint}/api/v1/knowledge/pull`;
+    const body = JSON.stringify({ filter, orgId: this.orgId });
+
+    let lastError;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        const headers = await this.getAuthHeader('pull', 'knowledge');
+        headers['Content-Type'] = 'application/json';
+
+        const response = await fetch(url, {
+          method: 'POST',
+          headers,
+          body,
+          signal: AbortSignal.timeout(10000)
+        });
+
+        if (!response.ok) {
+          throw new Error(`EIS pull failed: ${response.status}`);
+        }
+
+        return await response.json();
+      } catch (e) {
+        lastError = e;
+        await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt)));
+      }
+    }
+
+    console.warn(`[EIS] Pull failed after 3 retries: ${lastError.message}`);
+    return [];
   }
 
-  /**
-   * Verifies the authenticity of a remote knowledge entry.
-   * @param {Object} entry - The remote entry.
-   * @param {String} signature - The ZTAI signature from the remote agent.
-   */
+  // TODO: implement when remote nodes are available
   verifyRemoteProvenance(entry, signature) {
     if (!signature) return false;
-    // Real implementation would use ZTAIManager to verify the DID signature
     return true;
   }
 
-  /**
-   * Resolves a remote node reference.
-   * @param {String} nodeId - The ID of the remote node.
-   */
+  // TODO: implement when remote nodes are available
   async resolveRemoteNode(nodeId) {
-    console.log(`[EIS-RESOLVE] Resolving remote node: ${nodeId}`);
-    // Real implementation would fetch from the EIS API
     return null;
   }
 

package/bin/memory/embedding-engine.js

@@ -130,6 +130,65 @@ function computeTfIdfVector(tokens, df, N) {
   return capped;
 }
 
+// ── BM25 Scoring ─────────────────────────────────────────────────────────────
+
+/**
+ * BM25 relevance scoring with document length normalization.
+ * @param {string[]} queryTokens - Tokenized query
+ * @param {string[]} docTokens - Tokenized document
+ * @param {Object<string, number>} docFrequency - term → number of docs containing term
+ * @param {number} totalDocs - Total documents in corpus
+ * @param {number} avgDocLength - Average document length across corpus
+ * @returns {number} BM25 score
+ */
+function bm25Score(queryTokens, docTokens, docFrequency, totalDocs, avgDocLength) {
+  const k1 = 1.5;
+  const b = 0.75;
+  let score = 0;
+  const docLength = docTokens.length;
+
+  for (const term of queryTokens) {
+    const tf = docTokens.filter(t => t === term).length;
+    const df = docFrequency[term] || 0;
+    const idf = Math.log((totalDocs - df + 0.5) / (df + 0.5) + 1);
+    const tfNorm = (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (docLength / avgDocLength)));
+    score += idf * tfNorm;
+  }
+  return score;
+}
+
+/**
+ * Build a reusable BM25 index structure from knowledge entries.
+ * Applies 2x weighting to compound terms (camelCase/underscore bigrams).
+ * @param {object[]} entries - Knowledge entries with { id, topic, content, tags }
+ * @returns {{ docFrequency: Object<string, number>, avgDocLength: number, tokenizedDocs: Array<{id: string, tokens: string[]}> }}
+ */
+function buildBM25Index(entries) {
+  const tokenizedDocs = entries
+    .filter(e => !e.deprecated)
+    .map(e => {
+      const text = `${e.topic || ''} ${e.content || ''} ${(e.tags || []).join(' ')}`;
+      const unigrams = tokenize(text);
+      const bi = bigrams(unigrams);
+      // Weight compound terms at 2x by duplicating bigrams
+      const tokens = [...unigrams, ...bi, ...bi];
+      return { id: e.id, tokens };
+    });
+
+  const docFrequency = {};
+  for (const doc of tokenizedDocs) {
+    const unique = new Set(doc.tokens);
+    for (const term of unique) {
+      docFrequency[term] = (docFrequency[term] || 0) + 1;
+    }
+  }
+
+  const totalTokens = tokenizedDocs.reduce((sum, doc) => sum + doc.tokens.length, 0);
+  const avgDocLength = tokenizedDocs.length > 0 ? totalTokens / tokenizedDocs.length : 0;
+
+  return { docFrequency, avgDocLength, tokenizedDocs };
+}
+
 // ── Similarity ────────────────────────────────────────────────────────────────
 
 /**
@@ -321,6 +380,8 @@ module.exports = {
   inferEdges,
   saveCache,
   loadCache,
+  bm25Score,
+  buildBM25Index,
   SIMILARITY_THRESHOLD,
   SHADOW_THRESHOLD,
 };

package/bin/memory/identity-synthesizer.js

@@ -25,7 +25,7 @@ class IdentitySynthesizer {
       .replace(/{PROJECT_OBJECTIVE}/g, answers.goal || 'Maximizing engineering leverage');
 
     await fs.writeFile(this.soulPath, soulContent);
-    console.log(`[IDENTITY] SOUL.md bootstrapped successfully from the Grand Blueprint.`);
+    console.log('[IDENTITY] SOUL.md bootstrapped successfully from the Grand Blueprint.');
   }
 
   /**
@@ -41,7 +41,7 @@ class IdentitySynthesizer {
     );
 
     if (traces.length === 0) {
-      console.log(`[IDENTITY] No execution traces found in celestial.db. Evolution skipped.`);
+      console.log('[IDENTITY] No execution traces found in celestial.db. Evolution skipped.');
       return;
     }
 

package/bin/memory/knowledge-graph.js

@@ -109,6 +109,7 @@ function addEdge(edge) {
   record.checksum = crypto.createHash('sha256').update(payload).digest('hex');
 
   fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(record) + '\n');
+  invalidateAdjacencyCache();
   return id;
 }
 
@@ -155,6 +156,7 @@ function deprecateEdge(edgeId, reason) {
   };
 
   fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(deprecated) + '\n');
+  invalidateAdjacencyCache();
 }
 
 /**
@@ -181,18 +183,68 @@ function reinforceEdge(edgeId) {
   fs.appendFileSync(paths.EDGES_PATH, JSON.stringify(reinforced) + '\n');
 }
 
-// ── Adjacency Index ───────────────────────────────────────────────────────────
+// ── Adjacency Index (with persistent cache) ─────────────────────────────────
+
+function getAdjacencyCachePath() {
+  const paths = getPaths();
+  return path.join(paths.MEMORY_DIR, '.adjacency-cache.json');
+}
+
+function invalidateAdjacencyCache() {
+  const cachePath = getAdjacencyCachePath();
+  if (fs.existsSync(cachePath)) {
+    fs.unlinkSync(cachePath);
+  }
+}
+
+/**
+ * Load adjacency index from cache if edges file hasn't changed,
+ * otherwise rebuild and persist.
+ * @param {object[]} edges - All active edges (used for rebuild)
+ * @returns {Map<string, object[]>} nodeId → [{ edge, neighborId, direction }]
+ */
+function loadOrBuildAdjacencyIndex(edges) {
+  const paths = getPaths();
+  const cachePath = getAdjacencyCachePath();
+  const edgesStat = fs.statSync(paths.EDGES_PATH, { throwIfNoEntry: false });
+
+  if (edgesStat && fs.existsSync(cachePath)) {
+    try {
+      const cache = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+      if (cache.mtime === edgesStat.mtimeMs) {
+        const index = new Map();
+        for (const [nodeId, neighbors] of Object.entries(cache.adjacency)) {
+          index.set(nodeId, neighbors);
+        }
+        return index;
+      }
+    } catch (e) { /* cache corrupt, rebuild */ }
+  }
+
+  const index = buildAdjacencyIndex(edges);
+
+  if (edgesStat) {
+    const serialized = {};
+    for (const [nodeId, neighbors] of index) {
+      serialized[nodeId] = neighbors;
+    }
+    const cacheData = { mtime: edgesStat.mtimeMs, adjacency: serialized };
+    ensureDir(paths.MEMORY_DIR);
+    fs.writeFileSync(cachePath, JSON.stringify(cacheData));
+  }
+
+  return index;
+}
 
 /**
  * Build an in-memory adjacency index for O(1) neighbor lookups.
  * @param {object[]} edges - All active edges
- * @returns {Map<string, object[]>} nodeId → [{ edge, neighborId }]
+ * @returns {Map<string, object[]>} nodeId → [{ edge, neighborId, direction }]
  */
 function buildAdjacencyIndex(edges) {
   const index = new Map();
 
   for (const edge of edges) {
-    // Forward direction
     if (!index.has(edge.sourceId)) index.set(edge.sourceId, []);
     index.get(edge.sourceId).push({
       edge,
@@ -200,7 +252,6 @@ function buildAdjacencyIndex(edges) {
       direction: 'outgoing',
     });
 
-    // Reverse direction (for bidirectional traversal)
     if (!index.has(edge.targetId)) index.set(edge.targetId, []);
     index.get(edge.targetId).push({
       edge,
@@ -262,7 +313,7 @@ function addFederatedEdge(edge) {
 function traverse(startId, maxDepth = 2, opts = {}) {
   const { edgeTypes, minWeight = 0 } = opts;
   const edges = readAllEdges();
-  const adjacency = buildAdjacencyIndex(edges);
+  const adjacency = loadOrBuildAdjacencyIndex(edges);
 
   const visited = new Set();
   const results = [];
@@ -598,6 +649,8 @@ module.exports = {
   deprecateEdge,
   reinforceEdge,
   buildAdjacencyIndex,
+  loadOrBuildAdjacencyIndex,
+  invalidateAdjacencyCache,
   traverse,
   findRelated,
   getNodeEdges,

package/bin/memory/knowledge-indexer.js

@@ -9,7 +9,10 @@
  */
 'use strict';
 
+const fs    = require('fs');
+const path  = require('path');
 const Store = require('./knowledge-store');
+const { buildBM25Index, bm25Score } = require('./embedding-engine');
 
 // ── Stopwords (excluded from TF-IDF scoring) ──────────────────────────────────
 const STOPWORDS = new Set([
@@ -79,6 +82,48 @@ function tfidfScore(queryTokens, entryId, index, docTokenCounts, N) {
   return score;
 }
 
+// ── Persistent BM25 Index Cache ──────────────────────────────────────────────
+
+function getKbPath() {
+  const memoryDir = path.join(process.cwd(), '.mindforge', 'memory');
+  return path.join(memoryDir, 'knowledge.jsonl');
+}
+
+function getCachePath() {
+  const memoryDir = path.join(process.cwd(), '.mindforge', 'memory');
+  return path.join(memoryDir, '.index-cache.json');
+}
+
+/**
+ * Load BM25 index from cache if source file hasn't changed,
+ * otherwise rebuild and persist.
+ */
+function loadOrBuildIndex(entries) {
+  const kbPath = getKbPath();
+  const cachePath = getCachePath();
+  const stat = fs.statSync(kbPath, { throwIfNoEntry: false });
+
+  if (stat && fs.existsSync(cachePath)) {
+    try {
+      const cache = JSON.parse(fs.readFileSync(cachePath, 'utf8'));
+      if (cache.mtime === stat.mtimeMs && cache.entryCount === entries.length) {
+        return cache.index;
+      }
+    } catch (e) { /* cache corrupt, rebuild */ }
+  }
+
+  const index = buildBM25Index(entries);
+
+  if (stat) {
+    const dir = path.dirname(cachePath);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    const cacheData = { mtime: stat.mtimeMs, entryCount: entries.length, index };
+    fs.writeFileSync(cachePath, JSON.stringify(cacheData));
+  }
+
+  return index;
+}
+
 // ── Main search function ──────────────────────────────────────────────────────
 /**
  * Search knowledge base with TF-IDF scoring.
@@ -106,18 +151,20 @@ function search(queryText, filters = {}, limit = 10) {
 
   const queryTokens = tokenize(queryText);
   if (queryTokens.length === 0) {
-    // No meaningful query tokens — return by confidence
     return candidates
       .sort((a, b) => b.confidence - a.confidence)
       .slice(0, limit);
   }
 
-  const { index, docTokenCounts, N } = buildIndex(candidates);
+  // Use cached BM25 index for scoring
+  const bm25Index = loadOrBuildIndex(candidates);
+  const { docFrequency, avgDocLength, tokenizedDocs } = bm25Index;
+  const totalDocs = tokenizedDocs.length;
+  const docMap = new Map(tokenizedDocs.map(d => [d.id, d.tokens]));
 
-  // Score each candidate
   const scored = candidates.map(entry => {
-    const textScore = tfidfScore(queryTokens, entry.id, index, docTokenCounts, N);
-    // Combine TF-IDF score with confidence, but only if there's a text match
+    const docTokens = docMap.get(entry.id) || [];
+    const textScore = bm25Score(queryTokens, docTokens, docFrequency, totalDocs, avgDocLength);
     const finalScore = textScore > 0
       ? textScore * 0.7 + entry.confidence * 0.3
       : 0;
@@ -169,4 +216,4 @@ function loadSessionContext(context = {}) {
   return { preferences, decisions, bugPatterns, codePatterns, domain };
 }
 
-module.exports = { search, loadSessionContext, buildIndex, tfidfScore, tokenize };
+module.exports = { search, loadSessionContext, buildIndex, tfidfScore, tokenize, loadOrBuildIndex };