mindforge-cc 1.0.4 → 2.0.0-alpha.4

package/bin/memory/knowledge-capture.js

@@ -0,0 +1,278 @@
+/**
+ * MindForge v2 — Knowledge Capture Engine
+ * Automatically extracts and stores knowledge from MindForge lifecycle events.
+ */
+'use strict';
+
+const fs     = require('fs');
+const path   = require('path');
+const Store  = require('./knowledge-store');
+const Indexer = require('./knowledge-indexer');
+
+const PLANNING_DIR = path.join(process.cwd(), '.planning');
+const DECISIONS_DIR = path.join(PLANNING_DIR, 'decisions');
+
+// ── Capture helpers ───────────────────────────────────────────────────────────
+
+function getProjectName() {
+  const projectMd = path.join(PLANNING_DIR, 'PROJECT.md');
+  if (!fs.existsSync(projectMd)) return 'unknown';
+  const match = fs.readFileSync(projectMd, 'utf8').match(/^# (.+)/m);
+  return match?.[1]?.trim().slice(0, 50) || 'unknown';
+}
+
+function inferTagsFromText(text) {
+  const DOMAIN_TAGS = {
+    auth:        /auth|login|logout|jwt|token|session|password|bcrypt|argon/i,
+    database:    /database|sql|query|migration|prisma|drizzle|postgres|mysql|mongo/i,
+    api:         /api|endpoint|route|rest|graphql|webhook|request|response/i,
+    security:    /security|owasp|xss|csrf|injection|vulnerability|encryption/i,
+    performance: /performance|cache|cdn|lazy|async|concurrent|throttle|debounce/i,
+    testing:     /test|spec|mock|stub|fixture|coverage|jest|vitest|playwright/i,
+    ui:          /component|react|vue|svelte|css|style|render|layout/i,
+    infra:       /docker|kubernetes|ci|deploy|environment|config|env/i,
+  };
+
+  const tags = [];
+  for (const [tag, pattern] of Object.entries(DOMAIN_TAGS)) {
+    if (pattern.test(text)) tags.push(tag);
+  }
+  return tags;
+}
+
+function deduplicateOrAdd(entry) {
+  const existing = Indexer.search(`${entry.topic} ${entry.content}`, {
+    type: entry.type,
+    minConfidence: 0.0,
+    includeGlobal: false,
+  }, 3);
+
+  // Check if we have a near-duplicate
+  for (const e of existing) {
+    if (!e.deprecated && e.id) {
+      // High similarity — reinforce instead of duplicate
+      if (e.confidence >= entry.confidence) {
+        Store.reinforce(e.id);
+        return { action: 'reinforced', id: e.id };
+      } else {
+        // New entry has higher confidence — supersede old
+        Store.deprecate(e.id, 'Superseded by higher-confidence entry', null);
+      }
+    }
+  }
+
+  const id = Store.add(entry);
+  return { action: 'added', id };
+}
+
+// ── Event-specific capture functions ─────────────────────────────────────────
+
+/**
+ * Capture architectural decisions from ADR files after phase completion.
+ */
+function captureFromPhaseCompletion(phaseNum) {
+  if (!fs.existsSync(DECISIONS_DIR)) return [];
+
+  const captured = [];
+  const project  = getProjectName();
+
+  const adrFiles = fs.readdirSync(DECISIONS_DIR)
+    .filter(f => f.startsWith('ADR-') && f.endsWith('.md'))
+    .sort();
+
+  for (const adrFile of adrFiles) {
+    const filePath = path.join(DECISIONS_DIR, adrFile);
+    const content = fs.readFileSync(filePath, 'utf8');
+
+    // Extract decision content
+    const titleMatch   = content.match(/^# ADR-\d+: (.+)/m);
+    const decision     = (content.match(/## Decision\n+([\s\S]*?)(?=\n##)/)?.[1] || '').trim().slice(0, 500);
+    const rationale    = (content.match(/## Rationale\n+([\s\S]*?)(?=\n##)/)?.[1] || '').trim().slice(0, 500);
+    const status       = (content.match(/\*\*Status:\*\*\s*(.+)/)?.[1] || 'Unknown').trim();
+
+    if (!decision || status === 'Superseded') continue;
+
+    const topic = titleMatch?.[1]?.trim() || adrFile.replace('.md', '');
+
+    const result = deduplicateOrAdd({
+      type:          'architectural_decision',
+      topic:         topic.slice(0, 80),
+      content:       `${decision}\n\nRationale: ${rationale}`,
+      source:        `${adrFile} (Phase ${phaseNum})`,
+      project,
+      confidence:    0.90,
+      tags:          inferTagsFromText(content),
+      linked_adrs:   [adrFile.replace('.md', '')],
+      adr_reference: adrFile.replace('.md', ''),
+      decision,
+      rationale,
+    });
+
+    captured.push({ file: adrFile, ...result });
+  }
+
+  return captured;
+}
+
+/**
+ * Capture domain knowledge from smart compaction Block D (implicit knowledge).
+ */
+function captureFromCompaction(handoffPath) {
+  if (!fs.existsSync(handoffPath)) return [];
+
+  const handoff = JSON.parse(fs.readFileSync(handoffPath, 'utf8'));
+  const items   = handoff.implicit_knowledge || [];
+  const project = getProjectName();
+  const captured = [];
+
+  for (const item of items) {
+    if (!item || typeof item !== 'object') continue;
+
+    const confidence = item.confidence ?? 0.5;
+    if (confidence < 0.5) continue; // Skip low-confidence items
+
+    const result = deduplicateOrAdd({
+      type:       'domain_knowledge',
+      topic:      item.topic || item.text?.slice(0, 80) || 'Unknown topic',
+      content:    item.content || item.text || String(item),
+      source:     'Smart compaction Block D',
+      project,
+      confidence: confidence * 0.9, // Slight discount for auto-captured
+      tags:       inferTagsFromText(item.content || item.text || ''),
+    });
+
+    captured.push(result);
+  }
+
+  return captured;
+}
+
+/**
+ * Capture bug patterns from debug reports.
+ */
+function captureFromDebugReport(debugReportPath) {
+  if (!fs.existsSync(debugReportPath)) return null;
+
+  const content = fs.readFileSync(debugReportPath, 'utf8');
+  const project = getProjectName();
+
+  const rootCause = (content.match(/## Root [Cc]ause\n+([\s\S]*?)(?=\n##)/)?.[1] || '').trim();
+  const fix       = (content.match(/## Fix\n+([\s\S]*?)(?=\n##)/)?.[1] || '').trim();
+  const title     = (content.match(/^# Debug[:\s]+(.+)/m)?.[1] || 'Unknown bug').trim();
+
+  if (!rootCause) return null;
+
+  const result = deduplicateOrAdd({
+    type:        'bug_pattern',
+    topic:       title.slice(0, 80),
+    content:     `Root cause: ${rootCause}\n\nFix: ${fix}`,
+    source:      `Debug session: ${path.basename(debugReportPath)}`,
+    project,
+    confidence:  0.80,
+    tags:        inferTagsFromText(content),
+    bug_category: inferBugCategory(content),
+    root_cause:  rootCause.slice(0, 500),
+    fix:         fix.slice(0, 500),
+  });
+
+  return result;
+}
+
+/**
+ * Capture team preferences from retrospective reports.
+ */
+function captureFromRetrospective(retroReportPath) {
+  if (!fs.existsSync(retroReportPath)) return [];
+
+  const content = fs.readFileSync(retroReportPath, 'utf8');
+  const project = getProjectName();
+  const captured = [];
+
+  // Extract "keep doing" items (positive practices)
+  const keepSection = content.match(/## (Keep|What (?:we )?should we keep|Plus|Went well)\n+([\s\S]*?)(?=\n##)/i);
+  if (keepSection) {
+    const items = keepSection[2].split('\n')
+      .filter(l => l.startsWith('- ') || l.startsWith('* '))
+      .map(l => l.replace(/^[-*]\s+/, '').trim())
+      .filter(l => l.length > 20); // Skip trivial items
+
+    for (const item of items.slice(0, 5)) {
+      const result = deduplicateOrAdd({
+        type:        'team_preference',
+        topic:       item.slice(0, 80),
+        content:     item,
+        source:      `Retrospective: ${path.basename(retroReportPath)}`,
+        project,
+        confidence:  0.70,
+        tags:        inferTagsFromText(item),
+        preference:  item,
+        strength:    'moderate',
+        preference_type: 'process',
+      });
+      captured.push(result);
+    }
+  }
+
+  return captured;
+}
+
+/**
+ * Capture from cross-review consensus findings.
+ */
+function captureFromCrossReview(crossReviewPath) {
+  if (!fs.existsSync(crossReviewPath)) return [];
+
+  const content = fs.readFileSync(crossReviewPath, 'utf8');
+  const project = getProjectName();
+  const captured = [];
+
+  // Extract consensus findings table rows
+  const tableRows = content.match(/\|\s*\d+\s*\|\s*\*\*\w+\*\*\s*\|.+/g) || [];
+
+  for (const row of tableRows.slice(0, 10)) {
+    const cells = row.split('|').map(c => c.trim()).filter(Boolean);
+    if (cells.length < 3) continue;
+
+    const severity    = cells[1]?.replace(/\*\*/g, '') || 'MEDIUM';
+    const description = cells[2] || '';
+    if (description.length < 20) continue;
+
+    const result = deduplicateOrAdd({
+      type:        'bug_pattern',
+      topic:       description.slice(0, 80),
+      content:     description,
+      source:      `Cross-review consensus: ${path.basename(crossReviewPath)}`,
+      project,
+      confidence:  0.80,
+      tags:        [...inferTagsFromText(description), 'security'],
+      bug_category: 'security',
+      root_cause:  description,
+      severity_when_missed: severity,
+    });
+
+    captured.push(result);
+  }
+
+  return captured;
+}
+
+function inferBugCategory(text) {
+  if (/auth|login|session|jwt|token|password/i.test(text)) return 'auth';
+  if (/sql|database|query|migration/i.test(text))           return 'database';
+  if (/api|endpoint|route|request/i.test(text))             return 'api';
+  if (/ui|component|render|css/i.test(text))                return 'ui';
+  if (/performance|slow|timeout/i.test(text))               return 'performance';
+  if (/security|xss|injection|csrf/i.test(text))            return 'security';
+  return 'general';
+}
+
+module.exports = {
+  captureFromPhaseCompletion,
+  captureFromCompaction,
+  captureFromDebugReport,
+  captureFromRetrospective,
+  captureFromCrossReview,
+  deduplicateOrAdd,
+  inferTagsFromText,
+  inferBugCategory,
+};

package/bin/memory/knowledge-indexer.js

@@ -0,0 +1,172 @@
+/**
+ * MindForge v2 — Knowledge Indexer
+ * TF-IDF inspired relevance scoring for fast knowledge retrieval.
+ * Provides tag-based and text-based search across the knowledge graph.
+ *
+ * Design note: We use a simple in-memory index rebuilt on each query
+ * (not persisted) because the knowledge base stays small (< 10K entries
+ * for a typical project). Rebuild time < 50ms for 1K entries.
+ */
+'use strict';
+
+const Store = require('./knowledge-store');
+
+// ── Stopwords (excluded from TF-IDF scoring) ──────────────────────────────────
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'is', 'it', 'in', 'on', 'at', 'to', 'for', 'of', 'and',
+  'or', 'but', 'not', 'this', 'that', 'with', 'from', 'by', 'be', 'are',
+  'was', 'were', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
+  'could', 'should', 'may', 'might', 'can', 'use', 'using', 'used', 'when',
+  'where', 'which', 'what', 'how', 'why', 'who', 'all', 'any', 'some', 'we',
+  'our', 'they', 'their', 'we', 'you', 'your', 'my', 'its',
+]);
+
+// ── Tokenizer ─────────────────────────────────────────────────────────────────
+function tokenize(text) {
+  if (!text) return [];
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s_-]/g, ' ')
+    .split(/\s+/)
+    .filter(w => w.length > 2 && !STOPWORDS.has(w));
+}
+
+// ── Build in-memory index ─────────────────────────────────────────────────────
+function buildIndex(entries) {
+  const index = new Map(); // token → [{ id, count }]
+  const docTokenCounts = new Map(); // id → token count
+
+  for (const entry of entries) {
+    if (entry.deprecated) continue;
+
+    const text   = `${entry.topic} ${entry.content} ${(entry.tags || []).join(' ')}`;
+    const tokens = tokenize(text);
+    const counts = {};
+
+    for (const tok of tokens) {
+      counts[tok] = (counts[tok] || 0) + 1;
+    }
+
+    docTokenCounts.set(entry.id, tokens.length);
+
+    for (const [tok, count] of Object.entries(counts)) {
+      if (!index.has(tok)) index.set(tok, []);
+      index.get(tok).push({ id: entry.id, count });
+    }
+  }
+
+  return { index, docTokenCounts, N: entries.length };
+}
+
+// ── TF-IDF scoring ────────────────────────────────────────────────────────────
+function tfidfScore(queryTokens, entryId, index, docTokenCounts, N) {
+  let score = 0;
+  const docLen = docTokenCounts.get(entryId) || 1;
+
+  for (const qTok of queryTokens) {
+    const postings = index.get(qTok) || [];
+    const df       = postings.length; // Document frequency
+    if (df === 0) continue;
+
+    const posting  = postings.find(p => p.id === entryId);
+    if (!posting) continue;
+
+    const tf  = posting.count / docLen;              // Term frequency (normalized)
+    const idf = Math.log((N + 1) / (df + 1)) + 1;   // Smoothed IDF
+    score += tf * idf;
+  }
+
+  return score;
+}
+
+// ── Main search function ──────────────────────────────────────────────────────
+/**
+ * Search knowledge base with TF-IDF scoring.
+ * @param {string} queryText - Natural language query
+ * @param {object} filters   - Optional filters { type, tags, minConfidence }
+ * @param {number} limit     - Max results to return
+ * @returns {object[]} Ranked results
+ */
+function search(queryText, filters = {}, limit = 10) {
+  const allEntries = Store.readAll(filters.includeGlobal);
+  const active     = allEntries.filter(e => !e.deprecated);
+
+  // Apply filters
+  let candidates = active;
+  if (filters.type)          candidates = candidates.filter(e => e.type === filters.type);
+  if (filters.minConfidence) candidates = candidates.filter(e => e.confidence >= filters.minConfidence);
+  if (filters.tags?.length) {
+    const filterTags = filters.tags.map(t => t.toLowerCase());
+    candidates = candidates.filter(e =>
+      (e.tags || []).some(t => filterTags.includes(t.toLowerCase()))
+    );
+  }
+
+  if (candidates.length === 0) return [];
+
+  const queryTokens = tokenize(queryText);
+  if (queryTokens.length === 0) {
+    // No meaningful query tokens — return by confidence
+    return candidates
+      .sort((a, b) => b.confidence - a.confidence)
+      .slice(0, limit);
+  }
+
+  const { index, docTokenCounts, N } = buildIndex(candidates);
+
+  // Score each candidate
+  const scored = candidates.map(entry => {
+    const textScore = tfidfScore(queryTokens, entry.id, index, docTokenCounts, N);
+    // Combine TF-IDF score with confidence, but only if there's a text match
+    const finalScore = textScore > 0
+      ? textScore * 0.7 + entry.confidence * 0.3
+      : 0;
+    return { entry, score: finalScore };
+  });
+
+  return scored
+    .filter(s => s.score > 0)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit)
+    .map(s => s.entry);
+}
+
+/**
+ * Load session context: retrieve the most relevant memories for the current session.
+ * @param {object} context - { techStack, phase, topic, project }
+ * @returns {object} Categorized memories for session start display
+ */
+function loadSessionContext(context = {}) {
+  const { techStack = [], phase, topic = '', project } = context;
+
+  const allEntries = Store.readAll(true); // Include global knowledge
+  const active     = allEntries.filter(e => !e.deprecated && e.confidence >= 0.5);
+
+  // Build query from context
+  const queryText = [
+    topic,
+    ...(techStack || []),
+  ].join(' ');
+
+  const { index, docTokenCounts, N } = buildIndex(active);
+  const queryTokens = tokenize(queryText);
+
+  // Score all active entries
+  const scored = active.map(e => ({
+    entry: e,
+    score: queryTokens.length > 0
+      ? tfidfScore(queryTokens, e.id, index, docTokenCounts, N) * 0.6 + e.confidence * 0.4
+      : e.confidence,
+  })).sort((a, b) => b.score - a.score);
+
+  // Bucket by type, top N per bucket
+  const preferences  = scored.filter(s => s.entry.type === 'team_preference').slice(0, 5).map(s => s.entry);
+  const decisions    = scored.filter(s => s.entry.type === 'architectural_decision').slice(0, 8).map(s => s.entry);
+  const bugPatterns  = scored.filter(s => s.entry.type === 'bug_pattern').slice(0, 5).map(s => s.entry);
+  const codePatterns = scored.filter(s => s.entry.type === 'code_pattern').slice(0, 5).map(s => s.entry);
+  const domain       = scored.filter(s => s.entry.type === 'domain_knowledge').slice(0, 3).map(s => s.entry);
+
+  return { preferences, decisions, bugPatterns, codePatterns, domain };
+}
+
+module.exports = { search, loadSessionContext, buildIndex, tfidfScore, tokenize };