lore-memory 0.1.0

package/src/mcp/tools/search.js

@@ -0,0 +1,96 @@
+'use strict';
+
+const { readIndex } = require('../../lib/index');
+const { readEntry } = require('../../lib/entries');
+const { enforceBudget, formatEntry } = require('../../lib/budget');
+const { readConfig } = require('../../lib/config');
+
+const toolDefinition = {
+  name: 'lore_search',
+  description: 'Search Lore entries by keyword or semantic meaning. Returns matching architectural decisions, invariants, gotchas, and graveyard entries.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      query: {
+        type: 'string',
+        description: 'Search query — keyword or natural language description',
+      },
+      type: {
+        type: 'string',
+        enum: ['decision', 'invariant', 'gotcha', 'graveyard'],
+        description: 'Optional: filter by entry type',
+      },
+    },
+    required: ['query'],
+  },
+};
+
+async function handler(args) {
+  const { query, type: filterType } = args;
+  const config = readConfig();
+  const budget = (config.mcp && config.mcp.tokenBudget) ? config.mcp.tokenBudget : 4000;
+
+  try {
+    const index = readIndex();
+    const q = query.toLowerCase();
+    const matches = [];
+
+    for (const entryPath of Object.values(index.entries)) {
+      const entry = readEntry(entryPath);
+      if (!entry) continue;
+      if (filterType && entry.type !== filterType) continue;
+
+      const searchable = [
+        entry.title,
+        entry.context,
+        ...(entry.alternatives || []),
+        entry.tradeoffs || '',
+        ...(entry.tags || []),
+      ].join(' ').toLowerCase();
+
+      if (searchable.includes(q)) {
+        matches.push(entry);
+      }
+    }
+
+    // Try semantic search if text search found nothing and embeddings exist
+    if (matches.length === 0) {
+      try {
+        const { findSimilar } = require('../../lib/embeddings');
+        const allIds = Object.keys(index.entries);
+        const similar = await findSimilar(query, allIds, 5);
+        for (const { id, score } of similar) {
+          if (score < 0.5) continue;
+          const entry = readEntry(index.entries[id]);
+          if (entry) {
+            if (!filterType || entry.type === filterType) {
+              matches.push(entry);
+            }
+          }
+        }
+      } catch (e) {
+        // Ollama not available — skip semantic search silently
+      }
+    }
+
+    if (matches.length === 0) {
+      return {
+        content: [{ type: 'text', text: `No entries found for: "${query}"` }],
+      };
+    }
+
+    const formatted = matches.map(e => formatEntry(e)).join('\n\n---\n\n');
+    const budgeted = enforceBudget(matches, budget);
+
+    return {
+      content: [{ type: 'text', text: budgeted || formatted }],
+    };
+  } catch (e) {
+    return {
+      content: [{ type: 'text', text: `Error: ${e.message}` }],
+      isError: true,
+    };
+  }
+}
+
+module.exports = { toolDefinition, handler };

package/src/mcp/tools/stale.js

@@ -0,0 +1,88 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const { readIndex } = require('../../lib/index');
+const { readEntry } = require('../../lib/entries');
+const { checkStaleness } = require('../../lib/stale');
+const { checkPatternStaleness } = require('../../watcher/staleness');
+
+const toolDefinition = {
+  name: 'lore_stale',
+  description: 'Check which Lore entries may be outdated. Combines mtime-based staleness (files changed since entry was written) with pattern-based semantic checks (e.g. new HTTP calls in performance paths, architecture shifts).',
+  inputSchema: {
+    type: 'object',
+    properties: {},
+    required: [],
+  },
+};
+
+async function handler() {
+  try {
+    const index = readIndex();
+    const projectRoot = process.cwd();
+    const staleItems = [];
+
+    for (const [id, entryPath] of Object.entries(index.entries)) {
+      const entry = readEntry(entryPath);
+      if (!entry) continue;
+
+      // mtime-based staleness (existing)
+      const staleFiles = checkStaleness(entry);
+      for (const s of staleFiles) {
+        const daysText = s.daysAgo === 0 ? 'today' : `${s.daysAgo} day${s.daysAgo === 1 ? '' : 's'} ago`;
+
+        // Pattern-based semantic staleness on the changed file
+        const reasons = [];
+        try {
+          const absPath = path.resolve(projectRoot, s.filepath);
+          if (fs.existsSync(absPath)) {
+            const code = fs.readFileSync(absPath, 'utf8');
+            const patternReasons = checkPatternStaleness(entry, s.filepath, code);
+            reasons.push(...patternReasons);
+          }
+        } catch (e) { /* ignore read errors */ }
+
+        staleItems.push({
+          id: entry.id,
+          title: entry.title,
+          type: entry.type,
+          file: s.filepath,
+          changedDaysAgo: s.daysAgo,
+          daysText,
+          reasons,
+        });
+      }
+    }
+
+    if (staleItems.length === 0) {
+      return {
+        content: [{ type: 'text', text: 'All Lore entries are up to date.' }],
+      };
+    }
+
+    const lines = ['The following Lore entries may be outdated:\n'];
+    for (const item of staleItems) {
+      lines.push(`• [${item.type.toUpperCase()}] ${item.title} (${item.id})`);
+      lines.push(`  File changed: ${item.file} (${item.daysText})`);
+      if (item.reasons.length > 0) {
+        for (const r of item.reasons) {
+          lines.push(`  ⚠  ${r}`);
+        }
+      }
+      lines.push(`  Review with: lore edit ${item.id}`);
+      lines.push('');
+    }
+
+    return {
+      content: [{ type: 'text', text: lines.join('\n') }],
+    };
+  } catch (e) {
+    return {
+      content: [{ type: 'text', text: `Error: ${e.message}` }],
+      isError: true,
+    };
+  }
+}
+
+module.exports = { toolDefinition, handler };

package/src/mcp/tools/why.js

@@ -0,0 +1,91 @@
+'use strict';
+
+const path = require('path');
+const { readIndex } = require('../../lib/index');
+const { readEntry } = require('../../lib/entries');
+const { loadGraph } = require('../../lib/graph');
+const { enforceBudget } = require('../../lib/budget');
+const { readConfig } = require('../../lib/config');
+
+const toolDefinition = {
+  name: 'lore_why',
+  description: 'Retrieve architectural decisions and context for a specific file or directory. Returns all relevant Lore entries, including graph-propagated context from imported and importing modules.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      filepath: {
+        type: 'string',
+        description: 'The file or directory path to look up context for (relative to project root)',
+      },
+    },
+    required: ['filepath'],
+  },
+};
+
+async function handler(args) {
+  const { filepath } = args;
+  const config = readConfig();
+  const budget = (config.mcp && config.mcp.tokenBudget) ? config.mcp.tokenBudget : 4000;
+
+  try {
+    const index = readIndex();
+    const graph = loadGraph();
+    const normalized = filepath.replace(/^\.\//, '');
+
+    // Collect entry IDs with weights: { id → maxWeight }
+    const weights = {};
+    function addIds(ids, w) {
+      for (const id of (ids || [])) {
+        weights[id] = Math.max(weights[id] || 0, w);
+      }
+    }
+
+    // Direct file match (1.0)
+    addIds(index.files[normalized], 1.0);
+    addIds(index.files[normalized + '/'], 1.0);
+
+    // Ancestor directory walk (0.7)
+    let current = normalized;
+    while (true) {
+      const parent = path.dirname(current);
+      if (parent === current || parent === '.') break;
+      addIds(index.files[parent + '/'], 0.7);
+      current = parent;
+    }
+
+    // Graph: files this file imports (0.3)
+    for (const dep of (graph.imports[normalized] || [])) {
+      addIds(index.files[dep], 0.3);
+    }
+
+    // Graph: files that import this file (0.2)
+    for (const dep of (graph.importedBy[normalized] || [])) {
+      addIds(index.files[dep], 0.2);
+    }
+
+    if (Object.keys(weights).length === 0) {
+      return {
+        content: [{ type: 'text', text: `No Lore entries found for: ${filepath}` }],
+      };
+    }
+
+    // Sort by weight descending
+    const entries = [];
+    for (const [id, weight] of Object.entries(weights)) {
+      const entryPath = index.entries[id];
+      const entry = readEntry(entryPath);
+      if (entry) entries.push(Object.assign({}, entry, { _score: weight }));
+    }
+    entries.sort((a, b) => b._score - a._score);
+
+    const context = enforceBudget(entries, budget);
+    return { content: [{ type: 'text', text: context }] };
+  } catch (e) {
+    return {
+      content: [{ type: 'text', text: `Error: ${e.message}` }],
+      isError: true,
+    };
+  }
+}
+
+module.exports = { toolDefinition, handler };

package/src/watcher/comments.js

@@ -0,0 +1,113 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const { detectType, extractTitle, scoreComment } = require('../lib/nlp');
+const { saveDraft } = require('../lib/drafts');
+
+/**
+ * Extract raw comment strings from source code.
+ * Tries Babel AST for JS/TS, falls back to regex for all languages.
+ */
+function extractComments(code, filePath) {
+  if (/\.(js|ts|jsx|tsx)$/.test(filePath)) {
+    try {
+      const babelParser = require('@babel/parser');
+      const ast = babelParser.parse(code, {
+        sourceType: 'unambiguous',
+        plugins: ['typescript', 'jsx', 'decorators-legacy'],
+        errorRecovery: true,
+      });
+      return (ast.comments || []).map(c => c.value.trim()).filter(Boolean);
+    } catch (e) {
+      // fall through
+    }
+  }
+
+  // Regex: covers JS, Python, Go, Rust, shell
+  const comments = [];
+  const patterns = [
+    /\/\/([^\n]+)/g,        // //
+    /\/\*([\s\S]*?)\*\//g,  // /* */
+    /#([^\n]+)/g,           // #
+    /--([^\n]+)/g,          // --
+  ];
+  for (const re of patterns) {
+    let m;
+    while ((m = re.exec(code)) !== null) {
+      const text = m[1].trim();
+      if (text) comments.push(text);
+    }
+  }
+  return comments;
+}
+
+/**
+ * Mine a single file for lore-worthy comments.
+ * Saves passing comments as drafts.
+ * @param {string} absFilePath
+ * @param {string} projectRoot
+ * @returns {object[]} created drafts
+ */
+function mineFile(absFilePath, projectRoot) {
+  let code = '';
+  try { code = fs.readFileSync(absFilePath, 'utf8'); } catch (e) { return []; }
+
+  const relativePath = path.relative(projectRoot, absFilePath).replace(/\\/g, '/');
+  const comments = extractComments(code, absFilePath);
+  const created = [];
+
+  // Deduplicate: skip if we have a recent draft from same file with very similar title
+  for (const comment of comments) {
+    const score = scoreComment(comment);
+    if (score < 0.5) continue;
+
+    const { type, confidence } = detectType(comment);
+    const title = extractTitle(comment);
+    if (!title || title.length < 3) continue;
+
+    const draft = {
+      draftId: `draft-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`,
+      suggestedType: type,
+      suggestedTitle: title,
+      evidence: comment.length > 300 ? comment.slice(0, 300) + '…' : comment,
+      files: [relativePath],
+      confidence: Math.min(1, (score + confidence) / 2),
+      createdAt: new Date().toISOString(),
+      status: 'pending',
+      source: 'comment-mine',
+    };
+
+    saveDraft(draft);
+    created.push(draft);
+  }
+
+  return created;
+}
+
+/**
+ * Mine all source files in a directory recursively.
+ * @param {string} absDirPath
+ * @param {string} projectRoot
+ * @param {string[]} ignore
+ * @returns {number} total drafts created
+ */
+function mineDirectory(absDirPath, projectRoot, ignore) {
+  const { globSync } = require('glob');
+  const ignoreList = ignore || ['node_modules', 'dist', '.git', '.lore', 'coverage'];
+  const ignorePats = ignoreList.map(i => `${i}/**`);
+
+  const files = globSync('**/*.{js,ts,jsx,tsx,py,go,rs}', {
+    cwd: absDirPath,
+    ignore: ignorePats,
+    absolute: true,
+  });
+
+  let total = 0;
+  for (const file of files) {
+    total += mineFile(file, projectRoot).length;
+  }
+  return total;
+}
+
+module.exports = { extractComments, mineFile, mineDirectory };

package/src/watcher/graph.js

@@ -0,0 +1,149 @@
+'use strict';
+
+const fs = require('fs-extra');
+const path = require('path');
+const { loadGraph, saveGraph } = require('../lib/graph');
+
+/**
+ * Resolve a relative import path to a project-relative file path.
+ */
+function resolveImport(fromFile, importPath, projectRoot) {
+  if (!importPath.startsWith('.') && !importPath.startsWith('/')) return null;
+
+  const fromDir = path.dirname(path.join(projectRoot, fromFile));
+  const resolved = path.resolve(fromDir, importPath);
+  const relative = path.relative(projectRoot, resolved);
+
+  const extensions = ['', '.js', '.ts', '.jsx', '.tsx', '/index.js', '/index.ts', '/index.jsx', '/index.tsx'];
+  for (const ext of extensions) {
+    const candidate = relative + ext;
+    if (fs.existsSync(path.join(projectRoot, candidate))) {
+      return candidate.replace(/\\/g, '/');
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract all imports from a source file.
+ * Uses @babel/parser for JS/TS, falls back to regex.
+ */
+function extractImports(code, filePath, projectRoot) {
+  const imports = [];
+
+  if (/\.(js|ts|jsx|tsx)$/.test(filePath)) {
+    try {
+      const babelParser = require('@babel/parser');
+      const ast = babelParser.parse(code, {
+        sourceType: 'unambiguous',
+        plugins: ['typescript', 'jsx', 'decorators-legacy'],
+        errorRecovery: true,
+      });
+
+      function visit(node) {
+        if (!node || typeof node !== 'object') return;
+
+        // ES import: import x from 'y'
+        if (node.type === 'ImportDeclaration' && node.source && node.source.value) {
+          const r = resolveImport(filePath, node.source.value, projectRoot);
+          if (r) imports.push(r);
+        }
+
+        // CJS require: require('y')
+        if (
+          node.type === 'CallExpression' &&
+          node.callee && node.callee.name === 'require' &&
+          node.arguments && node.arguments[0] &&
+          node.arguments[0].type === 'StringLiteral'
+        ) {
+          const r = resolveImport(filePath, node.arguments[0].value, projectRoot);
+          if (r) imports.push(r);
+        }
+
+        for (const key of Object.keys(node)) {
+          if (key === 'type' || key === 'loc' || key === 'start' || key === 'end') continue;
+          const val = node[key];
+          if (Array.isArray(val)) val.forEach(v => v && v.type && visit(v));
+          else if (val && typeof val === 'object' && val.type) visit(val);
+        }
+      }
+
+      visit(ast.program);
+      return imports;
+    } catch (e) {
+      // fall through to regex
+    }
+  }
+
+  // Regex fallback
+  const requireRe = /require\(['"]([^'"]+)['"]\)/g;
+  const importRe = /import(?:[^'"]*from)?\s*['"]([^'"]+)['"]/g;
+  for (const re of [requireRe, importRe]) {
+    let m;
+    while ((m = re.exec(code)) !== null) {
+      const r = resolveImport(filePath, m[1], projectRoot);
+      if (r) imports.push(r);
+    }
+  }
+  return imports;
+}
+
+/**
+ * Update the graph incrementally for a single changed file.
+ */
+function updateGraphForFile(absFilePath, projectRoot) {
+  const graph = loadGraph();
+  const filePath = path.relative(projectRoot, absFilePath).replace(/\\/g, '/');
+
+  // Remove old outgoing edges
+  const oldImports = graph.imports[filePath] || [];
+  for (const dep of oldImports) {
+    if (graph.importedBy[dep]) {
+      graph.importedBy[dep] = graph.importedBy[dep].filter(f => f !== filePath);
+    }
+  }
+
+  // Read and parse
+  let code = '';
+  try { code = fs.readFileSync(absFilePath, 'utf8'); } catch (e) { return; }
+
+  const newImports = extractImports(code, filePath, projectRoot);
+  graph.imports[filePath] = newImports;
+
+  for (const dep of newImports) {
+    if (!graph.importedBy[dep]) graph.importedBy[dep] = [];
+    if (!graph.importedBy[dep].includes(filePath)) {
+      graph.importedBy[dep].push(filePath);
+    }
+  }
+
+  saveGraph(graph);
+}
+
+/**
+ * Build the full import graph by scanning all JS/TS files.
+ */
+function buildFullGraph(projectRoot, ignorePatterns) {
+  const { globSync } = require('glob');
+  const ignore = (ignorePatterns || ['node_modules', 'dist', '.git', '.lore', 'coverage']).map(i => `${i}/**`);
+
+  const graph = { imports: {}, importedBy: {}, lastUpdated: new Date().toISOString() };
+  const files = globSync('**/*.{js,ts,jsx,tsx}', { cwd: projectRoot, ignore, absolute: false });
+
+  for (const file of files) {
+    let code = '';
+    try { code = fs.readFileSync(path.join(projectRoot, file), 'utf8'); } catch (e) { continue; }
+
+    const imports = extractImports(code, file, projectRoot);
+    graph.imports[file] = imports;
+
+    for (const dep of imports) {
+      if (!graph.importedBy[dep]) graph.importedBy[dep] = [];
+      if (!graph.importedBy[dep].includes(file)) graph.importedBy[dep].push(file);
+    }
+  }
+
+  return graph;
+}
+
+module.exports = { updateGraphForFile, buildFullGraph, extractImports };

package/src/watcher/index.js

@@ -0,0 +1,134 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs-extra');
+const chalk = require('chalk');
+const chokidar = require('chokidar');
+const { readConfig } = require('../lib/config');
+const { LORE_DIR } = require('../lib/index');
+const signals = require('./signals');
+const { mineFile } = require('./comments');
+const { updateGraphForFile } = require('./graph');
+
+function timestamp() {
+  return new Date().toLocaleTimeString('en-US', { hour12: false, hour: '2-digit', minute: '2-digit' });
+}
+
+/**
+ * Start the file watcher.
+ * @param {object} options - { quiet, logFile }
+ */
+function startWatcher(options = {}) {
+  const projectRoot = process.cwd();
+  const config = readConfig();
+  const ignore = config.watchIgnore || ['node_modules', 'dist', '.git', 'coverage'];
+  // Build regex: any segment matching ignore list or .lore
+  const ignoreRe = new RegExp(`(${[...ignore, '\\.lore'].map(s => s.replace('.', '\\.')).join('|')})`);
+
+  let draftCount = 0;
+
+  const log = options.logFile
+    ? (msg) => {
+        const plain = msg.replace(/\x1B\[[0-9;]*m/g, '');
+        fs.appendFileSync(options.logFile, `${new Date().toISOString()} ${plain}\n`);
+      }
+    : (msg) => console.log(msg);
+
+  if (!options.quiet) {
+    console.log(chalk.cyan('📖 Lore Watcher started'));
+    console.log(chalk.dim(`   Watching: ${projectRoot}`));
+    console.log(chalk.dim(`   Ignoring: ${ignore.join(', ')}`));
+    console.log(chalk.dim('   Press Ctrl+C to stop'));
+    console.log();
+  }
+
+  function recordDraft(draft, filepath) {
+    draftCount++;
+    const rel = path.relative(projectRoot, filepath) || filepath;
+    log(`${chalk.dim(`[${timestamp()}]`)} Signal detected in ${chalk.yellow(rel)} — queued for review`);
+  }
+
+  const watcher = chokidar.watch('.', {
+    cwd: projectRoot,
+    ignored: ignoreRe,
+    persistent: true,
+    ignoreInitial: true,
+    awaitWriteFinish: { stabilityThreshold: 500, pollInterval: 100 },
+  });
+
+  watcher.on('unlink', (relPath) => {
+    const abs = path.join(projectRoot, relPath);
+    const draft = signals.onFileDeletion(abs, projectRoot);
+    if (draft) recordDraft(draft, abs);
+  });
+
+  watcher.on('unlinkDir', (relPath) => {
+    const abs = path.join(projectRoot, relPath);
+    const draft = signals.onDirectoryDeletion(abs, projectRoot);
+    if (draft) recordDraft(draft, abs);
+  });
+
+  watcher.on('add', (relPath) => {
+    const abs = path.join(projectRoot, relPath);
+    const draft = signals.onNewFile(abs, projectRoot);
+    if (draft) recordDraft(draft, abs);
+  });
+
+  watcher.on('change', (relPath) => {
+    const abs = path.join(projectRoot, relPath);
+
+    // Repeated edit tracking
+    const editDraft = signals.trackFileEdit(abs, projectRoot);
+    if (editDraft) recordDraft(editDraft, abs);
+
+    // package.json changes
+    if (relPath.endsWith('package.json')) {
+      const pkgDrafts = signals.onPackageJsonChange(abs, projectRoot);
+      for (const d of pkgDrafts) recordDraft(d, abs);
+    }
+
+    // Comment mining + graph update for source files
+    if (/\.(js|ts|jsx|tsx|py|go|rs)$/.test(relPath)) {
+      const commentDrafts = mineFile(abs, projectRoot);
+      if (commentDrafts.length > 0) {
+        draftCount += commentDrafts.length;
+        log(`${chalk.dim(`[${timestamp()}]`)} Mined ${commentDrafts.length} comment${commentDrafts.length === 1 ? '' : 's'} from ${chalk.yellow(relPath)} — queued for review`);
+      }
+
+      try { updateGraphForFile(abs, projectRoot); } catch (e) {}
+    }
+  });
+
+  // Watch COMMIT_EDITMSG to detect new commits
+  const commitMsgPath = path.join(projectRoot, '.git', 'COMMIT_EDITMSG');
+  let gitWatcher = null;
+  if (fs.existsSync(path.join(projectRoot, '.git'))) {
+    gitWatcher = chokidar.watch(commitMsgPath, { persistent: true, ignoreInitial: true });
+    gitWatcher.on('change', () => {
+      try {
+        const message = fs.readFileSync(commitMsgPath, 'utf8').trim();
+        const drafts = signals.onCommitMessage(message, projectRoot);
+        for (const d of drafts) {
+          draftCount++;
+          log(`${chalk.dim(`[${timestamp()}]`)} Commit signal: "${message.slice(0, 60)}" — queued for review`);
+        }
+      } catch (e) {}
+    });
+  }
+
+  function shutdown() {
+    watcher.close();
+    if (gitWatcher) gitWatcher.close();
+    if (!options.quiet) {
+      console.log();
+      console.log(chalk.cyan(`📖 Lore captured ${draftCount} draft${draftCount === 1 ? '' : 's'} today. Review with: lore drafts`));
+    }
+  }
+
+  process.on('SIGINT', () => { shutdown(); process.exit(0); });
+  process.on('SIGTERM', () => { watcher.close(); if (gitWatcher) gitWatcher.close(); process.exit(0); });
+
+  return watcher;
+}
+
+module.exports = { startWatcher };