mindlore 0.7.0 → 0.7.1

package/hooks/src/mindlore-read-guard.cjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * mindlore-read-guard — PreToolUse hook (if: "Read")
+ *
+ * Repeated-read detection: detects files read multiple times
+ * in the same session and emits a soft warning.
+ * Does NOT block (exit 0) — advisory only.
+ *
+ * Storage: .mindlore/diary/_session-reads.json
+ * Cleanup: session-end hook writes stats to delta then deletes the file.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { findMindloreDir, readHookStdin, getProjectName, hookLog, extractSkeleton, withTelemetrySync } = require('./lib/mindlore-common.cjs');
+
+function main() {
+  const baseDir = findMindloreDir();
+  if (!baseDir) return;
+
+  const filePath = readHookStdin(['file_path', 'path']);
+  if (!filePath) return;
+
+  // Only track CWD-relative files, skip .mindlore/ internals
+  const cwd = process.cwd();
+  const resolved = path.resolve(filePath);
+  if (!resolved.startsWith(cwd)) return;
+  if (resolved.startsWith(path.resolve(baseDir))) return;
+
+  // Load or create session reads tracker
+  const diaryDir = path.join(baseDir, 'diary');
+  if (!fs.existsSync(diaryDir)) {
+    fs.mkdirSync(diaryDir, { recursive: true });
+  }
+
+  const readsPath = path.join(diaryDir, `_session-reads-${getProjectName()}.json`);
+  let reads = {};
+  if (fs.existsSync(readsPath)) {
+    try {
+      reads = JSON.parse(fs.readFileSync(readsPath, 'utf8'));
+    } catch (_err) {
+      reads = {};
+    }
+  }
+
+  const normalizedPath = path.resolve(filePath);
+  const existing = reads[normalizedPath];
+
+  // Support both old format (number) and new format (object with tokens)
+  let count, tokens;
+  if (typeof existing === 'number') {
+    count = existing + 1;
+    tokens = 0;
+    reads[normalizedPath] = { count, tokens: 0, chars: 0 };
+  } else if (existing && typeof existing === 'object') {
+    count = (existing.count || 0) + 1;
+    tokens = existing.tokens || 0;
+    existing.count = count;
+    reads[normalizedPath] = existing;
+  } else {
+    count = 1;
+    tokens = 0;
+    reads[normalizedPath] = { count, tokens: 0, chars: 0 };
+  }
+
+  // Write updated reads
+  fs.writeFileSync(readsPath, JSON.stringify(reads, null, 2), 'utf8');
+
+  const basename = path.basename(filePath);
+  const tokenInfo = tokens > 0 ? ` (~${tokens} token)` : '';
+
+  // Block on 3+ repeated reads (exit 2 = block tool call)
+  if (count >= 3) {
+    const totalWaste = tokens > 0 ? ` Toplam israf: ~${tokens * (count - 1)} token.` : '';
+    process.stderr.write(`[Mindlore BLOCK] ${basename}${tokenInfo} bu session'da ${count}. kez okunuyor.${totalWaste} Edit icin gerekiyorsa once degisikligini yap, sonra tekrar oku. Analiz icin ctx_execute_file kullan.`);
+    process.exit(2);
+  }
+
+  // Warn on 2nd read (exit 0 = allow but warn)
+  if (count > 1) {
+    const totalWaste = tokens > 0 ? ` Toplam tekrar: ~${tokens * (count - 1)} token.` : '';
+    let skeletonSection = '';
+    try {
+      const ext = path.extname(filePath).slice(1);
+      const fileContent = fs.readFileSync(filePath, 'utf8');
+      if (fileContent.length < 500_000) {
+        const skeleton = extractSkeleton(fileContent, ext);
+        if (skeleton !== fileContent) {
+          const truncated = skeleton.length > 2000 ? skeleton.slice(0, 2000) + '\n...[truncated]' : skeleton;
+          skeletonSection = '\n\n' + truncated;
+        }
+      }
+    } catch (_e) { /* unreadable/binary — skip */ }
+    process.stdout.write(JSON.stringify({
+      hookSpecificOutput: {
+        hookEventName: 'PreToolUse',
+        additionalContext: `[Mindlore: ${basename}${tokenInfo} bu session'da ${count}. kez okunuyor.${totalWaste} Bir sonraki okuma engellenecek — Edit gerekiyorsa simdi yap.]${skeletonSection}`
+      }
+    }));
+  }
+}
+
+try { withTelemetrySync('mindlore-read-guard', main); } catch (err) { hookLog('read-guard', 'error', err?.message ?? String(err)); }

package/hooks/src/mindlore-research-guard.cjs

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * mindlore-research-guard — PreToolUse (Agent) hook
+ *
+ * Before spawning a researcher agent, checks FTS5 for existing knowledge.
+ * - High quality + recent (30 days) match → exit 2 (block)
+ * - Old or low quality match → exit 0 with warning (additionalContext)
+ * - No match → silent pass
+ *
+ * Prevents redundant web research when knowledge already exists in DB.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { getAllDbs, requireDatabase, extractKeywords, sanitizeKeyword, hookLog, withTelemetrySync } = require('./lib/mindlore-common.cjs');
+
+// Keywords that signal a research/web-search intent in agent prompts
+// Note: entries with dots/stars are regex patterns, rest are literals
+const RESEARCH_SIGNALS = [
+  'research', 'araştır', 'arastir', 'investigate', 'search for',
+  'web search', 'websearch', 'webfetch', 'fetch.*url', 'look up',
+  'find out', 'check.*docs', 'documentation.*for',
+];
+const RESEARCH_REGEX = new RegExp(RESEARCH_SIGNALS.join('|'), 'i');
+
+// Exclude ingest/internal operations (they intentionally fetch URLs)
+const EXCLUDE_REGEX = /\[mindlore:|\bmindlore-ingest\b|ingest.*url|save.*raw|\[research-override\]/i;
+
+const MAX_AGE_DAYS = 30;
+
+function isRecent(dateStr) {
+  if (!dateStr) return false;
+  const d = new Date(dateStr);
+  if (isNaN(d.getTime())) return false;
+  const diff = (Date.now() - d.getTime()) / (1000 * 60 * 60 * 24);
+  return diff <= MAX_AGE_DAYS;
+}
+
+/**
+ * Search FTS5 using a single OR query instead of per-path×keyword loop.
+ * Returns top matches with quality and date from FTS5 columns (no file I/O).
+ */
+function searchDbs(keywords) {
+  const Database = requireDatabase();
+  if (!Database) return [];
+
+  const sanitized = keywords.map(sanitizeKeyword).filter(Boolean);
+  if (sanitized.length === 0) return [];
+
+  const matchQuery = sanitized.join(' OR ');
+  const dbPaths = getAllDbs();
+  const results = [];
+
+  for (const dbPath of dbPaths) {
+    try {
+      const db = new Database(dbPath, { readonly: true });
+
+      // Single FTS5 query — O(1) instead of O(paths × keywords)
+      const rows = db.prepare(
+        `SELECT path, slug, title, description, quality, date_captured, rank
+         FROM mindlore_fts
+         WHERE mindlore_fts MATCH ?
+         ORDER BY rank
+         LIMIT 10`
+      ).all(matchQuery);
+
+      for (const row of rows) {
+        const quality = (row.quality || 'medium').toLowerCase();
+        const date_captured = row.date_captured || null;
+
+        results.push({
+          slug: row.slug || path.basename(row.path, '.md'),
+          title: row.title || row.description || row.slug || '',
+          quality,
+          date_captured,
+          recent: isRecent(date_captured),
+          rank: row.rank,
+        });
+      }
+
+      db.close();
+    } catch (_err) { /* db open or query failed */ }
+  }
+
+  // Sort by rank (lower = better match in FTS5)
+  results.sort((a, b) => a.rank - b.rank);
+  return results.slice(0, 5);
+}
+
+function main() {
+  let input;
+  try {
+    const raw = fs.readFileSync(0, 'utf8').trim();
+    if (!raw) return;
+    input = JSON.parse(raw);
+  } catch (_err) {
+    return;
+  }
+
+  const toolName = input.tool_name || '';
+  if (toolName !== 'Agent') return;
+
+  const toolInput = input.tool_input || {};
+
+  // Only block agents with web access — let local-only agents pass
+  const WEB_CAPABLE_TYPES = ['researcher', 'general-purpose'];
+  const LOCAL_ONLY_TYPES = [
+    'Explore', 'coder', 'code-reviewer', 'Plan',
+    'bug-analyzer', 'security-reviewer', 'contrarian',
+    'scope-guardian', 'quality-gate', 'test-runner',
+  ];
+  const subagentType = toolInput.subagent_type || '';
+  const description = (toolInput.description || '').toLowerCase();
+
+  // Known local-only agent → always pass
+  if (LOCAL_ONLY_TYPES.includes(subagentType)) return;
+
+  // Known web-capable agent → continue to FTS5 check
+  // Unknown or empty subagent_type → check description for research intent
+  if (subagentType && !WEB_CAPABLE_TYPES.includes(subagentType)) return;
+
+  // If no subagent_type, check description for web research intent (reuse RESEARCH_REGEX)
+  if (!subagentType && !RESEARCH_REGEX.test(description)) return;
+
+  const prompt = (toolInput.prompt || '') + ' ' + (toolInput.description || '');
+
+  // Skip mindlore internal operations and explicit overrides
+  if (EXCLUDE_REGEX.test(prompt)) return;
+
+  // If subagent_type is a known research type, skip prompt-level regex check
+  // Otherwise require research signals in the prompt text
+  const isKnownResearchType = WEB_CAPABLE_TYPES.includes(subagentType);
+  if (!isKnownResearchType && !RESEARCH_REGEX.test(prompt)) return;
+
+  const keywords = extractKeywords(prompt, 10);
+  if (keywords.length < 2) return;
+
+  const matches = searchDbs(keywords);
+  if (matches.length === 0) return;
+
+  // Prevents false positives like "claude-code-repo" matching on generic words
+  const lcKeywords = keywords.map((k) => k.toLowerCase());
+  const relevantMatches = matches.filter((m) => {
+    const haystack = `${m.slug} ${m.title}`.toLowerCase();
+    const overlap = lcKeywords.filter((k) => haystack.includes(k));
+    return overlap.length >= 2;
+  });
+
+  if (relevantMatches.length === 0) return;
+
+  // Check for high-quality recent matches among relevant ones
+  const strongMatches = relevantMatches.filter((m) => m.quality === 'high' && m.recent);
+
+  if (strongMatches.length > 0) {
+    const slugList = strongMatches.map((m) => `  - ${m.slug} (${m.title})`).join('\n');
+    const msg = `[mindlore-research-guard] BLOK: Bu konuda guncel, yuksek kaliteli bilgi DB'de zaten var.\n` +
+      `Once mevcut bilgiyi oku:\n${slugList}\n` +
+      `Eger bilgi yetersizse, prompt'a "[research-override]" ekleyerek tekrar dene.`;
+    process.stderr.write(msg);
+    process.exit(2);
+  }
+
+  // WARN: relevant but old or low-quality matches exist
+  const slugList = relevantMatches.map((m) => `${m.slug} (${m.quality}, ${m.date_captured || 'tarih yok'})`).join(', ');
+  const output = {
+    hookSpecificOutput: {
+      hookEventName: 'PreToolUse',
+      additionalContext: `[mindlore-research-guard] DB'de ilgili bilgi var ama eski/dusuk kalite: ${slugList}. Guncelleme gerekebilir — arastirma sonrasi DB'yi guncelle.`,
+    },
+  };
+  process.stdout.write(JSON.stringify(output));
+}
+
+try { withTelemetrySync('mindlore-research-guard', main); } catch (err) { hookLog('research-guard', 'error', err?.message ?? String(err)); }

package/hooks/src/mindlore-search.cjs

@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * mindlore-search — UserPromptSubmit hook
+ *
+ * Thin wrapper over search-engine.ts pipeline.
+ * Extracts keywords from user prompt, delegates search to modular engine,
+ * injects top results with description + headings.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { getAllDbs, openDatabase, extractHeadings, readHookStdin, readConfig, hookLog, incrementRecallCount, withTelemetry } = require('./lib/mindlore-common.cjs');
+
+const MAX_RESULTS = 3;
+const MIN_QUERY_WORDS = 3;
+
+let searchEngineMod;
+try {
+  searchEngineMod = require('../dist/scripts/lib/search-engine.js');
+} catch (_err) {
+  // search-engine not built yet
+}
+
+let SearchCacheMod;
+try {
+  SearchCacheMod = require('../dist/scripts/lib/search-cache.js');
+} catch (_err) {
+  // search-cache not built yet
+}
+
+function main() {
+  const userMessage = readHookStdin(['prompt', 'content', 'message', 'query']);
+  if (!userMessage || userMessage.length < MIN_QUERY_WORDS) return;
+
+  const dbPaths = getAllDbs();
+  if (dbPaths.length === 0) return;
+
+  if (!searchEngineMod) {
+    hookLog('search', 'warn', 'search-engine module not available — skipping');
+    return;
+  }
+
+  const project = path.basename(process.cwd());
+  const config = readConfig(path.dirname(dbPaths[0]));
+  const synonyms = (config && config.synonyms) ? config.synonyms : {};
+
+  // Read session_id from stdin for throttling
+  let sessionId;
+  try {
+    const stdinData = JSON.parse(process.env.CLAUDE_HOOK_STDIN || '{}');
+    sessionId = stdinData.session_id || 'unknown';
+  } catch (_) {
+    sessionId = 'unknown';
+  }
+
+  const allResults = [];
+  for (const dbPath of dbPaths) {
+    const db = openDatabase(dbPath);
+    if (!db) continue;
+    try {
+      // Cache + throttle
+      let cache;
+      let effectiveMax = MAX_RESULTS;
+      if (SearchCacheMod) {
+        cache = new SearchCacheMod.SearchCache(db, { ttlMs: 300000 });
+        const throttle = new SearchCacheMod.SearchThrottle(db);
+        const callCount = throttle.incrementCallCount(sessionId);
+        effectiveMax = throttle.getMaxResults(callCount);
+        if (effectiveMax === 0) {
+          hookLog('search', 'info', `Throttled (call #${callCount})`);
+          db.close();
+          continue;
+        }
+        const cached = cache.get(userMessage);
+        if (cached) {
+          const baseDir = path.dirname(dbPath);
+          for (const r of cached) allResults.push({ ...r, baseDir });
+          db.close();
+          continue;
+        }
+      }
+
+      const results = searchEngineMod.search(db, userMessage, {
+        project,
+        maxResults: effectiveMax,
+        synonyms,
+      });
+
+      if (cache) cache.set(userMessage, results);
+
+      const baseDir = path.dirname(dbPath);
+      for (const r of results) {
+        allResults.push({ ...r, baseDir });
+      }
+
+      // Recall count inside loop — avoid reopening DB
+      try {
+        const txn = db.transaction(() => {
+          for (const r of results) incrementRecallCount(db, r.path);
+        });
+        txn();
+      } catch (_e) { /* graceful */ }
+    } catch (err) {
+      hookLog('search', 'warn', `Search error: ${err?.message || err}`);
+    } finally {
+      db.close();
+    }
+  }
+
+  // Deduplicate by full path
+  const seen = new Set();
+  const unique = [];
+  for (const r of allResults) {
+    const normalized = path.resolve(r.path);
+    if (!seen.has(normalized)) {
+      seen.add(normalized);
+      unique.push(r);
+    }
+  }
+
+  // Sort by score descending, take top N
+  unique.sort((a, b) => b.score - a.score);
+  const relevant = unique.slice(0, MAX_RESULTS);
+  if (relevant.length === 0) return;
+
+  // Token budget from config
+  const budget = (config && config.tokenBudget) || {};
+  const perResultChars = ((budget.perResult || 500) * 4);
+  const totalChars = ((budget.searchResults || 1500) * 4);
+
+  // Build output
+  const output = [];
+  let totalUsed = 0;
+  for (const r of relevant) {
+    if (totalUsed >= totalChars) break;
+    const relativePath = path.relative(r.baseDir, r.path).replace(/\\/g, '/');
+
+    let headings = [];
+    const contentStr = r.content || '';
+    if (contentStr) {
+      try {
+        headings = extractHeadings(contentStr, 3);
+      } catch (_err) { /* skip */ }
+    }
+
+    const category = r.category || path.dirname(relativePath).split('/')[0];
+    const title = r.title || r.slug || path.basename(r.path, '.md');
+    const description = r.description || '';
+
+    const headingStr = headings.length > 0 ? `\nBasliklar: ${headings.join(', ')}` : '';
+    const tagsStr = r.tags ? `\nTags: ${r.tags}` : '';
+    const snippetOrDesc = r.snippet || description;
+    const entry = `[Mindlore: ${category}/${title}] ${snippetOrDesc}\nDosya: ${relativePath}${tagsStr}${headingStr}`;
+    const truncated = entry.slice(0, perResultChars);
+    totalUsed += truncated.length;
+    output.push(truncated);
+  }
+
+  if (output.length > 0) {
+    let outputStr = output.join('\n\n') + '\n';
+
+    const OFFLOAD_THRESHOLD = 10240;
+    if (outputStr.length > OFFLOAD_THRESHOLD) {
+      const baseDir = path.dirname(dbPaths[0]);
+      const tmpDir = path.join(baseDir, 'tmp');
+      fs.mkdirSync(tmpDir, { recursive: true });
+
+      try {
+        const oneHourAgo = Date.now() - 3600000;
+        const files = fs.readdirSync(tmpDir)
+          .filter(f => f.startsWith('search-'))
+          .map(f => ({ name: f, mtime: fs.statSync(path.join(tmpDir, f)).mtimeMs }))
+          .sort((a, b) => b.mtime - a.mtime);
+        for (let i = 0; i < files.length; i++) {
+          if (i >= 20 || files[i].mtime < oneHourAgo) {
+            try { fs.unlinkSync(path.join(tmpDir, files[i].name)); } catch { /* ignore */ }
+          }
+        }
+      } catch { /* cleanup is best-effort */ }
+      const fileName = `search-${Date.now()}.md`;
+      const filePath = path.join(tmpDir, fileName);
+      fs.writeFileSync(filePath, outputStr, 'utf8');
+
+      const summary = outputStr.slice(0, 500).replace(/\n/g, ' ').trim();
+      outputStr = `[Mindlore Search: ${outputStr.length} chars offloaded to ${filePath}]\n` +
+                  `Summary: ${summary}...\n` +
+                  `[Read full results: ${filePath}]`;
+      hookLog('search', 'info', 'offloaded to tmp/ (' + outputStr.length + ' chars)');
+    }
+
+    process.stdout.write(outputStr);
+  }
+}
+
+withTelemetry('mindlore-search', main).catch(err => {
+  hookLog('mindlore-search', 'error', err?.message ?? String(err));
+  process.exit(0);
+});