mindlore 0.6.1 → 0.6.3

package/hooks/mindlore-pre-compact.cjs

@@ -11,7 +11,63 @@
 
 const fs = require('fs');
 const path = require('path');
-const { findMindloreDir, hookLog, withTelemetry } = require('./lib/mindlore-common.cjs');
+const { findMindloreDir, openDatabase, hookLog, withTelemetry, listSnapshots } = require('./lib/mindlore-common.cjs');
+
+function collectRecentEpisodes(baseDir) {
+  try {
+    const dbPath = path.join(baseDir, 'mindlore.db');
+    const db = openDatabase(dbPath, { readonly: true });
+    if (!db) return [];
+    try {
+      const episodes = db.prepare(
+        "SELECT kind, summary FROM episodes WHERE created_at > datetime('now', '-4 hours') ORDER BY created_at DESC LIMIT 20"
+      ).all();
+      if (episodes.length === 0) return [];
+      const grouped = {};
+      for (const ep of episodes) {
+        const kind = ep.kind || 'other';
+        if (!grouped[kind]) grouped[kind] = [];
+        grouped[kind].push(ep.summary);
+      }
+      const lines = ['## Session Episodes'];
+      for (const [kind, items] of Object.entries(grouped)) {
+        lines.push(`### ${kind}`);
+        for (const item of items) lines.push(`- ${item}`);
+      }
+      return lines;
+    } finally {
+      db.close();
+    }
+  } catch (_err) {
+    return [];
+  }
+}
+
+function collectGitDiff() {
+  try {
+    const { execSync } = require('child_process');
+    const diffStat = execSync('git diff --stat HEAD 2>/dev/null || echo ""', {
+      encoding: 'utf8', timeout: 2000, windowsHide: true,
+    }).trim();
+    if (diffStat) return ['## Changed Files (uncommitted)', '```', diffStat, '```'];
+    return [];
+  } catch (_err) {
+    return [];
+  }
+}
+
+function getActivePlan() {
+  try {
+    const plansDir = path.join(process.cwd(), '.claude', 'plans');
+    if (!fs.existsSync(plansDir)) return [];
+    const plans = fs.readdirSync(plansDir).filter(f => f.endsWith('.md'));
+    if (plans.length === 0) return [];
+    const latestPlan = plans.sort().pop();
+    return [`## Active Plan: ${latestPlan}`];
+  } catch (_err) {
+    return [];
+  }
+}
 
 function main() {
   const baseDir = findMindloreDir();
@@ -34,11 +90,10 @@ function main() {
 
   const now = new Date();
   const iso = now.toISOString();
+  const ts = iso.replace(/[:.]/g, '-');
 
-  // Write pre-compact episode
   const episodesDir = path.join(baseDir, 'episodes');
   try {
-    const ts = iso.replace(/[:.]/g, '-');
     const episodePath = path.join(episodesDir, `pre-compact-${ts}.md`);
     const content = [
       '---',
@@ -61,6 +116,34 @@ function main() {
     fs.appendFileSync(logPath, entry, 'utf8');
   } catch (_err) { /* log file may not exist */ }
 
+  // Build compaction snapshot (#17)
+  const diaryDir = path.join(baseDir, 'diary');
+  try {
+    const sections = [];
+    sections.push(...collectRecentEpisodes(baseDir));
+    sections.push(...collectGitDiff());
+    sections.push(...getActivePlan());
+
+    if (sections.length > 0) {
+      const snapshotContent = [
+        '---',
+        'type: compaction-snapshot',
+        `date: ${iso.slice(0, 10)}`,
+        `project: ${path.basename(process.cwd())}`,
+        '---',
+        '',
+        ...sections,
+      ].join('\n');
+      fs.writeFileSync(path.join(diaryDir, `compaction-snapshot-${ts}.md`), snapshotContent);
+    }
+
+    const snapshots = listSnapshots(diaryDir).filter(f => f.startsWith('compaction-'));
+    while (snapshots.length > 5) {
+      const oldest = snapshots.shift();
+      if (oldest) fs.unlinkSync(path.join(diaryDir, oldest));
+    }
+  } catch (_err) { /* snapshot is best-effort */ }
+
   process.stdout.write('[Mindlore: pre-compact FTS5 flush complete]\n');
 }
 

package/hooks/mindlore-search.cjs

@@ -4,206 +4,114 @@
 /**
  * mindlore-search — UserPromptSubmit hook
  *
- * Extracts keywords from user prompt, searches FTS5 with per-keyword scoring,
- * injects top results with description + headings (matching old knowledge system quality).
+ * Thin wrapper over search-engine.ts pipeline.
+ * Extracts keywords from user prompt, delegates search to modular engine,
+ * injects top results with description + headings.
  */
 
 const fs = require('fs');
 const path = require('path');
-const { getAllDbs, openDatabase, extractHeadings, readHookStdin, extractKeywords, sanitizeKeyword, readConfig, loadSqliteVecCjs, hasVecTableCjs, hookLog, incrementRecallCount, getDaemonPortFile, withTelemetry, fixVersionTokens } = require('./lib/mindlore-common.cjs');
-
-const { execFileSync } = require('child_process');
+const { getAllDbs, openDatabase, extractHeadings, readHookStdin, readConfig, hookLog, incrementRecallCount, withTelemetry } = require('./lib/mindlore-common.cjs');
 
 const MAX_RESULTS = 3;
 const MIN_QUERY_WORDS = 3;
 
-// Try to load hybrid search module (built TS)
-let hybridSearchMod;
+let searchEngineMod;
 try {
-  hybridSearchMod = require('../dist/scripts/lib/hybrid-search.js');
+  searchEngineMod = require('../dist/scripts/lib/search-engine.js');
 } catch (_err) {
-  // hybrid-search not built yet — pure FTS5 mode
+  // search-engine not built yet
 }
 
-// v0.5.5: Request embedding from daemon via execFileSync bridge
-function requestEmbeddingSync(query) {
-  try {
-    const portFile = getDaemonPortFile();
-    if (!fs.existsSync(portFile)) return null;
-    const clientScript = path.join(__dirname, '..', 'scripts', 'lib', 'daemon-client.js');
-    if (!fs.existsSync(clientScript)) return null;
-    const result = execFileSync(process.execPath, [clientScript, portFile, query, '300'], {
-      timeout: 500, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
-    });
-    const parsed = JSON.parse(result.trim());
-    return parsed.type === 'embedding' ? parsed.embedding : null;
-  } catch {
-    return null;
-  }
+let SearchCacheMod;
+try {
+  SearchCacheMod = require('../dist/scripts/lib/search-cache.js');
+} catch (_err) {
+  // search-cache not built yet
 }
 
-/**
- * Search a single DB and return scored results with their baseDir.
- */
-function searchDb(dbPath, keywords) {
-  const baseDir = path.dirname(dbPath);
-  const db = openDatabase(dbPath, { readonly: true });
-  if (!db) return [];
-  const results = [];
+function main() {
+  const userMessage = readHookStdin(['prompt', 'content', 'message', 'query']);
+  if (!userMessage || userMessage.length < MIN_QUERY_WORDS) return;
 
-  // v0.5.0: Try hybrid search with synonym expansion (no embedding — hooks are sync)
-  if (!hybridSearchMod) {
-    hookLog('search', 'info', 'No hybridSearchMod — FTS5-only mode');
-  }
-  if (hybridSearchMod && loadSqliteVecCjs(db) && hasVecTableCjs(db)) {
-    try {
-      const config = readConfig(baseDir);
-      const synonyms = (config && config.synonyms) ? config.synonyms : {};
+  const dbPaths = getAllDbs();
+  if (dbPaths.length === 0) return;
 
-      // Expand keywords with synonyms
-      const expandedTerms = keywords.slice();
-      for (const kw of keywords) {
-        const lower = kw.toLowerCase();
-        if (synonyms[lower]) {
-          expandedTerms.push(...synonyms[lower]);
-        }
-      }
+  if (!searchEngineMod) {
+    hookLog('search', 'warn', 'search-engine module not available — skipping');
+    return;
+  }
 
-      // v0.5.5: Try to get queryEmbedding from daemon
-      let queryEmbedding = null;
-      try {
-        queryEmbedding = requestEmbeddingSync(expandedTerms.join(' '));
-        if (!queryEmbedding) {
-          hookLog('search', 'info', 'Daemon not available — FTS5-only hybrid mode');
-        }
-      } catch {
-        hookLog('search', 'info', 'Daemon connection failed — FTS5-only hybrid mode');
-      }
+  const project = path.basename(process.cwd());
+  const config = readConfig(path.dirname(dbPaths[0]));
+  const synonyms = (config && config.synonyms) ? config.synonyms : {};
 
-      const fusedResults = hybridSearchMod.hybridSearch(db, expandedTerms.join(' '), {
-        maxResults: MAX_RESULTS,
-        project: path.basename(process.cwd()),
-        queryEmbedding,
-      });
+  // Read session_id from stdin for throttling
+  let sessionId;
+  try {
+    const stdinData = JSON.parse(process.env.CLAUDE_HOOK_STDIN || '{}');
+    sessionId = stdinData.session_id || 'unknown';
+  } catch (_) {
+    sessionId = 'unknown';
+  }
 
-      if (fusedResults.length > 0) {
-        for (const r of fusedResults) {
-          const filePath = r.path || '';
-          let headings = [];
-          if (filePath) {
-            try {
-              const content = fs.readFileSync(filePath, 'utf8');
-              headings = extractHeadings(content, 3);
-            } catch (_err) { /* file may have been deleted */ }
-          }
-          results.push({
-            path: filePath,
-            slug: r.slug,
-            description: r.description || '',
-            category: r.category || '',
-            title: r.title || '',
-            tags: r.tags || '',
-            headings,
-            hits: 1,
-            rank: r.score,
-            baseDir,
-          });
+  const allResults = [];
+  for (const dbPath of dbPaths) {
+    const db = openDatabase(dbPath);
+    if (!db) continue;
+    try {
+      // Cache + throttle
+      let cache;
+      let effectiveMax = MAX_RESULTS;
+      if (SearchCacheMod) {
+        cache = new SearchCacheMod.SearchCache(db, { ttlMs: 300000 });
+        const callCount = cache.incrementCallCount(sessionId);
+        effectiveMax = cache.getMaxResults(callCount);
+        if (effectiveMax === 0) {
+          hookLog('search', 'info', `Throttled (call #${callCount})`);
+          db.close();
+          continue;
+        }
+        const cached = cache.get(userMessage);
+        if (cached) {
+          const baseDir = path.dirname(dbPath);
+          for (const r of cached) allResults.push({ ...r, baseDir });
+          db.close();
+          continue;
         }
-        db.close();
-        return results;
       }
-    } catch (hybridErr) {
-      hookLog('search', 'warn', `Hybrid search fallback to FTS5: ${hybridErr?.message || hybridErr}`);
-    }
-  }
 
-  // FTS5-only fallback: OR-joined single query (replaces O(docs×keywords) nested loop)
-  try {
-    const sanitized = keywords.map(sanitizeKeyword).filter(Boolean);
-    if (sanitized.length === 0) { db.close(); return results; }
-
-    const ftsQuery = fixVersionTokens(sanitized.join(' OR '));
-    const project = path.basename(process.cwd());
-
-    // v0.6.1: Project-scoped search with global fallback
-    let rows = db.prepare(
-      `SELECT path, slug, description, category, title, tags, rank
-       FROM mindlore_fts WHERE project = ? AND mindlore_fts MATCH ? ORDER BY rank LIMIT ?`
-    ).all(project, ftsQuery, MAX_RESULTS * 2);
-
-    if (rows.length === 0) {
-      rows = db.prepare(
-        `SELECT path, slug, description, category, title, tags, rank
-         FROM mindlore_fts WHERE mindlore_fts MATCH ? ORDER BY rank LIMIT ?`
-      ).all(ftsQuery, MAX_RESULTS * 2);
-    }
-
-    for (const r of rows) {
-      results.push({
-        path: r.path || '',
-        slug: r.slug,
-        description: r.description || '',
-        category: r.category || '',
-        title: r.title || '',
-        tags: r.tags || '',
-        headings: [],  // populated later in main() after slicing
-        hits: sanitized.length,
-        rank: r.rank,
-        baseDir,
+      const results = searchEngineMod.search(db, userMessage, {
+        project,
+        maxResults: effectiveMax,
+        synonyms,
       });
-    }
-  } catch (_err) {
-    // FTS5 query error — silently skip
-  } finally {
-    db.close();
-  }
-
-  return results;
-}
 
-/**
- * Search episodes via FTS5 mirror (type = 'episode').
- * Reuses an already-open DB handle — no extra sqlite3_open.
- */
-function searchEpisodesFts(db, keywords) {
-  try {
-    const ftsQuery = keywords.map(sanitizeKeyword).filter(Boolean).join(' OR ');
-    const rows = db.prepare(
-      "SELECT title, category, slug, tags FROM mindlore_fts WHERE type = 'episode' AND mindlore_fts MATCH ? LIMIT 2"
-    ).all(ftsQuery);
-
-    return rows.map(r => {
-      const tags = r.tags || '';
-      const kind = tags.split(',')[0]?.trim() || 'episode';
-      return `[episode] ${kind}: ${r.title || r.slug}`;
-    });
-  } catch (_err) {
-    return [];
-  }
-}
-
-function main() {
-  const userMessage = readHookStdin(['prompt', 'content', 'message', 'query']);
-  if (!userMessage || userMessage.length < MIN_QUERY_WORDS) return;
+      if (cache) cache.set(userMessage, results);
 
-  const dbPaths = getAllDbs();
-  if (dbPaths.length === 0) return;
-
-  const keywords = extractKeywords(userMessage);
-  if (keywords.length < MIN_QUERY_WORDS) return;
+      const baseDir = path.dirname(dbPath);
+      for (const r of results) {
+        allResults.push({ ...r, baseDir });
+      }
 
-  const allScores = [];
-  for (const dbPath of dbPaths) {
-    allScores.push(...searchDb(dbPath, keywords));
+      // Recall count inside loop — avoid reopening DB
+      try {
+        const txn = db.transaction(() => {
+          for (const r of results) incrementRecallCount(db, r.path);
+        });
+        txn();
+      } catch (_e) { /* graceful */ }
+    } catch (err) {
+      hookLog('search', 'warn', `Search error: ${err?.message || err}`);
+    } finally {
+      db.close();
+    }
   }
 
-  // Sort: most keyword hits first, then best rank
-  allScores.sort((a, b) => b.hits - a.hits || a.rank - b.rank);
-
-  // Deduplicate by full path (project version wins — appears first in sort)
+  // Deduplicate by full path
   const seen = new Set();
   const unique = [];
-  for (const r of allScores) {
+  for (const r of allResults) {
     const normalized = path.resolve(r.path);
     if (!seen.has(normalized)) {
       seen.add(normalized);
@@ -211,45 +119,30 @@ function main() {
     }
   }
 
+  // Sort by score descending, take top N
+  unique.sort((a, b) => b.score - a.score);
   const relevant = unique.slice(0, MAX_RESULTS);
   if (relevant.length === 0) return;
 
-  try {
-    const db = openDatabase(dbPaths[0]);
-    if (db) {
-      const txn = db.transaction(() => {
-        for (const r of relevant) incrementRecallCount(db, r.path);
-      });
-      txn();
-      db.close();
-    }
-  } catch (_e) { /* graceful — never block search output */ }
-
-  // Populate headings only for final results (avoid reading extra files)
-  for (const r of relevant) {
-    if (r.path && r.headings.length === 0 && fs.existsSync(r.path)) {
-      try {
-        const content = fs.readFileSync(r.path, 'utf8');
-        r.headings = extractHeadings(content, 3);
-      } catch (_err) { /* skip */ }
-    }
-  }
-
   // Token budget from config
-  const config = readConfig(path.dirname(dbPaths[0]));
   const budget = (config && config.tokenBudget) || {};
-  // Defaults match DEFAULT_TOKEN_BUDGET in scripts/lib/constants.ts
-  const perResultChars = ((budget.perResult || 500) * 4); // ~4 chars/token
+  const perResultChars = ((budget.perResult || 500) * 4);
   const totalChars = ((budget.searchResults || 1500) * 4);
 
-  // Build rich inject output
+  // Build output
   const output = [];
   let totalUsed = 0;
   for (const r of relevant) {
     if (totalUsed >= totalChars) break;
     const relativePath = path.relative(r.baseDir, r.path).replace(/\\/g, '/');
 
-    const headings = r.headings || [];
+    let headings = [];
+    const contentStr = r.content || '';
+    if (contentStr) {
+      try {
+        headings = extractHeadings(contentStr, 3);
+      } catch (_err) { /* skip */ }
+    }
 
     const category = r.category || path.dirname(relativePath).split('/')[0];
     const title = r.title || r.slug || path.basename(r.path, '.md');
@@ -263,32 +156,15 @@ function main() {
     output.push(truncated);
   }
 
-  // v0.4.0: Search episode mirrors in FTS5 (reuses searchDb's DB path, no extra open)
-  if (relevant.length < MAX_RESULTS) {
-    for (const dbPath of dbPaths) {
-      try {
-        const db = openDatabase(dbPath, { readonly: true });
-        if (!db) continue;
-        const episodeResults = searchEpisodesFts(db, keywords);
-        db.close();
-        if (episodeResults.length > 0) {
-          output.push(`[Mindlore Episodes]\n${episodeResults.join('\n')}`);
-          break;
-        }
-      } catch (_err) { /* skip */ }
-    }
-  }
-
   if (output.length > 0) {
     let outputStr = output.join('\n\n') + '\n';
 
-    const OFFLOAD_THRESHOLD = 10240; // 10KB
+    const OFFLOAD_THRESHOLD = 10240;
     if (outputStr.length > OFFLOAD_THRESHOLD) {
       const baseDir = path.dirname(dbPaths[0]);
       const tmpDir = path.join(baseDir, 'tmp');
       fs.mkdirSync(tmpDir, { recursive: true });
 
-      // Cleanup stale tmp files before writing new one (>1h old, keep max 20)
       try {
         const oneHourAgo = Date.now() - 3600000;
         const files = fs.readdirSync(tmpDir)

package/hooks/mindlore-session-end.cjs

@@ -13,7 +13,7 @@ const fs = require('fs');
 const path = require('path');
 const os = require('os');
 const { execSync, execFileSync, spawn } = require('child_process');
-const { findMindloreDir, globalDir, getProjectName, openDatabase, ensureEpisodesTable, hasEpisodesTable, insertBareEpisode, insertFtsRow, hookLog, SHARED_EXPORT_DIRS, resolveWin32Bin, withTelemetry } = require('./lib/mindlore-common.cjs');
+const { findMindloreDir, globalDir, getProjectName, openDatabase, ensureEpisodesTable, hasEpisodesTable, insertBareEpisode, insertFtsRow, hookLog, SHARED_EXPORT_DIRS, resolveWin32Bin, withTelemetry, getUnpromotedRawFiles } = require('./lib/mindlore-common.cjs');
 
 const EXPORT_DIRS = SHARED_EXPORT_DIRS;
 
@@ -82,6 +82,14 @@ if (process.argv.includes('--worker')) {
       }
     }, 'embed-trigger');
 
+    // Raw accumulation warning (moved from main to worker — off hot path)
+    await safeRunAsync(() => {
+      const unpromoted = getUnpromotedRawFiles(baseDir);
+      if (unpromoted.length >= 5) {
+        hookLog('session-end', 'info', `${unpromoted.length} raw files unpromoted`);
+      }
+    }, 'raw-check');
+
     // Obsidian + git-sync are independent — run in parallel
     await Promise.allSettled([
       safeRunAsync(() => syncObsidian(baseDir), 'obsidian'),
@@ -436,29 +444,25 @@ function syncObsidian(baseDir) {
     const destBase = path.join(vaultPath, 'mindlore');
     let exported = 0;
 
-    for (const dir of EXPORT_DIRS) {
-      const srcDir = path.join(baseDir, dir);
-      if (!fs.existsSync(srcDir)) continue;
-
-      const destDir = path.join(destBase, dir);
+    function walkAndExport(srcDir, destDir) {
+      if (!fs.existsSync(srcDir)) return;
       fs.mkdirSync(destDir, { recursive: true });
-
-      for (const file of fs.readdirSync(srcDir).filter(f => f.endsWith('.md') && !f.startsWith('_'))) {
-        if (exportMdFile(path.join(srcDir, file), path.join(destDir, file), convertFn)) exported++;
-      }
-
-      // One-level subdirectories (e.g. diary/mindlore/)
       for (const entry of fs.readdirSync(srcDir, { withFileTypes: true })) {
-        if (!entry.isDirectory() || entry.name.startsWith('_') || entry.name.startsWith('.')) continue;
-        const subSrc = path.join(srcDir, entry.name);
-        const subDest = path.join(destDir, entry.name);
-        fs.mkdirSync(subDest, { recursive: true });
-        for (const file of fs.readdirSync(subSrc).filter(f => f.endsWith('.md') && !f.startsWith('_'))) {
-          if (exportMdFile(path.join(subSrc, file), path.join(subDest, file), convertFn)) exported++;
+        if (entry.name.startsWith('_') || entry.name.startsWith('.')) continue;
+        const srcPath = path.join(srcDir, entry.name);
+        const destPath = path.join(destDir, entry.name);
+        if (entry.isDirectory()) {
+          walkAndExport(srcPath, destPath);
+        } else if (entry.isFile() && entry.name.endsWith('.md')) {
+          if (exportMdFile(srcPath, destPath, convertFn)) exported++;
         }
       }
     }
 
+    for (const dir of EXPORT_DIRS) {
+      walkAndExport(path.join(baseDir, dir), path.join(destBase, dir));
+    }
+
     for (const rootFile of ['INDEX.md', 'log.md']) {
       const srcPath = path.join(baseDir, rootFile);
       if (!fs.existsSync(srcPath)) continue;