mindlore 0.6.2 → 0.6.3

package/hooks/mindlore-search.cjs

@@ -4,206 +4,114 @@
 /**
  * mindlore-search — UserPromptSubmit hook
  *
- * Extracts keywords from user prompt, searches FTS5 with per-keyword scoring,
- * injects top results with description + headings (matching old knowledge system quality).
+ * Thin wrapper over search-engine.ts pipeline.
+ * Extracts keywords from user prompt, delegates search to modular engine,
+ * injects top results with description + headings.
  */
 
 const fs = require('fs');
 const path = require('path');
-const { getAllDbs, openDatabase, extractHeadings, readHookStdin, extractKeywords, sanitizeKeyword, readConfig, loadSqliteVecCjs, hasVecTableCjs, hookLog, incrementRecallCount, getDaemonPortFile, withTelemetry, fixVersionTokens } = require('./lib/mindlore-common.cjs');
-
-const { execFileSync } = require('child_process');
+const { getAllDbs, openDatabase, extractHeadings, readHookStdin, readConfig, hookLog, incrementRecallCount, withTelemetry } = require('./lib/mindlore-common.cjs');
 
 const MAX_RESULTS = 3;
 const MIN_QUERY_WORDS = 3;
 
-// Try to load hybrid search module (built TS)
-let hybridSearchMod;
+let searchEngineMod;
 try {
-  hybridSearchMod = require('../dist/scripts/lib/hybrid-search.js');
+  searchEngineMod = require('../dist/scripts/lib/search-engine.js');
 } catch (_err) {
-  // hybrid-search not built yet — pure FTS5 mode
+  // search-engine not built yet
 }
 
-// v0.5.5: Request embedding from daemon via execFileSync bridge
-function requestEmbeddingSync(query) {
-  try {
-    const portFile = getDaemonPortFile();
-    if (!fs.existsSync(portFile)) return null;
-    const clientScript = path.join(__dirname, '..', 'scripts', 'lib', 'daemon-client.js');
-    if (!fs.existsSync(clientScript)) return null;
-    const result = execFileSync(process.execPath, [clientScript, portFile, query, '300'], {
-      timeout: 500, encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true,
-    });
-    const parsed = JSON.parse(result.trim());
-    return parsed.type === 'embedding' ? parsed.embedding : null;
-  } catch {
-    return null;
-  }
+let SearchCacheMod;
+try {
+  SearchCacheMod = require('../dist/scripts/lib/search-cache.js');
+} catch (_err) {
+  // search-cache not built yet
 }
 
-/**
- * Search a single DB and return scored results with their baseDir.
- */
-function searchDb(dbPath, keywords) {
-  const baseDir = path.dirname(dbPath);
-  const db = openDatabase(dbPath, { readonly: true });
-  if (!db) return [];
-  const results = [];
+function main() {
+  const userMessage = readHookStdin(['prompt', 'content', 'message', 'query']);
+  if (!userMessage || userMessage.length < MIN_QUERY_WORDS) return;
 
-  // v0.5.0: Try hybrid search with synonym expansion (no embedding — hooks are sync)
-  if (!hybridSearchMod) {
-    hookLog('search', 'info', 'No hybridSearchMod — FTS5-only mode');
-  }
-  if (hybridSearchMod && loadSqliteVecCjs(db) && hasVecTableCjs(db)) {
-    try {
-      const config = readConfig(baseDir);
-      const synonyms = (config && config.synonyms) ? config.synonyms : {};
+  const dbPaths = getAllDbs();
+  if (dbPaths.length === 0) return;
 
-      // Expand keywords with synonyms
-      const expandedTerms = keywords.slice();
-      for (const kw of keywords) {
-        const lower = kw.toLowerCase();
-        if (synonyms[lower]) {
-          expandedTerms.push(...synonyms[lower]);
-        }
-      }
+  if (!searchEngineMod) {
+    hookLog('search', 'warn', 'search-engine module not available — skipping');
+    return;
+  }
 
-      // v0.5.5: Try to get queryEmbedding from daemon
-      let queryEmbedding = null;
-      try {
-        queryEmbedding = requestEmbeddingSync(expandedTerms.join(' '));
-        if (!queryEmbedding) {
-          hookLog('search', 'info', 'Daemon not available — FTS5-only hybrid mode');
-        }
-      } catch {
-        hookLog('search', 'info', 'Daemon connection failed — FTS5-only hybrid mode');
-      }
+  const project = path.basename(process.cwd());
+  const config = readConfig(path.dirname(dbPaths[0]));
+  const synonyms = (config && config.synonyms) ? config.synonyms : {};
 
-      const fusedResults = hybridSearchMod.hybridSearch(db, expandedTerms.join(' '), {
-        maxResults: MAX_RESULTS,
-        project: path.basename(process.cwd()),
-        queryEmbedding,
-      });
+  // Read session_id from stdin for throttling
+  let sessionId;
+  try {
+    const stdinData = JSON.parse(process.env.CLAUDE_HOOK_STDIN || '{}');
+    sessionId = stdinData.session_id || 'unknown';
+  } catch (_) {
+    sessionId = 'unknown';
+  }
 
-      if (fusedResults.length > 0) {
-        for (const r of fusedResults) {
-          const filePath = r.path || '';
-          let headings = [];
-          if (filePath) {
-            try {
-              const content = fs.readFileSync(filePath, 'utf8');
-              headings = extractHeadings(content, 3);
-            } catch (_err) { /* file may have been deleted */ }
-          }
-          results.push({
-            path: filePath,
-            slug: r.slug,
-            description: r.description || '',
-            category: r.category || '',
-            title: r.title || '',
-            tags: r.tags || '',
-            headings,
-            hits: 1,
-            rank: r.score,
-            baseDir,
-          });
+  const allResults = [];
+  for (const dbPath of dbPaths) {
+    const db = openDatabase(dbPath);
+    if (!db) continue;
+    try {
+      // Cache + throttle
+      let cache;
+      let effectiveMax = MAX_RESULTS;
+      if (SearchCacheMod) {
+        cache = new SearchCacheMod.SearchCache(db, { ttlMs: 300000 });
+        const callCount = cache.incrementCallCount(sessionId);
+        effectiveMax = cache.getMaxResults(callCount);
+        if (effectiveMax === 0) {
+          hookLog('search', 'info', `Throttled (call #${callCount})`);
+          db.close();
+          continue;
+        }
+        const cached = cache.get(userMessage);
+        if (cached) {
+          const baseDir = path.dirname(dbPath);
+          for (const r of cached) allResults.push({ ...r, baseDir });
+          db.close();
+          continue;
         }
-        db.close();
-        return results;
       }
-    } catch (hybridErr) {
-      hookLog('search', 'warn', `Hybrid search fallback to FTS5: ${hybridErr?.message || hybridErr}`);
-    }
-  }
 
-  // FTS5-only fallback: OR-joined single query (replaces O(docs×keywords) nested loop)
-  try {
-    const sanitized = keywords.map(sanitizeKeyword).filter(Boolean);
-    if (sanitized.length === 0) { db.close(); return results; }
-
-    const ftsQuery = fixVersionTokens(sanitized.join(' OR '));
-    const project = path.basename(process.cwd());
-
-    // v0.6.1: Project-scoped search with global fallback
-    let rows = db.prepare(
-      `SELECT path, slug, description, category, title, tags, rank
-       FROM mindlore_fts WHERE project = ? AND mindlore_fts MATCH ? ORDER BY rank LIMIT ?`
-    ).all(project, ftsQuery, MAX_RESULTS * 2);
-
-    if (rows.length === 0) {
-      rows = db.prepare(
-        `SELECT path, slug, description, category, title, tags, rank
-         FROM mindlore_fts WHERE mindlore_fts MATCH ? ORDER BY rank LIMIT ?`
-      ).all(ftsQuery, MAX_RESULTS * 2);
-    }
-
-    for (const r of rows) {
-      results.push({
-        path: r.path || '',
-        slug: r.slug,
-        description: r.description || '',
-        category: r.category || '',
-        title: r.title || '',
-        tags: r.tags || '',
-        headings: [],  // populated later in main() after slicing
-        hits: sanitized.length,
-        rank: r.rank,
-        baseDir,
+      const results = searchEngineMod.search(db, userMessage, {
+        project,
+        maxResults: effectiveMax,
+        synonyms,
       });
-    }
-  } catch (_err) {
-    // FTS5 query error — silently skip
-  } finally {
-    db.close();
-  }
-
-  return results;
-}
 
-/**
- * Search episodes via FTS5 mirror (type = 'episode').
- * Reuses an already-open DB handle — no extra sqlite3_open.
- */
-function searchEpisodesFts(db, keywords) {
-  try {
-    const ftsQuery = keywords.map(sanitizeKeyword).filter(Boolean).join(' OR ');
-    const rows = db.prepare(
-      "SELECT title, category, slug, tags FROM mindlore_fts WHERE type = 'episode' AND mindlore_fts MATCH ? LIMIT 2"
-    ).all(ftsQuery);
-
-    return rows.map(r => {
-      const tags = r.tags || '';
-      const kind = tags.split(',')[0]?.trim() || 'episode';
-      return `[episode] ${kind}: ${r.title || r.slug}`;
-    });
-  } catch (_err) {
-    return [];
-  }
-}
-
-function main() {
-  const userMessage = readHookStdin(['prompt', 'content', 'message', 'query']);
-  if (!userMessage || userMessage.length < MIN_QUERY_WORDS) return;
+      if (cache) cache.set(userMessage, results);
 
-  const dbPaths = getAllDbs();
-  if (dbPaths.length === 0) return;
-
-  const keywords = extractKeywords(userMessage);
-  if (keywords.length < MIN_QUERY_WORDS) return;
+      const baseDir = path.dirname(dbPath);
+      for (const r of results) {
+        allResults.push({ ...r, baseDir });
+      }
 
-  const allScores = [];
-  for (const dbPath of dbPaths) {
-    allScores.push(...searchDb(dbPath, keywords));
+      // Recall count inside loop — avoid reopening DB
+      try {
+        const txn = db.transaction(() => {
+          for (const r of results) incrementRecallCount(db, r.path);
+        });
+        txn();
+      } catch (_e) { /* graceful */ }
+    } catch (err) {
+      hookLog('search', 'warn', `Search error: ${err?.message || err}`);
+    } finally {
+      db.close();
+    }
   }
 
-  // Sort: most keyword hits first, then best rank
-  allScores.sort((a, b) => b.hits - a.hits || a.rank - b.rank);
-
-  // Deduplicate by full path (project version wins — appears first in sort)
+  // Deduplicate by full path
   const seen = new Set();
   const unique = [];
-  for (const r of allScores) {
+  for (const r of allResults) {
     const normalized = path.resolve(r.path);
     if (!seen.has(normalized)) {
       seen.add(normalized);
@@ -211,45 +119,30 @@ function main() {
     }
   }
 
+  // Sort by score descending, take top N
+  unique.sort((a, b) => b.score - a.score);
   const relevant = unique.slice(0, MAX_RESULTS);
   if (relevant.length === 0) return;
 
-  try {
-    const db = openDatabase(dbPaths[0]);
-    if (db) {
-      const txn = db.transaction(() => {
-        for (const r of relevant) incrementRecallCount(db, r.path);
-      });
-      txn();
-      db.close();
-    }
-  } catch (_e) { /* graceful — never block search output */ }
-
-  // Populate headings only for final results (avoid reading extra files)
-  for (const r of relevant) {
-    if (r.path && r.headings.length === 0 && fs.existsSync(r.path)) {
-      try {
-        const content = fs.readFileSync(r.path, 'utf8');
-        r.headings = extractHeadings(content, 3);
-      } catch (_err) { /* skip */ }
-    }
-  }
-
   // Token budget from config
-  const config = readConfig(path.dirname(dbPaths[0]));
   const budget = (config && config.tokenBudget) || {};
-  // Defaults match DEFAULT_TOKEN_BUDGET in scripts/lib/constants.ts
-  const perResultChars = ((budget.perResult || 500) * 4); // ~4 chars/token
+  const perResultChars = ((budget.perResult || 500) * 4);
   const totalChars = ((budget.searchResults || 1500) * 4);
 
-  // Build rich inject output
+  // Build output
   const output = [];
   let totalUsed = 0;
   for (const r of relevant) {
     if (totalUsed >= totalChars) break;
     const relativePath = path.relative(r.baseDir, r.path).replace(/\\/g, '/');
 
-    const headings = r.headings || [];
+    let headings = [];
+    const contentStr = r.content || '';
+    if (contentStr) {
+      try {
+        headings = extractHeadings(contentStr, 3);
+      } catch (_err) { /* skip */ }
+    }
 
     const category = r.category || path.dirname(relativePath).split('/')[0];
     const title = r.title || r.slug || path.basename(r.path, '.md');
@@ -263,32 +156,15 @@ function main() {
     output.push(truncated);
   }
 
-  // v0.4.0: Search episode mirrors in FTS5 (reuses searchDb's DB path, no extra open)
-  if (relevant.length < MAX_RESULTS) {
-    for (const dbPath of dbPaths) {
-      try {
-        const db = openDatabase(dbPath, { readonly: true });
-        if (!db) continue;
-        const episodeResults = searchEpisodesFts(db, keywords);
-        db.close();
-        if (episodeResults.length > 0) {
-          output.push(`[Mindlore Episodes]\n${episodeResults.join('\n')}`);
-          break;
-        }
-      } catch (_err) { /* skip */ }
-    }
-  }
-
   if (output.length > 0) {
     let outputStr = output.join('\n\n') + '\n';
 
-    const OFFLOAD_THRESHOLD = 10240; // 10KB
+    const OFFLOAD_THRESHOLD = 10240;
     if (outputStr.length > OFFLOAD_THRESHOLD) {
       const baseDir = path.dirname(dbPaths[0]);
       const tmpDir = path.join(baseDir, 'tmp');
       fs.mkdirSync(tmpDir, { recursive: true });
 
-      // Cleanup stale tmp files before writing new one (>1h old, keep max 20)
       try {
         const oneHourAgo = Date.now() - 3600000;
         const files = fs.readdirSync(tmpDir)

package/hooks/mindlore-session-end.cjs

@@ -13,7 +13,7 @@ const fs = require('fs');
 const path = require('path');
 const os = require('os');
 const { execSync, execFileSync, spawn } = require('child_process');
-const { findMindloreDir, globalDir, getProjectName, openDatabase, ensureEpisodesTable, hasEpisodesTable, insertBareEpisode, insertFtsRow, hookLog, SHARED_EXPORT_DIRS, resolveWin32Bin, withTelemetry } = require('./lib/mindlore-common.cjs');
+const { findMindloreDir, globalDir, getProjectName, openDatabase, ensureEpisodesTable, hasEpisodesTable, insertBareEpisode, insertFtsRow, hookLog, SHARED_EXPORT_DIRS, resolveWin32Bin, withTelemetry, getUnpromotedRawFiles } = require('./lib/mindlore-common.cjs');
 
 const EXPORT_DIRS = SHARED_EXPORT_DIRS;
 
@@ -82,6 +82,14 @@ if (process.argv.includes('--worker')) {
       }
     }, 'embed-trigger');
 
+    // Raw accumulation warning (moved from main to worker — off hot path)
+    await safeRunAsync(() => {
+      const unpromoted = getUnpromotedRawFiles(baseDir);
+      if (unpromoted.length >= 5) {
+        hookLog('session-end', 'info', `${unpromoted.length} raw files unpromoted`);
+      }
+    }, 'raw-check');
+
     // Obsidian + git-sync are independent — run in parallel
     await Promise.allSettled([
       safeRunAsync(() => syncObsidian(baseDir), 'obsidian'),
@@ -221,20 +229,6 @@ function main() {
     fs.appendFileSync(logPath, logEntry, 'utf8');
   }
 
-  // Raw accumulation warning
-  try {
-    const rawDir = path.join(baseDir, 'raw');
-    const sourcesDir = path.join(baseDir, 'sources');
-    if (fs.existsSync(rawDir) && fs.existsSync(sourcesDir)) {
-      const rawFiles = fs.readdirSync(rawDir).filter(f => f.endsWith('.md'));
-      const sourceFiles = new Set(fs.readdirSync(sourcesDir).filter(f => f.endsWith('.md')));
-      const unpromoted = rawFiles.filter(f => !sourceFiles.has(f)).length;
-      if (unpromoted >= 5) {
-        process.stdout.write(`[Mindlore] ${unpromoted} raw dosya promote bekliyor — \`/mindlore-maintain triage\` ile listele\n`);
-      }
-    }
-  } catch (_err) { /* graceful skip */ }
-
   // Heavy ops: detach into child process so CC can exit immediately.
   // Fixes "Hook cancelled" when CC kills the hook before completion.
   // See: https://github.com/anthropics/claude-code/issues/41577

package/hooks/mindlore-session-focus.cjs

@@ -10,38 +10,22 @@
 
 const fs = require('fs');
 const path = require('path');
-const { findMindloreDir, readConfig, openDatabase, hasEpisodesTable, querySupersededChains, formatSupersededChains, hookLog, getProjectName, parseFrontmatter, withTelemetry, withTimeoutDb } = require('./lib/mindlore-common.cjs');
-
-function isCorruptionError(err) {
-  const code = err?.code ?? '';
-  const msg = String(err?.message ?? err);
-  return code === 'SQLITE_CORRUPT' || code === 'SQLITE_NOTADB' || /corrupt|malformed/i.test(msg);
-}
-
-function recoverCorruptDb(db, dbPath, reason) {
-  try { db.close(); } catch { /* already closed */ }
-  const bakPath = dbPath + '.corrupt.bak';
-  try { fs.copyFileSync(dbPath, bakPath); } catch { /* best effort */ }
-  try { fs.unlinkSync(dbPath); } catch { /* best effort */ }
-  hookLog('session-focus', 'warn', reason);
-}
+const { findMindloreDir, readConfig, openDatabase, hasEpisodesTable, querySupersededChains, formatSupersededChains, hookLog, getProjectName, parseFrontmatter, withTelemetry, withTimeoutDb, listSnapshots, isCorruptionError, recoverCorruptDb } = require('./lib/mindlore-common.cjs');
 
 function tryOpenDb(dbPath) {
   return openDatabase(dbPath, { readonly: true });
 }
 
-function loadDbContent(db, baseDir, config, output, timings) {
+function loadDbContent(db, baseDir, config, output, timings, latestDeltaContent) {
   // Session payload: Session summary, Decisions, Friction, Learnings
   const tPayload = Date.now();
   try {
     const { buildSessionPayload } = require('../dist/scripts/lib/session-payload.js');
     const project = path.basename(process.cwd());
     const payloadBudget = config?.tokenBudget?.sessionInject ?? 2000;
-    const payload = buildSessionPayload(db, baseDir, project, payloadBudget);
-    if (!payload.skipInjection) {
-      for (const section of payload.sections) {
-        output.push(`[Mindlore ${section.label}]\n${section.content}`);
-      }
+    const payload = buildSessionPayload(db, baseDir, project, payloadBudget, latestDeltaContent);
+    for (const section of payload.sections) {
+      output.push(`[Mindlore ${section.label}]\n${section.content}`);
     }
   } catch (_payloadErr) {
     // Session payload is optional — don't break session start
@@ -108,16 +92,17 @@ function main() {
   // Inject latest delta + reflect trigger (single readdirSync)
   const tDiary = Date.now();
   const diaryDir = path.join(baseDir, 'diary');
+  let latestDeltaContent = undefined;
   if (fs.existsSync(diaryDir)) {
     try {
-      const diaryFiles = fs.readdirSync(diaryDir).filter(f => f.startsWith('delta-') && f.endsWith('.md'));
+      const diaryFiles = listSnapshots(diaryDir).filter(f => f.startsWith('delta-'));
 
       if (diaryFiles.length > 0) {
-        const sorted = [...diaryFiles].sort();
-        const latestName = sorted[sorted.length - 1];
+        const latestName = diaryFiles[diaryFiles.length - 1];
         const latestPath = path.join(diaryDir, latestName);
         sourceChars += fs.statSync(latestPath).size;
         const deltaContent = fs.readFileSync(latestPath, 'utf8').trim();
+        latestDeltaContent = deltaContent;
         const { meta } = parseFrontmatter(deltaContent);
         const deltaProject = meta.project || null;
         const currentProject = getProjectName();
@@ -163,10 +148,10 @@ function main() {
 
     if (db) {
       try {
-        loadDbContent(db, baseDir, config, output, timings);
+        loadDbContent(db, baseDir, config, output, timings, latestDeltaContent);
       } catch (err) {
         if (isCorruptionError(err)) {
-          recoverCorruptDb(db, dbPath, `Corrupt DB detected during query: ${err?.message ?? err}`);
+          recoverCorruptDb(db, dbPath, 'session-focus');
         }
       } finally {
         try { db.close(); } catch { /* already closed by recovery */ }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mindlore",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "AI-native knowledge system for Claude Code",
   "type": "commonjs",
   "bin": {
@@ -36,7 +36,7 @@
     "wiki"
   ],
   "author": "omrfc",
-  "license": "MIT",
+  "license": "AGPL-3.0-only",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mindlore/mindlore.git"

package/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "mindlore",
   "description": "AI-native knowledge system for Claude Code. Persistent, searchable, evolving knowledge base with FTS5.",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "skills": [
     {
       "name": "mindlore-ingest",

package/templates/config.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.6.2",
+  "version": "0.6.3",
   "models": {
     "ingest": "haiku",
     "evolve": "sonnet",