mindlore 0.4.3 → 0.5.1

package/hooks/mindlore-search.cjs

@@ -10,11 +10,18 @@
 
 const fs = require('fs');
 const path = require('path');
-const { getAllDbs, requireDatabase, extractHeadings, readHookStdin, extractKeywords } = require('./lib/mindlore-common.cjs');
+const { getAllDbs, requireDatabase, extractHeadings, readHookStdin, extractKeywords, sanitizeKeyword, readConfig, loadSqliteVecCjs, hasVecTableCjs, hookLog } = require('./lib/mindlore-common.cjs');
 
 const MAX_RESULTS = 3;
 const MIN_QUERY_WORDS = 3;
-const MIN_KEYWORD_HITS = 2;
+
+// Try to load hybrid search module (built TS)
+let hybridSearchMod;
+try {
+  hybridSearchMod = require('../dist/scripts/lib/hybrid-search.js');
+} catch (_err) {
+  // hybrid-search not built yet — pure FTS5 mode
+}
 
 /**
  * Search a single DB and return scored results with their baseDir.
@@ -24,35 +31,79 @@ function searchDb(dbPath, keywords, Database) {
   const db = new Database(dbPath, { readonly: true });
   const results = [];
 
-  try {
-    const allPaths = db.prepare('SELECT DISTINCT path FROM mindlore_fts').all();
-    const matchStmt = db.prepare('SELECT rank FROM mindlore_fts WHERE path = ? AND mindlore_fts MATCH ?');
-    const metaStmt = db.prepare(
-      'SELECT slug, description, category, title, tags FROM mindlore_fts WHERE path = ?'
-    );
-
-    for (const row of allPaths) {
-      let hits = 0;
-      let totalRank = 0;
+  // v0.5.0: Try hybrid search with synonym expansion (no embedding — hooks are sync)
+  if (hybridSearchMod && loadSqliteVecCjs(db) && hasVecTableCjs(db)) {
+    try {
+      const config = readConfig(baseDir);
+      const synonyms = (config && config.synonyms) ? config.synonyms : {};
 
+      // Expand keywords with synonyms
+      const expandedTerms = keywords.slice();
       for (const kw of keywords) {
-        try {
-          const sanitized = kw.replace(/["*(){}[\]^~:]/g, '');
-          if (!sanitized) continue;
-          const r = matchStmt.get(row.path, '"' + sanitized + '"');
-          if (r) {
-            hits++;
-            totalRank += r.rank;
-          }
-        } catch (_err) {
-          // FTS5 query error for this keyword — skip
+        const lower = kw.toLowerCase();
+        if (synonyms[lower]) {
+          expandedTerms.push(...synonyms[lower]);
         }
       }
 
-      if (hits >= MIN_KEYWORD_HITS) {
-        const meta = metaStmt.get(row.path) || {};
-        results.push({ path: row.path, hits, totalRank, baseDir, meta });
+      const fusedResults = hybridSearchMod.hybridSearch(db, expandedTerms.join(' '), {
+        maxResults: MAX_RESULTS,
+        project: path.basename(process.cwd()),
+      });
+
+      if (fusedResults.length > 0) {
+        for (const r of fusedResults) {
+          const filePath = r.path || '';
+          let headings = [];
+          if (filePath && fs.existsSync(filePath)) {
+            const content = fs.readFileSync(filePath, 'utf8');
+            headings = extractHeadings(content, 3);
+          }
+          results.push({
+            path: filePath,
+            slug: r.slug,
+            description: r.description || '',
+            category: r.category || '',
+            title: r.title || '',
+            tags: r.tags || '',
+            headings,
+            hits: 1,
+            rank: r.score,
+            baseDir,
+          });
+        }
+        db.close();
+        return results;
       }
+    } catch (_err) {
+      // Hybrid search failed — fall through to FTS5
+    }
+  }
+
+  // FTS5-only fallback: OR-joined single query (replaces O(docs×keywords) nested loop)
+  try {
+    const sanitized = keywords.map(sanitizeKeyword).filter(Boolean);
+    if (sanitized.length === 0) { db.close(); return results; }
+
+    const ftsQuery = sanitized.join(' OR ');
+    const rows = db.prepare(
+      `SELECT path, slug, description, category, title, tags, rank
+       FROM mindlore_fts WHERE mindlore_fts MATCH ? ORDER BY rank LIMIT ?`
+    ).all(ftsQuery, MAX_RESULTS * 2);
+
+    for (const r of rows) {
+      results.push({
+        path: r.path || '',
+        slug: r.slug,
+        description: r.description || '',
+        category: r.category || '',
+        title: r.title || '',
+        tags: r.tags || '',
+        headings: [],  // populated later in main() after slicing
+        hits: sanitized.length,
+        rank: r.rank,
+        baseDir,
+      });
     }
   } catch (_err) {
     // FTS5 query error — silently skip
@@ -69,7 +120,7 @@ function searchDb(dbPath, keywords, Database) {
  */
 function searchEpisodesFts(db, keywords) {
   try {
-    const ftsQuery = keywords.map(kw => '"' + kw.replace(/["*(){}[\]^~:]/g, '') + '"').filter(q => q !== '""').join(' OR ');
+    const ftsQuery = keywords.map(sanitizeKeyword).filter(Boolean).join(' OR ');
     const rows = db.prepare(
       "SELECT title, category, slug, tags FROM mindlore_fts WHERE type = 'episode' AND mindlore_fts MATCH ? LIMIT 2"
     ).all(ftsQuery);
@@ -119,17 +170,32 @@ function main() {
   const relevant = unique.slice(0, MAX_RESULTS);
   if (relevant.length === 0) return;
 
+  // Populate headings only for final results (avoid reading extra files)
+  for (const r of relevant) {
+    if (r.path && r.headings.length === 0 && fs.existsSync(r.path)) {
+      try {
+        const content = fs.readFileSync(r.path, 'utf8');
+        r.headings = extractHeadings(content, 3);
+      } catch (_err) { /* skip */ }
+    }
+  }
+
+  // Token budget from config
+  const config = readConfig(path.dirname(dbPaths[0]));
+  const budget = (config && config.tokenBudget) || {};
+  // Defaults match DEFAULT_TOKEN_BUDGET in scripts/lib/constants.ts
+  const perResultChars = ((budget.perResult || 500) * 4); // ~4 chars/token
+  const totalChars = ((budget.searchResults || 1500) * 4);
+
   // Build rich inject output
   const output = [];
+  let totalUsed = 0;
   for (const r of relevant) {
+    if (totalUsed >= totalChars) break;
     const meta = r.meta || {};
     const relativePath = path.relative(r.baseDir, r.path).replace(/\\/g, '/');
 
-    let headings = [];
-    if (fs.existsSync(r.path)) {
-      const content = fs.readFileSync(r.path, 'utf8');
-      headings = extractHeadings(content, 5);
-    }
+    const headings = r.headings || [];
 
     const category = meta.category || path.dirname(relativePath).split('/')[0];
     const title = meta.title || meta.slug || path.basename(r.path, '.md');
@@ -137,9 +203,10 @@ function main() {
 
     const headingStr = headings.length > 0 ? `\nBasliklar: ${headings.join(', ')}` : '';
     const tagsStr = meta.tags ? `\nTags: ${meta.tags}` : '';
-    output.push(
-      `[Mindlore: ${category}/${title}] ${description}\nDosya: ${relativePath}${tagsStr}${headingStr}`
-    );
+    const entry = `[Mindlore: ${category}/${title}] ${description}\nDosya: ${relativePath}${tagsStr}${headingStr}`;
+    const truncated = entry.slice(0, perResultChars);
+    totalUsed += truncated.length;
+    output.push(truncated);
   }
 
   // v0.4.0: Search episode mirrors in FTS5 (reuses searchDb's DB path, no extra open)
@@ -162,4 +229,4 @@ function main() {
   }
 }
 
-main();
+try { main(); } catch (err) { hookLog('search', 'error', err?.message ?? String(err)); }

package/hooks/mindlore-session-end.cjs

@@ -13,22 +13,48 @@ const fs = require('fs');
 const path = require('path');
 const os = require('os');
 const { execSync, spawn } = require('child_process');
-const { findMindloreDir, globalDir, getProjectName, openDatabase, ensureEpisodesTable, hasEpisodesTable, insertBareEpisode, insertFtsRow } = require('./lib/mindlore-common.cjs');
+const { findMindloreDir, globalDir, getProjectName, openDatabase, ensureEpisodesTable, hasEpisodesTable, insertBareEpisode, insertFtsRow, hookLog, SHARED_EXPORT_DIRS, resolveWin32Bin } = require('./lib/mindlore-common.cjs');
+
+const EXPORT_DIRS = SHARED_EXPORT_DIRS;
 
 // --worker mode: heavy ops run in detached child process (survives parent exit)
 if (process.argv.includes('--worker')) {
+  hookLog('session-end', 'info', 'worker started, pid=' + process.pid);
   const dataPath = process.argv[process.argv.indexOf('--worker') + 1];
+  let payload;
   try {
     const raw = fs.readFileSync(dataPath, 'utf8');
-    fs.unlinkSync(dataPath); // cleanup temp file before any processing
-    const { baseDir, project, commits, changedFiles, reads } = JSON.parse(raw);
-    writeBareEpisode(baseDir, project, commits, changedFiles, reads);
-    syncObsidian(baseDir);
-    syncGlobalRepo();
+    fs.unlinkSync(dataPath);
+    payload = JSON.parse(raw);
   } catch (_err) {
-    // Graceful fail — worker errors are silent
+    hookLog('session-end', 'error', 'payload read failed: ' + (_err?.message ?? _err));
+    process.exit(0);
   }
-  process.exit(0);
+  const { baseDir, project, commits, changedFiles, reads } = payload;
+
+  async function safeRunAsync(fn, label) {
+    try {
+      await fn();
+      hookLog('session-end', 'info', label + ' OK');
+    } catch (e) {
+      hookLog('session-end', 'error', label + ' FAIL: ' + e?.message);
+    }
+  }
+
+  (async () => {
+    // Episode writes share DB — run sequentially first
+    await safeRunAsync(() => writeBareEpisode(baseDir, project, commits, changedFiles, reads), 'episode');
+    await safeRunAsync(() => writeEpisodeFile(baseDir, project, commits, changedFiles, reads), 'episode-file');
+
+    // Obsidian + git-sync are independent — run in parallel
+    await Promise.allSettled([
+      safeRunAsync(() => syncObsidian(baseDir), 'obsidian'),
+      safeRunAsync(() => syncGlobalRepo(), 'git-sync'),
+    ]);
+
+    hookLog('session-end', 'info', 'worker done');
+    process.exit(0);
+  })();
 }
 
 function formatDate(date) {
@@ -47,9 +73,10 @@ function formatDate(date) {
 function getRecentGitInfo() {
   try {
     // --name-only includes file names after each commit entry
-    const raw = execSync('git log --oneline -5 --name-only 2>/dev/null', {
+    const raw = execSync('git log --oneline -5 --name-only', {
       encoding: 'utf8',
       timeout: 5000,
+      stdio: ['pipe', 'pipe', 'pipe'],
     }).trim();
     if (!raw) return { commits: [], changedFiles: [] };
 
@@ -164,7 +191,11 @@ function main() {
     const workerData = JSON.stringify({ baseDir, project, commits, changedFiles, reads });
     const tmpFile = path.join(os.tmpdir(), `mindlore-worker-${Date.now()}.json`);
     fs.writeFileSync(tmpFile, workerData, 'utf8');
-    const child = spawn(process.execPath, [__filename, '--worker', tmpFile], {
+    // Use system node instead of process.execPath — CC's embedded Node
+    // may not work as a standalone binary for detached worker processes.
+    // Resolve full path to avoid shell:true deprecation warning on Windows.
+    const nodeBin = resolveWin32Bin('node');
+    const child = spawn(nodeBin, [__filename, '--worker', tmpFile], {
       detached: true,
       stdio: 'ignore',
       cwd: process.cwd(),
@@ -173,6 +204,7 @@ function main() {
   } catch (_err) {
     // Fallback: run inline if spawn fails
     writeBareEpisode(baseDir, project, commits, changedFiles, reads);
+    writeEpisodeFile(baseDir, project, commits, changedFiles, reads);
     syncObsidian(baseDir);
     syncGlobalRepo();
   }
@@ -247,11 +279,60 @@ function writeBareEpisode(baseDir, project, commits, changedFiles, reads) {
 
     db.close();
   } catch (err) {
-    process.stderr.write(`[mindlore] episode write failed: ${err?.message ?? err}\n`);
+    hookLog('session-end', 'error', `episode write failed: ${err?.message ?? err}`);
   }
 }
 
-const EXPORT_DIRS = ['analyses', 'decisions', 'diary', 'raw', 'sources', 'domains', 'connections', 'insights', 'learnings'];
+/**
+ * Write episode as .md file to diary/{project}/ for human-readable browsing.
+ * Complements the DB episode — same content, different medium.
+ */
+function writeEpisodeFile(baseDir, project, commits, changedFiles, reads) {
+  const projDir = path.join(baseDir, 'diary', project || 'unknown');
+  if (!fs.existsSync(projDir)) fs.mkdirSync(projDir, { recursive: true });
+
+  const now = new Date();
+  const ts = formatDate(now);
+  const filePath = path.join(projDir, `episode-${ts}.md`);
+  if (fs.existsSync(filePath)) return; // idempotent
+
+  const lines = [
+    '---',
+    `slug: episode-${ts}`,
+    'type: episode',
+    `date: ${now.toISOString().slice(0, 10)}`,
+    `project: ${project || 'unknown'}`,
+    '---',
+    '',
+    `# Episode — ${ts}`,
+    '',
+  ];
+
+  if (commits.length > 0) {
+    lines.push('## Commits');
+    for (const c of commits) lines.push(`- ${c}`);
+    lines.push('');
+  }
+
+  if (changedFiles.length > 0) {
+    lines.push('## Changed Files');
+    for (const f of changedFiles) lines.push(`- ${f}`);
+    lines.push('');
+  }
+
+  if (reads) {
+    lines.push('## Read Stats');
+    lines.push(`- ${reads.count} files read, ${reads.repeats} repeated`);
+    lines.push('');
+  }
+
+  if (commits.length === 0 && changedFiles.length === 0) {
+    lines.push('_Read-only session — no commits or file changes._');
+    lines.push('');
+  }
+
+  fs.writeFileSync(filePath, lines.join('\n'), 'utf8');
+}
 
 /**
  * Load obsidian-helpers from compiled dist (single source of truth for wikilink conversion).
@@ -316,27 +397,40 @@ function syncObsidian(baseDir) {
       if (!fs.existsSync(srcDir)) continue;
 
       const destDir = path.join(destBase, dir);
-      if (!fs.existsSync(destDir)) fs.mkdirSync(destDir, { recursive: true });
+      fs.mkdirSync(destDir, { recursive: true });
 
       for (const file of fs.readdirSync(srcDir).filter(f => f.endsWith('.md') && !f.startsWith('_'))) {
         if (exportMdFile(path.join(srcDir, file), path.join(destDir, file), convertFn)) exported++;
       }
+
+      // One-level subdirectories (e.g. diary/mindlore/)
+      for (const entry of fs.readdirSync(srcDir, { withFileTypes: true })) {
+        if (!entry.isDirectory() || entry.name.startsWith('_') || entry.name.startsWith('.')) continue;
+        const subSrc = path.join(srcDir, entry.name);
+        const subDest = path.join(destDir, entry.name);
+        fs.mkdirSync(subDest, { recursive: true });
+        for (const file of fs.readdirSync(subSrc).filter(f => f.endsWith('.md') && !f.startsWith('_'))) {
+          if (exportMdFile(path.join(subSrc, file), path.join(subDest, file), convertFn)) exported++;
+        }
+      }
     }
 
     for (const rootFile of ['INDEX.md', 'log.md']) {
       const srcPath = path.join(baseDir, rootFile);
       if (!fs.existsSync(srcPath)) continue;
-      if (!fs.existsSync(destBase)) fs.mkdirSync(destBase, { recursive: true });
+      fs.mkdirSync(destBase, { recursive: true });
       if (exportMdFile(srcPath, path.join(destBase, rootFile), convertFn)) exported++;
     }
 
+    hookLog('session-end', 'info', `obsidian exported=${exported}, dirs=${EXPORT_DIRS.length}, vault=${vaultPath}`);
     if (exported > 0) {
       config.obsidian.lastExport = new Date().toISOString();
       config.obsidian.lastExportCount = exported;
       fs.writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf8');
     }
   } catch (err) {
-    process.stderr.write(`[mindlore] obsidian sync failed: ${err?.message ?? err}\n`);
+    hookLog('session-end', 'error', `obsidian internal: ${err?.message ?? err}`);
+    throw err; // re-throw so safeRun logs FAIL
   }
 }
 
@@ -345,38 +439,34 @@ function syncObsidian(baseDir) {
  * Only runs for the global scope — project .mindlore/ is in the project's own git.
  * Push failure is graceful (offline support).
  */
+function resolveGitBin() {
+  return resolveWin32Bin('git');
+}
+
 function syncGlobalRepo() {
   const gDir = globalDir();
   const gitDir = path.join(gDir, '.git');
   if (!fs.existsSync(gitDir)) return;
 
-  try {
-    // Check for changes
-    const status = execSync('git status --porcelain', {
-      cwd: gDir,
-      encoding: 'utf8',
-      timeout: 5000,
-    }).trim();
+  const git = resolveGitBin();
+  const execOpts = (timeout) => ({ cwd: gDir, encoding: 'utf8', timeout, stdio: 'pipe' });
 
-    if (!status) return; // nothing to commit
+  // Check for changes
+  const status = execSync(`"${git}" status --porcelain`, execOpts(5000)).trim();
+  if (!status) return; // nothing to commit
 
-    execSync('git add *.md mindlore.db config.json diary/ sources/ domains/ analyses/ decisions/ raw/ connections/ insights/ learnings/', { cwd: gDir, timeout: 5000, stdio: 'pipe' });
-    const now = new Date().toISOString().slice(0, 19);
-    execSync(`git commit -m "mindlore auto-sync ${now}"`, {
-      cwd: gDir,
-      timeout: 10000,
-      stdio: 'pipe',
-    });
+  execSync(`"${git}" add *.md mindlore.db diary/ sources/ domains/ analyses/ decisions/ raw/ connections/ insights/ learnings/`, execOpts(10000));
+  const now = new Date().toISOString().slice(0, 19);
+  execSync(`"${git}" commit -m "mindlore auto-sync ${now}"`, execOpts(15000));
 
-    // Push — graceful fail if no remote or offline
-    try {
-      execSync('git push', { cwd: gDir, timeout: 15000, stdio: 'pipe' });
-    } catch (_pushErr) {
-      // Offline or no remote — silently continue
-    }
-  } catch (_err) {
-    // Git not available or commit failed — silently continue
+  // Push — graceful fail if no remote or offline
+  try {
+    execSync(`"${git}" push`, execOpts(15000));
+  } catch (_pushErr) {
+    hookLog('session-end', 'warn', 'git push failed (offline?): ' + (_pushErr?.message ?? '').slice(0, 100));
   }
 }
 
-main();
+if (!process.argv.includes('--worker')) {
+  main();
+}

package/hooks/mindlore-session-focus.cjs

@@ -10,7 +10,7 @@
 
 const fs = require('fs');
 const path = require('path');
-const { findMindloreDir, readConfig, openDatabase, hasEpisodesTable, queryRecentEpisodes, querySupersededChains, formatSupersededChains, queryMultiSessionEpisodes, formatMultiSessionEpisodes, getAllMdFiles } = require('./lib/mindlore-common.cjs');
+const { findMindloreDir, readConfig, openDatabase, hasEpisodesTable, queryRecentEpisodes, querySupersededChains, formatSupersededChains, queryMultiSessionEpisodes, formatMultiSessionEpisodes, getAllMdFiles, getRecentHookErrors, hookLog } = require('./lib/mindlore-common.cjs');
 
 function main() {
   const baseDir = findMindloreDir();
@@ -117,8 +117,29 @@ function main() {
     }
   } catch (_healthErr) { /* skip */ }
 
-  if (output.length > 0) {
-    process.stdout.write(output.join('\n\n') + '\n');
+  // Check for recent hook errors — inject warnings into CC context
+  try {
+    const errors = getRecentHookErrors();
+    if (errors.length > 0) {
+      const lines = errors.map(e => `- [${e.ts.slice(0, 19)}] **${e.hook}** (${e.level}): ${e.msg}`);
+      output.push(`[Mindlore Hook Alerts]\n${lines.join('\n')}`);
+    }
+  } catch (_hookLogErr) { /* skip */ }
+
+  hookLog('session-focus', 'info', 'session started');
+
+  // Token budget for session inject
+  // Defaults match DEFAULT_TOKEN_BUDGET in scripts/lib/constants.ts
+  const budgetConfig = config?.tokenBudget ?? {};
+  const maxInjectChars = (budgetConfig.sessionInject || 2000) * 4;
+
+  let joined = output.join('\n\n');
+  if (joined.length > maxInjectChars) {
+    joined = joined.slice(0, maxInjectChars) + '\n[...truncated by token budget]';
+  }
+
+  if (joined.length > 0) {
+    process.stdout.write(joined + '\n');
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mindlore",
-  "version": "0.4.3",
+  "version": "0.5.1",
   "description": "AI-native knowledge system for Claude Code",
   "type": "commonjs",
   "bin": {
@@ -43,7 +43,7 @@
     "node": ">=20.0.0"
   },
   "dependencies": {
-    "better-sqlite3": "^11.0.0",
+    "better-sqlite3": "^12.9.0",
     "zod": "^4.3.6"
   },
   "devDependencies": {
@@ -52,9 +52,11 @@
     "@types/node": "^25.6.0",
     "@typescript-eslint/eslint-plugin": "^8.58.1",
     "@typescript-eslint/parser": "^8.58.1",
-    "eslint": "^9.0.0",
-    "globals": "^15.0.0",
+    "@xenova/transformers": "^2.17.2",
+    "eslint": "^10.2.0",
+    "globals": "^17.5.0",
     "jest": "^29.7.0",
+    "sqlite-vec": "^0.1.9",
     "ts-jest": "^29.4.9",
     "typescript": "^6.0.2"
   },

package/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "mindlore",
   "description": "AI-native knowledge system for Claude Code. Persistent, searchable, evolving knowledge base with FTS5.",
-  "version": "0.4.3",
+  "version": "0.5.1",
   "skills": [
     {
       "name": "mindlore-ingest",

package/skills/mindlore-ingest/SKILL.md

@@ -129,8 +129,14 @@ N source, N analysis, N total
 
 ## Post-Ingest Quality Gate
 
-After every ingest, verify all 6 checkpoints before reporting success:
+After every ingest, verify all 7 checkpoints before reporting success:
 
+0. **Duplicate check** — Ingest öncesi mevcut DB'de benzer içerik ara:
+   ```bash
+   node dist/scripts/lib/similarity.js "<title or first 100 chars>"
+   ```
+   Eğer score > 0.7 olan sonuç varsa KULLANICIYA SOR: "Bu içerik '${slug}' ile benzer görünüyor. Yine de eklensin mi?"
+   Kullanıcı onaylarsa devam et, yoksa atla.
 1. **raw/ file exists** — immutable capture written with frontmatter (slug, type, source_url)
 2. **sources/ summary exists** — processed summary with full frontmatter (slug, type, title, tags, quality, description)
 3. **INDEX.md updated** — stats line incremented, Recent section has new entry

package/templates/config.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.4.3",
+  "version": "0.5.1",
   "models": {
     "ingest": "haiku",
     "evolve": "sonnet",
@@ -12,5 +12,24 @@
   "session_focus": {
     "max_episodes": 3,
     "multi_session_days": 3
+  },
+  "synonyms": {
+    "auth": ["authentication", "login", "kimlik doğrulama"],
+    "güvenlik": ["security", "hardening", "sertleştirme"],
+    "db": ["database", "veritabanı", "sqlite"],
+    "api": ["endpoint", "rest", "graphql"],
+    "ui": ["frontend", "arayüz", "interface"],
+    "test": ["testing", "jest", "unit test"]
+  },
+  "hybrid": {
+    "enabled": true,
+    "ftsWeight": 0.4,
+    "vecWeight": 0.6,
+    "k": 60
+  },
+  "tokenBudget": {
+    "sessionInject": 2000,
+    "searchResults": 1500,
+    "perResult": 500
   }
 }