@booklib/core 2.0.0

package/lib/engine/query-expander.js

@@ -0,0 +1,42 @@
+export const STOPWORDS = new Set([
+  'a', 'an', 'the', 'in', 'on', 'at', 'to', 'for', 'of', 'and', 'or',
+  'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had',
+  'do', 'does', 'did', 'will', 'would', 'shall', 'should', 'may', 'might',
+  'must', 'can', 'could', 'not', 'no', 'nor', 'so', 'yet', 'both', 'with',
+  'about', 'from', 'up', 'down', 'out', 'how', 'what', 'when', 'where',
+  'who', 'why', 'which', 'by', 'as', 'if', 'then', 'than', 'too', 'very',
+  'just', 'more', 'also', 'its', 'it',
+]);
+
+export function extractKeywords(query) {
+  const tokens = query
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(token => token.length > 1 && !STOPWORDS.has(token));
+
+  return tokens;
+}
+
+export function expandQuery(query) {
+  const keywords = extractKeywords(query);
+
+  const expanded = [];
+
+  if (keywords.length > 0) {
+    const keywordJoined = keywords.join(' ');
+    const candidates = [
+      keywordJoined,
+      `best practices for ${keywordJoined}`,
+      `how to ${keywordJoined}`,
+    ];
+    for (const v of candidates) {
+      if (v !== query) expanded.push(v);
+    }
+  }
+
+  return {
+    original: query,
+    keywords,
+    expanded,
+  };
+}

package/lib/engine/reranker.js

@@ -0,0 +1,40 @@
+import { pipeline } from '@huggingface/transformers';
+
+const MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2';
+
+export class Reranker {
+  constructor() {
+    this._pipeline = null;
+  }
+
+  async _ensurePipelineLoaded() {
+    if (this._pipeline === null) {
+      this._pipeline = await pipeline('text-classification', MODEL);
+    }
+  }
+
+  async rerank(query, candidates) {
+    // Return empty array early without loading model if candidates is empty
+    if (candidates.length === 0) {
+      return [];
+    }
+
+    // Load the model on first use
+    await this._ensurePipelineLoaded();
+
+    // Create query-passage pairs
+    const pairs = candidates.map(c => [query, c.text]);
+
+    // Call the cross-encoder model
+    const outputs = await this._pipeline(pairs);
+
+    return candidates
+      .map((c, i) => {
+        const raw = outputs[i];
+        const scores = Array.isArray(raw) ? raw : [raw];
+        const best = scores.reduce((a, b) => (a.score > b.score ? a : b));
+        return { ...c, score: best.score };
+      })
+      .sort((a, b) => b.score - a.score);
+  }
+}

package/lib/engine/rrf.js

@@ -0,0 +1,59 @@
+/**
+ * Reciprocal Rank Fusion (RRF) — merge multiple ranked lists into a single sorted list.
+ * Items appearing in multiple lists accumulate scores; deduplication uses `text` as the identity key.
+ *
+ * Formula: contribution of item at rank r (0-indexed) in list i = weights[i] / (k + r + 1)
+ *
+ * @param {Array<Array<{score, text, metadata}>>} resultLists - One ranked list per retrieval source
+ * @param {Object} options
+ * @param {number} [options.k=60] - Smoothing constant (from RRF literature)
+ * @param {number[]} [options.weights] - Per-list multipliers (default: all 1)
+ * @returns {Array<{score, text, metadata}>} Merged and sorted descending by RRF score
+ */
+export function reciprocalRankFusion(resultLists, { k = 60, weights } = {}) {
+  // Handle empty input
+  if (!resultLists || resultLists.length === 0) {
+    return [];
+  }
+
+  // Filter out empty lists and compute default weights
+  const nonEmptyLists = resultLists.filter(list => list && list.length > 0);
+  if (nonEmptyLists.length === 0) {
+    return [];
+  }
+
+  // Use provided weights or default to all 1s
+  const finalWeights = weights || Array(resultLists.length).fill(1);
+
+  // Accumulator: map from text → { text, metadata, score }
+  const scoreMap = new Map();
+
+  // Process each list
+  for (let i = 0; i < resultLists.length; i++) {
+    const list = resultLists[i];
+    const weight = finalWeights[i] !== undefined ? finalWeights[i] : 1;
+
+    for (let rank = 0; rank < list.length; rank++) {
+      const item = list[rank];
+      const contribution = weight / (k + rank + 1);
+
+      if (scoreMap.has(item.text)) {
+        // Item already seen; accumulate score
+        const existing = scoreMap.get(item.text);
+        existing.score += contribution;
+      } else {
+        // First time seeing this item; store it
+        scoreMap.set(item.text, {
+          text: item.text,
+          metadata: item.metadata,
+          score: contribution,
+        });
+      }
+    }
+  }
+
+  // Convert map to sorted array (descending by score)
+  const result = Array.from(scoreMap.values()).sort((a, b) => b.score - a.score);
+
+  return result;
+}

package/lib/engine/scanner.js

@@ -0,0 +1,151 @@
+import fs from 'fs';
+import path from 'path';
+import { resolveBookLibPaths } from '../paths.js';
+
+/**
+ * Handles project-wide compliance scanning against the entire BookLib library.
+ * Uses audit.json rule sets for fast, accurate static analysis.
+ * Falls back to skipping files whose skill has no audit.json.
+ */
+export class BookLibScanner {
+  constructor() {}
+
+  /**
+   * Scans a directory and generates a 'Wisdom Compliance' heatmap.
+   *
+   * @param {string} dirPath - The project directory to scan.
+   * @param {object} opts
+   * @param {string} opts.mode - 'code' (default) or 'docs' (scans .md/.txt files)
+   * @returns {string} - A comprehensive project health report.
+   */
+  scan(dirPath, { mode = 'code' } = {}) {
+    const files = this.getFiles(dirPath);
+    const reports = [];
+    const stats = {
+      filesScanned: files.length,
+      filesMatched: 0,
+      filesAudited: 0,
+      violations: 0,
+      healthyFiles: 0,
+      debtBySkill: {},
+      noRulesSkills: new Set(),
+    };
+
+    console.log(`Starting deep scan of ${files.length} files...`);
+
+    for (const file of files) {
+      const skillName = this.detectSkill(file, mode);
+      if (!skillName) continue;
+
+      stats.filesMatched++;
+      const auditJsonPath = this._findAuditJson(skillName);
+      if (!auditJsonPath) {
+        stats.noRulesSkills.add(skillName);
+        continue; // skill has no static rules — skip
+      }
+
+      let rules;
+      try {
+        ({ rules } = JSON.parse(fs.readFileSync(auditJsonPath, 'utf8')));
+      } catch { continue; }
+
+      const lines = fs.readFileSync(file, 'utf8').split('\n');
+      let violationCount = 0;
+      for (const rule of rules) {
+        const regex = new RegExp(rule.pattern, 'g');
+        for (const line of lines) {
+          if (regex.test(line)) violationCount++;
+        }
+      }
+
+      stats.filesAudited++;
+      stats.violations += violationCount;
+      if (violationCount === 0) stats.healthyFiles++;
+      stats.debtBySkill[skillName] = (stats.debtBySkill[skillName] || 0) + violationCount;
+      if (violationCount > 0) {
+        reports.push({ file, skill: skillName, violationCount });
+      }
+    }
+
+    return this.formatDashboard(stats, reports);
+  }
+
+  /**
+   * Detects which BookLib skill is most relevant to a file extension.
+   */
+  detectSkill(filePath, mode = 'code') {
+    const ext = path.extname(filePath);
+    if (mode === 'docs') {
+      if (ext === '.md' || ext === '.mdx' || ext === '.mdc') return 'writing-plans';
+      if (ext === '.txt') return 'writing-plans';
+      return null;
+    }
+    if (ext === '.kt' || ext === '.kts') return 'effective-kotlin';
+    if (ext === '.ts' || ext === '.tsx') return 'effective-typescript';
+    if (ext === '.java') return 'effective-java';
+    if (ext === '.py') return 'effective-python';
+    if (ext === '.js' || ext === '.mjs' || ext === '.cjs') return 'clean-code-reviewer';
+    return null;
+  }
+
+  /**
+   * Finds audit.json for a skill — checks bundled skills first, then community cache.
+   */
+  _findAuditJson(skillName) {
+    const { skillsPath, cachePath } = resolveBookLibPaths();
+    const bundled = path.join(skillsPath, skillName, 'audit.json');
+    if (fs.existsSync(bundled)) return bundled;
+    const community = path.join(cachePath, 'skills', skillName, 'audit.json');
+    if (fs.existsSync(community)) return community;
+    return null;
+  }
+
+  /**
+   * Helper to recursively list files, excluding common noise directories.
+   */
+  getFiles(dir) {
+    let results = [];
+    const list = fs.readdirSync(dir);
+    const ignore = ['.git', 'node_modules', '.booklib', '.claude', '.idea', 'dist', 'build', 'coverage', '.cache'];
+
+    list.forEach(file => {
+      if (ignore.includes(file)) return;
+
+      const fullPath = path.join(dir, file);
+      const stat = fs.statSync(fullPath);
+      if (stat && stat.isDirectory()) {
+        results = results.concat(this.getFiles(fullPath));
+      } else {
+        results.push(fullPath);
+      }
+    });
+    return results;
+  }
+
+  formatDashboard(stats, reports) {
+    const sortedReports = reports.sort((a, b) => b.violationCount - a.violationCount);
+    const topDebt = sortedReports.slice(0, 5);
+
+    return `
+# 📊 BookLib Wisdom Heatmap
+**Project Health Dashboard**
+
+## 📈 Executive Summary
+- **Files Scanned**: ${stats.filesScanned} (${stats.filesMatched} matched a skill lens, ${stats.filesAudited} audited)
+- **Healthy Files**: ${stats.healthyFiles} / ${stats.filesAudited} audited
+- **Total Violations**: ${stats.violations}
+- **Compliance**: ${((stats.healthyFiles / (stats.filesAudited || 1)) * 100).toFixed(1)}% of audited files violation-free
+
+## 🛠 Architectural Debt by Skill
+${Object.entries(stats.debtBySkill).map(([skill, count]) => `- **${skill}**: ${count} violations`).join('\n')}
+
+## 🚨 Top 5 Refactoring Priorities (High Debt)
+${topDebt.map(r => `- \`${path.relative(process.cwd(), r.file)}\` (${r.violationCount} violations | Lens: ${r.skill})`).join('\n')}
+
+---
+> **Note**: This heatmap is based on 'Light Audits' (Static Pattern Detection).
+> Use \`booklib audit <skill> <file>\` for a deep-reasoning review of high-priority files.
+${stats.noRulesSkills.size > 0 ? `> **No static rules loaded for:** ${[...stats.noRulesSkills].join(', ')} — add an \`audit.json\` to enable pattern checks.` : ''}
+`;
+  }
+}

package/lib/engine/searcher.js

@@ -0,0 +1,139 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { LocalIndex } from 'vectra';
+import { pipeline } from '@huggingface/transformers';
+import { resolveBookLibPaths } from '../paths.js';
+import { BM25Index } from './bm25-index.js';
+import { reciprocalRankFusion } from './rrf.js';
+import { expandQuery } from './query-expander.js';
+import { Reranker } from './reranker.js';
+import { loadEdges, traverseEdges } from './graph.js';
+
+const RRF_ORIGINAL_WEIGHT = 2;
+const RRF_EXPANDED_WEIGHT = 1;
+const RERANK_CANDIDATES = 20;
+
+export class BookLibSearcher {
+  constructor(indexPath) {
+    this.indexPath = indexPath ?? resolveBookLibPaths().indexPath;
+    this.index = new LocalIndex(this.indexPath);
+    this.extractor = null;
+    this.reranker = new Reranker();
+  }
+
+  async loadModel() {
+    if (!this.extractor) {
+      this.extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+    }
+  }
+
+  async getEmbedding(text) {
+    await this.loadModel();
+    const output = await this.extractor(text, { pooling: 'mean', normalize: true });
+    return Array.from(output.data);
+  }
+
+  get bm25Path() {
+    return path.join(path.dirname(this.indexPath), 'bm25.json');
+  }
+
+  get graphFile() {
+    return path.join(path.dirname(this.indexPath), 'knowledge', 'graph.jsonl');
+  }
+
+  async _vectorSearch(query, topK) {
+    const vector = await this.getEmbedding(query);
+    const results = await this.index.queryItems(vector, '', topK);
+    return results.map(r => ({
+      score: r.score,
+      text: r.item.metadata.text,
+      metadata: { ...r.item.metadata, text: undefined },
+    }));
+  }
+
+  async search(query, limit = 5, minScore = 0.5, options = {}) {
+    const { useGraph = false } = options;
+    if (!(await this.index.isIndexCreated())) {
+      throw new Error('Index not found. Please run "booklib index" first.');
+    }
+
+    // Fallback: no BM25 index present — use pure vector search (backwards compatible)
+    if (!fs.existsSync(this.bm25Path)) {
+      const vector = await this.getEmbedding(query);
+      const results = await this.index.queryItems(vector, '', limit);
+      const filtered = results
+        .filter(r => r.score >= minScore)
+        .map(r => ({
+          score: r.score,
+          text: r.item.metadata.text,
+          metadata: { ...r.item.metadata, text: undefined },
+        }));
+      return addDisplayScores(useGraph ? this._appendGraphResults(filtered) : filtered);
+    }
+
+    const bm25 = BM25Index.load(this.bm25Path);
+    const { expanded } = expandQuery(query);
+    const allQueries = [query, ...expanded];
+
+    const resultLists = [];
+    const weights = [];
+
+    for (let i = 0; i < allQueries.length; i++) {
+      const q = allQueries[i];
+      const w = i === 0 ? RRF_ORIGINAL_WEIGHT : RRF_EXPANDED_WEIGHT;
+
+      const [vecResults, bm25Results] = await Promise.all([
+        this._vectorSearch(q, RERANK_CANDIDATES),
+        Promise.resolve(bm25.search(q, RERANK_CANDIDATES)),
+      ]);
+
+      resultLists.push(vecResults, bm25Results);
+      weights.push(w, w);
+    }
+
+    const merged = reciprocalRankFusion(resultLists, { weights });
+    const candidates = merged.slice(0, RERANK_CANDIDATES);
+    const reranked = await this.reranker.rerank(query, candidates);
+
+    const results = reranked.filter(r => r.score >= minScore).slice(0, limit);
+    return addDisplayScores(useGraph ? this._appendGraphResults(results) : results);
+  }
+
+  _appendGraphResults(results) {
+    const GRAPH_EDGE_TYPES = new Set(['see-also', 'applies-to', 'extends']);
+    let edges;
+    try {
+      edges = loadEdges({ graphFile: this.graphFile });
+    } catch {
+      return results;
+    }
+    if (edges.length === 0) return results;
+
+    const seenIds = new Set(
+      results.map(r => r.metadata?.name ?? r.metadata?.id).filter(Boolean)
+    );
+    const graphLinked = [];
+
+    for (const result of results) {
+      const nodeId = result.metadata?.name ?? result.metadata?.id;
+      if (!nodeId) continue;
+
+      for (const { id: neighborId, edge } of traverseEdges(nodeId, edges, 1)) {
+        if (!GRAPH_EDGE_TYPES.has(edge.type)) continue;
+        if (seenIds.has(neighborId)) continue;
+        seenIds.add(neighborId);
+        graphLinked.push({
+          score: 0,
+          text: '',
+          metadata: { name: neighborId, source: 'graph', edgeType: edge.type },
+        });
+      }
+    }
+
+    return [...results, ...graphLinked];
+  }
+}
+
+function addDisplayScores(results) {
+  return results.map((r, i) => ({ ...r, displayScore: r.displayScore ?? Math.round(100 / (i + 1)) }));
+}