@mseep/core 3.0.0

package/lib/engine/gap-detector.js

@@ -0,0 +1,138 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { scanDependencies, checkPublishDate, CUTOFF_DATE, CONCURRENCY } from './registries.js';
+
+/**
+ * Detect knowledge gaps in the current project by checking dependency
+ * publish dates against the model training cutoff.
+ */
+export class GapDetector {
+  constructor(opts = {}) {
+    this.cutoffDate = opts.cutoffDate ?? CUTOFF_DATE;
+    this.cachePath = opts.cachePath ?? path.join(process.cwd(), '.booklib', 'version-cache.json');
+    this.cacheTtlMs = opts.cacheTtlMs ?? 24 * 60 * 60 * 1000; // 24h
+  }
+
+  /**
+   * Scan project and detect all gaps.
+   * @param {string} projectDir
+   * @returns {Promise<{postTraining: Array, uncapturedDocs: Array, ecosystems: string[], totalDeps: number, checkedDeps: number}>}
+   */
+  async detect(projectDir) {
+    const deps = scanDependencies(projectDir);
+    const ecosystems = [...new Set(deps.map(d => d.ecosystem))];
+
+    const cache = this._loadCache();
+
+    // Process deps in batches to avoid blasting registries with hundreds of concurrent requests
+    const results = [];
+    for (let i = 0; i < deps.length; i += CONCURRENCY) {
+      const batch = deps.slice(i, i + CONCURRENCY);
+      const batchResults = await Promise.all(batch.map(dep => this._checkDep(dep, cache)));
+      results.push(...batchResults);
+      // Save cache after each batch so progress survives crashes
+      this._saveCache(cache);
+    }
+
+    // Aggregate — no shared mutable state during async work
+    const postTraining = results.filter(r => r.postTraining).map(r => r.postTraining);
+    const checked = results.map(r => r.dep);
+
+    const uncapturedDocs = this._scanProjectDocs(projectDir);
+
+    return {
+      postTraining,
+      uncapturedDocs,
+      ecosystems,
+      totalDeps: deps.length,
+      checkedDeps: checked.length,
+    };
+  }
+
+  /** Check a single dep against cache or registry, returning result without side effects. */
+  async _checkDep(dep, cache) {
+    const cacheKey = `${dep.ecosystem}:${dep.name}@${dep.version}`;
+
+    if (cache[cacheKey] && Date.now() - cache[cacheKey].checkedAt < this.cacheTtlMs) {
+      if (cache[cacheKey].publishDate) {
+        const pubDate = new Date(cache[cacheKey].publishDate);
+        if (pubDate > this.cutoffDate) {
+          return { dep, postTraining: { ...dep, publishDate: pubDate } };
+        }
+      }
+      return { dep, postTraining: null };
+    }
+
+    const publishDate = await checkPublishDate(dep);
+    cache[cacheKey] = {
+      publishDate: publishDate?.toISOString() ?? null,
+      checkedAt: Date.now(),
+    };
+
+    if (publishDate && publishDate > this.cutoffDate) {
+      return { dep, postTraining: { ...dep, publishDate } };
+    }
+    return { dep, postTraining: null };
+  }
+
+  /**
+   * Find project documentation directories and files that are not yet
+   * connected to BookLib as knowledge sources.
+   * @param {string} projectDir
+   * @returns {Array<{path: string, type: string, fileCount: number}>}
+   */
+  _scanProjectDocs(projectDir) {
+    const docPaths = ['docs', 'decisions', 'specs', 'adrs', 'architecture'];
+    const docFiles = ['ARCHITECTURE.md', 'CONVENTIONS.md', 'DECISIONS.md', 'ADR.md'];
+
+    // Check which docs are already connected as BookLib sources
+    const connectedNames = new Set();
+    try {
+      const sourcesPath = path.join(projectDir, '.booklib', 'sources.json');
+      if (fs.existsSync(sourcesPath)) {
+        const registry = JSON.parse(fs.readFileSync(sourcesPath, 'utf8'));
+        for (const s of registry.sources ?? []) {
+          connectedNames.add(s.name);
+        }
+      }
+    } catch { /* best effort */ }
+
+    const found = [];
+
+    for (const dir of docPaths) {
+      if (connectedNames.has(dir)) continue; // already connected
+      const full = path.join(projectDir, dir);
+      if (fs.existsSync(full) && fs.statSync(full).isDirectory()) {
+        const fileCount = fs.readdirSync(full).filter(f => /\.(md|mdx|txt)$/i.test(f)).length;
+        if (fileCount > 0) {
+          found.push({ path: dir, type: 'directory', fileCount });
+        }
+      }
+    }
+
+    for (const file of docFiles) {
+      if (connectedNames.has(file)) continue;
+      if (fs.existsSync(path.join(projectDir, file))) {
+        found.push({ path: file, type: 'file', fileCount: 1 });
+      }
+    }
+
+    return found;
+  }
+
+  _loadCache() {
+    try {
+      return JSON.parse(fs.readFileSync(this.cachePath, 'utf8'));
+    } catch {
+      return {};
+    }
+  }
+
+  _saveCache(cache) {
+    try {
+      const dir = path.dirname(this.cachePath);
+      fs.mkdirSync(dir, { recursive: true });
+      fs.writeFileSync(this.cachePath, JSON.stringify(cache, null, 2));
+    } catch { /* best effort */ }
+  }
+}

package/lib/engine/gap-resolver.js

@@ -0,0 +1,135 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Resolve knowledge gaps by trying multiple sources in priority order.
+ * Context7 (instant) -> GitHub releases -> manual suggestion.
+ */
+export class GapResolver {
+  constructor(opts = {}) {
+    this.outputBase = opts.outputBase ?? path.join(process.cwd(), '.booklib', 'sources');
+  }
+
+  /**
+   * Resolve a single dependency gap.
+   * @param {{ name: string, version: string, ecosystem: string }} dep
+   * @returns {Promise<{ resolved: boolean, source: string, pageCount: number, suggestion?: string }>}
+   */
+  async resolve(dep) {
+    // Try Context7 first — fastest, broadest coverage
+    const ctx7Result = await this._tryContext7(dep);
+    if (ctx7Result.resolved) return ctx7Result;
+
+    // Fall back to GitHub releases
+    const ghResult = await this._tryGitHub(dep);
+    if (ghResult.resolved) return ghResult;
+
+    // Last resort: manual suggestion with ecosystem-specific URL
+    return this._suggestManual(dep);
+  }
+
+  /**
+   * Resolve multiple gaps, returning results for each.
+   * @param {Array<{ name: string, version: string, ecosystem: string }>} deps
+   * @param {Function} [onProgress] - called with { dep, result, index, total }
+   * @returns {Promise<Array<{ dep, result }>>}
+   */
+  async resolveAll(deps, onProgress) {
+    const results = [];
+    for (let i = 0; i < deps.length; i++) {
+      const dep = deps[i];
+      const result = await this.resolve(dep);
+      results.push({ dep, result });
+      onProgress?.({ dep, result, index: i, total: deps.length });
+    }
+    return results;
+  }
+
+  /** Attempt resolution via Context7 API. */
+  async _tryContext7(dep) {
+    try {
+      const { Context7Connector } = await import('../connectors/context7.js');
+      const ctx7 = new Context7Connector();
+      if (!ctx7.checkAuth().ok) return { resolved: false, source: 'context7' };
+
+      const sourceName = `ctx7-${dep.name.replace(/[@/]/g, '_').replace(/^_+/, '')}`;
+      const outputDir = path.join(this.outputBase, sourceName);
+      const result = await ctx7.resolveAndFetch(
+        dep.name,
+        outputDir,
+        `${dep.name} v${dep.version} API`,
+      );
+
+      if (result.resolved && result.pageCount > 0) {
+        return {
+          resolved: true,
+          source: 'context7',
+          pageCount: result.pageCount,
+          sourceName,
+          outputDir,
+        };
+      }
+    } catch { /* Context7 unavailable */ }
+    return { resolved: false, source: 'context7' };
+  }
+
+  /** Attempt resolution via GitHub release notes. npm-only. */
+  async _tryGitHub(dep) {
+    if (dep.ecosystem !== 'npm') return { resolved: false, source: 'github' };
+
+    try {
+      const { GitHubConnector } = await import('../connectors/github.js');
+      const gh = new GitHubConnector();
+      if (!gh.checkAuth().ok) return { resolved: false, source: 'github' };
+
+      // Resolve npm package -> GitHub repo via npm registry
+      const res = await fetch(
+        `https://registry.npmjs.org/${encodeURIComponent(dep.name)}`,
+        {
+          headers: { 'Accept': 'application/vnd.npm.install-v1+json' },
+          signal: AbortSignal.timeout(5000),
+        },
+      );
+      if (!res.ok) return { resolved: false, source: 'github' };
+
+      const data = await res.json();
+      const repoUrl = data.repository?.url ?? '';
+      const match = repoUrl.match(/github\.com[/:]([^/]+\/[^/.]+)/);
+      if (!match) return { resolved: false, source: 'github' };
+
+      const repo = match[1];
+      const sourceName = `gh-${dep.name}-releases`;
+      const outputDir = path.join(this.outputBase, sourceName);
+      const result = await gh.fetchReleases(repo, outputDir, { limit: 5 });
+
+      if (result.pageCount > 0) {
+        return {
+          resolved: true,
+          source: 'github',
+          pageCount: result.pageCount,
+          sourceName,
+          outputDir,
+        };
+      }
+    } catch { /* GitHub unavailable */ }
+    return { resolved: false, source: 'github' };
+  }
+
+  /** Build a manual suggestion with ecosystem-specific URL. */
+  _suggestManual(dep) {
+    const suggestions = {
+      npm: `booklib connect https://www.npmjs.com/package/${dep.name} --type=framework-docs`,
+      pypi: `booklib connect https://pypi.org/project/${dep.name}/ --type=framework-docs`,
+      crates: `booklib connect https://docs.rs/${dep.name} --type=framework-docs`,
+      go: `booklib connect https://pkg.go.dev/${dep.name} --type=framework-docs`,
+      rubygems: `booklib connect https://rubygems.org/gems/${dep.name} --type=framework-docs`,
+      maven: `booklib connect https://search.maven.org/artifact/${dep.name.replace(':', '/')} --type=framework-docs`,
+    };
+    return {
+      resolved: false,
+      source: 'manual',
+      pageCount: 0,
+      suggestion: suggestions[dep.ecosystem] ?? 'booklib connect <docs-url> --type=framework-docs',
+    };
+  }
+}

package/lib/engine/graph-injector.js

@@ -0,0 +1,137 @@
+// lib/engine/graph-injector.js
+import { minimatch } from 'minimatch';
+import {
+  listNodes, loadNode, loadEdges, traverseEdges,
+  parseNodeFrontmatter, resolveKnowledgePaths,
+} from './graph.js';
+
+// ── Component matching ────────────────────────────────────────────────────────
+
+/**
+ * Returns component nodes whose path globs match filePath.
+ * @param {string} filePath
+ * @param {Array<{id: string, paths: string[], title: string}>} components
+ */
+export function findOwningComponents(filePath, components) {
+  return components.filter(comp =>
+    (comp.paths ?? []).some(glob => minimatch(filePath, glob, { matchBase: true }))
+  );
+}
+
+// ── Ranking ───────────────────────────────────────────────────────────────────
+
+/**
+ * Deduplicates nodes by id (keeping highest score) and sorts descending by score.
+ * @param {Array<{id: string, score: number, text: string, hop: number}>} nodes
+ */
+export function scoreAndRankNodes(nodes) {
+  const best = new Map();
+  for (const node of nodes) {
+    const existing = best.get(node.id);
+    if (!existing || node.score > existing.score) {
+      best.set(node.id, node);
+    }
+  }
+  return [...best.values()].sort((a, b) => b.score - a.score);
+}
+
+// ── Main injection pipeline ───────────────────────────────────────────────────
+
+/**
+ * Builds a ranked list of relevant knowledge nodes for the given context.
+ *
+ * Pipeline:
+ * 1. Find component nodes that own the current file (path matching)
+ * 2. Semantic search for nodes matching the task context
+ * 3. BFS graph traversal from all start nodes (components + semantic hits)
+ * 4. Deduplicate and rank by score
+ * 5. Return top N with full content loaded
+ *
+ * @param {object} opts
+ * @param {string|null} opts.filePath - File being edited (used for component matching)
+ * @param {string} opts.taskContext - Task description for semantic search
+ * @param {object} opts.searcher - BookLibSearcher instance
+ * @param {number} [opts.limit=8] - Max nodes to return
+ * @param {number} [opts.minScore=0.35] - Minimum semantic similarity score
+ * @returns {Promise<Array<{id: string, title: string, type: string, body: string, score: number}>>}
+ */
+export async function buildGraphContext({ filePath, taskContext, searcher, limit = 8, minScore = 0.35 }) {
+  const { nodesDir } = resolveKnowledgePaths();
+  const allNodeIds = listNodes({ nodesDir });
+  if (allNodeIds.length === 0) return [];
+
+  const edges = loadEdges();
+
+  // Load all component nodes for path matching
+  const componentNodes = allNodeIds
+    .map(id => {
+      const raw = loadNode(id, { nodesDir });
+      return raw ? parseNodeFrontmatter(raw) : null;
+    })
+    .filter(n => n?.type === 'component');
+
+  // 1. Component nodes that own the current file
+  const owningComponents = filePath
+    ? findOwningComponents(filePath, componentNodes)
+    : [];
+
+  // 2. Semantic search — only knowledge nodes (nodeKind: 'knowledge')
+  let semanticResults = [];
+  if (taskContext) {
+    try {
+      const raw = await searcher.search(taskContext, 20, minScore);
+      semanticResults = raw
+        .filter(r => r.metadata?.nodeKind === 'knowledge' && r.metadata?.id)
+        .map(r => ({
+          id: r.metadata.id,
+          title: r.metadata.title,
+          type: r.metadata.type,
+          text: r.text,
+          score: r.score,
+          hop: 0,
+        }));
+    } catch {
+      // Index may not exist yet — skip semantic step gracefully
+    }
+  }
+
+  // 3. BFS traversal from all start nodes
+  const startIds = new Set([
+    ...owningComponents.map(c => c.id),
+    ...semanticResults.map(r => r.id),
+  ]);
+
+  const traversalHits = [];
+  for (const startId of startIds) {
+    const hops = traverseEdges(startId, edges, 2);
+    for (const { id, hop } of hops) {
+      const raw = loadNode(id, { nodesDir });
+      if (!raw) continue;
+      const parsed = parseNodeFrontmatter(raw);
+      traversalHits.push({
+        id,
+        title: parsed.title,
+        type: parsed.type,
+        text: parsed.body ?? '',
+        score: 0.5 / hop, // distance penalty: hop 1 → 0.5, hop 2 → 0.25
+        hop,
+      });
+    }
+  }
+
+  // 4. Merge, deduplicate, rank
+  const ranked = scoreAndRankNodes([...semanticResults, ...traversalHits]);
+
+  // 5. Load full content for top results
+  return ranked.slice(0, limit).map(node => {
+    const raw = loadNode(node.id, { nodesDir });
+    const parsed = raw ? parseNodeFrontmatter(raw) : {};
+    return {
+      id: node.id,
+      title: parsed.title ?? node.title,
+      type: parsed.type ?? node.type,
+      body: parsed.body ?? node.text,
+      score: node.score,
+    };
+  });
+}

package/lib/engine/graph-search.js

@@ -0,0 +1,183 @@
+import { listNodes, loadNode, parseNodeFrontmatter, loadEdges, traverseEdges } from './graph.js';
+import { extractKeywords } from './query-expander.js';
+
+/**
+ * Multi-dimensional graph-activated search.
+ * Parses query into concepts, activates graph subregions for each,
+ * scores nodes by how many concepts they connect to, merges with text search results.
+ *
+ * @param {string} query
+ * @param {Array} textSearchResults - results from BookLibSearcher.search()
+ * @param {object} opts
+ * @param {string} [opts.nodesDir] - knowledge nodes directory
+ * @param {string} [opts.graphFile] - graph.jsonl path
+ * @returns {object} { concepts, graphResults, mergedResults, activated }
+ */
+export function graphActivatedSearch(query, textSearchResults = [], opts = {}) {
+  const { nodesDir, graphFile } = opts;
+
+  const concepts = extractConcepts(query);
+
+  // Single concept or no concepts: skip graph activation — regular search is sufficient
+  if (concepts.length < 2) {
+    return {
+      concepts,
+      graphResults: [],
+      mergedResults: textSearchResults,
+      activated: false,
+    };
+  }
+
+  const allNodeIds = listNodes({ nodesDir });
+  const edges = loadEdges({ graphFile });
+  const subgraphs = new Map();
+
+  for (const concept of concepts) {
+    const activated = activateSubgraph(concept, allNodeIds, edges, { nodesDir });
+    subgraphs.set(concept, activated);
+  }
+
+  const nodeScores = scoreIntersections(subgraphs);
+
+  const graphResults = [];
+  for (const [nodeId, { score: intersectionScore, matchedConcepts }] of nodeScores.entries()) {
+    if (intersectionScore < 2) continue;
+
+    const raw = loadNode(nodeId, { nodesDir });
+    if (!raw) continue;
+    const parsed = parseNodeFrontmatter(raw);
+
+    graphResults.push({
+      principle: parsed.title ?? nodeId,
+      context: (parsed.body ?? '').slice(0, 150),
+      source: `project ${parsed.type ?? 'knowledge'}: ${nodeId}`,
+      section: 'knowledge',
+      matchedConcepts,
+      intersectionScore,
+    });
+  }
+
+  graphResults.sort((a, b) => b.intersectionScore - a.intersectionScore);
+
+  const textPrinciples = textSearchResults.map(r => ({
+    principle: r.text?.slice(0, 150) ?? '',
+    context: '',
+    source: r.metadata?.name ?? 'unknown',
+    section: r.metadata?.type ?? 'content',
+    matchedConcepts: [],
+    intersectionScore: 0,
+    score: r.score,
+  }));
+
+  const mergedResults = [...graphResults, ...textPrinciples];
+
+  return {
+    concepts,
+    graphResults,
+    mergedResults,
+    activated: true,
+  };
+}
+
+/**
+ * Extract semantic concepts from a query.
+ * Groups consecutive keywords into compound concepts.
+ *
+ * @param {string} query
+ * @returns {string[]}
+ */
+export function extractConcepts(query) {
+  const keywords = extractKeywords(query);
+  if (keywords.length === 0) return [];
+
+  // Collect runs of consecutive keywords from the original query
+  const words = query.toLowerCase().split(/\s+/);
+  const runs = [];
+  let current = [];
+
+  for (const word of words) {
+    const isKeyword = keywords.includes(word);
+    if (isKeyword) {
+      current.push(word);
+    } else if (current.length > 0) {
+      runs.push(current);
+      current = [];
+    }
+  }
+  if (current.length > 0) {
+    runs.push(current);
+  }
+
+  // Split runs into concepts: compound terms (max 2 words) stay grouped,
+  // longer runs are split into individual concepts
+  const concepts = [];
+  for (const run of runs) {
+    if (run.length <= 2) {
+      concepts.push(run.join(' '));
+    } else {
+      for (const word of run) {
+        concepts.push(word);
+      }
+    }
+  }
+
+  return [...new Set(concepts)];
+}
+
+/**
+ * Activate a subgraph for a single concept.
+ * Finds matching nodes + their 1-hop neighbors.
+ *
+ * @param {string} concept
+ * @param {string[]} allNodeIds
+ * @param {Array} edges
+ * @param {{ nodesDir?: string }} opts
+ * @returns {Set<string>} activated node IDs
+ */
+function activateSubgraph(concept, allNodeIds, edges, { nodesDir } = {}) {
+  const activated = new Set();
+  const conceptWords = concept.split(/\s+/);
+
+  for (const nodeId of allNodeIds) {
+    const raw = loadNode(nodeId, { nodesDir });
+    if (!raw) continue;
+    const parsed = parseNodeFrontmatter(raw);
+
+    const nodeText = `${parsed.title ?? ''} ${parsed.body ?? ''} ${(parsed.tags ?? []).join(' ')}`.toLowerCase();
+
+    const matches = conceptWords.some(w => w.length > 2 && nodeText.includes(w));
+    if (matches) {
+      activated.add(nodeId);
+
+      const neighbors = traverseEdges(nodeId, edges, 1);
+      for (const { id } of neighbors) {
+        activated.add(id);
+      }
+    }
+  }
+
+  return activated;
+}
+
+/**
+ * Score nodes by how many concept subgraphs they appear in.
+ *
+ * @param {Map<string, Set<string>>} subgraphs
+ * @returns {Map<string, { score: number, matchedConcepts: string[] }>}
+ */
+function scoreIntersections(subgraphs) {
+  const scores = new Map();
+
+  for (const [concept, activated] of subgraphs.entries()) {
+    for (const nodeId of activated) {
+      if (!scores.has(nodeId)) {
+        scores.set(nodeId, { score: 0, matchedConcepts: [] });
+      }
+      const entry = scores.get(nodeId);
+      entry.score++;
+      entry.matchedConcepts.push(concept);
+    }
+  }
+
+  return scores;
+}