@mseep/core 3.0.0

package/lib/engine/principle-extractor.js

@@ -0,0 +1,116 @@
+/**
+ * Extracts individual principles from a search result chunk.
+ * A chunk is typically the content of one XML tag section.
+ * Principles are bullet points, numbered items, or paragraphs.
+ *
+ * @param {string} text - the chunk text
+ * @param {object} metadata - chunk metadata (name, type, section, etc.)
+ * @returns {Array<{principle: string, context: string, source: string, section: string}>}
+ */
+export function extractPrinciples(text, metadata = {}) {
+  if (!text || typeof text !== 'string') return [];
+
+  const source = metadata.name ?? metadata.title ?? 'unknown';
+  const section = metadata.type ?? metadata.originalTag ?? 'content';
+  const lines = text.split('\n');
+
+  const principles = [];
+
+  // Strategy 1: Bullet points and numbered items
+  let currentPrinciple = null;
+  let currentContext = [];
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      // Blank line — flush current principle
+      if (currentPrinciple) {
+        principles.push({
+          principle: currentPrinciple,
+          context: currentContext.join(' ').trim(),
+          source,
+          section,
+        });
+        currentPrinciple = null;
+        currentContext = [];
+      }
+      continue;
+    }
+
+    // New bullet/numbered item
+    const bulletMatch = trimmed.match(/^[-*]\s+\*?\*?(.+?)(\*?\*?\s*[:—–-]\s*(.+))?$/);
+    const numberedMatch = trimmed.match(/^\d+\.\s+\*?\*?(.+?)(\*?\*?\s*[:—–-]\s*(.+))?$/);
+    const boldMatch = trimmed.match(/^\*\*(.+?)\*\*\s*[:—–-]?\s*(.*)/);
+
+    if (bulletMatch || numberedMatch || boldMatch) {
+      // Flush previous
+      if (currentPrinciple) {
+        principles.push({
+          principle: currentPrinciple,
+          context: currentContext.join(' ').trim(),
+          source,
+          section,
+        });
+      }
+
+      const match = boldMatch || bulletMatch || numberedMatch;
+      currentPrinciple = (match[1] ?? '').replace(/\*\*/g, '').trim();
+      currentContext = match[3] ? [match[3].trim()] : match[2] ? [match[2].replace(/\*\*/g, '').trim()] : [];
+    } else if (currentPrinciple) {
+      // Continuation line — add to context
+      currentContext.push(trimmed);
+    }
+  }
+
+  // Flush last
+  if (currentPrinciple) {
+    principles.push({
+      principle: currentPrinciple,
+      context: currentContext.join(' ').trim(),
+      source,
+      section,
+    });
+  }
+
+  // Strategy 2: If no structured items found, treat the whole chunk as one principle
+  if (principles.length === 0 && text.trim().length > 0) {
+    // Take first sentence or first 150 chars as the principle
+    const firstSentence = text.trim().split(/[.!?]\s/)[0];
+    const principle = firstSentence.length > 150 ? firstSentence.slice(0, 147) + '...' : firstSentence;
+    principles.push({
+      principle,
+      context: text.trim().slice(principle.length).trim().slice(0, 200),
+      source,
+      section,
+    });
+  }
+
+  return principles;
+}
+
+/**
+ * Extracts principles from multiple search results.
+ * Deduplicates by principle text.
+ *
+ * @param {Array<{text: string, metadata: object, score: number}>} results
+ * @param {number} [maxPrinciples=5] - max total principles to return
+ * @returns {Array<{principle: string, context: string, source: string, section: string}>}
+ */
+export function extractFromResults(results, maxPrinciples = 5) {
+  const all = [];
+  const seen = new Set();
+
+  for (const result of results) {
+    const principles = extractPrinciples(result.text, result.metadata);
+    for (const p of principles) {
+      const key = p.principle.toLowerCase().slice(0, 50);
+      if (!seen.has(key)) {
+        seen.add(key);
+        all.push(p);
+      }
+      if (all.length >= maxPrinciples) return all;
+    }
+  }
+
+  return all;
+}

package/lib/engine/project-analyzer.js

@@ -0,0 +1,353 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseImports, detectLanguage } from './import-parser.js';
+import { GapDetector } from './gap-detector.js';
+
+const SKIP_DIRS = new Set([
+  'node_modules', '.git', '.booklib', 'dist', 'build', 'vendor',
+  '.next', '__pycache__', '.venv', 'venv',
+  'out', '.claude', '.idea', '.vscode', '.agents',
+  '.junie', '.specify', '.gemini', '.github',
+]);
+
+/**
+ * Icon library packages whose imports inflate API counts.
+ * Individual icon imports (e.g., `import { ArrowLeft } from 'lucide-react'`)
+ * are not meaningful API surface to track.
+ */
+const ICON_LIBRARIES = new Set([
+  'lucide-react', 'lucide-vue', 'lucide-svelte', 'lucide',
+  '@heroicons/react', '@heroicons/vue',
+  'react-icons',
+  '@phosphor-icons/react', '@phosphor-icons/vue',
+  '@tabler/icons-react', '@tabler/icons-vue',
+  '@radix-ui/react-icons',
+  'ionicons',
+  '@mdi/js', '@mdi/react',
+  'feather-icons', 'react-feather',
+]);
+const CODE_EXTENSIONS = /\.(js|mjs|cjs|ts|tsx|jsx|py|go|rs|java|kt|kts|rb|php|cs|swift|dart)$/i;
+const MAX_FILES = 500;
+const MAX_FILE_SIZE = 500_000;
+
+/**
+ * Cross-references gap detection with import detection to identify
+ * exactly which files and APIs in the project are affected by
+ * post-training knowledge gaps.
+ */
+export class ProjectAnalyzer {
+  constructor(opts = {}) {
+    this.gapDetector = opts.gapDetector ?? new GapDetector(opts);
+  }
+
+  /**
+   * Full project analysis: find post-training deps, then find which
+   * source files import APIs from those deps.
+   * @param {string} projectDir
+   * @returns {Promise<{affected: Array, totalFiles: number, totalApis: number, gaps: object}>}
+   */
+  async analyze(projectDir) {
+    const gaps = await this.gapDetector.detect(projectDir);
+    if (gaps.postTraining.length === 0) {
+      return { affected: [], totalFiles: 0, totalApis: 0, gaps };
+    }
+
+    const postTrainingDeps = new Map();
+    // Exclude the project itself so its own source files aren't flagged as "affected"
+    const selfName = readProjectName(projectDir);
+    for (const dep of gaps.postTraining) {
+      const normalized = normalizePkgName(dep.name);
+      if (selfName && normalized === selfName) continue;
+
+      postTrainingDeps.set(normalized, dep);
+
+      // Maven/Gradle deps use groupId:artifactId (e.g. com.google.code.gson:gson).
+      // Java imports use dot-notation (e.g. com.google.gson.Gson -> pkg com.google.gson).
+      // Add artifactId and groupId as extra keys so the lookup can match.
+      if (dep.name.includes(':')) {
+        const [groupId, artifactId] = dep.name.split(':');
+        if (artifactId) postTrainingDeps.set(normalizePkgName(artifactId), dep);
+        if (groupId) postTrainingDeps.set(normalizePkgName(groupId), dep);
+      }
+    }
+
+    const sourceFiles = findSourceFiles(projectDir);
+    const affected = [];
+
+    for (const filePath of sourceFiles) {
+      const language = detectLanguage(filePath);
+      if (!language) continue;
+
+      let code;
+      try {
+        const stat = fs.statSync(filePath);
+        if (stat.size > MAX_FILE_SIZE) continue;
+        code = fs.readFileSync(filePath, 'utf8');
+      } catch { continue; }
+
+      const imports = parseImports(code, language);
+
+      for (const imp of imports) {
+        const normalized = normalizePkgName(imp.module);
+        let dep = postTrainingDeps.get(normalized);
+
+        // For Java/Kotlin imports like com.google.gson, also try the last segment
+        // (e.g. "gson") which may match a Maven artifactId key.
+        if (!dep && (language === 'java' || language === 'kotlin') && imp.module.includes('.')) {
+          const lastSegment = imp.module.split('.').pop();
+          dep = postTrainingDeps.get(normalizePkgName(lastSegment));
+        }
+
+        // PHP: `illuminate/support` (from import) may not match dep `laravel/framework`
+        // directly. Check if any dep name contains the import as a segment.
+        if (!dep && language === 'php') {
+          const importSegment = normalized.split('-')[0]; // first segment, e.g. "illuminate"
+          for (const [depKey, depVal] of postTrainingDeps) {
+            if (depKey.includes(importSegment)) {
+              dep = depVal;
+              break;
+            }
+          }
+        }
+
+        if (!dep) continue;
+
+        const apis = extractApiNames(code, imp.module, language);
+        const relPath = path.relative(projectDir, filePath);
+
+        const existing = affected.find(a => a.file === relPath && a.dep.name === dep.name);
+        if (existing) {
+          for (const api of apis) {
+            if (!existing.apis.includes(api)) existing.apis.push(api);
+          }
+        } else {
+          affected.push({
+            file: relPath,
+            dep: {
+              name: dep.name,
+              version: dep.version,
+              ecosystem: dep.ecosystem,
+              publishDate: dep.publishDate,
+            },
+            apis,
+          });
+        }
+      }
+    }
+
+    // Separate icon library imports from framework APIs
+    let iconApis = 0;
+    let frameworkApis = 0;
+    for (const entry of affected) {
+      const depName = normalizePkgName(entry.dep.name);
+      const isIcon = ICON_LIBRARIES.has(entry.dep.name) ||
+        [...ICON_LIBRARIES].some(lib => depName === normalizePkgName(lib));
+      entry.isIconLibrary = isIcon;
+      if (isIcon) {
+        iconApis += entry.apis.length;
+      } else {
+        frameworkApis += entry.apis.length;
+      }
+    }
+
+    const totalApis = frameworkApis;
+    return { affected, totalFiles: affected.length, totalApis, iconApis, frameworkApis, gaps };
+  }
+}
+
+/**
+ * Read the project's own name from the root manifest file.
+ * Returns normalized name or null if not found.
+ * @param {string} projectDir
+ * @returns {string|null}
+ */
+function readProjectName(projectDir) {
+  // package.json (npm)
+  try {
+    const pkg = JSON.parse(fs.readFileSync(path.join(projectDir, 'package.json'), 'utf8'));
+    if (pkg.name) return normalizePkgName(pkg.name);
+  } catch { /* no package.json */ }
+
+  // pyproject.toml (Python)
+  try {
+    const toml = fs.readFileSync(path.join(projectDir, 'pyproject.toml'), 'utf8');
+    const nameMatch = toml.match(/\[project\][\s\S]*?name\s*=\s*"([^"]+)"/);
+    if (nameMatch) return normalizePkgName(nameMatch[1]);
+    const poetryMatch = toml.match(/\[tool\.poetry\][\s\S]*?name\s*=\s*"([^"]+)"/);
+    if (poetryMatch) return normalizePkgName(poetryMatch[1]);
+  } catch { /* no pyproject.toml */ }
+
+  // Cargo.toml (Rust)
+  try {
+    const cargo = fs.readFileSync(path.join(projectDir, 'Cargo.toml'), 'utf8');
+    const nameMatch = cargo.match(/\[package\][\s\S]*?name\s*=\s*"([^"]+)"/);
+    if (nameMatch) return normalizePkgName(nameMatch[1]);
+  } catch { /* no Cargo.toml */ }
+
+  return null;
+}
+
+/**
+ * Normalize package name for cross-ecosystem matching.
+ * Python deps use hyphens (typing-inspection) but imports use underscores (typing_inspection).
+ * Rust crates have the same mismatch (tokio-tungstenite vs tokio_tungstenite).
+ * Ruby requires use slashes (dry/types) but gems use hyphens (dry-types).
+ */
+function normalizePkgName(name) {
+  return name.toLowerCase().replace(/[-_/]+/g, '-');
+}
+
+/** Escape special regex characters in a string. */
+function escapeRegex(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Extract specific API/function names imported from a module.
+ * Looks for destructured imports like: import { cacheLife, after } from 'next/cache'
+ * @param {string} code - full source code
+ * @param {string} moduleName - the top-level package name
+ * @param {string} language - language key
+ * @returns {string[]} specific API names, or [moduleName] if none found
+ */
+function extractApiNames(code, moduleName, language) {
+  const apis = [];
+
+  if (language === 'js') {
+    // Destructured: import { X, Y } from 'module' or 'module/subpath'
+    const destructuredRe = new RegExp(
+      `import\\s*\\{([^}]+)\\}\\s*from\\s*['"]${escapeRegex(moduleName)}(?:/[^'"]*)?['"]`,
+      'gs',
+    );
+    let match;
+    while ((match = destructuredRe.exec(code)) !== null) {
+      const names = match[1].split(',').map(n => n.trim().split(/\s+as\s+/)[0].trim()).filter(Boolean);
+      apis.push(...names);
+    }
+
+    // Default: import React from 'react'
+    const defaultRe = new RegExp(
+      `import\\s+(\\w+)\\s+from\\s*['"]${escapeRegex(moduleName)}['"]`,
+      'g',
+    );
+    while ((match = defaultRe.exec(code)) !== null) {
+      apis.push(match[1]);
+    }
+
+    // Require destructured: const { X, Y } = require('module')
+    const requireRe = new RegExp(
+      `\\{([^}]+)\\}\\s*=\\s*require\\s*\\(\\s*['"]${escapeRegex(moduleName)}['"]`,
+      'g',
+    );
+    while ((match = requireRe.exec(code)) !== null) {
+      const names = match[1].split(',').map(n => n.trim().split(/\s+as\s+/)[0].trim()).filter(Boolean);
+      apis.push(...names);
+    }
+  }
+
+  if (language === 'python') {
+    // from module import X, Y, Z
+    const fromRe = new RegExp(
+      `from\\s+${escapeRegex(moduleName)}(?:\\.\\S+)?\\s+import\\s+([^\\n]+)`,
+      'g',
+    );
+    let match;
+    while ((match = fromRe.exec(code)) !== null) {
+      const names = match[1].split(',').map(n => n.trim().split(/\s+as\s+/)[0].trim()).filter(Boolean);
+      apis.push(...names);
+    }
+  }
+
+  if (language === 'go') {
+    // Match pkg.FuncName( usage — extract FuncName
+    const lastPart = moduleName.split('/').pop();
+    const goRe = new RegExp(`${escapeRegex(lastPart)}\\.(\\w+)`, 'g');
+    let match;
+    while ((match = goRe.exec(code)) !== null) {
+      apis.push(match[1]);
+    }
+  }
+
+  if (language === 'rust') {
+    // use crate::module::{Type1, Type2}
+    const rustRe = new RegExp(
+      `use\\s+${escapeRegex(moduleName)}(?:::\\w+)*::\\{([^}]+)\\}`,
+      'gs',
+    );
+    let match;
+    while ((match = rustRe.exec(code)) !== null) {
+      const names = match[1].split(',').map(n => n.trim()).filter(Boolean);
+      apis.push(...names);
+    }
+    // use module::SpecificType;
+    const rustSingleRe = new RegExp(
+      `use\\s+${escapeRegex(moduleName)}(?:::\\w+)*::(\\w+)\\s*;`,
+      'g',
+    );
+    while ((match = rustSingleRe.exec(code)) !== null) {
+      apis.push(match[1]);
+    }
+  }
+
+  if (language === 'java' || language === 'kotlin') {
+    // import com.google.gson.Gson — extract Gson (last segment)
+    const javaRe = new RegExp(
+      `import\\s+(?:static\\s+)?${escapeRegex(moduleName)}\\.([\\w.]+)`,
+      'gm',
+    );
+    let match;
+    while ((match = javaRe.exec(code)) !== null) {
+      // Take the last segment: com.google.gson.Gson -> Gson
+      const segments = match[1].split('.');
+      apis.push(segments[segments.length - 1]);
+    }
+  }
+
+  if (language === 'php') {
+    // use Namespace\Class; — extract Class (last segment)
+    // use Namespace\Class as Alias; — extract Alias
+    const phpRe = /use\s+[\w\\]+\\(\w+)(?:\s+as\s+(\w+))?\s*;/g;
+    let match;
+    while ((match = phpRe.exec(code)) !== null) {
+      apis.push(match[2] || match[1]);
+    }
+    // use Namespace\{Class1, Class2}; — extract all
+    const phpGroupRe = /use\s+[\w\\]+\\{([^}]+)}/g;
+    while ((match = phpGroupRe.exec(code)) !== null) {
+      const names = match[1].split(',').map(n => n.trim().split(/\s+as\s+/).pop().trim()).filter(Boolean);
+      apis.push(...names);
+    }
+  }
+
+  const unique = [...new Set(apis)];
+  return unique.length > 0 ? unique : [moduleName];
+}
+
+/**
+ * Recursively find source code files in a project directory.
+ * Skips node_modules, .git, .booklib, dist, build, vendor, etc.
+ * Caps at MAX_FILES for performance.
+ * @param {string} dirPath
+ * @returns {string[]}
+ */
+function findSourceFiles(dirPath) {
+  const results = [];
+
+  const walk = (dir) => {
+    if (results.length >= MAX_FILES) return;
+    let entries;
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      if (results.length >= MAX_FILES) return;
+      if (entry.isDirectory()) {
+        if (!SKIP_DIRS.has(entry.name)) walk(path.join(dir, entry.name));
+      } else if (CODE_EXTENSIONS.test(entry.name)) {
+        results.push(path.join(dir, entry.name));
+      }
+    }
+  };
+  walk(dirPath);
+  return results;
+}
+
+export { findSourceFiles, extractApiNames, normalizePkgName, readProjectName };

package/lib/engine/query-expander.js

@@ -0,0 +1,42 @@
+export const STOPWORDS = new Set([
+  'a', 'an', 'the', 'in', 'on', 'at', 'to', 'for', 'of', 'and', 'or',
+  'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had',
+  'do', 'does', 'did', 'will', 'would', 'shall', 'should', 'may', 'might',
+  'must', 'can', 'could', 'not', 'no', 'nor', 'so', 'yet', 'both', 'with',
+  'about', 'from', 'up', 'down', 'out', 'how', 'what', 'when', 'where',
+  'who', 'why', 'which', 'by', 'as', 'if', 'then', 'than', 'too', 'very',
+  'just', 'more', 'also', 'its', 'it',
+]);
+
+export function extractKeywords(query) {
+  const tokens = query
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(token => token.length > 1 && !STOPWORDS.has(token));
+
+  return tokens;
+}
+
+export function expandQuery(query) {
+  const keywords = extractKeywords(query);
+
+  const expanded = [];
+
+  if (keywords.length > 0) {
+    const keywordJoined = keywords.join(' ');
+    const candidates = [
+      keywordJoined,
+      `best practices for ${keywordJoined}`,
+      `how to ${keywordJoined}`,
+    ];
+    for (const v of candidates) {
+      if (v !== query) expanded.push(v);
+    }
+  }
+
+  return {
+    original: query,
+    keywords,
+    expanded,
+  };
+}