@optave/codegraph 2.3.0 → 2.3.1-dev.1aeea34

package/src/cochange.js

@@ -0,0 +1,498 @@
+/**
+ * Git co-change analysis — surfaces files that historically change together.
+ *
+ * Uses git log to find temporal coupling between files, computes Jaccard
+ * similarity coefficients, and stores results in the codegraph database.
+ */
+
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { normalizePath } from './constants.js';
+import { findDbPath, initSchema, openDb, openReadonlyOrFail } from './db.js';
+import { warn } from './logger.js';
+import { isTestFile } from './queries.js';
+
+/**
+ * Scan git history and return parsed commit data.
+ * @param {string} repoRoot - Absolute path to the git repo root
+ * @param {object} [opts]
+ * @param {string} [opts.since] - Git date expression (e.g. "1 year ago")
+ * @param {string} [opts.afterSha] - Only include commits after this SHA
+ * @returns {{ commits: Array<{sha: string, epoch: number, files: string[]}> }}
+ */
+export function scanGitHistory(repoRoot, opts = {}) {
+  const args = [
+    'log',
+    '--name-only',
+    '--pretty=format:%H%n%at',
+    '--no-merges',
+    '--diff-filter=AMRC',
+  ];
+  if (opts.since) args.push(`--since=${opts.since}`);
+  if (opts.afterSha) args.push(`${opts.afterSha}..HEAD`);
+  args.push('--', '.');
+
+  let output;
+  try {
+    output = execFileSync('git', args, {
+      cwd: repoRoot,
+      encoding: 'utf-8',
+      maxBuffer: 50 * 1024 * 1024,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+  } catch (e) {
+    warn(`Failed to scan git history: ${e.message}`);
+    return { commits: [] };
+  }
+
+  if (!output.trim()) return { commits: [] };
+
+  const commits = [];
+  // Split on double newlines to get blocks; each block is sha\nepoch\nfile1\nfile2...
+  const blocks = output.trim().split(/\n\n+/);
+  for (const block of blocks) {
+    const lines = block.split('\n').filter((l) => l.length > 0);
+    if (lines.length < 2) continue;
+    const sha = lines[0];
+    const epoch = parseInt(lines[1], 10);
+    if (Number.isNaN(epoch)) continue;
+    const files = lines.slice(2).map((f) => normalizePath(f));
+    if (files.length > 0) {
+      commits.push({ sha, epoch, files });
+    }
+  }
+
+  return { commits };
+}
+
+/**
+ * Compute co-change pairs from parsed commit data.
+ * @param {Array<{sha: string, epoch: number, files: string[]}>} commits
+ * @param {object} [opts]
+ * @param {number} [opts.minSupport=3] - Minimum number of co-occurrences
+ * @param {number} [opts.maxFilesPerCommit=50] - Skip commits with too many files
+ * @param {Set<string>} [opts.knownFiles] - If provided, only include pairs where both files are in this set
+ * @returns {Map<string, {commitCount: number, jaccard: number, lastEpoch: number}>}
+ */
+export function computeCoChanges(commits, opts = {}) {
+  const minSupport = opts.minSupport ?? 3;
+  const maxFilesPerCommit = opts.maxFilesPerCommit ?? 50;
+  const knownFiles = opts.knownFiles || null;
+
+  const fileCommitCounts = new Map();
+  const pairCounts = new Map();
+  const pairLastEpoch = new Map();
+
+  for (const commit of commits) {
+    let { files } = commit;
+    if (files.length > maxFilesPerCommit) continue;
+
+    if (knownFiles) {
+      files = files.filter((f) => knownFiles.has(f));
+    }
+
+    // Count per-file commits
+    for (const f of files) {
+      fileCommitCounts.set(f, (fileCommitCounts.get(f) || 0) + 1);
+    }
+
+    // Generate all unique pairs (canonical: a < b)
+    const sorted = [...new Set(files)].sort();
+    for (let i = 0; i < sorted.length; i++) {
+      for (let j = i + 1; j < sorted.length; j++) {
+        const key = `${sorted[i]}\0${sorted[j]}`;
+        pairCounts.set(key, (pairCounts.get(key) || 0) + 1);
+        const prev = pairLastEpoch.get(key) || 0;
+        if (commit.epoch > prev) pairLastEpoch.set(key, commit.epoch);
+      }
+    }
+  }
+
+  // Filter by minSupport and compute Jaccard
+  const results = new Map();
+  for (const [key, count] of pairCounts) {
+    if (count < minSupport) continue;
+    const [fileA, fileB] = key.split('\0');
+    const countA = fileCommitCounts.get(fileA) || 0;
+    const countB = fileCommitCounts.get(fileB) || 0;
+    const jaccard = count / (countA + countB - count);
+    results.set(key, {
+      commitCount: count,
+      jaccard,
+      lastEpoch: pairLastEpoch.get(key) || 0,
+    });
+  }
+
+  return { pairs: results, fileCommitCounts };
+}
+
+/**
+ * Analyze git history and populate co-change data in the database.
+ * @param {string} [customDbPath] - Path to graph.db
+ * @param {object} [opts]
+ * @param {string} [opts.since] - Git date expression
+ * @param {number} [opts.minSupport] - Minimum co-occurrence count
+ * @param {number} [opts.maxFilesPerCommit] - Max files per commit
+ * @param {boolean} [opts.full] - Force full re-scan
+ * @returns {{ pairsFound: number, commitsScanned: number, since: string, minSupport: number }}
+ */
+export function analyzeCoChanges(customDbPath, opts = {}) {
+  const dbPath = findDbPath(customDbPath);
+  const db = openDb(dbPath);
+  initSchema(db);
+
+  const repoRoot = path.resolve(path.dirname(dbPath), '..');
+
+  if (!fs.existsSync(path.join(repoRoot, '.git'))) {
+    db.close();
+    return { error: `Not a git repository: ${repoRoot}` };
+  }
+
+  const since = opts.since || '1 year ago';
+  const minSupport = opts.minSupport ?? 3;
+  const maxFilesPerCommit = opts.maxFilesPerCommit ?? 50;
+
+  // Check for incremental state
+  let afterSha = null;
+  if (!opts.full) {
+    try {
+      const row = db
+        .prepare("SELECT value FROM co_change_meta WHERE key = 'last_analyzed_commit'")
+        .get();
+      if (row) afterSha = row.value;
+    } catch {
+      /* table may not exist yet */
+    }
+  }
+
+  // If full re-scan, clear existing data
+  if (opts.full) {
+    db.exec('DELETE FROM co_changes');
+    db.exec('DELETE FROM co_change_meta');
+    db.exec('DELETE FROM file_commit_counts');
+  }
+
+  // Collect known files from the graph for filtering
+  let knownFiles = null;
+  try {
+    const rows = db.prepare('SELECT DISTINCT file FROM nodes').all();
+    knownFiles = new Set(rows.map((r) => r.file));
+  } catch {
+    /* nodes table may not exist */
+  }
+
+  const { commits } = scanGitHistory(repoRoot, { since, afterSha });
+  const { pairs: coChanges, fileCommitCounts } = computeCoChanges(commits, {
+    minSupport,
+    maxFilesPerCommit,
+    knownFiles,
+  });
+
+  // Upsert per-file commit counts so Jaccard can be recomputed from totals
+  const fileCountUpsert = db.prepare(`
+    INSERT INTO file_commit_counts (file, commit_count) VALUES (?, ?)
+    ON CONFLICT(file) DO UPDATE SET commit_count = commit_count + excluded.commit_count
+  `);
+
+  // Upsert pair counts (accumulate commit_count, jaccard placeholder — recomputed below)
+  const pairUpsert = db.prepare(`
+    INSERT INTO co_changes (file_a, file_b, commit_count, jaccard, last_commit_epoch)
+    VALUES (?, ?, ?, 0, ?)
+    ON CONFLICT(file_a, file_b) DO UPDATE SET
+      commit_count = commit_count + excluded.commit_count,
+      last_commit_epoch = MAX(co_changes.last_commit_epoch, excluded.last_commit_epoch)
+  `);
+
+  const insertMany = db.transaction(() => {
+    for (const [file, count] of fileCommitCounts) {
+      fileCountUpsert.run(file, count);
+    }
+    for (const [key, data] of coChanges) {
+      const [fileA, fileB] = key.split('\0');
+      pairUpsert.run(fileA, fileB, data.commitCount, data.lastEpoch);
+    }
+  });
+  insertMany();
+
+  // Recompute Jaccard for all affected pairs from total file commit counts
+  const affectedFiles = [...fileCommitCounts.keys()];
+  if (affectedFiles.length > 0) {
+    const ph = affectedFiles.map(() => '?').join(',');
+    db.prepare(`
+      UPDATE co_changes SET jaccard = (
+        SELECT CAST(co_changes.commit_count AS REAL) / (
+          COALESCE(fa.commit_count, 0) + COALESCE(fb.commit_count, 0) - co_changes.commit_count
+        )
+        FROM file_commit_counts fa, file_commit_counts fb
+        WHERE fa.file = co_changes.file_a AND fb.file = co_changes.file_b
+      )
+      WHERE file_a IN (${ph}) OR file_b IN (${ph})
+    `).run(...affectedFiles, ...affectedFiles);
+  }
+
+  // Update metadata
+  const metaUpsert = db.prepare(`
+    INSERT INTO co_change_meta (key, value) VALUES (?, ?)
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+  `);
+  if (commits.length > 0) {
+    metaUpsert.run('last_analyzed_commit', commits[0].sha);
+  }
+  metaUpsert.run('analyzed_at', new Date().toISOString());
+  metaUpsert.run('since', since);
+  metaUpsert.run('min_support', String(minSupport));
+
+  const totalPairs = db.prepare('SELECT COUNT(*) as cnt FROM co_changes').get().cnt;
+
+  db.close();
+
+  return {
+    pairsFound: totalPairs,
+    commitsScanned: commits.length,
+    since,
+    minSupport,
+  };
+}
+
+/**
+ * Query co-change partners for a specific file.
+ * @param {string} file - File path (partial match supported)
+ * @param {string} [customDbPath]
+ * @param {object} [opts]
+ * @param {number} [opts.limit=20]
+ * @param {number} [opts.minJaccard=0.3]
+ * @param {boolean} [opts.noTests]
+ * @returns {{ file: string, partners: Array, meta: object }}
+ */
+export function coChangeData(file, customDbPath, opts = {}) {
+  const db = openReadonlyOrFail(customDbPath);
+  const limit = opts.limit || 20;
+  const minJaccard = opts.minJaccard ?? 0.3;
+  const noTests = opts.noTests || false;
+
+  // Check if co_changes table exists
+  try {
+    db.prepare('SELECT 1 FROM co_changes LIMIT 1').get();
+  } catch {
+    db.close();
+    return { error: 'No co-change data found. Run `codegraph co-change --analyze` first.' };
+  }
+
+  // Resolve file via partial match
+  const resolvedFile = resolveCoChangeFile(db, file);
+  if (!resolvedFile) {
+    db.close();
+    return { error: `No co-change data found for file matching "${file}"` };
+  }
+
+  const rows = db
+    .prepare(
+      `SELECT file_a, file_b, commit_count, jaccard, last_commit_epoch
+       FROM co_changes
+       WHERE (file_a = ? OR file_b = ?) AND jaccard >= ?
+       ORDER BY jaccard DESC`,
+    )
+    .all(resolvedFile, resolvedFile, minJaccard);
+
+  const partners = [];
+  for (const row of rows) {
+    const partner = row.file_a === resolvedFile ? row.file_b : row.file_a;
+    if (noTests && isTestFile(partner)) continue;
+    partners.push({
+      file: partner,
+      commitCount: row.commit_count,
+      jaccard: row.jaccard,
+      lastCommitDate: row.last_commit_epoch
+        ? new Date(row.last_commit_epoch * 1000).toISOString().slice(0, 10)
+        : null,
+    });
+    if (partners.length >= limit) break;
+  }
+
+  const meta = getCoChangeMeta(db);
+  db.close();
+
+  return { file: resolvedFile, partners, meta };
+}
+
+/**
+ * Query top global co-change pairs.
+ * @param {string} [customDbPath]
+ * @param {object} [opts]
+ * @param {number} [opts.limit=20]
+ * @param {number} [opts.minJaccard=0.3]
+ * @param {boolean} [opts.noTests]
+ * @returns {{ pairs: Array, meta: object }}
+ */
+export function coChangeTopData(customDbPath, opts = {}) {
+  const db = openReadonlyOrFail(customDbPath);
+  const limit = opts.limit || 20;
+  const minJaccard = opts.minJaccard ?? 0.3;
+  const noTests = opts.noTests || false;
+
+  try {
+    db.prepare('SELECT 1 FROM co_changes LIMIT 1').get();
+  } catch {
+    db.close();
+    return { error: 'No co-change data found. Run `codegraph co-change --analyze` first.' };
+  }
+
+  const rows = db
+    .prepare(
+      `SELECT file_a, file_b, commit_count, jaccard, last_commit_epoch
+       FROM co_changes
+       WHERE jaccard >= ?
+       ORDER BY jaccard DESC`,
+    )
+    .all(minJaccard);
+
+  const pairs = [];
+  for (const row of rows) {
+    if (noTests && (isTestFile(row.file_a) || isTestFile(row.file_b))) continue;
+    pairs.push({
+      fileA: row.file_a,
+      fileB: row.file_b,
+      commitCount: row.commit_count,
+      jaccard: row.jaccard,
+      lastCommitDate: row.last_commit_epoch
+        ? new Date(row.last_commit_epoch * 1000).toISOString().slice(0, 10)
+        : null,
+    });
+    if (pairs.length >= limit) break;
+  }
+
+  const meta = getCoChangeMeta(db);
+  db.close();
+
+  return { pairs, meta };
+}
+
+/**
+ * Batch-query co-change partners for a set of files.
+ * Takes an already-open readonly DB handle (for diff-impact integration).
+ * @param {string[]} files - File paths to query
+ * @param {import('better-sqlite3').Database} db - Already-open DB handle
+ * @param {object} [opts]
+ * @param {number} [opts.minJaccard=0.3]
+ * @param {number} [opts.limit=20]
+ * @param {boolean} [opts.noTests]
+ * @returns {Array<{file: string, coupledWith: string, commitCount: number, jaccard: number}>}
+ */
+export function coChangeForFiles(files, db, opts = {}) {
+  const minJaccard = opts.minJaccard ?? 0.3;
+  const limit = opts.limit ?? 20;
+  const noTests = opts.noTests || false;
+  const inputSet = new Set(files);
+
+  if (files.length === 0) return [];
+
+  const placeholders = files.map(() => '?').join(',');
+  const rows = db
+    .prepare(
+      `SELECT file_a, file_b, commit_count, jaccard
+       FROM co_changes
+       WHERE (file_a IN (${placeholders}) OR file_b IN (${placeholders}))
+         AND jaccard >= ?
+       ORDER BY jaccard DESC
+       LIMIT ?`,
+    )
+    .all(...files, ...files, minJaccard, limit);
+
+  const results = [];
+  for (const row of rows) {
+    const partner = inputSet.has(row.file_a) ? row.file_b : row.file_a;
+    const source = inputSet.has(row.file_a) ? row.file_a : row.file_b;
+    if (inputSet.has(partner)) continue;
+    if (noTests && isTestFile(partner)) continue;
+    results.push({
+      file: partner,
+      coupledWith: source,
+      commitCount: row.commit_count,
+      jaccard: row.jaccard,
+    });
+  }
+
+  return results;
+}
+
+/**
+ * Format co-change data for CLI output (single file).
+ */
+export function formatCoChange(data) {
+  if (data.error) return data.error;
+  if (data.partners.length === 0) return `No co-change partners found for ${data.file}`;
+
+  const lines = [`\nCo-change partners for ${data.file}:\n`];
+  for (const p of data.partners) {
+    const pct = `${(p.jaccard * 100).toFixed(0)}%`.padStart(4);
+    const commits = `${p.commitCount} commits`.padStart(12);
+    lines.push(`  ${pct}  ${commits}  ${p.file}`);
+  }
+  if (data.meta?.analyzedAt) {
+    lines.push(`\n  Analyzed: ${data.meta.analyzedAt} | Window: ${data.meta.since || 'all'}`);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Format top co-change pairs for CLI output (global view).
+ */
+export function formatCoChangeTop(data) {
+  if (data.error) return data.error;
+  if (data.pairs.length === 0) return 'No co-change pairs found.';
+
+  const lines = ['\nTop co-change pairs:\n'];
+  for (const p of data.pairs) {
+    const pct = `${(p.jaccard * 100).toFixed(0)}%`.padStart(4);
+    const commits = `${p.commitCount} commits`.padStart(12);
+    lines.push(`  ${pct}  ${commits}  ${p.fileA}  <->  ${p.fileB}`);
+  }
+  if (data.meta?.analyzedAt) {
+    lines.push(`\n  Analyzed: ${data.meta.analyzedAt} | Window: ${data.meta.since || 'all'}`);
+  }
+  return lines.join('\n');
+}
+
+// ─── Internal Helpers ────────────────────────────────────────────────────
+
+function resolveCoChangeFile(db, file) {
+  // Exact match first
+  const exact = db
+    .prepare(
+      'SELECT file_a FROM co_changes WHERE file_a = ? UNION SELECT file_b FROM co_changes WHERE file_b = ? LIMIT 1',
+    )
+    .get(file, file);
+  if (exact) return exact.file_a;
+
+  // Partial match (ends with)
+  const partial = db
+    .prepare(
+      `SELECT file_a AS file FROM co_changes WHERE file_a LIKE ?
+       UNION
+       SELECT file_b AS file FROM co_changes WHERE file_b LIKE ?
+       LIMIT 1`,
+    )
+    .get(`%${file}`, `%${file}`);
+  if (partial) return partial.file;
+
+  return null;
+}
+
+function getCoChangeMeta(db) {
+  try {
+    const rows = db.prepare('SELECT key, value FROM co_change_meta').all();
+    const meta = {};
+    for (const row of rows) {
+      meta[row.key] = row.value;
+    }
+    return {
+      analyzedAt: meta.analyzed_at || null,
+      since: meta.since || null,
+      minSupport: meta.min_support ? parseInt(meta.min_support, 10) : null,
+      lastCommit: meta.last_analyzed_commit || null,
+    };
+  } catch {
+    return null;
+  }
+}

package/src/config.js

@@ -24,6 +24,12 @@ export const DEFAULTS = {
   llm: { provider: null, model: null, baseUrl: null, apiKey: null, apiKeyCommand: null },
   search: { defaultMinScore: 0.2, rrfK: 60, topK: 15 },
   ci: { failOnCycles: false, impactThreshold: null },
+  coChange: {
+    since: '1 year ago',
+    minSupport: 3,
+    minJaccard: 0.3,
+    maxFilesPerCommit: 50,
+  },
 };
 
 /**

package/src/db.js

@@ -71,6 +71,36 @@ export const MIGRATIONS = [
     version: 4,
     up: `ALTER TABLE file_hashes ADD COLUMN size INTEGER DEFAULT 0;`,
   },
+  {
+    version: 5,
+    up: `
+      CREATE TABLE IF NOT EXISTS co_changes (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        file_a TEXT NOT NULL,
+        file_b TEXT NOT NULL,
+        commit_count INTEGER NOT NULL,
+        jaccard REAL NOT NULL,
+        last_commit_epoch INTEGER,
+        UNIQUE(file_a, file_b)
+      );
+      CREATE INDEX IF NOT EXISTS idx_co_changes_file_a ON co_changes(file_a);
+      CREATE INDEX IF NOT EXISTS idx_co_changes_file_b ON co_changes(file_b);
+      CREATE INDEX IF NOT EXISTS idx_co_changes_jaccard ON co_changes(jaccard DESC);
+      CREATE TABLE IF NOT EXISTS co_change_meta (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+    `,
+  },
+  {
+    version: 6,
+    up: `
+      CREATE TABLE IF NOT EXISTS file_commit_counts (
+        file TEXT PRIMARY KEY,
+        commit_count INTEGER NOT NULL DEFAULT 0
+      );
+    `,
+  },
 ];
 
 export function openDb(dbPath) {
@@ -115,6 +145,16 @@ export function initSchema(db) {
   } catch {
     /* already exists */
   }
+  try {
+    db.exec('ALTER TABLE nodes ADD COLUMN role TEXT');
+  } catch {
+    /* already exists */
+  }
+  try {
+    db.exec('CREATE INDEX IF NOT EXISTS idx_nodes_role ON nodes(role)');
+  } catch {
+    /* already exists */
+  }
 }
 
 export function findDbPath(customPath) {

package/src/embedder.js

@@ -16,6 +16,28 @@ function splitIdentifier(name) {
     .trim();
 }
 
+/**
+ * Match a file path against a glob pattern.
+ * Supports *, **, and ? wildcards. Zero dependencies.
+ */
+function globMatch(filePath, pattern) {
+  // Normalize separators to forward slashes
+  const normalized = filePath.replace(/\\/g, '/');
+  // Escape regex specials except glob chars
+  let regex = pattern.replace(/\\/g, '/').replace(/[.+^${}()|[\]\\]/g, '\\$&');
+  // Replace ** first (matches any path segment), then * and ?
+  regex = regex.replace(/\*\*/g, '\0');
+  regex = regex.replace(/\*/g, '[^/]*');
+  regex = regex.replace(/\0/g, '.*');
+  regex = regex.replace(/\?/g, '[^/]');
+  try {
+    return new RegExp(`^${regex}$`).test(normalized);
+  } catch {
+    // Malformed pattern — fall back to substring match
+    return normalized.includes(pattern);
+  }
+}
+
 // Lazy-load transformers (heavy, optional module)
 let pipeline = null;
 let _cos_sim = null;
@@ -76,7 +98,7 @@ export const MODELS = {
 
 export const EMBEDDING_STRATEGIES = ['structured', 'source'];
 
-export const DEFAULT_MODEL = 'minilm';
+export const DEFAULT_MODEL = 'nomic-v1.5';
 const BATCH_SIZE_MAP = {
   minilm: 32,
   'jina-small': 16,
@@ -216,11 +238,26 @@ async function loadTransformers() {
   }
 }
 
+/**
+ * Dispose the current ONNX session and free memory.
+ * Safe to call when no model is loaded (no-op).
+ */
+export async function disposeModel() {
+  if (extractor) {
+    await extractor.dispose();
+    extractor = null;
+  }
+  activeModel = null;
+}
+
 async function loadModel(modelKey) {
   const config = getModelConfig(modelKey);
 
   if (extractor && activeModel === config.name) return { extractor, config };
 
+  // Dispose previous model before loading a different one
+  await disposeModel();
+
   const transformers = await loadTransformers();
   pipeline = transformers.pipeline;
   _cos_sim = transformers.cos_sim;
@@ -496,7 +533,8 @@ function _prepareSearch(customDbPath, opts = {}) {
     conditions.push('n.kind = ?');
     params.push(opts.kind);
   }
-  if (opts.filePattern) {
+  const isGlob = opts.filePattern && /[*?[\]]/.test(opts.filePattern);
+  if (opts.filePattern && !isGlob) {
     conditions.push('n.file LIKE ?');
     params.push(`%${opts.filePattern}%`);
   }
@@ -505,6 +543,9 @@ function _prepareSearch(customDbPath, opts = {}) {
   }
 
   let rows = db.prepare(sql).all(...params);
+  if (isGlob) {
+    rows = rows.filter((row) => globMatch(row.file, opts.filePattern));
+  }
   if (noTests) {
     rows = rows.filter((row) => !TEST_PATTERN.test(row.file));
   }
@@ -668,6 +709,11 @@ export async function search(query, customDbPath, opts = {}) {
     const data = await searchData(singleQuery, customDbPath, opts);
     if (!data) return;
 
+    if (opts.json) {
+      console.log(JSON.stringify(data, null, 2));
+      return;
+    }
+
     console.log(`\nSemantic search: "${singleQuery}"\n`);
 
     if (data.results.length === 0) {
@@ -687,6 +733,11 @@ export async function search(query, customDbPath, opts = {}) {
     const data = await multiSearchData(queries, customDbPath, opts);
     if (!data) return;
 
+    if (opts.json) {
+      console.log(JSON.stringify(data, null, 2));
+      return;
+    }
+
     console.log(`\nMulti-query semantic search (RRF, k=${opts.rrfK || 60}):`);
     queries.forEach((q, i) => {
       console.log(`  [${i + 1}] "${q}"`);