forge-server 0.1.0

package/src/ingestion/git-analyzer.ts

@@ -0,0 +1,545 @@
+/**
+ * Git history analyzer for co-modification edges (Phase 2 full implementation).
+ * Runs `git log --name-only` to find files changed together frequently,
+ * creates CO_MODIFIED edges, and detects test file relationships.
+ *
+ * Phase 3 additions:
+ *   - analyzeFileStats(): per-file commit count, stability, velocity
+ *   - extractCommitRecords(): parse git log into CommitRecord[]
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { logger } from '../util/logger.js';
+import type { GraphStore } from '../storage/interfaces.js';
+import type { FileChangeStats, CommitRecord } from '../util/types.js';
+import { UPSERT_CO_MODIFIED } from '../query/graph-queries.js';
+
+const execAsync = promisify(exec);
+
+export interface CoModificationPair {
+  fileA: string;
+  fileB: string;
+  count: number;
+  lastCommit: string;
+}
+
+export interface GitAnalysisResult {
+  coModifications: CoModificationPair[];
+  testPairs: Array<{ testFile: string; sourceFile: string }>;
+  totalCommitsAnalyzed: number;
+  errors: string[];
+}
+
+/**
+ * Analyze git history to find files frequently changed together.
+ * Parses `git log --name-only` output for the last `historyDepth` commits,
+ * counts co-occurrence pairs, and returns pairs with count >= minComodCount.
+ */
+export async function analyzeCoModifications(
+  repoPath: string,
+  historyDepth: number = 500,
+  minComodCount: number = 3
+): Promise<CoModificationPair[]> {
+  logger.debug('git-analyzer: starting co-modification analysis', { repoPath, historyDepth });
+
+  let stdout: string;
+  try {
+    const { stdout: out } = await execAsync(
+      `git -C "${repoPath}" log --name-only --format="%H" -n ${historyDepth}`,
+      { maxBuffer: 50 * 1024 * 1024 } // 50 MB buffer for large repos
+    );
+    stdout = out;
+  } catch (err) {
+    logger.warn('git log failed — not a git repo or git not available', {
+      repoPath,
+      error: String(err),
+    });
+    return [];
+  }
+
+  const commitGroups = parseGitLog(stdout);
+  const pairCounts = new Map<string, { count: number; lastCommit: string }>();
+
+  for (const { hash, files } of commitGroups) {
+    // Only consider commits with 2-50 files (ignore mega-commits like reformats)
+    if (files.length < 2 || files.length > 50) continue;
+
+    // Count every ordered pair (lexicographically sorted to ensure canonical key)
+    for (let i = 0; i < files.length; i++) {
+      for (let j = i + 1; j < files.length; j++) {
+        const [a, b] = [files[i], files[j]].sort();
+        const key = `${a}\x00${b}`;
+        const existing = pairCounts.get(key);
+        if (existing) {
+          existing.count++;
+          existing.lastCommit = hash;
+        } else {
+          pairCounts.set(key, { count: 1, lastCommit: hash });
+        }
+      }
+    }
+  }
+
+  const result: CoModificationPair[] = [];
+  for (const [key, { count, lastCommit }] of pairCounts) {
+    if (count >= minComodCount) {
+      const parts = key.split('\x00');
+      const fileA = parts[0] ?? '';
+      const fileB = parts[1] ?? '';
+      result.push({ fileA, fileB, count, lastCommit });
+    }
+  }
+
+  logger.debug('git-analyzer: co-modification analysis complete', {
+    repoPath,
+    totalCommits: commitGroups.length,
+    uniquePairs: pairCounts.size,
+    qualifyingPairs: result.length,
+  });
+
+  return result;
+}
+
+/**
+ * Parse `git log --name-only --format="%H"` output into commit groups.
+ * Each group contains a commit hash and the list of files changed.
+ */
+export function parseGitLog(
+  stdout: string
+): Array<{ hash: string; files: string[] }> {
+  const groups: Array<{ hash: string; files: string[] }> = [];
+  const lines = stdout.split('\n');
+
+  let currentHash = '';
+  let currentFiles: string[] = [];
+
+  for (const rawLine of lines) {
+    const line = rawLine.trim();
+    if (!line) {
+      // Blank line separates commits; flush current group
+      if (currentHash && currentFiles.length > 0) {
+        groups.push({ hash: currentHash, files: currentFiles });
+        currentFiles = [];
+      }
+      continue;
+    }
+
+    // A commit hash is exactly 40 hex chars
+    if (/^[0-9a-f]{40}$/i.test(line)) {
+      // Flush previous group before starting a new commit
+      if (currentHash && currentFiles.length > 0) {
+        groups.push({ hash: currentHash, files: currentFiles });
+        currentFiles = [];
+      }
+      currentHash = line;
+    } else {
+      // This is a file name line
+      if (currentHash) {
+        currentFiles.push(line);
+      }
+    }
+  }
+
+  // Flush final group
+  if (currentHash && currentFiles.length > 0) {
+    groups.push({ hash: currentHash, files: currentFiles });
+  }
+
+  return groups;
+}
+
+/**
+ * Detect test files and their corresponding source files.
+ * Returns pairs: { testFile, sourceFile }
+ *
+ * Handles patterns:
+ *   foo.spec.ts  -> foo.ts
+ *   foo.test.ts  -> foo.ts
+ *   __tests__/foo.ts -> ../foo.ts (sibling detection)
+ */
+export async function detectTestFiles(
+  files: string[]
+): Promise<Array<{ testFile: string; sourceFile: string }>> {
+  const fileSet = new Set(files);
+  const pairs: Array<{ testFile: string; sourceFile: string }> = [];
+
+  for (const file of files) {
+    // Pattern 1: foo.spec.{ext} -> foo.{ext}
+    const specMatch = file.match(/^(.+)\.spec\.(ts|tsx|js|jsx|mts|cts)$/);
+    if (specMatch) {
+      const [, base, ext] = specMatch;
+      const candidate = `${base}.${ext}`;
+      if (fileSet.has(candidate)) {
+        pairs.push({ testFile: file, sourceFile: candidate });
+      }
+      continue;
+    }
+
+    // Pattern 2: foo.test.{ext} -> foo.{ext}
+    const testMatch = file.match(/^(.+)\.test\.(ts|tsx|js|jsx|mts|cts)$/);
+    if (testMatch) {
+      const [, base, ext] = testMatch;
+      const candidate = `${base}.${ext}`;
+      if (fileSet.has(candidate)) {
+        pairs.push({ testFile: file, sourceFile: candidate });
+      }
+      continue;
+    }
+
+    // Pattern 3: __tests__/foo.ts -> ../foo.ts (try parent dir)
+    const testsMatch = file.match(/^(.*\/)__tests__\/([^/]+)$/);
+    if (testsMatch) {
+      const [, dir, filename] = testsMatch;
+      const candidate = `${dir}${filename}`;
+      if (fileSet.has(candidate)) {
+        pairs.push({ testFile: file, sourceFile: candidate });
+      }
+    }
+  }
+
+  return pairs;
+}
+
+/**
+ * Run full git analysis for a repo: co-modifications + test file detection.
+ * Then write the edges to FalkorDB.
+ */
+export async function runGitAnalysis(
+  repoPath: string,
+  repoId: string,
+  allFiles: string[],
+  graphStore: GraphStore,
+  historyDepth: number = 500,
+  minComodCount: number = 3
+): Promise<GitAnalysisResult> {
+  const errors: string[] = [];
+
+  // 1. Analyze co-modifications from git log
+  let coModifications: CoModificationPair[] = [];
+  let totalCommitsAnalyzed = 0;
+
+  try {
+    coModifications = await analyzeCoModifications(repoPath, historyDepth, minComodCount);
+
+    // Count commits (re-parse log for count only)
+    try {
+      const { stdout } = await execAsync(
+        `git -C "${repoPath}" rev-list --count HEAD -n ${historyDepth}`,
+        { maxBuffer: 1024 * 1024 }
+      );
+      totalCommitsAnalyzed = parseInt(stdout.trim(), 10) || 0;
+    } catch {
+      totalCommitsAnalyzed = 0;
+    }
+  } catch (err) {
+    errors.push(`Co-modification analysis failed: ${String(err)}`);
+  }
+
+  // 2. Write CO_MODIFIED edges to graph
+  for (const pair of coModifications) {
+    try {
+      await graphStore.query(
+        UPSERT_CO_MODIFIED(pair.fileA, pair.fileB, repoId, pair.count, pair.lastCommit)
+      );
+    } catch (err) {
+      logger.warn('Failed to write CO_MODIFIED edge', {
+        fileA: pair.fileA,
+        fileB: pair.fileB,
+        error: String(err),
+      });
+    }
+  }
+
+  // 3. Detect test pairs
+  const testPairs = await detectTestFiles(allFiles);
+
+  // 4. Write TESTS edges to graph
+  for (const { testFile, sourceFile } of testPairs) {
+    try {
+      await graphStore.upsertEdge(
+        'File', { path: testFile, repo_id: repoId },
+        'TESTS',
+        {},
+        'File', { path: sourceFile, repo_id: repoId }
+      );
+    } catch (err) {
+      logger.warn('Failed to write TESTS edge', {
+        testFile,
+        sourceFile,
+        error: String(err),
+      });
+    }
+  }
+
+  logger.info('git analysis complete', {
+    repoId,
+    coModPairs: coModifications.length,
+    testPairs: testPairs.length,
+    errors: errors.length,
+  });
+
+  return { coModifications, testPairs, totalCommitsAnalyzed, errors };
+}
+
+// ============================================================
+// Phase 3: Git History Enrichment
+// ============================================================
+
+/**
+ * Analyze per-file change statistics from git history.
+ * Runs a second git log with `%H|%s|%an|%aI` format (hash, subject, author, ISO date)
+ * plus `--name-only` to map commits to files.
+ *
+ * Computes per-file:
+ *   - commitCount: total commits touching this file
+ *   - stabilityScore: 0-1, higher = fewer recent changes relative to history
+ *   - changeVelocity: -1 to +1, positive = accelerating changes
+ *   - lastCommitHash, lastCommitTs, recentMessages (last 5)
+ */
+export async function analyzeFileStats(
+  repoPath: string,
+  historyDepth: number = 500
+): Promise<Map<string, FileChangeStats>> {
+  const normalizedPath = repoPath.replace(/\\/g, '/');
+  logger.debug('git-analyzer: starting file stats analysis', { repoPath: normalizedPath, historyDepth });
+
+  let stdout: string;
+  try {
+    const { stdout: out } = await execAsync(
+      `git -C "${normalizedPath}" log --format="%H|%s|%an|%aI" --name-only -n ${historyDepth}`,
+      { maxBuffer: 50 * 1024 * 1024 }
+    );
+    stdout = out;
+  } catch (err) {
+    logger.warn('git log (file stats) failed', { repoPath: normalizedPath, error: String(err) });
+    return new Map();
+  }
+
+  // Parse into commit records with files
+  const commits = parseStatsGitLog(stdout);
+
+  // Build per-file stats
+  const fileStatsMap = new Map<string, {
+    commitCount: number;
+    commitTimestamps: number[];
+    lastCommitHash: string;
+    lastCommitTs: number;
+    recentMessages: string[];
+  }>();
+
+  for (const commit of commits) {
+    const ts = commit.timestamp;
+
+    for (const file of commit.files) {
+      const existing = fileStatsMap.get(file);
+      if (existing) {
+        existing.commitCount++;
+        existing.commitTimestamps.push(ts);
+        if (ts > existing.lastCommitTs) {
+          existing.lastCommitHash = commit.hash;
+          existing.lastCommitTs = ts;
+        }
+        // Keep only last 5 messages
+        if (existing.recentMessages.length < 5) {
+          existing.recentMessages.push(commit.message);
+        }
+      } else {
+        fileStatsMap.set(file, {
+          commitCount: 1,
+          commitTimestamps: [ts],
+          lastCommitHash: commit.hash,
+          lastCommitTs: ts,
+          recentMessages: [commit.message],
+        });
+      }
+    }
+  }
+
+  // Compute stability and velocity for each file
+  const result = new Map<string, FileChangeStats>();
+  const now = Date.now();
+
+  for (const [filePath, raw] of fileStatsMap) {
+    const stability = computeStabilityScore(raw.commitTimestamps, now);
+    const velocity = computeChangeVelocity(raw.commitTimestamps);
+
+    result.set(filePath, {
+      commitCount: raw.commitCount,
+      stabilityScore: stability,
+      changeVelocity: velocity,
+      lastCommitHash: raw.lastCommitHash,
+      lastCommitTs: raw.lastCommitTs,
+      recentMessages: raw.recentMessages,
+    });
+  }
+
+  logger.debug('git-analyzer: file stats analysis complete', {
+    repoPath: normalizedPath,
+    filesWithStats: result.size,
+    totalCommits: commits.length,
+  });
+
+  return result;
+}
+
+/**
+ * Extract commit records from git history.
+ * Parses git log into CommitRecord[], filtering merge commits and
+ * bulk reformats (commits touching >50 files).
+ */
+export async function extractCommitRecords(
+  repoPath: string,
+  historyDepth: number = 500
+): Promise<CommitRecord[]> {
+  const normalizedPath = repoPath.replace(/\\/g, '/');
+  logger.debug('git-analyzer: extracting commit records', { repoPath: normalizedPath, historyDepth });
+
+  let stdout: string;
+  try {
+    const { stdout: out } = await execAsync(
+      `git -C "${normalizedPath}" log --format="%H|%s|%an|%aI" --name-only -n ${historyDepth}`,
+      { maxBuffer: 50 * 1024 * 1024 }
+    );
+    stdout = out;
+  } catch (err) {
+    logger.warn('git log (commit records) failed', { repoPath: normalizedPath, error: String(err) });
+    return [];
+  }
+
+  const allCommits = parseStatsGitLog(stdout);
+
+  // Filter out merge commits and bulk reformats (>50 files)
+  const filtered = allCommits.filter(c => {
+    // Skip merge commits (common patterns)
+    const lowerMsg = c.message.toLowerCase();
+    if (lowerMsg.startsWith('merge ') || lowerMsg.startsWith('merge:')) return false;
+
+    // Skip bulk reformats (>50 files changed)
+    if (c.files.length > 50) return false;
+
+    // Skip commits with no files (empty commits)
+    if (c.files.length === 0) return false;
+
+    return true;
+  });
+
+  logger.debug('git-analyzer: commit records extracted', {
+    repoPath: normalizedPath,
+    totalCommits: allCommits.length,
+    filteredCommits: filtered.length,
+  });
+
+  return filtered;
+}
+
+/**
+ * Parse `git log --format="%H|%s|%an|%aI" --name-only` output into commit records.
+ * Each commit block is: header line (hash|subject|author|date) followed by blank + file lines.
+ */
+function parseStatsGitLog(stdout: string): CommitRecord[] {
+  const commits: CommitRecord[] = [];
+  const lines = stdout.split('\n');
+
+  let currentCommit: CommitRecord | null = null;
+
+  for (const rawLine of lines) {
+    const line = rawLine.trim();
+
+    if (!line) {
+      // Blank line separates commit header from files, or between commits
+      continue;
+    }
+
+    // Try to parse as a commit header (hash|subject|author|date)
+    const pipeIndex = line.indexOf('|');
+    if (pipeIndex === 40 && /^[0-9a-f]{40}$/i.test(line.slice(0, 40))) {
+      // Flush previous commit
+      if (currentCommit) {
+        commits.push(currentCommit);
+      }
+
+      const parts = line.split('|');
+      const hash = parts[0] ?? '';
+      const message = parts[1] ?? '';
+      const author = parts[2] ?? '';
+      const dateStr = parts.slice(3).join('|'); // ISO date may contain no pipes but be safe
+
+      currentCommit = {
+        hash,
+        message,
+        author,
+        timestamp: dateStr ? new Date(dateStr).getTime() : 0,
+        files: [],
+      };
+    } else if (currentCommit) {
+      // This is a file path line
+      currentCommit.files.push(line);
+    }
+  }
+
+  // Flush final commit
+  if (currentCommit) {
+    commits.push(currentCommit);
+  }
+
+  return commits;
+}
+
+/**
+ * Compute stability score (0-1). Higher = more stable (fewer recent changes).
+ * Uses the ratio of commits in the recent half of the time window vs total commits.
+ * A file with all changes in the distant past is stable (score ~1.0).
+ * A file with all changes recently is unstable (score ~0.0).
+ */
+function computeStabilityScore(timestamps: number[], now: number): number {
+  if (timestamps.length <= 1) return 1.0;
+
+  const sorted = [...timestamps].sort((a, b) => a - b);
+  const oldest = sorted[0]!;
+  const timeRange = now - oldest;
+  if (timeRange <= 0) return 1.0;
+
+  const midpoint = oldest + timeRange / 2;
+  const recentCount = sorted.filter(t => t > midpoint).length;
+  const recentRatio = recentCount / sorted.length;
+
+  // Invert: fewer recent changes = more stable
+  return Math.max(0, Math.min(1, 1.0 - recentRatio));
+}
+
+/**
+ * Compute change velocity (-1 to +1).
+ * Compares the rate of changes in the first half vs second half of the commit history.
+ * Positive = changes are accelerating. Negative = changes are decelerating.
+ */
+function computeChangeVelocity(timestamps: number[]): number {
+  if (timestamps.length < 4) return 0;
+
+  const sorted = [...timestamps].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  const firstHalf = sorted.slice(0, mid);
+  const secondHalf = sorted.slice(mid);
+
+  // Compute average inter-commit interval for each half
+  const avgInterval = (arr: number[]): number => {
+    if (arr.length < 2) return Infinity;
+    let totalGap = 0;
+    for (let i = 1; i < arr.length; i++) {
+      totalGap += arr[i]! - arr[i - 1]!;
+    }
+    return totalGap / (arr.length - 1);
+  };
+
+  const firstAvg = avgInterval(firstHalf);
+  const secondAvg = avgInterval(secondHalf);
+
+  if (firstAvg === Infinity && secondAvg === Infinity) return 0;
+  if (firstAvg === Infinity) return 1; // No first-half activity, all second-half
+  if (secondAvg === Infinity) return -1; // No second-half activity
+
+  // Shorter intervals in second half = accelerating
+  // Normalize to -1..+1 range
+  const ratio = (firstAvg - secondAvg) / Math.max(firstAvg, secondAvg);
+  return Math.max(-1, Math.min(1, ratio));
+}