memorylink 1.0.0

package/src/utils/code-structure.ts

@@ -0,0 +1,686 @@
+/**
+ * MemoryLink - Code Structure Utilities
+ * AST parsing and dependency graph building for code structure memory
+ * 
+ * v1.2: Code Structure Memory feature
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as ts from 'typescript';
+import { glob } from 'glob';
+import { normalizePathForStorage } from '../utils.js';
+
+// ============================================
+// TYPES
+// ============================================
+
+export interface FileNode {
+  path: string;
+  imports: string[];
+  exports: string[];
+  functions: string[];
+  classes: string[];
+  last_modified: string;
+  last_modified_by?: string;
+  risk_score: number; // Based on dependents
+  complexity: number;
+  lines: number;
+}
+
+export interface CodeGraph {
+  files: Map<string, FileNode>;
+  dependencies: Map<string, string[]>; // file -> imports
+  dependents: Map<string, string[]>;   // file -> imported_by
+  complexity: Map<string, number>;
+  change_frequency: Map<string, number>;
+}
+
+export interface BreakingChange {
+  file: string;
+  reason: string;
+  migration_needed: boolean;
+  estimated_effort: string;
+}
+
+export interface ChangeImpact {
+  file: string;
+  proposed_change?: string;
+  affected_files: string[];
+  breaking_changes: BreakingChange[];
+  test_coverage?: number;
+  risk_level: 'LOW' | 'MEDIUM' | 'HIGH';
+  recommendations: string[];
+}
+
+// ============================================
+// AST PARSING
+// ============================================
+
+/**
+ * Parse a file and extract structure (supports TS/JS/Python/Go)
+ * v1.3: Week 7-8 - Added Python/Go support via Tree-sitter
+ */
+export async function parseFile(filepath: string, rootDir: string): Promise<FileNode | null> {
+  try {
+    const content = fs.readFileSync(filepath, 'utf-8');
+    const stats = fs.statSync(filepath);
+    
+    // Check file type
+    const isTypeScript = filepath.endsWith('.ts') || filepath.endsWith('.tsx');
+    const isJavaScript = filepath.endsWith('.js') || filepath.endsWith('.jsx');
+    const isPython = filepath.endsWith('.py');
+    const isGo = filepath.endsWith('.go');
+    
+    // Try Tree-sitter for Python/Go first (v1.3)
+    if (isPython) {
+      try {
+        const { parsePythonFile } = await import('./tree-sitter-parser.js');
+        const result = await parsePythonFile(filepath, rootDir);
+        if (result) return result;
+      } catch {
+        // Fall through to return null if Tree-sitter not available
+      }
+      return null;
+    }
+    
+    if (isGo) {
+      try {
+        const { parseGoFile } = await import('./tree-sitter-parser.js');
+        const result = await parseGoFile(filepath, rootDir);
+        if (result) return result;
+      } catch {
+        // Fall through to return null if Tree-sitter not available
+      }
+      return null;
+    }
+    
+    if (!isTypeScript && !isJavaScript) {
+      return null;
+    }
+    
+    let imports: string[] = [];
+    let exports: string[] = [];
+    let functions: string[] = [];
+    let classes: string[] = [];
+    
+    if (isTypeScript) {
+      const sourceFile = ts.createSourceFile(
+        filepath,
+        content,
+        ts.ScriptTarget.Latest,
+        true
+      );
+      
+      const result = extractFromTS(sourceFile, rootDir);
+      imports = result.imports;
+      exports = result.exports;
+      functions = result.functions;
+      classes = result.classes;
+    } else {
+      // For JavaScript, use simple regex parsing (can be enhanced with Babel later)
+      imports = extractImportsRegex(content, rootDir, filepath);
+      exports = extractExportsRegex(content);
+      functions = extractFunctionsRegex(content);
+      classes = extractClassesRegex(content);
+    }
+    
+    const lines = content.split('\n').length;
+    const complexity = calculateComplexity(content);
+    
+    return {
+      path: normalizePathForGraph(filepath, rootDir),
+      imports,
+      exports,
+      functions,
+      classes,
+      last_modified: stats.mtime.toISOString(),
+      risk_score: 0, // Will be calculated after graph is built
+      complexity,
+      lines,
+    };
+  } catch (error) {
+    // Skip the file but do not fail the graph build
+    console.warn(`Skipping file (parse error): ${filepath} -> ${error instanceof Error ? error.message : 'Unknown error'}`);
+    return null;
+  }
+}
+
+/**
+ * Extract imports, exports, functions, and classes from TypeScript AST
+ */
+function extractFromTS(
+  sourceFile: ts.SourceFile,
+  rootDir: string
+): {
+  imports: string[];
+  exports: string[];
+  functions: string[];
+  classes: string[];
+} {
+  const imports: string[] = [];
+  const exports: string[] = [];
+  const functions: string[] = [];
+  const classes: string[] = [];
+  
+  function visit(node: ts.Node) {
+    // Extract imports
+    if (ts.isImportDeclaration(node) && node.moduleSpecifier) {
+      const moduleSpec = node.moduleSpecifier;
+      if (ts.isStringLiteral(moduleSpec)) {
+        const importPath = moduleSpec.text;
+        // Resolve relative imports
+        if (importPath.startsWith('.')) {
+          const resolved = resolveImportPath(importPath, sourceFile.fileName, rootDir);
+          if (resolved) imports.push(resolved);
+        } else if (!importPath.startsWith('@') && !importPath.includes('node_modules')) {
+          // Local imports (not node_modules)
+          imports.push(importPath);
+        }
+      }
+    }
+    
+    // Extract exports
+    if (ts.isExportDeclaration(node)) {
+      if (node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier)) {
+        const exportPath = node.moduleSpecifier.text;
+        if (exportPath.startsWith('.')) {
+          const resolved = resolveImportPath(exportPath, sourceFile.fileName, rootDir);
+          if (resolved) exports.push(resolved);
+        }
+      }
+    }
+    
+    // Extract named exports
+    if (ts.isExportDeclaration(node) && node.exportClause) {
+      if (ts.isNamedExports(node.exportClause)) {
+        node.exportClause.elements.forEach(elem => {
+          exports.push(elem.name.text);
+        });
+      }
+    }
+    
+    // Extract function declarations
+    if (ts.isFunctionDeclaration(node) && node.name) {
+      functions.push(node.name.text);
+    }
+    
+    // Extract class declarations
+    if (ts.isClassDeclaration(node) && node.name) {
+      classes.push(node.name.text);
+    }
+    
+    // Extract arrow functions (if named)
+    if (ts.isVariableDeclaration(node) && node.name && ts.isIdentifier(node.name)) {
+      const initializer = node.initializer;
+      if (initializer && (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer))) {
+        functions.push(node.name.text);
+      }
+    }
+    
+    ts.forEachChild(node, visit);
+  }
+  
+  visit(sourceFile);
+  
+  return { imports, exports, functions, classes };
+}
+
+/**
+ * Extract imports using regex (for JavaScript files)
+ */
+function extractImportsRegex(content: string, rootDir: string, filepath: string): string[] {
+  const imports: string[] = [];
+  
+  // Match ES6 imports: import ... from '...'
+  const es6ImportRegex = /import\s+.*?\s+from\s+['"]([^'"]+)['"]/g;
+  let match;
+  while ((match = es6ImportRegex.exec(content)) !== null) {
+    const importPath = match[1];
+    if (importPath.startsWith('.')) {
+      const resolved = resolveImportPath(importPath, filepath, rootDir);
+      if (resolved) imports.push(resolved);
+    }
+  }
+  
+  // Match require() calls: require('...')
+  const requireRegex = /require\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
+  while ((match = requireRegex.exec(content)) !== null) {
+    const importPath = match[1];
+    if (importPath.startsWith('.')) {
+      const resolved = resolveImportPath(importPath, filepath, rootDir);
+      if (resolved) imports.push(resolved);
+    }
+  }
+  
+  return imports;
+}
+
+/**
+ * Extract exports using regex
+ */
+function extractExportsRegex(content: string): string[] {
+  const exports: string[] = [];
+  
+  // Match export declarations
+  const exportRegex = /export\s+(?:const|let|var|function|class|default)\s+(\w+)/g;
+  let match;
+  while ((match = exportRegex.exec(content)) !== null) {
+    exports.push(match[1]);
+  }
+  
+  return exports;
+}
+
+/**
+ * Extract function names using regex
+ */
+function extractFunctionsRegex(content: string): string[] {
+  const functions: string[] = [];
+  
+  // Match function declarations: function name() or const name = () =>
+  const functionRegex = /(?:function\s+(\w+)|const\s+(\w+)\s*=\s*(?:\(|function))/g;
+  let match;
+  while ((match = functionRegex.exec(content)) !== null) {
+    const name = match[1] || match[2];
+    if (name) functions.push(name);
+  }
+  
+  return functions;
+}
+
+/**
+ * Extract class names using regex
+ */
+function extractClassesRegex(content: string): string[] {
+  const classes: string[] = [];
+  
+  const classRegex = /class\s+(\w+)/g;
+  let match;
+  while ((match = classRegex.exec(content)) !== null) {
+    classes.push(match[1]);
+  }
+  
+  return classes;
+}
+
+/**
+ * Resolve import path to actual file path
+ */
+function resolveImportPath(importPath: string, fromFile: string, rootDir: string): string | null {
+  try {
+    const fromDir = path.dirname(fromFile);
+    let resolved = path.resolve(fromDir, importPath);
+    
+    // Try with .ts, .tsx, .js, .jsx extensions
+    const extensions = ['.ts', '.tsx', '.js', '.jsx', '/index.ts', '/index.tsx', '/index.js', '/index.jsx'];
+    
+    for (const ext of extensions) {
+      const withExt = resolved + ext;
+      if (fs.existsSync(withExt)) {
+        return normalizePathForGraph(withExt, rootDir);
+      }
+    }
+    
+    // Try as-is
+    if (fs.existsSync(resolved)) {
+      return normalizePathForGraph(resolved, rootDir);
+    }
+    
+    // Try directory with index
+    if (fs.statSync(resolved).isDirectory()) {
+      for (const ext of extensions.slice(4)) {
+        const indexFile = path.join(resolved, ext.slice(1));
+        if (fs.existsSync(indexFile)) {
+          return normalizePathForGraph(indexFile, rootDir);
+        }
+      }
+    }
+    
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Normalize path for graph storage (always relative to root, forward slashes)
+ */
+function normalizePathForGraph(filepath: string, rootDir: string): string {
+  return normalizePathForStorage(filepath, rootDir);
+}
+
+/**
+ * Calculate code complexity (simple metric)
+ */
+function calculateComplexity(content: string): number {
+  let complexity = 1; // Base complexity
+  
+  // Count control flow statements
+  const controlFlowRegex = /(if|else|for|while|switch|case|catch|try)\s*\(/g;
+  const matches = content.match(controlFlowRegex);
+  if (matches) complexity += matches.length;
+  
+  // Count nested structures
+  const nestedRegex = /\{[^}]*\{/g;
+  const nested = content.match(nestedRegex);
+  if (nested) complexity += nested.length;
+  
+  return complexity;
+}
+
+// ============================================
+// DEPENDENCY GRAPH BUILDING
+// ============================================
+
+/**
+ * Build dependency graph for entire codebase
+ */
+export async function buildDependencyGraph(rootDir: string): Promise<CodeGraph> {
+  const graph: CodeGraph = {
+    files: new Map(),
+    dependencies: new Map(),
+    dependents: new Map(),
+    complexity: new Map(),
+    change_frequency: new Map(),
+  };
+  
+  // Find all source files
+  const sourceFiles = await findSourceFiles(rootDir);
+  
+  // Parse all files (async for Tree-sitter support)
+  for (const file of sourceFiles) {
+    try {
+      const node = await parseFile(file, rootDir);
+      if (node) {
+        graph.files.set(node.path, node);
+        graph.dependencies.set(node.path, node.imports);
+        graph.complexity.set(node.path, node.complexity);
+        
+        // Initialize dependents map
+        if (!graph.dependents.has(node.path)) {
+          graph.dependents.set(node.path, []);
+        }
+      }
+    } catch (error) {
+      // Non-fatal: skip files that can't be parsed
+      console.warn(`Skipping ${file}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+  
+  // Build dependents map (reverse of dependencies)
+  for (const [file, imports] of graph.dependencies.entries()) {
+    for (const imp of imports) {
+      if (!graph.dependents.has(imp)) {
+        graph.dependents.set(imp, []);
+      }
+      const dependents = graph.dependents.get(imp)!;
+      if (!dependents.includes(file)) {
+        dependents.push(file);
+      }
+    }
+  }
+  
+  // Calculate risk scores based on dependents
+  for (const [file, node] of graph.files.entries()) {
+    const dependents = graph.dependents.get(file) || [];
+    node.risk_score = dependents.length;
+  }
+  
+  return graph;
+}
+
+/**
+ * Find all source files in codebase (supports TS/JS/Python/Go)
+ * v1.3: Week 7-8 - Added Python/Go support
+ */
+async function findSourceFiles(rootDir: string): Promise<string[]> {
+  const patterns = [
+    '**/*.ts',
+    '**/*.tsx',
+    '**/*.js',
+    '**/*.jsx',
+    '**/*.py',  // Python (v1.3)
+    '**/*.go',  // Go (v1.3)
+  ];
+  
+  const ignorePatterns = [
+    '**/node_modules/**',
+    '**/dist/**',
+    '**/build/**',
+    '**/.git/**',
+    '**/.memorylink/**',
+    '**/__pycache__/**',  // Python cache
+    '**/venv/**',         // Python virtual env
+    '**/vendor/**',       // Go vendor
+    '**/*.test.ts',
+    '**/*.test.tsx',
+    '**/*.test.js',
+    '**/*.test.jsx',
+    '**/*.spec.ts',
+    '**/*.spec.tsx',
+    '**/*_test.py',       // Python tests
+    '**/*_test.go',       // Go tests
+  ];
+  
+  const allFiles: string[] = [];
+  
+  for (const pattern of patterns) {
+    const files = await glob(pattern, {
+      cwd: rootDir,
+      ignore: ignorePatterns,
+      absolute: true,
+    });
+    allFiles.push(...files);
+  }
+  
+  return [...new Set(allFiles)];
+}
+
+// ============================================
+// IMPACT ANALYSIS
+// ============================================
+
+/**
+ * Analyze impact of changing a file
+ */
+export function analyzeImpact(
+  file: string,
+  graph: CodeGraph,
+  proposedChange?: string
+): ChangeImpact {
+  const normalizedFile = file.replace(/\\/g, '/');
+  
+  // Find direct dependents
+  const directDependents = graph.dependents.get(normalizedFile) || [];
+  
+  // Find indirect dependents (transitive)
+  const indirectDependents = findIndirectDependents(normalizedFile, graph);
+  
+  // All affected files
+  const affectedFiles = [...new Set([...directDependents, ...indirectDependents])];
+  
+  // Detect breaking changes
+  const breakingChanges = detectBreakingChanges(normalizedFile, graph, proposedChange);
+  
+  // Calculate risk level
+  const riskLevel = calculateRiskLevel(directDependents.length, breakingChanges.length);
+  
+  // Generate recommendations
+  const recommendations = generateRecommendations(
+    normalizedFile,
+    directDependents,
+    breakingChanges,
+    graph
+  );
+  
+  return {
+    file: normalizedFile,
+    proposed_change: proposedChange,
+    affected_files: affectedFiles,
+    breaking_changes: breakingChanges,
+    risk_level: riskLevel,
+    recommendations,
+  };
+}
+
+/**
+ * Find indirect dependents (transitive dependencies)
+ */
+function findIndirectDependents(file: string, graph: CodeGraph): string[] {
+  const visited = new Set<string>();
+  const indirect: string[] = [];
+  
+  function traverse(current: string) {
+    if (visited.has(current)) return;
+    visited.add(current);
+    
+    const dependents = graph.dependents.get(current) || [];
+    for (const dep of dependents) {
+      if (dep !== file && !indirect.includes(dep)) {
+        indirect.push(dep);
+        traverse(dep);
+      }
+    }
+  }
+  
+  const directDependents = graph.dependents.get(file) || [];
+  for (const dep of directDependents) {
+    traverse(dep);
+  }
+  
+  return indirect;
+}
+
+/**
+ * Detect potential breaking changes
+ */
+function detectBreakingChanges(
+  file: string,
+  graph: CodeGraph,
+  proposedChange?: string
+): BreakingChange[] {
+  const breaking: BreakingChange[] = [];
+  const dependents = graph.dependents.get(file) || [];
+  
+  const node = graph.files.get(file);
+  if (!node) return breaking;
+  
+  // If file has many exports and dependents, changes are likely breaking
+  if (node.exports.length > 0 && dependents.length > 5) {
+    breaking.push({
+      file,
+      reason: `File has ${node.exports.length} exports and ${dependents.length} dependents. Changes to exports may break dependent files.`,
+      migration_needed: true,
+      estimated_effort: `${dependents.length} files may need updates`,
+    });
+  }
+  
+  // If proposed change mentions breaking keywords
+  if (proposedChange) {
+    const breakingKeywords = ['remove', 'delete', 'rename', 'change signature', 'deprecate'];
+    const lowerChange = proposedChange.toLowerCase();
+    if (breakingKeywords.some(keyword => lowerChange.includes(keyword))) {
+      breaking.push({
+        file,
+        reason: `Proposed change contains breaking keywords: ${proposedChange}`,
+        migration_needed: true,
+        estimated_effort: 'Manual review required',
+      });
+    }
+  }
+  
+  return breaking;
+}
+
+/**
+ * Calculate risk level
+ */
+function calculateRiskLevel(dependentsCount: number, breakingCount: number): 'LOW' | 'MEDIUM' | 'HIGH' {
+  if (dependentsCount === 0) return 'LOW';
+  if (dependentsCount > 10 || breakingCount > 0) return 'HIGH';
+  if (dependentsCount > 5) return 'MEDIUM';
+  return 'LOW';
+}
+
+/**
+ * Generate recommendations
+ */
+function generateRecommendations(
+  file: string,
+  directDependents: string[],
+  breakingChanges: BreakingChange[],
+  graph: CodeGraph
+): string[] {
+  const recommendations: string[] = [];
+  
+  if (directDependents.length > 0) {
+    recommendations.push(`This file is imported by ${directDependents.length} file(s).`);
+  }
+  
+  if (breakingChanges.length > 0) {
+    recommendations.push('⚠️  Breaking changes detected. Consider:');
+    recommendations.push('  1. Create tests for this file first');
+    recommendations.push('  2. Add deprecation warnings before breaking changes');
+    recommendations.push('  3. Update dependent files simultaneously');
+  }
+  
+  if (directDependents.length > 5) {
+    recommendations.push(`⚠️  HIGH RISK: ${directDependents.length} files depend on this. Notify team before changes.`);
+  }
+  
+  const node = graph.files.get(file);
+  if (node && node.complexity > 50) {
+    recommendations.push(`File complexity is ${node.complexity}. Consider refactoring.`);
+  }
+  
+  return recommendations;
+}
+
+// ============================================
+// GRAPH STORAGE
+// ============================================
+
+/**
+ * Save graph to JSON file
+ */
+export function saveGraph(graph: CodeGraph, filepath: string): void {
+  const serializable = {
+    files: Object.fromEntries(
+      Array.from(graph.files.entries()).map(([k, v]) => [k, v])
+    ),
+    dependencies: Object.fromEntries(graph.dependencies),
+    dependents: Object.fromEntries(graph.dependents),
+    complexity: Object.fromEntries(graph.complexity),
+    change_frequency: Object.fromEntries(graph.change_frequency),
+    generated_at: new Date().toISOString(),
+  };
+  
+  fs.writeFileSync(filepath, JSON.stringify(serializable, null, 2));
+}
+
+/**
+ * Load graph from JSON file
+ */
+export function loadGraph(filepath: string): CodeGraph | null {
+  try {
+    if (!fs.existsSync(filepath)) return null;
+    
+    const content = fs.readFileSync(filepath, 'utf-8');
+    const data = JSON.parse(content);
+    
+    const graph: CodeGraph = {
+      files: new Map(Object.entries(data.files)),
+      dependencies: new Map(Object.entries(data.dependencies)),
+      dependents: new Map(Object.entries(data.dependents)),
+      complexity: new Map(Object.entries(data.complexity)),
+      change_frequency: new Map(Object.entries(data.change_frequency || {})),
+    };
+    
+    return graph;
+  } catch (error) {
+    console.error(`Error loading graph from ${filepath}:`, error);
+    return null;
+  }
+}
+