@grafema/core 0.1.0-alpha.5 → 0.2.0-beta

package/src/core/CoverageAnalyzer.ts

@@ -0,0 +1,243 @@
+/**
+ * CoverageAnalyzer - Calculates analysis coverage for a project
+ *
+ * Determines which files in a project have been analyzed and categorizes
+ * files into three groups:
+ * - Analyzed: Files that appear as MODULE nodes in the graph
+ * - Unsupported: Files with extensions that no indexer can handle
+ * - Unreachable: Files with supported extensions but not in the graph
+ *
+ * Usage:
+ *   const analyzer = new CoverageAnalyzer(graphBackend, '/path/to/project');
+ *   const result = await analyzer.analyze();
+ */
+
+import { readdirSync, statSync, existsSync, lstatSync } from 'fs';
+import { join, relative, extname } from 'path';
+import type { GraphBackend, NodeRecord } from '@grafema/types';
+
+/**
+ * Supported file extensions by language
+ */
+const JS_SUPPORTED = ['.js', '.mjs', '.cjs', '.jsx', '.ts', '.tsx'];
+const RUST_SUPPORTED = ['.rs'];
+const SUPPORTED_EXTENSIONS = new Set([...JS_SUPPORTED, ...RUST_SUPPORTED]);
+
+/**
+ * Known code file extensions (for scanning)
+ * Files with these extensions are considered source code
+ */
+const CODE_EXTENSIONS = new Set([
+  // Supported
+  ...JS_SUPPORTED,
+  ...RUST_SUPPORTED,
+  // Unsupported but tracked
+  '.go', '.kt', '.java', '.py', '.rb', '.php', '.c', '.cpp', '.h', '.hpp',
+  '.cs', '.swift', '.scala', '.sql', '.graphql', '.gql',
+]);
+
+/**
+ * Directories to always skip during scanning
+ */
+const SKIP_DIRS = new Set([
+  'node_modules',
+  '.git',
+  '.grafema',
+  'dist',
+  'build',
+  'out',
+  'coverage',
+  '.next',
+  '.nuxt',
+  '__pycache__',
+  'target', // Rust
+  'vendor',
+]);
+
+/**
+ * Coverage analysis result
+ */
+export interface CoverageResult {
+  projectPath: string;
+  total: number;
+  analyzed: {
+    count: number;
+    files: string[];
+  };
+  unsupported: {
+    count: number;
+    byExtension: Record<string, string[]>;
+  };
+  unreachable: {
+    count: number;
+    files: string[];
+    byExtension: Record<string, string[]>;
+  };
+  percentages: {
+    analyzed: number;
+    unsupported: number;
+    unreachable: number;
+  };
+}
+
+/**
+ * CoverageAnalyzer calculates what percentage of a codebase has been analyzed.
+ */
+export class CoverageAnalyzer {
+  private graph: GraphBackend;
+  private projectPath: string;
+
+  constructor(graph: GraphBackend, projectPath: string) {
+    this.graph = graph;
+    this.projectPath = projectPath;
+  }
+
+  /**
+   * Analyze the project and return coverage statistics
+   */
+  async analyze(): Promise<CoverageResult> {
+    // Step 1: Get all MODULE nodes from the graph
+    const analyzedFiles = await this.getAnalyzedFiles();
+    const analyzedSet = new Set(analyzedFiles);
+
+    // Step 2: Scan project for all code files
+    const allCodeFiles = this.scanProjectFiles();
+
+    // Step 3: Categorize files
+    const unsupportedByExt: Record<string, string[]> = {};
+    const unreachableByExt: Record<string, string[]> = {};
+    const unreachableFiles: string[] = [];
+
+    for (const file of allCodeFiles) {
+      if (analyzedSet.has(file)) {
+        continue; // Already analyzed
+      }
+
+      const ext = extname(file).toLowerCase();
+
+      if (SUPPORTED_EXTENSIONS.has(ext)) {
+        // Supported but not in graph = unreachable
+        unreachableFiles.push(file);
+        if (!unreachableByExt[ext]) {
+          unreachableByExt[ext] = [];
+        }
+        unreachableByExt[ext].push(file);
+      } else {
+        // Not supported = unsupported
+        if (!unsupportedByExt[ext]) {
+          unsupportedByExt[ext] = [];
+        }
+        unsupportedByExt[ext].push(file);
+      }
+    }
+
+    // Calculate counts
+    const unsupportedCount = Object.values(unsupportedByExt)
+      .reduce((sum, files) => sum + files.length, 0);
+    const unreachableCount = unreachableFiles.length;
+    const analyzedCount = analyzedFiles.length;
+    const total = analyzedCount + unsupportedCount + unreachableCount;
+
+    // Calculate percentages (handle division by zero)
+    const percentages = {
+      analyzed: total > 0 ? Math.round((analyzedCount / total) * 100) : 0,
+      unsupported: total > 0 ? Math.round((unsupportedCount / total) * 100) : 0,
+      unreachable: total > 0 ? Math.round((unreachableCount / total) * 100) : 0,
+    };
+
+    return {
+      projectPath: this.projectPath,
+      total,
+      analyzed: {
+        count: analyzedCount,
+        files: analyzedFiles,
+      },
+      unsupported: {
+        count: unsupportedCount,
+        byExtension: unsupportedByExt,
+      },
+      unreachable: {
+        count: unreachableCount,
+        files: unreachableFiles,
+        byExtension: unreachableByExt,
+      },
+      percentages,
+    };
+  }
+
+  /**
+   * Get list of analyzed files from the graph (MODULE nodes)
+   */
+  private async getAnalyzedFiles(): Promise<string[]> {
+    const files: string[] = [];
+
+    // Use queryNodes with type: 'MODULE'
+    for await (const node of this.graph.queryNodes({ type: 'MODULE' })) {
+      if (node.file) {
+        // Store relative path for consistency
+        const relativePath = node.file.startsWith(this.projectPath)
+          ? relative(this.projectPath, node.file)
+          : node.file;
+        files.push(relativePath);
+      }
+    }
+
+    return files;
+  }
+
+  /**
+   * Scan project directory for all code files
+   * Respects common ignore patterns (node_modules, .git, etc.)
+   */
+  private scanProjectFiles(): string[] {
+    const files: string[] = [];
+    this.walkDirectory(this.projectPath, files);
+    return files;
+  }
+
+  /**
+   * Recursively walk directory and collect code files
+   */
+  private walkDirectory(dir: string, files: string[], depth = 0): void {
+    // Safety: limit recursion depth
+    if (depth > 20) return;
+
+    if (!existsSync(dir)) return;
+
+    let entries: string[];
+    try {
+      entries = readdirSync(dir);
+    } catch {
+      return; // Skip directories we can't read
+    }
+
+    for (const entry of entries) {
+      // Skip hidden files and directories
+      if (entry.startsWith('.')) continue;
+
+      // Skip known non-source directories
+      if (SKIP_DIRS.has(entry)) continue;
+
+      const fullPath = join(dir, entry);
+
+      // Skip symlinks to avoid infinite loops
+      try {
+        const stat = lstatSync(fullPath);
+        if (stat.isSymbolicLink()) continue;
+
+        if (stat.isDirectory()) {
+          this.walkDirectory(fullPath, files, depth + 1);
+        } else if (stat.isFile()) {
+          const ext = extname(entry).toLowerCase();
+          if (CODE_EXTENSIONS.has(ext)) {
+            // Store relative path
+            files.push(relative(this.projectPath, fullPath));
+          }
+        }
+      } catch {
+        // Skip files we can't stat
+        continue;
+      }
+    }
+  }
+}

package/src/core/FileExplainer.ts

@@ -0,0 +1,179 @@
+/**
+ * FileExplainer - Show what nodes exist in a file for discovery
+ *
+ * Purpose: Help users discover what nodes exist in the graph for a file,
+ * displaying semantic IDs so users can query them.
+ *
+ * This addresses the core UX problem: users can't find nodes because
+ * semantic IDs are opaque (e.g., "src/app.ts->fetchData->try#0->VARIABLE->response").
+ * This tool shows what's in the graph so users know what they can query.
+ *
+ * @see _tasks/REG-177/006-don-revised-plan.md for design rationale
+ */
+
+import type { GraphBackend, BaseNodeRecord, NodeFilter } from '@grafema/types';
+
+/**
+ * Result of explaining a file's graph contents
+ */
+export interface FileExplainResult {
+  /** The file path that was explained */
+  file: string;
+  /** Whether the file has been analyzed */
+  status: 'ANALYZED' | 'NOT_ANALYZED';
+  /** All nodes in the graph for this file, enhanced with context */
+  nodes: EnhancedNode[];
+  /** Node counts grouped by type */
+  byType: Record<string, number>;
+  /** Total number of nodes in the file */
+  totalCount: number;
+}
+
+/**
+ * A node record enhanced with scope context information.
+ *
+ * The context field provides human-readable information about
+ * where the node appears (e.g., "inside try block", "catch parameter").
+ */
+export interface EnhancedNode extends BaseNodeRecord {
+  /** Human-readable context about the node's scope */
+  context?: string;
+}
+
+/**
+ * Scope patterns to detect from semantic IDs.
+ * Order matters - more specific patterns should come first.
+ */
+const SCOPE_PATTERNS: Array<{ pattern: RegExp; context: string }> = [
+  { pattern: /->catch#\d+->/, context: 'inside catch block' },
+  { pattern: /->try#\d+->/, context: 'inside try block' },
+  { pattern: /->if#\d+->/, context: 'inside conditional' },
+  { pattern: /->else#\d+->/, context: 'inside else block' },
+  { pattern: /->for#\d+->/, context: 'inside loop' },
+  { pattern: /->while#\d+->/, context: 'inside loop' },
+  { pattern: /->switch#\d+->/, context: 'inside switch' },
+];
+
+/**
+ * FileExplainer class - explains what nodes exist in a file's graph.
+ *
+ * Use this when:
+ * - User can't find a variable/function they expect to be in the graph
+ * - User wants to understand what's been analyzed for a file
+ * - User needs semantic IDs to construct queries
+ *
+ * Example:
+ * ```typescript
+ * const explainer = new FileExplainer(graphBackend);
+ * const result = await explainer.explain('src/app.ts');
+ *
+ * if (result.status === 'NOT_ANALYZED') {
+ *   console.log('File not in graph. Run: grafema analyze');
+ * } else {
+ *   for (const node of result.nodes) {
+ *     console.log(`[${node.type}] ${node.name}`);
+ *     console.log(`  ID: ${node.id}`);
+ *     if (node.context) {
+ *       console.log(`  Context: ${node.context}`);
+ *     }
+ *   }
+ * }
+ * ```
+ */
+export class FileExplainer {
+  constructor(private graph: GraphBackend) {}
+
+  /**
+   * Explain what nodes exist in the graph for a file.
+   *
+   * @param filePath - The file path to explain (relative or absolute)
+   * @returns FileExplainResult with all nodes, grouped by type, with context
+   */
+  async explain(filePath: string): Promise<FileExplainResult> {
+    // Query graph for all nodes in this file
+    const nodes = await this.getNodesForFile(filePath);
+
+    // Group by type
+    const byType = this.groupByType(nodes);
+
+    // Enhance with context from semantic ID parsing
+    const enhanced = this.enhanceWithContext(nodes);
+
+    // Sort nodes: by type, then by name
+    enhanced.sort((a, b) => {
+      const typeCompare = a.type.localeCompare(b.type);
+      if (typeCompare !== 0) return typeCompare;
+      return (a.name || '').localeCompare(b.name || '');
+    });
+
+    return {
+      file: filePath,
+      status: nodes.length > 0 ? 'ANALYZED' : 'NOT_ANALYZED',
+      nodes: enhanced,
+      byType,
+      totalCount: nodes.length,
+    };
+  }
+
+  /**
+   * Query graph for all nodes in a file
+   *
+   * Note: The server-side file filter may not work correctly in all cases,
+   * so we also filter client-side to ensure only nodes from the requested file are returned.
+   */
+  private async getNodesForFile(filePath: string): Promise<BaseNodeRecord[]> {
+    const filter: NodeFilter = { file: filePath };
+    const nodes: BaseNodeRecord[] = [];
+
+    for await (const node of this.graph.queryNodes(filter)) {
+      // Client-side filter as backup (server filter may not work correctly)
+      if (node.file === filePath) {
+        nodes.push(node);
+      }
+    }
+
+    return nodes;
+  }
+
+  /**
+   * Group nodes by type, counting occurrences
+   */
+  private groupByType(nodes: BaseNodeRecord[]): Record<string, number> {
+    const counts: Record<string, number> = {};
+
+    for (const node of nodes) {
+      const type = node.type || 'UNKNOWN';
+      counts[type] = (counts[type] || 0) + 1;
+    }
+
+    return counts;
+  }
+
+  /**
+   * Enhance nodes with human-readable scope context.
+   *
+   * Detects patterns in semantic IDs like:
+   * - "file->func->try#0->VARIABLE->x" → "inside try block"
+   * - "file->func->catch#0->VARIABLE->error" → "inside catch block"
+   */
+  private enhanceWithContext(nodes: BaseNodeRecord[]): EnhancedNode[] {
+    return nodes.map((node) => {
+      const context = this.detectScopeContext(node.id);
+      return context ? { ...node, context } : { ...node };
+    });
+  }
+
+  /**
+   * Detect scope context from semantic ID patterns.
+   *
+   * Returns human-readable context string or undefined if no special scope.
+   */
+  private detectScopeContext(semanticId: string): string | undefined {
+    for (const { pattern, context } of SCOPE_PATTERNS) {
+      if (pattern.test(semanticId)) {
+        return context;
+      }
+    }
+    return undefined;
+  }
+}

package/src/core/FileNodeManager.ts

@@ -0,0 +1,100 @@
+/**
+ * FileNodeManager - utility for idempotent file node clearing
+ *
+ * Problem: Multiple phases create nodes for the same file:
+ * - INDEXING creates MODULE nodes
+ * - ANALYSIS creates FUNCTION, CLASS, SCOPE, etc. nodes
+ *
+ * When re-analyzing with forceAnalysis=true, we need to clear existing
+ * nodes BEFORE any phase creates new nodes for that file.
+ *
+ * Solution: Track "touched" files. First touch clears all nodes for that file.
+ * Subsequent touches (from other phases) are no-ops.
+ */
+
+import type { GraphBackend } from '@grafema/types';
+
+/**
+ * Clear all nodes for a file if it hasn't been touched yet in this analysis run.
+ *
+ * Thread-safety note: The touchedFiles Set is shared across concurrent Promise.all
+ * calls, but this is safe because:
+ * 1. The check (has) and add are synchronous operations
+ * 2. We add to the set BEFORE the async clear operation
+ * 3. Other concurrent calls will see the file as touched immediately
+ *
+ * @param graph - Graph backend with deleteNode support
+ * @param file - Absolute file path to clear nodes for
+ * @param touchedFiles - Set tracking files already touched in this run
+ * @returns Number of nodes deleted (0 if file was already touched or backend doesn't support delete)
+ */
+export async function clearFileNodesIfNeeded(
+  graph: GraphBackend,
+  file: string,
+  touchedFiles: Set<string>
+): Promise<number> {
+  // Already touched in this run - nothing to clear
+  if (touchedFiles.has(file)) {
+    return 0;
+  }
+
+  // Mark as touched BEFORE clearing (sync operation, makes subsequent concurrent calls no-op)
+  touchedFiles.add(file);
+
+  // Skip if backend doesn't support deletion
+  if (!graph.deleteNode) {
+    return 0;
+  }
+
+  // Collect all nodes for this file
+  const nodesToDelete: string[] = [];
+  for await (const node of graph.queryNodes({ file })) {
+    nodesToDelete.push(node.id);
+  }
+
+  // Delete all of them - NO EXCLUSIONS
+  // MODULE nodes will be recreated by INDEXING phase
+  // FUNCTION/CLASS/etc will be recreated by ANALYSIS phase
+  for (const id of nodesToDelete) {
+    try {
+      await graph.deleteNode(id);
+    } catch (err) {
+      // Log but continue - node might already be deleted by concurrent operation
+      console.warn(`[FileNodeManager] Failed to delete ${id}:`, (err as Error).message);
+    }
+  }
+
+  if (nodesToDelete.length > 0) {
+    const fileName = file.split('/').pop() || file;
+    console.log(`[FileNodeManager] Cleared ${nodesToDelete.length} nodes for ${fileName}`);
+  }
+
+  return nodesToDelete.length;
+}
+
+/**
+ * Clear a SERVICE node by ID.
+ * SERVICE nodes have file=directory_path (not individual files), so they need
+ * explicit clearing separate from file-based clearing.
+ *
+ * @param graph - Graph backend with deleteNode support
+ * @param serviceId - SERVICE node ID (e.g., "SERVICE:apps/api")
+ * @returns true if node was deleted, false otherwise
+ */
+export async function clearServiceNodeIfExists(
+  graph: GraphBackend,
+  serviceId: string
+): Promise<boolean> {
+  if (!graph.deleteNode) {
+    return false;
+  }
+
+  try {
+    await graph.deleteNode(serviceId);
+    console.log(`[FileNodeManager] Cleared SERVICE node: ${serviceId}`);
+    return true;
+  } catch (err) {
+    // Node might not exist on fresh analysis - that's OK
+    return false;
+  }
+}

package/src/core/GraphFreshnessChecker.ts

@@ -0,0 +1,143 @@
+/**
+ * GraphFreshnessChecker - checks if graph data matches current files
+ *
+ * Compares contentHash stored in MODULE nodes against current file hashes.
+ * Used by `grafema check` to detect when files have changed since analysis.
+ */
+
+import { access, constants } from 'fs/promises';
+import { calculateFileHashAsync } from './HashUtils.js';
+import type { NodeRecord } from '@grafema/types';
+
+export interface StaleModule {
+  id: string;
+  file: string;
+  storedHash: string;
+  currentHash: string | null;
+  reason: 'changed' | 'deleted' | 'unreadable';
+}
+
+export interface FreshnessResult {
+  isFresh: boolean;
+  staleModules: StaleModule[];
+  freshCount: number;
+  staleCount: number;
+  deletedCount: number;
+  checkDurationMs: number;
+}
+
+export interface FreshnessGraph {
+  queryNodes(query: { type: string }): AsyncGenerator<NodeRecord, void, unknown>;
+}
+
+interface ModuleInfo {
+  id: string;
+  file: string;
+  contentHash: string;
+}
+
+const BATCH_SIZE = 50;
+
+export class GraphFreshnessChecker {
+  async checkFreshness(graph: FreshnessGraph): Promise<FreshnessResult> {
+    const startTime = Date.now();
+
+    const modules: ModuleInfo[] = [];
+    for await (const node of graph.queryNodes({ type: 'MODULE' })) {
+      if (node.file && typeof node.contentHash === 'string') {
+        modules.push({
+          id: node.id,
+          file: node.file,
+          contentHash: node.contentHash
+        });
+      }
+    }
+
+    if (modules.length === 0) {
+      return {
+        isFresh: true,
+        staleModules: [],
+        freshCount: 0,
+        staleCount: 0,
+        deletedCount: 0,
+        checkDurationMs: Date.now() - startTime
+      };
+    }
+
+    const staleModules: StaleModule[] = [];
+    let freshCount = 0;
+    let deletedCount = 0;
+
+    for (let i = 0; i < modules.length; i += BATCH_SIZE) {
+      const batch = modules.slice(i, i + BATCH_SIZE);
+      const results = await Promise.all(
+        batch.map(module => this._checkModuleFreshness(module))
+      );
+
+      for (const result of results) {
+        if (result === null) {
+          freshCount++;
+        } else {
+          staleModules.push(result);
+          if (result.reason === 'deleted') {
+            deletedCount++;
+          }
+        }
+      }
+    }
+
+    return {
+      isFresh: staleModules.length === 0,
+      staleModules,
+      freshCount,
+      staleCount: staleModules.length,
+      deletedCount,
+      checkDurationMs: Date.now() - startTime
+    };
+  }
+
+  private async _checkModuleFreshness(module: ModuleInfo): Promise<StaleModule | null> {
+    const exists = await this._fileExists(module.file);
+    if (!exists) {
+      return {
+        id: module.id,
+        file: module.file,
+        storedHash: module.contentHash,
+        currentHash: null,
+        reason: 'deleted'
+      };
+    }
+
+    const currentHash = await calculateFileHashAsync(module.file);
+    if (currentHash === null) {
+      return {
+        id: module.id,
+        file: module.file,
+        storedHash: module.contentHash,
+        currentHash: null,
+        reason: 'unreadable'
+      };
+    }
+
+    if (currentHash !== module.contentHash) {
+      return {
+        id: module.id,
+        file: module.file,
+        storedHash: module.contentHash,
+        currentHash,
+        reason: 'changed'
+      };
+    }
+
+    return null;
+  }
+
+  private async _fileExists(filePath: string): Promise<boolean> {
+    try {
+      await access(filePath, constants.R_OK);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}

package/src/core/HashUtils.ts

@@ -0,0 +1,48 @@
+/**
+ * HashUtils - unified hash computation for Grafema
+ *
+ * WHY THIS EXISTS:
+ * - 6 copies of the same hash computation existed across the codebase
+ * - Single source of truth ensures consistent hashing everywhere
+ * - Makes future algorithm changes (e.g., SHA-256 -> BLAKE3) trivial
+ */
+
+import { createHash } from 'crypto';
+import { readFileSync } from 'fs';
+import { readFile } from 'fs/promises';
+
+const HASH_ALGORITHM = 'sha256';
+
+/**
+ * Calculate hash from a file path (synchronous).
+ * Returns null if file doesn't exist or is unreadable.
+ */
+export function calculateFileHash(filePath: string): string | null {
+  try {
+    const content = readFileSync(filePath, 'utf-8');
+    return createHash(HASH_ALGORITHM).update(content).digest('hex');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Calculate hash from a file path (asynchronous).
+ * Returns null if file doesn't exist or is unreadable.
+ */
+export async function calculateFileHashAsync(filePath: string): Promise<string | null> {
+  try {
+    const content = await readFile(filePath, 'utf-8');
+    return createHash(HASH_ALGORITHM).update(content).digest('hex');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Calculate hash from content string.
+ * Always returns a hash (never null).
+ */
+export function calculateContentHash(content: string): string {
+  return createHash(HASH_ALGORITHM).update(content).digest('hex');
+}