@optave/codegraph 3.1.1 → 3.1.3

package/README.md

@@ -562,14 +562,14 @@ Self-measured on every release via CI ([build benchmarks](generated/benchmarks/B
 
 | Metric | Latest |
 |---|---|
-| Build speed (native) | **6.1 ms/file** |
-| Build speed (WASM) | **16.5 ms/file** |
-| Query time | **3ms** |
-| No-op rebuild (native) | **5ms** |
-| 1-file rebuild (native) | **332ms** |
+| Build speed (native) | **5.2 ms/file** |
+| Build speed (WASM) | **15 ms/file** |
+| Query time | **4ms** |
+| No-op rebuild (native) | **6ms** |
+| 1-file rebuild (native) | **296ms** |
 | Query: fn-deps | **0.8ms** |
 | Query: path | **0.8ms** |
-| ~50,000 files (est.) | **~305.0s build** |
+| ~50,000 files (est.) | **~260.0s build** |
 
 Metrics are normalized per file for cross-version comparability. Times above are for a full initial build — incremental rebuilds only re-parse changed files.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@optave/codegraph",
-  "version": "3.1.1",
+  "version": "3.1.3",
   "description": "Local code graph CLI — parse codebases with tree-sitter, build dependency graphs, query them",
   "type": "module",
   "main": "src/index.js",
@@ -71,12 +71,12 @@
   },
   "optionalDependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0",
-    "@optave/codegraph-darwin-arm64": "3.1.1",
-    "@optave/codegraph-darwin-x64": "3.1.1",
-    "@optave/codegraph-linux-arm64-gnu": "3.1.1",
-    "@optave/codegraph-linux-x64-gnu": "3.1.1",
-    "@optave/codegraph-linux-x64-musl": "3.1.1",
-    "@optave/codegraph-win32-x64-msvc": "3.1.1"
+    "@optave/codegraph-darwin-arm64": "3.1.3",
+    "@optave/codegraph-darwin-x64": "3.1.3",
+    "@optave/codegraph-linux-arm64-gnu": "3.1.3",
+    "@optave/codegraph-linux-x64-gnu": "3.1.3",
+    "@optave/codegraph-linux-x64-musl": "3.1.3",
+    "@optave/codegraph-win32-x64-msvc": "3.1.3"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.4",

package/src/ast-analysis/engine.js

@@ -0,0 +1,365 @@
+/**
+ * Unified AST analysis engine — orchestrates all analysis passes in one file-iteration loop.
+ *
+ * Replaces the 4 sequential buildXxx calls in builder.js with a single coordinated pass:
+ *   - AST node extraction (calls, new, string, regex, throw, await)
+ *   - Complexity metrics (cognitive, cyclomatic, nesting, Halstead, MI)
+ *   - CFG construction (basic blocks + edges)
+ *   - Dataflow analysis (define-use chains, arg flows, mutations)
+ *
+ * All 4 analyses run as visitors in a single DFS walk via walkWithVisitors.
+ *
+ * Optimization strategy: for files with WASM trees, run all applicable visitors
+ * in a single walkWithVisitors call. Store results in the format that buildXxx
+ * functions already expect as pre-computed data (same fields as native engine
+ * output). This eliminates redundant tree traversals per file.
+ */
+
+import path from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { bulkNodeIdsByFile } from '../db.js';
+import { debug } from '../logger.js';
+import { computeLOCMetrics, computeMaintainabilityIndex } from './metrics.js';
+import {
+  AST_TYPE_MAPS,
+  CFG_RULES,
+  COMPLEXITY_RULES,
+  DATAFLOW_RULES,
+  HALSTEAD_RULES,
+} from './rules/index.js';
+import { buildExtensionSet, buildExtToLangMap } from './shared.js';
+import { walkWithVisitors } from './visitor.js';
+import { functionName as getFuncName } from './visitor-utils.js';
+import { createAstStoreVisitor } from './visitors/ast-store-visitor.js';
+import { createCfgVisitor } from './visitors/cfg-visitor.js';
+import { createComplexityVisitor } from './visitors/complexity-visitor.js';
+import { createDataflowVisitor } from './visitors/dataflow-visitor.js';
+
+// ─── Extension sets for quick language-support checks ────────────────────
+
+const CFG_EXTENSIONS = buildExtensionSet(CFG_RULES);
+const DATAFLOW_EXTENSIONS = buildExtensionSet(DATAFLOW_RULES);
+const WALK_EXTENSIONS = buildExtensionSet(AST_TYPE_MAPS);
+
+// ─── Lazy imports (heavy modules loaded only when needed) ────────────────
+
+let _parserModule = null;
+async function getParserModule() {
+  if (!_parserModule) _parserModule = await import('../parser.js');
+  return _parserModule;
+}
+
+// ─── Public API ──────────────────────────────────────────────────────────
+
+/**
+ * Run all enabled AST analyses in a coordinated pass.
+ *
+ * @param {object} db - open better-sqlite3 database (read-write)
+ * @param {Map<string, object>} fileSymbols - Map<relPath, { definitions, calls, _tree, _langId, ... }>
+ * @param {string} rootDir - absolute project root path
+ * @param {object} opts - build options (ast, complexity, cfg, dataflow toggles)
+ * @param {object} [engineOpts] - engine options
+ * @returns {Promise<{ astMs: number, complexityMs: number, cfgMs: number, dataflowMs: number }>}
+ */
+export async function runAnalyses(db, fileSymbols, rootDir, opts, engineOpts) {
+  const timing = { astMs: 0, complexityMs: 0, cfgMs: 0, dataflowMs: 0 };
+
+  const doAst = opts.ast !== false;
+  const doComplexity = opts.complexity !== false;
+  const doCfg = opts.cfg !== false;
+  const doDataflow = opts.dataflow !== false;
+
+  if (!doAst && !doComplexity && !doCfg && !doDataflow) return timing;
+
+  const extToLang = buildExtToLangMap();
+
+  // ── WASM pre-parse for files that need it ───────────────────────────
+  // CFG now runs as a visitor in the unified walk, so only dataflow
+  // triggers WASM pre-parse when no tree exists.
+  if (doDataflow) {
+    let needsWasmTrees = false;
+    for (const [relPath, symbols] of fileSymbols) {
+      if (symbols._tree) continue;
+      const ext = path.extname(relPath).toLowerCase();
+
+      if (!symbols.dataflow && DATAFLOW_EXTENSIONS.has(ext)) {
+        needsWasmTrees = true;
+        break;
+      }
+    }
+
+    if (needsWasmTrees) {
+      try {
+        const { ensureWasmTrees } = await getParserModule();
+        await ensureWasmTrees(fileSymbols, rootDir);
+      } catch (err) {
+        debug(`ensureWasmTrees failed: ${err.message}`);
+      }
+    }
+  }
+
+  // ── Phase 7 Optimization: Unified pre-walk ─────────────────────────
+  // For files with WASM trees, run all applicable visitors in a SINGLE
+  // walkWithVisitors call. Store results in the format that buildXxx
+  // functions already expect as pre-computed data (same fields as native
+  // engine output). This eliminates ~3 redundant tree traversals per file.
+  const t0walk = performance.now();
+
+  for (const [relPath, symbols] of fileSymbols) {
+    if (!symbols._tree) continue; // No WASM tree — native path handles it
+
+    const ext = path.extname(relPath).toLowerCase();
+    const langId = symbols._langId || extToLang.get(ext);
+    if (!langId) continue;
+
+    const defs = symbols.definitions || [];
+    const visitors = [];
+    const walkerOpts = {
+      functionNodeTypes: new Set(),
+      nestingNodeTypes: new Set(),
+      getFunctionName: (_node) => null,
+    };
+
+    // ─ AST-store visitor ─
+    const astTypeMap = AST_TYPE_MAPS.get(langId);
+    let astVisitor = null;
+    if (doAst && astTypeMap && WALK_EXTENSIONS.has(ext) && !symbols.astNodes?.length) {
+      const nodeIdMap = new Map();
+      for (const row of bulkNodeIdsByFile(db, relPath)) {
+        nodeIdMap.set(`${row.name}|${row.kind}|${row.line}`, row.id);
+      }
+      astVisitor = createAstStoreVisitor(astTypeMap, defs, relPath, nodeIdMap);
+      visitors.push(astVisitor);
+    }
+
+    // ─ Complexity visitor (file-level mode) ─
+    const cRules = COMPLEXITY_RULES.get(langId);
+    const hRules = HALSTEAD_RULES.get(langId);
+    let complexityVisitor = null;
+    if (doComplexity && cRules) {
+      // Only use visitor if some functions lack pre-computed complexity
+      const needsWasmComplexity = defs.some(
+        (d) => (d.kind === 'function' || d.kind === 'method') && d.line && !d.complexity,
+      );
+      if (needsWasmComplexity) {
+        complexityVisitor = createComplexityVisitor(cRules, hRules, {
+          fileLevelWalk: true,
+          langId,
+        });
+        visitors.push(complexityVisitor);
+
+        // Merge nesting nodes for complexity tracking
+        // NOTE: do NOT add functionNodes here — funcDepth in the complexity
+        // visitor already tracks function-level nesting.  Adding them to
+        // nestingNodeTypes would inflate context.nestingLevel by +1 inside
+        // every function body, double-counting in cognitive += 1 + nestingLevel.
+        for (const t of cRules.nestingNodes) walkerOpts.nestingNodeTypes.add(t);
+
+        // Provide getFunctionName for complexity visitor
+        const dfRules = DATAFLOW_RULES.get(langId);
+        walkerOpts.getFunctionName = (node) => {
+          // Try complexity rules' function name field first
+          const nameNode = node.childForFieldName('name');
+          if (nameNode) return nameNode.text;
+          // Fall back to dataflow rules' richer name extraction
+          if (dfRules) return getFuncName(node, dfRules);
+          return null;
+        };
+      }
+    }
+
+    // ─ CFG visitor ─
+    const cfgRulesForLang = CFG_RULES.get(langId);
+    let cfgVisitor = null;
+    if (doCfg && cfgRulesForLang && CFG_EXTENSIONS.has(ext)) {
+      // Only use visitor if some functions lack pre-computed CFG
+      const needsWasmCfg = defs.some(
+        (d) =>
+          (d.kind === 'function' || d.kind === 'method') &&
+          d.line &&
+          d.cfg !== null &&
+          !Array.isArray(d.cfg?.blocks),
+      );
+      if (needsWasmCfg) {
+        cfgVisitor = createCfgVisitor(cfgRulesForLang);
+        visitors.push(cfgVisitor);
+      }
+    }
+
+    // ─ Dataflow visitor ─
+    const dfRules = DATAFLOW_RULES.get(langId);
+    let dataflowVisitor = null;
+    if (doDataflow && dfRules && DATAFLOW_EXTENSIONS.has(ext) && !symbols.dataflow) {
+      dataflowVisitor = createDataflowVisitor(dfRules);
+      visitors.push(dataflowVisitor);
+    }
+
+    // ─ Run unified walk if we have visitors ─
+    if (visitors.length === 0) continue;
+
+    const results = walkWithVisitors(symbols._tree.rootNode, visitors, langId, walkerOpts);
+
+    // ─ Store AST results (buildAstNodes will find symbols.astNodes and skip its walk) ─
+    if (astVisitor) {
+      const astRows = results['ast-store'] || [];
+      if (astRows.length > 0) {
+        // Store in the format buildAstNodes expects for the native path
+        symbols.astNodes = astRows;
+      }
+    }
+
+    // ─ Store complexity results on definitions (buildComplexityMetrics will find def.complexity) ─
+    if (complexityVisitor) {
+      const complexityResults = results.complexity || [];
+      // Match results back to definitions by function start line
+      // Store the full result (metrics + funcNode) for O(1) lookup
+      const resultByLine = new Map();
+      for (const r of complexityResults) {
+        if (r.funcNode) {
+          const line = r.funcNode.startPosition.row + 1;
+          if (!resultByLine.has(line)) resultByLine.set(line, []);
+          resultByLine.get(line).push(r);
+        }
+      }
+      for (const def of defs) {
+        if ((def.kind === 'function' || def.kind === 'method') && def.line && !def.complexity) {
+          const candidates = resultByLine.get(def.line);
+          const funcResult = !candidates
+            ? undefined
+            : candidates.length === 1
+              ? candidates[0]
+              : (candidates.find((r) => {
+                  const n = r.funcNode.childForFieldName('name');
+                  return n && n.text === def.name;
+                }) ?? candidates[0]);
+          if (funcResult) {
+            const { metrics } = funcResult;
+            const loc = computeLOCMetrics(funcResult.funcNode, langId);
+            const volume = metrics.halstead ? metrics.halstead.volume : 0;
+            const commentRatio = loc.loc > 0 ? loc.commentLines / loc.loc : 0;
+            const mi = computeMaintainabilityIndex(
+              volume,
+              metrics.cyclomatic,
+              loc.sloc,
+              commentRatio,
+            );
+
+            def.complexity = {
+              cognitive: metrics.cognitive,
+              cyclomatic: metrics.cyclomatic,
+              maxNesting: metrics.maxNesting,
+              halstead: metrics.halstead,
+              loc,
+              maintainabilityIndex: mi,
+            };
+          }
+        }
+      }
+    }
+
+    // ─ Store CFG results on definitions (buildCFGData will find def.cfg and skip its walk) ─
+    if (cfgVisitor) {
+      const cfgResults = results.cfg || [];
+      const cfgByLine = new Map();
+      for (const r of cfgResults) {
+        if (r.funcNode) {
+          const line = r.funcNode.startPosition.row + 1;
+          if (!cfgByLine.has(line)) cfgByLine.set(line, []);
+          cfgByLine.get(line).push(r);
+        }
+      }
+      for (const def of defs) {
+        if (
+          (def.kind === 'function' || def.kind === 'method') &&
+          def.line &&
+          !def.cfg?.blocks?.length
+        ) {
+          const candidates = cfgByLine.get(def.line);
+          const cfgResult = !candidates
+            ? undefined
+            : candidates.length === 1
+              ? candidates[0]
+              : (candidates.find((r) => {
+                  const n = r.funcNode.childForFieldName('name');
+                  return n && n.text === def.name;
+                }) ?? candidates[0]);
+          if (cfgResult) {
+            def.cfg = { blocks: cfgResult.blocks, edges: cfgResult.edges };
+
+            // Override complexity's cyclomatic with CFG-derived value (single source of truth)
+            // and recompute maintainability index to stay consistent
+            if (def.complexity && cfgResult.cyclomatic != null) {
+              def.complexity.cyclomatic = cfgResult.cyclomatic;
+              const { loc, halstead } = def.complexity;
+              const volume = halstead ? halstead.volume : 0;
+              const commentRatio = loc?.loc > 0 ? loc.commentLines / loc.loc : 0;
+              def.complexity.maintainabilityIndex = computeMaintainabilityIndex(
+                volume,
+                cfgResult.cyclomatic,
+                loc?.sloc ?? 0,
+                commentRatio,
+              );
+            }
+          }
+        }
+      }
+    }
+
+    // ─ Store dataflow results (buildDataflowEdges will find symbols.dataflow and skip its walk) ─
+    if (dataflowVisitor) {
+      symbols.dataflow = results.dataflow;
+    }
+  }
+
+  timing._unifiedWalkMs = performance.now() - t0walk;
+
+  // ── Delegate to buildXxx functions ─────────────────────────────────
+  // Each function finds pre-computed data from the unified walk above
+  // (or from the native engine) and only does DB writes + native fallback.
+
+  if (doAst) {
+    const t0 = performance.now();
+    try {
+      const { buildAstNodes } = await import('../ast.js');
+      await buildAstNodes(db, fileSymbols, rootDir, engineOpts);
+    } catch (err) {
+      debug(`buildAstNodes failed: ${err.message}`);
+    }
+    timing.astMs = performance.now() - t0;
+  }
+
+  if (doComplexity) {
+    const t0 = performance.now();
+    try {
+      const { buildComplexityMetrics } = await import('../complexity.js');
+      await buildComplexityMetrics(db, fileSymbols, rootDir, engineOpts);
+    } catch (err) {
+      debug(`buildComplexityMetrics failed: ${err.message}`);
+    }
+    timing.complexityMs = performance.now() - t0;
+  }
+
+  if (doCfg) {
+    const t0 = performance.now();
+    try {
+      const { buildCFGData } = await import('../cfg.js');
+      await buildCFGData(db, fileSymbols, rootDir, engineOpts);
+    } catch (err) {
+      debug(`buildCFGData failed: ${err.message}`);
+    }
+    timing.cfgMs = performance.now() - t0;
+  }
+
+  if (doDataflow) {
+    const t0 = performance.now();
+    try {
+      const { buildDataflowEdges } = await import('../dataflow.js');
+      await buildDataflowEdges(db, fileSymbols, rootDir, engineOpts);
+    } catch (err) {
+      debug(`buildDataflowEdges failed: ${err.message}`);
+    }
+    timing.dataflowMs = performance.now() - t0;
+  }
+
+  return timing;
+}

package/src/ast-analysis/metrics.js

@@ -0,0 +1,118 @@
+/**
+ * Pure metric computations extracted from complexity.js.
+ *
+ * Contains Halstead derived metrics, LOC metrics, and Maintainability Index —
+ * all stateless math that can be reused by visitor-based and standalone paths.
+ */
+
+// ─── Halstead Derived Metrics ─────────────────────────────────────────────
+
+/**
+ * Compute Halstead derived metrics from raw operator/operand counts.
+ *
+ * @param {Map<string, number>} operators - operator type/text → count
+ * @param {Map<string, number>} operands  - operand text → count
+ * @returns {{ n1: number, n2: number, bigN1: number, bigN2: number, vocabulary: number, length: number, volume: number, difficulty: number, effort: number, bugs: number }}
+ */
+export function computeHalsteadDerived(operators, operands) {
+  const n1 = operators.size;
+  const n2 = operands.size;
+  let bigN1 = 0;
+  for (const c of operators.values()) bigN1 += c;
+  let bigN2 = 0;
+  for (const c of operands.values()) bigN2 += c;
+
+  const vocabulary = n1 + n2;
+  const length = bigN1 + bigN2;
+  const volume = vocabulary > 0 ? length * Math.log2(vocabulary) : 0;
+  const difficulty = n2 > 0 ? (n1 / 2) * (bigN2 / n2) : 0;
+  const effort = difficulty * volume;
+  const bugs = volume / 3000;
+
+  return {
+    n1,
+    n2,
+    bigN1,
+    bigN2,
+    vocabulary,
+    length,
+    volume: +volume.toFixed(2),
+    difficulty: +difficulty.toFixed(2),
+    effort: +effort.toFixed(2),
+    bugs: +bugs.toFixed(4),
+  };
+}
+
+// ─── LOC Metrics ──────────────────────────────────────────────────────────
+
+const C_STYLE_PREFIXES = ['//', '/*', '*', '*/'];
+
+const COMMENT_PREFIXES = new Map([
+  ['javascript', C_STYLE_PREFIXES],
+  ['typescript', C_STYLE_PREFIXES],
+  ['tsx', C_STYLE_PREFIXES],
+  ['go', C_STYLE_PREFIXES],
+  ['rust', C_STYLE_PREFIXES],
+  ['java', C_STYLE_PREFIXES],
+  ['csharp', C_STYLE_PREFIXES],
+  ['python', ['#']],
+  ['ruby', ['#']],
+  ['php', ['//', '#', '/*', '*', '*/']],
+]);
+
+/**
+ * Compute LOC metrics from a function node's source text.
+ *
+ * @param {object} functionNode - tree-sitter node
+ * @param {string} [language] - Language ID (falls back to C-style prefixes)
+ * @returns {{ loc: number, sloc: number, commentLines: number }}
+ */
+export function computeLOCMetrics(functionNode, language) {
+  const text = functionNode.text;
+  const lines = text.split('\n');
+  const loc = lines.length;
+  const prefixes = (language && COMMENT_PREFIXES.get(language)) || C_STYLE_PREFIXES;
+
+  let commentLines = 0;
+  let blankLines = 0;
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed === '') {
+      blankLines++;
+    } else if (prefixes.some((p) => trimmed.startsWith(p))) {
+      commentLines++;
+    }
+  }
+
+  const sloc = Math.max(1, loc - blankLines - commentLines);
+  return { loc, sloc, commentLines };
+}
+
+// ─── Maintainability Index ────────────────────────────────────────────────
+
+/**
+ * Compute normalized Maintainability Index (0-100 scale).
+ *
+ * Original SEI formula: MI = 171 - 5.2*ln(V) - 0.23*G - 16.2*ln(LOC) + 50*sin(sqrt(2.4*CM))
+ * Microsoft normalization: max(0, min(100, MI * 100/171))
+ *
+ * @param {number} volume - Halstead volume
+ * @param {number} cyclomatic - Cyclomatic complexity
+ * @param {number} sloc - Source lines of code
+ * @param {number} [commentRatio] - Comment ratio (0-1), optional
+ * @returns {number} Normalized MI (0-100)
+ */
+export function computeMaintainabilityIndex(volume, cyclomatic, sloc, commentRatio) {
+  const safeVolume = Math.max(volume, 1);
+  const safeSLOC = Math.max(sloc, 1);
+
+  let mi = 171 - 5.2 * Math.log(safeVolume) - 0.23 * cyclomatic - 16.2 * Math.log(safeSLOC);
+
+  if (commentRatio != null && commentRatio > 0) {
+    mi += 50 * Math.sin(Math.sqrt(2.4 * commentRatio));
+  }
+
+  const normalized = Math.max(0, Math.min(100, (mi * 100) / 171));
+  return +normalized.toFixed(1);
+}