@optave/codegraph 3.11.0 → 3.11.2

package/src/features/structure-query.ts

@@ -227,6 +227,96 @@ interface HotspotsDataOpts {
   noTests?: boolean;
 }
 
+type HotspotEntry = {
+  name: string;
+  kind: string;
+  lineCount: number | null;
+  symbolCount: number | null;
+  importCount: number | null;
+  exportCount: number | null;
+  fanIn: number | null;
+  fanOut: number | null;
+  cohesion: number | null;
+  fileCount: number | null;
+  density: number;
+  coupling: number;
+};
+
+/** Compute density from either fileCount/symbolCount or lineCount/symbolCount. */
+function computeHotspotDensity(
+  symbolCount: number | null,
+  fileCount: number | null,
+  lineCount: number | null,
+): number {
+  if ((fileCount ?? 0) > 0) return (symbolCount || 0) / (fileCount ?? 1);
+  if ((lineCount ?? 0) > 0) return (symbolCount || 0) / (lineCount ?? 1);
+  return 0;
+}
+
+/** Map a native-engine hotspot row (camelCase keys) to the public HotspotEntry shape. */
+function mapNativeHotspotRow(r: {
+  name: string;
+  kind: string;
+  lineCount: number | null;
+  symbolCount: number | null;
+  importCount: number | null;
+  exportCount: number | null;
+  fanIn: number | null;
+  fanOut: number | null;
+  cohesion: number | null;
+  fileCount: number | null;
+}): HotspotEntry {
+  return {
+    name: r.name,
+    kind: r.kind,
+    lineCount: r.lineCount,
+    symbolCount: r.symbolCount,
+    importCount: r.importCount,
+    exportCount: r.exportCount,
+    fanIn: r.fanIn,
+    fanOut: r.fanOut,
+    cohesion: r.cohesion,
+    fileCount: r.fileCount,
+    density: computeHotspotDensity(r.symbolCount, r.fileCount, r.lineCount),
+    coupling: (r.fanIn || 0) + (r.fanOut || 0),
+  };
+}
+
+/** Map a JS-path hotspot row (snake_case keys from SQLite) to the public HotspotEntry shape. */
+function mapJsHotspotRow(r: HotspotRow): HotspotEntry {
+  return {
+    name: r.name,
+    kind: r.kind,
+    lineCount: r.line_count,
+    symbolCount: r.symbol_count,
+    importCount: r.import_count,
+    exportCount: r.export_count,
+    fanIn: r.fan_in,
+    fanOut: r.fan_out,
+    cohesion: r.cohesion,
+    fileCount: r.file_count,
+    density: computeHotspotDensity(r.symbol_count, r.file_count, r.line_count),
+    coupling: (r.fan_in || 0) + (r.fan_out || 0),
+  };
+}
+
+/** ORDER BY clause for each ranking dimension (strategy pattern). */
+const HOTSPOT_ORDER_BY: Record<string, string> = {
+  'fan-in': 'nm.fan_in DESC NULLS LAST',
+  'fan-out': 'nm.fan_out DESC NULLS LAST',
+  density: 'nm.symbol_count DESC NULLS LAST',
+  coupling: '(COALESCE(nm.fan_in, 0) + COALESCE(nm.fan_out, 0)) DESC NULLS LAST',
+};
+
+/** Build the JS-path SQL query for a given metric and test filter. */
+function buildHotspotQuery(metric: string, testFilter: string): string {
+  const orderBy = HOTSPOT_ORDER_BY[metric] ?? HOTSPOT_ORDER_BY['fan-in'];
+  return `SELECT n.name, n.kind, nm.line_count, nm.symbol_count, nm.import_count, nm.export_count,
+                 nm.fan_in, nm.fan_out, nm.cohesion, nm.file_count
+          FROM nodes n JOIN node_metrics nm ON n.id = nm.node_id
+          WHERE n.kind = ? ${testFilter} ORDER BY ${orderBy} LIMIT ?`;
+}
+
 export function hotspotsData(
   customDbPath?: string,
   opts: HotspotsDataOpts = {},
@@ -242,96 +332,21 @@ export function hotspotsData(
     const level = opts.level || 'file';
     const limit = opts.limit || 10;
     const noTests = opts.noTests || false;
-
     const kind = level === 'directory' ? 'directory' : 'file';
 
-    const mapRow = (r: {
-      name: string;
-      kind: string;
-      lineCount: number | null;
-      symbolCount: number | null;
-      importCount: number | null;
-      exportCount: number | null;
-      fanIn: number | null;
-      fanOut: number | null;
-      cohesion: number | null;
-      fileCount: number | null;
-    }) => ({
-      name: r.name,
-      kind: r.kind,
-      lineCount: r.lineCount,
-      symbolCount: r.symbolCount,
-      importCount: r.importCount,
-      exportCount: r.exportCount,
-      fanIn: r.fanIn,
-      fanOut: r.fanOut,
-      cohesion: r.cohesion,
-      fileCount: r.fileCount,
-      density:
-        (r.fileCount ?? 0) > 0
-          ? (r.symbolCount || 0) / (r.fileCount ?? 1)
-          : (r.lineCount ?? 0) > 0
-            ? (r.symbolCount || 0) / (r.lineCount ?? 1)
-            : 0,
-      coupling: (r.fanIn || 0) + (r.fanOut || 0),
-    });
-
     // ── Native fast path: single query instead of 4 eagerly prepared ──
     if (nativeDb?.getHotspots) {
       const rows = nativeDb.getHotspots(kind, metric, noTests, limit);
-      const hotspots = rows.map(mapRow);
+      const hotspots = rows.map(mapNativeHotspotRow);
       const base = { metric, level, limit, hotspots };
       return paginateResult(base, 'hotspots', { limit: opts.limit, offset: opts.offset });
     }
 
     // ── JS fallback ───────────────────────────────────────────────────
     const testFilter = testFilterSQL('n.name', noTests && kind === 'file');
-
-    const HOTSPOT_QUERIES: Record<string, { all(...params: unknown[]): HotspotRow[] }> = {
-      'fan-in': db.prepare(`
-        SELECT n.name, n.kind, nm.line_count, nm.symbol_count, nm.import_count, nm.export_count,
-               nm.fan_in, nm.fan_out, nm.cohesion, nm.file_count
-        FROM nodes n JOIN node_metrics nm ON n.id = nm.node_id
-        WHERE n.kind = ? ${testFilter} ORDER BY nm.fan_in DESC NULLS LAST LIMIT ?`),
-      'fan-out': db.prepare(`
-        SELECT n.name, n.kind, nm.line_count, nm.symbol_count, nm.import_count, nm.export_count,
-               nm.fan_in, nm.fan_out, nm.cohesion, nm.file_count
-        FROM nodes n JOIN node_metrics nm ON n.id = nm.node_id
-        WHERE n.kind = ? ${testFilter} ORDER BY nm.fan_out DESC NULLS LAST LIMIT ?`),
-      density: db.prepare(`
-        SELECT n.name, n.kind, nm.line_count, nm.symbol_count, nm.import_count, nm.export_count,
-               nm.fan_in, nm.fan_out, nm.cohesion, nm.file_count
-        FROM nodes n JOIN node_metrics nm ON n.id = nm.node_id
-        WHERE n.kind = ? ${testFilter} ORDER BY nm.symbol_count DESC NULLS LAST LIMIT ?`),
-      coupling: db.prepare(`
-        SELECT n.name, n.kind, nm.line_count, nm.symbol_count, nm.import_count, nm.export_count,
-               nm.fan_in, nm.fan_out, nm.cohesion, nm.file_count
-        FROM nodes n JOIN node_metrics nm ON n.id = nm.node_id
-        WHERE n.kind = ? ${testFilter} ORDER BY (COALESCE(nm.fan_in, 0) + COALESCE(nm.fan_out, 0)) DESC NULLS LAST LIMIT ?`),
-    };
-
-    const stmt = HOTSPOT_QUERIES[metric] ?? HOTSPOT_QUERIES['fan-in'];
-    const rows = stmt!.all(kind, limit);
-
-    const hotspots = rows.map((r) => ({
-      name: r.name,
-      kind: r.kind,
-      lineCount: r.line_count,
-      symbolCount: r.symbol_count,
-      importCount: r.import_count,
-      exportCount: r.export_count,
-      fanIn: r.fan_in,
-      fanOut: r.fan_out,
-      cohesion: r.cohesion,
-      fileCount: r.file_count,
-      density:
-        (r.file_count ?? 0) > 0
-          ? (r.symbol_count || 0) / (r.file_count ?? 1)
-          : (r.line_count ?? 0) > 0
-            ? (r.symbol_count || 0) / (r.line_count ?? 1)
-            : 0,
-      coupling: (r.fan_in || 0) + (r.fan_out || 0),
-    }));
+    const stmt = db.prepare(buildHotspotQuery(metric, testFilter));
+    const rows = stmt.all(kind, limit) as HotspotRow[];
+    const hotspots = rows.map(mapJsHotspotRow);
 
     const base = { metric, level, limit, hotspots };
     return paginateResult(base, 'hotspots', { limit: opts.limit, offset: opts.offset });

package/src/features/structure.ts

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import { getNodeId, testFilterSQL } from '../db/index.js';
+import { getBuildMeta, getNodeId, setBuildMeta, testFilterSQL } from '../db/index.js';
 import { debug } from '../infrastructure/logger.js';
 import { normalizePath } from '../shared/constants.js';
 import type { BetterSqlite3Database } from '../types.js';
@@ -532,6 +532,161 @@ function batchUpdateRoles(
   })();
 }
 
+interface CallableNodeRow {
+  id: number;
+  name: string;
+  kind: string;
+  file: string;
+  fan_in: number;
+  fan_out: number;
+}
+
+/** Build the activeFiles set: files with at least one callable connected to the graph. */
+function buildActiveFilesSet(rows: CallableNodeRow[]): Set<string> {
+  const activeFiles = new Set<string>();
+  for (const r of rows) {
+    if ((r.fan_in > 0 || r.fan_out > 0) && r.kind !== 'constant') {
+      activeFiles.add(r.file);
+    }
+  }
+  return activeFiles;
+}
+
+/** Map callable rows to classifier input objects, attaching exported/prod-fan-in/active-file metadata. */
+function buildClassifierInput(
+  rows: CallableNodeRow[],
+  exportedIds: Set<number>,
+  prodFanInMap: Map<number, number>,
+  activeFiles: Set<string>,
+): Array<{
+  id: string;
+  name: string;
+  kind: string;
+  file: string;
+  fanIn: number;
+  fanOut: number;
+  isExported: boolean;
+  productionFanIn: number;
+  hasActiveFileSiblings: boolean | undefined;
+}> {
+  return rows.map((r) => ({
+    id: String(r.id),
+    name: r.name,
+    kind: r.kind,
+    file: r.file,
+    fanIn: r.fan_in,
+    fanOut: r.fan_out,
+    isExported: exportedIds.has(r.id),
+    productionFanIn: prodFanInMap.get(r.id) || 0,
+    hasActiveFileSiblings: r.kind === 'constant' ? activeFiles.has(r.file) : undefined,
+  }));
+}
+
+// ─── Median cache helpers ─────────────────────────────────────────────────────
+
+const ROLES_MEDIANS_KEY = 'roles_medians';
+
+// Invalidate cached medians when the edge count drifts past this threshold.
+// A 1-file rebuild adds/removes < 100 edges — well within the margin.
+const MEDIAN_INVALIDATION_DELTA = 500;
+
+/**
+ * Full edge-table GROUP BY scan — O(M). Only runs on cache miss.
+ *
+ * Joins `nodes` to restrict to the same non-leaf kinds that
+ * `classifyNodeRolesFull` uses when computing medians from in-memory rows
+ * (excludes 'file', 'directory', 'parameter', 'property'). This keeps the
+ * two paths consistent so a cold-cache fallback produces the same distribution
+ * as the full-build cached value.
+ *
+ * Also returns the filtered edge count used for computing the medians so the
+ * caller can pass it directly to `writeMedianCache` without a second query.
+ */
+function computeGlobalMediansFromEdges(db: BetterSqlite3Database): {
+  fanIn: number;
+  fanOut: number;
+  edgeCount: number;
+} {
+  const excludedKinds = `('file', 'directory', 'parameter', 'property')`;
+  const fanInRows = db
+    .prepare(
+      `SELECT COUNT(*) AS cnt FROM edges e
+       JOIN nodes t ON e.target_id = t.id
+       WHERE e.kind IN ('calls', 'imports-type')
+         AND t.kind NOT IN ${excludedKinds}
+       GROUP BY e.target_id`,
+    )
+    .all() as { cnt: number }[];
+  const fanOutRows = db
+    .prepare(
+      `SELECT COUNT(*) AS cnt FROM edges e
+       JOIN nodes s ON e.source_id = s.id
+       WHERE e.kind = 'calls'
+         AND s.kind NOT IN ${excludedKinds}
+       GROUP BY e.source_id`,
+    )
+    .all() as { cnt: number }[];
+  const fanInDist = fanInRows.map((r) => r.cnt).sort((a, b) => a - b);
+  const fanOutDist = fanOutRows.map((r) => r.cnt).sort((a, b) => a - b);
+  // Sum of fanInRows[*].cnt equals the total edge count for the relevant
+  // edge subset — no extra COUNT query needed.
+  const edgeCount = fanInRows.reduce((acc, r) => acc + r.cnt, 0);
+  return { fanIn: median(fanInDist), fanOut: median(fanOutDist), edgeCount };
+}
+
+/**
+ * Read cached role medians from build_meta. Returns null when absent or stale
+ * (edge count moved beyond MEDIAN_INVALIDATION_DELTA from the cached value).
+ *
+ * The staleness check uses the same edge subset (calls + imports-type) that
+ * the medians are derived from, so only changes to the edges that actually
+ * influence fan-in/fan-out can evict the cache.
+ */
+function readCachedMedians(db: BetterSqlite3Database): { fanIn: number; fanOut: number } | null {
+  const raw = getBuildMeta(db, ROLES_MEDIANS_KEY);
+  if (!raw) return null;
+  try {
+    const cached = JSON.parse(raw) as { fanIn: number; fanOut: number; edgeCount: number };
+    // Count only the edge kinds that drive median computation — same subset
+    // used by computeGlobalMediansFromEdges and classifyNodeRolesFull.
+    const currentCount = (
+      db
+        .prepare(`SELECT COUNT(*) AS cnt FROM edges WHERE kind IN ('calls', 'imports-type')`)
+        .get() as { cnt: number }
+    ).cnt;
+    if (
+      Math.abs(currentCount - cached.edgeCount) >
+      Math.max(MEDIAN_INVALIDATION_DELTA, cached.edgeCount * 0.1)
+    )
+      return null;
+    return { fanIn: cached.fanIn, fanOut: cached.fanOut };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Persist global role medians + current edge count to build_meta.
+ *
+ * @param edgeCount - pre-computed calls+imports-type edge count. When provided,
+ *   the function skips the COUNT query entirely. Pass when the count is already
+ *   known at the call site (e.g. from `computeGlobalMediansFromEdges`).
+ */
+function writeMedianCache(
+  db: BetterSqlite3Database,
+  medians: { fanIn: number; fanOut: number },
+  edgeCount?: number,
+): void {
+  const cnt =
+    edgeCount ??
+    (
+      db
+        .prepare(`SELECT COUNT(*) AS cnt FROM edges WHERE kind IN ('calls', 'imports-type')`)
+        .get() as { cnt: number }
+    ).cnt;
+  setBuildMeta(db, { [ROLES_MEDIANS_KEY]: JSON.stringify({ ...medians, edgeCount: cnt }) });
+}
+
 function classifyNodeRolesFull(db: BetterSqlite3Database, emptySummary: RoleSummary): RoleSummary {
   // Leaf kinds (parameter, property) can never have callers/callees.
   // Classify them directly as dead-leaf without the expensive fan-in/fan-out JOINs.
@@ -558,14 +713,7 @@ function classifyNodeRolesFull(db: BetterSqlite3Database, emptySummary: RoleSumm
       ) fo ON n.id = fo.source_id
       WHERE n.kind NOT IN ('file', 'directory', 'parameter', 'property')`,
     )
-    .all() as {
-    id: number;
-    name: string;
-    kind: string;
-    file: string;
-    fan_in: number;
-    fan_out: number;
-  }[];
+    .all() as CallableNodeRow[];
 
   if (rows.length === 0 && leafRows.length === 0) return emptySummary;
 
@@ -629,29 +777,28 @@ function classifyNodeRolesFull(db: BetterSqlite3Database, emptySummary: RoleSumm
     prodFanInMap.set(r.target_id, r.cnt);
   }
 
-  // Files with at least one callable (non-constant) connected to the graph.
-  // Constants in these files are likely consumed locally via identifier reference.
-  const activeFiles = new Set<string>();
-  for (const r of rows) {
-    if ((r.fan_in > 0 || r.fan_out > 0) && r.kind !== 'constant') {
-      activeFiles.add(r.file);
-    }
-  }
-
-  // Delegate classification to the pure-logic classifier
-  const classifierInput = rows.map((r) => ({
-    id: String(r.id),
-    name: r.name,
-    kind: r.kind,
-    file: r.file,
-    fanIn: r.fan_in,
-    fanOut: r.fan_out,
-    isExported: exportedIds.has(r.id),
-    productionFanIn: prodFanInMap.get(r.id) || 0,
-    hasActiveFileSiblings: r.kind === 'constant' ? activeFiles.has(r.file) : undefined,
-  }));
-
-  const roleMap = classifyRoles(classifierInput);
+  // Delegate classification to the pure-logic classifier.
+  // Compute medians from the already-loaded rows (no extra DB round-trip),
+  // pass them as overrides to avoid recomputing inside classifyRoles,
+  // and cache them for subsequent incremental builds.
+  const activeFiles = buildActiveFilesSet(rows);
+  const classifierInput = buildClassifierInput(rows, exportedIds, prodFanInMap, activeFiles);
+  const nonZeroFanIn = classifierInput
+    .filter((n) => n.fanIn > 0)
+    .map((n) => n.fanIn)
+    .sort((a, b) => a - b);
+  const nonZeroFanOut = classifierInput
+    .filter((n) => n.fanOut > 0)
+    .map((n) => n.fanOut)
+    .sort((a, b) => a - b);
+  const globalMedians = { fanIn: median(nonZeroFanIn), fanOut: median(nonZeroFanOut) };
+  const roleMap = classifyRoles(classifierInput, globalMedians);
+  // Derive the edge count from already-loaded in-memory rows: summing fan_in
+  // across all nodes equals COUNT(*) FROM edges WHERE kind IN ('calls','imports-type'),
+  // since the full-build query left-joins every matching edge exactly once per target.
+  // Passing this avoids an extra COUNT query on the full-build path.
+  const inMemoryEdgeCount = rows.reduce((acc, r) => acc + r.fan_in, 0);
+  writeMedianCache(db, globalMedians, inMemoryEdgeCount);
 
   const { summary, idsByRole } = buildRoleSummary(rows, leafRows, roleMap, emptySummary);
 
@@ -667,7 +814,9 @@ function classifyNodeRolesFull(db: BetterSqlite3Database, emptySummary: RoleSumm
  * plus their immediate edge neighbours (callers and callees in other files).
  *
  * Uses indexed point lookups for fan-in/fan-out instead of full table scans.
- * Global medians are computed from edge distribution (fast GROUP BY on index).
+ * Global medians are read from the build_meta cache written by the last full
+ * classification; the cache is only recomputed when the edge count drifts
+ * beyond MEDIAN_INVALIDATION_DELTA (i.e. large structural changes).
  * Unchanged files not connected to changed files keep their roles from the
  * previous build.
  */
@@ -694,25 +843,20 @@ function classifyNodeRolesIncremental(
   const allAffectedFiles = [...changedFiles, ...neighbourFiles.map((r) => r.file)];
   const placeholders = allAffectedFiles.map(() => '?').join(',');
 
-  // 1. Compute global medians from edge distribution (fast: scans edge index, no node join)
-  const fanInDist = (
-    db
-      .prepare(
-        `SELECT COUNT(*) AS cnt FROM edges WHERE kind IN ('calls', 'imports-type') GROUP BY target_id`,
-      )
-      .all() as { cnt: number }[]
-  )
-    .map((r) => r.cnt)
-    .sort((a, b) => a - b);
-  const fanOutDist = (
-    db
-      .prepare(`SELECT COUNT(*) AS cnt FROM edges WHERE kind = 'calls' GROUP BY source_id`)
-      .all() as { cnt: number }[]
-  )
-    .map((r) => r.cnt)
-    .sort((a, b) => a - b);
-
-  const globalMedians = { fanIn: median(fanInDist), fanOut: median(fanOutDist) };
+  // 1. Read global medians from cache; fall back to full edge scan only on miss.
+  // The median barely moves for a 1-file change, so the cache is almost always
+  // valid, eliminating 2× full edge-table GROUP BY queries (~10-15 ms on large graphs).
+  const cachedMedians = readCachedMedians(db);
+  let globalMedians: { fanIn: number; fanOut: number };
+  if (cachedMedians) {
+    globalMedians = cachedMedians;
+  } else {
+    const computed = computeGlobalMediansFromEdges(db);
+    // Pass the edgeCount returned by computeGlobalMediansFromEdges so
+    // writeMedianCache does not issue a second COUNT query.
+    writeMedianCache(db, computed, computed.edgeCount);
+    globalMedians = computed;
+  }
 
   // 2a. Leaf kinds (parameter, property) in affected files — always dead-leaf
   const leafRows = db
@@ -733,14 +877,7 @@ function classifyNodeRolesIncremental(
       WHERE n.kind NOT IN ('file', 'directory', 'parameter', 'property')
         AND n.file IN (${placeholders})`,
     )
-    .all(...allAffectedFiles) as {
-    id: number;
-    name: string;
-    kind: string;
-    file: string;
-    fan_in: number;
-    fan_out: number;
-  }[];
+    .all(...allAffectedFiles) as CallableNodeRow[];
 
   if (rows.length === 0 && leafRows.length === 0) return emptySummary;
 
@@ -810,25 +947,8 @@ function classifyNodeRolesIncremental(
   }
 
   // 5. Classify affected nodes using global medians
-  const activeFiles = new Set<string>();
-  for (const r of rows) {
-    if ((r.fan_in > 0 || r.fan_out > 0) && r.kind !== 'constant') {
-      activeFiles.add(r.file);
-    }
-  }
-
-  const classifierInput = rows.map((r) => ({
-    id: String(r.id),
-    name: r.name,
-    kind: r.kind,
-    file: r.file,
-    fanIn: r.fan_in,
-    fanOut: r.fan_out,
-    isExported: exportedIds.has(r.id),
-    productionFanIn: prodFanInMap.get(r.id) || 0,
-    hasActiveFileSiblings: r.kind === 'constant' ? activeFiles.has(r.file) : undefined,
-  }));
-
+  const activeFiles = buildActiveFilesSet(rows);
+  const classifierInput = buildClassifierInput(rows, exportedIds, prodFanInMap, activeFiles);
   const roleMap = classifyRoles(classifierInput, globalMedians);
 
   // 6. Build summary (only for affected nodes) and update only those nodes