@optave/codegraph 3.4.1 → 3.6.0

package/src/domain/graph/builder/stages/collect-files.ts

@@ -2,14 +2,78 @@
  * Stage: collectFiles
  *
  * Collects all source files to process. Handles both normal and scoped rebuilds.
+ * For incremental builds with a valid journal, reconstructs the file list from
+ * the DB's file_hashes table + journal deltas, skipping the filesystem scan.
  */
 import fs from 'node:fs';
 import path from 'node:path';
-import { info } from '../../../../infrastructure/logger.js';
+import { debug, info } from '../../../../infrastructure/logger.js';
 import { normalizePath } from '../../../../shared/constants.js';
+import { readJournal } from '../../journal.js';
 import type { PipelineContext } from '../context.js';
 import { collectFiles as collectFilesUtil } from '../helpers.js';
 
+/**
+ * Reconstruct allFiles from DB file_hashes + journal deltas.
+ * Returns null when the fast path isn't applicable (first build, no journal, etc).
+ */
+function tryFastCollect(
+  ctx: PipelineContext,
+): { files: string[]; directories: Set<string> } | null {
+  const { db, rootDir } = ctx;
+
+  // 1. Check that file_hashes table exists and has entries
+  let dbFileCount: number;
+  try {
+    dbFileCount = (db.prepare('SELECT COUNT(*) as c FROM file_hashes').get() as { c: number }).c;
+  } catch {
+    return null;
+  }
+  if (dbFileCount === 0) return null;
+
+  // 2. Read the journal — only use fast path when journal has entries,
+  // proving the watcher was active and tracking changes. An empty-but-valid
+  // journal (no watcher) could miss file deletions.
+  const journal = readJournal(rootDir);
+  if (!journal.valid) return null;
+  const hasEntries =
+    (journal.changed && journal.changed.length > 0) ||
+    (journal.removed && journal.removed.length > 0);
+  if (!hasEntries) return null;
+
+  // 3. Load existing file list from file_hashes (relative paths)
+  const dbFiles = (db.prepare('SELECT file FROM file_hashes').all() as Array<{ file: string }>).map(
+    (r) => r.file,
+  );
+
+  // 4. Apply journal deltas: remove deleted files, add new/changed files
+  const fileSet = new Set(dbFiles);
+  if (journal.removed) {
+    for (const removed of journal.removed) {
+      fileSet.delete(removed);
+    }
+  }
+  if (journal.changed) {
+    for (const changed of journal.changed) {
+      fileSet.add(changed);
+    }
+  }
+
+  // 5. Convert to absolute paths and compute directories
+  const files: string[] = [];
+  const directories = new Set<string>();
+  for (const relPath of fileSet) {
+    const absPath = path.join(rootDir, relPath);
+    files.push(absPath);
+    directories.add(path.dirname(absPath));
+  }
+
+  debug(
+    `collectFiles fast path: ${dbFiles.length} from DB, journal: +${journal.changed?.length ?? 0}/-${journal.removed?.length ?? 0} → ${files.length} files`,
+  );
+  return { files, directories };
+}
+
 export async function collectFiles(ctx: PipelineContext): Promise<void> {
   const { rootDir, config, opts } = ctx;
 
@@ -33,10 +97,23 @@ export async function collectFiles(ctx: PipelineContext): Promise<void> {
     ctx.removed = missing;
     ctx.isFullBuild = false;
     info(`Scoped rebuild: ${existing.length} files to rebuild, ${missing.length} to purge`);
-  } else {
-    const collected = collectFilesUtil(rootDir, [], config, new Set<string>());
-    ctx.allFiles = collected.files;
-    ctx.discoveredDirs = collected.directories;
-    info(`Found ${ctx.allFiles.length} files to parse`);
+    return;
   }
+
+  // Incremental fast path: reconstruct file list from DB + journal deltas
+  // instead of full recursive filesystem scan (~8ms savings on 473 files).
+  if (ctx.incremental && !ctx.forceFullRebuild) {
+    const fast = tryFastCollect(ctx);
+    if (fast) {
+      ctx.allFiles = fast.files;
+      ctx.discoveredDirs = fast.directories;
+      info(`Found ${ctx.allFiles.length} files (cached)`);
+      return;
+    }
+  }
+
+  const collected = collectFilesUtil(rootDir, [], config, new Set<string>());
+  ctx.allFiles = collected.files;
+  ctx.discoveredDirs = collected.directories;
+  info(`Found ${ctx.allFiles.length} files to parse`);
 }

package/src/domain/graph/builder/stages/detect-changes.ts

@@ -10,7 +10,7 @@ import path from 'node:path';
 import { closeDb } from '../../../../db/index.js';
 import { debug, info } from '../../../../infrastructure/logger.js';
 import { normalizePath } from '../../../../shared/constants.js';
-import type { BetterSqlite3Database, ExtractorOutput } from '../../../../types.js';
+import type { BetterSqlite3Database, ExtractorOutput, NativeDatabase } from '../../../../types.js';
 import { parseFilesAuto } from '../../../parser.js';
 import { readJournal, writeJournalHeader } from '../../journal.js';
 import type { PipelineContext } from '../context.js';
@@ -58,10 +58,16 @@ function getChangedFiles(
   db: BetterSqlite3Database,
   allFiles: string[],
   rootDir: string,
+  nativeDb?: NativeDatabase,
 ): ChangeResult {
   let hasTable = false;
   try {
-    db.prepare('SELECT 1 FROM file_hashes LIMIT 1').get();
+    if (nativeDb) {
+      nativeDb.queryGet('SELECT 1 FROM file_hashes LIMIT 1', []);
+    } else {
+      db.prepare('SELECT 1 FROM file_hashes LIMIT 1').get();
+    }
+    // Query succeeded → table exists (result may be undefined if table is empty)
     hasTable = true;
   } catch {
     /* table doesn't exist */
@@ -75,11 +81,11 @@ function getChangedFiles(
     };
   }
 
-  const existing = new Map<string, FileHashRow>(
-    (db.prepare('SELECT file, hash, mtime, size FROM file_hashes').all() as FileHashRow[]).map(
-      (r) => [r.file, r],
-    ),
-  );
+  const sql = 'SELECT file, hash, mtime, size FROM file_hashes';
+  const rows = nativeDb
+    ? (nativeDb.queryAll(sql, []) as unknown as FileHashRow[])
+    : (db.prepare(sql).all() as FileHashRow[]);
+  const existing = new Map<string, FileHashRow>(rows.map((r) => [r.file, r]));
 
   const removed = detectRemovedFiles(existing, allFiles, rootDir);
   const journalResult = tryJournalTier(db, existing, rootDir, removed);
@@ -325,21 +331,33 @@ function purgeAndAddReverseDeps(
   reverseDeps: Set<string>,
 ): void {
   const { db, rootDir } = ctx;
-  if (changePaths.length > 0 || ctx.removed.length > 0) {
-    purgeFilesFromGraph(db, [...ctx.removed, ...changePaths], { purgeHashes: false });
-  }
-  if (reverseDeps.size > 0) {
-    const deleteOutgoingEdgesForFile = db.prepare(
-      'DELETE FROM edges WHERE source_id IN (SELECT id FROM nodes WHERE file = ?)',
-    );
-    for (const relPath of reverseDeps) {
-      deleteOutgoingEdgesForFile.run(relPath);
-    }
-    for (const relPath of reverseDeps) {
-      const absPath = path.join(rootDir, relPath);
-      ctx.parseChanges.push({ file: absPath, relPath, _reverseDepOnly: true });
+  const hasPurge = changePaths.length > 0 || ctx.removed.length > 0;
+  const hasReverseDeps = reverseDeps.size > 0;
+  const reverseDepList = hasReverseDeps ? [...reverseDeps] : [];
+
+  if (hasPurge || hasReverseDeps) {
+    const filesToPurge = hasPurge ? [...ctx.removed, ...changePaths] : [];
+    // Prefer NativeDatabase: purge + reverse-dep edge deletion in one transaction (#670)
+    if (ctx.engineName === 'native' && ctx.nativeDb?.purgeFilesData) {
+      ctx.nativeDb.purgeFilesData(filesToPurge, false, hasReverseDeps ? reverseDepList : undefined);
+    } else {
+      if (hasPurge) {
+        purgeFilesFromGraph(db, filesToPurge, { purgeHashes: false });
+      }
+      if (hasReverseDeps) {
+        const deleteOutgoingEdgesForFile = db.prepare(
+          'DELETE FROM edges WHERE source_id IN (SELECT id FROM nodes WHERE file = ?)',
+        );
+        for (const relPath of reverseDepList) {
+          deleteOutgoingEdgesForFile.run(relPath);
+        }
+      }
     }
   }
+  for (const relPath of reverseDeps) {
+    const absPath = path.join(rootDir, relPath);
+    ctx.parseChanges.push({ file: absPath, relPath, _reverseDepOnly: true });
+  }
 }
 
 function detectHasEmbeddings(db: BetterSqlite3Database): boolean {
@@ -415,7 +433,12 @@ export async function detectChanges(ctx: PipelineContext): Promise<void> {
   }
   const increResult =
     incremental && !forceFullRebuild
-      ? getChangedFiles(db, allFiles, rootDir)
+      ? getChangedFiles(
+          db,
+          allFiles,
+          rootDir,
+          ctx.engineName === 'native' ? ctx.nativeDb : undefined,
+        )
       : {
           changed: allFiles.map((f): ChangedFile => ({ file: f })),
           removed: [] as string[],

package/src/domain/graph/builder/stages/finalize.ts

@@ -6,18 +6,19 @@
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { performance } from 'node:perf_hooks';
-import { closeDb, closeDbDeferred, getBuildMeta, setBuildMeta } from '../../../../db/index.js';
+import {
+  closeDbPair,
+  closeDbPairDeferred,
+  getBuildMeta,
+  setBuildMeta,
+} from '../../../../db/index.js';
 import { debug, info, warn } from '../../../../infrastructure/logger.js';
 import { CODEGRAPH_VERSION } from '../../../../shared/version.js';
 import { writeJournalHeader } from '../../journal.js';
 import type { PipelineContext } from '../context.js';
 
-export async function finalize(ctx: PipelineContext): Promise<void> {
-  const { db, allSymbols, rootDir, isFullBuild, hasEmbeddings, config, opts, schemaVersion } = ctx;
-
-  const t0 = performance.now();
-
-  // Release cached WASM trees
+/** Release cached WASM parse trees to free memory. */
+function releaseWasmTrees(allSymbols: PipelineContext['allSymbols']): void {
   for (const [, symbols] of allSymbols) {
     const tree = symbols._tree as { delete?: () => void } | undefined;
     if (tree && typeof tree.delete === 'function') {
@@ -30,113 +31,141 @@ export async function finalize(ctx: PipelineContext): Promise<void> {
     symbols._tree = undefined;
     symbols._langId = undefined;
   }
+}
 
-  // Capture a single wall-clock timestamp for the current build — used for
-  // both the stale-embeddings comparison and the persisted built_at metadata.
-  const buildNow = new Date();
+/**
+ * Detect significant drift between current and previous node/edge counts.
+ * Skipped for small incremental changes where count fluctuation is expected.
+ */
+function detectIncrementalDrift(
+  ctx: PipelineContext,
+  nodeCount: number,
+  actualEdgeCount: number,
+): void {
+  const { db, allSymbols, config } = ctx;
+  const useNativeDb = ctx.engineName === 'native' && !!ctx.nativeDb;
+  if (ctx.isFullBuild || allSymbols.size <= 3) return;
 
-  const nodeCount = (db.prepare('SELECT COUNT(*) as c FROM nodes').get() as { c: number }).c;
-  const actualEdgeCount = (db.prepare('SELECT COUNT(*) as c FROM edges').get() as { c: number }).c;
-  info(`Graph built: ${nodeCount} nodes, ${actualEdgeCount} edges`);
-  info(`Stored in ${ctx.dbPath}`);
+  const prevNodes = useNativeDb
+    ? ctx.nativeDb!.getBuildMeta('node_count')
+    : getBuildMeta(db, 'node_count');
+  const prevEdges = useNativeDb
+    ? ctx.nativeDb!.getBuildMeta('edge_count')
+    : getBuildMeta(db, 'edge_count');
+  if (!prevNodes || !prevEdges) return;
 
-  // Incremental drift detection — skip for small incremental changes where
-  // count fluctuation is expected (reverse-dep edge churn).
-  if (!isFullBuild && allSymbols.size > 3) {
-    const prevNodes = getBuildMeta(db, 'node_count');
-    const prevEdges = getBuildMeta(db, 'edge_count');
-    if (prevNodes && prevEdges) {
-      const prevN = Number(prevNodes);
-      const prevE = Number(prevEdges);
-      if (prevN > 0) {
-        const nodeDrift = Math.abs(nodeCount - prevN) / prevN;
-        const edgeDrift = prevE > 0 ? Math.abs(actualEdgeCount - prevE) / prevE : 0;
-        const driftThreshold =
-          (config as { build?: { driftThreshold?: number } }).build?.driftThreshold ?? 0.2;
-        if (nodeDrift > driftThreshold || edgeDrift > driftThreshold) {
-          warn(
-            `Incremental build diverged significantly from previous counts (nodes: ${prevN}\u2192${nodeCount} [${(nodeDrift * 100).toFixed(1)}%], edges: ${prevE}\u2192${actualEdgeCount} [${(edgeDrift * 100).toFixed(1)}%], threshold: ${(driftThreshold * 100).toFixed(0)}%). Consider rebuilding with --no-incremental.`,
-          );
-        }
-      }
-    }
+  const prevN = Number(prevNodes);
+  const prevE = Number(prevEdges);
+  if (prevN <= 0) return;
+
+  const nodeDrift = Math.abs(nodeCount - prevN) / prevN;
+  const edgeDrift = prevE > 0 ? Math.abs(actualEdgeCount - prevE) / prevE : 0;
+  const driftThreshold =
+    (config as { build?: { driftThreshold?: number } }).build?.driftThreshold ?? 0.2;
+  if (nodeDrift > driftThreshold || edgeDrift > driftThreshold) {
+    warn(
+      `Incremental build diverged significantly from previous counts (nodes: ${prevN}\u2192${nodeCount} [${(nodeDrift * 100).toFixed(1)}%], edges: ${prevE}\u2192${actualEdgeCount} [${(edgeDrift * 100).toFixed(1)}%], threshold: ${(driftThreshold * 100).toFixed(0)}%). Consider rebuilding with --no-incremental.`,
+    );
   }
+}
 
-  // For small incremental builds, skip persisting build metadata — the
-  // engine/version/schema haven't changed (would have triggered a full rebuild),
-  // built_at is only used by stale-embeddings check (skipped for incremental),
-  // and counts are only used by drift detection (skipped for ≤3 files).
-  // This avoids a transaction commit + WAL fsync (~15-30ms).
-  if (isFullBuild || allSymbols.size > 5) {
-    try {
-      setBuildMeta(db, {
+/**
+ * Persist build metadata (engine, version, counts, timestamp).
+ * Skipped for small incremental builds to avoid WAL fsync cost.
+ */
+function persistBuildMetadata(
+  ctx: PipelineContext,
+  nodeCount: number,
+  actualEdgeCount: number,
+  buildNow: Date,
+): void {
+  const useNativeDb = ctx.engineName === 'native' && !!ctx.nativeDb;
+  if (!ctx.isFullBuild && ctx.allSymbols.size <= 3) return;
+  try {
+    if (useNativeDb) {
+      ctx.nativeDb!.setBuildMeta(
+        Object.entries({
+          engine: ctx.engineName,
+          engine_version: ctx.engineVersion || '',
+          codegraph_version: CODEGRAPH_VERSION,
+          schema_version: String(ctx.schemaVersion),
+          built_at: buildNow.toISOString(),
+          node_count: String(nodeCount),
+          edge_count: String(actualEdgeCount),
+        }).map(([key, value]) => ({ key, value: String(value) })),
+      );
+    } else {
+      setBuildMeta(ctx.db, {
         engine: ctx.engineName,
         engine_version: ctx.engineVersion || '',
         codegraph_version: CODEGRAPH_VERSION,
-        schema_version: String(schemaVersion),
+        schema_version: String(ctx.schemaVersion),
         built_at: buildNow.toISOString(),
         node_count: nodeCount,
         edge_count: actualEdgeCount,
       });
-    } catch (err) {
-      warn(`Failed to write build metadata: ${(err as Error).message}`);
     }
+  } catch (err) {
+    warn(`Failed to write build metadata: ${(err as Error).message}`);
   }
+}
 
-  // Skip expensive advisory queries for incremental builds — these are
-  // informational warnings that don't affect correctness and cost ~40-60ms.
-  if (!isFullBuild) {
-    debug(
-      'Finalize: skipping advisory queries (orphaned/stale embeddings, unused exports) for incremental build',
-    );
-  } else {
-    // Orphaned embeddings warning
-    if (hasEmbeddings) {
-      try {
-        const orphaned = (
-          db
-            .prepare(
-              'SELECT COUNT(*) as c FROM embeddings WHERE node_id NOT IN (SELECT id FROM nodes)',
-            )
-            .get() as { c: number }
-        ).c;
-        if (orphaned > 0) {
-          warn(
-            `${orphaned} embeddings are orphaned (nodes changed). Run "codegraph embed" to refresh.`,
-          );
-        }
-      } catch {
-        /* ignore - embeddings table may have been dropped */
+/**
+ * Run advisory checks on full builds: orphaned embeddings, stale embeddings,
+ * and unused exports. Informational only — does not affect correctness.
+ */
+function runAdvisoryChecks(
+  db: PipelineContext['db'],
+  hasEmbeddings: boolean,
+  buildNow: Date,
+): void {
+  // Orphaned embeddings warning
+  if (hasEmbeddings) {
+    try {
+      const orphaned = (
+        db
+          .prepare(
+            'SELECT COUNT(*) as c FROM embeddings WHERE node_id NOT IN (SELECT id FROM nodes)',
+          )
+          .get() as { c: number }
+      ).c;
+      if (orphaned > 0) {
+        warn(
+          `${orphaned} embeddings are orphaned (nodes changed). Run "codegraph embed" to refresh.`,
+        );
       }
+    } catch {
+      /* ignore - embeddings table may have been dropped */
     }
+  }
 
-    // Stale embeddings warning (built before current graph rebuild)
-    if (hasEmbeddings) {
-      try {
-        const embedBuiltAt = (
-          db.prepare("SELECT value FROM embedding_meta WHERE key = 'built_at'").get() as
-            | { value: string }
-            | undefined
-        )?.value;
-        if (embedBuiltAt) {
-          const embedTime = new Date(embedBuiltAt).getTime();
-          if (!Number.isNaN(embedTime) && embedTime < buildNow.getTime()) {
-            warn(
-              'Embeddings were built before the last graph rebuild. Run "codegraph embed" to update.',
-            );
-          }
+  // Stale embeddings warning (built before current graph rebuild)
+  if (hasEmbeddings) {
+    try {
+      const embedBuiltAt = (
+        db.prepare("SELECT value FROM embedding_meta WHERE key = 'built_at'").get() as
+          | { value: string }
+          | undefined
+      )?.value;
+      if (embedBuiltAt) {
+        const embedTime = new Date(embedBuiltAt).getTime();
+        if (!Number.isNaN(embedTime) && embedTime < buildNow.getTime()) {
+          warn(
+            'Embeddings were built before the last graph rebuild. Run "codegraph embed" to update.',
+          );
         }
-      } catch {
-        /* ignore - embedding_meta table may not exist */
       }
+    } catch {
+      /* ignore - embedding_meta table may not exist */
     }
+  }
 
-    // Unused exports warning
-    try {
-      const unusedCount = (
-        db
-          .prepare(
-            `SELECT COUNT(*) as c FROM nodes
+  // Unused exports warning
+  try {
+    const unusedCount = (
+      db
+        .prepare(
+          `SELECT COUNT(*) as c FROM nodes
          WHERE exported = 1 AND kind != 'file'
            AND id NOT IN (
              SELECT DISTINCT e.target_id FROM edges e
@@ -144,30 +173,65 @@ export async function finalize(ctx: PipelineContext): Promise<void> {
              JOIN nodes target ON e.target_id = target.id
              WHERE e.kind = 'calls' AND caller.file != target.file
            )`,
-          )
-          .get() as { c: number }
-      ).c;
-      if (unusedCount > 0) {
-        warn(
-          `${unusedCount} exported symbol${unusedCount > 1 ? 's have' : ' has'} zero cross-file consumers. Run "codegraph exports <file> --unused" to inspect.`,
-        );
-      }
-    } catch {
-      /* exported column may not exist on older DBs */
+        )
+        .get() as { c: number }
+    ).c;
+    if (unusedCount > 0) {
+      warn(
+        `${unusedCount} exported symbol${unusedCount > 1 ? 's have' : ' has'} zero cross-file consumers. Run "codegraph exports <file> --unused" to inspect.`,
+      );
     }
+  } catch {
+    /* exported column may not exist on older DBs */
+  }
+}
+
+export async function finalize(ctx: PipelineContext): Promise<void> {
+  const { allSymbols, rootDir, isFullBuild, hasEmbeddings, opts } = ctx;
+
+  const t0 = performance.now();
+
+  releaseWasmTrees(allSymbols);
+
+  // Capture a single wall-clock timestamp for the current build — used for
+  // both the stale-embeddings comparison and the persisted built_at metadata.
+  const buildNow = new Date();
+
+  const nodeCount = (ctx.db.prepare('SELECT COUNT(*) as c FROM nodes').get() as { c: number }).c;
+  const actualEdgeCount = (ctx.db.prepare('SELECT COUNT(*) as c FROM edges').get() as { c: number })
+    .c;
+  info(`Graph built: ${nodeCount} nodes, ${actualEdgeCount} edges`);
+  info(`Stored in ${ctx.dbPath}`);
+
+  detectIncrementalDrift(ctx, nodeCount, actualEdgeCount);
+  persistBuildMetadata(ctx, nodeCount, actualEdgeCount, buildNow);
+
+  // Skip expensive advisory queries for incremental builds — these are
+  // informational warnings that don't affect correctness and cost ~40-60ms.
+  if (!isFullBuild) {
+    debug(
+      'Finalize: skipping advisory queries (orphaned/stale embeddings, unused exports) for incremental build',
+    );
+  } else {
+    runAdvisoryChecks(ctx.db, hasEmbeddings, buildNow);
   }
 
+  // Intentionally measured before closeDb / writeJournalHeader / auto-registration:
+  // for the deferred-close path the close is async (setImmediate), and for full
+  // builds the metric captures finalize logic only — DB close cost is tracked
+  // separately via timing.closeDbMs when available.
   ctx.timing.finalizeMs = performance.now() - t0;
 
-  // For small incremental builds, defer db.close() to the next event loop tick.
-  // The WAL checkpoint in db.close() costs ~250ms on Windows NTFS due to fsync.
-  // Deferring lets buildGraph() return immediately; the checkpoint runs after.
-  // Skip for temp directories (tests) — they rmSync immediately after build.
+  // Close NativeDatabase (fast, ~1ms) then better-sqlite3 (WAL checkpoint).
+  // For small incremental builds, defer the expensive WAL checkpoint to the
+  // next event loop tick. Skip for temp directories (tests) — they rmSync
+  // immediately after build.
+  const pair = { db: ctx.db, nativeDb: ctx.nativeDb };
   const isTempDir = path.resolve(rootDir).startsWith(path.resolve(tmpdir()));
   if (!isFullBuild && allSymbols.size <= 5 && !isTempDir) {
-    closeDbDeferred(db);
+    closeDbPairDeferred(pair);
   } else {
-    closeDb(db);
+    closeDbPair(pair);
   }
 
   // Write journal header after successful build
@@ -177,7 +241,6 @@ export async function finalize(ctx: PipelineContext): Promise<void> {
   // registered during the initial full build. The dynamic import + file I/O
   // costs ~100ms which dominates incremental finalize time.
   if (!opts.skipRegistry && isFullBuild) {
-    const { tmpdir } = await import('node:os');
     const tmpDir = path.resolve(tmpdir());
     const resolvedRoot = path.resolve(rootDir);
     if (resolvedRoot.startsWith(tmpDir)) {