@optave/codegraph 3.7.0 → 3.8.1

package/src/domain/graph/builder/stages/collect-files.ts

@@ -21,13 +21,22 @@ function tryFastCollect(
   ctx: PipelineContext,
 ): { files: string[]; directories: Set<string> } | null {
   const { db, rootDir } = ctx;
+  const useNative = ctx.engineName === 'native' && !!ctx.nativeDb?.getCollectFilesData;
 
   // 1. Check that file_hashes table exists and has entries
   let dbFileCount: number;
-  try {
-    dbFileCount = (db.prepare('SELECT COUNT(*) as c FROM file_hashes').get() as { c: number }).c;
-  } catch {
-    return null;
+  let dbFiles: string[];
+  if (useNative) {
+    const data = ctx.nativeDb!.getCollectFilesData!();
+    dbFileCount = data.count;
+    dbFiles = data.files;
+  } else {
+    try {
+      dbFileCount = (db.prepare('SELECT COUNT(*) as c FROM file_hashes').get() as { c: number }).c;
+    } catch {
+      return null;
+    }
+    dbFiles = []; // deferred — loaded below only if needed
   }
   if (dbFileCount === 0) return null;
 
@@ -42,9 +51,11 @@ function tryFastCollect(
   if (!hasEntries) return null;
 
   // 3. Load existing file list from file_hashes (relative paths)
-  const dbFiles = (db.prepare('SELECT file FROM file_hashes').all() as Array<{ file: string }>).map(
-    (r) => r.file,
-  );
+  if (!useNative) {
+    dbFiles = (db.prepare('SELECT file FROM file_hashes').all() as Array<{ file: string }>).map(
+      (r) => r.file,
+    );
+  }
 
   // 4. Apply journal deltas: remove deleted files, add new/changed files
   const fileSet = new Set(dbFiles);

package/src/domain/graph/builder/stages/detect-changes.ts

@@ -60,14 +60,27 @@ function getChangedFiles(
   rootDir: string,
   nativeDb?: NativeDatabase,
 ): ChangeResult {
+  // Batched native path: single napi call for table check + all rows + max mtime
+  if (nativeDb?.getFileHashData) {
+    const data = nativeDb.getFileHashData();
+    if (!data.exists) {
+      return {
+        changed: allFiles.map((f) => ({ file: f })),
+        removed: [],
+        isFullBuild: true,
+      };
+    }
+    const existing = new Map<string, FileHashRow>(data.rows.map((r) => [r.file, r]));
+    const removed = detectRemovedFiles(existing, allFiles, rootDir);
+    const journalResult = tryJournalTier(db, existing, rootDir, removed, data.maxMtime);
+    if (journalResult) return journalResult;
+    return mtimeAndHashTiers(existing, allFiles, rootDir, removed);
+  }
+
+  // WASM / fallback path
   let hasTable = false;
   try {
-    if (nativeDb) {
-      nativeDb.queryGet('SELECT 1 FROM file_hashes LIMIT 1', []);
-    } else {
-      db.prepare('SELECT 1 FROM file_hashes LIMIT 1').get();
-    }
-    // Query succeeded → table exists (result may be undefined if table is empty)
+    db.prepare('SELECT 1 FROM file_hashes LIMIT 1').get();
     hasTable = true;
   } catch {
     /* table doesn't exist */
@@ -81,10 +94,7 @@ function getChangedFiles(
     };
   }
 
-  const sql = 'SELECT file, hash, mtime, size FROM file_hashes';
-  const rows = nativeDb
-    ? (nativeDb.queryAll(sql, []) as unknown as FileHashRow[])
-    : (db.prepare(sql).all() as FileHashRow[]);
+  const rows = db.prepare('SELECT file, hash, mtime, size FROM file_hashes').all() as FileHashRow[];
   const existing = new Map<string, FileHashRow>(rows.map((r) => [r.file, r]));
 
   const removed = detectRemovedFiles(existing, allFiles, rootDir);
@@ -116,14 +126,19 @@ function tryJournalTier(
   existing: Map<string, FileHashRow>,
   rootDir: string,
   removed: string[],
+  precomputedMaxMtime?: number,
 ): ChangeResult | null {
   const journal = readJournal(rootDir);
   if (!journal.valid) return null;
 
-  const dbMtimes = db.prepare('SELECT MAX(mtime) as latest FROM file_hashes').get() as
-    | { latest: number | null }
-    | undefined;
-  const latestDbMtime = dbMtimes?.latest || 0;
+  const latestDbMtime =
+    precomputedMaxMtime ??
+    ((
+      db.prepare('SELECT MAX(mtime) as latest FROM file_hashes').get() as
+        | { latest: number | null }
+        | undefined
+    )?.latest ||
+      0);
   const hasJournalEntries = journal.changed!.length > 0 || journal.removed!.length > 0;
 
   if (!hasJournalEntries || journal.timestamp! < latestDbMtime) {
@@ -231,30 +246,42 @@ function mtimeAndHashTiers(
 
 async function runPendingAnalysis(ctx: PipelineContext): Promise<boolean> {
   const { db, opts, engineOpts, allFiles, rootDir } = ctx;
-  const needsCfg =
-    (opts as Record<string, unknown>).cfg !== false &&
-    (() => {
-      try {
-        return (
-          (db.prepare('SELECT COUNT(*) as c FROM cfg_blocks').get() as { c: number } | undefined)
-            ?.c === 0
-        );
-      } catch {
-        return true;
-      }
-    })();
-  const needsDataflow =
-    (opts as Record<string, unknown>).dataflow !== false &&
-    (() => {
-      try {
-        return (
-          (db.prepare('SELECT COUNT(*) as c FROM dataflow').get() as { c: number } | undefined)
-            ?.c === 0
-        );
-      } catch {
-        return true;
-      }
-    })();
+  const useNative = ctx.engineName === 'native' && !!ctx.nativeDb?.checkPendingAnalysis;
+
+  let needsCfg: boolean;
+  let needsDataflow: boolean;
+
+  if (useNative) {
+    const counts = ctx.nativeDb!.checkPendingAnalysis!();
+    needsCfg = (opts as Record<string, unknown>).cfg !== false && counts.cfgCount <= 0;
+    needsDataflow =
+      (opts as Record<string, unknown>).dataflow !== false && counts.dataflowCount <= 0;
+  } else {
+    needsCfg =
+      (opts as Record<string, unknown>).cfg !== false &&
+      (() => {
+        try {
+          return (
+            (db.prepare('SELECT COUNT(*) as c FROM cfg_blocks').get() as { c: number } | undefined)
+              ?.c === 0
+          );
+        } catch {
+          return true;
+        }
+      })();
+    needsDataflow =
+      (opts as Record<string, unknown>).dataflow !== false &&
+      (() => {
+        try {
+          return (
+            (db.prepare('SELECT COUNT(*) as c FROM dataflow').get() as { c: number } | undefined)
+              ?.c === 0
+          );
+        } catch {
+          return true;
+        }
+      })();
+  }
   if (!needsCfg && !needsDataflow) return false;
 
   info('No file changes. Running pending analysis pass...');
@@ -282,17 +309,27 @@ function healMetadata(ctx: PipelineContext): void {
   const { db, metadataUpdates } = ctx;
   if (!metadataUpdates || metadataUpdates.length === 0) return;
   try {
-    const healHash = db.prepare(
-      'INSERT OR REPLACE INTO file_hashes (file, hash, mtime, size) VALUES (?, ?, ?, ?)',
-    );
-    const healTx = db.transaction(() => {
-      for (const item of metadataUpdates) {
-        const mtime = item.stat ? Math.floor(item.stat.mtime) : 0;
-        const size = item.stat ? item.stat.size : 0;
-        healHash.run(item.relPath, item.hash, mtime, size);
-      }
-    });
-    healTx();
+    if (ctx.engineName === 'native' && ctx.nativeDb?.healFileMetadata) {
+      const entries = metadataUpdates.map((item) => ({
+        file: item.relPath,
+        hash: item.hash,
+        mtime: item.stat ? Math.floor(item.stat.mtime) : 0,
+        size: item.stat ? item.stat.size : 0,
+      }));
+      ctx.nativeDb.healFileMetadata(entries);
+    } else {
+      const healHash = db.prepare(
+        'INSERT OR REPLACE INTO file_hashes (file, hash, mtime, size) VALUES (?, ?, ?, ?)',
+      );
+      const healTx = db.transaction(() => {
+        for (const item of metadataUpdates) {
+          const mtime = item.stat ? Math.floor(item.stat.mtime) : 0;
+          const size = item.stat ? item.stat.size : 0;
+          healHash.run(item.relPath, item.hash, mtime, size);
+        }
+      });
+      healTx();
+    }
     debug(`Self-healed mtime/size for ${metadataUpdates.length} files`);
   } catch {
     /* ignore heal errors */
@@ -303,9 +340,23 @@ function findReverseDependencies(
   db: BetterSqlite3Database,
   changedRelPaths: Set<string>,
   rootDir: string,
+  nativeDb?: NativeDatabase,
 ): Set<string> {
   const reverseDeps = new Set<string>();
   if (changedRelPaths.size === 0) return reverseDeps;
+
+  if (nativeDb?.findReverseDependencies) {
+    const changedArray = [...changedRelPaths];
+    const nativeResults = nativeDb.findReverseDependencies(changedArray);
+    for (const dep of nativeResults) {
+      const absPath = path.isAbsolute(dep) ? dep : path.join(rootDir, dep);
+      if (fs.existsSync(absPath)) {
+        reverseDeps.add(dep);
+      }
+    }
+    return reverseDeps;
+  }
+
   const findReverseDepsStmt = db.prepare(`
     SELECT DISTINCT n_src.file FROM edges e
     JOIN nodes n_src ON e.source_id = n_src.id
@@ -315,7 +366,7 @@ function findReverseDependencies(
   for (const relPath of changedRelPaths) {
     for (const row of findReverseDepsStmt.all(relPath) as Array<{ file: string }>) {
       if (!changedRelPaths.has(row.file) && !reverseDeps.has(row.file)) {
-        const absPath = path.join(rootDir, row.file);
+        const absPath = path.isAbsolute(row.file) ? row.file : path.join(rootDir, row.file);
         if (fs.existsSync(absPath)) {
           reverseDeps.add(row.file);
         }
@@ -360,7 +411,10 @@ function purgeAndAddReverseDeps(
   }
 }
 
-function detectHasEmbeddings(db: BetterSqlite3Database): boolean {
+function detectHasEmbeddings(db: BetterSqlite3Database, nativeDb?: NativeDatabase): boolean {
+  if (nativeDb?.hasEmbeddings) {
+    return nativeDb.hasEmbeddings();
+  }
   try {
     db.prepare('SELECT 1 FROM embeddings LIMIT 1').get();
     return true;
@@ -371,14 +425,14 @@ function detectHasEmbeddings(db: BetterSqlite3Database): boolean {
 
 function handleScopedBuild(ctx: PipelineContext): void {
   const { db, rootDir, opts } = ctx;
-  ctx.hasEmbeddings = detectHasEmbeddings(db);
+  ctx.hasEmbeddings = detectHasEmbeddings(db, ctx.nativeDb);
   const changePaths = ctx.parseChanges.map(
     (item) => item.relPath || normalizePath(path.relative(rootDir, item.file)),
   );
   let reverseDeps = new Set<string>();
   if (!(opts as Record<string, unknown>).noReverseDeps) {
     const changedRelPaths = new Set<string>([...changePaths, ...ctx.removed]);
-    reverseDeps = findReverseDependencies(db, changedRelPaths, rootDir);
+    reverseDeps = findReverseDependencies(db, changedRelPaths, rootDir, ctx.nativeDb);
   }
   purgeAndAddReverseDeps(ctx, changePaths, reverseDeps);
   info(
@@ -388,7 +442,7 @@ function handleScopedBuild(ctx: PipelineContext): void {
 
 function handleFullBuild(ctx: PipelineContext): void {
   const { db } = ctx;
-  const hasEmbeddings = detectHasEmbeddings(db);
+  const hasEmbeddings = detectHasEmbeddings(db, ctx.nativeDb);
   ctx.hasEmbeddings = hasEmbeddings;
   const deletions =
     'PRAGMA foreign_keys = OFF; DELETE FROM cfg_edges; DELETE FROM cfg_blocks; DELETE FROM node_metrics; DELETE FROM edges; DELETE FROM function_complexity; DELETE FROM dataflow; DELETE FROM ast_nodes; DELETE FROM nodes; PRAGMA foreign_keys = ON;';
@@ -401,7 +455,7 @@ function handleFullBuild(ctx: PipelineContext): void {
 
 function handleIncrementalBuild(ctx: PipelineContext): void {
   const { db, rootDir, opts } = ctx;
-  ctx.hasEmbeddings = detectHasEmbeddings(db);
+  ctx.hasEmbeddings = detectHasEmbeddings(db, ctx.nativeDb);
   let reverseDeps = new Set<string>();
   if (!(opts as Record<string, unknown>).noReverseDeps) {
     const changedRelPaths = new Set<string>();
@@ -411,7 +465,7 @@ function handleIncrementalBuild(ctx: PipelineContext): void {
     for (const relPath of ctx.removed) {
       changedRelPaths.add(relPath);
     }
-    reverseDeps = findReverseDependencies(db, changedRelPaths, rootDir);
+    reverseDeps = findReverseDependencies(db, changedRelPaths, rootDir, ctx.nativeDb);
   }
   info(
     `Incremental: ${ctx.parseChanges.length} changed, ${ctx.removed.length} removed${reverseDeps.size > 0 ? `, ${reverseDeps.size} reverse-deps` : ''}`,

package/src/domain/graph/builder/stages/finalize.ts

@@ -86,7 +86,7 @@ function persistBuildMetadata(
       ctx.nativeDb!.setBuildMeta(
         Object.entries({
           engine: ctx.engineName,
-          engine_version: ctx.engineVersion || '',
+          engine_version: CODEGRAPH_VERSION,
           codegraph_version: CODEGRAPH_VERSION,
           schema_version: String(ctx.schemaVersion),
           built_at: buildNow.toISOString(),
@@ -97,7 +97,7 @@ function persistBuildMetadata(
     } else {
       setBuildMeta(ctx.db, {
         engine: ctx.engineName,
-        engine_version: ctx.engineVersion || '',
+        engine_version: CODEGRAPH_VERSION,
         codegraph_version: CODEGRAPH_VERSION,
         schema_version: String(ctx.schemaVersion),
         built_at: buildNow.toISOString(),
@@ -114,11 +114,33 @@ function persistBuildMetadata(
  * Run advisory checks on full builds: orphaned embeddings, stale embeddings,
  * and unused exports. Informational only — does not affect correctness.
  */
-function runAdvisoryChecks(
-  db: PipelineContext['db'],
-  hasEmbeddings: boolean,
-  buildNow: Date,
-): void {
+function runAdvisoryChecks(ctx: PipelineContext, hasEmbeddings: boolean, buildNow: Date): void {
+  // Batched native path: single napi call for all 3 advisory checks
+  if (ctx.engineName === 'native' && ctx.nativeDb?.runAdvisoryChecks) {
+    const result = ctx.nativeDb.runAdvisoryChecks(hasEmbeddings);
+    if (result.orphanedEmbeddings > 0) {
+      warn(
+        `${result.orphanedEmbeddings} embeddings are orphaned (nodes changed). Run "codegraph embed" to refresh.`,
+      );
+    }
+    if (result.embedBuiltAt) {
+      const embedTime = new Date(result.embedBuiltAt).getTime();
+      if (!Number.isNaN(embedTime) && embedTime < buildNow.getTime()) {
+        warn(
+          'Embeddings were built before the last graph rebuild. Run "codegraph embed" to update.',
+        );
+      }
+    }
+    if (result.unusedExports > 0) {
+      warn(
+        `${result.unusedExports} exported symbol${result.unusedExports > 1 ? 's have' : ' has'} zero cross-file consumers. Run "codegraph exports <file> --unused" to inspect.`,
+      );
+    }
+    return;
+  }
+
+  const { db } = ctx;
+
   // Orphaned embeddings warning
   if (hasEmbeddings) {
     try {
@@ -197,9 +219,17 @@ export async function finalize(ctx: PipelineContext): Promise<void> {
   // both the stale-embeddings comparison and the persisted built_at metadata.
   const buildNow = new Date();
 
-  const nodeCount = (ctx.db.prepare('SELECT COUNT(*) as c FROM nodes').get() as { c: number }).c;
-  const actualEdgeCount = (ctx.db.prepare('SELECT COUNT(*) as c FROM edges').get() as { c: number })
-    .c;
+  const useNative = ctx.engineName === 'native' && !!ctx.nativeDb?.getFinalizeCounts;
+  let nodeCount: number;
+  let actualEdgeCount: number;
+  if (useNative) {
+    const counts = ctx.nativeDb!.getFinalizeCounts!();
+    nodeCount = counts.nodeCount;
+    actualEdgeCount = counts.edgeCount;
+  } else {
+    nodeCount = (ctx.db.prepare('SELECT COUNT(*) as c FROM nodes').get() as { c: number }).c;
+    actualEdgeCount = (ctx.db.prepare('SELECT COUNT(*) as c FROM edges').get() as { c: number }).c;
+  }
   info(`Graph built: ${nodeCount} nodes, ${actualEdgeCount} edges`);
   info(`Stored in ${ctx.dbPath}`);
 
@@ -213,7 +243,7 @@ export async function finalize(ctx: PipelineContext): Promise<void> {
       'Finalize: skipping advisory queries (orphaned/stale embeddings, unused exports) for incremental build',
     );
   } else {
-    runAdvisoryChecks(ctx.db, hasEmbeddings, buildNow);
+    runAdvisoryChecks(ctx, hasEmbeddings, buildNow);
   }
 
   // Intentionally measured before closeDb / writeJournalHeader / auto-registration:

package/src/domain/graph/builder/stages/insert-nodes.ts

@@ -11,6 +11,8 @@
 import path from 'node:path';
 import { performance } from 'node:perf_hooks';
 import { bulkNodeIdsByFile } from '../../../../db/index.js';
+import { debug } from '../../../../infrastructure/logger.js';
+import { toErrorMessage } from '../../../../shared/errors.js';
 import type {
   BetterSqlite3Database,
   ExtractorOutput,
@@ -36,41 +38,31 @@ interface PrecomputedFileData {
   _reverseDepOnly?: boolean;
 }
 
-// ── Native fast-path ─────────────────────────────────────────────────
-
-function tryNativeInsert(ctx: PipelineContext): boolean {
-  // Disabled: bulkInsertNodes corrupts the DB when both the JS (better-sqlite3)
-  // and Rust (rusqlite) connections are open to the same WAL-mode file.
-  // The native path was never operational before — it always crashed on null
-  // visibility serialisation. See #696 for the dual-connection fix.
-  if (ctx.db) return false;
-
-  // Use NativeDatabase persistent connection (Phase 6.15+).
-  // Standalone napi functions were removed in 6.17 — falls through to JS if nativeDb unavailable.
-  if (!ctx.nativeDb?.bulkInsertNodes) return false;
-
-  const { allSymbols, filesToParse, metadataUpdates, rootDir, removed } = ctx;
+// ── Native fast-path helpers ─────────────────────────────────────────
 
-  // Marshal allSymbols → InsertNodesBatch[]
-  const batches: Array<{
-    file: string;
-    definitions: Array<{
+/** Shape of a marshaled batch for native bulk insert. */
+interface InsertNodesBatch {
+  file: string;
+  definitions: Array<{
+    name: string;
+    kind: string;
+    line: number;
+    endLine?: number;
+    visibility?: string;
+    children: Array<{
       name: string;
       kind: string;
       line: number;
       endLine?: number;
       visibility?: string;
-      children: Array<{
-        name: string;
-        kind: string;
-        line: number;
-        endLine?: number;
-        visibility?: string;
-      }>;
     }>;
-    exports: Array<{ name: string; kind: string; line: number }>;
-  }> = [];
+  }>;
+  exports: Array<{ name: string; kind: string; line: number }>;
+}
 
+/** Marshal allSymbols into the batch format expected by native bulkInsertNodes. */
+function marshalSymbolBatches(allSymbols: Map<string, ExtractorOutput>): InsertNodesBatch[] {
+  const batches: InsertNodesBatch[] = [];
   for (const [relPath, symbols] of allSymbols) {
     batches.push({
       file: relPath,
@@ -95,14 +87,18 @@ function tryNativeInsert(ctx: PipelineContext): boolean {
       })),
     });
   }
+  return batches;
+}
 
-  // Build file hash entries
-  const precomputedData = new Map<string, PrecomputedFileData>();
-  for (const item of filesToParse) {
-    if (item.relPath) precomputedData.set(item.relPath, item as PrecomputedFileData);
-  }
-
+/** Build file hash entries from parsed symbols and precomputed/metadata sources. */
+function buildFileHashes(
+  allSymbols: Map<string, ExtractorOutput>,
+  precomputedData: Map<string, PrecomputedFileData>,
+  metadataUpdates: MetadataUpdate[],
+  rootDir: string,
+): Array<{ file: string; hash: string; mtime: number; size: number }> {
   const fileHashes: Array<{ file: string; hash: string; mtime: number; size: number }> = [];
+
   for (const [relPath] of allSymbols) {
     const precomputed = precomputedData.get(relPath);
     if (precomputed?._reverseDepOnly) {
@@ -125,7 +121,8 @@ function tryNativeInsert(ctx: PipelineContext): boolean {
       let code: string | null;
       try {
         code = readFileSafe(absPath);
-      } catch {
+      } catch (e) {
+        debug(`buildFileHashes: readFileSafe failed for ${relPath}: ${toErrorMessage(e)}`);
         code = null;
       }
       if (code !== null) {
@@ -144,7 +141,44 @@ function tryNativeInsert(ctx: PipelineContext): boolean {
     fileHashes.push({ file: item.relPath, hash: item.hash, mtime, size });
   }
 
-  return ctx.nativeDb!.bulkInsertNodes(batches, fileHashes, removed);
+  return fileHashes;
+}
+
+// ── Native fast-path ─────────────────────────────────────────────────
+
+function tryNativeInsert(ctx: PipelineContext): boolean {
+  if (!ctx.nativeDb?.bulkInsertNodes) return false;
+
+  const { allSymbols, filesToParse, metadataUpdates, rootDir, removed } = ctx;
+
+  const batches = marshalSymbolBatches(allSymbols);
+
+  const precomputedData = new Map<string, PrecomputedFileData>();
+  for (const item of filesToParse) {
+    if (item.relPath) precomputedData.set(item.relPath, item as PrecomputedFileData);
+  }
+  const fileHashes = buildFileHashes(allSymbols, precomputedData, metadataUpdates, rootDir);
+
+  // WAL guard: same suspendJsDb/resumeJsDb pattern used by feature modules
+  // (ast, cfg, complexity, dataflow). Checkpoint JS side before native write,
+  // then checkpoint native side after, so neither library reads WAL frames
+  // written by the other (#696, #709, #715, #717).
+  let result: boolean;
+  try {
+    if (ctx.db) {
+      ctx.db.pragma('wal_checkpoint(TRUNCATE)');
+    }
+    result = ctx.nativeDb!.bulkInsertNodes(batches, fileHashes, removed);
+  } finally {
+    try {
+      ctx.nativeDb?.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+    } catch (e) {
+      debug(
+        `tryNativeInsert: WAL checkpoint failed (nativeDb may already be closed): ${toErrorMessage(e)}`,
+      );
+    }
+  }
+  return result;
 }
 
 // ── JS fallback: Phase 1 ────────────────────────────────────────────
@@ -313,7 +347,8 @@ function updateFileHashes(
       let code: string | null;
       try {
         code = readFileSafe(absPath);
-      } catch {
+      } catch (e) {
+        debug(`updateFileHashes: readFileSafe failed for ${relPath}: ${toErrorMessage(e)}`);
         code = null;
       }
       if (code !== null) {
@@ -353,8 +388,8 @@ export async function insertNodes(ctx: PipelineContext): Promise<void> {
         // Removed-file hash cleanup is handled inside the native call
         return;
       }
-    } catch {
-      // Native insert failed — fall through to JS implementation
+    } catch (e) {
+      debug(`insertNodes: native insert failed, falling back to JS: ${toErrorMessage(e)}`);
     }
   }
 
@@ -369,7 +404,8 @@ export async function insertNodes(ctx: PipelineContext): Promise<void> {
     upsertHash = ctx.db.prepare(
       'INSERT OR REPLACE INTO file_hashes (file, hash, mtime, size) VALUES (?, ?, ?, ?)',
     );
-  } catch {
+  } catch (e) {
+    debug(`insertNodes: file_hashes prepare failed (table may not exist): ${toErrorMessage(e)}`);
     upsertHash = null;
   }