@optave/codegraph 3.9.3 → 3.9.5

package/src/domain/graph/journal.ts

@@ -1,9 +1,224 @@
+import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
 import { debug, warn } from '../../infrastructure/logger.js';
 
 export const JOURNAL_FILENAME = 'changes.journal';
 const HEADER_PREFIX = '# codegraph-journal v1 ';
+const LOCK_SUFFIX = '.lock';
+const LOCK_TIMEOUT_MS = 5_000;
+const LOCK_STALE_MS = 30_000;
+const LOCK_RETRY_MS = 25;
+
+// Busy-spin sleep avoids blocking the Node.js event loop (unlike Atomics.wait,
+// which freezes all I/O and timer callbacks). The retry interval is short
+// (25ms), so the CPU cost is negligible while keeping unrelated callbacks
+// responsive in watcher processes.
+function sleepSync(ms: number): void {
+  const end = process.hrtime.bigint() + BigInt(ms) * 1_000_000n;
+  while (process.hrtime.bigint() < end) {
+    /* spin */
+  }
+}
+
+function isPidAlive(pid: number): boolean {
+  if (!Number.isFinite(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    // EPERM means the process exists but we lack permission — still alive.
+    return (e as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+interface AcquiredLock {
+  fd: number;
+  nonce: string;
+}
+
+/**
+ * Steal a stale lockfile atomically via write-tmp + rename.
+ *
+ * Using rename (which is atomic on POSIX and Windows) avoids the TOCTOU race
+ * inherent to the unlink + openSync('wx') pattern: if two stealers both
+ * observed the same stale holder, one's unlink could cross the other's fresh
+ * acquisition, admitting two writers into the critical section.
+ *
+ * After rename, we re-read the lockfile to confirm our nonce — if another
+ * stealer's rename landed after ours, they own the lock and we retry.
+ */
+function trySteal(lockPath: string): AcquiredLock | null {
+  const nonce = `${process.pid}-${crypto.randomBytes(8).toString('hex')}`;
+  const tmpPath = `${lockPath}.${nonce}.tmp`;
+  try {
+    fs.writeFileSync(tmpPath, `${process.pid}\n${nonce}\n`, { flag: 'w' });
+  } catch {
+    return null;
+  }
+
+  try {
+    // Atomic replace: overwrites the stale lockfile.
+    fs.renameSync(tmpPath, lockPath);
+  } catch {
+    try {
+      fs.unlinkSync(tmpPath);
+    } catch {
+      /* ignore */
+    }
+    return null;
+  }
+
+  // Verify the nonce — another stealer's rename may have landed after ours.
+  let content: string;
+  try {
+    content = fs.readFileSync(lockPath, 'utf-8');
+  } catch {
+    return null;
+  }
+  if (!content.includes(nonce)) {
+    // Lost the race to another stealer; do NOT unlink their live lockfile.
+    return null;
+  }
+
+  let fd: number;
+  try {
+    // Re-open r+ so we have a persistent fd the caller can close on release.
+    fd = fs.openSync(lockPath, 'r+');
+  } catch {
+    return null;
+  }
+  return { fd, nonce };
+}
+
+function acquireJournalLock(lockPath: string): AcquiredLock {
+  const start = Date.now();
+  for (;;) {
+    const nonce = `${process.pid}-${crypto.randomBytes(8).toString('hex')}`;
+    try {
+      const fd = fs.openSync(lockPath, 'wx');
+      try {
+        fs.writeSync(fd, `${process.pid}\n${nonce}\n`);
+      } catch {
+        // Stamp write failed (ENOSPC, I/O error). An empty lockfile would
+        // look stale to concurrent waiters (Number('') === 0, isPidAlive(0)
+        // returns false), so they'd steal our live lock. Release and retry.
+        try {
+          fs.closeSync(fd);
+        } catch {
+          /* ignore */
+        }
+        try {
+          fs.unlinkSync(lockPath);
+        } catch {
+          /* ignore */
+        }
+        if (Date.now() - start > LOCK_TIMEOUT_MS) {
+          throw new Error(
+            `Failed to acquire journal lock at ${lockPath} within ${LOCK_TIMEOUT_MS}ms`,
+          );
+        }
+        sleepSync(LOCK_RETRY_MS);
+        continue;
+      }
+      return { fd, nonce };
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e;
+    }
+
+    let holderAlive = true;
+    try {
+      const pidContent = fs.readFileSync(lockPath, 'utf-8').split('\n')[0]!.trim();
+      holderAlive = isPidAlive(Number(pidContent));
+    } catch {
+      /* unreadable — fall through to age check */
+    }
+
+    let shouldSteal = !holderAlive;
+    if (holderAlive) {
+      try {
+        const stat = fs.statSync(lockPath);
+        if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
+          shouldSteal = true;
+        }
+      } catch {
+        /* stat failed — keep retrying */
+      }
+    }
+
+    if (shouldSteal) {
+      const stolen = trySteal(lockPath);
+      if (stolen) return stolen;
+      // Steal failed or lost the race — fall through to timeout check & retry.
+    }
+
+    if (Date.now() - start > LOCK_TIMEOUT_MS) {
+      throw new Error(`Failed to acquire journal lock at ${lockPath} within ${LOCK_TIMEOUT_MS}ms`);
+    }
+    sleepSync(LOCK_RETRY_MS);
+  }
+}
+
+function releaseJournalLock(lockPath: string, lock: AcquiredLock): void {
+  try {
+    fs.closeSync(lock.fd);
+  } catch {
+    /* ignore */
+  }
+  // Only unlink if the lockfile still carries our nonce — if another stealer
+  // decided we were stale and replaced it, we must not unlink their live lock.
+  try {
+    const content = fs.readFileSync(lockPath, 'utf-8');
+    if (content.includes(lock.nonce)) {
+      fs.unlinkSync(lockPath);
+    }
+  } catch {
+    /* lockfile gone or unreadable — nothing to unlink */
+  }
+}
+
+function sweepStaleTmpFiles(dir: string): void {
+  // Clean up orphaned .tmp files left behind when a process is killed after
+  // writeFileSync(tmpPath, ...) succeeds but before renameSync(tmpPath, lockPath)
+  // completes (trySteal path). Without this, tmp files accumulate silently in
+  // .codegraph/ across crash cycles. Only sweep ones older than LOCK_STALE_MS
+  // so we don't race an in-flight steal on another process.
+  let entries: fs.Dirent[];
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  const now = Date.now();
+  const prefix = `${JOURNAL_FILENAME}${LOCK_SUFFIX}.`;
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.startsWith(prefix) || !entry.name.endsWith('.tmp')) {
+      continue;
+    }
+    const tmpPath = path.join(dir, entry.name);
+    try {
+      const stat = fs.statSync(tmpPath);
+      if (now - stat.mtimeMs > LOCK_STALE_MS) {
+        fs.unlinkSync(tmpPath);
+      }
+    } catch {
+      /* stat/unlink raced another cleaner or was already removed — ignore */
+    }
+  }
+}
+
+function withJournalLock<T>(rootDir: string, fn: () => T): T {
+  const dir = path.join(rootDir, '.codegraph');
+  fs.mkdirSync(dir, { recursive: true });
+  sweepStaleTmpFiles(dir);
+  const lockPath = path.join(dir, `${JOURNAL_FILENAME}${LOCK_SUFFIX}`);
+  const lock = acquireJournalLock(lockPath);
+  try {
+    return fn();
+  } finally {
+    releaseJournalLock(lockPath, lock);
+  }
+}
 
 interface JournalResult {
   valid: boolean;
@@ -63,43 +278,85 @@ export function appendJournalEntries(
   rootDir: string,
   entries: Array<{ file: string; deleted?: boolean }>,
 ): void {
-  const dir = path.join(rootDir, '.codegraph');
-  const journalPath = path.join(dir, JOURNAL_FILENAME);
+  withJournalLock(rootDir, () => {
+    const journalPath = path.join(rootDir, '.codegraph', JOURNAL_FILENAME);
 
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true });
-  }
+    if (!fs.existsSync(journalPath)) {
+      fs.writeFileSync(journalPath, `${HEADER_PREFIX}0\n`);
+    }
 
-  if (!fs.existsSync(journalPath)) {
-    fs.writeFileSync(journalPath, `${HEADER_PREFIX}0\n`);
-  }
+    const lines = entries.map((e) => {
+      if (e.deleted) return `DELETED ${e.file}`;
+      return e.file;
+    });
 
-  const lines = entries.map((e) => {
-    if (e.deleted) return `DELETED ${e.file}`;
-    return e.file;
+    fs.appendFileSync(journalPath, `${lines.join('\n')}\n`);
   });
-
-  fs.appendFileSync(journalPath, `${lines.join('\n')}\n`);
 }
 
 export function writeJournalHeader(rootDir: string, timestamp: number): void {
-  const dir = path.join(rootDir, '.codegraph');
-  const journalPath = path.join(dir, JOURNAL_FILENAME);
-  const tmpPath = `${journalPath}.tmp`;
+  withJournalLock(rootDir, () => {
+    const journalPath = path.join(rootDir, '.codegraph', JOURNAL_FILENAME);
+    const tmpPath = `${journalPath}.tmp`;
 
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true });
-  }
+    try {
+      fs.writeFileSync(tmpPath, `${HEADER_PREFIX}${timestamp}\n`);
+      fs.renameSync(tmpPath, journalPath);
+    } catch (err) {
+      warn(`Failed to write journal header: ${(err as Error).message}`);
+      try {
+        fs.unlinkSync(tmpPath);
+      } catch {
+        /* ignore */
+      }
+    }
+  });
+}
 
-  try {
-    fs.writeFileSync(tmpPath, `${HEADER_PREFIX}${timestamp}\n`);
-    fs.renameSync(tmpPath, journalPath);
-  } catch (err) {
-    warn(`Failed to write journal header: ${(err as Error).message}`);
+/**
+ * Atomically append entries while advancing the header timestamp.
+ *
+ * Used by the watcher: without this, the header timestamp stays frozen at the
+ * last build's finalize time while entries accumulate, so the next build's
+ * Tier 0 check sees `journal.timestamp < MAX(file_hashes.mtime)`, rejects the
+ * journal, and falls through to the expensive mtime+size / hash scan.
+ *
+ * Writes a tmp file then renames — a crash mid-rename leaves the previous
+ * journal state intact.
+ */
+export function appendJournalEntriesAndStampHeader(
+  rootDir: string,
+  entries: Array<{ file: string; deleted?: boolean }>,
+  timestamp: number,
+): void {
+  withJournalLock(rootDir, () => {
+    const journalPath = path.join(rootDir, '.codegraph', JOURNAL_FILENAME);
+    const tmpPath = `${journalPath}.tmp`;
+
+    let existingBody = '';
     try {
-      fs.unlinkSync(tmpPath);
+      const content = fs.readFileSync(journalPath, 'utf-8');
+      const newlineIdx = content.indexOf('\n');
+      if (newlineIdx >= 0) existingBody = content.slice(newlineIdx + 1);
     } catch {
-      /* ignore */
+      /* no existing journal — fall through to write header + new entries */
     }
-  }
+    if (existingBody && !existingBody.endsWith('\n')) existingBody = `${existingBody}\n`;
+
+    const newLines = entries.map((e) => (e.deleted ? `DELETED ${e.file}` : e.file));
+    const appended = newLines.length > 0 ? `${newLines.join('\n')}\n` : '';
+    const content = `${HEADER_PREFIX}${timestamp}\n${existingBody}${appended}`;
+
+    try {
+      fs.writeFileSync(tmpPath, content);
+      fs.renameSync(tmpPath, journalPath);
+    } catch (err) {
+      warn(`Failed to update journal: ${(err as Error).message}`);
+      try {
+        fs.unlinkSync(tmpPath);
+      } catch {
+        /* ignore */
+      }
+    }
+  });
 }

package/src/domain/graph/watcher.ts

@@ -7,7 +7,7 @@ import { DbError } from '../../shared/errors.js';
 import { createParseTreeCache, getActiveEngine } from '../parser.js';
 import { type IncrementalStmts, rebuildFile } from './builder/incremental.js';
 import { appendChangeEvents, buildChangeEvent, diffSymbols } from './change-journal.js';
-import { appendJournalEntries } from './journal.js';
+import { appendJournalEntriesAndStampHeader } from './journal.js';
 
 function shouldIgnorePath(filePath: string): boolean {
   const parts = filePath.split(path.sep);
@@ -100,7 +100,7 @@ function writeJournalAndChangeEvents(rootDir: string, updates: RebuildResult[]):
     deleted: r.deleted || false,
   }));
   try {
-    appendJournalEntries(rootDir, entries);
+    appendJournalEntriesAndStampHeader(rootDir, entries, Date.now());
   } catch (e: unknown) {
     debug(`Journal write failed (non-fatal): ${(e as Error).message}`);
   }
@@ -165,8 +165,8 @@ interface WatcherContext {
 }
 
 /** Initialize DB, engine, cache, and statements for watch mode. */
-function setupWatcher(rootDir: string, opts: { engine?: string }): WatcherContext {
-  const dbPath = path.join(rootDir, '.codegraph', 'graph.db');
+function setupWatcher(rootDir: string, opts: { engine?: string; dbPath?: string }): WatcherContext {
+  const dbPath = opts.dbPath ?? path.join(rootDir, '.codegraph', 'graph.db');
   if (!fs.existsSync(dbPath)) {
     throw new DbError('No graph.db found. Run `codegraph build` first.', { file: dbPath });
   }
@@ -274,17 +274,37 @@ function startNativeWatcher(ctx: WatcherContext): () => void {
   return () => watcher.close();
 }
 
+/**
+ * Build journal entries for a pending-path set, detecting deletions by
+ * existence check.
+ *
+ * `ctx.pending` is an untyped `Set<string>` — it carries no event-type
+ * metadata. Without this check, a file deleted during the watch session
+ * would be journaled as "changed", causing the next incremental build to
+ * try to re-parse a non-existent file instead of removing it from the graph.
+ * Mirrors the deletion detection in `rebuildFile` (see builder/incremental.ts).
+ *
+ * Exported for unit-testing; prefer `setupShutdownHandler` in production paths.
+ */
+export function buildFlushEntriesFromPending(
+  rootDir: string,
+  pending: Iterable<string>,
+): Array<{ file: string; deleted: boolean }> {
+  return [...pending].map((filePath) => ({
+    file: normalizePath(path.relative(rootDir, filePath)),
+    deleted: !fs.existsSync(filePath),
+  }));
+}
+
 /** Register SIGINT handler to flush journal and clean up. */
 function setupShutdownHandler(ctx: WatcherContext, cleanup: () => void): void {
   process.once('SIGINT', () => {
     info('Stopping watcher...');
     cleanup();
     if (ctx.pending.size > 0) {
-      const entries = [...ctx.pending].map((filePath) => ({
-        file: normalizePath(path.relative(ctx.rootDir, filePath)),
-      }));
+      const entries = buildFlushEntriesFromPending(ctx.rootDir, ctx.pending);
       try {
-        appendJournalEntries(ctx.rootDir, entries);
+        appendJournalEntriesAndStampHeader(ctx.rootDir, entries, Date.now());
       } catch (e: unknown) {
         debug(`Journal flush on exit failed (non-fatal): ${(e as Error).message}`);
       }
@@ -297,7 +317,7 @@ function setupShutdownHandler(ctx: WatcherContext, cleanup: () => void): void {
 
 export async function watchProject(
   rootDir: string,
-  opts: { engine?: string; poll?: boolean; pollInterval?: number } = {},
+  opts: { engine?: string; poll?: boolean; pollInterval?: number; dbPath?: string } = {},
 ): Promise<void> {
   const ctx = setupWatcher(rootDir, opts);