doc2vec 2.13.0 → 2.13.1

package/README.md

@@ -767,6 +767,12 @@ If you don't specify a config path, it will look for config.yaml in the current
 
 ## Recent Changes
 
+### Code Scans: Dangling Symlinks and Incomplete Walks
+- **Dangling symlinks no longer abort a directory:** `statSync` follows symlinks, so a link whose target isn't in the clone (common in vendored forks, e.g. `crates/cel-fork/cel/README.md`) threw `ENOENT` and abandoned every remaining entry in that directory. Broken entries are now skipped with a warning and the walk continues
+- **Incomplete scans never delete chunks:** if a directory genuinely can't be listed, the scan is reported as incomplete — obsolete-file cleanup, the tracked-file list, the last-mtime marker, and the last-synced git SHA are all skipped, so unscanned files keep their chunks and the next run rescans them
+- **Incomplete scans fail the source:** a partial ingest is reported as a failed source instead of a silent success
+- **Quieter crawls:** planned browser restarts (initial launch, the every-50-pages recycle) log at `info`; only unplanned restarts (disconnected or degraded browser) warn
+
 ### Searchable Controller Run Logs
 - **Server-side log filtering:** Level chips and the keyword filter now query Postgres instead of the browser's in-memory buffer, so errors and warnings from the start of a long run are reachable (`GET /api/runs/:id/logs?levels=error,warn&q=...`, paged with **Load 2,000 more**)
 - **Whole-run level counts:** New `GET /api/runs/:id/logs/counts` endpoint backs the level chips, which previously only counted the lines still held in the browser

package/dist/content-processor.js

@@ -462,8 +462,14 @@ class ContentProcessor {
         let pagesSinceRestart = 0;
         // Restart the browser every N pages to prevent memory accumulation
         const PAGES_BETWEEN_RESTARTS = 50;
-        const restartBrowser = async (reason) => {
-            logger.warn(`Restarting browser: ${reason}`);
+        // Planned restarts (first launch, the every-N-pages recycle) are routine
+        // and would otherwise dominate a crawl's warning count; only unplanned
+        // ones — a disconnected or degraded browser — deserve a warning.
+        const restartBrowser = async (reason, planned = false) => {
+            if (planned)
+                logger.info(`Restarting browser: ${reason}`);
+            else
+                logger.warn(`Restarting browser: ${reason}`);
             if (browser) {
                 try {
                     await browser.close();
@@ -485,11 +491,11 @@ class ContentProcessor {
                 const reason = needsBrowserRestart
                     ? `protocol errors persisting after page recreation (${consecutiveProtocolErrors} consecutive)`
                     : browser ? 'browser disconnected' : 'initial launch';
-                await restartBrowser(reason);
+                await restartBrowser(reason, !browser && !needsBrowserRestart);
             }
             else if (pagesSinceRestart >= PAGES_BETWEEN_RESTARTS) {
                 // Periodic restart to prevent memory accumulation from long crawls
-                await restartBrowser(`periodic restart after ${pagesSinceRestart} pages`);
+                await restartBrowser(`periodic restart after ${pagesSinceRestart} pages`, true);
             }
             else if (pageNeedsRecreation) {
                 // The previous page.evaluate or navigation timed out / errored.
@@ -1539,7 +1545,17 @@ class ContentProcessor {
             let skippedFiles = 0;
             for (const file of files) {
                 const filePath = path.join(dirPath, file);
-                const stat = fs.statSync(filePath);
+                // A dangling symlink makes statSync throw ENOENT; skip the
+                // entry rather than abandoning the rest of the directory
+                let stat;
+                try {
+                    stat = fs.statSync(filePath);
+                }
+                catch (error) {
+                    logger.warn(`Skipping unreadable entry ${filePath}:`, error);
+                    skippedFiles++;
+                    continue;
+                }
                 // Skip already visited paths
                 if (visitedPaths.has(filePath)) {
                     logger.debug(`Skipping already visited path: ${filePath}`);
@@ -1629,6 +1645,7 @@ class ContentProcessor {
         const excludeExtensions = config.exclude_extensions || [];
         const encoding = config.encoding || 'utf8';
         let maxMtime = 0;
+        let incomplete = false;
         try {
             const files = fs.readdirSync(dirPath);
             let processedFiles = 0;
@@ -1638,7 +1655,19 @@ class ContentProcessor {
             const trackFiles = options?.trackFiles;
             for (const file of files) {
                 const filePath = path.join(dirPath, file);
-                const stat = fs.statSync(filePath);
+                // statSync follows symlinks, so a dangling one (common in
+                // vendored forks) throws ENOENT. Skipping the entry keeps the
+                // rest of the directory — letting it throw used to abandon
+                // every sibling after it, and cleanup then deleted their chunks.
+                let stat;
+                try {
+                    stat = fs.statSync(filePath);
+                }
+                catch (error) {
+                    logger.warn(`Skipping unreadable entry ${filePath}:`, error);
+                    skippedFiles++;
+                    continue;
+                }
                 if (visitedPaths.has(filePath)) {
                     logger.debug(`Skipping already visited path: ${filePath}`);
                     continue;
@@ -1650,6 +1679,7 @@ class ContentProcessor {
                         processedFiles += childResult.processedFiles;
                         skippedFiles += childResult.skippedFiles;
                         maxMtime = Math.max(maxMtime, childResult.maxMtime);
+                        incomplete = incomplete || childResult.incomplete;
                     }
                     else {
                         logger.debug(`Skipping directory ${filePath} (recursive=false)`);
@@ -1694,11 +1724,14 @@ class ContentProcessor {
                 }
             }
             logger.info(`Code directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
-            return { processedFiles, skippedFiles, maxMtime };
+            return { processedFiles, skippedFiles, maxMtime, incomplete };
         }
         catch (error) {
+            // The directory could not be listed at all — report the scan as
+            // incomplete so the caller skips cleanup instead of treating every
+            // file under this path as deleted
             logger.error(`Error reading code directory ${dirPath}:`, error);
-            return { processedFiles: 0, skippedFiles: 0, maxMtime };
+            return { processedFiles: 0, skippedFiles: 0, maxMtime, incomplete: true };
         }
     }
     async getTokenChunker(chunkSize) {

package/dist/doc2vec.js

@@ -1051,6 +1051,10 @@ class Doc2Vec {
         let lastMtimeKey;
         let trackedFiles;
         let maxObservedMtime = 0;
+        // Set when a directory could not be listed: the files we saw are only a
+        // subset of what is on disk, so nothing may be treated as deleted and no
+        // "last scanned" marker may advance past this run
+        let scanIncomplete = false;
         if (config.source === 'local_directory') {
             if (!config.path) {
                 logger.error('Code source type local_directory requires a path.');
@@ -1132,13 +1136,20 @@ class Doc2Vec {
                 mtimeCutoff,
                 trackFiles: trackedFiles
             });
+            scanIncomplete = scanResult.incomplete;
+            if (scanIncomplete) {
+                logger.error('Directory scan was incomplete — skipping cleanup so unscanned files keep their chunks');
+            }
             if (trackedFiles) {
                 maxObservedMtime = scanResult.maxMtime;
             }
         }
         finally {
             logger.section('CLEANUP');
-            if (incrementalMode) {
+            if (scanIncomplete) {
+                logger.warn('Skipping cleanup and last-scanned markers: the scan did not cover the whole tree');
+            }
+            else if (incrementalMode) {
                 if (deleteUrls.length > 0) {
                     logger.info(`Cleaning up ${deleteUrls.length} deleted/renamed files`);
                     for (const url of deleteUrls) {
@@ -1197,7 +1208,9 @@ class Doc2Vec {
                 this.counters.items_deleted += removed.items;
                 this.counters.chunks_deleted += removed.chunks;
             }
-            if (config.source === 'github' && basePath && repoBranch) {
+            // Storing the SHA after an incomplete scan would make the next run
+            // diff from it and never revisit the files this run missed
+            if (config.source === 'github' && basePath && repoBranch && !scanIncomplete) {
                 const headSha = await this.getRepoHeadSha(basePath, logger);
                 if (headSha) {
                     const shaKey = this.buildCodeShaMetadataKey(config.repo, repoBranch);
@@ -1214,6 +1227,11 @@ class Doc2Vec {
                 }
             }
         }
+        // Fail the source rather than reporting a partial ingest as a success —
+        // the collection is still serving whatever it had for the unscanned files
+        if (scanIncomplete) {
+            throw new Error(`Code scan of ${basePath} was incomplete: at least one directory could not be read`);
+        }
         logger.info(`Finished processing code source (${config.source})`);
     }
     buildCodeShaMetadataKey(repo, branch) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.13.0",
+    "version": "2.13.1",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",