@karmaniverous/jeeves-watcher 0.18.8 → 0.18.9

package/dist/index.js

@@ -1,13 +1,15 @@
 import { join, dirname, resolve, relative, extname, basename, isAbsolute, sep } from 'node:path';
 import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, createConfigApplyHandler, substituteEnvVars, DEFAULT_PORTS, getBindAddress, postJson, fetchJson, getPackageVersion } from '@karmaniverous/jeeves';
-import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsAuthorConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
+import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsRetentionConfigSchema, vcsAuthorConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
 export { apiConfigSchema, configWatchConfigSchema, embeddingConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema, loggingConfigSchema, vectorStoreConfigSchema, watchConfigSchema } from '@karmaniverous/jeeves-watcher-core';
 import Fastify from 'fastify';
-import { readdir, stat, readFile, writeFile, mkdir, appendFile, access, rm } from 'node:fs/promises';
+import { readdir, stat, access, readFile, writeFile, mkdir, appendFile, rm } from 'node:fs/promises';
 import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
 import { existsSync, statSync, readFileSync, readdirSync, mkdirSync, writeFileSync, rmSync, renameSync } from 'node:fs';
 import ignore from 'ignore';
 import picomatch from 'picomatch';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import { z, ZodError } from 'zod';
@@ -21,8 +23,6 @@ import { toMarkdown } from 'mdast-util-to-markdown';
 import rehypeParse from 'rehype-parse';
 import { unified } from 'unified';
 import yaml from 'js-yaml';
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
 import crypto from 'crypto';
 import { packageDirectorySync } from 'package-directory';
 import { homedir } from 'node:os';
@@ -421,6 +421,77 @@ async function retry(fn, options) {
     throw lastError;
 }
 
+/**
+ * @module vcs/gitExec
+ * Shared git execution utilities.
+ */
+/** Promisified execFile for git commands. */
+const execFileAsync = promisify(execFile);
+/**
+ * Stage files via stdin to avoid ENAMETOOLONG on Windows.
+ *
+ * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
+ * through stdin (NUL-delimited) instead of passed as command-line
+ * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
+ * limit that triggers ENAMETOOLONG when batches contain many files
+ * with long absolute paths.
+ *
+ * @param files - Absolute paths of files to stage.
+ * @param cwd   - Repository root (working directory for git).
+ */
+function gitAddViaStdin(files, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
+            if (error)
+                reject(error);
+            else
+                resolve();
+        });
+        if (!child.stdin) {
+            reject(new Error('Failed to open stdin for git add'));
+            return;
+        }
+        // Suppress EPIPE — expected if git exits before we finish writing
+        child.stdin.on('error', (err) => {
+            if (err.code !== 'EPIPE') {
+                reject(err);
+            }
+        });
+        child.stdin.end(files.join('\0'));
+    });
+}
+/**
+ * Normalize a path for case-insensitive comparison on Windows.
+ * On Windows, lowercases the entire path; on other platforms, returns as-is.
+ *
+ * @param p - The path string to normalize.
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The normalized path.
+ */
+function normalizePathCase(p, platform = process.platform) {
+    return platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Find the longest-prefix-match root for a normalized path.
+ * Roots must be sorted longest-first for correct nested matching.
+ *
+ * @param roots - Root paths sorted longest-first.
+ * @param normalizedPath - Normalized absolute path (forward slashes).
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The matching root, or undefined.
+ */
+function findRootForPath(roots, normalizedPath, platform = process.platform) {
+    const comparePath = normalizePathCase(normalizedPath, platform);
+    for (const root of roots) {
+        const compareRoot = normalizePathCase(root, platform);
+        if (comparePath === compareRoot ||
+            comparePath.startsWith(compareRoot + '/')) {
+            return root;
+        }
+    }
+    return undefined;
+}
+
 /**
  * Best-effort base directory inference for a glob pattern.
  *
@@ -638,7 +709,7 @@ function groupByRoot(filePaths, watchPaths) {
 /** Default page size for prune scroll (smaller than search to reduce connection strain). */
 const PRUNE_SCROLL_PAGE_SIZE = 500;
 /** Max retry attempts for scroll page failures. */
-const SCROLL_RETRY_ATTEMPTS = 3;
+const SCROLL_RETRY_ATTEMPTS = 5;
 /** Base delay for scroll retry backoff (ms). */
 const SCROLL_RETRY_BASE_DELAY_MS = 1000;
 /**
@@ -664,26 +735,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
     }
     throw lastError;
 }
+/** Concurrency limit for fs.access() disk-existence checks. */
+const FS_ACCESS_CONCURRENCY = 100;
 /**
- * Compute the plan for a prune operation.
- * Scrolls all Qdrant points page-by-page with retry/resume,
- * checks each file_path against watch scope + gitignore,
- * returns the list of orphaned point IDs and the plan.
+ * Compute the plan for a prune operation using a filter-first approach.
+ *
+ * 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
+ * 2. Scroll Qdrant points. For each point:
+ *    - No `file_path` → orphan.
+ *    - `file_path` not in `watchedFiles` → orphan (out of watch scope).
+ *    - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
+ *      If missing from disk → orphan (deleted file still in index).
+ * 3. Batch `fs.access()` checks with bounded concurrency.
  */
 async function computePrunePlan(deps) {
     const { config, vectorStore, gitignoreFilter, logger } = deps;
     if (!vectorStore) {
         throw new Error('vectorStore is required for prune scope');
     }
+    // Step 1: Build set of watched files from filesystem
+    const isGitignored = createIsGitignored(gitignoreFilter);
+    const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
+    const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
+    logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
+    // Step 2: Scroll Qdrant and classify points
     const seenFiles = new Set();
     const orphanedFiles = new Set();
+    const diskCheckFiles = new Set();
     const orphanedIds = [];
+    // Map from file_path → point IDs, for points needing disk-existence check
+    const diskCheckPointIds = new Map();
     let totalPoints = 0;
     let incomplete = false;
     let cursor;
     try {
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cursor is updated each iteration
-        while (true) {
+        let hasMore = true;
+        while (hasMore) {
             const page = await scrollPageWithRetry(vectorStore, cursor, logger);
             for (const point of page.points) {
                 totalPoints++;
@@ -692,32 +779,62 @@ async function computePrunePlan(deps) {
                     orphanedIds.push(point.id);
                     continue;
                 }
-                if (seenFiles.has(filePath)) {
-                    if (orphanedFiles.has(filePath)) {
+                const normalized = normalizePathCase(normalizeSlashes(filePath));
+                if (seenFiles.has(normalized)) {
+                    if (orphanedFiles.has(normalized)) {
                         orphanedIds.push(point.id);
                     }
+                    else if (diskCheckFiles.has(normalized)) {
+                        // Already queued for disk check — track its ID
+                        const ids = diskCheckPointIds.get(normalized);
+                        if (ids)
+                            ids.push(point.id);
+                    }
                     continue;
                 }
-                seenFiles.add(filePath);
-                const watched = isPathWatched(filePath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
-                const gitignored = gitignoreFilter
-                    ? gitignoreFilter.isIgnored(filePath)
-                    : false;
-                if (!watched || gitignored) {
-                    orphanedFiles.add(filePath);
+                seenFiles.add(normalized);
+                if (!watchedFiles.has(normalized)) {
+                    // Not in watch scope → definitely orphaned
+                    orphanedFiles.add(normalized);
                     orphanedIds.push(point.id);
                 }
+                else {
+                    // In watch scope → needs disk-existence check
+                    diskCheckFiles.add(normalized);
+                    diskCheckPointIds.set(normalized, [point.id]);
+                }
+            }
+            // Progress logging every 100K points
+            if (totalPoints > 0 && totalPoints % 100000 === 0) {
+                logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
+            }
+            if (!page.nextCursor) {
+                hasMore = false;
+            }
+            else {
+                cursor = page.nextCursor;
             }
-            if (!page.nextCursor)
-                break;
-            cursor = page.nextCursor;
         }
     }
     catch (error) {
-        // All retries exhausted - return partial results
         logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
         incomplete = true;
     }
+    // Step 3: Batch fs.access() checks for files in watch scope
+    const filesToCheck = [...diskCheckFiles];
+    await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
+        try {
+            await access(filePath);
+        }
+        catch {
+            // File doesn't exist on disk → orphaned
+            orphanedFiles.add(filePath);
+            const pointIds = diskCheckPointIds.get(filePath);
+            if (pointIds) {
+                orphanedIds.push(...pointIds);
+            }
+        }
+    });
     const orphanedFileList = [...orphanedFiles];
     const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
     return {
@@ -775,6 +892,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
+        // Abort live prune when plan is incomplete — never silently delete partial data
+        if (plan.incomplete) {
+            deps.queue?.resume();
+            logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
+            return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
+        }
         // Execute prune
         reindexTracker?.start(scope);
         const startTime = Date.now();
@@ -3369,77 +3492,6 @@ function createSearchHandler(deps) {
     }, deps.logger, 'Search');
 }
 
-/**
- * @module vcs/gitExec
- * Shared git execution utilities.
- */
-/** Promisified execFile for git commands. */
-const execFileAsync = promisify(execFile);
-/**
- * Stage files via stdin to avoid ENAMETOOLONG on Windows.
- *
- * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
- * through stdin (NUL-delimited) instead of passed as command-line
- * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
- * limit that triggers ENAMETOOLONG when batches contain many files
- * with long absolute paths.
- *
- * @param files - Absolute paths of files to stage.
- * @param cwd   - Repository root (working directory for git).
- */
-function gitAddViaStdin(files, cwd) {
-    return new Promise((resolve, reject) => {
-        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
-            if (error)
-                reject(error);
-            else
-                resolve();
-        });
-        if (!child.stdin) {
-            reject(new Error('Failed to open stdin for git add'));
-            return;
-        }
-        // Suppress EPIPE — expected if git exits before we finish writing
-        child.stdin.on('error', (err) => {
-            if (err.code !== 'EPIPE') {
-                reject(err);
-            }
-        });
-        child.stdin.end(files.join('\0'));
-    });
-}
-/**
- * Normalize a path for case-insensitive comparison on Windows.
- * On Windows, lowercases the entire path; on other platforms, returns as-is.
- *
- * @param p - The path string to normalize.
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The normalized path.
- */
-function normalizePathCase(p, platform = process.platform) {
-    return platform === 'win32' ? p.toLowerCase() : p;
-}
-/**
- * Find the longest-prefix-match root for a normalized path.
- * Roots must be sorted longest-first for correct nested matching.
- *
- * @param roots - Root paths sorted longest-first.
- * @param normalizedPath - Normalized absolute path (forward slashes).
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The matching root, or undefined.
- */
-function findRootForPath(roots, normalizedPath, platform = process.platform) {
-    const comparePath = normalizePathCase(normalizedPath, platform);
-    for (const root of roots) {
-        const compareRoot = normalizePathCase(root, platform);
-        if (comparePath === compareRoot ||
-            comparePath.startsWith(compareRoot + '/')) {
-            return root;
-        }
-    }
-    return undefined;
-}
-
 /**
  * @module vcs/resolveWatchRoot
  * Resolves which VCS-enabled watch root(s) own a given absolute path or glob.
@@ -4975,6 +5027,7 @@ class VcsManager {
     squashManager;
     debounceTimer;
     commitInFlight = Promise.resolve();
+    consecutiveCommitFailures = 0;
     started = false;
     isBaseline = true;
     _lastPushTime = null;
@@ -5023,6 +5076,11 @@ class VcsManager {
     fileChanged(filePath) {
         if (!this.started)
             return;
+        // Reset circuit breaker on new file change so the system can recover
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.info({ root: this.rootPath }, 'VCS circuit breaker reset — new file change received, retrying commits');
+            this.consecutiveCommitFailures = 0;
+        }
         this.pending.add(filePath);
         if (this.pending.size >= this.config.maxBatchSize) {
             this.clearDebounce();
@@ -5110,12 +5168,44 @@ class VcsManager {
         }, this.config.commitDebounceMs);
     }
     /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Check for and remove stale index.lock files before retrying.
+     */
+    async removeStaleLock() {
+        const lockPath = join(this.rootPath, '.git', 'index.lock');
+        try {
+            const lockStat = await stat(lockPath);
+            const ageMs = Date.now() - lockStat.mtimeMs;
+            if (ageMs > this.config.staleLockThresholdMs) {
+                await rm(lockPath, { force: true });
+                this.logger.warn({ root: this.rootPath, ageMs }, 'Removed stale index.lock');
+            }
+        }
+        catch (error) {
+            const code = error.code;
+            if (code !== 'ENOENT') {
+                this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Unable to remove stale index.lock');
+            }
+        }
+    }
+    /**
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     async commitBatch(files) {
         if (files.length === 0)
             return;
+        // Circuit breaker: skip if too many consecutive failures
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.error({
+                root: this.rootPath,
+                consecutiveFailures: this.consecutiveCommitFailures,
+                discardedFiles: files.length,
+            }, 'VCS circuit breaker tripped — pending files discarded until next successful commit');
+            return;
+        }
         try {
+            // Check for stale lock before first attempt
+            await this.removeStaleLock();
             await retry(async (attempt) => {
                 await gitAddViaStdin(files, this.rootPath);
                 // Build message after staging so getStagedDiff can see the changes.
@@ -5141,12 +5231,22 @@ class VcsManager {
                     this.logger.warn({ root: this.rootPath, attempt, delay: delayMs }, 'index.lock contention, retrying');
                 },
             });
+            // Success — reset circuit breaker
+            this.consecutiveCommitFailures = 0;
         }
         catch (error) {
-            for (const f of files) {
+            this.consecutiveCommitFailures++;
+            // Re-queue cap: only re-queue up to maxBatchSize files
+            const cap = this.config.maxBatchSize;
+            const toRequeue = files.slice(0, cap);
+            const overflow = files.length - toRequeue.length;
+            for (const f of toRequeue) {
                 this.pending.add(f);
             }
-            this.logger.warn({ root: this.rootPath, fileCount: files.length }, 'Re-queued files after commit failure');
+            if (overflow > 0) {
+                this.logger.warn({ root: this.rootPath, overflow, cap }, 'Re-queue cap exceeded — discarded overflow files');
+            }
+            this.logger.warn({ root: this.rootPath, fileCount: toRequeue.length }, 'Re-queued files after commit failure');
             this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS commit failed');
         }
     }
@@ -5178,10 +5278,14 @@ class VcsCoordinator {
             // Deduplicate: skip if another glob already resolved to this root.
             if (this.managers.has(rootKey))
                 continue;
+            const mergedRetention = entry.vcs?.retention ??
+                config.vcs.retention ??
+                vcsRetentionConfigSchema.parse({});
             const mergedConfig = {
                 ...config.vcs,
                 ...entry.vcs,
                 enabled: true,
+                retention: mergedRetention,
             };
             // Create CommitMessageGenerator if AI commit messages are configured
             const cmConfig = mergedConfig.commitMessage;
@@ -5315,10 +5419,10 @@ function createWatcher(config, factories, queue, processor, logger, runtimeOptio
  * Tear down and rebuild the filesystem watcher with new config.
  * Falls back to the old watcher if the new one fails to start.
  */
-async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache) {
+async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
     try {
         await oldState.watcher.stop();
-        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache);
+        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete);
         newWatcher.start();
         logger.info('Filesystem watcher rebuilt successfully');
         return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
@@ -6459,7 +6563,7 @@ async function reloadConfig(state, deps) {
         // Rebuild filesystem watcher if watch config changed
         if (watchConfigChanged(oldConfig, newConfig)) {
             logger.info('Watch config changed, rebuilding filesystem watcher...');
-            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache);
+            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache, deps.onVcsFileChange, deps.onInitialScanComplete);
             state.watcher = newState.watcher;
             state.gitignoreFilter = newState.gitignoreFilter;
         }
@@ -9140,6 +9244,8 @@ class JeevesWatcher {
             contentHashCache: this.contentHashCache,
             valuesManager: this.valuesManager,
             issuesManager: this.issuesManager,
+            onVcsFileChange: (filePath, event) => this.vcsCoordinator?.onFileChange(filePath, event),
+            onInitialScanComplete: () => this.vcsCoordinator?.onInitialScanComplete(),
         });
         this.config = state.config;
         this.watcher = state.watcher;

package/package.json

@@ -9,7 +9,7 @@
   "dependencies": {
     "@commander-js/extra-typings": "^14.0.0",
     "@karmaniverous/jeeves": "^0.5.12",
-    "@karmaniverous/jeeves-watcher-core": "^0.2.2",
+    "@karmaniverous/jeeves-watcher-core": "^0.2.3",
     "@karmaniverous/jsonmap": "^2.1.1",
     "@langchain/textsplitters": "*",
     "@qdrant/js-client-rest": "*",
@@ -149,5 +149,5 @@
   },
   "type": "module",
   "types": "dist/index.d.ts",
-  "version": "0.18.8"
+  "version": "0.18.9"
 }