@karmaniverous/jeeves-watcher 0.18.8 → 0.18.10

package/dist/cli/jeeves-watcher/index.js

@@ -7,10 +7,12 @@ import { packageDirectorySync } from 'package-directory';
 import { readFileSync, existsSync, mkdirSync, renameSync, statSync, readdirSync } from 'node:fs';
 import { cosmiconfig } from 'cosmiconfig';
 import { ZodError } from 'zod';
-import { readdir, stat } from 'node:fs/promises';
+import { readdir, stat, access } from 'node:fs/promises';
 import { parallel } from 'radash';
 import ignore from 'ignore';
 import picomatch from 'picomatch';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 
 /**
  * @module api/handlers/configMerge
@@ -224,7 +226,7 @@ async function startFromConfig(configPath, descriptor) {
     const config = await loadConfig(resolvedPath);
     // Dynamic import breaks the circular dependency between this module
     // and app/index.ts (which defines JeevesWatcher and re-exports startFromConfig).
-    const { JeevesWatcher } = await import('./index-BjghvUnc.js');
+    const { JeevesWatcher } = await import('./index-B_g6RfR8.js');
     const app = new JeevesWatcher(config, resolvedPath, descriptor);
     installShutdownHandlers(() => app.stop());
     await app.start();
@@ -613,6 +615,77 @@ async function retry(fn, options) {
     throw lastError;
 }
 
+/**
+ * @module vcs/gitExec
+ * Shared git execution utilities.
+ */
+/** Promisified execFile for git commands. */
+const execFileAsync = promisify(execFile);
+/**
+ * Stage files via stdin to avoid ENAMETOOLONG on Windows.
+ *
+ * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
+ * through stdin (NUL-delimited) instead of passed as command-line
+ * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
+ * limit that triggers ENAMETOOLONG when batches contain many files
+ * with long absolute paths.
+ *
+ * @param files - Absolute paths of files to stage.
+ * @param cwd   - Repository root (working directory for git).
+ */
+function gitAddViaStdin(files, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
+            if (error)
+                reject(error);
+            else
+                resolve();
+        });
+        if (!child.stdin) {
+            reject(new Error('Failed to open stdin for git add'));
+            return;
+        }
+        // Suppress EPIPE — expected if git exits before we finish writing
+        child.stdin.on('error', (err) => {
+            if (err.code !== 'EPIPE') {
+                reject(err);
+            }
+        });
+        child.stdin.end(files.join('\0'));
+    });
+}
+/**
+ * Normalize a path for case-insensitive comparison on Windows.
+ * On Windows, lowercases the entire path; on other platforms, returns as-is.
+ *
+ * @param p - The path string to normalize.
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The normalized path.
+ */
+function normalizePathCase(p, platform = process.platform) {
+    return platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Find the longest-prefix-match root for a normalized path.
+ * Roots must be sorted longest-first for correct nested matching.
+ *
+ * @param roots - Root paths sorted longest-first.
+ * @param normalizedPath - Normalized absolute path (forward slashes).
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The matching root, or undefined.
+ */
+function findRootForPath(roots, normalizedPath, platform = process.platform) {
+    const comparePath = normalizePathCase(normalizedPath, platform);
+    for (const root of roots) {
+        const compareRoot = normalizePathCase(root, platform);
+        if (comparePath === compareRoot ||
+            comparePath.startsWith(compareRoot + '/')) {
+            return root;
+        }
+    }
+    return undefined;
+}
+
 /**
  * Best-effort base directory inference for a glob pattern.
  *
@@ -830,7 +903,7 @@ function groupByRoot(filePaths, watchPaths) {
 /** Default page size for prune scroll (smaller than search to reduce connection strain). */
 const PRUNE_SCROLL_PAGE_SIZE = 500;
 /** Max retry attempts for scroll page failures. */
-const SCROLL_RETRY_ATTEMPTS = 3;
+const SCROLL_RETRY_ATTEMPTS = 5;
 /** Base delay for scroll retry backoff (ms). */
 const SCROLL_RETRY_BASE_DELAY_MS = 1000;
 /**
@@ -856,26 +929,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
     }
     throw lastError;
 }
+/** Concurrency limit for fs.access() disk-existence checks. */
+const FS_ACCESS_CONCURRENCY = 100;
 /**
- * Compute the plan for a prune operation.
- * Scrolls all Qdrant points page-by-page with retry/resume,
- * checks each file_path against watch scope + gitignore,
- * returns the list of orphaned point IDs and the plan.
+ * Compute the plan for a prune operation using a filter-first approach.
+ *
+ * 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
+ * 2. Scroll Qdrant points. For each point:
+ *    - No `file_path` → orphan.
+ *    - `file_path` not in `watchedFiles` → orphan (out of watch scope).
+ *    - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
+ *      If missing from disk → orphan (deleted file still in index).
+ * 3. Batch `fs.access()` checks with bounded concurrency.
  */
 async function computePrunePlan(deps) {
     const { config, vectorStore, gitignoreFilter, logger } = deps;
     if (!vectorStore) {
         throw new Error('vectorStore is required for prune scope');
     }
+    // Step 1: Build set of watched files from filesystem
+    const isGitignored = createIsGitignored(gitignoreFilter);
+    const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
+    const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
+    logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
+    // Step 2: Scroll Qdrant and classify points
     const seenFiles = new Set();
     const orphanedFiles = new Set();
+    const diskCheckFiles = new Set();
     const orphanedIds = [];
+    // Map from file_path → point IDs, for points needing disk-existence check
+    const diskCheckPointIds = new Map();
     let totalPoints = 0;
     let incomplete = false;
     let cursor;
     try {
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cursor is updated each iteration
-        while (true) {
+        let hasMore = true;
+        while (hasMore) {
             const page = await scrollPageWithRetry(vectorStore, cursor, logger);
             for (const point of page.points) {
                 totalPoints++;
@@ -884,32 +973,62 @@ async function computePrunePlan(deps) {
                     orphanedIds.push(point.id);
                     continue;
                 }
-                if (seenFiles.has(filePath)) {
-                    if (orphanedFiles.has(filePath)) {
+                const normalized = normalizePathCase(normalizeSlashes(filePath));
+                if (seenFiles.has(normalized)) {
+                    if (orphanedFiles.has(normalized)) {
                         orphanedIds.push(point.id);
                     }
+                    else if (diskCheckFiles.has(normalized)) {
+                        // Already queued for disk check — track its ID
+                        const ids = diskCheckPointIds.get(normalized);
+                        if (ids)
+                            ids.push(point.id);
+                    }
                     continue;
                 }
-                seenFiles.add(filePath);
-                const watched = isPathWatched(filePath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
-                const gitignored = gitignoreFilter
-                    ? gitignoreFilter.isIgnored(filePath)
-                    : false;
-                if (!watched || gitignored) {
-                    orphanedFiles.add(filePath);
+                seenFiles.add(normalized);
+                if (!watchedFiles.has(normalized)) {
+                    // Not in watch scope → definitely orphaned
+                    orphanedFiles.add(normalized);
                     orphanedIds.push(point.id);
                 }
+                else {
+                    // In watch scope → needs disk-existence check
+                    diskCheckFiles.add(normalized);
+                    diskCheckPointIds.set(normalized, [point.id]);
+                }
+            }
+            // Progress logging every 100K points
+            if (totalPoints > 0 && totalPoints % 100000 === 0) {
+                logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
+            }
+            if (!page.nextCursor) {
+                hasMore = false;
+            }
+            else {
+                cursor = page.nextCursor;
             }
-            if (!page.nextCursor)
-                break;
-            cursor = page.nextCursor;
         }
     }
     catch (error) {
-        // All retries exhausted - return partial results
         logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
         incomplete = true;
     }
+    // Step 3: Batch fs.access() checks for files in watch scope
+    const filesToCheck = [...diskCheckFiles];
+    await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
+        try {
+            await access(filePath);
+        }
+        catch {
+            // File doesn't exist on disk → orphaned
+            orphanedFiles.add(filePath);
+            const pointIds = diskCheckPointIds.get(filePath);
+            if (pointIds) {
+                orphanedIds.push(...pointIds);
+            }
+        }
+    });
     const orphanedFileList = [...orphanedFiles];
     const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
     return {
@@ -967,6 +1086,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
+        // Abort live prune when plan is incomplete — never silently delete partial data
+        if (plan.incomplete) {
+            deps.queue?.resume();
+            logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
+            return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
+        }
         // Execute prune
         reindexTracker?.start(scope);
         const startTime = Date.now();
@@ -1456,4 +1581,4 @@ const watcherDescriptor = {
 const program = createServiceCli(watcherDescriptor);
 program.parse();
 
-export { CONFIG_WATCH_VALID_SCOPES as C, GitignoreFilter as G, VALID_REINDEX_SCOPES as V, normalizeSlashes as a, startFromConfig as b, executeReindex as e, getWatchRootBases as g, isPathWatched as i, loadConfig as l, mergeInferenceRules as m, normalizeError as n, retry as r, sleep as s };
+export { CONFIG_WATCH_VALID_SCOPES as C, GitignoreFilter as G, VALID_REINDEX_SCOPES as V, executeReindex as a, gitAddViaStdin as b, normalizePathCase as c, normalizeSlashes as d, execFileAsync as e, findRootForPath as f, getWatchRootBases as g, startFromConfig as h, isPathWatched as i, loadConfig as l, mergeInferenceRules as m, normalizeError as n, retry as r, sleep as s };

package/dist/index.d.ts

@@ -1276,7 +1276,7 @@ interface PushError {
  * Per-root VCS manager for git-backed content versioning.
  *
  * Each VCS-enabled watch root gets its own VcsManager instance, which owns
- * a debounced commit pipeline: file changes are batched, staged, committed
+ * a throttled commit pipeline: file changes are batched, staged, committed
  * (with optional AI-generated messages), and optionally pushed to a remote.
  *
  * Concurrency: only one commit is in-flight at a time per root. Index.lock
@@ -1293,8 +1293,9 @@ declare class VcsManager {
     private readonly pendingReversions;
     private readonly _pushErrors;
     private readonly squashManager;
-    private debounceTimer;
+    private throttleTimer;
     private commitInFlight;
+    private consecutiveCommitFailures;
     private started;
     private isBaseline;
     private _lastPushTime;
@@ -1315,7 +1316,7 @@ declare class VcsManager {
      */
     endBaseline(): void;
     /**
-     * Add a file to the pending set and reset the debounce timer.
+     * Add a file to the pending set and start the throttle timer.
      * If the pending set exceeds maxBatchSize, flush immediately.
      *
      * @param filePath - Absolute path of the changed file.
@@ -1347,15 +1348,21 @@ declare class VcsManager {
      */
     private takeBatch;
     /**
-     * Clear the debounce timer.
+     * Clear the throttle timer.
      */
-    private clearDebounce;
+    private clearThrottle;
     /**
-     * Reset the debounce timer to fire flush() after commitDebounceMs.
+     * Start the throttle timer to fire flush() after commitThrottleMs.
+     * If a timer is already running, do not reset it — this is throttle, not debounce.
      */
-    private resetDebounce;
+    private resetThrottle;
     /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Check for and remove stale index.lock files before retrying.
+     */
+    private removeStaleLock;
+    /**
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     private commitBatch;
 }
@@ -1388,8 +1395,8 @@ declare class VcsCoordinator {
     /**
      * Signal that the initial filesystem scan is complete.
      *
-     * Flushes each manager's debounce buffer before ending baseline mode, ensuring
-     * any files still pending in the VCS debounce are committed with a
+     * Flushes each manager's throttle buffer before ending baseline mode, ensuring
+     * any files still pending in the VCS throttle are committed with a
      * `"baseline:"` prefix rather than `"watcher:"`. AI generation is not enabled
      * until all managers have completed their flush.
      */