@karmaniverous/jeeves-watcher 0.18.8 → 0.18.9

package/dist/cli/jeeves-watcher/{index-BjghvUnc.js → index-xP2XrV4W.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, createConfigApplyHandler, DEFAULT_PORTS, getBindAddress } from '@karmaniverous/jeeves';
-import { a as normalizeSlashes, n as normalizeError, r as retry, G as GitignoreFilter, e as executeReindex, V as VALID_REINDEX_SCOPES, m as mergeInferenceRules, i as isPathWatched, g as getWatchRootBases, C as CONFIG_WATCH_VALID_SCOPES, s as sleep, l as loadConfig } from './index.js';
-export { b as startFromConfig } from './index.js';
+import { d as normalizeSlashes, n as normalizeError, e as execFileAsync, r as retry, b as gitAddViaStdin, c as normalizePathCase, f as findRootForPath, G as GitignoreFilter, a as executeReindex, V as VALID_REINDEX_SCOPES, m as mergeInferenceRules, i as isPathWatched, g as getWatchRootBases, C as CONFIG_WATCH_VALID_SCOPES, s as sleep, l as loadConfig } from './index.js';
+export { h as startFromConfig } from './index.js';
 import { mkdirSync, existsSync, readFileSync, writeFileSync, rmSync, renameSync, readdirSync, statSync } from 'node:fs';
 import { join, dirname, resolve, extname, basename, relative, sep, isAbsolute } from 'node:path';
 import Database from 'better-sqlite3';
@@ -10,11 +10,9 @@ import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import picomatch from 'picomatch';
 import { deserialize, serialize } from 'node:v8';
-import { normalizeWatchPaths, extractWatchPathStrings, vcsAuthorConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema } from '@karmaniverous/jeeves-watcher-core';
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
+import { normalizeWatchPaths, vcsRetentionConfigSchema, extractWatchPathStrings, vcsAuthorConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema } from '@karmaniverous/jeeves-watcher-core';
 import { homedir } from 'node:os';
-import { access, rm, readFile, writeFile, appendFile, mkdir, stat } from 'node:fs/promises';
+import { access, rm, stat, readFile, writeFile, appendFile, mkdir } from 'node:fs/promises';
 import { pathToFileURL, fileURLToPath } from 'node:url';
 import { packageDirectorySync } from 'package-directory';
 import { JsonMap } from '@karmaniverous/jsonmap';
@@ -40,6 +38,8 @@ import mammoth from 'mammoth';
 import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import { QdrantClient } from '@qdrant/js-client-rest';
 import 'cosmiconfig';
+import 'node:child_process';
+import 'node:util';
 
 /**
  * @module api/InitialScanTracker
@@ -928,77 +928,6 @@ class CommitMessageGenerator {
     }
 }
 
-/**
- * @module vcs/gitExec
- * Shared git execution utilities.
- */
-/** Promisified execFile for git commands. */
-const execFileAsync = promisify(execFile);
-/**
- * Stage files via stdin to avoid ENAMETOOLONG on Windows.
- *
- * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
- * through stdin (NUL-delimited) instead of passed as command-line
- * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
- * limit that triggers ENAMETOOLONG when batches contain many files
- * with long absolute paths.
- *
- * @param files - Absolute paths of files to stage.
- * @param cwd   - Repository root (working directory for git).
- */
-function gitAddViaStdin(files, cwd) {
-    return new Promise((resolve, reject) => {
-        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
-            if (error)
-                reject(error);
-            else
-                resolve();
-        });
-        if (!child.stdin) {
-            reject(new Error('Failed to open stdin for git add'));
-            return;
-        }
-        // Suppress EPIPE — expected if git exits before we finish writing
-        child.stdin.on('error', (err) => {
-            if (err.code !== 'EPIPE') {
-                reject(err);
-            }
-        });
-        child.stdin.end(files.join('\0'));
-    });
-}
-/**
- * Normalize a path for case-insensitive comparison on Windows.
- * On Windows, lowercases the entire path; on other platforms, returns as-is.
- *
- * @param p - The path string to normalize.
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The normalized path.
- */
-function normalizePathCase(p, platform = process.platform) {
-    return platform === 'win32' ? p.toLowerCase() : p;
-}
-/**
- * Find the longest-prefix-match root for a normalized path.
- * Roots must be sorted longest-first for correct nested matching.
- *
- * @param roots - Root paths sorted longest-first.
- * @param normalizedPath - Normalized absolute path (forward slashes).
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The matching root, or undefined.
- */
-function findRootForPath(roots, normalizedPath, platform = process.platform) {
-    const comparePath = normalizePathCase(normalizedPath, platform);
-    for (const root of roots) {
-        const compareRoot = normalizePathCase(root, platform);
-        if (comparePath === compareRoot ||
-            comparePath.startsWith(compareRoot + '/')) {
-            return root;
-        }
-    }
-    return undefined;
-}
-
 /**
  * @module vcs/resolveCommitMessageApiKey
  * Resolves the API key for AI commit message generation.
@@ -1479,6 +1408,7 @@ class VcsManager {
     squashManager;
     debounceTimer;
     commitInFlight = Promise.resolve();
+    consecutiveCommitFailures = 0;
     started = false;
     isBaseline = true;
     _lastPushTime = null;
@@ -1527,6 +1457,11 @@ class VcsManager {
     fileChanged(filePath) {
         if (!this.started)
             return;
+        // Reset circuit breaker on new file change so the system can recover
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.info({ root: this.rootPath }, 'VCS circuit breaker reset — new file change received, retrying commits');
+            this.consecutiveCommitFailures = 0;
+        }
         this.pending.add(filePath);
         if (this.pending.size >= this.config.maxBatchSize) {
             this.clearDebounce();
@@ -1614,12 +1549,44 @@ class VcsManager {
         }, this.config.commitDebounceMs);
     }
     /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Check for and remove stale index.lock files before retrying.
+     */
+    async removeStaleLock() {
+        const lockPath = join(this.rootPath, '.git', 'index.lock');
+        try {
+            const lockStat = await stat(lockPath);
+            const ageMs = Date.now() - lockStat.mtimeMs;
+            if (ageMs > this.config.staleLockThresholdMs) {
+                await rm(lockPath, { force: true });
+                this.logger.warn({ root: this.rootPath, ageMs }, 'Removed stale index.lock');
+            }
+        }
+        catch (error) {
+            const code = error.code;
+            if (code !== 'ENOENT') {
+                this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Unable to remove stale index.lock');
+            }
+        }
+    }
+    /**
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     async commitBatch(files) {
         if (files.length === 0)
             return;
+        // Circuit breaker: skip if too many consecutive failures
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.error({
+                root: this.rootPath,
+                consecutiveFailures: this.consecutiveCommitFailures,
+                discardedFiles: files.length,
+            }, 'VCS circuit breaker tripped — pending files discarded until next successful commit');
+            return;
+        }
         try {
+            // Check for stale lock before first attempt
+            await this.removeStaleLock();
             await retry(async (attempt) => {
                 await gitAddViaStdin(files, this.rootPath);
                 // Build message after staging so getStagedDiff can see the changes.
@@ -1645,12 +1612,22 @@ class VcsManager {
                     this.logger.warn({ root: this.rootPath, attempt, delay: delayMs }, 'index.lock contention, retrying');
                 },
             });
+            // Success — reset circuit breaker
+            this.consecutiveCommitFailures = 0;
         }
         catch (error) {
-            for (const f of files) {
+            this.consecutiveCommitFailures++;
+            // Re-queue cap: only re-queue up to maxBatchSize files
+            const cap = this.config.maxBatchSize;
+            const toRequeue = files.slice(0, cap);
+            const overflow = files.length - toRequeue.length;
+            for (const f of toRequeue) {
                 this.pending.add(f);
             }
-            this.logger.warn({ root: this.rootPath, fileCount: files.length }, 'Re-queued files after commit failure');
+            if (overflow > 0) {
+                this.logger.warn({ root: this.rootPath, overflow, cap }, 'Re-queue cap exceeded — discarded overflow files');
+            }
+            this.logger.warn({ root: this.rootPath, fileCount: toRequeue.length }, 'Re-queued files after commit failure');
             this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS commit failed');
         }
     }
@@ -1682,10 +1659,14 @@ class VcsCoordinator {
             // Deduplicate: skip if another glob already resolved to this root.
             if (this.managers.has(rootKey))
                 continue;
+            const mergedRetention = entry.vcs?.retention ??
+                config.vcs.retention ??
+                vcsRetentionConfigSchema.parse({});
             const mergedConfig = {
                 ...config.vcs,
                 ...entry.vcs,
                 enabled: true,
+                retention: mergedRetention,
             };
             // Create CommitMessageGenerator if AI commit messages are configured
             const cmConfig = mergedConfig.commitMessage;
@@ -3179,10 +3160,10 @@ function createWatcher(config, factories, queue, processor, logger, runtimeOptio
  * Tear down and rebuild the filesystem watcher with new config.
  * Falls back to the old watcher if the new one fails to start.
  */
-async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache) {
+async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
     try {
         await oldState.watcher.stop();
-        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache);
+        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete);
         newWatcher.start();
         logger.info('Filesystem watcher rebuilt successfully');
         return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
@@ -3377,7 +3358,7 @@ async function reloadConfig(state, deps) {
         // Rebuild filesystem watcher if watch config changed
         if (watchConfigChanged(oldConfig, newConfig)) {
             logger.info('Watch config changed, rebuilding filesystem watcher...');
-            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache);
+            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache, deps.onVcsFileChange, deps.onInitialScanComplete);
             state.watcher = newState.watcher;
             state.gitignoreFilter = newState.gitignoreFilter;
         }
@@ -7989,6 +7970,8 @@ class JeevesWatcher {
             contentHashCache: this.contentHashCache,
             valuesManager: this.valuesManager,
             issuesManager: this.issuesManager,
+            onVcsFileChange: (filePath, event) => this.vcsCoordinator?.onFileChange(filePath, event),
+            onInitialScanComplete: () => this.vcsCoordinator?.onInitialScanComplete(),
         });
         this.config = state.config;
         this.watcher = state.watcher;

package/dist/cli/jeeves-watcher/index.js

@@ -7,10 +7,12 @@ import { packageDirectorySync } from 'package-directory';
 import { readFileSync, existsSync, mkdirSync, renameSync, statSync, readdirSync } from 'node:fs';
 import { cosmiconfig } from 'cosmiconfig';
 import { ZodError } from 'zod';
-import { readdir, stat } from 'node:fs/promises';
+import { readdir, stat, access } from 'node:fs/promises';
 import { parallel } from 'radash';
 import ignore from 'ignore';
 import picomatch from 'picomatch';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 
 /**
  * @module api/handlers/configMerge
@@ -224,7 +226,7 @@ async function startFromConfig(configPath, descriptor) {
     const config = await loadConfig(resolvedPath);
     // Dynamic import breaks the circular dependency between this module
     // and app/index.ts (which defines JeevesWatcher and re-exports startFromConfig).
-    const { JeevesWatcher } = await import('./index-BjghvUnc.js');
+    const { JeevesWatcher } = await import('./index-xP2XrV4W.js');
     const app = new JeevesWatcher(config, resolvedPath, descriptor);
     installShutdownHandlers(() => app.stop());
     await app.start();
@@ -613,6 +615,77 @@ async function retry(fn, options) {
     throw lastError;
 }
 
+/**
+ * @module vcs/gitExec
+ * Shared git execution utilities.
+ */
+/** Promisified execFile for git commands. */
+const execFileAsync = promisify(execFile);
+/**
+ * Stage files via stdin to avoid ENAMETOOLONG on Windows.
+ *
+ * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
+ * through stdin (NUL-delimited) instead of passed as command-line
+ * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
+ * limit that triggers ENAMETOOLONG when batches contain many files
+ * with long absolute paths.
+ *
+ * @param files - Absolute paths of files to stage.
+ * @param cwd   - Repository root (working directory for git).
+ */
+function gitAddViaStdin(files, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
+            if (error)
+                reject(error);
+            else
+                resolve();
+        });
+        if (!child.stdin) {
+            reject(new Error('Failed to open stdin for git add'));
+            return;
+        }
+        // Suppress EPIPE — expected if git exits before we finish writing
+        child.stdin.on('error', (err) => {
+            if (err.code !== 'EPIPE') {
+                reject(err);
+            }
+        });
+        child.stdin.end(files.join('\0'));
+    });
+}
+/**
+ * Normalize a path for case-insensitive comparison on Windows.
+ * On Windows, lowercases the entire path; on other platforms, returns as-is.
+ *
+ * @param p - The path string to normalize.
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The normalized path.
+ */
+function normalizePathCase(p, platform = process.platform) {
+    return platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Find the longest-prefix-match root for a normalized path.
+ * Roots must be sorted longest-first for correct nested matching.
+ *
+ * @param roots - Root paths sorted longest-first.
+ * @param normalizedPath - Normalized absolute path (forward slashes).
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The matching root, or undefined.
+ */
+function findRootForPath(roots, normalizedPath, platform = process.platform) {
+    const comparePath = normalizePathCase(normalizedPath, platform);
+    for (const root of roots) {
+        const compareRoot = normalizePathCase(root, platform);
+        if (comparePath === compareRoot ||
+            comparePath.startsWith(compareRoot + '/')) {
+            return root;
+        }
+    }
+    return undefined;
+}
+
 /**
  * Best-effort base directory inference for a glob pattern.
  *
@@ -830,7 +903,7 @@ function groupByRoot(filePaths, watchPaths) {
 /** Default page size for prune scroll (smaller than search to reduce connection strain). */
 const PRUNE_SCROLL_PAGE_SIZE = 500;
 /** Max retry attempts for scroll page failures. */
-const SCROLL_RETRY_ATTEMPTS = 3;
+const SCROLL_RETRY_ATTEMPTS = 5;
 /** Base delay for scroll retry backoff (ms). */
 const SCROLL_RETRY_BASE_DELAY_MS = 1000;
 /**
@@ -856,26 +929,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
     }
     throw lastError;
 }
+/** Concurrency limit for fs.access() disk-existence checks. */
+const FS_ACCESS_CONCURRENCY = 100;
 /**
- * Compute the plan for a prune operation.
- * Scrolls all Qdrant points page-by-page with retry/resume,
- * checks each file_path against watch scope + gitignore,
- * returns the list of orphaned point IDs and the plan.
+ * Compute the plan for a prune operation using a filter-first approach.
+ *
+ * 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
+ * 2. Scroll Qdrant points. For each point:
+ *    - No `file_path` → orphan.
+ *    - `file_path` not in `watchedFiles` → orphan (out of watch scope).
+ *    - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
+ *      If missing from disk → orphan (deleted file still in index).
+ * 3. Batch `fs.access()` checks with bounded concurrency.
  */
 async function computePrunePlan(deps) {
     const { config, vectorStore, gitignoreFilter, logger } = deps;
     if (!vectorStore) {
         throw new Error('vectorStore is required for prune scope');
     }
+    // Step 1: Build set of watched files from filesystem
+    const isGitignored = createIsGitignored(gitignoreFilter);
+    const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
+    const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
+    logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
+    // Step 2: Scroll Qdrant and classify points
     const seenFiles = new Set();
     const orphanedFiles = new Set();
+    const diskCheckFiles = new Set();
     const orphanedIds = [];
+    // Map from file_path → point IDs, for points needing disk-existence check
+    const diskCheckPointIds = new Map();
     let totalPoints = 0;
     let incomplete = false;
     let cursor;
     try {
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cursor is updated each iteration
-        while (true) {
+        let hasMore = true;
+        while (hasMore) {
             const page = await scrollPageWithRetry(vectorStore, cursor, logger);
             for (const point of page.points) {
                 totalPoints++;
@@ -884,32 +973,62 @@ async function computePrunePlan(deps) {
                     orphanedIds.push(point.id);
                     continue;
                 }
-                if (seenFiles.has(filePath)) {
-                    if (orphanedFiles.has(filePath)) {
+                const normalized = normalizePathCase(normalizeSlashes(filePath));
+                if (seenFiles.has(normalized)) {
+                    if (orphanedFiles.has(normalized)) {
                         orphanedIds.push(point.id);
                     }
+                    else if (diskCheckFiles.has(normalized)) {
+                        // Already queued for disk check — track its ID
+                        const ids = diskCheckPointIds.get(normalized);
+                        if (ids)
+                            ids.push(point.id);
+                    }
                     continue;
                 }
-                seenFiles.add(filePath);
-                const watched = isPathWatched(filePath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
-                const gitignored = gitignoreFilter
-                    ? gitignoreFilter.isIgnored(filePath)
-                    : false;
-                if (!watched || gitignored) {
-                    orphanedFiles.add(filePath);
+                seenFiles.add(normalized);
+                if (!watchedFiles.has(normalized)) {
+                    // Not in watch scope → definitely orphaned
+                    orphanedFiles.add(normalized);
                     orphanedIds.push(point.id);
                 }
+                else {
+                    // In watch scope → needs disk-existence check
+                    diskCheckFiles.add(normalized);
+                    diskCheckPointIds.set(normalized, [point.id]);
+                }
+            }
+            // Progress logging every 100K points
+            if (totalPoints > 0 && totalPoints % 100000 === 0) {
+                logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
+            }
+            if (!page.nextCursor) {
+                hasMore = false;
+            }
+            else {
+                cursor = page.nextCursor;
             }
-            if (!page.nextCursor)
-                break;
-            cursor = page.nextCursor;
         }
     }
     catch (error) {
-        // All retries exhausted - return partial results
         logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
         incomplete = true;
     }
+    // Step 3: Batch fs.access() checks for files in watch scope
+    const filesToCheck = [...diskCheckFiles];
+    await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
+        try {
+            await access(filePath);
+        }
+        catch {
+            // File doesn't exist on disk → orphaned
+            orphanedFiles.add(filePath);
+            const pointIds = diskCheckPointIds.get(filePath);
+            if (pointIds) {
+                orphanedIds.push(...pointIds);
+            }
+        }
+    });
     const orphanedFileList = [...orphanedFiles];
     const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
     return {
@@ -967,6 +1086,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
+        // Abort live prune when plan is incomplete — never silently delete partial data
+        if (plan.incomplete) {
+            deps.queue?.resume();
+            logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
+            return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
+        }
         // Execute prune
         reindexTracker?.start(scope);
         const startTime = Date.now();
@@ -1456,4 +1581,4 @@ const watcherDescriptor = {
 const program = createServiceCli(watcherDescriptor);
 program.parse();
 
-export { CONFIG_WATCH_VALID_SCOPES as C, GitignoreFilter as G, VALID_REINDEX_SCOPES as V, normalizeSlashes as a, startFromConfig as b, executeReindex as e, getWatchRootBases as g, isPathWatched as i, loadConfig as l, mergeInferenceRules as m, normalizeError as n, retry as r, sleep as s };
+export { CONFIG_WATCH_VALID_SCOPES as C, GitignoreFilter as G, VALID_REINDEX_SCOPES as V, executeReindex as a, gitAddViaStdin as b, normalizePathCase as c, normalizeSlashes as d, execFileAsync as e, findRootForPath as f, getWatchRootBases as g, startFromConfig as h, isPathWatched as i, loadConfig as l, mergeInferenceRules as m, normalizeError as n, retry as r, sleep as s };

package/dist/index.d.ts

@@ -1295,6 +1295,7 @@ declare class VcsManager {
     private readonly squashManager;
     private debounceTimer;
     private commitInFlight;
+    private consecutiveCommitFailures;
     private started;
     private isBaseline;
     private _lastPushTime;
@@ -1355,7 +1356,12 @@ declare class VcsManager {
      */
     private resetDebounce;
     /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Check for and remove stale index.lock files before retrying.
+     */
+    private removeStaleLock;
+    /**
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     private commitBatch;
 }