@karmaniverous/jeeves-watcher 0.18.7 → 0.18.9

package/dist/cli/jeeves-watcher/index.js

@@ -7,10 +7,12 @@ import { packageDirectorySync } from 'package-directory';
 import { readFileSync, existsSync, mkdirSync, renameSync, statSync, readdirSync } from 'node:fs';
 import { cosmiconfig } from 'cosmiconfig';
 import { ZodError } from 'zod';
-import { readdir, stat } from 'node:fs/promises';
+import { readdir, stat, access } from 'node:fs/promises';
 import { parallel } from 'radash';
 import ignore from 'ignore';
 import picomatch from 'picomatch';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 
 /**
  * @module api/handlers/configMerge
@@ -224,7 +226,7 @@ async function startFromConfig(configPath, descriptor) {
     const config = await loadConfig(resolvedPath);
     // Dynamic import breaks the circular dependency between this module
     // and app/index.ts (which defines JeevesWatcher and re-exports startFromConfig).
-    const { JeevesWatcher } = await import('./index-CS1wrtes.js');
+    const { JeevesWatcher } = await import('./index-xP2XrV4W.js');
     const app = new JeevesWatcher(config, resolvedPath, descriptor);
     installShutdownHandlers(() => app.stop());
     await app.start();
@@ -613,6 +615,77 @@ async function retry(fn, options) {
     throw lastError;
 }
 
+/**
+ * @module vcs/gitExec
+ * Shared git execution utilities.
+ */
+/** Promisified execFile for git commands. */
+const execFileAsync = promisify(execFile);
+/**
+ * Stage files via stdin to avoid ENAMETOOLONG on Windows.
+ *
+ * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
+ * through stdin (NUL-delimited) instead of passed as command-line
+ * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
+ * limit that triggers ENAMETOOLONG when batches contain many files
+ * with long absolute paths.
+ *
+ * @param files - Absolute paths of files to stage.
+ * @param cwd   - Repository root (working directory for git).
+ */
+function gitAddViaStdin(files, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
+            if (error)
+                reject(error);
+            else
+                resolve();
+        });
+        if (!child.stdin) {
+            reject(new Error('Failed to open stdin for git add'));
+            return;
+        }
+        // Suppress EPIPE — expected if git exits before we finish writing
+        child.stdin.on('error', (err) => {
+            if (err.code !== 'EPIPE') {
+                reject(err);
+            }
+        });
+        child.stdin.end(files.join('\0'));
+    });
+}
+/**
+ * Normalize a path for case-insensitive comparison on Windows.
+ * On Windows, lowercases the entire path; on other platforms, returns as-is.
+ *
+ * @param p - The path string to normalize.
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The normalized path.
+ */
+function normalizePathCase(p, platform = process.platform) {
+    return platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Find the longest-prefix-match root for a normalized path.
+ * Roots must be sorted longest-first for correct nested matching.
+ *
+ * @param roots - Root paths sorted longest-first.
+ * @param normalizedPath - Normalized absolute path (forward slashes).
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The matching root, or undefined.
+ */
+function findRootForPath(roots, normalizedPath, platform = process.platform) {
+    const comparePath = normalizePathCase(normalizedPath, platform);
+    for (const root of roots) {
+        const compareRoot = normalizePathCase(root, platform);
+        if (comparePath === compareRoot ||
+            comparePath.startsWith(compareRoot + '/')) {
+            return root;
+        }
+    }
+    return undefined;
+}
+
 /**
  * Best-effort base directory inference for a glob pattern.
  *
@@ -830,7 +903,7 @@ function groupByRoot(filePaths, watchPaths) {
 /** Default page size for prune scroll (smaller than search to reduce connection strain). */
 const PRUNE_SCROLL_PAGE_SIZE = 500;
 /** Max retry attempts for scroll page failures. */
-const SCROLL_RETRY_ATTEMPTS = 3;
+const SCROLL_RETRY_ATTEMPTS = 5;
 /** Base delay for scroll retry backoff (ms). */
 const SCROLL_RETRY_BASE_DELAY_MS = 1000;
 /**
@@ -856,26 +929,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
     }
     throw lastError;
 }
+/** Concurrency limit for fs.access() disk-existence checks. */
+const FS_ACCESS_CONCURRENCY = 100;
 /**
- * Compute the plan for a prune operation.
- * Scrolls all Qdrant points page-by-page with retry/resume,
- * checks each file_path against watch scope + gitignore,
- * returns the list of orphaned point IDs and the plan.
+ * Compute the plan for a prune operation using a filter-first approach.
+ *
+ * 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
+ * 2. Scroll Qdrant points. For each point:
+ *    - No `file_path` → orphan.
+ *    - `file_path` not in `watchedFiles` → orphan (out of watch scope).
+ *    - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
+ *      If missing from disk → orphan (deleted file still in index).
+ * 3. Batch `fs.access()` checks with bounded concurrency.
  */
 async function computePrunePlan(deps) {
     const { config, vectorStore, gitignoreFilter, logger } = deps;
     if (!vectorStore) {
         throw new Error('vectorStore is required for prune scope');
     }
+    // Step 1: Build set of watched files from filesystem
+    const isGitignored = createIsGitignored(gitignoreFilter);
+    const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
+    const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
+    logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
+    // Step 2: Scroll Qdrant and classify points
     const seenFiles = new Set();
     const orphanedFiles = new Set();
+    const diskCheckFiles = new Set();
     const orphanedIds = [];
+    // Map from file_path → point IDs, for points needing disk-existence check
+    const diskCheckPointIds = new Map();
     let totalPoints = 0;
     let incomplete = false;
     let cursor;
     try {
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cursor is updated each iteration
-        while (true) {
+        let hasMore = true;
+        while (hasMore) {
             const page = await scrollPageWithRetry(vectorStore, cursor, logger);
             for (const point of page.points) {
                 totalPoints++;
@@ -884,32 +973,62 @@ async function computePrunePlan(deps) {
                     orphanedIds.push(point.id);
                     continue;
                 }
-                if (seenFiles.has(filePath)) {
-                    if (orphanedFiles.has(filePath)) {
+                const normalized = normalizePathCase(normalizeSlashes(filePath));
+                if (seenFiles.has(normalized)) {
+                    if (orphanedFiles.has(normalized)) {
                         orphanedIds.push(point.id);
                     }
+                    else if (diskCheckFiles.has(normalized)) {
+                        // Already queued for disk check — track its ID
+                        const ids = diskCheckPointIds.get(normalized);
+                        if (ids)
+                            ids.push(point.id);
+                    }
                     continue;
                 }
-                seenFiles.add(filePath);
-                const watched = isPathWatched(filePath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
-                const gitignored = gitignoreFilter
-                    ? gitignoreFilter.isIgnored(filePath)
-                    : false;
-                if (!watched || gitignored) {
-                    orphanedFiles.add(filePath);
+                seenFiles.add(normalized);
+                if (!watchedFiles.has(normalized)) {
+                    // Not in watch scope → definitely orphaned
+                    orphanedFiles.add(normalized);
                     orphanedIds.push(point.id);
                 }
+                else {
+                    // In watch scope → needs disk-existence check
+                    diskCheckFiles.add(normalized);
+                    diskCheckPointIds.set(normalized, [point.id]);
+                }
+            }
+            // Progress logging every 100K points
+            if (totalPoints > 0 && totalPoints % 100000 === 0) {
+                logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
+            }
+            if (!page.nextCursor) {
+                hasMore = false;
+            }
+            else {
+                cursor = page.nextCursor;
             }
-            if (!page.nextCursor)
-                break;
-            cursor = page.nextCursor;
         }
     }
     catch (error) {
-        // All retries exhausted - return partial results
         logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
         incomplete = true;
     }
+    // Step 3: Batch fs.access() checks for files in watch scope
+    const filesToCheck = [...diskCheckFiles];
+    await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
+        try {
+            await access(filePath);
+        }
+        catch {
+            // File doesn't exist on disk → orphaned
+            orphanedFiles.add(filePath);
+            const pointIds = diskCheckPointIds.get(filePath);
+            if (pointIds) {
+                orphanedIds.push(...pointIds);
+            }
+        }
+    });
     const orphanedFileList = [...orphanedFiles];
     const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
     return {
@@ -967,6 +1086,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
+        // Abort live prune when plan is incomplete — never silently delete partial data
+        if (plan.incomplete) {
+            deps.queue?.resume();
+            logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
+            return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
+        }
         // Execute prune
         reindexTracker?.start(scope);
         const startTime = Date.now();
@@ -1456,4 +1581,4 @@ const watcherDescriptor = {
 const program = createServiceCli(watcherDescriptor);
 program.parse();
 
-export { CONFIG_WATCH_VALID_SCOPES as C, GitignoreFilter as G, VALID_REINDEX_SCOPES as V, normalizeSlashes as a, startFromConfig as b, executeReindex as e, getWatchRootBases as g, isPathWatched as i, loadConfig as l, mergeInferenceRules as m, normalizeError as n, retry as r, sleep as s };
+export { CONFIG_WATCH_VALID_SCOPES as C, GitignoreFilter as G, VALID_REINDEX_SCOPES as V, executeReindex as a, gitAddViaStdin as b, normalizePathCase as c, normalizeSlashes as d, execFileAsync as e, findRootForPath as f, getWatchRootBases as g, startFromConfig as h, isPathWatched as i, loadConfig as l, mergeInferenceRules as m, normalizeError as n, retry as r, sleep as s };

package/dist/index.d.ts

@@ -1252,10 +1252,9 @@ declare class CommitMessageGenerator {
 }
 
 /**
- * @module vcs/VcsManager
- * Per-root VCS manager for git-backed content versioning.
+ * @module vcs/types
+ * Shared types for the VCS subsystem.
  */
-
 /** Metadata for a pending reversion to include in the commit message. */
 interface PendingReversion {
     glob: string;
@@ -1267,6 +1266,12 @@ interface PushError {
     timestamp: string;
     message: string;
 }
+
+/**
+ * @module vcs/VcsManager
+ * Per-root VCS manager for git-backed content versioning.
+ */
+
 /**
  * Per-root VCS manager for git-backed content versioning.
  *
@@ -1283,13 +1288,14 @@ declare class VcsManager {
     readonly remoteUrl: string | undefined;
     private readonly accessToken;
     private readonly logger;
-    private readonly commitMessageGenerator;
+    private readonly commitMessageBuilder;
     private readonly pending;
     private readonly pendingReversions;
     private readonly _pushErrors;
     private readonly squashManager;
     private debounceTimer;
     private commitInFlight;
+    private consecutiveCommitFailures;
     private started;
     private isBaseline;
     private _lastPushTime;
@@ -1297,15 +1303,18 @@ declare class VcsManager {
     get lastPushTime(): string | null;
     get pushErrors(): readonly PushError[];
     /**
-     * Begin accepting file changes. Sets up the debounce mechanism.
-     * Enqueues a baseline commit for pre-existing files if the repo has no commits.
+     * Begin accepting file changes. Sets the manager to started state and
+     * starts the squash manager if configured.
+     *
+     * Baseline mode (commit prefix `"baseline:"`) remains active until
+     * {@link endBaseline} is called by the coordinator after the initial scan.
      */
-    start(): Promise<void>;
+    start(): void;
     /**
-     * If the repo has no commits, enumerate untracked files and feed them
-     * through fileChanged() to create a baseline commit.
+     * Signal that the initial filesystem scan is complete.
+     * Clears the baseline flag so subsequent commits use normal "watcher: batch" messages.
      */
-    private enqueueBaselineIfNeeded;
+    endBaseline(): void;
     /**
      * Add a file to the pending set and reset the debounce timer.
      * If the pending set exceeds maxBatchSize, flush immediately.
@@ -1347,26 +1356,14 @@ declare class VcsManager {
      */
     private resetDebounce;
     /**
-     * Get the cached diff for staged files, for AI context.
+     * Check for and remove stale index.lock files before retrying.
      */
-    private getStagedDiff;
+    private removeStaleLock;
     /**
-     * Build the template commit message (no AI).
-     */
-    private buildTemplateMessage;
-    /**
-     * Build the commit message, using AI when available with template fallback.
-     */
-    private buildCommitMessage;
-    /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     private commitBatch;
-    /**
-     * Push to the configured remote if remoteUrl is set.
-     * On failure: logs the error and records it in pushErrors; never throws.
-     */
-    private pushIfConfigured;
 }
 
 /**
@@ -1386,7 +1383,7 @@ declare class VcsCoordinator {
     /**
      * Start all VcsManager instances.
      */
-    start(): Promise<void>;
+    start(): void;
     /**
      * Route a file event to the correct root's VcsManager.
      *
@@ -1394,6 +1391,15 @@ declare class VcsCoordinator {
      * @param event - The event type: add, change, or unlink.
      */
     onFileChange(filePath: string, event: 'add' | 'change' | 'unlink'): void;
+    /**
+     * Signal that the initial filesystem scan is complete.
+     *
+     * Flushes each manager's debounce buffer before ending baseline mode, ensuring
+     * any files still pending in the VCS debounce are committed with a
+     * `"baseline:"` prefix rather than `"watcher:"`. AI generation is not enabled
+     * until all managers have completed their flush.
+     */
+    onInitialScanComplete(): Promise<void>;
     /**
      * Flush all managers and clean up.
      */
@@ -1539,6 +1545,8 @@ interface FileSystemWatcherOptions {
     contentHashCache?: ContentHashCache;
     /** Optional callback for VCS file change notifications. */
     onVcsFileChange?: (filePath: string, event: 'add' | 'change' | 'unlink') => void;
+    /** Optional callback invoked when the initial filesystem scan completes. */
+    onInitialScanComplete?: () => void | Promise<void>;
 }
 /**
  * Filesystem watcher that maps chokidar events to the processing queue.
@@ -1553,6 +1561,7 @@ declare class FileSystemWatcher {
     private readonly initialScanTracker?;
     private readonly contentHashCache?;
     private readonly onVcsFileChange?;
+    private readonly onInitialScanComplete?;
     private moveCorrelator?;
     private globMatches;
     private watcher;
@@ -1808,6 +1817,12 @@ declare class JeevesWatcher {
      * Start the watcher, API server, and all components.
      */
     start(): Promise<void>;
+    /** Set up logger, VCS, embedding provider, and vector store. */
+    private initInfrastructure;
+    /** Build the processing pipeline: rules, templates, processor, queue, VCS. */
+    private initPipeline;
+    /** Wire up watcher and VCS coordinator, then start all services. */
+    private startServicesAndWatcher;
     /**
      * Gracefully stop all components.
      */