@karmaniverous/jeeves-watcher 0.18.7 → 0.18.9

package/dist/index.js

@@ -1,13 +1,15 @@
 import { join, dirname, resolve, relative, extname, basename, isAbsolute, sep } from 'node:path';
 import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, createConfigApplyHandler, substituteEnvVars, DEFAULT_PORTS, getBindAddress, postJson, fetchJson, getPackageVersion } from '@karmaniverous/jeeves';
-import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsAuthorConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
+import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsRetentionConfigSchema, vcsAuthorConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
 export { apiConfigSchema, configWatchConfigSchema, embeddingConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema, loggingConfigSchema, vectorStoreConfigSchema, watchConfigSchema } from '@karmaniverous/jeeves-watcher-core';
 import Fastify from 'fastify';
-import { readdir, stat, readFile, writeFile, mkdir, appendFile, access, rm } from 'node:fs/promises';
+import { readdir, stat, access, readFile, writeFile, mkdir, appendFile, rm } from 'node:fs/promises';
 import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
 import { existsSync, statSync, readFileSync, readdirSync, mkdirSync, writeFileSync, rmSync, renameSync } from 'node:fs';
 import ignore from 'ignore';
 import picomatch from 'picomatch';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import { z, ZodError } from 'zod';
@@ -21,8 +23,6 @@ import { toMarkdown } from 'mdast-util-to-markdown';
 import rehypeParse from 'rehype-parse';
 import { unified } from 'unified';
 import yaml from 'js-yaml';
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
 import crypto from 'crypto';
 import { packageDirectorySync } from 'package-directory';
 import { homedir } from 'node:os';
@@ -421,6 +421,77 @@ async function retry(fn, options) {
     throw lastError;
 }
 
+/**
+ * @module vcs/gitExec
+ * Shared git execution utilities.
+ */
+/** Promisified execFile for git commands. */
+const execFileAsync = promisify(execFile);
+/**
+ * Stage files via stdin to avoid ENAMETOOLONG on Windows.
+ *
+ * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
+ * through stdin (NUL-delimited) instead of passed as command-line
+ * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
+ * limit that triggers ENAMETOOLONG when batches contain many files
+ * with long absolute paths.
+ *
+ * @param files - Absolute paths of files to stage.
+ * @param cwd   - Repository root (working directory for git).
+ */
+function gitAddViaStdin(files, cwd) {
+    return new Promise((resolve, reject) => {
+        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
+            if (error)
+                reject(error);
+            else
+                resolve();
+        });
+        if (!child.stdin) {
+            reject(new Error('Failed to open stdin for git add'));
+            return;
+        }
+        // Suppress EPIPE — expected if git exits before we finish writing
+        child.stdin.on('error', (err) => {
+            if (err.code !== 'EPIPE') {
+                reject(err);
+            }
+        });
+        child.stdin.end(files.join('\0'));
+    });
+}
+/**
+ * Normalize a path for case-insensitive comparison on Windows.
+ * On Windows, lowercases the entire path; on other platforms, returns as-is.
+ *
+ * @param p - The path string to normalize.
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The normalized path.
+ */
+function normalizePathCase(p, platform = process.platform) {
+    return platform === 'win32' ? p.toLowerCase() : p;
+}
+/**
+ * Find the longest-prefix-match root for a normalized path.
+ * Roots must be sorted longest-first for correct nested matching.
+ *
+ * @param roots - Root paths sorted longest-first.
+ * @param normalizedPath - Normalized absolute path (forward slashes).
+ * @param platform - The platform to check against (default: `process.platform`).
+ * @returns The matching root, or undefined.
+ */
+function findRootForPath(roots, normalizedPath, platform = process.platform) {
+    const comparePath = normalizePathCase(normalizedPath, platform);
+    for (const root of roots) {
+        const compareRoot = normalizePathCase(root, platform);
+        if (comparePath === compareRoot ||
+            comparePath.startsWith(compareRoot + '/')) {
+            return root;
+        }
+    }
+    return undefined;
+}
+
 /**
  * Best-effort base directory inference for a glob pattern.
  *
@@ -638,7 +709,7 @@ function groupByRoot(filePaths, watchPaths) {
 /** Default page size for prune scroll (smaller than search to reduce connection strain). */
 const PRUNE_SCROLL_PAGE_SIZE = 500;
 /** Max retry attempts for scroll page failures. */
-const SCROLL_RETRY_ATTEMPTS = 3;
+const SCROLL_RETRY_ATTEMPTS = 5;
 /** Base delay for scroll retry backoff (ms). */
 const SCROLL_RETRY_BASE_DELAY_MS = 1000;
 /**
@@ -664,26 +735,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
     }
     throw lastError;
 }
+/** Concurrency limit for fs.access() disk-existence checks. */
+const FS_ACCESS_CONCURRENCY = 100;
 /**
- * Compute the plan for a prune operation.
- * Scrolls all Qdrant points page-by-page with retry/resume,
- * checks each file_path against watch scope + gitignore,
- * returns the list of orphaned point IDs and the plan.
+ * Compute the plan for a prune operation using a filter-first approach.
+ *
+ * 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
+ * 2. Scroll Qdrant points. For each point:
+ *    - No `file_path` → orphan.
+ *    - `file_path` not in `watchedFiles` → orphan (out of watch scope).
+ *    - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
+ *      If missing from disk → orphan (deleted file still in index).
+ * 3. Batch `fs.access()` checks with bounded concurrency.
  */
 async function computePrunePlan(deps) {
     const { config, vectorStore, gitignoreFilter, logger } = deps;
     if (!vectorStore) {
         throw new Error('vectorStore is required for prune scope');
     }
+    // Step 1: Build set of watched files from filesystem
+    const isGitignored = createIsGitignored(gitignoreFilter);
+    const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
+    const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
+    logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
+    // Step 2: Scroll Qdrant and classify points
     const seenFiles = new Set();
     const orphanedFiles = new Set();
+    const diskCheckFiles = new Set();
     const orphanedIds = [];
+    // Map from file_path → point IDs, for points needing disk-existence check
+    const diskCheckPointIds = new Map();
     let totalPoints = 0;
     let incomplete = false;
     let cursor;
     try {
-        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cursor is updated each iteration
-        while (true) {
+        let hasMore = true;
+        while (hasMore) {
             const page = await scrollPageWithRetry(vectorStore, cursor, logger);
             for (const point of page.points) {
                 totalPoints++;
@@ -692,32 +779,62 @@ async function computePrunePlan(deps) {
                     orphanedIds.push(point.id);
                     continue;
                 }
-                if (seenFiles.has(filePath)) {
-                    if (orphanedFiles.has(filePath)) {
+                const normalized = normalizePathCase(normalizeSlashes(filePath));
+                if (seenFiles.has(normalized)) {
+                    if (orphanedFiles.has(normalized)) {
                         orphanedIds.push(point.id);
                     }
+                    else if (diskCheckFiles.has(normalized)) {
+                        // Already queued for disk check — track its ID
+                        const ids = diskCheckPointIds.get(normalized);
+                        if (ids)
+                            ids.push(point.id);
+                    }
                     continue;
                 }
-                seenFiles.add(filePath);
-                const watched = isPathWatched(filePath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
-                const gitignored = gitignoreFilter
-                    ? gitignoreFilter.isIgnored(filePath)
-                    : false;
-                if (!watched || gitignored) {
-                    orphanedFiles.add(filePath);
+                seenFiles.add(normalized);
+                if (!watchedFiles.has(normalized)) {
+                    // Not in watch scope → definitely orphaned
+                    orphanedFiles.add(normalized);
                     orphanedIds.push(point.id);
                 }
+                else {
+                    // In watch scope → needs disk-existence check
+                    diskCheckFiles.add(normalized);
+                    diskCheckPointIds.set(normalized, [point.id]);
+                }
+            }
+            // Progress logging every 100K points
+            if (totalPoints > 0 && totalPoints % 100000 === 0) {
+                logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
+            }
+            if (!page.nextCursor) {
+                hasMore = false;
+            }
+            else {
+                cursor = page.nextCursor;
             }
-            if (!page.nextCursor)
-                break;
-            cursor = page.nextCursor;
         }
     }
     catch (error) {
-        // All retries exhausted - return partial results
         logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
         incomplete = true;
     }
+    // Step 3: Batch fs.access() checks for files in watch scope
+    const filesToCheck = [...diskCheckFiles];
+    await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
+        try {
+            await access(filePath);
+        }
+        catch {
+            // File doesn't exist on disk → orphaned
+            orphanedFiles.add(filePath);
+            const pointIds = diskCheckPointIds.get(filePath);
+            if (pointIds) {
+                orphanedIds.push(...pointIds);
+            }
+        }
+    });
     const orphanedFileList = [...orphanedFiles];
     const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
     return {
@@ -775,6 +892,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
+        // Abort live prune when plan is incomplete — never silently delete partial data
+        if (plan.incomplete) {
+            deps.queue?.resume();
+            logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
+            return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
+        }
         // Execute prune
         reindexTracker?.start(scope);
         const startTime = Date.now();
@@ -3369,77 +3492,6 @@ function createSearchHandler(deps) {
     }, deps.logger, 'Search');
 }
 
-/**
- * @module vcs/gitExec
- * Shared git execution utilities.
- */
-/** Promisified execFile for git commands. */
-const execFileAsync = promisify(execFile);
-/**
- * Stage files via stdin to avoid ENAMETOOLONG on Windows.
- *
- * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
- * through stdin (NUL-delimited) instead of passed as command-line
- * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
- * limit that triggers ENAMETOOLONG when batches contain many files
- * with long absolute paths.
- *
- * @param files - Absolute paths of files to stage.
- * @param cwd   - Repository root (working directory for git).
- */
-function gitAddViaStdin(files, cwd) {
-    return new Promise((resolve, reject) => {
-        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
-            if (error)
-                reject(error);
-            else
-                resolve();
-        });
-        if (!child.stdin) {
-            reject(new Error('Failed to open stdin for git add'));
-            return;
-        }
-        // Suppress EPIPE — expected if git exits before we finish writing
-        child.stdin.on('error', (err) => {
-            if (err.code !== 'EPIPE') {
-                reject(err);
-            }
-        });
-        child.stdin.end(files.join('\0'));
-    });
-}
-/**
- * Normalize a path for case-insensitive comparison on Windows.
- * On Windows, lowercases the entire path; on other platforms, returns as-is.
- *
- * @param p - The path string to normalize.
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The normalized path.
- */
-function normalizePathCase(p, platform = process.platform) {
-    return platform === 'win32' ? p.toLowerCase() : p;
-}
-/**
- * Find the longest-prefix-match root for a normalized path.
- * Roots must be sorted longest-first for correct nested matching.
- *
- * @param roots - Root paths sorted longest-first.
- * @param normalizedPath - Normalized absolute path (forward slashes).
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The matching root, or undefined.
- */
-function findRootForPath(roots, normalizedPath, platform = process.platform) {
-    const comparePath = normalizePathCase(normalizedPath, platform);
-    for (const root of roots) {
-        const compareRoot = normalizePathCase(root, platform);
-        if (comparePath === compareRoot ||
-            comparePath.startsWith(compareRoot + '/')) {
-            return root;
-        }
-    }
-    return undefined;
-}
-
 /**
  * @module vcs/resolveWatchRoot
  * Resolves which VCS-enabled watch root(s) own a given absolute path or glob.
@@ -4193,6 +4245,91 @@ async function introspectAllHelpers(config, configDir) {
     return result;
 }
 
+/**
+ * @module vcs/CommitMessageBuilder
+ * Builds commit messages — both template-based and AI-generated.
+ */
+/**
+ * Builds commit messages for VCS commits.
+ * Supports template-based messages and AI-generated messages with fallback.
+ */
+class CommitMessageBuilder {
+    rootPath;
+    logger;
+    generator;
+    constructor(rootPath, logger, generator) {
+        this.rootPath = rootPath;
+        this.logger = logger;
+        this.generator = generator;
+    }
+    /**
+     * Build the template commit message (no AI).
+     */
+    buildTemplateMessage(fileCount, isBaseline, pendingReversions) {
+        if (pendingReversions.length === 0) {
+            if (isBaseline) {
+                return `baseline: batch (${String(fileCount)} files)`;
+            }
+            const timestamp = new Date().toISOString();
+            return `watcher: batch ${timestamp} (${String(fileCount)} files)`;
+        }
+        // Count total reverted files across all pending reversions
+        const revertedPaths = new Set(pendingReversions.flatMap((r) => r.paths));
+        const revertedCount = revertedPaths.size;
+        const otherCount = fileCount - revertedCount;
+        // Use the first reversion's glob and commit for the message
+        const { glob, commit } = pendingReversions[0];
+        const shortCommit = commit.slice(0, 7);
+        let message = `revert: ${glob} to ${shortCommit} — restored ${String(revertedCount)} files`;
+        if (otherCount > 0) {
+            message += ` (+ ${String(otherCount)} other changes)`;
+        }
+        return message;
+    }
+    /**
+     * Build the commit message, using AI when available with template fallback.
+     */
+    async buildCommitMessage(files, isBaseline, pendingReversions) {
+        const fileCount = files.length;
+        const templateMessage = this.buildTemplateMessage(fileCount, isBaseline, pendingReversions);
+        if (!this.generator) {
+            return templateMessage;
+        }
+        try {
+            const diffSummary = await this.getStagedDiff();
+            const aiMessage = await this.generator.generate(files, diffSummary);
+            if (!aiMessage) {
+                this.logger.warn('AI commit message generation returned null; using template');
+                return templateMessage;
+            }
+            // For reversions: prefix with revert info + AI description
+            if (pendingReversions.length > 0) {
+                const { glob, commit } = pendingReversions[0];
+                const shortCommit = commit.slice(0, 7);
+                return `revert: ${glob} to ${shortCommit} — ${aiMessage}`;
+            }
+            return aiMessage;
+        }
+        catch (error) {
+            this.logger.warn({ err: normalizeError(error) }, 'AI commit message generation failed; using template');
+            return templateMessage;
+        }
+    }
+    /**
+     * Get the staged diff for AI context.
+     */
+    async getStagedDiff() {
+        try {
+            const { stdout: stat } = await execFileAsync('git', ['diff', '--cached', '--stat'], { cwd: this.rootPath });
+            const { stdout: patch } = await execFileAsync('git', ['diff', '--cached'], { cwd: this.rootPath });
+            return `${stat}\n${patch}`;
+        }
+        catch {
+            return '';
+        }
+    }
+}
+
 /**
  * @module vcs/CommitMessageGenerator
  * Generates AI-powered commit messages via the Anthropic Messages API.
@@ -4819,6 +4956,50 @@ function resolveIgnored(ignored) {
     });
 }
 
+/**
+ * @module vcs/vcsPush
+ * Push logic for VCS commits.
+ */
+/**
+ * Push to the configured remote if remoteUrl is set.
+ *
+ * On failure: logs the error, appends to pushErrors, and returns undefined.
+ * Never throws.
+ *
+ * @param rootPath - Absolute path of the git repository.
+ * @param remoteUrl - Remote URL to push to, or undefined to skip.
+ * @param accessToken - Optional token for HTTPS authentication.
+ * @param pushErrors - Mutable array where push errors are recorded.
+ * @param logger - Logger instance.
+ * @returns ISO timestamp of a successful push, or undefined if skipped/failed.
+ */
+async function pushToRemote(rootPath, remoteUrl, accessToken, pushErrors, logger) {
+    if (!remoteUrl)
+        return undefined;
+    try {
+        // Build the authenticated URL if a token is available
+        const pushUrl = accessToken
+            ? remoteUrl.replace(/^https:\/\//, `https://${encodeURIComponent(accessToken)}@`)
+            : remoteUrl;
+        await execFileAsync('git', ['push', pushUrl, 'HEAD'], {
+            cwd: rootPath,
+            env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
+        });
+        const timestamp = new Date().toISOString();
+        logger.info({ root: rootPath, remote: remoteUrl }, 'VCS push succeeded');
+        return timestamp;
+    }
+    catch (error) {
+        const pushError = {
+            timestamp: new Date().toISOString(),
+            message: normalizeError(error).message,
+        };
+        pushErrors.push(pushError);
+        logger.error({ root: rootPath, err: normalizeError(error) }, 'VCS push failed');
+        return undefined;
+    }
+}
+
 /**
  * @module vcs/VcsManager
  * Per-root VCS manager for git-backed content versioning.
@@ -4839,23 +5020,24 @@ class VcsManager {
     remoteUrl;
     accessToken;
     logger;
-    commitMessageGenerator;
+    commitMessageBuilder;
     pending = new Set();
     pendingReversions = [];
     _pushErrors = [];
     squashManager;
     debounceTimer;
     commitInFlight = Promise.resolve();
+    consecutiveCommitFailures = 0;
     started = false;
-    isBaseline = false;
+    isBaseline = true;
     _lastPushTime = null;
     constructor(rootPath, config, logger, commitMessageGenerator, remoteUrl, accessToken) {
         this.rootPath = rootPath;
         this.config = config;
         this.logger = logger;
-        this.commitMessageGenerator = commitMessageGenerator;
         this.remoteUrl = remoteUrl;
         this.accessToken = accessToken;
+        this.commitMessageBuilder = new CommitMessageBuilder(rootPath, logger, commitMessageGenerator);
         if (config.retention) {
             this.squashManager = new SquashManager(rootPath, config.retention, logger, remoteUrl, accessToken);
         }
@@ -4867,48 +5049,23 @@ class VcsManager {
         return this._pushErrors;
     }
     /**
-     * Begin accepting file changes. Sets up the debounce mechanism.
-     * Enqueues a baseline commit for pre-existing files if the repo has no commits.
+     * Begin accepting file changes. Sets the manager to started state and
+     * starts the squash manager if configured.
+     *
+     * Baseline mode (commit prefix `"baseline:"`) remains active until
+     * {@link endBaseline} is called by the coordinator after the initial scan.
      */
-    async start() {
+    start() {
         this.started = true;
         this.squashManager?.start();
         this.logger.info({ root: this.rootPath }, 'VcsManager started');
-        await this.enqueueBaselineIfNeeded();
     }
     /**
-     * If the repo has no commits, enumerate untracked files and feed them
-     * through fileChanged() to create a baseline commit.
+     * Signal that the initial filesystem scan is complete.
+     * Clears the baseline flag so subsequent commits use normal "watcher: batch" messages.
      */
-    async enqueueBaselineIfNeeded() {
-        try {
-            await execFileAsync('git', ['rev-parse', 'HEAD'], {
-                cwd: this.rootPath,
-            });
-            // HEAD exists — repo already has commits
-            return;
-        }
-        catch {
-            // No commits yet — proceed with baseline
-        }
-        try {
-            const { stdout } = await execFileAsync('git', ['ls-files', '--others', '--exclude-standard', '-z'], { cwd: this.rootPath });
-            const files = stdout
-                .split('\0')
-                .map((line) => line.trim())
-                .filter((line) => line.length > 0);
-            if (files.length === 0)
-                return;
-            this.logger.info({ root: this.rootPath, fileCount: files.length }, 'Enqueuing baseline commit for pre-existing files');
-            this.isBaseline = true;
-            const absolutePaths = files.map((f) => this.rootPath + '/' + f);
-            for (const filePath of absolutePaths) {
-                this.fileChanged(filePath);
-            }
-        }
-        catch (error) {
-            this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Failed to enumerate untracked files for baseline commit');
-        }
+    endBaseline() {
+        this.isBaseline = false;
     }
     /**
      * Add a file to the pending set and reset the debounce timer.
@@ -4919,6 +5076,11 @@ class VcsManager {
     fileChanged(filePath) {
         if (!this.started)
             return;
+        // Reset circuit breaker on new file change so the system can recover
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.info({ root: this.rootPath }, 'VCS circuit breaker reset — new file change received, retrying commits');
+            this.consecutiveCommitFailures = 0;
+        }
         this.pending.add(filePath);
         if (this.pending.size >= this.config.maxBatchSize) {
             this.clearDebounce();
@@ -5006,97 +5168,61 @@ class VcsManager {
         }, this.config.commitDebounceMs);
     }
     /**
-     * Get the cached diff for staged files, for AI context.
+     * Check for and remove stale index.lock files before retrying.
      */
-    async getStagedDiff() {
-        try {
-            const { stdout: stat } = await execFileAsync('git', ['diff', '--cached', '--stat'], { cwd: this.rootPath });
-            const { stdout: patch } = await execFileAsync('git', ['diff', '--cached'], { cwd: this.rootPath });
-            return `${stat}\n${patch}`;
-        }
-        catch {
-            return '';
-        }
-    }
-    /**
-     * Build the template commit message (no AI).
-     */
-    buildTemplateMessage(fileCount) {
-        if (this.pendingReversions.length === 0) {
-            if (this.isBaseline) {
-                return `baseline: batch (${String(fileCount)} files)`;
-            }
-            const timestamp = new Date().toISOString();
-            return `watcher: batch ${timestamp} (${String(fileCount)} files)`;
-        }
-        // Count total reverted files across all pending reversions
-        const revertedPaths = new Set(this.pendingReversions.flatMap((r) => r.paths));
-        const revertedCount = revertedPaths.size;
-        const otherCount = fileCount - revertedCount;
-        // Use the first reversion's glob and commit for the message
-        const { glob, commit } = this.pendingReversions[0];
-        const shortCommit = commit.slice(0, 7);
-        let message = `revert: ${glob} to ${shortCommit} — restored ${String(revertedCount)} files`;
-        if (otherCount > 0) {
-            message += ` (+ ${String(otherCount)} other changes)`;
-        }
-        return message;
-    }
-    /**
-     * Build the commit message, using AI when available with template fallback.
-     */
-    async buildCommitMessage(files) {
-        const fileCount = files.length;
-        const templateMessage = this.buildTemplateMessage(fileCount);
-        if (!this.commitMessageGenerator) {
-            return templateMessage;
-        }
+    async removeStaleLock() {
+        const lockPath = join(this.rootPath, '.git', 'index.lock');
         try {
-            const diffSummary = await this.getStagedDiff();
-            const aiMessage = await this.commitMessageGenerator.generate(files, diffSummary);
-            if (!aiMessage) {
-                this.logger.warn('AI commit message generation returned null; using template');
-                return templateMessage;
+            const lockStat = await stat(lockPath);
+            const ageMs = Date.now() - lockStat.mtimeMs;
+            if (ageMs > this.config.staleLockThresholdMs) {
+                await rm(lockPath, { force: true });
+                this.logger.warn({ root: this.rootPath, ageMs }, 'Removed stale index.lock');
             }
-            // For reversions: prefix with revert info + AI description
-            if (this.pendingReversions.length > 0) {
-                const { glob, commit } = this.pendingReversions[0];
-                const shortCommit = commit.slice(0, 7);
-                return `revert: ${glob} to ${shortCommit} — ${aiMessage}`;
-            }
-            return aiMessage;
         }
         catch (error) {
-            this.logger.warn({ err: normalizeError(error) }, 'AI commit message generation failed; using template');
-            return templateMessage;
+            const code = error.code;
+            if (code !== 'ENOENT') {
+                this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Unable to remove stale index.lock');
+            }
         }
     }
     /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     async commitBatch(files) {
         if (files.length === 0)
             return;
+        // Circuit breaker: skip if too many consecutive failures
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.error({
+                root: this.rootPath,
+                consecutiveFailures: this.consecutiveCommitFailures,
+                discardedFiles: files.length,
+            }, 'VCS circuit breaker tripped — pending files discarded until next successful commit');
+            return;
+        }
         try {
+            // Check for stale lock before first attempt
+            await this.removeStaleLock();
             await retry(async (attempt) => {
                 await gitAddViaStdin(files, this.rootPath);
-                // Build message after staging so getStagedDiff can see the changes
-                // Skip AI for baselines and retries — use template directly
+                // Build message after staging so getStagedDiff can see the changes.
+                // Skip AI for baselines and retries — use template directly.
                 const message = this.isBaseline || attempt > 1
-                    ? this.buildTemplateMessage(files.length)
-                    : await this.buildCommitMessage(files);
+                    ? this.commitMessageBuilder.buildTemplateMessage(files.length, this.isBaseline, this.pendingReversions)
+                    : await this.commitMessageBuilder.buildCommitMessage(files, this.isBaseline, this.pendingReversions);
                 this.pendingReversions.length = 0;
-                // Only clear baseline flag when all baseline files have been committed
-                if (this.pending.size === 0) {
-                    this.isBaseline = false;
-                }
                 await execFileAsync('git', ['commit', '-m', message], {
                     cwd: this.rootPath,
                 });
                 const { stdout: hashOut } = await execFileAsync('git', ['rev-parse', '--short', 'HEAD'], { cwd: this.rootPath });
                 const hash = hashOut.trim();
                 this.logger.info({ root: this.rootPath, hash, fileCount: files.length }, 'VCS commit created');
-                await this.pushIfConfigured();
+                const pushTime = await pushToRemote(this.rootPath, this.remoteUrl, this.accessToken, this._pushErrors, this.logger);
+                if (pushTime)
+                    this._lastPushTime = pushTime;
             }, {
                 attempts: 4,
                 baseDelayMs: 500,
@@ -5105,43 +5231,25 @@ class VcsManager {
                     this.logger.warn({ root: this.rootPath, attempt, delay: delayMs }, 'index.lock contention, retrying');
                 },
             });
+            // Success — reset circuit breaker
+            this.consecutiveCommitFailures = 0;
         }
         catch (error) {
-            for (const f of files) {
+            this.consecutiveCommitFailures++;
+            // Re-queue cap: only re-queue up to maxBatchSize files
+            const cap = this.config.maxBatchSize;
+            const toRequeue = files.slice(0, cap);
+            const overflow = files.length - toRequeue.length;
+            for (const f of toRequeue) {
                 this.pending.add(f);
             }
-            this.logger.warn({ root: this.rootPath, fileCount: files.length }, 'Re-queued files after commit failure');
+            if (overflow > 0) {
+                this.logger.warn({ root: this.rootPath, overflow, cap }, 'Re-queue cap exceeded — discarded overflow files');
+            }
+            this.logger.warn({ root: this.rootPath, fileCount: toRequeue.length }, 'Re-queued files after commit failure');
             this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS commit failed');
         }
     }
-    /**
-     * Push to the configured remote if remoteUrl is set.
-     * On failure: logs the error and records it in pushErrors; never throws.
-     */
-    async pushIfConfigured() {
-        if (!this.remoteUrl)
-            return;
-        try {
-            // Build the authenticated URL if a token is available
-            const pushUrl = this.accessToken
-                ? this.remoteUrl.replace(/^https:\/\//, `https://${encodeURIComponent(this.accessToken)}@`)
-                : this.remoteUrl;
-            await execFileAsync('git', ['push', pushUrl, 'HEAD'], {
-                cwd: this.rootPath,
-                env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
-            });
-            this._lastPushTime = new Date().toISOString();
-            this.logger.info({ root: this.rootPath, remote: this.remoteUrl }, 'VCS push succeeded');
-        }
-        catch (error) {
-            const pushError = {
-                timestamp: new Date().toISOString(),
-                message: normalizeError(error).message,
-            };
-            this._pushErrors.push(pushError);
-            this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS push failed');
-        }
-    }
 }
 
 /**
@@ -5170,10 +5278,14 @@ class VcsCoordinator {
             // Deduplicate: skip if another glob already resolved to this root.
             if (this.managers.has(rootKey))
                 continue;
+            const mergedRetention = entry.vcs?.retention ??
+                config.vcs.retention ??
+                vcsRetentionConfigSchema.parse({});
             const mergedConfig = {
                 ...config.vcs,
                 ...entry.vcs,
                 enabled: true,
+                retention: mergedRetention,
             };
             // Create CommitMessageGenerator if AI commit messages are configured
             const cmConfig = mergedConfig.commitMessage;
@@ -5197,12 +5309,10 @@ class VcsCoordinator {
     /**
      * Start all VcsManager instances.
      */
-    async start() {
-        const startPromises = [];
+    start() {
         this.managers.forEach((manager) => {
-            startPromises.push(manager.start());
+            manager.start();
         });
-        await Promise.all(startPromises);
         this.logger.info({ rootCount: this.managers.size }, 'VcsCoordinator started');
     }
     /**
@@ -5223,6 +5333,21 @@ class VcsCoordinator {
             manager.fileChanged(normalizedPath);
         }
     }
+    /**
+     * Signal that the initial filesystem scan is complete.
+     *
+     * Flushes each manager's debounce buffer before ending baseline mode, ensuring
+     * any files still pending in the VCS debounce are committed with a
+     * `"baseline:"` prefix rather than `"watcher:"`. AI generation is not enabled
+     * until all managers have completed their flush.
+     */
+    async onInitialScanComplete() {
+        for (const manager of this.managers.values()) {
+            await manager.flush();
+            manager.endBaseline();
+        }
+        this.logger.debug('VcsCoordinator: initial scan complete, baseline ended');
+    }
     /**
      * Flush all managers and clean up.
      */
@@ -5265,6 +5390,68 @@ class VcsCoordinator {
     }
 }
 
+/**
+ * @module app/watcherFactory
+ * Factory helpers for creating and rebuilding the filesystem watcher.
+ * Extracted from initialization.ts to follow SRP.
+ */
+/**
+ * Create file system watcher with gitignore filtering.
+ */
+function createWatcher(config, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
+    const respectGitignore = config.watch.respectGitignore ?? true;
+    const gitignoreFilter = respectGitignore
+        ? new GitignoreFilter(extractWatchPathStrings(config.watch.paths))
+        : undefined;
+    const watcher = factories.createFileSystemWatcher(config.watch, queue, processor, logger, {
+        maxRetries: config.maxRetries ?? runtimeOptions.maxRetries,
+        maxBackoffMs: config.maxBackoffMs ?? runtimeOptions.maxBackoffMs,
+        onFatalError: runtimeOptions.onFatalError,
+        gitignoreFilter,
+        initialScanTracker,
+        contentHashCache,
+        onVcsFileChange,
+        onInitialScanComplete,
+    });
+    return { watcher, gitignoreFilter };
+}
+/**
+ * Tear down and rebuild the filesystem watcher with new config.
+ * Falls back to the old watcher if the new one fails to start.
+ */
+async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
+    try {
+        await oldState.watcher.stop();
+        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete);
+        newWatcher.start();
+        logger.info('Filesystem watcher rebuilt successfully');
+        return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
+    }
+    catch (error) {
+        logger.error({ err: normalizeError(error) }, 'Failed to rebuild watcher, restoring previous');
+        try {
+            oldState.watcher.start();
+        }
+        catch (restartError) {
+            logger.error({ err: normalizeError(restartError) }, 'Failed to restart previous watcher');
+        }
+        return oldState;
+    }
+}
+/**
+ * Check whether watch-relevant config fields changed between old and new config.
+ */
+function watchConfigChanged(oldConfig, newConfig) {
+    return (JSON.stringify(oldConfig.watch.paths) !==
+        JSON.stringify(newConfig.watch.paths) ||
+        JSON.stringify(oldConfig.watch.ignored) !==
+            JSON.stringify(newConfig.watch.ignored) ||
+        JSON.stringify(oldConfig.watch.moveDetection) !==
+            JSON.stringify(newConfig.watch.moveDetection) ||
+        (oldConfig.watch.respectGitignore ?? true) !==
+            (newConfig.watch.respectGitignore ?? true));
+}
+
 /**
  * @module app/initialization
  * Initialization helpers for JeevesWatcher. Extracted to follow SRP.
@@ -5329,25 +5516,6 @@ function createProcessorConfig(config, configDir, customMapLib) {
         globalSchemas: config.schemas,
     };
 }
-/**
- * Create file system watcher with gitignore filtering.
- */
-function createWatcher(config, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange) {
-    const respectGitignore = config.watch.respectGitignore ?? true;
-    const gitignoreFilter = respectGitignore
-        ? new GitignoreFilter(extractWatchPathStrings(config.watch.paths))
-        : undefined;
-    const watcher = factories.createFileSystemWatcher(config.watch, queue, processor, logger, {
-        maxRetries: config.maxRetries ?? runtimeOptions.maxRetries,
-        maxBackoffMs: config.maxBackoffMs ?? runtimeOptions.maxBackoffMs,
-        onFatalError: runtimeOptions.onFatalError,
-        gitignoreFilter,
-        initialScanTracker,
-        contentHashCache,
-        onVcsFileChange,
-    });
-    return { watcher, gitignoreFilter };
-}
 /**
  * Introspect all helpers for merged document.
  */
@@ -5363,42 +5531,6 @@ async function introspectHelpers(config, configDir) {
 function getConfigDir(configPath) {
     return configPath ? dirname(configPath) : '.';
 }
-/**
- * Tear down and rebuild the filesystem watcher with new config.
- * Falls back to the old watcher if the new one fails to start.
- */
-async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache) {
-    try {
-        await oldState.watcher.stop();
-        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache);
-        newWatcher.start();
-        logger.info('Filesystem watcher rebuilt successfully');
-        return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
-    }
-    catch (error) {
-        logger.error({ err: normalizeError(error) }, 'Failed to rebuild watcher, restoring previous');
-        try {
-            oldState.watcher.start();
-        }
-        catch (restartError) {
-            logger.error({ err: normalizeError(restartError) }, 'Failed to restart previous watcher');
-        }
-        return oldState;
-    }
-}
-/**
- * Check whether watch-relevant config fields changed between old and new config.
- */
-function watchConfigChanged(oldConfig, newConfig) {
-    return (JSON.stringify(oldConfig.watch.paths) !==
-        JSON.stringify(newConfig.watch.paths) ||
-        JSON.stringify(oldConfig.watch.ignored) !==
-            JSON.stringify(newConfig.watch.ignored) ||
-        JSON.stringify(oldConfig.watch.moveDetection) !==
-            JSON.stringify(newConfig.watch.moveDetection) ||
-        (oldConfig.watch.respectGitignore ?? true) !==
-            (newConfig.watch.respectGitignore ?? true));
-}
 /**
  * Resolve package version from nearest package.json.
  */
@@ -6431,7 +6563,7 @@ async function reloadConfig(state, deps) {
         // Rebuild filesystem watcher if watch config changed
         if (watchConfigChanged(oldConfig, newConfig)) {
             logger.info('Watch config changed, rebuilding filesystem watcher...');
-            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache);
+            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache, deps.onVcsFileChange, deps.onInitialScanComplete);
             state.watcher = newState.watcher;
             state.gitignoreFilter = newState.gitignoreFilter;
         }
@@ -8470,6 +8602,37 @@ class MoveCorrelator {
     }
 }
 
+/**
+ * @module watcher/ScanStats
+ * Tracks initial filesystem scan statistics for diagnostics.
+ */
+/**
+ * Mutable counter bag updated during the chokidar initial scan.
+ * Extracted from the watcher start() method for clarity.
+ */
+class ScanStats {
+    total = 0;
+    matched = 0;
+    globRejected = 0;
+    gitignored = 0;
+    byRoot;
+    constructor(roots) {
+        this.byRoot = Object.fromEntries(roots.map((r) => [r, 0]));
+    }
+    /**
+     * Attribute a newly discovered file to its root for per-root diagnostics.
+     */
+    classifyByRoot(path) {
+        const normalized = path.replace(/\\/g, '/').toLowerCase();
+        for (const root of Object.keys(this.byRoot)) {
+            if (normalized.startsWith(`${root}/`) || normalized === root) {
+                this.byRoot[root] = (this.byRoot[root] ?? 0) + 1;
+                break;
+            }
+        }
+    }
+}
+
 /**
  * @module watcher
  * Filesystem watcher wrapping chokidar. I/O: watches files/directories for add/change/unlink events, enqueues to processing queue.
@@ -8487,6 +8650,7 @@ class FileSystemWatcher {
     initialScanTracker;
     contentHashCache;
     onVcsFileChange;
+    onInitialScanComplete;
     moveCorrelator;
     globMatches;
     watcher;
@@ -8508,6 +8672,7 @@ class FileSystemWatcher {
         this.initialScanTracker = options.initialScanTracker;
         this.contentHashCache = options.contentHashCache;
         this.onVcsFileChange = options.onVcsFileChange;
+        this.onInitialScanComplete = options.onInitialScanComplete;
         this.globMatches = () => true;
         const healthOptions = {
             maxRetries: options.maxRetries,
@@ -8535,24 +8700,9 @@ class FileSystemWatcher {
             ? resolveIgnored(this.config.ignored)
             : undefined;
         // Track initial scan statistics per root for diagnostics.
-        const scanStats = {
-            total: 0,
-            matched: 0,
-            globRejected: 0,
-            gitignored: 0,
-            byRoot: Object.fromEntries(roots.map((r) => [r, 0])),
-        };
+        const scanStats = new ScanStats(roots);
         let initialScanComplete = false;
         this.initialScanTracker?.start();
-        const classifyByRoot = (path) => {
-            const normalized = path.replace(/\\/g, '/').toLowerCase();
-            for (const root of roots) {
-                if (normalized.startsWith(root + '/') || normalized === root) {
-                    scanStats.byRoot[root] = (scanStats.byRoot[root] ?? 0) + 1;
-                    break;
-                }
-            }
-        };
         // Create move correlator if move detection is configured and cache is available.
         const moveConfig = this.config.moveDetection;
         if (moveConfig?.enabled && this.contentHashCache) {
@@ -8585,7 +8735,7 @@ class FileSystemWatcher {
         this.watcher.on('add', (path) => {
             if (!initialScanComplete) {
                 scanStats.total++;
-                classifyByRoot(path);
+                scanStats.classifyByRoot(path);
             }
             this.handleGitignoreChange(path);
             if (!this.globMatches(path)) {
@@ -8606,8 +8756,7 @@ class FileSystemWatcher {
                 this.initialScanTracker?.incrementEnqueued();
             }
             this.logger.debug({ path }, 'File added');
-            if (initialScanComplete)
-                this.onVcsFileChange?.(path, 'add');
+            this.onVcsFileChange?.(path, 'add');
             if (correlator && initialScanComplete) {
                 void correlator.handleAdd(path);
             }
@@ -8646,6 +8795,7 @@ class FileSystemWatcher {
             initialScanComplete = true;
             this.initialScanTracker?.complete();
             this.logger.info({ scanStats }, 'Initial scan complete');
+            void this.onInitialScanComplete?.();
         });
         this.watcher.on('error', (error) => {
             this.logger.error({ err: normalizeError(error) }, 'Watcher error');
@@ -8919,6 +9069,12 @@ class JeevesWatcher {
      * Start the watcher, API server, and all components.
      */
     async start() {
+        await this.initInfrastructure();
+        await this.initPipeline();
+        await this.startServicesAndWatcher();
+    }
+    /** Set up logger, VCS, embedding provider, and vector store. */
+    async initInfrastructure() {
         const logger = this.factories.createLogger(this.config.logging);
         this.logger = logger;
         await initVcs(this.config, logger);
@@ -8928,6 +9084,10 @@ class JeevesWatcher {
         if (this.config.search?.hybrid?.enabled) {
             await vectorStore.ensureTextIndex('chunk_text');
         }
+    }
+    /** Build the processing pipeline: rules, templates, processor, queue, VCS. */
+    async initPipeline() {
+        const logger = this.logger;
         const compiledRules = this.factories.compileRules(this.config.inferenceRules ?? []);
         const configDir = getConfigDir(this.configPath);
         const { templateEngine, customMapLib } = await buildTemplateEngineAndCustomMapLib(this.config, configDir);
@@ -8937,36 +9097,39 @@ class JeevesWatcher {
         this.issuesManager = new IssuesManager(stateDir, logger);
         this.valuesManager = new ValuesManager(stateDir, logger);
         this.enrichmentStore = new EnrichmentStore(stateDir, logger);
-        const enrichmentStore = this.enrichmentStore;
         this.contentHashCache = new ContentHashCache();
-        const contentHashCache = this.contentHashCache;
-        const processor = this.factories.createDocumentProcessor({
+        this.processor = this.factories.createDocumentProcessor({
             config: processorConfig,
-            embeddingProvider,
-            vectorStore,
+            embeddingProvider: this.embeddingProvider,
+            vectorStore: this.vectorStore,
             compiledRules,
             logger,
             templateEngine,
-            enrichmentStore,
+            enrichmentStore: this.enrichmentStore,
             issuesManager: this.issuesManager,
             valuesManager: this.valuesManager,
-            contentHashCache,
+            contentHashCache: this.contentHashCache,
         });
-        this.processor = processor;
         this.queue = this.factories.createEventQueue({
             debounceMs: this.config.watch.debounceMs ?? 2000,
             concurrency: this.config.embedding.concurrency ?? 5,
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
-        const vcsCoordinator = new VcsCoordinator(this.config, logger);
-        this.vcsCoordinator = vcsCoordinator;
-        const { watcher, gitignoreFilter } = createWatcher(this.config, this.factories, this.queue, processor, logger, this.runtimeOptions, this.initialScanTracker, contentHashCache, (filePath, event) => {
+        this.vcsCoordinator = new VcsCoordinator(this.config, logger);
+    }
+    /** Wire up watcher and VCS coordinator, then start all services. */
+    async startServicesAndWatcher() {
+        const logger = this.logger;
+        const vcsCoordinator = this.vcsCoordinator;
+        const { watcher, gitignoreFilter } = createWatcher(this.config, this.factories, this.queue, this.processor, logger, this.runtimeOptions, this.initialScanTracker, this.contentHashCache, (filePath, event) => {
             vcsCoordinator.onFileChange(filePath, event);
+        }, () => {
+            void vcsCoordinator.onInitialScanComplete();
         });
         this.watcher = watcher;
         this.gitignoreFilter = gitignoreFilter;
         this.server = await this.startApiServer();
-        await vcsCoordinator.start();
+        vcsCoordinator.start();
         this.watcher.start();
         this.startConfigWatch();
         logger.info('jeeves-watcher started');
@@ -9081,6 +9244,8 @@ class JeevesWatcher {
             contentHashCache: this.contentHashCache,
             valuesManager: this.valuesManager,
             issuesManager: this.issuesManager,
+            onVcsFileChange: (filePath, event) => this.vcsCoordinator?.onFileChange(filePath, event),
+            onInitialScanComplete: () => this.vcsCoordinator?.onInitialScanComplete(),
         });
         this.config = state.config;
         this.watcher = state.watcher;