@karmaniverous/jeeves-watcher 0.18.8 → 0.18.10

package/dist/cli/jeeves-watcher/{index-BjghvUnc.js → index-B_g6RfR8.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, createConfigApplyHandler, DEFAULT_PORTS, getBindAddress } from '@karmaniverous/jeeves';
-import { a as normalizeSlashes, n as normalizeError, r as retry, G as GitignoreFilter, e as executeReindex, V as VALID_REINDEX_SCOPES, m as mergeInferenceRules, i as isPathWatched, g as getWatchRootBases, C as CONFIG_WATCH_VALID_SCOPES, s as sleep, l as loadConfig } from './index.js';
-export { b as startFromConfig } from './index.js';
+import { d as normalizeSlashes, n as normalizeError, e as execFileAsync, r as retry, b as gitAddViaStdin, c as normalizePathCase, f as findRootForPath, G as GitignoreFilter, a as executeReindex, V as VALID_REINDEX_SCOPES, m as mergeInferenceRules, i as isPathWatched, g as getWatchRootBases, C as CONFIG_WATCH_VALID_SCOPES, s as sleep, l as loadConfig } from './index.js';
+export { h as startFromConfig } from './index.js';
 import { mkdirSync, existsSync, readFileSync, writeFileSync, rmSync, renameSync, readdirSync, statSync } from 'node:fs';
 import { join, dirname, resolve, extname, basename, relative, sep, isAbsolute } from 'node:path';
 import Database from 'better-sqlite3';
@@ -10,11 +10,9 @@ import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import picomatch from 'picomatch';
 import { deserialize, serialize } from 'node:v8';
-import { normalizeWatchPaths, extractWatchPathStrings, vcsAuthorConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema } from '@karmaniverous/jeeves-watcher-core';
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
+import { normalizeWatchPaths, vcsRetentionConfigSchema, extractWatchPathStrings, vcsAuthorConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema, getEndpoint } from '@karmaniverous/jeeves-watcher-core';
 import { homedir } from 'node:os';
-import { access, rm, readFile, writeFile, appendFile, mkdir, stat } from 'node:fs/promises';
+import { access, rm, stat, readFile, writeFile, appendFile, mkdir } from 'node:fs/promises';
 import { pathToFileURL, fileURLToPath } from 'node:url';
 import { packageDirectorySync } from 'package-directory';
 import { JsonMap } from '@karmaniverous/jsonmap';
@@ -40,6 +38,8 @@ import mammoth from 'mammoth';
 import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import { QdrantClient } from '@qdrant/js-client-rest';
 import 'cosmiconfig';
+import 'node:child_process';
+import 'node:util';
 
 /**
  * @module api/InitialScanTracker
@@ -928,77 +928,6 @@ class CommitMessageGenerator {
     }
 }
 
-/**
- * @module vcs/gitExec
- * Shared git execution utilities.
- */
-/** Promisified execFile for git commands. */
-const execFileAsync = promisify(execFile);
-/**
- * Stage files via stdin to avoid ENAMETOOLONG on Windows.
- *
- * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
- * through stdin (NUL-delimited) instead of passed as command-line
- * arguments.  This sidesteps the Windows CreateProcessW 32 767-char
- * limit that triggers ENAMETOOLONG when batches contain many files
- * with long absolute paths.
- *
- * @param files - Absolute paths of files to stage.
- * @param cwd   - Repository root (working directory for git).
- */
-function gitAddViaStdin(files, cwd) {
-    return new Promise((resolve, reject) => {
-        const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
-            if (error)
-                reject(error);
-            else
-                resolve();
-        });
-        if (!child.stdin) {
-            reject(new Error('Failed to open stdin for git add'));
-            return;
-        }
-        // Suppress EPIPE — expected if git exits before we finish writing
-        child.stdin.on('error', (err) => {
-            if (err.code !== 'EPIPE') {
-                reject(err);
-            }
-        });
-        child.stdin.end(files.join('\0'));
-    });
-}
-/**
- * Normalize a path for case-insensitive comparison on Windows.
- * On Windows, lowercases the entire path; on other platforms, returns as-is.
- *
- * @param p - The path string to normalize.
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The normalized path.
- */
-function normalizePathCase(p, platform = process.platform) {
-    return platform === 'win32' ? p.toLowerCase() : p;
-}
-/**
- * Find the longest-prefix-match root for a normalized path.
- * Roots must be sorted longest-first for correct nested matching.
- *
- * @param roots - Root paths sorted longest-first.
- * @param normalizedPath - Normalized absolute path (forward slashes).
- * @param platform - The platform to check against (default: `process.platform`).
- * @returns The matching root, or undefined.
- */
-function findRootForPath(roots, normalizedPath, platform = process.platform) {
-    const comparePath = normalizePathCase(normalizedPath, platform);
-    for (const root of roots) {
-        const compareRoot = normalizePathCase(root, platform);
-        if (comparePath === compareRoot ||
-            comparePath.startsWith(compareRoot + '/')) {
-            return root;
-        }
-    }
-    return undefined;
-}
-
 /**
  * @module vcs/resolveCommitMessageApiKey
  * Resolves the API key for AI commit message generation.
@@ -1460,7 +1389,7 @@ async function pushToRemote(rootPath, remoteUrl, accessToken, pushErrors, logger
  * Per-root VCS manager for git-backed content versioning.
  *
  * Each VCS-enabled watch root gets its own VcsManager instance, which owns
- * a debounced commit pipeline: file changes are batched, staged, committed
+ * a throttled commit pipeline: file changes are batched, staged, committed
  * (with optional AI-generated messages), and optionally pushed to a remote.
  *
  * Concurrency: only one commit is in-flight at a time per root. Index.lock
@@ -1477,8 +1406,9 @@ class VcsManager {
     pendingReversions = [];
     _pushErrors = [];
     squashManager;
-    debounceTimer;
+    throttleTimer;
     commitInFlight = Promise.resolve();
+    consecutiveCommitFailures = 0;
     started = false;
     isBaseline = true;
     _lastPushTime = null;
@@ -1519,7 +1449,7 @@ class VcsManager {
         this.isBaseline = false;
     }
     /**
-     * Add a file to the pending set and reset the debounce timer.
+     * Add a file to the pending set and start the throttle timer.
      * If the pending set exceeds maxBatchSize, flush immediately.
      *
      * @param filePath - Absolute path of the changed file.
@@ -1527,17 +1457,22 @@ class VcsManager {
     fileChanged(filePath) {
         if (!this.started)
             return;
+        // Reset circuit breaker on new file change so the system can recover
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.info({ root: this.rootPath }, 'VCS circuit breaker reset — new file change received, retrying commits');
+            this.consecutiveCommitFailures = 0;
+        }
         this.pending.add(filePath);
         if (this.pending.size >= this.config.maxBatchSize) {
-            this.clearDebounce();
+            this.clearThrottle();
             const batch = this.takeBatch(this.config.maxBatchSize);
             this.commitInFlight = this.commitInFlight.then(() => this.commitBatch(batch));
             if (this.pending.size > 0) {
-                this.resetDebounce();
+                this.resetThrottle();
             }
             return;
         }
-        this.resetDebounce();
+        this.resetThrottle();
     }
     /**
      * Record reversion metadata so the next commit uses a revert-prefixed message.
@@ -1562,7 +1497,7 @@ class VcsManager {
      * Flush all pending files immediately.
      */
     async flush() {
-        this.clearDebounce();
+        this.clearThrottle();
         if (this.pending.size > 0) {
             const batch = [...this.pending];
             this.pending.clear();
@@ -1595,31 +1530,65 @@ class VcsManager {
         return batch;
     }
     /**
-     * Clear the debounce timer.
+     * Clear the throttle timer.
      */
-    clearDebounce() {
-        if (this.debounceTimer !== undefined) {
-            clearTimeout(this.debounceTimer);
-            this.debounceTimer = undefined;
+    clearThrottle() {
+        if (this.throttleTimer !== undefined) {
+            clearTimeout(this.throttleTimer);
+            this.throttleTimer = undefined;
         }
     }
     /**
-     * Reset the debounce timer to fire flush() after commitDebounceMs.
+     * Start the throttle timer to fire flush() after commitThrottleMs.
+     * If a timer is already running, do not reset it — this is throttle, not debounce.
      */
-    resetDebounce() {
-        this.clearDebounce();
-        this.debounceTimer = setTimeout(() => {
-            this.debounceTimer = undefined;
+    resetThrottle() {
+        if (this.throttleTimer !== undefined)
+            return;
+        this.throttleTimer = setTimeout(() => {
+            this.throttleTimer = undefined;
             void this.flush();
-        }, this.config.commitDebounceMs);
+        }, this.config.commitThrottleMs);
     }
     /**
-     * Commit a batch of files to the git repo with index.lock retry.
+     * Check for and remove stale index.lock files before retrying.
+     */
+    async removeStaleLock() {
+        const lockPath = join(this.rootPath, '.git', 'index.lock');
+        try {
+            const lockStat = await stat(lockPath);
+            const ageMs = Date.now() - lockStat.mtimeMs;
+            if (ageMs > this.config.staleLockThresholdMs) {
+                await rm(lockPath, { force: true });
+                this.logger.warn({ root: this.rootPath, ageMs }, 'Removed stale index.lock');
+            }
+        }
+        catch (error) {
+            const code = error.code;
+            if (code !== 'ENOENT') {
+                this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Unable to remove stale index.lock');
+            }
+        }
+    }
+    /**
+     * Commit a batch of files to the git repo with index.lock retry,
+     * stale lock detection, circuit breaker, and re-queue cap.
      */
     async commitBatch(files) {
         if (files.length === 0)
             return;
+        // Circuit breaker: skip if too many consecutive failures
+        if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
+            this.logger.error({
+                root: this.rootPath,
+                consecutiveFailures: this.consecutiveCommitFailures,
+                discardedFiles: files.length,
+            }, 'VCS circuit breaker tripped — pending files discarded until next successful commit');
+            return;
+        }
         try {
+            // Check for stale lock before first attempt
+            await this.removeStaleLock();
             await retry(async (attempt) => {
                 await gitAddViaStdin(files, this.rootPath);
                 // Build message after staging so getStagedDiff can see the changes.
@@ -1645,12 +1614,22 @@ class VcsManager {
                     this.logger.warn({ root: this.rootPath, attempt, delay: delayMs }, 'index.lock contention, retrying');
                 },
             });
+            // Success — reset circuit breaker
+            this.consecutiveCommitFailures = 0;
         }
         catch (error) {
-            for (const f of files) {
+            this.consecutiveCommitFailures++;
+            // Re-queue cap: only re-queue up to maxBatchSize files
+            const cap = this.config.maxBatchSize;
+            const toRequeue = files.slice(0, cap);
+            const overflow = files.length - toRequeue.length;
+            for (const f of toRequeue) {
                 this.pending.add(f);
             }
-            this.logger.warn({ root: this.rootPath, fileCount: files.length }, 'Re-queued files after commit failure');
+            if (overflow > 0) {
+                this.logger.warn({ root: this.rootPath, overflow, cap }, 'Re-queue cap exceeded — discarded overflow files');
+            }
+            this.logger.warn({ root: this.rootPath, fileCount: toRequeue.length }, 'Re-queued files after commit failure');
             this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS commit failed');
         }
     }
@@ -1682,10 +1661,14 @@ class VcsCoordinator {
             // Deduplicate: skip if another glob already resolved to this root.
             if (this.managers.has(rootKey))
                 continue;
+            const mergedRetention = entry.vcs?.retention ??
+                config.vcs.retention ??
+                vcsRetentionConfigSchema.parse({});
             const mergedConfig = {
                 ...config.vcs,
                 ...entry.vcs,
                 enabled: true,
+                retention: mergedRetention,
             };
             // Create CommitMessageGenerator if AI commit messages are configured
             const cmConfig = mergedConfig.commitMessage;
@@ -1736,8 +1719,8 @@ class VcsCoordinator {
     /**
      * Signal that the initial filesystem scan is complete.
      *
-     * Flushes each manager's debounce buffer before ending baseline mode, ensuring
-     * any files still pending in the VCS debounce are committed with a
+     * Flushes each manager's throttle buffer before ending baseline mode, ensuring
+     * any files still pending in the VCS throttle are committed with a
      * `"baseline:"` prefix rather than `"watcher:"`. AI generation is not enabled
      * until all managers have completed their flush.
      */
@@ -3179,10 +3162,10 @@ function createWatcher(config, factories, queue, processor, logger, runtimeOptio
  * Tear down and rebuild the filesystem watcher with new config.
  * Falls back to the old watcher if the new one fails to start.
  */
-async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache) {
+async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
     try {
         await oldState.watcher.stop();
-        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache);
+        const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete);
         newWatcher.start();
         logger.info('Filesystem watcher rebuilt successfully');
         return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
@@ -3377,7 +3360,7 @@ async function reloadConfig(state, deps) {
         // Rebuild filesystem watcher if watch config changed
         if (watchConfigChanged(oldConfig, newConfig)) {
             logger.info('Watch config changed, rebuilding filesystem watcher...');
-            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache);
+            const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache, deps.onVcsFileChange, deps.onInitialScanComplete);
             state.watcher = newState.watcher;
             state.gitignoreFilter = newState.gitignoreFilter;
         }
@@ -5431,36 +5414,36 @@ function createApiServer(options) {
             };
         },
     });
-    app.get('/status', withCache(cacheTtlMs, async () => {
+    app.get(getEndpoint('status').path, withCache(cacheTtlMs, async () => {
         const result = await coreStatusHandler();
         return result.body;
     }));
-    app.post('/metadata', createMetadataHandler({
+    app.post(getEndpoint('metadata').path, createMetadataHandler({
         processor,
         getConfig,
         logger,
         configDir: dirname(configPath),
     }));
-    app.post('/render', withCache(cacheTtlMs, createRenderHandler({
+    app.post(getEndpoint('render').path, withCache(cacheTtlMs, createRenderHandler({
         processor,
         getWatch: () => getConfig().watch,
         logger,
     })));
-    app.get('/search/facets', createFacetsHandler({
+    app.get(getEndpoint('searchFacets').path, createFacetsHandler({
         getConfig,
         valuesManager,
         configDir: dirname(configPath),
     }));
-    app.post('/scan', createScanHandler({
+    app.post(getEndpoint('scan').path, createScanHandler({
         vectorStore,
         logger,
     }));
-    app.post('/walk', createWalkHandler({
+    app.post(getEndpoint('walk').path, createWalkHandler({
         getWatchPaths: () => extractWatchPathStrings(getConfig().watch.paths),
         getFileSystemWatcher,
         logger,
     }));
-    app.post('/search', createSearchHandler({
+    app.post(getEndpoint('search').path, createSearchHandler({
         embeddingProvider,
         vectorStore,
         logger,
@@ -5471,12 +5454,12 @@ function createApiServer(options) {
                 : undefined;
         },
     }));
-    app.post('/rebuild-metadata', createRebuildMetadataHandler({
+    app.post(getEndpoint('rebuildMetadata').path, createRebuildMetadataHandler({
         enrichmentStore: options.enrichmentStore,
         vectorStore,
         logger,
     }));
-    app.post('/reindex', createConfigReindexHandler({
+    app.post(getEndpoint('reindex').path, createConfigReindexHandler({
         getConfig,
         processor,
         logger,
@@ -5487,10 +5470,10 @@ function createApiServer(options) {
         vectorStore,
         queue,
     }));
-    app.get('/issues', withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
-    app.get('/config/schema', withCache(cacheTtlMs, createConfigSchemaHandler()));
-    app.post('/config/match', createConfigMatchHandler({ getConfig, logger }));
-    app.get('/config', withCache(cacheTtlMs, createConfigQueryHandler({
+    app.get(getEndpoint('issues').path, withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
+    app.get(getEndpoint('configSchema').path, withCache(cacheTtlMs, createConfigSchemaHandler()));
+    app.post(getEndpoint('configMatch').path, createConfigMatchHandler({ getConfig, logger }));
+    app.get(getEndpoint('config').path, withCache(cacheTtlMs, createConfigQueryHandler({
         getConfig,
         valuesManager,
         issuesManager,
@@ -5500,7 +5483,7 @@ function createApiServer(options) {
             ? () => virtualRuleStore.getAll()
             : undefined,
     })));
-    app.post('/config/validate', createConfigValidateHandler({
+    app.post(getEndpoint('configValidate').path, createConfigValidateHandler({
         getConfig,
         logger,
         configDir: dirname(configPath),
@@ -5514,7 +5497,7 @@ function createApiServer(options) {
             return Promise.resolve();
         },
     });
-    app.post('/config/apply', async (request, reply) => {
+    app.post(getEndpoint('configApply').path, async (request, reply) => {
         const { patch, replace } = request.body;
         const result = await coreConfigApplyHandler({ patch, replace });
         return reply.status(result.status).send(result.body);
@@ -5533,34 +5516,34 @@ function createApiServer(options) {
             gitignoreFilter,
             vectorStore,
         });
-        app.post('/rules/register', createRulesRegisterHandler({
+        app.post(getEndpoint('rulesRegister').path, createRulesRegisterHandler({
             virtualRuleStore,
             logger,
             onRulesChanged,
         }));
-        app.delete('/rules/unregister', createRulesUnregisterHandler({
+        app.delete(getEndpoint('rulesUnregister').path, createRulesUnregisterHandler({
             virtualRuleStore,
             logger,
             onRulesChanged,
         }));
-        app.delete('/rules/unregister/:source', createRulesUnregisterParamHandler({
+        app.delete(getEndpoint('rulesUnregisterBySource').path, createRulesUnregisterParamHandler({
             virtualRuleStore,
             logger,
             onRulesChanged,
         }));
-        app.post('/points/delete', createPointsDeleteHandler({ vectorStore, logger }));
-        app.post('/rules/reapply', createRulesReapplyHandler({ processor, vectorStore, logger }));
+        app.post(getEndpoint('pointsDelete').path, createPointsDeleteHandler({ vectorStore, logger }));
+        app.post(getEndpoint('rulesReapply').path, createRulesReapplyHandler({ processor, vectorStore, logger }));
     }
     // VCS routes — only registered when a VcsCoordinator is provided
     if (options.vcsCoordinator) {
         const coordinator = options.vcsCoordinator;
-        app.get('/vcs/status', createVcsStatusHandler({ coordinator, logger }));
-        app.get('/vcs/history', createVcsHistoryHandler({ coordinator, logger }));
-        app.get('/vcs/show', createVcsShowHandler({ coordinator, logger }));
-        app.get('/vcs/diff', createVcsDiffHandler({ coordinator, logger }));
-        app.get('/vcs/check-exclusion', createVcsCheckExclusionHandler({ coordinator, logger }));
-        app.post('/vcs/revert', createVcsRevertHandler({ coordinator, logger }));
-        app.post('/vcs/exclude', createVcsExcludeHandler({ coordinator, logger }));
+        app.get(getEndpoint('vcsStatus').path, createVcsStatusHandler({ coordinator, logger }));
+        app.get(getEndpoint('vcsHistory').path, createVcsHistoryHandler({ coordinator, logger }));
+        app.get(getEndpoint('vcsShow').path, createVcsShowHandler({ coordinator, logger }));
+        app.get(getEndpoint('vcsDiff').path, createVcsDiffHandler({ coordinator, logger }));
+        app.get(getEndpoint('vcsCheckExclusion').path, createVcsCheckExclusionHandler({ coordinator, logger }));
+        app.post(getEndpoint('vcsRevert').path, createVcsRevertHandler({ coordinator, logger }));
+        app.post(getEndpoint('vcsExclude').path, createVcsExcludeHandler({ coordinator, logger }));
     }
     return app;
 }
@@ -7989,6 +7972,8 @@ class JeevesWatcher {
             contentHashCache: this.contentHashCache,
             valuesManager: this.valuesManager,
             issuesManager: this.issuesManager,
+            onVcsFileChange: (filePath, event) => this.vcsCoordinator?.onFileChange(filePath, event),
+            onInitialScanComplete: () => this.vcsCoordinator?.onInitialScanComplete(),
         });
         this.config = state.config;
         this.watcher = state.watcher;