@karmaniverous/jeeves-watcher 0.18.8 → 0.18.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,13 +1,15 @@
1
1
  import { join, dirname, resolve, relative, extname, basename, isAbsolute, sep } from 'node:path';
2
2
  import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, createConfigApplyHandler, substituteEnvVars, DEFAULT_PORTS, getBindAddress, postJson, fetchJson, getPackageVersion } from '@karmaniverous/jeeves';
3
- import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsAuthorConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
3
+ import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsRetentionConfigSchema, vcsAuthorConfigSchema, getEndpoint, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
4
4
  export { apiConfigSchema, configWatchConfigSchema, embeddingConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema, loggingConfigSchema, vectorStoreConfigSchema, watchConfigSchema } from '@karmaniverous/jeeves-watcher-core';
5
5
  import Fastify from 'fastify';
6
- import { readdir, stat, readFile, writeFile, mkdir, appendFile, access, rm } from 'node:fs/promises';
6
+ import { readdir, stat, access, readFile, writeFile, mkdir, appendFile, rm } from 'node:fs/promises';
7
7
  import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
8
8
  import { existsSync, statSync, readFileSync, readdirSync, mkdirSync, writeFileSync, rmSync, renameSync } from 'node:fs';
9
9
  import ignore from 'ignore';
10
10
  import picomatch from 'picomatch';
11
+ import { execFile } from 'node:child_process';
12
+ import { promisify } from 'node:util';
11
13
  import Ajv from 'ajv';
12
14
  import addFormats from 'ajv-formats';
13
15
  import { z, ZodError } from 'zod';
@@ -21,8 +23,6 @@ import { toMarkdown } from 'mdast-util-to-markdown';
21
23
  import rehypeParse from 'rehype-parse';
22
24
  import { unified } from 'unified';
23
25
  import yaml from 'js-yaml';
24
- import { execFile } from 'node:child_process';
25
- import { promisify } from 'node:util';
26
26
  import crypto from 'crypto';
27
27
  import { packageDirectorySync } from 'package-directory';
28
28
  import { homedir } from 'node:os';
@@ -421,6 +421,77 @@ async function retry(fn, options) {
421
421
  throw lastError;
422
422
  }
423
423
 
424
+ /**
425
+ * @module vcs/gitExec
426
+ * Shared git execution utilities.
427
+ */
428
+ /** Promisified execFile for git commands. */
429
+ const execFileAsync = promisify(execFile);
430
+ /**
431
+ * Stage files via stdin to avoid ENAMETOOLONG on Windows.
432
+ *
433
+ * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
434
+ * through stdin (NUL-delimited) instead of passed as command-line
435
+ * arguments. This sidesteps the Windows CreateProcessW 32 767-char
436
+ * limit that triggers ENAMETOOLONG when batches contain many files
437
+ * with long absolute paths.
438
+ *
439
+ * @param files - Absolute paths of files to stage.
440
+ * @param cwd - Repository root (working directory for git).
441
+ */
442
+ function gitAddViaStdin(files, cwd) {
443
+ return new Promise((resolve, reject) => {
444
+ const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
445
+ if (error)
446
+ reject(error);
447
+ else
448
+ resolve();
449
+ });
450
+ if (!child.stdin) {
451
+ reject(new Error('Failed to open stdin for git add'));
452
+ return;
453
+ }
454
+ // Suppress EPIPE — expected if git exits before we finish writing
455
+ child.stdin.on('error', (err) => {
456
+ if (err.code !== 'EPIPE') {
457
+ reject(err);
458
+ }
459
+ });
460
+ child.stdin.end(files.join('\0'));
461
+ });
462
+ }
463
+ /**
464
+ * Normalize a path for case-insensitive comparison on Windows.
465
+ * On Windows, lowercases the entire path; on other platforms, returns as-is.
466
+ *
467
+ * @param p - The path string to normalize.
468
+ * @param platform - The platform to check against (default: `process.platform`).
469
+ * @returns The normalized path.
470
+ */
471
+ function normalizePathCase(p, platform = process.platform) {
472
+ return platform === 'win32' ? p.toLowerCase() : p;
473
+ }
474
+ /**
475
+ * Find the longest-prefix-match root for a normalized path.
476
+ * Roots must be sorted longest-first for correct nested matching.
477
+ *
478
+ * @param roots - Root paths sorted longest-first.
479
+ * @param normalizedPath - Normalized absolute path (forward slashes).
480
+ * @param platform - The platform to check against (default: `process.platform`).
481
+ * @returns The matching root, or undefined.
482
+ */
483
+ function findRootForPath(roots, normalizedPath, platform = process.platform) {
484
+ const comparePath = normalizePathCase(normalizedPath, platform);
485
+ for (const root of roots) {
486
+ const compareRoot = normalizePathCase(root, platform);
487
+ if (comparePath === compareRoot ||
488
+ comparePath.startsWith(compareRoot + '/')) {
489
+ return root;
490
+ }
491
+ }
492
+ return undefined;
493
+ }
494
+
424
495
  /**
425
496
  * Best-effort base directory inference for a glob pattern.
426
497
  *
@@ -638,7 +709,7 @@ function groupByRoot(filePaths, watchPaths) {
638
709
  /** Default page size for prune scroll (smaller than search to reduce connection strain). */
639
710
  const PRUNE_SCROLL_PAGE_SIZE = 500;
640
711
  /** Max retry attempts for scroll page failures. */
641
- const SCROLL_RETRY_ATTEMPTS = 3;
712
+ const SCROLL_RETRY_ATTEMPTS = 5;
642
713
  /** Base delay for scroll retry backoff (ms). */
643
714
  const SCROLL_RETRY_BASE_DELAY_MS = 1000;
644
715
  /**
@@ -664,26 +735,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
664
735
  }
665
736
  throw lastError;
666
737
  }
738
+ /** Concurrency limit for fs.access() disk-existence checks. */
739
+ const FS_ACCESS_CONCURRENCY = 100;
667
740
  /**
668
- * Compute the plan for a prune operation.
669
- * Scrolls all Qdrant points page-by-page with retry/resume,
670
- * checks each file_path against watch scope + gitignore,
671
- * returns the list of orphaned point IDs and the plan.
741
+ * Compute the plan for a prune operation using a filter-first approach.
742
+ *
743
+ * 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
744
+ * 2. Scroll Qdrant points. For each point:
745
+ * - No `file_path` → orphan.
746
+ * - `file_path` not in `watchedFiles` → orphan (out of watch scope).
747
+ * - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
748
+ * If missing from disk → orphan (deleted file still in index).
749
+ * 3. Batch `fs.access()` checks with bounded concurrency.
672
750
  */
673
751
  async function computePrunePlan(deps) {
674
752
  const { config, vectorStore, gitignoreFilter, logger } = deps;
675
753
  if (!vectorStore) {
676
754
  throw new Error('vectorStore is required for prune scope');
677
755
  }
756
+ // Step 1: Build set of watched files from filesystem
757
+ const isGitignored = createIsGitignored(gitignoreFilter);
758
+ const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
759
+ const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
760
+ logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
761
+ // Step 2: Scroll Qdrant and classify points
678
762
  const seenFiles = new Set();
679
763
  const orphanedFiles = new Set();
764
+ const diskCheckFiles = new Set();
680
765
  const orphanedIds = [];
766
+ // Map from file_path → point IDs, for points needing disk-existence check
767
+ const diskCheckPointIds = new Map();
681
768
  let totalPoints = 0;
682
769
  let incomplete = false;
683
770
  let cursor;
684
771
  try {
685
- // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cursor is updated each iteration
686
- while (true) {
772
+ let hasMore = true;
773
+ while (hasMore) {
687
774
  const page = await scrollPageWithRetry(vectorStore, cursor, logger);
688
775
  for (const point of page.points) {
689
776
  totalPoints++;
@@ -692,32 +779,62 @@ async function computePrunePlan(deps) {
692
779
  orphanedIds.push(point.id);
693
780
  continue;
694
781
  }
695
- if (seenFiles.has(filePath)) {
696
- if (orphanedFiles.has(filePath)) {
782
+ const normalized = normalizePathCase(normalizeSlashes(filePath));
783
+ if (seenFiles.has(normalized)) {
784
+ if (orphanedFiles.has(normalized)) {
697
785
  orphanedIds.push(point.id);
698
786
  }
787
+ else if (diskCheckFiles.has(normalized)) {
788
+ // Already queued for disk check — track its ID
789
+ const ids = diskCheckPointIds.get(normalized);
790
+ if (ids)
791
+ ids.push(point.id);
792
+ }
699
793
  continue;
700
794
  }
701
- seenFiles.add(filePath);
702
- const watched = isPathWatched(filePath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
703
- const gitignored = gitignoreFilter
704
- ? gitignoreFilter.isIgnored(filePath)
705
- : false;
706
- if (!watched || gitignored) {
707
- orphanedFiles.add(filePath);
795
+ seenFiles.add(normalized);
796
+ if (!watchedFiles.has(normalized)) {
797
+ // Not in watch scope → definitely orphaned
798
+ orphanedFiles.add(normalized);
708
799
  orphanedIds.push(point.id);
709
800
  }
801
+ else {
802
+ // In watch scope → needs disk-existence check
803
+ diskCheckFiles.add(normalized);
804
+ diskCheckPointIds.set(normalized, [point.id]);
805
+ }
806
+ }
807
+ // Progress logging every 100K points
808
+ if (totalPoints > 0 && totalPoints % 100000 === 0) {
809
+ logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
810
+ }
811
+ if (!page.nextCursor) {
812
+ hasMore = false;
813
+ }
814
+ else {
815
+ cursor = page.nextCursor;
710
816
  }
711
- if (!page.nextCursor)
712
- break;
713
- cursor = page.nextCursor;
714
817
  }
715
818
  }
716
819
  catch (error) {
717
- // All retries exhausted - return partial results
718
820
  logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
719
821
  incomplete = true;
720
822
  }
823
+ // Step 3: Batch fs.access() checks for files in watch scope
824
+ const filesToCheck = [...diskCheckFiles];
825
+ await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
826
+ try {
827
+ await access(filePath);
828
+ }
829
+ catch {
830
+ // File doesn't exist on disk → orphaned
831
+ orphanedFiles.add(filePath);
832
+ const pointIds = diskCheckPointIds.get(filePath);
833
+ if (pointIds) {
834
+ orphanedIds.push(...pointIds);
835
+ }
836
+ }
837
+ });
721
838
  const orphanedFileList = [...orphanedFiles];
722
839
  const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
723
840
  return {
@@ -775,6 +892,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
775
892
  deps.queue?.resume();
776
893
  return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
777
894
  }
895
+ // Abort live prune when plan is incomplete — never silently delete partial data
896
+ if (plan.incomplete) {
897
+ deps.queue?.resume();
898
+ logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
899
+ return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
900
+ }
778
901
  // Execute prune
779
902
  reindexTracker?.start(scope);
780
903
  const startTime = Date.now();
@@ -3369,77 +3492,6 @@ function createSearchHandler(deps) {
3369
3492
  }, deps.logger, 'Search');
3370
3493
  }
3371
3494
 
3372
- /**
3373
- * @module vcs/gitExec
3374
- * Shared git execution utilities.
3375
- */
3376
- /** Promisified execFile for git commands. */
3377
- const execFileAsync = promisify(execFile);
3378
- /**
3379
- * Stage files via stdin to avoid ENAMETOOLONG on Windows.
3380
- *
3381
- * Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
3382
- * through stdin (NUL-delimited) instead of passed as command-line
3383
- * arguments. This sidesteps the Windows CreateProcessW 32 767-char
3384
- * limit that triggers ENAMETOOLONG when batches contain many files
3385
- * with long absolute paths.
3386
- *
3387
- * @param files - Absolute paths of files to stage.
3388
- * @param cwd - Repository root (working directory for git).
3389
- */
3390
- function gitAddViaStdin(files, cwd) {
3391
- return new Promise((resolve, reject) => {
3392
- const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
3393
- if (error)
3394
- reject(error);
3395
- else
3396
- resolve();
3397
- });
3398
- if (!child.stdin) {
3399
- reject(new Error('Failed to open stdin for git add'));
3400
- return;
3401
- }
3402
- // Suppress EPIPE — expected if git exits before we finish writing
3403
- child.stdin.on('error', (err) => {
3404
- if (err.code !== 'EPIPE') {
3405
- reject(err);
3406
- }
3407
- });
3408
- child.stdin.end(files.join('\0'));
3409
- });
3410
- }
3411
- /**
3412
- * Normalize a path for case-insensitive comparison on Windows.
3413
- * On Windows, lowercases the entire path; on other platforms, returns as-is.
3414
- *
3415
- * @param p - The path string to normalize.
3416
- * @param platform - The platform to check against (default: `process.platform`).
3417
- * @returns The normalized path.
3418
- */
3419
- function normalizePathCase(p, platform = process.platform) {
3420
- return platform === 'win32' ? p.toLowerCase() : p;
3421
- }
3422
- /**
3423
- * Find the longest-prefix-match root for a normalized path.
3424
- * Roots must be sorted longest-first for correct nested matching.
3425
- *
3426
- * @param roots - Root paths sorted longest-first.
3427
- * @param normalizedPath - Normalized absolute path (forward slashes).
3428
- * @param platform - The platform to check against (default: `process.platform`).
3429
- * @returns The matching root, or undefined.
3430
- */
3431
- function findRootForPath(roots, normalizedPath, platform = process.platform) {
3432
- const comparePath = normalizePathCase(normalizedPath, platform);
3433
- for (const root of roots) {
3434
- const compareRoot = normalizePathCase(root, platform);
3435
- if (comparePath === compareRoot ||
3436
- comparePath.startsWith(compareRoot + '/')) {
3437
- return root;
3438
- }
3439
- }
3440
- return undefined;
3441
- }
3442
-
3443
3495
  /**
3444
3496
  * @module vcs/resolveWatchRoot
3445
3497
  * Resolves which VCS-enabled watch root(s) own a given absolute path or glob.
@@ -4956,7 +5008,7 @@ async function pushToRemote(rootPath, remoteUrl, accessToken, pushErrors, logger
4956
5008
  * Per-root VCS manager for git-backed content versioning.
4957
5009
  *
4958
5010
  * Each VCS-enabled watch root gets its own VcsManager instance, which owns
4959
- * a debounced commit pipeline: file changes are batched, staged, committed
5011
+ * a throttled commit pipeline: file changes are batched, staged, committed
4960
5012
  * (with optional AI-generated messages), and optionally pushed to a remote.
4961
5013
  *
4962
5014
  * Concurrency: only one commit is in-flight at a time per root. Index.lock
@@ -4973,8 +5025,9 @@ class VcsManager {
4973
5025
  pendingReversions = [];
4974
5026
  _pushErrors = [];
4975
5027
  squashManager;
4976
- debounceTimer;
5028
+ throttleTimer;
4977
5029
  commitInFlight = Promise.resolve();
5030
+ consecutiveCommitFailures = 0;
4978
5031
  started = false;
4979
5032
  isBaseline = true;
4980
5033
  _lastPushTime = null;
@@ -5015,7 +5068,7 @@ class VcsManager {
5015
5068
  this.isBaseline = false;
5016
5069
  }
5017
5070
  /**
5018
- * Add a file to the pending set and reset the debounce timer.
5071
+ * Add a file to the pending set and start the throttle timer.
5019
5072
  * If the pending set exceeds maxBatchSize, flush immediately.
5020
5073
  *
5021
5074
  * @param filePath - Absolute path of the changed file.
@@ -5023,17 +5076,22 @@ class VcsManager {
5023
5076
  fileChanged(filePath) {
5024
5077
  if (!this.started)
5025
5078
  return;
5079
+ // Reset circuit breaker on new file change so the system can recover
5080
+ if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
5081
+ this.logger.info({ root: this.rootPath }, 'VCS circuit breaker reset — new file change received, retrying commits');
5082
+ this.consecutiveCommitFailures = 0;
5083
+ }
5026
5084
  this.pending.add(filePath);
5027
5085
  if (this.pending.size >= this.config.maxBatchSize) {
5028
- this.clearDebounce();
5086
+ this.clearThrottle();
5029
5087
  const batch = this.takeBatch(this.config.maxBatchSize);
5030
5088
  this.commitInFlight = this.commitInFlight.then(() => this.commitBatch(batch));
5031
5089
  if (this.pending.size > 0) {
5032
- this.resetDebounce();
5090
+ this.resetThrottle();
5033
5091
  }
5034
5092
  return;
5035
5093
  }
5036
- this.resetDebounce();
5094
+ this.resetThrottle();
5037
5095
  }
5038
5096
  /**
5039
5097
  * Record reversion metadata so the next commit uses a revert-prefixed message.
@@ -5058,7 +5116,7 @@ class VcsManager {
5058
5116
  * Flush all pending files immediately.
5059
5117
  */
5060
5118
  async flush() {
5061
- this.clearDebounce();
5119
+ this.clearThrottle();
5062
5120
  if (this.pending.size > 0) {
5063
5121
  const batch = [...this.pending];
5064
5122
  this.pending.clear();
@@ -5091,31 +5149,65 @@ class VcsManager {
5091
5149
  return batch;
5092
5150
  }
5093
5151
  /**
5094
- * Clear the debounce timer.
5152
+ * Clear the throttle timer.
5095
5153
  */
5096
- clearDebounce() {
5097
- if (this.debounceTimer !== undefined) {
5098
- clearTimeout(this.debounceTimer);
5099
- this.debounceTimer = undefined;
5154
+ clearThrottle() {
5155
+ if (this.throttleTimer !== undefined) {
5156
+ clearTimeout(this.throttleTimer);
5157
+ this.throttleTimer = undefined;
5100
5158
  }
5101
5159
  }
5102
5160
  /**
5103
- * Reset the debounce timer to fire flush() after commitDebounceMs.
5161
+ * Start the throttle timer to fire flush() after commitThrottleMs.
5162
+ * If a timer is already running, do not reset it — this is throttle, not debounce.
5104
5163
  */
5105
- resetDebounce() {
5106
- this.clearDebounce();
5107
- this.debounceTimer = setTimeout(() => {
5108
- this.debounceTimer = undefined;
5164
+ resetThrottle() {
5165
+ if (this.throttleTimer !== undefined)
5166
+ return;
5167
+ this.throttleTimer = setTimeout(() => {
5168
+ this.throttleTimer = undefined;
5109
5169
  void this.flush();
5110
- }, this.config.commitDebounceMs);
5170
+ }, this.config.commitThrottleMs);
5171
+ }
5172
+ /**
5173
+ * Check for and remove stale index.lock files before retrying.
5174
+ */
5175
+ async removeStaleLock() {
5176
+ const lockPath = join(this.rootPath, '.git', 'index.lock');
5177
+ try {
5178
+ const lockStat = await stat(lockPath);
5179
+ const ageMs = Date.now() - lockStat.mtimeMs;
5180
+ if (ageMs > this.config.staleLockThresholdMs) {
5181
+ await rm(lockPath, { force: true });
5182
+ this.logger.warn({ root: this.rootPath, ageMs }, 'Removed stale index.lock');
5183
+ }
5184
+ }
5185
+ catch (error) {
5186
+ const code = error.code;
5187
+ if (code !== 'ENOENT') {
5188
+ this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Unable to remove stale index.lock');
5189
+ }
5190
+ }
5111
5191
  }
5112
5192
  /**
5113
- * Commit a batch of files to the git repo with index.lock retry.
5193
+ * Commit a batch of files to the git repo with index.lock retry,
5194
+ * stale lock detection, circuit breaker, and re-queue cap.
5114
5195
  */
5115
5196
  async commitBatch(files) {
5116
5197
  if (files.length === 0)
5117
5198
  return;
5199
+ // Circuit breaker: skip if too many consecutive failures
5200
+ if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
5201
+ this.logger.error({
5202
+ root: this.rootPath,
5203
+ consecutiveFailures: this.consecutiveCommitFailures,
5204
+ discardedFiles: files.length,
5205
+ }, 'VCS circuit breaker tripped — pending files discarded until next successful commit');
5206
+ return;
5207
+ }
5118
5208
  try {
5209
+ // Check for stale lock before first attempt
5210
+ await this.removeStaleLock();
5119
5211
  await retry(async (attempt) => {
5120
5212
  await gitAddViaStdin(files, this.rootPath);
5121
5213
  // Build message after staging so getStagedDiff can see the changes.
@@ -5141,12 +5233,22 @@ class VcsManager {
5141
5233
  this.logger.warn({ root: this.rootPath, attempt, delay: delayMs }, 'index.lock contention, retrying');
5142
5234
  },
5143
5235
  });
5236
+ // Success — reset circuit breaker
5237
+ this.consecutiveCommitFailures = 0;
5144
5238
  }
5145
5239
  catch (error) {
5146
- for (const f of files) {
5240
+ this.consecutiveCommitFailures++;
5241
+ // Re-queue cap: only re-queue up to maxBatchSize files
5242
+ const cap = this.config.maxBatchSize;
5243
+ const toRequeue = files.slice(0, cap);
5244
+ const overflow = files.length - toRequeue.length;
5245
+ for (const f of toRequeue) {
5147
5246
  this.pending.add(f);
5148
5247
  }
5149
- this.logger.warn({ root: this.rootPath, fileCount: files.length }, 'Re-queued files after commit failure');
5248
+ if (overflow > 0) {
5249
+ this.logger.warn({ root: this.rootPath, overflow, cap }, 'Re-queue cap exceeded — discarded overflow files');
5250
+ }
5251
+ this.logger.warn({ root: this.rootPath, fileCount: toRequeue.length }, 'Re-queued files after commit failure');
5150
5252
  this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS commit failed');
5151
5253
  }
5152
5254
  }
@@ -5178,10 +5280,14 @@ class VcsCoordinator {
5178
5280
  // Deduplicate: skip if another glob already resolved to this root.
5179
5281
  if (this.managers.has(rootKey))
5180
5282
  continue;
5283
+ const mergedRetention = entry.vcs?.retention ??
5284
+ config.vcs.retention ??
5285
+ vcsRetentionConfigSchema.parse({});
5181
5286
  const mergedConfig = {
5182
5287
  ...config.vcs,
5183
5288
  ...entry.vcs,
5184
5289
  enabled: true,
5290
+ retention: mergedRetention,
5185
5291
  };
5186
5292
  // Create CommitMessageGenerator if AI commit messages are configured
5187
5293
  const cmConfig = mergedConfig.commitMessage;
@@ -5232,8 +5338,8 @@ class VcsCoordinator {
5232
5338
  /**
5233
5339
  * Signal that the initial filesystem scan is complete.
5234
5340
  *
5235
- * Flushes each manager's debounce buffer before ending baseline mode, ensuring
5236
- * any files still pending in the VCS debounce are committed with a
5341
+ * Flushes each manager's throttle buffer before ending baseline mode, ensuring
5342
+ * any files still pending in the VCS throttle are committed with a
5237
5343
  * `"baseline:"` prefix rather than `"watcher:"`. AI generation is not enabled
5238
5344
  * until all managers have completed their flush.
5239
5345
  */
@@ -5315,10 +5421,10 @@ function createWatcher(config, factories, queue, processor, logger, runtimeOptio
5315
5421
  * Tear down and rebuild the filesystem watcher with new config.
5316
5422
  * Falls back to the old watcher if the new one fails to start.
5317
5423
  */
5318
- async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache) {
5424
+ async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
5319
5425
  try {
5320
5426
  await oldState.watcher.stop();
5321
- const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache);
5427
+ const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete);
5322
5428
  newWatcher.start();
5323
5429
  logger.info('Filesystem watcher rebuilt successfully');
5324
5430
  return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
@@ -5797,36 +5903,36 @@ function createApiServer(options) {
5797
5903
  };
5798
5904
  },
5799
5905
  });
5800
- app.get('/status', withCache(cacheTtlMs, async () => {
5906
+ app.get(getEndpoint('status').path, withCache(cacheTtlMs, async () => {
5801
5907
  const result = await coreStatusHandler();
5802
5908
  return result.body;
5803
5909
  }));
5804
- app.post('/metadata', createMetadataHandler({
5910
+ app.post(getEndpoint('metadata').path, createMetadataHandler({
5805
5911
  processor,
5806
5912
  getConfig,
5807
5913
  logger,
5808
5914
  configDir: dirname(configPath),
5809
5915
  }));
5810
- app.post('/render', withCache(cacheTtlMs, createRenderHandler({
5916
+ app.post(getEndpoint('render').path, withCache(cacheTtlMs, createRenderHandler({
5811
5917
  processor,
5812
5918
  getWatch: () => getConfig().watch,
5813
5919
  logger,
5814
5920
  })));
5815
- app.get('/search/facets', createFacetsHandler({
5921
+ app.get(getEndpoint('searchFacets').path, createFacetsHandler({
5816
5922
  getConfig,
5817
5923
  valuesManager,
5818
5924
  configDir: dirname(configPath),
5819
5925
  }));
5820
- app.post('/scan', createScanHandler({
5926
+ app.post(getEndpoint('scan').path, createScanHandler({
5821
5927
  vectorStore,
5822
5928
  logger,
5823
5929
  }));
5824
- app.post('/walk', createWalkHandler({
5930
+ app.post(getEndpoint('walk').path, createWalkHandler({
5825
5931
  getWatchPaths: () => extractWatchPathStrings(getConfig().watch.paths),
5826
5932
  getFileSystemWatcher,
5827
5933
  logger,
5828
5934
  }));
5829
- app.post('/search', createSearchHandler({
5935
+ app.post(getEndpoint('search').path, createSearchHandler({
5830
5936
  embeddingProvider,
5831
5937
  vectorStore,
5832
5938
  logger,
@@ -5837,12 +5943,12 @@ function createApiServer(options) {
5837
5943
  : undefined;
5838
5944
  },
5839
5945
  }));
5840
- app.post('/rebuild-metadata', createRebuildMetadataHandler({
5946
+ app.post(getEndpoint('rebuildMetadata').path, createRebuildMetadataHandler({
5841
5947
  enrichmentStore: options.enrichmentStore,
5842
5948
  vectorStore,
5843
5949
  logger,
5844
5950
  }));
5845
- app.post('/reindex', createConfigReindexHandler({
5951
+ app.post(getEndpoint('reindex').path, createConfigReindexHandler({
5846
5952
  getConfig,
5847
5953
  processor,
5848
5954
  logger,
@@ -5853,10 +5959,10 @@ function createApiServer(options) {
5853
5959
  vectorStore,
5854
5960
  queue,
5855
5961
  }));
5856
- app.get('/issues', withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
5857
- app.get('/config/schema', withCache(cacheTtlMs, createConfigSchemaHandler()));
5858
- app.post('/config/match', createConfigMatchHandler({ getConfig, logger }));
5859
- app.get('/config', withCache(cacheTtlMs, createConfigQueryHandler({
5962
+ app.get(getEndpoint('issues').path, withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
5963
+ app.get(getEndpoint('configSchema').path, withCache(cacheTtlMs, createConfigSchemaHandler()));
5964
+ app.post(getEndpoint('configMatch').path, createConfigMatchHandler({ getConfig, logger }));
5965
+ app.get(getEndpoint('config').path, withCache(cacheTtlMs, createConfigQueryHandler({
5860
5966
  getConfig,
5861
5967
  valuesManager,
5862
5968
  issuesManager,
@@ -5866,7 +5972,7 @@ function createApiServer(options) {
5866
5972
  ? () => virtualRuleStore.getAll()
5867
5973
  : undefined,
5868
5974
  })));
5869
- app.post('/config/validate', createConfigValidateHandler({
5975
+ app.post(getEndpoint('configValidate').path, createConfigValidateHandler({
5870
5976
  getConfig,
5871
5977
  logger,
5872
5978
  configDir: dirname(configPath),
@@ -5880,7 +5986,7 @@ function createApiServer(options) {
5880
5986
  return Promise.resolve();
5881
5987
  },
5882
5988
  });
5883
- app.post('/config/apply', async (request, reply) => {
5989
+ app.post(getEndpoint('configApply').path, async (request, reply) => {
5884
5990
  const { patch, replace } = request.body;
5885
5991
  const result = await coreConfigApplyHandler({ patch, replace });
5886
5992
  return reply.status(result.status).send(result.body);
@@ -5899,34 +6005,34 @@ function createApiServer(options) {
5899
6005
  gitignoreFilter,
5900
6006
  vectorStore,
5901
6007
  });
5902
- app.post('/rules/register', createRulesRegisterHandler({
6008
+ app.post(getEndpoint('rulesRegister').path, createRulesRegisterHandler({
5903
6009
  virtualRuleStore,
5904
6010
  logger,
5905
6011
  onRulesChanged,
5906
6012
  }));
5907
- app.delete('/rules/unregister', createRulesUnregisterHandler({
6013
+ app.delete(getEndpoint('rulesUnregister').path, createRulesUnregisterHandler({
5908
6014
  virtualRuleStore,
5909
6015
  logger,
5910
6016
  onRulesChanged,
5911
6017
  }));
5912
- app.delete('/rules/unregister/:source', createRulesUnregisterParamHandler({
6018
+ app.delete(getEndpoint('rulesUnregisterBySource').path, createRulesUnregisterParamHandler({
5913
6019
  virtualRuleStore,
5914
6020
  logger,
5915
6021
  onRulesChanged,
5916
6022
  }));
5917
- app.post('/points/delete', createPointsDeleteHandler({ vectorStore, logger }));
5918
- app.post('/rules/reapply', createRulesReapplyHandler({ processor, vectorStore, logger }));
6023
+ app.post(getEndpoint('pointsDelete').path, createPointsDeleteHandler({ vectorStore, logger }));
6024
+ app.post(getEndpoint('rulesReapply').path, createRulesReapplyHandler({ processor, vectorStore, logger }));
5919
6025
  }
5920
6026
  // VCS routes — only registered when a VcsCoordinator is provided
5921
6027
  if (options.vcsCoordinator) {
5922
6028
  const coordinator = options.vcsCoordinator;
5923
- app.get('/vcs/status', createVcsStatusHandler({ coordinator, logger }));
5924
- app.get('/vcs/history', createVcsHistoryHandler({ coordinator, logger }));
5925
- app.get('/vcs/show', createVcsShowHandler({ coordinator, logger }));
5926
- app.get('/vcs/diff', createVcsDiffHandler({ coordinator, logger }));
5927
- app.get('/vcs/check-exclusion', createVcsCheckExclusionHandler({ coordinator, logger }));
5928
- app.post('/vcs/revert', createVcsRevertHandler({ coordinator, logger }));
5929
- app.post('/vcs/exclude', createVcsExcludeHandler({ coordinator, logger }));
6029
+ app.get(getEndpoint('vcsStatus').path, createVcsStatusHandler({ coordinator, logger }));
6030
+ app.get(getEndpoint('vcsHistory').path, createVcsHistoryHandler({ coordinator, logger }));
6031
+ app.get(getEndpoint('vcsShow').path, createVcsShowHandler({ coordinator, logger }));
6032
+ app.get(getEndpoint('vcsDiff').path, createVcsDiffHandler({ coordinator, logger }));
6033
+ app.get(getEndpoint('vcsCheckExclusion').path, createVcsCheckExclusionHandler({ coordinator, logger }));
6034
+ app.post(getEndpoint('vcsRevert').path, createVcsRevertHandler({ coordinator, logger }));
6035
+ app.post(getEndpoint('vcsExclude').path, createVcsExcludeHandler({ coordinator, logger }));
5930
6036
  }
5931
6037
  return app;
5932
6038
  }
@@ -6459,7 +6565,7 @@ async function reloadConfig(state, deps) {
6459
6565
  // Rebuild filesystem watcher if watch config changed
6460
6566
  if (watchConfigChanged(oldConfig, newConfig)) {
6461
6567
  logger.info('Watch config changed, rebuilding filesystem watcher...');
6462
- const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache);
6568
+ const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache, deps.onVcsFileChange, deps.onInitialScanComplete);
6463
6569
  state.watcher = newState.watcher;
6464
6570
  state.gitignoreFilter = newState.gitignoreFilter;
6465
6571
  }
@@ -9140,6 +9246,8 @@ class JeevesWatcher {
9140
9246
  contentHashCache: this.contentHashCache,
9141
9247
  valuesManager: this.valuesManager,
9142
9248
  issuesManager: this.issuesManager,
9249
+ onVcsFileChange: (filePath, event) => this.vcsCoordinator?.onFileChange(filePath, event),
9250
+ onInitialScanComplete: () => this.vcsCoordinator?.onInitialScanComplete(),
9143
9251
  });
9144
9252
  this.config = state.config;
9145
9253
  this.watcher = state.watcher;