@karmaniverous/jeeves-watcher 0.18.8 → 0.18.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/config.schema.json +231 -183
- package/dist/cli/jeeves-watcher/{index-BjghvUnc.js → index-B_g6RfR8.js} +115 -130
- package/dist/cli/jeeves-watcher/index.js +148 -23
- package/dist/index.d.ts +17 -10
- package/dist/index.js +256 -148
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { join, dirname, resolve, relative, extname, basename, isAbsolute, sep } from 'node:path';
|
|
2
2
|
import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, createConfigApplyHandler, substituteEnvVars, DEFAULT_PORTS, getBindAddress, postJson, fetchJson, getPackageVersion } from '@karmaniverous/jeeves';
|
|
3
|
-
import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsAuthorConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
|
|
3
|
+
import { extractWatchPathStrings, inferenceRuleSchema, jeevesWatcherConfigSchema, normalizeWatchPaths, vcsRetentionConfigSchema, vcsAuthorConfigSchema, getEndpoint, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
|
|
4
4
|
export { apiConfigSchema, configWatchConfigSchema, embeddingConfigSchema, inferenceRuleSchema, jeevesWatcherConfigSchema, loggingConfigSchema, vectorStoreConfigSchema, watchConfigSchema } from '@karmaniverous/jeeves-watcher-core';
|
|
5
5
|
import Fastify from 'fastify';
|
|
6
|
-
import { readdir, stat, readFile, writeFile, mkdir, appendFile,
|
|
6
|
+
import { readdir, stat, access, readFile, writeFile, mkdir, appendFile, rm } from 'node:fs/promises';
|
|
7
7
|
import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
|
|
8
8
|
import { existsSync, statSync, readFileSync, readdirSync, mkdirSync, writeFileSync, rmSync, renameSync } from 'node:fs';
|
|
9
9
|
import ignore from 'ignore';
|
|
10
10
|
import picomatch from 'picomatch';
|
|
11
|
+
import { execFile } from 'node:child_process';
|
|
12
|
+
import { promisify } from 'node:util';
|
|
11
13
|
import Ajv from 'ajv';
|
|
12
14
|
import addFormats from 'ajv-formats';
|
|
13
15
|
import { z, ZodError } from 'zod';
|
|
@@ -21,8 +23,6 @@ import { toMarkdown } from 'mdast-util-to-markdown';
|
|
|
21
23
|
import rehypeParse from 'rehype-parse';
|
|
22
24
|
import { unified } from 'unified';
|
|
23
25
|
import yaml from 'js-yaml';
|
|
24
|
-
import { execFile } from 'node:child_process';
|
|
25
|
-
import { promisify } from 'node:util';
|
|
26
26
|
import crypto from 'crypto';
|
|
27
27
|
import { packageDirectorySync } from 'package-directory';
|
|
28
28
|
import { homedir } from 'node:os';
|
|
@@ -421,6 +421,77 @@ async function retry(fn, options) {
|
|
|
421
421
|
throw lastError;
|
|
422
422
|
}
|
|
423
423
|
|
|
424
|
+
/**
|
|
425
|
+
* @module vcs/gitExec
|
|
426
|
+
* Shared git execution utilities.
|
|
427
|
+
*/
|
|
428
|
+
/** Promisified execFile for git commands. */
|
|
429
|
+
const execFileAsync = promisify(execFile);
|
|
430
|
+
/**
|
|
431
|
+
* Stage files via stdin to avoid ENAMETOOLONG on Windows.
|
|
432
|
+
*
|
|
433
|
+
* Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
|
|
434
|
+
* through stdin (NUL-delimited) instead of passed as command-line
|
|
435
|
+
* arguments. This sidesteps the Windows CreateProcessW 32 767-char
|
|
436
|
+
* limit that triggers ENAMETOOLONG when batches contain many files
|
|
437
|
+
* with long absolute paths.
|
|
438
|
+
*
|
|
439
|
+
* @param files - Absolute paths of files to stage.
|
|
440
|
+
* @param cwd - Repository root (working directory for git).
|
|
441
|
+
*/
|
|
442
|
+
function gitAddViaStdin(files, cwd) {
|
|
443
|
+
return new Promise((resolve, reject) => {
|
|
444
|
+
const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
|
|
445
|
+
if (error)
|
|
446
|
+
reject(error);
|
|
447
|
+
else
|
|
448
|
+
resolve();
|
|
449
|
+
});
|
|
450
|
+
if (!child.stdin) {
|
|
451
|
+
reject(new Error('Failed to open stdin for git add'));
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
// Suppress EPIPE — expected if git exits before we finish writing
|
|
455
|
+
child.stdin.on('error', (err) => {
|
|
456
|
+
if (err.code !== 'EPIPE') {
|
|
457
|
+
reject(err);
|
|
458
|
+
}
|
|
459
|
+
});
|
|
460
|
+
child.stdin.end(files.join('\0'));
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Normalize a path for case-insensitive comparison on Windows.
|
|
465
|
+
* On Windows, lowercases the entire path; on other platforms, returns as-is.
|
|
466
|
+
*
|
|
467
|
+
* @param p - The path string to normalize.
|
|
468
|
+
* @param platform - The platform to check against (default: `process.platform`).
|
|
469
|
+
* @returns The normalized path.
|
|
470
|
+
*/
|
|
471
|
+
function normalizePathCase(p, platform = process.platform) {
|
|
472
|
+
return platform === 'win32' ? p.toLowerCase() : p;
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Find the longest-prefix-match root for a normalized path.
|
|
476
|
+
* Roots must be sorted longest-first for correct nested matching.
|
|
477
|
+
*
|
|
478
|
+
* @param roots - Root paths sorted longest-first.
|
|
479
|
+
* @param normalizedPath - Normalized absolute path (forward slashes).
|
|
480
|
+
* @param platform - The platform to check against (default: `process.platform`).
|
|
481
|
+
* @returns The matching root, or undefined.
|
|
482
|
+
*/
|
|
483
|
+
function findRootForPath(roots, normalizedPath, platform = process.platform) {
|
|
484
|
+
const comparePath = normalizePathCase(normalizedPath, platform);
|
|
485
|
+
for (const root of roots) {
|
|
486
|
+
const compareRoot = normalizePathCase(root, platform);
|
|
487
|
+
if (comparePath === compareRoot ||
|
|
488
|
+
comparePath.startsWith(compareRoot + '/')) {
|
|
489
|
+
return root;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
return undefined;
|
|
493
|
+
}
|
|
494
|
+
|
|
424
495
|
/**
|
|
425
496
|
* Best-effort base directory inference for a glob pattern.
|
|
426
497
|
*
|
|
@@ -638,7 +709,7 @@ function groupByRoot(filePaths, watchPaths) {
|
|
|
638
709
|
/** Default page size for prune scroll (smaller than search to reduce connection strain). */
|
|
639
710
|
const PRUNE_SCROLL_PAGE_SIZE = 500;
|
|
640
711
|
/** Max retry attempts for scroll page failures. */
|
|
641
|
-
const SCROLL_RETRY_ATTEMPTS =
|
|
712
|
+
const SCROLL_RETRY_ATTEMPTS = 5;
|
|
642
713
|
/** Base delay for scroll retry backoff (ms). */
|
|
643
714
|
const SCROLL_RETRY_BASE_DELAY_MS = 1000;
|
|
644
715
|
/**
|
|
@@ -664,26 +735,42 @@ async function scrollPageWithRetry(vectorStore, cursor, logger) {
|
|
|
664
735
|
}
|
|
665
736
|
throw lastError;
|
|
666
737
|
}
|
|
738
|
+
/** Concurrency limit for fs.access() disk-existence checks. */
|
|
739
|
+
const FS_ACCESS_CONCURRENCY = 100;
|
|
667
740
|
/**
|
|
668
|
-
* Compute the plan for a prune operation.
|
|
669
|
-
*
|
|
670
|
-
*
|
|
671
|
-
*
|
|
741
|
+
* Compute the plan for a prune operation using a filter-first approach.
|
|
742
|
+
*
|
|
743
|
+
* 1. Enumerate all watched files from the filesystem → `watchedFiles` set.
|
|
744
|
+
* 2. Scroll Qdrant points. For each point:
|
|
745
|
+
* - No `file_path` → orphan.
|
|
746
|
+
* - `file_path` not in `watchedFiles` → orphan (out of watch scope).
|
|
747
|
+
* - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
|
|
748
|
+
* If missing from disk → orphan (deleted file still in index).
|
|
749
|
+
* 3. Batch `fs.access()` checks with bounded concurrency.
|
|
672
750
|
*/
|
|
673
751
|
async function computePrunePlan(deps) {
|
|
674
752
|
const { config, vectorStore, gitignoreFilter, logger } = deps;
|
|
675
753
|
if (!vectorStore) {
|
|
676
754
|
throw new Error('vectorStore is required for prune scope');
|
|
677
755
|
}
|
|
756
|
+
// Step 1: Build set of watched files from filesystem
|
|
757
|
+
const isGitignored = createIsGitignored(gitignoreFilter);
|
|
758
|
+
const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
|
|
759
|
+
const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
|
|
760
|
+
logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
|
|
761
|
+
// Step 2: Scroll Qdrant and classify points
|
|
678
762
|
const seenFiles = new Set();
|
|
679
763
|
const orphanedFiles = new Set();
|
|
764
|
+
const diskCheckFiles = new Set();
|
|
680
765
|
const orphanedIds = [];
|
|
766
|
+
// Map from file_path → point IDs, for points needing disk-existence check
|
|
767
|
+
const diskCheckPointIds = new Map();
|
|
681
768
|
let totalPoints = 0;
|
|
682
769
|
let incomplete = false;
|
|
683
770
|
let cursor;
|
|
684
771
|
try {
|
|
685
|
-
|
|
686
|
-
while (
|
|
772
|
+
let hasMore = true;
|
|
773
|
+
while (hasMore) {
|
|
687
774
|
const page = await scrollPageWithRetry(vectorStore, cursor, logger);
|
|
688
775
|
for (const point of page.points) {
|
|
689
776
|
totalPoints++;
|
|
@@ -692,32 +779,62 @@ async function computePrunePlan(deps) {
|
|
|
692
779
|
orphanedIds.push(point.id);
|
|
693
780
|
continue;
|
|
694
781
|
}
|
|
695
|
-
|
|
696
|
-
|
|
782
|
+
const normalized = normalizePathCase(normalizeSlashes(filePath));
|
|
783
|
+
if (seenFiles.has(normalized)) {
|
|
784
|
+
if (orphanedFiles.has(normalized)) {
|
|
697
785
|
orphanedIds.push(point.id);
|
|
698
786
|
}
|
|
787
|
+
else if (diskCheckFiles.has(normalized)) {
|
|
788
|
+
// Already queued for disk check — track its ID
|
|
789
|
+
const ids = diskCheckPointIds.get(normalized);
|
|
790
|
+
if (ids)
|
|
791
|
+
ids.push(point.id);
|
|
792
|
+
}
|
|
699
793
|
continue;
|
|
700
794
|
}
|
|
701
|
-
seenFiles.add(
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
: false;
|
|
706
|
-
if (!watched || gitignored) {
|
|
707
|
-
orphanedFiles.add(filePath);
|
|
795
|
+
seenFiles.add(normalized);
|
|
796
|
+
if (!watchedFiles.has(normalized)) {
|
|
797
|
+
// Not in watch scope → definitely orphaned
|
|
798
|
+
orphanedFiles.add(normalized);
|
|
708
799
|
orphanedIds.push(point.id);
|
|
709
800
|
}
|
|
801
|
+
else {
|
|
802
|
+
// In watch scope → needs disk-existence check
|
|
803
|
+
diskCheckFiles.add(normalized);
|
|
804
|
+
diskCheckPointIds.set(normalized, [point.id]);
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
// Progress logging every 100K points
|
|
808
|
+
if (totalPoints > 0 && totalPoints % 100000 === 0) {
|
|
809
|
+
logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
|
|
810
|
+
}
|
|
811
|
+
if (!page.nextCursor) {
|
|
812
|
+
hasMore = false;
|
|
813
|
+
}
|
|
814
|
+
else {
|
|
815
|
+
cursor = page.nextCursor;
|
|
710
816
|
}
|
|
711
|
-
if (!page.nextCursor)
|
|
712
|
-
break;
|
|
713
|
-
cursor = page.nextCursor;
|
|
714
817
|
}
|
|
715
818
|
}
|
|
716
819
|
catch (error) {
|
|
717
|
-
// All retries exhausted - return partial results
|
|
718
820
|
logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
|
|
719
821
|
incomplete = true;
|
|
720
822
|
}
|
|
823
|
+
// Step 3: Batch fs.access() checks for files in watch scope
|
|
824
|
+
const filesToCheck = [...diskCheckFiles];
|
|
825
|
+
await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
|
|
826
|
+
try {
|
|
827
|
+
await access(filePath);
|
|
828
|
+
}
|
|
829
|
+
catch {
|
|
830
|
+
// File doesn't exist on disk → orphaned
|
|
831
|
+
orphanedFiles.add(filePath);
|
|
832
|
+
const pointIds = diskCheckPointIds.get(filePath);
|
|
833
|
+
if (pointIds) {
|
|
834
|
+
orphanedIds.push(...pointIds);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
});
|
|
721
838
|
const orphanedFileList = [...orphanedFiles];
|
|
722
839
|
const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
|
|
723
840
|
return {
|
|
@@ -775,6 +892,12 @@ async function executeReindex(deps, scope, path, dryRun = false) {
|
|
|
775
892
|
deps.queue?.resume();
|
|
776
893
|
return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
|
|
777
894
|
}
|
|
895
|
+
// Abort live prune when plan is incomplete — never silently delete partial data
|
|
896
|
+
if (plan.incomplete) {
|
|
897
|
+
deps.queue?.resume();
|
|
898
|
+
logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
|
|
899
|
+
return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
|
|
900
|
+
}
|
|
778
901
|
// Execute prune
|
|
779
902
|
reindexTracker?.start(scope);
|
|
780
903
|
const startTime = Date.now();
|
|
@@ -3369,77 +3492,6 @@ function createSearchHandler(deps) {
|
|
|
3369
3492
|
}, deps.logger, 'Search');
|
|
3370
3493
|
}
|
|
3371
3494
|
|
|
3372
|
-
/**
|
|
3373
|
-
* @module vcs/gitExec
|
|
3374
|
-
* Shared git execution utilities.
|
|
3375
|
-
*/
|
|
3376
|
-
/** Promisified execFile for git commands. */
|
|
3377
|
-
const execFileAsync = promisify(execFile);
|
|
3378
|
-
/**
|
|
3379
|
-
* Stage files via stdin to avoid ENAMETOOLONG on Windows.
|
|
3380
|
-
*
|
|
3381
|
-
* Uses `git add --pathspec-from-file=- --pathspec-file-nul` so the file list is piped
|
|
3382
|
-
* through stdin (NUL-delimited) instead of passed as command-line
|
|
3383
|
-
* arguments. This sidesteps the Windows CreateProcessW 32 767-char
|
|
3384
|
-
* limit that triggers ENAMETOOLONG when batches contain many files
|
|
3385
|
-
* with long absolute paths.
|
|
3386
|
-
*
|
|
3387
|
-
* @param files - Absolute paths of files to stage.
|
|
3388
|
-
* @param cwd - Repository root (working directory for git).
|
|
3389
|
-
*/
|
|
3390
|
-
function gitAddViaStdin(files, cwd) {
|
|
3391
|
-
return new Promise((resolve, reject) => {
|
|
3392
|
-
const child = execFile('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], { cwd }, (error) => {
|
|
3393
|
-
if (error)
|
|
3394
|
-
reject(error);
|
|
3395
|
-
else
|
|
3396
|
-
resolve();
|
|
3397
|
-
});
|
|
3398
|
-
if (!child.stdin) {
|
|
3399
|
-
reject(new Error('Failed to open stdin for git add'));
|
|
3400
|
-
return;
|
|
3401
|
-
}
|
|
3402
|
-
// Suppress EPIPE — expected if git exits before we finish writing
|
|
3403
|
-
child.stdin.on('error', (err) => {
|
|
3404
|
-
if (err.code !== 'EPIPE') {
|
|
3405
|
-
reject(err);
|
|
3406
|
-
}
|
|
3407
|
-
});
|
|
3408
|
-
child.stdin.end(files.join('\0'));
|
|
3409
|
-
});
|
|
3410
|
-
}
|
|
3411
|
-
/**
|
|
3412
|
-
* Normalize a path for case-insensitive comparison on Windows.
|
|
3413
|
-
* On Windows, lowercases the entire path; on other platforms, returns as-is.
|
|
3414
|
-
*
|
|
3415
|
-
* @param p - The path string to normalize.
|
|
3416
|
-
* @param platform - The platform to check against (default: `process.platform`).
|
|
3417
|
-
* @returns The normalized path.
|
|
3418
|
-
*/
|
|
3419
|
-
function normalizePathCase(p, platform = process.platform) {
|
|
3420
|
-
return platform === 'win32' ? p.toLowerCase() : p;
|
|
3421
|
-
}
|
|
3422
|
-
/**
|
|
3423
|
-
* Find the longest-prefix-match root for a normalized path.
|
|
3424
|
-
* Roots must be sorted longest-first for correct nested matching.
|
|
3425
|
-
*
|
|
3426
|
-
* @param roots - Root paths sorted longest-first.
|
|
3427
|
-
* @param normalizedPath - Normalized absolute path (forward slashes).
|
|
3428
|
-
* @param platform - The platform to check against (default: `process.platform`).
|
|
3429
|
-
* @returns The matching root, or undefined.
|
|
3430
|
-
*/
|
|
3431
|
-
function findRootForPath(roots, normalizedPath, platform = process.platform) {
|
|
3432
|
-
const comparePath = normalizePathCase(normalizedPath, platform);
|
|
3433
|
-
for (const root of roots) {
|
|
3434
|
-
const compareRoot = normalizePathCase(root, platform);
|
|
3435
|
-
if (comparePath === compareRoot ||
|
|
3436
|
-
comparePath.startsWith(compareRoot + '/')) {
|
|
3437
|
-
return root;
|
|
3438
|
-
}
|
|
3439
|
-
}
|
|
3440
|
-
return undefined;
|
|
3441
|
-
}
|
|
3442
|
-
|
|
3443
3495
|
/**
|
|
3444
3496
|
* @module vcs/resolveWatchRoot
|
|
3445
3497
|
* Resolves which VCS-enabled watch root(s) own a given absolute path or glob.
|
|
@@ -4956,7 +5008,7 @@ async function pushToRemote(rootPath, remoteUrl, accessToken, pushErrors, logger
|
|
|
4956
5008
|
* Per-root VCS manager for git-backed content versioning.
|
|
4957
5009
|
*
|
|
4958
5010
|
* Each VCS-enabled watch root gets its own VcsManager instance, which owns
|
|
4959
|
-
* a
|
|
5011
|
+
* a throttled commit pipeline: file changes are batched, staged, committed
|
|
4960
5012
|
* (with optional AI-generated messages), and optionally pushed to a remote.
|
|
4961
5013
|
*
|
|
4962
5014
|
* Concurrency: only one commit is in-flight at a time per root. Index.lock
|
|
@@ -4973,8 +5025,9 @@ class VcsManager {
|
|
|
4973
5025
|
pendingReversions = [];
|
|
4974
5026
|
_pushErrors = [];
|
|
4975
5027
|
squashManager;
|
|
4976
|
-
|
|
5028
|
+
throttleTimer;
|
|
4977
5029
|
commitInFlight = Promise.resolve();
|
|
5030
|
+
consecutiveCommitFailures = 0;
|
|
4978
5031
|
started = false;
|
|
4979
5032
|
isBaseline = true;
|
|
4980
5033
|
_lastPushTime = null;
|
|
@@ -5015,7 +5068,7 @@ class VcsManager {
|
|
|
5015
5068
|
this.isBaseline = false;
|
|
5016
5069
|
}
|
|
5017
5070
|
/**
|
|
5018
|
-
* Add a file to the pending set and
|
|
5071
|
+
* Add a file to the pending set and start the throttle timer.
|
|
5019
5072
|
* If the pending set exceeds maxBatchSize, flush immediately.
|
|
5020
5073
|
*
|
|
5021
5074
|
* @param filePath - Absolute path of the changed file.
|
|
@@ -5023,17 +5076,22 @@ class VcsManager {
|
|
|
5023
5076
|
fileChanged(filePath) {
|
|
5024
5077
|
if (!this.started)
|
|
5025
5078
|
return;
|
|
5079
|
+
// Reset circuit breaker on new file change so the system can recover
|
|
5080
|
+
if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
|
|
5081
|
+
this.logger.info({ root: this.rootPath }, 'VCS circuit breaker reset — new file change received, retrying commits');
|
|
5082
|
+
this.consecutiveCommitFailures = 0;
|
|
5083
|
+
}
|
|
5026
5084
|
this.pending.add(filePath);
|
|
5027
5085
|
if (this.pending.size >= this.config.maxBatchSize) {
|
|
5028
|
-
this.
|
|
5086
|
+
this.clearThrottle();
|
|
5029
5087
|
const batch = this.takeBatch(this.config.maxBatchSize);
|
|
5030
5088
|
this.commitInFlight = this.commitInFlight.then(() => this.commitBatch(batch));
|
|
5031
5089
|
if (this.pending.size > 0) {
|
|
5032
|
-
this.
|
|
5090
|
+
this.resetThrottle();
|
|
5033
5091
|
}
|
|
5034
5092
|
return;
|
|
5035
5093
|
}
|
|
5036
|
-
this.
|
|
5094
|
+
this.resetThrottle();
|
|
5037
5095
|
}
|
|
5038
5096
|
/**
|
|
5039
5097
|
* Record reversion metadata so the next commit uses a revert-prefixed message.
|
|
@@ -5058,7 +5116,7 @@ class VcsManager {
|
|
|
5058
5116
|
* Flush all pending files immediately.
|
|
5059
5117
|
*/
|
|
5060
5118
|
async flush() {
|
|
5061
|
-
this.
|
|
5119
|
+
this.clearThrottle();
|
|
5062
5120
|
if (this.pending.size > 0) {
|
|
5063
5121
|
const batch = [...this.pending];
|
|
5064
5122
|
this.pending.clear();
|
|
@@ -5091,31 +5149,65 @@ class VcsManager {
|
|
|
5091
5149
|
return batch;
|
|
5092
5150
|
}
|
|
5093
5151
|
/**
|
|
5094
|
-
* Clear the
|
|
5152
|
+
* Clear the throttle timer.
|
|
5095
5153
|
*/
|
|
5096
|
-
|
|
5097
|
-
if (this.
|
|
5098
|
-
clearTimeout(this.
|
|
5099
|
-
this.
|
|
5154
|
+
clearThrottle() {
|
|
5155
|
+
if (this.throttleTimer !== undefined) {
|
|
5156
|
+
clearTimeout(this.throttleTimer);
|
|
5157
|
+
this.throttleTimer = undefined;
|
|
5100
5158
|
}
|
|
5101
5159
|
}
|
|
5102
5160
|
/**
|
|
5103
|
-
*
|
|
5161
|
+
* Start the throttle timer to fire flush() after commitThrottleMs.
|
|
5162
|
+
* If a timer is already running, do not reset it — this is throttle, not debounce.
|
|
5104
5163
|
*/
|
|
5105
|
-
|
|
5106
|
-
this.
|
|
5107
|
-
|
|
5108
|
-
|
|
5164
|
+
resetThrottle() {
|
|
5165
|
+
if (this.throttleTimer !== undefined)
|
|
5166
|
+
return;
|
|
5167
|
+
this.throttleTimer = setTimeout(() => {
|
|
5168
|
+
this.throttleTimer = undefined;
|
|
5109
5169
|
void this.flush();
|
|
5110
|
-
}, this.config.
|
|
5170
|
+
}, this.config.commitThrottleMs);
|
|
5171
|
+
}
|
|
5172
|
+
/**
|
|
5173
|
+
* Check for and remove stale index.lock files before retrying.
|
|
5174
|
+
*/
|
|
5175
|
+
async removeStaleLock() {
|
|
5176
|
+
const lockPath = join(this.rootPath, '.git', 'index.lock');
|
|
5177
|
+
try {
|
|
5178
|
+
const lockStat = await stat(lockPath);
|
|
5179
|
+
const ageMs = Date.now() - lockStat.mtimeMs;
|
|
5180
|
+
if (ageMs > this.config.staleLockThresholdMs) {
|
|
5181
|
+
await rm(lockPath, { force: true });
|
|
5182
|
+
this.logger.warn({ root: this.rootPath, ageMs }, 'Removed stale index.lock');
|
|
5183
|
+
}
|
|
5184
|
+
}
|
|
5185
|
+
catch (error) {
|
|
5186
|
+
const code = error.code;
|
|
5187
|
+
if (code !== 'ENOENT') {
|
|
5188
|
+
this.logger.warn({ root: this.rootPath, err: normalizeError(error) }, 'Unable to remove stale index.lock');
|
|
5189
|
+
}
|
|
5190
|
+
}
|
|
5111
5191
|
}
|
|
5112
5192
|
/**
|
|
5113
|
-
* Commit a batch of files to the git repo with index.lock retry
|
|
5193
|
+
* Commit a batch of files to the git repo with index.lock retry,
|
|
5194
|
+
* stale lock detection, circuit breaker, and re-queue cap.
|
|
5114
5195
|
*/
|
|
5115
5196
|
async commitBatch(files) {
|
|
5116
5197
|
if (files.length === 0)
|
|
5117
5198
|
return;
|
|
5199
|
+
// Circuit breaker: skip if too many consecutive failures
|
|
5200
|
+
if (this.consecutiveCommitFailures >= this.config.maxConsecutiveFailures) {
|
|
5201
|
+
this.logger.error({
|
|
5202
|
+
root: this.rootPath,
|
|
5203
|
+
consecutiveFailures: this.consecutiveCommitFailures,
|
|
5204
|
+
discardedFiles: files.length,
|
|
5205
|
+
}, 'VCS circuit breaker tripped — pending files discarded until next successful commit');
|
|
5206
|
+
return;
|
|
5207
|
+
}
|
|
5118
5208
|
try {
|
|
5209
|
+
// Check for stale lock before first attempt
|
|
5210
|
+
await this.removeStaleLock();
|
|
5119
5211
|
await retry(async (attempt) => {
|
|
5120
5212
|
await gitAddViaStdin(files, this.rootPath);
|
|
5121
5213
|
// Build message after staging so getStagedDiff can see the changes.
|
|
@@ -5141,12 +5233,22 @@ class VcsManager {
|
|
|
5141
5233
|
this.logger.warn({ root: this.rootPath, attempt, delay: delayMs }, 'index.lock contention, retrying');
|
|
5142
5234
|
},
|
|
5143
5235
|
});
|
|
5236
|
+
// Success — reset circuit breaker
|
|
5237
|
+
this.consecutiveCommitFailures = 0;
|
|
5144
5238
|
}
|
|
5145
5239
|
catch (error) {
|
|
5146
|
-
|
|
5240
|
+
this.consecutiveCommitFailures++;
|
|
5241
|
+
// Re-queue cap: only re-queue up to maxBatchSize files
|
|
5242
|
+
const cap = this.config.maxBatchSize;
|
|
5243
|
+
const toRequeue = files.slice(0, cap);
|
|
5244
|
+
const overflow = files.length - toRequeue.length;
|
|
5245
|
+
for (const f of toRequeue) {
|
|
5147
5246
|
this.pending.add(f);
|
|
5148
5247
|
}
|
|
5149
|
-
|
|
5248
|
+
if (overflow > 0) {
|
|
5249
|
+
this.logger.warn({ root: this.rootPath, overflow, cap }, 'Re-queue cap exceeded — discarded overflow files');
|
|
5250
|
+
}
|
|
5251
|
+
this.logger.warn({ root: this.rootPath, fileCount: toRequeue.length }, 'Re-queued files after commit failure');
|
|
5150
5252
|
this.logger.error({ root: this.rootPath, err: normalizeError(error) }, 'VCS commit failed');
|
|
5151
5253
|
}
|
|
5152
5254
|
}
|
|
@@ -5178,10 +5280,14 @@ class VcsCoordinator {
|
|
|
5178
5280
|
// Deduplicate: skip if another glob already resolved to this root.
|
|
5179
5281
|
if (this.managers.has(rootKey))
|
|
5180
5282
|
continue;
|
|
5283
|
+
const mergedRetention = entry.vcs?.retention ??
|
|
5284
|
+
config.vcs.retention ??
|
|
5285
|
+
vcsRetentionConfigSchema.parse({});
|
|
5181
5286
|
const mergedConfig = {
|
|
5182
5287
|
...config.vcs,
|
|
5183
5288
|
...entry.vcs,
|
|
5184
5289
|
enabled: true,
|
|
5290
|
+
retention: mergedRetention,
|
|
5185
5291
|
};
|
|
5186
5292
|
// Create CommitMessageGenerator if AI commit messages are configured
|
|
5187
5293
|
const cmConfig = mergedConfig.commitMessage;
|
|
@@ -5232,8 +5338,8 @@ class VcsCoordinator {
|
|
|
5232
5338
|
/**
|
|
5233
5339
|
* Signal that the initial filesystem scan is complete.
|
|
5234
5340
|
*
|
|
5235
|
-
* Flushes each manager's
|
|
5236
|
-
* any files still pending in the VCS
|
|
5341
|
+
* Flushes each manager's throttle buffer before ending baseline mode, ensuring
|
|
5342
|
+
* any files still pending in the VCS throttle are committed with a
|
|
5237
5343
|
* `"baseline:"` prefix rather than `"watcher:"`. AI generation is not enabled
|
|
5238
5344
|
* until all managers have completed their flush.
|
|
5239
5345
|
*/
|
|
@@ -5315,10 +5421,10 @@ function createWatcher(config, factories, queue, processor, logger, runtimeOptio
|
|
|
5315
5421
|
* Tear down and rebuild the filesystem watcher with new config.
|
|
5316
5422
|
* Falls back to the old watcher if the new one fails to start.
|
|
5317
5423
|
*/
|
|
5318
|
-
async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache) {
|
|
5424
|
+
async function rebuildWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, oldState, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete) {
|
|
5319
5425
|
try {
|
|
5320
5426
|
await oldState.watcher.stop();
|
|
5321
|
-
const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache);
|
|
5427
|
+
const { watcher: newWatcher, gitignoreFilter: newGitignoreFilter } = createWatcher(newConfig, factories, queue, processor, logger, runtimeOptions, initialScanTracker, contentHashCache, onVcsFileChange, onInitialScanComplete);
|
|
5322
5428
|
newWatcher.start();
|
|
5323
5429
|
logger.info('Filesystem watcher rebuilt successfully');
|
|
5324
5430
|
return { watcher: newWatcher, gitignoreFilter: newGitignoreFilter };
|
|
@@ -5797,36 +5903,36 @@ function createApiServer(options) {
|
|
|
5797
5903
|
};
|
|
5798
5904
|
},
|
|
5799
5905
|
});
|
|
5800
|
-
app.get('
|
|
5906
|
+
app.get(getEndpoint('status').path, withCache(cacheTtlMs, async () => {
|
|
5801
5907
|
const result = await coreStatusHandler();
|
|
5802
5908
|
return result.body;
|
|
5803
5909
|
}));
|
|
5804
|
-
app.post('
|
|
5910
|
+
app.post(getEndpoint('metadata').path, createMetadataHandler({
|
|
5805
5911
|
processor,
|
|
5806
5912
|
getConfig,
|
|
5807
5913
|
logger,
|
|
5808
5914
|
configDir: dirname(configPath),
|
|
5809
5915
|
}));
|
|
5810
|
-
app.post('
|
|
5916
|
+
app.post(getEndpoint('render').path, withCache(cacheTtlMs, createRenderHandler({
|
|
5811
5917
|
processor,
|
|
5812
5918
|
getWatch: () => getConfig().watch,
|
|
5813
5919
|
logger,
|
|
5814
5920
|
})));
|
|
5815
|
-
app.get('
|
|
5921
|
+
app.get(getEndpoint('searchFacets').path, createFacetsHandler({
|
|
5816
5922
|
getConfig,
|
|
5817
5923
|
valuesManager,
|
|
5818
5924
|
configDir: dirname(configPath),
|
|
5819
5925
|
}));
|
|
5820
|
-
app.post('
|
|
5926
|
+
app.post(getEndpoint('scan').path, createScanHandler({
|
|
5821
5927
|
vectorStore,
|
|
5822
5928
|
logger,
|
|
5823
5929
|
}));
|
|
5824
|
-
app.post('
|
|
5930
|
+
app.post(getEndpoint('walk').path, createWalkHandler({
|
|
5825
5931
|
getWatchPaths: () => extractWatchPathStrings(getConfig().watch.paths),
|
|
5826
5932
|
getFileSystemWatcher,
|
|
5827
5933
|
logger,
|
|
5828
5934
|
}));
|
|
5829
|
-
app.post('
|
|
5935
|
+
app.post(getEndpoint('search').path, createSearchHandler({
|
|
5830
5936
|
embeddingProvider,
|
|
5831
5937
|
vectorStore,
|
|
5832
5938
|
logger,
|
|
@@ -5837,12 +5943,12 @@ function createApiServer(options) {
|
|
|
5837
5943
|
: undefined;
|
|
5838
5944
|
},
|
|
5839
5945
|
}));
|
|
5840
|
-
app.post('
|
|
5946
|
+
app.post(getEndpoint('rebuildMetadata').path, createRebuildMetadataHandler({
|
|
5841
5947
|
enrichmentStore: options.enrichmentStore,
|
|
5842
5948
|
vectorStore,
|
|
5843
5949
|
logger,
|
|
5844
5950
|
}));
|
|
5845
|
-
app.post('
|
|
5951
|
+
app.post(getEndpoint('reindex').path, createConfigReindexHandler({
|
|
5846
5952
|
getConfig,
|
|
5847
5953
|
processor,
|
|
5848
5954
|
logger,
|
|
@@ -5853,10 +5959,10 @@ function createApiServer(options) {
|
|
|
5853
5959
|
vectorStore,
|
|
5854
5960
|
queue,
|
|
5855
5961
|
}));
|
|
5856
|
-
app.get('
|
|
5857
|
-
app.get('
|
|
5858
|
-
app.post('
|
|
5859
|
-
app.get('
|
|
5962
|
+
app.get(getEndpoint('issues').path, withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
|
|
5963
|
+
app.get(getEndpoint('configSchema').path, withCache(cacheTtlMs, createConfigSchemaHandler()));
|
|
5964
|
+
app.post(getEndpoint('configMatch').path, createConfigMatchHandler({ getConfig, logger }));
|
|
5965
|
+
app.get(getEndpoint('config').path, withCache(cacheTtlMs, createConfigQueryHandler({
|
|
5860
5966
|
getConfig,
|
|
5861
5967
|
valuesManager,
|
|
5862
5968
|
issuesManager,
|
|
@@ -5866,7 +5972,7 @@ function createApiServer(options) {
|
|
|
5866
5972
|
? () => virtualRuleStore.getAll()
|
|
5867
5973
|
: undefined,
|
|
5868
5974
|
})));
|
|
5869
|
-
app.post('
|
|
5975
|
+
app.post(getEndpoint('configValidate').path, createConfigValidateHandler({
|
|
5870
5976
|
getConfig,
|
|
5871
5977
|
logger,
|
|
5872
5978
|
configDir: dirname(configPath),
|
|
@@ -5880,7 +5986,7 @@ function createApiServer(options) {
|
|
|
5880
5986
|
return Promise.resolve();
|
|
5881
5987
|
},
|
|
5882
5988
|
});
|
|
5883
|
-
app.post('
|
|
5989
|
+
app.post(getEndpoint('configApply').path, async (request, reply) => {
|
|
5884
5990
|
const { patch, replace } = request.body;
|
|
5885
5991
|
const result = await coreConfigApplyHandler({ patch, replace });
|
|
5886
5992
|
return reply.status(result.status).send(result.body);
|
|
@@ -5899,34 +6005,34 @@ function createApiServer(options) {
|
|
|
5899
6005
|
gitignoreFilter,
|
|
5900
6006
|
vectorStore,
|
|
5901
6007
|
});
|
|
5902
|
-
app.post('
|
|
6008
|
+
app.post(getEndpoint('rulesRegister').path, createRulesRegisterHandler({
|
|
5903
6009
|
virtualRuleStore,
|
|
5904
6010
|
logger,
|
|
5905
6011
|
onRulesChanged,
|
|
5906
6012
|
}));
|
|
5907
|
-
app.delete('
|
|
6013
|
+
app.delete(getEndpoint('rulesUnregister').path, createRulesUnregisterHandler({
|
|
5908
6014
|
virtualRuleStore,
|
|
5909
6015
|
logger,
|
|
5910
6016
|
onRulesChanged,
|
|
5911
6017
|
}));
|
|
5912
|
-
app.delete('
|
|
6018
|
+
app.delete(getEndpoint('rulesUnregisterBySource').path, createRulesUnregisterParamHandler({
|
|
5913
6019
|
virtualRuleStore,
|
|
5914
6020
|
logger,
|
|
5915
6021
|
onRulesChanged,
|
|
5916
6022
|
}));
|
|
5917
|
-
app.post('
|
|
5918
|
-
app.post('
|
|
6023
|
+
app.post(getEndpoint('pointsDelete').path, createPointsDeleteHandler({ vectorStore, logger }));
|
|
6024
|
+
app.post(getEndpoint('rulesReapply').path, createRulesReapplyHandler({ processor, vectorStore, logger }));
|
|
5919
6025
|
}
|
|
5920
6026
|
// VCS routes — only registered when a VcsCoordinator is provided
|
|
5921
6027
|
if (options.vcsCoordinator) {
|
|
5922
6028
|
const coordinator = options.vcsCoordinator;
|
|
5923
|
-
app.get('
|
|
5924
|
-
app.get('
|
|
5925
|
-
app.get('
|
|
5926
|
-
app.get('
|
|
5927
|
-
app.get('
|
|
5928
|
-
app.post('
|
|
5929
|
-
app.post('
|
|
6029
|
+
app.get(getEndpoint('vcsStatus').path, createVcsStatusHandler({ coordinator, logger }));
|
|
6030
|
+
app.get(getEndpoint('vcsHistory').path, createVcsHistoryHandler({ coordinator, logger }));
|
|
6031
|
+
app.get(getEndpoint('vcsShow').path, createVcsShowHandler({ coordinator, logger }));
|
|
6032
|
+
app.get(getEndpoint('vcsDiff').path, createVcsDiffHandler({ coordinator, logger }));
|
|
6033
|
+
app.get(getEndpoint('vcsCheckExclusion').path, createVcsCheckExclusionHandler({ coordinator, logger }));
|
|
6034
|
+
app.post(getEndpoint('vcsRevert').path, createVcsRevertHandler({ coordinator, logger }));
|
|
6035
|
+
app.post(getEndpoint('vcsExclude').path, createVcsExcludeHandler({ coordinator, logger }));
|
|
5930
6036
|
}
|
|
5931
6037
|
return app;
|
|
5932
6038
|
}
|
|
@@ -6459,7 +6565,7 @@ async function reloadConfig(state, deps) {
|
|
|
6459
6565
|
// Rebuild filesystem watcher if watch config changed
|
|
6460
6566
|
if (watchConfigChanged(oldConfig, newConfig)) {
|
|
6461
6567
|
logger.info('Watch config changed, rebuilding filesystem watcher...');
|
|
6462
|
-
const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache);
|
|
6568
|
+
const newState = await rebuildWatcher(newConfig, deps.factories, deps.queue, processor, logger, deps.runtimeOptions, { watcher: state.watcher, gitignoreFilter: state.gitignoreFilter }, deps.initialScanTracker, deps.contentHashCache, deps.onVcsFileChange, deps.onInitialScanComplete);
|
|
6463
6569
|
state.watcher = newState.watcher;
|
|
6464
6570
|
state.gitignoreFilter = newState.gitignoreFilter;
|
|
6465
6571
|
}
|
|
@@ -9140,6 +9246,8 @@ class JeevesWatcher {
|
|
|
9140
9246
|
contentHashCache: this.contentHashCache,
|
|
9141
9247
|
valuesManager: this.valuesManager,
|
|
9142
9248
|
issuesManager: this.issuesManager,
|
|
9249
|
+
onVcsFileChange: (filePath, event) => this.vcsCoordinator?.onFileChange(filePath, event),
|
|
9250
|
+
onInitialScanComplete: () => this.vcsCoordinator?.onInitialScanComplete(),
|
|
9143
9251
|
});
|
|
9144
9252
|
this.config = state.config;
|
|
9145
9253
|
this.watcher = state.watcher;
|