@karmaniverous/jeeves-watcher 0.2.3 → 0.2.5

package/dist/index.iife.js

@@ -1,4 +1,4 @@
-(function (exports, Fastify, promises, node_path, picomatch, radash, node_crypto, cosmiconfig, zod, jsonmap, googleGenai, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest, chokidar) {
+(function (exports, Fastify, promises, node_path, picomatch, radash, node_crypto, cosmiconfig, zod, jsonmap, googleGenai, node_fs, ignore, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest, chokidar) {
     'use strict';
 
     function _interopNamespaceDefault(e) {
@@ -434,6 +434,7 @@
         stabilityThresholdMs: 500,
         usePolling: false,
         pollIntervalMs: 1000,
+        respectGitignore: true,
     };
     /** Default embedding configuration. */
     const EMBEDDING_DEFAULTS = {
@@ -478,6 +479,11 @@
             .number()
             .optional()
             .describe('Time in milliseconds a file must remain unchanged before processing.'),
+        /** Whether to respect .gitignore files when processing. */
+        respectGitignore: zod.z
+            .boolean()
+            .optional()
+            .describe('Skip files ignored by .gitignore in git repositories. Only applies to repos with a .git directory. Default: true.'),
     });
     /**
      * Configuration watch settings.
@@ -645,6 +651,16 @@
             .number()
             .optional()
             .describe('Timeout in milliseconds for graceful shutdown.'),
+        /** Maximum consecutive system-level failures before triggering fatal error. Default: Infinity. */
+        maxRetries: zod.z
+            .number()
+            .optional()
+            .describe('Maximum consecutive system-level failures before triggering fatal error. Default: Infinity.'),
+        /** Maximum backoff delay in milliseconds for system errors. Default: 60000. */
+        maxBackoffMs: zod.z
+            .number()
+            .optional()
+            .describe('Maximum backoff delay in milliseconds for system errors. Default: 60000.'),
     });
 
     /**
@@ -933,6 +949,212 @@
         return factory(config, logger);
     }
 
+    /**
+     * @module gitignore
+     * Processor-level gitignore filtering. Scans watched paths for `.gitignore` files in git repos, caches parsed patterns, and exposes `isIgnored()` for path checking.
+     */
+    /**
+     * Find the git repo root by walking up from `startDir` looking for `.git/`.
+     * Returns `undefined` if no repo is found.
+     */
+    function findRepoRoot(startDir) {
+        let dir = node_path.resolve(startDir);
+        const root = node_path.resolve('/');
+        while (dir !== root) {
+            if (node_fs.existsSync(node_path.join(dir, '.git')) &&
+                node_fs.statSync(node_path.join(dir, '.git')).isDirectory()) {
+                return dir;
+            }
+            const parent = node_path.dirname(dir);
+            if (parent === dir)
+                break;
+            dir = parent;
+        }
+        return undefined;
+    }
+    /**
+     * Convert a watch path (directory, file path, or glob) to a concrete directory
+     * that can be scanned for a repo root.
+     */
+    function watchPathToScanDir(watchPath) {
+        const absPath = node_path.resolve(watchPath);
+        try {
+            return node_fs.statSync(absPath).isDirectory() ? absPath : node_path.dirname(absPath);
+        }
+        catch {
+            // ignore
+        }
+        // If this is a glob, fall back to the non-glob prefix.
+        const globMatch = /[*?[{]/.exec(watchPath);
+        if (!globMatch)
+            return undefined;
+        const prefix = watchPath.slice(0, globMatch.index);
+        const trimmed = prefix.trim();
+        const baseDir = trimmed.length === 0
+            ? '.'
+            : trimmed.endsWith('/') || trimmed.endsWith('\\')
+                ? trimmed
+                : node_path.dirname(trimmed);
+        const resolved = node_path.resolve(baseDir);
+        if (!node_fs.existsSync(resolved))
+            return undefined;
+        return resolved;
+    }
+    /**
+     * Recursively find all `.gitignore` files under `dir`.
+     * Skips `.git` and `node_modules` directories for performance.
+     */
+    function findGitignoreFiles(dir) {
+        const results = [];
+        const gitignorePath = node_path.join(dir, '.gitignore');
+        if (node_fs.existsSync(gitignorePath)) {
+            results.push(gitignorePath);
+        }
+        let entries;
+        try {
+            entries = node_fs.readdirSync(dir);
+        }
+        catch {
+            return results;
+        }
+        for (const entry of entries) {
+            if (entry === '.git' || entry === 'node_modules')
+                continue;
+            const fullPath = node_path.join(dir, entry);
+            try {
+                if (node_fs.statSync(fullPath).isDirectory()) {
+                    results.push(...findGitignoreFiles(fullPath));
+                }
+            }
+            catch {
+                // Skip inaccessible entries
+            }
+        }
+        return results;
+    }
+    /**
+     * Parse a `.gitignore` file into an `ignore` instance.
+     */
+    function parseGitignore(gitignorePath) {
+        const content = node_fs.readFileSync(gitignorePath, 'utf8');
+        return ignore().add(content);
+    }
+    /**
+     * Normalize a path to use forward slashes (required by `ignore` package).
+     */
+    function toForwardSlash(p) {
+        return p.replace(/\\/g, '/');
+    }
+    /**
+     * Processor-level gitignore filter. Checks file paths against the nearest
+     * `.gitignore` chain in git repositories.
+     */
+    class GitignoreFilter {
+        repos = new Map();
+        /**
+         * Create a GitignoreFilter by scanning watched paths for `.gitignore` files.
+         *
+         * @param watchPaths - Absolute paths being watched (directories or globs resolved to roots).
+         */
+        constructor(watchPaths) {
+            this.scan(watchPaths);
+        }
+        /**
+         * Scan paths for git repos and their `.gitignore` files.
+         */
+        scan(watchPaths) {
+            this.repos.clear();
+            const scannedDirs = new Set();
+            for (const watchPath of watchPaths) {
+                const scanDir = watchPathToScanDir(watchPath);
+                if (!scanDir)
+                    continue;
+                if (scannedDirs.has(scanDir))
+                    continue;
+                scannedDirs.add(scanDir);
+                const repoRoot = findRepoRoot(scanDir);
+                if (!repoRoot)
+                    continue;
+                if (this.repos.has(repoRoot))
+                    continue;
+                const gitignoreFiles = findGitignoreFiles(repoRoot);
+                const entries = gitignoreFiles.map((gf) => ({
+                    dir: node_path.dirname(gf),
+                    ig: parseGitignore(gf),
+                }));
+                // Sort deepest-first so nested `.gitignore` files are checked first
+                entries.sort((a, b) => b.dir.length - a.dir.length);
+                this.repos.set(repoRoot, { root: repoRoot, entries });
+            }
+        }
+        /**
+         * Check whether a file path is ignored by any applicable `.gitignore`.
+         *
+         * @param filePath - Absolute file path to check.
+         * @returns `true` if the file should be ignored.
+         */
+        isIgnored(filePath) {
+            const absPath = node_path.resolve(filePath);
+            for (const [, repo] of this.repos) {
+                // Check if file is within this repo
+                const relToRepo = node_path.relative(repo.root, absPath);
+                if (relToRepo.startsWith('..') || relToRepo.startsWith(node_path.resolve('/'))) {
+                    continue;
+                }
+                // Check each `.gitignore` entry (deepest-first)
+                for (const entry of repo.entries) {
+                    const relToEntry = node_path.relative(entry.dir, absPath);
+                    if (relToEntry.startsWith('..'))
+                        continue;
+                    const normalized = toForwardSlash(relToEntry);
+                    if (entry.ig.ignores(normalized)) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+        /**
+         * Invalidate and re-parse a specific `.gitignore` file.
+         * Call when a `.gitignore` file is added, changed, or removed.
+         *
+         * @param gitignorePath - Absolute path to the `.gitignore` file that changed.
+         */
+        invalidate(gitignorePath) {
+            const absPath = node_path.resolve(gitignorePath);
+            const gitignoreDir = node_path.dirname(absPath);
+            for (const [, repo] of this.repos) {
+                const relToRepo = node_path.relative(repo.root, gitignoreDir);
+                if (relToRepo.startsWith('..'))
+                    continue;
+                // Remove old entry for this directory
+                repo.entries = repo.entries.filter((e) => e.dir !== gitignoreDir);
+                // Re-parse if file still exists
+                if (node_fs.existsSync(absPath)) {
+                    repo.entries.push({ dir: gitignoreDir, ig: parseGitignore(absPath) });
+                    // Re-sort deepest-first
+                    repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+                }
+                return;
+            }
+            // If not in any known repo, check if it's in a repo we haven't scanned
+            const repoRoot = findRepoRoot(gitignoreDir);
+            if (repoRoot && node_fs.existsSync(absPath)) {
+                const entries = [
+                    { dir: gitignoreDir, ig: parseGitignore(absPath) },
+                ];
+                if (this.repos.has(repoRoot)) {
+                    const repo = this.repos.get(repoRoot);
+                    repo.entries.push(entries[0]);
+                    repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+                }
+                else {
+                    this.repos.set(repoRoot, { root: repoRoot, entries });
+                }
+            }
+        }
+    }
+
     /**
      * @module logger
      * Creates pino logger instances. I/O: optionally writes logs to file via pino/file transport. Defaults to stdout at info level.
@@ -1049,11 +1271,11 @@
     }
     async function extractPlaintext(filePath) {
         const raw = await promises.readFile(filePath, 'utf8');
-        return { text: raw };
+        return { text: raw.replace(/^\uFEFF/, '') };
     }
     async function extractJson(filePath) {
         const raw = await promises.readFile(filePath, 'utf8');
-        const parsed = JSON.parse(raw);
+        const parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
         const json = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
             ? parsed
             : undefined;
@@ -1075,7 +1297,7 @@
     }
     async function extractHtml(filePath) {
         const raw = await promises.readFile(filePath, 'utf8');
-        const $ = cheerio__namespace.load(raw);
+        const $ = cheerio__namespace.load(raw.replace(/^\uFEFF/, ''));
         $('script, style').remove();
         const text = $('body').text().trim() || $.text().trim();
         return { text };
@@ -1905,6 +2127,112 @@
         }
     }
 
+    /**
+     * @module health
+     * Tracks consecutive system-level failures and applies exponential backoff.
+     * Triggers fatal error callback when maxRetries is exceeded.
+     */
+    /**
+     * Tracks system health via consecutive failure count and exponential backoff.
+     */
+    class SystemHealth {
+        consecutiveFailures = 0;
+        maxRetries;
+        maxBackoffMs;
+        baseDelayMs;
+        onFatalError;
+        logger;
+        constructor(options) {
+            this.maxRetries = options.maxRetries ?? Number.POSITIVE_INFINITY;
+            this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+            this.baseDelayMs = options.baseDelayMs ?? 1000;
+            this.onFatalError = options.onFatalError;
+            this.logger = options.logger;
+        }
+        /**
+         * Record a successful system operation. Resets the failure counter.
+         */
+        recordSuccess() {
+            if (this.consecutiveFailures > 0) {
+                this.logger.info({ previousFailures: this.consecutiveFailures }, 'System health recovered');
+            }
+            this.consecutiveFailures = 0;
+        }
+        /**
+         * Record a system-level failure. If maxRetries is exceeded, triggers fatal error.
+         *
+         * @param error - The error that occurred.
+         * @returns Whether the watcher should continue (false = fatal).
+         */
+        recordFailure(error) {
+            this.consecutiveFailures += 1;
+            this.logger.error({
+                consecutiveFailures: this.consecutiveFailures,
+                maxRetries: this.maxRetries,
+                err: normalizeError(error),
+            }, 'System-level failure recorded');
+            if (this.consecutiveFailures >= this.maxRetries) {
+                this.logger.fatal({ consecutiveFailures: this.consecutiveFailures }, 'Maximum retries exceeded, triggering fatal error');
+                if (this.onFatalError) {
+                    this.onFatalError(error);
+                    return false;
+                }
+                throw error instanceof Error
+                    ? error
+                    : new Error(`Fatal system error: ${String(error)}`);
+            }
+            return true;
+        }
+        /**
+         * Compute the current backoff delay based on consecutive failures.
+         *
+         * @returns Delay in milliseconds.
+         */
+        get currentBackoffMs() {
+            if (this.consecutiveFailures === 0)
+                return 0;
+            const exp = Math.max(0, this.consecutiveFailures - 1);
+            return Math.min(this.maxBackoffMs, this.baseDelayMs * 2 ** exp);
+        }
+        /**
+         * Sleep for the current backoff duration.
+         *
+         * @param signal - Optional abort signal.
+         */
+        async backoff(signal) {
+            const delay = this.currentBackoffMs;
+            if (delay <= 0)
+                return;
+            this.logger.warn({ delayMs: delay, consecutiveFailures: this.consecutiveFailures }, 'Backing off before next attempt');
+            await new Promise((resolve, reject) => {
+                const timer = setTimeout(() => {
+                    cleanup();
+                    resolve();
+                }, delay);
+                const onAbort = () => {
+                    cleanup();
+                    reject(new Error('Backoff aborted'));
+                };
+                const cleanup = () => {
+                    clearTimeout(timer);
+                    if (signal)
+                        signal.removeEventListener('abort', onAbort);
+                };
+                if (signal) {
+                    if (signal.aborted) {
+                        onAbort();
+                        return;
+                    }
+                    signal.addEventListener('abort', onAbort, { once: true });
+                }
+            });
+        }
+        /** Current consecutive failure count. */
+        get failures() {
+            return this.consecutiveFailures;
+        }
+    }
+
     /**
      * @module watcher
      * Filesystem watcher wrapping chokidar. I/O: watches files/directories for add/change/unlink events, enqueues to processing queue.
@@ -1917,6 +2245,8 @@
         queue;
         processor;
         logger;
+        health;
+        gitignoreFilter;
         watcher;
         /**
          * Create a new FileSystemWatcher.
@@ -1925,12 +2255,21 @@
          * @param queue - The event queue.
          * @param processor - The document processor.
          * @param logger - The logger instance.
+         * @param options - Optional health/fatal error options.
          */
-        constructor(config, queue, processor, logger) {
+        constructor(config, queue, processor, logger, options = {}) {
             this.config = config;
             this.queue = queue;
             this.processor = processor;
             this.logger = logger;
+            this.gitignoreFilter = options.gitignoreFilter;
+            const healthOptions = {
+                maxRetries: options.maxRetries,
+                maxBackoffMs: options.maxBackoffMs,
+                onFatalError: options.onFatalError,
+                logger,
+            };
+            this.health = new SystemHealth(healthOptions);
         }
         /**
          * Start watching the filesystem and processing events.
@@ -1946,19 +2285,29 @@
                 ignoreInitial: false,
             });
             this.watcher.on('add', (path) => {
+                this.handleGitignoreChange(path);
+                if (this.isGitignored(path))
+                    return;
                 this.logger.debug({ path }, 'File added');
-                this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.processor.processFile(path));
+                this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
             });
             this.watcher.on('change', (path) => {
+                this.handleGitignoreChange(path);
+                if (this.isGitignored(path))
+                    return;
                 this.logger.debug({ path }, 'File changed');
-                this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.processor.processFile(path));
+                this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
             });
             this.watcher.on('unlink', (path) => {
+                this.handleGitignoreChange(path);
+                if (this.isGitignored(path))
+                    return;
                 this.logger.debug({ path }, 'File removed');
-                this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.processor.deleteFile(path));
+                this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
             });
             this.watcher.on('error', (error) => {
                 this.logger.error({ err: normalizeError(error) }, 'Watcher error');
+                this.health.recordFailure(error);
             });
             this.queue.process();
             this.logger.info({ paths: this.config.paths }, 'Filesystem watcher started');
@@ -1973,6 +2322,53 @@
                 this.logger.info('Filesystem watcher stopped');
             }
         }
+        /**
+         * Get the system health tracker.
+         */
+        get systemHealth() {
+            return this.health;
+        }
+        /**
+         * Check if a path is gitignored and should be skipped.
+         */
+        isGitignored(path) {
+            if (!this.gitignoreFilter)
+                return false;
+            const ignored = this.gitignoreFilter.isIgnored(path);
+            if (ignored) {
+                this.logger.debug({ path }, 'Skipping gitignored file');
+            }
+            return ignored;
+        }
+        /**
+         * If the changed file is a `.gitignore`, invalidate the filter cache.
+         */
+        handleGitignoreChange(path) {
+            if (!this.gitignoreFilter)
+                return;
+            if (path.endsWith('.gitignore')) {
+                this.logger.info({ path }, 'Gitignore file changed, refreshing filter');
+                this.gitignoreFilter.invalidate(path);
+            }
+        }
+        /**
+         * Wrap a processing operation with health tracking.
+         * On success, resets the failure counter.
+         * On failure, records the failure and applies backoff.
+         */
+        async wrapProcessing(fn) {
+            try {
+                await this.health.backoff();
+                await fn();
+                this.health.recordSuccess();
+            }
+            catch (error) {
+                const shouldContinue = this.health.recordFailure(error);
+                if (!shouldContinue) {
+                    await this.stop();
+                }
+            }
+        }
     }
 
     /**
@@ -2048,7 +2444,7 @@
         compileRules,
         createDocumentProcessor: (config, embeddingProvider, vectorStore, compiledRules, logger) => new DocumentProcessor(config, embeddingProvider, vectorStore, compiledRules, logger),
         createEventQueue: (options) => new EventQueue(options),
-        createFileSystemWatcher: (config, queue, processor, logger) => new FileSystemWatcher(config, queue, processor, logger),
+        createFileSystemWatcher: (config, queue, processor, logger, options) => new FileSystemWatcher(config, queue, processor, logger, options),
         createApiServer,
     };
     /**
@@ -2058,6 +2454,7 @@
         config;
         configPath;
         factories;
+        runtimeOptions;
         logger;
         watcher;
         queue;
@@ -2070,11 +2467,13 @@
          * @param config - The application configuration.
          * @param configPath - Optional config file path to watch for changes.
          * @param factories - Optional component factories (for dependency injection).
+         * @param runtimeOptions - Optional runtime-only options (e.g., onFatalError).
          */
-        constructor(config, configPath, factories = {}) {
+        constructor(config, configPath, factories = {}, runtimeOptions = {}) {
             this.config = config;
             this.configPath = configPath;
             this.factories = { ...defaultFactories, ...factories };
+            this.runtimeOptions = runtimeOptions;
         }
         /**
          * Start the watcher, API server, and all components.
@@ -2107,7 +2506,16 @@
                 rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
             });
             this.queue = queue;
-            const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger);
+            const respectGitignore = this.config.watch.respectGitignore ?? true;
+            const gitignoreFilter = respectGitignore
+                ? new GitignoreFilter(this.config.watch.paths)
+                : undefined;
+            const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
+                maxRetries: this.config.maxRetries,
+                maxBackoffMs: this.config.maxBackoffMs,
+                onFatalError: this.runtimeOptions.onFatalError,
+                gitignoreFilter,
+            });
             this.watcher = watcher;
             const server = this.factories.createApiServer({
                 processor,
@@ -2215,7 +2623,9 @@
     exports.DocumentProcessor = DocumentProcessor;
     exports.EventQueue = EventQueue;
     exports.FileSystemWatcher = FileSystemWatcher;
+    exports.GitignoreFilter = GitignoreFilter;
     exports.JeevesWatcher = JeevesWatcher;
+    exports.SystemHealth = SystemHealth;
     exports.VectorStoreClient = VectorStoreClient;
     exports.apiConfigSchema = apiConfigSchema;
     exports.applyRules = applyRules;
@@ -2241,4 +2651,4 @@
     exports.watchConfigSchema = watchConfigSchema;
     exports.writeMetadata = writeMetadata;
 
-})(this["jeeves-watcher"] = this["jeeves-watcher"] || {}, Fastify, promises, node_path, picomatch, radash, node_crypto, cosmiconfig, zod, jsonmap, googleGenai, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest, chokidar);
+})(this["jeeves-watcher"] = this["jeeves-watcher"] || {}, Fastify, promises, node_path, picomatch, radash, node_crypto, cosmiconfig, zod, jsonmap, googleGenai, node_fs, ignore, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest, chokidar);