@karmaniverous/jeeves-watcher 0.2.3 → 0.2.5

package/dist/mjs/index.js

@@ -1,6 +1,6 @@
 import Fastify from 'fastify';
 import { readdir, stat, rm, readFile, mkdir, writeFile } from 'node:fs/promises';
-import { resolve, dirname, join, extname, basename } from 'node:path';
+import { resolve, dirname, join, relative, extname, basename } from 'node:path';
 import picomatch from 'picomatch';
 import { omit, get } from 'radash';
 import { createHash } from 'node:crypto';
@@ -8,6 +8,8 @@ import { cosmiconfig } from 'cosmiconfig';
 import { z, ZodError } from 'zod';
 import { jsonMapMapSchema, JsonMap } from '@karmaniverous/jsonmap';
 import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
+import { existsSync, statSync, readdirSync, readFileSync } from 'node:fs';
+import ignore from 'ignore';
 import pino from 'pino';
 import { v5 } from 'uuid';
 import * as cheerio from 'cheerio';
@@ -433,6 +435,7 @@ const WATCH_DEFAULTS = {
     stabilityThresholdMs: 500,
     usePolling: false,
     pollIntervalMs: 1000,
+    respectGitignore: true,
 };
 /** Default embedding configuration. */
 const EMBEDDING_DEFAULTS = {
@@ -477,6 +480,11 @@ const watchConfigSchema = z.object({
         .number()
         .optional()
         .describe('Time in milliseconds a file must remain unchanged before processing.'),
+    /** Whether to respect .gitignore files when processing. */
+    respectGitignore: z
+        .boolean()
+        .optional()
+        .describe('Skip files ignored by .gitignore in git repositories. Only applies to repos with a .git directory. Default: true.'),
 });
 /**
  * Configuration watch settings.
@@ -644,6 +652,16 @@ const jeevesWatcherConfigSchema = z.object({
         .number()
         .optional()
         .describe('Timeout in milliseconds for graceful shutdown.'),
+    /** Maximum consecutive system-level failures before triggering fatal error. Default: Infinity. */
+    maxRetries: z
+        .number()
+        .optional()
+        .describe('Maximum consecutive system-level failures before triggering fatal error. Default: Infinity.'),
+    /** Maximum backoff delay in milliseconds for system errors. Default: 60000. */
+    maxBackoffMs: z
+        .number()
+        .optional()
+        .describe('Maximum backoff delay in milliseconds for system errors. Default: 60000.'),
 });
 
 /**
@@ -932,6 +950,212 @@ function createEmbeddingProvider(config, logger) {
     return factory(config, logger);
 }
 
+/**
+ * @module gitignore
+ * Processor-level gitignore filtering. Scans watched paths for `.gitignore` files in git repos, caches parsed patterns, and exposes `isIgnored()` for path checking.
+ */
+/**
+ * Find the git repo root by walking up from `startDir` looking for `.git/`.
+ * Returns `undefined` if no repo is found.
+ */
+function findRepoRoot(startDir) {
+    let dir = resolve(startDir);
+    const root = resolve('/');
+    while (dir !== root) {
+        if (existsSync(join(dir, '.git')) &&
+            statSync(join(dir, '.git')).isDirectory()) {
+            return dir;
+        }
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return undefined;
+}
+/**
+ * Convert a watch path (directory, file path, or glob) to a concrete directory
+ * that can be scanned for a repo root.
+ */
+function watchPathToScanDir(watchPath) {
+    const absPath = resolve(watchPath);
+    try {
+        return statSync(absPath).isDirectory() ? absPath : dirname(absPath);
+    }
+    catch {
+        // ignore
+    }
+    // If this is a glob, fall back to the non-glob prefix.
+    const globMatch = /[*?[{]/.exec(watchPath);
+    if (!globMatch)
+        return undefined;
+    const prefix = watchPath.slice(0, globMatch.index);
+    const trimmed = prefix.trim();
+    const baseDir = trimmed.length === 0
+        ? '.'
+        : trimmed.endsWith('/') || trimmed.endsWith('\\')
+            ? trimmed
+            : dirname(trimmed);
+    const resolved = resolve(baseDir);
+    if (!existsSync(resolved))
+        return undefined;
+    return resolved;
+}
+/**
+ * Recursively find all `.gitignore` files under `dir`.
+ * Skips `.git` and `node_modules` directories for performance.
+ */
+function findGitignoreFiles(dir) {
+    const results = [];
+    const gitignorePath = join(dir, '.gitignore');
+    if (existsSync(gitignorePath)) {
+        results.push(gitignorePath);
+    }
+    let entries;
+    try {
+        entries = readdirSync(dir);
+    }
+    catch {
+        return results;
+    }
+    for (const entry of entries) {
+        if (entry === '.git' || entry === 'node_modules')
+            continue;
+        const fullPath = join(dir, entry);
+        try {
+            if (statSync(fullPath).isDirectory()) {
+                results.push(...findGitignoreFiles(fullPath));
+            }
+        }
+        catch {
+            // Skip inaccessible entries
+        }
+    }
+    return results;
+}
+/**
+ * Parse a `.gitignore` file into an `ignore` instance.
+ */
+function parseGitignore(gitignorePath) {
+    const content = readFileSync(gitignorePath, 'utf8');
+    return ignore().add(content);
+}
+/**
+ * Normalize a path to use forward slashes (required by `ignore` package).
+ */
+function toForwardSlash(p) {
+    return p.replace(/\\/g, '/');
+}
+/**
+ * Processor-level gitignore filter. Checks file paths against the nearest
+ * `.gitignore` chain in git repositories.
+ */
+class GitignoreFilter {
+    repos = new Map();
+    /**
+     * Create a GitignoreFilter by scanning watched paths for `.gitignore` files.
+     *
+     * @param watchPaths - Absolute paths being watched (directories or globs resolved to roots).
+     */
+    constructor(watchPaths) {
+        this.scan(watchPaths);
+    }
+    /**
+     * Scan paths for git repos and their `.gitignore` files.
+     */
+    scan(watchPaths) {
+        this.repos.clear();
+        const scannedDirs = new Set();
+        for (const watchPath of watchPaths) {
+            const scanDir = watchPathToScanDir(watchPath);
+            if (!scanDir)
+                continue;
+            if (scannedDirs.has(scanDir))
+                continue;
+            scannedDirs.add(scanDir);
+            const repoRoot = findRepoRoot(scanDir);
+            if (!repoRoot)
+                continue;
+            if (this.repos.has(repoRoot))
+                continue;
+            const gitignoreFiles = findGitignoreFiles(repoRoot);
+            const entries = gitignoreFiles.map((gf) => ({
+                dir: dirname(gf),
+                ig: parseGitignore(gf),
+            }));
+            // Sort deepest-first so nested `.gitignore` files are checked first
+            entries.sort((a, b) => b.dir.length - a.dir.length);
+            this.repos.set(repoRoot, { root: repoRoot, entries });
+        }
+    }
+    /**
+     * Check whether a file path is ignored by any applicable `.gitignore`.
+     *
+     * @param filePath - Absolute file path to check.
+     * @returns `true` if the file should be ignored.
+     */
+    isIgnored(filePath) {
+        const absPath = resolve(filePath);
+        for (const [, repo] of this.repos) {
+            // Check if file is within this repo
+            const relToRepo = relative(repo.root, absPath);
+            if (relToRepo.startsWith('..') || relToRepo.startsWith(resolve('/'))) {
+                continue;
+            }
+            // Check each `.gitignore` entry (deepest-first)
+            for (const entry of repo.entries) {
+                const relToEntry = relative(entry.dir, absPath);
+                if (relToEntry.startsWith('..'))
+                    continue;
+                const normalized = toForwardSlash(relToEntry);
+                if (entry.ig.ignores(normalized)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    /**
+     * Invalidate and re-parse a specific `.gitignore` file.
+     * Call when a `.gitignore` file is added, changed, or removed.
+     *
+     * @param gitignorePath - Absolute path to the `.gitignore` file that changed.
+     */
+    invalidate(gitignorePath) {
+        const absPath = resolve(gitignorePath);
+        const gitignoreDir = dirname(absPath);
+        for (const [, repo] of this.repos) {
+            const relToRepo = relative(repo.root, gitignoreDir);
+            if (relToRepo.startsWith('..'))
+                continue;
+            // Remove old entry for this directory
+            repo.entries = repo.entries.filter((e) => e.dir !== gitignoreDir);
+            // Re-parse if file still exists
+            if (existsSync(absPath)) {
+                repo.entries.push({ dir: gitignoreDir, ig: parseGitignore(absPath) });
+                // Re-sort deepest-first
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            return;
+        }
+        // If not in any known repo, check if it's in a repo we haven't scanned
+        const repoRoot = findRepoRoot(gitignoreDir);
+        if (repoRoot && existsSync(absPath)) {
+            const entries = [
+                { dir: gitignoreDir, ig: parseGitignore(absPath) },
+            ];
+            if (this.repos.has(repoRoot)) {
+                const repo = this.repos.get(repoRoot);
+                repo.entries.push(entries[0]);
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            else {
+                this.repos.set(repoRoot, { root: repoRoot, entries });
+            }
+        }
+    }
+}
+
 /**
  * @module logger
  * Creates pino logger instances. I/O: optionally writes logs to file via pino/file transport. Defaults to stdout at info level.
@@ -1048,11 +1272,11 @@ async function extractMarkdown(filePath) {
 }
 async function extractPlaintext(filePath) {
     const raw = await readFile(filePath, 'utf8');
-    return { text: raw };
+    return { text: raw.replace(/^\uFEFF/, '') };
 }
 async function extractJson(filePath) {
     const raw = await readFile(filePath, 'utf8');
-    const parsed = JSON.parse(raw);
+    const parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
     const json = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
         ? parsed
         : undefined;
@@ -1074,7 +1298,7 @@ async function extractDocx(filePath) {
 }
 async function extractHtml(filePath) {
     const raw = await readFile(filePath, 'utf8');
-    const $ = cheerio.load(raw);
+    const $ = cheerio.load(raw.replace(/^\uFEFF/, ''));
     $('script, style').remove();
     const text = $('body').text().trim() || $.text().trim();
     return { text };
@@ -1904,6 +2128,112 @@ class VectorStoreClient {
     }
 }
 
+/**
+ * @module health
+ * Tracks consecutive system-level failures and applies exponential backoff.
+ * Triggers fatal error callback when maxRetries is exceeded.
+ */
+/**
+ * Tracks system health via consecutive failure count and exponential backoff.
+ */
+class SystemHealth {
+    consecutiveFailures = 0;
+    maxRetries;
+    maxBackoffMs;
+    baseDelayMs;
+    onFatalError;
+    logger;
+    constructor(options) {
+        this.maxRetries = options.maxRetries ?? Number.POSITIVE_INFINITY;
+        this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+        this.baseDelayMs = options.baseDelayMs ?? 1000;
+        this.onFatalError = options.onFatalError;
+        this.logger = options.logger;
+    }
+    /**
+     * Record a successful system operation. Resets the failure counter.
+     */
+    recordSuccess() {
+        if (this.consecutiveFailures > 0) {
+            this.logger.info({ previousFailures: this.consecutiveFailures }, 'System health recovered');
+        }
+        this.consecutiveFailures = 0;
+    }
+    /**
+     * Record a system-level failure. If maxRetries is exceeded, triggers fatal error.
+     *
+     * @param error - The error that occurred.
+     * @returns Whether the watcher should continue (false = fatal).
+     */
+    recordFailure(error) {
+        this.consecutiveFailures += 1;
+        this.logger.error({
+            consecutiveFailures: this.consecutiveFailures,
+            maxRetries: this.maxRetries,
+            err: normalizeError(error),
+        }, 'System-level failure recorded');
+        if (this.consecutiveFailures >= this.maxRetries) {
+            this.logger.fatal({ consecutiveFailures: this.consecutiveFailures }, 'Maximum retries exceeded, triggering fatal error');
+            if (this.onFatalError) {
+                this.onFatalError(error);
+                return false;
+            }
+            throw error instanceof Error
+                ? error
+                : new Error(`Fatal system error: ${String(error)}`);
+        }
+        return true;
+    }
+    /**
+     * Compute the current backoff delay based on consecutive failures.
+     *
+     * @returns Delay in milliseconds.
+     */
+    get currentBackoffMs() {
+        if (this.consecutiveFailures === 0)
+            return 0;
+        const exp = Math.max(0, this.consecutiveFailures - 1);
+        return Math.min(this.maxBackoffMs, this.baseDelayMs * 2 ** exp);
+    }
+    /**
+     * Sleep for the current backoff duration.
+     *
+     * @param signal - Optional abort signal.
+     */
+    async backoff(signal) {
+        const delay = this.currentBackoffMs;
+        if (delay <= 0)
+            return;
+        this.logger.warn({ delayMs: delay, consecutiveFailures: this.consecutiveFailures }, 'Backing off before next attempt');
+        await new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                cleanup();
+                resolve();
+            }, delay);
+            const onAbort = () => {
+                cleanup();
+                reject(new Error('Backoff aborted'));
+            };
+            const cleanup = () => {
+                clearTimeout(timer);
+                if (signal)
+                    signal.removeEventListener('abort', onAbort);
+            };
+            if (signal) {
+                if (signal.aborted) {
+                    onAbort();
+                    return;
+                }
+                signal.addEventListener('abort', onAbort, { once: true });
+            }
+        });
+    }
+    /** Current consecutive failure count. */
+    get failures() {
+        return this.consecutiveFailures;
+    }
+}
+
 /**
  * @module watcher
  * Filesystem watcher wrapping chokidar. I/O: watches files/directories for add/change/unlink events, enqueues to processing queue.
@@ -1916,6 +2246,8 @@ class FileSystemWatcher {
     queue;
     processor;
     logger;
+    health;
+    gitignoreFilter;
     watcher;
     /**
      * Create a new FileSystemWatcher.
@@ -1924,12 +2256,21 @@ class FileSystemWatcher {
      * @param queue - The event queue.
      * @param processor - The document processor.
      * @param logger - The logger instance.
+     * @param options - Optional health/fatal error options.
      */
-    constructor(config, queue, processor, logger) {
+    constructor(config, queue, processor, logger, options = {}) {
         this.config = config;
         this.queue = queue;
         this.processor = processor;
         this.logger = logger;
+        this.gitignoreFilter = options.gitignoreFilter;
+        const healthOptions = {
+            maxRetries: options.maxRetries,
+            maxBackoffMs: options.maxBackoffMs,
+            onFatalError: options.onFatalError,
+            logger,
+        };
+        this.health = new SystemHealth(healthOptions);
     }
     /**
      * Start watching the filesystem and processing events.
@@ -1945,19 +2286,29 @@ class FileSystemWatcher {
             ignoreInitial: false,
         });
         this.watcher.on('add', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File added');
-            this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.processor.processFile(path));
+            this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('change', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File changed');
-            this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.processor.processFile(path));
+            this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('unlink', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File removed');
-            this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.processor.deleteFile(path));
+            this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
         });
         this.watcher.on('error', (error) => {
             this.logger.error({ err: normalizeError(error) }, 'Watcher error');
+            this.health.recordFailure(error);
         });
         this.queue.process();
         this.logger.info({ paths: this.config.paths }, 'Filesystem watcher started');
@@ -1972,6 +2323,53 @@ class FileSystemWatcher {
             this.logger.info('Filesystem watcher stopped');
         }
     }
+    /**
+     * Get the system health tracker.
+     */
+    get systemHealth() {
+        return this.health;
+    }
+    /**
+     * Check if a path is gitignored and should be skipped.
+     */
+    isGitignored(path) {
+        if (!this.gitignoreFilter)
+            return false;
+        const ignored = this.gitignoreFilter.isIgnored(path);
+        if (ignored) {
+            this.logger.debug({ path }, 'Skipping gitignored file');
+        }
+        return ignored;
+    }
+    /**
+     * If the changed file is a `.gitignore`, invalidate the filter cache.
+     */
+    handleGitignoreChange(path) {
+        if (!this.gitignoreFilter)
+            return;
+        if (path.endsWith('.gitignore')) {
+            this.logger.info({ path }, 'Gitignore file changed, refreshing filter');
+            this.gitignoreFilter.invalidate(path);
+        }
+    }
+    /**
+     * Wrap a processing operation with health tracking.
+     * On success, resets the failure counter.
+     * On failure, records the failure and applies backoff.
+     */
+    async wrapProcessing(fn) {
+        try {
+            await this.health.backoff();
+            await fn();
+            this.health.recordSuccess();
+        }
+        catch (error) {
+            const shouldContinue = this.health.recordFailure(error);
+            if (!shouldContinue) {
+                await this.stop();
+            }
+        }
+    }
 }
 
 /**
@@ -2047,7 +2445,7 @@ const defaultFactories = {
     compileRules,
     createDocumentProcessor: (config, embeddingProvider, vectorStore, compiledRules, logger) => new DocumentProcessor(config, embeddingProvider, vectorStore, compiledRules, logger),
     createEventQueue: (options) => new EventQueue(options),
-    createFileSystemWatcher: (config, queue, processor, logger) => new FileSystemWatcher(config, queue, processor, logger),
+    createFileSystemWatcher: (config, queue, processor, logger, options) => new FileSystemWatcher(config, queue, processor, logger, options),
     createApiServer,
 };
 /**
@@ -2057,6 +2455,7 @@ class JeevesWatcher {
     config;
     configPath;
     factories;
+    runtimeOptions;
     logger;
     watcher;
     queue;
@@ -2069,11 +2468,13 @@ class JeevesWatcher {
      * @param config - The application configuration.
      * @param configPath - Optional config file path to watch for changes.
      * @param factories - Optional component factories (for dependency injection).
+     * @param runtimeOptions - Optional runtime-only options (e.g., onFatalError).
      */
-    constructor(config, configPath, factories = {}) {
+    constructor(config, configPath, factories = {}, runtimeOptions = {}) {
         this.config = config;
         this.configPath = configPath;
         this.factories = { ...defaultFactories, ...factories };
+        this.runtimeOptions = runtimeOptions;
     }
     /**
      * Start the watcher, API server, and all components.
@@ -2106,7 +2507,16 @@ class JeevesWatcher {
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
         this.queue = queue;
-        const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger);
+        const respectGitignore = this.config.watch.respectGitignore ?? true;
+        const gitignoreFilter = respectGitignore
+            ? new GitignoreFilter(this.config.watch.paths)
+            : undefined;
+        const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
+            maxRetries: this.config.maxRetries,
+            maxBackoffMs: this.config.maxBackoffMs,
+            onFatalError: this.runtimeOptions.onFatalError,
+            gitignoreFilter,
+        });
         this.watcher = watcher;
         const server = this.factories.createApiServer({
             processor,
@@ -2211,4 +2621,4 @@ async function startFromConfig(configPath) {
     return app;
 }
 
-export { DocumentProcessor, EventQueue, FileSystemWatcher, JeevesWatcher, VectorStoreClient, apiConfigSchema, applyRules, buildAttributes, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createLogger, deleteMetadata, embeddingConfigSchema, extractText, inferenceRuleSchema, jeevesWatcherConfigSchema, loadConfig, loggingConfigSchema, metadataPath, pointId, readMetadata, startFromConfig, vectorStoreConfigSchema, watchConfigSchema, writeMetadata };
+export { DocumentProcessor, EventQueue, FileSystemWatcher, GitignoreFilter, JeevesWatcher, SystemHealth, VectorStoreClient, apiConfigSchema, applyRules, buildAttributes, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createLogger, deleteMetadata, embeddingConfigSchema, extractText, inferenceRuleSchema, jeevesWatcherConfigSchema, loadConfig, loggingConfigSchema, metadataPath, pointId, readMetadata, startFromConfig, vectorStoreConfigSchema, watchConfigSchema, writeMetadata };

package/package.json

@@ -26,6 +26,7 @@
     "commander": "^14.0.3",
     "cosmiconfig": "*",
     "fastify": "*",
+    "ignore": "^7.0.5",
     "js-yaml": "*",
     "json5": "*",
     "mammoth": "^1.11.0",
@@ -171,5 +172,5 @@
   },
   "type": "module",
   "types": "dist/index.d.ts",
-  "version": "0.2.3"
+  "version": "0.2.5"
 }