@karmaniverous/jeeves-watcher 0.2.4 → 0.2.5

package/dist/mjs/index.js

@@ -1,6 +1,6 @@
 import Fastify from 'fastify';
 import { readdir, stat, rm, readFile, mkdir, writeFile } from 'node:fs/promises';
-import { resolve, dirname, join, extname, basename } from 'node:path';
+import { resolve, dirname, join, relative, extname, basename } from 'node:path';
 import picomatch from 'picomatch';
 import { omit, get } from 'radash';
 import { createHash } from 'node:crypto';
@@ -8,6 +8,8 @@ import { cosmiconfig } from 'cosmiconfig';
 import { z, ZodError } from 'zod';
 import { jsonMapMapSchema, JsonMap } from '@karmaniverous/jsonmap';
 import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
+import { existsSync, statSync, readdirSync, readFileSync } from 'node:fs';
+import ignore from 'ignore';
 import pino from 'pino';
 import { v5 } from 'uuid';
 import * as cheerio from 'cheerio';
@@ -433,6 +435,7 @@ const WATCH_DEFAULTS = {
     stabilityThresholdMs: 500,
     usePolling: false,
     pollIntervalMs: 1000,
+    respectGitignore: true,
 };
 /** Default embedding configuration. */
 const EMBEDDING_DEFAULTS = {
@@ -477,6 +480,11 @@ const watchConfigSchema = z.object({
         .number()
         .optional()
         .describe('Time in milliseconds a file must remain unchanged before processing.'),
+    /** Whether to respect .gitignore files when processing. */
+    respectGitignore: z
+        .boolean()
+        .optional()
+        .describe('Skip files ignored by .gitignore in git repositories. Only applies to repos with a .git directory. Default: true.'),
 });
 /**
  * Configuration watch settings.
@@ -942,6 +950,212 @@ function createEmbeddingProvider(config, logger) {
     return factory(config, logger);
 }
 
+/**
+ * @module gitignore
+ * Processor-level gitignore filtering. Scans watched paths for `.gitignore` files in git repos, caches parsed patterns, and exposes `isIgnored()` for path checking.
+ */
+/**
+ * Find the git repo root by walking up from `startDir` looking for `.git/`.
+ * Returns `undefined` if no repo is found.
+ */
+function findRepoRoot(startDir) {
+    let dir = resolve(startDir);
+    const root = resolve('/');
+    while (dir !== root) {
+        if (existsSync(join(dir, '.git')) &&
+            statSync(join(dir, '.git')).isDirectory()) {
+            return dir;
+        }
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return undefined;
+}
+/**
+ * Convert a watch path (directory, file path, or glob) to a concrete directory
+ * that can be scanned for a repo root.
+ */
+function watchPathToScanDir(watchPath) {
+    const absPath = resolve(watchPath);
+    try {
+        return statSync(absPath).isDirectory() ? absPath : dirname(absPath);
+    }
+    catch {
+        // ignore
+    }
+    // If this is a glob, fall back to the non-glob prefix.
+    const globMatch = /[*?[{]/.exec(watchPath);
+    if (!globMatch)
+        return undefined;
+    const prefix = watchPath.slice(0, globMatch.index);
+    const trimmed = prefix.trim();
+    const baseDir = trimmed.length === 0
+        ? '.'
+        : trimmed.endsWith('/') || trimmed.endsWith('\\')
+            ? trimmed
+            : dirname(trimmed);
+    const resolved = resolve(baseDir);
+    if (!existsSync(resolved))
+        return undefined;
+    return resolved;
+}
+/**
+ * Recursively find all `.gitignore` files under `dir`.
+ * Skips `.git` and `node_modules` directories for performance.
+ */
+function findGitignoreFiles(dir) {
+    const results = [];
+    const gitignorePath = join(dir, '.gitignore');
+    if (existsSync(gitignorePath)) {
+        results.push(gitignorePath);
+    }
+    let entries;
+    try {
+        entries = readdirSync(dir);
+    }
+    catch {
+        return results;
+    }
+    for (const entry of entries) {
+        if (entry === '.git' || entry === 'node_modules')
+            continue;
+        const fullPath = join(dir, entry);
+        try {
+            if (statSync(fullPath).isDirectory()) {
+                results.push(...findGitignoreFiles(fullPath));
+            }
+        }
+        catch {
+            // Skip inaccessible entries
+        }
+    }
+    return results;
+}
+/**
+ * Parse a `.gitignore` file into an `ignore` instance.
+ */
+function parseGitignore(gitignorePath) {
+    const content = readFileSync(gitignorePath, 'utf8');
+    return ignore().add(content);
+}
+/**
+ * Normalize a path to use forward slashes (required by `ignore` package).
+ */
+function toForwardSlash(p) {
+    return p.replace(/\\/g, '/');
+}
+/**
+ * Processor-level gitignore filter. Checks file paths against the nearest
+ * `.gitignore` chain in git repositories.
+ */
+class GitignoreFilter {
+    repos = new Map();
+    /**
+     * Create a GitignoreFilter by scanning watched paths for `.gitignore` files.
+     *
+     * @param watchPaths - Absolute paths being watched (directories or globs resolved to roots).
+     */
+    constructor(watchPaths) {
+        this.scan(watchPaths);
+    }
+    /**
+     * Scan paths for git repos and their `.gitignore` files.
+     */
+    scan(watchPaths) {
+        this.repos.clear();
+        const scannedDirs = new Set();
+        for (const watchPath of watchPaths) {
+            const scanDir = watchPathToScanDir(watchPath);
+            if (!scanDir)
+                continue;
+            if (scannedDirs.has(scanDir))
+                continue;
+            scannedDirs.add(scanDir);
+            const repoRoot = findRepoRoot(scanDir);
+            if (!repoRoot)
+                continue;
+            if (this.repos.has(repoRoot))
+                continue;
+            const gitignoreFiles = findGitignoreFiles(repoRoot);
+            const entries = gitignoreFiles.map((gf) => ({
+                dir: dirname(gf),
+                ig: parseGitignore(gf),
+            }));
+            // Sort deepest-first so nested `.gitignore` files are checked first
+            entries.sort((a, b) => b.dir.length - a.dir.length);
+            this.repos.set(repoRoot, { root: repoRoot, entries });
+        }
+    }
+    /**
+     * Check whether a file path is ignored by any applicable `.gitignore`.
+     *
+     * @param filePath - Absolute file path to check.
+     * @returns `true` if the file should be ignored.
+     */
+    isIgnored(filePath) {
+        const absPath = resolve(filePath);
+        for (const [, repo] of this.repos) {
+            // Check if file is within this repo
+            const relToRepo = relative(repo.root, absPath);
+            if (relToRepo.startsWith('..') || relToRepo.startsWith(resolve('/'))) {
+                continue;
+            }
+            // Check each `.gitignore` entry (deepest-first)
+            for (const entry of repo.entries) {
+                const relToEntry = relative(entry.dir, absPath);
+                if (relToEntry.startsWith('..'))
+                    continue;
+                const normalized = toForwardSlash(relToEntry);
+                if (entry.ig.ignores(normalized)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    /**
+     * Invalidate and re-parse a specific `.gitignore` file.
+     * Call when a `.gitignore` file is added, changed, or removed.
+     *
+     * @param gitignorePath - Absolute path to the `.gitignore` file that changed.
+     */
+    invalidate(gitignorePath) {
+        const absPath = resolve(gitignorePath);
+        const gitignoreDir = dirname(absPath);
+        for (const [, repo] of this.repos) {
+            const relToRepo = relative(repo.root, gitignoreDir);
+            if (relToRepo.startsWith('..'))
+                continue;
+            // Remove old entry for this directory
+            repo.entries = repo.entries.filter((e) => e.dir !== gitignoreDir);
+            // Re-parse if file still exists
+            if (existsSync(absPath)) {
+                repo.entries.push({ dir: gitignoreDir, ig: parseGitignore(absPath) });
+                // Re-sort deepest-first
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            return;
+        }
+        // If not in any known repo, check if it's in a repo we haven't scanned
+        const repoRoot = findRepoRoot(gitignoreDir);
+        if (repoRoot && existsSync(absPath)) {
+            const entries = [
+                { dir: gitignoreDir, ig: parseGitignore(absPath) },
+            ];
+            if (this.repos.has(repoRoot)) {
+                const repo = this.repos.get(repoRoot);
+                repo.entries.push(entries[0]);
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            else {
+                this.repos.set(repoRoot, { root: repoRoot, entries });
+            }
+        }
+    }
+}
+
 /**
  * @module logger
  * Creates pino logger instances. I/O: optionally writes logs to file via pino/file transport. Defaults to stdout at info level.
@@ -2033,6 +2247,7 @@ class FileSystemWatcher {
     processor;
     logger;
     health;
+    gitignoreFilter;
     watcher;
     /**
      * Create a new FileSystemWatcher.
@@ -2048,6 +2263,7 @@ class FileSystemWatcher {
         this.queue = queue;
         this.processor = processor;
         this.logger = logger;
+        this.gitignoreFilter = options.gitignoreFilter;
         const healthOptions = {
             maxRetries: options.maxRetries,
             maxBackoffMs: options.maxBackoffMs,
@@ -2070,14 +2286,23 @@ class FileSystemWatcher {
             ignoreInitial: false,
         });
         this.watcher.on('add', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File added');
             this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('change', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File changed');
             this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('unlink', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File removed');
             this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
         });
@@ -2104,6 +2329,29 @@ class FileSystemWatcher {
     get systemHealth() {
         return this.health;
     }
+    /**
+     * Check if a path is gitignored and should be skipped.
+     */
+    isGitignored(path) {
+        if (!this.gitignoreFilter)
+            return false;
+        const ignored = this.gitignoreFilter.isIgnored(path);
+        if (ignored) {
+            this.logger.debug({ path }, 'Skipping gitignored file');
+        }
+        return ignored;
+    }
+    /**
+     * If the changed file is a `.gitignore`, invalidate the filter cache.
+     */
+    handleGitignoreChange(path) {
+        if (!this.gitignoreFilter)
+            return;
+        if (path.endsWith('.gitignore')) {
+            this.logger.info({ path }, 'Gitignore file changed, refreshing filter');
+            this.gitignoreFilter.invalidate(path);
+        }
+    }
     /**
      * Wrap a processing operation with health tracking.
      * On success, resets the failure counter.
@@ -2259,10 +2507,15 @@ class JeevesWatcher {
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
         this.queue = queue;
+        const respectGitignore = this.config.watch.respectGitignore ?? true;
+        const gitignoreFilter = respectGitignore
+            ? new GitignoreFilter(this.config.watch.paths)
+            : undefined;
         const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
             maxRetries: this.config.maxRetries,
             maxBackoffMs: this.config.maxBackoffMs,
             onFatalError: this.runtimeOptions.onFatalError,
+            gitignoreFilter,
         });
         this.watcher = watcher;
         const server = this.factories.createApiServer({
@@ -2368,4 +2621,4 @@ async function startFromConfig(configPath) {
     return app;
 }
 
-export { DocumentProcessor, EventQueue, FileSystemWatcher, JeevesWatcher, SystemHealth, VectorStoreClient, apiConfigSchema, applyRules, buildAttributes, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createLogger, deleteMetadata, embeddingConfigSchema, extractText, inferenceRuleSchema, jeevesWatcherConfigSchema, loadConfig, loggingConfigSchema, metadataPath, pointId, readMetadata, startFromConfig, vectorStoreConfigSchema, watchConfigSchema, writeMetadata };
+export { DocumentProcessor, EventQueue, FileSystemWatcher, GitignoreFilter, JeevesWatcher, SystemHealth, VectorStoreClient, apiConfigSchema, applyRules, buildAttributes, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createLogger, deleteMetadata, embeddingConfigSchema, extractText, inferenceRuleSchema, jeevesWatcherConfigSchema, loadConfig, loggingConfigSchema, metadataPath, pointId, readMetadata, startFromConfig, vectorStoreConfigSchema, watchConfigSchema, writeMetadata };

package/package.json

@@ -26,6 +26,7 @@
     "commander": "^14.0.3",
     "cosmiconfig": "*",
     "fastify": "*",
+    "ignore": "^7.0.5",
     "js-yaml": "*",
     "json5": "*",
     "mammoth": "^1.11.0",
@@ -171,5 +172,5 @@
   },
   "type": "module",
   "types": "dist/index.d.ts",
-  "version": "0.2.4"
+  "version": "0.2.5"
 }