@karmaniverous/jeeves-watcher 0.2.4 → 0.2.6

package/dist/cjs/index.js

@@ -10,6 +10,8 @@ var cosmiconfig = require('cosmiconfig');
 var zod = require('zod');
 var jsonmap = require('@karmaniverous/jsonmap');
 var googleGenai = require('@langchain/google-genai');
+var node_fs = require('node:fs');
+var ignore = require('ignore');
 var pino = require('pino');
 var uuid = require('uuid');
 var cheerio = require('cheerio');
@@ -454,6 +456,7 @@ const WATCH_DEFAULTS = {
     stabilityThresholdMs: 500,
     usePolling: false,
     pollIntervalMs: 1000,
+    respectGitignore: true,
 };
 /** Default embedding configuration. */
 const EMBEDDING_DEFAULTS = {
@@ -498,6 +501,11 @@ const watchConfigSchema = zod.z.object({
         .number()
         .optional()
         .describe('Time in milliseconds a file must remain unchanged before processing.'),
+    /** Whether to respect .gitignore files when processing. */
+    respectGitignore: zod.z
+        .boolean()
+        .optional()
+        .describe('Skip files ignored by .gitignore in git repositories. Only applies to repos with a .git directory. Default: true.'),
 });
 /**
  * Configuration watch settings.
@@ -963,6 +971,212 @@ function createEmbeddingProvider(config, logger) {
     return factory(config, logger);
 }
 
+/**
+ * @module gitignore
+ * Processor-level gitignore filtering. Scans watched paths for `.gitignore` files in git repos, caches parsed patterns, and exposes `isIgnored()` for path checking.
+ */
+/**
+ * Find the git repo root by walking up from `startDir` looking for `.git/`.
+ * Returns `undefined` if no repo is found.
+ */
+function findRepoRoot(startDir) {
+    let dir = node_path.resolve(startDir);
+    const root = node_path.resolve('/');
+    while (dir !== root) {
+        if (node_fs.existsSync(node_path.join(dir, '.git')) &&
+            node_fs.statSync(node_path.join(dir, '.git')).isDirectory()) {
+            return dir;
+        }
+        const parent = node_path.dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return undefined;
+}
+/**
+ * Convert a watch path (directory, file path, or glob) to a concrete directory
+ * that can be scanned for a repo root.
+ */
+function watchPathToScanDir(watchPath) {
+    const absPath = node_path.resolve(watchPath);
+    try {
+        return node_fs.statSync(absPath).isDirectory() ? absPath : node_path.dirname(absPath);
+    }
+    catch {
+        // ignore
+    }
+    // If this is a glob, fall back to the non-glob prefix.
+    const globMatch = /[*?[{]/.exec(watchPath);
+    if (!globMatch)
+        return undefined;
+    const prefix = watchPath.slice(0, globMatch.index);
+    const trimmed = prefix.trim();
+    const baseDir = trimmed.length === 0
+        ? '.'
+        : trimmed.endsWith('/') || trimmed.endsWith('\\')
+            ? trimmed
+            : node_path.dirname(trimmed);
+    const resolved = node_path.resolve(baseDir);
+    if (!node_fs.existsSync(resolved))
+        return undefined;
+    return resolved;
+}
+/**
+ * Recursively find all `.gitignore` files under `dir`.
+ * Skips `.git` and `node_modules` directories for performance.
+ */
+function findGitignoreFiles(dir) {
+    const results = [];
+    const gitignorePath = node_path.join(dir, '.gitignore');
+    if (node_fs.existsSync(gitignorePath)) {
+        results.push(gitignorePath);
+    }
+    let entries;
+    try {
+        entries = node_fs.readdirSync(dir);
+    }
+    catch {
+        return results;
+    }
+    for (const entry of entries) {
+        if (entry === '.git' || entry === 'node_modules')
+            continue;
+        const fullPath = node_path.join(dir, entry);
+        try {
+            if (node_fs.statSync(fullPath).isDirectory()) {
+                results.push(...findGitignoreFiles(fullPath));
+            }
+        }
+        catch {
+            // Skip inaccessible entries
+        }
+    }
+    return results;
+}
+/**
+ * Parse a `.gitignore` file into an `ignore` instance.
+ */
+function parseGitignore(gitignorePath) {
+    const content = node_fs.readFileSync(gitignorePath, 'utf8');
+    return ignore().add(content);
+}
+/**
+ * Normalize a path to use forward slashes (required by `ignore` package).
+ */
+function toForwardSlash(p) {
+    return p.replace(/\\/g, '/');
+}
+/**
+ * Processor-level gitignore filter. Checks file paths against the nearest
+ * `.gitignore` chain in git repositories.
+ */
+class GitignoreFilter {
+    repos = new Map();
+    /**
+     * Create a GitignoreFilter by scanning watched paths for `.gitignore` files.
+     *
+     * @param watchPaths - Absolute paths being watched (directories or globs resolved to roots).
+     */
+    constructor(watchPaths) {
+        this.scan(watchPaths);
+    }
+    /**
+     * Scan paths for git repos and their `.gitignore` files.
+     */
+    scan(watchPaths) {
+        this.repos.clear();
+        const scannedDirs = new Set();
+        for (const watchPath of watchPaths) {
+            const scanDir = watchPathToScanDir(watchPath);
+            if (!scanDir)
+                continue;
+            if (scannedDirs.has(scanDir))
+                continue;
+            scannedDirs.add(scanDir);
+            const repoRoot = findRepoRoot(scanDir);
+            if (!repoRoot)
+                continue;
+            if (this.repos.has(repoRoot))
+                continue;
+            const gitignoreFiles = findGitignoreFiles(repoRoot);
+            const entries = gitignoreFiles.map((gf) => ({
+                dir: node_path.dirname(gf),
+                ig: parseGitignore(gf),
+            }));
+            // Sort deepest-first so nested `.gitignore` files are checked first
+            entries.sort((a, b) => b.dir.length - a.dir.length);
+            this.repos.set(repoRoot, { root: repoRoot, entries });
+        }
+    }
+    /**
+     * Check whether a file path is ignored by any applicable `.gitignore`.
+     *
+     * @param filePath - Absolute file path to check.
+     * @returns `true` if the file should be ignored.
+     */
+    isIgnored(filePath) {
+        const absPath = node_path.resolve(filePath);
+        for (const [, repo] of this.repos) {
+            // Check if file is within this repo
+            const relToRepo = node_path.relative(repo.root, absPath);
+            if (relToRepo.startsWith('..') || relToRepo.startsWith(node_path.resolve('/'))) {
+                continue;
+            }
+            // Check each `.gitignore` entry (deepest-first)
+            for (const entry of repo.entries) {
+                const relToEntry = node_path.relative(entry.dir, absPath);
+                if (relToEntry.startsWith('..'))
+                    continue;
+                const normalized = toForwardSlash(relToEntry);
+                if (entry.ig.ignores(normalized)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    /**
+     * Invalidate and re-parse a specific `.gitignore` file.
+     * Call when a `.gitignore` file is added, changed, or removed.
+     *
+     * @param gitignorePath - Absolute path to the `.gitignore` file that changed.
+     */
+    invalidate(gitignorePath) {
+        const absPath = node_path.resolve(gitignorePath);
+        const gitignoreDir = node_path.dirname(absPath);
+        for (const [, repo] of this.repos) {
+            const relToRepo = node_path.relative(repo.root, gitignoreDir);
+            if (relToRepo.startsWith('..'))
+                continue;
+            // Remove old entry for this directory
+            repo.entries = repo.entries.filter((e) => e.dir !== gitignoreDir);
+            // Re-parse if file still exists
+            if (node_fs.existsSync(absPath)) {
+                repo.entries.push({ dir: gitignoreDir, ig: parseGitignore(absPath) });
+                // Re-sort deepest-first
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            return;
+        }
+        // If not in any known repo, check if it's in a repo we haven't scanned
+        const repoRoot = findRepoRoot(gitignoreDir);
+        if (repoRoot && node_fs.existsSync(absPath)) {
+            const entries = [
+                { dir: gitignoreDir, ig: parseGitignore(absPath) },
+            ];
+            if (this.repos.has(repoRoot)) {
+                const repo = this.repos.get(repoRoot);
+                repo.entries.push(entries[0]);
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            else {
+                this.repos.set(repoRoot, { root: repoRoot, entries });
+            }
+        }
+    }
+}
+
 /**
  * @module logger
  * Creates pino logger instances. I/O: optionally writes logs to file via pino/file transport. Defaults to stdout at info level.
@@ -2054,6 +2268,7 @@ class FileSystemWatcher {
     processor;
     logger;
     health;
+    gitignoreFilter;
     watcher;
     /**
      * Create a new FileSystemWatcher.
@@ -2069,6 +2284,7 @@ class FileSystemWatcher {
         this.queue = queue;
         this.processor = processor;
         this.logger = logger;
+        this.gitignoreFilter = options.gitignoreFilter;
         const healthOptions = {
             maxRetries: options.maxRetries,
             maxBackoffMs: options.maxBackoffMs,
@@ -2091,14 +2307,23 @@ class FileSystemWatcher {
             ignoreInitial: false,
         });
         this.watcher.on('add', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File added');
             this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('change', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File changed');
             this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('unlink', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File removed');
             this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
         });
@@ -2125,6 +2350,29 @@ class FileSystemWatcher {
     get systemHealth() {
         return this.health;
     }
+    /**
+     * Check if a path is gitignored and should be skipped.
+     */
+    isGitignored(path) {
+        if (!this.gitignoreFilter)
+            return false;
+        const ignored = this.gitignoreFilter.isIgnored(path);
+        if (ignored) {
+            this.logger.debug({ path }, 'Skipping gitignored file');
+        }
+        return ignored;
+    }
+    /**
+     * If the changed file is a `.gitignore`, invalidate the filter cache.
+     */
+    handleGitignoreChange(path) {
+        if (!this.gitignoreFilter)
+            return;
+        if (path.endsWith('.gitignore')) {
+            this.logger.info({ path }, 'Gitignore file changed, refreshing filter');
+            this.gitignoreFilter.invalidate(path);
+        }
+    }
     /**
      * Wrap a processing operation with health tracking.
      * On success, resets the failure counter.
@@ -2280,10 +2528,15 @@ class JeevesWatcher {
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
         this.queue = queue;
+        const respectGitignore = this.config.watch.respectGitignore ?? true;
+        const gitignoreFilter = respectGitignore
+            ? new GitignoreFilter(this.config.watch.paths)
+            : undefined;
         const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
             maxRetries: this.config.maxRetries,
             maxBackoffMs: this.config.maxBackoffMs,
             onFatalError: this.runtimeOptions.onFatalError,
+            gitignoreFilter,
         });
         this.watcher = watcher;
         const server = this.factories.createApiServer({
@@ -2392,6 +2645,7 @@ async function startFromConfig(configPath) {
 exports.DocumentProcessor = DocumentProcessor;
 exports.EventQueue = EventQueue;
 exports.FileSystemWatcher = FileSystemWatcher;
+exports.GitignoreFilter = GitignoreFilter;
 exports.JeevesWatcher = JeevesWatcher;
 exports.SystemHealth = SystemHealth;
 exports.VectorStoreClient = VectorStoreClient;

package/dist/cli/jeeves-watcher/index.js

@@ -1,9 +1,7 @@
-#!/usr/bin/env node
-#!/usr/bin/env node
 import { Command } from '@commander-js/extra-typings';
 import Fastify from 'fastify';
 import { readdir, stat, rm, readFile, mkdir, writeFile } from 'node:fs/promises';
-import { resolve, dirname, join, extname, basename } from 'node:path';
+import { resolve, dirname, join, relative, extname, basename } from 'node:path';
 import picomatch from 'picomatch';
 import { omit, get } from 'radash';
 import { createHash } from 'node:crypto';
@@ -11,6 +9,8 @@ import { cosmiconfig } from 'cosmiconfig';
 import { z, ZodError } from 'zod';
 import { jsonMapMapSchema, JsonMap } from '@karmaniverous/jsonmap';
 import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
+import { existsSync, statSync, readdirSync, readFileSync } from 'node:fs';
+import ignore from 'ignore';
 import pino from 'pino';
 import { v5 } from 'uuid';
 import * as cheerio from 'cheerio';
@@ -436,6 +436,7 @@ const WATCH_DEFAULTS = {
     stabilityThresholdMs: 500,
     usePolling: false,
     pollIntervalMs: 1000,
+    respectGitignore: true,
 };
 /** Default embedding configuration. */
 const EMBEDDING_DEFAULTS = {
@@ -501,6 +502,11 @@ const watchConfigSchema = z.object({
         .number()
         .optional()
         .describe('Time in milliseconds a file must remain unchanged before processing.'),
+    /** Whether to respect .gitignore files when processing. */
+    respectGitignore: z
+        .boolean()
+        .optional()
+        .describe('Skip files ignored by .gitignore in git repositories. Only applies to repos with a .git directory. Default: true.'),
 });
 /**
  * Configuration watch settings.
@@ -966,6 +972,212 @@ function createEmbeddingProvider(config, logger) {
     return factory(config, logger);
 }
 
+/**
+ * @module gitignore
+ * Processor-level gitignore filtering. Scans watched paths for `.gitignore` files in git repos, caches parsed patterns, and exposes `isIgnored()` for path checking.
+ */
+/**
+ * Find the git repo root by walking up from `startDir` looking for `.git/`.
+ * Returns `undefined` if no repo is found.
+ */
+function findRepoRoot(startDir) {
+    let dir = resolve(startDir);
+    const root = resolve('/');
+    while (dir !== root) {
+        if (existsSync(join(dir, '.git')) &&
+            statSync(join(dir, '.git')).isDirectory()) {
+            return dir;
+        }
+        const parent = dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return undefined;
+}
+/**
+ * Convert a watch path (directory, file path, or glob) to a concrete directory
+ * that can be scanned for a repo root.
+ */
+function watchPathToScanDir(watchPath) {
+    const absPath = resolve(watchPath);
+    try {
+        return statSync(absPath).isDirectory() ? absPath : dirname(absPath);
+    }
+    catch {
+        // ignore
+    }
+    // If this is a glob, fall back to the non-glob prefix.
+    const globMatch = /[*?[{]/.exec(watchPath);
+    if (!globMatch)
+        return undefined;
+    const prefix = watchPath.slice(0, globMatch.index);
+    const trimmed = prefix.trim();
+    const baseDir = trimmed.length === 0
+        ? '.'
+        : trimmed.endsWith('/') || trimmed.endsWith('\\')
+            ? trimmed
+            : dirname(trimmed);
+    const resolved = resolve(baseDir);
+    if (!existsSync(resolved))
+        return undefined;
+    return resolved;
+}
+/**
+ * Recursively find all `.gitignore` files under `dir`.
+ * Skips `.git` and `node_modules` directories for performance.
+ */
+function findGitignoreFiles(dir) {
+    const results = [];
+    const gitignorePath = join(dir, '.gitignore');
+    if (existsSync(gitignorePath)) {
+        results.push(gitignorePath);
+    }
+    let entries;
+    try {
+        entries = readdirSync(dir);
+    }
+    catch {
+        return results;
+    }
+    for (const entry of entries) {
+        if (entry === '.git' || entry === 'node_modules')
+            continue;
+        const fullPath = join(dir, entry);
+        try {
+            if (statSync(fullPath).isDirectory()) {
+                results.push(...findGitignoreFiles(fullPath));
+            }
+        }
+        catch {
+            // Skip inaccessible entries
+        }
+    }
+    return results;
+}
+/**
+ * Parse a `.gitignore` file into an `ignore` instance.
+ */
+function parseGitignore(gitignorePath) {
+    const content = readFileSync(gitignorePath, 'utf8');
+    return ignore().add(content);
+}
+/**
+ * Normalize a path to use forward slashes (required by `ignore` package).
+ */
+function toForwardSlash(p) {
+    return p.replace(/\\/g, '/');
+}
+/**
+ * Processor-level gitignore filter. Checks file paths against the nearest
+ * `.gitignore` chain in git repositories.
+ */
+class GitignoreFilter {
+    repos = new Map();
+    /**
+     * Create a GitignoreFilter by scanning watched paths for `.gitignore` files.
+     *
+     * @param watchPaths - Absolute paths being watched (directories or globs resolved to roots).
+     */
+    constructor(watchPaths) {
+        this.scan(watchPaths);
+    }
+    /**
+     * Scan paths for git repos and their `.gitignore` files.
+     */
+    scan(watchPaths) {
+        this.repos.clear();
+        const scannedDirs = new Set();
+        for (const watchPath of watchPaths) {
+            const scanDir = watchPathToScanDir(watchPath);
+            if (!scanDir)
+                continue;
+            if (scannedDirs.has(scanDir))
+                continue;
+            scannedDirs.add(scanDir);
+            const repoRoot = findRepoRoot(scanDir);
+            if (!repoRoot)
+                continue;
+            if (this.repos.has(repoRoot))
+                continue;
+            const gitignoreFiles = findGitignoreFiles(repoRoot);
+            const entries = gitignoreFiles.map((gf) => ({
+                dir: dirname(gf),
+                ig: parseGitignore(gf),
+            }));
+            // Sort deepest-first so nested `.gitignore` files are checked first
+            entries.sort((a, b) => b.dir.length - a.dir.length);
+            this.repos.set(repoRoot, { root: repoRoot, entries });
+        }
+    }
+    /**
+     * Check whether a file path is ignored by any applicable `.gitignore`.
+     *
+     * @param filePath - Absolute file path to check.
+     * @returns `true` if the file should be ignored.
+     */
+    isIgnored(filePath) {
+        const absPath = resolve(filePath);
+        for (const [, repo] of this.repos) {
+            // Check if file is within this repo
+            const relToRepo = relative(repo.root, absPath);
+            if (relToRepo.startsWith('..') || relToRepo.startsWith(resolve('/'))) {
+                continue;
+            }
+            // Check each `.gitignore` entry (deepest-first)
+            for (const entry of repo.entries) {
+                const relToEntry = relative(entry.dir, absPath);
+                if (relToEntry.startsWith('..'))
+                    continue;
+                const normalized = toForwardSlash(relToEntry);
+                if (entry.ig.ignores(normalized)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+    /**
+     * Invalidate and re-parse a specific `.gitignore` file.
+     * Call when a `.gitignore` file is added, changed, or removed.
+     *
+     * @param gitignorePath - Absolute path to the `.gitignore` file that changed.
+     */
+    invalidate(gitignorePath) {
+        const absPath = resolve(gitignorePath);
+        const gitignoreDir = dirname(absPath);
+        for (const [, repo] of this.repos) {
+            const relToRepo = relative(repo.root, gitignoreDir);
+            if (relToRepo.startsWith('..'))
+                continue;
+            // Remove old entry for this directory
+            repo.entries = repo.entries.filter((e) => e.dir !== gitignoreDir);
+            // Re-parse if file still exists
+            if (existsSync(absPath)) {
+                repo.entries.push({ dir: gitignoreDir, ig: parseGitignore(absPath) });
+                // Re-sort deepest-first
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            return;
+        }
+        // If not in any known repo, check if it's in a repo we haven't scanned
+        const repoRoot = findRepoRoot(gitignoreDir);
+        if (repoRoot && existsSync(absPath)) {
+            const entries = [
+                { dir: gitignoreDir, ig: parseGitignore(absPath) },
+            ];
+            if (this.repos.has(repoRoot)) {
+                const repo = this.repos.get(repoRoot);
+                repo.entries.push(entries[0]);
+                repo.entries.sort((a, b) => b.dir.length - a.dir.length);
+            }
+            else {
+                this.repos.set(repoRoot, { root: repoRoot, entries });
+            }
+        }
+    }
+}
+
 /**
  * @module logger
  * Creates pino logger instances. I/O: optionally writes logs to file via pino/file transport. Defaults to stdout at info level.
@@ -2057,6 +2269,7 @@ class FileSystemWatcher {
     processor;
     logger;
     health;
+    gitignoreFilter;
     watcher;
     /**
      * Create a new FileSystemWatcher.
@@ -2072,6 +2285,7 @@ class FileSystemWatcher {
         this.queue = queue;
         this.processor = processor;
         this.logger = logger;
+        this.gitignoreFilter = options.gitignoreFilter;
         const healthOptions = {
             maxRetries: options.maxRetries,
             maxBackoffMs: options.maxBackoffMs,
@@ -2094,14 +2308,23 @@ class FileSystemWatcher {
             ignoreInitial: false,
         });
         this.watcher.on('add', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File added');
             this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('change', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File changed');
             this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('unlink', (path) => {
+            this.handleGitignoreChange(path);
+            if (this.isGitignored(path))
+                return;
             this.logger.debug({ path }, 'File removed');
             this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
         });
@@ -2128,6 +2351,29 @@ class FileSystemWatcher {
     get systemHealth() {
         return this.health;
     }
+    /**
+     * Check if a path is gitignored and should be skipped.
+     */
+    isGitignored(path) {
+        if (!this.gitignoreFilter)
+            return false;
+        const ignored = this.gitignoreFilter.isIgnored(path);
+        if (ignored) {
+            this.logger.debug({ path }, 'Skipping gitignored file');
+        }
+        return ignored;
+    }
+    /**
+     * If the changed file is a `.gitignore`, invalidate the filter cache.
+     */
+    handleGitignoreChange(path) {
+        if (!this.gitignoreFilter)
+            return;
+        if (path.endsWith('.gitignore')) {
+            this.logger.info({ path }, 'Gitignore file changed, refreshing filter');
+            this.gitignoreFilter.invalidate(path);
+        }
+    }
     /**
      * Wrap a processing operation with health tracking.
      * On success, resets the failure counter.
@@ -2283,10 +2529,15 @@ class JeevesWatcher {
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
         this.queue = queue;
+        const respectGitignore = this.config.watch.respectGitignore ?? true;
+        const gitignoreFilter = respectGitignore
+            ? new GitignoreFilter(this.config.watch.paths)
+            : undefined;
         const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
             maxRetries: this.config.maxRetries,
             maxBackoffMs: this.config.maxBackoffMs,
             onFatalError: this.runtimeOptions.onFatalError,
+            gitignoreFilter,
         });
         this.watcher = watcher;
         const server = this.factories.createApiServer({