@karmaniverous/jeeves-watcher 0.2.2 → 0.2.4

package/README.md

@@ -94,6 +94,20 @@ The watcher will:
 
 ## Configuration
 
+### Environment Variable Substitution
+
+Config strings support `${VAR_NAME}` syntax for environment variable injection:
+
+```json
+{
+  "embedding": {
+    "apiKey": "${GOOGLE_API_KEY}"
+  }
+}
+```
+
+If `GOOGLE_API_KEY` is set in the environment, the value is substituted at config load time. **Unresolvable expressions are left untouched** — this allows `${...}` template syntax used in inference rule `set` values (e.g. `${frontmatter.title}`, `${file.path}`) to pass through for later resolution by the rules engine.
+
 ### Watch Paths
 
 ```json

package/config.schema.json

@@ -141,6 +141,22 @@
           "$ref": "#/definitions/__schema52"
         }
       ]
+    },
+    "maxRetries": {
+      "description": "Maximum consecutive system-level failures before triggering fatal error. Default: Infinity.",
+      "allOf": [
+        {
+          "$ref": "#/definitions/__schema53"
+        }
+      ]
+    },
+    "maxBackoffMs": {
+      "description": "Maximum backoff delay in milliseconds for system errors. Default: 60000.",
+      "allOf": [
+        {
+          "$ref": "#/definitions/__schema54"
+        }
+      ]
     }
   },
   "required": [
@@ -572,6 +588,12 @@
     },
     "__schema52": {
       "type": "number"
+    },
+    "__schema53": {
+      "type": "number"
+    },
+    "__schema54": {
+      "type": "number"
     }
   }
 }

package/dist/cjs/index.js

@@ -665,6 +665,16 @@ const jeevesWatcherConfigSchema = zod.z.object({
         .number()
         .optional()
         .describe('Timeout in milliseconds for graceful shutdown.'),
+    /** Maximum consecutive system-level failures before triggering fatal error. Default: Infinity. */
+    maxRetries: zod.z
+        .number()
+        .optional()
+        .describe('Maximum consecutive system-level failures before triggering fatal error. Default: Infinity.'),
+    /** Maximum backoff delay in milliseconds for system errors. Default: 60000. */
+    maxBackoffMs: zod.z
+        .number()
+        .optional()
+        .describe('Maximum backoff delay in milliseconds for system errors. Default: 60000.'),
 });
 
 /**
@@ -677,15 +687,13 @@ const ENV_PATTERN = /\$\{([^}]+)\}/g;
  * Replace `${VAR_NAME}` patterns in a string with `process.env.VAR_NAME`.
  *
  * @param value - The string to process.
- * @returns The string with env vars substituted.
- * @throws If a referenced env var is not set.
+ * @returns The string with resolved env vars; unresolvable expressions left untouched.
  */
 function substituteString(value) {
     return value.replace(ENV_PATTERN, (match, varName) => {
         const envValue = process.env[varName];
-        if (envValue === undefined) {
-            throw new Error(`Environment variable \${${varName}} referenced in config is not set.`);
-        }
+        if (envValue === undefined)
+            return match;
         return envValue;
     });
 }
@@ -1071,11 +1079,11 @@ async function extractMarkdown(filePath) {
 }
 async function extractPlaintext(filePath) {
     const raw = await promises.readFile(filePath, 'utf8');
-    return { text: raw };
+    return { text: raw.replace(/^\uFEFF/, '') };
 }
 async function extractJson(filePath) {
     const raw = await promises.readFile(filePath, 'utf8');
-    const parsed = JSON.parse(raw);
+    const parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
     const json = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
         ? parsed
         : undefined;
@@ -1097,7 +1105,7 @@ async function extractDocx(filePath) {
 }
 async function extractHtml(filePath) {
     const raw = await promises.readFile(filePath, 'utf8');
-    const $ = cheerio__namespace.load(raw);
+    const $ = cheerio__namespace.load(raw.replace(/^\uFEFF/, ''));
     $('script, style').remove();
     const text = $('body').text().trim() || $.text().trim();
     return { text };
@@ -1927,6 +1935,112 @@ class VectorStoreClient {
     }
 }
 
+/**
+ * @module health
+ * Tracks consecutive system-level failures and applies exponential backoff.
+ * Triggers fatal error callback when maxRetries is exceeded.
+ */
+/**
+ * Tracks system health via consecutive failure count and exponential backoff.
+ */
+class SystemHealth {
+    consecutiveFailures = 0;
+    maxRetries;
+    maxBackoffMs;
+    baseDelayMs;
+    onFatalError;
+    logger;
+    constructor(options) {
+        this.maxRetries = options.maxRetries ?? Number.POSITIVE_INFINITY;
+        this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+        this.baseDelayMs = options.baseDelayMs ?? 1000;
+        this.onFatalError = options.onFatalError;
+        this.logger = options.logger;
+    }
+    /**
+     * Record a successful system operation. Resets the failure counter.
+     */
+    recordSuccess() {
+        if (this.consecutiveFailures > 0) {
+            this.logger.info({ previousFailures: this.consecutiveFailures }, 'System health recovered');
+        }
+        this.consecutiveFailures = 0;
+    }
+    /**
+     * Record a system-level failure. If maxRetries is exceeded, triggers fatal error.
+     *
+     * @param error - The error that occurred.
+     * @returns Whether the watcher should continue (false = fatal).
+     */
+    recordFailure(error) {
+        this.consecutiveFailures += 1;
+        this.logger.error({
+            consecutiveFailures: this.consecutiveFailures,
+            maxRetries: this.maxRetries,
+            err: normalizeError(error),
+        }, 'System-level failure recorded');
+        if (this.consecutiveFailures >= this.maxRetries) {
+            this.logger.fatal({ consecutiveFailures: this.consecutiveFailures }, 'Maximum retries exceeded, triggering fatal error');
+            if (this.onFatalError) {
+                this.onFatalError(error);
+                return false;
+            }
+            throw error instanceof Error
+                ? error
+                : new Error(`Fatal system error: ${String(error)}`);
+        }
+        return true;
+    }
+    /**
+     * Compute the current backoff delay based on consecutive failures.
+     *
+     * @returns Delay in milliseconds.
+     */
+    get currentBackoffMs() {
+        if (this.consecutiveFailures === 0)
+            return 0;
+        const exp = Math.max(0, this.consecutiveFailures - 1);
+        return Math.min(this.maxBackoffMs, this.baseDelayMs * 2 ** exp);
+    }
+    /**
+     * Sleep for the current backoff duration.
+     *
+     * @param signal - Optional abort signal.
+     */
+    async backoff(signal) {
+        const delay = this.currentBackoffMs;
+        if (delay <= 0)
+            return;
+        this.logger.warn({ delayMs: delay, consecutiveFailures: this.consecutiveFailures }, 'Backing off before next attempt');
+        await new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                cleanup();
+                resolve();
+            }, delay);
+            const onAbort = () => {
+                cleanup();
+                reject(new Error('Backoff aborted'));
+            };
+            const cleanup = () => {
+                clearTimeout(timer);
+                if (signal)
+                    signal.removeEventListener('abort', onAbort);
+            };
+            if (signal) {
+                if (signal.aborted) {
+                    onAbort();
+                    return;
+                }
+                signal.addEventListener('abort', onAbort, { once: true });
+            }
+        });
+    }
+    /** Current consecutive failure count. */
+    get failures() {
+        return this.consecutiveFailures;
+    }
+}
+
 /**
  * @module watcher
  * Filesystem watcher wrapping chokidar. I/O: watches files/directories for add/change/unlink events, enqueues to processing queue.
@@ -1939,6 +2053,7 @@ class FileSystemWatcher {
     queue;
     processor;
     logger;
+    health;
     watcher;
     /**
      * Create a new FileSystemWatcher.
@@ -1947,12 +2062,20 @@ class FileSystemWatcher {
      * @param queue - The event queue.
      * @param processor - The document processor.
      * @param logger - The logger instance.
+     * @param options - Optional health/fatal error options.
      */
-    constructor(config, queue, processor, logger) {
+    constructor(config, queue, processor, logger, options = {}) {
         this.config = config;
         this.queue = queue;
         this.processor = processor;
         this.logger = logger;
+        const healthOptions = {
+            maxRetries: options.maxRetries,
+            maxBackoffMs: options.maxBackoffMs,
+            onFatalError: options.onFatalError,
+            logger,
+        };
+        this.health = new SystemHealth(healthOptions);
     }
     /**
      * Start watching the filesystem and processing events.
@@ -1969,18 +2092,19 @@ class FileSystemWatcher {
         });
         this.watcher.on('add', (path) => {
             this.logger.debug({ path }, 'File added');
-            this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.processor.processFile(path));
+            this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('change', (path) => {
             this.logger.debug({ path }, 'File changed');
-            this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.processor.processFile(path));
+            this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('unlink', (path) => {
             this.logger.debug({ path }, 'File removed');
-            this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.processor.deleteFile(path));
+            this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
         });
         this.watcher.on('error', (error) => {
             this.logger.error({ err: normalizeError(error) }, 'Watcher error');
+            this.health.recordFailure(error);
         });
         this.queue.process();
         this.logger.info({ paths: this.config.paths }, 'Filesystem watcher started');
@@ -1995,6 +2119,30 @@ class FileSystemWatcher {
             this.logger.info('Filesystem watcher stopped');
         }
     }
+    /**
+     * Get the system health tracker.
+     */
+    get systemHealth() {
+        return this.health;
+    }
+    /**
+     * Wrap a processing operation with health tracking.
+     * On success, resets the failure counter.
+     * On failure, records the failure and applies backoff.
+     */
+    async wrapProcessing(fn) {
+        try {
+            await this.health.backoff();
+            await fn();
+            this.health.recordSuccess();
+        }
+        catch (error) {
+            const shouldContinue = this.health.recordFailure(error);
+            if (!shouldContinue) {
+                await this.stop();
+            }
+        }
+    }
 }
 
 /**
@@ -2070,7 +2218,7 @@ const defaultFactories = {
     compileRules,
     createDocumentProcessor: (config, embeddingProvider, vectorStore, compiledRules, logger) => new DocumentProcessor(config, embeddingProvider, vectorStore, compiledRules, logger),
     createEventQueue: (options) => new EventQueue(options),
-    createFileSystemWatcher: (config, queue, processor, logger) => new FileSystemWatcher(config, queue, processor, logger),
+    createFileSystemWatcher: (config, queue, processor, logger, options) => new FileSystemWatcher(config, queue, processor, logger, options),
     createApiServer,
 };
 /**
@@ -2080,6 +2228,7 @@ class JeevesWatcher {
     config;
     configPath;
     factories;
+    runtimeOptions;
     logger;
     watcher;
     queue;
@@ -2092,11 +2241,13 @@ class JeevesWatcher {
      * @param config - The application configuration.
      * @param configPath - Optional config file path to watch for changes.
      * @param factories - Optional component factories (for dependency injection).
+     * @param runtimeOptions - Optional runtime-only options (e.g., onFatalError).
      */
-    constructor(config, configPath, factories = {}) {
+    constructor(config, configPath, factories = {}, runtimeOptions = {}) {
         this.config = config;
         this.configPath = configPath;
         this.factories = { ...defaultFactories, ...factories };
+        this.runtimeOptions = runtimeOptions;
     }
     /**
      * Start the watcher, API server, and all components.
@@ -2129,7 +2280,11 @@ class JeevesWatcher {
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
         this.queue = queue;
-        const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger);
+        const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
+            maxRetries: this.config.maxRetries,
+            maxBackoffMs: this.config.maxBackoffMs,
+            onFatalError: this.runtimeOptions.onFatalError,
+        });
         this.watcher = watcher;
         const server = this.factories.createApiServer({
             processor,
@@ -2238,6 +2393,7 @@ exports.DocumentProcessor = DocumentProcessor;
 exports.EventQueue = EventQueue;
 exports.FileSystemWatcher = FileSystemWatcher;
 exports.JeevesWatcher = JeevesWatcher;
+exports.SystemHealth = SystemHealth;
 exports.VectorStoreClient = VectorStoreClient;
 exports.apiConfigSchema = apiConfigSchema;
 exports.applyRules = applyRules;

package/dist/cli/jeeves-watcher/index.js

@@ -668,6 +668,16 @@ const jeevesWatcherConfigSchema = z.object({
         .number()
         .optional()
         .describe('Timeout in milliseconds for graceful shutdown.'),
+    /** Maximum consecutive system-level failures before triggering fatal error. Default: Infinity. */
+    maxRetries: z
+        .number()
+        .optional()
+        .describe('Maximum consecutive system-level failures before triggering fatal error. Default: Infinity.'),
+    /** Maximum backoff delay in milliseconds for system errors. Default: 60000. */
+    maxBackoffMs: z
+        .number()
+        .optional()
+        .describe('Maximum backoff delay in milliseconds for system errors. Default: 60000.'),
 });
 
 /**
@@ -680,15 +690,13 @@ const ENV_PATTERN = /\$\{([^}]+)\}/g;
  * Replace `${VAR_NAME}` patterns in a string with `process.env.VAR_NAME`.
  *
  * @param value - The string to process.
- * @returns The string with env vars substituted.
- * @throws If a referenced env var is not set.
+ * @returns The string with resolved env vars; unresolvable expressions left untouched.
  */
 function substituteString(value) {
     return value.replace(ENV_PATTERN, (match, varName) => {
         const envValue = process.env[varName];
-        if (envValue === undefined) {
-            throw new Error(`Environment variable \${${varName}} referenced in config is not set.`);
-        }
+        if (envValue === undefined)
+            return match;
         return envValue;
     });
 }
@@ -1074,11 +1082,11 @@ async function extractMarkdown(filePath) {
 }
 async function extractPlaintext(filePath) {
     const raw = await readFile(filePath, 'utf8');
-    return { text: raw };
+    return { text: raw.replace(/^\uFEFF/, '') };
 }
 async function extractJson(filePath) {
     const raw = await readFile(filePath, 'utf8');
-    const parsed = JSON.parse(raw);
+    const parsed = JSON.parse(raw.replace(/^\uFEFF/, ''));
     const json = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
         ? parsed
         : undefined;
@@ -1100,7 +1108,7 @@ async function extractDocx(filePath) {
 }
 async function extractHtml(filePath) {
     const raw = await readFile(filePath, 'utf8');
-    const $ = cheerio.load(raw);
+    const $ = cheerio.load(raw.replace(/^\uFEFF/, ''));
     $('script, style').remove();
     const text = $('body').text().trim() || $.text().trim();
     return { text };
@@ -1930,6 +1938,112 @@ class VectorStoreClient {
     }
 }
 
+/**
+ * @module health
+ * Tracks consecutive system-level failures and applies exponential backoff.
+ * Triggers fatal error callback when maxRetries is exceeded.
+ */
+/**
+ * Tracks system health via consecutive failure count and exponential backoff.
+ */
+class SystemHealth {
+    consecutiveFailures = 0;
+    maxRetries;
+    maxBackoffMs;
+    baseDelayMs;
+    onFatalError;
+    logger;
+    constructor(options) {
+        this.maxRetries = options.maxRetries ?? Number.POSITIVE_INFINITY;
+        this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+        this.baseDelayMs = options.baseDelayMs ?? 1000;
+        this.onFatalError = options.onFatalError;
+        this.logger = options.logger;
+    }
+    /**
+     * Record a successful system operation. Resets the failure counter.
+     */
+    recordSuccess() {
+        if (this.consecutiveFailures > 0) {
+            this.logger.info({ previousFailures: this.consecutiveFailures }, 'System health recovered');
+        }
+        this.consecutiveFailures = 0;
+    }
+    /**
+     * Record a system-level failure. If maxRetries is exceeded, triggers fatal error.
+     *
+     * @param error - The error that occurred.
+     * @returns Whether the watcher should continue (false = fatal).
+     */
+    recordFailure(error) {
+        this.consecutiveFailures += 1;
+        this.logger.error({
+            consecutiveFailures: this.consecutiveFailures,
+            maxRetries: this.maxRetries,
+            err: normalizeError(error),
+        }, 'System-level failure recorded');
+        if (this.consecutiveFailures >= this.maxRetries) {
+            this.logger.fatal({ consecutiveFailures: this.consecutiveFailures }, 'Maximum retries exceeded, triggering fatal error');
+            if (this.onFatalError) {
+                this.onFatalError(error);
+                return false;
+            }
+            throw error instanceof Error
+                ? error
+                : new Error(`Fatal system error: ${String(error)}`);
+        }
+        return true;
+    }
+    /**
+     * Compute the current backoff delay based on consecutive failures.
+     *
+     * @returns Delay in milliseconds.
+     */
+    get currentBackoffMs() {
+        if (this.consecutiveFailures === 0)
+            return 0;
+        const exp = Math.max(0, this.consecutiveFailures - 1);
+        return Math.min(this.maxBackoffMs, this.baseDelayMs * 2 ** exp);
+    }
+    /**
+     * Sleep for the current backoff duration.
+     *
+     * @param signal - Optional abort signal.
+     */
+    async backoff(signal) {
+        const delay = this.currentBackoffMs;
+        if (delay <= 0)
+            return;
+        this.logger.warn({ delayMs: delay, consecutiveFailures: this.consecutiveFailures }, 'Backing off before next attempt');
+        await new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                cleanup();
+                resolve();
+            }, delay);
+            const onAbort = () => {
+                cleanup();
+                reject(new Error('Backoff aborted'));
+            };
+            const cleanup = () => {
+                clearTimeout(timer);
+                if (signal)
+                    signal.removeEventListener('abort', onAbort);
+            };
+            if (signal) {
+                if (signal.aborted) {
+                    onAbort();
+                    return;
+                }
+                signal.addEventListener('abort', onAbort, { once: true });
+            }
+        });
+    }
+    /** Current consecutive failure count. */
+    get failures() {
+        return this.consecutiveFailures;
+    }
+}
+
 /**
  * @module watcher
  * Filesystem watcher wrapping chokidar. I/O: watches files/directories for add/change/unlink events, enqueues to processing queue.
@@ -1942,6 +2056,7 @@ class FileSystemWatcher {
     queue;
     processor;
     logger;
+    health;
     watcher;
     /**
      * Create a new FileSystemWatcher.
@@ -1950,12 +2065,20 @@ class FileSystemWatcher {
      * @param queue - The event queue.
      * @param processor - The document processor.
      * @param logger - The logger instance.
+     * @param options - Optional health/fatal error options.
      */
-    constructor(config, queue, processor, logger) {
+    constructor(config, queue, processor, logger, options = {}) {
         this.config = config;
         this.queue = queue;
         this.processor = processor;
         this.logger = logger;
+        const healthOptions = {
+            maxRetries: options.maxRetries,
+            maxBackoffMs: options.maxBackoffMs,
+            onFatalError: options.onFatalError,
+            logger,
+        };
+        this.health = new SystemHealth(healthOptions);
     }
     /**
      * Start watching the filesystem and processing events.
@@ -1972,18 +2095,19 @@ class FileSystemWatcher {
         });
         this.watcher.on('add', (path) => {
             this.logger.debug({ path }, 'File added');
-            this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.processor.processFile(path));
+            this.queue.enqueue({ type: 'create', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('change', (path) => {
             this.logger.debug({ path }, 'File changed');
-            this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.processor.processFile(path));
+            this.queue.enqueue({ type: 'modify', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.processFile(path)));
         });
         this.watcher.on('unlink', (path) => {
             this.logger.debug({ path }, 'File removed');
-            this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.processor.deleteFile(path));
+            this.queue.enqueue({ type: 'delete', path, priority: 'normal' }, () => this.wrapProcessing(() => this.processor.deleteFile(path)));
         });
         this.watcher.on('error', (error) => {
             this.logger.error({ err: normalizeError(error) }, 'Watcher error');
+            this.health.recordFailure(error);
         });
         this.queue.process();
         this.logger.info({ paths: this.config.paths }, 'Filesystem watcher started');
@@ -1998,6 +2122,30 @@ class FileSystemWatcher {
             this.logger.info('Filesystem watcher stopped');
         }
     }
+    /**
+     * Get the system health tracker.
+     */
+    get systemHealth() {
+        return this.health;
+    }
+    /**
+     * Wrap a processing operation with health tracking.
+     * On success, resets the failure counter.
+     * On failure, records the failure and applies backoff.
+     */
+    async wrapProcessing(fn) {
+        try {
+            await this.health.backoff();
+            await fn();
+            this.health.recordSuccess();
+        }
+        catch (error) {
+            const shouldContinue = this.health.recordFailure(error);
+            if (!shouldContinue) {
+                await this.stop();
+            }
+        }
+    }
 }
 
 /**
@@ -2073,7 +2221,7 @@ const defaultFactories = {
     compileRules,
     createDocumentProcessor: (config, embeddingProvider, vectorStore, compiledRules, logger) => new DocumentProcessor(config, embeddingProvider, vectorStore, compiledRules, logger),
     createEventQueue: (options) => new EventQueue(options),
-    createFileSystemWatcher: (config, queue, processor, logger) => new FileSystemWatcher(config, queue, processor, logger),
+    createFileSystemWatcher: (config, queue, processor, logger, options) => new FileSystemWatcher(config, queue, processor, logger, options),
     createApiServer,
 };
 /**
@@ -2083,6 +2231,7 @@ class JeevesWatcher {
     config;
     configPath;
     factories;
+    runtimeOptions;
     logger;
     watcher;
     queue;
@@ -2095,11 +2244,13 @@ class JeevesWatcher {
      * @param config - The application configuration.
      * @param configPath - Optional config file path to watch for changes.
      * @param factories - Optional component factories (for dependency injection).
+     * @param runtimeOptions - Optional runtime-only options (e.g., onFatalError).
      */
-    constructor(config, configPath, factories = {}) {
+    constructor(config, configPath, factories = {}, runtimeOptions = {}) {
         this.config = config;
         this.configPath = configPath;
         this.factories = { ...defaultFactories, ...factories };
+        this.runtimeOptions = runtimeOptions;
     }
     /**
      * Start the watcher, API server, and all components.
@@ -2132,7 +2283,11 @@ class JeevesWatcher {
             rateLimitPerMinute: this.config.embedding.rateLimitPerMinute,
         });
         this.queue = queue;
-        const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger);
+        const watcher = this.factories.createFileSystemWatcher(this.config.watch, queue, processor, logger, {
+            maxRetries: this.config.maxRetries,
+            maxBackoffMs: this.config.maxBackoffMs,
+            onFatalError: this.runtimeOptions.onFatalError,
+        });
         this.watcher = watcher;
         const server = this.factories.createApiServer({
             processor,