@karmaniverous/jeeves-watcher 0.9.4 → 0.9.5

package/config.schema.json

@@ -189,7 +189,7 @@
       "description": "Named Qdrant filter patterns for skill-activated behaviors.",
       "allOf": [
         {
-          "$ref": "#/definitions/__schema82"
+          "$ref": "#/definitions/__schema83"
         }
       ]
     },
@@ -197,7 +197,7 @@
       "description": "Search configuration including score thresholds and hybrid search.",
       "allOf": [
         {
-          "$ref": "#/definitions/__schema83"
+          "$ref": "#/definitions/__schema84"
         }
       ]
     },
@@ -205,7 +205,7 @@
       "description": "Logging configuration.",
       "allOf": [
         {
-          "$ref": "#/definitions/__schema84"
+          "$ref": "#/definitions/__schema85"
         }
       ]
     },
@@ -213,7 +213,7 @@
       "description": "Timeout in milliseconds for graceful shutdown.",
       "allOf": [
         {
-          "$ref": "#/definitions/__schema87"
+          "$ref": "#/definitions/__schema88"
         }
       ]
     },
@@ -221,7 +221,7 @@
       "description": "Maximum consecutive system-level failures before triggering fatal error. Default: Infinity.",
       "allOf": [
         {
-          "$ref": "#/definitions/__schema88"
+          "$ref": "#/definitions/__schema89"
         }
       ]
     },
@@ -229,7 +229,7 @@
       "description": "Maximum backoff delay in milliseconds for system errors. Default: 60000.",
       "allOf": [
         {
-          "$ref": "#/definitions/__schema89"
+          "$ref": "#/definitions/__schema90"
         }
       ]
     }
@@ -1046,17 +1046,31 @@
         "callbackUrl": {
           "type": "string",
           "format": "uri"
+        },
+        "concurrency": {
+          "default": 50,
+          "description": "Maximum concurrent file operations during reindex (default 50).",
+          "allOf": [
+            {
+              "$ref": "#/definitions/__schema82"
+            }
+          ]
         }
       }
     },
     "__schema82": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991
+    },
+    "__schema83": {
       "type": "object",
       "propertyNames": {
         "type": "string"
       },
       "additionalProperties": {}
     },
-    "__schema83": {
+    "__schema84": {
       "type": "object",
       "properties": {
         "scoreThresholds": {
@@ -1101,14 +1115,14 @@
         }
       }
     },
-    "__schema84": {
+    "__schema85": {
       "type": "object",
       "properties": {
         "level": {
           "description": "Logging level (trace, debug, info, warn, error, fatal).",
           "allOf": [
             {
-              "$ref": "#/definitions/__schema85"
+              "$ref": "#/definitions/__schema86"
             }
           ]
         },
@@ -1116,26 +1130,26 @@
           "description": "Path to log file (logs to stdout if omitted).",
           "allOf": [
             {
-              "$ref": "#/definitions/__schema86"
+              "$ref": "#/definitions/__schema87"
             }
           ]
         }
       }
     },
-    "__schema85": {
-      "type": "string"
-    },
     "__schema86": {
       "type": "string"
     },
     "__schema87": {
-      "type": "number"
+      "type": "string"
     },
     "__schema88": {
       "type": "number"
     },
     "__schema89": {
       "type": "number"
+    },
+    "__schema90": {
+      "type": "number"
     }
   }
 }

package/dist/cli/jeeves-watcher/index.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
 import { Command } from '@commander-js/extra-typings';
+import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
 import { readdir, stat, writeFile, rm, readFile, mkdir } from 'node:fs/promises';
 import { resolve, dirname, join, extname, basename, isAbsolute, relative } from 'node:path';
 import picomatch from 'picomatch';
@@ -17,7 +18,6 @@ import dayjs from 'dayjs';
 import { toMdast } from 'hast-util-to-mdast';
 import { fromADF } from 'mdast-util-from-adf';
 import { toMarkdown } from 'mdast-util-to-markdown';
-import { capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
 import rehypeParse from 'rehype-parse';
 import { unified } from 'unified';
 import yaml from 'js-yaml';
@@ -225,6 +225,8 @@ async function listFilesFromGlobs(patterns, ignored = []) {
  *
  * Shared helper for processing all files matching configured globs.
  */
+/** Default concurrency limit for reindex operations. */
+const DEFAULT_REINDEX_CONCURRENCY = 50;
 /**
  * Process all files from globs using the specified processor method.
  *
@@ -232,15 +234,17 @@ async function listFilesFromGlobs(patterns, ignored = []) {
  * @param ignoredPaths - The glob patterns to ignore.
  * @param processor - The document processor instance.
  * @param method - The processor method to call ('processFile' or 'processRulesUpdate').
+ * @param concurrency - Maximum concurrent file operations (default 50).
+ * @param callbacks - Optional progress tracking callbacks.
  * @returns The number of files processed.
  */
-async function processAllFiles(watchPaths, ignoredPaths, processor, method) {
+async function processAllFiles(watchPaths, ignoredPaths, processor, method, concurrency = DEFAULT_REINDEX_CONCURRENCY, callbacks) {
     const files = await listFilesFromGlobs(watchPaths, ignoredPaths);
-    for (const file of files) {
-        // Sequential on purpose to avoid surprising load.
-        // Queue integration can come later.
+    callbacks?.onTotal?.(files.length);
+    await parallel(concurrency, files, async (file) => {
         await processor[method](file);
-    }
+        callbacks?.onFileProcessed?.();
+    });
     return files.length;
 }
 
@@ -289,8 +293,8 @@ async function executeReindex(deps, scope) {
             // Reprocess only files with issues
             const issues = deps.issuesManager.getAll();
             const issuePaths = Object.keys(issues);
-            filesProcessed = 0;
-            for (const filePath of issuePaths) {
+            const concurrency = config.reindex?.concurrency ?? 50;
+            await parallel(concurrency, issuePaths, async (filePath) => {
                 try {
                     await processor.processFile(filePath);
                     filesProcessed++;
@@ -299,11 +303,15 @@ async function executeReindex(deps, scope) {
                     errors++;
                     logger.warn({ filePath, err: normalizeError(error) }, 'Failed to reprocess issue file');
                 }
-            }
+            });
         }
         else {
             // Full reindex - process all watched files
-            filesProcessed = await processAllFiles(config.watch.paths, config.watch.ignored, processor, 'processFile');
+            const concurrency = config.reindex?.concurrency ?? 50;
+            filesProcessed = await processAllFiles(config.watch.paths, config.watch.ignored, processor, 'processFile', concurrency, {
+                onTotal: (total) => reindexTracker?.setTotal(total),
+                onFileProcessed: () => reindexTracker?.incrementProcessed(),
+            });
         }
         const durationMs = Date.now() - startTime;
         logger.info({ scope, filesProcessed, durationMs }, `Reindex (${scope}) completed`);
@@ -2156,6 +2164,13 @@ const jeevesWatcherConfigSchema = z.object({
         .object({
         /** URL to call when reindex completes. */
         callbackUrl: z.url().optional(),
+        /** Maximum concurrent file operations during reindex. */
+        concurrency: z
+            .number()
+            .int()
+            .min(1)
+            .default(50)
+            .describe('Maximum concurrent file operations during reindex (default 50).'),
     })
         .optional()
         .describe('Reindex configuration.'),
@@ -3209,7 +3224,7 @@ function createRebuildMetadataHandler(deps) {
  */
 function createReindexHandler(deps) {
     return wrapHandler(async (_request, reply) => {
-        const count = await processAllFiles(deps.watch.paths, deps.watch.ignored, deps.processor, 'processFile');
+        const count = await processAllFiles(deps.watch.paths, deps.watch.ignored, deps.processor, 'processFile', deps.concurrency);
         return await reply.status(200).send({ ok: true, filesIndexed: count });
     }, deps.logger, 'Reindex');
 }
@@ -3544,17 +3559,31 @@ class ReindexTracker {
     _active = false;
     _scope;
     _startedAt;
+    _filesProcessed = 0;
+    _totalFiles = 0;
     /** Mark a reindex as started. */
     start(scope) {
         this._active = true;
         this._scope = scope;
         this._startedAt = new Date().toISOString();
+        this._filesProcessed = 0;
+        this._totalFiles = 0;
+    }
+    /** Set the total number of files to process. */
+    setTotal(total) {
+        this._totalFiles = total;
+    }
+    /** Increment the processed file count. */
+    incrementProcessed() {
+        this._filesProcessed++;
     }
     /** Mark the current reindex as complete. */
     complete() {
         this._active = false;
         this._scope = undefined;
         this._startedAt = undefined;
+        this._filesProcessed = 0;
+        this._totalFiles = 0;
     }
     /** Get current reindex status. */
     getStatus() {
@@ -3564,6 +3593,8 @@ class ReindexTracker {
             active: true,
             scope: this._scope,
             startedAt: this._startedAt,
+            filesProcessed: this._filesProcessed,
+            totalFiles: this._totalFiles,
         };
     }
 }
@@ -3623,7 +3654,12 @@ function createApiServer(options) {
         logger,
         hybridConfig,
     }));
-    app.post('/reindex', createReindexHandler({ watch: config.watch, processor, logger }));
+    app.post('/reindex', createReindexHandler({
+        watch: config.watch,
+        processor,
+        logger,
+        concurrency: config.reindex?.concurrency ?? 50,
+    }));
     app.post('/rebuild-metadata', createRebuildMetadataHandler({
         metadataDir: config.metadataDir,
         vectorStore,

package/dist/index.d.ts

@@ -186,6 +186,7 @@ declare const jeevesWatcherConfigSchema: z.ZodObject<{
     }, z.core.$strip>>>;
     reindex: z.ZodOptional<z.ZodObject<{
         callbackUrl: z.ZodOptional<z.ZodURL>;
+        concurrency: z.ZodDefault<z.ZodNumber>;
     }, z.core.$strip>>;
     slots: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     search: z.ZodOptional<z.ZodObject<{
@@ -1240,6 +1241,10 @@ interface ReindexStatus {
     scope?: string;
     /** ISO 8601 timestamp when the current reindex started (when {@link active} is true). */
     startedAt?: string;
+    /** Number of files processed so far (when {@link active} is true). */
+    filesProcessed?: number;
+    /** Total number of files to process (when {@link active} is true). */
+    totalFiles?: number;
 }
 /**
  * Tracks the state of reindex operations.
@@ -1248,8 +1253,14 @@ declare class ReindexTracker {
     private _active;
     private _scope?;
     private _startedAt?;
+    private _filesProcessed;
+    private _totalFiles;
     /** Mark a reindex as started. */
     start(scope: 'issues' | 'full'): void;
+    /** Set the total number of files to process. */
+    setTotal(total: number): void;
+    /** Increment the processed file count. */
+    incrementProcessed(): void;
     /** Mark the current reindex as complete. */
     complete(): void;
     /** Get current reindex status. */

package/dist/index.js

@@ -8,7 +8,7 @@ import dayjs from 'dayjs';
 import { toMdast } from 'hast-util-to-mdast';
 import { fromADF } from 'mdast-util-from-adf';
 import { toMarkdown } from 'mdast-util-to-markdown';
-import { capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
+import { capitalize, title, camel, snake, dash, isEqual, get, parallel, omit } from 'radash';
 import rehypeParse from 'rehype-parse';
 import { unified } from 'unified';
 import yaml from 'js-yaml';
@@ -1317,6 +1317,8 @@ async function listFilesFromGlobs(patterns, ignored = []) {
  *
  * Shared helper for processing all files matching configured globs.
  */
+/** Default concurrency limit for reindex operations. */
+const DEFAULT_REINDEX_CONCURRENCY = 50;
 /**
  * Process all files from globs using the specified processor method.
  *
@@ -1324,15 +1326,17 @@ async function listFilesFromGlobs(patterns, ignored = []) {
  * @param ignoredPaths - The glob patterns to ignore.
  * @param processor - The document processor instance.
  * @param method - The processor method to call ('processFile' or 'processRulesUpdate').
+ * @param concurrency - Maximum concurrent file operations (default 50).
+ * @param callbacks - Optional progress tracking callbacks.
  * @returns The number of files processed.
  */
-async function processAllFiles(watchPaths, ignoredPaths, processor, method) {
+async function processAllFiles(watchPaths, ignoredPaths, processor, method, concurrency = DEFAULT_REINDEX_CONCURRENCY, callbacks) {
     const files = await listFilesFromGlobs(watchPaths, ignoredPaths);
-    for (const file of files) {
-        // Sequential on purpose to avoid surprising load.
-        // Queue integration can come later.
+    callbacks?.onTotal?.(files.length);
+    await parallel(concurrency, files, async (file) => {
         await processor[method](file);
-    }
+        callbacks?.onFileProcessed?.();
+    });
     return files.length;
 }
 
@@ -1381,8 +1385,8 @@ async function executeReindex(deps, scope) {
             // Reprocess only files with issues
             const issues = deps.issuesManager.getAll();
             const issuePaths = Object.keys(issues);
-            filesProcessed = 0;
-            for (const filePath of issuePaths) {
+            const concurrency = config.reindex?.concurrency ?? 50;
+            await parallel(concurrency, issuePaths, async (filePath) => {
                 try {
                     await processor.processFile(filePath);
                     filesProcessed++;
@@ -1391,11 +1395,15 @@ async function executeReindex(deps, scope) {
                     errors++;
                     logger.warn({ filePath, err: normalizeError(error) }, 'Failed to reprocess issue file');
                 }
-            }
+            });
         }
         else {
             // Full reindex - process all watched files
-            filesProcessed = await processAllFiles(config.watch.paths, config.watch.ignored, processor, 'processFile');
+            const concurrency = config.reindex?.concurrency ?? 50;
+            filesProcessed = await processAllFiles(config.watch.paths, config.watch.ignored, processor, 'processFile', concurrency, {
+                onTotal: (total) => reindexTracker?.setTotal(total),
+                onFileProcessed: () => reindexTracker?.incrementProcessed(),
+            });
         }
         const durationMs = Date.now() - startTime;
         logger.info({ scope, filesProcessed, durationMs }, `Reindex (${scope}) completed`);
@@ -1842,6 +1850,13 @@ const jeevesWatcherConfigSchema = z.object({
         .object({
         /** URL to call when reindex completes. */
         callbackUrl: z.url().optional(),
+        /** Maximum concurrent file operations during reindex. */
+        concurrency: z
+            .number()
+            .int()
+            .min(1)
+            .default(50)
+            .describe('Maximum concurrent file operations during reindex (default 50).'),
     })
         .optional()
         .describe('Reindex configuration.'),
@@ -2895,7 +2910,7 @@ function createRebuildMetadataHandler(deps) {
  */
 function createReindexHandler(deps) {
     return wrapHandler(async (_request, reply) => {
-        const count = await processAllFiles(deps.watch.paths, deps.watch.ignored, deps.processor, 'processFile');
+        const count = await processAllFiles(deps.watch.paths, deps.watch.ignored, deps.processor, 'processFile', deps.concurrency);
         return await reply.status(200).send({ ok: true, filesIndexed: count });
     }, deps.logger, 'Reindex');
 }
@@ -3230,17 +3245,31 @@ class ReindexTracker {
     _active = false;
     _scope;
     _startedAt;
+    _filesProcessed = 0;
+    _totalFiles = 0;
     /** Mark a reindex as started. */
     start(scope) {
         this._active = true;
         this._scope = scope;
         this._startedAt = new Date().toISOString();
+        this._filesProcessed = 0;
+        this._totalFiles = 0;
+    }
+    /** Set the total number of files to process. */
+    setTotal(total) {
+        this._totalFiles = total;
+    }
+    /** Increment the processed file count. */
+    incrementProcessed() {
+        this._filesProcessed++;
     }
     /** Mark the current reindex as complete. */
     complete() {
         this._active = false;
         this._scope = undefined;
         this._startedAt = undefined;
+        this._filesProcessed = 0;
+        this._totalFiles = 0;
     }
     /** Get current reindex status. */
     getStatus() {
@@ -3250,6 +3279,8 @@ class ReindexTracker {
             active: true,
             scope: this._scope,
             startedAt: this._startedAt,
+            filesProcessed: this._filesProcessed,
+            totalFiles: this._totalFiles,
         };
     }
 }
@@ -3309,7 +3340,12 @@ function createApiServer(options) {
         logger,
         hybridConfig,
     }));
-    app.post('/reindex', createReindexHandler({ watch: config.watch, processor, logger }));
+    app.post('/reindex', createReindexHandler({
+        watch: config.watch,
+        processor,
+        logger,
+        concurrency: config.reindex?.concurrency ?? 50,
+    }));
     app.post('/rebuild-metadata', createRebuildMetadataHandler({
         metadataDir: config.metadataDir,
         vectorStore,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@karmaniverous/jeeves-watcher",
-  "version": "0.9.4",
+  "version": "0.9.5",
   "author": "Jason Williscroft",
   "description": "Filesystem watcher that keeps a Qdrant vector store in sync with document changes",
   "license": "BSD-3-Clause",