@karmaniverous/jeeves-watcher 0.15.1 → 0.15.2

package/dist/cli/jeeves-watcher/index.js

@@ -1,13 +1,14 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
 import { Command } from '@commander-js/extra-typings';
-import { mkdirSync, existsSync, readFileSync, writeFileSync, statSync, readdirSync } from 'node:fs';
+import { mkdirSync, existsSync, readFileSync, writeFileSync, rmSync, renameSync, statSync, readdirSync } from 'node:fs';
 import { join, dirname, resolve, relative, extname, basename, isAbsolute } from 'node:path';
 import Database from 'better-sqlite3';
 import { z, ZodError } from 'zod';
 import Ajv from 'ajv';
 import addFormats from 'ajv-formats';
 import picomatch from 'picomatch';
+import { deserialize, serialize } from 'node:v8';
 import { readdir, stat, writeFile, readFile } from 'node:fs/promises';
 import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
 import ignore from 'ignore';
@@ -207,21 +208,32 @@ class ContentHashCache {
  * @module enrichment/EnrichmentStore
  * SQLite-backed enrichment metadata store. Persists path-keyed metadata at stateDir/enrichments.sqlite. Atomic writes, supports move.
  */
+const BUSY_TIMEOUT_MS = 5000;
 /**
  * SQLite-backed enrichment metadata store.
  */
 class EnrichmentStore {
     db;
+    logger;
     /**
      * Create or open the enrichment store.
      *
      * @param stateDir - Directory for the SQLite database file.
      */
-    constructor(stateDir) {
+    constructor(stateDir, logger) {
+        this.logger = logger;
         mkdirSync(stateDir, { recursive: true });
         const dbPath = join(stateDir, 'enrichments.sqlite');
         this.db = new Database(dbPath);
         this.db.pragma('journal_mode = WAL');
+        this.db.pragma('busy_timeout = ' + BUSY_TIMEOUT_MS.toString());
+        const [checkpointStatus] = this.db.pragma('wal_checkpoint(TRUNCATE)');
+        if (checkpointStatus && checkpointStatus.busy > 0) {
+            // EnrichmentStore is expected to be single-writer. If we see a busy WAL
+            // checkpoint at startup, it's most likely from an unclean shutdown where
+            // the OS hasn't yet released file handles.
+            this.logger?.warn({ checkpointStatus }, 'WAL checkpoint busy at startup; OS may still be releasing file handles');
+        }
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS enrichments (
         path TEXT PRIMARY KEY,
@@ -578,6 +590,92 @@ class VirtualRuleStore {
     }
 }
 
+/**
+ * @module util/BinaryFileStore
+ * Binary-backed read/modify/write store with in-memory caching and debounced flush.
+ *
+ * Persists a single JS object to disk using V8 structured clone serialization.
+ * I/O: synchronous fs read/write.
+ */
+/**
+ * Base class for binary file stores.
+ *
+ * @typeParam T - The stored data structure.
+ */
+class BinaryFileStore {
+    /** Path to the binary file on disk. */
+    filePath;
+    /** In-memory cache of the file contents, or `null` if not yet loaded. */
+    cache = null;
+    /** Logger instance for warnings and diagnostics. */
+    logger;
+    flushDebounceMs;
+    flushTimer;
+    dirty = false;
+    constructor(options) {
+        this.filePath = options.filePath;
+        this.logger = options.logger;
+        this.flushDebounceMs = options.flushDebounceMs ?? 5000;
+        mkdirSync(dirname(this.filePath), { recursive: true });
+    }
+    /** Load from disk into cache if not already loaded. */
+    load() {
+        if (this.cache)
+            return this.cache;
+        try {
+            if (existsSync(this.filePath)) {
+                const raw = readFileSync(this.filePath);
+                this.cache = deserialize(raw);
+            }
+            else {
+                this.cache = this.createEmpty();
+            }
+        }
+        catch (error) {
+            this.logger.warn({ filePath: this.filePath, err: error }, 'Failed to read binary store file, starting fresh');
+            this.cache = this.createEmpty();
+        }
+        return this.cache;
+    }
+    /**
+     * Mark the store dirty and schedule a debounced flush.
+     */
+    markDirty() {
+        this.dirty = true;
+        if (this.flushTimer)
+            return;
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = undefined;
+            this.flush();
+        }, this.flushDebounceMs);
+    }
+    /**
+     * Flush cache to disk if dirty.
+     *
+     * Uses an atomic write (tmp + rename) to avoid partial files.
+     */
+    flush() {
+        if (!this.dirty)
+            return;
+        this.stopAutoFlush();
+        const value = this.cache ?? this.createEmpty();
+        const tmpPath = `${this.filePath}.tmp`;
+        const payload = serialize(value);
+        writeFileSync(tmpPath, payload);
+        // renameSync does not reliably overwrite on Windows. Remove target first.
+        rmSync(this.filePath, { force: true });
+        renameSync(tmpPath, this.filePath);
+        this.dirty = false;
+    }
+    /** Stop any pending scheduled flush. Does not flush automatically. */
+    stopAutoFlush() {
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = undefined;
+        }
+    }
+}
+
 /**
  * @module values/ValuesManager
  * Manages per-rule distinct metadata value tracking. Persists to disk with in-memory caching and sorted deduplication.
@@ -585,9 +683,9 @@ class VirtualRuleStore {
 /**
  * Manages a persistent values.json file tracking distinct metadata values per rule.
  */
-class ValuesManager extends JsonFileStore {
+class ValuesManager extends BinaryFileStore {
     constructor(stateDir, logger) {
-        super({ filePath: join(stateDir, 'values.json'), logger });
+        super({ filePath: join(stateDir, 'values.v8'), logger });
     }
     createEmpty() {
         return {};
@@ -602,6 +700,7 @@ class ValuesManager extends JsonFileStore {
         const index = this.load();
         index[ruleName] ??= {};
         const ruleValues = index[ruleName];
+        let changed = false;
         for (const [key, value] of Object.entries(metadata)) {
             // Decompose arrays into individual trackable elements so that
             // array-typed fields (e.g. domains: ["email"]) are indexed.
@@ -619,15 +718,18 @@ class ValuesManager extends JsonFileStore {
                         }
                         return typeof a < typeof b ? -1 : 1;
                     });
+                    changed = true;
                 }
             }
         }
-        this.save();
+        if (changed)
+            this.markDirty();
     }
     /** Wipe all values (called on full reindex start). */
     clearAll() {
         this.cache = {};
-        this.save();
+        this.markDirty();
+        this.flush();
         this.logger.debug('All values cleared');
     }
     /** Get all current values. */
@@ -1361,9 +1463,19 @@ async function executeReindex(deps, scope, path, dryRun = false) {
     // Compute plan before starting async work
     let plan;
     if (scope === 'prune') {
-        const pruneResult = await computePrunePlan(deps);
+        deps.queue?.pause();
+        await deps.queue?.drain();
+        let pruneResult;
+        try {
+            pruneResult = await computePrunePlan(deps);
+        }
+        catch (err) {
+            deps.queue?.resume();
+            throw err;
+        }
         plan = pruneResult.plan;
         if (dryRun) {
+            deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
         // Execute prune
@@ -1410,6 +1522,9 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             }
             return { filesProcessed: 0, durationMs, errors: 1, plan };
         }
+        finally {
+            deps.queue?.resume();
+        }
     }
     // Non-prune scopes: compute plan from file lists
     let fileList;
@@ -3908,6 +4023,7 @@ function createConfigReindexHandler(deps) {
             issuesManager: deps.issuesManager,
             gitignoreFilter: deps.gitignoreFilter,
             vectorStore: deps.vectorStore,
+            queue: deps.queue,
         };
         // Pass path for 'path' and 'rules' scopes
         const pathParam = validScope === 'path' || validScope === 'rules'
@@ -4726,19 +4842,20 @@ function createWalkHandler(deps) {
                 message: 'The "globs" field is required and must be a non-empty string array.',
             });
         }
-        if (!deps.fileSystemWatcher) {
+        const fileSystemWatcher = deps.getFileSystemWatcher?.();
+        if (!fileSystemWatcher) {
             return await reply.status(503).send({
                 error: 'Watcher unavailable',
                 message: 'Filesystem watcher is not initialized.',
             });
         }
-        if (!deps.fileSystemWatcher.isReady) {
+        if (!fileSystemWatcher.isReady) {
             return await reply.status(503).send({
                 error: 'Scan in progress',
                 message: 'Initial filesystem scan is still active. Try again after scan completes.',
             });
         }
-        const watchedFiles = deps.fileSystemWatcher.getWatchedFiles();
+        const watchedFiles = fileSystemWatcher.getWatchedFiles();
         const normGlobs = globs.map((g) => normalizeSlashes(g));
         const matchGlobs = picomatch(normGlobs, { dot: true, nocase: true });
         const paths = watchedFiles.filter((f) => matchGlobs(normalizeSlashes(f)));
@@ -4944,8 +5061,9 @@ class ReindexTracker {
  * @returns A configured Fastify instance.
  */
 function createApiServer(options) {
-    const { processor, vectorStore, embeddingProvider, logger, config, issuesManager, valuesManager, configPath, helperIntrospection, virtualRuleStore, gitignoreFilter, version, initialScanTracker, } = options;
+    const { processor, vectorStore, embeddingProvider, queue, logger, config, issuesManager, valuesManager, configPath, helperIntrospection, virtualRuleStore, gitignoreFilter, version, initialScanTracker, } = options;
     const getConfig = options.getConfig ?? (() => config);
+    const getFileSystemWatcher = options.getFileSystemWatcher ?? (() => options.fileSystemWatcher);
     const reindexTracker = options.reindexTracker ?? new ReindexTracker();
     const app = Fastify({ logger: false });
     const triggerReindex = (scope) => {
@@ -4962,6 +5080,7 @@ function createApiServer(options) {
             issuesManager,
             gitignoreFilter,
             vectorStore,
+            queue,
         }, scope);
     };
     const cacheTtlMs = config.api?.cacheTtlMs ?? 30000;
@@ -4994,7 +5113,7 @@ function createApiServer(options) {
     }));
     app.post('/walk', createWalkHandler({
         getWatchPaths: () => getConfig().watch.paths,
-        fileSystemWatcher: options.fileSystemWatcher,
+        getFileSystemWatcher,
         logger,
     }));
     app.post('/search', createSearchHandler({
@@ -5022,6 +5141,7 @@ function createApiServer(options) {
         issuesManager,
         gitignoreFilter,
         vectorStore,
+        queue,
     }));
     app.get('/issues', withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
     app.get('/config/schema', withCache(cacheTtlMs, createConfigSchemaHandler()));
@@ -6154,6 +6274,7 @@ class EventQueue {
     concurrency;
     rateLimitPerMinute;
     started = false;
+    paused = false;
     active = 0;
     debounceTimers = new Map();
     latestByKey = new Map();
@@ -6203,6 +6324,19 @@ class EventQueue {
         this.started = true;
         this.pump();
     }
+    /**
+     * Pause processing events. Events can still be enqueued but will not be processed.
+     */
+    pause() {
+        this.paused = true;
+    }
+    /**
+     * Resume processing events.
+     */
+    resume() {
+        this.paused = false;
+        this.pump();
+    }
     /**
      * Wait for the queue to become idle (no pending debounces, no queued items, no active work).
      *
@@ -6242,7 +6376,7 @@ class EventQueue {
         return this.normalQueue.shift() ?? this.lowQueue.shift();
     }
     pump() {
-        if (!this.started)
+        if (!this.started || this.paused)
             return;
         while (this.active < this.concurrency) {
             const item = this.nextItem();
@@ -6555,6 +6689,7 @@ async function* scrollCollection(client, collectionName, filter, limit = 100) {
  */
 class VectorStoreClient {
     client;
+    writeClient;
     clientConfig;
     collectionName;
     dims;
@@ -6570,6 +6705,7 @@ class VectorStoreClient {
     constructor(config, dimensions, logger) {
         this.clientConfig = { url: config.url, apiKey: config.apiKey };
         this.client = this.createClient();
+        this.writeClient = this.client;
         this.collectionName = config.collectionName;
         this.dims = dimensions;
         this.log = getLogger(logger);
@@ -6626,7 +6762,7 @@ class VectorStoreClient {
             if (attempt > 1) {
                 this.log.warn({ attempt, operation: `qdrant.${operation}` }, `Retrying Qdrant ${operation}`);
             }
-            await fn();
+            await fn(attempt);
         }, {
             attempts: 5,
             baseDelayMs: 500,
@@ -6642,21 +6778,27 @@ class VectorStoreClient {
             },
         });
     }
+    getWriteClient(attempt) {
+        if (attempt > 1) {
+            this.pinoLogger?.info('Created fresh Qdrant client for retry');
+            this.writeClient = this.createClient();
+        }
+        return this.writeClient;
+    }
     /**
      * Upsert points into the collection.
      *
-     * Uses a fresh QdrantClient per attempt to avoid stale keep-alive connections.
-     * Between embedding calls and upserts, idle connections may be closed by the
-     * server, causing ECONNRESET on reuse.
+     * Uses the shared client. On retry (after ECONNRESET from stale connections),
+     * creates a fresh client to recover.
      *
      * @param points - The points to upsert.
      */
     async upsert(points) {
         if (points.length === 0)
             return;
-        await this.retryOperation('upsert', async () => {
-            const freshClient = this.createClient();
-            await freshClient.upsert(this.collectionName, {
+        await this.retryOperation('upsert', async (attempt) => {
+            const client = this.getWriteClient(attempt);
+            await client.upsert(this.collectionName, {
                 wait: true,
                 points: points.map((p) => ({
                     id: p.id,
@@ -6669,16 +6811,16 @@ class VectorStoreClient {
     /**
      * Delete points by their IDs.
      *
-     * Uses a fresh QdrantClient per attempt to avoid stale keep-alive connections.
+     * Uses the shared client. On retry, creates a fresh client to recover.
      *
      * @param ids - The point IDs to delete.
      */
     async delete(ids) {
         if (ids.length === 0)
             return;
-        await this.retryOperation('delete', async () => {
-            const freshClient = this.createClient();
-            await freshClient.delete(this.collectionName, {
+        await this.retryOperation('delete', async (attempt) => {
+            const client = this.getWriteClient(attempt);
+            await client.delete(this.collectionName, {
                 wait: true,
                 points: ids,
             });
@@ -7546,7 +7688,7 @@ class JeevesWatcher {
         const stateDir = this.config.stateDir ?? '.jeeves-metadata';
         this.issuesManager = new IssuesManager(stateDir, logger);
         this.valuesManager = new ValuesManager(stateDir, logger);
-        this.enrichmentStore = new EnrichmentStore(stateDir);
+        this.enrichmentStore = new EnrichmentStore(stateDir, logger);
         const enrichmentStore = this.enrichmentStore;
         this.contentHashCache = new ContentHashCache();
         const contentHashCache = this.contentHashCache;
@@ -7598,6 +7740,9 @@ class JeevesWatcher {
                 this.logger?.warn({ timeoutMs: timeout }, 'Queue drain timeout hit, forcing shutdown');
             }
         }
+        // Flush in-memory state caches before shutdown.
+        this.valuesManager?.stopAutoFlush();
+        this.valuesManager?.flush();
         this.enrichmentStore?.close();
         if (this.server) {
             await this.server.close();
@@ -7622,6 +7767,7 @@ class JeevesWatcher {
             version: this.version,
             initialScanTracker: this.initialScanTracker,
             fileSystemWatcher: this.watcher,
+            getFileSystemWatcher: () => this.watcher,
             enrichmentStore: this.enrichmentStore,
         });
         await server.listen({

package/dist/index.d.ts

@@ -300,6 +300,7 @@ declare function createEmbeddingProvider(config: EmbeddingConfig, logger?: pino.
  * @module enrichment/EnrichmentStore
  * SQLite-backed enrichment metadata store. Persists path-keyed metadata at stateDir/enrichments.sqlite. Atomic writes, supports move.
  */
+
 /**
  * Interface for enrichment metadata persistence.
  */
@@ -322,12 +323,13 @@ interface EnrichmentStoreInterface {
  */
 declare class EnrichmentStore implements EnrichmentStoreInterface {
     private readonly db;
+    private readonly logger?;
     /**
      * Create or open the enrichment store.
      *
      * @param stateDir - Directory for the SQLite database file.
      */
-    constructor(stateDir: string);
+    constructor(stateDir: string, logger?: pino.Logger);
     get(path: string): Record<string, unknown> | null;
     set(path: string, metadata: Record<string, unknown>): void;
     delete(path: string): void;
@@ -806,6 +808,57 @@ declare class VirtualRuleStore {
     get size(): number;
 }
 
+/**
+ * @module util/BinaryFileStore
+ * Binary-backed read/modify/write store with in-memory caching and debounced flush.
+ *
+ * Persists a single JS object to disk using V8 structured clone serialization.
+ * I/O: synchronous fs read/write.
+ */
+
+/** Options for {@link BinaryFileStore}. */
+interface BinaryFileStoreOptions {
+    /** Path to the binary file on disk. */
+    filePath: string;
+    /** Logger for warnings. */
+    logger: pino.Logger;
+    /** Debounce interval in ms for flushing dirty state. Default: 5000. */
+    flushDebounceMs?: number;
+}
+/**
+ * Base class for binary file stores.
+ *
+ * @typeParam T - The stored data structure.
+ */
+declare abstract class BinaryFileStore<T> {
+    /** Path to the binary file on disk. */
+    protected readonly filePath: string;
+    /** In-memory cache of the file contents, or `null` if not yet loaded. */
+    protected cache: T | null;
+    /** Logger instance for warnings and diagnostics. */
+    protected readonly logger: pino.Logger;
+    private readonly flushDebounceMs;
+    private flushTimer;
+    private dirty;
+    protected constructor(options: BinaryFileStoreOptions);
+    /** Create an empty default value when file is missing or unreadable. */
+    protected abstract createEmpty(): T;
+    /** Load from disk into cache if not already loaded. */
+    protected load(): T;
+    /**
+     * Mark the store dirty and schedule a debounced flush.
+     */
+    protected markDirty(): void;
+    /**
+     * Flush cache to disk if dirty.
+     *
+     * Uses an atomic write (tmp + rename) to avoid partial files.
+     */
+    flush(): void;
+    /** Stop any pending scheduled flush. Does not flush automatically. */
+    stopAutoFlush(): void;
+}
+
 /**
  * @module values/ValuesManager
  * Manages per-rule distinct metadata value tracking. Persists to disk with in-memory caching and sorted deduplication.
@@ -816,7 +869,7 @@ type ValuesIndex = Record<string, Record<string, unknown[]>>;
 /**
  * Manages a persistent values.json file tracking distinct metadata values per rule.
  */
-declare class ValuesManager extends JsonFileStore<ValuesIndex> {
+declare class ValuesManager extends BinaryFileStore<ValuesIndex> {
     constructor(stateDir: string, logger: pino.Logger);
     protected createEmpty(): ValuesIndex;
     /** Check if a value is a trackable primitive (string, number, boolean). */
@@ -997,6 +1050,7 @@ interface VectorStore {
  */
 declare class VectorStoreClient implements VectorStore {
     private readonly client;
+    private writeClient;
     private readonly clientConfig;
     private readonly collectionName;
     private readonly dims;
@@ -1037,12 +1091,12 @@ declare class VectorStoreClient implements VectorStore {
      * @param fn - Async function to retry.
      */
     private retryOperation;
+    private getWriteClient;
     /**
      * Upsert points into the collection.
      *
-     * Uses a fresh QdrantClient per attempt to avoid stale keep-alive connections.
-     * Between embedding calls and upserts, idle connections may be closed by the
-     * server, causing ECONNRESET on reuse.
+     * Uses the shared client. On retry (after ECONNRESET from stale connections),
+     * creates a fresh client to recover.
      *
      * @param points - The points to upsert.
      */
@@ -1050,7 +1104,7 @@ declare class VectorStoreClient implements VectorStore {
     /**
      * Delete points by their IDs.
      *
-     * Uses a fresh QdrantClient per attempt to avoid stale keep-alive connections.
+     * Uses the shared client. On retry, creates a fresh client to recover.
      *
      * @param ids - The point IDs to delete.
      */
@@ -1360,6 +1414,7 @@ declare class EventQueue {
     private readonly concurrency;
     private readonly rateLimitPerMinute?;
     private started;
+    private paused;
     private active;
     private readonly debounceTimers;
     private readonly latestByKey;
@@ -1385,6 +1440,14 @@ declare class EventQueue {
      * Start processing events.
      */
     process(): void;
+    /**
+     * Pause processing events. Events can still be enqueued but will not be processed.
+     */
+    pause(): void;
+    /**
+     * Resume processing events.
+     */
+    resume(): void;
     /**
      * Wait for the queue to become idle (no pending debounces, no queued items, no active work).
      *
@@ -1663,6 +1726,8 @@ interface ApiServerOptions {
     initialScanTracker?: InitialScanTracker;
     /** Filesystem watcher instance for /walk endpoint (in-memory file list). */
     fileSystemWatcher?: FileSystemWatcher;
+    /** Getter for live filesystem watcher access after hot-reload rebuilds. */
+    getFileSystemWatcher?: () => FileSystemWatcher | undefined;
     /** Optional enrichment store for persisted enrichment metadata. */
     enrichmentStore?: EnrichmentStoreInterface;
 }
@@ -1836,5 +1901,5 @@ declare function contentHash(text: string): string;
  */
 declare function pointId(filePath: string, chunkIndex?: number): string;
 
-export { DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SystemHealth, TemplateEngine, ValuesManager, VectorStoreClient, VirtualRuleStore, apiConfigSchema, applyRules, buildAttributes, buildTemplateEngine, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, embeddingConfigSchema, extractText, inferenceRuleSchema, issueRecordSchema, jeevesWatcherConfigSchema, loadConfig, loadCustomHelpers, loggingConfigSchema, mergeEnrichment, pointId, registerBuiltinHelpers, resolveTemplateSource, startFromConfig, vectorStoreConfigSchema, watchConfigSchema };
+export { ContentHashCache, DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SystemHealth, TemplateEngine, ValuesManager, VectorStoreClient, VirtualRuleStore, apiConfigSchema, applyRules, buildAttributes, buildTemplateEngine, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, embeddingConfigSchema, extractText, inferenceRuleSchema, issueRecordSchema, jeevesWatcherConfigSchema, loadConfig, loadCustomHelpers, loggingConfigSchema, mergeEnrichment, pointId, registerBuiltinHelpers, resolveTemplateSource, startFromConfig, vectorStoreConfigSchema, watchConfigSchema };
 export type { AllHelpersIntrospection, ApiConfig, ApiServerOptions, ApplyRulesOptions, ApplyRulesResult, CollectionInfo, CompiledRule, CompiledTemplate, ConfigWatchConfig, DocumentProcessorDeps, DocumentProcessorInterface, EmbeddingConfig, EmbeddingProvider, EnrichmentStoreInterface, EventQueueOptions, ExtractedText, Extractor, FileAttributes, FileSystemWatcherOptions, HelperModuleIntrospection, InferenceRule, InitialScanStatus, IssueRecord, IssuesFile, JeevesWatcherConfig, JeevesWatcherConfigInput, JeevesWatcherFactories, JeevesWatcherRuntimeOptions, LoggingConfig, PayloadFieldSchema, ProcessFn, ProcessorConfig, ProviderFactory, ReindexStatus, RenderResult, RuleLogger, ScrollPageResult, ScrolledPoint, SearchResult, SystemHealthOptions, ValuesIndex, VectorPoint, VectorStore, VectorStoreConfig, WatchConfig, WatchEvent };

package/dist/index.js

@@ -2,7 +2,7 @@ import { join, dirname, resolve, relative, extname, basename, isAbsolute } from
 import Fastify from 'fastify';
 import { readdir, stat, writeFile, readFile } from 'node:fs/promises';
 import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
-import { existsSync, statSync, readFileSync, readdirSync, mkdirSync, writeFileSync } from 'node:fs';
+import { existsSync, statSync, readFileSync, readdirSync, mkdirSync, writeFileSync, rmSync, renameSync } from 'node:fs';
 import ignore from 'ignore';
 import picomatch from 'picomatch';
 import { z, ZodError } from 'zod';
@@ -22,6 +22,7 @@ import yaml from 'js-yaml';
 import crypto from 'crypto';
 import { packageDirectorySync } from 'package-directory';
 import Database from 'better-sqlite3';
+import { deserialize, serialize } from 'node:v8';
 import chokidar from 'chokidar';
 import { cosmiconfig } from 'cosmiconfig';
 import https from 'node:https';
@@ -753,9 +754,19 @@ async function executeReindex(deps, scope, path, dryRun = false) {
     // Compute plan before starting async work
     let plan;
     if (scope === 'prune') {
-        const pruneResult = await computePrunePlan(deps);
+        deps.queue?.pause();
+        await deps.queue?.drain();
+        let pruneResult;
+        try {
+            pruneResult = await computePrunePlan(deps);
+        }
+        catch (err) {
+            deps.queue?.resume();
+            throw err;
+        }
         plan = pruneResult.plan;
         if (dryRun) {
+            deps.queue?.resume();
             return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
         }
         // Execute prune
@@ -802,6 +813,9 @@ async function executeReindex(deps, scope, path, dryRun = false) {
             }
             return { filesProcessed: 0, durationMs, errors: 1, plan };
         }
+        finally {
+            deps.queue?.resume();
+        }
     }
     // Non-prune scopes: compute plan from file lists
     let fileList;
@@ -1986,6 +2000,7 @@ function createConfigReindexHandler(deps) {
             issuesManager: deps.issuesManager,
             gitignoreFilter: deps.gitignoreFilter,
             vectorStore: deps.vectorStore,
+            queue: deps.queue,
         };
         // Pass path for 'path' and 'rules' scopes
         const pathParam = validScope === 'path' || validScope === 'rules'
@@ -3853,19 +3868,20 @@ function createWalkHandler(deps) {
                 message: 'The "globs" field is required and must be a non-empty string array.',
             });
         }
-        if (!deps.fileSystemWatcher) {
+        const fileSystemWatcher = deps.getFileSystemWatcher?.();
+        if (!fileSystemWatcher) {
             return await reply.status(503).send({
                 error: 'Watcher unavailable',
                 message: 'Filesystem watcher is not initialized.',
             });
         }
-        if (!deps.fileSystemWatcher.isReady) {
+        if (!fileSystemWatcher.isReady) {
             return await reply.status(503).send({
                 error: 'Scan in progress',
                 message: 'Initial filesystem scan is still active. Try again after scan completes.',
             });
         }
-        const watchedFiles = deps.fileSystemWatcher.getWatchedFiles();
+        const watchedFiles = fileSystemWatcher.getWatchedFiles();
         const normGlobs = globs.map((g) => normalizeSlashes(g));
         const matchGlobs = picomatch(normGlobs, { dot: true, nocase: true });
         const paths = watchedFiles.filter((f) => matchGlobs(normalizeSlashes(f)));
@@ -4446,8 +4462,9 @@ class InitialScanTracker {
  * @returns A configured Fastify instance.
  */
 function createApiServer(options) {
-    const { processor, vectorStore, embeddingProvider, logger, config, issuesManager, valuesManager, configPath, helperIntrospection, virtualRuleStore, gitignoreFilter, version, initialScanTracker, } = options;
+    const { processor, vectorStore, embeddingProvider, queue, logger, config, issuesManager, valuesManager, configPath, helperIntrospection, virtualRuleStore, gitignoreFilter, version, initialScanTracker, } = options;
     const getConfig = options.getConfig ?? (() => config);
+    const getFileSystemWatcher = options.getFileSystemWatcher ?? (() => options.fileSystemWatcher);
     const reindexTracker = options.reindexTracker ?? new ReindexTracker();
     const app = Fastify({ logger: false });
     const triggerReindex = (scope) => {
@@ -4464,6 +4481,7 @@ function createApiServer(options) {
             issuesManager,
             gitignoreFilter,
             vectorStore,
+            queue,
         }, scope);
     };
     const cacheTtlMs = config.api?.cacheTtlMs ?? 30000;
@@ -4496,7 +4514,7 @@ function createApiServer(options) {
     }));
     app.post('/walk', createWalkHandler({
         getWatchPaths: () => getConfig().watch.paths,
-        fileSystemWatcher: options.fileSystemWatcher,
+        getFileSystemWatcher,
         logger,
     }));
     app.post('/search', createSearchHandler({
@@ -4524,6 +4542,7 @@ function createApiServer(options) {
         issuesManager,
         gitignoreFilter,
         vectorStore,
+        queue,
     }));
     app.get('/issues', withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
     app.get('/config/schema', withCache(cacheTtlMs, createConfigSchemaHandler()));
@@ -4692,21 +4711,32 @@ class ContentHashCache {
  * @module enrichment/EnrichmentStore
  * SQLite-backed enrichment metadata store. Persists path-keyed metadata at stateDir/enrichments.sqlite. Atomic writes, supports move.
  */
+const BUSY_TIMEOUT_MS = 5000;
 /**
  * SQLite-backed enrichment metadata store.
  */
 class EnrichmentStore {
     db;
+    logger;
     /**
      * Create or open the enrichment store.
      *
      * @param stateDir - Directory for the SQLite database file.
      */
-    constructor(stateDir) {
+    constructor(stateDir, logger) {
+        this.logger = logger;
         mkdirSync(stateDir, { recursive: true });
         const dbPath = join(stateDir, 'enrichments.sqlite');
         this.db = new Database(dbPath);
         this.db.pragma('journal_mode = WAL');
+        this.db.pragma('busy_timeout = ' + BUSY_TIMEOUT_MS.toString());
+        const [checkpointStatus] = this.db.pragma('wal_checkpoint(TRUNCATE)');
+        if (checkpointStatus && checkpointStatus.busy > 0) {
+            // EnrichmentStore is expected to be single-writer. If we see a busy WAL
+            // checkpoint at startup, it's most likely from an unclean shutdown where
+            // the OS hasn't yet released file handles.
+            this.logger?.warn({ checkpointStatus }, 'WAL checkpoint busy at startup; OS may still be releasing file handles');
+        }
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS enrichments (
         path TEXT PRIMARY KEY,
@@ -4926,6 +4956,92 @@ const issueRecordSchema = z.object({
     timestamp: z.union([z.number(), z.string()]),
 });
 
+/**
+ * @module util/BinaryFileStore
+ * Binary-backed read/modify/write store with in-memory caching and debounced flush.
+ *
+ * Persists a single JS object to disk using V8 structured clone serialization.
+ * I/O: synchronous fs read/write.
+ */
+/**
+ * Base class for binary file stores.
+ *
+ * @typeParam T - The stored data structure.
+ */
+class BinaryFileStore {
+    /** Path to the binary file on disk. */
+    filePath;
+    /** In-memory cache of the file contents, or `null` if not yet loaded. */
+    cache = null;
+    /** Logger instance for warnings and diagnostics. */
+    logger;
+    flushDebounceMs;
+    flushTimer;
+    dirty = false;
+    constructor(options) {
+        this.filePath = options.filePath;
+        this.logger = options.logger;
+        this.flushDebounceMs = options.flushDebounceMs ?? 5000;
+        mkdirSync(dirname(this.filePath), { recursive: true });
+    }
+    /** Load from disk into cache if not already loaded. */
+    load() {
+        if (this.cache)
+            return this.cache;
+        try {
+            if (existsSync(this.filePath)) {
+                const raw = readFileSync(this.filePath);
+                this.cache = deserialize(raw);
+            }
+            else {
+                this.cache = this.createEmpty();
+            }
+        }
+        catch (error) {
+            this.logger.warn({ filePath: this.filePath, err: error }, 'Failed to read binary store file, starting fresh');
+            this.cache = this.createEmpty();
+        }
+        return this.cache;
+    }
+    /**
+     * Mark the store dirty and schedule a debounced flush.
+     */
+    markDirty() {
+        this.dirty = true;
+        if (this.flushTimer)
+            return;
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = undefined;
+            this.flush();
+        }, this.flushDebounceMs);
+    }
+    /**
+     * Flush cache to disk if dirty.
+     *
+     * Uses an atomic write (tmp + rename) to avoid partial files.
+     */
+    flush() {
+        if (!this.dirty)
+            return;
+        this.stopAutoFlush();
+        const value = this.cache ?? this.createEmpty();
+        const tmpPath = `${this.filePath}.tmp`;
+        const payload = serialize(value);
+        writeFileSync(tmpPath, payload);
+        // renameSync does not reliably overwrite on Windows. Remove target first.
+        rmSync(this.filePath, { force: true });
+        renameSync(tmpPath, this.filePath);
+        this.dirty = false;
+    }
+    /** Stop any pending scheduled flush. Does not flush automatically. */
+    stopAutoFlush() {
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = undefined;
+        }
+    }
+}
+
 /**
  * @module values/ValuesManager
  * Manages per-rule distinct metadata value tracking. Persists to disk with in-memory caching and sorted deduplication.
@@ -4933,9 +5049,9 @@ const issueRecordSchema = z.object({
 /**
  * Manages a persistent values.json file tracking distinct metadata values per rule.
  */
-class ValuesManager extends JsonFileStore {
+class ValuesManager extends BinaryFileStore {
     constructor(stateDir, logger) {
-        super({ filePath: join(stateDir, 'values.json'), logger });
+        super({ filePath: join(stateDir, 'values.v8'), logger });
     }
     createEmpty() {
         return {};
@@ -4950,6 +5066,7 @@ class ValuesManager extends JsonFileStore {
         const index = this.load();
         index[ruleName] ??= {};
         const ruleValues = index[ruleName];
+        let changed = false;
         for (const [key, value] of Object.entries(metadata)) {
             // Decompose arrays into individual trackable elements so that
             // array-typed fields (e.g. domains: ["email"]) are indexed.
@@ -4967,15 +5084,18 @@ class ValuesManager extends JsonFileStore {
                         }
                         return typeof a < typeof b ? -1 : 1;
                     });
+                    changed = true;
                 }
             }
         }
-        this.save();
+        if (changed)
+            this.markDirty();
     }
     /** Wipe all values (called on full reindex start). */
     clearAll() {
         this.cache = {};
-        this.save();
+        this.markDirty();
+        this.flush();
         this.logger.debug('All values cleared');
     }
     /** Get all current values. */
@@ -6132,6 +6252,7 @@ class EventQueue {
     concurrency;
     rateLimitPerMinute;
     started = false;
+    paused = false;
     active = 0;
     debounceTimers = new Map();
     latestByKey = new Map();
@@ -6181,6 +6302,19 @@ class EventQueue {
         this.started = true;
         this.pump();
     }
+    /**
+     * Pause processing events. Events can still be enqueued but will not be processed.
+     */
+    pause() {
+        this.paused = true;
+    }
+    /**
+     * Resume processing events.
+     */
+    resume() {
+        this.paused = false;
+        this.pump();
+    }
     /**
      * Wait for the queue to become idle (no pending debounces, no queued items, no active work).
      *
@@ -6220,7 +6354,7 @@ class EventQueue {
         return this.normalQueue.shift() ?? this.lowQueue.shift();
     }
     pump() {
-        if (!this.started)
+        if (!this.started || this.paused)
             return;
         while (this.active < this.concurrency) {
             const item = this.nextItem();
@@ -6533,6 +6667,7 @@ async function* scrollCollection(client, collectionName, filter, limit = 100) {
  */
 class VectorStoreClient {
     client;
+    writeClient;
     clientConfig;
     collectionName;
     dims;
@@ -6548,6 +6683,7 @@ class VectorStoreClient {
     constructor(config, dimensions, logger) {
         this.clientConfig = { url: config.url, apiKey: config.apiKey };
         this.client = this.createClient();
+        this.writeClient = this.client;
         this.collectionName = config.collectionName;
         this.dims = dimensions;
         this.log = getLogger(logger);
@@ -6604,7 +6740,7 @@ class VectorStoreClient {
             if (attempt > 1) {
                 this.log.warn({ attempt, operation: `qdrant.${operation}` }, `Retrying Qdrant ${operation}`);
             }
-            await fn();
+            await fn(attempt);
         }, {
             attempts: 5,
             baseDelayMs: 500,
@@ -6620,21 +6756,27 @@ class VectorStoreClient {
             },
         });
     }
+    getWriteClient(attempt) {
+        if (attempt > 1) {
+            this.pinoLogger?.info('Created fresh Qdrant client for retry');
+            this.writeClient = this.createClient();
+        }
+        return this.writeClient;
+    }
     /**
      * Upsert points into the collection.
      *
-     * Uses a fresh QdrantClient per attempt to avoid stale keep-alive connections.
-     * Between embedding calls and upserts, idle connections may be closed by the
-     * server, causing ECONNRESET on reuse.
+     * Uses the shared client. On retry (after ECONNRESET from stale connections),
+     * creates a fresh client to recover.
      *
      * @param points - The points to upsert.
      */
     async upsert(points) {
         if (points.length === 0)
             return;
-        await this.retryOperation('upsert', async () => {
-            const freshClient = this.createClient();
-            await freshClient.upsert(this.collectionName, {
+        await this.retryOperation('upsert', async (attempt) => {
+            const client = this.getWriteClient(attempt);
+            await client.upsert(this.collectionName, {
                 wait: true,
                 points: points.map((p) => ({
                     id: p.id,
@@ -6647,16 +6789,16 @@ class VectorStoreClient {
     /**
      * Delete points by their IDs.
      *
-     * Uses a fresh QdrantClient per attempt to avoid stale keep-alive connections.
+     * Uses the shared client. On retry, creates a fresh client to recover.
      *
      * @param ids - The point IDs to delete.
      */
     async delete(ids) {
         if (ids.length === 0)
             return;
-        await this.retryOperation('delete', async () => {
-            const freshClient = this.createClient();
-            await freshClient.delete(this.collectionName, {
+        await this.retryOperation('delete', async (attempt) => {
+            const client = this.getWriteClient(attempt);
+            await client.delete(this.collectionName, {
                 wait: true,
                 points: ids,
             });
@@ -7524,7 +7666,7 @@ class JeevesWatcher {
         const stateDir = this.config.stateDir ?? '.jeeves-metadata';
         this.issuesManager = new IssuesManager(stateDir, logger);
         this.valuesManager = new ValuesManager(stateDir, logger);
-        this.enrichmentStore = new EnrichmentStore(stateDir);
+        this.enrichmentStore = new EnrichmentStore(stateDir, logger);
         const enrichmentStore = this.enrichmentStore;
         this.contentHashCache = new ContentHashCache();
         const contentHashCache = this.contentHashCache;
@@ -7576,6 +7718,9 @@ class JeevesWatcher {
                 this.logger?.warn({ timeoutMs: timeout }, 'Queue drain timeout hit, forcing shutdown');
             }
         }
+        // Flush in-memory state caches before shutdown.
+        this.valuesManager?.stopAutoFlush();
+        this.valuesManager?.flush();
         this.enrichmentStore?.close();
         if (this.server) {
             await this.server.close();
@@ -7600,6 +7745,7 @@ class JeevesWatcher {
             version: this.version,
             initialScanTracker: this.initialScanTracker,
             fileSystemWatcher: this.watcher,
+            getFileSystemWatcher: () => this.watcher,
             enrichmentStore: this.enrichmentStore,
         });
         await server.listen({
@@ -7662,4 +7808,4 @@ class JeevesWatcher {
     }
 }
 
-export { DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SystemHealth, TemplateEngine, ValuesManager, VectorStoreClient, VirtualRuleStore, apiConfigSchema, applyRules, buildAttributes, buildTemplateEngine, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, embeddingConfigSchema, extractText, inferenceRuleSchema, issueRecordSchema, jeevesWatcherConfigSchema, loadConfig, loadCustomHelpers, loggingConfigSchema, mergeEnrichment, pointId, registerBuiltinHelpers, resolveTemplateSource, startFromConfig, vectorStoreConfigSchema, watchConfigSchema };
+export { ContentHashCache, DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SystemHealth, TemplateEngine, ValuesManager, VectorStoreClient, VirtualRuleStore, apiConfigSchema, applyRules, buildAttributes, buildTemplateEngine, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, embeddingConfigSchema, extractText, inferenceRuleSchema, issueRecordSchema, jeevesWatcherConfigSchema, loadConfig, loadCustomHelpers, loggingConfigSchema, mergeEnrichment, pointId, registerBuiltinHelpers, resolveTemplateSource, startFromConfig, vectorStoreConfigSchema, watchConfigSchema };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@karmaniverous/jeeves-watcher",
-  "version": "0.15.1",
+  "version": "0.15.2",
   "author": "Jason Williscroft",
   "description": "Filesystem watcher that keeps a Qdrant vector store in sync with document changes",
   "license": "BSD-3-Clause",