@mcp-monorepo/notion-query 1.2.0 → 1.3.0

package/dist/local-rag/utils/pool.js

@@ -0,0 +1,140 @@
+import { cpus } from 'node:os';
+import { Worker } from 'node:worker_threads';
+import { logger } from '@mcp-monorepo/shared';
+import { EmbeddingError } from '../errors.js';
+export class WorkerPool {
+    workerPath;
+    workerData;
+    config;
+    workers = [];
+    taskQueue = [];
+    taskIdCounter = 0;
+    workerIdCounter = 0;
+    activeTasks = new Map();
+    idleCheckInterval;
+    constructor(workerPath, workerData, config = {}) {
+        this.workerPath = workerPath;
+        this.workerData = workerData;
+        this.config = {
+            maxWorkers: config.maxWorkers ?? Math.max(1, cpus().length - 1),
+            minWorkers: config.minWorkers ?? 0,
+            idleTimeoutMs: config.idleTimeoutMs ?? 1_800_000,
+        };
+        if (this.config.idleTimeoutMs > 0) {
+            this.idleCheckInterval = setInterval(() => this._reapIdleWorkers(), this.config.idleTimeoutMs / 2);
+        }
+    }
+    runTask(task) {
+        const taskId = this.taskIdCounter++;
+        return new Promise((resolve, reject) => {
+            this.activeTasks.set(taskId, { resolve, reject });
+            this.taskQueue.push({ taskId, task, resolve, reject });
+            this._dispatch();
+        });
+    }
+    _dispatch() {
+        if (this.taskQueue.length === 0) {
+            return;
+        }
+        // 1. Prioritize using an existing idle worker
+        const availableWorker = this.workers.find((w) => w.currentTaskId === undefined);
+        if (availableWorker) {
+            const taskInfo = this.taskQueue.shift();
+            if (taskInfo) {
+                this._assignTask(availableWorker, taskInfo);
+            }
+            return;
+        }
+        // 2. If no idle worker, create a new one if we're not at the max limit
+        if (this.workers.length < this.config.maxWorkers) {
+            const taskInfo = this.taskQueue.shift();
+            if (taskInfo) {
+                logger.info(`Scaling up workers to ${this.workers.length + 1} to handle load...`);
+                const newWorker = this._createWorker();
+                this.workers.push(newWorker);
+                this._assignTask(newWorker, taskInfo);
+            }
+        }
+        // 3. If we are at the max limit and all workers are busy, the task remains queued.
+        // It will be picked up when a worker finishes its current task.
+    }
+    _createWorker() {
+        const worker = new Worker(this.workerPath, { workerData: this.workerData });
+        const trackedWorker = {
+            id: ++this.workerIdCounter,
+            worker,
+            currentTaskId: undefined,
+            lastUsed: Date.now(),
+        };
+        worker.on('message', (message) => {
+            this._handleTaskCompletion(trackedWorker, message);
+        });
+        worker.on('error', (err) => {
+            this._handleWorkerCrash(trackedWorker, err);
+        });
+        return trackedWorker;
+    }
+    _assignTask(worker, taskInfo) {
+        worker.currentTaskId = taskInfo.taskId;
+        worker.worker.postMessage({ task: taskInfo.task, taskId: taskInfo.taskId });
+    }
+    _handleTaskCompletion(worker, message) {
+        if (worker.currentTaskId === undefined)
+            return; // Should not happen
+        const taskCallbacks = this.activeTasks.get(worker.currentTaskId);
+        if (taskCallbacks) {
+            if (message.status === 'success') {
+                taskCallbacks.resolve(message.result);
+            }
+            else {
+                taskCallbacks.reject(new EmbeddingError(message.error.message, message.error));
+            }
+            this.activeTasks.delete(worker.currentTaskId);
+        }
+        worker.currentTaskId = undefined;
+        worker.lastUsed = Date.now();
+        // After finishing, check if there are more tasks to run.
+        this._dispatch();
+    }
+    _handleWorkerCrash(crashedWorker, err) {
+        logger.error(`Worker #${crashedWorker.id} crashed.`, err);
+        // Reject the task it was working on
+        if (crashedWorker.currentTaskId !== undefined) {
+            const taskCallbacks = this.activeTasks.get(crashedWorker.currentTaskId);
+            taskCallbacks?.reject(new EmbeddingError(`Worker #${crashedWorker.id} crashed while processing task.`, err));
+            this.activeTasks.delete(crashedWorker.currentTaskId);
+        }
+        // Remove the dead worker from the pool
+        const workerIndex = this.workers.findIndex((w) => w.id === crashedWorker.id);
+        if (workerIndex !== -1) {
+            this.workers.splice(workerIndex, 1);
+        }
+        // Immediately try to dispatch another task, which may create a new worker if needed
+        this._dispatch();
+    }
+    _reapIdleWorkers() {
+        const now = Date.now();
+        const idleWorkers = this.workers.filter((w) => w.currentTaskId === undefined);
+        for (const worker of idleWorkers) {
+            if (this.workers.length <= this.config.minWorkers) {
+                break; // Stop reaping if we're at the minimum
+            }
+            if (now - worker.lastUsed > this.config.idleTimeoutMs) {
+                logger.info(`Scaling down: Terminating idle worker #${worker.id}.`);
+                worker.worker.terminate();
+                const workerIndex = this.workers.findIndex((w) => w.id === worker.id);
+                if (workerIndex !== -1) {
+                    this.workers.splice(workerIndex, 1);
+                }
+            }
+        }
+    }
+    async destroy() {
+        if (this.idleCheckInterval) {
+            clearInterval(this.idleCheckInterval);
+        }
+        await Promise.all(this.workers.map((w) => w.worker.terminate()));
+        this.workers = [];
+    }
+}
+//# sourceMappingURL=pool.js.map

package/dist/local-rag/utils/pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pool.js","sourceRoot":"","sources":["../../../src/local-rag/utils/pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAE5C,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAkC7C,MAAM,OAAO,UAAU;IACJ,UAAU,CAAK;IACf,UAAU,CAAQ;IAClB,MAAM,CAAsB;IAErC,OAAO,GAAoB,EAAE,CAAA;IACpB,SAAS,GAAe,EAAE,CAAA;IACnC,aAAa,GAAG,CAAC,CAAA;IACjB,eAAe,GAAG,CAAC,CAAA;IACV,WAAW,GAAG,IAAI,GAAG,EAAgD,CAAA;IAC9E,iBAAiB,CAA4B;IAErD,YAAY,UAAe,EAAE,UAAkB,EAAE,SAAqB,EAAE;QACtE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAE5B,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC/D,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,SAAS;SACjD,CAAA;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,CAAA;QACpG,CAAC;IACH,CAAC;IAIM,OAAO,CAAC,IAAmB;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;YACjD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;YACtD,IAAI,CAAC,SAAS,EAAE,CAAA;QAClB,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAM;QACR,CAAC;QAED,8CAA8C;QAC9C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAA;QAC/E,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACvC,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAA;YAC7C,CAAC;YACD,OAAM;QACR,CAAC;QAED,uEAAuE;QACvE,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;YACvC,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,oBAAoB,CAAC,CAAA;gBACjF,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;gBACtC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC5B,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;YACvC,CAAC;QACH,CAAC;QAED,mFAAmF;QACnF,gEAAgE;IAClE,CAAC;IAEO,aAAa;QACnB,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QAC3E,MAAM,aAAa,GAAkB;YACnC,EAAE,EAAE,EAAE,IAAI,CAAC,eAAe;YAC1B,MAAM;YACN,aAAa,EAAE,SAAS;YACxB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAA;QAED,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAwB,EAAE,EAAE;YAChD,IAAI,CAAC,qBAAqB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QAEF,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,IAAI,CAAC,kBAAkB,CAAC,aAAa,EAAE,GAAG,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;QAEF,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,WAAW,CAAC,MAAqB,EAAE,QAAkB;QAC3D,MAAM,CAAC,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAA;QACtC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7E,CAAC;IAEO,qBAAqB,CAAC,MAAqB,EAAE,OAAwB;QAC3E,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS;YAAE,OAAM,CAAC,oBAAoB;QAEnE,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAChE,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACjC,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACvC,CAAC;iBAAM,CAAC;gBACN,aAAa,CAAC,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;YAChF,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/C,CAAC;QAED,MAAM,CAAC,aAAa,GAAG,SAAS,CAAA;QAChC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAE5B,yDAAyD;QACzD,IAAI,CAAC,SAAS,EAAE,CAAA;IAClB,CAAC;IAEO,kBAAkB,CAAC,aAA4B,EAAE,GAAU;QACjE,MAAM,CAAC,KAAK,CAAC,WAAW,aAAa,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,CAAA;QACzD,oCAAoC;QACpC,IAAI,aAAa,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;YACvE,aAAa,EAAE,MAAM,CAAC,IAAI,cAAc,CAAC,WAAW,aAAa,CAAC,EAAE,iCAAiC,EAAE,GAAG,CAAC,CAAC,CAAA;YAC5G,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;QACtD,CAAC;QAED,uCAAuC;QACvC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,aAAa,CAAC,EAAE,CAAC,CAAA;QAC5E,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QACrC,CAAC;QAED,oFAAoF;QACpF,IAAI,CAAC,SAAS,EAAE,CAAA;IAClB,CAAC;IAEO,gBAAgB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,CAAC,CAAA;QAE7E,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBAClD,MAAK,CAAC,uCAAuC;YAC/C,CAAC;YAED,IAAI,GAAG,GAAG,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,0CAA0C,MAAM,CAAC,EAAE,GAAG,CAAC,CAAA;gBACnE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAA;gBACzB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAA;gBACrE,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;oBACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,OAAO;QAClB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACvC,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QAChE,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;CACF"}

package/dist/local-rag/utils/typed-emitter.d.ts

@@ -0,0 +1,28 @@
+type Listener<T extends unknown[]> = (...args: T) => void;
+/**
+ * A generic, type-safe event emitter.
+ * Ensures that event names and their payload types are strictly checked at compile time.
+ * @template T - A map of event names to their listener argument types.
+ */
+export declare class TypedEventEmitter<T extends Record<string, unknown[]>> {
+    private readonly listeners;
+    /**
+     * Registers an event listener for a given event.
+     * @param event - The name of the event to listen for.
+     * @param listener - The callback function to execute when the event is emitted.
+     */
+    on<E extends keyof T>(event: E, listener: Listener<T[E]>): void;
+    /**
+     * Emits an event, calling all registered listeners with the provided arguments.
+     * @param event - The name of the event to emit.
+     * @param args - The arguments to pass to the event listeners.
+     */
+    emit<E extends keyof T>(event: E, ...args: T[E]): void;
+    /**
+     * Removes all listeners, or all listeners for a specific event.
+     * @param event - Optional. The name of the event to remove listeners for.
+     */
+    off<E extends keyof T>(event?: E): void;
+}
+export {};
+//# sourceMappingURL=typed-emitter.d.ts.map

package/dist/local-rag/utils/typed-emitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"typed-emitter.d.ts","sourceRoot":"","sources":["../../../src/local-rag/utils/typed-emitter.ts"],"names":[],"mappings":"AAAA,KAAK,QAAQ,CAAC,CAAC,SAAS,OAAO,EAAE,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,KAAK,IAAI,CAAA;AAEzD;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IAChE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6C;IAEvE;;;;OAIG;IACI,EAAE,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;IAMtE;;;;OAIG;IACI,IAAI,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;IAS7D;;;OAGG;IACI,GAAG,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,GAAG,IAAI;CAO/C"}

package/dist/local-rag/utils/typed-emitter.js

@@ -0,0 +1,44 @@
+/**
+ * A generic, type-safe event emitter.
+ * Ensures that event names and their payload types are strictly checked at compile time.
+ * @template T - A map of event names to their listener argument types.
+ */
+export class TypedEventEmitter {
+    listeners = new Map();
+    /**
+     * Registers an event listener for a given event.
+     * @param event - The name of the event to listen for.
+     * @param listener - The callback function to execute when the event is emitted.
+     */
+    on(event, listener) {
+        const existing = this.listeners.get(event) ?? [];
+        // @ts-expect-error Undefined subtype
+        this.listeners.set(event, [...existing, listener]);
+    }
+    /**
+     * Emits an event, calling all registered listeners with the provided arguments.
+     * @param event - The name of the event to emit.
+     * @param args - The arguments to pass to the event listeners.
+     */
+    emit(event, ...args) {
+        const eventListeners = this.listeners.get(event);
+        if (eventListeners) {
+            for (const listener of eventListeners) {
+                listener(...args);
+            }
+        }
+    }
+    /**
+     * Removes all listeners, or all listeners for a specific event.
+     * @param event - Optional. The name of the event to remove listeners for.
+     */
+    off(event) {
+        if (event) {
+            this.listeners.delete(event);
+        }
+        else {
+            this.listeners.clear();
+        }
+    }
+}
+//# sourceMappingURL=typed-emitter.js.map

package/dist/local-rag/utils/typed-emitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"typed-emitter.js","sourceRoot":"","sources":["../../../src/local-rag/utils/typed-emitter.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,OAAO,iBAAiB;IACX,SAAS,GAAG,IAAI,GAAG,EAAmC,CAAA;IAEvE;;;;OAIG;IACI,EAAE,CAAoB,KAAQ,EAAE,QAAwB;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAChD,qCAAqC;QACrC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;IACpD,CAAC;IAED;;;;OAIG;IACI,IAAI,CAAoB,KAAQ,EAAE,GAAG,IAAU;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAChD,IAAI,cAAc,EAAE,CAAC;YACnB,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACtC,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAA;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,GAAG,CAAoB,KAAS;QACrC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC9B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAA;QACxB,CAAC;IACH,CAAC;CACF"}

package/dist/local-rag/vectordb/index.d.ts

@@ -0,0 +1,91 @@
+import { type Table } from '@lancedb/lancedb';
+import { StoreManager } from './manager.js';
+import { Retriever } from './retriever.js';
+import type { DocumentMetadata, ListItem, ListOptions, QueryFilters, QueryResult, StatusReport, VectorChunk, WatchedPath } from '../types.js';
+export interface VectorStoreConfig {
+    dbPath: string;
+    hybridWeight?: number;
+    maxDistance?: number;
+    grouping?: 'similar' | 'related';
+}
+/**
+ * Manages all interactions with the LanceDB vector database.
+ * This class serves as a facade, coordinating migration, retrieval, and management components.
+ * An instance must be initialized via the `initialize()` method before use.
+ */
+export declare class VectorStore {
+    private readonly config;
+    private db;
+    table: Table;
+    retriever: Retriever;
+    manager: StoreManager;
+    private ftsEnabled;
+    private constructor();
+    static create(config: VectorStoreConfig): Promise<VectorStore>;
+    close(): void;
+    /**
+     * Connects to the database, ensures all necessary tables exist and are migrated,
+     * and initializes all components. This must be called before any other method.
+     */
+    private initialize;
+    /**
+     * Inserts an array of vector chunks into the database.
+     * @param chunks - An array of vector chunks to insert.
+     */
+    insertChunks(chunks: VectorChunk[]): Promise<void>;
+    /**
+     * Deletes all chunks associated with a given file path from the documents table.
+     * @param filePath - The absolute or synthetic path of the document to delete.
+     */
+    deleteChunks(filePath: string): Promise<void>;
+    /**
+     * Executes a search query against the vector store.
+     */
+    search(queryVector: number[], queryText: string, limit?: number, filters?: QueryFilters): Promise<QueryResult[]>;
+    /**
+     * Lists all items in the vector store, with optional filtering and pagination.
+     */
+    listFiles(options?: ListOptions): Promise<ListItem[]>;
+    /**
+     * Retrieves the current status of the RAG system.
+     */
+    getStatus(): Promise<StatusReport>;
+    /**
+     * Removes all expired text snippets from the database.
+     */
+    cleanupExpired(): Promise<number>;
+    /**
+     * Optimizes the database table, which compacts segments and rebuilds indexes.
+     * This can improve query performance after many writes.
+     */
+    optimize(): Promise<void>;
+    /**
+     * Retrieves the most recently updated metadata for a specific file path.
+     * This is a read-only operation and does not use the write queue.
+     * @param filePath - The full path of the document.
+     * @returns A promise that resolves to the document's metadata or undefined if not found.
+     */
+    getLatestMetadata(filePath: string): Promise<DocumentMetadata | undefined>;
+    /**
+     * Retrieves all chunks for a specific document path.
+     */
+    getChunksByPath(filePath: string): Promise<VectorChunk[]>;
+    /**
+     * Retrieves all persisted watched paths from the database.
+     * @returns A promise that resolves to an array of watched path configurations.
+     */
+    getWatchedPaths(): Promise<WatchedPath[]>;
+    /**
+     * Adds or updates a path to be watched in the persistence layer.
+     * This operation is idempotent.
+     */
+    addWatchedPath(path: string, type: 'file' | 'folder', recursive: boolean): Promise<void>;
+    /**
+     * Removes a path from the persistence layer.
+     */
+    removeWatchedPath(path: string): Promise<void>;
+    private _ensureWatchedPathsTable;
+    private _setupComponents;
+    private _ensureFtsIndex;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/local-rag/vectordb/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/local-rag/vectordb/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAA0B,KAAK,KAAK,EAAW,MAAM,kBAAkB,CAAA;AAI9E,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAG1C,OAAO,KAAK,EACV,gBAAgB,EAChB,QAAQ,EACR,WAAW,EACX,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,WAAW,EACZ,MAAM,aAAa,CAAA;AAMpB,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;CACjC;AAED;;;;GAIG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,EAAE,CAAY;IACf,KAAK,EAAG,KAAK,CAAA;IACb,SAAS,EAAG,SAAS,CAAA;IACrB,OAAO,EAAG,YAAY,CAAA;IAC7B,OAAO,CAAC,UAAU,CAAQ;IAE1B,OAAO;WAKM,MAAM,CAAC,MAAM,EAAE,iBAAiB;IAOtC,KAAK,IAAI,IAAI;IAKpB;;;OAGG;YACW,UAAU;IA2BxB;;;OAGG;IACU,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/D;;;OAGG;IACU,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS1D;;OAEG;IACU,MAAM,CACjB,WAAW,EAAE,MAAM,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,KAAK,SAAK,EACV,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,WAAW,EAAE,CAAC;IAOzB;;OAEG;IACU,SAAS,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAOlE;;OAEG;IACU,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;IA0B/C;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAY9C;;;OAGG;IACU,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAetC;;;;;OAKG;IACU,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAwBvF;;OAEG;IACU,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAOtE;;;OAGG;IACU,eAAe,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAWtD;;;OAGG;IACU,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAQrG;;OAEG;IACU,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAK7C,wBAAwB;IAYtC,OAAO,CAAC,gBAAgB;YASV,eAAe;CAqB9B"}

package/dist/local-rag/vectordb/index.js

@@ -0,0 +1,278 @@
+import { Index, connect } from '@lancedb/lancedb';
+import { logger } from '@mcp-monorepo/shared';
+import { DatabaseError } from '../errors.js';
+import { StoreManager } from './manager.js';
+import { SchemaMigrator } from './migration.js';
+import { Retriever } from './retriever.js';
+import { DataMapper, getDocumentsSchema } from './schema.js';
+// Table names are now internal constants, not part of the public config.
+const DOCUMENTS_TABLE_NAME = 'documents';
+const WATCHED_PATHS_TABLE_NAME = 'watched_paths';
+/**
+ * Manages all interactions with the LanceDB vector database.
+ * This class serves as a facade, coordinating migration, retrieval, and management components.
+ * An instance must be initialized via the `initialize()` method before use.
+ */
+export class VectorStore {
+    config;
+    db;
+    table;
+    retriever;
+    manager;
+    ftsEnabled = false;
+    constructor(config, db) {
+        this.config = config;
+        this.db = db;
+    }
+    static async create(config) {
+        const db = await connect(config.dbPath);
+        const vectorStore = new VectorStore(config, db);
+        await vectorStore.initialize();
+        return vectorStore;
+    }
+    close() {
+        logger.info('VectorStore: Closing database connection.');
+        this.db.close();
+    }
+    /**
+     * Connects to the database, ensures all necessary tables exist and are migrated,
+     * and initializes all components. This must be called before any other method.
+     */
+    async initialize() {
+        try {
+            const migrator = new SchemaMigrator(this.db, DOCUMENTS_TABLE_NAME);
+            let documentsTable = await migrator.run();
+            // If the table doesn't exist after migration (or at all), create it with the correct schema.
+            if (!documentsTable) {
+                logger.info(`VectorStore: Creating new table "${DOCUMENTS_TABLE_NAME}" with predefined schema.`);
+                // Create the table with an empty array and the explicit schema.
+                // This forces LanceDB to use our schema instead of inferring it.
+                documentsTable = await this.db.createTable(DOCUMENTS_TABLE_NAME, [], { schema: getDocumentsSchema() });
+            }
+            this._setupComponents(documentsTable);
+            await this._ensureFtsIndex();
+            await this._ensureWatchedPathsTable();
+            logger.info(`VectorStore initialized at: ${this.config.dbPath}`);
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('Migration resulted in an empty table')) {
+                logger.warn('A migration resulted in an empty table, which was recreated.');
+                return;
+            }
+            throw new DatabaseError('Failed to initialize VectorStore', error);
+        }
+    }
+    /**
+     * Inserts an array of vector chunks into the database.
+     * @param chunks - An array of vector chunks to insert.
+     */
+    async insertChunks(chunks) {
+        if (chunks.length === 0)
+            return;
+        try {
+            await this.table.add(chunks);
+        }
+        catch (error) {
+            throw new DatabaseError('Failed to insert chunks into the database.', error);
+        }
+    }
+    /**
+     * Deletes all chunks associated with a given file path from the documents table.
+     * @param filePath - The absolute or synthetic path of the document to delete.
+     */
+    async deleteChunks(filePath) {
+        try {
+            const escapedFilePath = filePath.replace(/'/g, "''");
+            await this.table.delete(`\`filePath\` = '${escapedFilePath}'`);
+        }
+        catch (error) {
+            throw new DatabaseError(`Failed to delete chunks for file: ${filePath}`, error);
+        }
+    }
+    /**
+     * Executes a search query against the vector store.
+     */
+    async search(queryVector, queryText, limit = 10, filters) {
+        if (!this.retriever) {
+            return []; // If no table/retriever, search returns no results.
+        }
+        return this.retriever.search(queryVector, queryText, limit, filters);
+    }
+    /**
+     * Lists all items in the vector store, with optional filtering and pagination.
+     */
+    async listFiles(options) {
+        if (!this.manager) {
+            return []; // If no table/manager, list returns no results.
+        }
+        return this.manager.listFiles(options);
+    }
+    /**
+     * Retrieves the current status of the RAG system.
+     */
+    async getStatus() {
+        if (!this.table) {
+            return {
+                documentCount: 0,
+                chunkCount: 0,
+                memoryUsageMb: process.memoryUsage().heapUsed / 1024 / 1024,
+                uptimeSeconds: process.uptime(),
+                ftsIndexEnabled: false,
+                searchMode: 'vector-only',
+            };
+        }
+        const chunkCount = await this.table.countRows();
+        const list = await this.listFiles({ limit: 1_000_000 }); // A pragmatic way to count unique files
+        const searchMode = this.ftsEnabled && (this.config.hybridWeight ?? 0.6) > 0 ? 'hybrid' : 'vector-only';
+        return {
+            documentCount: list.length,
+            chunkCount,
+            memoryUsageMb: process.memoryUsage().heapUsed / 1024 / 1024,
+            uptimeSeconds: process.uptime(),
+            ftsIndexEnabled: this.ftsEnabled,
+            searchMode,
+        };
+    }
+    /**
+     * Removes all expired text snippets from the database.
+     */
+    async cleanupExpired() {
+        if (!this.manager) {
+            return 0;
+        }
+        const deletedCount = await this.manager.cleanupExpired();
+        if (deletedCount > 0) {
+            logger.info(`VectorStore: Triggering optimization after deleting ${deletedCount} documents.`);
+            await this.optimize();
+        }
+        return deletedCount;
+    }
+    /**
+     * Optimizes the database table, which compacts segments and rebuilds indexes.
+     * This can improve query performance after many writes.
+     */
+    async optimize() {
+        if (!this.table) {
+            logger.info('VectorStore: Skipping optimization as table does not exist.');
+            return;
+        }
+        try {
+            logger.info('VectorStore: Starting database optimization...');
+            await this.table.optimize();
+            logger.info('VectorStore: Database optimization complete.');
+        }
+        catch (error) {
+            logger.error('VectorStore: Database optimization failed.', error);
+            throw new DatabaseError('Failed to optimize the database.', error);
+        }
+    }
+    /**
+     * Retrieves the most recently updated metadata for a specific file path.
+     * This is a read-only operation and does not use the write queue.
+     * @param filePath - The full path of the document.
+     * @returns A promise that resolves to the document's metadata or undefined if not found.
+     */
+    async getLatestMetadata(filePath) {
+        if (!this.table) {
+            return undefined;
+        }
+        const records = await this.table
+            .query()
+            .select(['metadata'])
+            .where(`\`filePath\` = '${filePath.replace(/'/g, "''")}'`)
+            .toArray();
+        if (records.length === 0) {
+            return undefined;
+        }
+        records.sort((a, b) => {
+            const dateA = a.metadata?.updatedAt ?? '';
+            const dateB = b.metadata?.updatedAt ?? '';
+            return dateB.localeCompare(dateA);
+        });
+        return DataMapper.toDocumentMetadata(records[0].metadata);
+    }
+    /**
+     * Retrieves all chunks for a specific document path.
+     */
+    async getChunksByPath(filePath) {
+        if (!this.manager) {
+            return [];
+        }
+        return this.manager.getChunksByPath(filePath);
+    }
+    /**
+     * Retrieves all persisted watched paths from the database.
+     * @returns A promise that resolves to an array of watched path configurations.
+     */
+    async getWatchedPaths() {
+        const table = await this.db.openTable(WATCHED_PATHS_TABLE_NAME);
+        const records = await table.query().toArray();
+        return records.map((r) => ({
+            path: r.path,
+            type: r.type,
+            recursive: r.recursive,
+            addedAt: r.addedAt,
+        }));
+    }
+    /**
+     * Adds or updates a path to be watched in the persistence layer.
+     * This operation is idempotent.
+     */
+    async addWatchedPath(path, type, recursive) {
+        const table = await this.db.openTable(WATCHED_PATHS_TABLE_NAME);
+        await table.delete(`path = '${path.replace(/'/g, "''")}'`).catch(() => {
+            /* ignore if it doesn't exist */
+        });
+        await table.add([{ path, type, recursive, addedAt: new Date().toISOString() }]);
+    }
+    /**
+     * Removes a path from the persistence layer.
+     */
+    async removeWatchedPath(path) {
+        const table = await this.db.openTable(WATCHED_PATHS_TABLE_NAME);
+        await table.delete(`path = '${path.replace(/'/g, "''")}'`);
+    }
+    async _ensureWatchedPathsTable() {
+        const tableNames = await this.db.tableNames();
+        if (!tableNames.includes(WATCHED_PATHS_TABLE_NAME)) {
+            logger.info(`Creating new '${WATCHED_PATHS_TABLE_NAME}' table for persistence.`);
+            // Create table with a dummy record to define the schema, then delete it.
+            const table = await this.db.createTable(WATCHED_PATHS_TABLE_NAME, [
+                { path: '__dummy__', type: 'file', recursive: false, addedAt: '' },
+            ]);
+            await table.delete("path = '__dummy__'");
+        }
+    }
+    _setupComponents(table) {
+        this.table = table;
+        this.retriever = new Retriever(this.table, {
+            ...this.config,
+            ftsEnabled: this.ftsEnabled,
+        });
+        this.manager = new StoreManager(this.table);
+    }
+    async _ensureFtsIndex() {
+        if (!this.table)
+            return;
+        try {
+            await this.table.createIndex('text', { config: Index.fts() });
+            this.ftsEnabled = true;
+            logger.info('VectorStore: FTS index is active.');
+        }
+        catch (error) {
+            const msg = error.message;
+            if (msg.includes('already exists')) {
+                this.ftsEnabled = true;
+                logger.info('VectorStore: FTS index already exists and is active.');
+            }
+            else {
+                logger.error('VectorStore: FTS index creation failed. Hybrid search will be disabled.', error);
+                this.ftsEnabled = false;
+            }
+        }
+        // Update the retriever with the correct FTS status
+        if (this.retriever)
+            this.retriever.setFtsEnabled(this.ftsEnabled);
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/local-rag/vectordb/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/local-rag/vectordb/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,EAAc,OAAO,EAAE,MAAM,kBAAkB,CAAA;AAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAa5D,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,WAAW,CAAA;AACxC,MAAM,wBAAwB,GAAG,eAAe,CAAA;AAShD;;;;GAIG;AACH,MAAM,OAAO,WAAW;IACL,MAAM,CAAmB;IAClC,EAAE,CAAY;IACf,KAAK,CAAQ;IACb,SAAS,CAAY;IACrB,OAAO,CAAe;IACrB,UAAU,GAAG,KAAK,CAAA;IAE1B,YAAoB,MAAyB,EAAE,EAAc;QAC3D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAA;IACd,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAyB;QAC3C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACvC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QAC/C,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;QAC9B,OAAO,WAAW,CAAA;IACpB,CAAC;IAEM,KAAK;QACV,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;QACxD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAA;YAElE,IAAI,cAAc,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,CAAA;YACzC,6FAA6F;YAC7F,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,oCAAoC,oBAAoB,2BAA2B,CAAC,CAAA;gBAChG,gEAAgE;gBAChE,iEAAiE;gBACjE,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,oBAAoB,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAA;YACxG,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAA;YACrC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;YAC5B,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAA;YAErC,MAAM,CAAC,IAAI,CAAC,+BAA+B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;QAClE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,sCAAsC,CAAC,EAAE,CAAC;gBAC7F,MAAM,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAA;gBAC3E,OAAM;YACR,CAAC;YACD,MAAM,IAAI,aAAa,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAA;QACpE,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,YAAY,CAAC,MAAqB;QAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE/B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAA8C,CAAC,CAAA;QACtE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,IAAI,aAAa,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAA;QAC9E,CAAC;IACH,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,YAAY,CAAC,QAAgB;QACxC,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YACpD,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,eAAe,GAAG,CAAC,CAAA;QAChE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,IAAI,aAAa,CAAC,qCAAqC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QACjF,CAAC;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,MAAM,CACjB,WAAqB,EACrB,SAAiB,EACjB,KAAK,GAAG,EAAE,EACV,OAAsB;QAEtB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO,EAAE,CAAA,CAAC,oDAAoD;QAChE,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IACtE,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,SAAS,CAAC,OAAqB;QAC1C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,EAAE,CAAA,CAAC,gDAAgD;QAC5D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IACxC,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,SAAS;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;gBACL,aAAa,EAAE,CAAC;gBAChB,UAAU,EAAE,CAAC;gBACb,aAAa,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI;gBAC3D,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE;gBAC/B,eAAe,EAAE,KAAK;gBACtB,UAAU,EAAE,aAAa;aAC1B,CAAA;QACH,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAA;QAC/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA,CAAC,wCAAwC;QAChG,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAA;QAEtG,OAAO;YACL,aAAa,EAAE,IAAI,CAAC,MAAM;YAC1B,UAAU;YACV,aAAa,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI;YAC3D,aAAa,EAAE,OAAO,CAAC,MAAM,EAAE;YAC/B,eAAe,EAAE,IAAI,CAAC,UAAU;YAChC,UAAU;SACX,CAAA;IACH,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,cAAc;QACzB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,CAAC,CAAA;QACV,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAA;QACxD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,uDAAuD,YAAY,aAAa,CAAC,CAAA;YAC7F,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACvB,CAAC;QACD,OAAO,YAAY,CAAA;IACrB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,QAAQ;QACnB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAA;YAC1E,OAAM;QACR,CAAC;QACD,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAA;YAC7D,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAA;YAC3B,MAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAA;QAC7D,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAA;YACjE,MAAM,IAAI,aAAa,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAA;QACpE,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,iBAAiB,CAAC,QAAgB;QAC7C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK;aAC7B,KAAK,EAAE;aACP,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;aACpB,KAAK,CAAC,mBAAmB,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;aACzD,OAAO,EAAE,CAAA;QAEZ,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACpB,MAAM,KAAK,GAAI,CAAC,CAAC,QAA6B,EAAE,SAAS,IAAI,EAAE,CAAA;YAC/D,MAAM,KAAK,GAAI,CAAC,CAAC,QAA6B,EAAE,SAAS,IAAI,EAAE,CAAA;YAC/D,OAAO,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,OAAO,UAAU,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;IAC3D,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,eAAe,CAAC,QAAgB;QAC3C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO,EAAE,CAAA;QACX,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,eAAe;QAC1B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;QAC/D,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAA;QAC7C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzB,IAAI,EAAE,CAAC,CAAC,IAAc;YACtB,IAAI,EAAE,CAAC,CAAC,IAAyB;YACjC,SAAS,EAAE,CAAC,CAAC,SAAoB;YACjC,OAAO,EAAE,CAAC,CAAC,OAAiB;SAC7B,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,cAAc,CAAC,IAAY,EAAE,IAAuB,EAAE,SAAkB;QACnF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;QAC/D,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACpE,gCAAgC;QAClC,CAAC,CAAC,CAAA;QACF,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAA;IACjF,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,iBAAiB,CAAC,IAAY;QACzC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAA;QAC/D,MAAM,KAAK,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;IAC5D,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAA;QAC7C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,iBAAiB,wBAAwB,0BAA0B,CAAC,CAAA;YAChF,yEAAyE;YACzE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,wBAAwB,EAAE;gBAChE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE;aACnE,CAAC,CAAA;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAA;QAC1C,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,KAAY;QACnC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,SAAS,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE;YACzC,GAAG,IAAI,CAAC,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC7C,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAM;QAEvB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;YAC7D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;YACtB,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAA;QAClD,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,GAAG,GAAI,KAAe,CAAC,OAAO,CAAA;YACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;gBACtB,MAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAA;YACrE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,yEAAyE,EAAE,KAAK,CAAC,CAAA;gBAC9F,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;YACzB,CAAC;QACH,CAAC;QAED,mDAAmD;QACnD,IAAI,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACnE,CAAC;CACF"}

package/dist/local-rag/vectordb/manager.d.ts

@@ -0,0 +1,28 @@
+import type { ListItem, ListOptions, VectorChunk } from '../types.js';
+import type { Table } from '@lancedb/lancedb';
+/**
+ * Handles administrative and management tasks for the vector store, like listing and cleaning documents.
+ */
+export declare class StoreManager {
+    private readonly table;
+    constructor(table: Table);
+    /**
+     * Lists all unique documents in the store, with optional filtering and pagination.
+     * This method first filters in the DB, then aggregates unique files in memory.
+     * @param options - Optional filters and pagination settings.
+     * @returns A promise that resolves to an array of list items.
+     */
+    listFiles(options?: ListOptions): Promise<ListItem[]>;
+    /**
+     * Removes all documents (and their chunks) that have expired.
+     * @returns A promise that resolves to the number of documents deleted.
+     */
+    cleanupExpired(): Promise<number>;
+    /**
+     * Retrieves all chunks associated with a specific file path.
+     * @param filePath - The full path of the document (e.g., '/path/to/file.pdf' or 'memory://label').
+     * @returns A promise that resolves to an array of vector chunks.
+     */
+    getChunksByPath(filePath: string): Promise<VectorChunk[]>;
+}
+//# sourceMappingURL=manager.d.ts.map

package/dist/local-rag/vectordb/manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manager.d.ts","sourceRoot":"","sources":["../../../src/local-rag/vectordb/manager.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACrE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AAE7C;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAO;gBAEjB,KAAK,EAAE,KAAK;IAIxB;;;;;OAKG;IACU,SAAS,CAAC,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAkDtE;;;OAGG;IACU,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAyB9C;;;;OAIG;IACU,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAOvE"}