@mcp-monorepo/notion-query 1.1.0 → 1.3.0

package/dist/local-rag/html-parser.d.ts

@@ -0,0 +1,2 @@
+export declare function parseHtmlContent(url: string, htmlContent: string): Promise<string>;
+//# sourceMappingURL=html-parser.d.ts.map

package/dist/local-rag/html-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-parser.d.ts","sourceRoot":"","sources":["../../src/local-rag/html-parser.ts"],"names":[],"mappings":"AAIA,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBxF"}

package/dist/local-rag/html-parser.js

@@ -0,0 +1,32 @@
+import { logger } from '@mcp-monorepo/shared';
+import { Readability, isProbablyReaderable } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+export async function parseHtmlContent(url, htmlContent) {
+    const doc = new JSDOM(htmlContent, { url });
+    // First, check if the page is likely to be readable
+    if (!isProbablyReaderable(doc.window.document)) {
+        logger.warn(`Page at ${url} is not considered readerable. Falling back to basic parsing.`);
+        // Fallback to the naive implementation if not readerable
+        return defaultParseHtmlContent(htmlContent);
+    }
+    const reader = new Readability(doc.window.document);
+    const article = reader.parse();
+    const textContent = article?.textContent?.trim();
+    if (textContent) {
+        // Successfully parsed, return the clean text content
+        // For Markdown, you would convert article.content here with another library
+        return textContent;
+    }
+    else {
+        // Readability failed to parse an article, use the fallback
+        logger.warn(`Readability.js could not parse an article from ${url}. Falling back to basic parsing.`);
+        return defaultParseHtmlContent(htmlContent);
+    }
+}
+function defaultParseHtmlContent(htmlContent) {
+    return htmlContent
+        .replace(/<[^>]*>/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+//# sourceMappingURL=html-parser.js.map

package/dist/local-rag/html-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-parser.js","sourceRoot":"","sources":["../../src/local-rag/html-parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AACxE,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAA;AAE7B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,WAAmB;IACrE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IAE3C,oDAAoD;IACpD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC,WAAW,GAAG,+DAA+D,CAAC,CAAA;QAC1F,yDAAyD;QACzD,OAAO,uBAAuB,CAAC,WAAW,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAA;IAE9B,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;IAChD,IAAI,WAAW,EAAE,CAAC;QAChB,qDAAqD;QACrD,4EAA4E;QAC5E,OAAO,WAAW,CAAA;IACpB,CAAC;SAAM,CAAC;QACN,2DAA2D;QAC3D,MAAM,CAAC,IAAI,CAAC,kDAAkD,GAAG,kCAAkC,CAAC,CAAA;QACpG,OAAO,uBAAuB,CAAC,WAAW,CAAC,CAAA;IAC7C,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,WAAmB;IAClD,OAAO,WAAW;SACf,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAA;AACX,CAAC"}

package/dist/local-rag/index.d.ts

@@ -0,0 +1,67 @@
+import type { IngestFileInput, IngestFolderInput, IngestTextInupt, IngestUrlInput, ListItem, ListOptions, LocalRAGConfig, QueryInput, QueryResult, StatusReport, UpdateMemoryInput, WatchOptions } from './types.js';
+/**
+ * A powerful, private, and configurable local RAG library.
+ */
+export declare class LocalRAG {
+    private readonly config;
+    private readonly parser;
+    private readonly chunker;
+    private readonly embedder;
+    private readonly vectorStore;
+    private readonly watcher;
+    private readonly writeExecutor;
+    private cleanupIntervalId?;
+    private optimizeIntervalId?;
+    private constructor();
+    static create(config: LocalRAGConfig): Promise<LocalRAG>;
+    /**
+     * Initializes the database connection and starts background jobs. Must be called before any other methods.
+     */
+    private initialize;
+    /**
+     * Gracefully shuts down all background processes like the watcher, embedder pool, and periodic jobs.
+     */
+    shutdown(): Promise<void>;
+    ingestFile(input: IngestFileInput): Promise<void>;
+    ingestFolder(input: IngestFolderInput): Promise<void>;
+    /**
+     * Ingests a raw text snippet into the vector store.
+     */
+    ingestText(input: IngestTextInupt): Promise<void>;
+    /**
+     * Ingests content from a URL.
+     */
+    ingestUrl(input: IngestUrlInput): Promise<void>;
+    /**
+     * Searches the vector store with a natural language query.
+     */
+    query(input: QueryInput): Promise<QueryResult[]>;
+    /**
+     * Updates an existing text snippet (memory).
+     */
+    updateMemory(input: UpdateMemoryInput): Promise<void>;
+    /**
+     * Deletes a file or snippet from the vector store.
+     */
+    delete(path: string): Promise<void>;
+    /**
+     * Lists all items in the vector store, with optional filtering and pagination.
+     */
+    list(options?: ListOptions): Promise<ListItem[]>;
+    /**
+     * Retrieves the current status of the RAG system.
+     */
+    getStatus(): Promise<StatusReport>;
+    /**
+     * Manually triggers a cleanup of all expired text snippets from the database.
+     */
+    cleanupExpired(): Promise<number>;
+    watch(path: string, options?: WatchOptions): Promise<void>;
+    unwatch(path: string): Promise<void>;
+    private startPeriodicJobs;
+    private findSupportedFiles;
+    private setupWatcherListeners;
+    private calculateExpiresAt;
+    private _createVectorChunks;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/local-rag/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/local-rag/index.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAEV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,QAAQ,EACR,WAAW,EACX,cAAc,EACd,UAAU,EACV,WAAW,EACX,YAAY,EACZ,iBAAiB,EAEjB,YAAY,EACb,MAAM,YAAY,CAAA;AAEnB;;GAEG;AACH,qBAAa,QAAQ;IACnB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAmB;IAEjD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,kBAAkB,CAAC,CAAgB;IAE3C,OAAO;WAqBM,MAAM,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC;IAgC9D;;OAEG;YACW,UAAU;IAWxB;;OAEG;IACU,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAW/B,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAgD3C,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlE;;OAEG;IACI,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BxD;;OAEG;IACI,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BtD;;OAEG;IACU,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAM7D;;OAEG;IACI,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqC5D;;OAEG;IACI,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO1C;;OAEG;IACU,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAI7D;;OAEG;IACU,SAAS,IAAI,OAAO,CAAC,YAAY,CAAC;IAI/C;;OAEG;IACI,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAIjC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAe9D,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa3C,OAAO,CAAC,iBAAiB;YA4BX,kBAAkB;IAgBhC,OAAO,CAAC,qBAAqB;IAqB7B,OAAO,CAAC,kBAAkB;YAeZ,mBAAmB;CA4BlC"}

package/dist/local-rag/index.js

@@ -0,0 +1,410 @@
+// noinspection JSUnusedGlobalSymbols
+import { randomUUID } from 'node:crypto';
+import { readdir, stat } from 'node:fs/promises';
+import { basename, extname, join, resolve } from 'node:path';
+import { ThrottledExecutor, logger } from '@mcp-monorepo/shared';
+import { DocumentChunker } from './chunker.js';
+import { Embedder } from './embedder.js';
+import { ValidationError } from './errors.js';
+import { parseHtmlContent } from './html-parser.js';
+import { DocumentParser } from './parser.js';
+import { VectorStore } from './vectordb/index.js';
+import { DirectoryWatcher } from './watcher.js';
+/**
+ * A powerful, private, and configurable local RAG library.
+ */
+export class LocalRAG {
+    config;
+    parser;
+    chunker;
+    embedder;
+    vectorStore;
+    watcher;
+    writeExecutor;
+    cleanupIntervalId;
+    optimizeIntervalId;
+    constructor(config, embedder, vectorStore) {
+        this.config = config;
+        this.writeExecutor = new ThrottledExecutor(0);
+        // Initialize components
+        this.parser = new DocumentParser({
+            baseDir: this.config.baseDir,
+            maxFileSize: this.config.maxFileSize,
+        });
+        this.chunker = new DocumentChunker({
+            chunkSize: this.config.chunkSize,
+            chunkOverlap: this.config.chunkOverlap,
+        });
+        this.embedder = embedder;
+        this.vectorStore = vectorStore;
+        // Initialize the watcher
+        this.watcher = new DirectoryWatcher(this.config.debounceMs, this.parser.getSupportedExtensions());
+        this.setupWatcherListeners();
+    }
+    static async create(config) {
+        const baseDir = resolve(config.baseDir ?? process.cwd());
+        const requiredConfig = {
+            baseDir,
+            dbPath: config.dbPath,
+            cacheDir: config.cacheDir ?? resolve(baseDir, '.cache', 'models'),
+            modelName: config.modelName ?? 'Xenova/all-MiniLM-L6-v2',
+            maxFileSize: config.maxFileSize ?? 100 * 1024 * 1024, // 100MB
+            chunkSize: config.chunkSize ?? 512,
+            chunkOverlap: config.chunkOverlap ?? 100,
+            debounceMs: config.debounceMs ?? 5000,
+            cleanupIntervalMs: config.cleanupIntervalMs ?? 0, // Default to disabled
+            optimizeIntervalMs: config.optimizeIntervalMs ?? 0, // Default to disabled
+            retrievalOptions: config.retrievalOptions ?? {},
+            poolConfig: config.poolConfig ?? {},
+        };
+        const embedder = await Embedder.create({
+            modelPath: requiredConfig.modelName,
+            batchSize: 32,
+            cacheDir: requiredConfig.cacheDir,
+            poolConfig: requiredConfig.poolConfig,
+        });
+        const vectorStore = await VectorStore.create({
+            dbPath: requiredConfig.dbPath,
+            ...requiredConfig.retrievalOptions,
+        });
+        const rag = new LocalRAG(requiredConfig, embedder, vectorStore);
+        await rag.initialize();
+        return rag;
+    }
+    /**
+     * Initializes the database connection and starts background jobs. Must be called before any other methods.
+     */
+    async initialize() {
+        // Hydrate the watcher with persisted paths
+        const watchedPaths = await this.vectorStore.getWatchedPaths();
+        for (const item of watchedPaths) {
+            this.watcher.watch(item.path);
+        }
+        logger.info(`Watcher initialized and now monitoring ${watchedPaths.length} persisted paths.`);
+        this.startPeriodicJobs();
+    }
+    /**
+     * Gracefully shuts down all background processes like the watcher, embedder pool, and periodic jobs.
+     */
+    async shutdown() {
+        if (this.cleanupIntervalId)
+            clearInterval(this.cleanupIntervalId);
+        if (this.optimizeIntervalId)
+            clearInterval(this.optimizeIntervalId);
+        await this.watcher.close();
+        await this.embedder.destroy();
+        await this.writeExecutor.onIdle();
+        this.vectorStore.close();
+        logger.info('LocalRAG shutdown complete.');
+    }
+    ingestFile(input) {
+        return this.writeExecutor.execute(async () => {
+            // Callers are responsible for handling errors, including those from unsupported file types.
+            // The watcher listener has special handling to ignore unsupported files.
+            const { filePath, tags, project, watch } = input;
+            const absolutePath = resolve(filePath);
+            const stats = await this.parser.validateAndGetFileStats(absolutePath);
+            const currentMtime = stats.fileModifiedAt;
+            const existingMetadata = await this.vectorStore.getLatestMetadata(absolutePath);
+            if (existingMetadata?.fileModifiedAt === currentMtime) {
+                logger.info(`Skipping ingestion for unchanged file: ${absolutePath}`);
+            }
+            else {
+                logger.info(`Ingesting changed or new file: ${absolutePath}`);
+                const { text, language, fileSize, metadata: fileMetadata } = await this.parser.parseFile(absolutePath);
+                const metadata = {
+                    ...fileMetadata,
+                    fileName: basename(absolutePath),
+                    fileSize,
+                    fileType: extname(absolutePath).slice(1),
+                    language,
+                    tags: tags ?? [],
+                    project,
+                    memoryType: 'file',
+                };
+                const vectorChunks = await this._createVectorChunks(text, metadata, absolutePath);
+                if (vectorChunks.length === 0) {
+                    logger.warn(`Skipping ingestion for ${absolutePath} as it produced no valid chunks (file might be empty or too short).`);
+                    await this.vectorStore.deleteChunks(absolutePath);
+                    return;
+                }
+                await this.vectorStore.deleteChunks(absolutePath);
+                await this.vectorStore.insertChunks(vectorChunks);
+            }
+            if (watch && !this.watcher.isWatching(absolutePath)) {
+                await this.watch(absolutePath);
+            }
+        });
+    }
+    async ingestFolder(input) {
+        const { folderPath, watch, recursive = false, project, tags } = input;
+        const absolutePath = resolve(folderPath);
+        if (watch) {
+            // If watching, we just attach the watcher. Chokidar with `ignoreInitial: false`
+            // will emit 'add' events for all existing files, which our listeners will handle.
+            logger.info(`Attaching watcher to ${absolutePath}. It will process existing and new files (recursive: ${recursive}).`);
+            await this.watch(absolutePath, { recursive });
+        }
+        else {
+            // If not watching, perform a one-time manual scan and ingestion.
+            logger.info(`Performing one-time ingestion for ${absolutePath} (recursive: ${recursive}).`);
+            const filesToIndex = await this.findSupportedFiles(absolutePath, recursive);
+            logger.info(`Found ${filesToIndex.length} supported files to ingest.`);
+            await Promise.all(filesToIndex.map((filePath) => this.ingestFile({ filePath, project, tags, watch: false })));
+        }
+    }
+    /**
+     * Ingests a raw text snippet into the vector store.
+     */
+    ingestText(input) {
+        return this.writeExecutor.execute(async () => {
+            const { text, label, language, tags, project, ttl } = input;
+            if (!label.match(/^[\w.-]+$/)) {
+                throw new ValidationError('Label must contain only alphanumeric characters, hyphens, underscores, and dots.');
+            }
+            const syntheticPath = `memory://${label}`;
+            const metadata = {
+                fileName: label,
+                fileSize: text.length,
+                fileType: 'text-snippet',
+                language,
+                tags: tags ?? [],
+                project,
+                memoryType: 'text',
+                expiresAt: ttl ? this.calculateExpiresAt(ttl) : undefined,
+            };
+            const vectorChunks = await this._createVectorChunks(text, metadata, syntheticPath);
+            await this.vectorStore.deleteChunks(syntheticPath);
+            await this.vectorStore.insertChunks(vectorChunks);
+        });
+    }
+    /**
+     * Ingests content from a URL.
+     */
+    ingestUrl(input) {
+        return this.writeExecutor.execute(async () => {
+            const { url, tags, project, ttl } = input;
+            const label = basename(new URL(url).pathname) || `web-${Date.now()}`;
+            const response = await fetch(url);
+            if (!response.ok)
+                throw new Error(`Failed to fetch URL: ${response.statusText}`);
+            const htmlContent = await response.text();
+            const text = await parseHtmlContent(url, htmlContent);
+            const syntheticPath = `url://${label}`;
+            const metadata = {
+                fileName: label,
+                fileSize: text.length,
+                fileType: 'web-page',
+                tags: tags ?? [],
+                project,
+                memoryType: 'url',
+                expiresAt: ttl ? this.calculateExpiresAt(ttl) : undefined,
+                sourceUrl: url,
+            };
+            const vectorChunks = await this._createVectorChunks(text, metadata, syntheticPath);
+            await this.vectorStore.deleteChunks(syntheticPath);
+            await this.vectorStore.insertChunks(vectorChunks);
+        });
+    }
+    /**
+     * Searches the vector store with a natural language query.
+     */
+    async query(input) {
+        const { query, limit, filters } = input;
+        const queryVector = await this.embedder.embed(query);
+        return this.vectorStore.search(queryVector, query, limit, filters);
+    }
+    /**
+     * Updates an existing text snippet (memory).
+     */
+    updateMemory(input) {
+        return this.writeExecutor.execute(async () => {
+            const { label, mode = 'replace', text: newText, tags, addTags, removeTags } = input;
+            const syntheticPath = `memory://${label}`;
+            const existingChunks = await this.vectorStore.getChunksByPath(syntheticPath);
+            if (existingChunks.length === 0)
+                throw new Error(`Memory with label "${label}" not found.`);
+            const oldMetadata = existingChunks[0].metadata;
+            let fullText = existingChunks
+                .sort((a, b) => a.chunkIndex - b.chunkIndex)
+                .map((c) => c.text)
+                .join('');
+            if (newText) {
+                if (mode === 'append')
+                    fullText += newText;
+                else if (mode === 'prepend')
+                    fullText = newText + fullText;
+                else
+                    fullText = newText;
+            }
+            let newTags = oldMetadata?.tags ?? [];
+            if (tags)
+                newTags = tags; // Replace
+            if (addTags)
+                newTags = [...new Set([...newTags, ...addTags])];
+            if (removeTags)
+                newTags = newTags.filter((t) => !removeTags.includes(t));
+            const updatedMetadata = {
+                ...oldMetadata,
+                tags: newTags,
+                fileSize: fullText.length,
+            };
+            const vectorChunks = await this._createVectorChunks(fullText, updatedMetadata, syntheticPath);
+            await this.vectorStore.deleteChunks(syntheticPath);
+            await this.vectorStore.insertChunks(vectorChunks);
+        });
+    }
+    /**
+     * Deletes a file or snippet from the vector store.
+     */
+    delete(path) {
+        return this.writeExecutor.execute(async () => {
+            // Path can be an absolute file path or a synthetic path like `memory://label`
+            await this.vectorStore.deleteChunks(path);
+        });
+    }
+    /**
+     * Lists all items in the vector store, with optional filtering and pagination.
+     */
+    async list(options) {
+        return this.vectorStore.listFiles(options);
+    }
+    /**
+     * Retrieves the current status of the RAG system.
+     */
+    async getStatus() {
+        return this.vectorStore.getStatus();
+    }
+    /**
+     * Manually triggers a cleanup of all expired text snippets from the database.
+     */
+    cleanupExpired() {
+        return this.writeExecutor.execute(() => this.vectorStore.cleanupExpired());
+    }
+    watch(path, options = {}) {
+        return this.writeExecutor.execute(async () => {
+            const absolutePath = resolve(path);
+            if (this.watcher.isWatching(absolutePath))
+                return;
+            const stats = await stat(absolutePath);
+            const type = stats.isDirectory() ? 'folder' : 'file';
+            const recursive = type === 'folder' ? (options.recursive ?? false) : false;
+            await this.vectorStore.addWatchedPath(absolutePath, type, recursive);
+            this.watcher.watch(absolutePath);
+            logger.info(`Now watching path: ${absolutePath}`);
+        });
+    }
+    unwatch(path) {
+        return this.writeExecutor.execute(async () => {
+            const absolutePath = resolve(path);
+            await this.vectorStore.removeWatchedPath(absolutePath);
+            this.watcher.unwatch(absolutePath);
+            logger.info(`Stopped watching path: ${absolutePath}`);
+        });
+    }
+    // ============================================
+    // Private Helper Methods
+    // ============================================
+    startPeriodicJobs() {
+        if (this.config.cleanupIntervalMs && this.config.cleanupIntervalMs > 0) {
+            logger.info(`Starting periodic cleanup job every ${this.config.cleanupIntervalMs}ms.`);
+            this.cleanupIntervalId = setInterval(async () => {
+                try {
+                    logger.info('Running periodic cleanup job...');
+                    const deletedCount = await this.cleanupExpired();
+                    if (deletedCount > 0) {
+                        logger.info(`Periodic cleanup job finished. Removed ${deletedCount} expired items.`);
+                    }
+                }
+                catch (error) {
+                    logger.error('Periodic cleanup job failed.', error);
+                }
+            }, this.config.cleanupIntervalMs);
+        }
+        if (this.config.optimizeIntervalMs && this.config.optimizeIntervalMs > 0) {
+            logger.info(`Starting periodic DB optimization every ${this.config.optimizeIntervalMs}ms.`);
+            this.optimizeIntervalId = setInterval(async () => {
+                try {
+                    await this.vectorStore.optimize();
+                }
+                catch (error) {
+                    logger.error('Periodic database optimization failed.', error);
+                }
+            }, this.config.optimizeIntervalMs);
+        }
+    }
+    async findSupportedFiles(folderPath, recursive) {
+        const supportedExtensions = new Set(this.parser.getSupportedExtensions());
+        const files = [];
+        const entries = await readdir(folderPath, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = join(folderPath, entry.name);
+            if (entry.isDirectory() && recursive) {
+                files.push(...(await this.findSupportedFiles(fullPath, true)));
+            }
+            else if (entry.isFile() && supportedExtensions.has(extname(entry.name).toLowerCase())) {
+                files.push(fullPath);
+            }
+        }
+        return files;
+    }
+    setupWatcherListeners() {
+        const handleIngest = async (filePath) => {
+            try {
+                await this.ingestFile({ filePath });
+            }
+            catch (error) {
+                if (error instanceof ValidationError && error.message.includes('Unsupported file format')) {
+                    // This is expected when the watcher picks up an unsupported file. Ignore it.
+                    logger.debug(`Watcher ignored unsupported file: ${filePath}`);
+                }
+                else {
+                    // Log any other unexpected errors during ingestion.
+                    logger.error(`Watcher failed to process file ${filePath}.`, error);
+                }
+            }
+        };
+        this.watcher.on('file-added', handleIngest);
+        this.watcher.on('file-changed', handleIngest);
+        this.watcher.on('file-deleted', (filePath) => this.delete(filePath).catch((e) => logger.error(e, `Watcher failed to delete file: ${filePath}`)));
+    }
+    calculateExpiresAt(ttl) {
+        const match = ttl.match(/^(\d+)([dhy])$/);
+        if (!match)
+            throw new ValidationError(`Invalid TTL format: ${ttl}. Use '1d', '7h', '1y' etc.`);
+        const value = parseInt(match[1], 10);
+        const unit = match[2];
+        const date = new Date();
+        if (unit === 'd')
+            date.setDate(date.getDate() + value);
+        else if (unit === 'h')
+            date.setHours(date.getHours() + value);
+        else if (unit === 'y')
+            date.setFullYear(date.getFullYear() + value);
+        return date.toISOString();
+    }
+    async _createVectorChunks(text, baseMetadata, path) {
+        const chunks = await this.chunker.chunkText(text);
+        if (chunks.length === 0)
+            return [];
+        const embeddings = await this.embedder.embedBatch(chunks.map((c) => c.text));
+        const now = new Date().toISOString();
+        return chunks.map((chunk, i) => ({
+            id: randomUUID(),
+            filePath: path,
+            chunkIndex: chunk.index,
+            text: chunk.text,
+            vector: embeddings[i],
+            metadata: {
+                fileName: '', // Default values, will be overwritten
+                fileSize: 0,
+                fileType: '',
+                createdAt: now,
+                updatedAt: now,
+                ...baseMetadata,
+            },
+            timestamp: now,
+        }));
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/local-rag/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/local-rag/index.ts"],"names":[],"mappings":"AAAA,qCAAqC;AAErC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAE5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAEhE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAmB/C;;GAEG;AACH,MAAM,OAAO,QAAQ;IACF,MAAM,CAA0B;IAChC,MAAM,CAAgB;IACtB,OAAO,CAAiB;IACxB,QAAQ,CAAU;IAClB,WAAW,CAAa;IACxB,OAAO,CAAkB;IACzB,aAAa,CAAmB;IAEzC,iBAAiB,CAAiB;IAClC,kBAAkB,CAAiB;IAE3C,YAAoB,MAAgC,EAAE,QAAkB,EAAE,WAAwB;QAChG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,aAAa,GAAG,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAA;QAE7C,wBAAwB;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,cAAc,CAAC;YAC/B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,eAAe,CAAC;YACjC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;SACvC,CAAC,CAAA;QACF,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;QACxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAE9B,yBAAyB;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC,CAAA;QACjG,IAAI,CAAC,qBAAqB,EAAE,CAAA;IAC9B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAsB;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAA;QACxD,MAAM,cAAc,GAA6B;YAC/C,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC;YACjE,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,yBAAyB;YACxD,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ;YAC9D,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG;YAClC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;YACxC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;YACrC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,CAAC,EAAE,sBAAsB;YACxE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC,EAAE,sBAAsB;YAC1E,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;YAC/C,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE;SACpC,CAAA;QAED,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACrC,SAAS,EAAE,cAAc,CAAC,SAAS;YACnC,SAAS,EAAE,EAAE;YACb,QAAQ,EAAE,cAAc,CAAC,QAAQ;YACjC,UAAU,EAAE,cAAc,CAAC,UAAU;SACtC,CAAC,CAAA;QACF,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC;YAC3C,MAAM,EAAE,cAAc,CAAC,MAAM;YAC7B,GAAG,cAAc,CAAC,gBAAgB;SACnC,CAAC,CAAA;QACF,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,cAAc,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAA;QAC/D,MAAM,GAAG,CAAC,UAAU,EAAE,CAAA;QACtB,OAAO,GAAG,CAAA;IACZ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU;QACtB,2CAA2C;QAC3C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAA;QAC7D,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,0CAA0C,YAAY,CAAC,MAAM,mBAAmB,CAAC,CAAA;QAE7F,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC1B,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ;QACnB,IAAI,IAAI,CAAC,iBAAiB;YAAE,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACjE,IAAI,IAAI,CAAC,kBAAkB;YAAE,aAAa,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;QAEnE,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;QAC1B,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;QAC7B,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAA;QACjC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;QACxB,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;IAC5C,CAAC;IAEM,UAAU,CAAC,KAAsB;QACtC,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,4FAA4F;YAC5F,yEAAyE;YACzE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,KAAK,CAAA;YAChD,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;YAEtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,uBAAuB,CAAC,YAAY,CAAC,CAAA;YACrE,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAA;YAEzC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;YAE/E,IAAI,gBAAgB,EAAE,cAAc,KAAK,YAAY,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAA;YACvE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAA;gBAC7D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAA;gBAEtG,MAAM,QAAQ,GAA8B;oBAC1C,GAAG,YAAY;oBACf,QAAQ,EAAE,QAAQ,CAAC,YAAY,CAAC;oBAChC,QAAQ;oBACR,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBACxC,QAAQ;oBACR,IAAI,EAAE,IAAI,IAAI,EAAE;oBAChB,OAAO;oBACP,UAAU,EAAE,MAAM;iBACnB,CAAA;gBAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAA;gBAEjF,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC9B,MAAM,CAAC,IAAI,CACT,0BAA0B,YAAY,qEAAqE,CAC5G,CAAA;oBACD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;oBACjD,OAAM;gBACR,CAAC;gBACD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;gBACjD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;YACnD,CAAC;YAED,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpD,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAChC,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,KAAwB;QAChD,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAA;QACrE,MAAM,YAAY,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;QAExC,IAAI,KAAK,EAAE,CAAC;YACV,gFAAgF;YAChF,kFAAkF;YAClF,MAAM,CAAC,IAAI,CACT,wBAAwB,YAAY,wDAAwD,SAAS,IAAI,CAC1G,CAAA;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;QAC/C,CAAC;aAAM,CAAC;YACN,iEAAiE;YACjE,MAAM,CAAC,IAAI,CAAC,qCAAqC,YAAY,gBAAgB,SAAS,IAAI,CAAC,CAAA;YAC3F,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;YAC3E,MAAM,CAAC,IAAI,CAAC,SAAS,YAAY,CAAC,MAAM,6BAA6B,CAAC,CAAA;YACtE,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QAC/G,CAAC;IACH,CAAC;IAED;;OAEG;IACI,UAAU,CAAC,KAAsB;QACtC,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;YAC3D,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,eAAe,CAAC,kFAAkF,CAAC,CAAA;YAC/G,CAAC;YAED,MAAM,aAAa,GAAG,YAAY,KAAK,EAAE,CAAA;YAEzC,MAAM,QAAQ,GAA8B;gBAC1C,QAAQ,EAAE,KAAK;gBACf,QAAQ,EAAE,IAAI,CAAC,MAAM;gBACrB,QAAQ,EAAE,cAAc;gBACxB,QAAQ;gBACR,IAAI,EAAE,IAAI,IAAI,EAAE;gBAChB,OAAO;gBACP,UAAU,EAAE,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;aAC1D,CAAA;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;YAClF,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACI,SAAS,CAAC,KAAqB;QACpC,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;YACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,OAAO,IAAI,CAAC,GAAG,EAAE,EAAE,CAAA;YAEpE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;YACjC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;YAChF,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YACzC,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;YAErD,MAAM,aAAa,GAAG,SAAS,KAAK,EAAE,CAAA;YAEtC,MAAM,QAAQ,GAA8B;gBAC1C,QAAQ,EAAE,KAAK;gBACf,QAAQ,EAAE,IAAI,CAAC,MAAM;gBACrB,QAAQ,EAAE,UAAU;gBACpB,IAAI,EAAE,IAAI,IAAI,EAAE;gBAChB,OAAO;gBACP,UAAU,EAAE,KAAK;gBACjB,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;gBACzD,SAAS,EAAE,GAAG;aACf,CAAA;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;YAClF,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,KAAK,CAAC,KAAiB;QAClC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;QACvC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACpD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAED;;OAEG;IACI,YAAY,CAAC,KAAwB;QAC1C,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,MAAM,EAAE,KAAK,EAAE,IAAI,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,KAAK,CAAA;YACnF,MAAM,aAAa,GAAG,YAAY,KAAK,EAAE,CAAA;YAEzC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,aAAa,CAAC,CAAA;YAC5E,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,cAAc,CAAC,CAAA;YAE3F,MAAM,WAAW,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;YAC9C,IAAI,QAAQ,GAAG,cAAc;iBAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;iBAC3C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,IAAI,CAAC,EAAE,CAAC,CAAA;YAEX,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,IAAI,KAAK,QAAQ;oBAAE,QAAQ,IAAI,OAAO,CAAA;qBACrC,IAAI,IAAI,KAAK,SAAS;oBAAE,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAA;;oBACrD,QAAQ,GAAG,OAAO,CAAA;YACzB,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,IAAI,IAAI,EAAE,CAAA;YACrC,IAAI,IAAI;gBAAE,OAAO,GAAG,IAAI,CAAA,CAAC,UAAU;YACnC,IAAI,OAAO;gBAAE,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAC7D,IAAI,UAAU;gBAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAExE,MAAM,eAAe,GAA8B;gBACjD,GAAG,WAAW;gBACd,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,QAAQ,CAAC,MAAM;aAC1B,CAAA;YAED,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,aAAa,CAAC,CAAA;YAC7F,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,8EAA8E;YAC9E,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,IAAI,CAAC,OAAqB;QACrC,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAC5C,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,SAAS;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAA;IACrC,CAAC;IAED;;OAEG;IACI,cAAc;QACnB,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAA;IAC5E,CAAC;IAEM,KAAK,CAAC,IAAY,EAAE,UAAwB,EAAE;QACnD,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;YAClC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC;gBAAE,OAAM;YAEjD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAA;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;YAE1E,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC,CAAA;YACpE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAChC,MAAM,CAAC,IAAI,CAAC,sBAAsB,YAAY,EAAE,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;IACJ,CAAC;IAEM,OAAO,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;YAClC,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;YACtD,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;YAClC,MAAM,CAAC,IAAI,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;QACvD,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,+CAA+C;IAC/C,yBAAyB;IACzB,+CAA+C;IAEvC,iBAAiB;QACvB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,uCAAuC,IAAI,CAAC,MAAM,CAAC,iBAAiB,KAAK,CAAC,CAAA;YACtF,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;gBAC9C,IAAI,CAAC;oBACH,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;oBAC9C,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAA;oBAChD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;wBACrB,MAAM,CAAC,IAAI,CAAC,0CAA0C,YAAY,iBAAiB,CAAC,CAAA;oBACtF,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAc,EAAE,CAAC;oBACxB,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAA;gBACrD,CAAC;YACH,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;QACnC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,2CAA2C,IAAI,CAAC,MAAM,CAAC,kBAAkB,KAAK,CAAC,CAAA;YAC3F,IAAI,CAAC,kBAAkB,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;gBAC/C,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAA;gBACnC,CAAC;gBAAC,OAAO,KAAc,EAAE,CAAC;oBACxB,MAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAA;gBAC/D,CAAC;YACH,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,UAAkB,EAAE,SAAkB;QACrE,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,sBAAsB,EAAE,CAAC,CAAA;QACzE,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QAElE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YAC7C,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,SAAS,EAAE,CAAC;gBACrC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAChE,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBACxF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACtB,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,qBAAqB;QAC3B,MAAM,YAAY,GAAG,KAAK,EAAE,QAAgB,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAA;YACrC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;oBAC1F,6EAA6E;oBAC7E,MAAM,CAAC,KAAK,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAA;gBAC/D,CAAC;qBAAM,CAAC;oBACN,oDAAoD;oBACpD,MAAM,CAAC,KAAK,CAAC,kCAAkC,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAA;gBACpE,CAAC;YACH,CAAC;QACH,CAAC,CAAA;QACD,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC,CAAA;QAC3C,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,YAAY,CAAC,CAAA;QAC7C,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE,CAC3C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,kCAAkC,QAAQ,EAAE,CAAC,CAAC,CAClG,CAAA;IACH,CAAC;IAEO,kBAAkB,CAAC,GAAW;QACpC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;QACzC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,eAAe,CAAC,uBAAuB,GAAG,6BAA6B,CAAC,CAAA;QAE9F,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,IAAI,IAAI,EAAE,CAAA;QAEvB,IAAI,IAAI,KAAK,GAAG;YAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,CAAA;aACjD,IAAI,IAAI,KAAK,GAAG;YAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,KAAK,CAAC,CAAA;aACxD,IAAI,IAAI,KAAK,GAAG;YAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,CAAA;QAEnE,OAAO,IAAI,CAAC,WAAW,EAAE,CAAA;IAC3B,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,IAAY,EACZ,YAAuC,EACvC,IAAY;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAA;QAElC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC5E,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAEpC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,EAAE,EAAE,UAAU,EAAE;YAChB,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,KAAK,CAAC,KAAK;YACvB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;YACrB,QAAQ,EAAE;gBACR,QAAQ,EAAE,EAAE,EAAE,sCAAsC;gBACpD,QAAQ,EAAE,CAAC;gBACX,QAAQ,EAAE,EAAE;gBACZ,SAAS,EAAE,GAAG;gBACd,SAAS,EAAE,GAAG;gBACd,GAAG,YAAY;aAChB;YACD,SAAS,EAAE,GAAG;SACf,CAAC,CAAC,CAAA;IACL,CAAC;CACF"}

package/dist/local-rag/parser.d.ts

@@ -0,0 +1,59 @@
+import type { DocumentMetadata } from './types.js';
+export interface ParserConfig {
+    baseDir: string;
+    maxFileSize: number;
+}
+export interface ParsedFile {
+    text: string;
+    language?: string;
+    fileSize: number;
+    metadata: Partial<DocumentMetadata>;
+}
+/**
+ * Parses text content from various file formats.
+ * Extracts basic metadata and transforms structured content for better RAG performance.
+ */
+export declare class DocumentParser {
+    private readonly config;
+    constructor(config: ParserConfig);
+    /**
+     * Parses a file, automatically detecting its format.
+     * @param filePath - The path to the file to parse.
+     * @returns The extracted text, detected language (for code files), and file size.
+     */
+    parseFile(filePath: string): Promise<ParsedFile>;
+    /**
+     * Returns a list of all supported file extensions.
+     * @returns An array of file extensions (e.g., ['.pdf', '.ts', '.md']).
+     */
+    getSupportedExtensions(): string[];
+    /**
+     * Validates that a file path is absolute and within the configured `baseDir`.
+     * @param filePath - The absolute path to validate.
+     * @throws {ValidationError} If the path is invalid or outside the security boundary.
+     */
+    validateFilePath(filePath: string): void;
+    /**
+     * Validates that a file's size is within the configured limit.
+     * @param filePath - The path to the file to check.
+     * @returns The file size in bytes.
+     * @throws {ValidationError} If the file is too large.
+     * @throws {FileOperationError} If the file stats cannot be read.
+     */
+    validateAndGetFileStats(filePath: string): Promise<{
+        fileSize: number;
+        fileCreatedAt: string;
+        fileModifiedAt: string;
+    }>;
+    private detectLanguage;
+    private parsePdf;
+    private parseDocx;
+    private parseJson;
+    /**
+     * Parses CSV into a text format where each row preserves the header context.
+     * This is crucial for the semantic meaning of chunks.
+     */
+    private parseStructuredCsv;
+    private parseAsPlainText;
+}
+//# sourceMappingURL=parser.d.ts.map

package/dist/local-rag/parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/local-rag/parser.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AA6BlD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;CACpC;AAED;;;GAGG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;gBAEzB,MAAM,EAAE,YAAY;IAIhC;;;;OAIG;IACU,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IA4C7D;;;OAGG;IACI,sBAAsB,IAAI,MAAM,EAAE;IAMzC;;;;OAIG;IACI,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAa/C;;;;;;OAMG;IACU,uBAAuB,CAClC,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC;IAiB/E,OAAO,CAAC,cAAc;YAIR,QAAQ;YAaR,SAAS;YAWT,SAAS;IAWvB;;;OAGG;YACW,kBAAkB;YAuBlB,gBAAgB;CAO/B"}