@sunub/obsidian-mcp-server 0.1.1 → 0.3.0

package/build/utils/RAGIndexer.js

@@ -0,0 +1,206 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
+import { encodingForModel } from "js-tiktoken";
+import ora, {} from "ora";
+import { DirectoryWalker } from "./DirectoryWalker.js";
+import { llmClient } from "./LLMClient.js";
+import { parse as parseMatter } from "./processor/MatterParser.js";
+import { Semaphore } from "./semaphore.js";
+import { vectorDB } from "./VectorDB.js";
+function extractHeadingsWithPositions(body) {
+    const headingRegex = /^(#{1,3})\s+(.+)$/gm;
+    const results = [];
+    const matches = body.matchAll(headingRegex);
+    for (const match of matches) {
+        results.push({
+            heading: match[2].trim(),
+            pos: match.index,
+            depth: match[1].length,
+        });
+    }
+    return results;
+}
+function findSectionForChunk(body, chunk, headings) {
+    if (headings.length === 0)
+        return null;
+    const pos = body.indexOf(chunk.slice(0, 60));
+    if (pos === -1)
+        return null;
+    let section = null;
+    for (const h of headings) {
+        if (h.pos <= pos)
+            section = h.heading;
+    }
+    return section;
+}
+export class RAGIndexer {
+    splitter;
+    embeddingSemaphore;
+    ioSemaphore;
+    spinner = null;
+    totalFiles = 0;
+    processedFiles = 0;
+    enc = encodingForModel("gpt-3.5-turbo");
+    constructor() {
+        this.splitter = new RecursiveCharacterTextSplitter({
+            chunkSize: 500,
+            chunkOverlap: 50,
+            lengthFunction: (text) => {
+                return this.enc.encode(text).length;
+            },
+        });
+        this.embeddingSemaphore = new Semaphore(3);
+        this.ioSemaphore = new Semaphore(10);
+    }
+    setSpinner(total) {
+        this.totalFiles = total;
+        this.processedFiles = 0;
+        this.spinner = ora({
+            text: `Initializing RAG indexing (0/${total})...`,
+            stream: process.stderr,
+        }).start();
+    }
+    updateProgress(filePath) {
+        this.processedFiles++;
+        if (this.spinner) {
+            this.spinner.text = `[${this.processedFiles}/${this.totalFiles}] Indexing: ${path.basename(filePath)}`;
+        }
+        if (this.processedFiles >= this.totalFiles && this.spinner) {
+            this.spinner.succeed(`Successfully indexed ${this.totalFiles} files.`);
+            this.spinner = null;
+        }
+    }
+    async buildFileRecords(filePath, content) {
+        const fileStat = await fs.stat(filePath);
+        const fileContent = content ?? (await fs.readFile(filePath, "utf-8"));
+        const { frontmatter, content: body } = parseMatter(filePath, fileStat.birthtime.toISOString(), fileContent);
+        const fileName = path.basename(filePath);
+        const headings = extractHeadingsWithPositions(body);
+        const titleFromBody = headings.find((h) => h.depth === 1)?.heading ?? null;
+        const docTitle = frontmatter.title || titleFromBody || fileName;
+        const docStructure = headings
+            .filter((h) => h.depth <= 2)
+            .slice(0, 4)
+            .map((h) => h.heading.slice(0, 25));
+        const metadataPrefix = [
+            `Title: ${docTitle.slice(0, 50)}`,
+            frontmatter.tags?.length
+                ? `Tags: ${frontmatter.tags.slice(0, 3).join(", ")}`
+                : "",
+            frontmatter.summary ? `Summary: ${frontmatter.summary.slice(0, 60)}` : "",
+        ]
+            .filter(Boolean)
+            .join("\n");
+        const chunks = await this.splitter.splitText(body);
+        const records = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const sectionHeading = findSectionForChunk(body, chunk, headings);
+            const context = [
+                sectionHeading ? `Section: ${sectionHeading.slice(0, 30)}` : "",
+                docStructure.length > 1
+                    ? `Outline: ${docStructure.slice(0, 3).join(" > ")}`
+                    : "",
+            ]
+                .filter(Boolean)
+                .join("\n");
+            const combined = [metadataPrefix, context, chunk]
+                .filter(Boolean)
+                .join("\n\n");
+            const MAX_EMBED_TOKENS = 500;
+            const combinedTokenCount = this.enc.encode(combined).length;
+            const safeText = combinedTokenCount > MAX_EMBED_TOKENS
+                ? combined.slice(0, Math.floor(combined.length * (MAX_EMBED_TOKENS / combinedTokenCount)))
+                : combined;
+            await this.embeddingSemaphore.acquire();
+            let vector;
+            try {
+                vector = await llmClient.generateEmbedding(`search_document: ${safeText}`);
+            }
+            finally {
+                this.embeddingSemaphore.release();
+            }
+            records.push({
+                id: `${filePath}_chunk_${i}`,
+                filePath,
+                fileName,
+                chunkIndex: i,
+                content: chunk,
+                context,
+                vector,
+                metadata: {
+                    title: docTitle,
+                    date: frontmatter.date instanceof Date
+                        ? frontmatter.date.toISOString()
+                        : frontmatter.date || fileStat.birthtime.toISOString(),
+                    tags: frontmatter.tags?.join(", ") ?? "",
+                    summary: frontmatter.summary || "",
+                    slug: frontmatter.slug || "",
+                    category: frontmatter.category || "any",
+                    completed: frontmatter.completed ?? false,
+                },
+            });
+        }
+        return { records, mtime: fileStat.mtime.toISOString() };
+    }
+    async processFile(filePath, content) {
+        try {
+            const { records, mtime } = await this.buildFileRecords(filePath, content);
+            if (records.length > 0) {
+                await vectorDB.upsertChunks(records);
+            }
+            // 청크 생성 여부와 상관없이 파일 메타데이터(수정 시각)는 항상 업데이트
+            await vectorDB.updateFileMeta(filePath, mtime);
+        }
+        catch (error) {
+            console.error(`\nError processing file for RAG: ${filePath}`, error);
+        }
+        finally {
+            this.updateProgress(filePath);
+        }
+    }
+    async processFileInMemory(filePath, content) {
+        try {
+            return await this.buildFileRecords(filePath, content);
+        }
+        catch (error) {
+            console.error(`\nError processing file for RAG: ${filePath}`, error);
+            return null;
+        }
+        finally {
+            this.updateProgress(filePath);
+        }
+    }
+    async indexAll(vaultPath) {
+        const walker = new DirectoryWalker();
+        const filePaths = await walker.walk(vaultPath, this.ioSemaphore);
+        const markdownFiles = filePaths.filter((f) => f.endsWith(".md") || f.endsWith(".mdx"));
+        this.setSpinner(markdownFiles.length);
+        const allRecords = [];
+        const allMeta = [];
+        const fileSemaphore = new Semaphore(8);
+        await Promise.all(markdownFiles.map(async (filePath) => {
+            await fileSemaphore.acquire();
+            try {
+                const result = await this.processFileInMemory(filePath);
+                if (result && result.records.length > 0) {
+                    allRecords.push(...result.records);
+                    allMeta.push({ filePath, mtime: result.mtime });
+                }
+            }
+            finally {
+                fileSemaphore.release();
+            }
+        }));
+        if (allRecords.length > 0) {
+            await vectorDB.upsertChunks(allRecords);
+            await vectorDB.updateFileMetaBatch(allMeta);
+        }
+    }
+    async deleteFile(filePath) {
+        await vectorDB.deleteByFilePath(filePath);
+    }
+}
+export const ragIndexer = new RAGIndexer();
+//# sourceMappingURL=RAGIndexer.js.map

package/build/utils/RAGIndexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RAGIndexer.js","sourceRoot":"","sources":["../../src/utils/RAGIndexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,GAAG,EAAE,EAAY,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAqB,QAAQ,EAAE,MAAM,eAAe,CAAC;AAI5D,SAAS,4BAA4B,CAAC,IAAY;IACjD,MAAM,YAAY,GAAG,qBAAqB,CAAC;IAC3C,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC5C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;YACxB,GAAG,EAAE,KAAK,CAAC,KAAK;YAChB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;SACtB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAC3B,IAAY,EACZ,KAAa,EACb,QAAwB;IAExB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7C,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,OAAO,GAAkB,IAAI,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;IACvC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,UAAU;IACd,QAAQ,CAAiC;IACzC,kBAAkB,CAAY;IAC9B,WAAW,CAAY;IACvB,OAAO,GAAe,IAAI,CAAC;IAC3B,UAAU,GAAG,CAAC,CAAC;IACf,cAAc,GAAG,CAAC,CAAC;IACnB,GAAG,GAAG,gBAAgB,CAAC,eAAe,CAAC,CAAC;IAEhD;QACC,IAAI,CAAC,QAAQ,GAAG,IAAI,8BAA8B,CAAC;YAClD,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;YAChB,cAAc,EAAE,CAAC,IAAY,EAAE,EAAE;gBAChC,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACrC,CAAC;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAI,SAAS,CAAC,EAAE,CAAC,CAAC;IACtC,CAAC;IAEM,UAAU,CAAC,KAAa;QAC9B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC;YAClB,IAAI,EAAE,gCAAgC,KAAK,MAAM;YACjD,MAAM,EAAE,OAAO,CAAC,MAAM;SACtB,CAAC,CAAC,KAAK,EAAE,CAAC;IACZ,CAAC;IAEO,cAAc,CAAC,QAAgB;QACtC,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,UAAU,eAAe,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxG,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC5D,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,wBAAwB,IAAI,CAAC,UAAU,SAAS,CAAC,CAAC;YACvE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACrB,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC7B,QAAgB,EAChB,OAAgB;QAEhB,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,WAAW,CACjD,QAAQ,EACR,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE,EAChC,WAAW,CACX,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEzC,MAAM,QAAQ,GAAG,4BAA4B,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC;QAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,IAAI,aAAa,IAAI,QAAQ,CAAC;QAChE,MAAM,YAAY,GAAG,QAAQ;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAErC,MAAM,cAAc,GAAG;YACtB,UAAU,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;YACjC,WAAW,CAAC,IAAI,EAAE,MAAM;gBACvB,CAAC,CAAC,SAAS,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpD,CAAC,CAAC,EAAE;YACL,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;SACzE;aACC,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;QAEb,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YAExB,MAAM,cAAc,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClE,MAAM,OAAO,GAAG;gBACf,cAAc,CAAC,CAAC,CAAC,YAAY,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;gBAC/D,YAAY,CAAC,MAAM,GAAG,CAAC;oBACtB,CAAC,CAAC,YAAY,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBACpD,CAAC,CAAC,EAAE;aACL;iBACC,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,CAAC;YAEb,MAAM,QAAQ,GAAG,CAAC,cAAc,EAAE,OAAO,EAAE,KAAK,CAAC;iBAC/C,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,MAAM,CAAC,CAAC;YAEf,MAAM,gBAAgB,GAAG,GAAG,CAAC;YAC7B,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;YAC5D,MAAM,QAAQ,GACb,kBAAkB,GAAG,gBAAgB;gBACpC,CAAC,CAAC,QAAQ,CAAC,KAAK,CACd,CAAC,EACD,IAAI,CAAC,KAAK,CACT,QAAQ,CAAC,MAAM,GAAG,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CACzD,CACD;gBACF,CAAC,CAAC,QAAQ,CAAC;YAEb,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,MAAgB,CAAC;YACrB,IAAI,CAAC;gBACJ,MAAM,GAAG,MAAM,SAAS,CAAC,iBAAiB,CACzC,oBAAoB,QAAQ,EAAE,CAC9B,CAAC;YACH,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC;YACnC,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACZ,EAAE,EAAE,GAAG,QAAQ,UAAU,CAAC,EAAE;gBAC5B,QAAQ;gBACR,QAAQ;gBACR,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,KAAK;gBACd,OAAO;gBACP,MAAM;gBACN,QAAQ,EAAE;oBACT,KAAK,EAAE,QAAQ;oBACf,IAAI,EACH,WAAW,CAAC,IAAI,YAAY,IAAI;wBAC/B,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE;wBAChC,CAAC,CAAC,WAAW,CAAC,IAAI,IAAI,QAAQ,CAAC,SAAS,CAAC,WAAW,EAAE;oBACxD,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACxC,OAAO,EAAE,WAAW,CAAC,OAAO,IAAI,EAAE;oBAClC,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,EAAE;oBAC5B,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,KAAK;oBACvC,SAAS,EAAE,WAAW,CAAC,SAAS,IAAI,KAAK;iBACzC;aACD,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,OAAgB;QACnD,IAAI,CAAC;YACJ,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC1E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YACtC,CAAC;YACD,0CAA0C;YAC1C,MAAM,QAAQ,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,oCAAoC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;QACtE,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,mBAAmB,CACxB,QAAgB,EAChB,OAAgB;QAEhB,IAAI,CAAC;YACJ,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,oCAAoC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACb,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB;QAC/B,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACjE,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC9C,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAEtC,MAAM,UAAU,GAAmB,EAAE,CAAC;QACtC,MAAM,OAAO,GAA0C,EAAE,CAAC;QAC1D,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;QAEvC,MAAM,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YACpC,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACJ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;gBACxD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzC,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;oBACnC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;gBACjD,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,aAAa,CAAC,OAAO,EAAE,CAAC;YACzB,CAAC;QACF,CAAC,CAAC,CACF,CAAC;QAEF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,QAAQ,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAAgB;QAChC,MAAM,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;CACD;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,UAAU,EAAE,CAAC"}

package/build/utils/RerankerClient.d.ts

@@ -0,0 +1,27 @@
+import { z } from "zod/v4";
+export declare const RerankResponseSchema: z.ZodObject<{
+    model: z.ZodString;
+    object: z.ZodString;
+    usage: z.ZodObject<{
+        prompt_tokens: z.ZodNumber;
+        total_tokens: z.ZodNumber;
+    }, z.core.$strip>;
+    results: z.ZodArray<z.ZodObject<{
+        index: z.ZodNumber;
+        relevance_score: z.ZodNumber;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type RerankResult = {
+    index: number;
+    relevance_score: number;
+    document: string;
+};
+declare class RerankerClient {
+    private endpoint;
+    private model;
+    constructor();
+    rerank(query: string, documents: string[], topN?: number): Promise<RerankResult[]>;
+    private sigmoid;
+}
+export declare const rerankerClient: RerankerClient;
+export {};

package/build/utils/RerankerClient.js

@@ -0,0 +1,69 @@
+import { z } from "zod/v4";
+import { debugLogger } from "../cli/utils/debugLogger.js";
+import state from "../config.js";
+export const RerankResponseSchema = z.object({
+    model: z.string(),
+    object: z.string(),
+    usage: z.object({
+        prompt_tokens: z.number(),
+        total_tokens: z.number(),
+    }),
+    results: z.array(z.object({
+        index: z.number(),
+        relevance_score: z.number(),
+    })),
+});
+class RerankerClient {
+    endpoint;
+    model;
+    constructor() {
+        this.endpoint = state.llmRerankerApiUrl;
+        this.model = "bge-reranker-v2-m3-GGUF";
+    }
+    async rerank(query, documents, topN = 5) {
+        if (documents.length === 0) {
+            debugLogger.warn("No documents to rerank, returning empty results.");
+            return [];
+        }
+        try {
+            const response = await fetch(`${this.endpoint}/v1/rerank`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    query,
+                    documents,
+                    top_n: topN,
+                }),
+            });
+            if (!response.ok) {
+                throw new Error(`Reranker API error: ${response.status} ${response.statusText}`);
+            }
+            const data = await response.json();
+            const parsed = RerankResponseSchema.parse(data);
+            const rerankingResults = parsed.results.map((result) => ({
+                index: result.index,
+                relevance_score: this.sigmoid(result.relevance_score),
+                document: documents[result.index],
+            }));
+            return rerankingResults
+                .filter(({ relevance_score }) => relevance_score > 0.5)
+                .sort((a, b) => b.relevance_score - a.relevance_score);
+        }
+        catch (error) {
+            debugLogger.error("Reranking failed, falling back to original order:", error);
+            return documents.map((doc, index) => ({
+                index,
+                relevance_score: 0,
+                document: doc,
+            }));
+        }
+    }
+    sigmoid(score) {
+        return 1 / (1 + Math.exp(-score));
+    }
+}
+export const rerankerClient = new RerankerClient();
+//# sourceMappingURL=RerankerClient.js.map

package/build/utils/RerankerClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RerankerClient.js","sourceRoot":"","sources":["../../src/utils/RerankerClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,QAAQ,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,KAAK,MAAM,cAAc,CAAC;AAEjC,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;KACxB,CAAC;IACF,OAAO,EAAE,CAAC,CAAC,KAAK,CACf,CAAC,CAAC,MAAM,CAAC;QACR,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE;KAC3B,CAAC,CACF;CACD,CAAC,CAAC;AAQH,MAAM,cAAc;IACX,QAAQ,CAAS;IACjB,KAAK,CAAS;IAEtB;QACC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,yBAAyB,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,MAAM,CACX,KAAa,EACb,SAAmB,EACnB,OAAe,CAAC;QAEhB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,WAAW,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;YACrE,OAAO,EAAE,CAAC;QACX,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,YAAY,EAAE;gBAC1D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACR,cAAc,EAAE,kBAAkB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK;oBACL,SAAS;oBACT,KAAK,EAAE,IAAI;iBACX,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACd,uBAAuB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC/D,CAAC;YACH,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACxD,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC;gBACrD,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;aACjC,CAAC,CAAC,CAAC;YAEJ,OAAO,gBAAgB;iBACrB,MAAM,CAAC,CAAC,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,eAAe,GAAG,GAAG,CAAC;iBACtD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,WAAW,CAAC,KAAK,CAChB,mDAAmD,EACnD,KAAK,CACL,CAAC;YACF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACrC,KAAK;gBACL,eAAe,EAAE,CAAC;gBAClB,QAAQ,EAAE,GAAG;aACb,CAAC,CAAC,CAAC;QACL,CAAC;IACF,CAAC;IAEO,OAAO,CAAC,KAAa;QAC5B,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnC,CAAC;CACD;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC"}

package/build/utils/VaultManger/VaultManager.d.ts

@@ -0,0 +1,34 @@
+import type { DocumentIndex } from "../processor/types.js";
+import type { EnrichedDocument } from "./types.js";
+export declare class VaultManager {
+    private vaultPath;
+    private isInitialized;
+    private walker;
+    private indexer;
+    private ioSemaphore;
+    constructor(vaultPath: string, maxConcurrentIO?: number);
+    initialize(): Promise<void>;
+    getAllDocuments(): Promise<DocumentIndex[]>;
+    searchDocuments(keyword: string): Promise<DocumentIndex[]>;
+    getDocumentInfo(filename: string, options?: {
+        includeStats?: boolean;
+        includeBacklinks?: boolean;
+        maxContentPreview?: number;
+        includeContentHash?: boolean;
+    }): Promise<EnrichedDocument | null>;
+    writeDocument(fullPath: string, frontmatter: Record<string, unknown>): Promise<void>;
+    writeRawDocument(fullPath: string, content: string): Promise<void>;
+    refresh(): Promise<void>;
+    getStats(): {
+        totalFiles: number;
+        isInitialized: boolean;
+        vaultPath: string;
+    };
+    private parseFilenameToFullPath;
+    private resolvePathForWrite;
+    private isPathInsideVault;
+    private readDocumentContent;
+    private getDocumentContent;
+    private addStats;
+    private addBacklinks;
+}

package/build/utils/VaultManger/VaultManager.js

@@ -201,3 +201,4 @@ export class VaultManager {
         });
     }
 }
+//# sourceMappingURL=VaultManager.js.map

package/build/utils/VaultManger/VaultManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VaultManager.js","sourceRoot":"","sources":["../../../src/utils/VaultManger/VaultManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACN,QAAQ,EACR,OAAO,EACP,OAAO,EACP,UAAU,EACV,IAAI,EACJ,QAAQ,EACR,OAAO,GACP,MAAM,WAAW,CAAC;AACnB,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,OAAO,YAAY;IAChB,SAAS,CAAS;IAClB,aAAa,GAAY,KAAK,CAAC;IAC/B,MAAM,CAAkB;IACxB,OAAO,CAAU;IACjB,WAAW,CAAY;IAE/B,YAAY,SAAiB,EAAE,kBAA0B,EAAE;QAC1D,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,UAAU;QACf,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,OAAO;QACR,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3E,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC3B,CAAC;IAEM,KAAK,CAAC,eAAe;QAC3B,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;IACvC,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,OAAe;QAC3C,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAEM,KAAK,CAAC,eAAe,CAC3B,QAAgB,EAChB,UAKI,EAAE;QAEN,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC;QACb,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,MAAM,WAAW,GAAqB;YACrC,GAAG,KAAK;YACR,OAAO,EAAE,OAAO,CAAC,iBAAiB;gBACjC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBACjD,CAAC,CAAC,OAAO;SACV,CAAC;QAEF,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YAChC,WAAW,CAAC,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC;iBAC5C,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC;iBACvB,MAAM,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,WAAW,CAAC;IACpB,CAAC;IAEM,KAAK,CAAC,aAAa,CACzB,QAAgB,EAChB,WAAoC;QAEpC,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;YACrE,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC3D,MAAM,SAAS,CAAC,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC;IAEM,KAAK,CAAC,gBAAgB,CAC5B,QAAgB,EAChB,OAAe;QAEf,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAExD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,IAAI,CAAC;YACJ,MAAM,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACxD,MAAM,SAAS,CAAC,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAC3B,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IACzB,CAAC;IAEM,QAAQ;QACd,OAAO;YACN,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;YACnC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,SAAS,EAAE,IAAI,CAAC,SAAS;SACzB,CAAC;IACH,CAAC;IAEO,uBAAuB,CAAC,QAAgB;QAC/C,8BAA8B;QAC9B,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9D,OAAO,QAAQ,CAAC;YACjB,CAAC;YACD,OAAO,EAAE,CAAC;QACX,CAAC;QAED,qCAAqC;QACrC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACX,CAAC;QACD,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QAClB,CAAC;QAED,MAAM,UAAU,GAAG,EAAE,CAAC;QACtB,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACP,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC;YAC3D,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,QAAQ,MAAM,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChE,OAAO,SAAS,CAAC;YAClB,CAAC;QACF,CAAC;QAED,MAAM,WAAW,GAAG;YACnB,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YAC/B,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;SACrC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1E,IAAI,QAAQ,EAAE,CAAC;gBACd,OAAO,QAAQ,CAAC,QAAQ,CAAC;YAC1B,CAAC;QACF,CAAC;QACD,OAAO,EAAE,CAAC;IACX,CAAC;IAEO,mBAAmB,CAAC,SAAiB;QAC5C,MAAM,YAAY,GAAG,UAAU,CAAC,SAAS,CAAC;YACzC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;YACpB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAEtC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,YAAY,CAAC;IACrB,CAAC;IAEO,iBAAiB,CAAC,aAAqB;QAC9C,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;QAC7D,OAAO,CACN,YAAY,KAAK,EAAE;YACnB,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,QAAgB;QACjD,IAAI,CAAC;YACJ,OAAO,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,sBAAsB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,QAAgB;QAChD,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,IAAI,CAAC;YACJ,OAAO,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QAC5B,CAAC;IACF,CAAC;IAEO,QAAQ,CAAC,GAAqB,EAAE,OAAe;QACtD,GAAG,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;YACtD,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;YACrC,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,UAAU,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YACrC,cAAc,EAAE,OAAO,CAAC,MAAM;SAC9B,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,GAAqB,EAAE,QAAgB;QAC3D,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC1D,GAAG,CAAC,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAChD,OAAO;gBACN,QAAQ,EAAE,IAAI;gBACd,KAAK,EACJ,QAAQ,EAAE,WAAW,EAAE,KAAK;oBAC5B,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;oBACrD,UAAU;aACX,CAAC;QACH,CAAC,CAAC,CAAC;IACJ,CAAC;CACD"}

package/build/utils/VaultManger/VaultPathError.d.ts

@@ -0,0 +1,6 @@
+export declare class VaultPathError extends Error {
+    readonly inputPath: string;
+    readonly resolvedPath: string;
+    readonly vaultPath: string;
+    constructor(inputPath: string, resolvedPath: string, vaultPath: string);
+}

package/build/utils/VaultManger/VaultPathError.js

@@ -10,3 +10,4 @@ export class VaultPathError extends Error {
         this.vaultPath = vaultPath;
     }
 }
+//# sourceMappingURL=VaultPathError.js.map

package/build/utils/VaultManger/VaultPathError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VaultPathError.js","sourceRoot":"","sources":["../../../src/utils/VaultManger/VaultPathError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,cAAe,SAAQ,KAAK;IACxB,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,SAAS,CAAS;IAElC,YAAY,SAAiB,EAAE,YAAoB,EAAE,SAAiB;QACrE,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC5B,CAAC;CACD"}

package/build/utils/VaultManger/index.d.ts

@@ -0,0 +1,3 @@
+export type { EnrichedDocument } from "./types.js";
+export { VaultManager } from "./VaultManager.js";
+export { VaultPathError } from "./VaultPathError.js";

package/build/utils/VaultManger/index.js

@@ -1,2 +1,3 @@
 export { VaultManager } from "./VaultManager.js";
 export { VaultPathError } from "./VaultPathError.js";
+//# sourceMappingURL=index.js.map

package/build/utils/VaultManger/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/utils/VaultManger/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC"}

package/build/utils/VaultManger/types.d.ts

@@ -0,0 +1,16 @@
+import type { DocumentIndex } from "../processor/types.js";
+export interface EnrichedDocument extends DocumentIndex {
+    content: string;
+    contentHash?: string;
+    stats?: {
+        wordCount: number;
+        lineCount: number;
+        characterCount: number;
+        contentLength: number;
+        hasContent: boolean;
+    };
+    backlinks?: {
+        filePath: string;
+        title: string;
+    }[];
+}

package/build/utils/VaultManger/types.js

@@ -1 +1,2 @@
 export {};
+//# sourceMappingURL=types.js.map

package/build/utils/VaultManger/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/utils/VaultManger/types.ts"],"names":[],"mappings":""}

package/build/utils/VaultWatcher.d.ts

@@ -0,0 +1,8 @@
+export declare class VaultWatcher {
+    private watcher;
+    private forceReindex;
+    start(vaultPath: string): Promise<void>;
+    stop(): Promise<void>;
+    private isMarkdown;
+}
+export declare const vaultWatcher: VaultWatcher;

package/build/utils/VaultWatcher.js

@@ -0,0 +1,96 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import chokidar, {} from "chokidar";
+import { DirectoryWalker } from "./DirectoryWalker.js";
+import { ragIndexer } from "./RAGIndexer.js";
+import { Semaphore } from "./semaphore.js";
+import { vectorDB } from "./VectorDB.js";
+export class VaultWatcher {
+    watcher = null;
+    forceReindex = false;
+    async start(vaultPath) {
+        if (this.watcher) {
+            await this.watcher.close();
+        }
+        this.forceReindex = await vectorDB.checkAndMigrateIfNeeded();
+        if (this.forceReindex) {
+            console.error("[VaultWatcher] Index version changed — all files will be re-indexed on this startup.");
+        }
+        const walker = new DirectoryWalker();
+        const ioSemaphore = new Semaphore(10);
+        const allFiles = await walker.walk(vaultPath, ioSemaphore);
+        const markdownFiles = allFiles.filter((f) => this.isMarkdown(f));
+        const filesToProcess = [];
+        for (const filePath of markdownFiles) {
+            if (this.forceReindex) {
+                filesToProcess.push(filePath);
+            }
+            else {
+                const stats = await fs.stat(filePath);
+                const storedMtimeStr = await vectorDB.getFileMtime(filePath);
+                if (storedMtimeStr) {
+                    const storedTime = new Date(storedMtimeStr).getTime();
+                    const fileTime = stats.mtime.getTime();
+                    if (Math.abs(storedTime - fileTime) > 1000) {
+                        filesToProcess.push(filePath);
+                    }
+                }
+                else {
+                    filesToProcess.push(filePath);
+                }
+            }
+        }
+        if (filesToProcess.length > 0) {
+            console.error(`[VaultWatcher] Found ${filesToProcess.length} files to index.`);
+            ragIndexer.setSpinner(filesToProcess.length);
+            for (const filePath of filesToProcess) {
+                await ragIndexer.processFile(filePath);
+            }
+            await vectorDB.createVectorIndex();
+        }
+        else {
+            console.error("[VaultWatcher] No files need indexing.");
+        }
+        this.forceReindex = false;
+        console.error(`Starting Vault Watcher for real-time sync: ${vaultPath}`);
+        this.watcher = chokidar.watch(vaultPath, {
+            ignored: [/(^|[/\\])\../, "**/node_modules/**", "**/.obsidian/**"],
+            persistent: true,
+            ignoreInitial: true,
+        });
+        this.watcher
+            .on("add", async (filePath) => {
+            if (this.isMarkdown(filePath)) {
+                console.error(`File added: ${filePath}`);
+                await ragIndexer.processFile(filePath);
+            }
+        })
+            .on("change", async (filePath) => {
+            if (this.isMarkdown(filePath)) {
+                console.error(`File changed: ${filePath}`);
+                await ragIndexer.processFile(filePath);
+            }
+        })
+            .on("unlink", async (filePath) => {
+            if (this.isMarkdown(filePath)) {
+                console.error(`File deleted: ${filePath}`);
+                await ragIndexer.deleteFile(filePath);
+            }
+        });
+        this.watcher.on("error", (error) => {
+            console.error(`Watcher error: ${error}`);
+        });
+    }
+    async stop() {
+        if (this.watcher) {
+            await this.watcher.close();
+            this.watcher = null;
+        }
+    }
+    isMarkdown(filePath) {
+        const ext = path.extname(filePath).toLowerCase();
+        return ext === ".md" || ext === ".mdx";
+    }
+}
+export const vaultWatcher = new VaultWatcher();
+//# sourceMappingURL=VaultWatcher.js.map

package/build/utils/VaultWatcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"VaultWatcher.js","sourceRoot":"","sources":["../../src/utils/VaultWatcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,QAAQ,EAAE,EAAkB,MAAM,UAAU,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,MAAM,OAAO,YAAY;IAChB,OAAO,GAAqB,IAAI,CAAC;IACjC,YAAY,GAAG,KAAK,CAAC;IAE7B,KAAK,CAAC,KAAK,CAAC,SAAiB;QAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,QAAQ,CAAC,uBAAuB,EAAE,CAAC;QAC7D,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CACZ,sFAAsF,CACtF,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,IAAI,SAAS,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAEjE,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvB,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACP,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtC,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAE7D,IAAI,cAAc,EAAE,CAAC;oBACpB,MAAM,UAAU,GAAG,IAAI,IAAI,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC;oBACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBAEvC,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,QAAQ,CAAC,GAAG,IAAI,EAAE,CAAC;wBAC5C,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC/B,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/B,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CACZ,wBAAwB,cAAc,CAAC,MAAM,kBAAkB,CAC/D,CAAC;YACF,UAAU,CAAC,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YAC7C,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACvC,MAAM,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;YACD,MAAM,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAE1B,OAAO,CAAC,KAAK,CAAC,8CAA8C,SAAS,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE;YACxC,OAAO,EAAE,CAAC,cAAc,EAAE,oBAAoB,EAAE,iBAAiB,CAAC;YAClE,UAAU,EAAE,IAAI;YAChB,aAAa,EAAE,IAAI;SACnB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO;aACV,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;YACrC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,OAAO,CAAC,KAAK,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;gBACzC,MAAM,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;QACF,CAAC,CAAC;aACD,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;YACxC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,OAAO,CAAC,KAAK,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;gBAC3C,MAAM,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC;QACF,CAAC,CAAC;aACD,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;YACxC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC/B,OAAO,CAAC,KAAK,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;gBAC3C,MAAM,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;QACF,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE;YAC3C,OAAO,CAAC,KAAK,CAAC,kBAAkB,KAAK,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACrB,CAAC;IACF,CAAC;IAEO,UAAU,CAAC,QAAgB;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,CAAC;IACxC,CAAC;CACD;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC"}

package/build/utils/VectorDB.d.ts

@@ -0,0 +1,53 @@
+import type { Connection } from "@lancedb/lancedb";
+export declare const INDEX_VERSION = 5;
+export interface ChunkMetadata {
+    id: string;
+    filePath: string;
+    fileName: string;
+    chunkIndex: number;
+    content: string;
+    context?: string;
+    metadata: {
+        title: string;
+        date: string;
+        tags: string;
+        summary: string;
+        slug: string;
+        category: string;
+        completed: boolean;
+    };
+    [key: string]: unknown;
+}
+export interface VectorRecord extends ChunkMetadata {
+    vector: number[];
+}
+export declare class VectorDB {
+    private dbPath;
+    private tableName;
+    private metaTableName;
+    private fileMetaTableName;
+    private db;
+    constructor();
+    private ensureDir;
+    connect(): Promise<Connection>;
+    private getDb;
+    getTable(): Promise<import("@lancedb/lancedb").Table | null>;
+    checkAndMigrateIfNeeded(): Promise<boolean>;
+    private readMetadata;
+    private writeMetadata;
+    createVectorIndex(): Promise<void>;
+    upsertChunks(records: VectorRecord[]): Promise<void>;
+    updateFileMeta(filePath: string, mtime: string): Promise<void>;
+    updateFileMetaBatch(entries: {
+        filePath: string;
+        mtime: string;
+    }[]): Promise<void>;
+    getFileMtime(filePath: string): Promise<string | null>;
+    deleteByFilePath(filePath: string): Promise<void>;
+    hasFile(filePath: string): Promise<boolean>;
+    search(queryVector: number[], limit?: number): Promise<(ChunkMetadata & {
+        _distance: number;
+        vector: number[];
+    })[]>;
+}
+export declare const vectorDB: VectorDB;