@robthepcguy/rag-vault 1.0.0

package/dist/parser/pdf-filter.d.ts

@@ -0,0 +1,89 @@
+import type { EmbedderInterface } from '../chunker/semantic-chunker.js';
+export type { EmbedderInterface };
+/**
+ * Text item with position information from PDF
+ */
+export interface TextItemWithPosition {
+    text: string;
+    x: number;
+    y: number;
+    fontSize: number;
+    hasEOL: boolean;
+}
+/**
+ * Page data containing positioned text items
+ */
+export interface PageData {
+    pageNum: number;
+    items: TextItemWithPosition[];
+}
+/**
+ * Join filtered pages into text
+ *
+ * @param pages - Filtered page data
+ * @returns Joined text with proper line breaks
+ */
+export declare function joinFilteredPages(pages: PageData[]): string;
+/**
+ * Configuration for sentence-level pattern detection
+ */
+export interface SentencePatternConfig {
+    /** Similarity threshold for pattern detection (default: 0.85) */
+    similarityThreshold: number;
+    /** Minimum pages required for pattern detection (default: 3) */
+    minPages: number;
+    /** Number of pages to sample from center for pattern detection (default: 5) */
+    samplePages: number;
+}
+/** Default configuration for sentence-level pattern detection */
+export declare const DEFAULT_SENTENCE_PATTERN_CONFIG: SentencePatternConfig;
+/**
+ * Result of sentence-level pattern detection
+ */
+export interface SentencePatternResult {
+    /** Whether first sentences should be removed (detected as header) */
+    removeFirstSentence: boolean;
+    /** Whether last sentences should be removed (detected as footer) */
+    removeLastSentence: boolean;
+    /** Median similarity of first sentences */
+    headerSimilarity: number;
+    /** Median similarity of last sentences */
+    footerSimilarity: number;
+}
+/**
+ * Detect header/footer patterns at sentence level
+ *
+ * Algorithm:
+ * 1. Sample pages from the CENTER of the document (guaranteed to be content pages)
+ * 2. Split each page into sentences with Y coordinate
+ * 3. Collect first/last sentences from sampled pages
+ * 4. Embed and calculate median pairwise similarity
+ * 5. If similarity > threshold, mark as header/footer
+ *
+ * Key insight: Middle pages are always content pages (cover, TOC, index are at edges).
+ * Using median instead of mean provides robustness against outliers.
+ *
+ * This approach handles variable content like page numbers ("7 of 75")
+ * by using semantic similarity instead of exact text matching.
+ *
+ * @param pages - Array of page data
+ * @param embedder - Embedder for generating embeddings
+ * @param config - Configuration options
+ * @returns Detection result
+ */
+export declare function detectSentencePatterns(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<SentencePatternResult>;
+/**
+ * Filter page boundary sentences and join into text
+ *
+ * This is the main entry point for sentence-level header/footer filtering.
+ * It detects and removes repeating sentence patterns at page boundaries.
+ *
+ * Use this instead of joinFilteredPages when embedder is available.
+ *
+ * @param pages - Array of page data
+ * @param embedder - Embedder for generating embeddings
+ * @param config - Configuration options
+ * @returns Filtered text with header/footer sentences removed
+ */
+export declare function filterPageBoundarySentences(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<string>;
+//# sourceMappingURL=pdf-filter.d.ts.map

package/dist/parser/pdf-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-filter.d.ts","sourceRoot":"","sources":["../../src/parser/pdf-filter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AAIvE,YAAY,EAAE,iBAAiB,EAAE,CAAA;AAMjC;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAA;IACZ,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,oBAAoB,EAAE,CAAA;CAC9B;AAoCD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAK3D;AA6JD;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,iEAAiE;IACjE,mBAAmB,EAAE,MAAM,CAAA;IAC3B,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAA;IAChB,+EAA+E;IAC/E,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,iEAAiE;AACjE,eAAO,MAAM,+BAA+B,EAAE,qBAI7C,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,qEAAqE;IACrE,mBAAmB,EAAE,OAAO,CAAA;IAC5B,oEAAoE;IACpE,kBAAkB,EAAE,OAAO,CAAA;IAC3B,2CAA2C;IAC3C,gBAAgB,EAAE,MAAM,CAAA;IACxB,0CAA0C;IAC1C,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,qBAAqB,CAAC,CAsEhC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,MAAM,CAAC,CAyCjB"}

package/dist/parser/pdf-filter.js

@@ -0,0 +1,304 @@
+"use strict";
+// PDF Header/Footer Filter
+// - Detects and removes repeating patterns across pages
+// - Semantic similarity-based header/footer detection (sentence-level)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_SENTENCE_PATTERN_CONFIG = void 0;
+exports.joinFilteredPages = joinFilteredPages;
+exports.detectSentencePatterns = detectSentencePatterns;
+exports.filterPageBoundarySentences = filterPageBoundarySentences;
+const sentence_splitter_js_1 = require("../chunker/sentence-splitter.js");
+// ============================================
+// Text Joining
+// ============================================
+/**
+ * Join page items into text
+ *
+ * Groups items by Y coordinate (same Y = same line),
+ * sorts each group by X coordinate (left to right),
+ * then joins groups with newlines (top to bottom).
+ */
+function joinPageItems(items) {
+    // Group by Y coordinate (rounded to handle minor variations)
+    const yGroups = new Map();
+    for (const item of items) {
+        const y = Math.round(item.y);
+        const group = yGroups.get(y) || [];
+        group.push(item);
+        yGroups.set(y, group);
+    }
+    // Sort groups by Y descending (top to bottom), items by X ascending (left to right)
+    return [...yGroups.entries()]
+        .sort((a, b) => b[0] - a[0])
+        .map(([_, group]) => group
+        .sort((a, b) => a.x - b.x)
+        .map((i) => i.text)
+        .join(' '))
+        .join('\n')
+        .trim();
+}
+/**
+ * Join filtered pages into text
+ *
+ * @param pages - Filtered page data
+ * @returns Joined text with proper line breaks
+ */
+function joinFilteredPages(pages) {
+    return pages
+        .map((page) => joinPageItems(page.items))
+        .filter((text) => text.length > 0)
+        .join('\n\n');
+}
+/**
+ * Split page items into sentences with Y coordinate
+ *
+ * 1. Join items into text (preserving item boundaries)
+ * 2. Split into sentences using splitIntoSentences
+ * 3. Map each sentence to the Y coordinate of its first item
+ * 4. Merge sentences with same Y coordinate
+ *
+ * @param items - Text items with position
+ * @returns Sentences with Y coordinate (merged by Y)
+ */
+function splitItemsIntoSentencesWithY(items) {
+    if (items.length === 0)
+        return [];
+    // Sort items by Y descending, then X ascending (reading order)
+    const sortedItems = [...items].sort((a, b) => {
+        const yDiff = b.y - a.y;
+        if (Math.abs(yDiff) > 1)
+            return yDiff;
+        return a.x - b.x;
+    });
+    // Build text and track character positions to item mapping
+    const charToItem = [];
+    let fullText = '';
+    let prevY = null;
+    for (const item of sortedItems) {
+        // Insert newline when Y coordinate changes (different line)
+        // This matches joinPageItems behavior: same Y = space, different Y = newline
+        if (prevY !== null && Math.abs(prevY - item.y) > 1) {
+            fullText = `${fullText.trimEnd()}\n`;
+        }
+        charToItem.push({ start: fullText.length, item });
+        fullText += `${item.text} `;
+        prevY = item.y;
+    }
+    // Split into sentences
+    const sentences = (0, sentence_splitter_js_1.splitIntoSentences)(fullText);
+    // Map each sentence to Y coordinate of its first character's item
+    const sentencesWithY = [];
+    let searchStart = 0;
+    for (const sentence of sentences) {
+        // Find where this sentence starts in fullText
+        const sentenceStart = fullText.indexOf(sentence.trim(), searchStart);
+        if (sentenceStart === -1)
+            continue;
+        // Find the item that contains this position
+        let firstItemY = sortedItems[0]?.y ?? 0;
+        for (let i = charToItem.length - 1; i >= 0; i--) {
+            const entry = charToItem[i];
+            if (entry && entry.start <= sentenceStart) {
+                firstItemY = Math.round(entry.item.y);
+                break;
+            }
+        }
+        sentencesWithY.push({ text: sentence, y: firstItemY });
+        searchStart = sentenceStart + sentence.length;
+    }
+    // Merge sentences with same Y coordinate
+    return mergeSentencesByY(sentencesWithY);
+}
+/**
+ * Merge sentences with same Y coordinate
+ *
+ * @param sentences - Sentences with Y coordinate
+ * @returns Merged sentences (same Y = one sentence)
+ */
+function mergeSentencesByY(sentences) {
+    if (sentences.length === 0)
+        return [];
+    const merged = [];
+    let current = null;
+    for (const sentence of sentences) {
+        if (current === null) {
+            current = { ...sentence };
+        }
+        else if (current.y === sentence.y) {
+            // Same Y: merge text
+            current.text += ` ${sentence.text}`;
+        }
+        else {
+            // Different Y: push current and start new
+            merged.push(current);
+            current = { ...sentence };
+        }
+    }
+    if (current !== null) {
+        merged.push(current);
+    }
+    return merged;
+}
+// ============================================
+// Sentence-Level Header/Footer Detection
+// ============================================
+// Use shared cosine similarity function
+const math_js_1 = require("../utils/math.js");
+/**
+ * Calculate median pairwise similarity for a list of embeddings
+ *
+ * Uses median instead of mean for robustness against outliers.
+ * This handles cases where some pages have different header content
+ * (e.g., chapter title changes) that would otherwise drag down the average.
+ */
+function medianPairwiseSimilarity(embeddings) {
+    if (embeddings.length < 2)
+        return 1.0;
+    const similarities = [];
+    for (let i = 0; i < embeddings.length; i++) {
+        for (let j = i + 1; j < embeddings.length; j++) {
+            const embI = embeddings[i];
+            const embJ = embeddings[j];
+            if (embI && embJ) {
+                similarities.push((0, math_js_1.cosineSimilarity)(embI, embJ));
+            }
+        }
+    }
+    if (similarities.length === 0)
+        return 0;
+    // Sort and find median
+    similarities.sort((a, b) => a - b);
+    const mid = Math.floor(similarities.length / 2);
+    if (similarities.length % 2 === 0) {
+        // Even: average of two middle values
+        return ((similarities[mid - 1] ?? 0) + (similarities[mid] ?? 0)) / 2;
+    }
+    // Odd: middle value
+    return similarities[mid] ?? 0;
+}
+/** Default configuration for sentence-level pattern detection */
+exports.DEFAULT_SENTENCE_PATTERN_CONFIG = {
+    similarityThreshold: 0.85,
+    minPages: 3,
+    samplePages: 5,
+};
+/**
+ * Detect header/footer patterns at sentence level
+ *
+ * Algorithm:
+ * 1. Sample pages from the CENTER of the document (guaranteed to be content pages)
+ * 2. Split each page into sentences with Y coordinate
+ * 3. Collect first/last sentences from sampled pages
+ * 4. Embed and calculate median pairwise similarity
+ * 5. If similarity > threshold, mark as header/footer
+ *
+ * Key insight: Middle pages are always content pages (cover, TOC, index are at edges).
+ * Using median instead of mean provides robustness against outliers.
+ *
+ * This approach handles variable content like page numbers ("7 of 75")
+ * by using semantic similarity instead of exact text matching.
+ *
+ * @param pages - Array of page data
+ * @param embedder - Embedder for generating embeddings
+ * @param config - Configuration options
+ * @returns Detection result
+ */
+async function detectSentencePatterns(pages, embedder, config = {}) {
+    const cfg = { ...exports.DEFAULT_SENTENCE_PATTERN_CONFIG, ...config };
+    const result = {
+        removeFirstSentence: false,
+        removeLastSentence: false,
+        headerSimilarity: 0,
+        footerSimilarity: 0,
+    };
+    // Need minimum pages to detect patterns reliably
+    if (pages.length < cfg.minPages) {
+        return result;
+    }
+    // 1. Sample pages from the CENTER of the document
+    // Middle pages are guaranteed to be content (not cover, TOC, or index)
+    const centerIndex = Math.floor(pages.length / 2);
+    const halfSample = Math.floor(cfg.samplePages / 2);
+    const startIndex = Math.max(0, centerIndex - halfSample);
+    const endIndex = Math.min(pages.length, startIndex + cfg.samplePages);
+    const samplePages = pages.slice(startIndex, endIndex);
+    // 2. Split each page into sentences with Y coordinate (merged by Y)
+    const pageSentences = samplePages.map((page) => splitItemsIntoSentencesWithY(page.items));
+    // 3. Collect first and last sentences from sampled pages
+    const firstSentences = [];
+    const lastSentences = [];
+    for (const sentences of pageSentences) {
+        if (sentences.length > 0) {
+            firstSentences.push(sentences[0].text);
+            if (sentences.length > 1) {
+                lastSentences.push(sentences[sentences.length - 1].text);
+            }
+        }
+    }
+    // 5. Detect header pattern (sampled first sentences are semantically similar)
+    if (firstSentences.length >= cfg.minPages) {
+        const embeddings = await embedder.embedBatch(firstSentences);
+        const medianSim = medianPairwiseSimilarity(embeddings);
+        result.headerSimilarity = medianSim;
+        if (medianSim >= cfg.similarityThreshold) {
+            result.removeFirstSentence = true;
+            console.error(`Sentence header detected: sampled ${firstSentences.length} center pages (${startIndex + 1}-${endIndex}), median similarity: ${medianSim.toFixed(3)}`);
+        }
+    }
+    // 6. Detect footer pattern (sampled last sentences are semantically similar)
+    if (lastSentences.length >= cfg.minPages) {
+        const embeddings = await embedder.embedBatch(lastSentences);
+        const medianSim = medianPairwiseSimilarity(embeddings);
+        result.footerSimilarity = medianSim;
+        if (medianSim >= cfg.similarityThreshold) {
+            result.removeLastSentence = true;
+            console.error(`Sentence footer detected: sampled ${lastSentences.length} center pages (${startIndex + 1}-${endIndex}), median similarity: ${medianSim.toFixed(3)}`);
+        }
+    }
+    return result;
+}
+/**
+ * Filter page boundary sentences and join into text
+ *
+ * This is the main entry point for sentence-level header/footer filtering.
+ * It detects and removes repeating sentence patterns at page boundaries.
+ *
+ * Use this instead of joinFilteredPages when embedder is available.
+ *
+ * @param pages - Array of page data
+ * @param embedder - Embedder for generating embeddings
+ * @param config - Configuration options
+ * @returns Filtered text with header/footer sentences removed
+ */
+async function filterPageBoundarySentences(pages, embedder, config = {}) {
+    const cfg = { ...exports.DEFAULT_SENTENCE_PATTERN_CONFIG, ...config };
+    // Need minimum pages to detect patterns
+    if (pages.length < cfg.minPages) {
+        return joinFilteredPages(pages);
+    }
+    // Detect patterns
+    const patterns = await detectSentencePatterns(pages, embedder, cfg);
+    // If no patterns detected, return normally joined text
+    if (!patterns.removeFirstSentence && !patterns.removeLastSentence) {
+        return joinFilteredPages(pages);
+    }
+    // Split each page into sentences with Y coordinate (merged by Y)
+    const pageSentences = pages.map((page) => splitItemsIntoSentencesWithY(page.items));
+    // Remove detected patterns from page sentences
+    const cleanedPageSentences = pageSentences.map((sentences) => {
+        let cleaned = [...sentences];
+        if (patterns.removeFirstSentence && cleaned.length > 0) {
+            cleaned = cleaned.slice(1);
+        }
+        if (patterns.removeLastSentence && cleaned.length > 0) {
+            cleaned = cleaned.slice(0, -1);
+        }
+        return cleaned;
+    });
+    // Join back into final text
+    return cleanedPageSentences
+        .map((sentences) => sentences.map((s) => s.text).join(' '))
+        .filter((text) => text.length > 0)
+        .join('\n\n');
+}
+//# sourceMappingURL=pdf-filter.js.map

package/dist/parser/pdf-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-filter.js","sourceRoot":"","sources":["../../src/parser/pdf-filter.ts"],"names":[],"mappings":";AAAA,2BAA2B;AAC3B,wDAAwD;AACxD,uEAAuE;;;AAuEvE,8CAKC;AAmND,wDA0EC;AAeD,kEA6CC;AAlaD,0EAAoE;AA4BpE,+CAA+C;AAC/C,eAAe;AACf,+CAA+C;AAE/C;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,KAA6B;IAClD,6DAA6D;IAC7D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkC,CAAA;IACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAClC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;IACvB,CAAC;IAED,oFAAoF;IACpF,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAClB,KAAK;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SAClB,IAAI,CAAC,GAAG,CAAC,CACb;SACA,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,EAAE,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,KAAiB;IACjD,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,MAAM,CAAC,CAAA;AACjB,CAAC;AAcD;;;;;;;;;;GAUG;AACH,SAAS,4BAA4B,CAAC,KAA6B;IACjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAEjC,+DAA+D;IAC/D,MAAM,WAAW,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC3C,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QACvB,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAA;QACrC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAClB,CAAC,CAAC,CAAA;IAEF,2DAA2D;IAC3D,MAAM,UAAU,GAAyD,EAAE,CAAA;IAC3E,IAAI,QAAQ,GAAG,EAAE,CAAA;IACjB,IAAI,KAAK,GAAkB,IAAI,CAAA;IAE/B,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,4DAA4D;QAC5D,6EAA6E;QAC7E,IAAI,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACnD,QAAQ,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAA;QACtC,CAAC;QAED,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACjD,QAAQ,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,CAAA;QAC3B,KAAK,GAAG,IAAI,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,uBAAuB;IACvB,MAAM,SAAS,GAAG,IAAA,yCAAkB,EAAC,QAAQ,CAAC,CAAA;IAE9C,kEAAkE;IAClE,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,8CAA8C;QAC9C,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,WAAW,CAAC,CAAA;QACpE,IAAI,aAAa,KAAK,CAAC,CAAC;YAAE,SAAQ;QAElC,4CAA4C;QAC5C,IAAI,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QACvC,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAC3B,IAAI,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;gBAC1C,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACrC,MAAK;YACP,CAAC;QACH,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAA;QACtD,WAAW,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC/C,CAAC;IAED,yCAAyC;IACzC,OAAO,iBAAiB,CAAC,cAAc,CAAC,CAAA;AAC1C,CAAC;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,SAA0B;IACnD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IAErC,MAAM,MAAM,GAAoB,EAAE,CAAA;IAClC,IAAI,OAAO,GAAyB,IAAI,CAAA;IAExC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;QAC3B,CAAC;aAAM,IAAI,OAAO,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,EAAE,CAAC;YACpC,qBAAqB;YACrB,OAAO,CAAC,IAAI,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAA;QACrC,CAAC;aAAM,CAAC;YACN,0CAA0C;YAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACpB,OAAO,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAA;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACtB,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+CAA+C;AAC/C,yCAAyC;AACzC,+CAA+C;AAE/C,wCAAwC;AACxC,8CAAmD;AAEnD;;;;;;GAMG;AACH,SAAS,wBAAwB,CAAC,UAAsB;IACtD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAA;IAErC,MAAM,YAAY,GAAa,EAAE,CAAA;IAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAC1B,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAC1B,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,YAAY,CAAC,IAAI,CAAC,IAAA,0BAAgB,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IAEvC,uBAAuB;IACvB,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE/C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAClC,qCAAqC;QACrC,OAAO,CAAC,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACtE,CAAC;IACD,oBAAoB;IACpB,OAAO,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAcD,iEAAiE;AACpD,QAAA,+BAA+B,GAA0B;IACpE,mBAAmB,EAAE,IAAI;IACzB,QAAQ,EAAE,CAAC;IACX,WAAW,EAAE,CAAC;CACf,CAAA;AAgBD;;;;;;;;;;;;;;;;;;;;GAoBG;AACI,KAAK,UAAU,sBAAsB,CAC1C,KAAiB,EACjB,QAA2B,EAC3B,SAAyC,EAAE;IAE3C,MAAM,GAAG,GAAG,EAAE,GAAG,uCAA+B,EAAE,GAAG,MAAM,EAAE,CAAA;IAE7D,MAAM,MAAM,GAA0B;QACpC,mBAAmB,EAAE,KAAK;QAC1B,kBAAkB,EAAE,KAAK;QACzB,gBAAgB,EAAE,CAAC;QACnB,gBAAgB,EAAE,CAAC;KACpB,CAAA;IAED,iDAAiD;IACjD,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,OAAO,MAAM,CAAA;IACf,CAAC;IAED,kDAAkD;IAClD,uEAAuE;IACvE,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAA;IAClD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,UAAU,CAAC,CAAA;IACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAA;IACrE,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;IAErD,oEAAoE;IACpE,MAAM,aAAa,GAAsB,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChE,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,CACzC,CAAA;IAED,yDAAyD;IACzD,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,MAAM,aAAa,GAAa,EAAE,CAAA;IAElC,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;QACtC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAA;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC,CAAA;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,IAAI,cAAc,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;QAC5D,MAAM,SAAS,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAA;QACtD,MAAM,CAAC,gBAAgB,GAAG,SAAS,CAAA;QAEnC,IAAI,SAAS,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;YACzC,MAAM,CAAC,mBAAmB,GAAG,IAAI,CAAA;YACjC,OAAO,CAAC,KAAK,CACX,qCAAqC,cAAc,CAAC,MAAM,kBAAkB,UAAU,GAAG,CAAC,IAAI,QAAQ,yBAAyB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACtJ,CAAA;QACH,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,IAAI,aAAa,CAAC,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAA;QAC3D,MAAM,SAAS,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAA;QACtD,MAAM,CAAC,gBAAgB,GAAG,SAAS,CAAA;QAEnC,IAAI,SAAS,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;YACzC,MAAM,CAAC,kBAAkB,GAAG,IAAI,CAAA;YAChC,OAAO,CAAC,KAAK,CACX,qCAAqC,aAAa,CAAC,MAAM,kBAAkB,UAAU,GAAG,CAAC,IAAI,QAAQ,yBAAyB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACrJ,CAAA;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;;;;;GAYG;AACI,KAAK,UAAU,2BAA2B,CAC/C,KAAiB,EACjB,QAA2B,EAC3B,SAAyC,EAAE;IAE3C,MAAM,GAAG,GAAG,EAAE,GAAG,uCAA+B,EAAE,GAAG,MAAM,EAAE,CAAA;IAE7D,wCAAwC;IACxC,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACjC,CAAC;IAED,kBAAkB;IAClB,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEnE,uDAAuD;IACvD,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;QAClE,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACjC,CAAC;IAED,iEAAiE;IACjE,MAAM,aAAa,GAAsB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC1D,4BAA4B,CAAC,IAAI,CAAC,KAAK,CAAC,CACzC,CAAA;IAED,+CAA+C;IAC/C,MAAM,oBAAoB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;QAC3D,IAAI,OAAO,GAAG,CAAC,GAAG,SAAS,CAAC,CAAA;QAE5B,IAAI,QAAQ,CAAC,mBAAmB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,QAAQ,CAAC,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAChC,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IAEF,4BAA4B;IAC5B,OAAO,oBAAoB;SACxB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAC1D,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;SACjC,IAAI,CAAC,MAAM,CAAC,CAAA;AACjB,CAAC"}

package/dist/server/index.d.ts

@@ -0,0 +1,144 @@
+import { type GroupingMode } from '../vectordb/index.js';
+import { type DeleteFileInput, type IngestDataInput, type IngestFileInput, type QueryDocumentsInput } from './schemas.js';
+/**
+ * RAGServer configuration
+ */
+export interface RAGServerConfig {
+    /** LanceDB database path */
+    dbPath: string;
+    /** Transformers.js model path */
+    modelName: string;
+    /** Model cache directory */
+    cacheDir: string;
+    /** Document base directory */
+    baseDir: string;
+    /** Maximum file size (100MB) */
+    maxFileSize: number;
+    /** Maximum distance threshold for quality filtering (optional) */
+    maxDistance?: number;
+    /** Grouping mode for quality filtering (optional) */
+    grouping?: GroupingMode;
+    /** Hybrid search weight for BM25 (0.0 = vector only, 1.0 = BM25 only, default 0.6) */
+    hybridWeight?: number;
+}
+/**
+ * RAG server compliant with MCP Protocol
+ *
+ * Responsibilities:
+ * - MCP tool integration (6 tools)
+ * - Tool handler implementation with Zod validation
+ * - Error handling
+ * - Initialization (LanceDB, Transformers.js)
+ */
+export declare class RAGServer {
+    private readonly server;
+    private readonly vectorStore;
+    private readonly embedder;
+    private readonly chunker;
+    private readonly parser;
+    private readonly dbPath;
+    constructor(config: RAGServerConfig);
+    /**
+     * Set up MCP handlers using tool() API
+     * Note: Type casts are used to work around Zod version compatibility between project and SDK
+     */
+    private setupHandlers;
+    /**
+     * Initialization
+     */
+    initialize(): Promise<void>;
+    /**
+     * Close the server and release resources
+     */
+    close(): Promise<void>;
+    /**
+     * Get the current database configuration
+     */
+    getConfig(): {
+        dbPath: string;
+        modelName: string;
+    };
+    /**
+     * Execute query_documents logic (returns plain data)
+     */
+    private executeQueryDocuments;
+    /**
+     * query_documents tool handler (for test compatibility)
+     */
+    handleQueryDocuments(args: QueryDocumentsInput): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * Execute ingest_file logic (returns plain data)
+     */
+    private executeIngestFile;
+    /**
+     * ingest_file tool handler (for test compatibility)
+     */
+    handleIngestFile(args: IngestFileInput): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * Execute ingest_data logic (returns plain data)
+     */
+    private executeIngestData;
+    /**
+     * ingest_data tool handler (for test compatibility)
+     */
+    handleIngestData(args: IngestDataInput): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * Execute list_files logic (returns plain data)
+     */
+    private executeListFiles;
+    /**
+     * list_files tool handler (for test compatibility)
+     */
+    handleListFiles(): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * Execute status logic (returns plain data)
+     */
+    private executeStatus;
+    /**
+     * status tool handler (for test compatibility)
+     */
+    handleStatus(): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * Execute delete_file logic (returns plain data)
+     */
+    private executeDeleteFile;
+    /**
+     * delete_file tool handler (for test compatibility)
+     */
+    handleDeleteFile(args: DeleteFileInput): Promise<{
+        content: [{
+            type: 'text';
+            text: string;
+        }];
+    }>;
+    /**
+     * Start the server
+     */
+    run(): Promise<void>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/server/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,YAAY,EAAiC,MAAM,sBAAsB,CAAA;AAQvF,OAAO,EACL,KAAK,eAAe,EAIpB,KAAK,eAAe,EACpB,KAAK,eAAe,EAEpB,KAAK,mBAAmB,EAGzB,MAAM,cAAc,CAAA;AAMrB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAA;IACnB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB,sFAAsF;IACtF,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAMD;;;;;;;;GAQG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAU;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;gBAEnB,MAAM,EAAE,eAAe;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAuGrB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAKjC;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B;;OAEG;IACH,SAAS,IAAI;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;IAOlD;;OAEG;YACW,qBAAqB;IA4BnC;;OAEG;IACG,oBAAoB,CACxB,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiBzD;;OAEG;YACW,iBAAiB;IA6G/B;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAkBzD;;OAEG;YACW,iBAAiB;IA6C/B;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAkBzD;;OAEG;YACW,gBAAgB;IAe9B;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB/E;;OAEG;YACW,aAAa;IAI3B;;OAEG;IACG,YAAY,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAiB5E;;OAEG;YACW,iBAAiB;IAyC/B;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,eAAe,GACpB,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAkBzD;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAK3B"}