@ssmeo/mcp-local-rag 0.18.6

package/dist/parser/pdf-extract.js

@@ -0,0 +1,114 @@
+// Shared per-page PDF extraction for the parser module.
+//
+// Lifts the per-page `toStructuredText` + header/footer-filtering loop out of
+// `DocumentParser` so both `parsePdf` and `parsePdfPages` consume one helper.
+// The two callers differ ONLY in the `stextOptions` they pass:
+//   - `parsePdf` passes `'preserve-whitespace'`;
+//   - `parsePdfPages` passes `'preserve-whitespace,preserve-images'` so mupdf
+//     emits `block.type === 'image'` entries for the downstream
+//     visual-candidate detector.
+import { filterPageBoundaryLayouts, } from './pdf-filter.js';
+/**
+ * Per-page extraction shared by `parsePdf` and `parsePdfPages`, which differ
+ * only in `stextOptions`: `parsePdfPages` adds `preserve-images` so mupdf
+ * emits the image blocks the visual detector needs.
+ *
+ * Disposal of `doc` stays with the caller.
+ */
+/**
+ * Read one page's structured text, releasing the native handle either way.
+ *
+ * `asJSON()` is typed `string`, so this is where MuPDF's documented output
+ * contract meets {@link StextJson}. Dropping malformed elements instead would
+ * shift the `blockOrdinal`/`lineOrdinal` provenance recorded downstream.
+ *
+ * @see https://mupdf.readthedocs.io/en/1.28.0/reference/javascript/types/StructuredText.html
+ */
+function readPageStext(page, stextOptions) {
+    const stext = page.toStructuredText(stextOptions);
+    try {
+        // biome-ignore lint/nursery/noUnsafeTypeAssertion: connects MuPDF's documented asJSON() contract to StextJson
+        return JSON.parse(stext.asJSON());
+    }
+    finally {
+        stext.destroy();
+    }
+}
+/** One structured-text line as the layout filters consume it. */
+function toLineItem(line, pageHeight, blockOrdinal, lineOrdinal) {
+    const bbox = line.bbox;
+    return {
+        text: line.text.replace(/\t/g, ' '),
+        x: line.x,
+        // Invert Y only for the legacy boundary detector; bbox remains in MuPDF coordinates.
+        y: pageHeight - line.y,
+        fontSize: line.font.size,
+        hasEOL: true,
+        ...(line.font.name !== undefined ? { fontName: line.font.name } : {}),
+        ...(line.font.weight !== undefined ? { fontWeight: line.font.weight } : {}),
+        blockOrdinal,
+        lineOrdinal,
+        bbox: bbox
+            ? [bbox.x, bbox.y, bbox.x + bbox.w, bbox.y + bbox.h]
+            : [line.x, line.y, line.x, line.y],
+    };
+}
+function collectLineItems(json, pageHeight) {
+    const items = [];
+    json.blocks.forEach((block, blockOrdinal) => {
+        if (block.type !== 'text' || !block.lines) {
+            return;
+        }
+        block.lines.forEach((line, lineOrdinal) => {
+            items.push(toLineItem(line, pageHeight, blockOrdinal, lineOrdinal));
+        });
+    });
+    return items;
+}
+/** Concatenate page 1's consecutive largest-font lines as a title hint. */
+function largestFontTitleHint(page1Items) {
+    const maxFontSize = page1Items.reduce((max, item) => Math.max(max, item.fontSize), 0);
+    if (maxFontSize <= 0) {
+        return undefined;
+    }
+    const titleLines = [];
+    for (const item of page1Items) {
+        if (item.fontSize === maxFontSize) {
+            titleLines.push(item.text.trim());
+        }
+        else if (titleLines.length > 0) {
+            break;
+        }
+    }
+    return titleLines.length > 0 ? { text: titleLines.join(' '), fontSize: maxFontSize } : undefined;
+}
+export async function extractPdfPages(doc, embedder, stextOptions) {
+    const numPages = doc.countPages();
+    const metadataTitle = doc.getMetaData('info:Title') || undefined;
+    const pageDataList = [];
+    const stextJsonList = [];
+    for (let i = 0; i < numPages; i++) {
+        const page = doc.loadPage(i);
+        try {
+            const bounds = page.getBounds(); // [x0, y0, x1, y1]
+            const pageHeight = bounds[3] - bounds[1];
+            const json = readPageStext(page, stextOptions);
+            pageDataList.push({ pageNum: i + 1, items: collectLineItems(json, pageHeight), pageHeight });
+            stextJsonList.push(json);
+        }
+        finally {
+            page.destroy();
+        }
+    }
+    // Apply sentence-level header/footer filtering while retaining each survivor's layout data.
+    const filteredPages = await filterPageBoundaryLayouts(pageDataList, embedder);
+    const page1FontHint = largestFontTitleHint(pageDataList[0]?.items ?? []);
+    const pages = pageDataList.map((p, idx) => ({
+        pageNum: p.pageNum,
+        text: filteredPages[idx]?.text ?? '',
+        textFragments: filteredPages[idx]?.textFragments ?? [],
+        stextJson: stextJsonList[idx] ?? { blocks: [] },
+    }));
+    return { pages, metadataTitle, page1FontHint };
+}
+//# sourceMappingURL=pdf-extract.js.map

package/dist/parser/pdf-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.js","sourceRoot":"","sources":["../../src/parser/pdf-extract.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,EAAE;AACF,8EAA8E;AAC9E,8EAA8E;AAC9E,+DAA+D;AAC/D,iDAAiD;AACjD,8EAA8E;AAC9E,gEAAgE;AAChE,iCAAiC;AAIjC,OAAO,EAGL,yBAAyB,GAE1B,MAAM,iBAAiB,CAAA;AA2CxB;;;;;;GAMG;AACH;;;;;;;;GAQG;AACH,SAAS,aAAa,CAAC,IAAe,EAAE,YAAoB;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAA;IACjD,IAAI,CAAC;QACH,8GAA8G;QAC9G,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAc,CAAA;IAChD,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,OAAO,EAAE,CAAA;IACjB,CAAC;AACH,CAAC;AAKD,iEAAiE;AACjE,SAAS,UAAU,CACjB,IAAe,EACf,UAAkB,EAClB,YAAoB,EACpB,WAAmB;IAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;IACtB,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;QACnC,CAAC,EAAE,IAAI,CAAC,CAAC;QACT,qFAAqF;QACrF,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC;QACtB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;QACxB,MAAM,EAAE,IAAI;QACZ,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,YAAY;QACZ,WAAW;QACX,IAAI,EAAE,IAAI;YACR,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;KACrC,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAe,EAAE,UAAkB;IAC3D,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;QAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAC1C,OAAM;QACR,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE;YACxC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC;AAED,2EAA2E;AAC3E,SAAS,oBAAoB,CAC3B,UAA6B;IAE7B,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAA;IACrF,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;QACnC,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAK;QACP,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;AAClG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAkB,EAClB,QAA2B,EAC3B,YAAoB;IAEpB,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IACjC,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,SAAS,CAAA;IAEhE,MAAM,YAAY,GAAe,EAAE,CAAA;IACnC,MAAM,aAAa,GAAgB,EAAE,CAAA;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA,CAAC,mBAAmB;YACnD,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;YAC9C,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,UAAU,EAAE,CAAC,CAAA;YAC5F,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAA;QAChB,CAAC;IACH,CAAC;IAED,4FAA4F;IAC5F,MAAM,aAAa,GAAG,MAAM,yBAAyB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE7E,MAAM,aAAa,GAAG,oBAAoB,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC,CAAA;IAExE,MAAM,KAAK,GAAoB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC3D,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE;QACpC,aAAa,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,aAAa,IAAI,EAAE;QACtD,SAAS,EAAE,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE;KAChD,CAAC,CAAC,CAAA;IAEH,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,CAAA;AAChD,CAAC"}

package/dist/parser/pdf-filter.d.ts

@@ -0,0 +1,91 @@
+import type { EmbedderInterface } from '../chunker/semantic-chunker.js';
+export type { EmbedderInterface };
+/**
+ * Text item with position information from PDF
+ */
+interface TextItemWithPosition {
+    text: string;
+    x: number;
+    y: number;
+    fontSize: number;
+    hasEOL: boolean;
+    fontName?: string;
+    fontWeight?: string;
+    blockOrdinal?: number;
+    lineOrdinal?: number;
+    bbox?: [number, number, number, number];
+}
+/**
+ * Page data containing positioned text items
+ */
+export interface PageData {
+    pageNum: number;
+    items: TextItemWithPosition[];
+    pageHeight?: number;
+}
+export interface FilteredTextFragment {
+    pageNum: number;
+    blockOrdinal: number;
+    lineOrdinal: number;
+    fragmentOrdinal: number;
+    bbox: [number, number, number, number];
+    text: string;
+    pageTextStart: number;
+    pageTextEnd: number;
+}
+export interface FilteredPageLayout {
+    text: string;
+    textFragments: FilteredTextFragment[];
+}
+/** Join filtered pages into one text. */
+export declare function joinFilteredPages(pages: PageData[]): string;
+/**
+ * Configuration for sentence-level pattern detection
+ */
+interface SentencePatternConfig {
+    /** Similarity threshold for pattern detection (default: 0.85) */
+    similarityThreshold: number;
+    /** Minimum pages required for pattern detection (default: 3) */
+    minPages: number;
+    /** Number of pages to sample from center for pattern detection (default: 5) */
+    samplePages: number;
+    /** Block attribute hints for boosted threshold (from detectBlockAttributeCandidates) */
+    blockHints?: BlockAttributeHints;
+    /** Boosted similarity threshold when block hints match (default: 0.75) */
+    boostedThreshold?: number;
+}
+/**
+ * Hints for block-level header/footer detection based on font attributes
+ */
+export interface BlockAttributeHints {
+    medianFontSize: number;
+    headerCandidateYs: Set<number>;
+    footerCandidateYs: Set<number>;
+}
+export declare function detectBlockAttributeCandidates(pages: PageData[], config?: Partial<Pick<SentencePatternConfig, 'minPages' | 'samplePages'>>): BlockAttributeHints;
+/**
+ * Result of sentence-level pattern detection
+ */
+interface SentencePatternResult {
+    /** Whether first sentences should be removed (detected as header) */
+    removeFirstSentence: boolean;
+    /** Whether last sentences should be removed (detected as footer) */
+    removeLastSentence: boolean;
+    /** Median similarity of first sentences */
+    headerSimilarity: number;
+    /** Median similarity of last sentences */
+    footerSimilarity: number;
+}
+export declare function detectSentencePatterns(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<SentencePatternResult>;
+/**
+ * Main entry point for sentence-level header/footer filtering: removes
+ * repeating boundary sentences and returns one filtered text per page.
+ *
+ * Use this rather than {@link joinFilteredPages} when an embedder is available.
+ */
+export declare function filterPageBoundarySentences(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<string[]>;
+/**
+ * Filter page boundaries while retaining native provenance for every survivor.
+ */
+export declare function filterPageBoundaryLayouts(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<FilteredPageLayout[]>;
+//# sourceMappingURL=pdf-filter.d.ts.map

package/dist/parser/pdf-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-filter.d.ts","sourceRoot":"","sources":["../../src/parser/pdf-filter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AAIvE,YAAY,EAAE,iBAAiB,EAAE,CAAA;AAMjC;;GAEG;AACH,UAAU,oBAAoB;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,oBAAoB,EAAE,CAAA;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,oBAAoB,EAAE,CAAA;CACtC;AAwID,yCAAyC;AACzC,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAK3D;AA8MD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,iEAAiE;IACjE,mBAAmB,EAAE,MAAM,CAAA;IAC3B,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAA;IAChB,+EAA+E;IAC/E,WAAW,EAAE,MAAM,CAAA;IACnB,wFAAwF;IACxF,UAAU,CAAC,EAAE,mBAAmB,CAAA;IAChC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAcD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,MAAM,CAAA;IACtB,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC9B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAC/B;AAsED,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,GAAE,OAAO,CAAC,IAAI,CAAC,qBAAqB,EAAE,UAAU,GAAG,aAAa,CAAC,CAAM,GAC5E,mBAAmB,CA2BrB;AAED;;GAEG;AACH,UAAU,qBAAqB;IAC7B,qEAAqE;IACrE,mBAAmB,EAAE,OAAO,CAAA;IAC5B,oEAAoE;IACpE,kBAAkB,EAAE,OAAO,CAAA;IAC3B,2CAA2C;IAC3C,gBAAgB,EAAE,MAAM,CAAA;IACxB,0CAA0C;IAC1C,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAyDD,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,qBAAqB,CAAC,CAmFhC;AAED;;;;;GAKG;AACH,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,MAAM,EAAE,CAAC,CAEnB;AAED;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAwD/B"}