abb-opencode-local-rag 0.1.0

package/dist/parser/pdf-extract.js

@@ -0,0 +1,112 @@
+// Shared per-page PDF extraction for the parser module.
+//
+// Lifts the per-page `toStructuredText` + header/footer-filtering loop out of
+// `DocumentParser` so both `parsePdf` and `parsePdfPages` consume one helper.
+// The two callers differ ONLY in the `stextOptions` they pass:
+//   - `parsePdf` passes `'preserve-whitespace'`;
+//   - `parsePdfPages` passes `'preserve-whitespace,preserve-images'` so mupdf
+//     emits `block.type === 'image'` entries for the downstream
+//     visual-candidate detector.
+import { filterPageBoundaryLayouts, } from './pdf-filter.js';
+/**
+ * Per-page extraction shared by `parsePdf` and `parsePdfPages`.
+ *
+ * Takes an already-open mupdf `Document` and:
+ *   - reads `info:Title` once,
+ *   - iterates pages calling `toStructuredText(stextOptions)`,
+ *   - builds `PageData` items (only `block.type === 'text'` lines),
+ *   - runs `filterPageBoundarySentences` to drop semantic headers/footers,
+ *   - derives `page1FontHint` from page 1's largest-font lines.
+ *
+ * The two callers differ ONLY in `stextOptions`: `parsePdf` passes
+ * `'preserve-whitespace'`;
+ * `parsePdfPages` passes `'preserve-whitespace,preserve-images'` so mupdf
+ * emits `block.type === 'image'` entries for the downstream visual-candidate
+ * detector.
+ *
+ * Lifecycle: this helper does NOT call `doc.destroy()` — disposal stays
+ * with the caller.
+ */
+export async function extractPdfPages(doc, embedder, stextOptions) {
+    const numPages = doc.countPages();
+    const metadataTitle = doc.getMetaData('info:Title') || undefined;
+    const pageDataList = [];
+    const stextJsonList = [];
+    for (let i = 0; i < numPages; i++) {
+        const page = doc.loadPage(i);
+        try {
+            const bounds = page.getBounds(); // [x0, y0, x1, y1]
+            const pageHeight = bounds[3] - bounds[1];
+            let json;
+            const stext = page.toStructuredText(stextOptions);
+            try {
+                json = JSON.parse(stext.asJSON());
+            }
+            finally {
+                stext.destroy?.();
+            }
+            const items = [];
+            for (let blockOrdinal = 0; blockOrdinal < json.blocks.length; blockOrdinal++) {
+                const block = json.blocks[blockOrdinal];
+                if (block?.type !== 'text' || !block.lines)
+                    continue;
+                for (let lineOrdinal = 0; lineOrdinal < block.lines.length; lineOrdinal++) {
+                    const line = block.lines[lineOrdinal];
+                    if (!line)
+                        continue;
+                    const bbox = line.bbox;
+                    items.push({
+                        text: line.text.replace(/\t/g, ' '),
+                        x: line.x,
+                        // Invert Y only for the legacy boundary detector; bbox remains in MuPDF coordinates.
+                        y: pageHeight - line.y,
+                        fontSize: line.font.size,
+                        hasEOL: true,
+                        ...(line.font.name !== undefined ? { fontName: line.font.name } : {}),
+                        ...(line.font.weight !== undefined ? { fontWeight: line.font.weight } : {}),
+                        blockOrdinal,
+                        lineOrdinal,
+                        bbox: bbox
+                            ? [bbox.x, bbox.y, bbox.x + bbox.w, bbox.y + bbox.h]
+                            : [line.x, line.y, line.x, line.y],
+                    });
+                }
+            }
+            pageDataList.push({
+                pageNum: i + 1,
+                items,
+                pageHeight,
+            });
+            stextJsonList.push(json);
+        }
+        finally {
+            page.destroy?.();
+        }
+    }
+    // Apply sentence-level header/footer filtering while retaining each survivor's layout data.
+    const filteredPages = await filterPageBoundaryLayouts(pageDataList, embedder);
+    // Extract largest-font lines from page 1 for title hint.
+    // Concatenate all consecutive lines with the largest font size (covers multi-line titles).
+    const page1Items = pageDataList[0]?.items ?? [];
+    const maxFontSize = page1Items.reduce((max, item) => Math.max(max, item.fontSize), 0);
+    const titleLines = [];
+    if (maxFontSize > 0) {
+        for (const item of page1Items) {
+            if (item.fontSize === maxFontSize) {
+                titleLines.push(item.text.trim());
+            }
+            else if (titleLines.length > 0) {
+                break;
+            }
+        }
+    }
+    const page1FontHint = titleLines.length > 0 ? { text: titleLines.join(' '), fontSize: maxFontSize } : undefined;
+    const pages = pageDataList.map((p, idx) => ({
+        pageNum: p.pageNum,
+        text: filteredPages[idx]?.text ?? '',
+        textFragments: filteredPages[idx]?.textFragments ?? [],
+        stextJson: stextJsonList[idx],
+    }));
+    return { pages, metadataTitle, page1FontHint };
+}
+//# sourceMappingURL=pdf-extract.js.map

package/dist/parser/pdf-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.js","sourceRoot":"","sources":["../../src/parser/pdf-extract.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,EAAE;AACF,8EAA8E;AAC9E,8EAA8E;AAC9E,+DAA+D;AAC/D,iDAAiD;AACjD,8EAA8E;AAC9E,gEAAgE;AAChE,iCAAiC;AAGjC,OAAO,EAGL,yBAAyB,GAE1B,MAAM,iBAAiB,CAAA;AAwDxB;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,GAAkB,EAClB,QAA2B,EAC3B,YAAoB;IAEpB,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IACjC,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,SAAS,CAAA;IAEhE,MAAM,YAAY,GAAe,EAAE,CAAA;IACnC,MAAM,aAAa,GAAgB,EAAE,CAAA;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA,CAAC,mBAAmB;YACnD,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YACxC,IAAI,IAAe,CAAA;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAA;YACjD,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAc,CAAA;YAChD,CAAC;oBAAS,CAAC;gBACT,KAAK,CAAC,OAAO,EAAE,EAAE,CAAA;YACnB,CAAC;YAED,MAAM,KAAK,GAAsB,EAAE,CAAA;YACnC,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,CAAC;gBAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;gBACvC,IAAI,KAAK,EAAE,IAAI,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK;oBAAE,SAAQ;gBACpD,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC;oBAC1E,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;oBACrC,IAAI,CAAC,IAAI;wBAAE,SAAQ;oBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;oBACtB,KAAK,CAAC,IAAI,CAAC;wBACT,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;wBACnC,CAAC,EAAE,IAAI,CAAC,CAAC;wBACT,qFAAqF;wBACrF,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,CAAC;wBACtB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI;wBACxB,MAAM,EAAE,IAAI;wBACZ,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACrE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC3E,YAAY;wBACZ,WAAW;wBACX,IAAI,EAAE,IAAI;4BACR,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;4BACpD,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;qBACrC,CAAC,CAAA;gBACJ,CAAC;YACH,CAAC;YAED,YAAY,CAAC,IAAI,CAAC;gBAChB,OAAO,EAAE,CAAC,GAAG,CAAC;gBACd,KAAK;gBACL,UAAU;aACX,CAAC,CAAA;YACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,EAAE,CAAA;QAClB,CAAC;IACH,CAAC;IAED,4FAA4F;IAC5F,MAAM,aAAa,GAAG,MAAM,yBAAyB,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE7E,yDAAyD;IACzD,2FAA2F;IAC3F,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,EAAE,CAAA;IAC/C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAA;IACrF,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAClC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YACnC,CAAC;iBAAM,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjC,MAAK;YACP,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,aAAa,GACjB,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAA;IAE3F,MAAM,KAAK,GAAoB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC3D,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,IAAI,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE;QACpC,aAAa,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,aAAa,IAAI,EAAE;QACtD,SAAS,EAAE,aAAa,CAAC,GAAG,CAAc;KAC3C,CAAC,CAAC,CAAA;IAEH,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,CAAA;AAChD,CAAC"}

package/dist/parser/pdf-filter.d.ts

@@ -0,0 +1,117 @@
+import type { EmbedderInterface } from '../chunker/semantic-chunker.js';
+export type { EmbedderInterface };
+/**
+ * Text item with position information from PDF
+ */
+interface TextItemWithPosition {
+    text: string;
+    x: number;
+    y: number;
+    fontSize: number;
+    hasEOL: boolean;
+    fontName?: string;
+    fontWeight?: string;
+    blockOrdinal?: number;
+    lineOrdinal?: number;
+    bbox?: [number, number, number, number];
+}
+/**
+ * Page data containing positioned text items
+ */
+export interface PageData {
+    pageNum: number;
+    items: TextItemWithPosition[];
+    pageHeight?: number;
+}
+export interface FilteredTextFragment {
+    pageNum: number;
+    blockOrdinal: number;
+    lineOrdinal: number;
+    fragmentOrdinal: number;
+    bbox: [number, number, number, number];
+    text: string;
+    pageTextStart: number;
+    pageTextEnd: number;
+}
+export interface FilteredPageLayout {
+    text: string;
+    textFragments: FilteredTextFragment[];
+}
+/**
+ * Join filtered pages into text
+ *
+ * @param pages - Filtered page data
+ * @returns Joined text with proper line breaks
+ */
+export declare function joinFilteredPages(pages: PageData[]): string;
+/**
+ * Configuration for sentence-level pattern detection
+ */
+interface SentencePatternConfig {
+    /** Similarity threshold for pattern detection (default: 0.85) */
+    similarityThreshold: number;
+    /** Minimum pages required for pattern detection (default: 3) */
+    minPages: number;
+    /** Number of pages to sample from center for pattern detection (default: 5) */
+    samplePages: number;
+    /** Block attribute hints for boosted threshold (from detectBlockAttributeCandidates) */
+    blockHints?: BlockAttributeHints;
+    /** Boosted similarity threshold when block hints match (default: 0.75) */
+    boostedThreshold?: number;
+}
+/**
+ * Hints for block-level header/footer detection based on font attributes
+ */
+export interface BlockAttributeHints {
+    medianFontSize: number;
+    headerCandidateYs: Set<number>;
+    footerCandidateYs: Set<number>;
+}
+/**
+ * Detect candidate header/footer lines based on font size and Y position
+ *
+ * Stage 1 of the 2-stage header/footer detection:
+ * 1. Sample center pages (same logic as detectSentencePatterns)
+ * 2. Calculate median font size from all items across sampled pages
+ * 3. Identify header candidates: fontSize < medianFontSize * 0.7 AND y > pageHeight * 0.9
+ * 4. Identify footer candidates: fontSize < medianFontSize * 0.7 AND y < pageHeight * 0.1
+ *
+ * @param pages - Array of page data
+ * @param config - Configuration options (minPages, samplePages)
+ * @returns Block attribute hints with candidate Y positions
+ */
+export declare function detectBlockAttributeCandidates(pages: PageData[], config?: Partial<Pick<SentencePatternConfig, 'minPages' | 'samplePages'>>): BlockAttributeHints;
+/**
+ * Result of sentence-level pattern detection
+ */
+interface SentencePatternResult {
+    /** Whether first sentences should be removed (detected as header) */
+    removeFirstSentence: boolean;
+    /** Whether last sentences should be removed (detected as footer) */
+    removeLastSentence: boolean;
+    /** Median similarity of first sentences */
+    headerSimilarity: number;
+    /** Median similarity of last sentences */
+    footerSimilarity: number;
+}
+export declare function detectSentencePatterns(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<SentencePatternResult>;
+/**
+ * Filter page boundary sentences and return per-page filtered text
+ *
+ * This is the main entry point for sentence-level header/footer filtering.
+ * It detects and removes repeating sentence patterns at page boundaries.
+ * Returns an array of filtered text per page, preserving page boundaries.
+ *
+ * Use this instead of joinFilteredPages when embedder is available.
+ *
+ * @param pages - Array of page data
+ * @param embedder - Embedder for generating embeddings
+ * @param config - Configuration options
+ * @returns Array of filtered text strings, one per page
+ */
+export declare function filterPageBoundarySentences(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<string[]>;
+/**
+ * Filter page boundaries while retaining native provenance for every survivor.
+ */
+export declare function filterPageBoundaryLayouts(pages: PageData[], embedder: EmbedderInterface, config?: Partial<SentencePatternConfig>): Promise<FilteredPageLayout[]>;
+//# sourceMappingURL=pdf-filter.d.ts.map

package/dist/parser/pdf-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-filter.d.ts","sourceRoot":"","sources":["../../src/parser/pdf-filter.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AAIvE,YAAY,EAAE,iBAAiB,EAAE,CAAA;AAMjC;;GAEG;AACH,UAAU,oBAAoB;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,oBAAoB,EAAE,CAAA;IAC7B,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,oBAAoB,EAAE,CAAA;CACtC;AA2HD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAK3D;AAuMD;;GAEG;AACH,UAAU,qBAAqB;IAC7B,iEAAiE;IACjE,mBAAmB,EAAE,MAAM,CAAA;IAC3B,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAA;IAChB,+EAA+E;IAC/E,WAAW,EAAE,MAAM,CAAA;IACnB,wFAAwF;IACxF,UAAU,CAAC,EAAE,mBAAmB,CAAA;IAChC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAcD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,MAAM,CAAA;IACtB,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IAC9B,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAC/B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,GAAE,OAAO,CAAC,IAAI,CAAC,qBAAqB,EAAE,UAAU,GAAG,aAAa,CAAC,CAAM,GAC5E,mBAAmB,CAqErB;AAED;;GAEG;AACH,UAAU,qBAAqB;IAC7B,qEAAqE;IACrE,mBAAmB,EAAE,OAAO,CAAA;IAC5B,oEAAoE;IACpE,kBAAkB,EAAE,OAAO,CAAA;IAC3B,2CAA2C;IAC3C,gBAAgB,EAAE,MAAM,CAAA;IACxB,0CAA0C;IAC1C,gBAAgB,EAAE,MAAM,CAAA;CACzB;AA2ED,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,qBAAqB,CAAC,CA+EhC;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,MAAM,EAAE,CAAC,CAEnB;AAED;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAC1C,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA8B/B"}