@llamaindex/liteparse 1.0.0

package/dist/src/processing/bbox.d.ts

@@ -0,0 +1,20 @@
+import { TextItem, BoundingBox, ProjectionTextBox, LiteParseConfig } from "../core/types.js";
+import { PageData, Image } from "../engines/pdf/interface.js";
+/**
+ * Filters images that should not be OCR'd based on various criteria.
+ * Returns the filtered array of images that should be processed.
+ */
+export declare function filterImagesForOCR(images: Image[], page: {
+    width: number;
+    height: number;
+}): Image[];
+/**
+ * Build projection text boxes from page data, including OCR results
+ * This is the complete implementation from buildBbox.ts
+ */
+export declare function buildBbox(pageData: PageData, config: LiteParseConfig): ProjectionTextBox[];
+/**
+ * Build bounding boxes from text items
+ */
+export declare function buildBoundingBoxes(textItems: TextItem[]): BoundingBox[];
+//# sourceMappingURL=bbox.d.ts.map

package/dist/src/processing/bbox.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bbox.d.ts","sourceRoot":"","sources":["../../../src/processing/bbox.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,EACR,WAAW,EACX,iBAAiB,EAEjB,eAAe,EAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAC;AAgC9D;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,KAAK,EAAE,EACf,IAAI,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtC,KAAK,EAAE,CA2DT;AAsFD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,GAAG,iBAAiB,EAAE,CAmH1F;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,WAAW,EAAE,CAiBvE"}

package/dist/src/processing/bbox.js

@@ -0,0 +1,258 @@
+import { parseImageOcrBlocks } from "./ocrUtils.js";
+import { cleanOcrTableArtifacts } from "./textUtils.js";
+const OCR_CONFIDENCE_THRESHOLD = 0.1;
+/**
+ * Minimum overlap ratio (0-1) required to consider an OCR block as duplicate of existing text.
+ * An OCR block is filtered out if:
+ * - Total overlap with all text items covers more than this ratio of the OCR block area
+ * - OR the OCR block covers more than this ratio of any single text item
+ */
+const OCR_OVERLAP_THRESHOLD = 0.5;
+/**
+ * Maximum number of embedded images to process for OCR per page.
+ * Keeps the largest images when limit is exceeded.
+ */
+const MAX_IMAGES_PER_PAGE = 10;
+/**
+ * Minimum image dimensions for OCR processing
+ */
+const MIN_IMAGE_DIMENSION = 12;
+const MIN_IMAGE_AREA = 200;
+/**
+ * Minimum rendered image dimensions for OCR processing
+ */
+const MIN_RENDERED_DIMENSION = 6;
+const MIN_RENDERED_AREA = 200;
+/**
+ * Filters images that should not be OCR'd based on various criteria.
+ * Returns the filtered array of images that should be processed.
+ */
+export function filterImagesForOCR(images, page) {
+    // Filter images that start with g_ or pattern_ (generated/pattern images)
+    let filtered = images.filter((image) => !image.type?.startsWith("g_") && !image.type?.startsWith("pattern_"));
+    // Limit to max images per page, keeping the largest ones
+    if (filtered.length > MAX_IMAGES_PER_PAGE) {
+        filtered.sort((a, b) => b.width * b.height - a.width * a.height);
+        filtered = filtered.slice(0, MAX_IMAGES_PER_PAGE);
+    }
+    // Apply additional filtering criteria
+    filtered = filtered.filter((image) => {
+        // Ignore layout extracted images
+        if (image.type?.includes("layout_")) {
+            return false;
+        }
+        // Get image coords (use image dimensions if coords not set)
+        const coords = image.coords || {
+            x: image.x,
+            y: image.y,
+            w: image.width,
+            h: image.height,
+        };
+        // Skip images that are out of viewport
+        if (coords.x + coords.w < 0 || // left of page
+            coords.y + coords.h < 0 || // above page
+            coords.x > page.width || // right of page
+            coords.y > page.height // below page
+        ) {
+            return false;
+        }
+        // Skip small images (raw dimensions)
+        if (image.width < MIN_IMAGE_DIMENSION ||
+            image.height < MIN_IMAGE_DIMENSION ||
+            image.width * image.height < MIN_IMAGE_AREA) {
+            return false;
+        }
+        // Skip images that render too small in the viewport
+        if (coords.w < MIN_RENDERED_DIMENSION ||
+            coords.h < MIN_RENDERED_DIMENSION ||
+            coords.w * coords.h < MIN_RENDERED_AREA) {
+            return false;
+        }
+        return true;
+    });
+    return filtered;
+}
+/**
+ * Checks if two bounding boxes overlap and returns the overlap area.
+ */
+function getOverlapArea(box1, box2) {
+    const left = Math.max(box1.x, box2.x);
+    const right = Math.min(box1.x + box1.w, box2.x + box2.w);
+    const top = Math.max(box1.y, box2.y);
+    const bottom = Math.min(box1.y + box1.h, box2.y + box2.h);
+    if (left >= right || top >= bottom) {
+        return 0;
+    }
+    return (right - left) * (bottom - top);
+}
+/**
+ * Filters out OCR blocks that significantly overlap with already-extracted text items.
+ * This prevents duplicate text when both document text extraction and OCR detect the same content.
+ * Prefers document-extracted text over OCR text.
+ *
+ * An OCR block is rejected if:
+ * 1. The total overlap with all text items covers more than 50% of the OCR block area
+ * 2. OR the OCR block covers more than 50% of any single text item's area
+ */
+function filterOcrBlocksOverlappingWithText(ocrBlocks, textItems) {
+    if (!textItems.length || !ocrBlocks.length) {
+        return ocrBlocks;
+    }
+    return ocrBlocks.filter((ocrBlock) => {
+        const ocrBox = {
+            x: ocrBlock.x,
+            y: ocrBlock.y,
+            w: ocrBlock.w,
+            h: ocrBlock.h,
+        };
+        const ocrArea = ocrBlock.w * ocrBlock.h;
+        if (ocrArea <= 0) {
+            return false;
+        }
+        let totalOverlapArea = 0;
+        // Check overlap with each text item
+        for (const textItem of textItems) {
+            const textBox = {
+                x: textItem.x,
+                y: textItem.y,
+                w: textItem.w,
+                h: textItem.h,
+            };
+            const textItemArea = textItem.w * textItem.h;
+            const overlapArea = getOverlapArea(ocrBox, textBox);
+            if (overlapArea > 0) {
+                // Accumulate total overlap for condition 1
+                totalOverlapArea += overlapArea;
+                // Condition 2: Reject if OCR block covers more than 50% of any single text item
+                if (textItemArea > 0 && overlapArea / textItemArea >= OCR_OVERLAP_THRESHOLD) {
+                    return false;
+                }
+            }
+        }
+        // Condition 1: Reject if total overlap covers more than 50% of the OCR block
+        const totalOverlapRatio = totalOverlapArea / ocrArea;
+        if (totalOverlapRatio >= OCR_OVERLAP_THRESHOLD) {
+            return false;
+        }
+        return true;
+    });
+}
+/**
+ * Build projection text boxes from page data, including OCR results
+ * This is the complete implementation from buildBbox.ts
+ */
+export function buildBbox(pageData, config) {
+    const lines = [];
+    // Process all extracted text items
+    for (const item of pageData.textItems) {
+        const line = {
+            x: item.x,
+            y: item.y,
+            rx: item.rx || 0,
+            ry: item.ry || 0,
+            w: Math.round(item.w || item.width),
+            h: Math.round(item.h || item.height),
+            r: item.r || 0,
+            str: item.str,
+            strLength: [...item.str].length, // Handle multi-byte characters correctly
+            pageBbox: {
+                x: item.x,
+                y: item.y,
+                w: item.w || item.width,
+                h: item.h || item.height,
+            },
+            vgap: item.vgap,
+            isPlaceholder: item.isPlaceholder,
+        };
+        lines.push(line);
+    }
+    // Process OCR data if images are present
+    if (pageData.images.length && config.ocrEnabled) {
+        // Filter images that should be processed for OCR
+        const imagesToProcess = filterImagesForOCR(pageData.images, {
+            width: pageData.width,
+            height: pageData.height,
+        });
+        // Collect text item bounding boxes for overlap checking
+        const textItemBoxes = pageData.textItems.map((item) => ({
+            x: item.x,
+            y: item.y,
+            w: item.w || item.width,
+            h: item.h || item.height,
+        }));
+        // Collect text content for content-based deduplication (normalized lowercase)
+        const existingTextContent = new Set(pageData.textItems.map((item) => item.str.trim().toLowerCase()).filter((s) => s.length > 0));
+        for (const image of imagesToProcess) {
+            // Parse OCR blocks from image
+            let ocrData = parseImageOcrBlocks(image);
+            // Filter by confidence threshold
+            ocrData = ocrData.filter((block) => parseFloat(block.confidence.toString()) >= OCR_CONFIDENCE_THRESHOLD);
+            // Filter out OCR blocks that overlap with already-extracted text
+            ocrData = filterOcrBlocksOverlappingWithText(ocrData, textItemBoxes);
+            // Filter out OCR blocks whose text content already exists in native PDF text
+            // This catches duplicates that are at different positions (e.g., watermarks, repeated headers)
+            ocrData = ocrData.filter((block) => {
+                const ocrText = block.c.trim().toLowerCase();
+                return ocrText.length > 0 && !existingTextContent.has(ocrText);
+            });
+            const ocrParsed = [];
+            for (const block of ocrData) {
+                const confidenceRounded = Math.round(parseFloat(block.confidence.toString()) * 1000) / 1000;
+                // Clean OCR artifacts from table border misreads
+                const cleanedText = cleanOcrTableArtifacts(block.c);
+                // Skip if cleaning removed all content
+                if (cleanedText.length === 0) {
+                    continue;
+                }
+                const line = {
+                    fromOCR: true,
+                    x: block.x,
+                    y: block.y,
+                    w: block.w,
+                    h: block.h,
+                    r: image.originalOrientationAngle || 0,
+                    str: cleanedText,
+                    strLength: [...cleanedText].length,
+                    pageBbox: {
+                        x: block.x,
+                        y: block.y,
+                        w: block.w,
+                        h: block.h,
+                    },
+                };
+                lines.push(line);
+                ocrParsed.push({
+                    x: block.rx,
+                    y: block.ry,
+                    w: block.rw,
+                    h: block.rh,
+                    confidence: confidenceRounded,
+                    text: cleanedText,
+                });
+            }
+            if (ocrParsed.length) {
+                image.ocrParsed = ocrParsed;
+            }
+        }
+    }
+    return lines;
+}
+/**
+ * Build bounding boxes from text items
+ */
+export function buildBoundingBoxes(textItems) {
+    const bboxes = [];
+    for (const item of textItems) {
+        if (item.str.trim() === "") {
+            continue;
+        }
+        bboxes.push({
+            x1: item.x,
+            y1: item.y,
+            x2: item.x + (item.w || item.width),
+            y2: item.y + (item.h || item.height),
+        });
+    }
+    return bboxes;
+}
+//# sourceMappingURL=bbox.js.map

package/dist/src/processing/bbox.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bbox.js","sourceRoot":"","sources":["../../../src/processing/bbox.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,mBAAmB,EAAY,MAAM,eAAe,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAErC;;;;;GAKG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC;;;GAGG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B;;GAEG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;GAEG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAe,EACf,IAAuC;IAEvC,0EAA0E;IAC1E,IAAI,QAAQ,GAAG,MAAM,CAAC,MAAM,CAC1B,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,CAChF,CAAC;IAEF,yDAAyD;IACzD,IAAI,QAAQ,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QAC1C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QACjE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACpD,CAAC;IAED,sCAAsC;IACtC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QACnC,iCAAiC;QACjC,IAAI,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,4DAA4D;QAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI;YAC7B,CAAC,EAAE,KAAK,CAAC,CAAC;YACV,CAAC,EAAE,KAAK,CAAC,CAAC;YACV,CAAC,EAAE,KAAK,CAAC,KAAK;YACd,CAAC,EAAE,KAAK,CAAC,MAAM;SAChB,CAAC;QAEF,uCAAuC;QACvC,IACE,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,eAAe;YAC1C,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,aAAa;YACxC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,gBAAgB;YACzC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa;UACpC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,qCAAqC;QACrC,IACE,KAAK,CAAC,KAAK,GAAG,mBAAmB;YACjC,KAAK,CAAC,MAAM,GAAG,mBAAmB;YAClC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,cAAc,EAC3C,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,oDAAoD;QACpD,IACE,MAAM,CAAC,CAAC,GAAG,sBAAsB;YACjC,MAAM,CAAC,CAAC,GAAG,sBAAsB;YACjC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,iBAAiB,EACvC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,IAAoD,EACpD,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAE1D,IAAI,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC;QACnC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,kCAAkC,CACzC,SAAqB,EACrB,SAAgE;IAEhE,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QAC3C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE;QACnC,MAAM,MAAM,GAAG;YACb,CAAC,EAAE,QAAQ,CAAC,CAAC;YACb,CAAC,EAAE,QAAQ,CAAC,CAAC;YACb,CAAC,EAAE,QAAQ,CAAC,CAAC;YACb,CAAC,EAAE,QAAQ,CAAC,CAAC;SACd,CAAC;QACF,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;QAExC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,oCAAoC;QACpC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG;gBACd,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACb,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACb,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACb,CAAC,EAAE,QAAQ,CAAC,CAAC;aACd,CAAC;YACF,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;YAE7C,MAAM,WAAW,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAEpD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBACpB,2CAA2C;gBAC3C,gBAAgB,IAAI,WAAW,CAAC;gBAEhC,gFAAgF;gBAChF,IAAI,YAAY,GAAG,CAAC,IAAI,WAAW,GAAG,YAAY,IAAI,qBAAqB,EAAE,CAAC;oBAC5E,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;QACH,CAAC;QAED,6EAA6E;QAC7E,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,OAAO,CAAC;QACrD,IAAI,iBAAiB,IAAI,qBAAqB,EAAE,CAAC;YAC/C,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,QAAkB,EAAE,MAAuB;IACnE,MAAM,KAAK,GAAwB,EAAE,CAAC;IAEtC,mCAAmC;IACnC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QACtC,MAAM,IAAI,GAAsB;YAC9B,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;YAChB,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;YAChB,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC;YACnC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;YACpC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;YACd,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,SAAS,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,yCAAyC;YAC1E,QAAQ,EAAE;gBACR,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK;gBACvB,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;aACzB;YACD,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,yCAAyC;IACzC,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAChD,iDAAiD;QACjD,MAAM,eAAe,GAAG,kBAAkB,CAAC,QAAQ,CAAC,MAAM,EAAE;YAC1D,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;SACxB,CAAC,CAAC;QAEH,wDAAwD;QACxD,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACtD,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK;YACvB,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM;SACzB,CAAC,CAAC,CAAC;QAEJ,8EAA8E;QAC9E,MAAM,mBAAmB,GAAG,IAAI,GAAG,CACjC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC5F,CAAC;QAEF,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,8BAA8B;YAC9B,IAAI,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAEzC,iCAAiC;YACjC,OAAO,GAAG,OAAO,CAAC,MAAM,CACtB,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,IAAI,wBAAwB,CAC/E,CAAC;YAEF,iEAAiE;YACjE,OAAO,GAAG,kCAAkC,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YAErE,6EAA6E;YAC7E,+FAA+F;YAC/F,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAC7C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;YAEH,MAAM,SAAS,GAAc,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;gBAE5F,iDAAiD;gBACjD,MAAM,WAAW,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEpD,uCAAuC;gBACvC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,MAAM,IAAI,GAAsB;oBAC9B,OAAO,EAAE,IAAI;oBACb,CAAC,EAAE,KAAK,CAAC,CAAC;oBACV,CAAC,EAAE,KAAK,CAAC,CAAC;oBACV,CAAC,EAAE,KAAK,CAAC,CAAC;oBACV,CAAC,EAAE,KAAK,CAAC,CAAC;oBACV,CAAC,EAAE,KAAK,CAAC,wBAAwB,IAAI,CAAC;oBACtC,GAAG,EAAE,WAAW;oBAChB,SAAS,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM;oBAClC,QAAQ,EAAE;wBACR,CAAC,EAAE,KAAK,CAAC,CAAC;wBACV,CAAC,EAAE,KAAK,CAAC,CAAC;wBACV,CAAC,EAAE,KAAK,CAAC,CAAC;wBACV,CAAC,EAAE,KAAK,CAAC,CAAC;qBACX;iBACF,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEjB,SAAS,CAAC,IAAI,CAAC;oBACb,CAAC,EAAE,KAAK,CAAC,EAAE;oBACX,CAAC,EAAE,KAAK,CAAC,EAAE;oBACX,CAAC,EAAE,KAAK,CAAC,EAAE;oBACX,CAAC,EAAE,KAAK,CAAC,EAAE;oBACX,UAAU,EAAE,iBAAiB;oBAC7B,IAAI,EAAE,WAAW;iBAClB,CAAC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAqB;IACtD,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,CAAC;YACV,EAAE,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC;YACnC,EAAE,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/processing/bbox.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=bbox.test.d.ts.map

package/dist/src/processing/bbox.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bbox.test.d.ts","sourceRoot":"","sources":["../../../src/processing/bbox.test.ts"],"names":[],"mappings":""}

package/dist/src/processing/bbox.test.js

@@ -0,0 +1,334 @@
+import { expect, describe, it } from "vitest";
+import { buildBbox, buildBoundingBoxes, filterImagesForOCR } from "./bbox";
+import { DEFAULT_CONFIG } from "../core/config";
+describe("test filterImagesForOCR", () => {
+    it("test valid image", () => {
+        const images = [
+            {
+                type: "photo",
+                width: 200,
+                height: 200,
+                x: 10,
+                y: 10,
+                coords: { x: 10, y: 10, w: 200, h: 200 },
+            },
+        ];
+        const page = { width: 1000, height: 1000 };
+        const result = filterImagesForOCR(images, page);
+        expect(result).toStrictEqual(images);
+    });
+    it("test filter on patterns", () => {
+        const images = [
+            { type: "g_background", width: 200, height: 200, x: 0, y: 0 },
+            { type: "pattern_stripe", width: 200, height: 200, x: 0, y: 0 },
+            {
+                type: "photo",
+                width: 200,
+                height: 200,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 200, h: 200 },
+            },
+        ];
+        const page = { width: 1000, height: 1000 };
+        const result = filterImagesForOCR(images, page);
+        expect(result).toStrictEqual([images[2]]);
+    });
+    it("test filter layout", () => {
+        const images = [
+            {
+                type: "layout_header",
+                width: 300,
+                height: 300,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 300, h: 300 },
+            },
+        ];
+        const page = { width: 1000, height: 1000 };
+        const result = filterImagesForOCR(images, page);
+        expect(result.length).toBe(0);
+    });
+    it("test out of viewport", () => {
+        const images = [
+            {
+                type: "photo",
+                width: 200,
+                height: 200,
+                coords: { x: 1100, y: 0, w: 200, h: 200 },
+                x: 0,
+                y: 0,
+            },
+        ];
+        const page = { width: 1000, height: 1000 };
+        const result = filterImagesForOCR(images, page);
+        expect(result.length).toBe(0);
+    });
+    it("test min dimensions", () => {
+        const images = [
+            { type: "photo", width: 5, height: 5, x: 0, y: 0, coords: { x: 0, y: 0, w: 5, h: 5 } },
+        ];
+        const page = { width: 1000, height: 1000 };
+        const result = filterImagesForOCR(images, page);
+        expect(result.length).toBe(0);
+    });
+    it("test max images per page", () => {
+        const images = [
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 100, h: 100 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 500, h: 500 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 300, h: 300 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 100, h: 100 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 500, h: 500 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 300, h: 300 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 100, h: 100 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 500, h: 500 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 300, h: 300 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 100, h: 100 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 500, h: 500 },
+            },
+            {
+                type: "photo",
+                width: 100,
+                height: 100,
+                x: 0,
+                y: 0,
+                coords: { x: 0, y: 0, w: 300, h: 300 },
+            },
+        ];
+        const page = { width: 1000, height: 1000 };
+        const result = filterImagesForOCR(images, page);
+        expect(result).toStrictEqual(images.slice(0, 10));
+    });
+});
+describe("test buildBox", () => {
+    it("test with OCR disabled", () => {
+        const pageData = {
+            pageNum: 1,
+            width: 612,
+            height: 792,
+            textItems: [
+                { str: "Hello World", x: 50, y: 100, width: 120, height: 14, w: 120, h: 14 },
+                { str: "Some body text", x: 50, y: 130, width: 200, height: 14, w: 200, h: 14 },
+            ],
+            images: [],
+        };
+        const config = { ...DEFAULT_CONFIG, ocrEnabled: false };
+        const expectedOutput = [
+            {
+                x: 50,
+                y: 100,
+                rx: 0,
+                ry: 0,
+                w: 120,
+                h: 14,
+                r: 0,
+                str: "Hello World",
+                strLength: 11,
+                pageBbox: { x: 50, y: 100, w: 120, h: 14 },
+                vgap: undefined,
+                isPlaceholder: undefined,
+            },
+            {
+                x: 50,
+                y: 130,
+                rx: 0,
+                ry: 0,
+                w: 200,
+                h: 14,
+                r: 0,
+                str: "Some body text",
+                strLength: 14,
+                pageBbox: { x: 50, y: 130, w: 200, h: 14 },
+                vgap: undefined,
+                isPlaceholder: undefined,
+            },
+        ];
+        const result = buildBbox(pageData, config);
+        expect(result).toStrictEqual(expectedOutput);
+    });
+    it("test with OCR enabled", () => {
+        const pageData = {
+            pageNum: 1,
+            width: 612,
+            height: 792,
+            textItems: [
+                // Native PDF text (top-left)
+                { str: "Hello World", x: 50, y: 100, width: 120, height: 14, w: 120, h: 14 },
+            ],
+            images: [
+                {
+                    x: 0,
+                    y: 200,
+                    width: 612,
+                    height: 400,
+                    originalOrientationAngle: 0,
+                    // parseImageOcrBlocks() reads this internally:
+                    ocrRaw: [
+                        // Block A: no spatial overlap with native text, unique content → KEPT
+                        [
+                            [
+                                [50, 50],
+                                [250, 50],
+                                [250, 70],
+                                [50, 70],
+                            ],
+                            "Scanned paragraph text",
+                            0.95,
+                            // resolved by parseImageOcrBlocks to absolute page coords:
+                            // x:50, y:50, w:200, h:20, rx/ry/rw/rh for rotated coords
+                        ],
+                        // Block B: text already exists in native items ("hello world") → FILTERED (content dedup)
+                        [
+                            [
+                                [50, 0],
+                                [170, 0],
+                                [170, 14],
+                                [50, 14],
+                            ],
+                            "Hello World",
+                            0.97,
+                            // x:50, y:200, w:120, h:14 — also overlaps native text box
+                        ],
+                        // Block C: low confidence → FILTERED (below threshold)
+                        [
+                            [
+                                [300, 100],
+                                [500, 100],
+                                [500, 120],
+                                [300, 120],
+                            ],
+                            "Low confidence text",
+                            0.05,
+                        ],
+                    ],
+                },
+            ],
+        };
+        const config = { ...DEFAULT_CONFIG, ocrEnabled: true };
+        const expectedOutput = [
+            // ── Native text item ──────────────────────────────────────────────
+            {
+                x: 50,
+                y: 100,
+                rx: 0,
+                ry: 0,
+                w: 120,
+                h: 14,
+                r: 0,
+                str: "Hello World",
+                strLength: 11,
+                pageBbox: { x: 50, y: 100, w: 120, h: 14 },
+                vgap: undefined,
+                isPlaceholder: undefined,
+            },
+            // ── OCR block A (passed all filters) ─────────────────────────────
+            {
+                fromOCR: true,
+                x: 50,
+                y: 50,
+                w: 200,
+                h: 20,
+                r: 0,
+                str: "Scanned paragraph text",
+                strLength: 22,
+                pageBbox: { x: 50, y: 50, w: 200, h: 20 },
+            },
+            // Block B removed: spatial overlap >50% of native text item AND content dedup match
+            // Block C removed: confidence 0.40 < OCR_CONFIDENCE_THRESHOLD (0.5)
+        ];
+        const result = buildBbox(pageData, config);
+        expect(result).toStrictEqual(expectedOutput);
+    });
+});
+describe("test buildBoundingBoxes", () => {
+    it("test buildBoundingBoxes success", () => {
+        const textItems = [
+            { str: "Hello", x: 50, y: 100, width: 60, height: 14, w: 60, h: 14 },
+            { str: "   ", x: 50, y: 120, width: 30, height: 14, w: 30, h: 14 },
+            { str: "World", x: 50, y: 140, width: 80, height: 14, w: 80, h: 14 },
+        ];
+        const expectedOutput = [
+            { x1: 50, y1: 100, x2: 110, y2: 114 },
+            { x1: 50, y1: 140, x2: 130, y2: 154 },
+        ];
+        const result = buildBoundingBoxes(textItems);
+        expect(result).toStrictEqual(expectedOutput);
+    });
+});
+//# sourceMappingURL=bbox.test.js.map

package/dist/src/processing/bbox.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bbox.test.js","sourceRoot":"","sources":["../../../src/processing/bbox.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,QAAQ,CAAC;AAE3E,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,MAAM,GAAG;YACb;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,EAAE;gBACL,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACzC;SACF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG;YACb,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;YAC7D,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE;YAC/D;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;SACF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAC5B,MAAM,MAAM,GAAG;YACb;gBACE,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;SACF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,MAAM,GAAG;YACb;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,MAAM,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;gBACzC,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;aACL;SACF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC7B,MAAM,MAAM,GAAG;YACb,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;SACvF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,MAAM,GAAG;YACb;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;YACD;gBACE,IAAI,EAAE,OAAO;gBACb,KAAK,EAAE,GAAG;gBACV,MAAM,EAAE,GAAG;gBACX,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE;aACvC;SACF,CAAC;QACF,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,MAAM,QAAQ,GAAG;YACf,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,GAAG;YACX,SAAS,EAAE;gBACT,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;gBAC5E,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;aAChF;YACD,MAAM,EAAE,EAAE;SACX,CAAC;QACF,MAAM,MAAM,GAAoB,EAAE,GAAG,cAAc,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QAEzE,MAAM,cAAc,GAAG;YACrB;gBACE,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,GAAG;gBACN,EAAE,EAAE,CAAC;gBACL,EAAE,EAAE,CAAC;gBACL,CAAC,EAAE,GAAG;gBACN,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,CAAC;gBACJ,GAAG,EAAE,aAAa;gBAClB,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;gBAC1C,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,SAAS;aACzB;YACD;gBACE,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,GAAG;gBACN,EAAE,EAAE,CAAC;gBACL,EAAE,EAAE,CAAC;gBACL,CAAC,EAAE,GAAG;gBACN,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,CAAC;gBACJ,GAAG,EAAE,gBAAgB;gBACrB,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;gBAC1C,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,SAAS;aACzB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,QAAQ,GAAG;YACf,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,GAAG;YACX,SAAS,EAAE;gBACT,6BAA6B;gBAC7B,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;aAC7E;YACD,MAAM,EAAE;gBACN;oBACE,CAAC,EAAE,CAAC;oBACJ,CAAC,EAAE,GAAG;oBACN,KAAK,EAAE,GAAG;oBACV,MAAM,EAAE,GAAG;oBACX,wBAAwB,EAAE,CAAC;oBAC3B,+CAA+C;oBAC/C,MAAM,EAAE;wBACN,sEAAsE;wBACtE;4BACE;gCACE,CAAC,EAAE,EAAE,EAAE,CAAC;gCACR,CAAC,GAAG,EAAE,EAAE,CAAC;gCACT,CAAC,GAAG,EAAE,EAAE,CAAC;gCACT,CAAC,EAAE,EAAE,EAAE,CAAC;6BACT;4BACD,wBAAwB;4BACxB,IAAI;4BACJ,2DAA2D;4BAC3D,0DAA0D;yBACtC;wBACtB,0FAA0F;wBAC1F;4BACE;gCACE,CAAC,EAAE,EAAE,CAAC,CAAC;gCACP,CAAC,GAAG,EAAE,CAAC,CAAC;gCACR,CAAC,GAAG,EAAE,EAAE,CAAC;gCACT,CAAC,EAAE,EAAE,EAAE,CAAC;6BACT;4BACD,aAAa;4BACb,IAAI;4BACJ,2DAA2D;yBACvC;wBACtB,uDAAuD;wBACvD;4BACE;gCACE,CAAC,GAAG,EAAE,GAAG,CAAC;gCACV,CAAC,GAAG,EAAE,GAAG,CAAC;gCACV,CAAC,GAAG,EAAE,GAAG,CAAC;gCACV,CAAC,GAAG,EAAE,GAAG,CAAC;6BACX;4BACD,qBAAqB;4BACrB,IAAI;yBACgB;qBACvB;iBACF;aACF;SACF,CAAC;QACF,MAAM,MAAM,GAAoB,EAAE,GAAG,cAAc,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAExE,MAAM,cAAc,GAAG;YACrB,qEAAqE;YACrE;gBACE,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,GAAG;gBACN,EAAE,EAAE,CAAC;gBACL,EAAE,EAAE,CAAC;gBACL,CAAC,EAAE,GAAG;gBACN,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,CAAC;gBACJ,GAAG,EAAE,aAAa;gBAClB,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;gBAC1C,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,SAAS;aACzB;YAED,oEAAoE;YACpE;gBACE,OAAO,EAAE,IAAI;gBACb,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,GAAG;gBACN,CAAC,EAAE,EAAE;gBACL,CAAC,EAAE,CAAC;gBACJ,GAAG,EAAE,wBAAwB;gBAC7B,SAAS,EAAE,EAAE;gBACb,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;aAC1C;YAED,oFAAoF;YACpF,oEAAoE;SACrE,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,SAAS,GAAG;YAChB,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACpE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YAClE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACrE,CAAC;QAEF,MAAM,cAAc,GAAG;YACrB,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE;YACrC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE;SACtC,CAAC;QAEF,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/processing/cleanText.d.ts

@@ -0,0 +1,6 @@
+import { ParsedPage, LiteParseConfig } from "../core/types.js";
+/**
+ * Clean raw text output - removes margins, null characters
+ */
+export declare function cleanRawText(pages: ParsedPage[], _config: LiteParseConfig): void;
+//# sourceMappingURL=cleanText.d.ts.map

package/dist/src/processing/cleanText.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanText.d.ts","sourceRoot":"","sources":["../../../src/processing/cleanText.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AA2E/D;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,OAAO,EAAE,eAAe,GAAG,IAAI,CAShF"}