@llamaindex/liteparse 1.0.0

package/dist/src/processing/cleanText.js

@@ -0,0 +1,73 @@
+/**
+ * Detect and remove margins from a single page.
+ * Removes:
+ * - Left margin (consistent leading whitespace)
+ * - Top margin (empty lines at start)
+ * - Bottom margin (empty lines at end)
+ * - Right margin (trailing whitespace on each line)
+ */
+function detectAndRemoveMarginOnPage(page) {
+    const lines = page.text.split("\n");
+    let minX = undefined;
+    let minY = undefined;
+    let maxY = undefined;
+    // Find margins
+    for (let index = 0; index < lines.length; index++) {
+        const line = lines[index];
+        if (line.trim().length === 0) {
+            continue;
+        }
+        // Find first non-whitespace character position (left margin)
+        const x = line.search(/\S/);
+        if (minX === undefined || x < minX) {
+            minX = x;
+        }
+        // First non-empty line (top margin)
+        if (minY === undefined) {
+            minY = index;
+        }
+        // Last non-empty line (bottom margin)
+        maxY = index;
+    }
+    // If page is entirely empty, just return
+    if (minX === undefined || minY === undefined || maxY === undefined) {
+        page.text = "";
+        return;
+    }
+    // Remove margins
+    const newLines = [];
+    for (let index = 0; index < lines.length; index++) {
+        // Skip lines before first content (top margin) or after last content (bottom margin)
+        if (index < minY || index > maxY) {
+            continue;
+        }
+        let line = lines[index];
+        // Remove trailing whitespace (right margin)
+        line = line.trimEnd();
+        // Remove left margin
+        newLines.push(line.slice(minX));
+    }
+    page.text = newLines.join("\n");
+}
+/**
+ * Detect and remove margins from all pages.
+ * Processes each page independently.
+ */
+function detectAndRemoveMargin(pages) {
+    for (const page of pages) {
+        detectAndRemoveMarginOnPage(page);
+    }
+}
+/**
+ * Clean raw text output - removes margins, null characters
+ */
+export function cleanRawText(pages, _config) {
+    // Remove margins (per-page)
+    detectAndRemoveMargin(pages);
+    // Remove null characters
+    for (const page of pages) {
+        // eslint-disable-next-line no-control-regex
+        page.text = page.text.replace(/\u0000/g, " ");
+    }
+}
+//# sourceMappingURL=cleanText.js.map

package/dist/src/processing/cleanText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanText.js","sourceRoot":"","sources":["../../../src/processing/cleanText.ts"],"names":[],"mappings":"AAEA;;;;;;;GAOG;AACH,SAAS,2BAA2B,CAAC,IAAgB;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,IAAI,GAAuB,SAAS,CAAC;IACzC,IAAI,IAAI,GAAuB,SAAS,CAAC;IACzC,IAAI,IAAI,GAAuB,SAAS,CAAC;IAEzC,eAAe;IACf,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,6DAA6D;QAC7D,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;YACnC,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;QAED,sCAAsC;QACtC,IAAI,GAAG,KAAK,CAAC;IACf,CAAC;IAED,yCAAyC;IACzC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACnE,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,OAAO;IACT,CAAC;IAED,iBAAiB;IACjB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QAClD,qFAAqF;QACrF,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;YACjC,SAAS;QACX,CAAC;QAED,IAAI,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAExB,4CAA4C;QAC5C,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;QAEtB,qBAAqB;QACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,KAAmB;IAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,2BAA2B,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,OAAwB;IACxE,4BAA4B;IAC5B,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAE7B,yBAAyB;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,4CAA4C;QAC5C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;AACH,CAAC"}

package/dist/src/processing/cleanText.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cleanText.test.d.ts.map

package/dist/src/processing/cleanText.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanText.test.d.ts","sourceRoot":"","sources":["../../../src/processing/cleanText.test.ts"],"names":[],"mappings":""}

package/dist/src/processing/cleanText.test.js

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { cleanRawText } from "./cleanText.js";
+import { DEFAULT_CONFIG } from "../core/config.js";
+const mockPages = [
+    {
+        // Normal page with margins
+        pageNum: 1,
+        width: 612,
+        height: 792,
+        text: "   Hello World   \n   Foo Bar   \n",
+        textItems: [],
+    },
+    {
+        // Empty page
+        pageNum: 2,
+        width: 612,
+        height: 792,
+        text: "   \n   \n",
+        textItems: [],
+    },
+    {
+        // Single line
+        pageNum: 3,
+        width: 612,
+        height: 792,
+        text: "  Hello  ",
+        textItems: [],
+    },
+];
+const expectedTexts = [
+    // minX=3, minY=0, maxY=1 → slice(3) + trimEnd
+    "Hello World\nFoo Bar",
+    // entirely empty → ""
+    "",
+    // minX=2, minY=0, maxY=0 → slice(2) + trimEnd
+    "Hello",
+];
+describe("test cleanText", () => {
+    it("test cleanRawText", () => {
+        cleanRawText(mockPages, DEFAULT_CONFIG);
+        for (let i = 0; i < mockPages.length; i++) {
+            expect(mockPages[i].text).toBe(expectedTexts[i]);
+        }
+    });
+});
+//# sourceMappingURL=cleanText.test.js.map

package/dist/src/processing/cleanText.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanText.test.js","sourceRoot":"","sources":["../../../src/processing/cleanText.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,MAAM,SAAS,GAAiB;IAC9B;QACE,2BAA2B;QAC3B,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,GAAG;QACX,IAAI,EAAE,oCAAoC;QAC1C,SAAS,EAAE,EAAE;KACd;IACD;QACE,aAAa;QACb,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,GAAG;QACX,IAAI,EAAE,YAAY;QAClB,SAAS,EAAE,EAAE;KACd;IACD;QACE,cAAc;QACd,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,GAAG;QACX,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,EAAE;KACd;CACF,CAAC;AAEF,MAAM,aAAa,GAAa;IAC9B,8CAA8C;IAC9C,sBAAsB;IACtB,sBAAsB;IACtB,EAAE;IACF,8CAA8C;IAC9C,OAAO;CACR,CAAC;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,mBAAmB,EAAE,GAAG,EAAE;QAC3B,YAAY,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/processing/grid.d.ts

@@ -0,0 +1,7 @@
+import { PageData } from "../engines/pdf/interface.js";
+import { ParsedPage, LiteParseConfig } from "../core/types.js";
+/**
+ * Projects text items onto a grid for spatial text extraction
+ */
+export declare function projectPagesToGrid(pages: PageData[], config?: Partial<LiteParseConfig>): ParsedPage[];
+//# sourceMappingURL=grid.d.ts.map

package/dist/src/processing/grid.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grid.d.ts","sourceRoot":"","sources":["../../../src/processing/grid.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAI/D;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAChC,UAAU,EAAE,CAOd"}

package/dist/src/processing/grid.js

@@ -0,0 +1,13 @@
+import { projectPagesToGrid as projectPagesToGridComplete } from "./gridProjection.js";
+import { DEFAULT_CONFIG } from "../core/config.js";
+/**
+ * Projects text items onto a grid for spatial text extraction
+ */
+export function projectPagesToGrid(pages, config) {
+    const fullConfig = {
+        ...DEFAULT_CONFIG,
+        ...config,
+    };
+    return projectPagesToGridComplete(pages, fullConfig);
+}
+//# sourceMappingURL=grid.js.map

package/dist/src/processing/grid.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grid.js","sourceRoot":"","sources":["../../../src/processing/grid.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,IAAI,0BAA0B,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAiB,EACjB,MAAiC;IAEjC,MAAM,UAAU,GAAG;QACjB,GAAG,cAAc;QACjB,GAAG,MAAM;KACV,CAAC;IAEF,OAAO,0BAA0B,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;AACvD,CAAC"}

package/dist/src/processing/gridProjection.d.ts

@@ -0,0 +1,18 @@
+import { ProjectionTextBox, LiteParseConfig, ParsedPage } from "../core/types.js";
+import { PageData } from "../engines/pdf/interface.js";
+type ForwardAnchor = {
+    [key: string]: number;
+};
+type PrevAnchors = {
+    forwardAnchorLeft: ForwardAnchor;
+    forwardAnchorCenter: ForwardAnchor;
+    forwardAnchorRight: ForwardAnchor;
+};
+export declare function bboxToLine(textBbox: ProjectionTextBox[], medianWidth: number, medianHeight: number, pageWidth?: number): ProjectionTextBox[][];
+export declare function projectToGrid(config: LiteParseConfig, page: PageData, projectionBoxes: ProjectionTextBox[], prevAnchors: PrevAnchors, totalPages: number): {
+    text: string;
+    prevAnchors: PrevAnchors;
+};
+export declare function projectPagesToGrid(pages: PageData[], config: LiteParseConfig): ParsedPage[];
+export {};
+//# sourceMappingURL=gridProjection.d.ts.map

package/dist/src/processing/gridProjection.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gridProjection.d.ts","sourceRoot":"","sources":["../../../src/processing/gridProjection.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAe,eAAe,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAyBvD,KAAK,aAAa,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;CAAE,CAAC;AAE/C,KAAK,WAAW,GAAG;IACjB,iBAAiB,EAAE,aAAa,CAAC;IACjC,mBAAmB,EAAE,aAAa,CAAC;IACnC,kBAAkB,EAAE,aAAa,CAAC;CACnC,CAAC;AA0pBF,wBAAgB,UAAU,CACxB,QAAQ,EAAE,iBAAiB,EAAE,EAC7B,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,SAAS,CAAC,EAAE,MAAM,GACjB,iBAAiB,EAAE,EAAE,CAmSvB;AAwFD,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,QAAQ,EACd,eAAe,EAAE,iBAAiB,EAAE,EACpC,WAAW,EAAE,WAAW,EACxB,UAAU,EAAE,MAAM,GACjB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,WAAW,CAAA;CAAE,CAmjB5C;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,eAAe,GAAG,UAAU,EAAE,CAkD3F"}