@llamaindex/liteparse 1.0.0

package/dist/src/processing/markupUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markupUtils.js","sourceRoot":"","sources":["../../../src/processing/markupUtils.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,MAAkB,EAAE,IAAY;IAC9D,IAAI,MAAM,GAAG,IAAI,CAAC;IAElB,6BAA6B;IAC7B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,GAAG,KAAK,MAAM,IAAI,CAAC;IAC3B,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,GAAG,KAAK,MAAM,IAAI,CAAC;IAC3B,CAAC;IAED,yDAAyD;IACzD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,MAAM,GAAG,KAAK,MAAM,IAAI,CAAC;IAC3B,CAAC;IAED,6BAA6B;IAC7B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,MAAM,GAAG,KAAK,MAAM,IAAI,CAAC;IAC3B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/processing/markupUtils.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=markupUtils.test.d.ts.map

package/dist/src/processing/markupUtils.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markupUtils.test.d.ts","sourceRoot":"","sources":["../../../src/processing/markupUtils.test.ts"],"names":[],"mappings":""}

package/dist/src/processing/markupUtils.test.js

@@ -0,0 +1,26 @@
+import { describe, expect, it } from "vitest";
+import { applyMarkupTags } from "./markupUtils.js";
+const text = "hello";
+describe("test markupUtils", () => {
+    it("test strikeout", () => {
+        const data = { strikeout: true };
+        expect(applyMarkupTags(data, text)).toBe(`~~${text}~~`);
+    });
+    it("test underline", () => {
+        const data = { underline: true };
+        expect(applyMarkupTags(data, text)).toBe(`__${text}__`);
+    });
+    it("test squiggly", () => {
+        const data = { squiggly: true };
+        expect(applyMarkupTags(data, text)).toBe(`__${text}__`);
+    });
+    it("test highlight", () => {
+        const data = { highlight: "yes" };
+        expect(applyMarkupTags(data, text)).toBe(`==${text}==`);
+    });
+    it("test none", () => {
+        const data = {};
+        expect(applyMarkupTags(data, text)).toBe(text);
+    });
+});
+//# sourceMappingURL=markupUtils.test.js.map

package/dist/src/processing/markupUtils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markupUtils.test.js","sourceRoot":"","sources":["../../../src/processing/markupUtils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,IAAI,GAAG,OAAO,CAAC;AAErB,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,gBAAgB,EAAE,GAAG,EAAE;QACxB,MAAM,IAAI,GAAe,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC7C,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gBAAgB,EAAE,GAAG,EAAE;QACxB,MAAM,IAAI,GAAe,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC7C,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,eAAe,EAAE,GAAG,EAAE;QACvB,MAAM,IAAI,GAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gBAAgB,EAAE,GAAG,EAAE;QACxB,MAAM,IAAI,GAAe,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC9C,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,WAAW,EAAE,GAAG,EAAE;QACnB,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,MAAM,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/processing/ocrUtils.d.ts

@@ -0,0 +1,24 @@
+import { Image, EasyOcrResultLine } from "../engines/pdf/interface.js";
+export interface OcrBlock {
+    c: string;
+    x: number;
+    rx: number;
+    y: number;
+    ry: number;
+    w: number;
+    rw: number;
+    h: number;
+    rh: number;
+    confidence: string | number;
+    fromOcr: boolean;
+}
+/**
+ * Parse OCR blocks from image with OCR data
+ * Converts OCR bounding boxes from image space to page space
+ */
+export declare function parseImageOcrBlocks(image: Image): OcrBlock[];
+/**
+ * Parse EasyOCR stdout result into structured format
+ */
+export declare function easyOcrResultLinesToList(stdOutResult?: string): EasyOcrResultLine[];
+//# sourceMappingURL=ocrUtils.d.ts.map

package/dist/src/processing/ocrUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocrUtils.d.ts","sourceRoot":"","sources":["../../../src/processing/ocrUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAEvE,MAAM,WAAW,QAAQ;IACvB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,EAAE,EAAE,MAAM,CAAC;IACX,CAAC,EAAE,MAAM,CAAC;IACV,EAAE,EAAE,MAAM,CAAC;IACX,CAAC,EAAE,MAAM,CAAC;IACV,EAAE,EAAE,MAAM,CAAC;IACX,CAAC,EAAE,MAAM,CAAC;IACV,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,KAAK,GAAG,QAAQ,EAAE,CAqC5D;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,iBAAiB,EAAE,CA8CnF"}

package/dist/src/processing/ocrUtils.js

@@ -0,0 +1,79 @@
+/**
+ * Parse OCR blocks from image with OCR data
+ * Converts OCR bounding boxes from image space to page space
+ */
+export function parseImageOcrBlocks(image) {
+    if (!image || !image.ocrRaw?.length) {
+        return [];
+    }
+    const blocks = [];
+    // Calculate ratio to convert from OCR image coordinates to page coordinates
+    const coords = image.coords || { x: 0, y: 0, w: image.width, h: image.height };
+    const xRatio = image.width / coords.w;
+    const yRatio = image.height / coords.h;
+    for (const line of image.ocrRaw) {
+        // line format: [[[x1, y1], [x2, y2], [x3, y3], [x4, y4]], 'text', confidence]
+        const [x1, y1] = line[0][0];
+        const [x2, y2] = line[0][2]; // Use opposite corner for bbox
+        const text = line[1];
+        const confidence = line[2];
+        // Convert the bounding box to page space
+        const block = {
+            c: text,
+            x: x1 / xRatio + coords.x,
+            rx: Math.round(x1 / (image.scaleFactor || 1)),
+            y: y1 / yRatio + coords.y,
+            ry: Math.round(y1 / (image.scaleFactor || 1)),
+            w: Math.abs(x2 - x1) / xRatio,
+            rw: Math.round(Math.abs(x2 - x1) / (image.scaleFactor || 1)),
+            h: Math.abs(y2 - y1) / yRatio,
+            rh: Math.round(Math.abs(y2 - y1) / (image.scaleFactor || 1)),
+            confidence,
+            fromOcr: true,
+        };
+        blocks.push(block);
+    }
+    return blocks;
+}
+/**
+ * Parse EasyOCR stdout result into structured format
+ */
+export function easyOcrResultLinesToList(stdOutResult) {
+    if (!stdOutResult?.length) {
+        return [];
+    }
+    const lines = stdOutResult.split("\n");
+    const blocks = [];
+    for (const line of lines) {
+        if (!line.trim()) {
+            continue;
+        }
+        // Parse line format: ([[x1, y1], [x2, y2], [x3, y3], [x4, y4]], 'text', confidence)
+        const ocrMatch = line.match(/\[\[(.*?), (.*?)\], \[(.*?), (.*?)\], \[(.*?), (.*?)\], \[(.*?), (.*?)\]\], ['"](.*?)['"], (.*?)\)$/);
+        if (!ocrMatch) {
+            continue;
+        }
+        const x1 = Number(ocrMatch[1]);
+        const y1 = Number(ocrMatch[2]);
+        const x2 = Number(ocrMatch[3]);
+        const y2 = Number(ocrMatch[4]);
+        const x3 = Number(ocrMatch[5]);
+        const y3 = Number(ocrMatch[6]);
+        const x4 = Number(ocrMatch[7]);
+        const y4 = Number(ocrMatch[8]);
+        const text = ocrMatch[9];
+        const confidence = ocrMatch[10];
+        blocks.push([
+            [
+                [x1, y1],
+                [x2, y2],
+                [x3, y3],
+                [x4, y4],
+            ],
+            text,
+            confidence,
+        ]);
+    }
+    return blocks;
+}
+//# sourceMappingURL=ocrUtils.js.map

package/dist/src/processing/ocrUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocrUtils.js","sourceRoot":"","sources":["../../../src/processing/ocrUtils.ts"],"names":[],"mappings":"AAgBA;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAY;IAC9C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QACpC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAe,EAAE,CAAC;IAE9B,4EAA4E;IAC5E,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;IAC/E,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAChC,8EAA8E;QAC9E,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,+BAA+B;QAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAE3B,yCAAyC;QACzC,MAAM,KAAK,GAAa;YACtB,CAAC,EAAE,IAAI;YACP,CAAC,EAAE,EAAE,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;YACzB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;YAC7C,CAAC,EAAE,EAAE,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;YACzB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;YAC7C,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,MAAM;YAC7B,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;YAC5D,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,MAAM;YAC7B,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;YAC5D,UAAU;YACV,OAAO,EAAE,IAAI;SACd,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,YAAqB;IAC5D,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAwB,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,oFAAoF;QACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,qGAAqG,CACtG,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,UAAU,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEhC,MAAM,CAAC,IAAI,CAAC;YACV;gBACE,CAAC,EAAE,EAAE,EAAE,CAAC;gBACR,CAAC,EAAE,EAAE,EAAE,CAAC;gBACR,CAAC,EAAE,EAAE,EAAE,CAAC;gBACR,CAAC,EAAE,EAAE,EAAE,CAAC;aACT;YACD,IAAI;YACJ,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/processing/octUtils.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=octUtils.test.d.ts.map

package/dist/src/processing/octUtils.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"octUtils.test.d.ts","sourceRoot":"","sources":["../../../src/processing/octUtils.test.ts"],"names":[],"mappings":""}

package/dist/src/processing/octUtils.test.js

@@ -0,0 +1,72 @@
+import { describe, expect, it } from "vitest";
+import { parseImageOcrBlocks, easyOcrResultLinesToList } from "./ocrUtils";
+const mockImage = {
+    x: 0,
+    y: 0,
+    width: 200,
+    height: 100,
+    scaleFactor: 2,
+    coords: { x: 10, y: 20, w: 100, h: 50 },
+    ocrRaw: [
+        [
+            [
+                [0, 0],
+                [40, 0],
+                [40, 20],
+                [0, 20],
+            ],
+            "Hello",
+            0.95,
+        ],
+    ],
+};
+const expectedBlock = {
+    c: "Hello",
+    x: 0 / 2 + 10, // = 10
+    rx: Math.round(0 / 2), // = 0
+    y: 0 / 2 + 20, // = 20
+    ry: Math.round(0 / 2), // = 0
+    w: 40 / 2, // = 20
+    rw: Math.round(40 / 2), // = 20
+    h: 20 / 2, // = 10
+    rh: Math.round(20 / 2), // = 10
+    confidence: 0.95,
+    fromOcr: true,
+};
+const mockStdOut = "([[0, 10], [40, 10], [40, 30], [0, 30]], 'Hello', 0.95)";
+const expectedResult = [
+    [
+        [
+            [0, 10],
+            [40, 10],
+            [40, 30],
+            [0, 30],
+        ],
+        "Hello",
+        "0.95", // note: string, since ocrMatch[10] is not parsed with Number()
+    ],
+];
+describe("test ocrUtils", () => {
+    it("test parseImageOcrBlocks success", () => {
+        const result = parseImageOcrBlocks(mockImage);
+        expect(result.length).toBe(1);
+        expect(result[0]).toStrictEqual(expectedBlock);
+    });
+    it("test parseImageOcrBlocks failure", () => {
+        const imageNoRawOcr = {
+            ...mockImage,
+            ocrRaw: [],
+        };
+        const result = parseImageOcrBlocks(imageNoRawOcr);
+        expect(result.length).toBe(0);
+    });
+    it("test EasyOcrResultLine success", () => {
+        const result = easyOcrResultLinesToList(mockStdOut);
+        expect(result).toStrictEqual(expectedResult);
+    });
+    it("test EasyOcrResultLine success", () => {
+        const result = easyOcrResultLinesToList("");
+        expect(result.length).toBe(0);
+    });
+});
+//# sourceMappingURL=octUtils.test.js.map

package/dist/src/processing/octUtils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"octUtils.test.js","sourceRoot":"","sources":["../../../src/processing/octUtils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAG3E,MAAM,SAAS,GAAU;IACvB,CAAC,EAAE,CAAC;IACJ,CAAC,EAAE,CAAC;IACJ,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,WAAW,EAAE,CAAC;IACd,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE;IACvC,MAAM,EAAE;QACN;YACE;gBACE,CAAC,CAAC,EAAE,CAAC,CAAC;gBACN,CAAC,EAAE,EAAE,CAAC,CAAC;gBACP,CAAC,EAAE,EAAE,EAAE,CAAC;gBACR,CAAC,CAAC,EAAE,EAAE,CAAC;aACR;YACD,OAAO;YACP,IAAI;SACgB;KACvB;CACF,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,CAAC,EAAE,OAAO;IACV,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO;IACtB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM;IAC7B,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,OAAO;IACtB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM;IAC7B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,OAAO;IAClB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO;IAC/B,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,OAAO;IAClB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,OAAO;IAC/B,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE,IAAI;CACd,CAAC;AAEF,MAAM,UAAU,GAAG,yDAAyD,CAAC;AAE7E,MAAM,cAAc,GAAwB;IAC1C;QACE;YACE,CAAC,CAAC,EAAE,EAAE,CAAC;YACP,CAAC,EAAE,EAAE,EAAE,CAAC;YACR,CAAC,EAAE,EAAE,EAAE,CAAC;YACR,CAAC,CAAC,EAAE,EAAE,CAAC;SACR;QACD,OAAO;QACP,MAAM,EAAE,+DAA+D;KACnD;CACvB,CAAC;AAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,aAAa,GAAG;YACpB,GAAG,SAAS;YACZ,MAAM,EAAE,EAAE;SACX,CAAC;QACF,MAAM,MAAM,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,MAAM,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,MAAM,GAAG,wBAAwB,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/processing/textUtils.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Clean common OCR artifacts from table documents.
+ * OCR often misreads vertical table border lines as bracket-like characters.
+ * This is especially common with numbers adjacent to table cell borders.
+ *
+ * Examples:
+ * - "44520]" → "44,520" (vertical line misread as ])
+ * - "|123" → "123" (vertical line misread as |)
+ * - "0.3|" → "0.3" (vertical line misread as |)
+ */
+export declare function cleanOcrTableArtifacts(text: string): string;
+/**
+ * Convert string to subscript unicode characters
+ */
+export declare function strToSubscriptString(str: string): string;
+/**
+ * Convert string to superscript unicode characters
+ */
+export declare function strToPostScript(str: string): string;
+//# sourceMappingURL=textUtils.d.ts.map

package/dist/src/processing/textUtils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"textUtils.d.ts","sourceRoot":"","sources":["../../../src/processing/textUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAsB3D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAuCxD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAqEnD"}

package/dist/src/processing/textUtils.js

@@ -0,0 +1,142 @@
+/**
+ * Clean common OCR artifacts from table documents.
+ * OCR often misreads vertical table border lines as bracket-like characters.
+ * This is especially common with numbers adjacent to table cell borders.
+ *
+ * Examples:
+ * - "44520]" → "44,520" (vertical line misread as ])
+ * - "|123" → "123" (vertical line misread as |)
+ * - "0.3|" → "0.3" (vertical line misread as |)
+ */
+export function cleanOcrTableArtifacts(text) {
+    // Characters commonly misread from vertical table borders
+    // These typically appear at the start or end of cell content
+    const borderArtifacts = /^[|[\](){}]+|[|[\](){}]+$/g;
+    const cleaned = text.trim();
+    // Only clean if the core content looks like a number or short text
+    // This avoids incorrectly stripping brackets from actual content like "(note)"
+    const withoutArtifacts = cleaned.replace(borderArtifacts, "");
+    // If removing artifacts leaves us with something that looks like a number,
+    // statistical value, or percentage, use the cleaned version
+    if (withoutArtifacts.length > 0) {
+        // Check if core content is numeric-ish (numbers, commas, periods, asterisks, percent, minus, plus, Z, N/A)
+        const numericPattern = /^[*+-]?[\d,.\s]+[%]?$|^[*]?-?[\d,.\s]+$|^[ZN]\/A$|^[Z-]$/;
+        if (numericPattern.test(withoutArtifacts.trim())) {
+            return withoutArtifacts.trim();
+        }
+    }
+    return cleaned;
+}
+/**
+ * Convert string to subscript unicode characters
+ */
+export function strToSubscriptString(str) {
+    const sub = {
+        "0": "₀",
+        "1": "₁",
+        "2": "₂",
+        "3": "₃",
+        "4": "₄",
+        "5": "₅",
+        "6": "₆",
+        "7": "₇",
+        "8": "₈",
+        "9": "₉",
+        "+": "₊",
+        "-": "₋",
+        a: "ₐ",
+        e: "ₑ",
+        o: "ₒ",
+        x: "ₓ",
+        ə: "ₔ",
+        h: "ₕ",
+        k: "ₖ",
+        l: "ₗ",
+        m: "ₘ",
+        n: "ₙ",
+        p: "ₚ",
+        r: "ᵣ",
+        s: "ₛ",
+        t: "ₜ",
+    };
+    let subStr = "";
+    for (let i = 0; i < str.length; i++) {
+        if (!sub[str[i]]) {
+            return str;
+        }
+        subStr += sub[str[i]];
+    }
+    return subStr;
+}
+/**
+ * Convert string to superscript unicode characters
+ */
+export function strToPostScript(str) {
+    const post = {
+        "0": "⁰",
+        "1": "¹",
+        "2": "²",
+        "3": "³",
+        "4": "⁴",
+        "5": "⁵",
+        "6": "⁶",
+        "7": "⁷",
+        "8": "⁸",
+        "9": "⁹",
+        "+": "⁺",
+        "-": "⁻",
+        a: "ᵃ",
+        b: "ᵇ",
+        c: "ᶜ",
+        d: "ᵈ",
+        e: "ᵉ",
+        f: "ᶠ",
+        g: "ᵍ",
+        h: "ʰ",
+        i: "ⁱ",
+        j: "ʲ",
+        k: "ᵏ",
+        l: "ˡ",
+        m: "ᵐ",
+        n: "ⁿ",
+        o: "ᵒ",
+        p: "ᵖ",
+        r: "ʳ",
+        s: "ˢ",
+        t: "ᵗ",
+        u: "ᵘ",
+        v: "ᵛ",
+        w: "ʷ",
+        x: "ˣ",
+        y: "ʸ",
+        z: "ᶻ",
+        A: "ᴬ",
+        B: "ᴮ",
+        D: "ᴰ",
+        E: "ᴱ",
+        G: "ᴳ",
+        H: "ᴴ",
+        I: "ᴵ",
+        J: "ᴶ",
+        K: "ᴷ",
+        L: "ᴸ",
+        M: "ᴹ",
+        N: "ᴺ",
+        O: "ᴼ",
+        P: "ᴾ",
+        R: "ᴿ",
+        T: "ᵀ",
+        U: "ᵁ",
+        V: "ⱽ",
+        W: "ᵂ",
+    };
+    let postStr = "";
+    for (let i = 0; i < str.length; i++) {
+        if (!post[str[i]]) {
+            return str;
+        }
+        postStr += post[str[i]];
+    }
+    return postStr;
+}
+//# sourceMappingURL=textUtils.js.map

package/dist/src/processing/textUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textUtils.js","sourceRoot":"","sources":["../../../src/processing/textUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,0DAA0D;IAC1D,6DAA6D;IAC7D,MAAM,eAAe,GAAG,4BAA4B,CAAC;IAErD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,mEAAmE;IACnE,+EAA+E;IAC/E,MAAM,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAE9D,2EAA2E;IAC3E,4DAA4D;IAC5D,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,2GAA2G;QAC3G,MAAM,cAAc,GAAG,0DAA0D,CAAC;QAClF,IAAI,cAAc,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACjD,OAAO,gBAAgB,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAW;IAC9C,MAAM,GAAG,GAA8B;QACrC,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;KACP,CAAC;IAEF,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,IAAI,GAA8B;QACtC,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,GAAG,EAAE,GAAG;QACR,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;QACN,CAAC,EAAE,GAAG;KACP,CAAC;IAEF,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/processing/textUtils.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=textUtils.test.d.ts.map

package/dist/src/processing/textUtils.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"textUtils.test.d.ts","sourceRoot":"","sources":["../../../src/processing/textUtils.test.ts"],"names":[],"mappings":""}

package/dist/src/processing/textUtils.test.js

@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vitest";
+import { strToPostScript, strToSubscriptString, cleanOcrTableArtifacts } from "./textUtils";
+describe("test processing textUtils", () => {
+    it("test strToSubscriptString full conversion", () => {
+        const originalStr = "hello123";
+        const finalStr = strToSubscriptString(originalStr);
+        expect(finalStr).toBe("ₕₑₗₗₒ₁₂₃");
+    });
+    it("test strToSubscriptString fail conversion", () => {
+        const originalStr = "hello123!";
+        const finalStr = strToSubscriptString(originalStr);
+        expect(finalStr).toBe("hello123!");
+    });
+    it("test strToPostScript full conversion", () => {
+        const originalStr = "hello123";
+        const finalStr = strToPostScript(originalStr);
+        expect(finalStr).toBe("ʰᵉˡˡᵒ¹²³");
+    });
+    it("test strToPostScript fail conversion", () => {
+        const originalStr = "hello123!";
+        const finalStr = strToPostScript(originalStr);
+        expect(finalStr).toBe("hello123!");
+    });
+    it("test cleanOcrTableArtifacts removes trailing bracket from number", () => {
+        expect(cleanOcrTableArtifacts("44520]")).toBe("44520");
+        expect(cleanOcrTableArtifacts("9,674]")).toBe("9,674");
+        expect(cleanOcrTableArtifacts("0.3|")).toBe("0.3");
+        expect(cleanOcrTableArtifacts("63,790|")).toBe("63,790");
+    });
+    it("test cleanOcrTableArtifacts removes leading bracket from number", () => {
+        expect(cleanOcrTableArtifacts("|123")).toBe("123");
+        expect(cleanOcrTableArtifacts("[456")).toBe("456");
+    });
+    it("test cleanOcrTableArtifacts preserves brackets in non-numeric text", () => {
+        expect(cleanOcrTableArtifacts("(note)")).toBe("(note)");
+        expect(cleanOcrTableArtifacts("[ref]")).toBe("[ref]");
+        expect(cleanOcrTableArtifacts("hello]")).toBe("hello]");
+    });
+    it("test cleanOcrTableArtifacts handles special numeric patterns", () => {
+        expect(cleanOcrTableArtifacts("*-123")).toBe("*-123");
+        expect(cleanOcrTableArtifacts("Z")).toBe("Z");
+        expect(cleanOcrTableArtifacts("N/A")).toBe("N/A");
+    });
+});
+//# sourceMappingURL=textUtils.test.js.map

package/dist/src/processing/textUtils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textUtils.test.js","sourceRoot":"","sources":["../../../src/processing/textUtils.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAE5F,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,WAAW,GAAG,UAAU,CAAC;QAC/B,MAAM,QAAQ,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,WAAW,GAAG,WAAW,CAAC;QAChC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;QACnD,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,WAAW,GAAG,UAAU,CAAC;QAC/B,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,WAAW,GAAG,WAAW,CAAC;QAChC,MAAM,QAAQ,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,MAAM,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,MAAM,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}