@vertesia/converters 0.50.1 → 0.52.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/converters",
-  "version": "0.50.1",
+  "version": "0.52.0",
   "description": "Image and content converters",
   "type": "module",
   "types": "./lib/types/index.d.ts",
@@ -18,12 +18,9 @@
     "@types/tmp": "^0.2.6",
     "ts-dual-module": "^0.6.3",
     "typescript": "^5.0.2",
-    "vitest": "^2.1.8"
+    "vitest": "^3.0.9"
   },
   "dependencies": {
-    "@opendocsg/pdf2md": "0.2.0",
-    "@pdftron/pdfnet-node": "^10.11.0",
-    "mupdf": "^0.3.0",
     "sharp": "^0.33.5",
     "tmp": "^0.2.3"
   },

package/lib/esm/mutool2.js

@@ -1,16 +0,0 @@
-import * as mupdf from "mupdf";
-import fs from "fs";
-async function test() {
-    const doc = mupdf.Document.openDocument(fs.readFileSync("./fixtures/test-pdf1.pdf"), "application/pdf");
-    //const count = doc.countPages();
-    for (let i = 0; i < 5; i++) {
-        const page = doc.loadPage(i);
-        const stext = page.toStructuredText();
-        console.log("Page ================= ", i);
-        console.log("=================!!!!!!", stext.asText());
-        //console.log(JSON.stringify(JSON.parse(stext.asJSON()), undefined, 2));
-        //console.log("=================!!!!!!", stext);
-    }
-}
-test();
-//# sourceMappingURL=mutool2.js.map

package/lib/esm/mutool2.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"mutool2.js","sourceRoot":"","sources":["../../src/mutool2.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,KAAK,UAAU,IAAI;IAEf,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,YAAY,CAAC,0BAA0B,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAExG,iCAAiC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,CAAC,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,wEAAwE;QACxE,gDAAgD;IACpD,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC"}

package/lib/esm/pdf-test.js

@@ -1,15 +0,0 @@
-/**
- * IMPORTANT: DO NOT RUN IN VITEST, VITEST DOESN'T WORK WITH APRYSE
- */
-import fs from 'fs';
-import path from 'path';
-import { extractImagesFromPdfWithApryse } from "./pdf.js";
-const main = async () => {
-    const pdfPath = path.resolve(__dirname, '../../../fixtures', 'test-pdf2.pdf');
-    const pdfBuffer = fs.readFileSync(pdfPath);
-    console.log('start extracting images from pdf');
-    const result = await extractImagesFromPdfWithApryse(pdfBuffer);
-    console.log('result: ', result);
-};
-main();
-//# sourceMappingURL=pdf-test.js.map

package/lib/esm/pdf-test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf-test.js","sourceRoot":"","sources":["../../src/pdf-test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,8BAA8B,EAAE,MAAM,UAAU,CAAC;AAE1D,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IAEpB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,mBAAmB,EAAE,eAAe,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAE3C,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAChD,MAAM,MAAM,GAAQ,MAAM,8BAA8B,CAAC,SAAS,CAAC,CAAC;IAEpE,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;AAEpC,CAAC,CAAA;AAED,IAAI,EAAE,CAAC"}

package/lib/esm/pdf.js

@@ -1,75 +0,0 @@
-import pdf2md from "@opendocsg/pdf2md";
-import fs from 'fs';
-import os from 'os';
-import pkg from '@pdftron/pdfnet-node';
-const { PDFNet } = pkg;
-const pdf2mdFn = pdf2md;
-export function trasformPdfToMarkdown(buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}
-async function extractImages(buffer, minHw = 300) {
-    const doc = await PDFNet.PDFDoc.createFromBuffer(buffer);
-    const reader = await PDFNet.ElementReader.create();
-    const tmpDir = os.tmpdir();
-    const workingDir = fs.mkdtempSync(`${tmpDir}/pdfextract_`);
-    // Read page content on every page in the document
-    const itr = await doc.getPageIterator();
-    for (itr; await itr.hasNext(); itr.next()) {
-        // Read the page
-        const page = await itr.current();
-        const pageNumber = await page.getIndex();
-        reader.beginOnPage(page);
-        await ProcessElements(reader, pageNumber);
-        reader.end();
-    }
-    return { workingDir };
-    async function ProcessElements(reader, pageNumber) {
-        // Traverse the page display list
-        let imgCount = 1;
-        for (let element = await reader.next(); element !== null; element = await reader.next()) {
-            const elementType = await element.getType();
-            switch (elementType) {
-                case PDFNet.Element.Type.e_image:
-                    {
-                        const image = await PDFNet.Image.createFromObj(await element.getXObject());
-                        const h = await image.getImageHeight();
-                        const w = await image.getImageWidth();
-                        //console.log(`Image: width=${w}, height=${h}`);
-                        //do not extract if image is too small, likely not relevant
-                        //TODO: use LLM to decide if it matters?
-                        if (w < minHw && h < minHw) {
-                            break;
-                        }
-                        const imgName = `${workingDir}/img_${pageNumber}_${imgCount++}.png`;
-                        image.exportAsPng(imgName);
-                        break;
-                    }
-                case PDFNet.Element.Type.e_form:
-                    {
-                        reader.formBegin();
-                        ProcessElements(reader, pageNumber);
-                        reader.end();
-                        break;
-                    }
-            }
-        }
-    }
-}
-export async function extractImagesFromPdfWithApryse(buffer, minHw = 300) {
-    const APRYSE_KEY = process.env.APRYSE_KEY;
-    const extractImagesWrapper = async () => {
-        return await extractImages(buffer, minHw);
-    };
-    const res = await PDFNet.runWithCleanup(extractImagesWrapper, APRYSE_KEY).then((res) => {
-        return res;
-    }).finally(() => PDFNet.shutdown());
-    //read all images in the directory
-    const files = fs.readdirSync(res.workingDir);
-    const images = files.map((file) => {
-        const [pageNumber, imgCount] = file.split('.')[0].split('_').slice(1);
-        return { page: parseInt(pageNumber), imgCount: parseInt(imgCount), path: file };
-    });
-    return images;
-}
-//# sourceMappingURL=pdf.js.map

package/lib/esm/pdf.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../src/pdf.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,mBAAmB,CAAC;AAEvC,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,GAAG,MAAM,sBAAsB,CAAC;AACvC,MAAM,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;AAGvB,MAAM,QAAQ,GAAG,MAA4D,CAAC;AAE9E,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAChD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAID,KAAK,UAAU,aAAa,CAAC,MAAc,EAAE,QAAgB,GAAG;IAC5D,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;IAE3D,kDAAkD;IAClD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;IACxC,KAAK,GAAG,EAAE,MAAM,GAAG,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,gBAAgB;QAChB,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACzB,MAAM,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAC1C,MAAM,CAAC,GAAG,EAAE,CAAC;IACjB,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,KAAK,UAAU,eAAe,CAAC,MAA6B,EAAE,UAAkB;QAC5E,iCAAiC;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,OAAO,KAAK,IAAI,EAAE,OAAO,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YACtF,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAC5C,QAAQ,WAAW,EAAE,CAAC;gBAClB,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO;oBAC5B,CAAC;wBACG,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;wBAC3E,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;wBACvC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,CAAC;wBACtC,gDAAgD;wBAChD,2DAA2D;wBAC3D,wCAAwC;wBACxC,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC;4BACzB,MAAM;wBACV,CAAC;wBACD,MAAM,OAAO,GAAG,GAAG,UAAU,QAAQ,UAAU,IAAI,QAAQ,EAAE,MAAM,CAAC;wBACpE,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;wBAC3B,MAAM;oBACV,CAAC;gBACL,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM;oBAC3B,CAAC;wBACG,MAAM,CAAC,SAAS,EAAE,CAAC;wBACnB,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;wBACpC,MAAM,CAAC,GAAG,EAAE,CAAC;wBACb,MAAM;oBACV,CAAC;YACT,CAAC;QACL,CAAC;IACL,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,MAAc,EAAE,QAAgB,GAAG;IACpF,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;IAE1C,MAAM,oBAAoB,GAAG,KAAK,IAAI,EAAE;QACpC,OAAO,MAAM,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,oBAAoB,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QACnF,OAAO,GAAG,CAAC;IACf,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAGpC,kCAAkC;IAClC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC9B,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACtE,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/lib/types/mutool2.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=mutool2.d.ts.map

package/lib/types/mutool2.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"mutool2.d.ts","sourceRoot":"","sources":["../../src/mutool2.ts"],"names":[],"mappings":""}

package/lib/types/pdf-test.d.ts

@@ -1,5 +0,0 @@
-/**
- * IMPORTANT: DO NOT RUN IN VITEST, VITEST DOESN'T WORK WITH APRYSE
- */
-export {};
-//# sourceMappingURL=pdf-test.d.ts.map

package/lib/types/pdf-test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf-test.d.ts","sourceRoot":"","sources":["../../src/pdf-test.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/lib/types/pdf.d.ts

@@ -1,7 +0,0 @@
-export declare function trasformPdfToMarkdown(buffer: Buffer): Promise<string>;
-export declare function extractImagesFromPdfWithApryse(buffer: Buffer, minHw?: number): Promise<{
-    page: number;
-    imgCount: number;
-    path: string;
-}[]>;
-//# sourceMappingURL=pdf.d.ts.map

package/lib/types/pdf.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pdf.d.ts","sourceRoot":"","sources":["../../src/pdf.ts"],"names":[],"mappings":"AAWA,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,mBAGnD;AAyDD,wBAAsB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAAY;;;;KAqBvF"}

package/src/mutool2.ts

@@ -1,19 +0,0 @@
-import * as mupdf from "mupdf";
-import fs from "fs";
-
-async function test() {
-
-    const doc = mupdf.Document.openDocument(fs.readFileSync("./fixtures/test-pdf1.pdf"), "application/pdf");
-
-    //const count = doc.countPages();
-    for (let i = 0; i < 5; i++) {
-        const page = doc.loadPage(i);
-        const stext = page.toStructuredText();
-        console.log("Page ================= ", i);
-        console.log("=================!!!!!!", stext.asText());
-        //console.log(JSON.stringify(JSON.parse(stext.asJSON()), undefined, 2));
-        //console.log("=================!!!!!!", stext);
-    }
-}
-
-test();

package/src/pdf-test.ts

@@ -1,21 +0,0 @@
-/**
- * IMPORTANT: DO NOT RUN IN VITEST, VITEST DOESN'T WORK WITH APRYSE
- */
-
-import fs from 'fs';
-import path from 'path';
-import { extractImagesFromPdfWithApryse } from "./pdf.js";
-
-const main = async () => {
-
-    const pdfPath = path.resolve(__dirname, '../../../fixtures', 'test-pdf2.pdf');
-    const pdfBuffer = fs.readFileSync(pdfPath);
-    
-    console.log('start extracting images from pdf');
-    const result: any = await extractImagesFromPdfWithApryse(pdfBuffer);
-
-    console.log('result: ', result);
-
-}   
-
-main();

package/src/pdf.test.ts

@@ -1,25 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { expect, test } from 'vitest';
-import { trasformPdfToMarkdown } from './pdf';
-
-
-
-test('Converts a PDF to markdown', async () => {
-  const pdfPath = path.resolve(__dirname, '../fixtures', 'test-pdf1.pdf');
-  const pdfBuffer = fs.readFileSync(pdfPath);
-  const result = await trasformPdfToMarkdown(pdfBuffer);
-
-  expect(result).toContain('America');
-
-
-});
-
-test('Converts another PDF to markdown', async () => {
-  const pdfPath = path.resolve(__dirname, '../fixtures', 'test-pdf2.pdf');
-  const pdfBuffer = fs.readFileSync(pdfPath);
-  const result = await trasformPdfToMarkdown(pdfBuffer);
-
-  expect(result).toContain('America');
-
-});

package/src/pdf.ts

@@ -1,93 +0,0 @@
-import pdf2md from "@opendocsg/pdf2md";
-import type { PDFNet as PDFTron } from '@pdftron/pdfnet-node';
-import fs from 'fs';
-import os from 'os';
-
-import pkg from '@pdftron/pdfnet-node';
-const { PDFNet } = pkg;
-
-
-const pdf2mdFn = pdf2md as unknown as (buffer: Uint8Array) => Promise<string>;
-
-export function trasformPdfToMarkdown(buffer: Buffer) {
-    const arr = new Uint8Array(buffer);
-    return pdf2mdFn(arr);
-}
-
-
-
-async function extractImages(buffer: Buffer, minHw: number = 300) {
-    const doc = await PDFNet.PDFDoc.createFromBuffer(buffer);
-    const reader = await PDFNet.ElementReader.create();
-    const tmpDir = os.tmpdir()
-    const workingDir = fs.mkdtempSync(`${tmpDir}/pdfextract_`);
-
-    // Read page content on every page in the document
-    const itr = await doc.getPageIterator();
-    for (itr; await itr.hasNext(); itr.next()) {
-        // Read the page
-        const page = await itr.current();
-        const pageNumber = await page.getIndex();
-        reader.beginOnPage(page);
-        await ProcessElements(reader, pageNumber);
-        reader.end();
-    }
-
-    return { workingDir };
-
-    async function ProcessElements(reader: PDFTron.ElementReader, pageNumber: number) {
-        // Traverse the page display list
-        let imgCount = 1;
-
-        for (let element = await reader.next(); element !== null; element = await reader.next()) {
-            const elementType = await element.getType();
-            switch (elementType) {
-                case PDFNet.Element.Type.e_image:
-                    {
-                        const image = await PDFNet.Image.createFromObj(await element.getXObject());
-                        const h = await image.getImageHeight();
-                        const w = await image.getImageWidth();
-                        //console.log(`Image: width=${w}, height=${h}`);
-                        //do not extract if image is too small, likely not relevant
-                        //TODO: use LLM to decide if it matters?
-                        if (w < minHw && h < minHw) {
-                            break;
-                        }
-                        const imgName = `${workingDir}/img_${pageNumber}_${imgCount++}.png`;
-                        image.exportAsPng(imgName);
-                        break;
-                    }
-                case PDFNet.Element.Type.e_form:
-                    {
-                        reader.formBegin();
-                        ProcessElements(reader, pageNumber);
-                        reader.end();
-                        break;
-                    }
-            }
-        }
-    }
-}
-
-export async function extractImagesFromPdfWithApryse(buffer: Buffer, minHw: number = 300) {
-    const APRYSE_KEY = process.env.APRYSE_KEY;
-
-    const extractImagesWrapper = async () => {
-        return await extractImages(buffer, minHw);
-    };
-
-    const res = await PDFNet.runWithCleanup(extractImagesWrapper, APRYSE_KEY).then((res) => {
-        return res;
-    }).finally(() => PDFNet.shutdown());
-
-
-    //read all images in the directory
-    const files = fs.readdirSync(res.workingDir);
-
-    const images = files.map((file) => {
-        const [pageNumber, imgCount] = file.split('.')[0].split('_').slice(1);
-        return { page: parseInt(pageNumber), imgCount: parseInt(imgCount), path: file };
-    });
-
-    return images;
-}