extract-pdf 0.1.259 → 0.1.261

package/src/pdf-to-html.ts

@@ -33,6 +33,34 @@ import {
   convertPDFToHTMLWithLiteParseWasm,
   type LiteParseWasmHTMLOptions,
 } from "./liteparse-wasm-to-html";
+import { loadPdfJs } from "./utils/load-pdfjs";
+import {
+  scanPagesForOCR,
+  type OcrScanResult,
+  type ScanPagesForOCROptions,
+} from "./ocr-page-scan";
+import {
+  ocrPdfPagesWithDocling,
+  type DoclingOcrOptions,
+} from "./docling-ocr";
+
+/**
+ * Where the OCR-capable Docling processing happens for
+ * {@link convertPDFToHTML}.
+ * - `"frontend"` (default) — all pages parsed by the pure-JS text-layer
+ *   pipeline; no OCR, no model, works everywhere.
+ * - `"hybrid"` — every page goes through the JS pipeline, then a regex scan
+ *   ({@link scanPagesForOCR}) flags pages containing infographics/figures/
+ *   tables (or with no usable text layer) and only those pages are re-done
+ *   with the Granite Docling OCR model.
+ * - `"docling"` — every page is rasterized and OCR'd with Granite Docling
+ *   (in-process via the optional `@huggingface/transformers` dependency, or
+ *   remotely when `processorUrl` is set).
+ * - any `http(s)://` URL — like `"docling"`, but all pages are sent to that
+ *   docling-compatible processor API (this package's `server/`, or another
+ *   deployment).
+ */
+export type ProcessorMode = "frontend" | "hybrid" | "docling" | (string & {});
 
 /**
  * Which parsing engine {@link convertPDFToHTML} runs.
@@ -65,6 +93,16 @@ export type ParseMethod = "ts-block-algorithm" | "liteparse" | "liteparse-wasm";
  * @param {boolean} options.removePageHeaders default=true - Removes repeated headers found on each page
  * @param {ParseMethod} options.method default="ts-block-algorithm" - Parsing engine to use;
  *   `"liteparse"` delegates to LiteParse (Node.js only, see {@link ParseMethod})
+ * @param {ProcessorMode} options.processor default="frontend" - Where OCR happens:
+ *   `"frontend"` (all JS, no OCR), `"hybrid"` (regex-scan pages for
+ *   infographics/tables and OCR only those), `"docling"` (OCR every page), or
+ *   the URL of a docling-compatible processor API (see {@link ProcessorMode})
+ * @param {string} options.processorUrl - Remote docling-compatible API base URL
+ *   used by `"hybrid"`/`"docling"` instead of the in-process model
+ * @param {Object} options.ocrScanOptions - Threshold tuning for the hybrid
+ *   page scan, see {@link ScanPagesForOCROptions}
+ * @param {Object} options.doclingOptions - Prompt/maxTokens/scale for the OCR
+ *   model, see {@link DoclingOcrOptions}
  * @returns {string|Object} HTML formatted text
  * @category Extract
  * @author [vtempest (2025)](https://github.com/vtempest),
@@ -77,6 +115,10 @@ export async function convertPDFToHTML(
     addPageNumbers?: boolean;
     addCitation?: boolean;
     method?: ParseMethod;
+    processor?: ProcessorMode;
+    processorUrl?: string;
+    ocrScanOptions?: ScanPagesForOCROptions;
+    doclingOptions?: Omit<DoclingOcrOptions, "processorUrl">;
   } & Pick<LiteParseHTMLOptions, "liteParseOptions"> = {},
 ) {
   if (options.method === "liteparse") {
@@ -95,6 +137,16 @@ export async function convertPDFToHTML(
   // try {
   var { addPageNumbers = false, addCitation = true } = options;
 
+  // Resolve where OCR happens: "frontend" | "hybrid" | "docling" | a
+  // processor URL (which means "docling" against that remote API).
+  const processor = options.processor ?? "frontend";
+  const processorIsUrl = /^https?:\/\//i.test(processor);
+  const processorMode = processorIsUrl ? "docling" : processor;
+  const doclingOptions: DoclingOcrOptions = {
+    ...options.doclingOptions,
+    processorUrl: processorIsUrl ? processor : options.processorUrl,
+  };
+
   // pass in databuffer or download all pdf data
   // and convert to array buffer
   var buffer =
@@ -107,9 +159,7 @@ export async function convertPDFToHTML(
 
   let pdfDocument;
   try {
-    let { resolvePDFJS } = await import("https://cdn.jsdelivr.net/npm/pdfjs-serverless@1.1.0/+esm" as any);
-    
-    const { getDocument } = await resolvePDFJS();
+    const { getDocument } = await loadPdfJs();
     pdfDocument = await getDocument({
       data: new Uint8Array(buffer),
       useSystemFonts: true,
@@ -178,6 +228,19 @@ export async function convertPDFToHTML(
     pages[page.pageNumber - 1].items = textItems;
   }
 
+  // Raw per-page text (items grouped into lines by y) captured before the
+  // transforms mutate the pages — input for the OCR page scan.
+  const pageTexts = pages.map((page) =>
+    Object.values(
+      (page.items as any[]).reduce((lines: any, item: any) => {
+        (lines[item.y] = lines[item.y] || []).push(item.text);
+        return lines;
+      }, {}),
+    )
+      .map((line: any) => line.join(" "))
+      .join("\n"),
+  );
+
   var parseResult = new ParseResult({ pages });
 
   let lastTransformation: (typeof transformations)[number] | undefined,
@@ -206,14 +269,35 @@ export async function convertPDFToHTML(
     lastTransformation = transformation;
   });
 
-  var html = parseResult.pages.reduce((acc, page, pageNumber) => {
-    return (
-      acc +
+  var pageHtmls = parseResult.pages.map(
+    (page, pageNumber) =>
       `<p id="page-${pageNumber + 1}">${
         addPageNumbers ? ` [${pageNumber + 1}] ` : ""
-      }${page.items.join('</p><p id="page-' + pageNumber + '">')}</p>`
+      }${page.items.join('</p><p id="page-' + pageNumber + '">')}</p>`,
+  );
+
+  // Regex scan flagging pages with infographics/figures/tables (or no usable
+  // text layer) — the pages worth OCR'ing in hybrid mode.
+  const ocrScan = scanPagesForOCR(pageTexts, options.ocrScanOptions);
+
+  if (processorMode === "docling" || processorMode === "hybrid") {
+    const targetPages =
+      processorMode === "docling"
+        ? pageHtmls.map((_, index) => index + 1)
+        : ocrScan.pagesNeedingOcr;
+    const ocrResults = await ocrPdfPagesWithDocling(
+      pdfDocument,
+      targetPages,
+      doclingOptions,
     );
-  }, "");
+    // Pages whose OCR failed keep their frontend-parsed HTML.
+    for (const [pageNumber, ocrHtml] of ocrResults)
+      pageHtmls[pageNumber - 1] = `<section class="ocr-page" id="page-${pageNumber}">${
+        addPageNumbers ? ` [${pageNumber}] ` : ""
+      }${ocrHtml}</section>`;
+  }
+
+  var html = pageHtmls.join("");
 
   if (addCitation) {
     // Get metadata
@@ -234,7 +318,21 @@ export async function convertPDFToHTML(
     title = html.slice(0, 400).match(/<h[0-9]>(.*?)<\/h[0-9]>/)?.[1] || title;
   }
 
-  return { author, title, html, format: "pdf" };
+  return {
+    author,
+    title,
+    html,
+    format: "pdf",
+    processor: processorMode,
+    ocrScan,
+  } as {
+    author?: string;
+    title?: string;
+    html: string;
+    format: string;
+    processor: string;
+    ocrScan: OcrScanResult;
+  };
 }
 
 export { convertPDFToHTMLWithLiteParse };
@@ -246,4 +344,18 @@ export type {
   DetectPdfNeedsOcrOptions,
   PdfOcrAssessment,
 } from "./detect-needs-ocr";
+export { scanPagesForOCR } from "./ocr-page-scan";
+export type {
+  OcrScanResult,
+  PageOcrScan,
+  ScanPagesForOCROptions,
+} from "./ocr-page-scan";
+export {
+  doctagsToHtml,
+  ocrImageWithDocling,
+  ocrPdfPagesWithDocling,
+  renderPdfPageToPngBase64,
+} from "./docling-ocr";
+export type { DoclingOcrOptions } from "./docling-ocr";
+export { loadPdfJs } from "./utils/load-pdfjs";
 

package/src/utils/load-pdfjs.ts

@@ -0,0 +1,32 @@
+/**
+ * @fileoverview Lazy loader for the slim serverless PDF.js build.
+ *
+ * The package deliberately does NOT depend on `pdfjs-dist`: PDF.js is pulled
+ * at runtime from jsDelivr's ESM build of
+ * [pdfjs-serverless](https://github.com/johannschopplich/pdfjs-serverless) —
+ * a zero-dependency, single-file (~1.6 MB minified) redistribution of Mozilla
+ * PDF.js that works in Workers/edge runtimes, Node.js, and browsers. Pinned
+ * to the major version so compatible patch/minor releases update
+ * automatically. This keeps `extract-pdf`'s default export slim: nothing
+ * PDF.js-related is bundled or installed until a document is actually parsed.
+ */
+
+const PDFJS_SERVERLESS_CDN_URL =
+  "https://cdn.jsdelivr.net/npm/pdfjs-serverless@1/+esm";
+
+let pdfjsPromise: Promise<any> | null = null;
+
+/**
+ * Resolves the PDF.js API (`getDocument`, ...) from the pdfjs-serverless CDN
+ * build, caching the module after the first call. Node.js and Bun cannot
+ * import remote URLs, so when the CDN import throws we fall back to the
+ * locally-installed `pdfjs-serverless` optional dependency.
+ */
+export async function loadPdfJs(): Promise<any> {
+  if (!pdfjsPromise) {
+    pdfjsPromise = import(/* @vite-ignore */ PDFJS_SERVERLESS_CDN_URL as any)
+      .catch(() => import("pdfjs-serverless" as any))
+      .then(({ resolvePDFJS }) => resolvePDFJS());
+  }
+  return pdfjsPromise;
+}