formatferry 1.0.24 → 1.0.25

package/dist/index.js

@@ -5,9 +5,16 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
 var __commonJS = (cb, mod) => function __require() {
   return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
 };
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
 var __copyProps = (to, from, except, desc) => {
   if (from && typeof from === "object" || typeof from === "function") {
     for (let key of __getOwnPropNames(from))
@@ -2795,12 +2802,12 @@ var require_ChildNode = __commonJS({
     "use strict";
     var Node2 = require_Node();
     var LinkedList = require_LinkedList();
-    var createDocumentFragmentFromArguments = function(document2, args) {
-      var docFrag = document2.createDocumentFragment();
+    var createDocumentFragmentFromArguments = function(document3, args) {
+      var docFrag = document3.createDocumentFragment();
       for (var i = 0; i < args.length; i++) {
         var argItem = args[i];
         var isNode = argItem instanceof Node2;
-        docFrag.appendChild(isNode ? argItem : document2.createTextNode(String(argItem)));
+        docFrag.appendChild(isNode ? argItem : document3.createTextNode(String(argItem)));
       }
       return docFrag;
     };
@@ -3068,7 +3075,7 @@ var require_Element = __commonJS({
           return NodeUtils.serializeOne(this, { nodeType: 0 });
         },
         set: function(v) {
-          var document2 = this.ownerDocument;
+          var document3 = this.ownerDocument;
           var parent = this.parentNode;
           if (parent === null) {
             return;
@@ -3079,8 +3086,8 @@ var require_Element = __commonJS({
           if (parent.nodeType === Node2.DOCUMENT_FRAGMENT_NODE) {
             parent = parent.ownerDocument.createElement("body");
           }
-          var parser = document2.implementation.mozHTMLParser(
-            document2._address,
+          var parser = document3.implementation.mozHTMLParser(
+            document3._address,
             parent
           );
           parser.parse(v === null ? "" : String(v), true);
@@ -5665,9 +5672,9 @@ var require_defineElement = __commonJS({
       });
       return c;
     };
-    function EventHandlerBuilder(body, document2, form, element) {
+    function EventHandlerBuilder(body, document3, form, element) {
       this.body = body;
-      this.document = document2;
+      this.document = document3;
       this.form = form;
       this.element = element;
     }
@@ -16542,8 +16549,8 @@ var require_Window = __commonJS({
     var Location = require_Location();
     var utils = require_utils();
     module.exports = Window;
-    function Window(document2) {
-      this.document = document2 || new DOMImplementation(null).createHTMLDocument("");
+    function Window(document3) {
+      this.document = document3 || new DOMImplementation(null).createHTMLDocument("");
       this.document._scripting_enabled = true;
       this.document.defaultView = this;
       this.location = new Location(this, this.document._address || "about:blank");
@@ -16674,11 +16681,11 @@ var require_lib = __commonJS({
       };
     };
     exports.createWindow = function(html, address) {
-      var document2 = exports.createDocument(html);
+      var document3 = exports.createDocument(html);
       if (address !== void 0) {
-        document2._address = address;
+        document3._address = address;
       }
-      return new impl.Window(document2);
+      return new impl.Window(document3);
     };
     exports.impl = impl;
   }
@@ -17409,6 +17416,137 @@ var require_turndown_cjs = __commonJS({
   }
 });
 
+// ../../src/shared/feature-flags.ts
+function getFeatureFlags() {
+  return { ..._flags };
+}
+var _flags;
+var init_feature_flags = __esm({
+  "../../src/shared/feature-flags.ts"() {
+    _flags = {
+      enableDocx: true,
+      enableXlsx: true,
+      enablePdf: true,
+      enableHtml: true,
+      enablePdfTableDetection: true,
+      enableData: true,
+      enablePptx: true,
+      enablePdfEnhanced: true,
+      enablePdfMultiColumn: true,
+      enablePdfOcr: false
+    };
+  }
+});
+
+// ../../src/shared/adapters/pdf-ocr-adapter.ts
+var pdf_ocr_adapter_exports = {};
+__export(pdf_ocr_adapter_exports, {
+  isOcrAvailable: () => isOcrAvailable,
+  runOcr: () => runOcr
+});
+async function loadTesseract() {
+  try {
+    const mod = await import("tesseract.js");
+    if (mod && typeof mod.createWorker === "function") {
+      return mod;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+function preScaleCanvas(canvas) {
+  if (canvas.width <= MAX_IMAGE_WIDTH) return canvas;
+  const scale = MAX_IMAGE_WIDTH / canvas.width;
+  const scaled = document.createElement("canvas");
+  scaled.width = MAX_IMAGE_WIDTH;
+  scaled.height = Math.round(canvas.height * scale);
+  const ctx = scaled.getContext("2d");
+  if (ctx) {
+    ctx.drawImage(canvas, 0, 0, scaled.width, scaled.height);
+  }
+  return scaled;
+}
+function canvasToBlob(canvas) {
+  return new Promise((resolve2, reject) => {
+    canvas.toBlob(
+      (blob) => {
+        if (blob) resolve2(blob);
+        else reject(new Error("canvas.toBlob returned null"));
+      },
+      "image/png"
+    );
+  });
+}
+async function isOcrAvailable() {
+  if (!getFeatureFlags().enablePdfOcr) return false;
+  const lib = await loadTesseract();
+  return lib !== null;
+}
+async function runOcr(pageCanvases, options) {
+  const lib = await loadTesseract();
+  if (!lib) {
+    throw new Error(
+      "OCR is not available: tesseract.js could not be loaded. Ensure the tesseract.js package is installed."
+    );
+  }
+  const warnings = [];
+  const pages = [];
+  const worker = await lib.createWorker("eng", 1, {
+    logger: (m) => {
+      if (options?.onProgress) {
+        options.onProgress({
+          progress: m.progress,
+          status: m.status
+        });
+      }
+    }
+  });
+  try {
+    for (let i = 0; i < pageCanvases.length; i++) {
+      const { pageNumber, canvas } = pageCanvases[i];
+      const scaled = preScaleCanvas(canvas);
+      const blob = await canvasToBlob(scaled);
+      try {
+        const { data } = await worker.recognize(blob);
+        const text = data.text.trim();
+        if (data.confidence < MIN_CONFIDENCE) {
+          warnings.push(
+            `Page ${pageNumber}: low OCR confidence (${Math.round(data.confidence)}%). Text quality may be poor.`
+          );
+        }
+        pages.push({
+          pageNumber,
+          text: text || "",
+          confidence: data.confidence
+        });
+      } catch (err) {
+        warnings.push(
+          `Page ${pageNumber}: OCR failed \u2014 ${err instanceof Error ? err.message : String(err)}`
+        );
+        pages.push({ pageNumber, text: "", confidence: 0 });
+      }
+      if (options?.onProgress) {
+        options.onProgress({
+          progress: (i + 1) / pageCanvases.length,
+          status: `OCR page ${i + 1}/${pageCanvases.length}`
+        });
+      }
+    }
+  } finally {
+    await worker.terminate();
+  }
+  return { pages, warnings };
+}
+var MAX_IMAGE_WIDTH, MIN_CONFIDENCE;
+var init_pdf_ocr_adapter = __esm({
+  "../../src/shared/adapters/pdf-ocr-adapter.ts"() {
+    init_feature_flags();
+    MAX_IMAGE_WIDTH = 2e3;
+    MIN_CONFIDENCE = 40;
+  }
+});
+
 // ../../node_modules/@mozilla/readability/Readability.js
 var require_Readability = __commonJS({
   "../../node_modules/@mozilla/readability/Readability.js"(exports, module) {
@@ -19471,9 +19609,9 @@ var require_readability = __commonJS({
 // src/shim.ts
 import { parseHTML } from "linkedom";
 import createDOMPurify from "dompurify";
-var { document, window: window2 } = parseHTML("<!DOCTYPE html><html><body></body></html>");
+var { document: document2, window: window2 } = parseHTML("<!DOCTYPE html><html><body></body></html>");
 globalThis.window = window2;
-globalThis.document = document;
+globalThis.document = document2;
 globalThis.DOMParser = window2.DOMParser;
 globalThis.Node = window2.Node;
 globalThis.HTMLElement = window2.HTMLElement;
@@ -21449,21 +21587,8 @@ var DocxAdapter = class {
 };
 var docxAdapter = new DocxAdapter();
 
-// ../../src/shared/feature-flags.ts
-var _flags = {
-  enableDocx: true,
-  enableXlsx: true,
-  enablePdf: true,
-  enableHtml: true,
-  enablePdfTableDetection: true,
-  enableData: true,
-  enablePptx: true,
-  enablePdfEnhanced: true,
-  enablePdfMultiColumn: true
-};
-function getFeatureFlags() {
-  return { ..._flags };
-}
+// ../../src/shared/adapters/pdf-adapter.ts
+init_feature_flags();
 
 // ../../src/shared/adapters/pdf-table-detector.ts
 var CJK_REGEX = /[\u3400-\u9FFF\uF900-\uFAFF\u3040-\u30FF\uAC00-\uD7AF]/;
@@ -23044,8 +23169,9 @@ var PdfAdapter = class {
         onProgress?.(pageNum / numPages);
         const page = await pdf.getPage(pageNum);
         const viewport = page.getViewport({ scale: 1 });
+        const useMarkedContent = getFeatureFlags().enablePdfEnhanced && isTaggedPdf;
         const [textContent, annotations] = await Promise.all([
-          page.getTextContent(),
+          page.getTextContent(useMarkedContent ? { includeMarkedContent: true } : void 0),
           page.getAnnotations().catch((err) => {
             console.warn(`[PdfAdapter] getAnnotations failed for page ${pageNum}:`, err);
             return [];
@@ -23053,10 +23179,23 @@ var PdfAdapter = class {
         ]);
         const positionedText = [];
         const xPositions = [];
+        let currentMcid;
         for (const item of textContent.items) {
+          const itemType = "type" in item ? item.type : void 0;
+          if (itemType === "beginMarkedContentProps") {
+            currentMcid = item.id?.toString();
+            continue;
+          }
+          if (itemType === "endMarkedContent") {
+            currentMcid = void 0;
+            continue;
+          }
           if (!("str" in item) || typeof item.str !== "string") continue;
           const positioned = textItemToPositioned(item);
           if (!positioned) continue;
+          if (currentMcid !== void 0) {
+            positioned.mcid = currentMcid;
+          }
           positionedText.push(positioned);
           xPositions.push(positioned.x);
         }
@@ -23071,38 +23210,90 @@ var PdfAdapter = class {
         const renderItems = getFeatureFlags().enablePdfEnhanced && hfSuppressSet.size > 0 ? items.filter((it) => !isHeaderFooterItem(it, hfSuppressSet)) : items;
         if (getFeatureFlags().enablePdfEnhanced && isTaggedPdf) {
           let structHtml = null;
+          let claimedMcids = null;
           try {
             if (hasStructTree(page)) {
               const structResult = await structTreeToHtml(page);
               if (structResult !== null) {
                 structHtml = structResult.html;
+                claimedMcids = structResult.claimedMcids;
               }
             }
           } catch {
           }
-          if (structHtml !== null) {
-            const structTextLen = structHtml.replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim().length;
-            const geoTextLen = renderItems.map((it) => it.str).join(" ").length;
-            if (geoTextLen > 0 && structTextLen / geoTextLen < 0.7) {
-              structHtml = null;
+          let fusionStructHtml = null;
+          let fusionClaimedMcids = null;
+          if (structHtml !== null && claimedMcids !== null) {
+            const textItems = renderItems.filter((it) => it.source !== "acroform");
+            const covered = textItems.filter((it) => it.mcid != null && claimedMcids.has(it.mcid)).length;
+            const coverage = covered / Math.max(textItems.length, 1);
+            if (coverage >= 0.7) {
+              const plainText = structHtml.replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
+              totalTextLength += plainText.length;
+              let hasImages2 = false;
+              try {
+                const opList = await page.getOperatorList();
+                hasImages2 = opList.fnArray.some((op) => op === 85 || op === 86);
+              } catch {
+              }
+              pages.push({
+                pageNumber: pageNum,
+                textContent: plainText,
+                hasImages: hasImages2,
+                viewportWidth: viewport.width,
+                htmlBlocks: [structHtml],
+                tablesDetected: 0
+              });
+              continue;
+            } else if (coverage > 0) {
+              fusionStructHtml = structHtml;
+              fusionClaimedMcids = claimedMcids;
             }
           }
-          if (structHtml !== null) {
-            const plainText = structHtml.replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
-            totalTextLength += plainText.length;
-            let hasImages2 = false;
+          if (fusionStructHtml !== null && fusionClaimedMcids !== null) {
+            const unclaimedItems = renderItems.filter(
+              (it) => !it.mcid || !fusionClaimedMcids.has(it.mcid)
+            );
+            const fsSamples = unclaimedItems.map((it) => it.fontSize ?? 0).filter((f) => f > 0);
+            const medianFontSize = fsSamples.length > 0 ? [...fsSamples].sort((a, b) => a - b)[Math.floor(fsSamples.length / 2)] : 12;
+            const fusionEnhancedOpts = getFeatureFlags().enablePdfEnhanced ? { medianFontSize, outlineTitles } : void 0;
+            const pageBottomY2 = Array.isArray(page.view) && page.view.length >= 2 ? page.view[1] : 0;
+            let pageOpList2;
+            let ruleSeparators2;
             try {
-              const opList = await page.getOperatorList();
-              hasImages2 = opList.fnArray.some((op) => op === 85 || op === 86);
+              pageOpList2 = await page.getOperatorList();
+              if (getFeatureFlags().enablePdfEnhanced && tableDetectionEnabled) {
+                ruleSeparators2 = extractVectorLines(
+                  pageOpList2.fnArray,
+                  pageOpList2.argsArray ?? [],
+                  viewport.width,
+                  viewport.height
+                );
+              }
             } catch {
             }
+            const rendered2 = renderPage(
+              unclaimedItems,
+              pageNum,
+              pageNum < numPages ? pageNum + 1 : void 0,
+              tableDetectionEnabled,
+              translate,
+              pageBottomY2,
+              fusionEnhancedOpts,
+              ruleSeparators2
+            );
+            for (const w of rendered2.warnings) warnings.push(w);
+            const structPlainText = fusionStructHtml.replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim();
+            totalTextLength += structPlainText.length + rendered2.textContent.length;
+            const hasImages2 = pageOpList2?.fnArray.some((op) => op === 85 || op === 86) ?? false;
+            const mergedHtmlBlocks = [fusionStructHtml, ...rendered2.htmlBlocks];
             pages.push({
               pageNumber: pageNum,
-              textContent: plainText,
+              textContent: structPlainText + "\n" + rendered2.textContent,
               hasImages: hasImages2,
               viewportWidth: viewport.width,
-              htmlBlocks: [structHtml],
-              tablesDetected: 0
+              htmlBlocks: mergedHtmlBlocks,
+              tablesDetected: rendered2.tablesDetected
             });
             continue;
           }
@@ -23174,6 +23365,12 @@ var PdfAdapter = class {
       const hasNoText = totalTextLength === 0;
       const hasMinimalTextWithImages = totalTextLength < MIN_TEXT_CHARS_THRESHOLD && hasAnyImages;
       if (hasNoText || hasMinimalTextWithImages) {
+        if (getFeatureFlags().enablePdfOcr) {
+          const ocrResult = await this.runOcrOnPages(pdf, numPages);
+          if (ocrResult !== null) {
+            return ocrResult;
+          }
+        }
         throw new PdfParseError(
           "PDF contains only images \u2014 no text layer available.",
           "IMAGE_ONLY"
@@ -23279,6 +23476,65 @@ var PdfAdapter = class {
       }
     }
   }
+  /**
+   * Attempt OCR extraction on all pages of an image-only PDF.
+   *
+   * Renders each page to a canvas, passes them to Tesseract.js (lazy-loaded),
+   * and returns a standard PdfParseResult if text is found.
+   *
+   * Returns null if:
+   * - Tesseract.js cannot be loaded
+   * - OCR produces no usable text across all pages
+   */
+  async runOcrOnPages(pdf, numPages) {
+    const { isOcrAvailable: isOcrAvailable2, runOcr: runOcr2 } = await Promise.resolve().then(() => (init_pdf_ocr_adapter(), pdf_ocr_adapter_exports));
+    if (!await isOcrAvailable2()) {
+      return null;
+    }
+    const SCALE = 2;
+    const pageCanvases = [];
+    const viewportWidths = /* @__PURE__ */ new Map();
+    for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+      const page = await pdf.getPage(pageNum);
+      const viewport = page.getViewport({ scale: 1 });
+      viewportWidths.set(pageNum, viewport.width);
+      const renderViewport = page.getViewport({ scale: SCALE });
+      const canvas = document.createElement("canvas");
+      canvas.width = renderViewport.width;
+      canvas.height = renderViewport.height;
+      const ctx = canvas.getContext("2d");
+      if (!ctx) continue;
+      await page.render({ canvasContext: ctx, viewport: renderViewport }).promise;
+      pageCanvases.push({ pageNumber: pageNum, canvas });
+    }
+    if (pageCanvases.length === 0) return null;
+    const ocrResult = await runOcr2(pageCanvases);
+    const pagesWithText = ocrResult.pages.filter((p) => p.text.length > 0);
+    if (pagesWithText.length === 0) return null;
+    const ocrPages = ocrResult.pages.map((p) => ({
+      pageNumber: p.pageNumber,
+      textContent: p.text,
+      hasImages: true,
+      viewportWidth: viewportWidths.get(p.pageNumber),
+      tablesDetected: 0
+    }));
+    const html = this.pagesToHtml(ocrPages);
+    const allWarnings = [
+      "OCR was used to extract text from this image-only PDF. Quality may vary.",
+      ...ocrResult.warnings
+    ];
+    return {
+      html,
+      pages: ocrPages,
+      metadata: {
+        isTextBased: false,
+        isOcrBased: true,
+        pageCount: numPages
+      },
+      isTextBased: false,
+      warnings: allWarnings
+    };
+  }
 };
 var pdfAdapter = new PdfAdapter();
 
@@ -24682,12 +24938,12 @@ var ToonifyService = class {
       };
     }
     const blocks = this.parseMarkdown(markdown, opts);
-    const document2 = { t: "md", blocks };
+    const document3 = { t: "md", blocks };
     const encodeOptions = {
       keyFolding: opts.keyFolding,
       tableFormat: opts.tableFormat
     };
-    const compressed = ToonEncoder.encode(document2, encodeOptions);
+    const compressed = ToonEncoder.encode(document3, encodeOptions);
     const compressedTokenEstimate = estimateTokens(compressed);
     const reductionPercent = originalTokenEstimate > 0 ? Math.max(0, Math.min(100, Math.round(
       (originalTokenEstimate - compressedTokenEstimate) / originalTokenEstimate * 100
@@ -25505,8 +25761,8 @@ import { readFileSync as readFileSync4 } from "fs";
 import { resolve, dirname as dirname3 } from "path";
 import { fileURLToPath } from "url";
 var getVersion = () => {
-  if ("1.0.24") {
-    return "1.0.24";
+  if ("1.0.25") {
+    return "1.0.25";
   }
   try {
     const __dirname = dirname3(fileURLToPath(import.meta.url));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "formatferry",
-  "version": "1.0.24",
+  "version": "1.0.25",
   "description": "FormatFerry CLI — Convert HTML, DOCX, PDF, XLSX, CSV, JSON, XML, PPTX to Markdown",
   "keywords": [
     "markdown",