kordoc 2.8.0 → 2.9.0

package/dist/{parser-ZQQM6J7T.js → parser-OMPBVEFU.js}

@@ -6,7 +6,7 @@ import {
   blocksToMarkdown,
   safeMax,
   safeMin
-} from "./chunk-DLQY6FJH.js";
+} from "./chunk-RXZLTACX.js";
 import {
   parsePageRange
 } from "./chunk-SBVRCJFH.js";
@@ -1142,6 +1142,120 @@ function buildClusterTable(rows, columns, pageNum) {
   };
 }
 
+// src/pdf/quality.ts
+function computePageQuality(page, text) {
+  let total = 0;
+  let hangul = 0;
+  let control = 0;
+  let replacement = 0;
+  let pua = 0;
+  for (let i = 0; i < text.length; i++) {
+    const code = text.charCodeAt(i);
+    if (code === 32 || code === 9 || code === 10 || code === 13) continue;
+    total++;
+    if (code < 32 || code === 127 || code >= 128 && code <= 159) {
+      control++;
+      continue;
+    }
+    if (code === 65533) {
+      replacement++;
+      continue;
+    }
+    if (code >= 44032 && code <= 55203) {
+      hangul++;
+      continue;
+    }
+    if (code >= 57344 && code <= 63743 || code >= 56192 && code <= 56319) {
+      pua++;
+      continue;
+    }
+  }
+  const denom = total || 1;
+  const puaRatio = pua / denom;
+  const controlCharRatio = control / denom;
+  const replacementCharRatio = replacement / denom;
+  let needsOcr = false;
+  let ocrReason;
+  if (total < LOW_TEXT_THRESHOLD) {
+    needsOcr = true;
+    ocrReason = "low_text";
+  } else if (puaRatio >= HIGH_PUA_THRESHOLD) {
+    needsOcr = true;
+    ocrReason = "high_pua";
+  } else if (controlCharRatio >= HIGH_CONTROL_THRESHOLD) {
+    needsOcr = true;
+    ocrReason = "high_control";
+  } else if (replacementCharRatio >= HIGH_REPLACEMENT_THRESHOLD) {
+    needsOcr = true;
+    ocrReason = "high_replacement";
+  }
+  return {
+    page,
+    textChars: total,
+    hangulRatio: hangul / denom,
+    controlCharRatio,
+    replacementCharRatio,
+    puaRatio,
+    needsOcr,
+    ocrReason
+  };
+}
+var LOW_TEXT_THRESHOLD = 20;
+var HIGH_PUA_THRESHOLD = 0.2;
+var HIGH_CONTROL_THRESHOLD = 0.05;
+var HIGH_REPLACEMENT_THRESHOLD = 0.05;
+var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
+function stripControlChars(text) {
+  return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
+}
+function summarizeDocumentQuality(pages) {
+  if (pages.length === 0) {
+    return {
+      totalPages: 0,
+      totalTextChars: 0,
+      avgHangulRatio: 0,
+      avgControlCharRatio: 0,
+      avgReplacementCharRatio: 0,
+      avgPuaRatio: 0,
+      lowTextPageCount: 0,
+      highPuaPageCount: 0,
+      needsOcr: false,
+      ocrCandidatePages: []
+    };
+  }
+  let textChars = 0;
+  let hangul = 0;
+  let control = 0;
+  let replacement = 0;
+  let pua = 0;
+  let lowText = 0;
+  let highPua = 0;
+  const ocrCandidatePages = [];
+  for (const p of pages) {
+    textChars += p.textChars;
+    hangul += p.hangulRatio;
+    control += p.controlCharRatio;
+    replacement += p.replacementCharRatio;
+    pua += p.puaRatio;
+    if (p.textChars < LOW_TEXT_THRESHOLD) lowText++;
+    if (p.puaRatio >= HIGH_PUA_THRESHOLD) highPua++;
+    if (p.needsOcr) ocrCandidatePages.push(p.page);
+  }
+  const n = pages.length;
+  return {
+    totalPages: n,
+    totalTextChars: textChars,
+    avgHangulRatio: hangul / n,
+    avgControlCharRatio: control / n,
+    avgReplacementCharRatio: replacement / n,
+    avgPuaRatio: pua / n,
+    lowTextPageCount: lowText,
+    highPuaPageCount: highPua,
+    needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
+    ocrCandidatePages
+  };
+}
+
 // src/pdf/polyfill.ts
 import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
 var g = globalThis;
@@ -1197,6 +1311,7 @@ async function parsePdfDocument(buffer, options) {
     await extractPdfMetadata(doc, metadata);
     const blocks = [];
     const warnings = [];
+    const pageQuality = [];
     let totalChars = 0;
     let totalTextBytes = 0;
     const effectivePageCount = Math.min(pageCount, MAX_PAGES);
@@ -1224,11 +1339,14 @@ async function parsePdfDocument(buffer, options) {
         const opList = await page.getOperatorList();
         const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
         for (const b of pageBlocks) blocks.push(b);
+        let pageText = "";
         for (const b of pageBlocks) {
           const t = b.text || "";
           totalChars += t.replace(/\s/g, "").length;
           totalTextBytes += t.length * 2;
+          pageText += pageText ? "\n" + t : t;
         }
+        pageQuality.push(computePageQuality(i, pageText));
         if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
         parsedPages++;
         options?.onProgress?.(parsedPages, totalTarget);
@@ -1241,11 +1359,11 @@ async function parsePdfDocument(buffer, options) {
     if (totalChars / Math.max(parsedPageCount, 1) < 10) {
       if (options?.ocr) {
         try {
-          const { ocrPages } = await import("./provider-WPIYEALY.js");
+          const { ocrPages } = await import("./provider-2SEHU2FM.js");
           const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
           if (ocrBlocks.length > 0) {
             const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
-            return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true };
+            return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
           }
         } catch {
         }
@@ -1274,8 +1392,17 @@ async function parsePdfDocument(buffer, options) {
     }
     detectMarkerHeadings(blocks);
     const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
+    sanitizeBlockControlChars(blocks);
     let markdown = cleanPdfText(blocksToMarkdown(blocks));
-    return { markdown, blocks, metadata, outline: outline.length > 0 ? outline : void 0, warnings: warnings.length > 0 ? warnings : void 0 };
+    return {
+      markdown,
+      blocks,
+      metadata,
+      outline: outline.length > 0 ? outline : void 0,
+      warnings: warnings.length > 0 ? warnings : void 0,
+      pageQuality,
+      qualitySummary: summarizeDocumentQuality(pageQuality)
+    };
   } finally {
     await doc.destroy().catch(() => {
     });
@@ -2079,9 +2206,22 @@ function mergeLineSimple(items) {
   }
   return result;
 }
+function sanitizeBlockControlChars(blocks) {
+  for (const b of blocks) {
+    if (b.text) b.text = stripControlChars(b.text);
+    if (b.table) {
+      for (const row of b.table.cells) {
+        for (const cell of row) {
+          if (cell.text) cell.text = stripControlChars(cell.text);
+        }
+      }
+    }
+    if (b.children) sanitizeBlockControlChars(b.children);
+  }
+}
 function cleanPdfText(text) {
   return mergeKoreanLines(
-    text.replace(/^\d{1,4}\n/, "").replace(/^[\s]*[-–—]\s*[-–—]?\d+[-–—]?[\s]*[-–—]?[\s]*$/gm, "").replace(/^\s*\d+\s*\/\s*\d+\s*$/gm, "").replace(/\n\d{1,4}\n/g, "\n").replace(/\n\d{1,4}$/, "").replace(/^#{1,6}\s*\d{1,4}\s*$/gm, "")
+    stripControlChars(text).replace(/^\d{1,4}\n/, "").replace(/^[\s]*[-–—]\s*[-–—]?\d+[-–—]?[\s]*[-–—]?[\s]*$/gm, "").replace(/^\s*\d+\s*\/\s*\d+\s*$/gm, "").replace(/\n\d{1,4}\n/g, "\n").replace(/\n\d{1,4}$/, "").replace(/^#{1,6}\s*\d{1,4}\s*$/gm, "")
   ).replace(/^(?!\| ---).*$/gm, (line) => {
     if (/^\s*\${1,2}.+\${1,2}\s*$/.test(line)) return line;
     return collapseEvenSpacing(line);
@@ -2411,4 +2551,4 @@ export {
   extractPdfMetadataOnly,
   parsePdfDocument
 };
-//# sourceMappingURL=parser-ZQQM6J7T.js.map
+//# sourceMappingURL=parser-OMPBVEFU.js.map