kordoc 4.21.19 → 4.21.21

package/dist/{chunk-Y5OSPKBU.js → chunk-YQZOPT3A.js}

@@ -1,15 +1,88 @@
-#!/usr/bin/env node
 import {
   CELL_EDGES,
   HEADING_RATIO_H1,
   HEADING_RATIO_H2,
   HEADING_RATIO_H3,
   tidyScriptTags
-} from "./chunk-X6WXU4LA.js";
+} from "./chunk-KTLHADU4.js";
+import {
+  ensureModelsIn,
+  getModelsDir
+} from "./chunk-FBJVDBMO.js";
 import {
   safeMax,
   safeMin
-} from "./chunk-Z2OOYQ4I.js";
+} from "./chunk-ZZIQWSGC.js";
+
+// src/ocr/models.ts
+import { join } from "path";
+import { stat } from "fs/promises";
+var OCR_DET_MODEL = {
+  name: "PP-OCRv5 mobile det",
+  filename: "det.onnx",
+  url: "https://huggingface.co/PaddlePaddle/PP-OCRv5_mobile_det_onnx/resolve/main/inference.onnx",
+  sha256: "a431985659dc921974177a95adcfbb90fd9e51989a5e04d70d0b75f597b6e61d",
+  sizeMb: 5
+};
+var OCR_REC_MODEL = {
+  name: "PP-OCRv5 korean rec",
+  filename: "rec_korean.onnx",
+  url: "https://huggingface.co/PaddlePaddle/korean_PP-OCRv5_mobile_rec_onnx/resolve/main/inference.onnx",
+  sha256: "92f0b7785e64fc9090106a241cf4c1eb97472824558272751b88a2a4476d3a08",
+  sizeMb: 13
+};
+var OCR_REC_DICT = {
+  name: "PP-OCRv5 korean dict",
+  filename: "rec_korean.yml",
+  url: "https://huggingface.co/PaddlePaddle/korean_PP-OCRv5_mobile_rec_onnx/resolve/main/inference.yml",
+  sha256: "f757fa1c40e99edcf27e9cce879b93eb2a51fa46f5ef39095689b8c37dd75998",
+  sizeMb: 1
+};
+var ALL_OCR_MODELS = [OCR_DET_MODEL, OCR_REC_MODEL, OCR_REC_DICT];
+function getOcrModelsDir() {
+  return getModelsDir("ppocr");
+}
+async function ensureOcrModels(onProgress) {
+  return ensureModelsIn(getOcrModelsDir(), ALL_OCR_MODELS, onProgress);
+}
+async function ocrModelsCached() {
+  const dir = getOcrModelsDir();
+  for (const spec of ALL_OCR_MODELS) {
+    try {
+      if (!(await stat(join(dir, spec.filename))).size) return false;
+    } catch {
+      return false;
+    }
+  }
+  return true;
+}
+function parseCharacterDict(yml) {
+  const lines = yml.split("\n");
+  const chars = [];
+  let inDict = false;
+  let dictIndent = -1;
+  for (const line of lines) {
+    if (!inDict) {
+      const m2 = /^(\s*)character_dict:\s*$/.exec(line);
+      if (m2) {
+        inDict = true;
+        dictIndent = m2[1].length;
+      }
+      continue;
+    }
+    const m = /^(\s*)- (.*)$/.exec(line);
+    if (m && m[1].length >= dictIndent) {
+      let v = m[2];
+      if (v.length >= 2 && (v.startsWith("'") && v.endsWith("'") || v.startsWith('"') && v.endsWith('"'))) {
+        v = v.slice(1, -1).replace(/''/g, "'");
+      }
+      chars.push(v);
+      continue;
+    }
+    if (line.trim() !== "") break;
+  }
+  return chars;
+}
 
 // src/pdf/line-extract.ts
 import { OPS } from "pdfjs-dist/legacy/build/pdf.mjs";
@@ -1871,7 +1944,7 @@ function dropGridsInside(lineGrids, clipGrids, containers = [], verticals = [])
     const ga = area(g);
     return ga > 0 && ix * iy / ga >= 0.5;
   };
-  const inside = (c, g) => c.x1 >= g.x1 - 1 && c.x2 <= g.x2 + 1 && c.y1 >= g.y1 - 1 && c.y2 <= g.y2 + 1;
+  const inside = (c, g) => c.x1 >= g.x1 - 3 && c.x2 <= g.x2 + 3 && c.y1 >= g.y1 - 3 && c.y2 <= g.y2 + 3;
   const stackedOnly = (g) => {
     const stacked = clipGrids.filter((c) => inside(c.bbox, g.bbox)).sort((a, b) => a.bbox.y1 - b.bbox.y1);
     if (stacked.length < 2 || g.colXs.length < 3 || stacked.reduce((s, c) => s + area(c.bbox), 0) < area(g.bbox) * 0.5) return false;
@@ -3342,6 +3415,49 @@ function isExamLayoutTable(table) {
   return layout >= texts.length * 0.6;
 }
 
+// src/pdf/cluster-cell-text.ts
+function clusterCellLines(items) {
+  const hOf = (i) => i.h > 0 ? i.h : i.fontSize;
+  const lines = [];
+  for (const it of [...items].sort((a, b) => b.y - a.y || a.x - b.x)) {
+    const bottom = it.y, top = it.y + hOf(it);
+    const line = lines.find((l) => Math.min(l.top, top) - Math.max(l.bottom, bottom) >= Math.min(l.top - l.bottom, top - bottom) * 0.5);
+    if (line) {
+      line.items.push(it);
+      line.bottom = Math.min(line.bottom, bottom);
+      line.top = Math.max(line.top, top);
+    } else lines.push({ bottom, top, items: [it] });
+  }
+  return lines.sort((a, b) => b.top - a.top).map(({ items: line }) => {
+    line.sort((a, b) => a.x - b.x);
+    let s = line[0].text;
+    for (let i = 1; i < line.length; i++) {
+      const gap = line[i].x - (line[i - 1].x + line[i - 1].w);
+      const fs = (line[i].fontSize + line[i - 1].fontSize) / 2;
+      s += (!isCjkLatinAutospace(line[i - 1].text, line[i].text, gap, fs) && (line[i].hasSpaceBefore && gap >= fs * 0.05 || gap > spaceGapThreshold(fs)) ? " " : "") + line[i].text;
+    }
+    const last = line.reduce((a, b) => b.x + b.w > a.x + a.w ? b : a);
+    return {
+      text: s,
+      left: line[0].x,
+      right: last.x + last.w,
+      fontSize: line[0].fontSize,
+      spaceAfter: last.spaceAfter,
+      firstCharW: line[0].w / Math.max(1, [...line[0].text.replace(/<\/?u>|~~/g, "")].length)
+    };
+  });
+}
+var WRAP_INDENT_MAX = 3;
+function joinClusterCellLines(lines, box, lex) {
+  let out = lines[0]?.text ?? "";
+  for (let i = 1; i < lines.length; i++) {
+    const a = lines[i - 1], b = lines[i];
+    const join2 = !a.spaceAfter && cellLineWraps(box, box.x1, a.right, a.fontSize, b.firstCharW) && a.left - box.x1 <= WRAP_INDENT_MAX * a.fontSize && b.left - box.x1 <= WRAP_INDENT_MAX * b.fontSize && /<\/u>$/.test(a.text) === /^<u>/.test(b.text) && !startsNewItem(out, b.text) && wrapJoiner(out, b.text, lex, false) === "";
+    out += (join2 ? "" : " ") + b.text;
+  }
+  return out;
+}
+
 // src/pdf/cluster-detector.ts
 var Y_TOL = 3;
 var COL_CLUSTER_TOL = 15;
@@ -3350,7 +3466,7 @@ var MIN_COLS = 2;
 var MIN_GAP_FACTOR = 2;
 var MIN_GAP_ABSOLUTE = 20;
 var MIN_COL_FILL_RATIO = 0.4;
-function detectClusterTables(items, pageNum, rejected) {
+function detectClusterTables(items, pageNum, rejected, lex) {
   const tab = sideTabGlyphs(items);
   if (tab.size) items = items.filter((i) => !tab.has(i));
   if (items.length < MIN_ROWS * MIN_COLS) return [];
@@ -3371,7 +3487,7 @@ function detectClusterTables(items, pageNum, rejected) {
     const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
     const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
     for (const region of tableRegions) {
-      const table = buildClusterTable(region.rows, columns, pageNum);
+      const table = buildClusterTable(region.rows, columns, pageNum, lex);
       if (table) {
         expandUsedItems(table.usedItems, originMap);
         results.push(table);
@@ -3386,8 +3502,8 @@ function detectClusterTables(items, pageNum, rejected) {
         const tableRegions = findTableRegions(rows, columns);
         for (const region of tableRegions) {
           const own = extractColumnClusters(region.rows.filter((row) => hasSuspiciousGaps(row)));
-          const ownTable = own.length >= MIN_COLS && own.length < columns.length ? buildClusterTable(mergeMultiLineRows(region.rows, own), own, pageNum) : null;
-          const table = ownTable && ownTable.table.cells.every((row, r) => row.every((cell) => (r === 0 || cell.colSpan >= ownTable.table.cols || cell.text.trim().length <= 30) && !/\p{Co}/u.test(cell.text))) ? ownTable : buildClusterTable(mergeMultiLineRows(region.rows, columns), columns, pageNum);
+          const ownTable = own.length >= MIN_COLS && own.length < columns.length ? buildClusterTable(mergeMultiLineRows(region.rows, own), own, pageNum, lex) : null;
+          const table = ownTable && ownTable.table.cells.every((row, r) => row.every((cell) => (r === 0 || cell.colSpan >= ownTable.table.cols || cell.text.trim().length <= 30) && !/\p{Co}/u.test(cell.text))) ? ownTable : buildClusterTable(mergeMultiLineRows(region.rows, columns), columns, pageNum, lex);
           if (table) {
             expandUsedItems(table.usedItems, originMap);
             results.push(table);
@@ -4084,7 +4200,7 @@ function assignRowItems(items, columns, numCols, headerFace) {
   }
   return result;
 }
-function buildClusterTable(rows, columns, pageNum) {
+function buildClusterTable(rows, columns, pageNum, lex) {
   const numCols = columns.length;
   const numRows = rows.length;
   if (numRows < MIN_ROWS || numCols < MIN_COLS) return null;
@@ -4093,6 +4209,7 @@ function buildClusterTable(rows, columns, pageNum) {
     () => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
   );
   const usedItems = /* @__PURE__ */ new Set();
+  const lines = cells.map((row) => row.map(() => []));
   const headFaces = new Set(rows[0]?.items.map((i) => i.fontName));
   const headerFace = headFaces.size === 1 && rows.slice(1).every((row) => row.items.every((i) => !headFaces.has(i.fontName))) ? [...headFaces][0] : void 0;
   for (let r = 0; r < numRows; r++) {
@@ -4104,12 +4221,18 @@ function buildClusterTable(rows, columns, pageNum) {
     }
     const assignments = assignRowItems(row.items, columns, numCols, r === 0 ? headerFace : void 0);
     for (const { col, items } of assignments) {
-      const text = joinCellItems(items);
-      const existing = cells[r][col].text;
-      cells[r][col].text = existing ? existing + " " + text : text;
+      const cell = lines[r][col];
+      if (cell.length) cell[cell.length - 1].spaceAfter = true;
+      cell.push(...clusterCellLines(items));
       for (const item of items) usedItems.add(item);
     }
   }
+  const boxes = columns.map((_, c) => {
+    const col = lines.flatMap((row) => row[c]);
+    return { x1: Math.min(...col.map((l) => l.left)), x2: Math.max(...col.map((l) => l.right)) };
+  });
+  const textOf = (r, c) => joinClusterCellLines(lines[r][c], boxes[c], lex);
+  for (let r = 0; r < numRows; r++) for (let c = 0; c < numCols; c++) if (lines[r][c].length) cells[r][c].text = textOf(r, c);
   let emptyRows = 0;
   for (const row of cells) {
     if (row.every((c) => c.text === "")) emptyRows++;
@@ -4129,9 +4252,9 @@ function buildClusterTable(rows, columns, pageNum) {
       if (cells[pr].some((c) => c.text.trim())) {
         if (cells[pr][0].colSpan > 1) break;
         for (let c = 0; c < numCols; c++) {
-          const prev = cells[pr][c].text.trim();
-          const curr = cells[r][c].text.trim();
-          if (curr) cells[pr][c].text = prev ? prev + " " + curr : curr;
+          if (!cells[r][c].text.trim()) continue;
+          lines[pr][c].push(...lines[r][c]);
+          cells[pr][c].text = textOf(pr, c);
         }
         for (let c = 0; c < numCols; c++) cells[r][c].text = "";
         break;
@@ -4178,29 +4301,6 @@ function buildClusterTable(rows, columns, pageNum) {
     usedItems
   };
 }
-function joinCellItems(items) {
-  const hOf = (i) => i.h > 0 ? i.h : i.fontSize;
-  const lines = [];
-  for (const it of [...items].sort((a, b) => b.y - a.y || a.x - b.x)) {
-    const bottom = it.y, top = it.y + hOf(it);
-    const line = lines.find((l) => Math.min(l.top, top) - Math.max(l.bottom, bottom) >= Math.min(l.top - l.bottom, top - bottom) * 0.5);
-    if (line) {
-      line.items.push(it);
-      line.bottom = Math.min(line.bottom, bottom);
-      line.top = Math.max(line.top, top);
-    } else lines.push({ bottom, top, items: [it] });
-  }
-  return lines.sort((a, b) => b.top - a.top).map(({ items: line }) => {
-    line.sort((a, b) => a.x - b.x);
-    let s = line[0].text;
-    for (let i = 1; i < line.length; i++) {
-      const gap = line[i].x - (line[i - 1].x + line[i - 1].w);
-      const fs = (line[i].fontSize + line[i - 1].fontSize) / 2;
-      s += (!isCjkLatinAutospace(line[i - 1].text, line[i].text, gap, fs) && (line[i].hasSpaceBefore && gap >= fs * 0.05 || gap > spaceGapThreshold(fs)) ? " " : "") + line[i].text;
-    }
-    return s;
-  }).join(" ");
-}
 
 // src/pdf/page-regions.ts
 function splitTwoColumnProse(items, cutX) {
@@ -5644,6 +5744,20 @@ function detectSpecialKoreanTables(blocks) {
   flushKvTable();
   return result;
 }
+function tracksPages(occ) {
+  const nums = occ.map((o) => [...o.text.matchAll(/\d+/g)].map((m) => +m[0]));
+  if (!nums.length || nums.some((n) => n.length !== nums[0].length)) return false;
+  return nums[0].some((_, j) => new Set(occ.map((o, k) => o.page - nums[k][j])).size === 1);
+}
+function looseVaryingNumber(occ) {
+  const nums = occ.map((o) => [...o.text.matchAll(/\d+/g)]);
+  if (!nums.length || nums.some((n) => n.length !== nums[0].length)) return false;
+  const loose = (text, m) => !/\p{L}/u.test(text[m.index - 1] ?? " ") && !/\p{L}/u.test(text[m.index + m[0].length] ?? " ");
+  for (let j = 0; j < nums[0].length; j++) {
+    if (new Set(nums.map((n) => n[j][0])).size > 1 && occ.every((o, k) => loose(o.text, nums[k][j]))) return true;
+  }
+  return false;
+}
 function removeHeaderFooterBlocks(blocks, pageHeights, warnings, notes, tables = false, context = []) {
   if (context.length && !tables) {
     const removed = removeHeaderFooterBlocks([...blocks, ...context], pageHeights, [], notes).filter((index) => index < blocks.length);
@@ -5673,6 +5787,7 @@ function removeHeaderFooterBlocks(blocks, pageHeights, warnings, notes, tables =
     const patternCount = /* @__PURE__ */ new Map();
     const patternPages = /* @__PURE__ */ new Map();
     const patternNumbers = /* @__PURE__ */ new Map();
+    const patternOcc = /* @__PURE__ */ new Map();
     for (const e of entries) {
       const norm = e.text.replace(/\d+/g, "#");
       patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
@@ -5682,13 +5797,18 @@ function removeHeaderFooterBlocks(blocks, pageHeights, warnings, notes, tables =
       const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
       pages.add(e.page);
       patternPages.set(norm, pages);
+      patternOcc.set(norm, [...patternOcc.get(norm) ?? [], e]);
     }
     const repeatedPatterns = /* @__PURE__ */ new Set();
     for (const [p, count] of patternCount) {
       const pages = [...patternPages.get(p) ?? []];
       const span2 = pages.length ? Math.max(...pages) - Math.min(...pages) + 1 : 0;
-      const pageNumbered = !tables && (patternNumbers.get(p)?.size ?? 0) > 1;
-      if (count >= MIN_REPEAT2 && pages.length >= MIN_REPEAT2 && (pages.length >= span2 * 0.4 || pageNumbered)) {
+      const varying = !tables && (patternNumbers.get(p)?.size ?? 0) > 1;
+      const occ = patternOcc.get(p) ?? [];
+      const dense = pages.length >= span2 * 0.4;
+      const loose = looseVaryingNumber(occ) && (entries === bottomEntries || dense || tracksPages(occ));
+      const pageNumbered = varying && (loose || dense && new Set(occ.map((o) => o.text)).size * 2 <= occ.length);
+      if (count >= MIN_REPEAT2 && pages.length >= MIN_REPEAT2 && (varying ? pageNumbered : dense)) {
         repeatedPatterns.add(p);
       }
     }
@@ -5726,6 +5846,59 @@ function removeHeaderFooterBlocks(blocks, pageHeights, warnings, notes, tables =
   }
   return [...removeSet].sort((a, b) => a - b);
 }
+function runningLinePatterns(removed, pageHeights) {
+  const occ = /* @__PURE__ */ new Map();
+  for (const b of removed) {
+    const ph = b.bbox && b.pageNumber ? pageHeights.get(b.bbox.page) || pageHeights.get(b.pageNumber) : void 0;
+    const key = b.text ? runningKey(b.text) : "";
+    if (!ph || !b.bbox || !b.pageNumber || !/\p{L}/u.test(key) || ph - b.bbox.y <= ph * 0.5) continue;
+    (occ.get(key) ?? occ.set(key, []).get(key)).push({ page: b.pageNumber, nums: (b.text.match(/\d+/g) ?? []).join(",") });
+  }
+  const out = /* @__PURE__ */ new Map();
+  for (const [key, list] of occ) {
+    if (!pageNumberAtEnd(key)) continue;
+    const offsets = new Set(list.map((o) => o.page - +o.nums));
+    if (offsets.size === 1) out.set(key, [...offsets][0]);
+  }
+  return out;
+}
+var runningKey = (s) => s.replace(/\d+/g, "#").replace(/\s+/g, " ").trim();
+var pageNumberAtEnd = (key) => (key.match(/#/g) ?? []).length === 1 && /#\s*[)\]]?$/.test(key);
+function stripAbsorbedRunningLines(blocks, pageHeights, running) {
+  if (!running.size) return blocks;
+  const matches = (line, page) => {
+    const offset = running.get(runningKey(line));
+    return offset !== void 0 && page - +(line.match(/\d+/g) ?? []).pop() === offset;
+  };
+  const out = [];
+  for (const b of blocks) {
+    const ph = b.bbox && b.pageNumber ? pageHeights.get(b.bbox.page) || pageHeights.get(b.pageNumber) : void 0;
+    if (!ph || !b.bbox || !b.pageNumber || b.bbox.y > ph * 0.12) {
+      out.push(b);
+      continue;
+    }
+    if (b.type === "paragraph" && b.text) {
+      const lines = b.text.split("\n");
+      let hi = lines.length;
+      while (hi > 0 && !lines[hi - 1].trim()) hi--;
+      if (hi > 0 && matches(lines[hi - 1], b.pageNumber)) {
+        const text = lines.slice(0, hi - 1).join("\n").trim();
+        if (!text) continue;
+        b.text = text;
+      }
+    } else if (b.type === "table" && b.table && b.table.cells.length) {
+      const t = b.table, last = t.cells.length - 1;
+      if (matches(t.cells[last].map((c) => c.text.trim()).filter(Boolean).join(" "), b.pageNumber) && !t.cells.slice(0, last).some((row, i) => row.some((c) => i + (c.rowSpan || 1) - 1 >= last))) {
+        t.cells.pop();
+        if (!t.cells.length) continue;
+        t.rows = t.cells.length;
+        t.hasHeader = t.hasHeader && t.rows > 1;
+      }
+    }
+    out.push(b);
+  }
+  return out;
+}
 
 // src/pdf/table-meta.ts
 var CLIP_TABLES = /* @__PURE__ */ new WeakSet();
@@ -8201,7 +8374,7 @@ function extractBlocksWithGrids(items, pageNum, pageWidth, pageHeight, grids, ho
       spaceAfter: i.spaceAfter
     }));
     const proseColumns = findTwoColumnProseCutX(clusterItems) !== null || detectPersistentColumnGutter(remaining.map((i) => ({ x: i.x, y: i.y, w: i.w, h: i.h > 0 ? i.h : i.fontSize }))) !== null;
-    const clusterResults = (proseColumns ? [] : detectClusterTables(clusterItems, pageNum)).filter((cr) => {
+    const clusterResults = (proseColumns ? [] : detectClusterTables(clusterItems, pageNum, void 0, lex)).filter((cr) => {
       const b = cr.bbox;
       const overlapX = (x) => Math.min(x.x + x.width, b.x + b.width) - Math.max(x.x, b.x);
       return !blocks.some((x) => x.bbox && x.bbox.y >= b.y - 2 && x.bbox.y + x.bbox.height <= b.y + b.height + 2 && (x.type === "paragraph" ? (x.text?.match(/\p{L}/gu)?.length ?? 0) >= 10 && overlapX(x.bbox) >= 0.8 * b.width : x.type === "table" && !ocrPage && x.table?.cols === cr.table.cols && overlapX(x.bbox) >= 0.5 * Math.min(x.bbox.width, b.width)));
@@ -8553,7 +8726,7 @@ function extractPageBlocksFallback(items, pageNum, fullPage = false, detectTable
     if (regions) return regions.flatMap((region) => extractPageBlocksFallback(region, pageNum, false, detectTables, lex));
   }
   const rejected = { prose: 0 };
-  const clusterResults = detectTables ? detectClusterTables(clusterItems, pageNum, rejected) : [];
+  const clusterResults = detectTables ? detectClusterTables(clusterItems, pageNum, rejected, lex) : [];
   if (clusterResults.length > 0) {
     const ciToIdx = /* @__PURE__ */ new Map();
     for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
@@ -8738,6 +8911,13 @@ function rowParagraphs(block) {
 }
 
 export {
+  OCR_DET_MODEL,
+  OCR_REC_MODEL,
+  OCR_REC_DICT,
+  getOcrModelsDir,
+  ensureOcrModels,
+  ocrModelsCached,
+  parseCharacterDict,
   extractLines,
   extractImageRegions,
   filterHiddenText,
@@ -8767,6 +8947,8 @@ export {
   detectTableCaptions,
   detectKoreanListBlocks,
   removeHeaderFooterBlocks,
+  runningLinePatterns,
+  stripAbsorbedRunningLines,
   CLIP_TABLES,
   FRAME_TITLE_BLOCKS,
   TABLE_COLXS,
@@ -8789,4 +8971,4 @@ export {
   UNRULED_REGION_TABLES,
   mergeOcrImageRegions
 };
-//# sourceMappingURL=chunk-Y5OSPKBU.js.map
+//# sourceMappingURL=chunk-YQZOPT3A.js.map