kordoc 1.5.0 → 1.6.0

package/README.md

@@ -14,7 +14,15 @@
 
 ---
 
-## What's New in v1.5.0
+## What's New in v1.6.0
+
+- **Cluster-Based Table Detection (PDF)** — Detects borderless tables by analyzing text alignment patterns. Baseline grouping + X-coordinate clustering identifies 2+ column tables that line-based detection misses. Sort-and-split clustering for order-independent results.
+- **Korean Special Table Detection** — Automatically detects `구분/항목/종류`-style key-value patterns common in Korean government documents and converts them to structured 2-column tables.
+- **Korean Word-Break Recovery** — Improved merging of broken Korean words in PDF table cells. Handles character-level PDF rendering (micro-gaps between Hangul characters) and cell line-break artifacts up to 8 characters.
+- **Empty Table Filtering** — Tables with all-empty cells (from line detection of decorative borders) are now automatically removed.
+
+<details>
+<summary>v1.5.0 features</summary>
 
 - **Line-Based Table Detection (PDF)** — Ported from OpenDataLoader. Extracts horizontal/vertical lines from PDF graphics commands, builds grid via intersection vertices, maps text to cells by bbox overlap. Proper colspan/rowspan detection. Falls back to heuristic for line-free PDFs.
 - **IRBlock v2** — 6 block types: `heading`, `paragraph`, `table`, `list`, `image`, `separator`. New fields: `bbox`, `style`, `pageNumber`, `level`, `href`, `footnoteText`.
@@ -25,6 +33,8 @@
 - **List Detection** — Numbered paragraphs after tables auto-converted to ordered list blocks.
 - **MCP Server** — Now returns `outline` and `warnings` in parse_document responses.
 
+</details>
+
 <details>
 <summary>v1.4.x features</summary>
 

package/dist/{chunk-5SZWGBNL.js → chunk-TFGOV2ML.js}

@@ -185,7 +185,7 @@ function tableToMarkdown(table) {
 }
 
 // src/utils.ts
-var VERSION = true ? "1.5.0" : "0.0.0-dev";
+var VERSION = true ? "1.6.0" : "0.0.0-dev";
 function toArrayBuffer(buf) {
   if (buf.byteOffset === 0 && buf.byteLength === buf.buffer.byteLength) {
     return buf.buffer;
@@ -1609,14 +1609,25 @@ function cellTextToString(items) {
   lines.push(curLine);
   const textLines = lines.map((line) => {
     const s = line.sort((a, b) => a.x - b.x);
-    return s.map((i) => i.text).join(" ");
+    if (s.length === 1) return s[0].text;
+    let result = s[0].text;
+    for (let j = 1; j < s.length; j++) {
+      const gap = s[j].x - (s[j - 1].x + s[j - 1].w);
+      const avgFs = (s[j].fontSize + s[j - 1].fontSize) / 2;
+      if (gap < avgFs * 0.3 && /[가-힣]$/.test(result) && /^[가-힣]/.test(s[j].text)) {
+        result += s[j].text;
+      } else {
+        result += " " + s[j].text;
+      }
+    }
+    return result;
   });
   if (textLines.length <= 1) return textLines[0] || "";
   const merged = [textLines[0]];
   for (let i = 1; i < textLines.length; i++) {
     const prev = merged[merged.length - 1];
     const curr = textLines[i];
-    if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 5 && !curr.includes(" ")) {
+    if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
       merged[merged.length - 1] = prev + curr;
     } else {
       merged.push(curr);
@@ -1625,6 +1636,181 @@ function cellTextToString(items) {
   return merged.join("\n");
 }
 
+// src/pdf/cluster-detector.ts
+var Y_TOL = 3;
+var COL_CLUSTER_TOL = 15;
+var MIN_ROWS = 3;
+var MIN_COLS = 2;
+var MIN_GAP_FACTOR = 1.5;
+var MIN_COL_FILL_RATIO = 0.3;
+function detectClusterTables(items, pageNum) {
+  if (items.length < MIN_ROWS * MIN_COLS) return [];
+  const rows = groupByBaseline(items);
+  if (rows.length < MIN_ROWS) return [];
+  const suspiciousRows = rows.filter((row) => hasSuspiciousGaps(row));
+  if (suspiciousRows.length < MIN_ROWS) return [];
+  const columns = extractColumnClusters(suspiciousRows);
+  if (columns.length < MIN_COLS) return [];
+  const tableRegions = findTableRegions(rows, columns);
+  const results = [];
+  for (const region of tableRegions) {
+    const table = buildClusterTable(region.rows, columns, pageNum);
+    if (table) results.push(table);
+  }
+  return results;
+}
+function groupByBaseline(items) {
+  if (items.length === 0) return [];
+  const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
+  const rows = [];
+  let curItems = [sorted[0]];
+  let curY = sorted[0].y;
+  for (let i = 1; i < sorted.length; i++) {
+    if (Math.abs(sorted[i].y - curY) <= Y_TOL) {
+      curItems.push(sorted[i]);
+    } else {
+      rows.push({ y: curY, items: curItems });
+      curItems = [sorted[i]];
+      curY = sorted[i].y;
+    }
+  }
+  if (curItems.length > 0) rows.push({ y: curY, items: curItems });
+  return rows;
+}
+function hasSuspiciousGaps(row) {
+  if (row.items.length < 2) return false;
+  const sorted = [...row.items].sort((a, b) => a.x - b.x);
+  const avgFontSize = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
+  const minGap = avgFontSize * MIN_GAP_FACTOR;
+  for (let i = 1; i < sorted.length; i++) {
+    const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
+    if (gap >= minGap) return true;
+  }
+  return false;
+}
+function extractColumnClusters(rows) {
+  const allX = [];
+  for (const row of rows) {
+    for (const item of row.items) allX.push(item.x);
+  }
+  if (allX.length === 0) return [];
+  allX.sort((a, b) => a - b);
+  const clusters = [];
+  let clusterStart = 0;
+  for (let i = 1; i <= allX.length; i++) {
+    if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
+      const slice = allX.slice(clusterStart, i);
+      const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
+      clusters.push({ x: avg, count: slice.length });
+      clusterStart = i;
+    }
+  }
+  const minCount = Math.max(2, Math.floor(rows.length * MIN_COL_FILL_RATIO));
+  return clusters.filter((c) => c.count >= minCount).sort((a, b) => a.x - b.x);
+}
+function findTableRegions(allRows, columns) {
+  const regions = [];
+  let currentRegion = [];
+  for (const row of allRows) {
+    const matchedCols = countMatchedColumns(row, columns);
+    if (matchedCols >= MIN_COLS) {
+      currentRegion.push(row);
+    } else if (row.items.length === 1) {
+      if (currentRegion.length > 0) {
+        currentRegion.push(row);
+      }
+    } else {
+      if (currentRegion.length >= MIN_ROWS) {
+        regions.push({ rows: [...currentRegion] });
+      }
+      currentRegion = [];
+    }
+  }
+  if (currentRegion.length >= MIN_ROWS) {
+    regions.push({ rows: currentRegion });
+  }
+  return regions;
+}
+function countMatchedColumns(row, columns) {
+  const matched = /* @__PURE__ */ new Set();
+  for (const item of row.items) {
+    for (let ci = 0; ci < columns.length; ci++) {
+      if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
+        matched.add(ci);
+        break;
+      }
+    }
+  }
+  return matched.size;
+}
+function assignToColumn(item, columns) {
+  const MAX_DIST = COL_CLUSTER_TOL * 3;
+  let bestCol = -1;
+  let bestDist = Infinity;
+  for (let ci = 0; ci < columns.length; ci++) {
+    const dist = Math.abs(item.x - columns[ci].x);
+    if (dist < bestDist) {
+      bestDist = dist;
+      bestCol = ci;
+    }
+  }
+  return bestDist <= MAX_DIST ? bestCol : -1;
+}
+function buildClusterTable(rows, columns, pageNum) {
+  const numCols = columns.length;
+  const numRows = rows.length;
+  if (numRows < MIN_ROWS || numCols < MIN_COLS) return null;
+  const cells = Array.from(
+    { length: numRows },
+    () => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
+  );
+  const usedItems = /* @__PURE__ */ new Set();
+  for (let r = 0; r < numRows; r++) {
+    const row = rows[r];
+    if (row.items.length === 1 && numCols > 1) {
+      cells[r][0] = { text: row.items[0].text, colSpan: numCols, rowSpan: 1 };
+      usedItems.add(row.items[0]);
+      continue;
+    }
+    for (const item of row.items) {
+      const col = assignToColumn(item, columns);
+      if (col < 0) continue;
+      const existing = cells[r][col].text;
+      cells[r][col].text = existing ? existing + " " + item.text : item.text;
+      usedItems.add(item);
+    }
+  }
+  let emptyRows = 0;
+  for (const row of cells) {
+    if (row.every((c) => c.text === "")) emptyRows++;
+  }
+  if (emptyRows > numRows * 0.5) return null;
+  for (let c = 0; c < numCols; c++) {
+    const hasValue = cells.some((row) => row[c].text !== "");
+    if (!hasValue) return null;
+  }
+  const irTable = {
+    rows: numRows,
+    cols: numCols,
+    cells,
+    hasHeader: numRows > 1
+  };
+  const allItems = rows.flatMap((r) => r.items);
+  let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
+  for (const i of allItems) {
+    if (i.x < minX) minX = i.x;
+    if (i.y < minY) minY = i.y;
+    if (i.x + i.w > maxX) maxX = i.x + i.w;
+    const h = i.h > 0 ? i.h : i.fontSize;
+    if (i.y + h > maxY) maxY = i.y + h;
+  }
+  return {
+    table: irTable,
+    bbox: { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY },
+    usedItems
+  };
+}
+
 // src/pdf/polyfill.ts
 import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
 var g = globalThis;
@@ -1916,6 +2102,8 @@ function extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals) {
       cells: irGrid,
       hasHeader: numRows > 1
     };
+    const hasContent = irGrid.some((row) => row.some((cell) => cell.text.trim() !== ""));
+    if (!hasContent) continue;
     blocks.push({
       type: "table",
       table: irTable,
@@ -1953,19 +2141,28 @@ function extractPageBlocksFallback(items, pageNum) {
     const bbox = computeBBox(items, pageNum);
     blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(items) });
   } else {
-    const allY = items.map((i) => i.y);
-    const pageHeight = Math.max(...allY) - Math.min(...allY);
-    const gapThreshold = Math.max(15, pageHeight * 0.03);
-    const orderedGroups = xyCutOrder(items, gapThreshold);
-    for (const group of orderedGroups) {
-      if (group.length === 0) continue;
-      const yLines = groupByY(group);
-      const groupColumns = detectColumns(yLines);
-      if (groupColumns && groupColumns.length >= 3) {
-        const tableText = extractWithColumns(yLines, groupColumns);
-        const bbox = computeBBox(group, pageNum);
-        blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
-      } else {
+    const clusterItems = items.map((i) => ({
+      text: i.text,
+      x: i.x,
+      y: i.y,
+      w: i.w,
+      h: i.h,
+      fontSize: i.fontSize,
+      fontName: i.fontName
+    }));
+    const clusterResults = detectClusterTables(clusterItems, pageNum);
+    if (clusterResults.length > 0) {
+      const usedIndices = /* @__PURE__ */ new Set();
+      for (const cr of clusterResults) {
+        for (const ci of cr.usedItems) {
+          const idx = clusterItems.indexOf(ci);
+          if (idx >= 0) usedIndices.add(idx);
+        }
+        blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
+      }
+      const remaining = items.filter((_, idx) => !usedIndices.has(idx));
+      if (remaining.length > 0) {
+        const yLines = groupByY(remaining);
         for (const line of yLines) {
           const text = mergeLineSimple(line);
           if (!text.trim()) continue;
@@ -1973,9 +2170,36 @@ function extractPageBlocksFallback(items, pageNum) {
           blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
         }
       }
+      blocks.sort((a, b) => {
+        const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
+        const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
+        return by - ay;
+      });
+    } else {
+      const allY = items.map((i) => i.y);
+      const pageHeight = Math.max(...allY) - Math.min(...allY);
+      const gapThreshold = Math.max(15, pageHeight * 0.03);
+      const orderedGroups = xyCutOrder(items, gapThreshold);
+      for (const group of orderedGroups) {
+        if (group.length === 0) continue;
+        const yLines = groupByY(group);
+        const groupColumns = detectColumns(yLines);
+        if (groupColumns && groupColumns.length >= 3) {
+          const tableText = extractWithColumns(yLines, groupColumns);
+          const bbox = computeBBox(group, pageNum);
+          blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
+        } else {
+          for (const line of yLines) {
+            const text = mergeLineSimple(line);
+            if (!text.trim()) continue;
+            const bbox = computeBBox(line, pageNum);
+            blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
+          }
+        }
+      }
     }
   }
-  return blocks;
+  return detectSpecialKoreanTables(blocks);
 }
 function computeBBox(items, pageNum) {
   let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
@@ -2241,8 +2465,10 @@ function mergeLineSimple(items) {
   let result = sorted[0].text;
   for (let i = 1; i < sorted.length; i++) {
     const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
+    const avgFs = (sorted[i].fontSize + sorted[i - 1].fontSize) / 2;
     if (gap > 15) result += "	";
-    else if (gap > 3) result += " ";
+    else if (gap < avgFs * 0.3 && /[가-힣]$/.test(result) && /^[가-힣]/.test(sorted[i].text)) {
+    } else if (gap > 3) result += " ";
     result += sorted[i].text;
   }
   return result;
@@ -2280,6 +2506,90 @@ function detectListBlocks(blocks) {
   }
   return result;
 }
+var KOREAN_TABLE_HEADER_RE = /^\(?(구분|항목|종류|분류|유형|대상|내용|기간|금액|비율|방법|절차|요건|조건|근거|목적|범위|기준)\)?[:\s]/;
+function detectSpecialKoreanTables(blocks) {
+  const result = [];
+  let kvLines = [];
+  const flushKvTable = () => {
+    if (kvLines.length < 2) {
+      for (const kv of kvLines) result.push(kv.block);
+      kvLines = [];
+      return;
+    }
+    const cells = kvLines.map((kv) => {
+      if (kv.value) {
+        return [
+          { text: kv.key, colSpan: 1, rowSpan: 1 },
+          { text: kv.value, colSpan: 1, rowSpan: 1 }
+        ];
+      }
+      return [
+        { text: kv.key, colSpan: 2, rowSpan: 1 },
+        { text: "", colSpan: 1, rowSpan: 1 }
+      ];
+    });
+    const irTable = {
+      rows: cells.length,
+      cols: 2,
+      cells,
+      hasHeader: true
+    };
+    const firstBlock = kvLines[0].block;
+    result.push({
+      type: "table",
+      table: irTable,
+      pageNumber: firstBlock.pageNumber,
+      bbox: firstBlock.bbox
+    });
+    kvLines = [];
+  };
+  for (const block of blocks) {
+    if (block.type !== "paragraph" || !block.text) {
+      flushKvTable();
+      result.push(block);
+      continue;
+    }
+    const text = block.text.trim();
+    if (KOREAN_TABLE_HEADER_RE.test(text)) {
+      const colonIdx = text.indexOf(":");
+      if (colonIdx >= 0) {
+        kvLines.push({
+          key: text.slice(0, colonIdx).trim(),
+          value: text.slice(colonIdx + 1).trim(),
+          block
+        });
+      } else {
+        const spaceIdx = text.search(/\s/);
+        if (spaceIdx > 0) {
+          kvLines.push({
+            key: text.slice(0, spaceIdx).trim(),
+            value: text.slice(spaceIdx + 1).trim(),
+            block
+          });
+        } else {
+          kvLines.push({ key: text, value: "", block });
+        }
+      }
+      continue;
+    }
+    if (kvLines.length > 0 && text.includes(":") && !text.includes("(") && !text.includes(")")) {
+      const colonIdx = text.indexOf(":");
+      const key = text.slice(0, colonIdx).trim();
+      if (/^[가-힣]+$/.test(key) && key.length >= 2 && key.length <= 8) {
+        kvLines.push({
+          key,
+          value: text.slice(colonIdx + 1).trim(),
+          block
+        });
+        continue;
+      }
+    }
+    flushKvTable();
+    result.push(block);
+  }
+  flushKvTable();
+  return result;
+}
 function mergeKoreanLines(text) {
   if (!text) return "";
   const lines = text.split("\n");
@@ -2654,4 +2964,4 @@ export {
   extractFormFields,
   parse
 };
-//# sourceMappingURL=chunk-5SZWGBNL.js.map
+//# sourceMappingURL=chunk-TFGOV2ML.js.map