kordoc 3.7.0 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/{-7UC4ZWBS.js → -LD4BZDDJ.js} +3 -3
- package/dist/{chunk-IJJMVTU5.js → chunk-IFYJFWD2.js} +2 -2
- package/dist/{chunk-NXRABCWW.js → chunk-KT2BCHXI.js} +803 -703
- package/dist/chunk-KT2BCHXI.js.map +1 -0
- package/dist/{chunk-QZCP3UWU.cjs → chunk-LFCS3UVG.cjs} +2 -2
- package/dist/{chunk-QZCP3UWU.cjs.map → chunk-LFCS3UVG.cjs.map} +1 -1
- package/dist/{chunk-MEVKYW55.js → chunk-PELBIL4K.js} +2 -2
- package/dist/cli.js +4 -4
- package/dist/index.cjs +968 -868
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +802 -702
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +3 -3
- package/dist/{parser-DR5CTZ74.js → parser-FFEBMLSH.js} +1830 -1803
- package/dist/parser-FFEBMLSH.js.map +1 -0
- package/dist/{parser-RFLPUZ7P.cjs → parser-IXK5V7YG.cjs} +1830 -1803
- package/dist/parser-IXK5V7YG.cjs.map +1 -0
- package/dist/{parser-ZHJFQR44.js → parser-XEDROIM7.js} +1830 -1803
- package/dist/parser-XEDROIM7.js.map +1 -0
- package/dist/{watch-UIX447QV.js → watch-MAWCDNFI.js} +3 -3
- package/package.json +2 -2
- package/dist/chunk-NXRABCWW.js.map +0 -1
- package/dist/parser-DR5CTZ74.js.map +0 -1
- package/dist/parser-RFLPUZ7P.cjs.map +0 -1
- package/dist/parser-ZHJFQR44.js.map +0 -1
- /package/dist/{-7UC4ZWBS.js.map → -LD4BZDDJ.js.map} +0 -0
- /package/dist/{chunk-IJJMVTU5.js.map → chunk-IFYJFWD2.js.map} +0 -0
- /package/dist/{chunk-MEVKYW55.js.map → chunk-PELBIL4K.js.map} +0 -0
- /package/dist/{watch-UIX447QV.js.map → watch-MAWCDNFI.js.map} +0 -0
|
@@ -6,22 +6,16 @@ import {
|
|
|
6
6
|
blocksToMarkdown,
|
|
7
7
|
safeMax,
|
|
8
8
|
safeMin
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-PELBIL4K.js";
|
|
10
10
|
import {
|
|
11
11
|
parsePageRange
|
|
12
12
|
} from "./chunk-GE43BE46.js";
|
|
13
13
|
|
|
14
|
-
// src/pdf/line-
|
|
14
|
+
// src/pdf/line-extract.ts
|
|
15
15
|
import { OPS } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
16
16
|
var ORIENTATION_TOL = 2;
|
|
17
17
|
var MIN_LINE_LENGTH = 15;
|
|
18
18
|
var MAX_LINE_WIDTH = 5;
|
|
19
|
-
var CONNECT_TOL = 5;
|
|
20
|
-
var CELL_PADDING = 2;
|
|
21
|
-
var MIN_COL_WIDTH = 15;
|
|
22
|
-
var MIN_ROW_HEIGHT = 6;
|
|
23
|
-
var VERTEX_MERGE_FACTOR = 4;
|
|
24
|
-
var MIN_COORD_MERGE_TOL = 8;
|
|
25
19
|
function extractLines(fnArray, argsArray) {
|
|
26
20
|
const horizontals = [];
|
|
27
21
|
const verticals = [];
|
|
@@ -156,55 +150,6 @@ function extractLines(fnArray, argsArray) {
|
|
|
156
150
|
}
|
|
157
151
|
return { horizontals, verticals };
|
|
158
152
|
}
|
|
159
|
-
function multiplyTransform(m, t) {
|
|
160
|
-
return [
|
|
161
|
-
m[0] * t[0] + m[2] * t[1],
|
|
162
|
-
m[1] * t[0] + m[3] * t[1],
|
|
163
|
-
m[0] * t[2] + m[2] * t[3],
|
|
164
|
-
m[1] * t[2] + m[3] * t[3],
|
|
165
|
-
m[0] * t[4] + m[2] * t[5] + m[4],
|
|
166
|
-
m[1] * t[4] + m[3] * t[5] + m[5]
|
|
167
|
-
];
|
|
168
|
-
}
|
|
169
|
-
function extractImageRegions(fnArray, argsArray) {
|
|
170
|
-
const regions = [];
|
|
171
|
-
let ctm = [1, 0, 0, 1, 0, 0];
|
|
172
|
-
const stack = [];
|
|
173
|
-
for (let i = 0; i < fnArray.length; i++) {
|
|
174
|
-
const op = fnArray[i];
|
|
175
|
-
switch (op) {
|
|
176
|
-
case OPS.save:
|
|
177
|
-
stack.push(ctm);
|
|
178
|
-
break;
|
|
179
|
-
case OPS.restore:
|
|
180
|
-
ctm = stack.pop() || [1, 0, 0, 1, 0, 0];
|
|
181
|
-
break;
|
|
182
|
-
case OPS.transform: {
|
|
183
|
-
const t = argsArray[i];
|
|
184
|
-
if (Array.isArray(t) && t.length >= 6) ctm = multiplyTransform(ctm, t);
|
|
185
|
-
break;
|
|
186
|
-
}
|
|
187
|
-
case OPS.paintImageXObject:
|
|
188
|
-
case OPS.paintInlineImageXObject:
|
|
189
|
-
case OPS.paintImageMaskXObject:
|
|
190
|
-
case OPS.paintImageXObjectRepeat: {
|
|
191
|
-
const corners = [[0, 0], [1, 0], [0, 1], [1, 1]];
|
|
192
|
-
let x1 = Infinity, y1 = Infinity, x2 = -Infinity, y2 = -Infinity;
|
|
193
|
-
for (const [u, v] of corners) {
|
|
194
|
-
const x = ctm[0] * u + ctm[2] * v + ctm[4];
|
|
195
|
-
const y = ctm[1] * u + ctm[3] * v + ctm[5];
|
|
196
|
-
if (x < x1) x1 = x;
|
|
197
|
-
if (x > x2) x2 = x;
|
|
198
|
-
if (y < y1) y1 = y;
|
|
199
|
-
if (y > y2) y2 = y;
|
|
200
|
-
}
|
|
201
|
-
if (x2 - x1 > 0 && y2 - y1 > 0) regions.push({ x1, y1, x2, y2 });
|
|
202
|
-
break;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
return regions;
|
|
207
|
-
}
|
|
208
153
|
function classifyAndAdd(seg, lineWidth, horizontals, verticals) {
|
|
209
154
|
const dx = Math.abs(seg.x2 - seg.x1);
|
|
210
155
|
const dy = Math.abs(seg.y2 - seg.y1);
|
|
@@ -282,6 +227,67 @@ function filterPageBorderLines(horizontals, verticals, pageWidth, pageHeight) {
|
|
|
282
227
|
)
|
|
283
228
|
};
|
|
284
229
|
}
|
|
230
|
+
|
|
231
|
+
// src/pdf/image-regions.ts
|
|
232
|
+
import { OPS as OPS2 } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
233
|
+
function multiplyTransform(m, t) {
|
|
234
|
+
return [
|
|
235
|
+
m[0] * t[0] + m[2] * t[1],
|
|
236
|
+
m[1] * t[0] + m[3] * t[1],
|
|
237
|
+
m[0] * t[2] + m[2] * t[3],
|
|
238
|
+
m[1] * t[2] + m[3] * t[3],
|
|
239
|
+
m[0] * t[4] + m[2] * t[5] + m[4],
|
|
240
|
+
m[1] * t[4] + m[3] * t[5] + m[5]
|
|
241
|
+
];
|
|
242
|
+
}
|
|
243
|
+
function extractImageRegions(fnArray, argsArray) {
|
|
244
|
+
const regions = [];
|
|
245
|
+
let ctm = [1, 0, 0, 1, 0, 0];
|
|
246
|
+
const stack = [];
|
|
247
|
+
for (let i = 0; i < fnArray.length; i++) {
|
|
248
|
+
const op = fnArray[i];
|
|
249
|
+
switch (op) {
|
|
250
|
+
case OPS2.save:
|
|
251
|
+
stack.push(ctm);
|
|
252
|
+
break;
|
|
253
|
+
case OPS2.restore:
|
|
254
|
+
ctm = stack.pop() || [1, 0, 0, 1, 0, 0];
|
|
255
|
+
break;
|
|
256
|
+
case OPS2.transform: {
|
|
257
|
+
const t = argsArray[i];
|
|
258
|
+
if (Array.isArray(t) && t.length >= 6) ctm = multiplyTransform(ctm, t);
|
|
259
|
+
break;
|
|
260
|
+
}
|
|
261
|
+
case OPS2.paintImageXObject:
|
|
262
|
+
case OPS2.paintInlineImageXObject:
|
|
263
|
+
case OPS2.paintImageMaskXObject:
|
|
264
|
+
case OPS2.paintImageXObjectRepeat: {
|
|
265
|
+
const corners = [[0, 0], [1, 0], [0, 1], [1, 1]];
|
|
266
|
+
let x1 = Infinity, y1 = Infinity, x2 = -Infinity, y2 = -Infinity;
|
|
267
|
+
for (const [u, v] of corners) {
|
|
268
|
+
const x = ctm[0] * u + ctm[2] * v + ctm[4];
|
|
269
|
+
const y = ctm[1] * u + ctm[3] * v + ctm[5];
|
|
270
|
+
if (x < x1) x1 = x;
|
|
271
|
+
if (x > x2) x2 = x;
|
|
272
|
+
if (y < y1) y1 = y;
|
|
273
|
+
if (y > y2) y2 = y;
|
|
274
|
+
}
|
|
275
|
+
if (x2 - x1 > 0 && y2 - y1 > 0) regions.push({ x1, y1, x2, y2 });
|
|
276
|
+
break;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return regions;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// src/pdf/line-types.ts
|
|
284
|
+
var VERTEX_MERGE_FACTOR = 4;
|
|
285
|
+
|
|
286
|
+
// src/pdf/table-grid.ts
|
|
287
|
+
var CONNECT_TOL = 5;
|
|
288
|
+
var MIN_COL_WIDTH = 15;
|
|
289
|
+
var MIN_ROW_HEIGHT = 6;
|
|
290
|
+
var MIN_COORD_MERGE_TOL = 8;
|
|
285
291
|
function buildVertices(horizontals, verticals) {
|
|
286
292
|
const vertices = [];
|
|
287
293
|
const tol = CONNECT_TOL;
|
|
@@ -493,6 +499,8 @@ function linesIntersect(a, b) {
|
|
|
493
499
|
const tol = CONNECT_TOL;
|
|
494
500
|
return v.x1 >= h.x1 - tol && v.x1 <= h.x2 + tol && h.y1 >= v.y1 - tol && h.y1 <= v.y2 + tol;
|
|
495
501
|
}
|
|
502
|
+
|
|
503
|
+
// src/pdf/cell-extract.ts
|
|
496
504
|
function extractCells(grid, horizontals, verticals) {
|
|
497
505
|
const { rowYs, colXs } = grid;
|
|
498
506
|
const numRows = rowYs.length - 1;
|
|
@@ -590,6 +598,9 @@ function hasHorizontalLine(horizontals, y, leftX, rightX, vertexRadius) {
|
|
|
590
598
|
}
|
|
591
599
|
return false;
|
|
592
600
|
}
|
|
601
|
+
|
|
602
|
+
// src/pdf/cell-text.ts
|
|
603
|
+
var CELL_PADDING = 2;
|
|
593
604
|
var SPACE_GAP_RATIO = 0.17;
|
|
594
605
|
function spaceGapThreshold(fontSize) {
|
|
595
606
|
return Math.max(fontSize * SPACE_GAP_RATIO, 1);
|
|
@@ -718,6 +729,28 @@ function markEvenRun(items, result, start, end) {
|
|
|
718
729
|
}
|
|
719
730
|
}
|
|
720
731
|
}
|
|
732
|
+
function mergeCellTextLines(textLines) {
|
|
733
|
+
if (textLines.length <= 1) return textLines[0] || "";
|
|
734
|
+
const merged = [textLines[0]];
|
|
735
|
+
for (let i = 1; i < textLines.length; i++) {
|
|
736
|
+
const prev = merged[merged.length - 1];
|
|
737
|
+
const curr = textLines[i];
|
|
738
|
+
if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
|
|
739
|
+
merged[merged.length - 1] = prev + curr;
|
|
740
|
+
} else if (curr.trim().length <= 3 && /^[)\]%}]/.test(curr.trim())) {
|
|
741
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
742
|
+
} else if (/[,(]$/.test(prev.trim()) && curr.trim().length <= 15) {
|
|
743
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
744
|
+
} else if (/[\d,]$/.test(prev) && /^[\d,]+[)\]]?$/.test(curr.trim()) && curr.trim().length <= 10) {
|
|
745
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
746
|
+
} else {
|
|
747
|
+
merged.push(curr);
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
return merged.join("\n");
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// src/pdf/undersegmented.ts
|
|
721
754
|
var MAX_UNDERSEGMENTED_ROWS = 2;
|
|
722
755
|
var MIN_UNDERSEGMENTED_COLUMNS = 3;
|
|
723
756
|
var MIN_UNDERSEGMENTED_TEXT_LINES = 8;
|
|
@@ -831,1608 +864,981 @@ function normalizeUndersegmentedTable(originalCells, colXs, items) {
|
|
|
831
864
|
if (countNonEmptyCols(rebuilt, numCols) < countNonEmptyCols(originalCells, numCols)) return null;
|
|
832
865
|
return rebuilt;
|
|
833
866
|
}
|
|
834
|
-
function mergeCellTextLines(textLines) {
|
|
835
|
-
if (textLines.length <= 1) return textLines[0] || "";
|
|
836
|
-
const merged = [textLines[0]];
|
|
837
|
-
for (let i = 1; i < textLines.length; i++) {
|
|
838
|
-
const prev = merged[merged.length - 1];
|
|
839
|
-
const curr = textLines[i];
|
|
840
|
-
if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
|
|
841
|
-
merged[merged.length - 1] = prev + curr;
|
|
842
|
-
} else if (curr.trim().length <= 3 && /^[)\]%}]/.test(curr.trim())) {
|
|
843
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
844
|
-
} else if (/[,(]$/.test(prev.trim()) && curr.trim().length <= 15) {
|
|
845
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
846
|
-
} else if (/[\d,]$/.test(prev) && /^[\d,]+[)\]]?$/.test(curr.trim()) && curr.trim().length <= 10) {
|
|
847
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
848
|
-
} else {
|
|
849
|
-
merged.push(curr);
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
return merged.join("\n");
|
|
853
|
-
}
|
|
854
867
|
|
|
855
|
-
// src/pdf/
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
const headerResult = detectHeaderRow(rows);
|
|
870
|
-
if (headerResult) {
|
|
871
|
-
const { columns, headerIdx } = headerResult;
|
|
872
|
-
const headerRow = rows[headerIdx];
|
|
873
|
-
const headerItems = [...headerRow.items].sort((a, b) => a.x - b.x);
|
|
874
|
-
const headerAndBelow = rows.slice(headerIdx);
|
|
875
|
-
const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
|
|
876
|
-
const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
|
|
877
|
-
for (const region of tableRegions) {
|
|
878
|
-
const table = buildClusterTable(region.rows, columns, pageNum);
|
|
879
|
-
if (table) {
|
|
880
|
-
expandUsedItems(table.usedItems, originMap);
|
|
881
|
-
results.push(table);
|
|
882
|
-
}
|
|
868
|
+
// src/pdf/quality.ts
|
|
869
|
+
function computePageQuality(page, text) {
|
|
870
|
+
let total = 0;
|
|
871
|
+
let hangul = 0;
|
|
872
|
+
let control = 0;
|
|
873
|
+
let replacement = 0;
|
|
874
|
+
let pua = 0;
|
|
875
|
+
for (let i = 0; i < text.length; i++) {
|
|
876
|
+
const code = text.charCodeAt(i);
|
|
877
|
+
if (code === 32 || code === 9 || code === 10 || code === 13) continue;
|
|
878
|
+
total++;
|
|
879
|
+
if (code < 32 || code === 127 || code >= 128 && code <= 159) {
|
|
880
|
+
control++;
|
|
881
|
+
continue;
|
|
883
882
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
expandUsedItems(table.usedItems, originMap);
|
|
896
|
-
results.push(table);
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
}
|
|
883
|
+
if (code === 65533) {
|
|
884
|
+
replacement++;
|
|
885
|
+
continue;
|
|
886
|
+
}
|
|
887
|
+
if (code >= 44032 && code <= 55203) {
|
|
888
|
+
hangul++;
|
|
889
|
+
continue;
|
|
890
|
+
}
|
|
891
|
+
if (code >= 57344 && code <= 63743 || code >= 56192 && code <= 56319) {
|
|
892
|
+
pua++;
|
|
893
|
+
continue;
|
|
900
894
|
}
|
|
901
895
|
}
|
|
902
|
-
|
|
896
|
+
const denom = total || 1;
|
|
897
|
+
const puaRatio = pua / denom;
|
|
898
|
+
const controlCharRatio = control / denom;
|
|
899
|
+
const replacementCharRatio = replacement / denom;
|
|
900
|
+
let needsOcr = false;
|
|
901
|
+
let ocrReason;
|
|
902
|
+
if (total < LOW_TEXT_THRESHOLD) {
|
|
903
|
+
needsOcr = true;
|
|
904
|
+
ocrReason = "low_text";
|
|
905
|
+
} else if (puaRatio >= HIGH_PUA_THRESHOLD) {
|
|
906
|
+
needsOcr = true;
|
|
907
|
+
ocrReason = "high_pua";
|
|
908
|
+
} else if (controlCharRatio >= HIGH_CONTROL_THRESHOLD) {
|
|
909
|
+
needsOcr = true;
|
|
910
|
+
ocrReason = "high_control";
|
|
911
|
+
} else if (replacementCharRatio >= HIGH_REPLACEMENT_THRESHOLD) {
|
|
912
|
+
needsOcr = true;
|
|
913
|
+
ocrReason = "high_replacement";
|
|
914
|
+
}
|
|
915
|
+
return {
|
|
916
|
+
page,
|
|
917
|
+
textChars: total,
|
|
918
|
+
hangulRatio: hangul / denom,
|
|
919
|
+
controlCharRatio,
|
|
920
|
+
replacementCharRatio,
|
|
921
|
+
puaRatio,
|
|
922
|
+
needsOcr,
|
|
923
|
+
ocrReason
|
|
924
|
+
};
|
|
903
925
|
}
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
while (i < sorted.length) {
|
|
912
|
-
if (/^[가-힣\d]$/.test(sorted[i].text)) {
|
|
913
|
-
let runEnd = i + 1;
|
|
914
|
-
while (runEnd < sorted.length && /^[가-힣\d]$/.test(sorted[runEnd].text)) {
|
|
915
|
-
if (sorted[runEnd].hasSpaceBefore) break;
|
|
916
|
-
const gap = sorted[runEnd].x - (sorted[runEnd - 1].x + sorted[runEnd - 1].w);
|
|
917
|
-
const fs = sorted[runEnd].fontSize;
|
|
918
|
-
if (gap < fs * 0.1 || gap > fs * 3) break;
|
|
919
|
-
runEnd++;
|
|
920
|
-
}
|
|
921
|
-
if (runEnd - i >= 3) {
|
|
922
|
-
const gaps = [];
|
|
923
|
-
for (let g2 = i + 1; g2 < runEnd; g2++) {
|
|
924
|
-
gaps.push(sorted[g2].x - (sorted[g2 - 1].x + sorted[g2 - 1].w));
|
|
925
|
-
}
|
|
926
|
-
let minG = Infinity, maxG = -Infinity;
|
|
927
|
-
for (const g2 of gaps) {
|
|
928
|
-
if (g2 < minG) minG = g2;
|
|
929
|
-
if (g2 > maxG) maxG = g2;
|
|
930
|
-
}
|
|
931
|
-
if (minG > 0 && maxG / minG <= 3) {
|
|
932
|
-
const run = sorted.slice(i, runEnd);
|
|
933
|
-
const text = run.map((r) => r.text).join("");
|
|
934
|
-
const first = run[0], last = run[runEnd - i - 1];
|
|
935
|
-
const item = {
|
|
936
|
-
text,
|
|
937
|
-
x: first.x,
|
|
938
|
-
y: first.y,
|
|
939
|
-
w: last.x + last.w - first.x,
|
|
940
|
-
h: first.h,
|
|
941
|
-
fontSize: first.fontSize,
|
|
942
|
-
fontName: first.fontName
|
|
943
|
-
};
|
|
944
|
-
originMap.set(item, run);
|
|
945
|
-
merged.push(item);
|
|
946
|
-
i = runEnd;
|
|
947
|
-
continue;
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
merged.push(sorted[i]);
|
|
952
|
-
i++;
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
return { merged, originMap };
|
|
926
|
+
var LOW_TEXT_THRESHOLD = 20;
|
|
927
|
+
var HIGH_PUA_THRESHOLD = 0.2;
|
|
928
|
+
var HIGH_CONTROL_THRESHOLD = 0.05;
|
|
929
|
+
var HIGH_REPLACEMENT_THRESHOLD = 0.05;
|
|
930
|
+
var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
|
|
931
|
+
function stripControlChars(text) {
|
|
932
|
+
return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
|
|
956
933
|
}
|
|
957
|
-
function
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
934
|
+
function summarizeDocumentQuality(pages) {
|
|
935
|
+
if (pages.length === 0) {
|
|
936
|
+
return {
|
|
937
|
+
totalPages: 0,
|
|
938
|
+
totalTextChars: 0,
|
|
939
|
+
avgHangulRatio: 0,
|
|
940
|
+
avgControlCharRatio: 0,
|
|
941
|
+
avgReplacementCharRatio: 0,
|
|
942
|
+
avgPuaRatio: 0,
|
|
943
|
+
lowTextPageCount: 0,
|
|
944
|
+
highPuaPageCount: 0,
|
|
945
|
+
needsOcr: false,
|
|
946
|
+
ocrCandidatePages: []
|
|
947
|
+
};
|
|
962
948
|
}
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
let
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
949
|
+
let textChars = 0;
|
|
950
|
+
let hangul = 0;
|
|
951
|
+
let control = 0;
|
|
952
|
+
let replacement = 0;
|
|
953
|
+
let pua = 0;
|
|
954
|
+
let lowText = 0;
|
|
955
|
+
let highPua = 0;
|
|
956
|
+
const ocrCandidatePages = [];
|
|
957
|
+
for (const p of pages) {
|
|
958
|
+
textChars += p.textChars;
|
|
959
|
+
hangul += p.hangulRatio;
|
|
960
|
+
control += p.controlCharRatio;
|
|
961
|
+
replacement += p.replacementCharRatio;
|
|
962
|
+
pua += p.puaRatio;
|
|
963
|
+
if (p.textChars < LOW_TEXT_THRESHOLD) lowText++;
|
|
964
|
+
if (p.puaRatio >= HIGH_PUA_THRESHOLD) highPua++;
|
|
965
|
+
if (p.needsOcr) ocrCandidatePages.push(p.page);
|
|
973
966
|
}
|
|
974
|
-
const
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
967
|
+
const n = pages.length;
|
|
968
|
+
return {
|
|
969
|
+
totalPages: n,
|
|
970
|
+
totalTextChars: textChars,
|
|
971
|
+
avgHangulRatio: hangul / n,
|
|
972
|
+
avgControlCharRatio: control / n,
|
|
973
|
+
avgReplacementCharRatio: replacement / n,
|
|
974
|
+
avgPuaRatio: pua / n,
|
|
975
|
+
lowTextPageCount: lowText,
|
|
976
|
+
highPuaPageCount: highPua,
|
|
977
|
+
needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
|
|
978
|
+
ocrCandidatePages
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
// src/pdf/text-line.ts
|
|
983
|
+
function filterHiddenText(items, pageWidth, pageHeight) {
|
|
984
|
+
let hiddenCount = 0;
|
|
985
|
+
const visible = [];
|
|
986
|
+
for (const item of items) {
|
|
987
|
+
if (item.isHidden) {
|
|
988
|
+
hiddenCount++;
|
|
989
|
+
continue;
|
|
993
990
|
}
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
const matched = countMatchedColumnsRange(rows[j], columns, sorted);
|
|
999
|
-
if (matched >= MIN_COLS) matchCount++;
|
|
991
|
+
const margin = Math.max(pageWidth, pageHeight) * 0.1;
|
|
992
|
+
if (item.x < -margin || item.x > pageWidth + margin || item.y < -margin || item.y > pageHeight + margin) {
|
|
993
|
+
hiddenCount++;
|
|
994
|
+
continue;
|
|
1000
995
|
}
|
|
1001
|
-
|
|
1002
|
-
return { columns, headerIdx: ri };
|
|
996
|
+
visible.push(item);
|
|
1003
997
|
}
|
|
1004
|
-
return
|
|
998
|
+
return { visible, hiddenCount };
|
|
1005
999
|
}
|
|
1006
|
-
function
|
|
1007
|
-
|
|
1008
|
-
const
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
const curr = rows[i];
|
|
1012
|
-
const a = rowBand(prev);
|
|
1013
|
-
const b = rowBand(curr);
|
|
1014
|
-
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1015
|
-
const prevIsFrag = isFragmentRow(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1016
|
-
const currIsFrag = isFragmentRow(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1017
|
-
if (prevIsFrag || currIsFrag) {
|
|
1018
|
-
const baseY = prevIsFrag ? curr.y : prev.y;
|
|
1019
|
-
result[result.length - 1] = { y: baseY, items: [...prev.items, ...curr.items] };
|
|
1020
|
-
} else {
|
|
1021
|
-
result.push(curr);
|
|
1022
|
-
}
|
|
1000
|
+
function collapseEvenSpacing(text) {
|
|
1001
|
+
const tokens = text.split(" ");
|
|
1002
|
+
const singleCharCount = tokens.filter((t) => t.length === 1).length;
|
|
1003
|
+
if (tokens.length >= 3 && singleCharCount / tokens.length >= 0.7) {
|
|
1004
|
+
return tokens.join("");
|
|
1023
1005
|
}
|
|
1024
|
-
return
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
|
|
1006
|
+
return text.replace(
|
|
1007
|
+
/(?<![가-힣])[가-힣](?: [가-힣\d]){2,}(?![가-힣])/g,
|
|
1008
|
+
(match) => match.replace(/ /g, "")
|
|
1009
|
+
);
|
|
1028
1010
|
}
|
|
1029
|
-
function
|
|
1030
|
-
let
|
|
1031
|
-
for (const i of
|
|
1032
|
-
|
|
1033
|
-
if (i.y <
|
|
1034
|
-
if (i.
|
|
1011
|
+
function computeBBox(items, pageNum) {
|
|
1012
|
+
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
1013
|
+
for (const i of items) {
|
|
1014
|
+
if (i.x < minX) minX = i.x;
|
|
1015
|
+
if (i.y < minY) minY = i.y;
|
|
1016
|
+
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
1017
|
+
const effectiveH = i.h > 0 ? i.h : i.fontSize;
|
|
1018
|
+
if (i.y + effectiveH > maxY) maxY = i.y + effectiveH;
|
|
1035
1019
|
}
|
|
1036
|
-
return {
|
|
1020
|
+
return { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY };
|
|
1037
1021
|
}
|
|
1038
|
-
function
|
|
1039
|
-
if (
|
|
1040
|
-
const
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
const
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
result[result.length - 1] = {
|
|
1050
|
-
y: prev.y,
|
|
1051
|
-
items: [...prev.items, ...curr.items]
|
|
1052
|
-
};
|
|
1053
|
-
} else {
|
|
1054
|
-
result.push(curr);
|
|
1022
|
+
function dominantStyle(items) {
|
|
1023
|
+
if (items.length === 0) return void 0;
|
|
1024
|
+
const freq = /* @__PURE__ */ new Map();
|
|
1025
|
+
let maxCount = 0, dominantSize = 0;
|
|
1026
|
+
for (const i of items) {
|
|
1027
|
+
if (i.fontSize <= 0) continue;
|
|
1028
|
+
const count = (freq.get(i.fontSize) || 0) + 1;
|
|
1029
|
+
freq.set(i.fontSize, count);
|
|
1030
|
+
if (count > maxCount) {
|
|
1031
|
+
maxCount = count;
|
|
1032
|
+
dominantSize = i.fontSize;
|
|
1055
1033
|
}
|
|
1056
1034
|
}
|
|
1057
|
-
return
|
|
1035
|
+
if (dominantSize === 0) return void 0;
|
|
1036
|
+
const fontName = items.find((i) => i.fontSize === dominantSize)?.fontName || void 0;
|
|
1037
|
+
return { fontSize: dominantSize, fontName };
|
|
1058
1038
|
}
|
|
1059
|
-
function
|
|
1060
|
-
|
|
1061
|
-
const
|
|
1062
|
-
const
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
if (
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
rows.push({ y: curY, items: curItems });
|
|
1070
|
-
curItems = [sorted[i]];
|
|
1071
|
-
curY = sorted[i].y;
|
|
1039
|
+
function normalizeItems(rawItems) {
|
|
1040
|
+
const items = [];
|
|
1041
|
+
const spacePositions = [];
|
|
1042
|
+
for (const i of rawItems) {
|
|
1043
|
+
if (typeof i.str !== "string") continue;
|
|
1044
|
+
const x = Math.round(i.transform[4]);
|
|
1045
|
+
const y = Math.round(i.transform[5]);
|
|
1046
|
+
if (!i.str.trim()) {
|
|
1047
|
+
spacePositions.push({ x, y });
|
|
1048
|
+
continue;
|
|
1072
1049
|
}
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
const minGap = Math.max(avgFontSize * MIN_GAP_FACTOR, MIN_GAP_ABSOLUTE);
|
|
1083
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
1084
|
-
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1085
|
-
if (gap >= minGap) return true;
|
|
1086
|
-
}
|
|
1087
|
-
return false;
|
|
1088
|
-
}
|
|
1089
|
-
function extractColumnClusters(rows) {
|
|
1090
|
-
const allX = [];
|
|
1091
|
-
for (const row of rows) {
|
|
1092
|
-
for (const item of row.items) allX.push(item.x);
|
|
1093
|
-
}
|
|
1094
|
-
if (allX.length === 0) return [];
|
|
1095
|
-
allX.sort((a, b) => a - b);
|
|
1096
|
-
const clusters = [];
|
|
1097
|
-
let clusterStart = 0;
|
|
1098
|
-
for (let i = 1; i <= allX.length; i++) {
|
|
1099
|
-
if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
|
|
1100
|
-
const slice = allX.slice(clusterStart, i);
|
|
1101
|
-
const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
|
|
1102
|
-
clusters.push({ x: avg, count: slice.length });
|
|
1103
|
-
clusterStart = i;
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
const minCount = Math.max(2, Math.floor(rows.length * MIN_COL_FILL_RATIO));
|
|
1107
|
-
return clusters.filter((c) => c.count >= minCount).sort((a, b) => a.x - b.x);
|
|
1108
|
-
}
|
|
1109
|
-
function findTableRegionsByHeader(allRows, columns, headerItems) {
|
|
1110
|
-
const regions = [];
|
|
1111
|
-
let currentRegion = [];
|
|
1112
|
-
let missStreak = 0;
|
|
1113
|
-
for (const row of allRows) {
|
|
1114
|
-
const matchedCols = countMatchedColumnsRange(row, columns, headerItems);
|
|
1115
|
-
if (matchedCols >= MIN_COLS) {
|
|
1116
|
-
currentRegion.push(row);
|
|
1117
|
-
missStreak = 0;
|
|
1118
|
-
} else if (currentRegion.length > 0 && (row.items.length <= 2 || missStreak === 0)) {
|
|
1119
|
-
currentRegion.push(row);
|
|
1120
|
-
missStreak++;
|
|
1121
|
-
} else {
|
|
1122
|
-
while (currentRegion.length > 0) {
|
|
1123
|
-
const last = currentRegion[currentRegion.length - 1];
|
|
1124
|
-
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1125
|
-
currentRegion.pop();
|
|
1126
|
-
}
|
|
1127
|
-
if (currentRegion.length >= MIN_ROWS) {
|
|
1128
|
-
regions.push({ rows: [...currentRegion] });
|
|
1129
|
-
}
|
|
1130
|
-
currentRegion = [];
|
|
1131
|
-
missStreak = 0;
|
|
1050
|
+
const scaleX = Math.hypot(i.transform[0], i.transform[1]);
|
|
1051
|
+
const scaleY = Math.hypot(i.transform[2], i.transform[3]);
|
|
1052
|
+
const fontSize = Math.round(Math.max(scaleY, scaleX));
|
|
1053
|
+
const w = Math.round(i.width);
|
|
1054
|
+
const h = Math.round(i.height);
|
|
1055
|
+
const isHidden = fontSize === 0 || i.width === 0 && i.str.trim().length > 0;
|
|
1056
|
+
let text = i.str.trim();
|
|
1057
|
+
if (/^[\d\s\-().·,☎]+$/.test(text) && /\d/.test(text) && / /.test(text)) {
|
|
1058
|
+
text = text.replace(/ /g, "");
|
|
1132
1059
|
}
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
currentRegion.pop();
|
|
1138
|
-
}
|
|
1139
|
-
if (currentRegion.length >= MIN_ROWS) {
|
|
1140
|
-
regions.push({ rows: currentRegion });
|
|
1141
|
-
}
|
|
1142
|
-
return regions;
|
|
1143
|
-
}
|
|
1144
|
-
function findTableRegions(allRows, columns) {
|
|
1145
|
-
const regions = [];
|
|
1146
|
-
let currentRegion = [];
|
|
1147
|
-
for (const row of allRows) {
|
|
1148
|
-
const matchedCols = countMatchedColumns(row, columns);
|
|
1149
|
-
if (matchedCols >= MIN_COLS) {
|
|
1150
|
-
currentRegion.push(row);
|
|
1151
|
-
} else if (row.items.length === 1) {
|
|
1152
|
-
if (currentRegion.length > 0) {
|
|
1153
|
-
currentRegion.push(row);
|
|
1060
|
+
const split = splitEvenSpacedItem(text, x, w, fontSize);
|
|
1061
|
+
if (split) {
|
|
1062
|
+
for (const s of split) {
|
|
1063
|
+
items.push({ text: s.text, x: s.x, y, w: s.w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
1154
1064
|
}
|
|
1155
1065
|
} else {
|
|
1156
|
-
|
|
1157
|
-
regions.push({ rows: [...currentRegion] });
|
|
1158
|
-
}
|
|
1159
|
-
currentRegion = [];
|
|
1066
|
+
items.push({ text, x, y, w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
1160
1067
|
}
|
|
1161
1068
|
}
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
|
|
1172
|
-
matched.add(ci);
|
|
1069
|
+
const sorted = items.sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1070
|
+
const deduped = [];
|
|
1071
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
1072
|
+
let isDup = false;
|
|
1073
|
+
for (let j = deduped.length - 1; j >= 0; j--) {
|
|
1074
|
+
const prev = deduped[j];
|
|
1075
|
+
if (prev.y - sorted[i].y > 3) break;
|
|
1076
|
+
if (Math.abs(prev.y - sorted[i].y) <= 3 && prev.text === sorted[i].text && Math.abs(prev.x - sorted[i].x) <= 3) {
|
|
1077
|
+
isDup = true;
|
|
1173
1078
|
break;
|
|
1174
1079
|
}
|
|
1175
1080
|
}
|
|
1081
|
+
if (!isDup) deduped.push(sorted[i]);
|
|
1176
1082
|
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
const matched = /* @__PURE__ */ new Set();
|
|
1187
|
-
for (const item of row.items) {
|
|
1188
|
-
for (let ci = 0; ci < boundaries.length; ci++) {
|
|
1189
|
-
if (item.x >= boundaries[ci].left && item.x < boundaries[ci].right) {
|
|
1190
|
-
matched.add(ci);
|
|
1191
|
-
break;
|
|
1083
|
+
if (spacePositions.length > 0) {
|
|
1084
|
+
for (const sp of spacePositions) {
|
|
1085
|
+
let nearest = null;
|
|
1086
|
+
for (const item of deduped) {
|
|
1087
|
+
if (Math.abs(sp.y - item.y) > 3) continue;
|
|
1088
|
+
const dist = item.x - sp.x;
|
|
1089
|
+
if (dist >= -1 && dist <= 20 && (!nearest || item.x < nearest.x)) {
|
|
1090
|
+
nearest = item;
|
|
1091
|
+
}
|
|
1192
1092
|
}
|
|
1093
|
+
if (nearest) nearest.hasSpaceBefore = true;
|
|
1193
1094
|
}
|
|
1194
1095
|
}
|
|
1195
|
-
return
|
|
1096
|
+
return deduped;
|
|
1196
1097
|
}
|
|
1197
|
-
function
|
|
1098
|
+
function splitEvenSpacedItem(text, itemX, itemW, fontSize) {
|
|
1099
|
+
if (!/^[가-힣\d](?: [가-힣\d]){2,}$/.test(text)) return null;
|
|
1100
|
+
const chars = text.split(" ");
|
|
1101
|
+
if (chars.length < 3) return null;
|
|
1102
|
+
const charW = itemW / chars.length;
|
|
1103
|
+
if (charW > fontSize * 2) return null;
|
|
1104
|
+
return chars.map((ch, idx) => ({
|
|
1105
|
+
text: ch,
|
|
1106
|
+
x: Math.round(itemX + idx * charW),
|
|
1107
|
+
w: Math.round(charW * 0.8)
|
|
1108
|
+
// 실제 글자 폭은 간격보다 좁음
|
|
1109
|
+
}));
|
|
1110
|
+
}
|
|
1111
|
+
function groupByY(items) {
|
|
1198
1112
|
if (items.length === 0) return [];
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
for (let i = 1; i <
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
const gapThreshold = sorted.length <= numCols + 1 ? 12 : Math.max(medianGap * 2.5, 12);
|
|
1208
|
-
const significantGaps = gaps.filter((g2) => g2.size >= gapThreshold).sort((a, b) => b.size - a.size).slice(0, numCols - 1).sort((a, b) => a.idx - b.idx);
|
|
1209
|
-
const groups = [];
|
|
1210
|
-
let start = 0;
|
|
1211
|
-
for (const gap of significantGaps) {
|
|
1212
|
-
groups.push(sorted.slice(start, gap.idx));
|
|
1213
|
-
start = gap.idx;
|
|
1214
|
-
}
|
|
1215
|
-
groups.push(sorted.slice(start));
|
|
1216
|
-
const result = [];
|
|
1217
|
-
const usedCols = /* @__PURE__ */ new Set();
|
|
1218
|
-
const groupCenters = groups.map((g2) => {
|
|
1219
|
-
let minX = Infinity, maxX = -Infinity;
|
|
1220
|
-
for (const i of g2) {
|
|
1221
|
-
if (i.x < minX) minX = i.x;
|
|
1222
|
-
const r = i.x + i.w;
|
|
1223
|
-
if (r > maxX) maxX = r;
|
|
1224
|
-
}
|
|
1225
|
-
return (minX + maxX) / 2;
|
|
1226
|
-
});
|
|
1227
|
-
const assignments = [];
|
|
1228
|
-
for (let gi = 0; gi < groups.length; gi++) {
|
|
1229
|
-
for (let ci = 0; ci < numCols; ci++) {
|
|
1230
|
-
assignments.push({ gi, ci, dist: Math.abs(groupCenters[gi] - colCenters[ci]) });
|
|
1113
|
+
const lines = [];
|
|
1114
|
+
let curY = items[0].y;
|
|
1115
|
+
let curLine = [items[0]];
|
|
1116
|
+
for (let i = 1; i < items.length; i++) {
|
|
1117
|
+
if (Math.abs(items[i].y - curY) > 3) {
|
|
1118
|
+
lines.push(curLine);
|
|
1119
|
+
curLine = [];
|
|
1120
|
+
curY = items[i].y;
|
|
1231
1121
|
}
|
|
1122
|
+
curLine.push(items[i]);
|
|
1232
1123
|
}
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1124
|
+
if (curLine.length > 0) lines.push(curLine);
|
|
1125
|
+
return lines;
|
|
1126
|
+
}
|
|
1127
|
+
function mergeSuperscriptLines(lines) {
|
|
1128
|
+
if (lines.length <= 1) return lines;
|
|
1129
|
+
const band = (line) => {
|
|
1130
|
+
let bottom = Infinity, top = -Infinity;
|
|
1131
|
+
for (const i of line) {
|
|
1132
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1133
|
+
if (i.y < bottom) bottom = i.y;
|
|
1134
|
+
if (i.y + h > top) top = i.y + h;
|
|
1135
|
+
}
|
|
1136
|
+
return { bottom, top, height: top - bottom };
|
|
1137
|
+
};
|
|
1138
|
+
const isFrag = (line) => line.length <= 3 && line.every((i) => i.text.trim().length <= 8);
|
|
1139
|
+
const result = [lines[0]];
|
|
1140
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1141
|
+
const prev = result[result.length - 1];
|
|
1142
|
+
const curr = lines[i];
|
|
1143
|
+
const a = band(prev);
|
|
1144
|
+
const b = band(curr);
|
|
1145
|
+
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1146
|
+
const prevIsFrag = isFrag(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1147
|
+
const currIsFrag = isFrag(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1148
|
+
if (prevIsFrag || currIsFrag) {
|
|
1149
|
+
result[result.length - 1] = [...prev, ...curr];
|
|
1150
|
+
} else {
|
|
1151
|
+
result.push(curr);
|
|
1250
1152
|
}
|
|
1251
|
-
result.push({ col: bestCol, items: groups[gi] });
|
|
1252
1153
|
}
|
|
1253
1154
|
return result;
|
|
1254
1155
|
}
|
|
1255
|
-
function
|
|
1256
|
-
|
|
1257
|
-
const
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
usedItems.add(row.items[0]);
|
|
1269
|
-
continue;
|
|
1270
|
-
}
|
|
1271
|
-
const assignments = assignRowItems(row.items, columns, numCols);
|
|
1272
|
-
for (const { col, items } of assignments) {
|
|
1273
|
-
const text = items.map((i) => i.text).join(" ");
|
|
1274
|
-
const existing = cells[r][col].text;
|
|
1275
|
-
cells[r][col].text = existing ? existing + " " + text : text;
|
|
1276
|
-
for (const item of items) usedItems.add(item);
|
|
1156
|
+
function mergeLineSimple(items) {
|
|
1157
|
+
if (items.length <= 1) return items[0]?.text || "";
|
|
1158
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1159
|
+
const isEvenSpaced = detectEvenSpacedItems(sorted);
|
|
1160
|
+
let result = sorted[0].text;
|
|
1161
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1162
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1163
|
+
const avgFs = (sorted[i].fontSize + sorted[i - 1].fontSize) / 2;
|
|
1164
|
+
const tabThreshold = Math.max(avgFs * 2, 30);
|
|
1165
|
+
if (gap > tabThreshold) {
|
|
1166
|
+
result += " ";
|
|
1167
|
+
result += sorted[i].text;
|
|
1168
|
+
continue;
|
|
1277
1169
|
}
|
|
1170
|
+
if (isEvenSpaced[i]) {
|
|
1171
|
+
result += sorted[i].text;
|
|
1172
|
+
continue;
|
|
1173
|
+
}
|
|
1174
|
+
if (sorted[i].hasSpaceBefore && gap >= avgFs * 0.05) {
|
|
1175
|
+
result += " ";
|
|
1176
|
+
result += sorted[i].text;
|
|
1177
|
+
continue;
|
|
1178
|
+
}
|
|
1179
|
+
if (/[□■○●▶◆◇ㅇ]$/.test(sorted[i - 1].text) && /^[가-힣]/.test(sorted[i].text) && gap > 1) {
|
|
1180
|
+
result += " ";
|
|
1181
|
+
result += sorted[i].text;
|
|
1182
|
+
continue;
|
|
1183
|
+
}
|
|
1184
|
+
if (gap > spaceGapThreshold(avgFs)) result += " ";
|
|
1185
|
+
result += sorted[i].text;
|
|
1278
1186
|
}
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1187
|
+
return result;
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
// src/pdf/cluster-detector.ts
|
|
1191
|
+
var Y_TOL = 3;
|
|
1192
|
+
var COL_CLUSTER_TOL = 15;
|
|
1193
|
+
var MIN_ROWS = 3;
|
|
1194
|
+
var MIN_COLS = 2;
|
|
1195
|
+
var MIN_GAP_FACTOR = 2;
|
|
1196
|
+
var MIN_GAP_ABSOLUTE = 20;
|
|
1197
|
+
var MIN_COL_FILL_RATIO = 0.4;
|
|
1198
|
+
function detectClusterTables(items, pageNum) {
|
|
1199
|
+
if (items.length < MIN_ROWS * MIN_COLS) return [];
|
|
1200
|
+
const { merged, originMap } = mergeEvenSpacedClusters(items);
|
|
1201
|
+
const rows = mergeOverlappingRows(groupByBaseline(merged));
|
|
1202
|
+
if (rows.length < MIN_ROWS) return [];
|
|
1203
|
+
const results = [];
|
|
1204
|
+
const headerResult = detectHeaderRow(rows);
|
|
1205
|
+
if (headerResult) {
|
|
1206
|
+
const { columns, headerIdx } = headerResult;
|
|
1207
|
+
const headerRow = rows[headerIdx];
|
|
1208
|
+
const headerItems = [...headerRow.items].sort((a, b) => a.x - b.x);
|
|
1209
|
+
const headerAndBelow = rows.slice(headerIdx);
|
|
1210
|
+
const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
|
|
1211
|
+
const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
|
|
1212
|
+
for (const region of tableRegions) {
|
|
1213
|
+
const table = buildClusterTable(region.rows, columns, pageNum);
|
|
1214
|
+
if (table) {
|
|
1215
|
+
expandUsedItems(table.usedItems, originMap);
|
|
1216
|
+
results.push(table);
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1287
1219
|
}
|
|
1288
|
-
|
|
1289
|
-
const
|
|
1290
|
-
if (
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1220
|
+
if (results.length === 0) {
|
|
1221
|
+
const suspiciousRows = rows.filter((row) => hasSuspiciousGaps(row));
|
|
1222
|
+
if (suspiciousRows.length >= MIN_ROWS) {
|
|
1223
|
+
const columns = extractColumnClusters(suspiciousRows);
|
|
1224
|
+
if (columns.length >= MIN_COLS) {
|
|
1225
|
+
const tableRegions = findTableRegions(rows, columns);
|
|
1226
|
+
for (const region of tableRegions) {
|
|
1227
|
+
const mergedRows = mergeMultiLineRows(region.rows, columns);
|
|
1228
|
+
const table = buildClusterTable(mergedRows, columns, pageNum);
|
|
1229
|
+
if (table) {
|
|
1230
|
+
expandUsedItems(table.usedItems, originMap);
|
|
1231
|
+
results.push(table);
|
|
1232
|
+
}
|
|
1300
1233
|
}
|
|
1301
|
-
for (let c = 0; c < numCols; c++) cells[r][c].text = "";
|
|
1302
|
-
break;
|
|
1303
1234
|
}
|
|
1304
1235
|
}
|
|
1305
1236
|
}
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1237
|
+
return results;
|
|
1238
|
+
}
|
|
1239
|
+
function mergeEvenSpacedClusters(items) {
|
|
1240
|
+
const originMap = /* @__PURE__ */ new Map();
|
|
1241
|
+
const rows = groupByBaseline(items);
|
|
1242
|
+
const merged = [];
|
|
1243
|
+
for (const row of rows) {
|
|
1244
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1245
|
+
let i = 0;
|
|
1246
|
+
while (i < sorted.length) {
|
|
1247
|
+
if (/^[가-힣\d]$/.test(sorted[i].text)) {
|
|
1248
|
+
let runEnd = i + 1;
|
|
1249
|
+
while (runEnd < sorted.length && /^[가-힣\d]$/.test(sorted[runEnd].text)) {
|
|
1250
|
+
if (sorted[runEnd].hasSpaceBefore) break;
|
|
1251
|
+
const gap = sorted[runEnd].x - (sorted[runEnd - 1].x + sorted[runEnd - 1].w);
|
|
1252
|
+
const fs = sorted[runEnd].fontSize;
|
|
1253
|
+
if (gap < fs * 0.1 || gap > fs * 3) break;
|
|
1254
|
+
runEnd++;
|
|
1255
|
+
}
|
|
1256
|
+
if (runEnd - i >= 3) {
|
|
1257
|
+
const gaps = [];
|
|
1258
|
+
for (let g2 = i + 1; g2 < runEnd; g2++) {
|
|
1259
|
+
gaps.push(sorted[g2].x - (sorted[g2 - 1].x + sorted[g2 - 1].w));
|
|
1260
|
+
}
|
|
1261
|
+
let minG = Infinity, maxG = -Infinity;
|
|
1262
|
+
for (const g2 of gaps) {
|
|
1263
|
+
if (g2 < minG) minG = g2;
|
|
1264
|
+
if (g2 > maxG) maxG = g2;
|
|
1265
|
+
}
|
|
1266
|
+
if (minG > 0 && maxG / minG <= 3) {
|
|
1267
|
+
const run = sorted.slice(i, runEnd);
|
|
1268
|
+
const text = run.map((r) => r.text).join("");
|
|
1269
|
+
const first = run[0], last = run[runEnd - i - 1];
|
|
1270
|
+
const item = {
|
|
1271
|
+
text,
|
|
1272
|
+
x: first.x,
|
|
1273
|
+
y: first.y,
|
|
1274
|
+
w: last.x + last.w - first.x,
|
|
1275
|
+
h: first.h,
|
|
1276
|
+
fontSize: first.fontSize,
|
|
1277
|
+
fontName: first.fontName
|
|
1278
|
+
};
|
|
1279
|
+
originMap.set(item, run);
|
|
1280
|
+
merged.push(item);
|
|
1281
|
+
i = runEnd;
|
|
1282
|
+
continue;
|
|
1283
|
+
}
|
|
1317
1284
|
}
|
|
1318
|
-
for (let c = 0; c < numCols; c++) next[c].text = "";
|
|
1319
1285
|
}
|
|
1286
|
+
merged.push(sorted[i]);
|
|
1287
|
+
i++;
|
|
1320
1288
|
}
|
|
1321
1289
|
}
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
const
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1290
|
+
return { merged, originMap };
|
|
1291
|
+
}
|
|
1292
|
+
function expandUsedItems(usedItems, originMap) {
|
|
1293
|
+
const toAdd = [];
|
|
1294
|
+
for (const item of usedItems) {
|
|
1295
|
+
const origins = originMap.get(item);
|
|
1296
|
+
if (origins) for (const o of origins) toAdd.push(o);
|
|
1297
|
+
}
|
|
1298
|
+
for (const a of toAdd) usedItems.add(a);
|
|
1299
|
+
}
|
|
1300
|
+
function detectHeaderRow(rows) {
|
|
1331
1301
|
const allItems = rows.flatMap((r) => r.items);
|
|
1332
|
-
|
|
1302
|
+
if (allItems.length === 0) return null;
|
|
1303
|
+
let allMinX = Infinity, allMaxX = -Infinity;
|
|
1333
1304
|
for (const i of allItems) {
|
|
1334
|
-
if (i.x <
|
|
1335
|
-
|
|
1336
|
-
if (
|
|
1337
|
-
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1338
|
-
if (i.y + h > maxY) maxY = i.y + h;
|
|
1305
|
+
if (i.x < allMinX) allMinX = i.x;
|
|
1306
|
+
const r = i.x + i.w;
|
|
1307
|
+
if (r > allMaxX) allMaxX = r;
|
|
1339
1308
|
}
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
control++;
|
|
1360
|
-
continue;
|
|
1361
|
-
}
|
|
1362
|
-
if (code === 65533) {
|
|
1363
|
-
replacement++;
|
|
1364
|
-
continue;
|
|
1365
|
-
}
|
|
1366
|
-
if (code >= 44032 && code <= 55203) {
|
|
1367
|
-
hangul++;
|
|
1368
|
-
continue;
|
|
1309
|
+
const pageSpan = allMaxX - allMinX;
|
|
1310
|
+
if (pageSpan <= 0) return null;
|
|
1311
|
+
for (let ri = 0; ri < rows.length; ri++) {
|
|
1312
|
+
const row = rows[ri];
|
|
1313
|
+
if (row.items.length < MIN_COLS || row.items.length > 6) continue;
|
|
1314
|
+
if (row.items.some((i) => i.text.length > 8)) continue;
|
|
1315
|
+
if (!row.items.some((i) => /[가-힣]/.test(i.text))) continue;
|
|
1316
|
+
if (row.items.some((i) => /^[□■○●·※▶▷◆◇\-]/.test(i.text))) continue;
|
|
1317
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1318
|
+
const xSpan = sorted[sorted.length - 1].x + sorted[sorted.length - 1].w - sorted[0].x;
|
|
1319
|
+
if (xSpan / pageSpan < 0.4) continue;
|
|
1320
|
+
const avgFs = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
|
|
1321
|
+
let hasLargeGap = false;
|
|
1322
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1323
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1324
|
+
if (gap >= avgFs * 2.5) {
|
|
1325
|
+
hasLargeGap = true;
|
|
1326
|
+
break;
|
|
1327
|
+
}
|
|
1369
1328
|
}
|
|
1370
|
-
if (
|
|
1371
|
-
|
|
1372
|
-
|
|
1329
|
+
if (!hasLargeGap) continue;
|
|
1330
|
+
const columns = sorted.map((item) => ({ x: item.x, count: 0 }));
|
|
1331
|
+
let matchCount = 0;
|
|
1332
|
+
for (let j = ri + 1; j < rows.length && matchCount < MIN_ROWS + 2; j++) {
|
|
1333
|
+
const matched = countMatchedColumnsRange(rows[j], columns, sorted);
|
|
1334
|
+
if (matched >= MIN_COLS) matchCount++;
|
|
1373
1335
|
}
|
|
1336
|
+
if (matchCount < MIN_ROWS) continue;
|
|
1337
|
+
return { columns, headerIdx: ri };
|
|
1374
1338
|
}
|
|
1375
|
-
|
|
1376
|
-
const puaRatio = pua / denom;
|
|
1377
|
-
const controlCharRatio = control / denom;
|
|
1378
|
-
const replacementCharRatio = replacement / denom;
|
|
1379
|
-
let needsOcr = false;
|
|
1380
|
-
let ocrReason;
|
|
1381
|
-
if (total < LOW_TEXT_THRESHOLD) {
|
|
1382
|
-
needsOcr = true;
|
|
1383
|
-
ocrReason = "low_text";
|
|
1384
|
-
} else if (puaRatio >= HIGH_PUA_THRESHOLD) {
|
|
1385
|
-
needsOcr = true;
|
|
1386
|
-
ocrReason = "high_pua";
|
|
1387
|
-
} else if (controlCharRatio >= HIGH_CONTROL_THRESHOLD) {
|
|
1388
|
-
needsOcr = true;
|
|
1389
|
-
ocrReason = "high_control";
|
|
1390
|
-
} else if (replacementCharRatio >= HIGH_REPLACEMENT_THRESHOLD) {
|
|
1391
|
-
needsOcr = true;
|
|
1392
|
-
ocrReason = "high_replacement";
|
|
1393
|
-
}
|
|
1394
|
-
return {
|
|
1395
|
-
page,
|
|
1396
|
-
textChars: total,
|
|
1397
|
-
hangulRatio: hangul / denom,
|
|
1398
|
-
controlCharRatio,
|
|
1399
|
-
replacementCharRatio,
|
|
1400
|
-
puaRatio,
|
|
1401
|
-
needsOcr,
|
|
1402
|
-
ocrReason
|
|
1403
|
-
};
|
|
1404
|
-
}
|
|
1405
|
-
var LOW_TEXT_THRESHOLD = 20;
|
|
1406
|
-
var HIGH_PUA_THRESHOLD = 0.2;
|
|
1407
|
-
var HIGH_CONTROL_THRESHOLD = 0.05;
|
|
1408
|
-
var HIGH_REPLACEMENT_THRESHOLD = 0.05;
|
|
1409
|
-
var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
|
|
1410
|
-
function stripControlChars(text) {
|
|
1411
|
-
return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
|
|
1412
|
-
}
|
|
1413
|
-
function summarizeDocumentQuality(pages) {
|
|
1414
|
-
if (pages.length === 0) {
|
|
1415
|
-
return {
|
|
1416
|
-
totalPages: 0,
|
|
1417
|
-
totalTextChars: 0,
|
|
1418
|
-
avgHangulRatio: 0,
|
|
1419
|
-
avgControlCharRatio: 0,
|
|
1420
|
-
avgReplacementCharRatio: 0,
|
|
1421
|
-
avgPuaRatio: 0,
|
|
1422
|
-
lowTextPageCount: 0,
|
|
1423
|
-
highPuaPageCount: 0,
|
|
1424
|
-
needsOcr: false,
|
|
1425
|
-
ocrCandidatePages: []
|
|
1426
|
-
};
|
|
1427
|
-
}
|
|
1428
|
-
let textChars = 0;
|
|
1429
|
-
let hangul = 0;
|
|
1430
|
-
let control = 0;
|
|
1431
|
-
let replacement = 0;
|
|
1432
|
-
let pua = 0;
|
|
1433
|
-
let lowText = 0;
|
|
1434
|
-
let highPua = 0;
|
|
1435
|
-
const ocrCandidatePages = [];
|
|
1436
|
-
for (const p of pages) {
|
|
1437
|
-
textChars += p.textChars;
|
|
1438
|
-
hangul += p.hangulRatio;
|
|
1439
|
-
control += p.controlCharRatio;
|
|
1440
|
-
replacement += p.replacementCharRatio;
|
|
1441
|
-
pua += p.puaRatio;
|
|
1442
|
-
if (p.textChars < LOW_TEXT_THRESHOLD) lowText++;
|
|
1443
|
-
if (p.puaRatio >= HIGH_PUA_THRESHOLD) highPua++;
|
|
1444
|
-
if (p.needsOcr) ocrCandidatePages.push(p.page);
|
|
1445
|
-
}
|
|
1446
|
-
const n = pages.length;
|
|
1447
|
-
return {
|
|
1448
|
-
totalPages: n,
|
|
1449
|
-
totalTextChars: textChars,
|
|
1450
|
-
avgHangulRatio: hangul / n,
|
|
1451
|
-
avgControlCharRatio: control / n,
|
|
1452
|
-
avgReplacementCharRatio: replacement / n,
|
|
1453
|
-
avgPuaRatio: pua / n,
|
|
1454
|
-
lowTextPageCount: lowText,
|
|
1455
|
-
highPuaPageCount: highPua,
|
|
1456
|
-
needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
|
|
1457
|
-
ocrCandidatePages
|
|
1458
|
-
};
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
// src/pdf/polyfill.ts
|
|
1462
|
-
import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
|
|
1463
|
-
var g = globalThis;
|
|
1464
|
-
if (typeof g.DOMMatrix === "undefined") {
|
|
1465
|
-
g.DOMMatrix = class DOMMatrix {
|
|
1466
|
-
m = [1, 0, 0, 1, 0, 0];
|
|
1467
|
-
constructor(init) {
|
|
1468
|
-
if (init) this.m = init;
|
|
1469
|
-
}
|
|
1470
|
-
};
|
|
1471
|
-
}
|
|
1472
|
-
if (typeof g.Path2D === "undefined") {
|
|
1473
|
-
g.Path2D = class Path2D {
|
|
1474
|
-
};
|
|
1475
|
-
}
|
|
1476
|
-
g.pdfjsWorker = pdfjsWorker;
|
|
1477
|
-
|
|
1478
|
-
// src/pdf/parser.ts
|
|
1479
|
-
import { getDocument, GlobalWorkerOptions } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
1480
|
-
GlobalWorkerOptions.workerSrc = "";
|
|
1481
|
-
var MAX_PAGES = 5e3;
|
|
1482
|
-
var MAX_TOTAL_TEXT = 100 * 1024 * 1024;
|
|
1483
|
-
var PDF_LOAD_TIMEOUT_MS = 3e4;
|
|
1484
|
-
async function loadPdfWithTimeout(buffer) {
|
|
1485
|
-
const loadingTask = getDocument({
|
|
1486
|
-
data: new Uint8Array(buffer),
|
|
1487
|
-
useSystemFonts: true,
|
|
1488
|
-
disableFontFace: true,
|
|
1489
|
-
isEvalSupported: false
|
|
1490
|
-
});
|
|
1491
|
-
let timer;
|
|
1492
|
-
try {
|
|
1493
|
-
return await Promise.race([
|
|
1494
|
-
loadingTask.promise,
|
|
1495
|
-
new Promise((_, reject) => {
|
|
1496
|
-
timer = setTimeout(() => {
|
|
1497
|
-
loadingTask.destroy();
|
|
1498
|
-
reject(new KordocError("PDF \uB85C\uB529 \uD0C0\uC784\uC544\uC6C3 (30\uCD08 \uCD08\uACFC)"));
|
|
1499
|
-
}, PDF_LOAD_TIMEOUT_MS);
|
|
1500
|
-
})
|
|
1501
|
-
]);
|
|
1502
|
-
} finally {
|
|
1503
|
-
if (timer !== void 0) clearTimeout(timer);
|
|
1504
|
-
}
|
|
1505
|
-
}
|
|
1506
|
-
async function parsePdfDocument(buffer, options) {
|
|
1507
|
-
const formulaBuffer = options?.formulaOcr ? buffer.slice(0) : null;
|
|
1508
|
-
const doc = await loadPdfWithTimeout(buffer);
|
|
1509
|
-
try {
|
|
1510
|
-
const pageCount = doc.numPages;
|
|
1511
|
-
if (pageCount === 0) throw new KordocError("PDF\uC5D0 \uD398\uC774\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
|
|
1512
|
-
const metadata = { pageCount };
|
|
1513
|
-
await extractPdfMetadata(doc, metadata);
|
|
1514
|
-
const blocks = [];
|
|
1515
|
-
const warnings = [];
|
|
1516
|
-
const pageQuality = [];
|
|
1517
|
-
let totalChars = 0;
|
|
1518
|
-
let totalTextBytes = 0;
|
|
1519
|
-
const effectivePageCount = Math.min(pageCount, MAX_PAGES);
|
|
1520
|
-
const pageFilter = options?.pages ? parsePageRange(options.pages, effectivePageCount) : null;
|
|
1521
|
-
const totalTarget = pageFilter ? pageFilter.size : effectivePageCount;
|
|
1522
|
-
const fontSizeFreq = /* @__PURE__ */ new Map();
|
|
1523
|
-
const pageHeights = /* @__PURE__ */ new Map();
|
|
1524
|
-
const pagesWithLargeImage = /* @__PURE__ */ new Set();
|
|
1525
|
-
const skippedImagePages = /* @__PURE__ */ new Map();
|
|
1526
|
-
let parsedPages = 0;
|
|
1527
|
-
for (let i = 1; i <= effectivePageCount; i++) {
|
|
1528
|
-
if (pageFilter && !pageFilter.has(i)) continue;
|
|
1529
|
-
try {
|
|
1530
|
-
const page = await doc.getPage(i);
|
|
1531
|
-
const tc = await page.getTextContent();
|
|
1532
|
-
const viewport = page.getViewport({ scale: 1 });
|
|
1533
|
-
pageHeights.set(i, viewport.height);
|
|
1534
|
-
const rawItems = tc.items;
|
|
1535
|
-
const items = normalizeItems(rawItems);
|
|
1536
|
-
const { visible, hiddenCount } = filterHiddenText(items, viewport.width, viewport.height);
|
|
1537
|
-
if (hiddenCount > 0) {
|
|
1538
|
-
warnings.push({ page: i, message: `${hiddenCount}\uAC1C \uC228\uACA8\uC9C4 \uD14D\uC2A4\uD2B8 \uC694\uC18C \uD544\uD130\uB9C1\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
1539
|
-
}
|
|
1540
|
-
for (const item of visible) {
|
|
1541
|
-
if (item.fontSize > 0) fontSizeFreq.set(item.fontSize, (fontSizeFreq.get(item.fontSize) || 0) + 1);
|
|
1542
|
-
}
|
|
1543
|
-
const opList = await page.getOperatorList();
|
|
1544
|
-
const pageArea = viewport.width * viewport.height;
|
|
1545
|
-
if (pageArea > 0) {
|
|
1546
|
-
const imageRegions = extractImageRegions(opList.fnArray, opList.argsArray);
|
|
1547
|
-
let uncovered = 0;
|
|
1548
|
-
for (const r of imageRegions) {
|
|
1549
|
-
const area = (r.x2 - r.x1) * (r.y2 - r.y1);
|
|
1550
|
-
if (area < pageArea * 0.05) continue;
|
|
1551
|
-
pagesWithLargeImage.add(i);
|
|
1552
|
-
const hasText = visible.some((it) => {
|
|
1553
|
-
const cx = it.x + it.w / 2;
|
|
1554
|
-
const cy = it.y + (it.h || it.fontSize) / 2;
|
|
1555
|
-
return cx >= r.x1 && cx <= r.x2 && cy >= r.y1 && cy <= r.y2;
|
|
1556
|
-
});
|
|
1557
|
-
if (!hasText) uncovered++;
|
|
1558
|
-
}
|
|
1559
|
-
if (uncovered > 0) skippedImagePages.set(i, uncovered);
|
|
1560
|
-
}
|
|
1561
|
-
const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
|
|
1562
|
-
for (const b of pageBlocks) blocks.push(b);
|
|
1563
|
-
let pageText = "";
|
|
1564
|
-
for (const b of pageBlocks) {
|
|
1565
|
-
const t = b.text || "";
|
|
1566
|
-
totalChars += t.replace(/\s/g, "").length;
|
|
1567
|
-
totalTextBytes += t.length * 2;
|
|
1568
|
-
pageText += pageText ? "\n" + t : t;
|
|
1569
|
-
}
|
|
1570
|
-
pageQuality.push(computePageQuality(i, pageText));
|
|
1571
|
-
if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
|
|
1572
|
-
parsedPages++;
|
|
1573
|
-
options?.onProgress?.(parsedPages, totalTarget);
|
|
1574
|
-
} catch (pageErr) {
|
|
1575
|
-
if (pageErr instanceof KordocError) throw pageErr;
|
|
1576
|
-
warnings.push({ page: i, message: `\uD398\uC774\uC9C0 ${i} \uD30C\uC2F1 \uC2E4\uD328: ${pageErr instanceof Error ? pageErr.message : "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958"}`, code: "PARTIAL_PARSE" });
|
|
1577
|
-
}
|
|
1578
|
-
}
|
|
1579
|
-
const parsedPageCount = parsedPages || (pageFilter ? pageFilter.size : effectivePageCount);
|
|
1580
|
-
let isImageBased = false;
|
|
1581
|
-
if (totalChars / Math.max(parsedPageCount, 1) < 10) {
|
|
1582
|
-
if (options?.ocr) {
|
|
1583
|
-
try {
|
|
1584
|
-
const { ocrPages } = await import("./provider-4ZJKV3DC.js");
|
|
1585
|
-
const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
|
|
1586
|
-
if (ocrBlocks.length > 0) {
|
|
1587
|
-
const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
|
|
1588
|
-
return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
|
|
1589
|
-
}
|
|
1590
|
-
} catch {
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
isImageBased = true;
|
|
1594
|
-
warnings.push({
|
|
1595
|
-
message: `\uC774\uBBF8\uC9C0 \uAE30\uBC18 PDF (${pageCount}\uD398\uC774\uC9C0, \uD14D\uC2A4\uD2B8 ${totalChars}\uC790) \u2014 \uD14D\uC2A4\uD2B8 \uB808\uC774\uC5B4\uAC00 \uC5C6\uC5B4 OCR\uC774 \uD544\uC694\uD569\uB2C8\uB2E4`,
|
|
1596
|
-
code: "NEEDS_OCR"
|
|
1597
|
-
});
|
|
1598
|
-
}
|
|
1599
|
-
if (!isImageBased) {
|
|
1600
|
-
const OCR_REASON_MESSAGES = {
|
|
1601
|
-
low_text: "\uD14D\uC2A4\uD2B8\uAC00 \uAC70\uC758 \uC5C6\uB294 \uD398\uC774\uC9C0 (\uC2A4\uCE94/\uC774\uBBF8\uC9C0 \uCD94\uC815)",
|
|
1602
|
-
high_pua: "\uAE00\uAF34 \uB9E4\uD551 \uC2E4\uD328 (PUA \uBE44\uC728 \uB192\uC74C) \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
1603
|
-
high_control: "\uC81C\uC5B4\uBB38\uC790 \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
1604
|
-
high_replacement: "\uB300\uCCB4\uBB38\uC790(U+FFFD) \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00"
|
|
1605
|
-
};
|
|
1606
|
-
for (const pq of pageQuality) {
|
|
1607
|
-
if (!pq.needsOcr || !pq.ocrReason) continue;
|
|
1608
|
-
if (pq.ocrReason === "low_text" && !pagesWithLargeImage.has(pq.page)) continue;
|
|
1609
|
-
warnings.push({ page: pq.page, message: `${OCR_REASON_MESSAGES[pq.ocrReason]} \u2014 OCR \uAC80\uD1A0 \uD544\uC694`, code: "NEEDS_OCR" });
|
|
1610
|
-
}
|
|
1611
|
-
}
|
|
1612
|
-
if (!isImageBased) {
|
|
1613
|
-
for (const [page, count] of [...skippedImagePages.entries()].sort((a, b) => a[0] - b[0])) {
|
|
1614
|
-
warnings.push({ page, message: `${count}\uAC1C \uC774\uBBF8\uC9C0 \uC601\uC5ED\uC5D0 \uCD94\uCD9C \uAC00\uB2A5\uD55C \uD14D\uC2A4\uD2B8 \uC5C6\uC74C (\uADF8\uB9BC/\uCC28\uD2B8/\uB3C4\uC7A5 \uB0B4\uC6A9 \uB204\uB77D \uAC00\uB2A5)`, code: "SKIPPED_IMAGE" });
|
|
1615
|
-
}
|
|
1616
|
-
}
|
|
1617
|
-
if (options?.removeHeaderFooter !== false && parsedPageCount >= 3) {
|
|
1618
|
-
const removed = removeHeaderFooterBlocks(blocks, pageHeights, warnings);
|
|
1619
|
-
for (let ri = removed.length - 1; ri >= 0; ri--) {
|
|
1620
|
-
blocks.splice(removed[ri], 1);
|
|
1621
|
-
}
|
|
1622
|
-
}
|
|
1623
|
-
mergeCrossPageTables(blocks);
|
|
1624
|
-
if (options?.formulaOcr && formulaBuffer) {
|
|
1625
|
-
try {
|
|
1626
|
-
await applyFormulaOcr(formulaBuffer, blocks, pageFilter, effectivePageCount, warnings, options.onProgress);
|
|
1627
|
-
} catch (e) {
|
|
1628
|
-
warnings.push({
|
|
1629
|
-
message: `\uC218\uC2DD OCR \uC2E4\uD328: ${e instanceof Error ? e.message : String(e)}`,
|
|
1630
|
-
code: "PARTIAL_PARSE"
|
|
1631
|
-
});
|
|
1632
|
-
}
|
|
1633
|
-
}
|
|
1634
|
-
const medianFontSize = computeMedianFontSizeFromFreq(fontSizeFreq);
|
|
1635
|
-
if (medianFontSize > 0) {
|
|
1636
|
-
detectHeadings(blocks, medianFontSize);
|
|
1637
|
-
}
|
|
1638
|
-
detectMarkerHeadings(blocks);
|
|
1639
|
-
detectTableCaptions(blocks);
|
|
1640
|
-
detectKoreanListBlocks(blocks);
|
|
1641
|
-
const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
|
|
1642
|
-
sanitizeBlockControlChars(blocks);
|
|
1643
|
-
let markdown = cleanPdfText(blocksToMarkdown(blocks));
|
|
1644
|
-
return {
|
|
1645
|
-
markdown,
|
|
1646
|
-
blocks,
|
|
1647
|
-
metadata,
|
|
1648
|
-
outline: outline.length > 0 ? outline : void 0,
|
|
1649
|
-
warnings: warnings.length > 0 ? warnings : void 0,
|
|
1650
|
-
isImageBased: isImageBased || void 0,
|
|
1651
|
-
pageQuality,
|
|
1652
|
-
qualitySummary: summarizeDocumentQuality(pageQuality)
|
|
1653
|
-
};
|
|
1654
|
-
} finally {
|
|
1655
|
-
await doc.destroy().catch(() => {
|
|
1656
|
-
});
|
|
1657
|
-
}
|
|
1658
|
-
}
|
|
1659
|
-
async function extractPdfMetadata(doc, metadata) {
|
|
1660
|
-
try {
|
|
1661
|
-
const result = await doc.getMetadata();
|
|
1662
|
-
if (!result?.info) return;
|
|
1663
|
-
const info = result.info;
|
|
1664
|
-
if (typeof info.Title === "string" && info.Title.trim()) metadata.title = info.Title.trim();
|
|
1665
|
-
if (typeof info.Author === "string" && info.Author.trim()) metadata.author = info.Author.trim();
|
|
1666
|
-
if (typeof info.Creator === "string" && info.Creator.trim()) metadata.creator = info.Creator.trim();
|
|
1667
|
-
if (typeof info.Subject === "string" && info.Subject.trim()) metadata.description = info.Subject.trim();
|
|
1668
|
-
if (typeof info.Keywords === "string" && info.Keywords.trim()) {
|
|
1669
|
-
metadata.keywords = info.Keywords.split(/[,;]/).map((k) => k.trim()).filter(Boolean);
|
|
1670
|
-
}
|
|
1671
|
-
if (typeof info.CreationDate === "string") metadata.createdAt = parsePdfDate(info.CreationDate);
|
|
1672
|
-
if (typeof info.ModDate === "string") metadata.modifiedAt = parsePdfDate(info.ModDate);
|
|
1673
|
-
} catch {
|
|
1674
|
-
}
|
|
1675
|
-
}
|
|
1676
|
-
function parsePdfDate(dateStr) {
|
|
1677
|
-
const m = dateStr.match(/D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/);
|
|
1678
|
-
if (!m) return void 0;
|
|
1679
|
-
const [, year, month = "01", day = "01", hour = "00", min = "00", sec = "00"] = m;
|
|
1680
|
-
return `${year}-${month}-${day}T${hour}:${min}:${sec}`;
|
|
1681
|
-
}
|
|
1682
|
-
async function extractPdfMetadataOnly(buffer) {
|
|
1683
|
-
const doc = await loadPdfWithTimeout(buffer);
|
|
1684
|
-
try {
|
|
1685
|
-
const metadata = { pageCount: doc.numPages };
|
|
1686
|
-
await extractPdfMetadata(doc, metadata);
|
|
1687
|
-
return metadata;
|
|
1688
|
-
} finally {
|
|
1689
|
-
await doc.destroy().catch(() => {
|
|
1690
|
-
});
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
function filterHiddenText(items, pageWidth, pageHeight) {
|
|
1694
|
-
let hiddenCount = 0;
|
|
1695
|
-
const visible = [];
|
|
1696
|
-
for (const item of items) {
|
|
1697
|
-
if (item.isHidden) {
|
|
1698
|
-
hiddenCount++;
|
|
1699
|
-
continue;
|
|
1700
|
-
}
|
|
1701
|
-
const margin = Math.max(pageWidth, pageHeight) * 0.1;
|
|
1702
|
-
if (item.x < -margin || item.x > pageWidth + margin || item.y < -margin || item.y > pageHeight + margin) {
|
|
1703
|
-
hiddenCount++;
|
|
1704
|
-
continue;
|
|
1705
|
-
}
|
|
1706
|
-
visible.push(item);
|
|
1707
|
-
}
|
|
1708
|
-
return { visible, hiddenCount };
|
|
1709
|
-
}
|
|
1710
|
-
function computeMedianFontSizeFromFreq(freq) {
|
|
1711
|
-
if (freq.size === 0) return 0;
|
|
1712
|
-
let total = 0;
|
|
1713
|
-
for (const count of freq.values()) total += count;
|
|
1714
|
-
const sorted = [...freq.entries()].sort((a, b) => a[0] - b[0]);
|
|
1715
|
-
const mid = Math.floor(total / 2);
|
|
1716
|
-
let cumulative = 0;
|
|
1717
|
-
for (const [size, count] of sorted) {
|
|
1718
|
-
cumulative += count;
|
|
1719
|
-
if (cumulative > mid) return size;
|
|
1720
|
-
}
|
|
1721
|
-
return sorted[sorted.length - 1][0];
|
|
1722
|
-
}
|
|
1723
|
-
function detectHeadings(blocks, medianFontSize) {
|
|
1724
|
-
for (const block of blocks) {
|
|
1725
|
-
if (block.type !== "paragraph" || !block.text || !block.style?.fontSize) continue;
|
|
1726
|
-
const text = block.text.trim();
|
|
1727
|
-
if (text.length === 0 || text.length > 200) continue;
|
|
1728
|
-
if (/^\d+$/.test(text)) continue;
|
|
1729
|
-
const ratio = block.style.fontSize / medianFontSize;
|
|
1730
|
-
let level = 0;
|
|
1731
|
-
if (ratio >= HEADING_RATIO_H1) level = 1;
|
|
1732
|
-
else if (ratio >= HEADING_RATIO_H2) level = 2;
|
|
1733
|
-
else if (ratio >= HEADING_RATIO_H3) level = 3;
|
|
1734
|
-
if (level > 0) {
|
|
1735
|
-
block.type = "heading";
|
|
1736
|
-
block.level = level;
|
|
1737
|
-
block.text = collapseEvenSpacing(text);
|
|
1738
|
-
}
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
function collapseEvenSpacing(text) {
|
|
1742
|
-
const tokens = text.split(" ");
|
|
1743
|
-
const singleCharCount = tokens.filter((t) => t.length === 1).length;
|
|
1744
|
-
if (tokens.length >= 3 && singleCharCount / tokens.length >= 0.7) {
|
|
1745
|
-
return tokens.join("");
|
|
1746
|
-
}
|
|
1747
|
-
return text.replace(
|
|
1748
|
-
/(?<![가-힣])[가-힣](?: [가-힣\d]){2,}(?![가-힣])/g,
|
|
1749
|
-
(match) => match.replace(/ /g, "")
|
|
1750
|
-
);
|
|
1751
|
-
}
|
|
1752
|
-
function shouldDemoteTable(table) {
|
|
1753
|
-
const allCells = table.cells.flatMap((row) => row.map((c) => c.text.trim())).filter(Boolean);
|
|
1754
|
-
const allText = allCells.join(" ");
|
|
1755
|
-
if (table.rows <= 3 && table.cols <= 3) {
|
|
1756
|
-
const totalCells2 = table.rows * table.cols;
|
|
1757
|
-
const emptyCells2 = totalCells2 - allCells.length;
|
|
1758
|
-
if (emptyCells2 >= totalCells2 * 0.3) return true;
|
|
1759
|
-
if (/[□■◆○●▶ㅇ]/.test(allText)) return true;
|
|
1760
|
-
if (/<[^>]+>/.test(allText)) return true;
|
|
1761
|
-
}
|
|
1762
|
-
if (allText.length > 200) return false;
|
|
1763
|
-
if (/[□■◆○●▶]/.test(allText) && table.rows <= 3) return true;
|
|
1764
|
-
const totalCells = table.rows * table.cols;
|
|
1765
|
-
const emptyCells = totalCells - allCells.length;
|
|
1766
|
-
if (table.rows <= 2 && emptyCells > totalCells * 0.5) return true;
|
|
1767
|
-
if (table.rows === 1 && !/\d{2,}/.test(allText)) return true;
|
|
1768
|
-
return false;
|
|
1769
|
-
}
|
|
1770
|
-
function demoteTableToText(table) {
|
|
1771
|
-
const lines = [];
|
|
1772
|
-
for (let r = 0; r < table.rows; r++) {
|
|
1773
|
-
const cells = table.cells[r].map((c) => c.text.trim()).filter(Boolean);
|
|
1774
|
-
if (cells.length === 0) continue;
|
|
1775
|
-
if (table.cols === 2 && cells.length === 2) {
|
|
1776
|
-
lines.push(`${cells[0]} : ${cells[1]}`);
|
|
1777
|
-
} else {
|
|
1778
|
-
lines.push(cells.join(" "));
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
return lines.join("\n");
|
|
1782
|
-
}
|
|
1783
|
-
function detectMarkerHeadings(blocks) {
|
|
1784
|
-
for (let i = 0; i < blocks.length; i++) {
|
|
1785
|
-
const block = blocks[i];
|
|
1786
|
-
if (block.type !== "paragraph" || !block.text) continue;
|
|
1787
|
-
const text = block.text.trim();
|
|
1788
|
-
if (text.length < 50 && /^[□■◆◇▶]\s*[가-힣]/.test(text)) {
|
|
1789
|
-
block.type = "heading";
|
|
1790
|
-
block.level = 4;
|
|
1791
|
-
continue;
|
|
1792
|
-
}
|
|
1793
|
-
if (/^[가-힣]{2,6}$/.test(text) && block.style?.fontSize) {
|
|
1794
|
-
const prev = blocks[i - 1];
|
|
1795
|
-
const next = blocks[i + 1];
|
|
1796
|
-
const prevIsStructural = !prev || prev.type === "table" || prev.type === "heading" || prev.type === "separator";
|
|
1797
|
-
const nextIsStructural = !next || next.type === "table" || next.type === "heading" || next.type === "paragraph" && next.text && /^[□■◆○●]/.test(next.text.trim());
|
|
1798
|
-
if (prevIsStructural || nextIsStructural) {
|
|
1799
|
-
block.type = "heading";
|
|
1800
|
-
block.level = 3;
|
|
1801
|
-
}
|
|
1802
|
-
}
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
var MAX_XYCUT_DEPTH = 50;
|
|
1806
|
-
var XYCUT_MIN_GAP = 5;
|
|
1807
|
-
var CROSS_LAYOUT_BETA = 2;
|
|
1808
|
-
var CROSS_OVERLAP_RATIO = 0.1;
|
|
1809
|
-
var CROSS_MIN_OVERLAPS = 2;
|
|
1810
|
-
var CROSS_MAX_MASK_RATIO = 0.2;
|
|
1811
|
-
var NARROW_ELEMENT_WIDTH_RATIO = 0.1;
|
|
1812
|
-
function xyCutOrder(items, gapThreshold, depth = 0) {
|
|
1813
|
-
if (items.length === 0) return [];
|
|
1814
|
-
if (items.length <= 2 || depth >= MAX_XYCUT_DEPTH) return [items];
|
|
1815
|
-
if (depth === 0 && items.length >= 3) {
|
|
1816
|
-
const cross = identifyCrossLayoutItems(items);
|
|
1817
|
-
if (cross.size > 0 && cross.size <= items.length * CROSS_MAX_MASK_RATIO) {
|
|
1818
|
-
const rest = items.filter((i) => !cross.has(i));
|
|
1819
|
-
if (rest.length > 0) {
|
|
1820
|
-
const groups = xyCutOrder(rest, gapThreshold, 1);
|
|
1821
|
-
return mergeCrossLayoutGroups(groups, [...cross]);
|
|
1822
|
-
}
|
|
1823
|
-
}
|
|
1824
|
-
}
|
|
1825
|
-
const minGap = Math.max(XYCUT_MIN_GAP, gapThreshold);
|
|
1826
|
-
const hCut = findHorizontalCut(items);
|
|
1827
|
-
const vCut = findVerticalCutWithOutlierFilter(items, minGap);
|
|
1828
|
-
const hValid = hCut.gap >= minGap;
|
|
1829
|
-
const vValid = vCut.gap >= minGap;
|
|
1830
|
-
let useHorizontal;
|
|
1831
|
-
if (hValid && vValid) useHorizontal = vCut.gap <= hCut.gap * 1.5;
|
|
1832
|
-
else if (hValid) useHorizontal = true;
|
|
1833
|
-
else if (vValid) useHorizontal = false;
|
|
1834
|
-
else return [items];
|
|
1835
|
-
if (useHorizontal) {
|
|
1836
|
-
const upper = items.filter((i) => i.y > hCut.position);
|
|
1837
|
-
const lower = items.filter((i) => i.y <= hCut.position);
|
|
1838
|
-
if (upper.length > 0 && lower.length > 0 && upper.length < items.length) {
|
|
1839
|
-
return [...xyCutOrder(upper, gapThreshold, depth + 1), ...xyCutOrder(lower, gapThreshold, depth + 1)];
|
|
1840
|
-
}
|
|
1841
|
-
} else {
|
|
1842
|
-
const left = items.filter((i) => i.x + i.w / 2 < vCut.position);
|
|
1843
|
-
const right = items.filter((i) => i.x + i.w / 2 >= vCut.position);
|
|
1844
|
-
if (left.length > 0 && right.length > 0 && left.length < items.length) {
|
|
1845
|
-
return [...xyCutOrder(left, gapThreshold, depth + 1), ...xyCutOrder(right, gapThreshold, depth + 1)];
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1848
|
-
return [items];
|
|
1849
|
-
}
|
|
1850
|
-
function identifyCrossLayoutItems(items) {
|
|
1851
|
-
const cross = /* @__PURE__ */ new Set();
|
|
1852
|
-
if (items.length < 3) return cross;
|
|
1853
|
-
let maxWidth = 0;
|
|
1854
|
-
for (const i of items) {
|
|
1855
|
-
if (i.w > maxWidth) maxWidth = i.w;
|
|
1856
|
-
}
|
|
1857
|
-
const threshold = CROSS_LAYOUT_BETA * maxWidth;
|
|
1858
|
-
for (const item of items) {
|
|
1859
|
-
if (item.w < threshold) continue;
|
|
1860
|
-
let overlaps = 0;
|
|
1861
|
-
for (const other of items) {
|
|
1862
|
-
if (other === item) continue;
|
|
1863
|
-
const left = Math.max(item.x, other.x);
|
|
1864
|
-
const right = Math.min(item.x + item.w, other.x + other.w);
|
|
1865
|
-
const overlapW = right - left;
|
|
1866
|
-
if (overlapW <= 0) continue;
|
|
1867
|
-
const smaller = Math.min(item.w, other.w);
|
|
1868
|
-
if (smaller > 0 && overlapW / smaller >= CROSS_OVERLAP_RATIO) {
|
|
1869
|
-
overlaps++;
|
|
1870
|
-
if (overlaps >= CROSS_MIN_OVERLAPS) break;
|
|
1871
|
-
}
|
|
1872
|
-
}
|
|
1873
|
-
if (overlaps >= CROSS_MIN_OVERLAPS) cross.add(item);
|
|
1874
|
-
}
|
|
1875
|
-
return cross;
|
|
1339
|
+
return null;
|
|
1876
1340
|
}
|
|
1877
|
-
function
|
|
1878
|
-
if (
|
|
1879
|
-
const
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
result.push(
|
|
1893
|
-
continue;
|
|
1894
|
-
}
|
|
1895
|
-
if (gi >= groups.length) {
|
|
1896
|
-
result.push([sortedCross[ci++]]);
|
|
1897
|
-
continue;
|
|
1341
|
+
function mergeOverlappingRows(rows) {
|
|
1342
|
+
if (rows.length <= 1) return rows;
|
|
1343
|
+
const result = [rows[0]];
|
|
1344
|
+
for (let i = 1; i < rows.length; i++) {
|
|
1345
|
+
const prev = result[result.length - 1];
|
|
1346
|
+
const curr = rows[i];
|
|
1347
|
+
const a = rowBand(prev);
|
|
1348
|
+
const b = rowBand(curr);
|
|
1349
|
+
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1350
|
+
const prevIsFrag = isFragmentRow(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1351
|
+
const currIsFrag = isFragmentRow(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1352
|
+
if (prevIsFrag || currIsFrag) {
|
|
1353
|
+
const baseY = prevIsFrag ? curr.y : prev.y;
|
|
1354
|
+
result[result.length - 1] = { y: baseY, items: [...prev.items, ...curr.items] };
|
|
1355
|
+
} else {
|
|
1356
|
+
result.push(curr);
|
|
1898
1357
|
}
|
|
1899
|
-
const crossTop = sortedCross[ci].y + sortedCross[ci].h;
|
|
1900
|
-
if (crossTop >= groupTop(groups[gi])) result.push([sortedCross[ci++]]);
|
|
1901
|
-
else result.push(groups[gi++]);
|
|
1902
1358
|
}
|
|
1903
1359
|
return result;
|
|
1904
1360
|
}
|
|
1905
|
-
function
|
|
1906
|
-
|
|
1907
|
-
const sorted = [...items].sort((a, b) => b.y - a.y);
|
|
1908
|
-
let largestGap = 0;
|
|
1909
|
-
let position = 0;
|
|
1910
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
1911
|
-
const prevBottom = sorted[i - 1].y - sorted[i - 1].h;
|
|
1912
|
-
const currTop = sorted[i].y;
|
|
1913
|
-
const gap = prevBottom - currTop;
|
|
1914
|
-
if (gap > largestGap) {
|
|
1915
|
-
largestGap = gap;
|
|
1916
|
-
position = (prevBottom + currTop) / 2;
|
|
1917
|
-
}
|
|
1918
|
-
}
|
|
1919
|
-
return { position, gap: largestGap };
|
|
1361
|
+
function isFragmentRow(row) {
|
|
1362
|
+
return row.items.length <= 3 && row.items.every((i) => i.text.length <= 8);
|
|
1920
1363
|
}
|
|
1921
|
-
function
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
if (i.x < minX) minX = i.x;
|
|
1928
|
-
const r = i.x + i.w;
|
|
1929
|
-
if (r > maxX) maxX = r;
|
|
1930
|
-
}
|
|
1931
|
-
const narrowThreshold = (maxX - minX) * NARROW_ELEMENT_WIDTH_RATIO;
|
|
1932
|
-
const filtered = items.filter((i) => i.w >= narrowThreshold);
|
|
1933
|
-
if (filtered.length >= 2 && filtered.length < items.length && filtered.length >= items.length * 0.7) {
|
|
1934
|
-
const filteredCut = findVerticalCut(filtered);
|
|
1935
|
-
if (filteredCut.gap > edgeCut.gap && filteredCut.gap >= minGap) {
|
|
1936
|
-
return filteredCut;
|
|
1937
|
-
}
|
|
1938
|
-
}
|
|
1364
|
+
function rowBand(row) {
|
|
1365
|
+
let bottom = Infinity, top = -Infinity;
|
|
1366
|
+
for (const i of row.items) {
|
|
1367
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1368
|
+
if (i.y < bottom) bottom = i.y;
|
|
1369
|
+
if (i.y + h > top) top = i.y + h;
|
|
1939
1370
|
}
|
|
1940
|
-
return
|
|
1371
|
+
return { bottom, top, height: top - bottom };
|
|
1941
1372
|
}
|
|
1942
|
-
function
|
|
1943
|
-
if (
|
|
1944
|
-
const
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
let
|
|
1948
|
-
|
|
1949
|
-
const
|
|
1950
|
-
const
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
}
|
|
1373
|
+
function mergeMultiLineRows(rows, columns) {
|
|
1374
|
+
if (rows.length <= 1) return rows;
|
|
1375
|
+
const result = [rows[0]];
|
|
1376
|
+
const allFontSizes = rows.flatMap((r) => r.items).map((i) => i.fontSize);
|
|
1377
|
+
const avgFontSize = allFontSizes.length > 0 ? allFontSizes.reduce((s, v) => s + v, 0) / allFontSizes.length : 12;
|
|
1378
|
+
for (let i = 1; i < rows.length; i++) {
|
|
1379
|
+
const prev = result[result.length - 1];
|
|
1380
|
+
const curr = rows[i];
|
|
1381
|
+
const yGap = Math.abs(prev.y - curr.y);
|
|
1382
|
+
const matchedCols = countMatchedColumns(curr, columns);
|
|
1383
|
+
if (yGap < avgFontSize * 1.8 && curr.items.length <= 2 && (matchedCols < MIN_COLS || curr.items.length === 1)) {
|
|
1384
|
+
result[result.length - 1] = {
|
|
1385
|
+
y: prev.y,
|
|
1386
|
+
items: [...prev.items, ...curr.items]
|
|
1387
|
+
};
|
|
1388
|
+
} else {
|
|
1389
|
+
result.push(curr);
|
|
1957
1390
|
}
|
|
1958
|
-
prevRight = prevRight === null ? right : Math.max(prevRight, right);
|
|
1959
1391
|
}
|
|
1960
|
-
return
|
|
1392
|
+
return result;
|
|
1961
1393
|
}
|
|
1962
|
-
function
|
|
1394
|
+
function groupByBaseline(items) {
|
|
1963
1395
|
if (items.length === 0) return [];
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1396
|
+
const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1397
|
+
const rows = [];
|
|
1398
|
+
let curItems = [sorted[0]];
|
|
1399
|
+
let curY = sorted[0].y;
|
|
1400
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1401
|
+
if (Math.abs(sorted[i].y - curY) <= Y_TOL) {
|
|
1402
|
+
curItems.push(sorted[i]);
|
|
1403
|
+
} else {
|
|
1404
|
+
rows.push({ y: curY, items: curItems });
|
|
1405
|
+
curItems = [sorted[i]];
|
|
1406
|
+
curY = sorted[i].y;
|
|
1407
|
+
}
|
|
1972
1408
|
}
|
|
1973
|
-
|
|
1409
|
+
if (curItems.length > 0) rows.push({ y: curY, items: curItems });
|
|
1410
|
+
return rows;
|
|
1974
1411
|
}
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
if (
|
|
1984
|
-
const matches = [];
|
|
1985
|
-
for (const item of items) {
|
|
1986
|
-
const h = item.h > 0 ? item.h : item.fontSize;
|
|
1987
|
-
if (h <= 0 || item.w <= 0) continue;
|
|
1988
|
-
if (line.lineWidth > h * STRIKE_MAX_THICKNESS_RATIO) continue;
|
|
1989
|
-
const centerY = item.y + h * 0.4;
|
|
1990
|
-
if (Math.abs(line.y1 - centerY) > h * STRIKE_CENTER_TOLERANCE) continue;
|
|
1991
|
-
const overlap = Math.min(line.x2, item.x + item.w) - Math.max(line.x1, item.x);
|
|
1992
|
-
if (overlap / item.w < STRIKE_MIN_OVERLAP_RATIO) continue;
|
|
1993
|
-
matches.push(item);
|
|
1994
|
-
}
|
|
1995
|
-
if (matches.length === 0) continue;
|
|
1996
|
-
let totalW = 0;
|
|
1997
|
-
for (const m of matches) totalW += m.w;
|
|
1998
|
-
if (totalW <= 0 || (line.x2 - line.x1) / totalW > STRIKE_MAX_LINE_TO_TEXT_RATIO) continue;
|
|
1999
|
-
for (const m of matches) m.strike = true;
|
|
1412
|
+
function hasSuspiciousGaps(row) {
|
|
1413
|
+
if (row.items.length < 2) return false;
|
|
1414
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1415
|
+
if (sorted.length === 2 && sorted[1].text.length > 20) return false;
|
|
1416
|
+
const avgFontSize = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
|
|
1417
|
+
const minGap = Math.max(avgFontSize * MIN_GAP_FACTOR, MIN_GAP_ABSOLUTE);
|
|
1418
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1419
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1420
|
+
if (gap >= minGap) return true;
|
|
2000
1421
|
}
|
|
1422
|
+
return false;
|
|
2001
1423
|
}
|
|
2002
|
-
function
|
|
2003
|
-
const
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
for (const item of struck) {
|
|
2007
|
-
const key = Math.round(item.y / 3);
|
|
2008
|
-
const arr = lines.get(key) || [];
|
|
2009
|
-
arr.push(item);
|
|
2010
|
-
lines.set(key, arr);
|
|
1424
|
+
function extractColumnClusters(rows) {
|
|
1425
|
+
const allX = [];
|
|
1426
|
+
for (const row of rows) {
|
|
1427
|
+
for (const item of row.items) allX.push(item.x);
|
|
2011
1428
|
}
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
1429
|
+
if (allX.length === 0) return [];
|
|
1430
|
+
allX.sort((a, b) => a - b);
|
|
1431
|
+
const clusters = [];
|
|
1432
|
+
let clusterStart = 0;
|
|
1433
|
+
for (let i = 1; i <= allX.length; i++) {
|
|
1434
|
+
if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
|
|
1435
|
+
const slice = allX.slice(clusterStart, i);
|
|
1436
|
+
const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
|
|
1437
|
+
clusters.push({ x: avg, count: slice.length });
|
|
1438
|
+
clusterStart = i;
|
|
1439
|
+
}
|
|
2016
1440
|
}
|
|
1441
|
+
const minCount = Math.max(2, Math.floor(rows.length * MIN_COL_FILL_RATIO));
|
|
1442
|
+
return clusters.filter((c) => c.count >= minCount).sort((a, b) => a.x - b.x);
|
|
2017
1443
|
}
|
|
2018
|
-
function
|
|
2019
|
-
const
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
for (const
|
|
2023
|
-
const
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
const cells = extractCells(grid, horizontals, verticals);
|
|
2039
|
-
if (cells.length === 0) continue;
|
|
2040
|
-
const textItems = tableItems.map((i) => ({
|
|
2041
|
-
text: i.text,
|
|
2042
|
-
x: i.x,
|
|
2043
|
-
y: i.y,
|
|
2044
|
-
w: i.w,
|
|
2045
|
-
h: i.h,
|
|
2046
|
-
fontSize: i.fontSize,
|
|
2047
|
-
fontName: i.fontName,
|
|
2048
|
-
hasSpaceBefore: i.hasSpaceBefore
|
|
2049
|
-
}));
|
|
2050
|
-
const cellTextMap = mapTextToCells(textItems, cells);
|
|
2051
|
-
const numRows = grid.rowYs.length - 1;
|
|
2052
|
-
const numCols = grid.colXs.length - 1;
|
|
2053
|
-
const irGrid = Array.from(
|
|
2054
|
-
{ length: numRows },
|
|
2055
|
-
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
2056
|
-
);
|
|
2057
|
-
for (const cell of cells) {
|
|
2058
|
-
const cellItems = cellTextMap.get(cell) || [];
|
|
2059
|
-
let text = cellTextToString(cellItems);
|
|
2060
|
-
text = text.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2061
|
-
text = text.split("\n").map((line) => collapseEvenSpacing(line)).join("\n");
|
|
2062
|
-
irGrid[cell.row][cell.col] = {
|
|
2063
|
-
text,
|
|
2064
|
-
colSpan: cell.colSpan,
|
|
2065
|
-
rowSpan: cell.rowSpan
|
|
2066
|
-
};
|
|
2067
|
-
}
|
|
2068
|
-
let finalGrid = irGrid;
|
|
2069
|
-
let finalRows = numRows;
|
|
2070
|
-
if (numRows <= 2 && numCols >= 3) {
|
|
2071
|
-
const rebuilt = normalizeUndersegmentedTable(irGrid, grid.colXs, textItems);
|
|
2072
|
-
if (rebuilt) {
|
|
2073
|
-
finalGrid = rebuilt.map((row) => row.map((rawText) => {
|
|
2074
|
-
const cleaned = rawText.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2075
|
-
return {
|
|
2076
|
-
text: cleaned.split("\n").map((line) => collapseEvenSpacing(line)).join("\n"),
|
|
2077
|
-
colSpan: 1,
|
|
2078
|
-
rowSpan: 1
|
|
2079
|
-
};
|
|
2080
|
-
}));
|
|
2081
|
-
finalRows = finalGrid.length;
|
|
1444
|
+
function findTableRegionsByHeader(allRows, columns, headerItems) {
|
|
1445
|
+
const regions = [];
|
|
1446
|
+
let currentRegion = [];
|
|
1447
|
+
let missStreak = 0;
|
|
1448
|
+
for (const row of allRows) {
|
|
1449
|
+
const matchedCols = countMatchedColumnsRange(row, columns, headerItems);
|
|
1450
|
+
if (matchedCols >= MIN_COLS) {
|
|
1451
|
+
currentRegion.push(row);
|
|
1452
|
+
missStreak = 0;
|
|
1453
|
+
} else if (currentRegion.length > 0 && (row.items.length <= 2 || missStreak === 0)) {
|
|
1454
|
+
currentRegion.push(row);
|
|
1455
|
+
missStreak++;
|
|
1456
|
+
} else {
|
|
1457
|
+
while (currentRegion.length > 0) {
|
|
1458
|
+
const last = currentRegion[currentRegion.length - 1];
|
|
1459
|
+
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1460
|
+
currentRegion.pop();
|
|
1461
|
+
}
|
|
1462
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1463
|
+
regions.push({ rows: [...currentRegion] });
|
|
2082
1464
|
}
|
|
1465
|
+
currentRegion = [];
|
|
1466
|
+
missStreak = 0;
|
|
2083
1467
|
}
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
1468
|
+
}
|
|
1469
|
+
while (currentRegion.length > 0) {
|
|
1470
|
+
const last = currentRegion[currentRegion.length - 1];
|
|
1471
|
+
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1472
|
+
currentRegion.pop();
|
|
1473
|
+
}
|
|
1474
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1475
|
+
regions.push({ rows: currentRegion });
|
|
1476
|
+
}
|
|
1477
|
+
return regions;
|
|
1478
|
+
}
|
|
1479
|
+
function findTableRegions(allRows, columns) {
|
|
1480
|
+
const regions = [];
|
|
1481
|
+
let currentRegion = [];
|
|
1482
|
+
for (const row of allRows) {
|
|
1483
|
+
const matchedCols = countMatchedColumns(row, columns);
|
|
1484
|
+
if (matchedCols >= MIN_COLS) {
|
|
1485
|
+
currentRegion.push(row);
|
|
1486
|
+
} else if (row.items.length === 1) {
|
|
1487
|
+
if (currentRegion.length > 0) {
|
|
1488
|
+
currentRegion.push(row);
|
|
2104
1489
|
}
|
|
2105
|
-
|
|
1490
|
+
} else {
|
|
1491
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1492
|
+
regions.push({ rows: [...currentRegion] });
|
|
1493
|
+
}
|
|
1494
|
+
currentRegion = [];
|
|
2106
1495
|
}
|
|
2107
|
-
blocks.push({ type: "table", table: irTable, pageNumber: pageNum, bbox: tableBbox });
|
|
2108
1496
|
}
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
}));
|
|
2122
|
-
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2123
|
-
if (clusterResults.length > 0) {
|
|
2124
|
-
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2125
|
-
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2126
|
-
const usedClusterIndices = /* @__PURE__ */ new Set();
|
|
2127
|
-
for (const cr of clusterResults) {
|
|
2128
|
-
for (const ci of cr.usedItems) {
|
|
2129
|
-
const idx = ciToIdx.get(ci);
|
|
2130
|
-
if (idx !== void 0) usedClusterIndices.add(idx);
|
|
2131
|
-
}
|
|
2132
|
-
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
1497
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1498
|
+
regions.push({ rows: currentRegion });
|
|
1499
|
+
}
|
|
1500
|
+
return regions;
|
|
1501
|
+
}
|
|
1502
|
+
function countMatchedColumns(row, columns) {
|
|
1503
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1504
|
+
for (const item of row.items) {
|
|
1505
|
+
for (let ci = 0; ci < columns.length; ci++) {
|
|
1506
|
+
if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
|
|
1507
|
+
matched.add(ci);
|
|
1508
|
+
break;
|
|
2133
1509
|
}
|
|
2134
|
-
remaining = remaining.filter((_, idx) => !usedClusterIndices.has(idx));
|
|
2135
1510
|
}
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
1511
|
+
}
|
|
1512
|
+
return matched.size;
|
|
1513
|
+
}
|
|
1514
|
+
function countMatchedColumnsRange(row, columns, headerItems) {
|
|
1515
|
+
const boundaries = [];
|
|
1516
|
+
for (let ci = 0; ci < headerItems.length; ci++) {
|
|
1517
|
+
const left = ci === 0 ? 0 : (headerItems[ci - 1].x + headerItems[ci - 1].w + headerItems[ci].x) / 2;
|
|
1518
|
+
const right = ci === headerItems.length - 1 ? Infinity : (headerItems[ci].x + headerItems[ci].w + headerItems[ci + 1].x) / 2;
|
|
1519
|
+
boundaries.push({ left, right });
|
|
1520
|
+
}
|
|
1521
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1522
|
+
for (const item of row.items) {
|
|
1523
|
+
for (let ci = 0; ci < boundaries.length; ci++) {
|
|
1524
|
+
if (item.x >= boundaries[ci].left && item.x < boundaries[ci].right) {
|
|
1525
|
+
matched.add(ci);
|
|
1526
|
+
break;
|
|
2145
1527
|
}
|
|
2146
|
-
const finalTextBlocks = detectListBlocks(textBlocks);
|
|
2147
|
-
for (const b of finalTextBlocks) blocks.push(b);
|
|
2148
1528
|
}
|
|
2149
|
-
blocks.sort((a, b) => {
|
|
2150
|
-
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2151
|
-
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2152
|
-
return by - ay;
|
|
2153
|
-
});
|
|
2154
|
-
return mergeAdjacentTableBlocks(blocks);
|
|
2155
1529
|
}
|
|
2156
|
-
return
|
|
1530
|
+
return matched.size;
|
|
2157
1531
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
1532
|
+
function assignRowItems(items, columns, numCols) {
|
|
1533
|
+
if (items.length === 0) return [];
|
|
1534
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1535
|
+
const colCenters = columns.map((c) => c.x);
|
|
1536
|
+
const gaps = [];
|
|
1537
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1538
|
+
gaps.push({ idx: i, size: sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w) });
|
|
1539
|
+
}
|
|
1540
|
+
const gapSizes = gaps.map((g2) => g2.size).sort((a, b) => a - b);
|
|
1541
|
+
const medianGap = gapSizes.length > 0 ? gapSizes[Math.floor(gapSizes.length / 2)] : 0;
|
|
1542
|
+
const gapThreshold = sorted.length <= numCols + 1 ? 12 : Math.max(medianGap * 2.5, 12);
|
|
1543
|
+
const significantGaps = gaps.filter((g2) => g2.size >= gapThreshold).sort((a, b) => b.size - a.size).slice(0, numCols - 1).sort((a, b) => a.idx - b.idx);
|
|
1544
|
+
const groups = [];
|
|
1545
|
+
let start = 0;
|
|
1546
|
+
for (const gap of significantGaps) {
|
|
1547
|
+
groups.push(sorted.slice(start, gap.idx));
|
|
1548
|
+
start = gap.idx;
|
|
1549
|
+
}
|
|
1550
|
+
groups.push(sorted.slice(start));
|
|
1551
|
+
const result = [];
|
|
1552
|
+
const usedCols = /* @__PURE__ */ new Set();
|
|
1553
|
+
const groupCenters = groups.map((g2) => {
|
|
1554
|
+
let minX = Infinity, maxX = -Infinity;
|
|
1555
|
+
for (const i of g2) {
|
|
1556
|
+
if (i.x < minX) minX = i.x;
|
|
1557
|
+
const r = i.x + i.w;
|
|
1558
|
+
if (r > maxX) maxX = r;
|
|
2174
1559
|
}
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
1560
|
+
return (minX + maxX) / 2;
|
|
1561
|
+
});
|
|
1562
|
+
const assignments = [];
|
|
1563
|
+
for (let gi = 0; gi < groups.length; gi++) {
|
|
1564
|
+
for (let ci = 0; ci < numCols; ci++) {
|
|
1565
|
+
assignments.push({ gi, ci, dist: Math.abs(groupCenters[gi] - colCenters[ci]) });
|
|
2178
1566
|
}
|
|
2179
|
-
const merged = {
|
|
2180
|
-
rows: prev.table.rows + currCells.length,
|
|
2181
|
-
cols: prev.table.cols,
|
|
2182
|
-
cells: [...prev.table.cells, ...currCells],
|
|
2183
|
-
hasHeader: prev.table.hasHeader,
|
|
2184
|
-
caption: prev.table.caption
|
|
2185
|
-
};
|
|
2186
|
-
blocks[i] = { ...prev, table: merged };
|
|
2187
|
-
blocks.splice(i + 1, 1);
|
|
2188
1567
|
}
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
1568
|
+
assignments.sort((a, b) => a.dist - b.dist);
|
|
1569
|
+
const assignedGroups = /* @__PURE__ */ new Set();
|
|
1570
|
+
for (const { gi, ci } of assignments) {
|
|
1571
|
+
if (assignedGroups.has(gi) || usedCols.has(ci)) continue;
|
|
1572
|
+
result.push({ col: ci, items: groups[gi] });
|
|
1573
|
+
assignedGroups.add(gi);
|
|
1574
|
+
usedCols.add(ci);
|
|
2195
1575
|
}
|
|
2196
|
-
|
|
1576
|
+
for (let gi = 0; gi < groups.length; gi++) {
|
|
1577
|
+
if (assignedGroups.has(gi)) continue;
|
|
1578
|
+
let bestCol = 0, bestDist = Infinity;
|
|
1579
|
+
for (let ci = 0; ci < numCols; ci++) {
|
|
1580
|
+
const d = Math.abs(groupCenters[gi] - colCenters[ci]);
|
|
1581
|
+
if (d < bestDist) {
|
|
1582
|
+
bestDist = d;
|
|
1583
|
+
bestCol = ci;
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
result.push({ col: bestCol, items: groups[gi] });
|
|
1587
|
+
}
|
|
1588
|
+
return result;
|
|
2197
1589
|
}
|
|
2198
|
-
function
|
|
2199
|
-
|
|
2200
|
-
const
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
};
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
1590
|
+
function buildClusterTable(rows, columns, pageNum) {
|
|
1591
|
+
const numCols = columns.length;
|
|
1592
|
+
const numRows = rows.length;
|
|
1593
|
+
if (numRows < MIN_ROWS || numCols < MIN_COLS) return null;
|
|
1594
|
+
const cells = Array.from(
|
|
1595
|
+
{ length: numRows },
|
|
1596
|
+
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
1597
|
+
);
|
|
1598
|
+
const usedItems = /* @__PURE__ */ new Set();
|
|
1599
|
+
for (let r = 0; r < numRows; r++) {
|
|
1600
|
+
const row = rows[r];
|
|
1601
|
+
if (row.items.length === 1 && numCols > 1) {
|
|
1602
|
+
cells[r][0] = { text: row.items[0].text, colSpan: numCols, rowSpan: 1 };
|
|
1603
|
+
usedItems.add(row.items[0]);
|
|
1604
|
+
continue;
|
|
1605
|
+
}
|
|
1606
|
+
const assignments = assignRowItems(row.items, columns, numCols);
|
|
1607
|
+
for (const { col, items } of assignments) {
|
|
1608
|
+
const text = items.map((i) => i.text).join(" ");
|
|
1609
|
+
const existing = cells[r][col].text;
|
|
1610
|
+
cells[r][col].text = existing ? existing + " " + text : text;
|
|
1611
|
+
for (const item of items) usedItems.add(item);
|
|
2214
1612
|
}
|
|
2215
1613
|
}
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
}
|
|
2241
|
-
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2242
|
-
}
|
|
2243
|
-
const remaining = items.filter((_, idx) => !usedIndices.has(idx));
|
|
2244
|
-
if (remaining.length > 0) {
|
|
2245
|
-
const yLines = mergeSuperscriptLines(groupByY(remaining));
|
|
2246
|
-
for (const line of yLines) {
|
|
2247
|
-
const text = mergeLineSimple(line);
|
|
2248
|
-
if (!text.trim()) continue;
|
|
2249
|
-
const bbox = computeBBox(line, pageNum);
|
|
2250
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
1614
|
+
let emptyRows = 0;
|
|
1615
|
+
for (const row of cells) {
|
|
1616
|
+
if (row.every((c) => c.text === "")) emptyRows++;
|
|
1617
|
+
}
|
|
1618
|
+
if (emptyRows > numRows * 0.5) return null;
|
|
1619
|
+
for (let c = 0; c < numCols; c++) {
|
|
1620
|
+
const hasValue = cells.some((row) => row[c].text !== "");
|
|
1621
|
+
if (!hasValue) return null;
|
|
1622
|
+
}
|
|
1623
|
+
for (let r = numRows - 1; r >= 1; r--) {
|
|
1624
|
+
const nonEmptyCols = cells[r].filter((c) => c.text.trim()).length;
|
|
1625
|
+
if (nonEmptyCols !== 1) continue;
|
|
1626
|
+
if (cells[r][0].text.trim() !== "") continue;
|
|
1627
|
+
const contentText = cells[r].find((c) => c.text.trim())?.text.trim() || "";
|
|
1628
|
+
if (/^[○●▶\-·]/.test(contentText)) continue;
|
|
1629
|
+
for (let pr = r - 1; pr >= 0; pr--) {
|
|
1630
|
+
if (cells[pr].some((c) => c.text.trim())) {
|
|
1631
|
+
for (let c = 0; c < numCols; c++) {
|
|
1632
|
+
const prev = cells[pr][c].text.trim();
|
|
1633
|
+
const curr = cells[r][c].text.trim();
|
|
1634
|
+
if (curr) cells[pr][c].text = prev ? prev + " " + curr : curr;
|
|
1635
|
+
}
|
|
1636
|
+
for (let c = 0; c < numCols; c++) cells[r][c].text = "";
|
|
1637
|
+
break;
|
|
2251
1638
|
}
|
|
2252
1639
|
}
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
} else {
|
|
2266
|
-
const allY = items.map((i) => i.y);
|
|
2267
|
-
const pageHeight = safeMax(allY) - safeMin(allY);
|
|
2268
|
-
const gapThreshold = Math.max(15, pageHeight * 0.03);
|
|
2269
|
-
const orderedGroups = xyCutOrder(items, gapThreshold);
|
|
2270
|
-
for (const group of orderedGroups) {
|
|
2271
|
-
if (group.length === 0) continue;
|
|
2272
|
-
const yLines = mergeSuperscriptLines(groupByY(group));
|
|
2273
|
-
const groupColumns = detectColumns(yLines);
|
|
2274
|
-
if (groupColumns && groupColumns.length >= 3) {
|
|
2275
|
-
const tableText = extractWithColumns(yLines, groupColumns);
|
|
2276
|
-
const bbox = computeBBox(group, pageNum);
|
|
2277
|
-
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
|
|
2278
|
-
} else {
|
|
2279
|
-
for (const line of yLines) {
|
|
2280
|
-
const text = mergeLineSimple(line);
|
|
2281
|
-
if (!text.trim()) continue;
|
|
2282
|
-
const bbox = computeBBox(line, pageNum);
|
|
2283
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2284
|
-
}
|
|
1640
|
+
}
|
|
1641
|
+
for (let r = 0; r < cells.length - 1; r++) {
|
|
1642
|
+
const row = cells[r];
|
|
1643
|
+
const hasCol0 = row[0].text.trim() !== "";
|
|
1644
|
+
const hasColLast = numCols > 1 && row[numCols - 1].text.trim() !== "";
|
|
1645
|
+
const midEmpty = row.slice(1, numCols - 1).every((c) => c.text.trim() === "");
|
|
1646
|
+
if (hasCol0 && hasColLast && midEmpty) {
|
|
1647
|
+
const next = cells[r + 1];
|
|
1648
|
+
if (next[0].text.trim() === "" && next.some((c) => c.text.trim())) {
|
|
1649
|
+
for (let c = 1; c < numCols; c++) {
|
|
1650
|
+
const curr = next[c].text.trim();
|
|
1651
|
+
if (curr) row[c].text = row[c].text.trim() ? row[c].text.trim() + " " + curr : curr;
|
|
2285
1652
|
}
|
|
1653
|
+
for (let c = 0; c < numCols; c++) next[c].text = "";
|
|
2286
1654
|
}
|
|
2287
1655
|
}
|
|
2288
1656
|
}
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
1657
|
+
const filteredCells = cells.filter((row) => row.some((c) => c.text.trim()));
|
|
1658
|
+
const finalRowCount = filteredCells.length;
|
|
1659
|
+
if (finalRowCount < MIN_ROWS) return null;
|
|
1660
|
+
const irTable = {
|
|
1661
|
+
rows: finalRowCount,
|
|
1662
|
+
cols: numCols,
|
|
1663
|
+
cells: filteredCells,
|
|
1664
|
+
hasHeader: finalRowCount > 1
|
|
1665
|
+
};
|
|
1666
|
+
const allItems = rows.flatMap((r) => r.items);
|
|
2292
1667
|
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
2293
|
-
for (const i of
|
|
1668
|
+
for (const i of allItems) {
|
|
2294
1669
|
if (i.x < minX) minX = i.x;
|
|
2295
1670
|
if (i.y < minY) minY = i.y;
|
|
2296
1671
|
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
2297
|
-
const
|
|
2298
|
-
if (i.y +
|
|
1672
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1673
|
+
if (i.y + h > maxY) maxY = i.y + h;
|
|
2299
1674
|
}
|
|
2300
|
-
return {
|
|
1675
|
+
return {
|
|
1676
|
+
table: irTable,
|
|
1677
|
+
bbox: { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY },
|
|
1678
|
+
usedItems
|
|
1679
|
+
};
|
|
2301
1680
|
}
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
1681
|
+
|
|
1682
|
+
// src/pdf/xy-cut.ts
|
|
1683
|
+
var MAX_XYCUT_DEPTH = 50;
|
|
1684
|
+
var XYCUT_MIN_GAP = 5;
|
|
1685
|
+
var CROSS_LAYOUT_BETA = 2;
|
|
1686
|
+
var CROSS_OVERLAP_RATIO = 0.1;
|
|
1687
|
+
var CROSS_MIN_OVERLAPS = 2;
|
|
1688
|
+
var CROSS_MAX_MASK_RATIO = 0.2;
|
|
1689
|
+
var NARROW_ELEMENT_WIDTH_RATIO = 0.1;
|
|
1690
|
+
function xyCutOrder(items, gapThreshold, depth = 0) {
|
|
1691
|
+
if (items.length === 0) return [];
|
|
1692
|
+
if (items.length <= 2 || depth >= MAX_XYCUT_DEPTH) return [items];
|
|
1693
|
+
if (depth === 0 && items.length >= 3) {
|
|
1694
|
+
const cross = identifyCrossLayoutItems(items);
|
|
1695
|
+
if (cross.size > 0 && cross.size <= items.length * CROSS_MAX_MASK_RATIO) {
|
|
1696
|
+
const rest = items.filter((i) => !cross.has(i));
|
|
1697
|
+
if (rest.length > 0) {
|
|
1698
|
+
const groups = xyCutOrder(rest, gapThreshold, 1);
|
|
1699
|
+
return mergeCrossLayoutGroups(groups, [...cross]);
|
|
1700
|
+
}
|
|
2313
1701
|
}
|
|
2314
1702
|
}
|
|
2315
|
-
|
|
2316
|
-
const
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
1703
|
+
const minGap = Math.max(XYCUT_MIN_GAP, gapThreshold);
|
|
1704
|
+
const hCut = findHorizontalCut(items);
|
|
1705
|
+
const vCut = findVerticalCutWithOutlierFilter(items, minGap);
|
|
1706
|
+
const hValid = hCut.gap >= minGap;
|
|
1707
|
+
const vValid = vCut.gap >= minGap;
|
|
1708
|
+
let useHorizontal;
|
|
1709
|
+
if (hValid && vValid) useHorizontal = vCut.gap <= hCut.gap * 1.5;
|
|
1710
|
+
else if (hValid) useHorizontal = true;
|
|
1711
|
+
else if (vValid) useHorizontal = false;
|
|
1712
|
+
else return [items];
|
|
1713
|
+
if (useHorizontal) {
|
|
1714
|
+
const upper = items.filter((i) => i.y > hCut.position);
|
|
1715
|
+
const lower = items.filter((i) => i.y <= hCut.position);
|
|
1716
|
+
if (upper.length > 0 && lower.length > 0 && upper.length < items.length) {
|
|
1717
|
+
return [...xyCutOrder(upper, gapThreshold, depth + 1), ...xyCutOrder(lower, gapThreshold, depth + 1)];
|
|
2329
1718
|
}
|
|
2330
|
-
|
|
2331
|
-
const
|
|
2332
|
-
const
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
const isHidden = fontSize === 0 || i.width === 0 && i.str.trim().length > 0;
|
|
2336
|
-
let text = i.str.trim();
|
|
2337
|
-
if (/^[\d\s\-().·,☎]+$/.test(text) && /\d/.test(text) && / /.test(text)) {
|
|
2338
|
-
text = text.replace(/ /g, "");
|
|
1719
|
+
} else {
|
|
1720
|
+
const left = items.filter((i) => i.x + i.w / 2 < vCut.position);
|
|
1721
|
+
const right = items.filter((i) => i.x + i.w / 2 >= vCut.position);
|
|
1722
|
+
if (left.length > 0 && right.length > 0 && left.length < items.length) {
|
|
1723
|
+
return [...xyCutOrder(left, gapThreshold, depth + 1), ...xyCutOrder(right, gapThreshold, depth + 1)];
|
|
2339
1724
|
}
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
1725
|
+
}
|
|
1726
|
+
return [items];
|
|
1727
|
+
}
|
|
1728
|
+
function identifyCrossLayoutItems(items) {
|
|
1729
|
+
const cross = /* @__PURE__ */ new Set();
|
|
1730
|
+
if (items.length < 3) return cross;
|
|
1731
|
+
let maxWidth = 0;
|
|
1732
|
+
for (const i of items) {
|
|
1733
|
+
if (i.w > maxWidth) maxWidth = i.w;
|
|
1734
|
+
}
|
|
1735
|
+
const threshold = CROSS_LAYOUT_BETA * maxWidth;
|
|
1736
|
+
for (const item of items) {
|
|
1737
|
+
if (item.w < threshold) continue;
|
|
1738
|
+
let overlaps = 0;
|
|
1739
|
+
for (const other of items) {
|
|
1740
|
+
if (other === item) continue;
|
|
1741
|
+
const left = Math.max(item.x, other.x);
|
|
1742
|
+
const right = Math.min(item.x + item.w, other.x + other.w);
|
|
1743
|
+
const overlapW = right - left;
|
|
1744
|
+
if (overlapW <= 0) continue;
|
|
1745
|
+
const smaller = Math.min(item.w, other.w);
|
|
1746
|
+
if (smaller > 0 && overlapW / smaller >= CROSS_OVERLAP_RATIO) {
|
|
1747
|
+
overlaps++;
|
|
1748
|
+
if (overlaps >= CROSS_MIN_OVERLAPS) break;
|
|
2344
1749
|
}
|
|
2345
|
-
} else {
|
|
2346
|
-
items.push({ text, x, y, w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
2347
1750
|
}
|
|
1751
|
+
if (overlaps >= CROSS_MIN_OVERLAPS) cross.add(item);
|
|
2348
1752
|
}
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
1753
|
+
return cross;
|
|
1754
|
+
}
|
|
1755
|
+
function mergeCrossLayoutGroups(groups, cross) {
|
|
1756
|
+
if (cross.length === 0) return groups;
|
|
1757
|
+
const sortedCross = [...cross].sort((a, b) => b.y + b.h - (a.y + a.h) || a.x - b.x);
|
|
1758
|
+
const groupTop = (g2) => {
|
|
1759
|
+
let top = -Infinity;
|
|
1760
|
+
for (const i of g2) {
|
|
1761
|
+
const t = i.y + i.h;
|
|
1762
|
+
if (t > top) top = t;
|
|
1763
|
+
}
|
|
1764
|
+
return top;
|
|
1765
|
+
};
|
|
1766
|
+
const result = [];
|
|
1767
|
+
let gi = 0, ci = 0;
|
|
1768
|
+
while (gi < groups.length || ci < sortedCross.length) {
|
|
1769
|
+
if (ci >= sortedCross.length) {
|
|
1770
|
+
result.push(groups[gi++]);
|
|
1771
|
+
continue;
|
|
2360
1772
|
}
|
|
2361
|
-
if (
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
for (const sp of spacePositions) {
|
|
2365
|
-
let nearest = null;
|
|
2366
|
-
for (const item of deduped) {
|
|
2367
|
-
if (Math.abs(sp.y - item.y) > 3) continue;
|
|
2368
|
-
const dist = item.x - sp.x;
|
|
2369
|
-
if (dist >= -1 && dist <= 20 && (!nearest || item.x < nearest.x)) {
|
|
2370
|
-
nearest = item;
|
|
2371
|
-
}
|
|
2372
|
-
}
|
|
2373
|
-
if (nearest) nearest.hasSpaceBefore = true;
|
|
1773
|
+
if (gi >= groups.length) {
|
|
1774
|
+
result.push([sortedCross[ci++]]);
|
|
1775
|
+
continue;
|
|
2374
1776
|
}
|
|
1777
|
+
const crossTop = sortedCross[ci].y + sortedCross[ci].h;
|
|
1778
|
+
if (crossTop >= groupTop(groups[gi])) result.push([sortedCross[ci++]]);
|
|
1779
|
+
else result.push(groups[gi++]);
|
|
2375
1780
|
}
|
|
2376
|
-
return
|
|
2377
|
-
}
|
|
2378
|
-
function splitEvenSpacedItem(text, itemX, itemW, fontSize) {
|
|
2379
|
-
if (!/^[가-힣\d](?: [가-힣\d]){2,}$/.test(text)) return null;
|
|
2380
|
-
const chars = text.split(" ");
|
|
2381
|
-
if (chars.length < 3) return null;
|
|
2382
|
-
const charW = itemW / chars.length;
|
|
2383
|
-
if (charW > fontSize * 2) return null;
|
|
2384
|
-
return chars.map((ch, idx) => ({
|
|
2385
|
-
text: ch,
|
|
2386
|
-
x: Math.round(itemX + idx * charW),
|
|
2387
|
-
w: Math.round(charW * 0.8)
|
|
2388
|
-
// 실제 글자 폭은 간격보다 좁음
|
|
2389
|
-
}));
|
|
1781
|
+
return result;
|
|
2390
1782
|
}
|
|
2391
|
-
function
|
|
2392
|
-
if (items.length
|
|
2393
|
-
const
|
|
2394
|
-
let
|
|
2395
|
-
let
|
|
2396
|
-
for (let i = 1; i <
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
1783
|
+
function findHorizontalCut(items) {
|
|
1784
|
+
if (items.length < 2) return { position: 0, gap: 0 };
|
|
1785
|
+
const sorted = [...items].sort((a, b) => b.y - a.y);
|
|
1786
|
+
let largestGap = 0;
|
|
1787
|
+
let position = 0;
|
|
1788
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1789
|
+
const prevBottom = sorted[i - 1].y - sorted[i - 1].h;
|
|
1790
|
+
const currTop = sorted[i].y;
|
|
1791
|
+
const gap = prevBottom - currTop;
|
|
1792
|
+
if (gap > largestGap) {
|
|
1793
|
+
largestGap = gap;
|
|
1794
|
+
position = (prevBottom + currTop) / 2;
|
|
2401
1795
|
}
|
|
2402
|
-
curLine.push(items[i]);
|
|
2403
1796
|
}
|
|
2404
|
-
|
|
2405
|
-
return lines;
|
|
1797
|
+
return { position, gap: largestGap };
|
|
2406
1798
|
}
|
|
2407
|
-
function
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
if (i.
|
|
2414
|
-
|
|
1799
|
+
function findVerticalCutWithOutlierFilter(items, minGap) {
|
|
1800
|
+
const edgeCut = findVerticalCut(items);
|
|
1801
|
+
if (edgeCut.gap >= minGap) return edgeCut;
|
|
1802
|
+
if (items.length >= 3) {
|
|
1803
|
+
let minX = Infinity, maxX = -Infinity;
|
|
1804
|
+
for (const i of items) {
|
|
1805
|
+
if (i.x < minX) minX = i.x;
|
|
1806
|
+
const r = i.x + i.w;
|
|
1807
|
+
if (r > maxX) maxX = r;
|
|
2415
1808
|
}
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
const a = band(prev);
|
|
2424
|
-
const b = band(curr);
|
|
2425
|
-
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
2426
|
-
const prevIsFrag = isFrag(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
2427
|
-
const currIsFrag = isFrag(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
2428
|
-
if (prevIsFrag || currIsFrag) {
|
|
2429
|
-
result[result.length - 1] = [...prev, ...curr];
|
|
2430
|
-
} else {
|
|
2431
|
-
result.push(curr);
|
|
1809
|
+
const narrowThreshold = (maxX - minX) * NARROW_ELEMENT_WIDTH_RATIO;
|
|
1810
|
+
const filtered = items.filter((i) => i.w >= narrowThreshold);
|
|
1811
|
+
if (filtered.length >= 2 && filtered.length < items.length && filtered.length >= items.length * 0.7) {
|
|
1812
|
+
const filteredCut = findVerticalCut(filtered);
|
|
1813
|
+
if (filteredCut.gap > edgeCut.gap && filteredCut.gap >= minGap) {
|
|
1814
|
+
return filteredCut;
|
|
1815
|
+
}
|
|
2432
1816
|
}
|
|
2433
1817
|
}
|
|
2434
|
-
return
|
|
1818
|
+
return edgeCut;
|
|
1819
|
+
}
|
|
1820
|
+
function findVerticalCut(items) {
|
|
1821
|
+
if (items.length < 2) return { position: 0, gap: 0 };
|
|
1822
|
+
const sorted = [...items].sort((a, b) => a.x - b.x || a.x + a.w - (b.x + b.w));
|
|
1823
|
+
let largestGap = 0;
|
|
1824
|
+
let position = 0;
|
|
1825
|
+
let prevRight = null;
|
|
1826
|
+
for (const it of sorted) {
|
|
1827
|
+
const left = it.x;
|
|
1828
|
+
const right = it.x + it.w;
|
|
1829
|
+
if (prevRight !== null && left > prevRight) {
|
|
1830
|
+
const gap = left - prevRight;
|
|
1831
|
+
if (gap > largestGap) {
|
|
1832
|
+
largestGap = gap;
|
|
1833
|
+
position = (prevRight + left) / 2;
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
prevRight = prevRight === null ? right : Math.max(prevRight, right);
|
|
1837
|
+
}
|
|
1838
|
+
return { position, gap: largestGap };
|
|
2435
1839
|
}
|
|
1840
|
+
|
|
1841
|
+
// src/pdf/columns.ts
|
|
2436
1842
|
function isProseSpread(items) {
|
|
2437
1843
|
if (items.length < 4) return false;
|
|
2438
1844
|
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
@@ -2646,66 +2052,91 @@ function buildGridTable(lines, columns) {
|
|
|
2646
2052
|
}
|
|
2647
2053
|
return md.join("\n");
|
|
2648
2054
|
}
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
let
|
|
2654
|
-
for (
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
continue;
|
|
2662
|
-
}
|
|
2663
|
-
if (isEvenSpaced[i]) {
|
|
2664
|
-
result += sorted[i].text;
|
|
2665
|
-
continue;
|
|
2666
|
-
}
|
|
2667
|
-
if (sorted[i].hasSpaceBefore && gap >= avgFs * 0.05) {
|
|
2668
|
-
result += " ";
|
|
2669
|
-
result += sorted[i].text;
|
|
2670
|
-
continue;
|
|
2671
|
-
}
|
|
2672
|
-
if (/[□■○●▶◆◇ㅇ]$/.test(sorted[i - 1].text) && /^[가-힣]/.test(sorted[i].text) && gap > 1) {
|
|
2673
|
-
result += " ";
|
|
2674
|
-
result += sorted[i].text;
|
|
2675
|
-
continue;
|
|
2676
|
-
}
|
|
2677
|
-
if (gap > spaceGapThreshold(avgFs)) result += " ";
|
|
2678
|
-
result += sorted[i].text;
|
|
2055
|
+
|
|
2056
|
+
// src/pdf/block-detect.ts
|
|
2057
|
+
function computeMedianFontSizeFromFreq(freq) {
|
|
2058
|
+
if (freq.size === 0) return 0;
|
|
2059
|
+
let total = 0;
|
|
2060
|
+
for (const count of freq.values()) total += count;
|
|
2061
|
+
const sorted = [...freq.entries()].sort((a, b) => a[0] - b[0]);
|
|
2062
|
+
const mid = Math.floor(total / 2);
|
|
2063
|
+
let cumulative = 0;
|
|
2064
|
+
for (const [size, count] of sorted) {
|
|
2065
|
+
cumulative += count;
|
|
2066
|
+
if (cumulative > mid) return size;
|
|
2679
2067
|
}
|
|
2680
|
-
return
|
|
2068
|
+
return sorted[sorted.length - 1][0];
|
|
2681
2069
|
}
|
|
2682
|
-
function
|
|
2683
|
-
for (const
|
|
2684
|
-
if (
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2070
|
+
function detectHeadings(blocks, medianFontSize) {
|
|
2071
|
+
for (const block of blocks) {
|
|
2072
|
+
if (block.type !== "paragraph" || !block.text || !block.style?.fontSize) continue;
|
|
2073
|
+
const text = block.text.trim();
|
|
2074
|
+
if (text.length === 0 || text.length > 200) continue;
|
|
2075
|
+
if (/^\d+$/.test(text)) continue;
|
|
2076
|
+
const ratio = block.style.fontSize / medianFontSize;
|
|
2077
|
+
let level = 0;
|
|
2078
|
+
if (ratio >= HEADING_RATIO_H1) level = 1;
|
|
2079
|
+
else if (ratio >= HEADING_RATIO_H2) level = 2;
|
|
2080
|
+
else if (ratio >= HEADING_RATIO_H3) level = 3;
|
|
2081
|
+
if (level > 0) {
|
|
2082
|
+
block.type = "heading";
|
|
2083
|
+
block.level = level;
|
|
2084
|
+
block.text = collapseEvenSpacing(text);
|
|
2691
2085
|
}
|
|
2692
|
-
if (b.children) sanitizeBlockControlChars(b.children);
|
|
2693
2086
|
}
|
|
2694
2087
|
}
|
|
2695
|
-
function
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2088
|
+
function shouldDemoteTable(table) {
|
|
2089
|
+
const allCells = table.cells.flatMap((row) => row.map((c) => c.text.trim())).filter(Boolean);
|
|
2090
|
+
const allText = allCells.join(" ");
|
|
2091
|
+
if (table.rows <= 3 && table.cols <= 3) {
|
|
2092
|
+
const totalCells2 = table.rows * table.cols;
|
|
2093
|
+
const emptyCells2 = totalCells2 - allCells.length;
|
|
2094
|
+
if (emptyCells2 >= totalCells2 * 0.3) return true;
|
|
2095
|
+
if (/[□■◆○●▶ㅇ]/.test(allText)) return true;
|
|
2096
|
+
if (/<[^>]+>/.test(allText)) return true;
|
|
2097
|
+
}
|
|
2098
|
+
if (allText.length > 200) return false;
|
|
2099
|
+
if (/[□■◆○●▶]/.test(allText) && table.rows <= 3) return true;
|
|
2100
|
+
const totalCells = table.rows * table.cols;
|
|
2101
|
+
const emptyCells = totalCells - allCells.length;
|
|
2102
|
+
if (table.rows <= 2 && emptyCells > totalCells * 0.5) return true;
|
|
2103
|
+
if (table.rows === 1 && !/\d{2,}/.test(allText)) return true;
|
|
2104
|
+
return false;
|
|
2702
2105
|
}
|
|
2703
|
-
function
|
|
2704
|
-
const
|
|
2705
|
-
|
|
2106
|
+
function demoteTableToText(table) {
|
|
2107
|
+
const lines = [];
|
|
2108
|
+
for (let r = 0; r < table.rows; r++) {
|
|
2109
|
+
const cells = table.cells[r].map((c) => c.text.trim()).filter(Boolean);
|
|
2110
|
+
if (cells.length === 0) continue;
|
|
2111
|
+
if (table.cols === 2 && cells.length === 2) {
|
|
2112
|
+
lines.push(`${cells[0]} : ${cells[1]}`);
|
|
2113
|
+
} else {
|
|
2114
|
+
lines.push(cells.join(" "));
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
return lines.join("\n");
|
|
2706
2118
|
}
|
|
2707
|
-
function
|
|
2708
|
-
|
|
2119
|
+
function detectMarkerHeadings(blocks) {
|
|
2120
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2121
|
+
const block = blocks[i];
|
|
2122
|
+
if (block.type !== "paragraph" || !block.text) continue;
|
|
2123
|
+
const text = block.text.trim();
|
|
2124
|
+
if (text.length < 50 && /^[□■◆◇▶]\s*[가-힣]/.test(text)) {
|
|
2125
|
+
block.type = "heading";
|
|
2126
|
+
block.level = 4;
|
|
2127
|
+
continue;
|
|
2128
|
+
}
|
|
2129
|
+
if (/^[가-힣]{2,6}$/.test(text) && block.style?.fontSize) {
|
|
2130
|
+
const prev = blocks[i - 1];
|
|
2131
|
+
const next = blocks[i + 1];
|
|
2132
|
+
const prevIsStructural = !prev || prev.type === "table" || prev.type === "heading" || prev.type === "separator";
|
|
2133
|
+
const nextIsStructural = !next || next.type === "table" || next.type === "heading" || next.type === "paragraph" && next.text && /^[□■◆○●]/.test(next.text.trim());
|
|
2134
|
+
if (prevIsStructural || nextIsStructural) {
|
|
2135
|
+
block.type = "heading";
|
|
2136
|
+
block.level = 3;
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2709
2140
|
}
|
|
2710
2141
|
var TABLE_CAPTION_RE = /^[<\[(【〈]?\s*(표|그림|도표|Table|Figure|Fig\.?)\s*[\d①-⑮][\d.\-]*\s*[\])】〉>]?[.:]?\s*/i;
|
|
2711
2142
|
var CAPTION_MAX_LENGTH = 100;
|
|
@@ -2742,250 +2173,611 @@ function detectTableCaptions(blocks) {
|
|
|
2742
2173
|
}
|
|
2743
2174
|
}
|
|
2744
2175
|
}
|
|
2745
|
-
var KOREAN_LIST_SEQ = "\uAC00\uB098\uB2E4\uB77C\uB9C8\uBC14\uC0AC\uC544\uC790\uCC28\uCE74\uD0C0\uD30C\uD558";
|
|
2746
|
-
function parseListLabel(text) {
|
|
2747
|
-
let m = text.match(/^(\d{1,2})\.(?!\d)\s+/);
|
|
2748
|
-
if (m) return { family: "arabicDot", ord: parseInt(m[1], 10) };
|
|
2749
|
-
m = text.match(/^([가-하])\.\s+/);
|
|
2750
|
-
if (m) {
|
|
2751
|
-
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2752
|
-
if (idx >= 0) return { family: "korDot", ord: idx + 1 };
|
|
2753
|
-
}
|
|
2754
|
-
m = text.match(/^(\d{1,2})\)\s*/);
|
|
2755
|
-
if (m) return { family: "arabicParen", ord: parseInt(m[1], 10) };
|
|
2756
|
-
m = text.match(/^([가-하])\)\s*/);
|
|
2757
|
-
if (m) {
|
|
2758
|
-
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2759
|
-
if (idx >= 0) return { family: "korParen", ord: idx + 1 };
|
|
2176
|
+
var KOREAN_LIST_SEQ = "\uAC00\uB098\uB2E4\uB77C\uB9C8\uBC14\uC0AC\uC544\uC790\uCC28\uCE74\uD0C0\uD30C\uD558";
|
|
2177
|
+
function parseListLabel(text) {
|
|
2178
|
+
let m = text.match(/^(\d{1,2})\.(?!\d)\s+/);
|
|
2179
|
+
if (m) return { family: "arabicDot", ord: parseInt(m[1], 10) };
|
|
2180
|
+
m = text.match(/^([가-하])\.\s+/);
|
|
2181
|
+
if (m) {
|
|
2182
|
+
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2183
|
+
if (idx >= 0) return { family: "korDot", ord: idx + 1 };
|
|
2184
|
+
}
|
|
2185
|
+
m = text.match(/^(\d{1,2})\)\s*/);
|
|
2186
|
+
if (m) return { family: "arabicParen", ord: parseInt(m[1], 10) };
|
|
2187
|
+
m = text.match(/^([가-하])\)\s*/);
|
|
2188
|
+
if (m) {
|
|
2189
|
+
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2190
|
+
if (idx >= 0) return { family: "korParen", ord: idx + 1 };
|
|
2191
|
+
}
|
|
2192
|
+
m = text.match(/^([①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮])\s*/);
|
|
2193
|
+
if (m) return { family: "circled", ord: m[1].charCodeAt(0) - 9312 + 1 };
|
|
2194
|
+
return null;
|
|
2195
|
+
}
|
|
2196
|
+
var ATTACHMENT_RE = /^붙\s*임\s*(\d+[.:]?)?\s/;
|
|
2197
|
+
function detectKoreanListBlocks(blocks) {
|
|
2198
|
+
const labeled = [];
|
|
2199
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2200
|
+
const b = blocks[i];
|
|
2201
|
+
if (b.type !== "paragraph" && b.type !== "list" || !b.text) continue;
|
|
2202
|
+
const label = parseListLabel(b.text.trim());
|
|
2203
|
+
if (label) labeled.push({ idx: i, label });
|
|
2204
|
+
}
|
|
2205
|
+
const validated = /* @__PURE__ */ new Set();
|
|
2206
|
+
const byFamily = /* @__PURE__ */ new Map();
|
|
2207
|
+
for (const l of labeled) {
|
|
2208
|
+
const arr = byFamily.get(l.label.family) || [];
|
|
2209
|
+
arr.push(l);
|
|
2210
|
+
byFamily.set(l.label.family, arr);
|
|
2211
|
+
}
|
|
2212
|
+
for (const arr of byFamily.values()) {
|
|
2213
|
+
let chain = [];
|
|
2214
|
+
for (const item of arr) {
|
|
2215
|
+
const prev = chain[chain.length - 1];
|
|
2216
|
+
if (prev && item.label.ord === prev.label.ord + 1 && item.idx - prev.idx <= 20) {
|
|
2217
|
+
chain.push(item);
|
|
2218
|
+
} else {
|
|
2219
|
+
if (chain.length >= 2) for (const c of chain) validated.add(c.idx);
|
|
2220
|
+
chain = [item];
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
if (chain.length >= 2) for (const c of chain) validated.add(c.idx);
|
|
2224
|
+
}
|
|
2225
|
+
let familyStack = [];
|
|
2226
|
+
let lastTopLevelList = null;
|
|
2227
|
+
const toRemove = /* @__PURE__ */ new Set();
|
|
2228
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2229
|
+
const b = blocks[i];
|
|
2230
|
+
if (b.type === "table" || b.type === "heading" || b.type === "separator") {
|
|
2231
|
+
familyStack = [];
|
|
2232
|
+
lastTopLevelList = null;
|
|
2233
|
+
continue;
|
|
2234
|
+
}
|
|
2235
|
+
if (b.type !== "paragraph" && b.type !== "list" || !b.text) continue;
|
|
2236
|
+
const text = b.text.trim();
|
|
2237
|
+
if (b.type === "paragraph" && ATTACHMENT_RE.test(text)) {
|
|
2238
|
+
blocks[i] = { ...b, type: "list", listType: "unordered" };
|
|
2239
|
+
continue;
|
|
2240
|
+
}
|
|
2241
|
+
if (!validated.has(i)) continue;
|
|
2242
|
+
const label = parseListLabel(text);
|
|
2243
|
+
let depth = familyStack.indexOf(label.family);
|
|
2244
|
+
if (depth < 0) {
|
|
2245
|
+
familyStack.push(label.family);
|
|
2246
|
+
depth = familyStack.length - 1;
|
|
2247
|
+
} else {
|
|
2248
|
+
familyStack = familyStack.slice(0, depth + 1);
|
|
2249
|
+
}
|
|
2250
|
+
const listType = label.family === "arabicDot" ? "ordered" : "unordered";
|
|
2251
|
+
const listBlock = { ...b, type: "list", listType };
|
|
2252
|
+
if (depth === 0) {
|
|
2253
|
+
blocks[i] = listBlock;
|
|
2254
|
+
lastTopLevelList = listBlock;
|
|
2255
|
+
} else if (lastTopLevelList) {
|
|
2256
|
+
if (!lastTopLevelList.children) lastTopLevelList.children = [];
|
|
2257
|
+
lastTopLevelList.children.push(listBlock);
|
|
2258
|
+
toRemove.add(i);
|
|
2259
|
+
} else {
|
|
2260
|
+
blocks[i] = listBlock;
|
|
2261
|
+
lastTopLevelList = listBlock;
|
|
2262
|
+
}
|
|
2263
|
+
}
|
|
2264
|
+
if (toRemove.size > 0) {
|
|
2265
|
+
const sorted = [...toRemove].sort((a, b) => b - a);
|
|
2266
|
+
for (const idx of sorted) blocks.splice(idx, 1);
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
function detectListBlocks(blocks) {
|
|
2270
|
+
const result = [];
|
|
2271
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2272
|
+
const block = blocks[i];
|
|
2273
|
+
if (block.type === "paragraph" && block.text) {
|
|
2274
|
+
const text = block.text.trim();
|
|
2275
|
+
if (/^\d+\.\s/.test(text)) {
|
|
2276
|
+
result.push({ ...block, type: "list", listType: "ordered", text: block.text });
|
|
2277
|
+
continue;
|
|
2278
|
+
}
|
|
2279
|
+
if (/^[○●·※▶▷◆◇\-]\s/.test(text)) {
|
|
2280
|
+
result.push({ ...block, type: "list", listType: "unordered", text: block.text });
|
|
2281
|
+
continue;
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
result.push(block);
|
|
2285
|
+
}
|
|
2286
|
+
return result;
|
|
2287
|
+
}
|
|
2288
|
+
var KOREAN_TABLE_HEADER_RE = /^\(?(구분|항목|종류|분류|유형|대상|내용|기간|금액|비율|방법|절차|요건|조건|근거|목적|범위|기준)\)?[:\s]/;
|
|
2289
|
+
var KV_FALSE_POSITIVE_RE = /\d{1,2}:\d{2}|:\/\/|\d+:\d+/;
|
|
2290
|
+
function detectSpecialKoreanTables(blocks) {
|
|
2291
|
+
const result = [];
|
|
2292
|
+
let kvLines = [];
|
|
2293
|
+
const flushKvTable = () => {
|
|
2294
|
+
if (kvLines.length < 2) {
|
|
2295
|
+
for (const kv of kvLines) result.push(kv.block);
|
|
2296
|
+
kvLines = [];
|
|
2297
|
+
return;
|
|
2298
|
+
}
|
|
2299
|
+
const cells = kvLines.map((kv) => {
|
|
2300
|
+
if (kv.value) {
|
|
2301
|
+
return [
|
|
2302
|
+
{ text: kv.key, colSpan: 1, rowSpan: 1 },
|
|
2303
|
+
{ text: kv.value, colSpan: 1, rowSpan: 1 }
|
|
2304
|
+
];
|
|
2305
|
+
}
|
|
2306
|
+
return [
|
|
2307
|
+
{ text: kv.key, colSpan: 2, rowSpan: 1 },
|
|
2308
|
+
{ text: "", colSpan: 1, rowSpan: 1 }
|
|
2309
|
+
];
|
|
2310
|
+
});
|
|
2311
|
+
const irTable = {
|
|
2312
|
+
rows: cells.length,
|
|
2313
|
+
cols: 2,
|
|
2314
|
+
cells,
|
|
2315
|
+
hasHeader: true
|
|
2316
|
+
};
|
|
2317
|
+
const firstBlock = kvLines[0].block;
|
|
2318
|
+
result.push({
|
|
2319
|
+
type: "table",
|
|
2320
|
+
table: irTable,
|
|
2321
|
+
pageNumber: firstBlock.pageNumber,
|
|
2322
|
+
bbox: firstBlock.bbox
|
|
2323
|
+
});
|
|
2324
|
+
kvLines = [];
|
|
2325
|
+
};
|
|
2326
|
+
for (const block of blocks) {
|
|
2327
|
+
if (block.type !== "paragraph" || !block.text) {
|
|
2328
|
+
flushKvTable();
|
|
2329
|
+
result.push(block);
|
|
2330
|
+
continue;
|
|
2331
|
+
}
|
|
2332
|
+
const text = block.text.trim();
|
|
2333
|
+
if (KOREAN_TABLE_HEADER_RE.test(text)) {
|
|
2334
|
+
const colonIdx = text.indexOf(":");
|
|
2335
|
+
if (colonIdx >= 0) {
|
|
2336
|
+
kvLines.push({
|
|
2337
|
+
key: text.slice(0, colonIdx).trim(),
|
|
2338
|
+
value: text.slice(colonIdx + 1).trim(),
|
|
2339
|
+
block
|
|
2340
|
+
});
|
|
2341
|
+
} else {
|
|
2342
|
+
const spaceIdx = text.search(/\s/);
|
|
2343
|
+
if (spaceIdx > 0) {
|
|
2344
|
+
kvLines.push({
|
|
2345
|
+
key: text.slice(0, spaceIdx).trim(),
|
|
2346
|
+
value: text.slice(spaceIdx + 1).trim(),
|
|
2347
|
+
block
|
|
2348
|
+
});
|
|
2349
|
+
} else {
|
|
2350
|
+
kvLines.push({ key: text, value: "", block });
|
|
2351
|
+
}
|
|
2352
|
+
}
|
|
2353
|
+
continue;
|
|
2354
|
+
}
|
|
2355
|
+
if (kvLines.length > 0 && text.includes(":")) {
|
|
2356
|
+
if (!KV_FALSE_POSITIVE_RE.test(text) && !text.includes("(") && !text.includes(")")) {
|
|
2357
|
+
const colonIdx = text.indexOf(":");
|
|
2358
|
+
const key = text.slice(0, colonIdx).trim();
|
|
2359
|
+
if (/^[가-힣]+$/.test(key) && key.length >= 2 && key.length <= 8) {
|
|
2360
|
+
kvLines.push({
|
|
2361
|
+
key,
|
|
2362
|
+
value: text.slice(colonIdx + 1).trim(),
|
|
2363
|
+
block
|
|
2364
|
+
});
|
|
2365
|
+
continue;
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
flushKvTable();
|
|
2370
|
+
result.push(block);
|
|
2371
|
+
}
|
|
2372
|
+
flushKvTable();
|
|
2373
|
+
return result;
|
|
2374
|
+
}
|
|
2375
|
+
function removeHeaderFooterBlocks(blocks, pageHeights, warnings) {
|
|
2376
|
+
const ZONE_RATIO = 0.12;
|
|
2377
|
+
const MIN_REPEAT = 3;
|
|
2378
|
+
const topEntries = [];
|
|
2379
|
+
const bottomEntries = [];
|
|
2380
|
+
for (let bi = 0; bi < blocks.length; bi++) {
|
|
2381
|
+
const b = blocks[bi];
|
|
2382
|
+
if (!b.bbox || !b.pageNumber || !b.text?.trim()) continue;
|
|
2383
|
+
const ph = pageHeights.get(b.bbox.page) || pageHeights.get(b.pageNumber);
|
|
2384
|
+
if (!ph) continue;
|
|
2385
|
+
const blockTop = ph - (b.bbox.y + b.bbox.height);
|
|
2386
|
+
const blockBottom = ph - b.bbox.y;
|
|
2387
|
+
const entry = { blockIdx: bi, page: b.pageNumber, text: b.text.trim() };
|
|
2388
|
+
if (blockBottom <= ph * ZONE_RATIO) bottomEntries.push(entry);
|
|
2389
|
+
else if (blockTop >= ph * (1 - ZONE_RATIO)) topEntries.push(entry);
|
|
2390
|
+
}
|
|
2391
|
+
const removeSet = /* @__PURE__ */ new Set();
|
|
2392
|
+
for (const entries of [topEntries, bottomEntries]) {
|
|
2393
|
+
if (entries.length === 0) continue;
|
|
2394
|
+
const patternCount = /* @__PURE__ */ new Map();
|
|
2395
|
+
const patternPages = /* @__PURE__ */ new Map();
|
|
2396
|
+
for (const e of entries) {
|
|
2397
|
+
const norm = e.text.replace(/\d+/g, "#");
|
|
2398
|
+
patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
|
|
2399
|
+
const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
|
|
2400
|
+
pages.add(e.page);
|
|
2401
|
+
patternPages.set(norm, pages);
|
|
2402
|
+
}
|
|
2403
|
+
const repeatedPatterns = /* @__PURE__ */ new Set();
|
|
2404
|
+
for (const [p, count] of patternCount) {
|
|
2405
|
+
if (count >= MIN_REPEAT && (patternPages.get(p)?.size ?? 0) >= MIN_REPEAT) {
|
|
2406
|
+
repeatedPatterns.add(p);
|
|
2407
|
+
}
|
|
2408
|
+
}
|
|
2409
|
+
for (const e of entries) {
|
|
2410
|
+
const norm = e.text.replace(/\d+/g, "#");
|
|
2411
|
+
if (repeatedPatterns.has(norm)) {
|
|
2412
|
+
removeSet.add(e.blockIdx);
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
if (removeSet.size > 0) {
|
|
2417
|
+
warnings.push({ message: `${removeSet.size}\uAC1C \uBA38\uB9AC\uAE00/\uBC14\uB2E5\uAE00 \uC694\uC18C \uC81C\uAC70\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
2418
|
+
}
|
|
2419
|
+
return [...removeSet].sort((a, b) => a - b);
|
|
2420
|
+
}
|
|
2421
|
+
|
|
2422
|
+
// src/pdf/page-blocks.ts
|
|
2423
|
+
function extractPageBlocksWithLines(items, pageNum, opList, pageWidth, pageHeight) {
|
|
2424
|
+
if (items.length === 0) return [];
|
|
2425
|
+
let { horizontals, verticals } = extractLines(opList.fnArray, opList.argsArray);
|
|
2426
|
+
({ horizontals, verticals } = filterPageBorderLines(horizontals, verticals, pageWidth, pageHeight));
|
|
2427
|
+
({ horizontals, verticals } = preprocessLines(horizontals, verticals));
|
|
2428
|
+
markStrikethroughItems(items, horizontals);
|
|
2429
|
+
wrapStrikethroughRuns(items);
|
|
2430
|
+
const grids = buildTableGrids(horizontals, verticals);
|
|
2431
|
+
if (grids.length > 0) {
|
|
2432
|
+
return extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals);
|
|
2433
|
+
}
|
|
2434
|
+
return extractPageBlocksFallback(items, pageNum);
|
|
2435
|
+
}
|
|
2436
|
+
var STRIKE_MAX_THICKNESS = 2;
|
|
2437
|
+
var STRIKE_MAX_THICKNESS_RATIO = 0.25;
|
|
2438
|
+
var STRIKE_CENTER_TOLERANCE = 0.25;
|
|
2439
|
+
var STRIKE_MIN_OVERLAP_RATIO = 0.8;
|
|
2440
|
+
var STRIKE_MAX_LINE_TO_TEXT_RATIO = 1.5;
|
|
2441
|
+
function markStrikethroughItems(items, horizontals) {
|
|
2442
|
+
if (items.length === 0 || horizontals.length === 0) return;
|
|
2443
|
+
for (const line of horizontals) {
|
|
2444
|
+
if (line.lineWidth > STRIKE_MAX_THICKNESS) continue;
|
|
2445
|
+
const matches = [];
|
|
2446
|
+
for (const item of items) {
|
|
2447
|
+
const h = item.h > 0 ? item.h : item.fontSize;
|
|
2448
|
+
if (h <= 0 || item.w <= 0) continue;
|
|
2449
|
+
if (line.lineWidth > h * STRIKE_MAX_THICKNESS_RATIO) continue;
|
|
2450
|
+
const centerY = item.y + h * 0.4;
|
|
2451
|
+
if (Math.abs(line.y1 - centerY) > h * STRIKE_CENTER_TOLERANCE) continue;
|
|
2452
|
+
const overlap = Math.min(line.x2, item.x + item.w) - Math.max(line.x1, item.x);
|
|
2453
|
+
if (overlap / item.w < STRIKE_MIN_OVERLAP_RATIO) continue;
|
|
2454
|
+
matches.push(item);
|
|
2455
|
+
}
|
|
2456
|
+
if (matches.length === 0) continue;
|
|
2457
|
+
let totalW = 0;
|
|
2458
|
+
for (const m of matches) totalW += m.w;
|
|
2459
|
+
if (totalW <= 0 || (line.x2 - line.x1) / totalW > STRIKE_MAX_LINE_TO_TEXT_RATIO) continue;
|
|
2460
|
+
for (const m of matches) m.strike = true;
|
|
2760
2461
|
}
|
|
2761
|
-
m = text.match(/^([①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮])\s*/);
|
|
2762
|
-
if (m) return { family: "circled", ord: m[1].charCodeAt(0) - 9312 + 1 };
|
|
2763
|
-
return null;
|
|
2764
2462
|
}
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
const
|
|
2772
|
-
|
|
2463
|
+
function wrapStrikethroughRuns(items) {
|
|
2464
|
+
const struck = items.filter((i) => i.strike);
|
|
2465
|
+
if (struck.length === 0) return;
|
|
2466
|
+
const lines = /* @__PURE__ */ new Map();
|
|
2467
|
+
for (const item of struck) {
|
|
2468
|
+
const key = Math.round(item.y / 3);
|
|
2469
|
+
const arr = lines.get(key) || [];
|
|
2470
|
+
arr.push(item);
|
|
2471
|
+
lines.set(key, arr);
|
|
2773
2472
|
}
|
|
2774
|
-
const
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
arr.push(l);
|
|
2779
|
-
byFamily.set(l.label.family, arr);
|
|
2473
|
+
for (const arr of lines.values()) {
|
|
2474
|
+
arr.sort((a, b) => a.x - b.x);
|
|
2475
|
+
arr[0].text = "~~" + arr[0].text;
|
|
2476
|
+
arr[arr.length - 1].text = arr[arr.length - 1].text + "~~";
|
|
2780
2477
|
}
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2478
|
+
}
|
|
2479
|
+
function extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals) {
|
|
2480
|
+
const blocks = [];
|
|
2481
|
+
const usedItems = /* @__PURE__ */ new Set();
|
|
2482
|
+
const sortedGrids = [...grids].sort((a, b) => b.bbox.y2 - a.bbox.y2);
|
|
2483
|
+
for (const grid of sortedGrids) {
|
|
2484
|
+
const numGridRows = grid.rowYs.length - 1;
|
|
2485
|
+
const numGridCols = grid.colXs.length - 1;
|
|
2486
|
+
if (numGridRows === 1 && numGridCols >= 2) continue;
|
|
2487
|
+
if (numGridCols === 1 && numGridRows >= 2) continue;
|
|
2488
|
+
const tableItems = [];
|
|
2489
|
+
const pad = 3;
|
|
2490
|
+
const gridW = grid.bbox.x2 - grid.bbox.x1;
|
|
2491
|
+
for (const item of items) {
|
|
2492
|
+
if (usedItems.has(item)) continue;
|
|
2493
|
+
if (item.y < grid.bbox.y1 - pad || item.y > grid.bbox.y2 + pad) continue;
|
|
2494
|
+
if (item.x < grid.bbox.x1 - pad || item.x + item.w > grid.bbox.x2 + pad) continue;
|
|
2495
|
+
if (gridW < 120 && item.x + item.w > grid.bbox.x2 - 2) continue;
|
|
2496
|
+
tableItems.push(item);
|
|
2497
|
+
usedItems.add(item);
|
|
2498
|
+
}
|
|
2499
|
+
const cells = extractCells(grid, horizontals, verticals);
|
|
2500
|
+
if (cells.length === 0) continue;
|
|
2501
|
+
const textItems = tableItems.map((i) => ({
|
|
2502
|
+
text: i.text,
|
|
2503
|
+
x: i.x,
|
|
2504
|
+
y: i.y,
|
|
2505
|
+
w: i.w,
|
|
2506
|
+
h: i.h,
|
|
2507
|
+
fontSize: i.fontSize,
|
|
2508
|
+
fontName: i.fontName,
|
|
2509
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2510
|
+
}));
|
|
2511
|
+
const cellTextMap = mapTextToCells(textItems, cells);
|
|
2512
|
+
const numRows = grid.rowYs.length - 1;
|
|
2513
|
+
const numCols = grid.colXs.length - 1;
|
|
2514
|
+
const irGrid = Array.from(
|
|
2515
|
+
{ length: numRows },
|
|
2516
|
+
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
2517
|
+
);
|
|
2518
|
+
for (const cell of cells) {
|
|
2519
|
+
const cellItems = cellTextMap.get(cell) || [];
|
|
2520
|
+
let text = cellTextToString(cellItems);
|
|
2521
|
+
text = text.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2522
|
+
text = text.split("\n").map((line) => collapseEvenSpacing(line)).join("\n");
|
|
2523
|
+
irGrid[cell.row][cell.col] = {
|
|
2524
|
+
text,
|
|
2525
|
+
colSpan: cell.colSpan,
|
|
2526
|
+
rowSpan: cell.rowSpan
|
|
2527
|
+
};
|
|
2528
|
+
}
|
|
2529
|
+
let finalGrid = irGrid;
|
|
2530
|
+
let finalRows = numRows;
|
|
2531
|
+
if (numRows <= 2 && numCols >= 3) {
|
|
2532
|
+
const rebuilt = normalizeUndersegmentedTable(irGrid, grid.colXs, textItems);
|
|
2533
|
+
if (rebuilt) {
|
|
2534
|
+
finalGrid = rebuilt.map((row) => row.map((rawText) => {
|
|
2535
|
+
const cleaned = rawText.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2536
|
+
return {
|
|
2537
|
+
text: cleaned.split("\n").map((line) => collapseEvenSpacing(line)).join("\n"),
|
|
2538
|
+
colSpan: 1,
|
|
2539
|
+
rowSpan: 1
|
|
2540
|
+
};
|
|
2541
|
+
}));
|
|
2542
|
+
finalRows = finalGrid.length;
|
|
2790
2543
|
}
|
|
2791
2544
|
}
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
const
|
|
2799
|
-
if (
|
|
2800
|
-
|
|
2801
|
-
|
|
2545
|
+
const irTable = {
|
|
2546
|
+
rows: finalRows,
|
|
2547
|
+
cols: numCols,
|
|
2548
|
+
cells: finalGrid,
|
|
2549
|
+
hasHeader: finalRows > 1
|
|
2550
|
+
};
|
|
2551
|
+
const hasContent = finalGrid.some((row) => row.some((cell) => cell.text.trim() !== ""));
|
|
2552
|
+
if (!hasContent) continue;
|
|
2553
|
+
const tableBbox = {
|
|
2554
|
+
page: pageNum,
|
|
2555
|
+
x: grid.bbox.x1,
|
|
2556
|
+
y: grid.bbox.y1,
|
|
2557
|
+
width: grid.bbox.x2 - grid.bbox.x1,
|
|
2558
|
+
height: grid.bbox.y2 - grid.bbox.y1
|
|
2559
|
+
};
|
|
2560
|
+
if (shouldDemoteTable(irTable)) {
|
|
2561
|
+
const demoted = demoteTableToText(irTable);
|
|
2562
|
+
if (demoted) {
|
|
2563
|
+
const text = numGridRows === 1 ? "\n" + demoted + "\n" : demoted;
|
|
2564
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox: tableBbox, style: dominantStyle(tableItems) });
|
|
2565
|
+
}
|
|
2802
2566
|
continue;
|
|
2803
2567
|
}
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2568
|
+
blocks.push({ type: "table", table: irTable, pageNumber: pageNum, bbox: tableBbox });
|
|
2569
|
+
}
|
|
2570
|
+
let remaining = items.filter((i) => !usedItems.has(i));
|
|
2571
|
+
if (remaining.length > 0) {
|
|
2572
|
+
remaining.sort((a, b) => b.y - a.y || a.x - b.x);
|
|
2573
|
+
const clusterItems = remaining.map((i) => ({
|
|
2574
|
+
text: i.text,
|
|
2575
|
+
x: i.x,
|
|
2576
|
+
y: i.y,
|
|
2577
|
+
w: i.w,
|
|
2578
|
+
h: i.h,
|
|
2579
|
+
fontSize: i.fontSize,
|
|
2580
|
+
fontName: i.fontName,
|
|
2581
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2582
|
+
}));
|
|
2583
|
+
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2584
|
+
if (clusterResults.length > 0) {
|
|
2585
|
+
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2586
|
+
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2587
|
+
const usedClusterIndices = /* @__PURE__ */ new Set();
|
|
2588
|
+
for (const cr of clusterResults) {
|
|
2589
|
+
for (const ci of cr.usedItems) {
|
|
2590
|
+
const idx = ciToIdx.get(ci);
|
|
2591
|
+
if (idx !== void 0) usedClusterIndices.add(idx);
|
|
2592
|
+
}
|
|
2593
|
+
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2594
|
+
}
|
|
2595
|
+
remaining = remaining.filter((_, idx) => !usedClusterIndices.has(idx));
|
|
2809
2596
|
}
|
|
2810
|
-
if (
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2597
|
+
if (remaining.length > 0) {
|
|
2598
|
+
const allY = remaining.map((i) => i.y);
|
|
2599
|
+
const pageH = safeMax(allY) - safeMin(allY);
|
|
2600
|
+
const groups = xyCutOrder(remaining, Math.max(15, pageH * 0.03));
|
|
2601
|
+
const textBlocks = [];
|
|
2602
|
+
for (const group of groups) {
|
|
2603
|
+
if (group.length === 0) continue;
|
|
2604
|
+
const groupBlocks = extractPageBlocksFallback(group, pageNum);
|
|
2605
|
+
for (const b of groupBlocks) textBlocks.push(b);
|
|
2606
|
+
}
|
|
2607
|
+
const finalTextBlocks = detectListBlocks(textBlocks);
|
|
2608
|
+
for (const b of finalTextBlocks) blocks.push(b);
|
|
2609
|
+
}
|
|
2610
|
+
blocks.sort((a, b) => {
|
|
2611
|
+
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2612
|
+
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2613
|
+
return by - ay;
|
|
2614
|
+
});
|
|
2615
|
+
return mergeAdjacentTableBlocks(blocks);
|
|
2616
|
+
}
|
|
2617
|
+
return mergeAdjacentTableBlocks(blocks);
|
|
2618
|
+
}
|
|
2619
|
+
var NEIGHBOR_TABLE_EPSILON = 0.2;
|
|
2620
|
+
function mergeCrossPageTables(blocks) {
|
|
2621
|
+
for (let i = blocks.length - 2; i >= 0; i--) {
|
|
2622
|
+
const prev = blocks[i];
|
|
2623
|
+
const curr = blocks[i + 1];
|
|
2624
|
+
if (prev.type !== "table" || curr.type !== "table" || !prev.table || !curr.table) continue;
|
|
2625
|
+
if (!prev.pageNumber || !curr.pageNumber || curr.pageNumber !== prev.pageNumber + 1) continue;
|
|
2626
|
+
if (prev.table.cols !== curr.table.cols) continue;
|
|
2627
|
+
if (!prev.bbox || !curr.bbox) continue;
|
|
2628
|
+
const width = Math.max(prev.bbox.width, curr.bbox.width, 1);
|
|
2629
|
+
const leftDiff = Math.abs(prev.bbox.x - curr.bbox.x);
|
|
2630
|
+
const rightDiff = Math.abs(prev.bbox.x + prev.bbox.width - (curr.bbox.x + curr.bbox.width));
|
|
2631
|
+
if (leftDiff > width * NEIGHBOR_TABLE_EPSILON || rightDiff > width * NEIGHBOR_TABLE_EPSILON) continue;
|
|
2632
|
+
let currCells = curr.table.cells;
|
|
2633
|
+
if (currCells.length > 1 && prev.table.cells.length > 0 && rowTextsEqual(prev.table.cells[0], currCells[0])) {
|
|
2634
|
+
currCells = currCells.slice(1);
|
|
2818
2635
|
}
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
blocks[i] = listBlock;
|
|
2823
|
-
lastTopLevelList = listBlock;
|
|
2824
|
-
} else if (lastTopLevelList) {
|
|
2825
|
-
if (!lastTopLevelList.children) lastTopLevelList.children = [];
|
|
2826
|
-
lastTopLevelList.children.push(listBlock);
|
|
2827
|
-
toRemove.add(i);
|
|
2828
|
-
} else {
|
|
2829
|
-
blocks[i] = listBlock;
|
|
2830
|
-
lastTopLevelList = listBlock;
|
|
2636
|
+
if (currCells.length === 0) {
|
|
2637
|
+
blocks.splice(i + 1, 1);
|
|
2638
|
+
continue;
|
|
2831
2639
|
}
|
|
2640
|
+
const merged = {
|
|
2641
|
+
rows: prev.table.rows + currCells.length,
|
|
2642
|
+
cols: prev.table.cols,
|
|
2643
|
+
cells: [...prev.table.cells, ...currCells],
|
|
2644
|
+
hasHeader: prev.table.hasHeader,
|
|
2645
|
+
caption: prev.table.caption
|
|
2646
|
+
};
|
|
2647
|
+
blocks[i] = { ...prev, table: merged };
|
|
2648
|
+
blocks.splice(i + 1, 1);
|
|
2832
2649
|
}
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2650
|
+
}
|
|
2651
|
+
function rowTextsEqual(a, b) {
|
|
2652
|
+
if (a.length !== b.length) return false;
|
|
2653
|
+
const norm = (t) => t.replace(/\s+/g, "");
|
|
2654
|
+
for (let i = 0; i < a.length; i++) {
|
|
2655
|
+
if (norm(a[i].text) !== norm(b[i].text)) return false;
|
|
2836
2656
|
}
|
|
2657
|
+
return a.some((c) => c.text.trim() !== "");
|
|
2837
2658
|
}
|
|
2838
|
-
function
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
}
|
|
2659
|
+
function mergeAdjacentTableBlocks(blocks) {
|
|
2660
|
+
if (blocks.length <= 1) return blocks;
|
|
2661
|
+
const result = [blocks[0]];
|
|
2662
|
+
for (let i = 1; i < blocks.length; i++) {
|
|
2663
|
+
const prev = result[result.length - 1];
|
|
2664
|
+
const curr = blocks[i];
|
|
2665
|
+
if (prev.type === "table" && curr.type === "table" && prev.table && curr.table && prev.table.cols === curr.table.cols) {
|
|
2666
|
+
const merged = {
|
|
2667
|
+
rows: prev.table.rows + curr.table.rows,
|
|
2668
|
+
cols: prev.table.cols,
|
|
2669
|
+
cells: [...prev.table.cells, ...curr.table.cells],
|
|
2670
|
+
hasHeader: prev.table.hasHeader
|
|
2671
|
+
};
|
|
2672
|
+
result[result.length - 1] = { ...prev, table: merged };
|
|
2673
|
+
} else {
|
|
2674
|
+
result.push(curr);
|
|
2852
2675
|
}
|
|
2853
|
-
result.push(block);
|
|
2854
2676
|
}
|
|
2855
2677
|
return result;
|
|
2856
2678
|
}
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
const
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2679
|
+
function extractPageBlocksFallback(items, pageNum) {
|
|
2680
|
+
if (items.length === 0) return [];
|
|
2681
|
+
const blocks = [];
|
|
2682
|
+
const clusterItems = items.map((i) => ({
|
|
2683
|
+
text: i.text,
|
|
2684
|
+
x: i.x,
|
|
2685
|
+
y: i.y,
|
|
2686
|
+
w: i.w,
|
|
2687
|
+
h: i.h,
|
|
2688
|
+
fontSize: i.fontSize,
|
|
2689
|
+
fontName: i.fontName,
|
|
2690
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2691
|
+
}));
|
|
2692
|
+
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2693
|
+
if (clusterResults.length > 0) {
|
|
2694
|
+
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2695
|
+
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2696
|
+
const usedIndices = /* @__PURE__ */ new Set();
|
|
2697
|
+
for (const cr of clusterResults) {
|
|
2698
|
+
for (const ci of cr.usedItems) {
|
|
2699
|
+
const idx = ciToIdx.get(ci);
|
|
2700
|
+
if (idx !== void 0) usedIndices.add(idx);
|
|
2874
2701
|
}
|
|
2875
|
-
|
|
2876
|
-
{ text: kv.key, colSpan: 2, rowSpan: 1 },
|
|
2877
|
-
{ text: "", colSpan: 1, rowSpan: 1 }
|
|
2878
|
-
];
|
|
2879
|
-
});
|
|
2880
|
-
const irTable = {
|
|
2881
|
-
rows: cells.length,
|
|
2882
|
-
cols: 2,
|
|
2883
|
-
cells,
|
|
2884
|
-
hasHeader: true
|
|
2885
|
-
};
|
|
2886
|
-
const firstBlock = kvLines[0].block;
|
|
2887
|
-
result.push({
|
|
2888
|
-
type: "table",
|
|
2889
|
-
table: irTable,
|
|
2890
|
-
pageNumber: firstBlock.pageNumber,
|
|
2891
|
-
bbox: firstBlock.bbox
|
|
2892
|
-
});
|
|
2893
|
-
kvLines = [];
|
|
2894
|
-
};
|
|
2895
|
-
for (const block of blocks) {
|
|
2896
|
-
if (block.type !== "paragraph" || !block.text) {
|
|
2897
|
-
flushKvTable();
|
|
2898
|
-
result.push(block);
|
|
2899
|
-
continue;
|
|
2702
|
+
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2900
2703
|
}
|
|
2901
|
-
const
|
|
2902
|
-
if (
|
|
2903
|
-
const
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
});
|
|
2910
|
-
} else {
|
|
2911
|
-
const spaceIdx = text.search(/\s/);
|
|
2912
|
-
if (spaceIdx > 0) {
|
|
2913
|
-
kvLines.push({
|
|
2914
|
-
key: text.slice(0, spaceIdx).trim(),
|
|
2915
|
-
value: text.slice(spaceIdx + 1).trim(),
|
|
2916
|
-
block
|
|
2917
|
-
});
|
|
2918
|
-
} else {
|
|
2919
|
-
kvLines.push({ key: text, value: "", block });
|
|
2920
|
-
}
|
|
2704
|
+
const remaining = items.filter((_, idx) => !usedIndices.has(idx));
|
|
2705
|
+
if (remaining.length > 0) {
|
|
2706
|
+
const yLines = mergeSuperscriptLines(groupByY(remaining));
|
|
2707
|
+
for (const line of yLines) {
|
|
2708
|
+
const text = mergeLineSimple(line);
|
|
2709
|
+
if (!text.trim()) continue;
|
|
2710
|
+
const bbox = computeBBox(line, pageNum);
|
|
2711
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2921
2712
|
}
|
|
2922
|
-
continue;
|
|
2923
2713
|
}
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2714
|
+
blocks.sort((a, b) => {
|
|
2715
|
+
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2716
|
+
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2717
|
+
return by - ay;
|
|
2718
|
+
});
|
|
2719
|
+
} else {
|
|
2720
|
+
const allYLines = mergeSuperscriptLines(groupByY(items));
|
|
2721
|
+
const columns = detectColumns(allYLines);
|
|
2722
|
+
if (columns && columns.length >= 3) {
|
|
2723
|
+
const tableText = extractWithColumns(allYLines, columns);
|
|
2724
|
+
const bbox = computeBBox(items, pageNum);
|
|
2725
|
+
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(items) });
|
|
2726
|
+
} else {
|
|
2727
|
+
const allY = items.map((i) => i.y);
|
|
2728
|
+
const pageHeight = safeMax(allY) - safeMin(allY);
|
|
2729
|
+
const gapThreshold = Math.max(15, pageHeight * 0.03);
|
|
2730
|
+
const orderedGroups = xyCutOrder(items, gapThreshold);
|
|
2731
|
+
for (const group of orderedGroups) {
|
|
2732
|
+
if (group.length === 0) continue;
|
|
2733
|
+
const yLines = mergeSuperscriptLines(groupByY(group));
|
|
2734
|
+
const groupColumns = detectColumns(yLines);
|
|
2735
|
+
if (groupColumns && groupColumns.length >= 3) {
|
|
2736
|
+
const tableText = extractWithColumns(yLines, groupColumns);
|
|
2737
|
+
const bbox = computeBBox(group, pageNum);
|
|
2738
|
+
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
|
|
2739
|
+
} else {
|
|
2740
|
+
for (const line of yLines) {
|
|
2741
|
+
const text = mergeLineSimple(line);
|
|
2742
|
+
if (!text.trim()) continue;
|
|
2743
|
+
const bbox = computeBBox(line, pageNum);
|
|
2744
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2745
|
+
}
|
|
2935
2746
|
}
|
|
2936
2747
|
}
|
|
2937
2748
|
}
|
|
2938
|
-
flushKvTable();
|
|
2939
|
-
result.push(block);
|
|
2940
2749
|
}
|
|
2941
|
-
|
|
2942
|
-
return result;
|
|
2750
|
+
return detectSpecialKoreanTables(blocks);
|
|
2943
2751
|
}
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
const
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
const blockTop = ph - (b.bbox.y + b.bbox.height);
|
|
2955
|
-
const blockBottom = ph - b.bbox.y;
|
|
2956
|
-
const entry = { blockIdx: bi, page: b.pageNumber, text: b.text.trim() };
|
|
2957
|
-
if (blockBottom <= ph * ZONE_RATIO) bottomEntries.push(entry);
|
|
2958
|
-
else if (blockTop >= ph * (1 - ZONE_RATIO)) topEntries.push(entry);
|
|
2959
|
-
}
|
|
2960
|
-
const removeSet = /* @__PURE__ */ new Set();
|
|
2961
|
-
for (const entries of [topEntries, bottomEntries]) {
|
|
2962
|
-
if (entries.length === 0) continue;
|
|
2963
|
-
const patternCount = /* @__PURE__ */ new Map();
|
|
2964
|
-
const patternPages = /* @__PURE__ */ new Map();
|
|
2965
|
-
for (const e of entries) {
|
|
2966
|
-
const norm = e.text.replace(/\d+/g, "#");
|
|
2967
|
-
patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
|
|
2968
|
-
const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
|
|
2969
|
-
pages.add(e.page);
|
|
2970
|
-
patternPages.set(norm, pages);
|
|
2971
|
-
}
|
|
2972
|
-
const repeatedPatterns = /* @__PURE__ */ new Set();
|
|
2973
|
-
for (const [p, count] of patternCount) {
|
|
2974
|
-
if (count >= MIN_REPEAT && (patternPages.get(p)?.size ?? 0) >= MIN_REPEAT) {
|
|
2975
|
-
repeatedPatterns.add(p);
|
|
2976
|
-
}
|
|
2977
|
-
}
|
|
2978
|
-
for (const e of entries) {
|
|
2979
|
-
const norm = e.text.replace(/\d+/g, "#");
|
|
2980
|
-
if (repeatedPatterns.has(norm)) {
|
|
2981
|
-
removeSet.add(e.blockIdx);
|
|
2752
|
+
|
|
2753
|
+
// src/pdf/text-clean.ts
|
|
2754
|
+
function sanitizeBlockControlChars(blocks) {
|
|
2755
|
+
for (const b of blocks) {
|
|
2756
|
+
if (b.text) b.text = stripControlChars(b.text);
|
|
2757
|
+
if (b.table) {
|
|
2758
|
+
for (const row of b.table.cells) {
|
|
2759
|
+
for (const cell of row) {
|
|
2760
|
+
if (cell.text) cell.text = stripControlChars(cell.text);
|
|
2761
|
+
}
|
|
2982
2762
|
}
|
|
2983
2763
|
}
|
|
2764
|
+
if (b.children) sanitizeBlockControlChars(b.children);
|
|
2984
2765
|
}
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2766
|
+
}
|
|
2767
|
+
function cleanPdfText(text) {
|
|
2768
|
+
return mergeKoreanLines(
|
|
2769
|
+
stripControlChars(text).replace(/^\d{1,4}\n/, "").replace(/^[\s]*[-–—]\s*[-–—]?\d+[-–—]?[\s]*[-–—]?[\s]*$/gm, "").replace(/^\s*\d+\s*\/\s*\d+\s*$/gm, "").replace(/\n\d{1,4}\n/g, "\n").replace(/\n\d{1,4}$/, "").replace(/^#{1,6}\s*\d{1,4}\s*$/gm, "")
|
|
2770
|
+
).replace(/^(?!\| ---).*$/gm, (line) => {
|
|
2771
|
+
if (/^\s*\${1,2}.+\${1,2}\s*$/.test(line)) return line;
|
|
2772
|
+
return collapseEvenSpacing(line);
|
|
2773
|
+
}).replace(/([□■◆○●▶ㅇ])\s+([가-힣])\s+([가-힣])/g, "$1 $2$3").replace(/\\~\\~/g, "~~").replace(/~~~~/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
|
2774
|
+
}
|
|
2775
|
+
function startsWithMarker(line) {
|
|
2776
|
+
const t = line.trimStart();
|
|
2777
|
+
return /^[가-힣ㄱ-ㅎ][.)]/.test(t) || /^\d+[.)]/.test(t) || /^\([가-힣ㄱ-ㅎ\d]+\)/.test(t) || /^[○●※▶▷◆◇■□★☆\-·]\s/.test(t) || /^제\d+[조항호장절]/.test(t);
|
|
2778
|
+
}
|
|
2779
|
+
function isStandaloneHeader(line) {
|
|
2780
|
+
return /^제\d+[조항호장절](\([^)]*\))?(\s+\S+){0,7}$/.test(line.trim());
|
|
2989
2781
|
}
|
|
2990
2782
|
function mergeKoreanLines(text) {
|
|
2991
2783
|
if (!text) return "";
|
|
@@ -3016,6 +2808,8 @@ function mergeKoreanLines(text) {
|
|
|
3016
2808
|
}
|
|
3017
2809
|
return result.join("\n");
|
|
3018
2810
|
}
|
|
2811
|
+
|
|
2812
|
+
// src/pdf/formula-ocr.ts
|
|
3019
2813
|
async function applyFormulaOcr(buffer, blocks, pageFilter, effectivePageCount, warnings, _onProgress) {
|
|
3020
2814
|
const formulaMod = await import("./formula-RXVSQPXI.js");
|
|
3021
2815
|
const { FormulaPipeline, ensureFormulaModels } = formulaMod;
|
|
@@ -3134,6 +2928,239 @@ async function applyFormulaOcr(buffer, blocks, pageFilter, effectivePageCount, w
|
|
|
3134
2928
|
function formatMb(bytes) {
|
|
3135
2929
|
return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
|
|
3136
2930
|
}
|
|
2931
|
+
|
|
2932
|
+
// src/pdf/polyfill.ts
|
|
2933
|
+
import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
|
|
2934
|
+
var g = globalThis;
|
|
2935
|
+
if (typeof g.DOMMatrix === "undefined") {
|
|
2936
|
+
g.DOMMatrix = class DOMMatrix {
|
|
2937
|
+
m = [1, 0, 0, 1, 0, 0];
|
|
2938
|
+
constructor(init) {
|
|
2939
|
+
if (init) this.m = init;
|
|
2940
|
+
}
|
|
2941
|
+
};
|
|
2942
|
+
}
|
|
2943
|
+
if (typeof g.Path2D === "undefined") {
|
|
2944
|
+
g.Path2D = class Path2D {
|
|
2945
|
+
};
|
|
2946
|
+
}
|
|
2947
|
+
g.pdfjsWorker = pdfjsWorker;
|
|
2948
|
+
|
|
2949
|
+
// src/pdf/parser.ts
|
|
2950
|
+
import { getDocument, GlobalWorkerOptions } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
2951
|
+
GlobalWorkerOptions.workerSrc = "";
|
|
2952
|
+
var MAX_PAGES = 5e3;
|
|
2953
|
+
var MAX_TOTAL_TEXT = 100 * 1024 * 1024;
|
|
2954
|
+
var PDF_LOAD_TIMEOUT_MS = 3e4;
|
|
2955
|
+
async function loadPdfWithTimeout(buffer) {
|
|
2956
|
+
const loadingTask = getDocument({
|
|
2957
|
+
data: new Uint8Array(buffer),
|
|
2958
|
+
useSystemFonts: true,
|
|
2959
|
+
disableFontFace: true,
|
|
2960
|
+
isEvalSupported: false
|
|
2961
|
+
});
|
|
2962
|
+
let timer;
|
|
2963
|
+
try {
|
|
2964
|
+
return await Promise.race([
|
|
2965
|
+
loadingTask.promise,
|
|
2966
|
+
new Promise((_, reject) => {
|
|
2967
|
+
timer = setTimeout(() => {
|
|
2968
|
+
loadingTask.destroy();
|
|
2969
|
+
reject(new KordocError("PDF \uB85C\uB529 \uD0C0\uC784\uC544\uC6C3 (30\uCD08 \uCD08\uACFC)"));
|
|
2970
|
+
}, PDF_LOAD_TIMEOUT_MS);
|
|
2971
|
+
})
|
|
2972
|
+
]);
|
|
2973
|
+
} finally {
|
|
2974
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
2975
|
+
}
|
|
2976
|
+
}
|
|
2977
|
+
async function parsePdfDocument(buffer, options) {
|
|
2978
|
+
const formulaBuffer = options?.formulaOcr ? buffer.slice(0) : null;
|
|
2979
|
+
const doc = await loadPdfWithTimeout(buffer);
|
|
2980
|
+
try {
|
|
2981
|
+
const pageCount = doc.numPages;
|
|
2982
|
+
if (pageCount === 0) throw new KordocError("PDF\uC5D0 \uD398\uC774\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
|
|
2983
|
+
const metadata = { pageCount };
|
|
2984
|
+
await extractPdfMetadata(doc, metadata);
|
|
2985
|
+
const blocks = [];
|
|
2986
|
+
const warnings = [];
|
|
2987
|
+
const pageQuality = [];
|
|
2988
|
+
let totalChars = 0;
|
|
2989
|
+
let totalTextBytes = 0;
|
|
2990
|
+
const effectivePageCount = Math.min(pageCount, MAX_PAGES);
|
|
2991
|
+
const pageFilter = options?.pages ? parsePageRange(options.pages, effectivePageCount) : null;
|
|
2992
|
+
const totalTarget = pageFilter ? pageFilter.size : effectivePageCount;
|
|
2993
|
+
const fontSizeFreq = /* @__PURE__ */ new Map();
|
|
2994
|
+
const pageHeights = /* @__PURE__ */ new Map();
|
|
2995
|
+
const pagesWithLargeImage = /* @__PURE__ */ new Set();
|
|
2996
|
+
const skippedImagePages = /* @__PURE__ */ new Map();
|
|
2997
|
+
let parsedPages = 0;
|
|
2998
|
+
for (let i = 1; i <= effectivePageCount; i++) {
|
|
2999
|
+
if (pageFilter && !pageFilter.has(i)) continue;
|
|
3000
|
+
try {
|
|
3001
|
+
const page = await doc.getPage(i);
|
|
3002
|
+
const tc = await page.getTextContent();
|
|
3003
|
+
const viewport = page.getViewport({ scale: 1 });
|
|
3004
|
+
pageHeights.set(i, viewport.height);
|
|
3005
|
+
const rawItems = tc.items;
|
|
3006
|
+
const items = normalizeItems(rawItems);
|
|
3007
|
+
const { visible, hiddenCount } = filterHiddenText(items, viewport.width, viewport.height);
|
|
3008
|
+
if (hiddenCount > 0) {
|
|
3009
|
+
warnings.push({ page: i, message: `${hiddenCount}\uAC1C \uC228\uACA8\uC9C4 \uD14D\uC2A4\uD2B8 \uC694\uC18C \uD544\uD130\uB9C1\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
3010
|
+
}
|
|
3011
|
+
for (const item of visible) {
|
|
3012
|
+
if (item.fontSize > 0) fontSizeFreq.set(item.fontSize, (fontSizeFreq.get(item.fontSize) || 0) + 1);
|
|
3013
|
+
}
|
|
3014
|
+
const opList = await page.getOperatorList();
|
|
3015
|
+
const pageArea = viewport.width * viewport.height;
|
|
3016
|
+
if (pageArea > 0) {
|
|
3017
|
+
const imageRegions = extractImageRegions(opList.fnArray, opList.argsArray);
|
|
3018
|
+
let uncovered = 0;
|
|
3019
|
+
for (const r of imageRegions) {
|
|
3020
|
+
const area = (r.x2 - r.x1) * (r.y2 - r.y1);
|
|
3021
|
+
if (area < pageArea * 0.05) continue;
|
|
3022
|
+
pagesWithLargeImage.add(i);
|
|
3023
|
+
const hasText = visible.some((it) => {
|
|
3024
|
+
const cx = it.x + it.w / 2;
|
|
3025
|
+
const cy = it.y + (it.h || it.fontSize) / 2;
|
|
3026
|
+
return cx >= r.x1 && cx <= r.x2 && cy >= r.y1 && cy <= r.y2;
|
|
3027
|
+
});
|
|
3028
|
+
if (!hasText) uncovered++;
|
|
3029
|
+
}
|
|
3030
|
+
if (uncovered > 0) skippedImagePages.set(i, uncovered);
|
|
3031
|
+
}
|
|
3032
|
+
const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
|
|
3033
|
+
for (const b of pageBlocks) blocks.push(b);
|
|
3034
|
+
let pageText = "";
|
|
3035
|
+
for (const b of pageBlocks) {
|
|
3036
|
+
const t = b.text || "";
|
|
3037
|
+
totalChars += t.replace(/\s/g, "").length;
|
|
3038
|
+
totalTextBytes += t.length * 2;
|
|
3039
|
+
pageText += pageText ? "\n" + t : t;
|
|
3040
|
+
}
|
|
3041
|
+
pageQuality.push(computePageQuality(i, pageText));
|
|
3042
|
+
if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
|
|
3043
|
+
parsedPages++;
|
|
3044
|
+
options?.onProgress?.(parsedPages, totalTarget);
|
|
3045
|
+
} catch (pageErr) {
|
|
3046
|
+
if (pageErr instanceof KordocError) throw pageErr;
|
|
3047
|
+
warnings.push({ page: i, message: `\uD398\uC774\uC9C0 ${i} \uD30C\uC2F1 \uC2E4\uD328: ${pageErr instanceof Error ? pageErr.message : "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958"}`, code: "PARTIAL_PARSE" });
|
|
3048
|
+
}
|
|
3049
|
+
}
|
|
3050
|
+
const parsedPageCount = parsedPages || (pageFilter ? pageFilter.size : effectivePageCount);
|
|
3051
|
+
let isImageBased = false;
|
|
3052
|
+
if (totalChars / Math.max(parsedPageCount, 1) < 10) {
|
|
3053
|
+
if (options?.ocr) {
|
|
3054
|
+
try {
|
|
3055
|
+
const { ocrPages } = await import("./provider-4ZJKV3DC.js");
|
|
3056
|
+
const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
|
|
3057
|
+
if (ocrBlocks.length > 0) {
|
|
3058
|
+
const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
|
|
3059
|
+
return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
|
|
3060
|
+
}
|
|
3061
|
+
} catch {
|
|
3062
|
+
}
|
|
3063
|
+
}
|
|
3064
|
+
isImageBased = true;
|
|
3065
|
+
warnings.push({
|
|
3066
|
+
message: `\uC774\uBBF8\uC9C0 \uAE30\uBC18 PDF (${pageCount}\uD398\uC774\uC9C0, \uD14D\uC2A4\uD2B8 ${totalChars}\uC790) \u2014 \uD14D\uC2A4\uD2B8 \uB808\uC774\uC5B4\uAC00 \uC5C6\uC5B4 OCR\uC774 \uD544\uC694\uD569\uB2C8\uB2E4`,
|
|
3067
|
+
code: "NEEDS_OCR"
|
|
3068
|
+
});
|
|
3069
|
+
}
|
|
3070
|
+
if (!isImageBased) {
|
|
3071
|
+
const OCR_REASON_MESSAGES = {
|
|
3072
|
+
low_text: "\uD14D\uC2A4\uD2B8\uAC00 \uAC70\uC758 \uC5C6\uB294 \uD398\uC774\uC9C0 (\uC2A4\uCE94/\uC774\uBBF8\uC9C0 \uCD94\uC815)",
|
|
3073
|
+
high_pua: "\uAE00\uAF34 \uB9E4\uD551 \uC2E4\uD328 (PUA \uBE44\uC728 \uB192\uC74C) \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
3074
|
+
high_control: "\uC81C\uC5B4\uBB38\uC790 \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
3075
|
+
high_replacement: "\uB300\uCCB4\uBB38\uC790(U+FFFD) \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00"
|
|
3076
|
+
};
|
|
3077
|
+
for (const pq of pageQuality) {
|
|
3078
|
+
if (!pq.needsOcr || !pq.ocrReason) continue;
|
|
3079
|
+
if (pq.ocrReason === "low_text" && !pagesWithLargeImage.has(pq.page)) continue;
|
|
3080
|
+
warnings.push({ page: pq.page, message: `${OCR_REASON_MESSAGES[pq.ocrReason]} \u2014 OCR \uAC80\uD1A0 \uD544\uC694`, code: "NEEDS_OCR" });
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
if (!isImageBased) {
|
|
3084
|
+
for (const [page, count] of [...skippedImagePages.entries()].sort((a, b) => a[0] - b[0])) {
|
|
3085
|
+
warnings.push({ page, message: `${count}\uAC1C \uC774\uBBF8\uC9C0 \uC601\uC5ED\uC5D0 \uCD94\uCD9C \uAC00\uB2A5\uD55C \uD14D\uC2A4\uD2B8 \uC5C6\uC74C (\uADF8\uB9BC/\uCC28\uD2B8/\uB3C4\uC7A5 \uB0B4\uC6A9 \uB204\uB77D \uAC00\uB2A5)`, code: "SKIPPED_IMAGE" });
|
|
3086
|
+
}
|
|
3087
|
+
}
|
|
3088
|
+
if (options?.removeHeaderFooter !== false && parsedPageCount >= 3) {
|
|
3089
|
+
const removed = removeHeaderFooterBlocks(blocks, pageHeights, warnings);
|
|
3090
|
+
for (let ri = removed.length - 1; ri >= 0; ri--) {
|
|
3091
|
+
blocks.splice(removed[ri], 1);
|
|
3092
|
+
}
|
|
3093
|
+
}
|
|
3094
|
+
mergeCrossPageTables(blocks);
|
|
3095
|
+
if (options?.formulaOcr && formulaBuffer) {
|
|
3096
|
+
try {
|
|
3097
|
+
await applyFormulaOcr(formulaBuffer, blocks, pageFilter, effectivePageCount, warnings, options.onProgress);
|
|
3098
|
+
} catch (e) {
|
|
3099
|
+
warnings.push({
|
|
3100
|
+
message: `\uC218\uC2DD OCR \uC2E4\uD328: ${e instanceof Error ? e.message : String(e)}`,
|
|
3101
|
+
code: "PARTIAL_PARSE"
|
|
3102
|
+
});
|
|
3103
|
+
}
|
|
3104
|
+
}
|
|
3105
|
+
const medianFontSize = computeMedianFontSizeFromFreq(fontSizeFreq);
|
|
3106
|
+
if (medianFontSize > 0) {
|
|
3107
|
+
detectHeadings(blocks, medianFontSize);
|
|
3108
|
+
}
|
|
3109
|
+
detectMarkerHeadings(blocks);
|
|
3110
|
+
detectTableCaptions(blocks);
|
|
3111
|
+
detectKoreanListBlocks(blocks);
|
|
3112
|
+
const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
|
|
3113
|
+
sanitizeBlockControlChars(blocks);
|
|
3114
|
+
let markdown = cleanPdfText(blocksToMarkdown(blocks));
|
|
3115
|
+
return {
|
|
3116
|
+
markdown,
|
|
3117
|
+
blocks,
|
|
3118
|
+
metadata,
|
|
3119
|
+
outline: outline.length > 0 ? outline : void 0,
|
|
3120
|
+
warnings: warnings.length > 0 ? warnings : void 0,
|
|
3121
|
+
isImageBased: isImageBased || void 0,
|
|
3122
|
+
pageQuality,
|
|
3123
|
+
qualitySummary: summarizeDocumentQuality(pageQuality)
|
|
3124
|
+
};
|
|
3125
|
+
} finally {
|
|
3126
|
+
await doc.destroy().catch(() => {
|
|
3127
|
+
});
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
async function extractPdfMetadata(doc, metadata) {
|
|
3131
|
+
try {
|
|
3132
|
+
const result = await doc.getMetadata();
|
|
3133
|
+
if (!result?.info) return;
|
|
3134
|
+
const info = result.info;
|
|
3135
|
+
if (typeof info.Title === "string" && info.Title.trim()) metadata.title = info.Title.trim();
|
|
3136
|
+
if (typeof info.Author === "string" && info.Author.trim()) metadata.author = info.Author.trim();
|
|
3137
|
+
if (typeof info.Creator === "string" && info.Creator.trim()) metadata.creator = info.Creator.trim();
|
|
3138
|
+
if (typeof info.Subject === "string" && info.Subject.trim()) metadata.description = info.Subject.trim();
|
|
3139
|
+
if (typeof info.Keywords === "string" && info.Keywords.trim()) {
|
|
3140
|
+
metadata.keywords = info.Keywords.split(/[,;]/).map((k) => k.trim()).filter(Boolean);
|
|
3141
|
+
}
|
|
3142
|
+
if (typeof info.CreationDate === "string") metadata.createdAt = parsePdfDate(info.CreationDate);
|
|
3143
|
+
if (typeof info.ModDate === "string") metadata.modifiedAt = parsePdfDate(info.ModDate);
|
|
3144
|
+
} catch {
|
|
3145
|
+
}
|
|
3146
|
+
}
|
|
3147
|
+
function parsePdfDate(dateStr) {
|
|
3148
|
+
const m = dateStr.match(/D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/);
|
|
3149
|
+
if (!m) return void 0;
|
|
3150
|
+
const [, year, month = "01", day = "01", hour = "00", min = "00", sec = "00"] = m;
|
|
3151
|
+
return `${year}-${month}-${day}T${hour}:${min}:${sec}`;
|
|
3152
|
+
}
|
|
3153
|
+
async function extractPdfMetadataOnly(buffer) {
|
|
3154
|
+
const doc = await loadPdfWithTimeout(buffer);
|
|
3155
|
+
try {
|
|
3156
|
+
const metadata = { pageCount: doc.numPages };
|
|
3157
|
+
await extractPdfMetadata(doc, metadata);
|
|
3158
|
+
return metadata;
|
|
3159
|
+
} finally {
|
|
3160
|
+
await doc.destroy().catch(() => {
|
|
3161
|
+
});
|
|
3162
|
+
}
|
|
3163
|
+
}
|
|
3137
3164
|
export {
|
|
3138
3165
|
cleanPdfText,
|
|
3139
3166
|
detectKoreanListBlocks,
|
|
@@ -3143,4 +3170,4 @@ export {
|
|
|
3143
3170
|
parsePdfDocument,
|
|
3144
3171
|
removeHeaderFooterBlocks
|
|
3145
3172
|
};
|
|
3146
|
-
//# sourceMappingURL=parser-
|
|
3173
|
+
//# sourceMappingURL=parser-XEDROIM7.js.map
|