kordoc 3.7.0 → 3.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/{-7UC4ZWBS.js → -LD4BZDDJ.js} +3 -3
- package/dist/{chunk-IJJMVTU5.js → chunk-IFYJFWD2.js} +2 -2
- package/dist/{chunk-NXRABCWW.js → chunk-KT2BCHXI.js} +803 -703
- package/dist/chunk-KT2BCHXI.js.map +1 -0
- package/dist/{chunk-QZCP3UWU.cjs → chunk-LFCS3UVG.cjs} +2 -2
- package/dist/{chunk-QZCP3UWU.cjs.map → chunk-LFCS3UVG.cjs.map} +1 -1
- package/dist/{chunk-MEVKYW55.js → chunk-PELBIL4K.js} +2 -2
- package/dist/cli.js +4 -4
- package/dist/index.cjs +968 -868
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +802 -702
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +3 -3
- package/dist/{parser-DR5CTZ74.js → parser-FFEBMLSH.js} +1830 -1803
- package/dist/parser-FFEBMLSH.js.map +1 -0
- package/dist/{parser-RFLPUZ7P.cjs → parser-IXK5V7YG.cjs} +1830 -1803
- package/dist/parser-IXK5V7YG.cjs.map +1 -0
- package/dist/{parser-ZHJFQR44.js → parser-XEDROIM7.js} +1830 -1803
- package/dist/parser-XEDROIM7.js.map +1 -0
- package/dist/{watch-UIX447QV.js → watch-MAWCDNFI.js} +3 -3
- package/package.json +2 -2
- package/dist/chunk-NXRABCWW.js.map +0 -1
- package/dist/parser-DR5CTZ74.js.map +0 -1
- package/dist/parser-RFLPUZ7P.cjs.map +0 -1
- package/dist/parser-ZHJFQR44.js.map +0 -1
- /package/dist/{-7UC4ZWBS.js.map → -LD4BZDDJ.js.map} +0 -0
- /package/dist/{chunk-IJJMVTU5.js.map → chunk-IFYJFWD2.js.map} +0 -0
- /package/dist/{chunk-MEVKYW55.js.map → chunk-PELBIL4K.js.map} +0 -0
- /package/dist/{watch-UIX447QV.js.map → watch-MAWCDNFI.js.map} +0 -0
|
@@ -7,22 +7,16 @@ import {
|
|
|
7
7
|
blocksToMarkdown,
|
|
8
8
|
safeMax,
|
|
9
9
|
safeMin
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-IFYJFWD2.js";
|
|
11
11
|
import {
|
|
12
12
|
parsePageRange
|
|
13
13
|
} from "./chunk-MOL7MDBG.js";
|
|
14
14
|
|
|
15
|
-
// src/pdf/line-
|
|
15
|
+
// src/pdf/line-extract.ts
|
|
16
16
|
import { OPS } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
17
17
|
var ORIENTATION_TOL = 2;
|
|
18
18
|
var MIN_LINE_LENGTH = 15;
|
|
19
19
|
var MAX_LINE_WIDTH = 5;
|
|
20
|
-
var CONNECT_TOL = 5;
|
|
21
|
-
var CELL_PADDING = 2;
|
|
22
|
-
var MIN_COL_WIDTH = 15;
|
|
23
|
-
var MIN_ROW_HEIGHT = 6;
|
|
24
|
-
var VERTEX_MERGE_FACTOR = 4;
|
|
25
|
-
var MIN_COORD_MERGE_TOL = 8;
|
|
26
20
|
function extractLines(fnArray, argsArray) {
|
|
27
21
|
const horizontals = [];
|
|
28
22
|
const verticals = [];
|
|
@@ -157,55 +151,6 @@ function extractLines(fnArray, argsArray) {
|
|
|
157
151
|
}
|
|
158
152
|
return { horizontals, verticals };
|
|
159
153
|
}
|
|
160
|
-
function multiplyTransform(m, t) {
|
|
161
|
-
return [
|
|
162
|
-
m[0] * t[0] + m[2] * t[1],
|
|
163
|
-
m[1] * t[0] + m[3] * t[1],
|
|
164
|
-
m[0] * t[2] + m[2] * t[3],
|
|
165
|
-
m[1] * t[2] + m[3] * t[3],
|
|
166
|
-
m[0] * t[4] + m[2] * t[5] + m[4],
|
|
167
|
-
m[1] * t[4] + m[3] * t[5] + m[5]
|
|
168
|
-
];
|
|
169
|
-
}
|
|
170
|
-
function extractImageRegions(fnArray, argsArray) {
|
|
171
|
-
const regions = [];
|
|
172
|
-
let ctm = [1, 0, 0, 1, 0, 0];
|
|
173
|
-
const stack = [];
|
|
174
|
-
for (let i = 0; i < fnArray.length; i++) {
|
|
175
|
-
const op = fnArray[i];
|
|
176
|
-
switch (op) {
|
|
177
|
-
case OPS.save:
|
|
178
|
-
stack.push(ctm);
|
|
179
|
-
break;
|
|
180
|
-
case OPS.restore:
|
|
181
|
-
ctm = stack.pop() || [1, 0, 0, 1, 0, 0];
|
|
182
|
-
break;
|
|
183
|
-
case OPS.transform: {
|
|
184
|
-
const t = argsArray[i];
|
|
185
|
-
if (Array.isArray(t) && t.length >= 6) ctm = multiplyTransform(ctm, t);
|
|
186
|
-
break;
|
|
187
|
-
}
|
|
188
|
-
case OPS.paintImageXObject:
|
|
189
|
-
case OPS.paintInlineImageXObject:
|
|
190
|
-
case OPS.paintImageMaskXObject:
|
|
191
|
-
case OPS.paintImageXObjectRepeat: {
|
|
192
|
-
const corners = [[0, 0], [1, 0], [0, 1], [1, 1]];
|
|
193
|
-
let x1 = Infinity, y1 = Infinity, x2 = -Infinity, y2 = -Infinity;
|
|
194
|
-
for (const [u, v] of corners) {
|
|
195
|
-
const x = ctm[0] * u + ctm[2] * v + ctm[4];
|
|
196
|
-
const y = ctm[1] * u + ctm[3] * v + ctm[5];
|
|
197
|
-
if (x < x1) x1 = x;
|
|
198
|
-
if (x > x2) x2 = x;
|
|
199
|
-
if (y < y1) y1 = y;
|
|
200
|
-
if (y > y2) y2 = y;
|
|
201
|
-
}
|
|
202
|
-
if (x2 - x1 > 0 && y2 - y1 > 0) regions.push({ x1, y1, x2, y2 });
|
|
203
|
-
break;
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
return regions;
|
|
208
|
-
}
|
|
209
154
|
function classifyAndAdd(seg, lineWidth, horizontals, verticals) {
|
|
210
155
|
const dx = Math.abs(seg.x2 - seg.x1);
|
|
211
156
|
const dy = Math.abs(seg.y2 - seg.y1);
|
|
@@ -283,6 +228,67 @@ function filterPageBorderLines(horizontals, verticals, pageWidth, pageHeight) {
|
|
|
283
228
|
)
|
|
284
229
|
};
|
|
285
230
|
}
|
|
231
|
+
|
|
232
|
+
// src/pdf/image-regions.ts
|
|
233
|
+
import { OPS as OPS2 } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
234
|
+
function multiplyTransform(m, t) {
|
|
235
|
+
return [
|
|
236
|
+
m[0] * t[0] + m[2] * t[1],
|
|
237
|
+
m[1] * t[0] + m[3] * t[1],
|
|
238
|
+
m[0] * t[2] + m[2] * t[3],
|
|
239
|
+
m[1] * t[2] + m[3] * t[3],
|
|
240
|
+
m[0] * t[4] + m[2] * t[5] + m[4],
|
|
241
|
+
m[1] * t[4] + m[3] * t[5] + m[5]
|
|
242
|
+
];
|
|
243
|
+
}
|
|
244
|
+
function extractImageRegions(fnArray, argsArray) {
|
|
245
|
+
const regions = [];
|
|
246
|
+
let ctm = [1, 0, 0, 1, 0, 0];
|
|
247
|
+
const stack = [];
|
|
248
|
+
for (let i = 0; i < fnArray.length; i++) {
|
|
249
|
+
const op = fnArray[i];
|
|
250
|
+
switch (op) {
|
|
251
|
+
case OPS2.save:
|
|
252
|
+
stack.push(ctm);
|
|
253
|
+
break;
|
|
254
|
+
case OPS2.restore:
|
|
255
|
+
ctm = stack.pop() || [1, 0, 0, 1, 0, 0];
|
|
256
|
+
break;
|
|
257
|
+
case OPS2.transform: {
|
|
258
|
+
const t = argsArray[i];
|
|
259
|
+
if (Array.isArray(t) && t.length >= 6) ctm = multiplyTransform(ctm, t);
|
|
260
|
+
break;
|
|
261
|
+
}
|
|
262
|
+
case OPS2.paintImageXObject:
|
|
263
|
+
case OPS2.paintInlineImageXObject:
|
|
264
|
+
case OPS2.paintImageMaskXObject:
|
|
265
|
+
case OPS2.paintImageXObjectRepeat: {
|
|
266
|
+
const corners = [[0, 0], [1, 0], [0, 1], [1, 1]];
|
|
267
|
+
let x1 = Infinity, y1 = Infinity, x2 = -Infinity, y2 = -Infinity;
|
|
268
|
+
for (const [u, v] of corners) {
|
|
269
|
+
const x = ctm[0] * u + ctm[2] * v + ctm[4];
|
|
270
|
+
const y = ctm[1] * u + ctm[3] * v + ctm[5];
|
|
271
|
+
if (x < x1) x1 = x;
|
|
272
|
+
if (x > x2) x2 = x;
|
|
273
|
+
if (y < y1) y1 = y;
|
|
274
|
+
if (y > y2) y2 = y;
|
|
275
|
+
}
|
|
276
|
+
if (x2 - x1 > 0 && y2 - y1 > 0) regions.push({ x1, y1, x2, y2 });
|
|
277
|
+
break;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
return regions;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// src/pdf/line-types.ts
|
|
285
|
+
var VERTEX_MERGE_FACTOR = 4;
|
|
286
|
+
|
|
287
|
+
// src/pdf/table-grid.ts
|
|
288
|
+
var CONNECT_TOL = 5;
|
|
289
|
+
var MIN_COL_WIDTH = 15;
|
|
290
|
+
var MIN_ROW_HEIGHT = 6;
|
|
291
|
+
var MIN_COORD_MERGE_TOL = 8;
|
|
286
292
|
function buildVertices(horizontals, verticals) {
|
|
287
293
|
const vertices = [];
|
|
288
294
|
const tol = CONNECT_TOL;
|
|
@@ -494,6 +500,8 @@ function linesIntersect(a, b) {
|
|
|
494
500
|
const tol = CONNECT_TOL;
|
|
495
501
|
return v.x1 >= h.x1 - tol && v.x1 <= h.x2 + tol && h.y1 >= v.y1 - tol && h.y1 <= v.y2 + tol;
|
|
496
502
|
}
|
|
503
|
+
|
|
504
|
+
// src/pdf/cell-extract.ts
|
|
497
505
|
function extractCells(grid, horizontals, verticals) {
|
|
498
506
|
const { rowYs, colXs } = grid;
|
|
499
507
|
const numRows = rowYs.length - 1;
|
|
@@ -591,6 +599,9 @@ function hasHorizontalLine(horizontals, y, leftX, rightX, vertexRadius) {
|
|
|
591
599
|
}
|
|
592
600
|
return false;
|
|
593
601
|
}
|
|
602
|
+
|
|
603
|
+
// src/pdf/cell-text.ts
|
|
604
|
+
var CELL_PADDING = 2;
|
|
594
605
|
var SPACE_GAP_RATIO = 0.17;
|
|
595
606
|
function spaceGapThreshold(fontSize) {
|
|
596
607
|
return Math.max(fontSize * SPACE_GAP_RATIO, 1);
|
|
@@ -719,6 +730,28 @@ function markEvenRun(items, result, start, end) {
|
|
|
719
730
|
}
|
|
720
731
|
}
|
|
721
732
|
}
|
|
733
|
+
function mergeCellTextLines(textLines) {
|
|
734
|
+
if (textLines.length <= 1) return textLines[0] || "";
|
|
735
|
+
const merged = [textLines[0]];
|
|
736
|
+
for (let i = 1; i < textLines.length; i++) {
|
|
737
|
+
const prev = merged[merged.length - 1];
|
|
738
|
+
const curr = textLines[i];
|
|
739
|
+
if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
|
|
740
|
+
merged[merged.length - 1] = prev + curr;
|
|
741
|
+
} else if (curr.trim().length <= 3 && /^[)\]%}]/.test(curr.trim())) {
|
|
742
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
743
|
+
} else if (/[,(]$/.test(prev.trim()) && curr.trim().length <= 15) {
|
|
744
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
745
|
+
} else if (/[\d,]$/.test(prev) && /^[\d,]+[)\]]?$/.test(curr.trim()) && curr.trim().length <= 10) {
|
|
746
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
747
|
+
} else {
|
|
748
|
+
merged.push(curr);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
return merged.join("\n");
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// src/pdf/undersegmented.ts
|
|
722
755
|
var MAX_UNDERSEGMENTED_ROWS = 2;
|
|
723
756
|
var MIN_UNDERSEGMENTED_COLUMNS = 3;
|
|
724
757
|
var MIN_UNDERSEGMENTED_TEXT_LINES = 8;
|
|
@@ -832,1608 +865,981 @@ function normalizeUndersegmentedTable(originalCells, colXs, items) {
|
|
|
832
865
|
if (countNonEmptyCols(rebuilt, numCols) < countNonEmptyCols(originalCells, numCols)) return null;
|
|
833
866
|
return rebuilt;
|
|
834
867
|
}
|
|
835
|
-
function mergeCellTextLines(textLines) {
|
|
836
|
-
if (textLines.length <= 1) return textLines[0] || "";
|
|
837
|
-
const merged = [textLines[0]];
|
|
838
|
-
for (let i = 1; i < textLines.length; i++) {
|
|
839
|
-
const prev = merged[merged.length - 1];
|
|
840
|
-
const curr = textLines[i];
|
|
841
|
-
if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
|
|
842
|
-
merged[merged.length - 1] = prev + curr;
|
|
843
|
-
} else if (curr.trim().length <= 3 && /^[)\]%}]/.test(curr.trim())) {
|
|
844
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
845
|
-
} else if (/[,(]$/.test(prev.trim()) && curr.trim().length <= 15) {
|
|
846
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
847
|
-
} else if (/[\d,]$/.test(prev) && /^[\d,]+[)\]]?$/.test(curr.trim()) && curr.trim().length <= 10) {
|
|
848
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
849
|
-
} else {
|
|
850
|
-
merged.push(curr);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
return merged.join("\n");
|
|
854
|
-
}
|
|
855
868
|
|
|
856
|
-
// src/pdf/
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
const headerResult = detectHeaderRow(rows);
|
|
871
|
-
if (headerResult) {
|
|
872
|
-
const { columns, headerIdx } = headerResult;
|
|
873
|
-
const headerRow = rows[headerIdx];
|
|
874
|
-
const headerItems = [...headerRow.items].sort((a, b) => a.x - b.x);
|
|
875
|
-
const headerAndBelow = rows.slice(headerIdx);
|
|
876
|
-
const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
|
|
877
|
-
const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
|
|
878
|
-
for (const region of tableRegions) {
|
|
879
|
-
const table = buildClusterTable(region.rows, columns, pageNum);
|
|
880
|
-
if (table) {
|
|
881
|
-
expandUsedItems(table.usedItems, originMap);
|
|
882
|
-
results.push(table);
|
|
883
|
-
}
|
|
869
|
+
// src/pdf/quality.ts
|
|
870
|
+
function computePageQuality(page, text) {
|
|
871
|
+
let total = 0;
|
|
872
|
+
let hangul = 0;
|
|
873
|
+
let control = 0;
|
|
874
|
+
let replacement = 0;
|
|
875
|
+
let pua = 0;
|
|
876
|
+
for (let i = 0; i < text.length; i++) {
|
|
877
|
+
const code = text.charCodeAt(i);
|
|
878
|
+
if (code === 32 || code === 9 || code === 10 || code === 13) continue;
|
|
879
|
+
total++;
|
|
880
|
+
if (code < 32 || code === 127 || code >= 128 && code <= 159) {
|
|
881
|
+
control++;
|
|
882
|
+
continue;
|
|
884
883
|
}
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
expandUsedItems(table.usedItems, originMap);
|
|
897
|
-
results.push(table);
|
|
898
|
-
}
|
|
899
|
-
}
|
|
900
|
-
}
|
|
884
|
+
if (code === 65533) {
|
|
885
|
+
replacement++;
|
|
886
|
+
continue;
|
|
887
|
+
}
|
|
888
|
+
if (code >= 44032 && code <= 55203) {
|
|
889
|
+
hangul++;
|
|
890
|
+
continue;
|
|
891
|
+
}
|
|
892
|
+
if (code >= 57344 && code <= 63743 || code >= 56192 && code <= 56319) {
|
|
893
|
+
pua++;
|
|
894
|
+
continue;
|
|
901
895
|
}
|
|
902
896
|
}
|
|
903
|
-
|
|
897
|
+
const denom = total || 1;
|
|
898
|
+
const puaRatio = pua / denom;
|
|
899
|
+
const controlCharRatio = control / denom;
|
|
900
|
+
const replacementCharRatio = replacement / denom;
|
|
901
|
+
let needsOcr = false;
|
|
902
|
+
let ocrReason;
|
|
903
|
+
if (total < LOW_TEXT_THRESHOLD) {
|
|
904
|
+
needsOcr = true;
|
|
905
|
+
ocrReason = "low_text";
|
|
906
|
+
} else if (puaRatio >= HIGH_PUA_THRESHOLD) {
|
|
907
|
+
needsOcr = true;
|
|
908
|
+
ocrReason = "high_pua";
|
|
909
|
+
} else if (controlCharRatio >= HIGH_CONTROL_THRESHOLD) {
|
|
910
|
+
needsOcr = true;
|
|
911
|
+
ocrReason = "high_control";
|
|
912
|
+
} else if (replacementCharRatio >= HIGH_REPLACEMENT_THRESHOLD) {
|
|
913
|
+
needsOcr = true;
|
|
914
|
+
ocrReason = "high_replacement";
|
|
915
|
+
}
|
|
916
|
+
return {
|
|
917
|
+
page,
|
|
918
|
+
textChars: total,
|
|
919
|
+
hangulRatio: hangul / denom,
|
|
920
|
+
controlCharRatio,
|
|
921
|
+
replacementCharRatio,
|
|
922
|
+
puaRatio,
|
|
923
|
+
needsOcr,
|
|
924
|
+
ocrReason
|
|
925
|
+
};
|
|
904
926
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
while (i < sorted.length) {
|
|
913
|
-
if (/^[가-힣\d]$/.test(sorted[i].text)) {
|
|
914
|
-
let runEnd = i + 1;
|
|
915
|
-
while (runEnd < sorted.length && /^[가-힣\d]$/.test(sorted[runEnd].text)) {
|
|
916
|
-
if (sorted[runEnd].hasSpaceBefore) break;
|
|
917
|
-
const gap = sorted[runEnd].x - (sorted[runEnd - 1].x + sorted[runEnd - 1].w);
|
|
918
|
-
const fs = sorted[runEnd].fontSize;
|
|
919
|
-
if (gap < fs * 0.1 || gap > fs * 3) break;
|
|
920
|
-
runEnd++;
|
|
921
|
-
}
|
|
922
|
-
if (runEnd - i >= 3) {
|
|
923
|
-
const gaps = [];
|
|
924
|
-
for (let g2 = i + 1; g2 < runEnd; g2++) {
|
|
925
|
-
gaps.push(sorted[g2].x - (sorted[g2 - 1].x + sorted[g2 - 1].w));
|
|
926
|
-
}
|
|
927
|
-
let minG = Infinity, maxG = -Infinity;
|
|
928
|
-
for (const g2 of gaps) {
|
|
929
|
-
if (g2 < minG) minG = g2;
|
|
930
|
-
if (g2 > maxG) maxG = g2;
|
|
931
|
-
}
|
|
932
|
-
if (minG > 0 && maxG / minG <= 3) {
|
|
933
|
-
const run = sorted.slice(i, runEnd);
|
|
934
|
-
const text = run.map((r) => r.text).join("");
|
|
935
|
-
const first = run[0], last = run[runEnd - i - 1];
|
|
936
|
-
const item = {
|
|
937
|
-
text,
|
|
938
|
-
x: first.x,
|
|
939
|
-
y: first.y,
|
|
940
|
-
w: last.x + last.w - first.x,
|
|
941
|
-
h: first.h,
|
|
942
|
-
fontSize: first.fontSize,
|
|
943
|
-
fontName: first.fontName
|
|
944
|
-
};
|
|
945
|
-
originMap.set(item, run);
|
|
946
|
-
merged.push(item);
|
|
947
|
-
i = runEnd;
|
|
948
|
-
continue;
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
}
|
|
952
|
-
merged.push(sorted[i]);
|
|
953
|
-
i++;
|
|
954
|
-
}
|
|
955
|
-
}
|
|
956
|
-
return { merged, originMap };
|
|
927
|
+
var LOW_TEXT_THRESHOLD = 20;
|
|
928
|
+
var HIGH_PUA_THRESHOLD = 0.2;
|
|
929
|
+
var HIGH_CONTROL_THRESHOLD = 0.05;
|
|
930
|
+
var HIGH_REPLACEMENT_THRESHOLD = 0.05;
|
|
931
|
+
var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
|
|
932
|
+
function stripControlChars(text) {
|
|
933
|
+
return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
|
|
957
934
|
}
|
|
958
|
-
function
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
935
|
+
function summarizeDocumentQuality(pages) {
|
|
936
|
+
if (pages.length === 0) {
|
|
937
|
+
return {
|
|
938
|
+
totalPages: 0,
|
|
939
|
+
totalTextChars: 0,
|
|
940
|
+
avgHangulRatio: 0,
|
|
941
|
+
avgControlCharRatio: 0,
|
|
942
|
+
avgReplacementCharRatio: 0,
|
|
943
|
+
avgPuaRatio: 0,
|
|
944
|
+
lowTextPageCount: 0,
|
|
945
|
+
highPuaPageCount: 0,
|
|
946
|
+
needsOcr: false,
|
|
947
|
+
ocrCandidatePages: []
|
|
948
|
+
};
|
|
963
949
|
}
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
let
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
950
|
+
let textChars = 0;
|
|
951
|
+
let hangul = 0;
|
|
952
|
+
let control = 0;
|
|
953
|
+
let replacement = 0;
|
|
954
|
+
let pua = 0;
|
|
955
|
+
let lowText = 0;
|
|
956
|
+
let highPua = 0;
|
|
957
|
+
const ocrCandidatePages = [];
|
|
958
|
+
for (const p of pages) {
|
|
959
|
+
textChars += p.textChars;
|
|
960
|
+
hangul += p.hangulRatio;
|
|
961
|
+
control += p.controlCharRatio;
|
|
962
|
+
replacement += p.replacementCharRatio;
|
|
963
|
+
pua += p.puaRatio;
|
|
964
|
+
if (p.textChars < LOW_TEXT_THRESHOLD) lowText++;
|
|
965
|
+
if (p.puaRatio >= HIGH_PUA_THRESHOLD) highPua++;
|
|
966
|
+
if (p.needsOcr) ocrCandidatePages.push(p.page);
|
|
974
967
|
}
|
|
975
|
-
const
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
968
|
+
const n = pages.length;
|
|
969
|
+
return {
|
|
970
|
+
totalPages: n,
|
|
971
|
+
totalTextChars: textChars,
|
|
972
|
+
avgHangulRatio: hangul / n,
|
|
973
|
+
avgControlCharRatio: control / n,
|
|
974
|
+
avgReplacementCharRatio: replacement / n,
|
|
975
|
+
avgPuaRatio: pua / n,
|
|
976
|
+
lowTextPageCount: lowText,
|
|
977
|
+
highPuaPageCount: highPua,
|
|
978
|
+
needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
|
|
979
|
+
ocrCandidatePages
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
// src/pdf/text-line.ts
|
|
984
|
+
function filterHiddenText(items, pageWidth, pageHeight) {
|
|
985
|
+
let hiddenCount = 0;
|
|
986
|
+
const visible = [];
|
|
987
|
+
for (const item of items) {
|
|
988
|
+
if (item.isHidden) {
|
|
989
|
+
hiddenCount++;
|
|
990
|
+
continue;
|
|
994
991
|
}
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
const matched = countMatchedColumnsRange(rows[j], columns, sorted);
|
|
1000
|
-
if (matched >= MIN_COLS) matchCount++;
|
|
992
|
+
const margin = Math.max(pageWidth, pageHeight) * 0.1;
|
|
993
|
+
if (item.x < -margin || item.x > pageWidth + margin || item.y < -margin || item.y > pageHeight + margin) {
|
|
994
|
+
hiddenCount++;
|
|
995
|
+
continue;
|
|
1001
996
|
}
|
|
1002
|
-
|
|
1003
|
-
return { columns, headerIdx: ri };
|
|
997
|
+
visible.push(item);
|
|
1004
998
|
}
|
|
1005
|
-
return
|
|
999
|
+
return { visible, hiddenCount };
|
|
1006
1000
|
}
|
|
1007
|
-
function
|
|
1008
|
-
|
|
1009
|
-
const
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
const curr = rows[i];
|
|
1013
|
-
const a = rowBand(prev);
|
|
1014
|
-
const b = rowBand(curr);
|
|
1015
|
-
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1016
|
-
const prevIsFrag = isFragmentRow(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1017
|
-
const currIsFrag = isFragmentRow(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1018
|
-
if (prevIsFrag || currIsFrag) {
|
|
1019
|
-
const baseY = prevIsFrag ? curr.y : prev.y;
|
|
1020
|
-
result[result.length - 1] = { y: baseY, items: [...prev.items, ...curr.items] };
|
|
1021
|
-
} else {
|
|
1022
|
-
result.push(curr);
|
|
1023
|
-
}
|
|
1001
|
+
function collapseEvenSpacing(text) {
|
|
1002
|
+
const tokens = text.split(" ");
|
|
1003
|
+
const singleCharCount = tokens.filter((t) => t.length === 1).length;
|
|
1004
|
+
if (tokens.length >= 3 && singleCharCount / tokens.length >= 0.7) {
|
|
1005
|
+
return tokens.join("");
|
|
1024
1006
|
}
|
|
1025
|
-
return
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1007
|
+
return text.replace(
|
|
1008
|
+
/(?<![가-힣])[가-힣](?: [가-힣\d]){2,}(?![가-힣])/g,
|
|
1009
|
+
(match) => match.replace(/ /g, "")
|
|
1010
|
+
);
|
|
1029
1011
|
}
|
|
1030
|
-
function
|
|
1031
|
-
let
|
|
1032
|
-
for (const i of
|
|
1033
|
-
|
|
1034
|
-
if (i.y <
|
|
1035
|
-
if (i.
|
|
1012
|
+
function computeBBox(items, pageNum) {
|
|
1013
|
+
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
1014
|
+
for (const i of items) {
|
|
1015
|
+
if (i.x < minX) minX = i.x;
|
|
1016
|
+
if (i.y < minY) minY = i.y;
|
|
1017
|
+
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
1018
|
+
const effectiveH = i.h > 0 ? i.h : i.fontSize;
|
|
1019
|
+
if (i.y + effectiveH > maxY) maxY = i.y + effectiveH;
|
|
1036
1020
|
}
|
|
1037
|
-
return {
|
|
1021
|
+
return { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY };
|
|
1038
1022
|
}
|
|
1039
|
-
function
|
|
1040
|
-
if (
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
const
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
result[result.length - 1] = {
|
|
1051
|
-
y: prev.y,
|
|
1052
|
-
items: [...prev.items, ...curr.items]
|
|
1053
|
-
};
|
|
1054
|
-
} else {
|
|
1055
|
-
result.push(curr);
|
|
1023
|
+
function dominantStyle(items) {
|
|
1024
|
+
if (items.length === 0) return void 0;
|
|
1025
|
+
const freq = /* @__PURE__ */ new Map();
|
|
1026
|
+
let maxCount = 0, dominantSize = 0;
|
|
1027
|
+
for (const i of items) {
|
|
1028
|
+
if (i.fontSize <= 0) continue;
|
|
1029
|
+
const count = (freq.get(i.fontSize) || 0) + 1;
|
|
1030
|
+
freq.set(i.fontSize, count);
|
|
1031
|
+
if (count > maxCount) {
|
|
1032
|
+
maxCount = count;
|
|
1033
|
+
dominantSize = i.fontSize;
|
|
1056
1034
|
}
|
|
1057
1035
|
}
|
|
1058
|
-
return
|
|
1036
|
+
if (dominantSize === 0) return void 0;
|
|
1037
|
+
const fontName = items.find((i) => i.fontSize === dominantSize)?.fontName || void 0;
|
|
1038
|
+
return { fontSize: dominantSize, fontName };
|
|
1059
1039
|
}
|
|
1060
|
-
function
|
|
1061
|
-
|
|
1062
|
-
const
|
|
1063
|
-
const
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
if (
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
rows.push({ y: curY, items: curItems });
|
|
1071
|
-
curItems = [sorted[i]];
|
|
1072
|
-
curY = sorted[i].y;
|
|
1040
|
+
function normalizeItems(rawItems) {
|
|
1041
|
+
const items = [];
|
|
1042
|
+
const spacePositions = [];
|
|
1043
|
+
for (const i of rawItems) {
|
|
1044
|
+
if (typeof i.str !== "string") continue;
|
|
1045
|
+
const x = Math.round(i.transform[4]);
|
|
1046
|
+
const y = Math.round(i.transform[5]);
|
|
1047
|
+
if (!i.str.trim()) {
|
|
1048
|
+
spacePositions.push({ x, y });
|
|
1049
|
+
continue;
|
|
1073
1050
|
}
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
const minGap = Math.max(avgFontSize * MIN_GAP_FACTOR, MIN_GAP_ABSOLUTE);
|
|
1084
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
1085
|
-
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1086
|
-
if (gap >= minGap) return true;
|
|
1087
|
-
}
|
|
1088
|
-
return false;
|
|
1089
|
-
}
|
|
1090
|
-
function extractColumnClusters(rows) {
|
|
1091
|
-
const allX = [];
|
|
1092
|
-
for (const row of rows) {
|
|
1093
|
-
for (const item of row.items) allX.push(item.x);
|
|
1094
|
-
}
|
|
1095
|
-
if (allX.length === 0) return [];
|
|
1096
|
-
allX.sort((a, b) => a - b);
|
|
1097
|
-
const clusters = [];
|
|
1098
|
-
let clusterStart = 0;
|
|
1099
|
-
for (let i = 1; i <= allX.length; i++) {
|
|
1100
|
-
if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
|
|
1101
|
-
const slice = allX.slice(clusterStart, i);
|
|
1102
|
-
const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
|
|
1103
|
-
clusters.push({ x: avg, count: slice.length });
|
|
1104
|
-
clusterStart = i;
|
|
1105
|
-
}
|
|
1106
|
-
}
|
|
1107
|
-
const minCount = Math.max(2, Math.floor(rows.length * MIN_COL_FILL_RATIO));
|
|
1108
|
-
return clusters.filter((c) => c.count >= minCount).sort((a, b) => a.x - b.x);
|
|
1109
|
-
}
|
|
1110
|
-
function findTableRegionsByHeader(allRows, columns, headerItems) {
|
|
1111
|
-
const regions = [];
|
|
1112
|
-
let currentRegion = [];
|
|
1113
|
-
let missStreak = 0;
|
|
1114
|
-
for (const row of allRows) {
|
|
1115
|
-
const matchedCols = countMatchedColumnsRange(row, columns, headerItems);
|
|
1116
|
-
if (matchedCols >= MIN_COLS) {
|
|
1117
|
-
currentRegion.push(row);
|
|
1118
|
-
missStreak = 0;
|
|
1119
|
-
} else if (currentRegion.length > 0 && (row.items.length <= 2 || missStreak === 0)) {
|
|
1120
|
-
currentRegion.push(row);
|
|
1121
|
-
missStreak++;
|
|
1122
|
-
} else {
|
|
1123
|
-
while (currentRegion.length > 0) {
|
|
1124
|
-
const last = currentRegion[currentRegion.length - 1];
|
|
1125
|
-
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1126
|
-
currentRegion.pop();
|
|
1127
|
-
}
|
|
1128
|
-
if (currentRegion.length >= MIN_ROWS) {
|
|
1129
|
-
regions.push({ rows: [...currentRegion] });
|
|
1130
|
-
}
|
|
1131
|
-
currentRegion = [];
|
|
1132
|
-
missStreak = 0;
|
|
1051
|
+
const scaleX = Math.hypot(i.transform[0], i.transform[1]);
|
|
1052
|
+
const scaleY = Math.hypot(i.transform[2], i.transform[3]);
|
|
1053
|
+
const fontSize = Math.round(Math.max(scaleY, scaleX));
|
|
1054
|
+
const w = Math.round(i.width);
|
|
1055
|
+
const h = Math.round(i.height);
|
|
1056
|
+
const isHidden = fontSize === 0 || i.width === 0 && i.str.trim().length > 0;
|
|
1057
|
+
let text = i.str.trim();
|
|
1058
|
+
if (/^[\d\s\-().·,☎]+$/.test(text) && /\d/.test(text) && / /.test(text)) {
|
|
1059
|
+
text = text.replace(/ /g, "");
|
|
1133
1060
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
currentRegion.pop();
|
|
1139
|
-
}
|
|
1140
|
-
if (currentRegion.length >= MIN_ROWS) {
|
|
1141
|
-
regions.push({ rows: currentRegion });
|
|
1142
|
-
}
|
|
1143
|
-
return regions;
|
|
1144
|
-
}
|
|
1145
|
-
function findTableRegions(allRows, columns) {
|
|
1146
|
-
const regions = [];
|
|
1147
|
-
let currentRegion = [];
|
|
1148
|
-
for (const row of allRows) {
|
|
1149
|
-
const matchedCols = countMatchedColumns(row, columns);
|
|
1150
|
-
if (matchedCols >= MIN_COLS) {
|
|
1151
|
-
currentRegion.push(row);
|
|
1152
|
-
} else if (row.items.length === 1) {
|
|
1153
|
-
if (currentRegion.length > 0) {
|
|
1154
|
-
currentRegion.push(row);
|
|
1061
|
+
const split = splitEvenSpacedItem(text, x, w, fontSize);
|
|
1062
|
+
if (split) {
|
|
1063
|
+
for (const s of split) {
|
|
1064
|
+
items.push({ text: s.text, x: s.x, y, w: s.w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
1155
1065
|
}
|
|
1156
1066
|
} else {
|
|
1157
|
-
|
|
1158
|
-
regions.push({ rows: [...currentRegion] });
|
|
1159
|
-
}
|
|
1160
|
-
currentRegion = [];
|
|
1067
|
+
items.push({ text, x, y, w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
1161
1068
|
}
|
|
1162
1069
|
}
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
|
|
1173
|
-
matched.add(ci);
|
|
1070
|
+
const sorted = items.sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1071
|
+
const deduped = [];
|
|
1072
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
1073
|
+
let isDup = false;
|
|
1074
|
+
for (let j = deduped.length - 1; j >= 0; j--) {
|
|
1075
|
+
const prev = deduped[j];
|
|
1076
|
+
if (prev.y - sorted[i].y > 3) break;
|
|
1077
|
+
if (Math.abs(prev.y - sorted[i].y) <= 3 && prev.text === sorted[i].text && Math.abs(prev.x - sorted[i].x) <= 3) {
|
|
1078
|
+
isDup = true;
|
|
1174
1079
|
break;
|
|
1175
1080
|
}
|
|
1176
1081
|
}
|
|
1082
|
+
if (!isDup) deduped.push(sorted[i]);
|
|
1177
1083
|
}
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
const matched = /* @__PURE__ */ new Set();
|
|
1188
|
-
for (const item of row.items) {
|
|
1189
|
-
for (let ci = 0; ci < boundaries.length; ci++) {
|
|
1190
|
-
if (item.x >= boundaries[ci].left && item.x < boundaries[ci].right) {
|
|
1191
|
-
matched.add(ci);
|
|
1192
|
-
break;
|
|
1084
|
+
if (spacePositions.length > 0) {
|
|
1085
|
+
for (const sp of spacePositions) {
|
|
1086
|
+
let nearest = null;
|
|
1087
|
+
for (const item of deduped) {
|
|
1088
|
+
if (Math.abs(sp.y - item.y) > 3) continue;
|
|
1089
|
+
const dist = item.x - sp.x;
|
|
1090
|
+
if (dist >= -1 && dist <= 20 && (!nearest || item.x < nearest.x)) {
|
|
1091
|
+
nearest = item;
|
|
1092
|
+
}
|
|
1193
1093
|
}
|
|
1094
|
+
if (nearest) nearest.hasSpaceBefore = true;
|
|
1194
1095
|
}
|
|
1195
1096
|
}
|
|
1196
|
-
return
|
|
1097
|
+
return deduped;
|
|
1197
1098
|
}
|
|
1198
|
-
function
|
|
1099
|
+
function splitEvenSpacedItem(text, itemX, itemW, fontSize) {
|
|
1100
|
+
if (!/^[가-힣\d](?: [가-힣\d]){2,}$/.test(text)) return null;
|
|
1101
|
+
const chars = text.split(" ");
|
|
1102
|
+
if (chars.length < 3) return null;
|
|
1103
|
+
const charW = itemW / chars.length;
|
|
1104
|
+
if (charW > fontSize * 2) return null;
|
|
1105
|
+
return chars.map((ch, idx) => ({
|
|
1106
|
+
text: ch,
|
|
1107
|
+
x: Math.round(itemX + idx * charW),
|
|
1108
|
+
w: Math.round(charW * 0.8)
|
|
1109
|
+
// 실제 글자 폭은 간격보다 좁음
|
|
1110
|
+
}));
|
|
1111
|
+
}
|
|
1112
|
+
function groupByY(items) {
|
|
1199
1113
|
if (items.length === 0) return [];
|
|
1200
|
-
const
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
for (let i = 1; i <
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
const gapThreshold = sorted.length <= numCols + 1 ? 12 : Math.max(medianGap * 2.5, 12);
|
|
1209
|
-
const significantGaps = gaps.filter((g2) => g2.size >= gapThreshold).sort((a, b) => b.size - a.size).slice(0, numCols - 1).sort((a, b) => a.idx - b.idx);
|
|
1210
|
-
const groups = [];
|
|
1211
|
-
let start = 0;
|
|
1212
|
-
for (const gap of significantGaps) {
|
|
1213
|
-
groups.push(sorted.slice(start, gap.idx));
|
|
1214
|
-
start = gap.idx;
|
|
1215
|
-
}
|
|
1216
|
-
groups.push(sorted.slice(start));
|
|
1217
|
-
const result = [];
|
|
1218
|
-
const usedCols = /* @__PURE__ */ new Set();
|
|
1219
|
-
const groupCenters = groups.map((g2) => {
|
|
1220
|
-
let minX = Infinity, maxX = -Infinity;
|
|
1221
|
-
for (const i of g2) {
|
|
1222
|
-
if (i.x < minX) minX = i.x;
|
|
1223
|
-
const r = i.x + i.w;
|
|
1224
|
-
if (r > maxX) maxX = r;
|
|
1225
|
-
}
|
|
1226
|
-
return (minX + maxX) / 2;
|
|
1227
|
-
});
|
|
1228
|
-
const assignments = [];
|
|
1229
|
-
for (let gi = 0; gi < groups.length; gi++) {
|
|
1230
|
-
for (let ci = 0; ci < numCols; ci++) {
|
|
1231
|
-
assignments.push({ gi, ci, dist: Math.abs(groupCenters[gi] - colCenters[ci]) });
|
|
1114
|
+
const lines = [];
|
|
1115
|
+
let curY = items[0].y;
|
|
1116
|
+
let curLine = [items[0]];
|
|
1117
|
+
for (let i = 1; i < items.length; i++) {
|
|
1118
|
+
if (Math.abs(items[i].y - curY) > 3) {
|
|
1119
|
+
lines.push(curLine);
|
|
1120
|
+
curLine = [];
|
|
1121
|
+
curY = items[i].y;
|
|
1232
1122
|
}
|
|
1123
|
+
curLine.push(items[i]);
|
|
1233
1124
|
}
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1125
|
+
if (curLine.length > 0) lines.push(curLine);
|
|
1126
|
+
return lines;
|
|
1127
|
+
}
|
|
1128
|
+
function mergeSuperscriptLines(lines) {
|
|
1129
|
+
if (lines.length <= 1) return lines;
|
|
1130
|
+
const band = (line) => {
|
|
1131
|
+
let bottom = Infinity, top = -Infinity;
|
|
1132
|
+
for (const i of line) {
|
|
1133
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1134
|
+
if (i.y < bottom) bottom = i.y;
|
|
1135
|
+
if (i.y + h > top) top = i.y + h;
|
|
1136
|
+
}
|
|
1137
|
+
return { bottom, top, height: top - bottom };
|
|
1138
|
+
};
|
|
1139
|
+
const isFrag = (line) => line.length <= 3 && line.every((i) => i.text.trim().length <= 8);
|
|
1140
|
+
const result = [lines[0]];
|
|
1141
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1142
|
+
const prev = result[result.length - 1];
|
|
1143
|
+
const curr = lines[i];
|
|
1144
|
+
const a = band(prev);
|
|
1145
|
+
const b = band(curr);
|
|
1146
|
+
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1147
|
+
const prevIsFrag = isFrag(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1148
|
+
const currIsFrag = isFrag(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1149
|
+
if (prevIsFrag || currIsFrag) {
|
|
1150
|
+
result[result.length - 1] = [...prev, ...curr];
|
|
1151
|
+
} else {
|
|
1152
|
+
result.push(curr);
|
|
1251
1153
|
}
|
|
1252
|
-
result.push({ col: bestCol, items: groups[gi] });
|
|
1253
1154
|
}
|
|
1254
1155
|
return result;
|
|
1255
1156
|
}
|
|
1256
|
-
function
|
|
1257
|
-
|
|
1258
|
-
const
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
usedItems.add(row.items[0]);
|
|
1270
|
-
continue;
|
|
1271
|
-
}
|
|
1272
|
-
const assignments = assignRowItems(row.items, columns, numCols);
|
|
1273
|
-
for (const { col, items } of assignments) {
|
|
1274
|
-
const text = items.map((i) => i.text).join(" ");
|
|
1275
|
-
const existing = cells[r][col].text;
|
|
1276
|
-
cells[r][col].text = existing ? existing + " " + text : text;
|
|
1277
|
-
for (const item of items) usedItems.add(item);
|
|
1157
|
+
function mergeLineSimple(items) {
|
|
1158
|
+
if (items.length <= 1) return items[0]?.text || "";
|
|
1159
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1160
|
+
const isEvenSpaced = detectEvenSpacedItems(sorted);
|
|
1161
|
+
let result = sorted[0].text;
|
|
1162
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1163
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1164
|
+
const avgFs = (sorted[i].fontSize + sorted[i - 1].fontSize) / 2;
|
|
1165
|
+
const tabThreshold = Math.max(avgFs * 2, 30);
|
|
1166
|
+
if (gap > tabThreshold) {
|
|
1167
|
+
result += " ";
|
|
1168
|
+
result += sorted[i].text;
|
|
1169
|
+
continue;
|
|
1278
1170
|
}
|
|
1171
|
+
if (isEvenSpaced[i]) {
|
|
1172
|
+
result += sorted[i].text;
|
|
1173
|
+
continue;
|
|
1174
|
+
}
|
|
1175
|
+
if (sorted[i].hasSpaceBefore && gap >= avgFs * 0.05) {
|
|
1176
|
+
result += " ";
|
|
1177
|
+
result += sorted[i].text;
|
|
1178
|
+
continue;
|
|
1179
|
+
}
|
|
1180
|
+
if (/[□■○●▶◆◇ㅇ]$/.test(sorted[i - 1].text) && /^[가-힣]/.test(sorted[i].text) && gap > 1) {
|
|
1181
|
+
result += " ";
|
|
1182
|
+
result += sorted[i].text;
|
|
1183
|
+
continue;
|
|
1184
|
+
}
|
|
1185
|
+
if (gap > spaceGapThreshold(avgFs)) result += " ";
|
|
1186
|
+
result += sorted[i].text;
|
|
1279
1187
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1188
|
+
return result;
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
// src/pdf/cluster-detector.ts
|
|
1192
|
+
var Y_TOL = 3;
|
|
1193
|
+
var COL_CLUSTER_TOL = 15;
|
|
1194
|
+
var MIN_ROWS = 3;
|
|
1195
|
+
var MIN_COLS = 2;
|
|
1196
|
+
var MIN_GAP_FACTOR = 2;
|
|
1197
|
+
var MIN_GAP_ABSOLUTE = 20;
|
|
1198
|
+
var MIN_COL_FILL_RATIO = 0.4;
|
|
1199
|
+
function detectClusterTables(items, pageNum) {
|
|
1200
|
+
if (items.length < MIN_ROWS * MIN_COLS) return [];
|
|
1201
|
+
const { merged, originMap } = mergeEvenSpacedClusters(items);
|
|
1202
|
+
const rows = mergeOverlappingRows(groupByBaseline(merged));
|
|
1203
|
+
if (rows.length < MIN_ROWS) return [];
|
|
1204
|
+
const results = [];
|
|
1205
|
+
const headerResult = detectHeaderRow(rows);
|
|
1206
|
+
if (headerResult) {
|
|
1207
|
+
const { columns, headerIdx } = headerResult;
|
|
1208
|
+
const headerRow = rows[headerIdx];
|
|
1209
|
+
const headerItems = [...headerRow.items].sort((a, b) => a.x - b.x);
|
|
1210
|
+
const headerAndBelow = rows.slice(headerIdx);
|
|
1211
|
+
const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
|
|
1212
|
+
const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
|
|
1213
|
+
for (const region of tableRegions) {
|
|
1214
|
+
const table = buildClusterTable(region.rows, columns, pageNum);
|
|
1215
|
+
if (table) {
|
|
1216
|
+
expandUsedItems(table.usedItems, originMap);
|
|
1217
|
+
results.push(table);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1288
1220
|
}
|
|
1289
|
-
|
|
1290
|
-
const
|
|
1291
|
-
if (
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1221
|
+
if (results.length === 0) {
|
|
1222
|
+
const suspiciousRows = rows.filter((row) => hasSuspiciousGaps(row));
|
|
1223
|
+
if (suspiciousRows.length >= MIN_ROWS) {
|
|
1224
|
+
const columns = extractColumnClusters(suspiciousRows);
|
|
1225
|
+
if (columns.length >= MIN_COLS) {
|
|
1226
|
+
const tableRegions = findTableRegions(rows, columns);
|
|
1227
|
+
for (const region of tableRegions) {
|
|
1228
|
+
const mergedRows = mergeMultiLineRows(region.rows, columns);
|
|
1229
|
+
const table = buildClusterTable(mergedRows, columns, pageNum);
|
|
1230
|
+
if (table) {
|
|
1231
|
+
expandUsedItems(table.usedItems, originMap);
|
|
1232
|
+
results.push(table);
|
|
1233
|
+
}
|
|
1301
1234
|
}
|
|
1302
|
-
for (let c = 0; c < numCols; c++) cells[r][c].text = "";
|
|
1303
|
-
break;
|
|
1304
1235
|
}
|
|
1305
1236
|
}
|
|
1306
1237
|
}
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1238
|
+
return results;
|
|
1239
|
+
}
|
|
1240
|
+
function mergeEvenSpacedClusters(items) {
|
|
1241
|
+
const originMap = /* @__PURE__ */ new Map();
|
|
1242
|
+
const rows = groupByBaseline(items);
|
|
1243
|
+
const merged = [];
|
|
1244
|
+
for (const row of rows) {
|
|
1245
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1246
|
+
let i = 0;
|
|
1247
|
+
while (i < sorted.length) {
|
|
1248
|
+
if (/^[가-힣\d]$/.test(sorted[i].text)) {
|
|
1249
|
+
let runEnd = i + 1;
|
|
1250
|
+
while (runEnd < sorted.length && /^[가-힣\d]$/.test(sorted[runEnd].text)) {
|
|
1251
|
+
if (sorted[runEnd].hasSpaceBefore) break;
|
|
1252
|
+
const gap = sorted[runEnd].x - (sorted[runEnd - 1].x + sorted[runEnd - 1].w);
|
|
1253
|
+
const fs = sorted[runEnd].fontSize;
|
|
1254
|
+
if (gap < fs * 0.1 || gap > fs * 3) break;
|
|
1255
|
+
runEnd++;
|
|
1256
|
+
}
|
|
1257
|
+
if (runEnd - i >= 3) {
|
|
1258
|
+
const gaps = [];
|
|
1259
|
+
for (let g2 = i + 1; g2 < runEnd; g2++) {
|
|
1260
|
+
gaps.push(sorted[g2].x - (sorted[g2 - 1].x + sorted[g2 - 1].w));
|
|
1261
|
+
}
|
|
1262
|
+
let minG = Infinity, maxG = -Infinity;
|
|
1263
|
+
for (const g2 of gaps) {
|
|
1264
|
+
if (g2 < minG) minG = g2;
|
|
1265
|
+
if (g2 > maxG) maxG = g2;
|
|
1266
|
+
}
|
|
1267
|
+
if (minG > 0 && maxG / minG <= 3) {
|
|
1268
|
+
const run = sorted.slice(i, runEnd);
|
|
1269
|
+
const text = run.map((r) => r.text).join("");
|
|
1270
|
+
const first = run[0], last = run[runEnd - i - 1];
|
|
1271
|
+
const item = {
|
|
1272
|
+
text,
|
|
1273
|
+
x: first.x,
|
|
1274
|
+
y: first.y,
|
|
1275
|
+
w: last.x + last.w - first.x,
|
|
1276
|
+
h: first.h,
|
|
1277
|
+
fontSize: first.fontSize,
|
|
1278
|
+
fontName: first.fontName
|
|
1279
|
+
};
|
|
1280
|
+
originMap.set(item, run);
|
|
1281
|
+
merged.push(item);
|
|
1282
|
+
i = runEnd;
|
|
1283
|
+
continue;
|
|
1284
|
+
}
|
|
1318
1285
|
}
|
|
1319
|
-
for (let c = 0; c < numCols; c++) next[c].text = "";
|
|
1320
1286
|
}
|
|
1287
|
+
merged.push(sorted[i]);
|
|
1288
|
+
i++;
|
|
1321
1289
|
}
|
|
1322
1290
|
}
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
const
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1291
|
+
return { merged, originMap };
|
|
1292
|
+
}
|
|
1293
|
+
function expandUsedItems(usedItems, originMap) {
|
|
1294
|
+
const toAdd = [];
|
|
1295
|
+
for (const item of usedItems) {
|
|
1296
|
+
const origins = originMap.get(item);
|
|
1297
|
+
if (origins) for (const o of origins) toAdd.push(o);
|
|
1298
|
+
}
|
|
1299
|
+
for (const a of toAdd) usedItems.add(a);
|
|
1300
|
+
}
|
|
1301
|
+
function detectHeaderRow(rows) {
|
|
1332
1302
|
const allItems = rows.flatMap((r) => r.items);
|
|
1333
|
-
|
|
1303
|
+
if (allItems.length === 0) return null;
|
|
1304
|
+
let allMinX = Infinity, allMaxX = -Infinity;
|
|
1334
1305
|
for (const i of allItems) {
|
|
1335
|
-
if (i.x <
|
|
1336
|
-
|
|
1337
|
-
if (
|
|
1338
|
-
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1339
|
-
if (i.y + h > maxY) maxY = i.y + h;
|
|
1306
|
+
if (i.x < allMinX) allMinX = i.x;
|
|
1307
|
+
const r = i.x + i.w;
|
|
1308
|
+
if (r > allMaxX) allMaxX = r;
|
|
1340
1309
|
}
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
control++;
|
|
1361
|
-
continue;
|
|
1362
|
-
}
|
|
1363
|
-
if (code === 65533) {
|
|
1364
|
-
replacement++;
|
|
1365
|
-
continue;
|
|
1366
|
-
}
|
|
1367
|
-
if (code >= 44032 && code <= 55203) {
|
|
1368
|
-
hangul++;
|
|
1369
|
-
continue;
|
|
1310
|
+
const pageSpan = allMaxX - allMinX;
|
|
1311
|
+
if (pageSpan <= 0) return null;
|
|
1312
|
+
for (let ri = 0; ri < rows.length; ri++) {
|
|
1313
|
+
const row = rows[ri];
|
|
1314
|
+
if (row.items.length < MIN_COLS || row.items.length > 6) continue;
|
|
1315
|
+
if (row.items.some((i) => i.text.length > 8)) continue;
|
|
1316
|
+
if (!row.items.some((i) => /[가-힣]/.test(i.text))) continue;
|
|
1317
|
+
if (row.items.some((i) => /^[□■○●·※▶▷◆◇\-]/.test(i.text))) continue;
|
|
1318
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1319
|
+
const xSpan = sorted[sorted.length - 1].x + sorted[sorted.length - 1].w - sorted[0].x;
|
|
1320
|
+
if (xSpan / pageSpan < 0.4) continue;
|
|
1321
|
+
const avgFs = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
|
|
1322
|
+
let hasLargeGap = false;
|
|
1323
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1324
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1325
|
+
if (gap >= avgFs * 2.5) {
|
|
1326
|
+
hasLargeGap = true;
|
|
1327
|
+
break;
|
|
1328
|
+
}
|
|
1370
1329
|
}
|
|
1371
|
-
if (
|
|
1372
|
-
|
|
1373
|
-
|
|
1330
|
+
if (!hasLargeGap) continue;
|
|
1331
|
+
const columns = sorted.map((item) => ({ x: item.x, count: 0 }));
|
|
1332
|
+
let matchCount = 0;
|
|
1333
|
+
for (let j = ri + 1; j < rows.length && matchCount < MIN_ROWS + 2; j++) {
|
|
1334
|
+
const matched = countMatchedColumnsRange(rows[j], columns, sorted);
|
|
1335
|
+
if (matched >= MIN_COLS) matchCount++;
|
|
1374
1336
|
}
|
|
1337
|
+
if (matchCount < MIN_ROWS) continue;
|
|
1338
|
+
return { columns, headerIdx: ri };
|
|
1375
1339
|
}
|
|
1376
|
-
|
|
1377
|
-
const puaRatio = pua / denom;
|
|
1378
|
-
const controlCharRatio = control / denom;
|
|
1379
|
-
const replacementCharRatio = replacement / denom;
|
|
1380
|
-
let needsOcr = false;
|
|
1381
|
-
let ocrReason;
|
|
1382
|
-
if (total < LOW_TEXT_THRESHOLD) {
|
|
1383
|
-
needsOcr = true;
|
|
1384
|
-
ocrReason = "low_text";
|
|
1385
|
-
} else if (puaRatio >= HIGH_PUA_THRESHOLD) {
|
|
1386
|
-
needsOcr = true;
|
|
1387
|
-
ocrReason = "high_pua";
|
|
1388
|
-
} else if (controlCharRatio >= HIGH_CONTROL_THRESHOLD) {
|
|
1389
|
-
needsOcr = true;
|
|
1390
|
-
ocrReason = "high_control";
|
|
1391
|
-
} else if (replacementCharRatio >= HIGH_REPLACEMENT_THRESHOLD) {
|
|
1392
|
-
needsOcr = true;
|
|
1393
|
-
ocrReason = "high_replacement";
|
|
1394
|
-
}
|
|
1395
|
-
return {
|
|
1396
|
-
page,
|
|
1397
|
-
textChars: total,
|
|
1398
|
-
hangulRatio: hangul / denom,
|
|
1399
|
-
controlCharRatio,
|
|
1400
|
-
replacementCharRatio,
|
|
1401
|
-
puaRatio,
|
|
1402
|
-
needsOcr,
|
|
1403
|
-
ocrReason
|
|
1404
|
-
};
|
|
1405
|
-
}
|
|
1406
|
-
var LOW_TEXT_THRESHOLD = 20;
|
|
1407
|
-
var HIGH_PUA_THRESHOLD = 0.2;
|
|
1408
|
-
var HIGH_CONTROL_THRESHOLD = 0.05;
|
|
1409
|
-
var HIGH_REPLACEMENT_THRESHOLD = 0.05;
|
|
1410
|
-
var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
|
|
1411
|
-
function stripControlChars(text) {
|
|
1412
|
-
return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
|
|
1413
|
-
}
|
|
1414
|
-
function summarizeDocumentQuality(pages) {
|
|
1415
|
-
if (pages.length === 0) {
|
|
1416
|
-
return {
|
|
1417
|
-
totalPages: 0,
|
|
1418
|
-
totalTextChars: 0,
|
|
1419
|
-
avgHangulRatio: 0,
|
|
1420
|
-
avgControlCharRatio: 0,
|
|
1421
|
-
avgReplacementCharRatio: 0,
|
|
1422
|
-
avgPuaRatio: 0,
|
|
1423
|
-
lowTextPageCount: 0,
|
|
1424
|
-
highPuaPageCount: 0,
|
|
1425
|
-
needsOcr: false,
|
|
1426
|
-
ocrCandidatePages: []
|
|
1427
|
-
};
|
|
1428
|
-
}
|
|
1429
|
-
let textChars = 0;
|
|
1430
|
-
let hangul = 0;
|
|
1431
|
-
let control = 0;
|
|
1432
|
-
let replacement = 0;
|
|
1433
|
-
let pua = 0;
|
|
1434
|
-
let lowText = 0;
|
|
1435
|
-
let highPua = 0;
|
|
1436
|
-
const ocrCandidatePages = [];
|
|
1437
|
-
for (const p of pages) {
|
|
1438
|
-
textChars += p.textChars;
|
|
1439
|
-
hangul += p.hangulRatio;
|
|
1440
|
-
control += p.controlCharRatio;
|
|
1441
|
-
replacement += p.replacementCharRatio;
|
|
1442
|
-
pua += p.puaRatio;
|
|
1443
|
-
if (p.textChars < LOW_TEXT_THRESHOLD) lowText++;
|
|
1444
|
-
if (p.puaRatio >= HIGH_PUA_THRESHOLD) highPua++;
|
|
1445
|
-
if (p.needsOcr) ocrCandidatePages.push(p.page);
|
|
1446
|
-
}
|
|
1447
|
-
const n = pages.length;
|
|
1448
|
-
return {
|
|
1449
|
-
totalPages: n,
|
|
1450
|
-
totalTextChars: textChars,
|
|
1451
|
-
avgHangulRatio: hangul / n,
|
|
1452
|
-
avgControlCharRatio: control / n,
|
|
1453
|
-
avgReplacementCharRatio: replacement / n,
|
|
1454
|
-
avgPuaRatio: pua / n,
|
|
1455
|
-
lowTextPageCount: lowText,
|
|
1456
|
-
highPuaPageCount: highPua,
|
|
1457
|
-
needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
|
|
1458
|
-
ocrCandidatePages
|
|
1459
|
-
};
|
|
1460
|
-
}
|
|
1461
|
-
|
|
1462
|
-
// src/pdf/polyfill.ts
|
|
1463
|
-
import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
|
|
1464
|
-
var g = globalThis;
|
|
1465
|
-
if (typeof g.DOMMatrix === "undefined") {
|
|
1466
|
-
g.DOMMatrix = class DOMMatrix {
|
|
1467
|
-
m = [1, 0, 0, 1, 0, 0];
|
|
1468
|
-
constructor(init) {
|
|
1469
|
-
if (init) this.m = init;
|
|
1470
|
-
}
|
|
1471
|
-
};
|
|
1472
|
-
}
|
|
1473
|
-
if (typeof g.Path2D === "undefined") {
|
|
1474
|
-
g.Path2D = class Path2D {
|
|
1475
|
-
};
|
|
1476
|
-
}
|
|
1477
|
-
g.pdfjsWorker = pdfjsWorker;
|
|
1478
|
-
|
|
1479
|
-
// src/pdf/parser.ts
|
|
1480
|
-
import { getDocument, GlobalWorkerOptions } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
1481
|
-
GlobalWorkerOptions.workerSrc = "";
|
|
1482
|
-
var MAX_PAGES = 5e3;
|
|
1483
|
-
var MAX_TOTAL_TEXT = 100 * 1024 * 1024;
|
|
1484
|
-
var PDF_LOAD_TIMEOUT_MS = 3e4;
|
|
1485
|
-
async function loadPdfWithTimeout(buffer) {
|
|
1486
|
-
const loadingTask = getDocument({
|
|
1487
|
-
data: new Uint8Array(buffer),
|
|
1488
|
-
useSystemFonts: true,
|
|
1489
|
-
disableFontFace: true,
|
|
1490
|
-
isEvalSupported: false
|
|
1491
|
-
});
|
|
1492
|
-
let timer;
|
|
1493
|
-
try {
|
|
1494
|
-
return await Promise.race([
|
|
1495
|
-
loadingTask.promise,
|
|
1496
|
-
new Promise((_, reject) => {
|
|
1497
|
-
timer = setTimeout(() => {
|
|
1498
|
-
loadingTask.destroy();
|
|
1499
|
-
reject(new KordocError("PDF \uB85C\uB529 \uD0C0\uC784\uC544\uC6C3 (30\uCD08 \uCD08\uACFC)"));
|
|
1500
|
-
}, PDF_LOAD_TIMEOUT_MS);
|
|
1501
|
-
})
|
|
1502
|
-
]);
|
|
1503
|
-
} finally {
|
|
1504
|
-
if (timer !== void 0) clearTimeout(timer);
|
|
1505
|
-
}
|
|
1506
|
-
}
|
|
1507
|
-
async function parsePdfDocument(buffer, options) {
|
|
1508
|
-
const formulaBuffer = options?.formulaOcr ? buffer.slice(0) : null;
|
|
1509
|
-
const doc = await loadPdfWithTimeout(buffer);
|
|
1510
|
-
try {
|
|
1511
|
-
const pageCount = doc.numPages;
|
|
1512
|
-
if (pageCount === 0) throw new KordocError("PDF\uC5D0 \uD398\uC774\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
|
|
1513
|
-
const metadata = { pageCount };
|
|
1514
|
-
await extractPdfMetadata(doc, metadata);
|
|
1515
|
-
const blocks = [];
|
|
1516
|
-
const warnings = [];
|
|
1517
|
-
const pageQuality = [];
|
|
1518
|
-
let totalChars = 0;
|
|
1519
|
-
let totalTextBytes = 0;
|
|
1520
|
-
const effectivePageCount = Math.min(pageCount, MAX_PAGES);
|
|
1521
|
-
const pageFilter = options?.pages ? parsePageRange(options.pages, effectivePageCount) : null;
|
|
1522
|
-
const totalTarget = pageFilter ? pageFilter.size : effectivePageCount;
|
|
1523
|
-
const fontSizeFreq = /* @__PURE__ */ new Map();
|
|
1524
|
-
const pageHeights = /* @__PURE__ */ new Map();
|
|
1525
|
-
const pagesWithLargeImage = /* @__PURE__ */ new Set();
|
|
1526
|
-
const skippedImagePages = /* @__PURE__ */ new Map();
|
|
1527
|
-
let parsedPages = 0;
|
|
1528
|
-
for (let i = 1; i <= effectivePageCount; i++) {
|
|
1529
|
-
if (pageFilter && !pageFilter.has(i)) continue;
|
|
1530
|
-
try {
|
|
1531
|
-
const page = await doc.getPage(i);
|
|
1532
|
-
const tc = await page.getTextContent();
|
|
1533
|
-
const viewport = page.getViewport({ scale: 1 });
|
|
1534
|
-
pageHeights.set(i, viewport.height);
|
|
1535
|
-
const rawItems = tc.items;
|
|
1536
|
-
const items = normalizeItems(rawItems);
|
|
1537
|
-
const { visible, hiddenCount } = filterHiddenText(items, viewport.width, viewport.height);
|
|
1538
|
-
if (hiddenCount > 0) {
|
|
1539
|
-
warnings.push({ page: i, message: `${hiddenCount}\uAC1C \uC228\uACA8\uC9C4 \uD14D\uC2A4\uD2B8 \uC694\uC18C \uD544\uD130\uB9C1\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
1540
|
-
}
|
|
1541
|
-
for (const item of visible) {
|
|
1542
|
-
if (item.fontSize > 0) fontSizeFreq.set(item.fontSize, (fontSizeFreq.get(item.fontSize) || 0) + 1);
|
|
1543
|
-
}
|
|
1544
|
-
const opList = await page.getOperatorList();
|
|
1545
|
-
const pageArea = viewport.width * viewport.height;
|
|
1546
|
-
if (pageArea > 0) {
|
|
1547
|
-
const imageRegions = extractImageRegions(opList.fnArray, opList.argsArray);
|
|
1548
|
-
let uncovered = 0;
|
|
1549
|
-
for (const r of imageRegions) {
|
|
1550
|
-
const area = (r.x2 - r.x1) * (r.y2 - r.y1);
|
|
1551
|
-
if (area < pageArea * 0.05) continue;
|
|
1552
|
-
pagesWithLargeImage.add(i);
|
|
1553
|
-
const hasText = visible.some((it) => {
|
|
1554
|
-
const cx = it.x + it.w / 2;
|
|
1555
|
-
const cy = it.y + (it.h || it.fontSize) / 2;
|
|
1556
|
-
return cx >= r.x1 && cx <= r.x2 && cy >= r.y1 && cy <= r.y2;
|
|
1557
|
-
});
|
|
1558
|
-
if (!hasText) uncovered++;
|
|
1559
|
-
}
|
|
1560
|
-
if (uncovered > 0) skippedImagePages.set(i, uncovered);
|
|
1561
|
-
}
|
|
1562
|
-
const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
|
|
1563
|
-
for (const b of pageBlocks) blocks.push(b);
|
|
1564
|
-
let pageText = "";
|
|
1565
|
-
for (const b of pageBlocks) {
|
|
1566
|
-
const t = b.text || "";
|
|
1567
|
-
totalChars += t.replace(/\s/g, "").length;
|
|
1568
|
-
totalTextBytes += t.length * 2;
|
|
1569
|
-
pageText += pageText ? "\n" + t : t;
|
|
1570
|
-
}
|
|
1571
|
-
pageQuality.push(computePageQuality(i, pageText));
|
|
1572
|
-
if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
|
|
1573
|
-
parsedPages++;
|
|
1574
|
-
options?.onProgress?.(parsedPages, totalTarget);
|
|
1575
|
-
} catch (pageErr) {
|
|
1576
|
-
if (pageErr instanceof KordocError) throw pageErr;
|
|
1577
|
-
warnings.push({ page: i, message: `\uD398\uC774\uC9C0 ${i} \uD30C\uC2F1 \uC2E4\uD328: ${pageErr instanceof Error ? pageErr.message : "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958"}`, code: "PARTIAL_PARSE" });
|
|
1578
|
-
}
|
|
1579
|
-
}
|
|
1580
|
-
const parsedPageCount = parsedPages || (pageFilter ? pageFilter.size : effectivePageCount);
|
|
1581
|
-
let isImageBased = false;
|
|
1582
|
-
if (totalChars / Math.max(parsedPageCount, 1) < 10) {
|
|
1583
|
-
if (options?.ocr) {
|
|
1584
|
-
try {
|
|
1585
|
-
const { ocrPages } = await import("./provider-AKROB7WQ.js");
|
|
1586
|
-
const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
|
|
1587
|
-
if (ocrBlocks.length > 0) {
|
|
1588
|
-
const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
|
|
1589
|
-
return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
|
|
1590
|
-
}
|
|
1591
|
-
} catch {
|
|
1592
|
-
}
|
|
1593
|
-
}
|
|
1594
|
-
isImageBased = true;
|
|
1595
|
-
warnings.push({
|
|
1596
|
-
message: `\uC774\uBBF8\uC9C0 \uAE30\uBC18 PDF (${pageCount}\uD398\uC774\uC9C0, \uD14D\uC2A4\uD2B8 ${totalChars}\uC790) \u2014 \uD14D\uC2A4\uD2B8 \uB808\uC774\uC5B4\uAC00 \uC5C6\uC5B4 OCR\uC774 \uD544\uC694\uD569\uB2C8\uB2E4`,
|
|
1597
|
-
code: "NEEDS_OCR"
|
|
1598
|
-
});
|
|
1599
|
-
}
|
|
1600
|
-
if (!isImageBased) {
|
|
1601
|
-
const OCR_REASON_MESSAGES = {
|
|
1602
|
-
low_text: "\uD14D\uC2A4\uD2B8\uAC00 \uAC70\uC758 \uC5C6\uB294 \uD398\uC774\uC9C0 (\uC2A4\uCE94/\uC774\uBBF8\uC9C0 \uCD94\uC815)",
|
|
1603
|
-
high_pua: "\uAE00\uAF34 \uB9E4\uD551 \uC2E4\uD328 (PUA \uBE44\uC728 \uB192\uC74C) \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
1604
|
-
high_control: "\uC81C\uC5B4\uBB38\uC790 \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
1605
|
-
high_replacement: "\uB300\uCCB4\uBB38\uC790(U+FFFD) \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00"
|
|
1606
|
-
};
|
|
1607
|
-
for (const pq of pageQuality) {
|
|
1608
|
-
if (!pq.needsOcr || !pq.ocrReason) continue;
|
|
1609
|
-
if (pq.ocrReason === "low_text" && !pagesWithLargeImage.has(pq.page)) continue;
|
|
1610
|
-
warnings.push({ page: pq.page, message: `${OCR_REASON_MESSAGES[pq.ocrReason]} \u2014 OCR \uAC80\uD1A0 \uD544\uC694`, code: "NEEDS_OCR" });
|
|
1611
|
-
}
|
|
1612
|
-
}
|
|
1613
|
-
if (!isImageBased) {
|
|
1614
|
-
for (const [page, count] of [...skippedImagePages.entries()].sort((a, b) => a[0] - b[0])) {
|
|
1615
|
-
warnings.push({ page, message: `${count}\uAC1C \uC774\uBBF8\uC9C0 \uC601\uC5ED\uC5D0 \uCD94\uCD9C \uAC00\uB2A5\uD55C \uD14D\uC2A4\uD2B8 \uC5C6\uC74C (\uADF8\uB9BC/\uCC28\uD2B8/\uB3C4\uC7A5 \uB0B4\uC6A9 \uB204\uB77D \uAC00\uB2A5)`, code: "SKIPPED_IMAGE" });
|
|
1616
|
-
}
|
|
1617
|
-
}
|
|
1618
|
-
if (options?.removeHeaderFooter !== false && parsedPageCount >= 3) {
|
|
1619
|
-
const removed = removeHeaderFooterBlocks(blocks, pageHeights, warnings);
|
|
1620
|
-
for (let ri = removed.length - 1; ri >= 0; ri--) {
|
|
1621
|
-
blocks.splice(removed[ri], 1);
|
|
1622
|
-
}
|
|
1623
|
-
}
|
|
1624
|
-
mergeCrossPageTables(blocks);
|
|
1625
|
-
if (options?.formulaOcr && formulaBuffer) {
|
|
1626
|
-
try {
|
|
1627
|
-
await applyFormulaOcr(formulaBuffer, blocks, pageFilter, effectivePageCount, warnings, options.onProgress);
|
|
1628
|
-
} catch (e) {
|
|
1629
|
-
warnings.push({
|
|
1630
|
-
message: `\uC218\uC2DD OCR \uC2E4\uD328: ${e instanceof Error ? e.message : String(e)}`,
|
|
1631
|
-
code: "PARTIAL_PARSE"
|
|
1632
|
-
});
|
|
1633
|
-
}
|
|
1634
|
-
}
|
|
1635
|
-
const medianFontSize = computeMedianFontSizeFromFreq(fontSizeFreq);
|
|
1636
|
-
if (medianFontSize > 0) {
|
|
1637
|
-
detectHeadings(blocks, medianFontSize);
|
|
1638
|
-
}
|
|
1639
|
-
detectMarkerHeadings(blocks);
|
|
1640
|
-
detectTableCaptions(blocks);
|
|
1641
|
-
detectKoreanListBlocks(blocks);
|
|
1642
|
-
const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
|
|
1643
|
-
sanitizeBlockControlChars(blocks);
|
|
1644
|
-
let markdown = cleanPdfText(blocksToMarkdown(blocks));
|
|
1645
|
-
return {
|
|
1646
|
-
markdown,
|
|
1647
|
-
blocks,
|
|
1648
|
-
metadata,
|
|
1649
|
-
outline: outline.length > 0 ? outline : void 0,
|
|
1650
|
-
warnings: warnings.length > 0 ? warnings : void 0,
|
|
1651
|
-
isImageBased: isImageBased || void 0,
|
|
1652
|
-
pageQuality,
|
|
1653
|
-
qualitySummary: summarizeDocumentQuality(pageQuality)
|
|
1654
|
-
};
|
|
1655
|
-
} finally {
|
|
1656
|
-
await doc.destroy().catch(() => {
|
|
1657
|
-
});
|
|
1658
|
-
}
|
|
1659
|
-
}
|
|
1660
|
-
async function extractPdfMetadata(doc, metadata) {
|
|
1661
|
-
try {
|
|
1662
|
-
const result = await doc.getMetadata();
|
|
1663
|
-
if (!result?.info) return;
|
|
1664
|
-
const info = result.info;
|
|
1665
|
-
if (typeof info.Title === "string" && info.Title.trim()) metadata.title = info.Title.trim();
|
|
1666
|
-
if (typeof info.Author === "string" && info.Author.trim()) metadata.author = info.Author.trim();
|
|
1667
|
-
if (typeof info.Creator === "string" && info.Creator.trim()) metadata.creator = info.Creator.trim();
|
|
1668
|
-
if (typeof info.Subject === "string" && info.Subject.trim()) metadata.description = info.Subject.trim();
|
|
1669
|
-
if (typeof info.Keywords === "string" && info.Keywords.trim()) {
|
|
1670
|
-
metadata.keywords = info.Keywords.split(/[,;]/).map((k) => k.trim()).filter(Boolean);
|
|
1671
|
-
}
|
|
1672
|
-
if (typeof info.CreationDate === "string") metadata.createdAt = parsePdfDate(info.CreationDate);
|
|
1673
|
-
if (typeof info.ModDate === "string") metadata.modifiedAt = parsePdfDate(info.ModDate);
|
|
1674
|
-
} catch {
|
|
1675
|
-
}
|
|
1676
|
-
}
|
|
1677
|
-
function parsePdfDate(dateStr) {
|
|
1678
|
-
const m = dateStr.match(/D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/);
|
|
1679
|
-
if (!m) return void 0;
|
|
1680
|
-
const [, year, month = "01", day = "01", hour = "00", min = "00", sec = "00"] = m;
|
|
1681
|
-
return `${year}-${month}-${day}T${hour}:${min}:${sec}`;
|
|
1682
|
-
}
|
|
1683
|
-
async function extractPdfMetadataOnly(buffer) {
|
|
1684
|
-
const doc = await loadPdfWithTimeout(buffer);
|
|
1685
|
-
try {
|
|
1686
|
-
const metadata = { pageCount: doc.numPages };
|
|
1687
|
-
await extractPdfMetadata(doc, metadata);
|
|
1688
|
-
return metadata;
|
|
1689
|
-
} finally {
|
|
1690
|
-
await doc.destroy().catch(() => {
|
|
1691
|
-
});
|
|
1692
|
-
}
|
|
1693
|
-
}
|
|
1694
|
-
function filterHiddenText(items, pageWidth, pageHeight) {
|
|
1695
|
-
let hiddenCount = 0;
|
|
1696
|
-
const visible = [];
|
|
1697
|
-
for (const item of items) {
|
|
1698
|
-
if (item.isHidden) {
|
|
1699
|
-
hiddenCount++;
|
|
1700
|
-
continue;
|
|
1701
|
-
}
|
|
1702
|
-
const margin = Math.max(pageWidth, pageHeight) * 0.1;
|
|
1703
|
-
if (item.x < -margin || item.x > pageWidth + margin || item.y < -margin || item.y > pageHeight + margin) {
|
|
1704
|
-
hiddenCount++;
|
|
1705
|
-
continue;
|
|
1706
|
-
}
|
|
1707
|
-
visible.push(item);
|
|
1708
|
-
}
|
|
1709
|
-
return { visible, hiddenCount };
|
|
1710
|
-
}
|
|
1711
|
-
function computeMedianFontSizeFromFreq(freq) {
|
|
1712
|
-
if (freq.size === 0) return 0;
|
|
1713
|
-
let total = 0;
|
|
1714
|
-
for (const count of freq.values()) total += count;
|
|
1715
|
-
const sorted = [...freq.entries()].sort((a, b) => a[0] - b[0]);
|
|
1716
|
-
const mid = Math.floor(total / 2);
|
|
1717
|
-
let cumulative = 0;
|
|
1718
|
-
for (const [size, count] of sorted) {
|
|
1719
|
-
cumulative += count;
|
|
1720
|
-
if (cumulative > mid) return size;
|
|
1721
|
-
}
|
|
1722
|
-
return sorted[sorted.length - 1][0];
|
|
1723
|
-
}
|
|
1724
|
-
function detectHeadings(blocks, medianFontSize) {
|
|
1725
|
-
for (const block of blocks) {
|
|
1726
|
-
if (block.type !== "paragraph" || !block.text || !block.style?.fontSize) continue;
|
|
1727
|
-
const text = block.text.trim();
|
|
1728
|
-
if (text.length === 0 || text.length > 200) continue;
|
|
1729
|
-
if (/^\d+$/.test(text)) continue;
|
|
1730
|
-
const ratio = block.style.fontSize / medianFontSize;
|
|
1731
|
-
let level = 0;
|
|
1732
|
-
if (ratio >= HEADING_RATIO_H1) level = 1;
|
|
1733
|
-
else if (ratio >= HEADING_RATIO_H2) level = 2;
|
|
1734
|
-
else if (ratio >= HEADING_RATIO_H3) level = 3;
|
|
1735
|
-
if (level > 0) {
|
|
1736
|
-
block.type = "heading";
|
|
1737
|
-
block.level = level;
|
|
1738
|
-
block.text = collapseEvenSpacing(text);
|
|
1739
|
-
}
|
|
1740
|
-
}
|
|
1741
|
-
}
|
|
1742
|
-
function collapseEvenSpacing(text) {
|
|
1743
|
-
const tokens = text.split(" ");
|
|
1744
|
-
const singleCharCount = tokens.filter((t) => t.length === 1).length;
|
|
1745
|
-
if (tokens.length >= 3 && singleCharCount / tokens.length >= 0.7) {
|
|
1746
|
-
return tokens.join("");
|
|
1747
|
-
}
|
|
1748
|
-
return text.replace(
|
|
1749
|
-
/(?<![가-힣])[가-힣](?: [가-힣\d]){2,}(?![가-힣])/g,
|
|
1750
|
-
(match) => match.replace(/ /g, "")
|
|
1751
|
-
);
|
|
1752
|
-
}
|
|
1753
|
-
function shouldDemoteTable(table) {
|
|
1754
|
-
const allCells = table.cells.flatMap((row) => row.map((c) => c.text.trim())).filter(Boolean);
|
|
1755
|
-
const allText = allCells.join(" ");
|
|
1756
|
-
if (table.rows <= 3 && table.cols <= 3) {
|
|
1757
|
-
const totalCells2 = table.rows * table.cols;
|
|
1758
|
-
const emptyCells2 = totalCells2 - allCells.length;
|
|
1759
|
-
if (emptyCells2 >= totalCells2 * 0.3) return true;
|
|
1760
|
-
if (/[□■◆○●▶ㅇ]/.test(allText)) return true;
|
|
1761
|
-
if (/<[^>]+>/.test(allText)) return true;
|
|
1762
|
-
}
|
|
1763
|
-
if (allText.length > 200) return false;
|
|
1764
|
-
if (/[□■◆○●▶]/.test(allText) && table.rows <= 3) return true;
|
|
1765
|
-
const totalCells = table.rows * table.cols;
|
|
1766
|
-
const emptyCells = totalCells - allCells.length;
|
|
1767
|
-
if (table.rows <= 2 && emptyCells > totalCells * 0.5) return true;
|
|
1768
|
-
if (table.rows === 1 && !/\d{2,}/.test(allText)) return true;
|
|
1769
|
-
return false;
|
|
1770
|
-
}
|
|
1771
|
-
function demoteTableToText(table) {
|
|
1772
|
-
const lines = [];
|
|
1773
|
-
for (let r = 0; r < table.rows; r++) {
|
|
1774
|
-
const cells = table.cells[r].map((c) => c.text.trim()).filter(Boolean);
|
|
1775
|
-
if (cells.length === 0) continue;
|
|
1776
|
-
if (table.cols === 2 && cells.length === 2) {
|
|
1777
|
-
lines.push(`${cells[0]} : ${cells[1]}`);
|
|
1778
|
-
} else {
|
|
1779
|
-
lines.push(cells.join(" "));
|
|
1780
|
-
}
|
|
1781
|
-
}
|
|
1782
|
-
return lines.join("\n");
|
|
1783
|
-
}
|
|
1784
|
-
function detectMarkerHeadings(blocks) {
|
|
1785
|
-
for (let i = 0; i < blocks.length; i++) {
|
|
1786
|
-
const block = blocks[i];
|
|
1787
|
-
if (block.type !== "paragraph" || !block.text) continue;
|
|
1788
|
-
const text = block.text.trim();
|
|
1789
|
-
if (text.length < 50 && /^[□■◆◇▶]\s*[가-힣]/.test(text)) {
|
|
1790
|
-
block.type = "heading";
|
|
1791
|
-
block.level = 4;
|
|
1792
|
-
continue;
|
|
1793
|
-
}
|
|
1794
|
-
if (/^[가-힣]{2,6}$/.test(text) && block.style?.fontSize) {
|
|
1795
|
-
const prev = blocks[i - 1];
|
|
1796
|
-
const next = blocks[i + 1];
|
|
1797
|
-
const prevIsStructural = !prev || prev.type === "table" || prev.type === "heading" || prev.type === "separator";
|
|
1798
|
-
const nextIsStructural = !next || next.type === "table" || next.type === "heading" || next.type === "paragraph" && next.text && /^[□■◆○●]/.test(next.text.trim());
|
|
1799
|
-
if (prevIsStructural || nextIsStructural) {
|
|
1800
|
-
block.type = "heading";
|
|
1801
|
-
block.level = 3;
|
|
1802
|
-
}
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
var MAX_XYCUT_DEPTH = 50;
|
|
1807
|
-
var XYCUT_MIN_GAP = 5;
|
|
1808
|
-
var CROSS_LAYOUT_BETA = 2;
|
|
1809
|
-
var CROSS_OVERLAP_RATIO = 0.1;
|
|
1810
|
-
var CROSS_MIN_OVERLAPS = 2;
|
|
1811
|
-
var CROSS_MAX_MASK_RATIO = 0.2;
|
|
1812
|
-
var NARROW_ELEMENT_WIDTH_RATIO = 0.1;
|
|
1813
|
-
function xyCutOrder(items, gapThreshold, depth = 0) {
|
|
1814
|
-
if (items.length === 0) return [];
|
|
1815
|
-
if (items.length <= 2 || depth >= MAX_XYCUT_DEPTH) return [items];
|
|
1816
|
-
if (depth === 0 && items.length >= 3) {
|
|
1817
|
-
const cross = identifyCrossLayoutItems(items);
|
|
1818
|
-
if (cross.size > 0 && cross.size <= items.length * CROSS_MAX_MASK_RATIO) {
|
|
1819
|
-
const rest = items.filter((i) => !cross.has(i));
|
|
1820
|
-
if (rest.length > 0) {
|
|
1821
|
-
const groups = xyCutOrder(rest, gapThreshold, 1);
|
|
1822
|
-
return mergeCrossLayoutGroups(groups, [...cross]);
|
|
1823
|
-
}
|
|
1824
|
-
}
|
|
1825
|
-
}
|
|
1826
|
-
const minGap = Math.max(XYCUT_MIN_GAP, gapThreshold);
|
|
1827
|
-
const hCut = findHorizontalCut(items);
|
|
1828
|
-
const vCut = findVerticalCutWithOutlierFilter(items, minGap);
|
|
1829
|
-
const hValid = hCut.gap >= minGap;
|
|
1830
|
-
const vValid = vCut.gap >= minGap;
|
|
1831
|
-
let useHorizontal;
|
|
1832
|
-
if (hValid && vValid) useHorizontal = vCut.gap <= hCut.gap * 1.5;
|
|
1833
|
-
else if (hValid) useHorizontal = true;
|
|
1834
|
-
else if (vValid) useHorizontal = false;
|
|
1835
|
-
else return [items];
|
|
1836
|
-
if (useHorizontal) {
|
|
1837
|
-
const upper = items.filter((i) => i.y > hCut.position);
|
|
1838
|
-
const lower = items.filter((i) => i.y <= hCut.position);
|
|
1839
|
-
if (upper.length > 0 && lower.length > 0 && upper.length < items.length) {
|
|
1840
|
-
return [...xyCutOrder(upper, gapThreshold, depth + 1), ...xyCutOrder(lower, gapThreshold, depth + 1)];
|
|
1841
|
-
}
|
|
1842
|
-
} else {
|
|
1843
|
-
const left = items.filter((i) => i.x + i.w / 2 < vCut.position);
|
|
1844
|
-
const right = items.filter((i) => i.x + i.w / 2 >= vCut.position);
|
|
1845
|
-
if (left.length > 0 && right.length > 0 && left.length < items.length) {
|
|
1846
|
-
return [...xyCutOrder(left, gapThreshold, depth + 1), ...xyCutOrder(right, gapThreshold, depth + 1)];
|
|
1847
|
-
}
|
|
1848
|
-
}
|
|
1849
|
-
return [items];
|
|
1850
|
-
}
|
|
1851
|
-
function identifyCrossLayoutItems(items) {
|
|
1852
|
-
const cross = /* @__PURE__ */ new Set();
|
|
1853
|
-
if (items.length < 3) return cross;
|
|
1854
|
-
let maxWidth = 0;
|
|
1855
|
-
for (const i of items) {
|
|
1856
|
-
if (i.w > maxWidth) maxWidth = i.w;
|
|
1857
|
-
}
|
|
1858
|
-
const threshold = CROSS_LAYOUT_BETA * maxWidth;
|
|
1859
|
-
for (const item of items) {
|
|
1860
|
-
if (item.w < threshold) continue;
|
|
1861
|
-
let overlaps = 0;
|
|
1862
|
-
for (const other of items) {
|
|
1863
|
-
if (other === item) continue;
|
|
1864
|
-
const left = Math.max(item.x, other.x);
|
|
1865
|
-
const right = Math.min(item.x + item.w, other.x + other.w);
|
|
1866
|
-
const overlapW = right - left;
|
|
1867
|
-
if (overlapW <= 0) continue;
|
|
1868
|
-
const smaller = Math.min(item.w, other.w);
|
|
1869
|
-
if (smaller > 0 && overlapW / smaller >= CROSS_OVERLAP_RATIO) {
|
|
1870
|
-
overlaps++;
|
|
1871
|
-
if (overlaps >= CROSS_MIN_OVERLAPS) break;
|
|
1872
|
-
}
|
|
1873
|
-
}
|
|
1874
|
-
if (overlaps >= CROSS_MIN_OVERLAPS) cross.add(item);
|
|
1875
|
-
}
|
|
1876
|
-
return cross;
|
|
1340
|
+
return null;
|
|
1877
1341
|
}
|
|
1878
|
-
function
|
|
1879
|
-
if (
|
|
1880
|
-
const
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
result.push(
|
|
1894
|
-
continue;
|
|
1895
|
-
}
|
|
1896
|
-
if (gi >= groups.length) {
|
|
1897
|
-
result.push([sortedCross[ci++]]);
|
|
1898
|
-
continue;
|
|
1342
|
+
function mergeOverlappingRows(rows) {
|
|
1343
|
+
if (rows.length <= 1) return rows;
|
|
1344
|
+
const result = [rows[0]];
|
|
1345
|
+
for (let i = 1; i < rows.length; i++) {
|
|
1346
|
+
const prev = result[result.length - 1];
|
|
1347
|
+
const curr = rows[i];
|
|
1348
|
+
const a = rowBand(prev);
|
|
1349
|
+
const b = rowBand(curr);
|
|
1350
|
+
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1351
|
+
const prevIsFrag = isFragmentRow(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1352
|
+
const currIsFrag = isFragmentRow(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1353
|
+
if (prevIsFrag || currIsFrag) {
|
|
1354
|
+
const baseY = prevIsFrag ? curr.y : prev.y;
|
|
1355
|
+
result[result.length - 1] = { y: baseY, items: [...prev.items, ...curr.items] };
|
|
1356
|
+
} else {
|
|
1357
|
+
result.push(curr);
|
|
1899
1358
|
}
|
|
1900
|
-
const crossTop = sortedCross[ci].y + sortedCross[ci].h;
|
|
1901
|
-
if (crossTop >= groupTop(groups[gi])) result.push([sortedCross[ci++]]);
|
|
1902
|
-
else result.push(groups[gi++]);
|
|
1903
1359
|
}
|
|
1904
1360
|
return result;
|
|
1905
1361
|
}
|
|
1906
|
-
function
|
|
1907
|
-
|
|
1908
|
-
const sorted = [...items].sort((a, b) => b.y - a.y);
|
|
1909
|
-
let largestGap = 0;
|
|
1910
|
-
let position = 0;
|
|
1911
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
1912
|
-
const prevBottom = sorted[i - 1].y - sorted[i - 1].h;
|
|
1913
|
-
const currTop = sorted[i].y;
|
|
1914
|
-
const gap = prevBottom - currTop;
|
|
1915
|
-
if (gap > largestGap) {
|
|
1916
|
-
largestGap = gap;
|
|
1917
|
-
position = (prevBottom + currTop) / 2;
|
|
1918
|
-
}
|
|
1919
|
-
}
|
|
1920
|
-
return { position, gap: largestGap };
|
|
1362
|
+
function isFragmentRow(row) {
|
|
1363
|
+
return row.items.length <= 3 && row.items.every((i) => i.text.length <= 8);
|
|
1921
1364
|
}
|
|
1922
|
-
function
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
if (i.x < minX) minX = i.x;
|
|
1929
|
-
const r = i.x + i.w;
|
|
1930
|
-
if (r > maxX) maxX = r;
|
|
1931
|
-
}
|
|
1932
|
-
const narrowThreshold = (maxX - minX) * NARROW_ELEMENT_WIDTH_RATIO;
|
|
1933
|
-
const filtered = items.filter((i) => i.w >= narrowThreshold);
|
|
1934
|
-
if (filtered.length >= 2 && filtered.length < items.length && filtered.length >= items.length * 0.7) {
|
|
1935
|
-
const filteredCut = findVerticalCut(filtered);
|
|
1936
|
-
if (filteredCut.gap > edgeCut.gap && filteredCut.gap >= minGap) {
|
|
1937
|
-
return filteredCut;
|
|
1938
|
-
}
|
|
1939
|
-
}
|
|
1365
|
+
function rowBand(row) {
|
|
1366
|
+
let bottom = Infinity, top = -Infinity;
|
|
1367
|
+
for (const i of row.items) {
|
|
1368
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1369
|
+
if (i.y < bottom) bottom = i.y;
|
|
1370
|
+
if (i.y + h > top) top = i.y + h;
|
|
1940
1371
|
}
|
|
1941
|
-
return
|
|
1372
|
+
return { bottom, top, height: top - bottom };
|
|
1942
1373
|
}
|
|
1943
|
-
function
|
|
1944
|
-
if (
|
|
1945
|
-
const
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
let
|
|
1949
|
-
|
|
1950
|
-
const
|
|
1951
|
-
const
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
}
|
|
1374
|
+
function mergeMultiLineRows(rows, columns) {
|
|
1375
|
+
if (rows.length <= 1) return rows;
|
|
1376
|
+
const result = [rows[0]];
|
|
1377
|
+
const allFontSizes = rows.flatMap((r) => r.items).map((i) => i.fontSize);
|
|
1378
|
+
const avgFontSize = allFontSizes.length > 0 ? allFontSizes.reduce((s, v) => s + v, 0) / allFontSizes.length : 12;
|
|
1379
|
+
for (let i = 1; i < rows.length; i++) {
|
|
1380
|
+
const prev = result[result.length - 1];
|
|
1381
|
+
const curr = rows[i];
|
|
1382
|
+
const yGap = Math.abs(prev.y - curr.y);
|
|
1383
|
+
const matchedCols = countMatchedColumns(curr, columns);
|
|
1384
|
+
if (yGap < avgFontSize * 1.8 && curr.items.length <= 2 && (matchedCols < MIN_COLS || curr.items.length === 1)) {
|
|
1385
|
+
result[result.length - 1] = {
|
|
1386
|
+
y: prev.y,
|
|
1387
|
+
items: [...prev.items, ...curr.items]
|
|
1388
|
+
};
|
|
1389
|
+
} else {
|
|
1390
|
+
result.push(curr);
|
|
1958
1391
|
}
|
|
1959
|
-
prevRight = prevRight === null ? right : Math.max(prevRight, right);
|
|
1960
1392
|
}
|
|
1961
|
-
return
|
|
1393
|
+
return result;
|
|
1962
1394
|
}
|
|
1963
|
-
function
|
|
1395
|
+
function groupByBaseline(items) {
|
|
1964
1396
|
if (items.length === 0) return [];
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1397
|
+
const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1398
|
+
const rows = [];
|
|
1399
|
+
let curItems = [sorted[0]];
|
|
1400
|
+
let curY = sorted[0].y;
|
|
1401
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1402
|
+
if (Math.abs(sorted[i].y - curY) <= Y_TOL) {
|
|
1403
|
+
curItems.push(sorted[i]);
|
|
1404
|
+
} else {
|
|
1405
|
+
rows.push({ y: curY, items: curItems });
|
|
1406
|
+
curItems = [sorted[i]];
|
|
1407
|
+
curY = sorted[i].y;
|
|
1408
|
+
}
|
|
1973
1409
|
}
|
|
1974
|
-
|
|
1410
|
+
if (curItems.length > 0) rows.push({ y: curY, items: curItems });
|
|
1411
|
+
return rows;
|
|
1975
1412
|
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
if (
|
|
1985
|
-
const matches = [];
|
|
1986
|
-
for (const item of items) {
|
|
1987
|
-
const h = item.h > 0 ? item.h : item.fontSize;
|
|
1988
|
-
if (h <= 0 || item.w <= 0) continue;
|
|
1989
|
-
if (line.lineWidth > h * STRIKE_MAX_THICKNESS_RATIO) continue;
|
|
1990
|
-
const centerY = item.y + h * 0.4;
|
|
1991
|
-
if (Math.abs(line.y1 - centerY) > h * STRIKE_CENTER_TOLERANCE) continue;
|
|
1992
|
-
const overlap = Math.min(line.x2, item.x + item.w) - Math.max(line.x1, item.x);
|
|
1993
|
-
if (overlap / item.w < STRIKE_MIN_OVERLAP_RATIO) continue;
|
|
1994
|
-
matches.push(item);
|
|
1995
|
-
}
|
|
1996
|
-
if (matches.length === 0) continue;
|
|
1997
|
-
let totalW = 0;
|
|
1998
|
-
for (const m of matches) totalW += m.w;
|
|
1999
|
-
if (totalW <= 0 || (line.x2 - line.x1) / totalW > STRIKE_MAX_LINE_TO_TEXT_RATIO) continue;
|
|
2000
|
-
for (const m of matches) m.strike = true;
|
|
1413
|
+
function hasSuspiciousGaps(row) {
|
|
1414
|
+
if (row.items.length < 2) return false;
|
|
1415
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1416
|
+
if (sorted.length === 2 && sorted[1].text.length > 20) return false;
|
|
1417
|
+
const avgFontSize = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
|
|
1418
|
+
const minGap = Math.max(avgFontSize * MIN_GAP_FACTOR, MIN_GAP_ABSOLUTE);
|
|
1419
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1420
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1421
|
+
if (gap >= minGap) return true;
|
|
2001
1422
|
}
|
|
1423
|
+
return false;
|
|
2002
1424
|
}
|
|
2003
|
-
function
|
|
2004
|
-
const
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
for (const item of struck) {
|
|
2008
|
-
const key = Math.round(item.y / 3);
|
|
2009
|
-
const arr = lines.get(key) || [];
|
|
2010
|
-
arr.push(item);
|
|
2011
|
-
lines.set(key, arr);
|
|
1425
|
+
function extractColumnClusters(rows) {
|
|
1426
|
+
const allX = [];
|
|
1427
|
+
for (const row of rows) {
|
|
1428
|
+
for (const item of row.items) allX.push(item.x);
|
|
2012
1429
|
}
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
1430
|
+
if (allX.length === 0) return [];
|
|
1431
|
+
allX.sort((a, b) => a - b);
|
|
1432
|
+
const clusters = [];
|
|
1433
|
+
let clusterStart = 0;
|
|
1434
|
+
for (let i = 1; i <= allX.length; i++) {
|
|
1435
|
+
if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
|
|
1436
|
+
const slice = allX.slice(clusterStart, i);
|
|
1437
|
+
const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
|
|
1438
|
+
clusters.push({ x: avg, count: slice.length });
|
|
1439
|
+
clusterStart = i;
|
|
1440
|
+
}
|
|
2017
1441
|
}
|
|
1442
|
+
const minCount = Math.max(2, Math.floor(rows.length * MIN_COL_FILL_RATIO));
|
|
1443
|
+
return clusters.filter((c) => c.count >= minCount).sort((a, b) => a.x - b.x);
|
|
2018
1444
|
}
|
|
2019
|
-
function
|
|
2020
|
-
const
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
for (const
|
|
2024
|
-
const
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
const cells = extractCells(grid, horizontals, verticals);
|
|
2040
|
-
if (cells.length === 0) continue;
|
|
2041
|
-
const textItems = tableItems.map((i) => ({
|
|
2042
|
-
text: i.text,
|
|
2043
|
-
x: i.x,
|
|
2044
|
-
y: i.y,
|
|
2045
|
-
w: i.w,
|
|
2046
|
-
h: i.h,
|
|
2047
|
-
fontSize: i.fontSize,
|
|
2048
|
-
fontName: i.fontName,
|
|
2049
|
-
hasSpaceBefore: i.hasSpaceBefore
|
|
2050
|
-
}));
|
|
2051
|
-
const cellTextMap = mapTextToCells(textItems, cells);
|
|
2052
|
-
const numRows = grid.rowYs.length - 1;
|
|
2053
|
-
const numCols = grid.colXs.length - 1;
|
|
2054
|
-
const irGrid = Array.from(
|
|
2055
|
-
{ length: numRows },
|
|
2056
|
-
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
2057
|
-
);
|
|
2058
|
-
for (const cell of cells) {
|
|
2059
|
-
const cellItems = cellTextMap.get(cell) || [];
|
|
2060
|
-
let text = cellTextToString(cellItems);
|
|
2061
|
-
text = text.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2062
|
-
text = text.split("\n").map((line) => collapseEvenSpacing(line)).join("\n");
|
|
2063
|
-
irGrid[cell.row][cell.col] = {
|
|
2064
|
-
text,
|
|
2065
|
-
colSpan: cell.colSpan,
|
|
2066
|
-
rowSpan: cell.rowSpan
|
|
2067
|
-
};
|
|
2068
|
-
}
|
|
2069
|
-
let finalGrid = irGrid;
|
|
2070
|
-
let finalRows = numRows;
|
|
2071
|
-
if (numRows <= 2 && numCols >= 3) {
|
|
2072
|
-
const rebuilt = normalizeUndersegmentedTable(irGrid, grid.colXs, textItems);
|
|
2073
|
-
if (rebuilt) {
|
|
2074
|
-
finalGrid = rebuilt.map((row) => row.map((rawText) => {
|
|
2075
|
-
const cleaned = rawText.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2076
|
-
return {
|
|
2077
|
-
text: cleaned.split("\n").map((line) => collapseEvenSpacing(line)).join("\n"),
|
|
2078
|
-
colSpan: 1,
|
|
2079
|
-
rowSpan: 1
|
|
2080
|
-
};
|
|
2081
|
-
}));
|
|
2082
|
-
finalRows = finalGrid.length;
|
|
1445
|
+
function findTableRegionsByHeader(allRows, columns, headerItems) {
|
|
1446
|
+
const regions = [];
|
|
1447
|
+
let currentRegion = [];
|
|
1448
|
+
let missStreak = 0;
|
|
1449
|
+
for (const row of allRows) {
|
|
1450
|
+
const matchedCols = countMatchedColumnsRange(row, columns, headerItems);
|
|
1451
|
+
if (matchedCols >= MIN_COLS) {
|
|
1452
|
+
currentRegion.push(row);
|
|
1453
|
+
missStreak = 0;
|
|
1454
|
+
} else if (currentRegion.length > 0 && (row.items.length <= 2 || missStreak === 0)) {
|
|
1455
|
+
currentRegion.push(row);
|
|
1456
|
+
missStreak++;
|
|
1457
|
+
} else {
|
|
1458
|
+
while (currentRegion.length > 0) {
|
|
1459
|
+
const last = currentRegion[currentRegion.length - 1];
|
|
1460
|
+
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1461
|
+
currentRegion.pop();
|
|
1462
|
+
}
|
|
1463
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1464
|
+
regions.push({ rows: [...currentRegion] });
|
|
2083
1465
|
}
|
|
1466
|
+
currentRegion = [];
|
|
1467
|
+
missStreak = 0;
|
|
2084
1468
|
}
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
1469
|
+
}
|
|
1470
|
+
while (currentRegion.length > 0) {
|
|
1471
|
+
const last = currentRegion[currentRegion.length - 1];
|
|
1472
|
+
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1473
|
+
currentRegion.pop();
|
|
1474
|
+
}
|
|
1475
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1476
|
+
regions.push({ rows: currentRegion });
|
|
1477
|
+
}
|
|
1478
|
+
return regions;
|
|
1479
|
+
}
|
|
1480
|
+
function findTableRegions(allRows, columns) {
|
|
1481
|
+
const regions = [];
|
|
1482
|
+
let currentRegion = [];
|
|
1483
|
+
for (const row of allRows) {
|
|
1484
|
+
const matchedCols = countMatchedColumns(row, columns);
|
|
1485
|
+
if (matchedCols >= MIN_COLS) {
|
|
1486
|
+
currentRegion.push(row);
|
|
1487
|
+
} else if (row.items.length === 1) {
|
|
1488
|
+
if (currentRegion.length > 0) {
|
|
1489
|
+
currentRegion.push(row);
|
|
2105
1490
|
}
|
|
2106
|
-
|
|
1491
|
+
} else {
|
|
1492
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1493
|
+
regions.push({ rows: [...currentRegion] });
|
|
1494
|
+
}
|
|
1495
|
+
currentRegion = [];
|
|
2107
1496
|
}
|
|
2108
|
-
blocks.push({ type: "table", table: irTable, pageNumber: pageNum, bbox: tableBbox });
|
|
2109
1497
|
}
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
}));
|
|
2123
|
-
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2124
|
-
if (clusterResults.length > 0) {
|
|
2125
|
-
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2126
|
-
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2127
|
-
const usedClusterIndices = /* @__PURE__ */ new Set();
|
|
2128
|
-
for (const cr of clusterResults) {
|
|
2129
|
-
for (const ci of cr.usedItems) {
|
|
2130
|
-
const idx = ciToIdx.get(ci);
|
|
2131
|
-
if (idx !== void 0) usedClusterIndices.add(idx);
|
|
2132
|
-
}
|
|
2133
|
-
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
1498
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1499
|
+
regions.push({ rows: currentRegion });
|
|
1500
|
+
}
|
|
1501
|
+
return regions;
|
|
1502
|
+
}
|
|
1503
|
+
function countMatchedColumns(row, columns) {
|
|
1504
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1505
|
+
for (const item of row.items) {
|
|
1506
|
+
for (let ci = 0; ci < columns.length; ci++) {
|
|
1507
|
+
if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
|
|
1508
|
+
matched.add(ci);
|
|
1509
|
+
break;
|
|
2134
1510
|
}
|
|
2135
|
-
remaining = remaining.filter((_, idx) => !usedClusterIndices.has(idx));
|
|
2136
1511
|
}
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
1512
|
+
}
|
|
1513
|
+
return matched.size;
|
|
1514
|
+
}
|
|
1515
|
+
function countMatchedColumnsRange(row, columns, headerItems) {
|
|
1516
|
+
const boundaries = [];
|
|
1517
|
+
for (let ci = 0; ci < headerItems.length; ci++) {
|
|
1518
|
+
const left = ci === 0 ? 0 : (headerItems[ci - 1].x + headerItems[ci - 1].w + headerItems[ci].x) / 2;
|
|
1519
|
+
const right = ci === headerItems.length - 1 ? Infinity : (headerItems[ci].x + headerItems[ci].w + headerItems[ci + 1].x) / 2;
|
|
1520
|
+
boundaries.push({ left, right });
|
|
1521
|
+
}
|
|
1522
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1523
|
+
for (const item of row.items) {
|
|
1524
|
+
for (let ci = 0; ci < boundaries.length; ci++) {
|
|
1525
|
+
if (item.x >= boundaries[ci].left && item.x < boundaries[ci].right) {
|
|
1526
|
+
matched.add(ci);
|
|
1527
|
+
break;
|
|
2146
1528
|
}
|
|
2147
|
-
const finalTextBlocks = detectListBlocks(textBlocks);
|
|
2148
|
-
for (const b of finalTextBlocks) blocks.push(b);
|
|
2149
1529
|
}
|
|
2150
|
-
blocks.sort((a, b) => {
|
|
2151
|
-
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2152
|
-
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2153
|
-
return by - ay;
|
|
2154
|
-
});
|
|
2155
|
-
return mergeAdjacentTableBlocks(blocks);
|
|
2156
1530
|
}
|
|
2157
|
-
return
|
|
1531
|
+
return matched.size;
|
|
2158
1532
|
}
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
1533
|
+
function assignRowItems(items, columns, numCols) {
|
|
1534
|
+
if (items.length === 0) return [];
|
|
1535
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1536
|
+
const colCenters = columns.map((c) => c.x);
|
|
1537
|
+
const gaps = [];
|
|
1538
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1539
|
+
gaps.push({ idx: i, size: sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w) });
|
|
1540
|
+
}
|
|
1541
|
+
const gapSizes = gaps.map((g2) => g2.size).sort((a, b) => a - b);
|
|
1542
|
+
const medianGap = gapSizes.length > 0 ? gapSizes[Math.floor(gapSizes.length / 2)] : 0;
|
|
1543
|
+
const gapThreshold = sorted.length <= numCols + 1 ? 12 : Math.max(medianGap * 2.5, 12);
|
|
1544
|
+
const significantGaps = gaps.filter((g2) => g2.size >= gapThreshold).sort((a, b) => b.size - a.size).slice(0, numCols - 1).sort((a, b) => a.idx - b.idx);
|
|
1545
|
+
const groups = [];
|
|
1546
|
+
let start = 0;
|
|
1547
|
+
for (const gap of significantGaps) {
|
|
1548
|
+
groups.push(sorted.slice(start, gap.idx));
|
|
1549
|
+
start = gap.idx;
|
|
1550
|
+
}
|
|
1551
|
+
groups.push(sorted.slice(start));
|
|
1552
|
+
const result = [];
|
|
1553
|
+
const usedCols = /* @__PURE__ */ new Set();
|
|
1554
|
+
const groupCenters = groups.map((g2) => {
|
|
1555
|
+
let minX = Infinity, maxX = -Infinity;
|
|
1556
|
+
for (const i of g2) {
|
|
1557
|
+
if (i.x < minX) minX = i.x;
|
|
1558
|
+
const r = i.x + i.w;
|
|
1559
|
+
if (r > maxX) maxX = r;
|
|
2175
1560
|
}
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
1561
|
+
return (minX + maxX) / 2;
|
|
1562
|
+
});
|
|
1563
|
+
const assignments = [];
|
|
1564
|
+
for (let gi = 0; gi < groups.length; gi++) {
|
|
1565
|
+
for (let ci = 0; ci < numCols; ci++) {
|
|
1566
|
+
assignments.push({ gi, ci, dist: Math.abs(groupCenters[gi] - colCenters[ci]) });
|
|
2179
1567
|
}
|
|
2180
|
-
const merged = {
|
|
2181
|
-
rows: prev.table.rows + currCells.length,
|
|
2182
|
-
cols: prev.table.cols,
|
|
2183
|
-
cells: [...prev.table.cells, ...currCells],
|
|
2184
|
-
hasHeader: prev.table.hasHeader,
|
|
2185
|
-
caption: prev.table.caption
|
|
2186
|
-
};
|
|
2187
|
-
blocks[i] = { ...prev, table: merged };
|
|
2188
|
-
blocks.splice(i + 1, 1);
|
|
2189
1568
|
}
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
1569
|
+
assignments.sort((a, b) => a.dist - b.dist);
|
|
1570
|
+
const assignedGroups = /* @__PURE__ */ new Set();
|
|
1571
|
+
for (const { gi, ci } of assignments) {
|
|
1572
|
+
if (assignedGroups.has(gi) || usedCols.has(ci)) continue;
|
|
1573
|
+
result.push({ col: ci, items: groups[gi] });
|
|
1574
|
+
assignedGroups.add(gi);
|
|
1575
|
+
usedCols.add(ci);
|
|
2196
1576
|
}
|
|
2197
|
-
|
|
1577
|
+
for (let gi = 0; gi < groups.length; gi++) {
|
|
1578
|
+
if (assignedGroups.has(gi)) continue;
|
|
1579
|
+
let bestCol = 0, bestDist = Infinity;
|
|
1580
|
+
for (let ci = 0; ci < numCols; ci++) {
|
|
1581
|
+
const d = Math.abs(groupCenters[gi] - colCenters[ci]);
|
|
1582
|
+
if (d < bestDist) {
|
|
1583
|
+
bestDist = d;
|
|
1584
|
+
bestCol = ci;
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
result.push({ col: bestCol, items: groups[gi] });
|
|
1588
|
+
}
|
|
1589
|
+
return result;
|
|
2198
1590
|
}
|
|
2199
|
-
function
|
|
2200
|
-
|
|
2201
|
-
const
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
};
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
1591
|
+
function buildClusterTable(rows, columns, pageNum) {
|
|
1592
|
+
const numCols = columns.length;
|
|
1593
|
+
const numRows = rows.length;
|
|
1594
|
+
if (numRows < MIN_ROWS || numCols < MIN_COLS) return null;
|
|
1595
|
+
const cells = Array.from(
|
|
1596
|
+
{ length: numRows },
|
|
1597
|
+
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
1598
|
+
);
|
|
1599
|
+
const usedItems = /* @__PURE__ */ new Set();
|
|
1600
|
+
for (let r = 0; r < numRows; r++) {
|
|
1601
|
+
const row = rows[r];
|
|
1602
|
+
if (row.items.length === 1 && numCols > 1) {
|
|
1603
|
+
cells[r][0] = { text: row.items[0].text, colSpan: numCols, rowSpan: 1 };
|
|
1604
|
+
usedItems.add(row.items[0]);
|
|
1605
|
+
continue;
|
|
1606
|
+
}
|
|
1607
|
+
const assignments = assignRowItems(row.items, columns, numCols);
|
|
1608
|
+
for (const { col, items } of assignments) {
|
|
1609
|
+
const text = items.map((i) => i.text).join(" ");
|
|
1610
|
+
const existing = cells[r][col].text;
|
|
1611
|
+
cells[r][col].text = existing ? existing + " " + text : text;
|
|
1612
|
+
for (const item of items) usedItems.add(item);
|
|
2215
1613
|
}
|
|
2216
1614
|
}
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
}
|
|
2242
|
-
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2243
|
-
}
|
|
2244
|
-
const remaining = items.filter((_, idx) => !usedIndices.has(idx));
|
|
2245
|
-
if (remaining.length > 0) {
|
|
2246
|
-
const yLines = mergeSuperscriptLines(groupByY(remaining));
|
|
2247
|
-
for (const line of yLines) {
|
|
2248
|
-
const text = mergeLineSimple(line);
|
|
2249
|
-
if (!text.trim()) continue;
|
|
2250
|
-
const bbox = computeBBox(line, pageNum);
|
|
2251
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
1615
|
+
let emptyRows = 0;
|
|
1616
|
+
for (const row of cells) {
|
|
1617
|
+
if (row.every((c) => c.text === "")) emptyRows++;
|
|
1618
|
+
}
|
|
1619
|
+
if (emptyRows > numRows * 0.5) return null;
|
|
1620
|
+
for (let c = 0; c < numCols; c++) {
|
|
1621
|
+
const hasValue = cells.some((row) => row[c].text !== "");
|
|
1622
|
+
if (!hasValue) return null;
|
|
1623
|
+
}
|
|
1624
|
+
for (let r = numRows - 1; r >= 1; r--) {
|
|
1625
|
+
const nonEmptyCols = cells[r].filter((c) => c.text.trim()).length;
|
|
1626
|
+
if (nonEmptyCols !== 1) continue;
|
|
1627
|
+
if (cells[r][0].text.trim() !== "") continue;
|
|
1628
|
+
const contentText = cells[r].find((c) => c.text.trim())?.text.trim() || "";
|
|
1629
|
+
if (/^[○●▶\-·]/.test(contentText)) continue;
|
|
1630
|
+
for (let pr = r - 1; pr >= 0; pr--) {
|
|
1631
|
+
if (cells[pr].some((c) => c.text.trim())) {
|
|
1632
|
+
for (let c = 0; c < numCols; c++) {
|
|
1633
|
+
const prev = cells[pr][c].text.trim();
|
|
1634
|
+
const curr = cells[r][c].text.trim();
|
|
1635
|
+
if (curr) cells[pr][c].text = prev ? prev + " " + curr : curr;
|
|
1636
|
+
}
|
|
1637
|
+
for (let c = 0; c < numCols; c++) cells[r][c].text = "";
|
|
1638
|
+
break;
|
|
2252
1639
|
}
|
|
2253
1640
|
}
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
} else {
|
|
2267
|
-
const allY = items.map((i) => i.y);
|
|
2268
|
-
const pageHeight = safeMax(allY) - safeMin(allY);
|
|
2269
|
-
const gapThreshold = Math.max(15, pageHeight * 0.03);
|
|
2270
|
-
const orderedGroups = xyCutOrder(items, gapThreshold);
|
|
2271
|
-
for (const group of orderedGroups) {
|
|
2272
|
-
if (group.length === 0) continue;
|
|
2273
|
-
const yLines = mergeSuperscriptLines(groupByY(group));
|
|
2274
|
-
const groupColumns = detectColumns(yLines);
|
|
2275
|
-
if (groupColumns && groupColumns.length >= 3) {
|
|
2276
|
-
const tableText = extractWithColumns(yLines, groupColumns);
|
|
2277
|
-
const bbox = computeBBox(group, pageNum);
|
|
2278
|
-
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
|
|
2279
|
-
} else {
|
|
2280
|
-
for (const line of yLines) {
|
|
2281
|
-
const text = mergeLineSimple(line);
|
|
2282
|
-
if (!text.trim()) continue;
|
|
2283
|
-
const bbox = computeBBox(line, pageNum);
|
|
2284
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2285
|
-
}
|
|
1641
|
+
}
|
|
1642
|
+
for (let r = 0; r < cells.length - 1; r++) {
|
|
1643
|
+
const row = cells[r];
|
|
1644
|
+
const hasCol0 = row[0].text.trim() !== "";
|
|
1645
|
+
const hasColLast = numCols > 1 && row[numCols - 1].text.trim() !== "";
|
|
1646
|
+
const midEmpty = row.slice(1, numCols - 1).every((c) => c.text.trim() === "");
|
|
1647
|
+
if (hasCol0 && hasColLast && midEmpty) {
|
|
1648
|
+
const next = cells[r + 1];
|
|
1649
|
+
if (next[0].text.trim() === "" && next.some((c) => c.text.trim())) {
|
|
1650
|
+
for (let c = 1; c < numCols; c++) {
|
|
1651
|
+
const curr = next[c].text.trim();
|
|
1652
|
+
if (curr) row[c].text = row[c].text.trim() ? row[c].text.trim() + " " + curr : curr;
|
|
2286
1653
|
}
|
|
1654
|
+
for (let c = 0; c < numCols; c++) next[c].text = "";
|
|
2287
1655
|
}
|
|
2288
1656
|
}
|
|
2289
1657
|
}
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
1658
|
+
const filteredCells = cells.filter((row) => row.some((c) => c.text.trim()));
|
|
1659
|
+
const finalRowCount = filteredCells.length;
|
|
1660
|
+
if (finalRowCount < MIN_ROWS) return null;
|
|
1661
|
+
const irTable = {
|
|
1662
|
+
rows: finalRowCount,
|
|
1663
|
+
cols: numCols,
|
|
1664
|
+
cells: filteredCells,
|
|
1665
|
+
hasHeader: finalRowCount > 1
|
|
1666
|
+
};
|
|
1667
|
+
const allItems = rows.flatMap((r) => r.items);
|
|
2293
1668
|
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
2294
|
-
for (const i of
|
|
1669
|
+
for (const i of allItems) {
|
|
2295
1670
|
if (i.x < minX) minX = i.x;
|
|
2296
1671
|
if (i.y < minY) minY = i.y;
|
|
2297
1672
|
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
2298
|
-
const
|
|
2299
|
-
if (i.y +
|
|
1673
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1674
|
+
if (i.y + h > maxY) maxY = i.y + h;
|
|
2300
1675
|
}
|
|
2301
|
-
return {
|
|
1676
|
+
return {
|
|
1677
|
+
table: irTable,
|
|
1678
|
+
bbox: { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY },
|
|
1679
|
+
usedItems
|
|
1680
|
+
};
|
|
2302
1681
|
}
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
1682
|
+
|
|
1683
|
+
// src/pdf/xy-cut.ts
|
|
1684
|
+
var MAX_XYCUT_DEPTH = 50;
|
|
1685
|
+
var XYCUT_MIN_GAP = 5;
|
|
1686
|
+
var CROSS_LAYOUT_BETA = 2;
|
|
1687
|
+
var CROSS_OVERLAP_RATIO = 0.1;
|
|
1688
|
+
var CROSS_MIN_OVERLAPS = 2;
|
|
1689
|
+
var CROSS_MAX_MASK_RATIO = 0.2;
|
|
1690
|
+
var NARROW_ELEMENT_WIDTH_RATIO = 0.1;
|
|
1691
|
+
function xyCutOrder(items, gapThreshold, depth = 0) {
|
|
1692
|
+
if (items.length === 0) return [];
|
|
1693
|
+
if (items.length <= 2 || depth >= MAX_XYCUT_DEPTH) return [items];
|
|
1694
|
+
if (depth === 0 && items.length >= 3) {
|
|
1695
|
+
const cross = identifyCrossLayoutItems(items);
|
|
1696
|
+
if (cross.size > 0 && cross.size <= items.length * CROSS_MAX_MASK_RATIO) {
|
|
1697
|
+
const rest = items.filter((i) => !cross.has(i));
|
|
1698
|
+
if (rest.length > 0) {
|
|
1699
|
+
const groups = xyCutOrder(rest, gapThreshold, 1);
|
|
1700
|
+
return mergeCrossLayoutGroups(groups, [...cross]);
|
|
1701
|
+
}
|
|
2314
1702
|
}
|
|
2315
1703
|
}
|
|
2316
|
-
|
|
2317
|
-
const
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
1704
|
+
const minGap = Math.max(XYCUT_MIN_GAP, gapThreshold);
|
|
1705
|
+
const hCut = findHorizontalCut(items);
|
|
1706
|
+
const vCut = findVerticalCutWithOutlierFilter(items, minGap);
|
|
1707
|
+
const hValid = hCut.gap >= minGap;
|
|
1708
|
+
const vValid = vCut.gap >= minGap;
|
|
1709
|
+
let useHorizontal;
|
|
1710
|
+
if (hValid && vValid) useHorizontal = vCut.gap <= hCut.gap * 1.5;
|
|
1711
|
+
else if (hValid) useHorizontal = true;
|
|
1712
|
+
else if (vValid) useHorizontal = false;
|
|
1713
|
+
else return [items];
|
|
1714
|
+
if (useHorizontal) {
|
|
1715
|
+
const upper = items.filter((i) => i.y > hCut.position);
|
|
1716
|
+
const lower = items.filter((i) => i.y <= hCut.position);
|
|
1717
|
+
if (upper.length > 0 && lower.length > 0 && upper.length < items.length) {
|
|
1718
|
+
return [...xyCutOrder(upper, gapThreshold, depth + 1), ...xyCutOrder(lower, gapThreshold, depth + 1)];
|
|
2330
1719
|
}
|
|
2331
|
-
|
|
2332
|
-
const
|
|
2333
|
-
const
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
const isHidden = fontSize === 0 || i.width === 0 && i.str.trim().length > 0;
|
|
2337
|
-
let text = i.str.trim();
|
|
2338
|
-
if (/^[\d\s\-().·,☎]+$/.test(text) && /\d/.test(text) && / /.test(text)) {
|
|
2339
|
-
text = text.replace(/ /g, "");
|
|
1720
|
+
} else {
|
|
1721
|
+
const left = items.filter((i) => i.x + i.w / 2 < vCut.position);
|
|
1722
|
+
const right = items.filter((i) => i.x + i.w / 2 >= vCut.position);
|
|
1723
|
+
if (left.length > 0 && right.length > 0 && left.length < items.length) {
|
|
1724
|
+
return [...xyCutOrder(left, gapThreshold, depth + 1), ...xyCutOrder(right, gapThreshold, depth + 1)];
|
|
2340
1725
|
}
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
1726
|
+
}
|
|
1727
|
+
return [items];
|
|
1728
|
+
}
|
|
1729
|
+
function identifyCrossLayoutItems(items) {
|
|
1730
|
+
const cross = /* @__PURE__ */ new Set();
|
|
1731
|
+
if (items.length < 3) return cross;
|
|
1732
|
+
let maxWidth = 0;
|
|
1733
|
+
for (const i of items) {
|
|
1734
|
+
if (i.w > maxWidth) maxWidth = i.w;
|
|
1735
|
+
}
|
|
1736
|
+
const threshold = CROSS_LAYOUT_BETA * maxWidth;
|
|
1737
|
+
for (const item of items) {
|
|
1738
|
+
if (item.w < threshold) continue;
|
|
1739
|
+
let overlaps = 0;
|
|
1740
|
+
for (const other of items) {
|
|
1741
|
+
if (other === item) continue;
|
|
1742
|
+
const left = Math.max(item.x, other.x);
|
|
1743
|
+
const right = Math.min(item.x + item.w, other.x + other.w);
|
|
1744
|
+
const overlapW = right - left;
|
|
1745
|
+
if (overlapW <= 0) continue;
|
|
1746
|
+
const smaller = Math.min(item.w, other.w);
|
|
1747
|
+
if (smaller > 0 && overlapW / smaller >= CROSS_OVERLAP_RATIO) {
|
|
1748
|
+
overlaps++;
|
|
1749
|
+
if (overlaps >= CROSS_MIN_OVERLAPS) break;
|
|
2345
1750
|
}
|
|
2346
|
-
} else {
|
|
2347
|
-
items.push({ text, x, y, w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
2348
1751
|
}
|
|
1752
|
+
if (overlaps >= CROSS_MIN_OVERLAPS) cross.add(item);
|
|
2349
1753
|
}
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
1754
|
+
return cross;
|
|
1755
|
+
}
|
|
1756
|
+
function mergeCrossLayoutGroups(groups, cross) {
|
|
1757
|
+
if (cross.length === 0) return groups;
|
|
1758
|
+
const sortedCross = [...cross].sort((a, b) => b.y + b.h - (a.y + a.h) || a.x - b.x);
|
|
1759
|
+
const groupTop = (g2) => {
|
|
1760
|
+
let top = -Infinity;
|
|
1761
|
+
for (const i of g2) {
|
|
1762
|
+
const t = i.y + i.h;
|
|
1763
|
+
if (t > top) top = t;
|
|
1764
|
+
}
|
|
1765
|
+
return top;
|
|
1766
|
+
};
|
|
1767
|
+
const result = [];
|
|
1768
|
+
let gi = 0, ci = 0;
|
|
1769
|
+
while (gi < groups.length || ci < sortedCross.length) {
|
|
1770
|
+
if (ci >= sortedCross.length) {
|
|
1771
|
+
result.push(groups[gi++]);
|
|
1772
|
+
continue;
|
|
2361
1773
|
}
|
|
2362
|
-
if (
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
for (const sp of spacePositions) {
|
|
2366
|
-
let nearest = null;
|
|
2367
|
-
for (const item of deduped) {
|
|
2368
|
-
if (Math.abs(sp.y - item.y) > 3) continue;
|
|
2369
|
-
const dist = item.x - sp.x;
|
|
2370
|
-
if (dist >= -1 && dist <= 20 && (!nearest || item.x < nearest.x)) {
|
|
2371
|
-
nearest = item;
|
|
2372
|
-
}
|
|
2373
|
-
}
|
|
2374
|
-
if (nearest) nearest.hasSpaceBefore = true;
|
|
1774
|
+
if (gi >= groups.length) {
|
|
1775
|
+
result.push([sortedCross[ci++]]);
|
|
1776
|
+
continue;
|
|
2375
1777
|
}
|
|
1778
|
+
const crossTop = sortedCross[ci].y + sortedCross[ci].h;
|
|
1779
|
+
if (crossTop >= groupTop(groups[gi])) result.push([sortedCross[ci++]]);
|
|
1780
|
+
else result.push(groups[gi++]);
|
|
2376
1781
|
}
|
|
2377
|
-
return
|
|
2378
|
-
}
|
|
2379
|
-
function splitEvenSpacedItem(text, itemX, itemW, fontSize) {
|
|
2380
|
-
if (!/^[가-힣\d](?: [가-힣\d]){2,}$/.test(text)) return null;
|
|
2381
|
-
const chars = text.split(" ");
|
|
2382
|
-
if (chars.length < 3) return null;
|
|
2383
|
-
const charW = itemW / chars.length;
|
|
2384
|
-
if (charW > fontSize * 2) return null;
|
|
2385
|
-
return chars.map((ch, idx) => ({
|
|
2386
|
-
text: ch,
|
|
2387
|
-
x: Math.round(itemX + idx * charW),
|
|
2388
|
-
w: Math.round(charW * 0.8)
|
|
2389
|
-
// 실제 글자 폭은 간격보다 좁음
|
|
2390
|
-
}));
|
|
1782
|
+
return result;
|
|
2391
1783
|
}
|
|
2392
|
-
function
|
|
2393
|
-
if (items.length
|
|
2394
|
-
const
|
|
2395
|
-
let
|
|
2396
|
-
let
|
|
2397
|
-
for (let i = 1; i <
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
1784
|
+
function findHorizontalCut(items) {
|
|
1785
|
+
if (items.length < 2) return { position: 0, gap: 0 };
|
|
1786
|
+
const sorted = [...items].sort((a, b) => b.y - a.y);
|
|
1787
|
+
let largestGap = 0;
|
|
1788
|
+
let position = 0;
|
|
1789
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1790
|
+
const prevBottom = sorted[i - 1].y - sorted[i - 1].h;
|
|
1791
|
+
const currTop = sorted[i].y;
|
|
1792
|
+
const gap = prevBottom - currTop;
|
|
1793
|
+
if (gap > largestGap) {
|
|
1794
|
+
largestGap = gap;
|
|
1795
|
+
position = (prevBottom + currTop) / 2;
|
|
2402
1796
|
}
|
|
2403
|
-
curLine.push(items[i]);
|
|
2404
1797
|
}
|
|
2405
|
-
|
|
2406
|
-
return lines;
|
|
1798
|
+
return { position, gap: largestGap };
|
|
2407
1799
|
}
|
|
2408
|
-
function
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
if (i.
|
|
2415
|
-
|
|
1800
|
+
function findVerticalCutWithOutlierFilter(items, minGap) {
|
|
1801
|
+
const edgeCut = findVerticalCut(items);
|
|
1802
|
+
if (edgeCut.gap >= minGap) return edgeCut;
|
|
1803
|
+
if (items.length >= 3) {
|
|
1804
|
+
let minX = Infinity, maxX = -Infinity;
|
|
1805
|
+
for (const i of items) {
|
|
1806
|
+
if (i.x < minX) minX = i.x;
|
|
1807
|
+
const r = i.x + i.w;
|
|
1808
|
+
if (r > maxX) maxX = r;
|
|
2416
1809
|
}
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
const a = band(prev);
|
|
2425
|
-
const b = band(curr);
|
|
2426
|
-
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
2427
|
-
const prevIsFrag = isFrag(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
2428
|
-
const currIsFrag = isFrag(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
2429
|
-
if (prevIsFrag || currIsFrag) {
|
|
2430
|
-
result[result.length - 1] = [...prev, ...curr];
|
|
2431
|
-
} else {
|
|
2432
|
-
result.push(curr);
|
|
1810
|
+
const narrowThreshold = (maxX - minX) * NARROW_ELEMENT_WIDTH_RATIO;
|
|
1811
|
+
const filtered = items.filter((i) => i.w >= narrowThreshold);
|
|
1812
|
+
if (filtered.length >= 2 && filtered.length < items.length && filtered.length >= items.length * 0.7) {
|
|
1813
|
+
const filteredCut = findVerticalCut(filtered);
|
|
1814
|
+
if (filteredCut.gap > edgeCut.gap && filteredCut.gap >= minGap) {
|
|
1815
|
+
return filteredCut;
|
|
1816
|
+
}
|
|
2433
1817
|
}
|
|
2434
1818
|
}
|
|
2435
|
-
return
|
|
1819
|
+
return edgeCut;
|
|
1820
|
+
}
|
|
1821
|
+
function findVerticalCut(items) {
|
|
1822
|
+
if (items.length < 2) return { position: 0, gap: 0 };
|
|
1823
|
+
const sorted = [...items].sort((a, b) => a.x - b.x || a.x + a.w - (b.x + b.w));
|
|
1824
|
+
let largestGap = 0;
|
|
1825
|
+
let position = 0;
|
|
1826
|
+
let prevRight = null;
|
|
1827
|
+
for (const it of sorted) {
|
|
1828
|
+
const left = it.x;
|
|
1829
|
+
const right = it.x + it.w;
|
|
1830
|
+
if (prevRight !== null && left > prevRight) {
|
|
1831
|
+
const gap = left - prevRight;
|
|
1832
|
+
if (gap > largestGap) {
|
|
1833
|
+
largestGap = gap;
|
|
1834
|
+
position = (prevRight + left) / 2;
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
prevRight = prevRight === null ? right : Math.max(prevRight, right);
|
|
1838
|
+
}
|
|
1839
|
+
return { position, gap: largestGap };
|
|
2436
1840
|
}
|
|
1841
|
+
|
|
1842
|
+
// src/pdf/columns.ts
|
|
2437
1843
|
function isProseSpread(items) {
|
|
2438
1844
|
if (items.length < 4) return false;
|
|
2439
1845
|
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
@@ -2647,66 +2053,91 @@ function buildGridTable(lines, columns) {
|
|
|
2647
2053
|
}
|
|
2648
2054
|
return md.join("\n");
|
|
2649
2055
|
}
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
let
|
|
2655
|
-
for (
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
continue;
|
|
2663
|
-
}
|
|
2664
|
-
if (isEvenSpaced[i]) {
|
|
2665
|
-
result += sorted[i].text;
|
|
2666
|
-
continue;
|
|
2667
|
-
}
|
|
2668
|
-
if (sorted[i].hasSpaceBefore && gap >= avgFs * 0.05) {
|
|
2669
|
-
result += " ";
|
|
2670
|
-
result += sorted[i].text;
|
|
2671
|
-
continue;
|
|
2672
|
-
}
|
|
2673
|
-
if (/[□■○●▶◆◇ㅇ]$/.test(sorted[i - 1].text) && /^[가-힣]/.test(sorted[i].text) && gap > 1) {
|
|
2674
|
-
result += " ";
|
|
2675
|
-
result += sorted[i].text;
|
|
2676
|
-
continue;
|
|
2677
|
-
}
|
|
2678
|
-
if (gap > spaceGapThreshold(avgFs)) result += " ";
|
|
2679
|
-
result += sorted[i].text;
|
|
2056
|
+
|
|
2057
|
+
// src/pdf/block-detect.ts
|
|
2058
|
+
function computeMedianFontSizeFromFreq(freq) {
|
|
2059
|
+
if (freq.size === 0) return 0;
|
|
2060
|
+
let total = 0;
|
|
2061
|
+
for (const count of freq.values()) total += count;
|
|
2062
|
+
const sorted = [...freq.entries()].sort((a, b) => a[0] - b[0]);
|
|
2063
|
+
const mid = Math.floor(total / 2);
|
|
2064
|
+
let cumulative = 0;
|
|
2065
|
+
for (const [size, count] of sorted) {
|
|
2066
|
+
cumulative += count;
|
|
2067
|
+
if (cumulative > mid) return size;
|
|
2680
2068
|
}
|
|
2681
|
-
return
|
|
2069
|
+
return sorted[sorted.length - 1][0];
|
|
2682
2070
|
}
|
|
2683
|
-
function
|
|
2684
|
-
for (const
|
|
2685
|
-
if (
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2071
|
+
function detectHeadings(blocks, medianFontSize) {
|
|
2072
|
+
for (const block of blocks) {
|
|
2073
|
+
if (block.type !== "paragraph" || !block.text || !block.style?.fontSize) continue;
|
|
2074
|
+
const text = block.text.trim();
|
|
2075
|
+
if (text.length === 0 || text.length > 200) continue;
|
|
2076
|
+
if (/^\d+$/.test(text)) continue;
|
|
2077
|
+
const ratio = block.style.fontSize / medianFontSize;
|
|
2078
|
+
let level = 0;
|
|
2079
|
+
if (ratio >= HEADING_RATIO_H1) level = 1;
|
|
2080
|
+
else if (ratio >= HEADING_RATIO_H2) level = 2;
|
|
2081
|
+
else if (ratio >= HEADING_RATIO_H3) level = 3;
|
|
2082
|
+
if (level > 0) {
|
|
2083
|
+
block.type = "heading";
|
|
2084
|
+
block.level = level;
|
|
2085
|
+
block.text = collapseEvenSpacing(text);
|
|
2692
2086
|
}
|
|
2693
|
-
if (b.children) sanitizeBlockControlChars(b.children);
|
|
2694
2087
|
}
|
|
2695
2088
|
}
|
|
2696
|
-
function
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2089
|
+
function shouldDemoteTable(table) {
|
|
2090
|
+
const allCells = table.cells.flatMap((row) => row.map((c) => c.text.trim())).filter(Boolean);
|
|
2091
|
+
const allText = allCells.join(" ");
|
|
2092
|
+
if (table.rows <= 3 && table.cols <= 3) {
|
|
2093
|
+
const totalCells2 = table.rows * table.cols;
|
|
2094
|
+
const emptyCells2 = totalCells2 - allCells.length;
|
|
2095
|
+
if (emptyCells2 >= totalCells2 * 0.3) return true;
|
|
2096
|
+
if (/[□■◆○●▶ㅇ]/.test(allText)) return true;
|
|
2097
|
+
if (/<[^>]+>/.test(allText)) return true;
|
|
2098
|
+
}
|
|
2099
|
+
if (allText.length > 200) return false;
|
|
2100
|
+
if (/[□■◆○●▶]/.test(allText) && table.rows <= 3) return true;
|
|
2101
|
+
const totalCells = table.rows * table.cols;
|
|
2102
|
+
const emptyCells = totalCells - allCells.length;
|
|
2103
|
+
if (table.rows <= 2 && emptyCells > totalCells * 0.5) return true;
|
|
2104
|
+
if (table.rows === 1 && !/\d{2,}/.test(allText)) return true;
|
|
2105
|
+
return false;
|
|
2703
2106
|
}
|
|
2704
|
-
function
|
|
2705
|
-
const
|
|
2706
|
-
|
|
2107
|
+
function demoteTableToText(table) {
|
|
2108
|
+
const lines = [];
|
|
2109
|
+
for (let r = 0; r < table.rows; r++) {
|
|
2110
|
+
const cells = table.cells[r].map((c) => c.text.trim()).filter(Boolean);
|
|
2111
|
+
if (cells.length === 0) continue;
|
|
2112
|
+
if (table.cols === 2 && cells.length === 2) {
|
|
2113
|
+
lines.push(`${cells[0]} : ${cells[1]}`);
|
|
2114
|
+
} else {
|
|
2115
|
+
lines.push(cells.join(" "));
|
|
2116
|
+
}
|
|
2117
|
+
}
|
|
2118
|
+
return lines.join("\n");
|
|
2707
2119
|
}
|
|
2708
|
-
function
|
|
2709
|
-
|
|
2120
|
+
function detectMarkerHeadings(blocks) {
|
|
2121
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2122
|
+
const block = blocks[i];
|
|
2123
|
+
if (block.type !== "paragraph" || !block.text) continue;
|
|
2124
|
+
const text = block.text.trim();
|
|
2125
|
+
if (text.length < 50 && /^[□■◆◇▶]\s*[가-힣]/.test(text)) {
|
|
2126
|
+
block.type = "heading";
|
|
2127
|
+
block.level = 4;
|
|
2128
|
+
continue;
|
|
2129
|
+
}
|
|
2130
|
+
if (/^[가-힣]{2,6}$/.test(text) && block.style?.fontSize) {
|
|
2131
|
+
const prev = blocks[i - 1];
|
|
2132
|
+
const next = blocks[i + 1];
|
|
2133
|
+
const prevIsStructural = !prev || prev.type === "table" || prev.type === "heading" || prev.type === "separator";
|
|
2134
|
+
const nextIsStructural = !next || next.type === "table" || next.type === "heading" || next.type === "paragraph" && next.text && /^[□■◆○●]/.test(next.text.trim());
|
|
2135
|
+
if (prevIsStructural || nextIsStructural) {
|
|
2136
|
+
block.type = "heading";
|
|
2137
|
+
block.level = 3;
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
2140
|
+
}
|
|
2710
2141
|
}
|
|
2711
2142
|
var TABLE_CAPTION_RE = /^[<\[(【〈]?\s*(표|그림|도표|Table|Figure|Fig\.?)\s*[\d①-⑮][\d.\-]*\s*[\])】〉>]?[.:]?\s*/i;
|
|
2712
2143
|
var CAPTION_MAX_LENGTH = 100;
|
|
@@ -2743,250 +2174,611 @@ function detectTableCaptions(blocks) {
|
|
|
2743
2174
|
}
|
|
2744
2175
|
}
|
|
2745
2176
|
}
|
|
2746
|
-
var KOREAN_LIST_SEQ = "\uAC00\uB098\uB2E4\uB77C\uB9C8\uBC14\uC0AC\uC544\uC790\uCC28\uCE74\uD0C0\uD30C\uD558";
|
|
2747
|
-
function parseListLabel(text) {
|
|
2748
|
-
let m = text.match(/^(\d{1,2})\.(?!\d)\s+/);
|
|
2749
|
-
if (m) return { family: "arabicDot", ord: parseInt(m[1], 10) };
|
|
2750
|
-
m = text.match(/^([가-하])\.\s+/);
|
|
2751
|
-
if (m) {
|
|
2752
|
-
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2753
|
-
if (idx >= 0) return { family: "korDot", ord: idx + 1 };
|
|
2754
|
-
}
|
|
2755
|
-
m = text.match(/^(\d{1,2})\)\s*/);
|
|
2756
|
-
if (m) return { family: "arabicParen", ord: parseInt(m[1], 10) };
|
|
2757
|
-
m = text.match(/^([가-하])\)\s*/);
|
|
2758
|
-
if (m) {
|
|
2759
|
-
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2760
|
-
if (idx >= 0) return { family: "korParen", ord: idx + 1 };
|
|
2177
|
+
var KOREAN_LIST_SEQ = "\uAC00\uB098\uB2E4\uB77C\uB9C8\uBC14\uC0AC\uC544\uC790\uCC28\uCE74\uD0C0\uD30C\uD558";
|
|
2178
|
+
function parseListLabel(text) {
|
|
2179
|
+
let m = text.match(/^(\d{1,2})\.(?!\d)\s+/);
|
|
2180
|
+
if (m) return { family: "arabicDot", ord: parseInt(m[1], 10) };
|
|
2181
|
+
m = text.match(/^([가-하])\.\s+/);
|
|
2182
|
+
if (m) {
|
|
2183
|
+
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2184
|
+
if (idx >= 0) return { family: "korDot", ord: idx + 1 };
|
|
2185
|
+
}
|
|
2186
|
+
m = text.match(/^(\d{1,2})\)\s*/);
|
|
2187
|
+
if (m) return { family: "arabicParen", ord: parseInt(m[1], 10) };
|
|
2188
|
+
m = text.match(/^([가-하])\)\s*/);
|
|
2189
|
+
if (m) {
|
|
2190
|
+
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2191
|
+
if (idx >= 0) return { family: "korParen", ord: idx + 1 };
|
|
2192
|
+
}
|
|
2193
|
+
m = text.match(/^([①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮])\s*/);
|
|
2194
|
+
if (m) return { family: "circled", ord: m[1].charCodeAt(0) - 9312 + 1 };
|
|
2195
|
+
return null;
|
|
2196
|
+
}
|
|
2197
|
+
var ATTACHMENT_RE = /^붙\s*임\s*(\d+[.:]?)?\s/;
|
|
2198
|
+
function detectKoreanListBlocks(blocks) {
|
|
2199
|
+
const labeled = [];
|
|
2200
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2201
|
+
const b = blocks[i];
|
|
2202
|
+
if (b.type !== "paragraph" && b.type !== "list" || !b.text) continue;
|
|
2203
|
+
const label = parseListLabel(b.text.trim());
|
|
2204
|
+
if (label) labeled.push({ idx: i, label });
|
|
2205
|
+
}
|
|
2206
|
+
const validated = /* @__PURE__ */ new Set();
|
|
2207
|
+
const byFamily = /* @__PURE__ */ new Map();
|
|
2208
|
+
for (const l of labeled) {
|
|
2209
|
+
const arr = byFamily.get(l.label.family) || [];
|
|
2210
|
+
arr.push(l);
|
|
2211
|
+
byFamily.set(l.label.family, arr);
|
|
2212
|
+
}
|
|
2213
|
+
for (const arr of byFamily.values()) {
|
|
2214
|
+
let chain = [];
|
|
2215
|
+
for (const item of arr) {
|
|
2216
|
+
const prev = chain[chain.length - 1];
|
|
2217
|
+
if (prev && item.label.ord === prev.label.ord + 1 && item.idx - prev.idx <= 20) {
|
|
2218
|
+
chain.push(item);
|
|
2219
|
+
} else {
|
|
2220
|
+
if (chain.length >= 2) for (const c of chain) validated.add(c.idx);
|
|
2221
|
+
chain = [item];
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
if (chain.length >= 2) for (const c of chain) validated.add(c.idx);
|
|
2225
|
+
}
|
|
2226
|
+
let familyStack = [];
|
|
2227
|
+
let lastTopLevelList = null;
|
|
2228
|
+
const toRemove = /* @__PURE__ */ new Set();
|
|
2229
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2230
|
+
const b = blocks[i];
|
|
2231
|
+
if (b.type === "table" || b.type === "heading" || b.type === "separator") {
|
|
2232
|
+
familyStack = [];
|
|
2233
|
+
lastTopLevelList = null;
|
|
2234
|
+
continue;
|
|
2235
|
+
}
|
|
2236
|
+
if (b.type !== "paragraph" && b.type !== "list" || !b.text) continue;
|
|
2237
|
+
const text = b.text.trim();
|
|
2238
|
+
if (b.type === "paragraph" && ATTACHMENT_RE.test(text)) {
|
|
2239
|
+
blocks[i] = { ...b, type: "list", listType: "unordered" };
|
|
2240
|
+
continue;
|
|
2241
|
+
}
|
|
2242
|
+
if (!validated.has(i)) continue;
|
|
2243
|
+
const label = parseListLabel(text);
|
|
2244
|
+
let depth = familyStack.indexOf(label.family);
|
|
2245
|
+
if (depth < 0) {
|
|
2246
|
+
familyStack.push(label.family);
|
|
2247
|
+
depth = familyStack.length - 1;
|
|
2248
|
+
} else {
|
|
2249
|
+
familyStack = familyStack.slice(0, depth + 1);
|
|
2250
|
+
}
|
|
2251
|
+
const listType = label.family === "arabicDot" ? "ordered" : "unordered";
|
|
2252
|
+
const listBlock = { ...b, type: "list", listType };
|
|
2253
|
+
if (depth === 0) {
|
|
2254
|
+
blocks[i] = listBlock;
|
|
2255
|
+
lastTopLevelList = listBlock;
|
|
2256
|
+
} else if (lastTopLevelList) {
|
|
2257
|
+
if (!lastTopLevelList.children) lastTopLevelList.children = [];
|
|
2258
|
+
lastTopLevelList.children.push(listBlock);
|
|
2259
|
+
toRemove.add(i);
|
|
2260
|
+
} else {
|
|
2261
|
+
blocks[i] = listBlock;
|
|
2262
|
+
lastTopLevelList = listBlock;
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
if (toRemove.size > 0) {
|
|
2266
|
+
const sorted = [...toRemove].sort((a, b) => b - a);
|
|
2267
|
+
for (const idx of sorted) blocks.splice(idx, 1);
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2270
|
+
function detectListBlocks(blocks) {
|
|
2271
|
+
const result = [];
|
|
2272
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2273
|
+
const block = blocks[i];
|
|
2274
|
+
if (block.type === "paragraph" && block.text) {
|
|
2275
|
+
const text = block.text.trim();
|
|
2276
|
+
if (/^\d+\.\s/.test(text)) {
|
|
2277
|
+
result.push({ ...block, type: "list", listType: "ordered", text: block.text });
|
|
2278
|
+
continue;
|
|
2279
|
+
}
|
|
2280
|
+
if (/^[○●·※▶▷◆◇\-]\s/.test(text)) {
|
|
2281
|
+
result.push({ ...block, type: "list", listType: "unordered", text: block.text });
|
|
2282
|
+
continue;
|
|
2283
|
+
}
|
|
2284
|
+
}
|
|
2285
|
+
result.push(block);
|
|
2286
|
+
}
|
|
2287
|
+
return result;
|
|
2288
|
+
}
|
|
2289
|
+
var KOREAN_TABLE_HEADER_RE = /^\(?(구분|항목|종류|분류|유형|대상|내용|기간|금액|비율|방법|절차|요건|조건|근거|목적|범위|기준)\)?[:\s]/;
|
|
2290
|
+
var KV_FALSE_POSITIVE_RE = /\d{1,2}:\d{2}|:\/\/|\d+:\d+/;
|
|
2291
|
+
function detectSpecialKoreanTables(blocks) {
|
|
2292
|
+
const result = [];
|
|
2293
|
+
let kvLines = [];
|
|
2294
|
+
const flushKvTable = () => {
|
|
2295
|
+
if (kvLines.length < 2) {
|
|
2296
|
+
for (const kv of kvLines) result.push(kv.block);
|
|
2297
|
+
kvLines = [];
|
|
2298
|
+
return;
|
|
2299
|
+
}
|
|
2300
|
+
const cells = kvLines.map((kv) => {
|
|
2301
|
+
if (kv.value) {
|
|
2302
|
+
return [
|
|
2303
|
+
{ text: kv.key, colSpan: 1, rowSpan: 1 },
|
|
2304
|
+
{ text: kv.value, colSpan: 1, rowSpan: 1 }
|
|
2305
|
+
];
|
|
2306
|
+
}
|
|
2307
|
+
return [
|
|
2308
|
+
{ text: kv.key, colSpan: 2, rowSpan: 1 },
|
|
2309
|
+
{ text: "", colSpan: 1, rowSpan: 1 }
|
|
2310
|
+
];
|
|
2311
|
+
});
|
|
2312
|
+
const irTable = {
|
|
2313
|
+
rows: cells.length,
|
|
2314
|
+
cols: 2,
|
|
2315
|
+
cells,
|
|
2316
|
+
hasHeader: true
|
|
2317
|
+
};
|
|
2318
|
+
const firstBlock = kvLines[0].block;
|
|
2319
|
+
result.push({
|
|
2320
|
+
type: "table",
|
|
2321
|
+
table: irTable,
|
|
2322
|
+
pageNumber: firstBlock.pageNumber,
|
|
2323
|
+
bbox: firstBlock.bbox
|
|
2324
|
+
});
|
|
2325
|
+
kvLines = [];
|
|
2326
|
+
};
|
|
2327
|
+
for (const block of blocks) {
|
|
2328
|
+
if (block.type !== "paragraph" || !block.text) {
|
|
2329
|
+
flushKvTable();
|
|
2330
|
+
result.push(block);
|
|
2331
|
+
continue;
|
|
2332
|
+
}
|
|
2333
|
+
const text = block.text.trim();
|
|
2334
|
+
if (KOREAN_TABLE_HEADER_RE.test(text)) {
|
|
2335
|
+
const colonIdx = text.indexOf(":");
|
|
2336
|
+
if (colonIdx >= 0) {
|
|
2337
|
+
kvLines.push({
|
|
2338
|
+
key: text.slice(0, colonIdx).trim(),
|
|
2339
|
+
value: text.slice(colonIdx + 1).trim(),
|
|
2340
|
+
block
|
|
2341
|
+
});
|
|
2342
|
+
} else {
|
|
2343
|
+
const spaceIdx = text.search(/\s/);
|
|
2344
|
+
if (spaceIdx > 0) {
|
|
2345
|
+
kvLines.push({
|
|
2346
|
+
key: text.slice(0, spaceIdx).trim(),
|
|
2347
|
+
value: text.slice(spaceIdx + 1).trim(),
|
|
2348
|
+
block
|
|
2349
|
+
});
|
|
2350
|
+
} else {
|
|
2351
|
+
kvLines.push({ key: text, value: "", block });
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
continue;
|
|
2355
|
+
}
|
|
2356
|
+
if (kvLines.length > 0 && text.includes(":")) {
|
|
2357
|
+
if (!KV_FALSE_POSITIVE_RE.test(text) && !text.includes("(") && !text.includes(")")) {
|
|
2358
|
+
const colonIdx = text.indexOf(":");
|
|
2359
|
+
const key = text.slice(0, colonIdx).trim();
|
|
2360
|
+
if (/^[가-힣]+$/.test(key) && key.length >= 2 && key.length <= 8) {
|
|
2361
|
+
kvLines.push({
|
|
2362
|
+
key,
|
|
2363
|
+
value: text.slice(colonIdx + 1).trim(),
|
|
2364
|
+
block
|
|
2365
|
+
});
|
|
2366
|
+
continue;
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
flushKvTable();
|
|
2371
|
+
result.push(block);
|
|
2372
|
+
}
|
|
2373
|
+
flushKvTable();
|
|
2374
|
+
return result;
|
|
2375
|
+
}
|
|
2376
|
+
function removeHeaderFooterBlocks(blocks, pageHeights, warnings) {
|
|
2377
|
+
const ZONE_RATIO = 0.12;
|
|
2378
|
+
const MIN_REPEAT = 3;
|
|
2379
|
+
const topEntries = [];
|
|
2380
|
+
const bottomEntries = [];
|
|
2381
|
+
for (let bi = 0; bi < blocks.length; bi++) {
|
|
2382
|
+
const b = blocks[bi];
|
|
2383
|
+
if (!b.bbox || !b.pageNumber || !b.text?.trim()) continue;
|
|
2384
|
+
const ph = pageHeights.get(b.bbox.page) || pageHeights.get(b.pageNumber);
|
|
2385
|
+
if (!ph) continue;
|
|
2386
|
+
const blockTop = ph - (b.bbox.y + b.bbox.height);
|
|
2387
|
+
const blockBottom = ph - b.bbox.y;
|
|
2388
|
+
const entry = { blockIdx: bi, page: b.pageNumber, text: b.text.trim() };
|
|
2389
|
+
if (blockBottom <= ph * ZONE_RATIO) bottomEntries.push(entry);
|
|
2390
|
+
else if (blockTop >= ph * (1 - ZONE_RATIO)) topEntries.push(entry);
|
|
2391
|
+
}
|
|
2392
|
+
const removeSet = /* @__PURE__ */ new Set();
|
|
2393
|
+
for (const entries of [topEntries, bottomEntries]) {
|
|
2394
|
+
if (entries.length === 0) continue;
|
|
2395
|
+
const patternCount = /* @__PURE__ */ new Map();
|
|
2396
|
+
const patternPages = /* @__PURE__ */ new Map();
|
|
2397
|
+
for (const e of entries) {
|
|
2398
|
+
const norm = e.text.replace(/\d+/g, "#");
|
|
2399
|
+
patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
|
|
2400
|
+
const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
|
|
2401
|
+
pages.add(e.page);
|
|
2402
|
+
patternPages.set(norm, pages);
|
|
2403
|
+
}
|
|
2404
|
+
const repeatedPatterns = /* @__PURE__ */ new Set();
|
|
2405
|
+
for (const [p, count] of patternCount) {
|
|
2406
|
+
if (count >= MIN_REPEAT && (patternPages.get(p)?.size ?? 0) >= MIN_REPEAT) {
|
|
2407
|
+
repeatedPatterns.add(p);
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2410
|
+
for (const e of entries) {
|
|
2411
|
+
const norm = e.text.replace(/\d+/g, "#");
|
|
2412
|
+
if (repeatedPatterns.has(norm)) {
|
|
2413
|
+
removeSet.add(e.blockIdx);
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
if (removeSet.size > 0) {
|
|
2418
|
+
warnings.push({ message: `${removeSet.size}\uAC1C \uBA38\uB9AC\uAE00/\uBC14\uB2E5\uAE00 \uC694\uC18C \uC81C\uAC70\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
2419
|
+
}
|
|
2420
|
+
return [...removeSet].sort((a, b) => a - b);
|
|
2421
|
+
}
|
|
2422
|
+
|
|
2423
|
+
// src/pdf/page-blocks.ts
|
|
2424
|
+
function extractPageBlocksWithLines(items, pageNum, opList, pageWidth, pageHeight) {
|
|
2425
|
+
if (items.length === 0) return [];
|
|
2426
|
+
let { horizontals, verticals } = extractLines(opList.fnArray, opList.argsArray);
|
|
2427
|
+
({ horizontals, verticals } = filterPageBorderLines(horizontals, verticals, pageWidth, pageHeight));
|
|
2428
|
+
({ horizontals, verticals } = preprocessLines(horizontals, verticals));
|
|
2429
|
+
markStrikethroughItems(items, horizontals);
|
|
2430
|
+
wrapStrikethroughRuns(items);
|
|
2431
|
+
const grids = buildTableGrids(horizontals, verticals);
|
|
2432
|
+
if (grids.length > 0) {
|
|
2433
|
+
return extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals);
|
|
2434
|
+
}
|
|
2435
|
+
return extractPageBlocksFallback(items, pageNum);
|
|
2436
|
+
}
|
|
2437
|
+
var STRIKE_MAX_THICKNESS = 2;
|
|
2438
|
+
var STRIKE_MAX_THICKNESS_RATIO = 0.25;
|
|
2439
|
+
var STRIKE_CENTER_TOLERANCE = 0.25;
|
|
2440
|
+
var STRIKE_MIN_OVERLAP_RATIO = 0.8;
|
|
2441
|
+
var STRIKE_MAX_LINE_TO_TEXT_RATIO = 1.5;
|
|
2442
|
+
function markStrikethroughItems(items, horizontals) {
|
|
2443
|
+
if (items.length === 0 || horizontals.length === 0) return;
|
|
2444
|
+
for (const line of horizontals) {
|
|
2445
|
+
if (line.lineWidth > STRIKE_MAX_THICKNESS) continue;
|
|
2446
|
+
const matches = [];
|
|
2447
|
+
for (const item of items) {
|
|
2448
|
+
const h = item.h > 0 ? item.h : item.fontSize;
|
|
2449
|
+
if (h <= 0 || item.w <= 0) continue;
|
|
2450
|
+
if (line.lineWidth > h * STRIKE_MAX_THICKNESS_RATIO) continue;
|
|
2451
|
+
const centerY = item.y + h * 0.4;
|
|
2452
|
+
if (Math.abs(line.y1 - centerY) > h * STRIKE_CENTER_TOLERANCE) continue;
|
|
2453
|
+
const overlap = Math.min(line.x2, item.x + item.w) - Math.max(line.x1, item.x);
|
|
2454
|
+
if (overlap / item.w < STRIKE_MIN_OVERLAP_RATIO) continue;
|
|
2455
|
+
matches.push(item);
|
|
2456
|
+
}
|
|
2457
|
+
if (matches.length === 0) continue;
|
|
2458
|
+
let totalW = 0;
|
|
2459
|
+
for (const m of matches) totalW += m.w;
|
|
2460
|
+
if (totalW <= 0 || (line.x2 - line.x1) / totalW > STRIKE_MAX_LINE_TO_TEXT_RATIO) continue;
|
|
2461
|
+
for (const m of matches) m.strike = true;
|
|
2761
2462
|
}
|
|
2762
|
-
m = text.match(/^([①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮])\s*/);
|
|
2763
|
-
if (m) return { family: "circled", ord: m[1].charCodeAt(0) - 9312 + 1 };
|
|
2764
|
-
return null;
|
|
2765
2463
|
}
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
|
|
2772
|
-
const
|
|
2773
|
-
|
|
2464
|
+
function wrapStrikethroughRuns(items) {
|
|
2465
|
+
const struck = items.filter((i) => i.strike);
|
|
2466
|
+
if (struck.length === 0) return;
|
|
2467
|
+
const lines = /* @__PURE__ */ new Map();
|
|
2468
|
+
for (const item of struck) {
|
|
2469
|
+
const key = Math.round(item.y / 3);
|
|
2470
|
+
const arr = lines.get(key) || [];
|
|
2471
|
+
arr.push(item);
|
|
2472
|
+
lines.set(key, arr);
|
|
2774
2473
|
}
|
|
2775
|
-
const
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
arr.push(l);
|
|
2780
|
-
byFamily.set(l.label.family, arr);
|
|
2474
|
+
for (const arr of lines.values()) {
|
|
2475
|
+
arr.sort((a, b) => a.x - b.x);
|
|
2476
|
+
arr[0].text = "~~" + arr[0].text;
|
|
2477
|
+
arr[arr.length - 1].text = arr[arr.length - 1].text + "~~";
|
|
2781
2478
|
}
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2479
|
+
}
|
|
2480
|
+
function extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals) {
|
|
2481
|
+
const blocks = [];
|
|
2482
|
+
const usedItems = /* @__PURE__ */ new Set();
|
|
2483
|
+
const sortedGrids = [...grids].sort((a, b) => b.bbox.y2 - a.bbox.y2);
|
|
2484
|
+
for (const grid of sortedGrids) {
|
|
2485
|
+
const numGridRows = grid.rowYs.length - 1;
|
|
2486
|
+
const numGridCols = grid.colXs.length - 1;
|
|
2487
|
+
if (numGridRows === 1 && numGridCols >= 2) continue;
|
|
2488
|
+
if (numGridCols === 1 && numGridRows >= 2) continue;
|
|
2489
|
+
const tableItems = [];
|
|
2490
|
+
const pad = 3;
|
|
2491
|
+
const gridW = grid.bbox.x2 - grid.bbox.x1;
|
|
2492
|
+
for (const item of items) {
|
|
2493
|
+
if (usedItems.has(item)) continue;
|
|
2494
|
+
if (item.y < grid.bbox.y1 - pad || item.y > grid.bbox.y2 + pad) continue;
|
|
2495
|
+
if (item.x < grid.bbox.x1 - pad || item.x + item.w > grid.bbox.x2 + pad) continue;
|
|
2496
|
+
if (gridW < 120 && item.x + item.w > grid.bbox.x2 - 2) continue;
|
|
2497
|
+
tableItems.push(item);
|
|
2498
|
+
usedItems.add(item);
|
|
2499
|
+
}
|
|
2500
|
+
const cells = extractCells(grid, horizontals, verticals);
|
|
2501
|
+
if (cells.length === 0) continue;
|
|
2502
|
+
const textItems = tableItems.map((i) => ({
|
|
2503
|
+
text: i.text,
|
|
2504
|
+
x: i.x,
|
|
2505
|
+
y: i.y,
|
|
2506
|
+
w: i.w,
|
|
2507
|
+
h: i.h,
|
|
2508
|
+
fontSize: i.fontSize,
|
|
2509
|
+
fontName: i.fontName,
|
|
2510
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2511
|
+
}));
|
|
2512
|
+
const cellTextMap = mapTextToCells(textItems, cells);
|
|
2513
|
+
const numRows = grid.rowYs.length - 1;
|
|
2514
|
+
const numCols = grid.colXs.length - 1;
|
|
2515
|
+
const irGrid = Array.from(
|
|
2516
|
+
{ length: numRows },
|
|
2517
|
+
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
2518
|
+
);
|
|
2519
|
+
for (const cell of cells) {
|
|
2520
|
+
const cellItems = cellTextMap.get(cell) || [];
|
|
2521
|
+
let text = cellTextToString(cellItems);
|
|
2522
|
+
text = text.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2523
|
+
text = text.split("\n").map((line) => collapseEvenSpacing(line)).join("\n");
|
|
2524
|
+
irGrid[cell.row][cell.col] = {
|
|
2525
|
+
text,
|
|
2526
|
+
colSpan: cell.colSpan,
|
|
2527
|
+
rowSpan: cell.rowSpan
|
|
2528
|
+
};
|
|
2529
|
+
}
|
|
2530
|
+
let finalGrid = irGrid;
|
|
2531
|
+
let finalRows = numRows;
|
|
2532
|
+
if (numRows <= 2 && numCols >= 3) {
|
|
2533
|
+
const rebuilt = normalizeUndersegmentedTable(irGrid, grid.colXs, textItems);
|
|
2534
|
+
if (rebuilt) {
|
|
2535
|
+
finalGrid = rebuilt.map((row) => row.map((rawText) => {
|
|
2536
|
+
const cleaned = rawText.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2537
|
+
return {
|
|
2538
|
+
text: cleaned.split("\n").map((line) => collapseEvenSpacing(line)).join("\n"),
|
|
2539
|
+
colSpan: 1,
|
|
2540
|
+
rowSpan: 1
|
|
2541
|
+
};
|
|
2542
|
+
}));
|
|
2543
|
+
finalRows = finalGrid.length;
|
|
2791
2544
|
}
|
|
2792
2545
|
}
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
const
|
|
2800
|
-
if (
|
|
2801
|
-
|
|
2802
|
-
|
|
2546
|
+
const irTable = {
|
|
2547
|
+
rows: finalRows,
|
|
2548
|
+
cols: numCols,
|
|
2549
|
+
cells: finalGrid,
|
|
2550
|
+
hasHeader: finalRows > 1
|
|
2551
|
+
};
|
|
2552
|
+
const hasContent = finalGrid.some((row) => row.some((cell) => cell.text.trim() !== ""));
|
|
2553
|
+
if (!hasContent) continue;
|
|
2554
|
+
const tableBbox = {
|
|
2555
|
+
page: pageNum,
|
|
2556
|
+
x: grid.bbox.x1,
|
|
2557
|
+
y: grid.bbox.y1,
|
|
2558
|
+
width: grid.bbox.x2 - grid.bbox.x1,
|
|
2559
|
+
height: grid.bbox.y2 - grid.bbox.y1
|
|
2560
|
+
};
|
|
2561
|
+
if (shouldDemoteTable(irTable)) {
|
|
2562
|
+
const demoted = demoteTableToText(irTable);
|
|
2563
|
+
if (demoted) {
|
|
2564
|
+
const text = numGridRows === 1 ? "\n" + demoted + "\n" : demoted;
|
|
2565
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox: tableBbox, style: dominantStyle(tableItems) });
|
|
2566
|
+
}
|
|
2803
2567
|
continue;
|
|
2804
2568
|
}
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2569
|
+
blocks.push({ type: "table", table: irTable, pageNumber: pageNum, bbox: tableBbox });
|
|
2570
|
+
}
|
|
2571
|
+
let remaining = items.filter((i) => !usedItems.has(i));
|
|
2572
|
+
if (remaining.length > 0) {
|
|
2573
|
+
remaining.sort((a, b) => b.y - a.y || a.x - b.x);
|
|
2574
|
+
const clusterItems = remaining.map((i) => ({
|
|
2575
|
+
text: i.text,
|
|
2576
|
+
x: i.x,
|
|
2577
|
+
y: i.y,
|
|
2578
|
+
w: i.w,
|
|
2579
|
+
h: i.h,
|
|
2580
|
+
fontSize: i.fontSize,
|
|
2581
|
+
fontName: i.fontName,
|
|
2582
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2583
|
+
}));
|
|
2584
|
+
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2585
|
+
if (clusterResults.length > 0) {
|
|
2586
|
+
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2587
|
+
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2588
|
+
const usedClusterIndices = /* @__PURE__ */ new Set();
|
|
2589
|
+
for (const cr of clusterResults) {
|
|
2590
|
+
for (const ci of cr.usedItems) {
|
|
2591
|
+
const idx = ciToIdx.get(ci);
|
|
2592
|
+
if (idx !== void 0) usedClusterIndices.add(idx);
|
|
2593
|
+
}
|
|
2594
|
+
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2595
|
+
}
|
|
2596
|
+
remaining = remaining.filter((_, idx) => !usedClusterIndices.has(idx));
|
|
2810
2597
|
}
|
|
2811
|
-
if (
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2598
|
+
if (remaining.length > 0) {
|
|
2599
|
+
const allY = remaining.map((i) => i.y);
|
|
2600
|
+
const pageH = safeMax(allY) - safeMin(allY);
|
|
2601
|
+
const groups = xyCutOrder(remaining, Math.max(15, pageH * 0.03));
|
|
2602
|
+
const textBlocks = [];
|
|
2603
|
+
for (const group of groups) {
|
|
2604
|
+
if (group.length === 0) continue;
|
|
2605
|
+
const groupBlocks = extractPageBlocksFallback(group, pageNum);
|
|
2606
|
+
for (const b of groupBlocks) textBlocks.push(b);
|
|
2607
|
+
}
|
|
2608
|
+
const finalTextBlocks = detectListBlocks(textBlocks);
|
|
2609
|
+
for (const b of finalTextBlocks) blocks.push(b);
|
|
2610
|
+
}
|
|
2611
|
+
blocks.sort((a, b) => {
|
|
2612
|
+
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2613
|
+
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2614
|
+
return by - ay;
|
|
2615
|
+
});
|
|
2616
|
+
return mergeAdjacentTableBlocks(blocks);
|
|
2617
|
+
}
|
|
2618
|
+
return mergeAdjacentTableBlocks(blocks);
|
|
2619
|
+
}
|
|
2620
|
+
var NEIGHBOR_TABLE_EPSILON = 0.2;
|
|
2621
|
+
function mergeCrossPageTables(blocks) {
|
|
2622
|
+
for (let i = blocks.length - 2; i >= 0; i--) {
|
|
2623
|
+
const prev = blocks[i];
|
|
2624
|
+
const curr = blocks[i + 1];
|
|
2625
|
+
if (prev.type !== "table" || curr.type !== "table" || !prev.table || !curr.table) continue;
|
|
2626
|
+
if (!prev.pageNumber || !curr.pageNumber || curr.pageNumber !== prev.pageNumber + 1) continue;
|
|
2627
|
+
if (prev.table.cols !== curr.table.cols) continue;
|
|
2628
|
+
if (!prev.bbox || !curr.bbox) continue;
|
|
2629
|
+
const width = Math.max(prev.bbox.width, curr.bbox.width, 1);
|
|
2630
|
+
const leftDiff = Math.abs(prev.bbox.x - curr.bbox.x);
|
|
2631
|
+
const rightDiff = Math.abs(prev.bbox.x + prev.bbox.width - (curr.bbox.x + curr.bbox.width));
|
|
2632
|
+
if (leftDiff > width * NEIGHBOR_TABLE_EPSILON || rightDiff > width * NEIGHBOR_TABLE_EPSILON) continue;
|
|
2633
|
+
let currCells = curr.table.cells;
|
|
2634
|
+
if (currCells.length > 1 && prev.table.cells.length > 0 && rowTextsEqual(prev.table.cells[0], currCells[0])) {
|
|
2635
|
+
currCells = currCells.slice(1);
|
|
2819
2636
|
}
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
blocks[i] = listBlock;
|
|
2824
|
-
lastTopLevelList = listBlock;
|
|
2825
|
-
} else if (lastTopLevelList) {
|
|
2826
|
-
if (!lastTopLevelList.children) lastTopLevelList.children = [];
|
|
2827
|
-
lastTopLevelList.children.push(listBlock);
|
|
2828
|
-
toRemove.add(i);
|
|
2829
|
-
} else {
|
|
2830
|
-
blocks[i] = listBlock;
|
|
2831
|
-
lastTopLevelList = listBlock;
|
|
2637
|
+
if (currCells.length === 0) {
|
|
2638
|
+
blocks.splice(i + 1, 1);
|
|
2639
|
+
continue;
|
|
2832
2640
|
}
|
|
2641
|
+
const merged = {
|
|
2642
|
+
rows: prev.table.rows + currCells.length,
|
|
2643
|
+
cols: prev.table.cols,
|
|
2644
|
+
cells: [...prev.table.cells, ...currCells],
|
|
2645
|
+
hasHeader: prev.table.hasHeader,
|
|
2646
|
+
caption: prev.table.caption
|
|
2647
|
+
};
|
|
2648
|
+
blocks[i] = { ...prev, table: merged };
|
|
2649
|
+
blocks.splice(i + 1, 1);
|
|
2833
2650
|
}
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2651
|
+
}
|
|
2652
|
+
function rowTextsEqual(a, b) {
|
|
2653
|
+
if (a.length !== b.length) return false;
|
|
2654
|
+
const norm = (t) => t.replace(/\s+/g, "");
|
|
2655
|
+
for (let i = 0; i < a.length; i++) {
|
|
2656
|
+
if (norm(a[i].text) !== norm(b[i].text)) return false;
|
|
2837
2657
|
}
|
|
2658
|
+
return a.some((c) => c.text.trim() !== "");
|
|
2838
2659
|
}
|
|
2839
|
-
function
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
}
|
|
2660
|
+
function mergeAdjacentTableBlocks(blocks) {
|
|
2661
|
+
if (blocks.length <= 1) return blocks;
|
|
2662
|
+
const result = [blocks[0]];
|
|
2663
|
+
for (let i = 1; i < blocks.length; i++) {
|
|
2664
|
+
const prev = result[result.length - 1];
|
|
2665
|
+
const curr = blocks[i];
|
|
2666
|
+
if (prev.type === "table" && curr.type === "table" && prev.table && curr.table && prev.table.cols === curr.table.cols) {
|
|
2667
|
+
const merged = {
|
|
2668
|
+
rows: prev.table.rows + curr.table.rows,
|
|
2669
|
+
cols: prev.table.cols,
|
|
2670
|
+
cells: [...prev.table.cells, ...curr.table.cells],
|
|
2671
|
+
hasHeader: prev.table.hasHeader
|
|
2672
|
+
};
|
|
2673
|
+
result[result.length - 1] = { ...prev, table: merged };
|
|
2674
|
+
} else {
|
|
2675
|
+
result.push(curr);
|
|
2853
2676
|
}
|
|
2854
|
-
result.push(block);
|
|
2855
2677
|
}
|
|
2856
2678
|
return result;
|
|
2857
2679
|
}
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
const
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2680
|
+
function extractPageBlocksFallback(items, pageNum) {
|
|
2681
|
+
if (items.length === 0) return [];
|
|
2682
|
+
const blocks = [];
|
|
2683
|
+
const clusterItems = items.map((i) => ({
|
|
2684
|
+
text: i.text,
|
|
2685
|
+
x: i.x,
|
|
2686
|
+
y: i.y,
|
|
2687
|
+
w: i.w,
|
|
2688
|
+
h: i.h,
|
|
2689
|
+
fontSize: i.fontSize,
|
|
2690
|
+
fontName: i.fontName,
|
|
2691
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2692
|
+
}));
|
|
2693
|
+
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2694
|
+
if (clusterResults.length > 0) {
|
|
2695
|
+
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2696
|
+
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2697
|
+
const usedIndices = /* @__PURE__ */ new Set();
|
|
2698
|
+
for (const cr of clusterResults) {
|
|
2699
|
+
for (const ci of cr.usedItems) {
|
|
2700
|
+
const idx = ciToIdx.get(ci);
|
|
2701
|
+
if (idx !== void 0) usedIndices.add(idx);
|
|
2875
2702
|
}
|
|
2876
|
-
|
|
2877
|
-
{ text: kv.key, colSpan: 2, rowSpan: 1 },
|
|
2878
|
-
{ text: "", colSpan: 1, rowSpan: 1 }
|
|
2879
|
-
];
|
|
2880
|
-
});
|
|
2881
|
-
const irTable = {
|
|
2882
|
-
rows: cells.length,
|
|
2883
|
-
cols: 2,
|
|
2884
|
-
cells,
|
|
2885
|
-
hasHeader: true
|
|
2886
|
-
};
|
|
2887
|
-
const firstBlock = kvLines[0].block;
|
|
2888
|
-
result.push({
|
|
2889
|
-
type: "table",
|
|
2890
|
-
table: irTable,
|
|
2891
|
-
pageNumber: firstBlock.pageNumber,
|
|
2892
|
-
bbox: firstBlock.bbox
|
|
2893
|
-
});
|
|
2894
|
-
kvLines = [];
|
|
2895
|
-
};
|
|
2896
|
-
for (const block of blocks) {
|
|
2897
|
-
if (block.type !== "paragraph" || !block.text) {
|
|
2898
|
-
flushKvTable();
|
|
2899
|
-
result.push(block);
|
|
2900
|
-
continue;
|
|
2703
|
+
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2901
2704
|
}
|
|
2902
|
-
const
|
|
2903
|
-
if (
|
|
2904
|
-
const
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
});
|
|
2911
|
-
} else {
|
|
2912
|
-
const spaceIdx = text.search(/\s/);
|
|
2913
|
-
if (spaceIdx > 0) {
|
|
2914
|
-
kvLines.push({
|
|
2915
|
-
key: text.slice(0, spaceIdx).trim(),
|
|
2916
|
-
value: text.slice(spaceIdx + 1).trim(),
|
|
2917
|
-
block
|
|
2918
|
-
});
|
|
2919
|
-
} else {
|
|
2920
|
-
kvLines.push({ key: text, value: "", block });
|
|
2921
|
-
}
|
|
2705
|
+
const remaining = items.filter((_, idx) => !usedIndices.has(idx));
|
|
2706
|
+
if (remaining.length > 0) {
|
|
2707
|
+
const yLines = mergeSuperscriptLines(groupByY(remaining));
|
|
2708
|
+
for (const line of yLines) {
|
|
2709
|
+
const text = mergeLineSimple(line);
|
|
2710
|
+
if (!text.trim()) continue;
|
|
2711
|
+
const bbox = computeBBox(line, pageNum);
|
|
2712
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2922
2713
|
}
|
|
2923
|
-
continue;
|
|
2924
2714
|
}
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2715
|
+
blocks.sort((a, b) => {
|
|
2716
|
+
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2717
|
+
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2718
|
+
return by - ay;
|
|
2719
|
+
});
|
|
2720
|
+
} else {
|
|
2721
|
+
const allYLines = mergeSuperscriptLines(groupByY(items));
|
|
2722
|
+
const columns = detectColumns(allYLines);
|
|
2723
|
+
if (columns && columns.length >= 3) {
|
|
2724
|
+
const tableText = extractWithColumns(allYLines, columns);
|
|
2725
|
+
const bbox = computeBBox(items, pageNum);
|
|
2726
|
+
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(items) });
|
|
2727
|
+
} else {
|
|
2728
|
+
const allY = items.map((i) => i.y);
|
|
2729
|
+
const pageHeight = safeMax(allY) - safeMin(allY);
|
|
2730
|
+
const gapThreshold = Math.max(15, pageHeight * 0.03);
|
|
2731
|
+
const orderedGroups = xyCutOrder(items, gapThreshold);
|
|
2732
|
+
for (const group of orderedGroups) {
|
|
2733
|
+
if (group.length === 0) continue;
|
|
2734
|
+
const yLines = mergeSuperscriptLines(groupByY(group));
|
|
2735
|
+
const groupColumns = detectColumns(yLines);
|
|
2736
|
+
if (groupColumns && groupColumns.length >= 3) {
|
|
2737
|
+
const tableText = extractWithColumns(yLines, groupColumns);
|
|
2738
|
+
const bbox = computeBBox(group, pageNum);
|
|
2739
|
+
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
|
|
2740
|
+
} else {
|
|
2741
|
+
for (const line of yLines) {
|
|
2742
|
+
const text = mergeLineSimple(line);
|
|
2743
|
+
if (!text.trim()) continue;
|
|
2744
|
+
const bbox = computeBBox(line, pageNum);
|
|
2745
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2746
|
+
}
|
|
2936
2747
|
}
|
|
2937
2748
|
}
|
|
2938
2749
|
}
|
|
2939
|
-
flushKvTable();
|
|
2940
|
-
result.push(block);
|
|
2941
2750
|
}
|
|
2942
|
-
|
|
2943
|
-
return result;
|
|
2751
|
+
return detectSpecialKoreanTables(blocks);
|
|
2944
2752
|
}
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
const
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
const blockTop = ph - (b.bbox.y + b.bbox.height);
|
|
2956
|
-
const blockBottom = ph - b.bbox.y;
|
|
2957
|
-
const entry = { blockIdx: bi, page: b.pageNumber, text: b.text.trim() };
|
|
2958
|
-
if (blockBottom <= ph * ZONE_RATIO) bottomEntries.push(entry);
|
|
2959
|
-
else if (blockTop >= ph * (1 - ZONE_RATIO)) topEntries.push(entry);
|
|
2960
|
-
}
|
|
2961
|
-
const removeSet = /* @__PURE__ */ new Set();
|
|
2962
|
-
for (const entries of [topEntries, bottomEntries]) {
|
|
2963
|
-
if (entries.length === 0) continue;
|
|
2964
|
-
const patternCount = /* @__PURE__ */ new Map();
|
|
2965
|
-
const patternPages = /* @__PURE__ */ new Map();
|
|
2966
|
-
for (const e of entries) {
|
|
2967
|
-
const norm = e.text.replace(/\d+/g, "#");
|
|
2968
|
-
patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
|
|
2969
|
-
const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
|
|
2970
|
-
pages.add(e.page);
|
|
2971
|
-
patternPages.set(norm, pages);
|
|
2972
|
-
}
|
|
2973
|
-
const repeatedPatterns = /* @__PURE__ */ new Set();
|
|
2974
|
-
for (const [p, count] of patternCount) {
|
|
2975
|
-
if (count >= MIN_REPEAT && (patternPages.get(p)?.size ?? 0) >= MIN_REPEAT) {
|
|
2976
|
-
repeatedPatterns.add(p);
|
|
2977
|
-
}
|
|
2978
|
-
}
|
|
2979
|
-
for (const e of entries) {
|
|
2980
|
-
const norm = e.text.replace(/\d+/g, "#");
|
|
2981
|
-
if (repeatedPatterns.has(norm)) {
|
|
2982
|
-
removeSet.add(e.blockIdx);
|
|
2753
|
+
|
|
2754
|
+
// src/pdf/text-clean.ts
|
|
2755
|
+
function sanitizeBlockControlChars(blocks) {
|
|
2756
|
+
for (const b of blocks) {
|
|
2757
|
+
if (b.text) b.text = stripControlChars(b.text);
|
|
2758
|
+
if (b.table) {
|
|
2759
|
+
for (const row of b.table.cells) {
|
|
2760
|
+
for (const cell of row) {
|
|
2761
|
+
if (cell.text) cell.text = stripControlChars(cell.text);
|
|
2762
|
+
}
|
|
2983
2763
|
}
|
|
2984
2764
|
}
|
|
2765
|
+
if (b.children) sanitizeBlockControlChars(b.children);
|
|
2985
2766
|
}
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2767
|
+
}
|
|
2768
|
+
function cleanPdfText(text) {
|
|
2769
|
+
return mergeKoreanLines(
|
|
2770
|
+
stripControlChars(text).replace(/^\d{1,4}\n/, "").replace(/^[\s]*[-–—]\s*[-–—]?\d+[-–—]?[\s]*[-–—]?[\s]*$/gm, "").replace(/^\s*\d+\s*\/\s*\d+\s*$/gm, "").replace(/\n\d{1,4}\n/g, "\n").replace(/\n\d{1,4}$/, "").replace(/^#{1,6}\s*\d{1,4}\s*$/gm, "")
|
|
2771
|
+
).replace(/^(?!\| ---).*$/gm, (line) => {
|
|
2772
|
+
if (/^\s*\${1,2}.+\${1,2}\s*$/.test(line)) return line;
|
|
2773
|
+
return collapseEvenSpacing(line);
|
|
2774
|
+
}).replace(/([□■◆○●▶ㅇ])\s+([가-힣])\s+([가-힣])/g, "$1 $2$3").replace(/\\~\\~/g, "~~").replace(/~~~~/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
|
2775
|
+
}
|
|
2776
|
+
function startsWithMarker(line) {
|
|
2777
|
+
const t = line.trimStart();
|
|
2778
|
+
return /^[가-힣ㄱ-ㅎ][.)]/.test(t) || /^\d+[.)]/.test(t) || /^\([가-힣ㄱ-ㅎ\d]+\)/.test(t) || /^[○●※▶▷◆◇■□★☆\-·]\s/.test(t) || /^제\d+[조항호장절]/.test(t);
|
|
2779
|
+
}
|
|
2780
|
+
function isStandaloneHeader(line) {
|
|
2781
|
+
return /^제\d+[조항호장절](\([^)]*\))?(\s+\S+){0,7}$/.test(line.trim());
|
|
2990
2782
|
}
|
|
2991
2783
|
function mergeKoreanLines(text) {
|
|
2992
2784
|
if (!text) return "";
|
|
@@ -3017,6 +2809,8 @@ function mergeKoreanLines(text) {
|
|
|
3017
2809
|
}
|
|
3018
2810
|
return result.join("\n");
|
|
3019
2811
|
}
|
|
2812
|
+
|
|
2813
|
+
// src/pdf/formula-ocr.ts
|
|
3020
2814
|
async function applyFormulaOcr(buffer, blocks, pageFilter, effectivePageCount, warnings, _onProgress) {
|
|
3021
2815
|
const formulaMod = await import("./formula-JCNF43NE.js");
|
|
3022
2816
|
const { FormulaPipeline, ensureFormulaModels } = formulaMod;
|
|
@@ -3135,6 +2929,239 @@ async function applyFormulaOcr(buffer, blocks, pageFilter, effectivePageCount, w
|
|
|
3135
2929
|
function formatMb(bytes) {
|
|
3136
2930
|
return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
|
|
3137
2931
|
}
|
|
2932
|
+
|
|
2933
|
+
// src/pdf/polyfill.ts
|
|
2934
|
+
import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
|
|
2935
|
+
var g = globalThis;
|
|
2936
|
+
if (typeof g.DOMMatrix === "undefined") {
|
|
2937
|
+
g.DOMMatrix = class DOMMatrix {
|
|
2938
|
+
m = [1, 0, 0, 1, 0, 0];
|
|
2939
|
+
constructor(init) {
|
|
2940
|
+
if (init) this.m = init;
|
|
2941
|
+
}
|
|
2942
|
+
};
|
|
2943
|
+
}
|
|
2944
|
+
if (typeof g.Path2D === "undefined") {
|
|
2945
|
+
g.Path2D = class Path2D {
|
|
2946
|
+
};
|
|
2947
|
+
}
|
|
2948
|
+
g.pdfjsWorker = pdfjsWorker;
|
|
2949
|
+
|
|
2950
|
+
// src/pdf/parser.ts
|
|
2951
|
+
import { getDocument, GlobalWorkerOptions } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
2952
|
+
GlobalWorkerOptions.workerSrc = "";
|
|
2953
|
+
var MAX_PAGES = 5e3;
|
|
2954
|
+
var MAX_TOTAL_TEXT = 100 * 1024 * 1024;
|
|
2955
|
+
var PDF_LOAD_TIMEOUT_MS = 3e4;
|
|
2956
|
+
async function loadPdfWithTimeout(buffer) {
|
|
2957
|
+
const loadingTask = getDocument({
|
|
2958
|
+
data: new Uint8Array(buffer),
|
|
2959
|
+
useSystemFonts: true,
|
|
2960
|
+
disableFontFace: true,
|
|
2961
|
+
isEvalSupported: false
|
|
2962
|
+
});
|
|
2963
|
+
let timer;
|
|
2964
|
+
try {
|
|
2965
|
+
return await Promise.race([
|
|
2966
|
+
loadingTask.promise,
|
|
2967
|
+
new Promise((_, reject) => {
|
|
2968
|
+
timer = setTimeout(() => {
|
|
2969
|
+
loadingTask.destroy();
|
|
2970
|
+
reject(new KordocError("PDF \uB85C\uB529 \uD0C0\uC784\uC544\uC6C3 (30\uCD08 \uCD08\uACFC)"));
|
|
2971
|
+
}, PDF_LOAD_TIMEOUT_MS);
|
|
2972
|
+
})
|
|
2973
|
+
]);
|
|
2974
|
+
} finally {
|
|
2975
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
async function parsePdfDocument(buffer, options) {
|
|
2979
|
+
const formulaBuffer = options?.formulaOcr ? buffer.slice(0) : null;
|
|
2980
|
+
const doc = await loadPdfWithTimeout(buffer);
|
|
2981
|
+
try {
|
|
2982
|
+
const pageCount = doc.numPages;
|
|
2983
|
+
if (pageCount === 0) throw new KordocError("PDF\uC5D0 \uD398\uC774\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
|
|
2984
|
+
const metadata = { pageCount };
|
|
2985
|
+
await extractPdfMetadata(doc, metadata);
|
|
2986
|
+
const blocks = [];
|
|
2987
|
+
const warnings = [];
|
|
2988
|
+
const pageQuality = [];
|
|
2989
|
+
let totalChars = 0;
|
|
2990
|
+
let totalTextBytes = 0;
|
|
2991
|
+
const effectivePageCount = Math.min(pageCount, MAX_PAGES);
|
|
2992
|
+
const pageFilter = options?.pages ? parsePageRange(options.pages, effectivePageCount) : null;
|
|
2993
|
+
const totalTarget = pageFilter ? pageFilter.size : effectivePageCount;
|
|
2994
|
+
const fontSizeFreq = /* @__PURE__ */ new Map();
|
|
2995
|
+
const pageHeights = /* @__PURE__ */ new Map();
|
|
2996
|
+
const pagesWithLargeImage = /* @__PURE__ */ new Set();
|
|
2997
|
+
const skippedImagePages = /* @__PURE__ */ new Map();
|
|
2998
|
+
let parsedPages = 0;
|
|
2999
|
+
for (let i = 1; i <= effectivePageCount; i++) {
|
|
3000
|
+
if (pageFilter && !pageFilter.has(i)) continue;
|
|
3001
|
+
try {
|
|
3002
|
+
const page = await doc.getPage(i);
|
|
3003
|
+
const tc = await page.getTextContent();
|
|
3004
|
+
const viewport = page.getViewport({ scale: 1 });
|
|
3005
|
+
pageHeights.set(i, viewport.height);
|
|
3006
|
+
const rawItems = tc.items;
|
|
3007
|
+
const items = normalizeItems(rawItems);
|
|
3008
|
+
const { visible, hiddenCount } = filterHiddenText(items, viewport.width, viewport.height);
|
|
3009
|
+
if (hiddenCount > 0) {
|
|
3010
|
+
warnings.push({ page: i, message: `${hiddenCount}\uAC1C \uC228\uACA8\uC9C4 \uD14D\uC2A4\uD2B8 \uC694\uC18C \uD544\uD130\uB9C1\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
3011
|
+
}
|
|
3012
|
+
for (const item of visible) {
|
|
3013
|
+
if (item.fontSize > 0) fontSizeFreq.set(item.fontSize, (fontSizeFreq.get(item.fontSize) || 0) + 1);
|
|
3014
|
+
}
|
|
3015
|
+
const opList = await page.getOperatorList();
|
|
3016
|
+
const pageArea = viewport.width * viewport.height;
|
|
3017
|
+
if (pageArea > 0) {
|
|
3018
|
+
const imageRegions = extractImageRegions(opList.fnArray, opList.argsArray);
|
|
3019
|
+
let uncovered = 0;
|
|
3020
|
+
for (const r of imageRegions) {
|
|
3021
|
+
const area = (r.x2 - r.x1) * (r.y2 - r.y1);
|
|
3022
|
+
if (area < pageArea * 0.05) continue;
|
|
3023
|
+
pagesWithLargeImage.add(i);
|
|
3024
|
+
const hasText = visible.some((it) => {
|
|
3025
|
+
const cx = it.x + it.w / 2;
|
|
3026
|
+
const cy = it.y + (it.h || it.fontSize) / 2;
|
|
3027
|
+
return cx >= r.x1 && cx <= r.x2 && cy >= r.y1 && cy <= r.y2;
|
|
3028
|
+
});
|
|
3029
|
+
if (!hasText) uncovered++;
|
|
3030
|
+
}
|
|
3031
|
+
if (uncovered > 0) skippedImagePages.set(i, uncovered);
|
|
3032
|
+
}
|
|
3033
|
+
const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
|
|
3034
|
+
for (const b of pageBlocks) blocks.push(b);
|
|
3035
|
+
let pageText = "";
|
|
3036
|
+
for (const b of pageBlocks) {
|
|
3037
|
+
const t = b.text || "";
|
|
3038
|
+
totalChars += t.replace(/\s/g, "").length;
|
|
3039
|
+
totalTextBytes += t.length * 2;
|
|
3040
|
+
pageText += pageText ? "\n" + t : t;
|
|
3041
|
+
}
|
|
3042
|
+
pageQuality.push(computePageQuality(i, pageText));
|
|
3043
|
+
if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
|
|
3044
|
+
parsedPages++;
|
|
3045
|
+
options?.onProgress?.(parsedPages, totalTarget);
|
|
3046
|
+
} catch (pageErr) {
|
|
3047
|
+
if (pageErr instanceof KordocError) throw pageErr;
|
|
3048
|
+
warnings.push({ page: i, message: `\uD398\uC774\uC9C0 ${i} \uD30C\uC2F1 \uC2E4\uD328: ${pageErr instanceof Error ? pageErr.message : "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958"}`, code: "PARTIAL_PARSE" });
|
|
3049
|
+
}
|
|
3050
|
+
}
|
|
3051
|
+
const parsedPageCount = parsedPages || (pageFilter ? pageFilter.size : effectivePageCount);
|
|
3052
|
+
let isImageBased = false;
|
|
3053
|
+
if (totalChars / Math.max(parsedPageCount, 1) < 10) {
|
|
3054
|
+
if (options?.ocr) {
|
|
3055
|
+
try {
|
|
3056
|
+
const { ocrPages } = await import("./provider-AKROB7WQ.js");
|
|
3057
|
+
const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
|
|
3058
|
+
if (ocrBlocks.length > 0) {
|
|
3059
|
+
const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
|
|
3060
|
+
return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
|
|
3061
|
+
}
|
|
3062
|
+
} catch {
|
|
3063
|
+
}
|
|
3064
|
+
}
|
|
3065
|
+
isImageBased = true;
|
|
3066
|
+
warnings.push({
|
|
3067
|
+
message: `\uC774\uBBF8\uC9C0 \uAE30\uBC18 PDF (${pageCount}\uD398\uC774\uC9C0, \uD14D\uC2A4\uD2B8 ${totalChars}\uC790) \u2014 \uD14D\uC2A4\uD2B8 \uB808\uC774\uC5B4\uAC00 \uC5C6\uC5B4 OCR\uC774 \uD544\uC694\uD569\uB2C8\uB2E4`,
|
|
3068
|
+
code: "NEEDS_OCR"
|
|
3069
|
+
});
|
|
3070
|
+
}
|
|
3071
|
+
if (!isImageBased) {
|
|
3072
|
+
const OCR_REASON_MESSAGES = {
|
|
3073
|
+
low_text: "\uD14D\uC2A4\uD2B8\uAC00 \uAC70\uC758 \uC5C6\uB294 \uD398\uC774\uC9C0 (\uC2A4\uCE94/\uC774\uBBF8\uC9C0 \uCD94\uC815)",
|
|
3074
|
+
high_pua: "\uAE00\uAF34 \uB9E4\uD551 \uC2E4\uD328 (PUA \uBE44\uC728 \uB192\uC74C) \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
3075
|
+
high_control: "\uC81C\uC5B4\uBB38\uC790 \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
3076
|
+
high_replacement: "\uB300\uCCB4\uBB38\uC790(U+FFFD) \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00"
|
|
3077
|
+
};
|
|
3078
|
+
for (const pq of pageQuality) {
|
|
3079
|
+
if (!pq.needsOcr || !pq.ocrReason) continue;
|
|
3080
|
+
if (pq.ocrReason === "low_text" && !pagesWithLargeImage.has(pq.page)) continue;
|
|
3081
|
+
warnings.push({ page: pq.page, message: `${OCR_REASON_MESSAGES[pq.ocrReason]} \u2014 OCR \uAC80\uD1A0 \uD544\uC694`, code: "NEEDS_OCR" });
|
|
3082
|
+
}
|
|
3083
|
+
}
|
|
3084
|
+
if (!isImageBased) {
|
|
3085
|
+
for (const [page, count] of [...skippedImagePages.entries()].sort((a, b) => a[0] - b[0])) {
|
|
3086
|
+
warnings.push({ page, message: `${count}\uAC1C \uC774\uBBF8\uC9C0 \uC601\uC5ED\uC5D0 \uCD94\uCD9C \uAC00\uB2A5\uD55C \uD14D\uC2A4\uD2B8 \uC5C6\uC74C (\uADF8\uB9BC/\uCC28\uD2B8/\uB3C4\uC7A5 \uB0B4\uC6A9 \uB204\uB77D \uAC00\uB2A5)`, code: "SKIPPED_IMAGE" });
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
if (options?.removeHeaderFooter !== false && parsedPageCount >= 3) {
|
|
3090
|
+
const removed = removeHeaderFooterBlocks(blocks, pageHeights, warnings);
|
|
3091
|
+
for (let ri = removed.length - 1; ri >= 0; ri--) {
|
|
3092
|
+
blocks.splice(removed[ri], 1);
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
mergeCrossPageTables(blocks);
|
|
3096
|
+
if (options?.formulaOcr && formulaBuffer) {
|
|
3097
|
+
try {
|
|
3098
|
+
await applyFormulaOcr(formulaBuffer, blocks, pageFilter, effectivePageCount, warnings, options.onProgress);
|
|
3099
|
+
} catch (e) {
|
|
3100
|
+
warnings.push({
|
|
3101
|
+
message: `\uC218\uC2DD OCR \uC2E4\uD328: ${e instanceof Error ? e.message : String(e)}`,
|
|
3102
|
+
code: "PARTIAL_PARSE"
|
|
3103
|
+
});
|
|
3104
|
+
}
|
|
3105
|
+
}
|
|
3106
|
+
const medianFontSize = computeMedianFontSizeFromFreq(fontSizeFreq);
|
|
3107
|
+
if (medianFontSize > 0) {
|
|
3108
|
+
detectHeadings(blocks, medianFontSize);
|
|
3109
|
+
}
|
|
3110
|
+
detectMarkerHeadings(blocks);
|
|
3111
|
+
detectTableCaptions(blocks);
|
|
3112
|
+
detectKoreanListBlocks(blocks);
|
|
3113
|
+
const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
|
|
3114
|
+
sanitizeBlockControlChars(blocks);
|
|
3115
|
+
let markdown = cleanPdfText(blocksToMarkdown(blocks));
|
|
3116
|
+
return {
|
|
3117
|
+
markdown,
|
|
3118
|
+
blocks,
|
|
3119
|
+
metadata,
|
|
3120
|
+
outline: outline.length > 0 ? outline : void 0,
|
|
3121
|
+
warnings: warnings.length > 0 ? warnings : void 0,
|
|
3122
|
+
isImageBased: isImageBased || void 0,
|
|
3123
|
+
pageQuality,
|
|
3124
|
+
qualitySummary: summarizeDocumentQuality(pageQuality)
|
|
3125
|
+
};
|
|
3126
|
+
} finally {
|
|
3127
|
+
await doc.destroy().catch(() => {
|
|
3128
|
+
});
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
3131
|
+
async function extractPdfMetadata(doc, metadata) {
|
|
3132
|
+
try {
|
|
3133
|
+
const result = await doc.getMetadata();
|
|
3134
|
+
if (!result?.info) return;
|
|
3135
|
+
const info = result.info;
|
|
3136
|
+
if (typeof info.Title === "string" && info.Title.trim()) metadata.title = info.Title.trim();
|
|
3137
|
+
if (typeof info.Author === "string" && info.Author.trim()) metadata.author = info.Author.trim();
|
|
3138
|
+
if (typeof info.Creator === "string" && info.Creator.trim()) metadata.creator = info.Creator.trim();
|
|
3139
|
+
if (typeof info.Subject === "string" && info.Subject.trim()) metadata.description = info.Subject.trim();
|
|
3140
|
+
if (typeof info.Keywords === "string" && info.Keywords.trim()) {
|
|
3141
|
+
metadata.keywords = info.Keywords.split(/[,;]/).map((k) => k.trim()).filter(Boolean);
|
|
3142
|
+
}
|
|
3143
|
+
if (typeof info.CreationDate === "string") metadata.createdAt = parsePdfDate(info.CreationDate);
|
|
3144
|
+
if (typeof info.ModDate === "string") metadata.modifiedAt = parsePdfDate(info.ModDate);
|
|
3145
|
+
} catch {
|
|
3146
|
+
}
|
|
3147
|
+
}
|
|
3148
|
+
function parsePdfDate(dateStr) {
|
|
3149
|
+
const m = dateStr.match(/D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/);
|
|
3150
|
+
if (!m) return void 0;
|
|
3151
|
+
const [, year, month = "01", day = "01", hour = "00", min = "00", sec = "00"] = m;
|
|
3152
|
+
return `${year}-${month}-${day}T${hour}:${min}:${sec}`;
|
|
3153
|
+
}
|
|
3154
|
+
async function extractPdfMetadataOnly(buffer) {
|
|
3155
|
+
const doc = await loadPdfWithTimeout(buffer);
|
|
3156
|
+
try {
|
|
3157
|
+
const metadata = { pageCount: doc.numPages };
|
|
3158
|
+
await extractPdfMetadata(doc, metadata);
|
|
3159
|
+
return metadata;
|
|
3160
|
+
} finally {
|
|
3161
|
+
await doc.destroy().catch(() => {
|
|
3162
|
+
});
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3138
3165
|
export {
|
|
3139
3166
|
cleanPdfText,
|
|
3140
3167
|
detectKoreanListBlocks,
|
|
@@ -3144,4 +3171,4 @@ export {
|
|
|
3144
3171
|
parsePdfDocument,
|
|
3145
3172
|
removeHeaderFooterBlocks
|
|
3146
3173
|
};
|
|
3147
|
-
//# sourceMappingURL=parser-
|
|
3174
|
+
//# sourceMappingURL=parser-FFEBMLSH.js.map
|