kordoc 3.8.0 → 3.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/dist/{-ATVQYFSW.js → -FEHSMPVO.js} +3 -3
- package/dist/{chunk-QV25HMU7.js → chunk-553VTUVP.js} +2 -2
- package/dist/{chunk-SLKF72QF.js → chunk-DP37KF2X.js} +1985 -1954
- package/dist/chunk-DP37KF2X.js.map +1 -0
- package/dist/{chunk-3R3YK7EM.js → chunk-JHZUFBUV.js} +2 -2
- package/dist/{chunk-ITJIALN5.cjs → chunk-YBPNKFJW.cjs} +2 -2
- package/dist/{chunk-ITJIALN5.cjs.map → chunk-YBPNKFJW.cjs.map} +1 -1
- package/dist/cli.js +4 -4
- package/dist/index.cjs +2059 -2028
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +17 -8
- package/dist/index.d.ts +17 -8
- package/dist/index.js +1974 -1943
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +3 -3
- package/dist/{parser-7G5F7PT2.js → parser-KNQDRLZQ.js} +1930 -1863
- package/dist/parser-KNQDRLZQ.js.map +1 -0
- package/dist/{parser-DCK42RMA.js → parser-NR2TYGO3.js} +1930 -1863
- package/dist/parser-NR2TYGO3.js.map +1 -0
- package/dist/{parser-GUSJH44K.cjs → parser-NS4ZPD7B.cjs} +1933 -1864
- package/dist/parser-NS4ZPD7B.cjs.map +1 -0
- package/dist/{watch-GVZESOCE.js → watch-XCWADLPU.js} +3 -3
- package/package.json +2 -2
- package/dist/chunk-SLKF72QF.js.map +0 -1
- package/dist/parser-7G5F7PT2.js.map +0 -1
- package/dist/parser-DCK42RMA.js.map +0 -1
- package/dist/parser-GUSJH44K.cjs.map +0 -1
- /package/dist/{-ATVQYFSW.js.map → -FEHSMPVO.js.map} +0 -0
- /package/dist/{chunk-QV25HMU7.js.map → chunk-553VTUVP.js.map} +0 -0
- /package/dist/{chunk-3R3YK7EM.js.map → chunk-JHZUFBUV.js.map} +0 -0
- /package/dist/{watch-GVZESOCE.js.map → watch-XCWADLPU.js.map} +0 -0
|
@@ -6,41 +6,44 @@ import {
|
|
|
6
6
|
blocksToMarkdown,
|
|
7
7
|
safeMax,
|
|
8
8
|
safeMin
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-553VTUVP.js";
|
|
10
10
|
import {
|
|
11
11
|
parsePageRange
|
|
12
12
|
} from "./chunk-GE43BE46.js";
|
|
13
13
|
|
|
14
|
-
// src/pdf/line-
|
|
14
|
+
// src/pdf/line-extract.ts
|
|
15
15
|
import { OPS } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
16
16
|
var ORIENTATION_TOL = 2;
|
|
17
17
|
var MIN_LINE_LENGTH = 15;
|
|
18
18
|
var MAX_LINE_WIDTH = 5;
|
|
19
|
-
var CONNECT_TOL = 5;
|
|
20
|
-
var CELL_PADDING = 2;
|
|
21
|
-
var MIN_COL_WIDTH = 15;
|
|
22
|
-
var MIN_ROW_HEIGHT = 6;
|
|
23
|
-
var VERTEX_MERGE_FACTOR = 4;
|
|
24
|
-
var MIN_COORD_MERGE_TOL = 8;
|
|
25
19
|
function extractLines(fnArray, argsArray) {
|
|
26
20
|
const horizontals = [];
|
|
27
21
|
const verticals = [];
|
|
28
22
|
let lineWidth = 1;
|
|
23
|
+
let ctm = [1, 0, 0, 1, 0, 0];
|
|
24
|
+
const ctmStack = [];
|
|
25
|
+
const applyCtm = (x, y) => [ctm[0] * x + ctm[2] * y + ctm[4], ctm[1] * x + ctm[3] * y + ctm[5]];
|
|
26
|
+
const ctmScale = () => (Math.hypot(ctm[0], ctm[1]) + Math.hypot(ctm[2], ctm[3])) / 2;
|
|
29
27
|
let currentPath = [];
|
|
30
28
|
let pathStartX = 0, pathStartY = 0;
|
|
31
29
|
let curX = 0, curY = 0;
|
|
32
|
-
function
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
function pushSeg(x1, y1, x2, y2) {
|
|
31
|
+
const [tx1, ty1] = applyCtm(x1, y1);
|
|
32
|
+
const [tx2, ty2] = applyCtm(x2, y2);
|
|
33
|
+
currentPath.push({ x1: tx1, y1: ty1, x2: tx2, y2: ty2 });
|
|
34
|
+
}
|
|
35
|
+
function pushRectangle(rx, ry, rw, rh) {
|
|
36
|
+
const effH = Math.abs(rh) * Math.hypot(ctm[2], ctm[3]);
|
|
37
|
+
const effW = Math.abs(rw) * Math.hypot(ctm[0], ctm[1]);
|
|
38
|
+
if (effH < ORIENTATION_TOL * 2) {
|
|
39
|
+
pushSeg(rx, ry + rh / 2, rx + rw, ry + rh / 2);
|
|
40
|
+
} else if (effW < ORIENTATION_TOL * 2) {
|
|
41
|
+
pushSeg(rx + rw / 2, ry, rx + rw / 2, ry + rh);
|
|
37
42
|
} else {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
{ x1: rx, y1: ry + rh, x2: rx, y2: ry }
|
|
43
|
-
);
|
|
43
|
+
pushSeg(rx, ry, rx + rw, ry);
|
|
44
|
+
pushSeg(rx + rw, ry, rx + rw, ry + rh);
|
|
45
|
+
pushSeg(rx + rw, ry + rh, rx, ry + rh);
|
|
46
|
+
pushSeg(rx, ry + rh, rx, ry);
|
|
44
47
|
}
|
|
45
48
|
}
|
|
46
49
|
function flushPath(isStroke) {
|
|
@@ -48,8 +51,9 @@ function extractLines(fnArray, argsArray) {
|
|
|
48
51
|
currentPath = [];
|
|
49
52
|
return;
|
|
50
53
|
}
|
|
54
|
+
const effWidth = lineWidth * ctmScale();
|
|
51
55
|
for (const seg of currentPath) {
|
|
52
|
-
classifyAndAdd(seg,
|
|
56
|
+
classifyAndAdd(seg, effWidth, horizontals, verticals);
|
|
53
57
|
}
|
|
54
58
|
currentPath = [];
|
|
55
59
|
}
|
|
@@ -60,6 +64,24 @@ function extractLines(fnArray, argsArray) {
|
|
|
60
64
|
case OPS.setLineWidth:
|
|
61
65
|
lineWidth = args[0] || 1;
|
|
62
66
|
break;
|
|
67
|
+
case OPS.save:
|
|
68
|
+
ctmStack.push(ctm.slice());
|
|
69
|
+
break;
|
|
70
|
+
case OPS.restore:
|
|
71
|
+
ctm = ctmStack.pop() ?? [1, 0, 0, 1, 0, 0];
|
|
72
|
+
break;
|
|
73
|
+
case OPS.transform: {
|
|
74
|
+
const t = args;
|
|
75
|
+
ctm = [
|
|
76
|
+
ctm[0] * t[0] + ctm[2] * t[1],
|
|
77
|
+
ctm[1] * t[0] + ctm[3] * t[1],
|
|
78
|
+
ctm[0] * t[2] + ctm[2] * t[3],
|
|
79
|
+
ctm[1] * t[2] + ctm[3] * t[3],
|
|
80
|
+
ctm[0] * t[4] + ctm[2] * t[5] + ctm[4],
|
|
81
|
+
ctm[1] * t[4] + ctm[3] * t[5] + ctm[5]
|
|
82
|
+
];
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
63
85
|
case OPS.constructPath: {
|
|
64
86
|
const arg0 = args[0];
|
|
65
87
|
if (Array.isArray(arg0)) {
|
|
@@ -74,16 +96,16 @@ function extractLines(fnArray, argsArray) {
|
|
|
74
96
|
pathStartY = curY;
|
|
75
97
|
} else if (subOp === OPS.lineTo) {
|
|
76
98
|
const x2 = coords[ci++], y2 = coords[ci++];
|
|
77
|
-
|
|
99
|
+
pushSeg(curX, curY, x2, y2);
|
|
78
100
|
curX = x2;
|
|
79
101
|
curY = y2;
|
|
80
102
|
} else if (subOp === OPS.rectangle) {
|
|
81
103
|
const rx = coords[ci++], ry = coords[ci++];
|
|
82
104
|
const rw = coords[ci++], rh = coords[ci++];
|
|
83
|
-
pushRectangle(
|
|
105
|
+
pushRectangle(rx, ry, rw, rh);
|
|
84
106
|
} else if (subOp === OPS.closePath) {
|
|
85
107
|
if (curX !== pathStartX || curY !== pathStartY) {
|
|
86
|
-
|
|
108
|
+
pushSeg(curX, curY, pathStartX, pathStartY);
|
|
87
109
|
}
|
|
88
110
|
curX = pathStartX;
|
|
89
111
|
curY = pathStartY;
|
|
@@ -109,7 +131,7 @@ function extractLines(fnArray, argsArray) {
|
|
|
109
131
|
pathStartY = curY;
|
|
110
132
|
} else if (drawOp === 1 /* lineTo */) {
|
|
111
133
|
const x2 = pathData[di++], y2 = pathData[di++];
|
|
112
|
-
|
|
134
|
+
pushSeg(curX, curY, x2, y2);
|
|
113
135
|
curX = x2;
|
|
114
136
|
curY = y2;
|
|
115
137
|
} else if (drawOp === 2 /* curveTo */) {
|
|
@@ -118,7 +140,7 @@ function extractLines(fnArray, argsArray) {
|
|
|
118
140
|
di += 4;
|
|
119
141
|
} else if (drawOp === 4 /* closePath */) {
|
|
120
142
|
if (curX !== pathStartX || curY !== pathStartY) {
|
|
121
|
-
|
|
143
|
+
pushSeg(curX, curY, pathStartX, pathStartY);
|
|
122
144
|
}
|
|
123
145
|
curX = pathStartX;
|
|
124
146
|
curY = pathStartY;
|
|
@@ -156,55 +178,6 @@ function extractLines(fnArray, argsArray) {
|
|
|
156
178
|
}
|
|
157
179
|
return { horizontals, verticals };
|
|
158
180
|
}
|
|
159
|
-
function multiplyTransform(m, t) {
|
|
160
|
-
return [
|
|
161
|
-
m[0] * t[0] + m[2] * t[1],
|
|
162
|
-
m[1] * t[0] + m[3] * t[1],
|
|
163
|
-
m[0] * t[2] + m[2] * t[3],
|
|
164
|
-
m[1] * t[2] + m[3] * t[3],
|
|
165
|
-
m[0] * t[4] + m[2] * t[5] + m[4],
|
|
166
|
-
m[1] * t[4] + m[3] * t[5] + m[5]
|
|
167
|
-
];
|
|
168
|
-
}
|
|
169
|
-
function extractImageRegions(fnArray, argsArray) {
|
|
170
|
-
const regions = [];
|
|
171
|
-
let ctm = [1, 0, 0, 1, 0, 0];
|
|
172
|
-
const stack = [];
|
|
173
|
-
for (let i = 0; i < fnArray.length; i++) {
|
|
174
|
-
const op = fnArray[i];
|
|
175
|
-
switch (op) {
|
|
176
|
-
case OPS.save:
|
|
177
|
-
stack.push(ctm);
|
|
178
|
-
break;
|
|
179
|
-
case OPS.restore:
|
|
180
|
-
ctm = stack.pop() || [1, 0, 0, 1, 0, 0];
|
|
181
|
-
break;
|
|
182
|
-
case OPS.transform: {
|
|
183
|
-
const t = argsArray[i];
|
|
184
|
-
if (Array.isArray(t) && t.length >= 6) ctm = multiplyTransform(ctm, t);
|
|
185
|
-
break;
|
|
186
|
-
}
|
|
187
|
-
case OPS.paintImageXObject:
|
|
188
|
-
case OPS.paintInlineImageXObject:
|
|
189
|
-
case OPS.paintImageMaskXObject:
|
|
190
|
-
case OPS.paintImageXObjectRepeat: {
|
|
191
|
-
const corners = [[0, 0], [1, 0], [0, 1], [1, 1]];
|
|
192
|
-
let x1 = Infinity, y1 = Infinity, x2 = -Infinity, y2 = -Infinity;
|
|
193
|
-
for (const [u, v] of corners) {
|
|
194
|
-
const x = ctm[0] * u + ctm[2] * v + ctm[4];
|
|
195
|
-
const y = ctm[1] * u + ctm[3] * v + ctm[5];
|
|
196
|
-
if (x < x1) x1 = x;
|
|
197
|
-
if (x > x2) x2 = x;
|
|
198
|
-
if (y < y1) y1 = y;
|
|
199
|
-
if (y > y2) y2 = y;
|
|
200
|
-
}
|
|
201
|
-
if (x2 - x1 > 0 && y2 - y1 > 0) regions.push({ x1, y1, x2, y2 });
|
|
202
|
-
break;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
return regions;
|
|
207
|
-
}
|
|
208
181
|
function classifyAndAdd(seg, lineWidth, horizontals, verticals) {
|
|
209
182
|
const dx = Math.abs(seg.x2 - seg.x1);
|
|
210
183
|
const dy = Math.abs(seg.y2 - seg.y1);
|
|
@@ -282,6 +255,67 @@ function filterPageBorderLines(horizontals, verticals, pageWidth, pageHeight) {
|
|
|
282
255
|
)
|
|
283
256
|
};
|
|
284
257
|
}
|
|
258
|
+
|
|
259
|
+
// src/pdf/image-regions.ts
|
|
260
|
+
import { OPS as OPS2 } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
261
|
+
function multiplyTransform(m, t) {
|
|
262
|
+
return [
|
|
263
|
+
m[0] * t[0] + m[2] * t[1],
|
|
264
|
+
m[1] * t[0] + m[3] * t[1],
|
|
265
|
+
m[0] * t[2] + m[2] * t[3],
|
|
266
|
+
m[1] * t[2] + m[3] * t[3],
|
|
267
|
+
m[0] * t[4] + m[2] * t[5] + m[4],
|
|
268
|
+
m[1] * t[4] + m[3] * t[5] + m[5]
|
|
269
|
+
];
|
|
270
|
+
}
|
|
271
|
+
function extractImageRegions(fnArray, argsArray) {
|
|
272
|
+
const regions = [];
|
|
273
|
+
let ctm = [1, 0, 0, 1, 0, 0];
|
|
274
|
+
const stack = [];
|
|
275
|
+
for (let i = 0; i < fnArray.length; i++) {
|
|
276
|
+
const op = fnArray[i];
|
|
277
|
+
switch (op) {
|
|
278
|
+
case OPS2.save:
|
|
279
|
+
stack.push(ctm);
|
|
280
|
+
break;
|
|
281
|
+
case OPS2.restore:
|
|
282
|
+
ctm = stack.pop() || [1, 0, 0, 1, 0, 0];
|
|
283
|
+
break;
|
|
284
|
+
case OPS2.transform: {
|
|
285
|
+
const t = argsArray[i];
|
|
286
|
+
if (Array.isArray(t) && t.length >= 6) ctm = multiplyTransform(ctm, t);
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
case OPS2.paintImageXObject:
|
|
290
|
+
case OPS2.paintInlineImageXObject:
|
|
291
|
+
case OPS2.paintImageMaskXObject:
|
|
292
|
+
case OPS2.paintImageXObjectRepeat: {
|
|
293
|
+
const corners = [[0, 0], [1, 0], [0, 1], [1, 1]];
|
|
294
|
+
let x1 = Infinity, y1 = Infinity, x2 = -Infinity, y2 = -Infinity;
|
|
295
|
+
for (const [u, v] of corners) {
|
|
296
|
+
const x = ctm[0] * u + ctm[2] * v + ctm[4];
|
|
297
|
+
const y = ctm[1] * u + ctm[3] * v + ctm[5];
|
|
298
|
+
if (x < x1) x1 = x;
|
|
299
|
+
if (x > x2) x2 = x;
|
|
300
|
+
if (y < y1) y1 = y;
|
|
301
|
+
if (y > y2) y2 = y;
|
|
302
|
+
}
|
|
303
|
+
if (x2 - x1 > 0 && y2 - y1 > 0) regions.push({ x1, y1, x2, y2 });
|
|
304
|
+
break;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return regions;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// src/pdf/line-types.ts
|
|
312
|
+
var VERTEX_MERGE_FACTOR = 4;
|
|
313
|
+
|
|
314
|
+
// src/pdf/table-grid.ts
|
|
315
|
+
var CONNECT_TOL = 5;
|
|
316
|
+
var MIN_COL_WIDTH = 15;
|
|
317
|
+
var MIN_ROW_HEIGHT = 6;
|
|
318
|
+
var MIN_COORD_MERGE_TOL = 8;
|
|
285
319
|
function buildVertices(horizontals, verticals) {
|
|
286
320
|
const vertices = [];
|
|
287
321
|
const tol = CONNECT_TOL;
|
|
@@ -493,6 +527,8 @@ function linesIntersect(a, b) {
|
|
|
493
527
|
const tol = CONNECT_TOL;
|
|
494
528
|
return v.x1 >= h.x1 - tol && v.x1 <= h.x2 + tol && h.y1 >= v.y1 - tol && h.y1 <= v.y2 + tol;
|
|
495
529
|
}
|
|
530
|
+
|
|
531
|
+
// src/pdf/cell-extract.ts
|
|
496
532
|
function extractCells(grid, horizontals, verticals) {
|
|
497
533
|
const { rowYs, colXs } = grid;
|
|
498
534
|
const numRows = rowYs.length - 1;
|
|
@@ -590,6 +626,9 @@ function hasHorizontalLine(horizontals, y, leftX, rightX, vertexRadius) {
|
|
|
590
626
|
}
|
|
591
627
|
return false;
|
|
592
628
|
}
|
|
629
|
+
|
|
630
|
+
// src/pdf/cell-text.ts
|
|
631
|
+
var CELL_PADDING = 2;
|
|
593
632
|
var SPACE_GAP_RATIO = 0.17;
|
|
594
633
|
function spaceGapThreshold(fontSize) {
|
|
595
634
|
return Math.max(fontSize * SPACE_GAP_RATIO, 1);
|
|
@@ -718,6 +757,28 @@ function markEvenRun(items, result, start, end) {
|
|
|
718
757
|
}
|
|
719
758
|
}
|
|
720
759
|
}
|
|
760
|
+
function mergeCellTextLines(textLines) {
|
|
761
|
+
if (textLines.length <= 1) return textLines[0] || "";
|
|
762
|
+
const merged = [textLines[0]];
|
|
763
|
+
for (let i = 1; i < textLines.length; i++) {
|
|
764
|
+
const prev = merged[merged.length - 1];
|
|
765
|
+
const curr = textLines[i];
|
|
766
|
+
if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
|
|
767
|
+
merged[merged.length - 1] = prev + curr;
|
|
768
|
+
} else if (curr.trim().length <= 3 && /^[)\]%}]/.test(curr.trim())) {
|
|
769
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
770
|
+
} else if (/[,(]$/.test(prev.trim()) && curr.trim().length <= 15) {
|
|
771
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
772
|
+
} else if (/[\d,]$/.test(prev) && /^[\d,]+[)\]]?$/.test(curr.trim()) && curr.trim().length <= 10) {
|
|
773
|
+
merged[merged.length - 1] = prev + curr.trim();
|
|
774
|
+
} else {
|
|
775
|
+
merged.push(curr);
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return merged.join("\n");
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// src/pdf/undersegmented.ts
|
|
721
782
|
var MAX_UNDERSEGMENTED_ROWS = 2;
|
|
722
783
|
var MIN_UNDERSEGMENTED_COLUMNS = 3;
|
|
723
784
|
var MIN_UNDERSEGMENTED_TEXT_LINES = 8;
|
|
@@ -831,977 +892,822 @@ function normalizeUndersegmentedTable(originalCells, colXs, items) {
|
|
|
831
892
|
if (countNonEmptyCols(rebuilt, numCols) < countNonEmptyCols(originalCells, numCols)) return null;
|
|
832
893
|
return rebuilt;
|
|
833
894
|
}
|
|
834
|
-
function mergeCellTextLines(textLines) {
|
|
835
|
-
if (textLines.length <= 1) return textLines[0] || "";
|
|
836
|
-
const merged = [textLines[0]];
|
|
837
|
-
for (let i = 1; i < textLines.length; i++) {
|
|
838
|
-
const prev = merged[merged.length - 1];
|
|
839
|
-
const curr = textLines[i];
|
|
840
|
-
if (/[가-힣]$/.test(prev) && /^[가-힣]+$/.test(curr) && curr.length <= 8 && !curr.includes(" ")) {
|
|
841
|
-
merged[merged.length - 1] = prev + curr;
|
|
842
|
-
} else if (curr.trim().length <= 3 && /^[)\]%}]/.test(curr.trim())) {
|
|
843
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
844
|
-
} else if (/[,(]$/.test(prev.trim()) && curr.trim().length <= 15) {
|
|
845
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
846
|
-
} else if (/[\d,]$/.test(prev) && /^[\d,]+[)\]]?$/.test(curr.trim()) && curr.trim().length <= 10) {
|
|
847
|
-
merged[merged.length - 1] = prev + curr.trim();
|
|
848
|
-
} else {
|
|
849
|
-
merged.push(curr);
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
return merged.join("\n");
|
|
853
|
-
}
|
|
854
895
|
|
|
855
|
-
// src/pdf/
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
const headerResult = detectHeaderRow(rows);
|
|
870
|
-
if (headerResult) {
|
|
871
|
-
const { columns, headerIdx } = headerResult;
|
|
872
|
-
const headerRow = rows[headerIdx];
|
|
873
|
-
const headerItems = [...headerRow.items].sort((a, b) => a.x - b.x);
|
|
874
|
-
const headerAndBelow = rows.slice(headerIdx);
|
|
875
|
-
const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
|
|
876
|
-
const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
|
|
877
|
-
for (const region of tableRegions) {
|
|
878
|
-
const table = buildClusterTable(region.rows, columns, pageNum);
|
|
879
|
-
if (table) {
|
|
880
|
-
expandUsedItems(table.usedItems, originMap);
|
|
881
|
-
results.push(table);
|
|
882
|
-
}
|
|
896
|
+
// src/pdf/quality.ts
|
|
897
|
+
function computePageQuality(page, text) {
|
|
898
|
+
let total = 0;
|
|
899
|
+
let hangul = 0;
|
|
900
|
+
let control = 0;
|
|
901
|
+
let replacement = 0;
|
|
902
|
+
let pua = 0;
|
|
903
|
+
for (let i = 0; i < text.length; i++) {
|
|
904
|
+
const code = text.charCodeAt(i);
|
|
905
|
+
if (code === 32 || code === 9 || code === 10 || code === 13) continue;
|
|
906
|
+
total++;
|
|
907
|
+
if (code < 32 || code === 127 || code >= 128 && code <= 159) {
|
|
908
|
+
control++;
|
|
909
|
+
continue;
|
|
883
910
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
expandUsedItems(table.usedItems, originMap);
|
|
896
|
-
results.push(table);
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
}
|
|
911
|
+
if (code === 65533) {
|
|
912
|
+
replacement++;
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
if (code >= 44032 && code <= 55203) {
|
|
916
|
+
hangul++;
|
|
917
|
+
continue;
|
|
918
|
+
}
|
|
919
|
+
if (code >= 57344 && code <= 63743 || code >= 56192 && code <= 56319) {
|
|
920
|
+
pua++;
|
|
921
|
+
continue;
|
|
900
922
|
}
|
|
901
923
|
}
|
|
902
|
-
|
|
924
|
+
const denom = total || 1;
|
|
925
|
+
const puaRatio = pua / denom;
|
|
926
|
+
const controlCharRatio = control / denom;
|
|
927
|
+
const replacementCharRatio = replacement / denom;
|
|
928
|
+
let needsOcr = false;
|
|
929
|
+
let ocrReason;
|
|
930
|
+
if (total < LOW_TEXT_THRESHOLD) {
|
|
931
|
+
needsOcr = true;
|
|
932
|
+
ocrReason = "low_text";
|
|
933
|
+
} else if (puaRatio >= HIGH_PUA_THRESHOLD) {
|
|
934
|
+
needsOcr = true;
|
|
935
|
+
ocrReason = "high_pua";
|
|
936
|
+
} else if (controlCharRatio >= HIGH_CONTROL_THRESHOLD) {
|
|
937
|
+
needsOcr = true;
|
|
938
|
+
ocrReason = "high_control";
|
|
939
|
+
} else if (replacementCharRatio >= HIGH_REPLACEMENT_THRESHOLD) {
|
|
940
|
+
needsOcr = true;
|
|
941
|
+
ocrReason = "high_replacement";
|
|
942
|
+
}
|
|
943
|
+
return {
|
|
944
|
+
page,
|
|
945
|
+
textChars: total,
|
|
946
|
+
hangulRatio: hangul / denom,
|
|
947
|
+
controlCharRatio,
|
|
948
|
+
replacementCharRatio,
|
|
949
|
+
puaRatio,
|
|
950
|
+
needsOcr,
|
|
951
|
+
ocrReason
|
|
952
|
+
};
|
|
903
953
|
}
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
while (i < sorted.length) {
|
|
912
|
-
if (/^[가-힣\d]$/.test(sorted[i].text)) {
|
|
913
|
-
let runEnd = i + 1;
|
|
914
|
-
while (runEnd < sorted.length && /^[가-힣\d]$/.test(sorted[runEnd].text)) {
|
|
915
|
-
if (sorted[runEnd].hasSpaceBefore) break;
|
|
916
|
-
const gap = sorted[runEnd].x - (sorted[runEnd - 1].x + sorted[runEnd - 1].w);
|
|
917
|
-
const fs = sorted[runEnd].fontSize;
|
|
918
|
-
if (gap < fs * 0.1 || gap > fs * 3) break;
|
|
919
|
-
runEnd++;
|
|
920
|
-
}
|
|
921
|
-
if (runEnd - i >= 3) {
|
|
922
|
-
const gaps = [];
|
|
923
|
-
for (let g2 = i + 1; g2 < runEnd; g2++) {
|
|
924
|
-
gaps.push(sorted[g2].x - (sorted[g2 - 1].x + sorted[g2 - 1].w));
|
|
925
|
-
}
|
|
926
|
-
let minG = Infinity, maxG = -Infinity;
|
|
927
|
-
for (const g2 of gaps) {
|
|
928
|
-
if (g2 < minG) minG = g2;
|
|
929
|
-
if (g2 > maxG) maxG = g2;
|
|
930
|
-
}
|
|
931
|
-
if (minG > 0 && maxG / minG <= 3) {
|
|
932
|
-
const run = sorted.slice(i, runEnd);
|
|
933
|
-
const text = run.map((r) => r.text).join("");
|
|
934
|
-
const first = run[0], last = run[runEnd - i - 1];
|
|
935
|
-
const item = {
|
|
936
|
-
text,
|
|
937
|
-
x: first.x,
|
|
938
|
-
y: first.y,
|
|
939
|
-
w: last.x + last.w - first.x,
|
|
940
|
-
h: first.h,
|
|
941
|
-
fontSize: first.fontSize,
|
|
942
|
-
fontName: first.fontName
|
|
943
|
-
};
|
|
944
|
-
originMap.set(item, run);
|
|
945
|
-
merged.push(item);
|
|
946
|
-
i = runEnd;
|
|
947
|
-
continue;
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
merged.push(sorted[i]);
|
|
952
|
-
i++;
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
return { merged, originMap };
|
|
956
|
-
}
|
|
957
|
-
function expandUsedItems(usedItems, originMap) {
|
|
958
|
-
const toAdd = [];
|
|
959
|
-
for (const item of usedItems) {
|
|
960
|
-
const origins = originMap.get(item);
|
|
961
|
-
if (origins) for (const o of origins) toAdd.push(o);
|
|
962
|
-
}
|
|
963
|
-
for (const a of toAdd) usedItems.add(a);
|
|
954
|
+
var LOW_TEXT_THRESHOLD = 20;
|
|
955
|
+
var HIGH_PUA_THRESHOLD = 0.2;
|
|
956
|
+
var HIGH_CONTROL_THRESHOLD = 0.05;
|
|
957
|
+
var HIGH_REPLACEMENT_THRESHOLD = 0.05;
|
|
958
|
+
var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
|
|
959
|
+
function stripControlChars(text) {
|
|
960
|
+
return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
|
|
964
961
|
}
|
|
965
|
-
function
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
962
|
+
function summarizeDocumentQuality(pages) {
|
|
963
|
+
if (pages.length === 0) {
|
|
964
|
+
return {
|
|
965
|
+
totalPages: 0,
|
|
966
|
+
totalTextChars: 0,
|
|
967
|
+
avgHangulRatio: 0,
|
|
968
|
+
avgControlCharRatio: 0,
|
|
969
|
+
avgReplacementCharRatio: 0,
|
|
970
|
+
avgPuaRatio: 0,
|
|
971
|
+
lowTextPageCount: 0,
|
|
972
|
+
highPuaPageCount: 0,
|
|
973
|
+
needsOcr: false,
|
|
974
|
+
ocrCandidatePages: []
|
|
975
|
+
};
|
|
973
976
|
}
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
break;
|
|
992
|
-
}
|
|
993
|
-
}
|
|
994
|
-
if (!hasLargeGap) continue;
|
|
995
|
-
const columns = sorted.map((item) => ({ x: item.x, count: 0 }));
|
|
996
|
-
let matchCount = 0;
|
|
997
|
-
for (let j = ri + 1; j < rows.length && matchCount < MIN_ROWS + 2; j++) {
|
|
998
|
-
const matched = countMatchedColumnsRange(rows[j], columns, sorted);
|
|
999
|
-
if (matched >= MIN_COLS) matchCount++;
|
|
1000
|
-
}
|
|
1001
|
-
if (matchCount < MIN_ROWS) continue;
|
|
1002
|
-
return { columns, headerIdx: ri };
|
|
977
|
+
let textChars = 0;
|
|
978
|
+
let hangul = 0;
|
|
979
|
+
let control = 0;
|
|
980
|
+
let replacement = 0;
|
|
981
|
+
let pua = 0;
|
|
982
|
+
let lowText = 0;
|
|
983
|
+
let highPua = 0;
|
|
984
|
+
const ocrCandidatePages = [];
|
|
985
|
+
for (const p of pages) {
|
|
986
|
+
textChars += p.textChars;
|
|
987
|
+
hangul += p.hangulRatio;
|
|
988
|
+
control += p.controlCharRatio;
|
|
989
|
+
replacement += p.replacementCharRatio;
|
|
990
|
+
pua += p.puaRatio;
|
|
991
|
+
if (p.textChars < LOW_TEXT_THRESHOLD) lowText++;
|
|
992
|
+
if (p.puaRatio >= HIGH_PUA_THRESHOLD) highPua++;
|
|
993
|
+
if (p.needsOcr) ocrCandidatePages.push(p.page);
|
|
1003
994
|
}
|
|
1004
|
-
|
|
995
|
+
const n = pages.length;
|
|
996
|
+
return {
|
|
997
|
+
totalPages: n,
|
|
998
|
+
totalTextChars: textChars,
|
|
999
|
+
avgHangulRatio: hangul / n,
|
|
1000
|
+
avgControlCharRatio: control / n,
|
|
1001
|
+
avgReplacementCharRatio: replacement / n,
|
|
1002
|
+
avgPuaRatio: pua / n,
|
|
1003
|
+
lowTextPageCount: lowText,
|
|
1004
|
+
highPuaPageCount: highPua,
|
|
1005
|
+
needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
|
|
1006
|
+
ocrCandidatePages
|
|
1007
|
+
};
|
|
1005
1008
|
}
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
const
|
|
1017
|
-
if (
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
} else {
|
|
1021
|
-
result.push(curr);
|
|
1009
|
+
|
|
1010
|
+
// src/pdf/text-line.ts
|
|
1011
|
+
function filterHiddenText(items, pageWidth, pageHeight) {
|
|
1012
|
+
let hiddenCount = 0;
|
|
1013
|
+
const visible = [];
|
|
1014
|
+
for (const item of items) {
|
|
1015
|
+
if (item.isHidden) {
|
|
1016
|
+
hiddenCount++;
|
|
1017
|
+
continue;
|
|
1018
|
+
}
|
|
1019
|
+
const margin = Math.max(pageWidth, pageHeight) * 0.1;
|
|
1020
|
+
if (item.x < -margin || item.x > pageWidth + margin || item.y < -margin || item.y > pageHeight + margin) {
|
|
1021
|
+
hiddenCount++;
|
|
1022
|
+
continue;
|
|
1022
1023
|
}
|
|
1024
|
+
visible.push(item);
|
|
1023
1025
|
}
|
|
1024
|
-
return
|
|
1025
|
-
}
|
|
1026
|
-
function isFragmentRow(row) {
|
|
1027
|
-
return row.items.length <= 3 && row.items.every((i) => i.text.length <= 8);
|
|
1026
|
+
return { visible, hiddenCount };
|
|
1028
1027
|
}
|
|
1029
|
-
function
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
if (i.y + h > top) top = i.y + h;
|
|
1028
|
+
function collapseEvenSpacing(text) {
|
|
1029
|
+
const tokens = text.split(" ");
|
|
1030
|
+
const singleCharCount = tokens.filter((t) => t.length === 1).length;
|
|
1031
|
+
if (tokens.length >= 3 && singleCharCount / tokens.length >= 0.7) {
|
|
1032
|
+
return tokens.join("");
|
|
1035
1033
|
}
|
|
1036
|
-
return
|
|
1034
|
+
return text.replace(
|
|
1035
|
+
/(?<![가-힣])[가-힣](?: [가-힣\d]){2,}(?![가-힣])/g,
|
|
1036
|
+
(match) => match.replace(/ /g, "")
|
|
1037
|
+
);
|
|
1037
1038
|
}
|
|
1038
|
-
function
|
|
1039
|
-
|
|
1040
|
-
const
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
const
|
|
1045
|
-
|
|
1046
|
-
const yGap = Math.abs(prev.y - curr.y);
|
|
1047
|
-
const matchedCols = countMatchedColumns(curr, columns);
|
|
1048
|
-
if (yGap < avgFontSize * 1.8 && curr.items.length <= 2 && (matchedCols < MIN_COLS || curr.items.length === 1)) {
|
|
1049
|
-
result[result.length - 1] = {
|
|
1050
|
-
y: prev.y,
|
|
1051
|
-
items: [...prev.items, ...curr.items]
|
|
1052
|
-
};
|
|
1053
|
-
} else {
|
|
1054
|
-
result.push(curr);
|
|
1055
|
-
}
|
|
1039
|
+
function computeBBox(items, pageNum) {
|
|
1040
|
+
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
1041
|
+
for (const i of items) {
|
|
1042
|
+
if (i.x < minX) minX = i.x;
|
|
1043
|
+
if (i.y < minY) minY = i.y;
|
|
1044
|
+
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
1045
|
+
const effectiveH = i.h > 0 ? i.h : i.fontSize;
|
|
1046
|
+
if (i.y + effectiveH > maxY) maxY = i.y + effectiveH;
|
|
1056
1047
|
}
|
|
1057
|
-
return
|
|
1048
|
+
return { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY };
|
|
1058
1049
|
}
|
|
1059
|
-
function
|
|
1060
|
-
if (items.length === 0) return
|
|
1061
|
-
const
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
curItems = [sorted[i]];
|
|
1071
|
-
curY = sorted[i].y;
|
|
1050
|
+
function dominantStyle(items) {
|
|
1051
|
+
if (items.length === 0) return void 0;
|
|
1052
|
+
const freq = /* @__PURE__ */ new Map();
|
|
1053
|
+
let maxCount = 0, dominantSize = 0;
|
|
1054
|
+
for (const i of items) {
|
|
1055
|
+
if (i.fontSize <= 0) continue;
|
|
1056
|
+
const count = (freq.get(i.fontSize) || 0) + 1;
|
|
1057
|
+
freq.set(i.fontSize, count);
|
|
1058
|
+
if (count > maxCount) {
|
|
1059
|
+
maxCount = count;
|
|
1060
|
+
dominantSize = i.fontSize;
|
|
1072
1061
|
}
|
|
1073
1062
|
}
|
|
1074
|
-
if (
|
|
1075
|
-
|
|
1063
|
+
if (dominantSize === 0) return void 0;
|
|
1064
|
+
const fontName = items.find((i) => i.fontSize === dominantSize)?.fontName || void 0;
|
|
1065
|
+
return { fontSize: dominantSize, fontName };
|
|
1076
1066
|
}
|
|
1077
|
-
function
|
|
1078
|
-
|
|
1079
|
-
const
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
return false;
|
|
1088
|
-
}
|
|
1089
|
-
function extractColumnClusters(rows) {
|
|
1090
|
-
const allX = [];
|
|
1091
|
-
for (const row of rows) {
|
|
1092
|
-
for (const item of row.items) allX.push(item.x);
|
|
1093
|
-
}
|
|
1094
|
-
if (allX.length === 0) return [];
|
|
1095
|
-
allX.sort((a, b) => a - b);
|
|
1096
|
-
const clusters = [];
|
|
1097
|
-
let clusterStart = 0;
|
|
1098
|
-
for (let i = 1; i <= allX.length; i++) {
|
|
1099
|
-
if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
|
|
1100
|
-
const slice = allX.slice(clusterStart, i);
|
|
1101
|
-
const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
|
|
1102
|
-
clusters.push({ x: avg, count: slice.length });
|
|
1103
|
-
clusterStart = i;
|
|
1067
|
+
function normalizeItems(rawItems) {
|
|
1068
|
+
const items = [];
|
|
1069
|
+
const spacePositions = [];
|
|
1070
|
+
for (const i of rawItems) {
|
|
1071
|
+
if (typeof i.str !== "string") continue;
|
|
1072
|
+
const x = Math.round(i.transform[4]);
|
|
1073
|
+
const y = Math.round(i.transform[5]);
|
|
1074
|
+
if (!i.str.trim()) {
|
|
1075
|
+
spacePositions.push({ x, y });
|
|
1076
|
+
continue;
|
|
1104
1077
|
}
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
const matchedCols = countMatchedColumnsRange(row, columns, headerItems);
|
|
1115
|
-
if (matchedCols >= MIN_COLS) {
|
|
1116
|
-
currentRegion.push(row);
|
|
1117
|
-
missStreak = 0;
|
|
1118
|
-
} else if (currentRegion.length > 0 && (row.items.length <= 2 || missStreak === 0)) {
|
|
1119
|
-
currentRegion.push(row);
|
|
1120
|
-
missStreak++;
|
|
1121
|
-
} else {
|
|
1122
|
-
while (currentRegion.length > 0) {
|
|
1123
|
-
const last = currentRegion[currentRegion.length - 1];
|
|
1124
|
-
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1125
|
-
currentRegion.pop();
|
|
1126
|
-
}
|
|
1127
|
-
if (currentRegion.length >= MIN_ROWS) {
|
|
1128
|
-
regions.push({ rows: [...currentRegion] });
|
|
1129
|
-
}
|
|
1130
|
-
currentRegion = [];
|
|
1131
|
-
missStreak = 0;
|
|
1078
|
+
const scaleX = Math.hypot(i.transform[0], i.transform[1]);
|
|
1079
|
+
const scaleY = Math.hypot(i.transform[2], i.transform[3]);
|
|
1080
|
+
const fontSize = Math.round(Math.max(scaleY, scaleX));
|
|
1081
|
+
const w = Math.round(i.width);
|
|
1082
|
+
const h = Math.round(i.height);
|
|
1083
|
+
const isHidden = fontSize === 0 || i.width === 0 && i.str.trim().length > 0;
|
|
1084
|
+
let text = i.str.trim();
|
|
1085
|
+
if (/^[\d\s\-().·,☎]+$/.test(text) && /\d/.test(text) && / /.test(text)) {
|
|
1086
|
+
text = text.replace(/ /g, "");
|
|
1132
1087
|
}
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
currentRegion.pop();
|
|
1138
|
-
}
|
|
1139
|
-
if (currentRegion.length >= MIN_ROWS) {
|
|
1140
|
-
regions.push({ rows: currentRegion });
|
|
1141
|
-
}
|
|
1142
|
-
return regions;
|
|
1143
|
-
}
|
|
1144
|
-
function findTableRegions(allRows, columns) {
|
|
1145
|
-
const regions = [];
|
|
1146
|
-
let currentRegion = [];
|
|
1147
|
-
for (const row of allRows) {
|
|
1148
|
-
const matchedCols = countMatchedColumns(row, columns);
|
|
1149
|
-
if (matchedCols >= MIN_COLS) {
|
|
1150
|
-
currentRegion.push(row);
|
|
1151
|
-
} else if (row.items.length === 1) {
|
|
1152
|
-
if (currentRegion.length > 0) {
|
|
1153
|
-
currentRegion.push(row);
|
|
1088
|
+
const split = splitEvenSpacedItem(text, x, w, fontSize);
|
|
1089
|
+
if (split) {
|
|
1090
|
+
for (const s of split) {
|
|
1091
|
+
items.push({ text: s.text, x: s.x, y, w: s.w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
1154
1092
|
}
|
|
1155
1093
|
} else {
|
|
1156
|
-
|
|
1157
|
-
regions.push({ rows: [...currentRegion] });
|
|
1158
|
-
}
|
|
1159
|
-
currentRegion = [];
|
|
1094
|
+
items.push({ text, x, y, w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
1160
1095
|
}
|
|
1161
1096
|
}
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
|
|
1172
|
-
matched.add(ci);
|
|
1097
|
+
const sorted = items.sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1098
|
+
const deduped = [];
|
|
1099
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
1100
|
+
let isDup = false;
|
|
1101
|
+
for (let j = deduped.length - 1; j >= 0; j--) {
|
|
1102
|
+
const prev = deduped[j];
|
|
1103
|
+
if (prev.y - sorted[i].y > 3) break;
|
|
1104
|
+
if (Math.abs(prev.y - sorted[i].y) <= 3 && prev.text === sorted[i].text && Math.abs(prev.x - sorted[i].x) <= 3) {
|
|
1105
|
+
isDup = true;
|
|
1173
1106
|
break;
|
|
1174
1107
|
}
|
|
1175
1108
|
}
|
|
1109
|
+
if (!isDup) deduped.push(sorted[i]);
|
|
1176
1110
|
}
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
const matched = /* @__PURE__ */ new Set();
|
|
1187
|
-
for (const item of row.items) {
|
|
1188
|
-
for (let ci = 0; ci < boundaries.length; ci++) {
|
|
1189
|
-
if (item.x >= boundaries[ci].left && item.x < boundaries[ci].right) {
|
|
1190
|
-
matched.add(ci);
|
|
1191
|
-
break;
|
|
1111
|
+
if (spacePositions.length > 0) {
|
|
1112
|
+
for (const sp of spacePositions) {
|
|
1113
|
+
let nearest = null;
|
|
1114
|
+
for (const item of deduped) {
|
|
1115
|
+
if (Math.abs(sp.y - item.y) > 3) continue;
|
|
1116
|
+
const dist = item.x - sp.x;
|
|
1117
|
+
if (dist >= -1 && dist <= 20 && (!nearest || item.x < nearest.x)) {
|
|
1118
|
+
nearest = item;
|
|
1119
|
+
}
|
|
1192
1120
|
}
|
|
1121
|
+
if (nearest) nearest.hasSpaceBefore = true;
|
|
1193
1122
|
}
|
|
1194
1123
|
}
|
|
1195
|
-
return
|
|
1124
|
+
return deduped;
|
|
1196
1125
|
}
|
|
1197
|
-
function
|
|
1126
|
+
function splitEvenSpacedItem(text, itemX, itemW, fontSize) {
|
|
1127
|
+
if (!/^[가-힣\d](?: [가-힣\d]){2,}$/.test(text)) return null;
|
|
1128
|
+
const chars = text.split(" ");
|
|
1129
|
+
if (chars.length < 3) return null;
|
|
1130
|
+
const charW = itemW / chars.length;
|
|
1131
|
+
if (charW > fontSize * 2) return null;
|
|
1132
|
+
return chars.map((ch, idx) => ({
|
|
1133
|
+
text: ch,
|
|
1134
|
+
x: Math.round(itemX + idx * charW),
|
|
1135
|
+
w: Math.round(charW * 0.8)
|
|
1136
|
+
// 실제 글자 폭은 간격보다 좁음
|
|
1137
|
+
}));
|
|
1138
|
+
}
|
|
1139
|
+
function groupByY(items) {
|
|
1198
1140
|
if (items.length === 0) return [];
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
for (let i = 1; i <
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
const gapThreshold = sorted.length <= numCols + 1 ? 12 : Math.max(medianGap * 2.5, 12);
|
|
1208
|
-
const significantGaps = gaps.filter((g2) => g2.size >= gapThreshold).sort((a, b) => b.size - a.size).slice(0, numCols - 1).sort((a, b) => a.idx - b.idx);
|
|
1209
|
-
const groups = [];
|
|
1210
|
-
let start = 0;
|
|
1211
|
-
for (const gap of significantGaps) {
|
|
1212
|
-
groups.push(sorted.slice(start, gap.idx));
|
|
1213
|
-
start = gap.idx;
|
|
1214
|
-
}
|
|
1215
|
-
groups.push(sorted.slice(start));
|
|
1216
|
-
const result = [];
|
|
1217
|
-
const usedCols = /* @__PURE__ */ new Set();
|
|
1218
|
-
const groupCenters = groups.map((g2) => {
|
|
1219
|
-
let minX = Infinity, maxX = -Infinity;
|
|
1220
|
-
for (const i of g2) {
|
|
1221
|
-
if (i.x < minX) minX = i.x;
|
|
1222
|
-
const r = i.x + i.w;
|
|
1223
|
-
if (r > maxX) maxX = r;
|
|
1224
|
-
}
|
|
1225
|
-
return (minX + maxX) / 2;
|
|
1226
|
-
});
|
|
1227
|
-
const assignments = [];
|
|
1228
|
-
for (let gi = 0; gi < groups.length; gi++) {
|
|
1229
|
-
for (let ci = 0; ci < numCols; ci++) {
|
|
1230
|
-
assignments.push({ gi, ci, dist: Math.abs(groupCenters[gi] - colCenters[ci]) });
|
|
1141
|
+
const lines = [];
|
|
1142
|
+
let curY = items[0].y;
|
|
1143
|
+
let curLine = [items[0]];
|
|
1144
|
+
for (let i = 1; i < items.length; i++) {
|
|
1145
|
+
if (Math.abs(items[i].y - curY) > 3) {
|
|
1146
|
+
lines.push(curLine);
|
|
1147
|
+
curLine = [];
|
|
1148
|
+
curY = items[i].y;
|
|
1231
1149
|
}
|
|
1150
|
+
curLine.push(items[i]);
|
|
1232
1151
|
}
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1152
|
+
if (curLine.length > 0) lines.push(curLine);
|
|
1153
|
+
return lines;
|
|
1154
|
+
}
|
|
1155
|
+
function mergeSuperscriptLines(lines) {
|
|
1156
|
+
if (lines.length <= 1) return lines;
|
|
1157
|
+
const band = (line) => {
|
|
1158
|
+
let bottom = Infinity, top = -Infinity;
|
|
1159
|
+
for (const i of line) {
|
|
1160
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1161
|
+
if (i.y < bottom) bottom = i.y;
|
|
1162
|
+
if (i.y + h > top) top = i.y + h;
|
|
1163
|
+
}
|
|
1164
|
+
return { bottom, top, height: top - bottom };
|
|
1165
|
+
};
|
|
1166
|
+
const isFrag = (line) => line.length <= 3 && line.every((i) => i.text.trim().length <= 8);
|
|
1167
|
+
const result = [lines[0]];
|
|
1168
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1169
|
+
const prev = result[result.length - 1];
|
|
1170
|
+
const curr = lines[i];
|
|
1171
|
+
const a = band(prev);
|
|
1172
|
+
const b = band(curr);
|
|
1173
|
+
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1174
|
+
const prevIsFrag = isFrag(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1175
|
+
const currIsFrag = isFrag(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1176
|
+
if (prevIsFrag || currIsFrag) {
|
|
1177
|
+
result[result.length - 1] = [...prev, ...curr];
|
|
1178
|
+
} else {
|
|
1179
|
+
result.push(curr);
|
|
1250
1180
|
}
|
|
1251
|
-
result.push({ col: bestCol, items: groups[gi] });
|
|
1252
1181
|
}
|
|
1253
1182
|
return result;
|
|
1254
1183
|
}
|
|
1255
|
-
function
|
|
1256
|
-
|
|
1257
|
-
const
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
cells[r][0] = { text: row.items[0].text, colSpan: numCols, rowSpan: 1 };
|
|
1268
|
-
usedItems.add(row.items[0]);
|
|
1184
|
+
function mergeLineSimple(items) {
|
|
1185
|
+
if (items.length <= 1) return items[0]?.text || "";
|
|
1186
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1187
|
+
const isEvenSpaced = detectEvenSpacedItems(sorted);
|
|
1188
|
+
let result = sorted[0].text;
|
|
1189
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1190
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1191
|
+
const avgFs = (sorted[i].fontSize + sorted[i - 1].fontSize) / 2;
|
|
1192
|
+
const tabThreshold = Math.max(avgFs * 2, 30);
|
|
1193
|
+
if (gap > tabThreshold) {
|
|
1194
|
+
result += " ";
|
|
1195
|
+
result += sorted[i].text;
|
|
1269
1196
|
continue;
|
|
1270
1197
|
}
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
const existing = cells[r][col].text;
|
|
1275
|
-
cells[r][col].text = existing ? existing + " " + text : text;
|
|
1276
|
-
for (const item of items) usedItems.add(item);
|
|
1198
|
+
if (isEvenSpaced[i]) {
|
|
1199
|
+
result += sorted[i].text;
|
|
1200
|
+
continue;
|
|
1277
1201
|
}
|
|
1202
|
+
if (sorted[i].hasSpaceBefore && gap >= avgFs * 0.05) {
|
|
1203
|
+
result += " ";
|
|
1204
|
+
result += sorted[i].text;
|
|
1205
|
+
continue;
|
|
1206
|
+
}
|
|
1207
|
+
if (/[□■○●▶◆◇ㅇ]$/.test(sorted[i - 1].text) && /^[가-힣]/.test(sorted[i].text) && gap > 1) {
|
|
1208
|
+
result += " ";
|
|
1209
|
+
result += sorted[i].text;
|
|
1210
|
+
continue;
|
|
1211
|
+
}
|
|
1212
|
+
if (gap > spaceGapThreshold(avgFs)) result += " ";
|
|
1213
|
+
result += sorted[i].text;
|
|
1278
1214
|
}
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1215
|
+
return result;
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
// src/pdf/cluster-detector.ts
|
|
1219
|
+
var Y_TOL = 3;
|
|
1220
|
+
var COL_CLUSTER_TOL = 15;
|
|
1221
|
+
var MIN_ROWS = 3;
|
|
1222
|
+
var MIN_COLS = 2;
|
|
1223
|
+
var MIN_GAP_FACTOR = 2;
|
|
1224
|
+
var MIN_GAP_ABSOLUTE = 20;
|
|
1225
|
+
var MIN_COL_FILL_RATIO = 0.4;
|
|
1226
|
+
function detectClusterTables(items, pageNum) {
|
|
1227
|
+
if (items.length < MIN_ROWS * MIN_COLS) return [];
|
|
1228
|
+
const { merged, originMap } = mergeEvenSpacedClusters(items);
|
|
1229
|
+
const rows = mergeOverlappingRows(groupByBaseline(merged));
|
|
1230
|
+
if (rows.length < MIN_ROWS) return [];
|
|
1231
|
+
const results = [];
|
|
1232
|
+
const headerResult = detectHeaderRow(rows);
|
|
1233
|
+
if (headerResult) {
|
|
1234
|
+
const { columns, headerIdx } = headerResult;
|
|
1235
|
+
const headerRow = rows[headerIdx];
|
|
1236
|
+
const headerItems = [...headerRow.items].sort((a, b) => a.x - b.x);
|
|
1237
|
+
const headerAndBelow = rows.slice(headerIdx);
|
|
1238
|
+
const mergedRows = mergeMultiLineRows(headerAndBelow, columns);
|
|
1239
|
+
const tableRegions = findTableRegionsByHeader(mergedRows, columns, headerItems);
|
|
1240
|
+
for (const region of tableRegions) {
|
|
1241
|
+
const table = buildClusterTable(region.rows, columns, pageNum);
|
|
1242
|
+
if (table) {
|
|
1243
|
+
expandUsedItems(table.usedItems, originMap);
|
|
1244
|
+
results.push(table);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1287
1247
|
}
|
|
1288
|
-
|
|
1289
|
-
const
|
|
1290
|
-
if (
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1248
|
+
if (results.length === 0) {
|
|
1249
|
+
const suspiciousRows = rows.filter((row) => hasSuspiciousGaps(row));
|
|
1250
|
+
if (suspiciousRows.length >= MIN_ROWS) {
|
|
1251
|
+
const columns = extractColumnClusters(suspiciousRows);
|
|
1252
|
+
if (columns.length >= MIN_COLS) {
|
|
1253
|
+
const tableRegions = findTableRegions(rows, columns);
|
|
1254
|
+
for (const region of tableRegions) {
|
|
1255
|
+
const mergedRows = mergeMultiLineRows(region.rows, columns);
|
|
1256
|
+
const table = buildClusterTable(mergedRows, columns, pageNum);
|
|
1257
|
+
if (table) {
|
|
1258
|
+
expandUsedItems(table.usedItems, originMap);
|
|
1259
|
+
results.push(table);
|
|
1260
|
+
}
|
|
1300
1261
|
}
|
|
1301
|
-
for (let c = 0; c < numCols; c++) cells[r][c].text = "";
|
|
1302
|
-
break;
|
|
1303
1262
|
}
|
|
1304
1263
|
}
|
|
1305
1264
|
}
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1265
|
+
return results;
|
|
1266
|
+
}
|
|
1267
|
+
function mergeEvenSpacedClusters(items) {
|
|
1268
|
+
const originMap = /* @__PURE__ */ new Map();
|
|
1269
|
+
const rows = groupByBaseline(items);
|
|
1270
|
+
const merged = [];
|
|
1271
|
+
for (const row of rows) {
|
|
1272
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1273
|
+
let i = 0;
|
|
1274
|
+
while (i < sorted.length) {
|
|
1275
|
+
if (/^[가-힣\d]$/.test(sorted[i].text)) {
|
|
1276
|
+
let runEnd = i + 1;
|
|
1277
|
+
while (runEnd < sorted.length && /^[가-힣\d]$/.test(sorted[runEnd].text)) {
|
|
1278
|
+
if (sorted[runEnd].hasSpaceBefore) break;
|
|
1279
|
+
const gap = sorted[runEnd].x - (sorted[runEnd - 1].x + sorted[runEnd - 1].w);
|
|
1280
|
+
const fs = sorted[runEnd].fontSize;
|
|
1281
|
+
if (gap < fs * 0.1 || gap > fs * 3) break;
|
|
1282
|
+
runEnd++;
|
|
1283
|
+
}
|
|
1284
|
+
if (runEnd - i >= 3) {
|
|
1285
|
+
const gaps = [];
|
|
1286
|
+
for (let g2 = i + 1; g2 < runEnd; g2++) {
|
|
1287
|
+
gaps.push(sorted[g2].x - (sorted[g2 - 1].x + sorted[g2 - 1].w));
|
|
1288
|
+
}
|
|
1289
|
+
let minG = Infinity, maxG = -Infinity;
|
|
1290
|
+
for (const g2 of gaps) {
|
|
1291
|
+
if (g2 < minG) minG = g2;
|
|
1292
|
+
if (g2 > maxG) maxG = g2;
|
|
1293
|
+
}
|
|
1294
|
+
if (minG > 0 && maxG / minG <= 3) {
|
|
1295
|
+
const run = sorted.slice(i, runEnd);
|
|
1296
|
+
const text = run.map((r) => r.text).join("");
|
|
1297
|
+
const first = run[0], last = run[runEnd - i - 1];
|
|
1298
|
+
const item = {
|
|
1299
|
+
text,
|
|
1300
|
+
x: first.x,
|
|
1301
|
+
y: first.y,
|
|
1302
|
+
w: last.x + last.w - first.x,
|
|
1303
|
+
h: first.h,
|
|
1304
|
+
fontSize: first.fontSize,
|
|
1305
|
+
fontName: first.fontName
|
|
1306
|
+
};
|
|
1307
|
+
originMap.set(item, run);
|
|
1308
|
+
merged.push(item);
|
|
1309
|
+
i = runEnd;
|
|
1310
|
+
continue;
|
|
1311
|
+
}
|
|
1317
1312
|
}
|
|
1318
|
-
for (let c = 0; c < numCols; c++) next[c].text = "";
|
|
1319
1313
|
}
|
|
1314
|
+
merged.push(sorted[i]);
|
|
1315
|
+
i++;
|
|
1320
1316
|
}
|
|
1321
1317
|
}
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
const
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1318
|
+
return { merged, originMap };
|
|
1319
|
+
}
|
|
1320
|
+
function expandUsedItems(usedItems, originMap) {
|
|
1321
|
+
const toAdd = [];
|
|
1322
|
+
for (const item of usedItems) {
|
|
1323
|
+
const origins = originMap.get(item);
|
|
1324
|
+
if (origins) for (const o of origins) toAdd.push(o);
|
|
1325
|
+
}
|
|
1326
|
+
for (const a of toAdd) usedItems.add(a);
|
|
1327
|
+
}
|
|
1328
|
+
function detectHeaderRow(rows) {
|
|
1331
1329
|
const allItems = rows.flatMap((r) => r.items);
|
|
1332
|
-
|
|
1330
|
+
if (allItems.length === 0) return null;
|
|
1331
|
+
let allMinX = Infinity, allMaxX = -Infinity;
|
|
1333
1332
|
for (const i of allItems) {
|
|
1334
|
-
if (i.x <
|
|
1335
|
-
|
|
1336
|
-
if (
|
|
1337
|
-
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1338
|
-
if (i.y + h > maxY) maxY = i.y + h;
|
|
1333
|
+
if (i.x < allMinX) allMinX = i.x;
|
|
1334
|
+
const r = i.x + i.w;
|
|
1335
|
+
if (r > allMaxX) allMaxX = r;
|
|
1339
1336
|
}
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
control++;
|
|
1360
|
-
continue;
|
|
1361
|
-
}
|
|
1362
|
-
if (code === 65533) {
|
|
1363
|
-
replacement++;
|
|
1364
|
-
continue;
|
|
1337
|
+
const pageSpan = allMaxX - allMinX;
|
|
1338
|
+
if (pageSpan <= 0) return null;
|
|
1339
|
+
for (let ri = 0; ri < rows.length; ri++) {
|
|
1340
|
+
const row = rows[ri];
|
|
1341
|
+
if (row.items.length < MIN_COLS || row.items.length > 6) continue;
|
|
1342
|
+
if (row.items.some((i) => i.text.length > 8)) continue;
|
|
1343
|
+
if (!row.items.some((i) => /[가-힣]/.test(i.text))) continue;
|
|
1344
|
+
if (row.items.some((i) => /^[□■○●·※▶▷◆◇\-]/.test(i.text))) continue;
|
|
1345
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1346
|
+
const xSpan = sorted[sorted.length - 1].x + sorted[sorted.length - 1].w - sorted[0].x;
|
|
1347
|
+
if (xSpan / pageSpan < 0.4) continue;
|
|
1348
|
+
const avgFs = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
|
|
1349
|
+
let hasLargeGap = false;
|
|
1350
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1351
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1352
|
+
if (gap >= avgFs * 2.5) {
|
|
1353
|
+
hasLargeGap = true;
|
|
1354
|
+
break;
|
|
1355
|
+
}
|
|
1365
1356
|
}
|
|
1366
|
-
if (
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
continue;
|
|
1357
|
+
if (!hasLargeGap) continue;
|
|
1358
|
+
const columns = sorted.map((item) => ({ x: item.x, count: 0 }));
|
|
1359
|
+
let matchCount = 0;
|
|
1360
|
+
for (let j = ri + 1; j < rows.length && matchCount < MIN_ROWS + 2; j++) {
|
|
1361
|
+
const matched = countMatchedColumnsRange(rows[j], columns, sorted);
|
|
1362
|
+
if (matched >= MIN_COLS) matchCount++;
|
|
1373
1363
|
}
|
|
1364
|
+
if (matchCount < MIN_ROWS) continue;
|
|
1365
|
+
return { columns, headerIdx: ri };
|
|
1374
1366
|
}
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
let
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1367
|
+
return null;
|
|
1368
|
+
}
|
|
1369
|
+
function mergeOverlappingRows(rows) {
|
|
1370
|
+
if (rows.length <= 1) return rows;
|
|
1371
|
+
const result = [rows[0]];
|
|
1372
|
+
for (let i = 1; i < rows.length; i++) {
|
|
1373
|
+
const prev = result[result.length - 1];
|
|
1374
|
+
const curr = rows[i];
|
|
1375
|
+
const a = rowBand(prev);
|
|
1376
|
+
const b = rowBand(curr);
|
|
1377
|
+
const overlap = Math.min(a.top, b.top) - Math.max(a.bottom, b.bottom);
|
|
1378
|
+
const prevIsFrag = isFragmentRow(prev) && a.height <= b.height * 0.8 && overlap >= a.height * 0.5;
|
|
1379
|
+
const currIsFrag = isFragmentRow(curr) && b.height <= a.height * 0.8 && overlap >= b.height * 0.5;
|
|
1380
|
+
if (prevIsFrag || currIsFrag) {
|
|
1381
|
+
const baseY = prevIsFrag ? curr.y : prev.y;
|
|
1382
|
+
result[result.length - 1] = { y: baseY, items: [...prev.items, ...curr.items] };
|
|
1383
|
+
} else {
|
|
1384
|
+
result.push(curr);
|
|
1385
|
+
}
|
|
1393
1386
|
}
|
|
1394
|
-
return
|
|
1395
|
-
page,
|
|
1396
|
-
textChars: total,
|
|
1397
|
-
hangulRatio: hangul / denom,
|
|
1398
|
-
controlCharRatio,
|
|
1399
|
-
replacementCharRatio,
|
|
1400
|
-
puaRatio,
|
|
1401
|
-
needsOcr,
|
|
1402
|
-
ocrReason
|
|
1403
|
-
};
|
|
1387
|
+
return result;
|
|
1404
1388
|
}
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
var HIGH_CONTROL_THRESHOLD = 0.05;
|
|
1408
|
-
var HIGH_REPLACEMENT_THRESHOLD = 0.05;
|
|
1409
|
-
var DOC_NEEDS_OCR_PAGE_RATIO = 0.3;
|
|
1410
|
-
function stripControlChars(text) {
|
|
1411
|
-
return text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x80-\x9F]/g, "");
|
|
1389
|
+
function isFragmentRow(row) {
|
|
1390
|
+
return row.items.length <= 3 && row.items.every((i) => i.text.length <= 8);
|
|
1412
1391
|
}
|
|
1413
|
-
function
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
avgControlCharRatio: 0,
|
|
1420
|
-
avgReplacementCharRatio: 0,
|
|
1421
|
-
avgPuaRatio: 0,
|
|
1422
|
-
lowTextPageCount: 0,
|
|
1423
|
-
highPuaPageCount: 0,
|
|
1424
|
-
needsOcr: false,
|
|
1425
|
-
ocrCandidatePages: []
|
|
1426
|
-
};
|
|
1392
|
+
function rowBand(row) {
|
|
1393
|
+
let bottom = Infinity, top = -Infinity;
|
|
1394
|
+
for (const i of row.items) {
|
|
1395
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1396
|
+
if (i.y < bottom) bottom = i.y;
|
|
1397
|
+
if (i.y + h > top) top = i.y + h;
|
|
1427
1398
|
}
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1399
|
+
return { bottom, top, height: top - bottom };
|
|
1400
|
+
}
|
|
1401
|
+
function mergeMultiLineRows(rows, columns) {
|
|
1402
|
+
if (rows.length <= 1) return rows;
|
|
1403
|
+
const result = [rows[0]];
|
|
1404
|
+
const allFontSizes = rows.flatMap((r) => r.items).map((i) => i.fontSize);
|
|
1405
|
+
const avgFontSize = allFontSizes.length > 0 ? allFontSizes.reduce((s, v) => s + v, 0) / allFontSizes.length : 12;
|
|
1406
|
+
for (let i = 1; i < rows.length; i++) {
|
|
1407
|
+
const prev = result[result.length - 1];
|
|
1408
|
+
const curr = rows[i];
|
|
1409
|
+
const yGap = Math.abs(prev.y - curr.y);
|
|
1410
|
+
const matchedCols = countMatchedColumns(curr, columns);
|
|
1411
|
+
if (yGap < avgFontSize * 1.8 && curr.items.length <= 2 && (matchedCols < MIN_COLS || curr.items.length === 1)) {
|
|
1412
|
+
result[result.length - 1] = {
|
|
1413
|
+
y: prev.y,
|
|
1414
|
+
items: [...prev.items, ...curr.items]
|
|
1415
|
+
};
|
|
1416
|
+
} else {
|
|
1417
|
+
result.push(curr);
|
|
1418
|
+
}
|
|
1445
1419
|
}
|
|
1446
|
-
|
|
1447
|
-
return {
|
|
1448
|
-
totalPages: n,
|
|
1449
|
-
totalTextChars: textChars,
|
|
1450
|
-
avgHangulRatio: hangul / n,
|
|
1451
|
-
avgControlCharRatio: control / n,
|
|
1452
|
-
avgReplacementCharRatio: replacement / n,
|
|
1453
|
-
avgPuaRatio: pua / n,
|
|
1454
|
-
lowTextPageCount: lowText,
|
|
1455
|
-
highPuaPageCount: highPua,
|
|
1456
|
-
needsOcr: ocrCandidatePages.length / n >= DOC_NEEDS_OCR_PAGE_RATIO,
|
|
1457
|
-
ocrCandidatePages
|
|
1458
|
-
};
|
|
1420
|
+
return result;
|
|
1459
1421
|
}
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1422
|
+
function groupByBaseline(items) {
|
|
1423
|
+
if (items.length === 0) return [];
|
|
1424
|
+
const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1425
|
+
const rows = [];
|
|
1426
|
+
let curItems = [sorted[0]];
|
|
1427
|
+
let curY = sorted[0].y;
|
|
1428
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1429
|
+
if (Math.abs(sorted[i].y - curY) <= Y_TOL) {
|
|
1430
|
+
curItems.push(sorted[i]);
|
|
1431
|
+
} else {
|
|
1432
|
+
rows.push({ y: curY, items: curItems });
|
|
1433
|
+
curItems = [sorted[i]];
|
|
1434
|
+
curY = sorted[i].y;
|
|
1469
1435
|
}
|
|
1470
|
-
}
|
|
1436
|
+
}
|
|
1437
|
+
if (curItems.length > 0) rows.push({ y: curY, items: curItems });
|
|
1438
|
+
return rows;
|
|
1471
1439
|
}
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1440
|
+
function hasSuspiciousGaps(row) {
|
|
1441
|
+
if (row.items.length < 2) return false;
|
|
1442
|
+
const sorted = [...row.items].sort((a, b) => a.x - b.x);
|
|
1443
|
+
if (sorted.length === 2 && sorted[1].text.length > 20) return false;
|
|
1444
|
+
const avgFontSize = sorted.reduce((s, i) => s + i.fontSize, 0) / sorted.length;
|
|
1445
|
+
const minGap = Math.max(avgFontSize * MIN_GAP_FACTOR, MIN_GAP_ABSOLUTE);
|
|
1446
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1447
|
+
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
1448
|
+
if (gap >= minGap) return true;
|
|
1449
|
+
}
|
|
1450
|
+
return false;
|
|
1475
1451
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
try {
|
|
1493
|
-
return await Promise.race([
|
|
1494
|
-
loadingTask.promise,
|
|
1495
|
-
new Promise((_, reject) => {
|
|
1496
|
-
timer = setTimeout(() => {
|
|
1497
|
-
loadingTask.destroy();
|
|
1498
|
-
reject(new KordocError("PDF \uB85C\uB529 \uD0C0\uC784\uC544\uC6C3 (30\uCD08 \uCD08\uACFC)"));
|
|
1499
|
-
}, PDF_LOAD_TIMEOUT_MS);
|
|
1500
|
-
})
|
|
1501
|
-
]);
|
|
1502
|
-
} finally {
|
|
1503
|
-
if (timer !== void 0) clearTimeout(timer);
|
|
1452
|
+
function extractColumnClusters(rows) {
|
|
1453
|
+
const allX = [];
|
|
1454
|
+
for (const row of rows) {
|
|
1455
|
+
for (const item of row.items) allX.push(item.x);
|
|
1456
|
+
}
|
|
1457
|
+
if (allX.length === 0) return [];
|
|
1458
|
+
allX.sort((a, b) => a - b);
|
|
1459
|
+
const clusters = [];
|
|
1460
|
+
let clusterStart = 0;
|
|
1461
|
+
for (let i = 1; i <= allX.length; i++) {
|
|
1462
|
+
if (i === allX.length || allX[i] - allX[i - 1] > COL_CLUSTER_TOL) {
|
|
1463
|
+
const slice = allX.slice(clusterStart, i);
|
|
1464
|
+
const avg = Math.round(slice.reduce((s, v) => s + v, 0) / slice.length);
|
|
1465
|
+
clusters.push({ x: avg, count: slice.length });
|
|
1466
|
+
clusterStart = i;
|
|
1467
|
+
}
|
|
1504
1468
|
}
|
|
1469
|
+
const minCount = Math.max(2, Math.floor(rows.length * MIN_COL_FILL_RATIO));
|
|
1470
|
+
return clusters.filter((c) => c.count >= minCount).sort((a, b) => a.x - b.x);
|
|
1505
1471
|
}
|
|
1506
|
-
|
|
1507
|
-
const
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
const pageHeights = /* @__PURE__ */ new Map();
|
|
1524
|
-
const pagesWithLargeImage = /* @__PURE__ */ new Set();
|
|
1525
|
-
const skippedImagePages = /* @__PURE__ */ new Map();
|
|
1526
|
-
let parsedPages = 0;
|
|
1527
|
-
for (let i = 1; i <= effectivePageCount; i++) {
|
|
1528
|
-
if (pageFilter && !pageFilter.has(i)) continue;
|
|
1529
|
-
try {
|
|
1530
|
-
const page = await doc.getPage(i);
|
|
1531
|
-
const tc = await page.getTextContent();
|
|
1532
|
-
const viewport = page.getViewport({ scale: 1 });
|
|
1533
|
-
pageHeights.set(i, viewport.height);
|
|
1534
|
-
const rawItems = tc.items;
|
|
1535
|
-
const items = normalizeItems(rawItems);
|
|
1536
|
-
const { visible, hiddenCount } = filterHiddenText(items, viewport.width, viewport.height);
|
|
1537
|
-
if (hiddenCount > 0) {
|
|
1538
|
-
warnings.push({ page: i, message: `${hiddenCount}\uAC1C \uC228\uACA8\uC9C4 \uD14D\uC2A4\uD2B8 \uC694\uC18C \uD544\uD130\uB9C1\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
1539
|
-
}
|
|
1540
|
-
for (const item of visible) {
|
|
1541
|
-
if (item.fontSize > 0) fontSizeFreq.set(item.fontSize, (fontSizeFreq.get(item.fontSize) || 0) + 1);
|
|
1542
|
-
}
|
|
1543
|
-
const opList = await page.getOperatorList();
|
|
1544
|
-
const pageArea = viewport.width * viewport.height;
|
|
1545
|
-
if (pageArea > 0) {
|
|
1546
|
-
const imageRegions = extractImageRegions(opList.fnArray, opList.argsArray);
|
|
1547
|
-
let uncovered = 0;
|
|
1548
|
-
for (const r of imageRegions) {
|
|
1549
|
-
const area = (r.x2 - r.x1) * (r.y2 - r.y1);
|
|
1550
|
-
if (area < pageArea * 0.05) continue;
|
|
1551
|
-
pagesWithLargeImage.add(i);
|
|
1552
|
-
const hasText = visible.some((it) => {
|
|
1553
|
-
const cx = it.x + it.w / 2;
|
|
1554
|
-
const cy = it.y + (it.h || it.fontSize) / 2;
|
|
1555
|
-
return cx >= r.x1 && cx <= r.x2 && cy >= r.y1 && cy <= r.y2;
|
|
1556
|
-
});
|
|
1557
|
-
if (!hasText) uncovered++;
|
|
1558
|
-
}
|
|
1559
|
-
if (uncovered > 0) skippedImagePages.set(i, uncovered);
|
|
1560
|
-
}
|
|
1561
|
-
const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
|
|
1562
|
-
for (const b of pageBlocks) blocks.push(b);
|
|
1563
|
-
let pageText = "";
|
|
1564
|
-
for (const b of pageBlocks) {
|
|
1565
|
-
const t = b.text || "";
|
|
1566
|
-
totalChars += t.replace(/\s/g, "").length;
|
|
1567
|
-
totalTextBytes += t.length * 2;
|
|
1568
|
-
pageText += pageText ? "\n" + t : t;
|
|
1569
|
-
}
|
|
1570
|
-
pageQuality.push(computePageQuality(i, pageText));
|
|
1571
|
-
if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
|
|
1572
|
-
parsedPages++;
|
|
1573
|
-
options?.onProgress?.(parsedPages, totalTarget);
|
|
1574
|
-
} catch (pageErr) {
|
|
1575
|
-
if (pageErr instanceof KordocError) throw pageErr;
|
|
1576
|
-
warnings.push({ page: i, message: `\uD398\uC774\uC9C0 ${i} \uD30C\uC2F1 \uC2E4\uD328: ${pageErr instanceof Error ? pageErr.message : "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958"}`, code: "PARTIAL_PARSE" });
|
|
1577
|
-
}
|
|
1578
|
-
}
|
|
1579
|
-
const parsedPageCount = parsedPages || (pageFilter ? pageFilter.size : effectivePageCount);
|
|
1580
|
-
let isImageBased = false;
|
|
1581
|
-
if (totalChars / Math.max(parsedPageCount, 1) < 10) {
|
|
1582
|
-
if (options?.ocr) {
|
|
1583
|
-
try {
|
|
1584
|
-
const { ocrPages } = await import("./provider-4ZJKV3DC.js");
|
|
1585
|
-
const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
|
|
1586
|
-
if (ocrBlocks.length > 0) {
|
|
1587
|
-
const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
|
|
1588
|
-
return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
|
|
1589
|
-
}
|
|
1590
|
-
} catch {
|
|
1591
|
-
}
|
|
1472
|
+
function findTableRegionsByHeader(allRows, columns, headerItems) {
|
|
1473
|
+
const regions = [];
|
|
1474
|
+
let currentRegion = [];
|
|
1475
|
+
let missStreak = 0;
|
|
1476
|
+
for (const row of allRows) {
|
|
1477
|
+
const matchedCols = countMatchedColumnsRange(row, columns, headerItems);
|
|
1478
|
+
if (matchedCols >= MIN_COLS) {
|
|
1479
|
+
currentRegion.push(row);
|
|
1480
|
+
missStreak = 0;
|
|
1481
|
+
} else if (currentRegion.length > 0 && (row.items.length <= 2 || missStreak === 0)) {
|
|
1482
|
+
currentRegion.push(row);
|
|
1483
|
+
missStreak++;
|
|
1484
|
+
} else {
|
|
1485
|
+
while (currentRegion.length > 0) {
|
|
1486
|
+
const last = currentRegion[currentRegion.length - 1];
|
|
1487
|
+
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1488
|
+
currentRegion.pop();
|
|
1592
1489
|
}
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
message: `\uC774\uBBF8\uC9C0 \uAE30\uBC18 PDF (${pageCount}\uD398\uC774\uC9C0, \uD14D\uC2A4\uD2B8 ${totalChars}\uC790) \u2014 \uD14D\uC2A4\uD2B8 \uB808\uC774\uC5B4\uAC00 \uC5C6\uC5B4 OCR\uC774 \uD544\uC694\uD569\uB2C8\uB2E4`,
|
|
1596
|
-
code: "NEEDS_OCR"
|
|
1597
|
-
});
|
|
1598
|
-
}
|
|
1599
|
-
if (!isImageBased) {
|
|
1600
|
-
const OCR_REASON_MESSAGES = {
|
|
1601
|
-
low_text: "\uD14D\uC2A4\uD2B8\uAC00 \uAC70\uC758 \uC5C6\uB294 \uD398\uC774\uC9C0 (\uC2A4\uCE94/\uC774\uBBF8\uC9C0 \uCD94\uC815)",
|
|
1602
|
-
high_pua: "\uAE00\uAF34 \uB9E4\uD551 \uC2E4\uD328 (PUA \uBE44\uC728 \uB192\uC74C) \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
1603
|
-
high_control: "\uC81C\uC5B4\uBB38\uC790 \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
1604
|
-
high_replacement: "\uB300\uCCB4\uBB38\uC790(U+FFFD) \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00"
|
|
1605
|
-
};
|
|
1606
|
-
for (const pq of pageQuality) {
|
|
1607
|
-
if (!pq.needsOcr || !pq.ocrReason) continue;
|
|
1608
|
-
if (pq.ocrReason === "low_text" && !pagesWithLargeImage.has(pq.page)) continue;
|
|
1609
|
-
warnings.push({ page: pq.page, message: `${OCR_REASON_MESSAGES[pq.ocrReason]} \u2014 OCR \uAC80\uD1A0 \uD544\uC694`, code: "NEEDS_OCR" });
|
|
1490
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1491
|
+
regions.push({ rows: [...currentRegion] });
|
|
1610
1492
|
}
|
|
1493
|
+
currentRegion = [];
|
|
1494
|
+
missStreak = 0;
|
|
1611
1495
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1496
|
+
}
|
|
1497
|
+
while (currentRegion.length > 0) {
|
|
1498
|
+
const last = currentRegion[currentRegion.length - 1];
|
|
1499
|
+
if (countMatchedColumnsRange(last, columns, headerItems) >= MIN_COLS) break;
|
|
1500
|
+
currentRegion.pop();
|
|
1501
|
+
}
|
|
1502
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1503
|
+
regions.push({ rows: currentRegion });
|
|
1504
|
+
}
|
|
1505
|
+
return regions;
|
|
1506
|
+
}
|
|
1507
|
+
function findTableRegions(allRows, columns) {
|
|
1508
|
+
const regions = [];
|
|
1509
|
+
let currentRegion = [];
|
|
1510
|
+
for (const row of allRows) {
|
|
1511
|
+
const matchedCols = countMatchedColumns(row, columns);
|
|
1512
|
+
if (matchedCols >= MIN_COLS) {
|
|
1513
|
+
currentRegion.push(row);
|
|
1514
|
+
} else if (row.items.length === 1) {
|
|
1515
|
+
if (currentRegion.length > 0) {
|
|
1516
|
+
currentRegion.push(row);
|
|
1615
1517
|
}
|
|
1616
|
-
}
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
for (let ri = removed.length - 1; ri >= 0; ri--) {
|
|
1620
|
-
blocks.splice(removed[ri], 1);
|
|
1518
|
+
} else {
|
|
1519
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1520
|
+
regions.push({ rows: [...currentRegion] });
|
|
1621
1521
|
}
|
|
1522
|
+
currentRegion = [];
|
|
1622
1523
|
}
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1524
|
+
}
|
|
1525
|
+
if (currentRegion.length >= MIN_ROWS) {
|
|
1526
|
+
regions.push({ rows: currentRegion });
|
|
1527
|
+
}
|
|
1528
|
+
return regions;
|
|
1529
|
+
}
|
|
1530
|
+
function countMatchedColumns(row, columns) {
|
|
1531
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1532
|
+
for (const item of row.items) {
|
|
1533
|
+
for (let ci = 0; ci < columns.length; ci++) {
|
|
1534
|
+
if (Math.abs(item.x - columns[ci].x) <= COL_CLUSTER_TOL * 2) {
|
|
1535
|
+
matched.add(ci);
|
|
1536
|
+
break;
|
|
1632
1537
|
}
|
|
1633
1538
|
}
|
|
1634
|
-
const medianFontSize = computeMedianFontSizeFromFreq(fontSizeFreq);
|
|
1635
|
-
if (medianFontSize > 0) {
|
|
1636
|
-
detectHeadings(blocks, medianFontSize);
|
|
1637
|
-
}
|
|
1638
|
-
detectMarkerHeadings(blocks);
|
|
1639
|
-
detectTableCaptions(blocks);
|
|
1640
|
-
detectKoreanListBlocks(blocks);
|
|
1641
|
-
const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
|
|
1642
|
-
sanitizeBlockControlChars(blocks);
|
|
1643
|
-
let markdown = cleanPdfText(blocksToMarkdown(blocks));
|
|
1644
|
-
return {
|
|
1645
|
-
markdown,
|
|
1646
|
-
blocks,
|
|
1647
|
-
metadata,
|
|
1648
|
-
outline: outline.length > 0 ? outline : void 0,
|
|
1649
|
-
warnings: warnings.length > 0 ? warnings : void 0,
|
|
1650
|
-
isImageBased: isImageBased || void 0,
|
|
1651
|
-
pageQuality,
|
|
1652
|
-
qualitySummary: summarizeDocumentQuality(pageQuality)
|
|
1653
|
-
};
|
|
1654
|
-
} finally {
|
|
1655
|
-
await doc.destroy().catch(() => {
|
|
1656
|
-
});
|
|
1657
1539
|
}
|
|
1540
|
+
return matched.size;
|
|
1658
1541
|
}
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
const
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1542
|
+
function countMatchedColumnsRange(row, columns, headerItems) {
|
|
1543
|
+
const boundaries = [];
|
|
1544
|
+
for (let ci = 0; ci < headerItems.length; ci++) {
|
|
1545
|
+
const left = ci === 0 ? 0 : (headerItems[ci - 1].x + headerItems[ci - 1].w + headerItems[ci].x) / 2;
|
|
1546
|
+
const right = ci === headerItems.length - 1 ? Infinity : (headerItems[ci].x + headerItems[ci].w + headerItems[ci + 1].x) / 2;
|
|
1547
|
+
boundaries.push({ left, right });
|
|
1548
|
+
}
|
|
1549
|
+
const matched = /* @__PURE__ */ new Set();
|
|
1550
|
+
for (const item of row.items) {
|
|
1551
|
+
for (let ci = 0; ci < boundaries.length; ci++) {
|
|
1552
|
+
if (item.x >= boundaries[ci].left && item.x < boundaries[ci].right) {
|
|
1553
|
+
matched.add(ci);
|
|
1554
|
+
break;
|
|
1555
|
+
}
|
|
1670
1556
|
}
|
|
1671
|
-
if (typeof info.CreationDate === "string") metadata.createdAt = parsePdfDate(info.CreationDate);
|
|
1672
|
-
if (typeof info.ModDate === "string") metadata.modifiedAt = parsePdfDate(info.ModDate);
|
|
1673
|
-
} catch {
|
|
1674
1557
|
}
|
|
1558
|
+
return matched.size;
|
|
1675
1559
|
}
|
|
1676
|
-
function
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
const
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
const doc = await loadPdfWithTimeout(buffer);
|
|
1684
|
-
try {
|
|
1685
|
-
const metadata = { pageCount: doc.numPages };
|
|
1686
|
-
await extractPdfMetadata(doc, metadata);
|
|
1687
|
-
return metadata;
|
|
1688
|
-
} finally {
|
|
1689
|
-
await doc.destroy().catch(() => {
|
|
1690
|
-
});
|
|
1560
|
+
function assignRowItems(items, columns, numCols) {
|
|
1561
|
+
if (items.length === 0) return [];
|
|
1562
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1563
|
+
const colCenters = columns.map((c) => c.x);
|
|
1564
|
+
const gaps = [];
|
|
1565
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1566
|
+
gaps.push({ idx: i, size: sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w) });
|
|
1691
1567
|
}
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
const
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1568
|
+
const gapSizes = gaps.map((g2) => g2.size).sort((a, b) => a - b);
|
|
1569
|
+
const medianGap = gapSizes.length > 0 ? gapSizes[Math.floor(gapSizes.length / 2)] : 0;
|
|
1570
|
+
const gapThreshold = sorted.length <= numCols + 1 ? 12 : Math.max(medianGap * 2.5, 12);
|
|
1571
|
+
const significantGaps = gaps.filter((g2) => g2.size >= gapThreshold).sort((a, b) => b.size - a.size).slice(0, numCols - 1).sort((a, b) => a.idx - b.idx);
|
|
1572
|
+
const groups = [];
|
|
1573
|
+
let start = 0;
|
|
1574
|
+
for (const gap of significantGaps) {
|
|
1575
|
+
groups.push(sorted.slice(start, gap.idx));
|
|
1576
|
+
start = gap.idx;
|
|
1577
|
+
}
|
|
1578
|
+
groups.push(sorted.slice(start));
|
|
1579
|
+
const result = [];
|
|
1580
|
+
const usedCols = /* @__PURE__ */ new Set();
|
|
1581
|
+
const groupCenters = groups.map((g2) => {
|
|
1582
|
+
let minX = Infinity, maxX = -Infinity;
|
|
1583
|
+
for (const i of g2) {
|
|
1584
|
+
if (i.x < minX) minX = i.x;
|
|
1585
|
+
const r = i.x + i.w;
|
|
1586
|
+
if (r > maxX) maxX = r;
|
|
1700
1587
|
}
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1588
|
+
return (minX + maxX) / 2;
|
|
1589
|
+
});
|
|
1590
|
+
const assignments = [];
|
|
1591
|
+
for (let gi = 0; gi < groups.length; gi++) {
|
|
1592
|
+
for (let ci = 0; ci < numCols; ci++) {
|
|
1593
|
+
assignments.push({ gi, ci, dist: Math.abs(groupCenters[gi] - colCenters[ci]) });
|
|
1705
1594
|
}
|
|
1706
|
-
visible.push(item);
|
|
1707
1595
|
}
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
const mid = Math.floor(total / 2);
|
|
1716
|
-
let cumulative = 0;
|
|
1717
|
-
for (const [size, count] of sorted) {
|
|
1718
|
-
cumulative += count;
|
|
1719
|
-
if (cumulative > mid) return size;
|
|
1596
|
+
assignments.sort((a, b) => a.dist - b.dist);
|
|
1597
|
+
const assignedGroups = /* @__PURE__ */ new Set();
|
|
1598
|
+
for (const { gi, ci } of assignments) {
|
|
1599
|
+
if (assignedGroups.has(gi) || usedCols.has(ci)) continue;
|
|
1600
|
+
result.push({ col: ci, items: groups[gi] });
|
|
1601
|
+
assignedGroups.add(gi);
|
|
1602
|
+
usedCols.add(ci);
|
|
1720
1603
|
}
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
let level = 0;
|
|
1731
|
-
if (ratio >= HEADING_RATIO_H1) level = 1;
|
|
1732
|
-
else if (ratio >= HEADING_RATIO_H2) level = 2;
|
|
1733
|
-
else if (ratio >= HEADING_RATIO_H3) level = 3;
|
|
1734
|
-
if (level > 0) {
|
|
1735
|
-
block.type = "heading";
|
|
1736
|
-
block.level = level;
|
|
1737
|
-
block.text = collapseEvenSpacing(text);
|
|
1604
|
+
for (let gi = 0; gi < groups.length; gi++) {
|
|
1605
|
+
if (assignedGroups.has(gi)) continue;
|
|
1606
|
+
let bestCol = 0, bestDist = Infinity;
|
|
1607
|
+
for (let ci = 0; ci < numCols; ci++) {
|
|
1608
|
+
const d = Math.abs(groupCenters[gi] - colCenters[ci]);
|
|
1609
|
+
if (d < bestDist) {
|
|
1610
|
+
bestDist = d;
|
|
1611
|
+
bestCol = ci;
|
|
1612
|
+
}
|
|
1738
1613
|
}
|
|
1614
|
+
result.push({ col: bestCol, items: groups[gi] });
|
|
1739
1615
|
}
|
|
1616
|
+
return result;
|
|
1740
1617
|
}
|
|
1741
|
-
function
|
|
1742
|
-
const
|
|
1743
|
-
const
|
|
1744
|
-
if (
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
/(?<![가-힣])[가-힣](?: [가-힣\d]){2,}(?![가-힣])/g,
|
|
1749
|
-
(match) => match.replace(/ /g, "")
|
|
1618
|
+
function buildClusterTable(rows, columns, pageNum) {
|
|
1619
|
+
const numCols = columns.length;
|
|
1620
|
+
const numRows = rows.length;
|
|
1621
|
+
if (numRows < MIN_ROWS || numCols < MIN_COLS) return null;
|
|
1622
|
+
const cells = Array.from(
|
|
1623
|
+
{ length: numRows },
|
|
1624
|
+
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
1750
1625
|
);
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
const emptyCells = totalCells - allCells.length;
|
|
1766
|
-
if (table.rows <= 2 && emptyCells > totalCells * 0.5) return true;
|
|
1767
|
-
if (table.rows === 1 && !/\d{2,}/.test(allText)) return true;
|
|
1768
|
-
return false;
|
|
1769
|
-
}
|
|
1770
|
-
function demoteTableToText(table) {
|
|
1771
|
-
const lines = [];
|
|
1772
|
-
for (let r = 0; r < table.rows; r++) {
|
|
1773
|
-
const cells = table.cells[r].map((c) => c.text.trim()).filter(Boolean);
|
|
1774
|
-
if (cells.length === 0) continue;
|
|
1775
|
-
if (table.cols === 2 && cells.length === 2) {
|
|
1776
|
-
lines.push(`${cells[0]} : ${cells[1]}`);
|
|
1777
|
-
} else {
|
|
1778
|
-
lines.push(cells.join(" "));
|
|
1626
|
+
const usedItems = /* @__PURE__ */ new Set();
|
|
1627
|
+
for (let r = 0; r < numRows; r++) {
|
|
1628
|
+
const row = rows[r];
|
|
1629
|
+
if (row.items.length === 1 && numCols > 1) {
|
|
1630
|
+
cells[r][0] = { text: row.items[0].text, colSpan: numCols, rowSpan: 1 };
|
|
1631
|
+
usedItems.add(row.items[0]);
|
|
1632
|
+
continue;
|
|
1633
|
+
}
|
|
1634
|
+
const assignments = assignRowItems(row.items, columns, numCols);
|
|
1635
|
+
for (const { col, items } of assignments) {
|
|
1636
|
+
const text = items.map((i) => i.text).join(" ");
|
|
1637
|
+
const existing = cells[r][col].text;
|
|
1638
|
+
cells[r][col].text = existing ? existing + " " + text : text;
|
|
1639
|
+
for (const item of items) usedItems.add(item);
|
|
1779
1640
|
}
|
|
1780
1641
|
}
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
const
|
|
1788
|
-
if (
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1642
|
+
let emptyRows = 0;
|
|
1643
|
+
for (const row of cells) {
|
|
1644
|
+
if (row.every((c) => c.text === "")) emptyRows++;
|
|
1645
|
+
}
|
|
1646
|
+
if (emptyRows > numRows * 0.5) return null;
|
|
1647
|
+
for (let c = 0; c < numCols; c++) {
|
|
1648
|
+
const hasValue = cells.some((row) => row[c].text !== "");
|
|
1649
|
+
if (!hasValue) return null;
|
|
1650
|
+
}
|
|
1651
|
+
for (let r = numRows - 1; r >= 1; r--) {
|
|
1652
|
+
const nonEmptyCols = cells[r].filter((c) => c.text.trim()).length;
|
|
1653
|
+
if (nonEmptyCols !== 1) continue;
|
|
1654
|
+
if (cells[r][0].text.trim() !== "") continue;
|
|
1655
|
+
const contentText = cells[r].find((c) => c.text.trim())?.text.trim() || "";
|
|
1656
|
+
if (/^[○●▶\-·]/.test(contentText)) continue;
|
|
1657
|
+
for (let pr = r - 1; pr >= 0; pr--) {
|
|
1658
|
+
if (cells[pr].some((c) => c.text.trim())) {
|
|
1659
|
+
for (let c = 0; c < numCols; c++) {
|
|
1660
|
+
const prev = cells[pr][c].text.trim();
|
|
1661
|
+
const curr = cells[r][c].text.trim();
|
|
1662
|
+
if (curr) cells[pr][c].text = prev ? prev + " " + curr : curr;
|
|
1663
|
+
}
|
|
1664
|
+
for (let c = 0; c < numCols; c++) cells[r][c].text = "";
|
|
1665
|
+
break;
|
|
1666
|
+
}
|
|
1792
1667
|
}
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1668
|
+
}
|
|
1669
|
+
for (let r = 0; r < cells.length - 1; r++) {
|
|
1670
|
+
const row = cells[r];
|
|
1671
|
+
const hasCol0 = row[0].text.trim() !== "";
|
|
1672
|
+
const hasColLast = numCols > 1 && row[numCols - 1].text.trim() !== "";
|
|
1673
|
+
const midEmpty = row.slice(1, numCols - 1).every((c) => c.text.trim() === "");
|
|
1674
|
+
if (hasCol0 && hasColLast && midEmpty) {
|
|
1675
|
+
const next = cells[r + 1];
|
|
1676
|
+
if (next[0].text.trim() === "" && next.some((c) => c.text.trim())) {
|
|
1677
|
+
for (let c = 1; c < numCols; c++) {
|
|
1678
|
+
const curr = next[c].text.trim();
|
|
1679
|
+
if (curr) row[c].text = row[c].text.trim() ? row[c].text.trim() + " " + curr : curr;
|
|
1680
|
+
}
|
|
1681
|
+
for (let c = 0; c < numCols; c++) next[c].text = "";
|
|
1801
1682
|
}
|
|
1802
1683
|
}
|
|
1803
1684
|
}
|
|
1685
|
+
const filteredCells = cells.filter((row) => row.some((c) => c.text.trim()));
|
|
1686
|
+
const finalRowCount = filteredCells.length;
|
|
1687
|
+
if (finalRowCount < MIN_ROWS) return null;
|
|
1688
|
+
const irTable = {
|
|
1689
|
+
rows: finalRowCount,
|
|
1690
|
+
cols: numCols,
|
|
1691
|
+
cells: filteredCells,
|
|
1692
|
+
hasHeader: finalRowCount > 1
|
|
1693
|
+
};
|
|
1694
|
+
const allItems = rows.flatMap((r) => r.items);
|
|
1695
|
+
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
1696
|
+
for (const i of allItems) {
|
|
1697
|
+
if (i.x < minX) minX = i.x;
|
|
1698
|
+
if (i.y < minY) minY = i.y;
|
|
1699
|
+
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
1700
|
+
const h = i.h > 0 ? i.h : i.fontSize;
|
|
1701
|
+
if (i.y + h > maxY) maxY = i.y + h;
|
|
1702
|
+
}
|
|
1703
|
+
return {
|
|
1704
|
+
table: irTable,
|
|
1705
|
+
bbox: { page: pageNum, x: minX, y: minY, width: maxX - minX, height: maxY - minY },
|
|
1706
|
+
usedItems
|
|
1707
|
+
};
|
|
1804
1708
|
}
|
|
1709
|
+
|
|
1710
|
+
// src/pdf/xy-cut.ts
|
|
1805
1711
|
var MAX_XYCUT_DEPTH = 50;
|
|
1806
1712
|
var XYCUT_MIN_GAP = 5;
|
|
1807
1713
|
var CROSS_LAYOUT_BETA = 2;
|
|
@@ -1959,1033 +1865,947 @@ function findVerticalCut(items) {
|
|
|
1959
1865
|
}
|
|
1960
1866
|
return { position, gap: largestGap };
|
|
1961
1867
|
}
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
(
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
if (grids.length > 0) {
|
|
1971
|
-
return extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals);
|
|
1868
|
+
|
|
1869
|
+
// src/pdf/columns.ts
|
|
1870
|
+
function isProseSpread(items) {
|
|
1871
|
+
if (items.length < 4) return false;
|
|
1872
|
+
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
1873
|
+
const gaps = [];
|
|
1874
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
1875
|
+
gaps.push(sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w));
|
|
1972
1876
|
}
|
|
1973
|
-
|
|
1877
|
+
const maxGap = safeMax(gaps);
|
|
1878
|
+
const avgLen = items.reduce((s, i) => s + i.text.length, 0) / items.length;
|
|
1879
|
+
return maxGap < 40 && avgLen < 5;
|
|
1974
1880
|
}
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
for (const item of items) {
|
|
1986
|
-
const h = item.h > 0 ? item.h : item.fontSize;
|
|
1987
|
-
if (h <= 0 || item.w <= 0) continue;
|
|
1988
|
-
if (line.lineWidth > h * STRIKE_MAX_THICKNESS_RATIO) continue;
|
|
1989
|
-
const centerY = item.y + h * 0.4;
|
|
1990
|
-
if (Math.abs(line.y1 - centerY) > h * STRIKE_CENTER_TOLERANCE) continue;
|
|
1991
|
-
const overlap = Math.min(line.x2, item.x + item.w) - Math.max(line.x1, item.x);
|
|
1992
|
-
if (overlap / item.w < STRIKE_MIN_OVERLAP_RATIO) continue;
|
|
1993
|
-
matches.push(item);
|
|
1881
|
+
function detectColumns(yLines) {
|
|
1882
|
+
const allItems = yLines.flat();
|
|
1883
|
+
if (allItems.length === 0) return null;
|
|
1884
|
+
const pageWidth = safeMax(allItems.map((i) => i.x + i.w)) - safeMin(allItems.map((i) => i.x));
|
|
1885
|
+
if (pageWidth < 100) return null;
|
|
1886
|
+
let bigoLineIdx = -1;
|
|
1887
|
+
for (let i = 0; i < yLines.length; i++) {
|
|
1888
|
+
if (yLines[i].length <= 2 && yLines[i].some((item) => item.text === "\uBE44\uACE0")) {
|
|
1889
|
+
bigoLineIdx = i;
|
|
1890
|
+
break;
|
|
1994
1891
|
}
|
|
1995
|
-
if (matches.length === 0) continue;
|
|
1996
|
-
let totalW = 0;
|
|
1997
|
-
for (const m of matches) totalW += m.w;
|
|
1998
|
-
if (totalW <= 0 || (line.x2 - line.x1) / totalW > STRIKE_MAX_LINE_TO_TEXT_RATIO) continue;
|
|
1999
|
-
for (const m of matches) m.strike = true;
|
|
2000
|
-
}
|
|
2001
|
-
}
|
|
2002
|
-
function wrapStrikethroughRuns(items) {
|
|
2003
|
-
const struck = items.filter((i) => i.strike);
|
|
2004
|
-
if (struck.length === 0) return;
|
|
2005
|
-
const lines = /* @__PURE__ */ new Map();
|
|
2006
|
-
for (const item of struck) {
|
|
2007
|
-
const key = Math.round(item.y / 3);
|
|
2008
|
-
const arr = lines.get(key) || [];
|
|
2009
|
-
arr.push(item);
|
|
2010
|
-
lines.set(key, arr);
|
|
2011
|
-
}
|
|
2012
|
-
for (const arr of lines.values()) {
|
|
2013
|
-
arr.sort((a, b) => a.x - b.x);
|
|
2014
|
-
arr[0].text = "~~" + arr[0].text;
|
|
2015
|
-
arr[arr.length - 1].text = arr[arr.length - 1].text + "~~";
|
|
2016
1892
|
}
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
const
|
|
2020
|
-
const
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
if (
|
|
2034
|
-
|
|
2035
|
-
tableItems.push(item);
|
|
2036
|
-
usedItems.add(item);
|
|
2037
|
-
}
|
|
2038
|
-
const cells = extractCells(grid, horizontals, verticals);
|
|
2039
|
-
if (cells.length === 0) continue;
|
|
2040
|
-
const textItems = tableItems.map((i) => ({
|
|
2041
|
-
text: i.text,
|
|
2042
|
-
x: i.x,
|
|
2043
|
-
y: i.y,
|
|
2044
|
-
w: i.w,
|
|
2045
|
-
h: i.h,
|
|
2046
|
-
fontSize: i.fontSize,
|
|
2047
|
-
fontName: i.fontName,
|
|
2048
|
-
hasSpaceBefore: i.hasSpaceBefore
|
|
2049
|
-
}));
|
|
2050
|
-
const cellTextMap = mapTextToCells(textItems, cells);
|
|
2051
|
-
const numRows = grid.rowYs.length - 1;
|
|
2052
|
-
const numCols = grid.colXs.length - 1;
|
|
2053
|
-
const irGrid = Array.from(
|
|
2054
|
-
{ length: numRows },
|
|
2055
|
-
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
2056
|
-
);
|
|
2057
|
-
for (const cell of cells) {
|
|
2058
|
-
const cellItems = cellTextMap.get(cell) || [];
|
|
2059
|
-
let text = cellTextToString(cellItems);
|
|
2060
|
-
text = text.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2061
|
-
text = text.split("\n").map((line) => collapseEvenSpacing(line)).join("\n");
|
|
2062
|
-
irGrid[cell.row][cell.col] = {
|
|
2063
|
-
text,
|
|
2064
|
-
colSpan: cell.colSpan,
|
|
2065
|
-
rowSpan: cell.rowSpan
|
|
2066
|
-
};
|
|
2067
|
-
}
|
|
2068
|
-
let finalGrid = irGrid;
|
|
2069
|
-
let finalRows = numRows;
|
|
2070
|
-
if (numRows <= 2 && numCols >= 3) {
|
|
2071
|
-
const rebuilt = normalizeUndersegmentedTable(irGrid, grid.colXs, textItems);
|
|
2072
|
-
if (rebuilt) {
|
|
2073
|
-
finalGrid = rebuilt.map((row) => row.map((rawText) => {
|
|
2074
|
-
const cleaned = rawText.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2075
|
-
return {
|
|
2076
|
-
text: cleaned.split("\n").map((line) => collapseEvenSpacing(line)).join("\n"),
|
|
2077
|
-
colSpan: 1,
|
|
2078
|
-
rowSpan: 1
|
|
2079
|
-
};
|
|
2080
|
-
}));
|
|
2081
|
-
finalRows = finalGrid.length;
|
|
1893
|
+
const tableYLines = bigoLineIdx >= 0 ? yLines.slice(0, bigoLineIdx) : yLines;
|
|
1894
|
+
const CLUSTER_TOL = 22;
|
|
1895
|
+
const xClusters = [];
|
|
1896
|
+
for (const line of tableYLines) {
|
|
1897
|
+
if (isProseSpread(line)) continue;
|
|
1898
|
+
for (const item of line) {
|
|
1899
|
+
let found = false;
|
|
1900
|
+
for (const c of xClusters) {
|
|
1901
|
+
if (Math.abs(item.x - c.center) <= CLUSTER_TOL) {
|
|
1902
|
+
c.center = Math.round((c.center * c.count + item.x) / (c.count + 1));
|
|
1903
|
+
c.minX = Math.min(c.minX, item.x);
|
|
1904
|
+
c.count++;
|
|
1905
|
+
found = true;
|
|
1906
|
+
break;
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
if (!found) {
|
|
1910
|
+
xClusters.push({ center: item.x, count: 1, minX: item.x });
|
|
2082
1911
|
}
|
|
2083
1912
|
}
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
const
|
|
2091
|
-
if (
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
x: grid.bbox.x1,
|
|
2095
|
-
y: grid.bbox.y1,
|
|
2096
|
-
width: grid.bbox.x2 - grid.bbox.x1,
|
|
2097
|
-
height: grid.bbox.y2 - grid.bbox.y1
|
|
2098
|
-
};
|
|
2099
|
-
if (shouldDemoteTable(irTable)) {
|
|
2100
|
-
const demoted = demoteTableToText(irTable);
|
|
2101
|
-
if (demoted) {
|
|
2102
|
-
const text = numGridRows === 1 ? "\n" + demoted + "\n" : demoted;
|
|
2103
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox: tableBbox, style: dominantStyle(tableItems) });
|
|
1913
|
+
}
|
|
1914
|
+
const peaks = xClusters.filter((c) => c.count >= 3).sort((a, b) => a.minX - b.minX);
|
|
1915
|
+
if (peaks.length < 3) return null;
|
|
1916
|
+
const MERGE_TOL = 40;
|
|
1917
|
+
const merged = [peaks[0]];
|
|
1918
|
+
for (let i = 1; i < peaks.length; i++) {
|
|
1919
|
+
const prev = merged[merged.length - 1];
|
|
1920
|
+
if (peaks[i].minX - prev.minX < MERGE_TOL) {
|
|
1921
|
+
if (peaks[i].count > prev.count) {
|
|
1922
|
+
prev.center = peaks[i].center;
|
|
2104
1923
|
}
|
|
2105
|
-
|
|
1924
|
+
prev.count += peaks[i].count;
|
|
1925
|
+
prev.minX = Math.min(prev.minX, peaks[i].minX);
|
|
1926
|
+
} else {
|
|
1927
|
+
merged.push({ ...peaks[i] });
|
|
2106
1928
|
}
|
|
2107
|
-
blocks.push({ type: "table", table: irTable, pageNumber: pageNum, bbox: tableBbox });
|
|
2108
1929
|
}
|
|
2109
|
-
|
|
2110
|
-
if (
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
1930
|
+
const rawColumns = merged.filter((c) => c.count >= 3).map((c) => c.minX);
|
|
1931
|
+
if (rawColumns.length < 3) return null;
|
|
1932
|
+
const MIN_DETECT_COL_WIDTH = 30;
|
|
1933
|
+
const columns = [rawColumns[0]];
|
|
1934
|
+
for (let i = 1; i < rawColumns.length; i++) {
|
|
1935
|
+
if (rawColumns[i] - columns[columns.length - 1] < MIN_DETECT_COL_WIDTH) continue;
|
|
1936
|
+
columns.push(rawColumns[i]);
|
|
1937
|
+
}
|
|
1938
|
+
return columns.length >= 3 ? columns : null;
|
|
1939
|
+
}
|
|
1940
|
+
function findColumn(x, columns) {
|
|
1941
|
+
for (let i = columns.length - 1; i >= 0; i--) {
|
|
1942
|
+
if (x >= columns[i] - 10) return i;
|
|
1943
|
+
}
|
|
1944
|
+
return 0;
|
|
1945
|
+
}
|
|
1946
|
+
function extractWithColumns(yLines, columns) {
|
|
1947
|
+
const result = [];
|
|
1948
|
+
const colMin = columns[0];
|
|
1949
|
+
const colMax = columns[columns.length - 1];
|
|
1950
|
+
let bigoIdx = -1;
|
|
1951
|
+
for (let i = 0; i < yLines.length; i++) {
|
|
1952
|
+
if (yLines[i].length <= 2 && yLines[i].some((item) => item.text === "\uBE44\uACE0")) {
|
|
1953
|
+
bigoIdx = i;
|
|
1954
|
+
break;
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
let tableStart = -1;
|
|
1958
|
+
for (let i = 0; i < (bigoIdx >= 0 ? bigoIdx : yLines.length); i++) {
|
|
1959
|
+
const usedCols = new Set(yLines[i].map((item) => findColumn(item.x, columns)));
|
|
1960
|
+
if (usedCols.size >= 3) {
|
|
1961
|
+
tableStart = i;
|
|
1962
|
+
break;
|
|
1963
|
+
}
|
|
1964
|
+
}
|
|
1965
|
+
const tableEnd = bigoIdx >= 0 ? bigoIdx : yLines.length;
|
|
1966
|
+
for (let i = 0; i < (tableStart >= 0 ? tableStart : tableEnd); i++) {
|
|
1967
|
+
result.push(mergeLineSimple(yLines[i]));
|
|
1968
|
+
}
|
|
1969
|
+
if (tableStart >= 0) {
|
|
1970
|
+
const tableLines = yLines.slice(tableStart, tableEnd);
|
|
1971
|
+
const gridLines = [];
|
|
1972
|
+
for (const line of tableLines) {
|
|
1973
|
+
const inRange = line.some(
|
|
1974
|
+
(item) => item.x >= colMin - 20 && item.x <= colMax + 200
|
|
1975
|
+
);
|
|
1976
|
+
if (inRange && !isProseSpread(line)) {
|
|
1977
|
+
gridLines.push(line);
|
|
1978
|
+
} else {
|
|
1979
|
+
if (gridLines.length > 0) {
|
|
1980
|
+
result.push(buildGridTable(gridLines.splice(0), columns));
|
|
2131
1981
|
}
|
|
2132
|
-
|
|
1982
|
+
result.push(mergeLineSimple(line));
|
|
2133
1983
|
}
|
|
2134
|
-
remaining = remaining.filter((_, idx) => !usedClusterIndices.has(idx));
|
|
2135
1984
|
}
|
|
2136
|
-
if (
|
|
2137
|
-
|
|
2138
|
-
const pageH = safeMax(allY) - safeMin(allY);
|
|
2139
|
-
const groups = xyCutOrder(remaining, Math.max(15, pageH * 0.03));
|
|
2140
|
-
const textBlocks = [];
|
|
2141
|
-
for (const group of groups) {
|
|
2142
|
-
if (group.length === 0) continue;
|
|
2143
|
-
const groupBlocks = extractPageBlocksFallback(group, pageNum);
|
|
2144
|
-
for (const b of groupBlocks) textBlocks.push(b);
|
|
2145
|
-
}
|
|
2146
|
-
const finalTextBlocks = detectListBlocks(textBlocks);
|
|
2147
|
-
for (const b of finalTextBlocks) blocks.push(b);
|
|
1985
|
+
if (gridLines.length > 0) {
|
|
1986
|
+
result.push(buildGridTable(gridLines, columns));
|
|
2148
1987
|
}
|
|
2149
|
-
blocks.sort((a, b) => {
|
|
2150
|
-
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2151
|
-
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2152
|
-
return by - ay;
|
|
2153
|
-
});
|
|
2154
|
-
return mergeAdjacentTableBlocks(blocks);
|
|
2155
1988
|
}
|
|
2156
|
-
|
|
1989
|
+
if (bigoIdx >= 0) {
|
|
1990
|
+
result.push("");
|
|
1991
|
+
for (let i = bigoIdx; i < yLines.length; i++) {
|
|
1992
|
+
result.push(mergeLineSimple(yLines[i]));
|
|
1993
|
+
}
|
|
1994
|
+
}
|
|
1995
|
+
return result.join("\n");
|
|
2157
1996
|
}
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
const
|
|
2162
|
-
const
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
if (prev.table.cols !== curr.table.cols) continue;
|
|
2166
|
-
if (!prev.bbox || !curr.bbox) continue;
|
|
2167
|
-
const width = Math.max(prev.bbox.width, curr.bbox.width, 1);
|
|
2168
|
-
const leftDiff = Math.abs(prev.bbox.x - curr.bbox.x);
|
|
2169
|
-
const rightDiff = Math.abs(prev.bbox.x + prev.bbox.width - (curr.bbox.x + curr.bbox.width));
|
|
2170
|
-
if (leftDiff > width * NEIGHBOR_TABLE_EPSILON || rightDiff > width * NEIGHBOR_TABLE_EPSILON) continue;
|
|
2171
|
-
let currCells = curr.table.cells;
|
|
2172
|
-
if (currCells.length > 1 && prev.table.cells.length > 0 && rowTextsEqual(prev.table.cells[0], currCells[0])) {
|
|
2173
|
-
currCells = currCells.slice(1);
|
|
1997
|
+
function buildGridTable(lines, columns) {
|
|
1998
|
+
const numCols = columns.length;
|
|
1999
|
+
const yRows = lines.map((items) => {
|
|
2000
|
+
const row = Array(numCols).fill("");
|
|
2001
|
+
for (const item of items) {
|
|
2002
|
+
const col = findColumn(item.x, columns);
|
|
2003
|
+
row[col] = row[col] ? row[col] + " " + item.text : item.text;
|
|
2174
2004
|
}
|
|
2175
|
-
|
|
2176
|
-
|
|
2005
|
+
return row;
|
|
2006
|
+
});
|
|
2007
|
+
const dataColStart = Math.max(2, Math.floor(numCols / 2));
|
|
2008
|
+
const merged = [];
|
|
2009
|
+
for (const row of yRows) {
|
|
2010
|
+
if (row.every((c) => c === "")) continue;
|
|
2011
|
+
if (merged.length === 0) {
|
|
2012
|
+
merged.push([...row]);
|
|
2177
2013
|
continue;
|
|
2178
2014
|
}
|
|
2179
|
-
const
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
};
|
|
2186
|
-
blocks[i] = { ...prev, table: merged };
|
|
2187
|
-
blocks.splice(i + 1, 1);
|
|
2188
|
-
}
|
|
2189
|
-
}
|
|
2190
|
-
function rowTextsEqual(a, b) {
|
|
2191
|
-
if (a.length !== b.length) return false;
|
|
2192
|
-
const norm = (t) => t.replace(/\s+/g, "");
|
|
2193
|
-
for (let i = 0; i < a.length; i++) {
|
|
2194
|
-
if (norm(a[i].text) !== norm(b[i].text)) return false;
|
|
2195
|
-
}
|
|
2196
|
-
return a.some((c) => c.text.trim() !== "");
|
|
2197
|
-
}
|
|
2198
|
-
function mergeAdjacentTableBlocks(blocks) {
|
|
2199
|
-
if (blocks.length <= 1) return blocks;
|
|
2200
|
-
const result = [blocks[0]];
|
|
2201
|
-
for (let i = 1; i < blocks.length; i++) {
|
|
2202
|
-
const prev = result[result.length - 1];
|
|
2203
|
-
const curr = blocks[i];
|
|
2204
|
-
if (prev.type === "table" && curr.type === "table" && prev.table && curr.table && prev.table.cols === curr.table.cols) {
|
|
2205
|
-
const merged = {
|
|
2206
|
-
rows: prev.table.rows + curr.table.rows,
|
|
2207
|
-
cols: prev.table.cols,
|
|
2208
|
-
cells: [...prev.table.cells, ...curr.table.cells],
|
|
2209
|
-
hasHeader: prev.table.hasHeader
|
|
2210
|
-
};
|
|
2211
|
-
result[result.length - 1] = { ...prev, table: merged };
|
|
2212
|
-
} else {
|
|
2213
|
-
result.push(curr);
|
|
2015
|
+
const prev = merged[merged.length - 1];
|
|
2016
|
+
const filledCols = row.map((c, i) => c ? i : -1).filter((i) => i >= 0);
|
|
2017
|
+
const filledCount = filledCols.length;
|
|
2018
|
+
let isNewRow = false;
|
|
2019
|
+
if (row[0] && row[0].length >= 3) {
|
|
2020
|
+
isNewRow = true;
|
|
2214
2021
|
}
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
}
|
|
2218
|
-
function extractPageBlocksFallback(items, pageNum) {
|
|
2219
|
-
if (items.length === 0) return [];
|
|
2220
|
-
const blocks = [];
|
|
2221
|
-
const clusterItems = items.map((i) => ({
|
|
2222
|
-
text: i.text,
|
|
2223
|
-
x: i.x,
|
|
2224
|
-
y: i.y,
|
|
2225
|
-
w: i.w,
|
|
2226
|
-
h: i.h,
|
|
2227
|
-
fontSize: i.fontSize,
|
|
2228
|
-
fontName: i.fontName,
|
|
2229
|
-
hasSpaceBefore: i.hasSpaceBefore
|
|
2230
|
-
}));
|
|
2231
|
-
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2232
|
-
if (clusterResults.length > 0) {
|
|
2233
|
-
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2234
|
-
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2235
|
-
const usedIndices = /* @__PURE__ */ new Set();
|
|
2236
|
-
for (const cr of clusterResults) {
|
|
2237
|
-
for (const ci of cr.usedItems) {
|
|
2238
|
-
const idx = ciToIdx.get(ci);
|
|
2239
|
-
if (idx !== void 0) usedIndices.add(idx);
|
|
2240
|
-
}
|
|
2241
|
-
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2022
|
+
if (!isNewRow && numCols > 1 && row[1]) {
|
|
2023
|
+
isNewRow = true;
|
|
2242
2024
|
}
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
const
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
if (!text.trim()) continue;
|
|
2249
|
-
const bbox = computeBBox(line, pageNum);
|
|
2250
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2025
|
+
if (!isNewRow) {
|
|
2026
|
+
const hasData = row.slice(dataColStart).some((c) => c !== "");
|
|
2027
|
+
const prevHasData = prev.slice(dataColStart).some((c) => c !== "");
|
|
2028
|
+
if (hasData && prevHasData) {
|
|
2029
|
+
isNewRow = true;
|
|
2251
2030
|
}
|
|
2252
2031
|
}
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
} else {
|
|
2259
|
-
const allYLines = mergeSuperscriptLines(groupByY(items));
|
|
2260
|
-
const columns = detectColumns(allYLines);
|
|
2261
|
-
if (columns && columns.length >= 3) {
|
|
2262
|
-
const tableText = extractWithColumns(allYLines, columns);
|
|
2263
|
-
const bbox = computeBBox(items, pageNum);
|
|
2264
|
-
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(items) });
|
|
2032
|
+
if (isNewRow && filledCount === 1 && row[0] && row[0].length <= 2) {
|
|
2033
|
+
isNewRow = false;
|
|
2034
|
+
}
|
|
2035
|
+
if (isNewRow) {
|
|
2036
|
+
merged.push([...row]);
|
|
2265
2037
|
} else {
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
const orderedGroups = xyCutOrder(items, gapThreshold);
|
|
2270
|
-
for (const group of orderedGroups) {
|
|
2271
|
-
if (group.length === 0) continue;
|
|
2272
|
-
const yLines = mergeSuperscriptLines(groupByY(group));
|
|
2273
|
-
const groupColumns = detectColumns(yLines);
|
|
2274
|
-
if (groupColumns && groupColumns.length >= 3) {
|
|
2275
|
-
const tableText = extractWithColumns(yLines, groupColumns);
|
|
2276
|
-
const bbox = computeBBox(group, pageNum);
|
|
2277
|
-
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
|
|
2278
|
-
} else {
|
|
2279
|
-
for (const line of yLines) {
|
|
2280
|
-
const text = mergeLineSimple(line);
|
|
2281
|
-
if (!text.trim()) continue;
|
|
2282
|
-
const bbox = computeBBox(line, pageNum);
|
|
2283
|
-
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2284
|
-
}
|
|
2038
|
+
for (let c = 0; c < numCols; c++) {
|
|
2039
|
+
if (row[c]) {
|
|
2040
|
+
prev[c] = prev[c] ? prev[c] + " " + row[c] : row[c];
|
|
2285
2041
|
}
|
|
2286
2042
|
}
|
|
2287
2043
|
}
|
|
2288
2044
|
}
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
function computeBBox(items, pageNum) {
|
|
2292
|
-
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
|
2293
|
-
for (const i of items) {
|
|
2294
|
-
if (i.x < minX) minX = i.x;
|
|
2295
|
-
if (i.y < minY) minY = i.y;
|
|
2296
|
-
if (i.x + i.w > maxX) maxX = i.x + i.w;
|
|
2297
|
-
const effectiveH = i.h > 0 ? i.h : i.fontSize;
|
|
2298
|
-
if (i.y + effectiveH > maxY) maxY = i.y + effectiveH;
|
|
2045
|
+
if (merged.length < 2) {
|
|
2046
|
+
return merged.map((r) => r.filter((c) => c).join(" ")).join("\n");
|
|
2299
2047
|
}
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
let maxCount = 0, dominantSize = 0;
|
|
2306
|
-
for (const i of items) {
|
|
2307
|
-
if (i.fontSize <= 0) continue;
|
|
2308
|
-
const count = (freq.get(i.fontSize) || 0) + 1;
|
|
2309
|
-
freq.set(i.fontSize, count);
|
|
2310
|
-
if (count > maxCount) {
|
|
2311
|
-
maxCount = count;
|
|
2312
|
-
dominantSize = i.fontSize;
|
|
2313
|
-
}
|
|
2048
|
+
let headerEnd = 0;
|
|
2049
|
+
for (let r = 0; r < merged.length; r++) {
|
|
2050
|
+
const hasDataValues = merged[r].slice(dataColStart).some((c) => c && /\d/.test(c));
|
|
2051
|
+
if (hasDataValues) break;
|
|
2052
|
+
headerEnd = r + 1;
|
|
2314
2053
|
}
|
|
2315
|
-
if (
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
for (const i of rawItems) {
|
|
2323
|
-
if (typeof i.str !== "string") continue;
|
|
2324
|
-
const x = Math.round(i.transform[4]);
|
|
2325
|
-
const y = Math.round(i.transform[5]);
|
|
2326
|
-
if (!i.str.trim()) {
|
|
2327
|
-
spacePositions.push({ x, y });
|
|
2328
|
-
continue;
|
|
2329
|
-
}
|
|
2330
|
-
const scaleY = Math.abs(i.transform[3]);
|
|
2331
|
-
const scaleX = Math.abs(i.transform[0]);
|
|
2332
|
-
const fontSize = Math.round(Math.max(scaleY, scaleX));
|
|
2333
|
-
const w = Math.round(i.width);
|
|
2334
|
-
const h = Math.round(i.height);
|
|
2335
|
-
const isHidden = fontSize === 0 || i.width === 0 && i.str.trim().length > 0;
|
|
2336
|
-
let text = i.str.trim();
|
|
2337
|
-
if (/^[\d\s\-().·,☎]+$/.test(text) && /\d/.test(text) && / /.test(text)) {
|
|
2338
|
-
text = text.replace(/ /g, "");
|
|
2339
|
-
}
|
|
2340
|
-
const split = splitEvenSpacedItem(text, x, w, fontSize);
|
|
2341
|
-
if (split) {
|
|
2342
|
-
for (const s of split) {
|
|
2343
|
-
items.push({ text: s.text, x: s.x, y, w: s.w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
2054
|
+
if (headerEnd > 1) {
|
|
2055
|
+
const headerRow = Array(numCols).fill("");
|
|
2056
|
+
for (let r = 0; r < headerEnd; r++) {
|
|
2057
|
+
for (let c = 0; c < numCols; c++) {
|
|
2058
|
+
if (merged[r][c]) {
|
|
2059
|
+
headerRow[c] = headerRow[c] ? headerRow[c] + " " + merged[r][c] : merged[r][c];
|
|
2060
|
+
}
|
|
2344
2061
|
}
|
|
2345
|
-
} else {
|
|
2346
|
-
items.push({ text, x, y, w, h, fontSize, fontName: i.fontName || "", isHidden });
|
|
2347
2062
|
}
|
|
2063
|
+
merged.splice(0, headerEnd, headerRow);
|
|
2348
2064
|
}
|
|
2349
|
-
const
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
let isDup = false;
|
|
2353
|
-
for (let j = deduped.length - 1; j >= 0; j--) {
|
|
2354
|
-
const prev = deduped[j];
|
|
2355
|
-
if (prev.y - sorted[i].y > 3) break;
|
|
2356
|
-
if (Math.abs(prev.y - sorted[i].y) <= 3 && prev.text === sorted[i].text && Math.abs(prev.x - sorted[i].x) <= 3) {
|
|
2357
|
-
isDup = true;
|
|
2358
|
-
break;
|
|
2359
|
-
}
|
|
2065
|
+
for (const row of merged) {
|
|
2066
|
+
for (let c = 0; c < row.length; c++) {
|
|
2067
|
+
if (row[c]) row[c] = collapseEvenSpacing(row[c]);
|
|
2360
2068
|
}
|
|
2361
|
-
if (!isDup) deduped.push(sorted[i]);
|
|
2362
2069
|
}
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
if (Math.abs(sp.y - item.y) > 3) continue;
|
|
2368
|
-
const dist = item.x - sp.x;
|
|
2369
|
-
if (dist >= -1 && dist <= 20 && (!nearest || item.x < nearest.x)) {
|
|
2370
|
-
nearest = item;
|
|
2371
|
-
}
|
|
2372
|
-
}
|
|
2373
|
-
if (nearest) nearest.hasSpaceBefore = true;
|
|
2374
|
-
}
|
|
2070
|
+
const totalCells = merged.length * numCols;
|
|
2071
|
+
const filledCells = merged.reduce((s, row) => s + row.filter((c) => c).length, 0);
|
|
2072
|
+
if (filledCells < totalCells * 0.35 || merged.length < 2 || merged.length <= 3 && numCols >= 7) {
|
|
2073
|
+
return merged.map((r) => r.filter((c) => c).join(" ")).join("\n");
|
|
2375
2074
|
}
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
if (chars.length < 3) return null;
|
|
2382
|
-
const charW = itemW / chars.length;
|
|
2383
|
-
if (charW > fontSize * 2) return null;
|
|
2384
|
-
return chars.map((ch, idx) => ({
|
|
2385
|
-
text: ch,
|
|
2386
|
-
x: Math.round(itemX + idx * charW),
|
|
2387
|
-
w: Math.round(charW * 0.8)
|
|
2388
|
-
// 실제 글자 폭은 간격보다 좁음
|
|
2389
|
-
}));
|
|
2390
|
-
}
|
|
2391
|
-
function groupByY(items) {
|
|
2392
|
-
if (items.length === 0) return [];
|
|
2393
|
-
const lines = [];
|
|
2394
|
-
let curY = items[0].y;
|
|
2395
|
-
let curLine = [items[0]];
|
|
2396
|
-
for (let i = 1; i < items.length; i++) {
|
|
2397
|
-
if (Math.abs(items[i].y - curY) > 3) {
|
|
2398
|
-
lines.push(curLine);
|
|
2399
|
-
curLine = [];
|
|
2400
|
-
curY = items[i].y;
|
|
2401
|
-
}
|
|
2402
|
-
curLine.push(items[i]);
|
|
2075
|
+
const md = [];
|
|
2076
|
+
md.push("| " + merged[0].join(" | ") + " |");
|
|
2077
|
+
md.push("| " + merged[0].map(() => "---").join(" | ") + " |");
|
|
2078
|
+
for (let r = 1; r < merged.length; r++) {
|
|
2079
|
+
md.push("| " + merged[r].join(" | ") + " |");
|
|
2403
2080
|
}
|
|
2404
|
-
|
|
2405
|
-
return lines;
|
|
2081
|
+
return md.join("\n");
|
|
2406
2082
|
}
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
const
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2083
|
+
|
|
2084
|
+
// src/pdf/block-detect.ts
|
|
2085
|
+
function computeMedianFontSizeFromFreq(freq) {
|
|
2086
|
+
if (freq.size === 0) return 0;
|
|
2087
|
+
let total = 0;
|
|
2088
|
+
for (const count of freq.values()) total += count;
|
|
2089
|
+
const sorted = [...freq.entries()].sort((a, b) => a[0] - b[0]);
|
|
2090
|
+
const mid = Math.floor(total / 2);
|
|
2091
|
+
let cumulative = 0;
|
|
2092
|
+
for (const [size, count] of sorted) {
|
|
2093
|
+
cumulative += count;
|
|
2094
|
+
if (cumulative > mid) return size;
|
|
2095
|
+
}
|
|
2096
|
+
return sorted[sorted.length - 1][0];
|
|
2097
|
+
}
|
|
2098
|
+
function detectHeadings(blocks, medianFontSize) {
|
|
2099
|
+
for (const block of blocks) {
|
|
2100
|
+
if (block.type !== "paragraph" || !block.text || !block.style?.fontSize) continue;
|
|
2101
|
+
const text = block.text.trim();
|
|
2102
|
+
if (text.length === 0 || text.length > 200) continue;
|
|
2103
|
+
if (/^\d+$/.test(text)) continue;
|
|
2104
|
+
const ratio = block.style.fontSize / medianFontSize;
|
|
2105
|
+
let level = 0;
|
|
2106
|
+
if (ratio >= HEADING_RATIO_H1) level = 1;
|
|
2107
|
+
else if (ratio >= HEADING_RATIO_H2) level = 2;
|
|
2108
|
+
else if (ratio >= HEADING_RATIO_H3) level = 3;
|
|
2109
|
+
if (level > 0) {
|
|
2110
|
+
block.type = "heading";
|
|
2111
|
+
block.level = level;
|
|
2112
|
+
block.text = collapseEvenSpacing(text);
|
|
2432
2113
|
}
|
|
2433
2114
|
}
|
|
2434
|
-
return result;
|
|
2435
2115
|
}
|
|
2436
|
-
function
|
|
2437
|
-
|
|
2438
|
-
const
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2116
|
+
function shouldDemoteTable(table) {
|
|
2117
|
+
const allCells = table.cells.flatMap((row) => row.map((c) => c.text.trim())).filter(Boolean);
|
|
2118
|
+
const allText = allCells.join(" ");
|
|
2119
|
+
if (table.rows <= 3 && table.cols <= 3) {
|
|
2120
|
+
const totalCells2 = table.rows * table.cols;
|
|
2121
|
+
const emptyCells2 = totalCells2 - allCells.length;
|
|
2122
|
+
if (emptyCells2 >= totalCells2 * 0.3) return true;
|
|
2123
|
+
if (/[□■◆○●▶ㅇ]/.test(allText)) return true;
|
|
2124
|
+
if (/<[^>]+>/.test(allText)) return true;
|
|
2442
2125
|
}
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2126
|
+
if (allText.length > 200) return false;
|
|
2127
|
+
if (/[□■◆○●▶]/.test(allText) && table.rows <= 3) return true;
|
|
2128
|
+
const totalCells = table.rows * table.cols;
|
|
2129
|
+
const emptyCells = totalCells - allCells.length;
|
|
2130
|
+
if (table.rows <= 2 && emptyCells > totalCells * 0.5) return true;
|
|
2131
|
+
if (table.rows === 1 && !/\d{2,}/.test(allText)) return true;
|
|
2132
|
+
return false;
|
|
2446
2133
|
}
|
|
2447
|
-
function
|
|
2448
|
-
const
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
break;
|
|
2134
|
+
function demoteTableToText(table) {
|
|
2135
|
+
const lines = [];
|
|
2136
|
+
for (let r = 0; r < table.rows; r++) {
|
|
2137
|
+
const cells = table.cells[r].map((c) => c.text.trim()).filter(Boolean);
|
|
2138
|
+
if (cells.length === 0) continue;
|
|
2139
|
+
if (table.cols === 2 && cells.length === 2) {
|
|
2140
|
+
lines.push(`${cells[0]} : ${cells[1]}`);
|
|
2141
|
+
} else {
|
|
2142
|
+
lines.push(cells.join(" "));
|
|
2457
2143
|
}
|
|
2458
2144
|
}
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
for (
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
c.count++;
|
|
2471
|
-
found = true;
|
|
2472
|
-
break;
|
|
2473
|
-
}
|
|
2474
|
-
}
|
|
2475
|
-
if (!found) {
|
|
2476
|
-
xClusters.push({ center: item.x, count: 1, minX: item.x });
|
|
2477
|
-
}
|
|
2145
|
+
return lines.join("\n");
|
|
2146
|
+
}
|
|
2147
|
+
function detectMarkerHeadings(blocks) {
|
|
2148
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2149
|
+
const block = blocks[i];
|
|
2150
|
+
if (block.type !== "paragraph" || !block.text) continue;
|
|
2151
|
+
const text = block.text.trim();
|
|
2152
|
+
if (text.length < 50 && /^[□■◆◇▶]\s*[가-힣]/.test(text)) {
|
|
2153
|
+
block.type = "heading";
|
|
2154
|
+
block.level = 4;
|
|
2155
|
+
continue;
|
|
2478
2156
|
}
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
if (peaks[i].count > prev.count) {
|
|
2488
|
-
prev.center = peaks[i].center;
|
|
2157
|
+
if (/^[가-힣]{2,6}$/.test(text) && block.style?.fontSize) {
|
|
2158
|
+
const prev = blocks[i - 1];
|
|
2159
|
+
const next = blocks[i + 1];
|
|
2160
|
+
const prevIsStructural = !prev || prev.type === "table" || prev.type === "heading" || prev.type === "separator";
|
|
2161
|
+
const nextIsStructural = !next || next.type === "table" || next.type === "heading" || next.type === "paragraph" && next.text && /^[□■◆○●]/.test(next.text.trim());
|
|
2162
|
+
if (prevIsStructural || nextIsStructural) {
|
|
2163
|
+
block.type = "heading";
|
|
2164
|
+
block.level = 3;
|
|
2489
2165
|
}
|
|
2490
|
-
prev.count += peaks[i].count;
|
|
2491
|
-
prev.minX = Math.min(prev.minX, peaks[i].minX);
|
|
2492
|
-
} else {
|
|
2493
|
-
merged.push({ ...peaks[i] });
|
|
2494
2166
|
}
|
|
2495
2167
|
}
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2168
|
+
}
|
|
2169
|
+
var TABLE_CAPTION_RE = /^[<\[(【〈]?\s*(표|그림|도표|Table|Figure|Fig\.?)\s*[\d①-⑮][\d.\-]*\s*[\])】〉>]?[.:]?\s*/i;
|
|
2170
|
+
var CAPTION_MAX_LENGTH = 100;
|
|
2171
|
+
var CAPTION_MAX_GAP = 30;
|
|
2172
|
+
function detectTableCaptions(blocks) {
|
|
2173
|
+
const isCaptionCandidate = (b, table) => {
|
|
2174
|
+
if (!b || b.type !== "paragraph" || !b.text) return false;
|
|
2175
|
+
if (b.pageNumber !== table.pageNumber) return false;
|
|
2176
|
+
const text = b.text.trim();
|
|
2177
|
+
if (!text || text.length > CAPTION_MAX_LENGTH || text.includes("\n")) return false;
|
|
2178
|
+
if (!TABLE_CAPTION_RE.test(text)) return false;
|
|
2179
|
+
if (b.bbox && table.bbox) {
|
|
2180
|
+
const capTop = b.bbox.y + b.bbox.height;
|
|
2181
|
+
const capBottom = b.bbox.y;
|
|
2182
|
+
const tblTop = table.bbox.y + table.bbox.height;
|
|
2183
|
+
const tblBottom = table.bbox.y;
|
|
2184
|
+
const gap = capBottom >= tblTop ? capBottom - tblTop : tblBottom - capTop;
|
|
2185
|
+
if (gap > CAPTION_MAX_GAP) return false;
|
|
2186
|
+
const overlap = Math.min(b.bbox.x + b.bbox.width, table.bbox.x + table.bbox.width) - Math.max(b.bbox.x, table.bbox.x);
|
|
2187
|
+
if (overlap < Math.min(b.bbox.width, table.bbox.width) * 0.3) return false;
|
|
2188
|
+
}
|
|
2189
|
+
return true;
|
|
2190
|
+
};
|
|
2191
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2192
|
+
const block = blocks[i];
|
|
2193
|
+
if (block.type !== "table" || !block.table || block.table.caption) continue;
|
|
2194
|
+
if (isCaptionCandidate(blocks[i - 1], block)) {
|
|
2195
|
+
block.table.caption = blocks[i - 1].text.trim();
|
|
2196
|
+
blocks.splice(i - 1, 1);
|
|
2197
|
+
i--;
|
|
2198
|
+
} else if (isCaptionCandidate(blocks[i + 1], block)) {
|
|
2199
|
+
block.table.caption = blocks[i + 1].text.trim();
|
|
2200
|
+
blocks.splice(i + 1, 1);
|
|
2201
|
+
}
|
|
2503
2202
|
}
|
|
2504
|
-
return columns.length >= 3 ? columns : null;
|
|
2505
2203
|
}
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2204
|
+
var KOREAN_LIST_SEQ = "\uAC00\uB098\uB2E4\uB77C\uB9C8\uBC14\uC0AC\uC544\uC790\uCC28\uCE74\uD0C0\uD30C\uD558";
|
|
2205
|
+
function parseListLabel(text) {
|
|
2206
|
+
let m = text.match(/^(\d{1,2})\.(?!\d)\s+/);
|
|
2207
|
+
if (m) return { family: "arabicDot", ord: parseInt(m[1], 10) };
|
|
2208
|
+
m = text.match(/^([가-하])\.\s+/);
|
|
2209
|
+
if (m) {
|
|
2210
|
+
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2211
|
+
if (idx >= 0) return { family: "korDot", ord: idx + 1 };
|
|
2509
2212
|
}
|
|
2510
|
-
|
|
2213
|
+
m = text.match(/^(\d{1,2})\)\s*/);
|
|
2214
|
+
if (m) return { family: "arabicParen", ord: parseInt(m[1], 10) };
|
|
2215
|
+
m = text.match(/^([가-하])\)\s*/);
|
|
2216
|
+
if (m) {
|
|
2217
|
+
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2218
|
+
if (idx >= 0) return { family: "korParen", ord: idx + 1 };
|
|
2219
|
+
}
|
|
2220
|
+
m = text.match(/^([①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮])\s*/);
|
|
2221
|
+
if (m) return { family: "circled", ord: m[1].charCodeAt(0) - 9312 + 1 };
|
|
2222
|
+
return null;
|
|
2511
2223
|
}
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
const
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
break;
|
|
2521
|
-
}
|
|
2224
|
+
var ATTACHMENT_RE = /^붙\s*임\s*(\d+[.:]?)?\s/;
|
|
2225
|
+
function detectKoreanListBlocks(blocks) {
|
|
2226
|
+
const labeled = [];
|
|
2227
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2228
|
+
const b = blocks[i];
|
|
2229
|
+
if (b.type !== "paragraph" && b.type !== "list" || !b.text) continue;
|
|
2230
|
+
const label = parseListLabel(b.text.trim());
|
|
2231
|
+
if (label) labeled.push({ idx: i, label });
|
|
2522
2232
|
}
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2233
|
+
const validated = /* @__PURE__ */ new Set();
|
|
2234
|
+
const byFamily = /* @__PURE__ */ new Map();
|
|
2235
|
+
for (const l of labeled) {
|
|
2236
|
+
const arr = byFamily.get(l.label.family) || [];
|
|
2237
|
+
arr.push(l);
|
|
2238
|
+
byFamily.set(l.label.family, arr);
|
|
2239
|
+
}
|
|
2240
|
+
for (const arr of byFamily.values()) {
|
|
2241
|
+
let chain = [];
|
|
2242
|
+
for (const item of arr) {
|
|
2243
|
+
const prev = chain[chain.length - 1];
|
|
2244
|
+
if (prev && item.label.ord === prev.label.ord + 1 && item.idx - prev.idx <= 20) {
|
|
2245
|
+
chain.push(item);
|
|
2246
|
+
} else {
|
|
2247
|
+
if (chain.length >= 2) for (const c of chain) validated.add(c.idx);
|
|
2248
|
+
chain = [item];
|
|
2249
|
+
}
|
|
2529
2250
|
}
|
|
2251
|
+
if (chain.length >= 2) for (const c of chain) validated.add(c.idx);
|
|
2530
2252
|
}
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
(item) => item.x >= colMin - 20 && item.x <= colMax + 200
|
|
2541
|
-
);
|
|
2542
|
-
if (inRange && !isProseSpread(line)) {
|
|
2543
|
-
gridLines.push(line);
|
|
2544
|
-
} else {
|
|
2545
|
-
if (gridLines.length > 0) {
|
|
2546
|
-
result.push(buildGridTable(gridLines.splice(0), columns));
|
|
2547
|
-
}
|
|
2548
|
-
result.push(mergeLineSimple(line));
|
|
2549
|
-
}
|
|
2253
|
+
let familyStack = [];
|
|
2254
|
+
let lastTopLevelList = null;
|
|
2255
|
+
const toRemove = /* @__PURE__ */ new Set();
|
|
2256
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2257
|
+
const b = blocks[i];
|
|
2258
|
+
if (b.type === "table" || b.type === "heading" || b.type === "separator") {
|
|
2259
|
+
familyStack = [];
|
|
2260
|
+
lastTopLevelList = null;
|
|
2261
|
+
continue;
|
|
2550
2262
|
}
|
|
2551
|
-
if (
|
|
2552
|
-
|
|
2263
|
+
if (b.type !== "paragraph" && b.type !== "list" || !b.text) continue;
|
|
2264
|
+
const text = b.text.trim();
|
|
2265
|
+
if (b.type === "paragraph" && ATTACHMENT_RE.test(text)) {
|
|
2266
|
+
blocks[i] = { ...b, type: "list", listType: "unordered" };
|
|
2267
|
+
continue;
|
|
2268
|
+
}
|
|
2269
|
+
if (!validated.has(i)) continue;
|
|
2270
|
+
const label = parseListLabel(text);
|
|
2271
|
+
let depth = familyStack.indexOf(label.family);
|
|
2272
|
+
if (depth < 0) {
|
|
2273
|
+
familyStack.push(label.family);
|
|
2274
|
+
depth = familyStack.length - 1;
|
|
2275
|
+
} else {
|
|
2276
|
+
familyStack = familyStack.slice(0, depth + 1);
|
|
2277
|
+
}
|
|
2278
|
+
const listType = label.family === "arabicDot" ? "ordered" : "unordered";
|
|
2279
|
+
const listBlock = { ...b, type: "list", listType };
|
|
2280
|
+
if (depth === 0) {
|
|
2281
|
+
blocks[i] = listBlock;
|
|
2282
|
+
lastTopLevelList = listBlock;
|
|
2283
|
+
} else if (lastTopLevelList) {
|
|
2284
|
+
if (!lastTopLevelList.children) lastTopLevelList.children = [];
|
|
2285
|
+
lastTopLevelList.children.push(listBlock);
|
|
2286
|
+
toRemove.add(i);
|
|
2287
|
+
} else {
|
|
2288
|
+
blocks[i] = listBlock;
|
|
2289
|
+
lastTopLevelList = listBlock;
|
|
2553
2290
|
}
|
|
2554
2291
|
}
|
|
2555
|
-
if (
|
|
2556
|
-
|
|
2557
|
-
for (
|
|
2558
|
-
|
|
2292
|
+
if (toRemove.size > 0) {
|
|
2293
|
+
const sorted = [...toRemove].sort((a, b) => b - a);
|
|
2294
|
+
for (const idx of sorted) blocks.splice(idx, 1);
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
function detectListBlocks(blocks) {
|
|
2298
|
+
const result = [];
|
|
2299
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2300
|
+
const block = blocks[i];
|
|
2301
|
+
if (block.type === "paragraph" && block.text) {
|
|
2302
|
+
const text = block.text.trim();
|
|
2303
|
+
if (/^\d+\.\s/.test(text)) {
|
|
2304
|
+
result.push({ ...block, type: "list", listType: "ordered", text: block.text });
|
|
2305
|
+
continue;
|
|
2306
|
+
}
|
|
2307
|
+
if (/^[○●·※▶▷◆◇\-]\s/.test(text)) {
|
|
2308
|
+
result.push({ ...block, type: "list", listType: "unordered", text: block.text });
|
|
2309
|
+
continue;
|
|
2310
|
+
}
|
|
2559
2311
|
}
|
|
2312
|
+
result.push(block);
|
|
2560
2313
|
}
|
|
2561
|
-
return result
|
|
2314
|
+
return result;
|
|
2562
2315
|
}
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2316
|
+
var KOREAN_TABLE_HEADER_RE = /^\(?(구분|항목|종류|분류|유형|대상|내용|기간|금액|비율|방법|절차|요건|조건|근거|목적|범위|기준)\)?[:\s]/;
|
|
2317
|
+
var KV_FALSE_POSITIVE_RE = /\d{1,2}:\d{2}|:\/\/|\d+:\d+/;
|
|
2318
|
+
function detectSpecialKoreanTables(blocks) {
|
|
2319
|
+
const result = [];
|
|
2320
|
+
let kvLines = [];
|
|
2321
|
+
const flushKvTable = () => {
|
|
2322
|
+
if (kvLines.length < 2) {
|
|
2323
|
+
for (const kv of kvLines) result.push(kv.block);
|
|
2324
|
+
kvLines = [];
|
|
2325
|
+
return;
|
|
2570
2326
|
}
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2327
|
+
const cells = kvLines.map((kv) => {
|
|
2328
|
+
if (kv.value) {
|
|
2329
|
+
return [
|
|
2330
|
+
{ text: kv.key, colSpan: 1, rowSpan: 1 },
|
|
2331
|
+
{ text: kv.value, colSpan: 1, rowSpan: 1 }
|
|
2332
|
+
];
|
|
2333
|
+
}
|
|
2334
|
+
return [
|
|
2335
|
+
{ text: kv.key, colSpan: 2, rowSpan: 1 },
|
|
2336
|
+
{ text: "", colSpan: 1, rowSpan: 1 }
|
|
2337
|
+
];
|
|
2338
|
+
});
|
|
2339
|
+
const irTable = {
|
|
2340
|
+
rows: cells.length,
|
|
2341
|
+
cols: 2,
|
|
2342
|
+
cells,
|
|
2343
|
+
hasHeader: true
|
|
2344
|
+
};
|
|
2345
|
+
const firstBlock = kvLines[0].block;
|
|
2346
|
+
result.push({
|
|
2347
|
+
type: "table",
|
|
2348
|
+
table: irTable,
|
|
2349
|
+
pageNumber: firstBlock.pageNumber,
|
|
2350
|
+
bbox: firstBlock.bbox
|
|
2351
|
+
});
|
|
2352
|
+
kvLines = [];
|
|
2353
|
+
};
|
|
2354
|
+
for (const block of blocks) {
|
|
2355
|
+
if (block.type !== "paragraph" || !block.text) {
|
|
2356
|
+
flushKvTable();
|
|
2357
|
+
result.push(block);
|
|
2579
2358
|
continue;
|
|
2580
2359
|
}
|
|
2581
|
-
const
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2360
|
+
const text = block.text.trim();
|
|
2361
|
+
if (KOREAN_TABLE_HEADER_RE.test(text)) {
|
|
2362
|
+
const colonIdx = text.indexOf(":");
|
|
2363
|
+
if (colonIdx >= 0) {
|
|
2364
|
+
kvLines.push({
|
|
2365
|
+
key: text.slice(0, colonIdx).trim(),
|
|
2366
|
+
value: text.slice(colonIdx + 1).trim(),
|
|
2367
|
+
block
|
|
2368
|
+
});
|
|
2369
|
+
} else {
|
|
2370
|
+
const spaceIdx = text.search(/\s/);
|
|
2371
|
+
if (spaceIdx > 0) {
|
|
2372
|
+
kvLines.push({
|
|
2373
|
+
key: text.slice(0, spaceIdx).trim(),
|
|
2374
|
+
value: text.slice(spaceIdx + 1).trim(),
|
|
2375
|
+
block
|
|
2376
|
+
});
|
|
2377
|
+
} else {
|
|
2378
|
+
kvLines.push({ key: text, value: "", block });
|
|
2379
|
+
}
|
|
2596
2380
|
}
|
|
2381
|
+
continue;
|
|
2597
2382
|
}
|
|
2598
|
-
if (
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2383
|
+
if (kvLines.length > 0 && text.includes(":")) {
|
|
2384
|
+
if (!KV_FALSE_POSITIVE_RE.test(text) && !text.includes("(") && !text.includes(")")) {
|
|
2385
|
+
const colonIdx = text.indexOf(":");
|
|
2386
|
+
const key = text.slice(0, colonIdx).trim();
|
|
2387
|
+
if (/^[가-힣]+$/.test(key) && key.length >= 2 && key.length <= 8) {
|
|
2388
|
+
kvLines.push({
|
|
2389
|
+
key,
|
|
2390
|
+
value: text.slice(colonIdx + 1).trim(),
|
|
2391
|
+
block
|
|
2392
|
+
});
|
|
2393
|
+
continue;
|
|
2607
2394
|
}
|
|
2608
2395
|
}
|
|
2609
2396
|
}
|
|
2397
|
+
flushKvTable();
|
|
2398
|
+
result.push(block);
|
|
2610
2399
|
}
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2400
|
+
flushKvTable();
|
|
2401
|
+
return result;
|
|
2402
|
+
}
|
|
2403
|
+
function removeHeaderFooterBlocks(blocks, pageHeights, warnings) {
|
|
2404
|
+
const ZONE_RATIO = 0.12;
|
|
2405
|
+
const MIN_REPEAT = 3;
|
|
2406
|
+
const topEntries = [];
|
|
2407
|
+
const bottomEntries = [];
|
|
2408
|
+
for (let bi = 0; bi < blocks.length; bi++) {
|
|
2409
|
+
const b = blocks[bi];
|
|
2410
|
+
if (!b.bbox || !b.pageNumber || !b.text?.trim()) continue;
|
|
2411
|
+
const ph = pageHeights.get(b.bbox.page) || pageHeights.get(b.pageNumber);
|
|
2412
|
+
if (!ph) continue;
|
|
2413
|
+
const blockTop = ph - (b.bbox.y + b.bbox.height);
|
|
2414
|
+
const blockBottom = ph - b.bbox.y;
|
|
2415
|
+
const entry = { blockIdx: bi, page: b.pageNumber, text: b.text.trim() };
|
|
2416
|
+
if (blockBottom <= ph * ZONE_RATIO) bottomEntries.push(entry);
|
|
2417
|
+
else if (blockTop >= ph * (1 - ZONE_RATIO)) topEntries.push(entry);
|
|
2619
2418
|
}
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2419
|
+
const removeSet = /* @__PURE__ */ new Set();
|
|
2420
|
+
for (const entries of [topEntries, bottomEntries]) {
|
|
2421
|
+
if (entries.length === 0) continue;
|
|
2422
|
+
const patternCount = /* @__PURE__ */ new Map();
|
|
2423
|
+
const patternPages = /* @__PURE__ */ new Map();
|
|
2424
|
+
for (const e of entries) {
|
|
2425
|
+
const norm = e.text.replace(/\d+/g, "#");
|
|
2426
|
+
patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
|
|
2427
|
+
const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
|
|
2428
|
+
pages.add(e.page);
|
|
2429
|
+
patternPages.set(norm, pages);
|
|
2430
|
+
}
|
|
2431
|
+
const repeatedPatterns = /* @__PURE__ */ new Set();
|
|
2432
|
+
for (const [p, count] of patternCount) {
|
|
2433
|
+
if (count >= MIN_REPEAT && (patternPages.get(p)?.size ?? 0) >= MIN_REPEAT) {
|
|
2434
|
+
repeatedPatterns.add(p);
|
|
2627
2435
|
}
|
|
2628
2436
|
}
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2437
|
+
for (const e of entries) {
|
|
2438
|
+
const norm = e.text.replace(/\d+/g, "#");
|
|
2439
|
+
if (repeatedPatterns.has(norm)) {
|
|
2440
|
+
removeSet.add(e.blockIdx);
|
|
2441
|
+
}
|
|
2634
2442
|
}
|
|
2635
2443
|
}
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
if (filledCells < totalCells * 0.35 || merged.length < 2 || merged.length <= 3 && numCols >= 7) {
|
|
2639
|
-
return merged.map((r) => r.filter((c) => c).join(" ")).join("\n");
|
|
2640
|
-
}
|
|
2641
|
-
const md = [];
|
|
2642
|
-
md.push("| " + merged[0].join(" | ") + " |");
|
|
2643
|
-
md.push("| " + merged[0].map(() => "---").join(" | ") + " |");
|
|
2644
|
-
for (let r = 1; r < merged.length; r++) {
|
|
2645
|
-
md.push("| " + merged[r].join(" | ") + " |");
|
|
2646
|
-
}
|
|
2647
|
-
return md.join("\n");
|
|
2648
|
-
}
|
|
2649
|
-
function mergeLineSimple(items) {
|
|
2650
|
-
if (items.length <= 1) return items[0]?.text || "";
|
|
2651
|
-
const sorted = [...items].sort((a, b) => a.x - b.x);
|
|
2652
|
-
const isEvenSpaced = detectEvenSpacedItems(sorted);
|
|
2653
|
-
let result = sorted[0].text;
|
|
2654
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
2655
|
-
const gap = sorted[i].x - (sorted[i - 1].x + sorted[i - 1].w);
|
|
2656
|
-
const avgFs = (sorted[i].fontSize + sorted[i - 1].fontSize) / 2;
|
|
2657
|
-
const tabThreshold = Math.max(avgFs * 2, 30);
|
|
2658
|
-
if (gap > tabThreshold) {
|
|
2659
|
-
result += " ";
|
|
2660
|
-
result += sorted[i].text;
|
|
2661
|
-
continue;
|
|
2662
|
-
}
|
|
2663
|
-
if (isEvenSpaced[i]) {
|
|
2664
|
-
result += sorted[i].text;
|
|
2665
|
-
continue;
|
|
2666
|
-
}
|
|
2667
|
-
if (sorted[i].hasSpaceBefore && gap >= avgFs * 0.05) {
|
|
2668
|
-
result += " ";
|
|
2669
|
-
result += sorted[i].text;
|
|
2670
|
-
continue;
|
|
2671
|
-
}
|
|
2672
|
-
if (/[□■○●▶◆◇ㅇ]$/.test(sorted[i - 1].text) && /^[가-힣]/.test(sorted[i].text) && gap > 1) {
|
|
2673
|
-
result += " ";
|
|
2674
|
-
result += sorted[i].text;
|
|
2675
|
-
continue;
|
|
2676
|
-
}
|
|
2677
|
-
if (gap > spaceGapThreshold(avgFs)) result += " ";
|
|
2678
|
-
result += sorted[i].text;
|
|
2444
|
+
if (removeSet.size > 0) {
|
|
2445
|
+
warnings.push({ message: `${removeSet.size}\uAC1C \uBA38\uB9AC\uAE00/\uBC14\uB2E5\uAE00 \uC694\uC18C \uC81C\uAC70\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
2679
2446
|
}
|
|
2680
|
-
return
|
|
2447
|
+
return [...removeSet].sort((a, b) => a - b);
|
|
2681
2448
|
}
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2449
|
+
|
|
2450
|
+
// src/pdf/page-blocks.ts
|
|
2451
|
+
function extractPageBlocksWithLines(items, pageNum, opList, pageWidth, pageHeight) {
|
|
2452
|
+
if (items.length === 0) return [];
|
|
2453
|
+
let { horizontals, verticals } = extractLines(opList.fnArray, opList.argsArray);
|
|
2454
|
+
({ horizontals, verticals } = filterPageBorderLines(horizontals, verticals, pageWidth, pageHeight));
|
|
2455
|
+
({ horizontals, verticals } = preprocessLines(horizontals, verticals));
|
|
2456
|
+
markStrikethroughItems(items, horizontals);
|
|
2457
|
+
wrapStrikethroughRuns(items);
|
|
2458
|
+
const grids = buildTableGrids(horizontals, verticals);
|
|
2459
|
+
if (grids.length > 0) {
|
|
2460
|
+
return extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals);
|
|
2693
2461
|
}
|
|
2462
|
+
return extractPageBlocksFallback(items, pageNum);
|
|
2694
2463
|
}
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
const isCaptionCandidate = (b, table) => {
|
|
2715
|
-
if (!b || b.type !== "paragraph" || !b.text) return false;
|
|
2716
|
-
if (b.pageNumber !== table.pageNumber) return false;
|
|
2717
|
-
const text = b.text.trim();
|
|
2718
|
-
if (!text || text.length > CAPTION_MAX_LENGTH || text.includes("\n")) return false;
|
|
2719
|
-
if (!TABLE_CAPTION_RE.test(text)) return false;
|
|
2720
|
-
if (b.bbox && table.bbox) {
|
|
2721
|
-
const capTop = b.bbox.y + b.bbox.height;
|
|
2722
|
-
const capBottom = b.bbox.y;
|
|
2723
|
-
const tblTop = table.bbox.y + table.bbox.height;
|
|
2724
|
-
const tblBottom = table.bbox.y;
|
|
2725
|
-
const gap = capBottom >= tblTop ? capBottom - tblTop : tblBottom - capTop;
|
|
2726
|
-
if (gap > CAPTION_MAX_GAP) return false;
|
|
2727
|
-
const overlap = Math.min(b.bbox.x + b.bbox.width, table.bbox.x + table.bbox.width) - Math.max(b.bbox.x, table.bbox.x);
|
|
2728
|
-
if (overlap < Math.min(b.bbox.width, table.bbox.width) * 0.3) return false;
|
|
2729
|
-
}
|
|
2730
|
-
return true;
|
|
2731
|
-
};
|
|
2732
|
-
for (let i = 0; i < blocks.length; i++) {
|
|
2733
|
-
const block = blocks[i];
|
|
2734
|
-
if (block.type !== "table" || !block.table || block.table.caption) continue;
|
|
2735
|
-
if (isCaptionCandidate(blocks[i - 1], block)) {
|
|
2736
|
-
block.table.caption = blocks[i - 1].text.trim();
|
|
2737
|
-
blocks.splice(i - 1, 1);
|
|
2738
|
-
i--;
|
|
2739
|
-
} else if (isCaptionCandidate(blocks[i + 1], block)) {
|
|
2740
|
-
block.table.caption = blocks[i + 1].text.trim();
|
|
2741
|
-
blocks.splice(i + 1, 1);
|
|
2464
|
+
var STRIKE_MAX_THICKNESS = 2;
|
|
2465
|
+
var STRIKE_MAX_THICKNESS_RATIO = 0.25;
|
|
2466
|
+
var STRIKE_CENTER_TOLERANCE = 0.25;
|
|
2467
|
+
var STRIKE_MIN_OVERLAP_RATIO = 0.8;
|
|
2468
|
+
var STRIKE_MAX_LINE_TO_TEXT_RATIO = 1.5;
|
|
2469
|
+
function markStrikethroughItems(items, horizontals) {
|
|
2470
|
+
if (items.length === 0 || horizontals.length === 0) return;
|
|
2471
|
+
for (const line of horizontals) {
|
|
2472
|
+
if (line.lineWidth > STRIKE_MAX_THICKNESS) continue;
|
|
2473
|
+
const matches = [];
|
|
2474
|
+
for (const item of items) {
|
|
2475
|
+
const h = item.h > 0 ? item.h : item.fontSize;
|
|
2476
|
+
if (h <= 0 || item.w <= 0) continue;
|
|
2477
|
+
if (line.lineWidth > h * STRIKE_MAX_THICKNESS_RATIO) continue;
|
|
2478
|
+
const centerY = item.y + h * 0.4;
|
|
2479
|
+
if (Math.abs(line.y1 - centerY) > h * STRIKE_CENTER_TOLERANCE) continue;
|
|
2480
|
+
const overlap = Math.min(line.x2, item.x + item.w) - Math.max(line.x1, item.x);
|
|
2481
|
+
if (overlap / item.w < STRIKE_MIN_OVERLAP_RATIO) continue;
|
|
2482
|
+
matches.push(item);
|
|
2742
2483
|
}
|
|
2484
|
+
if (matches.length === 0) continue;
|
|
2485
|
+
let totalW = 0;
|
|
2486
|
+
for (const m of matches) totalW += m.w;
|
|
2487
|
+
if (totalW <= 0 || (line.x2 - line.x1) / totalW > STRIKE_MAX_LINE_TO_TEXT_RATIO) continue;
|
|
2488
|
+
for (const m of matches) m.strike = true;
|
|
2743
2489
|
}
|
|
2744
2490
|
}
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
const
|
|
2752
|
-
|
|
2491
|
+
function wrapStrikethroughRuns(items) {
|
|
2492
|
+
const struck = items.filter((i) => i.strike);
|
|
2493
|
+
if (struck.length === 0) return;
|
|
2494
|
+
const lines = /* @__PURE__ */ new Map();
|
|
2495
|
+
for (const item of struck) {
|
|
2496
|
+
const key = Math.round(item.y / 3);
|
|
2497
|
+
const arr = lines.get(key) || [];
|
|
2498
|
+
arr.push(item);
|
|
2499
|
+
lines.set(key, arr);
|
|
2753
2500
|
}
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
const idx = KOREAN_LIST_SEQ.indexOf(m[1]);
|
|
2759
|
-
if (idx >= 0) return { family: "korParen", ord: idx + 1 };
|
|
2501
|
+
for (const arr of lines.values()) {
|
|
2502
|
+
arr.sort((a, b) => a.x - b.x);
|
|
2503
|
+
arr[0].text = "~~" + arr[0].text;
|
|
2504
|
+
arr[arr.length - 1].text = arr[arr.length - 1].text + "~~";
|
|
2760
2505
|
}
|
|
2761
|
-
m = text.match(/^([①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮])\s*/);
|
|
2762
|
-
if (m) return { family: "circled", ord: m[1].charCodeAt(0) - 9312 + 1 };
|
|
2763
|
-
return null;
|
|
2764
2506
|
}
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
const
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
|
|
2771
|
-
const
|
|
2772
|
-
if (
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
const
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2507
|
+
function extractBlocksWithGrids(items, pageNum, grids, horizontals, verticals) {
|
|
2508
|
+
const blocks = [];
|
|
2509
|
+
const usedItems = /* @__PURE__ */ new Set();
|
|
2510
|
+
const sortedGrids = [...grids].sort((a, b) => b.bbox.y2 - a.bbox.y2);
|
|
2511
|
+
for (const grid of sortedGrids) {
|
|
2512
|
+
const numGridRows = grid.rowYs.length - 1;
|
|
2513
|
+
const numGridCols = grid.colXs.length - 1;
|
|
2514
|
+
if (numGridRows === 1 && numGridCols >= 2) continue;
|
|
2515
|
+
if (numGridCols === 1 && numGridRows >= 2) continue;
|
|
2516
|
+
const tableItems = [];
|
|
2517
|
+
const pad = 3;
|
|
2518
|
+
const gridW = grid.bbox.x2 - grid.bbox.x1;
|
|
2519
|
+
for (const item of items) {
|
|
2520
|
+
if (usedItems.has(item)) continue;
|
|
2521
|
+
if (item.y < grid.bbox.y1 - pad || item.y > grid.bbox.y2 + pad) continue;
|
|
2522
|
+
if (item.x < grid.bbox.x1 - pad || item.x + item.w > grid.bbox.x2 + pad) continue;
|
|
2523
|
+
if (gridW < 120 && item.x + item.w > grid.bbox.x2 - 2) continue;
|
|
2524
|
+
tableItems.push(item);
|
|
2525
|
+
usedItems.add(item);
|
|
2526
|
+
}
|
|
2527
|
+
const cells = extractCells(grid, horizontals, verticals);
|
|
2528
|
+
if (cells.length === 0) continue;
|
|
2529
|
+
const textItems = tableItems.map((i) => ({
|
|
2530
|
+
text: i.text,
|
|
2531
|
+
x: i.x,
|
|
2532
|
+
y: i.y,
|
|
2533
|
+
w: i.w,
|
|
2534
|
+
h: i.h,
|
|
2535
|
+
fontSize: i.fontSize,
|
|
2536
|
+
fontName: i.fontName,
|
|
2537
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2538
|
+
}));
|
|
2539
|
+
const cellTextMap = mapTextToCells(textItems, cells);
|
|
2540
|
+
const numRows = grid.rowYs.length - 1;
|
|
2541
|
+
const numCols = grid.colXs.length - 1;
|
|
2542
|
+
const irGrid = Array.from(
|
|
2543
|
+
{ length: numRows },
|
|
2544
|
+
() => Array.from({ length: numCols }, () => ({ text: "", colSpan: 1, rowSpan: 1 }))
|
|
2545
|
+
);
|
|
2546
|
+
for (const cell of cells) {
|
|
2547
|
+
const cellItems = cellTextMap.get(cell) || [];
|
|
2548
|
+
let text = cellTextToString(cellItems);
|
|
2549
|
+
text = text.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2550
|
+
text = text.split("\n").map((line) => collapseEvenSpacing(line)).join("\n");
|
|
2551
|
+
irGrid[cell.row][cell.col] = {
|
|
2552
|
+
text,
|
|
2553
|
+
colSpan: cell.colSpan,
|
|
2554
|
+
rowSpan: cell.rowSpan
|
|
2555
|
+
};
|
|
2556
|
+
}
|
|
2557
|
+
let finalGrid = irGrid;
|
|
2558
|
+
let finalRows = numRows;
|
|
2559
|
+
if (numRows <= 2 && numCols >= 3) {
|
|
2560
|
+
const rebuilt = normalizeUndersegmentedTable(irGrid, grid.colXs, textItems);
|
|
2561
|
+
if (rebuilt) {
|
|
2562
|
+
finalGrid = rebuilt.map((row) => row.map((rawText) => {
|
|
2563
|
+
const cleaned = rawText.replace(/^[\s]*[-–—]\s*\d+\s*[-–—][\s]*$/gm, "").trim();
|
|
2564
|
+
return {
|
|
2565
|
+
text: cleaned.split("\n").map((line) => collapseEvenSpacing(line)).join("\n"),
|
|
2566
|
+
colSpan: 1,
|
|
2567
|
+
rowSpan: 1
|
|
2568
|
+
};
|
|
2569
|
+
}));
|
|
2570
|
+
finalRows = finalGrid.length;
|
|
2790
2571
|
}
|
|
2791
2572
|
}
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
const
|
|
2799
|
-
if (
|
|
2800
|
-
|
|
2801
|
-
|
|
2573
|
+
const irTable = {
|
|
2574
|
+
rows: finalRows,
|
|
2575
|
+
cols: numCols,
|
|
2576
|
+
cells: finalGrid,
|
|
2577
|
+
hasHeader: finalRows > 1
|
|
2578
|
+
};
|
|
2579
|
+
const hasContent = finalGrid.some((row) => row.some((cell) => cell.text.trim() !== ""));
|
|
2580
|
+
if (!hasContent) continue;
|
|
2581
|
+
const tableBbox = {
|
|
2582
|
+
page: pageNum,
|
|
2583
|
+
x: grid.bbox.x1,
|
|
2584
|
+
y: grid.bbox.y1,
|
|
2585
|
+
width: grid.bbox.x2 - grid.bbox.x1,
|
|
2586
|
+
height: grid.bbox.y2 - grid.bbox.y1
|
|
2587
|
+
};
|
|
2588
|
+
if (shouldDemoteTable(irTable)) {
|
|
2589
|
+
const demoted = demoteTableToText(irTable);
|
|
2590
|
+
if (demoted) {
|
|
2591
|
+
const text = numGridRows === 1 ? "\n" + demoted + "\n" : demoted;
|
|
2592
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox: tableBbox, style: dominantStyle(tableItems) });
|
|
2593
|
+
}
|
|
2802
2594
|
continue;
|
|
2803
2595
|
}
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2596
|
+
blocks.push({ type: "table", table: irTable, pageNumber: pageNum, bbox: tableBbox });
|
|
2597
|
+
}
|
|
2598
|
+
let remaining = items.filter((i) => !usedItems.has(i));
|
|
2599
|
+
if (remaining.length > 0) {
|
|
2600
|
+
remaining.sort((a, b) => b.y - a.y || a.x - b.x);
|
|
2601
|
+
const clusterItems = remaining.map((i) => ({
|
|
2602
|
+
text: i.text,
|
|
2603
|
+
x: i.x,
|
|
2604
|
+
y: i.y,
|
|
2605
|
+
w: i.w,
|
|
2606
|
+
h: i.h,
|
|
2607
|
+
fontSize: i.fontSize,
|
|
2608
|
+
fontName: i.fontName,
|
|
2609
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2610
|
+
}));
|
|
2611
|
+
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2612
|
+
if (clusterResults.length > 0) {
|
|
2613
|
+
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2614
|
+
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2615
|
+
const usedClusterIndices = /* @__PURE__ */ new Set();
|
|
2616
|
+
for (const cr of clusterResults) {
|
|
2617
|
+
for (const ci of cr.usedItems) {
|
|
2618
|
+
const idx = ciToIdx.get(ci);
|
|
2619
|
+
if (idx !== void 0) usedClusterIndices.add(idx);
|
|
2620
|
+
}
|
|
2621
|
+
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2622
|
+
}
|
|
2623
|
+
remaining = remaining.filter((_, idx) => !usedClusterIndices.has(idx));
|
|
2809
2624
|
}
|
|
2810
|
-
if (
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2625
|
+
if (remaining.length > 0) {
|
|
2626
|
+
const allY = remaining.map((i) => i.y);
|
|
2627
|
+
const pageH = safeMax(allY) - safeMin(allY);
|
|
2628
|
+
const groups = xyCutOrder(remaining, Math.max(15, pageH * 0.03));
|
|
2629
|
+
const textBlocks = [];
|
|
2630
|
+
for (const group of groups) {
|
|
2631
|
+
if (group.length === 0) continue;
|
|
2632
|
+
const groupBlocks = extractPageBlocksFallback(group, pageNum);
|
|
2633
|
+
for (const b of groupBlocks) textBlocks.push(b);
|
|
2634
|
+
}
|
|
2635
|
+
const finalTextBlocks = detectListBlocks(textBlocks);
|
|
2636
|
+
for (const b of finalTextBlocks) blocks.push(b);
|
|
2637
|
+
}
|
|
2638
|
+
blocks.sort((a, b) => {
|
|
2639
|
+
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2640
|
+
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2641
|
+
return by - ay;
|
|
2642
|
+
});
|
|
2643
|
+
return mergeAdjacentTableBlocks(blocks);
|
|
2644
|
+
}
|
|
2645
|
+
return mergeAdjacentTableBlocks(blocks);
|
|
2646
|
+
}
|
|
2647
|
+
var NEIGHBOR_TABLE_EPSILON = 0.2;
|
|
2648
|
+
function mergeCrossPageTables(blocks) {
|
|
2649
|
+
for (let i = blocks.length - 2; i >= 0; i--) {
|
|
2650
|
+
const prev = blocks[i];
|
|
2651
|
+
const curr = blocks[i + 1];
|
|
2652
|
+
if (prev.type !== "table" || curr.type !== "table" || !prev.table || !curr.table) continue;
|
|
2653
|
+
if (!prev.pageNumber || !curr.pageNumber || curr.pageNumber !== prev.pageNumber + 1) continue;
|
|
2654
|
+
if (prev.table.cols !== curr.table.cols) continue;
|
|
2655
|
+
if (!prev.bbox || !curr.bbox) continue;
|
|
2656
|
+
const width = Math.max(prev.bbox.width, curr.bbox.width, 1);
|
|
2657
|
+
const leftDiff = Math.abs(prev.bbox.x - curr.bbox.x);
|
|
2658
|
+
const rightDiff = Math.abs(prev.bbox.x + prev.bbox.width - (curr.bbox.x + curr.bbox.width));
|
|
2659
|
+
if (leftDiff > width * NEIGHBOR_TABLE_EPSILON || rightDiff > width * NEIGHBOR_TABLE_EPSILON) continue;
|
|
2660
|
+
let currCells = curr.table.cells;
|
|
2661
|
+
if (currCells.length > 1 && prev.table.cells.length > 0 && rowTextsEqual(prev.table.cells[0], currCells[0])) {
|
|
2662
|
+
currCells = currCells.slice(1);
|
|
2818
2663
|
}
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
blocks[i] = listBlock;
|
|
2823
|
-
lastTopLevelList = listBlock;
|
|
2824
|
-
} else if (lastTopLevelList) {
|
|
2825
|
-
if (!lastTopLevelList.children) lastTopLevelList.children = [];
|
|
2826
|
-
lastTopLevelList.children.push(listBlock);
|
|
2827
|
-
toRemove.add(i);
|
|
2828
|
-
} else {
|
|
2829
|
-
blocks[i] = listBlock;
|
|
2830
|
-
lastTopLevelList = listBlock;
|
|
2664
|
+
if (currCells.length === 0) {
|
|
2665
|
+
blocks.splice(i + 1, 1);
|
|
2666
|
+
continue;
|
|
2831
2667
|
}
|
|
2668
|
+
const merged = {
|
|
2669
|
+
rows: prev.table.rows + currCells.length,
|
|
2670
|
+
cols: prev.table.cols,
|
|
2671
|
+
cells: [...prev.table.cells, ...currCells],
|
|
2672
|
+
hasHeader: prev.table.hasHeader,
|
|
2673
|
+
caption: prev.table.caption
|
|
2674
|
+
};
|
|
2675
|
+
blocks[i] = { ...prev, table: merged };
|
|
2676
|
+
blocks.splice(i + 1, 1);
|
|
2832
2677
|
}
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2678
|
+
}
|
|
2679
|
+
function rowTextsEqual(a, b) {
|
|
2680
|
+
if (a.length !== b.length) return false;
|
|
2681
|
+
const norm = (t) => t.replace(/\s+/g, "");
|
|
2682
|
+
for (let i = 0; i < a.length; i++) {
|
|
2683
|
+
if (norm(a[i].text) !== norm(b[i].text)) return false;
|
|
2836
2684
|
}
|
|
2685
|
+
return a.some((c) => c.text.trim() !== "");
|
|
2837
2686
|
}
|
|
2838
|
-
function
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
}
|
|
2687
|
+
function mergeAdjacentTableBlocks(blocks) {
|
|
2688
|
+
if (blocks.length <= 1) return blocks;
|
|
2689
|
+
const result = [blocks[0]];
|
|
2690
|
+
for (let i = 1; i < blocks.length; i++) {
|
|
2691
|
+
const prev = result[result.length - 1];
|
|
2692
|
+
const curr = blocks[i];
|
|
2693
|
+
if (prev.type === "table" && curr.type === "table" && prev.table && curr.table && prev.table.cols === curr.table.cols) {
|
|
2694
|
+
const merged = {
|
|
2695
|
+
rows: prev.table.rows + curr.table.rows,
|
|
2696
|
+
cols: prev.table.cols,
|
|
2697
|
+
cells: [...prev.table.cells, ...curr.table.cells],
|
|
2698
|
+
hasHeader: prev.table.hasHeader
|
|
2699
|
+
};
|
|
2700
|
+
result[result.length - 1] = { ...prev, table: merged };
|
|
2701
|
+
} else {
|
|
2702
|
+
result.push(curr);
|
|
2852
2703
|
}
|
|
2853
|
-
result.push(block);
|
|
2854
2704
|
}
|
|
2855
2705
|
return result;
|
|
2856
2706
|
}
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
const
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2707
|
+
function extractPageBlocksFallback(items, pageNum) {
|
|
2708
|
+
if (items.length === 0) return [];
|
|
2709
|
+
const blocks = [];
|
|
2710
|
+
const clusterItems = items.map((i) => ({
|
|
2711
|
+
text: i.text,
|
|
2712
|
+
x: i.x,
|
|
2713
|
+
y: i.y,
|
|
2714
|
+
w: i.w,
|
|
2715
|
+
h: i.h,
|
|
2716
|
+
fontSize: i.fontSize,
|
|
2717
|
+
fontName: i.fontName,
|
|
2718
|
+
hasSpaceBefore: i.hasSpaceBefore
|
|
2719
|
+
}));
|
|
2720
|
+
const clusterResults = detectClusterTables(clusterItems, pageNum);
|
|
2721
|
+
if (clusterResults.length > 0) {
|
|
2722
|
+
const ciToIdx = /* @__PURE__ */ new Map();
|
|
2723
|
+
for (let ci = 0; ci < clusterItems.length; ci++) ciToIdx.set(clusterItems[ci], ci);
|
|
2724
|
+
const usedIndices = /* @__PURE__ */ new Set();
|
|
2725
|
+
for (const cr of clusterResults) {
|
|
2726
|
+
for (const ci of cr.usedItems) {
|
|
2727
|
+
const idx = ciToIdx.get(ci);
|
|
2728
|
+
if (idx !== void 0) usedIndices.add(idx);
|
|
2874
2729
|
}
|
|
2875
|
-
|
|
2876
|
-
{ text: kv.key, colSpan: 2, rowSpan: 1 },
|
|
2877
|
-
{ text: "", colSpan: 1, rowSpan: 1 }
|
|
2878
|
-
];
|
|
2879
|
-
});
|
|
2880
|
-
const irTable = {
|
|
2881
|
-
rows: cells.length,
|
|
2882
|
-
cols: 2,
|
|
2883
|
-
cells,
|
|
2884
|
-
hasHeader: true
|
|
2885
|
-
};
|
|
2886
|
-
const firstBlock = kvLines[0].block;
|
|
2887
|
-
result.push({
|
|
2888
|
-
type: "table",
|
|
2889
|
-
table: irTable,
|
|
2890
|
-
pageNumber: firstBlock.pageNumber,
|
|
2891
|
-
bbox: firstBlock.bbox
|
|
2892
|
-
});
|
|
2893
|
-
kvLines = [];
|
|
2894
|
-
};
|
|
2895
|
-
for (const block of blocks) {
|
|
2896
|
-
if (block.type !== "paragraph" || !block.text) {
|
|
2897
|
-
flushKvTable();
|
|
2898
|
-
result.push(block);
|
|
2899
|
-
continue;
|
|
2730
|
+
blocks.push({ type: "table", table: cr.table, pageNumber: pageNum, bbox: cr.bbox });
|
|
2900
2731
|
}
|
|
2901
|
-
const
|
|
2902
|
-
if (
|
|
2903
|
-
const
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
});
|
|
2910
|
-
} else {
|
|
2911
|
-
const spaceIdx = text.search(/\s/);
|
|
2912
|
-
if (spaceIdx > 0) {
|
|
2913
|
-
kvLines.push({
|
|
2914
|
-
key: text.slice(0, spaceIdx).trim(),
|
|
2915
|
-
value: text.slice(spaceIdx + 1).trim(),
|
|
2916
|
-
block
|
|
2917
|
-
});
|
|
2918
|
-
} else {
|
|
2919
|
-
kvLines.push({ key: text, value: "", block });
|
|
2920
|
-
}
|
|
2732
|
+
const remaining = items.filter((_, idx) => !usedIndices.has(idx));
|
|
2733
|
+
if (remaining.length > 0) {
|
|
2734
|
+
const yLines = mergeSuperscriptLines(groupByY(remaining));
|
|
2735
|
+
for (const line of yLines) {
|
|
2736
|
+
const text = mergeLineSimple(line);
|
|
2737
|
+
if (!text.trim()) continue;
|
|
2738
|
+
const bbox = computeBBox(line, pageNum);
|
|
2739
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2921
2740
|
}
|
|
2922
|
-
continue;
|
|
2923
2741
|
}
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2742
|
+
blocks.sort((a, b) => {
|
|
2743
|
+
const ay = a.bbox ? a.bbox.y + a.bbox.height : 0;
|
|
2744
|
+
const by = b.bbox ? b.bbox.y + b.bbox.height : 0;
|
|
2745
|
+
return by - ay;
|
|
2746
|
+
});
|
|
2747
|
+
} else {
|
|
2748
|
+
const allYLines = mergeSuperscriptLines(groupByY(items));
|
|
2749
|
+
const columns = detectColumns(allYLines);
|
|
2750
|
+
if (columns && columns.length >= 3) {
|
|
2751
|
+
const tableText = extractWithColumns(allYLines, columns);
|
|
2752
|
+
const bbox = computeBBox(items, pageNum);
|
|
2753
|
+
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(items) });
|
|
2754
|
+
} else {
|
|
2755
|
+
const allY = items.map((i) => i.y);
|
|
2756
|
+
const pageHeight = safeMax(allY) - safeMin(allY);
|
|
2757
|
+
const gapThreshold = Math.max(15, pageHeight * 0.03);
|
|
2758
|
+
const orderedGroups = xyCutOrder(items, gapThreshold);
|
|
2759
|
+
for (const group of orderedGroups) {
|
|
2760
|
+
if (group.length === 0) continue;
|
|
2761
|
+
const yLines = mergeSuperscriptLines(groupByY(group));
|
|
2762
|
+
const groupColumns = detectColumns(yLines);
|
|
2763
|
+
if (groupColumns && groupColumns.length >= 3) {
|
|
2764
|
+
const tableText = extractWithColumns(yLines, groupColumns);
|
|
2765
|
+
const bbox = computeBBox(group, pageNum);
|
|
2766
|
+
blocks.push({ type: "paragraph", text: tableText, pageNumber: pageNum, bbox, style: dominantStyle(group) });
|
|
2767
|
+
} else {
|
|
2768
|
+
for (const line of yLines) {
|
|
2769
|
+
const text = mergeLineSimple(line);
|
|
2770
|
+
if (!text.trim()) continue;
|
|
2771
|
+
const bbox = computeBBox(line, pageNum);
|
|
2772
|
+
blocks.push({ type: "paragraph", text, pageNumber: pageNum, bbox, style: dominantStyle(line) });
|
|
2773
|
+
}
|
|
2935
2774
|
}
|
|
2936
2775
|
}
|
|
2937
2776
|
}
|
|
2938
|
-
flushKvTable();
|
|
2939
|
-
result.push(block);
|
|
2940
2777
|
}
|
|
2941
|
-
|
|
2942
|
-
return result;
|
|
2778
|
+
return detectSpecialKoreanTables(blocks);
|
|
2943
2779
|
}
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
const
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
const blockTop = ph - (b.bbox.y + b.bbox.height);
|
|
2955
|
-
const blockBottom = ph - b.bbox.y;
|
|
2956
|
-
const entry = { blockIdx: bi, page: b.pageNumber, text: b.text.trim() };
|
|
2957
|
-
if (blockBottom <= ph * ZONE_RATIO) bottomEntries.push(entry);
|
|
2958
|
-
else if (blockTop >= ph * (1 - ZONE_RATIO)) topEntries.push(entry);
|
|
2959
|
-
}
|
|
2960
|
-
const removeSet = /* @__PURE__ */ new Set();
|
|
2961
|
-
for (const entries of [topEntries, bottomEntries]) {
|
|
2962
|
-
if (entries.length === 0) continue;
|
|
2963
|
-
const patternCount = /* @__PURE__ */ new Map();
|
|
2964
|
-
const patternPages = /* @__PURE__ */ new Map();
|
|
2965
|
-
for (const e of entries) {
|
|
2966
|
-
const norm = e.text.replace(/\d+/g, "#");
|
|
2967
|
-
patternCount.set(norm, (patternCount.get(norm) || 0) + 1);
|
|
2968
|
-
const pages = patternPages.get(norm) || /* @__PURE__ */ new Set();
|
|
2969
|
-
pages.add(e.page);
|
|
2970
|
-
patternPages.set(norm, pages);
|
|
2971
|
-
}
|
|
2972
|
-
const repeatedPatterns = /* @__PURE__ */ new Set();
|
|
2973
|
-
for (const [p, count] of patternCount) {
|
|
2974
|
-
if (count >= MIN_REPEAT && (patternPages.get(p)?.size ?? 0) >= MIN_REPEAT) {
|
|
2975
|
-
repeatedPatterns.add(p);
|
|
2976
|
-
}
|
|
2977
|
-
}
|
|
2978
|
-
for (const e of entries) {
|
|
2979
|
-
const norm = e.text.replace(/\d+/g, "#");
|
|
2980
|
-
if (repeatedPatterns.has(norm)) {
|
|
2981
|
-
removeSet.add(e.blockIdx);
|
|
2780
|
+
|
|
2781
|
+
// src/pdf/text-clean.ts
|
|
2782
|
+
function sanitizeBlockControlChars(blocks) {
|
|
2783
|
+
for (const b of blocks) {
|
|
2784
|
+
if (b.text) b.text = stripControlChars(b.text);
|
|
2785
|
+
if (b.table) {
|
|
2786
|
+
for (const row of b.table.cells) {
|
|
2787
|
+
for (const cell of row) {
|
|
2788
|
+
if (cell.text) cell.text = stripControlChars(cell.text);
|
|
2789
|
+
}
|
|
2982
2790
|
}
|
|
2983
2791
|
}
|
|
2792
|
+
if (b.children) sanitizeBlockControlChars(b.children);
|
|
2984
2793
|
}
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2794
|
+
}
|
|
2795
|
+
function cleanPdfText(text) {
|
|
2796
|
+
return mergeKoreanLines(
|
|
2797
|
+
stripControlChars(text).replace(/^\d{1,4}\n/, "").replace(/^[\s]*[-–—]\s*[-–—]?\d+[-–—]?[\s]*[-–—]?[\s]*$/gm, "").replace(/^\s*\d+\s*\/\s*\d+\s*$/gm, "").replace(/\n\d{1,4}\n/g, "\n").replace(/\n\d{1,4}$/, "").replace(/^#{1,6}\s*\d{1,4}\s*$/gm, "")
|
|
2798
|
+
).replace(/^(?!\| ---).*$/gm, (line) => {
|
|
2799
|
+
if (/^\s*\${1,2}.+\${1,2}\s*$/.test(line)) return line;
|
|
2800
|
+
return collapseEvenSpacing(line);
|
|
2801
|
+
}).replace(/([□■◆○●▶ㅇ])\s+([가-힣])\s+([가-힣])/g, "$1 $2$3").replace(/\\~\\~/g, "~~").replace(/~~~~/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
|
2802
|
+
}
|
|
2803
|
+
function startsWithMarker(line) {
|
|
2804
|
+
const t = line.trimStart();
|
|
2805
|
+
return /^[가-힣ㄱ-ㅎ][.)]/.test(t) || /^\d+[.)]/.test(t) || /^\([가-힣ㄱ-ㅎ\d]+\)/.test(t) || /^[○●※▶▷◆◇■□★☆\-·]\s/.test(t) || /^제\d+[조항호장절]/.test(t);
|
|
2806
|
+
}
|
|
2807
|
+
function isStandaloneHeader(line) {
|
|
2808
|
+
return /^제\d+[조항호장절](\([^)]*\))?(\s+\S+){0,7}$/.test(line.trim());
|
|
2989
2809
|
}
|
|
2990
2810
|
function mergeKoreanLines(text) {
|
|
2991
2811
|
if (!text) return "";
|
|
@@ -3016,6 +2836,8 @@ function mergeKoreanLines(text) {
|
|
|
3016
2836
|
}
|
|
3017
2837
|
return result.join("\n");
|
|
3018
2838
|
}
|
|
2839
|
+
|
|
2840
|
+
// src/pdf/formula-ocr.ts
|
|
3019
2841
|
async function applyFormulaOcr(buffer, blocks, pageFilter, effectivePageCount, warnings, _onProgress) {
|
|
3020
2842
|
const formulaMod = await import("./formula-RXVSQPXI.js");
|
|
3021
2843
|
const { FormulaPipeline, ensureFormulaModels } = formulaMod;
|
|
@@ -3134,6 +2956,251 @@ async function applyFormulaOcr(buffer, blocks, pageFilter, effectivePageCount, w
|
|
|
3134
2956
|
function formatMb(bytes) {
|
|
3135
2957
|
return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
|
|
3136
2958
|
}
|
|
2959
|
+
|
|
2960
|
+
// src/pdf/polyfill.ts
|
|
2961
|
+
import * as pdfjsWorker from "pdfjs-dist/legacy/build/pdf.worker.mjs";
|
|
2962
|
+
var g = globalThis;
|
|
2963
|
+
if (typeof g.DOMMatrix === "undefined") {
|
|
2964
|
+
g.DOMMatrix = class DOMMatrix {
|
|
2965
|
+
m = [1, 0, 0, 1, 0, 0];
|
|
2966
|
+
constructor(init) {
|
|
2967
|
+
if (init) this.m = init;
|
|
2968
|
+
}
|
|
2969
|
+
};
|
|
2970
|
+
}
|
|
2971
|
+
if (typeof g.Path2D === "undefined") {
|
|
2972
|
+
g.Path2D = class Path2D {
|
|
2973
|
+
};
|
|
2974
|
+
}
|
|
2975
|
+
g.pdfjsWorker = pdfjsWorker;
|
|
2976
|
+
|
|
2977
|
+
// src/pdf/parser.ts
|
|
2978
|
+
import { getDocument, GlobalWorkerOptions } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
2979
|
+
import { createRequire } from "module";
|
|
2980
|
+
import { dirname, join } from "path";
|
|
2981
|
+
GlobalWorkerOptions.workerSrc = "";
|
|
2982
|
+
var MAX_PAGES = 5e3;
|
|
2983
|
+
var MAX_TOTAL_TEXT = 100 * 1024 * 1024;
|
|
2984
|
+
var PDF_LOAD_TIMEOUT_MS = 3e4;
|
|
2985
|
+
var pdfjsAssets = {};
|
|
2986
|
+
try {
|
|
2987
|
+
const _require = createRequire(import.meta.url);
|
|
2988
|
+
const pkgDir = dirname(_require.resolve("pdfjs-dist/package.json"));
|
|
2989
|
+
pdfjsAssets.cMapUrl = join(pkgDir, "cmaps") + "/";
|
|
2990
|
+
pdfjsAssets.cMapPacked = true;
|
|
2991
|
+
pdfjsAssets.standardFontDataUrl = join(pkgDir, "standard_fonts") + "/";
|
|
2992
|
+
} catch {
|
|
2993
|
+
}
|
|
2994
|
+
async function loadPdfWithTimeout(buffer) {
|
|
2995
|
+
const loadingTask = getDocument({
|
|
2996
|
+
data: new Uint8Array(buffer),
|
|
2997
|
+
useSystemFonts: true,
|
|
2998
|
+
disableFontFace: true,
|
|
2999
|
+
isEvalSupported: false,
|
|
3000
|
+
...pdfjsAssets
|
|
3001
|
+
});
|
|
3002
|
+
let timer;
|
|
3003
|
+
try {
|
|
3004
|
+
return await Promise.race([
|
|
3005
|
+
loadingTask.promise,
|
|
3006
|
+
new Promise((_, reject) => {
|
|
3007
|
+
timer = setTimeout(() => {
|
|
3008
|
+
loadingTask.destroy();
|
|
3009
|
+
reject(new KordocError("PDF \uB85C\uB529 \uD0C0\uC784\uC544\uC6C3 (30\uCD08 \uCD08\uACFC)"));
|
|
3010
|
+
}, PDF_LOAD_TIMEOUT_MS);
|
|
3011
|
+
})
|
|
3012
|
+
]);
|
|
3013
|
+
} finally {
|
|
3014
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
3015
|
+
}
|
|
3016
|
+
}
|
|
3017
|
+
async function parsePdfDocument(buffer, options) {
|
|
3018
|
+
const formulaBuffer = options?.formulaOcr ? buffer.slice(0) : null;
|
|
3019
|
+
const doc = await loadPdfWithTimeout(buffer);
|
|
3020
|
+
try {
|
|
3021
|
+
const pageCount = doc.numPages;
|
|
3022
|
+
if (pageCount === 0) throw new KordocError("PDF\uC5D0 \uD398\uC774\uC9C0\uAC00 \uC5C6\uC2B5\uB2C8\uB2E4.");
|
|
3023
|
+
const metadata = { pageCount };
|
|
3024
|
+
await extractPdfMetadata(doc, metadata);
|
|
3025
|
+
const blocks = [];
|
|
3026
|
+
const warnings = [];
|
|
3027
|
+
const pageQuality = [];
|
|
3028
|
+
let totalChars = 0;
|
|
3029
|
+
let totalTextBytes = 0;
|
|
3030
|
+
const effectivePageCount = Math.min(pageCount, MAX_PAGES);
|
|
3031
|
+
const pageFilter = options?.pages ? parsePageRange(options.pages, effectivePageCount) : null;
|
|
3032
|
+
const totalTarget = pageFilter ? pageFilter.size : effectivePageCount;
|
|
3033
|
+
const fontSizeFreq = /* @__PURE__ */ new Map();
|
|
3034
|
+
const pageHeights = /* @__PURE__ */ new Map();
|
|
3035
|
+
const pagesWithLargeImage = /* @__PURE__ */ new Set();
|
|
3036
|
+
const skippedImagePages = /* @__PURE__ */ new Map();
|
|
3037
|
+
let parsedPages = 0;
|
|
3038
|
+
for (let i = 1; i <= effectivePageCount; i++) {
|
|
3039
|
+
if (pageFilter && !pageFilter.has(i)) continue;
|
|
3040
|
+
try {
|
|
3041
|
+
const page = await doc.getPage(i);
|
|
3042
|
+
const tc = await page.getTextContent();
|
|
3043
|
+
const viewport = page.getViewport({ scale: 1 });
|
|
3044
|
+
pageHeights.set(i, viewport.height);
|
|
3045
|
+
const rawItems = tc.items;
|
|
3046
|
+
const items = normalizeItems(rawItems);
|
|
3047
|
+
const { visible, hiddenCount } = filterHiddenText(items, viewport.width, viewport.height);
|
|
3048
|
+
if (hiddenCount > 0) {
|
|
3049
|
+
warnings.push({ page: i, message: `${hiddenCount}\uAC1C \uC228\uACA8\uC9C4 \uD14D\uC2A4\uD2B8 \uC694\uC18C \uD544\uD130\uB9C1\uB428`, code: "HIDDEN_TEXT_FILTERED" });
|
|
3050
|
+
}
|
|
3051
|
+
for (const item of visible) {
|
|
3052
|
+
if (item.fontSize > 0) fontSizeFreq.set(item.fontSize, (fontSizeFreq.get(item.fontSize) || 0) + 1);
|
|
3053
|
+
}
|
|
3054
|
+
const opList = await page.getOperatorList();
|
|
3055
|
+
const pageArea = viewport.width * viewport.height;
|
|
3056
|
+
if (pageArea > 0) {
|
|
3057
|
+
const imageRegions = extractImageRegions(opList.fnArray, opList.argsArray);
|
|
3058
|
+
let uncovered = 0;
|
|
3059
|
+
for (const r of imageRegions) {
|
|
3060
|
+
const area = (r.x2 - r.x1) * (r.y2 - r.y1);
|
|
3061
|
+
if (area < pageArea * 0.05) continue;
|
|
3062
|
+
pagesWithLargeImage.add(i);
|
|
3063
|
+
const hasText = visible.some((it) => {
|
|
3064
|
+
const cx = it.x + it.w / 2;
|
|
3065
|
+
const cy = it.y + (it.h || it.fontSize) / 2;
|
|
3066
|
+
return cx >= r.x1 && cx <= r.x2 && cy >= r.y1 && cy <= r.y2;
|
|
3067
|
+
});
|
|
3068
|
+
if (!hasText) uncovered++;
|
|
3069
|
+
}
|
|
3070
|
+
if (uncovered > 0) skippedImagePages.set(i, uncovered);
|
|
3071
|
+
}
|
|
3072
|
+
const pageBlocks = extractPageBlocksWithLines(visible, i, opList, viewport.width, viewport.height);
|
|
3073
|
+
for (const b of pageBlocks) blocks.push(b);
|
|
3074
|
+
let pageText = "";
|
|
3075
|
+
for (const b of pageBlocks) {
|
|
3076
|
+
const t = b.text || "";
|
|
3077
|
+
totalChars += t.replace(/\s/g, "").length;
|
|
3078
|
+
totalTextBytes += t.length * 2;
|
|
3079
|
+
pageText += pageText ? "\n" + t : t;
|
|
3080
|
+
}
|
|
3081
|
+
pageQuality.push(computePageQuality(i, pageText));
|
|
3082
|
+
if (totalTextBytes > MAX_TOTAL_TEXT) throw new KordocError("\uD14D\uC2A4\uD2B8 \uCD94\uCD9C \uD06C\uAE30 \uCD08\uACFC");
|
|
3083
|
+
parsedPages++;
|
|
3084
|
+
options?.onProgress?.(parsedPages, totalTarget);
|
|
3085
|
+
} catch (pageErr) {
|
|
3086
|
+
if (pageErr instanceof KordocError) throw pageErr;
|
|
3087
|
+
warnings.push({ page: i, message: `\uD398\uC774\uC9C0 ${i} \uD30C\uC2F1 \uC2E4\uD328: ${pageErr instanceof Error ? pageErr.message : "\uC54C \uC218 \uC5C6\uB294 \uC624\uB958"}`, code: "PARTIAL_PARSE" });
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
const parsedPageCount = parsedPages || (pageFilter ? pageFilter.size : effectivePageCount);
|
|
3091
|
+
let isImageBased = false;
|
|
3092
|
+
if (totalChars / Math.max(parsedPageCount, 1) < 10) {
|
|
3093
|
+
if (options?.ocr) {
|
|
3094
|
+
try {
|
|
3095
|
+
const { ocrPages } = await import("./provider-4ZJKV3DC.js");
|
|
3096
|
+
const ocrBlocks = await ocrPages(doc, options.ocr, pageFilter, effectivePageCount);
|
|
3097
|
+
if (ocrBlocks.length > 0) {
|
|
3098
|
+
const ocrMarkdown = ocrBlocks.map((b) => b.text || "").filter(Boolean).join("\n\n");
|
|
3099
|
+
return { markdown: ocrMarkdown, blocks: ocrBlocks, metadata, warnings, isImageBased: true, pageQuality, qualitySummary: summarizeDocumentQuality(pageQuality) };
|
|
3100
|
+
}
|
|
3101
|
+
} catch {
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
isImageBased = true;
|
|
3105
|
+
warnings.push({
|
|
3106
|
+
message: `\uC774\uBBF8\uC9C0 \uAE30\uBC18 PDF (${pageCount}\uD398\uC774\uC9C0, \uD14D\uC2A4\uD2B8 ${totalChars}\uC790) \u2014 \uD14D\uC2A4\uD2B8 \uB808\uC774\uC5B4\uAC00 \uC5C6\uC5B4 OCR\uC774 \uD544\uC694\uD569\uB2C8\uB2E4`,
|
|
3107
|
+
code: "NEEDS_OCR"
|
|
3108
|
+
});
|
|
3109
|
+
}
|
|
3110
|
+
if (!isImageBased) {
|
|
3111
|
+
const OCR_REASON_MESSAGES = {
|
|
3112
|
+
low_text: "\uD14D\uC2A4\uD2B8\uAC00 \uAC70\uC758 \uC5C6\uB294 \uD398\uC774\uC9C0 (\uC2A4\uCE94/\uC774\uBBF8\uC9C0 \uCD94\uC815)",
|
|
3113
|
+
high_pua: "\uAE00\uAF34 \uB9E4\uD551 \uC2E4\uD328 (PUA \uBE44\uC728 \uB192\uC74C) \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
3114
|
+
high_control: "\uC81C\uC5B4\uBB38\uC790 \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00",
|
|
3115
|
+
high_replacement: "\uB300\uCCB4\uBB38\uC790(U+FFFD) \uBE44\uC728 \uB192\uC74C \u2014 \uCD94\uCD9C \uD14D\uC2A4\uD2B8 \uC2E0\uB8B0 \uBD88\uAC00"
|
|
3116
|
+
};
|
|
3117
|
+
for (const pq of pageQuality) {
|
|
3118
|
+
if (!pq.needsOcr || !pq.ocrReason) continue;
|
|
3119
|
+
if (pq.ocrReason === "low_text" && !pagesWithLargeImage.has(pq.page)) continue;
|
|
3120
|
+
warnings.push({ page: pq.page, message: `${OCR_REASON_MESSAGES[pq.ocrReason]} \u2014 OCR \uAC80\uD1A0 \uD544\uC694`, code: "NEEDS_OCR" });
|
|
3121
|
+
}
|
|
3122
|
+
}
|
|
3123
|
+
if (!isImageBased) {
|
|
3124
|
+
for (const [page, count] of [...skippedImagePages.entries()].sort((a, b) => a[0] - b[0])) {
|
|
3125
|
+
warnings.push({ page, message: `${count}\uAC1C \uC774\uBBF8\uC9C0 \uC601\uC5ED\uC5D0 \uCD94\uCD9C \uAC00\uB2A5\uD55C \uD14D\uC2A4\uD2B8 \uC5C6\uC74C (\uADF8\uB9BC/\uCC28\uD2B8/\uB3C4\uC7A5 \uB0B4\uC6A9 \uB204\uB77D \uAC00\uB2A5)`, code: "SKIPPED_IMAGE" });
|
|
3126
|
+
}
|
|
3127
|
+
}
|
|
3128
|
+
if (options?.removeHeaderFooter !== false && parsedPageCount >= 3) {
|
|
3129
|
+
const removed = removeHeaderFooterBlocks(blocks, pageHeights, warnings);
|
|
3130
|
+
for (let ri = removed.length - 1; ri >= 0; ri--) {
|
|
3131
|
+
blocks.splice(removed[ri], 1);
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
mergeCrossPageTables(blocks);
|
|
3135
|
+
if (options?.formulaOcr && formulaBuffer) {
|
|
3136
|
+
try {
|
|
3137
|
+
await applyFormulaOcr(formulaBuffer, blocks, pageFilter, effectivePageCount, warnings, options.onProgress);
|
|
3138
|
+
} catch (e) {
|
|
3139
|
+
warnings.push({
|
|
3140
|
+
message: `\uC218\uC2DD OCR \uC2E4\uD328: ${e instanceof Error ? e.message : String(e)}`,
|
|
3141
|
+
code: "PARTIAL_PARSE"
|
|
3142
|
+
});
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
const medianFontSize = computeMedianFontSizeFromFreq(fontSizeFreq);
|
|
3146
|
+
if (medianFontSize > 0) {
|
|
3147
|
+
detectHeadings(blocks, medianFontSize);
|
|
3148
|
+
}
|
|
3149
|
+
detectMarkerHeadings(blocks);
|
|
3150
|
+
detectTableCaptions(blocks);
|
|
3151
|
+
detectKoreanListBlocks(blocks);
|
|
3152
|
+
const outline = blocks.filter((b) => b.type === "heading" && b.level && b.text).map((b) => ({ level: b.level, text: b.text, pageNumber: b.pageNumber }));
|
|
3153
|
+
sanitizeBlockControlChars(blocks);
|
|
3154
|
+
let markdown = cleanPdfText(blocksToMarkdown(blocks));
|
|
3155
|
+
return {
|
|
3156
|
+
markdown,
|
|
3157
|
+
blocks,
|
|
3158
|
+
metadata,
|
|
3159
|
+
outline: outline.length > 0 ? outline : void 0,
|
|
3160
|
+
warnings: warnings.length > 0 ? warnings : void 0,
|
|
3161
|
+
isImageBased: isImageBased || void 0,
|
|
3162
|
+
pageQuality,
|
|
3163
|
+
qualitySummary: summarizeDocumentQuality(pageQuality)
|
|
3164
|
+
};
|
|
3165
|
+
} finally {
|
|
3166
|
+
await doc.destroy().catch(() => {
|
|
3167
|
+
});
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
3170
|
+
async function extractPdfMetadata(doc, metadata) {
|
|
3171
|
+
try {
|
|
3172
|
+
const result = await doc.getMetadata();
|
|
3173
|
+
if (!result?.info) return;
|
|
3174
|
+
const info = result.info;
|
|
3175
|
+
if (typeof info.Title === "string" && info.Title.trim()) metadata.title = info.Title.trim();
|
|
3176
|
+
if (typeof info.Author === "string" && info.Author.trim()) metadata.author = info.Author.trim();
|
|
3177
|
+
if (typeof info.Creator === "string" && info.Creator.trim()) metadata.creator = info.Creator.trim();
|
|
3178
|
+
if (typeof info.Subject === "string" && info.Subject.trim()) metadata.description = info.Subject.trim();
|
|
3179
|
+
if (typeof info.Keywords === "string" && info.Keywords.trim()) {
|
|
3180
|
+
metadata.keywords = info.Keywords.split(/[,;]/).map((k) => k.trim()).filter(Boolean);
|
|
3181
|
+
}
|
|
3182
|
+
if (typeof info.CreationDate === "string") metadata.createdAt = parsePdfDate(info.CreationDate);
|
|
3183
|
+
if (typeof info.ModDate === "string") metadata.modifiedAt = parsePdfDate(info.ModDate);
|
|
3184
|
+
} catch {
|
|
3185
|
+
}
|
|
3186
|
+
}
|
|
3187
|
+
function parsePdfDate(dateStr) {
|
|
3188
|
+
const m = dateStr.match(/D:(\d{4})(\d{2})?(\d{2})?(\d{2})?(\d{2})?(\d{2})?/);
|
|
3189
|
+
if (!m) return void 0;
|
|
3190
|
+
const [, year, month = "01", day = "01", hour = "00", min = "00", sec = "00"] = m;
|
|
3191
|
+
return `${year}-${month}-${day}T${hour}:${min}:${sec}`;
|
|
3192
|
+
}
|
|
3193
|
+
async function extractPdfMetadataOnly(buffer) {
|
|
3194
|
+
const doc = await loadPdfWithTimeout(buffer);
|
|
3195
|
+
try {
|
|
3196
|
+
const metadata = { pageCount: doc.numPages };
|
|
3197
|
+
await extractPdfMetadata(doc, metadata);
|
|
3198
|
+
return metadata;
|
|
3199
|
+
} finally {
|
|
3200
|
+
await doc.destroy().catch(() => {
|
|
3201
|
+
});
|
|
3202
|
+
}
|
|
3203
|
+
}
|
|
3137
3204
|
export {
|
|
3138
3205
|
cleanPdfText,
|
|
3139
3206
|
detectKoreanListBlocks,
|
|
@@ -3143,4 +3210,4 @@ export {
|
|
|
3143
3210
|
parsePdfDocument,
|
|
3144
3211
|
removeHeaderFooterBlocks
|
|
3145
3212
|
};
|
|
3146
|
-
//# sourceMappingURL=parser-
|
|
3213
|
+
//# sourceMappingURL=parser-NR2TYGO3.js.map
|