@effortlessmotion/pdf-to-md 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -0
- package/bin/cli.js +9 -0
- package/dist/cjs/analyze.d.cts +35 -0
- package/dist/cjs/analyze.d.ts +35 -0
- package/dist/cjs/analyze.js +620 -0
- package/dist/cjs/analyze.js.map +1 -0
- package/dist/cjs/audit.d.cts +38 -0
- package/dist/cjs/audit.d.ts +38 -0
- package/dist/cjs/audit.js +252 -0
- package/dist/cjs/audit.js.map +1 -0
- package/dist/cjs/cli/audit.d.cts +3 -0
- package/dist/cjs/cli/audit.d.ts +3 -0
- package/dist/cjs/cli/audit.js +269 -0
- package/dist/cjs/cli/audit.js.map +1 -0
- package/dist/cjs/cli/extract.d.cts +3 -0
- package/dist/cjs/cli/extract.d.ts +3 -0
- package/dist/cjs/cli/extract.js +704 -0
- package/dist/cjs/cli/extract.js.map +1 -0
- package/dist/cjs/cli/index.d.cts +5 -0
- package/dist/cjs/cli/index.d.ts +5 -0
- package/dist/cjs/cli/index.js +58 -0
- package/dist/cjs/cli/index.js.map +1 -0
- package/dist/cjs/cli/render.d.cts +3 -0
- package/dist/cjs/cli/render.d.ts +3 -0
- package/dist/cjs/cli/render.js +361 -0
- package/dist/cjs/cli/render.js.map +1 -0
- package/dist/cjs/cli/types.d.cts +10 -0
- package/dist/cjs/cli/types.d.ts +10 -0
- package/dist/cjs/cli/types.js +7 -0
- package/dist/cjs/cli/types.js.map +1 -0
- package/dist/cjs/cli.d.cts +2 -0
- package/dist/cjs/cli.d.ts +2 -0
- package/dist/cjs/cli.js +345 -0
- package/dist/cjs/cli.js.map +1 -0
- package/dist/cjs/collect.d.cts +6 -0
- package/dist/cjs/collect.d.ts +6 -0
- package/dist/cjs/collect.js +370 -0
- package/dist/cjs/collect.js.map +1 -0
- package/dist/cjs/dict.d.cts +8 -0
- package/dist/cjs/dict.d.ts +8 -0
- package/dist/cjs/dict.js +55 -0
- package/dist/cjs/dict.js.map +1 -0
- package/dist/cjs/emit.d.cts +27 -0
- package/dist/cjs/emit.d.ts +27 -0
- package/dist/cjs/emit.js +187 -0
- package/dist/cjs/emit.js.map +1 -0
- package/dist/cjs/engines/tesseract.d.cts +30 -0
- package/dist/cjs/engines/tesseract.d.ts +30 -0
- package/dist/cjs/engines/tesseract.js +650 -0
- package/dist/cjs/engines/tesseract.js.map +1 -0
- package/dist/cjs/errors.d.cts +21 -0
- package/dist/cjs/errors.d.ts +21 -0
- package/dist/cjs/errors.js +124 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/extract.d.cts +88 -0
- package/dist/cjs/extract.d.ts +88 -0
- package/dist/cjs/extract.js +436 -0
- package/dist/cjs/extract.js.map +1 -0
- package/dist/cjs/geometry.d.cts +63 -0
- package/dist/cjs/geometry.d.ts +63 -0
- package/dist/cjs/geometry.js +521 -0
- package/dist/cjs/geometry.js.map +1 -0
- package/dist/cjs/index.d.cts +15 -0
- package/dist/cjs/index.d.ts +15 -0
- package/dist/cjs/index.js +56 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/lines.d.cts +3 -0
- package/dist/cjs/lines.d.ts +3 -0
- package/dist/cjs/lines.js +181 -0
- package/dist/cjs/lines.js.map +1 -0
- package/dist/cjs/markdown.d.cts +2 -0
- package/dist/cjs/markdown.d.ts +2 -0
- package/dist/cjs/markdown.js +222 -0
- package/dist/cjs/markdown.js.map +1 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/page-marker.d.cts +16 -0
- package/dist/cjs/page-marker.d.ts +16 -0
- package/dist/cjs/page-marker.js +85 -0
- package/dist/cjs/page-marker.js.map +1 -0
- package/dist/cjs/pdf-open.d.cts +12 -0
- package/dist/cjs/pdf-open.d.ts +12 -0
- package/dist/cjs/pdf-open.js +257 -0
- package/dist/cjs/pdf-open.js.map +1 -0
- package/dist/cjs/raster.d.cts +26 -0
- package/dist/cjs/raster.d.ts +26 -0
- package/dist/cjs/raster.js +262 -0
- package/dist/cjs/raster.js.map +1 -0
- package/dist/cjs/render-analysis.d.cts +25 -0
- package/dist/cjs/render-analysis.d.ts +25 -0
- package/dist/cjs/render-analysis.js +319 -0
- package/dist/cjs/render-analysis.js.map +1 -0
- package/dist/cjs/report.d.cts +12 -0
- package/dist/cjs/report.d.ts +12 -0
- package/dist/cjs/report.js +392 -0
- package/dist/cjs/report.js.map +1 -0
- package/dist/cjs/resolve.d.cts +1 -0
- package/dist/cjs/resolve.d.ts +1 -0
- package/dist/cjs/resolve.js +18 -0
- package/dist/cjs/resolve.js.map +1 -0
- package/dist/cjs/types.d.cts +171 -0
- package/dist/cjs/types.d.ts +171 -0
- package/dist/cjs/types.js +9 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/cjs/validate.d.cts +14 -0
- package/dist/cjs/validate.d.ts +14 -0
- package/dist/cjs/validate.js +96 -0
- package/dist/cjs/validate.js.map +1 -0
- package/dist/esm/analyze.d.ts +35 -0
- package/dist/esm/analyze.js +270 -0
- package/dist/esm/analyze.js.map +1 -0
- package/dist/esm/audit.d.ts +38 -0
- package/dist/esm/audit.js +113 -0
- package/dist/esm/audit.js.map +1 -0
- package/dist/esm/cli/audit.d.ts +3 -0
- package/dist/esm/cli/audit.js +20 -0
- package/dist/esm/cli/audit.js.map +1 -0
- package/dist/esm/cli/extract.d.ts +3 -0
- package/dist/esm/cli/extract.js +313 -0
- package/dist/esm/cli/extract.js.map +1 -0
- package/dist/esm/cli/index.d.ts +5 -0
- package/dist/esm/cli/index.js +7 -0
- package/dist/esm/cli/index.js.map +1 -0
- package/dist/esm/cli/render.d.ts +3 -0
- package/dist/esm/cli/render.js +77 -0
- package/dist/esm/cli/render.js.map +1 -0
- package/dist/esm/cli/types.d.ts +10 -0
- package/dist/esm/cli/types.js +3 -0
- package/dist/esm/cli/types.js.map +1 -0
- package/dist/esm/cli.d.ts +2 -0
- package/dist/esm/cli.js +90 -0
- package/dist/esm/cli.js.map +1 -0
- package/dist/esm/collect.d.ts +6 -0
- package/dist/esm/collect.js +80 -0
- package/dist/esm/collect.js.map +1 -0
- package/dist/esm/dict.d.ts +8 -0
- package/dist/esm/dict.js +28 -0
- package/dist/esm/dict.js.map +1 -0
- package/dist/esm/emit.d.ts +27 -0
- package/dist/esm/emit.js +131 -0
- package/dist/esm/emit.js.map +1 -0
- package/dist/esm/engines/tesseract.d.ts +30 -0
- package/dist/esm/engines/tesseract.js +190 -0
- package/dist/esm/engines/tesseract.js.map +1 -0
- package/dist/esm/errors.d.ts +21 -0
- package/dist/esm/errors.js +9 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/extract.d.ts +88 -0
- package/dist/esm/extract.js +147 -0
- package/dist/esm/extract.js.map +1 -0
- package/dist/esm/geometry.d.ts +63 -0
- package/dist/esm/geometry.js +279 -0
- package/dist/esm/geometry.js.map +1 -0
- package/dist/esm/index.d.ts +15 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/lines.d.ts +3 -0
- package/dist/esm/lines.js +67 -0
- package/dist/esm/lines.js.map +1 -0
- package/dist/esm/markdown.d.ts +2 -0
- package/dist/esm/markdown.js +153 -0
- package/dist/esm/markdown.js.map +1 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/page-marker.d.ts +16 -0
- package/dist/esm/page-marker.js +40 -0
- package/dist/esm/page-marker.js.map +1 -0
- package/dist/esm/pdf-open.d.ts +12 -0
- package/dist/esm/pdf-open.js +61 -0
- package/dist/esm/pdf-open.js.map +1 -0
- package/dist/esm/raster.d.ts +26 -0
- package/dist/esm/raster.js +60 -0
- package/dist/esm/raster.js.map +1 -0
- package/dist/esm/render-analysis.d.ts +25 -0
- package/dist/esm/render-analysis.js +115 -0
- package/dist/esm/render-analysis.js.map +1 -0
- package/dist/esm/report.d.ts +12 -0
- package/dist/esm/report.js +182 -0
- package/dist/esm/report.js.map +1 -0
- package/dist/esm/resolve.d.ts +1 -0
- package/dist/esm/resolve.js +7 -0
- package/dist/esm/resolve.js.map +1 -0
- package/dist/esm/types.d.ts +171 -0
- package/dist/esm/types.js +6 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/validate.d.ts +14 -0
- package/dist/esm/validate.js +56 -0
- package/dist/esm/validate.js.map +1 -0
- package/package.json +67 -0
- package/skills/pdf-to-md/SKILL.md +102 -0
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// geometry.ts — OCR-path pass 2: words -> lines/columns/floats. Input is
|
|
2
|
+
// flat words (normalized page coords, origin bottom-left, y up), not pre-clustered line observations.
|
|
3
|
+
/** Measured defaults; derivation in `test/unit/tuning.test.ts`. lineHeightRatio
|
|
4
|
+
* 2.4 = pair-F1 plateau edge (0.924@1.6 -> 0.995@2.4, 9,471 pairs, 2026-08-16). */ export const DEFAULT_TUNING = {
|
|
5
|
+
floatMaxWords: 3,
|
|
6
|
+
floatMaxWidth: 0.15,
|
|
7
|
+
floatMargin: 0.008,
|
|
8
|
+
paraGap: 1.6,
|
|
9
|
+
headingScale: 2.0,
|
|
10
|
+
colMaxWidth: 0.55,
|
|
11
|
+
colSplit: 0.5,
|
|
12
|
+
lineYTol: 0.006,
|
|
13
|
+
lineHeightRatio: 2.4
|
|
14
|
+
};
|
|
15
|
+
// Two-column decision (test/unit/column-break.test.ts): gutter ink <=0.017
|
|
16
|
+
// (2-col) vs >=0.047 (1-col) measured over 100 baseline pages, cut 0.03.
|
|
17
|
+
export const MAX_GUTTER_INK_FRAC = 0.03;
|
|
18
|
+
// Near-empty pages (e.g. a 2-word title page) have no gutter ink either, so
|
|
19
|
+
// a word-count floor is needed too; genuine columns measure >=76 words.
|
|
20
|
+
export const MIN_COL_WORDS = 10;
|
|
21
|
+
// Measured: a thin data rail is 11% of body words; the thinnest genuine
|
|
22
|
+
// column is 31% — 0.2 sits between.
|
|
23
|
+
export const MIN_COL_SHARE = 0.2;
|
|
24
|
+
const maxX = (r)=>r.x + r.w;
|
|
25
|
+
const maxY = (r)=>r.y + r.h;
|
|
26
|
+
const midX = (r)=>r.x + r.w / 2;
|
|
27
|
+
const midY = (r)=>r.y + r.h / 2;
|
|
28
|
+
function union(a, b) {
|
|
29
|
+
const x0 = Math.min(a.x, b.x);
|
|
30
|
+
const y0 = Math.min(a.y, b.y);
|
|
31
|
+
const x1 = Math.max(maxX(a), maxX(b));
|
|
32
|
+
const y1 = Math.max(maxY(a), maxY(b));
|
|
33
|
+
return {
|
|
34
|
+
x: x0,
|
|
35
|
+
y: y0,
|
|
36
|
+
w: x1 - x0,
|
|
37
|
+
h: y1 - y0
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
// Overlap of a rect's x-span with a column x-range.
|
|
41
|
+
function xOverlap(r, range) {
|
|
42
|
+
return Math.max(0, Math.min(maxX(r), range[1]) - Math.max(r.x, range[0]));
|
|
43
|
+
}
|
|
44
|
+
function median(xs) {
|
|
45
|
+
if (!xs.length) return undefined;
|
|
46
|
+
return [
|
|
47
|
+
...xs
|
|
48
|
+
].sort((a, b)=>a - b)[Math.floor(xs.length / 2)];
|
|
49
|
+
}
|
|
50
|
+
// Char-weighted dominant height: most characters wins (ties keep first) —
|
|
51
|
+
// same rule as src/lines.ts, so a small merged fragment can't promote a heading.
|
|
52
|
+
function dominantHeight(parts) {
|
|
53
|
+
let best = parts[0];
|
|
54
|
+
for(let i = 1; i < parts.length; i++){
|
|
55
|
+
if (parts[i].text.length > best.text.length) best = parts[i];
|
|
56
|
+
}
|
|
57
|
+
return best.h;
|
|
58
|
+
}
|
|
59
|
+
function assignColumn(rect, colRanges) {
|
|
60
|
+
if (colRanges.length !== 2) return 0;
|
|
61
|
+
const l = xOverlap(rect, colRanges[0]);
|
|
62
|
+
const r = xOverlap(rect, colRanges[1]);
|
|
63
|
+
if (l !== r) return l > r ? 0 : 1;
|
|
64
|
+
const centers = colRanges.map((rg)=>(rg[0] + rg[1]) / 2);
|
|
65
|
+
return Math.abs(midX(rect) - centers[0]) <= Math.abs(midX(rect) - centers[1]) ? 0 : 1;
|
|
66
|
+
}
|
|
67
|
+
// A column x-range from full-width body rows: short paragraph-final rows and
|
|
68
|
+
// decorative fragments must not stretch it.
|
|
69
|
+
function range(rects) {
|
|
70
|
+
if (rects.length < 2) return undefined;
|
|
71
|
+
return [
|
|
72
|
+
Math.min(...rects.map((r)=>r.x)),
|
|
73
|
+
Math.max(...rects.map(maxX))
|
|
74
|
+
];
|
|
75
|
+
}
|
|
76
|
+
// Groups same-visual-line words via lineYTol/lineHeightRatio, requiring band
|
|
77
|
+
// agreement; searches ALL open rows since per-word y drifts ~0.001 across a
|
|
78
|
+
// printed line, enough to interleave two lines' words in y-sorted order.
|
|
79
|
+
function buildRows(words, tuning, bandOf) {
|
|
80
|
+
const rows = [];
|
|
81
|
+
// Sort order only affects which compatible row a tie lands in (every row
|
|
82
|
+
// is searched) — top-to-bottom, left-to-right keeps results deterministic.
|
|
83
|
+
const sorted = [
|
|
84
|
+
...words
|
|
85
|
+
].sort((a, b)=>midY(b) - midY(a) || a.x - b.x);
|
|
86
|
+
for (const w of sorted){
|
|
87
|
+
const band = bandOf(w);
|
|
88
|
+
const target = rows.find((r)=>r.band === band && Math.abs(midY(r.rect) - midY(w)) <= tuning.lineYTol && Math.max(r.rect.h, w.h) < Math.min(r.rect.h, w.h) * tuning.lineHeightRatio);
|
|
89
|
+
if (target) {
|
|
90
|
+
target.words.push(w);
|
|
91
|
+
target.rect = union(target.rect, w);
|
|
92
|
+
} else {
|
|
93
|
+
rows.push({
|
|
94
|
+
rect: {
|
|
95
|
+
x: w.x,
|
|
96
|
+
y: w.y,
|
|
97
|
+
w: w.w,
|
|
98
|
+
h: w.h
|
|
99
|
+
},
|
|
100
|
+
words: [
|
|
101
|
+
w
|
|
102
|
+
],
|
|
103
|
+
band
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return rows;
|
|
108
|
+
}
|
|
109
|
+
function makeLine(parts, rect, col) {
|
|
110
|
+
const sorted = [
|
|
111
|
+
...parts
|
|
112
|
+
].sort((a, b)=>a.x - b.x);
|
|
113
|
+
const text = sorted.map((p)=>p.text).join(' ');
|
|
114
|
+
return {
|
|
115
|
+
text,
|
|
116
|
+
x: rect.x,
|
|
117
|
+
width: rect.w,
|
|
118
|
+
midY: midY(rect),
|
|
119
|
+
h: dominantHeight(parts),
|
|
120
|
+
col,
|
|
121
|
+
words: sorted
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
// Word-geometry pass (render/recognize happens upstream, in the OCR engine).
|
|
125
|
+
// Body-height words band left/right before row-grouping; heading-height words group across the full width, so a title stays one line.
|
|
126
|
+
export function buildPageOCR(words, tuning) {
|
|
127
|
+
var _median, _median1, _median2;
|
|
128
|
+
if (!words.length) return {
|
|
129
|
+
lines: [],
|
|
130
|
+
floats: [],
|
|
131
|
+
pitch: 0.012
|
|
132
|
+
};
|
|
133
|
+
const colBand = 0.02;
|
|
134
|
+
const colLeftCut = tuning.colSplit - colBand;
|
|
135
|
+
const colRightCut = tuning.colSplit + colBand;
|
|
136
|
+
const medianH = (_median = median(words.map((w)=>w.h))) !== null && _median !== void 0 ? _median : 0.017;
|
|
137
|
+
// Same literal `medianH * 2` used elsewhere for this gate — not
|
|
138
|
+
// `tuning.headingScale`, a separate constant that happens to share the 2.0 default.
|
|
139
|
+
const tallGate = medianH * 2;
|
|
140
|
+
// Two-column decision (test/unit/column-break.test.ts): gutter ink <=0.017
|
|
141
|
+
// (2-col) vs >=0.047 (1-col), cut 0.03; floors >=0.2 share/>=10 words per side.
|
|
142
|
+
const bodyWords = words.filter((w)=>w.h < tallGate);
|
|
143
|
+
const gutterInk = bodyWords.filter((w)=>w.x < tuning.colSplit && maxX(w) > tuning.colSplit).length;
|
|
144
|
+
const leftPop = bodyWords.filter((w)=>midX(w) < colLeftCut).length;
|
|
145
|
+
const rightPop = bodyWords.filter((w)=>midX(w) > colRightCut).length;
|
|
146
|
+
const twoCol = leftPop >= MIN_COL_WORDS && rightPop >= MIN_COL_WORDS && leftPop >= bodyWords.length * MIN_COL_SHARE && rightPop >= bodyWords.length * MIN_COL_SHARE && gutterInk <= bodyWords.length * MAX_GUTTER_INK_FRAC;
|
|
147
|
+
// Band by word CENTER, no dead band: a word box can touch the split exactly
|
|
148
|
+
// (a box can touch the split exactly) and still belongs to its center's column — the twoCol
|
|
149
|
+
// gate above already guarantees near-zero gutter ink.
|
|
150
|
+
const bandOf = (w)=>{
|
|
151
|
+
if (!twoCol) return 0;
|
|
152
|
+
return midX(w) < tuning.colSplit ? 0 : 1;
|
|
153
|
+
};
|
|
154
|
+
const tallRows = buildRows(words.filter((w)=>w.h >= tallGate), tuning, ()=>0);
|
|
155
|
+
const normalRows = buildRows(words.filter((w)=>w.h < tallGate), tuning, bandOf);
|
|
156
|
+
const rows = [
|
|
157
|
+
...tallRows,
|
|
158
|
+
...normalRows
|
|
159
|
+
];
|
|
160
|
+
// Column ranges from body rows (height <= medianH*1.6): two-col ranges use
|
|
161
|
+
// rows 0.2-0.55 wide; single-col has no upper bound (full-width rows ARE the column).
|
|
162
|
+
let colRanges;
|
|
163
|
+
if (twoCol) {
|
|
164
|
+
const wideRows = rows.filter((r)=>r.rect.w >= 0.2 && r.rect.w <= 0.55 && r.rect.h <= medianH * 1.6);
|
|
165
|
+
const left = range(wideRows.filter((r)=>r.band === 0).map((r)=>r.rect));
|
|
166
|
+
const right = range(wideRows.filter((r)=>r.band === 1).map((r)=>r.rect));
|
|
167
|
+
colRanges = [
|
|
168
|
+
left,
|
|
169
|
+
right
|
|
170
|
+
].filter((r)=>r !== undefined);
|
|
171
|
+
} else {
|
|
172
|
+
const wideRows = rows.filter((r)=>r.rect.w >= 0.2 && r.rect.h <= medianH * 1.6);
|
|
173
|
+
const single = range(wideRows.map((r)=>r.rect));
|
|
174
|
+
colRanges = single !== undefined ? [
|
|
175
|
+
single
|
|
176
|
+
] : [];
|
|
177
|
+
}
|
|
178
|
+
const lines = rows.map((row)=>makeLine(row.words, row.rect, assignColumn(row.rect, colRanges)));
|
|
179
|
+
lines.sort((a, b)=>a.col - b.col || b.midY - a.midY);
|
|
180
|
+
// Float detection: short+narrow+misaligned, rescued into body when it
|
|
181
|
+
// hangs under a body line like a paragraph continuation (same col, gap<=paraGap x pitch, x-overlap) — protects a centered paragraph's short last line.
|
|
182
|
+
const preGaps = [];
|
|
183
|
+
for(let k = 1; k < lines.length; k++){
|
|
184
|
+
if (lines[k].col === lines[k - 1].col) {
|
|
185
|
+
const g = lines[k - 1].midY - lines[k].midY;
|
|
186
|
+
if (g > 0.0005) preGaps.push(g);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
const provisionalPitch = (_median1 = median(preGaps)) !== null && _median1 !== void 0 ? _median1 : 0.012;
|
|
190
|
+
const colStarts = new Map();
|
|
191
|
+
colRanges.forEach((r, i)=>{
|
|
192
|
+
colStarts.set(i, r[0]);
|
|
193
|
+
});
|
|
194
|
+
const body = [];
|
|
195
|
+
const floatCandidates = [];
|
|
196
|
+
// Display-size junk (e.g. "1/3" at 68pt, 0.179 wide vs the 0.15 cap)
|
|
197
|
+
// escapes the narrow float test; caught here by junk text + isolation (no adjacent tall line within 2.2x height).
|
|
198
|
+
const tallLines = lines.filter((l)=>l.h >= tallGate);
|
|
199
|
+
for (const line of lines){
|
|
200
|
+
const wordCount = line.text.split(' ').filter(Boolean).length;
|
|
201
|
+
const start = colStarts.get(line.col);
|
|
202
|
+
const short = wordCount <= tuning.floatMaxWords && start !== undefined && Math.abs(line.x - start) > tuning.floatMargin;
|
|
203
|
+
const displayJunkFragment = short && line.h >= tallGate && !/[A-Za-z]{3}/.test(line.text) && // Same-column only, matching emitPage's heading-run gather — a heading
|
|
204
|
+
// in the OTHER column can be y-adjacent without composing with this one.
|
|
205
|
+
!tallLines.some((other)=>other !== line && other.col === line.col && Math.abs(other.midY - line.midY) < Math.max(other.h, line.h) * 2.2);
|
|
206
|
+
const looksDecorative = displayJunkFragment || short && line.width < tuning.floatMaxWidth;
|
|
207
|
+
const paragraphAdjacent = looksDecorative && body.some((b)=>{
|
|
208
|
+
if (b.col !== line.col) return false;
|
|
209
|
+
const gap = b.midY - line.midY;
|
|
210
|
+
if (gap <= 0 || gap > tuning.paraGap * provisionalPitch) return false;
|
|
211
|
+
const overlap = Math.min(b.x + b.width, line.x + line.width) - Math.max(b.x, line.x);
|
|
212
|
+
return overlap >= line.width * 0.5;
|
|
213
|
+
});
|
|
214
|
+
if (looksDecorative && !paragraphAdjacent) {
|
|
215
|
+
floatCandidates.push(line);
|
|
216
|
+
} else {
|
|
217
|
+
body.push(line);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
// One float per candidate row, in line order — no clustering of adjacent
|
|
221
|
+
// candidates: replay against the frozen baseline regressed ~30 pages when tried.
|
|
222
|
+
const floats = floatCandidates;
|
|
223
|
+
// Line pitch = median vertical gap between consecutive same-column lines.
|
|
224
|
+
const gaps = [];
|
|
225
|
+
for(let k = 1; k < body.length; k++){
|
|
226
|
+
if (body[k].col === body[k - 1].col) {
|
|
227
|
+
const g = body[k - 1].midY - body[k].midY;
|
|
228
|
+
if (g > 0.0005) gaps.push(g);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
lines: body,
|
|
233
|
+
floats,
|
|
234
|
+
pitch: (_median2 = median(gaps)) !== null && _median2 !== void 0 ? _median2 : 0.012
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
// Document-wide height histogram -> heading levels
|
|
238
|
+
// Bucket heights to 0.002 of page height so histogram keys are stable.
|
|
239
|
+
export function bucket(h) {
|
|
240
|
+
return Math.round(h * 500) / 500;
|
|
241
|
+
}
|
|
242
|
+
// `pages` are the already-body-only (post-float-filtering) PageOCR results.
|
|
243
|
+
export function headingLevels(pages, tuning) {
|
|
244
|
+
const chars = new Map();
|
|
245
|
+
for (const p of pages){
|
|
246
|
+
for (const l of p.lines){
|
|
247
|
+
var _chars_get;
|
|
248
|
+
const b = bucket(l.h);
|
|
249
|
+
chars.set(b, ((_chars_get = chars.get(b)) !== null && _chars_get !== void 0 ? _chars_get : 0) + l.text.length);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
// Most-common height wins; ties broken by first-encountered (deterministic;
|
|
253
|
+
// char-count ties across real documents are not observed).
|
|
254
|
+
let bodyH = 0;
|
|
255
|
+
let bestCount = -1;
|
|
256
|
+
for (const [k, v] of chars){
|
|
257
|
+
if (v > bestCount) {
|
|
258
|
+
bestCount = v;
|
|
259
|
+
bodyH = k;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
const levelOf = new Map();
|
|
263
|
+
let level = 3; // "### pN" page markers occupy level 3
|
|
264
|
+
let rep = null;
|
|
265
|
+
const sizes = [
|
|
266
|
+
...chars.keys()
|
|
267
|
+
].filter((s)=>s > bodyH * tuning.headingScale).sort((a, b)=>b - a);
|
|
268
|
+
for (const s of sizes){
|
|
269
|
+
if (rep === null || s < rep * 0.8) {
|
|
270
|
+
level = Math.min(6, level + 1);
|
|
271
|
+
rep = s;
|
|
272
|
+
}
|
|
273
|
+
levelOf.set(s, level);
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
bodyH,
|
|
277
|
+
levelOf
|
|
278
|
+
};
|
|
279
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/geometry.ts"],"sourcesContent":["// geometry.ts — OCR-path pass 2: words -> lines/columns/floats. Input is\n// flat words (normalized page coords, origin bottom-left, y up), not pre-clustered line observations.\nexport interface Word {\n text: string;\n x: number; // rect.minX\n y: number; // rect.minY\n w: number; // rect.width\n h: number; // rect.height\n // 0-100 (tesseract.js's own `confidence` field, not `conf`). Optional —\n // passthrough only; see report.ts's LOW_CONFIDENCE_THRESHOLD for the consumer.\n confidence?: number;\n}\n\n/** A `Word` with its page number — the shared shape behind both the OCR\n * engine's own output (tesseract.ts) and the replay input contract (extract.ts's OcrWordInput). */\nexport interface PageWord extends Word {\n page: number;\n}\n\nexport interface OcrLine {\n text: string;\n x: number; // rect.minX, used for float-alignment and indentation checks\n width: number; // rect.width, used for the float width check\n midY: number;\n h: number; // char-weighted dominant constituent height\n col: number;\n // Constituent words, left-to-right (same order as `text`'s tokens) — carries\n // word identity through row-building for analyze()'s block->word linkage.\n words: Word[];\n}\n\nexport interface PageOCR {\n lines: OcrLine[];\n // Decorative fragment candidates (src/emit.ts's floats aside), kept as full\n // OcrLine so callers recover both the printed text and constituent words.\n floats: OcrLine[];\n pitch: number;\n}\n\n/** Tuning knobs for the OCR-path geometry pass (CLI: `--float-*`/`--para-gap`/\n * `--heading-scale`/`--col-*`/`--line-*`). See DEFAULT_TUNING for defaults. */\nexport interface Tuning {\n /** fragments with more words than this are body text, not decoration */\n floatMaxWords: number;\n /** fraction of page width above which a fragment is too wide to be\n * decoration */\n floatMaxWidth: number;\n /** how far a fragment's left edge must sit from its column edge to be\n * decoration */\n floatMargin: number;\n /** a vertical gap this many times the line pitch is a paragraph break */\n paraGap: number;\n /** lines taller than this times body height are headings */\n headingScale: number;\n /** boxes wider than this are excluded from gutter detection */\n colMaxWidth: number;\n /** fractional x of the two-column midpoint */\n colSplit: number;\n /** vertical distance within which two boxes are the same line */\n lineYTol: number;\n /** same-line boxes with a larger height ratio don't merge */\n lineHeightRatio: number;\n}\n\n/** Measured defaults; derivation in `test/unit/tuning.test.ts`. lineHeightRatio\n * 2.4 = pair-F1 plateau edge (0.924@1.6 -> 0.995@2.4, 9,471 pairs, 2026-08-16). */\nexport const DEFAULT_TUNING: Tuning = {\n floatMaxWords: 3,\n floatMaxWidth: 0.15,\n floatMargin: 0.008,\n paraGap: 1.6,\n headingScale: 2.0,\n colMaxWidth: 0.55,\n colSplit: 0.5,\n lineYTol: 0.006,\n lineHeightRatio: 2.4,\n};\n\n// Two-column decision (test/unit/column-break.test.ts): gutter ink <=0.017\n// (2-col) vs >=0.047 (1-col) measured over 100 baseline pages, cut 0.03.\nexport const MAX_GUTTER_INK_FRAC = 0.03;\n\n// Near-empty pages (e.g. a 2-word title page) have no gutter ink either, so\n// a word-count floor is needed too; genuine columns measure >=76 words.\nexport const MIN_COL_WORDS = 10;\n\n// Measured: a thin data rail is 11% of body words; the thinnest genuine\n// column is 31% — 0.2 sits between.\nexport const MIN_COL_SHARE = 0.2;\n\ninterface Rect {\n x: number;\n y: number;\n w: number;\n h: number;\n}\ntype Range = readonly [number, number];\n\nconst maxX = (r: Rect): number => r.x + r.w;\nconst maxY = (r: Rect): number => r.y + r.h;\nconst midX = (r: Rect): number => r.x + r.w / 2;\nconst midY = (r: Rect): number => r.y + r.h / 2;\n\nfunction union(a: Rect, b: Rect): Rect {\n const x0 = Math.min(a.x, b.x);\n const y0 = Math.min(a.y, b.y);\n const x1 = Math.max(maxX(a), maxX(b));\n const y1 = Math.max(maxY(a), maxY(b));\n return { x: x0, y: y0, w: x1 - x0, h: y1 - y0 };\n}\n\n// Overlap of a rect's x-span with a column x-range.\nfunction xOverlap(r: Rect, range: Range): number {\n return Math.max(0, Math.min(maxX(r), range[1]) - Math.max(r.x, range[0]));\n}\n\nfunction median(xs: number[]): number | undefined {\n if (!xs.length) return undefined;\n return [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)];\n}\n\n// Char-weighted dominant height: most characters wins (ties keep first) —\n// same rule as src/lines.ts, so a small merged fragment can't promote a heading.\nfunction dominantHeight(parts: Word[]): number {\n let best = parts[0];\n for (let i = 1; i < parts.length; i++) {\n if (parts[i].text.length > best.text.length) best = parts[i];\n }\n return best.h;\n}\n\nfunction assignColumn(rect: Rect, colRanges: Range[]): number {\n if (colRanges.length !== 2) return 0;\n const l = xOverlap(rect, colRanges[0]);\n const r = xOverlap(rect, colRanges[1]);\n if (l !== r) return l > r ? 0 : 1;\n const centers = colRanges.map((rg) => (rg[0] + rg[1]) / 2);\n return Math.abs(midX(rect) - centers[0]) <= Math.abs(midX(rect) - centers[1]) ? 0 : 1;\n}\n\n// A column x-range from full-width body rows: short paragraph-final rows and\n// decorative fragments must not stretch it.\nfunction range(rects: Rect[]): Range | undefined {\n if (rects.length < 2) return undefined;\n return [Math.min(...rects.map((r) => r.x)), Math.max(...rects.map(maxX))];\n}\n\ninterface Row {\n rect: Rect;\n words: Word[];\n band: number; // -1 (gutter band / n/a), 0 (left or single column), 1 (right)\n}\n\n// Groups same-visual-line words via lineYTol/lineHeightRatio, requiring band\n// agreement; searches ALL open rows since per-word y drifts ~0.001 across a\n// printed line, enough to interleave two lines' words in y-sorted order.\nfunction buildRows(words: Word[], tuning: Tuning, bandOf: (w: Word) => number): Row[] {\n const rows: Row[] = [];\n // Sort order only affects which compatible row a tie lands in (every row\n // is searched) — top-to-bottom, left-to-right keeps results deterministic.\n const sorted = [...words].sort((a, b) => midY(b) - midY(a) || a.x - b.x);\n for (const w of sorted) {\n const band = bandOf(w);\n const target = rows.find((r) => r.band === band && Math.abs(midY(r.rect) - midY(w)) <= tuning.lineYTol && Math.max(r.rect.h, w.h) < Math.min(r.rect.h, w.h) * tuning.lineHeightRatio);\n if (target) {\n target.words.push(w);\n target.rect = union(target.rect, w);\n } else {\n rows.push({ rect: { x: w.x, y: w.y, w: w.w, h: w.h }, words: [w], band });\n }\n }\n return rows;\n}\n\nfunction makeLine(parts: Word[], rect: Rect, col: number): OcrLine {\n const sorted = [...parts].sort((a, b) => a.x - b.x);\n const text = sorted.map((p) => p.text).join(' ');\n return {\n text,\n x: rect.x,\n width: rect.w,\n midY: midY(rect),\n h: dominantHeight(parts),\n col,\n words: sorted,\n };\n}\n\n// Word-geometry pass (render/recognize happens upstream, in the OCR engine).\n// Body-height words band left/right before row-grouping; heading-height words group across the full width, so a title stays one line.\nexport function buildPageOCR(words: Word[], tuning: Tuning): PageOCR {\n if (!words.length) return { lines: [], floats: [], pitch: 0.012 };\n\n const colBand = 0.02;\n const colLeftCut = tuning.colSplit - colBand;\n const colRightCut = tuning.colSplit + colBand;\n\n const medianH = median(words.map((w) => w.h)) ?? 0.017;\n // Same literal `medianH * 2` used elsewhere for this gate — not\n // `tuning.headingScale`, a separate constant that happens to share the 2.0 default.\n const tallGate = medianH * 2;\n\n // Two-column decision (test/unit/column-break.test.ts): gutter ink <=0.017\n // (2-col) vs >=0.047 (1-col), cut 0.03; floors >=0.2 share/>=10 words per side.\n const bodyWords = words.filter((w) => w.h < tallGate);\n const gutterInk = bodyWords.filter((w) => w.x < tuning.colSplit && maxX(w) > tuning.colSplit).length;\n const leftPop = bodyWords.filter((w) => midX(w) < colLeftCut).length;\n const rightPop = bodyWords.filter((w) => midX(w) > colRightCut).length;\n const twoCol = leftPop >= MIN_COL_WORDS && rightPop >= MIN_COL_WORDS && leftPop >= bodyWords.length * MIN_COL_SHARE && rightPop >= bodyWords.length * MIN_COL_SHARE && gutterInk <= bodyWords.length * MAX_GUTTER_INK_FRAC;\n\n // Band by word CENTER, no dead band: a word box can touch the split exactly\n // (a box can touch the split exactly) and still belongs to its center's column — the twoCol\n // gate above already guarantees near-zero gutter ink.\n const bandOf = (w: Word): number => {\n if (!twoCol) return 0;\n return midX(w) < tuning.colSplit ? 0 : 1;\n };\n const tallRows = buildRows(\n words.filter((w) => w.h >= tallGate),\n tuning,\n () => 0\n );\n const normalRows = buildRows(\n words.filter((w) => w.h < tallGate),\n tuning,\n bandOf\n );\n const rows = [...tallRows, ...normalRows];\n\n // Column ranges from body rows (height <= medianH*1.6): two-col ranges use\n // rows 0.2-0.55 wide; single-col has no upper bound (full-width rows ARE the column).\n let colRanges: Range[];\n if (twoCol) {\n const wideRows = rows.filter((r) => r.rect.w >= 0.2 && r.rect.w <= 0.55 && r.rect.h <= medianH * 1.6);\n const left = range(wideRows.filter((r) => r.band === 0).map((r) => r.rect));\n const right = range(wideRows.filter((r) => r.band === 1).map((r) => r.rect));\n colRanges = [left, right].filter((r): r is Range => r !== undefined);\n } else {\n const wideRows = rows.filter((r) => r.rect.w >= 0.2 && r.rect.h <= medianH * 1.6);\n const single = range(wideRows.map((r) => r.rect));\n colRanges = single !== undefined ? [single] : [];\n }\n\n const lines: OcrLine[] = rows.map((row) => makeLine(row.words, row.rect, assignColumn(row.rect, colRanges)));\n lines.sort((a, b) => a.col - b.col || b.midY - a.midY);\n\n // Float detection: short+narrow+misaligned, rescued into body when it\n // hangs under a body line like a paragraph continuation (same col, gap<=paraGap x pitch, x-overlap) — protects a centered paragraph's short last line.\n const preGaps: number[] = [];\n for (let k = 1; k < lines.length; k++) {\n if (lines[k].col === lines[k - 1].col) {\n const g = lines[k - 1].midY - lines[k].midY;\n if (g > 0.0005) preGaps.push(g);\n }\n }\n const provisionalPitch = median(preGaps) ?? 0.012;\n const colStarts = new Map<number, number>();\n colRanges.forEach((r, i) => {\n colStarts.set(i, r[0]);\n });\n const body: OcrLine[] = [];\n const floatCandidates: OcrLine[] = [];\n // Display-size junk (e.g. \"1/3\" at 68pt, 0.179 wide vs the 0.15 cap)\n // escapes the narrow float test; caught here by junk text + isolation (no adjacent tall line within 2.2x height).\n const tallLines = lines.filter((l) => l.h >= tallGate);\n for (const line of lines) {\n const wordCount = line.text.split(' ').filter(Boolean).length;\n const start = colStarts.get(line.col);\n const short = wordCount <= tuning.floatMaxWords && start !== undefined && Math.abs(line.x - start) > tuning.floatMargin;\n const displayJunkFragment =\n short &&\n line.h >= tallGate &&\n !/[A-Za-z]{3}/.test(line.text) &&\n // Same-column only, matching emitPage's heading-run gather — a heading\n // in the OTHER column can be y-adjacent without composing with this one.\n !tallLines.some((other) => other !== line && other.col === line.col && Math.abs(other.midY - line.midY) < Math.max(other.h, line.h) * 2.2);\n const looksDecorative = displayJunkFragment || (short && line.width < tuning.floatMaxWidth);\n const paragraphAdjacent =\n looksDecorative &&\n body.some((b) => {\n if (b.col !== line.col) return false;\n const gap = b.midY - line.midY;\n if (gap <= 0 || gap > tuning.paraGap * provisionalPitch) return false;\n const overlap = Math.min(b.x + b.width, line.x + line.width) - Math.max(b.x, line.x);\n return overlap >= line.width * 0.5;\n });\n if (looksDecorative && !paragraphAdjacent) {\n floatCandidates.push(line);\n } else {\n body.push(line);\n }\n }\n // One float per candidate row, in line order — no clustering of adjacent\n // candidates: replay against the frozen baseline regressed ~30 pages when tried.\n const floats = floatCandidates;\n\n // Line pitch = median vertical gap between consecutive same-column lines.\n const gaps: number[] = [];\n for (let k = 1; k < body.length; k++) {\n if (body[k].col === body[k - 1].col) {\n const g = body[k - 1].midY - body[k].midY;\n if (g > 0.0005) gaps.push(g);\n }\n }\n return { lines: body, floats, pitch: median(gaps) ?? 0.012 };\n}\n\n// Document-wide height histogram -> heading levels\n\n// Bucket heights to 0.002 of page height so histogram keys are stable.\nexport function bucket(h: number): number {\n return Math.round(h * 500) / 500;\n}\n\n// `pages` are the already-body-only (post-float-filtering) PageOCR results.\nexport function headingLevels(pages: PageOCR[], tuning: Tuning): { bodyH: number; levelOf: Map<number, number> } {\n const chars = new Map<number, number>();\n for (const p of pages) {\n for (const l of p.lines) {\n const b = bucket(l.h);\n chars.set(b, (chars.get(b) ?? 0) + l.text.length);\n }\n }\n // Most-common height wins; ties broken by first-encountered (deterministic;\n // char-count ties across real documents are not observed).\n let bodyH = 0;\n let bestCount = -1;\n for (const [k, v] of chars) {\n if (v > bestCount) {\n bestCount = v;\n bodyH = k;\n }\n }\n\n const levelOf = new Map<number, number>();\n let level = 3; // \"### pN\" page markers occupy level 3\n let rep: number | null = null;\n const sizes = [...chars.keys()].filter((s) => s > bodyH * tuning.headingScale).sort((a, b) => b - a);\n for (const s of sizes) {\n if (rep === null || s < rep * 0.8) {\n level = Math.min(6, level + 1);\n rep = s;\n }\n levelOf.set(s, level);\n }\n return { bodyH, levelOf };\n}\n"],"names":["DEFAULT_TUNING","floatMaxWords","floatMaxWidth","floatMargin","paraGap","headingScale","colMaxWidth","colSplit","lineYTol","lineHeightRatio","MAX_GUTTER_INK_FRAC","MIN_COL_WORDS","MIN_COL_SHARE","maxX","r","x","w","maxY","y","h","midX","midY","union","a","b","x0","Math","min","y0","x1","max","y1","xOverlap","range","median","xs","length","undefined","sort","floor","dominantHeight","parts","best","i","text","assignColumn","rect","colRanges","l","centers","map","rg","abs","rects","buildRows","words","tuning","bandOf","rows","sorted","band","target","find","push","makeLine","col","p","join","width","buildPageOCR","lines","floats","pitch","colBand","colLeftCut","colRightCut","medianH","tallGate","bodyWords","filter","gutterInk","leftPop","rightPop","twoCol","tallRows","normalRows","wideRows","left","right","single","row","preGaps","k","g","provisionalPitch","colStarts","Map","forEach","set","body","floatCandidates","tallLines","line","wordCount","split","Boolean","start","get","short","displayJunkFragment","test","some","other","looksDecorative","paragraphAdjacent","gap","overlap","gaps","bucket","round","headingLevels","pages","chars","bodyH","bestCount","v","levelOf","level","rep","sizes","keys","s"],"mappings":"AAAA,yEAAyE;AACzE,sGAAsG;AA+DtG;iFACiF,GACjF,OAAO,MAAMA,iBAAyB;IACpCC,eAAe;IACfC,eAAe;IACfC,aAAa;IACbC,SAAS;IACTC,cAAc;IACdC,aAAa;IACbC,UAAU;IACVC,UAAU;IACVC,iBAAiB;AACnB,EAAE;AAEF,2EAA2E;AAC3E,yEAAyE;AACzE,OAAO,MAAMC,sBAAsB,KAAK;AAExC,4EAA4E;AAC5E,wEAAwE;AACxE,OAAO,MAAMC,gBAAgB,GAAG;AAEhC,wEAAwE;AACxE,oCAAoC;AACpC,OAAO,MAAMC,gBAAgB,IAAI;AAUjC,MAAMC,OAAO,CAACC,IAAoBA,EAAEC,CAAC,GAAGD,EAAEE,CAAC;AAC3C,MAAMC,OAAO,CAACH,IAAoBA,EAAEI,CAAC,GAAGJ,EAAEK,CAAC;AAC3C,MAAMC,OAAO,CAACN,IAAoBA,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAG;AAC9C,MAAMK,OAAO,CAACP,IAAoBA,EAAEI,CAAC,GAAGJ,EAAEK,CAAC,GAAG;AAE9C,SAASG,MAAMC,CAAO,EAAEC,CAAO;IAC7B,MAAMC,KAAKC,KAAKC,GAAG,CAACJ,EAAER,CAAC,EAAES,EAAET,CAAC;IAC5B,MAAMa,KAAKF,KAAKC,GAAG,CAACJ,EAAEL,CAAC,EAAEM,EAAEN,CAAC;IAC5B,MAAMW,KAAKH,KAAKI,GAAG,CAACjB,KAAKU,IAAIV,KAAKW;IAClC,MAAMO,KAAKL,KAAKI,GAAG,CAACb,KAAKM,IAAIN,KAAKO;IAClC,OAAO;QAAET,GAAGU;QAAIP,GAAGU;QAAIZ,GAAGa,KAAKJ;QAAIN,GAAGY,KAAKH;IAAG;AAChD;AAEA,oDAAoD;AACpD,SAASI,SAASlB,CAAO,EAAEmB,KAAY;IACrC,OAAOP,KAAKI,GAAG,CAAC,GAAGJ,KAAKC,GAAG,CAACd,KAAKC,IAAImB,KAAK,CAAC,EAAE,IAAIP,KAAKI,GAAG,CAAChB,EAAEC,CAAC,EAAEkB,KAAK,CAAC,EAAE;AACzE;AAEA,SAASC,OAAOC,EAAY;IAC1B,IAAI,CAACA,GAAGC,MAAM,EAAE,OAAOC;IACvB,OAAO;WAAIF;KAAG,CAACG,IAAI,CAAC,CAACf,GAAGC,IAAMD,IAAIC,EAAE,CAACE,KAAKa,KAAK,CAACJ,GAAGC,MAAM,GAAG,GAAG;AACjE;AAEA,0EAA0E;AAC1E,iFAAiF;AACjF,SAASI,eAAeC,KAAa;IACnC,IAAIC,OAAOD,KAAK,CAAC,EAAE;IACnB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,MAAML,MAAM,EAAEO,IAAK;QACrC,IAAIF,KAAK,CAACE,EAAE,CAACC,IAAI,CAACR,MAAM,GAAGM,KAAKE,IAAI,CAACR,MAAM,EAAEM,OAAOD,KAAK,CAACE,EAAE;IAC9D;IACA,OAAOD,KAAKvB,CAAC;AACf;AAEA,SAAS0B,aAAaC,IAAU,EAAEC,SAAkB;IAClD,IAAIA,UAAUX,MAAM,KAAK,GAAG,OAAO;IACnC,MAAMY,IAAIhB,SAASc,MAAMC,SAAS,CAAC,EAAE;IACrC,MAAMjC,IAAIkB,SAASc,MAAMC,SAAS,CAAC,EAAE;IACrC,IAAIC,MAAMlC,GAAG,OAAOkC,IAAIlC,IAAI,IAAI;IAChC,MAAMmC,UAAUF,UAAUG,GAAG,CAAC,CAACC,KAAO,AAACA,CAAAA,EAAE,CAAC,EAAE,GAAGA,EAAE,CAAC,EAAE,AAAD,IAAK;IACxD,OAAOzB,KAAK0B,GAAG,CAAChC,KAAK0B,QAAQG,OAAO,CAAC,EAAE,KAAKvB,KAAK0B,GAAG,CAAChC,KAAK0B,QAAQG,OAAO,CAAC,EAAE,IAAI,IAAI;AACtF;AAEA,6EAA6E;AAC7E,4CAA4C;AAC5C,SAAShB,MAAMoB,KAAa;IAC1B,IAAIA,MAAMjB,MAAM,GAAG,GAAG,OAAOC;IAC7B,OAAO;QAACX,KAAKC,GAAG,IAAI0B,MAAMH,GAAG,CAAC,CAACpC,IAAMA,EAAEC,CAAC;QAAIW,KAAKI,GAAG,IAAIuB,MAAMH,GAAG,CAACrC;KAAO;AAC3E;AAQA,6EAA6E;AAC7E,4EAA4E;AAC5E,yEAAyE;AACzE,SAASyC,UAAUC,KAAa,EAAEC,MAAc,EAAEC,MAA2B;IAC3E,MAAMC,OAAc,EAAE;IACtB,yEAAyE;IACzE,2EAA2E;IAC3E,MAAMC,SAAS;WAAIJ;KAAM,CAACjB,IAAI,CAAC,CAACf,GAAGC,IAAMH,KAAKG,KAAKH,KAAKE,MAAMA,EAAER,CAAC,GAAGS,EAAET,CAAC;IACvE,KAAK,MAAMC,KAAK2C,OAAQ;QACtB,MAAMC,OAAOH,OAAOzC;QACpB,MAAM6C,SAASH,KAAKI,IAAI,CAAC,CAAChD,IAAMA,EAAE8C,IAAI,KAAKA,QAAQlC,KAAK0B,GAAG,CAAC/B,KAAKP,EAAEgC,IAAI,IAAIzB,KAAKL,OAAOwC,OAAOhD,QAAQ,IAAIkB,KAAKI,GAAG,CAAChB,EAAEgC,IAAI,CAAC3B,CAAC,EAAEH,EAAEG,CAAC,IAAIO,KAAKC,GAAG,CAACb,EAAEgC,IAAI,CAAC3B,CAAC,EAAEH,EAAEG,CAAC,IAAIqC,OAAO/C,eAAe;QACpL,IAAIoD,QAAQ;YACVA,OAAON,KAAK,CAACQ,IAAI,CAAC/C;YAClB6C,OAAOf,IAAI,GAAGxB,MAAMuC,OAAOf,IAAI,EAAE9B;QACnC,OAAO;YACL0C,KAAKK,IAAI,CAAC;gBAAEjB,MAAM;oBAAE/B,GAAGC,EAAED,CAAC;oBAAEG,GAAGF,EAAEE,CAAC;oBAAEF,GAAGA,EAAEA,CAAC;oBAAEG,GAAGH,EAAEG,CAAC;gBAAC;gBAAGoC,OAAO;oBAACvC;iBAAE;gBAAE4C;YAAK;QACzE;IACF;IACA,OAAOF;AACT;AAEA,SAASM,SAASvB,KAAa,EAAEK,IAAU,EAAEmB,GAAW;IACtD,MAAMN,SAAS;WAAIlB;KAAM,CAACH,IAAI,CAAC,CAACf,GAAGC,IAAMD,EAAER,CAAC,GAAGS,EAAET,CAAC;IAClD,MAAM6B,OAAOe,OAAOT,GAAG,CAAC,CAACgB,IAAMA,EAAEtB,IAAI,EAAEuB,IAAI,CAAC;IAC5C,OAAO;QACLvB;QACA7B,GAAG+B,KAAK/B,CAAC;QACTqD,OAAOtB,KAAK9B,CAAC;QACbK,MAAMA,KAAKyB;QACX3B,GAAGqB,eAAeC;QAClBwB;QACAV,OAAOI;IACT;AACF;AAEA,6EAA6E;AAC7E,sIAAsI;AACtI,OAAO,SAASU,aAAad,KAAa,EAAEC,MAAc;QAOxCtB,SA0DSA,UAiDYA;IAjHrC,IAAI,CAACqB,MAAMnB,MAAM,EAAE,OAAO;QAAEkC,OAAO,EAAE;QAAEC,QAAQ,EAAE;QAAEC,OAAO;IAAM;IAEhE,MAAMC,UAAU;IAChB,MAAMC,aAAalB,OAAOjD,QAAQ,GAAGkE;IACrC,MAAME,cAAcnB,OAAOjD,QAAQ,GAAGkE;IAEtC,MAAMG,WAAU1C,UAAAA,OAAOqB,MAAML,GAAG,CAAC,CAAClC,IAAMA,EAAEG,CAAC,gBAA3Be,qBAAAA,UAAiC;IACjD,gEAAgE;IAChE,oFAAoF;IACpF,MAAM2C,WAAWD,UAAU;IAE3B,2EAA2E;IAC3E,gFAAgF;IAChF,MAAME,YAAYvB,MAAMwB,MAAM,CAAC,CAAC/D,IAAMA,EAAEG,CAAC,GAAG0D;IAC5C,MAAMG,YAAYF,UAAUC,MAAM,CAAC,CAAC/D,IAAMA,EAAED,CAAC,GAAGyC,OAAOjD,QAAQ,IAAIM,KAAKG,KAAKwC,OAAOjD,QAAQ,EAAE6B,MAAM;IACpG,MAAM6C,UAAUH,UAAUC,MAAM,CAAC,CAAC/D,IAAMI,KAAKJ,KAAK0D,YAAYtC,MAAM;IACpE,MAAM8C,WAAWJ,UAAUC,MAAM,CAAC,CAAC/D,IAAMI,KAAKJ,KAAK2D,aAAavC,MAAM;IACtE,MAAM+C,SAASF,WAAWtE,iBAAiBuE,YAAYvE,iBAAiBsE,WAAWH,UAAU1C,MAAM,GAAGxB,iBAAiBsE,YAAYJ,UAAU1C,MAAM,GAAGxB,iBAAiBoE,aAAaF,UAAU1C,MAAM,GAAG1B;IAEvM,4EAA4E;IAC5E,4FAA4F;IAC5F,sDAAsD;IACtD,MAAM+C,SAAS,CAACzC;QACd,IAAI,CAACmE,QAAQ,OAAO;QACpB,OAAO/D,KAAKJ,KAAKwC,OAAOjD,QAAQ,GAAG,IAAI;IACzC;IACA,MAAM6E,WAAW9B,UACfC,MAAMwB,MAAM,CAAC,CAAC/D,IAAMA,EAAEG,CAAC,IAAI0D,WAC3BrB,QACA,IAAM;IAER,MAAM6B,aAAa/B,UACjBC,MAAMwB,MAAM,CAAC,CAAC/D,IAAMA,EAAEG,CAAC,GAAG0D,WAC1BrB,QACAC;IAEF,MAAMC,OAAO;WAAI0B;WAAaC;KAAW;IAEzC,2EAA2E;IAC3E,sFAAsF;IACtF,IAAItC;IACJ,IAAIoC,QAAQ;QACV,MAAMG,WAAW5B,KAAKqB,MAAM,CAAC,CAACjE,IAAMA,EAAEgC,IAAI,CAAC9B,CAAC,IAAI,OAAOF,EAAEgC,IAAI,CAAC9B,CAAC,IAAI,QAAQF,EAAEgC,IAAI,CAAC3B,CAAC,IAAIyD,UAAU;QACjG,MAAMW,OAAOtD,MAAMqD,SAASP,MAAM,CAAC,CAACjE,IAAMA,EAAE8C,IAAI,KAAK,GAAGV,GAAG,CAAC,CAACpC,IAAMA,EAAEgC,IAAI;QACzE,MAAM0C,QAAQvD,MAAMqD,SAASP,MAAM,CAAC,CAACjE,IAAMA,EAAE8C,IAAI,KAAK,GAAGV,GAAG,CAAC,CAACpC,IAAMA,EAAEgC,IAAI;QAC1EC,YAAY;YAACwC;YAAMC;SAAM,CAACT,MAAM,CAAC,CAACjE,IAAkBA,MAAMuB;IAC5D,OAAO;QACL,MAAMiD,WAAW5B,KAAKqB,MAAM,CAAC,CAACjE,IAAMA,EAAEgC,IAAI,CAAC9B,CAAC,IAAI,OAAOF,EAAEgC,IAAI,CAAC3B,CAAC,IAAIyD,UAAU;QAC7E,MAAMa,SAASxD,MAAMqD,SAASpC,GAAG,CAAC,CAACpC,IAAMA,EAAEgC,IAAI;QAC/CC,YAAY0C,WAAWpD,YAAY;YAACoD;SAAO,GAAG,EAAE;IAClD;IAEA,MAAMnB,QAAmBZ,KAAKR,GAAG,CAAC,CAACwC,MAAQ1B,SAAS0B,IAAInC,KAAK,EAAEmC,IAAI5C,IAAI,EAAED,aAAa6C,IAAI5C,IAAI,EAAEC;IAChGuB,MAAMhC,IAAI,CAAC,CAACf,GAAGC,IAAMD,EAAE0C,GAAG,GAAGzC,EAAEyC,GAAG,IAAIzC,EAAEH,IAAI,GAAGE,EAAEF,IAAI;IAErD,sEAAsE;IACtE,uJAAuJ;IACvJ,MAAMsE,UAAoB,EAAE;IAC5B,IAAK,IAAIC,IAAI,GAAGA,IAAItB,MAAMlC,MAAM,EAAEwD,IAAK;QACrC,IAAItB,KAAK,CAACsB,EAAE,CAAC3B,GAAG,KAAKK,KAAK,CAACsB,IAAI,EAAE,CAAC3B,GAAG,EAAE;YACrC,MAAM4B,IAAIvB,KAAK,CAACsB,IAAI,EAAE,CAACvE,IAAI,GAAGiD,KAAK,CAACsB,EAAE,CAACvE,IAAI;YAC3C,IAAIwE,IAAI,QAAQF,QAAQ5B,IAAI,CAAC8B;QAC/B;IACF;IACA,MAAMC,oBAAmB5D,WAAAA,OAAOyD,sBAAPzD,sBAAAA,WAAmB;IAC5C,MAAM6D,YAAY,IAAIC;IACtBjD,UAAUkD,OAAO,CAAC,CAACnF,GAAG6B;QACpBoD,UAAUG,GAAG,CAACvD,GAAG7B,CAAC,CAAC,EAAE;IACvB;IACA,MAAMqF,OAAkB,EAAE;IAC1B,MAAMC,kBAA6B,EAAE;IACrC,qEAAqE;IACrE,kHAAkH;IAClH,MAAMC,YAAY/B,MAAMS,MAAM,CAAC,CAAC/B,IAAMA,EAAE7B,CAAC,IAAI0D;IAC7C,KAAK,MAAMyB,QAAQhC,MAAO;QACxB,MAAMiC,YAAYD,KAAK1D,IAAI,CAAC4D,KAAK,CAAC,KAAKzB,MAAM,CAAC0B,SAASrE,MAAM;QAC7D,MAAMsE,QAAQX,UAAUY,GAAG,CAACL,KAAKrC,GAAG;QACpC,MAAM2C,QAAQL,aAAa/C,OAAOvD,aAAa,IAAIyG,UAAUrE,aAAaX,KAAK0B,GAAG,CAACkD,KAAKvF,CAAC,GAAG2F,SAASlD,OAAOrD,WAAW;QACvH,MAAM0G,sBACJD,SACAN,KAAKnF,CAAC,IAAI0D,YACV,CAAC,cAAciC,IAAI,CAACR,KAAK1D,IAAI,KAC7B,uEAAuE;QACvE,yEAAyE;QACzE,CAACyD,UAAUU,IAAI,CAAC,CAACC,QAAUA,UAAUV,QAAQU,MAAM/C,GAAG,KAAKqC,KAAKrC,GAAG,IAAIvC,KAAK0B,GAAG,CAAC4D,MAAM3F,IAAI,GAAGiF,KAAKjF,IAAI,IAAIK,KAAKI,GAAG,CAACkF,MAAM7F,CAAC,EAAEmF,KAAKnF,CAAC,IAAI;QACxI,MAAM8F,kBAAkBJ,uBAAwBD,SAASN,KAAKlC,KAAK,GAAGZ,OAAOtD,aAAa;QAC1F,MAAMgH,oBACJD,mBACAd,KAAKY,IAAI,CAAC,CAACvF;YACT,IAAIA,EAAEyC,GAAG,KAAKqC,KAAKrC,GAAG,EAAE,OAAO;YAC/B,MAAMkD,MAAM3F,EAAEH,IAAI,GAAGiF,KAAKjF,IAAI;YAC9B,IAAI8F,OAAO,KAAKA,MAAM3D,OAAOpD,OAAO,GAAG0F,kBAAkB,OAAO;YAChE,MAAMsB,UAAU1F,KAAKC,GAAG,CAACH,EAAET,CAAC,GAAGS,EAAE4C,KAAK,EAAEkC,KAAKvF,CAAC,GAAGuF,KAAKlC,KAAK,IAAI1C,KAAKI,GAAG,CAACN,EAAET,CAAC,EAAEuF,KAAKvF,CAAC;YACnF,OAAOqG,WAAWd,KAAKlC,KAAK,GAAG;QACjC;QACF,IAAI6C,mBAAmB,CAACC,mBAAmB;YACzCd,gBAAgBrC,IAAI,CAACuC;QACvB,OAAO;YACLH,KAAKpC,IAAI,CAACuC;QACZ;IACF;IACA,yEAAyE;IACzE,iFAAiF;IACjF,MAAM/B,SAAS6B;IAEf,0EAA0E;IAC1E,MAAMiB,OAAiB,EAAE;IACzB,IAAK,IAAIzB,IAAI,GAAGA,IAAIO,KAAK/D,MAAM,EAAEwD,IAAK;QACpC,IAAIO,IAAI,CAACP,EAAE,CAAC3B,GAAG,KAAKkC,IAAI,CAACP,IAAI,EAAE,CAAC3B,GAAG,EAAE;YACnC,MAAM4B,IAAIM,IAAI,CAACP,IAAI,EAAE,CAACvE,IAAI,GAAG8E,IAAI,CAACP,EAAE,CAACvE,IAAI;YACzC,IAAIwE,IAAI,QAAQwB,KAAKtD,IAAI,CAAC8B;QAC5B;IACF;IACA,OAAO;QAAEvB,OAAO6B;QAAM5B;QAAQC,KAAK,GAAEtC,WAAAA,OAAOmF,mBAAPnF,sBAAAA,WAAgB;IAAM;AAC7D;AAEA,mDAAmD;AAEnD,uEAAuE;AACvE,OAAO,SAASoF,OAAOnG,CAAS;IAC9B,OAAOO,KAAK6F,KAAK,CAACpG,IAAI,OAAO;AAC/B;AAEA,4EAA4E;AAC5E,OAAO,SAASqG,cAAcC,KAAgB,EAAEjE,MAAc;IAC5D,MAAMkE,QAAQ,IAAI1B;IAClB,KAAK,MAAM9B,KAAKuD,MAAO;QACrB,KAAK,MAAMzE,KAAKkB,EAAEI,KAAK,CAAE;gBAEToD;YADd,MAAMlG,IAAI8F,OAAOtE,EAAE7B,CAAC;YACpBuG,MAAMxB,GAAG,CAAC1E,GAAG,EAACkG,aAAAA,MAAMf,GAAG,CAACnF,gBAAVkG,wBAAAA,aAAgB,KAAK1E,EAAEJ,IAAI,CAACR,MAAM;QAClD;IACF;IACA,4EAA4E;IAC5E,2DAA2D;IAC3D,IAAIuF,QAAQ;IACZ,IAAIC,YAAY,CAAC;IACjB,KAAK,MAAM,CAAChC,GAAGiC,EAAE,IAAIH,MAAO;QAC1B,IAAIG,IAAID,WAAW;YACjBA,YAAYC;YACZF,QAAQ/B;QACV;IACF;IAEA,MAAMkC,UAAU,IAAI9B;IACpB,IAAI+B,QAAQ,GAAG,uCAAuC;IACtD,IAAIC,MAAqB;IACzB,MAAMC,QAAQ;WAAIP,MAAMQ,IAAI;KAAG,CAACnD,MAAM,CAAC,CAACoD,IAAMA,IAAIR,QAAQnE,OAAOnD,YAAY,EAAEiC,IAAI,CAAC,CAACf,GAAGC,IAAMA,IAAID;IAClG,KAAK,MAAM4G,KAAKF,MAAO;QACrB,IAAID,QAAQ,QAAQG,IAAIH,MAAM,KAAK;YACjCD,QAAQrG,KAAKC,GAAG,CAAC,GAAGoG,QAAQ;YAC5BC,MAAMG;QACR;QACAL,QAAQ5B,GAAG,CAACiC,GAAGJ;IACjB;IACA,OAAO;QAAEJ;QAAOG;IAAQ;AAC1B"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type { AnalyzeOcrOptions, AnalyzeOptions, AnalyzeTextOptions } from './analyze.js';
|
|
2
|
+
export { analyze } from './analyze.js';
|
|
3
|
+
export type { AuditResult } from './audit.js';
|
|
4
|
+
export { auditWords } from './audit.js';
|
|
5
|
+
export type { PdfToMdErrorCode } from './errors.js';
|
|
6
|
+
export { PdfToMdError } from './errors.js';
|
|
7
|
+
export type { ExtractOcrOptions, ExtractResult, ExtractTextOptions, OcrProgressEvent, OcrWordInput, PageSelection, PdfToMarkdownOptions, } from './extract.js';
|
|
8
|
+
export { extractOcr, extractText, pdfToMarkdown } from './extract.js';
|
|
9
|
+
export type { PageWord, Tuning } from './geometry.js';
|
|
10
|
+
export { DEFAULT_TUNING } from './geometry.js';
|
|
11
|
+
export type { RenderPageResult } from './raster.js';
|
|
12
|
+
export { renderPageToPNG } from './raster.js';
|
|
13
|
+
export { toMarkdown, toText } from './render-analysis.js';
|
|
14
|
+
export { OUTPUT_VERSION } from './report.js';
|
|
15
|
+
export type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord, PageStatsRow, Report } from './types.js';
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Public library API. Deliberately small: every export is a stability
|
|
2
|
+
// promise; internals stay module-private behind these.
|
|
3
|
+
export { analyze } from './analyze.js';
|
|
4
|
+
export { auditWords } from './audit.js';
|
|
5
|
+
export { PdfToMdError } from './errors.js';
|
|
6
|
+
export { extractOcr, extractText, pdfToMarkdown } from './extract.js';
|
|
7
|
+
export { DEFAULT_TUNING } from './geometry.js';
|
|
8
|
+
export { renderPageToPNG } from './raster.js';
|
|
9
|
+
export { toMarkdown, toText } from './render-analysis.js';
|
|
10
|
+
export { OUTPUT_VERSION } from './report.js';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability\n// promise; internals stay module-private behind these.\n\nexport type { AnalyzeOcrOptions, AnalyzeOptions, AnalyzeTextOptions } from './analyze.ts';\nexport { analyze } from './analyze.ts';\nexport type { AuditResult } from './audit.ts';\nexport { auditWords } from './audit.ts';\nexport type { PdfToMdErrorCode } from './errors.ts';\nexport { PdfToMdError } from './errors.ts';\nexport type {\n ExtractOcrOptions,\n ExtractResult,\n ExtractTextOptions,\n OcrProgressEvent,\n OcrWordInput,\n PageSelection,\n PdfToMarkdownOptions,\n} from './extract.ts';\nexport { extractOcr, extractText, pdfToMarkdown } from './extract.ts';\nexport type { PageWord, Tuning } from './geometry.ts';\nexport { DEFAULT_TUNING } from './geometry.ts';\nexport type { RenderPageResult } from './raster.ts';\nexport { renderPageToPNG } from './raster.ts';\nexport { toMarkdown, toText } from './render-analysis.ts';\nexport { OUTPUT_VERSION } from './report.ts';\nexport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord, PageStatsRow, Report } from './types.ts';\n"],"names":["analyze","auditWords","PdfToMdError","extractOcr","extractText","pdfToMarkdown","DEFAULT_TUNING","renderPageToPNG","toMarkdown","toText","OUTPUT_VERSION"],"mappings":"AAAA,sEAAsE;AACtE,uDAAuD;AAGvD,SAASA,OAAO,QAAQ,eAAe;AAEvC,SAASC,UAAU,QAAQ,aAAa;AAExC,SAASC,YAAY,QAAQ,cAAc;AAU3C,SAASC,UAAU,EAAEC,WAAW,EAAEC,aAAa,QAAQ,eAAe;AAEtE,SAASC,cAAc,QAAQ,gBAAgB;AAE/C,SAASC,eAAe,QAAQ,cAAc;AAC9C,SAASC,UAAU,EAAEC,MAAM,QAAQ,uBAAuB;AAC1D,SAASC,cAAc,QAAQ,cAAc"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// Pass 2: glyphs -> visual lines — de-braids two-column pages (spanning
|
|
2
|
+
// lines like headings sort with the left column), joins words by measured horizontal gap, and folds typographic codepoints to ASCII so search doesn't miss half the hits.
|
|
3
|
+
const FOLD = {
|
|
4
|
+
ʼ: "'",
|
|
5
|
+
'’': "'",
|
|
6
|
+
'‘': "'",
|
|
7
|
+
'“': '"',
|
|
8
|
+
'”': '"',
|
|
9
|
+
'–': '-'
|
|
10
|
+
};
|
|
11
|
+
// Exported for the OCR engine seam (src/engines/tesseract.ts): both paths
|
|
12
|
+
// must fold identically or the same printed apostrophe surfaces as two spellings depending on which path read the page.
|
|
13
|
+
export const foldTypographic = (s)=>s.replace(/[ʼ‘’“”–]/g, (c)=>FOLD[c]);
|
|
14
|
+
const normalize = foldTypographic;
|
|
15
|
+
export function toLines({ glyphs, width }) {
|
|
16
|
+
if (!glyphs.length) return [];
|
|
17
|
+
const mid = width / 2;
|
|
18
|
+
// A real two-column page has a gutter: clusters each side, almost nothing crossing mid.
|
|
19
|
+
// Label/value layouts also cluster but their values cross mid — de-braiding would reorder them.
|
|
20
|
+
const twoCol = glyphs.filter((g)=>g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g)=>g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g)=>g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;
|
|
21
|
+
// Group on y with a tolerance that scales with font size.
|
|
22
|
+
const lines = [];
|
|
23
|
+
for (const g of [
|
|
24
|
+
...glyphs
|
|
25
|
+
].sort((a, b)=>a.y - b.y || a.x - b.x)){
|
|
26
|
+
// NOTE: start-x assignment can misfile a gutter-straddling glyph; an
|
|
27
|
+
// overlap-based alternative was tried and reshuffled stray numerals with
|
|
28
|
+
// no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).
|
|
29
|
+
const col = twoCol && g.x > mid ? 1 : 0;
|
|
30
|
+
const last = lines.at(-1);
|
|
31
|
+
if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {
|
|
32
|
+
last.parts.push(g);
|
|
33
|
+
last.y = (last.y + g.y) / 2;
|
|
34
|
+
} else lines.push({
|
|
35
|
+
col,
|
|
36
|
+
y: g.y,
|
|
37
|
+
parts: [
|
|
38
|
+
g
|
|
39
|
+
]
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
lines.sort((a, b)=>a.col - b.col || a.y - b.y); // column, then down the page
|
|
43
|
+
return lines.map((l)=>{
|
|
44
|
+
const parts = l.parts.sort((a, b)=>a.x - b.x);
|
|
45
|
+
let t = '';
|
|
46
|
+
let prev = null;
|
|
47
|
+
for (const cur of parts){
|
|
48
|
+
// insert a space when the measured gap is wider than intra-word kerning
|
|
49
|
+
if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\s$/.test(t) && !/^\s/.test(cur.s)) t += ' ';
|
|
50
|
+
t += cur.s;
|
|
51
|
+
prev = cur;
|
|
52
|
+
}
|
|
53
|
+
// Line height = the char-weighted dominant height, not the max: a small
|
|
54
|
+
// decorative large-face numeral must not promote a body line to a heading.
|
|
55
|
+
const byH = new Map();
|
|
56
|
+
for (const p of parts)byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);
|
|
57
|
+
return {
|
|
58
|
+
text: normalize(t).replace(/\s+/g, ' ').trim(),
|
|
59
|
+
x: parts[0].x,
|
|
60
|
+
y: l.y,
|
|
61
|
+
h: [
|
|
62
|
+
...byH.entries()
|
|
63
|
+
].sort((a, b)=>b[1] - a[1])[0][0],
|
|
64
|
+
col: l.col
|
|
65
|
+
};
|
|
66
|
+
}).filter((l)=>l.text);
|
|
67
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/lines.ts"],"sourcesContent":["// Pass 2: glyphs -> visual lines — de-braids two-column pages (spanning\n// lines like headings sort with the left column), joins words by measured horizontal gap, and folds typographic codepoints to ASCII so search doesn't miss half the hits.\nimport type { CollectedPage, Glyph, Line } from './types.ts';\n\nconst FOLD: Record<string, string> = {\n ʼ: \"'\",\n '’': \"'\",\n '‘': \"'\",\n '“': '\"',\n '”': '\"',\n '–': '-',\n};\n// Exported for the OCR engine seam (src/engines/tesseract.ts): both paths\n// must fold identically or the same printed apostrophe surfaces as two spellings depending on which path read the page.\nexport const foldTypographic = (s: string): string => s.replace(/[ʼ‘’“”–]/g, (c) => FOLD[c]);\nconst normalize = foldTypographic;\n\n// Intermediate row while grouping glyphs by visual line, before the final\n// text/height reduction below.\ninterface LineDraft {\n col: number;\n y: number;\n parts: Glyph[];\n}\n\nexport function toLines({ glyphs, width }: Pick<CollectedPage, 'glyphs' | 'width'>): Line[] {\n if (!glyphs.length) return [];\n\n const mid = width / 2;\n // A real two-column page has a gutter: clusters each side, almost nothing crossing mid.\n // Label/value layouts also cluster but their values cross mid — de-braiding would reorder them.\n const twoCol = glyphs.filter((g) => g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;\n\n // Group on y with a tolerance that scales with font size.\n const lines: LineDraft[] = [];\n for (const g of [...glyphs].sort((a, b) => a.y - b.y || a.x - b.x)) {\n // NOTE: start-x assignment can misfile a gutter-straddling glyph; an\n // overlap-based alternative was tried and reshuffled stray numerals with\n // no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).\n const col = twoCol && g.x > mid ? 1 : 0;\n const last = lines.at(-1);\n if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {\n last.parts.push(g);\n last.y = (last.y + g.y) / 2;\n } else lines.push({ col, y: g.y, parts: [g] });\n }\n lines.sort((a, b) => a.col - b.col || a.y - b.y); // column, then down the page\n\n return lines\n .map((l): Line => {\n const parts = l.parts.sort((a, b) => a.x - b.x);\n let t = '';\n let prev: Glyph | null = null;\n for (const cur of parts) {\n // insert a space when the measured gap is wider than intra-word kerning\n if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\\s$/.test(t) && !/^\\s/.test(cur.s)) t += ' ';\n t += cur.s;\n prev = cur;\n }\n // Line height = the char-weighted dominant height, not the max: a small\n // decorative large-face numeral must not promote a body line to a heading.\n const byH = new Map<number, number>();\n for (const p of parts) byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);\n return {\n text: normalize(t).replace(/\\s+/g, ' ').trim(),\n x: parts[0].x,\n y: l.y,\n h: [...byH.entries()].sort((a, b) => b[1] - a[1])[0][0],\n col: l.col,\n };\n })\n .filter((l) => l.text);\n}\n"],"names":["FOLD","ʼ","foldTypographic","s","replace","c","normalize","toLines","glyphs","width","length","mid","twoCol","filter","g","x","w","lines","sort","a","b","y","col","last","at","Math","abs","max","h","parts","push","map","l","t","prev","cur","test","byH","Map","p","set","get","text","trim","entries"],"mappings":"AAAA,wEAAwE;AACxE,0KAA0K;AAG1K,MAAMA,OAA+B;IACnCC,GAAG;IACH,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;AACP;AACA,0EAA0E;AAC1E,wHAAwH;AACxH,OAAO,MAAMC,kBAAkB,CAACC,IAAsBA,EAAEC,OAAO,CAAC,aAAa,CAACC,IAAML,IAAI,CAACK,EAAE,EAAE;AAC7F,MAAMC,YAAYJ;AAUlB,OAAO,SAASK,QAAQ,EAAEC,MAAM,EAAEC,KAAK,EAA2C;IAChF,IAAI,CAACD,OAAOE,MAAM,EAAE,OAAO,EAAE;IAE7B,MAAMC,MAAMF,QAAQ;IACpB,wFAAwF;IACxF,gGAAgG;IAChG,MAAMG,SAASJ,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,OAAOG,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG;IAEnO,0DAA0D;IAC1D,MAAMO,QAAqB,EAAE;IAC7B,KAAK,MAAMH,KAAK;WAAIN;KAAO,CAACU,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEE,CAAC,GAAGD,EAAEC,CAAC,IAAIF,EAAEJ,CAAC,GAAGK,EAAEL,CAAC,EAAG;QAClE,qEAAqE;QACrE,yEAAyE;QACzE,0IAA0I;QAC1I,MAAMO,MAAMV,UAAUE,EAAEC,CAAC,GAAGJ,MAAM,IAAI;QACtC,MAAMY,OAAON,MAAMO,EAAE,CAAC,CAAC;QACvB,IAAID,QAAQA,KAAKD,GAAG,KAAKA,OAAOG,KAAKC,GAAG,CAACH,KAAKF,CAAC,GAAGP,EAAEO,CAAC,IAAII,KAAKE,GAAG,CAAC,GAAGb,EAAEc,CAAC,GAAG,MAAM;YAC/EL,KAAKM,KAAK,CAACC,IAAI,CAAChB;YAChBS,KAAKF,CAAC,GAAG,AAACE,CAAAA,KAAKF,CAAC,GAAGP,EAAEO,CAAC,AAADA,IAAK;QAC5B,OAAOJ,MAAMa,IAAI,CAAC;YAAER;YAAKD,GAAGP,EAAEO,CAAC;YAAEQ,OAAO;gBAACf;aAAE;QAAC;IAC9C;IACAG,MAAMC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEG,GAAG,GAAGF,EAAEE,GAAG,IAAIH,EAAEE,CAAC,GAAGD,EAAEC,CAAC,GAAG,6BAA6B;IAE/E,OAAOJ,MACJc,GAAG,CAAC,CAACC;QACJ,MAAMH,QAAQG,EAAEH,KAAK,CAACX,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEJ,CAAC,GAAGK,EAAEL,CAAC;QAC9C,IAAIkB,IAAI;QACR,IAAIC,OAAqB;QACzB,KAAK,MAAMC,OAAON,MAAO;YACvB,wEAAwE;YACxE,IAAIK,QAAQC,IAAIpB,CAAC,GAAImB,CAAAA,KAAKnB,CAAC,GAAGmB,KAAKlB,CAAC,AAADA,IAAKkB,KAAKN,CAAC,GAAG,QAAQ,CAAC,MAAMQ,IAAI,CAACH,MAAM,CAAC,MAAMG,IAAI,CAACD,IAAIhC,CAAC,GAAG8B,KAAK;YACpGA,KAAKE,IAAIhC,CAAC;YACV+B,OAAOC;QACT;QACA,wEAAwE;QACxE,2EAA2E;QAC3E,MAAME,MAAM,IAAIC;QAChB,KAAK,MAAMC,KAAKV,MAAOQ,IAAIG,GAAG,CAACD,EAAEX,CAAC,EAAE,AAACS,CAAAA,IAAII,GAAG,CAACF,EAAEX,CAAC,KAAK,CAAA,IAAKW,EAAEpC,CAAC,CAACO,MAAM;QACpE,OAAO;YACLgC,MAAMpC,UAAU2B,GAAG7B,OAAO,CAAC,QAAQ,KAAKuC,IAAI;YAC5C5B,GAAGc,KAAK,CAAC,EAAE,CAACd,CAAC;YACbM,GAAGW,EAAEX,CAAC;YACNO,GAAG;mBAAIS,IAAIO,OAAO;aAAG,CAAC1B,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;YACvDG,KAAKU,EAAEV,GAAG;QACZ;IACF,GACCT,MAAM,CAAC,CAACmB,IAAMA,EAAEU,IAAI;AACzB"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// Pass 3: lines -> markdown. Structure is measured, not guessed — heading
|
|
2
|
+
// levels come from the font-height histogram, hyphen-closes only against a dictionary (see the inline comments below for each rule).
|
|
3
|
+
import { getDict } from './dict.js';
|
|
4
|
+
import { pageMarkerLine } from './page-marker.js';
|
|
5
|
+
const GLYPH = /^[•◦▪▫●○·‣⁃]$/;
|
|
6
|
+
const BULLETS = /^(?:[•◦▪▫●○·‣⁃]\s*)+/;
|
|
7
|
+
export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
8
|
+
var _ref;
|
|
9
|
+
var _sort_;
|
|
10
|
+
// body height = char-weighted mode; headings are the rarer, larger sizes.
|
|
11
|
+
// The 1.35 multiplier keeps emphasized body text out of the outline.
|
|
12
|
+
const bodyH = (_ref = (_sort_ = [
|
|
13
|
+
...heightChars.entries()
|
|
14
|
+
].sort((a, b)=>b[1] - a[1])[0]) === null || _sort_ === void 0 ? void 0 : _sort_[0]) !== null && _ref !== void 0 ? _ref : 0;
|
|
15
|
+
const headingSizes = [
|
|
16
|
+
...heightChars.keys()
|
|
17
|
+
].filter((h)=>h > bodyH * 1.35).sort((a, b)=>b - a);
|
|
18
|
+
// with page markers on, content headings start below the page ### level;
|
|
19
|
+
// sizes beyond the ###### budget share level 6 rather than being dropped
|
|
20
|
+
const offset = opts.pageMarkers ? 3 : 0;
|
|
21
|
+
const levelOf = new Map(headingSizes.slice(0, 6).map((h, i)=>[
|
|
22
|
+
h,
|
|
23
|
+
Math.min(6, i + 1 + offset)
|
|
24
|
+
]));
|
|
25
|
+
const out = [];
|
|
26
|
+
const stats = {
|
|
27
|
+
headings: 0,
|
|
28
|
+
listItems: 0,
|
|
29
|
+
paragraphs: 0,
|
|
30
|
+
joinedPairs: 0,
|
|
31
|
+
reflowed: 0,
|
|
32
|
+
callouts: 0,
|
|
33
|
+
closedHyphens: 0
|
|
34
|
+
};
|
|
35
|
+
let para = [];
|
|
36
|
+
// a label line ending in ':' whose next line is a bare number belongs on one line
|
|
37
|
+
function attachValue(label, lines, get, set) {
|
|
38
|
+
const j = get() + 1;
|
|
39
|
+
if (/[::]\s*$/.test(label) && j < lines.length && /^[0-9][\d.,]*%?$/.test(lines[j].text)) {
|
|
40
|
+
set(j);
|
|
41
|
+
stats.joinedPairs++;
|
|
42
|
+
return `${label} ${lines[j].text}`;
|
|
43
|
+
}
|
|
44
|
+
return label;
|
|
45
|
+
}
|
|
46
|
+
const flush = ()=>{
|
|
47
|
+
if (!para.length) return;
|
|
48
|
+
out.push(para.join(' ').replace(/\s+([,.;:%])/g, '$1'));
|
|
49
|
+
out.push('');
|
|
50
|
+
stats.paragraphs++;
|
|
51
|
+
para = [];
|
|
52
|
+
};
|
|
53
|
+
// append a wrapped line to the open paragraph, resolving a line-final hyphen
|
|
54
|
+
const appendToPara = (text)=>{
|
|
55
|
+
const prev = para.at(-1);
|
|
56
|
+
if (prev && /\w-$/.test(prev)) {
|
|
57
|
+
var _text_split_;
|
|
58
|
+
// a bare " -" is punctuation, not a wrapped word
|
|
59
|
+
const head = prev.replace(/-$/, '').split(/\s/).pop();
|
|
60
|
+
const tail = ((_text_split_ = text.split(/\s/)[0]) !== null && _text_split_ !== void 0 ? _text_split_ : '').replace(/[^A-Za-z-]/g, '').split('-')[0];
|
|
61
|
+
const close = getDict().has((head + tail).toLowerCase());
|
|
62
|
+
if (close) stats.closedHyphens++;
|
|
63
|
+
para[para.length - 1] = prev.replace(/-$/, close ? '' : '-') + text;
|
|
64
|
+
} else para.push(text);
|
|
65
|
+
};
|
|
66
|
+
for (const { page, lines } of pageLines){
|
|
67
|
+
if (opts.pageMarkers) {
|
|
68
|
+
flush();
|
|
69
|
+
out.push(pageMarkerLine(page));
|
|
70
|
+
out.push('');
|
|
71
|
+
}
|
|
72
|
+
if (!lines.length) continue;
|
|
73
|
+
const indentBase = Math.min(...lines.map((l)=>l.x));
|
|
74
|
+
for(let i = 0; i < lines.length; i++){
|
|
75
|
+
let { text, h, x } = lines[i];
|
|
76
|
+
const prevLine = i > 0 ? lines[i - 1] : null;
|
|
77
|
+
// a column change or a vertical jump wider than a blank line ends the paragraph
|
|
78
|
+
if (para.length && prevLine && (lines[i].col !== prevLine.col || lines[i].y - prevLine.y > h * 2.2)) flush();
|
|
79
|
+
// A large-font line opens a run: gather the consecutive same-size lines
|
|
80
|
+
// below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.
|
|
81
|
+
if (levelOf.has(h) && !BULLETS.test(text)) {
|
|
82
|
+
const run = [
|
|
83
|
+
text
|
|
84
|
+
];
|
|
85
|
+
while(i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text))run.push(lines[++i].text);
|
|
86
|
+
// join wrapped title lines, resolving line-final hyphens like body text
|
|
87
|
+
const merged = run.reduce((acc, part)=>{
|
|
88
|
+
var _part_split_;
|
|
89
|
+
return /\w-$/.test(acc) ? acc.replace(/-$/, getDict().has((acc.replace(/-$/, '').split(/\s/).pop() + ((_part_split_ = part.split(/\s/)[0]) !== null && _part_split_ !== void 0 ? _part_split_ : '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`;
|
|
90
|
+
}).replace(/\s+([,.;:%])/g, '$1');
|
|
91
|
+
flush();
|
|
92
|
+
if (merged.length > 150) {
|
|
93
|
+
out.push(`> ${merged}`);
|
|
94
|
+
out.push('');
|
|
95
|
+
stats.callouts++;
|
|
96
|
+
} else {
|
|
97
|
+
out.push(`${'#'.repeat(levelOf.get(h))} ${merged}`);
|
|
98
|
+
out.push('');
|
|
99
|
+
stats.headings++;
|
|
100
|
+
}
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
// Lines can carry several stacked glyphs ("• ▪") from nested source lists.
|
|
104
|
+
const stripped = text.replace(BULLETS, '').trim();
|
|
105
|
+
const wasBullet = stripped !== text;
|
|
106
|
+
if (wasBullet) text = stripped;
|
|
107
|
+
// bullet with nothing left on the line: attach the following line's text
|
|
108
|
+
if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {
|
|
109
|
+
flush();
|
|
110
|
+
const depth = Math.max(0, Math.round((x - indentBase) / 18));
|
|
111
|
+
let label = lines[++i].text.replace(BULLETS, '').trim();
|
|
112
|
+
while(!label && i + 1 < lines.length)// stacked glyphs
|
|
113
|
+
label = lines[++i].text.replace(BULLETS, '').trim();
|
|
114
|
+
label = attachValue(label, lines, ()=>i, (n)=>{
|
|
115
|
+
i = n;
|
|
116
|
+
});
|
|
117
|
+
out.push(`${' '.repeat(depth)}- ${label}`);
|
|
118
|
+
stats.listItems++;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
// glyph(s) followed by text on the same line
|
|
122
|
+
if (wasBullet) {
|
|
123
|
+
flush();
|
|
124
|
+
const depth = Math.max(0, Math.round((x - indentBase) / 18));
|
|
125
|
+
out.push(`${' '.repeat(depth)}- ${attachValue(text, lines, ()=>i, (n)=>{
|
|
126
|
+
i = n;
|
|
127
|
+
})}`);
|
|
128
|
+
stats.listItems++;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
// `label:` on its own line followed by a bare value -> keep together
|
|
132
|
+
if (/[::]\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\d.,]*%?$/.test(lines[i + 1].text)) {
|
|
133
|
+
flush();
|
|
134
|
+
out.push(`- ${text} ${lines[++i].text}`);
|
|
135
|
+
stats.joinedPairs++;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
// otherwise body text: re-flow wrapped lines into a paragraph
|
|
139
|
+
appendToPara(text);
|
|
140
|
+
if (/[.!?:]$/.test(text)) flush();
|
|
141
|
+
else stats.reflowed++;
|
|
142
|
+
}
|
|
143
|
+
flush();
|
|
144
|
+
}
|
|
145
|
+
flush();
|
|
146
|
+
const md = `${out.join('\n').replace(/\n{3,}/g, '\n\n').trim()}\n`;
|
|
147
|
+
return {
|
|
148
|
+
md,
|
|
149
|
+
stats,
|
|
150
|
+
bodyH,
|
|
151
|
+
headingSizes
|
|
152
|
+
};
|
|
153
|
+
}
|