@effortlessmotion/pdf-to-md 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -0
- package/bin/cli.js +9 -0
- package/dist/cjs/analyze.d.cts +35 -0
- package/dist/cjs/analyze.d.ts +35 -0
- package/dist/cjs/analyze.js +620 -0
- package/dist/cjs/analyze.js.map +1 -0
- package/dist/cjs/audit.d.cts +38 -0
- package/dist/cjs/audit.d.ts +38 -0
- package/dist/cjs/audit.js +252 -0
- package/dist/cjs/audit.js.map +1 -0
- package/dist/cjs/cli/audit.d.cts +3 -0
- package/dist/cjs/cli/audit.d.ts +3 -0
- package/dist/cjs/cli/audit.js +269 -0
- package/dist/cjs/cli/audit.js.map +1 -0
- package/dist/cjs/cli/extract.d.cts +3 -0
- package/dist/cjs/cli/extract.d.ts +3 -0
- package/dist/cjs/cli/extract.js +704 -0
- package/dist/cjs/cli/extract.js.map +1 -0
- package/dist/cjs/cli/index.d.cts +5 -0
- package/dist/cjs/cli/index.d.ts +5 -0
- package/dist/cjs/cli/index.js +58 -0
- package/dist/cjs/cli/index.js.map +1 -0
- package/dist/cjs/cli/render.d.cts +3 -0
- package/dist/cjs/cli/render.d.ts +3 -0
- package/dist/cjs/cli/render.js +361 -0
- package/dist/cjs/cli/render.js.map +1 -0
- package/dist/cjs/cli/types.d.cts +10 -0
- package/dist/cjs/cli/types.d.ts +10 -0
- package/dist/cjs/cli/types.js +7 -0
- package/dist/cjs/cli/types.js.map +1 -0
- package/dist/cjs/cli.d.cts +2 -0
- package/dist/cjs/cli.d.ts +2 -0
- package/dist/cjs/cli.js +345 -0
- package/dist/cjs/cli.js.map +1 -0
- package/dist/cjs/collect.d.cts +6 -0
- package/dist/cjs/collect.d.ts +6 -0
- package/dist/cjs/collect.js +370 -0
- package/dist/cjs/collect.js.map +1 -0
- package/dist/cjs/dict.d.cts +8 -0
- package/dist/cjs/dict.d.ts +8 -0
- package/dist/cjs/dict.js +55 -0
- package/dist/cjs/dict.js.map +1 -0
- package/dist/cjs/emit.d.cts +27 -0
- package/dist/cjs/emit.d.ts +27 -0
- package/dist/cjs/emit.js +187 -0
- package/dist/cjs/emit.js.map +1 -0
- package/dist/cjs/engines/tesseract.d.cts +30 -0
- package/dist/cjs/engines/tesseract.d.ts +30 -0
- package/dist/cjs/engines/tesseract.js +650 -0
- package/dist/cjs/engines/tesseract.js.map +1 -0
- package/dist/cjs/errors.d.cts +21 -0
- package/dist/cjs/errors.d.ts +21 -0
- package/dist/cjs/errors.js +124 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/extract.d.cts +88 -0
- package/dist/cjs/extract.d.ts +88 -0
- package/dist/cjs/extract.js +436 -0
- package/dist/cjs/extract.js.map +1 -0
- package/dist/cjs/geometry.d.cts +63 -0
- package/dist/cjs/geometry.d.ts +63 -0
- package/dist/cjs/geometry.js +521 -0
- package/dist/cjs/geometry.js.map +1 -0
- package/dist/cjs/index.d.cts +15 -0
- package/dist/cjs/index.d.ts +15 -0
- package/dist/cjs/index.js +56 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/lines.d.cts +3 -0
- package/dist/cjs/lines.d.ts +3 -0
- package/dist/cjs/lines.js +181 -0
- package/dist/cjs/lines.js.map +1 -0
- package/dist/cjs/markdown.d.cts +2 -0
- package/dist/cjs/markdown.d.ts +2 -0
- package/dist/cjs/markdown.js +222 -0
- package/dist/cjs/markdown.js.map +1 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/page-marker.d.cts +16 -0
- package/dist/cjs/page-marker.d.ts +16 -0
- package/dist/cjs/page-marker.js +85 -0
- package/dist/cjs/page-marker.js.map +1 -0
- package/dist/cjs/pdf-open.d.cts +12 -0
- package/dist/cjs/pdf-open.d.ts +12 -0
- package/dist/cjs/pdf-open.js +257 -0
- package/dist/cjs/pdf-open.js.map +1 -0
- package/dist/cjs/raster.d.cts +26 -0
- package/dist/cjs/raster.d.ts +26 -0
- package/dist/cjs/raster.js +262 -0
- package/dist/cjs/raster.js.map +1 -0
- package/dist/cjs/render-analysis.d.cts +25 -0
- package/dist/cjs/render-analysis.d.ts +25 -0
- package/dist/cjs/render-analysis.js +319 -0
- package/dist/cjs/render-analysis.js.map +1 -0
- package/dist/cjs/report.d.cts +12 -0
- package/dist/cjs/report.d.ts +12 -0
- package/dist/cjs/report.js +392 -0
- package/dist/cjs/report.js.map +1 -0
- package/dist/cjs/resolve.d.cts +1 -0
- package/dist/cjs/resolve.d.ts +1 -0
- package/dist/cjs/resolve.js +18 -0
- package/dist/cjs/resolve.js.map +1 -0
- package/dist/cjs/types.d.cts +171 -0
- package/dist/cjs/types.d.ts +171 -0
- package/dist/cjs/types.js +9 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/cjs/validate.d.cts +14 -0
- package/dist/cjs/validate.d.ts +14 -0
- package/dist/cjs/validate.js +96 -0
- package/dist/cjs/validate.js.map +1 -0
- package/dist/esm/analyze.d.ts +35 -0
- package/dist/esm/analyze.js +270 -0
- package/dist/esm/analyze.js.map +1 -0
- package/dist/esm/audit.d.ts +38 -0
- package/dist/esm/audit.js +113 -0
- package/dist/esm/audit.js.map +1 -0
- package/dist/esm/cli/audit.d.ts +3 -0
- package/dist/esm/cli/audit.js +20 -0
- package/dist/esm/cli/audit.js.map +1 -0
- package/dist/esm/cli/extract.d.ts +3 -0
- package/dist/esm/cli/extract.js +313 -0
- package/dist/esm/cli/extract.js.map +1 -0
- package/dist/esm/cli/index.d.ts +5 -0
- package/dist/esm/cli/index.js +7 -0
- package/dist/esm/cli/index.js.map +1 -0
- package/dist/esm/cli/render.d.ts +3 -0
- package/dist/esm/cli/render.js +77 -0
- package/dist/esm/cli/render.js.map +1 -0
- package/dist/esm/cli/types.d.ts +10 -0
- package/dist/esm/cli/types.js +3 -0
- package/dist/esm/cli/types.js.map +1 -0
- package/dist/esm/cli.d.ts +2 -0
- package/dist/esm/cli.js +90 -0
- package/dist/esm/cli.js.map +1 -0
- package/dist/esm/collect.d.ts +6 -0
- package/dist/esm/collect.js +80 -0
- package/dist/esm/collect.js.map +1 -0
- package/dist/esm/dict.d.ts +8 -0
- package/dist/esm/dict.js +28 -0
- package/dist/esm/dict.js.map +1 -0
- package/dist/esm/emit.d.ts +27 -0
- package/dist/esm/emit.js +131 -0
- package/dist/esm/emit.js.map +1 -0
- package/dist/esm/engines/tesseract.d.ts +30 -0
- package/dist/esm/engines/tesseract.js +190 -0
- package/dist/esm/engines/tesseract.js.map +1 -0
- package/dist/esm/errors.d.ts +21 -0
- package/dist/esm/errors.js +9 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/extract.d.ts +88 -0
- package/dist/esm/extract.js +147 -0
- package/dist/esm/extract.js.map +1 -0
- package/dist/esm/geometry.d.ts +63 -0
- package/dist/esm/geometry.js +279 -0
- package/dist/esm/geometry.js.map +1 -0
- package/dist/esm/index.d.ts +15 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/lines.d.ts +3 -0
- package/dist/esm/lines.js +67 -0
- package/dist/esm/lines.js.map +1 -0
- package/dist/esm/markdown.d.ts +2 -0
- package/dist/esm/markdown.js +153 -0
- package/dist/esm/markdown.js.map +1 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/page-marker.d.ts +16 -0
- package/dist/esm/page-marker.js +40 -0
- package/dist/esm/page-marker.js.map +1 -0
- package/dist/esm/pdf-open.d.ts +12 -0
- package/dist/esm/pdf-open.js +61 -0
- package/dist/esm/pdf-open.js.map +1 -0
- package/dist/esm/raster.d.ts +26 -0
- package/dist/esm/raster.js +60 -0
- package/dist/esm/raster.js.map +1 -0
- package/dist/esm/render-analysis.d.ts +25 -0
- package/dist/esm/render-analysis.js +115 -0
- package/dist/esm/render-analysis.js.map +1 -0
- package/dist/esm/report.d.ts +12 -0
- package/dist/esm/report.js +182 -0
- package/dist/esm/report.js.map +1 -0
- package/dist/esm/resolve.d.ts +1 -0
- package/dist/esm/resolve.js +7 -0
- package/dist/esm/resolve.js.map +1 -0
- package/dist/esm/types.d.ts +171 -0
- package/dist/esm/types.js +6 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/validate.d.ts +14 -0
- package/dist/esm/validate.js +56 -0
- package/dist/esm/validate.js.map +1 -0
- package/package.json +67 -0
- package/skills/pdf-to-md/SKILL.md +102 -0
package/dist/cjs/emit.js
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// emit.ts — OCR-path pass 3: lines -> markdown blocks.
|
|
2
|
+
"use strict";
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
function _export(target, all) {
|
|
7
|
+
for(var name in all)Object.defineProperty(target, name, {
|
|
8
|
+
enumerable: true,
|
|
9
|
+
get: Object.getOwnPropertyDescriptor(all, name).get
|
|
10
|
+
});
|
|
11
|
+
}
|
|
12
|
+
_export(exports, {
|
|
13
|
+
get UNSTRUCTURED_WORDLIKE_FRACTION () {
|
|
14
|
+
return UNSTRUCTURED_WORDLIKE_FRACTION;
|
|
15
|
+
},
|
|
16
|
+
get emitPage () {
|
|
17
|
+
return emitPage;
|
|
18
|
+
},
|
|
19
|
+
get renderEmitBlock () {
|
|
20
|
+
return renderEmitBlock;
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
var _dictts = require("./dict.js");
|
|
24
|
+
var _geometryts = require("./geometry.js");
|
|
25
|
+
function _array_like_to_array(arr, len) {
|
|
26
|
+
if (len == null || len > arr.length) len = arr.length;
|
|
27
|
+
for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
|
|
28
|
+
return arr2;
|
|
29
|
+
}
|
|
30
|
+
function _array_without_holes(arr) {
|
|
31
|
+
if (Array.isArray(arr)) return _array_like_to_array(arr);
|
|
32
|
+
}
|
|
33
|
+
function _iterable_to_array(iter) {
|
|
34
|
+
if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
|
|
35
|
+
return Array.from(iter);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function _non_iterable_spread() {
|
|
39
|
+
throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
|
|
40
|
+
}
|
|
41
|
+
function _to_consumable_array(arr) {
|
|
42
|
+
return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
|
|
43
|
+
}
|
|
44
|
+
function _unsupported_iterable_to_array(o, minLen) {
|
|
45
|
+
if (!o) return;
|
|
46
|
+
if (typeof o === "string") return _array_like_to_array(o, minLen);
|
|
47
|
+
var n = Object.prototype.toString.call(o).slice(8, -1);
|
|
48
|
+
if (n === "Object" && o.constructor) n = o.constructor.name;
|
|
49
|
+
if (n === "Map" || n === "Set") return Array.from(n);
|
|
50
|
+
if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
|
|
51
|
+
}
|
|
52
|
+
// Join a wrapped line onto the open paragraph, closing a line-final hyphen
|
|
53
|
+
// only when the dictionary confirms the joined form is a word.
|
|
54
|
+
function hyphenJoin(acc, next) {
|
|
55
|
+
if (acc.endsWith('-')) {
|
|
56
|
+
var withoutDash = acc.slice(0, -1);
|
|
57
|
+
var pen = withoutDash.at(-1);
|
|
58
|
+
if (pen && /[A-Za-z0-9]/.test(pen)) {
|
|
59
|
+
var _withoutDash_split_pop, _next_split_, _filtered_split_;
|
|
60
|
+
var head = (_withoutDash_split_pop = withoutDash.split(' ').pop()) !== null && _withoutDash_split_pop !== void 0 ? _withoutDash_split_pop : '';
|
|
61
|
+
var firstTok = (_next_split_ = next.split(' ')[0]) !== null && _next_split_ !== void 0 ? _next_split_ : '';
|
|
62
|
+
var filtered = _to_consumable_array(firstTok).filter(function(c) {
|
|
63
|
+
return /[A-Za-z-]/.test(c);
|
|
64
|
+
}).join('');
|
|
65
|
+
var tail = (_filtered_split_ = filtered.split('-')[0]) !== null && _filtered_split_ !== void 0 ? _filtered_split_ : '';
|
|
66
|
+
var close = (0, _dictts.getDict)().has((head + tail).toLowerCase());
|
|
67
|
+
return (close ? withoutDash : acc) + next;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return "".concat(acc, " ").concat(next);
|
|
71
|
+
}
|
|
72
|
+
// Blocks with too few word-like tokens to trust as prose (see
|
|
73
|
+
// UNSTRUCTURED_WORDLIKE_FRACTION below) are labeled `> [review: reason] text` instead of guessed at.
|
|
74
|
+
var REVIEW_REASON = {
|
|
75
|
+
unstructuredLabels: 'unstructured labels'
|
|
76
|
+
};
|
|
77
|
+
var UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;
|
|
78
|
+
function isUnstructured(text) {
|
|
79
|
+
var tokens = text.split(/\s+/).filter(Boolean);
|
|
80
|
+
if (tokens.length === 0) return false; // never mark an empty block
|
|
81
|
+
var wordLike = tokens.filter(function(t) {
|
|
82
|
+
return /[A-Za-z]{3}/.test(t);
|
|
83
|
+
}).length;
|
|
84
|
+
return wordLike / tokens.length < UNSTRUCTURED_WORDLIKE_FRACTION;
|
|
85
|
+
}
|
|
86
|
+
function reviewLine(reason, text) {
|
|
87
|
+
return "> [review: ".concat(reason, "] ").concat(text);
|
|
88
|
+
}
|
|
89
|
+
function renderEmitBlock(b) {
|
|
90
|
+
switch(b.kind){
|
|
91
|
+
case 'heading':
|
|
92
|
+
return "".concat('#'.repeat(b.level), " ").concat(b.text);
|
|
93
|
+
case 'quote':
|
|
94
|
+
return "> ".concat(b.text);
|
|
95
|
+
case 'review':
|
|
96
|
+
return reviewLine(b.reason, b.text);
|
|
97
|
+
default:
|
|
98
|
+
return b.text;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
function emitPage(p, levelOf, tuning) {
|
|
102
|
+
var blocks = [];
|
|
103
|
+
var para = [];
|
|
104
|
+
var flush = function flush() {
|
|
105
|
+
if (!para.length) return;
|
|
106
|
+
var texts = para.map(function(l) {
|
|
107
|
+
return l.text;
|
|
108
|
+
});
|
|
109
|
+
var merged = texts.slice(1).reduce(hyphenJoin, texts[0]);
|
|
110
|
+
var words = para.flatMap(function(l) {
|
|
111
|
+
return l.words;
|
|
112
|
+
});
|
|
113
|
+
// Same review-marker rule as the heading path below: no word may be
|
|
114
|
+
// dropped or reordered here, only how the block renders changes.
|
|
115
|
+
blocks.push(isUnstructured(merged) ? {
|
|
116
|
+
kind: 'review',
|
|
117
|
+
text: merged,
|
|
118
|
+
reason: REVIEW_REASON.unstructuredLabels,
|
|
119
|
+
words: words
|
|
120
|
+
} : {
|
|
121
|
+
kind: 'paragraph',
|
|
122
|
+
text: merged,
|
|
123
|
+
words: words
|
|
124
|
+
});
|
|
125
|
+
para = [];
|
|
126
|
+
};
|
|
127
|
+
var i = 0;
|
|
128
|
+
while(i < p.lines.length){
|
|
129
|
+
var line = p.lines[i];
|
|
130
|
+
var prev = i > 0 ? p.lines[i - 1] : null;
|
|
131
|
+
if (prev && (line.col !== prev.col || prev.midY - line.midY > p.pitch * tuning.paraGap)) {
|
|
132
|
+
// Join across a column break (instead of flushing) only when the column
|
|
133
|
+
// changed, the paragraph dangles (no terminal punct), and the new text starts lowercase; a miss stays visible via pageStats.danglingLong.
|
|
134
|
+
var danglingEnd = para.length > 0 && !/[.!?:;"'”’)\]]$/.test(para[para.length - 1].text);
|
|
135
|
+
var continuationStart = /^[a-z]/.test(line.text);
|
|
136
|
+
var joinAcrossColumns = line.col !== prev.col && danglingEnd && continuationStart && !levelOf.has((0, _geometryts.bucket)(line.h));
|
|
137
|
+
if (!joinAcrossColumns) flush();
|
|
138
|
+
}
|
|
139
|
+
if (levelOf.has((0, _geometryts.bucket)(line.h))) {
|
|
140
|
+
// Gather the run of consecutive heading-size lines: a wrapped title is
|
|
141
|
+
// one heading, a long run is a pull quote in a bigger face.
|
|
142
|
+
flush();
|
|
143
|
+
var run = [
|
|
144
|
+
line
|
|
145
|
+
];
|
|
146
|
+
var maxB = (0, _geometryts.bucket)(line.h);
|
|
147
|
+
while(i + 1 < p.lines.length && levelOf.has((0, _geometryts.bucket)(p.lines[i + 1].h)) && p.lines[i + 1].col === p.lines[i].col && p.lines[i].midY - p.lines[i + 1].midY < Math.max(p.lines[i].h, p.lines[i + 1].h) * 2.2){
|
|
148
|
+
i++;
|
|
149
|
+
run.push(p.lines[i]);
|
|
150
|
+
maxB = Math.max(maxB, (0, _geometryts.bucket)(p.lines[i].h));
|
|
151
|
+
}
|
|
152
|
+
var runTexts = run.map(function(l) {
|
|
153
|
+
return l.text;
|
|
154
|
+
});
|
|
155
|
+
var merged = runTexts.slice(1).reduce(hyphenJoin, runTexts[0]);
|
|
156
|
+
var words = run.flatMap(function(l) {
|
|
157
|
+
return l.words;
|
|
158
|
+
});
|
|
159
|
+
// Below UNSTRUCTURED_WORDLIKE_FRACTION = chart labels/stray numerals,
|
|
160
|
+
// not a heading; checked FIRST so a long numeral run isn't asserted as a pull quote.
|
|
161
|
+
var junk = isUnstructured(merged);
|
|
162
|
+
if (junk) blocks.push({
|
|
163
|
+
kind: 'review',
|
|
164
|
+
text: merged,
|
|
165
|
+
reason: REVIEW_REASON.unstructuredLabels,
|
|
166
|
+
words: words
|
|
167
|
+
});
|
|
168
|
+
else if (merged.length > 150) blocks.push({
|
|
169
|
+
kind: 'quote',
|
|
170
|
+
text: merged,
|
|
171
|
+
words: words
|
|
172
|
+
});
|
|
173
|
+
else blocks.push({
|
|
174
|
+
kind: 'heading',
|
|
175
|
+
text: merged,
|
|
176
|
+
level: levelOf.get(maxB),
|
|
177
|
+
words: words
|
|
178
|
+
});
|
|
179
|
+
} else {
|
|
180
|
+
para.push(line);
|
|
181
|
+
}
|
|
182
|
+
i++;
|
|
183
|
+
}
|
|
184
|
+
flush();
|
|
185
|
+
return blocks;
|
|
186
|
+
}
|
|
187
|
+
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/emit.ts"],"sourcesContent":["// emit.ts — OCR-path pass 3: lines -> markdown blocks.\nimport { getDict } from './dict.ts';\nimport type { OcrLine, PageOCR, Tuning, Word } from './geometry.ts';\nimport { bucket } from './geometry.ts';\n\n// Join a wrapped line onto the open paragraph, closing a line-final hyphen\n// only when the dictionary confirms the joined form is a word.\nfunction hyphenJoin(acc: string, next: string): string {\n if (acc.endsWith('-')) {\n const withoutDash = acc.slice(0, -1);\n const pen = withoutDash.at(-1);\n if (pen && /[A-Za-z0-9]/.test(pen)) {\n const head = withoutDash.split(' ').pop() ?? '';\n const firstTok = next.split(' ')[0] ?? '';\n const filtered = [...firstTok].filter((c) => /[A-Za-z-]/.test(c)).join('');\n const tail = filtered.split('-')[0] ?? '';\n const close = getDict().has((head + tail).toLowerCase());\n return (close ? withoutDash : acc) + next;\n }\n }\n return `${acc} ${next}`;\n}\n\n// Blocks with too few word-like tokens to trust as prose (see\n// UNSTRUCTURED_WORDLIKE_FRACTION below) are labeled `> [review: reason] text` instead of guessed at.\nconst REVIEW_REASON = {\n unstructuredLabels: 'unstructured labels',\n} as const;\n\n/** Share of word-like tokens (3+ letter run) below which a block counts as\n * unstructured. Measured (`review-blocks.test.ts`): junk 0.000-0.100 vs prose 0.743-0.857, cut 0.4. */\nexport const UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;\nfunction isUnstructured(text: string): boolean {\n const tokens = text.split(/\\s+/).filter(Boolean);\n if (tokens.length === 0) return false; // never mark an empty block\n const wordLike = tokens.filter((t) => /[A-Za-z]{3}/.test(t)).length;\n return wordLike / tokens.length < UNSTRUCTURED_WORDLIKE_FRACTION;\n}\n\nfunction reviewLine(reason: string, text: string): string {\n return `> [review: ${reason}] ${text}`;\n}\n\n// One classified block, carrying constituent words for analyze()'s\n// block->word linkage; discriminated on `kind` so `reason`/`level` are compile-time guaranteed present on their variants.\nexport type EmitBlock =\n | { kind: 'heading'; text: string; level: number; words: Word[] }\n | { kind: 'paragraph'; text: string; words: Word[] }\n | { kind: 'quote'; text: string; words: Word[] }\n | {\n kind: 'review';\n text: string;\n /** The parseable reason inside `> [review: ...]`, carried on the block\n * so more reasons can be added without changing the render format. */\n reason: string;\n words: Word[];\n };\n\n// Renders one EmitBlock to the exact markdown emitPage used to push\n// directly — shared so extract.ts and analyze.ts produce byte-identical text.\nexport function renderEmitBlock(b: EmitBlock): string {\n switch (b.kind) {\n case 'heading':\n return `${'#'.repeat(b.level)} ${b.text}`;\n case 'quote':\n return `> ${b.text}`;\n case 'review':\n return reviewLine(b.reason, b.text);\n default:\n return b.text;\n }\n}\n\nexport function emitPage(p: PageOCR, levelOf: Map<number, number>, tuning: Tuning): EmitBlock[] {\n const blocks: EmitBlock[] = [];\n let para: OcrLine[] = [];\n const flush = () => {\n if (!para.length) return;\n const texts = para.map((l) => l.text);\n const merged = texts.slice(1).reduce(hyphenJoin, texts[0]);\n const words = para.flatMap((l) => l.words);\n // Same review-marker rule as the heading path below: no word may be\n // dropped or reordered here, only how the block renders changes.\n blocks.push(isUnstructured(merged) ? { kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words } : { kind: 'paragraph', text: merged, words });\n para = [];\n };\n\n let i = 0;\n while (i < p.lines.length) {\n const line: OcrLine = p.lines[i];\n const prev = i > 0 ? p.lines[i - 1] : null;\n if (prev && (line.col !== prev.col || prev.midY - line.midY > p.pitch * tuning.paraGap)) {\n // Join across a column break (instead of flushing) only when the column\n // changed, the paragraph dangles (no terminal punct), and the new text starts lowercase; a miss stays visible via pageStats.danglingLong.\n const danglingEnd = para.length > 0 && !/[.!?:;\"'”’)\\]]$/.test(para[para.length - 1].text);\n const continuationStart = /^[a-z]/.test(line.text);\n const joinAcrossColumns = line.col !== prev.col && danglingEnd && continuationStart && !levelOf.has(bucket(line.h));\n if (!joinAcrossColumns) flush();\n }\n\n if (levelOf.has(bucket(line.h))) {\n // Gather the run of consecutive heading-size lines: a wrapped title is\n // one heading, a long run is a pull quote in a bigger face.\n flush();\n const run: OcrLine[] = [line];\n let maxB = bucket(line.h);\n while (i + 1 < p.lines.length && levelOf.has(bucket(p.lines[i + 1].h)) && p.lines[i + 1].col === p.lines[i].col && p.lines[i].midY - p.lines[i + 1].midY < Math.max(p.lines[i].h, p.lines[i + 1].h) * 2.2) {\n i++;\n run.push(p.lines[i]);\n maxB = Math.max(maxB, bucket(p.lines[i].h));\n }\n const runTexts = run.map((l) => l.text);\n const merged = runTexts.slice(1).reduce(hyphenJoin, runTexts[0]);\n const words = run.flatMap((l) => l.words);\n // Below UNSTRUCTURED_WORDLIKE_FRACTION = chart labels/stray numerals,\n // not a heading; checked FIRST so a long numeral run isn't asserted as a pull quote.\n const junk = isUnstructured(merged);\n if (junk) blocks.push({ kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words });\n else if (merged.length > 150) blocks.push({ kind: 'quote', text: merged, words });\n else blocks.push({ kind: 'heading', text: merged, level: levelOf.get(maxB) as number, words });\n } else {\n para.push(line);\n }\n i++;\n }\n flush();\n return blocks;\n}\n"],"names":["UNSTRUCTURED_WORDLIKE_FRACTION","emitPage","renderEmitBlock","hyphenJoin","acc","next","endsWith","withoutDash","slice","pen","at","test","filtered","head","split","pop","firstTok","filter","c","join","tail","close","getDict","has","toLowerCase","REVIEW_REASON","unstructuredLabels","isUnstructured","text","tokens","Boolean","length","wordLike","t","reviewLine","reason","b","kind","repeat","level","p","levelOf","tuning","blocks","para","flush","texts","map","l","merged","reduce","words","flatMap","push","i","lines","line","prev","col","midY","pitch","paraGap","danglingEnd","continuationStart","joinAcrossColumns","bucket","h","run","maxB","Math","max","runTexts","junk","get"],"mappings":"AAAA,uDAAuD;;;;;;;;;;;;QA+B1CA;eAAAA;;QA0CGC;eAAAA;;QAbAC;eAAAA;;;sBA3DQ;0BAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvB,2EAA2E;AAC3E,+DAA+D;AAC/D,SAASC,WAAWC,GAAW,EAAEC,IAAY;IAC3C,IAAID,IAAIE,QAAQ,CAAC,MAAM;QACrB,IAAMC,cAAcH,IAAII,KAAK,CAAC,GAAG,CAAC;QAClC,IAAMC,MAAMF,YAAYG,EAAE,CAAC,CAAC;QAC5B,IAAID,OAAO,cAAcE,IAAI,CAACF,MAAM;gBACrBF,wBACIF,cAEJO;YAHb,IAAMC,QAAON,yBAAAA,YAAYO,KAAK,CAAC,KAAKC,GAAG,gBAA1BR,oCAAAA,yBAAgC;YAC7C,IAAMS,YAAWX,eAAAA,KAAKS,KAAK,CAAC,IAAI,CAAC,EAAE,cAAlBT,0BAAAA,eAAsB;YACvC,IAAMO,WAAW,AAAC,qBAAGI,UAAUC,MAAM,CAAC,SAACC;uBAAM,YAAYP,IAAI,CAACO;eAAIC,IAAI,CAAC;YACvE,IAAMC,QAAOR,mBAAAA,SAASE,KAAK,CAAC,IAAI,CAAC,EAAE,cAAtBF,8BAAAA,mBAA0B;YACvC,IAAMS,QAAQC,IAAAA,eAAO,IAAGC,GAAG,CAAC,AAACV,CAAAA,OAAOO,IAAG,EAAGI,WAAW;YACrD,OAAO,AAACH,CAAAA,QAAQd,cAAcH,GAAE,IAAKC;QACvC;IACF;IACA,OAAO,AAAC,GAASA,OAAPD,KAAI,KAAQ,OAALC;AACnB;AAEA,8DAA8D;AAC9D,qGAAqG;AACrG,IAAMoB,gBAAgB;IACpBC,oBAAoB;AACtB;AAIO,IAAM1B,iCAAiC;AAC9C,SAAS2B,eAAeC,IAAY;IAClC,IAAMC,SAASD,KAAKd,KAAK,CAAC,OAAOG,MAAM,CAACa;IACxC,IAAID,OAAOE,MAAM,KAAK,GAAG,OAAO,OAAO,4BAA4B;IACnE,IAAMC,WAAWH,OAAOZ,MAAM,CAAC,SAACgB;eAAM,cAActB,IAAI,CAACsB;OAAIF,MAAM;IACnE,OAAOC,WAAWH,OAAOE,MAAM,GAAG/B;AACpC;AAEA,SAASkC,WAAWC,MAAc,EAAEP,IAAY;IAC9C,OAAO,AAAC,cAAwBA,OAAXO,QAAO,MAAS,OAALP;AAClC;AAmBO,SAAS1B,gBAAgBkC,CAAY;IAC1C,OAAQA,EAAEC,IAAI;QACZ,KAAK;YACH,OAAO,AAAC,GAAyBD,OAAvB,IAAIE,MAAM,CAACF,EAAEG,KAAK,GAAE,KAAU,OAAPH,EAAER,IAAI;QACzC,KAAK;YACH,OAAO,AAAC,KAAW,OAAPQ,EAAER,IAAI;QACpB,KAAK;YACH,OAAOM,WAAWE,EAAED,MAAM,EAAEC,EAAER,IAAI;QACpC;YACE,OAAOQ,EAAER,IAAI;IACjB;AACF;AAEO,SAAS3B,SAASuC,CAAU,EAAEC,OAA4B,EAAEC,MAAc;IAC/E,IAAMC,SAAsB,EAAE;IAC9B,IAAIC,OAAkB,EAAE;IACxB,IAAMC,QAAQ;QACZ,IAAI,CAACD,KAAKb,MAAM,EAAE;QAClB,IAAMe,QAAQF,KAAKG,GAAG,CAAC,SAACC;mBAAMA,EAAEpB,IAAI;;QACpC,IAAMqB,SAASH,MAAMtC,KAAK,CAAC,GAAG0C,MAAM,CAAC/C,YAAY2C,KAAK,CAAC,EAAE;QACzD,IAAMK,QAAQP,KAAKQ,OAAO,CAAC,SAACJ;mBAAMA,EAAEG,KAAK;;QACzC,oEAAoE;QACpE,iEAAiE;QACjER,OAAOU,IAAI,CAAC1B,eAAesB,UAAU;YAAEZ,MAAM;YAAUT,MAAMqB;YAAQd,QAAQV,cAAcC,kBAAkB;YAAEyB,OAAAA;QAAM,IAAI;YAAEd,MAAM;YAAaT,MAAMqB;YAAQE,OAAAA;QAAM;QAClKP,OAAO,EAAE;IACX;IAEA,IAAIU,IAAI;IACR,MAAOA,IAAId,EAAEe,KAAK,CAACxB,MAAM,CAAE;QACzB,IAAMyB,OAAgBhB,EAAEe,KAAK,CAACD,EAAE;QAChC,IAAMG,OAAOH,IAAI,IAAId,EAAEe,KAAK,CAACD,IAAI,EAAE,GAAG;QACtC,IAAIG,QAASD,CAAAA,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAID,KAAKE,IAAI,GAAGH,KAAKG,IAAI,GAAGnB,EAAEoB,KAAK,GAAGlB,OAAOmB,OAAO,AAAD,GAAI;YACvF,wEAAwE;YACxE,0IAA0I;YAC1I,IAAMC,cAAclB,KAAKb,MAAM,GAAG,KAAK,CAAC,kBAAkBpB,IAAI,CAACiC,IAAI,CAACA,KAAKb,MAAM,GAAG,EAAE,CAACH,IAAI;YACzF,IAAMmC,oBAAoB,SAASpD,IAAI,CAAC6C,KAAK5B,IAAI;YACjD,IAAMoC,oBAAoBR,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAII,eAAeC,qBAAqB,CAACtB,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACT,KAAKU,CAAC;YACjH,IAAI,CAACF,mBAAmBnB;QAC1B;QAEA,IAAIJ,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACT,KAAKU,CAAC,IAAI;YAC/B,uEAAuE;YACvE,4DAA4D;YAC5DrB;YACA,IAAMsB,MAAiB;gBAACX;aAAK;YAC7B,IAAIY,OAAOH,IAAAA,kBAAM,EAACT,KAAKU,CAAC;YACxB,MAAOZ,IAAI,IAAId,EAAEe,KAAK,CAACxB,MAAM,IAAIU,QAAQlB,GAAG,CAAC0C,IAAAA,kBAAM,EAACzB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACY,CAAC,MAAM1B,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACI,GAAG,KAAKlB,EAAEe,KAAK,CAACD,EAAE,CAACI,GAAG,IAAIlB,EAAEe,KAAK,CAACD,EAAE,CAACK,IAAI,GAAGnB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACK,IAAI,GAAGU,KAAKC,GAAG,CAAC9B,EAAEe,KAAK,CAACD,EAAE,CAACY,CAAC,EAAE1B,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACY,CAAC,IAAI,IAAK;gBACzMZ;gBACAa,IAAId,IAAI,CAACb,EAAEe,KAAK,CAACD,EAAE;gBACnBc,OAAOC,KAAKC,GAAG,CAACF,MAAMH,IAAAA,kBAAM,EAACzB,EAAEe,KAAK,CAACD,EAAE,CAACY,CAAC;YAC3C;YACA,IAAMK,WAAWJ,IAAIpB,GAAG,CAAC,SAACC;uBAAMA,EAAEpB,IAAI;;YACtC,IAAMqB,SAASsB,SAAS/D,KAAK,CAAC,GAAG0C,MAAM,CAAC/C,YAAYoE,QAAQ,CAAC,EAAE;YAC/D,IAAMpB,QAAQgB,IAAIf,OAAO,CAAC,SAACJ;uBAAMA,EAAEG,KAAK;;YACxC,sEAAsE;YACtE,qFAAqF;YACrF,IAAMqB,OAAO7C,eAAesB;YAC5B,IAAIuB,MAAM7B,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAUT,MAAMqB;gBAAQd,QAAQV,cAAcC,kBAAkB;gBAAEyB,OAAAA;YAAM;iBACjG,IAAIF,OAAOlB,MAAM,GAAG,KAAKY,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAST,MAAMqB;gBAAQE,OAAAA;YAAM;iBAC1ER,OAAOU,IAAI,CAAC;gBAAEhB,MAAM;gBAAWT,MAAMqB;gBAAQV,OAAOE,QAAQgC,GAAG,CAACL;gBAAiBjB,OAAAA;YAAM;QAC9F,OAAO;YACLP,KAAKS,IAAI,CAACG;QACZ;QACAF;IACF;IACAT;IACA,OAAOF;AACT"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PageWord } from '../geometry.js';
|
|
2
|
+
export declare const OCR_RENDER_DPI = 288;
|
|
3
|
+
export declare const SPARSE_RETRY_PROSE_FRACTION = 0.6;
|
|
4
|
+
/** Does this page's PSM.AUTO pass look chart-like enough to re-recognize
|
|
5
|
+
* with PSM.SPARSE_TEXT? A page with 0 words never retries (nothing to take a prose fraction of). */
|
|
6
|
+
export declare function shouldRetrySparse(words: {
|
|
7
|
+
text: string;
|
|
8
|
+
}[]): boolean;
|
|
9
|
+
/** Per-page recognition progress, delivered once per page as it finishes.
|
|
10
|
+
* Engine chatter goes through `onWarning`/`warnings` instead — this is timing/count data only. */
|
|
11
|
+
export type OcrProgressEvent = {
|
|
12
|
+
type: 'page';
|
|
13
|
+
page: number;
|
|
14
|
+
words: number;
|
|
15
|
+
ms: number;
|
|
16
|
+
};
|
|
17
|
+
export interface RecognizeResult {
|
|
18
|
+
words: PageWord[];
|
|
19
|
+
/** pages (1-based) where the engine returned zero words */
|
|
20
|
+
emptyPages: number[];
|
|
21
|
+
/** wall-clock ms per page (render + recognize), keyed by page number */
|
|
22
|
+
msPerPage: Map<number, number>;
|
|
23
|
+
}
|
|
24
|
+
/** Recognize words on pages [first..last] of a PDF (renders at `dpi`,
|
|
25
|
+
* default OCR_RENDER_DPI). Progress via `onProgress`; diagnostics via `onWarning` — this module never writes to stdout/stderr itself. */
|
|
26
|
+
export declare function recognizePdf(pdfPath: string, first: number, last: number, { dpi, onProgress, onWarning }?: {
|
|
27
|
+
dpi?: number;
|
|
28
|
+
onProgress?: (e: OcrProgressEvent) => void;
|
|
29
|
+
onWarning?: (message: string) => void;
|
|
30
|
+
}): Promise<RecognizeResult>;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PageWord } from '../geometry.js';
|
|
2
|
+
export declare const OCR_RENDER_DPI = 288;
|
|
3
|
+
export declare const SPARSE_RETRY_PROSE_FRACTION = 0.6;
|
|
4
|
+
/** Does this page's PSM.AUTO pass look chart-like enough to re-recognize
|
|
5
|
+
* with PSM.SPARSE_TEXT? A page with 0 words never retries (nothing to take a prose fraction of). */
|
|
6
|
+
export declare function shouldRetrySparse(words: {
|
|
7
|
+
text: string;
|
|
8
|
+
}[]): boolean;
|
|
9
|
+
/** Per-page recognition progress, delivered once per page as it finishes.
|
|
10
|
+
* Engine chatter goes through `onWarning`/`warnings` instead — this is timing/count data only. */
|
|
11
|
+
export type OcrProgressEvent = {
|
|
12
|
+
type: 'page';
|
|
13
|
+
page: number;
|
|
14
|
+
words: number;
|
|
15
|
+
ms: number;
|
|
16
|
+
};
|
|
17
|
+
export interface RecognizeResult {
|
|
18
|
+
words: PageWord[];
|
|
19
|
+
/** pages (1-based) where the engine returned zero words */
|
|
20
|
+
emptyPages: number[];
|
|
21
|
+
/** wall-clock ms per page (render + recognize), keyed by page number */
|
|
22
|
+
msPerPage: Map<number, number>;
|
|
23
|
+
}
|
|
24
|
+
/** Recognize words on pages [first..last] of a PDF (renders at `dpi`,
|
|
25
|
+
* default OCR_RENDER_DPI). Progress via `onProgress`; diagnostics via `onWarning` — this module never writes to stdout/stderr itself. */
|
|
26
|
+
export declare function recognizePdf(pdfPath: string, first: number, last: number, { dpi, onProgress, onWarning }?: {
|
|
27
|
+
dpi?: number;
|
|
28
|
+
onProgress?: (e: OcrProgressEvent) => void;
|
|
29
|
+
onWarning?: (message: string) => void;
|
|
30
|
+
}): Promise<RecognizeResult>;
|