@effortlessmotion/pdf-to-md 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/package.json +1 -1
- package/skills/pdf-to-md/SKILL.md +39 -66
package/dist/cjs/extract.d.cts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { OcrProgressEvent } from './engines/tesseract.js';
|
|
2
2
|
import type { PageWord, Tuning } from './geometry.js';
|
|
3
|
-
import type { Report } from './types.js';
|
|
3
|
+
import type { Analysis, Report } from './types.js';
|
|
4
4
|
export type { OcrProgressEvent } from './engines/tesseract.js';
|
|
5
5
|
/** One recognized word with its page — the OCR replay input contract (same
|
|
6
6
|
* shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
|
|
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
|
|
|
34
34
|
/** emit a `### pN` marker before each page's content (output formatting
|
|
35
35
|
* only — `report.pageStats` is always populated regardless) */
|
|
36
36
|
pageMarkers?: boolean;
|
|
37
|
+
/** source PDF text runs entering layout, delivered once for the selected
|
|
38
|
+
* final result. Runs retain measured bounds and are not fabricated words. */
|
|
39
|
+
onWords?: (words: OcrWordInput[]) => void;
|
|
37
40
|
}
|
|
38
41
|
/** Options for {@link extractOcr}. */
|
|
39
42
|
export interface ExtractOcrOptions extends CommonOptions {
|
|
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
|
|
|
59
62
|
/** never fall back to OCR, even for image-only pages */
|
|
60
63
|
noOcr?: boolean;
|
|
61
64
|
}
|
|
65
|
+
/** Extract embedded text with a QA report and warnings; never runs OCR.
|
|
66
|
+
* @param pdfPath path to the PDF file
|
|
67
|
+
* @param opts see {@link ExtractTextOptions}
|
|
68
|
+
*/
|
|
62
69
|
export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
|
|
63
70
|
/**
|
|
64
71
|
* OCR extraction. Input is either a PDF (pages render at `dpi` and the
|
|
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
|
|
|
72
79
|
} | {
|
|
73
80
|
words: OcrWordInput[];
|
|
74
81
|
}, opts?: ExtractOcrOptions): Promise<ExtractResult>;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
82
|
+
interface AutoAnalysisResult {
|
|
83
|
+
analysis: Analysis;
|
|
84
|
+
markdown: string;
|
|
85
|
+
warnings: string[];
|
|
86
|
+
}
|
|
87
|
+
/** Shared automatic route for the library's Markdown result and the CLI's
|
|
88
|
+
* raw/plain-text formats. A page changes producer only when it has image
|
|
89
|
+
* evidence and an unusable text layer, so a blank or vector-only page never
|
|
90
|
+
* becomes speculative OCR input. */
|
|
91
|
+
export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
|
|
81
92
|
/**
|
|
82
|
-
*
|
|
83
|
-
*
|
|
93
|
+
* Extract each page from embedded text, using OCR only when the page has
|
|
94
|
+
* image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
|
|
84
95
|
*
|
|
85
96
|
* @param pdfPath path to the PDF file
|
|
86
97
|
* @param opts see {@link PdfToMarkdownOptions}
|
package/dist/cjs/extract.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { OcrProgressEvent } from './engines/tesseract.js';
|
|
2
2
|
import type { PageWord, Tuning } from './geometry.js';
|
|
3
|
-
import type { Report } from './types.js';
|
|
3
|
+
import type { Analysis, Report } from './types.js';
|
|
4
4
|
export type { OcrProgressEvent } from './engines/tesseract.js';
|
|
5
5
|
/** One recognized word with its page — the OCR replay input contract (same
|
|
6
6
|
* shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
|
|
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
|
|
|
34
34
|
/** emit a `### pN` marker before each page's content (output formatting
|
|
35
35
|
* only — `report.pageStats` is always populated regardless) */
|
|
36
36
|
pageMarkers?: boolean;
|
|
37
|
+
/** source PDF text runs entering layout, delivered once for the selected
|
|
38
|
+
* final result. Runs retain measured bounds and are not fabricated words. */
|
|
39
|
+
onWords?: (words: OcrWordInput[]) => void;
|
|
37
40
|
}
|
|
38
41
|
/** Options for {@link extractOcr}. */
|
|
39
42
|
export interface ExtractOcrOptions extends CommonOptions {
|
|
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
|
|
|
59
62
|
/** never fall back to OCR, even for image-only pages */
|
|
60
63
|
noOcr?: boolean;
|
|
61
64
|
}
|
|
65
|
+
/** Extract embedded text with a QA report and warnings; never runs OCR.
|
|
66
|
+
* @param pdfPath path to the PDF file
|
|
67
|
+
* @param opts see {@link ExtractTextOptions}
|
|
68
|
+
*/
|
|
62
69
|
export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
|
|
63
70
|
/**
|
|
64
71
|
* OCR extraction. Input is either a PDF (pages render at `dpi` and the
|
|
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
|
|
|
72
79
|
} | {
|
|
73
80
|
words: OcrWordInput[];
|
|
74
81
|
}, opts?: ExtractOcrOptions): Promise<ExtractResult>;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
82
|
+
interface AutoAnalysisResult {
|
|
83
|
+
analysis: Analysis;
|
|
84
|
+
markdown: string;
|
|
85
|
+
warnings: string[];
|
|
86
|
+
}
|
|
87
|
+
/** Shared automatic route for the library's Markdown result and the CLI's
|
|
88
|
+
* raw/plain-text formats. A page changes producer only when it has image
|
|
89
|
+
* evidence and an unusable text layer, so a blank or vector-only page never
|
|
90
|
+
* becomes speculative OCR input. */
|
|
91
|
+
export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
|
|
81
92
|
/**
|
|
82
|
-
*
|
|
83
|
-
*
|
|
93
|
+
* Extract each page from embedded text, using OCR only when the page has
|
|
94
|
+
* image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
|
|
84
95
|
*
|
|
85
96
|
* @param pdfPath path to the PDF file
|
|
86
97
|
* @param opts see {@link PdfToMarkdownOptions}
|
package/dist/cjs/extract.js
CHANGED
|
@@ -14,18 +14,15 @@ function _export(target, all) {
|
|
|
14
14
|
});
|
|
15
15
|
}
|
|
16
16
|
_export(exports, {
|
|
17
|
+
get analyzePdfAuto () {
|
|
18
|
+
return analyzePdfAuto;
|
|
19
|
+
},
|
|
17
20
|
get extractOcr () {
|
|
18
21
|
return extractOcr;
|
|
19
22
|
},
|
|
20
23
|
get extractText () {
|
|
21
24
|
return extractText;
|
|
22
25
|
},
|
|
23
|
-
get needsOcrFallback () {
|
|
24
|
-
return needsOcrFallback;
|
|
25
|
-
},
|
|
26
|
-
get ocrFallbackNotice () {
|
|
27
|
-
return ocrFallbackNotice;
|
|
28
|
-
},
|
|
29
26
|
get pdfToMarkdown () {
|
|
30
27
|
return pdfToMarkdown;
|
|
31
28
|
}
|
|
@@ -33,11 +30,18 @@ _export(exports, {
|
|
|
33
30
|
var _analyzets = require("./analyze.js");
|
|
34
31
|
var _collectts = require("./collect.js");
|
|
35
32
|
var _dictts = require("./dict.js");
|
|
36
|
-
var _linests = require("./lines.js");
|
|
37
|
-
var _markdownts = require("./markdown.js");
|
|
38
33
|
var _renderanalysists = require("./render-analysis.js");
|
|
39
34
|
var _reportts = require("./report.js");
|
|
35
|
+
var _textanalysists = require("./text-analysis.js");
|
|
40
36
|
var _validatets = require("./validate.js");
|
|
37
|
+
function _array_like_to_array(arr, len) {
|
|
38
|
+
if (len == null || len > arr.length) len = arr.length;
|
|
39
|
+
for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
|
|
40
|
+
return arr2;
|
|
41
|
+
}
|
|
42
|
+
function _array_without_holes(arr) {
|
|
43
|
+
if (Array.isArray(arr)) return _array_like_to_array(arr);
|
|
44
|
+
}
|
|
41
45
|
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
42
46
|
try {
|
|
43
47
|
var info = gen[key](arg);
|
|
@@ -75,6 +79,14 @@ function _define_property(obj, key, value) {
|
|
|
75
79
|
} else obj[key] = value;
|
|
76
80
|
return obj;
|
|
77
81
|
}
|
|
82
|
+
function _iterable_to_array(iter) {
|
|
83
|
+
if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
|
|
84
|
+
return Array.from(iter);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
function _non_iterable_spread() {
|
|
88
|
+
throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
|
|
89
|
+
}
|
|
78
90
|
function _object_spread(target) {
|
|
79
91
|
for(var i = 1; i < arguments.length; i++){
|
|
80
92
|
var source = arguments[i] != null ? arguments[i] : {};
|
|
@@ -113,6 +125,9 @@ function _object_spread_props(target, source) {
|
|
|
113
125
|
}
|
|
114
126
|
return target;
|
|
115
127
|
}
|
|
128
|
+
function _to_consumable_array(arr) {
|
|
129
|
+
return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
|
|
130
|
+
}
|
|
116
131
|
function _ts_generator(thisArg, body) {
|
|
117
132
|
var f, y, t, _ = {
|
|
118
133
|
label: 0,
|
|
@@ -212,13 +227,15 @@ function _ts_generator(thisArg, body) {
|
|
|
212
227
|
};
|
|
213
228
|
}
|
|
214
229
|
}
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
230
|
+
function _unsupported_iterable_to_array(o, minLen) {
|
|
231
|
+
if (!o) return;
|
|
232
|
+
if (typeof o === "string") return _array_like_to_array(o, minLen);
|
|
233
|
+
var n = Object.prototype.toString.call(o).slice(8, -1);
|
|
234
|
+
if (n === "Object" && o.constructor) n = o.constructor.name;
|
|
235
|
+
if (n === "Map" || n === "Set") return Array.from(n);
|
|
236
|
+
if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
|
|
237
|
+
}
|
|
238
|
+
// One collector per entry point: collects into `warnings`, streams via
|
|
222
239
|
// onWarning, and emits the dictionary-unavailable notice exactly once up front.
|
|
223
240
|
function warnCollector(onWarning) {
|
|
224
241
|
var warnings = [];
|
|
@@ -234,7 +251,7 @@ function warnCollector(onWarning) {
|
|
|
234
251
|
}
|
|
235
252
|
function extractText(_0) {
|
|
236
253
|
return _async_to_generator(function(pdfPath) {
|
|
237
|
-
var opts, _opts_pages_last, _opts_pageMarkers, _warnCollector, warnings, warn, range,
|
|
254
|
+
var opts, _opts_pages_last, _opts_pageMarkers, _opts_onWords, _warnCollector, warnings, warn, range, collected, text;
|
|
238
255
|
var _arguments = arguments;
|
|
239
256
|
return _ts_generator(this, function(_state) {
|
|
240
257
|
switch(_state.label){
|
|
@@ -251,64 +268,15 @@ function extractText(_0) {
|
|
|
251
268
|
(0, _collectts.collect)(pdfPath, range)
|
|
252
269
|
];
|
|
253
270
|
case 1:
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
lines: (0, _linests.toLines)(p)
|
|
259
|
-
};
|
|
260
|
-
});
|
|
261
|
-
pageMarkers = (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false;
|
|
262
|
-
_buildMarkdown = (0, _markdownts.buildMarkdown)(pageLines, heightChars, {
|
|
263
|
-
pageMarkers: pageMarkers
|
|
264
|
-
}), md = _buildMarkdown.md, stats = _buildMarkdown.stats, bodyH = _buildMarkdown.bodyH, headingSizes = _buildMarkdown.headingSizes;
|
|
265
|
-
// pageStats needs "### pN" markers to segment content; when the caller
|
|
266
|
-
// didn't request them, build a throwaway marked variant (same heightChars) purely to measure stats from, so pageStats is always populated.
|
|
267
|
-
statsMd = pageMarkers ? md : (0, _markdownts.buildMarkdown)(pageLines, heightChars, {
|
|
268
|
-
pageMarkers: true
|
|
269
|
-
}).md;
|
|
270
|
-
// Advisory, not prescriptive: `hasLargeImage` fires on any large image,
|
|
271
|
-
// even one with no baked-in text, where `--ocr` recovers nothing and can
|
|
272
|
-
// actively degrade a correct result (measured: "I" -> "|", stray em-dashes) — so this tells the caller to COMPARE the OCR rerun, never blindly replace.
|
|
273
|
-
hybridPages = pages.filter(function(p) {
|
|
274
|
-
return p.largeImage;
|
|
275
|
-
}).map(function(p) {
|
|
276
|
-
return p.page;
|
|
277
|
-
});
|
|
278
|
-
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
279
|
-
try {
|
|
280
|
-
for(_iterator = hybridPages[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
281
|
-
n = _step.value;
|
|
282
|
-
warn("p".concat(n, ": text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ").concat(n, " --ocr, which can itself introduce errors"));
|
|
283
|
-
}
|
|
284
|
-
} catch (err) {
|
|
285
|
-
_didIteratorError = true;
|
|
286
|
-
_iteratorError = err;
|
|
287
|
-
} finally{
|
|
288
|
-
try {
|
|
289
|
-
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
290
|
-
_iterator.return();
|
|
291
|
-
}
|
|
292
|
-
} finally{
|
|
293
|
-
if (_didIteratorError) {
|
|
294
|
-
throw _iteratorError;
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
report = (0, _reportts.buildReport)({
|
|
299
|
-
numPages: numPages,
|
|
300
|
-
md: md,
|
|
301
|
-
statsMd: statsMd,
|
|
302
|
-
stats: stats,
|
|
303
|
-
bodyH: bodyH,
|
|
304
|
-
headingSizes: headingSizes,
|
|
305
|
-
hybridPages: hybridPages
|
|
306
|
-
});
|
|
271
|
+
collected = _state.sent();
|
|
272
|
+
text = (0, _textanalysists.buildTextAnalysis)(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
|
|
273
|
+
textNotices(collected, new Set(), warn);
|
|
274
|
+
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, text.words);
|
|
307
275
|
return [
|
|
308
276
|
2,
|
|
309
277
|
{
|
|
310
|
-
markdown:
|
|
311
|
-
report: report,
|
|
278
|
+
markdown: text.markdown,
|
|
279
|
+
report: text.analysis.report,
|
|
312
280
|
warnings: warnings
|
|
313
281
|
}
|
|
314
282
|
];
|
|
@@ -355,61 +323,171 @@ function extractOcr(_0) {
|
|
|
355
323
|
});
|
|
356
324
|
}).apply(this, arguments);
|
|
357
325
|
}
|
|
358
|
-
|
|
359
|
-
|
|
326
|
+
/** A page needs OCR only when its image signal accompanies an unusable text
|
|
327
|
+
* layer. The caller supplies the image part of that decision. */ function needsPageOcr(chars) {
|
|
328
|
+
return chars < _reportts.AUTO_OCR_CHARS_PER_PAGE;
|
|
360
329
|
}
|
|
361
|
-
function
|
|
362
|
-
|
|
330
|
+
function textNotices(collected, ocrPages, warn) {
|
|
331
|
+
var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
332
|
+
try {
|
|
333
|
+
for(var _iterator = collected.pages[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
334
|
+
var page = _step.value;
|
|
335
|
+
if (page.textChars !== 0 || ocrPages.has(page.page)) continue;
|
|
336
|
+
if (page.hasImage) warn("p".concat(page.page, ": no extractable text; page has image content and OCR was not selected"));
|
|
337
|
+
else warn("p".concat(page.page, ": no extractable text; the page may be blank or contain only vector content"));
|
|
338
|
+
}
|
|
339
|
+
} catch (err) {
|
|
340
|
+
_didIteratorError = true;
|
|
341
|
+
_iteratorError = err;
|
|
342
|
+
} finally{
|
|
343
|
+
try {
|
|
344
|
+
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
345
|
+
_iterator.return();
|
|
346
|
+
}
|
|
347
|
+
} finally{
|
|
348
|
+
if (_didIteratorError) {
|
|
349
|
+
throw _iteratorError;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
|
|
354
|
+
try {
|
|
355
|
+
for(var _iterator1 = collected.pages[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
|
|
356
|
+
var page1 = _step1.value;
|
|
357
|
+
if (!page1.largeImage || ocrPages.has(page1.page)) continue;
|
|
358
|
+
warn("p".concat(page1.page, ": text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ").concat(page1.page, " --ocr, which can itself introduce errors"));
|
|
359
|
+
}
|
|
360
|
+
} catch (err) {
|
|
361
|
+
_didIteratorError1 = true;
|
|
362
|
+
_iteratorError1 = err;
|
|
363
|
+
} finally{
|
|
364
|
+
try {
|
|
365
|
+
if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
|
|
366
|
+
_iterator1.return();
|
|
367
|
+
}
|
|
368
|
+
} finally{
|
|
369
|
+
if (_didIteratorError1) {
|
|
370
|
+
throw _iteratorError1;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
363
374
|
}
|
|
364
|
-
function
|
|
375
|
+
function analysisWords(pages) {
|
|
376
|
+
return pages.flatMap(function(page) {
|
|
377
|
+
return page.words.map(function(word) {
|
|
378
|
+
return _object_spread_props(_object_spread({
|
|
379
|
+
page: page.page,
|
|
380
|
+
text: word.text
|
|
381
|
+
}, word.box), {
|
|
382
|
+
confidence: word.confidence
|
|
383
|
+
});
|
|
384
|
+
});
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
function analyzePdfAuto(_0) {
|
|
365
388
|
return _async_to_generator(function(pdfPath) {
|
|
366
|
-
var opts,
|
|
389
|
+
var opts, _opts_pages_last, _opts_onWords, _warnCollector, warnings, warn, range, _opts_onWords1, _ref, pages, words, markdown, report, analysis, collected, ocrPageNumbers, _opts_pageMarkers, _opts_onWords2, text, textPageNumbers, text1, textByPage, ocrByPage, ocrWords, ocrRanges, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, page, lastRange, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, range1, _ocrWords, result, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, page1, err, pages1, markdown1, pagePaths, report1, analysis1;
|
|
367
390
|
var _arguments = arguments;
|
|
368
391
|
return _ts_generator(this, function(_state) {
|
|
369
392
|
switch(_state.label){
|
|
370
393
|
case 0:
|
|
371
394
|
opts = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
return [
|
|
379
|
-
4,
|
|
380
|
-
extractText(pdfPath, _object_spread_props(_object_spread({}, opts), {
|
|
381
|
-
onWarning: undefined
|
|
382
|
-
}))
|
|
383
|
-
];
|
|
384
|
-
case 1:
|
|
385
|
-
text = _state.sent();
|
|
386
|
-
if (!(needsOcrFallback(text.report.charsPerPage) && !opts.noOcr)) return [
|
|
395
|
+
(0, _validatets.validatePages)(opts.pages);
|
|
396
|
+
_warnCollector = warnCollector(opts.onWarning), warnings = _warnCollector.warnings, warn = _warnCollector.warn;
|
|
397
|
+
range = opts.pages ? {
|
|
398
|
+
first: opts.pages.first,
|
|
399
|
+
last: (_opts_pages_last = opts.pages.last) !== null && _opts_pages_last !== void 0 ? _opts_pages_last : opts.pages.first
|
|
400
|
+
} : {};
|
|
401
|
+
if (!opts.ocr) return [
|
|
387
402
|
3,
|
|
388
|
-
|
|
403
|
+
2
|
|
389
404
|
];
|
|
390
|
-
notice = ocrFallbackNotice(text.report.charsPerPage);
|
|
391
|
-
(_opts_onWarning1 = opts.onWarning) === null || _opts_onWarning1 === void 0 ? void 0 : _opts_onWarning1.call(opts, notice);
|
|
392
405
|
return [
|
|
393
406
|
4,
|
|
394
|
-
|
|
407
|
+
(0, _analyzets.buildOcrAnalysisPages)({
|
|
395
408
|
pdfPath: pdfPath
|
|
396
409
|
}, _object_spread_props(_object_spread({}, opts), {
|
|
397
|
-
|
|
398
|
-
}))
|
|
410
|
+
onWords: undefined
|
|
411
|
+
}), warn)
|
|
399
412
|
];
|
|
400
|
-
case
|
|
401
|
-
|
|
402
|
-
|
|
413
|
+
case 1:
|
|
414
|
+
_ref = _state.sent(), pages = _ref.pages, words = _ref.words;
|
|
415
|
+
markdown = (0, _renderanalysists.renderPagesToMarkdown)(pages, {
|
|
416
|
+
forcePageMarkers: true,
|
|
417
|
+
pageMarkers: true
|
|
418
|
+
});
|
|
419
|
+
report = (0, _reportts.buildOcrReport)({
|
|
420
|
+
md: markdown,
|
|
421
|
+
words: words
|
|
422
|
+
});
|
|
423
|
+
analysis = {
|
|
424
|
+
outputVersion: report.outputVersion,
|
|
425
|
+
report: report,
|
|
426
|
+
pages: pages
|
|
427
|
+
};
|
|
428
|
+
(_opts_onWords1 = opts.onWords) === null || _opts_onWords1 === void 0 ? void 0 : _opts_onWords1.call(opts, words);
|
|
403
429
|
return [
|
|
404
430
|
2,
|
|
405
|
-
|
|
431
|
+
{
|
|
432
|
+
analysis: analysis,
|
|
433
|
+
markdown: markdown,
|
|
434
|
+
warnings: warnings
|
|
435
|
+
}
|
|
436
|
+
];
|
|
437
|
+
case 2:
|
|
438
|
+
return [
|
|
439
|
+
4,
|
|
440
|
+
(0, _collectts.collect)(pdfPath, range)
|
|
406
441
|
];
|
|
407
442
|
case 3:
|
|
443
|
+
collected = _state.sent();
|
|
444
|
+
ocrPageNumbers = new Set(!opts.noOcr ? collected.pages.filter(function(page) {
|
|
445
|
+
return page.hasImage && needsPageOcr(page.textChars);
|
|
446
|
+
}).map(function(page) {
|
|
447
|
+
return page.page;
|
|
448
|
+
}) : []);
|
|
449
|
+
if (!ocrPageNumbers.size) {
|
|
450
|
+
;
|
|
451
|
+
;
|
|
452
|
+
text = (0, _textanalysists.buildTextAnalysis)(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
|
|
453
|
+
textNotices(collected, ocrPageNumbers, warn);
|
|
454
|
+
(_opts_onWords2 = opts.onWords) === null || _opts_onWords2 === void 0 ? void 0 : _opts_onWords2.call(opts, text.words);
|
|
455
|
+
return [
|
|
456
|
+
2,
|
|
457
|
+
{
|
|
458
|
+
analysis: text.analysis,
|
|
459
|
+
markdown: text.markdown,
|
|
460
|
+
warnings: warnings
|
|
461
|
+
}
|
|
462
|
+
];
|
|
463
|
+
}
|
|
464
|
+
textPageNumbers = new Set(collected.pages.filter(function(page) {
|
|
465
|
+
return !ocrPageNumbers.has(page.page);
|
|
466
|
+
}).map(function(page) {
|
|
467
|
+
return page.page;
|
|
468
|
+
}));
|
|
469
|
+
text1 = textPageNumbers.size ? (0, _textanalysists.buildTextAnalysis)((0, _textanalysists.selectCollectedPages)(collected, textPageNumbers), true) : undefined;
|
|
470
|
+
textByPage = new Map(text1 === null || text1 === void 0 ? void 0 : text1.analysis.pages.map(function(page) {
|
|
471
|
+
return [
|
|
472
|
+
page.page,
|
|
473
|
+
page
|
|
474
|
+
];
|
|
475
|
+
}));
|
|
476
|
+
ocrByPage = new Map();
|
|
477
|
+
ocrWords = [];
|
|
478
|
+
ocrRanges = [];
|
|
408
479
|
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
409
480
|
try {
|
|
410
|
-
for(_iterator =
|
|
411
|
-
|
|
412
|
-
(
|
|
481
|
+
for(_iterator = collected.pages[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
482
|
+
page = _step.value;
|
|
483
|
+
if (!ocrPageNumbers.has(page.page)) continue;
|
|
484
|
+
warn("p".concat(page.page, ": sparse text layer with image content; using OCR for this page"));
|
|
485
|
+
lastRange = ocrRanges.at(-1);
|
|
486
|
+
if (lastRange && lastRange.last + 1 === page.page) lastRange.last = page.page;
|
|
487
|
+
else ocrRanges.push({
|
|
488
|
+
first: page.page,
|
|
489
|
+
last: page.page
|
|
490
|
+
});
|
|
413
491
|
}
|
|
414
492
|
} catch (err) {
|
|
415
493
|
_didIteratorError = true;
|
|
@@ -425,9 +503,158 @@ function pdfToMarkdown(_0) {
|
|
|
425
503
|
}
|
|
426
504
|
}
|
|
427
505
|
}
|
|
506
|
+
_iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
|
|
507
|
+
_state.label = 4;
|
|
508
|
+
case 4:
|
|
509
|
+
_state.trys.push([
|
|
510
|
+
4,
|
|
511
|
+
9,
|
|
512
|
+
10,
|
|
513
|
+
11
|
|
514
|
+
]);
|
|
515
|
+
_iterator1 = ocrRanges[Symbol.iterator]();
|
|
516
|
+
_state.label = 5;
|
|
517
|
+
case 5:
|
|
518
|
+
if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
|
|
519
|
+
3,
|
|
520
|
+
8
|
|
521
|
+
];
|
|
522
|
+
range1 = _step1.value;
|
|
523
|
+
return [
|
|
524
|
+
4,
|
|
525
|
+
(0, _analyzets.buildOcrAnalysisPages)({
|
|
526
|
+
pdfPath: pdfPath
|
|
527
|
+
}, _object_spread_props(_object_spread({}, opts), {
|
|
528
|
+
pages: range1,
|
|
529
|
+
onWords: undefined
|
|
530
|
+
}), warn)
|
|
531
|
+
];
|
|
532
|
+
case 6:
|
|
533
|
+
result = _state.sent();
|
|
534
|
+
_iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
|
|
535
|
+
try {
|
|
536
|
+
for(_iterator2 = result.pages[Symbol.iterator](); !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
|
|
537
|
+
page1 = _step2.value;
|
|
538
|
+
ocrByPage.set(page1.page, page1);
|
|
539
|
+
}
|
|
540
|
+
} catch (err) {
|
|
541
|
+
_didIteratorError2 = true;
|
|
542
|
+
_iteratorError2 = err;
|
|
543
|
+
} finally{
|
|
544
|
+
try {
|
|
545
|
+
if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
|
|
546
|
+
_iterator2.return();
|
|
547
|
+
}
|
|
548
|
+
} finally{
|
|
549
|
+
if (_didIteratorError2) {
|
|
550
|
+
throw _iteratorError2;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
(_ocrWords = ocrWords).push.apply(_ocrWords, _to_consumable_array(result.words));
|
|
555
|
+
_state.label = 7;
|
|
556
|
+
case 7:
|
|
557
|
+
_iteratorNormalCompletion1 = true;
|
|
558
|
+
return [
|
|
559
|
+
3,
|
|
560
|
+
5
|
|
561
|
+
];
|
|
562
|
+
case 8:
|
|
563
|
+
return [
|
|
564
|
+
3,
|
|
565
|
+
11
|
|
566
|
+
];
|
|
567
|
+
case 9:
|
|
568
|
+
err = _state.sent();
|
|
569
|
+
_didIteratorError1 = true;
|
|
570
|
+
_iteratorError1 = err;
|
|
571
|
+
return [
|
|
572
|
+
3,
|
|
573
|
+
11
|
|
574
|
+
];
|
|
575
|
+
case 10:
|
|
576
|
+
try {
|
|
577
|
+
if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
|
|
578
|
+
_iterator1.return();
|
|
579
|
+
}
|
|
580
|
+
} finally{
|
|
581
|
+
if (_didIteratorError1) {
|
|
582
|
+
throw _iteratorError1;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
return [
|
|
586
|
+
7
|
|
587
|
+
];
|
|
588
|
+
case 11:
|
|
589
|
+
textNotices(collected, ocrPageNumbers, warn);
|
|
590
|
+
pages1 = collected.pages.map(function(page) {
|
|
591
|
+
var _ocrByPage_get;
|
|
592
|
+
return (_ocrByPage_get = ocrByPage.get(page.page)) !== null && _ocrByPage_get !== void 0 ? _ocrByPage_get : textByPage.get(page.page);
|
|
593
|
+
});
|
|
594
|
+
markdown1 = (0, _renderanalysists.renderPagesToMarkdown)(pages1, {
|
|
595
|
+
forcePageMarkers: true,
|
|
596
|
+
pageMarkers: true
|
|
597
|
+
});
|
|
598
|
+
pagePaths = new Map(pages1.map(function(page) {
|
|
599
|
+
return [
|
|
600
|
+
page.page,
|
|
601
|
+
page.path
|
|
602
|
+
];
|
|
603
|
+
}));
|
|
604
|
+
report1 = (0, _reportts.buildOcrReport)({
|
|
605
|
+
md: markdown1,
|
|
606
|
+
fallback: true,
|
|
607
|
+
words: ocrWords,
|
|
608
|
+
pagePaths: pagePaths,
|
|
609
|
+
hybridPages: text1 === null || text1 === void 0 ? void 0 : text1.analysis.report.hybridPages
|
|
610
|
+
});
|
|
611
|
+
if (text1) {
|
|
612
|
+
report1.bodyFontHeight = text1.analysis.report.bodyFontHeight;
|
|
613
|
+
report1.headingSizes = text1.analysis.report.headingSizes;
|
|
614
|
+
report1.listItems = text1.analysis.report.listItems;
|
|
615
|
+
report1.joinedPairs = text1.analysis.report.joinedPairs;
|
|
616
|
+
report1.closedHyphens = text1.analysis.report.closedHyphens;
|
|
617
|
+
report1.hybridPages = text1.analysis.report.hybridPages;
|
|
618
|
+
}
|
|
619
|
+
analysis1 = {
|
|
620
|
+
outputVersion: report1.outputVersion,
|
|
621
|
+
report: report1,
|
|
622
|
+
pages: pages1
|
|
623
|
+
};
|
|
624
|
+
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, analysisWords(pages1));
|
|
625
|
+
return [
|
|
626
|
+
2,
|
|
627
|
+
{
|
|
628
|
+
analysis: analysis1,
|
|
629
|
+
markdown: markdown1,
|
|
630
|
+
warnings: warnings
|
|
631
|
+
}
|
|
632
|
+
];
|
|
633
|
+
}
|
|
634
|
+
});
|
|
635
|
+
}).apply(this, arguments);
|
|
636
|
+
}
|
|
637
|
+
function pdfToMarkdown(_0) {
|
|
638
|
+
return _async_to_generator(function(pdfPath) {
|
|
639
|
+
var opts, result;
|
|
640
|
+
var _arguments = arguments;
|
|
641
|
+
return _ts_generator(this, function(_state) {
|
|
642
|
+
switch(_state.label){
|
|
643
|
+
case 0:
|
|
644
|
+
opts = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
|
|
645
|
+
return [
|
|
646
|
+
4,
|
|
647
|
+
analyzePdfAuto(pdfPath, opts)
|
|
648
|
+
];
|
|
649
|
+
case 1:
|
|
650
|
+
result = _state.sent();
|
|
428
651
|
return [
|
|
429
652
|
2,
|
|
430
|
-
|
|
653
|
+
{
|
|
654
|
+
markdown: result.markdown,
|
|
655
|
+
report: result.analysis.report,
|
|
656
|
+
warnings: result.warnings
|
|
657
|
+
}
|
|
431
658
|
];
|
|
432
659
|
}
|
|
433
660
|
});
|