@effortlessmotion/pdf-to-md 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +5 -5
  2. package/dist/cjs/analyze.d.cts +3 -1
  3. package/dist/cjs/analyze.d.ts +3 -1
  4. package/dist/cjs/analyze.js +38 -100
  5. package/dist/cjs/analyze.js.map +1 -1
  6. package/dist/cjs/audit.d.cts +6 -5
  7. package/dist/cjs/audit.d.ts +6 -5
  8. package/dist/cjs/audit.js +368 -60
  9. package/dist/cjs/audit.js.map +1 -1
  10. package/dist/cjs/cli/extract.js +13 -48
  11. package/dist/cjs/cli/extract.js.map +1 -1
  12. package/dist/cjs/collect.js +91 -35
  13. package/dist/cjs/collect.js.map +1 -1
  14. package/dist/cjs/errors.d.cts +2 -0
  15. package/dist/cjs/errors.d.ts +2 -0
  16. package/dist/cjs/errors.js.map +1 -1
  17. package/dist/cjs/extract.d.cts +20 -9
  18. package/dist/cjs/extract.d.ts +20 -9
  19. package/dist/cjs/extract.js +333 -106
  20. package/dist/cjs/extract.js.map +1 -1
  21. package/dist/cjs/lines.js +21 -10
  22. package/dist/cjs/lines.js.map +1 -1
  23. package/dist/cjs/markdown.js +140 -54
  24. package/dist/cjs/markdown.js.map +1 -1
  25. package/dist/cjs/page-marker.d.cts +1 -1
  26. package/dist/cjs/page-marker.d.ts +1 -1
  27. package/dist/cjs/page-marker.js +39 -8
  28. package/dist/cjs/page-marker.js.map +1 -1
  29. package/dist/cjs/render-analysis.d.cts +4 -2
  30. package/dist/cjs/render-analysis.d.ts +4 -2
  31. package/dist/cjs/render-analysis.js +5 -3
  32. package/dist/cjs/render-analysis.js.map +1 -1
  33. package/dist/cjs/report.d.cts +2 -2
  34. package/dist/cjs/report.d.ts +2 -2
  35. package/dist/cjs/report.js +27 -8
  36. package/dist/cjs/report.js.map +1 -1
  37. package/dist/cjs/text-analysis.d.cts +12 -0
  38. package/dist/cjs/text-analysis.d.ts +12 -0
  39. package/dist/cjs/text-analysis.js +175 -0
  40. package/dist/cjs/text-analysis.js.map +1 -0
  41. package/dist/cjs/types.d.cts +40 -3
  42. package/dist/cjs/types.d.ts +40 -3
  43. package/dist/cjs/types.js +2 -3
  44. package/dist/cjs/types.js.map +1 -1
  45. package/dist/esm/analyze.d.ts +3 -1
  46. package/dist/esm/analyze.js +18 -89
  47. package/dist/esm/analyze.js.map +1 -1
  48. package/dist/esm/audit.d.ts +6 -5
  49. package/dist/esm/audit.js +236 -29
  50. package/dist/esm/audit.js.map +1 -1
  51. package/dist/esm/cli/extract.js +10 -23
  52. package/dist/esm/cli/extract.js.map +1 -1
  53. package/dist/esm/collect.js +64 -15
  54. package/dist/esm/collect.js.map +1 -1
  55. package/dist/esm/errors.d.ts +2 -0
  56. package/dist/esm/errors.js.map +1 -1
  57. package/dist/esm/extract.d.ts +20 -9
  58. package/dist/esm/extract.js +167 -76
  59. package/dist/esm/extract.js.map +1 -1
  60. package/dist/esm/lines.js +14 -10
  61. package/dist/esm/lines.js.map +1 -1
  62. package/dist/esm/markdown.js +110 -27
  63. package/dist/esm/markdown.js.map +1 -1
  64. package/dist/esm/page-marker.d.ts +1 -1
  65. package/dist/esm/page-marker.js +15 -9
  66. package/dist/esm/page-marker.js.map +1 -1
  67. package/dist/esm/render-analysis.d.ts +4 -2
  68. package/dist/esm/render-analysis.js +9 -5
  69. package/dist/esm/render-analysis.js.map +1 -1
  70. package/dist/esm/report.d.ts +2 -2
  71. package/dist/esm/report.js +12 -7
  72. package/dist/esm/report.js.map +1 -1
  73. package/dist/esm/text-analysis.d.ts +12 -0
  74. package/dist/esm/text-analysis.js +81 -0
  75. package/dist/esm/text-analysis.js.map +1 -0
  76. package/dist/esm/types.d.ts +40 -3
  77. package/dist/esm/types.js +4 -4
  78. package/dist/esm/types.js.map +1 -1
  79. package/package.json +1 -1
  80. package/skills/pdf-to-md/SKILL.md +39 -66
@@ -1,6 +1,6 @@
1
1
  import type { OcrProgressEvent } from './engines/tesseract.js';
2
2
  import type { PageWord, Tuning } from './geometry.js';
3
- import type { Report } from './types.js';
3
+ import type { Analysis, Report } from './types.js';
4
4
  export type { OcrProgressEvent } from './engines/tesseract.js';
5
5
  /** One recognized word with its page — the OCR replay input contract (same
6
6
  * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
34
34
  /** emit a `### pN` marker before each page's content (output formatting
35
35
  * only — `report.pageStats` is always populated regardless) */
36
36
  pageMarkers?: boolean;
37
+ /** source PDF text runs entering layout, delivered once for the selected
38
+ * final result. Runs retain measured bounds and are not fabricated words. */
39
+ onWords?: (words: OcrWordInput[]) => void;
37
40
  }
38
41
  /** Options for {@link extractOcr}. */
39
42
  export interface ExtractOcrOptions extends CommonOptions {
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
59
62
  /** never fall back to OCR, even for image-only pages */
60
63
  noOcr?: boolean;
61
64
  }
65
+ /** Extract embedded text with a QA report and warnings; never runs OCR.
66
+ * @param pdfPath path to the PDF file
67
+ * @param opts see {@link ExtractTextOptions}
68
+ */
62
69
  export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
63
70
  /**
64
71
  * OCR extraction. Input is either a PDF (pages render at `dpi` and the
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
72
79
  } | {
73
80
  words: OcrWordInput[];
74
81
  }, opts?: ExtractOcrOptions): Promise<ExtractResult>;
75
- /** pdfToMarkdown's auto-OCR fallback predicate, shared with the CLI's
76
- * --format=raw|txt path so the two call sites can never drift apart. Module-internal: not re-exported by `src/index.ts`. */
77
- export declare function needsOcrFallback(charsPerPage: number): boolean;
78
- /** The exact stderr/warnings notice printed when needsOcrFallback fires —
79
- * one copy of the string for both call sites (asserted verbatim by the suite). */
80
- export declare function ocrFallbackNotice(charsPerPage: number): string;
82
+ interface AutoAnalysisResult {
83
+ analysis: Analysis;
84
+ markdown: string;
85
+ warnings: string[];
86
+ }
87
+ /** Shared automatic route for the library's Markdown result and the CLI's
88
+ * raw/plain-text formats. A page changes producer only when it has image
89
+ * evidence and an unusable text layer, so a blank or vector-only page never
90
+ * becomes speculative OCR input. */
91
+ export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
81
92
  /**
82
- * The headline API: extract a PDF to markdown, text layer first, falling
83
- * back to OCR when the text layer is empty enough the pages are certainly images of text (< `AUTO_OCR_CHARS_PER_PAGE` chars/page) — the same decision the CLI makes.
93
+ * Extract each page from embedded text, using OCR only when the page has
94
+ * image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
84
95
  *
85
96
  * @param pdfPath path to the PDF file
86
97
  * @param opts see {@link PdfToMarkdownOptions}
@@ -1,6 +1,6 @@
1
1
  import type { OcrProgressEvent } from './engines/tesseract.js';
2
2
  import type { PageWord, Tuning } from './geometry.js';
3
- import type { Report } from './types.js';
3
+ import type { Analysis, Report } from './types.js';
4
4
  export type { OcrProgressEvent } from './engines/tesseract.js';
5
5
  /** One recognized word with its page — the OCR replay input contract (same
6
6
  * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
34
34
  /** emit a `### pN` marker before each page's content (output formatting
35
35
  * only — `report.pageStats` is always populated regardless) */
36
36
  pageMarkers?: boolean;
37
+ /** source PDF text runs entering layout, delivered once for the selected
38
+ * final result. Runs retain measured bounds and are not fabricated words. */
39
+ onWords?: (words: OcrWordInput[]) => void;
37
40
  }
38
41
  /** Options for {@link extractOcr}. */
39
42
  export interface ExtractOcrOptions extends CommonOptions {
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
59
62
  /** never fall back to OCR, even for image-only pages */
60
63
  noOcr?: boolean;
61
64
  }
65
+ /** Extract embedded text with a QA report and warnings; never runs OCR.
66
+ * @param pdfPath path to the PDF file
67
+ * @param opts see {@link ExtractTextOptions}
68
+ */
62
69
  export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
63
70
  /**
64
71
  * OCR extraction. Input is either a PDF (pages render at `dpi` and the
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
72
79
  } | {
73
80
  words: OcrWordInput[];
74
81
  }, opts?: ExtractOcrOptions): Promise<ExtractResult>;
75
- /** pdfToMarkdown's auto-OCR fallback predicate, shared with the CLI's
76
- * --format=raw|txt path so the two call sites can never drift apart. Module-internal: not re-exported by `src/index.ts`. */
77
- export declare function needsOcrFallback(charsPerPage: number): boolean;
78
- /** The exact stderr/warnings notice printed when needsOcrFallback fires —
79
- * one copy of the string for both call sites (asserted verbatim by the suite). */
80
- export declare function ocrFallbackNotice(charsPerPage: number): string;
82
+ interface AutoAnalysisResult {
83
+ analysis: Analysis;
84
+ markdown: string;
85
+ warnings: string[];
86
+ }
87
+ /** Shared automatic route for the library's Markdown result and the CLI's
88
+ * raw/plain-text formats. A page changes producer only when it has image
89
+ * evidence and an unusable text layer, so a blank or vector-only page never
90
+ * becomes speculative OCR input. */
91
+ export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
81
92
  /**
82
- * The headline API: extract a PDF to markdown, text layer first, falling
83
- * back to OCR when the text layer is empty enough the pages are certainly images of text (< `AUTO_OCR_CHARS_PER_PAGE` chars/page) — the same decision the CLI makes.
93
+ * Extract each page from embedded text, using OCR only when the page has
94
+ * image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
84
95
  *
85
96
  * @param pdfPath path to the PDF file
86
97
  * @param opts see {@link PdfToMarkdownOptions}
@@ -14,18 +14,15 @@ function _export(target, all) {
14
14
  });
15
15
  }
16
16
  _export(exports, {
17
+ get analyzePdfAuto () {
18
+ return analyzePdfAuto;
19
+ },
17
20
  get extractOcr () {
18
21
  return extractOcr;
19
22
  },
20
23
  get extractText () {
21
24
  return extractText;
22
25
  },
23
- get needsOcrFallback () {
24
- return needsOcrFallback;
25
- },
26
- get ocrFallbackNotice () {
27
- return ocrFallbackNotice;
28
- },
29
26
  get pdfToMarkdown () {
30
27
  return pdfToMarkdown;
31
28
  }
@@ -33,11 +30,18 @@ _export(exports, {
33
30
  var _analyzets = require("./analyze.js");
34
31
  var _collectts = require("./collect.js");
35
32
  var _dictts = require("./dict.js");
36
- var _linests = require("./lines.js");
37
- var _markdownts = require("./markdown.js");
38
33
  var _renderanalysists = require("./render-analysis.js");
39
34
  var _reportts = require("./report.js");
35
+ var _textanalysists = require("./text-analysis.js");
40
36
  var _validatets = require("./validate.js");
37
+ function _array_like_to_array(arr, len) {
38
+ if (len == null || len > arr.length) len = arr.length;
39
+ for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
40
+ return arr2;
41
+ }
42
+ function _array_without_holes(arr) {
43
+ if (Array.isArray(arr)) return _array_like_to_array(arr);
44
+ }
41
45
  function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
42
46
  try {
43
47
  var info = gen[key](arg);
@@ -75,6 +79,14 @@ function _define_property(obj, key, value) {
75
79
  } else obj[key] = value;
76
80
  return obj;
77
81
  }
82
+ function _iterable_to_array(iter) {
83
+ if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
84
+ return Array.from(iter);
85
+ }
86
+ }
87
+ function _non_iterable_spread() {
88
+ throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
89
+ }
78
90
  function _object_spread(target) {
79
91
  for(var i = 1; i < arguments.length; i++){
80
92
  var source = arguments[i] != null ? arguments[i] : {};
@@ -113,6 +125,9 @@ function _object_spread_props(target, source) {
113
125
  }
114
126
  return target;
115
127
  }
128
+ function _to_consumable_array(arr) {
129
+ return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
130
+ }
116
131
  function _ts_generator(thisArg, body) {
117
132
  var f, y, t, _ = {
118
133
  label: 0,
@@ -212,13 +227,15 @@ function _ts_generator(thisArg, body) {
212
227
  };
213
228
  }
214
229
  }
215
- /**
216
- * Text-layer extraction. Returns markdown + QA report + warnings (hybrid
217
- * pages carrying large images whose baked-in text the text layer can never see). Makes no OCR decision — see pdfToMarkdown for the fallback.
218
- *
219
- * @param pdfPath path to the PDF file
220
- * @param opts see {@link ExtractTextOptions}
221
- */ // One collector per entry point: collects into `warnings`, streams via
230
+ function _unsupported_iterable_to_array(o, minLen) {
231
+ if (!o) return;
232
+ if (typeof o === "string") return _array_like_to_array(o, minLen);
233
+ var n = Object.prototype.toString.call(o).slice(8, -1);
234
+ if (n === "Object" && o.constructor) n = o.constructor.name;
235
+ if (n === "Map" || n === "Set") return Array.from(n);
236
+ if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
237
+ }
238
+ // One collector per entry point: collects into `warnings`, streams via
222
239
  // onWarning, and emits the dictionary-unavailable notice exactly once up front.
223
240
  function warnCollector(onWarning) {
224
241
  var warnings = [];
@@ -234,7 +251,7 @@ function warnCollector(onWarning) {
234
251
  }
235
252
  function extractText(_0) {
236
253
  return _async_to_generator(function(pdfPath) {
237
- var opts, _opts_pages_last, _opts_pageMarkers, _warnCollector, warnings, warn, range, _ref, numPages, pages, heightChars, pageLines, pageMarkers, _buildMarkdown, md, stats, bodyH, headingSizes, statsMd, hybridPages, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, n, report;
254
+ var opts, _opts_pages_last, _opts_pageMarkers, _opts_onWords, _warnCollector, warnings, warn, range, collected, text;
238
255
  var _arguments = arguments;
239
256
  return _ts_generator(this, function(_state) {
240
257
  switch(_state.label){
@@ -251,64 +268,15 @@ function extractText(_0) {
251
268
  (0, _collectts.collect)(pdfPath, range)
252
269
  ];
253
270
  case 1:
254
- _ref = _state.sent(), numPages = _ref.numPages, pages = _ref.pages, heightChars = _ref.heightChars;
255
- pageLines = pages.map(function(p) {
256
- return {
257
- page: p.page,
258
- lines: (0, _linests.toLines)(p)
259
- };
260
- });
261
- pageMarkers = (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false;
262
- _buildMarkdown = (0, _markdownts.buildMarkdown)(pageLines, heightChars, {
263
- pageMarkers: pageMarkers
264
- }), md = _buildMarkdown.md, stats = _buildMarkdown.stats, bodyH = _buildMarkdown.bodyH, headingSizes = _buildMarkdown.headingSizes;
265
- // pageStats needs "### pN" markers to segment content; when the caller
266
- // didn't request them, build a throwaway marked variant (same heightChars) purely to measure stats from, so pageStats is always populated.
267
- statsMd = pageMarkers ? md : (0, _markdownts.buildMarkdown)(pageLines, heightChars, {
268
- pageMarkers: true
269
- }).md;
270
- // Advisory, not prescriptive: `hasLargeImage` fires on any large image,
271
- // even one with no baked-in text, where `--ocr` recovers nothing and can
272
- // actively degrade a correct result (measured: "I" -> "|", stray em-dashes) — so this tells the caller to COMPARE the OCR rerun, never blindly replace.
273
- hybridPages = pages.filter(function(p) {
274
- return p.largeImage;
275
- }).map(function(p) {
276
- return p.page;
277
- });
278
- _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
279
- try {
280
- for(_iterator = hybridPages[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
281
- n = _step.value;
282
- warn("p".concat(n, ": text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ").concat(n, " --ocr, which can itself introduce errors"));
283
- }
284
- } catch (err) {
285
- _didIteratorError = true;
286
- _iteratorError = err;
287
- } finally{
288
- try {
289
- if (!_iteratorNormalCompletion && _iterator.return != null) {
290
- _iterator.return();
291
- }
292
- } finally{
293
- if (_didIteratorError) {
294
- throw _iteratorError;
295
- }
296
- }
297
- }
298
- report = (0, _reportts.buildReport)({
299
- numPages: numPages,
300
- md: md,
301
- statsMd: statsMd,
302
- stats: stats,
303
- bodyH: bodyH,
304
- headingSizes: headingSizes,
305
- hybridPages: hybridPages
306
- });
271
+ collected = _state.sent();
272
+ text = (0, _textanalysists.buildTextAnalysis)(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
273
+ textNotices(collected, new Set(), warn);
274
+ (_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, text.words);
307
275
  return [
308
276
  2,
309
277
  {
310
- markdown: md,
311
- report: report,
278
+ markdown: text.markdown,
279
+ report: text.analysis.report,
312
280
  warnings: warnings
313
281
  }
314
282
  ];
@@ -355,61 +323,171 @@ function extractOcr(_0) {
355
323
  });
356
324
  }).apply(this, arguments);
357
325
  }
358
- function needsOcrFallback(charsPerPage) {
359
- return charsPerPage < _reportts.AUTO_OCR_CHARS_PER_PAGE;
326
+ /** A page needs OCR only when its image signal accompanies an unusable text
327
+ * layer. The caller supplies the image part of that decision. */ function needsPageOcr(chars) {
328
+ return chars < _reportts.AUTO_OCR_CHARS_PER_PAGE;
360
329
  }
361
- function ocrFallbackNotice(charsPerPage) {
362
- return "text layer yields ~".concat(charsPerPage, " chars/page (< ").concat(_reportts.AUTO_OCR_CHARS_PER_PAGE, ") — pages are images of text; falling back to OCR (suppress with --no-ocr)");
330
+ function textNotices(collected, ocrPages, warn) {
331
+ var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
332
+ try {
333
+ for(var _iterator = collected.pages[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
334
+ var page = _step.value;
335
+ if (page.textChars !== 0 || ocrPages.has(page.page)) continue;
336
+ if (page.hasImage) warn("p".concat(page.page, ": no extractable text; page has image content and OCR was not selected"));
337
+ else warn("p".concat(page.page, ": no extractable text; the page may be blank or contain only vector content"));
338
+ }
339
+ } catch (err) {
340
+ _didIteratorError = true;
341
+ _iteratorError = err;
342
+ } finally{
343
+ try {
344
+ if (!_iteratorNormalCompletion && _iterator.return != null) {
345
+ _iterator.return();
346
+ }
347
+ } finally{
348
+ if (_didIteratorError) {
349
+ throw _iteratorError;
350
+ }
351
+ }
352
+ }
353
+ var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
354
+ try {
355
+ for(var _iterator1 = collected.pages[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
356
+ var page1 = _step1.value;
357
+ if (!page1.largeImage || ocrPages.has(page1.page)) continue;
358
+ warn("p".concat(page1.page, ": text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ").concat(page1.page, " --ocr, which can itself introduce errors"));
359
+ }
360
+ } catch (err) {
361
+ _didIteratorError1 = true;
362
+ _iteratorError1 = err;
363
+ } finally{
364
+ try {
365
+ if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
366
+ _iterator1.return();
367
+ }
368
+ } finally{
369
+ if (_didIteratorError1) {
370
+ throw _iteratorError1;
371
+ }
372
+ }
373
+ }
363
374
  }
364
- function pdfToMarkdown(_0) {
375
+ function analysisWords(pages) {
376
+ return pages.flatMap(function(page) {
377
+ return page.words.map(function(word) {
378
+ return _object_spread_props(_object_spread({
379
+ page: page.page,
380
+ text: word.text
381
+ }, word.box), {
382
+ confidence: word.confidence
383
+ });
384
+ });
385
+ });
386
+ }
387
+ function analyzePdfAuto(_0) {
365
388
  return _async_to_generator(function(pdfPath) {
366
- var opts, _opts_onWarning, text, _opts_onWarning1, notice, ocr, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, w;
389
+ var opts, _opts_pages_last, _opts_onWords, _warnCollector, warnings, warn, range, _opts_onWords1, _ref, pages, words, markdown, report, analysis, collected, ocrPageNumbers, _opts_pageMarkers, _opts_onWords2, text, textPageNumbers, text1, textByPage, ocrByPage, ocrWords, ocrRanges, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, page, lastRange, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, range1, _ocrWords, result, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, page1, err, pages1, markdown1, pagePaths, report1, analysis1;
367
390
  var _arguments = arguments;
368
391
  return _ts_generator(this, function(_state) {
369
392
  switch(_state.label){
370
393
  case 0:
371
394
  opts = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
372
- if (opts.ocr) return [
373
- 2,
374
- extractOcr({
375
- pdfPath: pdfPath
376
- }, opts)
377
- ];
378
- return [
379
- 4,
380
- extractText(pdfPath, _object_spread_props(_object_spread({}, opts), {
381
- onWarning: undefined
382
- }))
383
- ];
384
- case 1:
385
- text = _state.sent();
386
- if (!(needsOcrFallback(text.report.charsPerPage) && !opts.noOcr)) return [
395
+ (0, _validatets.validatePages)(opts.pages);
396
+ _warnCollector = warnCollector(opts.onWarning), warnings = _warnCollector.warnings, warn = _warnCollector.warn;
397
+ range = opts.pages ? {
398
+ first: opts.pages.first,
399
+ last: (_opts_pages_last = opts.pages.last) !== null && _opts_pages_last !== void 0 ? _opts_pages_last : opts.pages.first
400
+ } : {};
401
+ if (!opts.ocr) return [
387
402
  3,
388
- 3
403
+ 2
389
404
  ];
390
- notice = ocrFallbackNotice(text.report.charsPerPage);
391
- (_opts_onWarning1 = opts.onWarning) === null || _opts_onWarning1 === void 0 ? void 0 : _opts_onWarning1.call(opts, notice);
392
405
  return [
393
406
  4,
394
- extractOcr({
407
+ (0, _analyzets.buildOcrAnalysisPages)({
395
408
  pdfPath: pdfPath
396
409
  }, _object_spread_props(_object_spread({}, opts), {
397
- fallback: true
398
- }))
410
+ onWords: undefined
411
+ }), warn)
399
412
  ];
400
- case 2:
401
- ocr = _state.sent();
402
- ocr.warnings.unshift(notice);
413
+ case 1:
414
+ _ref = _state.sent(), pages = _ref.pages, words = _ref.words;
415
+ markdown = (0, _renderanalysists.renderPagesToMarkdown)(pages, {
416
+ forcePageMarkers: true,
417
+ pageMarkers: true
418
+ });
419
+ report = (0, _reportts.buildOcrReport)({
420
+ md: markdown,
421
+ words: words
422
+ });
423
+ analysis = {
424
+ outputVersion: report.outputVersion,
425
+ report: report,
426
+ pages: pages
427
+ };
428
+ (_opts_onWords1 = opts.onWords) === null || _opts_onWords1 === void 0 ? void 0 : _opts_onWords1.call(opts, words);
403
429
  return [
404
430
  2,
405
- ocr
431
+ {
432
+ analysis: analysis,
433
+ markdown: markdown,
434
+ warnings: warnings
435
+ }
436
+ ];
437
+ case 2:
438
+ return [
439
+ 4,
440
+ (0, _collectts.collect)(pdfPath, range)
406
441
  ];
407
442
  case 3:
443
+ collected = _state.sent();
444
+ ocrPageNumbers = new Set(!opts.noOcr ? collected.pages.filter(function(page) {
445
+ return page.hasImage && needsPageOcr(page.textChars);
446
+ }).map(function(page) {
447
+ return page.page;
448
+ }) : []);
449
+ if (!ocrPageNumbers.size) {
450
+ ;
451
+ ;
452
+ text = (0, _textanalysists.buildTextAnalysis)(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
453
+ textNotices(collected, ocrPageNumbers, warn);
454
+ (_opts_onWords2 = opts.onWords) === null || _opts_onWords2 === void 0 ? void 0 : _opts_onWords2.call(opts, text.words);
455
+ return [
456
+ 2,
457
+ {
458
+ analysis: text.analysis,
459
+ markdown: text.markdown,
460
+ warnings: warnings
461
+ }
462
+ ];
463
+ }
464
+ textPageNumbers = new Set(collected.pages.filter(function(page) {
465
+ return !ocrPageNumbers.has(page.page);
466
+ }).map(function(page) {
467
+ return page.page;
468
+ }));
469
+ text1 = textPageNumbers.size ? (0, _textanalysists.buildTextAnalysis)((0, _textanalysists.selectCollectedPages)(collected, textPageNumbers), true) : undefined;
470
+ textByPage = new Map(text1 === null || text1 === void 0 ? void 0 : text1.analysis.pages.map(function(page) {
471
+ return [
472
+ page.page,
473
+ page
474
+ ];
475
+ }));
476
+ ocrByPage = new Map();
477
+ ocrWords = [];
478
+ ocrRanges = [];
408
479
  _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
409
480
  try {
410
- for(_iterator = text.warnings[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
411
- w = _step.value;
412
- (_opts_onWarning = opts.onWarning) === null || _opts_onWarning === void 0 ? void 0 : _opts_onWarning.call(opts, w);
481
+ for(_iterator = collected.pages[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
482
+ page = _step.value;
483
+ if (!ocrPageNumbers.has(page.page)) continue;
484
+ warn("p".concat(page.page, ": sparse text layer with image content; using OCR for this page"));
485
+ lastRange = ocrRanges.at(-1);
486
+ if (lastRange && lastRange.last + 1 === page.page) lastRange.last = page.page;
487
+ else ocrRanges.push({
488
+ first: page.page,
489
+ last: page.page
490
+ });
413
491
  }
414
492
  } catch (err) {
415
493
  _didIteratorError = true;
@@ -425,9 +503,158 @@ function pdfToMarkdown(_0) {
425
503
  }
426
504
  }
427
505
  }
506
+ _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
507
+ _state.label = 4;
508
+ case 4:
509
+ _state.trys.push([
510
+ 4,
511
+ 9,
512
+ 10,
513
+ 11
514
+ ]);
515
+ _iterator1 = ocrRanges[Symbol.iterator]();
516
+ _state.label = 5;
517
+ case 5:
518
+ if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
519
+ 3,
520
+ 8
521
+ ];
522
+ range1 = _step1.value;
523
+ return [
524
+ 4,
525
+ (0, _analyzets.buildOcrAnalysisPages)({
526
+ pdfPath: pdfPath
527
+ }, _object_spread_props(_object_spread({}, opts), {
528
+ pages: range1,
529
+ onWords: undefined
530
+ }), warn)
531
+ ];
532
+ case 6:
533
+ result = _state.sent();
534
+ _iteratorNormalCompletion2 = true, _didIteratorError2 = false, _iteratorError2 = undefined;
535
+ try {
536
+ for(_iterator2 = result.pages[Symbol.iterator](); !(_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done); _iteratorNormalCompletion2 = true){
537
+ page1 = _step2.value;
538
+ ocrByPage.set(page1.page, page1);
539
+ }
540
+ } catch (err) {
541
+ _didIteratorError2 = true;
542
+ _iteratorError2 = err;
543
+ } finally{
544
+ try {
545
+ if (!_iteratorNormalCompletion2 && _iterator2.return != null) {
546
+ _iterator2.return();
547
+ }
548
+ } finally{
549
+ if (_didIteratorError2) {
550
+ throw _iteratorError2;
551
+ }
552
+ }
553
+ }
554
+ (_ocrWords = ocrWords).push.apply(_ocrWords, _to_consumable_array(result.words));
555
+ _state.label = 7;
556
+ case 7:
557
+ _iteratorNormalCompletion1 = true;
558
+ return [
559
+ 3,
560
+ 5
561
+ ];
562
+ case 8:
563
+ return [
564
+ 3,
565
+ 11
566
+ ];
567
+ case 9:
568
+ err = _state.sent();
569
+ _didIteratorError1 = true;
570
+ _iteratorError1 = err;
571
+ return [
572
+ 3,
573
+ 11
574
+ ];
575
+ case 10:
576
+ try {
577
+ if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
578
+ _iterator1.return();
579
+ }
580
+ } finally{
581
+ if (_didIteratorError1) {
582
+ throw _iteratorError1;
583
+ }
584
+ }
585
+ return [
586
+ 7
587
+ ];
588
+ case 11:
589
+ textNotices(collected, ocrPageNumbers, warn);
590
+ pages1 = collected.pages.map(function(page) {
591
+ var _ocrByPage_get;
592
+ return (_ocrByPage_get = ocrByPage.get(page.page)) !== null && _ocrByPage_get !== void 0 ? _ocrByPage_get : textByPage.get(page.page);
593
+ });
594
+ markdown1 = (0, _renderanalysists.renderPagesToMarkdown)(pages1, {
595
+ forcePageMarkers: true,
596
+ pageMarkers: true
597
+ });
598
+ pagePaths = new Map(pages1.map(function(page) {
599
+ return [
600
+ page.page,
601
+ page.path
602
+ ];
603
+ }));
604
+ report1 = (0, _reportts.buildOcrReport)({
605
+ md: markdown1,
606
+ fallback: true,
607
+ words: ocrWords,
608
+ pagePaths: pagePaths,
609
+ hybridPages: text1 === null || text1 === void 0 ? void 0 : text1.analysis.report.hybridPages
610
+ });
611
+ if (text1) {
612
+ report1.bodyFontHeight = text1.analysis.report.bodyFontHeight;
613
+ report1.headingSizes = text1.analysis.report.headingSizes;
614
+ report1.listItems = text1.analysis.report.listItems;
615
+ report1.joinedPairs = text1.analysis.report.joinedPairs;
616
+ report1.closedHyphens = text1.analysis.report.closedHyphens;
617
+ report1.hybridPages = text1.analysis.report.hybridPages;
618
+ }
619
+ analysis1 = {
620
+ outputVersion: report1.outputVersion,
621
+ report: report1,
622
+ pages: pages1
623
+ };
624
+ (_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, analysisWords(pages1));
625
+ return [
626
+ 2,
627
+ {
628
+ analysis: analysis1,
629
+ markdown: markdown1,
630
+ warnings: warnings
631
+ }
632
+ ];
633
+ }
634
+ });
635
+ }).apply(this, arguments);
636
+ }
637
+ function pdfToMarkdown(_0) {
638
+ return _async_to_generator(function(pdfPath) {
639
+ var opts, result;
640
+ var _arguments = arguments;
641
+ return _ts_generator(this, function(_state) {
642
+ switch(_state.label){
643
+ case 0:
644
+ opts = _arguments.length > 1 && _arguments[1] !== void 0 ? _arguments[1] : {};
645
+ return [
646
+ 4,
647
+ analyzePdfAuto(pdfPath, opts)
648
+ ];
649
+ case 1:
650
+ result = _state.sent();
428
651
  return [
429
652
  2,
430
- text
653
+ {
654
+ markdown: result.markdown,
655
+ report: result.analysis.report,
656
+ warnings: result.warnings
657
+ }
431
658
  ];
432
659
  }
433
660
  });