@effortlessmotion/pdf-to-md 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -76
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/package.json +2 -8
- package/skills/pdf-to-md/SKILL.md +39 -66
package/README.md
CHANGED
|
@@ -1,116 +1,109 @@
|
|
|
1
|
-
# pdf-to-md
|
|
1
|
+
# @effortlessmotion/pdf-to-md
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
> contract: recognized text is never silently lost, unrecoverable structure is marked rather
|
|
5
|
-
> than guessed, and every claim the output makes is mechanically checkable. When the defaults
|
|
6
|
-
> misread your document, you get the measured knobs, the detectors that point at exactly what to
|
|
7
|
-
> distrust, and the raw analysis to converge on a lossless conversion yourself.
|
|
3
|
+
Convert born-digital and scanned PDFs to Markdown. `pdf-to-md` reads each page's text layer and uses local OCR where needed. It returns a QA report; the OCR path marks text it cannot place reliably for review.
|
|
8
4
|
|
|
9
|
-
|
|
5
|
+
Use it when you need editable Markdown plus evidence about conversion quality. OCR runs locally and makes no cloud calls.
|
|
6
|
+
|
|
7
|
+
## Requirements
|
|
8
|
+
|
|
9
|
+
- Node.js 22.13 or newer
|
|
10
|
+
- English text for the bundled OCR model. Text-layer extraction can preserve other languages.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
10
13
|
|
|
11
14
|
```bash
|
|
12
|
-
npm install pdf-to-md
|
|
15
|
+
npm install @effortlessmotion/pdf-to-md
|
|
13
16
|
```
|
|
14
17
|
|
|
15
|
-
|
|
18
|
+
## Convert a PDF
|
|
16
19
|
|
|
17
|
-
|
|
20
|
+
The package installs the `pdf-to-md` executable:
|
|
18
21
|
|
|
19
22
|
```bash
|
|
20
|
-
npx pdf-to-md book.pdf
|
|
21
|
-
npx pdf-to-md scan.pdf out.md --ocr # force the OCR path
|
|
23
|
+
npx pdf-to-md book.pdf
|
|
22
24
|
```
|
|
23
25
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
This writes `book.md`. Pages with too little embedded text are assessed individually for OCR, so one document can use both extraction paths. The command reports those choices on stderr.
|
|
27
|
+
|
|
28
|
+
Specify an output path or force OCR when needed:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npx pdf-to-md scan.pdf notes.md --ocr
|
|
29
32
|
```
|
|
30
33
|
|
|
31
|
-
|
|
34
|
+
A conversion can produce ordinary Markdown plus visible review markers:
|
|
32
35
|
|
|
33
|
-
|
|
36
|
+
```text
|
|
37
|
+
## DAILY TALLIES
|
|
34
38
|
|
|
35
|
-
|
|
36
|
-
import { pdfToMarkdown } from 'pdf-to-md';
|
|
39
|
+
Every skiff was counted at the north pier.
|
|
37
40
|
|
|
38
|
-
|
|
41
|
+
> [review: unstructured labels] 12 47 380 5 91
|
|
39
42
|
```
|
|
40
43
|
|
|
41
|
-
|
|
44
|
+
The excerpt is illustrative. Review markers preserve text whose structure could not be recovered reliably.
|
|
42
45
|
|
|
43
|
-
|
|
46
|
+
## Library use
|
|
44
47
|
|
|
45
|
-
|
|
48
|
+
```js
|
|
49
|
+
import { pdfToMarkdown } from '@effortlessmotion/pdf-to-md';
|
|
46
50
|
|
|
47
|
-
|
|
51
|
+
const { markdown, report, warnings } = await pdfToMarkdown('book.pdf');
|
|
48
52
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
and exits non-zero on any loss (`MISSING > 0`) — a first-class part of the product, usable as a CI
|
|
54
|
-
gate, not a path into the source tree.
|
|
55
|
-
2. **Invisible corruption is minimized and made detectable.** Right-words-wrong-order failures
|
|
56
|
-
(column braiding, mid-sentence splits) are what the geometry works hardest to prevent; a second
|
|
57
|
-
class — right order, WRONG VALUE (an OCR misread spliced into otherwise word-like prose) — is
|
|
58
|
-
what confidence scoring catches. **Enforced by:** `report.danglingLong` (the order-failure
|
|
59
|
-
detector no output-only check can prove), `report.reviewBlocks` (blocks the geometry couldn't
|
|
60
|
-
order, labeled rather than guessed at), and `report.lowConfidenceWords` /
|
|
61
|
-
`pageStats[].lowConfidenceSample` (words the engine itself scored below confidence 60 — see
|
|
62
|
-
`LOW_CONFIDENCE_THRESHOLD`'s test in `test/unit/review-blocks.test.ts` for the measured derivation).
|
|
63
|
-
3. **Visible misplacement is acceptable.** A fragment in the floats aside, a `> [review: ...]`
|
|
64
|
-
block, or an odd paragraph break is something a caller can see and edit; the two classes above
|
|
65
|
-
are not. **Enforced by:** the `> [floats]` / `> [review: reason]` markers themselves — regular
|
|
66
|
-
enough (`^> \[(floats|review: [^\]]+)\] (.*)$`) for a downstream agent to find and handle
|
|
67
|
-
without guessing — surfaced in `--json` as `report.floats` / `report.reviewBlocks`.
|
|
53
|
+
console.log(markdown);
|
|
54
|
+
console.log(report.path); // "text", "ocr", or "mixed"
|
|
55
|
+
console.warn(warnings);
|
|
56
|
+
```
|
|
68
57
|
|
|
69
|
-
|
|
58
|
+
`pdfToMarkdown()` uses the same text-first, automatic-OCR behavior as the CLI. It returns data and does not write files or exit the process. Conversion failures use `PdfToMdError` where the library assigns a package error code; native filesystem errors can also surface when inputs cannot be read.
|
|
70
59
|
|
|
71
|
-
|
|
60
|
+
Use `extractText()` or `extractOcr()` when you need to choose the path yourself. The [API documentation](https://kmalakoff.github.io/pdf-to-md/) lists the public functions, options and return types.
|
|
72
61
|
|
|
73
|
-
|
|
62
|
+
## Know which path ran
|
|
74
63
|
|
|
75
|
-
|
|
64
|
+
The QA report identifies the extraction path and highlights output that deserves review. Important cases:
|
|
76
65
|
|
|
77
|
-
|
|
66
|
+
- The bundled OCR model recognizes English. Use text-layer extraction for PDFs whose embedded text is in another language.
|
|
67
|
+
- A page can contain both embedded text and a large image with additional text. The text path warns about these hybrid pages so you can compare a targeted OCR pass.
|
|
68
|
+
- `> [floats]` and `> [review: ...]` blocks contain recognized text that the converter could not place confidently. They are intentionally visible.
|
|
69
|
+
- OCR quality depends on the scan. Check low-confidence words and rendered pages before relying on the result.
|
|
78
70
|
|
|
79
|
-
|
|
80
|
-
|---|---|---|
|
|
81
|
-
| `--float-max-words` | fragments with more words than this are body text, not decoration | `3` |
|
|
82
|
-
| `--float-max-width` | fraction of page width above which a fragment is too wide to be decoration | `0.15` |
|
|
83
|
-
| `--float-margin` | how far a fragment's left edge must sit from its column edge to be decoration | `0.008` |
|
|
84
|
-
| `--para-gap` | a vertical gap this many times the line pitch is a paragraph break | `1.6` |
|
|
85
|
-
| `--heading-scale` | lines taller than this times body height are headings | `2.0` |
|
|
86
|
-
| `--col-max-width` | boxes wider than this are excluded from gutter detection | `0.55` |
|
|
87
|
-
| `--col-split` | fractional x of the two-column midpoint | `0.5` |
|
|
88
|
-
| `--line-y-tol` | vertical distance within which two boxes are the same line | `0.006` |
|
|
89
|
-
| `--line-height-ratio` | same-line boxes with a larger height ratio don't merge | `2.4` |
|
|
90
|
-
| `--dpi` | OCR page-render resolution; raise for tiny print or chart/label pages, `>~400` measured worse on ordinary prose pages (`test/unit/sparse-retry.test.ts`) | `288` |
|
|
71
|
+
Use `--json` to print the report:
|
|
91
72
|
|
|
92
|
-
|
|
73
|
+
```bash
|
|
74
|
+
npx pdf-to-md book.pdf --json
|
|
75
|
+
```
|
|
93
76
|
|
|
94
|
-
|
|
77
|
+
See [quality checks and tuning](https://github.com/kmalakoff/pdf-to-md/blob/master/QUALITY.md) for the audit workflow, report fields and OCR controls.
|
|
95
78
|
|
|
96
|
-
##
|
|
79
|
+
## Common CLI options
|
|
97
80
|
|
|
98
|
-
```
|
|
99
|
-
|
|
81
|
+
```text
|
|
82
|
+
pdf-to-md <file.pdf> [out.md] [flags]
|
|
83
|
+
pdf-to-md render <file.pdf> <page>[-<page>]
|
|
84
|
+
pdf-to-md audit <words.jsonl> <file.md>
|
|
100
85
|
```
|
|
101
86
|
|
|
102
|
-
|
|
87
|
+
Common flags:
|
|
88
|
+
|
|
89
|
+
- `--ocr` forces OCR; `--no-ocr` disables automatic OCR fallback.
|
|
90
|
+
- `--pages N[-M]` converts a page range.
|
|
91
|
+
- `--stdout` writes the converted content to stdout.
|
|
92
|
+
- `--json` prints the QA report.
|
|
93
|
+
- `--format md|txt|raw` selects Markdown, plain text or analysis JSON.
|
|
94
|
+
- `--debug-words=FILE` writes source text items or recognized OCR words for `pdf-to-md audit`. Use `--page-markers` when producing Markdown for a page-by-page audit.
|
|
95
|
+
- `--dpi=N` changes OCR render resolution. The default is 288; higher values are not always more accurate.
|
|
96
|
+
|
|
97
|
+
Run `npx pdf-to-md --help` for the complete option list.
|
|
98
|
+
|
|
99
|
+
## Agent skill
|
|
103
100
|
|
|
104
|
-
|
|
101
|
+
An optional skill teaches compatible coding agents how to inspect the report, render suspect pages and run targeted conversions:
|
|
105
102
|
|
|
106
103
|
```bash
|
|
107
|
-
|
|
108
|
-
npm test # synthetic fixtures (ground truth by construction), a fetched+sha256-pinned
|
|
109
|
-
# public corpus (skips offline), real engine runs, plus self-skipping
|
|
110
|
-
# private-document extras when available locally
|
|
111
|
-
npm run prepublishOnly # tsds validate: build + biome + depcheck + docs
|
|
104
|
+
npx skills add kmalakoff/pdf-to-md
|
|
112
105
|
```
|
|
113
106
|
|
|
114
|
-
|
|
107
|
+
## Support and license
|
|
115
108
|
|
|
116
|
-
|
|
109
|
+
Report problems through the [GitHub issue tracker](https://github.com/kmalakoff/pdf-to-md/issues). See [CONTRIBUTING.md](https://github.com/kmalakoff/pdf-to-md/blob/master/CONTRIBUTING.md) to work on the package. The package is licensed under the [MIT License](https://github.com/kmalakoff/pdf-to-md/blob/master/LICENSE).
|
package/dist/cjs/analyze.d.cts
CHANGED
|
@@ -28,7 +28,9 @@ export declare function buildOcrAnalysisPages(input: {
|
|
|
28
28
|
* `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).
|
|
29
29
|
*
|
|
30
30
|
* @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`
|
|
31
|
-
* @param opts see {@link AnalyzeOptions}; for a
|
|
31
|
+
* @param opts see {@link AnalyzeOptions}; for a PDF path, `path` explicitly
|
|
32
|
+
* selects text (default) or OCR. Automatic per-page routing belongs to
|
|
33
|
+
* pdfToMarkdown and the CLI output formats.
|
|
32
34
|
*/
|
|
33
35
|
export declare function analyze(input: string | {
|
|
34
36
|
words: OcrWordInput[];
|
package/dist/cjs/analyze.d.ts
CHANGED
|
@@ -28,7 +28,9 @@ export declare function buildOcrAnalysisPages(input: {
|
|
|
28
28
|
* `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).
|
|
29
29
|
*
|
|
30
30
|
* @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`
|
|
31
|
-
* @param opts see {@link AnalyzeOptions}; for a
|
|
31
|
+
* @param opts see {@link AnalyzeOptions}; for a PDF path, `path` explicitly
|
|
32
|
+
* selects text (default) or OCR. Automatic per-page routing belongs to
|
|
33
|
+
* pdfToMarkdown and the CLI output formats.
|
|
32
34
|
*/
|
|
33
35
|
export declare function analyze(input: string | {
|
|
34
36
|
words: OcrWordInput[];
|
package/dist/cjs/analyze.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr
|
|
2
|
-
// shares with it
|
|
2
|
+
// shares with it. Text analysis uses text-analysis.ts so it captures the same
|
|
3
|
+
// blocks, source runs, and report input as extractText.
|
|
3
4
|
"use strict";
|
|
4
5
|
Object.defineProperty(exports, "__esModule", {
|
|
5
6
|
value: true
|
|
@@ -23,10 +24,9 @@ var _dictts = require("./dict.js");
|
|
|
23
24
|
var _emitts = require("./emit.js");
|
|
24
25
|
var _tesseractts = require("./engines/tesseract.js");
|
|
25
26
|
var _geometryts = require("./geometry.js");
|
|
26
|
-
var _linests = require("./lines.js");
|
|
27
|
-
var _markdownts = require("./markdown.js");
|
|
28
27
|
var _renderanalysists = require("./render-analysis.js");
|
|
29
28
|
var _reportts = require("./report.js");
|
|
29
|
+
var _textanalysists = require("./text-analysis.js");
|
|
30
30
|
var _validatets = require("./validate.js");
|
|
31
31
|
function _array_like_to_array(arr, len) {
|
|
32
32
|
if (len == null || len > arr.length) len = arr.length;
|
|
@@ -293,6 +293,9 @@ function buildOcrAnalysisPages(input, opts, warn) {
|
|
|
293
293
|
words = result.words;
|
|
294
294
|
_state.label = 4;
|
|
295
295
|
case 4:
|
|
296
|
+
words = words.filter(function(word) {
|
|
297
|
+
return word.page >= first && word.page <= last;
|
|
298
|
+
});
|
|
296
299
|
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, words);
|
|
297
300
|
// Stable per-page index, assigned in recognition/replay order — this IS
|
|
298
301
|
// the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).
|
|
@@ -357,6 +360,7 @@ function buildOcrAnalysisPages(input, opts, warn) {
|
|
|
357
360
|
}
|
|
358
361
|
pages.push({
|
|
359
362
|
page: pn1,
|
|
363
|
+
path: 'ocr',
|
|
360
364
|
words: ((_byPage_get1 = byPage.get(pn1)) !== null && _byPage_get1 !== void 0 ? _byPage_get1 : []).map(toAnalysisWord),
|
|
361
365
|
blocks: blocks
|
|
362
366
|
});
|
|
@@ -410,22 +414,9 @@ function analyzeOcr(input, opts) {
|
|
|
410
414
|
});
|
|
411
415
|
})();
|
|
412
416
|
}
|
|
413
|
-
// TEXT path: deliberately NOT routed through the OCR shared-core pattern —
|
|
414
|
-
// buildMarkdown's reflow/hyphen-rejoin/list rules don't fit the 4-type Analysis block union, so this is an independent, intentionally coarser reconstruction (one word/block per visual LINE) — honest, not byte-identical to extractText (see AnalysisPage, src/types.ts).
|
|
415
|
-
// Converts a text-path Line's page-point geometry (src/lines.ts) into the
|
|
416
|
-
// same normalized box contract the OCR path uses (see AnalysisPage.words) — only the coordinate SYSTEM changes; `box.y` anchors on the line's baseline, `box.w` stays 0.
|
|
417
|
-
function textLineBox(l, dims) {
|
|
418
|
-
var width = dims.width, height = dims.height;
|
|
419
|
-
return {
|
|
420
|
-
x: width > 0 ? l.x / width : 0,
|
|
421
|
-
y: height > 0 ? (height - l.y) / height : 0,
|
|
422
|
-
w: 0,
|
|
423
|
-
h: height > 0 ? l.h / height : 0
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
417
|
function analyzeText(pdfPath, opts) {
|
|
427
418
|
return _async_to_generator(function() {
|
|
428
|
-
var _opts_pages_last, _opts_pageMarkers, warn, range,
|
|
419
|
+
var _opts_pages_last, _opts_pageMarkers, _opts_onWords, warn, range, collected, result, hybridPages, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, n, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, page;
|
|
429
420
|
return _ts_generator(this, function(_state) {
|
|
430
421
|
switch(_state.label){
|
|
431
422
|
case 0:
|
|
@@ -443,33 +434,12 @@ function analyzeText(pdfPath, opts) {
|
|
|
443
434
|
(0, _collectts.collect)(pdfPath, range)
|
|
444
435
|
];
|
|
445
436
|
case 1:
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
height: p.height
|
|
453
|
-
}
|
|
454
|
-
];
|
|
455
|
-
}));
|
|
456
|
-
pageLines = pages.map(function(p) {
|
|
457
|
-
return {
|
|
458
|
-
page: p.page,
|
|
459
|
-
lines: (0, _linests.toLines)(p)
|
|
460
|
-
};
|
|
461
|
-
});
|
|
462
|
-
pageMarkers = (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false;
|
|
463
|
-
_buildMarkdown = (0, _markdownts.buildMarkdown)(pageLines, heightChars, {
|
|
464
|
-
pageMarkers: pageMarkers
|
|
465
|
-
}), md = _buildMarkdown.md, stats = _buildMarkdown.stats, bodyH = _buildMarkdown.bodyH, headingSizes = _buildMarkdown.headingSizes;
|
|
466
|
-
statsMd = pageMarkers ? md : (0, _markdownts.buildMarkdown)(pageLines, heightChars, {
|
|
467
|
-
pageMarkers: true
|
|
468
|
-
}).md;
|
|
469
|
-
hybridPages = pages.filter(function(p) {
|
|
470
|
-
return p.largeImage;
|
|
471
|
-
}).map(function(p) {
|
|
472
|
-
return p.page;
|
|
437
|
+
collected = _state.sent();
|
|
438
|
+
result = (0, _textanalysists.buildTextAnalysis)(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
|
|
439
|
+
hybridPages = collected.pages.filter(function(page) {
|
|
440
|
+
return page.largeImage;
|
|
441
|
+
}).map(function(page) {
|
|
442
|
+
return page.page;
|
|
473
443
|
});
|
|
474
444
|
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
475
445
|
try {
|
|
@@ -491,64 +461,32 @@ function analyzeText(pdfPath, opts) {
|
|
|
491
461
|
}
|
|
492
462
|
}
|
|
493
463
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
return {
|
|
518
|
-
text: l.text,
|
|
519
|
-
box: textLineBox(l, dims)
|
|
520
|
-
};
|
|
521
|
-
});
|
|
522
|
-
var blocks = lines.map(function(l, i) {
|
|
523
|
-
var level = levelOf.get(l.h);
|
|
524
|
-
return level !== undefined ? {
|
|
525
|
-
type: 'heading',
|
|
526
|
-
text: l.text,
|
|
527
|
-
level: level,
|
|
528
|
-
wordIndexes: [
|
|
529
|
-
i
|
|
530
|
-
]
|
|
531
|
-
} : {
|
|
532
|
-
type: 'paragraph',
|
|
533
|
-
text: l.text,
|
|
534
|
-
wordIndexes: [
|
|
535
|
-
i
|
|
536
|
-
]
|
|
537
|
-
};
|
|
538
|
-
});
|
|
539
|
-
return {
|
|
540
|
-
page: page,
|
|
541
|
-
words: words,
|
|
542
|
-
blocks: blocks
|
|
543
|
-
};
|
|
544
|
-
});
|
|
464
|
+
_iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
|
|
465
|
+
try {
|
|
466
|
+
for(_iterator1 = collected.pages.filter(function(page) {
|
|
467
|
+
return page.textChars === 0 && !page.hasImage;
|
|
468
|
+
})[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
|
|
469
|
+
page = _step1.value;
|
|
470
|
+
warn("p".concat(page.page, ": no extractable text; the page may be blank or contain only vector content"));
|
|
471
|
+
}
|
|
472
|
+
} catch (err) {
|
|
473
|
+
_didIteratorError1 = true;
|
|
474
|
+
_iteratorError1 = err;
|
|
475
|
+
} finally{
|
|
476
|
+
try {
|
|
477
|
+
if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
|
|
478
|
+
_iterator1.return();
|
|
479
|
+
}
|
|
480
|
+
} finally{
|
|
481
|
+
if (_didIteratorError1) {
|
|
482
|
+
throw _iteratorError1;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, result.words);
|
|
545
487
|
return [
|
|
546
488
|
2,
|
|
547
|
-
|
|
548
|
-
outputVersion: _reportts.OUTPUT_VERSION,
|
|
549
|
-
report: report,
|
|
550
|
-
pages: analysisPages
|
|
551
|
-
}
|
|
489
|
+
result.analysis
|
|
552
490
|
];
|
|
553
491
|
}
|
|
554
492
|
});
|
package/dist/cjs/analyze.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/analyze.ts"],"sourcesContent":["// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr\n// shares with it — same buildPageOCR -> emitPage -> renderPagesToMarkdown pipeline, so OCR-path markdown is byte-identical by construction, not two implementations kept in sync by hand. The TEXT path is NOT shared this way (see analyzeText).\nimport { collect, pageCount } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { EmitBlock } from './emit.ts';\nimport { emitPage, renderEmitBlock } from './emit.ts';\nimport { recognizePdf } from './engines/tesseract.ts';\n// Type-only: erased at compile time, so this does NOT create a runtime\n// circular dependency even though src/extract.ts imports (value-level) from this module.\nimport type { ExtractOcrOptions, ExtractTextOptions, OcrWordInput } from './extract.ts';\nimport type { PageOCR, Tuning, Word } from './geometry.ts';\nimport { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { buildOcrReport, buildReport, OUTPUT_VERSION } from './report.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord, Line, PageLines } from './types.ts';\nimport { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.ts';\n\n/** analyze(pdf) with the default/'text' path — OCR-only fields don't exist\n * on this branch, so passing e.g. `tuning` without `path: 'ocr'` is a compile error. */\nexport type AnalyzeTextOptions = ExtractTextOptions & { path?: 'text' };\n\n/** analyze(pdf, {path:'ocr'}) — same fields as {@link ExtractOcrOptions},\n * unlocked only once `path: 'ocr'` is set. */\nexport type AnalyzeOcrOptions = ExtractOcrOptions & { path: 'ocr' };\n\n/** Options for {@link analyze}: which fields type-check depends on `path`\n * (ignored, forced 'ocr', when `input` is a `{ words }` replay). */\nexport type AnalyzeOptions = AnalyzeTextOptions | AnalyzeOcrOptions;\n\nfunction toAnalysisWord(w: OcrWordInput): AnalysisWord {\n const box = { x: w.x, y: w.y, w: w.w, h: w.h };\n return w.confidence === undefined ? { text: w.text, box } : { text: w.text, box, confidence: w.confidence };\n}\n\n// EmitBlock -> AnalysisBlock: collapses emit.ts's 'quote' kind (a pull-quote)\n// into Analysis's 'paragraph' type, `text` keeping its literal '> ' prefix — see AnalysisBlock's doc comment (src/types.ts) for why.\nfunction toAnalysisBlock(b: EmitBlock, wordIndex: Map<Word, number>): AnalysisBlock {\n const wordIndexes = b.words.map((w) => wordIndex.get(w) as number);\n if (b.kind === 'heading') return { type: 'heading', text: b.text, level: b.level, wordIndexes };\n if (b.kind === 'review') return { type: 'review', text: renderEmitBlock(b), wordIndexes };\n return { type: 'paragraph', text: renderEmitBlock(b), wordIndexes };\n}\n\n/** The OCR-path core shared by extractOcr and analyze(..., {path:'ocr'}):\n * recognizes (or replays) words and runs the unchanged buildPageOCR/emitPage pipeline. Word IDENTITY (object reference), not value, carries indices through into the emitted blocks — see `geometry.ts`'s OcrLine.words. */\nexport async function buildOcrAnalysisPages(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions, warn: (m: string) => void): Promise<{ pages: AnalysisPage[]; words: OcrWordInput[] }> {\n const tuning: Tuning = { ...DEFAULT_TUNING, ...opts.tuning };\n const validated = validateOcrAnalysisInput(input);\n\n let words: OcrWordInput[];\n let first: number;\n let last: number;\n if ('words' in validated) {\n // Replay: page range from --pages if given, else every page up to the highest one the dump mentions.\n words = validated.words;\n first = opts.pages?.first ?? 1;\n last = opts.pages?.last ?? opts.pages?.first ?? Math.max(1, ...words.map((w) => w.page));\n } else {\n const totalPages = await pageCount(validated.pdfPath);\n const rawFirst = opts.pages?.first ?? 1;\n const rawLast = opts.pages?.last ?? (opts.pages ? rawFirst : totalPages);\n first = Math.min(Math.max(1, rawFirst), Math.max(1, totalPages));\n last = Math.min(Math.max(first, rawLast), totalPages);\n const result = await recognizePdf(validated.pdfPath, first, last, {\n dpi: opts.dpi,\n onProgress: opts.onProgress,\n onWarning: warn,\n });\n words = result.words;\n }\n\n opts.onWords?.(words);\n\n // Stable per-page index, assigned in recognition/replay order — this IS\n // the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).\n const byPage = new Map<number, OcrWordInput[]>();\n const wordIndex = new Map<Word, number>();\n for (const w of words) {\n const list = byPage.get(w.page);\n if (list) {\n wordIndex.set(w, list.length);\n list.push(w);\n } else {\n wordIndex.set(w, 0);\n byPage.set(w.page, [w]);\n }\n }\n\n const pageOcrs = new Map<number, PageOCR>();\n for (let pn = first; pn <= last; pn++) pageOcrs.set(pn, buildPageOCR(byPage.get(pn) ?? [], tuning));\n const { levelOf } = headingLevels([...pageOcrs.values()], tuning);\n\n const pages: AnalysisPage[] = [];\n for (let pn = first; pn <= last; pn++) {\n const p = pageOcrs.get(pn) as PageOCR;\n // An in-range page with no recognized words stays visible (marker +\n // warning), never silently absent — same rule as extractOcr.\n if (!byPage.has(pn)) warn(`p${pn}: no recognized words, page emitted with no OCR text`);\n\n const blocks: AnalysisBlock[] = emitPage(p, levelOf, tuning).map((b) => toAnalysisBlock(b, wordIndex));\n // Decorative fragments always render as one visible \"> [floats]\" aside\n // per page (matching the one printed line) — see geometry.ts's PageOCR.floats.\n if (p.floats.length) {\n blocks.push({\n type: 'float',\n text: `> [floats] ${p.floats.map((f) => f.text).join(' · ')}`,\n wordIndexes: p.floats.flatMap((f) => f.words).map((w) => wordIndex.get(w) as number),\n });\n }\n\n pages.push({\n page: pn,\n words: (byPage.get(pn) ?? []).map(toAnalysisWord),\n blocks,\n });\n }\n\n return { pages, words };\n}\n\nasync function analyzeOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md, fallback: opts.fallback, words });\n return { outputVersion: OUTPUT_VERSION, report, pages };\n}\n\n// TEXT path: deliberately NOT routed through the OCR shared-core pattern —\n// buildMarkdown's reflow/hyphen-rejoin/list rules don't fit the 4-type Analysis block union, so this is an independent, intentionally coarser reconstruction (one word/block per visual LINE) — honest, not byte-identical to extractText (see AnalysisPage, src/types.ts).\n\n// Converts a text-path Line's page-point geometry (src/lines.ts) into the\n// same normalized box contract the OCR path uses (see AnalysisPage.words) — only the coordinate SYSTEM changes; `box.y` anchors on the line's baseline, `box.w` stays 0.\nfunction textLineBox(l: Line, dims: { width: number; height: number }): AnalysisWord['box'] {\n const { width, height } = dims;\n return {\n x: width > 0 ? l.x / width : 0,\n y: height > 0 ? (height - l.y) / height : 0,\n w: 0,\n h: height > 0 ? l.h / height : 0,\n };\n}\n\nasync function analyzeText(pdfPath: string, opts: ExtractTextOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const { numPages, pages, heightChars } = await collect(pdfPath, range);\n const pageDims = new Map(pages.map((p) => [p.page, { width: p.width, height: p.height }]));\n const pageLines: PageLines[] = pages.map((p) => ({ page: p.page, lines: toLines(p) }));\n const pageMarkers = opts.pageMarkers ?? false;\n const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, { pageMarkers });\n const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, { pageMarkers: true }).md;\n\n const hybridPages = pages.filter((p) => p.largeImage).map((p) => p.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n\n // Report is measured from the REAL markdown above — accurate regardless of\n // the coarser block/word reconstruction below.\n const report = buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages });\n\n // headingSizes is offset-independent — reused directly, then re-leveled\n // here with offset 0 regardless of `opts.pageMarkers` (see AnalysisPage).\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1)]));\n const analysisPages: AnalysisPage[] = pageLines.map(({ page, lines }) => {\n const dims = pageDims.get(page) as { width: number; height: number };\n const words: AnalysisWord[] = lines.map((l) => ({ text: l.text, box: textLineBox(l, dims) }));\n const blocks: AnalysisBlock[] = lines.map((l, i) => {\n const level = levelOf.get(l.h);\n return level !== undefined ? { type: 'heading' as const, text: l.text, level, wordIndexes: [i] } : { type: 'paragraph' as const, text: l.text, wordIndexes: [i] };\n });\n return { page, words, blocks };\n });\n\n return { outputVersion: OUTPUT_VERSION, report, pages: analysisPages };\n}\n\n/**\n * Analyze a PDF (or replay an already-recognized OCR word dump) into\n * `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).\n *\n * @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`\n * @param opts see {@link AnalyzeOptions}; for a pdf path, `opts.path` picks 'text' (default) or 'ocr' — analyze() makes no auto-fallback decision (that's pdfToMarkdown's job)\n */\nexport async function analyze(input: string | { words: OcrWordInput[] }, opts: AnalyzeOptions = {}): Promise<Analysis> {\n validatePages(opts.pages);\n if (typeof input !== 'string') return analyzeOcr({ words: validateWordsInput(input) }, opts);\n if (opts.path === 'ocr') return analyzeOcr({ pdfPath: input }, opts);\n return analyzeText(input, opts);\n}\n"],"names":["analyze","buildOcrAnalysisPages","toAnalysisWord","w","box","x","y","h","confidence","undefined","text","toAnalysisBlock","b","wordIndex","wordIndexes","words","map","get","kind","type","level","renderEmitBlock","input","opts","warn","byPage","tuning","validated","first","last","Math","totalPages","rawFirst","rawLast","result","list","pageOcrs","pn","levelOf","pages","p","blocks","DEFAULT_TUNING","validateOcrAnalysisInput","max","page","pageCount","pdfPath","min","recognizePdf","dpi","onProgress","onWarning","onWords","Map","set","length","push","buildPageOCR","headingLevels","values","has","emitPage","floats","f","join","flatMap","analyzeOcr","md","report","m","dictAvailable","dictUnavailableWarning","renderPagesToMarkdown","forcePageMarkers","pageMarkers","buildOcrReport","fallback","outputVersion","OUTPUT_VERSION","textLineBox","l","dims","width","height","analyzeText","range","numPages","heightChars","pageDims","pageLines","buildMarkdown","stats","bodyH","headingSizes","statsMd","hybridPages","n","analysisPages","collect","lines","toLines","filter","largeImage","buildReport","slice","i","validatePages","validateWordsInput","path"],"mappings":"AAAA,2EAA2E;AAC3E,kPAAkP;;;;;;;;;;;;QA4L5NA;eAAAA;;QA9IAC;eAAAA;;;yBA7Ca;sBACmB;sBAEZ;2BACb;0BAK+B;uBACpC;0BACM;gCACQ;wBACsB;0BAEgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc5E,SAASC,eAAeC,CAAe;IACrC,IAAMC,MAAM;QAAEC,GAAGF,EAAEE,CAAC;QAAEC,GAAGH,EAAEG,CAAC;QAAEH,GAAGA,EAAEA,CAAC;QAAEI,GAAGJ,EAAEI,CAAC;IAAC;IAC7C,OAAOJ,EAAEK,UAAU,KAAKC,YAAY;QAAEC,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;IAAI,IAAI;QAAEM,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;QAAKI,YAAYL,EAAEK,UAAU;IAAC;AAC5G;AAEA,8EAA8E;AAC9E,qIAAqI;AACrI,SAASG,gBAAgBC,CAAY,EAAEC,SAA4B;IACjE,IAAMC,cAAcF,EAAEG,KAAK,CAACC,GAAG,CAAC,SAACb;eAAMU,UAAUI,GAAG,CAACd;;IACrD,IAAIS,EAAEM,IAAI,KAAK,WAAW,OAAO;QAAEC,MAAM;QAAWT,MAAME,EAAEF,IAAI;QAAEU,OAAOR,EAAEQ,KAAK;QAAEN,aAAAA;IAAY;IAC9F,IAAIF,EAAEM,IAAI,KAAK,UAAU,OAAO;QAAEC,MAAM;QAAUT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;IACxF,OAAO;QAAEK,MAAM;QAAaT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;AACpE;AAIO,SAAeb,sBAAsBqB,KAAsD,EAAEC,IAAuB,EAAEC,IAAyB;;YA4C/EC,aAlBrEF,eAzBMG,QACAC,WAEFZ,OACAa,OACAC,MAK8CC,aAAzCP,cADCA,aACDA,cAAoBA,4BAGVA,cACDA,cAFVQ,YACAC,UACAC,SAGAC,QAYFT,QACAZ,WACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,GACHgC,MAUFC,UACGC,IACDC,SAEFC,OACGF,KAmBGZ,cAlBJe,GAKAC;;;;oBArDFf,SAAiB,mBAAKgB,0BAAc,EAAKnB,KAAKG,MAAM;oBACpDC,YAAYgB,IAAAA,oCAAwB,EAACrB;yBAKvC,CAAA,WAAWK,SAAQ,GAAnB;;;;oBACF,qGAAqG;oBACrGZ,QAAQY,UAAUZ,KAAK;oBACvBa,iBAAQL,cAAAA,KAAKgB,KAAK,cAAVhB,kCAAAA,YAAYK,KAAK,uCAAI;oBAC7BC,QAAON,kBAAAA,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYM,IAAI,0CAAIN,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYK,KAAK,cAArCL,mBAAAA,QAAyCO,CAAAA,QAAAA,MAAKc,GAAG,OAARd,OAAAA;wBAAS;sBAATA,OAAY,qBAAGf,MAAMC,GAAG,CAAC,SAACb;+BAAMA,EAAE0C,IAAI;;;;;;;oBAEnE;;wBAAMC,IAAAA,oBAAS,EAACnB,UAAUoB,OAAO;;;oBAA9ChB,aAAa;oBACbC,qBAAWT,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYK,KAAK,yCAAI;oBAChCK,oBAAUV,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYM,IAAI,yCAAKN,KAAKgB,KAAK,GAAGP,WAAWD;oBAC7DH,QAAQE,KAAKkB,GAAG,CAAClB,KAAKc,GAAG,CAAC,GAAGZ,WAAWF,KAAKc,GAAG,CAAC,GAAGb;oBACpDF,OAAOC,KAAKkB,GAAG,CAAClB,KAAKc,GAAG,CAAChB,OAAOK,UAAUF;oBAC3B;;wBAAMkB,IAAAA,yBAAY,EAACtB,UAAUoB,OAAO,EAAEnB,OAAOC,MAAM;4BAChEqB,KAAK3B,KAAK2B,GAAG;4BACbC,YAAY5B,KAAK4B,UAAU;4BAC3BC,WAAW5B;wBACb;;;oBAJMU,SAAS;oBAKfnB,QAAQmB,OAAOnB,KAAK;;;qBAGtBQ,gBAAAA,KAAK8B,OAAO,cAAZ9B,oCAAAA,mBAAAA,MAAeR;oBAEf,wEAAwE;oBACxE,mJAAmJ;oBAC7IU,SAAS,IAAI6B;oBACbzC,YAAY,IAAIyC;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAWvC,4BAAX,6BAAA,QAAA,yBAAA,iCAAkB;4BAAZZ,IAAN;4BACGgC,OAAOV,OAAOR,GAAG,CAACd,EAAE0C,IAAI;4BAC9B,IAAIV,MAAM;gCACRtB,UAAU0C,GAAG,CAACpD,GAAGgC,KAAKqB,MAAM;gCAC5BrB,KAAKsB,IAAI,CAACtD;4BACZ,OAAO;gCACLU,UAAU0C,GAAG,CAACpD,GAAG;gCACjBsB,OAAO8B,GAAG,CAACpD,EAAE0C,IAAI;oCAAG1C;;4BACtB;wBACF;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWCiC,WAAW,IAAIkB;oBACrB,IAASjB,KAAKT,OAAOS,MAAMR,MAAMQ,KAAMD,SAASmB,GAAG,CAAClB,IAAIqB,IAAAA,wBAAY,GAACjC,cAAAA,OAAOR,GAAG,CAACoB,iBAAXZ,yBAAAA,kBAAsBC;oBACnFY,UAAYqB,IAAAA,yBAAa,EAAE,qBAAGvB,SAASwB,MAAM,KAAKlC,QAAlDY;oBAEFC;oBACN,IAASF,MAAKT,OAAOS,OAAMR,MAAMQ,MAAM;;wBAC/BG,IAAIJ,SAASnB,GAAG,CAACoB;wBACvB,oEAAoE;wBACpE,6DAA6D;wBAC7D,IAAI,CAACZ,OAAOoC,GAAG,CAACxB,MAAKb,KAAK,AAAC,IAAM,OAAHa,KAAG;wBAE3BI,SAA0BqB,IAAAA,gBAAQ,EAACtB,GAAGF,SAASZ,QAAQV,GAAG,CAAC,SAACJ;mCAAMD,gBAAgBC,GAAGC;;wBAC3F,uEAAuE;wBACvE,+EAA+E;wBAC/E,IAAI2B,EAAEuB,MAAM,CAACP,MAAM,EAAE;4BACnBf,OAAOgB,IAAI,CAAC;gCACVtC,MAAM;gCACNT,MAAM,AAAC,cAAqD,OAAxC8B,EAAEuB,MAAM,CAAC/C,GAAG,CAAC,SAACgD;2CAAMA,EAAEtD,IAAI;mCAAEuD,IAAI,CAAC;gCACrDnD,aAAa0B,EAAEuB,MAAM,CAACG,OAAO,CAAC,SAACF;2CAAMA,EAAEjD,KAAK;mCAAEC,GAAG,CAAC,SAACb;2CAAMU,UAAUI,GAAG,CAACd;;4BACzE;wBACF;wBAEAoC,MAAMkB,IAAI,CAAC;4BACTZ,MAAMR;4BACNtB,OAAO,EAACU,eAAAA,OAAOR,GAAG,CAACoB,kBAAXZ,0BAAAA,mBAAsBT,GAAG,CAACd;4BAClCuC,QAAAA;wBACF;oBACF;oBAEA;;wBAAO;4BAAEF,OAAAA;4BAAOxB,OAAAA;wBAAM;;;;IACxB;;AAEA,SAAeoD,WAAW7C,KAAsD,EAAEC,IAAuB;;YACjGC,MAEmB,MAAjBe,OAAOxB,OACTqD,IACAC;;;;oBAJA7C,OAAO,cAAC8C;4BAAc/C;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiB+C;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAI/C,KAAKgD,IAAAA,8BAAsB;oBACxB;;wBAAMvE,sBAAsBqB,OAAOC,MAAMC;;;oBAAzC,OAAA,eAAjBe,QAAiB,KAAjBA,OAAOxB,QAAU,KAAVA;oBACTqD,KAAKK,IAAAA,uCAAqB,EAAClC,OAAO;wBAAEmC,kBAAkB;wBAAMC,aAAa;oBAAK;oBAC9EN,SAASO,IAAAA,wBAAc,EAAC;wBAAER,IAAAA;wBAAIS,UAAUtD,KAAKsD,QAAQ;wBAAE9D,OAAAA;oBAAM;oBACnE;;wBAAO;4BAAE+D,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQ9B,OAAAA;wBAAM;;;;IACxD;;AAEA,2EAA2E;AAC3E,4QAA4Q;AAE5Q,0EAA0E;AAC1E,yKAAyK;AACzK,SAASyC,YAAYC,CAAO,EAAEC,IAAuC;IACnE,IAAQC,QAAkBD,KAAlBC,OAAOC,SAAWF,KAAXE;IACf,OAAO;QACL/E,GAAG8E,QAAQ,IAAIF,EAAE5E,CAAC,GAAG8E,QAAQ;QAC7B7E,GAAG8E,SAAS,IAAI,AAACA,CAAAA,SAASH,EAAE3E,CAAC,AAADA,IAAK8E,SAAS;QAC1CjF,GAAG;QACHI,GAAG6E,SAAS,IAAIH,EAAE1E,CAAC,GAAG6E,SAAS;IACjC;AACF;AAEA,SAAeC,YAAYtC,OAAe,EAAExB,IAAwB;;YAGNA,kBAIxCA,mBANdC,MAEA8D,OACmC,MAAjCC,UAAUhD,OAAOiD,aACnBC,UACAC,WACAf,aACqCgB,gBAAnCvB,IAAIwB,OAAOC,OAAOC,cACpBC,SAEAC,aACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,GAML5B,QAIA/B,SACA4D;;;;oBAtBA1E,OAAO,cAAC8C;4BAAc/C;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiB+C;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAI/C,KAAKgD,IAAAA,8BAAsB;oBAC3Cc,QAAQ/D,KAAKgB,KAAK,GAAG;wBAAEX,OAAOL,KAAKgB,KAAK,CAACX,KAAK;wBAAEC,IAAI,GAAEN,mBAAAA,KAAKgB,KAAK,CAACV,IAAI,cAAfN,8BAAAA,mBAAmBA,KAAKgB,KAAK,CAACX,KAAK;oBAAC,IAAI,CAAC;oBAC5D;;wBAAMuE,IAAAA,kBAAO,EAACpD,SAASuC;;;oBAAvB,OAAA,eAAjCC,WAAiC,KAAjCA,UAAUhD,QAAuB,KAAvBA,OAAOiD,cAAgB,KAAhBA;oBACnBC,WAAW,IAAInC,IAAIf,MAAMvB,GAAG,CAAC,SAACwB;+BAAM;4BAACA,EAAEK,IAAI;4BAAE;gCAAEsC,OAAO3C,EAAE2C,KAAK;gCAAEC,QAAQ5C,EAAE4C,MAAM;4BAAC;yBAAE;;oBAClFM,YAAyBnD,MAAMvB,GAAG,CAAC,SAACwB;+BAAO;4BAAEK,MAAML,EAAEK,IAAI;4BAAEuD,OAAOC,IAAAA,gBAAO,EAAC7D;wBAAG;;oBAC7EmC,eAAcpD,oBAAAA,KAAKoD,WAAW,cAAhBpD,+BAAAA,oBAAoB;oBACGoE,iBAAAA,IAAAA,yBAAa,EAACD,WAAWF,aAAa;wBAAEb,aAAAA;oBAAY,IAAvFP,KAAmCuB,eAAnCvB,IAAIwB,QAA+BD,eAA/BC,OAAOC,QAAwBF,eAAxBE,OAAOC,eAAiBH,eAAjBG;oBACpBC,UAAUpB,cAAcP,KAAKuB,IAAAA,yBAAa,EAACD,WAAWF,aAAa;wBAAEb,aAAa;oBAAK,GAAGP,EAAE;oBAE5F4B,cAAczD,MAAM+D,MAAM,CAAC,SAAC9D;+BAAMA,EAAE+D,UAAU;uBAAEvF,GAAG,CAAC,SAACwB;+BAAMA,EAAEK,IAAI;;oBAClE,kCAAA,2BAAA;;wBAAL,IAAK,YAAWmD,kCAAX,6BAAA,QAAA,yBAAA,iCAAwB;4BAAlBC,IAAN;4BACHzE,KAAK,AAAC,IAA8JyE,OAA3JA,GAAE,2JAA2J,OAAFA,GAAE;wBACxK;;wBAFK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAIL,2EAA2E;oBAC3E,+CAA+C;oBACzC5B,SAASmC,IAAAA,qBAAW,EAAC;wBAAEjB,UAAAA;wBAAUnB,IAAAA;wBAAI2B,SAAAA;wBAASH,OAAAA;wBAAOC,OAAAA;wBAAOC,cAAAA;wBAAcE,aAAAA;oBAAY;oBAE5F,wEAAwE;oBACxE,0EAA0E;oBACpE1D,UAAU,IAAIgB,IAAIwC,aAAaW,KAAK,CAAC,GAAG,GAAGzF,GAAG,CAAC,SAACT,GAAGmG;+BAAM;4BAACnG;4BAAGuB,KAAKkB,GAAG,CAAC,GAAG0D,IAAI;yBAAG;;oBAChFR,gBAAgCR,UAAU1E,GAAG,CAAC;4BAAG6B,aAAAA,MAAMuD,cAAAA;wBAC3D,IAAMlB,OAAOO,SAASxE,GAAG,CAAC4B;wBAC1B,IAAM9B,QAAwBqF,MAAMpF,GAAG,CAAC,SAACiE;mCAAO;gCAAEvE,MAAMuE,EAAEvE,IAAI;gCAAEN,KAAK4E,YAAYC,GAAGC;4BAAM;;wBAC1F,IAAMzC,SAA0B2D,MAAMpF,GAAG,CAAC,SAACiE,GAAGyB;4BAC5C,IAAMtF,QAAQkB,QAAQrB,GAAG,CAACgE,EAAE1E,CAAC;4BAC7B,OAAOa,UAAUX,YAAY;gCAAEU,MAAM;gCAAoBT,MAAMuE,EAAEvE,IAAI;gCAAEU,OAAAA;gCAAON,aAAa;oCAAC4F;iCAAE;4BAAC,IAAI;gCAAEvF,MAAM;gCAAsBT,MAAMuE,EAAEvE,IAAI;gCAAEI,aAAa;oCAAC4F;iCAAE;4BAAC;wBAClK;wBACA,OAAO;4BAAE7D,MAAAA;4BAAM9B,OAAAA;4BAAO0B,QAAAA;wBAAO;oBAC/B;oBAEA;;wBAAO;4BAAEqC,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQ9B,OAAO2D;wBAAc;;;;IACvE;;AASO,SAAelG;wCAAQsB,KAAyC;YAAEC;;;YAAAA,OAAAA,oEAAuB,CAAC;YAC/FoF,IAAAA,yBAAa,EAACpF,KAAKgB,KAAK;YACxB,IAAI,OAAOjB,UAAU,UAAU;;gBAAO6C,WAAW;oBAAEpD,OAAO6F,IAAAA,8BAAkB,EAACtF;gBAAO,GAAGC;;YACvF,IAAIA,KAAKsF,IAAI,KAAK,OAAO;;gBAAO1C,WAAW;oBAAEpB,SAASzB;gBAAM,GAAGC;;YAC/D;;gBAAO8D,YAAY/D,OAAOC;;;IAC5B"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/analyze.ts"],"sourcesContent":["// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr\n// shares with it. Text analysis uses text-analysis.ts so it captures the same\n// blocks, source runs, and report input as extractText.\nimport { collect, pageCount } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { EmitBlock } from './emit.ts';\nimport { emitPage, renderEmitBlock } from './emit.ts';\nimport { recognizePdf } from './engines/tesseract.ts';\n// Type-only: erased at compile time, so this does NOT create a runtime\n// circular dependency even though src/extract.ts imports (value-level) from this module.\nimport type { ExtractOcrOptions, ExtractTextOptions, OcrWordInput } from './extract.ts';\nimport type { PageOCR, Tuning, Word } from './geometry.ts';\nimport { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { buildOcrReport, OUTPUT_VERSION } from './report.ts';\nimport { buildTextAnalysis } from './text-analysis.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord } from './types.ts';\nimport { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.ts';\n\n/** analyze(pdf) with the default/'text' path — OCR-only fields don't exist\n * on this branch, so passing e.g. `tuning` without `path: 'ocr'` is a compile error. */\nexport type AnalyzeTextOptions = ExtractTextOptions & { path?: 'text' };\n\n/** analyze(pdf, {path:'ocr'}) — same fields as {@link ExtractOcrOptions},\n * unlocked only once `path: 'ocr'` is set. */\nexport type AnalyzeOcrOptions = ExtractOcrOptions & { path: 'ocr' };\n\n/** Options for {@link analyze}: which fields type-check depends on `path`\n * (ignored, forced 'ocr', when `input` is a `{ words }` replay). */\nexport type AnalyzeOptions = AnalyzeTextOptions | AnalyzeOcrOptions;\n\nfunction toAnalysisWord(w: OcrWordInput): AnalysisWord {\n const box = { x: w.x, y: w.y, w: w.w, h: w.h };\n return w.confidence === undefined ? { text: w.text, box } : { text: w.text, box, confidence: w.confidence };\n}\n\n// EmitBlock -> AnalysisBlock: collapses emit.ts's 'quote' kind (a pull-quote)\n// into Analysis's 'paragraph' type, `text` keeping its literal '> ' prefix — see AnalysisBlock's doc comment (src/types.ts) for why.\nfunction toAnalysisBlock(b: EmitBlock, wordIndex: Map<Word, number>): AnalysisBlock {\n const wordIndexes = b.words.map((w) => wordIndex.get(w) as number);\n if (b.kind === 'heading') return { type: 'heading', text: b.text, level: b.level, wordIndexes };\n if (b.kind === 'review') return { type: 'review', text: renderEmitBlock(b), wordIndexes };\n return { type: 'paragraph', text: renderEmitBlock(b), wordIndexes };\n}\n\n/** The OCR-path core shared by extractOcr and analyze(..., {path:'ocr'}):\n * recognizes (or replays) words and runs the unchanged buildPageOCR/emitPage pipeline. Word IDENTITY (object reference), not value, carries indices through into the emitted blocks — see `geometry.ts`'s OcrLine.words. */\nexport async function buildOcrAnalysisPages(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions, warn: (m: string) => void): Promise<{ pages: AnalysisPage[]; words: OcrWordInput[] }> {\n const tuning: Tuning = { ...DEFAULT_TUNING, ...opts.tuning };\n const validated = validateOcrAnalysisInput(input);\n\n let words: OcrWordInput[];\n let first: number;\n let last: number;\n if ('words' in validated) {\n // Replay: page range from --pages if given, else every page up to the highest one the dump mentions.\n words = validated.words;\n first = opts.pages?.first ?? 1;\n last = opts.pages?.last ?? opts.pages?.first ?? Math.max(1, ...words.map((w) => w.page));\n } else {\n const totalPages = await pageCount(validated.pdfPath);\n const rawFirst = opts.pages?.first ?? 1;\n const rawLast = opts.pages?.last ?? (opts.pages ? rawFirst : totalPages);\n first = Math.min(Math.max(1, rawFirst), Math.max(1, totalPages));\n last = Math.min(Math.max(first, rawLast), totalPages);\n const result = await recognizePdf(validated.pdfPath, first, last, {\n dpi: opts.dpi,\n onProgress: opts.onProgress,\n onWarning: warn,\n });\n words = result.words;\n }\n\n words = words.filter((word) => word.page >= first && word.page <= last);\n opts.onWords?.(words);\n\n // Stable per-page index, assigned in recognition/replay order — this IS\n // the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).\n const byPage = new Map<number, OcrWordInput[]>();\n const wordIndex = new Map<Word, number>();\n for (const w of words) {\n const list = byPage.get(w.page);\n if (list) {\n wordIndex.set(w, list.length);\n list.push(w);\n } else {\n wordIndex.set(w, 0);\n byPage.set(w.page, [w]);\n }\n }\n\n const pageOcrs = new Map<number, PageOCR>();\n for (let pn = first; pn <= last; pn++) pageOcrs.set(pn, buildPageOCR(byPage.get(pn) ?? [], tuning));\n const { levelOf } = headingLevels([...pageOcrs.values()], tuning);\n\n const pages: AnalysisPage[] = [];\n for (let pn = first; pn <= last; pn++) {\n const p = pageOcrs.get(pn) as PageOCR;\n // An in-range page with no recognized words stays visible (marker +\n // warning), never silently absent — same rule as extractOcr.\n if (!byPage.has(pn)) warn(`p${pn}: no recognized words, page emitted with no OCR text`);\n\n const blocks: AnalysisBlock[] = emitPage(p, levelOf, tuning).map((b) => toAnalysisBlock(b, wordIndex));\n // Decorative fragments always render as one visible \"> [floats]\" aside\n // per page (matching the one printed line) — see geometry.ts's PageOCR.floats.\n if (p.floats.length) {\n blocks.push({\n type: 'float',\n text: `> [floats] ${p.floats.map((f) => f.text).join(' · ')}`,\n wordIndexes: p.floats.flatMap((f) => f.words).map((w) => wordIndex.get(w) as number),\n });\n }\n\n pages.push({\n page: pn,\n path: 'ocr',\n words: (byPage.get(pn) ?? []).map(toAnalysisWord),\n blocks,\n });\n }\n\n return { pages, words };\n}\n\nasync function analyzeOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md, fallback: opts.fallback, words });\n return { outputVersion: OUTPUT_VERSION, report, pages };\n}\n\nasync function analyzeText(pdfPath: string, opts: ExtractTextOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const collected = await collect(pdfPath, range);\n const result = buildTextAnalysis(collected, opts.pageMarkers ?? false);\n const hybridPages = collected.pages.filter((page) => page.largeImage).map((page) => page.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n for (const page of collected.pages.filter((page) => page.textChars === 0 && !page.hasImage)) {\n warn(`p${page.page}: no extractable text; the page may be blank or contain only vector content`);\n }\n opts.onWords?.(result.words);\n return result.analysis;\n}\n\n/**\n * Analyze a PDF (or replay an already-recognized OCR word dump) into\n * `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).\n *\n * @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`\n * @param opts see {@link AnalyzeOptions}; for a PDF path, `path` explicitly\n * selects text (default) or OCR. Automatic per-page routing belongs to\n * pdfToMarkdown and the CLI output formats.\n */\nexport async function analyze(input: string | { words: OcrWordInput[] }, opts: AnalyzeOptions = {}): Promise<Analysis> {\n validatePages(opts.pages);\n if (typeof input !== 'string') return analyzeOcr({ words: validateWordsInput(input) }, opts);\n if (opts.path === 'ocr') return analyzeOcr({ pdfPath: input }, opts);\n return analyzeText(input, opts);\n}\n"],"names":["analyze","buildOcrAnalysisPages","toAnalysisWord","w","box","x","y","h","confidence","undefined","text","toAnalysisBlock","b","wordIndex","wordIndexes","words","map","get","kind","type","level","renderEmitBlock","input","opts","warn","byPage","tuning","validated","first","last","Math","totalPages","rawFirst","rawLast","result","list","pageOcrs","pn","levelOf","pages","p","blocks","DEFAULT_TUNING","validateOcrAnalysisInput","max","page","pageCount","pdfPath","min","recognizePdf","dpi","onProgress","onWarning","filter","word","onWords","Map","set","length","push","buildPageOCR","headingLevels","values","has","emitPage","floats","f","join","flatMap","path","analyzeOcr","md","report","m","dictAvailable","dictUnavailableWarning","renderPagesToMarkdown","forcePageMarkers","pageMarkers","buildOcrReport","fallback","outputVersion","OUTPUT_VERSION","analyzeText","range","collected","hybridPages","n","collect","buildTextAnalysis","largeImage","textChars","hasImage","analysis","validatePages","validateWordsInput"],"mappings":"AAAA,2EAA2E;AAC3E,8EAA8E;AAC9E,wDAAwD;;;;;;;;;;;;QA6JlCA;eAAAA;;QAhHAC;eAAAA;;;yBA5Ca;sBACmB;sBAEZ;2BACb;0BAK+B;gCACtB;wBACS;8BACb;0BAE0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc5E,SAASC,eAAeC,CAAe;IACrC,IAAMC,MAAM;QAAEC,GAAGF,EAAEE,CAAC;QAAEC,GAAGH,EAAEG,CAAC;QAAEH,GAAGA,EAAEA,CAAC;QAAEI,GAAGJ,EAAEI,CAAC;IAAC;IAC7C,OAAOJ,EAAEK,UAAU,KAAKC,YAAY;QAAEC,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;IAAI,IAAI;QAAEM,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;QAAKI,YAAYL,EAAEK,UAAU;IAAC;AAC5G;AAEA,8EAA8E;AAC9E,qIAAqI;AACrI,SAASG,gBAAgBC,CAAY,EAAEC,SAA4B;IACjE,IAAMC,cAAcF,EAAEG,KAAK,CAACC,GAAG,CAAC,SAACb;eAAMU,UAAUI,GAAG,CAACd;;IACrD,IAAIS,EAAEM,IAAI,KAAK,WAAW,OAAO;QAAEC,MAAM;QAAWT,MAAME,EAAEF,IAAI;QAAEU,OAAOR,EAAEQ,KAAK;QAAEN,aAAAA;IAAY;IAC9F,IAAIF,EAAEM,IAAI,KAAK,UAAU,OAAO;QAAEC,MAAM;QAAUT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;IACxF,OAAO;QAAEK,MAAM;QAAaT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;AACpE;AAIO,SAAeb,sBAAsBqB,KAAsD,EAAEC,IAAuB,EAAEC,IAAyB;;YA6C/EC,aAlBrEF,eA1BMG,QACAC,WAEFZ,OACAa,OACAC,MAK8CC,aAAzCP,cADCA,aACDA,cAAoBA,4BAGVA,cACDA,cAFVQ,YACAC,UACAC,SAGAC,QAaFT,QACAZ,WACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,GACHgC,MAUFC,UACGC,IACDC,SAEFC,OACGF,KAoBGZ,cAnBJe,GAKAC;;;;oBAtDFf,SAAiB,mBAAKgB,0BAAc,EAAKnB,KAAKG,MAAM;oBACpDC,YAAYgB,IAAAA,oCAAwB,EAACrB;yBAKvC,CAAA,WAAWK,SAAQ,GAAnB;;;;oBACF,qGAAqG;oBACrGZ,QAAQY,UAAUZ,KAAK;oBACvBa,iBAAQL,cAAAA,KAAKgB,KAAK,cAAVhB,kCAAAA,YAAYK,KAAK,uCAAI;oBAC7BC,QAAON,kBAAAA,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYM,IAAI,0CAAIN,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYK,KAAK,cAArCL,mBAAAA,QAAyCO,CAAAA,QAAAA,MAAKc,GAAG,OAARd,OAAAA;wBAAS;sBAATA,OAAY,qBAAGf,MAAMC,GAAG,CAAC,SAACb;+BAAMA,EAAE0C,IAAI;;;;;;;oBAEnE;;wBAAMC,IAAAA,oBAAS,EAACnB,UAAUoB,OAAO;;;oBAA9ChB,aAAa;oBACbC,qBAAWT,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYK,KAAK,yCAAI;oBAChCK,oBAAUV,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYM,IAAI,yCAAKN,KAAKgB,KAAK,GAAGP,WAAWD;oBAC7DH,QAAQE,KAAKkB,GAAG,CAAClB,KAAKc,GAAG,CAAC,GAAGZ,WAAWF,KAAKc,GAAG,CAAC,GAAGb;oBACpDF,OAAOC,KAAKkB,GAAG,CAAClB,KAAKc,GAAG,CAAChB,OAAOK,UAAUF;oBAC3B;;wBAAMkB,IAAAA,yBAAY,EAACtB,UAAUoB,OAAO,EAAEnB,OAAOC,MAAM;4BAChEqB,KAAK3B,KAAK2B,GAAG;4BACbC,YAAY5B,KAAK4B,UAAU;4BAC3BC,WAAW5B;wBACb;;;oBAJMU,SAAS;oBAKfnB,QAAQmB,OAAOnB,KAAK;;;oBAGtBA,QAAQA,MAAMsC,MAAM,CAAC,SAACC;+BAASA,KAAKT,IAAI,IAAIjB,SAAS0B,KAAKT,IAAI,IAAIhB;;qBAClEN,gBAAAA,KAAKgC,OAAO,cAAZhC,oCAAAA,mBAAAA,MAAeR;oBAEf,wEAAwE;oBACxE,mJAAmJ;oBAC7IU,SAAS,IAAI+B;oBACb3C,YAAY,IAAI2C;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAWzC,4BAAX,6BAAA,QAAA,yBAAA,iCAAkB;4BAAZZ,IAAN;4BACGgC,OAAOV,OAAOR,GAAG,CAACd,EAAE0C,IAAI;4BAC9B,IAAIV,MAAM;gCACRtB,UAAU4C,GAAG,CAACtD,GAAGgC,KAAKuB,MAAM;gCAC5BvB,KAAKwB,IAAI,CAACxD;4BACZ,OAAO;gCACLU,UAAU4C,GAAG,CAACtD,GAAG;gCACjBsB,OAAOgC,GAAG,CAACtD,EAAE0C,IAAI;oCAAG1C;;4BACtB;wBACF;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWCiC,WAAW,IAAIoB;oBACrB,IAASnB,KAAKT,OAAOS,MAAMR,MAAMQ,KAAMD,SAASqB,GAAG,CAACpB,IAAIuB,IAAAA,wBAAY,GAACnC,cAAAA,OAAOR,GAAG,CAACoB,iBAAXZ,yBAAAA,kBAAsBC;oBACnFY,UAAYuB,IAAAA,yBAAa,EAAE,qBAAGzB,SAAS0B,MAAM,KAAKpC,QAAlDY;oBAEFC;oBACN,IAASF,MAAKT,OAAOS,OAAMR,MAAMQ,MAAM;;wBAC/BG,IAAIJ,SAASnB,GAAG,CAACoB;wBACvB,oEAAoE;wBACpE,6DAA6D;wBAC7D,IAAI,CAACZ,OAAOsC,GAAG,CAAC1B,MAAKb,KAAK,AAAC,IAAM,OAAHa,KAAG;wBAE3BI,SAA0BuB,IAAAA,gBAAQ,EAACxB,GAAGF,SAASZ,QAAQV,GAAG,CAAC,SAACJ;mCAAMD,gBAAgBC,GAAGC;;wBAC3F,uEAAuE;wBACvE,+EAA+E;wBAC/E,IAAI2B,EAAEyB,MAAM,CAACP,MAAM,EAAE;4BACnBjB,OAAOkB,IAAI,CAAC;gCACVxC,MAAM;gCACNT,MAAM,AAAC,cAAqD,OAAxC8B,EAAEyB,MAAM,CAACjD,GAAG,CAAC,SAACkD;2CAAMA,EAAExD,IAAI;mCAAEyD,IAAI,CAAC;gCACrDrD,aAAa0B,EAAEyB,MAAM,CAACG,OAAO,CAAC,SAACF;2CAAMA,EAAEnD,KAAK;mCAAEC,GAAG,CAAC,SAACb;2CAAMU,UAAUI,GAAG,CAACd;;4BACzE;wBACF;wBAEAoC,MAAMoB,IAAI,CAAC;4BACTd,MAAMR;4BACNgC,MAAM;4BACNtD,OAAO,EAACU,eAAAA,OAAOR,GAAG,CAACoB,kBAAXZ,0BAAAA,mBAAsBT,GAAG,CAACd;4BAClCuC,QAAAA;wBACF;oBACF;oBAEA;;wBAAO;4BAAEF,OAAAA;4BAAOxB,OAAAA;wBAAM;;;;IACxB;;AAEA,SAAeuD,WAAWhD,KAAsD,EAAEC,IAAuB;;YACjGC,MAEmB,MAAjBe,OAAOxB,OACTwD,IACAC;;;;oBAJAhD,OAAO,cAACiD;4BAAclD;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiBkD;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAIlD,KAAKmD,IAAAA,8BAAsB;oBACxB;;wBAAM1E,sBAAsBqB,OAAOC,MAAMC;;;oBAAzC,OAAA,eAAjBe,QAAiB,KAAjBA,OAAOxB,QAAU,KAAVA;oBACTwD,KAAKK,IAAAA,uCAAqB,EAACrC,OAAO;wBAAEsC,kBAAkB;wBAAMC,aAAa;oBAAK;oBAC9EN,SAASO,IAAAA,wBAAc,EAAC;wBAAER,IAAAA;wBAAIS,UAAUzD,KAAKyD,QAAQ;wBAAEjE,OAAAA;oBAAM;oBACnE;;wBAAO;4BAAEkE,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQjC,OAAAA;wBAAM;;;;IACxD;;AAEA,SAAe4C,YAAYpC,OAAe,EAAExB,IAAwB;;YAGNA,kBAEhBA,mBAQ5CA,eAZMC,MAEA4D,OACAC,WACAnD,QACAoD,aACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,GAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAM1C;;;;oBATLrB,OAAO,cAACiD;4BAAclD;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiBkD;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAIlD,KAAKmD,IAAAA,8BAAsB;oBAC3CS,QAAQ7D,KAAKgB,KAAK,GAAG;wBAAEX,OAAOL,KAAKgB,KAAK,CAACX,KAAK;wBAAEC,IAAI,GAAEN,mBAAAA,KAAKgB,KAAK,CAACV,IAAI,cAAfN,8BAAAA,mBAAmBA,KAAKgB,KAAK,CAACX,KAAK;oBAAC,IAAI,CAAC;oBACnF;;wBAAM4D,IAAAA,kBAAO,EAACzC,SAASqC;;;oBAAnCC,YAAY;oBACZnD,SAASuD,IAAAA,iCAAiB,EAACJ,YAAW9D,oBAAAA,KAAKuD,WAAW,cAAhBvD,+BAAAA,oBAAoB;oBAC1D+D,cAAcD,UAAU9C,KAAK,CAACc,MAAM,CAAC,SAACR;+BAASA,KAAK6C,UAAU;uBAAE1E,GAAG,CAAC,SAAC6B;+BAASA,KAAKA,IAAI;;oBACxF,kCAAA,2BAAA;;wBAAL,IAAK,YAAWyC,kCAAX,6BAAA,QAAA,yBAAA,iCAAwB;4BAAlBC,IAAN;4BACH/D,KAAK,AAAC,IAA8J+D,OAA3JA,GAAE,2JAA2J,OAAFA,GAAE;wBACxK;;wBAFK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAGA,mCAAA,4BAAA;;wBAAL,IAAK,aAAcF,UAAU9C,KAAK,CAACc,MAAM,CAAC,SAACR;mCAASA,KAAK8C,SAAS,KAAK,KAAK,CAAC9C,KAAK+C,QAAQ;iDAArF,8BAAA,SAAA,0BAAA,kCAAwF;4BAAlF/C,OAAN;4BACHrB,KAAK,AAAC,IAAa,OAAVqB,KAAKA,IAAI,EAAC;wBACrB;;wBAFK;wBAAA;;;iCAAA,8BAAA;gCAAA;;;gCAAA;sCAAA;;;;qBAGLtB,gBAAAA,KAAKgC,OAAO,cAAZhC,oCAAAA,mBAAAA,MAAeW,OAAOnB,KAAK;oBAC3B;;wBAAOmB,OAAO2D,QAAQ;;;;IACxB;;AAWO,SAAe7F;wCAAQsB,KAAyC;YAAEC;;;YAAAA,OAAAA,oEAAuB,CAAC;YAC/FuE,IAAAA,yBAAa,EAACvE,KAAKgB,KAAK;YACxB,IAAI,OAAOjB,UAAU,UAAU;;gBAAOgD,WAAW;oBAAEvD,OAAOgF,IAAAA,8BAAkB,EAACzE;gBAAO,GAAGC;;YACvF,IAAIA,KAAK8C,IAAI,KAAK,OAAO;;gBAAOC,WAAW;oBAAEvB,SAASzB;gBAAM,GAAGC;;YAC/D;;gBAAO4D,YAAY7D,OAAOC;;;IAC5B"}
|
package/dist/cjs/audit.d.cts
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
export interface AuditResult {
|
|
3
3
|
/** JSON lines successfully parsed from the word dump */
|
|
4
4
|
parsed: number;
|
|
5
|
-
/** lines in the dump that were not parseable JSON
|
|
6
|
-
*
|
|
5
|
+
/** lines in the dump that were not parseable JSON; auditWords rejects
|
|
6
|
+
* evidence with any such line */
|
|
7
7
|
nonJsonSkipped: number;
|
|
8
8
|
/** distinct page numbers present in the dump */
|
|
9
9
|
pages: number;
|
|
@@ -13,7 +13,7 @@ export interface AuditResult {
|
|
|
13
13
|
* hyphen-join/split repair is applied; always `>= missing` */
|
|
14
14
|
strictDeficits: number;
|
|
15
15
|
/** words genuinely absent from the markdown after repair-checking; `0` is
|
|
16
|
-
* the only passing value (`pdf-to-md audit` exits
|
|
16
|
+
* the only passing value (`pdf-to-md audit` exits nonzero otherwise) */
|
|
17
17
|
missing: number;
|
|
18
18
|
/** per-page detail for every page with at least one missing word (capped
|
|
19
19
|
* sample of the missing tokens per page) */
|
|
@@ -29,8 +29,9 @@ export interface AuditResult {
|
|
|
29
29
|
lines: string[];
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
|
-
* Audit a --debug-words dump against a
|
|
33
|
-
* must land
|
|
32
|
+
* Audit a --debug-words dump against a Markdown file. Every recognized word
|
|
33
|
+
* must land on its page. Invalid evidence or absent page provenance throws a
|
|
34
|
+
* PdfToMdError with code `AUDIT_INPUT`.
|
|
34
35
|
*
|
|
35
36
|
* @param wordsPath path to a --debug-words JSON-lines dump (`{page, text, ...}` per line)
|
|
36
37
|
* @param mdPath path to the markdown file to check the dump against
|
package/dist/cjs/audit.d.ts
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
export interface AuditResult {
|
|
3
3
|
/** JSON lines successfully parsed from the word dump */
|
|
4
4
|
parsed: number;
|
|
5
|
-
/** lines in the dump that were not parseable JSON
|
|
6
|
-
*
|
|
5
|
+
/** lines in the dump that were not parseable JSON; auditWords rejects
|
|
6
|
+
* evidence with any such line */
|
|
7
7
|
nonJsonSkipped: number;
|
|
8
8
|
/** distinct page numbers present in the dump */
|
|
9
9
|
pages: number;
|
|
@@ -13,7 +13,7 @@ export interface AuditResult {
|
|
|
13
13
|
* hyphen-join/split repair is applied; always `>= missing` */
|
|
14
14
|
strictDeficits: number;
|
|
15
15
|
/** words genuinely absent from the markdown after repair-checking; `0` is
|
|
16
|
-
* the only passing value (`pdf-to-md audit` exits
|
|
16
|
+
* the only passing value (`pdf-to-md audit` exits nonzero otherwise) */
|
|
17
17
|
missing: number;
|
|
18
18
|
/** per-page detail for every page with at least one missing word (capped
|
|
19
19
|
* sample of the missing tokens per page) */
|
|
@@ -29,8 +29,9 @@ export interface AuditResult {
|
|
|
29
29
|
lines: string[];
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
|
-
* Audit a --debug-words dump against a
|
|
33
|
-
* must land
|
|
32
|
+
* Audit a --debug-words dump against a Markdown file. Every recognized word
|
|
33
|
+
* must land on its page. Invalid evidence or absent page provenance throws a
|
|
34
|
+
* PdfToMdError with code `AUDIT_INPUT`.
|
|
34
35
|
*
|
|
35
36
|
* @param wordsPath path to a --debug-words JSON-lines dump (`{page, text, ...}` per line)
|
|
36
37
|
* @param mdPath path to the markdown file to check the dump against
|