@effortlessmotion/pdf-to-md 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/analyze.js +8 -45
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/cli/audit.js +34 -74
- package/dist/cjs/cli/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +40 -99
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/cli/render.js +23 -67
- package/dist/cjs/cli/render.js.map +1 -1
- package/dist/esm/analyze.js +1 -1
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/cli/audit.js +1 -3
- package/dist/esm/cli/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +3 -6
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/cli/render.js +2 -3
- package/dist/esm/cli/render.js.map +1 -1
- package/package.json +1 -1
package/dist/cjs/analyze.js
CHANGED
|
@@ -21,6 +21,7 @@ _export(exports, {
|
|
|
21
21
|
var _collectts = require("./collect.js");
|
|
22
22
|
var _dictts = require("./dict.js");
|
|
23
23
|
var _emitts = require("./emit.js");
|
|
24
|
+
var _tesseractts = require("./engines/tesseract.js");
|
|
24
25
|
var _geometryts = require("./geometry.js");
|
|
25
26
|
var _linests = require("./lines.js");
|
|
26
27
|
var _markdownts = require("./markdown.js");
|
|
@@ -72,36 +73,6 @@ function _define_property(obj, key, value) {
|
|
|
72
73
|
} else obj[key] = value;
|
|
73
74
|
return obj;
|
|
74
75
|
}
|
|
75
|
-
function _getRequireWildcardCache(nodeInterop) {
|
|
76
|
-
if (typeof WeakMap !== "function") return null;
|
|
77
|
-
var cacheBabelInterop = new WeakMap();
|
|
78
|
-
var cacheNodeInterop = new WeakMap();
|
|
79
|
-
return (_getRequireWildcardCache = function(nodeInterop) {
|
|
80
|
-
return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
|
|
81
|
-
})(nodeInterop);
|
|
82
|
-
}
|
|
83
|
-
function _interop_require_wildcard(obj, nodeInterop) {
|
|
84
|
-
if (!nodeInterop && obj && obj.__esModule) return obj;
|
|
85
|
-
if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
|
|
86
|
-
default: obj
|
|
87
|
-
};
|
|
88
|
-
var cache = _getRequireWildcardCache(nodeInterop);
|
|
89
|
-
if (cache && cache.has(obj)) return cache.get(obj);
|
|
90
|
-
var newObj = {
|
|
91
|
-
__proto__: null
|
|
92
|
-
};
|
|
93
|
-
var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
|
|
94
|
-
for(var key in obj){
|
|
95
|
-
if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
96
|
-
var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
|
|
97
|
-
if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
|
|
98
|
-
else newObj[key] = obj[key];
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
newObj.default = obj;
|
|
102
|
-
if (cache) cache.set(obj, newObj);
|
|
103
|
-
return newObj;
|
|
104
|
-
}
|
|
105
76
|
function _iterable_to_array(iter) {
|
|
106
77
|
if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
|
|
107
78
|
return Array.from(iter);
|
|
@@ -276,7 +247,7 @@ function toAnalysisBlock(b, wordIndex) {
|
|
|
276
247
|
}
|
|
277
248
|
function buildOcrAnalysisPages(input, opts, warn) {
|
|
278
249
|
return _async_to_generator(function() {
|
|
279
|
-
var _byPage_get, _opts_onWords, tuning, validated, words, first, last, _Math, _ref, _ref1, _ref2, _opts_pages, _opts_pages1, _opts_pages2, _ref3, _ref4, _opts_pages3, _opts_pages4,
|
|
250
|
+
var _byPage_get, _opts_onWords, tuning, validated, words, first, last, _Math, _ref, _ref1, _ref2, _opts_pages, _opts_pages1, _opts_pages2, _ref3, _ref4, _opts_pages3, _opts_pages4, totalPages, rawFirst, rawLast, result, byPage, wordIndex, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, w, list, pageOcrs, pn, levelOf, pages, pn1, _byPage_get1, p, blocks;
|
|
280
251
|
return _ts_generator(this, function(_state) {
|
|
281
252
|
switch(_state.label){
|
|
282
253
|
case 0:
|
|
@@ -296,22 +267,14 @@ function buildOcrAnalysisPages(input, opts, warn) {
|
|
|
296
267
|
}))));
|
|
297
268
|
return [
|
|
298
269
|
3,
|
|
299
|
-
|
|
270
|
+
4
|
|
300
271
|
];
|
|
301
272
|
case 1:
|
|
302
|
-
return [
|
|
303
|
-
4,
|
|
304
|
-
Promise.resolve().then(function() {
|
|
305
|
-
return /*#__PURE__*/ _interop_require_wildcard(require("./engines/tesseract.js"));
|
|
306
|
-
})
|
|
307
|
-
];
|
|
308
|
-
case 2:
|
|
309
|
-
recognizePdf = _state.sent().recognizePdf;
|
|
310
273
|
return [
|
|
311
274
|
4,
|
|
312
275
|
(0, _collectts.pageCount)(validated.pdfPath)
|
|
313
276
|
];
|
|
314
|
-
case
|
|
277
|
+
case 2:
|
|
315
278
|
totalPages = _state.sent();
|
|
316
279
|
rawFirst = (_ref3 = (_opts_pages3 = opts.pages) === null || _opts_pages3 === void 0 ? void 0 : _opts_pages3.first) !== null && _ref3 !== void 0 ? _ref3 : 1;
|
|
317
280
|
rawLast = (_ref4 = (_opts_pages4 = opts.pages) === null || _opts_pages4 === void 0 ? void 0 : _opts_pages4.last) !== null && _ref4 !== void 0 ? _ref4 : opts.pages ? rawFirst : totalPages;
|
|
@@ -319,17 +282,17 @@ function buildOcrAnalysisPages(input, opts, warn) {
|
|
|
319
282
|
last = Math.min(Math.max(first, rawLast), totalPages);
|
|
320
283
|
return [
|
|
321
284
|
4,
|
|
322
|
-
recognizePdf(validated.pdfPath, first, last, {
|
|
285
|
+
(0, _tesseractts.recognizePdf)(validated.pdfPath, first, last, {
|
|
323
286
|
dpi: opts.dpi,
|
|
324
287
|
onProgress: opts.onProgress,
|
|
325
288
|
onWarning: warn
|
|
326
289
|
})
|
|
327
290
|
];
|
|
328
|
-
case
|
|
291
|
+
case 3:
|
|
329
292
|
result = _state.sent();
|
|
330
293
|
words = result.words;
|
|
331
|
-
_state.label =
|
|
332
|
-
case
|
|
294
|
+
_state.label = 4;
|
|
295
|
+
case 4:
|
|
333
296
|
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, words);
|
|
334
297
|
// Stable per-page index, assigned in recognition/replay order — this IS
|
|
335
298
|
// the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).
|
package/dist/cjs/analyze.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/analyze.ts"],"sourcesContent":["// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr\n// shares with it — same buildPageOCR -> emitPage -> renderPagesToMarkdown pipeline, so OCR-path markdown is byte-identical by construction, not two implementations kept in sync by hand. The TEXT path is NOT shared this way (see analyzeText).\nimport { collect, pageCount } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { EmitBlock } from './emit.ts';\nimport { emitPage, renderEmitBlock } from './emit.ts';\n// Type-only: erased at compile time, so this does NOT create a runtime\n// circular dependency even though src/extract.ts imports (value-level) from this module.\nimport type { ExtractOcrOptions, ExtractTextOptions, OcrWordInput } from './extract.ts';\nimport type { PageOCR, Tuning, Word } from './geometry.ts';\nimport { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { buildOcrReport, buildReport, OUTPUT_VERSION } from './report.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord, Line, PageLines } from './types.ts';\nimport { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.ts';\n\n/** analyze(pdf) with the default/'text' path — OCR-only fields don't exist\n * on this branch, so passing e.g. `tuning` without `path: 'ocr'` is a compile error. */\nexport type AnalyzeTextOptions = ExtractTextOptions & { path?: 'text' };\n\n/** analyze(pdf, {path:'ocr'}) — same fields as {@link ExtractOcrOptions},\n * unlocked only once `path: 'ocr'` is set. */\nexport type AnalyzeOcrOptions = ExtractOcrOptions & { path: 'ocr' };\n\n/** Options for {@link analyze}: which fields type-check depends on `path`\n * (ignored, forced 'ocr', when `input` is a `{ words }` replay). */\nexport type AnalyzeOptions = AnalyzeTextOptions | AnalyzeOcrOptions;\n\nfunction toAnalysisWord(w: OcrWordInput): AnalysisWord {\n const box = { x: w.x, y: w.y, w: w.w, h: w.h };\n return w.confidence === undefined ? { text: w.text, box } : { text: w.text, box, confidence: w.confidence };\n}\n\n// EmitBlock -> AnalysisBlock: collapses emit.ts's 'quote' kind (a pull-quote)\n// into Analysis's 'paragraph' type, `text` keeping its literal '> ' prefix — see AnalysisBlock's doc comment (src/types.ts) for why.\nfunction toAnalysisBlock(b: EmitBlock, wordIndex: Map<Word, number>): AnalysisBlock {\n const wordIndexes = b.words.map((w) => wordIndex.get(w) as number);\n if (b.kind === 'heading') return { type: 'heading', text: b.text, level: b.level, wordIndexes };\n if (b.kind === 'review') return { type: 'review', text: renderEmitBlock(b), wordIndexes };\n return { type: 'paragraph', text: renderEmitBlock(b), wordIndexes };\n}\n\n/** The OCR-path core shared by extractOcr and analyze(..., {path:'ocr'}):\n * recognizes (or replays) words and runs the unchanged buildPageOCR/emitPage pipeline. Word IDENTITY (object reference), not value, carries indices through into the emitted blocks — see `geometry.ts`'s OcrLine.words. */\nexport async function buildOcrAnalysisPages(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions, warn: (m: string) => void): Promise<{ pages: AnalysisPage[]; words: OcrWordInput[] }> {\n const tuning: Tuning = { ...DEFAULT_TUNING, ...opts.tuning };\n const validated = validateOcrAnalysisInput(input);\n\n let words: OcrWordInput[];\n let first: number;\n let last: number;\n if ('words' in validated) {\n // Replay: page range from --pages if given, else every page up to the highest one the dump mentions.\n words = validated.words;\n first = opts.pages?.first ?? 1;\n last = opts.pages?.last ?? opts.pages?.first ?? Math.max(1, ...words.map((w) => w.page));\n } else {\n const { recognizePdf } = await import('./engines/tesseract.ts');\n const totalPages = await pageCount(validated.pdfPath);\n const rawFirst = opts.pages?.first ?? 1;\n const rawLast = opts.pages?.last ?? (opts.pages ? rawFirst : totalPages);\n first = Math.min(Math.max(1, rawFirst), Math.max(1, totalPages));\n last = Math.min(Math.max(first, rawLast), totalPages);\n const result = await recognizePdf(validated.pdfPath, first, last, {\n dpi: opts.dpi,\n onProgress: opts.onProgress,\n onWarning: warn,\n });\n words = result.words;\n }\n\n opts.onWords?.(words);\n\n // Stable per-page index, assigned in recognition/replay order — this IS\n // the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).\n const byPage = new Map<number, OcrWordInput[]>();\n const wordIndex = new Map<Word, number>();\n for (const w of words) {\n const list = byPage.get(w.page);\n if (list) {\n wordIndex.set(w, list.length);\n list.push(w);\n } else {\n wordIndex.set(w, 0);\n byPage.set(w.page, [w]);\n }\n }\n\n const pageOcrs = new Map<number, PageOCR>();\n for (let pn = first; pn <= last; pn++) pageOcrs.set(pn, buildPageOCR(byPage.get(pn) ?? [], tuning));\n const { levelOf } = headingLevels([...pageOcrs.values()], tuning);\n\n const pages: AnalysisPage[] = [];\n for (let pn = first; pn <= last; pn++) {\n const p = pageOcrs.get(pn) as PageOCR;\n // An in-range page with no recognized words stays visible (marker +\n // warning), never silently absent — same rule as extractOcr.\n if (!byPage.has(pn)) warn(`p${pn}: no recognized words, page emitted with no OCR text`);\n\n const blocks: AnalysisBlock[] = emitPage(p, levelOf, tuning).map((b) => toAnalysisBlock(b, wordIndex));\n // Decorative fragments always render as one visible \"> [floats]\" aside\n // per page (matching the one printed line) — see geometry.ts's PageOCR.floats.\n if (p.floats.length) {\n blocks.push({\n type: 'float',\n text: `> [floats] ${p.floats.map((f) => f.text).join(' · ')}`,\n wordIndexes: p.floats.flatMap((f) => f.words).map((w) => wordIndex.get(w) as number),\n });\n }\n\n pages.push({\n page: pn,\n words: (byPage.get(pn) ?? []).map(toAnalysisWord),\n blocks,\n });\n }\n\n return { pages, words };\n}\n\nasync function analyzeOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md, fallback: opts.fallback, words });\n return { outputVersion: OUTPUT_VERSION, report, pages };\n}\n\n// TEXT path: deliberately NOT routed through the OCR shared-core pattern —\n// buildMarkdown's reflow/hyphen-rejoin/list rules don't fit the 4-type Analysis block union, so this is an independent, intentionally coarser reconstruction (one word/block per visual LINE) — honest, not byte-identical to extractText (see AnalysisPage, src/types.ts).\n\n// Converts a text-path Line's page-point geometry (src/lines.ts) into the\n// same normalized box contract the OCR path uses (see AnalysisPage.words) — only the coordinate SYSTEM changes; `box.y` anchors on the line's baseline, `box.w` stays 0.\nfunction textLineBox(l: Line, dims: { width: number; height: number }): AnalysisWord['box'] {\n const { width, height } = dims;\n return {\n x: width > 0 ? l.x / width : 0,\n y: height > 0 ? (height - l.y) / height : 0,\n w: 0,\n h: height > 0 ? l.h / height : 0,\n };\n}\n\nasync function analyzeText(pdfPath: string, opts: ExtractTextOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const { numPages, pages, heightChars } = await collect(pdfPath, range);\n const pageDims = new Map(pages.map((p) => [p.page, { width: p.width, height: p.height }]));\n const pageLines: PageLines[] = pages.map((p) => ({ page: p.page, lines: toLines(p) }));\n const pageMarkers = opts.pageMarkers ?? false;\n const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, { pageMarkers });\n const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, { pageMarkers: true }).md;\n\n const hybridPages = pages.filter((p) => p.largeImage).map((p) => p.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n\n // Report is measured from the REAL markdown above — accurate regardless of\n // the coarser block/word reconstruction below.\n const report = buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages });\n\n // headingSizes is offset-independent — reused directly, then re-leveled\n // here with offset 0 regardless of `opts.pageMarkers` (see AnalysisPage).\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1)]));\n const analysisPages: AnalysisPage[] = pageLines.map(({ page, lines }) => {\n const dims = pageDims.get(page) as { width: number; height: number };\n const words: AnalysisWord[] = lines.map((l) => ({ text: l.text, box: textLineBox(l, dims) }));\n const blocks: AnalysisBlock[] = lines.map((l, i) => {\n const level = levelOf.get(l.h);\n return level !== undefined ? { type: 'heading' as const, text: l.text, level, wordIndexes: [i] } : { type: 'paragraph' as const, text: l.text, wordIndexes: [i] };\n });\n return { page, words, blocks };\n });\n\n return { outputVersion: OUTPUT_VERSION, report, pages: analysisPages };\n}\n\n/**\n * Analyze a PDF (or replay an already-recognized OCR word dump) into\n * `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).\n *\n * @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`\n * @param opts see {@link AnalyzeOptions}; for a pdf path, `opts.path` picks 'text' (default) or 'ocr' — analyze() makes no auto-fallback decision (that's pdfToMarkdown's job)\n */\nexport async function analyze(input: string | { words: OcrWordInput[] }, opts: AnalyzeOptions = {}): Promise<Analysis> {\n validatePages(opts.pages);\n if (typeof input !== 'string') return analyzeOcr({ words: validateWordsInput(input) }, opts);\n if (opts.path === 'ocr') return analyzeOcr({ pdfPath: input }, opts);\n return analyzeText(input, opts);\n}\n"],"names":["analyze","buildOcrAnalysisPages","toAnalysisWord","w","box","x","y","h","confidence","undefined","text","toAnalysisBlock","b","wordIndex","wordIndexes","words","map","get","kind","type","level","renderEmitBlock","input","opts","warn","byPage","tuning","validated","first","last","Math","recognizePdf","totalPages","rawFirst","rawLast","result","list","pageOcrs","pn","levelOf","pages","p","blocks","DEFAULT_TUNING","validateOcrAnalysisInput","max","page","pageCount","pdfPath","min","dpi","onProgress","onWarning","onWords","Map","set","length","push","buildPageOCR","headingLevels","values","has","emitPage","floats","f","join","flatMap","analyzeOcr","md","report","m","dictAvailable","dictUnavailableWarning","renderPagesToMarkdown","forcePageMarkers","pageMarkers","buildOcrReport","fallback","outputVersion","OUTPUT_VERSION","textLineBox","l","dims","width","height","analyzeText","range","numPages","heightChars","pageDims","pageLines","buildMarkdown","stats","bodyH","headingSizes","statsMd","hybridPages","n","analysisPages","collect","lines","toLines","filter","largeImage","buildReport","slice","i","validatePages","validateWordsInput","path"],"mappings":"AAAA,2EAA2E;AAC3E,kPAAkP;;;;;;;;;;;;QA4L5NA;eAAAA;;QA/IAC;eAAAA;;;yBA5Ca;sBACmB;sBAEZ;0BAKkB;uBACpC;0BACM;gCACQ;wBACsB;0BAEgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc5E,SAASC,eAAeC,CAAe;IACrC,IAAMC,MAAM;QAAEC,GAAGF,EAAEE,CAAC;QAAEC,GAAGH,EAAEG,CAAC;QAAEH,GAAGA,EAAEA,CAAC;QAAEI,GAAGJ,EAAEI,CAAC;IAAC;IAC7C,OAAOJ,EAAEK,UAAU,KAAKC,YAAY;QAAEC,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;IAAI,IAAI;QAAEM,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;QAAKI,YAAYL,EAAEK,UAAU;IAAC;AAC5G;AAEA,8EAA8E;AAC9E,qIAAqI;AACrI,SAASG,gBAAgBC,CAAY,EAAEC,SAA4B;IACjE,IAAMC,cAAcF,EAAEG,KAAK,CAACC,GAAG,CAAC,SAACb;eAAMU,UAAUI,GAAG,CAACd;;IACrD,IAAIS,EAAEM,IAAI,KAAK,WAAW,OAAO;QAAEC,MAAM;QAAWT,MAAME,EAAEF,IAAI;QAAEU,OAAOR,EAAEQ,KAAK;QAAEN,aAAAA;IAAY;IAC9F,IAAIF,EAAEM,IAAI,KAAK,UAAU,OAAO;QAAEC,MAAM;QAAUT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;IACxF,OAAO;QAAEK,MAAM;QAAaT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;AACpE;AAIO,SAAeb,sBAAsBqB,KAAsD,EAAEC,IAAuB,EAAEC,IAAyB;;YA6C/EC,aAlBrEF,eA1BMG,QACAC,WAEFZ,OACAa,OACAC,MAK8CC,aAAzCP,cADCA,aACDA,cAAoBA,4BAIVA,cACDA,cAHRQ,cACFC,YACAC,UACAC,SAGAC,QAYFV,QACAZ,WACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,GACHiC,MAUFC,UACGC,IACDC,SAEFC,OACGF,KAmBGb,cAlBJgB,GAKAC;;;;oBAtDFhB,SAAiB,mBAAKiB,0BAAc,EAAKpB,KAAKG,MAAM;oBACpDC,YAAYiB,IAAAA,oCAAwB,EAACtB;yBAKvC,CAAA,WAAWK,SAAQ,GAAnB;;;;oBACF,qGAAqG;oBACrGZ,QAAQY,UAAUZ,KAAK;oBACvBa,iBAAQL,cAAAA,KAAKiB,KAAK,cAAVjB,kCAAAA,YAAYK,KAAK,uCAAI;oBAC7BC,QAAON,kBAAAA,eAAAA,KAAKiB,KAAK,cAAVjB,mCAAAA,aAAYM,IAAI,0CAAIN,eAAAA,KAAKiB,KAAK,cAAVjB,mCAAAA,aAAYK,KAAK,cAArCL,mBAAAA,QAAyCO,CAAAA,QAAAA,MAAKe,GAAG,OAARf,OAAAA;wBAAS;sBAATA,OAAY,qBAAGf,MAAMC,GAAG,CAAC,SAACb;+BAAMA,EAAE2C,IAAI;;;;;;;oBAE7D;;wBAAM;2EAAA,QAAO;;;;oBAA9Bf,eAAiB,cAAjBA;oBACW;;wBAAMgB,IAAAA,oBAAS,EAACpB,UAAUqB,OAAO;;;oBAA9ChB,aAAa;oBACbC,qBAAWV,eAAAA,KAAKiB,KAAK,cAAVjB,mCAAAA,aAAYK,KAAK,yCAAI;oBAChCM,oBAAUX,eAAAA,KAAKiB,KAAK,cAAVjB,mCAAAA,aAAYM,IAAI,yCAAKN,KAAKiB,KAAK,GAAGP,WAAWD;oBAC7DJ,QAAQE,KAAKmB,GAAG,CAACnB,KAAKe,GAAG,CAAC,GAAGZ,WAAWH,KAAKe,GAAG,CAAC,GAAGb;oBACpDH,OAAOC,KAAKmB,GAAG,CAACnB,KAAKe,GAAG,CAACjB,OAAOM,UAAUF;oBAC3B;;wBAAMD,aAAaJ,UAAUqB,OAAO,EAAEpB,OAAOC,MAAM;4BAChEqB,KAAK3B,KAAK2B,GAAG;4BACbC,YAAY5B,KAAK4B,UAAU;4BAC3BC,WAAW5B;wBACb;;;oBAJMW,SAAS;oBAKfpB,QAAQoB,OAAOpB,KAAK;;;qBAGtBQ,gBAAAA,KAAK8B,OAAO,cAAZ9B,oCAAAA,mBAAAA,MAAeR;oBAEf,wEAAwE;oBACxE,mJAAmJ;oBAC7IU,SAAS,IAAI6B;oBACbzC,YAAY,IAAIyC;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAWvC,4BAAX,6BAAA,QAAA,yBAAA,iCAAkB;4BAAZZ,IAAN;4BACGiC,OAAOX,OAAOR,GAAG,CAACd,EAAE2C,IAAI;4BAC9B,IAAIV,MAAM;gCACRvB,UAAU0C,GAAG,CAACpD,GAAGiC,KAAKoB,MAAM;gCAC5BpB,KAAKqB,IAAI,CAACtD;4BACZ,OAAO;gCACLU,UAAU0C,GAAG,CAACpD,GAAG;gCACjBsB,OAAO8B,GAAG,CAACpD,EAAE2C,IAAI;oCAAG3C;;4BACtB;wBACF;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWCkC,WAAW,IAAIiB;oBACrB,IAAShB,KAAKV,OAAOU,MAAMT,MAAMS,KAAMD,SAASkB,GAAG,CAACjB,IAAIoB,IAAAA,wBAAY,GAACjC,cAAAA,OAAOR,GAAG,CAACqB,iBAAXb,yBAAAA,kBAAsBC;oBACnFa,UAAYoB,IAAAA,yBAAa,EAAE,qBAAGtB,SAASuB,MAAM,KAAKlC,QAAlDa;oBAEFC;oBACN,IAASF,MAAKV,OAAOU,OAAMT,MAAMS,MAAM;;wBAC/BG,IAAIJ,SAASpB,GAAG,CAACqB;wBACvB,oEAAoE;wBACpE,6DAA6D;wBAC7D,IAAI,CAACb,OAAOoC,GAAG,CAACvB,MAAKd,KAAK,AAAC,IAAM,OAAHc,KAAG;wBAE3BI,SAA0BoB,IAAAA,gBAAQ,EAACrB,GAAGF,SAASb,QAAQV,GAAG,CAAC,SAACJ;mCAAMD,gBAAgBC,GAAGC;;wBAC3F,uEAAuE;wBACvE,+EAA+E;wBAC/E,IAAI4B,EAAEsB,MAAM,CAACP,MAAM,EAAE;4BACnBd,OAAOe,IAAI,CAAC;gCACVtC,MAAM;gCACNT,MAAM,AAAC,cAAqD,OAAxC+B,EAAEsB,MAAM,CAAC/C,GAAG,CAAC,SAACgD;2CAAMA,EAAEtD,IAAI;mCAAEuD,IAAI,CAAC;gCACrDnD,aAAa2B,EAAEsB,MAAM,CAACG,OAAO,CAAC,SAACF;2CAAMA,EAAEjD,KAAK;mCAAEC,GAAG,CAAC,SAACb;2CAAMU,UAAUI,GAAG,CAACd;;4BACzE;wBACF;wBAEAqC,MAAMiB,IAAI,CAAC;4BACTX,MAAMR;4BACNvB,OAAO,EAACU,eAAAA,OAAOR,GAAG,CAACqB,kBAAXb,0BAAAA,mBAAsBT,GAAG,CAACd;4BAClCwC,QAAAA;wBACF;oBACF;oBAEA;;wBAAO;4BAAEF,OAAAA;4BAAOzB,OAAAA;wBAAM;;;;IACxB;;AAEA,SAAeoD,WAAW7C,KAAsD,EAAEC,IAAuB;;YACjGC,MAEmB,MAAjBgB,OAAOzB,OACTqD,IACAC;;;;oBAJA7C,OAAO,cAAC8C;4BAAc/C;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiB+C;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAI/C,KAAKgD,IAAAA,8BAAsB;oBACxB;;wBAAMvE,sBAAsBqB,OAAOC,MAAMC;;;oBAAzC,OAAA,eAAjBgB,QAAiB,KAAjBA,OAAOzB,QAAU,KAAVA;oBACTqD,KAAKK,IAAAA,uCAAqB,EAACjC,OAAO;wBAAEkC,kBAAkB;wBAAMC,aAAa;oBAAK;oBAC9EN,SAASO,IAAAA,wBAAc,EAAC;wBAAER,IAAAA;wBAAIS,UAAUtD,KAAKsD,QAAQ;wBAAE9D,OAAAA;oBAAM;oBACnE;;wBAAO;4BAAE+D,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQ7B,OAAAA;wBAAM;;;;IACxD;;AAEA,2EAA2E;AAC3E,4QAA4Q;AAE5Q,0EAA0E;AAC1E,yKAAyK;AACzK,SAASwC,YAAYC,CAAO,EAAEC,IAAuC;IACnE,IAAQC,QAAkBD,KAAlBC,OAAOC,SAAWF,KAAXE;IACf,OAAO;QACL/E,GAAG8E,QAAQ,IAAIF,EAAE5E,CAAC,GAAG8E,QAAQ;QAC7B7E,GAAG8E,SAAS,IAAI,AAACA,CAAAA,SAASH,EAAE3E,CAAC,AAADA,IAAK8E,SAAS;QAC1CjF,GAAG;QACHI,GAAG6E,SAAS,IAAIH,EAAE1E,CAAC,GAAG6E,SAAS;IACjC;AACF;AAEA,SAAeC,YAAYrC,OAAe,EAAEzB,IAAwB;;YAGNA,kBAIxCA,mBANdC,MAEA8D,OACmC,MAAjCC,UAAU/C,OAAOgD,aACnBC,UACAC,WACAf,aACqCgB,gBAAnCvB,IAAIwB,OAAOC,OAAOC,cACpBC,SAEAC,aACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,GAML5B,QAIA9B,SACA2D;;;;oBAtBA1E,OAAO,cAAC8C;4BAAc/C;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiB+C;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAI/C,KAAKgD,IAAAA,8BAAsB;oBAC3Cc,QAAQ/D,KAAKiB,KAAK,GAAG;wBAAEZ,OAAOL,KAAKiB,KAAK,CAACZ,KAAK;wBAAEC,IAAI,GAAEN,mBAAAA,KAAKiB,KAAK,CAACX,IAAI,cAAfN,8BAAAA,mBAAmBA,KAAKiB,KAAK,CAACZ,KAAK;oBAAC,IAAI,CAAC;oBAC5D;;wBAAMuE,IAAAA,kBAAO,EAACnD,SAASsC;;;oBAAvB,OAAA,eAAjCC,WAAiC,KAAjCA,UAAU/C,QAAuB,KAAvBA,OAAOgD,cAAgB,KAAhBA;oBACnBC,WAAW,IAAInC,IAAId,MAAMxB,GAAG,CAAC,SAACyB;+BAAM;4BAACA,EAAEK,IAAI;4BAAE;gCAAEqC,OAAO1C,EAAE0C,KAAK;gCAAEC,QAAQ3C,EAAE2C,MAAM;4BAAC;yBAAE;;oBAClFM,YAAyBlD,MAAMxB,GAAG,CAAC,SAACyB;+BAAO;4BAAEK,MAAML,EAAEK,IAAI;4BAAEsD,OAAOC,IAAAA,gBAAO,EAAC5D;wBAAG;;oBAC7EkC,eAAcpD,oBAAAA,KAAKoD,WAAW,cAAhBpD,+BAAAA,oBAAoB;oBACGoE,iBAAAA,IAAAA,yBAAa,EAACD,WAAWF,aAAa;wBAAEb,aAAAA;oBAAY,IAAvFP,KAAmCuB,eAAnCvB,IAAIwB,QAA+BD,eAA/BC,OAAOC,QAAwBF,eAAxBE,OAAOC,eAAiBH,eAAjBG;oBACpBC,UAAUpB,cAAcP,KAAKuB,IAAAA,yBAAa,EAACD,WAAWF,aAAa;wBAAEb,aAAa;oBAAK,GAAGP,EAAE;oBAE5F4B,cAAcxD,MAAM8D,MAAM,CAAC,SAAC7D;+BAAMA,EAAE8D,UAAU;uBAAEvF,GAAG,CAAC,SAACyB;+BAAMA,EAAEK,IAAI;;oBAClE,kCAAA,2BAAA;;wBAAL,IAAK,YAAWkD,kCAAX,6BAAA,QAAA,yBAAA,iCAAwB;4BAAlBC,IAAN;4BACHzE,KAAK,AAAC,IAA8JyE,OAA3JA,GAAE,2JAA2J,OAAFA,GAAE;wBACxK;;wBAFK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAIL,2EAA2E;oBAC3E,+CAA+C;oBACzC5B,SAASmC,IAAAA,qBAAW,EAAC;wBAAEjB,UAAAA;wBAAUnB,IAAAA;wBAAI2B,SAAAA;wBAASH,OAAAA;wBAAOC,OAAAA;wBAAOC,cAAAA;wBAAcE,aAAAA;oBAAY;oBAE5F,wEAAwE;oBACxE,0EAA0E;oBACpEzD,UAAU,IAAIe,IAAIwC,aAAaW,KAAK,CAAC,GAAG,GAAGzF,GAAG,CAAC,SAACT,GAAGmG;+BAAM;4BAACnG;4BAAGuB,KAAKmB,GAAG,CAAC,GAAGyD,IAAI;yBAAG;;oBAChFR,gBAAgCR,UAAU1E,GAAG,CAAC;4BAAG8B,aAAAA,MAAMsD,cAAAA;wBAC3D,IAAMlB,OAAOO,SAASxE,GAAG,CAAC6B;wBAC1B,IAAM/B,QAAwBqF,MAAMpF,GAAG,CAAC,SAACiE;mCAAO;gCAAEvE,MAAMuE,EAAEvE,IAAI;gCAAEN,KAAK4E,YAAYC,GAAGC;4BAAM;;wBAC1F,IAAMxC,SAA0B0D,MAAMpF,GAAG,CAAC,SAACiE,GAAGyB;4BAC5C,IAAMtF,QAAQmB,QAAQtB,GAAG,CAACgE,EAAE1E,CAAC;4BAC7B,OAAOa,UAAUX,YAAY;gCAAEU,MAAM;gCAAoBT,MAAMuE,EAAEvE,IAAI;gCAAEU,OAAAA;gCAAON,aAAa;oCAAC4F;iCAAE;4BAAC,IAAI;gCAAEvF,MAAM;gCAAsBT,MAAMuE,EAAEvE,IAAI;gCAAEI,aAAa;oCAAC4F;iCAAE;4BAAC;wBAClK;wBACA,OAAO;4BAAE5D,MAAAA;4BAAM/B,OAAAA;4BAAO2B,QAAAA;wBAAO;oBAC/B;oBAEA;;wBAAO;4BAAEoC,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQ7B,OAAO0D;wBAAc;;;;IACvE;;AASO,SAAelG;wCAAQsB,KAAyC;YAAEC;;;YAAAA,OAAAA,oEAAuB,CAAC;YAC/FoF,IAAAA,yBAAa,EAACpF,KAAKiB,KAAK;YACxB,IAAI,OAAOlB,UAAU,UAAU;;gBAAO6C,WAAW;oBAAEpD,OAAO6F,IAAAA,8BAAkB,EAACtF;gBAAO,GAAGC;;YACvF,IAAIA,KAAKsF,IAAI,KAAK,OAAO;;gBAAO1C,WAAW;oBAAEnB,SAAS1B;gBAAM,GAAGC;;YAC/D;;gBAAO8D,YAAY/D,OAAOC;;;IAC5B"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/analyze.ts"],"sourcesContent":["// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr\n// shares with it — same buildPageOCR -> emitPage -> renderPagesToMarkdown pipeline, so OCR-path markdown is byte-identical by construction, not two implementations kept in sync by hand. The TEXT path is NOT shared this way (see analyzeText).\nimport { collect, pageCount } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { EmitBlock } from './emit.ts';\nimport { emitPage, renderEmitBlock } from './emit.ts';\nimport { recognizePdf } from './engines/tesseract.ts';\n// Type-only: erased at compile time, so this does NOT create a runtime\n// circular dependency even though src/extract.ts imports (value-level) from this module.\nimport type { ExtractOcrOptions, ExtractTextOptions, OcrWordInput } from './extract.ts';\nimport type { PageOCR, Tuning, Word } from './geometry.ts';\nimport { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { buildOcrReport, buildReport, OUTPUT_VERSION } from './report.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord, Line, PageLines } from './types.ts';\nimport { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.ts';\n\n/** analyze(pdf) with the default/'text' path — OCR-only fields don't exist\n * on this branch, so passing e.g. `tuning` without `path: 'ocr'` is a compile error. */\nexport type AnalyzeTextOptions = ExtractTextOptions & { path?: 'text' };\n\n/** analyze(pdf, {path:'ocr'}) — same fields as {@link ExtractOcrOptions},\n * unlocked only once `path: 'ocr'` is set. */\nexport type AnalyzeOcrOptions = ExtractOcrOptions & { path: 'ocr' };\n\n/** Options for {@link analyze}: which fields type-check depends on `path`\n * (ignored, forced 'ocr', when `input` is a `{ words }` replay). */\nexport type AnalyzeOptions = AnalyzeTextOptions | AnalyzeOcrOptions;\n\nfunction toAnalysisWord(w: OcrWordInput): AnalysisWord {\n const box = { x: w.x, y: w.y, w: w.w, h: w.h };\n return w.confidence === undefined ? { text: w.text, box } : { text: w.text, box, confidence: w.confidence };\n}\n\n// EmitBlock -> AnalysisBlock: collapses emit.ts's 'quote' kind (a pull-quote)\n// into Analysis's 'paragraph' type, `text` keeping its literal '> ' prefix — see AnalysisBlock's doc comment (src/types.ts) for why.\nfunction toAnalysisBlock(b: EmitBlock, wordIndex: Map<Word, number>): AnalysisBlock {\n const wordIndexes = b.words.map((w) => wordIndex.get(w) as number);\n if (b.kind === 'heading') return { type: 'heading', text: b.text, level: b.level, wordIndexes };\n if (b.kind === 'review') return { type: 'review', text: renderEmitBlock(b), wordIndexes };\n return { type: 'paragraph', text: renderEmitBlock(b), wordIndexes };\n}\n\n/** The OCR-path core shared by extractOcr and analyze(..., {path:'ocr'}):\n * recognizes (or replays) words and runs the unchanged buildPageOCR/emitPage pipeline. Word IDENTITY (object reference), not value, carries indices through into the emitted blocks — see `geometry.ts`'s OcrLine.words. */\nexport async function buildOcrAnalysisPages(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions, warn: (m: string) => void): Promise<{ pages: AnalysisPage[]; words: OcrWordInput[] }> {\n const tuning: Tuning = { ...DEFAULT_TUNING, ...opts.tuning };\n const validated = validateOcrAnalysisInput(input);\n\n let words: OcrWordInput[];\n let first: number;\n let last: number;\n if ('words' in validated) {\n // Replay: page range from --pages if given, else every page up to the highest one the dump mentions.\n words = validated.words;\n first = opts.pages?.first ?? 1;\n last = opts.pages?.last ?? opts.pages?.first ?? Math.max(1, ...words.map((w) => w.page));\n } else {\n const totalPages = await pageCount(validated.pdfPath);\n const rawFirst = opts.pages?.first ?? 1;\n const rawLast = opts.pages?.last ?? (opts.pages ? rawFirst : totalPages);\n first = Math.min(Math.max(1, rawFirst), Math.max(1, totalPages));\n last = Math.min(Math.max(first, rawLast), totalPages);\n const result = await recognizePdf(validated.pdfPath, first, last, {\n dpi: opts.dpi,\n onProgress: opts.onProgress,\n onWarning: warn,\n });\n words = result.words;\n }\n\n opts.onWords?.(words);\n\n // Stable per-page index, assigned in recognition/replay order — this IS\n // the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).\n const byPage = new Map<number, OcrWordInput[]>();\n const wordIndex = new Map<Word, number>();\n for (const w of words) {\n const list = byPage.get(w.page);\n if (list) {\n wordIndex.set(w, list.length);\n list.push(w);\n } else {\n wordIndex.set(w, 0);\n byPage.set(w.page, [w]);\n }\n }\n\n const pageOcrs = new Map<number, PageOCR>();\n for (let pn = first; pn <= last; pn++) pageOcrs.set(pn, buildPageOCR(byPage.get(pn) ?? [], tuning));\n const { levelOf } = headingLevels([...pageOcrs.values()], tuning);\n\n const pages: AnalysisPage[] = [];\n for (let pn = first; pn <= last; pn++) {\n const p = pageOcrs.get(pn) as PageOCR;\n // An in-range page with no recognized words stays visible (marker +\n // warning), never silently absent — same rule as extractOcr.\n if (!byPage.has(pn)) warn(`p${pn}: no recognized words, page emitted with no OCR text`);\n\n const blocks: AnalysisBlock[] = emitPage(p, levelOf, tuning).map((b) => toAnalysisBlock(b, wordIndex));\n // Decorative fragments always render as one visible \"> [floats]\" aside\n // per page (matching the one printed line) — see geometry.ts's PageOCR.floats.\n if (p.floats.length) {\n blocks.push({\n type: 'float',\n text: `> [floats] ${p.floats.map((f) => f.text).join(' · ')}`,\n wordIndexes: p.floats.flatMap((f) => f.words).map((w) => wordIndex.get(w) as number),\n });\n }\n\n pages.push({\n page: pn,\n words: (byPage.get(pn) ?? []).map(toAnalysisWord),\n blocks,\n });\n }\n\n return { pages, words };\n}\n\nasync function analyzeOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md, fallback: opts.fallback, words });\n return { outputVersion: OUTPUT_VERSION, report, pages };\n}\n\n// TEXT path: deliberately NOT routed through the OCR shared-core pattern —\n// buildMarkdown's reflow/hyphen-rejoin/list rules don't fit the 4-type Analysis block union, so this is an independent, intentionally coarser reconstruction (one word/block per visual LINE) — honest, not byte-identical to extractText (see AnalysisPage, src/types.ts).\n\n// Converts a text-path Line's page-point geometry (src/lines.ts) into the\n// same normalized box contract the OCR path uses (see AnalysisPage.words) — only the coordinate SYSTEM changes; `box.y` anchors on the line's baseline, `box.w` stays 0.\nfunction textLineBox(l: Line, dims: { width: number; height: number }): AnalysisWord['box'] {\n const { width, height } = dims;\n return {\n x: width > 0 ? l.x / width : 0,\n y: height > 0 ? (height - l.y) / height : 0,\n w: 0,\n h: height > 0 ? l.h / height : 0,\n };\n}\n\nasync function analyzeText(pdfPath: string, opts: ExtractTextOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const { numPages, pages, heightChars } = await collect(pdfPath, range);\n const pageDims = new Map(pages.map((p) => [p.page, { width: p.width, height: p.height }]));\n const pageLines: PageLines[] = pages.map((p) => ({ page: p.page, lines: toLines(p) }));\n const pageMarkers = opts.pageMarkers ?? false;\n const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, { pageMarkers });\n const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, { pageMarkers: true }).md;\n\n const hybridPages = pages.filter((p) => p.largeImage).map((p) => p.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n\n // Report is measured from the REAL markdown above — accurate regardless of\n // the coarser block/word reconstruction below.\n const report = buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages });\n\n // headingSizes is offset-independent — reused directly, then re-leveled\n // here with offset 0 regardless of `opts.pageMarkers` (see AnalysisPage).\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1)]));\n const analysisPages: AnalysisPage[] = pageLines.map(({ page, lines }) => {\n const dims = pageDims.get(page) as { width: number; height: number };\n const words: AnalysisWord[] = lines.map((l) => ({ text: l.text, box: textLineBox(l, dims) }));\n const blocks: AnalysisBlock[] = lines.map((l, i) => {\n const level = levelOf.get(l.h);\n return level !== undefined ? { type: 'heading' as const, text: l.text, level, wordIndexes: [i] } : { type: 'paragraph' as const, text: l.text, wordIndexes: [i] };\n });\n return { page, words, blocks };\n });\n\n return { outputVersion: OUTPUT_VERSION, report, pages: analysisPages };\n}\n\n/**\n * Analyze a PDF (or replay an already-recognized OCR word dump) into\n * `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).\n *\n * @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`\n * @param opts see {@link AnalyzeOptions}; for a pdf path, `opts.path` picks 'text' (default) or 'ocr' — analyze() makes no auto-fallback decision (that's pdfToMarkdown's job)\n */\nexport async function analyze(input: string | { words: OcrWordInput[] }, opts: AnalyzeOptions = {}): Promise<Analysis> {\n validatePages(opts.pages);\n if (typeof input !== 'string') return analyzeOcr({ words: validateWordsInput(input) }, opts);\n if (opts.path === 'ocr') return analyzeOcr({ pdfPath: input }, opts);\n return analyzeText(input, opts);\n}\n"],"names":["analyze","buildOcrAnalysisPages","toAnalysisWord","w","box","x","y","h","confidence","undefined","text","toAnalysisBlock","b","wordIndex","wordIndexes","words","map","get","kind","type","level","renderEmitBlock","input","opts","warn","byPage","tuning","validated","first","last","Math","totalPages","rawFirst","rawLast","result","list","pageOcrs","pn","levelOf","pages","p","blocks","DEFAULT_TUNING","validateOcrAnalysisInput","max","page","pageCount","pdfPath","min","recognizePdf","dpi","onProgress","onWarning","onWords","Map","set","length","push","buildPageOCR","headingLevels","values","has","emitPage","floats","f","join","flatMap","analyzeOcr","md","report","m","dictAvailable","dictUnavailableWarning","renderPagesToMarkdown","forcePageMarkers","pageMarkers","buildOcrReport","fallback","outputVersion","OUTPUT_VERSION","textLineBox","l","dims","width","height","analyzeText","range","numPages","heightChars","pageDims","pageLines","buildMarkdown","stats","bodyH","headingSizes","statsMd","hybridPages","n","analysisPages","collect","lines","toLines","filter","largeImage","buildReport","slice","i","validatePages","validateWordsInput","path"],"mappings":"AAAA,2EAA2E;AAC3E,kPAAkP;;;;;;;;;;;;QA4L5NA;eAAAA;;QA9IAC;eAAAA;;;yBA7Ca;sBACmB;sBAEZ;2BACb;0BAK+B;uBACpC;0BACM;gCACQ;wBACsB;0BAEgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAc5E,SAASC,eAAeC,CAAe;IACrC,IAAMC,MAAM;QAAEC,GAAGF,EAAEE,CAAC;QAAEC,GAAGH,EAAEG,CAAC;QAAEH,GAAGA,EAAEA,CAAC;QAAEI,GAAGJ,EAAEI,CAAC;IAAC;IAC7C,OAAOJ,EAAEK,UAAU,KAAKC,YAAY;QAAEC,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;IAAI,IAAI;QAAEM,MAAMP,EAAEO,IAAI;QAAEN,KAAAA;QAAKI,YAAYL,EAAEK,UAAU;IAAC;AAC5G;AAEA,8EAA8E;AAC9E,qIAAqI;AACrI,SAASG,gBAAgBC,CAAY,EAAEC,SAA4B;IACjE,IAAMC,cAAcF,EAAEG,KAAK,CAACC,GAAG,CAAC,SAACb;eAAMU,UAAUI,GAAG,CAACd;;IACrD,IAAIS,EAAEM,IAAI,KAAK,WAAW,OAAO;QAAEC,MAAM;QAAWT,MAAME,EAAEF,IAAI;QAAEU,OAAOR,EAAEQ,KAAK;QAAEN,aAAAA;IAAY;IAC9F,IAAIF,EAAEM,IAAI,KAAK,UAAU,OAAO;QAAEC,MAAM;QAAUT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;IACxF,OAAO;QAAEK,MAAM;QAAaT,MAAMW,IAAAA,uBAAe,EAACT;QAAIE,aAAAA;IAAY;AACpE;AAIO,SAAeb,sBAAsBqB,KAAsD,EAAEC,IAAuB,EAAEC,IAAyB;;YA4C/EC,aAlBrEF,eAzBMG,QACAC,WAEFZ,OACAa,OACAC,MAK8CC,aAAzCP,cADCA,aACDA,cAAoBA,4BAGVA,cACDA,cAFVQ,YACAC,UACAC,SAGAC,QAYFT,QACAZ,WACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMV,GACHgC,MAUFC,UACGC,IACDC,SAEFC,OACGF,KAmBGZ,cAlBJe,GAKAC;;;;oBArDFf,SAAiB,mBAAKgB,0BAAc,EAAKnB,KAAKG,MAAM;oBACpDC,YAAYgB,IAAAA,oCAAwB,EAACrB;yBAKvC,CAAA,WAAWK,SAAQ,GAAnB;;;;oBACF,qGAAqG;oBACrGZ,QAAQY,UAAUZ,KAAK;oBACvBa,iBAAQL,cAAAA,KAAKgB,KAAK,cAAVhB,kCAAAA,YAAYK,KAAK,uCAAI;oBAC7BC,QAAON,kBAAAA,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYM,IAAI,0CAAIN,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYK,KAAK,cAArCL,mBAAAA,QAAyCO,CAAAA,QAAAA,MAAKc,GAAG,OAARd,OAAAA;wBAAS;sBAATA,OAAY,qBAAGf,MAAMC,GAAG,CAAC,SAACb;+BAAMA,EAAE0C,IAAI;;;;;;;oBAEnE;;wBAAMC,IAAAA,oBAAS,EAACnB,UAAUoB,OAAO;;;oBAA9ChB,aAAa;oBACbC,qBAAWT,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYK,KAAK,yCAAI;oBAChCK,oBAAUV,eAAAA,KAAKgB,KAAK,cAAVhB,mCAAAA,aAAYM,IAAI,yCAAKN,KAAKgB,KAAK,GAAGP,WAAWD;oBAC7DH,QAAQE,KAAKkB,GAAG,CAAClB,KAAKc,GAAG,CAAC,GAAGZ,WAAWF,KAAKc,GAAG,CAAC,GAAGb;oBACpDF,OAAOC,KAAKkB,GAAG,CAAClB,KAAKc,GAAG,CAAChB,OAAOK,UAAUF;oBAC3B;;wBAAMkB,IAAAA,yBAAY,EAACtB,UAAUoB,OAAO,EAAEnB,OAAOC,MAAM;4BAChEqB,KAAK3B,KAAK2B,GAAG;4BACbC,YAAY5B,KAAK4B,UAAU;4BAC3BC,WAAW5B;wBACb;;;oBAJMU,SAAS;oBAKfnB,QAAQmB,OAAOnB,KAAK;;;qBAGtBQ,gBAAAA,KAAK8B,OAAO,cAAZ9B,oCAAAA,mBAAAA,MAAeR;oBAEf,wEAAwE;oBACxE,mJAAmJ;oBAC7IU,SAAS,IAAI6B;oBACbzC,YAAY,IAAIyC;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAWvC,4BAAX,6BAAA,QAAA,yBAAA,iCAAkB;4BAAZZ,IAAN;4BACGgC,OAAOV,OAAOR,GAAG,CAACd,EAAE0C,IAAI;4BAC9B,IAAIV,MAAM;gCACRtB,UAAU0C,GAAG,CAACpD,GAAGgC,KAAKqB,MAAM;gCAC5BrB,KAAKsB,IAAI,CAACtD;4BACZ,OAAO;gCACLU,UAAU0C,GAAG,CAACpD,GAAG;gCACjBsB,OAAO8B,GAAG,CAACpD,EAAE0C,IAAI;oCAAG1C;;4BACtB;wBACF;;wBATK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAWCiC,WAAW,IAAIkB;oBACrB,IAASjB,KAAKT,OAAOS,MAAMR,MAAMQ,KAAMD,SAASmB,GAAG,CAAClB,IAAIqB,IAAAA,wBAAY,GAACjC,cAAAA,OAAOR,GAAG,CAACoB,iBAAXZ,yBAAAA,kBAAsBC;oBACnFY,UAAYqB,IAAAA,yBAAa,EAAE,qBAAGvB,SAASwB,MAAM,KAAKlC,QAAlDY;oBAEFC;oBACN,IAASF,MAAKT,OAAOS,OAAMR,MAAMQ,MAAM;;wBAC/BG,IAAIJ,SAASnB,GAAG,CAACoB;wBACvB,oEAAoE;wBACpE,6DAA6D;wBAC7D,IAAI,CAACZ,OAAOoC,GAAG,CAACxB,MAAKb,KAAK,AAAC,IAAM,OAAHa,KAAG;wBAE3BI,SAA0BqB,IAAAA,gBAAQ,EAACtB,GAAGF,SAASZ,QAAQV,GAAG,CAAC,SAACJ;mCAAMD,gBAAgBC,GAAGC;;wBAC3F,uEAAuE;wBACvE,+EAA+E;wBAC/E,IAAI2B,EAAEuB,MAAM,CAACP,MAAM,EAAE;4BACnBf,OAAOgB,IAAI,CAAC;gCACVtC,MAAM;gCACNT,MAAM,AAAC,cAAqD,OAAxC8B,EAAEuB,MAAM,CAAC/C,GAAG,CAAC,SAACgD;2CAAMA,EAAEtD,IAAI;mCAAEuD,IAAI,CAAC;gCACrDnD,aAAa0B,EAAEuB,MAAM,CAACG,OAAO,CAAC,SAACF;2CAAMA,EAAEjD,KAAK;mCAAEC,GAAG,CAAC,SAACb;2CAAMU,UAAUI,GAAG,CAACd;;4BACzE;wBACF;wBAEAoC,MAAMkB,IAAI,CAAC;4BACTZ,MAAMR;4BACNtB,OAAO,EAACU,eAAAA,OAAOR,GAAG,CAACoB,kBAAXZ,0BAAAA,mBAAsBT,GAAG,CAACd;4BAClCuC,QAAAA;wBACF;oBACF;oBAEA;;wBAAO;4BAAEF,OAAAA;4BAAOxB,OAAAA;wBAAM;;;;IACxB;;AAEA,SAAeoD,WAAW7C,KAAsD,EAAEC,IAAuB;;YACjGC,MAEmB,MAAjBe,OAAOxB,OACTqD,IACAC;;;;oBAJA7C,OAAO,cAAC8C;4BAAc/C;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiB+C;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAI/C,KAAKgD,IAAAA,8BAAsB;oBACxB;;wBAAMvE,sBAAsBqB,OAAOC,MAAMC;;;oBAAzC,OAAA,eAAjBe,QAAiB,KAAjBA,OAAOxB,QAAU,KAAVA;oBACTqD,KAAKK,IAAAA,uCAAqB,EAAClC,OAAO;wBAAEmC,kBAAkB;wBAAMC,aAAa;oBAAK;oBAC9EN,SAASO,IAAAA,wBAAc,EAAC;wBAAER,IAAAA;wBAAIS,UAAUtD,KAAKsD,QAAQ;wBAAE9D,OAAAA;oBAAM;oBACnE;;wBAAO;4BAAE+D,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQ9B,OAAAA;wBAAM;;;;IACxD;;AAEA,2EAA2E;AAC3E,4QAA4Q;AAE5Q,0EAA0E;AAC1E,yKAAyK;AACzK,SAASyC,YAAYC,CAAO,EAAEC,IAAuC;IACnE,IAAQC,QAAkBD,KAAlBC,OAAOC,SAAWF,KAAXE;IACf,OAAO;QACL/E,GAAG8E,QAAQ,IAAIF,EAAE5E,CAAC,GAAG8E,QAAQ;QAC7B7E,GAAG8E,SAAS,IAAI,AAACA,CAAAA,SAASH,EAAE3E,CAAC,AAADA,IAAK8E,SAAS;QAC1CjF,GAAG;QACHI,GAAG6E,SAAS,IAAIH,EAAE1E,CAAC,GAAG6E,SAAS;IACjC;AACF;AAEA,SAAeC,YAAYtC,OAAe,EAAExB,IAAwB;;YAGNA,kBAIxCA,mBANdC,MAEA8D,OACmC,MAAjCC,UAAUhD,OAAOiD,aACnBC,UACAC,WACAf,aACqCgB,gBAAnCvB,IAAIwB,OAAOC,OAAOC,cACpBC,SAEAC,aACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,GAML5B,QAIA/B,SACA4D;;;;oBAtBA1E,OAAO,cAAC8C;4BAAc/C;gCAAAA,kBAAAA,KAAK6B,SAAS,cAAd7B,sCAAAA,qBAAAA,MAAiB+C;;oBAC7C,IAAI,CAACC,IAAAA,qBAAa,KAAI/C,KAAKgD,IAAAA,8BAAsB;oBAC3Cc,QAAQ/D,KAAKgB,KAAK,GAAG;wBAAEX,OAAOL,KAAKgB,KAAK,CAACX,KAAK;wBAAEC,IAAI,GAAEN,mBAAAA,KAAKgB,KAAK,CAACV,IAAI,cAAfN,8BAAAA,mBAAmBA,KAAKgB,KAAK,CAACX,KAAK;oBAAC,IAAI,CAAC;oBAC5D;;wBAAMuE,IAAAA,kBAAO,EAACpD,SAASuC;;;oBAAvB,OAAA,eAAjCC,WAAiC,KAAjCA,UAAUhD,QAAuB,KAAvBA,OAAOiD,cAAgB,KAAhBA;oBACnBC,WAAW,IAAInC,IAAIf,MAAMvB,GAAG,CAAC,SAACwB;+BAAM;4BAACA,EAAEK,IAAI;4BAAE;gCAAEsC,OAAO3C,EAAE2C,KAAK;gCAAEC,QAAQ5C,EAAE4C,MAAM;4BAAC;yBAAE;;oBAClFM,YAAyBnD,MAAMvB,GAAG,CAAC,SAACwB;+BAAO;4BAAEK,MAAML,EAAEK,IAAI;4BAAEuD,OAAOC,IAAAA,gBAAO,EAAC7D;wBAAG;;oBAC7EmC,eAAcpD,oBAAAA,KAAKoD,WAAW,cAAhBpD,+BAAAA,oBAAoB;oBACGoE,iBAAAA,IAAAA,yBAAa,EAACD,WAAWF,aAAa;wBAAEb,aAAAA;oBAAY,IAAvFP,KAAmCuB,eAAnCvB,IAAIwB,QAA+BD,eAA/BC,OAAOC,QAAwBF,eAAxBE,OAAOC,eAAiBH,eAAjBG;oBACpBC,UAAUpB,cAAcP,KAAKuB,IAAAA,yBAAa,EAACD,WAAWF,aAAa;wBAAEb,aAAa;oBAAK,GAAGP,EAAE;oBAE5F4B,cAAczD,MAAM+D,MAAM,CAAC,SAAC9D;+BAAMA,EAAE+D,UAAU;uBAAEvF,GAAG,CAAC,SAACwB;+BAAMA,EAAEK,IAAI;;oBAClE,kCAAA,2BAAA;;wBAAL,IAAK,YAAWmD,kCAAX,6BAAA,QAAA,yBAAA,iCAAwB;4BAAlBC,IAAN;4BACHzE,KAAK,AAAC,IAA8JyE,OAA3JA,GAAE,2JAA2J,OAAFA,GAAE;wBACxK;;wBAFK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAIL,2EAA2E;oBAC3E,+CAA+C;oBACzC5B,SAASmC,IAAAA,qBAAW,EAAC;wBAAEjB,UAAAA;wBAAUnB,IAAAA;wBAAI2B,SAAAA;wBAASH,OAAAA;wBAAOC,OAAAA;wBAAOC,cAAAA;wBAAcE,aAAAA;oBAAY;oBAE5F,wEAAwE;oBACxE,0EAA0E;oBACpE1D,UAAU,IAAIgB,IAAIwC,aAAaW,KAAK,CAAC,GAAG,GAAGzF,GAAG,CAAC,SAACT,GAAGmG;+BAAM;4BAACnG;4BAAGuB,KAAKkB,GAAG,CAAC,GAAG0D,IAAI;yBAAG;;oBAChFR,gBAAgCR,UAAU1E,GAAG,CAAC;4BAAG6B,aAAAA,MAAMuD,cAAAA;wBAC3D,IAAMlB,OAAOO,SAASxE,GAAG,CAAC4B;wBAC1B,IAAM9B,QAAwBqF,MAAMpF,GAAG,CAAC,SAACiE;mCAAO;gCAAEvE,MAAMuE,EAAEvE,IAAI;gCAAEN,KAAK4E,YAAYC,GAAGC;4BAAM;;wBAC1F,IAAMzC,SAA0B2D,MAAMpF,GAAG,CAAC,SAACiE,GAAGyB;4BAC5C,IAAMtF,QAAQkB,QAAQrB,GAAG,CAACgE,EAAE1E,CAAC;4BAC7B,OAAOa,UAAUX,YAAY;gCAAEU,MAAM;gCAAoBT,MAAMuE,EAAEvE,IAAI;gCAAEU,OAAAA;gCAAON,aAAa;oCAAC4F;iCAAE;4BAAC,IAAI;gCAAEvF,MAAM;gCAAsBT,MAAMuE,EAAEvE,IAAI;gCAAEI,aAAa;oCAAC4F;iCAAE;4BAAC;wBAClK;wBACA,OAAO;4BAAE7D,MAAAA;4BAAM9B,OAAAA;4BAAO0B,QAAAA;wBAAO;oBAC/B;oBAEA;;wBAAO;4BAAEqC,eAAeC,wBAAc;4BAAEV,QAAAA;4BAAQ9B,OAAO2D;wBAAc;;;;IACvE;;AASO,SAAelG;wCAAQsB,KAAyC;YAAEC;;;YAAAA,OAAAA,oEAAuB,CAAC;YAC/FoF,IAAAA,yBAAa,EAACpF,KAAKgB,KAAK;YACxB,IAAI,OAAOjB,UAAU,UAAU;;gBAAO6C,WAAW;oBAAEpD,OAAO6F,IAAAA,8BAAkB,EAACtF;gBAAO,GAAGC;;YACvF,IAAIA,KAAKsF,IAAI,KAAK,OAAO;;gBAAO1C,WAAW;oBAAEpB,SAASzB;gBAAM,GAAGC;;YAC/D;;gBAAO8D,YAAY/D,OAAOC;;;IAC5B"}
|
package/dist/cjs/cli/audit.js
CHANGED
|
@@ -10,6 +10,7 @@ Object.defineProperty(exports, "default", {
|
|
|
10
10
|
return _default;
|
|
11
11
|
}
|
|
12
12
|
});
|
|
13
|
+
var _auditts = require("../audit.js");
|
|
13
14
|
function _array_like_to_array(arr, len) {
|
|
14
15
|
if (len == null || len > arr.length) len = arr.length;
|
|
15
16
|
for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
|
|
@@ -44,36 +45,6 @@ function _async_to_generator(fn) {
|
|
|
44
45
|
});
|
|
45
46
|
};
|
|
46
47
|
}
|
|
47
|
-
function _getRequireWildcardCache(nodeInterop) {
|
|
48
|
-
if (typeof WeakMap !== "function") return null;
|
|
49
|
-
var cacheBabelInterop = new WeakMap();
|
|
50
|
-
var cacheNodeInterop = new WeakMap();
|
|
51
|
-
return (_getRequireWildcardCache = function(nodeInterop) {
|
|
52
|
-
return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
|
|
53
|
-
})(nodeInterop);
|
|
54
|
-
}
|
|
55
|
-
function _interop_require_wildcard(obj, nodeInterop) {
|
|
56
|
-
if (!nodeInterop && obj && obj.__esModule) return obj;
|
|
57
|
-
if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
|
|
58
|
-
default: obj
|
|
59
|
-
};
|
|
60
|
-
var cache = _getRequireWildcardCache(nodeInterop);
|
|
61
|
-
if (cache && cache.has(obj)) return cache.get(obj);
|
|
62
|
-
var newObj = {
|
|
63
|
-
__proto__: null
|
|
64
|
-
};
|
|
65
|
-
var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
|
|
66
|
-
for(var key in obj){
|
|
67
|
-
if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
68
|
-
var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
|
|
69
|
-
if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
|
|
70
|
-
else newObj[key] = obj[key];
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
newObj.default = obj;
|
|
74
|
-
if (cache) cache.set(obj, newObj);
|
|
75
|
-
return newObj;
|
|
76
|
-
}
|
|
77
48
|
function _iterable_to_array_limit(arr, i) {
|
|
78
49
|
var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
|
|
79
50
|
if (_i == null) return;
|
|
@@ -214,54 +185,43 @@ function _unsupported_iterable_to_array(o, minLen) {
|
|
|
214
185
|
var USAGE = 'usage: pdf-to-md audit <words.jsonl> <file.md>';
|
|
215
186
|
var audit = function audit(ctx) {
|
|
216
187
|
return _async_to_generator(function() {
|
|
217
|
-
var _ctx_rest, wordsPath, mdPath,
|
|
188
|
+
var _ctx_rest, wordsPath, mdPath, result, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, line;
|
|
218
189
|
return _ts_generator(this, function(_state) {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
190
|
+
if (ctx.rest.includes('-h') || ctx.rest.includes('--help')) {
|
|
191
|
+
console.log(USAGE);
|
|
192
|
+
return [
|
|
193
|
+
2
|
|
194
|
+
];
|
|
195
|
+
}
|
|
196
|
+
_ctx_rest = _sliced_to_array(ctx.rest, 2), wordsPath = _ctx_rest[0], mdPath = _ctx_rest[1];
|
|
197
|
+
if (!wordsPath || !mdPath) ctx.usageError(USAGE);
|
|
198
|
+
result = (0, _auditts.auditWords)(wordsPath, mdPath);
|
|
199
|
+
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
200
|
+
try {
|
|
201
|
+
for(_iterator = result.lines[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
202
|
+
line = _step.value;
|
|
203
|
+
console.log(line);
|
|
204
|
+
}
|
|
205
|
+
} catch (err) {
|
|
206
|
+
_didIteratorError = true;
|
|
207
|
+
_iteratorError = err;
|
|
208
|
+
} finally{
|
|
209
|
+
try {
|
|
210
|
+
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
211
|
+
_iterator.return();
|
|
226
212
|
}
|
|
227
|
-
|
|
228
|
-
if (
|
|
229
|
-
|
|
230
|
-
4,
|
|
231
|
-
Promise.resolve().then(function() {
|
|
232
|
-
return /*#__PURE__*/ _interop_require_wildcard(require("../audit.js"));
|
|
233
|
-
})
|
|
234
|
-
];
|
|
235
|
-
case 1:
|
|
236
|
-
auditWords = _state.sent().auditWords;
|
|
237
|
-
result = auditWords(wordsPath, mdPath);
|
|
238
|
-
_iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
239
|
-
try {
|
|
240
|
-
for(_iterator = result.lines[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
241
|
-
line = _step.value;
|
|
242
|
-
console.log(line);
|
|
243
|
-
}
|
|
244
|
-
} catch (err) {
|
|
245
|
-
_didIteratorError = true;
|
|
246
|
-
_iteratorError = err;
|
|
247
|
-
} finally{
|
|
248
|
-
try {
|
|
249
|
-
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
250
|
-
_iterator.return();
|
|
251
|
-
}
|
|
252
|
-
} finally{
|
|
253
|
-
if (_didIteratorError) {
|
|
254
|
-
throw _iteratorError;
|
|
255
|
-
}
|
|
256
|
-
}
|
|
213
|
+
} finally{
|
|
214
|
+
if (_didIteratorError) {
|
|
215
|
+
throw _iteratorError;
|
|
257
216
|
}
|
|
258
|
-
|
|
259
|
-
// lost. Exit 1 so the subcommand is usable as a CI gate; strict_deficits from hyphen-join repairs are reported but not failures.
|
|
260
|
-
if (result.missing > 0) process.exitCode = 1;
|
|
261
|
-
return [
|
|
262
|
-
2
|
|
263
|
-
];
|
|
217
|
+
}
|
|
264
218
|
}
|
|
219
|
+
// MISSING>0 is the severity-1 fail signal: a recognized word this markdown
|
|
220
|
+
// lost. Exit 1 so the subcommand is usable as a CI gate; strict_deficits from hyphen-join repairs are reported but not failures.
|
|
221
|
+
if (result.missing > 0) process.exitCode = 1;
|
|
222
|
+
return [
|
|
223
|
+
2
|
|
224
|
+
];
|
|
265
225
|
});
|
|
266
226
|
})();
|
|
267
227
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/cli/audit.ts"],"sourcesContent":["// audit command — the severity-1 audit instrument (README's \"Design\n// contract\") as a first-class subcommand. Thin wrapper over src/audit.ts's auditWords (the library returns data, only the CLI prints and exits).\nimport type { Command } from './types.ts';\n\nconst USAGE = 'usage: pdf-to-md audit <words.jsonl> <file.md>';\n\nconst audit: Command = async (ctx) => {\n if (ctx.rest.includes('-h') || ctx.rest.includes('--help')) {\n console.log(USAGE);\n return;\n }\n const [wordsPath, mdPath] = ctx.rest;\n if (!wordsPath || !mdPath) ctx.usageError(USAGE);\n\n
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/cli/audit.ts"],"sourcesContent":["// audit command — the severity-1 audit instrument (README's \"Design\n// contract\") as a first-class subcommand. Thin wrapper over src/audit.ts's auditWords (the library returns data, only the CLI prints and exits).\nimport { auditWords } from '../audit.ts';\nimport type { Command } from './types.ts';\n\nconst USAGE = 'usage: pdf-to-md audit <words.jsonl> <file.md>';\n\nconst audit: Command = async (ctx) => {\n if (ctx.rest.includes('-h') || ctx.rest.includes('--help')) {\n console.log(USAGE);\n return;\n }\n const [wordsPath, mdPath] = ctx.rest;\n if (!wordsPath || !mdPath) ctx.usageError(USAGE);\n\n const result = auditWords(wordsPath, mdPath);\n for (const line of result.lines) console.log(line);\n\n // MISSING>0 is the severity-1 fail signal: a recognized word this markdown\n // lost. Exit 1 so the subcommand is usable as a CI gate; strict_deficits from hyphen-join repairs are reported but not failures.\n if (result.missing > 0) process.exitCode = 1;\n};\n\nexport default audit;\n"],"names":["USAGE","audit","ctx","wordsPath","mdPath","result","line","rest","includes","console","log","usageError","auditWords","lines","missing","process","exitCode"],"mappings":"AAAA,oEAAoE;AACpE,iJAAiJ;;;;;+BAsBjJ;;;eAAA;;;uBArB2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG3B,IAAMA,QAAQ;AAEd,IAAMC,QAAiB,eAAOC;;YAKAA,WAArBC,WAAWC,QAGZC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;YARX,IAAIJ,IAAIK,IAAI,CAACC,QAAQ,CAAC,SAASN,IAAIK,IAAI,CAACC,QAAQ,CAAC,WAAW;gBAC1DC,QAAQC,GAAG,CAACV;gBACZ;;;YACF;YAC4BE,6BAAAA,IAAIK,IAAI,MAA7BJ,YAAqBD,cAAVE,SAAUF;YAC5B,IAAI,CAACC,aAAa,CAACC,QAAQF,IAAIS,UAAU,CAACX;YAEpCK,SAASO,IAAAA,mBAAU,EAACT,WAAWC;YAChC,kCAAA,2BAAA;;gBAAL,IAAK,YAAcC,OAAOQ,KAAK,uBAA1B,6BAAA,QAAA,yBAAA;oBAAMP,OAAN;oBAA4BG,QAAQC,GAAG,CAACJ;;;gBAAxC;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAEL,2EAA2E;YAC3E,iIAAiI;YACjI,IAAID,OAAOS,OAAO,GAAG,GAAGC,QAAQC,QAAQ,GAAG;;;;;IAC7C;;IAEA,WAAef"}
|
package/dist/cjs/cli/extract.js
CHANGED
|
@@ -13,6 +13,9 @@ Object.defineProperty(exports, "default", {
|
|
|
13
13
|
var _nodefs = require("node:fs");
|
|
14
14
|
var _nodepath = /*#__PURE__*/ _interop_require_default(require("node:path"));
|
|
15
15
|
var _nodeutil = require("node:util");
|
|
16
|
+
var _analyzets = require("../analyze.js");
|
|
17
|
+
var _extractts = require("../extract.js");
|
|
18
|
+
var _renderanalysists = require("../render-analysis.js");
|
|
16
19
|
function _array_like_to_array(arr, len) {
|
|
17
20
|
if (len == null || len > arr.length) len = arr.length;
|
|
18
21
|
for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
|
|
@@ -63,36 +66,6 @@ function _interop_require_default(obj) {
|
|
|
63
66
|
default: obj
|
|
64
67
|
};
|
|
65
68
|
}
|
|
66
|
-
function _getRequireWildcardCache(nodeInterop) {
|
|
67
|
-
if (typeof WeakMap !== "function") return null;
|
|
68
|
-
var cacheBabelInterop = new WeakMap();
|
|
69
|
-
var cacheNodeInterop = new WeakMap();
|
|
70
|
-
return (_getRequireWildcardCache = function(nodeInterop) {
|
|
71
|
-
return nodeInterop ? cacheNodeInterop : cacheBabelInterop;
|
|
72
|
-
})(nodeInterop);
|
|
73
|
-
}
|
|
74
|
-
function _interop_require_wildcard(obj, nodeInterop) {
|
|
75
|
-
if (!nodeInterop && obj && obj.__esModule) return obj;
|
|
76
|
-
if (obj === null || typeof obj !== "object" && typeof obj !== "function") return {
|
|
77
|
-
default: obj
|
|
78
|
-
};
|
|
79
|
-
var cache = _getRequireWildcardCache(nodeInterop);
|
|
80
|
-
if (cache && cache.has(obj)) return cache.get(obj);
|
|
81
|
-
var newObj = {
|
|
82
|
-
__proto__: null
|
|
83
|
-
};
|
|
84
|
-
var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor;
|
|
85
|
-
for(var key in obj){
|
|
86
|
-
if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) {
|
|
87
|
-
var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null;
|
|
88
|
-
if (desc && (desc.get || desc.set)) Object.defineProperty(newObj, key, desc);
|
|
89
|
-
else newObj[key] = obj[key];
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
newObj.default = obj;
|
|
93
|
-
if (cache) cache.set(obj, newObj);
|
|
94
|
-
return newObj;
|
|
95
|
-
}
|
|
96
69
|
function _iterable_to_array_limit(arr, i) {
|
|
97
70
|
var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
|
|
98
71
|
if (_i == null) return;
|
|
@@ -317,7 +290,7 @@ var TUNING_FLAGS = [
|
|
|
317
290
|
];
|
|
318
291
|
var extract = function extract(ctx) {
|
|
319
292
|
return _async_to_generator(function() {
|
|
320
|
-
var parsed, values, positionals, _positionals, src, outArg, wordsPath, formatRaw, format, pages, m, first, last, tuning, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, spec, raw, value, ok, dpi, raw1, value1, source, defaultExt, dest, onWarning, showProgress, onProgress, debugWordsPath, onWords, common, ocrCommon, words, skipped, ref, content, report,
|
|
293
|
+
var parsed, values, positionals, _positionals, src, outArg, wordsPath, formatRaw, format, pages, m, first, last, tuning, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, spec, raw, value, ok, dpi, raw1, value1, source, defaultExt, dest, onWarning, showProgress, onProgress, debugWordsPath, onWords, common, ocrCommon, words, skipped, ref, content, report, markdown, ref1, ref2, analysis, text, _report_floats;
|
|
321
294
|
return _ts_generator(this, function(_state) {
|
|
322
295
|
switch(_state.label){
|
|
323
296
|
case 0:
|
|
@@ -504,146 +477,114 @@ var extract = function extract(ctx) {
|
|
|
504
477
|
}
|
|
505
478
|
if (!(format === 'md')) return [
|
|
506
479
|
3,
|
|
507
|
-
|
|
508
|
-
];
|
|
509
|
-
return [
|
|
510
|
-
4,
|
|
511
|
-
Promise.resolve().then(function() {
|
|
512
|
-
return /*#__PURE__*/ _interop_require_wildcard(require("../extract.js"));
|
|
513
|
-
})
|
|
480
|
+
5
|
|
514
481
|
];
|
|
515
|
-
case 1:
|
|
516
|
-
_ref = _state.sent(), extractOcr = _ref.extractOcr, pdfToMarkdown = _ref.pdfToMarkdown;
|
|
517
482
|
if (!(words !== undefined)) return [
|
|
518
483
|
3,
|
|
519
|
-
|
|
484
|
+
2
|
|
520
485
|
];
|
|
521
486
|
return [
|
|
522
487
|
4,
|
|
523
|
-
extractOcr({
|
|
488
|
+
(0, _extractts.extractOcr)({
|
|
524
489
|
words: words
|
|
525
490
|
}, ocrCommon)
|
|
526
491
|
];
|
|
527
|
-
case
|
|
492
|
+
case 1:
|
|
528
493
|
ref1 = _state.sent(), markdown = ref1.markdown, report = ref1.report, ref1;
|
|
529
494
|
return [
|
|
530
495
|
3,
|
|
531
|
-
|
|
496
|
+
4
|
|
532
497
|
];
|
|
533
|
-
case
|
|
498
|
+
case 2:
|
|
534
499
|
return [
|
|
535
500
|
4,
|
|
536
|
-
pdfToMarkdown(src, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
501
|
+
(0, _extractts.pdfToMarkdown)(src, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
537
502
|
ocr: values.ocr,
|
|
538
503
|
noOcr: values['no-ocr'],
|
|
539
504
|
pageMarkers: values['page-markers']
|
|
540
505
|
}))
|
|
541
506
|
];
|
|
542
|
-
case
|
|
507
|
+
case 3:
|
|
543
508
|
ref2 = _state.sent(), markdown = ref2.markdown, report = ref2.report, ref2;
|
|
544
|
-
_state.label =
|
|
545
|
-
case
|
|
509
|
+
_state.label = 4;
|
|
510
|
+
case 4:
|
|
546
511
|
content = markdown;
|
|
547
512
|
return [
|
|
548
513
|
3,
|
|
549
|
-
|
|
550
|
-
];
|
|
551
|
-
case 6:
|
|
552
|
-
return [
|
|
553
|
-
4,
|
|
554
|
-
Promise.resolve().then(function() {
|
|
555
|
-
return /*#__PURE__*/ _interop_require_wildcard(require("../analyze.js"));
|
|
556
|
-
})
|
|
557
|
-
];
|
|
558
|
-
case 7:
|
|
559
|
-
analyze = _state.sent().analyze;
|
|
560
|
-
return [
|
|
561
|
-
4,
|
|
562
|
-
Promise.resolve().then(function() {
|
|
563
|
-
return /*#__PURE__*/ _interop_require_wildcard(require("../render-analysis.js"));
|
|
564
|
-
})
|
|
565
|
-
];
|
|
566
|
-
case 8:
|
|
567
|
-
toText = _state.sent().toText;
|
|
568
|
-
return [
|
|
569
|
-
4,
|
|
570
|
-
Promise.resolve().then(function() {
|
|
571
|
-
return /*#__PURE__*/ _interop_require_wildcard(require("../extract.js"));
|
|
572
|
-
})
|
|
514
|
+
14
|
|
573
515
|
];
|
|
574
|
-
case
|
|
575
|
-
_ref1 = _state.sent(), needsOcrFallback = _ref1.needsOcrFallback, ocrFallbackNotice = _ref1.ocrFallbackNotice;
|
|
516
|
+
case 5:
|
|
576
517
|
if (!(words !== undefined)) return [
|
|
577
518
|
3,
|
|
578
|
-
|
|
519
|
+
7
|
|
579
520
|
];
|
|
580
521
|
return [
|
|
581
522
|
4,
|
|
582
|
-
analyze({
|
|
523
|
+
(0, _analyzets.analyze)({
|
|
583
524
|
words: words
|
|
584
525
|
}, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
585
526
|
path: 'ocr'
|
|
586
527
|
}))
|
|
587
528
|
];
|
|
588
|
-
case
|
|
529
|
+
case 6:
|
|
589
530
|
analysis = _state.sent();
|
|
590
531
|
return [
|
|
591
532
|
3,
|
|
592
|
-
|
|
533
|
+
13
|
|
593
534
|
];
|
|
594
|
-
case
|
|
535
|
+
case 7:
|
|
595
536
|
if (!values.ocr) return [
|
|
596
537
|
3,
|
|
597
|
-
|
|
538
|
+
9
|
|
598
539
|
];
|
|
599
540
|
return [
|
|
600
541
|
4,
|
|
601
|
-
analyze(src, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
542
|
+
(0, _analyzets.analyze)(src, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
602
543
|
path: 'ocr'
|
|
603
544
|
}))
|
|
604
545
|
];
|
|
605
|
-
case
|
|
546
|
+
case 8:
|
|
606
547
|
analysis = _state.sent();
|
|
607
548
|
return [
|
|
608
549
|
3,
|
|
609
|
-
|
|
550
|
+
13
|
|
610
551
|
];
|
|
611
|
-
case
|
|
552
|
+
case 9:
|
|
612
553
|
return [
|
|
613
554
|
4,
|
|
614
|
-
analyze(src, _object_spread_props(_object_spread({}, common), {
|
|
555
|
+
(0, _analyzets.analyze)(src, _object_spread_props(_object_spread({}, common), {
|
|
615
556
|
path: 'text',
|
|
616
557
|
pageMarkers: values['page-markers']
|
|
617
558
|
}))
|
|
618
559
|
];
|
|
619
|
-
case
|
|
560
|
+
case 10:
|
|
620
561
|
text = _state.sent();
|
|
621
|
-
if (!(needsOcrFallback(text.report.charsPerPage) && !values['no-ocr'])) return [
|
|
562
|
+
if (!((0, _extractts.needsOcrFallback)(text.report.charsPerPage) && !values['no-ocr'])) return [
|
|
622
563
|
3,
|
|
623
|
-
|
|
564
|
+
12
|
|
624
565
|
];
|
|
625
|
-
onWarning(ocrFallbackNotice(text.report.charsPerPage));
|
|
566
|
+
onWarning((0, _extractts.ocrFallbackNotice)(text.report.charsPerPage));
|
|
626
567
|
return [
|
|
627
568
|
4,
|
|
628
|
-
analyze(src, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
569
|
+
(0, _analyzets.analyze)(src, _object_spread_props(_object_spread({}, ocrCommon), {
|
|
629
570
|
path: 'ocr',
|
|
630
571
|
fallback: true
|
|
631
572
|
}))
|
|
632
573
|
];
|
|
633
|
-
case
|
|
574
|
+
case 11:
|
|
634
575
|
analysis = _state.sent();
|
|
635
576
|
return [
|
|
636
577
|
3,
|
|
637
|
-
|
|
578
|
+
13
|
|
638
579
|
];
|
|
639
|
-
case
|
|
580
|
+
case 12:
|
|
640
581
|
analysis = text;
|
|
641
|
-
_state.label =
|
|
642
|
-
case
|
|
582
|
+
_state.label = 13;
|
|
583
|
+
case 13:
|
|
643
584
|
report = analysis.report;
|
|
644
|
-
content = format === 'raw' ? JSON.stringify(analysis, null, 1) : toText(analysis);
|
|
645
|
-
_state.label =
|
|
646
|
-
case
|
|
585
|
+
content = format === 'raw' ? JSON.stringify(analysis, null, 1) : (0, _renderanalysists.toText)(analysis);
|
|
586
|
+
_state.label = 14;
|
|
587
|
+
case 14:
|
|
647
588
|
if (values.stdout) process.stdout.write(content);
|
|
648
589
|
else (0, _nodefs.writeFileSync)(dest, content);
|
|
649
590
|
// The library's Report stays clean of source/out; --json consumers keep
|