@effortlessmotion/pdf-to-md 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -76
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/package.json +2 -8
- package/skills/pdf-to-md/SKILL.md +39 -66
package/dist/esm/analyze.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/analyze.ts"],"sourcesContent":["// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr\n// shares with it — same buildPageOCR -> emitPage -> renderPagesToMarkdown pipeline, so OCR-path markdown is byte-identical by construction, not two implementations kept in sync by hand. The TEXT path is NOT shared this way (see analyzeText).\nimport { collect, pageCount } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { EmitBlock } from './emit.ts';\nimport { emitPage, renderEmitBlock } from './emit.ts';\nimport { recognizePdf } from './engines/tesseract.ts';\n// Type-only: erased at compile time, so this does NOT create a runtime\n// circular dependency even though src/extract.ts imports (value-level) from this module.\nimport type { ExtractOcrOptions, ExtractTextOptions, OcrWordInput } from './extract.ts';\nimport type { PageOCR, Tuning, Word } from './geometry.ts';\nimport { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { buildOcrReport, buildReport, OUTPUT_VERSION } from './report.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord, Line, PageLines } from './types.ts';\nimport { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.ts';\n\n/** analyze(pdf) with the default/'text' path — OCR-only fields don't exist\n * on this branch, so passing e.g. `tuning` without `path: 'ocr'` is a compile error. */\nexport type AnalyzeTextOptions = ExtractTextOptions & { path?: 'text' };\n\n/** analyze(pdf, {path:'ocr'}) — same fields as {@link ExtractOcrOptions},\n * unlocked only once `path: 'ocr'` is set. */\nexport type AnalyzeOcrOptions = ExtractOcrOptions & { path: 'ocr' };\n\n/** Options for {@link analyze}: which fields type-check depends on `path`\n * (ignored, forced 'ocr', when `input` is a `{ words }` replay). */\nexport type AnalyzeOptions = AnalyzeTextOptions | AnalyzeOcrOptions;\n\nfunction toAnalysisWord(w: OcrWordInput): AnalysisWord {\n const box = { x: w.x, y: w.y, w: w.w, h: w.h };\n return w.confidence === undefined ? { text: w.text, box } : { text: w.text, box, confidence: w.confidence };\n}\n\n// EmitBlock -> AnalysisBlock: collapses emit.ts's 'quote' kind (a pull-quote)\n// into Analysis's 'paragraph' type, `text` keeping its literal '> ' prefix — see AnalysisBlock's doc comment (src/types.ts) for why.\nfunction toAnalysisBlock(b: EmitBlock, wordIndex: Map<Word, number>): AnalysisBlock {\n const wordIndexes = b.words.map((w) => wordIndex.get(w) as number);\n if (b.kind === 'heading') return { type: 'heading', text: b.text, level: b.level, wordIndexes };\n if (b.kind === 'review') return { type: 'review', text: renderEmitBlock(b), wordIndexes };\n return { type: 'paragraph', text: renderEmitBlock(b), wordIndexes };\n}\n\n/** The OCR-path core shared by extractOcr and analyze(..., {path:'ocr'}):\n * recognizes (or replays) words and runs the unchanged buildPageOCR/emitPage pipeline. Word IDENTITY (object reference), not value, carries indices through into the emitted blocks — see `geometry.ts`'s OcrLine.words. */\nexport async function buildOcrAnalysisPages(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions, warn: (m: string) => void): Promise<{ pages: AnalysisPage[]; words: OcrWordInput[] }> {\n const tuning: Tuning = { ...DEFAULT_TUNING, ...opts.tuning };\n const validated = validateOcrAnalysisInput(input);\n\n let words: OcrWordInput[];\n let first: number;\n let last: number;\n if ('words' in validated) {\n // Replay: page range from --pages if given, else every page up to the highest one the dump mentions.\n words = validated.words;\n first = opts.pages?.first ?? 1;\n last = opts.pages?.last ?? opts.pages?.first ?? Math.max(1, ...words.map((w) => w.page));\n } else {\n const totalPages = await pageCount(validated.pdfPath);\n const rawFirst = opts.pages?.first ?? 1;\n const rawLast = opts.pages?.last ?? (opts.pages ? rawFirst : totalPages);\n first = Math.min(Math.max(1, rawFirst), Math.max(1, totalPages));\n last = Math.min(Math.max(first, rawLast), totalPages);\n const result = await recognizePdf(validated.pdfPath, first, last, {\n dpi: opts.dpi,\n onProgress: opts.onProgress,\n onWarning: warn,\n });\n words = result.words;\n }\n\n opts.onWords?.(words);\n\n // Stable per-page index, assigned in recognition/replay order — this IS\n // the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).\n const byPage = new Map<number, OcrWordInput[]>();\n const wordIndex = new Map<Word, number>();\n for (const w of words) {\n const list = byPage.get(w.page);\n if (list) {\n wordIndex.set(w, list.length);\n list.push(w);\n } else {\n wordIndex.set(w, 0);\n byPage.set(w.page, [w]);\n }\n }\n\n const pageOcrs = new Map<number, PageOCR>();\n for (let pn = first; pn <= last; pn++) pageOcrs.set(pn, buildPageOCR(byPage.get(pn) ?? [], tuning));\n const { levelOf } = headingLevels([...pageOcrs.values()], tuning);\n\n const pages: AnalysisPage[] = [];\n for (let pn = first; pn <= last; pn++) {\n const p = pageOcrs.get(pn) as PageOCR;\n // An in-range page with no recognized words stays visible (marker +\n // warning), never silently absent — same rule as extractOcr.\n if (!byPage.has(pn)) warn(`p${pn}: no recognized words, page emitted with no OCR text`);\n\n const blocks: AnalysisBlock[] = emitPage(p, levelOf, tuning).map((b) => toAnalysisBlock(b, wordIndex));\n // Decorative fragments always render as one visible \"> [floats]\" aside\n // per page (matching the one printed line) — see geometry.ts's PageOCR.floats.\n if (p.floats.length) {\n blocks.push({\n type: 'float',\n text: `> [floats] ${p.floats.map((f) => f.text).join(' · ')}`,\n wordIndexes: p.floats.flatMap((f) => f.words).map((w) => wordIndex.get(w) as number),\n });\n }\n\n pages.push({\n page: pn,\n words: (byPage.get(pn) ?? []).map(toAnalysisWord),\n blocks,\n });\n }\n\n return { pages, words };\n}\n\nasync function analyzeOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md, fallback: opts.fallback, words });\n return { outputVersion: OUTPUT_VERSION, report, pages };\n}\n\n// TEXT path: deliberately NOT routed through the OCR shared-core pattern —\n// buildMarkdown's reflow/hyphen-rejoin/list rules don't fit the 4-type Analysis block union, so this is an independent, intentionally coarser reconstruction (one word/block per visual LINE) — honest, not byte-identical to extractText (see AnalysisPage, src/types.ts).\n\n// Converts a text-path Line's page-point geometry (src/lines.ts) into the\n// same normalized box contract the OCR path uses (see AnalysisPage.words) — only the coordinate SYSTEM changes; `box.y` anchors on the line's baseline, `box.w` stays 0.\nfunction textLineBox(l: Line, dims: { width: number; height: number }): AnalysisWord['box'] {\n const { width, height } = dims;\n return {\n x: width > 0 ? l.x / width : 0,\n y: height > 0 ? (height - l.y) / height : 0,\n w: 0,\n h: height > 0 ? l.h / height : 0,\n };\n}\n\nasync function analyzeText(pdfPath: string, opts: ExtractTextOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const { numPages, pages, heightChars } = await collect(pdfPath, range);\n const pageDims = new Map(pages.map((p) => [p.page, { width: p.width, height: p.height }]));\n const pageLines: PageLines[] = pages.map((p) => ({ page: p.page, lines: toLines(p) }));\n const pageMarkers = opts.pageMarkers ?? false;\n const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, { pageMarkers });\n const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, { pageMarkers: true }).md;\n\n const hybridPages = pages.filter((p) => p.largeImage).map((p) => p.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n\n // Report is measured from the REAL markdown above — accurate regardless of\n // the coarser block/word reconstruction below.\n const report = buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages });\n\n // headingSizes is offset-independent — reused directly, then re-leveled\n // here with offset 0 regardless of `opts.pageMarkers` (see AnalysisPage).\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1)]));\n const analysisPages: AnalysisPage[] = pageLines.map(({ page, lines }) => {\n const dims = pageDims.get(page) as { width: number; height: number };\n const words: AnalysisWord[] = lines.map((l) => ({ text: l.text, box: textLineBox(l, dims) }));\n const blocks: AnalysisBlock[] = lines.map((l, i) => {\n const level = levelOf.get(l.h);\n return level !== undefined ? { type: 'heading' as const, text: l.text, level, wordIndexes: [i] } : { type: 'paragraph' as const, text: l.text, wordIndexes: [i] };\n });\n return { page, words, blocks };\n });\n\n return { outputVersion: OUTPUT_VERSION, report, pages: analysisPages };\n}\n\n/**\n * Analyze a PDF (or replay an already-recognized OCR word dump) into\n * `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).\n *\n * @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`\n * @param opts see {@link AnalyzeOptions}; for a pdf path, `opts.path` picks 'text' (default) or 'ocr' — analyze() makes no auto-fallback decision (that's pdfToMarkdown's job)\n */\nexport async function analyze(input: string | { words: OcrWordInput[] }, opts: AnalyzeOptions = {}): Promise<Analysis> {\n validatePages(opts.pages);\n if (typeof input !== 'string') return analyzeOcr({ words: validateWordsInput(input) }, opts);\n if (opts.path === 'ocr') return analyzeOcr({ pdfPath: input }, opts);\n return analyzeText(input, opts);\n}\n"],"names":["collect","pageCount","dictAvailable","dictUnavailableWarning","emitPage","renderEmitBlock","recognizePdf","buildPageOCR","DEFAULT_TUNING","headingLevels","toLines","buildMarkdown","renderPagesToMarkdown","buildOcrReport","buildReport","OUTPUT_VERSION","validateOcrAnalysisInput","validatePages","validateWordsInput","toAnalysisWord","w","box","x","y","h","confidence","undefined","text","toAnalysisBlock","b","wordIndex","wordIndexes","words","map","get","kind","type","level","buildOcrAnalysisPages","input","opts","warn","byPage","tuning","validated","first","last","pages","Math","max","page","totalPages","pdfPath","rawFirst","rawLast","min","result","dpi","onProgress","onWarning","onWords","Map","list","set","length","push","pageOcrs","pn","levelOf","values","p","has","blocks","floats","f","join","flatMap","analyzeOcr","m","md","forcePageMarkers","pageMarkers","report","fallback","outputVersion","textLineBox","l","dims","width","height","analyzeText","range","numPages","heightChars","pageDims","pageLines","lines","stats","bodyH","headingSizes","statsMd","hybridPages","filter","largeImage","n","slice","i","analysisPages","analyze","path"],"mappings":"AAAA,2EAA2E;AAC3E,kPAAkP;AAClP,SAASA,OAAO,EAAEC,SAAS,QAAQ,eAAe;AAClD,SAASC,aAAa,EAAEC,sBAAsB,QAAQ,YAAY;AAElE,SAASC,QAAQ,EAAEC,eAAe,QAAQ,YAAY;AACtD,SAASC,YAAY,QAAQ,yBAAyB;AAKtD,SAASC,YAAY,EAAEC,cAAc,EAAEC,aAAa,QAAQ,gBAAgB;AAC5E,SAASC,OAAO,QAAQ,aAAa;AACrC,SAASC,aAAa,QAAQ,gBAAgB;AAC9C,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,cAAc,EAAEC,WAAW,EAAEC,cAAc,QAAQ,cAAc;AAE1E,SAASC,wBAAwB,EAAEC,aAAa,EAAEC,kBAAkB,QAAQ,gBAAgB;AAc5F,SAASC,eAAeC,CAAe;IACrC,MAAMC,MAAM;QAAEC,GAAGF,EAAEE,CAAC;QAAEC,GAAGH,EAAEG,CAAC;QAAEH,GAAGA,EAAEA,CAAC;QAAEI,GAAGJ,EAAEI,CAAC;IAAC;IAC7C,OAAOJ,EAAEK,UAAU,KAAKC,YAAY;QAAEC,MAAMP,EAAEO,IAAI;QAAEN;IAAI,IAAI;QAAEM,MAAMP,EAAEO,IAAI;QAAEN;QAAKI,YAAYL,EAAEK,UAAU;IAAC;AAC5G;AAEA,8EAA8E;AAC9E,qIAAqI;AACrI,SAASG,gBAAgBC,CAAY,EAAEC,SAA4B;IACjE,MAAMC,cAAcF,EAAEG,KAAK,CAACC,GAAG,CAAC,CAACb,IAAMU,UAAUI,GAAG,CAACd;IACrD,IAAIS,EAAEM,IAAI,KAAK,WAAW,OAAO;QAAEC,MAAM;QAAWT,MAAME,EAAEF,IAAI;QAAEU,OAAOR,EAAEQ,KAAK;QAAEN;IAAY;IAC9F,IAAIF,EAAEM,IAAI,KAAK,UAAU,OAAO;QAAEC,MAAM;QAAUT,MAAMtB,gBAAgBwB;QAAIE;IAAY;IACxF,OAAO;QAAEK,MAAM;QAAaT,MAAMtB,gBAAgBwB;QAAIE;IAAY;AACpE;AAEA;0NAC0N,GAC1N,OAAO,eAAeO,sBAAsBC,KAAsD,EAAEC,IAAuB,EAAEC,IAAyB;QA4C/EC;QAlBrEF;IAzBA,MAAMG,SAAiB;QAAE,GAAGnC,cAAc;QAAE,GAAGgC,KAAKG,MAAM;IAAC;IAC3D,MAAMC,YAAY5B,yBAAyBuB;IAE3C,IAAIP;IACJ,IAAIa;IACJ,IAAIC;IACJ,IAAI,WAAWF,WAAW;kBAIjBJ;YADCA,aACDA,cAAoBA;QAH3B,qGAAqG;QACrGR,QAAQY,UAAUZ,KAAK;QACvBa,iBAAQL,cAAAA,KAAKO,KAAK,cAAVP,kCAAAA,YAAYK,KAAK,uCAAI;QAC7BC,QAAON,kBAAAA,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYM,IAAI,0CAAIN,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYK,KAAK,cAArCL,mBAAAA,QAAyCQ,KAAKC,GAAG,CAAC,MAAMjB,MAAMC,GAAG,CAAC,CAACb,IAAMA,EAAE8B,IAAI;IACxF,OAAO;;YAEYV,cACDA;QAFhB,MAAMW,aAAa,MAAMlD,UAAU2C,UAAUQ,OAAO;QACpD,MAAMC,qBAAWb,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYK,KAAK,yCAAI;QACtC,MAAMS,oBAAUd,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYM,IAAI,yCAAKN,KAAKO,KAAK,GAAGM,WAAWF;QAC7DN,QAAQG,KAAKO,GAAG,CAACP,KAAKC,GAAG,CAAC,GAAGI,WAAWL,KAAKC,GAAG,CAAC,GAAGE;QACpDL,OAAOE,KAAKO,GAAG,CAACP,KAAKC,GAAG,CAACJ,OAAOS,UAAUH;QAC1C,MAAMK,SAAS,MAAMlD,aAAasC,UAAUQ,OAAO,EAAEP,OAAOC,MAAM;YAChEW,KAAKjB,KAAKiB,GAAG;YACbC,YAAYlB,KAAKkB,UAAU;YAC3BC,WAAWlB;QACb;QACAT,QAAQwB,OAAOxB,KAAK;IACtB;KAEAQ,gBAAAA,KAAKoB,OAAO,cAAZpB,oCAAAA,mBAAAA,MAAeR;IAEf,wEAAwE;IACxE,mJAAmJ;IACnJ,MAAMU,SAAS,IAAImB;IACnB,MAAM/B,YAAY,IAAI+B;IACtB,KAAK,MAAMzC,KAAKY,MAAO;QACrB,MAAM8B,OAAOpB,OAAOR,GAAG,CAACd,EAAE8B,IAAI;QAC9B,IAAIY,MAAM;YACRhC,UAAUiC,GAAG,CAAC3C,GAAG0C,KAAKE,MAAM;YAC5BF,KAAKG,IAAI,CAAC7C;QACZ,OAAO;YACLU,UAAUiC,GAAG,CAAC3C,GAAG;YACjBsB,OAAOqB,GAAG,CAAC3C,EAAE8B,IAAI,EAAE;gBAAC9B;aAAE;QACxB;IACF;IAEA,MAAM8C,WAAW,IAAIL;IACrB,IAAK,IAAIM,KAAKtB,OAAOsB,MAAMrB,MAAMqB,KAAMD,SAASH,GAAG,CAACI,IAAI5D,cAAamC,cAAAA,OAAOR,GAAG,CAACiC,iBAAXzB,yBAAAA,cAAkB,EAAE,EAAEC;IAC3F,MAAM,EAAEyB,OAAO,EAAE,GAAG3D,cAAc;WAAIyD,SAASG,MAAM;KAAG,EAAE1B;IAE1D,MAAMI,QAAwB,EAAE;IAChC,IAAK,IAAIoB,KAAKtB,OAAOsB,MAAMrB,MAAMqB,KAAM;YAmB3BzB;QAlBV,MAAM4B,IAAIJ,SAAShC,GAAG,CAACiC;QACvB,oEAAoE;QACpE,6DAA6D;QAC7D,IAAI,CAACzB,OAAO6B,GAAG,CAACJ,KAAK1B,KAAK,CAAC,CAAC,EAAE0B,GAAG,oDAAoD,CAAC;QAEtF,MAAMK,SAA0BpE,SAASkE,GAAGF,SAASzB,QAAQV,GAAG,CAAC,CAACJ,IAAMD,gBAAgBC,GAAGC;QAC3F,uEAAuE;QACvE,+EAA+E;QAC/E,IAAIwC,EAAEG,MAAM,CAACT,MAAM,EAAE;YACnBQ,OAAOP,IAAI,CAAC;gBACV7B,MAAM;gBACNT,MAAM,CAAC,WAAW,EAAE2C,EAAEG,MAAM,CAACxC,GAAG,CAAC,CAACyC,IAAMA,EAAE/C,IAAI,EAAEgD,IAAI,CAAC,QAAQ;gBAC7D5C,aAAauC,EAAEG,MAAM,CAACG,OAAO,CAAC,CAACF,IAAMA,EAAE1C,KAAK,EAAEC,GAAG,CAAC,CAACb,IAAMU,UAAUI,GAAG,CAACd;YACzE;QACF;QAEA2B,MAAMkB,IAAI,CAAC;YACTf,MAAMiB;YACNnC,OAAO,EAACU,eAAAA,OAAOR,GAAG,CAACiC,iBAAXzB,0BAAAA,eAAkB,EAAE,EAAET,GAAG,CAACd;YAClCqD;QACF;IACF;IAEA,OAAO;QAAEzB;QAAOf;IAAM;AACxB;AAEA,eAAe6C,WAAWtC,KAAsD,EAAEC,IAAuB;IACvG,MAAMC,OAAO,CAACqC;YAActC;gBAAAA,kBAAAA,KAAKmB,SAAS,cAAdnB,sCAAAA,qBAAAA,MAAiBsC;;IAC7C,IAAI,CAAC5E,iBAAiBuC,KAAKtC;IAC3B,MAAM,EAAE4C,KAAK,EAAEf,KAAK,EAAE,GAAG,MAAMM,sBAAsBC,OAAOC,MAAMC;IAClE,MAAMsC,KAAKnE,sBAAsBmC,OAAO;QAAEiC,kBAAkB;QAAMC,aAAa;IAAK;IACpF,MAAMC,SAASrE,eAAe;QAAEkE;QAAII,UAAU3C,KAAK2C,QAAQ;QAAEnD;IAAM;IACnE,OAAO;QAAEoD,eAAerE;QAAgBmE;QAAQnC;IAAM;AACxD;AAEA,2EAA2E;AAC3E,4QAA4Q;AAE5Q,0EAA0E;AAC1E,yKAAyK;AACzK,SAASsC,YAAYC,CAAO,EAAEC,IAAuC;IACnE,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,OAAO;QACLjE,GAAGkE,QAAQ,IAAIF,EAAEhE,CAAC,GAAGkE,QAAQ;QAC7BjE,GAAGkE,SAAS,IAAI,AAACA,CAAAA,SAASH,EAAE/D,CAAC,AAADA,IAAKkE,SAAS;QAC1CrE,GAAG;QACHI,GAAGiE,SAAS,IAAIH,EAAE9D,CAAC,GAAGiE,SAAS;IACjC;AACF;AAEA,eAAeC,YAAYtC,OAAe,EAAEZ,IAAwB;QAGNA,kBAIxCA;IANpB,MAAMC,OAAO,CAACqC;YAActC;gBAAAA,kBAAAA,KAAKmB,SAAS,cAAdnB,sCAAAA,qBAAAA,MAAiBsC;;IAC7C,IAAI,CAAC5E,iBAAiBuC,KAAKtC;IAC3B,MAAMwF,QAAQnD,KAAKO,KAAK,GAAG;QAAEF,OAAOL,KAAKO,KAAK,CAACF,KAAK;QAAEC,IAAI,GAAEN,mBAAAA,KAAKO,KAAK,CAACD,IAAI,cAAfN,8BAAAA,mBAAmBA,KAAKO,KAAK,CAACF,KAAK;IAAC,IAAI,CAAC;IACrG,MAAM,EAAE+C,QAAQ,EAAE7C,KAAK,EAAE8C,WAAW,EAAE,GAAG,MAAM7F,QAAQoD,SAASuC;IAChE,MAAMG,WAAW,IAAIjC,IAAId,MAAMd,GAAG,CAAC,CAACqC,IAAM;YAACA,EAAEpB,IAAI;YAAE;gBAAEsC,OAAOlB,EAAEkB,KAAK;gBAAEC,QAAQnB,EAAEmB,MAAM;YAAC;SAAE;IACxF,MAAMM,YAAyBhD,MAAMd,GAAG,CAAC,CAACqC,IAAO,CAAA;YAAEpB,MAAMoB,EAAEpB,IAAI;YAAE8C,OAAOtF,QAAQ4D;QAAG,CAAA;IACnF,MAAMW,eAAczC,oBAAAA,KAAKyC,WAAW,cAAhBzC,+BAAAA,oBAAoB;IACxC,MAAM,EAAEuC,EAAE,EAAEkB,KAAK,EAAEC,KAAK,EAAEC,YAAY,EAAE,GAAGxF,cAAcoF,WAAWF,aAAa;QAAEZ;IAAY;IAC/F,MAAMmB,UAAUnB,cAAcF,KAAKpE,cAAcoF,WAAWF,aAAa;QAAEZ,aAAa;IAAK,GAAGF,EAAE;IAElG,MAAMsB,cAActD,MAAMuD,MAAM,CAAC,CAAChC,IAAMA,EAAEiC,UAAU,EAAEtE,GAAG,CAAC,CAACqC,IAAMA,EAAEpB,IAAI;IACvE,KAAK,MAAMsD,KAAKH,YAAa;QAC3B5D,KAAK,CAAC,CAAC,EAAE+D,EAAE,uJAAuJ,EAAEA,EAAE,yCAAyC,CAAC;IAClN;IAEA,2EAA2E;IAC3E,+CAA+C;IAC/C,MAAMtB,SAASpE,YAAY;QAAE8E;QAAUb;QAAIqB;QAASH;QAAOC;QAAOC;QAAcE;IAAY;IAE5F,wEAAwE;IACxE,0EAA0E;IAC1E,MAAMjC,UAAU,IAAIP,IAAIsC,aAAaM,KAAK,CAAC,GAAG,GAAGxE,GAAG,CAAC,CAACT,GAAGkF,IAAM;YAAClF;YAAGwB,KAAKO,GAAG,CAAC,GAAGmD,IAAI;SAAG;IACtF,MAAMC,gBAAgCZ,UAAU9D,GAAG,CAAC,CAAC,EAAEiB,IAAI,EAAE8C,KAAK,EAAE;QAClE,MAAMT,OAAOO,SAAS5D,GAAG,CAACgB;QAC1B,MAAMlB,QAAwBgE,MAAM/D,GAAG,CAAC,CAACqD,IAAO,CAAA;gBAAE3D,MAAM2D,EAAE3D,IAAI;gBAAEN,KAAKgE,YAAYC,GAAGC;YAAM,CAAA;QAC1F,MAAMf,SAA0BwB,MAAM/D,GAAG,CAAC,CAACqD,GAAGoB;YAC5C,MAAMrE,QAAQ+B,QAAQlC,GAAG,CAACoD,EAAE9D,CAAC;YAC7B,OAAOa,UAAUX,YAAY;gBAAEU,MAAM;gBAAoBT,MAAM2D,EAAE3D,IAAI;gBAAEU;gBAAON,aAAa;oBAAC2E;iBAAE;YAAC,IAAI;gBAAEtE,MAAM;gBAAsBT,MAAM2D,EAAE3D,IAAI;gBAAEI,aAAa;oBAAC2E;iBAAE;YAAC;QAClK;QACA,OAAO;YAAExD;YAAMlB;YAAOwC;QAAO;IAC/B;IAEA,OAAO;QAAEY,eAAerE;QAAgBmE;QAAQnC,OAAO4D;IAAc;AACvE;AAEA;;;;;;CAMC,GACD,OAAO,eAAeC,QAAQrE,KAAyC,EAAEC,OAAuB,CAAC,CAAC;IAChGvB,cAAcuB,KAAKO,KAAK;IACxB,IAAI,OAAOR,UAAU,UAAU,OAAOsC,WAAW;QAAE7C,OAAOd,mBAAmBqB;IAAO,GAAGC;IACvF,IAAIA,KAAKqE,IAAI,KAAK,OAAO,OAAOhC,WAAW;QAAEzB,SAASb;IAAM,GAAGC;IAC/D,OAAOkD,YAAYnD,OAAOC;AAC5B"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/analyze.ts"],"sourcesContent":["// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr\n// shares with it. Text analysis uses text-analysis.ts so it captures the same\n// blocks, source runs, and report input as extractText.\nimport { collect, pageCount } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { EmitBlock } from './emit.ts';\nimport { emitPage, renderEmitBlock } from './emit.ts';\nimport { recognizePdf } from './engines/tesseract.ts';\n// Type-only: erased at compile time, so this does NOT create a runtime\n// circular dependency even though src/extract.ts imports (value-level) from this module.\nimport type { ExtractOcrOptions, ExtractTextOptions, OcrWordInput } from './extract.ts';\nimport type { PageOCR, Tuning, Word } from './geometry.ts';\nimport { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { buildOcrReport, OUTPUT_VERSION } from './report.ts';\nimport { buildTextAnalysis } from './text-analysis.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage, AnalysisWord } from './types.ts';\nimport { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.ts';\n\n/** analyze(pdf) with the default/'text' path — OCR-only fields don't exist\n * on this branch, so passing e.g. `tuning` without `path: 'ocr'` is a compile error. */\nexport type AnalyzeTextOptions = ExtractTextOptions & { path?: 'text' };\n\n/** analyze(pdf, {path:'ocr'}) — same fields as {@link ExtractOcrOptions},\n * unlocked only once `path: 'ocr'` is set. */\nexport type AnalyzeOcrOptions = ExtractOcrOptions & { path: 'ocr' };\n\n/** Options for {@link analyze}: which fields type-check depends on `path`\n * (ignored, forced 'ocr', when `input` is a `{ words }` replay). */\nexport type AnalyzeOptions = AnalyzeTextOptions | AnalyzeOcrOptions;\n\nfunction toAnalysisWord(w: OcrWordInput): AnalysisWord {\n const box = { x: w.x, y: w.y, w: w.w, h: w.h };\n return w.confidence === undefined ? { text: w.text, box } : { text: w.text, box, confidence: w.confidence };\n}\n\n// EmitBlock -> AnalysisBlock: collapses emit.ts's 'quote' kind (a pull-quote)\n// into Analysis's 'paragraph' type, `text` keeping its literal '> ' prefix — see AnalysisBlock's doc comment (src/types.ts) for why.\nfunction toAnalysisBlock(b: EmitBlock, wordIndex: Map<Word, number>): AnalysisBlock {\n const wordIndexes = b.words.map((w) => wordIndex.get(w) as number);\n if (b.kind === 'heading') return { type: 'heading', text: b.text, level: b.level, wordIndexes };\n if (b.kind === 'review') return { type: 'review', text: renderEmitBlock(b), wordIndexes };\n return { type: 'paragraph', text: renderEmitBlock(b), wordIndexes };\n}\n\n/** The OCR-path core shared by extractOcr and analyze(..., {path:'ocr'}):\n * recognizes (or replays) words and runs the unchanged buildPageOCR/emitPage pipeline. Word IDENTITY (object reference), not value, carries indices through into the emitted blocks — see `geometry.ts`'s OcrLine.words. */\nexport async function buildOcrAnalysisPages(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions, warn: (m: string) => void): Promise<{ pages: AnalysisPage[]; words: OcrWordInput[] }> {\n const tuning: Tuning = { ...DEFAULT_TUNING, ...opts.tuning };\n const validated = validateOcrAnalysisInput(input);\n\n let words: OcrWordInput[];\n let first: number;\n let last: number;\n if ('words' in validated) {\n // Replay: page range from --pages if given, else every page up to the highest one the dump mentions.\n words = validated.words;\n first = opts.pages?.first ?? 1;\n last = opts.pages?.last ?? opts.pages?.first ?? Math.max(1, ...words.map((w) => w.page));\n } else {\n const totalPages = await pageCount(validated.pdfPath);\n const rawFirst = opts.pages?.first ?? 1;\n const rawLast = opts.pages?.last ?? (opts.pages ? rawFirst : totalPages);\n first = Math.min(Math.max(1, rawFirst), Math.max(1, totalPages));\n last = Math.min(Math.max(first, rawLast), totalPages);\n const result = await recognizePdf(validated.pdfPath, first, last, {\n dpi: opts.dpi,\n onProgress: opts.onProgress,\n onWarning: warn,\n });\n words = result.words;\n }\n\n words = words.filter((word) => word.page >= first && word.page <= last);\n opts.onWords?.(words);\n\n // Stable per-page index, assigned in recognition/replay order — this IS\n // the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).\n const byPage = new Map<number, OcrWordInput[]>();\n const wordIndex = new Map<Word, number>();\n for (const w of words) {\n const list = byPage.get(w.page);\n if (list) {\n wordIndex.set(w, list.length);\n list.push(w);\n } else {\n wordIndex.set(w, 0);\n byPage.set(w.page, [w]);\n }\n }\n\n const pageOcrs = new Map<number, PageOCR>();\n for (let pn = first; pn <= last; pn++) pageOcrs.set(pn, buildPageOCR(byPage.get(pn) ?? [], tuning));\n const { levelOf } = headingLevels([...pageOcrs.values()], tuning);\n\n const pages: AnalysisPage[] = [];\n for (let pn = first; pn <= last; pn++) {\n const p = pageOcrs.get(pn) as PageOCR;\n // An in-range page with no recognized words stays visible (marker +\n // warning), never silently absent — same rule as extractOcr.\n if (!byPage.has(pn)) warn(`p${pn}: no recognized words, page emitted with no OCR text`);\n\n const blocks: AnalysisBlock[] = emitPage(p, levelOf, tuning).map((b) => toAnalysisBlock(b, wordIndex));\n // Decorative fragments always render as one visible \"> [floats]\" aside\n // per page (matching the one printed line) — see geometry.ts's PageOCR.floats.\n if (p.floats.length) {\n blocks.push({\n type: 'float',\n text: `> [floats] ${p.floats.map((f) => f.text).join(' · ')}`,\n wordIndexes: p.floats.flatMap((f) => f.words).map((w) => wordIndex.get(w) as number),\n });\n }\n\n pages.push({\n page: pn,\n path: 'ocr',\n words: (byPage.get(pn) ?? []).map(toAnalysisWord),\n blocks,\n });\n }\n\n return { pages, words };\n}\n\nasync function analyzeOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md, fallback: opts.fallback, words });\n return { outputVersion: OUTPUT_VERSION, report, pages };\n}\n\nasync function analyzeText(pdfPath: string, opts: ExtractTextOptions): Promise<Analysis> {\n const warn = (m: string) => opts.onWarning?.(m);\n if (!dictAvailable()) warn(dictUnavailableWarning());\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const collected = await collect(pdfPath, range);\n const result = buildTextAnalysis(collected, opts.pageMarkers ?? false);\n const hybridPages = collected.pages.filter((page) => page.largeImage).map((page) => page.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n for (const page of collected.pages.filter((page) => page.textChars === 0 && !page.hasImage)) {\n warn(`p${page.page}: no extractable text; the page may be blank or contain only vector content`);\n }\n opts.onWords?.(result.words);\n return result.analysis;\n}\n\n/**\n * Analyze a PDF (or replay an already-recognized OCR word dump) into\n * `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).\n *\n * @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`\n * @param opts see {@link AnalyzeOptions}; for a PDF path, `path` explicitly\n * selects text (default) or OCR. Automatic per-page routing belongs to\n * pdfToMarkdown and the CLI output formats.\n */\nexport async function analyze(input: string | { words: OcrWordInput[] }, opts: AnalyzeOptions = {}): Promise<Analysis> {\n validatePages(opts.pages);\n if (typeof input !== 'string') return analyzeOcr({ words: validateWordsInput(input) }, opts);\n if (opts.path === 'ocr') return analyzeOcr({ pdfPath: input }, opts);\n return analyzeText(input, opts);\n}\n"],"names":["collect","pageCount","dictAvailable","dictUnavailableWarning","emitPage","renderEmitBlock","recognizePdf","buildPageOCR","DEFAULT_TUNING","headingLevels","renderPagesToMarkdown","buildOcrReport","OUTPUT_VERSION","buildTextAnalysis","validateOcrAnalysisInput","validatePages","validateWordsInput","toAnalysisWord","w","box","x","y","h","confidence","undefined","text","toAnalysisBlock","b","wordIndex","wordIndexes","words","map","get","kind","type","level","buildOcrAnalysisPages","input","opts","warn","byPage","tuning","validated","first","last","pages","Math","max","page","totalPages","pdfPath","rawFirst","rawLast","min","result","dpi","onProgress","onWarning","filter","word","onWords","Map","list","set","length","push","pageOcrs","pn","levelOf","values","p","has","blocks","floats","f","join","flatMap","path","analyzeOcr","m","md","forcePageMarkers","pageMarkers","report","fallback","outputVersion","analyzeText","range","collected","hybridPages","largeImage","n","textChars","hasImage","analysis","analyze"],"mappings":"AAAA,2EAA2E;AAC3E,8EAA8E;AAC9E,wDAAwD;AACxD,SAASA,OAAO,EAAEC,SAAS,QAAQ,eAAe;AAClD,SAASC,aAAa,EAAEC,sBAAsB,QAAQ,YAAY;AAElE,SAASC,QAAQ,EAAEC,eAAe,QAAQ,YAAY;AACtD,SAASC,YAAY,QAAQ,yBAAyB;AAKtD,SAASC,YAAY,EAAEC,cAAc,EAAEC,aAAa,QAAQ,gBAAgB;AAC5E,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,cAAc,EAAEC,cAAc,QAAQ,cAAc;AAC7D,SAASC,iBAAiB,QAAQ,qBAAqB;AAEvD,SAASC,wBAAwB,EAAEC,aAAa,EAAEC,kBAAkB,QAAQ,gBAAgB;AAc5F,SAASC,eAAeC,CAAe;IACrC,MAAMC,MAAM;QAAEC,GAAGF,EAAEE,CAAC;QAAEC,GAAGH,EAAEG,CAAC;QAAEH,GAAGA,EAAEA,CAAC;QAAEI,GAAGJ,EAAEI,CAAC;IAAC;IAC7C,OAAOJ,EAAEK,UAAU,KAAKC,YAAY;QAAEC,MAAMP,EAAEO,IAAI;QAAEN;IAAI,IAAI;QAAEM,MAAMP,EAAEO,IAAI;QAAEN;QAAKI,YAAYL,EAAEK,UAAU;IAAC;AAC5G;AAEA,8EAA8E;AAC9E,qIAAqI;AACrI,SAASG,gBAAgBC,CAAY,EAAEC,SAA4B;IACjE,MAAMC,cAAcF,EAAEG,KAAK,CAACC,GAAG,CAAC,CAACb,IAAMU,UAAUI,GAAG,CAACd;IACrD,IAAIS,EAAEM,IAAI,KAAK,WAAW,OAAO;QAAEC,MAAM;QAAWT,MAAME,EAAEF,IAAI;QAAEU,OAAOR,EAAEQ,KAAK;QAAEN;IAAY;IAC9F,IAAIF,EAAEM,IAAI,KAAK,UAAU,OAAO;QAAEC,MAAM;QAAUT,MAAMpB,gBAAgBsB;QAAIE;IAAY;IACxF,OAAO;QAAEK,MAAM;QAAaT,MAAMpB,gBAAgBsB;QAAIE;IAAY;AACpE;AAEA;0NAC0N,GAC1N,OAAO,eAAeO,sBAAsBC,KAAsD,EAAEC,IAAuB,EAAEC,IAAyB;QA6C/EC;QAlBrEF;IA1BA,MAAMG,SAAiB;QAAE,GAAGjC,cAAc;QAAE,GAAG8B,KAAKG,MAAM;IAAC;IAC3D,MAAMC,YAAY5B,yBAAyBuB;IAE3C,IAAIP;IACJ,IAAIa;IACJ,IAAIC;IACJ,IAAI,WAAWF,WAAW;kBAIjBJ;YADCA,aACDA,cAAoBA;QAH3B,qGAAqG;QACrGR,QAAQY,UAAUZ,KAAK;QACvBa,iBAAQL,cAAAA,KAAKO,KAAK,cAAVP,kCAAAA,YAAYK,KAAK,uCAAI;QAC7BC,QAAON,kBAAAA,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYM,IAAI,0CAAIN,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYK,KAAK,cAArCL,mBAAAA,QAAyCQ,KAAKC,GAAG,CAAC,MAAMjB,MAAMC,GAAG,CAAC,CAACb,IAAMA,EAAE8B,IAAI;IACxF,OAAO;;YAEYV,cACDA;QAFhB,MAAMW,aAAa,MAAMhD,UAAUyC,UAAUQ,OAAO;QACpD,MAAMC,qBAAWb,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYK,KAAK,yCAAI;QACtC,MAAMS,oBAAUd,eAAAA,KAAKO,KAAK,cAAVP,mCAAAA,aAAYM,IAAI,yCAAKN,KAAKO,KAAK,GAAGM,WAAWF;QAC7DN,QAAQG,KAAKO,GAAG,CAACP,KAAKC,GAAG,CAAC,GAAGI,WAAWL,KAAKC,GAAG,CAAC,GAAGE;QACpDL,OAAOE,KAAKO,GAAG,CAACP,KAAKC,GAAG,CAACJ,OAAOS,UAAUH;QAC1C,MAAMK,SAAS,MAAMhD,aAAaoC,UAAUQ,OAAO,EAAEP,OAAOC,MAAM;YAChEW,KAAKjB,KAAKiB,GAAG;YACbC,YAAYlB,KAAKkB,UAAU;YAC3BC,WAAWlB;QACb;QACAT,QAAQwB,OAAOxB,KAAK;IACtB;IAEAA,QAAQA,MAAM4B,MAAM,CAAC,CAACC,OAASA,KAAKX,IAAI,IAAIL,SAASgB,KAAKX,IAAI,IAAIJ;KAClEN,gBAAAA,KAAKsB,OAAO,cAAZtB,oCAAAA,mBAAAA,MAAeR;IAEf,wEAAwE;IACxE,mJAAmJ;IACnJ,MAAMU,SAAS,IAAIqB;IACnB,MAAMjC,YAAY,IAAIiC;IACtB,KAAK,MAAM3C,KAAKY,MAAO;QACrB,MAAMgC,OAAOtB,OAAOR,GAAG,CAACd,EAAE8B,IAAI;QAC9B,IAAIc,MAAM;YACRlC,UAAUmC,GAAG,CAAC7C,GAAG4C,KAAKE,MAAM;YAC5BF,KAAKG,IAAI,CAAC/C;QACZ,OAAO;YACLU,UAAUmC,GAAG,CAAC7C,GAAG;YACjBsB,OAAOuB,GAAG,CAAC7C,EAAE8B,IAAI,EAAE;gBAAC9B;aAAE;QACxB;IACF;IAEA,MAAMgD,WAAW,IAAIL;IACrB,IAAK,IAAIM,KAAKxB,OAAOwB,MAAMvB,MAAMuB,KAAMD,SAASH,GAAG,CAACI,IAAI5D,cAAaiC,cAAAA,OAAOR,GAAG,CAACmC,iBAAX3B,yBAAAA,cAAkB,EAAE,EAAEC;IAC3F,MAAM,EAAE2B,OAAO,EAAE,GAAG3D,cAAc;WAAIyD,SAASG,MAAM;KAAG,EAAE5B;IAE1D,MAAMI,QAAwB,EAAE;IAChC,IAAK,IAAIsB,KAAKxB,OAAOwB,MAAMvB,MAAMuB,KAAM;YAoB3B3B;QAnBV,MAAM8B,IAAIJ,SAASlC,GAAG,CAACmC;QACvB,oEAAoE;QACpE,6DAA6D;QAC7D,IAAI,CAAC3B,OAAO+B,GAAG,CAACJ,KAAK5B,KAAK,CAAC,CAAC,EAAE4B,GAAG,oDAAoD,CAAC;QAEtF,MAAMK,SAA0BpE,SAASkE,GAAGF,SAAS3B,QAAQV,GAAG,CAAC,CAACJ,IAAMD,gBAAgBC,GAAGC;QAC3F,uEAAuE;QACvE,+EAA+E;QAC/E,IAAI0C,EAAEG,MAAM,CAACT,MAAM,EAAE;YACnBQ,OAAOP,IAAI,CAAC;gBACV/B,MAAM;gBACNT,MAAM,CAAC,WAAW,EAAE6C,EAAEG,MAAM,CAAC1C,GAAG,CAAC,CAAC2C,IAAMA,EAAEjD,IAAI,EAAEkD,IAAI,CAAC,QAAQ;gBAC7D9C,aAAayC,EAAEG,MAAM,CAACG,OAAO,CAAC,CAACF,IAAMA,EAAE5C,KAAK,EAAEC,GAAG,CAAC,CAACb,IAAMU,UAAUI,GAAG,CAACd;YACzE;QACF;QAEA2B,MAAMoB,IAAI,CAAC;YACTjB,MAAMmB;YACNU,MAAM;YACN/C,OAAO,EAACU,eAAAA,OAAOR,GAAG,CAACmC,iBAAX3B,0BAAAA,eAAkB,EAAE,EAAET,GAAG,CAACd;YAClCuD;QACF;IACF;IAEA,OAAO;QAAE3B;QAAOf;IAAM;AACxB;AAEA,eAAegD,WAAWzC,KAAsD,EAAEC,IAAuB;IACvG,MAAMC,OAAO,CAACwC;YAAczC;gBAAAA,kBAAAA,KAAKmB,SAAS,cAAdnB,sCAAAA,qBAAAA,MAAiByC;;IAC7C,IAAI,CAAC7E,iBAAiBqC,KAAKpC;IAC3B,MAAM,EAAE0C,KAAK,EAAEf,KAAK,EAAE,GAAG,MAAMM,sBAAsBC,OAAOC,MAAMC;IAClE,MAAMyC,KAAKtE,sBAAsBmC,OAAO;QAAEoC,kBAAkB;QAAMC,aAAa;IAAK;IACpF,MAAMC,SAASxE,eAAe;QAAEqE;QAAII,UAAU9C,KAAK8C,QAAQ;QAAEtD;IAAM;IACnE,OAAO;QAAEuD,eAAezE;QAAgBuE;QAAQtC;IAAM;AACxD;AAEA,eAAeyC,YAAYpC,OAAe,EAAEZ,IAAwB;QAGNA,kBAEhBA;QAQ5CA;IAZA,MAAMC,OAAO,CAACwC;YAAczC;gBAAAA,kBAAAA,KAAKmB,SAAS,cAAdnB,sCAAAA,qBAAAA,MAAiByC;;IAC7C,IAAI,CAAC7E,iBAAiBqC,KAAKpC;IAC3B,MAAMoF,QAAQjD,KAAKO,KAAK,GAAG;QAAEF,OAAOL,KAAKO,KAAK,CAACF,KAAK;QAAEC,IAAI,GAAEN,mBAAAA,KAAKO,KAAK,CAACD,IAAI,cAAfN,8BAAAA,mBAAmBA,KAAKO,KAAK,CAACF,KAAK;IAAC,IAAI,CAAC;IACrG,MAAM6C,YAAY,MAAMxF,QAAQkD,SAASqC;IACzC,MAAMjC,SAASzC,kBAAkB2E,YAAWlD,oBAAAA,KAAK4C,WAAW,cAAhB5C,+BAAAA,oBAAoB;IAChE,MAAMmD,cAAcD,UAAU3C,KAAK,CAACa,MAAM,CAAC,CAACV,OAASA,KAAK0C,UAAU,EAAE3D,GAAG,CAAC,CAACiB,OAASA,KAAKA,IAAI;IAC7F,KAAK,MAAM2C,KAAKF,YAAa;QAC3BlD,KAAK,CAAC,CAAC,EAAEoD,EAAE,uJAAuJ,EAAEA,EAAE,yCAAyC,CAAC;IAClN;IACA,KAAK,MAAM3C,QAAQwC,UAAU3C,KAAK,CAACa,MAAM,CAAC,CAACV,OAASA,KAAK4C,SAAS,KAAK,KAAK,CAAC5C,KAAK6C,QAAQ,EAAG;QAC3FtD,KAAK,CAAC,CAAC,EAAES,KAAKA,IAAI,CAAC,2EAA2E,CAAC;IACjG;KACAV,gBAAAA,KAAKsB,OAAO,cAAZtB,oCAAAA,mBAAAA,MAAegB,OAAOxB,KAAK;IAC3B,OAAOwB,OAAOwC,QAAQ;AACxB;AAEA;;;;;;;;CAQC,GACD,OAAO,eAAeC,QAAQ1D,KAAyC,EAAEC,OAAuB,CAAC,CAAC;IAChGvB,cAAcuB,KAAKO,KAAK;IACxB,IAAI,OAAOR,UAAU,UAAU,OAAOyC,WAAW;QAAEhD,OAAOd,mBAAmBqB;IAAO,GAAGC;IACvF,IAAIA,KAAKuC,IAAI,KAAK,OAAO,OAAOC,WAAW;QAAE5B,SAASb;IAAM,GAAGC;IAC/D,OAAOgD,YAAYjD,OAAOC;AAC5B"}
|
package/dist/esm/audit.d.ts
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
export interface AuditResult {
|
|
3
3
|
/** JSON lines successfully parsed from the word dump */
|
|
4
4
|
parsed: number;
|
|
5
|
-
/** lines in the dump that were not parseable JSON
|
|
6
|
-
*
|
|
5
|
+
/** lines in the dump that were not parseable JSON; auditWords rejects
|
|
6
|
+
* evidence with any such line */
|
|
7
7
|
nonJsonSkipped: number;
|
|
8
8
|
/** distinct page numbers present in the dump */
|
|
9
9
|
pages: number;
|
|
@@ -13,7 +13,7 @@ export interface AuditResult {
|
|
|
13
13
|
* hyphen-join/split repair is applied; always `>= missing` */
|
|
14
14
|
strictDeficits: number;
|
|
15
15
|
/** words genuinely absent from the markdown after repair-checking; `0` is
|
|
16
|
-
* the only passing value (`pdf-to-md audit` exits
|
|
16
|
+
* the only passing value (`pdf-to-md audit` exits nonzero otherwise) */
|
|
17
17
|
missing: number;
|
|
18
18
|
/** per-page detail for every page with at least one missing word (capped
|
|
19
19
|
* sample of the missing tokens per page) */
|
|
@@ -29,8 +29,9 @@ export interface AuditResult {
|
|
|
29
29
|
lines: string[];
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
|
-
* Audit a --debug-words dump against a
|
|
33
|
-
* must land
|
|
32
|
+
* Audit a --debug-words dump against a Markdown file. Every recognized word
|
|
33
|
+
* must land on its page. Invalid evidence or absent page provenance throws a
|
|
34
|
+
* PdfToMdError with code `AUDIT_INPUT`.
|
|
34
35
|
*
|
|
35
36
|
* @param wordsPath path to a --debug-words JSON-lines dump (`{page, text, ...}` per line)
|
|
36
37
|
* @param mdPath path to the markdown file to check the dump against
|
package/dist/esm/audit.js
CHANGED
|
@@ -2,14 +2,40 @@
|
|
|
2
2
|
// shipped as part of the product, not just a path into the source tree.
|
|
3
3
|
//
|
|
4
4
|
// Every recognized word (a --debug-words dump, or an Analysis page's
|
|
5
|
-
// `words`) must appear in its page's
|
|
5
|
+
// `words`) must appear in its page's Markdown. Matching consumes evidence so
|
|
6
|
+
// a single output occurrence cannot satisfy repeated source occurrences.
|
|
6
7
|
//
|
|
7
8
|
// NOTE: no producer guarantees JSON key order in a --debug-words dump —
|
|
8
9
|
// parse, never grep.
|
|
9
10
|
import { readFileSync } from 'node:fs';
|
|
11
|
+
import { PdfToMdError } from './errors.js';
|
|
12
|
+
import { foldTypographic } from './lines.js';
|
|
10
13
|
import { bodyByPage } from './page-marker.js';
|
|
11
|
-
|
|
12
|
-
|
|
14
|
+
const WORD_RE = /[\p{L}\p{N}][\p{L}\p{N}\p{M}]*/gu;
|
|
15
|
+
const HYPHEN_RE = /[\p{Pd}\u2212]\s*$/u;
|
|
16
|
+
const LINE_HYPHEN_RE = /^[\p{Pd}\u2212][ \t]*\r?\n[ \t]*$/u;
|
|
17
|
+
const INLINE_HYPHEN_RE = /^[\p{Pd}\u2212][ \t]*$/u;
|
|
18
|
+
function tokens(text) {
|
|
19
|
+
const folded = foldTypographic(text);
|
|
20
|
+
WORD_RE.lastIndex = 0;
|
|
21
|
+
const matches = [
|
|
22
|
+
...folded.matchAll(WORD_RE)
|
|
23
|
+
];
|
|
24
|
+
return matches.map((match, index)=>{
|
|
25
|
+
var _ref;
|
|
26
|
+
const start = match.index;
|
|
27
|
+
const end = start + match[0].length;
|
|
28
|
+
const next = matches[index + 1];
|
|
29
|
+
return {
|
|
30
|
+
text: match[0].normalize('NFC').toLowerCase(),
|
|
31
|
+
start,
|
|
32
|
+
end,
|
|
33
|
+
gapAfter: folded.slice(end, (_ref = next === null || next === void 0 ? void 0 : next.index) !== null && _ref !== void 0 ? _ref : folded.length)
|
|
34
|
+
};
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
function markdownText(md) {
|
|
38
|
+
return md.replace(/<!--[\s\S]*?-->/gu, ' ').replace(/!\[[^\]]*\]\([^)]*\)/gu, ' ').replace(/!\[[^\]]*\]\[[^\]]*\]/gu, ' ').replace(/<img\b[^>]*>/giu, ' ').replace(/<a\b[^>]*>/giu, ' ').replace(/<\/a>/giu, ' ').replace(/^\s*\[[^\]]+\]:[^\n]*$/gmu, ' ').replace(/^>\s*\[(?:review:[^\]]*|floats)\]\s*/gmu, '').replace(/\[([^\]]*)\]\([^)]*\)/gu, '$1').replace(/\[([^\]]*)\]\[[^\]]*\]/gu, '$1');
|
|
13
39
|
}
|
|
14
40
|
function counter(items) {
|
|
15
41
|
var _m_get;
|
|
@@ -18,8 +44,9 @@ function counter(items) {
|
|
|
18
44
|
return m;
|
|
19
45
|
}
|
|
20
46
|
/**
|
|
21
|
-
* Audit a --debug-words dump against a
|
|
22
|
-
* must land
|
|
47
|
+
* Audit a --debug-words dump against a Markdown file. Every recognized word
|
|
48
|
+
* must land on its page. Invalid evidence or absent page provenance throws a
|
|
49
|
+
* PdfToMdError with code `AUDIT_INPUT`.
|
|
23
50
|
*
|
|
24
51
|
* @param wordsPath path to a --debug-words JSON-lines dump (`{page, text, ...}` per line)
|
|
25
52
|
* @param mdPath path to the markdown file to check the dump against
|
|
@@ -27,15 +54,18 @@ function counter(items) {
|
|
|
27
54
|
const rec = new Map();
|
|
28
55
|
let parsed = 0;
|
|
29
56
|
let skipped = 0;
|
|
57
|
+
const errors = [];
|
|
58
|
+
const runsByPage = new Map();
|
|
30
59
|
const wordsRaw = readFileSync(wordsPath, 'utf8');
|
|
31
60
|
// A trailing "\n" at EOF must not yield a spurious final empty line; a
|
|
32
61
|
// genuine blank line mid-file does count, as skipped.
|
|
33
62
|
const rawLines = wordsRaw.split('\n');
|
|
34
63
|
if (rawLines.length > 0 && rawLines[rawLines.length - 1] === '') rawLines.pop();
|
|
35
|
-
for (
|
|
36
|
-
line =
|
|
64
|
+
for (const [lineIndex, rawLine] of rawLines.entries()){
|
|
65
|
+
const line = rawLine.trim();
|
|
37
66
|
if (!line.startsWith('{')) {
|
|
38
67
|
skipped++;
|
|
68
|
+
errors.push(`line ${lineIndex + 1} is not valid JSON`);
|
|
39
69
|
continue;
|
|
40
70
|
}
|
|
41
71
|
let o;
|
|
@@ -43,18 +73,87 @@ function counter(items) {
|
|
|
43
73
|
o = JSON.parse(line);
|
|
44
74
|
} catch {
|
|
45
75
|
skipped++;
|
|
76
|
+
errors.push(`line ${lineIndex + 1} is not valid JSON`);
|
|
46
77
|
continue;
|
|
47
78
|
}
|
|
48
79
|
parsed++;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
80
|
+
if (!isAuditRecord(o)) {
|
|
81
|
+
errors.push(`line ${lineIndex + 1} must contain a positive integer page and non-empty text`);
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
let evidence = rec.get(o.page);
|
|
85
|
+
if (!evidence) {
|
|
86
|
+
evidence = {
|
|
87
|
+
tokens: [],
|
|
88
|
+
sourcePairs: []
|
|
89
|
+
};
|
|
90
|
+
rec.set(o.page, evidence);
|
|
91
|
+
}
|
|
92
|
+
const runTokenDetails = tokens(o.text);
|
|
93
|
+
const runTokens = runTokenDetails.map((token)=>token.text);
|
|
94
|
+
const start = evidence.tokens.length;
|
|
95
|
+
evidence.tokens.push(...runTokens);
|
|
96
|
+
for(let index = 0; index + 1 < runTokenDetails.length; index++){
|
|
97
|
+
if (INLINE_HYPHEN_RE.test(runTokenDetails[index].gapAfter)) {
|
|
98
|
+
evidence.sourcePairs.push({
|
|
99
|
+
first: start + index,
|
|
100
|
+
second: start + index + 1,
|
|
101
|
+
explicitHyphen: true
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const run = {
|
|
106
|
+
page: o.page,
|
|
107
|
+
tokens: runTokens.map((_, index)=>start + index),
|
|
108
|
+
terminalHyphen: HYPHEN_RE.test(o.text),
|
|
109
|
+
hyphenOnly: /^[\p{Pd}\u2212]\s*$/u.test(o.text),
|
|
110
|
+
leadingWhitespace: /^\s/u.test(o.text),
|
|
111
|
+
trailingWhitespace: /\s$/u.test(o.text),
|
|
112
|
+
box: recordBox(o)
|
|
113
|
+
};
|
|
114
|
+
const pageRuns = runsByPage.get(o.page);
|
|
115
|
+
if (pageRuns) pageRuns.push(run);
|
|
116
|
+
else runsByPage.set(o.page, [
|
|
117
|
+
run
|
|
118
|
+
]);
|
|
119
|
+
}
|
|
120
|
+
if (parsed === 0) errors.push('word dump contains no JSON records');
|
|
121
|
+
if (rec.size === 0) errors.push('word dump contains no valid records');
|
|
122
|
+
if ([
|
|
123
|
+
...rec.values()
|
|
124
|
+
].every((evidence)=>evidence.tokens.length === 0)) errors.push('word dump contains no lexical tokens');
|
|
125
|
+
for (const [page, runs] of runsByPage){
|
|
126
|
+
for(let index = 0; index + 1 < runs.length; index++){
|
|
127
|
+
const current = runs[index];
|
|
128
|
+
const next = runs[index + 1];
|
|
129
|
+
if (current.tokens.length > 0 && current.terminalHyphen && next.tokens.length > 0) {
|
|
130
|
+
const evidence = rec.get(page);
|
|
131
|
+
evidence.sourcePairs.push({
|
|
132
|
+
first: current.tokens[current.tokens.length - 1],
|
|
133
|
+
second: next.tokens[0],
|
|
134
|
+
explicitHyphen: true
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (current.tokens.length === 1 && next.tokens.length === 1 && !current.terminalHyphen && !next.hyphenOnly && !current.trailingWhitespace && !next.leadingWhitespace && touchingFragments(current.box, next.box)) {
|
|
138
|
+
const evidence = rec.get(page);
|
|
139
|
+
evidence.sourcePairs.push({
|
|
140
|
+
first: current.tokens[0],
|
|
141
|
+
second: next.tokens[0],
|
|
142
|
+
explicitHyphen: false
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (current.tokens.length === 0 || !next.hyphenOnly || index + 2 >= runs.length) continue;
|
|
146
|
+
const afterHyphen = runs[index + 2];
|
|
147
|
+
if (afterHyphen.tokens.length === 0) continue;
|
|
148
|
+
const evidence = rec.get(page);
|
|
149
|
+
evidence.sourcePairs.push({
|
|
150
|
+
first: current.tokens[current.tokens.length - 1],
|
|
151
|
+
second: afterHyphen.tokens[0],
|
|
152
|
+
explicitHyphen: true
|
|
153
|
+
});
|
|
56
154
|
}
|
|
57
155
|
}
|
|
156
|
+
if (errors.length > 0) throw auditInputError(errors);
|
|
58
157
|
const md = readFileSync(mdPath, 'utf8');
|
|
59
158
|
const out = bodyByPage(md);
|
|
60
159
|
const lines = [];
|
|
@@ -64,21 +163,20 @@ function counter(items) {
|
|
|
64
163
|
const pages = [
|
|
65
164
|
...rec.keys()
|
|
66
165
|
].sort((a, b)=>a - b);
|
|
166
|
+
const absent = pages.filter((page)=>!out.has(page));
|
|
167
|
+
if (absent.length > 0) {
|
|
168
|
+
const markers = absent.map((page)=>`p${page}`).join(', ');
|
|
169
|
+
throw auditInputError([
|
|
170
|
+
`Markdown has no provenance marker for ${markers}; add "### pN" or "<!-- pN -->" for every dumped page`
|
|
171
|
+
]);
|
|
172
|
+
}
|
|
67
173
|
for (const page of pages){
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
const missing =
|
|
74
|
-
for (const [tok, n] of counter((_rec_get = rec.get(page)) !== null && _rec_get !== void 0 ? _rec_get : [])){
|
|
75
|
-
var _ms_get;
|
|
76
|
-
const d = n - ((_ms_get = ms.get(tok)) !== null && _ms_get !== void 0 ? _ms_get : 0);
|
|
77
|
-
if (d <= 0) continue;
|
|
78
|
-
strict += d;
|
|
79
|
-
if (joined.includes(tok)) continue; // hyphen-join / split repair
|
|
80
|
-
for(let i = 0; i < d; i++)missing.push(tok);
|
|
81
|
-
}
|
|
174
|
+
const evidence = rec.get(page);
|
|
175
|
+
const output = tokens(markdownText(out.get(page)));
|
|
176
|
+
const outputTokens = output.map((token)=>token.text);
|
|
177
|
+
const strictMissing = strictDeficits(evidence.tokens, outputTokens);
|
|
178
|
+
strict += strictMissing;
|
|
179
|
+
const missing = missingTokens(evidence, output);
|
|
82
180
|
if (missing.length > 0) {
|
|
83
181
|
total += missing.length;
|
|
84
182
|
perPage.push({
|
|
@@ -91,7 +189,7 @@ function counter(items) {
|
|
|
91
189
|
const words = pages.reduce((sum, p)=>{
|
|
92
190
|
var _ref;
|
|
93
191
|
var _rec_get;
|
|
94
|
-
return sum + ((_ref = (_rec_get = rec.get(p)) === null || _rec_get === void 0 ? void 0 : _rec_get.length) !== null && _ref !== void 0 ? _ref : 0);
|
|
192
|
+
return sum + ((_ref = (_rec_get = rec.get(p)) === null || _rec_get === void 0 ? void 0 : _rec_get.tokens.length) !== null && _ref !== void 0 ? _ref : 0);
|
|
95
193
|
}, 0);
|
|
96
194
|
const summaryLine = `parsed=${parsed} non-json-skipped=${skipped} pages=${rec.size} words=${words} strict_deficits=${strict} MISSING=${total}`;
|
|
97
195
|
lines.push(summaryLine);
|
|
@@ -107,6 +205,115 @@ function counter(items) {
|
|
|
107
205
|
lines
|
|
108
206
|
};
|
|
109
207
|
}
|
|
208
|
+
function isAuditRecord(value) {
|
|
209
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) return false;
|
|
210
|
+
const record = value;
|
|
211
|
+
if (!(typeof record.page === 'number' && Number.isSafeInteger(record.page) && record.page > 0 && typeof record.text === 'string' && record.text.trim().length > 0)) return false;
|
|
212
|
+
const geometry = [
|
|
213
|
+
record.x,
|
|
214
|
+
record.y,
|
|
215
|
+
record.w,
|
|
216
|
+
record.h
|
|
217
|
+
];
|
|
218
|
+
return geometry.every((field)=>field === undefined) || geometry.every((field)=>typeof field === 'number' && Number.isFinite(field)) && record.w >= 0 && record.h >= 0;
|
|
219
|
+
}
|
|
220
|
+
function recordBox(record) {
|
|
221
|
+
if (record.x === undefined || record.y === undefined || record.w === undefined || record.h === undefined) return undefined;
|
|
222
|
+
return {
|
|
223
|
+
x: record.x,
|
|
224
|
+
y: record.y,
|
|
225
|
+
w: record.w,
|
|
226
|
+
h: record.h
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
function touchingFragments(first, second) {
|
|
230
|
+
if (!first || !second) return false;
|
|
231
|
+
const height = Math.max(first.h, second.h);
|
|
232
|
+
if (height === 0) return false;
|
|
233
|
+
const gap = second.x - (first.x + first.w);
|
|
234
|
+
// Fragment repairs are restricted to nearly touching runs on one baseline.
|
|
235
|
+
return Math.abs(first.y - second.y) <= height * 0.25 && Math.abs(gap) <= height * 0.15;
|
|
236
|
+
}
|
|
237
|
+
function auditInputError(errors) {
|
|
238
|
+
return new PdfToMdError('AUDIT_INPUT', `audit evidence is invalid: ${errors.join('; ')}`);
|
|
239
|
+
}
|
|
240
|
+
function strictDeficits(source, output) {
|
|
241
|
+
var _outputCounts_get;
|
|
242
|
+
let deficits = 0;
|
|
243
|
+
const outputCounts = counter(output);
|
|
244
|
+
for (const [token, count] of counter(source))deficits += Math.max(0, count - ((_outputCounts_get = outputCounts.get(token)) !== null && _outputCounts_get !== void 0 ? _outputCounts_get : 0));
|
|
245
|
+
return deficits;
|
|
246
|
+
}
|
|
247
|
+
function missingTokens(evidence, output) {
|
|
248
|
+
const sourceAvailable = new Set(evidence.tokens.map((_, index)=>index));
|
|
249
|
+
const outputAvailable = new Set(output.map((_, index)=>index));
|
|
250
|
+
const linePairs = lineHyphenPairs(output);
|
|
251
|
+
const inlinePairs = inlineHyphenPairs(output);
|
|
252
|
+
// Reserve literal occurrences before attempting a repair. A repair may join
|
|
253
|
+
// several source tokens into one output token, so doing it first can steal
|
|
254
|
+
// an occurrence required by an unrelated literal source token.
|
|
255
|
+
for (const sourceIndex of [
|
|
256
|
+
...sourceAvailable
|
|
257
|
+
]){
|
|
258
|
+
const direct = output.findIndex((token, index)=>outputAvailable.has(index) && token.text === evidence.tokens[sourceIndex]);
|
|
259
|
+
if (direct < 0) continue;
|
|
260
|
+
sourceAvailable.delete(sourceIndex);
|
|
261
|
+
outputAvailable.delete(direct);
|
|
262
|
+
}
|
|
263
|
+
for (const pair of evidence.sourcePairs){
|
|
264
|
+
if (!sourceAvailable.has(pair.first) || !sourceAvailable.has(pair.second)) continue;
|
|
265
|
+
const joined = evidence.tokens[pair.first] + evidence.tokens[pair.second];
|
|
266
|
+
if (pair.explicitHyphen) {
|
|
267
|
+
var _inlinePairs_find;
|
|
268
|
+
const hyphenated = (_inlinePairs_find = inlinePairs.find(([first, second])=>outputAvailable.has(first) && outputAvailable.has(second) && output[first].text + output[second].text === joined)) !== null && _inlinePairs_find !== void 0 ? _inlinePairs_find : linePairs.find(([first, second])=>outputAvailable.has(first) && outputAvailable.has(second) && output[first].text + output[second].text === joined);
|
|
269
|
+
if (hyphenated) {
|
|
270
|
+
sourceAvailable.delete(pair.first);
|
|
271
|
+
sourceAvailable.delete(pair.second);
|
|
272
|
+
outputAvailable.delete(hyphenated[0]);
|
|
273
|
+
outputAvailable.delete(hyphenated[1]);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
const direct = output.findIndex((token, index)=>outputAvailable.has(index) && token.text === joined);
|
|
278
|
+
if (direct >= 0) {
|
|
279
|
+
sourceAvailable.delete(pair.first);
|
|
280
|
+
sourceAvailable.delete(pair.second);
|
|
281
|
+
outputAvailable.delete(direct);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
for (const sourceIndex of [
|
|
285
|
+
...sourceAvailable
|
|
286
|
+
]){
|
|
287
|
+
const split = linePairs.find(([first, second])=>outputAvailable.has(first) && outputAvailable.has(second) && output[first].text + output[second].text === evidence.tokens[sourceIndex]);
|
|
288
|
+
if (!split) continue;
|
|
289
|
+
sourceAvailable.delete(sourceIndex);
|
|
290
|
+
outputAvailable.delete(split[0]);
|
|
291
|
+
outputAvailable.delete(split[1]);
|
|
292
|
+
}
|
|
293
|
+
return [
|
|
294
|
+
...sourceAvailable
|
|
295
|
+
].map((index)=>evidence.tokens[index]);
|
|
296
|
+
}
|
|
297
|
+
function lineHyphenPairs(output) {
|
|
298
|
+
const pairs = [];
|
|
299
|
+
for(let index = 0; index + 1 < output.length; index++){
|
|
300
|
+
if (LINE_HYPHEN_RE.test(output[index].gapAfter)) pairs.push([
|
|
301
|
+
index,
|
|
302
|
+
index + 1
|
|
303
|
+
]);
|
|
304
|
+
}
|
|
305
|
+
return pairs;
|
|
306
|
+
}
|
|
307
|
+
function inlineHyphenPairs(output) {
|
|
308
|
+
const pairs = [];
|
|
309
|
+
for(let index = 0; index + 1 < output.length; index++){
|
|
310
|
+
if (INLINE_HYPHEN_RE.test(output[index].gapAfter)) pairs.push([
|
|
311
|
+
index,
|
|
312
|
+
index + 1
|
|
313
|
+
]);
|
|
314
|
+
}
|
|
315
|
+
return pairs;
|
|
316
|
+
}
|
|
110
317
|
// Renders a string list as ['a', 'b'] (single-quoted) to match frozen baseline comparisons byte-for-byte.
|
|
111
318
|
function pyRepr(items) {
|
|
112
319
|
return `[${items.map((s)=>`'${s.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`).join(', ')}]`;
|
package/dist/esm/audit.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/audit.ts"],"sourcesContent":["// audit.ts — the severity-1 audit instrument (README's \"Design contract\"),\n// shipped as part of the product, not just a path into the source tree.\n//\n// Every recognized word (a --debug-words dump, or an Analysis page's\n// `words`) must appear in its page's markdown — normalized, compared as a per-page multiset, with substring repair for hyphen joins.\n//\n// NOTE: no producer guarantees JSON key order in a --debug-words dump —\n// parse, never grep.\nimport { readFileSync } from 'node:fs';\nimport { bodyByPage } from './page-marker.ts';\n\nfunction norm(s: string): string {\n return s\n .normalize('NFKD')\n .replace(/\\p{M}/gu, '')\n .toLowerCase()\n .replace(/[^a-z0-9]/g, '');\n}\n\nfunction counter(items: Iterable<string>): Map<string, number> {\n const m = new Map<string, number>();\n for (const item of items) m.set(item, (m.get(item) ?? 0) + 1);\n return m;\n}\n\n/** auditWords()'s result — the severity-1 no-silent-loss check. */\nexport interface AuditResult {\n /** JSON lines successfully parsed from the word dump */\n parsed: number;\n /** lines in the dump that were not parseable JSON, skipped rather than\n * failing the whole audit (same policy `--words-json` itself uses) */\n nonJsonSkipped: number;\n /** distinct page numbers present in the dump */\n pages: number;\n /** total recognized words across all pages in the dump */\n words: number;\n /** words present in the dump but not found verbatim on their page, BEFORE\n * hyphen-join/split repair is applied; always `>= missing` */\n strictDeficits: number;\n /** words genuinely absent from the markdown after repair-checking; `0` is\n * the only passing value (`pdf-to-md audit` exits 1 otherwise) */\n missing: number;\n /** per-page detail for every page with at least one missing word (capped\n * sample of the missing tokens per page) */\n perPage: { page: number; missing: string[] }[];\n /** the one-line `parsed=… pages=… words=… strict_deficits=… MISSING=…`\n * summary, also the last entry of `lines` */\n summaryLine: string;\n /** every line `pdf-to-md audit` prints, in order: one per page with\n * missing words, then `summaryLine` */\n lines: string[];\n}\n\n/**\n * Audit a --debug-words dump against a markdown file: every recognized word\n * must land somewhere in its page's markdown. `result.missing === 0` is the pass/fail signal.\n *\n * @param wordsPath path to a --debug-words JSON-lines dump (`{page, text, ...}` per line)\n * @param mdPath path to the markdown file to check the dump against\n */\nexport function auditWords(wordsPath: string, mdPath: string): AuditResult {\n const rec = new Map<number, string[]>();\n let parsed = 0;\n let skipped = 0;\n\n const wordsRaw = readFileSync(wordsPath, 'utf8');\n // A trailing \"\\n\" at EOF must not yield a spurious final empty line; a\n // genuine blank line mid-file does count, as skipped.\n const rawLines = wordsRaw.split('\\n');\n if (rawLines.length > 0 && rawLines[rawLines.length - 1] === '') rawLines.pop();\n for (let line of rawLines) {\n line = line.trim();\n if (!line.startsWith('{')) {\n skipped++;\n continue;\n }\n let o: { page: number; text: string };\n try {\n o = JSON.parse(line);\n } catch {\n skipped++;\n continue;\n }\n parsed++;\n const t = norm(o.text);\n if (t) {\n const arr = rec.get(o.page);\n if (arr) arr.push(t);\n else rec.set(o.page, [t]);\n }\n }\n\n const md = readFileSync(mdPath, 'utf8');\n const out = bodyByPage(md);\n\n const lines: string[] = [];\n let total = 0;\n let strict = 0;\n const perPage: { page: number; missing: string[] }[] = [];\n const pages = [...rec.keys()].sort((a, b) => a - b);\n\n for (const page of pages) {\n const toks = (out.get(page)?.match(/\\S+/g) ?? []).map(norm).filter((t) => t.length > 0);\n const ms = counter(toks);\n const joined = toks.join('');\n const missing: string[] = [];\n for (const [tok, n] of counter(rec.get(page) ?? [])) {\n const d = n - (ms.get(tok) ?? 0);\n if (d <= 0) continue;\n strict += d;\n if (joined.includes(tok)) continue; // hyphen-join / split repair\n for (let i = 0; i < d; i++) missing.push(tok);\n }\n if (missing.length > 0) {\n total += missing.length;\n perPage.push({ page, missing });\n lines.push(`p${page}: MISSING ${missing.length}: ${pyRepr(missing.slice(0, 12))}`);\n }\n }\n\n const words = pages.reduce((sum, p) => sum + (rec.get(p)?.length ?? 0), 0);\n const summaryLine = `parsed=${parsed} non-json-skipped=${skipped} pages=${rec.size} words=${words} strict_deficits=${strict} MISSING=${total}`;\n lines.push(summaryLine);\n\n return {\n parsed,\n nonJsonSkipped: skipped,\n pages: rec.size,\n words,\n strictDeficits: strict,\n missing: total,\n perPage,\n summaryLine,\n lines,\n };\n}\n\n// Renders a string list as ['a', 'b'] (single-quoted) to match frozen baseline comparisons byte-for-byte.\nfunction pyRepr(items: string[]): string {\n return `[${items.map((s) => `'${s.replace(/\\\\/g, '\\\\\\\\').replace(/'/g, \"\\\\'\")}'`).join(', ')}]`;\n}\n"],"names":["readFileSync","bodyByPage","norm","s","normalize","replace","toLowerCase","counter","items","m","Map","item","set","get","auditWords","wordsPath","mdPath","rec","parsed","skipped","wordsRaw","rawLines","split","length","pop","line","trim","startsWith","o","JSON","parse","t","text","arr","page","push","md","out","lines","total","strict","perPage","pages","keys","sort","a","b","toks","match","map","filter","ms","joined","join","missing","tok","n","d","includes","i","pyRepr","slice","words","reduce","sum","p","summaryLine","size","nonJsonSkipped","strictDeficits"],"mappings":"AAAA,2EAA2E;AAC3E,wEAAwE;AACxE,EAAE;AACF,qEAAqE;AACrE,qIAAqI;AACrI,EAAE;AACF,wEAAwE;AACxE,qBAAqB;AACrB,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,UAAU,QAAQ,mBAAmB;AAE9C,SAASC,KAAKC,CAAS;IACrB,OAAOA,EACJC,SAAS,CAAC,QACVC,OAAO,CAAC,WAAW,IACnBC,WAAW,GACXD,OAAO,CAAC,cAAc;AAC3B;AAEA,SAASE,QAAQC,KAAuB;QAECC;IADvC,MAAMA,IAAI,IAAIC;IACd,KAAK,MAAMC,QAAQH,MAAOC,EAAEG,GAAG,CAACD,MAAM,EAACF,SAAAA,EAAEI,GAAG,CAACF,mBAANF,oBAAAA,SAAe,KAAK;IAC3D,OAAOA;AACT;AA8BA;;;;;;CAMC,GACD,OAAO,SAASK,WAAWC,SAAiB,EAAEC,MAAc;IAC1D,MAAMC,MAAM,IAAIP;IAChB,IAAIQ,SAAS;IACb,IAAIC,UAAU;IAEd,MAAMC,WAAWpB,aAAae,WAAW;IACzC,uEAAuE;IACvE,sDAAsD;IACtD,MAAMM,WAAWD,SAASE,KAAK,CAAC;IAChC,IAAID,SAASE,MAAM,GAAG,KAAKF,QAAQ,CAACA,SAASE,MAAM,GAAG,EAAE,KAAK,IAAIF,SAASG,GAAG;IAC7E,KAAK,IAAIC,QAAQJ,SAAU;QACzBI,OAAOA,KAAKC,IAAI;QAChB,IAAI,CAACD,KAAKE,UAAU,CAAC,MAAM;YACzBR;YACA;QACF;QACA,IAAIS;QACJ,IAAI;YACFA,IAAIC,KAAKC,KAAK,CAACL;QACjB,EAAE,OAAM;YACNN;YACA;QACF;QACAD;QACA,MAAMa,IAAI7B,KAAK0B,EAAEI,IAAI;QACrB,IAAID,GAAG;YACL,MAAME,MAAMhB,IAAIJ,GAAG,CAACe,EAAEM,IAAI;YAC1B,IAAID,KAAKA,IAAIE,IAAI,CAACJ;iBACbd,IAAIL,GAAG,CAACgB,EAAEM,IAAI,EAAE;gBAACH;aAAE;QAC1B;IACF;IAEA,MAAMK,KAAKpC,aAAagB,QAAQ;IAChC,MAAMqB,MAAMpC,WAAWmC;IAEvB,MAAME,QAAkB,EAAE;IAC1B,IAAIC,QAAQ;IACZ,IAAIC,SAAS;IACb,MAAMC,UAAiD,EAAE;IACzD,MAAMC,QAAQ;WAAIzB,IAAI0B,IAAI;KAAG,CAACC,IAAI,CAAC,CAACC,GAAGC,IAAMD,IAAIC;IAEjD,KAAK,MAAMZ,QAAQQ,MAAO;kBAKOzB;YAJjBoB;QAAd,MAAMU,OAAO,UAACV,WAAAA,IAAIxB,GAAG,CAACqB,mBAARG,+BAAAA,SAAeW,KAAK,CAAC,8CAAW,EAAE,EAAEC,GAAG,CAAC/C,MAAMgD,MAAM,CAAC,CAACnB,IAAMA,EAAER,MAAM,GAAG;QACrF,MAAM4B,KAAK5C,QAAQwC;QACnB,MAAMK,SAASL,KAAKM,IAAI,CAAC;QACzB,MAAMC,UAAoB,EAAE;QAC5B,KAAK,MAAM,CAACC,KAAKC,EAAE,IAAIjD,SAAQU,WAAAA,IAAIJ,GAAG,CAACqB,mBAARjB,sBAAAA,WAAiB,EAAE,EAAG;gBACpCkC;YAAf,MAAMM,IAAID,MAAKL,UAAAA,GAAGtC,GAAG,CAAC0C,kBAAPJ,qBAAAA,UAAe;YAC9B,IAAIM,KAAK,GAAG;YACZjB,UAAUiB;YACV,IAAIL,OAAOM,QAAQ,CAACH,MAAM,UAAU,6BAA6B;YACjE,IAAK,IAAII,IAAI,GAAGA,IAAIF,GAAGE,IAAKL,QAAQnB,IAAI,CAACoB;QAC3C;QACA,IAAID,QAAQ/B,MAAM,GAAG,GAAG;YACtBgB,SAASe,QAAQ/B,MAAM;YACvBkB,QAAQN,IAAI,CAAC;gBAAED;gBAAMoB;YAAQ;YAC7BhB,MAAMH,IAAI,CAAC,CAAC,CAAC,EAAED,KAAK,UAAU,EAAEoB,QAAQ/B,MAAM,CAAC,EAAE,EAAEqC,OAAON,QAAQO,KAAK,CAAC,GAAG,MAAM;QACnF;IACF;IAEA,MAAMC,QAAQpB,MAAMqB,MAAM,CAAC,CAACC,KAAKC;;YAAahD;eAAP+C,gBAAO/C,WAAAA,IAAIJ,GAAG,CAACoD,gBAARhD,+BAAAA,SAAYM,MAAM,uCAAI;OAAI;IACxE,MAAM2C,cAAc,CAAC,OAAO,EAAEhD,OAAO,kBAAkB,EAAEC,QAAQ,OAAO,EAAEF,IAAIkD,IAAI,CAAC,OAAO,EAAEL,MAAM,iBAAiB,EAAEtB,OAAO,SAAS,EAAED,OAAO;IAC9ID,MAAMH,IAAI,CAAC+B;IAEX,OAAO;QACLhD;QACAkD,gBAAgBjD;QAChBuB,OAAOzB,IAAIkD,IAAI;QACfL;QACAO,gBAAgB7B;QAChBc,SAASf;QACTE;QACAyB;QACA5B;IACF;AACF;AAEA,0GAA0G;AAC1G,SAASsB,OAAOpD,KAAe;IAC7B,OAAO,CAAC,CAAC,EAAEA,MAAMyC,GAAG,CAAC,CAAC9C,IAAM,CAAC,CAAC,EAAEA,EAAEE,OAAO,CAAC,OAAO,QAAQA,OAAO,CAAC,MAAM,OAAO,CAAC,CAAC,EAAEgD,IAAI,CAAC,MAAM,CAAC,CAAC;AACjG"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/audit.ts"],"sourcesContent":["// audit.ts — the severity-1 audit instrument (README's \"Design contract\"),\n// shipped as part of the product, not just a path into the source tree.\n//\n// Every recognized word (a --debug-words dump, or an Analysis page's\n// `words`) must appear in its page's Markdown. Matching consumes evidence so\n// a single output occurrence cannot satisfy repeated source occurrences.\n//\n// NOTE: no producer guarantees JSON key order in a --debug-words dump —\n// parse, never grep.\nimport { readFileSync } from 'node:fs';\nimport { PdfToMdError } from './errors.ts';\nimport { foldTypographic } from './lines.ts';\nimport { bodyByPage } from './page-marker.ts';\n\nconst WORD_RE = /[\\p{L}\\p{N}][\\p{L}\\p{N}\\p{M}]*/gu;\nconst HYPHEN_RE = /[\\p{Pd}\\u2212]\\s*$/u;\nconst LINE_HYPHEN_RE = /^[\\p{Pd}\\u2212][ \\t]*\\r?\\n[ \\t]*$/u;\nconst INLINE_HYPHEN_RE = /^[\\p{Pd}\\u2212][ \\t]*$/u;\n\ninterface Token {\n text: string;\n start: number;\n end: number;\n gapAfter: string;\n}\n\ninterface SourceRun {\n page: number;\n tokens: number[];\n terminalHyphen: boolean;\n hyphenOnly: boolean;\n leadingWhitespace: boolean;\n trailingWhitespace: boolean;\n box?: Box;\n}\n\ninterface PageEvidence {\n tokens: string[];\n sourcePairs: SourcePair[];\n}\n\ninterface SourcePair {\n first: number;\n second: number;\n explicitHyphen: boolean;\n}\n\ninterface Box {\n x: number;\n y: number;\n w: number;\n h: number;\n}\n\ninterface AuditRecord {\n page: number;\n text: string;\n x?: number;\n y?: number;\n w?: number;\n h?: number;\n}\n\nfunction tokens(text: string): Token[] {\n const folded = foldTypographic(text);\n WORD_RE.lastIndex = 0;\n const matches = [...folded.matchAll(WORD_RE)];\n return matches.map((match, index) => {\n const start = match.index as number;\n const end = start + match[0].length;\n const next = matches[index + 1];\n return {\n text: match[0].normalize('NFC').toLowerCase(),\n start,\n end,\n gapAfter: folded.slice(end, next?.index ?? folded.length),\n };\n });\n}\n\nfunction markdownText(md: string): string {\n return md\n .replace(/<!--[\\s\\S]*?-->/gu, ' ')\n .replace(/!\\[[^\\]]*\\]\\([^)]*\\)/gu, ' ')\n .replace(/!\\[[^\\]]*\\]\\[[^\\]]*\\]/gu, ' ')\n .replace(/<img\\b[^>]*>/giu, ' ')\n .replace(/<a\\b[^>]*>/giu, ' ')\n .replace(/<\\/a>/giu, ' ')\n .replace(/^\\s*\\[[^\\]]+\\]:[^\\n]*$/gmu, ' ')\n .replace(/^>\\s*\\[(?:review:[^\\]]*|floats)\\]\\s*/gmu, '')\n .replace(/\\[([^\\]]*)\\]\\([^)]*\\)/gu, '$1')\n .replace(/\\[([^\\]]*)\\]\\[[^\\]]*\\]/gu, '$1');\n}\n\nfunction counter(items: Iterable<string>): Map<string, number> {\n const m = new Map<string, number>();\n for (const item of items) m.set(item, (m.get(item) ?? 0) + 1);\n return m;\n}\n\n/** auditWords()'s result — the severity-1 no-silent-loss check. */\nexport interface AuditResult {\n /** JSON lines successfully parsed from the word dump */\n parsed: number;\n /** lines in the dump that were not parseable JSON; auditWords rejects\n * evidence with any such line */\n nonJsonSkipped: number;\n /** distinct page numbers present in the dump */\n pages: number;\n /** total recognized words across all pages in the dump */\n words: number;\n /** words present in the dump but not found verbatim on their page, BEFORE\n * hyphen-join/split repair is applied; always `>= missing` */\n strictDeficits: number;\n /** words genuinely absent from the markdown after repair-checking; `0` is\n * the only passing value (`pdf-to-md audit` exits nonzero otherwise) */\n missing: number;\n /** per-page detail for every page with at least one missing word (capped\n * sample of the missing tokens per page) */\n perPage: { page: number; missing: string[] }[];\n /** the one-line `parsed=… pages=… words=… strict_deficits=… MISSING=…`\n * summary, also the last entry of `lines` */\n summaryLine: string;\n /** every line `pdf-to-md audit` prints, in order: one per page with\n * missing words, then `summaryLine` */\n lines: string[];\n}\n\n/**\n * Audit a --debug-words dump against a Markdown file. Every recognized word\n * must land on its page. Invalid evidence or absent page provenance throws a\n * PdfToMdError with code `AUDIT_INPUT`.\n *\n * @param wordsPath path to a --debug-words JSON-lines dump (`{page, text, ...}` per line)\n * @param mdPath path to the markdown file to check the dump against\n */\nexport function auditWords(wordsPath: string, mdPath: string): AuditResult {\n const rec = new Map<number, PageEvidence>();\n let parsed = 0;\n let skipped = 0;\n const errors: string[] = [];\n const runsByPage = new Map<number, SourceRun[]>();\n\n const wordsRaw = readFileSync(wordsPath, 'utf8');\n // A trailing \"\\n\" at EOF must not yield a spurious final empty line; a\n // genuine blank line mid-file does count, as skipped.\n const rawLines = wordsRaw.split('\\n');\n if (rawLines.length > 0 && rawLines[rawLines.length - 1] === '') rawLines.pop();\n for (const [lineIndex, rawLine] of rawLines.entries()) {\n const line = rawLine.trim();\n if (!line.startsWith('{')) {\n skipped++;\n errors.push(`line ${lineIndex + 1} is not valid JSON`);\n continue;\n }\n let o: unknown;\n try {\n o = JSON.parse(line);\n } catch {\n skipped++;\n errors.push(`line ${lineIndex + 1} is not valid JSON`);\n continue;\n }\n parsed++;\n if (!isAuditRecord(o)) {\n errors.push(`line ${lineIndex + 1} must contain a positive integer page and non-empty text`);\n continue;\n }\n let evidence = rec.get(o.page);\n if (!evidence) {\n evidence = { tokens: [], sourcePairs: [] };\n rec.set(o.page, evidence);\n }\n const runTokenDetails = tokens(o.text);\n const runTokens = runTokenDetails.map((token) => token.text);\n const start = evidence.tokens.length;\n evidence.tokens.push(...runTokens);\n for (let index = 0; index + 1 < runTokenDetails.length; index++) {\n if (INLINE_HYPHEN_RE.test(runTokenDetails[index].gapAfter)) {\n evidence.sourcePairs.push({ first: start + index, second: start + index + 1, explicitHyphen: true });\n }\n }\n const run = {\n page: o.page,\n tokens: runTokens.map((_, index) => start + index),\n terminalHyphen: HYPHEN_RE.test(o.text),\n hyphenOnly: /^[\\p{Pd}\\u2212]\\s*$/u.test(o.text),\n leadingWhitespace: /^\\s/u.test(o.text),\n trailingWhitespace: /\\s$/u.test(o.text),\n box: recordBox(o),\n };\n const pageRuns = runsByPage.get(o.page);\n if (pageRuns) pageRuns.push(run);\n else runsByPage.set(o.page, [run]);\n }\n\n if (parsed === 0) errors.push('word dump contains no JSON records');\n if (rec.size === 0) errors.push('word dump contains no valid records');\n if ([...rec.values()].every((evidence) => evidence.tokens.length === 0)) errors.push('word dump contains no lexical tokens');\n for (const [page, runs] of runsByPage) {\n for (let index = 0; index + 1 < runs.length; index++) {\n const current = runs[index];\n const next = runs[index + 1];\n if (current.tokens.length > 0 && current.terminalHyphen && next.tokens.length > 0) {\n const evidence = rec.get(page) as PageEvidence;\n evidence.sourcePairs.push({ first: current.tokens[current.tokens.length - 1], second: next.tokens[0], explicitHyphen: true });\n }\n if (current.tokens.length === 1 && next.tokens.length === 1 && !current.terminalHyphen && !next.hyphenOnly && !current.trailingWhitespace && !next.leadingWhitespace && touchingFragments(current.box, next.box)) {\n const evidence = rec.get(page) as PageEvidence;\n evidence.sourcePairs.push({ first: current.tokens[0], second: next.tokens[0], explicitHyphen: false });\n }\n if (current.tokens.length === 0 || !next.hyphenOnly || index + 2 >= runs.length) continue;\n const afterHyphen = runs[index + 2];\n if (afterHyphen.tokens.length === 0) continue;\n const evidence = rec.get(page) as PageEvidence;\n evidence.sourcePairs.push({ first: current.tokens[current.tokens.length - 1], second: afterHyphen.tokens[0], explicitHyphen: true });\n }\n }\n if (errors.length > 0) throw auditInputError(errors);\n\n const md = readFileSync(mdPath, 'utf8');\n const out = bodyByPage(md);\n\n const lines: string[] = [];\n let total = 0;\n let strict = 0;\n const perPage: { page: number; missing: string[] }[] = [];\n const pages = [...rec.keys()].sort((a, b) => a - b);\n\n const absent = pages.filter((page) => !out.has(page));\n if (absent.length > 0) {\n const markers = absent.map((page) => `p${page}`).join(', ');\n throw auditInputError([`Markdown has no provenance marker for ${markers}; add \"### pN\" or \"<!-- pN -->\" for every dumped page`]);\n }\n\n for (const page of pages) {\n const evidence = rec.get(page) as PageEvidence;\n const output = tokens(markdownText(out.get(page) as string));\n const outputTokens = output.map((token) => token.text);\n const strictMissing = strictDeficits(evidence.tokens, outputTokens);\n strict += strictMissing;\n const missing = missingTokens(evidence, output);\n if (missing.length > 0) {\n total += missing.length;\n perPage.push({ page, missing });\n lines.push(`p${page}: MISSING ${missing.length}: ${pyRepr(missing.slice(0, 12))}`);\n }\n }\n\n const words = pages.reduce((sum, p) => sum + (rec.get(p)?.tokens.length ?? 0), 0);\n const summaryLine = `parsed=${parsed} non-json-skipped=${skipped} pages=${rec.size} words=${words} strict_deficits=${strict} MISSING=${total}`;\n lines.push(summaryLine);\n\n return {\n parsed,\n nonJsonSkipped: skipped,\n pages: rec.size,\n words,\n strictDeficits: strict,\n missing: total,\n perPage,\n summaryLine,\n lines,\n };\n}\n\nfunction isAuditRecord(value: unknown): value is AuditRecord {\n if (typeof value !== 'object' || value === null || Array.isArray(value)) return false;\n const record = value as Partial<AuditRecord>;\n if (!(typeof record.page === 'number' && Number.isSafeInteger(record.page) && record.page > 0 && typeof record.text === 'string' && record.text.trim().length > 0)) return false;\n const geometry = [record.x, record.y, record.w, record.h];\n return geometry.every((field) => field === undefined) || (geometry.every((field) => typeof field === 'number' && Number.isFinite(field)) && (record.w as number) >= 0 && (record.h as number) >= 0);\n}\n\nfunction recordBox(record: AuditRecord): Box | undefined {\n if (record.x === undefined || record.y === undefined || record.w === undefined || record.h === undefined) return undefined;\n return { x: record.x, y: record.y, w: record.w, h: record.h };\n}\n\nfunction touchingFragments(first: Box | undefined, second: Box | undefined): boolean {\n if (!first || !second) return false;\n const height = Math.max(first.h, second.h);\n if (height === 0) return false;\n const gap = second.x - (first.x + first.w);\n // Fragment repairs are restricted to nearly touching runs on one baseline.\n return Math.abs(first.y - second.y) <= height * 0.25 && Math.abs(gap) <= height * 0.15;\n}\n\nfunction auditInputError(errors: string[]): PdfToMdError {\n return new PdfToMdError('AUDIT_INPUT', `audit evidence is invalid: ${errors.join('; ')}`);\n}\n\nfunction strictDeficits(source: string[], output: string[]): number {\n let deficits = 0;\n const outputCounts = counter(output);\n for (const [token, count] of counter(source)) deficits += Math.max(0, count - (outputCounts.get(token) ?? 0));\n return deficits;\n}\n\nfunction missingTokens(evidence: PageEvidence, output: Token[]): string[] {\n const sourceAvailable = new Set(evidence.tokens.map((_, index) => index));\n const outputAvailable = new Set(output.map((_, index) => index));\n const linePairs = lineHyphenPairs(output);\n const inlinePairs = inlineHyphenPairs(output);\n\n // Reserve literal occurrences before attempting a repair. A repair may join\n // several source tokens into one output token, so doing it first can steal\n // an occurrence required by an unrelated literal source token.\n for (const sourceIndex of [...sourceAvailable]) {\n const direct = output.findIndex((token, index) => outputAvailable.has(index) && token.text === evidence.tokens[sourceIndex]);\n if (direct < 0) continue;\n sourceAvailable.delete(sourceIndex);\n outputAvailable.delete(direct);\n }\n\n for (const pair of evidence.sourcePairs) {\n if (!sourceAvailable.has(pair.first) || !sourceAvailable.has(pair.second)) continue;\n const joined = evidence.tokens[pair.first] + evidence.tokens[pair.second];\n if (pair.explicitHyphen) {\n const hyphenated =\n inlinePairs.find(([first, second]) => outputAvailable.has(first) && outputAvailable.has(second) && output[first].text + output[second].text === joined) ?? linePairs.find(([first, second]) => outputAvailable.has(first) && outputAvailable.has(second) && output[first].text + output[second].text === joined);\n if (hyphenated) {\n sourceAvailable.delete(pair.first);\n sourceAvailable.delete(pair.second);\n outputAvailable.delete(hyphenated[0]);\n outputAvailable.delete(hyphenated[1]);\n continue;\n }\n }\n const direct = output.findIndex((token, index) => outputAvailable.has(index) && token.text === joined);\n if (direct >= 0) {\n sourceAvailable.delete(pair.first);\n sourceAvailable.delete(pair.second);\n outputAvailable.delete(direct);\n }\n }\n\n for (const sourceIndex of [...sourceAvailable]) {\n const split = linePairs.find(([first, second]) => outputAvailable.has(first) && outputAvailable.has(second) && output[first].text + output[second].text === evidence.tokens[sourceIndex]);\n if (!split) continue;\n sourceAvailable.delete(sourceIndex);\n outputAvailable.delete(split[0]);\n outputAvailable.delete(split[1]);\n }\n\n return [...sourceAvailable].map((index) => evidence.tokens[index]);\n}\n\nfunction lineHyphenPairs(output: Token[]): [number, number][] {\n const pairs: [number, number][] = [];\n for (let index = 0; index + 1 < output.length; index++) {\n if (LINE_HYPHEN_RE.test(output[index].gapAfter)) pairs.push([index, index + 1]);\n }\n return pairs;\n}\n\nfunction inlineHyphenPairs(output: Token[]): [number, number][] {\n const pairs: [number, number][] = [];\n for (let index = 0; index + 1 < output.length; index++) {\n if (INLINE_HYPHEN_RE.test(output[index].gapAfter)) pairs.push([index, index + 1]);\n }\n return pairs;\n}\n\n// Renders a string list as ['a', 'b'] (single-quoted) to match frozen baseline comparisons byte-for-byte.\nfunction pyRepr(items: string[]): string {\n return `[${items.map((s) => `'${s.replace(/\\\\/g, '\\\\\\\\').replace(/'/g, \"\\\\'\")}'`).join(', ')}]`;\n}\n"],"names":["readFileSync","PdfToMdError","foldTypographic","bodyByPage","WORD_RE","HYPHEN_RE","LINE_HYPHEN_RE","INLINE_HYPHEN_RE","tokens","text","folded","lastIndex","matches","matchAll","map","match","index","start","end","length","next","normalize","toLowerCase","gapAfter","slice","markdownText","md","replace","counter","items","m","Map","item","set","get","auditWords","wordsPath","mdPath","rec","parsed","skipped","errors","runsByPage","wordsRaw","rawLines","split","pop","lineIndex","rawLine","entries","line","trim","startsWith","push","o","JSON","parse","isAuditRecord","evidence","page","sourcePairs","runTokenDetails","runTokens","token","test","first","second","explicitHyphen","run","_","terminalHyphen","hyphenOnly","leadingWhitespace","trailingWhitespace","box","recordBox","pageRuns","size","values","every","runs","current","touchingFragments","afterHyphen","auditInputError","out","lines","total","strict","perPage","pages","keys","sort","a","b","absent","filter","has","markers","join","output","outputTokens","strictMissing","strictDeficits","missing","missingTokens","pyRepr","words","reduce","sum","p","summaryLine","nonJsonSkipped","value","Array","isArray","record","Number","isSafeInteger","geometry","x","y","w","h","field","undefined","isFinite","height","Math","max","gap","abs","source","outputCounts","deficits","count","sourceAvailable","Set","outputAvailable","linePairs","lineHyphenPairs","inlinePairs","inlineHyphenPairs","sourceIndex","direct","findIndex","delete","pair","joined","hyphenated","find","pairs","s"],"mappings":"AAAA,2EAA2E;AAC3E,wEAAwE;AACxE,EAAE;AACF,qEAAqE;AACrE,6EAA6E;AAC7E,yEAAyE;AACzE,EAAE;AACF,wEAAwE;AACxE,qBAAqB;AACrB,SAASA,YAAY,QAAQ,UAAU;AACvC,SAASC,YAAY,QAAQ,cAAc;AAC3C,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,UAAU,QAAQ,mBAAmB;AAE9C,MAAMC,UAAU;AAChB,MAAMC,YAAY;AAClB,MAAMC,iBAAiB;AACvB,MAAMC,mBAAmB;AA8CzB,SAASC,OAAOC,IAAY;IAC1B,MAAMC,SAASR,gBAAgBO;IAC/BL,QAAQO,SAAS,GAAG;IACpB,MAAMC,UAAU;WAAIF,OAAOG,QAAQ,CAACT;KAAS;IAC7C,OAAOQ,QAAQE,GAAG,CAAC,CAACC,OAAOC;;QACzB,MAAMC,QAAQF,MAAMC,KAAK;QACzB,MAAME,MAAMD,QAAQF,KAAK,CAAC,EAAE,CAACI,MAAM;QACnC,MAAMC,OAAOR,OAAO,CAACI,QAAQ,EAAE;QAC/B,OAAO;YACLP,MAAMM,KAAK,CAAC,EAAE,CAACM,SAAS,CAAC,OAAOC,WAAW;YAC3CL;YACAC;YACAK,UAAUb,OAAOc,KAAK,CAACN,aAAKE,iBAAAA,2BAAAA,KAAMJ,KAAK,uCAAIN,OAAOS,MAAM;QAC1D;IACF;AACF;AAEA,SAASM,aAAaC,EAAU;IAC9B,OAAOA,GACJC,OAAO,CAAC,qBAAqB,KAC7BA,OAAO,CAAC,0BAA0B,KAClCA,OAAO,CAAC,2BAA2B,KACnCA,OAAO,CAAC,mBAAmB,KAC3BA,OAAO,CAAC,iBAAiB,KACzBA,OAAO,CAAC,YAAY,KACpBA,OAAO,CAAC,6BAA6B,KACrCA,OAAO,CAAC,2CAA2C,IACnDA,OAAO,CAAC,2BAA2B,MACnCA,OAAO,CAAC,4BAA4B;AACzC;AAEA,SAASC,QAAQC,KAAuB;QAECC;IADvC,MAAMA,IAAI,IAAIC;IACd,KAAK,MAAMC,QAAQH,MAAOC,EAAEG,GAAG,CAACD,MAAM,EAACF,SAAAA,EAAEI,GAAG,CAACF,mBAANF,oBAAAA,SAAe,KAAK;IAC3D,OAAOA;AACT;AA8BA;;;;;;;CAOC,GACD,OAAO,SAASK,WAAWC,SAAiB,EAAEC,MAAc;IAC1D,MAAMC,MAAM,IAAIP;IAChB,IAAIQ,SAAS;IACb,IAAIC,UAAU;IACd,MAAMC,SAAmB,EAAE;IAC3B,MAAMC,aAAa,IAAIX;IAEvB,MAAMY,WAAW3C,aAAaoC,WAAW;IACzC,uEAAuE;IACvE,sDAAsD;IACtD,MAAMQ,WAAWD,SAASE,KAAK,CAAC;IAChC,IAAID,SAASzB,MAAM,GAAG,KAAKyB,QAAQ,CAACA,SAASzB,MAAM,GAAG,EAAE,KAAK,IAAIyB,SAASE,GAAG;IAC7E,KAAK,MAAM,CAACC,WAAWC,QAAQ,IAAIJ,SAASK,OAAO,GAAI;QACrD,MAAMC,OAAOF,QAAQG,IAAI;QACzB,IAAI,CAACD,KAAKE,UAAU,CAAC,MAAM;YACzBZ;YACAC,OAAOY,IAAI,CAAC,CAAC,KAAK,EAAEN,YAAY,EAAE,kBAAkB,CAAC;YACrD;QACF;QACA,IAAIO;QACJ,IAAI;YACFA,IAAIC,KAAKC,KAAK,CAACN;QACjB,EAAE,OAAM;YACNV;YACAC,OAAOY,IAAI,CAAC,CAAC,KAAK,EAAEN,YAAY,EAAE,kBAAkB,CAAC;YACrD;QACF;QACAR;QACA,IAAI,CAACkB,cAAcH,IAAI;YACrBb,OAAOY,IAAI,CAAC,CAAC,KAAK,EAAEN,YAAY,EAAE,wDAAwD,CAAC;YAC3F;QACF;QACA,IAAIW,WAAWpB,IAAIJ,GAAG,CAACoB,EAAEK,IAAI;QAC7B,IAAI,CAACD,UAAU;YACbA,WAAW;gBAAElD,QAAQ,EAAE;gBAAEoD,aAAa,EAAE;YAAC;YACzCtB,IAAIL,GAAG,CAACqB,EAAEK,IAAI,EAAED;QAClB;QACA,MAAMG,kBAAkBrD,OAAO8C,EAAE7C,IAAI;QACrC,MAAMqD,YAAYD,gBAAgB/C,GAAG,CAAC,CAACiD,QAAUA,MAAMtD,IAAI;QAC3D,MAAMQ,QAAQyC,SAASlD,MAAM,CAACW,MAAM;QACpCuC,SAASlD,MAAM,CAAC6C,IAAI,IAAIS;QACxB,IAAK,IAAI9C,QAAQ,GAAGA,QAAQ,IAAI6C,gBAAgB1C,MAAM,EAAEH,QAAS;YAC/D,IAAIT,iBAAiByD,IAAI,CAACH,eAAe,CAAC7C,MAAM,CAACO,QAAQ,GAAG;gBAC1DmC,SAASE,WAAW,CAACP,IAAI,CAAC;oBAAEY,OAAOhD,QAAQD;oBAAOkD,QAAQjD,QAAQD,QAAQ;oBAAGmD,gBAAgB;gBAAK;YACpG;QACF;QACA,MAAMC,MAAM;YACVT,MAAML,EAAEK,IAAI;YACZnD,QAAQsD,UAAUhD,GAAG,CAAC,CAACuD,GAAGrD,QAAUC,QAAQD;YAC5CsD,gBAAgBjE,UAAU2D,IAAI,CAACV,EAAE7C,IAAI;YACrC8D,YAAY,uBAAuBP,IAAI,CAACV,EAAE7C,IAAI;YAC9C+D,mBAAmB,OAAOR,IAAI,CAACV,EAAE7C,IAAI;YACrCgE,oBAAoB,OAAOT,IAAI,CAACV,EAAE7C,IAAI;YACtCiE,KAAKC,UAAUrB;QACjB;QACA,MAAMsB,WAAWlC,WAAWR,GAAG,CAACoB,EAAEK,IAAI;QACtC,IAAIiB,UAAUA,SAASvB,IAAI,CAACe;aACvB1B,WAAWT,GAAG,CAACqB,EAAEK,IAAI,EAAE;YAACS;SAAI;IACnC;IAEA,IAAI7B,WAAW,GAAGE,OAAOY,IAAI,CAAC;IAC9B,IAAIf,IAAIuC,IAAI,KAAK,GAAGpC,OAAOY,IAAI,CAAC;IAChC,IAAI;WAAIf,IAAIwC,MAAM;KAAG,CAACC,KAAK,CAAC,CAACrB,WAAaA,SAASlD,MAAM,CAACW,MAAM,KAAK,IAAIsB,OAAOY,IAAI,CAAC;IACrF,KAAK,MAAM,CAACM,MAAMqB,KAAK,IAAItC,WAAY;QACrC,IAAK,IAAI1B,QAAQ,GAAGA,QAAQ,IAAIgE,KAAK7D,MAAM,EAAEH,QAAS;YACpD,MAAMiE,UAAUD,IAAI,CAAChE,MAAM;YAC3B,MAAMI,OAAO4D,IAAI,CAAChE,QAAQ,EAAE;YAC5B,IAAIiE,QAAQzE,MAAM,CAACW,MAAM,GAAG,KAAK8D,QAAQX,cAAc,IAAIlD,KAAKZ,MAAM,CAACW,MAAM,GAAG,GAAG;gBACjF,MAAMuC,WAAWpB,IAAIJ,GAAG,CAACyB;gBACzBD,SAASE,WAAW,CAACP,IAAI,CAAC;oBAAEY,OAAOgB,QAAQzE,MAAM,CAACyE,QAAQzE,MAAM,CAACW,MAAM,GAAG,EAAE;oBAAE+C,QAAQ9C,KAAKZ,MAAM,CAAC,EAAE;oBAAE2D,gBAAgB;gBAAK;YAC7H;YACA,IAAIc,QAAQzE,MAAM,CAACW,MAAM,KAAK,KAAKC,KAAKZ,MAAM,CAACW,MAAM,KAAK,KAAK,CAAC8D,QAAQX,cAAc,IAAI,CAAClD,KAAKmD,UAAU,IAAI,CAACU,QAAQR,kBAAkB,IAAI,CAACrD,KAAKoD,iBAAiB,IAAIU,kBAAkBD,QAAQP,GAAG,EAAEtD,KAAKsD,GAAG,GAAG;gBAChN,MAAMhB,WAAWpB,IAAIJ,GAAG,CAACyB;gBACzBD,SAASE,WAAW,CAACP,IAAI,CAAC;oBAAEY,OAAOgB,QAAQzE,MAAM,CAAC,EAAE;oBAAE0D,QAAQ9C,KAAKZ,MAAM,CAAC,EAAE;oBAAE2D,gBAAgB;gBAAM;YACtG;YACA,IAAIc,QAAQzE,MAAM,CAACW,MAAM,KAAK,KAAK,CAACC,KAAKmD,UAAU,IAAIvD,QAAQ,KAAKgE,KAAK7D,MAAM,EAAE;YACjF,MAAMgE,cAAcH,IAAI,CAAChE,QAAQ,EAAE;YACnC,IAAImE,YAAY3E,MAAM,CAACW,MAAM,KAAK,GAAG;YACrC,MAAMuC,WAAWpB,IAAIJ,GAAG,CAACyB;YACzBD,SAASE,WAAW,CAACP,IAAI,CAAC;gBAAEY,OAAOgB,QAAQzE,MAAM,CAACyE,QAAQzE,MAAM,CAACW,MAAM,GAAG,EAAE;gBAAE+C,QAAQiB,YAAY3E,MAAM,CAAC,EAAE;gBAAE2D,gBAAgB;YAAK;QACpI;IACF;IACA,IAAI1B,OAAOtB,MAAM,GAAG,GAAG,MAAMiE,gBAAgB3C;IAE7C,MAAMf,KAAK1B,aAAaqC,QAAQ;IAChC,MAAMgD,MAAMlF,WAAWuB;IAEvB,MAAM4D,QAAkB,EAAE;IAC1B,IAAIC,QAAQ;IACZ,IAAIC,SAAS;IACb,MAAMC,UAAiD,EAAE;IACzD,MAAMC,QAAQ;WAAIpD,IAAIqD,IAAI;KAAG,CAACC,IAAI,CAAC,CAACC,GAAGC,IAAMD,IAAIC;IAEjD,MAAMC,SAASL,MAAMM,MAAM,CAAC,CAACrC,OAAS,CAAC0B,IAAIY,GAAG,CAACtC;IAC/C,IAAIoC,OAAO5E,MAAM,GAAG,GAAG;QACrB,MAAM+E,UAAUH,OAAOjF,GAAG,CAAC,CAAC6C,OAAS,CAAC,CAAC,EAAEA,MAAM,EAAEwC,IAAI,CAAC;QACtD,MAAMf,gBAAgB;YAAC,CAAC,sCAAsC,EAAEc,QAAQ,qDAAqD,CAAC;SAAC;IACjI;IAEA,KAAK,MAAMvC,QAAQ+B,MAAO;QACxB,MAAMhC,WAAWpB,IAAIJ,GAAG,CAACyB;QACzB,MAAMyC,SAAS5F,OAAOiB,aAAa4D,IAAInD,GAAG,CAACyB;QAC3C,MAAM0C,eAAeD,OAAOtF,GAAG,CAAC,CAACiD,QAAUA,MAAMtD,IAAI;QACrD,MAAM6F,gBAAgBC,eAAe7C,SAASlD,MAAM,EAAE6F;QACtDb,UAAUc;QACV,MAAME,UAAUC,cAAc/C,UAAU0C;QACxC,IAAII,QAAQrF,MAAM,GAAG,GAAG;YACtBoE,SAASiB,QAAQrF,MAAM;YACvBsE,QAAQpC,IAAI,CAAC;gBAAEM;gBAAM6C;YAAQ;YAC7BlB,MAAMjC,IAAI,CAAC,CAAC,CAAC,EAAEM,KAAK,UAAU,EAAE6C,QAAQrF,MAAM,CAAC,EAAE,EAAEuF,OAAOF,QAAQhF,KAAK,CAAC,GAAG,MAAM;QACnF;IACF;IAEA,MAAMmF,QAAQjB,MAAMkB,MAAM,CAAC,CAACC,KAAKC;;YAAaxE;eAAPuE,gBAAOvE,WAAAA,IAAIJ,GAAG,CAAC4E,gBAARxE,+BAAAA,SAAY9B,MAAM,CAACW,MAAM,uCAAI;OAAI;IAC/E,MAAM4F,cAAc,CAAC,OAAO,EAAExE,OAAO,kBAAkB,EAAEC,QAAQ,OAAO,EAAEF,IAAIuC,IAAI,CAAC,OAAO,EAAE8B,MAAM,iBAAiB,EAAEnB,OAAO,SAAS,EAAED,OAAO;IAC9ID,MAAMjC,IAAI,CAAC0D;IAEX,OAAO;QACLxE;QACAyE,gBAAgBxE;QAChBkD,OAAOpD,IAAIuC,IAAI;QACf8B;QACAJ,gBAAgBf;QAChBgB,SAASjB;QACTE;QACAsB;QACAzB;IACF;AACF;AAEA,SAAS7B,cAAcwD,KAAc;IACnC,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ,OAAO;IAChF,MAAMG,SAASH;IACf,IAAI,CAAE,CAAA,OAAOG,OAAOzD,IAAI,KAAK,YAAY0D,OAAOC,aAAa,CAACF,OAAOzD,IAAI,KAAKyD,OAAOzD,IAAI,GAAG,KAAK,OAAOyD,OAAO3G,IAAI,KAAK,YAAY2G,OAAO3G,IAAI,CAAC0C,IAAI,GAAGhC,MAAM,GAAG,CAAA,GAAI,OAAO;IAC3K,MAAMoG,WAAW;QAACH,OAAOI,CAAC;QAAEJ,OAAOK,CAAC;QAAEL,OAAOM,CAAC;QAAEN,OAAOO,CAAC;KAAC;IACzD,OAAOJ,SAASxC,KAAK,CAAC,CAAC6C,QAAUA,UAAUC,cAAeN,SAASxC,KAAK,CAAC,CAAC6C,QAAU,OAAOA,UAAU,YAAYP,OAAOS,QAAQ,CAACF,WAAW,AAACR,OAAOM,CAAC,IAAe,KAAK,AAACN,OAAOO,CAAC,IAAe;AACnM;AAEA,SAAShD,UAAUyC,MAAmB;IACpC,IAAIA,OAAOI,CAAC,KAAKK,aAAaT,OAAOK,CAAC,KAAKI,aAAaT,OAAOM,CAAC,KAAKG,aAAaT,OAAOO,CAAC,KAAKE,WAAW,OAAOA;IACjH,OAAO;QAAEL,GAAGJ,OAAOI,CAAC;QAAEC,GAAGL,OAAOK,CAAC;QAAEC,GAAGN,OAAOM,CAAC;QAAEC,GAAGP,OAAOO,CAAC;IAAC;AAC9D;AAEA,SAASzC,kBAAkBjB,KAAsB,EAAEC,MAAuB;IACxE,IAAI,CAACD,SAAS,CAACC,QAAQ,OAAO;IAC9B,MAAM6D,SAASC,KAAKC,GAAG,CAAChE,MAAM0D,CAAC,EAAEzD,OAAOyD,CAAC;IACzC,IAAII,WAAW,GAAG,OAAO;IACzB,MAAMG,MAAMhE,OAAOsD,CAAC,GAAIvD,CAAAA,MAAMuD,CAAC,GAAGvD,MAAMyD,CAAC,AAADA;IACxC,2EAA2E;IAC3E,OAAOM,KAAKG,GAAG,CAAClE,MAAMwD,CAAC,GAAGvD,OAAOuD,CAAC,KAAKM,SAAS,QAAQC,KAAKG,GAAG,CAACD,QAAQH,SAAS;AACpF;AAEA,SAAS3C,gBAAgB3C,MAAgB;IACvC,OAAO,IAAIxC,aAAa,eAAe,CAAC,2BAA2B,EAAEwC,OAAO0D,IAAI,CAAC,OAAO;AAC1F;AAEA,SAASI,eAAe6B,MAAgB,EAAEhC,MAAgB;QAGuBiC;IAF/E,IAAIC,WAAW;IACf,MAAMD,eAAezG,QAAQwE;IAC7B,KAAK,MAAM,CAACrC,OAAOwE,MAAM,IAAI3G,QAAQwG,QAASE,YAAYN,KAAKC,GAAG,CAAC,GAAGM,UAASF,oBAAAA,aAAanG,GAAG,CAAC6B,oBAAjBsE,+BAAAA,oBAA2B;IAC1G,OAAOC;AACT;AAEA,SAAS7B,cAAc/C,QAAsB,EAAE0C,MAAe;IAC5D,MAAMoC,kBAAkB,IAAIC,IAAI/E,SAASlD,MAAM,CAACM,GAAG,CAAC,CAACuD,GAAGrD,QAAUA;IAClE,MAAM0H,kBAAkB,IAAID,IAAIrC,OAAOtF,GAAG,CAAC,CAACuD,GAAGrD,QAAUA;IACzD,MAAM2H,YAAYC,gBAAgBxC;IAClC,MAAMyC,cAAcC,kBAAkB1C;IAEtC,4EAA4E;IAC5E,2EAA2E;IAC3E,+DAA+D;IAC/D,KAAK,MAAM2C,eAAe;WAAIP;KAAgB,CAAE;QAC9C,MAAMQ,SAAS5C,OAAO6C,SAAS,CAAC,CAAClF,OAAO/C,QAAU0H,gBAAgBzC,GAAG,CAACjF,UAAU+C,MAAMtD,IAAI,KAAKiD,SAASlD,MAAM,CAACuI,YAAY;QAC3H,IAAIC,SAAS,GAAG;QAChBR,gBAAgBU,MAAM,CAACH;QACvBL,gBAAgBQ,MAAM,CAACF;IACzB;IAEA,KAAK,MAAMG,QAAQzF,SAASE,WAAW,CAAE;QACvC,IAAI,CAAC4E,gBAAgBvC,GAAG,CAACkD,KAAKlF,KAAK,KAAK,CAACuE,gBAAgBvC,GAAG,CAACkD,KAAKjF,MAAM,GAAG;QAC3E,MAAMkF,SAAS1F,SAASlD,MAAM,CAAC2I,KAAKlF,KAAK,CAAC,GAAGP,SAASlD,MAAM,CAAC2I,KAAKjF,MAAM,CAAC;QACzE,IAAIiF,KAAKhF,cAAc,EAAE;gBAErB0E;YADF,MAAMQ,cACJR,oBAAAA,YAAYS,IAAI,CAAC,CAAC,CAACrF,OAAOC,OAAO,GAAKwE,gBAAgBzC,GAAG,CAAChC,UAAUyE,gBAAgBzC,GAAG,CAAC/B,WAAWkC,MAAM,CAACnC,MAAM,CAACxD,IAAI,GAAG2F,MAAM,CAAClC,OAAO,CAACzD,IAAI,KAAK2I,qBAAhJP,+BAAAA,oBAA2JF,UAAUW,IAAI,CAAC,CAAC,CAACrF,OAAOC,OAAO,GAAKwE,gBAAgBzC,GAAG,CAAChC,UAAUyE,gBAAgBzC,GAAG,CAAC/B,WAAWkC,MAAM,CAACnC,MAAM,CAACxD,IAAI,GAAG2F,MAAM,CAAClC,OAAO,CAACzD,IAAI,KAAK2I;YAC3S,IAAIC,YAAY;gBACdb,gBAAgBU,MAAM,CAACC,KAAKlF,KAAK;gBACjCuE,gBAAgBU,MAAM,CAACC,KAAKjF,MAAM;gBAClCwE,gBAAgBQ,MAAM,CAACG,UAAU,CAAC,EAAE;gBACpCX,gBAAgBQ,MAAM,CAACG,UAAU,CAAC,EAAE;gBACpC;YACF;QACF;QACA,MAAML,SAAS5C,OAAO6C,SAAS,CAAC,CAAClF,OAAO/C,QAAU0H,gBAAgBzC,GAAG,CAACjF,UAAU+C,MAAMtD,IAAI,KAAK2I;QAC/F,IAAIJ,UAAU,GAAG;YACfR,gBAAgBU,MAAM,CAACC,KAAKlF,KAAK;YACjCuE,gBAAgBU,MAAM,CAACC,KAAKjF,MAAM;YAClCwE,gBAAgBQ,MAAM,CAACF;QACzB;IACF;IAEA,KAAK,MAAMD,eAAe;WAAIP;KAAgB,CAAE;QAC9C,MAAM3F,QAAQ8F,UAAUW,IAAI,CAAC,CAAC,CAACrF,OAAOC,OAAO,GAAKwE,gBAAgBzC,GAAG,CAAChC,UAAUyE,gBAAgBzC,GAAG,CAAC/B,WAAWkC,MAAM,CAACnC,MAAM,CAACxD,IAAI,GAAG2F,MAAM,CAAClC,OAAO,CAACzD,IAAI,KAAKiD,SAASlD,MAAM,CAACuI,YAAY;QACxL,IAAI,CAAClG,OAAO;QACZ2F,gBAAgBU,MAAM,CAACH;QACvBL,gBAAgBQ,MAAM,CAACrG,KAAK,CAAC,EAAE;QAC/B6F,gBAAgBQ,MAAM,CAACrG,KAAK,CAAC,EAAE;IACjC;IAEA,OAAO;WAAI2F;KAAgB,CAAC1H,GAAG,CAAC,CAACE,QAAU0C,SAASlD,MAAM,CAACQ,MAAM;AACnE;AAEA,SAAS4H,gBAAgBxC,MAAe;IACtC,MAAMmD,QAA4B,EAAE;IACpC,IAAK,IAAIvI,QAAQ,GAAGA,QAAQ,IAAIoF,OAAOjF,MAAM,EAAEH,QAAS;QACtD,IAAIV,eAAe0D,IAAI,CAACoC,MAAM,CAACpF,MAAM,CAACO,QAAQ,GAAGgI,MAAMlG,IAAI,CAAC;YAACrC;YAAOA,QAAQ;SAAE;IAChF;IACA,OAAOuI;AACT;AAEA,SAAST,kBAAkB1C,MAAe;IACxC,MAAMmD,QAA4B,EAAE;IACpC,IAAK,IAAIvI,QAAQ,GAAGA,QAAQ,IAAIoF,OAAOjF,MAAM,EAAEH,QAAS;QACtD,IAAIT,iBAAiByD,IAAI,CAACoC,MAAM,CAACpF,MAAM,CAACO,QAAQ,GAAGgI,MAAMlG,IAAI,CAAC;YAACrC;YAAOA,QAAQ;SAAE;IAClF;IACA,OAAOuI;AACT;AAEA,0GAA0G;AAC1G,SAAS7C,OAAO7E,KAAe;IAC7B,OAAO,CAAC,CAAC,EAAEA,MAAMf,GAAG,CAAC,CAAC0I,IAAM,CAAC,CAAC,EAAEA,EAAE7H,OAAO,CAAC,OAAO,QAAQA,OAAO,CAAC,MAAM,OAAO,CAAC,CAAC,EAAEwE,IAAI,CAAC,MAAM,CAAC,CAAC;AACjG"}
|
package/dist/esm/cli/extract.js
CHANGED
|
@@ -4,7 +4,7 @@ import { readFileSync, writeFileSync } from 'node:fs';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { parseArgs } from 'node:util';
|
|
6
6
|
import { analyze } from '../analyze.js';
|
|
7
|
-
import {
|
|
7
|
+
import { analyzePdfAuto, extractOcr, pdfToMarkdown } from '../extract.js';
|
|
8
8
|
import { toText } from '../render-analysis.js';
|
|
9
9
|
const USAGE = 'usage: pdf-to-md [extract] <file.pdf> [out.md] [--json] [--page-markers] [--ocr] [--no-ocr] ' + '[--pages N[-M]] [--stdout] [--words-json=FILE] [--debug-words=FILE] ' + '[--format md|txt|raw (default md; raw = Analysis JSON incl. the report; txt = reading-order plain text)] ' + '[--dpi=N (OCR render resolution, default 288; raise for tiny print or chart/label pages, ' + '>~400 measured worse on ordinary prose pages)] ' + '[--float-max-words=N] [--float-max-width=F] [--float-margin=F] [--para-gap=F] [--heading-scale=F] ' + '[--col-max-width=F] [--col-split=F] [--line-y-tol=F] [--line-height-ratio=F]';
|
|
10
10
|
const FORMATS = [
|
|
@@ -205,7 +205,8 @@ const extract = async (ctx)=>{
|
|
|
205
205
|
} : undefined;
|
|
206
206
|
const common = {
|
|
207
207
|
pages,
|
|
208
|
-
onWarning
|
|
208
|
+
onWarning,
|
|
209
|
+
onWords
|
|
209
210
|
};
|
|
210
211
|
const ocrCommon = {
|
|
211
212
|
...common,
|
|
@@ -238,8 +239,8 @@ const extract = async (ctx)=>{
|
|
|
238
239
|
}
|
|
239
240
|
content = markdown;
|
|
240
241
|
} else {
|
|
241
|
-
// raw/txt
|
|
242
|
-
//
|
|
242
|
+
// raw/txt render the same routed Analysis as the Markdown API, so a
|
|
243
|
+
// format switch cannot discard a sparse image page or change audit input.
|
|
243
244
|
let analysis;
|
|
244
245
|
if (words !== undefined) {
|
|
245
246
|
analysis = await analyze({
|
|
@@ -248,27 +249,13 @@ const extract = async (ctx)=>{
|
|
|
248
249
|
...ocrCommon,
|
|
249
250
|
path: 'ocr'
|
|
250
251
|
});
|
|
251
|
-
} else if (values.ocr) {
|
|
252
|
-
analysis = await analyze(src, {
|
|
253
|
-
...ocrCommon,
|
|
254
|
-
path: 'ocr'
|
|
255
|
-
});
|
|
256
252
|
} else {
|
|
257
|
-
|
|
258
|
-
...
|
|
259
|
-
|
|
253
|
+
analysis = (await analyzePdfAuto(src, {
|
|
254
|
+
...ocrCommon,
|
|
255
|
+
ocr: values.ocr,
|
|
256
|
+
noOcr: values['no-ocr'],
|
|
260
257
|
pageMarkers: values['page-markers']
|
|
261
|
-
});
|
|
262
|
-
if (needsOcrFallback(text.report.charsPerPage) && !values['no-ocr']) {
|
|
263
|
-
onWarning(ocrFallbackNotice(text.report.charsPerPage));
|
|
264
|
-
analysis = await analyze(src, {
|
|
265
|
-
...ocrCommon,
|
|
266
|
-
path: 'ocr',
|
|
267
|
-
fallback: true
|
|
268
|
-
});
|
|
269
|
-
} else {
|
|
270
|
-
analysis = text;
|
|
271
|
-
}
|
|
258
|
+
})).analysis;
|
|
272
259
|
}
|
|
273
260
|
report = analysis.report;
|
|
274
261
|
content = format === 'raw' ? JSON.stringify(analysis, null, 1) : toText(analysis);
|