@effortlessmotion/pdf-to-md 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -76
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/package.json +2 -8
- package/skills/pdf-to-md/SKILL.md +39 -66
package/dist/esm/extract.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/extract.ts"],"sourcesContent":["// The library layer for PDF -> markdown conversion: extractText/extractOcr/\n// pdfToMarkdown each take plain args, return plain data, and throw PdfToMdError — no process.exit, no console.*, no file writing.\n//\n// Warnings are DATA: every notice is returned in `warnings` verbatim and also\n// delivered incrementally via `onWarning` — the exact strings are load-bearing (the test suite asserts them).\nimport { buildOcrAnalysisPages } from './analyze.ts';\nimport { collect } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { OcrProgressEvent } from './engines/tesseract.ts';\nimport type { PageWord, Tuning } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { AUTO_OCR_CHARS_PER_PAGE, buildOcrReport, buildReport } from './report.ts';\nimport type { PageLines, Report } from './types.ts';\nimport { validatePages } from './validate.ts';\n\nexport type { OcrProgressEvent } from './engines/tesseract.ts';\n\n// A plain-JS caller (or `.ts` with `as any`) gets no compile-time check on\n// `pages`, so it's validated once at runtime — see src/validate.ts's `validatePages`.\n\n/** One recognized word with its page — the OCR replay input contract (same\n * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */\nexport type OcrWordInput = PageWord;\n\n/** A 1-based, inclusive page range — the shape every entry point's `pages`\n * option takes (the CLI's own `--pages N[-M]` parses into this). */\nexport interface PageSelection {\n first: number;\n /** inclusive; defaults to `first` */\n last?: number;\n}\n\n/** extractText/extractOcr/pdfToMarkdown's common return shape. */\nexport interface ExtractResult {\n markdown: string;\n /** per-document and per-page QA metrics — see `Report` (`src/types.ts`) */\n report: Report;\n /** every notice also delivered incrementally via `onWarning`, collected\n * here in full (hybrid pages, auto-OCR-fallback, empty OCR pages, sparse-retry) */\n warnings: string[];\n}\n\ninterface CommonOptions {\n /** convert only this page range (1-based, inclusive); defaults to the\n * whole document */\n pages?: PageSelection;\n /** delivered as each warning happens (also returned, in full, as\n * `warnings` on the resolved result) */\n onWarning?: (message: string) => void;\n}\n\n/** Options for {@link extractText}. */\nexport interface ExtractTextOptions extends CommonOptions {\n /** emit a `### pN` marker before each page's content (output formatting\n * only — `report.pageStats` is always populated regardless) */\n pageMarkers?: boolean;\n}\n\n/** Options for {@link extractOcr}. */\nexport interface ExtractOcrOptions extends CommonOptions {\n /** override any subset of the OCR-path geometry defaults — see\n * DEFAULT_TUNING (`src/geometry.ts`) for each knob's measured default */\n tuning?: Partial<Tuning>;\n /** page-render resolution for engine recognition (default 288, measured;\n * >~400 measured WORSE on ordinary pages — not a \"bigger is safer\" knob) */\n dpi?: number;\n /** marks report.ocrFallback — set by pdfToMarkdown's auto-fallback */\n fallback?: boolean;\n /** one event per page as recognition finishes (timing/count only — engine\n * chatter and the sparse-retry notice arrive via `onWarning` instead) */\n onProgress?: (e: OcrProgressEvent) => void;\n /** every recognized word entering the geometry, delivered once after\n * recognition — the severity-1 audit instrument (--debug-words) */\n onWords?: (words: OcrWordInput[]) => void;\n}\n\n/** Options for {@link pdfToMarkdown}. */\nexport interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOcrOptions, 'fallback'> {\n /** force the OCR path even when a text layer exists */\n ocr?: boolean;\n /** never fall back to OCR, even for image-only pages */\n noOcr?: boolean;\n}\n\n/**\n * Text-layer extraction. Returns markdown + QA report + warnings (hybrid\n * pages carrying large images whose baked-in text the text layer can never see). Makes no OCR decision — see pdfToMarkdown for the fallback.\n *\n * @param pdfPath path to the PDF file\n * @param opts see {@link ExtractTextOptions}\n */\n// One collector per entry point: collects into `warnings`, streams via\n// onWarning, and emits the dictionary-unavailable notice exactly once up front.\nfunction warnCollector(onWarning?: (message: string) => void): { warnings: string[]; warn: (m: string) => void } {\n const warnings: string[] = [];\n const warn = (m: string) => {\n warnings.push(m);\n onWarning?.(m);\n };\n if (!dictAvailable()) warn(dictUnavailableWarning());\n return { warnings, warn };\n}\n\nexport async function extractText(pdfPath: string, opts: ExtractTextOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const { numPages, pages, heightChars } = await collect(pdfPath, range);\n const pageLines: PageLines[] = pages.map((p) => ({\n page: p.page,\n lines: toLines(p),\n }));\n const pageMarkers = opts.pageMarkers ?? false;\n const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, { pageMarkers });\n // pageStats needs \"### pN\" markers to segment content; when the caller\n // didn't request them, build a throwaway marked variant (same heightChars) purely to measure stats from, so pageStats is always populated.\n const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, { pageMarkers: true }).md;\n\n // Advisory, not prescriptive: `hasLargeImage` fires on any large image,\n // even one with no baked-in text, where `--ocr` recovers nothing and can\n // actively degrade a correct result (measured: \"I\" -> \"|\", stray em-dashes) — so this tells the caller to COMPARE the OCR rerun, never blindly replace.\n const hybridPages = pages.filter((p) => p.largeImage).map((p) => p.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n\n const report = buildReport({\n numPages,\n md,\n statsMd,\n stats,\n bodyH,\n headingSizes,\n hybridPages,\n });\n return { markdown: md, report, warnings };\n}\n\n/**\n * OCR extraction. Input is either a PDF (pages render at `dpi` and the\n * engine recognizes them, imported lazily) or an already-recognized word dump (the offline replay input — see `--words-json`).\n *\n * @param input `{ pdfPath }` to recognize, or `{ words }` to replay a dump\n * @param opts see {@link ExtractOcrOptions}\n */\nexport async function extractOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n\n // Shared with analyze(pdf, {path:'ocr'}) — see that module's header for\n // why this makes the two byte-identical by construction, not by hand.\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n\n const report = buildOcrReport({\n md,\n fallback: opts.fallback,\n // Confidence is on `words` — see report.ts's low-confidence detector for\n // how it handles a confidence-less replay honestly.\n words,\n });\n return { markdown: md, report, warnings };\n}\n\n/** pdfToMarkdown's auto-OCR fallback predicate, shared with the CLI's\n * --format=raw|txt path so the two call sites can never drift apart. Module-internal: not re-exported by `src/index.ts`. */\nexport function needsOcrFallback(charsPerPage: number): boolean {\n return charsPerPage < AUTO_OCR_CHARS_PER_PAGE;\n}\n\n/** The exact stderr/warnings notice printed when needsOcrFallback fires —\n * one copy of the string for both call sites (asserted verbatim by the suite). */\nexport function ocrFallbackNotice(charsPerPage: number): string {\n return `text layer yields ~${charsPerPage} chars/page (< ${AUTO_OCR_CHARS_PER_PAGE}) — pages are images of text; falling back to OCR (suppress with --no-ocr)`;\n}\n\n/**\n * The headline API: extract a PDF to markdown, text layer first, falling\n * back to OCR when the text layer is empty enough the pages are certainly images of text (< `AUTO_OCR_CHARS_PER_PAGE` chars/page) — the same decision the CLI makes.\n *\n * @param pdfPath path to the PDF file\n * @param opts see {@link PdfToMarkdownOptions}\n */\nexport async function pdfToMarkdown(pdfPath: string, opts: PdfToMarkdownOptions = {}): Promise<ExtractResult> {\n if (opts.ocr) return extractOcr({ pdfPath }, opts);\n\n // Warning DELIVERY is deferred on the text pass: if the fallback fires,\n // those hybrid warnings describe output being discarded, so only a kept text result replays them.\n const text = await extractText(pdfPath, { ...opts, onWarning: undefined });\n if (needsOcrFallback(text.report.charsPerPage) && !opts.noOcr) {\n const notice = ocrFallbackNotice(text.report.charsPerPage);\n opts.onWarning?.(notice);\n const ocr = await extractOcr({ pdfPath }, { ...opts, fallback: true });\n ocr.warnings.unshift(notice);\n return ocr;\n }\n for (const w of text.warnings) opts.onWarning?.(w);\n return text;\n}\n"],"names":["buildOcrAnalysisPages","collect","dictAvailable","dictUnavailableWarning","toLines","buildMarkdown","renderPagesToMarkdown","AUTO_OCR_CHARS_PER_PAGE","buildOcrReport","buildReport","validatePages","warnCollector","onWarning","warnings","warn","m","push","extractText","pdfPath","opts","pages","range","first","last","numPages","heightChars","pageLines","map","p","page","lines","pageMarkers","md","stats","bodyH","headingSizes","statsMd","hybridPages","filter","largeImage","n","report","markdown","extractOcr","input","words","forcePageMarkers","fallback","needsOcrFallback","charsPerPage","ocrFallbackNotice","pdfToMarkdown","ocr","text","undefined","noOcr","notice","unshift","w"],"mappings":"AAAA,4EAA4E;AAC5E,kIAAkI;AAClI,EAAE;AACF,8EAA8E;AAC9E,8GAA8G;AAC9G,SAASA,qBAAqB,QAAQ,eAAe;AACrD,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,aAAa,EAAEC,sBAAsB,QAAQ,YAAY;AAGlE,SAASC,OAAO,QAAQ,aAAa;AACrC,SAASC,aAAa,QAAQ,gBAAgB;AAC9C,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,uBAAuB,EAAEC,cAAc,EAAEC,WAAW,QAAQ,cAAc;AAEnF,SAASC,aAAa,QAAQ,gBAAgB;AAuE9C;;;;;;CAMC,GACD,uEAAuE;AACvE,gFAAgF;AAChF,SAASC,cAAcC,SAAqC;IAC1D,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,OAAO,CAACC;QACZF,SAASG,IAAI,CAACD;QACdH,sBAAAA,gCAAAA,UAAYG;IACd;IACA,IAAI,CAACb,iBAAiBY,KAAKX;IAC3B,OAAO;QAAEU;QAAUC;IAAK;AAC1B;AAEA,OAAO,eAAeG,YAAYC,OAAe,EAAEC,OAA2B,CAAC,CAAC;QAIlBA,kBAMxCA;IATpBT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IAEvD,MAAMS,QAAQF,KAAKC,KAAK,GAAG;QAAEE,OAAOH,KAAKC,KAAK,CAACE,KAAK;QAAEC,IAAI,GAAEJ,mBAAAA,KAAKC,KAAK,CAACG,IAAI,cAAfJ,8BAAAA,mBAAmBA,KAAKC,KAAK,CAACE,KAAK;IAAC,IAAI,CAAC;IACrG,MAAM,EAAEE,QAAQ,EAAEJ,KAAK,EAAEK,WAAW,EAAE,GAAG,MAAMxB,QAAQiB,SAASG;IAChE,MAAMK,YAAyBN,MAAMO,GAAG,CAAC,CAACC,IAAO,CAAA;YAC/CC,MAAMD,EAAEC,IAAI;YACZC,OAAO1B,QAAQwB;QACjB,CAAA;IACA,MAAMG,eAAcZ,oBAAAA,KAAKY,WAAW,cAAhBZ,+BAAAA,oBAAoB;IACxC,MAAM,EAAEa,EAAE,EAAEC,KAAK,EAAEC,KAAK,EAAEC,YAAY,EAAE,GAAG9B,cAAcqB,WAAWD,aAAa;QAAEM;IAAY;IAC/F,uEAAuE;IACvE,2IAA2I;IAC3I,MAAMK,UAAUL,cAAcC,KAAK3B,cAAcqB,WAAWD,aAAa;QAAEM,aAAa;IAAK,GAAGC,EAAE;IAElG,wEAAwE;IACxE,yEAAyE;IACzE,wJAAwJ;IACxJ,MAAMK,cAAcjB,MAAMkB,MAAM,CAAC,CAACV,IAAMA,EAAEW,UAAU,EAAEZ,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvE,KAAK,MAAMW,KAAKH,YAAa;QAC3BvB,KAAK,CAAC,CAAC,EAAE0B,EAAE,uJAAuJ,EAAEA,EAAE,yCAAyC,CAAC;IAClN;IAEA,MAAMC,SAAShC,YAAY;QACzBe;QACAQ;QACAI;QACAH;QACAC;QACAC;QACAE;IACF;IACA,OAAO;QAAEK,UAAUV;QAAIS;QAAQ5B;IAAS;AAC1C;AAEA;;;;;;CAMC,GACD,OAAO,eAAe8B,WAAWC,KAAsD,EAAEzB,OAA0B,CAAC,CAAC;IACnHT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IAEvD,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,EAAEQ,KAAK,EAAEyB,KAAK,EAAE,GAAG,MAAM7C,sBAAsB4C,OAAOzB,MAAML;IAClE,MAAMkB,KAAK1B,sBAAsBc,OAAO;QAAE0B,kBAAkB;QAAMf,aAAa;IAAK;IAEpF,MAAMU,SAASjC,eAAe;QAC5BwB;QACAe,UAAU5B,KAAK4B,QAAQ;QACvB,yEAAyE;QACzE,oDAAoD;QACpDF;IACF;IACA,OAAO;QAAEH,UAAUV;QAAIS;QAAQ5B;IAAS;AAC1C;AAEA;0HAC0H,GAC1H,OAAO,SAASmC,iBAAiBC,YAAoB;IACnD,OAAOA,eAAe1C;AACxB;AAEA;gFACgF,GAChF,OAAO,SAAS2C,kBAAkBD,YAAoB;IACpD,OAAO,CAAC,mBAAmB,EAAEA,aAAa,eAAe,EAAE1C,wBAAwB,0EAA0E,CAAC;AAChK;AAEA;;;;;;CAMC,GACD,OAAO,eAAe4C,cAAcjC,OAAe,EAAEC,OAA6B,CAAC,CAAC;QAanDA;IAZ/B,IAAIA,KAAKiC,GAAG,EAAE,OAAOT,WAAW;QAAEzB;IAAQ,GAAGC;IAE7C,wEAAwE;IACxE,kGAAkG;IAClG,MAAMkC,OAAO,MAAMpC,YAAYC,SAAS;QAAE,GAAGC,IAAI;QAAEP,WAAW0C;IAAU;IACxE,IAAIN,iBAAiBK,KAAKZ,MAAM,CAACQ,YAAY,KAAK,CAAC9B,KAAKoC,KAAK,EAAE;YAE7DpC;QADA,MAAMqC,SAASN,kBAAkBG,KAAKZ,MAAM,CAACQ,YAAY;SACzD9B,mBAAAA,KAAKP,SAAS,cAAdO,uCAAAA,sBAAAA,MAAiBqC;QACjB,MAAMJ,MAAM,MAAMT,WAAW;YAAEzB;QAAQ,GAAG;YAAE,GAAGC,IAAI;YAAE4B,UAAU;QAAK;QACpEK,IAAIvC,QAAQ,CAAC4C,OAAO,CAACD;QACrB,OAAOJ;IACT;IACA,KAAK,MAAMM,KAAKL,KAAKxC,QAAQ,EAAEM,kBAAAA,KAAKP,SAAS,cAAdO,sCAAAA,qBAAAA,MAAiBuC;IAChD,OAAOL;AACT"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/extract.ts"],"sourcesContent":["// The library layer for PDF -> markdown conversion: extractText/extractOcr/\n// pdfToMarkdown each take plain args, return plain data, and throw PdfToMdError — no process.exit, no console.*, no file writing.\n//\n// Warnings are DATA: every notice is returned in `warnings` verbatim and also\n// delivered incrementally via `onWarning` — the exact strings are load-bearing (the test suite asserts them).\nimport { buildOcrAnalysisPages } from './analyze.ts';\nimport { collect } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { OcrProgressEvent } from './engines/tesseract.ts';\nimport type { PageWord, Tuning } from './geometry.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { AUTO_OCR_CHARS_PER_PAGE, buildOcrReport } from './report.ts';\nimport { buildTextAnalysis, selectCollectedPages } from './text-analysis.ts';\nimport type { Analysis, AnalysisPage, CollectResult, Report } from './types.ts';\nimport { validatePages } from './validate.ts';\n\nexport type { OcrProgressEvent } from './engines/tesseract.ts';\n\n// A plain-JS caller (or `.ts` with `as any`) gets no compile-time check on\n// `pages`, so it's validated once at runtime — see src/validate.ts's `validatePages`.\n\n/** One recognized word with its page — the OCR replay input contract (same\n * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */\nexport type OcrWordInput = PageWord;\n\n/** A 1-based, inclusive page range — the shape every entry point's `pages`\n * option takes (the CLI's own `--pages N[-M]` parses into this). */\nexport interface PageSelection {\n first: number;\n /** inclusive; defaults to `first` */\n last?: number;\n}\n\n/** extractText/extractOcr/pdfToMarkdown's common return shape. */\nexport interface ExtractResult {\n markdown: string;\n /** per-document and per-page QA metrics — see `Report` (`src/types.ts`) */\n report: Report;\n /** every notice also delivered incrementally via `onWarning`, collected\n * here in full (hybrid pages, auto-OCR-fallback, empty OCR pages, sparse-retry) */\n warnings: string[];\n}\n\ninterface CommonOptions {\n /** convert only this page range (1-based, inclusive); defaults to the\n * whole document */\n pages?: PageSelection;\n /** delivered as each warning happens (also returned, in full, as\n * `warnings` on the resolved result) */\n onWarning?: (message: string) => void;\n}\n\n/** Options for {@link extractText}. */\nexport interface ExtractTextOptions extends CommonOptions {\n /** emit a `### pN` marker before each page's content (output formatting\n * only — `report.pageStats` is always populated regardless) */\n pageMarkers?: boolean;\n /** source PDF text runs entering layout, delivered once for the selected\n * final result. Runs retain measured bounds and are not fabricated words. */\n onWords?: (words: OcrWordInput[]) => void;\n}\n\n/** Options for {@link extractOcr}. */\nexport interface ExtractOcrOptions extends CommonOptions {\n /** override any subset of the OCR-path geometry defaults — see\n * DEFAULT_TUNING (`src/geometry.ts`) for each knob's measured default */\n tuning?: Partial<Tuning>;\n /** page-render resolution for engine recognition (default 288, measured;\n * >~400 measured WORSE on ordinary pages — not a \"bigger is safer\" knob) */\n dpi?: number;\n /** marks report.ocrFallback — set by pdfToMarkdown's auto-fallback */\n fallback?: boolean;\n /** one event per page as recognition finishes (timing/count only — engine\n * chatter and the sparse-retry notice arrive via `onWarning` instead) */\n onProgress?: (e: OcrProgressEvent) => void;\n /** every recognized word entering the geometry, delivered once after\n * recognition — the severity-1 audit instrument (--debug-words) */\n onWords?: (words: OcrWordInput[]) => void;\n}\n\n/** Options for {@link pdfToMarkdown}. */\nexport interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOcrOptions, 'fallback'> {\n /** force the OCR path even when a text layer exists */\n ocr?: boolean;\n /** never fall back to OCR, even for image-only pages */\n noOcr?: boolean;\n}\n\n// One collector per entry point: collects into `warnings`, streams via\n// onWarning, and emits the dictionary-unavailable notice exactly once up front.\nfunction warnCollector(onWarning?: (message: string) => void): { warnings: string[]; warn: (m: string) => void } {\n const warnings: string[] = [];\n const warn = (m: string) => {\n warnings.push(m);\n onWarning?.(m);\n };\n if (!dictAvailable()) warn(dictUnavailableWarning());\n return { warnings, warn };\n}\n\n/** Extract embedded text with a QA report and warnings; never runs OCR.\n * @param pdfPath path to the PDF file\n * @param opts see {@link ExtractTextOptions}\n */\nexport async function extractText(pdfPath: string, opts: ExtractTextOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const collected = await collect(pdfPath, range);\n const text = buildTextAnalysis(collected, opts.pageMarkers ?? false);\n textNotices(collected, new Set(), warn);\n opts.onWords?.(text.words);\n return { markdown: text.markdown, report: text.analysis.report, warnings };\n}\n\n/**\n * OCR extraction. Input is either a PDF (pages render at `dpi` and the\n * engine recognizes them, imported lazily) or an already-recognized word dump (the offline replay input — see `--words-json`).\n *\n * @param input `{ pdfPath }` to recognize, or `{ words }` to replay a dump\n * @param opts see {@link ExtractOcrOptions}\n */\nexport async function extractOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n\n // Shared with analyze(pdf, {path:'ocr'}) — see that module's header for\n // why this makes the two byte-identical by construction, not by hand.\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n\n const report = buildOcrReport({\n md,\n fallback: opts.fallback,\n // Confidence is on `words` — see report.ts's low-confidence detector for\n // how it handles a confidence-less replay honestly.\n words,\n });\n return { markdown: md, report, warnings };\n}\n\n/** A page needs OCR only when its image signal accompanies an unusable text\n * layer. The caller supplies the image part of that decision. */\nfunction needsPageOcr(chars: number): boolean {\n return chars < AUTO_OCR_CHARS_PER_PAGE;\n}\n\nfunction textNotices(collected: CollectResult, ocrPages: Set<number>, warn: (message: string) => void): void {\n for (const page of collected.pages) {\n if (page.textChars !== 0 || ocrPages.has(page.page)) continue;\n if (page.hasImage) warn(`p${page.page}: no extractable text; page has image content and OCR was not selected`);\n else warn(`p${page.page}: no extractable text; the page may be blank or contain only vector content`);\n }\n for (const page of collected.pages) {\n if (!page.largeImage || ocrPages.has(page.page)) continue;\n warn(`p${page.page}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${page.page} --ocr, which can itself introduce errors`);\n }\n}\n\nfunction analysisWords(pages: AnalysisPage[]): OcrWordInput[] {\n return pages.flatMap((page) => page.words.map((word) => ({ page: page.page, text: word.text, ...word.box, confidence: word.confidence })));\n}\n\ninterface AutoAnalysisResult {\n analysis: Analysis;\n markdown: string;\n warnings: string[];\n}\n\n/** Shared automatic route for the library's Markdown result and the CLI's\n * raw/plain-text formats. A page changes producer only when it has image\n * evidence and an unusable text layer, so a blank or vector-only page never\n * becomes speculative OCR input. */\nexport async function analyzePdfAuto(pdfPath: string, opts: PdfToMarkdownOptions = {}): Promise<AutoAnalysisResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n\n if (opts.ocr) {\n const { pages, words } = await buildOcrAnalysisPages({ pdfPath }, { ...opts, onWords: undefined }, warn);\n const markdown = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md: markdown, words });\n const analysis: Analysis = { outputVersion: report.outputVersion, report, pages };\n opts.onWords?.(words);\n return { analysis, markdown, warnings };\n }\n\n const collected = await collect(pdfPath, range);\n const ocrPageNumbers = new Set(!opts.noOcr ? collected.pages.filter((page) => page.hasImage && needsPageOcr(page.textChars)).map((page) => page.page) : []);\n if (!ocrPageNumbers.size) {\n const text = buildTextAnalysis(collected, opts.pageMarkers ?? false);\n textNotices(collected, ocrPageNumbers, warn);\n opts.onWords?.(text.words);\n return { analysis: text.analysis, markdown: text.markdown, warnings };\n }\n\n const textPageNumbers = new Set(collected.pages.filter((page) => !ocrPageNumbers.has(page.page)).map((page) => page.page));\n const text = textPageNumbers.size ? buildTextAnalysis(selectCollectedPages(collected, textPageNumbers), true) : undefined;\n const textByPage = new Map(text?.analysis.pages.map((page) => [page.page, page]));\n const ocrByPage = new Map<number, AnalysisPage>();\n const ocrWords: OcrWordInput[] = [];\n const ocrRanges: Array<{ first: number; last: number }> = [];\n for (const page of collected.pages) {\n if (!ocrPageNumbers.has(page.page)) continue;\n warn(`p${page.page}: sparse text layer with image content; using OCR for this page`);\n const lastRange = ocrRanges.at(-1);\n if (lastRange && lastRange.last + 1 === page.page) lastRange.last = page.page;\n else ocrRanges.push({ first: page.page, last: page.page });\n }\n for (const range of ocrRanges) {\n const result = await buildOcrAnalysisPages({ pdfPath }, { ...opts, pages: range, onWords: undefined }, warn);\n for (const page of result.pages) ocrByPage.set(page.page, page);\n ocrWords.push(...result.words);\n }\n textNotices(collected, ocrPageNumbers, warn);\n const pages = collected.pages.map((page) => ocrByPage.get(page.page) ?? (textByPage.get(page.page) as AnalysisPage));\n const markdown = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const pagePaths = new Map(pages.map((page) => [page.page, page.path]));\n const report = buildOcrReport({\n md: markdown,\n fallback: true,\n words: ocrWords,\n pagePaths,\n hybridPages: text?.analysis.report.hybridPages,\n });\n if (text) {\n report.bodyFontHeight = text.analysis.report.bodyFontHeight;\n report.headingSizes = text.analysis.report.headingSizes;\n report.listItems = text.analysis.report.listItems;\n report.joinedPairs = text.analysis.report.joinedPairs;\n report.closedHyphens = text.analysis.report.closedHyphens;\n report.hybridPages = text.analysis.report.hybridPages;\n }\n const analysis: Analysis = { outputVersion: report.outputVersion, report, pages };\n opts.onWords?.(analysisWords(pages));\n return { analysis, markdown, warnings };\n}\n\n/**\n * Extract each page from embedded text, using OCR only when the page has\n * image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.\n *\n * @param pdfPath path to the PDF file\n * @param opts see {@link PdfToMarkdownOptions}\n */\nexport async function pdfToMarkdown(pdfPath: string, opts: PdfToMarkdownOptions = {}): Promise<ExtractResult> {\n const result = await analyzePdfAuto(pdfPath, opts);\n return { markdown: result.markdown, report: result.analysis.report, warnings: result.warnings };\n}\n"],"names":["buildOcrAnalysisPages","collect","dictAvailable","dictUnavailableWarning","renderPagesToMarkdown","AUTO_OCR_CHARS_PER_PAGE","buildOcrReport","buildTextAnalysis","selectCollectedPages","validatePages","warnCollector","onWarning","warnings","warn","m","push","extractText","pdfPath","opts","pages","range","first","last","collected","text","pageMarkers","textNotices","Set","onWords","words","markdown","report","analysis","extractOcr","input","md","forcePageMarkers","fallback","needsPageOcr","chars","ocrPages","page","textChars","has","hasImage","largeImage","analysisWords","flatMap","map","word","box","confidence","analyzePdfAuto","ocr","undefined","outputVersion","ocrPageNumbers","noOcr","filter","size","textPageNumbers","textByPage","Map","ocrByPage","ocrWords","ocrRanges","lastRange","at","result","set","get","pagePaths","path","hybridPages","bodyFontHeight","headingSizes","listItems","joinedPairs","closedHyphens","pdfToMarkdown"],"mappings":"AAAA,4EAA4E;AAC5E,kIAAkI;AAClI,EAAE;AACF,8EAA8E;AAC9E,8GAA8G;AAC9G,SAASA,qBAAqB,QAAQ,eAAe;AACrD,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,aAAa,EAAEC,sBAAsB,QAAQ,YAAY;AAGlE,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,uBAAuB,EAAEC,cAAc,QAAQ,cAAc;AACtE,SAASC,iBAAiB,EAAEC,oBAAoB,QAAQ,qBAAqB;AAE7E,SAASC,aAAa,QAAQ,gBAAgB;AA0E9C,uEAAuE;AACvE,gFAAgF;AAChF,SAASC,cAAcC,SAAqC;IAC1D,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,OAAO,CAACC;QACZF,SAASG,IAAI,CAACD;QACdH,sBAAAA,gCAAAA,UAAYG;IACd;IACA,IAAI,CAACZ,iBAAiBW,KAAKV;IAC3B,OAAO;QAAES;QAAUC;IAAK;AAC1B;AAEA;;;CAGC,GACD,OAAO,eAAeG,YAAYC,OAAe,EAAEC,OAA2B,CAAC,CAAC;QAGlBA,kBAElBA;QAE1CA;IANAT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IACvD,MAAMS,QAAQF,KAAKC,KAAK,GAAG;QAAEE,OAAOH,KAAKC,KAAK,CAACE,KAAK;QAAEC,IAAI,GAAEJ,mBAAAA,KAAKC,KAAK,CAACG,IAAI,cAAfJ,8BAAAA,mBAAmBA,KAAKC,KAAK,CAACE,KAAK;IAAC,IAAI,CAAC;IACrG,MAAME,YAAY,MAAMtB,QAAQgB,SAASG;IACzC,MAAMI,OAAOjB,kBAAkBgB,YAAWL,oBAAAA,KAAKO,WAAW,cAAhBP,+BAAAA,oBAAoB;IAC9DQ,YAAYH,WAAW,IAAII,OAAOd;KAClCK,gBAAAA,KAAKU,OAAO,cAAZV,oCAAAA,mBAAAA,MAAeM,KAAKK,KAAK;IACzB,OAAO;QAAEC,UAAUN,KAAKM,QAAQ;QAAEC,QAAQP,KAAKQ,QAAQ,CAACD,MAAM;QAAEnB;IAAS;AAC3E;AAEA;;;;;;CAMC,GACD,OAAO,eAAeqB,WAAWC,KAAsD,EAAEhB,OAA0B,CAAC,CAAC;IACnHT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IAEvD,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,EAAEQ,KAAK,EAAEU,KAAK,EAAE,GAAG,MAAM7B,sBAAsBkC,OAAOhB,MAAML;IAClE,MAAMsB,KAAK/B,sBAAsBe,OAAO;QAAEiB,kBAAkB;QAAMX,aAAa;IAAK;IAEpF,MAAMM,SAASzB,eAAe;QAC5B6B;QACAE,UAAUnB,KAAKmB,QAAQ;QACvB,yEAAyE;QACzE,oDAAoD;QACpDR;IACF;IACA,OAAO;QAAEC,UAAUK;QAAIJ;QAAQnB;IAAS;AAC1C;AAEA;+DAC+D,GAC/D,SAAS0B,aAAaC,KAAa;IACjC,OAAOA,QAAQlC;AACjB;AAEA,SAASqB,YAAYH,SAAwB,EAAEiB,QAAqB,EAAE3B,IAA+B;IACnG,KAAK,MAAM4B,QAAQlB,UAAUJ,KAAK,CAAE;QAClC,IAAIsB,KAAKC,SAAS,KAAK,KAAKF,SAASG,GAAG,CAACF,KAAKA,IAAI,GAAG;QACrD,IAAIA,KAAKG,QAAQ,EAAE/B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,sEAAsE,CAAC;aACxG5B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,2EAA2E,CAAC;IACtG;IACA,KAAK,MAAMA,QAAQlB,UAAUJ,KAAK,CAAE;QAClC,IAAI,CAACsB,KAAKI,UAAU,IAAIL,SAASG,GAAG,CAACF,KAAKA,IAAI,GAAG;QACjD5B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,uJAAuJ,EAAEA,KAAKA,IAAI,CAAC,yCAAyC,CAAC;IAClO;AACF;AAEA,SAASK,cAAc3B,KAAqB;IAC1C,OAAOA,MAAM4B,OAAO,CAAC,CAACN,OAASA,KAAKZ,KAAK,CAACmB,GAAG,CAAC,CAACC,OAAU,CAAA;gBAAER,MAAMA,KAAKA,IAAI;gBAAEjB,MAAMyB,KAAKzB,IAAI;gBAAE,GAAGyB,KAAKC,GAAG;gBAAEC,YAAYF,KAAKE,UAAU;YAAC,CAAA;AACxI;AAQA;;;kCAGkC,GAClC,OAAO,eAAeC,eAAenC,OAAe,EAAEC,OAA6B,CAAC,CAAC;QAGvBA;QA0D5DA;IA5DAT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IACvD,MAAMS,QAAQF,KAAKC,KAAK,GAAG;QAAEE,OAAOH,KAAKC,KAAK,CAACE,KAAK;QAAEC,IAAI,GAAEJ,mBAAAA,KAAKC,KAAK,CAACG,IAAI,cAAfJ,8BAAAA,mBAAmBA,KAAKC,KAAK,CAACE,KAAK;IAAC,IAAI,CAAC;IAErG,IAAIH,KAAKmC,GAAG,EAAE;YAKZnC;QAJA,MAAM,EAAEC,KAAK,EAAEU,KAAK,EAAE,GAAG,MAAM7B,sBAAsB;YAAEiB;QAAQ,GAAG;YAAE,GAAGC,IAAI;YAAEU,SAAS0B;QAAU,GAAGzC;QACnG,MAAMiB,WAAW1B,sBAAsBe,OAAO;YAAEiB,kBAAkB;YAAMX,aAAa;QAAK;QAC1F,MAAMM,SAASzB,eAAe;YAAE6B,IAAIL;YAAUD;QAAM;QACpD,MAAMG,WAAqB;YAAEuB,eAAexB,OAAOwB,aAAa;YAAExB;YAAQZ;QAAM;SAChFD,iBAAAA,KAAKU,OAAO,cAAZV,qCAAAA,oBAAAA,MAAeW;QACf,OAAO;YAAEG;YAAUF;YAAUlB;QAAS;IACxC;IAEA,MAAMW,YAAY,MAAMtB,QAAQgB,SAASG;IACzC,MAAMoC,iBAAiB,IAAI7B,IAAI,CAACT,KAAKuC,KAAK,GAAGlC,UAAUJ,KAAK,CAACuC,MAAM,CAAC,CAACjB,OAASA,KAAKG,QAAQ,IAAIN,aAAaG,KAAKC,SAAS,GAAGM,GAAG,CAAC,CAACP,OAASA,KAAKA,IAAI,IAAI,EAAE;IAC1J,IAAI,CAACe,eAAeG,IAAI,EAAE;YACkBzC;YAE1CA;QAFA,MAAMM,OAAOjB,kBAAkBgB,YAAWL,oBAAAA,KAAKO,WAAW,cAAhBP,+BAAAA,oBAAoB;QAC9DQ,YAAYH,WAAWiC,gBAAgB3C;SACvCK,iBAAAA,KAAKU,OAAO,cAAZV,qCAAAA,oBAAAA,MAAeM,KAAKK,KAAK;QACzB,OAAO;YAAEG,UAAUR,KAAKQ,QAAQ;YAAEF,UAAUN,KAAKM,QAAQ;YAAElB;QAAS;IACtE;IAEA,MAAMgD,kBAAkB,IAAIjC,IAAIJ,UAAUJ,KAAK,CAACuC,MAAM,CAAC,CAACjB,OAAS,CAACe,eAAeb,GAAG,CAACF,KAAKA,IAAI,GAAGO,GAAG,CAAC,CAACP,OAASA,KAAKA,IAAI;IACxH,MAAMjB,OAAOoC,gBAAgBD,IAAI,GAAGpD,kBAAkBC,qBAAqBe,WAAWqC,kBAAkB,QAAQN;IAChH,MAAMO,aAAa,IAAIC,IAAItC,iBAAAA,2BAAAA,KAAMQ,QAAQ,CAACb,KAAK,CAAC6B,GAAG,CAAC,CAACP,OAAS;YAACA,KAAKA,IAAI;YAAEA;SAAK;IAC/E,MAAMsB,YAAY,IAAID;IACtB,MAAME,WAA2B,EAAE;IACnC,MAAMC,YAAoD,EAAE;IAC5D,KAAK,MAAMxB,QAAQlB,UAAUJ,KAAK,CAAE;QAClC,IAAI,CAACqC,eAAeb,GAAG,CAACF,KAAKA,IAAI,GAAG;QACpC5B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,+DAA+D,CAAC;QACnF,MAAMyB,YAAYD,UAAUE,EAAE,CAAC,CAAC;QAChC,IAAID,aAAaA,UAAU5C,IAAI,GAAG,MAAMmB,KAAKA,IAAI,EAAEyB,UAAU5C,IAAI,GAAGmB,KAAKA,IAAI;aACxEwB,UAAUlD,IAAI,CAAC;YAAEM,OAAOoB,KAAKA,IAAI;YAAEnB,MAAMmB,KAAKA,IAAI;QAAC;IAC1D;IACA,KAAK,MAAMrB,SAAS6C,UAAW;QAC7B,MAAMG,SAAS,MAAMpE,sBAAsB;YAAEiB;QAAQ,GAAG;YAAE,GAAGC,IAAI;YAAEC,OAAOC;YAAOQ,SAAS0B;QAAU,GAAGzC;QACvG,KAAK,MAAM4B,QAAQ2B,OAAOjD,KAAK,CAAE4C,UAAUM,GAAG,CAAC5B,KAAKA,IAAI,EAAEA;QAC1DuB,SAASjD,IAAI,IAAIqD,OAAOvC,KAAK;IAC/B;IACAH,YAAYH,WAAWiC,gBAAgB3C;IACvC,MAAMM,QAAQI,UAAUJ,KAAK,CAAC6B,GAAG,CAAC,CAACP;YAASsB;gBAAAA,iBAAAA,UAAUO,GAAG,CAAC7B,KAAKA,IAAI,eAAvBsB,4BAAAA,iBAA6BF,WAAWS,GAAG,CAAC7B,KAAKA,IAAI;;IACjG,MAAMX,WAAW1B,sBAAsBe,OAAO;QAAEiB,kBAAkB;QAAMX,aAAa;IAAK;IAC1F,MAAM8C,YAAY,IAAIT,IAAI3C,MAAM6B,GAAG,CAAC,CAACP,OAAS;YAACA,KAAKA,IAAI;YAAEA,KAAK+B,IAAI;SAAC;IACpE,MAAMzC,SAASzB,eAAe;QAC5B6B,IAAIL;QACJO,UAAU;QACVR,OAAOmC;QACPO;QACAE,WAAW,EAAEjD,iBAAAA,2BAAAA,KAAMQ,QAAQ,CAACD,MAAM,CAAC0C,WAAW;IAChD;IACA,IAAIjD,MAAM;QACRO,OAAO2C,cAAc,GAAGlD,KAAKQ,QAAQ,CAACD,MAAM,CAAC2C,cAAc;QAC3D3C,OAAO4C,YAAY,GAAGnD,KAAKQ,QAAQ,CAACD,MAAM,CAAC4C,YAAY;QACvD5C,OAAO6C,SAAS,GAAGpD,KAAKQ,QAAQ,CAACD,MAAM,CAAC6C,SAAS;QACjD7C,OAAO8C,WAAW,GAAGrD,KAAKQ,QAAQ,CAACD,MAAM,CAAC8C,WAAW;QACrD9C,OAAO+C,aAAa,GAAGtD,KAAKQ,QAAQ,CAACD,MAAM,CAAC+C,aAAa;QACzD/C,OAAO0C,WAAW,GAAGjD,KAAKQ,QAAQ,CAACD,MAAM,CAAC0C,WAAW;IACvD;IACA,MAAMzC,WAAqB;QAAEuB,eAAexB,OAAOwB,aAAa;QAAExB;QAAQZ;IAAM;KAChFD,gBAAAA,KAAKU,OAAO,cAAZV,oCAAAA,mBAAAA,MAAe4B,cAAc3B;IAC7B,OAAO;QAAEa;QAAUF;QAAUlB;IAAS;AACxC;AAEA;;;;;;CAMC,GACD,OAAO,eAAemE,cAAc9D,OAAe,EAAEC,OAA6B,CAAC,CAAC;IAClF,MAAMkD,SAAS,MAAMhB,eAAenC,SAASC;IAC7C,OAAO;QAAEY,UAAUsC,OAAOtC,QAAQ;QAAEC,QAAQqC,OAAOpC,QAAQ,CAACD,MAAM;QAAEnB,UAAUwD,OAAOxD,QAAQ;IAAC;AAChG"}
|
package/dist/esm/lines.js
CHANGED
|
@@ -20,31 +20,34 @@ export function toLines({ glyphs, width }) {
|
|
|
20
20
|
const twoCol = glyphs.filter((g)=>g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g)=>g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g)=>g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;
|
|
21
21
|
// Group on y with a tolerance that scales with font size.
|
|
22
22
|
const lines = [];
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
const indexedGlyphs = glyphs.map((glyph, index)=>({
|
|
24
|
+
glyph,
|
|
25
|
+
index
|
|
26
|
+
}));
|
|
27
|
+
for (const part of indexedGlyphs.sort((a, b)=>a.glyph.y - b.glyph.y || a.glyph.x - b.glyph.x)){
|
|
28
|
+
const { glyph: g } = part;
|
|
26
29
|
// NOTE: start-x assignment can misfile a gutter-straddling glyph; an
|
|
27
30
|
// overlap-based alternative was tried and reshuffled stray numerals with
|
|
28
31
|
// no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).
|
|
29
32
|
const col = twoCol && g.x > mid ? 1 : 0;
|
|
30
33
|
const last = lines.at(-1);
|
|
31
34
|
if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {
|
|
32
|
-
last.parts.push(
|
|
35
|
+
last.parts.push(part);
|
|
33
36
|
last.y = (last.y + g.y) / 2;
|
|
34
37
|
} else lines.push({
|
|
35
38
|
col,
|
|
36
39
|
y: g.y,
|
|
37
40
|
parts: [
|
|
38
|
-
|
|
41
|
+
part
|
|
39
42
|
]
|
|
40
43
|
});
|
|
41
44
|
}
|
|
42
45
|
lines.sort((a, b)=>a.col - b.col || a.y - b.y); // column, then down the page
|
|
43
46
|
return lines.map((l)=>{
|
|
44
|
-
const parts = l.parts.sort((a, b)=>a.x - b.x);
|
|
47
|
+
const parts = l.parts.sort((a, b)=>a.glyph.x - b.glyph.x);
|
|
45
48
|
let t = '';
|
|
46
49
|
let prev = null;
|
|
47
|
-
for (const cur of parts){
|
|
50
|
+
for (const { glyph: cur } of parts){
|
|
48
51
|
// insert a space when the measured gap is wider than intra-word kerning
|
|
49
52
|
if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\s$/.test(t) && !/^\s/.test(cur.s)) t += ' ';
|
|
50
53
|
t += cur.s;
|
|
@@ -53,15 +56,16 @@ export function toLines({ glyphs, width }) {
|
|
|
53
56
|
// Line height = the char-weighted dominant height, not the max: a small
|
|
54
57
|
// decorative large-face numeral must not promote a body line to a heading.
|
|
55
58
|
const byH = new Map();
|
|
56
|
-
for (const p of parts)byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);
|
|
59
|
+
for (const { glyph: p } of parts)byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);
|
|
57
60
|
return {
|
|
58
61
|
text: normalize(t).replace(/\s+/g, ' ').trim(),
|
|
59
|
-
x: parts[0].x,
|
|
62
|
+
x: parts[0].glyph.x,
|
|
60
63
|
y: l.y,
|
|
61
64
|
h: [
|
|
62
65
|
...byH.entries()
|
|
63
66
|
].sort((a, b)=>b[1] - a[1])[0][0],
|
|
64
|
-
col: l.col
|
|
67
|
+
col: l.col,
|
|
68
|
+
wordIndexes: parts.map(({ index })=>index)
|
|
65
69
|
};
|
|
66
70
|
}).filter((l)=>l.text);
|
|
67
71
|
}
|
package/dist/esm/lines.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/lines.ts"],"sourcesContent":["// Pass 2: glyphs -> visual lines — de-braids two-column pages (spanning\n// lines like headings sort with the left column), joins words by measured horizontal gap, and folds typographic codepoints to ASCII so search doesn't miss half the hits.\nimport type { CollectedPage, Glyph, Line } from './types.ts';\n\nconst FOLD: Record<string, string> = {\n ʼ: \"'\",\n '’': \"'\",\n '‘': \"'\",\n '“': '\"',\n '”': '\"',\n '–': '-',\n};\n// Exported for the OCR engine seam (src/engines/tesseract.ts): both paths\n// must fold identically or the same printed apostrophe surfaces as two spellings depending on which path read the page.\nexport const foldTypographic = (s: string): string => s.replace(/[ʼ‘’“”–]/g, (c) => FOLD[c]);\nconst normalize = foldTypographic;\n\n// Intermediate row while grouping glyphs by visual line, before the final\n// text/height reduction below.\ninterface LineDraft {\n col: number;\n y: number;\n parts: Glyph
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/lines.ts"],"sourcesContent":["// Pass 2: glyphs -> visual lines — de-braids two-column pages (spanning\n// lines like headings sort with the left column), joins words by measured horizontal gap, and folds typographic codepoints to ASCII so search doesn't miss half the hits.\nimport type { CollectedPage, Glyph, Line } from './types.ts';\n\nconst FOLD: Record<string, string> = {\n ʼ: \"'\",\n '’': \"'\",\n '‘': \"'\",\n '“': '\"',\n '”': '\"',\n '–': '-',\n};\n// Exported for the OCR engine seam (src/engines/tesseract.ts): both paths\n// must fold identically or the same printed apostrophe surfaces as two spellings depending on which path read the page.\nexport const foldTypographic = (s: string): string => s.replace(/[ʼ‘’“”–]/g, (c) => FOLD[c]);\nconst normalize = foldTypographic;\n\n// Intermediate row while grouping glyphs by visual line, before the final\n// text/height reduction below.\ninterface LineDraft {\n col: number;\n y: number;\n parts: Array<{ glyph: Glyph; index: number }>;\n}\n\nexport function toLines({ glyphs, width }: Pick<CollectedPage, 'glyphs' | 'width'>): Line[] {\n if (!glyphs.length) return [];\n\n const mid = width / 2;\n // A real two-column page has a gutter: clusters each side, almost nothing crossing mid.\n // Label/value layouts also cluster but their values cross mid — de-braiding would reorder them.\n const twoCol = glyphs.filter((g) => g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;\n\n // Group on y with a tolerance that scales with font size.\n const lines: LineDraft[] = [];\n const indexedGlyphs = glyphs.map((glyph, index) => ({ glyph, index }));\n for (const part of indexedGlyphs.sort((a, b) => a.glyph.y - b.glyph.y || a.glyph.x - b.glyph.x)) {\n const { glyph: g } = part;\n // NOTE: start-x assignment can misfile a gutter-straddling glyph; an\n // overlap-based alternative was tried and reshuffled stray numerals with\n // no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).\n const col = twoCol && g.x > mid ? 1 : 0;\n const last = lines.at(-1);\n if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {\n last.parts.push(part);\n last.y = (last.y + g.y) / 2;\n } else lines.push({ col, y: g.y, parts: [part] });\n }\n lines.sort((a, b) => a.col - b.col || a.y - b.y); // column, then down the page\n\n return lines\n .map((l): Line => {\n const parts = l.parts.sort((a, b) => a.glyph.x - b.glyph.x);\n let t = '';\n let prev: Glyph | null = null;\n for (const { glyph: cur } of parts) {\n // insert a space when the measured gap is wider than intra-word kerning\n if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\\s$/.test(t) && !/^\\s/.test(cur.s)) t += ' ';\n t += cur.s;\n prev = cur;\n }\n // Line height = the char-weighted dominant height, not the max: a small\n // decorative large-face numeral must not promote a body line to a heading.\n const byH = new Map<number, number>();\n for (const { glyph: p } of parts) byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);\n return {\n text: normalize(t).replace(/\\s+/g, ' ').trim(),\n x: parts[0].glyph.x,\n y: l.y,\n h: [...byH.entries()].sort((a, b) => b[1] - a[1])[0][0],\n col: l.col,\n wordIndexes: parts.map(({ index }) => index),\n };\n })\n .filter((l) => l.text);\n}\n"],"names":["FOLD","ʼ","foldTypographic","s","replace","c","normalize","toLines","glyphs","width","length","mid","twoCol","filter","g","x","w","lines","indexedGlyphs","map","glyph","index","part","sort","a","b","y","col","last","at","Math","abs","max","h","parts","push","l","t","prev","cur","test","byH","Map","p","set","get","text","trim","entries","wordIndexes"],"mappings":"AAAA,wEAAwE;AACxE,0KAA0K;AAG1K,MAAMA,OAA+B;IACnCC,GAAG;IACH,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;AACP;AACA,0EAA0E;AAC1E,wHAAwH;AACxH,OAAO,MAAMC,kBAAkB,CAACC,IAAsBA,EAAEC,OAAO,CAAC,aAAa,CAACC,IAAML,IAAI,CAACK,EAAE,EAAE;AAC7F,MAAMC,YAAYJ;AAUlB,OAAO,SAASK,QAAQ,EAAEC,MAAM,EAAEC,KAAK,EAA2C;IAChF,IAAI,CAACD,OAAOE,MAAM,EAAE,OAAO,EAAE;IAE7B,MAAMC,MAAMF,QAAQ;IACpB,wFAAwF;IACxF,gGAAgG;IAChG,MAAMG,SAASJ,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,OAAOG,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG;IAEnO,0DAA0D;IAC1D,MAAMO,QAAqB,EAAE;IAC7B,MAAMC,gBAAgBV,OAAOW,GAAG,CAAC,CAACC,OAAOC,QAAW,CAAA;YAAED;YAAOC;QAAM,CAAA;IACnE,KAAK,MAAMC,QAAQJ,cAAcK,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEJ,KAAK,CAACM,CAAC,GAAGD,EAAEL,KAAK,CAACM,CAAC,IAAIF,EAAEJ,KAAK,CAACL,CAAC,GAAGU,EAAEL,KAAK,CAACL,CAAC,EAAG;QAC/F,MAAM,EAAEK,OAAON,CAAC,EAAE,GAAGQ;QACrB,qEAAqE;QACrE,yEAAyE;QACzE,0IAA0I;QAC1I,MAAMK,MAAMf,UAAUE,EAAEC,CAAC,GAAGJ,MAAM,IAAI;QACtC,MAAMiB,OAAOX,MAAMY,EAAE,CAAC,CAAC;QACvB,IAAID,QAAQA,KAAKD,GAAG,KAAKA,OAAOG,KAAKC,GAAG,CAACH,KAAKF,CAAC,GAAGZ,EAAEY,CAAC,IAAII,KAAKE,GAAG,CAAC,GAAGlB,EAAEmB,CAAC,GAAG,MAAM;YAC/EL,KAAKM,KAAK,CAACC,IAAI,CAACb;YAChBM,KAAKF,CAAC,GAAG,AAACE,CAAAA,KAAKF,CAAC,GAAGZ,EAAEY,CAAC,AAADA,IAAK;QAC5B,OAAOT,MAAMkB,IAAI,CAAC;YAAER;YAAKD,GAAGZ,EAAEY,CAAC;YAAEQ,OAAO;gBAACZ;aAAK;QAAC;IACjD;IACAL,MAAMM,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEG,GAAG,GAAGF,EAAEE,GAAG,IAAIH,EAAEE,CAAC,GAAGD,EAAEC,CAAC,GAAG,6BAA6B;IAE/E,OAAOT,MACJE,GAAG,CAAC,CAACiB;QACJ,MAAMF,QAAQE,EAAEF,KAAK,CAACX,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEJ,KAAK,CAACL,CAAC,GAAGU,EAAEL,KAAK,CAACL,CAAC;QAC1D,IAAIsB,IAAI;QACR,IAAIC,OAAqB;QACzB,KAAK,MAAM,EAAElB,OAAOmB,GAAG,EAAE,IAAIL,MAAO;YAClC,wEAAwE;YACxE,IAAII,QAAQC,IAAIxB,CAAC,GAAIuB,CAAAA,KAAKvB,CAAC,GAAGuB,KAAKtB,CAAC,AAADA,IAAKsB,KAAKL,CAAC,GAAG,QAAQ,CAAC,MAAMO,IAAI,CAACH,MAAM,CAAC,MAAMG,IAAI,CAACD,IAAIpC,CAAC,GAAGkC,KAAK;YACpGA,KAAKE,IAAIpC,CAAC;YACVmC,OAAOC;QACT;QACA,wEAAwE;QACxE,2EAA2E;QAC3E,MAAME,MAAM,IAAIC;QAChB,KAAK,MAAM,EAAEtB,OAAOuB,CAAC,EAAE,IAAIT,MAAOO,IAAIG,GAAG,CAACD,EAAEV,CAAC,EAAE,AAACQ,CAAAA,IAAII,GAAG,CAACF,EAAEV,CAAC,KAAK,CAAA,IAAKU,EAAExC,CAAC,CAACO,MAAM;QAC/E,OAAO;YACLoC,MAAMxC,UAAU+B,GAAGjC,OAAO,CAAC,QAAQ,KAAK2C,IAAI;YAC5ChC,GAAGmB,KAAK,CAAC,EAAE,CAACd,KAAK,CAACL,CAAC;YACnBW,GAAGU,EAAEV,CAAC;YACNO,GAAG;mBAAIQ,IAAIO,OAAO;aAAG,CAACzB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;YACvDG,KAAKS,EAAET,GAAG;YACVsB,aAAaf,MAAMf,GAAG,CAAC,CAAC,EAAEE,KAAK,EAAE,GAAKA;QACxC;IACF,GACCR,MAAM,CAAC,CAACuB,IAAMA,EAAEU,IAAI;AACzB"}
|
package/dist/esm/markdown.js
CHANGED
|
@@ -23,6 +23,7 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
|
23
23
|
Math.min(6, i + 1 + offset)
|
|
24
24
|
]));
|
|
25
25
|
const out = [];
|
|
26
|
+
const blockPages = new Map();
|
|
26
27
|
const stats = {
|
|
27
28
|
headings: 0,
|
|
28
29
|
listItems: 0,
|
|
@@ -32,38 +33,64 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
|
32
33
|
callouts: 0,
|
|
33
34
|
closedHyphens: 0
|
|
34
35
|
};
|
|
36
|
+
let currentPage = 0;
|
|
35
37
|
let para = [];
|
|
38
|
+
const addBlock = (block)=>{
|
|
39
|
+
const blocks = blockPages.get(currentPage);
|
|
40
|
+
if (blocks) blocks.push(block);
|
|
41
|
+
else blockPages.set(currentPage, [
|
|
42
|
+
block
|
|
43
|
+
]);
|
|
44
|
+
};
|
|
36
45
|
// a label line ending in ':' whose next line is a bare number belongs on one line
|
|
37
|
-
function attachValue(label, lines,
|
|
38
|
-
const j =
|
|
46
|
+
function attachValue(label, lines, i) {
|
|
47
|
+
const j = i + 1;
|
|
39
48
|
if (/[::]\s*$/.test(label) && j < lines.length && /^[0-9][\d.,]*%?$/.test(lines[j].text)) {
|
|
40
|
-
set(j);
|
|
41
49
|
stats.joinedPairs++;
|
|
42
|
-
return
|
|
50
|
+
return {
|
|
51
|
+
text: `${label} ${lines[j].text}`,
|
|
52
|
+
last: j
|
|
53
|
+
};
|
|
43
54
|
}
|
|
44
|
-
return
|
|
55
|
+
return {
|
|
56
|
+
text: label,
|
|
57
|
+
last: i
|
|
58
|
+
};
|
|
45
59
|
}
|
|
46
60
|
const flush = ()=>{
|
|
47
61
|
if (!para.length) return;
|
|
48
|
-
|
|
62
|
+
const text = para.map((part)=>part.text).join(' ').replace(/\s+([,.;:%])/g, '$1');
|
|
63
|
+
out.push(text);
|
|
49
64
|
out.push('');
|
|
65
|
+
addBlock({
|
|
66
|
+
type: 'paragraph',
|
|
67
|
+
text,
|
|
68
|
+
wordIndexes: para.flatMap((part)=>part.wordIndexes)
|
|
69
|
+
});
|
|
50
70
|
stats.paragraphs++;
|
|
51
71
|
para = [];
|
|
52
72
|
};
|
|
53
73
|
// append a wrapped line to the open paragraph, resolving a line-final hyphen
|
|
54
|
-
const appendToPara = (text)=>{
|
|
74
|
+
const appendToPara = (line, text)=>{
|
|
55
75
|
const prev = para.at(-1);
|
|
56
|
-
if (prev && /\w-$/.test(prev)) {
|
|
76
|
+
if (prev && /\w-$/.test(prev.text)) {
|
|
57
77
|
var _text_split_;
|
|
58
78
|
// a bare " -" is punctuation, not a wrapped word
|
|
59
|
-
const head = prev.replace(/-$/, '').split(/\s/).pop();
|
|
79
|
+
const head = prev.text.replace(/-$/, '').split(/\s/).pop();
|
|
60
80
|
const tail = ((_text_split_ = text.split(/\s/)[0]) !== null && _text_split_ !== void 0 ? _text_split_ : '').replace(/[^A-Za-z-]/g, '').split('-')[0];
|
|
61
81
|
const close = getDict().has((head + tail).toLowerCase());
|
|
62
82
|
if (close) stats.closedHyphens++;
|
|
63
|
-
para[para.length - 1] = prev.replace(/-$/, close ? '' : '-') + text;
|
|
64
|
-
|
|
83
|
+
para[para.length - 1].text = prev.text.replace(/-$/, close ? '' : '-') + text;
|
|
84
|
+
para[para.length - 1].wordIndexes.push(...line.wordIndexes);
|
|
85
|
+
} else para.push({
|
|
86
|
+
text,
|
|
87
|
+
wordIndexes: [
|
|
88
|
+
...line.wordIndexes
|
|
89
|
+
]
|
|
90
|
+
});
|
|
65
91
|
};
|
|
66
92
|
for (const { page, lines } of pageLines){
|
|
93
|
+
currentPage = page;
|
|
67
94
|
if (opts.pageMarkers) {
|
|
68
95
|
flush();
|
|
69
96
|
out.push(pageMarkerLine(page));
|
|
@@ -80,22 +107,35 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
|
80
107
|
// below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.
|
|
81
108
|
if (levelOf.has(h) && !BULLETS.test(text)) {
|
|
82
109
|
const run = [
|
|
83
|
-
|
|
110
|
+
lines[i]
|
|
84
111
|
];
|
|
85
|
-
while(i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text))run.push(lines[++i]
|
|
112
|
+
while(i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text))run.push(lines[++i]);
|
|
86
113
|
// join wrapped title lines, resolving line-final hyphens like body text
|
|
87
|
-
const merged = run.reduce((acc, part)=>{
|
|
114
|
+
const merged = run.map((line)=>line.text).reduce((acc, part)=>{
|
|
88
115
|
var _part_split_;
|
|
89
116
|
return /\w-$/.test(acc) ? acc.replace(/-$/, getDict().has((acc.replace(/-$/, '').split(/\s/).pop() + ((_part_split_ = part.split(/\s/)[0]) !== null && _part_split_ !== void 0 ? _part_split_ : '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`;
|
|
90
117
|
}).replace(/\s+([,.;:%])/g, '$1');
|
|
118
|
+
const wordIndexes = run.flatMap((line)=>line.wordIndexes);
|
|
91
119
|
flush();
|
|
92
120
|
if (merged.length > 150) {
|
|
93
121
|
out.push(`> ${merged}`);
|
|
94
122
|
out.push('');
|
|
123
|
+
addBlock({
|
|
124
|
+
type: 'paragraph',
|
|
125
|
+
text: `> ${merged}`,
|
|
126
|
+
wordIndexes
|
|
127
|
+
});
|
|
95
128
|
stats.callouts++;
|
|
96
129
|
} else {
|
|
97
|
-
|
|
130
|
+
const level = levelOf.get(h);
|
|
131
|
+
out.push(`${'#'.repeat(level)} ${merged}`);
|
|
98
132
|
out.push('');
|
|
133
|
+
addBlock({
|
|
134
|
+
type: 'heading',
|
|
135
|
+
text: merged,
|
|
136
|
+
level,
|
|
137
|
+
wordIndexes
|
|
138
|
+
});
|
|
99
139
|
stats.headings++;
|
|
100
140
|
}
|
|
101
141
|
continue;
|
|
@@ -108,13 +148,27 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
|
108
148
|
if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {
|
|
109
149
|
flush();
|
|
110
150
|
const depth = Math.max(0, Math.round((x - indentBase) / 18));
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
151
|
+
const itemLines = [
|
|
152
|
+
lines[i],
|
|
153
|
+
lines[++i]
|
|
154
|
+
];
|
|
155
|
+
let label = lines[i].text.replace(BULLETS, '').trim();
|
|
156
|
+
while(!label && i + 1 < lines.length){
|
|
157
|
+
// stacked glyphs
|
|
158
|
+
itemLines.push(lines[++i]);
|
|
159
|
+
label = lines[i].text.replace(BULLETS, '').trim();
|
|
160
|
+
}
|
|
161
|
+
const attached = attachValue(label, lines, i);
|
|
162
|
+
if (attached.last !== i) itemLines.push(lines[attached.last]);
|
|
163
|
+
i = attached.last;
|
|
164
|
+
const itemText = `${' '.repeat(depth)}- ${attached.text}`;
|
|
165
|
+
out.push(itemText);
|
|
166
|
+
addBlock({
|
|
167
|
+
type: 'paragraph',
|
|
168
|
+
text: itemText,
|
|
169
|
+
tight: true,
|
|
170
|
+
wordIndexes: itemLines.flatMap((line)=>line.wordIndexes)
|
|
116
171
|
});
|
|
117
|
-
out.push(`${' '.repeat(depth)}- ${label}`);
|
|
118
172
|
stats.listItems++;
|
|
119
173
|
continue;
|
|
120
174
|
}
|
|
@@ -122,21 +176,42 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
|
122
176
|
if (wasBullet) {
|
|
123
177
|
flush();
|
|
124
178
|
const depth = Math.max(0, Math.round((x - indentBase) / 18));
|
|
125
|
-
|
|
126
|
-
i
|
|
127
|
-
|
|
179
|
+
const itemLines = [
|
|
180
|
+
lines[i]
|
|
181
|
+
];
|
|
182
|
+
const attached = attachValue(text, lines, i);
|
|
183
|
+
if (attached.last !== i) itemLines.push(lines[attached.last]);
|
|
184
|
+
i = attached.last;
|
|
185
|
+
const itemText = `${' '.repeat(depth)}- ${attached.text}`;
|
|
186
|
+
out.push(itemText);
|
|
187
|
+
addBlock({
|
|
188
|
+
type: 'paragraph',
|
|
189
|
+
text: itemText,
|
|
190
|
+
tight: true,
|
|
191
|
+
wordIndexes: itemLines.flatMap((line)=>line.wordIndexes)
|
|
192
|
+
});
|
|
128
193
|
stats.listItems++;
|
|
129
194
|
continue;
|
|
130
195
|
}
|
|
131
196
|
// `label:` on its own line followed by a bare value -> keep together
|
|
132
197
|
if (/[::]\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\d.,]*%?$/.test(lines[i + 1].text)) {
|
|
133
198
|
flush();
|
|
134
|
-
|
|
199
|
+
const itemText = `- ${text} ${lines[++i].text}`;
|
|
200
|
+
out.push(itemText);
|
|
201
|
+
addBlock({
|
|
202
|
+
type: 'paragraph',
|
|
203
|
+
text: itemText,
|
|
204
|
+
tight: true,
|
|
205
|
+
wordIndexes: [
|
|
206
|
+
...lines[i - 1].wordIndexes,
|
|
207
|
+
...lines[i].wordIndexes
|
|
208
|
+
]
|
|
209
|
+
});
|
|
135
210
|
stats.joinedPairs++;
|
|
136
211
|
continue;
|
|
137
212
|
}
|
|
138
213
|
// otherwise body text: re-flow wrapped lines into a paragraph
|
|
139
|
-
appendToPara(text);
|
|
214
|
+
appendToPara(lines[i], text);
|
|
140
215
|
if (/[.!?:]$/.test(text)) flush();
|
|
141
216
|
else stats.reflowed++;
|
|
142
217
|
}
|
|
@@ -144,10 +219,18 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
|
|
|
144
219
|
}
|
|
145
220
|
flush();
|
|
146
221
|
const md = `${out.join('\n').replace(/\n{3,}/g, '\n\n').trim()}\n`;
|
|
222
|
+
const pages = pageLines.map(({ page })=>{
|
|
223
|
+
var _blockPages_get;
|
|
224
|
+
return {
|
|
225
|
+
page,
|
|
226
|
+
blocks: (_blockPages_get = blockPages.get(page)) !== null && _blockPages_get !== void 0 ? _blockPages_get : []
|
|
227
|
+
};
|
|
228
|
+
});
|
|
147
229
|
return {
|
|
148
230
|
md,
|
|
149
231
|
stats,
|
|
150
232
|
bodyH,
|
|
151
|
-
headingSizes
|
|
233
|
+
headingSizes,
|
|
234
|
+
pages
|
|
152
235
|
};
|
|
153
236
|
}
|
package/dist/esm/markdown.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/markdown.ts"],"sourcesContent":["// Pass 3: lines -> markdown. Structure is measured, not guessed — heading\n// levels come from the font-height histogram, hyphen-closes only against a dictionary (see the inline comments below for each rule).\nimport { getDict } from './dict.ts';\nimport { pageMarkerLine } from './page-marker.ts';\nimport type { BuildMarkdownOptions, Line, MarkdownResult, MarkdownStats, PageLines } from './types.ts';\n\nconst GLYPH = /^[•◦▪▫●○·‣⁃]$/;\nconst BULLETS = /^(?:[•◦▪▫●○·‣⁃]\\s*)+/;\n\nexport function buildMarkdown(pageLines: PageLines[], heightChars: Map<number, number>, opts: BuildMarkdownOptions = {}): MarkdownResult {\n // body height = char-weighted mode; headings are the rarer, larger sizes.\n // The 1.35 multiplier keeps emphasized body text out of the outline.\n const bodyH = [...heightChars.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? 0;\n const headingSizes = [...heightChars.keys()].filter((h) => h > bodyH * 1.35).sort((a, b) => b - a);\n // with page markers on, content headings start below the page ### level;\n // sizes beyond the ###### budget share level 6 rather than being dropped\n const offset = opts.pageMarkers ? 3 : 0;\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1 + offset)]));\n\n const out: string[] = [];\n const stats: MarkdownStats = {\n headings: 0,\n listItems: 0,\n paragraphs: 0,\n joinedPairs: 0,\n reflowed: 0,\n callouts: 0,\n closedHyphens: 0,\n };\n let para: string[] = [];\n\n // a label line ending in ':' whose next line is a bare number belongs on one line\n function attachValue(label: string, lines: Line[], get: () => number, set: (n: number) => void): string {\n const j = get() + 1;\n if (/[::]\\s*$/.test(label) && j < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[j].text)) {\n set(j);\n stats.joinedPairs++;\n return `${label} ${lines[j].text}`;\n }\n return label;\n }\n const flush = () => {\n if (!para.length) return;\n out.push(para.join(' ').replace(/\\s+([,.;:%])/g, '$1'));\n out.push('');\n stats.paragraphs++;\n para = [];\n };\n // append a wrapped line to the open paragraph, resolving a line-final hyphen\n const appendToPara = (text: string) => {\n const prev = para.at(-1);\n if (prev && /\\w-$/.test(prev)) {\n // a bare \" -\" is punctuation, not a wrapped word\n const head = prev.replace(/-$/, '').split(/\\s/).pop() as string;\n const tail = (text.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0];\n const close = getDict().has((head + tail).toLowerCase());\n if (close) stats.closedHyphens++;\n para[para.length - 1] = prev.replace(/-$/, close ? '' : '-') + text;\n } else para.push(text);\n };\n\n for (const { page, lines } of pageLines) {\n if (opts.pageMarkers) {\n flush();\n out.push(pageMarkerLine(page));\n out.push('');\n }\n if (!lines.length) continue;\n const indentBase = Math.min(...lines.map((l) => l.x));\n\n for (let i = 0; i < lines.length; i++) {\n let { text, h, x } = lines[i];\n const prevLine = i > 0 ? lines[i - 1] : null;\n // a column change or a vertical jump wider than a blank line ends the paragraph\n if (para.length && prevLine && (lines[i].col !== prevLine.col || lines[i].y - prevLine.y > h * 2.2)) flush();\n\n // A large-font line opens a run: gather the consecutive same-size lines\n // below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.\n if (levelOf.has(h) && !BULLETS.test(text)) {\n const run = [text];\n while (i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text)) run.push(lines[++i].text);\n // join wrapped title lines, resolving line-final hyphens like body text\n const merged = run.reduce((acc, part) => (/\\w-$/.test(acc) ? acc.replace(/-$/, getDict().has(((acc.replace(/-$/, '').split(/\\s/).pop() as string) + (part.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`)).replace(/\\s+([,.;:%])/g, '$1');\n flush();\n if (merged.length > 150) {\n out.push(`> ${merged}`);\n out.push('');\n stats.callouts++;\n } else {\n out.push(`${'#'.repeat(levelOf.get(h) as number)} ${merged}`);\n out.push('');\n stats.headings++;\n }\n continue;\n }\n\n // Lines can carry several stacked glyphs (\"• ▪\") from nested source lists.\n const stripped = text.replace(BULLETS, '').trim();\n const wasBullet = stripped !== text;\n if (wasBullet) text = stripped;\n\n // bullet with nothing left on the line: attach the following line's text\n if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n let label = lines[++i].text.replace(BULLETS, '').trim();\n while (!label && i + 1 < lines.length)\n // stacked glyphs\n label = lines[++i].text.replace(BULLETS, '').trim();\n label = attachValue(\n label,\n lines,\n () => i,\n (n) => {\n i = n;\n }\n );\n out.push(`${' '.repeat(depth)}- ${label}`);\n stats.listItems++;\n continue;\n }\n // glyph(s) followed by text on the same line\n if (wasBullet) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n out.push(\n `${' '.repeat(depth)}- ${attachValue(\n text,\n lines,\n () => i,\n (n) => {\n i = n;\n }\n )}`\n );\n stats.listItems++;\n continue;\n }\n // `label:` on its own line followed by a bare value -> keep together\n if (/[::]\\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[i + 1].text)) {\n flush();\n out.push(`- ${text} ${lines[++i].text}`);\n stats.joinedPairs++;\n continue;\n }\n // otherwise body text: re-flow wrapped lines into a paragraph\n appendToPara(text);\n if (/[.!?:]$/.test(text)) flush();\n else stats.reflowed++;\n }\n flush();\n }\n flush();\n\n const md = `${out\n .join('\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()}\\n`;\n return { md, stats, bodyH, headingSizes };\n}\n"],"names":["getDict","pageMarkerLine","GLYPH","BULLETS","buildMarkdown","pageLines","heightChars","opts","bodyH","entries","sort","a","b","headingSizes","keys","filter","h","offset","pageMarkers","levelOf","Map","slice","map","i","Math","min","out","stats","headings","listItems","paragraphs","joinedPairs","reflowed","callouts","closedHyphens","para","attachValue","label","lines","get","set","j","test","length","text","flush","push","join","replace","appendToPara","prev","at","head","split","pop","tail","close","has","toLowerCase","page","indentBase","l","x","prevLine","col","y","run","merged","reduce","acc","part","repeat","stripped","trim","wasBullet","depth","max","round","n","md"],"mappings":"AAAA,0EAA0E;AAC1E,qIAAqI;AACrI,SAASA,OAAO,QAAQ,YAAY;AACpC,SAASC,cAAc,QAAQ,mBAAmB;AAGlD,MAAMC,QAAQ;AACd,MAAMC,UAAU;AAEhB,OAAO,SAASC,cAAcC,SAAsB,EAAEC,WAAgC,EAAEC,OAA6B,CAAC,CAAC;;QAGvG;IAFd,0EAA0E;IAC1E,qEAAqE;IACrE,MAAMC,iBAAQ,SAAA;WAAIF,YAAYG,OAAO;KAAG,CAACC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,cAAzD,6BAAA,MAA2D,CAAC,EAAE,uCAAI;IAChF,MAAME,eAAe;WAAIP,YAAYQ,IAAI;KAAG,CAACC,MAAM,CAAC,CAACC,IAAMA,IAAIR,QAAQ,MAAME,IAAI,CAAC,CAACC,GAAGC,IAAMA,IAAID;IAChG,yEAAyE;IACzE,yEAAyE;IACzE,MAAMM,SAASV,KAAKW,WAAW,GAAG,IAAI;IACtC,MAAMC,UAAU,IAAIC,IAAIP,aAAaQ,KAAK,CAAC,GAAG,GAAGC,GAAG,CAAC,CAACN,GAAGO,IAAM;YAACP;YAAGQ,KAAKC,GAAG,CAAC,GAAGF,IAAI,IAAIN;SAAQ;IAE/F,MAAMS,MAAgB,EAAE;IACxB,MAAMC,QAAuB;QAC3BC,UAAU;QACVC,WAAW;QACXC,YAAY;QACZC,aAAa;QACbC,UAAU;QACVC,UAAU;QACVC,eAAe;IACjB;IACA,IAAIC,OAAiB,EAAE;IAEvB,kFAAkF;IAClF,SAASC,YAAYC,KAAa,EAAEC,KAAa,EAAEC,GAAiB,EAAEC,GAAwB;QAC5F,MAAMC,IAAIF,QAAQ;QAClB,IAAI,WAAWG,IAAI,CAACL,UAAUI,IAAIH,MAAMK,MAAM,IAAI,mBAAmBD,IAAI,CAACJ,KAAK,CAACG,EAAE,CAACG,IAAI,GAAG;YACxFJ,IAAIC;YACJd,MAAMI,WAAW;YACjB,OAAO,GAAGM,MAAM,CAAC,EAAEC,KAAK,CAACG,EAAE,CAACG,IAAI,EAAE;QACpC;QACA,OAAOP;IACT;IACA,MAAMQ,QAAQ;QACZ,IAAI,CAACV,KAAKQ,MAAM,EAAE;QAClBjB,IAAIoB,IAAI,CAACX,KAAKY,IAAI,CAAC,KAAKC,OAAO,CAAC,iBAAiB;QACjDtB,IAAIoB,IAAI,CAAC;QACTnB,MAAMG,UAAU;QAChBK,OAAO,EAAE;IACX;IACA,6EAA6E;IAC7E,MAAMc,eAAe,CAACL;QACpB,MAAMM,OAAOf,KAAKgB,EAAE,CAAC,CAAC;QACtB,IAAID,QAAQ,OAAOR,IAAI,CAACQ,OAAO;gBAGfN;YAFd,iDAAiD;YACjD,MAAMQ,OAAOF,KAAKF,OAAO,CAAC,MAAM,IAAIK,KAAK,CAAC,MAAMC,GAAG;YACnD,MAAMC,OAAO,EAACX,eAAAA,KAAKS,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBT,0BAAAA,eAAuB,IAAII,OAAO,CAAC,eAAe,IAAIK,KAAK,CAAC,IAAI,CAAC,EAAE;YACjF,MAAMG,QAAQxD,UAAUyD,GAAG,CAAC,AAACL,CAAAA,OAAOG,IAAG,EAAGG,WAAW;YACrD,IAAIF,OAAO7B,MAAMO,aAAa;YAC9BC,IAAI,CAACA,KAAKQ,MAAM,GAAG,EAAE,GAAGO,KAAKF,OAAO,CAAC,MAAMQ,QAAQ,KAAK,OAAOZ;QACjE,OAAOT,KAAKW,IAAI,CAACF;IACnB;IAEA,KAAK,MAAM,EAAEe,IAAI,EAAErB,KAAK,EAAE,IAAIjC,UAAW;QACvC,IAAIE,KAAKW,WAAW,EAAE;YACpB2B;YACAnB,IAAIoB,IAAI,CAAC7C,eAAe0D;YACxBjC,IAAIoB,IAAI,CAAC;QACX;QACA,IAAI,CAACR,MAAMK,MAAM,EAAE;QACnB,MAAMiB,aAAapC,KAAKC,GAAG,IAAIa,MAAMhB,GAAG,CAAC,CAACuC,IAAMA,EAAEC,CAAC;QAEnD,IAAK,IAAIvC,IAAI,GAAGA,IAAIe,MAAMK,MAAM,EAAEpB,IAAK;YACrC,IAAI,EAAEqB,IAAI,EAAE5B,CAAC,EAAE8C,CAAC,EAAE,GAAGxB,KAAK,CAACf,EAAE;YAC7B,MAAMwC,WAAWxC,IAAI,IAAIe,KAAK,CAACf,IAAI,EAAE,GAAG;YACxC,gFAAgF;YAChF,IAAIY,KAAKQ,MAAM,IAAIoB,YAAazB,CAAAA,KAAK,CAACf,EAAE,CAACyC,GAAG,KAAKD,SAASC,GAAG,IAAI1B,KAAK,CAACf,EAAE,CAAC0C,CAAC,GAAGF,SAASE,CAAC,GAAGjD,IAAI,GAAE,GAAI6B;YAErG,wEAAwE;YACxE,mLAAmL;YACnL,IAAI1B,QAAQsC,GAAG,CAACzC,MAAM,CAACb,QAAQuC,IAAI,CAACE,OAAO;gBACzC,MAAMsB,MAAM;oBAACtB;iBAAK;gBAClB,MAAOrB,IAAI,IAAIe,MAAMK,MAAM,IAAIL,KAAK,CAACf,IAAI,EAAE,CAACP,CAAC,KAAKA,KAAKsB,KAAK,CAACf,IAAI,EAAE,CAACyC,GAAG,KAAK1B,KAAK,CAACf,EAAE,CAACyC,GAAG,IAAI1B,KAAK,CAACf,IAAI,EAAE,CAAC0C,CAAC,GAAG3B,KAAK,CAACf,EAAE,CAAC0C,CAAC,GAAGjD,IAAI,OAAO,CAACb,QAAQuC,IAAI,CAACJ,KAAK,CAACf,IAAI,EAAE,CAACqB,IAAI,EAAGsB,IAAIpB,IAAI,CAACR,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI;gBAC/L,wEAAwE;gBACxE,MAAMuB,SAASD,IAAIE,MAAM,CAAC,CAACC,KAAKC;wBAAqHA;2BAA3G,OAAO5B,IAAI,CAAC2B,OAAOA,IAAIrB,OAAO,CAAC,MAAMhD,UAAUyD,GAAG,CAAC,AAAC,CAAA,AAACY,IAAIrB,OAAO,CAAC,MAAM,IAAIK,KAAK,CAAC,MAAMC,GAAG,KAAgB,EAACgB,eAAAA,KAAKjB,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBiB,0BAAAA,eAAuB,IAAItB,OAAO,CAAC,eAAe,IAAIK,KAAK,CAAC,IAAI,CAAC,EAAE,AAAD,EAAGK,WAAW,MAAM,KAAK,OAAOY,OAAO,GAAGD,IAAI,CAAC,EAAEC,MAAM;mBAAGtB,OAAO,CAAC,iBAAiB;gBAC1SH;gBACA,IAAIsB,OAAOxB,MAAM,GAAG,KAAK;oBACvBjB,IAAIoB,IAAI,CAAC,CAAC,EAAE,EAAEqB,QAAQ;oBACtBzC,IAAIoB,IAAI,CAAC;oBACTnB,MAAMM,QAAQ;gBAChB,OAAO;oBACLP,IAAIoB,IAAI,CAAC,GAAG,IAAIyB,MAAM,CAACpD,QAAQoB,GAAG,CAACvB,IAAc,CAAC,EAAEmD,QAAQ;oBAC5DzC,IAAIoB,IAAI,CAAC;oBACTnB,MAAMC,QAAQ;gBAChB;gBACA;YACF;YAEA,2EAA2E;YAC3E,MAAM4C,WAAW5B,KAAKI,OAAO,CAAC7C,SAAS,IAAIsE,IAAI;YAC/C,MAAMC,YAAYF,aAAa5B;YAC/B,IAAI8B,WAAW9B,OAAO4B;YAEtB,yEAAyE;YACzE,IAAI,AAAC,CAAA,CAAC5B,QAAQ1C,MAAMwC,IAAI,CAACE,KAAI,KAAMrB,IAAI,IAAIe,MAAMK,MAAM,EAAE;gBACvDE;gBACA,MAAM8B,QAAQnD,KAAKoD,GAAG,CAAC,GAAGpD,KAAKqD,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxD,IAAIvB,QAAQC,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI,CAACI,OAAO,CAAC7C,SAAS,IAAIsE,IAAI;gBACrD,MAAO,CAACpC,SAASd,IAAI,IAAIe,MAAMK,MAAM,CACnC,iBAAiB;gBACjBN,QAAQC,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI,CAACI,OAAO,CAAC7C,SAAS,IAAIsE,IAAI;gBACnDpC,QAAQD,YACNC,OACAC,OACA,IAAMf,GACN,CAACuD;oBACCvD,IAAIuD;gBACN;gBAEFpD,IAAIoB,IAAI,CAAC,GAAG,KAAKyB,MAAM,CAACI,OAAO,EAAE,EAAEtC,OAAO;gBAC1CV,MAAME,SAAS;gBACf;YACF;YACA,6CAA6C;YAC7C,IAAI6C,WAAW;gBACb7B;gBACA,MAAM8B,QAAQnD,KAAKoD,GAAG,CAAC,GAAGpD,KAAKqD,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxDlC,IAAIoB,IAAI,CACN,GAAG,KAAKyB,MAAM,CAACI,OAAO,EAAE,EAAEvC,YACxBQ,MACAN,OACA,IAAMf,GACN,CAACuD;oBACCvD,IAAIuD;gBACN,IACC;gBAELnD,MAAME,SAAS;gBACf;YACF;YACA,qEAAqE;YACrE,IAAI,WAAWa,IAAI,CAACE,SAASrB,IAAI,IAAIe,MAAMK,MAAM,IAAI,mBAAmBD,IAAI,CAACJ,KAAK,CAACf,IAAI,EAAE,CAACqB,IAAI,GAAG;gBAC/FC;gBACAnB,IAAIoB,IAAI,CAAC,CAAC,EAAE,EAAEF,KAAK,CAAC,EAAEN,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI,EAAE;gBACvCjB,MAAMI,WAAW;gBACjB;YACF;YACA,8DAA8D;YAC9DkB,aAAaL;YACb,IAAI,UAAUF,IAAI,CAACE,OAAOC;iBACrBlB,MAAMK,QAAQ;QACrB;QACAa;IACF;IACAA;IAEA,MAAMkC,KAAK,GAAGrD,IACXqB,IAAI,CAAC,MACLC,OAAO,CAAC,WAAW,QACnByB,IAAI,GAAG,EAAE,CAAC;IACb,OAAO;QAAEM;QAAIpD;QAAOnB;QAAOK;IAAa;AAC1C"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/markdown.ts"],"sourcesContent":["// Pass 3: lines -> markdown. Structure is measured, not guessed — heading\n// levels come from the font-height histogram, hyphen-closes only against a dictionary (see the inline comments below for each rule).\nimport { getDict } from './dict.ts';\nimport { pageMarkerLine } from './page-marker.ts';\nimport type { AnalysisBlock, BuildMarkdownOptions, Line, MarkdownPage, MarkdownResult, MarkdownStats, PageLines } from './types.ts';\n\nconst GLYPH = /^[•◦▪▫●○·‣⁃]$/;\nconst BULLETS = /^(?:[•◦▪▫●○·‣⁃]\\s*)+/;\n\nexport function buildMarkdown(pageLines: PageLines[], heightChars: Map<number, number>, opts: BuildMarkdownOptions = {}): MarkdownResult {\n // body height = char-weighted mode; headings are the rarer, larger sizes.\n // The 1.35 multiplier keeps emphasized body text out of the outline.\n const bodyH = [...heightChars.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? 0;\n const headingSizes = [...heightChars.keys()].filter((h) => h > bodyH * 1.35).sort((a, b) => b - a);\n // with page markers on, content headings start below the page ### level;\n // sizes beyond the ###### budget share level 6 rather than being dropped\n const offset = opts.pageMarkers ? 3 : 0;\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1 + offset)]));\n\n const out: string[] = [];\n const blockPages = new Map<number, AnalysisBlock[]>();\n const stats: MarkdownStats = {\n headings: 0,\n listItems: 0,\n paragraphs: 0,\n joinedPairs: 0,\n reflowed: 0,\n callouts: 0,\n closedHyphens: 0,\n };\n let currentPage = 0;\n let para: { text: string; wordIndexes: number[] }[] = [];\n\n const addBlock = (block: AnalysisBlock) => {\n const blocks = blockPages.get(currentPage);\n if (blocks) blocks.push(block);\n else blockPages.set(currentPage, [block]);\n };\n\n // a label line ending in ':' whose next line is a bare number belongs on one line\n function attachValue(label: string, lines: Line[], i: number): { text: string; last: number } {\n const j = i + 1;\n if (/[::]\\s*$/.test(label) && j < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[j].text)) {\n stats.joinedPairs++;\n return { text: `${label} ${lines[j].text}`, last: j };\n }\n return { text: label, last: i };\n }\n const flush = () => {\n if (!para.length) return;\n const text = para\n .map((part) => part.text)\n .join(' ')\n .replace(/\\s+([,.;:%])/g, '$1');\n out.push(text);\n out.push('');\n addBlock({ type: 'paragraph', text, wordIndexes: para.flatMap((part) => part.wordIndexes) });\n stats.paragraphs++;\n para = [];\n };\n // append a wrapped line to the open paragraph, resolving a line-final hyphen\n const appendToPara = (line: Line, text: string) => {\n const prev = para.at(-1);\n if (prev && /\\w-$/.test(prev.text)) {\n // a bare \" -\" is punctuation, not a wrapped word\n const head = prev.text.replace(/-$/, '').split(/\\s/).pop() as string;\n const tail = (text.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0];\n const close = getDict().has((head + tail).toLowerCase());\n if (close) stats.closedHyphens++;\n para[para.length - 1].text = prev.text.replace(/-$/, close ? '' : '-') + text;\n para[para.length - 1].wordIndexes.push(...line.wordIndexes);\n } else para.push({ text, wordIndexes: [...line.wordIndexes] });\n };\n\n for (const { page, lines } of pageLines) {\n currentPage = page;\n if (opts.pageMarkers) {\n flush();\n out.push(pageMarkerLine(page));\n out.push('');\n }\n if (!lines.length) continue;\n const indentBase = Math.min(...lines.map((l) => l.x));\n\n for (let i = 0; i < lines.length; i++) {\n let { text, h, x } = lines[i];\n const prevLine = i > 0 ? lines[i - 1] : null;\n // a column change or a vertical jump wider than a blank line ends the paragraph\n if (para.length && prevLine && (lines[i].col !== prevLine.col || lines[i].y - prevLine.y > h * 2.2)) flush();\n\n // A large-font line opens a run: gather the consecutive same-size lines\n // below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.\n if (levelOf.has(h) && !BULLETS.test(text)) {\n const run = [lines[i]];\n while (i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text)) run.push(lines[++i]);\n // join wrapped title lines, resolving line-final hyphens like body text\n const merged = run\n .map((line) => line.text)\n .reduce((acc, part) => (/\\w-$/.test(acc) ? acc.replace(/-$/, getDict().has(((acc.replace(/-$/, '').split(/\\s/).pop() as string) + (part.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`))\n .replace(/\\s+([,.;:%])/g, '$1');\n const wordIndexes = run.flatMap((line) => line.wordIndexes);\n flush();\n if (merged.length > 150) {\n out.push(`> ${merged}`);\n out.push('');\n addBlock({ type: 'paragraph', text: `> ${merged}`, wordIndexes });\n stats.callouts++;\n } else {\n const level = levelOf.get(h) as number;\n out.push(`${'#'.repeat(level)} ${merged}`);\n out.push('');\n addBlock({ type: 'heading', text: merged, level, wordIndexes });\n stats.headings++;\n }\n continue;\n }\n\n // Lines can carry several stacked glyphs (\"• ▪\") from nested source lists.\n const stripped = text.replace(BULLETS, '').trim();\n const wasBullet = stripped !== text;\n if (wasBullet) text = stripped;\n\n // bullet with nothing left on the line: attach the following line's text\n if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n const itemLines = [lines[i], lines[++i]];\n let label = lines[i].text.replace(BULLETS, '').trim();\n while (!label && i + 1 < lines.length) {\n // stacked glyphs\n itemLines.push(lines[++i]);\n label = lines[i].text.replace(BULLETS, '').trim();\n }\n const attached = attachValue(label, lines, i);\n if (attached.last !== i) itemLines.push(lines[attached.last]);\n i = attached.last;\n const itemText = `${' '.repeat(depth)}- ${attached.text}`;\n out.push(itemText);\n addBlock({ type: 'paragraph', text: itemText, tight: true, wordIndexes: itemLines.flatMap((line) => line.wordIndexes) });\n stats.listItems++;\n continue;\n }\n // glyph(s) followed by text on the same line\n if (wasBullet) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n const itemLines = [lines[i]];\n const attached = attachValue(text, lines, i);\n if (attached.last !== i) itemLines.push(lines[attached.last]);\n i = attached.last;\n const itemText = `${' '.repeat(depth)}- ${attached.text}`;\n out.push(itemText);\n addBlock({ type: 'paragraph', text: itemText, tight: true, wordIndexes: itemLines.flatMap((line) => line.wordIndexes) });\n stats.listItems++;\n continue;\n }\n // `label:` on its own line followed by a bare value -> keep together\n if (/[::]\\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[i + 1].text)) {\n flush();\n const itemText = `- ${text} ${lines[++i].text}`;\n out.push(itemText);\n addBlock({ type: 'paragraph', text: itemText, tight: true, wordIndexes: [...lines[i - 1].wordIndexes, ...lines[i].wordIndexes] });\n stats.joinedPairs++;\n continue;\n }\n // otherwise body text: re-flow wrapped lines into a paragraph\n appendToPara(lines[i], text);\n if (/[.!?:]$/.test(text)) flush();\n else stats.reflowed++;\n }\n flush();\n }\n flush();\n\n const md = `${out\n .join('\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()}\\n`;\n const pages: MarkdownPage[] = pageLines.map(({ page }) => ({ page, blocks: blockPages.get(page) ?? [] }));\n return { md, stats, bodyH, headingSizes, pages };\n}\n"],"names":["getDict","pageMarkerLine","GLYPH","BULLETS","buildMarkdown","pageLines","heightChars","opts","bodyH","entries","sort","a","b","headingSizes","keys","filter","h","offset","pageMarkers","levelOf","Map","slice","map","i","Math","min","out","blockPages","stats","headings","listItems","paragraphs","joinedPairs","reflowed","callouts","closedHyphens","currentPage","para","addBlock","block","blocks","get","push","set","attachValue","label","lines","j","test","length","text","last","flush","part","join","replace","type","wordIndexes","flatMap","appendToPara","line","prev","at","head","split","pop","tail","close","has","toLowerCase","page","indentBase","l","x","prevLine","col","y","run","merged","reduce","acc","level","repeat","stripped","trim","wasBullet","depth","max","round","itemLines","attached","itemText","tight","md","pages"],"mappings":"AAAA,0EAA0E;AAC1E,qIAAqI;AACrI,SAASA,OAAO,QAAQ,YAAY;AACpC,SAASC,cAAc,QAAQ,mBAAmB;AAGlD,MAAMC,QAAQ;AACd,MAAMC,UAAU;AAEhB,OAAO,SAASC,cAAcC,SAAsB,EAAEC,WAAgC,EAAEC,OAA6B,CAAC,CAAC;;QAGvG;IAFd,0EAA0E;IAC1E,qEAAqE;IACrE,MAAMC,iBAAQ,SAAA;WAAIF,YAAYG,OAAO;KAAG,CAACC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,cAAzD,6BAAA,MAA2D,CAAC,EAAE,uCAAI;IAChF,MAAME,eAAe;WAAIP,YAAYQ,IAAI;KAAG,CAACC,MAAM,CAAC,CAACC,IAAMA,IAAIR,QAAQ,MAAME,IAAI,CAAC,CAACC,GAAGC,IAAMA,IAAID;IAChG,yEAAyE;IACzE,yEAAyE;IACzE,MAAMM,SAASV,KAAKW,WAAW,GAAG,IAAI;IACtC,MAAMC,UAAU,IAAIC,IAAIP,aAAaQ,KAAK,CAAC,GAAG,GAAGC,GAAG,CAAC,CAACN,GAAGO,IAAM;YAACP;YAAGQ,KAAKC,GAAG,CAAC,GAAGF,IAAI,IAAIN;SAAQ;IAE/F,MAAMS,MAAgB,EAAE;IACxB,MAAMC,aAAa,IAAIP;IACvB,MAAMQ,QAAuB;QAC3BC,UAAU;QACVC,WAAW;QACXC,YAAY;QACZC,aAAa;QACbC,UAAU;QACVC,UAAU;QACVC,eAAe;IACjB;IACA,IAAIC,cAAc;IAClB,IAAIC,OAAkD,EAAE;IAExD,MAAMC,WAAW,CAACC;QAChB,MAAMC,SAASb,WAAWc,GAAG,CAACL;QAC9B,IAAII,QAAQA,OAAOE,IAAI,CAACH;aACnBZ,WAAWgB,GAAG,CAACP,aAAa;YAACG;SAAM;IAC1C;IAEA,kFAAkF;IAClF,SAASK,YAAYC,KAAa,EAAEC,KAAa,EAAEvB,CAAS;QAC1D,MAAMwB,IAAIxB,IAAI;QACd,IAAI,WAAWyB,IAAI,CAACH,UAAUE,IAAID,MAAMG,MAAM,IAAI,mBAAmBD,IAAI,CAACF,KAAK,CAACC,EAAE,CAACG,IAAI,GAAG;YACxFtB,MAAMI,WAAW;YACjB,OAAO;gBAAEkB,MAAM,GAAGL,MAAM,CAAC,EAAEC,KAAK,CAACC,EAAE,CAACG,IAAI,EAAE;gBAAEC,MAAMJ;YAAE;QACtD;QACA,OAAO;YAAEG,MAAML;YAAOM,MAAM5B;QAAE;IAChC;IACA,MAAM6B,QAAQ;QACZ,IAAI,CAACf,KAAKY,MAAM,EAAE;QAClB,MAAMC,OAAOb,KACVf,GAAG,CAAC,CAAC+B,OAASA,KAAKH,IAAI,EACvBI,IAAI,CAAC,KACLC,OAAO,CAAC,iBAAiB;QAC5B7B,IAAIgB,IAAI,CAACQ;QACTxB,IAAIgB,IAAI,CAAC;QACTJ,SAAS;YAAEkB,MAAM;YAAaN;YAAMO,aAAapB,KAAKqB,OAAO,CAAC,CAACL,OAASA,KAAKI,WAAW;QAAE;QAC1F7B,MAAMG,UAAU;QAChBM,OAAO,EAAE;IACX;IACA,6EAA6E;IAC7E,MAAMsB,eAAe,CAACC,MAAYV;QAChC,MAAMW,OAAOxB,KAAKyB,EAAE,CAAC,CAAC;QACtB,IAAID,QAAQ,OAAOb,IAAI,CAACa,KAAKX,IAAI,GAAG;gBAGpBA;YAFd,iDAAiD;YACjD,MAAMa,OAAOF,KAAKX,IAAI,CAACK,OAAO,CAAC,MAAM,IAAIS,KAAK,CAAC,MAAMC,GAAG;YACxD,MAAMC,OAAO,EAAChB,eAAAA,KAAKc,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBd,0BAAAA,eAAuB,IAAIK,OAAO,CAAC,eAAe,IAAIS,KAAK,CAAC,IAAI,CAAC,EAAE;YACjF,MAAMG,QAAQnE,UAAUoE,GAAG,CAAC,AAACL,CAAAA,OAAOG,IAAG,EAAGG,WAAW;YACrD,IAAIF,OAAOvC,MAAMO,aAAa;YAC9BE,IAAI,CAACA,KAAKY,MAAM,GAAG,EAAE,CAACC,IAAI,GAAGW,KAAKX,IAAI,CAACK,OAAO,CAAC,MAAMY,QAAQ,KAAK,OAAOjB;YACzEb,IAAI,CAACA,KAAKY,MAAM,GAAG,EAAE,CAACQ,WAAW,CAACf,IAAI,IAAIkB,KAAKH,WAAW;QAC5D,OAAOpB,KAAKK,IAAI,CAAC;YAAEQ;YAAMO,aAAa;mBAAIG,KAAKH,WAAW;aAAC;QAAC;IAC9D;IAEA,KAAK,MAAM,EAAEa,IAAI,EAAExB,KAAK,EAAE,IAAIzC,UAAW;QACvC+B,cAAckC;QACd,IAAI/D,KAAKW,WAAW,EAAE;YACpBkC;YACA1B,IAAIgB,IAAI,CAACzC,eAAeqE;YACxB5C,IAAIgB,IAAI,CAAC;QACX;QACA,IAAI,CAACI,MAAMG,MAAM,EAAE;QACnB,MAAMsB,aAAa/C,KAAKC,GAAG,IAAIqB,MAAMxB,GAAG,CAAC,CAACkD,IAAMA,EAAEC,CAAC;QAEnD,IAAK,IAAIlD,IAAI,GAAGA,IAAIuB,MAAMG,MAAM,EAAE1B,IAAK;YACrC,IAAI,EAAE2B,IAAI,EAAElC,CAAC,EAAEyD,CAAC,EAAE,GAAG3B,KAAK,CAACvB,EAAE;YAC7B,MAAMmD,WAAWnD,IAAI,IAAIuB,KAAK,CAACvB,IAAI,EAAE,GAAG;YACxC,gFAAgF;YAChF,IAAIc,KAAKY,MAAM,IAAIyB,YAAa5B,CAAAA,KAAK,CAACvB,EAAE,CAACoD,GAAG,KAAKD,SAASC,GAAG,IAAI7B,KAAK,CAACvB,EAAE,CAACqD,CAAC,GAAGF,SAASE,CAAC,GAAG5D,IAAI,GAAE,GAAIoC;YAErG,wEAAwE;YACxE,mLAAmL;YACnL,IAAIjC,QAAQiD,GAAG,CAACpD,MAAM,CAACb,QAAQ6C,IAAI,CAACE,OAAO;gBACzC,MAAM2B,MAAM;oBAAC/B,KAAK,CAACvB,EAAE;iBAAC;gBACtB,MAAOA,IAAI,IAAIuB,MAAMG,MAAM,IAAIH,KAAK,CAACvB,IAAI,EAAE,CAACP,CAAC,KAAKA,KAAK8B,KAAK,CAACvB,IAAI,EAAE,CAACoD,GAAG,KAAK7B,KAAK,CAACvB,EAAE,CAACoD,GAAG,IAAI7B,KAAK,CAACvB,IAAI,EAAE,CAACqD,CAAC,GAAG9B,KAAK,CAACvB,EAAE,CAACqD,CAAC,GAAG5D,IAAI,OAAO,CAACb,QAAQ6C,IAAI,CAACF,KAAK,CAACvB,IAAI,EAAE,CAAC2B,IAAI,EAAG2B,IAAInC,IAAI,CAACI,KAAK,CAAC,EAAEvB,EAAE;gBAC1L,wEAAwE;gBACxE,MAAMuD,SAASD,IACZvD,GAAG,CAAC,CAACsC,OAASA,KAAKV,IAAI,EACvB6B,MAAM,CAAC,CAACC,KAAK3B;wBAAqHA;2BAA3G,OAAOL,IAAI,CAACgC,OAAOA,IAAIzB,OAAO,CAAC,MAAMvD,UAAUoE,GAAG,CAAC,AAAC,CAAA,AAACY,IAAIzB,OAAO,CAAC,MAAM,IAAIS,KAAK,CAAC,MAAMC,GAAG,KAAgB,EAACZ,eAAAA,KAAKW,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBX,0BAAAA,eAAuB,IAAIE,OAAO,CAAC,eAAe,IAAIS,KAAK,CAAC,IAAI,CAAC,EAAE,AAAD,EAAGK,WAAW,MAAM,KAAK,OAAOhB,OAAO,GAAG2B,IAAI,CAAC,EAAE3B,MAAM;mBAC3PE,OAAO,CAAC,iBAAiB;gBAC5B,MAAME,cAAcoB,IAAInB,OAAO,CAAC,CAACE,OAASA,KAAKH,WAAW;gBAC1DL;gBACA,IAAI0B,OAAO7B,MAAM,GAAG,KAAK;oBACvBvB,IAAIgB,IAAI,CAAC,CAAC,EAAE,EAAEoC,QAAQ;oBACtBpD,IAAIgB,IAAI,CAAC;oBACTJ,SAAS;wBAAEkB,MAAM;wBAAaN,MAAM,CAAC,EAAE,EAAE4B,QAAQ;wBAAErB;oBAAY;oBAC/D7B,MAAMM,QAAQ;gBAChB,OAAO;oBACL,MAAM+C,QAAQ9D,QAAQsB,GAAG,CAACzB;oBAC1BU,IAAIgB,IAAI,CAAC,GAAG,IAAIwC,MAAM,CAACD,OAAO,CAAC,EAAEH,QAAQ;oBACzCpD,IAAIgB,IAAI,CAAC;oBACTJ,SAAS;wBAAEkB,MAAM;wBAAWN,MAAM4B;wBAAQG;wBAAOxB;oBAAY;oBAC7D7B,MAAMC,QAAQ;gBAChB;gBACA;YACF;YAEA,2EAA2E;YAC3E,MAAMsD,WAAWjC,KAAKK,OAAO,CAACpD,SAAS,IAAIiF,IAAI;YAC/C,MAAMC,YAAYF,aAAajC;YAC/B,IAAImC,WAAWnC,OAAOiC;YAEtB,yEAAyE;YACzE,IAAI,AAAC,CAAA,CAACjC,QAAQhD,MAAM8C,IAAI,CAACE,KAAI,KAAM3B,IAAI,IAAIuB,MAAMG,MAAM,EAAE;gBACvDG;gBACA,MAAMkC,QAAQ9D,KAAK+D,GAAG,CAAC,GAAG/D,KAAKgE,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxD,MAAMkB,YAAY;oBAAC3C,KAAK,CAACvB,EAAE;oBAAEuB,KAAK,CAAC,EAAEvB,EAAE;iBAAC;gBACxC,IAAIsB,QAAQC,KAAK,CAACvB,EAAE,CAAC2B,IAAI,CAACK,OAAO,CAACpD,SAAS,IAAIiF,IAAI;gBACnD,MAAO,CAACvC,SAAStB,IAAI,IAAIuB,MAAMG,MAAM,CAAE;oBACrC,iBAAiB;oBACjBwC,UAAU/C,IAAI,CAACI,KAAK,CAAC,EAAEvB,EAAE;oBACzBsB,QAAQC,KAAK,CAACvB,EAAE,CAAC2B,IAAI,CAACK,OAAO,CAACpD,SAAS,IAAIiF,IAAI;gBACjD;gBACA,MAAMM,WAAW9C,YAAYC,OAAOC,OAAOvB;gBAC3C,IAAImE,SAASvC,IAAI,KAAK5B,GAAGkE,UAAU/C,IAAI,CAACI,KAAK,CAAC4C,SAASvC,IAAI,CAAC;gBAC5D5B,IAAImE,SAASvC,IAAI;gBACjB,MAAMwC,WAAW,GAAG,KAAKT,MAAM,CAACI,OAAO,EAAE,EAAEI,SAASxC,IAAI,EAAE;gBAC1DxB,IAAIgB,IAAI,CAACiD;gBACTrD,SAAS;oBAAEkB,MAAM;oBAAaN,MAAMyC;oBAAUC,OAAO;oBAAMnC,aAAagC,UAAU/B,OAAO,CAAC,CAACE,OAASA,KAAKH,WAAW;gBAAE;gBACtH7B,MAAME,SAAS;gBACf;YACF;YACA,6CAA6C;YAC7C,IAAIuD,WAAW;gBACbjC;gBACA,MAAMkC,QAAQ9D,KAAK+D,GAAG,CAAC,GAAG/D,KAAKgE,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxD,MAAMkB,YAAY;oBAAC3C,KAAK,CAACvB,EAAE;iBAAC;gBAC5B,MAAMmE,WAAW9C,YAAYM,MAAMJ,OAAOvB;gBAC1C,IAAImE,SAASvC,IAAI,KAAK5B,GAAGkE,UAAU/C,IAAI,CAACI,KAAK,CAAC4C,SAASvC,IAAI,CAAC;gBAC5D5B,IAAImE,SAASvC,IAAI;gBACjB,MAAMwC,WAAW,GAAG,KAAKT,MAAM,CAACI,OAAO,EAAE,EAAEI,SAASxC,IAAI,EAAE;gBAC1DxB,IAAIgB,IAAI,CAACiD;gBACTrD,SAAS;oBAAEkB,MAAM;oBAAaN,MAAMyC;oBAAUC,OAAO;oBAAMnC,aAAagC,UAAU/B,OAAO,CAAC,CAACE,OAASA,KAAKH,WAAW;gBAAE;gBACtH7B,MAAME,SAAS;gBACf;YACF;YACA,qEAAqE;YACrE,IAAI,WAAWkB,IAAI,CAACE,SAAS3B,IAAI,IAAIuB,MAAMG,MAAM,IAAI,mBAAmBD,IAAI,CAACF,KAAK,CAACvB,IAAI,EAAE,CAAC2B,IAAI,GAAG;gBAC/FE;gBACA,MAAMuC,WAAW,CAAC,EAAE,EAAEzC,KAAK,CAAC,EAAEJ,KAAK,CAAC,EAAEvB,EAAE,CAAC2B,IAAI,EAAE;gBAC/CxB,IAAIgB,IAAI,CAACiD;gBACTrD,SAAS;oBAAEkB,MAAM;oBAAaN,MAAMyC;oBAAUC,OAAO;oBAAMnC,aAAa;2BAAIX,KAAK,CAACvB,IAAI,EAAE,CAACkC,WAAW;2BAAKX,KAAK,CAACvB,EAAE,CAACkC,WAAW;qBAAC;gBAAC;gBAC/H7B,MAAMI,WAAW;gBACjB;YACF;YACA,8DAA8D;YAC9D2B,aAAab,KAAK,CAACvB,EAAE,EAAE2B;YACvB,IAAI,UAAUF,IAAI,CAACE,OAAOE;iBACrBxB,MAAMK,QAAQ;QACrB;QACAmB;IACF;IACAA;IAEA,MAAMyC,KAAK,GAAGnE,IACX4B,IAAI,CAAC,MACLC,OAAO,CAAC,WAAW,QACnB6B,IAAI,GAAG,EAAE,CAAC;IACb,MAAMU,QAAwBzF,UAAUiB,GAAG,CAAC,CAAC,EAAEgD,IAAI,EAAE;YAAsB3C;eAAhB;YAAE2C;YAAM9B,MAAM,GAAEb,kBAAAA,WAAWc,GAAG,CAAC6B,mBAAf3C,6BAAAA,kBAAwB,EAAE;QAAC;;IACtG,OAAO;QAAEkE;QAAIjE;QAAOpB;QAAOK;QAAciF;IAAM;AACjD"}
|
|
@@ -12,5 +12,5 @@ export declare function splitByPageMarker(md: string): {
|
|
|
12
12
|
/** Per-page body text, duplicate-marker slices CONCATENATED — a word anywhere
|
|
13
13
|
* on page N counts as on page N (the auditor's membership semantics). */
|
|
14
14
|
export declare function bodyByPage(md: string): Map<number, string>;
|
|
15
|
-
/** Count of
|
|
15
|
+
/** Count of visible or invisible page-provenance marker lines in `md`. */
|
|
16
16
|
export declare function countPageMarkers(md: string): number;
|
package/dist/esm/page-marker.js
CHANGED
|
@@ -1,20 +1,24 @@
|
|
|
1
|
-
// page-marker.ts —
|
|
2
|
-
// the page-marker heading imports from here so the regex can't drift (report.ts once had a `\s+` variant).
|
|
1
|
+
// page-marker.ts — page provenance shared by Markdown producers and readers.
|
|
3
2
|
/** The marker heading for page N, with no surrounding whitespace/newlines
|
|
4
3
|
* (callers own their own blank-line formatting). */ export function pageMarkerLine(page) {
|
|
5
4
|
return `### p${page}`;
|
|
6
5
|
}
|
|
7
|
-
//
|
|
8
|
-
|
|
6
|
+
// Visible headings are emitted by this package; invisible comments preserve
|
|
7
|
+
// page provenance when a caller does not want headings in its final Markdown.
|
|
8
|
+
const MARKER_RE_SOURCE = '^(?:### p(\\d+)|<!-- p(\\d+) -->)\\r?$';
|
|
9
|
+
function markerPage(match) {
|
|
10
|
+
var _match_;
|
|
11
|
+
return Number((_match_ = match[1]) !== null && _match_ !== void 0 ? _match_ : match[2]);
|
|
12
|
+
}
|
|
9
13
|
/** Is this line (already split, no trailing newline) a page-marker heading? */ export function isPageMarkerLine(line) {
|
|
10
|
-
return
|
|
14
|
+
return new RegExp(MARKER_RE_SOURCE, 'u').test(line);
|
|
11
15
|
}
|
|
12
16
|
/** Split markdown into per-marker body slices, in document order — one entry
|
|
13
17
|
* PER OCCURRENCE (a duplicate "### pN" yields two entries; no slice is ever dropped). */ export function splitByPageMarker(md) {
|
|
14
|
-
const re =
|
|
18
|
+
const re = new RegExp(MARKER_RE_SOURCE, 'gmu');
|
|
15
19
|
const markers = [];
|
|
16
20
|
for(let m = re.exec(md); m !== null; m = re.exec(md))markers.push({
|
|
17
|
-
page:
|
|
21
|
+
page: markerPage(m),
|
|
18
22
|
index: m.index,
|
|
19
23
|
end: m.index + m[0].length
|
|
20
24
|
});
|
|
@@ -35,6 +39,8 @@ const MARKER_RE = /^### p(\d+)$/;
|
|
|
35
39
|
for (const { page, body } of splitByPageMarker(md))out.set(page, ((_out_get = out.get(page)) !== null && _out_get !== void 0 ? _out_get : '') + body);
|
|
36
40
|
return out;
|
|
37
41
|
}
|
|
38
|
-
/** Count of
|
|
39
|
-
return
|
|
42
|
+
/** Count of visible or invisible page-provenance marker lines in `md`. */ export function countPageMarkers(md) {
|
|
43
|
+
return [
|
|
44
|
+
...md.matchAll(new RegExp(MARKER_RE_SOURCE, 'gmu'))
|
|
45
|
+
].length;
|
|
40
46
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/page-marker.ts"],"sourcesContent":["// page-marker.ts —
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/page-marker.ts"],"sourcesContent":["// page-marker.ts — page provenance shared by Markdown producers and readers.\n\n/** The marker heading for page N, with no surrounding whitespace/newlines\n * (callers own their own blank-line formatting). */\nexport function pageMarkerLine(page: number): string {\n return `### p${page}`;\n}\n\n// Visible headings are emitted by this package; invisible comments preserve\n// page provenance when a caller does not want headings in its final Markdown.\nconst MARKER_RE_SOURCE = '^(?:### p(\\\\d+)|<!-- p(\\\\d+) -->)\\\\r?$';\n\nfunction markerPage(match: RegExpExecArray): number {\n return Number(match[1] ?? match[2]);\n}\n\n/** Is this line (already split, no trailing newline) a page-marker heading? */\nexport function isPageMarkerLine(line: string): boolean {\n return new RegExp(MARKER_RE_SOURCE, 'u').test(line);\n}\n\n/** Split markdown into per-marker body slices, in document order — one entry\n * PER OCCURRENCE (a duplicate \"### pN\" yields two entries; no slice is ever dropped). */\nexport function splitByPageMarker(md: string): { page: number; body: string }[] {\n const re = new RegExp(MARKER_RE_SOURCE, 'gmu');\n const markers: { page: number; index: number; end: number }[] = [];\n for (let m = re.exec(md); m !== null; m = re.exec(md)) markers.push({ page: markerPage(m), index: m.index, end: m.index + m[0].length });\n const out: { page: number; body: string }[] = [];\n for (let i = 0; i < markers.length; i++) {\n const end = i + 1 < markers.length ? markers[i + 1].index : md.length;\n out.push({ page: markers[i].page, body: md.slice(markers[i].end, end) });\n }\n return out;\n}\n\n/** Per-page body text, duplicate-marker slices CONCATENATED — a word anywhere\n * on page N counts as on page N (the auditor's membership semantics). */\nexport function bodyByPage(md: string): Map<number, string> {\n const out = new Map<number, string>();\n for (const { page, body } of splitByPageMarker(md)) out.set(page, (out.get(page) ?? '') + body);\n return out;\n}\n\n/** Count of visible or invisible page-provenance marker lines in `md`. */\nexport function countPageMarkers(md: string): number {\n return [...md.matchAll(new RegExp(MARKER_RE_SOURCE, 'gmu'))].length;\n}\n"],"names":["pageMarkerLine","page","MARKER_RE_SOURCE","markerPage","match","Number","isPageMarkerLine","line","RegExp","test","splitByPageMarker","md","re","markers","m","exec","push","index","end","length","out","i","body","slice","bodyByPage","Map","set","get","countPageMarkers","matchAll"],"mappings":"AAAA,6EAA6E;AAE7E;kDACkD,GAClD,OAAO,SAASA,eAAeC,IAAY;IACzC,OAAO,CAAC,KAAK,EAAEA,MAAM;AACvB;AAEA,4EAA4E;AAC5E,8EAA8E;AAC9E,MAAMC,mBAAmB;AAEzB,SAASC,WAAWC,KAAsB;QAC1BA;IAAd,OAAOC,QAAOD,UAAAA,KAAK,CAAC,EAAE,cAARA,qBAAAA,UAAYA,KAAK,CAAC,EAAE;AACpC;AAEA,6EAA6E,GAC7E,OAAO,SAASE,iBAAiBC,IAAY;IAC3C,OAAO,IAAIC,OAAON,kBAAkB,KAAKO,IAAI,CAACF;AAChD;AAEA;uFACuF,GACvF,OAAO,SAASG,kBAAkBC,EAAU;IAC1C,MAAMC,KAAK,IAAIJ,OAAON,kBAAkB;IACxC,MAAMW,UAA0D,EAAE;IAClE,IAAK,IAAIC,IAAIF,GAAGG,IAAI,CAACJ,KAAKG,MAAM,MAAMA,IAAIF,GAAGG,IAAI,CAACJ,IAAKE,QAAQG,IAAI,CAAC;QAAEf,MAAME,WAAWW;QAAIG,OAAOH,EAAEG,KAAK;QAAEC,KAAKJ,EAAEG,KAAK,GAAGH,CAAC,CAAC,EAAE,CAACK,MAAM;IAAC;IACtI,MAAMC,MAAwC,EAAE;IAChD,IAAK,IAAIC,IAAI,GAAGA,IAAIR,QAAQM,MAAM,EAAEE,IAAK;QACvC,MAAMH,MAAMG,IAAI,IAAIR,QAAQM,MAAM,GAAGN,OAAO,CAACQ,IAAI,EAAE,CAACJ,KAAK,GAAGN,GAAGQ,MAAM;QACrEC,IAAIJ,IAAI,CAAC;YAAEf,MAAMY,OAAO,CAACQ,EAAE,CAACpB,IAAI;YAAEqB,MAAMX,GAAGY,KAAK,CAACV,OAAO,CAACQ,EAAE,CAACH,GAAG,EAAEA;QAAK;IACxE;IACA,OAAOE;AACT;AAEA;uEACuE,GACvE,OAAO,SAASI,WAAWb,EAAU;QAEgCS;IADnE,MAAMA,MAAM,IAAIK;IAChB,KAAK,MAAM,EAAExB,IAAI,EAAEqB,IAAI,EAAE,IAAIZ,kBAAkBC,IAAKS,IAAIM,GAAG,CAACzB,MAAM,EAACmB,WAAAA,IAAIO,GAAG,CAAC1B,mBAARmB,sBAAAA,WAAiB,MAAME;IAC1F,OAAOF;AACT;AAEA,wEAAwE,GACxE,OAAO,SAASQ,iBAAiBjB,EAAU;IACzC,OAAO;WAAIA,GAAGkB,QAAQ,CAAC,IAAIrB,OAAON,kBAAkB;KAAQ,CAACiB,MAAM;AACrE"}
|
|
@@ -8,10 +8,12 @@ export declare function renderPagesToMarkdown(pages: AnalysisPage[], opts: {
|
|
|
8
8
|
export declare function renderPagesToText(pages: AnalysisPage[]): string;
|
|
9
9
|
/**
|
|
10
10
|
* Render an Analysis to markdown. PURE and fail-fast over any well-formed Analysis (including hand-built/spliced ones) — throws `PdfToMdError`/`'ANALYSIS_INPUT'` before any output on a malformed one.
|
|
11
|
-
* OCR-origin output is byte-identical to extractOcr's own markdown
|
|
11
|
+
* OCR-origin output is byte-identical to extractOcr's own markdown. Text
|
|
12
|
+
* output comes from the same captured blocks as extractText. Mixed output
|
|
13
|
+
* always includes page markers because adjacent pages have different producers.
|
|
12
14
|
*
|
|
13
15
|
* @param analysis the Analysis to render (from analyze(), or hand-built/spliced)
|
|
14
|
-
* @param opts.pageMarkers text-origin only;
|
|
16
|
+
* @param opts.pageMarkers text-origin only; OCR and mixed Analysis values always carry markers
|
|
15
17
|
*/
|
|
16
18
|
export declare function toMarkdown(analysis: Analysis, opts?: {
|
|
17
19
|
pageMarkers?: boolean;
|