@effortlessmotion/pdf-to-md 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +69 -76
  2. package/dist/cjs/analyze.d.cts +3 -1
  3. package/dist/cjs/analyze.d.ts +3 -1
  4. package/dist/cjs/analyze.js +38 -100
  5. package/dist/cjs/analyze.js.map +1 -1
  6. package/dist/cjs/audit.d.cts +6 -5
  7. package/dist/cjs/audit.d.ts +6 -5
  8. package/dist/cjs/audit.js +368 -60
  9. package/dist/cjs/audit.js.map +1 -1
  10. package/dist/cjs/cli/extract.js +13 -48
  11. package/dist/cjs/cli/extract.js.map +1 -1
  12. package/dist/cjs/collect.js +91 -35
  13. package/dist/cjs/collect.js.map +1 -1
  14. package/dist/cjs/errors.d.cts +2 -0
  15. package/dist/cjs/errors.d.ts +2 -0
  16. package/dist/cjs/errors.js.map +1 -1
  17. package/dist/cjs/extract.d.cts +20 -9
  18. package/dist/cjs/extract.d.ts +20 -9
  19. package/dist/cjs/extract.js +333 -106
  20. package/dist/cjs/extract.js.map +1 -1
  21. package/dist/cjs/lines.js +21 -10
  22. package/dist/cjs/lines.js.map +1 -1
  23. package/dist/cjs/markdown.js +140 -54
  24. package/dist/cjs/markdown.js.map +1 -1
  25. package/dist/cjs/page-marker.d.cts +1 -1
  26. package/dist/cjs/page-marker.d.ts +1 -1
  27. package/dist/cjs/page-marker.js +39 -8
  28. package/dist/cjs/page-marker.js.map +1 -1
  29. package/dist/cjs/render-analysis.d.cts +4 -2
  30. package/dist/cjs/render-analysis.d.ts +4 -2
  31. package/dist/cjs/render-analysis.js +5 -3
  32. package/dist/cjs/render-analysis.js.map +1 -1
  33. package/dist/cjs/report.d.cts +2 -2
  34. package/dist/cjs/report.d.ts +2 -2
  35. package/dist/cjs/report.js +27 -8
  36. package/dist/cjs/report.js.map +1 -1
  37. package/dist/cjs/text-analysis.d.cts +12 -0
  38. package/dist/cjs/text-analysis.d.ts +12 -0
  39. package/dist/cjs/text-analysis.js +175 -0
  40. package/dist/cjs/text-analysis.js.map +1 -0
  41. package/dist/cjs/types.d.cts +40 -3
  42. package/dist/cjs/types.d.ts +40 -3
  43. package/dist/cjs/types.js +2 -3
  44. package/dist/cjs/types.js.map +1 -1
  45. package/dist/esm/analyze.d.ts +3 -1
  46. package/dist/esm/analyze.js +18 -89
  47. package/dist/esm/analyze.js.map +1 -1
  48. package/dist/esm/audit.d.ts +6 -5
  49. package/dist/esm/audit.js +236 -29
  50. package/dist/esm/audit.js.map +1 -1
  51. package/dist/esm/cli/extract.js +10 -23
  52. package/dist/esm/cli/extract.js.map +1 -1
  53. package/dist/esm/collect.js +64 -15
  54. package/dist/esm/collect.js.map +1 -1
  55. package/dist/esm/errors.d.ts +2 -0
  56. package/dist/esm/errors.js.map +1 -1
  57. package/dist/esm/extract.d.ts +20 -9
  58. package/dist/esm/extract.js +167 -76
  59. package/dist/esm/extract.js.map +1 -1
  60. package/dist/esm/lines.js +14 -10
  61. package/dist/esm/lines.js.map +1 -1
  62. package/dist/esm/markdown.js +110 -27
  63. package/dist/esm/markdown.js.map +1 -1
  64. package/dist/esm/page-marker.d.ts +1 -1
  65. package/dist/esm/page-marker.js +15 -9
  66. package/dist/esm/page-marker.js.map +1 -1
  67. package/dist/esm/render-analysis.d.ts +4 -2
  68. package/dist/esm/render-analysis.js +9 -5
  69. package/dist/esm/render-analysis.js.map +1 -1
  70. package/dist/esm/report.d.ts +2 -2
  71. package/dist/esm/report.js +12 -7
  72. package/dist/esm/report.js.map +1 -1
  73. package/dist/esm/text-analysis.d.ts +12 -0
  74. package/dist/esm/text-analysis.js +81 -0
  75. package/dist/esm/text-analysis.js.map +1 -0
  76. package/dist/esm/types.d.ts +40 -3
  77. package/dist/esm/types.js +4 -4
  78. package/dist/esm/types.js.map +1 -1
  79. package/package.json +2 -8
  80. package/skills/pdf-to-md/SKILL.md +39 -66
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/extract.ts"],"sourcesContent":["// The library layer for PDF -> markdown conversion: extractText/extractOcr/\n// pdfToMarkdown each take plain args, return plain data, and throw PdfToMdError — no process.exit, no console.*, no file writing.\n//\n// Warnings are DATA: every notice is returned in `warnings` verbatim and also\n// delivered incrementally via `onWarning` — the exact strings are load-bearing (the test suite asserts them).\nimport { buildOcrAnalysisPages } from './analyze.ts';\nimport { collect } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { OcrProgressEvent } from './engines/tesseract.ts';\nimport type { PageWord, Tuning } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { AUTO_OCR_CHARS_PER_PAGE, buildOcrReport, buildReport } from './report.ts';\nimport type { PageLines, Report } from './types.ts';\nimport { validatePages } from './validate.ts';\n\nexport type { OcrProgressEvent } from './engines/tesseract.ts';\n\n// A plain-JS caller (or `.ts` with `as any`) gets no compile-time check on\n// `pages`, so it's validated once at runtime — see src/validate.ts's `validatePages`.\n\n/** One recognized word with its page — the OCR replay input contract (same\n * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */\nexport type OcrWordInput = PageWord;\n\n/** A 1-based, inclusive page range — the shape every entry point's `pages`\n * option takes (the CLI's own `--pages N[-M]` parses into this). */\nexport interface PageSelection {\n first: number;\n /** inclusive; defaults to `first` */\n last?: number;\n}\n\n/** extractText/extractOcr/pdfToMarkdown's common return shape. */\nexport interface ExtractResult {\n markdown: string;\n /** per-document and per-page QA metrics — see `Report` (`src/types.ts`) */\n report: Report;\n /** every notice also delivered incrementally via `onWarning`, collected\n * here in full (hybrid pages, auto-OCR-fallback, empty OCR pages, sparse-retry) */\n warnings: string[];\n}\n\ninterface CommonOptions {\n /** convert only this page range (1-based, inclusive); defaults to the\n * whole document */\n pages?: PageSelection;\n /** delivered as each warning happens (also returned, in full, as\n * `warnings` on the resolved result) */\n onWarning?: (message: string) => void;\n}\n\n/** Options for {@link extractText}. */\nexport interface ExtractTextOptions extends CommonOptions {\n /** emit a `### pN` marker before each page's content (output formatting\n * only — `report.pageStats` is always populated regardless) */\n pageMarkers?: boolean;\n}\n\n/** Options for {@link extractOcr}. */\nexport interface ExtractOcrOptions extends CommonOptions {\n /** override any subset of the OCR-path geometry defaults — see\n * DEFAULT_TUNING (`src/geometry.ts`) for each knob's measured default */\n tuning?: Partial<Tuning>;\n /** page-render resolution for engine recognition (default 288, measured;\n * >~400 measured WORSE on ordinary pages — not a \"bigger is safer\" knob) */\n dpi?: number;\n /** marks report.ocrFallback — set by pdfToMarkdown's auto-fallback */\n fallback?: boolean;\n /** one event per page as recognition finishes (timing/count only — engine\n * chatter and the sparse-retry notice arrive via `onWarning` instead) */\n onProgress?: (e: OcrProgressEvent) => void;\n /** every recognized word entering the geometry, delivered once after\n * recognition — the severity-1 audit instrument (--debug-words) */\n onWords?: (words: OcrWordInput[]) => void;\n}\n\n/** Options for {@link pdfToMarkdown}. */\nexport interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOcrOptions, 'fallback'> {\n /** force the OCR path even when a text layer exists */\n ocr?: boolean;\n /** never fall back to OCR, even for image-only pages */\n noOcr?: boolean;\n}\n\n/**\n * Text-layer extraction. Returns markdown + QA report + warnings (hybrid\n * pages carrying large images whose baked-in text the text layer can never see). Makes no OCR decision — see pdfToMarkdown for the fallback.\n *\n * @param pdfPath path to the PDF file\n * @param opts see {@link ExtractTextOptions}\n */\n// One collector per entry point: collects into `warnings`, streams via\n// onWarning, and emits the dictionary-unavailable notice exactly once up front.\nfunction warnCollector(onWarning?: (message: string) => void): { warnings: string[]; warn: (m: string) => void } {\n const warnings: string[] = [];\n const warn = (m: string) => {\n warnings.push(m);\n onWarning?.(m);\n };\n if (!dictAvailable()) warn(dictUnavailableWarning());\n return { warnings, warn };\n}\n\nexport async function extractText(pdfPath: string, opts: ExtractTextOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const { numPages, pages, heightChars } = await collect(pdfPath, range);\n const pageLines: PageLines[] = pages.map((p) => ({\n page: p.page,\n lines: toLines(p),\n }));\n const pageMarkers = opts.pageMarkers ?? false;\n const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, { pageMarkers });\n // pageStats needs \"### pN\" markers to segment content; when the caller\n // didn't request them, build a throwaway marked variant (same heightChars) purely to measure stats from, so pageStats is always populated.\n const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, { pageMarkers: true }).md;\n\n // Advisory, not prescriptive: `hasLargeImage` fires on any large image,\n // even one with no baked-in text, where `--ocr` recovers nothing and can\n // actively degrade a correct result (measured: \"I\" -> \"|\", stray em-dashes) — so this tells the caller to COMPARE the OCR rerun, never blindly replace.\n const hybridPages = pages.filter((p) => p.largeImage).map((p) => p.page);\n for (const n of hybridPages) {\n warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);\n }\n\n const report = buildReport({\n numPages,\n md,\n statsMd,\n stats,\n bodyH,\n headingSizes,\n hybridPages,\n });\n return { markdown: md, report, warnings };\n}\n\n/**\n * OCR extraction. Input is either a PDF (pages render at `dpi` and the\n * engine recognizes them, imported lazily) or an already-recognized word dump (the offline replay input — see `--words-json`).\n *\n * @param input `{ pdfPath }` to recognize, or `{ words }` to replay a dump\n * @param opts see {@link ExtractOcrOptions}\n */\nexport async function extractOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n\n // Shared with analyze(pdf, {path:'ocr'}) — see that module's header for\n // why this makes the two byte-identical by construction, not by hand.\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n\n const report = buildOcrReport({\n md,\n fallback: opts.fallback,\n // Confidence is on `words` — see report.ts's low-confidence detector for\n // how it handles a confidence-less replay honestly.\n words,\n });\n return { markdown: md, report, warnings };\n}\n\n/** pdfToMarkdown's auto-OCR fallback predicate, shared with the CLI's\n * --format=raw|txt path so the two call sites can never drift apart. Module-internal: not re-exported by `src/index.ts`. */\nexport function needsOcrFallback(charsPerPage: number): boolean {\n return charsPerPage < AUTO_OCR_CHARS_PER_PAGE;\n}\n\n/** The exact stderr/warnings notice printed when needsOcrFallback fires —\n * one copy of the string for both call sites (asserted verbatim by the suite). */\nexport function ocrFallbackNotice(charsPerPage: number): string {\n return `text layer yields ~${charsPerPage} chars/page (< ${AUTO_OCR_CHARS_PER_PAGE}) — pages are images of text; falling back to OCR (suppress with --no-ocr)`;\n}\n\n/**\n * The headline API: extract a PDF to markdown, text layer first, falling\n * back to OCR when the text layer is empty enough the pages are certainly images of text (< `AUTO_OCR_CHARS_PER_PAGE` chars/page) — the same decision the CLI makes.\n *\n * @param pdfPath path to the PDF file\n * @param opts see {@link PdfToMarkdownOptions}\n */\nexport async function pdfToMarkdown(pdfPath: string, opts: PdfToMarkdownOptions = {}): Promise<ExtractResult> {\n if (opts.ocr) return extractOcr({ pdfPath }, opts);\n\n // Warning DELIVERY is deferred on the text pass: if the fallback fires,\n // those hybrid warnings describe output being discarded, so only a kept text result replays them.\n const text = await extractText(pdfPath, { ...opts, onWarning: undefined });\n if (needsOcrFallback(text.report.charsPerPage) && !opts.noOcr) {\n const notice = ocrFallbackNotice(text.report.charsPerPage);\n opts.onWarning?.(notice);\n const ocr = await extractOcr({ pdfPath }, { ...opts, fallback: true });\n ocr.warnings.unshift(notice);\n return ocr;\n }\n for (const w of text.warnings) opts.onWarning?.(w);\n return text;\n}\n"],"names":["buildOcrAnalysisPages","collect","dictAvailable","dictUnavailableWarning","toLines","buildMarkdown","renderPagesToMarkdown","AUTO_OCR_CHARS_PER_PAGE","buildOcrReport","buildReport","validatePages","warnCollector","onWarning","warnings","warn","m","push","extractText","pdfPath","opts","pages","range","first","last","numPages","heightChars","pageLines","map","p","page","lines","pageMarkers","md","stats","bodyH","headingSizes","statsMd","hybridPages","filter","largeImage","n","report","markdown","extractOcr","input","words","forcePageMarkers","fallback","needsOcrFallback","charsPerPage","ocrFallbackNotice","pdfToMarkdown","ocr","text","undefined","noOcr","notice","unshift","w"],"mappings":"AAAA,4EAA4E;AAC5E,kIAAkI;AAClI,EAAE;AACF,8EAA8E;AAC9E,8GAA8G;AAC9G,SAASA,qBAAqB,QAAQ,eAAe;AACrD,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,aAAa,EAAEC,sBAAsB,QAAQ,YAAY;AAGlE,SAASC,OAAO,QAAQ,aAAa;AACrC,SAASC,aAAa,QAAQ,gBAAgB;AAC9C,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,uBAAuB,EAAEC,cAAc,EAAEC,WAAW,QAAQ,cAAc;AAEnF,SAASC,aAAa,QAAQ,gBAAgB;AAuE9C;;;;;;CAMC,GACD,uEAAuE;AACvE,gFAAgF;AAChF,SAASC,cAAcC,SAAqC;IAC1D,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,OAAO,CAACC;QACZF,SAASG,IAAI,CAACD;QACdH,sBAAAA,gCAAAA,UAAYG;IACd;IACA,IAAI,CAACb,iBAAiBY,KAAKX;IAC3B,OAAO;QAAEU;QAAUC;IAAK;AAC1B;AAEA,OAAO,eAAeG,YAAYC,OAAe,EAAEC,OAA2B,CAAC,CAAC;QAIlBA,kBAMxCA;IATpBT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IAEvD,MAAMS,QAAQF,KAAKC,KAAK,GAAG;QAAEE,OAAOH,KAAKC,KAAK,CAACE,KAAK;QAAEC,IAAI,GAAEJ,mBAAAA,KAAKC,KAAK,CAACG,IAAI,cAAfJ,8BAAAA,mBAAmBA,KAAKC,KAAK,CAACE,KAAK;IAAC,IAAI,CAAC;IACrG,MAAM,EAAEE,QAAQ,EAAEJ,KAAK,EAAEK,WAAW,EAAE,GAAG,MAAMxB,QAAQiB,SAASG;IAChE,MAAMK,YAAyBN,MAAMO,GAAG,CAAC,CAACC,IAAO,CAAA;YAC/CC,MAAMD,EAAEC,IAAI;YACZC,OAAO1B,QAAQwB;QACjB,CAAA;IACA,MAAMG,eAAcZ,oBAAAA,KAAKY,WAAW,cAAhBZ,+BAAAA,oBAAoB;IACxC,MAAM,EAAEa,EAAE,EAAEC,KAAK,EAAEC,KAAK,EAAEC,YAAY,EAAE,GAAG9B,cAAcqB,WAAWD,aAAa;QAAEM;IAAY;IAC/F,uEAAuE;IACvE,2IAA2I;IAC3I,MAAMK,UAAUL,cAAcC,KAAK3B,cAAcqB,WAAWD,aAAa;QAAEM,aAAa;IAAK,GAAGC,EAAE;IAElG,wEAAwE;IACxE,yEAAyE;IACzE,wJAAwJ;IACxJ,MAAMK,cAAcjB,MAAMkB,MAAM,CAAC,CAACV,IAAMA,EAAEW,UAAU,EAAEZ,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACvE,KAAK,MAAMW,KAAKH,YAAa;QAC3BvB,KAAK,CAAC,CAAC,EAAE0B,EAAE,uJAAuJ,EAAEA,EAAE,yCAAyC,CAAC;IAClN;IAEA,MAAMC,SAAShC,YAAY;QACzBe;QACAQ;QACAI;QACAH;QACAC;QACAC;QACAE;IACF;IACA,OAAO;QAAEK,UAAUV;QAAIS;QAAQ5B;IAAS;AAC1C;AAEA;;;;;;CAMC,GACD,OAAO,eAAe8B,WAAWC,KAAsD,EAAEzB,OAA0B,CAAC,CAAC;IACnHT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IAEvD,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,EAAEQ,KAAK,EAAEyB,KAAK,EAAE,GAAG,MAAM7C,sBAAsB4C,OAAOzB,MAAML;IAClE,MAAMkB,KAAK1B,sBAAsBc,OAAO;QAAE0B,kBAAkB;QAAMf,aAAa;IAAK;IAEpF,MAAMU,SAASjC,eAAe;QAC5BwB;QACAe,UAAU5B,KAAK4B,QAAQ;QACvB,yEAAyE;QACzE,oDAAoD;QACpDF;IACF;IACA,OAAO;QAAEH,UAAUV;QAAIS;QAAQ5B;IAAS;AAC1C;AAEA;0HAC0H,GAC1H,OAAO,SAASmC,iBAAiBC,YAAoB;IACnD,OAAOA,eAAe1C;AACxB;AAEA;gFACgF,GAChF,OAAO,SAAS2C,kBAAkBD,YAAoB;IACpD,OAAO,CAAC,mBAAmB,EAAEA,aAAa,eAAe,EAAE1C,wBAAwB,0EAA0E,CAAC;AAChK;AAEA;;;;;;CAMC,GACD,OAAO,eAAe4C,cAAcjC,OAAe,EAAEC,OAA6B,CAAC,CAAC;QAanDA;IAZ/B,IAAIA,KAAKiC,GAAG,EAAE,OAAOT,WAAW;QAAEzB;IAAQ,GAAGC;IAE7C,wEAAwE;IACxE,kGAAkG;IAClG,MAAMkC,OAAO,MAAMpC,YAAYC,SAAS;QAAE,GAAGC,IAAI;QAAEP,WAAW0C;IAAU;IACxE,IAAIN,iBAAiBK,KAAKZ,MAAM,CAACQ,YAAY,KAAK,CAAC9B,KAAKoC,KAAK,EAAE;YAE7DpC;QADA,MAAMqC,SAASN,kBAAkBG,KAAKZ,MAAM,CAACQ,YAAY;SACzD9B,mBAAAA,KAAKP,SAAS,cAAdO,uCAAAA,sBAAAA,MAAiBqC;QACjB,MAAMJ,MAAM,MAAMT,WAAW;YAAEzB;QAAQ,GAAG;YAAE,GAAGC,IAAI;YAAE4B,UAAU;QAAK;QACpEK,IAAIvC,QAAQ,CAAC4C,OAAO,CAACD;QACrB,OAAOJ;IACT;IACA,KAAK,MAAMM,KAAKL,KAAKxC,QAAQ,EAAEM,kBAAAA,KAAKP,SAAS,cAAdO,sCAAAA,qBAAAA,MAAiBuC;IAChD,OAAOL;AACT"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/extract.ts"],"sourcesContent":["// The library layer for PDF -> markdown conversion: extractText/extractOcr/\n// pdfToMarkdown each take plain args, return plain data, and throw PdfToMdError — no process.exit, no console.*, no file writing.\n//\n// Warnings are DATA: every notice is returned in `warnings` verbatim and also\n// delivered incrementally via `onWarning` — the exact strings are load-bearing (the test suite asserts them).\nimport { buildOcrAnalysisPages } from './analyze.ts';\nimport { collect } from './collect.ts';\nimport { dictAvailable, dictUnavailableWarning } from './dict.ts';\nimport type { OcrProgressEvent } from './engines/tesseract.ts';\nimport type { PageWord, Tuning } from './geometry.ts';\nimport { renderPagesToMarkdown } from './render-analysis.ts';\nimport { AUTO_OCR_CHARS_PER_PAGE, buildOcrReport } from './report.ts';\nimport { buildTextAnalysis, selectCollectedPages } from './text-analysis.ts';\nimport type { Analysis, AnalysisPage, CollectResult, Report } from './types.ts';\nimport { validatePages } from './validate.ts';\n\nexport type { OcrProgressEvent } from './engines/tesseract.ts';\n\n// A plain-JS caller (or `.ts` with `as any`) gets no compile-time check on\n// `pages`, so it's validated once at runtime — see src/validate.ts's `validatePages`.\n\n/** One recognized word with its page — the OCR replay input contract (same\n * shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */\nexport type OcrWordInput = PageWord;\n\n/** A 1-based, inclusive page range — the shape every entry point's `pages`\n * option takes (the CLI's own `--pages N[-M]` parses into this). */\nexport interface PageSelection {\n first: number;\n /** inclusive; defaults to `first` */\n last?: number;\n}\n\n/** extractText/extractOcr/pdfToMarkdown's common return shape. */\nexport interface ExtractResult {\n markdown: string;\n /** per-document and per-page QA metrics — see `Report` (`src/types.ts`) */\n report: Report;\n /** every notice also delivered incrementally via `onWarning`, collected\n * here in full (hybrid pages, auto-OCR-fallback, empty OCR pages, sparse-retry) */\n warnings: string[];\n}\n\ninterface CommonOptions {\n /** convert only this page range (1-based, inclusive); defaults to the\n * whole document */\n pages?: PageSelection;\n /** delivered as each warning happens (also returned, in full, as\n * `warnings` on the resolved result) */\n onWarning?: (message: string) => void;\n}\n\n/** Options for {@link extractText}. */\nexport interface ExtractTextOptions extends CommonOptions {\n /** emit a `### pN` marker before each page's content (output formatting\n * only — `report.pageStats` is always populated regardless) */\n pageMarkers?: boolean;\n /** source PDF text runs entering layout, delivered once for the selected\n * final result. Runs retain measured bounds and are not fabricated words. */\n onWords?: (words: OcrWordInput[]) => void;\n}\n\n/** Options for {@link extractOcr}. */\nexport interface ExtractOcrOptions extends CommonOptions {\n /** override any subset of the OCR-path geometry defaults — see\n * DEFAULT_TUNING (`src/geometry.ts`) for each knob's measured default */\n tuning?: Partial<Tuning>;\n /** page-render resolution for engine recognition (default 288, measured;\n * >~400 measured WORSE on ordinary pages — not a \"bigger is safer\" knob) */\n dpi?: number;\n /** marks report.ocrFallback — set by pdfToMarkdown's auto-fallback */\n fallback?: boolean;\n /** one event per page as recognition finishes (timing/count only — engine\n * chatter and the sparse-retry notice arrive via `onWarning` instead) */\n onProgress?: (e: OcrProgressEvent) => void;\n /** every recognized word entering the geometry, delivered once after\n * recognition — the severity-1 audit instrument (--debug-words) */\n onWords?: (words: OcrWordInput[]) => void;\n}\n\n/** Options for {@link pdfToMarkdown}. */\nexport interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOcrOptions, 'fallback'> {\n /** force the OCR path even when a text layer exists */\n ocr?: boolean;\n /** never fall back to OCR, even for image-only pages */\n noOcr?: boolean;\n}\n\n// One collector per entry point: collects into `warnings`, streams via\n// onWarning, and emits the dictionary-unavailable notice exactly once up front.\nfunction warnCollector(onWarning?: (message: string) => void): { warnings: string[]; warn: (m: string) => void } {\n const warnings: string[] = [];\n const warn = (m: string) => {\n warnings.push(m);\n onWarning?.(m);\n };\n if (!dictAvailable()) warn(dictUnavailableWarning());\n return { warnings, warn };\n}\n\n/** Extract embedded text with a QA report and warnings; never runs OCR.\n * @param pdfPath path to the PDF file\n * @param opts see {@link ExtractTextOptions}\n */\nexport async function extractText(pdfPath: string, opts: ExtractTextOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n const collected = await collect(pdfPath, range);\n const text = buildTextAnalysis(collected, opts.pageMarkers ?? false);\n textNotices(collected, new Set(), warn);\n opts.onWords?.(text.words);\n return { markdown: text.markdown, report: text.analysis.report, warnings };\n}\n\n/**\n * OCR extraction. Input is either a PDF (pages render at `dpi` and the\n * engine recognizes them, imported lazily) or an already-recognized word dump (the offline replay input — see `--words-json`).\n *\n * @param input `{ pdfPath }` to recognize, or `{ words }` to replay a dump\n * @param opts see {@link ExtractOcrOptions}\n */\nexport async function extractOcr(input: { pdfPath: string } | { words: OcrWordInput[] }, opts: ExtractOcrOptions = {}): Promise<ExtractResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n\n // Shared with analyze(pdf, {path:'ocr'}) — see that module's header for\n // why this makes the two byte-identical by construction, not by hand.\n const { pages, words } = await buildOcrAnalysisPages(input, opts, warn);\n const md = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n\n const report = buildOcrReport({\n md,\n fallback: opts.fallback,\n // Confidence is on `words` — see report.ts's low-confidence detector for\n // how it handles a confidence-less replay honestly.\n words,\n });\n return { markdown: md, report, warnings };\n}\n\n/** A page needs OCR only when its image signal accompanies an unusable text\n * layer. The caller supplies the image part of that decision. */\nfunction needsPageOcr(chars: number): boolean {\n return chars < AUTO_OCR_CHARS_PER_PAGE;\n}\n\nfunction textNotices(collected: CollectResult, ocrPages: Set<number>, warn: (message: string) => void): void {\n for (const page of collected.pages) {\n if (page.textChars !== 0 || ocrPages.has(page.page)) continue;\n if (page.hasImage) warn(`p${page.page}: no extractable text; page has image content and OCR was not selected`);\n else warn(`p${page.page}: no extractable text; the page may be blank or contain only vector content`);\n }\n for (const page of collected.pages) {\n if (!page.largeImage || ocrPages.has(page.page)) continue;\n warn(`p${page.page}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${page.page} --ocr, which can itself introduce errors`);\n }\n}\n\nfunction analysisWords(pages: AnalysisPage[]): OcrWordInput[] {\n return pages.flatMap((page) => page.words.map((word) => ({ page: page.page, text: word.text, ...word.box, confidence: word.confidence })));\n}\n\ninterface AutoAnalysisResult {\n analysis: Analysis;\n markdown: string;\n warnings: string[];\n}\n\n/** Shared automatic route for the library's Markdown result and the CLI's\n * raw/plain-text formats. A page changes producer only when it has image\n * evidence and an unusable text layer, so a blank or vector-only page never\n * becomes speculative OCR input. */\nexport async function analyzePdfAuto(pdfPath: string, opts: PdfToMarkdownOptions = {}): Promise<AutoAnalysisResult> {\n validatePages(opts.pages);\n const { warnings, warn } = warnCollector(opts.onWarning);\n const range = opts.pages ? { first: opts.pages.first, last: opts.pages.last ?? opts.pages.first } : {};\n\n if (opts.ocr) {\n const { pages, words } = await buildOcrAnalysisPages({ pdfPath }, { ...opts, onWords: undefined }, warn);\n const markdown = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const report = buildOcrReport({ md: markdown, words });\n const analysis: Analysis = { outputVersion: report.outputVersion, report, pages };\n opts.onWords?.(words);\n return { analysis, markdown, warnings };\n }\n\n const collected = await collect(pdfPath, range);\n const ocrPageNumbers = new Set(!opts.noOcr ? collected.pages.filter((page) => page.hasImage && needsPageOcr(page.textChars)).map((page) => page.page) : []);\n if (!ocrPageNumbers.size) {\n const text = buildTextAnalysis(collected, opts.pageMarkers ?? false);\n textNotices(collected, ocrPageNumbers, warn);\n opts.onWords?.(text.words);\n return { analysis: text.analysis, markdown: text.markdown, warnings };\n }\n\n const textPageNumbers = new Set(collected.pages.filter((page) => !ocrPageNumbers.has(page.page)).map((page) => page.page));\n const text = textPageNumbers.size ? buildTextAnalysis(selectCollectedPages(collected, textPageNumbers), true) : undefined;\n const textByPage = new Map(text?.analysis.pages.map((page) => [page.page, page]));\n const ocrByPage = new Map<number, AnalysisPage>();\n const ocrWords: OcrWordInput[] = [];\n const ocrRanges: Array<{ first: number; last: number }> = [];\n for (const page of collected.pages) {\n if (!ocrPageNumbers.has(page.page)) continue;\n warn(`p${page.page}: sparse text layer with image content; using OCR for this page`);\n const lastRange = ocrRanges.at(-1);\n if (lastRange && lastRange.last + 1 === page.page) lastRange.last = page.page;\n else ocrRanges.push({ first: page.page, last: page.page });\n }\n for (const range of ocrRanges) {\n const result = await buildOcrAnalysisPages({ pdfPath }, { ...opts, pages: range, onWords: undefined }, warn);\n for (const page of result.pages) ocrByPage.set(page.page, page);\n ocrWords.push(...result.words);\n }\n textNotices(collected, ocrPageNumbers, warn);\n const pages = collected.pages.map((page) => ocrByPage.get(page.page) ?? (textByPage.get(page.page) as AnalysisPage));\n const markdown = renderPagesToMarkdown(pages, { forcePageMarkers: true, pageMarkers: true });\n const pagePaths = new Map(pages.map((page) => [page.page, page.path]));\n const report = buildOcrReport({\n md: markdown,\n fallback: true,\n words: ocrWords,\n pagePaths,\n hybridPages: text?.analysis.report.hybridPages,\n });\n if (text) {\n report.bodyFontHeight = text.analysis.report.bodyFontHeight;\n report.headingSizes = text.analysis.report.headingSizes;\n report.listItems = text.analysis.report.listItems;\n report.joinedPairs = text.analysis.report.joinedPairs;\n report.closedHyphens = text.analysis.report.closedHyphens;\n report.hybridPages = text.analysis.report.hybridPages;\n }\n const analysis: Analysis = { outputVersion: report.outputVersion, report, pages };\n opts.onWords?.(analysisWords(pages));\n return { analysis, markdown, warnings };\n}\n\n/**\n * Extract each page from embedded text, using OCR only when the page has\n * image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.\n *\n * @param pdfPath path to the PDF file\n * @param opts see {@link PdfToMarkdownOptions}\n */\nexport async function pdfToMarkdown(pdfPath: string, opts: PdfToMarkdownOptions = {}): Promise<ExtractResult> {\n const result = await analyzePdfAuto(pdfPath, opts);\n return { markdown: result.markdown, report: result.analysis.report, warnings: result.warnings };\n}\n"],"names":["buildOcrAnalysisPages","collect","dictAvailable","dictUnavailableWarning","renderPagesToMarkdown","AUTO_OCR_CHARS_PER_PAGE","buildOcrReport","buildTextAnalysis","selectCollectedPages","validatePages","warnCollector","onWarning","warnings","warn","m","push","extractText","pdfPath","opts","pages","range","first","last","collected","text","pageMarkers","textNotices","Set","onWords","words","markdown","report","analysis","extractOcr","input","md","forcePageMarkers","fallback","needsPageOcr","chars","ocrPages","page","textChars","has","hasImage","largeImage","analysisWords","flatMap","map","word","box","confidence","analyzePdfAuto","ocr","undefined","outputVersion","ocrPageNumbers","noOcr","filter","size","textPageNumbers","textByPage","Map","ocrByPage","ocrWords","ocrRanges","lastRange","at","result","set","get","pagePaths","path","hybridPages","bodyFontHeight","headingSizes","listItems","joinedPairs","closedHyphens","pdfToMarkdown"],"mappings":"AAAA,4EAA4E;AAC5E,kIAAkI;AAClI,EAAE;AACF,8EAA8E;AAC9E,8GAA8G;AAC9G,SAASA,qBAAqB,QAAQ,eAAe;AACrD,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,aAAa,EAAEC,sBAAsB,QAAQ,YAAY;AAGlE,SAASC,qBAAqB,QAAQ,uBAAuB;AAC7D,SAASC,uBAAuB,EAAEC,cAAc,QAAQ,cAAc;AACtE,SAASC,iBAAiB,EAAEC,oBAAoB,QAAQ,qBAAqB;AAE7E,SAASC,aAAa,QAAQ,gBAAgB;AA0E9C,uEAAuE;AACvE,gFAAgF;AAChF,SAASC,cAAcC,SAAqC;IAC1D,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,OAAO,CAACC;QACZF,SAASG,IAAI,CAACD;QACdH,sBAAAA,gCAAAA,UAAYG;IACd;IACA,IAAI,CAACZ,iBAAiBW,KAAKV;IAC3B,OAAO;QAAES;QAAUC;IAAK;AAC1B;AAEA;;;CAGC,GACD,OAAO,eAAeG,YAAYC,OAAe,EAAEC,OAA2B,CAAC,CAAC;QAGlBA,kBAElBA;QAE1CA;IANAT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IACvD,MAAMS,QAAQF,KAAKC,KAAK,GAAG;QAAEE,OAAOH,KAAKC,KAAK,CAACE,KAAK;QAAEC,IAAI,GAAEJ,mBAAAA,KAAKC,KAAK,CAACG,IAAI,cAAfJ,8BAAAA,mBAAmBA,KAAKC,KAAK,CAACE,KAAK;IAAC,IAAI,CAAC;IACrG,MAAME,YAAY,MAAMtB,QAAQgB,SAASG;IACzC,MAAMI,OAAOjB,kBAAkBgB,YAAWL,oBAAAA,KAAKO,WAAW,cAAhBP,+BAAAA,oBAAoB;IAC9DQ,YAAYH,WAAW,IAAII,OAAOd;KAClCK,gBAAAA,KAAKU,OAAO,cAAZV,oCAAAA,mBAAAA,MAAeM,KAAKK,KAAK;IACzB,OAAO;QAAEC,UAAUN,KAAKM,QAAQ;QAAEC,QAAQP,KAAKQ,QAAQ,CAACD,MAAM;QAAEnB;IAAS;AAC3E;AAEA;;;;;;CAMC,GACD,OAAO,eAAeqB,WAAWC,KAAsD,EAAEhB,OAA0B,CAAC,CAAC;IACnHT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IAEvD,wEAAwE;IACxE,sEAAsE;IACtE,MAAM,EAAEQ,KAAK,EAAEU,KAAK,EAAE,GAAG,MAAM7B,sBAAsBkC,OAAOhB,MAAML;IAClE,MAAMsB,KAAK/B,sBAAsBe,OAAO;QAAEiB,kBAAkB;QAAMX,aAAa;IAAK;IAEpF,MAAMM,SAASzB,eAAe;QAC5B6B;QACAE,UAAUnB,KAAKmB,QAAQ;QACvB,yEAAyE;QACzE,oDAAoD;QACpDR;IACF;IACA,OAAO;QAAEC,UAAUK;QAAIJ;QAAQnB;IAAS;AAC1C;AAEA;+DAC+D,GAC/D,SAAS0B,aAAaC,KAAa;IACjC,OAAOA,QAAQlC;AACjB;AAEA,SAASqB,YAAYH,SAAwB,EAAEiB,QAAqB,EAAE3B,IAA+B;IACnG,KAAK,MAAM4B,QAAQlB,UAAUJ,KAAK,CAAE;QAClC,IAAIsB,KAAKC,SAAS,KAAK,KAAKF,SAASG,GAAG,CAACF,KAAKA,IAAI,GAAG;QACrD,IAAIA,KAAKG,QAAQ,EAAE/B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,sEAAsE,CAAC;aACxG5B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,2EAA2E,CAAC;IACtG;IACA,KAAK,MAAMA,QAAQlB,UAAUJ,KAAK,CAAE;QAClC,IAAI,CAACsB,KAAKI,UAAU,IAAIL,SAASG,GAAG,CAACF,KAAKA,IAAI,GAAG;QACjD5B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,uJAAuJ,EAAEA,KAAKA,IAAI,CAAC,yCAAyC,CAAC;IAClO;AACF;AAEA,SAASK,cAAc3B,KAAqB;IAC1C,OAAOA,MAAM4B,OAAO,CAAC,CAACN,OAASA,KAAKZ,KAAK,CAACmB,GAAG,CAAC,CAACC,OAAU,CAAA;gBAAER,MAAMA,KAAKA,IAAI;gBAAEjB,MAAMyB,KAAKzB,IAAI;gBAAE,GAAGyB,KAAKC,GAAG;gBAAEC,YAAYF,KAAKE,UAAU;YAAC,CAAA;AACxI;AAQA;;;kCAGkC,GAClC,OAAO,eAAeC,eAAenC,OAAe,EAAEC,OAA6B,CAAC,CAAC;QAGvBA;QA0D5DA;IA5DAT,cAAcS,KAAKC,KAAK;IACxB,MAAM,EAAEP,QAAQ,EAAEC,IAAI,EAAE,GAAGH,cAAcQ,KAAKP,SAAS;IACvD,MAAMS,QAAQF,KAAKC,KAAK,GAAG;QAAEE,OAAOH,KAAKC,KAAK,CAACE,KAAK;QAAEC,IAAI,GAAEJ,mBAAAA,KAAKC,KAAK,CAACG,IAAI,cAAfJ,8BAAAA,mBAAmBA,KAAKC,KAAK,CAACE,KAAK;IAAC,IAAI,CAAC;IAErG,IAAIH,KAAKmC,GAAG,EAAE;YAKZnC;QAJA,MAAM,EAAEC,KAAK,EAAEU,KAAK,EAAE,GAAG,MAAM7B,sBAAsB;YAAEiB;QAAQ,GAAG;YAAE,GAAGC,IAAI;YAAEU,SAAS0B;QAAU,GAAGzC;QACnG,MAAMiB,WAAW1B,sBAAsBe,OAAO;YAAEiB,kBAAkB;YAAMX,aAAa;QAAK;QAC1F,MAAMM,SAASzB,eAAe;YAAE6B,IAAIL;YAAUD;QAAM;QACpD,MAAMG,WAAqB;YAAEuB,eAAexB,OAAOwB,aAAa;YAAExB;YAAQZ;QAAM;SAChFD,iBAAAA,KAAKU,OAAO,cAAZV,qCAAAA,oBAAAA,MAAeW;QACf,OAAO;YAAEG;YAAUF;YAAUlB;QAAS;IACxC;IAEA,MAAMW,YAAY,MAAMtB,QAAQgB,SAASG;IACzC,MAAMoC,iBAAiB,IAAI7B,IAAI,CAACT,KAAKuC,KAAK,GAAGlC,UAAUJ,KAAK,CAACuC,MAAM,CAAC,CAACjB,OAASA,KAAKG,QAAQ,IAAIN,aAAaG,KAAKC,SAAS,GAAGM,GAAG,CAAC,CAACP,OAASA,KAAKA,IAAI,IAAI,EAAE;IAC1J,IAAI,CAACe,eAAeG,IAAI,EAAE;YACkBzC;YAE1CA;QAFA,MAAMM,OAAOjB,kBAAkBgB,YAAWL,oBAAAA,KAAKO,WAAW,cAAhBP,+BAAAA,oBAAoB;QAC9DQ,YAAYH,WAAWiC,gBAAgB3C;SACvCK,iBAAAA,KAAKU,OAAO,cAAZV,qCAAAA,oBAAAA,MAAeM,KAAKK,KAAK;QACzB,OAAO;YAAEG,UAAUR,KAAKQ,QAAQ;YAAEF,UAAUN,KAAKM,QAAQ;YAAElB;QAAS;IACtE;IAEA,MAAMgD,kBAAkB,IAAIjC,IAAIJ,UAAUJ,KAAK,CAACuC,MAAM,CAAC,CAACjB,OAAS,CAACe,eAAeb,GAAG,CAACF,KAAKA,IAAI,GAAGO,GAAG,CAAC,CAACP,OAASA,KAAKA,IAAI;IACxH,MAAMjB,OAAOoC,gBAAgBD,IAAI,GAAGpD,kBAAkBC,qBAAqBe,WAAWqC,kBAAkB,QAAQN;IAChH,MAAMO,aAAa,IAAIC,IAAItC,iBAAAA,2BAAAA,KAAMQ,QAAQ,CAACb,KAAK,CAAC6B,GAAG,CAAC,CAACP,OAAS;YAACA,KAAKA,IAAI;YAAEA;SAAK;IAC/E,MAAMsB,YAAY,IAAID;IACtB,MAAME,WAA2B,EAAE;IACnC,MAAMC,YAAoD,EAAE;IAC5D,KAAK,MAAMxB,QAAQlB,UAAUJ,KAAK,CAAE;QAClC,IAAI,CAACqC,eAAeb,GAAG,CAACF,KAAKA,IAAI,GAAG;QACpC5B,KAAK,CAAC,CAAC,EAAE4B,KAAKA,IAAI,CAAC,+DAA+D,CAAC;QACnF,MAAMyB,YAAYD,UAAUE,EAAE,CAAC,CAAC;QAChC,IAAID,aAAaA,UAAU5C,IAAI,GAAG,MAAMmB,KAAKA,IAAI,EAAEyB,UAAU5C,IAAI,GAAGmB,KAAKA,IAAI;aACxEwB,UAAUlD,IAAI,CAAC;YAAEM,OAAOoB,KAAKA,IAAI;YAAEnB,MAAMmB,KAAKA,IAAI;QAAC;IAC1D;IACA,KAAK,MAAMrB,SAAS6C,UAAW;QAC7B,MAAMG,SAAS,MAAMpE,sBAAsB;YAAEiB;QAAQ,GAAG;YAAE,GAAGC,IAAI;YAAEC,OAAOC;YAAOQ,SAAS0B;QAAU,GAAGzC;QACvG,KAAK,MAAM4B,QAAQ2B,OAAOjD,KAAK,CAAE4C,UAAUM,GAAG,CAAC5B,KAAKA,IAAI,EAAEA;QAC1DuB,SAASjD,IAAI,IAAIqD,OAAOvC,KAAK;IAC/B;IACAH,YAAYH,WAAWiC,gBAAgB3C;IACvC,MAAMM,QAAQI,UAAUJ,KAAK,CAAC6B,GAAG,CAAC,CAACP;YAASsB;gBAAAA,iBAAAA,UAAUO,GAAG,CAAC7B,KAAKA,IAAI,eAAvBsB,4BAAAA,iBAA6BF,WAAWS,GAAG,CAAC7B,KAAKA,IAAI;;IACjG,MAAMX,WAAW1B,sBAAsBe,OAAO;QAAEiB,kBAAkB;QAAMX,aAAa;IAAK;IAC1F,MAAM8C,YAAY,IAAIT,IAAI3C,MAAM6B,GAAG,CAAC,CAACP,OAAS;YAACA,KAAKA,IAAI;YAAEA,KAAK+B,IAAI;SAAC;IACpE,MAAMzC,SAASzB,eAAe;QAC5B6B,IAAIL;QACJO,UAAU;QACVR,OAAOmC;QACPO;QACAE,WAAW,EAAEjD,iBAAAA,2BAAAA,KAAMQ,QAAQ,CAACD,MAAM,CAAC0C,WAAW;IAChD;IACA,IAAIjD,MAAM;QACRO,OAAO2C,cAAc,GAAGlD,KAAKQ,QAAQ,CAACD,MAAM,CAAC2C,cAAc;QAC3D3C,OAAO4C,YAAY,GAAGnD,KAAKQ,QAAQ,CAACD,MAAM,CAAC4C,YAAY;QACvD5C,OAAO6C,SAAS,GAAGpD,KAAKQ,QAAQ,CAACD,MAAM,CAAC6C,SAAS;QACjD7C,OAAO8C,WAAW,GAAGrD,KAAKQ,QAAQ,CAACD,MAAM,CAAC8C,WAAW;QACrD9C,OAAO+C,aAAa,GAAGtD,KAAKQ,QAAQ,CAACD,MAAM,CAAC+C,aAAa;QACzD/C,OAAO0C,WAAW,GAAGjD,KAAKQ,QAAQ,CAACD,MAAM,CAAC0C,WAAW;IACvD;IACA,MAAMzC,WAAqB;QAAEuB,eAAexB,OAAOwB,aAAa;QAAExB;QAAQZ;IAAM;KAChFD,gBAAAA,KAAKU,OAAO,cAAZV,oCAAAA,mBAAAA,MAAe4B,cAAc3B;IAC7B,OAAO;QAAEa;QAAUF;QAAUlB;IAAS;AACxC;AAEA;;;;;;CAMC,GACD,OAAO,eAAemE,cAAc9D,OAAe,EAAEC,OAA6B,CAAC,CAAC;IAClF,MAAMkD,SAAS,MAAMhB,eAAenC,SAASC;IAC7C,OAAO;QAAEY,UAAUsC,OAAOtC,QAAQ;QAAEC,QAAQqC,OAAOpC,QAAQ,CAACD,MAAM;QAAEnB,UAAUwD,OAAOxD,QAAQ;IAAC;AAChG"}
package/dist/esm/lines.js CHANGED
@@ -20,31 +20,34 @@ export function toLines({ glyphs, width }) {
20
20
  const twoCol = glyphs.filter((g)=>g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g)=>g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g)=>g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;
21
21
  // Group on y with a tolerance that scales with font size.
22
22
  const lines = [];
23
- for (const g of [
24
- ...glyphs
25
- ].sort((a, b)=>a.y - b.y || a.x - b.x)){
23
+ const indexedGlyphs = glyphs.map((glyph, index)=>({
24
+ glyph,
25
+ index
26
+ }));
27
+ for (const part of indexedGlyphs.sort((a, b)=>a.glyph.y - b.glyph.y || a.glyph.x - b.glyph.x)){
28
+ const { glyph: g } = part;
26
29
  // NOTE: start-x assignment can misfile a gutter-straddling glyph; an
27
30
  // overlap-based alternative was tried and reshuffled stray numerals with
28
31
  // no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).
29
32
  const col = twoCol && g.x > mid ? 1 : 0;
30
33
  const last = lines.at(-1);
31
34
  if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {
32
- last.parts.push(g);
35
+ last.parts.push(part);
33
36
  last.y = (last.y + g.y) / 2;
34
37
  } else lines.push({
35
38
  col,
36
39
  y: g.y,
37
40
  parts: [
38
- g
41
+ part
39
42
  ]
40
43
  });
41
44
  }
42
45
  lines.sort((a, b)=>a.col - b.col || a.y - b.y); // column, then down the page
43
46
  return lines.map((l)=>{
44
- const parts = l.parts.sort((a, b)=>a.x - b.x);
47
+ const parts = l.parts.sort((a, b)=>a.glyph.x - b.glyph.x);
45
48
  let t = '';
46
49
  let prev = null;
47
- for (const cur of parts){
50
+ for (const { glyph: cur } of parts){
48
51
  // insert a space when the measured gap is wider than intra-word kerning
49
52
  if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\s$/.test(t) && !/^\s/.test(cur.s)) t += ' ';
50
53
  t += cur.s;
@@ -53,15 +56,16 @@ export function toLines({ glyphs, width }) {
53
56
  // Line height = the char-weighted dominant height, not the max: a small
54
57
  // decorative large-face numeral must not promote a body line to a heading.
55
58
  const byH = new Map();
56
- for (const p of parts)byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);
59
+ for (const { glyph: p } of parts)byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);
57
60
  return {
58
61
  text: normalize(t).replace(/\s+/g, ' ').trim(),
59
- x: parts[0].x,
62
+ x: parts[0].glyph.x,
60
63
  y: l.y,
61
64
  h: [
62
65
  ...byH.entries()
63
66
  ].sort((a, b)=>b[1] - a[1])[0][0],
64
- col: l.col
67
+ col: l.col,
68
+ wordIndexes: parts.map(({ index })=>index)
65
69
  };
66
70
  }).filter((l)=>l.text);
67
71
  }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/lines.ts"],"sourcesContent":["// Pass 2: glyphs -> visual lines — de-braids two-column pages (spanning\n// lines like headings sort with the left column), joins words by measured horizontal gap, and folds typographic codepoints to ASCII so search doesn't miss half the hits.\nimport type { CollectedPage, Glyph, Line } from './types.ts';\n\nconst FOLD: Record<string, string> = {\n ʼ: \"'\",\n '’': \"'\",\n '‘': \"'\",\n '“': '\"',\n '”': '\"',\n '–': '-',\n};\n// Exported for the OCR engine seam (src/engines/tesseract.ts): both paths\n// must fold identically or the same printed apostrophe surfaces as two spellings depending on which path read the page.\nexport const foldTypographic = (s: string): string => s.replace(/[ʼ‘’“”–]/g, (c) => FOLD[c]);\nconst normalize = foldTypographic;\n\n// Intermediate row while grouping glyphs by visual line, before the final\n// text/height reduction below.\ninterface LineDraft {\n col: number;\n y: number;\n parts: Glyph[];\n}\n\nexport function toLines({ glyphs, width }: Pick<CollectedPage, 'glyphs' | 'width'>): Line[] {\n if (!glyphs.length) return [];\n\n const mid = width / 2;\n // A real two-column page has a gutter: clusters each side, almost nothing crossing mid.\n // Label/value layouts also cluster but their values cross mid — de-braiding would reorder them.\n const twoCol = glyphs.filter((g) => g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;\n\n // Group on y with a tolerance that scales with font size.\n const lines: LineDraft[] = [];\n for (const g of [...glyphs].sort((a, b) => a.y - b.y || a.x - b.x)) {\n // NOTE: start-x assignment can misfile a gutter-straddling glyph; an\n // overlap-based alternative was tried and reshuffled stray numerals with\n // no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).\n const col = twoCol && g.x > mid ? 1 : 0;\n const last = lines.at(-1);\n if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {\n last.parts.push(g);\n last.y = (last.y + g.y) / 2;\n } else lines.push({ col, y: g.y, parts: [g] });\n }\n lines.sort((a, b) => a.col - b.col || a.y - b.y); // column, then down the page\n\n return lines\n .map((l): Line => {\n const parts = l.parts.sort((a, b) => a.x - b.x);\n let t = '';\n let prev: Glyph | null = null;\n for (const cur of parts) {\n // insert a space when the measured gap is wider than intra-word kerning\n if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\\s$/.test(t) && !/^\\s/.test(cur.s)) t += ' ';\n t += cur.s;\n prev = cur;\n }\n // Line height = the char-weighted dominant height, not the max: a small\n // decorative large-face numeral must not promote a body line to a heading.\n const byH = new Map<number, number>();\n for (const p of parts) byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);\n return {\n text: normalize(t).replace(/\\s+/g, ' ').trim(),\n x: parts[0].x,\n y: l.y,\n h: [...byH.entries()].sort((a, b) => b[1] - a[1])[0][0],\n col: l.col,\n };\n })\n .filter((l) => l.text);\n}\n"],"names":["FOLD","ʼ","foldTypographic","s","replace","c","normalize","toLines","glyphs","width","length","mid","twoCol","filter","g","x","w","lines","sort","a","b","y","col","last","at","Math","abs","max","h","parts","push","map","l","t","prev","cur","test","byH","Map","p","set","get","text","trim","entries"],"mappings":"AAAA,wEAAwE;AACxE,0KAA0K;AAG1K,MAAMA,OAA+B;IACnCC,GAAG;IACH,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;AACP;AACA,0EAA0E;AAC1E,wHAAwH;AACxH,OAAO,MAAMC,kBAAkB,CAACC,IAAsBA,EAAEC,OAAO,CAAC,aAAa,CAACC,IAAML,IAAI,CAACK,EAAE,EAAE;AAC7F,MAAMC,YAAYJ;AAUlB,OAAO,SAASK,QAAQ,EAAEC,MAAM,EAAEC,KAAK,EAA2C;IAChF,IAAI,CAACD,OAAOE,MAAM,EAAE,OAAO,EAAE;IAE7B,MAAMC,MAAMF,QAAQ;IACpB,wFAAwF;IACxF,gGAAgG;IAChG,MAAMG,SAASJ,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,OAAOG,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG;IAEnO,0DAA0D;IAC1D,MAAMO,QAAqB,EAAE;IAC7B,KAAK,MAAMH,KAAK;WAAIN;KAAO,CAACU,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEE,CAAC,GAAGD,EAAEC,CAAC,IAAIF,EAAEJ,CAAC,GAAGK,EAAEL,CAAC,EAAG;QAClE,qEAAqE;QACrE,yEAAyE;QACzE,0IAA0I;QAC1I,MAAMO,MAAMV,UAAUE,EAAEC,CAAC,GAAGJ,MAAM,IAAI;QACtC,MAAMY,OAAON,MAAMO,EAAE,CAAC,CAAC;QACvB,IAAID,QAAQA,KAAKD,GAAG,KAAKA,OAAOG,KAAKC,GAAG,CAACH,KAAKF,CAAC,GAAGP,EAAEO,CAAC,IAAII,KAAKE,GAAG,CAAC,GAAGb,EAAEc,CAAC,GAAG,MAAM;YAC/EL,KAAKM,KAAK,CAACC,IAAI,CAAChB;YAChBS,KAAKF,CAAC,GAAG,AAACE,CAAAA,KAAKF,CAAC,GAAGP,EAAEO,CAAC,AAADA,IAAK;QAC5B,OAAOJ,MAAMa,IAAI,CAAC;YAAER;YAAKD,GAAGP,EAAEO,CAAC;YAAEQ,OAAO;gBAACf;aAAE;QAAC;IAC9C;IACAG,MAAMC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEG,GAAG,GAAGF,EAAEE,GAAG,IAAIH,EAAEE,CAAC,GAAGD,EAAEC,CAAC,GAAG,6BAA6B;IAE/E,OAAOJ,MACJc,GAAG,CAAC,CAACC;QACJ,MAAMH,QAAQG,EAAEH,KAAK,CAACX,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEJ,CAAC,GAAGK,EAAEL,CAAC;QAC9C,IAAIkB,IAAI;QACR,IAAIC,OAAqB;QACzB,KAAK,MAAMC,OAAON,MAAO;YACvB,wEAAwE;YACxE,IAAIK,QAAQC,IAAIpB,CAAC,GAAImB,CAAAA,KAAKnB,CAAC,GAAGmB,KAAKlB,CAAC,AAADA,IAAKkB,KAAKN,CAAC,GAAG,QAAQ,CAAC,MAAMQ,IAAI,CAACH,MAAM,CAAC,MAAMG,IAAI,CAACD,IAAIhC,CAAC,GAAG8B,KAAK;YACpGA,KAAKE,IAAIhC,CAAC;YACV+B,OAAOC;QACT;QACA,wEAAwE;QACxE,2EAA2E;QAC3E,MAAME,MAAM,IAAIC;QAChB,KAAK,MAAMC,KAAKV,MAAOQ,IAAIG,GAAG,CAACD,EAAEX,CAAC,EAAE,AAACS,CAAAA,IAAII,GAAG,CAACF,EAAEX,CAAC,KAAK,CAAA,IAAKW,EAAEpC,CAAC,CAACO,MAAM;QACpE,OAAO;YACLgC,MAAMpC,UAAU2B,GAAG7B,OAAO,CAAC,QAAQ,KAAKuC,IAAI;YAC5C5B,GAAGc,KAAK,CAAC,EAAE,CAACd,CAAC;YACbM,GAAGW,EAAEX,CAAC;YACNO,GAAG;mBAAIS,IAAIO,OAAO;aAAG,CAAC1B,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;YACvDG,KAAKU,EAAEV,GAAG;QACZ;IACF,GACCT,MAAM,CAAC,CAACmB,IAAMA,EAAEU,IAAI;AACzB"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/lines.ts"],"sourcesContent":["// Pass 2: glyphs -> visual lines — de-braids two-column pages (spanning\n// lines like headings sort with the left column), joins words by measured horizontal gap, and folds typographic codepoints to ASCII so search doesn't miss half the hits.\nimport type { CollectedPage, Glyph, Line } from './types.ts';\n\nconst FOLD: Record<string, string> = {\n ʼ: \"'\",\n '’': \"'\",\n '‘': \"'\",\n '“': '\"',\n '”': '\"',\n '–': '-',\n};\n// Exported for the OCR engine seam (src/engines/tesseract.ts): both paths\n// must fold identically or the same printed apostrophe surfaces as two spellings depending on which path read the page.\nexport const foldTypographic = (s: string): string => s.replace(/[ʼ‘’“”–]/g, (c) => FOLD[c]);\nconst normalize = foldTypographic;\n\n// Intermediate row while grouping glyphs by visual line, before the final\n// text/height reduction below.\ninterface LineDraft {\n col: number;\n y: number;\n parts: Array<{ glyph: Glyph; index: number }>;\n}\n\nexport function toLines({ glyphs, width }: Pick<CollectedPage, 'glyphs' | 'width'>): Line[] {\n if (!glyphs.length) return [];\n\n const mid = width / 2;\n // A real two-column page has a gutter: clusters each side, almost nothing crossing mid.\n // Label/value layouts also cluster but their values cross mid — de-braiding would reorder them.\n const twoCol = glyphs.filter((g) => g.x + g.w < mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x > mid).length > glyphs.length * 0.25 && glyphs.filter((g) => g.x < mid && g.x + g.w > mid).length < glyphs.length * 0.15;\n\n // Group on y with a tolerance that scales with font size.\n const lines: LineDraft[] = [];\n const indexedGlyphs = glyphs.map((glyph, index) => ({ glyph, index }));\n for (const part of indexedGlyphs.sort((a, b) => a.glyph.y - b.glyph.y || a.glyph.x - b.glyph.x)) {\n const { glyph: g } = part;\n // NOTE: start-x assignment can misfile a gutter-straddling glyph; an\n // overlap-based alternative was tried and reshuffled stray numerals with\n // no measured win, so start-x stays (src/geometry.ts's OCR path assigns by x-span overlap instead, where the mis-assignment is measured).\n const col = twoCol && g.x > mid ? 1 : 0;\n const last = lines.at(-1);\n if (last && last.col === col && Math.abs(last.y - g.y) < Math.max(2, g.h * 0.5)) {\n last.parts.push(part);\n last.y = (last.y + g.y) / 2;\n } else lines.push({ col, y: g.y, parts: [part] });\n }\n lines.sort((a, b) => a.col - b.col || a.y - b.y); // column, then down the page\n\n return lines\n .map((l): Line => {\n const parts = l.parts.sort((a, b) => a.glyph.x - b.glyph.x);\n let t = '';\n let prev: Glyph | null = null;\n for (const { glyph: cur } of parts) {\n // insert a space when the measured gap is wider than intra-word kerning\n if (prev && cur.x - (prev.x + prev.w) > prev.h * 0.12 && !/\\s$/.test(t) && !/^\\s/.test(cur.s)) t += ' ';\n t += cur.s;\n prev = cur;\n }\n // Line height = the char-weighted dominant height, not the max: a small\n // decorative large-face numeral must not promote a body line to a heading.\n const byH = new Map<number, number>();\n for (const { glyph: p } of parts) byH.set(p.h, (byH.get(p.h) || 0) + p.s.length);\n return {\n text: normalize(t).replace(/\\s+/g, ' ').trim(),\n x: parts[0].glyph.x,\n y: l.y,\n h: [...byH.entries()].sort((a, b) => b[1] - a[1])[0][0],\n col: l.col,\n wordIndexes: parts.map(({ index }) => index),\n };\n })\n .filter((l) => l.text);\n}\n"],"names":["FOLD","ʼ","foldTypographic","s","replace","c","normalize","toLines","glyphs","width","length","mid","twoCol","filter","g","x","w","lines","indexedGlyphs","map","glyph","index","part","sort","a","b","y","col","last","at","Math","abs","max","h","parts","push","l","t","prev","cur","test","byH","Map","p","set","get","text","trim","entries","wordIndexes"],"mappings":"AAAA,wEAAwE;AACxE,0KAA0K;AAG1K,MAAMA,OAA+B;IACnCC,GAAG;IACH,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;AACP;AACA,0EAA0E;AAC1E,wHAAwH;AACxH,OAAO,MAAMC,kBAAkB,CAACC,IAAsBA,EAAEC,OAAO,CAAC,aAAa,CAACC,IAAML,IAAI,CAACK,EAAE,EAAE;AAC7F,MAAMC,YAAYJ;AAUlB,OAAO,SAASK,QAAQ,EAAEC,MAAM,EAAEC,KAAK,EAA2C;IAChF,IAAI,CAACD,OAAOE,MAAM,EAAE,OAAO,EAAE;IAE7B,MAAMC,MAAMF,QAAQ;IACpB,wFAAwF;IACxF,gGAAgG;IAChG,MAAMG,SAASJ,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG,QAAQF,OAAOK,MAAM,CAAC,CAACC,IAAMA,EAAEC,CAAC,GAAGJ,OAAOG,EAAEC,CAAC,GAAGD,EAAEE,CAAC,GAAGL,KAAKD,MAAM,GAAGF,OAAOE,MAAM,GAAG;IAEnO,0DAA0D;IAC1D,MAAMO,QAAqB,EAAE;IAC7B,MAAMC,gBAAgBV,OAAOW,GAAG,CAAC,CAACC,OAAOC,QAAW,CAAA;YAAED;YAAOC;QAAM,CAAA;IACnE,KAAK,MAAMC,QAAQJ,cAAcK,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEJ,KAAK,CAACM,CAAC,GAAGD,EAAEL,KAAK,CAACM,CAAC,IAAIF,EAAEJ,KAAK,CAACL,CAAC,GAAGU,EAAEL,KAAK,CAACL,CAAC,EAAG;QAC/F,MAAM,EAAEK,OAAON,CAAC,EAAE,GAAGQ;QACrB,qEAAqE;QACrE,yEAAyE;QACzE,0IAA0I;QAC1I,MAAMK,MAAMf,UAAUE,EAAEC,CAAC,GAAGJ,MAAM,IAAI;QACtC,MAAMiB,OAAOX,MAAMY,EAAE,CAAC,CAAC;QACvB,IAAID,QAAQA,KAAKD,GAAG,KAAKA,OAAOG,KAAKC,GAAG,CAACH,KAAKF,CAAC,GAAGZ,EAAEY,CAAC,IAAII,KAAKE,GAAG,CAAC,GAAGlB,EAAEmB,CAAC,GAAG,MAAM;YAC/EL,KAAKM,KAAK,CAACC,IAAI,CAACb;YAChBM,KAAKF,CAAC,GAAG,AAACE,CAAAA,KAAKF,CAAC,GAAGZ,EAAEY,CAAC,AAADA,IAAK;QAC5B,OAAOT,MAAMkB,IAAI,CAAC;YAAER;YAAKD,GAAGZ,EAAEY,CAAC;YAAEQ,OAAO;gBAACZ;aAAK;QAAC;IACjD;IACAL,MAAMM,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEG,GAAG,GAAGF,EAAEE,GAAG,IAAIH,EAAEE,CAAC,GAAGD,EAAEC,CAAC,GAAG,6BAA6B;IAE/E,OAAOT,MACJE,GAAG,CAAC,CAACiB;QACJ,MAAMF,QAAQE,EAAEF,KAAK,CAACX,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEJ,KAAK,CAACL,CAAC,GAAGU,EAAEL,KAAK,CAACL,CAAC;QAC1D,IAAIsB,IAAI;QACR,IAAIC,OAAqB;QACzB,KAAK,MAAM,EAAElB,OAAOmB,GAAG,EAAE,IAAIL,MAAO;YAClC,wEAAwE;YACxE,IAAII,QAAQC,IAAIxB,CAAC,GAAIuB,CAAAA,KAAKvB,CAAC,GAAGuB,KAAKtB,CAAC,AAADA,IAAKsB,KAAKL,CAAC,GAAG,QAAQ,CAAC,MAAMO,IAAI,CAACH,MAAM,CAAC,MAAMG,IAAI,CAACD,IAAIpC,CAAC,GAAGkC,KAAK;YACpGA,KAAKE,IAAIpC,CAAC;YACVmC,OAAOC;QACT;QACA,wEAAwE;QACxE,2EAA2E;QAC3E,MAAME,MAAM,IAAIC;QAChB,KAAK,MAAM,EAAEtB,OAAOuB,CAAC,EAAE,IAAIT,MAAOO,IAAIG,GAAG,CAACD,EAAEV,CAAC,EAAE,AAACQ,CAAAA,IAAII,GAAG,CAACF,EAAEV,CAAC,KAAK,CAAA,IAAKU,EAAExC,CAAC,CAACO,MAAM;QAC/E,OAAO;YACLoC,MAAMxC,UAAU+B,GAAGjC,OAAO,CAAC,QAAQ,KAAK2C,IAAI;YAC5ChC,GAAGmB,KAAK,CAAC,EAAE,CAACd,KAAK,CAACL,CAAC;YACnBW,GAAGU,EAAEV,CAAC;YACNO,GAAG;mBAAIQ,IAAIO,OAAO;aAAG,CAACzB,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;YACvDG,KAAKS,EAAET,GAAG;YACVsB,aAAaf,MAAMf,GAAG,CAAC,CAAC,EAAEE,KAAK,EAAE,GAAKA;QACxC;IACF,GACCR,MAAM,CAAC,CAACuB,IAAMA,EAAEU,IAAI;AACzB"}
@@ -23,6 +23,7 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
23
23
  Math.min(6, i + 1 + offset)
24
24
  ]));
25
25
  const out = [];
26
+ const blockPages = new Map();
26
27
  const stats = {
27
28
  headings: 0,
28
29
  listItems: 0,
@@ -32,38 +33,64 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
32
33
  callouts: 0,
33
34
  closedHyphens: 0
34
35
  };
36
+ let currentPage = 0;
35
37
  let para = [];
38
+ const addBlock = (block)=>{
39
+ const blocks = blockPages.get(currentPage);
40
+ if (blocks) blocks.push(block);
41
+ else blockPages.set(currentPage, [
42
+ block
43
+ ]);
44
+ };
36
45
  // a label line ending in ':' whose next line is a bare number belongs on one line
37
- function attachValue(label, lines, get, set) {
38
- const j = get() + 1;
46
+ function attachValue(label, lines, i) {
47
+ const j = i + 1;
39
48
  if (/[::]\s*$/.test(label) && j < lines.length && /^[0-9][\d.,]*%?$/.test(lines[j].text)) {
40
- set(j);
41
49
  stats.joinedPairs++;
42
- return `${label} ${lines[j].text}`;
50
+ return {
51
+ text: `${label} ${lines[j].text}`,
52
+ last: j
53
+ };
43
54
  }
44
- return label;
55
+ return {
56
+ text: label,
57
+ last: i
58
+ };
45
59
  }
46
60
  const flush = ()=>{
47
61
  if (!para.length) return;
48
- out.push(para.join(' ').replace(/\s+([,.;:%])/g, '$1'));
62
+ const text = para.map((part)=>part.text).join(' ').replace(/\s+([,.;:%])/g, '$1');
63
+ out.push(text);
49
64
  out.push('');
65
+ addBlock({
66
+ type: 'paragraph',
67
+ text,
68
+ wordIndexes: para.flatMap((part)=>part.wordIndexes)
69
+ });
50
70
  stats.paragraphs++;
51
71
  para = [];
52
72
  };
53
73
  // append a wrapped line to the open paragraph, resolving a line-final hyphen
54
- const appendToPara = (text)=>{
74
+ const appendToPara = (line, text)=>{
55
75
  const prev = para.at(-1);
56
- if (prev && /\w-$/.test(prev)) {
76
+ if (prev && /\w-$/.test(prev.text)) {
57
77
  var _text_split_;
58
78
  // a bare " -" is punctuation, not a wrapped word
59
- const head = prev.replace(/-$/, '').split(/\s/).pop();
79
+ const head = prev.text.replace(/-$/, '').split(/\s/).pop();
60
80
  const tail = ((_text_split_ = text.split(/\s/)[0]) !== null && _text_split_ !== void 0 ? _text_split_ : '').replace(/[^A-Za-z-]/g, '').split('-')[0];
61
81
  const close = getDict().has((head + tail).toLowerCase());
62
82
  if (close) stats.closedHyphens++;
63
- para[para.length - 1] = prev.replace(/-$/, close ? '' : '-') + text;
64
- } else para.push(text);
83
+ para[para.length - 1].text = prev.text.replace(/-$/, close ? '' : '-') + text;
84
+ para[para.length - 1].wordIndexes.push(...line.wordIndexes);
85
+ } else para.push({
86
+ text,
87
+ wordIndexes: [
88
+ ...line.wordIndexes
89
+ ]
90
+ });
65
91
  };
66
92
  for (const { page, lines } of pageLines){
93
+ currentPage = page;
67
94
  if (opts.pageMarkers) {
68
95
  flush();
69
96
  out.push(pageMarkerLine(page));
@@ -80,22 +107,35 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
80
107
  // below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.
81
108
  if (levelOf.has(h) && !BULLETS.test(text)) {
82
109
  const run = [
83
- text
110
+ lines[i]
84
111
  ];
85
- while(i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text))run.push(lines[++i].text);
112
+ while(i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text))run.push(lines[++i]);
86
113
  // join wrapped title lines, resolving line-final hyphens like body text
87
- const merged = run.reduce((acc, part)=>{
114
+ const merged = run.map((line)=>line.text).reduce((acc, part)=>{
88
115
  var _part_split_;
89
116
  return /\w-$/.test(acc) ? acc.replace(/-$/, getDict().has((acc.replace(/-$/, '').split(/\s/).pop() + ((_part_split_ = part.split(/\s/)[0]) !== null && _part_split_ !== void 0 ? _part_split_ : '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`;
90
117
  }).replace(/\s+([,.;:%])/g, '$1');
118
+ const wordIndexes = run.flatMap((line)=>line.wordIndexes);
91
119
  flush();
92
120
  if (merged.length > 150) {
93
121
  out.push(`> ${merged}`);
94
122
  out.push('');
123
+ addBlock({
124
+ type: 'paragraph',
125
+ text: `> ${merged}`,
126
+ wordIndexes
127
+ });
95
128
  stats.callouts++;
96
129
  } else {
97
- out.push(`${'#'.repeat(levelOf.get(h))} ${merged}`);
130
+ const level = levelOf.get(h);
131
+ out.push(`${'#'.repeat(level)} ${merged}`);
98
132
  out.push('');
133
+ addBlock({
134
+ type: 'heading',
135
+ text: merged,
136
+ level,
137
+ wordIndexes
138
+ });
99
139
  stats.headings++;
100
140
  }
101
141
  continue;
@@ -108,13 +148,27 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
108
148
  if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {
109
149
  flush();
110
150
  const depth = Math.max(0, Math.round((x - indentBase) / 18));
111
- let label = lines[++i].text.replace(BULLETS, '').trim();
112
- while(!label && i + 1 < lines.length)// stacked glyphs
113
- label = lines[++i].text.replace(BULLETS, '').trim();
114
- label = attachValue(label, lines, ()=>i, (n)=>{
115
- i = n;
151
+ const itemLines = [
152
+ lines[i],
153
+ lines[++i]
154
+ ];
155
+ let label = lines[i].text.replace(BULLETS, '').trim();
156
+ while(!label && i + 1 < lines.length){
157
+ // stacked glyphs
158
+ itemLines.push(lines[++i]);
159
+ label = lines[i].text.replace(BULLETS, '').trim();
160
+ }
161
+ const attached = attachValue(label, lines, i);
162
+ if (attached.last !== i) itemLines.push(lines[attached.last]);
163
+ i = attached.last;
164
+ const itemText = `${' '.repeat(depth)}- ${attached.text}`;
165
+ out.push(itemText);
166
+ addBlock({
167
+ type: 'paragraph',
168
+ text: itemText,
169
+ tight: true,
170
+ wordIndexes: itemLines.flatMap((line)=>line.wordIndexes)
116
171
  });
117
- out.push(`${' '.repeat(depth)}- ${label}`);
118
172
  stats.listItems++;
119
173
  continue;
120
174
  }
@@ -122,21 +176,42 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
122
176
  if (wasBullet) {
123
177
  flush();
124
178
  const depth = Math.max(0, Math.round((x - indentBase) / 18));
125
- out.push(`${' '.repeat(depth)}- ${attachValue(text, lines, ()=>i, (n)=>{
126
- i = n;
127
- })}`);
179
+ const itemLines = [
180
+ lines[i]
181
+ ];
182
+ const attached = attachValue(text, lines, i);
183
+ if (attached.last !== i) itemLines.push(lines[attached.last]);
184
+ i = attached.last;
185
+ const itemText = `${' '.repeat(depth)}- ${attached.text}`;
186
+ out.push(itemText);
187
+ addBlock({
188
+ type: 'paragraph',
189
+ text: itemText,
190
+ tight: true,
191
+ wordIndexes: itemLines.flatMap((line)=>line.wordIndexes)
192
+ });
128
193
  stats.listItems++;
129
194
  continue;
130
195
  }
131
196
  // `label:` on its own line followed by a bare value -> keep together
132
197
  if (/[::]\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\d.,]*%?$/.test(lines[i + 1].text)) {
133
198
  flush();
134
- out.push(`- ${text} ${lines[++i].text}`);
199
+ const itemText = `- ${text} ${lines[++i].text}`;
200
+ out.push(itemText);
201
+ addBlock({
202
+ type: 'paragraph',
203
+ text: itemText,
204
+ tight: true,
205
+ wordIndexes: [
206
+ ...lines[i - 1].wordIndexes,
207
+ ...lines[i].wordIndexes
208
+ ]
209
+ });
135
210
  stats.joinedPairs++;
136
211
  continue;
137
212
  }
138
213
  // otherwise body text: re-flow wrapped lines into a paragraph
139
- appendToPara(text);
214
+ appendToPara(lines[i], text);
140
215
  if (/[.!?:]$/.test(text)) flush();
141
216
  else stats.reflowed++;
142
217
  }
@@ -144,10 +219,18 @@ export function buildMarkdown(pageLines, heightChars, opts = {}) {
144
219
  }
145
220
  flush();
146
221
  const md = `${out.join('\n').replace(/\n{3,}/g, '\n\n').trim()}\n`;
222
+ const pages = pageLines.map(({ page })=>{
223
+ var _blockPages_get;
224
+ return {
225
+ page,
226
+ blocks: (_blockPages_get = blockPages.get(page)) !== null && _blockPages_get !== void 0 ? _blockPages_get : []
227
+ };
228
+ });
147
229
  return {
148
230
  md,
149
231
  stats,
150
232
  bodyH,
151
- headingSizes
233
+ headingSizes,
234
+ pages
152
235
  };
153
236
  }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/markdown.ts"],"sourcesContent":["// Pass 3: lines -> markdown. Structure is measured, not guessed — heading\n// levels come from the font-height histogram, hyphen-closes only against a dictionary (see the inline comments below for each rule).\nimport { getDict } from './dict.ts';\nimport { pageMarkerLine } from './page-marker.ts';\nimport type { BuildMarkdownOptions, Line, MarkdownResult, MarkdownStats, PageLines } from './types.ts';\n\nconst GLYPH = /^[•◦▪▫●○·‣⁃]$/;\nconst BULLETS = /^(?:[•◦▪▫●○·‣⁃]\\s*)+/;\n\nexport function buildMarkdown(pageLines: PageLines[], heightChars: Map<number, number>, opts: BuildMarkdownOptions = {}): MarkdownResult {\n // body height = char-weighted mode; headings are the rarer, larger sizes.\n // The 1.35 multiplier keeps emphasized body text out of the outline.\n const bodyH = [...heightChars.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? 0;\n const headingSizes = [...heightChars.keys()].filter((h) => h > bodyH * 1.35).sort((a, b) => b - a);\n // with page markers on, content headings start below the page ### level;\n // sizes beyond the ###### budget share level 6 rather than being dropped\n const offset = opts.pageMarkers ? 3 : 0;\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1 + offset)]));\n\n const out: string[] = [];\n const stats: MarkdownStats = {\n headings: 0,\n listItems: 0,\n paragraphs: 0,\n joinedPairs: 0,\n reflowed: 0,\n callouts: 0,\n closedHyphens: 0,\n };\n let para: string[] = [];\n\n // a label line ending in ':' whose next line is a bare number belongs on one line\n function attachValue(label: string, lines: Line[], get: () => number, set: (n: number) => void): string {\n const j = get() + 1;\n if (/[::]\\s*$/.test(label) && j < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[j].text)) {\n set(j);\n stats.joinedPairs++;\n return `${label} ${lines[j].text}`;\n }\n return label;\n }\n const flush = () => {\n if (!para.length) return;\n out.push(para.join(' ').replace(/\\s+([,.;:%])/g, '$1'));\n out.push('');\n stats.paragraphs++;\n para = [];\n };\n // append a wrapped line to the open paragraph, resolving a line-final hyphen\n const appendToPara = (text: string) => {\n const prev = para.at(-1);\n if (prev && /\\w-$/.test(prev)) {\n // a bare \" -\" is punctuation, not a wrapped word\n const head = prev.replace(/-$/, '').split(/\\s/).pop() as string;\n const tail = (text.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0];\n const close = getDict().has((head + tail).toLowerCase());\n if (close) stats.closedHyphens++;\n para[para.length - 1] = prev.replace(/-$/, close ? '' : '-') + text;\n } else para.push(text);\n };\n\n for (const { page, lines } of pageLines) {\n if (opts.pageMarkers) {\n flush();\n out.push(pageMarkerLine(page));\n out.push('');\n }\n if (!lines.length) continue;\n const indentBase = Math.min(...lines.map((l) => l.x));\n\n for (let i = 0; i < lines.length; i++) {\n let { text, h, x } = lines[i];\n const prevLine = i > 0 ? lines[i - 1] : null;\n // a column change or a vertical jump wider than a blank line ends the paragraph\n if (para.length && prevLine && (lines[i].col !== prevLine.col || lines[i].y - prevLine.y > h * 2.2)) flush();\n\n // A large-font line opens a run: gather the consecutive same-size lines\n // below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.\n if (levelOf.has(h) && !BULLETS.test(text)) {\n const run = [text];\n while (i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text)) run.push(lines[++i].text);\n // join wrapped title lines, resolving line-final hyphens like body text\n const merged = run.reduce((acc, part) => (/\\w-$/.test(acc) ? acc.replace(/-$/, getDict().has(((acc.replace(/-$/, '').split(/\\s/).pop() as string) + (part.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`)).replace(/\\s+([,.;:%])/g, '$1');\n flush();\n if (merged.length > 150) {\n out.push(`> ${merged}`);\n out.push('');\n stats.callouts++;\n } else {\n out.push(`${'#'.repeat(levelOf.get(h) as number)} ${merged}`);\n out.push('');\n stats.headings++;\n }\n continue;\n }\n\n // Lines can carry several stacked glyphs (\"• ▪\") from nested source lists.\n const stripped = text.replace(BULLETS, '').trim();\n const wasBullet = stripped !== text;\n if (wasBullet) text = stripped;\n\n // bullet with nothing left on the line: attach the following line's text\n if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n let label = lines[++i].text.replace(BULLETS, '').trim();\n while (!label && i + 1 < lines.length)\n // stacked glyphs\n label = lines[++i].text.replace(BULLETS, '').trim();\n label = attachValue(\n label,\n lines,\n () => i,\n (n) => {\n i = n;\n }\n );\n out.push(`${' '.repeat(depth)}- ${label}`);\n stats.listItems++;\n continue;\n }\n // glyph(s) followed by text on the same line\n if (wasBullet) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n out.push(\n `${' '.repeat(depth)}- ${attachValue(\n text,\n lines,\n () => i,\n (n) => {\n i = n;\n }\n )}`\n );\n stats.listItems++;\n continue;\n }\n // `label:` on its own line followed by a bare value -> keep together\n if (/[::]\\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[i + 1].text)) {\n flush();\n out.push(`- ${text} ${lines[++i].text}`);\n stats.joinedPairs++;\n continue;\n }\n // otherwise body text: re-flow wrapped lines into a paragraph\n appendToPara(text);\n if (/[.!?:]$/.test(text)) flush();\n else stats.reflowed++;\n }\n flush();\n }\n flush();\n\n const md = `${out\n .join('\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()}\\n`;\n return { md, stats, bodyH, headingSizes };\n}\n"],"names":["getDict","pageMarkerLine","GLYPH","BULLETS","buildMarkdown","pageLines","heightChars","opts","bodyH","entries","sort","a","b","headingSizes","keys","filter","h","offset","pageMarkers","levelOf","Map","slice","map","i","Math","min","out","stats","headings","listItems","paragraphs","joinedPairs","reflowed","callouts","closedHyphens","para","attachValue","label","lines","get","set","j","test","length","text","flush","push","join","replace","appendToPara","prev","at","head","split","pop","tail","close","has","toLowerCase","page","indentBase","l","x","prevLine","col","y","run","merged","reduce","acc","part","repeat","stripped","trim","wasBullet","depth","max","round","n","md"],"mappings":"AAAA,0EAA0E;AAC1E,qIAAqI;AACrI,SAASA,OAAO,QAAQ,YAAY;AACpC,SAASC,cAAc,QAAQ,mBAAmB;AAGlD,MAAMC,QAAQ;AACd,MAAMC,UAAU;AAEhB,OAAO,SAASC,cAAcC,SAAsB,EAAEC,WAAgC,EAAEC,OAA6B,CAAC,CAAC;;QAGvG;IAFd,0EAA0E;IAC1E,qEAAqE;IACrE,MAAMC,iBAAQ,SAAA;WAAIF,YAAYG,OAAO;KAAG,CAACC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,cAAzD,6BAAA,MAA2D,CAAC,EAAE,uCAAI;IAChF,MAAME,eAAe;WAAIP,YAAYQ,IAAI;KAAG,CAACC,MAAM,CAAC,CAACC,IAAMA,IAAIR,QAAQ,MAAME,IAAI,CAAC,CAACC,GAAGC,IAAMA,IAAID;IAChG,yEAAyE;IACzE,yEAAyE;IACzE,MAAMM,SAASV,KAAKW,WAAW,GAAG,IAAI;IACtC,MAAMC,UAAU,IAAIC,IAAIP,aAAaQ,KAAK,CAAC,GAAG,GAAGC,GAAG,CAAC,CAACN,GAAGO,IAAM;YAACP;YAAGQ,KAAKC,GAAG,CAAC,GAAGF,IAAI,IAAIN;SAAQ;IAE/F,MAAMS,MAAgB,EAAE;IACxB,MAAMC,QAAuB;QAC3BC,UAAU;QACVC,WAAW;QACXC,YAAY;QACZC,aAAa;QACbC,UAAU;QACVC,UAAU;QACVC,eAAe;IACjB;IACA,IAAIC,OAAiB,EAAE;IAEvB,kFAAkF;IAClF,SAASC,YAAYC,KAAa,EAAEC,KAAa,EAAEC,GAAiB,EAAEC,GAAwB;QAC5F,MAAMC,IAAIF,QAAQ;QAClB,IAAI,WAAWG,IAAI,CAACL,UAAUI,IAAIH,MAAMK,MAAM,IAAI,mBAAmBD,IAAI,CAACJ,KAAK,CAACG,EAAE,CAACG,IAAI,GAAG;YACxFJ,IAAIC;YACJd,MAAMI,WAAW;YACjB,OAAO,GAAGM,MAAM,CAAC,EAAEC,KAAK,CAACG,EAAE,CAACG,IAAI,EAAE;QACpC;QACA,OAAOP;IACT;IACA,MAAMQ,QAAQ;QACZ,IAAI,CAACV,KAAKQ,MAAM,EAAE;QAClBjB,IAAIoB,IAAI,CAACX,KAAKY,IAAI,CAAC,KAAKC,OAAO,CAAC,iBAAiB;QACjDtB,IAAIoB,IAAI,CAAC;QACTnB,MAAMG,UAAU;QAChBK,OAAO,EAAE;IACX;IACA,6EAA6E;IAC7E,MAAMc,eAAe,CAACL;QACpB,MAAMM,OAAOf,KAAKgB,EAAE,CAAC,CAAC;QACtB,IAAID,QAAQ,OAAOR,IAAI,CAACQ,OAAO;gBAGfN;YAFd,iDAAiD;YACjD,MAAMQ,OAAOF,KAAKF,OAAO,CAAC,MAAM,IAAIK,KAAK,CAAC,MAAMC,GAAG;YACnD,MAAMC,OAAO,EAACX,eAAAA,KAAKS,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBT,0BAAAA,eAAuB,IAAII,OAAO,CAAC,eAAe,IAAIK,KAAK,CAAC,IAAI,CAAC,EAAE;YACjF,MAAMG,QAAQxD,UAAUyD,GAAG,CAAC,AAACL,CAAAA,OAAOG,IAAG,EAAGG,WAAW;YACrD,IAAIF,OAAO7B,MAAMO,aAAa;YAC9BC,IAAI,CAACA,KAAKQ,MAAM,GAAG,EAAE,GAAGO,KAAKF,OAAO,CAAC,MAAMQ,QAAQ,KAAK,OAAOZ;QACjE,OAAOT,KAAKW,IAAI,CAACF;IACnB;IAEA,KAAK,MAAM,EAAEe,IAAI,EAAErB,KAAK,EAAE,IAAIjC,UAAW;QACvC,IAAIE,KAAKW,WAAW,EAAE;YACpB2B;YACAnB,IAAIoB,IAAI,CAAC7C,eAAe0D;YACxBjC,IAAIoB,IAAI,CAAC;QACX;QACA,IAAI,CAACR,MAAMK,MAAM,EAAE;QACnB,MAAMiB,aAAapC,KAAKC,GAAG,IAAIa,MAAMhB,GAAG,CAAC,CAACuC,IAAMA,EAAEC,CAAC;QAEnD,IAAK,IAAIvC,IAAI,GAAGA,IAAIe,MAAMK,MAAM,EAAEpB,IAAK;YACrC,IAAI,EAAEqB,IAAI,EAAE5B,CAAC,EAAE8C,CAAC,EAAE,GAAGxB,KAAK,CAACf,EAAE;YAC7B,MAAMwC,WAAWxC,IAAI,IAAIe,KAAK,CAACf,IAAI,EAAE,GAAG;YACxC,gFAAgF;YAChF,IAAIY,KAAKQ,MAAM,IAAIoB,YAAazB,CAAAA,KAAK,CAACf,EAAE,CAACyC,GAAG,KAAKD,SAASC,GAAG,IAAI1B,KAAK,CAACf,EAAE,CAAC0C,CAAC,GAAGF,SAASE,CAAC,GAAGjD,IAAI,GAAE,GAAI6B;YAErG,wEAAwE;YACxE,mLAAmL;YACnL,IAAI1B,QAAQsC,GAAG,CAACzC,MAAM,CAACb,QAAQuC,IAAI,CAACE,OAAO;gBACzC,MAAMsB,MAAM;oBAACtB;iBAAK;gBAClB,MAAOrB,IAAI,IAAIe,MAAMK,MAAM,IAAIL,KAAK,CAACf,IAAI,EAAE,CAACP,CAAC,KAAKA,KAAKsB,KAAK,CAACf,IAAI,EAAE,CAACyC,GAAG,KAAK1B,KAAK,CAACf,EAAE,CAACyC,GAAG,IAAI1B,KAAK,CAACf,IAAI,EAAE,CAAC0C,CAAC,GAAG3B,KAAK,CAACf,EAAE,CAAC0C,CAAC,GAAGjD,IAAI,OAAO,CAACb,QAAQuC,IAAI,CAACJ,KAAK,CAACf,IAAI,EAAE,CAACqB,IAAI,EAAGsB,IAAIpB,IAAI,CAACR,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI;gBAC/L,wEAAwE;gBACxE,MAAMuB,SAASD,IAAIE,MAAM,CAAC,CAACC,KAAKC;wBAAqHA;2BAA3G,OAAO5B,IAAI,CAAC2B,OAAOA,IAAIrB,OAAO,CAAC,MAAMhD,UAAUyD,GAAG,CAAC,AAAC,CAAA,AAACY,IAAIrB,OAAO,CAAC,MAAM,IAAIK,KAAK,CAAC,MAAMC,GAAG,KAAgB,EAACgB,eAAAA,KAAKjB,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBiB,0BAAAA,eAAuB,IAAItB,OAAO,CAAC,eAAe,IAAIK,KAAK,CAAC,IAAI,CAAC,EAAE,AAAD,EAAGK,WAAW,MAAM,KAAK,OAAOY,OAAO,GAAGD,IAAI,CAAC,EAAEC,MAAM;mBAAGtB,OAAO,CAAC,iBAAiB;gBAC1SH;gBACA,IAAIsB,OAAOxB,MAAM,GAAG,KAAK;oBACvBjB,IAAIoB,IAAI,CAAC,CAAC,EAAE,EAAEqB,QAAQ;oBACtBzC,IAAIoB,IAAI,CAAC;oBACTnB,MAAMM,QAAQ;gBAChB,OAAO;oBACLP,IAAIoB,IAAI,CAAC,GAAG,IAAIyB,MAAM,CAACpD,QAAQoB,GAAG,CAACvB,IAAc,CAAC,EAAEmD,QAAQ;oBAC5DzC,IAAIoB,IAAI,CAAC;oBACTnB,MAAMC,QAAQ;gBAChB;gBACA;YACF;YAEA,2EAA2E;YAC3E,MAAM4C,WAAW5B,KAAKI,OAAO,CAAC7C,SAAS,IAAIsE,IAAI;YAC/C,MAAMC,YAAYF,aAAa5B;YAC/B,IAAI8B,WAAW9B,OAAO4B;YAEtB,yEAAyE;YACzE,IAAI,AAAC,CAAA,CAAC5B,QAAQ1C,MAAMwC,IAAI,CAACE,KAAI,KAAMrB,IAAI,IAAIe,MAAMK,MAAM,EAAE;gBACvDE;gBACA,MAAM8B,QAAQnD,KAAKoD,GAAG,CAAC,GAAGpD,KAAKqD,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxD,IAAIvB,QAAQC,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI,CAACI,OAAO,CAAC7C,SAAS,IAAIsE,IAAI;gBACrD,MAAO,CAACpC,SAASd,IAAI,IAAIe,MAAMK,MAAM,CACnC,iBAAiB;gBACjBN,QAAQC,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI,CAACI,OAAO,CAAC7C,SAAS,IAAIsE,IAAI;gBACnDpC,QAAQD,YACNC,OACAC,OACA,IAAMf,GACN,CAACuD;oBACCvD,IAAIuD;gBACN;gBAEFpD,IAAIoB,IAAI,CAAC,GAAG,KAAKyB,MAAM,CAACI,OAAO,EAAE,EAAEtC,OAAO;gBAC1CV,MAAME,SAAS;gBACf;YACF;YACA,6CAA6C;YAC7C,IAAI6C,WAAW;gBACb7B;gBACA,MAAM8B,QAAQnD,KAAKoD,GAAG,CAAC,GAAGpD,KAAKqD,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxDlC,IAAIoB,IAAI,CACN,GAAG,KAAKyB,MAAM,CAACI,OAAO,EAAE,EAAEvC,YACxBQ,MACAN,OACA,IAAMf,GACN,CAACuD;oBACCvD,IAAIuD;gBACN,IACC;gBAELnD,MAAME,SAAS;gBACf;YACF;YACA,qEAAqE;YACrE,IAAI,WAAWa,IAAI,CAACE,SAASrB,IAAI,IAAIe,MAAMK,MAAM,IAAI,mBAAmBD,IAAI,CAACJ,KAAK,CAACf,IAAI,EAAE,CAACqB,IAAI,GAAG;gBAC/FC;gBACAnB,IAAIoB,IAAI,CAAC,CAAC,EAAE,EAAEF,KAAK,CAAC,EAAEN,KAAK,CAAC,EAAEf,EAAE,CAACqB,IAAI,EAAE;gBACvCjB,MAAMI,WAAW;gBACjB;YACF;YACA,8DAA8D;YAC9DkB,aAAaL;YACb,IAAI,UAAUF,IAAI,CAACE,OAAOC;iBACrBlB,MAAMK,QAAQ;QACrB;QACAa;IACF;IACAA;IAEA,MAAMkC,KAAK,GAAGrD,IACXqB,IAAI,CAAC,MACLC,OAAO,CAAC,WAAW,QACnByB,IAAI,GAAG,EAAE,CAAC;IACb,OAAO;QAAEM;QAAIpD;QAAOnB;QAAOK;IAAa;AAC1C"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/markdown.ts"],"sourcesContent":["// Pass 3: lines -> markdown. Structure is measured, not guessed — heading\n// levels come from the font-height histogram, hyphen-closes only against a dictionary (see the inline comments below for each rule).\nimport { getDict } from './dict.ts';\nimport { pageMarkerLine } from './page-marker.ts';\nimport type { AnalysisBlock, BuildMarkdownOptions, Line, MarkdownPage, MarkdownResult, MarkdownStats, PageLines } from './types.ts';\n\nconst GLYPH = /^[•◦▪▫●○·‣⁃]$/;\nconst BULLETS = /^(?:[•◦▪▫●○·‣⁃]\\s*)+/;\n\nexport function buildMarkdown(pageLines: PageLines[], heightChars: Map<number, number>, opts: BuildMarkdownOptions = {}): MarkdownResult {\n // body height = char-weighted mode; headings are the rarer, larger sizes.\n // The 1.35 multiplier keeps emphasized body text out of the outline.\n const bodyH = [...heightChars.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? 0;\n const headingSizes = [...heightChars.keys()].filter((h) => h > bodyH * 1.35).sort((a, b) => b - a);\n // with page markers on, content headings start below the page ### level;\n // sizes beyond the ###### budget share level 6 rather than being dropped\n const offset = opts.pageMarkers ? 3 : 0;\n const levelOf = new Map(headingSizes.slice(0, 6).map((h, i) => [h, Math.min(6, i + 1 + offset)]));\n\n const out: string[] = [];\n const blockPages = new Map<number, AnalysisBlock[]>();\n const stats: MarkdownStats = {\n headings: 0,\n listItems: 0,\n paragraphs: 0,\n joinedPairs: 0,\n reflowed: 0,\n callouts: 0,\n closedHyphens: 0,\n };\n let currentPage = 0;\n let para: { text: string; wordIndexes: number[] }[] = [];\n\n const addBlock = (block: AnalysisBlock) => {\n const blocks = blockPages.get(currentPage);\n if (blocks) blocks.push(block);\n else blockPages.set(currentPage, [block]);\n };\n\n // a label line ending in ':' whose next line is a bare number belongs on one line\n function attachValue(label: string, lines: Line[], i: number): { text: string; last: number } {\n const j = i + 1;\n if (/[::]\\s*$/.test(label) && j < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[j].text)) {\n stats.joinedPairs++;\n return { text: `${label} ${lines[j].text}`, last: j };\n }\n return { text: label, last: i };\n }\n const flush = () => {\n if (!para.length) return;\n const text = para\n .map((part) => part.text)\n .join(' ')\n .replace(/\\s+([,.;:%])/g, '$1');\n out.push(text);\n out.push('');\n addBlock({ type: 'paragraph', text, wordIndexes: para.flatMap((part) => part.wordIndexes) });\n stats.paragraphs++;\n para = [];\n };\n // append a wrapped line to the open paragraph, resolving a line-final hyphen\n const appendToPara = (line: Line, text: string) => {\n const prev = para.at(-1);\n if (prev && /\\w-$/.test(prev.text)) {\n // a bare \" -\" is punctuation, not a wrapped word\n const head = prev.text.replace(/-$/, '').split(/\\s/).pop() as string;\n const tail = (text.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0];\n const close = getDict().has((head + tail).toLowerCase());\n if (close) stats.closedHyphens++;\n para[para.length - 1].text = prev.text.replace(/-$/, close ? '' : '-') + text;\n para[para.length - 1].wordIndexes.push(...line.wordIndexes);\n } else para.push({ text, wordIndexes: [...line.wordIndexes] });\n };\n\n for (const { page, lines } of pageLines) {\n currentPage = page;\n if (opts.pageMarkers) {\n flush();\n out.push(pageMarkerLine(page));\n out.push('');\n }\n if (!lines.length) continue;\n const indentBase = Math.min(...lines.map((l) => l.x));\n\n for (let i = 0; i < lines.length; i++) {\n let { text, h, x } = lines[i];\n const prevLine = i > 0 ? lines[i - 1] : null;\n // a column change or a vertical jump wider than a blank line ends the paragraph\n if (para.length && prevLine && (lines[i].col !== prevLine.col || lines[i].y - prevLine.y > h * 2.2)) flush();\n\n // A large-font line opens a run: gather the consecutive same-size lines\n // below it (a title wraps across lines). A short merged run is one heading; a long one is a callout/pull-quote in a bigger face — prose, not structure — and becomes a blockquote.\n if (levelOf.has(h) && !BULLETS.test(text)) {\n const run = [lines[i]];\n while (i + 1 < lines.length && lines[i + 1].h === h && lines[i + 1].col === lines[i].col && lines[i + 1].y - lines[i].y < h * 2.2 && !BULLETS.test(lines[i + 1].text)) run.push(lines[++i]);\n // join wrapped title lines, resolving line-final hyphens like body text\n const merged = run\n .map((line) => line.text)\n .reduce((acc, part) => (/\\w-$/.test(acc) ? acc.replace(/-$/, getDict().has(((acc.replace(/-$/, '').split(/\\s/).pop() as string) + (part.split(/\\s/)[0] ?? '').replace(/[^A-Za-z-]/g, '').split('-')[0]).toLowerCase()) ? '' : '-') + part : `${acc} ${part}`))\n .replace(/\\s+([,.;:%])/g, '$1');\n const wordIndexes = run.flatMap((line) => line.wordIndexes);\n flush();\n if (merged.length > 150) {\n out.push(`> ${merged}`);\n out.push('');\n addBlock({ type: 'paragraph', text: `> ${merged}`, wordIndexes });\n stats.callouts++;\n } else {\n const level = levelOf.get(h) as number;\n out.push(`${'#'.repeat(level)} ${merged}`);\n out.push('');\n addBlock({ type: 'heading', text: merged, level, wordIndexes });\n stats.headings++;\n }\n continue;\n }\n\n // Lines can carry several stacked glyphs (\"• ▪\") from nested source lists.\n const stripped = text.replace(BULLETS, '').trim();\n const wasBullet = stripped !== text;\n if (wasBullet) text = stripped;\n\n // bullet with nothing left on the line: attach the following line's text\n if ((!text || GLYPH.test(text)) && i + 1 < lines.length) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n const itemLines = [lines[i], lines[++i]];\n let label = lines[i].text.replace(BULLETS, '').trim();\n while (!label && i + 1 < lines.length) {\n // stacked glyphs\n itemLines.push(lines[++i]);\n label = lines[i].text.replace(BULLETS, '').trim();\n }\n const attached = attachValue(label, lines, i);\n if (attached.last !== i) itemLines.push(lines[attached.last]);\n i = attached.last;\n const itemText = `${' '.repeat(depth)}- ${attached.text}`;\n out.push(itemText);\n addBlock({ type: 'paragraph', text: itemText, tight: true, wordIndexes: itemLines.flatMap((line) => line.wordIndexes) });\n stats.listItems++;\n continue;\n }\n // glyph(s) followed by text on the same line\n if (wasBullet) {\n flush();\n const depth = Math.max(0, Math.round((x - indentBase) / 18));\n const itemLines = [lines[i]];\n const attached = attachValue(text, lines, i);\n if (attached.last !== i) itemLines.push(lines[attached.last]);\n i = attached.last;\n const itemText = `${' '.repeat(depth)}- ${attached.text}`;\n out.push(itemText);\n addBlock({ type: 'paragraph', text: itemText, tight: true, wordIndexes: itemLines.flatMap((line) => line.wordIndexes) });\n stats.listItems++;\n continue;\n }\n // `label:` on its own line followed by a bare value -> keep together\n if (/[::]\\s*$/.test(text) && i + 1 < lines.length && /^[0-9][\\d.,]*%?$/.test(lines[i + 1].text)) {\n flush();\n const itemText = `- ${text} ${lines[++i].text}`;\n out.push(itemText);\n addBlock({ type: 'paragraph', text: itemText, tight: true, wordIndexes: [...lines[i - 1].wordIndexes, ...lines[i].wordIndexes] });\n stats.joinedPairs++;\n continue;\n }\n // otherwise body text: re-flow wrapped lines into a paragraph\n appendToPara(lines[i], text);\n if (/[.!?:]$/.test(text)) flush();\n else stats.reflowed++;\n }\n flush();\n }\n flush();\n\n const md = `${out\n .join('\\n')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()}\\n`;\n const pages: MarkdownPage[] = pageLines.map(({ page }) => ({ page, blocks: blockPages.get(page) ?? [] }));\n return { md, stats, bodyH, headingSizes, pages };\n}\n"],"names":["getDict","pageMarkerLine","GLYPH","BULLETS","buildMarkdown","pageLines","heightChars","opts","bodyH","entries","sort","a","b","headingSizes","keys","filter","h","offset","pageMarkers","levelOf","Map","slice","map","i","Math","min","out","blockPages","stats","headings","listItems","paragraphs","joinedPairs","reflowed","callouts","closedHyphens","currentPage","para","addBlock","block","blocks","get","push","set","attachValue","label","lines","j","test","length","text","last","flush","part","join","replace","type","wordIndexes","flatMap","appendToPara","line","prev","at","head","split","pop","tail","close","has","toLowerCase","page","indentBase","l","x","prevLine","col","y","run","merged","reduce","acc","level","repeat","stripped","trim","wasBullet","depth","max","round","itemLines","attached","itemText","tight","md","pages"],"mappings":"AAAA,0EAA0E;AAC1E,qIAAqI;AACrI,SAASA,OAAO,QAAQ,YAAY;AACpC,SAASC,cAAc,QAAQ,mBAAmB;AAGlD,MAAMC,QAAQ;AACd,MAAMC,UAAU;AAEhB,OAAO,SAASC,cAAcC,SAAsB,EAAEC,WAAgC,EAAEC,OAA6B,CAAC,CAAC;;QAGvG;IAFd,0EAA0E;IAC1E,qEAAqE;IACrE,MAAMC,iBAAQ,SAAA;WAAIF,YAAYG,OAAO;KAAG,CAACC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,cAAzD,6BAAA,MAA2D,CAAC,EAAE,uCAAI;IAChF,MAAME,eAAe;WAAIP,YAAYQ,IAAI;KAAG,CAACC,MAAM,CAAC,CAACC,IAAMA,IAAIR,QAAQ,MAAME,IAAI,CAAC,CAACC,GAAGC,IAAMA,IAAID;IAChG,yEAAyE;IACzE,yEAAyE;IACzE,MAAMM,SAASV,KAAKW,WAAW,GAAG,IAAI;IACtC,MAAMC,UAAU,IAAIC,IAAIP,aAAaQ,KAAK,CAAC,GAAG,GAAGC,GAAG,CAAC,CAACN,GAAGO,IAAM;YAACP;YAAGQ,KAAKC,GAAG,CAAC,GAAGF,IAAI,IAAIN;SAAQ;IAE/F,MAAMS,MAAgB,EAAE;IACxB,MAAMC,aAAa,IAAIP;IACvB,MAAMQ,QAAuB;QAC3BC,UAAU;QACVC,WAAW;QACXC,YAAY;QACZC,aAAa;QACbC,UAAU;QACVC,UAAU;QACVC,eAAe;IACjB;IACA,IAAIC,cAAc;IAClB,IAAIC,OAAkD,EAAE;IAExD,MAAMC,WAAW,CAACC;QAChB,MAAMC,SAASb,WAAWc,GAAG,CAACL;QAC9B,IAAII,QAAQA,OAAOE,IAAI,CAACH;aACnBZ,WAAWgB,GAAG,CAACP,aAAa;YAACG;SAAM;IAC1C;IAEA,kFAAkF;IAClF,SAASK,YAAYC,KAAa,EAAEC,KAAa,EAAEvB,CAAS;QAC1D,MAAMwB,IAAIxB,IAAI;QACd,IAAI,WAAWyB,IAAI,CAACH,UAAUE,IAAID,MAAMG,MAAM,IAAI,mBAAmBD,IAAI,CAACF,KAAK,CAACC,EAAE,CAACG,IAAI,GAAG;YACxFtB,MAAMI,WAAW;YACjB,OAAO;gBAAEkB,MAAM,GAAGL,MAAM,CAAC,EAAEC,KAAK,CAACC,EAAE,CAACG,IAAI,EAAE;gBAAEC,MAAMJ;YAAE;QACtD;QACA,OAAO;YAAEG,MAAML;YAAOM,MAAM5B;QAAE;IAChC;IACA,MAAM6B,QAAQ;QACZ,IAAI,CAACf,KAAKY,MAAM,EAAE;QAClB,MAAMC,OAAOb,KACVf,GAAG,CAAC,CAAC+B,OAASA,KAAKH,IAAI,EACvBI,IAAI,CAAC,KACLC,OAAO,CAAC,iBAAiB;QAC5B7B,IAAIgB,IAAI,CAACQ;QACTxB,IAAIgB,IAAI,CAAC;QACTJ,SAAS;YAAEkB,MAAM;YAAaN;YAAMO,aAAapB,KAAKqB,OAAO,CAAC,CAACL,OAASA,KAAKI,WAAW;QAAE;QAC1F7B,MAAMG,UAAU;QAChBM,OAAO,EAAE;IACX;IACA,6EAA6E;IAC7E,MAAMsB,eAAe,CAACC,MAAYV;QAChC,MAAMW,OAAOxB,KAAKyB,EAAE,CAAC,CAAC;QACtB,IAAID,QAAQ,OAAOb,IAAI,CAACa,KAAKX,IAAI,GAAG;gBAGpBA;YAFd,iDAAiD;YACjD,MAAMa,OAAOF,KAAKX,IAAI,CAACK,OAAO,CAAC,MAAM,IAAIS,KAAK,CAAC,MAAMC,GAAG;YACxD,MAAMC,OAAO,EAAChB,eAAAA,KAAKc,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBd,0BAAAA,eAAuB,IAAIK,OAAO,CAAC,eAAe,IAAIS,KAAK,CAAC,IAAI,CAAC,EAAE;YACjF,MAAMG,QAAQnE,UAAUoE,GAAG,CAAC,AAACL,CAAAA,OAAOG,IAAG,EAAGG,WAAW;YACrD,IAAIF,OAAOvC,MAAMO,aAAa;YAC9BE,IAAI,CAACA,KAAKY,MAAM,GAAG,EAAE,CAACC,IAAI,GAAGW,KAAKX,IAAI,CAACK,OAAO,CAAC,MAAMY,QAAQ,KAAK,OAAOjB;YACzEb,IAAI,CAACA,KAAKY,MAAM,GAAG,EAAE,CAACQ,WAAW,CAACf,IAAI,IAAIkB,KAAKH,WAAW;QAC5D,OAAOpB,KAAKK,IAAI,CAAC;YAAEQ;YAAMO,aAAa;mBAAIG,KAAKH,WAAW;aAAC;QAAC;IAC9D;IAEA,KAAK,MAAM,EAAEa,IAAI,EAAExB,KAAK,EAAE,IAAIzC,UAAW;QACvC+B,cAAckC;QACd,IAAI/D,KAAKW,WAAW,EAAE;YACpBkC;YACA1B,IAAIgB,IAAI,CAACzC,eAAeqE;YACxB5C,IAAIgB,IAAI,CAAC;QACX;QACA,IAAI,CAACI,MAAMG,MAAM,EAAE;QACnB,MAAMsB,aAAa/C,KAAKC,GAAG,IAAIqB,MAAMxB,GAAG,CAAC,CAACkD,IAAMA,EAAEC,CAAC;QAEnD,IAAK,IAAIlD,IAAI,GAAGA,IAAIuB,MAAMG,MAAM,EAAE1B,IAAK;YACrC,IAAI,EAAE2B,IAAI,EAAElC,CAAC,EAAEyD,CAAC,EAAE,GAAG3B,KAAK,CAACvB,EAAE;YAC7B,MAAMmD,WAAWnD,IAAI,IAAIuB,KAAK,CAACvB,IAAI,EAAE,GAAG;YACxC,gFAAgF;YAChF,IAAIc,KAAKY,MAAM,IAAIyB,YAAa5B,CAAAA,KAAK,CAACvB,EAAE,CAACoD,GAAG,KAAKD,SAASC,GAAG,IAAI7B,KAAK,CAACvB,EAAE,CAACqD,CAAC,GAAGF,SAASE,CAAC,GAAG5D,IAAI,GAAE,GAAIoC;YAErG,wEAAwE;YACxE,mLAAmL;YACnL,IAAIjC,QAAQiD,GAAG,CAACpD,MAAM,CAACb,QAAQ6C,IAAI,CAACE,OAAO;gBACzC,MAAM2B,MAAM;oBAAC/B,KAAK,CAACvB,EAAE;iBAAC;gBACtB,MAAOA,IAAI,IAAIuB,MAAMG,MAAM,IAAIH,KAAK,CAACvB,IAAI,EAAE,CAACP,CAAC,KAAKA,KAAK8B,KAAK,CAACvB,IAAI,EAAE,CAACoD,GAAG,KAAK7B,KAAK,CAACvB,EAAE,CAACoD,GAAG,IAAI7B,KAAK,CAACvB,IAAI,EAAE,CAACqD,CAAC,GAAG9B,KAAK,CAACvB,EAAE,CAACqD,CAAC,GAAG5D,IAAI,OAAO,CAACb,QAAQ6C,IAAI,CAACF,KAAK,CAACvB,IAAI,EAAE,CAAC2B,IAAI,EAAG2B,IAAInC,IAAI,CAACI,KAAK,CAAC,EAAEvB,EAAE;gBAC1L,wEAAwE;gBACxE,MAAMuD,SAASD,IACZvD,GAAG,CAAC,CAACsC,OAASA,KAAKV,IAAI,EACvB6B,MAAM,CAAC,CAACC,KAAK3B;wBAAqHA;2BAA3G,OAAOL,IAAI,CAACgC,OAAOA,IAAIzB,OAAO,CAAC,MAAMvD,UAAUoE,GAAG,CAAC,AAAC,CAAA,AAACY,IAAIzB,OAAO,CAAC,MAAM,IAAIS,KAAK,CAAC,MAAMC,GAAG,KAAgB,EAACZ,eAAAA,KAAKW,KAAK,CAAC,KAAK,CAAC,EAAE,cAAnBX,0BAAAA,eAAuB,IAAIE,OAAO,CAAC,eAAe,IAAIS,KAAK,CAAC,IAAI,CAAC,EAAE,AAAD,EAAGK,WAAW,MAAM,KAAK,OAAOhB,OAAO,GAAG2B,IAAI,CAAC,EAAE3B,MAAM;mBAC3PE,OAAO,CAAC,iBAAiB;gBAC5B,MAAME,cAAcoB,IAAInB,OAAO,CAAC,CAACE,OAASA,KAAKH,WAAW;gBAC1DL;gBACA,IAAI0B,OAAO7B,MAAM,GAAG,KAAK;oBACvBvB,IAAIgB,IAAI,CAAC,CAAC,EAAE,EAAEoC,QAAQ;oBACtBpD,IAAIgB,IAAI,CAAC;oBACTJ,SAAS;wBAAEkB,MAAM;wBAAaN,MAAM,CAAC,EAAE,EAAE4B,QAAQ;wBAAErB;oBAAY;oBAC/D7B,MAAMM,QAAQ;gBAChB,OAAO;oBACL,MAAM+C,QAAQ9D,QAAQsB,GAAG,CAACzB;oBAC1BU,IAAIgB,IAAI,CAAC,GAAG,IAAIwC,MAAM,CAACD,OAAO,CAAC,EAAEH,QAAQ;oBACzCpD,IAAIgB,IAAI,CAAC;oBACTJ,SAAS;wBAAEkB,MAAM;wBAAWN,MAAM4B;wBAAQG;wBAAOxB;oBAAY;oBAC7D7B,MAAMC,QAAQ;gBAChB;gBACA;YACF;YAEA,2EAA2E;YAC3E,MAAMsD,WAAWjC,KAAKK,OAAO,CAACpD,SAAS,IAAIiF,IAAI;YAC/C,MAAMC,YAAYF,aAAajC;YAC/B,IAAImC,WAAWnC,OAAOiC;YAEtB,yEAAyE;YACzE,IAAI,AAAC,CAAA,CAACjC,QAAQhD,MAAM8C,IAAI,CAACE,KAAI,KAAM3B,IAAI,IAAIuB,MAAMG,MAAM,EAAE;gBACvDG;gBACA,MAAMkC,QAAQ9D,KAAK+D,GAAG,CAAC,GAAG/D,KAAKgE,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxD,MAAMkB,YAAY;oBAAC3C,KAAK,CAACvB,EAAE;oBAAEuB,KAAK,CAAC,EAAEvB,EAAE;iBAAC;gBACxC,IAAIsB,QAAQC,KAAK,CAACvB,EAAE,CAAC2B,IAAI,CAACK,OAAO,CAACpD,SAAS,IAAIiF,IAAI;gBACnD,MAAO,CAACvC,SAAStB,IAAI,IAAIuB,MAAMG,MAAM,CAAE;oBACrC,iBAAiB;oBACjBwC,UAAU/C,IAAI,CAACI,KAAK,CAAC,EAAEvB,EAAE;oBACzBsB,QAAQC,KAAK,CAACvB,EAAE,CAAC2B,IAAI,CAACK,OAAO,CAACpD,SAAS,IAAIiF,IAAI;gBACjD;gBACA,MAAMM,WAAW9C,YAAYC,OAAOC,OAAOvB;gBAC3C,IAAImE,SAASvC,IAAI,KAAK5B,GAAGkE,UAAU/C,IAAI,CAACI,KAAK,CAAC4C,SAASvC,IAAI,CAAC;gBAC5D5B,IAAImE,SAASvC,IAAI;gBACjB,MAAMwC,WAAW,GAAG,KAAKT,MAAM,CAACI,OAAO,EAAE,EAAEI,SAASxC,IAAI,EAAE;gBAC1DxB,IAAIgB,IAAI,CAACiD;gBACTrD,SAAS;oBAAEkB,MAAM;oBAAaN,MAAMyC;oBAAUC,OAAO;oBAAMnC,aAAagC,UAAU/B,OAAO,CAAC,CAACE,OAASA,KAAKH,WAAW;gBAAE;gBACtH7B,MAAME,SAAS;gBACf;YACF;YACA,6CAA6C;YAC7C,IAAIuD,WAAW;gBACbjC;gBACA,MAAMkC,QAAQ9D,KAAK+D,GAAG,CAAC,GAAG/D,KAAKgE,KAAK,CAAC,AAACf,CAAAA,IAAIF,UAAS,IAAK;gBACxD,MAAMkB,YAAY;oBAAC3C,KAAK,CAACvB,EAAE;iBAAC;gBAC5B,MAAMmE,WAAW9C,YAAYM,MAAMJ,OAAOvB;gBAC1C,IAAImE,SAASvC,IAAI,KAAK5B,GAAGkE,UAAU/C,IAAI,CAACI,KAAK,CAAC4C,SAASvC,IAAI,CAAC;gBAC5D5B,IAAImE,SAASvC,IAAI;gBACjB,MAAMwC,WAAW,GAAG,KAAKT,MAAM,CAACI,OAAO,EAAE,EAAEI,SAASxC,IAAI,EAAE;gBAC1DxB,IAAIgB,IAAI,CAACiD;gBACTrD,SAAS;oBAAEkB,MAAM;oBAAaN,MAAMyC;oBAAUC,OAAO;oBAAMnC,aAAagC,UAAU/B,OAAO,CAAC,CAACE,OAASA,KAAKH,WAAW;gBAAE;gBACtH7B,MAAME,SAAS;gBACf;YACF;YACA,qEAAqE;YACrE,IAAI,WAAWkB,IAAI,CAACE,SAAS3B,IAAI,IAAIuB,MAAMG,MAAM,IAAI,mBAAmBD,IAAI,CAACF,KAAK,CAACvB,IAAI,EAAE,CAAC2B,IAAI,GAAG;gBAC/FE;gBACA,MAAMuC,WAAW,CAAC,EAAE,EAAEzC,KAAK,CAAC,EAAEJ,KAAK,CAAC,EAAEvB,EAAE,CAAC2B,IAAI,EAAE;gBAC/CxB,IAAIgB,IAAI,CAACiD;gBACTrD,SAAS;oBAAEkB,MAAM;oBAAaN,MAAMyC;oBAAUC,OAAO;oBAAMnC,aAAa;2BAAIX,KAAK,CAACvB,IAAI,EAAE,CAACkC,WAAW;2BAAKX,KAAK,CAACvB,EAAE,CAACkC,WAAW;qBAAC;gBAAC;gBAC/H7B,MAAMI,WAAW;gBACjB;YACF;YACA,8DAA8D;YAC9D2B,aAAab,KAAK,CAACvB,EAAE,EAAE2B;YACvB,IAAI,UAAUF,IAAI,CAACE,OAAOE;iBACrBxB,MAAMK,QAAQ;QACrB;QACAmB;IACF;IACAA;IAEA,MAAMyC,KAAK,GAAGnE,IACX4B,IAAI,CAAC,MACLC,OAAO,CAAC,WAAW,QACnB6B,IAAI,GAAG,EAAE,CAAC;IACb,MAAMU,QAAwBzF,UAAUiB,GAAG,CAAC,CAAC,EAAEgD,IAAI,EAAE;YAAsB3C;eAAhB;YAAE2C;YAAM9B,MAAM,GAAEb,kBAAAA,WAAWc,GAAG,CAAC6B,mBAAf3C,6BAAAA,kBAAwB,EAAE;QAAC;;IACtG,OAAO;QAAEkE;QAAIjE;QAAOpB;QAAOK;QAAciF;IAAM;AACjD"}
@@ -12,5 +12,5 @@ export declare function splitByPageMarker(md: string): {
12
12
  /** Per-page body text, duplicate-marker slices CONCATENATED — a word anywhere
13
13
  * on page N counts as on page N (the auditor's membership semantics). */
14
14
  export declare function bodyByPage(md: string): Map<number, string>;
15
- /** Count of "### pN" marker lines in `md`. */
15
+ /** Count of visible or invisible page-provenance marker lines in `md`. */
16
16
  export declare function countPageMarkers(md: string): number;
@@ -1,20 +1,24 @@
1
- // page-marker.ts — the single "### pN" convention: every producer/parser of
2
- // the page-marker heading imports from here so the regex can't drift (report.ts once had a `\s+` variant).
1
+ // page-marker.ts — page provenance shared by Markdown producers and readers.
3
2
  /** The marker heading for page N, with no surrounding whitespace/newlines
4
3
  * (callers own their own blank-line formatting). */ export function pageMarkerLine(page) {
5
4
  return `### p${page}`;
6
5
  }
7
- // Single space, matching pageMarkerLine's own output exactly.
8
- const MARKER_RE = /^### p(\d+)$/;
6
+ // Visible headings are emitted by this package; invisible comments preserve
7
+ // page provenance when a caller does not want headings in its final Markdown.
8
+ const MARKER_RE_SOURCE = '^(?:### p(\\d+)|<!-- p(\\d+) -->)\\r?$';
9
+ function markerPage(match) {
10
+ var _match_;
11
+ return Number((_match_ = match[1]) !== null && _match_ !== void 0 ? _match_ : match[2]);
12
+ }
9
13
  /** Is this line (already split, no trailing newline) a page-marker heading? */ export function isPageMarkerLine(line) {
10
- return MARKER_RE.test(line);
14
+ return new RegExp(MARKER_RE_SOURCE, 'u').test(line);
11
15
  }
12
16
  /** Split markdown into per-marker body slices, in document order — one entry
13
17
  * PER OCCURRENCE (a duplicate "### pN" yields two entries; no slice is ever dropped). */ export function splitByPageMarker(md) {
14
- const re = /^### p(\d+)$/gm;
18
+ const re = new RegExp(MARKER_RE_SOURCE, 'gmu');
15
19
  const markers = [];
16
20
  for(let m = re.exec(md); m !== null; m = re.exec(md))markers.push({
17
- page: Number(m[1]),
21
+ page: markerPage(m),
18
22
  index: m.index,
19
23
  end: m.index + m[0].length
20
24
  });
@@ -35,6 +39,8 @@ const MARKER_RE = /^### p(\d+)$/;
35
39
  for (const { page, body } of splitByPageMarker(md))out.set(page, ((_out_get = out.get(page)) !== null && _out_get !== void 0 ? _out_get : '') + body);
36
40
  return out;
37
41
  }
38
- /** Count of "### pN" marker lines in `md`. */ export function countPageMarkers(md) {
39
- return (md.match(/^### p\d+$/gm) || []).length;
42
+ /** Count of visible or invisible page-provenance marker lines in `md`. */ export function countPageMarkers(md) {
43
+ return [
44
+ ...md.matchAll(new RegExp(MARKER_RE_SOURCE, 'gmu'))
45
+ ].length;
40
46
  }
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/page-marker.ts"],"sourcesContent":["// page-marker.ts — the single \"### pN\" convention: every producer/parser of\n// the page-marker heading imports from here so the regex can't drift (report.ts once had a `\\s+` variant).\n\n/** The marker heading for page N, with no surrounding whitespace/newlines\n * (callers own their own blank-line formatting). */\nexport function pageMarkerLine(page: number): string {\n return `### p${page}`;\n}\n\n// Single space, matching pageMarkerLine's own output exactly.\nconst MARKER_RE = /^### p(\\d+)$/;\n\n/** Is this line (already split, no trailing newline) a page-marker heading? */\nexport function isPageMarkerLine(line: string): boolean {\n return MARKER_RE.test(line);\n}\n\n/** Split markdown into per-marker body slices, in document order — one entry\n * PER OCCURRENCE (a duplicate \"### pN\" yields two entries; no slice is ever dropped). */\nexport function splitByPageMarker(md: string): { page: number; body: string }[] {\n const re = /^### p(\\d+)$/gm;\n const markers: { page: number; index: number; end: number }[] = [];\n for (let m = re.exec(md); m !== null; m = re.exec(md)) markers.push({ page: Number(m[1]), index: m.index, end: m.index + m[0].length });\n const out: { page: number; body: string }[] = [];\n for (let i = 0; i < markers.length; i++) {\n const end = i + 1 < markers.length ? markers[i + 1].index : md.length;\n out.push({ page: markers[i].page, body: md.slice(markers[i].end, end) });\n }\n return out;\n}\n\n/** Per-page body text, duplicate-marker slices CONCATENATED — a word anywhere\n * on page N counts as on page N (the auditor's membership semantics). */\nexport function bodyByPage(md: string): Map<number, string> {\n const out = new Map<number, string>();\n for (const { page, body } of splitByPageMarker(md)) out.set(page, (out.get(page) ?? '') + body);\n return out;\n}\n\n/** Count of \"### pN\" marker lines in `md`. */\nexport function countPageMarkers(md: string): number {\n return (md.match(/^### p\\d+$/gm) || []).length;\n}\n"],"names":["pageMarkerLine","page","MARKER_RE","isPageMarkerLine","line","test","splitByPageMarker","md","re","markers","m","exec","push","Number","index","end","length","out","i","body","slice","bodyByPage","Map","set","get","countPageMarkers","match"],"mappings":"AAAA,4EAA4E;AAC5E,2GAA2G;AAE3G;kDACkD,GAClD,OAAO,SAASA,eAAeC,IAAY;IACzC,OAAO,CAAC,KAAK,EAAEA,MAAM;AACvB;AAEA,8DAA8D;AAC9D,MAAMC,YAAY;AAElB,6EAA6E,GAC7E,OAAO,SAASC,iBAAiBC,IAAY;IAC3C,OAAOF,UAAUG,IAAI,CAACD;AACxB;AAEA;uFACuF,GACvF,OAAO,SAASE,kBAAkBC,EAAU;IAC1C,MAAMC,KAAK;IACX,MAAMC,UAA0D,EAAE;IAClE,IAAK,IAAIC,IAAIF,GAAGG,IAAI,CAACJ,KAAKG,MAAM,MAAMA,IAAIF,GAAGG,IAAI,CAACJ,IAAKE,QAAQG,IAAI,CAAC;QAAEX,MAAMY,OAAOH,CAAC,CAAC,EAAE;QAAGI,OAAOJ,EAAEI,KAAK;QAAEC,KAAKL,EAAEI,KAAK,GAAGJ,CAAC,CAAC,EAAE,CAACM,MAAM;IAAC;IACrI,MAAMC,MAAwC,EAAE;IAChD,IAAK,IAAIC,IAAI,GAAGA,IAAIT,QAAQO,MAAM,EAAEE,IAAK;QACvC,MAAMH,MAAMG,IAAI,IAAIT,QAAQO,MAAM,GAAGP,OAAO,CAACS,IAAI,EAAE,CAACJ,KAAK,GAAGP,GAAGS,MAAM;QACrEC,IAAIL,IAAI,CAAC;YAAEX,MAAMQ,OAAO,CAACS,EAAE,CAACjB,IAAI;YAAEkB,MAAMZ,GAAGa,KAAK,CAACX,OAAO,CAACS,EAAE,CAACH,GAAG,EAAEA;QAAK;IACxE;IACA,OAAOE;AACT;AAEA;uEACuE,GACvE,OAAO,SAASI,WAAWd,EAAU;QAEgCU;IADnE,MAAMA,MAAM,IAAIK;IAChB,KAAK,MAAM,EAAErB,IAAI,EAAEkB,IAAI,EAAE,IAAIb,kBAAkBC,IAAKU,IAAIM,GAAG,CAACtB,MAAM,EAACgB,WAAAA,IAAIO,GAAG,CAACvB,mBAARgB,sBAAAA,WAAiB,MAAME;IAC1F,OAAOF;AACT;AAEA,4CAA4C,GAC5C,OAAO,SAASQ,iBAAiBlB,EAAU;IACzC,OAAO,AAACA,CAAAA,GAAGmB,KAAK,CAAC,mBAAmB,EAAE,AAAD,EAAGV,MAAM;AAChD"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/page-marker.ts"],"sourcesContent":["// page-marker.ts — page provenance shared by Markdown producers and readers.\n\n/** The marker heading for page N, with no surrounding whitespace/newlines\n * (callers own their own blank-line formatting). */\nexport function pageMarkerLine(page: number): string {\n return `### p${page}`;\n}\n\n// Visible headings are emitted by this package; invisible comments preserve\n// page provenance when a caller does not want headings in its final Markdown.\nconst MARKER_RE_SOURCE = '^(?:### p(\\\\d+)|<!-- p(\\\\d+) -->)\\\\r?$';\n\nfunction markerPage(match: RegExpExecArray): number {\n return Number(match[1] ?? match[2]);\n}\n\n/** Is this line (already split, no trailing newline) a page-marker heading? */\nexport function isPageMarkerLine(line: string): boolean {\n return new RegExp(MARKER_RE_SOURCE, 'u').test(line);\n}\n\n/** Split markdown into per-marker body slices, in document order — one entry\n * PER OCCURRENCE (a duplicate \"### pN\" yields two entries; no slice is ever dropped). */\nexport function splitByPageMarker(md: string): { page: number; body: string }[] {\n const re = new RegExp(MARKER_RE_SOURCE, 'gmu');\n const markers: { page: number; index: number; end: number }[] = [];\n for (let m = re.exec(md); m !== null; m = re.exec(md)) markers.push({ page: markerPage(m), index: m.index, end: m.index + m[0].length });\n const out: { page: number; body: string }[] = [];\n for (let i = 0; i < markers.length; i++) {\n const end = i + 1 < markers.length ? markers[i + 1].index : md.length;\n out.push({ page: markers[i].page, body: md.slice(markers[i].end, end) });\n }\n return out;\n}\n\n/** Per-page body text, duplicate-marker slices CONCATENATED — a word anywhere\n * on page N counts as on page N (the auditor's membership semantics). */\nexport function bodyByPage(md: string): Map<number, string> {\n const out = new Map<number, string>();\n for (const { page, body } of splitByPageMarker(md)) out.set(page, (out.get(page) ?? '') + body);\n return out;\n}\n\n/** Count of visible or invisible page-provenance marker lines in `md`. */\nexport function countPageMarkers(md: string): number {\n return [...md.matchAll(new RegExp(MARKER_RE_SOURCE, 'gmu'))].length;\n}\n"],"names":["pageMarkerLine","page","MARKER_RE_SOURCE","markerPage","match","Number","isPageMarkerLine","line","RegExp","test","splitByPageMarker","md","re","markers","m","exec","push","index","end","length","out","i","body","slice","bodyByPage","Map","set","get","countPageMarkers","matchAll"],"mappings":"AAAA,6EAA6E;AAE7E;kDACkD,GAClD,OAAO,SAASA,eAAeC,IAAY;IACzC,OAAO,CAAC,KAAK,EAAEA,MAAM;AACvB;AAEA,4EAA4E;AAC5E,8EAA8E;AAC9E,MAAMC,mBAAmB;AAEzB,SAASC,WAAWC,KAAsB;QAC1BA;IAAd,OAAOC,QAAOD,UAAAA,KAAK,CAAC,EAAE,cAARA,qBAAAA,UAAYA,KAAK,CAAC,EAAE;AACpC;AAEA,6EAA6E,GAC7E,OAAO,SAASE,iBAAiBC,IAAY;IAC3C,OAAO,IAAIC,OAAON,kBAAkB,KAAKO,IAAI,CAACF;AAChD;AAEA;uFACuF,GACvF,OAAO,SAASG,kBAAkBC,EAAU;IAC1C,MAAMC,KAAK,IAAIJ,OAAON,kBAAkB;IACxC,MAAMW,UAA0D,EAAE;IAClE,IAAK,IAAIC,IAAIF,GAAGG,IAAI,CAACJ,KAAKG,MAAM,MAAMA,IAAIF,GAAGG,IAAI,CAACJ,IAAKE,QAAQG,IAAI,CAAC;QAAEf,MAAME,WAAWW;QAAIG,OAAOH,EAAEG,KAAK;QAAEC,KAAKJ,EAAEG,KAAK,GAAGH,CAAC,CAAC,EAAE,CAACK,MAAM;IAAC;IACtI,MAAMC,MAAwC,EAAE;IAChD,IAAK,IAAIC,IAAI,GAAGA,IAAIR,QAAQM,MAAM,EAAEE,IAAK;QACvC,MAAMH,MAAMG,IAAI,IAAIR,QAAQM,MAAM,GAAGN,OAAO,CAACQ,IAAI,EAAE,CAACJ,KAAK,GAAGN,GAAGQ,MAAM;QACrEC,IAAIJ,IAAI,CAAC;YAAEf,MAAMY,OAAO,CAACQ,EAAE,CAACpB,IAAI;YAAEqB,MAAMX,GAAGY,KAAK,CAACV,OAAO,CAACQ,EAAE,CAACH,GAAG,EAAEA;QAAK;IACxE;IACA,OAAOE;AACT;AAEA;uEACuE,GACvE,OAAO,SAASI,WAAWb,EAAU;QAEgCS;IADnE,MAAMA,MAAM,IAAIK;IAChB,KAAK,MAAM,EAAExB,IAAI,EAAEqB,IAAI,EAAE,IAAIZ,kBAAkBC,IAAKS,IAAIM,GAAG,CAACzB,MAAM,EAACmB,WAAAA,IAAIO,GAAG,CAAC1B,mBAARmB,sBAAAA,WAAiB,MAAME;IAC1F,OAAOF;AACT;AAEA,wEAAwE,GACxE,OAAO,SAASQ,iBAAiBjB,EAAU;IACzC,OAAO;WAAIA,GAAGkB,QAAQ,CAAC,IAAIrB,OAAON,kBAAkB;KAAQ,CAACiB,MAAM;AACrE"}
@@ -8,10 +8,12 @@ export declare function renderPagesToMarkdown(pages: AnalysisPage[], opts: {
8
8
  export declare function renderPagesToText(pages: AnalysisPage[]): string;
9
9
  /**
10
10
  * Render an Analysis to markdown. PURE and fail-fast over any well-formed Analysis (including hand-built/spliced ones) — throws `PdfToMdError`/`'ANALYSIS_INPUT'` before any output on a malformed one.
11
- * OCR-origin output is byte-identical to extractOcr's own markdown (shared core); text-origin is NOT byte-identical to extractText's (see `AnalysisPage.words`, `src/types.ts`).
11
+ * OCR-origin output is byte-identical to extractOcr's own markdown. Text
12
+ * output comes from the same captured blocks as extractText. Mixed output
13
+ * always includes page markers because adjacent pages have different producers.
12
14
  *
13
15
  * @param analysis the Analysis to render (from analyze(), or hand-built/spliced)
14
- * @param opts.pageMarkers text-origin only; an OCR-origin Analysis always carries markers
16
+ * @param opts.pageMarkers text-origin only; OCR and mixed Analysis values always carry markers
15
17
  */
16
18
  export declare function toMarkdown(analysis: Analysis, opts?: {
17
19
  pageMarkers?: boolean;