@effortlessmotion/pdf-to-md 0.1.8 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/render.js.map +1 -1
- package/dist/cjs/cli/types.js.map +1 -1
- package/dist/cjs/cli.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/dict.js.map +1 -1
- package/dist/cjs/emit.js.map +1 -1
- package/dist/cjs/engines/tesseract.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/geometry.js.map +1 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/pdf-open.js.map +1 -1
- package/dist/cjs/raster.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/resolve.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/cjs/validate.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/render.js.map +1 -1
- package/dist/esm/cli/types.js.map +1 -1
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/dict.js.map +1 -1
- package/dist/esm/emit.js.map +1 -1
- package/dist/esm/engines/tesseract.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/geometry.js.map +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/pdf-open.js.map +1 -1
- package/dist/esm/raster.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/resolve.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/dist/esm/validate.js.map +1 -1
- package/package.json +1 -1
- package/skills/pdf-to-md/SKILL.md +39 -66
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/page-marker.ts"],"sourcesContent":["// page-marker.ts — page provenance shared by Markdown producers and readers.\n\n/** The marker heading for page N, with no surrounding whitespace/newlines\n * (callers own their own blank-line formatting). */\nexport function pageMarkerLine(page: number): string {\n return `### p${page}`;\n}\n\n// Visible headings are emitted by this package; invisible comments preserve\n// page provenance when a caller does not want headings in its final Markdown.\nconst MARKER_RE_SOURCE = '^(?:### p(\\\\d+)|<!-- p(\\\\d+) -->)\\\\r?$';\n\nfunction markerPage(match: RegExpExecArray): number {\n return Number(match[1] ?? match[2]);\n}\n\n/** Is this line (already split, no trailing newline) a page-marker heading? */\nexport function isPageMarkerLine(line: string): boolean {\n return new RegExp(MARKER_RE_SOURCE, 'u').test(line);\n}\n\n/** Split markdown into per-marker body slices, in document order — one entry\n * PER OCCURRENCE (a duplicate \"### pN\" yields two entries; no slice is ever dropped). */\nexport function splitByPageMarker(md: string): { page: number; body: string }[] {\n const re = new RegExp(MARKER_RE_SOURCE, 'gmu');\n const markers: { page: number; index: number; end: number }[] = [];\n for (let m = re.exec(md); m !== null; m = re.exec(md)) markers.push({ page: markerPage(m), index: m.index, end: m.index + m[0].length });\n const out: { page: number; body: string }[] = [];\n for (let i = 0; i < markers.length; i++) {\n const end = i + 1 < markers.length ? markers[i + 1].index : md.length;\n out.push({ page: markers[i].page, body: md.slice(markers[i].end, end) });\n }\n return out;\n}\n\n/** Per-page body text, duplicate-marker slices CONCATENATED — a word anywhere\n * on page N counts as on page N (the auditor's membership semantics). */\nexport function bodyByPage(md: string): Map<number, string> {\n const out = new Map<number, string>();\n for (const { page, body } of splitByPageMarker(md)) out.set(page, (out.get(page) ?? '') + body);\n return out;\n}\n\n/** Count of visible or invisible page-provenance marker lines in `md`. */\nexport function countPageMarkers(md: string): number {\n return [...md.matchAll(new RegExp(MARKER_RE_SOURCE, 'gmu'))].length;\n}\n"],"names":["pageMarkerLine","page","MARKER_RE_SOURCE","markerPage","match","Number","isPageMarkerLine","line","RegExp","test","splitByPageMarker","md","re","markers","m","exec","push","index","end","length","out","i","body","slice","bodyByPage","Map","set","get","countPageMarkers","matchAll"],"mappings":"AAAA,6EAA6E;AAE7E;kDACkD,GAClD,OAAO,SAASA,eAAeC,IAAY;IACzC,OAAO,CAAC,KAAK,EAAEA,MAAM;AACvB;AAEA,4EAA4E;AAC5E,8EAA8E;AAC9E,MAAMC,mBAAmB;AAEzB,SAASC,WAAWC,KAAsB;QAC1BA;IAAd,OAAOC,QAAOD,UAAAA,KAAK,CAAC,EAAE,cAARA,qBAAAA,UAAYA,KAAK,CAAC,EAAE;AACpC;AAEA,6EAA6E,GAC7E,OAAO,SAASE,iBAAiBC,IAAY;IAC3C,OAAO,IAAIC,OAAON,kBAAkB,KAAKO,IAAI,CAACF;AAChD;AAEA;uFACuF,GACvF,OAAO,SAASG,kBAAkBC,EAAU;IAC1C,MAAMC,KAAK,IAAIJ,OAAON,kBAAkB;IACxC,MAAMW,UAA0D,EAAE;IAClE,IAAK,IAAIC,IAAIF,GAAGG,IAAI,CAACJ,KAAKG,MAAM,MAAMA,IAAIF,GAAGG,IAAI,CAACJ,IAAKE,QAAQG,IAAI,CAAC;QAAEf,MAAME,WAAWW;QAAIG,OAAOH,EAAEG,KAAK;QAAEC,KAAKJ,EAAEG,KAAK,GAAGH,CAAC,CAAC,EAAE,CAACK,MAAM;IAAC;IACtI,MAAMC,MAAwC,EAAE;IAChD,IAAK,IAAIC,IAAI,GAAGA,IAAIR,QAAQM,MAAM,EAAEE,IAAK;QACvC,MAAMH,MAAMG,IAAI,IAAIR,QAAQM,MAAM,GAAGN,OAAO,CAACQ,IAAI,EAAE,CAACJ,KAAK,GAAGN,GAAGQ,MAAM;QACrEC,IAAIJ,IAAI,CAAC;YAAEf,MAAMY,OAAO,CAACQ,EAAE,CAACpB,IAAI;YAAEqB,MAAMX,GAAGY,KAAK,CAACV,OAAO,CAACQ,EAAE,CAACH,GAAG,EAAEA;QAAK;IACxE;IACA,OAAOE;AACT;AAEA;uEACuE,GACvE,OAAO,SAASI,WAAWb,EAAU;QAEgCS;IADnE,MAAMA,MAAM,IAAIK;IAChB,KAAK,MAAM,EAAExB,IAAI,EAAEqB,IAAI,EAAE,IAAIZ,kBAAkBC,IAAKS,IAAIM,GAAG,CAACzB,MAAM,EAACmB,WAAAA,IAAIO,GAAG,CAAC1B,mBAARmB,sBAAAA,WAAiB,MAAME;IAC1F,OAAOF;AACT;AAEA,wEAAwE,GACxE,OAAO,SAASQ,iBAAiBjB,EAAU;IACzC,OAAO;WAAIA,GAAGkB,QAAQ,CAAC,IAAIrB,OAAON,kBAAkB;KAAQ,CAACiB,MAAM;AACrE"}
|
package/dist/esm/pdf-open.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/pdf-open.ts"],"sourcesContent":["// pdf-open.ts — the one seam that opens a PDF via pdfjs-dist, in the two\n// shapes callers need: openDocument (data-based, already-open doc) and openPdfForRender (url-based, task you must await + destroy()).\n// Both wrap a failure the same way: PdfToMdError/PDF_OPEN, original message preserved.\nimport { readFileSync } from 'node:fs';\nimport path from 'node:path';\nimport type { PDFDocumentLoadingTask, PDFDocumentProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { getDocument, VerbosityLevel } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { PdfToMdError } from './errors.ts';\nimport { resolvePackagePath } from './resolve.ts';\n\n// `verbosity: VerbosityLevel.ERRORS` silences pdfjs's own console.warn calls\n// — the library's documented promise is it never writes to stdout/stderr on its own.\n\n/** Data-based open (collect.ts's seam): no cMaps/fonts — text-layer extraction\n * only, verified to decode CJK correctly. Wraps missing/unreadable and non-PDF files as PdfToMdError/PDF_OPEN. */\nexport async function openDocument(src: string): Promise<PDFDocumentProxy> {\n let data: Uint8Array;\n try {\n data = new Uint8Array(readFileSync(src));\n } catch (err) {\n throw new PdfToMdError('PDF_OPEN', `cannot read ${src}: ${(err as Error).message}`);\n }\n try {\n return await getDocument({\n data,\n useSystemFonts: true,\n verbosity: VerbosityLevel.ERRORS,\n }).promise;\n } catch (err) {\n throw new PdfToMdError('PDF_OPEN', `cannot open ${src} as a PDF: ${(err as Error).message}`);\n }\n}\n\n// Wires up pdfjs's bundled cmaps/standard_fonts so CJK and non-embedded\n// standard fonts resolve instead of falling back to substitute glyphs.\n//\n// GOTCHA: despite the `...Url` name, pdfjs's Node loader does `fs.readFile`\n// on the string directly — a `file://` URL silently fails every lookup (only a stderr warning); plain filesystem paths are required.\n// The trailing \"/\" must be literal (pdfjs rejects a trailing \"\\\"); Windows' fs accepts \"/\" after a \"\\\" path.\nconst PDFJS_DIST_ROOT = path.dirname(resolvePackagePath('pdfjs-dist/package.json'));\nconst CMAP_URL = `${path.join(PDFJS_DIST_ROOT, 'cmaps')}/`;\nconst STANDARD_FONT_DATA_URL = `${path.join(PDFJS_DIST_ROOT, 'standard_fonts')}/`;\n// JBIG2/JPEG2000 decoding lives in pdfjs's WASM modules: without this, bitonal\n// scans (the usual encoding for B&W page images) decode to a BLANK page and OCR sees nothing.\nconst WASM_URL = `${path.join(PDFJS_DIST_ROOT, 'wasm')}/`;\n\n/**\n * Open a PDF for rendering (raster.ts/tesseract.ts's seam): cMaps + standard\n * fonts wired up, url-based. Callers must `await` the returned task's `.promise` (via `awaitPdfOpen`) and call `.destroy()` when done.\n */\nexport function openPdfForRender(pdfPath: string): PDFDocumentLoadingTask {\n // No `worker` option: this pdfjs-dist Node build needs no real Worker thread wired up to run.\n return getDocument({\n url: pdfPath,\n cMapUrl: CMAP_URL,\n cMapPacked: true,\n standardFontDataUrl: STANDARD_FONT_DATA_URL,\n wasmUrl: WASM_URL,\n verbosity: VerbosityLevel.ERRORS,\n });\n}\n\n/** Shared error wrapping for `openPdfForRender`'s task: a raw pdfjs open\n * failure becomes PdfToMdError/PDF_OPEN, original message preserved. */\nexport async function awaitPdfOpen(loadingTask: PDFDocumentLoadingTask, pdfPath: string): Promise<PDFDocumentProxy> {\n try {\n return await loadingTask.promise;\n } catch (err) {\n throw new PdfToMdError('PDF_OPEN', `could not open PDF: ${pdfPath} (${(err as Error).message})`);\n }\n}\n"],"names":["readFileSync","path","getDocument","VerbosityLevel","PdfToMdError","resolvePackagePath","openDocument","src","data","Uint8Array","err","message","useSystemFonts","verbosity","ERRORS","promise","PDFJS_DIST_ROOT","dirname","CMAP_URL","join","STANDARD_FONT_DATA_URL","WASM_URL","openPdfForRender","pdfPath","url","cMapUrl","cMapPacked","standardFontDataUrl","wasmUrl","awaitPdfOpen","loadingTask"],"mappings":"AAAA,yEAAyE;AACzE,sIAAsI;AACtI,uFAAuF;AACvF,SAASA,YAAY,QAAQ,UAAU;AACvC,OAAOC,UAAU,YAAY;AAE7B,SAASC,WAAW,EAAEC,cAAc,QAAQ,kCAAkC;AAC9E,SAASC,YAAY,QAAQ,cAAc;AAC3C,SAASC,kBAAkB,QAAQ,eAAe;AAElD,6EAA6E;AAC7E,qFAAqF;AAErF;gHACgH,GAChH,OAAO,eAAeC,aAAaC,GAAW;IAC5C,IAAIC;IACJ,IAAI;QACFA,OAAO,IAAIC,WAAWT,aAAaO;IACrC,EAAE,OAAOG,KAAK;QACZ,MAAM,IAAIN,aAAa,YAAY,CAAC,YAAY,EAAEG,IAAI,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACpF;IACA,IAAI;QACF,OAAO,MAAMT,YAAY;YACvBM;YACAI,gBAAgB;YAChBC,WAAWV,eAAeW,MAAM;QAClC,GAAGC,OAAO;IACZ,EAAE,OAAOL,KAAK;QACZ,MAAM,IAAIN,aAAa,YAAY,CAAC,YAAY,EAAEG,IAAI,WAAW,EAAE,AAACG,IAAcC,OAAO,EAAE;IAC7F;AACF;AAEA,wEAAwE;AACxE,uEAAuE;AACvE,EAAE;AACF,4EAA4E;AAC5E,qIAAqI;AACrI,6GAA6G;AAC7G,MAAMK,kBAAkBf,KAAKgB,OAAO,CAACZ,mBAAmB;AACxD,MAAMa,WAAW,GAAGjB,KAAKkB,IAAI,CAACH,iBAAiB,SAAS,CAAC,CAAC;AAC1D,MAAMI,yBAAyB,GAAGnB,KAAKkB,IAAI,CAACH,iBAAiB,kBAAkB,CAAC,CAAC;AACjF,+EAA+E;AAC/E,8FAA8F;AAC9F,MAAMK,WAAW,GAAGpB,KAAKkB,IAAI,CAACH,iBAAiB,QAAQ,CAAC,CAAC;AAEzD;;;CAGC,GACD,OAAO,SAASM,iBAAiBC,OAAe;IAC9C,8FAA8F;IAC9F,OAAOrB,YAAY;QACjBsB,KAAKD;QACLE,SAASP;QACTQ,YAAY;QACZC,qBAAqBP;QACrBQ,SAASP;QACTR,WAAWV,eAAeW,MAAM;IAClC;AACF;AAEA;sEACsE,GACtE,OAAO,eAAee,aAAaC,WAAmC,EAAEP,OAAe;IACrF,IAAI;QACF,OAAO,MAAMO,YAAYf,OAAO;IAClC,EAAE,OAAOL,KAAK;QACZ,MAAM,IAAIN,aAAa,YAAY,CAAC,oBAAoB,EAAEmB,QAAQ,EAAE,EAAE,AAACb,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjG;AACF"}
|
package/dist/esm/raster.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/raster.ts"],"sourcesContent":["// Renders PDF pages to PNG with pdfjs-dist + @napi-rs/canvas. Shared by the\n// `render` CLI command (human/vision-model page verification) and src/engines/tesseract.ts (OCR page images).\n\nimport { createCanvas } from '@napi-rs/canvas';\nimport type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { PdfToMdError } from './errors.ts';\nimport { awaitPdfOpen, openPdfForRender } from './pdf-open.ts';\n\n// pdfjs-dist doesn't export RenderParameters from the package root; derive\n// it from PDFPageProxy['render'] rather than hand-declaring a shape that could drift.\ntype RenderParams = Parameters<PDFPageProxy['render']>[0];\n\n// No CanvasFactory shim needed: pdfjs-dist detects `@napi-rs/canvas` itself\n// and uses it internally (patterns, SMasks, Type3 glyphs); we only hand it a `canvasContext` for the page itself, created directly below.\n\nexport { openPdfForRender } from './pdf-open.ts';\n\n/** One rendered page, from renderPageToPNG (or the `pdf-to-md render` CLI\n * command it backs). */\nexport interface RenderPageResult {\n /** the rendered page, PNG-encoded */\n png: Buffer;\n /** pixel width at the requested dpi */\n width: number;\n /** pixel height at the requested dpi */\n height: number;\n /** the page's own /Rotate entry, baked into `png`/`width`/`height` already\n * — informational, not something a caller needs to apply again */\n rotation: number;\n}\n\n/** Render one already-open document's page to a PNG buffer at the given DPI. */\nexport async function renderLoadedPage(doc: PDFDocumentProxy, pageNum: number, dpi: number): Promise<RenderPageResult> {\n if (!(pageNum >= 1) || pageNum > doc.numPages) {\n throw new PdfToMdError('PAGE_RANGE', `page ${pageNum} is outside this document (1-${doc.numPages})`);\n }\n const page = await doc.getPage(pageNum);\n\n // getViewport({scale}) bakes in the page's own /Rotate entry AND its\n // mediaBox origin automatically — no manual origin translate needed on top of it.\n const scale = dpi / 72;\n const viewport = page.getViewport({ scale });\n const width = Math.round(viewport.width);\n const height = Math.round(viewport.height);\n if (width <= 0 || height <= 0) throw new PdfToMdError('PAGE_RENDER', `page ${pageNum} has non-positive dimensions ${width}x${height}`);\n\n const canvas = createCanvas(width, height);\n const context = canvas.getContext('2d');\n\n // Paint white first: an unpainted @napi-rs/canvas context is transparent,\n // which would encode as black/checkered rather than a white page.\n context.fillStyle = 'white';\n context.fillRect(0, 0, width, height);\n\n // pdfjs's RenderParameters types canvas/context against DOM types;\n // @napi-rs/canvas's are API-compatible at runtime but not that DOM type — the interop boundary the \"no any\" rule allows a cast for.\n const renderTask = page.render({\n canvas: canvas as unknown as RenderParams['canvas'],\n canvasContext: context as unknown as RenderParams['canvasContext'],\n viewport,\n });\n await renderTask.promise;\n\n const png = await canvas.encode('png');\n return { png, width, height, rotation: viewport.rotation };\n}\n\n/**\n * Render one page of a PDF to a PNG buffer at the given DPI. Opens and\n * closes the document itself — for more than one page from the same PDF, use `openPdfForRender` + `renderLoadedPage` instead.\n *\n * @param pdfPath path to the PDF file\n * @param pageNum 1-based page number\n * @param dpi PDF user space is 72 units/inch, so scale = dpi / 72\n */\nexport async function renderPageToPNG(pdfPath: string, pageNum: number, dpi: number): Promise<RenderPageResult> {\n const loadingTask = openPdfForRender(pdfPath);\n try {\n const doc = await awaitPdfOpen(loadingTask, pdfPath);\n return await renderLoadedPage(doc, pageNum, dpi);\n } finally {\n await loadingTask.destroy();\n }\n}\n"],"names":["createCanvas","PdfToMdError","awaitPdfOpen","openPdfForRender","renderLoadedPage","doc","pageNum","dpi","numPages","page","getPage","scale","viewport","getViewport","width","Math","round","height","canvas","context","getContext","fillStyle","fillRect","renderTask","render","canvasContext","promise","png","encode","rotation","renderPageToPNG","pdfPath","loadingTask","destroy"],"mappings":"AAAA,4EAA4E;AAC5E,8GAA8G;AAE9G,SAASA,YAAY,QAAQ,kBAAkB;AAE/C,SAASC,YAAY,QAAQ,cAAc;AAC3C,SAASC,YAAY,EAAEC,gBAAgB,QAAQ,gBAAgB;AAM/D,4EAA4E;AAC5E,0IAA0I;AAE1I,SAASA,gBAAgB,QAAQ,gBAAgB;AAgBjD,8EAA8E,GAC9E,OAAO,eAAeC,iBAAiBC,GAAqB,EAAEC,OAAe,EAAEC,GAAW;IACxF,IAAI,CAAED,CAAAA,WAAW,CAAA,KAAMA,UAAUD,IAAIG,QAAQ,EAAE;QAC7C,MAAM,IAAIP,aAAa,cAAc,CAAC,KAAK,EAAEK,QAAQ,6BAA6B,EAAED,IAAIG,QAAQ,CAAC,CAAC,CAAC;IACrG;IACA,MAAMC,OAAO,MAAMJ,IAAIK,OAAO,CAACJ;IAE/B,qEAAqE;IACrE,kFAAkF;IAClF,MAAMK,QAAQJ,MAAM;IACpB,MAAMK,WAAWH,KAAKI,WAAW,CAAC;QAAEF;IAAM;IAC1C,MAAMG,QAAQC,KAAKC,KAAK,CAACJ,SAASE,KAAK;IACvC,MAAMG,SAASF,KAAKC,KAAK,CAACJ,SAASK,MAAM;IACzC,IAAIH,SAAS,KAAKG,UAAU,GAAG,MAAM,IAAIhB,aAAa,eAAe,CAAC,KAAK,EAAEK,QAAQ,6BAA6B,EAAEQ,MAAM,CAAC,EAAEG,QAAQ;IAErI,MAAMC,SAASlB,aAAac,OAAOG;IACnC,MAAME,UAAUD,OAAOE,UAAU,CAAC;IAElC,0EAA0E;IAC1E,kEAAkE;IAClED,QAAQE,SAAS,GAAG;IACpBF,QAAQG,QAAQ,CAAC,GAAG,GAAGR,OAAOG;IAE9B,mEAAmE;IACnE,oIAAoI;IACpI,MAAMM,aAAad,KAAKe,MAAM,CAAC;QAC7BN,QAAQA;QACRO,eAAeN;QACfP;IACF;IACA,MAAMW,WAAWG,OAAO;IAExB,MAAMC,MAAM,MAAMT,OAAOU,MAAM,CAAC;IAChC,OAAO;QAAED;QAAKb;QAAOG;QAAQY,UAAUjB,SAASiB,QAAQ;IAAC;AAC3D;AAEA;;;;;;;CAOC,GACD,OAAO,eAAeC,gBAAgBC,OAAe,EAAEzB,OAAe,EAAEC,GAAW;IACjF,MAAMyB,cAAc7B,iBAAiB4B;IACrC,IAAI;QACF,MAAM1B,MAAM,MAAMH,aAAa8B,aAAaD;QAC5C,OAAO,MAAM3B,iBAAiBC,KAAKC,SAASC;IAC9C,SAAU;QACR,MAAMyB,YAAYC,OAAO;IAC3B;AACF"}
|
|
@@ -8,10 +8,12 @@ export declare function renderPagesToMarkdown(pages: AnalysisPage[], opts: {
|
|
|
8
8
|
export declare function renderPagesToText(pages: AnalysisPage[]): string;
|
|
9
9
|
/**
|
|
10
10
|
* Render an Analysis to markdown. PURE and fail-fast over any well-formed Analysis (including hand-built/spliced ones) — throws `PdfToMdError`/`'ANALYSIS_INPUT'` before any output on a malformed one.
|
|
11
|
-
* OCR-origin output is byte-identical to extractOcr's own markdown
|
|
11
|
+
* OCR-origin output is byte-identical to extractOcr's own markdown. Text
|
|
12
|
+
* output comes from the same captured blocks as extractText. Mixed output
|
|
13
|
+
* always includes page markers because adjacent pages have different producers.
|
|
12
14
|
*
|
|
13
15
|
* @param analysis the Analysis to render (from analyze(), or hand-built/spliced)
|
|
14
|
-
* @param opts.pageMarkers text-origin only;
|
|
16
|
+
* @param opts.pageMarkers text-origin only; OCR and mixed Analysis values always carry markers
|
|
15
17
|
*/
|
|
16
18
|
export declare function toMarkdown(analysis: Analysis, opts?: {
|
|
17
19
|
pageMarkers?: boolean;
|
|
@@ -18,12 +18,13 @@ function validateAnalysis(analysis) {
|
|
|
18
18
|
const a = analysis;
|
|
19
19
|
if (typeof a.report !== 'object' || a.report === null) fail('Analysis.report must be an object');
|
|
20
20
|
const path = a.report.path;
|
|
21
|
-
if (path !== 'text' && path !== 'ocr') fail(`Analysis.report.path must be "text" or "
|
|
21
|
+
if (path !== 'text' && path !== 'ocr' && path !== 'mixed') fail(`Analysis.report.path must be "text", "ocr", or "mixed" (got ${JSON.stringify(path)})`);
|
|
22
22
|
if (!Array.isArray(a.pages)) fail('Analysis.pages must be an array');
|
|
23
23
|
for (const [pi, page] of a.pages.entries()){
|
|
24
24
|
if (typeof page !== 'object' || page === null) fail(`Analysis.pages[${pi}] must be an object`);
|
|
25
25
|
const p = page;
|
|
26
26
|
if (typeof p.page !== 'number') fail(`Analysis.pages[${pi}].page must be a number`);
|
|
27
|
+
if (p.path !== 'text' && p.path !== 'ocr') fail(`Analysis.pages[${pi}].path must be "text" or "ocr" (got ${JSON.stringify(p.path)})`);
|
|
27
28
|
if (!Array.isArray(p.words)) fail(`Analysis.pages[${pi}].words must be an array`);
|
|
28
29
|
for (const [wi, w] of p.words.entries()){
|
|
29
30
|
if (typeof w !== 'object' || w === null || typeof w.text !== 'string') {
|
|
@@ -48,6 +49,7 @@ function validateAnalysis(analysis) {
|
|
|
48
49
|
} else if (block.level !== undefined) {
|
|
49
50
|
fail(`Analysis.pages[${pi}].blocks[${bi}].level is only allowed on heading blocks (got a ${JSON.stringify(block.type)} block with level ${JSON.stringify(block.level)})`);
|
|
50
51
|
}
|
|
52
|
+
if (block.tight !== undefined && typeof block.tight !== 'boolean') fail(`Analysis.pages[${pi}].blocks[${bi}].tight must be boolean when present`);
|
|
51
53
|
if (!Array.isArray(block.wordIndexes)) fail(`Analysis.pages[${pi}].blocks[${bi}].wordIndexes must be an array`);
|
|
52
54
|
for (const idx of block.wordIndexes){
|
|
53
55
|
if (typeof idx !== 'number' || !Number.isInteger(idx) || idx < 0 || idx >= wordCount) {
|
|
@@ -70,7 +72,7 @@ function renderBlockLiteral(block) {
|
|
|
70
72
|
for (const page of pages){
|
|
71
73
|
if (opts.forcePageMarkers) md += `\n${pageMarkerLine(page.page)}\n\n`;
|
|
72
74
|
else if (opts.pageMarkers) md += `${pageMarkerLine(page.page)}\n\n`;
|
|
73
|
-
for (const block of page.blocks)md += `${renderBlockLiteral(block)}\n\n`;
|
|
75
|
+
for (const block of page.blocks)md += `${renderBlockLiteral(block)}${block.tight ? '\n' : '\n\n'}`;
|
|
74
76
|
}
|
|
75
77
|
return md;
|
|
76
78
|
}
|
|
@@ -91,13 +93,15 @@ export function renderPagesToText(pages) {
|
|
|
91
93
|
}
|
|
92
94
|
/**
|
|
93
95
|
* Render an Analysis to markdown. PURE and fail-fast over any well-formed Analysis (including hand-built/spliced ones) — throws `PdfToMdError`/`'ANALYSIS_INPUT'` before any output on a malformed one.
|
|
94
|
-
* OCR-origin output is byte-identical to extractOcr's own markdown
|
|
96
|
+
* OCR-origin output is byte-identical to extractOcr's own markdown. Text
|
|
97
|
+
* output comes from the same captured blocks as extractText. Mixed output
|
|
98
|
+
* always includes page markers because adjacent pages have different producers.
|
|
95
99
|
*
|
|
96
100
|
* @param analysis the Analysis to render (from analyze(), or hand-built/spliced)
|
|
97
|
-
* @param opts.pageMarkers text-origin only;
|
|
101
|
+
* @param opts.pageMarkers text-origin only; OCR and mixed Analysis values always carry markers
|
|
98
102
|
*/ export function toMarkdown(analysis, opts = {}) {
|
|
99
103
|
validateAnalysis(analysis);
|
|
100
|
-
const isOcr = analysis.report.path
|
|
104
|
+
const isOcr = analysis.report.path !== 'text';
|
|
101
105
|
const md = renderPagesToMarkdown(analysis.pages, {
|
|
102
106
|
forcePageMarkers: isOcr,
|
|
103
107
|
pageMarkers: !!opts.pageMarkers
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/render-analysis.ts"],"sourcesContent":["// Pure renderers over the Analysis IR: toMarkdown()/toText() are the public,\n// validated entry points; renderPagesToMarkdown/renderPagesToText below are the unvalidated core, reused (unvalidated) by src/analyze.ts.\nimport { PdfToMdError } from './errors.ts';\nimport { pageMarkerLine } from './page-marker.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage } from './types.ts';\n\nconst BLOCK_TYPES = new Set(['heading', 'paragraph', 'float', 'review']);\n\n// Whole-object, fail-fast validation: every page/block/wordIndexes entry is\n// checked BEFORE toMarkdown/toText render a single character — a malformed Analysis must never produce partial output, only a clean throw.\nfunction validateAnalysis(analysis: unknown): asserts analysis is Analysis {\n const fail = (msg: string): never => {\n throw new PdfToMdError('ANALYSIS_INPUT', msg);\n };\n if (typeof analysis !== 'object' || analysis === null) fail('Analysis must be an object');\n const a = analysis as Record<string, unknown>;\n if (typeof a.report !== 'object' || a.report === null) fail('Analysis.report must be an object');\n const path = (a.report as Record<string, unknown>).path;\n if (path !== 'text' && path !== 'ocr') fail(`Analysis.report.path must be \"text\" or \"ocr\" (got ${JSON.stringify(path)})`);\n if (!Array.isArray(a.pages)) fail('Analysis.pages must be an array');\n\n for (const [pi, page] of (a.pages as unknown[]).entries()) {\n if (typeof page !== 'object' || page === null) fail(`Analysis.pages[${pi}] must be an object`);\n const p = page as Record<string, unknown>;\n if (typeof p.page !== 'number') fail(`Analysis.pages[${pi}].page must be a number`);\n if (!Array.isArray(p.words)) fail(`Analysis.pages[${pi}].words must be an array`);\n for (const [wi, w] of (p.words as unknown[]).entries()) {\n if (typeof w !== 'object' || w === null || typeof (w as Record<string, unknown>).text !== 'string') {\n fail(`Analysis.pages[${pi}].words[${wi}] must be an object with a string 'text'`);\n }\n }\n const wordCount = (p.words as unknown[]).length;\n\n if (!Array.isArray(p.blocks)) fail(`Analysis.pages[${pi}].blocks must be an array`);\n for (const [bi, b] of (p.blocks as unknown[]).entries()) {\n if (typeof b !== 'object' || b === null) fail(`Analysis.pages[${pi}].blocks[${bi}] must be an object`);\n const block = b as Record<string, unknown>;\n if (typeof block.type !== 'string' || !BLOCK_TYPES.has(block.type)) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].type must be one of heading|paragraph|float|review (got ${JSON.stringify(block.type)})`);\n }\n if (typeof block.text !== 'string') fail(`Analysis.pages[${pi}].blocks[${bi}].text must be a string`);\n // `level` contract (see AnalysisBlock, src/types.ts): required 1-6 on a\n // heading, forbidden otherwise — rejected, not ignored, since it likely means `type` was mistyped.\n if (block.type === 'heading') {\n if (!Number.isInteger(block.level) || (block.level as number) < 1 || (block.level as number) > 6) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].level must be an integer 1-6 on a heading block (got ${JSON.stringify(block.level)})`);\n }\n } else if (block.level !== undefined) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].level is only allowed on heading blocks (got a ${JSON.stringify(block.type)} block with level ${JSON.stringify(block.level)})`);\n }\n if (!Array.isArray(block.wordIndexes)) fail(`Analysis.pages[${pi}].blocks[${bi}].wordIndexes must be an array`);\n for (const idx of block.wordIndexes as unknown[]) {\n if (typeof idx !== 'number' || !Number.isInteger(idx) || idx < 0 || idx >= wordCount) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].wordIndexes has an out-of-range index (${JSON.stringify(idx)}; page ${pi} has ${wordCount} words)`);\n }\n }\n }\n }\n}\n\n// See AnalysisBlock's doc comment (src/types.ts) for the `text` contract this relies on.\nfunction renderBlockLiteral(block: AnalysisBlock): string {\n // A heading's `level` is guaranteed present (1-6): validateAnalysis\n // enforces it, and src/analyze.ts (the only unvalidated caller) always sets it.\n if (block.type === 'heading') return `${'#'.repeat(block.level as number)} ${block.text}`;\n return block.text;\n}\n\n/** Unvalidated core, reused by `src/analyze.ts`. `forcePageMarkers` is the\n * OCR path's always-on \"### pN\"; `pageMarkers` is the text path's opt-in flag, ignored when `forcePageMarkers` is set. */\nexport function renderPagesToMarkdown(pages: AnalysisPage[], opts: { forcePageMarkers: boolean; pageMarkers: boolean }): string {\n let md = '';\n for (const page of pages) {\n if (opts.forcePageMarkers) md += `\\n${pageMarkerLine(page.page)}\\n\\n`;\n else if (opts.pageMarkers) md += `${pageMarkerLine(page.page)}\\n\\n`;\n for (const block of page.blocks) md += `${renderBlockLiteral(block)}\\n\\n`;\n }\n return md;\n}\n\n// Strips a block's markdown decoration for plain-text rendering (the inverse\n// of AnalysisBlock's literal-text convention); headings need no stripping.\nfunction bareBlockText(block: AnalysisBlock): string {\n if (block.type === 'heading') return block.text;\n if (block.type === 'review') return block.text.replace(/^> \\[review: [^\\]]*\\] /, '');\n if (block.type === 'float') return block.text.replace(/^> \\[floats\\] /, '');\n return block.text.replace(/^> /, '');\n}\n\nexport function renderPagesToText(pages: AnalysisPage[]): string {\n const out: string[] = [];\n for (const page of pages) {\n for (const block of page.blocks) out.push(bareBlockText(block));\n }\n return out.length ? `${out.join('\\n\\n').trim()}\\n` : '';\n}\n\n/**\n * Render an Analysis to markdown. PURE and fail-fast over any well-formed Analysis (including hand-built/spliced ones) — throws `PdfToMdError`/`'ANALYSIS_INPUT'` before any output on a malformed one.\n * OCR-origin output is byte-identical to extractOcr's own markdown (shared core); text-origin is NOT byte-identical to extractText's (see `AnalysisPage.words`, `src/types.ts`).\n *\n * @param analysis the Analysis to render (from analyze(), or hand-built/spliced)\n * @param opts.pageMarkers text-origin only; an OCR-origin Analysis always carries markers\n */\nexport function toMarkdown(analysis: Analysis, opts: { pageMarkers?: boolean } = {}): string {\n validateAnalysis(analysis);\n const isOcr = analysis.report.path === 'ocr';\n const md = renderPagesToMarkdown(analysis.pages, { forcePageMarkers: isOcr, pageMarkers: !!opts.pageMarkers });\n return isOcr ? md : `${md.trim()}\\n`;\n}\n\n/**\n * Render an Analysis to reading-order plain text: headings drop their `#`;\n * float/review content is INCLUDED with only its markdown decoration stripped, so no content is lost silently. Same fail-fast validation as toMarkdown.\n *\n * @param analysis the Analysis to render (from analyze(), or hand-built/spliced)\n */\nexport function toText(analysis: Analysis): string {\n validateAnalysis(analysis);\n return renderPagesToText(analysis.pages);\n}\n"],"names":["PdfToMdError","pageMarkerLine","BLOCK_TYPES","Set","validateAnalysis","analysis","fail","msg","a","report","path","JSON","stringify","Array","isArray","pages","pi","page","entries","p","words","wi","w","text","wordCount","length","blocks","bi","b","block","type","has","Number","isInteger","level","undefined","wordIndexes","idx","renderBlockLiteral","repeat","renderPagesToMarkdown","opts","md","forcePageMarkers","pageMarkers","bareBlockText","replace","renderPagesToText","out","push","join","trim","toMarkdown","isOcr","toText"],"mappings":"AAAA,6EAA6E;AAC7E,0IAA0I;AAC1I,SAASA,YAAY,QAAQ,cAAc;AAC3C,SAASC,cAAc,QAAQ,mBAAmB;AAGlD,MAAMC,cAAc,IAAIC,IAAI;IAAC;IAAW;IAAa;IAAS;CAAS;AAEvE,4EAA4E;AAC5E,2IAA2I;AAC3I,SAASC,iBAAiBC,QAAiB;IACzC,MAAMC,OAAO,CAACC;QACZ,MAAM,IAAIP,aAAa,kBAAkBO;IAC3C;IACA,IAAI,OAAOF,aAAa,YAAYA,aAAa,MAAMC,KAAK;IAC5D,MAAME,IAAIH;IACV,IAAI,OAAOG,EAAEC,MAAM,KAAK,YAAYD,EAAEC,MAAM,KAAK,MAAMH,KAAK;IAC5D,MAAMI,OAAO,AAACF,EAAEC,MAAM,CAA6BC,IAAI;IACvD,IAAIA,SAAS,UAAUA,SAAS,OAAOJ,KAAK,CAAC,kDAAkD,EAAEK,KAAKC,SAAS,CAACF,MAAM,CAAC,CAAC;IACxH,IAAI,CAACG,MAAMC,OAAO,CAACN,EAAEO,KAAK,GAAGT,KAAK;IAElC,KAAK,MAAM,CAACU,IAAIC,KAAK,IAAI,AAACT,EAAEO,KAAK,CAAeG,OAAO,GAAI;QACzD,IAAI,OAAOD,SAAS,YAAYA,SAAS,MAAMX,KAAK,CAAC,eAAe,EAAEU,GAAG,mBAAmB,CAAC;QAC7F,MAAMG,IAAIF;QACV,IAAI,OAAOE,EAAEF,IAAI,KAAK,UAAUX,KAAK,CAAC,eAAe,EAAEU,GAAG,uBAAuB,CAAC;QAClF,IAAI,CAACH,MAAMC,OAAO,CAACK,EAAEC,KAAK,GAAGd,KAAK,CAAC,eAAe,EAAEU,GAAG,wBAAwB,CAAC;QAChF,KAAK,MAAM,CAACK,IAAIC,EAAE,IAAI,AAACH,EAAEC,KAAK,CAAeF,OAAO,GAAI;YACtD,IAAI,OAAOI,MAAM,YAAYA,MAAM,QAAQ,OAAO,AAACA,EAA8BC,IAAI,KAAK,UAAU;gBAClGjB,KAAK,CAAC,eAAe,EAAEU,GAAG,QAAQ,EAAEK,GAAG,wCAAwC,CAAC;YAClF;QACF;QACA,MAAMG,YAAY,AAACL,EAAEC,KAAK,CAAeK,MAAM;QAE/C,IAAI,CAACZ,MAAMC,OAAO,CAACK,EAAEO,MAAM,GAAGpB,KAAK,CAAC,eAAe,EAAEU,GAAG,yBAAyB,CAAC;QAClF,KAAK,MAAM,CAACW,IAAIC,EAAE,IAAI,AAACT,EAAEO,MAAM,CAAeR,OAAO,GAAI;YACvD,IAAI,OAAOU,MAAM,YAAYA,MAAM,MAAMtB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,mBAAmB,CAAC;YACrG,MAAME,QAAQD;YACd,IAAI,OAAOC,MAAMC,IAAI,KAAK,YAAY,CAAC5B,YAAY6B,GAAG,CAACF,MAAMC,IAAI,GAAG;gBAClExB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,0DAA0D,EAAEhB,KAAKC,SAAS,CAACiB,MAAMC,IAAI,EAAE,CAAC,CAAC;YACnI;YACA,IAAI,OAAOD,MAAMN,IAAI,KAAK,UAAUjB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,uBAAuB,CAAC;YACpG,wEAAwE;YACxE,mGAAmG;YACnG,IAAIE,MAAMC,IAAI,KAAK,WAAW;gBAC5B,IAAI,CAACE,OAAOC,SAAS,CAACJ,MAAMK,KAAK,KAAK,AAACL,MAAMK,KAAK,GAAc,KAAK,AAACL,MAAMK,KAAK,GAAc,GAAG;oBAChG5B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,uDAAuD,EAAEhB,KAAKC,SAAS,CAACiB,MAAMK,KAAK,EAAE,CAAC,CAAC;gBACjI;YACF,OAAO,IAAIL,MAAMK,KAAK,KAAKC,WAAW;gBACpC7B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,iDAAiD,EAAEhB,KAAKC,SAAS,CAACiB,MAAMC,IAAI,EAAE,kBAAkB,EAAEnB,KAAKC,SAAS,CAACiB,MAAMK,KAAK,EAAE,CAAC,CAAC;YAC1K;YACA,IAAI,CAACrB,MAAMC,OAAO,CAACe,MAAMO,WAAW,GAAG9B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,8BAA8B,CAAC;YAC9G,KAAK,MAAMU,OAAOR,MAAMO,WAAW,CAAe;gBAChD,IAAI,OAAOC,QAAQ,YAAY,CAACL,OAAOC,SAAS,CAACI,QAAQA,MAAM,KAAKA,OAAOb,WAAW;oBACpFlB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,yCAAyC,EAAEhB,KAAKC,SAAS,CAACyB,KAAK,OAAO,EAAErB,GAAG,KAAK,EAAEQ,UAAU,OAAO,CAAC;gBAC9I;YACF;QACF;IACF;AACF;AAEA,yFAAyF;AACzF,SAASc,mBAAmBT,KAAoB;IAC9C,oEAAoE;IACpE,gFAAgF;IAChF,IAAIA,MAAMC,IAAI,KAAK,WAAW,OAAO,GAAG,IAAIS,MAAM,CAACV,MAAMK,KAAK,EAAY,CAAC,EAAEL,MAAMN,IAAI,EAAE;IACzF,OAAOM,MAAMN,IAAI;AACnB;AAEA;wHACwH,GACxH,OAAO,SAASiB,sBAAsBzB,KAAqB,EAAE0B,IAAyD;IACpH,IAAIC,KAAK;IACT,KAAK,MAAMzB,QAAQF,MAAO;QACxB,IAAI0B,KAAKE,gBAAgB,EAAED,MAAM,CAAC,EAAE,EAAEzC,eAAegB,KAAKA,IAAI,EAAE,IAAI,CAAC;aAChE,IAAIwB,KAAKG,WAAW,EAAEF,MAAM,GAAGzC,eAAegB,KAAKA,IAAI,EAAE,IAAI,CAAC;QACnE,KAAK,MAAMY,SAASZ,KAAKS,MAAM,CAAEgB,MAAM,GAAGJ,mBAAmBT,OAAO,IAAI,CAAC;IAC3E;IACA,OAAOa;AACT;AAEA,6EAA6E;AAC7E,2EAA2E;AAC3E,SAASG,cAAchB,KAAoB;IACzC,IAAIA,MAAMC,IAAI,KAAK,WAAW,OAAOD,MAAMN,IAAI;IAC/C,IAAIM,MAAMC,IAAI,KAAK,UAAU,OAAOD,MAAMN,IAAI,CAACuB,OAAO,CAAC,0BAA0B;IACjF,IAAIjB,MAAMC,IAAI,KAAK,SAAS,OAAOD,MAAMN,IAAI,CAACuB,OAAO,CAAC,kBAAkB;IACxE,OAAOjB,MAAMN,IAAI,CAACuB,OAAO,CAAC,OAAO;AACnC;AAEA,OAAO,SAASC,kBAAkBhC,KAAqB;IACrD,MAAMiC,MAAgB,EAAE;IACxB,KAAK,MAAM/B,QAAQF,MAAO;QACxB,KAAK,MAAMc,SAASZ,KAAKS,MAAM,CAAEsB,IAAIC,IAAI,CAACJ,cAAchB;IAC1D;IACA,OAAOmB,IAAIvB,MAAM,GAAG,GAAGuB,IAAIE,IAAI,CAAC,QAAQC,IAAI,GAAG,EAAE,CAAC,GAAG;AACvD;AAEA;;;;;;CAMC,GACD,OAAO,SAASC,WAAW/C,QAAkB,EAAEoC,OAAkC,CAAC,CAAC;IACjFrC,iBAAiBC;IACjB,MAAMgD,QAAQhD,SAASI,MAAM,CAACC,IAAI,KAAK;IACvC,MAAMgC,KAAKF,sBAAsBnC,SAASU,KAAK,EAAE;QAAE4B,kBAAkBU;QAAOT,aAAa,CAAC,CAACH,KAAKG,WAAW;IAAC;IAC5G,OAAOS,QAAQX,KAAK,GAAGA,GAAGS,IAAI,GAAG,EAAE,CAAC;AACtC;AAEA;;;;;CAKC,GACD,OAAO,SAASG,OAAOjD,QAAkB;IACvCD,iBAAiBC;IACjB,OAAO0C,kBAAkB1C,SAASU,KAAK;AACzC"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/render-analysis.ts"],"sourcesContent":["// Pure renderers over the Analysis IR: toMarkdown()/toText() are the public,\n// validated entry points; renderPagesToMarkdown/renderPagesToText below are the unvalidated core, reused (unvalidated) by src/analyze.ts.\nimport { PdfToMdError } from './errors.ts';\nimport { pageMarkerLine } from './page-marker.ts';\nimport type { Analysis, AnalysisBlock, AnalysisPage } from './types.ts';\n\nconst BLOCK_TYPES = new Set(['heading', 'paragraph', 'float', 'review']);\n\n// Whole-object, fail-fast validation: every page/block/wordIndexes entry is\n// checked BEFORE toMarkdown/toText render a single character — a malformed Analysis must never produce partial output, only a clean throw.\nfunction validateAnalysis(analysis: unknown): asserts analysis is Analysis {\n const fail = (msg: string): never => {\n throw new PdfToMdError('ANALYSIS_INPUT', msg);\n };\n if (typeof analysis !== 'object' || analysis === null) fail('Analysis must be an object');\n const a = analysis as Record<string, unknown>;\n if (typeof a.report !== 'object' || a.report === null) fail('Analysis.report must be an object');\n const path = (a.report as Record<string, unknown>).path;\n if (path !== 'text' && path !== 'ocr' && path !== 'mixed') fail(`Analysis.report.path must be \"text\", \"ocr\", or \"mixed\" (got ${JSON.stringify(path)})`);\n if (!Array.isArray(a.pages)) fail('Analysis.pages must be an array');\n\n for (const [pi, page] of (a.pages as unknown[]).entries()) {\n if (typeof page !== 'object' || page === null) fail(`Analysis.pages[${pi}] must be an object`);\n const p = page as Record<string, unknown>;\n if (typeof p.page !== 'number') fail(`Analysis.pages[${pi}].page must be a number`);\n if (p.path !== 'text' && p.path !== 'ocr') fail(`Analysis.pages[${pi}].path must be \"text\" or \"ocr\" (got ${JSON.stringify(p.path)})`);\n if (!Array.isArray(p.words)) fail(`Analysis.pages[${pi}].words must be an array`);\n for (const [wi, w] of (p.words as unknown[]).entries()) {\n if (typeof w !== 'object' || w === null || typeof (w as Record<string, unknown>).text !== 'string') {\n fail(`Analysis.pages[${pi}].words[${wi}] must be an object with a string 'text'`);\n }\n }\n const wordCount = (p.words as unknown[]).length;\n\n if (!Array.isArray(p.blocks)) fail(`Analysis.pages[${pi}].blocks must be an array`);\n for (const [bi, b] of (p.blocks as unknown[]).entries()) {\n if (typeof b !== 'object' || b === null) fail(`Analysis.pages[${pi}].blocks[${bi}] must be an object`);\n const block = b as Record<string, unknown>;\n if (typeof block.type !== 'string' || !BLOCK_TYPES.has(block.type)) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].type must be one of heading|paragraph|float|review (got ${JSON.stringify(block.type)})`);\n }\n if (typeof block.text !== 'string') fail(`Analysis.pages[${pi}].blocks[${bi}].text must be a string`);\n // `level` contract (see AnalysisBlock, src/types.ts): required 1-6 on a\n // heading, forbidden otherwise — rejected, not ignored, since it likely means `type` was mistyped.\n if (block.type === 'heading') {\n if (!Number.isInteger(block.level) || (block.level as number) < 1 || (block.level as number) > 6) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].level must be an integer 1-6 on a heading block (got ${JSON.stringify(block.level)})`);\n }\n } else if (block.level !== undefined) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].level is only allowed on heading blocks (got a ${JSON.stringify(block.type)} block with level ${JSON.stringify(block.level)})`);\n }\n if (block.tight !== undefined && typeof block.tight !== 'boolean') fail(`Analysis.pages[${pi}].blocks[${bi}].tight must be boolean when present`);\n if (!Array.isArray(block.wordIndexes)) fail(`Analysis.pages[${pi}].blocks[${bi}].wordIndexes must be an array`);\n for (const idx of block.wordIndexes as unknown[]) {\n if (typeof idx !== 'number' || !Number.isInteger(idx) || idx < 0 || idx >= wordCount) {\n fail(`Analysis.pages[${pi}].blocks[${bi}].wordIndexes has an out-of-range index (${JSON.stringify(idx)}; page ${pi} has ${wordCount} words)`);\n }\n }\n }\n }\n}\n\n// See AnalysisBlock's doc comment (src/types.ts) for the `text` contract this relies on.\nfunction renderBlockLiteral(block: AnalysisBlock): string {\n // A heading's `level` is guaranteed present (1-6): validateAnalysis\n // enforces it, and src/analyze.ts (the only unvalidated caller) always sets it.\n if (block.type === 'heading') return `${'#'.repeat(block.level as number)} ${block.text}`;\n return block.text;\n}\n\n/** Unvalidated core, reused by `src/analyze.ts`. `forcePageMarkers` is the\n * OCR path's always-on \"### pN\"; `pageMarkers` is the text path's opt-in flag, ignored when `forcePageMarkers` is set. */\nexport function renderPagesToMarkdown(pages: AnalysisPage[], opts: { forcePageMarkers: boolean; pageMarkers: boolean }): string {\n let md = '';\n for (const page of pages) {\n if (opts.forcePageMarkers) md += `\\n${pageMarkerLine(page.page)}\\n\\n`;\n else if (opts.pageMarkers) md += `${pageMarkerLine(page.page)}\\n\\n`;\n for (const block of page.blocks) md += `${renderBlockLiteral(block)}${block.tight ? '\\n' : '\\n\\n'}`;\n }\n return md;\n}\n\n// Strips a block's markdown decoration for plain-text rendering (the inverse\n// of AnalysisBlock's literal-text convention); headings need no stripping.\nfunction bareBlockText(block: AnalysisBlock): string {\n if (block.type === 'heading') return block.text;\n if (block.type === 'review') return block.text.replace(/^> \\[review: [^\\]]*\\] /, '');\n if (block.type === 'float') return block.text.replace(/^> \\[floats\\] /, '');\n return block.text.replace(/^> /, '');\n}\n\nexport function renderPagesToText(pages: AnalysisPage[]): string {\n const out: string[] = [];\n for (const page of pages) {\n for (const block of page.blocks) out.push(bareBlockText(block));\n }\n return out.length ? `${out.join('\\n\\n').trim()}\\n` : '';\n}\n\n/**\n * Render an Analysis to markdown. PURE and fail-fast over any well-formed Analysis (including hand-built/spliced ones) — throws `PdfToMdError`/`'ANALYSIS_INPUT'` before any output on a malformed one.\n * OCR-origin output is byte-identical to extractOcr's own markdown. Text\n * output comes from the same captured blocks as extractText. Mixed output\n * always includes page markers because adjacent pages have different producers.\n *\n * @param analysis the Analysis to render (from analyze(), or hand-built/spliced)\n * @param opts.pageMarkers text-origin only; OCR and mixed Analysis values always carry markers\n */\nexport function toMarkdown(analysis: Analysis, opts: { pageMarkers?: boolean } = {}): string {\n validateAnalysis(analysis);\n const isOcr = analysis.report.path !== 'text';\n const md = renderPagesToMarkdown(analysis.pages, { forcePageMarkers: isOcr, pageMarkers: !!opts.pageMarkers });\n return isOcr ? md : `${md.trim()}\\n`;\n}\n\n/**\n * Render an Analysis to reading-order plain text: headings drop their `#`;\n * float/review content is INCLUDED with only its markdown decoration stripped, so no content is lost silently. Same fail-fast validation as toMarkdown.\n *\n * @param analysis the Analysis to render (from analyze(), or hand-built/spliced)\n */\nexport function toText(analysis: Analysis): string {\n validateAnalysis(analysis);\n return renderPagesToText(analysis.pages);\n}\n"],"names":["PdfToMdError","pageMarkerLine","BLOCK_TYPES","Set","validateAnalysis","analysis","fail","msg","a","report","path","JSON","stringify","Array","isArray","pages","pi","page","entries","p","words","wi","w","text","wordCount","length","blocks","bi","b","block","type","has","Number","isInteger","level","undefined","tight","wordIndexes","idx","renderBlockLiteral","repeat","renderPagesToMarkdown","opts","md","forcePageMarkers","pageMarkers","bareBlockText","replace","renderPagesToText","out","push","join","trim","toMarkdown","isOcr","toText"],"mappings":"AAAA,6EAA6E;AAC7E,0IAA0I;AAC1I,SAASA,YAAY,QAAQ,cAAc;AAC3C,SAASC,cAAc,QAAQ,mBAAmB;AAGlD,MAAMC,cAAc,IAAIC,IAAI;IAAC;IAAW;IAAa;IAAS;CAAS;AAEvE,4EAA4E;AAC5E,2IAA2I;AAC3I,SAASC,iBAAiBC,QAAiB;IACzC,MAAMC,OAAO,CAACC;QACZ,MAAM,IAAIP,aAAa,kBAAkBO;IAC3C;IACA,IAAI,OAAOF,aAAa,YAAYA,aAAa,MAAMC,KAAK;IAC5D,MAAME,IAAIH;IACV,IAAI,OAAOG,EAAEC,MAAM,KAAK,YAAYD,EAAEC,MAAM,KAAK,MAAMH,KAAK;IAC5D,MAAMI,OAAO,AAACF,EAAEC,MAAM,CAA6BC,IAAI;IACvD,IAAIA,SAAS,UAAUA,SAAS,SAASA,SAAS,SAASJ,KAAK,CAAC,4DAA4D,EAAEK,KAAKC,SAAS,CAACF,MAAM,CAAC,CAAC;IACtJ,IAAI,CAACG,MAAMC,OAAO,CAACN,EAAEO,KAAK,GAAGT,KAAK;IAElC,KAAK,MAAM,CAACU,IAAIC,KAAK,IAAI,AAACT,EAAEO,KAAK,CAAeG,OAAO,GAAI;QACzD,IAAI,OAAOD,SAAS,YAAYA,SAAS,MAAMX,KAAK,CAAC,eAAe,EAAEU,GAAG,mBAAmB,CAAC;QAC7F,MAAMG,IAAIF;QACV,IAAI,OAAOE,EAAEF,IAAI,KAAK,UAAUX,KAAK,CAAC,eAAe,EAAEU,GAAG,uBAAuB,CAAC;QAClF,IAAIG,EAAET,IAAI,KAAK,UAAUS,EAAET,IAAI,KAAK,OAAOJ,KAAK,CAAC,eAAe,EAAEU,GAAG,oCAAoC,EAAEL,KAAKC,SAAS,CAACO,EAAET,IAAI,EAAE,CAAC,CAAC;QACpI,IAAI,CAACG,MAAMC,OAAO,CAACK,EAAEC,KAAK,GAAGd,KAAK,CAAC,eAAe,EAAEU,GAAG,wBAAwB,CAAC;QAChF,KAAK,MAAM,CAACK,IAAIC,EAAE,IAAI,AAACH,EAAEC,KAAK,CAAeF,OAAO,GAAI;YACtD,IAAI,OAAOI,MAAM,YAAYA,MAAM,QAAQ,OAAO,AAACA,EAA8BC,IAAI,KAAK,UAAU;gBAClGjB,KAAK,CAAC,eAAe,EAAEU,GAAG,QAAQ,EAAEK,GAAG,wCAAwC,CAAC;YAClF;QACF;QACA,MAAMG,YAAY,AAACL,EAAEC,KAAK,CAAeK,MAAM;QAE/C,IAAI,CAACZ,MAAMC,OAAO,CAACK,EAAEO,MAAM,GAAGpB,KAAK,CAAC,eAAe,EAAEU,GAAG,yBAAyB,CAAC;QAClF,KAAK,MAAM,CAACW,IAAIC,EAAE,IAAI,AAACT,EAAEO,MAAM,CAAeR,OAAO,GAAI;YACvD,IAAI,OAAOU,MAAM,YAAYA,MAAM,MAAMtB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,mBAAmB,CAAC;YACrG,MAAME,QAAQD;YACd,IAAI,OAAOC,MAAMC,IAAI,KAAK,YAAY,CAAC5B,YAAY6B,GAAG,CAACF,MAAMC,IAAI,GAAG;gBAClExB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,0DAA0D,EAAEhB,KAAKC,SAAS,CAACiB,MAAMC,IAAI,EAAE,CAAC,CAAC;YACnI;YACA,IAAI,OAAOD,MAAMN,IAAI,KAAK,UAAUjB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,uBAAuB,CAAC;YACpG,wEAAwE;YACxE,mGAAmG;YACnG,IAAIE,MAAMC,IAAI,KAAK,WAAW;gBAC5B,IAAI,CAACE,OAAOC,SAAS,CAACJ,MAAMK,KAAK,KAAK,AAACL,MAAMK,KAAK,GAAc,KAAK,AAACL,MAAMK,KAAK,GAAc,GAAG;oBAChG5B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,uDAAuD,EAAEhB,KAAKC,SAAS,CAACiB,MAAMK,KAAK,EAAE,CAAC,CAAC;gBACjI;YACF,OAAO,IAAIL,MAAMK,KAAK,KAAKC,WAAW;gBACpC7B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,iDAAiD,EAAEhB,KAAKC,SAAS,CAACiB,MAAMC,IAAI,EAAE,kBAAkB,EAAEnB,KAAKC,SAAS,CAACiB,MAAMK,KAAK,EAAE,CAAC,CAAC;YAC1K;YACA,IAAIL,MAAMO,KAAK,KAAKD,aAAa,OAAON,MAAMO,KAAK,KAAK,WAAW9B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,oCAAoC,CAAC;YAChJ,IAAI,CAACd,MAAMC,OAAO,CAACe,MAAMQ,WAAW,GAAG/B,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,8BAA8B,CAAC;YAC9G,KAAK,MAAMW,OAAOT,MAAMQ,WAAW,CAAe;gBAChD,IAAI,OAAOC,QAAQ,YAAY,CAACN,OAAOC,SAAS,CAACK,QAAQA,MAAM,KAAKA,OAAOd,WAAW;oBACpFlB,KAAK,CAAC,eAAe,EAAEU,GAAG,SAAS,EAAEW,GAAG,yCAAyC,EAAEhB,KAAKC,SAAS,CAAC0B,KAAK,OAAO,EAAEtB,GAAG,KAAK,EAAEQ,UAAU,OAAO,CAAC;gBAC9I;YACF;QACF;IACF;AACF;AAEA,yFAAyF;AACzF,SAASe,mBAAmBV,KAAoB;IAC9C,oEAAoE;IACpE,gFAAgF;IAChF,IAAIA,MAAMC,IAAI,KAAK,WAAW,OAAO,GAAG,IAAIU,MAAM,CAACX,MAAMK,KAAK,EAAY,CAAC,EAAEL,MAAMN,IAAI,EAAE;IACzF,OAAOM,MAAMN,IAAI;AACnB;AAEA;wHACwH,GACxH,OAAO,SAASkB,sBAAsB1B,KAAqB,EAAE2B,IAAyD;IACpH,IAAIC,KAAK;IACT,KAAK,MAAM1B,QAAQF,MAAO;QACxB,IAAI2B,KAAKE,gBAAgB,EAAED,MAAM,CAAC,EAAE,EAAE1C,eAAegB,KAAKA,IAAI,EAAE,IAAI,CAAC;aAChE,IAAIyB,KAAKG,WAAW,EAAEF,MAAM,GAAG1C,eAAegB,KAAKA,IAAI,EAAE,IAAI,CAAC;QACnE,KAAK,MAAMY,SAASZ,KAAKS,MAAM,CAAEiB,MAAM,GAAGJ,mBAAmBV,SAASA,MAAMO,KAAK,GAAG,OAAO,QAAQ;IACrG;IACA,OAAOO;AACT;AAEA,6EAA6E;AAC7E,2EAA2E;AAC3E,SAASG,cAAcjB,KAAoB;IACzC,IAAIA,MAAMC,IAAI,KAAK,WAAW,OAAOD,MAAMN,IAAI;IAC/C,IAAIM,MAAMC,IAAI,KAAK,UAAU,OAAOD,MAAMN,IAAI,CAACwB,OAAO,CAAC,0BAA0B;IACjF,IAAIlB,MAAMC,IAAI,KAAK,SAAS,OAAOD,MAAMN,IAAI,CAACwB,OAAO,CAAC,kBAAkB;IACxE,OAAOlB,MAAMN,IAAI,CAACwB,OAAO,CAAC,OAAO;AACnC;AAEA,OAAO,SAASC,kBAAkBjC,KAAqB;IACrD,MAAMkC,MAAgB,EAAE;IACxB,KAAK,MAAMhC,QAAQF,MAAO;QACxB,KAAK,MAAMc,SAASZ,KAAKS,MAAM,CAAEuB,IAAIC,IAAI,CAACJ,cAAcjB;IAC1D;IACA,OAAOoB,IAAIxB,MAAM,GAAG,GAAGwB,IAAIE,IAAI,CAAC,QAAQC,IAAI,GAAG,EAAE,CAAC,GAAG;AACvD;AAEA;;;;;;;;CAQC,GACD,OAAO,SAASC,WAAWhD,QAAkB,EAAEqC,OAAkC,CAAC,CAAC;IACjFtC,iBAAiBC;IACjB,MAAMiD,QAAQjD,SAASI,MAAM,CAACC,IAAI,KAAK;IACvC,MAAMiC,KAAKF,sBAAsBpC,SAASU,KAAK,EAAE;QAAE6B,kBAAkBU;QAAOT,aAAa,CAAC,CAACH,KAAKG,WAAW;IAAC;IAC5G,OAAOS,QAAQX,KAAK,GAAGA,GAAGS,IAAI,GAAG,EAAE,CAAC;AACtC;AAEA;;;;;CAKC,GACD,OAAO,SAASG,OAAOlD,QAAkB;IACvCD,iBAAiBC;IACjB,OAAO2C,kBAAkB3C,SAASU,KAAK;AACzC"}
|
package/dist/esm/report.d.ts
CHANGED
|
@@ -3,10 +3,10 @@ export declare const SPARSE_CHARS_PER_PAGE = 200;
|
|
|
3
3
|
export declare const AUTO_OCR_CHARS_PER_PAGE = 20;
|
|
4
4
|
/** A manually-bumped integer, deliberately not the npm package version —
|
|
5
5
|
* bump whenever a change can alter output for identical input. See `CHANGELOG.md` for history. */
|
|
6
|
-
export declare const OUTPUT_VERSION =
|
|
6
|
+
export declare const OUTPUT_VERSION = 2;
|
|
7
7
|
/** Below this per-word confidence (0-100, tesseract.js's `confidence`), a
|
|
8
8
|
* word counts as low-confidence. Measured: junk max 43 vs prose min 76, cut 60, 2026-08-17. */
|
|
9
9
|
export declare const LOW_CONFIDENCE_THRESHOLD = 60;
|
|
10
10
|
export declare const LOW_CONFIDENCE_SAMPLE_CAP = 12;
|
|
11
11
|
export declare function buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages }: BuildReportInput): Report;
|
|
12
|
-
export declare function buildOcrReport({ md, fallback, words }: BuildOcrReportInput): Report;
|
|
12
|
+
export declare function buildOcrReport({ md, fallback, words, pagePaths, hybridPages }: BuildOcrReportInput): Report;
|
package/dist/esm/report.js
CHANGED
|
@@ -5,7 +5,7 @@ import { countPageMarkers, isPageMarkerLine, splitByPageMarker } from './page-ma
|
|
|
5
5
|
export const SPARSE_CHARS_PER_PAGE = 200;
|
|
6
6
|
export const AUTO_OCR_CHARS_PER_PAGE = 20;
|
|
7
7
|
/** A manually-bumped integer, deliberately not the npm package version —
|
|
8
|
-
* bump whenever a change can alter output for identical input. See `CHANGELOG.md` for history. */ export const OUTPUT_VERSION =
|
|
8
|
+
* bump whenever a change can alter output for identical input. See `CHANGELOG.md` for history. */ export const OUTPUT_VERSION = 2;
|
|
9
9
|
// A body paragraph line may end in sentence/clause punctuation, a closing
|
|
10
10
|
// quote/bracket, an ellipsis, or a digit (chart/label lines like "1/3" legitimately end in a number).
|
|
11
11
|
const TERMINAL_PUNCT = /[.!?:;"”'’)\]…0-9]$/;
|
|
@@ -96,14 +96,16 @@ function measurePage(body) {
|
|
|
96
96
|
//
|
|
97
97
|
// `hybridPages` (text path only) marks a row `largeImages: true` when that
|
|
98
98
|
// page used its text layer but also carries a large embedded image; omitted entirely on the OCR path, which has no equivalent signal.
|
|
99
|
-
function pageStats(md, hybridPages) {
|
|
99
|
+
function pageStats(md, path, hybridPages, pagePaths) {
|
|
100
100
|
const rows = [];
|
|
101
101
|
for (const { page, body } of splitByPageMarker(md)){
|
|
102
|
+
var _ref;
|
|
102
103
|
const row = {
|
|
103
104
|
page,
|
|
105
|
+
path: (_ref = pagePaths === null || pagePaths === void 0 ? void 0 : pagePaths.get(page)) !== null && _ref !== void 0 ? _ref : path,
|
|
104
106
|
...measurePage(body)
|
|
105
107
|
};
|
|
106
|
-
if (hybridPages) row.largeImages = hybridPages.includes(page);
|
|
108
|
+
if (hybridPages && row.path === 'text') row.largeImages = hybridPages.includes(page);
|
|
107
109
|
rows.push(row);
|
|
108
110
|
}
|
|
109
111
|
return rows;
|
|
@@ -111,7 +113,7 @@ function pageStats(md, hybridPages) {
|
|
|
111
113
|
export function buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages = [] }) {
|
|
112
114
|
const heads = (md.match(/^#{1,6}\s*(.+)$/gm) || []).filter((h)=>!isPageMarkerLine(h)); // page markers are provenance, not structure
|
|
113
115
|
const perPage = Math.round(md.length / numPages);
|
|
114
|
-
const perPageStats = pageStats(statsMd, hybridPages);
|
|
116
|
+
const perPageStats = pageStats(statsMd, 'text', hybridPages);
|
|
115
117
|
return {
|
|
116
118
|
outputVersion: OUTPUT_VERSION,
|
|
117
119
|
path: 'text',
|
|
@@ -140,11 +142,11 @@ export function buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes,
|
|
|
140
142
|
}
|
|
141
143
|
// QA report for the OCR path: measured from the emitted markdown text.
|
|
142
144
|
// `fallback` records that the text layer was tried first and found empty.
|
|
143
|
-
export function buildOcrReport({ md, fallback, words = [] }) {
|
|
145
|
+
export function buildOcrReport({ md, fallback, words = [], pagePaths, hybridPages }) {
|
|
144
146
|
const numPages = countPageMarkers(md);
|
|
145
147
|
const heads = (md.match(/^#{1,6}\s*(.+)$/gm) || []).filter((h)=>!isPageMarkerLine(h));
|
|
146
148
|
const perPage = numPages ? Math.round(md.length / numPages) : 0;
|
|
147
|
-
const perPageStats = pageStats(md);
|
|
149
|
+
const perPageStats = pageStats(md, 'ocr', hybridPages, pagePaths);
|
|
148
150
|
// Attach the low-confidence detector's per-page results after the fact:
|
|
149
151
|
// pageStats() has no confidence data (it reads markdown text only), so join it in here by page number.
|
|
150
152
|
const lowConf = lowConfidenceByPage(words);
|
|
@@ -155,9 +157,12 @@ export function buildOcrReport({ md, fallback, words = [] }) {
|
|
|
155
157
|
if (entry.sample.length) row.lowConfidenceSample = entry.sample;
|
|
156
158
|
}
|
|
157
159
|
}
|
|
160
|
+
const mixed = pagePaths !== undefined && [
|
|
161
|
+
...pagePaths.values()
|
|
162
|
+
].some((path)=>path === 'text');
|
|
158
163
|
return {
|
|
159
164
|
outputVersion: OUTPUT_VERSION,
|
|
160
|
-
path: 'ocr',
|
|
165
|
+
path: mixed ? 'mixed' : 'ocr',
|
|
161
166
|
pages: numPages,
|
|
162
167
|
ocrFallback: !!fallback,
|
|
163
168
|
chars: md.length,
|
package/dist/esm/report.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/report.ts"],"sourcesContent":["// report.ts — QA report over an extraction, machine-readable with --json.\nimport { countPageMarkers, isPageMarkerLine, splitByPageMarker } from './page-marker.ts';\nimport type { BuildOcrReportInput, BuildReportInput, PageStatsRow, Report, WordConfidenceInput } from './types.ts';\n\n// < SPARSE warns only; < AUTO_OCR means the pages are certainly images (measured:\n// an image-only 100-pager yields ~4 chars/page) and the auto-OCR fallback fires.\nexport const SPARSE_CHARS_PER_PAGE = 200;\nexport const AUTO_OCR_CHARS_PER_PAGE = 20;\n\n/** A manually-bumped integer, deliberately not the npm package version —\n * bump whenever a change can alter output for identical input. See `CHANGELOG.md` for history. */\nexport const OUTPUT_VERSION = 1;\n\n// A body paragraph line may end in sentence/clause punctuation, a closing\n// quote/bracket, an ellipsis, or a digit (chart/label lines like \"1/3\" legitimately end in a number).\nconst TERMINAL_PUNCT = /[.!?:;\"”'’)\\]…0-9]$/;\n\n// Lines that are not body prose at all: headings (incl. \"### pN\" markers),\n// floats/blockquotes, table rows, and list items — excluded before TERMINAL_PUNCT runs.\nconst NON_PROSE_LINE = /^[#>|*-]/;\n\n// A `> [review: reason] text` line (src/emit.ts). Shared by pageStats (per\n// page) and both top-level report builders (whole doc) so the counts can't drift apart.\nconst REVIEW_LINE = /^> \\[review: /gm;\n\n// Dangling lines split by length: long ones are the invisible column-break/splice\n// signal; short ones are usually a missed heading or an unjoined label.\nconst DANGLING_LONG_MIN = 60;\n\n/** Below this per-word confidence (0-100, tesseract.js's `confidence`), a\n * word counts as low-confidence. Measured: junk max 43 vs prose min 76, cut 60, 2026-08-17. */\nexport const LOW_CONFIDENCE_THRESHOLD = 60;\n\n// Cap on `lowConfidenceSample`'s length per page — a few representative\n// offenders, not the whole page re-emitted a second time.\nexport const LOW_CONFIDENCE_SAMPLE_CAP = 12;\n\ninterface LowConfidenceStats {\n measured: number; // words on this page that carried a confidence value at all\n low: number; // of those, how many scored below LOW_CONFIDENCE_THRESHOLD\n sample: string[]; // worst-confidence-first, capped list of the actual low-confidence tokens\n}\n\n// Grouped from the flat recognized-word list (confidence is per-word, never\n// merged into a line — see Word's comment). `measured` is tracked separately from `low` so a page with nothing to measure reports ABSENT, not a lying zero.\nfunction lowConfidenceByPage(words: WordConfidenceInput[]): Map<number, LowConfidenceStats> {\n const byPage = new Map<number, { measured: number; low: number; lowWords: { text: string; confidence: number }[] }>();\n for (const w of words) {\n if (w.confidence === undefined) continue;\n const entry = byPage.get(w.page) ?? { measured: 0, low: 0, lowWords: [] };\n entry.measured++;\n if (w.confidence < LOW_CONFIDENCE_THRESHOLD) {\n entry.low++;\n entry.lowWords.push({ text: w.text, confidence: w.confidence });\n }\n byPage.set(w.page, entry);\n }\n const result = new Map<number, LowConfidenceStats>();\n for (const [page, entry] of byPage) {\n // Sort ascending by confidence BEFORE capping, so the cap keeps the\n // worst offenders regardless of appearance order.\n const sample = entry.lowWords\n .slice()\n .sort((a, b) => a.confidence - b.confidence)\n .slice(0, LOW_CONFIDENCE_SAMPLE_CAP)\n .map((w) => w.text);\n result.set(page, { measured: entry.measured, low: entry.low, sample });\n }\n return result;\n}\n\ninterface DanglingCounts {\n long: number;\n short: number;\n}\n\n// Counts of body paragraph LINES ending WITHOUT terminal punctuation, split\n// by length: `long` is the invisible column-break/splice signal; `short` is usually a missed heading or unjoined label.\nfunction danglingCounts(body: string): DanglingCounts {\n let long = 0;\n let short = 0;\n for (const raw of body.split('\\n')) {\n const line = raw.trim();\n if (!line || NON_PROSE_LINE.test(line)) continue;\n if (TERMINAL_PUNCT.test(line)) continue;\n if (line.length >= DANGLING_LONG_MIN) long++;\n else short++;\n }\n return { long, short };\n}\n\n// One page's own measured stats, derivable from its body text alone —\n// shared by `pageStats` and any future caller that already has a page's body text in hand.\nfunction measurePage(body: string): Omit<PageStatsRow, 'page' | 'largeImages' | 'lowConfidenceWords' | 'lowConfidenceSample'> {\n const heads = body.match(/^#{1,6}\\s*(.+)$/gm) || [];\n const dangling = danglingCounts(body);\n return {\n chars: body.length,\n headings: heads.length,\n junkHeadings: heads.filter((h) => !/[A-Za-z]{3}/.test(h)).length,\n floats: (body.match(/^> \\[floats\\]/gm) || []).length,\n paragraphs: (body.match(/^(?![#>-])\\S.*$/gm) || []).length,\n danglingLong: dangling.long,\n danglingShort: dangling.short,\n reviewBlocks: (body.match(REVIEW_LINE) || []).length,\n };\n}\n\n// Per-page QA rows, one per \"### pN\" block. The text path always passes\n// markdown with markers forced on (`statsMd`) regardless of `--page-markers`, so pageStats is always populated.\n//\n// `hybridPages` (text path only) marks a row `largeImages: true` when that\n// page used its text layer but also carries a large embedded image; omitted entirely on the OCR path, which has no equivalent signal.\nfunction pageStats(md: string, hybridPages?: number[]): PageStatsRow[] {\n const rows: PageStatsRow[] = [];\n for (const { page, body } of splitByPageMarker(md)) {\n const row: PageStatsRow = { page, ...measurePage(body) };\n if (hybridPages) row.largeImages = hybridPages.includes(page);\n rows.push(row);\n }\n return rows;\n}\n\nexport function buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages = [] }: BuildReportInput): Report {\n const heads = (md.match(/^#{1,6}\\s*(.+)$/gm) || []).filter((h) => !isPageMarkerLine(h)); // page markers are provenance, not structure\n const perPage = Math.round(md.length / numPages);\n const perPageStats = pageStats(statsMd, hybridPages);\n return {\n outputVersion: OUTPUT_VERSION,\n path: 'text',\n pages: numPages,\n bodyFontHeight: bodyH,\n headingSizes: headingSizes.slice(0, 6),\n chars: md.length,\n charsPerPage: perPage,\n sparse: perPage < SPARSE_CHARS_PER_PAGE,\n headings: heads.length,\n junkHeadings: heads.filter((h) => !/[A-Za-z]{3}/.test(h)).length,\n listItems: stats.listItems,\n paragraphs: stats.paragraphs,\n joinedPairs: stats.joinedPairs,\n callouts: stats.callouts,\n closedHyphens: stats.closedHyphens,\n danglingLong: perPageStats.reduce((sum, r) => sum + r.danglingLong, 0),\n danglingShort: perPageStats.reduce((sum, r) => sum + r.danglingShort, 0),\n reviewBlocks: perPageStats.reduce((sum, r) => sum + r.reviewBlocks, 0),\n // The text path never sees per-word confidence at all; 0 here means\n // \"nothing to measure\" (see PageStatsRow's field doc), not \"measured 0\".\n lowConfidenceWords: 0,\n hybridPages,\n pageStats: perPageStats,\n };\n}\n\n// QA report for the OCR path: measured from the emitted markdown text.\n// `fallback` records that the text layer was tried first and found empty.\nexport function buildOcrReport({ md, fallback, words = [] }: BuildOcrReportInput): Report {\n const numPages = countPageMarkers(md);\n const heads = (md.match(/^#{1,6}\\s*(.+)$/gm) || []).filter((h) => !isPageMarkerLine(h));\n const perPage = numPages ? Math.round(md.length / numPages) : 0;\n const perPageStats = pageStats(md);\n // Attach the low-confidence detector's per-page results after the fact:\n // pageStats() has no confidence data (it reads markdown text only), so join it in here by page number.\n const lowConf = lowConfidenceByPage(words);\n for (const row of perPageStats) {\n const entry = lowConf.get(row.page);\n if (entry && entry.measured > 0) {\n row.lowConfidenceWords = entry.low;\n if (entry.sample.length) row.lowConfidenceSample = entry.sample;\n }\n }\n return {\n outputVersion: OUTPUT_VERSION,\n path: 'ocr',\n pages: numPages,\n ocrFallback: !!fallback,\n chars: md.length,\n charsPerPage: perPage,\n sparse: perPage < SPARSE_CHARS_PER_PAGE,\n headings: heads.length,\n junkHeadings: heads.filter((h) => !/[A-Za-z]{3}/.test(h)).length,\n paragraphs: (md.match(/^(?![#>-])\\S.*$/gm) || []).length,\n // Exclude BOTH \"[floats]\" and \"[review: ...]\" lines — a review marker is\n // a labeled unstructured block, not a genuine callout.\n callouts: (md.match(/^> (?!\\[floats\\]|\\[review:)/gm) || []).length,\n floats: (md.match(/^> \\[floats\\]/gm) || []).length,\n danglingLong: perPageStats.reduce((sum, r) => sum + r.danglingLong, 0),\n danglingShort: perPageStats.reduce((sum, r) => sum + r.danglingShort, 0),\n reviewBlocks: perPageStats.reduce((sum, r) => sum + r.reviewBlocks, 0),\n lowConfidenceWords: perPageStats.reduce((sum, r) => sum + (r.lowConfidenceWords ?? 0), 0),\n pageStats: perPageStats,\n };\n}\n"],"names":["countPageMarkers","isPageMarkerLine","splitByPageMarker","SPARSE_CHARS_PER_PAGE","AUTO_OCR_CHARS_PER_PAGE","OUTPUT_VERSION","TERMINAL_PUNCT","NON_PROSE_LINE","REVIEW_LINE","DANGLING_LONG_MIN","LOW_CONFIDENCE_THRESHOLD","LOW_CONFIDENCE_SAMPLE_CAP","lowConfidenceByPage","words","byPage","Map","w","confidence","undefined","entry","get","page","measured","low","lowWords","push","text","set","result","sample","slice","sort","a","b","map","danglingCounts","body","long","short","raw","split","line","trim","test","length","measurePage","heads","match","dangling","chars","headings","junkHeadings","filter","h","floats","paragraphs","danglingLong","danglingShort","reviewBlocks","pageStats","md","hybridPages","rows","row","largeImages","includes","buildReport","numPages","statsMd","stats","bodyH","headingSizes","perPage","Math","round","perPageStats","outputVersion","path","pages","bodyFontHeight","charsPerPage","sparse","listItems","joinedPairs","callouts","closedHyphens","reduce","sum","r","lowConfidenceWords","buildOcrReport","fallback","lowConf","lowConfidenceSample","ocrFallback"],"mappings":"AAAA,0EAA0E;AAC1E,SAASA,gBAAgB,EAAEC,gBAAgB,EAAEC,iBAAiB,QAAQ,mBAAmB;AAGzF,kFAAkF;AAClF,iFAAiF;AACjF,OAAO,MAAMC,wBAAwB,IAAI;AACzC,OAAO,MAAMC,0BAA0B,GAAG;AAE1C;gGACgG,GAChG,OAAO,MAAMC,iBAAiB,EAAE;AAEhC,0EAA0E;AAC1E,sGAAsG;AACtG,MAAMC,iBAAiB;AAEvB,2EAA2E;AAC3E,wFAAwF;AACxF,MAAMC,iBAAiB;AAEvB,2EAA2E;AAC3E,wFAAwF;AACxF,MAAMC,cAAc;AAEpB,kFAAkF;AAClF,wEAAwE;AACxE,MAAMC,oBAAoB;AAE1B;6FAC6F,GAC7F,OAAO,MAAMC,2BAA2B,GAAG;AAE3C,wEAAwE;AACxE,0DAA0D;AAC1D,OAAO,MAAMC,4BAA4B,GAAG;AAQ5C,4EAA4E;AAC5E,4JAA4J;AAC5J,SAASC,oBAAoBC,KAA4B;IACvD,MAAMC,SAAS,IAAIC;IACnB,KAAK,MAAMC,KAAKH,MAAO;YAEPC;QADd,IAAIE,EAAEC,UAAU,KAAKC,WAAW;QAChC,MAAMC,SAAQL,cAAAA,OAAOM,GAAG,CAACJ,EAAEK,IAAI,eAAjBP,yBAAAA,cAAsB;YAAEQ,UAAU;YAAGC,KAAK;YAAGC,UAAU,EAAE;QAAC;QACxEL,MAAMG,QAAQ;QACd,IAAIN,EAAEC,UAAU,GAAGP,0BAA0B;YAC3CS,MAAMI,GAAG;YACTJ,MAAMK,QAAQ,CAACC,IAAI,CAAC;gBAAEC,MAAMV,EAAEU,IAAI;gBAAET,YAAYD,EAAEC,UAAU;YAAC;QAC/D;QACAH,OAAOa,GAAG,CAACX,EAAEK,IAAI,EAAEF;IACrB;IACA,MAAMS,SAAS,IAAIb;IACnB,KAAK,MAAM,CAACM,MAAMF,MAAM,IAAIL,OAAQ;QAClC,oEAAoE;QACpE,kDAAkD;QAClD,MAAMe,SAASV,MAAMK,QAAQ,CAC1BM,KAAK,GACLC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEf,UAAU,GAAGgB,EAAEhB,UAAU,EAC1Ca,KAAK,CAAC,GAAGnB,2BACTuB,GAAG,CAAC,CAAClB,IAAMA,EAAEU,IAAI;QACpBE,OAAOD,GAAG,CAACN,MAAM;YAAEC,UAAUH,MAAMG,QAAQ;YAAEC,KAAKJ,MAAMI,GAAG;YAAEM;QAAO;IACtE;IACA,OAAOD;AACT;AAOA,4EAA4E;AAC5E,wHAAwH;AACxH,SAASO,eAAeC,IAAY;IAClC,IAAIC,OAAO;IACX,IAAIC,QAAQ;IACZ,KAAK,MAAMC,OAAOH,KAAKI,KAAK,CAAC,MAAO;QAClC,MAAMC,OAAOF,IAAIG,IAAI;QACrB,IAAI,CAACD,QAAQlC,eAAeoC,IAAI,CAACF,OAAO;QACxC,IAAInC,eAAeqC,IAAI,CAACF,OAAO;QAC/B,IAAIA,KAAKG,MAAM,IAAInC,mBAAmB4B;aACjCC;IACP;IACA,OAAO;QAAED;QAAMC;IAAM;AACvB;AAEA,sEAAsE;AACtE,2FAA2F;AAC3F,SAASO,YAAYT,IAAY;IAC/B,MAAMU,QAAQV,KAAKW,KAAK,CAAC,wBAAwB,EAAE;IACnD,MAAMC,WAAWb,eAAeC;IAChC,OAAO;QACLa,OAAOb,KAAKQ,MAAM;QAClBM,UAAUJ,MAAMF,MAAM;QACtBO,cAAcL,MAAMM,MAAM,CAAC,CAACC,IAAM,CAAC,cAAcV,IAAI,CAACU,IAAIT,MAAM;QAChEU,QAAQ,AAAClB,CAAAA,KAAKW,KAAK,CAAC,sBAAsB,EAAE,AAAD,EAAGH,MAAM;QACpDW,YAAY,AAACnB,CAAAA,KAAKW,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGH,MAAM;QAC1DY,cAAcR,SAASX,IAAI;QAC3BoB,eAAeT,SAASV,KAAK;QAC7BoB,cAAc,AAACtB,CAAAA,KAAKW,KAAK,CAACvC,gBAAgB,EAAE,AAAD,EAAGoC,MAAM;IACtD;AACF;AAEA,wEAAwE;AACxE,gHAAgH;AAChH,EAAE;AACF,2EAA2E;AAC3E,sIAAsI;AACtI,SAASe,UAAUC,EAAU,EAAEC,WAAsB;IACnD,MAAMC,OAAuB,EAAE;IAC/B,KAAK,MAAM,EAAEzC,IAAI,EAAEe,IAAI,EAAE,IAAIlC,kBAAkB0D,IAAK;QAClD,MAAMG,MAAoB;YAAE1C;YAAM,GAAGwB,YAAYT,KAAK;QAAC;QACvD,IAAIyB,aAAaE,IAAIC,WAAW,GAAGH,YAAYI,QAAQ,CAAC5C;QACxDyC,KAAKrC,IAAI,CAACsC;IACZ;IACA,OAAOD;AACT;AAEA,OAAO,SAASI,YAAY,EAAEC,QAAQ,EAAEP,EAAE,EAAEQ,OAAO,EAAEC,KAAK,EAAEC,KAAK,EAAEC,YAAY,EAAEV,cAAc,EAAE,EAAoB;IACnH,MAAMf,QAAQ,AAACc,CAAAA,GAAGb,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGK,MAAM,CAAC,CAACC,IAAM,CAACpD,iBAAiBoD,KAAK,6CAA6C;IACtI,MAAMmB,UAAUC,KAAKC,KAAK,CAACd,GAAGhB,MAAM,GAAGuB;IACvC,MAAMQ,eAAehB,UAAUS,SAASP;IACxC,OAAO;QACLe,eAAevE;QACfwE,MAAM;QACNC,OAAOX;QACPY,gBAAgBT;QAChBC,cAAcA,aAAazC,KAAK,CAAC,GAAG;QACpCmB,OAAOW,GAAGhB,MAAM;QAChBoC,cAAcR;QACdS,QAAQT,UAAUrE;QAClB+C,UAAUJ,MAAMF,MAAM;QACtBO,cAAcL,MAAMM,MAAM,CAAC,CAACC,IAAM,CAAC,cAAcV,IAAI,CAACU,IAAIT,MAAM;QAChEsC,WAAWb,MAAMa,SAAS;QAC1B3B,YAAYc,MAAMd,UAAU;QAC5B4B,aAAad,MAAMc,WAAW;QAC9BC,UAAUf,MAAMe,QAAQ;QACxBC,eAAehB,MAAMgB,aAAa;QAClC7B,cAAcmB,aAAaW,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAEhC,YAAY,EAAE;QACpEC,eAAekB,aAAaW,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAE/B,aAAa,EAAE;QACtEC,cAAciB,aAAaW,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAE9B,YAAY,EAAE;QACpE,oEAAoE;QACpE,yEAAyE;QACzE+B,oBAAoB;QACpB5B;QACAF,WAAWgB;IACb;AACF;AAEA,uEAAuE;AACvE,0EAA0E;AAC1E,OAAO,SAASe,eAAe,EAAE9B,EAAE,EAAE+B,QAAQ,EAAE9E,QAAQ,EAAE,EAAuB;IAC9E,MAAMsD,WAAWnE,iBAAiB4D;IAClC,MAAMd,QAAQ,AAACc,CAAAA,GAAGb,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGK,MAAM,CAAC,CAACC,IAAM,CAACpD,iBAAiBoD;IACpF,MAAMmB,UAAUL,WAAWM,KAAKC,KAAK,CAACd,GAAGhB,MAAM,GAAGuB,YAAY;IAC9D,MAAMQ,eAAehB,UAAUC;IAC/B,wEAAwE;IACxE,uGAAuG;IACvG,MAAMgC,UAAUhF,oBAAoBC;IACpC,KAAK,MAAMkD,OAAOY,aAAc;QAC9B,MAAMxD,QAAQyE,QAAQxE,GAAG,CAAC2C,IAAI1C,IAAI;QAClC,IAAIF,SAASA,MAAMG,QAAQ,GAAG,GAAG;YAC/ByC,IAAI0B,kBAAkB,GAAGtE,MAAMI,GAAG;YAClC,IAAIJ,MAAMU,MAAM,CAACe,MAAM,EAAEmB,IAAI8B,mBAAmB,GAAG1E,MAAMU,MAAM;QACjE;IACF;IACA,OAAO;QACL+C,eAAevE;QACfwE,MAAM;QACNC,OAAOX;QACP2B,aAAa,CAAC,CAACH;QACf1C,OAAOW,GAAGhB,MAAM;QAChBoC,cAAcR;QACdS,QAAQT,UAAUrE;QAClB+C,UAAUJ,MAAMF,MAAM;QACtBO,cAAcL,MAAMM,MAAM,CAAC,CAACC,IAAM,CAAC,cAAcV,IAAI,CAACU,IAAIT,MAAM;QAChEW,YAAY,AAACK,CAAAA,GAAGb,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGH,MAAM;QACxD,yEAAyE;QACzE,uDAAuD;QACvDwC,UAAU,AAACxB,CAAAA,GAAGb,KAAK,CAAC,oCAAoC,EAAE,AAAD,EAAGH,MAAM;QAClEU,QAAQ,AAACM,CAAAA,GAAGb,KAAK,CAAC,sBAAsB,EAAE,AAAD,EAAGH,MAAM;QAClDY,cAAcmB,aAAaW,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAEhC,YAAY,EAAE;QACpEC,eAAekB,aAAaW,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAE/B,aAAa,EAAE;QACtEC,cAAciB,aAAaW,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAE9B,YAAY,EAAE;QACpE+B,oBAAoBd,aAAaW,MAAM,CAAC,CAACC,KAAKC;gBAAaA;mBAAPD,QAAOC,wBAAAA,EAAEC,kBAAkB,cAApBD,mCAAAA,wBAAwB;WAAI;QACvF7B,WAAWgB;IACb;AACF"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/report.ts"],"sourcesContent":["// report.ts — QA report over an extraction, machine-readable with --json.\nimport { countPageMarkers, isPageMarkerLine, splitByPageMarker } from './page-marker.ts';\nimport type { BuildOcrReportInput, BuildReportInput, PageStatsRow, Report, WordConfidenceInput } from './types.ts';\n\n// < SPARSE warns only; < AUTO_OCR means the pages are certainly images (measured:\n// an image-only 100-pager yields ~4 chars/page) and the auto-OCR fallback fires.\nexport const SPARSE_CHARS_PER_PAGE = 200;\nexport const AUTO_OCR_CHARS_PER_PAGE = 20;\n\n/** A manually-bumped integer, deliberately not the npm package version —\n * bump whenever a change can alter output for identical input. See `CHANGELOG.md` for history. */\nexport const OUTPUT_VERSION = 2;\n\n// A body paragraph line may end in sentence/clause punctuation, a closing\n// quote/bracket, an ellipsis, or a digit (chart/label lines like \"1/3\" legitimately end in a number).\nconst TERMINAL_PUNCT = /[.!?:;\"”'’)\\]…0-9]$/;\n\n// Lines that are not body prose at all: headings (incl. \"### pN\" markers),\n// floats/blockquotes, table rows, and list items — excluded before TERMINAL_PUNCT runs.\nconst NON_PROSE_LINE = /^[#>|*-]/;\n\n// A `> [review: reason] text` line (src/emit.ts). Shared by pageStats (per\n// page) and both top-level report builders (whole doc) so the counts can't drift apart.\nconst REVIEW_LINE = /^> \\[review: /gm;\n\n// Dangling lines split by length: long ones are the invisible column-break/splice\n// signal; short ones are usually a missed heading or an unjoined label.\nconst DANGLING_LONG_MIN = 60;\n\n/** Below this per-word confidence (0-100, tesseract.js's `confidence`), a\n * word counts as low-confidence. Measured: junk max 43 vs prose min 76, cut 60, 2026-08-17. */\nexport const LOW_CONFIDENCE_THRESHOLD = 60;\n\n// Cap on `lowConfidenceSample`'s length per page — a few representative\n// offenders, not the whole page re-emitted a second time.\nexport const LOW_CONFIDENCE_SAMPLE_CAP = 12;\n\ninterface LowConfidenceStats {\n measured: number; // words on this page that carried a confidence value at all\n low: number; // of those, how many scored below LOW_CONFIDENCE_THRESHOLD\n sample: string[]; // worst-confidence-first, capped list of the actual low-confidence tokens\n}\n\n// Grouped from the flat recognized-word list (confidence is per-word, never\n// merged into a line — see Word's comment). `measured` is tracked separately from `low` so a page with nothing to measure reports ABSENT, not a lying zero.\nfunction lowConfidenceByPage(words: WordConfidenceInput[]): Map<number, LowConfidenceStats> {\n const byPage = new Map<number, { measured: number; low: number; lowWords: { text: string; confidence: number }[] }>();\n for (const w of words) {\n if (w.confidence === undefined) continue;\n const entry = byPage.get(w.page) ?? { measured: 0, low: 0, lowWords: [] };\n entry.measured++;\n if (w.confidence < LOW_CONFIDENCE_THRESHOLD) {\n entry.low++;\n entry.lowWords.push({ text: w.text, confidence: w.confidence });\n }\n byPage.set(w.page, entry);\n }\n const result = new Map<number, LowConfidenceStats>();\n for (const [page, entry] of byPage) {\n // Sort ascending by confidence BEFORE capping, so the cap keeps the\n // worst offenders regardless of appearance order.\n const sample = entry.lowWords\n .slice()\n .sort((a, b) => a.confidence - b.confidence)\n .slice(0, LOW_CONFIDENCE_SAMPLE_CAP)\n .map((w) => w.text);\n result.set(page, { measured: entry.measured, low: entry.low, sample });\n }\n return result;\n}\n\ninterface DanglingCounts {\n long: number;\n short: number;\n}\n\n// Counts of body paragraph LINES ending WITHOUT terminal punctuation, split\n// by length: `long` is the invisible column-break/splice signal; `short` is usually a missed heading or unjoined label.\nfunction danglingCounts(body: string): DanglingCounts {\n let long = 0;\n let short = 0;\n for (const raw of body.split('\\n')) {\n const line = raw.trim();\n if (!line || NON_PROSE_LINE.test(line)) continue;\n if (TERMINAL_PUNCT.test(line)) continue;\n if (line.length >= DANGLING_LONG_MIN) long++;\n else short++;\n }\n return { long, short };\n}\n\n// One page's own measured stats, derivable from its body text alone —\n// shared by `pageStats` and any future caller that already has a page's body text in hand.\nfunction measurePage(body: string): Omit<PageStatsRow, 'page' | 'path' | 'largeImages' | 'lowConfidenceWords' | 'lowConfidenceSample'> {\n const heads = body.match(/^#{1,6}\\s*(.+)$/gm) || [];\n const dangling = danglingCounts(body);\n return {\n chars: body.length,\n headings: heads.length,\n junkHeadings: heads.filter((h) => !/[A-Za-z]{3}/.test(h)).length,\n floats: (body.match(/^> \\[floats\\]/gm) || []).length,\n paragraphs: (body.match(/^(?![#>-])\\S.*$/gm) || []).length,\n danglingLong: dangling.long,\n danglingShort: dangling.short,\n reviewBlocks: (body.match(REVIEW_LINE) || []).length,\n };\n}\n\n// Per-page QA rows, one per \"### pN\" block. The text path always passes\n// markdown with markers forced on (`statsMd`) regardless of `--page-markers`, so pageStats is always populated.\n//\n// `hybridPages` (text path only) marks a row `largeImages: true` when that\n// page used its text layer but also carries a large embedded image; omitted entirely on the OCR path, which has no equivalent signal.\nfunction pageStats(md: string, path: 'text' | 'ocr', hybridPages?: number[], pagePaths?: Map<number, 'text' | 'ocr'>): PageStatsRow[] {\n const rows: PageStatsRow[] = [];\n for (const { page, body } of splitByPageMarker(md)) {\n const row: PageStatsRow = { page, path: pagePaths?.get(page) ?? path, ...measurePage(body) };\n if (hybridPages && row.path === 'text') row.largeImages = hybridPages.includes(page);\n rows.push(row);\n }\n return rows;\n}\n\nexport function buildReport({ numPages, md, statsMd, stats, bodyH, headingSizes, hybridPages = [] }: BuildReportInput): Report {\n const heads = (md.match(/^#{1,6}\\s*(.+)$/gm) || []).filter((h) => !isPageMarkerLine(h)); // page markers are provenance, not structure\n const perPage = Math.round(md.length / numPages);\n const perPageStats = pageStats(statsMd, 'text', hybridPages);\n return {\n outputVersion: OUTPUT_VERSION,\n path: 'text',\n pages: numPages,\n bodyFontHeight: bodyH,\n headingSizes: headingSizes.slice(0, 6),\n chars: md.length,\n charsPerPage: perPage,\n sparse: perPage < SPARSE_CHARS_PER_PAGE,\n headings: heads.length,\n junkHeadings: heads.filter((h) => !/[A-Za-z]{3}/.test(h)).length,\n listItems: stats.listItems,\n paragraphs: stats.paragraphs,\n joinedPairs: stats.joinedPairs,\n callouts: stats.callouts,\n closedHyphens: stats.closedHyphens,\n danglingLong: perPageStats.reduce((sum, r) => sum + r.danglingLong, 0),\n danglingShort: perPageStats.reduce((sum, r) => sum + r.danglingShort, 0),\n reviewBlocks: perPageStats.reduce((sum, r) => sum + r.reviewBlocks, 0),\n // The text path never sees per-word confidence at all; 0 here means\n // \"nothing to measure\" (see PageStatsRow's field doc), not \"measured 0\".\n lowConfidenceWords: 0,\n hybridPages,\n pageStats: perPageStats,\n };\n}\n\n// QA report for the OCR path: measured from the emitted markdown text.\n// `fallback` records that the text layer was tried first and found empty.\nexport function buildOcrReport({ md, fallback, words = [], pagePaths, hybridPages }: BuildOcrReportInput): Report {\n const numPages = countPageMarkers(md);\n const heads = (md.match(/^#{1,6}\\s*(.+)$/gm) || []).filter((h) => !isPageMarkerLine(h));\n const perPage = numPages ? Math.round(md.length / numPages) : 0;\n const perPageStats = pageStats(md, 'ocr', hybridPages, pagePaths);\n // Attach the low-confidence detector's per-page results after the fact:\n // pageStats() has no confidence data (it reads markdown text only), so join it in here by page number.\n const lowConf = lowConfidenceByPage(words);\n for (const row of perPageStats) {\n const entry = lowConf.get(row.page);\n if (entry && entry.measured > 0) {\n row.lowConfidenceWords = entry.low;\n if (entry.sample.length) row.lowConfidenceSample = entry.sample;\n }\n }\n const mixed = pagePaths !== undefined && [...pagePaths.values()].some((path) => path === 'text');\n return {\n outputVersion: OUTPUT_VERSION,\n path: mixed ? 'mixed' : 'ocr',\n pages: numPages,\n ocrFallback: !!fallback,\n chars: md.length,\n charsPerPage: perPage,\n sparse: perPage < SPARSE_CHARS_PER_PAGE,\n headings: heads.length,\n junkHeadings: heads.filter((h) => !/[A-Za-z]{3}/.test(h)).length,\n paragraphs: (md.match(/^(?![#>-])\\S.*$/gm) || []).length,\n // Exclude BOTH \"[floats]\" and \"[review: ...]\" lines — a review marker is\n // a labeled unstructured block, not a genuine callout.\n callouts: (md.match(/^> (?!\\[floats\\]|\\[review:)/gm) || []).length,\n floats: (md.match(/^> \\[floats\\]/gm) || []).length,\n danglingLong: perPageStats.reduce((sum, r) => sum + r.danglingLong, 0),\n danglingShort: perPageStats.reduce((sum, r) => sum + r.danglingShort, 0),\n reviewBlocks: perPageStats.reduce((sum, r) => sum + r.reviewBlocks, 0),\n lowConfidenceWords: perPageStats.reduce((sum, r) => sum + (r.lowConfidenceWords ?? 0), 0),\n pageStats: perPageStats,\n };\n}\n"],"names":["countPageMarkers","isPageMarkerLine","splitByPageMarker","SPARSE_CHARS_PER_PAGE","AUTO_OCR_CHARS_PER_PAGE","OUTPUT_VERSION","TERMINAL_PUNCT","NON_PROSE_LINE","REVIEW_LINE","DANGLING_LONG_MIN","LOW_CONFIDENCE_THRESHOLD","LOW_CONFIDENCE_SAMPLE_CAP","lowConfidenceByPage","words","byPage","Map","w","confidence","undefined","entry","get","page","measured","low","lowWords","push","text","set","result","sample","slice","sort","a","b","map","danglingCounts","body","long","short","raw","split","line","trim","test","length","measurePage","heads","match","dangling","chars","headings","junkHeadings","filter","h","floats","paragraphs","danglingLong","danglingShort","reviewBlocks","pageStats","md","path","hybridPages","pagePaths","rows","row","largeImages","includes","buildReport","numPages","statsMd","stats","bodyH","headingSizes","perPage","Math","round","perPageStats","outputVersion","pages","bodyFontHeight","charsPerPage","sparse","listItems","joinedPairs","callouts","closedHyphens","reduce","sum","r","lowConfidenceWords","buildOcrReport","fallback","lowConf","lowConfidenceSample","mixed","values","some","ocrFallback"],"mappings":"AAAA,0EAA0E;AAC1E,SAASA,gBAAgB,EAAEC,gBAAgB,EAAEC,iBAAiB,QAAQ,mBAAmB;AAGzF,kFAAkF;AAClF,iFAAiF;AACjF,OAAO,MAAMC,wBAAwB,IAAI;AACzC,OAAO,MAAMC,0BAA0B,GAAG;AAE1C;gGACgG,GAChG,OAAO,MAAMC,iBAAiB,EAAE;AAEhC,0EAA0E;AAC1E,sGAAsG;AACtG,MAAMC,iBAAiB;AAEvB,2EAA2E;AAC3E,wFAAwF;AACxF,MAAMC,iBAAiB;AAEvB,2EAA2E;AAC3E,wFAAwF;AACxF,MAAMC,cAAc;AAEpB,kFAAkF;AAClF,wEAAwE;AACxE,MAAMC,oBAAoB;AAE1B;6FAC6F,GAC7F,OAAO,MAAMC,2BAA2B,GAAG;AAE3C,wEAAwE;AACxE,0DAA0D;AAC1D,OAAO,MAAMC,4BAA4B,GAAG;AAQ5C,4EAA4E;AAC5E,4JAA4J;AAC5J,SAASC,oBAAoBC,KAA4B;IACvD,MAAMC,SAAS,IAAIC;IACnB,KAAK,MAAMC,KAAKH,MAAO;YAEPC;QADd,IAAIE,EAAEC,UAAU,KAAKC,WAAW;QAChC,MAAMC,SAAQL,cAAAA,OAAOM,GAAG,CAACJ,EAAEK,IAAI,eAAjBP,yBAAAA,cAAsB;YAAEQ,UAAU;YAAGC,KAAK;YAAGC,UAAU,EAAE;QAAC;QACxEL,MAAMG,QAAQ;QACd,IAAIN,EAAEC,UAAU,GAAGP,0BAA0B;YAC3CS,MAAMI,GAAG;YACTJ,MAAMK,QAAQ,CAACC,IAAI,CAAC;gBAAEC,MAAMV,EAAEU,IAAI;gBAAET,YAAYD,EAAEC,UAAU;YAAC;QAC/D;QACAH,OAAOa,GAAG,CAACX,EAAEK,IAAI,EAAEF;IACrB;IACA,MAAMS,SAAS,IAAIb;IACnB,KAAK,MAAM,CAACM,MAAMF,MAAM,IAAIL,OAAQ;QAClC,oEAAoE;QACpE,kDAAkD;QAClD,MAAMe,SAASV,MAAMK,QAAQ,CAC1BM,KAAK,GACLC,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEf,UAAU,GAAGgB,EAAEhB,UAAU,EAC1Ca,KAAK,CAAC,GAAGnB,2BACTuB,GAAG,CAAC,CAAClB,IAAMA,EAAEU,IAAI;QACpBE,OAAOD,GAAG,CAACN,MAAM;YAAEC,UAAUH,MAAMG,QAAQ;YAAEC,KAAKJ,MAAMI,GAAG;YAAEM;QAAO;IACtE;IACA,OAAOD;AACT;AAOA,4EAA4E;AAC5E,wHAAwH;AACxH,SAASO,eAAeC,IAAY;IAClC,IAAIC,OAAO;IACX,IAAIC,QAAQ;IACZ,KAAK,MAAMC,OAAOH,KAAKI,KAAK,CAAC,MAAO;QAClC,MAAMC,OAAOF,IAAIG,IAAI;QACrB,IAAI,CAACD,QAAQlC,eAAeoC,IAAI,CAACF,OAAO;QACxC,IAAInC,eAAeqC,IAAI,CAACF,OAAO;QAC/B,IAAIA,KAAKG,MAAM,IAAInC,mBAAmB4B;aACjCC;IACP;IACA,OAAO;QAAED;QAAMC;IAAM;AACvB;AAEA,sEAAsE;AACtE,2FAA2F;AAC3F,SAASO,YAAYT,IAAY;IAC/B,MAAMU,QAAQV,KAAKW,KAAK,CAAC,wBAAwB,EAAE;IACnD,MAAMC,WAAWb,eAAeC;IAChC,OAAO;QACLa,OAAOb,KAAKQ,MAAM;QAClBM,UAAUJ,MAAMF,MAAM;QACtBO,cAAcL,MAAMM,MAAM,CAAC,CAACC,IAAM,CAAC,cAAcV,IAAI,CAACU,IAAIT,MAAM;QAChEU,QAAQ,AAAClB,CAAAA,KAAKW,KAAK,CAAC,sBAAsB,EAAE,AAAD,EAAGH,MAAM;QACpDW,YAAY,AAACnB,CAAAA,KAAKW,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGH,MAAM;QAC1DY,cAAcR,SAASX,IAAI;QAC3BoB,eAAeT,SAASV,KAAK;QAC7BoB,cAAc,AAACtB,CAAAA,KAAKW,KAAK,CAACvC,gBAAgB,EAAE,AAAD,EAAGoC,MAAM;IACtD;AACF;AAEA,wEAAwE;AACxE,gHAAgH;AAChH,EAAE;AACF,2EAA2E;AAC3E,sIAAsI;AACtI,SAASe,UAAUC,EAAU,EAAEC,IAAoB,EAAEC,WAAsB,EAAEC,SAAuC;IAClH,MAAMC,OAAuB,EAAE;IAC/B,KAAK,MAAM,EAAE3C,IAAI,EAAEe,IAAI,EAAE,IAAIlC,kBAAkB0D,IAAK;;QAClD,MAAMK,MAAoB;YAAE5C;YAAMwC,IAAI,UAAEE,sBAAAA,gCAAAA,UAAW3C,GAAG,CAACC,4CAASwC;YAAM,GAAGhB,YAAYT,KAAK;QAAC;QAC3F,IAAI0B,eAAeG,IAAIJ,IAAI,KAAK,QAAQI,IAAIC,WAAW,GAAGJ,YAAYK,QAAQ,CAAC9C;QAC/E2C,KAAKvC,IAAI,CAACwC;IACZ;IACA,OAAOD;AACT;AAEA,OAAO,SAASI,YAAY,EAAEC,QAAQ,EAAET,EAAE,EAAEU,OAAO,EAAEC,KAAK,EAAEC,KAAK,EAAEC,YAAY,EAAEX,cAAc,EAAE,EAAoB;IACnH,MAAMhB,QAAQ,AAACc,CAAAA,GAAGb,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGK,MAAM,CAAC,CAACC,IAAM,CAACpD,iBAAiBoD,KAAK,6CAA6C;IACtI,MAAMqB,UAAUC,KAAKC,KAAK,CAAChB,GAAGhB,MAAM,GAAGyB;IACvC,MAAMQ,eAAelB,UAAUW,SAAS,QAAQR;IAChD,OAAO;QACLgB,eAAezE;QACfwD,MAAM;QACNkB,OAAOV;QACPW,gBAAgBR;QAChBC,cAAcA,aAAa3C,KAAK,CAAC,GAAG;QACpCmB,OAAOW,GAAGhB,MAAM;QAChBqC,cAAcP;QACdQ,QAAQR,UAAUvE;QAClB+C,UAAUJ,MAAMF,MAAM;QACtBO,cAAcL,MAAMM,MAAM,CAAC,CAACC,IAAM,CAAC,cAAcV,IAAI,CAACU,IAAIT,MAAM;QAChEuC,WAAWZ,MAAMY,SAAS;QAC1B5B,YAAYgB,MAAMhB,UAAU;QAC5B6B,aAAab,MAAMa,WAAW;QAC9BC,UAAUd,MAAMc,QAAQ;QACxBC,eAAef,MAAMe,aAAa;QAClC9B,cAAcqB,aAAaU,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAEjC,YAAY,EAAE;QACpEC,eAAeoB,aAAaU,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAEhC,aAAa,EAAE;QACtEC,cAAcmB,aAAaU,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAE/B,YAAY,EAAE;QACpE,oEAAoE;QACpE,yEAAyE;QACzEgC,oBAAoB;QACpB5B;QACAH,WAAWkB;IACb;AACF;AAEA,uEAAuE;AACvE,0EAA0E;AAC1E,OAAO,SAASc,eAAe,EAAE/B,EAAE,EAAEgC,QAAQ,EAAE/E,QAAQ,EAAE,EAAEkD,SAAS,EAAED,WAAW,EAAuB;IACtG,MAAMO,WAAWrE,iBAAiB4D;IAClC,MAAMd,QAAQ,AAACc,CAAAA,GAAGb,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGK,MAAM,CAAC,CAACC,IAAM,CAACpD,iBAAiBoD;IACpF,MAAMqB,UAAUL,WAAWM,KAAKC,KAAK,CAAChB,GAAGhB,MAAM,GAAGyB,YAAY;IAC9D,MAAMQ,eAAelB,UAAUC,IAAI,OAAOE,aAAaC;IACvD,wEAAwE;IACxE,uGAAuG;IACvG,MAAM8B,UAAUjF,oBAAoBC;IACpC,KAAK,MAAMoD,OAAOY,aAAc;QAC9B,MAAM1D,QAAQ0E,QAAQzE,GAAG,CAAC6C,IAAI5C,IAAI;QAClC,IAAIF,SAASA,MAAMG,QAAQ,GAAG,GAAG;YAC/B2C,IAAIyB,kBAAkB,GAAGvE,MAAMI,GAAG;YAClC,IAAIJ,MAAMU,MAAM,CAACe,MAAM,EAAEqB,IAAI6B,mBAAmB,GAAG3E,MAAMU,MAAM;QACjE;IACF;IACA,MAAMkE,QAAQhC,cAAc7C,aAAa;WAAI6C,UAAUiC,MAAM;KAAG,CAACC,IAAI,CAAC,CAACpC,OAASA,SAAS;IACzF,OAAO;QACLiB,eAAezE;QACfwD,MAAMkC,QAAQ,UAAU;QACxBhB,OAAOV;QACP6B,aAAa,CAAC,CAACN;QACf3C,OAAOW,GAAGhB,MAAM;QAChBqC,cAAcP;QACdQ,QAAQR,UAAUvE;QAClB+C,UAAUJ,MAAMF,MAAM;QACtBO,cAAcL,MAAMM,MAAM,CAAC,CAACC,IAAM,CAAC,cAAcV,IAAI,CAACU,IAAIT,MAAM;QAChEW,YAAY,AAACK,CAAAA,GAAGb,KAAK,CAAC,wBAAwB,EAAE,AAAD,EAAGH,MAAM;QACxD,yEAAyE;QACzE,uDAAuD;QACvDyC,UAAU,AAACzB,CAAAA,GAAGb,KAAK,CAAC,oCAAoC,EAAE,AAAD,EAAGH,MAAM;QAClEU,QAAQ,AAACM,CAAAA,GAAGb,KAAK,CAAC,sBAAsB,EAAE,AAAD,EAAGH,MAAM;QAClDY,cAAcqB,aAAaU,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAEjC,YAAY,EAAE;QACpEC,eAAeoB,aAAaU,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAEhC,aAAa,EAAE;QACtEC,cAAcmB,aAAaU,MAAM,CAAC,CAACC,KAAKC,IAAMD,MAAMC,EAAE/B,YAAY,EAAE;QACpEgC,oBAAoBb,aAAaU,MAAM,CAAC,CAACC,KAAKC;gBAAaA;mBAAPD,QAAOC,wBAAAA,EAAEC,kBAAkB,cAApBD,mCAAAA,wBAAwB;WAAI;QACvF9B,WAAWkB;IACb;AACF"}
|
package/dist/esm/resolve.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/resolve.ts"],"sourcesContent":["// Resolves an installed package file's on-disk path for both build targets:\n// ESM's import.meta.resolve returns a file:// URL, the CJS build's rewritten require.resolve a plain path — normalizes both to a plain path.\nimport { fileURLToPath } from 'node:url';\n\nexport function resolvePackagePath(specifier: string): string {\n const resolved = import.meta.resolve(specifier);\n return resolved.startsWith('file://') ? fileURLToPath(resolved) : resolved;\n}\n"],"names":["fileURLToPath","resolvePackagePath","specifier","resolved","resolve","startsWith"],"mappings":"AAAA,4EAA4E;AAC5E,6IAA6I;AAC7I,SAASA,aAAa,QAAQ,WAAW;AAEzC,OAAO,SAASC,mBAAmBC,SAAiB;IAClD,MAAMC,WAAW,YAAYC,OAAO,CAACF;IACrC,OAAOC,SAASE,UAAU,CAAC,aAAaL,cAAcG,YAAYA;AACpE"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { PageWord } from './geometry.js';
|
|
2
|
+
import type { Analysis, CollectResult } from './types.js';
|
|
3
|
+
export interface TextAnalysisResult {
|
|
4
|
+
analysis: Analysis;
|
|
5
|
+
markdown: string;
|
|
6
|
+
words: PageWord[];
|
|
7
|
+
}
|
|
8
|
+
export declare function selectCollectedPages(collected: CollectResult, pageNumbers: Set<number>): CollectResult;
|
|
9
|
+
/** Builds the text result once for Markdown, raw analysis, text callbacks,
|
|
10
|
+
* and reporting. The `pages` returned by buildMarkdown own the actual block
|
|
11
|
+
* construction, so this never reconstructs ownership from rendered text. */
|
|
12
|
+
export declare function buildTextAnalysis(collected: CollectResult, pageMarkers: boolean): TextAnalysisResult;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Shared text-layer analysis. It keeps PDF source runs separate from layout
|
|
2
|
+
// reconstruction, while Markdown blocks retain the source-run indexes that
|
|
3
|
+
// produced them.
|
|
4
|
+
import { toLines } from './lines.js';
|
|
5
|
+
import { buildMarkdown } from './markdown.js';
|
|
6
|
+
import { buildReport, OUTPUT_VERSION } from './report.js';
|
|
7
|
+
function sourceWord(glyph, dims) {
|
|
8
|
+
return {
|
|
9
|
+
text: glyph.s,
|
|
10
|
+
box: {
|
|
11
|
+
x: dims.width > 0 ? glyph.bounds.x / dims.width : 0,
|
|
12
|
+
y: dims.height > 0 ? (dims.height - glyph.bounds.y - glyph.bounds.h) / dims.height : 0,
|
|
13
|
+
w: dims.width > 0 ? glyph.bounds.w / dims.width : 0,
|
|
14
|
+
h: dims.height > 0 ? glyph.bounds.h / dims.height : 0
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function callbackWords(pages) {
|
|
19
|
+
return pages.flatMap((page)=>page.words.map((word)=>({
|
|
20
|
+
page: page.page,
|
|
21
|
+
text: word.text,
|
|
22
|
+
...word.box
|
|
23
|
+
})));
|
|
24
|
+
}
|
|
25
|
+
export function selectCollectedPages(collected, pageNumbers) {
|
|
26
|
+
const pages = collected.pages.filter((page)=>pageNumbers.has(page.page));
|
|
27
|
+
const heightChars = new Map();
|
|
28
|
+
for (const page of pages){
|
|
29
|
+
var _heightChars_get;
|
|
30
|
+
for (const glyph of page.glyphs)heightChars.set(glyph.h, ((_heightChars_get = heightChars.get(glyph.h)) !== null && _heightChars_get !== void 0 ? _heightChars_get : 0) + glyph.s.length);
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
numPages: pages.length,
|
|
34
|
+
pages,
|
|
35
|
+
heightChars
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** Builds the text result once for Markdown, raw analysis, text callbacks,
|
|
39
|
+
* and reporting. The `pages` returned by buildMarkdown own the actual block
|
|
40
|
+
* construction, so this never reconstructs ownership from rendered text. */ export function buildTextAnalysis(collected, pageMarkers) {
|
|
41
|
+
const pageLines = collected.pages.map((page)=>({
|
|
42
|
+
page: page.page,
|
|
43
|
+
lines: toLines(page)
|
|
44
|
+
}));
|
|
45
|
+
const rendered = buildMarkdown(pageLines, collected.heightChars, {
|
|
46
|
+
pageMarkers
|
|
47
|
+
});
|
|
48
|
+
const statsMd = pageMarkers ? rendered.md : buildMarkdown(pageLines, collected.heightChars, {
|
|
49
|
+
pageMarkers: true
|
|
50
|
+
}).md;
|
|
51
|
+
const hybridPages = collected.pages.filter((page)=>page.largeImage).map((page)=>page.page);
|
|
52
|
+
const report = buildReport({
|
|
53
|
+
numPages: collected.numPages,
|
|
54
|
+
md: rendered.md,
|
|
55
|
+
statsMd,
|
|
56
|
+
stats: rendered.stats,
|
|
57
|
+
bodyH: rendered.bodyH,
|
|
58
|
+
headingSizes: rendered.headingSizes,
|
|
59
|
+
hybridPages
|
|
60
|
+
});
|
|
61
|
+
const analysisPages = collected.pages.map((page)=>{
|
|
62
|
+
var _ref;
|
|
63
|
+
var _rendered_pages_find;
|
|
64
|
+
return {
|
|
65
|
+
page: page.page,
|
|
66
|
+
path: 'text',
|
|
67
|
+
words: page.glyphs.map((glyph)=>sourceWord(glyph, page)),
|
|
68
|
+
blocks: (_ref = (_rendered_pages_find = rendered.pages.find((renderedPage)=>renderedPage.page === page.page)) === null || _rendered_pages_find === void 0 ? void 0 : _rendered_pages_find.blocks) !== null && _ref !== void 0 ? _ref : []
|
|
69
|
+
};
|
|
70
|
+
});
|
|
71
|
+
const analysis = {
|
|
72
|
+
outputVersion: OUTPUT_VERSION,
|
|
73
|
+
report,
|
|
74
|
+
pages: analysisPages
|
|
75
|
+
};
|
|
76
|
+
return {
|
|
77
|
+
analysis,
|
|
78
|
+
markdown: rendered.md,
|
|
79
|
+
words: callbackWords(analysisPages)
|
|
80
|
+
};
|
|
81
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/open-source/formats/pdf-to-md/src/text-analysis.ts"],"sourcesContent":["// Shared text-layer analysis. It keeps PDF source runs separate from layout\n// reconstruction, while Markdown blocks retain the source-run indexes that\n// produced them.\nimport type { PageWord } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { buildReport, OUTPUT_VERSION } from './report.ts';\nimport type { Analysis, AnalysisPage, AnalysisWord, CollectResult, Glyph, PageLines } from './types.ts';\n\nfunction sourceWord(glyph: Glyph, dims: { width: number; height: number }): AnalysisWord {\n return {\n text: glyph.s,\n box: {\n x: dims.width > 0 ? glyph.bounds.x / dims.width : 0,\n y: dims.height > 0 ? (dims.height - glyph.bounds.y - glyph.bounds.h) / dims.height : 0,\n w: dims.width > 0 ? glyph.bounds.w / dims.width : 0,\n h: dims.height > 0 ? glyph.bounds.h / dims.height : 0,\n },\n };\n}\n\nfunction callbackWords(pages: AnalysisPage[]): PageWord[] {\n return pages.flatMap((page) => page.words.map((word) => ({ page: page.page, text: word.text, ...word.box })));\n}\n\nexport interface TextAnalysisResult {\n analysis: Analysis;\n markdown: string;\n words: PageWord[];\n}\n\nexport function selectCollectedPages(collected: CollectResult, pageNumbers: Set<number>): CollectResult {\n const pages = collected.pages.filter((page) => pageNumbers.has(page.page));\n const heightChars = new Map<number, number>();\n for (const page of pages) {\n for (const glyph of page.glyphs) heightChars.set(glyph.h, (heightChars.get(glyph.h) ?? 0) + glyph.s.length);\n }\n return { numPages: pages.length, pages, heightChars };\n}\n\n/** Builds the text result once for Markdown, raw analysis, text callbacks,\n * and reporting. The `pages` returned by buildMarkdown own the actual block\n * construction, so this never reconstructs ownership from rendered text. */\nexport function buildTextAnalysis(collected: CollectResult, pageMarkers: boolean): TextAnalysisResult {\n const pageLines: PageLines[] = collected.pages.map((page) => ({ page: page.page, lines: toLines(page) }));\n const rendered = buildMarkdown(pageLines, collected.heightChars, { pageMarkers });\n const statsMd = pageMarkers ? rendered.md : buildMarkdown(pageLines, collected.heightChars, { pageMarkers: true }).md;\n const hybridPages = collected.pages.filter((page) => page.largeImage).map((page) => page.page);\n const report = buildReport({\n numPages: collected.numPages,\n md: rendered.md,\n statsMd,\n stats: rendered.stats,\n bodyH: rendered.bodyH,\n headingSizes: rendered.headingSizes,\n hybridPages,\n });\n const analysisPages: AnalysisPage[] = collected.pages.map((page) => ({\n page: page.page,\n path: 'text',\n words: page.glyphs.map((glyph) => sourceWord(glyph, page)),\n blocks: rendered.pages.find((renderedPage) => renderedPage.page === page.page)?.blocks ?? [],\n }));\n const analysis: Analysis = { outputVersion: OUTPUT_VERSION, report, pages: analysisPages };\n return { analysis, markdown: rendered.md, words: callbackWords(analysisPages) };\n}\n"],"names":["toLines","buildMarkdown","buildReport","OUTPUT_VERSION","sourceWord","glyph","dims","text","s","box","x","width","bounds","y","height","h","w","callbackWords","pages","flatMap","page","words","map","word","selectCollectedPages","collected","pageNumbers","filter","has","heightChars","Map","glyphs","set","get","length","numPages","buildTextAnalysis","pageMarkers","pageLines","lines","rendered","statsMd","md","hybridPages","largeImage","report","stats","bodyH","headingSizes","analysisPages","path","blocks","find","renderedPage","analysis","outputVersion","markdown"],"mappings":"AAAA,4EAA4E;AAC5E,2EAA2E;AAC3E,iBAAiB;AAEjB,SAASA,OAAO,QAAQ,aAAa;AACrC,SAASC,aAAa,QAAQ,gBAAgB;AAC9C,SAASC,WAAW,EAAEC,cAAc,QAAQ,cAAc;AAG1D,SAASC,WAAWC,KAAY,EAAEC,IAAuC;IACvE,OAAO;QACLC,MAAMF,MAAMG,CAAC;QACbC,KAAK;YACHC,GAAGJ,KAAKK,KAAK,GAAG,IAAIN,MAAMO,MAAM,CAACF,CAAC,GAAGJ,KAAKK,KAAK,GAAG;YAClDE,GAAGP,KAAKQ,MAAM,GAAG,IAAI,AAACR,CAAAA,KAAKQ,MAAM,GAAGT,MAAMO,MAAM,CAACC,CAAC,GAAGR,MAAMO,MAAM,CAACG,CAAC,AAADA,IAAKT,KAAKQ,MAAM,GAAG;YACrFE,GAAGV,KAAKK,KAAK,GAAG,IAAIN,MAAMO,MAAM,CAACI,CAAC,GAAGV,KAAKK,KAAK,GAAG;YAClDI,GAAGT,KAAKQ,MAAM,GAAG,IAAIT,MAAMO,MAAM,CAACG,CAAC,GAAGT,KAAKQ,MAAM,GAAG;QACtD;IACF;AACF;AAEA,SAASG,cAAcC,KAAqB;IAC1C,OAAOA,MAAMC,OAAO,CAAC,CAACC,OAASA,KAAKC,KAAK,CAACC,GAAG,CAAC,CAACC,OAAU,CAAA;gBAAEH,MAAMA,KAAKA,IAAI;gBAAEb,MAAMgB,KAAKhB,IAAI;gBAAE,GAAGgB,KAAKd,GAAG;YAAC,CAAA;AAC3G;AAQA,OAAO,SAASe,qBAAqBC,SAAwB,EAAEC,WAAwB;IACrF,MAAMR,QAAQO,UAAUP,KAAK,CAACS,MAAM,CAAC,CAACP,OAASM,YAAYE,GAAG,CAACR,KAAKA,IAAI;IACxE,MAAMS,cAAc,IAAIC;IACxB,KAAK,MAAMV,QAAQF,MAAO;YACmCW;QAA3D,KAAK,MAAMxB,SAASe,KAAKW,MAAM,CAAEF,YAAYG,GAAG,CAAC3B,MAAMU,CAAC,EAAE,EAACc,mBAAAA,YAAYI,GAAG,CAAC5B,MAAMU,CAAC,eAAvBc,8BAAAA,mBAA4B,KAAKxB,MAAMG,CAAC,CAAC0B,MAAM;IAC5G;IACA,OAAO;QAAEC,UAAUjB,MAAMgB,MAAM;QAAEhB;QAAOW;IAAY;AACtD;AAEA;;0EAE0E,GAC1E,OAAO,SAASO,kBAAkBX,SAAwB,EAAEY,WAAoB;IAC9E,MAAMC,YAAyBb,UAAUP,KAAK,CAACI,GAAG,CAAC,CAACF,OAAU,CAAA;YAAEA,MAAMA,KAAKA,IAAI;YAAEmB,OAAOvC,QAAQoB;QAAM,CAAA;IACtG,MAAMoB,WAAWvC,cAAcqC,WAAWb,UAAUI,WAAW,EAAE;QAAEQ;IAAY;IAC/E,MAAMI,UAAUJ,cAAcG,SAASE,EAAE,GAAGzC,cAAcqC,WAAWb,UAAUI,WAAW,EAAE;QAAEQ,aAAa;IAAK,GAAGK,EAAE;IACrH,MAAMC,cAAclB,UAAUP,KAAK,CAACS,MAAM,CAAC,CAACP,OAASA,KAAKwB,UAAU,EAAEtB,GAAG,CAAC,CAACF,OAASA,KAAKA,IAAI;IAC7F,MAAMyB,SAAS3C,YAAY;QACzBiC,UAAUV,UAAUU,QAAQ;QAC5BO,IAAIF,SAASE,EAAE;QACfD;QACAK,OAAON,SAASM,KAAK;QACrBC,OAAOP,SAASO,KAAK;QACrBC,cAAcR,SAASQ,YAAY;QACnCL;IACF;IACA,MAAMM,gBAAgCxB,UAAUP,KAAK,CAACI,GAAG,CAAC,CAACF;;YAIjDoB;eAJ2D;YACnEpB,MAAMA,KAAKA,IAAI;YACf8B,MAAM;YACN7B,OAAOD,KAAKW,MAAM,CAACT,GAAG,CAAC,CAACjB,QAAUD,WAAWC,OAAOe;YACpD+B,MAAM,WAAEX,uBAAAA,SAAStB,KAAK,CAACkC,IAAI,CAAC,CAACC,eAAiBA,aAAajC,IAAI,KAAKA,KAAKA,IAAI,eAArEoB,2CAAAA,qBAAwEW,MAAM,uCAAI,EAAE;QAC9F;;IACA,MAAMG,WAAqB;QAAEC,eAAepD;QAAgB0C;QAAQ3B,OAAO+B;IAAc;IACzF,OAAO;QAAEK;QAAUE,UAAUhB,SAASE,EAAE;QAAErB,OAAOJ,cAAcgC;IAAe;AAChF"}
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -1,9 +1,18 @@
|
|
|
1
|
+
/** Viewport-point AABB for a source text run's transformed advance by
|
|
2
|
+
* font-height box. This is a run box, not fabricated per-token geometry. */
|
|
3
|
+
export interface GlyphBounds {
|
|
4
|
+
x: number;
|
|
5
|
+
y: number;
|
|
6
|
+
w: number;
|
|
7
|
+
h: number;
|
|
8
|
+
}
|
|
1
9
|
export interface Glyph {
|
|
2
10
|
s: string;
|
|
3
11
|
x: number;
|
|
4
12
|
y: number;
|
|
5
13
|
w: number;
|
|
6
14
|
h: number;
|
|
15
|
+
bounds: GlyphBounds;
|
|
7
16
|
}
|
|
8
17
|
export interface CollectedPage {
|
|
9
18
|
page: number;
|
|
@@ -11,6 +20,8 @@ export interface CollectedPage {
|
|
|
11
20
|
width: number;
|
|
12
21
|
height: number;
|
|
13
22
|
textChars: number;
|
|
23
|
+
/** any painted image operator, independent of text-layer density */
|
|
24
|
+
hasImage: boolean;
|
|
14
25
|
largeImage: boolean;
|
|
15
26
|
}
|
|
16
27
|
export interface CollectResult {
|
|
@@ -24,6 +35,9 @@ export interface Line {
|
|
|
24
35
|
y: number;
|
|
25
36
|
h: number;
|
|
26
37
|
col: number;
|
|
38
|
+
/** indexes of the collected PDF text runs that produced this line, in
|
|
39
|
+
* display order. OCR has its own source-word linkage. */
|
|
40
|
+
wordIndexes: number[];
|
|
27
41
|
}
|
|
28
42
|
export interface PageLines {
|
|
29
43
|
page: number;
|
|
@@ -46,11 +60,16 @@ export interface MarkdownResult {
|
|
|
46
60
|
stats: MarkdownStats;
|
|
47
61
|
bodyH: number;
|
|
48
62
|
headingSizes: number[];
|
|
63
|
+
/** The structural blocks that produced `md`, grouped by source page.
|
|
64
|
+
* Markers are rendering provenance and are deliberately not blocks. */
|
|
65
|
+
pages: MarkdownPage[];
|
|
49
66
|
}
|
|
50
67
|
/** One row of the per-page QA breakdown ("### pN" blocks) — used to decide
|
|
51
68
|
* which pages are worth rendering and re-checking (README's "Design contract"). */
|
|
52
69
|
export interface PageStatsRow {
|
|
53
70
|
page: number;
|
|
71
|
+
/** producer retained for this page, including in a mixed result */
|
|
72
|
+
path: 'text' | 'ocr';
|
|
54
73
|
chars: number;
|
|
55
74
|
headings: number;
|
|
56
75
|
junkHeadings: number;
|
|
@@ -92,6 +111,10 @@ export interface WordConfidenceInput {
|
|
|
92
111
|
export interface BuildOcrReportInput {
|
|
93
112
|
md: string;
|
|
94
113
|
fallback?: boolean;
|
|
114
|
+
/** Per-page producer map for a mixed result. Omit for OCR-only output. */
|
|
115
|
+
pagePaths?: Map<number, 'text' | 'ocr'>;
|
|
116
|
+
/** text pages retaining a large embedded image, for mixed per-page QA */
|
|
117
|
+
hybridPages?: number[];
|
|
95
118
|
/** every recognized word, for `report.ts`'s low-confidence detector.
|
|
96
119
|
* Defaults to [] (a pre-confidence --words-json replay), reporting the detector absent, never wrong. */
|
|
97
120
|
words?: WordConfidenceInput[];
|
|
@@ -102,7 +125,7 @@ export interface Report {
|
|
|
102
125
|
/** manually-bumped integer (not the npm package version) — see
|
|
103
126
|
* `report.ts`'s OUTPUT_VERSION for the bump rule; present on every Report so a frozen baseline's staleness is self-announcing. */
|
|
104
127
|
outputVersion: number;
|
|
105
|
-
path: 'text' | 'ocr';
|
|
128
|
+
path: 'text' | 'ocr' | 'mixed';
|
|
106
129
|
pages: number;
|
|
107
130
|
chars: number;
|
|
108
131
|
charsPerPage: number;
|
|
@@ -149,13 +172,20 @@ export interface AnalysisBlock {
|
|
|
149
172
|
/** Heading level (integer 1-6); REQUIRED when `type` is `'heading'`,
|
|
150
173
|
* FORBIDDEN otherwise — toMarkdown()/toText() validate this fail-fast (throws PdfToMdError/'ANALYSIS_INPUT'), never silently ignored. */
|
|
151
174
|
level?: number;
|
|
175
|
+
/** adjacent Markdown line with no blank separator, used for list items */
|
|
176
|
+
tight?: boolean;
|
|
152
177
|
wordIndexes: number[];
|
|
153
178
|
}
|
|
154
179
|
/** One page's flat words plus its structural blocks. */
|
|
155
180
|
export interface AnalysisPage {
|
|
156
181
|
page: number;
|
|
157
|
-
/**
|
|
158
|
-
*
|
|
182
|
+
/** producer retained for this page; `Analysis.report.path` is `'mixed'`
|
|
183
|
+
* when the document has more than one page producer. */
|
|
184
|
+
path: 'text' | 'ocr';
|
|
185
|
+
/** Flat, page-relative source items; `AnalysisBlock.wordIndexes` indexes
|
|
186
|
+
* into it. OCR boxes are normalized [0,1]. Text-run boxes are page
|
|
187
|
+
* fractions and may extend outside that range when the PDF run extends past
|
|
188
|
+
* the viewport. Both use a bottom-left origin with y up. */
|
|
159
189
|
words: AnalysisWord[];
|
|
160
190
|
blocks: AnalysisBlock[];
|
|
161
191
|
}
|
|
@@ -169,3 +199,10 @@ export interface Analysis {
|
|
|
169
199
|
report: Report;
|
|
170
200
|
pages: AnalysisPage[];
|
|
171
201
|
}
|
|
202
|
+
/** One page's blocks from the text Markdown builder. This is internal
|
|
203
|
+
* rendering evidence, but it is declared here because the shared text
|
|
204
|
+
* analysis consumes it without parsing Markdown back into structure. */
|
|
205
|
+
export interface MarkdownPage {
|
|
206
|
+
page: number;
|
|
207
|
+
blocks: AnalysisBlock[];
|
|
208
|
+
}
|