@effortlessmotion/pdf-to-md 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/package.json +1 -1
- package/skills/pdf-to-md/SKILL.md +39 -66
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/cli/extract.ts"],"sourcesContent":["// extract command — parse this command's flags, call src/extract.ts, print\n// and write the result. No business logic here — the library returns data and throws; only the CLI prints and exits.\nimport { readFileSync, writeFileSync } from 'node:fs';\nimport path from 'node:path';\nimport { parseArgs } from 'node:util';\nimport { analyze } from '../analyze.ts';\nimport type { OcrWordInput } from '../extract.ts';\nimport { extractOcr, needsOcrFallback, ocrFallbackNotice, pdfToMarkdown } from '../extract.ts';\nimport type { Tuning } from '../geometry.ts';\nimport { toText } from '../render-analysis.ts';\nimport type { Analysis, Report } from '../types.ts';\nimport type { Command } from './types.ts';\n\nconst USAGE =\n 'usage: pdf-to-md [extract] <file.pdf> [out.md] [--json] [--page-markers] [--ocr] [--no-ocr] ' +\n '[--pages N[-M]] [--stdout] [--words-json=FILE] [--debug-words=FILE] ' +\n '[--format md|txt|raw (default md; raw = Analysis JSON incl. the report; txt = reading-order plain text)] ' +\n '[--dpi=N (OCR render resolution, default 288; raise for tiny print or chart/label pages, ' +\n '>~400 measured worse on ordinary prose pages)] ' +\n '[--float-max-words=N] [--float-max-width=F] [--float-margin=F] [--para-gap=F] [--heading-scale=F] ' +\n '[--col-max-width=F] [--col-split=F] [--line-y-tol=F] [--line-height-ratio=F]';\n\nconst FORMATS = ['md', 'txt', 'raw'] as const;\ntype Format = (typeof FORMATS)[number];\n\n// A bad tuning-flag value (non-numeric, or <= 0 unless the flag allows zero) is a usage error, not a silent no-op.\nconst TUNING_FLAGS: {\n name: string;\n key: keyof Tuning;\n allowZero?: boolean;\n isInt?: boolean;\n}[] = [\n {\n name: 'float-max-words',\n key: 'floatMaxWords',\n allowZero: true,\n isInt: true,\n },\n { name: 'float-max-width', key: 'floatMaxWidth' },\n { name: 'float-margin', key: 'floatMargin' },\n { name: 'para-gap', key: 'paraGap' },\n { name: 'heading-scale', key: 'headingScale' },\n { name: 'col-max-width', key: 'colMaxWidth' },\n { name: 'col-split', key: 'colSplit' },\n { name: 'line-y-tol', key: 'lineYTol' },\n { name: 'line-height-ratio', key: 'lineHeightRatio' },\n];\n\nconst extract: Command = async (ctx) => {\n // --debug-words takes a required FILE value; a bare --debug-words is a\n // usage error, checked before parseArgs so it can't silently swallow the next token as its value.\n if (ctx.rest.includes('--debug-words')) {\n ctx.usageError(`${USAGE}\\n--debug-words requires a value: --debug-words=FILE`);\n }\n\n const parsed = (() => {\n try {\n return parseArgs({\n args: ctx.rest,\n options: {\n json: { type: 'boolean', default: false },\n 'page-markers': { type: 'boolean', default: false },\n ocr: { type: 'boolean', default: false },\n 'no-ocr': { type: 'boolean', default: false },\n stdout: { type: 'boolean', default: false },\n 'debug-words': { type: 'string' },\n help: { type: 'boolean', default: false, short: 'h' },\n pages: { type: 'string' },\n dpi: { type: 'string' },\n format: { type: 'string', default: 'md' },\n 'words-json': { type: 'string' },\n ...Object.fromEntries(TUNING_FLAGS.map((f) => [f.name, { type: 'string' as const }])),\n },\n allowPositionals: true,\n });\n } catch (err) {\n // usageError exits; the throw is dead code that tells the compiler so.\n throw ctx.usageError(`${(err as Error).message}\\n${USAGE}`);\n }\n })();\n const values = parsed.values as Record<string, string | boolean | undefined>;\n const positionals = parsed.positionals;\n\n if (values.help) {\n console.log(USAGE);\n return;\n }\n\n const [src, outArg] = positionals;\n const wordsPath = values['words-json'] as string | undefined;\n if (!src && !wordsPath) ctx.usageError(USAGE);\n\n const formatRaw = values.format as string;\n if (!(FORMATS as readonly string[]).includes(formatRaw)) {\n ctx.usageError(`${USAGE}\\n--format must be one of ${FORMATS.join('|')} (got ${JSON.stringify(formatRaw)})`);\n }\n const format = formatRaw as Format;\n\n // --pages: a malformed spec is a usage error; out-of-range page NUMBERS\n // clamp instead (collect.ts / the engine apply the same clamping policy).\n let pages: { first: number; last: number } | undefined;\n if (values.pages !== undefined) {\n const m = /^(\\d+)(?:-(\\d+))?$/.exec(values.pages as string);\n const first = m ? Number(m[1]) : Number.NaN;\n const last = m ? (m[2] !== undefined ? Number(m[2]) : first) : Number.NaN;\n if (!m || first < 1 || last < first) {\n ctx.usageError(`${USAGE}\\n--pages requires N or N-M, N >= 1, M >= N (got ${JSON.stringify(values.pages)})`);\n }\n pages = { first, last };\n }\n\n const tuning: Partial<Tuning> = {};\n for (const spec of TUNING_FLAGS) {\n const raw = values[spec.name] as string | undefined;\n if (raw === undefined) continue;\n const value = Number(raw);\n const ok = raw !== '' && Number.isFinite(value) && (spec.allowZero ? value >= 0 : value > 0);\n if (!ok) {\n ctx.usageError(`${USAGE}\\n--${spec.name} requires a ${spec.allowZero ? 'non-negative' : 'positive'} number (got ${JSON.stringify(raw)})`);\n }\n (tuning[spec.key] as number) = spec.isInt ? Math.trunc(value) : value;\n }\n\n // --dpi: page-render resolution, validated like the tuning flags. Default\n // 288 is the measured operating point; >~400 measured WORSE on ordinary pages — not a \"bigger is safer\" knob.\n let dpi: number | undefined;\n if (values.dpi !== undefined) {\n const raw = values.dpi as string;\n const value = Number(raw);\n if (raw === '' || !Number.isFinite(value) || value <= 0) {\n ctx.usageError(`${USAGE}\\n--dpi requires a positive number (got ${JSON.stringify(raw)})`);\n }\n dpi = value;\n }\n\n // In --words-json mode there may be no PDF positional; the words file\n // stands in for naming, extension stripped so the default dest can never overwrite the input. Default extension follows --format.\n const source = src ?? (wordsPath as string);\n const defaultExt = format === 'raw' ? 'json' : format === 'txt' ? 'txt' : 'md';\n const dest = outArg || `${source.replace(/\\.(pdf|jsonl)$/i, '')}.${defaultExt}`;\n\n const onWarning = (m: string) => console.error(m);\n const showProgress = process.stderr.isTTY;\n // onProgress carries per-page timing/count only; human-readable\n // diagnostics (e.g. the sparse-retry notice) arrive via onWarning instead, so they print regardless of TTY-ness.\n const onProgress = (e: { type: 'page'; page: number; words: number; ms: number }) => {\n if (showProgress) console.error(`p${e.page}: ${e.words} words, ${e.ms}ms`);\n };\n const debugWordsPath = values['debug-words'] as string | undefined;\n if (debugWordsPath !== undefined && debugWordsPath.trim() === '') {\n ctx.usageError(`${USAGE}\\n--debug-words requires a value: --debug-words=FILE`);\n }\n // The severity-1 audit instrument: every recognized word as clean JSON\n // lines to its own file. stderr stays a human channel (mixed prose), never a dump target — this file is guaranteed byte-for-byte parseable.\n const onWords = debugWordsPath\n ? (words: OcrWordInput[]) => {\n const lines = words.map((w) => {\n const entry: OcrWordInput = { page: w.page, text: w.text, x: w.x, y: w.y, w: w.w, h: w.h };\n if (w.confidence !== undefined) entry.confidence = w.confidence;\n return JSON.stringify(entry);\n });\n writeFileSync(debugWordsPath, lines.length ? `${lines.join('\\n')}\\n` : '');\n }\n : undefined;\n\n const common = { pages, onWarning };\n const ocrCommon = {\n ...common,\n tuning,\n dpi,\n onProgress,\n onWords,\n };\n\n let words: OcrWordInput[] | undefined;\n let skipped = 0;\n if (wordsPath !== undefined) {\n ({ words, skipped } = parseWordLines(readFileSync(wordsPath, 'utf8')));\n if (skipped > 0) console.error(`--words-json: skipped ${skipped} non-JSON line(s)`);\n }\n\n let content: string;\n let report: Report;\n if (format === 'md') {\n let markdown: string;\n if (words !== undefined) {\n ({ markdown, report } = await extractOcr({ words }, ocrCommon));\n } else {\n ({ markdown, report } = await pdfToMarkdown(src, {\n ...ocrCommon,\n ocr: values.ocr as boolean,\n noOcr: values['no-ocr'] as boolean,\n pageMarkers: values['page-markers'] as boolean,\n }));\n }\n content = markdown;\n } else {\n // raw/txt: both render from an Analysis — raw IS the Analysis as JSON;\n // txt is toText(analysis). Mirrors pdfToMarkdown's own auto-OCR fallback decision so a different --format doesn't silently pick a different path.\n let analysis: Analysis;\n if (words !== undefined) {\n analysis = await analyze({ words }, { ...ocrCommon, path: 'ocr' });\n } else if (values.ocr) {\n analysis = await analyze(src, { ...ocrCommon, path: 'ocr' });\n } else {\n const text = await analyze(src, { ...common, path: 'text', pageMarkers: values['page-markers'] as boolean });\n if (needsOcrFallback(text.report.charsPerPage) && !values['no-ocr']) {\n onWarning(ocrFallbackNotice(text.report.charsPerPage));\n analysis = await analyze(src, { ...ocrCommon, path: 'ocr', fallback: true });\n } else {\n analysis = text;\n }\n }\n report = analysis.report;\n content = format === 'raw' ? JSON.stringify(analysis, null, 1) : toText(analysis);\n }\n\n if (values.stdout) process.stdout.write(content);\n else writeFileSync(dest, content);\n\n // The library's Report stays clean of source/out; --json consumers keep\n // report.json files around and need to know which input produced each one, so the CLI (only) decorates the printed JSON with source/out here.\n if (values.json) console.log(JSON.stringify({ source, out: dest, ...report }, null, 1));\n else if (report.path === 'ocr') {\n console.error(`${path.basename(dest).padEnd(32)} OCR path heads=${report.headings}(junk ${report.junkHeadings}) ` + `paras=${report.paragraphs} callouts=${report.callouts} floats=${report.floats ?? 0} dangling=${report.danglingLong} (long) ` + `~${report.charsPerPage} chars/page`);\n } else {\n console.error(\n `${path.basename(dest).padEnd(32)} body=${report.bodyFontHeight}pt heads=${report.headings}(junk ${report.junkHeadings}) ` +\n `lists=${report.listItems} paras=${report.paragraphs} callouts=${report.callouts} pairs=${report.joinedPairs} ` +\n `~${report.charsPerPage} chars/page` +\n (report.sparse ? ' ⚠️ implausibly sparse — pages are likely images; re-run with --ocr' : '')\n );\n }\n};\n\n// Parse one word-dump JSON line per word — key order is not guaranteed, parse\n// never grep. Non-JSON lines are skipped with a count, not fatal (the auditor applies the same policy).\nfunction parseWordLines(raw: string): { words: OcrWordInput[]; skipped: number } {\n const words: OcrWordInput[] = [];\n let skipped = 0;\n for (const line of raw.split('\\n')) {\n if (!line.trim()) continue;\n try {\n words.push(JSON.parse(line));\n } catch {\n skipped++;\n }\n }\n return { words, skipped };\n}\n\nexport default extract;\n"],"names":["readFileSync","writeFileSync","path","parseArgs","analyze","extractOcr","needsOcrFallback","ocrFallbackNotice","pdfToMarkdown","toText","USAGE","FORMATS","TUNING_FLAGS","name","key","allowZero","isInt","extract","ctx","rest","includes","usageError","parsed","args","options","json","type","default","ocr","stdout","help","short","pages","dpi","format","Object","fromEntries","map","f","allowPositionals","err","message","values","positionals","console","log","src","outArg","wordsPath","formatRaw","join","JSON","stringify","undefined","m","exec","first","Number","NaN","last","tuning","spec","raw","value","ok","isFinite","Math","trunc","source","defaultExt","dest","replace","onWarning","error","showProgress","process","stderr","isTTY","onProgress","e","page","words","ms","debugWordsPath","trim","onWords","lines","w","entry","text","x","y","h","confidence","length","common","ocrCommon","skipped","parseWordLines","content","report","markdown","noOcr","pageMarkers","analysis","charsPerPage","fallback","write","out","basename","padEnd","headings","junkHeadings","paragraphs","callouts","floats","danglingLong","bodyFontHeight","listItems","joinedPairs","sparse","line","split","push","parse"],"mappings":"AAAA,2EAA2E;AAC3E,qHAAqH;AACrH,SAASA,YAAY,EAAEC,aAAa,QAAQ,UAAU;AACtD,OAAOC,UAAU,YAAY;AAC7B,SAASC,SAAS,QAAQ,YAAY;AACtC,SAASC,OAAO,QAAQ,gBAAgB;AAExC,SAASC,UAAU,EAAEC,gBAAgB,EAAEC,iBAAiB,EAAEC,aAAa,QAAQ,gBAAgB;AAE/F,SAASC,MAAM,QAAQ,wBAAwB;AAI/C,MAAMC,QACJ,iGACA,yEACA,8GACA,8FACA,oDACA,uGACA;AAEF,MAAMC,UAAU;IAAC;IAAM;IAAO;CAAM;AAGpC,mHAAmH;AACnH,MAAMC,eAKA;IACJ;QACEC,MAAM;QACNC,KAAK;QACLC,WAAW;QACXC,OAAO;IACT;IACA;QAAEH,MAAM;QAAmBC,KAAK;IAAgB;IAChD;QAAED,MAAM;QAAgBC,KAAK;IAAc;IAC3C;QAAED,MAAM;QAAYC,KAAK;IAAU;IACnC;QAAED,MAAM;QAAiBC,KAAK;IAAe;IAC7C;QAAED,MAAM;QAAiBC,KAAK;IAAc;IAC5C;QAAED,MAAM;QAAaC,KAAK;IAAW;IACrC;QAAED,MAAM;QAAcC,KAAK;IAAW;IACtC;QAAED,MAAM;QAAqBC,KAAK;IAAkB;CACrD;AAED,MAAMG,UAAmB,OAAOC;IAC9B,uEAAuE;IACvE,kGAAkG;IAClG,IAAIA,IAAIC,IAAI,CAACC,QAAQ,CAAC,kBAAkB;QACtCF,IAAIG,UAAU,CAAC,GAAGX,MAAM,oDAAoD,CAAC;IAC/E;IAEA,MAAMY,SAAS,AAAC,CAAA;QACd,IAAI;YACF,OAAOnB,UAAU;gBACfoB,MAAML,IAAIC,IAAI;gBACdK,SAAS;oBACPC,MAAM;wBAAEC,MAAM;wBAAWC,SAAS;oBAAM;oBACxC,gBAAgB;wBAAED,MAAM;wBAAWC,SAAS;oBAAM;oBAClDC,KAAK;wBAAEF,MAAM;wBAAWC,SAAS;oBAAM;oBACvC,UAAU;wBAAED,MAAM;wBAAWC,SAAS;oBAAM;oBAC5CE,QAAQ;wBAAEH,MAAM;wBAAWC,SAAS;oBAAM;oBAC1C,eAAe;wBAAED,MAAM;oBAAS;oBAChCI,MAAM;wBAAEJ,MAAM;wBAAWC,SAAS;wBAAOI,OAAO;oBAAI;oBACpDC,OAAO;wBAAEN,MAAM;oBAAS;oBACxBO,KAAK;wBAAEP,MAAM;oBAAS;oBACtBQ,QAAQ;wBAAER,MAAM;wBAAUC,SAAS;oBAAK;oBACxC,cAAc;wBAAED,MAAM;oBAAS;oBAC/B,GAAGS,OAAOC,WAAW,CAACxB,aAAayB,GAAG,CAAC,CAACC,IAAM;4BAACA,EAAEzB,IAAI;4BAAE;gCAAEa,MAAM;4BAAkB;yBAAE,EAAE;gBACvF;gBACAa,kBAAkB;YACpB;QACF,EAAE,OAAOC,KAAK;YACZ,uEAAuE;YACvE,MAAMtB,IAAIG,UAAU,CAAC,GAAG,AAACmB,IAAcC,OAAO,CAAC,EAAE,EAAE/B,OAAO;QAC5D;IACF,CAAA;IACA,MAAMgC,SAASpB,OAAOoB,MAAM;IAC5B,MAAMC,cAAcrB,OAAOqB,WAAW;IAEtC,IAAID,OAAOZ,IAAI,EAAE;QACfc,QAAQC,GAAG,CAACnC;QACZ;IACF;IAEA,MAAM,CAACoC,KAAKC,OAAO,GAAGJ;IACtB,MAAMK,YAAYN,MAAM,CAAC,aAAa;IACtC,IAAI,CAACI,OAAO,CAACE,WAAW9B,IAAIG,UAAU,CAACX;IAEvC,MAAMuC,YAAYP,OAAOR,MAAM;IAC/B,IAAI,CAAC,AAACvB,QAA8BS,QAAQ,CAAC6B,YAAY;QACvD/B,IAAIG,UAAU,CAAC,GAAGX,MAAM,0BAA0B,EAAEC,QAAQuC,IAAI,CAAC,KAAK,MAAM,EAAEC,KAAKC,SAAS,CAACH,WAAW,CAAC,CAAC;IAC5G;IACA,MAAMf,SAASe;IAEf,wEAAwE;IACxE,0EAA0E;IAC1E,IAAIjB;IACJ,IAAIU,OAAOV,KAAK,KAAKqB,WAAW;QAC9B,MAAMC,IAAI,qBAAqBC,IAAI,CAACb,OAAOV,KAAK;QAChD,MAAMwB,QAAQF,IAAIG,OAAOH,CAAC,CAAC,EAAE,IAAIG,OAAOC,GAAG;QAC3C,MAAMC,OAAOL,IAAKA,CAAC,CAAC,EAAE,KAAKD,YAAYI,OAAOH,CAAC,CAAC,EAAE,IAAIE,QAASC,OAAOC,GAAG;QACzE,IAAI,CAACJ,KAAKE,QAAQ,KAAKG,OAAOH,OAAO;YACnCtC,IAAIG,UAAU,CAAC,GAAGX,MAAM,iDAAiD,EAAEyC,KAAKC,SAAS,CAACV,OAAOV,KAAK,EAAE,CAAC,CAAC;QAC5G;QACAA,QAAQ;YAAEwB;YAAOG;QAAK;IACxB;IAEA,MAAMC,SAA0B,CAAC;IACjC,KAAK,MAAMC,QAAQjD,aAAc;QAC/B,MAAMkD,MAAMpB,MAAM,CAACmB,KAAKhD,IAAI,CAAC;QAC7B,IAAIiD,QAAQT,WAAW;QACvB,MAAMU,QAAQN,OAAOK;QACrB,MAAME,KAAKF,QAAQ,MAAML,OAAOQ,QAAQ,CAACF,UAAWF,CAAAA,KAAK9C,SAAS,GAAGgD,SAAS,IAAIA,QAAQ,CAAA;QAC1F,IAAI,CAACC,IAAI;YACP9C,IAAIG,UAAU,CAAC,GAAGX,MAAM,IAAI,EAAEmD,KAAKhD,IAAI,CAAC,YAAY,EAAEgD,KAAK9C,SAAS,GAAG,iBAAiB,WAAW,aAAa,EAAEoC,KAAKC,SAAS,CAACU,KAAK,CAAC,CAAC;QAC1I;QACCF,MAAM,CAACC,KAAK/C,GAAG,CAAC,GAAc+C,KAAK7C,KAAK,GAAGkD,KAAKC,KAAK,CAACJ,SAASA;IAClE;IAEA,0EAA0E;IAC1E,8GAA8G;IAC9G,IAAI9B;IACJ,IAAIS,OAAOT,GAAG,KAAKoB,WAAW;QAC5B,MAAMS,MAAMpB,OAAOT,GAAG;QACtB,MAAM8B,QAAQN,OAAOK;QACrB,IAAIA,QAAQ,MAAM,CAACL,OAAOQ,QAAQ,CAACF,UAAUA,SAAS,GAAG;YACvD7C,IAAIG,UAAU,CAAC,GAAGX,MAAM,wCAAwC,EAAEyC,KAAKC,SAAS,CAACU,KAAK,CAAC,CAAC;QAC1F;QACA7B,MAAM8B;IACR;IAEA,sEAAsE;IACtE,kIAAkI;IAClI,MAAMK,SAAStB,gBAAAA,iBAAAA,MAAQE;IACvB,MAAMqB,aAAanC,WAAW,QAAQ,SAASA,WAAW,QAAQ,QAAQ;IAC1E,MAAMoC,OAAOvB,UAAU,GAAGqB,OAAOG,OAAO,CAAC,mBAAmB,IAAI,CAAC,EAAEF,YAAY;IAE/E,MAAMG,YAAY,CAAClB,IAAcV,QAAQ6B,KAAK,CAACnB;IAC/C,MAAMoB,eAAeC,QAAQC,MAAM,CAACC,KAAK;IACzC,gEAAgE;IAChE,iHAAiH;IACjH,MAAMC,aAAa,CAACC;QAClB,IAAIL,cAAc9B,QAAQ6B,KAAK,CAAC,CAAC,CAAC,EAAEM,EAAEC,IAAI,CAAC,EAAE,EAAED,EAAEE,KAAK,CAAC,QAAQ,EAAEF,EAAEG,EAAE,CAAC,EAAE,CAAC;IAC3E;IACA,MAAMC,iBAAiBzC,MAAM,CAAC,cAAc;IAC5C,IAAIyC,mBAAmB9B,aAAa8B,eAAeC,IAAI,OAAO,IAAI;QAChElE,IAAIG,UAAU,CAAC,GAAGX,MAAM,oDAAoD,CAAC;IAC/E;IACA,uEAAuE;IACvE,4IAA4I;IAC5I,MAAM2E,UAAUF,iBACZ,CAACF;QACC,MAAMK,QAAQL,MAAM5C,GAAG,CAAC,CAACkD;YACvB,MAAMC,QAAsB;gBAAER,MAAMO,EAAEP,IAAI;gBAAES,MAAMF,EAAEE,IAAI;gBAAEC,GAAGH,EAAEG,CAAC;gBAAEC,GAAGJ,EAAEI,CAAC;gBAAEJ,GAAGA,EAAEA,CAAC;gBAAEK,GAAGL,EAAEK,CAAC;YAAC;YACzF,IAAIL,EAAEM,UAAU,KAAKxC,WAAWmC,MAAMK,UAAU,GAAGN,EAAEM,UAAU;YAC/D,OAAO1C,KAAKC,SAAS,CAACoC;QACxB;QACAvF,cAAckF,gBAAgBG,MAAMQ,MAAM,GAAG,GAAGR,MAAMpC,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG;IACzE,IACAG;IAEJ,MAAM0C,SAAS;QAAE/D;QAAOwC;IAAU;IAClC,MAAMwB,YAAY;QAChB,GAAGD,MAAM;QACTnC;QACA3B;QACA6C;QACAO;IACF;IAEA,IAAIJ;IACJ,IAAIgB,UAAU;IACd,IAAIjD,cAAcK,WAAW;QAC1B,CAAA,EAAE4B,KAAK,EAAEgB,OAAO,EAAE,GAAGC,eAAelG,aAAagD,WAAW,QAAO;QACpE,IAAIiD,UAAU,GAAGrD,QAAQ6B,KAAK,CAAC,CAAC,sBAAsB,EAAEwB,QAAQ,iBAAiB,CAAC;IACpF;IAEA,IAAIE;IACJ,IAAIC;IACJ,IAAIlE,WAAW,MAAM;QACnB,IAAImE;QACJ,IAAIpB,UAAU5B,WAAW;YACtB,CAAA,EAAEgD,QAAQ,EAAED,MAAM,EAAE,GAAG,MAAM/F,WAAW;gBAAE4E;YAAM,GAAGe,UAAS;QAC/D,OAAO;YACJ,CAAA,EAAEK,QAAQ,EAAED,MAAM,EAAE,GAAG,MAAM5F,cAAcsC,KAAK;gBAC/C,GAAGkD,SAAS;gBACZpE,KAAKc,OAAOd,GAAG;gBACf0E,OAAO5D,MAAM,CAAC,SAAS;gBACvB6D,aAAa7D,MAAM,CAAC,eAAe;YACrC,EAAC;QACH;QACAyD,UAAUE;IACZ,OAAO;QACL,uEAAuE;QACvE,kJAAkJ;QAClJ,IAAIG;QACJ,IAAIvB,UAAU5B,WAAW;YACvBmD,WAAW,MAAMpG,QAAQ;gBAAE6E;YAAM,GAAG;gBAAE,GAAGe,SAAS;gBAAE9F,MAAM;YAAM;QAClE,OAAO,IAAIwC,OAAOd,GAAG,EAAE;YACrB4E,WAAW,MAAMpG,QAAQ0C,KAAK;gBAAE,GAAGkD,SAAS;gBAAE9F,MAAM;YAAM;QAC5D,OAAO;YACL,MAAMuF,OAAO,MAAMrF,QAAQ0C,KAAK;gBAAE,GAAGiD,MAAM;gBAAE7F,MAAM;gBAAQqG,aAAa7D,MAAM,CAAC,eAAe;YAAY;YAC1G,IAAIpC,iBAAiBmF,KAAKW,MAAM,CAACK,YAAY,KAAK,CAAC/D,MAAM,CAAC,SAAS,EAAE;gBACnE8B,UAAUjE,kBAAkBkF,KAAKW,MAAM,CAACK,YAAY;gBACpDD,WAAW,MAAMpG,QAAQ0C,KAAK;oBAAE,GAAGkD,SAAS;oBAAE9F,MAAM;oBAAOwG,UAAU;gBAAK;YAC5E,OAAO;gBACLF,WAAWf;YACb;QACF;QACAW,SAASI,SAASJ,MAAM;QACxBD,UAAUjE,WAAW,QAAQiB,KAAKC,SAAS,CAACoD,UAAU,MAAM,KAAK/F,OAAO+F;IAC1E;IAEA,IAAI9D,OAAOb,MAAM,EAAE8C,QAAQ9C,MAAM,CAAC8E,KAAK,CAACR;SACnClG,cAAcqE,MAAM6B;IAEzB,wEAAwE;IACxE,8IAA8I;IAC9I,IAAIzD,OAAOjB,IAAI,EAAEmB,QAAQC,GAAG,CAACM,KAAKC,SAAS,CAAC;QAAEgB;QAAQwC,KAAKtC;QAAM,GAAG8B,MAAM;IAAC,GAAG,MAAM;SAC/E,IAAIA,OAAOlG,IAAI,KAAK,OAAO;YACuJkG;QAArLxD,QAAQ6B,KAAK,CAAC,GAAGvE,KAAK2G,QAAQ,CAACvC,MAAMwC,MAAM,CAAC,IAAI,gBAAgB,EAAEV,OAAOW,QAAQ,CAAC,MAAM,EAAEX,OAAOY,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAEZ,OAAOa,UAAU,CAAC,UAAU,EAAEb,OAAOc,QAAQ,CAAC,QAAQ,GAAEd,iBAAAA,OAAOe,MAAM,cAAbf,4BAAAA,iBAAiB,EAAE,UAAU,EAAEA,OAAOgB,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAEhB,OAAOK,YAAY,CAAC,WAAW,CAAC;IAC1R,OAAO;QACL7D,QAAQ6B,KAAK,CACX,GAAGvE,KAAK2G,QAAQ,CAACvC,MAAMwC,MAAM,CAAC,IAAI,MAAM,EAAEV,OAAOiB,cAAc,CAAC,SAAS,EAAEjB,OAAOW,QAAQ,CAAC,MAAM,EAAEX,OAAOY,YAAY,CAAC,EAAE,CAAC,GACxH,CAAC,MAAM,EAAEZ,OAAOkB,SAAS,CAAC,OAAO,EAAElB,OAAOa,UAAU,CAAC,UAAU,EAAEb,OAAOc,QAAQ,CAAC,OAAO,EAAEd,OAAOmB,WAAW,CAAC,CAAC,CAAC,GAC/G,CAAC,CAAC,EAAEnB,OAAOK,YAAY,CAAC,WAAW,CAAC,GACnCL,CAAAA,OAAOoB,MAAM,GAAG,0EAA0E,EAAC;IAElG;AACF;AAEA,8EAA8E;AAC9E,wGAAwG;AACxG,SAAStB,eAAepC,GAAW;IACjC,MAAMmB,QAAwB,EAAE;IAChC,IAAIgB,UAAU;IACd,KAAK,MAAMwB,QAAQ3D,IAAI4D,KAAK,CAAC,MAAO;QAClC,IAAI,CAACD,KAAKrC,IAAI,IAAI;QAClB,IAAI;YACFH,MAAM0C,IAAI,CAACxE,KAAKyE,KAAK,CAACH;QACxB,EAAE,OAAM;YACNxB;QACF;IACF;IACA,OAAO;QAAEhB;QAAOgB;IAAQ;AAC1B;AAEA,eAAehF,QAAQ"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/cli/extract.ts"],"sourcesContent":["// extract command — parse this command's flags, call src/extract.ts, print\n// and write the result. No business logic here — the library returns data and throws; only the CLI prints and exits.\nimport { readFileSync, writeFileSync } from 'node:fs';\nimport path from 'node:path';\nimport { parseArgs } from 'node:util';\nimport { analyze } from '../analyze.ts';\nimport type { OcrWordInput } from '../extract.ts';\nimport { analyzePdfAuto, extractOcr, pdfToMarkdown } from '../extract.ts';\nimport type { Tuning } from '../geometry.ts';\nimport { toText } from '../render-analysis.ts';\nimport type { Analysis, Report } from '../types.ts';\nimport type { Command } from './types.ts';\n\nconst USAGE =\n 'usage: pdf-to-md [extract] <file.pdf> [out.md] [--json] [--page-markers] [--ocr] [--no-ocr] ' +\n '[--pages N[-M]] [--stdout] [--words-json=FILE] [--debug-words=FILE] ' +\n '[--format md|txt|raw (default md; raw = Analysis JSON incl. the report; txt = reading-order plain text)] ' +\n '[--dpi=N (OCR render resolution, default 288; raise for tiny print or chart/label pages, ' +\n '>~400 measured worse on ordinary prose pages)] ' +\n '[--float-max-words=N] [--float-max-width=F] [--float-margin=F] [--para-gap=F] [--heading-scale=F] ' +\n '[--col-max-width=F] [--col-split=F] [--line-y-tol=F] [--line-height-ratio=F]';\n\nconst FORMATS = ['md', 'txt', 'raw'] as const;\ntype Format = (typeof FORMATS)[number];\n\n// A bad tuning-flag value (non-numeric, or <= 0 unless the flag allows zero) is a usage error, not a silent no-op.\nconst TUNING_FLAGS: {\n name: string;\n key: keyof Tuning;\n allowZero?: boolean;\n isInt?: boolean;\n}[] = [\n {\n name: 'float-max-words',\n key: 'floatMaxWords',\n allowZero: true,\n isInt: true,\n },\n { name: 'float-max-width', key: 'floatMaxWidth' },\n { name: 'float-margin', key: 'floatMargin' },\n { name: 'para-gap', key: 'paraGap' },\n { name: 'heading-scale', key: 'headingScale' },\n { name: 'col-max-width', key: 'colMaxWidth' },\n { name: 'col-split', key: 'colSplit' },\n { name: 'line-y-tol', key: 'lineYTol' },\n { name: 'line-height-ratio', key: 'lineHeightRatio' },\n];\n\nconst extract: Command = async (ctx) => {\n // --debug-words takes a required FILE value; a bare --debug-words is a\n // usage error, checked before parseArgs so it can't silently swallow the next token as its value.\n if (ctx.rest.includes('--debug-words')) {\n ctx.usageError(`${USAGE}\\n--debug-words requires a value: --debug-words=FILE`);\n }\n\n const parsed = (() => {\n try {\n return parseArgs({\n args: ctx.rest,\n options: {\n json: { type: 'boolean', default: false },\n 'page-markers': { type: 'boolean', default: false },\n ocr: { type: 'boolean', default: false },\n 'no-ocr': { type: 'boolean', default: false },\n stdout: { type: 'boolean', default: false },\n 'debug-words': { type: 'string' },\n help: { type: 'boolean', default: false, short: 'h' },\n pages: { type: 'string' },\n dpi: { type: 'string' },\n format: { type: 'string', default: 'md' },\n 'words-json': { type: 'string' },\n ...Object.fromEntries(TUNING_FLAGS.map((f) => [f.name, { type: 'string' as const }])),\n },\n allowPositionals: true,\n });\n } catch (err) {\n // usageError exits; the throw is dead code that tells the compiler so.\n throw ctx.usageError(`${(err as Error).message}\\n${USAGE}`);\n }\n })();\n const values = parsed.values as Record<string, string | boolean | undefined>;\n const positionals = parsed.positionals;\n\n if (values.help) {\n console.log(USAGE);\n return;\n }\n\n const [src, outArg] = positionals;\n const wordsPath = values['words-json'] as string | undefined;\n if (!src && !wordsPath) ctx.usageError(USAGE);\n\n const formatRaw = values.format as string;\n if (!(FORMATS as readonly string[]).includes(formatRaw)) {\n ctx.usageError(`${USAGE}\\n--format must be one of ${FORMATS.join('|')} (got ${JSON.stringify(formatRaw)})`);\n }\n const format = formatRaw as Format;\n\n // --pages: a malformed spec is a usage error; out-of-range page NUMBERS\n // clamp instead (collect.ts / the engine apply the same clamping policy).\n let pages: { first: number; last: number } | undefined;\n if (values.pages !== undefined) {\n const m = /^(\\d+)(?:-(\\d+))?$/.exec(values.pages as string);\n const first = m ? Number(m[1]) : Number.NaN;\n const last = m ? (m[2] !== undefined ? Number(m[2]) : first) : Number.NaN;\n if (!m || first < 1 || last < first) {\n ctx.usageError(`${USAGE}\\n--pages requires N or N-M, N >= 1, M >= N (got ${JSON.stringify(values.pages)})`);\n }\n pages = { first, last };\n }\n\n const tuning: Partial<Tuning> = {};\n for (const spec of TUNING_FLAGS) {\n const raw = values[spec.name] as string | undefined;\n if (raw === undefined) continue;\n const value = Number(raw);\n const ok = raw !== '' && Number.isFinite(value) && (spec.allowZero ? value >= 0 : value > 0);\n if (!ok) {\n ctx.usageError(`${USAGE}\\n--${spec.name} requires a ${spec.allowZero ? 'non-negative' : 'positive'} number (got ${JSON.stringify(raw)})`);\n }\n (tuning[spec.key] as number) = spec.isInt ? Math.trunc(value) : value;\n }\n\n // --dpi: page-render resolution, validated like the tuning flags. Default\n // 288 is the measured operating point; >~400 measured WORSE on ordinary pages — not a \"bigger is safer\" knob.\n let dpi: number | undefined;\n if (values.dpi !== undefined) {\n const raw = values.dpi as string;\n const value = Number(raw);\n if (raw === '' || !Number.isFinite(value) || value <= 0) {\n ctx.usageError(`${USAGE}\\n--dpi requires a positive number (got ${JSON.stringify(raw)})`);\n }\n dpi = value;\n }\n\n // In --words-json mode there may be no PDF positional; the words file\n // stands in for naming, extension stripped so the default dest can never overwrite the input. Default extension follows --format.\n const source = src ?? (wordsPath as string);\n const defaultExt = format === 'raw' ? 'json' : format === 'txt' ? 'txt' : 'md';\n const dest = outArg || `${source.replace(/\\.(pdf|jsonl)$/i, '')}.${defaultExt}`;\n\n const onWarning = (m: string) => console.error(m);\n const showProgress = process.stderr.isTTY;\n // onProgress carries per-page timing/count only; human-readable\n // diagnostics (e.g. the sparse-retry notice) arrive via onWarning instead, so they print regardless of TTY-ness.\n const onProgress = (e: { type: 'page'; page: number; words: number; ms: number }) => {\n if (showProgress) console.error(`p${e.page}: ${e.words} words, ${e.ms}ms`);\n };\n const debugWordsPath = values['debug-words'] as string | undefined;\n if (debugWordsPath !== undefined && debugWordsPath.trim() === '') {\n ctx.usageError(`${USAGE}\\n--debug-words requires a value: --debug-words=FILE`);\n }\n // The severity-1 audit instrument: every recognized word as clean JSON\n // lines to its own file. stderr stays a human channel (mixed prose), never a dump target — this file is guaranteed byte-for-byte parseable.\n const onWords = debugWordsPath\n ? (words: OcrWordInput[]) => {\n const lines = words.map((w) => {\n const entry: OcrWordInput = { page: w.page, text: w.text, x: w.x, y: w.y, w: w.w, h: w.h };\n if (w.confidence !== undefined) entry.confidence = w.confidence;\n return JSON.stringify(entry);\n });\n writeFileSync(debugWordsPath, lines.length ? `${lines.join('\\n')}\\n` : '');\n }\n : undefined;\n\n const common = { pages, onWarning, onWords };\n const ocrCommon = {\n ...common,\n tuning,\n dpi,\n onProgress,\n onWords,\n };\n\n let words: OcrWordInput[] | undefined;\n let skipped = 0;\n if (wordsPath !== undefined) {\n ({ words, skipped } = parseWordLines(readFileSync(wordsPath, 'utf8')));\n if (skipped > 0) console.error(`--words-json: skipped ${skipped} non-JSON line(s)`);\n }\n\n let content: string;\n let report: Report;\n if (format === 'md') {\n let markdown: string;\n if (words !== undefined) {\n ({ markdown, report } = await extractOcr({ words }, ocrCommon));\n } else {\n ({ markdown, report } = await pdfToMarkdown(src, {\n ...ocrCommon,\n ocr: values.ocr as boolean,\n noOcr: values['no-ocr'] as boolean,\n pageMarkers: values['page-markers'] as boolean,\n }));\n }\n content = markdown;\n } else {\n // raw/txt render the same routed Analysis as the Markdown API, so a\n // format switch cannot discard a sparse image page or change audit input.\n let analysis: Analysis;\n if (words !== undefined) {\n analysis = await analyze({ words }, { ...ocrCommon, path: 'ocr' });\n } else {\n analysis = (\n await analyzePdfAuto(src, {\n ...ocrCommon,\n ocr: values.ocr as boolean,\n noOcr: values['no-ocr'] as boolean,\n pageMarkers: values['page-markers'] as boolean,\n })\n ).analysis;\n }\n report = analysis.report;\n content = format === 'raw' ? JSON.stringify(analysis, null, 1) : toText(analysis);\n }\n\n if (values.stdout) process.stdout.write(content);\n else writeFileSync(dest, content);\n\n // The library's Report stays clean of source/out; --json consumers keep\n // report.json files around and need to know which input produced each one, so the CLI (only) decorates the printed JSON with source/out here.\n if (values.json) console.log(JSON.stringify({ source, out: dest, ...report }, null, 1));\n else if (report.path === 'ocr') {\n console.error(`${path.basename(dest).padEnd(32)} OCR path heads=${report.headings}(junk ${report.junkHeadings}) ` + `paras=${report.paragraphs} callouts=${report.callouts} floats=${report.floats ?? 0} dangling=${report.danglingLong} (long) ` + `~${report.charsPerPage} chars/page`);\n } else {\n console.error(\n `${path.basename(dest).padEnd(32)} body=${report.bodyFontHeight}pt heads=${report.headings}(junk ${report.junkHeadings}) ` +\n `lists=${report.listItems} paras=${report.paragraphs} callouts=${report.callouts} pairs=${report.joinedPairs} ` +\n `~${report.charsPerPage} chars/page` +\n (report.sparse ? ' ⚠️ implausibly sparse — pages are likely images; re-run with --ocr' : '')\n );\n }\n};\n\n// Parse one word-dump JSON line per word — key order is not guaranteed, parse\n// never grep. Non-JSON lines are skipped with a count, not fatal (the auditor applies the same policy).\nfunction parseWordLines(raw: string): { words: OcrWordInput[]; skipped: number } {\n const words: OcrWordInput[] = [];\n let skipped = 0;\n for (const line of raw.split('\\n')) {\n if (!line.trim()) continue;\n try {\n words.push(JSON.parse(line));\n } catch {\n skipped++;\n }\n }\n return { words, skipped };\n}\n\nexport default extract;\n"],"names":["readFileSync","writeFileSync","path","parseArgs","analyze","analyzePdfAuto","extractOcr","pdfToMarkdown","toText","USAGE","FORMATS","TUNING_FLAGS","name","key","allowZero","isInt","extract","ctx","rest","includes","usageError","parsed","args","options","json","type","default","ocr","stdout","help","short","pages","dpi","format","Object","fromEntries","map","f","allowPositionals","err","message","values","positionals","console","log","src","outArg","wordsPath","formatRaw","join","JSON","stringify","undefined","m","exec","first","Number","NaN","last","tuning","spec","raw","value","ok","isFinite","Math","trunc","source","defaultExt","dest","replace","onWarning","error","showProgress","process","stderr","isTTY","onProgress","e","page","words","ms","debugWordsPath","trim","onWords","lines","w","entry","text","x","y","h","confidence","length","common","ocrCommon","skipped","parseWordLines","content","report","markdown","noOcr","pageMarkers","analysis","write","out","basename","padEnd","headings","junkHeadings","paragraphs","callouts","floats","danglingLong","charsPerPage","bodyFontHeight","listItems","joinedPairs","sparse","line","split","push","parse"],"mappings":"AAAA,2EAA2E;AAC3E,qHAAqH;AACrH,SAASA,YAAY,EAAEC,aAAa,QAAQ,UAAU;AACtD,OAAOC,UAAU,YAAY;AAC7B,SAASC,SAAS,QAAQ,YAAY;AACtC,SAASC,OAAO,QAAQ,gBAAgB;AAExC,SAASC,cAAc,EAAEC,UAAU,EAAEC,aAAa,QAAQ,gBAAgB;AAE1E,SAASC,MAAM,QAAQ,wBAAwB;AAI/C,MAAMC,QACJ,iGACA,yEACA,8GACA,8FACA,oDACA,uGACA;AAEF,MAAMC,UAAU;IAAC;IAAM;IAAO;CAAM;AAGpC,mHAAmH;AACnH,MAAMC,eAKA;IACJ;QACEC,MAAM;QACNC,KAAK;QACLC,WAAW;QACXC,OAAO;IACT;IACA;QAAEH,MAAM;QAAmBC,KAAK;IAAgB;IAChD;QAAED,MAAM;QAAgBC,KAAK;IAAc;IAC3C;QAAED,MAAM;QAAYC,KAAK;IAAU;IACnC;QAAED,MAAM;QAAiBC,KAAK;IAAe;IAC7C;QAAED,MAAM;QAAiBC,KAAK;IAAc;IAC5C;QAAED,MAAM;QAAaC,KAAK;IAAW;IACrC;QAAED,MAAM;QAAcC,KAAK;IAAW;IACtC;QAAED,MAAM;QAAqBC,KAAK;IAAkB;CACrD;AAED,MAAMG,UAAmB,OAAOC;IAC9B,uEAAuE;IACvE,kGAAkG;IAClG,IAAIA,IAAIC,IAAI,CAACC,QAAQ,CAAC,kBAAkB;QACtCF,IAAIG,UAAU,CAAC,GAAGX,MAAM,oDAAoD,CAAC;IAC/E;IAEA,MAAMY,SAAS,AAAC,CAAA;QACd,IAAI;YACF,OAAOlB,UAAU;gBACfmB,MAAML,IAAIC,IAAI;gBACdK,SAAS;oBACPC,MAAM;wBAAEC,MAAM;wBAAWC,SAAS;oBAAM;oBACxC,gBAAgB;wBAAED,MAAM;wBAAWC,SAAS;oBAAM;oBAClDC,KAAK;wBAAEF,MAAM;wBAAWC,SAAS;oBAAM;oBACvC,UAAU;wBAAED,MAAM;wBAAWC,SAAS;oBAAM;oBAC5CE,QAAQ;wBAAEH,MAAM;wBAAWC,SAAS;oBAAM;oBAC1C,eAAe;wBAAED,MAAM;oBAAS;oBAChCI,MAAM;wBAAEJ,MAAM;wBAAWC,SAAS;wBAAOI,OAAO;oBAAI;oBACpDC,OAAO;wBAAEN,MAAM;oBAAS;oBACxBO,KAAK;wBAAEP,MAAM;oBAAS;oBACtBQ,QAAQ;wBAAER,MAAM;wBAAUC,SAAS;oBAAK;oBACxC,cAAc;wBAAED,MAAM;oBAAS;oBAC/B,GAAGS,OAAOC,WAAW,CAACxB,aAAayB,GAAG,CAAC,CAACC,IAAM;4BAACA,EAAEzB,IAAI;4BAAE;gCAAEa,MAAM;4BAAkB;yBAAE,EAAE;gBACvF;gBACAa,kBAAkB;YACpB;QACF,EAAE,OAAOC,KAAK;YACZ,uEAAuE;YACvE,MAAMtB,IAAIG,UAAU,CAAC,GAAG,AAACmB,IAAcC,OAAO,CAAC,EAAE,EAAE/B,OAAO;QAC5D;IACF,CAAA;IACA,MAAMgC,SAASpB,OAAOoB,MAAM;IAC5B,MAAMC,cAAcrB,OAAOqB,WAAW;IAEtC,IAAID,OAAOZ,IAAI,EAAE;QACfc,QAAQC,GAAG,CAACnC;QACZ;IACF;IAEA,MAAM,CAACoC,KAAKC,OAAO,GAAGJ;IACtB,MAAMK,YAAYN,MAAM,CAAC,aAAa;IACtC,IAAI,CAACI,OAAO,CAACE,WAAW9B,IAAIG,UAAU,CAACX;IAEvC,MAAMuC,YAAYP,OAAOR,MAAM;IAC/B,IAAI,CAAC,AAACvB,QAA8BS,QAAQ,CAAC6B,YAAY;QACvD/B,IAAIG,UAAU,CAAC,GAAGX,MAAM,0BAA0B,EAAEC,QAAQuC,IAAI,CAAC,KAAK,MAAM,EAAEC,KAAKC,SAAS,CAACH,WAAW,CAAC,CAAC;IAC5G;IACA,MAAMf,SAASe;IAEf,wEAAwE;IACxE,0EAA0E;IAC1E,IAAIjB;IACJ,IAAIU,OAAOV,KAAK,KAAKqB,WAAW;QAC9B,MAAMC,IAAI,qBAAqBC,IAAI,CAACb,OAAOV,KAAK;QAChD,MAAMwB,QAAQF,IAAIG,OAAOH,CAAC,CAAC,EAAE,IAAIG,OAAOC,GAAG;QAC3C,MAAMC,OAAOL,IAAKA,CAAC,CAAC,EAAE,KAAKD,YAAYI,OAAOH,CAAC,CAAC,EAAE,IAAIE,QAASC,OAAOC,GAAG;QACzE,IAAI,CAACJ,KAAKE,QAAQ,KAAKG,OAAOH,OAAO;YACnCtC,IAAIG,UAAU,CAAC,GAAGX,MAAM,iDAAiD,EAAEyC,KAAKC,SAAS,CAACV,OAAOV,KAAK,EAAE,CAAC,CAAC;QAC5G;QACAA,QAAQ;YAAEwB;YAAOG;QAAK;IACxB;IAEA,MAAMC,SAA0B,CAAC;IACjC,KAAK,MAAMC,QAAQjD,aAAc;QAC/B,MAAMkD,MAAMpB,MAAM,CAACmB,KAAKhD,IAAI,CAAC;QAC7B,IAAIiD,QAAQT,WAAW;QACvB,MAAMU,QAAQN,OAAOK;QACrB,MAAME,KAAKF,QAAQ,MAAML,OAAOQ,QAAQ,CAACF,UAAWF,CAAAA,KAAK9C,SAAS,GAAGgD,SAAS,IAAIA,QAAQ,CAAA;QAC1F,IAAI,CAACC,IAAI;YACP9C,IAAIG,UAAU,CAAC,GAAGX,MAAM,IAAI,EAAEmD,KAAKhD,IAAI,CAAC,YAAY,EAAEgD,KAAK9C,SAAS,GAAG,iBAAiB,WAAW,aAAa,EAAEoC,KAAKC,SAAS,CAACU,KAAK,CAAC,CAAC;QAC1I;QACCF,MAAM,CAACC,KAAK/C,GAAG,CAAC,GAAc+C,KAAK7C,KAAK,GAAGkD,KAAKC,KAAK,CAACJ,SAASA;IAClE;IAEA,0EAA0E;IAC1E,8GAA8G;IAC9G,IAAI9B;IACJ,IAAIS,OAAOT,GAAG,KAAKoB,WAAW;QAC5B,MAAMS,MAAMpB,OAAOT,GAAG;QACtB,MAAM8B,QAAQN,OAAOK;QACrB,IAAIA,QAAQ,MAAM,CAACL,OAAOQ,QAAQ,CAACF,UAAUA,SAAS,GAAG;YACvD7C,IAAIG,UAAU,CAAC,GAAGX,MAAM,wCAAwC,EAAEyC,KAAKC,SAAS,CAACU,KAAK,CAAC,CAAC;QAC1F;QACA7B,MAAM8B;IACR;IAEA,sEAAsE;IACtE,kIAAkI;IAClI,MAAMK,SAAStB,gBAAAA,iBAAAA,MAAQE;IACvB,MAAMqB,aAAanC,WAAW,QAAQ,SAASA,WAAW,QAAQ,QAAQ;IAC1E,MAAMoC,OAAOvB,UAAU,GAAGqB,OAAOG,OAAO,CAAC,mBAAmB,IAAI,CAAC,EAAEF,YAAY;IAE/E,MAAMG,YAAY,CAAClB,IAAcV,QAAQ6B,KAAK,CAACnB;IAC/C,MAAMoB,eAAeC,QAAQC,MAAM,CAACC,KAAK;IACzC,gEAAgE;IAChE,iHAAiH;IACjH,MAAMC,aAAa,CAACC;QAClB,IAAIL,cAAc9B,QAAQ6B,KAAK,CAAC,CAAC,CAAC,EAAEM,EAAEC,IAAI,CAAC,EAAE,EAAED,EAAEE,KAAK,CAAC,QAAQ,EAAEF,EAAEG,EAAE,CAAC,EAAE,CAAC;IAC3E;IACA,MAAMC,iBAAiBzC,MAAM,CAAC,cAAc;IAC5C,IAAIyC,mBAAmB9B,aAAa8B,eAAeC,IAAI,OAAO,IAAI;QAChElE,IAAIG,UAAU,CAAC,GAAGX,MAAM,oDAAoD,CAAC;IAC/E;IACA,uEAAuE;IACvE,4IAA4I;IAC5I,MAAM2E,UAAUF,iBACZ,CAACF;QACC,MAAMK,QAAQL,MAAM5C,GAAG,CAAC,CAACkD;YACvB,MAAMC,QAAsB;gBAAER,MAAMO,EAAEP,IAAI;gBAAES,MAAMF,EAAEE,IAAI;gBAAEC,GAAGH,EAAEG,CAAC;gBAAEC,GAAGJ,EAAEI,CAAC;gBAAEJ,GAAGA,EAAEA,CAAC;gBAAEK,GAAGL,EAAEK,CAAC;YAAC;YACzF,IAAIL,EAAEM,UAAU,KAAKxC,WAAWmC,MAAMK,UAAU,GAAGN,EAAEM,UAAU;YAC/D,OAAO1C,KAAKC,SAAS,CAACoC;QACxB;QACAtF,cAAciF,gBAAgBG,MAAMQ,MAAM,GAAG,GAAGR,MAAMpC,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG;IACzE,IACAG;IAEJ,MAAM0C,SAAS;QAAE/D;QAAOwC;QAAWa;IAAQ;IAC3C,MAAMW,YAAY;QAChB,GAAGD,MAAM;QACTnC;QACA3B;QACA6C;QACAO;IACF;IAEA,IAAIJ;IACJ,IAAIgB,UAAU;IACd,IAAIjD,cAAcK,WAAW;QAC1B,CAAA,EAAE4B,KAAK,EAAEgB,OAAO,EAAE,GAAGC,eAAejG,aAAa+C,WAAW,QAAO;QACpE,IAAIiD,UAAU,GAAGrD,QAAQ6B,KAAK,CAAC,CAAC,sBAAsB,EAAEwB,QAAQ,iBAAiB,CAAC;IACpF;IAEA,IAAIE;IACJ,IAAIC;IACJ,IAAIlE,WAAW,MAAM;QACnB,IAAImE;QACJ,IAAIpB,UAAU5B,WAAW;YACtB,CAAA,EAAEgD,QAAQ,EAAED,MAAM,EAAE,GAAG,MAAM7F,WAAW;gBAAE0E;YAAM,GAAGe,UAAS;QAC/D,OAAO;YACJ,CAAA,EAAEK,QAAQ,EAAED,MAAM,EAAE,GAAG,MAAM5F,cAAcsC,KAAK;gBAC/C,GAAGkD,SAAS;gBACZpE,KAAKc,OAAOd,GAAG;gBACf0E,OAAO5D,MAAM,CAAC,SAAS;gBACvB6D,aAAa7D,MAAM,CAAC,eAAe;YACrC,EAAC;QACH;QACAyD,UAAUE;IACZ,OAAO;QACL,oEAAoE;QACpE,0EAA0E;QAC1E,IAAIG;QACJ,IAAIvB,UAAU5B,WAAW;YACvBmD,WAAW,MAAMnG,QAAQ;gBAAE4E;YAAM,GAAG;gBAAE,GAAGe,SAAS;gBAAE7F,MAAM;YAAM;QAClE,OAAO;YACLqG,WAAW,AACT,CAAA,MAAMlG,eAAewC,KAAK;gBACxB,GAAGkD,SAAS;gBACZpE,KAAKc,OAAOd,GAAG;gBACf0E,OAAO5D,MAAM,CAAC,SAAS;gBACvB6D,aAAa7D,MAAM,CAAC,eAAe;YACrC,EAAC,EACD8D,QAAQ;QACZ;QACAJ,SAASI,SAASJ,MAAM;QACxBD,UAAUjE,WAAW,QAAQiB,KAAKC,SAAS,CAACoD,UAAU,MAAM,KAAK/F,OAAO+F;IAC1E;IAEA,IAAI9D,OAAOb,MAAM,EAAE8C,QAAQ9C,MAAM,CAAC4E,KAAK,CAACN;SACnCjG,cAAcoE,MAAM6B;IAEzB,wEAAwE;IACxE,8IAA8I;IAC9I,IAAIzD,OAAOjB,IAAI,EAAEmB,QAAQC,GAAG,CAACM,KAAKC,SAAS,CAAC;QAAEgB;QAAQsC,KAAKpC;QAAM,GAAG8B,MAAM;IAAC,GAAG,MAAM;SAC/E,IAAIA,OAAOjG,IAAI,KAAK,OAAO;YACuJiG;QAArLxD,QAAQ6B,KAAK,CAAC,GAAGtE,KAAKwG,QAAQ,CAACrC,MAAMsC,MAAM,CAAC,IAAI,gBAAgB,EAAER,OAAOS,QAAQ,CAAC,MAAM,EAAET,OAAOU,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAEV,OAAOW,UAAU,CAAC,UAAU,EAAEX,OAAOY,QAAQ,CAAC,QAAQ,GAAEZ,iBAAAA,OAAOa,MAAM,cAAbb,4BAAAA,iBAAiB,EAAE,UAAU,EAAEA,OAAOc,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAEd,OAAOe,YAAY,CAAC,WAAW,CAAC;IAC1R,OAAO;QACLvE,QAAQ6B,KAAK,CACX,GAAGtE,KAAKwG,QAAQ,CAACrC,MAAMsC,MAAM,CAAC,IAAI,MAAM,EAAER,OAAOgB,cAAc,CAAC,SAAS,EAAEhB,OAAOS,QAAQ,CAAC,MAAM,EAAET,OAAOU,YAAY,CAAC,EAAE,CAAC,GACxH,CAAC,MAAM,EAAEV,OAAOiB,SAAS,CAAC,OAAO,EAAEjB,OAAOW,UAAU,CAAC,UAAU,EAAEX,OAAOY,QAAQ,CAAC,OAAO,EAAEZ,OAAOkB,WAAW,CAAC,CAAC,CAAC,GAC/G,CAAC,CAAC,EAAElB,OAAOe,YAAY,CAAC,WAAW,CAAC,GACnCf,CAAAA,OAAOmB,MAAM,GAAG,0EAA0E,EAAC;IAElG;AACF;AAEA,8EAA8E;AAC9E,wGAAwG;AACxG,SAASrB,eAAepC,GAAW;IACjC,MAAMmB,QAAwB,EAAE;IAChC,IAAIgB,UAAU;IACd,KAAK,MAAMuB,QAAQ1D,IAAI2D,KAAK,CAAC,MAAO;QAClC,IAAI,CAACD,KAAKpC,IAAI,IAAI;QAClB,IAAI;YACFH,MAAMyC,IAAI,CAACvE,KAAKwE,KAAK,CAACH;QACxB,EAAE,OAAM;YACNvB;QACF;IACF;IACA,OAAO;QAAEhB;QAAOgB;IAAQ;AAC1B;AAEA,eAAehF,QAAQ"}
|
package/dist/esm/collect.js
CHANGED
|
@@ -1,23 +1,39 @@
|
|
|
1
|
-
// Pass 1: load
|
|
2
|
-
//
|
|
3
|
-
import { OPS } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
1
|
+
// Pass 1: load each PDF text run into viewport points, plus the document-wide
|
|
2
|
+
// char-weighted font-height histogram. Height rounds to 0.1pt for stable histogram keys.
|
|
3
|
+
import { OPS, Util } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
4
4
|
import { openDocument } from './pdf-open.js';
|
|
5
5
|
// HYBRID-PAGE DETECTION (README's "Design contract"): a page can carry a
|
|
6
6
|
// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
8
|
+
// Image presence uses the paint operators for XObjects, inline images, and masks.
|
|
9
|
+
// The large-image threshold uses `paintImageXObject` dimensions when available.
|
|
10
10
|
const LARGE_IMAGE_MIN_PX = 200;
|
|
11
|
-
|
|
11
|
+
const IMAGE_PAINT_OPS = new Set([
|
|
12
|
+
OPS.paintImageMaskXObject,
|
|
13
|
+
OPS.paintImageMaskXObjectGroup,
|
|
14
|
+
OPS.paintImageMaskXObjectRepeat,
|
|
15
|
+
OPS.paintImageXObject,
|
|
16
|
+
OPS.paintImageXObjectRepeat,
|
|
17
|
+
OPS.paintInlineImageXObject,
|
|
18
|
+
OPS.paintInlineImageXObjectGroup,
|
|
19
|
+
OPS.paintSolidColorImageMask
|
|
20
|
+
]);
|
|
21
|
+
async function getPageImageSignals(page, textChars) {
|
|
12
22
|
const { fnArray, argsArray } = await page.getOperatorList();
|
|
23
|
+
let hasImage = false;
|
|
24
|
+
let largeImage = false;
|
|
13
25
|
for(let i = 0; i < fnArray.length; i++){
|
|
26
|
+
if (IMAGE_PAINT_OPS.has(fnArray[i])) hasImage = true;
|
|
14
27
|
if (fnArray[i] !== OPS.paintImageXObject) continue;
|
|
15
28
|
const [, w, h] = argsArray[i];
|
|
16
29
|
if (typeof w === 'number' && typeof h === 'number') {
|
|
17
|
-
if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX)
|
|
18
|
-
} else
|
|
30
|
+
if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) largeImage = true;
|
|
31
|
+
} else largeImage = true; // dims unavailable: coarser "op present" signal
|
|
19
32
|
}
|
|
20
|
-
return
|
|
33
|
+
return {
|
|
34
|
+
hasImage,
|
|
35
|
+
largeImage: textChars > 80 && largeImage
|
|
36
|
+
};
|
|
21
37
|
}
|
|
22
38
|
// { first, last }: 1-based, inclusive page range (the CLI's --pages).
|
|
23
39
|
// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.
|
|
@@ -42,31 +58,64 @@ export async function collect(src, { first, last } = {}) {
|
|
|
42
58
|
const vp = page.getViewport({
|
|
43
59
|
scale: 1
|
|
44
60
|
});
|
|
61
|
+
const viewportScale = Math.hypot(vp.transform[0], vp.transform[1]);
|
|
45
62
|
const glyphs = [];
|
|
46
63
|
let textChars = 0;
|
|
47
64
|
for (const it of items){
|
|
48
65
|
if (!('str' in it) || typeof it.str !== 'string') continue;
|
|
49
66
|
textChars += it.str.length;
|
|
50
67
|
if (!it.str.trim()) continue;
|
|
51
|
-
const
|
|
68
|
+
const transform = Util.transform(vp.transform, it.transform);
|
|
69
|
+
const baselineLength = Math.hypot(transform[0], transform[1]);
|
|
70
|
+
const w = Math.abs(it.width) * viewportScale;
|
|
71
|
+
const h = Math.round(Math.hypot(transform[2], transform[3]) * 10) / 10;
|
|
72
|
+
const baselineX = baselineLength > 0 ? transform[0] / baselineLength * w : w;
|
|
73
|
+
const baselineY = baselineLength > 0 ? transform[1] / baselineLength * w : 0;
|
|
74
|
+
const corners = [
|
|
75
|
+
[
|
|
76
|
+
transform[4],
|
|
77
|
+
transform[5]
|
|
78
|
+
],
|
|
79
|
+
[
|
|
80
|
+
transform[4] + baselineX,
|
|
81
|
+
transform[5] + baselineY
|
|
82
|
+
],
|
|
83
|
+
[
|
|
84
|
+
transform[4] + baselineX + transform[2],
|
|
85
|
+
transform[5] + baselineY + transform[3]
|
|
86
|
+
],
|
|
87
|
+
[
|
|
88
|
+
transform[4] + transform[2],
|
|
89
|
+
transform[5] + transform[3]
|
|
90
|
+
]
|
|
91
|
+
];
|
|
92
|
+
const boundsX = corners.map(([x])=>x);
|
|
93
|
+
const boundsY = corners.map(([, y])=>y);
|
|
52
94
|
glyphs.push({
|
|
53
95
|
s: it.str,
|
|
54
|
-
x:
|
|
55
|
-
y:
|
|
56
|
-
w
|
|
57
|
-
h
|
|
96
|
+
x: transform[4],
|
|
97
|
+
y: transform[5],
|
|
98
|
+
w,
|
|
99
|
+
h,
|
|
100
|
+
bounds: {
|
|
101
|
+
x: Math.min(...boundsX),
|
|
102
|
+
y: Math.min(...boundsY),
|
|
103
|
+
w: Math.max(...boundsX) - Math.min(...boundsX),
|
|
104
|
+
h: Math.max(...boundsY) - Math.min(...boundsY)
|
|
105
|
+
}
|
|
58
106
|
});
|
|
59
107
|
heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);
|
|
60
108
|
}
|
|
61
109
|
// "text layer used" for hybrid-page purposes (page text > 80 chars) — a
|
|
62
110
|
// page with only a couple of stray scraps isn't "using" the text layer even if it also carries a large image.
|
|
63
|
-
const
|
|
111
|
+
const { hasImage, largeImage } = await getPageImageSignals(page, textChars);
|
|
64
112
|
pages.push({
|
|
65
113
|
page: p,
|
|
66
114
|
glyphs,
|
|
67
115
|
width: vp.width,
|
|
68
116
|
height: vp.height,
|
|
69
117
|
textChars,
|
|
118
|
+
hasImage,
|
|
70
119
|
largeImage
|
|
71
120
|
});
|
|
72
121
|
}
|
package/dist/esm/collect.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/collect.ts"],"sourcesContent":["// Pass 1: load the PDF and collect positioned glyphs per page, plus the\n// document-wide char-weighted font-height histogram. Heights round to 0.1pt for stable histogram keys; y is flipped so sorting ascending reads top-down.\nimport type { PDFPageProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { OPS } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { openDocument } from './pdf-open.ts';\nimport type { CollectedPage, CollectResult, Glyph } from './types.ts';\n\n// HYBRID-PAGE DETECTION (README's \"Design contract\"): a page can carry a\n// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.\n//\n// Signal: scan the operator list for `OPS.paintImageXObject`, whose args are\n// `[objId, width, height]` — available synchronously, `large` is >=200x200px. Masks/nested Form XObjects aren't walked (accepted gap); missing dims fall back to \"op present\" as the signal.\nconst LARGE_IMAGE_MIN_PX = 200;\nasync function hasLargeImage(page: PDFPageProxy): Promise<boolean> {\n const { fnArray, argsArray } = await page.getOperatorList();\n for (let i = 0; i < fnArray.length; i++) {\n if (fnArray[i] !== OPS.paintImageXObject) continue;\n const [, w, h] = argsArray[i];\n if (typeof w === 'number' && typeof h === 'number') {\n if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) return true;\n } else return true; // dims unavailable: coarser \"op present\" signal\n }\n return false;\n}\n\n// { first, last }: 1-based, inclusive page range (the CLI's --pages).\n// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.\n//\n// CAVEAT: `heightChars` is built ONLY from the pages actually collected, so\n// a `--pages` subset's heading levels can differ from a full run's — accepted since re-extracted pages get spliced back in for their prose, not diffed.\n\n// Cheap page-count-only load (no per-page text extraction) — the OCR path\n// needs it to resolve its own [first, last] clamping and print a \"### pN\" marker for every page in range, including ones with no recognized words.\nexport async function pageCount(src: string): Promise<number> {\n const doc = await openDocument(src);\n return doc.numPages;\n}\n\nexport async function collect(src: string, { first, last }: { first?: number; last?: number } = {}): Promise<CollectResult> {\n const doc = await openDocument(src);\n const from = Math.min(Math.max(1, first ?? 1), Math.max(1, doc.numPages));\n const to = Math.min(Math.max(from, last ?? doc.numPages), doc.numPages);\n const pages: CollectedPage[] = [];\n const heightChars = new Map<number, number>();\n for (let p = from; p <= to; p++) {\n const page = await doc.getPage(p);\n const { items } = await page.getTextContent();\n const vp = page.getViewport({ scale: 1 });\n const glyphs: Glyph[] = [];\n let textChars = 0;\n for (const it of items) {\n if (!('str' in it) || typeof it.str !== 'string') continue;\n textChars += it.str.length;\n if (!it.str.trim()) continue;\n const h = Math.round(Math.abs(it.transform[3]) * 10) / 10;\n glyphs.push({\n s: it.str,\n x: it.transform[4],\n y: vp.height - it.transform[5],\n w: it.width,\n h,\n });\n heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);\n }\n // \"text layer used\" for hybrid-page purposes (page text > 80 chars) — a\n // page with only a couple of stray scraps isn't \"using\" the text layer even if it also carries a large image.\n const largeImage = textChars > 80 && (await hasLargeImage(page));\n pages.push({\n page: p,\n glyphs,\n width: vp.width,\n height: vp.height,\n textChars,\n largeImage,\n });\n }\n // `numPages` is the SELECTED count, not doc.numPages: the auto-OCR\n // chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely \"sparse\".\n return { numPages: pages.length, pages, heightChars };\n}\n"],"names":["OPS","openDocument","LARGE_IMAGE_MIN_PX","hasLargeImage","page","fnArray","argsArray","getOperatorList","i","length","paintImageXObject","w","h","pageCount","src","doc","numPages","collect","first","last","from","Math","min","max","to","pages","heightChars","Map","p","getPage","items","getTextContent","vp","getViewport","scale","glyphs","textChars","it","str","trim","round","abs","transform","push","s","x","y","height","width","set","get","largeImage"],"mappings":"AAAA,wEAAwE;AACxE,yJAAyJ;AAEzJ,SAASA,GAAG,QAAQ,kCAAkC;AACtD,SAASC,YAAY,QAAQ,gBAAgB;AAG7C,yEAAyE;AACzE,2HAA2H;AAC3H,EAAE;AACF,6EAA6E;AAC7E,6LAA6L;AAC7L,MAAMC,qBAAqB;AAC3B,eAAeC,cAAcC,IAAkB;IAC7C,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAG,MAAMF,KAAKG,eAAe;IACzD,IAAK,IAAIC,IAAI,GAAGA,IAAIH,QAAQI,MAAM,EAAED,IAAK;QACvC,IAAIH,OAAO,CAACG,EAAE,KAAKR,IAAIU,iBAAiB,EAAE;QAC1C,MAAM,GAAGC,GAAGC,EAAE,GAAGN,SAAS,CAACE,EAAE;QAC7B,IAAI,OAAOG,MAAM,YAAY,OAAOC,MAAM,UAAU;YAClD,IAAID,KAAKT,sBAAsBU,KAAKV,oBAAoB,OAAO;QACjE,OAAO,OAAO,MAAM,gDAAgD;IACtE;IACA,OAAO;AACT;AAEA,sEAAsE;AACtE,wJAAwJ;AACxJ,EAAE;AACF,4EAA4E;AAC5E,wJAAwJ;AAExJ,0EAA0E;AAC1E,mJAAmJ;AACnJ,OAAO,eAAeW,UAAUC,GAAW;IACzC,MAAMC,MAAM,MAAMd,aAAaa;IAC/B,OAAOC,IAAIC,QAAQ;AACrB;AAEA,OAAO,eAAeC,QAAQH,GAAW,EAAE,EAAEI,KAAK,EAAEC,IAAI,EAAqC,GAAG,CAAC,CAAC;IAChG,MAAMJ,MAAM,MAAMd,aAAaa;IAC/B,MAAMM,OAAOC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,GAAGL,kBAAAA,mBAAAA,QAAS,IAAIG,KAAKE,GAAG,CAAC,GAAGR,IAAIC,QAAQ;IACvE,MAAMQ,KAAKH,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACH,MAAMD,iBAAAA,kBAAAA,OAAQJ,IAAIC,QAAQ,GAAGD,IAAIC,QAAQ;IACtE,MAAMS,QAAyB,EAAE;IACjC,MAAMC,cAAc,IAAIC;IACxB,IAAK,IAAIC,IAAIR,MAAMQ,KAAKJ,IAAII,IAAK;QAC/B,MAAMxB,OAAO,MAAMW,IAAIc,OAAO,CAACD;QAC/B,MAAM,EAAEE,KAAK,EAAE,GAAG,MAAM1B,KAAK2B,cAAc;QAC3C,MAAMC,KAAK5B,KAAK6B,WAAW,CAAC;YAAEC,OAAO;QAAE;QACvC,MAAMC,SAAkB,EAAE;QAC1B,IAAIC,YAAY;QAChB,KAAK,MAAMC,MAAMP,MAAO;YACtB,IAAI,CAAE,CAAA,SAASO,EAAC,KAAM,OAAOA,GAAGC,GAAG,KAAK,UAAU;YAClDF,aAAaC,GAAGC,GAAG,CAAC7B,MAAM;YAC1B,IAAI,CAAC4B,GAAGC,GAAG,CAACC,IAAI,IAAI;YACpB,MAAM3B,IAAIS,KAAKmB,KAAK,CAACnB,KAAKoB,GAAG,CAACJ,GAAGK,SAAS,CAAC,EAAE,IAAI,MAAM;YACvDP,OAAOQ,IAAI,CAAC;gBACVC,GAAGP,GAAGC,GAAG;gBACTO,GAAGR,GAAGK,SAAS,CAAC,EAAE;gBAClBI,GAAGd,GAAGe,MAAM,GAAGV,GAAGK,SAAS,CAAC,EAAE;gBAC9B/B,GAAG0B,GAAGW,KAAK;gBACXpC;YACF;YACAc,YAAYuB,GAAG,CAACrC,GAAG,AAACc,CAAAA,YAAYwB,GAAG,CAACtC,MAAM,CAAA,IAAKyB,GAAGC,GAAG,CAAC7B,MAAM;QAC9D;QACA,wEAAwE;QACxE,8GAA8G;QAC9G,MAAM0C,aAAaf,YAAY,MAAO,MAAMjC,cAAcC;QAC1DqB,MAAMkB,IAAI,CAAC;YACTvC,MAAMwB;YACNO;YACAa,OAAOhB,GAAGgB,KAAK;YACfD,QAAQf,GAAGe,MAAM;YACjBX;YACAe;QACF;IACF;IACA,mEAAmE;IACnE,sIAAsI;IACtI,OAAO;QAAEnC,UAAUS,MAAMhB,MAAM;QAAEgB;QAAOC;IAAY;AACtD"}
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/collect.ts"],"sourcesContent":["// Pass 1: load each PDF text run into viewport points, plus the document-wide\n// char-weighted font-height histogram. Height rounds to 0.1pt for stable histogram keys.\nimport type { PDFPageProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { OPS, Util } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { openDocument } from './pdf-open.ts';\nimport type { CollectedPage, CollectResult, Glyph } from './types.ts';\n\n// HYBRID-PAGE DETECTION (README's \"Design contract\"): a page can carry a\n// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.\n//\n// Image presence uses the paint operators for XObjects, inline images, and masks.\n// The large-image threshold uses `paintImageXObject` dimensions when available.\nconst LARGE_IMAGE_MIN_PX = 200;\ninterface PageImageSignals {\n hasImage: boolean;\n largeImage: boolean;\n}\n\nconst IMAGE_PAINT_OPS = new Set([OPS.paintImageMaskXObject, OPS.paintImageMaskXObjectGroup, OPS.paintImageMaskXObjectRepeat, OPS.paintImageXObject, OPS.paintImageXObjectRepeat, OPS.paintInlineImageXObject, OPS.paintInlineImageXObjectGroup, OPS.paintSolidColorImageMask]);\n\nasync function getPageImageSignals(page: PDFPageProxy, textChars: number): Promise<PageImageSignals> {\n const { fnArray, argsArray } = await page.getOperatorList();\n let hasImage = false;\n let largeImage = false;\n for (let i = 0; i < fnArray.length; i++) {\n if (IMAGE_PAINT_OPS.has(fnArray[i])) hasImage = true;\n if (fnArray[i] !== OPS.paintImageXObject) continue;\n const [, w, h] = argsArray[i];\n if (typeof w === 'number' && typeof h === 'number') {\n if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) largeImage = true;\n } else largeImage = true; // dims unavailable: coarser \"op present\" signal\n }\n return { hasImage, largeImage: textChars > 80 && largeImage };\n}\n\n// { first, last }: 1-based, inclusive page range (the CLI's --pages).\n// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.\n//\n// CAVEAT: `heightChars` is built ONLY from the pages actually collected, so\n// a `--pages` subset's heading levels can differ from a full run's — accepted since re-extracted pages get spliced back in for their prose, not diffed.\n\n// Cheap page-count-only load (no per-page text extraction) — the OCR path\n// needs it to resolve its own [first, last] clamping and print a \"### pN\" marker for every page in range, including ones with no recognized words.\nexport async function pageCount(src: string): Promise<number> {\n const doc = await openDocument(src);\n return doc.numPages;\n}\n\nexport async function collect(src: string, { first, last }: { first?: number; last?: number } = {}): Promise<CollectResult> {\n const doc = await openDocument(src);\n const from = Math.min(Math.max(1, first ?? 1), Math.max(1, doc.numPages));\n const to = Math.min(Math.max(from, last ?? doc.numPages), doc.numPages);\n const pages: CollectedPage[] = [];\n const heightChars = new Map<number, number>();\n for (let p = from; p <= to; p++) {\n const page = await doc.getPage(p);\n const { items } = await page.getTextContent();\n const vp = page.getViewport({ scale: 1 });\n const viewportScale = Math.hypot(vp.transform[0], vp.transform[1]);\n const glyphs: Glyph[] = [];\n let textChars = 0;\n for (const it of items) {\n if (!('str' in it) || typeof it.str !== 'string') continue;\n textChars += it.str.length;\n if (!it.str.trim()) continue;\n const transform = Util.transform(vp.transform, it.transform);\n const baselineLength = Math.hypot(transform[0], transform[1]);\n const w = Math.abs(it.width) * viewportScale;\n const h = Math.round(Math.hypot(transform[2], transform[3]) * 10) / 10;\n const baselineX = baselineLength > 0 ? (transform[0] / baselineLength) * w : w;\n const baselineY = baselineLength > 0 ? (transform[1] / baselineLength) * w : 0;\n const corners = [\n [transform[4], transform[5]],\n [transform[4] + baselineX, transform[5] + baselineY],\n [transform[4] + baselineX + transform[2], transform[5] + baselineY + transform[3]],\n [transform[4] + transform[2], transform[5] + transform[3]],\n ];\n const boundsX = corners.map(([x]) => x);\n const boundsY = corners.map(([, y]) => y);\n glyphs.push({\n s: it.str,\n x: transform[4],\n y: transform[5],\n w,\n h,\n bounds: {\n x: Math.min(...boundsX),\n y: Math.min(...boundsY),\n w: Math.max(...boundsX) - Math.min(...boundsX),\n h: Math.max(...boundsY) - Math.min(...boundsY),\n },\n });\n heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);\n }\n // \"text layer used\" for hybrid-page purposes (page text > 80 chars) — a\n // page with only a couple of stray scraps isn't \"using\" the text layer even if it also carries a large image.\n const { hasImage, largeImage } = await getPageImageSignals(page, textChars);\n pages.push({\n page: p,\n glyphs,\n width: vp.width,\n height: vp.height,\n textChars,\n hasImage,\n largeImage,\n });\n }\n // `numPages` is the SELECTED count, not doc.numPages: the auto-OCR\n // chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely \"sparse\".\n return { numPages: pages.length, pages, heightChars };\n}\n"],"names":["OPS","Util","openDocument","LARGE_IMAGE_MIN_PX","IMAGE_PAINT_OPS","Set","paintImageMaskXObject","paintImageMaskXObjectGroup","paintImageMaskXObjectRepeat","paintImageXObject","paintImageXObjectRepeat","paintInlineImageXObject","paintInlineImageXObjectGroup","paintSolidColorImageMask","getPageImageSignals","page","textChars","fnArray","argsArray","getOperatorList","hasImage","largeImage","i","length","has","w","h","pageCount","src","doc","numPages","collect","first","last","from","Math","min","max","to","pages","heightChars","Map","p","getPage","items","getTextContent","vp","getViewport","scale","viewportScale","hypot","transform","glyphs","it","str","trim","baselineLength","abs","width","round","baselineX","baselineY","corners","boundsX","map","x","boundsY","y","push","s","bounds","set","get","height"],"mappings":"AAAA,8EAA8E;AAC9E,yFAAyF;AAEzF,SAASA,GAAG,EAAEC,IAAI,QAAQ,kCAAkC;AAC5D,SAASC,YAAY,QAAQ,gBAAgB;AAG7C,yEAAyE;AACzE,2HAA2H;AAC3H,EAAE;AACF,kFAAkF;AAClF,gFAAgF;AAChF,MAAMC,qBAAqB;AAM3B,MAAMC,kBAAkB,IAAIC,IAAI;IAACL,IAAIM,qBAAqB;IAAEN,IAAIO,0BAA0B;IAAEP,IAAIQ,2BAA2B;IAAER,IAAIS,iBAAiB;IAAET,IAAIU,uBAAuB;IAAEV,IAAIW,uBAAuB;IAAEX,IAAIY,4BAA4B;IAAEZ,IAAIa,wBAAwB;CAAC;AAE7Q,eAAeC,oBAAoBC,IAAkB,EAAEC,SAAiB;IACtE,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAG,MAAMH,KAAKI,eAAe;IACzD,IAAIC,WAAW;IACf,IAAIC,aAAa;IACjB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,QAAQM,MAAM,EAAED,IAAK;QACvC,IAAIlB,gBAAgBoB,GAAG,CAACP,OAAO,CAACK,EAAE,GAAGF,WAAW;QAChD,IAAIH,OAAO,CAACK,EAAE,KAAKtB,IAAIS,iBAAiB,EAAE;QAC1C,MAAM,GAAGgB,GAAGC,EAAE,GAAGR,SAAS,CAACI,EAAE;QAC7B,IAAI,OAAOG,MAAM,YAAY,OAAOC,MAAM,UAAU;YAClD,IAAID,KAAKtB,sBAAsBuB,KAAKvB,oBAAoBkB,aAAa;QACvE,OAAOA,aAAa,MAAM,gDAAgD;IAC5E;IACA,OAAO;QAAED;QAAUC,YAAYL,YAAY,MAAMK;IAAW;AAC9D;AAEA,sEAAsE;AACtE,wJAAwJ;AACxJ,EAAE;AACF,4EAA4E;AAC5E,wJAAwJ;AAExJ,0EAA0E;AAC1E,mJAAmJ;AACnJ,OAAO,eAAeM,UAAUC,GAAW;IACzC,MAAMC,MAAM,MAAM3B,aAAa0B;IAC/B,OAAOC,IAAIC,QAAQ;AACrB;AAEA,OAAO,eAAeC,QAAQH,GAAW,EAAE,EAAEI,KAAK,EAAEC,IAAI,EAAqC,GAAG,CAAC,CAAC;IAChG,MAAMJ,MAAM,MAAM3B,aAAa0B;IAC/B,MAAMM,OAAOC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,GAAGL,kBAAAA,mBAAAA,QAAS,IAAIG,KAAKE,GAAG,CAAC,GAAGR,IAAIC,QAAQ;IACvE,MAAMQ,KAAKH,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACH,MAAMD,iBAAAA,kBAAAA,OAAQJ,IAAIC,QAAQ,GAAGD,IAAIC,QAAQ;IACtE,MAAMS,QAAyB,EAAE;IACjC,MAAMC,cAAc,IAAIC;IACxB,IAAK,IAAIC,IAAIR,MAAMQ,KAAKJ,IAAII,IAAK;QAC/B,MAAM3B,OAAO,MAAMc,IAAIc,OAAO,CAACD;QAC/B,MAAM,EAAEE,KAAK,EAAE,GAAG,MAAM7B,KAAK8B,cAAc;QAC3C,MAAMC,KAAK/B,KAAKgC,WAAW,CAAC;YAAEC,OAAO;QAAE;QACvC,MAAMC,gBAAgBd,KAAKe,KAAK,CAACJ,GAAGK,SAAS,CAAC,EAAE,EAAEL,GAAGK,SAAS,CAAC,EAAE;QACjE,MAAMC,SAAkB,EAAE;QAC1B,IAAIpC,YAAY;QAChB,KAAK,MAAMqC,MAAMT,MAAO;YACtB,IAAI,CAAE,CAAA,SAASS,EAAC,KAAM,OAAOA,GAAGC,GAAG,KAAK,UAAU;YAClDtC,aAAaqC,GAAGC,GAAG,CAAC/B,MAAM;YAC1B,IAAI,CAAC8B,GAAGC,GAAG,CAACC,IAAI,IAAI;YACpB,MAAMJ,YAAYlD,KAAKkD,SAAS,CAACL,GAAGK,SAAS,EAAEE,GAAGF,SAAS;YAC3D,MAAMK,iBAAiBrB,KAAKe,KAAK,CAACC,SAAS,CAAC,EAAE,EAAEA,SAAS,CAAC,EAAE;YAC5D,MAAM1B,IAAIU,KAAKsB,GAAG,CAACJ,GAAGK,KAAK,IAAIT;YAC/B,MAAMvB,IAAIS,KAAKwB,KAAK,CAACxB,KAAKe,KAAK,CAACC,SAAS,CAAC,EAAE,EAAEA,SAAS,CAAC,EAAE,IAAI,MAAM;YACpE,MAAMS,YAAYJ,iBAAiB,IAAI,AAACL,SAAS,CAAC,EAAE,GAAGK,iBAAkB/B,IAAIA;YAC7E,MAAMoC,YAAYL,iBAAiB,IAAI,AAACL,SAAS,CAAC,EAAE,GAAGK,iBAAkB/B,IAAI;YAC7E,MAAMqC,UAAU;gBACd;oBAACX,SAAS,CAAC,EAAE;oBAAEA,SAAS,CAAC,EAAE;iBAAC;gBAC5B;oBAACA,SAAS,CAAC,EAAE,GAAGS;oBAAWT,SAAS,CAAC,EAAE,GAAGU;iBAAU;gBACpD;oBAACV,SAAS,CAAC,EAAE,GAAGS,YAAYT,SAAS,CAAC,EAAE;oBAAEA,SAAS,CAAC,EAAE,GAAGU,YAAYV,SAAS,CAAC,EAAE;iBAAC;gBAClF;oBAACA,SAAS,CAAC,EAAE,GAAGA,SAAS,CAAC,EAAE;oBAAEA,SAAS,CAAC,EAAE,GAAGA,SAAS,CAAC,EAAE;iBAAC;aAC3D;YACD,MAAMY,UAAUD,QAAQE,GAAG,CAAC,CAAC,CAACC,EAAE,GAAKA;YACrC,MAAMC,UAAUJ,QAAQE,GAAG,CAAC,CAAC,GAAGG,EAAE,GAAKA;YACvCf,OAAOgB,IAAI,CAAC;gBACVC,GAAGhB,GAAGC,GAAG;gBACTW,GAAGd,SAAS,CAAC,EAAE;gBACfgB,GAAGhB,SAAS,CAAC,EAAE;gBACf1B;gBACAC;gBACA4C,QAAQ;oBACNL,GAAG9B,KAAKC,GAAG,IAAI2B;oBACfI,GAAGhC,KAAKC,GAAG,IAAI8B;oBACfzC,GAAGU,KAAKE,GAAG,IAAI0B,WAAW5B,KAAKC,GAAG,IAAI2B;oBACtCrC,GAAGS,KAAKE,GAAG,IAAI6B,WAAW/B,KAAKC,GAAG,IAAI8B;gBACxC;YACF;YACA1B,YAAY+B,GAAG,CAAC7C,GAAG,AAACc,CAAAA,YAAYgC,GAAG,CAAC9C,MAAM,CAAA,IAAK2B,GAAGC,GAAG,CAAC/B,MAAM;QAC9D;QACA,wEAAwE;QACxE,8GAA8G;QAC9G,MAAM,EAAEH,QAAQ,EAAEC,UAAU,EAAE,GAAG,MAAMP,oBAAoBC,MAAMC;QACjEuB,MAAM6B,IAAI,CAAC;YACTrD,MAAM2B;YACNU;YACAM,OAAOZ,GAAGY,KAAK;YACfe,QAAQ3B,GAAG2B,MAAM;YACjBzD;YACAI;YACAC;QACF;IACF;IACA,mEAAmE;IACnE,sIAAsI;IACtI,OAAO;QAAES,UAAUS,MAAMhB,MAAM;QAAEgB;QAAOC;IAAY;AACtD"}
|
package/dist/esm/errors.d.ts
CHANGED
|
@@ -11,6 +11,8 @@ export type PdfToMdErrorCode =
|
|
|
11
11
|
| 'OCR_PAGE_FAILED'
|
|
12
12
|
/** malformed word-dump input (--words-json / extractOcr words) */
|
|
13
13
|
| 'WORDS_INPUT'
|
|
14
|
+
/** malformed or insufficient evidence passed to auditWords */
|
|
15
|
+
| 'AUDIT_INPUT'
|
|
14
16
|
/** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range
|
|
15
17
|
* wordIndexes entry) — validated fail-fast, whole object checked before any output. */
|
|
16
18
|
| 'ANALYSIS_INPUT';
|
package/dist/esm/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/errors.ts"],"sourcesContent":["// Library code throws; only src/cli.ts prints and exits.\n\n/** The closed set of error codes every `PdfToMdError` carries — switch on\n * `code` instead of parsing `message`. */\nexport type PdfToMdErrorCode =\n /** input missing/unreadable/not a PDF */\n | 'PDF_OPEN'\n /** malformed page selection (validated before any work) */\n | 'PAGE_RANGE'\n /** rendering an opened, in-range page to PNG failed */\n | 'PAGE_RENDER'\n /** the OCR engine crashed twice on one page (message names it) */\n | 'OCR_PAGE_FAILED'\n /** malformed word-dump input (--words-json / extractOcr words) */\n | 'WORDS_INPUT'\n /** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range\n * wordIndexes entry) — validated fail-fast, whole object checked before any output. */\n | 'ANALYSIS_INPUT';\n\n/** The error every entry point throws — never a bare `Error`. */\nexport class PdfToMdError extends Error {\n code: PdfToMdErrorCode;\n\n constructor(code: PdfToMdErrorCode, message: string) {\n super(message);\n this.name = 'PdfToMdError';\n this.code = code;\n }\n}\n"],"names":["PdfToMdError","Error","code","message","name"],"mappings":"AAAA,yDAAyD;AAEzD;wCACwC,
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/errors.ts"],"sourcesContent":["// Library code throws; only src/cli.ts prints and exits.\n\n/** The closed set of error codes every `PdfToMdError` carries — switch on\n * `code` instead of parsing `message`. */\nexport type PdfToMdErrorCode =\n /** input missing/unreadable/not a PDF */\n | 'PDF_OPEN'\n /** malformed page selection (validated before any work) */\n | 'PAGE_RANGE'\n /** rendering an opened, in-range page to PNG failed */\n | 'PAGE_RENDER'\n /** the OCR engine crashed twice on one page (message names it) */\n | 'OCR_PAGE_FAILED'\n /** malformed word-dump input (--words-json / extractOcr words) */\n | 'WORDS_INPUT'\n /** malformed or insufficient evidence passed to auditWords */\n | 'AUDIT_INPUT'\n /** malformed Analysis handed to toMarkdown/toText (e.g. an out-of-range\n * wordIndexes entry) — validated fail-fast, whole object checked before any output. */\n | 'ANALYSIS_INPUT';\n\n/** The error every entry point throws — never a bare `Error`. */\nexport class PdfToMdError extends Error {\n code: PdfToMdErrorCode;\n\n constructor(code: PdfToMdErrorCode, message: string) {\n super(message);\n this.name = 'PdfToMdError';\n this.code = code;\n }\n}\n"],"names":["PdfToMdError","Error","code","message","name"],"mappings":"AAAA,yDAAyD;AAEzD;wCACwC,GAkBxC,+DAA+D,GAC/D,OAAO,MAAMA,qBAAqBC;IAGhC,YAAYC,IAAsB,EAAEC,OAAe,CAAE;QACnD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}
|
package/dist/esm/extract.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { OcrProgressEvent } from './engines/tesseract.js';
|
|
2
2
|
import type { PageWord, Tuning } from './geometry.js';
|
|
3
|
-
import type { Report } from './types.js';
|
|
3
|
+
import type { Analysis, Report } from './types.js';
|
|
4
4
|
export type { OcrProgressEvent } from './engines/tesseract.js';
|
|
5
5
|
/** One recognized word with its page — the OCR replay input contract (same
|
|
6
6
|
* shape as `geometry.ts`'s `PageWord`, re-exported under this module's public name). */
|
|
@@ -34,6 +34,9 @@ export interface ExtractTextOptions extends CommonOptions {
|
|
|
34
34
|
/** emit a `### pN` marker before each page's content (output formatting
|
|
35
35
|
* only — `report.pageStats` is always populated regardless) */
|
|
36
36
|
pageMarkers?: boolean;
|
|
37
|
+
/** source PDF text runs entering layout, delivered once for the selected
|
|
38
|
+
* final result. Runs retain measured bounds and are not fabricated words. */
|
|
39
|
+
onWords?: (words: OcrWordInput[]) => void;
|
|
37
40
|
}
|
|
38
41
|
/** Options for {@link extractOcr}. */
|
|
39
42
|
export interface ExtractOcrOptions extends CommonOptions {
|
|
@@ -59,6 +62,10 @@ export interface PdfToMarkdownOptions extends ExtractTextOptions, Omit<ExtractOc
|
|
|
59
62
|
/** never fall back to OCR, even for image-only pages */
|
|
60
63
|
noOcr?: boolean;
|
|
61
64
|
}
|
|
65
|
+
/** Extract embedded text with a QA report and warnings; never runs OCR.
|
|
66
|
+
* @param pdfPath path to the PDF file
|
|
67
|
+
* @param opts see {@link ExtractTextOptions}
|
|
68
|
+
*/
|
|
62
69
|
export declare function extractText(pdfPath: string, opts?: ExtractTextOptions): Promise<ExtractResult>;
|
|
63
70
|
/**
|
|
64
71
|
* OCR extraction. Input is either a PDF (pages render at `dpi` and the
|
|
@@ -72,15 +79,19 @@ export declare function extractOcr(input: {
|
|
|
72
79
|
} | {
|
|
73
80
|
words: OcrWordInput[];
|
|
74
81
|
}, opts?: ExtractOcrOptions): Promise<ExtractResult>;
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
82
|
+
interface AutoAnalysisResult {
|
|
83
|
+
analysis: Analysis;
|
|
84
|
+
markdown: string;
|
|
85
|
+
warnings: string[];
|
|
86
|
+
}
|
|
87
|
+
/** Shared automatic route for the library's Markdown result and the CLI's
|
|
88
|
+
* raw/plain-text formats. A page changes producer only when it has image
|
|
89
|
+
* evidence and an unusable text layer, so a blank or vector-only page never
|
|
90
|
+
* becomes speculative OCR input. */
|
|
91
|
+
export declare function analyzePdfAuto(pdfPath: string, opts?: PdfToMarkdownOptions): Promise<AutoAnalysisResult>;
|
|
81
92
|
/**
|
|
82
|
-
*
|
|
83
|
-
*
|
|
93
|
+
* Extract each page from embedded text, using OCR only when the page has
|
|
94
|
+
* image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
|
|
84
95
|
*
|
|
85
96
|
* @param pdfPath path to the PDF file
|
|
86
97
|
* @param opts see {@link PdfToMarkdownOptions}
|
package/dist/esm/extract.js
CHANGED
|
@@ -6,18 +6,11 @@
|
|
|
6
6
|
import { buildOcrAnalysisPages } from './analyze.js';
|
|
7
7
|
import { collect } from './collect.js';
|
|
8
8
|
import { dictAvailable, dictUnavailableWarning } from './dict.js';
|
|
9
|
-
import { toLines } from './lines.js';
|
|
10
|
-
import { buildMarkdown } from './markdown.js';
|
|
11
9
|
import { renderPagesToMarkdown } from './render-analysis.js';
|
|
12
|
-
import { AUTO_OCR_CHARS_PER_PAGE, buildOcrReport
|
|
10
|
+
import { AUTO_OCR_CHARS_PER_PAGE, buildOcrReport } from './report.js';
|
|
11
|
+
import { buildTextAnalysis, selectCollectedPages } from './text-analysis.js';
|
|
13
12
|
import { validatePages } from './validate.js';
|
|
14
|
-
|
|
15
|
-
* Text-layer extraction. Returns markdown + QA report + warnings (hybrid
|
|
16
|
-
* pages carrying large images whose baked-in text the text layer can never see). Makes no OCR decision — see pdfToMarkdown for the fallback.
|
|
17
|
-
*
|
|
18
|
-
* @param pdfPath path to the PDF file
|
|
19
|
-
* @param opts see {@link ExtractTextOptions}
|
|
20
|
-
*/ // One collector per entry point: collects into `warnings`, streams via
|
|
13
|
+
// One collector per entry point: collects into `warnings`, streams via
|
|
21
14
|
// onWarning, and emits the dictionary-unavailable notice exactly once up front.
|
|
22
15
|
function warnCollector(onWarning) {
|
|
23
16
|
const warnings = [];
|
|
@@ -31,47 +24,25 @@ function warnCollector(onWarning) {
|
|
|
31
24
|
warn
|
|
32
25
|
};
|
|
33
26
|
}
|
|
34
|
-
|
|
27
|
+
/** Extract embedded text with a QA report and warnings; never runs OCR.
|
|
28
|
+
* @param pdfPath path to the PDF file
|
|
29
|
+
* @param opts see {@link ExtractTextOptions}
|
|
30
|
+
*/ export async function extractText(pdfPath, opts = {}) {
|
|
35
31
|
var _opts_pages_last, _opts_pageMarkers;
|
|
32
|
+
var _opts_onWords;
|
|
36
33
|
validatePages(opts.pages);
|
|
37
34
|
const { warnings, warn } = warnCollector(opts.onWarning);
|
|
38
35
|
const range = opts.pages ? {
|
|
39
36
|
first: opts.pages.first,
|
|
40
37
|
last: (_opts_pages_last = opts.pages.last) !== null && _opts_pages_last !== void 0 ? _opts_pages_last : opts.pages.first
|
|
41
38
|
} : {};
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}));
|
|
47
|
-
const pageMarkers = (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false;
|
|
48
|
-
const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, {
|
|
49
|
-
pageMarkers
|
|
50
|
-
});
|
|
51
|
-
// pageStats needs "### pN" markers to segment content; when the caller
|
|
52
|
-
// didn't request them, build a throwaway marked variant (same heightChars) purely to measure stats from, so pageStats is always populated.
|
|
53
|
-
const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, {
|
|
54
|
-
pageMarkers: true
|
|
55
|
-
}).md;
|
|
56
|
-
// Advisory, not prescriptive: `hasLargeImage` fires on any large image,
|
|
57
|
-
// even one with no baked-in text, where `--ocr` recovers nothing and can
|
|
58
|
-
// actively degrade a correct result (measured: "I" -> "|", stray em-dashes) — so this tells the caller to COMPARE the OCR rerun, never blindly replace.
|
|
59
|
-
const hybridPages = pages.filter((p)=>p.largeImage).map((p)=>p.page);
|
|
60
|
-
for (const n of hybridPages){
|
|
61
|
-
warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);
|
|
62
|
-
}
|
|
63
|
-
const report = buildReport({
|
|
64
|
-
numPages,
|
|
65
|
-
md,
|
|
66
|
-
statsMd,
|
|
67
|
-
stats,
|
|
68
|
-
bodyH,
|
|
69
|
-
headingSizes,
|
|
70
|
-
hybridPages
|
|
71
|
-
});
|
|
39
|
+
const collected = await collect(pdfPath, range);
|
|
40
|
+
const text = buildTextAnalysis(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
|
|
41
|
+
textNotices(collected, new Set(), warn);
|
|
42
|
+
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, text.words);
|
|
72
43
|
return {
|
|
73
|
-
markdown:
|
|
74
|
-
report,
|
|
44
|
+
markdown: text.markdown,
|
|
45
|
+
report: text.analysis.report,
|
|
75
46
|
warnings
|
|
76
47
|
};
|
|
77
48
|
}
|
|
@@ -104,44 +75,164 @@ export async function extractText(pdfPath, opts = {}) {
|
|
|
104
75
|
warnings
|
|
105
76
|
};
|
|
106
77
|
}
|
|
107
|
-
/**
|
|
108
|
-
*
|
|
109
|
-
return
|
|
78
|
+
/** A page needs OCR only when its image signal accompanies an unusable text
|
|
79
|
+
* layer. The caller supplies the image part of that decision. */ function needsPageOcr(chars) {
|
|
80
|
+
return chars < AUTO_OCR_CHARS_PER_PAGE;
|
|
81
|
+
}
|
|
82
|
+
function textNotices(collected, ocrPages, warn) {
|
|
83
|
+
for (const page of collected.pages){
|
|
84
|
+
if (page.textChars !== 0 || ocrPages.has(page.page)) continue;
|
|
85
|
+
if (page.hasImage) warn(`p${page.page}: no extractable text; page has image content and OCR was not selected`);
|
|
86
|
+
else warn(`p${page.page}: no extractable text; the page may be blank or contain only vector content`);
|
|
87
|
+
}
|
|
88
|
+
for (const page of collected.pages){
|
|
89
|
+
if (!page.largeImage || ocrPages.has(page.page)) continue;
|
|
90
|
+
warn(`p${page.page}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${page.page} --ocr, which can itself introduce errors`);
|
|
91
|
+
}
|
|
110
92
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
93
|
+
function analysisWords(pages) {
|
|
94
|
+
return pages.flatMap((page)=>page.words.map((word)=>({
|
|
95
|
+
page: page.page,
|
|
96
|
+
text: word.text,
|
|
97
|
+
...word.box,
|
|
98
|
+
confidence: word.confidence
|
|
99
|
+
})));
|
|
114
100
|
}
|
|
115
|
-
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
onWarning: undefined
|
|
131
|
-
});
|
|
132
|
-
if (needsOcrFallback(text.report.charsPerPage) && !opts.noOcr) {
|
|
133
|
-
var _opts_onWarning1;
|
|
134
|
-
const notice = ocrFallbackNotice(text.report.charsPerPage);
|
|
135
|
-
(_opts_onWarning1 = opts.onWarning) === null || _opts_onWarning1 === void 0 ? void 0 : _opts_onWarning1.call(opts, notice);
|
|
136
|
-
const ocr = await extractOcr({
|
|
101
|
+
/** Shared automatic route for the library's Markdown result and the CLI's
|
|
102
|
+
* raw/plain-text formats. A page changes producer only when it has image
|
|
103
|
+
* evidence and an unusable text layer, so a blank or vector-only page never
|
|
104
|
+
* becomes speculative OCR input. */ export async function analyzePdfAuto(pdfPath, opts = {}) {
|
|
105
|
+
var _opts_pages_last;
|
|
106
|
+
var _opts_onWords;
|
|
107
|
+
validatePages(opts.pages);
|
|
108
|
+
const { warnings, warn } = warnCollector(opts.onWarning);
|
|
109
|
+
const range = opts.pages ? {
|
|
110
|
+
first: opts.pages.first,
|
|
111
|
+
last: (_opts_pages_last = opts.pages.last) !== null && _opts_pages_last !== void 0 ? _opts_pages_last : opts.pages.first
|
|
112
|
+
} : {};
|
|
113
|
+
if (opts.ocr) {
|
|
114
|
+
var _opts_onWords1;
|
|
115
|
+
const { pages, words } = await buildOcrAnalysisPages({
|
|
137
116
|
pdfPath
|
|
138
117
|
}, {
|
|
139
118
|
...opts,
|
|
140
|
-
|
|
119
|
+
onWords: undefined
|
|
120
|
+
}, warn);
|
|
121
|
+
const markdown = renderPagesToMarkdown(pages, {
|
|
122
|
+
forcePageMarkers: true,
|
|
123
|
+
pageMarkers: true
|
|
141
124
|
});
|
|
142
|
-
|
|
143
|
-
|
|
125
|
+
const report = buildOcrReport({
|
|
126
|
+
md: markdown,
|
|
127
|
+
words
|
|
128
|
+
});
|
|
129
|
+
const analysis = {
|
|
130
|
+
outputVersion: report.outputVersion,
|
|
131
|
+
report,
|
|
132
|
+
pages
|
|
133
|
+
};
|
|
134
|
+
(_opts_onWords1 = opts.onWords) === null || _opts_onWords1 === void 0 ? void 0 : _opts_onWords1.call(opts, words);
|
|
135
|
+
return {
|
|
136
|
+
analysis,
|
|
137
|
+
markdown,
|
|
138
|
+
warnings
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
const collected = await collect(pdfPath, range);
|
|
142
|
+
const ocrPageNumbers = new Set(!opts.noOcr ? collected.pages.filter((page)=>page.hasImage && needsPageOcr(page.textChars)).map((page)=>page.page) : []);
|
|
143
|
+
if (!ocrPageNumbers.size) {
|
|
144
|
+
var _opts_pageMarkers;
|
|
145
|
+
var _opts_onWords2;
|
|
146
|
+
const text = buildTextAnalysis(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
|
|
147
|
+
textNotices(collected, ocrPageNumbers, warn);
|
|
148
|
+
(_opts_onWords2 = opts.onWords) === null || _opts_onWords2 === void 0 ? void 0 : _opts_onWords2.call(opts, text.words);
|
|
149
|
+
return {
|
|
150
|
+
analysis: text.analysis,
|
|
151
|
+
markdown: text.markdown,
|
|
152
|
+
warnings
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
const textPageNumbers = new Set(collected.pages.filter((page)=>!ocrPageNumbers.has(page.page)).map((page)=>page.page));
|
|
156
|
+
const text = textPageNumbers.size ? buildTextAnalysis(selectCollectedPages(collected, textPageNumbers), true) : undefined;
|
|
157
|
+
const textByPage = new Map(text === null || text === void 0 ? void 0 : text.analysis.pages.map((page)=>[
|
|
158
|
+
page.page,
|
|
159
|
+
page
|
|
160
|
+
]));
|
|
161
|
+
const ocrByPage = new Map();
|
|
162
|
+
const ocrWords = [];
|
|
163
|
+
const ocrRanges = [];
|
|
164
|
+
for (const page of collected.pages){
|
|
165
|
+
if (!ocrPageNumbers.has(page.page)) continue;
|
|
166
|
+
warn(`p${page.page}: sparse text layer with image content; using OCR for this page`);
|
|
167
|
+
const lastRange = ocrRanges.at(-1);
|
|
168
|
+
if (lastRange && lastRange.last + 1 === page.page) lastRange.last = page.page;
|
|
169
|
+
else ocrRanges.push({
|
|
170
|
+
first: page.page,
|
|
171
|
+
last: page.page
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
for (const range of ocrRanges){
|
|
175
|
+
const result = await buildOcrAnalysisPages({
|
|
176
|
+
pdfPath
|
|
177
|
+
}, {
|
|
178
|
+
...opts,
|
|
179
|
+
pages: range,
|
|
180
|
+
onWords: undefined
|
|
181
|
+
}, warn);
|
|
182
|
+
for (const page of result.pages)ocrByPage.set(page.page, page);
|
|
183
|
+
ocrWords.push(...result.words);
|
|
184
|
+
}
|
|
185
|
+
textNotices(collected, ocrPageNumbers, warn);
|
|
186
|
+
const pages = collected.pages.map((page)=>{
|
|
187
|
+
var _ocrByPage_get;
|
|
188
|
+
return (_ocrByPage_get = ocrByPage.get(page.page)) !== null && _ocrByPage_get !== void 0 ? _ocrByPage_get : textByPage.get(page.page);
|
|
189
|
+
});
|
|
190
|
+
const markdown = renderPagesToMarkdown(pages, {
|
|
191
|
+
forcePageMarkers: true,
|
|
192
|
+
pageMarkers: true
|
|
193
|
+
});
|
|
194
|
+
const pagePaths = new Map(pages.map((page)=>[
|
|
195
|
+
page.page,
|
|
196
|
+
page.path
|
|
197
|
+
]));
|
|
198
|
+
const report = buildOcrReport({
|
|
199
|
+
md: markdown,
|
|
200
|
+
fallback: true,
|
|
201
|
+
words: ocrWords,
|
|
202
|
+
pagePaths,
|
|
203
|
+
hybridPages: text === null || text === void 0 ? void 0 : text.analysis.report.hybridPages
|
|
204
|
+
});
|
|
205
|
+
if (text) {
|
|
206
|
+
report.bodyFontHeight = text.analysis.report.bodyFontHeight;
|
|
207
|
+
report.headingSizes = text.analysis.report.headingSizes;
|
|
208
|
+
report.listItems = text.analysis.report.listItems;
|
|
209
|
+
report.joinedPairs = text.analysis.report.joinedPairs;
|
|
210
|
+
report.closedHyphens = text.analysis.report.closedHyphens;
|
|
211
|
+
report.hybridPages = text.analysis.report.hybridPages;
|
|
144
212
|
}
|
|
145
|
-
|
|
146
|
-
|
|
213
|
+
const analysis = {
|
|
214
|
+
outputVersion: report.outputVersion,
|
|
215
|
+
report,
|
|
216
|
+
pages
|
|
217
|
+
};
|
|
218
|
+
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, analysisWords(pages));
|
|
219
|
+
return {
|
|
220
|
+
analysis,
|
|
221
|
+
markdown,
|
|
222
|
+
warnings
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Extract each page from embedded text, using OCR only when the page has
|
|
227
|
+
* image content and fewer than `AUTO_OCR_CHARS_PER_PAGE` text characters.
|
|
228
|
+
*
|
|
229
|
+
* @param pdfPath path to the PDF file
|
|
230
|
+
* @param opts see {@link PdfToMarkdownOptions}
|
|
231
|
+
*/ export async function pdfToMarkdown(pdfPath, opts = {}) {
|
|
232
|
+
const result = await analyzePdfAuto(pdfPath, opts);
|
|
233
|
+
return {
|
|
234
|
+
markdown: result.markdown,
|
|
235
|
+
report: result.analysis.report,
|
|
236
|
+
warnings: result.warnings
|
|
237
|
+
};
|
|
147
238
|
}
|