@effortlessmotion/pdf-to-md 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -0
- package/bin/cli.js +9 -0
- package/dist/cjs/analyze.d.cts +35 -0
- package/dist/cjs/analyze.d.ts +35 -0
- package/dist/cjs/analyze.js +620 -0
- package/dist/cjs/analyze.js.map +1 -0
- package/dist/cjs/audit.d.cts +38 -0
- package/dist/cjs/audit.d.ts +38 -0
- package/dist/cjs/audit.js +252 -0
- package/dist/cjs/audit.js.map +1 -0
- package/dist/cjs/cli/audit.d.cts +3 -0
- package/dist/cjs/cli/audit.d.ts +3 -0
- package/dist/cjs/cli/audit.js +269 -0
- package/dist/cjs/cli/audit.js.map +1 -0
- package/dist/cjs/cli/extract.d.cts +3 -0
- package/dist/cjs/cli/extract.d.ts +3 -0
- package/dist/cjs/cli/extract.js +704 -0
- package/dist/cjs/cli/extract.js.map +1 -0
- package/dist/cjs/cli/index.d.cts +5 -0
- package/dist/cjs/cli/index.d.ts +5 -0
- package/dist/cjs/cli/index.js +58 -0
- package/dist/cjs/cli/index.js.map +1 -0
- package/dist/cjs/cli/render.d.cts +3 -0
- package/dist/cjs/cli/render.d.ts +3 -0
- package/dist/cjs/cli/render.js +361 -0
- package/dist/cjs/cli/render.js.map +1 -0
- package/dist/cjs/cli/types.d.cts +10 -0
- package/dist/cjs/cli/types.d.ts +10 -0
- package/dist/cjs/cli/types.js +7 -0
- package/dist/cjs/cli/types.js.map +1 -0
- package/dist/cjs/cli.d.cts +2 -0
- package/dist/cjs/cli.d.ts +2 -0
- package/dist/cjs/cli.js +345 -0
- package/dist/cjs/cli.js.map +1 -0
- package/dist/cjs/collect.d.cts +6 -0
- package/dist/cjs/collect.d.ts +6 -0
- package/dist/cjs/collect.js +370 -0
- package/dist/cjs/collect.js.map +1 -0
- package/dist/cjs/dict.d.cts +8 -0
- package/dist/cjs/dict.d.ts +8 -0
- package/dist/cjs/dict.js +55 -0
- package/dist/cjs/dict.js.map +1 -0
- package/dist/cjs/emit.d.cts +27 -0
- package/dist/cjs/emit.d.ts +27 -0
- package/dist/cjs/emit.js +187 -0
- package/dist/cjs/emit.js.map +1 -0
- package/dist/cjs/engines/tesseract.d.cts +30 -0
- package/dist/cjs/engines/tesseract.d.ts +30 -0
- package/dist/cjs/engines/tesseract.js +650 -0
- package/dist/cjs/engines/tesseract.js.map +1 -0
- package/dist/cjs/errors.d.cts +21 -0
- package/dist/cjs/errors.d.ts +21 -0
- package/dist/cjs/errors.js +124 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/extract.d.cts +88 -0
- package/dist/cjs/extract.d.ts +88 -0
- package/dist/cjs/extract.js +436 -0
- package/dist/cjs/extract.js.map +1 -0
- package/dist/cjs/geometry.d.cts +63 -0
- package/dist/cjs/geometry.d.ts +63 -0
- package/dist/cjs/geometry.js +521 -0
- package/dist/cjs/geometry.js.map +1 -0
- package/dist/cjs/index.d.cts +15 -0
- package/dist/cjs/index.d.ts +15 -0
- package/dist/cjs/index.js +56 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/lines.d.cts +3 -0
- package/dist/cjs/lines.d.ts +3 -0
- package/dist/cjs/lines.js +181 -0
- package/dist/cjs/lines.js.map +1 -0
- package/dist/cjs/markdown.d.cts +2 -0
- package/dist/cjs/markdown.d.ts +2 -0
- package/dist/cjs/markdown.js +222 -0
- package/dist/cjs/markdown.js.map +1 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/page-marker.d.cts +16 -0
- package/dist/cjs/page-marker.d.ts +16 -0
- package/dist/cjs/page-marker.js +85 -0
- package/dist/cjs/page-marker.js.map +1 -0
- package/dist/cjs/pdf-open.d.cts +12 -0
- package/dist/cjs/pdf-open.d.ts +12 -0
- package/dist/cjs/pdf-open.js +257 -0
- package/dist/cjs/pdf-open.js.map +1 -0
- package/dist/cjs/raster.d.cts +26 -0
- package/dist/cjs/raster.d.ts +26 -0
- package/dist/cjs/raster.js +262 -0
- package/dist/cjs/raster.js.map +1 -0
- package/dist/cjs/render-analysis.d.cts +25 -0
- package/dist/cjs/render-analysis.d.ts +25 -0
- package/dist/cjs/render-analysis.js +319 -0
- package/dist/cjs/render-analysis.js.map +1 -0
- package/dist/cjs/report.d.cts +12 -0
- package/dist/cjs/report.d.ts +12 -0
- package/dist/cjs/report.js +392 -0
- package/dist/cjs/report.js.map +1 -0
- package/dist/cjs/resolve.d.cts +1 -0
- package/dist/cjs/resolve.d.ts +1 -0
- package/dist/cjs/resolve.js +18 -0
- package/dist/cjs/resolve.js.map +1 -0
- package/dist/cjs/types.d.cts +171 -0
- package/dist/cjs/types.d.ts +171 -0
- package/dist/cjs/types.js +9 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/cjs/validate.d.cts +14 -0
- package/dist/cjs/validate.d.ts +14 -0
- package/dist/cjs/validate.js +96 -0
- package/dist/cjs/validate.js.map +1 -0
- package/dist/esm/analyze.d.ts +35 -0
- package/dist/esm/analyze.js +270 -0
- package/dist/esm/analyze.js.map +1 -0
- package/dist/esm/audit.d.ts +38 -0
- package/dist/esm/audit.js +113 -0
- package/dist/esm/audit.js.map +1 -0
- package/dist/esm/cli/audit.d.ts +3 -0
- package/dist/esm/cli/audit.js +20 -0
- package/dist/esm/cli/audit.js.map +1 -0
- package/dist/esm/cli/extract.d.ts +3 -0
- package/dist/esm/cli/extract.js +313 -0
- package/dist/esm/cli/extract.js.map +1 -0
- package/dist/esm/cli/index.d.ts +5 -0
- package/dist/esm/cli/index.js +7 -0
- package/dist/esm/cli/index.js.map +1 -0
- package/dist/esm/cli/render.d.ts +3 -0
- package/dist/esm/cli/render.js +77 -0
- package/dist/esm/cli/render.js.map +1 -0
- package/dist/esm/cli/types.d.ts +10 -0
- package/dist/esm/cli/types.js +3 -0
- package/dist/esm/cli/types.js.map +1 -0
- package/dist/esm/cli.d.ts +2 -0
- package/dist/esm/cli.js +90 -0
- package/dist/esm/cli.js.map +1 -0
- package/dist/esm/collect.d.ts +6 -0
- package/dist/esm/collect.js +80 -0
- package/dist/esm/collect.js.map +1 -0
- package/dist/esm/dict.d.ts +8 -0
- package/dist/esm/dict.js +28 -0
- package/dist/esm/dict.js.map +1 -0
- package/dist/esm/emit.d.ts +27 -0
- package/dist/esm/emit.js +131 -0
- package/dist/esm/emit.js.map +1 -0
- package/dist/esm/engines/tesseract.d.ts +30 -0
- package/dist/esm/engines/tesseract.js +190 -0
- package/dist/esm/engines/tesseract.js.map +1 -0
- package/dist/esm/errors.d.ts +21 -0
- package/dist/esm/errors.js +9 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/extract.d.ts +88 -0
- package/dist/esm/extract.js +147 -0
- package/dist/esm/extract.js.map +1 -0
- package/dist/esm/geometry.d.ts +63 -0
- package/dist/esm/geometry.js +279 -0
- package/dist/esm/geometry.js.map +1 -0
- package/dist/esm/index.d.ts +15 -0
- package/dist/esm/index.js +10 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/lines.d.ts +3 -0
- package/dist/esm/lines.js +67 -0
- package/dist/esm/lines.js.map +1 -0
- package/dist/esm/markdown.d.ts +2 -0
- package/dist/esm/markdown.js +153 -0
- package/dist/esm/markdown.js.map +1 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/page-marker.d.ts +16 -0
- package/dist/esm/page-marker.js +40 -0
- package/dist/esm/page-marker.js.map +1 -0
- package/dist/esm/pdf-open.d.ts +12 -0
- package/dist/esm/pdf-open.js +61 -0
- package/dist/esm/pdf-open.js.map +1 -0
- package/dist/esm/raster.d.ts +26 -0
- package/dist/esm/raster.js +60 -0
- package/dist/esm/raster.js.map +1 -0
- package/dist/esm/render-analysis.d.ts +25 -0
- package/dist/esm/render-analysis.js +115 -0
- package/dist/esm/render-analysis.js.map +1 -0
- package/dist/esm/report.d.ts +12 -0
- package/dist/esm/report.js +182 -0
- package/dist/esm/report.js.map +1 -0
- package/dist/esm/resolve.d.ts +1 -0
- package/dist/esm/resolve.js +7 -0
- package/dist/esm/resolve.js.map +1 -0
- package/dist/esm/types.d.ts +171 -0
- package/dist/esm/types.js +6 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/validate.d.ts +14 -0
- package/dist/esm/validate.js +56 -0
- package/dist/esm/validate.js.map +1 -0
- package/package.json +67 -0
- package/skills/pdf-to-md/SKILL.md +102 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// render command — render PDF pages to PNG for human/vision-model
|
|
2
|
+
// verification (extraction output is a claim; the page itself is the truth). Thin wrapper: parse flags, call src/raster.ts, write files.
|
|
3
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { parseArgs } from 'node:util';
|
|
6
|
+
import { PdfToMdError } from '../errors.js';
|
|
7
|
+
const USAGE = 'usage: pdf-to-md render <file.pdf> <page>[-<page>] [--dpi|-d N (default 288)] [--out|-o DIR (default .)]';
|
|
8
|
+
const render = async (ctx)=>{
|
|
9
|
+
var _values_out;
|
|
10
|
+
const parsed = (()=>{
|
|
11
|
+
try {
|
|
12
|
+
return parseArgs({
|
|
13
|
+
args: ctx.rest,
|
|
14
|
+
options: {
|
|
15
|
+
dpi: {
|
|
16
|
+
type: 'string',
|
|
17
|
+
default: '288',
|
|
18
|
+
short: 'd'
|
|
19
|
+
},
|
|
20
|
+
out: {
|
|
21
|
+
type: 'string',
|
|
22
|
+
short: 'o'
|
|
23
|
+
},
|
|
24
|
+
help: {
|
|
25
|
+
type: 'boolean',
|
|
26
|
+
default: false,
|
|
27
|
+
short: 'h'
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
allowPositionals: true
|
|
31
|
+
});
|
|
32
|
+
} catch (err) {
|
|
33
|
+
// usageError exits; the throw is dead code that tells the compiler so.
|
|
34
|
+
throw ctx.usageError(`${err.message}\n${USAGE}`);
|
|
35
|
+
}
|
|
36
|
+
})();
|
|
37
|
+
const { values, positionals } = parsed;
|
|
38
|
+
if (values.help) {
|
|
39
|
+
console.log(USAGE);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
const [pdfPath, spec] = positionals;
|
|
43
|
+
if (!pdfPath || !spec) ctx.usageError(USAGE);
|
|
44
|
+
const m = /^(\d+)(?:-(\d+))?$/.exec(spec);
|
|
45
|
+
if (m === null) throw ctx.usageError(`${USAGE}\ncould not parse page spec: ${spec}`);
|
|
46
|
+
const firstPage = Number(m[1]);
|
|
47
|
+
const lastPage = m[2] !== undefined ? Number(m[2]) : firstPage;
|
|
48
|
+
if (!(firstPage >= 1) || !(lastPage >= firstPage)) throw ctx.usageError(`${USAGE}\ninvalid page spec: ${spec}`);
|
|
49
|
+
const dpi = Number.parseFloat(values.dpi);
|
|
50
|
+
if (!(dpi > 0)) ctx.usageError(`${USAGE}\n--dpi must be positive`);
|
|
51
|
+
const outDir = (_values_out = values.out) !== null && _values_out !== void 0 ? _values_out : process.cwd();
|
|
52
|
+
mkdirSync(outDir, {
|
|
53
|
+
recursive: true
|
|
54
|
+
});
|
|
55
|
+
const stem = path.basename(pdfPath, path.extname(pdfPath)).replaceAll(' ', '-');
|
|
56
|
+
// Imported here, not at module top: keeps pdfjs/canvas out of every other command's startup.
|
|
57
|
+
const { awaitPdfOpen, openPdfForRender } = await import('../pdf-open.js');
|
|
58
|
+
const { renderLoadedPage } = await import('../raster.js');
|
|
59
|
+
const loadingTask = openPdfForRender(pdfPath);
|
|
60
|
+
try {
|
|
61
|
+
const doc = await awaitPdfOpen(loadingTask, pdfPath);
|
|
62
|
+
// Whole-range check before any file is written, so a partly-valid range
|
|
63
|
+
// fails atomically; renderLoadedPage re-validates per page.
|
|
64
|
+
if (lastPage > doc.numPages) {
|
|
65
|
+
throw new PdfToMdError('PAGE_RANGE', `page range ${firstPage}-${lastPage} is outside this document (1-${doc.numPages})`);
|
|
66
|
+
}
|
|
67
|
+
for(let n = firstPage; n <= lastPage; n++){
|
|
68
|
+
const { png } = await renderLoadedPage(doc, n, dpi);
|
|
69
|
+
const outPath = path.join(outDir, `${stem}-p${n}.png`);
|
|
70
|
+
writeFileSync(outPath, png);
|
|
71
|
+
console.log(outPath);
|
|
72
|
+
}
|
|
73
|
+
} finally{
|
|
74
|
+
await loadingTask.destroy();
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
export default render;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/cli/render.ts"],"sourcesContent":["// render command — render PDF pages to PNG for human/vision-model\n// verification (extraction output is a claim; the page itself is the truth). Thin wrapper: parse flags, call src/raster.ts, write files.\nimport { mkdirSync, writeFileSync } from 'node:fs';\nimport path from 'node:path';\nimport { parseArgs } from 'node:util';\nimport { PdfToMdError } from '../errors.ts';\nimport type { Command } from './types.ts';\n\nconst USAGE = 'usage: pdf-to-md render <file.pdf> <page>[-<page>] [--dpi|-d N (default 288)] [--out|-o DIR (default .)]';\n\nconst render: Command = async (ctx) => {\n const parsed = (() => {\n try {\n return parseArgs({\n args: ctx.rest,\n options: {\n dpi: { type: 'string', default: '288', short: 'd' },\n out: { type: 'string', short: 'o' },\n help: { type: 'boolean', default: false, short: 'h' },\n },\n allowPositionals: true,\n });\n } catch (err) {\n // usageError exits; the throw is dead code that tells the compiler so.\n throw ctx.usageError(`${(err as Error).message}\\n${USAGE}`);\n }\n })();\n const { values, positionals } = parsed;\n\n if (values.help) {\n console.log(USAGE);\n return;\n }\n\n const [pdfPath, spec] = positionals;\n if (!pdfPath || !spec) ctx.usageError(USAGE);\n\n const m = /^(\\d+)(?:-(\\d+))?$/.exec(spec);\n if (m === null) throw ctx.usageError(`${USAGE}\\ncould not parse page spec: ${spec}`);\n const firstPage = Number(m[1]);\n const lastPage = m[2] !== undefined ? Number(m[2]) : firstPage;\n if (!(firstPage >= 1) || !(lastPage >= firstPage)) throw ctx.usageError(`${USAGE}\\ninvalid page spec: ${spec}`);\n\n const dpi = Number.parseFloat(values.dpi as string);\n if (!(dpi > 0)) ctx.usageError(`${USAGE}\\n--dpi must be positive`);\n\n const outDir = values.out ?? process.cwd();\n mkdirSync(outDir, { recursive: true });\n const stem = path.basename(pdfPath, path.extname(pdfPath)).replaceAll(' ', '-');\n\n // Imported here, not at module top: keeps pdfjs/canvas out of every other command's startup.\n const { awaitPdfOpen, openPdfForRender } = await import('../pdf-open.ts');\n const { renderLoadedPage } = await import('../raster.ts');\n const loadingTask = openPdfForRender(pdfPath);\n try {\n const doc = await awaitPdfOpen(loadingTask, pdfPath);\n\n // Whole-range check before any file is written, so a partly-valid range\n // fails atomically; renderLoadedPage re-validates per page.\n if (lastPage > doc.numPages) {\n throw new PdfToMdError('PAGE_RANGE', `page range ${firstPage}-${lastPage} is outside this document (1-${doc.numPages})`);\n }\n for (let n = firstPage; n <= lastPage; n++) {\n const { png } = await renderLoadedPage(doc, n, dpi);\n const outPath = path.join(outDir, `${stem}-p${n}.png`);\n writeFileSync(outPath, png);\n console.log(outPath);\n }\n } finally {\n await loadingTask.destroy();\n }\n};\n\nexport default render;\n"],"names":["mkdirSync","writeFileSync","path","parseArgs","PdfToMdError","USAGE","render","ctx","values","parsed","args","rest","options","dpi","type","default","short","out","help","allowPositionals","err","usageError","message","positionals","console","log","pdfPath","spec","m","exec","firstPage","Number","lastPage","undefined","parseFloat","outDir","process","cwd","recursive","stem","basename","extname","replaceAll","awaitPdfOpen","openPdfForRender","renderLoadedPage","loadingTask","doc","numPages","n","png","outPath","join","destroy"],"mappings":"AAAA,kEAAkE;AAClE,yIAAyI;AACzI,SAASA,SAAS,EAAEC,aAAa,QAAQ,UAAU;AACnD,OAAOC,UAAU,YAAY;AAC7B,SAASC,SAAS,QAAQ,YAAY;AACtC,SAASC,YAAY,QAAQ,eAAe;AAG5C,MAAMC,QAAQ;AAEd,MAAMC,SAAkB,OAAOC;QAoCdC;IAnCf,MAAMC,SAAS,AAAC,CAAA;QACd,IAAI;YACF,OAAON,UAAU;gBACfO,MAAMH,IAAII,IAAI;gBACdC,SAAS;oBACPC,KAAK;wBAAEC,MAAM;wBAAUC,SAAS;wBAAOC,OAAO;oBAAI;oBAClDC,KAAK;wBAAEH,MAAM;wBAAUE,OAAO;oBAAI;oBAClCE,MAAM;wBAAEJ,MAAM;wBAAWC,SAAS;wBAAOC,OAAO;oBAAI;gBACtD;gBACAG,kBAAkB;YACpB;QACF,EAAE,OAAOC,KAAK;YACZ,uEAAuE;YACvE,MAAMb,IAAIc,UAAU,CAAC,GAAG,AAACD,IAAcE,OAAO,CAAC,EAAE,EAAEjB,OAAO;QAC5D;IACF,CAAA;IACA,MAAM,EAAEG,MAAM,EAAEe,WAAW,EAAE,GAAGd;IAEhC,IAAID,OAAOU,IAAI,EAAE;QACfM,QAAQC,GAAG,CAACpB;QACZ;IACF;IAEA,MAAM,CAACqB,SAASC,KAAK,GAAGJ;IACxB,IAAI,CAACG,WAAW,CAACC,MAAMpB,IAAIc,UAAU,CAAChB;IAEtC,MAAMuB,IAAI,qBAAqBC,IAAI,CAACF;IACpC,IAAIC,MAAM,MAAM,MAAMrB,IAAIc,UAAU,CAAC,GAAGhB,MAAM,6BAA6B,EAAEsB,MAAM;IACnF,MAAMG,YAAYC,OAAOH,CAAC,CAAC,EAAE;IAC7B,MAAMI,WAAWJ,CAAC,CAAC,EAAE,KAAKK,YAAYF,OAAOH,CAAC,CAAC,EAAE,IAAIE;IACrD,IAAI,CAAEA,CAAAA,aAAa,CAAA,KAAM,CAAEE,CAAAA,YAAYF,SAAQ,GAAI,MAAMvB,IAAIc,UAAU,CAAC,GAAGhB,MAAM,qBAAqB,EAAEsB,MAAM;IAE9G,MAAMd,MAAMkB,OAAOG,UAAU,CAAC1B,OAAOK,GAAG;IACxC,IAAI,CAAEA,CAAAA,MAAM,CAAA,GAAIN,IAAIc,UAAU,CAAC,GAAGhB,MAAM,wBAAwB,CAAC;IAEjE,MAAM8B,UAAS3B,cAAAA,OAAOS,GAAG,cAAVT,yBAAAA,cAAc4B,QAAQC,GAAG;IACxCrC,UAAUmC,QAAQ;QAAEG,WAAW;IAAK;IACpC,MAAMC,OAAOrC,KAAKsC,QAAQ,CAACd,SAASxB,KAAKuC,OAAO,CAACf,UAAUgB,UAAU,CAAC,KAAK;IAE3E,6FAA6F;IAC7F,MAAM,EAAEC,YAAY,EAAEC,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC;IACxD,MAAM,EAAEC,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC;IAC1C,MAAMC,cAAcF,iBAAiBlB;IACrC,IAAI;QACF,MAAMqB,MAAM,MAAMJ,aAAaG,aAAapB;QAE5C,wEAAwE;QACxE,4DAA4D;QAC5D,IAAIM,WAAWe,IAAIC,QAAQ,EAAE;YAC3B,MAAM,IAAI5C,aAAa,cAAc,CAAC,WAAW,EAAE0B,UAAU,CAAC,EAAEE,SAAS,6BAA6B,EAAEe,IAAIC,QAAQ,CAAC,CAAC,CAAC;QACzH;QACA,IAAK,IAAIC,IAAInB,WAAWmB,KAAKjB,UAAUiB,IAAK;YAC1C,MAAM,EAAEC,GAAG,EAAE,GAAG,MAAML,iBAAiBE,KAAKE,GAAGpC;YAC/C,MAAMsC,UAAUjD,KAAKkD,IAAI,CAACjB,QAAQ,GAAGI,KAAK,EAAE,EAAEU,EAAE,IAAI,CAAC;YACrDhD,cAAckD,SAASD;YACvB1B,QAAQC,GAAG,CAAC0B;QACd;IACF,SAAU;QACR,MAAML,YAAYO,OAAO;IAC3B;AACF;AAEA,eAAe/C,OAAO"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface Ctx {
|
|
2
|
+
/** binary name, for usage strings */
|
|
3
|
+
name: string;
|
|
4
|
+
/** argv after the command word */
|
|
5
|
+
rest: string[];
|
|
6
|
+
/** print the message and exit 2 — every malformed-invocation path.
|
|
7
|
+
* Readonly property (not method) syntax: TS only applies never-return control-flow analysis to readonly property-typed function members. */
|
|
8
|
+
readonly usageError: (message: string) => never;
|
|
9
|
+
}
|
|
10
|
+
export type Command = (ctx: Ctx) => Promise<void> | void;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/cli/types.ts"],"sourcesContent":["// Handed to every command; the command word is peeled off in src/cli.ts and\n// each command owns its own parseArgs table over `rest` (our commands' flags are disjoint, so a shared table would let `render --para-gap=2` parse silently as a no-op).\nexport interface Ctx {\n /** binary name, for usage strings */\n name: string;\n /** argv after the command word */\n rest: string[];\n /** print the message and exit 2 — every malformed-invocation path.\n * Readonly property (not method) syntax: TS only applies never-return control-flow analysis to readonly property-typed function members. */\n readonly usageError: (message: string) => never;\n}\n\nexport type Command = (ctx: Ctx) => Promise<void> | void;\n"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,yKAAyK;AAWzK,WAAyD"}
|
package/dist/esm/cli.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Parsing and dispatch only. Commands live in src/cli/, one file each,
|
|
2
|
+
// lazy-loaded — nothing tree- or dependency-heavy may be imported at the top of this file (`render` must never load the OCR engine, `--version` must load almost nothing).
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { COMMANDS } from './cli/index.js';
|
|
7
|
+
// __dirname (CJS) or its ESM equivalent, then two fixed hops up to the
|
|
8
|
+
// package root. NOT `import.meta.resolve('pdf-to-md/package.json')` — Node's self-reference lookup throws ERR_MODULE_NOT_FOUND from a file this deep; a location-relative path is stable across both build targets.
|
|
9
|
+
const __dirname = path.dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));
|
|
10
|
+
function packageVersion() {
|
|
11
|
+
try {
|
|
12
|
+
var _pkg_version;
|
|
13
|
+
const pkgPath = path.resolve(__dirname, '..', '..', 'package.json');
|
|
14
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
|
|
15
|
+
return (_pkg_version = pkg.version) !== null && _pkg_version !== void 0 ? _pkg_version : 'unknown';
|
|
16
|
+
} catch {
|
|
17
|
+
return 'unknown';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
// Exported so the comparison is unit-testable without spawning a subprocess
|
|
21
|
+
// with a faked Node version. null means unparseable input, not "not below".
|
|
22
|
+
export function isBelowNodeFloor(version, floor) {
|
|
23
|
+
const parts = (v)=>v.replace(/^v/, '').replace(/^>=/, '').split('.').map(Number);
|
|
24
|
+
const v = parts(version);
|
|
25
|
+
const f = parts(floor);
|
|
26
|
+
if (v.length !== 3 || f.length !== 3 || v.some(Number.isNaN) || f.some(Number.isNaN)) return null;
|
|
27
|
+
const [rMaj, rMin, rPat] = v;
|
|
28
|
+
const [fMaj, fMin, fPat] = f;
|
|
29
|
+
return rMaj !== fMaj ? rMaj < fMaj : rMin !== fMin ? rMin < fMin : rPat < fPat;
|
|
30
|
+
}
|
|
31
|
+
// Reads engines.node from package.json (kept as the single source of truth,
|
|
32
|
+
// not hardcoded here) and returns it iff process.version is below it.
|
|
33
|
+
function belowNodeFloor(version = process.version) {
|
|
34
|
+
try {
|
|
35
|
+
var _pkg_engines;
|
|
36
|
+
const pkgPath = path.resolve(__dirname, '..', '..', 'package.json');
|
|
37
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
|
|
38
|
+
const floor = (_pkg_engines = pkg.engines) === null || _pkg_engines === void 0 ? void 0 : _pkg_engines.node;
|
|
39
|
+
if (!floor) return null;
|
|
40
|
+
return isBelowNodeFloor(version, floor) ? floor.replace(/^>=/, '') : null;
|
|
41
|
+
} catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function usage(name) {
|
|
46
|
+
return `usage: ${name} <file.pdf> [out.md] [flags...] convert a PDF to markdown\n` + ` ${name} extract <file.pdf> [out.md] [flags...] (same, explicit)\n` + ` ${name} render <file.pdf> <page>[-<page>] [--dpi|-d N (default 288)] [--out|-o DIR (default .)]\n` + ` ${name} audit <words.jsonl> <file.md> severity-1 word-loss check\n` + ` ${name} --version | --help\n` + '\n' + 'extract flags:\n' + ' --ocr force the OCR path (image-only/scanned pages)\n' + ' --no-ocr never fall back to OCR; keep sparse text output\n' + ' --pages N[-M] convert only these 1-based source pages\n' + ' --stdout write markdown to stdout instead of a file\n' + ' --json print the full QA report as JSON\n' + ` --page-markers emit a "### pN" heading per page\n` + ' --format md|txt|raw output format (default md; raw = Analysis JSON incl. the\n' + ' report; txt = reading-order plain text)\n' + ' --words-json=FILE OCR path from a recognized-word dump (offline replay)\n' + ' --debug-words=FILE dump every recognized word as JSON lines to FILE\n' + ' --dpi=N OCR render resolution (default 288; raise for tiny print or\n' + ' chart/label pages, >~400 measured worse on ordinary prose)\n' + '\n' + `extract tuning (measured defaults; see README's tuning table):\n` + ' --float-max-words=N --float-max-width=F --float-margin=F --para-gap=F\n' + ' --heading-scale=F --col-max-width=F --col-split=F --line-y-tol=F\n' + ' --line-height-ratio=F\n' + '\n' + 'examples:\n' + ` ${name} book.pdf # text layer -> book.md (auto-OCR if scanned)\n` + ` ${name} scan.pdf out.md --ocr --json # force OCR, print the QA report\n` + ` ${name} book.pdf --pages 27-31 --stdout # re-extract a page range to stdout\n` + ` ${name} render book.pdf 27 # render p27 to PNG for eyeballing (288 dpi)\n` + ` ${name} audit words.jsonl book.md # check no recognized word was lost`;
|
|
47
|
+
}
|
|
48
|
+
// Thrown errors -> exit 1 with the message verbatim; usage errors exit 2.
|
|
49
|
+
export default async function cli(argv, name) {
|
|
50
|
+
const floor = belowNodeFloor();
|
|
51
|
+
if (floor) console.error(`${name}: Node ${process.version} is below the required >=${floor} — the OCR path can silently find no words instead of erroring; upgrade Node.`);
|
|
52
|
+
if (argv[0] === '--version' || argv[0] === '-v') {
|
|
53
|
+
console.log(packageVersion());
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (argv.length === 0 || argv[0] === '--help' || argv[0] === '-h') {
|
|
57
|
+
// No args is a usage ERROR (bare `pdf-to-md` can't do anything); asking
|
|
58
|
+
// for --help is not.
|
|
59
|
+
if (argv.length === 0) {
|
|
60
|
+
console.error(usage(name));
|
|
61
|
+
process.exit(2);
|
|
62
|
+
}
|
|
63
|
+
console.log(`${name} v${packageVersion()}`);
|
|
64
|
+
console.log('');
|
|
65
|
+
console.log(usage(name));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const ctx = {
|
|
69
|
+
name,
|
|
70
|
+
rest: argv.slice(1),
|
|
71
|
+
usageError (message) {
|
|
72
|
+
console.error(message);
|
|
73
|
+
process.exit(2);
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
try {
|
|
77
|
+
const load = COMMANDS[argv[0]];
|
|
78
|
+
if (load) {
|
|
79
|
+
await (await load()).default(ctx);
|
|
80
|
+
} else {
|
|
81
|
+
// Compat shim: `pdf-to-md file.pdf [...]` — any first word that isn't
|
|
82
|
+
// a reserved command is extract's own first argument.
|
|
83
|
+
ctx.rest = argv;
|
|
84
|
+
await (await COMMANDS.extract()).default(ctx);
|
|
85
|
+
}
|
|
86
|
+
} catch (err) {
|
|
87
|
+
console.error(err.message);
|
|
88
|
+
process.exit(1);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/cli.ts"],"sourcesContent":["// Parsing and dispatch only. Commands live in src/cli/, one file each,\n// lazy-loaded — nothing tree- or dependency-heavy may be imported at the top of this file (`render` must never load the OCR engine, `--version` must load almost nothing).\nimport { readFileSync } from 'node:fs';\nimport path from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { COMMANDS } from './cli/index.ts';\nimport type { Ctx } from './cli/types.ts';\n\n// __dirname (CJS) or its ESM equivalent, then two fixed hops up to the\n// package root. NOT `import.meta.resolve('pdf-to-md/package.json')` — Node's self-reference lookup throws ERR_MODULE_NOT_FOUND from a file this deep; a location-relative path is stable across both build targets.\nconst __dirname = path.dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\nfunction packageVersion(): string {\n try {\n const pkgPath = path.resolve(__dirname, '..', '..', 'package.json');\n const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { version?: string };\n return pkg.version ?? 'unknown';\n } catch {\n return 'unknown';\n }\n}\n\n// Exported so the comparison is unit-testable without spawning a subprocess\n// with a faked Node version. null means unparseable input, not \"not below\".\nexport function isBelowNodeFloor(version: string, floor: string): boolean | null {\n const parts = (v: string) => v.replace(/^v/, '').replace(/^>=/, '').split('.').map(Number);\n const v = parts(version);\n const f = parts(floor);\n if (v.length !== 3 || f.length !== 3 || v.some(Number.isNaN) || f.some(Number.isNaN)) return null;\n const [rMaj, rMin, rPat] = v;\n const [fMaj, fMin, fPat] = f;\n return rMaj !== fMaj ? rMaj < fMaj : rMin !== fMin ? rMin < fMin : rPat < fPat;\n}\n\n// Reads engines.node from package.json (kept as the single source of truth,\n// not hardcoded here) and returns it iff process.version is below it.\nfunction belowNodeFloor(version = process.version): string | null {\n try {\n const pkgPath = path.resolve(__dirname, '..', '..', 'package.json');\n const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { engines?: { node?: string } };\n const floor = pkg.engines?.node;\n if (!floor) return null;\n return isBelowNodeFloor(version, floor) ? floor.replace(/^>=/, '') : null;\n } catch {\n return null;\n }\n}\n\nfunction usage(name: string): string {\n return (\n `usage: ${name} <file.pdf> [out.md] [flags...] convert a PDF to markdown\\n` +\n ` ${name} extract <file.pdf> [out.md] [flags...] (same, explicit)\\n` +\n ` ${name} render <file.pdf> <page>[-<page>] [--dpi|-d N (default 288)] [--out|-o DIR (default .)]\\n` +\n ` ${name} audit <words.jsonl> <file.md> severity-1 word-loss check\\n` +\n ` ${name} --version | --help\\n` +\n '\\n' +\n 'extract flags:\\n' +\n ' --ocr force the OCR path (image-only/scanned pages)\\n' +\n ' --no-ocr never fall back to OCR; keep sparse text output\\n' +\n ' --pages N[-M] convert only these 1-based source pages\\n' +\n ' --stdout write markdown to stdout instead of a file\\n' +\n ' --json print the full QA report as JSON\\n' +\n ` --page-markers emit a \"### pN\" heading per page\\n` +\n ' --format md|txt|raw output format (default md; raw = Analysis JSON incl. the\\n' +\n ' report; txt = reading-order plain text)\\n' +\n ' --words-json=FILE OCR path from a recognized-word dump (offline replay)\\n' +\n ' --debug-words=FILE dump every recognized word as JSON lines to FILE\\n' +\n ' --dpi=N OCR render resolution (default 288; raise for tiny print or\\n' +\n ' chart/label pages, >~400 measured worse on ordinary prose)\\n' +\n '\\n' +\n `extract tuning (measured defaults; see README's tuning table):\\n` +\n ' --float-max-words=N --float-max-width=F --float-margin=F --para-gap=F\\n' +\n ' --heading-scale=F --col-max-width=F --col-split=F --line-y-tol=F\\n' +\n ' --line-height-ratio=F\\n' +\n '\\n' +\n 'examples:\\n' +\n ` ${name} book.pdf # text layer -> book.md (auto-OCR if scanned)\\n` +\n ` ${name} scan.pdf out.md --ocr --json # force OCR, print the QA report\\n` +\n ` ${name} book.pdf --pages 27-31 --stdout # re-extract a page range to stdout\\n` +\n ` ${name} render book.pdf 27 # render p27 to PNG for eyeballing (288 dpi)\\n` +\n ` ${name} audit words.jsonl book.md # check no recognized word was lost`\n );\n}\n\n// Thrown errors -> exit 1 with the message verbatim; usage errors exit 2.\nexport default async function cli(argv: string[], name: string): Promise<void> {\n const floor = belowNodeFloor();\n if (floor) console.error(`${name}: Node ${process.version} is below the required >=${floor} — the OCR path can silently find no words instead of erroring; upgrade Node.`);\n\n if (argv[0] === '--version' || argv[0] === '-v') {\n console.log(packageVersion());\n return;\n }\n if (argv.length === 0 || argv[0] === '--help' || argv[0] === '-h') {\n // No args is a usage ERROR (bare `pdf-to-md` can't do anything); asking\n // for --help is not.\n if (argv.length === 0) {\n console.error(usage(name));\n process.exit(2);\n }\n console.log(`${name} v${packageVersion()}`);\n console.log('');\n console.log(usage(name));\n return;\n }\n\n const ctx: Ctx = {\n name,\n rest: argv.slice(1),\n usageError(message) {\n console.error(message);\n process.exit(2);\n },\n };\n\n try {\n const load = COMMANDS[argv[0]];\n if (load) {\n await (await load()).default(ctx);\n } else {\n // Compat shim: `pdf-to-md file.pdf [...]` — any first word that isn't\n // a reserved command is extract's own first argument.\n ctx.rest = argv;\n await (await COMMANDS.extract()).default(ctx);\n }\n } catch (err) {\n console.error((err as Error).message);\n process.exit(1);\n }\n}\n"],"names":["readFileSync","path","fileURLToPath","COMMANDS","__dirname","dirname","__filename","url","packageVersion","pkg","pkgPath","resolve","JSON","parse","version","isBelowNodeFloor","floor","parts","v","replace","split","map","Number","f","length","some","isNaN","rMaj","rMin","rPat","fMaj","fMin","fPat","belowNodeFloor","process","engines","node","usage","name","cli","argv","console","error","log","exit","ctx","rest","slice","usageError","message","load","default","extract","err"],"mappings":"AAAA,uEAAuE;AACvE,2KAA2K;AAC3K,SAASA,YAAY,QAAQ,UAAU;AACvC,OAAOC,UAAU,YAAY;AAC7B,SAASC,aAAa,QAAQ,WAAW;AACzC,SAASC,QAAQ,QAAQ,iBAAiB;AAG1C,uEAAuE;AACvE,oNAAoN;AACpN,MAAMC,YAAYH,KAAKI,OAAO,CAAC,OAAOC,eAAe,cAAcA,aAAaJ,cAAc,YAAYK,GAAG;AAE7G,SAASC;IACP,IAAI;YAGKC;QAFP,MAAMC,UAAUT,KAAKU,OAAO,CAACP,WAAW,MAAM,MAAM;QACpD,MAAMK,MAAMG,KAAKC,KAAK,CAACb,aAAaU,SAAS;QAC7C,QAAOD,eAAAA,IAAIK,OAAO,cAAXL,0BAAAA,eAAe;IACxB,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,4EAA4E;AAC5E,4EAA4E;AAC5E,OAAO,SAASM,iBAAiBD,OAAe,EAAEE,KAAa;IAC7D,MAAMC,QAAQ,CAACC,IAAcA,EAAEC,OAAO,CAAC,MAAM,IAAIA,OAAO,CAAC,OAAO,IAAIC,KAAK,CAAC,KAAKC,GAAG,CAACC;IACnF,MAAMJ,IAAID,MAAMH;IAChB,MAAMS,IAAIN,MAAMD;IAChB,IAAIE,EAAEM,MAAM,KAAK,KAAKD,EAAEC,MAAM,KAAK,KAAKN,EAAEO,IAAI,CAACH,OAAOI,KAAK,KAAKH,EAAEE,IAAI,CAACH,OAAOI,KAAK,GAAG,OAAO;IAC7F,MAAM,CAACC,MAAMC,MAAMC,KAAK,GAAGX;IAC3B,MAAM,CAACY,MAAMC,MAAMC,KAAK,GAAGT;IAC3B,OAAOI,SAASG,OAAOH,OAAOG,OAAOF,SAASG,OAAOH,OAAOG,OAAOF,OAAOG;AAC5E;AAEA,4EAA4E;AAC5E,sEAAsE;AACtE,SAASC,eAAenB,UAAUoB,QAAQpB,OAAO;IAC/C,IAAI;YAGYL;QAFd,MAAMC,UAAUT,KAAKU,OAAO,CAACP,WAAW,MAAM,MAAM;QACpD,MAAMK,MAAMG,KAAKC,KAAK,CAACb,aAAaU,SAAS;QAC7C,MAAMM,SAAQP,eAAAA,IAAI0B,OAAO,cAAX1B,mCAAAA,aAAa2B,IAAI;QAC/B,IAAI,CAACpB,OAAO,OAAO;QACnB,OAAOD,iBAAiBD,SAASE,SAASA,MAAMG,OAAO,CAAC,OAAO,MAAM;IACvE,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASkB,MAAMC,IAAY;IACzB,OACE,CAAC,OAAO,EAAEA,KAAK,kEAAkE,CAAC,GAClF,CAAC,OAAO,EAAEA,KAAK,0DAA0D,CAAC,GAC1E,CAAC,OAAO,EAAEA,KAAK,0FAA0F,CAAC,GAC1G,CAAC,OAAO,EAAEA,KAAK,oEAAoE,CAAC,GACpF,CAAC,OAAO,EAAEA,KAAK,qBAAqB,CAAC,GACrC,OACA,qBACA,2EACA,6EACA,qEACA,wEACA,8DACA,CAAC,yDAAyD,CAAC,GAC3D,sFACA,qEACA,mFACA,8EACA,yFACA,wFACA,OACA,CAAC,gEAAgE,CAAC,GAClE,8EACA,yEACA,8BACA,OACA,gBACA,CAAC,EAAE,EAAEA,KAAK,gFAAgF,CAAC,GAC3F,CAAC,EAAE,EAAEA,KAAK,mEAAmE,CAAC,GAC9E,CAAC,EAAE,EAAEA,KAAK,sEAAsE,CAAC,GACjF,CAAC,EAAE,EAAEA,KAAK,+EAA+E,CAAC,GAC1F,CAAC,EAAE,EAAEA,KAAK,oEAAoE,CAAC;AAEnF;AAEA,0EAA0E;AAC1E,eAAe,eAAeC,IAAIC,IAAc,EAAEF,IAAY;IAC5D,MAAMtB,QAAQiB;IACd,IAAIjB,OAAOyB,QAAQC,KAAK,CAAC,GAAGJ,KAAK,OAAO,EAAEJ,QAAQpB,OAAO,CAAC,yBAAyB,EAAEE,MAAM,6EAA6E,CAAC;IAEzK,IAAIwB,IAAI,CAAC,EAAE,KAAK,eAAeA,IAAI,CAAC,EAAE,KAAK,MAAM;QAC/CC,QAAQE,GAAG,CAACnC;QACZ;IACF;IACA,IAAIgC,KAAKhB,MAAM,KAAK,KAAKgB,IAAI,CAAC,EAAE,KAAK,YAAYA,IAAI,CAAC,EAAE,KAAK,MAAM;QACjE,wEAAwE;QACxE,qBAAqB;QACrB,IAAIA,KAAKhB,MAAM,KAAK,GAAG;YACrBiB,QAAQC,KAAK,CAACL,MAAMC;YACpBJ,QAAQU,IAAI,CAAC;QACf;QACAH,QAAQE,GAAG,CAAC,GAAGL,KAAK,EAAE,EAAE9B,kBAAkB;QAC1CiC,QAAQE,GAAG,CAAC;QACZF,QAAQE,GAAG,CAACN,MAAMC;QAClB;IACF;IAEA,MAAMO,MAAW;QACfP;QACAQ,MAAMN,KAAKO,KAAK,CAAC;QACjBC,YAAWC,OAAO;YAChBR,QAAQC,KAAK,CAACO;YACdf,QAAQU,IAAI,CAAC;QACf;IACF;IAEA,IAAI;QACF,MAAMM,OAAO/C,QAAQ,CAACqC,IAAI,CAAC,EAAE,CAAC;QAC9B,IAAIU,MAAM;YACR,MAAM,AAAC,CAAA,MAAMA,MAAK,EAAGC,OAAO,CAACN;QAC/B,OAAO;YACL,sEAAsE;YACtE,sDAAsD;YACtDA,IAAIC,IAAI,GAAGN;YACX,MAAM,AAAC,CAAA,MAAMrC,SAASiD,OAAO,EAAC,EAAGD,OAAO,CAACN;QAC3C;IACF,EAAE,OAAOQ,KAAK;QACZZ,QAAQC,KAAK,CAAC,AAACW,IAAcJ,OAAO;QACpCf,QAAQU,IAAI,CAAC;IACf;AACF"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Pass 1: load the PDF and collect positioned glyphs per page, plus the
|
|
2
|
+
// document-wide char-weighted font-height histogram. Heights round to 0.1pt for stable histogram keys; y is flipped so sorting ascending reads top-down.
|
|
3
|
+
import { OPS } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
4
|
+
import { openDocument } from './pdf-open.js';
|
|
5
|
+
// HYBRID-PAGE DETECTION (README's "Design contract"): a page can carry a
|
|
6
|
+
// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.
|
|
7
|
+
//
|
|
8
|
+
// Signal: scan the operator list for `OPS.paintImageXObject`, whose args are
|
|
9
|
+
// `[objId, width, height]` — available synchronously, `large` is >=200x200px. Masks/nested Form XObjects aren't walked (accepted gap); missing dims fall back to "op present" as the signal.
|
|
10
|
+
const LARGE_IMAGE_MIN_PX = 200;
|
|
11
|
+
async function hasLargeImage(page) {
|
|
12
|
+
const { fnArray, argsArray } = await page.getOperatorList();
|
|
13
|
+
for(let i = 0; i < fnArray.length; i++){
|
|
14
|
+
if (fnArray[i] !== OPS.paintImageXObject) continue;
|
|
15
|
+
const [, w, h] = argsArray[i];
|
|
16
|
+
if (typeof w === 'number' && typeof h === 'number') {
|
|
17
|
+
if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) return true;
|
|
18
|
+
} else return true; // dims unavailable: coarser "op present" signal
|
|
19
|
+
}
|
|
20
|
+
return false;
|
|
21
|
+
}
|
|
22
|
+
// { first, last }: 1-based, inclusive page range (the CLI's --pages).
|
|
23
|
+
// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.
|
|
24
|
+
//
|
|
25
|
+
// CAVEAT: `heightChars` is built ONLY from the pages actually collected, so
|
|
26
|
+
// a `--pages` subset's heading levels can differ from a full run's — accepted since re-extracted pages get spliced back in for their prose, not diffed.
|
|
27
|
+
// Cheap page-count-only load (no per-page text extraction) — the OCR path
|
|
28
|
+
// needs it to resolve its own [first, last] clamping and print a "### pN" marker for every page in range, including ones with no recognized words.
|
|
29
|
+
export async function pageCount(src) {
|
|
30
|
+
const doc = await openDocument(src);
|
|
31
|
+
return doc.numPages;
|
|
32
|
+
}
|
|
33
|
+
export async function collect(src, { first, last } = {}) {
|
|
34
|
+
const doc = await openDocument(src);
|
|
35
|
+
const from = Math.min(Math.max(1, first !== null && first !== void 0 ? first : 1), Math.max(1, doc.numPages));
|
|
36
|
+
const to = Math.min(Math.max(from, last !== null && last !== void 0 ? last : doc.numPages), doc.numPages);
|
|
37
|
+
const pages = [];
|
|
38
|
+
const heightChars = new Map();
|
|
39
|
+
for(let p = from; p <= to; p++){
|
|
40
|
+
const page = await doc.getPage(p);
|
|
41
|
+
const { items } = await page.getTextContent();
|
|
42
|
+
const vp = page.getViewport({
|
|
43
|
+
scale: 1
|
|
44
|
+
});
|
|
45
|
+
const glyphs = [];
|
|
46
|
+
let textChars = 0;
|
|
47
|
+
for (const it of items){
|
|
48
|
+
if (!('str' in it) || typeof it.str !== 'string') continue;
|
|
49
|
+
textChars += it.str.length;
|
|
50
|
+
if (!it.str.trim()) continue;
|
|
51
|
+
const h = Math.round(Math.abs(it.transform[3]) * 10) / 10;
|
|
52
|
+
glyphs.push({
|
|
53
|
+
s: it.str,
|
|
54
|
+
x: it.transform[4],
|
|
55
|
+
y: vp.height - it.transform[5],
|
|
56
|
+
w: it.width,
|
|
57
|
+
h
|
|
58
|
+
});
|
|
59
|
+
heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);
|
|
60
|
+
}
|
|
61
|
+
// "text layer used" for hybrid-page purposes (page text > 80 chars) — a
|
|
62
|
+
// page with only a couple of stray scraps isn't "using" the text layer even if it also carries a large image.
|
|
63
|
+
const largeImage = textChars > 80 && await hasLargeImage(page);
|
|
64
|
+
pages.push({
|
|
65
|
+
page: p,
|
|
66
|
+
glyphs,
|
|
67
|
+
width: vp.width,
|
|
68
|
+
height: vp.height,
|
|
69
|
+
textChars,
|
|
70
|
+
largeImage
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
// `numPages` is the SELECTED count, not doc.numPages: the auto-OCR
|
|
74
|
+
// chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely "sparse".
|
|
75
|
+
return {
|
|
76
|
+
numPages: pages.length,
|
|
77
|
+
pages,
|
|
78
|
+
heightChars
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/collect.ts"],"sourcesContent":["// Pass 1: load the PDF and collect positioned glyphs per page, plus the\n// document-wide char-weighted font-height histogram. Heights round to 0.1pt for stable histogram keys; y is flipped so sorting ascending reads top-down.\nimport type { PDFPageProxy } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { OPS } from 'pdfjs-dist/legacy/build/pdf.mjs';\nimport { openDocument } from './pdf-open.ts';\nimport type { CollectedPage, CollectResult, Glyph } from './types.ts';\n\n// HYBRID-PAGE DETECTION (README's \"Design contract\"): a page can carry a\n// real text layer AND a separately-painted large image with its OWN baked-in text that `page.getTextContent()` never sees.\n//\n// Signal: scan the operator list for `OPS.paintImageXObject`, whose args are\n// `[objId, width, height]` — available synchronously, `large` is >=200x200px. Masks/nested Form XObjects aren't walked (accepted gap); missing dims fall back to \"op present\" as the signal.\nconst LARGE_IMAGE_MIN_PX = 200;\nasync function hasLargeImage(page: PDFPageProxy): Promise<boolean> {\n const { fnArray, argsArray } = await page.getOperatorList();\n for (let i = 0; i < fnArray.length; i++) {\n if (fnArray[i] !== OPS.paintImageXObject) continue;\n const [, w, h] = argsArray[i];\n if (typeof w === 'number' && typeof h === 'number') {\n if (w >= LARGE_IMAGE_MIN_PX && h >= LARGE_IMAGE_MIN_PX) return true;\n } else return true; // dims unavailable: coarser \"op present\" signal\n }\n return false;\n}\n\n// { first, last }: 1-based, inclusive page range (the CLI's --pages).\n// Out-of-range values CLAMP to the document's bounds; `page` on each returned entry is always the SOURCE page number, so markers/splicing stay correct.\n//\n// CAVEAT: `heightChars` is built ONLY from the pages actually collected, so\n// a `--pages` subset's heading levels can differ from a full run's — accepted since re-extracted pages get spliced back in for their prose, not diffed.\n\n// Cheap page-count-only load (no per-page text extraction) — the OCR path\n// needs it to resolve its own [first, last] clamping and print a \"### pN\" marker for every page in range, including ones with no recognized words.\nexport async function pageCount(src: string): Promise<number> {\n const doc = await openDocument(src);\n return doc.numPages;\n}\n\nexport async function collect(src: string, { first, last }: { first?: number; last?: number } = {}): Promise<CollectResult> {\n const doc = await openDocument(src);\n const from = Math.min(Math.max(1, first ?? 1), Math.max(1, doc.numPages));\n const to = Math.min(Math.max(from, last ?? doc.numPages), doc.numPages);\n const pages: CollectedPage[] = [];\n const heightChars = new Map<number, number>();\n for (let p = from; p <= to; p++) {\n const page = await doc.getPage(p);\n const { items } = await page.getTextContent();\n const vp = page.getViewport({ scale: 1 });\n const glyphs: Glyph[] = [];\n let textChars = 0;\n for (const it of items) {\n if (!('str' in it) || typeof it.str !== 'string') continue;\n textChars += it.str.length;\n if (!it.str.trim()) continue;\n const h = Math.round(Math.abs(it.transform[3]) * 10) / 10;\n glyphs.push({\n s: it.str,\n x: it.transform[4],\n y: vp.height - it.transform[5],\n w: it.width,\n h,\n });\n heightChars.set(h, (heightChars.get(h) || 0) + it.str.length);\n }\n // \"text layer used\" for hybrid-page purposes (page text > 80 chars) — a\n // page with only a couple of stray scraps isn't \"using\" the text layer even if it also carries a large image.\n const largeImage = textChars > 80 && (await hasLargeImage(page));\n pages.push({\n page: p,\n glyphs,\n width: vp.width,\n height: vp.height,\n textChars,\n largeImage,\n });\n }\n // `numPages` is the SELECTED count, not doc.numPages: the auto-OCR\n // chars/page floor must judge a `--pages` subset against its own page count, or a genuinely image-only page reads as merely \"sparse\".\n return { numPages: pages.length, pages, heightChars };\n}\n"],"names":["OPS","openDocument","LARGE_IMAGE_MIN_PX","hasLargeImage","page","fnArray","argsArray","getOperatorList","i","length","paintImageXObject","w","h","pageCount","src","doc","numPages","collect","first","last","from","Math","min","max","to","pages","heightChars","Map","p","getPage","items","getTextContent","vp","getViewport","scale","glyphs","textChars","it","str","trim","round","abs","transform","push","s","x","y","height","width","set","get","largeImage"],"mappings":"AAAA,wEAAwE;AACxE,yJAAyJ;AAEzJ,SAASA,GAAG,QAAQ,kCAAkC;AACtD,SAASC,YAAY,QAAQ,gBAAgB;AAG7C,yEAAyE;AACzE,2HAA2H;AAC3H,EAAE;AACF,6EAA6E;AAC7E,6LAA6L;AAC7L,MAAMC,qBAAqB;AAC3B,eAAeC,cAAcC,IAAkB;IAC7C,MAAM,EAAEC,OAAO,EAAEC,SAAS,EAAE,GAAG,MAAMF,KAAKG,eAAe;IACzD,IAAK,IAAIC,IAAI,GAAGA,IAAIH,QAAQI,MAAM,EAAED,IAAK;QACvC,IAAIH,OAAO,CAACG,EAAE,KAAKR,IAAIU,iBAAiB,EAAE;QAC1C,MAAM,GAAGC,GAAGC,EAAE,GAAGN,SAAS,CAACE,EAAE;QAC7B,IAAI,OAAOG,MAAM,YAAY,OAAOC,MAAM,UAAU;YAClD,IAAID,KAAKT,sBAAsBU,KAAKV,oBAAoB,OAAO;QACjE,OAAO,OAAO,MAAM,gDAAgD;IACtE;IACA,OAAO;AACT;AAEA,sEAAsE;AACtE,wJAAwJ;AACxJ,EAAE;AACF,4EAA4E;AAC5E,wJAAwJ;AAExJ,0EAA0E;AAC1E,mJAAmJ;AACnJ,OAAO,eAAeW,UAAUC,GAAW;IACzC,MAAMC,MAAM,MAAMd,aAAaa;IAC/B,OAAOC,IAAIC,QAAQ;AACrB;AAEA,OAAO,eAAeC,QAAQH,GAAW,EAAE,EAAEI,KAAK,EAAEC,IAAI,EAAqC,GAAG,CAAC,CAAC;IAChG,MAAMJ,MAAM,MAAMd,aAAaa;IAC/B,MAAMM,OAAOC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,GAAGL,kBAAAA,mBAAAA,QAAS,IAAIG,KAAKE,GAAG,CAAC,GAAGR,IAAIC,QAAQ;IACvE,MAAMQ,KAAKH,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAACH,MAAMD,iBAAAA,kBAAAA,OAAQJ,IAAIC,QAAQ,GAAGD,IAAIC,QAAQ;IACtE,MAAMS,QAAyB,EAAE;IACjC,MAAMC,cAAc,IAAIC;IACxB,IAAK,IAAIC,IAAIR,MAAMQ,KAAKJ,IAAII,IAAK;QAC/B,MAAMxB,OAAO,MAAMW,IAAIc,OAAO,CAACD;QAC/B,MAAM,EAAEE,KAAK,EAAE,GAAG,MAAM1B,KAAK2B,cAAc;QAC3C,MAAMC,KAAK5B,KAAK6B,WAAW,CAAC;YAAEC,OAAO;QAAE;QACvC,MAAMC,SAAkB,EAAE;QAC1B,IAAIC,YAAY;QAChB,KAAK,MAAMC,MAAMP,MAAO;YACtB,IAAI,CAAE,CAAA,SAASO,EAAC,KAAM,OAAOA,GAAGC,GAAG,KAAK,UAAU;YAClDF,aAAaC,GAAGC,GAAG,CAAC7B,MAAM;YAC1B,IAAI,CAAC4B,GAAGC,GAAG,CAACC,IAAI,IAAI;YACpB,MAAM3B,IAAIS,KAAKmB,KAAK,CAACnB,KAAKoB,GAAG,CAACJ,GAAGK,SAAS,CAAC,EAAE,IAAI,MAAM;YACvDP,OAAOQ,IAAI,CAAC;gBACVC,GAAGP,GAAGC,GAAG;gBACTO,GAAGR,GAAGK,SAAS,CAAC,EAAE;gBAClBI,GAAGd,GAAGe,MAAM,GAAGV,GAAGK,SAAS,CAAC,EAAE;gBAC9B/B,GAAG0B,GAAGW,KAAK;gBACXpC;YACF;YACAc,YAAYuB,GAAG,CAACrC,GAAG,AAACc,CAAAA,YAAYwB,GAAG,CAACtC,MAAM,CAAA,IAAKyB,GAAGC,GAAG,CAAC7B,MAAM;QAC9D;QACA,wEAAwE;QACxE,8GAA8G;QAC9G,MAAM0C,aAAaf,YAAY,MAAO,MAAMjC,cAAcC;QAC1DqB,MAAMkB,IAAI,CAAC;YACTvC,MAAMwB;YACNO;YACAa,OAAOhB,GAAGgB,KAAK;YACfD,QAAQf,GAAGe,MAAM;YACjBX;YACAe;QACF;IACF;IACA,mEAAmE;IACnE,sIAAsI;IACtI,OAAO;QAAEnC,UAAUS,MAAMhB,MAAM;QAAEgB;QAAOC;IAAY;AACtD"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const DICT_PATH: string;
|
|
2
|
+
/** The hyphen-rejoin dictionary; an empty set if none was found. */
|
|
3
|
+
export declare function getDict(): Set<string>;
|
|
4
|
+
/** False when no dictionary file exists at DICT_PATH — every line-final
|
|
5
|
+
* hyphen then stays unresolved. */
|
|
6
|
+
export declare function dictAvailable(): boolean;
|
|
7
|
+
/** The notice callers surface via warn() when dictAvailable() is false. */
|
|
8
|
+
export declare function dictUnavailableWarning(): string;
|
package/dist/esm/dict.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// The hyphen-rejoin dictionary, shared by src/emit.ts and src/markdown.ts.
|
|
2
|
+
// Loaded once, lazily, on first use — never at import — so importing the library does no file IO on its own.
|
|
3
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
4
|
+
import wordListPath from 'word-list';
|
|
5
|
+
// Bundled list (not the OS's /usr/share/dict/words) → identical rejoin decisions on every
|
|
6
|
+
// OS/machine. PDF_TO_MD_DICT_PATH overrides it — the test seam for simulating no dictionary.
|
|
7
|
+
export const DICT_PATH = process.env.PDF_TO_MD_DICT_PATH || wordListPath;
|
|
8
|
+
let cache = null;
|
|
9
|
+
function load() {
|
|
10
|
+
if (!cache) {
|
|
11
|
+
const available = existsSync(DICT_PATH);
|
|
12
|
+
cache = {
|
|
13
|
+
words: available ? new Set(readFileSync(DICT_PATH, 'utf8').toLowerCase().split('\n')) : new Set(),
|
|
14
|
+
available
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
return cache;
|
|
18
|
+
}
|
|
19
|
+
/** The hyphen-rejoin dictionary; an empty set if none was found. */ export function getDict() {
|
|
20
|
+
return load().words;
|
|
21
|
+
}
|
|
22
|
+
/** False when no dictionary file exists at DICT_PATH — every line-final
|
|
23
|
+
* hyphen then stays unresolved. */ export function dictAvailable() {
|
|
24
|
+
return load().available;
|
|
25
|
+
}
|
|
26
|
+
/** The notice callers surface via warn() when dictAvailable() is false. */ export function dictUnavailableWarning() {
|
|
27
|
+
return `no dictionary at ${DICT_PATH} — line-final hyphens kept unresolved`;
|
|
28
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/dict.ts"],"sourcesContent":["// The hyphen-rejoin dictionary, shared by src/emit.ts and src/markdown.ts.\n// Loaded once, lazily, on first use — never at import — so importing the library does no file IO on its own.\nimport { existsSync, readFileSync } from 'node:fs';\nimport wordListPath from 'word-list';\n\n// Bundled list (not the OS's /usr/share/dict/words) → identical rejoin decisions on every\n// OS/machine. PDF_TO_MD_DICT_PATH overrides it — the test seam for simulating no dictionary.\nexport const DICT_PATH = process.env.PDF_TO_MD_DICT_PATH || wordListPath;\n\nlet cache: { words: Set<string>; available: boolean } | null = null;\n\nfunction load(): { words: Set<string>; available: boolean } {\n if (!cache) {\n const available = existsSync(DICT_PATH);\n cache = { words: available ? new Set(readFileSync(DICT_PATH, 'utf8').toLowerCase().split('\\n')) : new Set(), available };\n }\n return cache;\n}\n\n/** The hyphen-rejoin dictionary; an empty set if none was found. */\nexport function getDict(): Set<string> {\n return load().words;\n}\n\n/** False when no dictionary file exists at DICT_PATH — every line-final\n * hyphen then stays unresolved. */\nexport function dictAvailable(): boolean {\n return load().available;\n}\n\n/** The notice callers surface via warn() when dictAvailable() is false. */\nexport function dictUnavailableWarning(): string {\n return `no dictionary at ${DICT_PATH} — line-final hyphens kept unresolved`;\n}\n"],"names":["existsSync","readFileSync","wordListPath","DICT_PATH","process","env","PDF_TO_MD_DICT_PATH","cache","load","available","words","Set","toLowerCase","split","getDict","dictAvailable","dictUnavailableWarning"],"mappings":"AAAA,2EAA2E;AAC3E,6GAA6G;AAC7G,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,kBAAkB,YAAY;AAErC,0FAA0F;AAC1F,6FAA6F;AAC7F,OAAO,MAAMC,YAAYC,QAAQC,GAAG,CAACC,mBAAmB,IAAIJ,aAAa;AAEzE,IAAIK,QAA2D;AAE/D,SAASC;IACP,IAAI,CAACD,OAAO;QACV,MAAME,YAAYT,WAAWG;QAC7BI,QAAQ;YAAEG,OAAOD,YAAY,IAAIE,IAAIV,aAAaE,WAAW,QAAQS,WAAW,GAAGC,KAAK,CAAC,SAAS,IAAIF;YAAOF;QAAU;IACzH;IACA,OAAOF;AACT;AAEA,kEAAkE,GAClE,OAAO,SAASO;IACd,OAAON,OAAOE,KAAK;AACrB;AAEA;iCACiC,GACjC,OAAO,SAASK;IACd,OAAOP,OAAOC,SAAS;AACzB;AAEA,yEAAyE,GACzE,OAAO,SAASO;IACd,OAAO,CAAC,iBAAiB,EAAEb,UAAU,qCAAqC,CAAC;AAC7E"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { PageOCR, Tuning, Word } from './geometry.js';
|
|
2
|
+
/** Share of word-like tokens (3+ letter run) below which a block counts as
|
|
3
|
+
* unstructured. Measured (`review-blocks.test.ts`): junk 0.000-0.100 vs prose 0.743-0.857, cut 0.4. */
|
|
4
|
+
export declare const UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;
|
|
5
|
+
export type EmitBlock = {
|
|
6
|
+
kind: 'heading';
|
|
7
|
+
text: string;
|
|
8
|
+
level: number;
|
|
9
|
+
words: Word[];
|
|
10
|
+
} | {
|
|
11
|
+
kind: 'paragraph';
|
|
12
|
+
text: string;
|
|
13
|
+
words: Word[];
|
|
14
|
+
} | {
|
|
15
|
+
kind: 'quote';
|
|
16
|
+
text: string;
|
|
17
|
+
words: Word[];
|
|
18
|
+
} | {
|
|
19
|
+
kind: 'review';
|
|
20
|
+
text: string;
|
|
21
|
+
/** The parseable reason inside `> [review: ...]`, carried on the block
|
|
22
|
+
* so more reasons can be added without changing the render format. */
|
|
23
|
+
reason: string;
|
|
24
|
+
words: Word[];
|
|
25
|
+
};
|
|
26
|
+
export declare function renderEmitBlock(b: EmitBlock): string;
|
|
27
|
+
export declare function emitPage(p: PageOCR, levelOf: Map<number, number>, tuning: Tuning): EmitBlock[];
|
package/dist/esm/emit.js
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// emit.ts — OCR-path pass 3: lines -> markdown blocks.
|
|
2
|
+
import { getDict } from './dict.js';
|
|
3
|
+
import { bucket } from './geometry.js';
|
|
4
|
+
// Join a wrapped line onto the open paragraph, closing a line-final hyphen
|
|
5
|
+
// only when the dictionary confirms the joined form is a word.
|
|
6
|
+
function hyphenJoin(acc, next) {
|
|
7
|
+
if (acc.endsWith('-')) {
|
|
8
|
+
const withoutDash = acc.slice(0, -1);
|
|
9
|
+
const pen = withoutDash.at(-1);
|
|
10
|
+
if (pen && /[A-Za-z0-9]/.test(pen)) {
|
|
11
|
+
var _withoutDash_split_pop, _next_split_, _filtered_split_;
|
|
12
|
+
const head = (_withoutDash_split_pop = withoutDash.split(' ').pop()) !== null && _withoutDash_split_pop !== void 0 ? _withoutDash_split_pop : '';
|
|
13
|
+
const firstTok = (_next_split_ = next.split(' ')[0]) !== null && _next_split_ !== void 0 ? _next_split_ : '';
|
|
14
|
+
const filtered = [
|
|
15
|
+
...firstTok
|
|
16
|
+
].filter((c)=>/[A-Za-z-]/.test(c)).join('');
|
|
17
|
+
const tail = (_filtered_split_ = filtered.split('-')[0]) !== null && _filtered_split_ !== void 0 ? _filtered_split_ : '';
|
|
18
|
+
const close = getDict().has((head + tail).toLowerCase());
|
|
19
|
+
return (close ? withoutDash : acc) + next;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return `${acc} ${next}`;
|
|
23
|
+
}
|
|
24
|
+
// Blocks with too few word-like tokens to trust as prose (see
|
|
25
|
+
// UNSTRUCTURED_WORDLIKE_FRACTION below) are labeled `> [review: reason] text` instead of guessed at.
|
|
26
|
+
const REVIEW_REASON = {
|
|
27
|
+
unstructuredLabels: 'unstructured labels'
|
|
28
|
+
};
|
|
29
|
+
/** Share of word-like tokens (3+ letter run) below which a block counts as
|
|
30
|
+
* unstructured. Measured (`review-blocks.test.ts`): junk 0.000-0.100 vs prose 0.743-0.857, cut 0.4. */ export const UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;
|
|
31
|
+
function isUnstructured(text) {
|
|
32
|
+
const tokens = text.split(/\s+/).filter(Boolean);
|
|
33
|
+
if (tokens.length === 0) return false; // never mark an empty block
|
|
34
|
+
const wordLike = tokens.filter((t)=>/[A-Za-z]{3}/.test(t)).length;
|
|
35
|
+
return wordLike / tokens.length < UNSTRUCTURED_WORDLIKE_FRACTION;
|
|
36
|
+
}
|
|
37
|
+
function reviewLine(reason, text) {
|
|
38
|
+
return `> [review: ${reason}] ${text}`;
|
|
39
|
+
}
|
|
40
|
+
// Renders one EmitBlock to the exact markdown emitPage used to push
|
|
41
|
+
// directly — shared so extract.ts and analyze.ts produce byte-identical text.
|
|
42
|
+
export function renderEmitBlock(b) {
|
|
43
|
+
switch(b.kind){
|
|
44
|
+
case 'heading':
|
|
45
|
+
return `${'#'.repeat(b.level)} ${b.text}`;
|
|
46
|
+
case 'quote':
|
|
47
|
+
return `> ${b.text}`;
|
|
48
|
+
case 'review':
|
|
49
|
+
return reviewLine(b.reason, b.text);
|
|
50
|
+
default:
|
|
51
|
+
return b.text;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export function emitPage(p, levelOf, tuning) {
|
|
55
|
+
const blocks = [];
|
|
56
|
+
let para = [];
|
|
57
|
+
const flush = ()=>{
|
|
58
|
+
if (!para.length) return;
|
|
59
|
+
const texts = para.map((l)=>l.text);
|
|
60
|
+
const merged = texts.slice(1).reduce(hyphenJoin, texts[0]);
|
|
61
|
+
const words = para.flatMap((l)=>l.words);
|
|
62
|
+
// Same review-marker rule as the heading path below: no word may be
|
|
63
|
+
// dropped or reordered here, only how the block renders changes.
|
|
64
|
+
blocks.push(isUnstructured(merged) ? {
|
|
65
|
+
kind: 'review',
|
|
66
|
+
text: merged,
|
|
67
|
+
reason: REVIEW_REASON.unstructuredLabels,
|
|
68
|
+
words
|
|
69
|
+
} : {
|
|
70
|
+
kind: 'paragraph',
|
|
71
|
+
text: merged,
|
|
72
|
+
words
|
|
73
|
+
});
|
|
74
|
+
para = [];
|
|
75
|
+
};
|
|
76
|
+
let i = 0;
|
|
77
|
+
while(i < p.lines.length){
|
|
78
|
+
const line = p.lines[i];
|
|
79
|
+
const prev = i > 0 ? p.lines[i - 1] : null;
|
|
80
|
+
if (prev && (line.col !== prev.col || prev.midY - line.midY > p.pitch * tuning.paraGap)) {
|
|
81
|
+
// Join across a column break (instead of flushing) only when the column
|
|
82
|
+
// changed, the paragraph dangles (no terminal punct), and the new text starts lowercase; a miss stays visible via pageStats.danglingLong.
|
|
83
|
+
const danglingEnd = para.length > 0 && !/[.!?:;"'”’)\]]$/.test(para[para.length - 1].text);
|
|
84
|
+
const continuationStart = /^[a-z]/.test(line.text);
|
|
85
|
+
const joinAcrossColumns = line.col !== prev.col && danglingEnd && continuationStart && !levelOf.has(bucket(line.h));
|
|
86
|
+
if (!joinAcrossColumns) flush();
|
|
87
|
+
}
|
|
88
|
+
if (levelOf.has(bucket(line.h))) {
|
|
89
|
+
// Gather the run of consecutive heading-size lines: a wrapped title is
|
|
90
|
+
// one heading, a long run is a pull quote in a bigger face.
|
|
91
|
+
flush();
|
|
92
|
+
const run = [
|
|
93
|
+
line
|
|
94
|
+
];
|
|
95
|
+
let maxB = bucket(line.h);
|
|
96
|
+
while(i + 1 < p.lines.length && levelOf.has(bucket(p.lines[i + 1].h)) && p.lines[i + 1].col === p.lines[i].col && p.lines[i].midY - p.lines[i + 1].midY < Math.max(p.lines[i].h, p.lines[i + 1].h) * 2.2){
|
|
97
|
+
i++;
|
|
98
|
+
run.push(p.lines[i]);
|
|
99
|
+
maxB = Math.max(maxB, bucket(p.lines[i].h));
|
|
100
|
+
}
|
|
101
|
+
const runTexts = run.map((l)=>l.text);
|
|
102
|
+
const merged = runTexts.slice(1).reduce(hyphenJoin, runTexts[0]);
|
|
103
|
+
const words = run.flatMap((l)=>l.words);
|
|
104
|
+
// Below UNSTRUCTURED_WORDLIKE_FRACTION = chart labels/stray numerals,
|
|
105
|
+
// not a heading; checked FIRST so a long numeral run isn't asserted as a pull quote.
|
|
106
|
+
const junk = isUnstructured(merged);
|
|
107
|
+
if (junk) blocks.push({
|
|
108
|
+
kind: 'review',
|
|
109
|
+
text: merged,
|
|
110
|
+
reason: REVIEW_REASON.unstructuredLabels,
|
|
111
|
+
words
|
|
112
|
+
});
|
|
113
|
+
else if (merged.length > 150) blocks.push({
|
|
114
|
+
kind: 'quote',
|
|
115
|
+
text: merged,
|
|
116
|
+
words
|
|
117
|
+
});
|
|
118
|
+
else blocks.push({
|
|
119
|
+
kind: 'heading',
|
|
120
|
+
text: merged,
|
|
121
|
+
level: levelOf.get(maxB),
|
|
122
|
+
words
|
|
123
|
+
});
|
|
124
|
+
} else {
|
|
125
|
+
para.push(line);
|
|
126
|
+
}
|
|
127
|
+
i++;
|
|
128
|
+
}
|
|
129
|
+
flush();
|
|
130
|
+
return blocks;
|
|
131
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Projects/pdf-to-md/src/emit.ts"],"sourcesContent":["// emit.ts — OCR-path pass 3: lines -> markdown blocks.\nimport { getDict } from './dict.ts';\nimport type { OcrLine, PageOCR, Tuning, Word } from './geometry.ts';\nimport { bucket } from './geometry.ts';\n\n// Join a wrapped line onto the open paragraph, closing a line-final hyphen\n// only when the dictionary confirms the joined form is a word.\nfunction hyphenJoin(acc: string, next: string): string {\n if (acc.endsWith('-')) {\n const withoutDash = acc.slice(0, -1);\n const pen = withoutDash.at(-1);\n if (pen && /[A-Za-z0-9]/.test(pen)) {\n const head = withoutDash.split(' ').pop() ?? '';\n const firstTok = next.split(' ')[0] ?? '';\n const filtered = [...firstTok].filter((c) => /[A-Za-z-]/.test(c)).join('');\n const tail = filtered.split('-')[0] ?? '';\n const close = getDict().has((head + tail).toLowerCase());\n return (close ? withoutDash : acc) + next;\n }\n }\n return `${acc} ${next}`;\n}\n\n// Blocks with too few word-like tokens to trust as prose (see\n// UNSTRUCTURED_WORDLIKE_FRACTION below) are labeled `> [review: reason] text` instead of guessed at.\nconst REVIEW_REASON = {\n unstructuredLabels: 'unstructured labels',\n} as const;\n\n/** Share of word-like tokens (3+ letter run) below which a block counts as\n * unstructured. Measured (`review-blocks.test.ts`): junk 0.000-0.100 vs prose 0.743-0.857, cut 0.4. */\nexport const UNSTRUCTURED_WORDLIKE_FRACTION = 0.4;\nfunction isUnstructured(text: string): boolean {\n const tokens = text.split(/\\s+/).filter(Boolean);\n if (tokens.length === 0) return false; // never mark an empty block\n const wordLike = tokens.filter((t) => /[A-Za-z]{3}/.test(t)).length;\n return wordLike / tokens.length < UNSTRUCTURED_WORDLIKE_FRACTION;\n}\n\nfunction reviewLine(reason: string, text: string): string {\n return `> [review: ${reason}] ${text}`;\n}\n\n// One classified block, carrying constituent words for analyze()'s\n// block->word linkage; discriminated on `kind` so `reason`/`level` are compile-time guaranteed present on their variants.\nexport type EmitBlock =\n | { kind: 'heading'; text: string; level: number; words: Word[] }\n | { kind: 'paragraph'; text: string; words: Word[] }\n | { kind: 'quote'; text: string; words: Word[] }\n | {\n kind: 'review';\n text: string;\n /** The parseable reason inside `> [review: ...]`, carried on the block\n * so more reasons can be added without changing the render format. */\n reason: string;\n words: Word[];\n };\n\n// Renders one EmitBlock to the exact markdown emitPage used to push\n// directly — shared so extract.ts and analyze.ts produce byte-identical text.\nexport function renderEmitBlock(b: EmitBlock): string {\n switch (b.kind) {\n case 'heading':\n return `${'#'.repeat(b.level)} ${b.text}`;\n case 'quote':\n return `> ${b.text}`;\n case 'review':\n return reviewLine(b.reason, b.text);\n default:\n return b.text;\n }\n}\n\nexport function emitPage(p: PageOCR, levelOf: Map<number, number>, tuning: Tuning): EmitBlock[] {\n const blocks: EmitBlock[] = [];\n let para: OcrLine[] = [];\n const flush = () => {\n if (!para.length) return;\n const texts = para.map((l) => l.text);\n const merged = texts.slice(1).reduce(hyphenJoin, texts[0]);\n const words = para.flatMap((l) => l.words);\n // Same review-marker rule as the heading path below: no word may be\n // dropped or reordered here, only how the block renders changes.\n blocks.push(isUnstructured(merged) ? { kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words } : { kind: 'paragraph', text: merged, words });\n para = [];\n };\n\n let i = 0;\n while (i < p.lines.length) {\n const line: OcrLine = p.lines[i];\n const prev = i > 0 ? p.lines[i - 1] : null;\n if (prev && (line.col !== prev.col || prev.midY - line.midY > p.pitch * tuning.paraGap)) {\n // Join across a column break (instead of flushing) only when the column\n // changed, the paragraph dangles (no terminal punct), and the new text starts lowercase; a miss stays visible via pageStats.danglingLong.\n const danglingEnd = para.length > 0 && !/[.!?:;\"'”’)\\]]$/.test(para[para.length - 1].text);\n const continuationStart = /^[a-z]/.test(line.text);\n const joinAcrossColumns = line.col !== prev.col && danglingEnd && continuationStart && !levelOf.has(bucket(line.h));\n if (!joinAcrossColumns) flush();\n }\n\n if (levelOf.has(bucket(line.h))) {\n // Gather the run of consecutive heading-size lines: a wrapped title is\n // one heading, a long run is a pull quote in a bigger face.\n flush();\n const run: OcrLine[] = [line];\n let maxB = bucket(line.h);\n while (i + 1 < p.lines.length && levelOf.has(bucket(p.lines[i + 1].h)) && p.lines[i + 1].col === p.lines[i].col && p.lines[i].midY - p.lines[i + 1].midY < Math.max(p.lines[i].h, p.lines[i + 1].h) * 2.2) {\n i++;\n run.push(p.lines[i]);\n maxB = Math.max(maxB, bucket(p.lines[i].h));\n }\n const runTexts = run.map((l) => l.text);\n const merged = runTexts.slice(1).reduce(hyphenJoin, runTexts[0]);\n const words = run.flatMap((l) => l.words);\n // Below UNSTRUCTURED_WORDLIKE_FRACTION = chart labels/stray numerals,\n // not a heading; checked FIRST so a long numeral run isn't asserted as a pull quote.\n const junk = isUnstructured(merged);\n if (junk) blocks.push({ kind: 'review', text: merged, reason: REVIEW_REASON.unstructuredLabels, words });\n else if (merged.length > 150) blocks.push({ kind: 'quote', text: merged, words });\n else blocks.push({ kind: 'heading', text: merged, level: levelOf.get(maxB) as number, words });\n } else {\n para.push(line);\n }\n i++;\n }\n flush();\n return blocks;\n}\n"],"names":["getDict","bucket","hyphenJoin","acc","next","endsWith","withoutDash","slice","pen","at","test","filtered","head","split","pop","firstTok","filter","c","join","tail","close","has","toLowerCase","REVIEW_REASON","unstructuredLabels","UNSTRUCTURED_WORDLIKE_FRACTION","isUnstructured","text","tokens","Boolean","length","wordLike","t","reviewLine","reason","renderEmitBlock","b","kind","repeat","level","emitPage","p","levelOf","tuning","blocks","para","flush","texts","map","l","merged","reduce","words","flatMap","push","i","lines","line","prev","col","midY","pitch","paraGap","danglingEnd","continuationStart","joinAcrossColumns","h","run","maxB","Math","max","runTexts","junk","get"],"mappings":"AAAA,uDAAuD;AACvD,SAASA,OAAO,QAAQ,YAAY;AAEpC,SAASC,MAAM,QAAQ,gBAAgB;AAEvC,2EAA2E;AAC3E,+DAA+D;AAC/D,SAASC,WAAWC,GAAW,EAAEC,IAAY;IAC3C,IAAID,IAAIE,QAAQ,CAAC,MAAM;QACrB,MAAMC,cAAcH,IAAII,KAAK,CAAC,GAAG,CAAC;QAClC,MAAMC,MAAMF,YAAYG,EAAE,CAAC,CAAC;QAC5B,IAAID,OAAO,cAAcE,IAAI,CAACF,MAAM;gBACrBF,wBACIF,cAEJO;YAHb,MAAMC,QAAON,yBAAAA,YAAYO,KAAK,CAAC,KAAKC,GAAG,gBAA1BR,oCAAAA,yBAAgC;YAC7C,MAAMS,YAAWX,eAAAA,KAAKS,KAAK,CAAC,IAAI,CAAC,EAAE,cAAlBT,0BAAAA,eAAsB;YACvC,MAAMO,WAAW;mBAAII;aAAS,CAACC,MAAM,CAAC,CAACC,IAAM,YAAYP,IAAI,CAACO,IAAIC,IAAI,CAAC;YACvE,MAAMC,QAAOR,mBAAAA,SAASE,KAAK,CAAC,IAAI,CAAC,EAAE,cAAtBF,8BAAAA,mBAA0B;YACvC,MAAMS,QAAQpB,UAAUqB,GAAG,CAAC,AAACT,CAAAA,OAAOO,IAAG,EAAGG,WAAW;YACrD,OAAO,AAACF,CAAAA,QAAQd,cAAcH,GAAE,IAAKC;QACvC;IACF;IACA,OAAO,GAAGD,IAAI,CAAC,EAAEC,MAAM;AACzB;AAEA,8DAA8D;AAC9D,qGAAqG;AACrG,MAAMmB,gBAAgB;IACpBC,oBAAoB;AACtB;AAEA;qGACqG,GACrG,OAAO,MAAMC,iCAAiC,IAAI;AAClD,SAASC,eAAeC,IAAY;IAClC,MAAMC,SAASD,KAAKd,KAAK,CAAC,OAAOG,MAAM,CAACa;IACxC,IAAID,OAAOE,MAAM,KAAK,GAAG,OAAO,OAAO,4BAA4B;IACnE,MAAMC,WAAWH,OAAOZ,MAAM,CAAC,CAACgB,IAAM,cAActB,IAAI,CAACsB,IAAIF,MAAM;IACnE,OAAOC,WAAWH,OAAOE,MAAM,GAAGL;AACpC;AAEA,SAASQ,WAAWC,MAAc,EAAEP,IAAY;IAC9C,OAAO,CAAC,WAAW,EAAEO,OAAO,EAAE,EAAEP,MAAM;AACxC;AAiBA,oEAAoE;AACpE,8EAA8E;AAC9E,OAAO,SAASQ,gBAAgBC,CAAY;IAC1C,OAAQA,EAAEC,IAAI;QACZ,KAAK;YACH,OAAO,GAAG,IAAIC,MAAM,CAACF,EAAEG,KAAK,EAAE,CAAC,EAAEH,EAAET,IAAI,EAAE;QAC3C,KAAK;YACH,OAAO,CAAC,EAAE,EAAES,EAAET,IAAI,EAAE;QACtB,KAAK;YACH,OAAOM,WAAWG,EAAEF,MAAM,EAAEE,EAAET,IAAI;QACpC;YACE,OAAOS,EAAET,IAAI;IACjB;AACF;AAEA,OAAO,SAASa,SAASC,CAAU,EAAEC,OAA4B,EAAEC,MAAc;IAC/E,MAAMC,SAAsB,EAAE;IAC9B,IAAIC,OAAkB,EAAE;IACxB,MAAMC,QAAQ;QACZ,IAAI,CAACD,KAAKf,MAAM,EAAE;QAClB,MAAMiB,QAAQF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEtB,IAAI;QACpC,MAAMuB,SAASH,MAAMxC,KAAK,CAAC,GAAG4C,MAAM,CAACjD,YAAY6C,KAAK,CAAC,EAAE;QACzD,MAAMK,QAAQP,KAAKQ,OAAO,CAAC,CAACJ,IAAMA,EAAEG,KAAK;QACzC,oEAAoE;QACpE,iEAAiE;QACjER,OAAOU,IAAI,CAAC5B,eAAewB,UAAU;YAAEb,MAAM;YAAUV,MAAMuB;YAAQhB,QAAQX,cAAcC,kBAAkB;YAAE4B;QAAM,IAAI;YAAEf,MAAM;YAAaV,MAAMuB;YAAQE;QAAM;QAClKP,OAAO,EAAE;IACX;IAEA,IAAIU,IAAI;IACR,MAAOA,IAAId,EAAEe,KAAK,CAAC1B,MAAM,CAAE;QACzB,MAAM2B,OAAgBhB,EAAEe,KAAK,CAACD,EAAE;QAChC,MAAMG,OAAOH,IAAI,IAAId,EAAEe,KAAK,CAACD,IAAI,EAAE,GAAG;QACtC,IAAIG,QAASD,CAAAA,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAID,KAAKE,IAAI,GAAGH,KAAKG,IAAI,GAAGnB,EAAEoB,KAAK,GAAGlB,OAAOmB,OAAO,AAAD,GAAI;YACvF,wEAAwE;YACxE,0IAA0I;YAC1I,MAAMC,cAAclB,KAAKf,MAAM,GAAG,KAAK,CAAC,kBAAkBpB,IAAI,CAACmC,IAAI,CAACA,KAAKf,MAAM,GAAG,EAAE,CAACH,IAAI;YACzF,MAAMqC,oBAAoB,SAAStD,IAAI,CAAC+C,KAAK9B,IAAI;YACjD,MAAMsC,oBAAoBR,KAAKE,GAAG,KAAKD,KAAKC,GAAG,IAAII,eAAeC,qBAAqB,CAACtB,QAAQrB,GAAG,CAACpB,OAAOwD,KAAKS,CAAC;YACjH,IAAI,CAACD,mBAAmBnB;QAC1B;QAEA,IAAIJ,QAAQrB,GAAG,CAACpB,OAAOwD,KAAKS,CAAC,IAAI;YAC/B,uEAAuE;YACvE,4DAA4D;YAC5DpB;YACA,MAAMqB,MAAiB;gBAACV;aAAK;YAC7B,IAAIW,OAAOnE,OAAOwD,KAAKS,CAAC;YACxB,MAAOX,IAAI,IAAId,EAAEe,KAAK,CAAC1B,MAAM,IAAIY,QAAQrB,GAAG,CAACpB,OAAOwC,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACW,CAAC,MAAMzB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACI,GAAG,KAAKlB,EAAEe,KAAK,CAACD,EAAE,CAACI,GAAG,IAAIlB,EAAEe,KAAK,CAACD,EAAE,CAACK,IAAI,GAAGnB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACK,IAAI,GAAGS,KAAKC,GAAG,CAAC7B,EAAEe,KAAK,CAACD,EAAE,CAACW,CAAC,EAAEzB,EAAEe,KAAK,CAACD,IAAI,EAAE,CAACW,CAAC,IAAI,IAAK;gBACzMX;gBACAY,IAAIb,IAAI,CAACb,EAAEe,KAAK,CAACD,EAAE;gBACnBa,OAAOC,KAAKC,GAAG,CAACF,MAAMnE,OAAOwC,EAAEe,KAAK,CAACD,EAAE,CAACW,CAAC;YAC3C;YACA,MAAMK,WAAWJ,IAAInB,GAAG,CAAC,CAACC,IAAMA,EAAEtB,IAAI;YACtC,MAAMuB,SAASqB,SAAShE,KAAK,CAAC,GAAG4C,MAAM,CAACjD,YAAYqE,QAAQ,CAAC,EAAE;YAC/D,MAAMnB,QAAQe,IAAId,OAAO,CAAC,CAACJ,IAAMA,EAAEG,KAAK;YACxC,sEAAsE;YACtE,qFAAqF;YACrF,MAAMoB,OAAO9C,eAAewB;YAC5B,IAAIsB,MAAM5B,OAAOU,IAAI,CAAC;gBAAEjB,MAAM;gBAAUV,MAAMuB;gBAAQhB,QAAQX,cAAcC,kBAAkB;gBAAE4B;YAAM;iBACjG,IAAIF,OAAOpB,MAAM,GAAG,KAAKc,OAAOU,IAAI,CAAC;gBAAEjB,MAAM;gBAASV,MAAMuB;gBAAQE;YAAM;iBAC1ER,OAAOU,IAAI,CAAC;gBAAEjB,MAAM;gBAAWV,MAAMuB;gBAAQX,OAAOG,QAAQ+B,GAAG,CAACL;gBAAiBhB;YAAM;QAC9F,OAAO;YACLP,KAAKS,IAAI,CAACG;QACZ;QACAF;IACF;IACAT;IACA,OAAOF;AACT"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PageWord } from '../geometry.js';
|
|
2
|
+
export declare const OCR_RENDER_DPI = 288;
|
|
3
|
+
export declare const SPARSE_RETRY_PROSE_FRACTION = 0.6;
|
|
4
|
+
/** Does this page's PSM.AUTO pass look chart-like enough to re-recognize
|
|
5
|
+
* with PSM.SPARSE_TEXT? A page with 0 words never retries (nothing to take a prose fraction of). */
|
|
6
|
+
export declare function shouldRetrySparse(words: {
|
|
7
|
+
text: string;
|
|
8
|
+
}[]): boolean;
|
|
9
|
+
/** Per-page recognition progress, delivered once per page as it finishes.
|
|
10
|
+
* Engine chatter goes through `onWarning`/`warnings` instead — this is timing/count data only. */
|
|
11
|
+
export type OcrProgressEvent = {
|
|
12
|
+
type: 'page';
|
|
13
|
+
page: number;
|
|
14
|
+
words: number;
|
|
15
|
+
ms: number;
|
|
16
|
+
};
|
|
17
|
+
export interface RecognizeResult {
|
|
18
|
+
words: PageWord[];
|
|
19
|
+
/** pages (1-based) where the engine returned zero words */
|
|
20
|
+
emptyPages: number[];
|
|
21
|
+
/** wall-clock ms per page (render + recognize), keyed by page number */
|
|
22
|
+
msPerPage: Map<number, number>;
|
|
23
|
+
}
|
|
24
|
+
/** Recognize words on pages [first..last] of a PDF (renders at `dpi`,
|
|
25
|
+
* default OCR_RENDER_DPI). Progress via `onProgress`; diagnostics via `onWarning` — this module never writes to stdout/stderr itself. */
|
|
26
|
+
export declare function recognizePdf(pdfPath: string, first: number, last: number, { dpi, onProgress, onWarning }?: {
|
|
27
|
+
dpi?: number;
|
|
28
|
+
onProgress?: (e: OcrProgressEvent) => void;
|
|
29
|
+
onWarning?: (message: string) => void;
|
|
30
|
+
}): Promise<RecognizeResult>;
|